From a31640ba8248ab94f9b9c0c672b02c9cbb5dd54a Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 5 Nov 2024 10:11:37 +0100 Subject: [PATCH 01/93] [OV JS] Simplify run js samples (#27206) ### Details: - Add script to fetch static - Update run instructions - Remove logic of downloading model from notebooks - Put image manipulation logic in one file - Refactor notebooks structure - Remove opencv-wasm dependency ### Tickets: - 154015 --- samples/js/node/.gitignore | 3 +- samples/js/node/README.md | 4 +- .../classification_sample_async/README.md | 7 +- .../classification_sample_async.js | 118 +++--- samples/js/node/fetch-samples-assets.js | 83 ++++ .../js/node/hello_classification/README.md | 8 +- .../hello_classification.js | 29 +- samples/js/node/hello_reshape_ssd/README.md | 8 +- .../hello_reshape_ssd/hello_reshape_ssd.js | 96 ++--- samples/js/node/helpers.js | 106 +---- samples/js/node/image.js | 275 ++++++++++++ samples/js/node/notebooks/hello-detection.nnb | 66 +-- .../js/node/notebooks/hello-segmentation.nnb | 77 +--- samples/js/node/notebooks/hello-world.nnb | 79 +--- .../optical-character-recognition.nnb | 195 ++++----- samples/js/node/notebooks/pose-estimation.nnb | 62 +-- .../js/node/notebooks/question-answering.nnb | 114 +---- .../{tokens_bert.js => tokenizer_bert.js} | 41 +- .../notebooks/vision-background-removal.nnb | 96 +---- .../optical_character_recognition/README.md | 10 +- .../optical-character-recognition.js | 399 ++++-------------- samples/js/node/package-lock.json | 260 ++++++++---- samples/js/node/package.json | 11 +- .../node/vision_background_removal/README.md | 10 +- .../vision_background_removal.js | 240 +++-------- 25 files changed, 1060 insertions(+), 1337 deletions(-) create mode 100644 samples/js/node/fetch-samples-assets.js create mode 100644 samples/js/node/image.js rename samples/js/node/notebooks/{tokens_bert.js => tokenizer_bert.js} (79%) diff --git a/samples/js/node/.gitignore b/samples/js/node/.gitignore index 2ab910508d5529..c836a5862da256 100644 --- a/samples/js/node/.gitignore +++ b/samples/js/node/.gitignore @@ -1,2 +1,3 @@ node_modules -hello_reshape_ssd/out.jpg +out*.jpg +output/ diff --git a/samples/js/node/README.md b/samples/js/node/README.md index 9e69769778d8a8..2fd4dbe76e5900 100644 --- a/samples/js/node/README.md +++ b/samples/js/node/README.md @@ -7,13 +7,15 @@ To run samples, install dependencies first. In current directory run: npm install ``` -Note: Perform these steps also before running notebooks. +Note: Perform this step also before running notebooks. ## Samples - hello_classification - hello_reshape_ssd - classification_sample_async + - optical_character_recognition + - vision_background_removal ## Notebooks diff --git a/samples/js/node/classification_sample_async/README.md b/samples/js/node/classification_sample_async/README.md index 0b19e908587505..6a0dcba6d16e02 100644 --- a/samples/js/node/classification_sample_async/README.md +++ b/samples/js/node/classification_sample_async/README.md @@ -4,7 +4,12 @@ Models with only 1 input and output are supported. Run: ```bash -node classification_sample_async.js -m *path_to_model_file* -i *path_to_img1* -i *path_to_img2* -d AUTO +node classification_sample_async.js -m ../../assets/models/v3-small_224_1.0_float.xml -i ../../assets/images/coco.jpg -i ../../assets/images/coco_hollywood.jpg -d AUTO +``` + +Where +```bash +node classification_sample_async.js -m *path_to_model_file* -i *path_to_img1* -i *path_to_img2* -d *device* ``` Other details see in [../../../python/classification_sample_async/README.md](../../../python/classification_sample_async/README.md) diff --git a/samples/js/node/classification_sample_async/classification_sample_async.js b/samples/js/node/classification_sample_async/classification_sample_async.js index 94f4d7828c95c2..96724be80d3b0b 100644 --- a/samples/js/node/classification_sample_async/classification_sample_async.js +++ b/samples/js/node/classification_sample_async/classification_sample_async.js @@ -1,39 +1,30 @@ const { addon: ov } = require('openvino-node'); const args = require('args'); -const { cv } = require('opencv-wasm'); -const { getImageData } = require('../helpers.js'); - -args.options([{ - name: 'img', - defaultValue: [], -}, { - name: 'model', -}, { - name: 'device', -}]); -const { model: modelPath, device: deviceName, img: images } = - args.parse(process.argv); - -main(modelPath, images, deviceName); - -function completionCallback(result, imagePath) { - const predictions = Array.from(result.data) - .map((prediction, classId) => ({ prediction, classId })) - .sort(({ prediction: predictionA }, { prediction: predictionB }) => - predictionA === predictionB ? 0 : predictionA > predictionB ? -1 : 1); - - console.log(`Image path: ${imagePath}`); - console.log('Top 10 results:'); - console.log('class_id probability'); - console.log('--------------------'); - predictions.slice(0, 10).forEach(({ classId, prediction }) => - console.log(`${classId}\t ${prediction.toFixed(7)}`), - ); - console.log(); -} - -async function main(modelPath, images, deviceName) { +const Image = require('../image.js'); +const imagenetClassesMap = require('../../assets/datasets/imagenet_class_index.json'); + +args.options([ + { + name: 'img', + defaultValue: [], + }, + { + name: 'model', + }, + { + name: 'device', + }, +]); +const { + model: modelPath, + device: deviceName, + img: imgPaths +} = args.parse(process.argv); + +main(modelPath, imgPaths, deviceName); + +async function main(modelPath, imgPaths, deviceName) { //----------- Step 1. Initialize OpenVINO Runtime Core ----------------------- console.log('Creating OpenVINO Runtime Core'); const core = new ov.Core(); @@ -42,8 +33,6 @@ async function main(modelPath, images, deviceName) { console.log(`Reading the model: ${modelPath}`); // (.xml and .bin files) or (.onnx file) const model = await core.readModel(modelPath); - const [h, w] = model.inputs[0].shape.slice(-2); - const tensorShape = [1, h, w, 3]; if (model.inputs.length !== 1) throw new Error('Sample supports only single input topologies'); @@ -52,56 +41,65 @@ async function main(modelPath, images, deviceName) { throw new Error('Sample supports only single output topologies'); //----------- Step 3. Set up input ------------------------------------------- - // Read input image - const imagesData = []; - - for (const imagePath of images) - imagesData.push(await getImageData(imagePath)); + const inputImages = []; + const [, inputHeight, inputWidth] = model.inputs[0].getShape(); - const preprocessedImages = imagesData.map((imgData) => { - // Use opencv-wasm to preprocess image. - const originalImage = cv.matFromImageData(imgData); - const image = new cv.Mat(); - // The MobileNet model expects images in RGB format. - cv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB); - cv.resize(image, image, new cv.Size(w, h)); + // Read input image, resize it to the model's input size and convert it to a tensor. + for (const path of imgPaths) { + const img = await Image.load(path); + const resized = img.resize(inputWidth, inputHeight); - return new Uint8Array(image.data); - }); + inputImages.push(resized); + } //----------- Step 4. Apply preprocessing ------------------------------------ const _ppp = new ov.preprocess.PrePostProcessor(model); _ppp.input().tensor().setLayout('NHWC').setElementType(ov.element.u8); - _ppp.input().model().setLayout('NCHW'); + _ppp.input().model().setLayout('NHWC'); _ppp.output().tensor().setElementType(ov.element.f32); _ppp.build(); - //----------------- Step 5. Loading model to the device ---------------------- + //----------- Step 5. Loading model to the device ---------------------------- console.log('Loading the model to the plugin'); const compiledModel = await core.compileModel(model, deviceName); const outputName = compiledModel.output(0).toString(); - //----------- Step 6. Collecting promises to react when they resolve --------- - console.log('Starting inference in asynchronous mode'); + //----------- Step 6. Do inference ------------------------------------------- + console.log('Starting inference\n'); // Create infer request const inferRequest = compiledModel.createInferRequest(); - - const promises = preprocessedImages.map((tensorData, i) => { - const inferPromise = inferRequest.inferAsync([ - new ov.Tensor(ov.element.u8, tensorShape, tensorData) - ]); + const promises = inputImages.map((img, i) => { + const inferPromise = inferRequest.inferAsync([img.toTensor()]); inferPromise.then(result => - completionCallback(result[outputName], images[i])); + completionCallback(result[outputName], imgPaths[i])); return inferPromise; }); - //----------- Step 7. Do inference ------------------------------------------- + //----------- Step 7. Wait till all inferences execute ----------------------- await Promise.all(promises); console.log('All inferences executed'); console.log('\nThis sample is an API example, for any performance ' + 'measurements please use the dedicated benchmark_app tool'); } + +function completionCallback(result, imagePath) { + const predictions = Array.from(result.data) + .map((prediction, classId) => ({ prediction, classId })) + .sort(({ prediction: predictionA }, { prediction: predictionB }) => + predictionA === predictionB ? 0 : predictionA > predictionB ? -1 : 1); + + const imagenetClasses = ['background', ...Object.values(imagenetClassesMap)]; + + console.log(`Image path: ${imagePath}`); + console.log('Top 5 results:\n'); + console.log('id\tprobability\tlabel'); + console.log('---------------------------------'); + predictions.slice(0, 5).forEach(({ classId, prediction }) => + console.log(`${classId}\t${prediction.toFixed(7)}\t${imagenetClasses[classId][1]}`), + ); + console.log(); +} diff --git a/samples/js/node/fetch-samples-assets.js b/samples/js/node/fetch-samples-assets.js new file mode 100644 index 00000000000000..33dd509a922f85 --- /dev/null +++ b/samples/js/node/fetch-samples-assets.js @@ -0,0 +1,83 @@ +const { downloadFile } = require('./helpers.js'); + +const host = 'https://storage.openvinotoolkit.org'; + +const models = [ + // hello classification + '/repositories/openvino_notebooks/models/mobelinet-v3-tf/FP32/v3-small_224_1.0_float.xml', + '/repositories/openvino_notebooks/models/mobelinet-v3-tf/FP32/v3-small_224_1.0_float.bin', + + // hello reshape ssd + '/repositories/open_model_zoo/2022.3/models_bin/1/road-segmentation-adas-0001/FP32/road-segmentation-adas-0001.xml', + '/repositories/open_model_zoo/2022.3/models_bin/1/road-segmentation-adas-0001/FP32/road-segmentation-adas-0001.bin', + + // hello detection, optical character recognition + '/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.xml', + '/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.bin', + + '/repositories/open_model_zoo/public/text-recognition-resnet-fc/text-recognition-resnet-fc.xml', + '/repositories/open_model_zoo/public/text-recognition-resnet-fc/text-recognition-resnet-fc.bin', + + // vision background removal + '/repositories/open_model_zoo/public/vision-background-removal/unet_ir_model.xml', + '/repositories/open_model_zoo/public/vision-background-removal/unet_ir_model.bin', + + // pose estimation + '/repositories/open_model_zoo/2022.1/models_bin/3/human-pose-estimation-0001/FP16-INT8/human-pose-estimation-0001.xml', + '/repositories/open_model_zoo/2022.1/models_bin/3/human-pose-estimation-0001/FP16-INT8/human-pose-estimation-0001.bin', + + // question answering + '/repositories/open_model_zoo/2022.3/models_bin/1/bert-small-uncased-whole-word-masking-squad-0001/FP16/bert-small-uncased-whole-word-masking-squad-0001.xml', +]; +const modelsDir = __dirname + '/../assets/models'; + + +const images = [ + // hello classification + '/repositories/openvino_notebooks/data/data/image/coco.jpg', + + // hello reshape ssd + '/repositories/openvino_notebooks/data/data/image/empty_road_mapillary.jpg', + + // hello detection, optical character recognition, pose estimation + '/repositories/openvino_notebooks/data/data/image/intel_rnb.jpg', + + // vision background removal + '/repositories/openvino_notebooks/data/data/image/coco_hollywood.jpg', + '/repositories/openvino_notebooks/data/data/image/wall.jpg', +]; +const imagesDir = __dirname + '/../assets/images'; + +const datasets = [ + // hello classification + '/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_class_index.json', +]; +const datasetsDir = __dirname + '/../assets/datasets'; + +const vocab = [ + '/repositories/openvino_notebooks/data/data/text/bert-uncased/vocab.txt', +]; +const vocabDir = __dirname + '/../assets/vocab'; + +try { + main(); +} catch(error) { + console.error('Error Occurred', error); +} + +async function main() { + await downloadAssets(models, modelsDir); + await downloadAssets(images, imagesDir); + await downloadAssets(datasets, datasetsDir); + await downloadAssets(vocab, vocabDir); +} + +async function downloadAssets(links, destinationDir) { + for (const link of links) { + const url = host + link; + const filename = link.split('/').pop(); + + await downloadFile(url, filename, destinationDir); + console.log(`Downloaded: ${filename} \n`); + } +} diff --git a/samples/js/node/hello_classification/README.md b/samples/js/node/hello_classification/README.md index 2de983af58334b..1d20bba8ac63e6 100644 --- a/samples/js/node/hello_classification/README.md +++ b/samples/js/node/hello_classification/README.md @@ -2,9 +2,13 @@ Models with only 1 input and output are supported. -Run: +Run sample: ```bash -node hello_classification.js *path_to_model_file* *path_to_img* AUTO +node hello_classification.js ../../assets/models/v3-small_224_1.0_float.xml ../../assets/images/coco.jpg AUTO +``` +Where +```bash +node hello_classification.js *path_to_model_file* *path_to_img* *device* ``` Other details see in [../../../python/hello_classification/README.md](../../../python/hello_classification/README.md) diff --git a/samples/js/node/hello_classification/hello_classification.js b/samples/js/node/hello_classification/hello_classification.js index 00ba868e40f3a0..9edbc998d006bd 100644 --- a/samples/js/node/hello_classification/hello_classification.js +++ b/samples/js/node/hello_classification/hello_classification.js @@ -1,7 +1,7 @@ const { addon: ov } = require('openvino-node'); -const { cv } = require('opencv-wasm'); -const { getImageData } = require('../helpers.js'); +const Image = require('../image.js'); +const imagenetClassesMap = require('../../assets/datasets/imagenet_class_index.json'); // Parsing and validation of input arguments if (process.argv.length !== 5) @@ -31,21 +31,12 @@ async function main(modelPath, imagePath, deviceName) { //----------------- Step 3. Set up input ------------------------------------- // Read input image - const imgData = await getImageData(imagePath); - - // Use opencv-wasm to preprocess image. - const originalImage = cv.matFromImageData(imgData); - const image = new cv.Mat(); - // The MobileNet model expects images in RGB format. - cv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB); - - const tensorData = new Float32Array(image.data); - const shape = [1, image.rows, image.cols, 3]; - const inputTensor = new ov.Tensor(ov.element.f32, shape, tensorData); + const img = await Image.load(imagePath); + const inputTensor = img.toTensor(); //----------------- Step 4. Apply preprocessing ------------------------------ const _ppp = new ov.preprocess.PrePostProcessor(model); - _ppp.input().tensor().setShape(shape).setLayout('NHWC'); + _ppp.input().tensor().setElementType(ov.element.u8).setShape(inputTensor.getShape()).setLayout('NHWC'); _ppp.input().preprocess().resize(ov.preprocess.resizeAlgorithm.RESIZE_LINEAR); _ppp.input().model().setLayout('NHWC'); _ppp.output().tensor().setElementType(ov.element.f32); @@ -69,12 +60,14 @@ async function main(modelPath, imagePath, deviceName) { .sort(({ prediction: predictionA }, { prediction: predictionB }) => predictionA === predictionB ? 0 : predictionA > predictionB ? -1 : 1); + const imagenetClasses = ['background', ...Object.values(imagenetClassesMap)]; + console.log(`Image path: ${imagePath}`); - console.log('Top 10 results:'); - console.log('class_id probability'); - console.log('--------------------'); + console.log('Top 10 results:\n'); + console.log('id\tprobability\tlabel'); + console.log('---------------------------------'); predictions.slice(0, 10).forEach(({ classId, prediction }) => - console.log(`${classId}\t ${prediction.toFixed(7)}`), + console.log(`${classId}\t${prediction.toFixed(7)}\t${imagenetClasses[classId][1]}`), ); console.log('\nThis sample is an API example, for any performance ' diff --git a/samples/js/node/hello_reshape_ssd/README.md b/samples/js/node/hello_reshape_ssd/README.md index 21d8be8ec4b50d..547cf989478a87 100644 --- a/samples/js/node/hello_reshape_ssd/README.md +++ b/samples/js/node/hello_reshape_ssd/README.md @@ -2,9 +2,13 @@ Models with only 1 input and output are supported. -Run: +Run sample: ```bash -node hello_reshape_ssd.js *path_to_model_file* *path_to_img* AUTO +node hello_reshape_ssd.js ../../assets/models/road-segmentation-adas-0001.xml ../../assets/images/empty_road_mapillary.jpg AUTO +``` +Where +```bash +node hello_reshape_ssd.js *path_to_model_file* *path_to_img* *device* ``` Other details see in [../../../python/hello_reshape_ssd/README.md](../../../python/hello_reshape_ssd/README.md) diff --git a/samples/js/node/hello_reshape_ssd/hello_reshape_ssd.js b/samples/js/node/hello_reshape_ssd/hello_reshape_ssd.js index 416e8bc8180668..ebcc8e83223853 100644 --- a/samples/js/node/hello_reshape_ssd/hello_reshape_ssd.js +++ b/samples/js/node/hello_reshape_ssd/hello_reshape_ssd.js @@ -1,13 +1,5 @@ const { addon: ov } = require('openvino-node'); - -const fs = require('node:fs/promises'); -const { cv } = require('opencv-wasm'); -const { - setShape, - getImageData, - getImageBuffer, - arrayToImageData, -} = require('../helpers.js'); +const Image = require('../image.js'); // Parsing and validation of input arguments if (process.argv.length !== 5) @@ -38,24 +30,15 @@ async function main(modelPath, imagePath, deviceName) { //----------------- Step 3. Set up input ------------------------------------- // Read input image - const imgData = await getImageData(imagePath); - - // Use opencv-wasm to preprocess image. - const originalImage = cv.matFromImageData(imgData); - const image = new cv.Mat(); - // The MobileNet model expects images in RGB format. - cv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB); - - const tensorData = new Uint8Array(image.data); - const shape = [1, image.rows, image.cols, 3]; - const inputTensor = new ov.Tensor(ov.element.u8, shape, tensorData); + const img = await Image.load(imagePath); + const inputTensor = img.toTensor(); //----------------- Step 4. Apply preprocessing ------------------------------ const _ppp = new ov.preprocess.PrePostProcessor(model); _ppp.input().preprocess().resize(ov.preprocess.resizeAlgorithm.RESIZE_LINEAR); _ppp.input().tensor() - .setShape(shape) + .setShape(inputTensor.getShape()) .setElementType(ov.element.u8) .setLayout('NHWC'); @@ -70,48 +53,51 @@ async function main(modelPath, imagePath, deviceName) { //---------------- Step 6. Create infer request and do inference synchronously console.log('Starting inference in synchronous mode'); const inferRequest = compiledModel.createInferRequest(); - inferRequest.setInputTensor(inputTensor); - inferRequest.infer(); + const outputs = inferRequest.infer([inputTensor]); //----------------- Step 7. Process output ----------------------------------- const outputLayer = compiledModel.outputs[0]; - const resultInfer = inferRequest.getTensor(outputLayer); - const predictions = Array.from(resultInfer.data); - const [height, width] = [originalImage.rows, originalImage.cols]; - - const detections = setShape(predictions, [100, 7]); - const color = [255, 0, 0, 255]; - const THROUGHPUT = 0.9; - - detections.forEach(detection => { - const [classId, confidence, xmin, ymin, xmax, ymax] = detection.slice(1); - - if (confidence < THROUGHPUT) return; - - console.log(`Found: classId = ${classId}, ` - + `confidence = ${confidence.toFixed(2)}, ` - + `coords = (${xmin}, ${ymin}), (${xmax}, ${ymax})`, - ); - - // Draw a bounding box on a output image - cv.rectangle(originalImage, - new cv.Point(xmin*width, ymin*height), - new cv.Point(xmax*width, ymax*height), - color, - 2, - ); - }); - - const resultImgData = arrayToImageData(originalImage.data, width, height); + const output = outputs[outputLayer]; + const outputData = output.data; + const resultLayer = []; + const colormap = [ + [68, 1, 84, 255], + [48, 103, 141, 255], + [53, 183, 120, 255], + [199, 216, 52, 255], + ]; + const size = outputData.length/4; + + for (let i = 0; i < size; i++) { + const valueAt = (i, number) => outputData[i + number*size]; + const currentValues = { + bg: valueAt(i, 0), + c: valueAt(i, 1), + h: valueAt(i, 2), + w: valueAt(i, 3), + }; + const values = Object.values(currentValues); + const maxIndex = values.indexOf(Math.max(...values)); + + resultLayer.push(maxIndex); + } + + const pixels = []; + resultLayer.forEach(i => pixels.push(...colormap[i])); + + const alpha = 0.6; const filename = 'out.jpg'; + const [, , H, W] = output.getShape(); - await fs.writeFile(`./${filename}`, getImageBuffer(resultImgData)); + const segmentsImg = Image.fromArray(pixels, W, H); + const resizedSegments = segmentsImg.resize(img.width, img.height); + const mergedImg = Image.overlay(img, resizedSegments, alpha); try { - await fs.readFile(filename); - console.log('Image out.jpg was created!'); + await mergedImg.save(filename); + console.log(`Image '${filename}' was created.`); } catch(err) { - console.log(`Image ${filename} was not created. Check your permissions.`); + console.log(`Image '${filename}' was not created. Check your permissions.`); } console.log('\nThis sample is an API example, for any performance ' diff --git a/samples/js/node/helpers.js b/samples/js/node/helpers.js index 5cbd6650495070..5d56d4c60a7139 100644 --- a/samples/js/node/helpers.js +++ b/samples/js/node/helpers.js @@ -1,17 +1,8 @@ const path = require('node:path'); -const { cv } = require('opencv-wasm'); const { createWriteStream } = require('node:fs'); const { mkdir, stat } = require('node:fs/promises'); const { HttpsProxyAgent } = require('https-proxy-agent'); -const { - Image, - ImageData, - loadImage, - createCanvas, - createImageData, -} = require('canvas'); - module.exports = { exp, sum, @@ -23,96 +14,29 @@ module.exports = { setShape, transform, downloadFile, - displayImage, - getImageData, extractValues, - getImageBuffer, - arrayToImageData, - displayArrayAsImage, matrixMultiplication, }; -function arrayToImageData(array, width, height) { - return createImageData(new Uint8ClampedArray(array), width, height); -} - -function getImageBuffer(imageOrImageData) { - const canvas = createCanvas(imageOrImageData.width, imageOrImageData.height); - const ctx = canvas.getContext('2d'); - - if (imageOrImageData instanceof Image) - ctx.drawImage(imageOrImageData, 0, 0); - else if (imageOrImageData instanceof ImageData) - ctx.putImageData(imageOrImageData, 0, 0); - else - throw Error(`Passed parameters has type '${typeof imageOrImageData}'. ` - + 'It is\'t supported.'); - - return canvas.toBuffer('image/jpeg'); -} - -function displayImage(imageOrImageData, display) { - const buffer = getImageBuffer(imageOrImageData); - - display.image(buffer); -} - -function displayArrayAsImage(arr, width, height, display) { - const alpha = 255; - const componentsPerPixel = arr.length / (width*height); - - try { - switch (componentsPerPixel) { - case 1: - arr = arr.reduce((acc, val) => { - acc.push(val, val, val, alpha); - - return acc; - }, []); - break; - - case 3: - arr = arr.reduce((acc, val, index) => { - if (index && index%3 === 0) acc.push(alpha); - - acc.push(val); - - return acc; - }, []); - break; - } - } catch(e) { - console.log(e); - } - - const imageData = arrayToImageData(arr, width, height); - - displayImage(imageData, display); -} - -async function getImageData(path) { - const image = await loadImage(path); - const { width, height } = image; - - const canvas = await createCanvas(width, height); - const ctx = canvas.getContext('2d'); - - ctx.drawImage(image, 0, 0); - - return ctx.getImageData(0, 0, width, height); -} - function transform(arr, { width, height }, order) { - const img = new cv.Mat(height, width, cv.CV_8UC3); - - img.data.set(arr, 0, arr.length); + // Calculate the number of pixels and the size of each channel + const numPixels = width * height; + const channels = [[], [], []]; + + // Separate RGB channels + for (let i = 0; i < numPixels; i++) { + channels[0].push(arr[i * 3]); // Red channel + channels[1].push(arr[i * 3 + 1]); // Green channel + channels[2].push(arr[i * 3 + 2]); // Blue channel + } - const channels = new cv.MatVector(); - cv.split(img, channels); + // Reorder channels based on the 'order' array + const reorderedChannels = order.map(num => channels[num]); - const val = order.map(num => [...channels.get(num).data]); + // Flatten reordered channels into a single array + const result = reorderedChannels.flat(); - return [].concat(...val); + return result; } async function downloadFile(url, filename, destination) { diff --git a/samples/js/node/image.js b/samples/js/node/image.js new file mode 100644 index 00000000000000..30ac35032d7ed7 --- /dev/null +++ b/samples/js/node/image.js @@ -0,0 +1,275 @@ +const { + ImageData, + loadImage, + createCanvas, +} = require('@napi-rs/canvas'); +const path = require('node:path'); +const fs = require('node:fs/promises'); +const { addon: ov } = require('openvino-node'); + +const codeENOENT = 'ENOENT'; + +class OvImage { + constructor(imageData) { + this.imageData = imageData; + this.channels = imageData.data.length / (this.width*this.height); + } + + get width() { + return this.imageData.width; + } + + get height() { + return this.imageData.height; + } + + get rgb() { + return this.imageData.data.filter((_, index) => index % 4 !== 3); + } + + get rgba() { + return this.imageData.data; + } + + get grayscale() { + const grayData = new Uint8ClampedArray(this.width * this.height); + + for (let i = 0; i < this.imageData.data.length; i += 4) { + const [r, g, b] = this.imageData.data.slice(i, i + 3); + const gray = 0.299 * r + 0.587 * g + 0.114 * b; + + grayData[i / 4] = gray; + } + + return grayData; + } + + get canvasCtx() { + const canvas = createCanvas(this.width, this.height); + const ctx = canvas.getContext('2d'); + + ctx.putImageData(this.imageData, 0, 0); + + return ctx; + } + + get buffer() { + return this.canvasCtx.canvas.toBuffer('image/jpeg'); + } + + drawRect(x, y, width, height, properties) { + const ctx = this.canvasCtx; + + ctx.strokeStyle = properties.color || 'red'; + ctx.lineWidth = properties.width || 1; + ctx.strokeRect(x, y, width, height); + + const imageData = ctx.getImageData(0, 0, this.width, this.height); + + return new OvImage(imageData); + } + + drawText(text, x, y, properties) { + const ctx = this.canvasCtx; + + ctx.font = properties.font || '30px Arial'; + ctx.fillStyle = properties.color || 'red'; + ctx.fillText(text, x, y); + + const imageData = ctx.getImageData(0, 0, this.width, this.height); + + return new OvImage(imageData); + } + + drawCircle(x, y, radius, properties) { + const ctx = this.canvasCtx; + + ctx.strokeStyle = properties.color || 'red'; + ctx.lineWidth = properties.width || 1; + ctx.beginPath(); + ctx.arc(x, y, radius, 0, 2 * Math.PI); + ctx.stroke(); + + const imageData = ctx.getImageData(0, 0, this.width, this.height); + + return new OvImage(imageData); + } + + drawLine(x1, y1, x2, y2, properties) { + const ctx = this.canvasCtx; + + ctx.strokeStyle = properties.color || 'red'; + ctx.lineWidth = properties.width || 1; + ctx.beginPath(); + ctx.moveTo(x1, y1); + ctx.lineTo(x2, y2); + ctx.stroke(); + + const imageData = ctx.getImageData(0, 0, this.width, this.height); + + return new OvImage(imageData); + } + + toTensor() { + return new ov.Tensor( + ov.element.u8, + [1, this.height, this.width, 3], + new Uint8ClampedArray(this.rgb), + ); + } + + resize(newWidth, newHeight) { + const ctx = this.canvasCtx; + + const canvas2 = createCanvas(newWidth, newHeight); + const ctx2 = canvas2.getContext('2d'); + ctx2.drawImage(ctx.canvas, 0, 0, newWidth, newHeight); + + const imageData = ctx2.getImageData(0, 0, newWidth, newHeight); + + return new OvImage(imageData); + } + + invert() { + const invertedData = this.rgba.map((value, index) => { + if (index % 4 === 3) + return 255; + + return 255 - value; + }); + + return OvImage.fromArray(invertedData, this.width, this.height); + } + + crop(x, y, width, height) { + const canvas2 = createCanvas(width, height); + const ctx2 = canvas2.getContext('2d'); + + ctx2.drawImage(this.canvasCtx.canvas, x, y, width, height, 0, 0, width, height); + + const imageData = ctx2.getImageData(0, 0, width, height); + + return new OvImage(imageData); + } + + async save(filepath) { + const destination = path.dirname(filepath); + + try { + await fs.access(destination); + } catch(error) { + if (error.code !== codeENOENT) throw error; + + await fs.mkdir(destination, { recursive: true }); + } + + return await fs.writeFile(filepath, this.buffer); + } + + // Display the image using the node notebook display object + display(display) { + display.image(this.buffer); + } + + static async load(path) { + const image = await loadImage(path); + const { width, height } = image; + + const canvas = await createCanvas(width, height); + const ctx = canvas.getContext('2d'); + + ctx.drawImage(image, 0, 0); + + return new OvImage(ctx.getImageData(0, 0, width, height)); + } + + static fromArray(arr, width, height) { + const canvas = createCanvas(width, height); + const ctx = canvas.getContext('2d'); + + const imageData = new ImageData( + new Uint8ClampedArray(arr), + width, + height, + ); + + ctx.putImageData(imageData, 0, 0); + + return new OvImage(ctx.getImageData(0, 0, width, height)); + } + + static merge(img1, img2) { + if (img1.width !== img2.width || img1.height !== img2.height) + throw new Error('Images should have the same size'); + + const canvas = createCanvas(img1.width, img1.height); + const ctx = canvas.getContext('2d'); + + const img1Data = img1.imageData.data; + const img2Data = img2.imageData.data; + + const mergedData = img1Data.map((_, index) => { + if (index % 4 === 3) + return 255; + + return (img1Data[index] + img2Data[index]); + }); + + const imageData = new ImageData( + new Uint8ClampedArray(mergedData), + img1.width, + img1.height, + ); + + ctx.putImageData(imageData, 0, 0); + + return new OvImage(ctx.getImageData(0, 0, img1.width, img1.height)); + } + + static mask(img1, img2) { + if (img1.width !== img2.width || img1.height !== img2.height) + throw new Error('Images should have the same size'); + + const canvas = createCanvas(img1.width, img1.height); + const ctx = canvas.getContext('2d'); + + const img1Data = img1.imageData.data; + const img2Data = img2.imageData.data; + + const subtractedData = img1Data.map((_, index) => { + if (index % 4 === 3) + return 255; + + return img1Data[index] * (img2Data[index] / 255); + }); + + const imageData = new ImageData( + new Uint8ClampedArray(subtractedData), + img1.width, + img1.height, + ); + + ctx.putImageData(imageData, 0, 0); + + return new OvImage(ctx.getImageData(0, 0, img1.width, img1.height)); + } + + static overlay(img1, img2, alpha) { + if (img1.width !== img2.width || img1.height !== img2.height) + throw new Error('Images should have the same size'); + + const img1Data = img1.imageData.data; + const img2Data = img2.imageData.data; + + const overlayedData = img1Data.map((_, index) => { + if (index % 4 === 3) + return 255; + + return img1Data[index] * (1 - alpha) + img2Data[index] * alpha; + }); + + return OvImage.fromArray(overlayedData, img1.width, img1.height); + } +} + +module.exports = OvImage; diff --git a/samples/js/node/notebooks/hello-detection.nnb b/samples/js/node/notebooks/hello-detection.nnb index f6139ec7ec8b98..60640b3bd042ea 100644 --- a/samples/js/node/notebooks/hello-detection.nnb +++ b/samples/js/node/notebooks/hello-detection.nnb @@ -17,57 +17,42 @@ { "language": "typescript", "source": [ - "const { cv } = require('opencv-wasm');\nconst { display } = require('node-kernel');\nconst { transform, getImageData, displayArrayAsImage, downloadFile } = require('../helpers.js');\n\nconst { addon: ov } = require('openvino-node'); \n" + "const { addon: ov } = require('openvino-node');\nconst { display } = require('node-kernel');\n\nconst Image = require('../image');\nconst { transform, argMax, setShape } = require('../helpers.js');\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "## Download the Model" + "## Load the Model" ], "outputs": [] }, { "language": "typescript", "source": [ - "const baseArtifactsDir = '../../assets/models';\n\nconst modelName = 'horizontal-text-detection-0001';\nconst modelXMLName = `${modelName}.xml`;\nconst modelBINName = `${modelName}.bin`;\n\nconst modelXMLPath = baseArtifactsDir + '/' + modelXMLName;\n\nconst baseURL = 'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/';\n\nawait downloadFile(baseURL + modelXMLName, modelXMLName, baseArtifactsDir);\nawait downloadFile(baseURL + modelBINName, modelBINName, baseArtifactsDir);\n" + "const modelXMLPath = '../../assets/models/horizontal-text-detection-0001.xml';\n\n// Initialize OpenVINO core and load the detection model\nconst core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'AUTO');\nconst inputLayer = compiledModel.input(0);\nconst outputLayer = compiledModel.output('boxes');\n" ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/models/horizontal-text-detection-0001.bin'", - "" - ] - } - ] - } - ] + "outputs": [] }, { "language": "markdown", "source": [ - "## Download an Image" + "## Load an Image" ], "outputs": [] }, { "language": "typescript", "source": [ - "const baseImagesDir = '../../assets/images';\nconst imgUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/intel_rnb.jpg';\n\nawait downloadFile(imgUrl, 'intel_rnb.jpg', baseImagesDir);\n" + "const imagePath = '../../assets/images/intel_rnb.jpg';\nconst img = await Image.load(imagePath);\nimg.display(display);\n\n// Resize the image to meet network input size\nconst [inputHeight, inputWidth] = inputLayer.shape.slice(2);\nconst resizedImg = img.resize(inputWidth, inputHeight);\n\n// Prepare input tensor\nconst inputImageTransformedData = transform(\n resizedImg.rgb,\n { width: inputWidth, height: inputHeight },\n [0, 1, 2],\n);\nconst tensorData = new Float32Array(inputImageTransformedData);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, tensorData);\n" ], "outputs": [ { "items": [ { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/images/intel_rnb.jpg'", - "" - ] + "mime": "image/jpeg", + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAIFArIDASIAAhEBAxEB/8QAHgAAAAYDAQEAAAAAAAAAAAAAAwQFBgcIAAECCQr/xABgEAABAwEGAwQHBQUEBwMHBg8BAgMRBAAFBhIhMQdBURMiYXEIFDKBkaHwI7HB0eEJFUJS8RYzYnIXJEOCkqLCJVOyGDRUY5PD0iZEc4OEo7PT1OIZdIWVJzY3V5S08v/EABwBAAEFAQEBAAAAAAAAAAAAAAMAAQIEBQYHCP/EAD4RAAEDAgQDBQcDAwMEAgMAAAEAAgMEEQUSITETQVEGFCJhkRUyQlJxgaEjsfAzwdE0U+EWJENicvGSouL/2gAMAwEAAhEDEQA/ALv1/D+mrU9hULp3EdolwdxxB7qgobOdQOVq9Yn/AGbfA7E9TW1jqr/pXq91b7qqW98vfUvOSA605Go22tbZ8fa/71u0J0gbRatBg9DSOJp4g2/QWR3YjUztAleTbqqJXn+yo4ePsli6cd4tpARlBdepaka8o7NuzTvr9lJei6ZdNc3GCqCDEesXAgnQ9U1U/LmbejyQRHxt2ArQCTY5pY0MTOXlje37Lvi/TBBu3HmHqkI5VFNWs5tD/I0sdBvtZr1/7PH0hrvcccYbw9X5hH+r3mWjpz/1hDfKLevIHMmeeot2LDNG081LjnmF4w1HoU+kvSNvLcwEl5ZJhLF9Xc6c07kJqCdieXOyRW+jP6Ql2LbNTwjxgtKVStbF0vPp2M95oLHIaz8be2a6dh0/asNrHigGxddyXO4cy7spSevZAG0O522KlxwvDauwBxAuB5X77wriKhGgJrbreZynX+dI8Nv0smJNQjMhyo74ChOpMg7wfjrb3dTcV2IENU6mo/7txSPuIsRvLA+Gb3SUXndrdWgjVNQO1HwXItE0juqQnC8PB61CApRTzJjWYnYeVjSVVIcbUgqUG0zEcx8iBPj8rex15ejpwbvYZavh1hpfPMq5KNR+JamzarfQ44D1qys4CudJUCO4w40Pgy4gWAaN6Mydo3XjxQvOhKmlPqH2rqTmBIKsx8OvP4dbKdO9UKZpEthUBgd6J3Agx0j7xb1Ir/2fnAV9KvUcKIpVKWVZmLzrkak6+264B8LNmu/ZwcL3UoRR11/UobASgNXw0pIgR/HSE7f4rAjpJI3XIVh9RG9tgV5udqsOPKIWpIIQQB7vdv8AdYBx57KEuKzSDlVlIMbaj4cuYt6BXn+zQusqUq68fXyyCdEuUlPUwPMONTy5WaN6/s0sZNoSi6cfMPZABNVdKmwQDMHs3XPHkbH4bhyVbMCqYKWorQS4ggGCDodjrHn15j3W6Qp1SlpDyoEDmQRIJnny+/xtaW8f2d/GmkWVU984ZqEjXKF1zZ6mJpoHxNmxePoMekJdqHPVcM3RWZ5ILN80zafCA8pB5DcWYsd0TgjdQGaioTT/AN6oEa97XKTy135fW4yHKtDqU9qojX2hsI/W0oXn6KvH+7cod4V3s+E/xUZaqxoJ2YUvn4WbtZwd4uXQpa704Y4wpUITGd25KpCTvuSn65WgWlTDrpqIWt9SgQmASCcp26+Gg+Xw7DrgCASMrgkBJkjToDvIsM/dd6UKEt3jR1FK4pRSUvMls94xpI+o8LcoacU4nOAvLuDrroOn1p5WGQUQboVLjhJSAqBqoAAxHkd/CzUvWqqWcX0QQSlRQkE7QIVoT7hpvoN7OtCVlRcSgGO6CI1Hx8febM/EBLd+UbqUhQUJCgrY5zEfAbdfG1eUXCLGdU76GtU7TD7QLhwpTyOsdP0sIagpAVmKhM6HfQxv7+nzsVutRZoxUgEqzwnUCNI+P11sKtZSlDAQo9p0AJ8dvH52hCLNsiyHXVbVUj2QpIA1OUkHnEe7x5Wxx1aU5t5IGYAHXff3ePnbSVIUtXdByawQY168v0Ng1KZLZW6gHPsCI1nQa+GvWxiNEJCLcfLhUVnQDcATvt8frmH2rmaXHezMlUeB6Dy1+HhbboST2crlROYgkDmdJ32J/rboM5kkmco7oGbTLO53Ez91hndSC4Ln93nUpCiQUjmPEbmJJ+duczql5lgpgbKIgDpHXzsL2S8wSCpStyRtt+Y0Fu22HUqJSQRlAiABtM6npPhaBcNlJYhS0biY1OxA5HXccrBPqU9TrSgkSkwCoxrET7wDYyEOBScyUgqMZYVqCdRPlpvbtxhwtrDgKwpBVAJ9xEjXn87BcbKYFwmRhmpeYXWoOYlRbKgBEjvEggeeh56eVnj6rTOhL4ZCAqVkEbp5nx0J+PnZn3W6tqvrmEtnrAXEEHrM7E+Pv0L+u3Mq7mDkOVxAmQdT16fXvsB3hddFaLtsk792MuZAplQJ5SdTpymenwtyu629crzomE6KEmfPnP1FnClpKEhRCEyARrAAjp7j9+lulst5R2bWYkTMkCfGPhZuImyJtm5/9iioUdBmBG35/jrYA3E4SSstqBHdzp021j3b+XhZ1GmaUSkgjpG5HXTy+t7Y3Slaxl1MTM7b6jp+ZNomXKpCO6a6MMKeAWpinWoQDqUyOp002+jbHsNLywijBhRMJX4bn4geVnuiiAjKkaSOkHmNbcLYKgpMAAxPOImfn+FhGZEEQUduXBUHRFI6r/ECJUJGkn+n4FH8PPpUFON1TQCtcyCIMQBr9RNpNXTlIJUnIUGRmA0P56H5+FgX6JzsFOBCjCTA10+6efztESlIQgqJUB5p6AM0EpIgkaGZjzO3gOeyi1QF9lClhtKuSp31289j8bavRnsalcpJBWYIPMkGfPT5ednphNmnqrkbS6w2qFrbUCnMd5A+cCzufsUzW8kx6rDLbbC3qm76VwJlUJbSSY8D7/iNptHWMbiLVKqqprqSyEqEFpvLpr0EQZHj7rT7f9LRFCqRphISDKykECZ9nT3/AI+EcYrogLnqUtp7qFhRO/ekfn5+VtKhlc1wN1TqWCxsoQLTzZlKFiOgItsVdYiQH3RJzEZjv1+6zkcpyT7a/tCdVDnJIOn1sdrFfVgtBQO9oSJTIgRJ193wt0YlB3WQWW5pFNbWaKLyjlgCdYtoVbs94A89RZTVSIyBwoSZB0CRvrp+nlbj1RKkSpjKNydhvtH1r7rSD2qNj1SaalZGoEnnbr1o5MhEgCBr9ePxsZVSN5ZUlSYE6a6bfX0LAPU6ENZkZt9ZtIFpTeJY447CXF5u8IBJ5W49ad73e0VuOth3vtKSmCie6lSB5hRP/ULFQyogmRoJ52cWKRuuxUlJzBInnbv11ZEZQJnYxYAtrHLxtrKomAk2ewTZijLVS2lQK0d2RpvYyq8myns20qy7ZiYMT4WTi24BqhW3S3MHpZsoKWYo76y2pARzHUaTJ3FtB5OoJEyNZixPWLZryFllCV0NUHNCpGvSwUzFtWy0homJQtNUO01S1UtLKVtLStJG4IMg2+kvhvik4u4d4XxU2sKTfFzUVfI59qwlf42+a8biLe+foIYmViz0ReGt5Lc7RynupV3rPT1Z5xgD/hbFuZ7Sx3jjk6Ej1/8ApaOHWLiCp+VUuRvYI1Du4NuTvbStbcmCtgNb0Wdu4T7Vs7dzUz8LcxztqCbGCnlC6YfcS+4gq0ISr7x+FjHbOz7RsTSSmoQdsyFD4EWHMzboaM5ogq8oAKG7Z06lRtoPu/zW4mRFsMnna0AhErrt3Brmtnbu8lbWD52zbnaQF026F9YdOmY2T7xecD9K8VnRZQdeRFjnw91iV6pWaUrbIlCkrnyNk4aJ26Fddu9Ht25NQ8P44i2gZTIG9uFD32YjRPsuxUukRPnYdNS6UTPIcrFI0MjWwrAlAHutXc2yI3VDGpeiCr52QcV3hetHh+9Lwbvc0RpaV2oStFPnKAhJVqDObbkBOtld5fZpkIKlKMJSOZ/AflZBxLhlvEVyV913tVPuIrmVtAIOVDBI0KUiMxBAPenUctrBcwyeEKZIaLqvtb6TdwIum777vC5Hq6rrgy+lhbDCV0ockF3OHVBJA1LaghY2UQdLN3iZifBF4JrUVGI78Xfq3E0jtGkLcNGcx7nYJhttSvaKHVoUEmTJ3auIeH1z4lwxfuE2r0dp8QYKfco/WnezUpTKUgrkhQBCyc0nUEZddU2rWzipVLSUGHnr6YdvO6K3KyENoU0wlDS1ZFFaSFpSvUEqKQqImYtnVmHiWBzSbaH1We6tkY4Ai4KngYnwGkBLlPifOBCovCmQJ5wlNIQPIEgdTvbLNxn0g6ltlDdVSXut5KQHFJSkgqjUgpWARPQAeA2tlvODhtbf3f8A9v8AhaPFh6r00fH2unU27bAUItw+T20RPW3aDpv/AFt9IO3XIsFghkjrHvt3AAtwklO9u0kWHZSXYEH9bbAjbblbkEKI193S3YPKySWwBO3vt0NtjbkRztsGySXUbWwTG58ranwi29Js1k4XQJne2wSeY00twPf1t2Dv94tEqS3MA6W34C3MmSAf6WyYmySXQI2Nt8teduZERMe+2TO+1kkujAOmltnvD5W4Jts+NmskuF01K5o5TtL/AMyQbBG67s3/AHfTiBulsA/EWMSAOnKbZmAHusrBPcom7ct2vI7J2nJR/KVqjfpNm9ePCXhve6iu88FXJVlW6qi72XifetBs7c2kZjbJjnItEsb0ThxHNRbeXoxcEbykO8NsNImT9nc1K375Q2k8zz52Zt9egj6N99uoqKrh7RpcbgJWzVVjMdNEPgTqeVrBzr52wq/mG1hGCM7hSEzxsVVyo/Z58Du+LuavigB1Cae815QY3+1S59RZtXr+zdwS8srujGl/0hCSEh19l8A6awGm522kWuQCImbZPLoLR7tH0UuO/mVQyv8A2a14pacRRcTXVqXr3rmQE+UipnkOVm1eX7OXiYktmgxdcjyWdU+sMVLc6RPcQ5yn5W9GQZETbJBM/jaBo4ypCpeF5fXr6BXHSgQ+umbw9XrI0DFc42VdD9s22PmLNS8PQ/473UhPaYBYdZbJUssXpRO6Cd0h2ddeXK3rbMG3C0NrTDiUqG3eFhOoWnmiNq3cwvHeo4BcaKdpVUvhPil1vdJprsee0Ame4kiCbIN4cPMa3PkbvjCV/UJBlRqbufZHiAVpHhrb2ZZu27lMJSugpiAIjsk/lbf7luoiRRpSTuUKUj7iLAOGki4KK2tA3C8T1MBLzheXBb2BV4anXXythpnCySh0qcXIToNR8NOWtvaerwtcVcCippVug6Qt1agfcSbNy8OCXC+9ZN4YMuV8k7vXVSOke9bRPzsB2GSciisrWdF4gFhdPfTzI1UtCspJ3Mjl9fKz/wAPhblz0yVKISjMQdUlRnbeNvhb1QvP0O/R5vZwvVfDO4e0IIzt0QYMHfVkoj3WRX/Qf4Glo093YeRRNme6xVVYVr0Kn1D5WBJhsxGiKysi5rziSD2WctaAADuwCeRjz+UeFhEUPahLJKte8VHyEyNxuOdr+13oD8MnVZ6C879Yyg5Ui8GsvwVTqPzshu+gPd1Mpxd3Yxr0FQyp7Snaeypjb2m7U34fUDYKwyrg6qlibto1OEqQgAAgefMkWGZu1lAAUkdo4ZlKpPu6jz62tzXegrfzYQKDGTbwSc3293hsTB17jy+fhys3rx9DTiRSl1ynrrlfWYCAE1Qnz+wI687U5KOpbqWlWWTwHZyrQaNtx09nmSGkgFSj7W5kGOXOwaqF0iO1IUuSMyY8tJHytPFb6J3F2myhvD1HUJTqpabwYQSfJakn5ToLIFVwB4qUJccdwBey1AQgUyA+B1nsyoHrvy91q5gnbu0+iIJIzsQohcu5RWpKMhABUpWUgDwHWRJ8osXdpXjTEEN96RouCdxG3OD8bSZX8LcdUK0t1WCsR0qUpCnFO3c+kdYkpj4aWalfc9TQKAqmalkpEJQtGVRPM6jx3iwjnG4RBl5KCsU0zjL6kqQjNmClFKpgRqT0P6WXMG3itm5nmwStaXjl0gQQNfHbeweM6dxpNSktDtnRmUQQTAI06RH32T8GEvUNQ2ZjMkgEzGhnfxA3tdY0uaqrjlfolioUVOFZEqV/MfGd/KRp1izQxRTtuUNal0SEtzmJ9525ab9Dpzh4LQsoKpGYmZjeOnWLNy/mu0aqUdmcq2VAa5pGXz1/XcWu0/hIsq0wuFEjzZglCSspBPtZp13j4+MneycQpQLWQ5VCSfEffEHz+FlRSsoSlJcBI5QoGefhtBkdegsmqKHESAdIkHmBz+ult9iyHIJ3u5ieQGpn5e4fpYN1KRlKwgKGh72/1pHlYQrTKlNrBV7KtfaHMjw0+VgFKXKc6CEq2GhMSeXLf52KAolaXkICey0He8hyHx+uhaqALTgTEggkk8/A/XOwy8wSCEqO8iZ25zYJzvIcCsw011nX6ixQo25INKpups5NGqhXe/zJGn/LbuG15UEBWYHUHWeW/kPnbinSV3bVoynuLbXoNvaH4i3KMywJB3g66H38rSKbktltmII3MEnQzt8jbMqUEJVEjoYiLYVkmVCJ8YERE/dbkgqOgGo73Ub/AJWSS2UBU8xpGuvPfrbSmkpJTJmYBI++3RK8xQpskHc89LczEFWhO08/ysklzlAJ0BIEeztblTWsKA12k2EzbnLmmCAVaQPvtpUT3ANehj6FnumsgFoSkab+dg/Owy092dbBAHUeFpAqBC0N7ezH7JXFKr79F6puNx3MvD2I6ylSn+VtxDTw/wCZxdvGcbi3p1+xnxOSxxNwa47oFXdebSJ/+mbWY/8AZ/K2PjrM9GT0IP8Ab+6uUDrTW6r0vVI3NucwG1tr2sGfPa3DhdC0XCE32tzEWxB2EW7AB2scC6bZBqGVbSjAhUfERYY6WAqP7on+UhQ9xsY20623KA/p2VeYc1mlt7crYJ520QCdvCbXwECywxFtDU23HIi2jPM2e1kitknb77BVCA4y4iRCkkRYUSdANbaUJGnjaYbcJr2SfTnMygkz3bbVJMxYKldQvtmEKSVMuqQoT7PMA+4/dYYyNOtoqV7Lka6WEY/iB1OhsGNz0t0FFJOQCSDEmAT0+hYMlgLlSjJvZDpbW4sISmVHQWLvVV1IafQ/XtJ7MFR74JSoDXQaz4WVKUXWy16zeVTTgKSSO1cASB5ExaLMR8TbgRjd3AhvKop3HmBVIbYIRnbGiw4QFOAAHNkSmf5ykA2zpcQFI3OSAeV/79FIjiuyhVX9Jhy8rt4vf21wJX0lO09dvqt4O1TDlI2t0lTZaW64lKZKM0EHMIIjQC1bsPcI8l5tY/xxjS6MOXLeLy00LLjwb/eBKiVFIAKyiDBUhJTmj2gDF4+L/EK5au42rmvPD9XeVI+UP01AxdqqenpRPcWXXWuxUtAhUZwiQUybUjxDwexbjYGvxNiynpO3fW8mvvJwUgLCRAAClAOEJ0OQEaEawTbDqMUdM0tdIGA7m2v23Qn07WPzBubyVgaK7vRhp6NhhuopXUNtJQlxdM6VLAAAJIqkiT/lHkNrZaGqTCuGaWlZpX8YXG44y2lta+wq+8oCCf7sfdbLckaB1/8AUv8A59la4v8A6BesLwPaanmbdpBjnpbToh3mJnlbtAAPzt9DO3XJN2WFRG9kyqqC2Xni66lLQJISqNAmdLKS5Inpysi1yS41VNE5QtKkExtKY/GxIxcoUhsod4b+mXwX4nX7Q4auDEN60V53ioN0tPeV3qaDqyJCM4zICjsASNdN9LTqxeFROV4gnlpaA8L+i/hC471uy+lOsVDt3PtVTbqWsqittQUk7nmkc7PX/SE3XY+VhGhpK5H7upi9VvVFK4whS1qGRLecDOISs5kynaCYMFyBw1CgHkHRSo29mAUIE2AfvLsl5Gmgop9okwBYGncIZnnvaOeOdJWXtwnxDdFDUOMu3my1RFbaoV2bzyEOAEaiUKUPI2GIxdTMhUgUWJ6SvQXaN6lqmkqyqXTPpcCVdCUzr4TZUbrG3Eg6687VI9HDgtWcKMcJvG66t1q77zpXaerpQtRaUQkrQqD/ABBSd/8AERzNrOitS1W+qkjVAWPfI/CzuiHJJshS84+00guLMJTvpNghedISCXonqg/lYlVuk0wE7qSPmLVy9KjGHGrCN4YUc4SX4xRodarnLwpnqFp9NUUFjsgStJKQJcHdInNrtYYiuph5vorQt1LDolp5CttjtYTMI018rMLCF+V1dhS48QX2wilra+hpn6poApDbriE5kgHUd5URZ2MP5kkg6EWgYraJxLdKMwRI+NsCjOhNmBjjE72FMI3/AItprqdvN66aKprk0TTnZrqS0lSg2lUGCcsTB32tBuG/TKevpxtlzg1jNhxUJKadYeKSTH8qbLhW3Kdry7YK2RV/NbM0gybJtHXLqWkO95IKQSFb2J1V6PNrqXe3LbVPMwmdAkKJ2nnZhGTsnMgG6Xs3Tbfe2FWu8C0K4B9Kbg5xLvZi48FcRqO8LwqZLNK7RP0y3YGYhPaoQCYBOUGYB0tLVNeSnQC4iDzA2sxjISEgKP5gedsJjnZEvfEdJdTjDTym0mozZO0dCAYAkCdzrYNGJ2nAFJplKB5pcCh7utmylSzBL+bqedsmNd5slUl+MVb3YJYdSYJlWWPvsoocSsSk2jZSQgV1tk8h77czHKwL9XT0ye0qX22kA6rcWEiTyk2iQkjClSf1tmfxsC3UMvJztuIWnqhQIPwt3mkSNpsyS6zdbbCup+NuJm2oJ5+dknWNHukTzI+dhM/ObF2v40nXvq++wgII3+NmCRQoV1A1tokg6TbknTytiTqQI31s6YLqdiZ+Ftheg59LBn5zbcgbCzFTBuhQrnGlszGelgkkbm25J062FayVkIFDcae+2ZzOlg+ZtgPjZJITNHP320tDbgyuISoHcETYMHnboK162YgIjSgTdt2LIUbvpyRseyEj5W5dui7Xk5F05KTyC1AfAGxgbaC2wR77QyN6I2Z3VNq9OFPDe+ypV8YLuWtKxCjVUDL+YdD2iTNmnU+ir6PtXnP+ivDjJXGb1e7WWCQNtWkpOk6WlIdbbBg2gYIzyT8R45qB709Bv0dryzKTgv1VZEZqevrEn4B6PlZl33+zb4HXqVOU15YkoSrkxeKIG+3aMuHmedrXZjpJkC2wrTQ+c2h3WLkFLjPtYlUDvj9kXw9fzC5OJeJqQFMJ9YFNVR5gNtE/HnZgX3+x5v1KZuHjN2ipJAqbiCEjXTVFUo6T/LyFvT1KhvFu0rGotPhW2KhfyXkBe/7I3j7TFa7rxvhCrSmcgWuvaWR7qZYB9/vswb7/AGY/pX3WhaafCtyXlGYJVT32w0I/+0Fsj4W9vgsH8rdJUN+VlZw5peE8l4AXz6DPpV3OV+t8F75qCJA9QfpqyYG8MOLmzQvf0afSFuQKVe/AzH1OlOpWvDdYUAdcwbKY9/nb6LVtsuj7VtCv8wBsXF13UTP7upgY5MpFkHvG6bhtK+Z57DGJLnTWMXzcN5XeoNHu1VI40ZChocwEc7I4SoNzty1t9Or1xXVUJKHqcqSd0B1YSfcDFmve/BLhNfxUb64fYfr83teuXWxUT/7RCrPxnDkkYgea+bMocEZtNNdCNOU20pRM9yATtNvoWvj0KvRdvxJFbwTwinNrNPc9PTn4tISfnZi3v+zK9EO9ypX+i+npFKMzTXjXNQfIP5flZd5tu1R4PmvCSCD3UxG2ttlLihCUmJ0jW3tTe/7JH0Y64f8AZzWI7ujb1a9zt/8AXNu2Y1+fscOGTqnF3BxDxTSAju+sLpaqD4jsmpG/OzGraNwUuCSvI9CDlzDQzrIjT6/C2dkiJKhGmo13FvTS+P2Nl8IBVcnF9xwnlVXA2B4SUVZJ/wCGzDvz9kVx2oCs3TjHCNcn+FLyK5pR/wCBhxM++yFXGU/AeqCqSj+ImT42BKSDFrhX/wDsvvSwuwRRYcuC8ttaa+m2gT/9o7I/LnaPr59Aj0uLiUF1fBi8qhJUB/qNbSVh+DLqiPeLFbPGfiCiYX9FXvUWu/8AsjcRqur0k7zuJTuVu+8M1TQST7S2nWXR8krtXC+/Rg9I3D6lG9eBOPWG0/7T+z9UtEf50oKfnaT/AEDW8TcOfS84fVN/3Fed1oq6167HPW6RxmfWGHGkg5gP4lJsGvyy0rwDyT07XMmabc17kKMggW4522VamLaHjbz4DVdKFnusIgzIFuBr4W2BrY7Eiu3EZ21IP8QIttpWdtCjzSLYk5hB3txTK+yUjTuLUny1kD4EW16A2JCrS6BCj3W3FtCDvpbYmNLaoCr3Wc5m2hlgcrbJ03tke+LOEyyCBIM25URtbek/ibYZ2FihRJ1SahIRW1CEpAzwvQb8rdq31GptzVAiubWD7aCkj527VKhMzaFksy5G4IG3Kx+6ruFW+X1tpyta7bq5CxFsdo+3TI1ccMJTv7/IWdvYeo0JbpU5lJRKdPaV1PvtXka06nkmfIWaDcpt3xQ3Ve7qziR5hm52VJbDbjvZpfeUYhZkSASEhM6qJkGBaMuLXD3hxdlwV9fT05w2mphp28aeudoHHgogqSMiVuPKMJ7mQ5tdbTS8GKTItbWcNkFAy5lZoiR46n4myPifCVFjK7Td+KqRD9A6tLhogshKinUBwpIzCYOX2dNZti1VFHIx0b2hznc7bfflbknZKWuBB0VSmlPXtThq7r0x5f13UjfaJXWMuPtKUlKyQy0ptxSFQ2YByLy90e1Brzih3CWIcbU9wVeNsVu1F/VyqZ2gqaPs1sNkpzuZE5glAyz3koACZCXIMXz4m33U1l10l1YBoFIYoX6dKKimYCmlpCi2tlkJICYSCCpRSjZIVMlNQ+N+M2g1fdVgm+Lpuy+H7qDFe3dRbdqWVtocYCHKhBkAdqhTeTKFDtNJFuIqaLu02Vri4deQPodlrts+LMdPJMuvYwNS11RTU/ol8SFNNOrQgqdr21FIJAlCachJjkCQNgTbLRX+4OMqu9dtVfIpDrTh15IWG/4cwKhBiJ0FssT2fBzkP/5uWfxz8v4C9jnxD0a26A0j522+gdqOptiRpJ3t7s46rmW7LgxyMWbGKgpdwXy2DGekqBv/AOrNnUpEg6TOwtCXEk8a2MUVFDhqruN3Dt4ZW+zqGFB9lCkhLneG5nMR0kTtYjHNb7xshPY5xsAq+cD+BGL+GuMbgv65MX3kKBuoZbq6NVQS08wSErSobHunSZiBGokW6vZtg3lQurbBXlcSkxqB3dPKxdjDNXR3C45QpbVXNMqUwlWiS4BKQT0kCzDwXjTifizEjd1Yu4d010U9Iha/XGq0LBWIGUIJkg7zOmWxjIzS7kMRv6KaaUxTb7J52b+M2/WMLwnvAu0yjPTtE62ctIwpLGRQ5Rp5WirEfFO6+Hle9hziFdte1dzhUaS82KdTzCmiZyOBPeSpO2gMgA6WUbgTome0gJ23Mwhp2mWlIgSNtu4q3dVUZ8TlCYhFK3OvPMo/daO3fSS4HXVS9uzi5yscAhqnp6J8uL/wjMgCfMjzs4OGtdeuLkvYtr7vdoU3m6XWaZ722mAAlsKn+LKkE+JNnkeL6pRxuAUiPkilbJH+0Ry/xC0I+kT6QaeCF44cp3eHL2Jqa+G6px19moCFUYZU1HdLas2btDzEZec6TNiC8KK5LndvS9KxqlpKYoU686oIQ2nMO8onQDW0Z4pc4WcRaqkqazG9wPCjbcbaSmtYcBzkT/Fp7ItAG+ykR1SrTvUHFDD+G+INzXveTFGtlN409GVBDTwUg6PJAkqTJjvQCNuYf11uq7FIVMxr1tGT+NcDYNuq6sB4cvOlrat9SKOkpqV9DikIJlbiwk9xITmOu5gDwki6AtVOhShEj32Y7papGxVV3Fd+FL7vDFLymbnpaSqevB0pUrLTJSpTioSCowgEwATpoLRThfiz6MFU8y3hvHFD2zykoaQ4y+gqUSAAM7Y1JI0m0t4hu5u/MN3xcamw569TVNNkVEKzhSYM6c+doywl6PuE7ir2rxew5QpeaUlSVdkhRBBBBkeNhSa7i6LFpzspqoT9mAgDLG4snOjMa9tWynCCfNtIsp0iUNtBAVoBFk1SAXbwTPtKPL/1abTZtZQfuoMwn6MWGsLYoubFDQp2qu6ahFShxlBCllMmNRsZg+FpaubGFLe991l3ULNUhN3rDLqqimWxmcgE5UrAJSAU96IM6EjW1UeF3B/HvDbGlwX1dGN73dudNY23VUD1UtbTjKyEqSRMHQzrMeYBtbspaNawsphZSoBUcpGnuk2ThlCduuyReLfCy4OKNysUN/UYfVSKU5TrzlJbUoQSIjcCLIfCzh/SYGwBW4QpFLS01UVgSSokgODNuTM942lhgZmQCSdLIaUpS/eLYG7xPxbT+VmGoNynvYgWULej5wovPAN6uXnUYmvq8EuU3YIarKtTjbYKgSQk8+6NfE2shTkBEEAWbGGqdtLDZSkeyPOzoQiBpp0Nq1iNCbqxodghiqzH4qKUq4Utg93t0KIGs2e5O8aWZnEtAXcZJ1hxJtNouDdRcU0sC4j/AHe47drildk/30CfZXGvxH3WTcMX1UDGia1T7qkLqnioZjBnMNttzaLqzGFdc3GsYMqZRS12H2Lyu1w/+kNPvJqEeeQsq9yrPHBztQqsadqW1NuevP6Hp2ysp96YjwsPZgBUhq64ViaR8OpmbGQZ0H62SLpUpTQk8hZVBgWThYpLlr2nJ5L5+QsLMaxYFsgLcGs5gY9wt2TPKbMBdJNvFGMjhypYYaoRUrdSVLlzJlHLkd9bD0+LqN65Dfbram0NtqU6gKzFJTumY1/UWaXEgH96056tHkep/OzIrMTMKwviJuiqgsUzL7TyQofZvNo1BHIxB8QRYLn8OpEbvdIVlsXEp87dwVM2HcYUWIu0SxSvsqaSlRz5dZ8jZaU8IBSPKbRZgt9VHVBLayQ418dRZ04wvx66cNVVayftQjKggxClGAfnNnpnGVlzuo1EbYn2CP1mOML0FSqkqr4aDqDlWlCVOZT0JSCAfDeynd18XZerPb3bXM1LYgEtqnKfEcvI2qDxIuviBfd2sNYAv83XUtqW86sNBRd07qJMwNyY128QXDwIxbiZVPc1fiIKbvJThorwSABmIWUEkDT+VWkCRpYTXvc3iHa9kQxNByDdWnW6lKcyiAneegttDza0haCCCORs0cd3dU4iwZel0UTpbeqadSWyFFPeGoBjkSAD1BItDvCfGLl3Vie3loOj1SrbUfZcSdCRyIPyUbWnxZWB/K9j5Ks05nFvNWQDiSqBbqQBJMWgyqxbed94pQunvWrYo+1S0hNO8pGZsHU6Eb6md9RaYUV6V06n8xCInvdLV43CVxDeSNJGYwCeaUA6Npt2FAiRHhaBeIPGupuC76zELL9Yi66SAhFFTh59+SACB4zO4AAk7TZV4QcaWsd0FBeLNWqpu+8wQy462EPNOAlOVYTpuCOfIgmwjKL2tpe10UQuAvfXeymYK26WwLAgmJsj35frNzXPVXq+kqRStlzKCAVRy9+gsyl8YKKmDZvCkYpw4cqM9aE5zEwJSJ0tOVzYQC/mmia6a4aNlJwUPL3W2FTZsYWxtQYnU8zSsqQplAWSXEqBBMbizjSqedmY8PF2qTmFhsUMF6aRbebxsDm00Pvsi1OMLhoqpyiqr4pWnmzC0rXGUxt0tImwuVEAk2CcIX4267TlrZJu2/bqvRRbobxpahYEkNPJUQPEAzY+XAEk9LR0cLhIgjdGQsEaG3Qc87FG3Q4mYOmltioSpWSdRaNk4dbdGguNfjbfaCJBsB2gSnMT520lwKEpMi0HAqV77IwHDoOVug6ORsWKwNSdLbCtgPvtGymLI12pnU6W32s7WLZo0m2ZoG5i0UrAoyVk24JB3sBnJnWTbO0nS0SErWQqjPT4WBcS05IU2lU9QDbCsnYC3Kj4izFo5qQNkWcu+hUcyqNgnr2YsA5dNCuCWimNgHFCPcDY6penP324Kj1sMxNI2RWvI5og5dFNlIbdfR4h0n75sCbqXMt3jUf7wQR/4bKSvPa3A9oePUWAaOF3wowleOaTV0Vcn+7rWj0zMT9yhbnsrzQdfVV+9SPwNlJY03iwWnvtDuER5KXHciYVXo1NG2SP5XvzAsQumpvdx2rVUXe00pTuYN9tJAiNwCD7PzstGedirY7OsWJjOJjyj9bJtI2IgtT8YuFiF0l2qjvUh8crgP5W2agpmaV8R0SD9xsPsJ5G28xHQWthh6oOZFjWsDRQdT5tK/K3IvGhKoVVIBA2On32NnXU21AIgj42bK4c09wghU07glD7avJYNuiZ2I+NuHKWncELZbV4lANgF3XdylKPqbQKtyEx91peJRJARe93006WH1BUJeSNBsCYtwKk1Ci3TQcuil7pT+ZsSv8Aw829ddSihedYdyKKFlxa0pMb5CqDz0sfo0JRTNJbEJyiABA26WXiLiEjlAuE4rkugUKDVOJJed9oq9qPwstKcSlkvKCiEAkhKSokeAGpPhZGpr9ZRTIRUZlOAQTHTY2Zt78U8WYXDlTeHDh28KMvFDT913gh4pRBILqHEtqSdDojPrAEkgWo1dXFRsMkxIA52JQhDJO6zRqn8+yoH1+oKm+5AQVd1tPjynqfdytEfFbGOJr/AHEYLwNSMlKSl2vqKxakN1TWaFU7aEguLSZBWpMDKCnMCrRuY746X1fFUjCzLFNcbrikuFovpXVvNgahKTATBIBJSqI2HJpXxj2uuK4n0U123I1U3k7kqF1iy6p4oSIU44rMpzQgDNISAQJ2tw2JY/NUPbHQg8M2u62p672stinwtzGZp7X6XQOJGL1RcdXduLcCVfaPBZq7wonGbxbJgAKNPmQy3DaUpSghRSAhIB7pNauINw8J8Mm7bwwtW3qugvFxLLpW0W3UvaOJcebXlSlBKoSlKEiFE6zaY73x/iRFEf3HeWHlVT6FNoRTXc0ptDkEpUVhsJyJMGSoa6QSYMNX9iKiqr27bGr9Vie+EEIStkqpbsIlJAKEJ7RWaNYDbhKT3VFOmTJO6STQG3RWZImxstcEqJ6ziHxOoKt+hpMNYmqGKdxTTTq+yWpxCTAUVZBJIEzAm2WnRdXg5S1KYwuA2SSjs7szoy8sqlZlKHQlRJ5k72y0Pa0X+x+/+VT4D+q9HK1ID5002AFuECdbD14+28zysEkaAje3v+65FYEnmLAP0TbypUmTY0lMxIt3Gm1mukiaaVKUZQnfSxZFzU7b5fQ0ArmQNbKuWPdbeWNxZiE40QCGsqYsmXzh6ivhos1bCHEq0IUkEGy0RbYRPKzGx0KVyNlGzPCTCtPV+ss3PSoWFTmSykH4gWeN3XU1RNpbbSEpSNNLK5QneLZk8LQbG1puAiF7njVJd93LRX5dNRdN40zVRTVKOzcacSFJUnoQbQzeXoxcPKl0rGH6VJJJ7qMv3WnuNY2i2lNpVrFk5ufmQosdl5KHsD8DsI4LrfXrpudhp+IDmXMuJ/mOvO0pU9Lkby5YseDCQZCBbYbA0jeyY3JzTvOc3UC8SeFPEC9cTVl9YW4o3/c9PVZVCkZdJabUEgHKkECDE+ZNmuzw89ICjVDXGa8lJ2lyhQ5/4jr8rWfXTocPeTYH1Fon2bJ3EJ0d+yk0sA1aku4qapRQMN1z6nn0NpStwpAK1AQVQOptHHEpji9SYlQvAlfc6bqqEIS81WUylLS5JClBSdxGXTqPG0wIYCNrcO0jb0ZgNPCxRI5vuoRY07hR5WYWq0YYqP3e22quaYK6cKkJLidUgnlqBZtcO8R8TMR3w5SYzwpd91sUqCpLzFSVKWskDKEEmExOpPIeNpoFIkJKQNCLF27saadLiUJBO8c7J0rzZIMYNVqnZKWwmZ05WijHPFCiwNit6467DWIKxValt1p+hou0aMjLlKiod7T5i0xJbCRBFk+8bjpq91LrzQUpBBEizGQtGgT5Gk6pMw2y6mmaUtBHdG/lZxIiwNPSpa7g0FjQSRzm0C6+6f6LShzizU4hKabuJx2ocQ022tJUtaglKdeZNnYYsh4uwzdmLLhqrhvilRUUlWkJcbWNDBBHzANptcBuolt9lWbi5hb+0NPRYtw0609emGyirplMrCs6Uk52iRyUhR06xZwYUvBurTQXi1HZPlpaOcTH17rSZgvhXhzBF3O3Xc13IZZdcU4oanMowJJOp0AHus0mvR2w5RYzTimmerkZaj1lNOH1BoLmQMo5Axp4RZnyMcwA30U2scHXBCk5m86e7KL1qqUoIGVOgkkkgAD42WKG8qeta7ZkrAnUKGvWzVxlharv7C1RdV2Xi9d1UooU1UsgFbakqB0kEco99g+GtwX/AHBcvqOI7+qb2qi4VqffQlBgwAkBIiBHnqbQc5ugCQaSLp7oIDqyD0sIYiB52DRHan/KPxsJrGny1sgbJlHvEVsGtoTqJSsH5G1abvYewpxvxVcN6OKduPiA0iupQZCUVbTCGX2v99AQeWxtYrjeMVU1zUt4YRuFF71bbxQthTgQA2RqZO50A352j29eHWIMb4GpLxvG7W7vxBQuJvCmbQ4Fll5JJCMw01T3Ty15xaMkQkmu46EW8wUeKUxxWA1BunnhOWXaRBWV5GuzzEbxGtl3iInPg2tVPsBCjH+dJtGnC/FGJ7/xCbvvDAd5XS3ToUXqipWAnMDAATAJJmegFpixDc7t8YavC7ElIXUU60IJ2C47vzi0aBhibleeaeueJHBzeijLBFO09UPIcAPdGh982bTVM3d2Jq9lgZQ3eJWlIIG+VRPz+fwRcP8AEy7bivF27b+fTdF70Us1FPW9xKj1B25Tv5SLK1y3xdGKMTFdBeVLUv1b4fcbpnA5lACQToTGiRv1tGcmGEwuGt1KJvElEoOllYW7yXmQlQ9oSbQZxCwyvC2PEV9Ex/qF/hanUpGiKhIkn/eE+ZJtOt0sqSykqSQSOlmbxkp2UYfp612Einq0qUTskEEHXkLXYiHtLDsQqjyWuDhyKaWA7sFXWpeCISyAkCI1/paVL3aW3h+tSgwoUzkeeU2YnCd5itpqlTS0ryPDVJB3SPyNpNqmQ9RONEGHEFJ99qFEwM1Kt1j87rKBLnuakv6nprtq2kONPISopIme7NiuA+Bt54Txg0u7sVVjNwU9Waxq7W2UBMk5spWQTGbpGgjxsewheLVI+2HVAu3e+umfSN0KBKdRuOtlurxXiF6/akXbWFijZDaWShCVdpIOcmZnUx7rQe0QGQOOt9kZhdNky7WT/wAdtF7Bd7J3PqjioPgmfwtA2LMCYf4i3PRUOI6RL7NO4mpRqQUqyFMgggjRRtYS+0+v3DXMAhXa0riSOspNq+X1hZeOcIUlzovK8LtWvsHA/RP9k6CncBQ5HW0q8foMcDazv3CjQn9V7SL6KRuBeB8H4Boai78M0iWn31BypXlVncAJCZUSdBJ08T1tMCFAjS0O8DOHDuBaasdqsQXxe79UUp7S8axT3ZoHJA2Enc76C0wtwIEWFFfLqbqUpu7QWXSzCSqBpaueL2K5yoxExQVPY1ZeqksO5QoocIORUHTQwddNLWLcEII8LQPithTOLb6Z2BdSsH/MgGxpm3pJD0sfyoU7rVTPuPwom9HzF/EWsYvJGPHQu+LjvINtVTdOGe0QUgiQmE/orW1wq6rd/ddS7TLyudgtSD0VlMWp1wu4mYsv7Gl64Oxfh2koK26EJq2HqZa+zeRngaL6778iItbS5a395XMzVEAdu0CoDkrmPKZtVpvDOQ4WvY25KxU6wgg3tcXUY8KL6ql3rRVT9W8760y424VrUqVQTJnnmTZOv3H2JWOIl7tJvaraZu6raShlt0oQWsoMFMwZk6n8NC/D4qo65uiUqDRXk9TGdP4yPuNknHbAo+Kd8NgkGtpaapGnROT/AKbWclu9M6EkeoKCTmdA7qNfQqx98Xv6hctbeaEhz1enceCSfaKUkx8rMThdjy/MQVqWr2qmltP0xeQlLSUwvunQjlBO/wCFnJQqF7YRZKiT63QpB/3m9fvtEXB+rLdddCFqMhC6dXmAoR8hYUv9Bsg+YflThH6pYeh/ClniTjheDropaxqkFS9VVKaZsKVlSkkE5jGpGnzFj2BsVuYquU3g/SIp3W3lMqQhZUNAkgz/AL1mNx+aWvBbFakEmjvFh0lPQkp+9Qsa4NvqFFeFMDol5Dm38wj/AKbKpbkbG4c7g/ZRhGYvB5WUp5tJJgWaD/EG6mrxqaBAqnV0zhbWUNjLmAkgSRMbWdWYlo+VoHraZpWI8QXW6ohDlQ8hcEpIDgmQeWhsKoJjpXys3bb97KcQDpmxu2P+FLl14zuutqE0peW24vRIdbKJPSdvnZwFYgkjYWptga5MUcOuKNXhC8MSXneV1V1CirovXX1OlohRBAKjpsqYiRl6Ta113166+5G3s5Lq2ilRj+ICCfiLAhdJxBFLbUAg+RRZGsDC5nI2QtBim4rxvB26qW+7verWZz0zdShTqY3lAMj4WUKioSykLV15m1P+ENcbs4q3U84YUt9ynVP+NC0ffFpW9J+sqk4GpW6d1xLTtXkdCVQFdxRTPUSLaXdCap1MDsqgmHAE3VTQKlCmi6NR52xp1LqSocjztSHBHF298J4GxHhlu8HGDUsioonkKOZp0KSFhJPs5kbbQU6b2kL0TsXXtX35f103tfFdWBVMh5hNRVLdSgIcIOXMSBIWnbpau6LLEJfOyKJLyZLcrqzi3ENpzKMCwYfaV3QseVos488W3uGuH2lXZSt1F5VoV2Ad/u2wClJWoSCYKhA0nrpatN1+krxfob7pUXlfgfRWoDjDdRdzaWXo3CSlIJGm4V4WC4ZCARujtOa5HJXqJ3iwcmY1+Nmxw1x3ScQsI0mJKdsNOOZmqhnNPZPJ9pM8xsR4KFnOZJ0982YG4upWI0K0TpYBaQKhtXXSw5sDUEwhZ/hVPu+hZiEhoULyg7W2OpFsgeVtDQWkm3Wz+FtGPxtsxbRjysklzy099sAG8aWw+0BbfLeLOAoE3QT6M6FIOx08bJd2LUujQD7TZLavAgxZYKQQfvsjXelbVZXU5QUgPB1JPRQn8LORqCkDcI3zsKxTJrWKilISSUZ0Trqk6WDI2jnbpisp6BZq6peRlCFZ1QTpHIDU68hqdhNq9ZE2SFzX7WSY4tddqhniRhbD143k7dV9XVR16ajI96u8hKikAZS7CtgkzqNdY52jOvwLetY0wrCGOr0u5EuEu1bKa9socVmJbS6QpCU6BOVaZAG8ZrTPWXRWXled84irxUUZqXQxT077Ce0ZaQhJQTvBJVmyzAJJUCRAZ2Kqegw/hWqqL5YcNG42pIVUBZ9ZUUmEFQEkqOm0axoLcuGxTh0j2hreS0XPdlGtyojvPC4xxfT12VN90JumgQH2001OA7VMKAGd5/MokLV2gGUoTAWQkQCqrvEa5r6fxRe9z4rqTddxpqVUFJV1N6rWt2hSv7HMlJU6lgZEqg5UGCRBM2t7dFNdd0XCqrvGqup29rzfFdXBx1CWqZRAgBH+AAJSk8xPIm1M/S4wPi7EGPX2MMvu3hddV6rXNGnqszBdUgNrWsiQpcok7kZwAABFhU9JTMcA0ht9z0VeWRwZnOqkWixXh2moqemp+M16sNNNIQhpm63ltoSAAEpUTKkgaA8xbLVcpMAY6apWWximlZyNpT2YVIRA9nQxptpbLVjglJf/AFP7f4Qu/SfKvoEvBMPTE62AQkAzEWjJXpNcMa5wFKr2ZkxDlIlUHxyLNjjHH7hY4QF37UNHnnu6o0+CDb03vtOfjCyjSTjXKVIoEamTboJ5RttZmMcZeGFSQG8YUqSqQA4063/4kiyizxI4ev8AsY2uQHoqtbSfmbSFVC7Z49VAwSjdp9E4x1tmnI2TKbFmE6whNJie6HyZIDda0o/JVlFqppXxLNS05PNCwfuNiCVjtio8N43C7CRz2tuBprbYBPl4W3GmvOz3B2UcpXIE6xbUbQPK3cC2FMiPDeyulYriATG1thM7i3Uaz1thERAsk65jprp8bZljXrboCOdsg8gPGySXOU7wbaIFhBPO2ojU2SS4CegtoDwsJGvnbUDysklxEn5b21APK3ZTrpbRGnKySXOWbajmBpbsiNbaAOmlkdklxHO248LbAJNsIFo3SWoi2iARFtkAWwxOhs4KSDLSCdtRbSmUkju2FjlbCOUWYpIEspIjLpbQp0o1Fh9+Vtc7MkDZAAQ9BH8H42F/G2iPtkx/Kfwt0Brz1sk5QD1Ml9JDmo8rF03e22kpSjQ6WUIEW1AsrJrpLRc9Oy8XktpCj0FjYYATE2NRO3lbRAA2swFtlIuJ3TJxXw0wrishd93FR1ixsp5lKiPIkfU2Awvwxw3hVR/c1z0tGknUMtJRPwGtn5CToQTbMoBmPjaDow7VFZI5osi7LJQkAJ2sk4uwtdeL7iqLivmlD1LUxnTJGxkajxFl4pk6aWzLB8D4WkPCLKGl7lMvAnD+5MCXebruKhRTsFwuqAklSjzJOpMAD3CztLYiCPfYfKBy3thSOQ3tBtwk52bkoP4j8EbwvG/nsW4HxG9cl5VA/wBZR2QdYfPUoMa9T74kkkrgnh5julvIO4sxGzWttnut09IGQT1UZJPkIFp4LSTopMjxtx6q0NQkWhM0zm7iiwzCEWCTW6E+qKZP8Scutq13hcXHG4apygu+5sP1dEyvKypanc5bB0JMgT8tedrUdkBpFi713svd5SATztKQufHw9LeaeKRrHZlBPCwcTqjFTCsS4fu+76BlKluONVClLWYISkJ23M67AdbT62DA3NizV2MsrzIQPdY4hJToLAiZwxZFkfxDdbIlO/haAOMd9U2EcWLrKu7rxdRXMIUFUtKp0ApEEqI0Gw8bWAKZG1k687npLzATUMpXG0ixi48NzLXuLIbG2kD77Kq1xYhwhe2I0Vt23fXqvh1j1QRQvJPZ580KMZRrzVEdetlcM0S6O56elM/Zog+Z1PzNuqfCN3MO9o3SoB8BZdYpEtJygaARanDE5kmd3TrdXJpmvZlCrJiK/KfAfEm+Lovl31RqufF40LzhhtzOdRPLvSP90+EgYgrWMQ4rZxGipQmKFNGptKgvOrOVAgj3iPH4z9jbhxhnHFMinxBdNPWdlPZqWnvInfKrcfGzQw5wAwPhi8mryu25kIfa9hS1rXk8RmJg2PVylxc6O4LhY7WKHTNaAA+xynTqnhhKlfpcM0FLUAhxthIUFbp8D5bWhfBxF14qdpFLCfVL8eaAHJPbEfibWJYpQ0yG4G0Wjp7gxci8cOYvcerlFx71j1YVKksdrM5igb6666WiXAU3Cd5fhJo/X4g8/wAo1xjpPXeHF8ICdWmkvjwyLSr8LNThhf1Ldb7q6qpbbbqWUKCnCEpkHqT4m0r3/cjV93FW3O7KU1lOthRGpGZJE2gB7gpxMoV9lQcQAWkJhKHLuQrbaDM/0s1TK3gsF9QSefP6JU8ZMjjyIVh7svmgvRtSaSupnlJElLbqVEDqQDaHcVA0nEq8Wtk1TTL4HIwkJJ+Is4OEeC8T4bdqa7FGIE3g+62GW0N0qWUITIJUYJJJgdAIOmuiRxiw1jI4go8R4Ru6krf9V9XeQ+8pEQokEQDO/wArMyVslNI19hcH/jdIxFszC3kU3cfVDC8f4IrGWzndbrGXCOY7JJGvSc1piwmvNc6G9SEkgSPGbQncrHE6tvBinvXB9EyAYNQasKDaecAJJMwNLTzcF3+pULbIT7KdfE8zbPjnfPUMcRYNbbe/O6tuiEUThe9zdVKeAw1xXVJyJor/AFK6AJD2Ye6Dad+P93/vDhpWPBoLVRvMVAA6Zwk/JRtFPF/B9/ucTqtu57irqpV4OtvsraYJaBKACSvYQpOsnTe0746uypvXh1fNGtuX3bucISnX7QIkAddRbojOzv7ZQd8t/QXWS2NwpTGR1Xn3ftYbuvWgoH0j1a8TUUi9IhyMyNtpCVj3i0w+i/WLu7iNR0jjgJqaR+mJ2JypChPn2YPvs06Lh5WcSlXpS3RSrefoGFVlK8EnK3UoUCElWwKgFp95PKx3g5UVl3cSsOOMsrU6qtbbcZSk50BRKVynlAKiegtWqWBocGciiwuNwXcwpY9LiiccauioB7qqd9M/ylK21fdaIb1fwtfPDvCF2VV41VLfNKmpcDrCEHIgPKRBKtJICYAk6eU2K9Jy53KzA9NeyEHLQVMOq/lbcTln/iy/G1QLzuK4b/w4u471oSa2kqUPUbwBKS2VpLjZ5xAJ6mTtZ5WtLYpfIi/Q/wAKlGSDIzzB+quT6NNRhZjCNTc2HReRVTPJeq3K0pzuuOJgKGQwBCIgAbe+0xe61a/RfvamVf183fTkBLtKhYAEAdm4Ux7s5FrJjQcrZcTrt1V6UAHRYSOf32AqhmZUNOXwsMTrbhYlBSIkgxYiDm1WNrK2kK6pBsII8D42BpyFMAARqR89LCwroRZwE91s21Bjb326A0g20Tl2tKyiSuYBVMW3sNLb3E21ykDezgXTFcxqYFkp0djfAg919nXzSfyNlcmdhZKvRLiKqifQUhIeKHJmcqknb3gWk4aKLSbowRAg2T2Ev1N8+tLATSUKkhkA+29rnUfBIISPHP0BsokHlbjKRyG8/Ow5Yw8C+ydr7LvGdEhTYVplLfTSR9C0J4wu2mW5QU79Q3UJ7cut0TErqKp5CcyG0iQIEZlE9wAakCbSvjS9MVLpWmLlwgK5KmQfWX69DDYVqCnQKV/CDtztFmOafElx3JeN6Ul3q9ZNGhVT2KF1JLpABbQoABDQWUnUAGFLOaABxWNYkyJggidlcfLofNaFMwubZ2qiS/KGpvrGBoMc381d131LZ9ToWVkBLTaAVl6pAISDmzFKMkCMyyBZIviguvEpv/1LBN03w4puoU9elJkcTd4SUpbCUalS1DM4ZDclQSP5rOPGF13jeDbNNft5LoLu7MdpSMinFStRITlSjOpEbqzJUSmQIJEGHMYUzhvN5q7OJCaO4qVCHEUdNUFpa3FJKjCSMzhSkI1JWSqAkTojh6yaorMzJTa3Mixv9v8A6V4MjA8IToprs4bMUzTD90XitxtCULUq87vSVKAgkjszHlJ8zbLNym9H+jrKZqrreJ1bTVD6EuOslxxRbWRKkyNDBJE+Fstl54v98+hUO7u+T8owmsfZcICjERqrbX7j+FlmivntIC3O8AJJ+ug+tbN2pCu1zoVKo1PLTwsNTKSAEkqSQDsTI57+/wCFvYHAOUmnVPRitQ6MyVGJ1P19bWOtvQkGTJgmPrzs0GKp1pRQZgSnQa6T+dlenr3EaFRKSPamD8vo2rubbZGCWezpVqIVTNEHmUja3aAw2oFDLaVcoSBr9fdYo1VJJAIzDaNtrDpey+zz3ObUifnYWoU7ApTp71raYZqevqmRMy08pPwIP1NlBjGmLKYhLGKb4T4CudEdf4rIGaUgKBMb89fzsIFKju5Y9qSdrSEjxsVHhsO4Top+I+Pqf2MXXvOvt1a1j5k2PU/GDiSyMreK6k9M7TS+n8yTNmSFEg6nfeZ/p+tuiSQCFGDMg8yenW0xUzN2cfVRMER3aPRSIzxz4jte1fLLw09uja/6Ug2UGfSEx63o43dLpEHv0yh80rFouJVIg65hEb23mUCM3MQCN4ixBXVLdnlCNHA74QphpvSMxUEkVFx3S5z7gcQfms2U2PSKvAEJqcJUyidJRXKTr0jIbQe0qFZjoJ5jb6+uVj4JTAKdY11+tbOcVq2bPSGG0zvhU3s+kRSE/wCsYTeRoD3K0K+9A+hY416QmG5+1uC9kx/KWlf9YtA6c2TMI0gAwNfgbdQRHIRr4fXSzjG6tu7vwonCKY8lYFnj3gtyO0ob4bkfxMNn7nDY6zxs4fOiV3jVs/56Nw/+EG1cBJiDOoI0ts94QR4b72KMeqh0QzgtPyurMt8W+HjsD+0aEE8nKd5B+aLHWuImBX9G8V3dP+J4I/8AFFqsBR1BXA315nmLYFZQf8R57WK3tDPzaEM4HEdnFWzaxdhN7RjE90r/AMta2fxsbavK7KkA095Uzs80PJI+RtUMx3cyhodlDy+Vs7MAQAhR2ggRYg7RP+JigcCbyerigpUJBB56W3lPIfG1O0uraMoWtvrCog2Ms3zfFPBp73rG/wCXK8tMHyBsQdohzZ+UP2EeT/wrdFJki2ssbDQ2qk3jHFzRQGsTXwIjatcjb/N9eVj7XEnHTMZMVV2n86grT/eB1sQdoYubCoHA5OTgrP5TMHW2ik7jytWxri9xBZ0OIe0H+KmZJ/8ABY61xux417VRQPf/AElKNf8AhIixW4/THcFCOCVA2IVh8p6WwDTa0Cs8fMYJGZ277ncA/wDUup/95Y4x6Qd8pI7fDNEsEEAoqFIn4hVjNxqkPP8ACGcIqhyU0uJPbNmZkED5W7KVbHpaF1ekM+p1n/5HNqgkri8iI0j/ALrrZRZ9IO6yQKvDFWgk69lUoWB8QmxBi9GfjQ3YZVN3apXiNbaA1tGzHH3CTpAdui+W5P8A3bKh0/7yx9njfgJ4AuVFcwSf46RR/wDDNitxKlds8IZoKkfAU+oi2Qf6WaDXGDh04Qn+0BQf8dG+n55LHGOJWAqgAIxVQCdB2iij/wAQFiCsp3bPHqhmlnG7D6JxgTuLYNwPwsks4ywe+qGcV3QszsK5v/4rHWr4uiogsXvROf5KhB+42IJozs4JcJ7dwUZG2otgTvbG1trEtuJUOqVA27CSBt8rIuB2KgWlcbxOtsIn87dZfD5W3AJ20s11GxQcTbevW3URrbeXpZJWXBAO41trKI1sIQR/W2o1328bLVTa0LkjlbI6i3RTbcGLRRRouI8LbCSNNyLdAco87YBzsk60ExuOW1ugBGmlsANuh1skwN1wUz5m2ZU9Ld76RbNedklsuABO9uQgTO9hIjS2tzZrJ1yUAjTnYJdM2r2kjXwsYgRobaMTaJATglAIpkI2AFtPU6HRC0CPHWxkjwtnKw8o2RLlJwuxgKzBsAjnFjDbSUaAaWHO1syjff8AG0crW7J85O6I1F3sPvJWpsFSdASNrCGnT2ZbIBB0jlYwBOgGk20oa6WiGAG6mHl26Rqa4KOjUv1ZhCAskmBGtiNLg+56C83Lzortp2ah0krWhsBSid5Is5YHObai0DE3opiRyTbxuqlvOgeu+tpkP076FNutuJCkqSRqCDuLQ7W+jNgYVxfpKWoaZJJ7BNQvswT0EyI5a6WnIwExAsGUCdQLJ7Mzct0mOyG9lG/DHg/hbhoHBcN3lC1jJ2jjilqCJkgEnQE6mNzvaQ9hFhCgDby2topgaHSzMjyCyd7y/dcGYtuARoLbgbC2xIG+tiAIaLUWhdb/AJXDHlt94NjPu+VizJy1rySPaAUPL+pNjOvOyGyc7rUSN9bZOu9tnTW2jyjz2s6ZYZ5W142yYNsnraTQouWaEWIXskqpFrTuhQcHuM2PconnYB9Hasrb2zpKfiLEy3CgTlIQRhUFPMSLckachNuKRZXSNFUzlAPmLdkEmAJ8rROyfmjlb2rmE6j1YgPtZktEicqtwdxtPUWiDGtNVVl206KW8lO0SaxmrdbZ7M+uBvcBS1ACeomTAkWlW7EOVtFUi8EAQA81TTIQobFRjVW2mw8YksDGLLLVU4cpzNkq0EyI+W/ytyNfhTquodODYGw213utCmmbG0sOvNRVju6E4iv25PW7iZ9RKFO1hL6G3oSUltpboClIaJCtG5Ku6CUpJUWTxFevNi6K697n4YYPRW3fSuKo6x55Lwpg02rs1hosCYCQQiYmJjUhzXzidiVUtwUK7zq0lSnmqNSUttLMqV2jqiEJ10iSrX2d7IqaavfadqcSvFKCoBNFQjMFZoHZqKhLkwRIyCCZGmYCdgVPw+NOcz7eVgkKt2bIwWCqSjj5d7aEt1fFHE5fSAHSze7oQVjfLodJ21Olss8qnhE05UurquO2D6J5S1FymRR3clLCp1QAgZAEnSE93TTS2W5rg4d/D/8Ayj3l6/z1T1eYUHFlTQKQqdQZ1t2w0AlJicumgH0bHq5tindAfdS0snKAo5ZPQdTbmmCFpBDyPNKwR7vr426dsheLhaXDDTqtttZRGUq015TznlH6+NhkII1CZI5kdBtHy+tRUNACEKhQgDKI5bfG3SWURKCBz5QAOXyn32fMeafRdsOrZhCgNwe7Ikz4an63scp34y50E6bxr9b2KJCSQG1BR035ba8uVttZoUEkpn2goROvl9fG0NSkldqqMyTOUaAySN+XusOHErSQVKGUCTvr9ac7IyXVg5pAB1E6jx2saS64mJJMwZj5f152ikErar7ySDpmnUx+e/L+uABI7yyRzER5WJofaJhJ19oGdrGEOEApHI+IPy91olSFkKNUnTNrI393lbsJGbvEnXQE9frnbhKkFWRSpJOutuoA2KSN9Oenj5fKzXS3XTcEhKVHTl9fX3WUmhrMnXcT9fQslpISuQIOhk9LKbIUUAAEc42/rYL3IzAhQOoAkb7gR02+vkISmII02idd+mxtwAAkETqRoPv+uttaD2UgEgASJ90WDdFXahGgIAAiR5W4zDUZRJB0Enl9fO2ylSBBOp11525WmJjKQkHTcz+VkHJrBYDmJAOg5qGk+62JSQhSAkE+HMG2kJJASVQDvpvbsZMw0zTuBaWZINCxKlRpA7pIO/1+ttgaKATMQBB05/X9bbjLJUdY0OWOmnytzMgkCN/hZsxSsF1KJzK5aGfmR9b25I/mA1122t0QZAlRJ130H1FtEmAZJBMz9ctrK6VliU6aiYMEA8o/rbYBUNADpzHL6i29IgQDy8vr7ra0UkhEQI25fX10srpZQszCQQZ06WxWaRm1zawZtmWMpKtQZJtikiTCfa05QLK6kGrFFM9IOk24KzBPdGmwkC3ZUCsAkQBsNR/T63sErVYTOm6tBr9aWldRLViElR0SASdQTbshMQSdNBPPS2IkLBBG0CTrbQSQMo8xH3zZXUcq6BSqZTuAIOh62wqCSTuROv3fn8bB5SATmk767eHjbM0kHLJ3GtlcqWVCBYJg6lPe+fP6/TEqnULO0z9fW9uQRIO8aax9dfoW5zEJ0ImAd+XSzJWCFzARlnXTnv8AU22EoUkhQBM7lPI2CzJTosHUaEk7aT9edsUojed40/P6/N7lItBQg7JKcyEAEHSBHXWxpm87wYEsXjUsj/1by0/cd7EEHXukzNu4TlKTpG3P6/WziRzdio8Np3CV28WYnp1DscS3sgdPX3o/8VjzPETHNPOTFd4nnCnQsj/imzbGpzATvtvr/S21AJGUp8ztaYqpW7OPqoOponaloTxZ4t8QWpBxGpR6Lp2DH/J+NjzXGnHjUZq6je1g9pSJE/8ADHjZhBZ1GoI3gW1miSspPU+E2IK+pGzz6oZoqc7sHopKa4642RAcobncJ5erOA//AITwPKxxnj7fyT9vcd3L/wAinEfeTaKUqjcmddYH1426zqQNTpzmJ/rYgxOrbs8qJw6mPwKYUekC+NHsKtkTEpriP/dmx6n4+3WrR/DtUjWPs30r+8C0I51d2ZHIjeT0torOsK0iTp93xtMYxVj4vwoeyqV3wqfWuOuFF6O3bezXU9m2Y/57HmeM+B3P7yorGZ/npyf/AAk2rulzU6AjlpGvS3ZdSpBHOSJ6+FiDHKpu9j9lE4NTHqrJM8WsAPaC/cp2IVTPDX/hix5riDgl7RGJaIHfvrKP/EBar3aHMCCDOxjf7rCIdIGYaggxztMdoKgbtCH7Cg5OKtM3jDCTh7mKbqM7f643+djbN83NUa097Ubv+R9B/G1UA8UjMeQ38Ldl4HURAnf8eXWxG9opObB6qBwFnJ5VtkKbWJQtKh/hM23lIExapCH0NglshMExACfr9bGmr7vOmEMXpVtRqMj6xHwNijtGOcf5UDgJ5P8AwrXR0FtAR0tWCmxtihkkJxHeXd1M1Kz+NlFniRjJtP2eIakEie+Qr/xA2mO0UR3YUM4FKNnBWOgzbIJ1mbV+p+LGNkGF34XP8zDX35bHW+MuLmwM7lG4T/M0PjpGlpjH6Y7gqJwaoGxCnQjS2iNBFoXa424hTo7Q3ev/AHFj7lfU2OtccK0D7e42FRuUOFI+c2K3HKR3NDOE1I5KWcvUW0RPMzytGLPG5lyO0w/5xUz/ANFjbXGi5lf3t0VYMx3FJV98WkMXoz8f7oZwyqb8KkAjTS2iJMmzHTxiworRxmvRPVpJ/wCqxpHFXBbioNe63/nYV+E2MMQpXbPCh3OoZuwp2Eac4to/UWbieI2C3YKL7QOWrLgj/lsOjHWEHTKcQ0Y6Z15fvi0hVwHZ49U3d5Ru0+iWSP6W0Rynaya3i3C1QYZxHdiyeSaxske6bG27yu5/RivpXOmV5J+42IJY3bEKBjeNwhjAi2trYClWqSFeRm2bH7jYgIPNRsRuizgy1rawfbSUx8TYyI5WKVi+zXTrzR9oEz9/3WNTNnGqYmyyR10trz52w2yZ52mBZQJutTrNs2E2yTrblci0gNVG9ltREwNR5W4JGwBtsmRpztogDcx52Khk3RCmkds0f4HVD3HX8bCmBztwRkrXdNFpSofdYRPZpUVrbKpjYkbG1eZz2x5ohmPS9lMAX1QlGspfE7KkHlvaMsV1uH8U1T9Ca9qoDCRT113heVaVAyc5BCikiIiARvIOkjBlpb6qjOpDqiCDnJJ126QLQnxi9G+7uJd5pxPhrFVRcN9pUFmraSpbShBSMpQpJQqYJjNMDQb283xTtTWUVSGVlOWMO/O/Qgj8rQZC0tzMNykl+gum7gKO7aJuhokLKGmGx3GUcthsAdTHxtEF+4ixtdV5uX41w0U9eDLgoEqTe4qqdpAJU6rsktoUnlmc7ydU94JylU23fw2xPgO54vjEVbjV5tta0gppqNU5MoRncKirvEwYHtTrlSmyFT3PfF9X+HXcMLud2jQ2F0nZ0ricsAlTjqUaoEQMqk90pISCqLZtT2tbi8nAo3ZWMF9Tv9lYipQwZ37lV3q8OYK9ae9evhwVPaK7bMaGc896YqDznnbLT07gThE46td5YdxG7VqUS+5+5anvuT3joobmeQ8hbLcycQk+f8q3wWdFBH7SN91PCzC6W82ZV+hRCZB0pXQNj1Vt42p9wmerH3qY+sPFJeQrL2qj/tIMz59LW3/aUOBOA8I00qld7Oq0J2SwP/i+dqqcHWypqgfcKSXX946rBgdfZ1529c7INDsNjzdT+6r9o3EVr7dB+ynBg1aSnsqmoSsHQJdIkTqLGU1F7wkKvOrmeTy4A6an8bD0rUkhPeBkgAagdT01H1yUUspCMypkDOQdQI5z9bW7gwxn4QuWbM/kUVRfGJGtr8rjJghSyoRr8o+8WHRiLFaQQL3dzDTvoSdtuVh/VW0d0IG+mms+X199hAwEk6SToBMAkeJ+tB7hmlhPwD0UxUzN2cVprFWKVGDXIWoEe20mT8o5WNN40xQ0hUrpVmQYLZ0PlPXz+63Ip4ICkknXUq1P0frnbXqiFRAjkIAMD+htHuFM7dgS79UDZxRxviJf7aIXS0SiNBAUP+rpOsdLG2eJt7ApR+6WN5hLhEn4GyP6ohw5VpB2khMdN45W36kjQGNDsdvl7vjZjhVI7dgUhiVSNnJyI4p1MlK7jBlO4fiOp1TYdPFunQftLkqJTz7RJgz7vocrNY0pSk6k6+Ex9bTYF6hbAkDluFbRH6WF7Eo3fCpDF6ofElq8/SMwlctYilvS5b6+0QHQppppxIExBlYM/XPR30PGLC9VStP+r3kgOoCk52U7ETOivK1UOLCRT4hpwkZZpUlJA55lDnqOlpXw/SB26KLMASqmbJnSZSOlq3/T1G9x0PqrIxyqY0bKZE8W8JjQOVgJM606uv4mLGWeKODXJi8XUkclUzgPxy+fw6Wif92JUAlCCDMzGp167WEF2AiASdYT3Znf9LRPZijOxPqnHaGp5gKXEcQ8HPwEX8wgnUhSSnUyOYsYRjfCjxTkxFQ6TMvJAHxOp3+jaGTdiUHIlESNp5bW4/dzcglIAUIOn42Cey0HJ5RG9o5ubQpyZxJh9xOZu+6JUDRQqEH7vxsbReF3KJyV7BJI2dST4c/D5WgIXQlZClogA8+Rg87JOIbjbN2jMhKe8IkjTQ76eVgP7LMtdsn4VhnaJ5OrPyrNJcbcTAWlQB3GutukwSkAwU6mOnl7rUTNzOoZxC0y842S52iileUwUDXQyPMW4oziK5GbnqF4ivZoq7R10Jr3QJUM2ozbDkDtFqR7OO5P/CsjHm82flXw7oKVE5Y5D7/G2ZSVnQCNByg/DwtQ+6cRcSW27xWrHGJEKfp/WWki9nzkBUqAJXA0jbw6WNU2OeLlHhWorE8Rb+VVracfC3atThQYVCQFzoJGnnaH/Tkx2cFMY9FzaVeiDIVl0gbGdPxtrJJGbXzHyNvPZv0kuN93qpEDiHUu/ZCUuUtOrOqSCTLcjXoeVnfeXpLcX2K1qhuzEKFpQ60h5bt3sqzZtx3UDl+O9gu7P1IOhCMMcp7agq7ZICYUox8PKLZlG6c0zp+Fqtscf+KTKJeXdLmm7lEtPjqErHSyW16VnFf94rof7PYaezVCkolh9spaBMrVDx5+7Q2G/AatvT1RI8bpT1Vt/AaxAIJIPX8rBN5ZnUk6zHW1c6v0mcXtU3af2XutYQMy4fcTmEbc4nTXW3OHfSxvW9aJVVWcPGachzIUN3sVcgebQ8R7rQOC1bfh/KmMXpHa5vwrH5SU7wIg68uluAFE66j2o52gCg9Lahr7zcov9H9elticzzde2sSOQlKZ+Njd1elphCurXrvqMI4hZcpwVKUkMLQqI2PaAn4WGcLqx8Cn7SpfnU7KkQE6dJGptoyQAJTsY5D32iVr0mMCLJbduq/mjMwaVpQ+Tp2/Gwa/Sp4QMPFirvC82SlWUlV2uKAOuncmfjaD6CpZuwqYrqZ2zwpdMqJSCRzkdYtpS1yZ0E6D7/rxtFdB6T/BO8nwyxix4Oq1SF3XVjlrqW4sof8AlBcGkDI9xCu5pQMw8lxv3d5I6iwzSVAFyw+iIKqA/GPVSHDgUQkanSPrfzt0ToCUkwdI62ZtLxk4UVlMKtniJh4spJClqr0JSI3kqgfXusoUfETh9XJD1Fjq4Hkq0Cm7yZXPl3rDMMo3afREEsR2cE4QtciVAd4zrEfWmvhbtBgyrQRvGvw87JTeJMOVaiilv67XiDJSirbUR8D9a2PNVDDolp9Czr7KgZ1/pYZjf0Uw5pO6MIUJjkByj5/E/O2AkaDSJ5zHOwYWFHMoqgHkeWn18LbzqSlRHvnX3wdrRylEuF2VBSgRruRH10toqC+6IgxB2A/KwfaFQEjz0PhOn1vbAtJMKEx3QFcvD5xZWUbC6FCiU59QowRPKLdQCSEg/pv9edggrMnvE67SLaUsAZSozEAcjvaJKmuwYICY1OoHwttJEzqIgSRM2DU5IOswdTER7vM2wuc8uhEifrxs6VkMleVZIUQRsR99ulOKyxlkRAnXnzjyFixWCISFaxuRB6W0FAR3Vd4bbkT9CyToyFKkwYMx1k6c+dug6QmZGus7/W1ipdSUwhwCOXIfUW2pxRJJ0O4gaxy++zWCSNKcIUcsmddtBytovpKjBjoQTOuv4WJl5RkgzsRJ6iwangYO2mYg/n7trSDVEmyP9uToTqfZ12130sEqpHsyRrIJHlNiaqlJ0JBCvjFgXXEpEkiZJiY08PfZw3VRLilNqpBVMgbETNjKKrUCeUab9bICKgSJJ3MTsff8LG0VEphIJJ1B8Pj7rMWJw5KwqjKjm55o5/hbaakwVZ9dBqoiPGyUupKUyVagxG2vT8LcqqFgSBqNJmPo/XiW4afNZLPrJjRZnWTGtsTVd3KIEDUn+vnZFFWUJ3X+m5+vDlbBWLygKJEHy1suGlnS6mqBmDHgJ+H11sIKyEgKOaTAmZ/TnZvpvA6lWyTuDpYRNcEggEkHQTp9fpaPCTh10uOVatSTqd4Jg2Lmr725lMbaT9RZLXWAolZ1Gm02LqrkhRVMfAg6aizhlki5ORutGxWMp8dvz/pZTo6sFtcFJnfys0KatzDTNodROtlGgqVnOkq0KfcRaBbZLdLpqydDG068tpsEp5gnVttUn+UHz+6yaag6EqV4gjQ2CXXJQM4OaRKoO4+hZ81gllBSsmoabOdpCUET7Jgz7udtP33VtKDSbyvELUe41TVTqVK2/lULI1JU1d6KPqaQloEzUESmP8P8x+X3Wcd03PTU+wzuq0W6rVa4Okn8LMJJHGzSmLY2i7gnDgdnEVbfTNfeV/3mplkLy0i61x1pRUIClhRMkSYjTztNaVZkg9QCLRth1k07YUBEQR10tIVAsqpW9ZKQU/AxPyt22FROjhAcbkrk8ReJJbhGDrbRVp525nTe2iQdd7a4aLLOJXRX15W0VTrtbmfG2ieVnAsok3XW1smIOmh2tzOsb+VsJ5mzlocLFMmRV3RjK5L2L1zXh69d7qlKbpnSntGZM5Ez7SZmNRAPQAErXY9vy43xTX1hvZIKnUdqgBJMaDIoEyU/xRvrMAviukobcSfYWknlodDbeYwRy3g7W4ebsxiFJK6TC6xzAfhd4mj6X2V1tVG4ASsv57KOF8fMI0rymaihqUAJIU4mHFZp0AToIy7nltrvZZf4/wDCu80UbbN8qYeeb7qC0SJy5ssolJVoRCSTNnU7hnDl407tRfVyUFQh320rpkFThG0kjxPxNilbwywFW3EXWMMUzKHKz1tbbRyDtu+kKJTEwHFGOsdLYsrMcjjk45ZIG3Bdb8bIzXQZ25QQmO5xF4fioXSoxdS17j7SnBSsKCh2YgKK1L7oyg7CI3M72MUmELqxAh69KW6WmrwdUGVoDhabyIMJBMFKgmVQcp1UYiZshYj4HcN27s7Wsqa+7RSNoWqqbqsqWw3uvvEhOkydI5QNLMa+cLYyuXDVy1XC2/7/AE0ja6pp1uoS409eXbrCWsichyjKlULzIIkuSmc1vKMQpe8zNe1uRx0NtB+FsZ3NYU4rxuTCtDeFVRHihTUBp3ltGkRWUiE0+VRHZhJZJSExEE6RbLEWeFHEBTKFJuTBNOCkEMvUNM843p7KnOz76hsVczrbLZ/cJvmP4S4jlVP9pNUpbuHA7BOUqrq1QBjk0z+dq28I6eWrt7MSDVg6bATBB9yp8vdawH7S99xDGAWCrRTt5LO2kJpgNDrz5HwtBPB1A7G68xzpU6lUE+z3x9Tb6R7JtLcNi+/7rP7Q61sn0H7KeKenSIUsSSo6DkdOtjrbYJC9DmICoET0tlM2pwDMgEkHNInT85mxxLYKc5B08OfXT+nhbuFygRfIrMogaROkR9fl8BEtNgxlJA32P1z+Fhy1kMqGmggE727CTIzJlKo5x0G5sk65CJMGTCgY3M68vra3RbkBAUZIzHymJ+vnbaUFaQsDQp1HP3WFSMqJ3BkwTt5zZDRIhFyjvQgRBInqT08bYWwO7Mge4T+n52M5MyjOm5nr9aWEbQQsyCDMGRyHM2kHFRyhFcpSc0kkecxufrzsAtrKMxIRzKuWn32PKbCiEqSCmZ328PDnYFxAyBKhA9k676nw8voaECE4KC+MrahiOiISdaQZknYnOrY7c/6c5cw22n9y0BKSqaVklQH+FM/1tFfGhJRiChJghVJ/ESNlqJJjfl8JtLeFU57luxYUrL6ozrm1IyD42gz3ipP9wJZbaISUlsTvH3/WlhlNKIyJza/ETH142xtGcAHQ/wAR2n4+7ysaaQYOg8OY+trFQwEWLQ7xJMnWIGn4HT77dFgwEwkbR4fp+djfZj2tI156T93LrztvsUrHsEDw+ulo3U7IoWUpJ7gzAyZEST9fdZMxAwTdxICNFpmY67n4ffZdDZXCkchtoeX52T8QJX+7nNUg5kED3jW0XHRTbuoZviA/eFIyAFVLzWYdEpRJPjsB7+Vhb2ok1d03YyhP2joQhJUdgpOp+HzNlKqpEO3teb24ap8gB5lSdfkPn4WTW+1rKq7lQezplMsmP5imTGmp0A+NqGbkrFlzeLTF33g72iE5fUuzSOW8DTz/AB5WLUNMheGkpyaBtQIPvH5Gyte9I1WX4xTqjKhBcWAYzZRIHjqQSPKyMmqcpbhbpaZMvvFSUgEEAZjJ6xFptfYprKBb3pHmK5Tysqe3ddcKdw0M+VKd5OieelpRormYo7rulpuCpTiHXFlMFaiJ931Fo6xE4WqhtDolP2kAkDXOSmeca9eZ87Sqwkqu26Vbn7E9f4Z002P4fCBsnA0TuqA1R0q6l0oyNpiTzPT5WRsN3emorK2ucT31wkEkbSZge762so1ua97yRdjTn2KCHH1Ae0BGnXpbq4WQ1X3gltEDtIUEnQROnhaRdchK1gha272QytISkqAMHmRGv3fGzWomE09xrZbQO1edUG4TrsNdeY39/us/H2dCHEgI1EnrtZGue7kVdT61ADFNDbKDBM/zH6+60HHVSaNLopdmHG7vo0oDaO1UQtxQT/F+m30DZKu66kJxLVKDY7wV3ADA202s+XWgDnI9rXaIjn91kFhnJiCoU6YzJUdTpGUbfP6FmcQLJgCUUvWjFJTKAZCVnRIVy8fAAeFmfUXE072knvkZySYzHX8/nZ53i4qodW842CkHKmfd18vqLFmbue9Vqatxak5Gzl19o/kPxsF7s5RWtyjVJFwYWpmKUVicgdfkkjUAdBOu/WzB4k0IbecEGcqenvEDXl89xaZLsSk3WxBJ0UN4jvfrtaMOKzYS444dEhgEmRroQJ58jItZAGSyCb3RTBN3ovLBbqlnMkLdSYSCSNT+nu52cd03MmiuVplLIzrnKnTNr0/LnvYjwkdQrDdWHmwsIqlSkEHTKkhPnr8xZ+XFTprX1VruVQbgITyST7unjy8bDIBUwSBdIl34KapmnHXG0l92C4qNhyA+P1AsYOHuzWClJSdwdR3hOvh8vyea2kkynNlnQxEdD+FuCyAdUcpiDKfeLTDG9FDO7qo9xFdl4Ju9Cma+rbhWYFD6kyNddCAN7GrmcxUzd1N2OJb3QpDeXMK51K9CRMhXn9buW/KUOXevKjREKk/l749/nYG6adJu5KSTCSrKOe8zPx+tLC4UZdqFMSvtoSiTN/cQWyezxtf4O4AvB7QjpKo6fKxhOPeKlMO7ja9lqTIBcqCsnror+ulj6aNKBJQoZRzgz0H3WTbwCQU09OkqcWSICeXTyGnwtF9PCfhHopiomGzj6oGt40ca2F+rXdjmpLuaO/TML0I3lTZ0++OQsoYp4u+krg/hg/xFXfrT1C1WNUYfqbqZCHXFSSlMIGgA1JO+mpmF7h1g7C1Vei3MXX01Q0zKO1VmHffXE5EkAgDqTy+QvpT4oocTcNqy4KKrpX6dllK2GaRRDFOlsQlOXROaNNtLUZKWHihjYx56K0yqmDC4vPqouub00+Nj1DSVFQbgdLxhal3eofxRPdcH3cjpaa7m4/cQ6ulpqiqoLmUt5tLpCWXEjUeKzHS1G7lX2V00iyUhbLyiNJKe94ef3WtzhRpp65aFUJI7IAGOmnP6/EU9BT5tGBWoK6e2ripCTx3xZlh24LscgScq3ESOe5Nkq/PSkvy4KhltfD9ipQ6gqzpvFTeUjSILZ/PXaxJujbKMyZlQjMNT8fePl7mvjS52nfVnSgfxJChGpOU/He1Y4bTk+6rPf57XDk6GPTGXH+s8M6hJ3IbvZKtCd9WR9T0saR6ZGHf/AJ7gS/mlRPdeZWNh1I18LRA7c7JQSGQMoBjw2Aj3jpvYBVxMKSW0tp7o1JOoPL36j4D3L2VTnl+VH2nUDmpuR6YeAlJHbYaxM1lmfsGD16PWNtel5woWtCapF9UxIMldATH/AAqMjpavb2HaYBLgZAVmgDLAEdAOcWKvYcplgpI12MEg+f3mzeyYPNP7UnHRWcR6VXBgpleJ6tnQBIcuupHjuEH4Hpbs+lBwOdjtOINKwSY+2pn2yNBvmQLVQqML07g6zCenWJJnx+jZo42wkGsNV1a0oyhtLglKYhKhMabxvEa++0m4LATuUxxidvIK9NL6QnBeoyqa4mXCkL0GasSjXc+15iy7S8YeGFWlKafiFhxcjSLzYk/80g/lbyjCQbt9sgtPkAQJOYfH+HytJdzYeYqqBiqzD7RlKylQkAESZ16CedmmwOJguHFPHjUh3aF6UMYzwvWpilxJdT0iU5K1pU9I71jib2onATT1jbo1EoWDO/IG3m8jB7Trax2bcKVuoazHL5WLVGEG0oIZSEagHQjWfDw/rztV9jt+b8I/tk/KvS1FQpau6k/zbG3K6gpCcxUDyMxH19b28z3LrvmnQUUF41dO4ASns6laSDsBIIj+lsums4puVCGbpxriRnZIKLzqEADzzac/CzDBbj3/AMKQxgbZPyvS4ViidVTqBBM/1sIKs5j3zoNzbznqsY8WsPIqUq4o4peebpi73r3dUEHKCAApWuubWOVvQv8AZ3XhT8WeCV4V3EBIvy97txA/Rqq6wZnuzNOw4hJOhgKWuPfajX0JoYeNe4urVPiLZn5Mtij5qEwUp0OsgdOmti5qiFHKdxsTv5bWss5wqwC6sFWHGgRp3XXBpHgqxKp4LcPHUn/sVxGsyiqdH/VbD76w7hXuICq/U1WlR9rSPM/W1lGmvBKVxmmQR3Ty52l1XAjA+cqbXejRIiE1UwPDMk2S724DXOKCoVdOIr4YqA2sslamVoC4MSOzBI20BGnjrYT62MorHC6jV683C6mnZK3HVaIQg5iT4R/TX4H6G4nqpSKm9FglGqaZJ+zH+b+Y+ER52Gum62LraCAkrd0Lji9VrPOSPHlZXZlXskkA9d7FAL9kYkN3RylaSAlISBAGg5eHh+ll27GSVhMjTWI2sl0jXf118J+tP0s4rsaOYCPr8OVtehpLuuVm1U9gnJdyQ20kZrPK63SqkE8o+YBsz6YykQIB3s5bidzNKTGwAHun8xbsKZuUWXLzuuSUqGB8banlOm9sVrvbU8totcVRbm2TG/W3JM79LamCRFkkuiRFtkwIm3OYDf3W0SOR2sRouoOKDqgVU6wD/DP425bV2gSQfaiwiwFAid50sVo1zTo1hSZBM7EGzSNuNEmmyUq+oKECjbVlSkArG0+FlC4FdvdlVTkAlBzDnuP0sgLJJJJ1O5ssYUdiqeZ3K2iY6kEfgbZbaFlPSmLfmfPqiGQl11H/ABGuqoqVJqQhD9Oy1nFKoEIUvcqXyUAIhJEAyTJCSmMLzvXE17KqLwF6OpZpHwXVKJyoQQo5UnRIVmLZyiSRrGk2mrHN2P19E7TuVK6OnAUlRa0dOh0kapBMajXe0d3Fd923vclbhp1TdGhNZTpT2RgqK1wkRtEiNAJ6zJt5RiBMMksnC8LXN3Gtr62C6mN+aJrk3W71xU42lxqVoUkKSrt3u8Dsf7s/fbLOV65eKSnVqoqinYpyollputShLaJ7qQk6pAECDtbLQ7/S/K3+fZQzu6/lUG/aVvKVeWBGEgZkNXkSOYzGmA+vC0QcIkZKe5pEAuiIGhAWB9eZtJH7Ri8k3jfOAXAsEO3bVVGUnRWdTUjwMDf87R/wUSU0tymA4hIlBI0iQdB7z7yT429I7Li2HQ38/wByqOPm9ZL/ADkp2aASNikpAO+qdY+jZQQlBg77FSYnTnYGmaTGpEATO342NMtpSoOJiRqe7rH1Nu0K5Zi67M5YkmAYB2nztgbymQM3Lp5/hYSFAaKEbQR7/fbbYUUTOidyTr4/cfrdlPZBoQgEbmTHiNNP6WES0kiQiSrQfXw/QW6y94AnbUbaH6Bt0EEJIBA0gac+k/W1m5p7LhSE54ClGFSRzjrtboJU4EqKozAiQNhpr8bCBoklKN9Y6+/4W6S0QrMCRGsxtvPO0m7qB8kGpvPBUIM6dB5xBPLf52LuAgArPdXvpPWx0tiCDlOgOmgGvv8ACxd9sEEpOkjnqfHxP1pYgKE7RQZxqQE3/QqOX/zQxzg51a+7r4Wl7CaALguskpBNIyCUjXRCefx+t4e45PKYxBQKUTrSGNZjvqncdPqdbTNg1Pa4ZupW00DJI13yDT7t9bQabuKd3uBLSQZKROWZAj4a/X42MBJI3WDI5azvrP1zsEhpSITm217p5yPzsZQjuyRmKRoDpE8vjH5WndIN6LBmTmkjWNfztsypMkH4cuk8v6+YFyiFKBSrcpgfKOv1pbZaK0yjRJEz9dPqLQzKeRcFtt2cqBr743EWTL/Qk3O4Z1BTBPPUH8N7KxCj9mEazESfLymCbJ+Ims91FRQUwtChBAHtb/daDz4bqQteyi90ttVV7uOQlKEImTzKNuvhYqpluguq7HFFKQt8PLOpjuk8t401t1eqKh6tr6OTlWQ+4QmYQhEhPxiw95ssv0V00IVlU/2YIgk5con4gx77Z4N1bLbJKo0OPXkqrqMx9aZcdyie6g+yPKBYvddChFG7eJbkraWhnT2UDUnwPP8ALmtXm63R3q4pxSAlqiCQBuCTAjnrG1ilKhCMLqTkDf2CiSeW/X6199pNOqidlX/GjRpn6YQFFRWVACSQSCPHcWk27Uo/svdlQ4AUtNNEZjrITt933+UZY9palm9Gal4q7J5aQwkz/dhIOY6fxEmz6w2upvDD7JGZNPQ07bZgxLmg5dPw8bO86aKLVI+H6FVPSl99RNRU/aKJBA6xB56j5W1c6CL5vBuSZykz0/rZQuxSnKKnehXebSYI8Jj77J5qEXbfNY45KR2IVrpJ7unhaDn2AUgCSu72ddqH27sphC1n7VSU6BM8/G3GG0LS1UtGe67IEb6Gxy46NSkLvCpb+2qSdDyG+lgsPqIfrG0ggBYJnzMHX6087QzG4KJawsj5aK+6ogAgaQTGo+o/OzdvbsKeqdQ2SpxyM5iMqQBp8RPw5WcNfVikpy4QS4rQCdvrrZqOrK3Vd2XHPdJ6WhLJyTxs5oJqkdrKhLSUwZIJCdBG+mll6spG6W7FtgiEtkeY+P177D3TdvqlPnWgdusBR5EeH3W6vQoNE8sEphByab9NPrn7nYLC5TO1KR7obP7oQlaZIKok+PT4j32jjiqwFtKKk589OSJ5mTy56H52kq6gP3bK1kgFYM777fO0XcWq0tVtBRM95VRmSokR3BqZ8ADr9G1kP8KCW3KT+EKXqluupW5Qy26HVkARqiABvqcvwFpQwgo9k/3CVSkGTBMg7cuQ26jbW0bcEXgp++KYO6lDK99J74jbTUm0l4UIVVVCTmAgEEkDYxH3fG0Bo4JyLhONLZCipSSeY1326/nbakhw6DWADzsOlGkAgAwIgQr3e/5W0sFMqCASZGu4672IXWQ7JLvBkqpnwRMpzCI0ixK4GvsXACkd8qM6GCBH3fA2W6hsKYWI1UCkTrB26a7WS6JtV2UxU6lQccOYJHtJ6DTn9eAA99ijMbosq3Q0kpaBUpZhISZPhGus6+VuWKBDRDrywt5emugG8gfP42NUtEsE1dSk9sU6g6ZQY+vo2HWhKjmKQogzod9R+vnr75NN9SmcCNAiLzZ3TGkakAacvo2bmM6P1nDN604JhdK4BG3sGB152dikQ2nMZCdo0PXp52SL3Z9YoqhkNkpW2pJ25jUx7+tjB10NVRuBIXcyykat1RV7MwCgfltztbjhs8iqwrdz8JKS0CAozlJE/cd9/K1RbkzBm8KZ0GW3EmOcDMAZPP3aT7rWo4K1SarBVGAUnswEiCemgM9NeVqE+hV+nN1IiEgJAUAQQSTOpEneOW39bI2LqcGibWmCpL0qy7TB2+htZaSTpBMTATMc+envslYoYcXdayNezWlUGZJ1n8dPO1cHW6snayZi2gV51AFIgrIIPL5crBqaSVShISrLMTPTY9NbGiVQViNeih8Pr8bAkKIEqBUvUEyDp4jz+fvtO6FlKKvNlXdciBAJ2B3nfwn6mxdynASI72ZRMxzjWden3WOrHIQpKRIMz8jz0+psC4DEEAzqSZUfv6zr158rSDkxBCSnWoKgITExlEyfDf7vush4hoVVWH70pkmC9RvBIAgHuqgT5x9HV0Og6JKSJEGBoT4AddLEFNNrXkUkZXDliNwRB/H84sVj0J4uquIAVd9W2D7K23B0AkiZ/wB4b/jaXsBn1jDdA8pBSpLYbHkFZZHwn32iembLSLwpCAn7I7nmlQPTwNpO4XuqXcLbaiAWX3G8pXBjQxHLVXzsSY+FDjGqejdMFNwptMxBAJkDbltPv0FsqaVJHdgGQoZun5ef42PUTC6hQQhuSNxMjnpZy0FwBoBdWiVghRBGsxvp5WomSytNjzJrXfhior3gqpQUN6QojU6A6/1jfay8i6mKJnsmGkJ3IjnJ308ffvZwBlKUZUaRHs7j6j3WK1DBSCoAFMSVAamPMeXwtHPcooZl2UW44owmrJKSQ/TwT0MFPlOgtbv9ktfpcuLiJhtxzv09Vdt4oSVa99DzazHL+7b+ItVbHNPlbpnUp0Wl1AMTB7p085+tbTV+yuvlV38cMV4acXlbvO4XyhEmFLYqWiOgJyrXrvali8fFw+QfdEpnZKhq9SlKzBKxztysbmNvlblBOTIRJSbdnYE9Lec5rhdBayLySY8LcOd5JB59bYruqPibcrJ6687VXuRwogxPdqqS+ahoAgFXaJI5pVr8NxYmw0SrmCBtFnrji7ypdNXBP8zSvvT+NmuhgJUCD7RCtetulwz9WJpSnfZHKNGylAizgu1CUictkiiaBIlMjTUdOdl6jQEpA20Bi3WUkVlg1Ut0rMCNJ8tbLlyOFLhRAgg/E6/gbIbO4kTtZTu1woqkkiNR8zH3G2ywWWU83Fk4s3L5xbQOk7W523tkzyG/S1iyAuzANtE6c7clXKPfbWbSQR02tMC6i46LeYz0i250gA624J10+RtudvqLEAtshreaNedijJyrebVoQskacjrY2ZA+VipBTVL6KQFaWdzdFEHVbVI11ix+4ngzezCp9olHxEWIK2gHytth0svNu6koWFAdYM2DK27SpDeyUsWs9omqYBGYgKA8x+dq135dj1ViE063VMtzmcWnVSUDfKB7SuSRzMWtDilphoLvR1xLbXq5zqJgBI1k/G1c78pqyrvh6vraL1elzzTU7iJW8mBC3kqEATJDfP8AiiChXA4g8xVRZzcNP2XTULg+nt0QPaYjGlFganXT/wCxUuhU6pSP4SVz3zEd7nvbLAOofcdW4uuqFKUokkrJJNstR9nu6D1KNlcqD/tFgaTGmEaBpay0xdT/AHQnUA1CgnUchlHysyuF1+MXTd1yPLoHnQzTNlWWO8ZPMxvHys6v2j7yFcVsOsqmUXGDIMb1Tw3jwP1oWRgVtCLgu7MpK1+rwVAbwpQ0A8vD3W3sHlMGFwuG9v7oGKxiXEJQVNdPxNuwIzLuq8UR3vYQfOO9ZQb4lXCtKZo7wEawGQdd/wCbxtGjJCtU6a5QdPrp8LH29dycpSFEnXz92/uNtH2vUeSoDDYTsFJKeIuG1EZk1if4ZNOdtddPrawox7hnNnFY+haeZZXEzt4/raNVEpkZuUkbe4/XW3TTn8JXqTMRI/TSyGMT9AkcNiHVSYMcYXWkAXmQo7Zm3E6c+Xn1sO3jDDIhRvZCTMGUKTp7x5fnaMkEBGZAhQ21Ik/ntpbBlKjISSNdYgjp7xtaYxiboE3s2MDcqV04pw2vX990yIOmZUfH65zYdWJMPLISL8oDP8PbDX3ToNrRGMqRGSQARp05aDe3C0oKspQEzoNDprO48/lYgxqQbtCGcNZyKmBV93MQUi9qEmJCvWExoeUmwLl6XcpJQmvpVyP++STHM/faHVBKgVkp03ISPGTp4jrZOqG0IGbKmBpqkSBsPf8ApY7cZdzahPwxp+JEvSEvClZxJdrjdQFAUiwSjvJJznQxv+k8rTrgOoZVhO53UPNyqgp9AQdChO9qecTKhulvWlhOUlgmAIJ1M/Ln/W0o4JQ1+4LtUlrL/qrZPjmSNz1384sV+KGJoky7oLaASeAHZWYbamCkkjeM31P62MpaWRGYmIJ3133tBbSyJKHFpIJEhagN9R9a2OsuViFQK6rBA0yvLEbfgRYftxp3Z+UZuFH5lNqUrACgjUGSSd9bcKQFhKUCCVd46yPH8LQ81WXoFAi96+I0ioWIO/M+djLd832lRIviviT7T50MzpPOyGNM5tKRwx4+JS6G3CAFEnmfHY2IYhZIu1yRGVSDB0nW0dtYhv8A3F9VszrLkgfEfXxt07ft9PMFt+931pJAhYSIM7xG82TsZiItlKQwuS97hE32A5UX5ULTK0NpQCBqAETHjr93xTqZtdVeFBXLJDbbiWGkjolBzGOWv11UC5VqU8fXCF1Byud0d6NNfjbhHrKAwhupBTTd5AKEwPqdz8LVe/tKOaN3VFr4o01l8upPeRTsdq4J10BgQOROvusRvBwUWEgy7MvU6kpOYaq1P6+6ymk1HbPVPaoLtQkocJQIUnb3crJN6Ui1sIYWpPZtJLaO54Dx6c7TbXNuhmjcBZQ7xUZ7J66lJJByiFCTpkG3U6c4G3lZ8YTpWlYNYDaEguskqjfNJnz208rR/wAYKqpbvOkaecZ7JDSShITBBGk76iJ+Nnnw9rKp7DVIhCmw0UrQAUmR3j+mnjaw+qaGhyC2nObKpDw04hV0UjhgwkT3toJ1+XzsDWpRVXy1Qg5QofaKJjNl5H652Bul6qoKRNIylooaGxmYOw315/EWEpaNVPVtVqVBS0KKzmJAVJ5fOwXVjDZGbSuGqcJROVKR3tk8tIgCOnvsjXQtDV716VqgDveIEz79Dv8AfY2bwqITmpW+8BJC4/CxBbalOVD7bZSuoA2cnTy8TFovrGHZJtM7mid61yqlztcwDY0SNNxGo+uXhYK7Uti8qclMyCSOR9qPmLDLux1xwLUgZZ7wB1jw6GxhxsCuZq2WMqGk5coAJPvJsFtQ1zrlT4JAsEuISMwgFI72ggz+ViN5qc9SqAncIUSE68j+thHL0bylAZeMa6QZ8NPrX4l3q5laFtFh0ZioSE/PQ9LWzUsI3QOA66Q2K4UlzmQEuFZSgH+Lx6xIsyMaXS4xd5r6pJdfcQ5EicgIGg+vus+W26cVIqKptamW0yygJkCOZ+uYs1uJV9XexdiVOdoCHMmrSiCMp0+AP67WnBOwmxKhJC7kEyOBr039eTedULpkkjlIWPd/EOfIWlXC5AveqYSpUAGQobd4T56RaFODla1R44qEuGErYdRojUnMlXlyO3M6bWmi5nWmL3cqDPZrCyFAEASZGmp5b+FiPka0i6i2IkJ6AKTA3MyUawR/S3OYgBOYka6kxA931HSwCK+kVC0uaSQRkIjwPxH0bcmto0JEOglIjQHX6+uVkZ29VEQO6IV5wNBRLkEnKJ1InfSyPdrvrN4rU+SVISSBG2oBP11saaNPVPqfqqlKUtGEJGhmN94PKxK7ghm+M7iClolSc61aGNdLAMoJRhGQFIGFsHVV/IXWVDTjVC2YK0IJ7VendSQfif0s9m8GUraRU1NMM5gtsKplAFPXYwPrlobuevw6zRU1QusoA22kIYYC2+8rbvQsGPPebNfGePGKtLt3XJUoGfR6oByk9W24J7um/uEWpcaSaTKDYKwI2sbcpv4weuamqF3fdTVO6AYdqENwkGdkaTp1+iyKtoOtKacSoiNiY57+X15KroQ6jKciSDoREDr9eNiL7SYIXCzqAAY+vO2vEWsFrrPeCTeyqOKcUWIL+oVpMocUlJCfZhyNdOh/K1h+AdSp3C6k9pmLTxgTMyVaef6+doHxcw1Q8R79SogBa3VeJkJVtvyPnAtL/o5VYcoK6mCtW3NRMwAE7aaaqI91gT66hWafQ2U5D2+6ZI7x05EffFid+JWq6X4Ue6kKBnQAEH362HaUEbKEAQOQI5gDblbi8kLVdz7eTMOzUkBPPQ6j77Vb9VbITEhQbUSCCkb69I5+/TpFtKme9OVEgac9dfo9LYMygM57vTbXrqZ5fK2nFQAoAZTIJjYzB9/62WYqGqDcSogbxJ0nlpt7jy8bF3CDIShSkxp4zr7/AOlhnDIClRMZRIJEDX387BrAGYrVPj7IJ16+X1ye9kxF0TdSCjNmMk/xaxA/IfebE1kpJVm0kyEmI8vl030sfWghBRlM6Du6H3aa+Fk5yCoqbJQREa6yNxy+Hl0ixGOUCxV3vOhFLi+9qKAn7eobTmJ5lUHbyP6WenBqgeq6SupUKUOzdSsmTpmTEf8AKLEca4ddYxtedcqEtpcbqEJIjtApCSr7zPzsucEHQxe16URUkKcaQ4AddEkif+bz2sSaUFhsoRM8YBUyXVd7NIj7BAznUqGm2u1lNORCIUQBvmgGfw6WLMBCTomQNBM+8DX6+VjTeZKgT7QEkAwAOURtr+NssvJWk0Zdl0AcoBSJBABA12/SxWraQpBJWJMhWpy7H8Z5/jY9nbyltXsjYlW+3vsA+vPLeaYEacvrTztNjkzhcKPcdtH1BLqMwyu97wKkq1G33WX/AEEL3GGvTAw4gryt3l6zRr5SHaRzLPh2iUgDqNhZPxe12l0PpTIIAUBmgykgTp4HkDZpcHr5GGPSM4dX2p1LSW7+uxTi9oR6ylKx55VQYsSZvFppGdQVXHhkafNe4TgCHjoIULdQcsg2yrEBKk/w6W0FZkZh1t5aTYkLpBq0FFnCM2k/G3Czp4m3bx72YHysC4tKUZlKypAkk8rVHHWysNSbf9Ga266hCRKmwHE+adfumzHbbBMQI3iNPdZ7XBii4cS1l4UNz1yav1HIHlISS33s2iVbK9kzEizYqqI0VY7Rx/dqKRP8o2+VuowAktLCg1ZLRYrVIyAEpywPvssUoAjY/hYlTtkiefjI8bH6cQZP6/X5W7inaucqDqj7UaAHaxymVkdBHIEj4WItgnU6x00mxxpQDiB0InS2gAVSKc8yJtgOu/jYGnWTTtkmTlAPiRp+Fu82k2sNF1XJXebbS3M+XwtqdJNskSdTFiAWUVu250gW0DztknXfSxGhQcV1uQZOti9QYfZPJWZJ+Gn42HzSIsWrzlbS5rCHEkkeOn42d2yiN12uY1ANuFE8rdr2iLBKj9LCIuFO9tU73w3WXPQ1C0BRQkakTBAyk/faNMQXWw9fKEO0rD6nVJQgPOFIBJ3ge1y+7ykW5HQ9cBaKSeycKYB6kH8bMfHtMUStsHTQkC3DdooA5wsbHa638IdmuwrhWL8IUyjTJuWgcS0cgWEQFAaT7FstH6KetCQBSlQjQyrX52y3M+xm/wC8fwr/AHZnX915kftFqlL3GW62CdG7gZASdv8AzioM/Dy3s1cFXrdVPcN3sKr6VCk0yM4U8kETrJE6fW+tnB+0Bq546tN51IyXDToVlOuVTjxI+B1tXJ1a26t4pUClCUyrUmAnkfceUdbdnhkAlw2Fu3hCq4i/JXSu81ZmnvK6VhJXelICdY9YTry/Hlzso095XcoS3XsKTtAcSY+B0P5m1VkrKSG8w6FJPOZPOBt91hA+4qftCkp2TG48+vLToLGND5quKu3JWtS6wCAFIlQiMwP9B+Ws2HadbKoBTpBOuny89otU3198O5mXVKyaAJJA322E2GF6VvtIraprN/K8oCYj4k7fpaPcD1TmrvrZWyQNsyMwG06/ERHW3RPfSSNCd55yBPv026/GqP7+vplGRF61k7oKX1yDrpM/P6I6MUX822rs74vAJjNHrKwJB6A9fx87IUThzS72DyVqM6T3lD3jofD32CXqBsVLmYHPQ7D3fpar6cZ4padStGILyTAlIFWoyfj5+8RGuopx7ixtMJxHeLgGhJfXI03k8pAP0ZXc39U3em9FZJxzQ6aCT3dJ2+WliNUoqbKW0knYJjbXl0HjaAP7fYxC0n+0NYRCRose6Ad9RHvNsc4i4vQslV/VCiNJUlJM890zGlptpXBRNQ0pV4tKP72ozMqDJ7qpIkL5+Pn+dpbwEQrDl2GFKmlakHQg5ROkfXlauV939XX4tp29Krt6hsZCopAAEzEAR46/dZy3XxKxPdFAxS0NSjsWWwltKmkqyoA6kbDTqeVizwOdG1oQ4pAxxKspTlJJBkEbkax+Gwjl7rH2YAKMvdOqRmmdPHztW9jjTjRvMf3hSLypEE0yRqRodNxzsKjjtjZmXlqoCEyO+wY8van4dbVO6SK02pYFZCAU9nlk6yYB2+/66WFUIJUmDodFJjLGsz00PxtXRHHvGTYSHmLqMjUqYWNOYHf1HUe7wsMj0hsWNjK7d90rIAEZXNRMT7fj9/jEe6SqXeWFWGQoIUkJSSTyKQJHTw+trDFwABMZgfCPfavzfpE4jcAcNz3Yon2gAsSf+Lp9dR//ACiL2ySrDtIoxMIeXrv4Gef0bD7tLfZOKlnVTwtaUAIKjvlgj6PU25LgUhJJVI7xM6fX6++DT6Rtdp2mGKdQkQU1SvgJR+PvsKn0iCSpbmGGxlISIqzqd49nw++bS7tKOSTp4zzU1F0qAIBUB3tOn46mxCqWSCEpmTEknnvvtytErfpF0+QH+y6wZ/8AS5IOw/gncx8bdOcfLpfht/D1WnOJBDiVQQR4Df3WkIJRyUOMzqkTjU+pm8KR5YzAtgmDrIKtvH56bWcnCt/PhOiJzjVYnb+NXnaM+ImN6PFq2amjo3EFpIQouK3Gs7acvv8AGy1gLiDd9yXQihrW3lLbUv2EAiM07kz03takY4wgc0Bj28UlTtROpQgtgTBmTv5aWPBQ9uD5QAOcjz8uvjaMabizhtJyPmqBO/2cx8DzIjrr5WUmOLeGFjv1bySZlKmTI09/3/pSLHdFazN6p/FacxCEKEkg+J6/L522H5Ep0iRI016WYSOLGEnUpzV7iQqRJaVpyMaaWEPFPCbkFN5Ebe02rpodvgfK0cjuibO3qnwp5OmZSSkaEE7W47RMEpIyzJiCT08LMxvifhRRE3qnfUqbWAPCfj99hDxFwrCc96sogTJKgTr8+vvssruie7eqd4cQSc2VI6zIUZ/rbha5OUyTsYO31+fLSzU/0hYVUlSf3ywMp1BBTMfXL9baXj/Cao/7bphEfxHmOfnHhys4DkszU4ahYKJkgREDfeeVmJxHR21wLhE5HUK05jXT4kedld3G+F1lShfVGTOxdA100jlpv/WGxjLEN01dz1TTF407q0EHKlxMjWNp87FizByE+xCjnho+lnHjICHAHUutAjYHISRt4WsPQuNyhJ7vOVbG1Z8D1JYx5QvO1CQ2XVQpaoE5CPhvHj77WJu+taUQA43ooQAsER4662tVRIcLdEGntlTnQ5pnWlWkEAnU6dfKdfEW6UmQc/dJiCInf+vysTafBR3VJWFbkqzGJ5df1sIV5+4UjMZISSZBOn5fCwMxR7BDZgDAkynT8o+vGwZCEg9xKsuswCNt/C3AczxIkAgAdNiR4crcrKoHdkxJEzI8Pz+hDOQU2UEoZSmZns0idJiBp9f1sCtDajqjUTpz32+o+duA8kyqQka6a7f0n4W5S4BJIEjqPqN/rS0muI1CcgELhxumBV9mdpVz005GxN9phMjMJkp7itD5/n42NuqXOdRmI7pJjQ/pPv52IVbpyaKHe89dNo+vdpYrZHIRYFAvE+mpqHGdcptpSHFobX3SRqpAGo+O362WOCdbUZLybYefaKVtkKbcKT3grQ/8E+4WKcYk9niFhax3XaZuROspWsae4ixDgpU9nfd5UqlFGdhCyADulUGf+I2vbw3VVotIp6o6+8sySL0q8hTmOZxXhr7/ABsfTeF4uoLZvN1SVe1KpnbSLItIoQCTBSnlMA7n4afDnNlFCiuCFpz5tTB+P3DyFqRerll03TpSezLiipJgz/KfuGse+3PqKW5Qe6YiZ6aDyibCtAwSQF5J1Pj1NuswBCUjPrzO+34j5WbiJZEVVQqcTCXzvr4Dw0+osEqiW0dHVBI1GbWPd462UFFQCAoBQG53I8dLcrAzEEwNpjNrHT42QkT5Aks0CiAAqFaScs/p1+hYSiuhIXKpUQSYB/A+Nji1JTKVAyQc2pAPv9/3W7pipOfKoZo06aTO3O0XSkJ2x3UUcWrvLF8B9KYL9CCCOUFSIHwHL9Gpwoq+yxmQCQmqYeSfEApVIkbaRp0jS0i8XGULN11KkZkkOskjYgKSf+oxaJ8CVDlNjG7H1riV5B3twoEdOZM8+vjY8RLoygvblkCsU25lgpUSAdkgwBrGsfUGx1oyBCDA2mfqbJTDxy6CEn4zHQDXysptuAqIWEjMZ0Op08/qLU72VyyHSsIkpEzqoETuOvvNizroCfZ2gGFcvKPD6i3bjyCsHRJ29qY/WxV5R7M5iND3jO/kQNOVlnsoloSHiFfb0NWyQe/TrMRJ2MfP66Qnfd4uUd5XPeVPUErpnCtKpGhBSUx4fmLTrXMocPYrEpV3DG4B029/1raAMUsqaoqdb+YervBKiZiYIJJnUfiogWuwOz+EqrM3Lqvfa6bzYvy4aG96ZQLV4UrVU2oHQpcQFA/A2Fp1ygojY2h3gLxRwvSei7w/xviu/qS7qJGHaKmefqHQAXWWw0tI5qVmbVoJOlo0xn6aGE6x9+7sK3i/RUmoNUunV2rgmJTyQN+qv8pt5lJSzOmc1jb2JC6OnAkaATb6qwuMcfXFhVBRUOmqrYJTSMEFw+KuSR4n3TaDca8TL0xJmarqkNUavYomSez8M5/2nvhO3d52hiv44YXqFKDV5KlxRUouBRUtU6lROpJgySZsg1fFi4qkdp+8RBIzTOg8J5/rYseGy3uWrZiNLAPeBKs56OmIlVeNrzutTuZNTd/bQf5m3EjT3LNpdxXR9neDdSgAJfRrA/iTpPwi1SvRp4i3NU8ZLku+nrkqcvFFRSxrJ+xUoD4pHytdHE1H6xdpdCZUwoLnwOhts4cw08wDgsnEXtkcS03TYp0QBEa766WNITA2JjeOtgGUmNSJiT52NIBMb6c58bdrBouXm1KGaPeGg+FjKCIjWbFQggwRIImw6IjqALXwqbjZOChcLjGm4Vrp11/GxgEga72IXWsFlSd/oj7osZqKlikZXUVTyGWkAqWtZypSOck6Wss91Vzuh5B1P3WyZMfGLErvvi6r0Cv3deDFTl3DTgJA8rCvV1JTKSioq2WlL9kLcCSfKbSBvqmNxojExtbYURubcTIzA6WxDrbkltSVwdcqpixAoG5QgIkfLzsBeCM1E9lE90ke7WwwMH8BbagFIKCB3tNbORfRMEXaUlxhC07KSDYNRBGwmwdAT6mlB1UglJ8wbdq02Ox1sMpwbpzYNd7RNZTKOikpUPmD+Fid90jK2jUVaQpFOMykKHtnkPrlNucIvdje4bUdHm1J167/AIWO3+3Vllxlqkbc1UVAu5Y6cukW5PtM0CAuIP2/4Wjh7iJALqO3cQ1/aryVlalOYwEvhKQJ5DLoPC2WFVQXlmMPJQJ9kPmE+A7tst5pxaXq78rq+DB0Xjt6d9QH/SBqEK2Zuuja1PIpKoj/AHrV+dRlrXg5kSZE6c51PUnfr5WnX04HQv0hL4gpJbo6FEEbf6sknbz++0Hpl111KgVZjJCANSCem34Tb1PCfDQQ/wDxCwcTN6yT6lY2tSVGRqARoRvyMcvHr87YoDdSlyRMcvH8fnYXskqUgnUgyNRrHIc+h8rbKBKUKRJUQcytCvkDa6SFVsuA6sGMqCcpMEkRJ+vEn3W5UVAlQIAXMwZBiD4jSwrjQQe1KgnNtJAn6gW6yntcx9pRIPlGs/D4RZgQmIRZK1JAJCgRJIUcsn3am2FAIhKVAHQEzJkcjtPOxwUxVBaSNJ0KYHXrrp9x6W0lghZMZyYEHkZ0jofxBsswTWRXMSU5Mqc2wI1PMj4R8Bbckp7NOYgA6H4R9c43saaZWoKzLKkKUE5MsyYO/Xx+hblDE5QpKpIAUCDKdNIB8h46WWYJWRRTnfUUglc666kzofv38ettFwk50JS2F65Rpm941A0ixkMFAyuwdu6pMwYgac4/XW3Cmm0ELDiRlB7u5MbR8+gtIOGyiWlFHu73ipPdUUyFeyRsB1Gm/vscbV9i2DrkTlPe08ZjfSefPxiwlz3BeeJ74u/D1wUIqryvCpTSUtOlQGd5ZASmSQBMxJIGnhaZ2vQz9IjKnPw+Da0gZpvOk1MAf97E/lYc9RDCAJHAfVEhgkk1YCVCZdKEw2DJE5ojXxP5xtYJeZa5LZMch8/Ly8bTw16FfpFPNKcVgVjKTAUL1o9CeUB3Tn9C2K9CL0j31AHAzKUmCVqvajEjkf7zpytXGIUvzj1R+51HyH0UFZ0hQWcqVJ001BPjHhpG3lrPJUOQzJUCU6mTrsfqNPC0/U/oH+ktUoUn+xdGlcSAb0pBqDpsvb8OlhFegj6S60JT/YmnGg0/elMZMc/tPDbUfOz+0aQacQeqXc6n5D6KAmaw0y1KU224E6QtJEgxt8v6WDcqFOKLuiQBO2kc48Plv1tOFV6FfpFUL6mXcEdotRnKiqaUEncR3o35gnWwafQ09It5SGG8BOdoo+0urZCBvBJKoH15WcVtKdQ8eqj3WcHVpUKh0lRCsoBGpSZ+Hl566+NtKqEg/wCGYWRroSOuh28vO01VXodekQy6pl7AL5KVAkoqmFpmBsQvpE6mxdfoh+kM0Y/0d1kbaPsxsdiF+PLws4q6b5x6pGnmHwlQ2y6ACjNJ1lXnMCOWg38B4WxFSkKSsoJCgQsq00k9fDTTx8LS4PRH9INtGdXDWrkzmBfYA9raAqwb/om8f0hShw3rRmyxDjPTX+PX+mtpiqpyffHqocCb5SogdcWlAEmJJgCMw6zy016fiPROuBpS0LJSdiDBkGYPjqLSo76JvpBPR2PDSvOg2daGwP8Ai+tfcNR+iP6RmY9vw0vGEnTK80ND1Gc9BaRq6fL749Uwpps3un0UXCoCRnCjEZVp0J0HhqNjH522mrWlsgFxOXuQFEx5dPaHnpaW/wDyTfSBUkp/0Y3jomP71oGQIjRX4fqGr0S+PpKh/o1vIlSZSSpoAe4K6Tt1sAVdOd3j1RO7zfKVE4qFElTqwCSMpKTsPCdLdIrFtNnvEKKdRO45e+NvM+FpSd9E3j+tAC+G145yNs7Yj/m5e/Qne2j6J/pC8uGV4qndRW0PCNFSZEbfPW0u80x+MeqjwJvlPoosNeIRKVBQTlCdZj3/AFM+Fuk16m2woAJUTqdZ023kdP62lRHol+kOMyTw4vDUnTtWRAMae34fRttHoj+kIUmeHFaNIhTrCuZnXPvy1/Oy7xTfOPVPwJj8JUTGqcKiVASdfHXnr110tyqsXlBUUnN3lZdSfr3be60wUvogekVVKLTfDerQcsqDlVTtgbjYrg6RYm96JvH8OhocOLyMzIDrcbbHvRrPXwsu9Uw0Lx6p+7T75T6KKfXnHVZjJ7+k90kdfDfn4205XJgJzglGgI0Khz193ytKi/RN9IVPcPDa8VFCoCwtBmdCR3o5DX3+YZ9E30hFZQeGl5gL9mVogR5K25bjraQq6T5x6hR7tP8AIfRQ4zULS+k5u9m2B56Tv7rCuVa/ZClCRqdI3IPly8rSqPRK9IcjKOGl4pymZzNiJJ/xfGwg9EX0hgkn/RtWmei0TtzGaTt99imtpL/1G+oUBTVA2YfRRSm8atB7RNStBBjRZSANfHwt3+/b3ZWEC8aoZdSA6RqBvvvtp4Wk1fokekIBH+jWvSMsHvI1A0/m3841t0fRJ9IVauzHDi8MwJUR2jYnx9rQ+z87OKuk+dvqEjT1Hyn0UcNYwxElv7K/7yGbUgVKwAdgfa6fR3sMnHWLUggYmvXKnQJFY5I16yR7uptITvolekOCVL4b14OnsutydDyzaa+Hwtx/5IvpEZj/APw2rIgwS41+KvGyFTRn42+oTd3qflPomEjH2L2niTiW98+bVKqxZkcxv52Ms8ScYoBR/aW8CdMoNQodQdeY5e/wmzzHokekQVZDw4rgNVBXaNxA2nveduv/ACRPSFSSP9H1WQZEesNCSdt1/Plz5WRqKP52+oTiCp+UpoDihjYDL/aWs7xMZoObXy89vH3iK4oY0nK7iKqVlGgUEq320KY6z1s8mPRA9I50Ky8OagQMxipagzznPrrv00sI/wCh36RlK6KdfDupJSAoFupZUCd9FBZE78/wtDj0fzt9Qn4FT8pUV37im/b7Uly+bwU+42ChtRbSCU9O6BOvUT8LB4exHeuGrxXX3a8EPOJUlX2YWMogzB05TItKb3oe+kS240g8PKllLyghKlVDYTqNicx+XjY8z6EnpFrcCP7FJlYE/wCvMpHQDVUWJ3ykAsXj1CYUlUTfIfRM+k40YtpUABVA6EjvBTG/iYVoNrH0ceMUJASaK7FEGP7lwaTy73h9a2XXvQr9I9Mf/IJE8z+8Kcgjr7evX6NiS/Q/9IlogKwCQnQAiupxv17/AI2HxqJ3xt9VPg1YPuH0QSOPt/JEOXNdpIJGUlyZjU+1Y4xx+rwuVYdpSEzJD6tDJ3H4+Gx2toeh76RmbshgY5hyTeFNA0HMOdB93U2T8XejJxxwPh+sxfibBC6O6buQlVQ/67TuhCSpKQYQszqRyMTaOajeQ1rhf6qWSpaLuafRLjHH15JLT1wU6lA65amB5+yfryEjDjrTrRK8NHltVDUdfZ+f32gpDqScoUZCdJJk6/1sMisWCA0U5iIJHj79dvusQ0jELvDlOjPHK7FhAcuWoTmTrlfCo59OQ/ppZSuzi5cr1WW03XVoJBKSFJ18teX1pE1+RUulXf1EglJBPj4eP4WV7jrlivbICiVEtgde78/zJsCWlAaSEWOc31Ur4/xjd9+0NMzTU1ShxpalhKgADKRpIO8gfQ0iukrBSYhYq+yJSxWdovL3tA5Ow8Of3aWXax5LjYhJWU+zmIBEadekWZ9epxFdUKIgFSSAkyIgRty0+Vo0guCE8ztQVYCnxtc0aLeSdDBR8Bryj4ab2UqfHdwZUpD7vd6smY2AiNbQ9TPj1ftE5hm/igxG8dI/DnBix/tlBKFgpIA0gkpBJ+ufI62pvaQUZshKls41uDIhSqogZjqELmdNzFuFYtuIqCPXBKgIJQrmPL9NLROmpKWgSVJMAeKvx5+Fki9cXXfdMtBYdeEdxBTod9Ty3PWyjifKbNCd8wbq5THV4iuQs9qLyQhIGq4IHI7kRv8Al1tA+PMTXS9X1t33YoPoeqSS4kygJzT3dBOs8ufibN2+cVXnfK1B15LTSlZuzbACfCepFkJwkmSZ1mbbNLRcLV+6zqiqzCzVKmAr341cQGKfB2GsSvP0eH6ZRpKOodSWqdtTpWoNpUCBK1knTnvtZ4VHDb0j2UFa3KF0azFOwZ08W+kfK0HXNiG+MM3i5W3FeD1G8oFGdpUEpkGPEaDeztp+PHFSmbDIxdUOJ59q22v/AMSfK0Kimnz3pwy3mNUemqKcMtOXX8inJX3Fx/pCfWKJokDVSael0Hnl0skOPcYKYAVFA2SZGrLIM6jlEGNNNbJr/GviHUp/1i/AuTJ/1dvU/DysQe4nYrqEdm7XpI6hpI6fDblppZmQVHxsYmfNBfwOcpd9HfG3ELD/AB84eXne1MGqRGJLvaqF5QPsnH0tq2P8qjb3KdYTUMuMLHdcSUH32+di7uI+IaO8qO8PW1BdLUIqElOhBSoEa+75+VvoiuO9Ke/blu++6Q5mbwpWqto8ihxAUD8CLY+LQujex5aB9FcoZQ4OaCT9UzUMqbWppYhTZKTpzBiw7YHjoI99j1/U3YXipYHddAXMc9j8wbEgNhyGnutqUzszQ5VphZ1l2lMkdPusKkaagadDYNPtDTT8bCjUTpBtoBU3C4SldLgJKBznX4R+NkXibct8X9hh2guJCXKntW1ltS8gWkKBIn3A+6ynd6gH0jxB+8fjY7etYLuu2prigrFOytzKNzlEx8rHZq2yrk2ddRTd11qwfii7qekrHl1D6GV1dOp3tQytRhQSdwkj+HbSQBNlHEmGb7vS+r2qat6poKdDanmK5JbWjRIypyqkx10G2h52TOGlddt5V1RinEd5sisdcK0pWSYJG/kBoB0FnLxFvdbyKO6KJ0BqqAdcczQnL/DJ6c/daLcoYTyRHE5gk2lvq8nsBVQLq0inqGm8xOobUpMp6xr8LJlzvV+Hb4u+9hea3ae8hPZFKQG8qsq0SB3gZB11BnXYB4C6rsRgqtua7K2nqX/V1uKLa0qUpwCZIHjHysxql1NXh673WlhS2Ktad9Q24jMD5EpsW1gChHc2Srik35e2ILypKGuS1UU4KqZK0lSIS2F5YBETr3ht42e3D2+n73uQesvKdcYUE5lGVKQQCCTz52adfUBnGN3Vyu4iupmHFA6DWUq+Ay2P8MnF0lfeF1qOjcoTP/q1lA+UWINDdQOye7cIdqGxpC8w94ti5UdNracBTeCyVaONgx5G21DXSyKYI1dLwprzpn9gl1M+RMGz4vCmCnV6e0i0diAd4jUG0lKWKmmYqQNHGwr4ibZWKQiSFwPRGhcWu0TGdoQHVjXRR5+Nss5F0iSonJubZbzI056Lb72V4K+mj3/SIxElBVPZUKYH/wCqtdbQ8UqcqHgW1KHakZ52n3dTG2utpu9L6hqLz9ILEdXdzJqqZfqWR5glaTlpmgdU+ItHQTdVI5UpqaRaX+1IjMBy13Ply1t3VC7JRxNt8I/YIdb4qqQ+ZTfNOvIVdmXIMEARoRoD4mCN+Vum0KUMqgYmSkIgnu6Hw3s42b+uWmSWkUhb7Q6gqSABy5zpztpNfhDR00jp00MmUmdNSoSefS1i7uir3HVN5NOtQSCFDIJJjbTczz11526TTKSQrs1qOgEE7xp5f1svJvDCi/sadp3vryJGZQiPMzz84B6ahorMNLUnKFoSvu7geMb7fITZru6Jhbe6RhTK7RBZUQANYTOu8nx25fp2lsOBbYBQghSVExB1PMzO55zrvZ3XFdFFftO7U0YORtZQVqWB3okiOehn8rKQwa0lKgnKIEAFcmQJjlGmthGUg2RRGXahR+pgEkZDm9jQDfw6a+7QG3JowBlghSN4BHI9OUH7+VpEZwSwk5XklKcxSR2qZBBI3jflz2EeBhGBLrUQlZcAk5T2qNTrtO58+gtHj2UxASoxUylWiGxKUEFMTJH6fHnIsG9TOskZ21QRA1ISTtM/dytK5wTcwSsOVDwC+jiMpJEztzn5ixd7h/cDo1vB5IMk6tkgbRtqNPhNk2qF05pnEaJF9Hp3Jx74al0pAViiggbjvPo01/W3uUmloHgla6RgnL7RbTP3W8VOGVzU9w+kPw2pGHnHUDEt2rK1lJOY1CBuPK3tMmqCW0x/CItQxSPjuY63JXMPcY2uaeqEN33aglQoKYf/AFQ/K3aqK7VaepsE7atpsA7ejKmxThiFzq5OkeVg1VC4y5pGkmd7ZxogFeFTfdDGiu4HSiY8fsxbZobuUO9Rsmf/AFY+elihvFFKStxCnJ0ATv524RXlYzGAN4mRZdyFr2T95J0uu13bQF7tU0rIUTybEnzNulUdINSw2I/wjewZqlHvAfG2i/Jjl1tNlKBpZMZ/NGW6Ojyx6ugQP5RvtYZNFRAEdg1GsDKLE26pI3V7tjYUVOmpGg2iyNN5JhPfmhzR0mqfV24Oh7oNgxd9Eoz2KPegfjbkVOYwUjrbn1sTEj4z8LIU9uSRm80KigokqJ7BsGI9gdbCt0tIP9mkTzyixUVAnp5CbdesmRoI5gmzmm8kuN5o0mnpEpILKRp/LblynpZksok7EpFi/rYgiQD+NuVVRmUmTt52j3XySE3mhPVmEq1QjQxqgeFuls0uiuxQSR/L87F1VSo9tXh4W4NSdif8Vpik8k3H80OaSjgD1VuR/hEWDVS0YBHq6IGh7osCKobAHa3HrRGvLfe0hS+SXeD1QppqQDL6syY/wWLVdNS9lpTteyf4BrbFVBgzJG2n19aWL1dQgNgmUwCJ8bQlo/Dspsqdd0LTtUxcWoMtzl0gC3KmKUttJLLRk7FINidNUIzokiVIn3zbhNUcyNZCSNZ6k/laiaMjkrPeB1Rw0lGWCo07R3klI5Hy1sXNBSBgqFM3Ko2SNPlYJVcAxCtyo87dPVI9VmRMA6ajey7nfkl3i3NGU0VFH/mrIEfyixdykpVLX/q7crIGqR8bYa0BvMCCI5mwCKxLj8rJGUEjl9fpYvc9ALKPeB1Q1VSUbaW/sG+kBA1t2qgoVJj1VnQDdsWLVtUFJSSNlHnrNuzWJUEqAExOlpso7HZQdU+aKu0NEapI9VbgJiEpAsVqqCkfUGkU7Mjc5IPKTYeqqwHisBMJTPgd9LF2XyUhawoKWrrMaHT5TaQpNU3eQEUqbsoWlJSmnYCYBICQDv5R9TzsiY3xHhPBlzLr75yUyHCllpTdOFLJUYlIjkDvECLK1+1rV3Ublc8VFDTS3lhOphIJ08Yn5+NoQ4ecV3Mb4pp75vAyUsetBouy2wlZAbaGsGJk76gnSNLMVAXnyUHVYbZPpGI7wxS8tGCOGN64gdY0RV1zq6eka/wJ7RRSo6knK2Umedg8S1XGbDF3euO8G2qtE5nDSVDTwbSQMwCMoV58trWCw1elB6m0xSMNiESCjQKOu/Unfzsevq8l07IU6ltKlwMugPv1m1luHQBuZyn3mRzg0BQNhDGOHcbUqwzdrdLXspHrNI8hCXEGNSUnvRPgR42PVV0UTy8qaJkRqqGx3dfdzstYmuS4W3nsUMXOzS1obzKrWBCnAnXKobEECPf8C13w9Tt1KSIdAXIiACJ395+hanHTtc4taE9RI6IAlEGLtpUVHZqp2vaBAygwLNT0grgoL34K43u8UjJAw9XLSnsk6KSwpaY6EFIPmB0s9KmvS24qmTJgZdAkDNmCs8xmmAU7xB2nWxHGVMi9sKXxdgSVeu0FSwZ6LaUn8TazFSBjwQFTkqszCF4h5UZoygAmDHx+vK3aCkyFLylJCZ2/pbpdMtsqVJkHv5tANSBPvHhbvsXHXClCBCk7JTqdfrbqbdXcLm7ELaELKu6tJCRvBhX1B+Njl1lTVYyvXuuJGZI0Eke75TNgGGlzmLYJBE5hOkgayfr7jbbaG4dQhKoVmkxAjYa76RMfKwJDcEIjBrdOqoUkJCFAZld4BSOh6jWdBGvPxs2r6UoV6Va+wI7s5tT79tyfnZ1VKE9iEtmUq0UCmZI0HPw38OtmriYgJaWlMjKqToZ1EHfTUqtSpCC+yszaNunDdLinKJhxMx2aO8lUkRp74GnusHeV60tCz6xVuZDHcj2lJ5QOc/gOmjTRiZdDQM0lOwntG5+0WJiSSBHPc7/CyLUvu1DxdfcLq1ays62tMosziXbKu6osLDdLd8YxrK9JZoppmtZP8Z8yPKzdKpOqjPjbR00Bi2jMa9NBa+yNsYs0Ko97n6uWxIgfdbSgZ1523Cuu56227BAM76xFiKFlysagpESBbREDS3RBCUK1gyAffbakZR9aWSVkGCelsOltwd7ZB2skloEzMW+gL0Q8TDGPoxcM7+zlS1YcpKRxUnVynR2C/wDmaNvn9O1vaj9lpiY396Jd13ctQKsPXzeN276wVioH/wDs/K2PjTM0LXdCtDDXWkLfJWgxLThdO3UBMFCsp8jZux7vfZ6V1P6zRusACVI08+VmbBB1EG1fDn3ZlVuqbZ110AZkEeXIGwiVEka6+f1424GU9QDbsCQRlPiTbXas8lDMOdm6FE6CSfdr+Flh9hqqYcp30BbTqShaeoIg2RRBOVZ7p0PlsbK9EtTlGwtftKaRm841+djRlVnjmmgxwgwXSyKRiuYBnRuvfA+SrKd74Dui+GqVpb1Wx6m32LamX1JOTSAeu1nGCYInztuYMDexg1vRQLnHmm5h3A1Bh2rcraetrH1LQW8r7gUACZ6eXwslvcKbr9eNTQ3pV01MpZcNIkIU2kkycpUkqSPAGBsIFnwd+VtiNgdtpNpBjbWso5nXum5iXA9DiGipWEVTtFU0IinfagkJIAKVA6KSYEjwBEHW3eEsHqw4XXaiu9bfcGUrDeQAeAk/fZwiAJmBbtKjvpaQaN02Y7IpWJy1NO71KkE+Yn8LdLSDuAD0i27wgNIcP8DiVfOD+NtrEAfGzEalK6BIg66Wf2H3RU3DTmZLctn3EgfKLMJXx/GzwwW6V3fUsGfs3Mw94/S1WqbmYQptKUSjU6Wyw5RJJtluU7i1Wsy+aKnqV0TLbTbriTJBKVQo6kgT79rYupf7ZSHUOJWhSgoFKe71GniTp5c7ErrRUPM06e0lSjHdIEHYDz8PlZw3dwt4kX7eTrd10HaPIQp05qpsbECSSdTJHnNulbAXOsAhGXKLkpKRXLJyw6cw7sjUHp9DWNfAVq8AiPsXE94SFIBJOkeOhP1vZyDgFxjQO9cyEiZM1rO//F4bmxpjgDxgUpSTddOTlJE1jJ13B38D9RYvc3n4UPvLPmTTTeBUlJLJjUElrUnQgGPKI8Oe1uvXErhCKdefY/ZSYIjpB+Pws5TwF4xpQS3crJCz/BVMTIn/ABCdLcI4KcY2kR+5BABTpUMczP8AN9TaJo5OicVLOqRmbzq20GlpnapCDKy220QFH+JRKfHLyHnYRF+X0FZU1l46gpGVTkDXbnHLygC0l4G4H41qKGpcxBhvtKhD47OKhsSMp7oKFa6xqfGzne4G30hSVUmFVNgCClNQCSZTrqvTTWdxpas+FzXWyq0x4cLhyg9q/b+efSUVd4EqTlBGeQR+oHT4myg1V4zWgGnTeeoBlLSwVancAHx8J1i0v0/BLETTqVKw3UNAKGoqEk6kyPa8APjY+nh3i6iWtNLh10BJlMZVctt/r52rSCQe7H+FYiyH3nqFm2+JTqClqhvtSBBMsLUZjQ6J93Tffe3NQviTQNhdTdd9ozGVLVSLlX/LpqPhNp2RceNqVK0OYXW5BmSyAdSdsvMQPjYk7duNy6Gqm461IcGfSnUlIEDaDpyA8rAa6e+seit8OntpIbqPeB9ZflRxzwCm8EVSHEYju1MvtFsgesNnmBvJ+t/Z9tw5QZiYOlvIzCN3V9Dx64fP3hQu0q1Ygu4gKbKCQKtI5nb9PG3rg2QUJGWFRO9lUsz5TayhTuyZgDddKUCRJJIHvm2jWtoEKGw+vvsE4HMhKUmBz6e/4WJL7RRJSox1ja0WQjZNJNbZGi4y4vOSNIk/XnbsOM5co166/dZPQh51RShJUqJIAtyXSFBK5B5z52L3cHRCFQQlULY3UOe9tlxoSQSn652S0vAjQg/hbC8JIzeR/GyFMEjVEpUFSxvqPCNhbBV0/JXh42SC6IJB5RpbkvnNGaROhn8LTFK0qPeiEtqqmSYMkjTe2GpZkEEn+tkUPySAr/NNsD+XvEyN43+jZd1AS70Ut+tMzEE7RbYqmTM6TvrZE7eUwTPPlpp1t0KgACSQPPlZxTCyXenJX9Zp5g8vvtpVVTzqPjZIL8GMxHnbXrIUrUieXl9Cy7q1LvTkrmqYzSNJtz6wxzSoRpZK7bTQK0H420qo2AmDG1n7sEu9OSsaljcoG3X87cmop5jKdZ5zZJU/zUrfXfS2lPnWDM89vCzimCgak8kql6mH8A8JNi1dUU5bIybggQbEvWFaAq0+Q+tLF66oCkJSN/Pwsn04spNqTdGG6uk7Rr7OPso+FuG6qm7ErCIlwDfTSyR61kQy6NB2SjbTbw9QJB2XJ1Gus2rmlaiiqclM1jJEZdZJ1OlhV1dMaUSn+GRrHQbWRS9nDywQYlI/H7rb7UmlCBBCkxE77WburVLvTilV+8KRLIT2YTmHUzpbVLW0xdCuzVKknUzIshPvqUqAe6BAVI0+NhkuparEQdkZddtv0FnbTC6Y1Tglm8axgNCG9AQLCJq2FITDWkdddtrIdc7LG+oMkGd55fXjYQ1IQ2FKVASnlB1+vv8AGxO7tCH3lyNVVZTrUlkszm8TPl8/qbDpepk5R2QypIhP4x9bWbztSMyVpJAKp9qSQDrz8fD32URUAnTWDBEnXr+Bs4p2pjUush72cpahCW1sIhYLahJ2Pj7vvtT2huS+Lr4o3VgG6rlvK7WzXKU89UUpYUulQohK050gqCjICkyIOh1M20q35Q2pKEnKdEqEz4b6frZCvbC9Bd9WxjCiTSsXVdjaW007Lcu1DrypfccUTOZICVCAPZJObQgUwMTSQFeobTu8RTpvaivdFzihuZNTkd7jy2qs0y8kAR22vZDQkkSoxAHSHeGuDb5XxCqnDTXZR05ePrDlHfFRebi2ZIAU8/MKjWABExrEmaqrE1Jh+6n61x1LyKZrN2TRAL6tMonb3k87RvVcT3aamqKujxzhK770Lkpu11lLzCWx/CEoU2oqMR2hJEzAgRbOa9rhYro44XXzNSDju7cV0HEKsu0O3k83TKytqpL1LJSwvvJdUye46M3dggkQdRNpWo1Jprup6NISSwy23sCAQmNuXW0TMcSr0xPeP72xHRXA4+hSWQihV2mQQIUFEmCJnyI90stMLcp01B9rKmRsdt4O36/CWG5JJH25Klj7XQsjzbn/AISReCyp9WQhPUAnWf6WFgrSG3CQFd0687Gl3VUVIeqaVJU2ynv94AzBMAc9EkwOlgygBgJOsJBEiNB+v1tbS4Wq59slwvNm8sCcHLvvu8LuqnrhFRTVbtO62qrSlSVIWpJBHaCCIiPC3bWC+CRICTcS0nvDLeBAPwc00+trNLjhwLvV7jDjN+kqHEt1F91j7Tfq61aOOFYggbDONf62YrnAnEAWA1UAhR0+yXMe4dCLVXQQk6zkFXmzSgf0AVOKeG3BVxtB7O6w257ATeSiFDYH2/EWOU/CfgzUFSUpoxlBKibzc/Az0087V/HAvEaRPraZ2MMOEBU7SUiPx194DvBbFDUjtmySDA7NzUjlMb/XOzGljdtUH+fdEFU9uppwrOK4c8ICnskNNqXlCQG7wcUmYEjzAFqc3iLxrb2qaAqW8tpbjaEN6gZZmAPBO/vsv1PCLFTCe0DjWTkTnEnp7NmZX0dVdN4u3fVAh6ncKFiTuD8bXsNpGQF2WTOT+Fn4jUumDbxZP7p9XDhe4a7ClYbxuS90X96wU072YCnDRA9pJ70iF+FuVYBu+FIKqxKgdsyT06J6fh0s38J4VxBiv1v9zPpHqSUrc7R0p0VIEfCysOGWMgExWsAiYAfVpzI2tZkeGOIMllTaC9oIZdSdgXhJwbxRW0lxPO4pTXvMkrcK2UMlaEyvLCVabxNpHc9D/hS072buIL7ZOmgrGCU+ctfWlq50XD3iOw+HrvvMtOgZQtutUg+QOnX52U2cJ8Z2wkM4orW8qcoAvZxOUCDG+m3ytmzRyudeOqsFq080TW2kprlT+PQ14WOphGKr/SIkS9TDX/2c/jpZgcePRkwhwy4fuYrw3iO8q2pYqmWXGKhbSkhC5E9xIMg5fjysyxh3j8ggJxjevcOYRfbndPX2tLB3lhHjtfV2que9MUV1fd7qg4unfvhTjRXMyUqVBPPa0YW1EUjXPqgWg6jTVFnkppYnMZSkOI0KiTIC0DmEhUWuxhD0DcD4rwtc2J0cTrxT+9rvp67sk0TXd7RtKymc+sTFqzI4GcRilSfUGACrUGpRqRtz8fnZ9XXdPpQXNd1Pdd1YwvVijpEBqnp270ORCEiAlImAkACBt87HxOZ07AKSoawjfbVVMMiZA5xq4C8HZTLWfs7bkRpR8RbyWraP3ahUq5ahyyY7+zrrQFOIx7VoQJGZdyk69P7wC0c1N+el/hygqr2/ttfzbFNTlx51F4gqS0kEqVvOgmSNbM930rPSHfc7V/ipfLqoABcUhcR0lOm3Le2dDT4zKLxVTXD6f8LQmqMIjNpaZzf59VNDn7PG/hAZx4XAoEibnWNBv/tOXO16/wBndwlvvgjgfFeBb2vYV7b96tXqwoUy2cpcaDaxCiZ/uU7W88sE8YPTexrdn77wfiW9K+iDymu2TRUhSFpAJT3m9BBGm2vna5/7PnGHpPXlxFv+6ePDla/dT1zFygcfp6ZsJqUPokS0kKJKVq3kd21eU4iHcOpqGOA3A979lMCgczPTQPaep2/dX0Eyd5s0L0p/V651uNCrMNeRs8Dpz+FkLEVPq1UAanuq/C1+hdlfbqqFS3M26RQkK84iI5WETA184jS2RsInn4fWttyDzIjebbjVllbAIOpI5WVaNwqZjmFKj3mR8jZK1EwD0Nj9CsdkpAPMHT4fhY8e6rP2R0RuAdrbBIEGfjbgGBoSIttKidBz6WsoS7EDczbAflbkdZ8NLbmdY162ko2QgO8iBbtJ1iRYESnQi3YOgPwtNMua5BcpXgPayGNOY1FuGFh6nadJMqSDYdRERBI6WKUcJpuyM/ZqUjfofytB26fkuliBFnFgd/JWVFPycbzecH9TZvOa9d+dlPCjpavlkkwFSk+MiwpRdqdqeZQqT52yxssgmbZbG7mUXiBfMtcjA7aibK0/3qClMcyofD63tZ3hc36xf1YhCkIHqyl94EQntE89eg+XlaugumruW/W7uvOkeo66iqEMvsutdm42sLAIUCMyTrz6aWsrwoYR+8a59TqJSwlGTKRoVSeWu3n+G3R6vVaq0jKkZdMppAl5CkKITlSqR4ae+PrUJIKHQsEpBVm2nQHr77HcjRkKfSJEknpPM9BbYZbGZfboBGkZuu8/XxtsgmyyQUSWlWYo7Mnp3oEfX0RbR70KKRBIkg6n3fXLxsdfYYOV3t0HQagmVEe7cQDrYFbSfZDiQIHsmAfL57/rZkjqlbDRSadyASUOpMxABj4fHxstKUlStOZlI57DSLJeHWULYeQFBQDiBGsDfpZXXTLSUkJOmo5SNJ8BztTl95aEHuIE6HOAZBJH/N0sGoLQoOIKCokknfpv99jCkZm0tBG0xGmup28/usGsJSoZkERqI122/wDDHxsFGGqKqzlPaTOxPPSDpHvB+teHRnhACQlMDKPMc/d91jAZUpvvpXqoJlU7aTYvV1LdHTO1VUshptBWoxpAE/jt99mNgNVMAnRQ/jNeT0hOE7cJzJvqjUop3A9aa+Wvvjwt6ctLSpIn3z99vLS9Gaq/fSD4bLW6Kd6uvyhT2rmoZ/1lsIEc430G8nc29L2KO+0o717g5tQex/W2FWuzPuFp0oLGkFONVe+ulVQJy9i5qrTWdOfwsWU1CVJMGBpZKFDfK+7+945/3c6WGFBewSqL0UDEQECwmEnkneAOaNMVzt1ZnWMvaKGUZkzAsjl9alKWtUqUdfPf8LFqu7b47WXL3cyFRgBAsSeu+8CIN8PgjnA87W27WsqjjqlX1kb5zvEDTlbPWcycyVE6TG9oM46Y9xDw0cuGoo8Q1KGq9x9t1HZoOfKEkHVM/wAR2NowX6SWLgsZb2UtMgqJCfdACeevuFkXBptZSa0kXureqqhmJzGAfo20qoE6Ekcp3+dqjtekdiEkhV8PAjT2Uyd9+7p52EHpE4i0y31UwqIhKJ+Q0mzZ/JPa+xVtPWhB36RI0+Fs9aSRJWZiNDGlqmH0hMQd0G/a47pgBMzoZOnnbn/T5f5Ol+VyRMESkH7tbNxQOSYMPVW2FWEyc2069NLb9bkwCmRpEWqV/p7xCdVX1XA8wVjTly8j8LCU/G3EdQSVX9XQTJUXfZ+G9mNQ0clIROPNWx9ZHXXTUW5VVECSRHhoZtU2q46XyVkN35eMCRIfIMjn4f1sGjjXfjhzfvu8SAQO8+rXXz+tLN3hvRLhHqrb+tDLpJBmYtwqqMiAdOfutWq6uKtdVFJevW8jO/8ArTiQJGmk2cCMcuVCErTe9fBEE+trnx5/XvtXfXtbyRmUbnbFTv6yVaJnoLaVUkCASOW0i0L1eKKlqlacN61ozpgkVi5knzsVRiJ51p1z98Xr3RoPXV9dt7C9qMteyMMNeeam81JzBOXUE6g2BrXz2QUOZ1kfXhaCXb8rKkBDWJLwaVkIbKqt1SSepAUNJj5CwN137eBeyvX9XVCk6k+sOhOgOuUqJGvj91huxmP5URuFP+ZTCuqUthoQZlSTO0TzsIalQpHgCqA5oY+utonZv+rKCo17pzLSgFVUvTnP6+Fi19YpfpXVoReD8KUSMtQuCB0B/HwsAYzETbKjHCpAL3UwNVLqg6QFEFRiIkfXW22X3vVUgNLmDMDSfD5Wgr+2zyVgmtq1CCTNQojXbn5/CxhnHDq25Qt5Qj2u1VqfKbSOLxj4VEYa/qpmaW84+ErScqQCox8vhNhHalxNa2pKFFJESR4nnaB7yxw42goXUOqJ3KXj+BiNvhZBXjZ4KPaVTpB1Ci+slPXn9aWePFmO+FDfQObzVl66rdLEgKI0Pd1+vr3l3K1x0NUqEmcokiQR8uUm1bnMc1IkGodAgCDUbTz38/DXzsUVjp4pKy+pUaH7YzHjrpzHw6WsiuaeSGaM9VZyrqlN9inUEGIAGpnffz+djhqnNglUyf4ZJPKBGvPlap7mOHgtPioEgLk85ETry+/yKHH4OZJU0O9qC7KcwjQ5fMaeHS021YOwUDSG2pVsbxqldmlHZqJBJyxOn19TaMPSJxTjnCnCisxZgJNcm8qNTLTqW1fYKpXe64X2j3FoTIVKhoY5G0LnHynm0eqqpShLkKcCgrNtpvE7QCJJOk2ScF8QF3txiw3ct+UZXdNRUVNNLi1BLlSqle7NfLKUODunNIORW4GW9Rs71K1hGhKE+8Au11lIXDT0m7qpqumwJxDvLtkhxNHdt91tKmkReCkJHdfaStaWnpAiFqSsQZSoFFrC0103FV3c/eVxpuVmsqCpbtSwsUzq1QdStEFS45yeWsW8++M/CKsww1+/ENVd/wCFryf1p6koTVXZUFagtKqpQygGdFOA5jEqBIKmRUcU77unA1ZePD7HS8PpuqsFEblvO+k1NWpvROdpl1lSijaQl6E5TCY1tOrwaKbNNSOA6tO/mtCmx2SjLYatpPQhXK4yYwwVwJuavx/WUl2VuIq1Xq9Gl51T9RVviCltS1KKuzBBJAMQdNYtYrBF93hiXBNz37fN0JuevvS72KqpoFOZjSuLQFFsqOuk8xO0i3kRgjF1XefECi4gcW37yxxdeG6+ncqEKr1BC2i5CcgdTAbDmUqTlSFARIkm3pNgb0sfR+xYlqkfvV3DTroPZi+qZLDKjvo8gqaSnoVqTNqFPQdzbrqTuVHEcUOIuBAs0bKV62rqKcpaoah5AWkoX2f8YJ1mNPr32KB1wU4TEkAjfb61+XSbGHsPUV50qX7rZpqhh5vtkLbKS2pEe0FTlKYjWY1FkdrD9K2pxtVGhKkKKFJUgAhWs7+/4GxXAhUWmyrnxXQzR8Rb57NKFhxTa0EkZUKU2gkx5n60s2A26j7ZbCcqlykFyCRzI0mPHaNLSHxpuVFJjlKadDbKXrvac/hSBlzJ5+CR8epsyGcqJyMl0+ylS/ZSNTt5C3F1pLJ3DzXdULeJTsd5LhgMoYzZVJcKYy66HqTv10+6LH2mXngKiqaUltBCUKU53SY/hAEmZ5bEWDZoXHSArNCtBmPdHn8fnZUTdL7pSy2pdQ5tDae6B4E6nWOUb2znSW5rTZH5ImKqgQpt31N1SGgdVuAIB0hWUCOnhFqN+kXTFvi/fj4IWKksv5+SiplEkeGabeiF13DX3dFYinpW3G5WHKhtDoSRyAUCkk77T5Wpb6bVHUjinQXvUKzLr7paK3A2lAUtDriSYAAGgTbb7N1F6wsvuFido6Y9zD+hC59C6n/e2OcSYVDvZt3lcrskqEDK4gAmd4Cztr94tUOHVUzeVHedNXUyl07yXyhLQyZwyhtSRprIC9uZGuhtUT0QHew9IC7LvIKhedNVsiDz7Eug7TugfI29J7rwq5UOJHYFSU+0IgRoTJ2+Nqnauc01YDfdoP8AZT7N0sdVR+MbEqPMJ4QbovWHb3u2jNZVViqjtGmwvs0FCU5RmAgyJmOehGlnpdeAsPVdO56xdVOiUkLqHmEFI1kBMe0dzz6xvL2p8POOSw02hLSo7jXsqM8zur39DtZdpsOUdMhJqFKWoaBpG4HjytxMuIk6grq2U8UIDbKPKbhjherMIuelcQFQ48ujbAGkTJM/j4WUqbhfhCmeCV4fuqoWUHKF3Y2RmJmAI1Gp5c9rSQzdzi8qWW1JCR3SqCoeXIfCyxQ4ddbkgZVnfWVK8z+dqJrppNiVCSaCIXcAo7oOFeCSpT14YQuxxajISqiaBPQkAaWXLv4V4LcgNYNuQKX7S13YxI8u7oPnaUrlwJWVagpTEN75z3RY9el64AwIAL3vBuqrNk0rHeUVchE2uwYdW1DeLK7IzqTb05n7LBqcZhzcOFuZ3QBRbV+jvhHFNxXncacHXW4LwpHaYlNE2kDOhQnNGhE7i3hzxG4dXxw2xpeeEb/aQ1V3dVO0TyErCgl1CilQmI3B+FvoO/fHE/HFP2OFbmbw3diyUioqgUuFP8yUDX/w+dvC70trkvXCPpA4uw7ftb65W3ffNQhx9ZBLoLkpcMndSSFak7629D7JRGlc6OMuc0jciwNul9Vy+LSvqG5pi3MOQNyPryV2P2ceMMVcTOCN68JcNcMrgcbuS80et3sWUoUtD4zhTqjmKnAUKAMeyEiNJN2uG/o7v8P6lzFl8YkeqrxCFgUzCA1SthW4Cd1eZPLa1Gf2K+NW7vvfijgV1SZfYoL1Z8m1utL/APwjdvTm8757VtTM6KFliUNFS1r5JSS7Sw5BCgr6p0Igi0bz80kDbaDYnerBfoXAAJRCx7rHJ1525MKGU7EG1+E5SCjPFwQmeOsgz42wTMjQGwlS0WnnGlE91ZE++wc5uZ/AW6JhuLrHcLGy6TqOZPjv9a2OUCu8U9Qfl/8A9WKSeW/LnYakUA8kAjptGkH9LGYbFVnahKUmYjY22FQJiPEWDzE6je28wBiLWggEXQmaZTvytuTGhmbBBRmY526Ss9ZtJOhAZ8bdpUOWgsCCTBB1Put2lR+OlphQQoXsZINiVIspq6xhX8LiVjyUkfkbG83LSxMyi95HsvMdOaVf/nWi8bFONkO7oJPxFh7pdLVc06P4FhW9gHNBofhbu7z/AKwD1NmcLhIbqVBqJtlk+nvBHq7WYiciZ+FstTUrFfNrWV1bXJRet51rtTVKQl156ocK1qypAEqJJMJGk9By0tYngjUprKSovBsaPU7CoIgiQTEe+LV2qGgLlSpBRm9WJUCYkZee/M+Um00ejVWqqcIOu5iXG1JpieYIJI6clC1ujFpECqN2KbC+5AheUwJmdD0+utuVPgAwkQFTM5RP19b2Khas2VCu8dBrB1+vvtwpRUCrkkhOXTveevWOlthZeVKDT2dpSQJKVBQGXSI18d48rAhwySnQq0EeHMfpYohamnFdwEK7p1Oug+YHw0tijlcJJUrNqeYIImNfIfnaOymnThSsy1D1KUAlwBaZPdGU6/DMT7rONbyCQgiVITAVO8FJ1+MWjm7KxNBWsPAEhtUkZdxz05GNvGz8SISEoXmSQIV/MO7B+61OYWKu05u2yMMvISsApygQRIB5R+I362AcLagcqSd1eB0PTTmbdylTaSBtuNoJIGg22tpRJUZUqFnb4Db37bWDcKyAuUaryqOhJJ18T87I2Lgs3E4gIGjyFwQNO9p8I+7nZYUnLlJXKTBGg20/GfzslYlyKud7MYSSg9Ce8PlqbQkPgKLH7wUMKqOx4/8AC6pOzV/UMgE/+lta+PP6iPUlpsxMEwCZHW3lZeYnjhw1hZEX5Qmcsf8Azps/n8fHT1bZQopAJFsaUXsrrTqUWUS0A2imzLJnPPs7frboBWVRB1Ow8rGVNDmN/C2nE9xUdOfSzxhQkKJ0ztE27F6MKdaA0SkwZshrQnOpSUQCo5QRqB4+NlRxBzKmd+n142LKbjQER4WtgWVRzrlVf9M656u9qHCTNO6tuKypzZACSOzSNbVsRgG+VFIcrKzSNCEkaeY6/WptfLiPhFjFt7YeoqmQGXKhxJH+VOnvANuKDgbd7jyHiwSknNISAfAQBZxTmQ3Tcfhiyo23w/vNcj1qtBOvdKNP+WfH32PIwJXJSCl+tOoCZCZ93d+tLXsXwHuju5GVSmAJTP8ATexlrgVdqDPqvxH4Wn3MqPegqIJwHeEBWesIBPtZdd9u6PxG9hhgO8gJCKzQiCVDUdfZ00+6192uB9zpVnTTQeo0sMOCVyDUUZgHTS0DQuUxVjmqG0GAK591IcTXgaD24jbbTw+tLLF58Na273TTUVNWrbCBqp32tj003td5rgvcqSFeqGec62Wn+EFyuHtnKRJlA3+dq76B+ZWI6tll51u4FvZHdXTVZnQ/aRI3108rHKHh7XLeSk0tVvAhwxH1+Fr5VPCO5ipR9QB3GwtjPCK6GzIowPCNjafs1xG6H31oOyq1hLg9QVYS7WM3hHMetKSZ6bfPe0iUvBPCqyls3ZW6gZj+8nzO8fxxawdz8OqKmPcpwnTTay1/YulzFXYjWIITFsqfCZS7RyvR4lGOSr89wiwfTtMIF2POyCoFV41BI8j2mnWwdTwswozROrauasStEJCfX6jIZ1JIK9d/qLTveuCahwJDLJjKU5kAfH6628z8cYZ/aNqxbfbWG2sYrulNc8KVSDShJRmUE5QUhRERB3587Uzg8p0LrfVWhiUe4F1ZxXDK6xUKdapX2kJbjWteMGN9VGyUOHFAp1akNqKgDqmqdkADX+L7vO1Y2cAftBK51v8AeNJj5wsHtRD9MgDSNI2JlQ05ac7KdRgn0z3r8uupYuDH10MNPI9Y9YvoOodGcSVZj7O8gRvYLsFfb+oFaZirR8CsJT4Fu9jLLJlpfdT27vd8fasUxJgpht7KGCtJzEkvLk6jUiYn65Wl+mulS6VXb06J9kefhYLENztIC1LYQgI0BI121PytmjDZA+4Kv99YWbKBmsFpddCVUIWCDI7VZg+czZbpuHt1qYBN1hWWIJWvlp18LSUzcbZeUUJQVpakwNvjZbp7nb9VQUtCAAkmOfjaE9DKdiox1cfRV8vrANMg5WrvSlQ1lWvyP32a39imC6lpd3MnOqAC0DAM/wBfoWshfVyoVVju6BJkH5mzZaw8isvgLQM6Ge9qoHcHl9a2NTUEttSgyVMbnbKMk8MbqLKC9dFKpSid2hr05bRbP9F91ggG6qYKTHeDYkyfHn9SbTe1cgdrkJUgJSlJy6aTyI87N3irXnA+B77xGyUdtRUi+zOSD2yhkbPuUR8PO15lJITa6Z08YbchV6xJWcN8LVC6KoqbtNY2ohbDbaFKCphXe29qAdZ5RaO784jYYbzO3ZdFIcpCSpNKlZSMxmAfE6SOXIG0H4mxA8isdqHHnFKSSnvKJKgrnr4qmPDaDYOnqRW0pccKpA0bIKshH8swTtMctDzgdLT4VGwAuJJXPT4i95s0WT3vjiPeb1S0G10rjTaloW2uUKAncKghIGndMb++xa4MT1lNja4L6CkMro71pS2oHulKXRmBnWSFDX/EnlNmiw/V0jpDThAI0UZJBkAgSNyY2nkecEdtpVMhT2dSHCgrStIyKROqCCBMSEEHQg8tBGxTMbBI1zRss6SV0gsSvQXGV4YGwhfd4UeKcU3BS0t5MN1FXd1dVM9sTBAWlgErIUIBASQYEHS1cr89HrhXiJmuxzhelCrsCnOzbu+uz0bikGFZA2SQSdShKkgA6AaS9MK8McO35h6nqWKSodVWzUVL1QhtLri1AZlOEqKnDMklYk69CLC02AcRYAVUN4Dfao1X19nWNICGGe6f78NQWe0RoElKEiFwSqBbpMS7LT1UxrovCHWJAR6XE2CIQSjNba6hG4LiwtgehvnDeJcO3y7WYkpy09R3fRJcRQtLOVDinCrVSRC8iUkqJEETJhVyuvW6w/dbNaXQw640VFpSFwkwVDMkLEmTCo6RuLXbu3hu5dzjiryzrUFFTpceU4tSlbrUtRJJOWdSdtY1mpPGTDLmGMf3zSIS6Gapw1rXbd4kLGYyfNIPKfG1TFcKdTUzZCNAbIImDnWCVeF/pBcUOE1b22DMT1NHTklL1F/eUTxO4cp3B2apEwSnN4ybXO4E+nhhziJiSgwnj+7kXdfl8VQYp6ykcz0j76zoFhfeaKlaAyqSRsLeatSvIwiFSRBUDtyEA78o91nFwh7ao4r4PaTmWVX7QAJG5PrCBvbl5IwRcozHkFeqnHG6qV7EN1V9StMKo3Gy3qCtSVmBMae2qzAYuXOlvtXWWGPZTlkz7iJVEa9J91pm4yXS9UouqrbpwsIddbzLXCAk5TmPX2fDW0dUNzrdWG0I7VU6JRJA10HMW82xl4jqnBem4I3iUbD/ADdFKG52QAmhbSt7LmlaSpXIa6QN+c+dly77odox6s4GHFE5g4pailCQNyI3205dOqvQ3HUttQ5UlKj3C21OWOijME/GzmufDi6hQWtpa0aEhLkfPbf62tzU9UG810LIRa6bzOHWnVhKn0vKMI7V3MAAdZSkCE/MxtaoX7RLB5uqowNiBqVtVbNdRlzIBJbU0rQHl3yfPpb0EpMOF9SEppwgJ2yDl4nfkLVj/aWYNSrhJhzEKlrcXd9+dgpaRo2l1hZiY6tJ26c7W+zeIEYpGL6G4/Cyu0LGPoHs57qkPox1f7u9JDh/UOr7tVejDEzH96C0Nv8ANb2WoriSWkNtAhAOYlSjAJ3EbTbxT4cOruXilgvEJSoC7r3oqlRAMFLT6Vmemk299Lrwo7VFMoXl5SJ0tr9uYJKmoh4QuSCPz/yuZwCvFFTva421v+E0KW4EKgoWrJoO5oTPLUaD32WrvwsXIbbbSE8zMk2kCnwlS0jIfvB5LLaBqVKAgW0nEFAyr1XCt1OXk+NO1AhtJ/zH8Lc9B2dcyzqx2QdN3H6Aaok+PSS3EAv58h90lXZgNcdpUIS02N1rMe+Bbt2+sK3C56ldFK5fdeNmqdOYA+JGgsqqwrfmISHMUXqtDMz6pTHIg/5jubL13XRc9xMdhd9G20kcwNTbp6PAxEA6JgjHzP1d9m7D7rEnxDP/AFnl56DQfc7n7JlvXBxDxn3b2vRFwXar/wCb0ozPKT4q2Hzsr4a4bYMwco1NBdqHKxft1T57R1XXvHbyECy9U3k2nTOJ8LI1Ze5JMKseWWgoTn/qSdXa+g2CCJamobw2+FvQaevM/dLNReLbSSlOg5AW8I/2m9ym5/S5xZUpSQ3eTdFeCARvnpmwT/xJV8Le1VZeiiCSvS3kn+1suZTfGTDWJUIHZ3jh9DKlcytl92f+VaLTwTGnVmJiNx3BspS0Jhp3PTX/AGW+Lk4a9Kr92lwJRiG4a6iAmASkIqBHX+5NvXqrvYTova3hB6IWJ3MJ+k1w4vhCgkOXyxQrI/kqJYJ+DlvbWprZTJWetsjt259PWxvbs5v7Fa3Z6AVMRvuCpDYcDrLbwiFpChbsn3WTsPVAqbmpXSZOTLPkY/CygRPS3RUcnFgY/qAgzNyPLTySDflNlqQ9ycHPwsnbQmbOC+WO1owoAyhU6dLIBkHTT3W6KlddiyahtnrBpqEgT1sIyrIsKUNiPv1+63Bk6A7cptitQcupIIHvFrgVN6UgsEQBbYPLadCbAoWlaEuCNRmAt2DPTzm1oFVl2DJ/W3QJjbQ2DzQJJ+dsMbREfK0wSUkNm1mBJ0tsKFgcwkazHhYRK4OvLnraQ3UShQfmOtita4GqmjcUDBcLStP5knf3gWHzToCd7Er5JTQKfjVhaHh/uqBPymzu2SG6PLAGsW6oD9tGYb/KwayFAKSeVuqEy/EjU9bPa6ZOHtgnTMNNLZbjtY0z/Xwtlo5U914CjDF51TzeGKVDS611RpQkudwrHdOp3E2lLgZgHEmBbtvS7sQNMtpfqUuMBt4OSQkydNv4dN7NCgvRq78XU9/PtvFCK1VQ4EEFWqiYGw/DW0nJ4tXD2edVBXBJ1MhBAGvRX1tZ4A1pzFBmzOGUBO/snDyVCvPrO0W0hAzQsaSJ2BNmsnilcqT9jQ16hoTojQ+Pe8LCOcVbneWEro64qjeGzz12V4/0td4zeqqmF/ROMIQE6iBB0A8NPLa2EAqKSJkgghQ1Ph+fhZqjiZcSgJu6vnLzCJI5D2vz2nlbj/Sdc0hSaCs7xgwhHPkdfL61tHjNPNNwXdE7MkEpAkgaCJ5bfXWz3uO8F1d1MBTiitr7NUSNdPwj52hb/SfcsEpu6uGwGUJKZ6b2WLl4zXNd6HBUXZeC0rWFApCNwddyLCme0jRWYGFh1UyNr7LveGmnlr9dLBqKe66VLOXnA8/wGnjaNFcecMoUpSLnvUhMn2UaaHX2uv10Bc4+YcUSpVyXnodT9mJG++bxtUuVcu1SgVlYKu6MiYSIAGk6eW3wslYsUHLldAaUkKLaRCZjvjU+8g+7pZhf6esOp7v7kvFUrAA7kHb/ABaR8fvsh4g9IXDNTQrpE3DeoUohRV9mUgAjcZtNh9A2hJctKnG5oIuka/n0N8aeHC0AJSq/aIZVT3pqmjqI93xPjb1gYTz8Zt450WPbvxxxmwC5QUdXTt0t/wBDKX8uZU1KehOkJj429jWMuUqtnSggC6sxkOJIXZA8bBuCUkT4jlFhZzDU8uti61GNRoREE2ePdQek9xJKjGvj+FgHEgjbflYw7lmBpvpG1gHCNSdevS1oKod1GnFvifcHCJVx4mxGwHqR6peowQ6hspUpoqBlZA2R5xNm9d/p4cHKJIDwS5IEA3ixz13BM9bS1eXD/BfEPs7sxvhW6r+oqUmpap7ypEVDSHQICwlYICsqlCY2JsJSejXwFa77fBnBDcq0/wCwabf3otaiY8i7TZAe9l7EKKnf2hHBptJKEUupgZr0QNJ30QbBp/aLcHogJu/XSDeit9dNGPC05U/o/cEWPY4QYKTGki4aX/4NbKlPwW4PsgdjwqwinyuKlH/RYhEt9XfhQvH0VdHP2j/CBoED90CP571dHkdKY2B//SQ8K1hRaRcRSBJJvZ+I6/8AmvutaFrhZwyb/uuHWGE9SLopxP8AyWOtcP8AAbOreCrhTH8t2sD/AKbMRKfi/CcGMclVBX7RnhslIXkw+kHc/vWo8Y19VjlZx8P/AE12eLl+jCPDzCt13xeop11RYavF5IDSN1Fa2EpGp2mTayzOD8JtK+zwxdKdI7tE0PuFkXEeEMOUuJMOX22qkudVK5VNKfZbaazBbJGUlSYiQNOoFo8OXcuUhKwaBqYycccd6lvtafgzd6klZSCb6b5Eg7gcxbpvGHH6deEN1thO83u2f+sTaQ7rvDBeH7vYuX+2NI8GQopdrL0bW8tSlFRKlFQkyr7rcv4vwBTuFD+MLjTG+e82Rr/x2nY295Qv/wCqZNLi7jyCFL4Z3WhM8qtC9f8A2wsrJxVxrfAIwPdSOpKkKAHT/wA4squ4/wCF7cJc4hYZQNfavdiYHjn8R8bFHuKHBptaVK4m4QTBMhV8Uxn/AJ/CwXRB27z+ERshHwoL+0fGwoQBhO6Ukci2g8v/ANatir341LaKv3DcbawJGdsR8qk2Bd4y8EEgpc4p4MSRv/2tTkj/AJvOxNfHLgCwpOfivg9J03vRgg/81h93b859Qp8U/KEkIxfxgqcT3lhdVHh9ipu6npqtwuAJQ42+XQnIe3JMFlc6CNPcBV0PF2tWS/V4XKCIID8Ry6mLN9rjv6PlBxlvm/HuI+GvU67Dd3MesNPBxtTjVTVnICkEZgHZj9LOR70p/RoZlw8T7k5SEtOqPyR9TYJpWOHiefVHFQ9uzB6JNGF+JACQqvwwCFSAKwiAP902AvLBXEqvIKL2ww0Jkk1qlSTz/uvL42UXfS79Ghru/wCkuhmD7FHUE6eTdi6vTF9GlOUJ4hNuFX/d3dVEdf8Au7B9mU97lx9UX2hPawaPRIqOH3FJDhJv7DQVGXuvk6dP7n8LKFFw84oBsoOJLhaB1MOFQA6asW096Z/o4NwP7Z1C9/YuqpP/AEWJu+mx6OiFjs8SXm4BAhN1Pa+RI8LI4bSu3J9UhX1LdgPRavXhpxCTUN0q8d3Il6pQtbaEoClLyiVR9lrAPum1frrvX0o3kmsu/gdip1twdol5D93ZXkkSFJzjY6EaDe063h6a3o3VgT67UXnWBsktpcucrCVbGMx00tN+EKmkvPCl13tQAopaykZqGUqTkKW1oCkSBtoQI5WYYZBfwE+qXf5j7w/CpWh/0vVLC08E8VMKCplVdc45eI5bWjT0hr69ISgwCpjibgW87humvq0NIfq667nQt5ILiW4pxnH92TOg7oHO3pe42lWuZSo130FqQftPb1FLhPBOH2n1j1mqraxeXcdmhtAO/wD61VpsoY43Ai6d1bI9pBXmpfleaipfo9VJBykEEco0iQdvOeY3OsPVqFt9lUIQS1o53u8Y7uvSZHWNTFi150jz7/qxfl9CU5QQSV6+0k7GdIjUiPKyVdC3Wb5WioQtCghasq1FMq3106Hn8RbSGipHVPNAh5KwpQJVBSQO0KVc/foROghR3GhxwsOMJNMlJSheckJBJVAjLGkbkxGknUmyQ68yKY5ohTeVKUxmiQZBJ5jLygag84VKdCHaft22kkvZVgxCpVqTqBMd1MAnaLEB1QyFdHgTiFN58Lborbrap6q8KakQy6pwCUqQMpkJPtd0HfprAssVd4XpUX3RLvOnYC3g8zlQgBJGRRA0HRJjrA15Wjf0XKxdZgN1ph5Lz1FeD9P2mYg5VK7RAkRMhwGfPraWb/rGbsoGUXheDC6x19DFKvLnzLPdEjQ5UiTPMa29ao5jNRRvJ5BUMoa8oq4xTUzLeSVvvqLhStSkjLmkb8tU8to6gCsnpY3O36/dWJkNKyPB6lXmlIJIC0ySY0g+Gm2lrTUIrXaVs1iaUuinCXXmimFLCf8AKmBuRoNOdoU9KO6BX4BcrUTnu91urQtAJIAVCh19lZ6T100p4vTiegkHlf01RGutILKlt6UjrKAtQCm80CE90adR7vL42mb0HMDPY39JPCVMFJDV1PKvh8qVEtsJKoAG8rKNLRXedCqppKgtKW6UDNoNIA0M89Ofh01tLvoD37T3H6UGFRVultNciqo0cgVuMLge8gfK3kktw0rVi94XXqxxKuxuvuimzoSQ3VJV3gVASkjbnrGm3zs1rnwglynQtCg3BBJVoOegSNTsLLPGPiLc/Dfh3e+PL9u+trbvuZLT77VIhJeKVOpQCkKUBoVgmSNAedq5UP7SXgLTJSo4Oxok5o7tJTHl/wDTjW3mfaHC6+qqeJSsJBC9EwTFKWlpOHM4B11ZGkwlffrlMzTqpRd0y6VpVnnllSNDz18rPShuFCkpabBCQZUY0nymLVPb/ah8BmVBDuEMbDkQKSmk+f2/SylS/tU/RrZCUqwjjxShqR6lSwP/AL+3MO7NYxKfFEbK3P2ggI8LwrfUVwNlSQlRjSTl3tDfp38Oq3EvozX8q76QvPXZUUlekIElKUOhK1dAAhaiT0BmwPDX9pF6K+KWKqorqnEFx+rIzobvG70y+RulPYrXB/zZQZ3tHPHz9qBwsvrAmI8D4Nw/frKr1u6oom6ldKypJzpKTmlzuggkTBInrbWwrs9NSyNlkvnB0AFz/gLAqcRkqTl+HmSV504YF13Nf91VF+NOLpaesCqhpqQSkqCjlUecAxuRmt9AuEMXf2uw3dN9YOut0UN50TFWzVVQy/ZuICkmJ1MEeFvnIvbFlPeLv+rsraV2gXJAiNY0HmfibeqPBz9rN6PODOEGDMJYnuTGH74uS4qK7a40l3MrZU8wyltakKLwJSSmRIB12t31TRyyObISR1sLn7dFgPkYW5Wi/wB/5dX/AP7Kt1boqL+rXK5Q1CFaNg+CRpZVQKGgaDdMyhtI0ASItRB79sX6MhScmH8en/8AZtP/APlFkiq/bDejg7OXDOPv/wB30v8A+UWpOZJS37pTku6kXPqULK6SwkeLdL6K/dVe6EJMKsjVd9Eg97a1DH/2u3o7uJ7uGMee+gpf/wAosnP/ALWn0fXdE4Yxz/8A4NL/APj7c/Ww41UknhOWhTxUrPeeFeuqveQTnsk1V7knRR91qNu/tXeASz//ACvjffnR03/4+xZz9qlwEUSRhbG2o50lN/8Aj7c/NgWNy/8Ahd+FsQzUDN5ArtVNepYMkxzt59ftWbjN4YfwPiBtAJpna+kUoH+YMrSP+RZjz6Wcq/2pvAY7YXxqf/sdNp/9/aEPSt9Nbg7x34fUuGrgw9iZmvpa9NU25WU7CEZOzWhQlLqjMqTy5G17s/geLUWJRVEsJDRufqFOuq6GWkfGyQXsqdYPvtzDmLbhxEyqHbqvGnq0naC26lY+63vO24qqbQtkKUHkhSMvMHa3gAXGoOUrCs8pECI8+u1vQ+6v2leC6PB9y3HWXfidytprspqas9VpmaZgvIaSlfeDxdcTmCjJWienK3U9ssFqMWEJp23Ivf72Wd2cxGKi4jZSBe269LsFFaLnNM8QHGXVJKc3eE96D0OuxsvH52qt6CXpE4W47XfjCmw7RVVG5cz9I+8y/TIa/vw4AUlK1Ff9zqVGflFqCTy57WVBTSUlOyGUWcBZFqpGTSufGbgoN9CXmltH+IEWajoKVqQT7Oh5WdiiYjptZuXs0WaxRSNDChHW27RHUhZlUNMyLFcmT8YtmbY6H3/hYMqG+Xa2T1M21AFmk3R2mUexSAT3ZR7gYsLPONTpFidOsHOkKJAVtzgpB++bGAox7VrLdQq50KFJO2sx0tgUDr99uArlI91thQnY2INEyECpgHXw526QSICTt99gSSYAMaW6SoDY677WkFBDBXXQddrc1CQ/TuMqAIWgp+ItyFDUdRtboq52kkuaR3taNlRkkoE+ca2M0J+3G8DlZPu+Ay5TwPsnVpA8Jn8bH6IgO6ybME3NOEAwNVjT+U2ywE/4494/O2WbVOvBLFj1RdVzVFVSOBL7CUqCikESVBJEDQTJ8JnY7POl4L8R32EKTjO6klSAsRSHTNy9n7vHaTZp4+addw1WNpRJWWyB/vp28fz8bXP4bcLsVcRlmlwpc79c5TMhx1LSUhKExuSSAJ1jXrFngYx9y9AqHvjtkVa0cEeJAUD/AGzurLvpSaAx5dOf52OM8B+IjmZs4yuwqA/9CO/w+ptYiuw7X3Jejl1XrTqZqadwtPIdTlUlQ3kdfMWn7g36OTeP7mXfTleKdpIyJzJzZiRsI5DT8bHfBCwZjsqzaiVxsvPRXAviUpIYRjCgccyapRQKJnTQaz8bBVfBLiRSns6nGFG0txEoCrtIPmJjQxHu+Fz+LnDF/h9fT13qqJLZA7RM7bxpr0+Nogryt1Z7RS3FqTlGYyoDkDrP372dtPG7UJd4kGhUCK4M4/UEhOPqGMuXS7xrvO/ntbocGcdOlLTnEOhSUKETdiSNNf5tt/qRaaydZKQJEaazpy+NgVCEgISZUNRMeOtp91jS7y++qjZPoy8THvs1cTbvPabkXYJI+Omn4eNu2vRj4gKzLd4p0kHci6ZEkbyVdJ1+dp/uCo9dulp7tSVtQ0oHckAR8RZWaaQlQCgCI70E6kae/e1F7A02WrGA5oKrafRZxmpUq4uU4MkkJuYaf8402+hZNvv0VMTUtGqre4rNumUpUn9yhJgq5faTufna0ak5kqDigNpBG8yNpsk4uJNxvvrKRlW1ISQZ+0SDHjpvYLxZpIKO1rSbWVJcH4aq8J8drjuCpvX15V34guuHUNBorCnG3CCkT1A3O3lb21p9WwSCR57728a6tzJ6TNG2lQM4hulRKdI1ZG46EgW9ladOVkJ13/G2fMSbFTjADiAuiBrCjO1i7o38NrGzH8p157WLPEkRPO0Y908mqT3x1PnYosD6HKx51IVqoSZsUcAzGSdTztdaqTt0g4jx+3w6p6e9nrpdrvWnxShDbiUFJKFLzEnl3PnZF/8AKhaCspwZUHKo6evJHOP5LJXHdSRhq7FBWSL0SJ56sPWhcrCFGUgjoTP1z+hbSpxdiz6hxDtFYJPpVZDCcBu6fzXkAZ9zdulelo4lHcwGD4KvOBtyhq1eC6ADCoyiD1n89futvPKiCZGx0kR8I6fCxiwKvxXBWEHpaXidRgGnTO03kr7+ysEr0tr7UTkwPd4CTBJvFZj/AO7tABUZKDBIMAADUe/3eNtFzLmKNdAJkb/X32YRhPxXFT9/5WeJCPssGXbmJ29ccP8A062jPjrxRxDx4uK6uHF43XQ3VRXheIW7UUzq1vJ7NpawAFaaqTB8LM5DqjCgkASIkCAI3tpt4C/bjWFjuVsQQebTg/H5+6zujBFikJHA3UeXn6N9yXS+lly/rwd7T2VBLYA5aSnoBYJv0fsMEBJve80iDoFNa6bexvaasbAB+mKz7aTlPUTIizadS6QlNMop111gTpYLqeIGwCK2olIvdR2jgBg1IPaXhei0lUd59AJJj/Dp/X3n6b0fcAuJUVPXooySo+tAAAeQ2+t7OpXrCNFE9yCYnnH6WU7sbqVqLocJZSTIBJ5dP0snQRgbJ2zyE7pns+j1w+ToBeUE6j1uOcRtrzEWFPo/cOwCVUlYRuQatR5eFn22+JHdAB0nbUTYwHhJSo5tYPXw/pYfCj6Iwlk6qKxwZwJS4zcpPUaxdIbiRU9gurWUl0VK2yrcH2SNJjawGI+F+AqGgNTRXIUlCtvWXTmHT2uWnjzs/wCpj+2NOAlQL2HqpGpOvZ1jB06aOfMWJYgbS/dlSkojKmdfDX3bfdZNiYWnRM6V9xqq+1F2YdocYpoauhSiiNKt3Kp1YCSJ55pnQWR10NNX095111uf6vQZ1JVmOZahEASdEgCyvxBCXLzu+7GVILl41CGSJM9mkyr7wIsnXShLFNjChIc+zUslO0pyHaNfD4+FgcME2Rw4gXuuL8TRMYQu69qenCXVuNhxRJMg7zr0+NlDF120jF001TdlKlBD7bi8p3RHekg7ADX39RZv19QHeEFI5lGdCWiJVlg9oNZ5acvEWGxxfjQwlS0/antH2mllKFDNlSBOnL9dLOWNASDyUvUNDSXpX0l5OUCWqBVShDKQTDqTzKehOvPa0gMekTxsurHtfguk4k3wxdF306WqSmYUhKWUpIQAFBIMACIn+GzPerW2rloVtRp2KgpIGUdwxrz57a/IWI0V309+8YMXFXaJdoH1BsNuCSlSlSTrBGgPu85rl7Y9URoLzYJ8Yy9IHjhQ1tyCk4uYkZZqKrsn0prSMwzJBG08ztraNePHEHFONa6hpMSYnvO+F3e0tTfrtWt/ss6xmCcxIAMD4DrZ/V+DLjvFFNU1dS9FKsKaIcgJjcQNP5flrFoa4x0dLcWJaxlpwllNGzUOBagpSlqJAA1E6GI6Zd7RiqGSPsCpuhfGLuCjW9WbmZYFZeTrocHeATEue1oJ13nXpBnqhXU+aquXUKDjbKGyEJMkJBVpvuAfu99hU3bX30+XnnA0ypQCUZ5AJPw2AJ8culuKBz1e/wBxpa0rIZCTlIOWNNCdPERtp0i1y6CAlStIQlTq3G0ZoIycuZ567mSTqCZ9rRQuFdL6g+yttE0jhCyCnKUqAKRJMGCFCdD4kwLJNQ/nRmU4MpOUzOkAAakwZjYfqQ7nr2qW9w0FJyvJLalZAOzcnTWJGo5awrrrZwUiLqyfow34xTLvi7XnlNKqa1DwUteYZVMp0JJ72bLI13OgAm0xYii8b1ZbpQrLRvNLSEoJBOcRkERAI3HkIM2gX0cK+hpb0v5h2mbfdaYZdZW23KQsqVMa92cqToNhB1Gtg2qijuG5U194IJqKlxBKFwIWVARMyImTOwSelvRsCcZ6JrCdAqsoyOzJTfevlbgRTILbTOeQFQVyUgqnfaN4jMN50j/jXR1tVgesoaynQl2qpXVEqCiJGx25k6CBAjbe0rUbl7UIFJeCkhltxZ7VB9mZEQBI/jnU6kbSRaD+OlVijGSqy67mqV0VPTJUkrRIUBGoEQBJJlWsDpbWrv06Z25JFrIAsXKoFI6s04qFZlFsqCgsghMAaEnlJnnv1Ih8ejXcldXekRgahuZKg+zf9NUJUknvMIV2ijOmyEqnaRaPaEKoXaijWQHWVqQogagpkE6eZ36aeE2ehzUXm16TmCn7vofW+0edZfIJORvsF5l+BSmTr0iZt49OMoK1Y9SAvUXiNw6Z4q4Fvvh69WmiRf8ASGj9ZDQd7ElQIWEEgKggGCeloJw3+yDuh8pdf4qVrrR1KlXO2lIEzElw/KbWww/VmjvelebTT5kOCO3MNEzAzdBrvy3i0tLwtf8AiBXaYsxKlFJ/6Dd0ttqH+JZ7yvkLcfiT6lsrRC426CwH1JO32W1E9jW+Kw8zcn7BULxx+zN4F0uGV4duHHF/Xli3OlLRoqZlxAE94KQfZ0nXtBqduVkPCX7GFm8UCsxbxdrLvS5B9XYuxtbgA2lRXA05Cbell20WHsOUwpboomKdCf5EiT4k87Y9fiQD3xapHX9yJdUTX/8AVu3qdf2UZJDMMsTPud/QaD8qgzv7GvhmyiaTjVilpyAMyqGnUNOUaWbOI/2OWDKS6KyspuOF+u1DFO4tptV1MhK1JSSkGF9dPfb0Nqb8PJXlZHr73C21IUuQQQdbZdX2pyG8Rsiw0U0lg7ZfOJiPD7mH3+xW4Xc5WknLlhQ/rPvsgkk762lHjldT12Y2vu63GC3+7r2raWTpOVwj/pn3j3RcbeiUcpmha87kLLqWCOQtCwEjrrbWpMzZwYXwBjbG63GsGYUva/nWBmeau2jcqXGkyAFLS2CUgkwCRBIMbWc7Po38f6gEs8E8dKCQVKIw/VQANyTk0HjYrp42GznAH6hDbC94uGkhRvHj8rZH1FnyngbxmVd5vUcKMX+pAhPrH7kqezzHQDNkixu7PR44730Cq6eDuMqpI1zN3JUER55LMamFupePUKXdpT8B9Co7n6m2ptKn/kq+kh//AGOxt/8AuV8f9NtK9Fj0jU78EMaj/wDYz/8A8Nm77Tf7jfUJu6VB+A+hUWTbJNpTT6LPpGKH/wDRTGI0nW6Xh96bIONOC/FLh1SUtdjvA173DT1y1t0zldTKaDq0pzFKZ3MWdlXTvdla8E/UJnUs7G5nMIH0KZxKOzSQCVGQokaDyta70W/Qww16QnDeoxxX8Qa66KilvV67V0jNCh1PcaaWFZisHUO7RytU4CTA10tfn9mji1LeHcb4UdcSOwq6O8Gkk6nOhaFn/wC7b+Ns7Hpp6aidLTmzhb91pYFDDU1jYpxcG6tD6Efow3T6N+NcQuXRjisvlrEF3NtOMP0iWkpW05mSoFKjJhaxEc7XEUAB0tAvDe+P/l3d7ZX3XS40T5tq08dQLT0sgm3N4ZVzVkZknN3XXQYlSRUcoZCLNsuTtp8bJd/U6i2h5P8ACcp8rKZE6T8bB1iQ9SLQRrEi21TuyuBWRKMzSE0gQRr8OQt0NQAk67W6W3lXlI3+G+otwQRzJ87bQN1jrthSU1CtNFtg/An/AOKxoEakjfexFJipaXOqgpGviJ/6bGgokyNxroLWI9UF267WsBClRMAxA18rETfjYb7Ruiqz3gkJyd6T4T4fUi3dQ3WLUg0tSlCdMySkEqPmdtJ1twwxegMvXmFgH+FoCR7/AH2sC3NQWv36Nhd1aZ00b8/H4+Nh03oolQRRvrCUhSTEBYMbTz12MWBTS3j2SgbyPakkpUGwQE6QI5nTfxtgpbxJSTeikpgZgECVEAbdOvTXytMZVDUJQpKpVQz2qqdbJkpyrGog7/jYxm55vOyQijvI6qvlRBER2YAmInrM62U0kaAnUDXxNmPknC4YHZ1dSkyO0SlwT7wfuFlCintZI21M2TVKCK9gn/aJWjptB/OynQwHdQNdpsyY7pYJcn+P/itlu8q/5j8D+Vsskrrwix0Ut3O64AmVvN5UgR3ivn035G12+DvGzFfCRblbhlVKU1bKG6hmoaztuAbEwQQRPI8/G1Hcfqi7MpTJNS2kykbZtvLzB5xvBtvgekbrr8u26n31N01ZUM0dQsESEKWATBHjt1Fo09g1xdsq1W7IA7pdODEWNr1xhieuxNfjra66ue7V7KnKgdIA5AD5a2lrhP6Rl84Au9y6aZNPUU7hBh0TCo3GotCOO2XbrxZet0VFUKhd1Vb1AhQBnIy4UjyGnzskCsW0vVR7u+5nra+A2WMdFRa/4gpE4k8R67Hl5PXneVSlx15ROZOgHwtHVSorzKKwrQ5v4swnX46726FQt0FOYBRkmDqBtH3/ADtyumdySUgxA8+WotJoDdAkdURX9mVBa0lJ1JmZ1+fPzsC4gJC+8mSDmnw5mfvsO8ypkKGcApOgkgzGu+362Kr7EgGPaVlgiYAPTfaetpqVku4OrS1eDlE4MyKgFQMmM41HxmPfZ8inKgtbYPdCiIEgaSPPW0UtPLYqEvtuFK0rCx5gjr5fK0rXZUoraFmub9h1tKiPZAJ3Hxtm1rcpzBalC7MC1a7BRILghMQCDHlHwsj4kZUbmqe0AUlKm1GdiM4PPx+FnI6krSY7oTJ1O2ug+unKyLiUpbuKtKkKOVvMAP8AMJ+Gvvtnvf4CtNjRmCp3eXZf+UpRuoSQRiC7EjSRopnaNDrPXTpb2UZ9kSdZt41XshbHpJ0a3m0JWu/LqJykAkHsTOn/ABafdNvZlhsqG8wdPC1N+wKdos931WKA3IA8bF3AJOXTrpY4ptIAJEE9dJsXdEbAd6zxqMlkQfT/ABGxNY1knfbWyg8kEa8uosVdGkHny+vfa4xUnbqLePCJwrRKRui8myD/APVOjl52gtSVESpwiecTawPGulNRhBsSO7XNK1H+FY5+doL9VOQ5IMGNOk/QtpU7hlWfUC7kSKHE+06SQZIHPQ6WyEhJK1bEqBj5WNCiKl5QrlIMbDmQfjp/Wwhu45gsZYVHTQ/Gxy8KuGEpOWnUHMesgb23lURmUo68pkedj3qLijlCisAGdJOnh7xZSGCsUuaowxehCgYUKNzaPLptZuIBumLcoSAAAoEOCR46HUaRHXS28sXpdBKlaXg0NY/ilMfOzhbwXiYmFYfvBASI7zKkjbTePGwNXh2su+puk19C5Tl28KbIF6EgOoCo672eQPa3MRomicx7soOqU8ZsNvOUzmZRAQuIMDkdf6WQH6SiYbJqHy2FzMqglQ5fX52kLHFFSU6KPMNcq9zue7zmdvrWzDqqKjrlBpzVKDMAxrEWBmzOurIZlbZI+e7O1ITUI7pglKzGpPTcfjZZux66lMFtypl9xXdbBIBP1Fk9Vx0CVplWm8lXQWV7pui6FINVCs7ZzIlU9I+vO0pDcJmA3Q/7uaUkuEKB5iSLdpu2kV3SkyQCRmyjx0sYbcaQgAqA0mI57zba3QFE5yEgHQnW1Y5lYBaEitXbTPcRcLUj8hiqprxpV5SQcpcpVEAnnAJtYo+jVw7rad6neqb5hYKFZ3kAHkdkTavS6lDGOcIVadOzq69uVA86dK//AHYteqnKF5VpU7KpC+6DPlP4WdhNyEQNadVX4ehbwGr69ivqrvvSrrKdyWXFVzqQ31PdTvFq6q4G4Iub01bw4Su3atzDF80dNUdh6wvOsKp3CZXIVPaI5Hp1t6FpLKKxFO264CpQMEgc9rU44vu09z/tEcEValhBvO5aJkAfxK7V9ESOcdfw0T9BdStqpFV6Gno5U92i42MDldCjRTLl41K0HWYADg5iy/Reh36NtRTobf4X3W8lXdX27j7q4GwSSuQOqZg6dLSUimSlbgNP2ozGSpBUd+tlKjWhitQ00ltDaVbjKB99nyhTK8oeJ1z3VcWLsSYeudopaui86mjaZjRLbT6kITEyISNz1+LYu6vbuPi/iu8jop+haqCDPeKmWzv/ALwPPU2lD0iOxTx/xndAZCnBej7hWhspzFZC9B5L3/O0NY8qv3VxJdfhpLddc1ACc2i4ZbAB96OfXxtgTvzZmq3C0MIcny/idx9r1dNMUzBUEg+O4O+p+pNq/cfMRF/ESqenq/tlsMioCTp3QrKPCM34crTSw6EpyoUSkkSY358vM8udq/Y7rbiuvEF4VlawuovJ+ocWULSFKPeMKHJIMCNJgDTe0MNitIXKxWyXYAmrQ3UpTX7wvmsU3StpDmVLkK1nQRt9QDNidxEVt6POMjszULCUJ1UQkqGnWdtfzsSve+7yvJXZ1CuzanMGhonpPjt+UWPYWYZX21TVEpYbGVRyk77bAzoDobbw1KyksVyUOiG15iqVKAVJEqVB3BIjXWJnxEAUNMyh1Km1r7SJXASoBO55kcyI59NbdVVQqtdW3TH7NRhKQgqBSNJA3946z5K1z3a52SHnyvtFpzNk5VH+Hr1kE6z5SDaQTXspU9G5+maxnedOtHZmooGnWyXEqSpSXCConY6q0HtR1nSxNUmkvq9WaN9C/VKZKgooBObQTsfPQ6E8zAip3CtxN3Yu7OnXToNTSko7QCCCpBBTH8UAxA6nu2tfgHLWAO1YDdMwTnzQSogGQJOqTJEa7jfe3oPZQ54cnmqdUbJ//vKlvC7qZ6vQhvM22pwkTBiCoHLI5eMAdNI1xBfdK8XqOnpEvOOLWArKYHidNDHUTIsttXbfl7FdTTlTFDXIb7FlDZSppABMgiFEkRoTpJ8ij46qbg4Y4bcvK8B2lW4nsaWnBSVrXlA7oMRoB3jEAa7ablTNI65Is0c1ARty76qkXEe628McQ7zoG3dA8KhfeMIUtIWU+4k/LeLWo/Z3cP8ACuLOId441r/WfX8IUjTt3ttuFDa1v9ohTi41VlSmAj2RnJM2pvi696+98U3hX3opHrD7pK8iRlkcgee0Ttpppa2H7NTGDN3cY7ywvU1CUpvu43ENIOmZ5haXAB45S6fcfK3kdeQ57y3a5WlT6EXXpWtJgoCimRGvI/XXp4Wk67cXisuqjqUOn7ZhDmvikG0aEZACJBkEHx008YsTwzfS0Xf6kt3WmqamnA27qHlpT8ki3l/bR0kMEckZ52XW4HStrHuY7lqpaN+5v9p75sWdvmBAXy62Z7F5LIkqgdbdprHnJKQVC3mMlZKRuumGFsZunE5eqlbqsUerVLzEqs3b+xJcOFKMV+K78orppyCQupeCCqOSU+0o+ABtENf6XOA6y+hhfh9QvX3eTiHC29VTTU2ZCFKjYrJISYBCZsSCkq6zVjTbqmLIotBuvO/0xbo/dHF3G9IJHZ349VJGXbtjnGu+zgtWVY70Hfr1tYr0l8Y3lj7G154xvqjpad+/GmansadGVIypCAIMqBhB5mTm8LV4fTleWidlHc297wMPbRMa/cALgcUt3hxCsl6CnGK9uEfEe+nLsulq8F3zcy6YIWCcqkOoWFRmE6BQ1nfa3qbw8xXjjiVgq578rXbvpGq9txLqC0HFMOtOFCoTAQc2ihKSRmiTpbxm9HV15PFy5KZh7s11Rep80SO80vlz1j329XPR4r7wuS7X7lqL0dqKRCzUNIWkZW1rAC8kCQDkTpMaeJtzHa+kbm47R4tLroOzt5Yw0C9id1NzOELnbfTW3s4/elSmMrlY52gR/lT7KfcLKq6plhIbYQlCQIASIFkdy9JTmK9vlYhUXmn2s5PlztwJc9xXVMpC7V6Vn7wzSUka87Enq/2hm/h6WQai+EJIBJ73dPn9ffYkq+0FwIBVOaDyOtpNieVbZAxqWHq9UnUkjpao37Raj/e3B65rwSlS1XdiGnKo0hDjTyDt4lNrF1N9CNUE+AE/G0Fel4j9/cBcTsgEuUSaeuRB0T2L7az5HLmtt4KHQVsT/MflVMWjbJRSNHReVbYAeSlQnvAGbWa9AG/1XZxbvS6VOEIvS5XUZR/EtDrah8s9q13m32N4VDfLtFERGxMjbTaLSn6Kt9m4+PeG6kKypqHX6YiND2jK0pHxKbep4lHxqORvUFeX4ZIYayN3mvU7DV6+oYruisU5AbrmCoyNElwA7+B/C1tVSBsR77UOcv1xpaVoPfTCkqTvprP0Putei6a5N7XPR17eqKumafTpyUkKHyNuLwhpYHNK7PGCHFrghdTO8W5UNNvObCEd3aOlgtSQdd9bbrVhOTerGexfcbKdAZFi0QSN4iLKl+tlLiHAJBTFkjOIiSZM22YTmYCsiZuVxWLARkUDP2iADPUx9xsaUDvrMWJvLzU7gTuEkiORAkfOw5UFd4bK1Enra0xVXLuNASB8LYJ3g624JCjIMQOm1sJOsWM0qKEBAOs+E23MjYx42DmCbZKgO8SOh6WkN0yHCjPSLdJOkA2AnQk26BA269LTCig6x0t9g+f9m8nNtoDofvstUSSHQqeetkWvldI6ge1lMeBFlW7XA8GnU7LAV5yLNzTFL47KBK1T/mtlg5XyHyNss6ivCPFLK3vUaZMKDl40yQo7jM4np5n7rWkpHWbrvWmeYqFuCkfQ7mCQmQkyCPhasV8oS5e9yNgI+1vqkB5SC6OWx5b9bWQCmO0zdoolMqmJkD7/AK5WalAc1wKDWNzANKdPEi9rtxBj2/7+w+4t2hvCsXVNKKCk98Z1Ag6yFFfwJs28xQjNBCZmD5fnYamepmVDMlBSVDvBOpj6H32AcUA+oMCRIKDoIT019/x+NyMcNoYOSoNjytsE48OUVG/XNCrcSlskZsx0I+/lt+trD8e+GXCfCGBbnvDCV8Nv3lUhJ7FL6Vl1BQSXYHswQB5ExsbVYZrw2orQrKoEd7WRtE8/CxuovesqEIQpwkbAE+/rpP4fFnMLnA32RGkAEEIjWFJWpCI906ifAfU2JLSqZIMHY7DQ/rYy48V5SlKes9Tty+uVgXYUpRgERsRqOUfW/usZMizhXn7qo5TI08fGz+wFePbUDtA6onsHMyEgwQFdNev32YqilQOaITsAr6+jZWwreKKC+W1OrCWXQWXCZ0Ctj7iB8LAqWcRhVmlfkkF1JqkrcGYwAdIB0kfW9kLFb7X7gr1OtlUMrJCUnSNP1s6G2iV95EmdQNgQfu+/TrYK9sM3Zet016qu6aisKWVuOZKtbAUjLqkwRp4jW3PyyBjTddLHCXOFlSPFLbbPpH0IzAkXvdJVuI1YOx+7wNvZil1bBIHnO9vIDiu7d49KKjF03O3dTIvO6AKZLzrwBlrMSpxRUVQPLUC3sDSp+yA89d7AzZmNPkhEFsjh5rtSYOUn5WLPIMd2QT1NjkaRqYsA8kkSYE87Ti3Q5EmPNFIzAeJsSVnBnUjf7rKTqVEEQdPr8rFFt9NLXWbKk7dR5xjH/wAjXiEgqTUNKAJ09qNPjaCVuKAJBBUI5ESRayWPcPV+Jbgdue7snbvLQW86so7qgTry0Fo5a4E44en7S7kpOgBfUYHlHnY8MgAsUJ1PJJq0XUasOvjtIy90DulBn2t7Yc6gFrg5f8IP1+VpSY9HzGc96tuxJI/71Z2/3bGEejpi05gu97tQCRMKcM/8utj8RvVN3Sb5VEzailSVJUBE94JEabfA2sXT0eK7wxQmloKaaJdIX4FIsoIyjvF+SkOZz7EezrZpN+jbf5JLt/0KZ1gNKMfd4WsZdzS6G76KkcKlqapm2yUqIEpSATYT421Dw6+yHJSvDbSCyhG88L40D5aXTVCW5knsZkEnQGCD8edmHxSpKukVh128KJ1hz1/RLqVAkB6n6jbU2ts25n1S0sjrnItG/HjBtDiSiuy8Kt6oaVdSXX2wgiCoKbVCpnmi2xUYiZqbu5b91m0eCiCp4zXX8lCPFNKWW6RYTnjPIgkgkTHytGaqvUpSiNRO43+ulrJYz4fXZfCKcVdTUoCe8cigJ0Om3ibNNHA7CHcdcvipaKpUEu1KUkbiYyzyNs2PIxt5CB9VozRPbqoZcUlSQ4CZGuuomRA+duEVTqUBJMAazGv1r99poqeDWGkLZWzXqcZz/bldQoymI0hO+1jl18FcM3nUep0dK6+s94Ht3ECJ11IAInz1NtBlI6RnEHu9VTEjS7IHC6gw1D/eUVJJnURofDW2hWPgZHFjLrBJiB5/C1jD6NtGqR+6WkAwBmr1H3xNhmvRwu1CMpuakWZ1mtWY133tAQs2LgjcF/VVmq6gi9sLOqUJavladejlDUjUcvZte+gqr0fZS7TM0wBQgpK1K1BSDMR4/K1YOIeCLjw3f+E6amoWmSjE9M2+lKyQZbebO/8Anj32ny7ajiDUJo13bQ3KqgU0gJU++4FhvL7WVOhOg0tQkZlkIBV1sZDBdK9Q9i92qC0+opQDKSS6dJ8B52qH6WNS7cXpl8EL7co0B+sRQUrrgkpyprlZo/49Z8OtrW1h4mIW23SLw602lJWo9g+dJ10I1PhOtof464WTfeLsIYnxvdty195Xa7/2LWMsvNmnUHUKJKFHecp1nnYEjAxhJd6osERc6w1U8VDvZVL7eUQVbEAzz5+djV2sMuTVVORKAuAcolR9ws0X7g4pOFC0YvuRKzq4Dd4USdNRKhpY5S0PEhthVLW44u5aVtlttLF1IC23FbLEOQYO4Omp02tF1Q5p0YT6f5RTCLe8FQf0uaNNL6TGJ6pFC6lD6adztFJAzBVOiQJ5Ei0C8YqepN8XDUUbai0bmZc7yohSHHUTm6kCBH9PQDjbw/YViCorsRUNFet6di0V1y6BtLiwG+6CNdojfpau67ipK647tcfu6kqE9g4wv1hsFZyvOGAVCYlWn6254y3neD1Wm2jORpB3ChO+7wqLqw3WXnRFPbtMFTMapCyAAfGJnTf5Wr7UC/l1r15XvW0pUtwrdccZQtZHMlRg+HI8tNBa7XGPDLTfCu8ks0NPTKFOw8XmmwVNthaFKOgkd2R4a9Dan99XBU1VQxSuU7nq7RzEBv8AvFbgRrp0G2scxOthzW5SRuqVe0scAUxqhVBeFS67TXYopAzFZWQCBuY9x08LH7gaQ5QrQpglSASQATBBiSD4yfjZRvlygu8pYapQl52EIbgIKiecEd3UDUkHWOWiHh6sSHgl2YKlSE6kSQTruToPhz2toWsVn8k46ah9Zd7R4BDaJAR3gNNdQdJ18PjFllYCiVaZQkKG6ZBTAPUgQRoTPhsUxiscK0DtUtkgFYCoVqRBkba7CBMKEnax5ASUoZMpaIGQa5SeWWBvCTtoQANgLEFlE3Tj4bXDdN74urn78qkMMMMNraUl/IrOVqBHMnRMayII91orgxhhGjSKKmPrbBCQvMSG1I1J1JkySN94IE2pLf8AeS7vvOhCbwq6dLzayXKdIcdCTAgBZGhKZ3j2tySbG2sSVq1oScT4gdDeV5X2jTGTvR/ATpr/ADJjXXXTdw3tAcKi4bGXPVDdTCY3cVf268dUCqJ1ugYbW+whxLVOkjtFd4htvr4DfztCmOMJXjeDzmLeKOMriw+04fsmaoOuFtG+QNgCVQrwPnpEc3Vf1PiKixJdt3XTd1319Fh929LrrKWtvNb6qhp1swntap5pRyKXqUJ1GhBAmE7zosUXrXrqb9avCsL4LKH3lFX2hQVIhexkiY5ifO0n9shW3Y+Pbz0Vp+H8FoLTundxMw7w9eZvC9sMX9eVbXUykEKcok01O4galSUFSlxEwSBOmmtkfgXxE/0XcV8MY3KVFu6rzYdfWCqfVpKHwE7HM2tW4nQeMtKvw5iK52FGpYcbQYzpQrMOe5GnXn18bJLSwk6xbBqp21bs4aB9FBoLF79NOs1KE1VK8HG30hxpaVEhaSNCPAg/pZAuy7qoXleqgclO1XKV2ijlQApCHDJOg1WdzaKPQk4kq4k+jth52rqA9XYezXBWKzEmWMvZE+bC2Z6kG0t1uBncS3sqprL6qGqBBT/qzaiO+EiTJ2+HO3n/AGsp2y0dnm1iux7NzGOpJbzCGvDHWEMPhLT9W5Xvo0DNImRMfzK0jxE2j7ilxJ4pHB1ZfeE7qcw9RUICnHw3mcWhSgkd9WoEqB7oB0tMFxYNwnh8A0V1sl0a9o4nOqfM7W3i3933pctXdNdStP01S0W3GVpCkqBG0Hxt5zEKencMjL/Vde4OmNivOq+nr7xa469fN6Vt51rpzFbq1KJ18dfjPjbrDGCr2ui86e+0kNGiUFgCRCVaHnyBtL153BRXPeT1PTUiGktuFIKURsdPh0/SyVXU6hdlU0CEq9XcAEazlJ0/rbtoA3J4RosmRmV2qq56TOC/7H3tdtI26p5NZdhdbKuudyR7ifnatlXlNQsp2nTvZvna5/paU6Lxw5ga/SkrU7SusuKE8uzIBPLVRtTSubKXSIjSD5jT8Lddgz81OLrkMZYGVBsnNwgvL908UsKV+bKG73pUqIEylTgSoR5E29acA1HqSllIIKgM0bbj9fhbxvu2tdoLxpaxpZSumeQ6gjkUqBHzt694Yr23WmH6cqyuN5wJ3BAItm9pY87W+a2Oy0li8dLKWk34exCo0MJiYsBUP1q2PWgw6pkAEqIMfHazOq6q+XrtfcuHsXK5KCtpt0ZkLIkhJE8wPKfhZHvG+8cUF4uKpahqqo/9vV1BZaUhsLSp7sk54WjsfZJH+1BPsxbhe6i9gu54lgnfVXkEyQRIEExuPfYj+9MiwpB1Cs0A8/H9bRD/AGl4h1VO4jtmmiw6puoqHHqYN07YeMqcIkocCMkgiIUo5QQAUh7FPENbfb017XZ2IabU4XKilKQpWUqCCFbDK4BOsLmTGlttJYIJnCma8azK+4BlgOKA8ACev1rZj8UaYX7w+xPci0hXrt0VTKQT/GplUfMj6i2sP3lfVTRLqL3eLy1vqNO8S2S4wQkpV9kcn8REjkATFh36kFDjTpSUqBzK3O3nqLGhZw3gjkUCd/EjLeoXk5e6Yqg5/wB62hcxEmNfPWdbLnDm9E4d4h4Xvxz+7pbypKhU6d1LozfcbEsXXau6b0eu10ELon36VUjctuqFkrO4E07wJGQlIM7EGf8Aqt6hYSR26heUaxzX6FeodTVBCnElSd9YInrqOXONbXi4L3qb34W4arcwWRQJYJmZ7Ils68/Zt543HiD99Yeu2+P4a2iYqTrM50BXLnJ+Pwtd30Ub6/evChmmKyTdt41NL7iQ6P8A8Lbh6QFkxaV29a7iQtcpiW4CdvjYIFIVoPjbpYUFEczYMq33HytsNWM5Fb3bD1MrmUCRFm2oiDuZnQ2dSxmSUnYgzZsPo7J9SI1m2lSvuMqz6pvxLgZTuBE7T7rapyoUzQVqoJynxI0M/C2gSnROkfX4Wxmci0ad1Zj394ffbQYs526GS57/AB52zMVAmdOnO3BBSMusb72TcU3hX3Xhu9rzupppyspKF9+mS6CUKdQhSkpUAQYJA5ix2tubBQJtqlfVXI6HnrbCZ1B28bUmwN6aPE3FeMrhw5XUOG6SmvW8qWjfebo3iptDrqUKUnM7EjNz6a6WDxn6ZnFPDuJ72uW66W43aOlq3WqV6pul9h15lKyErUguSkkDY9DttbSGFz5suipe0IcuZXeQogga/dbvMTqCNrRd6PXE29OK3DWlxXfTdI3eHrVRS1ApkFLYUhfdgEmO4pM6nWfK0mBeuadrVJGGJxY7cKyx4kaHN2KGIBBBOn4WM4bJVTNJIMtEtxG2Ux+FiYUNhPMGbD4c7tXVsSDldDg/3kj8QbD5pynWA3HtK9yv1tlszDnr4xbLOorwsvFXaYiw4JVlcvqkBT4Z59/9Da1eH+GeNMUYavbF9z4eU9c1xNlyvrFvtNtsjLmiVqGdUa5UAq1SI1AtVQFbuMMMJkEfvmnJzbEAq2/S1hRfF4GiN2+vVKaFbgeXTJeIZWsCErKNirlJE+O9npAcpshVRFxdGG6tpKx2uciYGVUEH6ixiorbsLY7FnQiZWSZMb/H8bJBUDrmUYMHkYOlhmUodKc1QEQogSnSIGvQ/pa6QqjTyQodbOZKlDQSlMcz9ffbSVlRMwDEJ6+7x+utjNRQ3Y3SreYvVLjgUkBIGqgQTIPh+J8rECiMoBCu0GwIJ30sg66T25U9qDhfjG98BVXEq7Ll9ZuKgdW1W1DLzalNQBOdrNnSnvA5ogAnYWZjry1AlZlJOxMg8pH1ysZo6pxumVSevPsUz5HbJbXKVBJkZkgwqInXaycpyFrzIBEEnWJ059LM24vdLQrvMZKiSDMmDvHn9b2CQ4CvNlBKZkjcnkfrxti1Zs06lRGXu6H8Im2kZcojvCD9T0tI6pWI1U7YWdF63LS3oO92rSUrlMnMNFfnqbLdddtSm5quW1rIYUCk7ER9eVmTwOvBNQqtuN4pzpAqWcx1UNlgf8p+Np2XcD711vuCQosKUJgBRAPTf9LcPis/dXuYV3+Exd5ia8Lzc4xsPp9J2k9Z7y03hdRKYnL/AHR1+Xu8pt7CUgHZJHKLeS3pH0iaf0r2xCoVW3OpRUQYJDUxA1Ezp4n3+tdKMzYG+nysSF+eCNw6LOqWZKiQeaEUCRziJ05WLupBkEbWNGCIIsXdAPKPfazGqkiIPJ3PP52LLBEEifPSx14abb8osUcQCSP62uA2Cpu3WqFkPXiw2oCFKj5Wcn7qSowgRz3slYdYadvqmS8FLBKoiAZymLPpFIxoMhKyYiLU5qjgvstWitw9U3EXTMCdRrvYYXdklWcCBFnA9dyWnMpSZiSI2tyaFKjog2Ea031VwFh1Cb4u9PenIqeZBsa7JP2aQFdxMaRt7zZV9RVICU2i3HDmK6fGdBddy3BUVdHVOhFRUpLiUsiEzqCEpjU67xG9iQV0mfLG3MT9lQxBrOHmUiIQkKBUpKI11KPwmzc4lJaqLgcCVJWpLLswoGO74WjnE+FMcLSpV0IrHFKgR2io35DMTtY1hK5MS0VNe7GIbqqGW/UwGnXUnvq70jUkE7W6uaiAoDVF4Bt7vNcxRYm+SubTGIgX35Jy32xmpaZZbnOAdvCwD+FL4vegut+6KV5Y7TKtbamk5AVnvqLg7yUxqkanNZdqGE1F20ZWAqUIOomO5ZzYZShOHWApWgcUkHwm2A3JVgRuO2q36uLLHqmHiHhxiFxeeidWoBKgA27oOYOWQfvtrBWGcTXDe5Vel2OIpFMhIqCtJ7+cQDBm0qJS0AVhC1eMH8LcXqibtT2QKSl1tUbfxCd7dGzEnx03dgBlXNtwqJtQJwTdNdLyytQXUBWXUd//APOtwy8DW5jVNhtAkjONPnYk/RVJrHUU93Z0CcpzkSN9Zt0xRrbqCp27UJAjve1+lqBcStUBQJxre7XFLIU2kCkxRdbydBpmqW0b/wD1nztO+GHH03Ddz6GkkCmSglRG4/py6WhDjDTmpxZeVIgKzt1t01MRoIraUz4/xWmvC6S7cdHTrCoZcdSvWIKHFJiPMfK2XT1YlqXRX1H9lp1VPkp2v+iVKt91xoF8tMgHKftNz8LRP6QLbBurDFc2FOdjeRBcSdEkgaT4wfhaXLxS0WW0qfcSdVQk/kLRpx1b7DAFNVFC3BTXmyvMqZEyB7rPi1+6PI5IeG/6hqkCrQczb6l/3qQQYnkN9LdsKQYCnFKVoBCDH3WApq0qoKN9DpPa0zSwnswqJSNjY41WVK4IQqOoSRra1GbsBVaQWcQo64y3RmvVpxPsv0yEqCzOska/PS1asH4fFdd6qAuNU4ZvKuaUsyUpAcJkgRrCveSbXJxvdibzTSvrpVnIjLtJkEHW0A4PwA5TX7iK565ioOS9XahpaUz3FpQobGNYjXnbzzEq4U9fM08l2VDGJaeJM1zCtXelz1VCqkYdaVRuBTTmVKXUZCChQXpqNIPWLecDSnqYrWoO9jT/ANy0lKlCOUQPADx13Jt6/wBVgKnWFNrYqlkpnRAmPEzuJt5U4twjWYZ4g35gdmlqnHqC9ai722A0pTi1B0pQEpA1JkQAJMjfSdns3iUdW57QdRZZ+P0xjDHBQ/iq76h8evu9x5HflAgpGndOs6dfPobN26socCSlI27xBEe8edrFYp9F3j7U3Sa1zgziwobaLsLup1Dkc4bKc51MwBPnFq6M07rFQ7T1QdYdaWULbWjIpKhoQRyMyCI/TqQ9rj4SuaLHN94WT4u9aUNJ7sh0FYSVZpSrQAjcmDoSNvdB1Va1TsIeqVko0Q7vGpKYgQdgDzOh9yJd9W6+wGkLLpXlAQO8YgbRvOnjqNZ1srU+R2mV2ri9ICpSUqjmJIHh4HveBBDtdDG64p7sujE9Yy/U09RVpbbDQyrCCoiSZ6SVSdASZM7gvuk4dYFBaqV3cEIBSvItySUyDrpsdevSRANm1hGlpW1ApaDebMstApgJEHUGe7toeg5mLSM1RKr6dxLVQ4ytSCEpSJlcER3dCJJ1/ITUe7VW42i109KzE2JmmBclTibEVRRJRl9UevqrdpMsD/YrcKCNAfZjwEgFrXlU1tSlVMzTodfXKVlxCSkEADSdxPvAMgawVpVCxUZnF0q3FK75Qaggie9B1mfZBET3T71+5bhYu/8A199hDLmUFAbKiEpTvqdZjXcaEbmZotj8eiuF4DdVD2O8Fk0tPdD9a7U11e+X3VqRo00kZin3lSZB5DTYi0PY7wevDlSHUCW1mCMuUDoRrqCQrXwtYzEGIk/vOqbuWlF43s6kNgAgopmwdSSJynfnpGuoi0eYk4XX256xeN7XoXHX0BbrSUjKnSDB9+mm0baC16N1tFTkZfVS3+zJ4mKuLiVfvDKsf/1XFFB61SIUY/1ymlUD/Myp2T/gTb0raqjTKypJAIzDSBPP8LeJXAPFiuHHHjB2JV1Abbu2/adFQudOwU4G3dehbUu3tXXNPdokMtrXlJCoExEW57tRFnonn6futrs3IG1jQfP9koKvEx3TJJnfn1s3sQ3oQ0pKjoNQZ36/jYRp0rdNNIzA+yDKp8vzs3sZpfoshcQtPaoIlSdd/wBRbzKCIOcAvSiWsGii3FzQF5uupJBc75HIA8/hZoOvMZ3GVuIyuJI1Me77/efGzb9K67r6vjC12uXGxW1LzVaha26VtayUlpUkhPSB4CfG1YF4Oxo+sJXhm+1KCVJCjROkHXxB3/LS3dUFG2SEOc6y5DEa10MxY1l1MfH5lq8eCOHa1PfN3XkunJSSYQQsa66juga8xalt7oW3UqDhJUFKSSdzBi1wqu67wHowXndl4UjtPU3Re2cNuJUlYBcTuk7e2dAOdqj4iaCax4hMDMFeyREjYj3D4m23g/gzMHIrAxe73NkPMBI6SAqSSdot6ncIbyF48PMKV/a5lVFz0ZUSdz2KZ+dvNDCmBMWYzZvFzC9zLrxdbBqKvItCS23B70KIJ2O08rX89GK/EVvBHCri1KUpll1hRV/D2by0/wDhSkRvFmx5odCCORVjs68smcDzCn27a5lBKHKgMBSZQ9knKQDv4QT8rM++8B4Vqyum/eVTUsuOPLU0lKUtJLq21OJAKSSCltLcaAIJAG0HVXgrtFIGXdQ3BB8/nrZNfrXHElOeRMdJ3Ma7D9PC3HiOxuF2xm0subnuOhw8p5TNU+92zaWk9uUkJSHHFxIGveeWdZ0IHIyjHCNzUzjLtNU1iTTsllAzpMBTDLJJBSQZQy3vIke6x9yqQYSVbQM0gn4/CyXV4iuikKzVXhTgMhIWC4BBUYEjoZgeJsUMdyQXStCUqNTV03bS3bTKcW1SMJZQXInIJAkJAGmmwiwb1YtQJCQOsjvEzHWbM7EnEu6bkHq6E1dbUtPFhTbKJIWUBUiY0gk2Srg4iKvi/XLlqLvcpklbgacK8wUps94EEaEDX32kIHWzWQHVLb5bqnPHy60XZxNxMw2Dl/ertQDyyvpDoH/MfjaPQqaMI17jkjwzD/8ANFpw9LC6fU+IVTWhGtfQ0lWVCd0lbROvLuJ+ItBrWYtupGoACjr4x+Nu+on56djvILzyubkqXjzV3uCN7qvLhRhx8rzKbpjTqE/904tsD4JFrw+hXfHbXXiq5Fr1YqaarSJ/nQpJI/8AZpt58+i7V+vcMCwXCDQXi80BMQClC9+Q75tdj0Oa/wBU4h3rdpOlfdK1gciW3UEH3BSrcvM0RVjh5rqYncSiafIK4qyTpHhtzsEogEgHTxFu1HQBR8NrckSJO4EG18bLOcgzptsLIV8NZX88ABQmy6RH8PlpZNvhouMByYKDNrVM7K9Vp25mpECtRtvbQJQ+sH+LKr8PwtsEjWI0+VuXTleSqVd5J1+H621mFZDguys+f3e62nEIdbU0sBSHElKgeYPKwZJMkE/CwiTm0J1262O02N1A6rzUwPdbuFcb41u67aNmoxJcNHWt3K280HIqWKlKXFNoUIU6KdL6k7kESNQmyxflK/x0uG9eIH7pormvqirspdqL4cNPeCFJW6622at5QQtnVYQghORRTlBCQXPxXr8H4L9ILElJia4Kena9YReVHeVKy+qr7Z7s3ZkPJSIKnYKQNUpEzJsTxFxi4ZY07BzFar5vI0zRSy3U0IeDaiolRBW6dFdySEySCSTpbr2vc7LIByC51zGtu0lSv6Bl9B7COJsOqOYUdezVtidMrreX4S1a0yTrBJtTf0T8c3JUcXr2w9h+7qWkuuvuou05TRpYeccbLZIcykgxmcjwtcVKhHdMDlH9bYWJDLUE23WrRG8IA5IYHmBFh7nWpN+Fue69TgjzSqP+qxUHcETpuR0t3TrLd60D2aE9oppXhmSY+aRairaeBWQSA2rToB+dst0Qf5h/w2y0rKC8LaOf7X4WbSqc15tqkmYTlJk++0+BtfZgI72YyVCPO0PcNbmpsR8Y8EXDVkhmrvQ5+zVBIS2smDvy15ee1rGY1wZeGCq5FIuoRV0r5Utl1RyqIEaKE77Dx08rBpaiNruETqVOopZHs4oGgTWKVJM5NJPLUHlI+vnbpCVKJQUkgaSOY15/UW6R2qYJG41/xfOetty9yCSD3Z11Gv4eNtJZpaQu10lY0pptTYCn2wtAS4DAOxMHQ848+tt/u+rU4hkkZlQQUid9uWp187cBb0ZgogDUgcunl+PjbpT78hXaGABqTJj6/K0EuS6coWmnVBSyttGsoSSR+E/lZWoMLU79PU1rd+XZFGwip9WeccbdfSokFDfdIK08xI0IidrJQeeyZS7qDBE6R0+t5t3mcy5+0He00HXz8J++zXKkwDdF/VXVuHK0pMKn2dNbdtXe+tU9ivvK8pPhPn/Wwmd0H+8gDunXYdT8/iLGqdx4K7ripI1jSInXp9G0XPsEVrASnVwvoL6oMY3VeNNd7zqUVCUuhJAzNqGVXxBP32vrS8Onl0hCqcErTAhMR7rU14QXY/VXwy4/fLlOG3EqVlGigCDB/L6HpTSdmqmaUgCCgEfC3I4pQjFanIXWsF0lLVy4bTjh/EV40emnhR7DvpetqRTqYQ47crgVObOe6JI5bK/4bemtOn7MJ6DkOVqM/tK6mjR6UGFWmwA8KC7M8AAk+tOkeegA+HS16qdMNAxvZ4IhBEIfl09Ez5DMeI7c6rsjTQzqOdgHQrmNzYyQAnSxd3RUgz52txoEmyJOpkculiqgORmeVjjwJBg/CxZWnKR0iLWhsqTt0tYLol1F/wBMAkkJzKPlFpPZuSmYcD7aPtB7JKiQPjZgcOnUt3+G51caWn7j+FpSsPu0cri526mZXMGVpSebtKlZlLBJ5xbf7sT/ADfKx+2rIUEPRR4z+qTlXUM2YLIs279o3Ke8EoB0WJnw+hZ7WZ+Mkreq0MIcLZKU99O41sanpY4X5mpnSueLORFDD5khf/MbA3ywwq7FNVL6EBXdClGBJ0H32GZSkaheb/Ny+VkDiI8lrC7z6icrbrStAN8w62JWXdA8A8kSlIErT5o3WUNPR0bNK64gdkhKVGf8Nhbpve6Ke620KveiZCXCSFupGnvsycUYuSjDf7wIkuNsn2huUjb42g/+1rt4VaqTs1JbWVQQqduX3Wy8MjeHZyVcr5gW5Fbqgvu5bzfNPQXtRVTqBnUht1KiBMSYGmpFkS/+JXD+kZqKB/F9zN1dK6hLzKqtCVIUlwBQIMajWRvpas3Cbig7c2Lr2W/RNNs09K4yt3OZUe0bUInwGv4c6t8bcVJr+JuLqZKHMwvm8WlgBJzD1hcRrpztYkxBzah0FtAL3VdlLeISL0lVxV4fpa7Y49w6lDmYoJrW06DnqbJTnHfhC2ShXFLDSldG61C5iZjL5W8h70oO2qCUERIJ0MfDmYmJ/OythdbtE620llspdJylwDTSO6T9a2aTEntFwAnZTNcbFXdvPHN1439IfEgufECLzuP+zrVVTqaI7EOsOsLVGkk906/mLTvcuOsPXbelfhyovkN1jV93gyGyy4oQp9a0gkJyjRQ3NqEcHa5268WXqUENn+z95pMd4KHq6laEHXUD4/G+PDSuFSziq8W1gqexC66DodFsskffbJoXSOry9thcFadU1vdAHa2Wrx48cOqSodbVfF7PFtICBTXHVuSfAhuOdo19IHjRhS9uF1Vdd33fipdY7UUz6FPXHUU7YTnT7S1pCRII3PO0+BbQcDvaLDmqvaB1syvSEu5F48DcTLb7RTrbTb4UqVDMh9CtZMAaeQt0c0b5InNcdLdFjxSCOQOaiOHeIt4MYdutit4XY6VUUtCyy8TdraE50oAJlTg5iy9SY+vSpQlTPC7FEFYB7RVG3kB0ky/pZ0sKdqKSnUFEoUyhXdJ1lIOuhsKGCtktFgqSSJCl7+6LRax4AAck6RpNyE78EIF53A3W3ldDtG84pYVT1Km1rbAMQShSkmQJ0POxS57suR3Gd+/6o0XXAw97MT9mEz4+zYvS38q7WPVpykoCgmR3dI/C0f4axg7RcTK9TrxKKi72gEk6ZkqXr12Atw+L4pTRVrYZGg5TdxIGv8utakop3xPkYbAjT9/7KblXFdJkmjb11Olq1VnAnAtwek7fnFZFxMuXrX09EaVSzKadwtOIdebSRotYbSkkawDEZlTNasdojQjpvaO+KOMW6FNPialby1LaFUxezJypBIUnMD7R0WEgc1e4zjxrDpZWx0zcpPMC1/JHoqGpZJeYXH81RPF9+12IcMX7T4NBqb8oO2p2WWnG0qTUR9mkl0pROqdyE672rB6TfoH3xxa4S1fEWg4WJu/izRULVVUuUlZToF7LRBeQ622tTa3CnNlUIUVBIKiNLWTu2+mMRXawzclTVqr1OGrZKHWwipZXBWwhSicqgqVpkEb6idJZurEdcu5KVF40LgrOySHEOuo1UBqQQTPLlbooXxRXcT/OibEGyEBoHlY/uvnBpVusVZaqEdmtpeVSH0/aJgxGXeQJERzMxNluirHXstE2mFKXBQlsJgkbzy3jXkRpOzn9KG/KW9/SQ4kXrTerIaqcT168tMhSWsweMmFEyoqCieWYqIgaBpYZZcrq1LbKApLac5BB1IGgE9Qk69NdptsB92By57JZ+VSNhu72WWUCqqEBoASlC/4oJUdCRqJ6ezI5w+aaqoqWlXVKUlDQQlkpJgulQBGg31JjSN/ez7mp1LyjtF5MwS6lRSEpRsN+U6g9DoNdT6a5V44gTdbCVrYokdstG/fBImY2Gux5DqZq3VkaJ60RqEIaebIbU0hpOQAJDZA7wmNgSfAkaaRY3iC8W0Xe4q8L2NDRDWurguFBKQO63J5qOmg0g6zIbLTl7NNLF2XJVXgtxa0MttMyAoFW6v4B8EiSQoEgWaF5Grvy/vVb+CMS3q2rPS4du5al01MJ9upcSCDB/gSSesa2ZrFJzwnBQXneeImVuYFo0YYw+2vK9fVa3nqahRkANiJBIPtEaZtxtZJxkL7ujDt40GErqQrtEOF28at1NQ/Ut6lSmiNBok+UGNdbO5N14+o0NM3veV9uZEBKLvuW7mkU7ImQgrdCW4BjmeZI11HdpXFJdvC9MP0tDQKKUupafVVuyBqXEspU2mYiSAJG82TrstZJnjuCqdds72/rGcl3Nnzc80zNvoF9GrE124qwzh7El8sMPJvzDlFeJ7RIWA6tlpaonbVareCuOruo7sxTX0920L1JRF0rp23TJCD0I3EzG/nb2B9FS/3mvRu4d3o24uWrgZYzeCMzceXd+QtVxYnujnsFyBdSoI884iJtc2VxqLDfDe7b1dvunuunTVOkklSipCSd8qCcqfcBaLPShq7lXh+53LuYp0OCrcQpTbYBCS2dJHkLN1eO3yDDx32n6+hZj8TcSO3xczTTjshmpC5JkbEfXu6284irp5rQuY0A9Au3iwcwSCd0hJHUqOLweDtO2tw7oIMiDueVkJ9BSolSArLoPET+Bsoh0u0pQQqW3VxI2mCJ95P1pYg+2VLStWqdoT49PrrbYiuBZHlN9UwMbUSarBPEW6gkSKduuAg6kIzT/wAlqE4jbUlxRJV3kpWJA6662v5i2+7iuWtv27r/AK/1YX7h91llJbWtTq05kFICQdgofRi1Br/RmJABzBGp111Hu6/K3QYOTc3XMY1Y5bKTvRKoLwvviJVYZu+vRRuXnQFGdwdxUOtkpV4QT9+1p79EaocHDy8rkWYVdN+1NLBABywhX3k6edqbYDxcrBOIBfIo/WkFhxhbXaZCQsRIVBggwoHqLWk9DDEH71XjlKW3GkuV7NehrNmy9r2siefsJExrFrmKxXgc76Krg8uWoa36qyt6VxobvfvAI7QMNqcKQTJhOYjnp+u/OKXOJd9OtuuKu5NI8232hBM7OJnfcFtYM7zNpWvSnNTdLjaQIfZUiPAgjTwPnZm4IuHB9+UqU30lvt3rpqiFB7sz2zdOS2TtOrZGXmRzNuQLmxNL3C9l0WJVZpI+JyCiitrcUX1dr1det+VgYYWioyJGQBC3uzUiQP4QJ/rZPvynXSVF6pbecW6aippnVrWZUlCEutz0Pd6fKz4xoxddC9fdz4feD121NO460M8hDiUhaoVzGk/U2Z98OGpvBxQImsdoX0idFdswW1QB8NuVr8JD2gqlHPxWB/VEm8Sl28HL+qAVBupoqpzTUgNlCwI0mT/WzjuasZXjynDTSm1OVq6pHad0hLrCTlMayT9GbNld1N3bc9JebLi1mtpFVOUgHIptbRKfHUKg2cD14C/LyexDc9PVlCU01QkFogqUhZQoaTOhB8uXKzyDTRGYTfVMT0xKMKvK461tvL6xRVVOpQBglpxLg/8AGr42rCx7SkxMoVa3vpZU9PV4auKtCwt1mvWyRm72R1pU6c9UJ18LVEpShNUguju5oPltbpMJdelaFzmMNy1RKsn6IV4lV34nukqJKHaaobRJ/iC0qP8AyptdT0ar2N28ZriClZUVSX6RwEiTmZVlHxSn428/vRUvVdDji8rvERW3avQjdSFoI18s1rfcL8UqpMZYWxEmnfpUNXpS1DjT7fZuNo7ZIUlQkxpM6++LY2KMLKvN9FtYY8Pow36r0tMjTTfabcBY2j3W7WkSRYMyZ10PysduyrOC0TCtJ1PhYCob7VpaOosKfvMGbcq10jcdbGjNihOF01lpyLj4zYs+uFsqJH95G/UH9LKN4tdnUGdcxneIsmV8ppysT3CFbHSDP4W2YzcArHlGUoYKiJM687Zmk/kbB5wdj4bW2FgiQDvqbHCCmhizg5wwxzexvzFuDaG8rwS2lnt3s2YoTsNCBpP1pZPb9Hrgg0YHDO4p379Pm+8mz/Jnmeu3O3WYEyCD0FjCaQCwcbKBjYTchNfDvCvhvhK8G73w1gi57srWkqSh+npkocSkiFAEa6jSzuziQTuTYFMyQDttboK5GY/CycXP1cUgwN2Q2aFQRqTuPrW3T6+xpjUwf9XUh4HwSoE/IGwSTr15AED8rDsNCqbdpiow4hSYHORFlyST2EKAISNddxbLItDfVKaKnLriCvskZiTBmBMi2WV1FeP3o+U5rfSMwIghRSmorHEk6exSPHTx0taf0gm+zVdcqUkqW7qkxO3S1VvR0ve67h9InCV7X0+lmnaF4ZlDUgKpVpCo6SqSbWo9Ie9aSrq7laoFesUzrLzwqmyC1qQIkHQwJ8iPK2Ixru/sNtLLZDmjD3i+t/8ACh9IX7KBBBkQN56/CbYEjMQolREA6fX38xbk+0CNUwTrt+mltLcWlTaEtjKYKl5hpA2321j426XMubcCdl2kFRJJUQCImYEa/f8AW8uzAnDDG/EapepMHXI5Xu0wSp1KXENlCTOpzqE7HbrZqNZipJJM5ZJI5gHTl0+tbTd6NvGC5eFWIKt+/qGofo6+mU248wpOdtQ9nuKgKBk89PGbVauaSOPNELn+XVeXOwXaE38V+jxxUwTci8QYkwuqiu9jV1xb7RjXbKlZOp+4b2jpTSm5JM6wdYMGfr+lrVcffSdwhxGwa5hu4aGuQ8taVhbyk5dCJkAnqfhaqzzrmVAbLamypWckwTtET7+fusGlqHygl4t0UoHOkLtNOS5SFkglSeUEjc/h9eNjdIUthJMAzIEjX8T52IHvJJkkRp9c9PvNt9u6xXU7Xao7BSV9oT7QgCPASJ38LWHG6vRgjdSPh2/F3U2FNqGZSTEbyIA+voej2E8Ts1OGrrqFrkuUbKyZ6oBt5QNYlpkP9mVkIKAoKC9zMbe75i19MKX5e6LhuarReTIu5FzNIcpewlxb2VGVwOZtAAFDLGs725DH5pKLLLEbHVdFh1MMQaYzyVF/2it9euemRdqUOEIapLmRy3DqlDfzOvU+63o9SkltMjcW8o/TfvFd4elqw+pKVDPdCM6lRslBj5/G3q5Sf3SDMyLFpZDNTxyO3IuhSx8GV0XQ2Q2w06WLuA6zNjPKEjaxdyZ9m12NAk2RR3TcTrr42Jr31A8NfxsddAiSqDYksiTvta23ZU3JawY/2GJKI5oCllOvOUkWl31jxtC2HXezvyi1g9smdfr6NpUD6dy58LU6ioMD7DmiRx8QXSqalI0zC2jUp/mFod4nUnGKuvZj/R/f130VDl+17UALnnMpM+H3WZ7GEvSVfWDU8S7uQJByoTBIkaSEdJ+Nsp2MyNcQGo/dNLqyJqQP4vnYnXO08turbbUpSwmSkEx52gqgwDx4NWp68OKbC2IP2Scw321y8rJ94cMOLTRbdreMlQ6jMEdmAsDMrTNoRsT9bgT8Ync33LJ20tzaysGp+kCNGWx/ui1esb3zUVN43pQu1jpaTUOBLRUcoAUY02sewvw8xtc1+sXve3E6trmGgUqpghWVWo17yjvEGQdzpZj40XWVN9XyKOhqHFduuDkVBM7gjlE2nSV76guEumnVWG0xis63NJGM8d3XdmGGbuvEuNBLTQUsxlToNTroNN7RZX3t6k8aynQVBIKgoRCkkcldI8Om9lnHmEl4muenpai6Hq1qqaTTVSFtqIXPdIOm28zZiX/g/GFEf3BdKaBmmQ203ShVSWWqdpDYRkInMDmIAM6xynS1R4jG0lhOqsS4aZrOCRWeIzVff980t1OJhmkfFWAUqOdCM0KAJymNYOugtE/F+voU8XsTUorKf1movSufQyVjP2ZeUVKjoPrY2dTmBcTNC9ayguBhip7Baq9+nvN1IcX2fe0CeaREK8OmpXHfCjGlTebNY5hJNTXN1Sah+obr6p5VQg6qTnU0BCiUkmd0iwH1Eb5y4m5VxtCWRBpNgowva96O7KZuoqKpLLfaNoJGuZSjAAjXf6HJfuPDF8OtuXulS0UqT2pKlFZypBKsqUyTok/E6WdNBwlxzfDjd3XZwifqKhDgdW6ahXsJBJnMkBIBiNOnnaTcK8D7+wjR3fjC8JpHqNDjjt2B5cBOZDYDpMZ9V5oAH5wmnDv049XdFXdTxxt4j3CybnDbD1et2nxGwAq77xuuqLKnEFDvZu0zgBKTqJkCCJtdXgOt1zB941TicwqaqmqTlOvfpGPDwtV66Kq9K69mVKcR6utS0qASJgpMDcmPrrazPoy1J/sS4AqAaK7HU5o1JpUjSP8AL+vQ2EFxnBeLbqpVEOgIBUpd1CR2jZEiRLh69ALN3jBUso4K4yLKkhtm6qh5ztZAyhMkydtAfhZx0bb9W8lIp3d9ZO3ysVxzcYvrhrii7aqnU2mruypSFJElEIMESNwdY2t1jj4T9FjN94XRPDb7l5YWuKspCpSaq66V3OlWhlsHcedlH93VrkypWvUkwfhZD4Xv0qcBYbz3klambqp21lSgCopQEydRExZ4IvO7SNK+nHPV5P3Taq2aMAXcEZ0br6BNLiJet8XPdrCsO3WzeN6mnSG6JyqDAcQlxCVrzkGMqVFW28DnaKa/EqKbF7TgPeQ0UKUDGmkDz1+dpVx6blvBNBfNLU0tY/RF+mbcacS4UBYSVpBB0koTPlaA8ZLCL0F4NABam1d5StQruAQBJjQyfC3lmPRsdizxvf8AwF22G3FCw9P8p3X5xOFz3O9ejjNQ+Gl99NOM69xsPeNN9LMy8uM9fetK/QU2EMRr7oIdFLKAdwcwmCIB948rIVRej5lSX8iErSciUkJEjaRrEjSIn4WI196vrYRDiUpUowhBgTpqdNSQBqZPwtnQ4a9sgkYbWVqWWQ+6bBP/AIVt4Hp8dsv3piJdy1NSs1BQ6RTMFxuFSFPDupWNwkpMjSRNrQU173RfDLFXdN8UlbTLK0h2lcDqCUyCMyZToQQddDahSHM9Q0673ZUMxUdIOwMnvbxETrazvo15XeG1OQSsUlbVNKTJAGYpXtsdFbW7ihmMjTG8LGr25iJb+S8P+MlSLz4vY2vFht4JqcR3m82lxICgFVLh7wGx1E9OtjfD9hosu16mo74SnUzGkQYjkd4k9NbC8W8L3oji7jOiapHVJaxHebUlIBEVbggHbX8bLWCqIXVRIS+UtFtQUCW90iJ170qjp0IjQgdTmGQALmADxCU5nKtu5qA1i1oKhK0hW4TuDrHXz1HhZLuCuF33YuuqWWy/WLcWolBQGxMgaRpAMT1POLI1/Xk9eTzVI04QiW1QYIMqiZ3/AJZnXqOvOIKpmkw+HJcSW2cspaAQTExmT4wec5pOthjdFJsjVxXXiHGFGq9FV1TS0Lsslqgbc7SobKl7qBykScvQDcjUWfOGsM44p2zdFwleGKN9QU52PedcIhOqUEob0P8AEpapPeVBtF3CK7qq91paN7Xsw00FJCKGrU2reYhOsakmNYCjytNN1YNuWqWad++Knt2SA4t++lFYVr1WFEzpGXqOoFgNQb3F067l4dXPSpZrL8qK283kAK7WtqFOpTGoGXNlT10EactBY7WjD1K+t4XK1U1a++z2DailsFOhzT4H/hjlYK7cBNNJC6LF1/IS6hRJavNahJJmBJEQUz4gzsAVZ24q+haQzTX/AFqnXXgS442hSkplXdlKR4GdfCdZUjfDspROs7UqsvpKYcxLT1d04kvRRXRvoXTN6x2apKwIJnUGOnct6N+jJTu0Pop4IQolJFwh+SJgFxbg+R+7laPuCeE8HYprsQ8PMZXK3iG7r/u9Dj6bxlwywsmUKJzNkdoohSSCCmZtZJjD90Ydwi3hfD9Cmhu27qD1KkpkEkNNJbISkE6mBzMzuddbUZf1IDGehVuL9OobIOoUcV5FdVUNa5WVDaqF1biUsvqS28S2pGV1I0WkSSEnZQB6WJ3zVLdplo2SCCNJjXx99gy8pRgDJOwUfl4bn5eFilYttynWrPIKQqCdQN/cfC3BMjs5eiufmam5cN3U92NV1LRh1XrFWurczuqclbk5ozGUiRokQByG9t3rdlFeKGU3gylxLL6KlsKUTldQQUKBG8HXXSw7S/tnQ3ErbSYAjYwJ5zqfrYvWr7QJypSYMjMdI3+vO15tyVTdYCyhL0kqd4XnhapabcUpw1LJCEZiYSDA66n4pFqkYnpFNVFQFtLQ4HlpWlehmOc68t/O3oNWZmMU4XrArKyiscZWSsZYWggEmY3199qZekLdyrp4l4qomwMiL0deBREK7TvSDrp350+Ok22cMltJw1zmLQWbxfNQwQQTr4Wsj6EN4djje/7tj+/utL2WNy26kR5982reqQpQGmpEWmn0RLyNDxmoqQEj94UVTTwEzJydoB/93barm5qZ48ljYe7JVMPmr7gzTJlQUFSI36Tr01tAV53NetNeFdRqq6amVSlLdGpx5I7YKccURG40Vt4Wn1umfUyUIacgHUpQTP1HX8rQ5ijBtU5eOJquku6tXeDDjVTTZW1ysBKHCECIVrmGnXe3HRWJ1XaVbBI0JpNV1x0d7Fb9/qqUIu9a1stNnvqLMKIJ5RqNvvsQbvzDz1Zh5dVdNcs0zCEJfdUlJCUuZUqWmNQDHhrrrbt7B1+rpKWMPXgHHqKkUVCkc0UUuNuJJywNCkkfG3N8XXiG8rquxScNXkzUN0gu97tadSQlaFtrzmYgGFAWttDQswNLdEPR33fd5oFNdd30TNNS3a/UBPZ51NkhaSEzP8Y6bGPCxOuvi8r6eZoWK55lirco1oDaoSnM2oECDp309eZspXVcWIbrxAhtCaVqmdeqKV9x99tKewU6F5gCoanOY+ob13XDfFA8k1C6EhpTCm0qr2IGVwnKZXvClHymzjLdTOYJv8QbscvDCd9XpeTT6bwu9ugrGw4TJBUlKiR1gn5+6ulYkMVzyUH2HVZSPA2tnxRauW9275ra+8qWlbFzvtt/620S4+kKyCAomDIjx6a2qhehBrFL0OcJVIMzoJ+c23cKdeMhYOKttICn7wJrU3fxbuwAqDb6n6cwRspteXz1g+61wUOuN/aBWVxEKkHUKGs7bjTx362ozg6vF044uC880JZrKVxRB5BSZ/G14iF5oiJ9lRI7wnr9fK1LGG2ka7yV/BX3ic3zXqDcF6ovq4LsvlBBRX0bFUkg7hxsK/Gxs7TGvKzD4DXsm+ODWEqsKBLd2opSQZ1ZJZ/93Z9FZid9dLQjN2hPJ7xWirkB+ttEpP8AEPIna2iZ1OnMGLYFCd9POxmoZSXe7RypcQNt/KyHUI7ZlTYGqgRt1s669pCqRUAT1FmusBILZMC2nTO8Nlm1Tdboow52tO2sgkrQJA1A0sMFAHVRsUpcyWygiezcWmOneMfhYcrAM5pOutrrdlSQoMjOTynW3Sl9IB6czYHNpJk/hbclIAkmBB+FnCS7B05TMwPhbpKh1FggqNQQB523PdM6DxM2I1MUYC5ESNNxY5d6z2gOokjlZNCifOx275L0pAI8TytJRRGrumtNU8WsmTtFZdtp0tlnWA0RJCjPOBbLNZJeKXDlSneKVzZs8JYrTqoCT2UT56/P32nK8alxRSlagQ37ObWNNuZ+vC0DcNcx4l3a+3KUilrIJGvsAQefX4fGbKxxa1ZlQM2kCdz49JJNo0oFlCdxCUbocbW/2D6Z7p15HedRvZeRTXWACppaxMQTJH1rZsXOHF1YQ0palAH2Rsn4672cKaWqKc2Rck90DWDp9fCzzuIcpU4BajDbF0iUpZQVQCN9Dv05eFszXXJUmmSBm012259NB8rFvVHSCANANSUyIHS2IoahSlZkEGTrEz89duVg5r7oxaOiMofoE94sCEkyZB26/UWDF5Ui2VBqnbQAZCQkRIjadOY91nPw9qcEXNelS9xBwnVX7d66VaGWKarVTqQ+fZVKSJGhB8510FmTTXah12pqKBisQhyoUpKHnw4ltkgZEDuzI70kmCI0EahbN+oW2+6TmWFwuqq9EtIU4G0IA0OgTpt+P6dYy4p4gdpzS9nWqpkPFTyHmnRmWpP8I0IiCNbSBfNCG7vc7ZKXEqRGVUqkbTHgT87RDjanaq7qVQ366oC7XnTTrGi0oDZKZ3OsHTnHhawZARYKGU2vZIlPi+svFppP+sLbbyOLcRGfss0cup8tjyNvTTB9/qewJdh76YuqmkH2ky0nffWD87eSeDayrXiO76Tt2mmmkn2gopeCQVJmJ0M/Lnz9OcJ1dS7hC7HA9lDlCx2qAmQfsk6AgCNfwty3aOzomA9Sun7NN/Uf9lSr0s60VHpMtOFXdVUXUIIn+FAPloPn5W9faNQDSAYPd59beO/pSOJHpCtuBSVRU3aIIlMZUa+MGLewbDhS0kJj2Qd+djU1m00Y8lUqjeqk+qPKUACSIsVUoKnvflbl2oUUkcuosVU64hJSlX6fUWssdqqj23QrokRPkBYi4VHz3t07WLSIUJI8N7AdoHBIBHWetrrDdUpBYo1dbmS9KRcah5G3mLSGKg8zaNKNeSupiFbPJOnnZUxNibGF2XimluDBjl5sqQFesB9KU5uYiZHvtlYi1zpGhvRaNDlEZJ6p31NR34nlbhl+VgZjaL7yxjxgCwKbhk2QU7qrET8M02BoMWcaXatpL/D1inaPtrcXlyn3qja2MaSYuzC3qtATRhtv7KW1vKSgjMdrJd5N1lRRGoaaUtDLralFMaALHKyRdF642fPY33h2iaaXr27dUJQOgSCcx+Fur8eWLvKEOEfbM6BW/wBom1acljsrlZgYXatR1L/aNgg9fvscuOhadSkqCSV5j46k2ZmIr3vq4LqevS7bpfvt5TjbbdE0Uo7KTClExmO45H3CbR1ScZuOd3VlWbo4G195sFYFPnrm2UpjQgZgCAT1J/C2fVtdkDgRa/UBGlBylrd1M9JhiidoqlKkJOSvqIPT7ZR/G1R6UJxyi+LwvwtJdTeFRQLQ0S2laW1pUhQ0VBnnaSLw4t+kPTUNS9h7hNSvu1L63KhirvFseqOk95sd9OYDQyOtmpwSwTjaiu2+Knifhpq7a9293Kmnp2KhDiFNuJSVElCl/wAQiCfjY+AU0T3yx3HEdbLqDte/4WHiz62FjXsJyjf+ybV716cM3XV0VPhxTlDWp7OoccUteQZcpgpSOXnZXwlg5ji5iW7rxxfdXq13hjsU+rvOtrWy2hRCozkgk8yNYtN9Lhi4b/q6S7r4uplyncWESUKGWTyOX6m0l3LgbC911blZd90MNONMt06kjNqEgK16+2belwQUdHTENjHFIPiXGvfiFVKOJMTGOSjvAvDC7LrpK5nBd1tXeisoEJbfqAtQP2hUoKUVHMYTHvsncVrsu5d5XDTVWHmKn1uqWXA62SFpU2yYGWZjoAdhadaFxhhqqSgobQ1XKQEto0CTlgADzsiXqhjtcN1ZS2VLUCFFrMrZA8+VqmGwtpJONKMzzzVqoa+WPhtcQFRu8KamuvFQu+gZaZY7dCuzSmISoAyOca7+NpF4EMUVVcVMa+9Kllmjw/RukMVq2QopcW1Kwg6gBOlov47XXfFNx4qnbuvtFIyxVsDsywnuNKS2VISqJAJBkeI6WlH0eGkm5qpxx1BULoeZzhOwavB1IHun5m1OZpFTdwtcrepzelsw3NlLdAnCa3pXeFWWzquK6pMjp7VsqsJ4Jvmgfu+gfrnELS6G2lV1VmUSgka9pBGmx0+JsiuM1AZTVrvdwoUY+ybI98AiBZy4R9TecDZvd5a+8hbTiT35SQIO552stc29i5voqnDqr3ymyZ/B7DGHqrBV33kq67uVVPNFFQ4872alFDigAdCDGo8Pufirmu6jpgtu7bndCEglAW2VHrr2ZJ5+O1mR6PwuHA3C6pbrqv1G7rqrK5bz9WvRH26lElR5SvTziyhx04iJuLAVNRYRq01V84yWm7rmNMrMVh2Ap5JHIBQg9VJsMRUkcHFkaCbeV1pRR1NVMI2E2JtfkEqXnc12oubtqO66WjWmpTHZJyyhQVI9lPOOXS0EcSmgy9TOSnIEqT2Z3Ptk68hoOn5Ta3hSswDwiu3Dtbez94Vl3UjIqap90uKcd7QZ0pnZAKiEjkkC0E8TH1OUaXjU5yXwUIygIGm3iZUfjbhu0UAjxKNzW5QWjRdLhjv+zezNms46pq1Abo6ZxReTAAcgrByJ13PT39LM+pxzhtKJReSld/8AkV3oHKQJ2NnQZU1Udq23k7spSJ128REffZvvXbdjIIRdtICSYKW0DWZ1kaj68LEpwAPEpSlxHhKkP0Z8aYGvPiarDeIjdtRQ31d7rSG7yZQWVvIKVoA7TQEgLgbmfK1tb3r8CcOblcdTW3Hhu6kK7VRSqnpGCoiMxUcqZgASSNhrbzsxM3R/uCuUzTtpCWs6YQBoFAx87QHiFls11KU0zchG4CR/EfEa6D4626XD8r4i0D7rn63M2UOJRHiS7RXzxExVe9LUest1d919Sl4rDhdS5ULWFlR3BzAyJ331tH2IA5R1ZpojOgGFwAQrMIJ1/lTHlZ+VFGpL3rDaCVAAajSRHyMRPgJsxMWKDeJFBKsy+waGgOgCBpGpI3119nkINtVrcoWc83KQ3hmrUhJAynRKZSDqCZBiOo6wOYAEmcEsM3Vj7jThHCl73QKyjcql1VbS1DYW06wyw46ttwCNFqSBtOuh1i0bOrX2wSXCpuMqe8SARodz3tQABuJMCZixPoG3Eu9uK1/YpcSSxclxinQQggJeqXkhJKjucjLo1AjYxFpDqhkqNOIfB9PAvi1WYXfp3H7hrgKy6aioTnTUUilSUHkpxCvs1bDuhRhKxMnYduvDdattHqt2LdVlQGQhHd8Ak7QD117u0Ws16RXCFzi5w+Xdt104cxFdLiq65CBlUt2BnpwojZ1ICde7nS2SSE2qbgLEVzVN20iL/KaepQEsIqHWygKAMJzEpEK12keybGifcoZGllI1BcFxUziBTXM0C0kEFSYSJA56k6jU7fCy5UNrQ0W6d0hIRIZUSYUNU6HTdI8N4sWo23Ujtm6gLQBIUkhUDKDA3nn120O9jVTUFhptyoqEBBIPeBOYyIAE9AOfwItYIuLJgbFGuHV/jD/EK5q5Cc3a1CaRSQJlD0NkSOYnN45d4tbtaJQUFO+ijprrHvO3hHzpVdwqKK9LvqlNKaFPVtrU84AhKQlwDSTzhI/C10pSrK4FTm1HMRH1tbNtqQrpOxUHvH1dbrZckpKkEHwO23hP42LuqzIUqICh7WojWx+/Gyi+bxaSAmKpzKZ27x26/lZJzAwcxCBokAkgc9umv4W4tzcryF38bs0Yd5JN7TI4QnMpGQjKdY035eO/3WAcCVNpIIABy5QfP6mwzhJeCciSO8IVqeev18drArCt15RrIA2+o8t7GZsgP3TZxhRIXc7LroMNVjS8wAB9qJ84J+t6n+krdzlHxCvRJVm7SnpnEZtCQltICtNDOVQ5bWuDitvPhyuAcEpbzgHXVJB1+Hjas/pYUA/tPQVoZJ9fuhKSQNMyVLI194+XS16gdaoBWVijc0BKrA8haXFhQykEgjpZ0cKn65jiJh/93VRp6l6uap23c5Rl7Q5NxqBCrNmqSUOEqEBRnT68LHsMXiu58R3Ve6NDR1rD6f8AcWFfhbqZBmjI8lycRyyA+avkcB8Q6snt8UU5CY0NQ4SSY3gfUWZuL8PX9c9Q5R3liZCFt0prEkqUO2E+wk81d0j3/GxNO+XEqaKWgEIUUrTOZwGDKuUg6CI5WinjI1R09XdlWu6KetUAsuFaiAlpMZo1EnvE8+fW3ER3z2XeSxNEeYKJ7nuK8r8vdm6Hr3dZcdbVJgqAytpVMTrmC/dHjYhTXPUP3iu7S7XOPs1RpnsrZKYlYSoKE6d0ctJHSzyp72vpgNXwtFDTtVDz9Aw602C4lSW1JTrvoUiNdbI9cnENQzVXjUX0+Qq7kV6G0EpEFYChAI1GuvOfO15oWcWhJDWFMTVdGPVqGrLy+zhLikpGUpJVuNO8I6ajxs4qXhHeDqUqqahxjvupV9olRDYSOzMZdTPtD87A1dEq4MS0dQ1XVDgbbo3UrWuc7anChcj3p15R7zNYbaW2IJgpmeu33a/GwZXFmyNDC2S91Cj/AAHZvWgTS3teS0rWpZcSyCAQYygTz05zyEaWqdfVO7T1CW3QQpsFozO6FFJ38vlb0cW20qcrpnmSnbnprpr91qFcX7pTc+Ob+ogILV61KgIjuLVnRp5K+uerg0znFzCsnGqdsbGuamcVlAp3UqBKR8CFE/ja91y3gm9LloL2GiKykaqCJ5KQlX4/1tQ/NNMkfyrPPqB+Vrn8IKxV5cNMP1K8qoouwOo/2alNj5JHxsfGGjhtd5oGCu8bmeS9DfRAvb94cIvUSpM3XelTTAA7JVld+9xXwtNizm1J6aC1YvQkvQKu/FdylWjb9LVgbRnStKjr/kTazZOoAn3b2oQG7Ar1QMrytEzpl+Vucxnn4k2w842+RtpUA6g7xa0Aqy2tRUjIdNLNytbLTziQefvNnCCOfLl0skXw2EL7QAz4Wt07rFVagXCbyDkqahE/xBfuIj/pNhe9src62De7l4FRgZ2jpt7J/wDzrCwQTvpOvW2iHLOy3WwVEHlB87YSCdNuttgADMIjz2tzIgGBtsTad1Aiy2CeRjwt1JKpBO+45W51gb9NLbEAkDlpvNpApIT+HnET9fXjY9duj0iRBkEediAgDNqNNTY5dqwXhJ36WnmTWS7mV/Mfj+tst3I5pPwtlnumsF4lcM3GnOI1AEADLSvwQNRKIIPXb63tM1aoq3A0VEjbNy/HS0BYbvNygxbTViS6FBtQBb3BJM7bDTUe60x/vt112nKlJyrTkUlZglXKD8eth0rxsh1DDZOfDlQxT1K3luQ2hKlFJ1JJgQP6bWcn9pLuTKU1QIkQAkjl9R5WZLVSHEKKHZSlI7wMneeX18LJTeMcNVN7G4W79o11qF5Q12kd4bpBmJB5WsSRNebkoUUjoxYKSP7RUQ0TUKhMykJJnpy+pFsOJbuQmEuvTvAT9x56j7rNBSiVZwuQDIG8Drpy1+psVfuulcS46C/nkkZHVAnw3sMwNGimKh19Urf6TLsDjyGk1LnZqUpUDKpIHICZMnN8+thrqxfdbLKnXlLQH5W2Csax3dNfC0SVdO/dle8s9tSvqqELJUSpKUKVBJB3EqEgHn8ClbVBq8Cp99aA3TqWWUA5RrmciNSIEg6e0Olq/CaDqrHFcVI9+cSsP1lSm53rurCqsR2aXM6YCjtPT2YHuHMWhXFN+Xo/iSouxlgU7iGFtLJIlYAGWY9rSfOT42K1d7NPYhReCnUClbzLKe1OZOwAk6pMgR4WPY0Zum/slZdVak1rTRcqsylZy2ACAnrHlyEkRYZYBqFIPJ0KalKw/QVNPedA0X00rAWVIQYCh3TMbDfUyBMyOfpdgqto3ME3MglfaKuumME6BRaSTz21+63nx6P9TUNcW7ho3nHTS1NS5TOtrkocQtKklJTEFJEA78t+d9cHMCmw1dTCNENUjKE/5UoEcvK3KdoJvdjXYdmodHSfZU59KJeXj8l/vKCa27svP/Zo938Q93vI9h6Mj1ZvqEj7rePHpPKqf9OzqUIzJS5dyiJ0ACEanl0+WtvYOhUPVWyI1QmBPha7F/po/osqoFqqT6owqCNI5TYBzQSdLDKJywJIsAvYySDy0sZirP3RV6NdhpYNLiCkSoA+zbp8xuY91iClALmSddLXojZUZBdDVThQyXBopIkR5WkGlrqenp6aodcCkuInmSqEyYtGN6OuGieS1mJKFZQnUzl5Dn5WUrqvCleoqO7b7rn6KspgShtxXZ9qlQIB130V7iLVa7KSLq9h4JBCetXilpK2k0jK051tpKS3lOVcwofCxV/FS+yWqldOcQpBUBCkhQB5/fZLNy3Q8ptZvioKkJaSgodEjs5g6DnOtuFque68110lPX1y3+8GUyoIBMzOkCeptS4cLtbG6vWc3pZLS67tLyqKdtKSkNJW5GgCyTG3MiyXfAJu1eckQttQI1iFgiyw1TU1KghDYT2hK3CTJKj1Nku/XqRu6aoh5uQmYKwCSCNBbnKt5mkuwaBalPZgsUE046WF5idHCNbaoHHHkqbaOy1b+dq6ekTjnGuEsROUVwYnq6KmdpmnOzZUEhMyM207gfU2jfhBxVx5U49aui9Ma3pUtLu+ocUlT6sodbQVBR2nbn5WjNgUtfC14cAE02IRwEtI1VnHa5aai+kZjCa9fLaW0H8bHw6Vsu6eyQR8bVh4c4mxFet63O/XX268X62pRUGqrFJ7Q5SBofbIED4WmzGWP2MAppPWLmrL3F6LUy2i7yFKS4mIB/zToBqSDarhOGvwzHImucDcHXbkoYlUNq8LeWN2t+4Up4VuBy96lhLzeRtSlKLhQdcsEhO31FpPpafsHKxEDVaDMa6oQPwtB/CXi5UXziClwreWCr0uSpZafqmU1jgzLbKADAA01AtJV74urrvvRh6iuVb7DzR7ZfaQltSVN5QdDuCr/hNvR3V0LpMmYE/j1XDilkYzOWlLlI6n1uqoUkh5yoDwCUCco7MHfnvZJdK313Ow2taV9g6EZkwEqCiAfA6WaFXxBrk3tX3bfLN33feqb1Fz0zqqpa2lLfp+3pyVISImEJMDcmDZKxNj/GN38PcI1tNclCbzevZFLUuJCl07QK1pJEkK9tI3smVcLn5A7UJjBIG5rKBOPl01DfFaqfBX2vZ06lkIkAgQSSdNYs4PRtdSfXaRxorKWb5bhaZHdvAqAjn7f9LJHFy9q+9sW3m5ejjIrWH105DTZSlQSs8iTr7/ANUbhhxBc4fCuvVq6m62L2vOlUyp3IPtC05MgHr03NsuWq48pudAVt08PBiFt1Z2vo6ZVO0aamoygCQU06QT/hM2N4ZaYbvNpZYaSudFBpIAPmNffaD1elqmlZWUYApiWXJ71URGXxyjfX4WLNembWF5akYBuxsurCmwqrUVDXfQa/Kyjpmh4kz7eSM6pcWFmXdSNw1obnvvCt/YexBdLV40D173jTPsKSFpWkOoIMcuoO4InQxZL4f8ArpwVxBRiqnvq8Lxuq7qdxq4rorSV/u1bipcKVkmRE5dAe8SSTraO8I8Yb0uKhbeorroS3f2IKhxxDgWexDic3dII00G/nZ0V3HrFN1ur9Ru67dSUgFtShrof4rAOO4bGQyYEub5KzHS4g1juA6zXDVTJj+rTWYPvNtLTKw2xmUW3kqUPtG9RHTXTwtV3G7VOKGDq32yMhUkFOoCfjP1vZ6o40Ymv27Ki7ayhutLVUytl0tMZVpCo2M7iB77R3j+oUigW8SsuEIKSkQE9/Ukxofuje2LjNdDilSyWC+gtqtHDaWWjp3Mk6psNqSSs5gc7KTqTqdDttH5/ErUJEE6CTII57zy3tujcZbqHRmbh5kqSQqTy5H38traeCQe8ZSUn2j4fp7rTjZZFDrjVIl9tKfuWuQoEDsHASE6zB6RHL4jxtAN7sk1lKpSYys65e8SokztqPx08IsVWILjDjOY6oIgqI7xEDz/AFtXu+kLTWMJUVo7iifDU7CR0jWN/cN3DOYWLiVrghFF0oUkNpSFBwkJEAzO2/h8hvrrGGOlrRi11lBStammSsAaEwdgdBvpHOPG0uZUrpgrPokkgJRG52mNdvkZtEeMW3TjKqWQ4AlDSQQDlCUpA5aJ2I35DlvsHTdZCbF6OJbZP2hZ7pAA1JG5AkxzOg8N51uv+z1uYMYAxbfq2Qh68L3p2ElUSW2WVZST/meVptNqZ1rZbU2pohpKt0hJJIG5VlkgQBp7zta/foe0hu3hLd7CgEKqGXH3FzqpZq3N9NwhTY66x0tAu0skWG11YBhYQ82SCiXEmDqNDr4zytVP0m+GrPDnisMT3ZSsvXPj0uVZp1mEtV6dahAIB9sqDyYTut0RATFplvB0LUTMmY3iB7Xuny36WaHpe4VdxNwWcvmmRnfwxVs3qlJQTDAOR/XfRCswjmkWJEOqC7dVZuE07bk3De9RdFSVpK6Z9P2RJg+yTkM/4eR5aS70VN5U7S3qxhLiwgFdTRZS6jaTlV5qO6p00A0sz7rW3e1GyatpLanUZgsIzNqVoIJ5HefETrsFemdvhl9NMilp3A0rKAvPlCZEEb6RBHPWPK6L2TaIKofoK+p9bpnK6pLRCg6XC4G1CAUuNASkTuop5yJMWurhu/rvxLcNFfV2OqVTPsjeApKhoUqHUEHTbmJFqlm82GElLFBSmqpEBZiRl05KOux5eNpn9HS+xel1X62spZeZrEKXThITlC0e2RuMxBH+5ram9mQ36q0w5m/Raxi0lnE94IU4QVrCogfxJBnw36c7IOYI0SsnUCCJ8tfrcdDZ0Y/T2eJVrGYdsy25A5wCPPl4ffZrrhJgKJ2ACTuDH9f6W4upGWdw813lG7NTMPkEnPRnJUnTNOVKtpMeHM/OwtfeNZe9WurvB4O1BSEKWlpLcwMo0SANhyHjYNxSFLKwQZTmnp9R7rBlCgtYUpMpJ1B0mdND+Vostuk7dEr2R211VzOUqK6dY3jXKYgfC1efSipE1eHcF3vlzJVTO05JBB0Q2R8cp67eOtkHEJdZW1MhcpPkRt06f0tAfHen9Y4MXHWnRVJeRZzAHMBDqY98C1qlOWZpVGubmgcFTKqSe0IPSBGs2DGmUjQ2N3o2WqtwHkpWk8pNiidd5gba87dkDcLiSLOXqZgWrRfGEbjvfKkmvu2nqNua2kn7z9a2bPEu72Ha253axoLaNQ4w4CO7C21AyI/w216O16m9uDGEKtRzKboU02sa9ktTfXX2fnZc4jXLXXtcLoutoO1lMtNQwhX8aknUE8pBNuGk/TnI6FehsPFp2kdAoJvairX6G9WqRbBu2mvhDlIECVAuGTqNCDnTGvw3tl1sCsuq6sySBVXVX0Cu9MlJKk6HWeeutnJR4CxtT0i7uacpBT1LbDruZZzJdayxAAOgPjysuU2AHmHmClxrsqavfqEADdl5JGSeRBPwB95zK0c1REDyb2UUXjdrzV23BeCax56aNbyw4rRYQtKy3PQSYHhadqZTb9Oy8mSy4hKwI5ESNNtrNC7+EjlIQivvp99torS22pEJQlaVJUnUnfMD7hZ63ZQKum7Ka7+0W6KZoNJW5BJCREnxgTpYUjw61kaCJzL3QBGubNmOYRJMe/Weg62pv6TV1+pcQ7xcQCG6xmlqxrz7Psjp5tm1zykOKJbGh1EmTzm1WvS4uxTWIrqvHISmsux1kDmC05m19zlr2Evy1FuoVHGGZqa/RVyT/drT4hX4fja23o4VvrXC1hs6+p1j7A00EqC//eH5dLVJb2WkH2k/jP4Wsv6KVcHsNX5dsyaatbfAnX7Rsgx/7O2zizb0xPQhYWDutUgdQVez0NLwNJxEva7VKlFbdSymNAVNuoI89FKNrhkCJBBnUfUWoj6Nt5fuvjVcUqKUVZfpVyrQ5mVgefeCffa9qtR1nmLYtG7MxbNa2z7rkkbTr4W1I/mHnbCevKbaKp1BB8elr4OqolYVayOXU2J3ijOxm0kGZsaOg0V+dg3EhbZTrtGtjxmzkKQZhZM68YRVUbs6Z1tzPVJP3pFhFKMT1nn8rc4hBapu1CR9g82uToAAoT8ptyXCdFKI1tpN1WY4WQwJ1BPWdbcZxOaee2/ysGFzuoaeFsSoxIjQT7rFQzqhCr/FAI+hboKGmo1sAVA6lQgn5W2les8wNT7rSCijCVCQTpr52ULuMvCSNfhFkpKjoYBGkb2PXYoh4lOkmzpJw9465FGedst0kqCQAQABoNfztlldNZeFt2uLcv5pTK4IbKjpmA7x5/Xy0khd4ssNttpWQ9HfVvsDoDytGVHUvM1ofadSCtoIiSTJUoZSI015b9edlOmeNRWINZV5G0rH2Sf4txlHnPS1OGTK+yPK3My6k2jr6JbdZW1dc3SMss9h2qnIT7OpknWPM2gfDt3qrMa0d00zrK4rUI7VAELSkznBAJ1ACtfnZ2OYgvHEeMF3XcDKKmmo6MIcpnmwWgSoZu6d+WsEjXTezfxdTVt3Xi5idvLRVhWCWko7MNKCP4QOQI5TuN7FkqA6QN6ITIMrMysxIQSpzTNuCPzg7ee1kqura3M9QUTjQeKc6Co65dR8otFGCOMVbfj7d24jdS28mSXUJAD7eypT/MBrp/LtZyXziWhorwarrqWqpWzqsZSQtOkjSZ1+467i1wzNIuqYiN9UnY14gLTVfutSG+1bb7+QEnLl5yJBnTTl1myNc+J6Jy7mBVQHIepS4rQhtfsE7SAREnXWzRxFUqvW+qq9A2XjUAx2h76UnyjbbrqLJ2d0NBzsysICSlREKzcwTtMz7o2tTdISbq01uiXroQxUVgNQouq7ZGUwDnSdNZ6aGenTeyxildwUNW0y48ilqEoUlKgmSdu8Fb7TodIJHMCzc7WjprnZrWQj1kLLayFGEEiN9xpr9GyFez66iriofZdVKUpWlRJAzTEDwjaRaObRSy6p8cEK5DPGHCyG4UF3glskCI7x2Hjrpp0629C8PN+rXbSU8AFDSW+6Omn15284eFFS5d/FDDAcKghV6UuZIVIyZ09OQIn3jxt6R3emClClAlKtYG5+63H9ov6jfou47MaxP+qpb6Uqn18a6ktsqWhpy7yswDlBSjWOWsDx8dLev1BPqTJ/9Wn36C3kd6TAW5xiv1ZnI0i7TIOhkIAnp3iPhy1t643WqbvpjG7SP/CLaUJ/7aP6LFqv9VJ9UOqSNyD0+NgFkDfWw6+u3vsA5CTMmJibHYqkm6KPE5SUzZOeJCjGkazPKyg+dCT9aWTniUqOb3C11ipO3QTrhQ2FlQ0Oxs8WWrvvIJRX0zTyBqlLiAoA2YV6LyUDqkaZUEaEb6/Gzeuzite1Cy1TrYpXltISC8oGVmBJIBAE+Atk4pFM97HRcls4U6MMeHqX6m57ru9SF0FC0yXQSrIInpZUuWpaep1oSrvIWUK5QbQ5WcWq2uhJp6VAAgKAOpmeulgUcWL8pyX6ZFKjMgIgNzoCYOusyTapNFPJGBzV8GMNsFNd7UBrLqrKbOUl5hxEjlKSJtVo4Pbu91utVedQsMLaecBEJCkKBI15aa2fo4uYweQWl1NN39IQ0n62szcTXhU1V2V3eUkOU7mZIG/dPTzsfDWvpY3sfzQJWZyHHku+MfCrEHFPFLd/4QraStusUYpHlIcQoB1ClHQ5t+8LRlhDgBi1jHlW3TPH1lmiqaVZW0W2/tG8hKVmUqIzgwPG0o4dxFeN00C7suyscpqUqkNNGE6gAkH3WUrprKt6822zVOlS1GVZzJlPz1tWjNVAzhtItyVl8EM13OCiGj4PYhw9fFLSP3i0XbgvIvVAiQtKwlcJHx+tLSPj55arvuZwCezvajVtzKo29/zshKvK9r6vm9GRTOl9KkpcyZipRSVJ133AHw0ssXtSX7eNzXfdzGH7yW+iop1rcLUJCUlKlGTB2PTpbncbZK6Rk0h1Gmi08P4LGOYxSBSqZp/SDuRmjBLdRddQgyddW0HSd9bSiH01dPWoomchd7GEDmQp1JP/ACWjVWH78XxlwriJikKqGmpnGHl9ohORSmgdZI0EGfMdbPbD96rp3GEKbLiKpx5sOZkgI7N11UHXmFC0KGbJAL9VmV8YfJp0UY49uTE7l7GtorsqKhTeKMN141T/AHaGcjitSPZjz87P3EN1XnW4Oorv9RUmopL6ZqchWB3PWnVTIO2UTYHHF93mwm7627rsadp6h9g1LinsuTIe7yPU2PVWOLpfVWpq6htAWhtKcn2moU500mFTaD8V4TiCfomiw5zg1zRf+BVx4533Q4fxpfDroLiKmudWgtgEkZp357je0S1eJXLpoH0N0CqhFdf6yAV5ezC6NhyTod491nfx8bqryvtxNyrccZbeKiVgCRA3H1vaPL5o36i5+wFOVuN3tQqUBEpzUWU6+bZtvYdM6ogEjhuq1VEIZDGOSb2JMYV1NT1BoWksuFKnEhcKElI2MdR05+GonBzEFXjd+8aS9lIS5QXS7XNOtpCe0eSpMSD/AAwdh4WCvnC9XeDislFVS02EfZvAJnXT2T9352C9HfCl8IxGhbjnqtPX0FQyVDvLKCYIhQiZSRPX4W0q0uZRPew2ICp0wDqprXbEqer6u6kuzBWE6ykzD1i82ah5RUT9opMEidBbeI+6pS0uyQZjYgdfhY7fl3preGN3sqcdSbuelDiCJCmlgDl+H6tzEN21KlLcXfdcsETBWmD8BbzaneZnEvOtyuykZw22aEo3BWOKd7k6J0kTOnz5fP3gY1vJ1y73W3E50ZQQG0gKQoTr1O/yFkXD93L9YQpdVXENnMiX1CDOugjoPrSx7GNSF0aUqUo5G1gBP8QJGubwk26CkiBKz55Dw7JisXvVqvMMGtdKEFSUjMkQlIICDpqJgzpYd696p7MguZTBk+6IB843/OxBwA3yQpaQkIJT706g/P424chuUjdMqlQ0jntt/X39XHC0hYgmdshHLxqYUpVQEwJJSqCTp7+XjytFWNKb1atafZbgLp0wmdSozJ2+fv01tJRgKPfMHRI3k6D6/rEf8QW15aEpJgJUkhM5dFQdSANo6bnxtfpmhrrBVKhxc3VJFMEIpmm21LkAAaxBPOY036xaK8XpbcxZWGVZVJbhSVFMSAd4EwI1E6AgRNpbaC2mmUqHdWghJExsNTtJ1+YtE+MwhGKKhIyolCJWI0ievgTvpqCTBtek2VJu6Qn2XO2Cex1EKgxMhOwlJkRHlJ9qNL9ei204zw+u9haCjJSONgT/ACs0qyI5d5S9DO3utQ9m6a687xZu2ha7SqqXfV2G0albqyUoGpGpKkADf8PSXhthFOEq2tw22nOi7r2q6EGCkqSGqhE/BlPht4Wz5KhrJWxk6lW2RF8bnDknu3lKMxKVSnbnO2u/L8h4vJFFSX7hp26a5rNT19Kad4RuFIjU78z8BrZnobAIyNJI1SZERp/QbeXQvK5HAq7KcK5JKSJ5Ax/UxbUh1WZICF58XfddZgvEd+YOqmlF26K0sFDumZqe4oAaGUxr5+NlleJ7rZayVzTiUBQlcCEyOfIa8vxOsm+lTge7bvxvdWMX2Cilv9tV3VjidCl5JKm1GOpJET+NoXqLrFCtNLfa1qQAEtPgZUkRPeG0iB8NbHFwmFk6ruqKGtYUKJxCg4ggrI74zHfSSdRy8LLvo6X5U3TxONzuvlabxYfoHpOuZCS6hRnUx2agPPxtFfqNTh50Xjd7yXqZRUVBObKRm/VXxEQLPHhverN3cWcP3wiE09bUtZikwJWOz+ROvny516gE2JVmAgXarFcT0lm86N5wlPa06kAdcqv/AM6zKeKs2ZRggRBO0HlI8BrtaQ+KDJW1dlQQDlU6gnXnlP4ffaPygwcq16HloCY+Ox+QtxuI+Gpcu3wo56RqKPaHRJzBWsA68p21On324CJ1UFRyjUkmNLGHUACCqVbkHefjr5/1tgbCSE95JMAg/Dn4H6Nq7XK4Qi6EQUiI1yzzPhHx+fvhvizSJc4JYkpkpK/3fexUkDU/+cJB+Sj9TaaQjvJ7qQSZM8vrX62jfG91GrwTxIuzRSktmqhWsS0F/enz0saJ9pAfNVqhmaMjyKoFiFtSK1xSgqVHMcw28PrwslAHefdZbxI1DyFlSdUaRzg/rZEgpIJ687dvEbsC4CQWcVfn0Nr0/eHBZimMKN3XlU04E/5Xf/ebeVp3q0BTi0lByqhRPnOuvl9C1YPQJvAv4YxXc+4pa6mqYB1HatqT/wC6tad5pQUkgBQKEkwNenwi3GYlaOqePP8Add9hX6lIw+SbrlMhJWooJSOZB1Ov19RYq6CO8Rv7RA2ssPMdmrKpKjBM6AHQn7+nn7yzjKkJyrJGx16dfL8LVA7RXS3qk1TRSSQYiBOUnx/GwLrZKQRqUiFJn3Rt1/Cx5bSoKAICTGUHw/LW3DrZUFDKESQAcsmf6n+vJZlAtSZ6uS4A5Ble400PMfXwtAfpa3IFYcuS8YUr1avcppSkzDzRny/ugPo2sT2aQoK1HMHcHmD4Wi30m7oVePC28qwA9pd9TTVaIO0OBJP/AArVa1QyhtSw+ao18OelePJUKYJQ6I3UCn4iOdp19EqtCcR35dhVHb0Tb4SZhRbXH3OG0G1QDNc6EiAh0keAnS0qejHW+pcWaej/APTaaop+uyc//Rbrq4ZqZ/0XF4e7JVM+qupgG8U3JjvDt7Z4FLedOtROpyh1M+Wk/fb0YXvChzNvNLsnEEFsAKBlPWdNQenj+tvR65q9N63LQXmmCmspWXxB3zpCp+duZoHXuF1GINtYo3JAIBI8baOUGSZg21oNTz5Dba2lanTxnztqBZRWiTEJi2oAgSdd7Yo6ae6bcEgezzHWxGqJSBiSnz0z6EjVbagPONLJLbwdbQ6lI7wCvlNnJejeZrMNdLNC7yPU22809kS2PJJI/C2lAbtWbM2zkcSsRqQR4W2VJTy36CwQWJ31md7aU4RBkE8tbWEA6IUuGdBrPPY24SogjLsI0sGXB/Fy6flbErJOunIeHnaQUCjGYSJ08bKF2Ll0EyD5WSg5qZ8rH7tX9qFE6zBnpFnSTlzeHzNssW7cnULV8Ytln0SXhtToPbdo0ohfZgRP8MqmPl7/ACsl3lWu094KDLigVEhIGggbxJ5DQfGbHlBDVQU5kqcKUicxJ/i0A8Bz+YmyHenapqyW2ySrUInTUbn63nebZLTd6su0YnJwCcbViy8nFKzFdJI7uYzmHx3g+fnZxcfblo7vu677yacf7esqXAtKl5kju6x02Gg+divo7Ubjd/X04R9iKZCSojmpUxrry+6yl6StTFDcCASmXXlKABmQE6jlztQe93tENG3/AAtBjWmhLioSoVLaq2HE9popBUYgDw8f00s9VXkWVpbWVpSQFpAOUK0Gh5kD8POzFpXXHXAylYUCCkyJA5j693msNorKgpyrygnUExJ8Ou/LXX3HZKyLXR+sfL6hUR33BqUkpTI2GvmfrWxVWoBb7yIzJkwBAPL4a/1sI25Rs0q2nHnO2S3lQotiJJ0Opk7CxFSlpUpS5kJCYBmTED89OQHuhdTDbIwe0RdrhS4stqdBykQNtY8do6jSxZtinS0CzVla0lIHcMHUGROxBnwiLH2altFw3i0pJDgdZyJHQEgxZz8EsM3NizEVbTX3SLep26PtUtpcUgZgtI1ymdj1IO9k52VtypRszuDQieCKRf8ApGwwUZVBN6UygpLk/wC0TOu4OpMHfXwt6W0yCH8gP8RBA5T4/X4WqtdfDnCNx1Ld4XRc/ZPMLS62ovLUc41ESo7ETaSKbHOKFVBm9VkKXJUEpGvPlHn525vFqV1c5pYbWXWYNUNoGua8XuoQ9KdxLPF++EFZyuNXaQiIChCQTOnWfdy5+uFzqP7tpTzLKJ5/wi3jF6Qt511dxMTU1tV6w4pdKlRJ1UAgQPPp529mLkcz3XRqECadskDxQLXWxmOBjTyCx5ZBLUSPHMo+rWRpr8rAr7o8Ok2FnSAJ8LAOGRqN9NbFjQZEUqFGCOY0+vjZLqVD8IspVE5SdxEWSqhQzd4kRpubXY9AqTt0n3oo+pPpAPsH7rQ+h0lITPLMNeVpRxHXpoqBS1ICpWlEGBuQLRNKgfsxGhO3142r1R1Cv4fsUdcccQcxMyIkjcHmPrewjLzimgAsnQRB0J5D7/hYFplxSA6dErEAzpp0svO3Cmjuakvc3jd7oqVFHqzTpXUNQN1pIgDfYncWqFwG61W2SahzKiCvNzBzbfptYO9Q47dtZlMDsHDrJjuWPCmQSomdR0Bjp+Nua1DblBVtoVlHYrBPuPT+thlymRoi1yqMLjVICVEk66izkuZIN7U6kdwJUXCCZ0G4n42bFwLbDUZs32bevIyD8LLDLrjJS+0ohSZgzqDtYLnIrBcI6w83QY4vp4QhJbaVPQZ3Nfus6arENHT3ezX07qFMpppWlImDlSPwFoxvVxSr6cedJUpymRmV4gkGfj99lJKj/ZJ4TCvVleMQJi3M4/8A0g4dVdw+IGQgqSq/Hl20artRU0j77hUpEpUEpyuMqSQZ15C0ccPsc4gdua86FVcrs6a9qppogSoJBB3InmfrSwmIFqULodTAipZKj7lD8vhZr8PVZP7RU6yqU3y8QCd5Sg/jbnWyvNM4XV10DGSDRPl2vqak/wCsvrUJmFK9+g5W7p3YYdIQQMyDodjrZNaWVAhASTJGxgHSxqnUBS1Ikx3DBGg70e/e2TLdw1WhDYHRRlxGUkXy6twEQpsoG4Pd+vnZl3kHm1VhfbLZ9aup5zTUSXUSnl0+PlLo4mlRvolIASWk7nQQd/62ZF9EG4Kp6A4ct3LKkqGsVKwN+ma3pmDEChYuPxJpNU8pRqmb+XeVQi5KtppbnZlYcUmDmCwD0Jifo2S+Da6qjxLQU1Y4e0acq2AAQYTnXtHKR87aIq26xaxTqWCykqlGmUKMHpsT+Z5FeF9YXMZMrS1l/wC0H2zrIzaz7vztp1MjX0kjPI/ss6KJzahjz1CnKq+0wFXtR3W6ipBG8aqj7jv0s27/AINMlSUpgpnMTvpsdPLpz87L6HO1wnfzAUD2dY7KSIiUn8x8fKzUvSpSu72FwlwqaQRJ2gDX4fC3m1K20h+q7Oc+FEbrdWiobT2mx25dDYXFwzXUah0pAMJUBoSJ2Hwsm0a0+uSkZioDbYGeYO4so40dyYfnLmUmSMoiBy8utupoG+KxWLMTkJTDcP8A25ToQhQJQ0mI2OQb/rbhxwZkyAqQBMxtrz3ttlxJxBdjgXJeYAhQkEAKTy/yi23EJQokmDPMyd+cW6dgsAsflqgCnXUgFWpJJjfrz3/CzSx6wHbqpQgFSjVlJA0kEKOvTX7x1s7VIVCdhAnVJn68bI2JKJFVRphCcqKyVDf2kxv4kp+FrcRs4IEgBCZ1e83S19LTtrjK2FEb6kdeoET5+M2iDHhdbxG8nLoUoVmgzlEpGgMjY+EdNZka/albmK1IAStLRyyQYAAG3vsweJNODfs/xFhCglOYKhRXrmAjkZAP6XpPdVNu6kf0PLsw1ePpD4OVid1pNBRmqvL/AFhxLaEVFOw48xmnUQ602YmDAE6xa/l2ss/2pvp5t9p5k4gbWlbawUkO5Z1iP/nJET5m3nH6P9MmuxZRvnuOUdHULCVGMwy5BvvPbTIHIWu1gXFfbqvChXlQqnqKKqSUp9kfYI2O393qPo85W0TjVNqw7YWt91r0k7QwxEbqTbwp+wyrbT3wqVFSiRPhOx3+t1jDzyl3cO8cyVLCiQBrM7ddRv42L37SuGhU4gpTACp9/LXx8PlYLDDx7BxsqIUl5Wk9QNduv1pNt6hkzBZlfHwym96QeEjjDhdezLDWaru1k19MOZLfeIka6p5c4i1U7sq3LzuxqqCkuO5EoWh0DvEAjyG3LbytetS21NkKKS2qJS4AQQRJBH1obUkxPhRzh7ja+MLk5EUb3a0bhRJXTOd9oH+Y5YB/xJMaW0gs4bJBXedBTuOU9XdymAFlLiuzKg31GmoH5xZHqaG8KWtpKy45cu9tZqWXkLBLK5BiCZyyJjw2s6y/SV+ZvtGfWhqhsHNJG8E7gz9aWal54aVRPKqWS5ShRAV2JhPgY8RpEWhKzOEWN2Q3Kt7f15t4jwLc1+tKT/rIaeBCtDmQZ+JFmaGFJSUhtWih4ieZ6cvv6W64VXw3fXB5u7lJKn7oqUsuyqIGeQRHgo/WljrrCCdjlgSkjUc/ONNrcLjX6dT9l32A+Ol06pPcazJyJGgjUmJ8umszt+NuAyUnWZMmeUjTfx91lJVIe/CSZOoA85+j08LYmmIJbVCcwJEk7z05fpbObItd0d0nFlQMhsAgbETzP17turQr7s9bvrFVzutApvG5wQAdT3FNny5fpZ+dkTIypzg6nWZ5e/WzbrGjT8QqMkAJrbrdaOb/AALBgctlT+liCSxugyRaLzRxQyWi0SFqWkrQSUwCfqfP42biSM2qrP7ijdq7sv29bvISBTXk+wU8wErUkbT05mzAJ/pbv6V2aIFebVTckpCtb6Ad4BOLMVXRrFTd7FT1gtu5Zj/62121s52wQAZBHKN9flbyg4f8QL44eV9Re1xXneNDVvs9gXaKoDKi2SCUklKtJSk7crPWp9JfiJUQHcT4nUiDAN+upJk/4Ep+VsXEcIlq6gysOi6DC8bio6YRPGoXozU0y85WUwqdQr66/Wtij7PY/arypB3JGUEzprbzWqOOWNKpZcevK83TO7t8Viv/AHosj1vEi+a8RVUtO9rIL7jz0f8AG4q1duAS83Ky/tLDyYvSupvS46VIU/fNDT6QS7VISB8TptZKq8dYBowUVeNbhYiUkLvJlO2n83X66eddXje713GxTUeH2Wb0SuXqpSGFtqTroGy1I5bqOtks40xBMtv07es9ykZT/wBNit7Pnm5Af2kA2YvRB/i3wqZyqd4hXDmA7wFe2qPckzZjcVuKXC2/sAX/AHBQ40oqmqrbsebp2WULWVu5CUJEJ3mN9Nd7UlGMsUBGRF9VCADPcVl193nYJeKcSL1Vf1frp/5wv87WIsDbG8Pzbfzoq0vaF0jCzLui16ILdYsFKhMKhQ12s8ODFci7+LOGanQJcr2mjJ/7zuHb/NZjvPPVDhdfdW4tW6lqJJ95spXFeK7ov26r3QrvUdQy+NIgoXP4W2pWZoizyWDC/LM1/mF6Mu02ZClJOsRJGgPu+t7Xg4LXsm+OFmHajtSotUYpD1lolvX/AIbU3NEh4Z0rCklIIUn+U6g/Py1taT0aaxT/AA4VRL3orwfbAGwCsrn3rVbiMPk/WLV3mJRfohyleQZVOswbaXA5HT7raKwDPQ8utuToOfibbwNlzxCwkc7cz3tVfrbZidJMTYMkphWliNKgUHUp7RoyIB982YiT2NZXU8Zcr+YCYEKAP3zZ/GeXdmzFvlBpcQvI2D7CFjT+UqBPzFrtOdbKpUt0uuSsT5bRbCsjUfxHTf6+jYuVQoiO9PP47212hKj3TrFr40CziTzRgLkxPObbCpkGR52Lhz2iTIB/O2docsCNefIWYJIyl3vROx5iOdjt3KQFhUgzpFkrPBJ02JMn8bHbvcJcmfHW0kk4gWo1Qj/hNssT7Y81GfP9LZZJLxJbShx1JGSEpRIPNZGumnnJ62bV89yoAUtJlSiDlMefnqZjpZ63E80KlxL1zIqiWkBvNKshCNB3dRvGvSzNvgMP3wrs8jas8DtDlAJM8xtrEx91spnvq04eBFKC+7wupLwu+scbK0pzAKgSkzm035n3CxW+sSXxfxR+9LzfqAyT2aHHVKCQd45chtZT/c4p7tdr3XkunIowkBSMoHL5a/1snYeo2K6vU3VoWUpSe6kgT4a6Ry958rWLNJzWQbuAy3QF3FCXD2iQVESkpgR5+74QdrLKVJUITUNpCEqJka+cxrZMokssXv2N4JBaT2mZJOhACiNt+Xy1sUefcIIWo5hy5DroP05b2kRdII6muLhLSFEpTKYGw3126A+dh/3felQhVSxTOLGhzhJJ0HMgfXU2IZEC71OhgjMYTz1nWOQEiB1087LeD6p9mrSz6yOzfaX3PaPdBIMHy+doW6KXJCUV2VIuS9XKqlXTuoDQQtwHWTqOh2nnZ+ejEioVxbui6C0H2b4cFFUtkqnIVicsHQyBy/KyDVsXtXUblFKEKbbShtZhISU9IE/HWBp4yZ6Il1Vdx8fsJ1FYtotpvBlSggkkAKkjY7wPz52pYo8x0cjgbWB1VzDh/wB1HpfUadVde8uBV2Xa+Gaq7rwoXCgKLTxKCUk6HUAxvZEPBmnrsU4cw3czvqb181RpA68VLShWWQTp4G1sMfuUeMMRm87vzqZDKG0lSCNt9PfaJeJiavA1fhTFtIgKdu29kvpaOgchCiUz4gEc97eN0ONVLsVFO2YvZcgG+hXqMppZ8NzGJrJSOmxXnp6aXDi8+FnHWqwdeNUxWv0hoXg4ylRQtLjKVCJ1G5BB3+/1pw+ofuahIE/6s1zn+EW8p/T54hL4m8fa3Fa7tTd6Q1RsBie1VlQ0BJJAAJ18reqWG1RcN3yFf+as79OzG9vXic0TSvMg1zXuDt+aVnVg6D5dLAqI8QbcrcMxynrYFaySRMWkxM8armpV3CkdJ1HSyPUqIVAEFPysovLTBnTUeBsl1RAzEDW12MKi/dIWIKf1qiIBAhxCpUdNCDaL2EoVGcQCN8vlr9fO0jYnrPV6VALaVBbyEAHkCRr9eFo9pUutOB0t5pGh3kf1tUrCGkXWhhoLmmyO05GVLKQQc4GYnXX6FjzFM+KlFKkoK3FJy/agATy6AfL3CxJgrWorICNgByj32N0TgaWQHgYJ0kc/K1FzxbdarWuulO/7jvbDV4uXZeHZOOJShYUy4HEFKkhSSCPP60NkqobfqmXUobVJbUI8YNjoq6hxtTDDKio7QkmPrWwoum9HmVLZpKrMpBBKWVHceAsDjR8yEcMdbVIuHrveQw0lTZbBZbk+Q/X7rOu5MP0d4vPU9ffjF2IbZK0uOtlYcVIhIA2Pn02Nke4MN32/TU7q6avP2CBm9XVqYH62Muqu273HEVF50bK2yUrS9WNoUDMEEKMgzPlYZka7QFTa1wbrokS9FNt3m2M8EsqAI0kBX6iw7UO4aqG1GT2LonmNCPOyRfuILjor5ZqHb+ursGad1LrjdcysI1QQDlUTJ1IToTBgGCbF/wDSRwvaw5WrqOI9yM1YaeU3SKdzLUrXKCAeZjlztj4xC+WAZATr0VuilZHIc7gE4b+dKrsuh4JWPtKUn/iGtm3ggOKvTE1OJBF55xoRllCfft+W9nVfxQrDVx1ly1CaxippqR9DoQUyhQBB1PL8bM6kvG9rtvm+zdxp1OOVKFLKyAIyR+FsKmpDJC5p/mqvzSDOCnmlt5AJyKKIkzuP6zY5RUL1RSVzil5S2ylwJGv+0SP+q0VY5xRxDwi6husfYQH0B1tLSWlHL0IOu0dbN65OOV+XI5XVmI311tK5SqaDLKmEqzykgwFTysX/AKenkjzssVU9rRRPyO0Tox9QuOXxTuLGqm0gSNTr/SzPxV/2XdV55cpKKKncCjpIRVoP/V8DtZKvrj1dN53k3U1WHqzsEo7PIKltCyudN0kRqevL3iO4ho8Y3BU11NRO0jNRd1ewGy6l1WZtxhU5gACNZ26a26mhopKenayQWssipqop5XFhusxTj5Vx3+miSFLF4MJpYBEQuCN+UT9CyTw3PZX88862EuG/DkSnUpSQkqMzp7XwmzjS3dKHGEFvtMzJV9otIM93WSFfdZIwq7SUWIrzcdbQyo16VIWupSmJbQZkogjx2kRrbS4DeE4W5KjxnGQa6XUv09Uhu7MRMOOIQfWUKTmIT7SBtOh/OzXdbcq7oYTTqbUvsEJKc6R3oEjfl+JstVV94GpmLxpqi9roSy+8pxIqLwRJzNhJgjLPPYWR0cScAXXddLRqxLcNMQgIQtNaSox0lcHQTNuJjw+UPJa0rqHVERaLuHqkekoa5FeCtATPtEyd99uUdJs4cS3S7V3ItlS0EKSOoPz23087N+o4rYHdWFnGVM62ZXmStS0wn2tUjoJ05DlYzenGG4Kijauiiv5au0CUJT2KkhR0PIagzGvjbbpaeZrgcpWbNLAGkZgmo7Shq65UELcQkoS4ESUp7RWk7jy8bBO01S8ciKOoXuYQhR5jw/H4WLXxiapu67luNVtUwh4uaNrUJggnQEfzaT4+Nm9XY0rEUxfCbxqkhJKW0Fbi1wNkJzySSdhbfaHbBZBcy2qdf7pvBaNLnqxlAMinVp4gjxn5b6WLVV2VyKZ9x+kXlYyrmBKOU685HMTr42IXthvjLd1DT3m/wYxE5TV1V6oyS/Rgl3IpUKHbkoASg95WUap/mTmJ0VxcXu3eeq+FrdBQMEKq6moxHdmZluJKg0l4rWe8YSkEkiBJ0tZjhmvchV3TRWsCojrHSvE1SkupPeIKdJAzRvv9bbWbeOkF2/G1KdCCKJnN0iVZTBBG86/fFlanqQb4vV4rlOdScs65dTE6gCBPXfbQFJxasm/EuIbIQKdhBOUd2UEjYHSfLzOlrsujFUjN3Jwej9CceN0RqG6cKp6oKU7IiAT3oB1OXpufOLM4arqejxHWsovFkrdoElBSXMqsgdVEkCCCkHWPZ+NXOGVQaXHdO8w+EF1l8pJCRP2e8mRsTtpoDrraacM3ktvH11tOOhYrqd5gAgjvKSoDWeYUI8uU2qGPO25RhLwnWVvb4xmVti6UUqcgbSguEnMRlTrH4xy+AmEapDi6hSQApCkkmIGoifr7osxTVIeTS1AM9pTMrnMBMoTp57n6NnJgysQF1TYRlJSDMHKTP6nbla/BE2P3VSnqHzHxJ9oeMZzIO0gwBvzPT8BavfpS3Kmnrrgxi1lQHELu2pUE8hLjRJHIZnRr/N8Z29ZSJCF6lWsj3/nFo74+3Z++eGd5paRLlEpFagp1goOoieh+uVpAG6qXe12sVawqmcVS1EaKRPfPP7t+eo0sni/sZYZa/wC2WG75u7ZS0GKhtG0xML2894mLOGhbpqgpadbHaglSeRI6ePnvYW87vfVTeriidfSvvJypzECOfMxMafqXHkpJ/cCcS3bUVN4XZRVIXR3xSds0UmSlxCk6EHYkQAD06zaXBdy1DOSRpmlQM7ePmP1tTHB9deuD8WC8qVxTK6KozOsQoJWhR1PwJ1O3vtKt9en7w8w/V1F2PYIxC7VsSHUqXStJzc4JcUfKRNuMx+iqaqdroG30Xa9nq+mpYHMndbVT2LuIGQJggxO493uHy5WaGO8QvYYqaZDFMFl1srClAAGCNPEwLQdV/tJbtaKv3bwukcu3vkJM+SGFzv1sy8W+n1eOIXm3f9H2HZaSpKS7UPumCRpJaTGw5WyYsFxIm5Zp9QtiXHcOAsJPwVNq+KN6dt2bNG0dAQDMDfwgmwNNi69r5xPc9QxR0xqEIfbbLtSmnRK49pxyEjQc+ZjzrbX+mtf9SkAYCwuSkHLnpnF5SZndQ+jZPc9M7GGdK2cF4RTCgdbsVt0/vBa2MFr3aFoH3VF2O0Pzk/ZNHjk4qoxziVwpQhx283nXG0LzpSpa5gHmNdCNNBaKVgpMEROsRZz44x/W49xBWYivWipqd6tKCtmjbDTKcqQnup1iconXcmzYdWVrJTMTp5W7KkidDC1j9wFxNZK2WZz2bXXEkn9LZJtsDwNukMur9htR8hNrVwFVAJ2QfKJtllWmwtiWuKfUsPXlUZtuypXFz8BZbpeDvFiuT2lHw0xQ8nqi6XyPjltEysG5CmIZHbNKZ5tlpFZ9HfjY8ATw3vlmf/SGex/8ZFlam9FTjfUkZsMUNOOtRflA1HuU9PysJ1XTt3ePUIjaKodsw+iiS2WnOl9Dviq8tCau8cKUebm7fbS4/wDZZj+NlFj0NMVqhVTxIwShEEns36x1WngKeD7jYRxKkG8gRhhlW7/xlV7sICezSeijr8LTze3oj3zRNJ/d2Pbnr3ioBSRSVaEBJHtZuzJJ20y9elku+fRxxDT09IKK9E1jwSUPpRRuIDcE5SFKjMDsIEiNQLMMSpXbPSOHVTN2KYLk9MbBVFhu7aGrwzez1UxSMsvLFQwkKcQhIUr251IJEj7rW89Abj1c/F9vGl03fdT1Cq6FUNRDr4cLgdDqSRAER2aRudxbzYZ9GnGjzq0LrqBhAPcLpczKHIwlJA5c+trZ+gDhtXAPiPfVXizFd1/u2/rp9WVkDoKX0OpU3Eo1MFYjxtkmnw+JxfEfEtdtTiEoEco8K9LiBBBMdPK3BEkFU2baOJODHBrfraY3lpwf9NuzxAwhl1vtpMSPZX/8NiAhQIKXzJGWNPG2lHUkiROulm8riHg3VJxBTyN9FA/MW0OIGDlwRf8AS6kcjr8rFaQoWS+ZOhI8bMzGgTT3vdtSRHbIdYn3JVP/ACmx88RcFqOUYho5iIkzt5WaHEDiTgZd30dQ3iu7ZZrW0yXgkAElB1O0E78rHhdZ4QJm3Yhy6mZzdd9ibaU8MsgQAJ56+P42Z6+JeAp7mMbnVzPZ1aF/cdbAL4r4GbGX9+hYEqPY0zzmkxplSZtqBwssstN09i6rKYIk6aibYHV7/GZtH7vGLCKE50G9HQRmHZ3dUQQPNAEe+xN/jXcwSVMYcxJVka5WqJIJBPLMsfdZZ29UshUmIdklWY66QB+djdC6rPvpO4M2h5fGrKErYwRfaUknV9bDWUzz7552CTxxr0OpbYw1StKkgiovZtKiY6JSo/XnZcRvVNlKnr1hX/dzbLQOfSEqp790UKVcx+8FaH/2Vss/EYnyFecWG73prsrnFVlQ23IahM75Up1kcgNbN7Eibscqn30XYhROZye2V3lFR1A6GR8eU2WLpuJi+u2XUuVCXG+zA7MBUS2CZ8dPmd7GXMNUhfX2y1uBKYcJA1010Gn142zmtANwjuJLbJnUrwVhZ1fZqbydqhISnSYOniYPwnyshXRU1LDjjlMklcphQBOn3/XlaWUYXu9V2Jo2WVntVLXlRmTMgCfl7p8LN7DGH6Zq9HkdittxKCCHASJCtToYO/hpaecBR4ZTFaQ/WV+iFF1ZIG4JVOnOehtst1DAW28nIuSlQOneG8+P66WkGluSiOK1JNK2odqVEqOitAoQM3htvvFiFdRUbVa4h6kZLnalJzJBJUegJ8fy52YypxGmW29V9kA6oRlkTrrPv5nfw+JzD5q/3nSpacASlckjURGun4+VnOlttlKUM01PCQCSGkASNYk6/lB62cFz3u9dylKPqq0ue0SClQMKlOngB578tYcbyRBEDuUSYvphwlo0z5LitSlgke6PADkd7SRwVxjdGC+JOG8X3o1XPUdNUmoeSzSlTmUaEAaA7zG/SyVdlfc9Y6C664lxWhBWU5vgZ5bG1g+B/EC+LsxDdVw4forhpS66hAqnLoZcdKSdu0UCrmee4tkYxUPFK9uW4IN9baLYwqlD52kOsbi31XpFwsuW7cb4RuvGFFUrapb1pkVLLbzOVxKFCQFCdD4WH4o8BGOIdzUd2ovlFCKWo9YU4aftMwCCIAzDmRzsNw2xOGblZVUpr62qKYcdRTHIs9QE90e6zyfxfUdiQ1h6vfCkkEJyII03OdQtw3Z+HAhA2SRtpOW5RMQqMQiq3AO2Pl/deGXppXF/ZnizUUyHy+lYp1BxKN9FAbmeU/Dbe3q5ht1IuG7gND6mx/8Ag028qfTwvBD/ABuqAzSGlSGaXKha0lSAlKhymNj7/O3qVh11P7gutYWNaGn08ezTzt3NEXyUUT37kKvW5WVcgG10vF47lQ0PKwC3SRvy26aWA7aJUVkE2CU+lIJBMza7GFSedLrbqlEEkiB87JtUuZB10iQRYR6qOoB+f5+6xB+qSoKhRB56xra9GqLymvjZ1SKBASVZy+0BB/xg/ODaPF8VqClSpn1/DzawezCTQUxUFD/MfM2fmL1F6gRlT7DzaxHgr5crVSqOG2Nqm9qx5i87oZZdqXFN9rWEHIVHTKlJ8PeLUq+kiqbcVXcOqZIL5AprPG1i7kKzYioGAhPtNUlKiI/yoNkCp9J6qF1m+6CqxjXUJLQRUUdxVaULLpR2QSoU6Ukq7VvKATOYRpZiNcF8Q1zeStxPcDJMSQ68RvB/2ImdrOO8eH/Em9sDVeA7x4yXPRUlaw005UXbcznbgoXmzJUXUgEnOSAAJcVAACMtCLDMPB8S0ZMQrLeEIWo9I3HdS4UU/D3iw8YnM5QuU6cpkSC4tGmh6bWTneNnG69FqbujhHfThADYFfiqjp1KUrTVCnySIIOk72I/6Dr/AHFKcqvSYxMhayVqNDc1PTHVOUmUKBEjf42Nsej5h+sR/wBuceOJ1eUjLJvdTaQOaQnNtoDHha2KLDI/hVc1mIO529Es4Yf9IrEWFf7QXXceC6dph5d3KTU4hdrSXm1BJlTDZb1Xoe/IyqBE5AuvLt53tfF4V15VdV2K7yfNU4yw+0tCFqkqCVQoECYkHUczM2sfdvArhlctyi56XHeN3qRTrtQpkXi0ErdchTi1/ZHOpRSkkmSYBNspeBnBK72g21SXupAGVKXbwCRHMJyISP6Wm2Wji/pMKG9lVN/VcoHo3nkUF5tpfcUvNTOZyUGVd8SMqfjp7usfXyy47er9Q4+lt4p7NYLhB5HknQ6+O3SbXd4f4C9HGmxXX0+MrvdaucXch1PaVdW4VPpdEA9monYmOXnZ+P3B6ANM+updwg1WOjvKKmrwcnTotUbco6+NrcTGyNz3A+qozF7HZSL2Rz0VsE4f4pej7cddf97pp74uxs0NQGHy802lMKZ33IbW3JmCQYkEG0k1Ho18PXX1O0+LLypnnoLq2nWyVmImFJIHlZiYT40eiZwsoHbt4e4Mq7ppKlztHWKG7ktocWB7RzOCTrv4mx2s9MLhAynPR4LvZ/SQVtsJnX/MTbn58BvMXRvAB+q0o8TmyAHN+Fxf3oRYIxRXiqvji1iJSEApaQ2ikSUg6xmLZnWTtZEqf2efCzs0pZ4jYodMqBUamjSQDMQBTGYnn0HvMv8ApuYNbVF3cO6lQCZBXWIT9yDZPe9OELBFBgBhvkC7WqUPkkWsMwt7G5eNb6BV3Tve7NlufOyUL4/Z58Kv3c0qhxZfoq0tgqcU/TkFQj+ENA6gkb2QMbei3dWDr+wnw+wPU19VR37QXww7X1ie1RTvrabKc/ZpASIb209k22z6auI6usRSruC5qJtaoLi0vOhOmg9tOvnG9ka8fSsxpfF+Plhu6kO4eD9dTON0xhZFO4lUhSiDovad7F7iGnMZnH7JMllB90LVL+zvxIxTNPI4o3SF5NUroXhlkaj2zrtrZMT6C18XZTXhdruH8N33WvU7zTd5OZp7ZeYh2VpCgUk7D+YiYSkWGvj0weJC8L0153XjGjF4uPllylTdjAI/xQUkxGszGsaGJZ1T6VfHerckY1daSdYboqdvMfcjwI+tbrIQzd5P2CBd7twPyjCPQh4vmmYo0YCwAk05KkOu1LRQoqHflJplHU6iSY1EmbLTHoX8cHDTn93cLaQMoS3lyv8AdSkJiOzZTGoJIECzIqeP/HSs7Qrx1iASIhtwoAPhlA6GyU9xT4xXiSKjGOJXAUzlNe8fCdDtp87HbwhyJQ8khO4CnK5fQ84yUlyLuivxtg9IXVLqlONUdQ4U5lBQSmcsgQB3sxICQfZTAFD6CD13qZqL14pUqDTlJATdsAkebvMeW9q+V+L8bvq/7QxJeCgBr214Lg6c8xjxsRbxrRUz03ziqgayJkJevRoHNGk5lTGg8dbJ72W9z8qTIjzcrcJ9E7A1WpuivLiQFoQFANs09OSAQJ0UVxsOX5W096IvCSmpyKziDWtrElFQxT07LqCdlBUGFDkY5C1dMN4lbTTIvK78V3C626jOgU19Urjzic0CGkuFyJ11T+dlml4juVbZQKhS+zJHNMwddSQCPLfflbn6yjq5ps8L8jei1qd8LWZZDc/ZTnjPg9w6xRdlNc2IuOuO6iloK1q8aZNFX0dO43UoCsqwpqmzpIzEyFTt0Fo+vTgRwHp2H6g4p4j3vUsIW4x69iuqca7USpMp7oV3tSCCDPjZpnHt3sNntjVJcAGvZiNJ9qFGPnYgriLQh3su1aWTplW8WyN+qDGmpHKLPFSVgILpnFT4lK0WDAql0CEsXrXNF9anUuHPpJUqdgBt77JOJqpFTiB2idVDblM2oK3IITOnTc+8bRBsp36WrmxtX05ZkOuqjMqcxMgCPu8OUmS2ceN1z96M1FG4e19RQ+dcoSQpYieum2sb8tOjlbdtlixkBylT0e37voeJFEbyu+mq30M1QaRVMJdQrMytSjkUCDEaeY2MWsXe98pTWUj7F03XTt09ShRRTU7bJJ1BEhO0aeHTSLU24ZYrql4mu2qpUst1zVQGlB/MpuSFAylIkjX5HQTFp/r7/wAT1jYp2a65UkfaZfVHEBX8MFUEjUg7fwaWqCIk3VgygCym648UhN30zDyuzNPTpQDnUUqygDSE9Bz+Vlq68e0NPVlDy3HEpbOYyNDIMTIJO2w+Vq502M8ZUSECruRFWB3CaN5KsoiR7WTSYnbysoXVjqrZce/fFxX22jIChKaBbpJBB2RmhUHrHnaXElZsVFscL9wrIP8AFK4kZkoS8SBKRkIjTp08fumyVf3Eq77wu6tu8Xc86h5pbZ+02RGnLUaRaETjWpeSr1bBOKXs47uWhQiZ0ABWtMeI5aDTkC7i3EeSWOGuJVAOEZnDRpCvH+/5yOWnSy7xKVMU8I2TPrHKuiUUUDSu1Z+ybCU5u0ImAQOcT+HMWHouIVdSU/7vvu7F0r7gKEPAKUiSJBmSY2jXTxto1lW3eKk19zuXa+9Ck0ry0qVB0IBQSD066crKrDRq6IU1Td7eUjKlK/aIOhkeGkxy99tJhLm6Kg8ZXWUbsLqaXE7zlfVmqdqGg4XE6COURuJHmOe4szMacGb9xziFD2GXKFtfZTUduvsUp2gzBnf9BaR8cfuG4nKOloe0L6AXEyvNkQDvqO7OnhoI3gH8DVT4vlhDbjSfWQCQqQkRIymNtz59elGokdELs3V2BjJSGv2UOU3olY+djtL9uBtRMZfWXFEDXUwjQaWU6X0PMSuEJrMW3a2SrKQ00tyI33jUWt61hWqWjtV1rKQoZpCSoRpB5HXTXXnPiZ/stTlcKeUdZlLStOeknr5fxG2OcTqid/wtgYZS72VTqf0OWQlKqvHbpGpPZXcNvCXPqD0soseiFhgNgOYivl1ZJkpYaRtyAk2tMjDTCE51POkhQUQEpkkxA5/QHSLcJuRCAQpDnfkxMFOwA2jX63tA19UfiUvZ9KNmqrTfonYZaWVGsvd5IPsqLYkf7v52UKT0fsNU/ZB3DNO4GNQVuLOYwPaP8Xlt8bWabuWlChNKoKToSXOe/wCP1rY03ctAkqBaHtaEpKpg6+fh+dl3uodu4pxRU7dgFCV24FwywUzgy4E6ADLc9OpQjlJaM/ebPq6KG5acBpDZpRySw0hoDyCEAnb5+cPj9z0rawOzAGWYCYKj+f1rYdm62ATm7MA84AMeX1vFqspkk3JVmJjGbJvsNXIdDWXs5rpFYuNOWg6fcfeaXdOHHgC/SV7p3lSlL000knXpy99loUVMSFtvZeUjlGkdPrW3WRhAKu1Qe9BzGAB4/XO1R0DirrZmgJDTh3CiYb/ckhQkjsEpgaiNQLGG7luZuOxuhttJgE5ANPLz+7lZQ7ahbQSKxlMmVBOUFXnHh0sGuqusArVeCE8zlTqIB/KfjZhSOdqUu8sCLqo6RlIZao0pTqlMNhURp1Ej68LBOUza4LTB/l9kR8fcLGFXrdyYArFLSToAFd0jTSdP6+8l3b9oUd3I8SgmZIBjkZOoOtjNondEJ1a3qiz91B0LUhpRUnYaajruR+FijtwuqKciQDvGYjofhpt1sOrEqKdXaN0rqwoc3SBttsQPMfGwasXvhGVqna1AgBRMGfCJ6WMKF3IIBrWdUUXhN5z2Xm2wZASSrcz+njbTWAXg+ioTWUh7NQISEqMkeM9bCOYsr4zAhsrICYRMyPnufxsFUYlvFxMh9Y1IEo0O3h4/fY7aF26A+sbZPz+2XElCswvO7VJTpoyohI02Mxy38ffbSuI/ElCEhN53eVgzm7AamBpPLytHSr/vInKu8HSknNCVQPl1/CwDl4vKTPrfaSRCQdxvvP18rXmQOG5VIzMOwUkDipxJaUkE3OEEz9oyTpM8+nu3sIrjfi9kBVUxdUHMZCwnnOkp2AP1Fovdq86lAJBInQmdvMzsBrrysTdUUq0akpGwMR0+psdsPVAdKOQUqVnH7EQaCWKOgDhAClSFCdNu7tr42a198Y8V3pdtVRFqk7VxIKc9OlbIWFDQpyDoedmWsvAKQGgkCJ70fHT62sEITmQAFDMNAZG+nysdkQaq75CUZVxEx/kaDDd0UyhGjdAgADfnr0/WxR7H3EhSxlvlhhajrkom4g+YsCoA6KUo5pEZjIJkfhYFRSk6uKWknuxp7v08PjYGgVY3ujBxVjpwfaYsrBmH+ybQmOswNP112sVqLyxHUFPbYsv/ACohWRqrLZUImO75/dbgF0FKQoBMmdNvGfP77chLqHFxmk7FKfa06c5185s4N0yDU7W5i65e18vKByku3g6sHTmCojb7zrbVJQUS3FOuUBVJzSZPIHmeo/pbS1uFEIKswMa7gzBiLd0rr6HA4VqMEbkHNy/Dpz99nUeaV1UtMklPqdNoY9q2WD7cjTt6Yf5iqffbLJSUBYTpKpPrLaLpqHytwFPZtKXICEyrSZEzp8jY1U3NiJ50qZuGvymFaUjpk7b5dedkJ2/b0Cm6egvyoQ0RAp6eoUWwEgbBKvLl46WDapMR3stTbVLetToR3ad5yYBGmWQTsCJ8bUuIAbkhHLdLWKcbVz4mYJdXd1akDSFslsak/wA3ujTbrNkeow/edO6qtp2qRhwuKWQutaSVAmCIJBBj5iJ6KFBws4g3o+kXbw9xJWKCVK+zut5YzR1y7jbSZjWdrOSl9HfjdVDNS8JcUw1lzZ7rcbnyzAT0/O0DUxDd49U4he4eFpTHqrhfdf8AW275oGnyElYVXMkGCCYyqJGw018bdVtzUtc0n1jEN1IegpW4ha1FQ3juIIO206T8ZFovRR9Iy80hFPwiv1CETlLyW2gnXYZ1pMf16Q57k9Cv0laqkFK5w9do0O95zt71pkpXERKQudvHkdp1G6tpm7yD1Ck2mmcbZCoKqcL3aw32T2LKNZjOvs23jIIBGnZwIE6cjHmCbtLctOMovqocy91RbpjuFTOsRz+trOL/AGenpC3mpta13KylOUlNTeZB/wAQSpCFxJJ8tOYss0n7M7ixUECuxphqkbUDIbW8sg+A7ODvvpqBpaHtKiaNZAiCiqTsxVIbqKBltCk11cQkDKCyk89/7wGZidPnZw3NxavfCt60943TVVyamnWFsudi1lCkkwqFZs3vEDpa2NN+zGvpsJVePE670LUZhmjWoZv94iRJPQ2TMRfs6blwxQqr774yKPfhDFPcwCl66JTmeUDtzSRYL8Uw+UZHOuDysjxUVcw5mCx+qhwenhx9pmSzR4wvVthIlXZerIyzAGgpxGp/PpZv4g9Mfjpf6CzeGNb7fQhSsgNeWpO0DsUo8eukbb2ma4PRQ4SXSnNfbF9X+oxnTU1XYtKUZIIS0kHmNCSB42Wbl9GTgvdJXWN4McfQ4uQ3V1LjsRuEdBrB5bedqjThEJvHCPQKz3fEX6ufv5qnbH9suOOM7ow9QUia6/L4rGaKkU7UOEurWopQlS3VGEjMdZ2mOQt7SXJhvFdLc9BQruJ4uU1Kyy4ovNgFSEAGJUOYtVvhlwr4WYQ4gXBjFOEbpuZu6rxYrF1qlrZRTpQoHtCoqCQkAH2hFrbv+klwIYPe4s4ZJBBhu8EOH4JJNtyidBXMuPCAsqsjmpH2dqSgxh3E5VBuohX/ANK3+CrYrCOKlIn1Sn22L+vyBFir/pR8CpAZx9T1OYSn1ajqXfeMjZ6WAX6T3CpwD1SsvurB1HY3DWxy1lTQHS1zgUzd3flUjJO7QNR5WAsVPCJoWxvq+o/9FsHCy/XR9re1I3m6NqXr8R4WSXPSZwiof6phXF7+4GW6w3P/ALRxNgV+k3duUKpuG+LFZpyqd9TQFQfF+0g+lZ8Q9UwhqH/CUp3hwSq7xZDNViZsSQruUZ3Go3c6gG1ZscYGvfBmNrzw/S+sXimmWhaakpS2FlaAsjeABmjmbT0/6S95rSk0XC6sUVSP9Zvhhr/wBdqocbuH2NuK/EO9eJLV3Ul0M3t6u09Qt3gt5SFMtJQlRWEICswQTsIg87VqmWleAA7VWaWGpiNy0p1Iu+/SEj/U2l7BK69lB3B5q8/rY41h29alAXUX7czM6FK7yQqDz9mfoWiZjhBjmmQlKr+eGQZUZaxeg/Lb4fFRa4L4mqaJyrfvlLrlOB2jfrC1fZye/udtARGkg9Tanan+ZXCKgn3VI5wuttwCpxVc6Ecsjrq9IP8AKj8bCfuW4WiXKjiHd7Ujf1d8GB5pHL62tF44NPPAl6tQhURISSofEHx+Fke/eEF7oeU5ddTX1lSEStptZSlwRqUDYK0iOc/GYfT7XUeHPvZTU2xw4aSpD/FBpa1CMrVMCJ8QpwWBdvnhPSJUl/GV5vKQdS1TMEGOUdrM9P6C1Z14dShYBaqMqVBCwXFdomdIIInQmOs6RYw1hdCnVKepMhnvZ1SUj+EHqJ6RGp8bSAi6KNpFLl+474aM1Ieu3EN4IQtvs3C/SElW2VGRtKlTuc22h11Fo9vPixciVrTSUd81SUHKFhumbS4BzAceSYPjrrrEGxWnuO7kKk0KAf4wo5ZPgTM6E6EcuekdruG7lhJpmG8hjMkwVpM8pGh5QdN9RFpBzNgFAxPKTjxKp3iS1hy9FOJBVDtVSIEDxQ6uN+li7vEyscTKML0YiCBUXupOUxpORhX1O1l5i56NLKltsU4UCQlWbKtI0kKH5aanewrVJRNlTdRT0JSD3ihsJIHOQN9ht8LPnaOSXCd1TUVxBxSs5qTDtxIJMgfvCpe93/m6RzB/G3CMe46qSDSsXAjLCu5d9S8oeEKdQeXSz0TQXeQFt9kpCBJOSU+Gk6aA+/3ACqpGgpCXVNO51EZ0yABGsGJn4fCz8RvRLhO6pkvYp4iVSM6bzo2FEQFMXFoUxvKqhfjuOVg6W9OKDJNSxiOrQp1tSF9nQ0SISqApMqbXuCefLY8n0UNtj7GrKm82Ud4kaK3zcvCZ326YtN3kOIDLqEoSJUEqUknfcanSN+k72WcHknEVuaYwRjqpSttWJL33yryro2gCf8SKcR7jYBVyYp0m9b/cBUZUq93U9Ng2pJPhFpCUllaA2h1LkwSYKUg7zIEQPvBsUUulpVQXSwUju65dI1PTmT8bLOeQUTE0c0yH8PXu+Slyvr+zMEB29q5QCifF73e7xsWTw+bMO1VPRvISSYqC4/mMHUdotUc9OelpAVeF3lkOvusLGXRSklJjTYyZkT5/G3CL8uLtBMoTsC4nuxGpJEjl79dtLPndyS4bBzTGRw1p1EH1G5BK9k3dT5iNdNUmDPjpPlYdrBaaAksIp2ERKizRtInXllSIj3naN7PE3vQq7zb5dBUDBToNDrJAkydPZHzkmu+Wm1JUgVaFH7Pvagxz3gQAddN+tnDnkJZWJCbw29TAJVfFfKoltuoPenkIIM6kRH52DXhRJKfWLwvFQX9p9pUq+ZnqOnMbQBZWXftRHbJSHz7MklKzroBMgnbXYR11sD++CSta6aoZW0Ncq8xSRz5wIB5nppzcZioEsCSncF0SnEreS4pYO+dR7x8NYI0Mc9ekW03hWhbC0JpAk94lJSSCkEEnczrzAIGtlJV5vOIWpCG3mykthOvtGQDI56fLwsXVV1jTZbZLK1ZSpKVNnQ6A7nYGOg231tINclmaNkwOJjBpryYrGGkDOhIzCEiBy+MmAdPLZIvqvURdFWhxWRbCmVKnTRYnfWe/8jPi7cYNO3wzSBCe0U2tSHFIQUwcoM7bkGI+in/6Nq+/rvy0VQhlxh9KgCNNRBHloOg02BsYtzNVfPlcmngfs7j4n3RWuUyX6f1hD5S4S2VpTm6+HUcx1tauj4i4RQM9XdTlKVgFJ9XQ4gzOwSoq2B3Gs6QbVWw4067fl3VTS1NVFAp1KylWVUFtXQydQIA5E+QerlVeLjy00ynV5CV5e3JWEiACQRO+kDSdRIJIrmLiI7ZuHsrP3XjnB1WhIZxDQU7RkZXkGmgiJMupHjqNJ18bOWlFDeQDtBX09Qkpkpb7N0AwRMp3G+418TtTF5+pUwXKt1LaHgMgeQEAhOg7pTBAHxB5wMum6ZapyJCnHSrs3GYCgYTqNATHdOnTnpAnUAJuHIza+27VdZN1A5JS2VA5VQlSZ1/wjl5jSdNLdKu9AOZypUkKVJCHACAVcp8lD3adLVAo7w4gtIU5dl9XwOzSM6E132aNDIyFW+3IiAIibLNPxB4m3WpNO9WULiFOe1VNZQrQj/ZpH825Mkk66pFoGiI5ogrmnkpl4jYeYqlU9VSvpFSiYzKSpQJgaeACR+PKzQa9YRlSpalPnuqQ2iQ3G+o03G8czvtZCo+KVbeakM112UruZc9uzXtJKQVCVFrMTHMQZ5cps5GKhupWVs5Ql6BKBqecCNOY/W2jTtyMylZ87g5+ZMa97lqXA5eNW12inHSvMUZlZUmBqRy2033625u6pXQv01UhnMqnUkkkGMydSNtOfWzrqKO8bzZe+0QaMKI7Rbm4EzEjUb2Zj6YfCCS4lKsgJOveJSojoTAn42BIy2hVmN3MKzWGsVXVV3QyFU7ilsogxkGh0CpmdgDz0B3spqxLSnNlpnFAnUKKYHnA00jYfHnCXDm+lCmNCXRkTISknvTPdnfl93xfPrpglTSBzCijuidDJ/p4TvaiaRm6vCsdbROtzFCASn93NgEc3BMAHUmBpJ/LbQsb/qHTnTTsJjXQkySNx4STy5WbqUuqzkJdgHVSRB93XlqND4cukKXklp+QCdMugnkep8Y5edmFKwJd5eUtuX/WlSiWkIIhIOhjXfU7mLF1XxeRVC6ooAJMBrSJ69dB8DZNQt9KZcQ2oKI2Okaajl9c97bUakgpLaTr3VJXJHl0P62mKdvRRNQ5GV3nWLzJU+ohRIgaKHhNgV1tSvKsOVZyynQnT7rBdmsKynMhJH8SRA6n77dlIH8KlJ2griZBmz8FvROJndUF6w+pJU6p9SojvqO/uNgCtbye1kFJAEFU6axzscLLKzk7IjWAUrIA6nU6b25XTEkykzoQT3h5SOf1rZxEByUeKSk5ZUqAqphZ/lVIJidtpP1vbaXHm0jsy4Qf5kyBt5QR5crGlsqSofaEzsVHRXn1+/SwC6dwgKSnTf2hOmm8aj8vO0xGFAyFABbgXmD6xlEzlj39I+X4cds85JCxrqDMeEEcxHzNjKYalYpgVezJ+/Qn657W5cUy6AFNrzSNMoB+Ea6eNlw+ibiFF3HCpJJQc0xOhM9frrblVW7JSpGaAU8oynwG3152NpbpJBzpSVCQI8ySdbcpaYSQpsz3dRl08NuW/n4WcMUeIgBeb6Up7enATvO8x4/X3W5XWBIhQVmJGkj468vlvYwpTYWEHs5BOp6chE+dglFOfIgIKpOZWWQT4wNPdaQYomRFV1UHMArKkDWNYnQx9eU2364gtlS+UaEgEmBrv9c7C5UhQDYQAAYypgADbQ+7624cQoFKkOazrJnpr+n5m0gwBRL7aoNT7LiSnKjTukBM6fU/py5X6stJiIImQ3O/u0+tbcvJcXK0rSUE5gc0Dr1j68rAO0j4UVpKErB0IPMg69Zm08qhxEIUsqCkpIBnSFEGdvyGnhzsEtDLqVBM6+ySomJ8J+XxtnY1bZWtVQkwCQpEj3dOR+fnYMpfy5VOI5zrpz1Gn1NnSJutBppMErSYEaD37b/1+IZQUAwsnKAMp03nTrYYiqOZBQ0FIESdpiOW/KwKwpCftWExMKTlMn468vlZhYJkCtxzMYdCiPEA7RMxvv4WCJyHs8+p0235zM9eU/GLGF1DSCEPUwCRoDsPOAI+EWCS8wrLKXVyiAUpMRHXbpp/Wz5gFEtKLOOEBZIWoZYIDJBPjvG3KwrK+8QUpgakwZ6g6672MpaSoAMB0IgElaD3p5z4WOU91rWnK0jInYBQiCT5T9RaOcAJ+ESi4LahmcYbzHVWnO2WWE3UnKPs6c6b5x+dss3ECfhFemDOCcMU7aW6a4ruZQkylKKRAAPkBY81c9G0kJS02ANgGwBbLZbysDmugzuWlUbZVAJSPC2l3a1spaiCLZbLFawHdFD3DYoBVOw0fYKvM20420kBRRO9stlpZQEZrid0A8hLeoSkgQdRbRcPZ5ghG8bWy2WgNCjt1tdJN8Xg5QUD1chtKi2kkJ2+doGxD65iWt9dvWvcWZKUoQkJSnyHutlssamF7kqyNkjN4epFvoQpxw5yATpznWwbt0UalZ1hZCQAlIV7I+vnNstlroUCuU3PQ1YVTPtl1CiEqDpzhW2457/Kwjty3RQVLtNRXZStBhZaBDKZOWNdt5M2y2WsROIGhQXtB3C6ZaOU9kUtgJBgJ01/rYaoazuoZmOzbSJA3mSfvtlssYOJ5oeUDkglICVBKiVSqN42BV/02FS029QNqKT3XlpTJkjQSfPS2WyxWqFkCmlRCtRokKHdH80fh87HKKjS6p1hSzl7FatBr3UlQ/8ADHvNstln5qKKu3ZTJUVkEgTI66n3cultUdOyxWJep0BK2pOsKCtNiOYiR77ZbLTCbmlG88NUdM+2thWRt9tLyUBPsBU92eYkf15kDhmjfLrSllJVELQkBSTMZgeotlss5UVGfE3BV1G4l42bKkVbDxp6pEAipCHuzCp/hXporXTQ5tCImcbp00yCadJLqdY00k7dNRMjrbLZbSpdWC6zanR+iIu3hTtU7MUeZNQ6lEKUk5SQVBRlOvsj36+FiSMQPs0jaktznqOxlRGYabzGuw0+62Wy10AKmXFGW78qX3mWlNt5ylHfKZ1UAdvqbEay/Lwp+wUFpyvgaJSEkBRPPYxlHK2WyxA0KDnFAu3/AHmahspqD2iVwFlKTptG22vPpYVmtvAVZoqmvdeStsOjXKBJiIHv+XSbZbLEDRZBLjddPdrTutvO1Dj32yGgCYgKSNZGvP5DxkOqU+h5omqdJdkI7x7iYBg9efTe2Wy0g0KNyiqAXqVpxMI7RAMCYB113125/pbpqrWhtmkqJcU64W86CG4iNSANfKbZbLPYJXKMIYaU6WinKSgjM3CfZ0H4W5ao1J0D6ozaAgHYkc9eXW2Wy0VFxKEVdrfbJSh1beUTKAAdCTE+87zvYuFR30rcIUsAZlSRKtRIAEeEWy2Weya5XCXKZDgacpyo1BDeYKAgDroZ38PnYneF4ponlqQ0tbhVBUtwGYT3dI1iOetstlpBMiDd8orVIUKFKFwO8VzG+2gj2jtZzXfdabyaCw4lkAxCWknqJncnTckna2WyzlRRm9sIquui/eD9e3UpXWIQEBgoIKp1nMQdxpHLygRm7BTkoafUC04jKYE7kfh87ZbLEbsou3UOfudqjxIwpDplFUlKyEiVy5BMGRyO4I185sXRcNcLVlR6oll5ChMkqTljLmgBKUn4kxPutlstUYd1ZIFlqqwFhy66pLCGKlRTupNa+gK15pC4P9etu2cLYdcZW45dTToBHdfJemT/AIyefSLZbLEuVEALlFw4SaUhJwtQBSnEolDLYGugJlBnfr12k2VH8PUN00yapimpBmyyEUyU848Rsek2y2WiN1MgLlla3m0qKjKBIkyJOm2x2G/TSLNN9S1XmrKoJ7VQTokaAkEbDkeke62WyxYt0OXqmnjTiJeVx3mMK0FIh1SkOPGoq3VuDRShAQkpj2eR2jpaPl8RXXXqti8rnadcpCUFdO6WkuAAH2VBcbxv162y2WHL7yND7icHDTiWbyv391MXMacBttXaGpzEkzEgIExB8dRqItY0pacAUtKoypV3VQZO/KPlbLZYbt1Jmy6FGlaA+lxaYbC45klAO/kQPdbWVxGqXlSO6CQDz8bZbLMnXSW1HtBn0B7o1EaeBH0Lbapg+o5XXEwspMkKBgnr5Wy2WSa5utuULjOVfrMhPRMHrvNiK3nUuDOsuELCDm1BM7m2Wy0gE91yupcStCFZVaADSBHl1sJSvl1bVOJCnAO+YMEgcotlsskgjd50TtNFQakKOYAAIIiVKHU8k/OyahxaiSFxBSmMoIPyn52y2WdJcuZ1uKSVDuiZj422ptJBJkkqMTEaAfr8bZbLJILHLuUlDjhfBAAhOTSJ23sVLUuKSCkajZO5m2WyzFMgkMdsla82UJ7sRJJBiZNtmmGVBSqAZ0InYn8vnbLZZX0SQakONFQS6QE6R74sNTtuuICy/MxlCkggDeOhtlstFPYI4i7FK/vXkrEbdnHQ8j42E/c1PGVxxZjQ5DEyJ5z0i2WyzOJT2C6YuGjqELWlx9OUjRTmaZ+Fjhw3SUrgQFlWROhjx5yTNstlhucUVrQtJudhTgADQUteSS0DBjewT9MhlGdpKE8kiDoQTrvbLZYYcbqQaEWSiUoeSopzlWmmgG2vXxsNT0frLvZLWkCSkdwHafytlss9ynaAumaCmeLjamkDTMTlmZIH5fCwzV20iSFKSqCf4SAdQDrIPW2WywwSpWF0IugpMxlmTO5O9stlstJPZf/Z" } ] } @@ -76,51 +61,28 @@ { "language": "markdown", "source": [ - "## Load the Model" + "## Do Inference" ], "outputs": [] }, { "language": "typescript", "source": [ - "const core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'CPU');\n\nconst inputLayer = compiledModel.input(0);\nconst outputLayer = compiledModel.output('boxes');\n" - ], - "outputs": [] - }, - { - "language": "markdown", - "source": [ - "## Load an Image" + "const inferRequest = compiledModel.createInferRequest();\nconst result = await inferRequest.inferAsync([tensor]);\n" ], "outputs": [] }, - { - "language": "typescript", - "source": [ - "const imgData = await getImageData('../../assets/images/intel_rnb.jpg');\nconst originalImage = cv.matFromImageData(imgData);\nconst { cols: originalWidth, rows: originalHeight } = originalImage;\n\nconst image = new cv.Mat();\nconst resizedImage = new cv.Mat();\ncv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB);\ncv.cvtColor(image, image, cv.COLOR_BGR2RGB);\n\nconst [B, C, H, W] = inputLayer.shape;\n\ncv.resize(image, resizedImage, new cv.Size(W, H));\n\nconst inputImage = transform(resizedImage.data, { width: W, height: H }, [0, 1, 2]); // NHWC to NCHW\n\ndisplayArrayAsImage(originalImage.data, originalWidth, originalHeight, display);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAIFArIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1SXSkkG1ihGQehHQ59a429+D3h+9klkY3SNKxZjHcY5Jz/Epr0RvvfjSgVEcPSg/djYp1pyWrPJpvgbpjKVh1G/j/AN5kf+grPufgXMYysGtvj/btR/R69rFLVciFzM8Cn+CetpjytRtHx/fSRc/kpqhL8JfE0RYqLSX/AHJtv/oQFfRtLS9mh8x8yP8ADjxXGrE6cGPolzE38nqtJ4M8TQld+i6gRnkrAzD81zX1IUVuqg/hTDbQHrEn/fNL2Ycx8py6VqVqx+0Wd3F/11gZcfmKgG4ZBbnmvrMWsI6KV/3SRUU2mWlwCJYhIPR/m/nS9mw5j5W+fAzxTwW3AjJAFfS03hHQpx8+l2Z9zbRn/wBlqjJ8PfDshz/Z1uPopX/0Eip9mylJHzZEzYI3H7x6/Wp0ZiseM/d6173L8KfDjA+XZhDnOUmkH82NUZfg/pDABJLpAOBtuAR+sdSqbTLc00eJbjuY8kdOKYWbHPPocV7DN8Gockw6lcL7NGj/ANVrOn+Dd8ABDqStj/npBj+TGqsyLnmJPI5FKCSSNxrvpvhJr0Zys9k49MyD/wBkqhN8MvE0IbZaW8mf7twgH/jxFFmByO9gn3j+PanAuGAyfxrem8DeJIcZ0edsd49sn/oJNUpPD2tW5Jm0nUEAHVrZwP5UrDuZ4JYnp+VLuPHofSnNBNEAJUdGzjDLjrQFJYZ5xUlCgnOOfes+d3XU4scHA/rWiAc5x7Vm3fF3EQM57596mQ0aMUm6PrnnAp2/AznNRwHbFv754pxOAFweaUdimBftkY9qCxAz+tAIJPtTSV25I6+tUSOJbOc03cc8nHelbHTn3o25Ht069qkYmfu5yD2oyScnj60u05xyTTgpB49KVxgCRTWO5Dj09afg5HAye1KVODnnIzUsZl2TsplHPbI/OtLYjYbbgHms2Bis0q4/WteHJgTjgipe41sQ+SpxlTSGAf3m9OtXAoAzwKUqMcLmi4WKX2f+EMffNM+ynPOCO2RWhsB4oCZPFLmHYoCyLAEqhP5UNZnHCd+gNagj9BSFc5qeYfKYxtW7Ix9/Wo2tGByQ6jPcVulMDnjHrTGjOwnB6Ucwcpzo3K/r2qdYtyAnAPrSTrtkPHetSwVXtACoPJB4zQ2CRlPZAIWaJD34ArF1C12xl0hC8/wjFdfdJHgoFGO+KxL6P/RZMdAc/jW1KTTM5o5bawPANHmOP4m/OrxT3PPrUezII6/hXZzHPYq+Y/8AePFHmHvU5jGM4H5Unlgj7uPei6EQbzS7+MU8xjHIIpjIAuRmnoGopJ4JzzTd555609uY09gR+v8A9eo9poAXfil8w+lNwaTB9KYXJFcA8jinmYYwM49ahwfQ0lKwXJd4IxRuFRUUWAc/NNoopiHI5SRXBwQcg19uaPffb9EsLwHInt45fzUGviGvrv4X3pv/AIa6HKTkrAYj/wAAYr/SuLGLRM2o7nXlzSbzSUhrhOiyF3GjcaSkqh2FViHYZ9DT9x9ajHDj3Bp1ddPWJEh24+tG4+tJRViF3H1o3NTaKYh28+tQzMd8bZ74qWopwfLyOxzQxoXc3rSb29aTtSGkAu808OcVHTl6VLQ0O3tVS+lmjsriUT+XsQsCEzjAz+NWGOB0yT0FVLyyF3aTQzOxEileOAv0Hf8AGpavoPY46TxpbC2huJLdpZJNrBSqgpn+LO44+nB9ap6ze6fKJQ11cm7JCNGMny/baOAT6MQcVn3ekwXlhd2YmZLzT2MfmNgkqBz379f0riFvtkUNs06tPBJhcAFVABODkc4PvjNY1KV4tGTqNM6z7bpwHK3ue+JUH6eXRVJfFjBQHS4LAckAdfzorx/Y1OxtzRPdG+9Sjmkb71KK+wZ56HClpBxSikMWlpOtLQAtFFFABS0lLQAUtJS0hhS0lFAC0UlFAC0daSloAaUQ9VU/UU3yIv8Anmn4CpKM0ARNbRMMFePTJqnNoOlXBzLYW0me7xK38xWhmilZDuYM3gvQJs7tKsx9LdB/ICsy5+GHha5YM+mRhh0KvIv8mrsaM1PKg5mcE/wm8P8APlC4i9kmOP8Ax4GqM/we09jmC/uo+ONzK3/sor0uijkQ+ZnkkvwclCsI9WYk+tuP/i6ozfCHVhjy722YL03q4/kDXtNLS9mh87PBp/hb4hiDlBaSn/ZlIz/30BWfL8P/ABFABnTVZRyds8bfpur6JpCARyAfrUukhqoz5sfwrryKXOjXzDtshZv5Cqcuk39vgT2V1F674WX+Yr6aWGIoAY0/75FH2aH+4B9Dip9iUqh8ulcM249PejYdpwck9K+n5LG3l4ZC31YmqUvhvSJ/9bYWzH1aBG/mtQ6LGqiPlfaUu2XuRxWxaZNqgPQZ9s179N8PvDM7bn0m13eqx7f/AEHFVW+Gnh/btithGvoryZ/VjUujIpVInig+7nH6U4RbgF5+tevy/CzSWOY5bpcdB5q4/wDQDVRvhZEhJivpRngZQNj9RWbpTLVSJ5gIULHIGKcsKjr1Poa9Gl+GNyMeXfBsf34sfyY1Sm+HeqpuKyWzHtw//wATWbpz7FqUe5w/lgtxkbR1PemmI468n1FdbJ4E1tMYto3A6kSqP5kVTfwrrEWSdNnJ7BBu/lmo5Zdh3RzhhOSBj3OKYyN5fbn3rcl0TUYiA9heIAOS0LD+lUJbd4j86upHQEYNLUehyd8hVyCBnOTg1a06YrasOpDcUmooVDjHzNyah075oXHuK0SuiHuWXOWz3PrWbeoDDKD0A61pEHGe9Urpcq4xwVPvWkCJHOsPTnHvmoOTxjg1YJwABn+dQHBFdaOdjW4zSMBxnGfrS56kHnofemknjI4NMBDjpj3pj/dbFOOcd6aeQetUIQH/AEccfdfr9R/9al4OB1zSIMwSD0INIMkUwDC0YwcGjNHX+tABjNBUDijnOCOKPrQAmP8AOKQrTs+2fxpD7UCGkYptOI4pKYhK+mfgNffafh89uWybW8kQD0BAb+ZNfM1e7fs73vya7Yk94plH/fQP9K58Ur0zSk/ePcTRmg0leYdaFopBS1QCHgqfenU1/u/TmnV1UtiJC0UUVqSFJS0lAC01xlSPUU6kNOwEKcoPpSmmowO5RjKsQfanUgEpy96bS5xnHWpY0OAJOAOaazwhXDSKMe/Q1Onkqu+V05H8R4rAvNathqzad5roWXeFTjI756n3wB9cVlKryK493Y4DxmZYfE32/TpI0VodkrSKYwTyMFiAOmfeuJtNAxOupahf29naysfKUtjzeecd8e4HX1r1XxBq0ElosEttJNG2GSJISiJ6HLLtyOvXFeWXfh+91L95d3iR7mLCWc+X8o+p549K5Z13JWbsS4JPa52McPhJIkUMjAKACUbJ/wDHxRXMx2Nokao19bFlABO1+f0org9k/wCdl83kfQjfepRQ33qUV9azgQZqB3xubLAD0NTmqsoysi9MgimhM5rR/iJoOtXkVrbXM8c8pwiTxFdx9M8jNdYsrfxVyFl4Ksra4gnJV2iZXDBcHI5Fav8Aawl1k2UaSjyk3SM6FQSTwBnr0PI4qrCub4bIpjTbTgDOOtNQ/JWL4mjefw5eQRsVaZRHkHnDMAf0JpWHc2I71JRlGR1BwSjA4/KpxICK878H+HH0LVhLC7LDMhSSPPB4yD9cj9a7rzNsuz2zQ4gmXCwUZPSm+cn979Kikb93+IrivHGoa9YT6edFuFjDLIZUaJWD4246jjv09aXKO53gdW6MDTs1kafcyS6da3NwoSWSJGdRxgkDI/M1oq2RS5Q5iajNY+p3rWOm3l6kLTNBG8giU4L7QTgH8K5Sz+IbXLBToWoqx4wh3Y/QUcoJnoeaM1BHKXUHkcd6jedgXO7Cr7e2aVh3LeaXNctpXjfQ9ZuUt7DVI5Zn+7G0bIW78bgK6JJtw5FFguTZoqrcXiQFAxA3ZxlsU0XoI+4SPY5osFy5miq8dysjbQrD61MDmkMdmikprSKgy7BR6k4pAPJozTQ6sMggj1Bpc0ALmjNJRQAi9KdmmL3HvTu1AC5opKBQAtGaSigY7NGaaKKkB2aM0lFAC5oIB4IBptLmgaG+TEefLT/vmka3iYYK8fU0+ilZFXZRn0LS7nPn2FtJnr5kStn8xWe/gbw1Jn/iT2a567IVX/0ECt+ilyod2clP8M/DE2SLDYfVJZB/7NWZc/B7w/OSVlvIif7kw/qpr0HNGaXJEOZnkNx8A9MbP2fVr2Pjjfsf+i1j3P7PtwB/o+u5P+3a4H6Oa93BpQafKI+b7j4CeJEyYdQ0+QDpkyqf/QDWRc/BfxlADts7ab0KXKr/AOhYr6ozQDRqGh8gXHwy8Y2+d+g3D/8AXFkk/wDQSazbjwb4mtgTN4e1VAO5s5MfyxX2iQrdQD9aZ5EP/PJP++RRdhZHw01ldW4lWe2miO3pJGV7+9VsHFfdjWsLjDLkem44qjceG9Gus+fplpLnr5kCv/MGjmYcp8RYPegn2r7IuPhx4RuQfM0CwGf7luif+ggVk3HwY8E3GT/ZCRk/3JpV/wDZqOfyFynybRgnpX1BcfAXwlL/AKoXkP8AuXH/AMUGrJuf2etJYk22p30fpvKP/wCyrR7RBynzqBxRtHrXulx+zzOMm31sn/rpaj+kn9KyLn4B+Ios+TfWEo7BhIp/RSKPaIfKzyEgd6bjmvSbr4KeMof9Xa2s3+5chf8A0PFY9x8LfGtqcvoUzjP/ACykjk/9BY01JdxcrONr1T4CXnkeO57fOBcWTrj1IZW/oa4m58FeKLQnzvDuqKB3+yOR+YFb3wuF3pHxL0d7m3mgDyNCfMjK/eUqOvuRU1bODCCakj6rNJRmivKO0KUUlFUgHEZBHrQpyoPtQOaRD8pHoSK6KREh1FFFbkBRRRQAUhpaKoRABiVwB15pTSP/AK5T6jFKeaQXCpYIfMfcQMCohy6oOp6CtHb5UOEGSBx7moaQN2KVxFDcMftTKtspA2lsBmPr/LFYWvaTpcNnNIq/Yw/ytMkrRFs9RwCWPTjHNdO22PBIzjpxk5qte2EeoweVeIGhYg+Vng49cdfp0rmnTTTTV2wUtTztS06Yim1S6hQZBkUspwD91SCQePY4471xt61ld6sls9/fM9zKUaJ48FRxk4GcDj0H0Net61cvJbxw6bGQkbKA6LkEZwVXHt3OB2z6ec+JdRG26ewnghuWg2yrb4Z1IBXBYf7wIxjPNebOnyysnc6FrG5lyrp6Sui+C9YKqxAy0o4+gTiiuf8AsuunmJ7jy/4NzDOO2eaKfso9/wAWY83kfSzfeopWHzUCvqGcaEqhfc2dyPWNv5VoEVy2sf2+uoPHavbNZS4GHU7lBGDz+dNNLclps47w14XvdG1Ozube9m8oOokjL8MvQg/hXos4Xz4iRzg4/SmLZPHZsYwDKFyoPTPasnTtR1a/vhDe6WluiAnzFkzz6Y/z0qrruKzOoT/V/hVLURv0/jn5kP6ir0akJiufvNch0mZrXU4pVhOfLnRCylfQ45BFCYmjRt1CtGf89DSu+dQwOyD+ZrGbxj4fgj3LemRv4USNsn25FXNGlmvw17JE0QmbcqN1Vegz74FDYJM2W/1Y+o/nXLeLvFn/AAjU9kraW16k4csyvgx7dvbBznP6V013LHbWrTTOqRpgszHAArDvjo+ryRtJqFq3lghQJFPX8fakMsI0et2VlqcE0yxFRKkecBsj+IeorXgY7BmsJtS0/Tra3061lSSRiI40jYEgdycdBjNbdvkoCaAK19JbxaddS3jFbZI3aVsE4QZJ6c9K56y17wk7KLXUItzEBQVYZP4iuiu4Rc2NzbkZ8xHTB75yKw7DwpZWsyytaxBgQQdoNSyonURdOOlQNz5wPc/0FTxgBcZqEj5ph6n+lNCZydh4LtbHULa7GxZIHDgqOTiuit9QS4u5Io1cCI7WLoVyfbPbpzXn2ieH9S0fVbO4h1C4a28wB4mclSp4r0bA81DjnHWhgirr2h22t2qR3Me4oSUOcYJqroelJpmjS2SZCq8mOfXn+tdCvK1UAw8w/wBr+goA5jwnoUulXDStd3MoKbQskhIH+cV2ydKoWaAIvHar4FQWPzWVrhJswO24GtSszWRm0/EU0Jmdpl55TNESdrcj2NQWVy39qiQsxBds8/WsGTUJLfxX9gfiOS0WaFv9oMwYfltP51paezGVWcEHzW/9COP0pdB9TtI23Cn1XgOVqehgC9T9aWmjqfrS0gKV7qP2N0UR7yRk84xTkv0a0+0EEAAlhnOMVn6x/wAfCf7tZUl6p0+9Eb5CKwYZ6MBUt2nZ7FqN4XR01pqEd3uCoykAHnFWi3FYGnMY5MA9Vq/qFy0FhJIv3sYH1NODuhTSTJpNTtInKPOu4dQATj8qnhuIp03RSK49j0rzjWINSuYEGm3PkOCWY7c7vQVc8L392UtpLrInJ8uUevOP8DUptq5XKtjvy2BntQGBGR0rO1SF7vS54I2wzoQOcc1zWhagYZRu+XPySA9iP8/rVuNlchb2O2zzS1yj38tzqAKzSLHuAARsZFdIJQULZ496lO70KasTbqXNcjq3iRrWGS5VpBAnQRJuZqseH/Ea6pDDKrl4ZvulhhlPTBxU8w+VnT5ozVa5ult7aSZhkIM49ayz4gjTHmoqZ4GZMZ/SnJqO4RTlsbuaM1RsdSjvSyopBUZ6g1dBoTuNqw7NGabmqz6hbRyGN50Vh1BPSgRczS5qtDdQznEcqOfRWBqbPFG4EmaM1GGyKN4JxSC5JmjNNzgUA5HFJjH5o3UzNGaQyTdS7qjzRmgB+aSm5ozSAcaYQD1AP1ozSGiwxhijPPlr+VNMEZ7Y+hNSE0manlQ0yI2644LD/gVM8g9pX/HH+FTmk71Ps4voVzMgMcg6SL+K/wD16TbKP7h/MVOabS9lEfMyPMg/gH4NUMDzlpC0SqS2cbv8+lWqjHEp96FTS2DmuKGfun5Gjfj+Bvyp9Ga0sTcZ5i/7Q/4CaTzo8/fFSUUrMY3ep6MD+NLSFFPVQfwphgiJPyLz7U9RDLhtgVjnhhSb95wn4nsKiurQNbuI2ZWxwckgfhmpowBGoHTFGtw0Lttb+UN5+8euetWSQFLHPHoM1WS6UIA2S1Zlxrl7Zbml0tpY92FaCUNgf7QIBH4ZrKpUjBXkTyuT0Ndl/wCWjZHHTPSud13ULu6YWGnouBgyvISA690AHJHqR24zzVLVPE89xILNVS2YkHaWzIw9h2/I1nXGqSWtm4SK2V5mw5lO4tgdSTkn8a8ytipSaVPbudEKDSvIbeLMLSSK906TLZ8yaIrMPrsyFHAAA5xwPSuH1a10ay8iWzknMMpCtlcMG6gsDxjnjAHWuluNVuhF/o8toZGBACQqQD2OcYwPrXM3V3G9zuv2e9uRwCuUh/IDJz+B46HFYOTbLcUkc9Jq2rRSNGlpeuiEqrHaSQO+cUV1hksSTts/l7bYcj8Cc5/Oil7eP8pnys9qkHz0gp0v36QV9UcAYpjRhjzUgFLQBEEAGKYLdQ5YDmrGKMUANC4FV7i0juFKuoIPqKtUuKQGGug2aSb1gQHPUKK04YBGoAGBVjFGKSSRV2yC5to7m2eGVFdHGCpGQa5ibwXpjtn7Mg+gxXX0hANDVxJ2Ob0zwzZadL5kMCq/rjJ/Ot9EwMYqXaPSjFCVgepyWsaFqU9/LPZ6vdW6Pg+WrfKDjsKoLpPiSM8a7MR/tRA/zruygPUU3yl9KHfuNW7Fe1RxCokYs4ABbHX3rF1ldbjvwdOktxbuBuWRCSD3P8q6QLikaMN1qrvoTZGNJYuNPfywDKFygPTI6VQ0i81a8uil9ZxQIgyGV8kn0x6V1HljGKYIVVsgChyYWQiLha5/U9bj0zUWt5LS7kMgDK8UeV9MZz1rpAMVDNbJKwZhkii9h2RBZqwjUkdquimogXin4pXADWfqxAsyzEKARkk4ArQqrf2UV/ZyW06B43HINNMVjhtfsftaRXtqytcWmHQqc5x1H4g1csZRIIZV+62CK3NO0O102BoYIgqkkn3NZy+EbWPVReI0ow+8IGO3P0pNqw0nc3VmWGLe+ccCrUUyyLuXP41n6jYvdae8MMrQycFXXqCKbo1rc2tr5d1cvPJnJdgB+HFJtBY1R9406mj734UtAGPq4/exfQ1w0StY+LNQtpiWtNTUSJ6B1UKw/EYrtPEv2xLWOWythcSBsFCccVjT6Rc6lpEcssQivIyJUUHO1h2z9OKHG8tSoytE07D5WjGc4XGTVvVxnS5T6YP61h6Je3d1e+VLp00CoDudzxn6V0t3bm4sJouMshA+tKkrKzHVd3dGFpqKzsD6VRVBDfzKvAE2QPyNVbTWYrWdorlhb3MfyukvANWba4hvb/McqO7tuIQ5x0/wpS92PKwiry5jsYvmUA965XVrI2OsCSNf3NzksB2YV1cCkIM1m+IUUWSSHjbICfatY6qxD0dzO0uHzJQ2OF4rfuFIspcddh/lWToTLJHIVIOG7Vuuu6Jh6jFZU1Yuo7s5G3t0ukSJwCrDpUel+GZrDU1MV5ItmsnmLCqjHrjP1qXT5hG4z96JijD0PSrcl9cteP5T7YlwFwAc+tJrlvcpXlaxr6ou7Srgf9Mya5K/0u21e1ijukDKpDj2OMf1rsbkebZyr13If5Vx1zYnU9MjgEssJO074m2tx70VfhTCl8TRteGNMsdKieK0QK7HLnByfSukBrm/DOjtpiSM9zcXDvgZmkLYHtXSClHYJbinpmuK1BZC96sb7ZNzhWxnB7Gu0PSuSvlK6ldL/tA/mKqS/dsUH76Oe8J6hqkiTjUTm5tpsK4TbkY9v8816TLIfs7lDhtpI/KvNdE1m8utVuLG9tY4pYAHVkJwwz716HbSedaq/wDeXms4aS1Ln8JhaFcubiJ2dm3qQcnPP/6xUN1qt0ut3KiaRVikUBQ2BjHpTNJzHMsZ6xzMn61W1RfL8Q3I/wCeiI/6Y/pV2+NE9Ys7a4uPKtZZQM7ULY9eKydE1S4u5QJnUqybgAoHPFXoj5+mrn+OIfqK5zw/Jia2B9Cp/WlL4Uxx+Kx0OsamdPto3Cb2dwgBOAPepdMvjfWnmMgRg20gHPpWV4qUnSkkH/LOZTx+X9ak8PN+6mT0YH8//wBVE1ZJijrc381mtq0KzvGN7FDg4HGa0M/LXJSIDfXkJPBdge3WpnpByXQcdZJM6KDUYZHCbiGPQMMVdzXmemW13pHiCSxluppreSIPH5rFtv5/jXoMUpltA2fmK4J96mLd7SKaVrodFfW80zQpcRNKvVFcFh+FTO4UZrzfQJfJ8RW7HqWKnPuCK6HxrI40iMKSFaTDYPXg1t7P33Az5vd5jpt4K5oVgwryzTdfnsNIvbQSld67o2B5VsjOPqP5Vs+BL+aW7u4Zp5ZMoGUO5YDB7Z+oqHH3blX1sd2SAOabuB4zWB4o15tGslMSB55M7d3Qdsn864aDxlrcV3GJbjcJBlQ8ICt9MD+tS9ClqesUlUNG1RNW0yO6UbScq65+6w6j/PrV+gYlNI+cU6mv2PoaQDqKKKYBRRRQAlFHeloENYZBFV4STEPUcGrOKrRArLKuMfNkfjTAkpyp5iOnHTIzTaVZFiJdzhQDk1FSKcWmCdmczrFjbTTtDPDHKGw2xgDjtu59KwpdMmkVPsWozwjn5pFEowTnjdyB6YI/rXTyW7zT3N1Jvj3ttRHUZUADH884/Os2+SO006Rp1PlkEZfPzn0z71xWjK7asjVt2Ocmsv7Tu2ie4i+zxDcAifM6nuzZPU59B149eC1e3uG1C4gvH8i1DmOOR5yS0YPy5AywXgH0r0e3SG3sy8rwNcTN5kuWACH6e3QCvM/HumXt3rDraM0sD7JF2PldxGCT6nj9amFOCemhMm7XNmO+tUiRF12dVVQAqwMQB6A9xRXBR6VqKxqPtiLgAbc9Paio+rU/5yfavsfYUo+emAVhHxnpMp6zr9Y8/wAjUq+KtHPW4dfrC/8AhXte0h3MPZy7G1S4rLXxFpD4xfIM/wB5SP5ip11jTG6X9t+MgFHPHuLll2LtFQJf2Un3Lu3b6SKf61Mro33XU/Q1XMhWYuKWil7UBYSiloxQA2lxRS0AJRilooATFFLRQA3FFLRQAlFLiigBMUlLRQAlFLRSASiiimAm0UhUU6ikA3aKTYBT6KAGfx/hTqP4h9KWgBjIGHNMEQAxipqKAK4t1ViwAzUm3ipKTtQO5l32jWd9zcW8ch9WUGm2Wi2tif3ECRj/AGVArWoxSauNNoYq4FV7+xh1Cze3nTdG/UVbxRigRl6XpVvpcBht4wiZyfc1o44p+KMUkDZyuseGpZrxr3T7pradvvjbuVvwqPTdJ1FJ83t0sijoEj211u0dxSbF9KUlzblRlykAi/dlfbFcRLa+ILWRo4oLSSJThSSc4rv9tMaJW6im7tWCLSdzk9D/ALWfUUN1bRRQqCSVckn2xXXCmLCqnIFSgYqYqxTdw7Vx/iG5Sw1Eu8UzCRQcxoW6etdjioJrdJhhlB+tVfRoSWtzgLW7sp74SRRSm5Zdn+qYcZzz2/Ou4sozHaonoKEsIlbIQZq2se0YrOMWndmkpJqxwt3crpeu3ME58tZG82Jj0OaZdyLd6it0GA/deWQDnJzmuw1LR7TU4wtzCkmOhI5H0NZtn4V0+ynWWKAB16EknH51VSV7tdSYJdehpWCMlhEjfeCjIPauY0/9xqLJnGy5Yfhurs1TauKxm8O251Y3xaU5bfs3kLn1xSv7nKwXxXH+IY/M0O5Hoob8iDWfot0kDsXcBXUHJ4FdDdWwubOWA8CRCp/EVx7eG9WiOI9S+UDgGEGiclyoILVnZw3Ec4ISRGI7K2a5q+/d69MvZ1Vv0xVzQNOu7NpJLy5ErsNoCoFAFV/ENnffbY7qyijk+TYwdiO9CknB3DltJWKWqup1nSnUcsJFP5D/AOvXSWBzagelcvbLq0kyLNYxqP7/AJmcfpXW2sXlwqvoKxUnKafZGjjaLPO2/wBD8RnsI7on8N2a63xVF5ugyNjJjZW/XH9a5/xBp9ydfkEFvK5lYMpVeOnr9RXW6nC8+h3MZHzmE8D1xXZzL2ql6GCT5LHjt1J5NxDG33JdyH69R/I10ngqQw65GhP342T9M/0rPj0l9ZNwkKFmjXzEbHAcHpn35FS+HneHXbIqpLGQAqByM8H+tRNb2HF9zofHsZK2zdijD6YINc3O1ncaJpsTyulygc7kA4G4jv8AhXaeNLcyaSkwHEb/ADH0BGP54rzia1t7qxNvNH+9Rw0bdsZGRRJK0WNdUel+DXs1014LXzsowaRpcZYkdePpXS1w/gqdTeXMa9CgOPocf1rt6wi9DWQU1+VNOpD0IqibgDlQfalpqcoKdzQAUdqWjpTEJ3ooooASq7fLcj0Zf5VZqvOCJImGMbsH6GmxIfUKhnut54jjI2+7dz+HT86mpMVMlcaY7UYwVHpiuW1CFC0KswcbtwiTlnYDIA/me1dDqM94Y1W3sfNBX77ShRn07ntWBqaXVtaTzJEd/lgvtBfn0B7LnH6nmvOxNZJcsXZmsFpqc5cxNc6n5eoXKwwuP3cSnooHO5+3XOBj3NV7iKG8N55dhBcEhi08eCIsYAwO5PJ7dce9XdQgllVUuJTFBjmNdu8npgDJHvkHiua1BD9oZYdVEdogBEaPtJJGenU4GPXnp7eZUlOd1I1suhfSHSlRVaCYsAASZohn9KKpJ4USRFeTVpEdgCy5JwfSisLx/mFyPsPEjK3X9asx3Gep5qk+d2R1pyEe4r6B6gjUWQN0NShuKzVcqcfhVlJSO/HrUtFFnCE8ov5Uo2g8KAfpUav+NPDY6fzqRk6TyJysjr/usRUy6jeJ928uB/21b/GqeeKdk9setO7CyL6axqSdL24/GQmpU8QaqowLx/xVT/MVl5pe1Pnl3Fyx7GyvibVF6zq31jX/AAqZfFmojqIG+qH+hrB/xoyaftZ9yfZx7HSJ4vvMfNbwH6ZH9asL4ul/jskP0lI/pXKKec9qm6Ue3qLqHsYPodUvi1P4rJh9JM/0qVfFlr/FbTj6bT/WuRGcZpaf1mp3D6vA7FfFNgesdwPqo/xqRfEmmN1lkX6xn+lcVR1p/Wpk/VoHdDXtMb/l6A+qMP6VIur6c3S8h/FsVwOfejOKpYuXYX1aPc9DW/sm+7dwH6SCpFmif7sqN9GFecemT+dGPoaf1t9hfVV3PSuDS4rzUMVPBI/GpFuJ0+7PIPTDEVX1vyF9V8z0XFGK8+GoXq4xd3H/AH8P+NTLrGor0vJfxOf50/rcewvqz7nd4oxXELr+pr/y85+qL/hUq+JdRXq0TfVP8KpYqBP1aZ2WKK5JfFN8OsVuR/usP61IviycfetIz9HI/wAapYmmL6vM6cj5lpcVzB8WsWX/AEEH1/ff/Y1OviyH+O0kH+64P+FP6xT7kujPsdBRWIviqyY8wXA/BT/Wpl8S6cw5aVfrGf6VSrQ7i9lPsatFZy+INLJ/4+cfWNh/SpF1nTX6XkX4nH86ftIdxckuxdoquuo2LH5by3P/AG1H+NSrcQP92aM/RxT5l3DlY+jFAIPQg/Q0uKLisJRS4ooEJRS0YoASjFLRQNISilxRSKEoxTqSgAxS0UUAGKTFOooAbRilooATFNKA9afRSAYEAoZA3UU+ilYoh8lc5xTwuKdRilZBcieJWYEjkUuwbcVJSGiw7lZLVIydqgZ9KiTT4IpzNHEiu3Ugcmr1JS5UO7IJoEmhaKRQyMMMpGQRXNyeDNP84siOq/3A5xXVdqTFDV1YE7GJovh+z0YH7NHgnjJJJxWzTsUmKSVgbuJRS0VQhkf8Q9DT6YvErD1p9ABRRRQAUlFFNCYtQzjMRI7HNS0xhuUj1FOwhOvSkpIzmNfpS0hkkm46c+z74zt9q5zUkeSBAkpaISK7KuP3mPcmt+EGSJ/NH+0qen196yNQVVkPHI5rgq0HObkawkkrHP6pbi7vLXfbr5WC0mWAbjGATyQOvTr7DmsvV2mW2lng0jTxLEhMcjMG2bQcHbt9ulXri9XlLaMzSDJZYiAFPU5J4H8/aqoSRlZ7psDPEUXOc9jnr+lS8LC3NLVh7R3sjzseKYgAH1e93j7224bGfaitN9ABdi/iLT42JOUEcQCn044/KiuPlo/1/wAMV7xqMvzHjjNKq8D2qWUKjfMQp9+KRMEfeH4Gu1O5tawBcds08D2pwX0PPtQFHai4xVYrwf0qVG6ZFRjHY5oXOD29c0gLKv8ApT8gjqeKqhj1qQMRQBY68j60dOpqMMP608HjFIB3b1pcc00EZxml/KgBR1xmp1quODU65xUtlIdTuKTtSfQVJQppM+1LgikI+nFFwDr/APXoA4IxQB+VLxn1ouFgB/lR68UdOv8AKii4Bx1o+tL+dJQAAUdaWk6jigLC5pD796XHSkI9utAWA4oz9KM80h64/OmFgAzS0DrSY4oFYXg5ozikx15ozQOwuaAfek70Z4pALn0pcAj/AOtTc460E0wsO4AyB+VPWaVfuyuv0Yioh7UvGKLsVkWBf3aH5bqcD/rq3+NTLq+oJ0vJvxbP86o+9B47U+eXcThF9DTXXtTX/l6J+qL/AIVKviPUV6yRt9Yx/SsjNGfXFP2s+4vZw7G4vie/HWO3P/AD/jUi+Krkfet4T9CRXPg0ucVXtqncXsYdjpB4rb+KzH4S/wD1qlTxVCfvWrj6MDXLZPFGfen9Yqdxewg+h1y+J7M9Yp1/Af41MviLTz1aRfqn+FcYDS7gQaf1mYvq8Dt117TW/wCXjH1Rv8KlXVrBul3H+JxXB55pQ1P63PsL6rHud8NQsj0vIP8Av4KkW4gf7s0bfRhXn27HNLuprFvsL6qu56ICD0IP0pcV50GA6cfpUi3MqfdmkX6Mar655C+q+Z6DRXBpqV2vS6m4/wBs1MusXwHF0/480/rcewvqsu521Fccmu3463Gfqq/4VKPEV6OpjP1Wn9agL6vM6uiuYXxJcjrHEfwP+NSr4lk/it1P0OKpYmmL2EzocUVhL4lU9bb8n/8ArVIviOA9YJPwINH1in3J9jPsbFJWWPENmeqyj/gI/wAaeNcsCf8AWMPqpqvawfUXs5roaNJVIavYN0uB/wB8n/CnjU7JulzH+JxT9pHuHJLsWaKgF/Zv926hP0kFSCaJvuyIfowp8yFZjqKOvTmiqEMPEoPqMU+o5DgoffFSUAFJRRTEFFFIaYhTTaWkpiIk/iHoxp1J0lb3ANKMA5IzUybSutRixnD/AFrDvpLa+kePzFfaNssOcEH39q2toLlskMfeuW8Q+D4tZnF3a3j2t0DnzFBKntxgjB/OvHr46pTnapCyNVFWuiu0UMPyRRiKIHCoOiiucurvUILhrgaUWmU+UCLjeqjqTjAI9z06c9M9RFo93pdr+/upNSYAkcJGemMZOc8/z9gKqpbz3N7uNobdowMx4QjHqWA6fQjt61jPHqs+Wm7JFxhbVnFyWdh5jeZOd+Tuz5fXv/HRXXNpeiFiZbW8aQn5j9mfk9+9FcXtX3NOVHJ/GFiPD1hjOTdZwP8AcP8AjXm+gs7MnzNjcDjcfWvRPjGcaPpyc8zk/kv/ANevP/DwysLHHLf1r38vV6KuRjH+8Z1S7xjDuD7NTw8+B+9k/wC+jT0Xr3qYKMZP1r0uVdjhUmMFxdL/AMvEv4nNOF3eD/lu2fcCn7AOMUu3Bpckew+eXcFvrw/8tAT7qKkGo3ag8ofwpNn+c0nlg0vZQ7B7WfclGr3IHKRn8/8AGpF1qbIHkr+Bqt5YPB/lR5Yo9hT7D9tPuXhrj9Db9uzf/Wp415R1t3yPcVn7MCmtEP8AJqfq1PsP6xPuWpvF9lbShJoLnkZyqqR/OtGLxDaPGrbZhkZGVH+Nefa6Nl6mOPkGPzNdBaR7raLPdB/Ko+qU2y/rM0jphr1l6yf98U9dbsT/AMtWHsUP+Fc95IPAFO8n/wCtxR9Sph9bmdENWsW6XCj6jFPGpWbEYuovxYVzPkgcAUnkj86n6jHuNYyXY6tby2I4uIz77xUgliPSRf8AvoVyH2cE5Iqvd2w8jkAc1DwK7lrFvsdyCCOoNKO3tXk/2chbwKxHOTg47Ukf2q2W2Y3U65yWxK3fn1rP6m+5f1pdj1rjjtRjmvJYLvVQsxOoXgLJvUee3HJ96emp61HpzuNUujIVLZaQnHtzS+py7j+tR7Hq9GPWvGx4x8QRGMf2m7fL0KIcn8q0pvGWtrKscVyCAwDFolOfyFS8JMr6zA9S7c0Y+ua4JfFWrqPmMB9zGR/I1XXxzrPnmP7NZtlyB8rDA9T81J4WoNYmB6LTRXFSeM71Uz9kgOOT8xGaLTx3NPEXfTFTnGBPn/2Wl9Wqdh/WKb6na44pOc1yEXj2OW4aP+zZQq9WWUH+gqSDx5ZSytE1ldqy8kjaQf1qfYVOw/bQ7nWGk/SueXxnpxODDdL/AMAU/wDs1IfHGiK215JlwcZMJP8AKk6U10H7WD6nRdTQSeawIvGugTPtW9bcemYH/wAKl/4SvQhw2pwqf9oEfzFL2c+xXPHubXOeKO1ZqeIdGkj3rqdptHUmUAVLHq+myjdHqNqwPcTKf60uWXYfNHuXcn9aUVXF5ayHCXELewkBqVXVujA/Q1NmO6Hg0Umc8nNGcA0WGLnJozmm5zS5Hf8AWgBc8Z70tNzkf40E9qQxaBSE0ZoAcDg0pJxTM/Wkz7GgCTJyaN3FR7uODSk0ASE88Um7mo91NLf407CJt3/1qaXqIuKaxAosFydX5qQPVMPUgfiiwXLO/rQH75/Wq5fApC5pWC5a3+9AfjFVfMwO9HmHHNFguWw9L5nHPNUxL604S4o5R3LRkpnmc+9QGTjmmGUZosFy6JPfip45PlPSs5JM+tTROefTFKwFvzKaWX0B/CoN/uaQygDPX1ouFiwHA5AA+lI1y6naJZsnoqOwJ/I1Wjd5z8gwv989Pw9auwW6J7serHqaLt7CsluXdMW6ku1kluZiq5xGZCynPrmuoByBWHaLsGa2YjmNfbivSoRajqcNZ3kPpM0UldFjEXNGaSigQUtJRRa4GVJb31tcloJfNhYnCNjK+w9RUcuqXFs4S4te3LDI/ofbvWrL0U+hpc15ssFVhJujUa8nqjRTT3RiHxTZIxVo3Axyw5OfpVlvFWjzCILOVZhwNvtnHHetBrK1mRmnt4nDdQUGTUcmi6dJZ7ltEUGTzCF4+bkZ4+prmksSk+azt1KTjdWMo6vpu8oL1JWZSfLQ5475J44p8enw3YaZIVWZjtIB2jA6fXHPbvVS88M6WINzvLDsAJkD4Ax356VlXFjfW1havpFzdCMF1YOCrTbjhccccA85Hr714VWHNJNKx0XaRdmtrOKaSP8AtdItrFfLEiAJg9MbeMUVCuhakVGLfTU4+60SMR7E45PvRWXspdwuzz/4xOBZ6UvTMsn/AKCtcRoCfLBj/np/n+ddh8ZWITSFz/FMf/QK5Pw8Plt88gsD9Oa+wwC/cxMsX/EZ1qIOp65qUDnPr1oQZ6j61IBxmvTOEZg5NOCj0p23HWnY9RwaAEx/OjHGM89aUDIz7U4cCgBmOeKMdqfjJNOA5NFxWI8YphXA9PepSMnpxTSOOfpVCZyniIf6dFx/yz5H4muisx/osPGfkXn8BWB4jGL2L3j7/U10ViM2kByceWvf2pLcHsWQvGMc08qenNIBke/c1IopiGbevrS7e3FSY70bQe1AyPaB25qC7X9wenUVbxnkVDdg+QenUUmNHM3H35kXq7DP4ClnjEltAo6nAGfcVO8Ya5nbsqY/SoBmSSH0QqvHrWVywmVYpmyBjy8Co4kBsAMdjVi4jEl2i9gMn3xVYOUs1RR87ZA/OmmI5G4jZZSxwNzE4/u84A/St+O3WO3t1HUkEnHU1i3Z2uoPTn+fFdAozBbn/d/lSGaL4jjLnGAKq2cQeWWQjk1PJm4nEIPyjlj60tqu2aYAd+cU7gLJEu08DNUI1CWhUD5mY44rXZeOelVbeESPv/gThRSYIjhsxFEBgbjyTjvVeGAC/kOOueK1mXv61UVcXrk9x/SkwIp4/LjPy4PbNZr2obPr1+tacxLsWI46CmLC3lu5JGBx71Ldykita2KLGH43NWRrEWGb6CumhA+zr+P86wddGCT221fQkj02ITaU2eRlu1XYLcR2qrt5PQVFoLA2Mm4Zw54/AVrWqCRzIcEDoPSkMqxaaqKSQN7dTT/smDxxWoVHbOKbt9v/AK1OyFdmPdwyiEFZJBznhiKfbm8WBNt1cAgYz5rA/wA6vXKZgPHTmmwIPIA9M4qeVXHzMiW61IdL+6+nmt/jTxqmsIOL+cn3fP8AOpRGB2PFQy9lUZJ9qHCPYfPLuNk8R6+p2Rag+7PdFP8AMVNfa/4r0/w++qG4VohII9zwLhiew4q3pGn2b3BN7cLEijcc9WPoKd44vY73QpLaN0ZFUFVjPyoB2x0rJwjzWSLU5WvcwLf4ja+0UbN9lbd1JiP9DXUW/irU5I0Z47clgDwpH9a8qtjttozxlW/LmvRbFQ1pEePu1MqUL7Fxqy7myPFF5jm2hP0JFV7nxvcWrqDpquGGciYjH6VEIxjP61Q1G3DbDj1Gaj2MOxftZdy+vxCOPn0lwfacH/2WpB8Q7X/lpp10p9mU1zjW64+70phtVIIAHFHsIB7aZ1I+IOmkfNaXq4/2V/8AiqkXx/oxIDi5T6xf4GuOa0Tg7ec+lRtZoaPYRD28juh450HHN3IvoDA/+FO/4TXw+2M6mi/7yMP5ivPXslP8qztSsNthLID0APQdjTWGiH1iR6unizQXwRq1qM+sgH86tp4g0mQALqdmf+2y/wCNfPmP3HX7rfz/AP1VuW9orwo/qoODSlhorqCxL7HuC6jaSD5LqBvTEin+tSCeM/dcMPUGvExp6sp4HJ6mo308AfKMVH1ddyvrHke4h8nj60hfGM5rw4wToMRyyIe2HIogk1guFhv7xfpMw/rR9W8x/WPI9x8w+tL5nPWvFn1DWrQPnV75mCZ5uGOPzP1r2T4SSrr3hSaTUh9puYrtozJLy2NqkD9TWVWl7OPMXCspO1ibeOgpm/nj9a7g6HprHm1X8GP+NRP4c0xh/wAe5H0dv8a5vaI1ucekgPepklwf8K6M+F9PySDOv0f/AOtUE/heDyX8m6uFfB25KkZ/KpdRFJmG0x3BVyWPQDmporVnIaY9OiDp+PrToIFgXGMt3J6mrK81W5WxIijgYq3CvNV415q7CvIrelT1MZyLsIworTgbMVZydKu2rZUivQgrHDJliikNFaEBRRSUALRRmkqkJiPyh+lIDnHvSnkGmRn5B6ik0CJpXwAgOB3qe1O63kXuOapn9as2LfvGXuV6ViqSjCw76mPq8DOQ+AyKufLPQn1Pr9P/AK2MGae7nLy+cwVG+YnoBzx9c4rqNThaWJlLGNOQSvX/AOtWNaxRXFpLaNiMCRANvfJ4rwqvuuT5dE0dyfuopCe8KgjkEcHc3P6UVea21gsTGyKmflUSAADsMUUvaw7IV2eQ/GRiZ9JUdQsv/slc5oAwlt6bv61t/F2bzrrSDn70Lvj1yVrG8NjEdr3A6V7GC/gxMsV/EkdavHtj9KmAHH6imooxT1GDn+leicKFxxRjHvTufWgZx9KBjQBmnBR6daXHNGOKQxCBnqetLjOD607b2HWgLzTQhCM0w9s9DUuPpTGHpTJZyniMYvIjx/q/6mujsB/ocHT/AFa9PpXNeJm23sJ/6Z/1NdPpw3WFuf8Apkv8qS3B7Foenan4+uaaFI4z+VPA49cUwsHIzR1FOx1PB9KNuRx0pXHYTAbtUN0B9mP4VY9sfhUN2ubcnGOQf1pPYZgthZLkngAD+VRlRFbwE4GW3GlnDNLKnY/MeOwHSnTKrRW8fQtj8sVkaWK8YLTl2z86k49B2pkEQETS46qQvsKtTMI7hiSMLHio0AGnnjHymmhHH6iux075zmt2ED+z4GPQKOv0rC1RHW4V2ztYjaPbHX8a1rMvLZL1CRoB+NDEjatIike5vvvyaLcf6VMKmhJMSNzyBUO8Q3UhPHy5/lSbGLOxdxCnU/eIHakswQsg9GqS2jJBlcfM9JaH55R70rjJdueDVOfakjAck9fYVclk8tCf4jWexyx9TSkxpDVjMjgD+VW5IwkBHoKfBD5aZI+c0s+PKY9OOKEJlW3H+jDI9axdcXKnjOUrbg/1HJ9awdek2ywxryWyD9Ku+hNiHQAziVBwobcfy/8ArVvaeflbjnisTw02XuUz2U/zrcseZH60uoy6BznFKRk04Dj2/nQeOcc1VySvKuY2+maitV+U9OuatOMofU1BGDDHlgcnnHpUNlJBI20YHJPTFIsQU5Y5Y1IkZ5dx82Pyp5AJzjNNAyFhVLUY99hcL2KH+VaJHyjPSq1wu+J1xwQRVXEefWozanHUPnp7V6Lo7B9OhbjG3v2rzm2zsmRuxH9a7/w44fSo+nHFZSNYG0Bxz0qrfp+6BHUNzirQ/wAiq96pNufYg1BZmFecnp3ppUZ44OKk56/1pvOOvJpk2I2GevSmFOPXJqU/mBTT/nvTuBXZevb6VVu4t9lOg/ijbH5Veb6VEVBOMcHiqTJZwQ5gkHuD/n866PS/nsYWxggY/XFc8g2iZOny/wAq3dEbNmAeqsRjNOWwkagTI6Ch09PrzUsalzgDmr0Vrt5cc9ayuWkUIrJpW+YYX1q2IFjXCgCrm0AYH6VG64BPb1ouVaxganHiXp95P/rV6N8B7rNnrVqTyrxSgZ9QwP8AIVwGpphYyB1DD+VdR8D7gxeLdQtScCa1bA9Srr/QmssQr0mODtNHvZOcGg0g6Y9KWvIudYzvSHkGg8GkNQ2Uc3ewmO6ce+R+NRqvNampxZKSf8BNUAuCPfmuyjrFBJkkYq7CMCq0a1bjGBXfTics5Fhat2xw2KqLU8JxIK6EYMu5opKKskWikzRmmJhmlpKKYgzUa8Fh71JUfSQ+4oaEKaltW23Ke/FRGhW2uG9DmpktBk9+uRIvfrXD3MLPe7SSo7kdQPb1Nd7fKqgzEgLs5J9K4q5R5LlpJI9kefkQjlh6sD/L8/Q+XVdp27nZSd4Dc3X/ACz09Cn8JMRYkdue/wBaKYwYsSZHJJ9aKy9kyrM8i+Lv7vVdOjBO1YG4x/tHH8qy9EuVggtWMbMFQZx3rQ+MDA+IrJT2tv8A2dqy9MAFnBnBOzr+Jrqw8uWhFk11erI6hNahxkwzDv0H+NTDWbYgfJL/AN8//XrDXnp9KlH6YzW31iZl7GJuDV7U9RIP+AUv9qWmc72BH+yaxDx3pVPbNH1iQexibn9p2ZH+t59wRTxqFpx++A/AisIdMjrR37U/rEg9ijoBfWp/5eEH1NPN5anj7RF9N1c7wB0ppAzjGKf1l9ifYo6Q3MH/AD2i+u8U0zxEYEiH/gQrmjg88flULgDnj8qpYh9hOiu5F4slRb6Ahs/uzyOe9dZpbqdNtiGGTEv8hXm2suEuY+3y1vaaF+xwHGPkH8qp17K9iVSvodyF9P51IFPrXJqfQkfialUuD/rJPwY0vrK7FKh5nUgHrjn3pCMgY9ea5pZJs/6+X/vs1ILi4B/18v4tR9ZXYPYvudFg96iu1/cH2IrGW7uf+fiT86Gurhkw07EehxQ8RHsHsGRsuXu2I5AA/SoEBeaGQ/dBCqPoKny5LfPy3B4600bxtAbhORxWftUV7NjLiPzLph1Crk1FKfL00Kf4kIHPepxu3M+RuYYJx1qtPGSgU4wowOKaqoXs2c3ri7Xt8enX8K1bBAdLTAHK8/WsfxA7i4jDFdoUYGK09JkdrCMDG3BGMVbmrEqGtjZsyDbRn29abIA90sfQdz602BniiCKFwvrSpGUkWQHJBzz3qXURSgy5joB17VWtyFuZgT71J5rcZQc+9REEs7AYLe9J1ECgyKeUu2c8dqbCB56cf55pxhJOccd6eR+9V1XAUYxUqauPlLYHI7daimJ8p8eh6UpnGMbWpjSqQRtbnPatOdEcrKqy+Xa+jZwPesrUYCsPmONzkH8K1QF8zc4JUD5RiqGs3MSwAnPXHK+1OMkJxZleGW/0ycZPKf1/+vXQWX/HzIoJ/H61y/h6QR6s+ehUjp7g109uwW5LfwnPNNtIFE1OR/hSZ4xmmCVDyD+GKPMQDr09KOZC5WKx2g889KrwtvnO7qBT12u5Z3AC9BUUOFuskYXkZJqeYqxsWOnvdAuwKxDuB94+grVGnIBvZOeylD0p9vLarFGxki2gYVcjn681R1HVFkDRW7jn7zjj8B7VnzOTsXZJFPUGgRzHCEb+84HH4VlSLuUgg1YbDDsKhZeuea3jZGLPO9nl3t3GR0JA46c12XhZy2nnnO1v8a5K/URa7dj1JP8AI10nhCTMMq56H/D/ABqZFwOr71Fcgm2fnoM09TikmBMDjGeCOKg0MnnB9qQ9fYUdevSgn8qLiGkGmHvwSKcf/rUh75P9KAImHH+NRnjnP5VMRxjH5VAeuRxTTFY4yaLZqdxH0+ZwM1qeHYmkjlQZ4YH86i1K0K6rPIeFBDAeuQM1a8NNtubiPIyVB59v/wBdVKWgorU6aCJYx8o59anGAP61GuAakHB96wubLQXtUcigjrz3qXIxg9PrTGOeKaYmY+qL+5BGeG5/KrfwvuPsfxNshnCzb4z/AMCQ/wBQKh1Bc2zjv161n+H7j7F440W4yFAuoST7bwD+lOSvBonZo+qDw31pe1EnYijORXiHZ0Iz1pD0pW600nAznArNlIhuo/Mt3HcciskCtW1vba8lmjgkEnl43EDjnPQ9+lUHj8uVk9DXdhdrE1NBI16cVaSokFSpXpQRyTJl7VKhw1QipVPIrYyL1FNQ/IPpS5qkSLmikoqhC0UUU0Ji1G/31980/NMl4UH0IoYhTTTTj0pppDNFsSWsTEZwP/rVh3cCtdAFFYsQBuOK2rZt1lj+6cVlaomORXmYuN2deHfQQ6hZIdggiIXjOOv6UVjhJMD5M/nRXF9XX8xryI8L+LrhvFEC9harx/wJ6z9NnhSzhUyICEGcsKufFaT/AIq9RkjFqgOPQlq4piRI2DwAOa9GjG9GK8jOs7VZHcpNCQMzR/8AfYqZJoiOJFI+tcADjjP4U7cT3xjtVeyJ9oegBl45H509WGe1ee+a27KknHpTvPk6iR1z6MaXsg9oehD6Zpe4rz77VOowJpPYhjTxe3IU4nlx1++aPZsPaHf5Hemnt6muDGo3isCLmYen7w07+1L0Di6mI/3jR7Nhzo7Yn/IqJzlcAfhXH/2rfZH+lSdu9KdXvQeblz9QP8Kagxc6J9eP+kxeu3ofrXRaXzYwdT8g/lXF3N1JclWmfc44yRV2DWbu3hRI3G1RgAqDgVUotpIUXZncJUy+mOO3NcSviO/Gf3sZ4/uCnDxPqC/MTF+K/wD16z9my1NHa9sU4+o/UdK4seKb4Y3LB+Kn/GnjxbejgxQH2wf8aPZyHzo7IHBHH6U7P41x48XXRwfIhPr1/wAad/wls2ObWMn2Y1PJIOdHWkgDGaM5A6+tcp/wl8ne0Q/Rz/hTh4t6k2g44/1n/wBanySByR0+7I9aic+lc6PFy4/49D/38/8ArUp8Uwtw1tIM+4NPlkLmRV8RsVnjY88Vd0Ns6dF17/zNYer6kl+VaNCMDByas6Xq0VtbCOQMSCeg96tp8pKa5jrY2AGKlz3rCTXrUcNv/Kpl160PV2HsVrOzLujYyM8A0bvSsga7ZMB+8Iz6qaX+3LM/8tfzBpWYXRqlunIxRkdulZY1qzJ/1w/EGnf2vZ8ZmUfnRZhoaWRznA/rTSe1Z/8Aa1mQR568fhQdVszj/SE/OjULouOeKydYG6yPHRgastqVoc/v4/8AvqqGo3cMltIFlRiOwIqo3uSzF0ZtusLweQw/SuziI4HT61w2mPt1iJmYbdx5J9q7OKQHHI6+tXPcmGxfB7nNBHHPFRq3HBBz75pc547+lTcsdn+VJwM8A0mc0h6UrisOJX0FNIB7Um4delIDTTGIQmTx9ajZVHf24NSMT1/SoZG46002TY5HWkSLVJSAQSAePcVZ8NyNicKzLgjkHHr/AIVH4hGL1CehQfzNReG3xdzoTjKg/kf/AK9a/ZIW510csuR++kxjPJqUSysMeaxB681VjP8AKpwc9xnNZ3NACAcZORR5QHHSnL+eKM/jSuFiMxZH3qQxlT94496mOcDPNIev+TRcLFfyuPf6U6O3GeeakJA6/jTkyM880OQJHPa9Ftutw6tF/iKz9Ck26p7OrD+Rra19QTA5GRyv8v8AGue0tymqQMT3x19aqOqJa1OzBx0P5VKv0/Oq6tx7VODzzjmszQeDjp+NMZuOn60pYZHb8ajY8HP40XCxVuzuikX1Q1y1zMY57aVW5Q5B/KuslUE7T0PFcffLtiQtn5Wwf8/561pF30M5I+vYJlubOKdPuyorg+xGach4IrmvC2t2kfw+0fULy5jhiFpGjO7fxKNpHucg1h6j8RbKR3is5Wjj/wCehQ7j9PT+f0rxnCXM0kdkNUdlqGq29iMMd8vaNOv4+lcpqWszXmRI22M9Il6fj6/yrmJfEto5OJTycknOSfeqkmu278+bx3pqjLsdEeSPU7rwjd+Zq08JOQ8W78iP8a6O+jxMrjow/UV534N1eB/FFrEsgJlDJ/46T/SvT72PfBu7qc100Vyy1MKzTehQQdKkApiipBXoxOKQ5etSCo8U4VqZsuRHKfjUlQwH5CKe7rGpZ2CqOSTwBVrYgfR3qKK4hnz5Uiv/ALppzSohAZ1Unpk4pgPozSe/agMD0IP0NMQ6mSjMTfSnUHkY9aYhqnKAjuKaabF/qgO44pxqRl7TmyJE9QDUVzGpXc/IXkg96LBttyAf4gRUt0H2soQH1+bFcONXu3NaT1MVruTccPIBns2P6UU4xS5+8B7Bun6UV43NDzO/lifNnxQfd40cdlgjH9f61xzDErZwK6z4ltnxrc9OI4x/44K5UfMzA859K9uhpSj6HLW/iMASDQfqadtBIpcdBjr3PetTMbuPt0pDnOfWnlcc5xmjHzZ7mkAwHHr+PFGOOAak2Z+7/KgLz60XAjz0xgZ9aXtjnFPVSc85BOMYpAvTIPv7UXAjJ5OOtGecgAZ7etP24GD+RFIVA5yOO1O4rEbcc5HXHXpUoPyj2GOtLb2st7dQ2ttHvnlcIiA9WPQV06/DvxPgZ0zBHX98n/xVKU4x3Y4xb2OW3YHH5005J6V1q/DjxQyknTlx6+fH/wDFUH4a+KWP/IPUD1M8f+NR7WHcv2c+xyeec8AijP4jtXXp8L/Fbgj7BGD/ANd0/wAaU/DDxWQB9gT/AL/p/jR7an3D2c+xyKybCSQD9RTS5Jz0rqn+HHieJyp0/JPYOpx+tIPh34oYhRpp3HuZFx/On7SHcXJLscvu57fhQXH4d66d/h74nViraa3B7Op/rTD4A8TL/wAwyT/vpf8AGj2kO4ckuxzSt2z+NAcZBx165rov+EC8TAZOlSe/zL6/WkbwJ4kGf+JVJzjoV/xp88O4uWXY5tiQPb+dPjY7SQeK328CeJW+7pUv/fS/40+PwF4oyd2lTcejL/jT9pC24ckr7GBvxznjuKBIQO4xx1/z610f/CCeJSD/AMSmbp/eH+NNPgPxJk/8Sqbkccr/AI1HtIdx8kuxz289z9OKUSFV684rfbwJ4kI50qbP1H+NJ/wgnib/AKBMx98j/Gnzw7i5ZdjA83pwc4xilEpC56Gt8eA/E/I/suX/AL6X/GlHgLxNg/8AErk/FlP9aOeHcfLLsc9vOeaaZDjnHPJxXSp8P/E7kgaVIOOcuo/rUTeBPEm7H9lzf99D/Gjnh3Dkl2Of80sfx+lIZRwM8jv610B8CeJhx/ZUxweuR/jSf8IJ4mOP+JVNz05H+NP2lPuLkl2OZVyHHrmnGQ9Mmug/4QPxP0/sqYY9x/jTv+EB8TY/5BUn5j/Gn7Sn3FyT7HPCZxyHI/HFO+1TqQPNfj/ardPgLxN/0CpRx6j/ABo/4QLxMTj+y5c9eo/xp+0p9w5J9jFXULoLxczc+jn/ABpw1O9A/wCPufA7eYa2G8B+J++lS/gw/wAaT/hAfE+f+QVJ+Y/xo56fdByT7GQNVvVb/j7uM56GQ1IusXwGPtcvt85rT/4QLxPnH9ly/XI/xpf+EB8TD/mGSf8Afa/40c9Pug5Z9jN/tvUOn2uT8e9OOt3/AEN05x64P9K0l+H/AIpbONLf/vtf8ac3w+8UI206Y+R6Op/XNLmp90Pln2MC6vri5IM8hZhwCQOn4U20vJrOcyRNhiCDxnit9vh94nBUf2Y6hjgEuMfzqVfht4oLAfYBz/01Uf1p+0p9xezn2M2PxHeoP+WTY65X/wCvUw8UXg48uEn/AHT/AI1bb4ceKR/zDh9fNT/Goj8P/E69dN4/66p/jS5qb6j5anYaPFVwOsEP05/xqRfFUmebVOP9o0f8K+8UZx/Z/PtKn+NQ3/gvxBpllJe3enmO3iALv5iNgZA7Glem+o7TXQtL4qYfK1shPs//ANan/wDCTqRzafk//wBauTDds9qcJD2xmn7NE87OsXxNEcZgcZHZs1NDr8DSY8mQenIrjw5zz+VWbaU+cOvPH6VMoaFKR0Gq6hFdQoqo4YHOD9KwI5PLvUfHCyZOOe9W5GyvTPpms2UkSv8AUdKVMJM7BNSgx1YfhUyapbYA3N/3zXNo3yZGee9TbuAeMVm0UmdF/aVtgHf39DSG/t848zr7GueD4XuD/Oq09/FBxnc3oMUKLew3Kx0sl3b7c+aAPWuS1S9haaWKI7gz/eHTr2qlcX01wTlgqk52jpVU9a6YU7bmM532N/S7jXtVRLG0u2aK1Q+XE7DagLZOAfc1pPo/ilQSTE3/AABf8K5W3u57KdpLeRo2PGVPatFPFGsIu37a5H+0Af6Upwlf3bFQnC3vXLstr4kjPzRr9Qif4VXLa2n3ox+KrUDeJNTcfNcZ/wCACoW1q8cYMg/75FJRn1SByj0bOj8Jalqdp4x0aWZNsYvIlc4/hLAH9DX1Yyh1KnoRivjCHWLmOeOXeco4YY9jX2bbTrdWsNwnKyorr9CM1z14tNOxpSle6MwLglT1FPFS3SbZyex5qKt4O6uRLcUCnCkHWlrUzZPAe1Vtatp7rT2jtwDJuBwTjIzU8R+cVLPJ5MDyYztUnH0q1sR1OehgOn6hCqOxdgpkQtu2k9ce1T3llcT3Vwzs8SAFllGCOnAwar6NLFNK95dSr5hOQDV/V7gsI4I2+V/mLZ4x2pK1inuQJcyto8nzEbXUZ9jjiq9u0lpdQzeaWSX+HA4wcEe9afkRDSpYIZEdthJ2kEk1kuwksoSDkrIR+BGf6VRJYvvtE97OkcgV05QEZHAzitTSblri0+dizKep7is+V9uqQSdBIik/yP8ASptFJjmmhPbgfgcUxGoOGce+aDyaDxMfcUGgCSBtlwjdgwrVlTLH3FYtbhO9Eb1GawrxvFlRdmZTRfMfrRV0xjJ4orxeQ6faHyP8RufG97jP3Yxj/gC1zeCXbgn5utdT4/iabxpevEu9D5eGXkcIvpWLiGMvvQh93rXqUnanFeRNTWbKew4zjPtSgZ4P5Y9qurdQINoTGfcUCWy67G/w/Wq1JKYQnHXigIQQcE1cEtmflUNycDk00SWhIxkA8UaiK2w5G0/pSgZyOg71o2tvHdIzp0BxknvU/wDZygHp+JqeYqxjleTxz0pPL7dxW0umqD8wwM4+8KeNLhPBzjt8wo5h8pglQeg6DpSMhXqD/Sug/s2DByzc+4xTG0q2Yf61h+VCmHIyt4TbHjHQ845vY/8A0IV9WBI2wSi5x1xXy9otutr420NFYsPtkRycdd4r6eD4Ue1ZV1zNM0pOyY7yogf9Wn/fNKY4j/Av5CmtOpG3bz/eppc1j7M15x3lxZ/1a/lS+VEesa/lUfnBOSCfpTRLnmj2Yc4phj37gi5+lKY0/uj8qTfSbqagHMSCOPH3R+VOEcf91fyqIOPWnb6OQOYf5af3R+VN8qM/wj8qN+aTzKOQOYcIowfuj8qcET0H5VHvpd9HIHMPCIB90flQUT+6PypnmUhejkDmHbFB6D8qCqf3R+VMLn1NJvp+zFzD/Lj/ALg/KmlE/uj8qbvpN9HIHOO2J/dX8qjkRNv3V6elBemSOMelKVPQamORU3H5R0pCqbVG1fyqNHGR7ikD8is/Zl85J5abD8q/lTPKTZnYMn2pDL8nPrQz/u/ej2Yc5II48fcX8qYY0JPyjJ9qTzPlzTRJl+e1P2Yc4540AHyj8qUxRkfcX8qZI+QPrS+YCAaapicyNoo/MHyDp2FMeJGOAq/lTnk+bPHApitxk5yTT9mLnI3hjUgBVx9Kq6leWenWpknwgPyqQmT+FWLqQQxGRs4VSxx7Vy2k66dS1BJ5OoXft3cKD0FXGlcTqGsLyW+YjT9Jnu2XpJKxRF9hk8/gMU28fXbKDedDWQdT5bq2B9MZrsbOePylVFHTt3qW5mKJyACe1WqMbD523Y5HT9QtdSjO2IJMo+eNgAR+FTPbxscCNffjpVm9trcM12sCpLjJlTqcdjTIvmQOP4uazUE3Yc20QrCgfBVevpWf4rtY7jwrqsWxf+PSQgbR1Ckj+Vajy4YqPp265zn19qi1FBPp1zFjPmROv5qRVxp2Zm53R8sYGf8AClGO5xjilKEZ/XNLtLNwO3YV3HGABzwRx+tSwfLKp9xyKaqnrj86eAByADzmpY0aDkYwep55FUbknzgfb061oOBtGOh68Vn3vRSPT/P9azp7ly2LsBJiQj0HINNmnSJNznHp6ms8XpihVFUZHc1VdmdtzHcferVO71Icy1cag8o2x/Iv61Sz70UVqklsQ22FIaWlamIQ0lL2BoIxQAlFFFABX2D4Avf7Q8AaFcZyTZohPug2n9Vr4+r6f+CN79q+G8ER62txLD+u/wD9nrnxKvFM1ovU7y8TKBvQ4qlWpKm+Jl9RWZ3qaL0NJrUWlBpKWt0YjlOGzVllDoVYZVhgiq3selWIzmJCepUZqkQzOXw/YJ9xZV/3ZW/xqe40uC4WNS0i+WNoKsRxV2iqshXZStNMjtJDIskjEjGGOagbQofOLxzSIhOfLGCB9MjIrUpadkF2UrzTI7uKNQ7RvH9x19PQ+opbDT/se5mk8xzxnGKuUoNOwrkcgxIh+opSKJfug+hBpTSAbWvaNvs09uKyDWlpzZgdfQ5qJq6GibFFPxRXD7JGlz4cRzGqgMR9DQXbcQwIIJzwOKigDMiDPPtV2HRNUup2EMeWAzy4rsUbsnmsVxKf9rnpTllx/CRz3FXf+EV1wD/UAD/rov8AjUi+FdbJI8lOn/PRar2b7C513M4S5A+X/wAdpfMzwFOf92r3/CLa4Bxbrz6Ov+NIPDeuKP8Aj34/31/xpezYc6KqzOBsUuB1wq9fU8fhThcz5wHm9OCa3NM8NX7wubm1y4bj5x6exq+3hmcEFLMj2D/T3qHFp7Fp36nKrdXLOMPLyMd6mWS+I+UTfgp5rpE8N3SsCbV15/vj/GpRpF7GSEtW46dDUO/RFxt1ZzAGqsOI7kj/AHSf6UjnVYhl4bkZ6kxnn9K6wW1+gINoT/wH/Comh1DdhreQZ5+4QKlOXY0tDuY/hqS4fxdpAlDgi8iHzLj+MV9Ng8V86WEMkXjHR2ljZCbuLqMfxivoodBxzRNXsKDtcUmk8wCmnOOBUZyTSURORJlSc0uVxUADMeBk0bucGq5Bc5PlaXK/SoA1Juo5A5yxvWjzF9arbutG7mnyIXOWi60b1qru/wDr0bsfSjkDnLW9aN61V3cUu+jkDnZZ3rSF1qvu96Tfk0ciDnZZ3rmk3r6GoN31pC9HIHOyxvX0pN61XLe9BanyC5yxuX0qOV129Kj3n1qOV8gUOA1MkEibl4/hpodNucd6r78BW/2TQG/cn61HIh87J/MX0pTInl9O1Vt2dx/Cjd+7x6ijkQ+dlhpUCYxjNIkibuh5FVWYk+3rTgwWUfTFCgHOyzNIu3pThIpA44qpK3yU7fhc+gp8iJ52SPIpIXb1pwZBjjgdqps/II6Z9am30+RBzsfOUcAFRg8GvNoraeDxBb6bDBNCPNJZnTaSgPBGRzmvRJG4BAHHY1Unso4pEvowi28IwEUcuzH5iT6jg/hUy0RrS956l+eOYWojg34bhismw4/3u3865rRtOnOtSHZDGm75zHcPMSvuzd66h71LS3eQsGCLnav8R7ViPrRSN3j1CwinzxAyhlA+gIOfesU0dai9ypqkN5FrckWZmVDgGOfbhTyCV6HniugjISBEGPlUCueXWZr2fzrqO1LghcRHOPfNdCqkoG74FOjZtmeKTikV5Tlzjil6jB78VIYGfc6cqo55/wA+lJj5PwrblORM8Qm0vQ4ruaJ2td6SMrAyAEEEj1pV07QO32UjrxL/APXrP8S+GJm8T6myMQr3MjKNhPU5/rWSfDFznhuv+yahxj/MaqUv5Tqho+glRxBtPTE3X9alTQtCckAR8df3xrj/APhGLofxj/vk/wCFMbw5dr/EPyNLkX84+d/yHdnSNExgAE4xxKTXmk3myXLx5LFSQAPb/wDVVx9AvFGcrj8f8KzJY3gnaJ/vKcGtaNNRvZ3Mq03K3u2NW1sraXTpPNt7gXe/CNn5NvuOvrQdKjwRmQH6j/CqdhY3N95nkMP3YyctirH9i33H7xf++jVt2e5mtVsbumaDod7LHbsb4TMvJyoXIHOODW2fh/oythrm5U/9dF4/8drio9J1RX3RS7W9RIRU62GujG28kHGP9eRisZKTekzeEopawOw/4V5o7Di8uh/wJP8ACsjxR4LsdF0Y3lrdTSOrqpVypGD9B9Ky/sniQdL6fj/p5PH60k1h4iuYDBNdyywsclGuCRn6E0oqaabnoVJwcWlDU5zHy/jXqOn/AAu0++0+2uxq0w8+JZNojXjIBx1rhh4Z1TBHlrjP98VrQQeLreBIYb6dY0G1EE/AHoParrScl7k0jOjFRb543Omk+EluPuapMT/1xB/rUDfCOTkjUZAPU23/ANesR7nxvZwyTfb7oIiZZhN0A61mt458TM2W1i4Y+5B/pWMYYh/DNM1lPDreDR05+Etx/DqOc+tuf8a9Y+Emg3HhrSdQ0+abzVadZlOwrjK4PX/dFeNab4g8f6lb/aLG6nli3FdwjjxkduRXp3wo1DxbNrl5D4iMjW7W+YiyIMOGH90ehP5VEvbXtOaY/wB01eEWj1ys2dNkzDtmtKql2n3W/A1rSdmZTWhVxmnCiiupGItTxnKfiagqWI/KRVIhktFFANWSLRSUUxDqUU3pS0wElGY29cUincit6inmo4+I9voSKTADV3TGxK6+ozVM1PYttul9+KmWw0aeDmipNtFc/sx3Phi2X5ohkfeGB+Nd3og33koGANmefqK4vyHtrxYpUaOWNwrKy4IOe/pXcaEo8+ViRwoGMe9dNPciextFCo+8CDxgGmjhs9Oc1Lgc5YfjRtHJ3D866DAiIOcY/WjrjipWVeDuH+NMKj1H4UAWbPGw+zCrJIJ/lUFooKMM55FWShGOKzluax2Gd8/570hyDkYzTyMgDFNOAen+f8ipKIznGaRueOMDtT9pK8g9cc0yRxHGzuflAyaQzm9ROPGvh4cZ+0xk4/3xXuykEV4HOr3XjTQyTsaS5jG4/wAPzjFe4rHcAf67r/s1zVHdm8NEXTKxjMYxtPWmFeCKr+VOf+W36U/ypsH98fyqUDHrK0GSuNx45FVtxJJJ5NMkhn3czHGfSomik/57tWiM2WN/vRvyODXKeJ9UutGa0ZLlwshYMMDnGPb3rBPjG9z/AK4kd+n+FFxpHo5fk0F686Xxfc952z9B/hS/8Jddf893x9BSuB6Jvo3jHWvO/wDhLLnj/SJfTjFH/CU3P/PxKKOYLHonmYo8z6V53/wlN13uJfzpyeJLp/8Al5l+u7pRzofKz0LfSF689fxPPk4uJv8AvqmjxJcHn7RN+LGlzoOU9F38UheuHg1yR8bppv8Avsirg1MuARPL/wB/DUuqkUqbZ1u/PSkL1zEl6yxqfOk5H/PQ1GLtmVj58/H/AE0NT7dFexZ1W/nGKbI3y5rk2uXfhbqVTjjMjEfzpkF1Luw1zK5H+22PyzSeIXYaoPudKXJRfypd58tuvWueW6fGfMbk4yXNMub5kYgSt17Oaj6xHsV7BnSK5O7r1oVm8scGuT/tJgf9ZIf+BmnrqZK8Fj77jT+sLsL2LOnUsX5HA60rORKCAcVyU2pkDBZj9GqodSbPLt9Sx4oVddhOkzuJZDs7/hTTIW2oB27VxB1N/wC83/fdRnU2xncT/wACq/aoXszupHI2ipd59D+VeenU2yPr2NR/2r1GV6/3u/4U1UF7M9Cmf5QMH6VheLr7ULHw5Je6d5onjKqwU/KUbg7l6EDrzXMf2qWA2FMA8nOc1W07VTP4nsoLiPNuzunzE4LlGwfbB/ofprTXPJIl+71NnRvGkKSpp2py7hkRw3UqCMS4HRlBIVvxIPseK7JILeSB5bcWyyNks6HYxP1HU1474i0B7JftAWS6sJm+4+A8LZOcuePxPX19ct9cuINIll0zUTaCGTy/ss9yHcjpkKVPH0bjHSnUw8ZXlB/I1hinC0Zo9M8Q6hYeGLWbUnSGS9kO2MMxZpG7AknOK7PTbmS80m2uJ4BbyzRK7xE52EjOM1856bfvNrUWpa002pwWkqlwZTgjPGMjpnGRxXt2meO/DV+FRpms2PQXSBVP/AhkY+pFZQpciCtX9q/I6CSRkwsbMMjBx3qPcdmKe1pHNGGhVHRhuBXGCPXPTFVltEBIKAEHBBFUzJHF66Fj1y5wAckEexwKoYI+YqME8ZNbXiO2EerjaAoaJT2HqP6VlLgdF3dgT0FebU0kz06SvBCLtCdCDjpUqqzfM4IA4BJ/lSLEW9efXpU4gZsKCXPsOKybN0iPfGCp2MQvqePrivKfFyY8TXTdd+1s+vyivZoLWSH5wqBhzucBsfnxXmHxJjf/AISGGduTJAuTjGSCR/hXTg5/vLHNjIfu7h8OU8/V76zzgS2x7+hH+Nd//ZDrPHKkiEqwbAXjO0Aj+dec/D9tvjOCPr5yOv8A47u/pXt8FiXYfLkDrWeOly1B4OCnT1Maw08R72nij8x5C25RnAwBjmtODS7aRG3QoOOXZRj8PWtRLQn5VACnsvQ/41bSzRAN2Sf7orzZVjuUIx0MZNFtJDxAhGeWMYqdNEskbBtoHOOMwittYScBQRjpnrVmK0I9j+prL2smJyitzGi0OwyWksoST2Ma1ai0OwPSxtsnqTCv+Fb9tpbyHO3j16VLPPpul/6+UPL2jXk1rGjUkrt2RyzxEb2irswJPCVlfWc9v9hgPmxsnEYHUHvXyrq+kT6Pqs9lcgLJE5jYA5wQcGvsX7Rq2ppts4BZwHjfJwfwH/6q+UfHttNYeM9RtbiTzJYrhgXP8XPB/EV62AjyNpXaOKvJyV5Wueo/CDULzWfCdxotppNqRbzDzLjaASG5+Y85PB/SvUtH8JNpUjXs90zzYPyINqDPt3ryj9nLUhFc69p5I+ZYpl/Asp/mK92muNylaK0acKjb3JjVny8sditUc67om9uakpOtaxKZm0tK67XIPY03rXWjnYo6VLEecVHToz8wqkQyejNJmjNaEi5pabmlBoAWlBptKDTEOzUUZxJKvuD+YqTNR9Ln2Zf5H/69DAc1PgbbMp9DmmGli+/SYHQUVCko2Ln0orMZ8RySySgTSyM8hAZmc5Jx7/Suz8NOJInlH8SKa4x1/wBFGMZ2c/lXT+DZN+mMc8ghP8/nWlPcmex1O4+uKQt7cZ+lR5OcDrSE5GfTjHrXQYWJlbKkehz0pueuO9RglSeOvFB4buaQy/YyYdkx15H4f/rq6WHTuB1rFhk8qVG9D6VrjpgHI9fXpWcjSGw9WAPTAphwc4HvS8FR/Kg9ep5qSxB15qtf5+yEY/iB/WrBGMc8VXvMG1b04/nSexS3OZL7fGegt6XUf/oa174or5/m/wCRt0Pn/l5j7f7Yr6CUHFc8jRDD8uAFyfX0pecGnlaCODQhMiRow375Sy+gqqRycDjPFTkcmmEVoQ2cH8Rbd54tOVWI/ePnH0FcQNKnOMvJ+letaxYLf3NnG3RSxH5Cki8MxFg23j2FHJcXNY8qGlSn+OT8Mf4VKNLkA+9J7f5xXrJ8Lw8YU8e1PXwxEP4Kfsw5zyUaXJ1zJ+OP8Kd/ZcvpJ+f/ANavXV8NQA5CU7/hG7f+5S9kx+0PJYtKkZhnzfzqzNo0kTFY1kK47t1r1NfDkA/gq03h+3PzFB0qXSdylUR4u2mTDqkn51LFpMhYfI/5162+gQZP7uhdAhH8FP2LF7RHAWHh+OQAusv/AH2RW0nhuzOB5Unv++b/ABrsbfSI06LirX9nJn7v6VhKhK+5qqyOPbQLFFQeUzd+Zn/xpr6HZrExEEgI4x5r4/nXWz6axA2r2xkV4bqdl8UTqV0tquoG3ErbCNmMZOMcZrP6vLuX7ZHdHRYd5KoyjH/PRv8AGq/9jxliQDn2dv8AGuFXSviVKy+amqnb833kH+e9TvpvjtruB1ttVt0Vhv33O4Nz3z2qXhn3LVddjsU0yJcfL0PA3Hj9ajvNNUNjbkf7xrpEgJjO5RnpTbu3AzlQAKx9i7mvtFY5JdODNjy8/wDAjVpNJhKD9znHua3FthuOAMhatJbjyxx7UpUpAqiOOudKUcCIA+9Z/wDZq7gDEvJ/u121zbAyfhVFbQSXWRyF561UKUiXNXMIaLDtG6FCT/s0f2JD/wA8UyO+K6pbbdMMjAA4qlrkv9maRdXS43RxnBx/EeB+taKmxOSscbeSaXYuY2eHzAeUABOe/NY1zq9oMmKGPjjIQHFcpe3bCVmLEkccnrn/APXSI/mRknP064+ldsKCW5ySrNmrcaxKzrgoVBIIPB/PtTLW9dNWs58hTHOmD2wG5/n+orNVnjbg8etPC7AWyQ2Mgjgj0/pXRBKLTRk5NnsWoy6fp93Ml5eWqRyqGkhldd31C9efpXFXPhPR7tZdQtEzBk4WGXMZI64x/LIrTsdFtbmyR1R2MnzOzgBiT1zzk/jTk0q60ouNOZYzccSKMKvH8W37uR2wB1712VsDKcvaLS5UKy5eV6nLWtrZ6ZFc2t3a3DSXabWjhjBESngEnPUdcAc1y5lmgDwrJu2sVztIPHfkZ/OvUodHMJJlyTnLbmyST3JP0rzvxDZGy1m5QBtrnzF3c9ef6VnXoOEEyea7LGieK9X0GXdY3bxp0aLrG31Q8fpXpvhj4oWur30VlqUQhu532pLGcozH1zyuT9a8Pc4Qc/Wrugbn8R6aBkk3UXA/3hXE0UmfQHiaBGvbeRyOYyMepB/+vWOttkDJVU6DH+ea6bxDAzi3cLnDMMk8dv8ACsWK3LHAG4+grxsQ7TZ7WGV6aI4rdekYBbHcZNW4rcx/IdpPXJJwBViK2YLy+D02r0/Gr9vZlzkgkexrjlOx1qJSW0DHG4Mem5s/oK84+Len+Q+lXI5V1kjzj0IP9a9ijs9xAC4x6Vwnxk04Hw1ZXOSTFc7SR2DKf8BWmDq/v0YYtJ0mjyzwXJ5PjrR2Y8POq/8AfXy/1r6YjtRtAHTrya+X9HY23iHS7nB/dXEbn6Bga+u4LFnxwcVvmcXKceU4sLV9nB3M5LUHoTj2qzFY5wABitlLBI13SsFUeppBdxqdlnCZn/vdh+NckcI1rUdhyxTfwleHSzjLAKvqac1zZ2reXAhuZv7qDNTmxuLvm7mIX/nmnA/GrcNvBaptiQKK7aeGtrFW83v9xzSq3+J3MxrXU9R4mmFrB/cj5Y/jVmz0ew08744gZT1kflj+NW3mA71VkuPerlKlT13fmTzTkrLRFp5go4r5M+NFt9n+JWouB8sojlH4oP6g19QST+9fO/x5t8eKLG6A+WW0Ck+6s39CKeGxLnWsxypcsGyh8Er8WfxE8rOBdWskf8n/APZa+jZJ+etfJngC9Nh4+0SccZuFjP0b5f619SPJx1rnzRuNRNdUb4SPNE2VO5Vb1GaWoLR99rGfbFTV103eKZMlZ2KlymH3etQ+1XLhd0X0NU67IPQwmtQpynBzSUH2rQzZNmlpgOQDS1ZA6ikzRTAdnmlzTM04GmIdUch2vEffB/KnZqO44hLd1Ib8jQwJTSxfe60080sX36ALm7Heik3e9FFgPkH7FK7i0QKZSdmM8Z6Vv+GdKutMguIrkKAzgrhs9qzopxFqSXDBsCTcQOvWt0a9bbc+XLj8P8aI2WpMrmltPvSAc89KoDW4B92OU/l/jSnXIGPKS5/D/GtOZEcrLuBijvj+Rqh/bNuf+WUvT2/xpv8AbUGeI5OfYUcyDlZo4rVtpTJbpknK8GuX/tqD/nlL+GKs23iKCINuhlIJzxipk0XFWOmB280044OTWGfFFoCcQT4HsP8AGmnxTanrbzfp/jWZpob2c88cDiq9+c2rcEZwOB71kf8ACU2o4+zzHn2/xqrd+LLR4Sgt58nvxj+dJ7DTRVumA8VaIRwPtUfB7/OtfQiivmuPVI9T8U6QY45ECXUfDYyfnH+FfSq4xWUi0LTT0p3WmE0ITISOaaRT2xTDVmZh69rVtoBtbu6XdGztH94DBK57/SqUXxQ0OMfNg/8AbZa6KbSrDVsRahZwXUSfOqTRhwD64PfmnR+DvDi8jQtNHP8Az6p/hWkU+hLaOfb4saEBwE/Gcf4U0fF3Q8dIv+/5/wDia6tPCnh9emiaaP8At1T/AAqwnhzQ1+7o9gP+3VP8KfvdxaHFH4waIv8Az7/jO3/xFN/4XBo5ztFrj/ru3/xFd6uh6SOml2Q/7d0/wqRdK05elhaj6Qr/AIUe93DQ8+Pxe0oDOLQD/ru//wARV3SviQuv3n2LTLOC4uNhfYszD5R3yVAruF0+zU8WkA+ka1WvNPtUv7K4Hl25QuCyhVzlenIpWl3HzLsZI1PxE43LoURGcf8AHyKUah4k/wCgJAMf9PA/xrZglsLSFIPt0bBc4aScFjk55OfekbUNNQkNfWw+sy/40/mL5GWl/wCI+p0mAD/roD/7NVgX2vt/zD4B+IP/ALPVhtV0gcHU7ID3uF/xqNtb0IEE6tp4+twn+NS4+Y0/Ib9s1/A/0KAfgP8A4ug3GvFf+Pa2B9x/9nTW8ReHxkHWNOH/AG3T/Goj4m8NqRnWdPH/AG3X/GlyLuPm8iuNQ1t7+e02WivEiOd3AIbdjHzf7JpkkWtyH5nssf71U18UeGYvFFzcNqln5clnEu9WyCVeTjjv81Xm8ceFF5/te2/BWP8ASp5E92VztdCD7FqvGZLLr/z0/wDrUybTdVlxiayX/toT/wCy1M3j7wov/MVi/CN/8KafiF4UGP8AiZgn2hf/AAqfYw7le1l2Ko0nVw3/AB82eenDf/Y1NHpOr7cfarVfxz/7LSN8RfCw/wCX9z9IH/wqNviR4XB4upj9IGo9jBh7WYT6NqYcIdRtgzgkADJOOv8ADXHQT+L2BeLw/fMp5DBosMPUZrrJfiP4WkA8xppMdAbfOPzrqtPdJtOt5o+I5I1ZQRjAIyKXsY9A9rLqeXhvGxORoF8pz3lt6w/Ftz4mi0Yrq2nTWtvJIFDySwtlhyB8vPb9K9yIB7k15X8ap9mnaXbBj87ySHHsAP8A2Y01SSYOo2jw+5l3yPH1HSktJAVwwGV688+lMmjZn2bvnA49/pVe3LLdEMCDgnBOOa2MzTH3gcnr074/z/WpTtKDaBgHPA71WZl2HPpgAdf89KsIAybgB82D78/5AqhHp3hi787w9bSQqjzJGFYnsRx/SrMksz3cRlVctuXAHsf8KxPBMhk0dgrBmjlZM598j+ddDdSLDCoklUyMwVDjOT0/Svepy5qaZlazGFURRjl2OcEkcZrhfHduPOt7sA4bchzx7iu+i8xo13hN2zDMuOTj6CuX8bW/m6M0g6xMHBH6/oazxEOakxp6nmE8ZUZ6rn0rpvhnpjal4705cjbCxuGyeyjP88VgTRb43xlsc10fwsultviBp+84EgeMfUqa8CWxvHc+gdZhEtsmQOHB557Vn2+ngoCDj6/4Va8Q6vBo+iXGo3EUkkMAVmWMDdgsBxk+9cVF8YvDiAH7DqI57Rp/8VXjYuhVnO8Eevhq8IU7SZ20dhcebGqlPJ/iyDn8BWpFaggAdO9eeD42eHVOGstS/wC/af8AxVTp8cfCqgf6FqhP/XNP/iq4ng8Q94lyxcejPR47UZGD+lc18UNIkvPAV2Yk3NC6SjHYBsE/kTTdG+MPg++SRpGu7bYMgTRD5vYbSf1xWL4p+Nej3Oj3un2NtdKZoWjDlFI5GOea3oYSUGpPc5Z1nLToeLWXk295btcAmNZMuq+mc8H/AD1r7C0+/wDt9jbz2MLeVNGrrJJxwRkV8Vz36zN8qlTnNe/eHvjv4Z07wzplld2+ofabe1jhl8uFSpZVAJB3dOK9WdOTaZytq1j2D7CJG3XMhlPoen5VOPLiXCqAPavJm/aE8J44ttU/78p/8XVZ/wBoLws3/Lpqv/fpP/i6zacPghqTZvdnrz3AA61WkufevJG+Pvhgji01T/v0n/xdQN8ePDTdLTU/+/Sf/FVyVI4mX2WawjBbs9Ze496rPcV5U3x08Nn/AJc9S/79p/8AFUw/HHw4f+XPUv8Av2n/AMVXJLC4l/ZZ0RlSXU9ReUmvHfjnbebZaVcgfcaVCfrtI/kavH43+HP+fTUv+/af/FVy3jr4j6H4o0WO0tra9WZJd4MiKBjBHZj6itcJhq9OtGUo6Dq1KTptJnmun3Js9StLpfvQzK4/Ag19bAlwCuSGGRivkDIxxnOeK9lg+MlhHplrbvHemVIUSTy0VF3BQD/FuIznuK7sxw063LyIywdaMLpnuOm5FqUbqrEYzz61crgPhh4us/E8OpJaxvGbdkZlZAv3s9ME5+73rvqKUHCCjLcc2pSbQjDcpHrWe3BI9K0TVGddspxXTTMZjM0ZpuaWtzEkQ/KPbinVGh6j3p+ferRAtLmm5pc0wFzSim0A0xD80jjchU9xikzS5pgJG26JT7VJF9+oYvulfRiKlj+9SAu9u/5UUz8aKAPka/ZoLV3Q4dQDnHvitNPDmqMgP2+AZGf9X/8AWrP1VS1hKMdcfzFem6Pol5q52WcDSsi5YKBwKIpPcmba2OIHhrVcj/T4Mf8AXOpF8L6mcj7dDn/rnXZy2kltcGGZSrqdrBhgg12Hh7wgNVtTOZNijgZGc1bjFakKcjxo+GNVI2i+iLY6CI0knhvVU4e+jUkcZhr03X9FbSbpot3TuK5uXLHkkkjHPWhQTDnZyR8O6kcY1KLp/wA8qP8AhHdRbCnU4hg/88R/jXUU09OBzT5EHOzEHgzVm4/taLn/AKY0q+C9SOSdYjx/1w/+vXX2r+ZbK2eR8pqyqgHn8ayasbrVHEf8IRfk860n4W//ANeoLnwNdpEXbWA3QEfZsd/rXekZByar3/8Ax6OxxwV6fUVL2KSR5dp9m9h4vtLZ5vNMV3D8wXbnJB6V9Rp92vmiQ48exAH/AJe4Dx/wGvpdBhcVlIa3FpjVJ+FMakgZC1RmpWGajPWtUZsqXmqjSESZoWl3tswpAxwTn9Kq/wDCbDOPsD8H/nqP8Kr+KD/oEBzj9+P/AEFq5jODW0NjKb1OxHjnH/MOb8Zv/rUp8eEDjTvzm/8Asa4zd+lGeaqxHMzsv+E8l/6BqD/tsf8A4mmnx7cHpp8X/f4/4VyGe1JnGcUrBzM7D/hPLr+Gwhz/ANdD/hWF4n1u68UWlvpcsMUEUs2WdGJYYBPf3FZoY9cUgb/TLQ56Sf0NNoLsxpvB9vA4U3MrZ6HA/wAKaPClp/z2mH4r/hXUal99M9xxVFgeNpxUuESlORjDwrYjrLORnuw/wqVPCmmkHLTn1+etA7h17VPCHJzn5RQ4oFJmavhPTR/z2/7+U7/hFNM7pJ/38Nawb2p+78anlRXMzA/4R3Tk1Rk8uQx/ZQ+wyHGd5GaZeaJpsUO+O3xg/wB88/rWu/8AyFE6/NaOPykX/GorsBrdxjoKFFWByZx7w2sWqCN4wIthbBY8frVYxLKk8kJ+SPJBzyT/AIVY1bBnhiUjdK4X8B1qGABU1KPngnj2xU2KuJciNdMhmVcMSMmpr+FFtkaFAPmBOPTvVOV93hmM45AHfHenancj+zY1z8zKDgHnFFkFy3FEk80cpjCwlwFA/iH0rYXxdr8GsS2CarcLbRIBGikAKBx1x/nFZrSBbWIj/ZOR06VFHEt14n1HqGiY42ntk1N0ilqauo+K/EEUtrs1q8VWfDASdeRWH4o1a81KaJLq7muDEpI82QttyecZrYl063mCO7t8hyvPT/PFcz4hjS1v5FU/KI1Y5OSSf8/ypRmmxuLRhzrAqb5S2euB361UgbfKWOQoHA9OacIZLl9zHapPAz/n/OKSI7L1lJB+XHHatCSxJwCSQM+n+f8AP41LalPJdSB8h56dD0/rVd2yMk8dKS3lCXOMjDDBOOh/z/OgDt/BVyqG5iZipeQNknPVf1ziulu/31woTOI2BGB79q5HwfLGlxdqyhmCqykDjOT+XQV2KulraiSUfO5HB9c/5/KvYwr5qaRnLRk7NOWwowq579enP+fWsfxJHI+kyRuoDOhPOa6CMzRfJLjaCfmHb/PNcr4ne71EywwOY0QHkf5/Wt6ukGT1POI2OwNycZzntWr4OtpJfG2lRwA71ukcEd1Byf0BrGi/dM8Z+8pIP4V1Hw9eVfH2ltHH5mWKt7Dacn8BXz8tDdHver6QuuaRdaY0nli5j8vft3beeuO9clZ/ACBsFtYkZfU24A/nXoVpJ5dzGwCZB/j+7+NdEbG5uzm9uwI/+eUPAP1PU159Zz5lys6YtJankep/Bnw8lgbW21C6m1HIC+Uike+R2/Oqth+zqsw33utyRA/wJCCfzzXuEMdrZpsgjVAPQUrXI9azVX2espfJCb5tEjyFv2eNJUfJr18repiQ1RvP2e7GO2ldPEF0zqhKgwLgkDjvXsr3PvVaW4yCCawqY63wlRpye58UXlobR9pO7OR0xzVSt/xNA0OrXUJXHlTyJ+v/ANasCvXpy5opmE1Z2CirllpV/qRYWFnPdMvLLBGXKj1IHSr6+D/Ej/d0DUzjr/oj/wCFU5JbsSi30MSitUeGddMPnf2NqHldN/2Z8fnipIfCXiK5GYdD1Fx6rbP/AIUuePcOSXYxaK6D/hBvFP8A0L+pf+Azf4Uh8D+KB/zL+pf+A7f4Ue0h3Qezn2MCit8eCPFB/wCYBqH/AH4b/Cqmo+HNX0iOOTUdPuLVJCQhlQruIGcChVIN2TBwkldozeNo9e9eg+CPh1aeLNCfUJNSlt3SdoTGsQYcBTnOfevPa9d+DV/ix1WzJHyyRyqPqCD/ACFZYqUo03KG5thYxlUSkd58NvBUPg/VbwwahJcLdQgFXjCgFTkHr7mvSjXI6Pcf8TiEZ4OR+hrrjXHRqSmry3OqtTjCVo7CVXuk+UMO3FWKbIN0RFdEHZmEtUZ1L2pSMHFJXSc4qnDn3H+f51JUI/1in1yP8/lUmatEsUngn+VRfaRtyI5OuMY5pXDkjY4A7jHWkVZs/NLn6LVCE+1f9MpPyp4n64RjgZHvTQku0/vfm7HHajZLx++IHfA609BE0b71ztK9sGn596rCOX/nufyqcUAIvErj1ANTR/eqAnEy+4IqxF96gCzz7/nRS4NFAHyZqeBbMeOWGB75r1Lw94kvNBLSWhTDqA6uuQa8q1U/uOn8YHI969F0yMS3cELMQkjrG5HoTShs7kVHbUuXepTahqEt3cEGWRtzYGBXRaF4vn0qAwoEdD/e9a5XVFMGpXEDPvMMjRA+ynFV/MKn6Vro0ZJ9TZ1jWJNUnaWVgWY9RWM5zk5z6980by3GeaQocU0BEeM5IxTSMZ5HvT2XbnnkVGdv5mmMt6fJtmMZ5DfzrV2ZyR2rn1Yq4YHBBzXQQuJIVkHRgD6VjUXU2pO+gbT36VWvFP2V88gEH9ausMj0AqreYFnLweBn9axb0N0tTzabb/wncRH/AD9w/wA1r6YXpXzPOCvjuIsACbmHp/wH/wDXX0yozWbBbsDTD7VIRTGpoTImFRGpmFRtWiM2YPigf8S6IjtMP/QWrkyD3Ndh4kTfpg9pQf51ymzjitoPQynuRYI6n8KO3J96k8vJxTvJ5zxg1VyLEBHvS4PepfKJOOtTf2beHpaT89/LNFwtYqdxzRj/AEi35P8ArV/Xiro067720oHupFMktHikt/MjKFpUxn/eFDulcUWm7FjUVDMhyeh6VTaONR8zYz7962NTjRBH9D/Ssl40lOD0FTe7LtZFbMW77w/A1ZhaEpgv85PAqE20YI5/WrMFvARv5yORzQwQ/wAlSM80ohQ8f1xTwQBQW5PPFRqXoVVhRtcsEb7jpKhx9UNdp/whumSIys9xzwcsP8K40uF1fTnHaSQZP+4D/SvWEwcEbsnrxTQ7I4//AIVz4clmSR455JVPysZWGPyFcWfDOn2/xVm0VoibG4jRtm85OUPfr1FeyDaJQoY8mvNfEDLb/G3SnJx51tGv1O5hQxmyfh34XSD7Oun5iHVTM5H86uR/D7wq6ANpEDA9d5Zj+HP6VuBACfl3c9xmp4yFlCjAUH2osM+e9at4bXUr22gGFgmeNV9ArED9Kowyi28TahL3aJX+uVH+Nbvi3aPGepwbct57HIGM55/rXNao/ka6zfKBJbx9+vyj/CuWTvdFxVtTVa9LLt2Y9cVx/im73XxVX+Yqu/H44/nXTq2BweK4/VJLeC9meRS87OTgjJ69falRjrcuo9DPigJXzZ5CIwM4BqK1/eXDFeCxwB171FcXMsxwxwvXaOlTWKqdzPwo4PFdRgWZQG6HOevP1pkSKGBBOe+MHildzIxCdD0GM8VYt4TtDNncRkd/SgDoPB7IuqTqRjdErDkEEg//AF67NwlzcKjA+Wg5x3rz3Qz5OpYUoN6cZ+o6e/8AnivQdKxJy/CL1z3/APrV62B1jYzmbHnJLAjSADIBP+PSsO7uUbdGqBmJPOKtLDcT5dcrFIBtUDBUfzqtqb22i2Blk5kI2onGScdv8a6Zyb9BWVjyzWIBZa3PGD/FuPPQnnFegfCTSrO/1ue/k3+dYxholU4BLZBJ9cAdOnNeaX9xJcahNJMRvZucDivQfg3qCw+KJ7RmAFxbEKPVlIP8s14NXVuxtA9xI7ZxW7DqHmW8bg/eUH9Kw+lRWVyRB5ZP3HdPwDED+VeJmLcYpo78NBTbTOi+1Z70xrj3rNWY+tKJGPTmvFdSR2ewSLpnJ71E0mc81Tury2sYvMvLiOBPV2xn6DvXNy+PtOkuhaaZG1zOQcM/yJwCfr29qcadSeyC0UeM/EK3+z+JdVT0uWcDH97n+tcMetdp4y1CXVdWnvp40R7hVfagwOBj+n864xhhyPevqcNdU0meVX+Nnb/DHxDNoGu3RhhWU3FuUwexDA56/WvfNJvtQ1nSra4kMUayghgVyVZTg8dOevTvXzN4RZh4ltVVsF9yZ+qmvoLwlLLbQPA0zPGDuUEcAnrj8hXFmFNX5ludeE1Vjql0+AOJJi07jo0hzj6DoKnLqowoAHtVcz8ZzULzD1ry7tncqd9yw0uaiaXrz2qo9wB+NRG5G4DnrQos0UUiy0tedfF2Pz/DFtKASYrtc49CrD/Cu0e59q5Px+PtXg6/X+KMLIPbawP8s104a8akWZ10nTaPn8ffGfWu6+FV0YfEtxCTxNbMMepBB/xriJhtnce9b/ga5+zeMbF84DMyfmpFe5WXNTaPEovlqJnvtnP5Wo20hPSVc/TNeiGvJTdFSCOo5BFerwSie1jkHR0DD8RmvOw6tdHo4jWzFoNL2ptdKOZlORdrkYplT3QwwPtVfPFdMdjnkrMDxg+4p5/Wo2OUOPSn5z+NWiGLRSdTRVIQtLTaKYD80Cm0UxCSNjY3owzVmMfNVaXmJh3xViE7tpHfmkBb49T+dFJzRTEfJl8pbykHOZkGfxFd9GywXCMrE7GDZxjpXCXAzc2o4+a5T+ddtld3U8c0oapk1Fcv6xPFd6zeXNsS0UshdTjHXk/rmqWcD2pyMintjPXFNJ+c7fwNaLRWMkrIu2ccbTLvIA75rsvFOi6Np+kW0tlOGnfHyhgdwx972rgVlwcg4PrUj3DuACaGrsaIpMZIFREGnls44FNbkmqAYc561r6XNuhaM/wnIH1rJPPXt71YsZhFdKScKflP41M1dFwdmbpBIzVS+YfYpSRn5T0FXwvPSmz2UU9tMXheQhSTiQrkY6VySdkdijdnlt8AvjmLn/l4hz/47X0yn3a+b9daL/hYMfkwCBfOg+QMzf3c8k19IIPlqb3SFs2OIqNhUtMYU4ksgZcc1Ec1Ow4qMitEZsxvEP8AyC245Dqa5Inj3rt9UtJLyzaCLG9iMZOO9Yq+GNQbvCB/vGqiyXBvYw1ZuenHbHvRyeT29q318J33eSEf8CP+FPHhG95zPCPxP+FXdB7OXY50HBHP44rtUjvJdQCRp+6Me7/VnHTru9c9qzh4Oue9zEP+Amu0hUxQxIckhAMg+gqWlJicHbU5aayv9+0o4X/drJ1uN4zZGSNlbzejA/3lr0QHPQH86xPFGnR3kUEjs6mEMwx+B5/Kuidbmhy2MaeG5Z8yZy2uDasZxnrWEZOeBXb6jpMVwE3u4Hsazh4ZsuCZ3XPOGcD+lYqyWprKLOZJyM0gcgYrp38O2gKlZCVz82XPT8qkg8OWkz7ERmPX7xFaqm2r9DO6va5ym9uTxR5jdCeK7T/hDkP/ACxUfWU05fB8QGPIjP8A20NLlXcrlZw0j/6Tp5J6XBH5xv8A4V61E8zKCqpjAxkn0rg9W023s7zTlSNVIvUDAH2Yf1rr4X1NxEYo7YwlRguxzjHoKya1NUtCw7XrSZHlAdvvV5z47c2vxR8K3BQb3ESMe3EnP869Bk/tYEBDaKoGT8rVzfiexFzqWm3eoRW0s8Tf6NKqsNhyD0P4VLVluOMbs6122yOMd6fCqn53wBn061nNa6uSCL22B7/us/1qVItUCFJNQhIIwAsAyCe/Wk5tdCuXzPIvH0YTx7fuI2AYKckdcoK5HxCjfarRowdv2dTyfQsOteweJNKU3ryXUcc9xtGZTEATxxXGG1SW0gLRRuNpU7xz94+v1rk5veZsqeiOXuZWgsZZo8b1XK+ma45/tJlaWeRMk5ZioJ/OvUfENkB4dnCxoh2K25RyBkE15xc2rO6IVOxeenU10UUrGdVWZkuY5XZkhOOuc1NaqDCQV5HaprgxxYVUwzcAdM/4VVtJBu56Z7VqZF1It7ZbhR25qyeT7Yz6VAshyOQPXnmpR0C9F7Dt+H5UxF3R7WG41KVrhwqKoKkNg5ya7211CyjHlp+8XjOehFeW3UxiuIsSyIGB+ZBk4/H6fzqRbyQkf6XdtjDHkLjn2/xFdVHF+xjZIThzbnr8GpxmJhGoLqCFQdTzwK5fU7CWVmvdXvrW0U/dWTJwPTHrzWLBdLdxX0UUMMU0do00EkckxYsrDpudlPBPYVy80d3PMWuBLJu+UOxzzjI5pvMfaaNFulyrQ0dZtNMZZprS5mklQjlowikfTJNVvDGr/wBieI7DUMHbDMrMefudG4+hNZ0tndW6HepA7gHNVlODXLOSm72EtD6+VlcB0bKsMqQeoqpDC/n3HZBLnJ4HIB/rXPfDbWDrHgiyZ23S2ubWQ5/u4x/46VropNMN5cl3ndYRj5FPevJx8E6ep6GDlaeg6XU7K0wGcyuP4Yx/WsfW9Y1f+y5bizhNpFHyWxkkZx1P17V0lrp1laf6uFd394jJpb/yp7WSGRFZHXBUjINeRHki9Eeg7yPF7lri/ZmnmkmlbnLEmiy02a3uEuB8vlnP510k1rHbzsqoFAPYVXlT/R5B0Ow/yr0o2toYNanA+M9O/s+5gQEsHhyM/U1xEn3zjp9c16d48QTWOlXGMlkKk/l/jXmcow1d+Hd4Hn4hWmXvD83keIdPk6YuEB+hODX0RpT+WT79a+aYZDFPHIDgowYfga+jrKUMqMucEZrHGK6R0YF7nQi5+X9Ka7SFN+1ivrWZI87QObbaZQMqG5B9qrzXOoRTsUZXi/ikfapAyC23nkben19q8zkPTuaLzYqLz8HI9c1zn2zU3QjcqlWIdyyYQbup9DjH5niqzX2pkbkmh27QTudMZ4zjn2P51oqZPMdNNJh26YyaytbT7Vot9bkZ8y3dR9SpotJp3iLTtuJY7G45XjH3eO9PZ+CDjB6mqirMmTurHzzcD95n+8Aat6RP9k1uwuD91JkY/QNzUV/CYLhoj1jZkP4Mar5OEb04/wA/nXtbo8LaR7y74LDivVfDk/2jw9YyZz+6C/8AfPH9K8atrv7TZQT9pI1f8xmvUvAtz5/hxUz/AKqZ0/8AZv8A2avNp6SselUd4pnSk03vSnrSZroRzsjuBujPtVI1fPIxVFhtcitoPoYzQn8qRP8AVrnrjFHShehHof8A69aoyY4GjOaTpUF9LJBY3EsIUypEzIG6EgEjNUkIs9aK8t0z4jatfapaWskdnHHNMkbMI2yAzAEj5vek1H4iaxaahcQQpbNEkhCM8DKWXPBIzxW3sJXsZe1ja56mDS5rB8J61Nrugx3k4jE29kcIMDIPH6EVuZqGrOxad1dD+1Ps+UUf3ePyqLNPs+JJF9Gz+YqRmhx6n86KM0UxHyhMc31l1wblOPxr0C00a/vbC4vYLYtbWwzLIWUBeM9zyfYe1ef8tqdgP+nhev412X2iXyvK8xxETuKBvlJ9cetFPYmY8SAHnP4GnvJFj5V/E1Xz7mnKA2Mtjn0rQhC7hzk/QUA5J/SnvFCIyyzAtkcetRY6d80XBqxqRaJfXGjyarFb77SJisjqwJX6rnIHPWsxmJ69PQ1JG5EZTzGVG+8AeDj1HeoCeTxQgFz3poPOaCc59+nFA6etMDrLFvPtY5uuVGeO/erUsLi1k4JO08Vl+GZQ5lt2xn7657+tdYbVmt3PfaTz3rzK8uRtHq0I88UzxLxCrDx/Hv5Ilh49Pu19JR/dFfO/jCMJ8Rl68yQE5/4DX0QnKinF3imYzVpscaYwqSmNVozZCwqM1K1Rkc1oZsIl3ToPervkA9KgtFVrpA2T1/lWqI19OazlPlZvT2KQgp3k45zV1oQrYxSeUCelL2hpoU/KHPQ1Jt6DngVP5R7CsHUzeJqsMMFs8kTth3GQF6fgKcarvZK5lVStc2QBnkgflVLWQHs25BIU9D7Vi3tjqBB8gSEn3P8AjUlhbXccdwtzC6jy/lZh1613Sp/uue5xU6zdVQ5S7crmNDjrTWsZ7iG3aBGPOCQQMc9TnqB6Vbdd8EWeeB1+lXrIAWKfUiuVWnozrqRsjIu9HuicxkkYPRv6UabZXdrckzRERlcbye+a6AAYzgmknH7gY4wwP612Ks1DkONUI8/MZ4Y5OWz+P/16RW/e/fGB71E0b+awWLI7c0qxkPzEB79ayubnI+JG3aivA+S9hYfi4H9a6yyLCzhYKPuAc1y3iBN+pToM5EkD/wDkRP8A69dRZDdaRKc/KWB/AkVhCpebRtOFoJliRiV+bavbrXPeKwv2ewkGTtm6jtXRTAbVG4g9eKw/E426Mj4J2TKcmiv/AA2Kj8aNiQcq2eopVI7kk/SmpJmGJgfvIDjGe1SLIxxwfyrRbEPcxfEVvm5UjoyDOa4fT7TzYTHlU2zSKT2HNemalCJhGxQ8DFchp+lFLy9gkV+J2ZSB2IBryK1XlqyR6FJXhEzDYvPayRlFZTGcq2AGGOhzXialkyTu2r91QM19HvpanIKuePSvn+/sHstau9PVHLRTvEF2kkndgYFdODrKbaMsVC1mc3fRM37w8MOeO1U4MbgMD612d94J8SPbGQ6Ff4C7uYGB/LGa4tUKuyvuVgcEEYIrtumcdmtzViICjjrzjOetSmQIgZjx0b+VVYpCyYB3Zxx1qwmGjOSffjH+f/11RIiQw3sqs6vIANvBxn/Oa1o9I07KsYsDrgntVGwRAeBjOTtGOn+Fbax+ahAYqccAev4Vm2aJGnJe3ap9ne7u3ixjy2uXaP8A75Jx+lUJnkcFFUM54OQKsmJXyShJPOC/4/4flVy2tVi/eMoU44xngCsktTS+hzeqad+7SFpGeWRtzEj7oHOP1Fc1qmnmzfI+6fbH+e9drd3f+kSCBPOuGGPZB/Ssa80S4O+WabLMMsoHArVMzaOi+DGsm11270l2/d3kW+MH/nonP6qW/IV7gr7Dj8a+W/C1/wD2P4w027LYWG6UOf8AZJw36E19QSq24bQTjrXJjY3ps6cG7VETGaqd3P8AKRSq2W2d/Qdap6iGjxkEZHcV4sY3Z7GiMC/X/SGI781nMy5KkjBFUfHUM9xp8Bt1kdhICQgJ4wa4M6ffsebS5JwRnyzXqUqacbtnBWqOMrJHTeKlE3hOykHPlTFePTn/AArzC4BDnPXJFekyQS/8K/nhkRkeCfO1gQeo/wAa85u1xK3Hf0rpw+l0cmI1aZXHWvfdAm87RNPk3ZLW8eT/AMBFeG2Ol3morMbSAy+Su+TBAwPXmvX/AAXciTwnp5JJKqVOe2GI/kKWKV4l4R2kzr4ZVBwW25HDY6Vm3Ol2cmU813UliVAAUbiCR09Bj6VIZTuI471C0hIxnivPselzCW9tHaFirs24BRuxwMk/zY1W+wQIylXkBVdo5HdVX09FFSlx0z+NQSXcEed8qfLjPPrVWZLkieMiCCOJMlUUKM+n4U1pCe1Zt5rMNsNoEkjq20qo74zVe11c3F4YGjKDJ2nOckdafKyXNbHmnimAQ6/fKOnnlvwb5v61j5/dY9D/AJ/lXVeO4PL1p5Mf62JHz9Mr/QVyi52sPxr1KbvBHkVVabPU/DVwZvDlk2ckJtP4Ej+leq/Di43W+oW5PKujj8QR/QV454Jk83w+Vz/q5mX+R/rXqPw9l8vW7iL/AJ6QE/kR/ia4pK1RndF3po9KNNNKfejtWpixtVLhcPn1q3UNwuUz6VcHqRJXRUz0o6OffBoobhh15FdCMGLmkIDKQeQRg0lKOapCPD9MgNjq2qRRIr3ttHILYMufnVwCQD32BiKs3KN4ns7jUvJjt7qOXG57g7JQcswHmMcFeuBxg9OlX9dlsdO8aXqXdsiLvEsc0asZNzYb+8B3PT0FRXfiHSdR2G8NxMUXCh4t2Dnnq3fjtXoJt2ZxtLY6H4XXO7Tb616+XKsgH+8Mf+y13wrzTwJqdu/iW4traJI7eWDKYjCsSMdcfU16UDXLW+M3p/COp9uSLsjsyfyP/wBeo6VDi4hbPGSD+NZGhpZ9jRS/j+lFMR8oR/8AITsAD1mBrr8HbxznvXN6NbpeeKNKt3+68/OD7Gu01LTpdNmCFhJG2SrHg1MJpPlHODa5jPwR2pQCeMUoyP8APWj5vb0rYxsOMbqVBHLDIwaPKcsF7n0pMt1zxSlm4OfzpABiCscnIHoKsxWKskkguIf3ah9jEhmz2HHUVX3NjGaXJxnNA0M2EnoRz6UqxMT909aXLevtUiFs9TSbGkaGiRTxanbyrEzAOA2O4PBr1xNIYxcr1HYV5n4fhZ7pS05TBBOO9e4R4MakelcFel7adrnZCpKlDTqfM3xGsWtPiYuFKgtbnPXPT/P4V7og+XFeUfGR0HxA08D73lQ5/wC+2r1hB8tEY8seXsDfNqxaa1P7UxutaIlkTCmGpGphqzJlnToy94nHTJreW2RTuA+btzWRpDYvMeqmt6lyJu7HzNEPk5Oc0eSKmoo9lEOZkJg5zmqN1GUmA9a1KzdRBaQKDjgciqhBRd0JybIgrev6024VTAQ7AZ4yacoHrn61T1dtunsx6BgePrTqaxZUPiRJJEscSoxHygZ/KnQXEKW4BnjUA92FZd7fgWPm+oXv7Vyv29pZSmCFOe9YUU73NKstLHo0VzBM+2OaN2HJCsDVW61nTY1eNr23EiMAymQAgg81wug62bfUrgtGoVUKls9eQa4LxJfCXX9RTBz9olU9OfmNW6rU3EhQ9257adc00Lu/tG0APTMgFQHxRog4Or2ZPtID/KvnKeLc/FWLItGwUKMN0z/Sh1mgUEepzanDqXja+8i5E1p9kV0K/d3Kyk/yrrLbU7aG4ltWnxILmVcFSerEjtivI/D0pg1K4xx/okw9c/KTXrWjS711CUHlrtm/NVrnpNurdG80vZ6hN4o0uN2BnnbA42Wzn+lYfivxHZz+H5IYor4yl0YFrZ0HUdSRiuvyM5yd31rL8WQibwjfEZLBQ2TzyGBrskm4tM54uzuiK01eVbKBZNI1MukSq37kAZA9zVuPVZnAK6Re4z3MYx/49V9SXjQ9ioPFO25XG3I9zSSfcG0aWmjzrISSwNGxJyjkEj8iRUdvDbtql38i7jtbp7YqNLryU2dOM49KyLPUDHr0xZuHiXj6E15uIrwVRRa23N6dOTi2jqTaw8/IK4eTwxp1r4/u9ZFupuJUj2E9EO0hmA9TgD/9ZrqDqgrG1vUBEEu0GJANm7IwO4z+v501iaLklBWKp0pp+8RahdSXen3a2Pz3Ue5VVSAQ3YfNgentXB+NPhfPr3huTVI9HEXiGOIO5jkUeeR94MASCcZweucc13ENyt3Ai27yGYneuGGHU9VBPQ55FdDBeSG0jEsZ8zaMhmHWuuLitQqp7HxQhKyYYYIOCGHI/CrUchbEYHJPQDFXvG1ylx461uZNgV72U4QEDO7375z+OazrJTLKAoyAM/5/Kui+lzktrY27OJVUb2G30B79/wDPtWqjxpGXOAuAuP71ZtuhOOTjOGBxgCphKZr0QqCUjG4j3qCzUj3AKw4KhRjpipLuYCE+bN5UX/LWXPOB2H41RUzKp8q3eUkkKqr3579v881mzb7m82XOLy4BylnCSUT3cj+QpJDbLkU0t2hOnILKzBw11KMu59v8feq+o/aLexnjsoQcg7ppGDs477fyrREGpRhVmluTgYENrCoVfbLYFPZCQ0k1skUJxuCsZD+IUECh6AtTzXcd+7PzZzmvsHwbexX1hZ3U6qwuLOOXkZ+YqpP8zXyRqcKQ6jMsUbRxbsorelfSHga6ZfAuizAn5bVVz9Mj+lRX/htoKSvLlPSo7PSoblrhYUEh9TkD6DoKwPG0kBsrYxKgPmEEgdsVTOqN/erJ1q8NxaqCfuvmvHjVlL3bI9OOH5XzNmLK25AT6VUYYPIzU27dHjnhjUTDJBPT2roiVIx9Sj36VrUOP4BJ+mf6V5HeAhj15ANevX9zb20t3FcybPtNoyqME7iMjHH1ryK6GfriuvDnFiehu+A4pbnW5LSKQRtNFjJ6HkcGuu8Auf7Entz1gunT+RrzTS786be+fs3jaVK5x196734dXfnnVQAVBlWULnON2f8AAVpXj7rZGHl76R3M8vlQvLjO0E4H0zXPHWZ2ViYhGwGf1H9DXQTpvtmA/iUis3TbWxuYwJwNzQPyGx8wXj+XSvPukrs661TkVznpJLu5gaSa4k2KQ2BxwWwR+FQ3KGN7gBiW3ujEnqAAwrV1FYYmuoLZt0DoWUZ6EDNZtwd8xP8Az0aNh77lwa1jqjNSurkQvN0xuW5w8bn8sGrtvIp1hMAgmQuM8cMo4qiYBDbRzKSfMj349CCv/wBerjS/aZ2uYFkwAjD5epBwabKRk/EGPM9pIBjdG6k/Qg/1NcIvU/SvRvHaLJYWkmcsspXrzhlP+ArzlMCQZ6ZrsoP3EceIXvnb+AJswX8OejIwH5g/yFeoeDZ/J8U2mTgOGQ/ipx/IV4/4GnMWrTxdpIT+YI/+vXo+iXxj1SwugrIFnRirDBA3DINc1dWqXOmi707HuNJmlIptWiGHemONykU6g1SJZnkYNMY8r9ammXDn3qCXhM+nNdCOeQ7NGeabmlzVEmdf+HtJ1O5+0XtjFNMAF3tnOKgHhPw+p/5BNr+KZrYpc1XM+4rIoWmh6VYTLPaafbwyqCA6IAQO9aOaaKM0O7Cw/PNDHam7+6Q35GkFOVd6smeoIoA1Ovaiq0VynlJkjO0ZooEfN3hNPM8caSOcb5CPwRq7/wAVjBg6jJPSuA8I3ENr4306adgqDzcn/gBFd/4tnSSS1EZ3oys3mD7tcyT9qjpuvZM5oZ6ClxzzzSd/agkggAcdzmu25xsUc+uK0NL0XUNXkZLG3MrJywBAx+ZrPXOR9K6nwd4gg0O9ka5jdo5EILKRkHtwetZ1JNK6IldbFO+8Jaxptobm6tDHCv3mLL/LNYpUivQPFXjSx1fS2tbaOUMSDliMVwDMcDGCM81MJt7ji27iDPtUkfGKh6il3FZkXI2kHPrVs0RtWlyYFyDyRXtlheq9hAxPWNT+lfPa3iB8Z4xnOa9csbmcWds4lXyRbgGPbyWwMHOfrxXn4qThaUTsow9orHlHxcufM+KEAB4EduP1Jr2tPuivnz4lTGX4kI3B5gGSfpX0HH90VUHzQTZMlyyaHUw0+mmtESyJqjNSt9aiNaIzZa05tt9F9cfpXR765e0bF3F/vCug3D1rOc+VjSuT76N49a5rWo9cluU/s24ijix827r/ACrNWw8Vsfn1WEewH/1qweId9ivZnb7/AHqKVlyCQCScZxXJxaV4i8wtJrClP7oz/hUMui60u1pNcdhnGBnqal4iVthqB2JZMfdH5Vx2pXDPPcRs7bQ5wueOtS2Wk6hb3iTzatJKi9UwcH8zWVqJd7q68uNydx7GnTquV7lKFitqOqQw2CxS5XCrk9hWBLP5beYozjnPrVnVLA3tqiNA0iuoRwQee1ZN1p99Gfs0PlKgACZfaqADGPXrWlOsti5UebUrLq6y3lykJHyxt5nQ8gZ59K57xBLGPE18nmJ5jTyMFzzjcc1fOmXa/aJI7ZVfaTK6TMMnHPb09aZqmhX73CubIPKHDs6yuxcdxkr34/Koc05XNFStGxgz3CQxhmcKMgfUmrdtZTsrT5IjHPJz09MVoRaDqFwwii0RncHcW3noPqOK3bHw1c2EUN9L+7aMEtBuPTIHzevXNKUr6LchwSV2ylo9pIWS6XmGWB9pIw2GQ9q9Q8LljpkzkZ3uj8e6LXBW7zS3KkkbCTnj2ruvBj/8So+nlwkZ/wBwf4VWHvzakT+E3uABkfrVLxA6jwrqe0jCwMx3elXow0jAbWqPU7b7ToN/E6kB4XGR24rvexzLcjs2M2n2kiZIeBGyD7VN5Mh9aqaIyDRrLMoJWBQcnrgYrSE0X/PRP++hUKS7lNMztXnnt4ENrCs1xsGImfbkZAJz9DmuflvAmpqe4XBNdBqnkSiKdHSR49yBlIOM4z/IVyOonFwJB1weSe/FeHiknXZ6VH+EjRuda+z2zTFXbaeQnJrMm8RSTxvGtleHj72zj86qPO3UNgAjgDiopZ2KDkAE9BWMaLTujSUn0NfQxp6awrTXRtnc7sN8i5HPVux9sV3iXEFwqPDPHIhyN0Z3D8xxXkgOXUtxzzn/ADzXdeDsNoSd9kjqR9cH+tenSldWZzVVfU+V/EL+d4m1SVQ2HvJmAI5++etSaUo2tJjvgU7XrKYeJdTjCMQt5MvT/bNWtNj8iIBsLg56dv15ruvocXUvGQW8JckZ6jPpUFrL5VuZHUbnJJ4xiqt1M0zqinjg0XbrHZZ5GFxwvH5j/PNSUPtYLrUIjMZHSI/LtiBy4ye/T2rWs7LUEBgts2UTHnbyx7dBwPxJPqawNAhe4IHnTqozxFIQf0/z1rqINOt3O1p33L1LXJzn8/6VdiTQttIgQK9w0kzDndK5YD8M4FSyfZkct5CvIeV2A4HH+fypsOlhRmO9ugCD92Yn/ParDWskShUuZCxb7zAEgc8cChrQcXqcN4xs7tJLe6m5jYFB7Hr/AJ+le0+C0MXw60odP9F3fqTWP4bsLG+mvdMvoBdw3MQLCbn7p7HqOp5HpXbraQ2mmC0toxFBFF5caD+EAcVlLWNi46TTMSX97JFIZHBjYkBWIDcEYI7j2qO4cshHam7ifb61HIQUPPbNeWlqeu3oUrWFYVlRNx3SFzlieT169PpRPDHMFEqghWDjJ6EdDT1PzNjuOlMkOQMY/GtUZs5bxijfaLB1BJO9eBmvO72Mq75BDbiCDXsUmV1CwfOFEhU8+orzLxZCYNe1CMdBOW4755/rXRRlrY5K8dLnMV2/w1l26tdxf3oN2Pow/wAa4k9TXT+AZvK8UxJ/z1jdOntn+ldNVXgzmpO00eu/wDnINchNbzJNKhdEKcRlmHzZJP8AWuvCMVwAfwFcze6c5nvnSKQzKQ6YB56Hj9a86J6VRXRnrLbx3OWuS4ERJVR1O3moRc2rS2ReGUlFADsQOM4BIpW0+4Mcf+iy5aNDnyz6EEdPpSXEF1NbwH7JMriPym3IRggg5/nWisYWHx3NxMNsMcaokLNjGSOo4/GopbiW5dY1kZUcoRg9Mg/1FTwWtzBegAIqMzoxdgPlLZz196pw2s8TDcYuCpAMq9j9fc0aD1KerQmXTbqaUMJohHIN31wa4uQbZmA7McV6Hra29wtzJJKiL9nYD94vLDOO9eez/wCtJ9cGuqg9DlrrU1/DEnleJIOuGLL+YNekBiOc4I/nXlOny+Rq1pLnhZEJ/MV6rzn+R9azxC1TNcM/daPeLWcXNnBOOksauPxGalrI8Lz/AGjwvpz56QhP++fl/pWtmkthvcTNHFFGapEle4XgEVVcblI9a0JVBjNUDxxW0HoYzRGp3Ip9RTs1GmQuPQkfrTs1ojMd2zSk03PFHSmAtKDTc0vamgH5qSI81BmpYvvUxEUkEnmNjGMnFFaHHvRSA+XtIJbxDbZzwsnf2rq5nJwCenTNcjo2f7egYcDZJ/KupkJJ570QFJk1uQX2sO1WwkPcE1Rt8mTAJJ9quBHx0PtRJ6jhsPCw9AozS5h6hB1qPyz/AJFAiYk5FTcqw8NGOdvApPOQqcKB7Yq/pL6fb3EjanZPdQlCFVJChDdjxWUkIZnaNZAC5IDNkBew6fWpUtbA0K84UE4A/SsLXLsp5eJCgb5gytySO1bFxFiFtwBGOh5rnNSQSW5juCf3THYe4GOKu4rFVNQkmVR85UYJYddte56fdbtIg6j9wn4cCvnfTpHN9Cm5VVR3zhscivdbCR20yA7sZiXcMewrhxmsUduDWrPL/Hkm/wAeqc8F4f5Cvo+M/KPpXzZ43I/4TUHIPzw/ToK+kVOFGPSqhpBGc/jZKTxTCc0jOcVGWIHBq0zNoc1RHNK0hA5pmc1ojNkkBxcRn/aFbO+sOM4lT/eFT3t7fQzhLaxMykZ37gBmsKybaNqVrGk780itz1rBm1DW8/JpI6d5B/jTYr/XmkUNpiovck4x+tc/s5XNeZWOiLECq8wd4twBIVhnH1qtbz37fLcWsaqf4w/T8O9LcsfJwD/Evf3FRLR2Lir7EobIqW2iDAZxzWXd3E9rbtNFC1yxIAiXjb6n1rGj8ReIYZZPJ8PSzJn5MyhQPzrKona5UtrI6aOyjaJxgcSt/wChGvOkxqYuZbjaGErRELwCAQQe9bcuveJ0idrXRUZnYl1kmH7s9x1Gaz/Dem38cFy+rWiwzNcF0RHBBBAz0J71WFhFuS6vY5q7qRSa2KNxKLK3kjW1JikGHLEnHGOwqzYaeuv38Mt7Dsh27RsZgSoB5611KWVtdSRxTwqUJx0PH6Vu22mWkEjPFAqsqhSOe3P9a9mMacIaL3u55zdWctZaGLpmixQRyrYQrEJIgAz5PfJyc89Kh12GIz2iPbK+9zncOuQtdXEVVZAMALKRgDtxVWcLusXwMk9duT2rOjFQfM9WXNNqyZ5VKiQaj5UaqqbgdoHY1teF1jezj8yZ1WO0jOEkK5wSvOPpWD4ngnTxhIYrgRqsi/LtHAIGQDW94SUfZZCSM/Z2XOPSVhWcl7+p1Q+DQ6OIWRbmWTHf96/+NI9hYXELxRtKQQ2FMr5PH1qsVbaHM5wf7oq7YbGOPPYnkFSOvFWmu6ItPsZ3h+ytn0qGUwwmRlw5ZsE4JrWNvFHHkRW7YHTIz/Ksrwp9m0zw84kfyoYZJCzSHp82ev41P4n1cWujpHZOHur8iK32HOd3Vh+f6ily01G7RtFTnKyJ5reIWu5IUjIcfdGOOfYVyWsrtZDxjBGPzrqRYvpXhqC1kmaWSKNd8jtnLZ5x7c8e1cnrLZiDbs/NwMcf55rzMXG1ZNKx2UX+7avcz3xGjHcPXr0FZr6nagcSk8+h5q/1V8gY44FU2hiXpFH+AFOHmOV+hs+DNR0+bXza3XkvFcRMAsyjaWGCOv416JcS6dpFoxEltZ24OTgrGv58V4xehPscu1QAFyOPeuRu1HnR4QdPau2lZxsclS/Nch1ho7jW9QnRt4e5kcNnO4Fyc5/Gse7zHJs9R398/wCFa7x4bcBzWRfnF+cHJ2j+X+fyrdIxZVbmUY7dhxW54bsodV8VadZzQeZGXLyRuMqyqpYg/UisRid3XI6Dmuz+F1qZ/Ed3eEfJb220cdGduP0VqBGHq3h8eGPEktmylrST95A7jIdCenuR0P0z0NbtpBayEDZCW4G3A4/D/Pau68XaAdf0UxQrm9gPmW3Yk91z/tDj0yB6V55pd3A8EYucI4wodhjPpnjr/hVRYjbitbdCNkC5UdxVpwQuFPGPun17fypkYIG4Nkeo57VI77VBZhj371Yh+kXX2TWraQc5cIR7Nx/n6V6ORxjH1ry+HdHcQvgrtkBLHgDB/wD1V6dwcHPWsTU5VvkLDPTIpjHIJ/WprkYupgOPnOPzqvn8u1ec1Znqp3VyDOG7kY6Uw4Kj06Ypx+9jA79aac98VSIZR1CMG1UnosinNee+MoTHrVwM5yiEZ+g5/Q16RfDNjLz0GcVw3juL/T4pNv8ArLcD8QTWlJ++YV1eJwjAhj2q/obSLrdn5T7HaVVDZxjPH9aouMNzUtlMbe+t5x/yzkVh+BzXc9UcMdGet/2Xqcn3rtMf75rM1C0ubd2jluwCE8wcn5vYe/FdmjZBGF4HBHf61z/iFUSSBzAkh5zk9FHX+deatz1JRVrnPW9rLc3KwNOylgffoAf1zUKW7NOYt0pdX2Ngcd+c/hWmk9woWciJFdmjVlHIwCB/Kq0ouXWSVp25hEoUcd60RjYrLY3ckXyRybjjhiBxjn9aupoErAFmK8sDyDx27fnTZI/st/Gwkc4EbAk9QTg/0rqMAr+FTJ2KjFM5dvC6zwhJpSCSchf0rzy5Qo4DdR8v5cV7WQD0P6V5H4ggFvq93H6TufwPI/nW+Hk3dGGJgkk0ZucbDnp/jXrNtKJ7WGbtJGrfmAa8lz+7Hsa9O8PyGbQbNjg/u9v5Ej+lXiFomRhnq0ey/D+fzfDXl8fuZ3T88N/U11B5rhPhtPmHUIM9GR/zBB/kK7qso7Gs9wpM0UGrIAnIxVKQbWIq7Va4GDmtIMiZSHEjj3zTqRuJvqv8v/106tbmNgo70UUxBS96SjvTAd2qWH71Q1LCfmp3At596KX8KKBHy3oxB1yHHZG/lXTSHP51yNnMYtSRxuzg9K6X7SWZMkYIwQfWlBimi9ZuqSFieAOlXvtkQyA/6VlK+QcHjHWoBqFo9z9nFxGZQcbc9/SrcUyYto2/tcfZj9MUfbIgOC35Vmk85zUbQIQx+bPsxqeVD52WP7Zi3MAHOCSccEU+DUIVUliRu5Az+Fc7IjQzMfmR94PPIAz/APXqKR9sxLMQAhO0fmf8+9TyovmZt3Os20jiBopMyDAbI6/5FcvfXMzXzxKuwhSpz39KjknDXolJHljJxu5H+FTaisN1h4ZB5oXL5Jzj2qbDuZ6K0UiSxjcEXOQO/SvcdNkQ6TbDncYE/PArx3wo7L4ktI2LeW7lGB6EEEYx6V63p67LC3UdFjUD6Yrhxctkehg47s818bHHjINycSRY/If419JR/wCrX6V82+NS3/CXsAMgGI/oK+kYj+7X6CtI/AjCfxseaYaeelMNUiGRtSAjFK1Qk81rEyY5zhc9xWwkqoiMxyCPz4rCnY+UwGc4OMfSp4JUaKKK4kaOROgJxuz/APrqKtjWkakl8AVCKRkgYxjg1G18dpKHnqM9+ag+zQMQfPfIAAw3pSEwQZhRZZS38I5xWVos01RZMuZ3UAY2gn61Bcf6g59Qf1qyqIgOBjPJ+tQXTILaT5h09a5Kj5nobw0GKTsOfWiIlgQPU1xfi7U7+wvmjtruSNCgO1TjFYnh/XNRfWFgmv53UxMSCxxkDOaUsLKpFO4SqqOh3TSkPdDPSU/yFTbsq1cJpF7dT3Fs0lwzbpHDmSQjP+NdRqOqrpQj3QSXHnHaBDyQf/r1FCi6WJimxVpqdB2Rv2NqbiRAwwCSckenpW8ibTKPcfyFcroOvtcXsdnLYT2zqrOolPJGK3Li/kiuEaOAsjL8xzwDkY/r+Veu6sW7XPM5Glexbjb95JH/ABF93A+lVmyxtlBIO1sZHQ5rOk1aQXM0U6xRXAn8hGLkqSybk6D6Cq97qt9FounSJbxee04R2GSq8kfXqKFUi3a4crtc5HxTAw8RSNzuwpPFW/Bzf61CucLOOR6S5/rVfX55J9SnMxXzVYr8oxnBqrourHSRLMIRJ+/mQqWx12msZT5pHTCPLE7uWNSi7Ejx6hB+VS2SqLhTtUH12iuVPjzYpxpqfKe7+n4VGvxEk3kjTYRk8ZkOaFBXvcpzdrWNvRooLnTru2uYVmha4lRkIyCMiq+leFYdN1oXizyzW8SFbW3l58knrg/y+tYth4gmtYg0cMWLm7YkNn5cjPFX5fFN5Ax8uKH05BP9aj61RWkt0WoVUnyvRnTarJ5mmTgBThcnDAnqK4LUlXyvbcMZHHpWoPEd3dQPE8cAV1KttXBGaxtVciEtzu4xjtzXPiKsas04mtGEoQaZQBHPuoqNxxSxlQ7dPmXI5pG/THemkVcq3KlrWUf7J7VyFwv72MkdF7c812cgyjLnt61x1yCJVByODXTROasRlMjAGc9KwdTJGpMowSVXNdHjMec9PQVzmoA/2pIeeAPpwK6DAozkBeu3ivUPhNb7dG1G4K4aW4VRn0VTj9WNeZyDBGPlB7Y/wr174fR+T4bhU8F1LE+p3n+hFK4WOwU4Ze3Irz/xpoy6R4i+1worW2pZfYeiyj7w/HO7p3b2rvS27JrO8f2JvfCrTqMvZyLOAR/D0b9Dn8KqJLOBtdob/Rp3t3zyjjj8un5Voh5UUtIoJxy8WNw/A/jWbARPEu8AEjOcZBNWUadXCBEIU45zjFaCGu0csm9DK+3ndnOPqO31r1Czuo7yziniYlGX8QfQ1535yqPljTzEGfp+NdP4RufPtrsHCssgymMYyOv4/wBKzasaLVBqC7dQmGepz+lU846Gr+qjF+Tz8yg8VQPB/wAK86ekmepTd4Ihbqc9M9AadLNJcSF5W3PjBIUD27U0kE59s0mDk5I4pIGRTjdbyr1JQ/yrjfG0YksdLnxkFGXn6D/Cu2IDKR68VyPihN/ha1k7pNtz3/iFXDSSMqq91nmTj5qb6VJONsjfU1GK9E83qe+aZILjTbSfAzLCr/moqjrMStLbFxld5U56cg/4UeEZ/P8AC2nOeSIgn/fJI/pVrV7aSezbyRulQhlB7kV5j0kestYI5OeORobhUK+QlwDHjrz/APrFEC+Zb2/+3BJEefTmrsel36RmIGPY4VmyeQwxVtNKZWTBXakrOP8AdYdKrmRnys56aFlgs5A7N+7LHJ64IOK61CGRWH3SAazYtBMfEk7MFyACOACCCP1rUhiMFukeS2xcAt7Um7lRi0N/HPNeZ+M4PL1uYjo6o/6bf6V6djJ4/WuB8ewlb63lxxJCy/8AfJz/AFrSg7TMsQvcOKH3SK9F8Hyb/DyD/nnIy/1/rXnQ7j2rufAsu6wu4v7kgbH1H/1q6a69w5cO/fPWPh3L5et3ER6SQHH1DD/E16VXkvg6byPFVpzgPuQ8+qn+uK9ZNc1N6HTUWolFFJmtTIXNRTDKZqSmkZFUtyWZs3EkTe5H6UpNF38secfdYH9aTPrWyMWOpM0maAaoQufelzTM+9ANMRIDUsP36gBqWE/NxQBc59DRSjOKKAPlCEk3ilTzj6962zMqqBn5u5rDjdllDBhyuP1NTo2+Ub3woP3R39qzi7MqS0N2OWMrJI8gjVV27ieOlcjaRGTVYoUKn94BuHcDv/WtE3ct5qZhtlDokeCjD5ffiqd+kkM5uxiOXP3QMbTjtVOd3YlRsrnc9OTVeWSTLRxld2MjPpXP6b4hkuXEV0QGH8QH3h9PWrtxeRxzLJCS5Xrx1FacyM+Uh1LVSJPJIG4DnH0qrb3sZgTf1+ZNx9D0rOu3M91JNjcW/vdQKhy23OM46E9azbNEi3bhXlG47juGPerN8baKRVLBHAwCB+uapbkS2WRcb84PPSqk7F5PmZWPQEGlcLGr4alC+J7ADnMoH617JaDZBGvouOK8U0JzF4gsM5wZ0yM9sivbYh0HvXn4vdHp4L4WeYeNyx8VyYUkAxZ9uBX0fF/ql+lfOvjPJ8T3Z7KIv6f1r6KgP7lP90fyraPwI5p/Gx5php5pp4qkQyJuhqFutTNULcGtUZMaxwua0lWObAkRWHYEZrInOIWI7CqcOuzRKqlUYgD5j3rnrxk2nE6aDVnc6R7eGIgxxqpPXFT2zqyEA8g4Nc1Jr0kv8KD3po124T5kCDjHSolGTRrpY6ieLzLeRM43KR+lcD/Z4iYSGVztIY+mQa1/7fvmBBdOfRRWZeys9vLyRlDkfhVUU4JpkSV9RfEOh3OuagLmxkjkt/L2MQQfmBPv71haf4VvV1iQI37xY3Q5XA5GOD0PWt+0u5YITFE5SPP3V6VNBI7XCje2Seuahc8VZFuMZas5yPw/dWlzGjSrutptz+4ODxW3qrEw2p9J4z+tVDNNc3U67W3ggHGcnHFWZ47ia1hiW2mLh1Jbbxxg1yYlSbUmb0uVJpGwhVPGlqqdGgYfoK3twkSURrjdt4H1I/pWGbS4PinT7pUzEiFWOQMHbWraTlCgIyHLDORxhmP9aVKVomFVXZg6pbXZufMjhdyL20k7dAuCfwrXu4JZNMii8sh0uVfGe29jTdTuZVEMkUSsjMu8lsYx0qV9ThYyh2AyABjnuf8AGk69mEaLdmjivE1zHaarck8h5WIK/WudkvDBC4EZcSXR74xmNTWl4pDzXZFuSVDZ5rGuI2e227csJ4yfbMeP6V1UZOUbkTjyuxSvNQkRH8tQpxkZ57U7w9dvqTzJMQGigMisBjLAj9KbcWTyk4jfIGOG4/lTfCVjOL4EnYkkTLnqcfj9K2qXVNtGcPjSZ11zCkOlac6Z+aZWY57mi84JOaluYhJoEK5YeU3BHqDVK7hckk3Ep/EV40HzPU9FqyJrWQluPSm6lMxhYHkY7Doaq2kJ3jLy4HT5jUuoPmIZzwD07/jXXTiYyehkrcObjb5jYGcc+nantcO2RnBqE/8AH1yRjHFIeOPT1ruUUc3MxxmbnLY+hrn9RTZKrKOqdPetzv1+lY+rA4i+hHHTrWsFZkTd0VkwI1AJrn9QAOpS9cEDkHFdGuVVc9COK57UcDUH6Dgc1qzJFRlO77vP/wBb6f5969e8EKV0WFSMYjI/8dQ/zzXkqwSTTrFGMyO21QO7HgfzFe3aPYCwlltQMiKd4/TIww/9lFZOaUkjRRumzUGMdjxWmI0urBoZFykibW/EVmgegHpWnbHNun0xW0TFnjkUEmnX13YuDugk24buvY/lVk3sKriQEDPX0rd8caZFFq1vfMuI7keVIR2Ycg1zDweUQlwSR0Vug/GrEaELxyIfLIOR1781b8I3Lwa/5DNkSq0TfUfMD+hrn/Ke0bzYmDRnrjOOv/6609HnWHxJZzjhJHXOPfiomXE7PWhtuImPGUx+R/8Ar1mN1z3rZ1tcrA3oSP5Vj444Jrzq2k2enQ1pojb6c5puPXNPYcdeaMY9R9alM0GAdPyzXNa9GD4Tvkxnyp8gf8CH+NdPjkcCsTUoPM0nW4epA38/TP8ASqi9SJrQ8guxiVuvPPNQVavF+YHjpVXpXpR2PKe5678PJ/N8KonXypnX+Tf1rrZBliMcGuD+Fsu7T9Qg/uSo/wCYI/8AZa79lOR34rzq2k2epQ1popFAM8cVG3r+dWWXBxzTCpA5rO5rYgK4/wD1U1hwP1FSle3p2pGGQeMfhRcViDZ83Pr2rkPHltmxtZeTslKZA/vD/wCxrs8DOawfGlv53h6d/wCKJ0cfnj+RNaUpWmjKrG8GeRrw314rrPAcmL67iz96MNj1wf8A69co/wAszY7NXQeC5PL8SIn/AD0Rl/TP9K76usGedSdpo9P0qYW2sWU2fuTKT9MivaTXh20jp17V7Xbyie0hlHSRFb8xmuKkztqolpO9FBrcwEooNJTQineJmNx6g1XDblBx15q7OMrms6L/AFSjP3ePyraOxjJakgNLnFNzSE1ZI7NID6U3PrSg0xD81NCfmFV81LCfmoAvZopm73NFAHymg+bI64/TmoJpCkx2k89KlOFfsTgd/rVWfIk4HXtWC3Lexd8KkHUpz1zH6Z71d8VWyRQQygtvkc5BOR0qPwjGReXR/h2AZ/Gp/GT/ALqzHT5mJ/Ssm37axqkvZXOWiJWRSM9RmtQzbSAcgdR2zWSjFjtByOlWQHfGDgV0GBNI247u57jioz7cjtSgosZBY7gOCRURJBJPpikOw/kQNgnBbpTAqhflfJGO1Sq4FnMD13LgCr/huygv72VLhCyCPcACRzkUN2QJXZHpsZ/tyxxg4nQ5B9xXuKD58e9efwaRZW0gkgg2spDA7iefzrbTU7wv/rjyeuBXHXg6jVjvw81TTTOV8cHb4muRngrFx69K+i7c/uI/90V8yeLJpJdeDyPvJKA+/FfTNsc28R/2B/KtUrRSOeTvNslNNPAp1MNNEsjc1A5qZ+lV3PNaozZDOf3Tj2rmw3AFb95KI4ScZ5Arne/FRM1pEpJHNOViV601VJAPY1ba1EdrHP5sTbzjYrZdfqKzubogBwOufxpJ8tBL/un+VS7Bk0kgBhkA4+U0rjGWx6+nBq7bj/SUI4wc1RtSNvXPAqyrFcMDyO9S2NEisItWum6DCn9TWg92iQrIrAqE5A+lYU5Juix5JQZNTA/8S1vXYa4sV8NzWlHU3JdUijMAdGZskcHHVSKxNJ1O5a1njMhwk7hfXFOuzn7MR/fXP61Q0k4+2qc8XDf0rkUnyM0cUmazSs/32JHuaVG+RuO4qBTkHGKkQ/u3/D+dYSNYmFq5H2pifUY/KsubcDLuGDvhY/8Ajw4q/rWftXHTaKyrn/jzkbrxEcg/7Z/xr2cP/CR59Ze+yd1uDO4t3VScZyR71X8PF47+FHPKl1/DJo+cSk7SRtGeKj0STOqKQMfvWH41tNp02jKMXzpnVvzo8w7B3/rVK6/1YPHTrVsHdpt2uekh4/Cs+d8wIeDlRXjwWp6MtiKBiHXmnX/NvuOPeoIz+945qbUWxZdMkeld1JanNLYyT/x9oAD0A/SkJ5HehSPtsBz95e/4igjBrtRzjcfmaztUXdbR45PmY/nWiR0/wqreRh4hwMCTkfUf/qq47kszZWCTRqD0HSub1TIvmHsDn26Vt3Tk6iRwQvFY+sJ/pnvtBwM55z3rV7GaNv4fQ2k3jbTTdlRDHvl+cgAMillz+KivX4VX+0bpgysv2sEEHj5sf/F14p4UTzdSibo0cbHB78Y/9mr1LTL7cZozgFXjcYHT7o/pXHUpvnU7m9OStym5Km3BA5zzk1atGzAOeQTmmXSHySRgd6bZN8jDuGNdVJ3MaqsU/Flh/aHh+4VVzJCvmp+HP8q8+hkM1ur5BbGCG716wSCOcYPY15be2J0nVbm06CNt0bY6oeR9eP1FbGJUM0aEq8RXnk4ziqzxSpLG9vzCDvVgfun/AArQ3JLkZXzOw61nTWZjYuuUz129KUlcpOx6PdTC80i2uBj58N19RWbtIGMHrS6HcC58MLF1eB9rc+9SMoz7eleZidJnq4XWBCVyMCk24qcx9aAnbpmsUzdoh2nPSs2WHzLrUICOJbf+hFa+36ZqjIuzWovSSArz7H/69VclxPDr1du3qSMjkVRHWtjW4TDeXEfHyTMuPoSKx69WDvE8eatI9B+FcuNS1CD+/Er/AJHH/s1epFcqK+fNK1WfSZnmt5ZopGXbuifacenQ+grTfxlqb9bu9I/6+mH8gK5q2HlOd0ddDExhDlZ7U6HOcc+9Rsu3k4H6V4e/ia/c5aWZj/tXEh/9mqtJrE8v30Rv94s38zULCy7lvGR7HuLz26AFp4k/3nAqvJqemxjD39qvbmZR/WvF5NSiNoiJbKtwD80hCkH8Nv8AWoP7RuezIPpGo/pVLCeZDxnkezNr2jrjOp2ue/70H+VZOu63pF1o15bR38bySQsEVQTlscdq8u/tG7xgTuPocU0310etxL/32auOGSd7kSxbatYjnGJTwfXmtLw7KIvEli/Yyhfz4/rWUzM5yxJJ7k1PazG3u7ecHlHVvyNdEleNjli7STPamTINep+HJxceHrJs5xHs/wC+eP6V5p5YYZzkY6j0rvvBshbQzGf+WcrDH15/qa82k/esepWj7tzoPeg0ZpK6jkCk70tN6c00IRxlayR8ssq9MNn861/0rJuBsvWH95QfyzWsCJoTNGf1pmefejPPStTEfmjNMz1ozxSAkDc1LCRnNV881JEfmpgXePQflRUe73ooA+WgAWHTgD86o3HD8kdT2rUtWHmHdAH+UYzzjis242tdHGAc9+KwW5b2I4rmSAN5bkZAzz6VHc3k90R50rPt6AsTip/s+yBpGYNx25GKhtI1lmIcHAHQVem5OuwyHAPI57YqyDn+IDANQRhVudsn3RnI/Oo2Y9+tAEolzwDwOKf5UzgsqEj1xUOB5JO3r0q1p7ssgXdwynjr0pDFjhf7JcF0KMMYJrX8Fhj4ltoMblnPluvPTPaqkizSxNHwCAAD0xitzwDA9t4y05nK7RKucfWs67tTbNKPxo9Rm8MRQvteKWI4+63HFVf+EdWXULK1gPltcPs3NkgHFehaqY9QvvNiyV2gDIrndZD6ZNp96gy0U4YL68HivnqWJn7fkUro9uXJKj8KUjxv4jaPNofi+SxldZHTy2yo4IKg19EWh/0WL/cX+VfPvxT1Y614ylvDEIhtjXZ944Ar3+zP+hw9fuL/ACr3/so8WzTdywxpp/WkJppNNAxHPFVnPNTsarv3rRGTKl2m+L8QeawVAPX+Vbd7JsjHAOWArGQFTuxms6mhtR2JU6BR69alVG3hOMkjHzVEuSc9KkjO0/erNs3SZPdW01nOYpdpYAHKnI5GagcM6sADnBqTzGKlVU59hTvImZSVR847KajmRVmVrSJgigjHyitC2tI5nZZLhYQFyGYZz7VWtbO4aNCUl+6Odhp7GKJiGljUjghpAD+tK6Y0nYqzkCdef4Tz+NPX5rBwf7pqtdXdvHdKxuINqoQxEqnHT0P6Uz+2NIWxlLapbLJtYiMtkk9q58RFuOhpTkk9WXLps29s3PVP51S03JuL5f8Aptn6cCr91g2Fo8DCRHRGDYxwazY5pobq68rYSXBOfpXLCneLRrJ6mmAw7HFSxxM8cpzjC5x+Irn9TvdTsGAdlG4bgFCmqdt4muLZpXumMkZQrtUqDnj3qvqkmroz9vFOzL+qxE3SE9wKzL79xbz4xxGpz9HH+NQXPimGacM9tJtxjG8A5/Kla7TULN5EjaNWhlXG4NyCprupU3GCTOec4yk7Bfaqba8EfJ81Qn51X0f5bxmIw32ngDsOM/zq6BCCgxnK55I9vrVexZI72ckBT5oIJcDsPatuVcrM+Z3OjRwLe9UkD5wRnjqKoMDJbIFIJ2jjI61ae509EmRprcKzZG+Udxj2qsNY02C3jQ3VqnGARJz/ADrzVSlfRHa5xtuVo4pBNyMepq5eQGS0KkjkVTfXdPY5+3Iy9cgkinz+ILZ4lgjuCc4AG0jNdMIST2MZSjbcz2Tbb9iwGAQOnJpGR24COfoDUdxevDASsjqGz0J/z3qnLqLhN375xjgDJJ+gzXUrnPdGh5EpH+ok4/2DUbwyCNyyHC4P0qKez12GJJW0K7KSPsX5o/vYJ5+bjgdTjt6jMcdrre9mfSBFCv8ArHe8hyo9dobJ69BVqMiXKJzkjZv3GR16fjVHUxuuwScfu1z+tWUf/Sbhs8Zxj2qtfn/SwQONijp04rSWxmty54U41gR7gnyPkt/X8q7qzlWO+lUSrlohjGcHG4/0/SuC0V9msIytjKtj8q6eymI1mAE58xGXn1IP+NRa6K5rM9HuNRyvkhBjAGe/QUtg4JcjqCKyd4YRt6opzn2FXtOkGZBjHH4VrGKWxnKbluawbvXHeN7YJLZ3wwMgwucfiv8ANvzrrN/XB71i+KoftGgz4HMZEgI9qsg88nhWQjaSj+o71D9qvrJf36i4g7kffA/rVyIK+AR83UU6aJjHt8tmB5GBmgZr+GLyJ5Joo2zHPHuXHYjFdH5JIz+PNeZafLNp+oiVCVMb5ZOcEGugufipplpI8TafdtIv3gSijP5mvOxVOc5JxR6WEqwhFqTOu8ntis3VLtrKRAqZ3DOTXKyfGKJc+VpH/fdxj+Smsu/+Kct26n+zLT5QQNzs3/sorCOGrdjoliqPc6k63NuwqLTUv5rjULZljTeAwG5wg59SeK4iX4j3Lgf8S6y46ZQnH61CfiLfZBWwsBz/AM8T/jWn1aq+hk8VS7md4mO/V744AJmYkA5wSa588Ve1PVZNUvZbqaNEaTGVjG1RgY4H4VRY5NehTi4xSZ5tSScm0NopaUKT0BrQgbRVhLG7lx5dtM+f7qE1aTw9rMozHpV6w9RA3+FLmQ+V9jNoraXwlr7f8wq4X/fXb/OrCeBvED9bSJP9+5iX+bVLqQXUapzfQ52iurT4fawxAeWxjz/euVP8s1Ovw7vDy+qaaB/stI3/ALJS9tT7lexqdjjad2H1rrZ/AU8ajytRt5WzyPLcDH1xVe48IXSJH5cwkbGGAjIx6cnrS9tB9Q9jNdDpbb4hWEdjBG9pO0ixqrHeoyQBnvXo3ws8UweIF1OGKFojAY2wzZzncP6CvEF8G37MQZIlHbdnJ/SvQ/hVZnwtrl097eQeRcwbDjdwwYEdvrWHJSTvHc6FOq9JbHuVJ9aojWLA/wDLwB/wE/4Uv9q2OP8Aj4Ufgf8ACmIuUh/SqZ1ax5H2lP1o/tWxP/LylUhFuszUcJcwN/eDL/I1N/a9gePtUeazdV1jTzDEwvIcrIBy2Paqi9SJLQfuoLcVmnWdN7X1ufpIDTDrunjj7Rnv8qMf5Ct7mNjU3cUbjWO3iGyAyPPb6Qt/hUbeJIMEra3knssY/qaLoLG4G75qSJua5s+I8YK6fcgf7ZVcfrTR4mkDALaIp/25wD+gNF0Kx128+lFcl/wlj94Igf8Arqf/AImii6HY8Us7hIZT5jAfd4/CqV4ITI7CIHqfvHk1ZgtVudxcuCMdOe1SGzTedxJwOaySKexmo2dPY4IxuAwKqW7urMUHPrXQiyiNuEVT8xJwMiqdlaItww2kMB3+tO4rGSoaSbodxowy5DDB6Ee9bKW0f9okbAfmzz37+tQyxxrKwZF3bscjvS5h2MsM+3k9qltN/wBojweAavgBQAqJ/wB8irdvcNCT9wg9ex78UuYfKRrcqTja2Sey1t+HNQg07XbK+mErRI+5gqZOKrwywSN8zEE++M12PhrVZ4b23traO1TcwHmG3Utj6nmsMRN8jVjooQvJantuh20WpaZBfRuVjmQOqsuCAfWna34WXVrWOITiLY+4tsznj60/R73baqXEskmOWCcH8uK0m1Btvy2srcdsD+ZrzMJHC8qbWo6s6qqPU+U/iNa/YvEbqG3A7TkD619BWbD7HD/1zX+Qr5/+KEobxY+1Ng2pgEjIxmvfLRh9jgOf+WS/yFenTu6cWyallUZb3e9MLUzd70hYCtEZMGPWoHNOZ6hZ855rVGbKGpNiEYzncv8AOsY67GgK+ZaA9MeUmc/jWtqB3QjA6MD+tefvo9+9zKyy26qzkjdJ2z6AVlVpxluaUZuOx1H/AAkiwg5uolwOqxoP6VTfxq/2f7RG+oSxfLh47VwDnG3B2Ac5GPrWSvhy5lGJLu1X/gTf/E1dm0rVZ9Ik06XXbeOORQpeG3O7g5yDuHv+Z9sZRo0ups6tToOfxfqLthdM11vcxFB+pFQt4k1+ckQaLcnt+9vo0JJ9i1Rf8Izckkv4rvQTyfKt0T27GpF8J20g/wBI8Ra1Ljv9oIH061p7OiuhHtKrLNk3ie7077TDb6ciqxiIe7MnzA46qMdffsfbPGNNNcTSyu+0ytvKqykAnrg8120PhjSba18hNR1JoyxYr5y4JPJJ+XknAoTwzoES4CXBHQBpcflgCmpU18KE1OW7OSjZhDOAxJyhyce/oKx7hS1w7FgGxg5P/wBavU9K0vwsmozLfRstt5II3SOcsD7GtZrX4bI5drJZG6nKyn+ZrSKTVzKV07EvgfTbbW/BdpJczBLmEeW+xtwAHK/oRW2/g7TGclbyZGb7xVhz+YrJsPEfgzQ4Wi0yxkgjc5ZYoQAT68mpZPiDoijKWE7fUKP61ySwvvXTNlWlYS6+Gun3swefWrsgcKAEGPxxVV/hNo+0BdUvW998Y/8AZKe3xJsQf3WluRjvIB/SoW+Jef8AV6ao/wB6TP8ASrVBpW5iHJt3sTXHwm0fyFMd5deYBySy9fptqnqXgiHT7zT9M095XiuYp1aWQbgjFRjOBx0/SlX4j3UkoQ21vGCfvEM2P1FVZvHN/cXbbRAGtd0iEJ1+Ug9T70/ZW+0wUpdgT4SXSxqw1eDOOhibj9agHwxnhjmiNtZ3MrIwE5z949+Rn/8AX7CnXHxA1U6eksN9H5xbaYxCv59KzX8c+IpD/wAf7KP9mNRn9K1UbdSdR4+Gut+WkY03Sht5DM4wc9eNhqyvw68QHZ+60SPaAMYbjH0UVlv4q8QyZzqN3+Bx/Kq7a5rk33r68PHTzWqlyiszq7b4fa5HaGCS/wBPGXLkrG5xk5x2/XPb0FMi+F7RFWm1dBtx0h/+yrjpdQ1Bj+9upT/vSn+tRDUo0b9/eRLgdGnHX8TQ2uwKPmeijwJp8hEcuq5AzwqL/XNI3gHRUQ79TkB7OiIpHvn1ri7O8AjEsV5asGGRsuULEf7oOf0q0msGQY3E4/CuSpTqSleLsjog4pWZ1eo+H9LvbdILrxDqjxxSLKgiljQhxnByEyOvrWNP4X8Oojt9s1i4kUEr5t85Geo44zzWf/akSr828N9P/r1CdXj3Y3KfYtj+lONOp1kx3h2PPIgFuJRuJbPPuar3rh71o2+6UBz+FT3WLfVpl29WPU9aoaoJGuFZD83lBvTHWuuS0OZbnQeE2ii12LzYkkcK+0OoYH5SehrtLi4xLGywwIFccIgX+leZ6LfOb+B02iUPtO/JH5CuwluruRdqyWwPXHlkZ7df89KjlL5jqra+xDGrHG1AOvBx+FWYNUjSQhiSAOa4tNRvowN9uJO37tgcfnipoNTdWbz7e5AxwBEW/lnmi8kJKLO2bW7cZwG9uKr3WsxywSxeUzBlI69q5b+0mYHZYXzZ6YiA/mRTWv7rHy6VeHnqfLGf/HqOeQ+SJmSF4ziMHcvAwM5p8erSRp5VxEUc8BuSKPMcTnzIDCzciNiCf0qdV8yLY8Yx0APWtlqjJ7mIpdNQYyPvZ1zkVmaj4duNTvQ1oYgdvz7ztArb1P7NatGked45HOcD+lS6Y7fakAKjf69PpWc21saRSejOaTwHqTdbi1B9N5P9KsJ8PrsnD3sIOcfKpNejLYuRkyKM88DNSfYVzyx/BTXP7aZ0exgeeJ8PVwC+ot/wGH/69Tr4AtNozdXDH2VRXfCzUDJZuuegpBbAZ4PP6Uvaz7j9lDscCPAlopzvuGHocf0qaPwpaJtzaIdvqTz9fWu6FsmfuHI96kFtGM/L37jNHtJ9w9nA5aHTLRcf6Baj6W6E/wDoNa1vFAgwBs9kUL/IVq/Z0B6dvTrTlgX2rOV2XFJFNVt/785/7aGpDBasPmSVvqSateWmchsUYUdx171DizVSRVFpZjj7P/46BTxbQD7sAA+lTbowPvqPXGOaaXh6+aB9BS9mw50NMaKNqoAOgwM0woDjavt0qQzxD+MkfjxUbXUY4w3FUqbJdRDGg3ZIBzUbWpOMD9aebwIchGP/AAKmnUGxgKv51XsmT7REZsGPRgPahdKbeGDx8Hpg0pv5OvTPTimveSkfeP5VapMh1Ea/9o6qDnzYSB/snig6xqoAxLFn121im6l6GVsexphmYj7+fatFFmfMjb/tzVVI/wCPfH+0tKfEt8uN6wd++P6VgtJkn1qNjg9OlWokuRvyeKrraNqRZ7nr/SqFz4hvJ4JI8R7iOMoCuc+mKzDu5GMfjTemR1GapRIbHnV9Swu0W6H0EQqNtV1UnidVJ9IxTT75596YcDuSO1WQSfbtRPW9k/4CAKjea6fG69usDnCyYz+VJzwO1Jghj1z7DrQA0tJnJmuGPT5pWP8AWiOKMkkx5oJOOM5pUZgc5P8AjQIsFEBx5afnRSbv9pPxzRQM5CwjcbwIXbJ4wpPYc1I9vdMxK28uOvEZ/wAKqNdTZCx3Dhf7iOccfQ01Y7qckBJ3+iMf5VncqxdW3u1OTFIPquP51We0lRjIojU5JwZFGani0PU53HlaZeSHB+7Ax/pV1PCWvv8Ac0W++XGcwEfzpc8e4creyMp7VmfeJ4lfjOZV/oaWS3SVRuuoA3cgk5/IVsx+BfFEwwuiXQA6bgFx+ZFX7b4ceK3j2HTGjB5O6dAD+GaTqQ7jUJdjk3solGGvYz3OA3+FRskCcee57HCe9d0fhN4mmIJNsoGOHm/PoDVmP4M6y+PMv7OMexY/0pe2prqP2c+x52HjVRiSXjp8o/xq5b69PY3CSwvKHU5U7RjPrznNehp8F5xgy6tED/sxk/zqC7+EVvZQmS4109eES35Ptyx/lUuvSejZcadVao5n/haHiRF2pfThO+3YMf8AjlU7v4heIbobZL+5YAnH73b/AOggV09r4F0WAZuFubo9w77VP4KKs23gvQYMuLAsCfuyOW/Ks19XjtEvkrPdnmy/bvE2qW1tGglu55FjjLOfmJ4AJY9Oa+nbazvEtoYzbtuRFUncOoH1rgtF0PR9P1qzvhZQW4hmWQyklQgB69cY+teiN4x8Or11my/4DKD/ACrppuNRdjCopQeon2S7z/qef94f40GwvCPuJ+LVG3jbw9/DqSPnpsjdv5CmHxro5+5Jcyf7trJ/Va05YLqZXk+hKdLvG/55D/gR/wAKX+w7hus0Y/Amqx8Z2RHyWeoN9IMfzIpp8aRYymlX5z0LeWM/+PU7wXUOWb6E8vht5l2vdj14j/8Ar1w2p6ZPp2rT2yb5ghBD4AzkZ/rXXN4ymIHl6RIc/wB+4Vf5Zrz3xLpN/rut3GqiKO3WfarRCUtgqAAc4Gc4qJyg+pcIzXQ0BFccf6tT6GVR/WpFtJnGWuLdfYzA/wAq55fD+oIADctxwMSGpl8OXbxF2nDFeo3k8etZ+53NPf7G39iIPzXluB7Mx/kKd9mtl5bU4l/4A3+FYH/COs33pAD9OarXXh+YMTC8skmOVU4B+nvTvAVpHUBdLUENq6knsqf4mmtcaNGDuvpmI/uov/xVcMbTB6PgHByTkU9bIFiWTHrk9PSn7oanRXWqaUsm6K5lAIwd8fX2wBWPNrtuCQkdw4HGcIM/mwqNLaIH/Vj3zxSm1iONqjHcdxRdC5WQ/wBsq33bWcsP7zxj+TGmNrLkcWkf/A7gjH5KatrboFJCpnsc4I+tOWOMZDLFjvgYp3QcrM86teHlLW1Ht5rt/wCyCmjVNRfGxbUY54idj+rCtQRRdRtIHtTjGMgHa2T1FF0HKzLa+1NxkTRqfVbXt+LmmpPq6/Mt1ICQQcRRjg9RyDWtgAfK+RnHWgiL5htYADrjIouHKZONRcEG6uPQ4Ma/qEpptrz/AJ7XR56m4YfyIrYIUgAMD+gphKIeu3HTtRcXKjKa0nbgyS7f9qeQ8/8AfVRjSR951jYD+/ls/mTWyZY9uWKnjqRimi5t93oP9ocUXYWRlDRlP/LO269oVz/Knrpwi+6EUd9saj+QrS+0RnkNuGfT/P8ASozcAEEeYD05p3YWRUFmyDHny89g/Wmmx6bpJjnnlzVg3Tfexu/Q037R1JV1I9DnFGotCu2nRkgtkn6nrQLGMZATHXjHapzMxBIAZcYxUZkdVwu0nHAIp2YXRkayuydZFA5A5HFVrmU/6O4JwVKk/j/9etHUFNwseBkgkEgY7VD/AGPJdQ4jYKVYGqtoTfUz9MxbeILaQoGTeGw3GQK9Aj1eyHLwlM9PkBH6HNcBZqzXcLglXjJBIOD0NaZeUsdhY45xu5xUctylKx3kGp2MgG25iRfRhs/9CFXk8ubmORHGOi4avMWZimXIAbpuGOntj/P8gIT0GS2cFevb/wCtUul5lKr5HqAg6cDPQ8Y/lSmId3Iyex9//wBdebxy6moJinuOByBLwPwzVpNW1aAhWeIjPWRcZ/Ie/wDnil7MftUdNq9orlHRhvHqQTWau4YGSW6EAdKqx63JMQskSNk/fWVRjnrtzV1XDklcYb0raCsrGUnd3Mq4tnO6V1yS2c4ycCkhcxOjhclSPzrQeOWZG5Hl56k1mMPnA6gHHPvwalotM7qzvoZLZcqSVHt+dTm8TnCk/XFctpFyfL8vPHYd/atXzPUD644rL2aNfaM0Dejp5Q/OmfambkKoqkATnhseopQTjhv0o5EHOy0bqTJ4A96YbiXPL4/4DUALAcgHNKd3TA9iDT5ELnY8zOcgseaaZHODmT0puDnHIHuKXHsSPrRyoOZjd7EHO4n3NNyWGe31qTap4x+RpCnt/WjlDmITk9X59jQCwAxn8RUhUg9fz70woeuP1p2Fcbk5zuP5Um5jnmnj5ednPSkJVuoOfpRYVxhOR05pDIfTPb8KlAT1AzSBVHT09KdguM85gBuXikMmOuc1ISMgcZ9aYcZwMZ7nFFhXGF/rgUeYNuTTsDPGPwFIR0INOwriFlI6D8qQ7CP/AK1IwJ5yMdetMaNs5GAadhXHYU5H9aaQrA4/DJo2uCSWH1FNw2OSKADaB3FJjHfp2p3z8jC5FNPA5Ue4xQAwk565pvTjNPLqOGXikDKccMeOoFFxWIyevU8f3acp56CpAoP3d2PUjrUiQEjAGB70XHyjODyVGe9FWRAMdE/OilcOU9yXTbRFAS3hUDoBGKlW3RRwB+VFFeGdd2BjGfSgwjuTRRVJDuxhRV7ZpCAOcUUU7FJjWAHYUmflzgUUUi0V7iUxQtIADgdK5K78y8l8yaQn0AGAKKKqBZVFohcAk80jW6E5OeOgz0oorQQgt45Mqw3A8HdzmnNbQRSMkcSLtOPujtRRVxZLQqrxxgcelOdcsF9AKKKq5NhpGDjrz/8AX/pTgoaFT6MQKKKaEMCDn6Z6VLHGGLKTxtJ/IZ/pRRTAjaFQc0RoqyhlGCPXnNFFMRNNZxo6leFYBgMdM9qi+xIxYE49wORRRQBha1psP2M34yJFba4/v4bGfY/55rniFCD5R8wooraGxjPciaVVRf3eQzY5PTvnp7VELtljBA6vt560UVoZ3Hi5dmUYGcDnHrUUlzImw5GG9Bj/AD0ooqrCbGNdS7xhvmB6kCnLJL5nlvIzAjd6UUU7E3FbKMGLFvmA/OmuWDr87fN056CiinYQwfNGp6ZFKshAVG5LHGRxRRQA4KC23GDjqOOlCx4/iPWiikJimEbhgke4pme4J69zRRTAQFA20rndxnNRyzeWxIBJz1JoooAhFwJCD5YB9c/59avxQCZc5C/RRRRTEPnsDBF5rSBwZAMbcdfxpVh2cBjwRg0UU0JnNfZxHfIQejgHjrzXaR6NZyPsCsD9RjpnsBRRWaLB9LtYJAoVyR3EjDP4ZpVsbUqSYVb2f5v50UVQCC1slIH2OLJOOFH+FWGtI4Iw6pHz6JiiikMRSWA56VnsSZzjjJx0ooqokyM7UdXltrj7HGgYkFt8jE9z2GPSsc6uWeRZYFYpxlG25/A5ooqZblR2Lmjaz5155KwbOAc78/0rtMAgZz0B4NFFJjQvlgjcCRxn9KMEdGP40UUAKAfm5+lCpuPBI5+tFFAAYiuDu/SoixDcnPOOaKKAELkEA4NORtxVe570UUASTRlPm3559Pr/AIVACSTz6DpRRTADksRmggfrRRQAGEgE7vwxUe35j0/KiikA0LuBOcYpdnA5oooAaQVzg8CnICRnd9MiiigCUQ56sD+FL9nXuT+FFFJjFW1RwSCwx6nNS/Y0Q4znAoopNjSEFupb+HJOM7aayBRkYHpRRSuOwwDgMDjOacke9sEj06UUUAhViVsjA9elOWFByQfwoopDHGJMn5aKKKAP/9k=" - } - ] - } - ] - }, { "language": "markdown", "source": [ - "## Do Inference" + "## Initialize helper functions\n" ], "outputs": [] }, { "language": "typescript", "source": [ - "const tensorData = new Float32Array(inputImage);\nconst tensor = new ov.Tensor(ov.element.f32, Int32Array.from(inputLayer.shape), tensorData);\n\nconst inferRequest = compiledModel.createInferRequest();\ninferRequest.setInputTensor(tensor);\ninferRequest.infer();\n\nconst output = inferRequest.getTensor(outputLayer);\nconst { data: boxes } = output;\n" + "// Function to extract bounding boxes from the model output\nfunction extractBoundingBoxes(output) {\n const { data: boxes } = output;\n const foldingCoefficient = 5;\n const numberOfBoxes = boxes.length / foldingCoefficient;\n\n return setShape(boxes, [numberOfBoxes, foldingCoefficient]);\n}\n\n// Function to adjust bounding box coordinates by a given ratio\nfunction multiplyByRatio(ratioX, ratioY, box) {\n const scaleShape = (shape, idx) => {\n const position = idx % 2\n ? Math.max(shape * ratioY, 10)\n : shape * ratioX;\n\n return Math.floor(position);\n }\n\n return box.map(scaleShape);\n}" ], "outputs": [] }, @@ -134,14 +96,14 @@ { "language": "typescript", "source": [ - "// For each detection, the description is in the [x_min, y_min, x_max, y_max, conf] format:\n// The image passed here is in BGR format with changed width and height. To display it in colors expected by matplotlib, use cvtColor function\nfunction convertResultToImage(bgrImage, resizedImage, boxes, componentsCount, options) {\n\tconst defaultOptions = { threshold: 0.3, confLabels: true };\n\tconst { threshold, confLabels } = Object.assign(defaultOptions, options);\n\n\t// Define colors for boxes and descriptions.\n\tconst colors = { red: [255, 0, 0, 255], green: [0, 255, 0, 255] };\n\n\t// Fetch the image shapes to calculate a ratio.\n\tconst [realY, realX] = [bgrImage.rows, bgrImage.cols];\n\tconst [resizedY, resizedX] = [resizedImage.rows, resizedImage.cols];\n\n\tconst [ratioX, ratioY] = [realX / resizedX, realY / resizedY];\n\n\t// Convert the base image from BGR to RGB format.\n\tconst rgbImage = new cv.Mat();\n\tcv.cvtColor(bgrImage, rgbImage, cv.COLOR_BGR2RGB);\n\n\t// Iterate through non-zero boxes.\n\tfor (let i = 0; i < boxes.length; i += componentsCount) {\n\t\tconst box = boxes.slice(i, i + componentsCount);\n\n\t\t// Pick a confidence factor from the last place in an array.\n\t\tconst conf = box[box.length - 1];\n\n\t\tif (conf < threshold) continue;\n\n\t\t// Convert float to int and multiply corner position of each box by x and y ratio.\n\t\t// If the bounding box is found at the top of the image, \n\t\t// position the upper box bar little lower to make it visible on the image. \n\t\tconst [xMin, yMin, xMax, yMax] = box.slice(0, -1).map((cornerPosition, idx) =>\n\t\t\tidx % 2\n\t\t\t\t? parseInt(Math.max(cornerPosition * ratioY, 10))\n\t\t\t\t: parseInt(cornerPosition * ratioX)\n\t\t);\n\t\t// Draw a box based on the position, parameters in rectangle function are: image, start_point, end_point, color, thickness.\n\t\tcv.rectangle(rgbImage, new cv.Point(xMin, yMin), new cv.Point(xMax, yMax), colors.green, 2);\n\n\t\t// Add text to the image based on position and confidence.\n\t\t// Parameters in text function are: image, text, bottom-left_corner_textfield, font, font_scale, color, thickness, line_type.\n\t\tif (confLabels)\n\t\t\tcv.putText(\n\t\t\t\trgbImage,\n\t\t\t\t`${conf.toFixed(2)}`,\n\t\t\t\tnew cv.Point(xMin, yMin - 10),\n\t\t\t\tcv.FONT_HERSHEY_SIMPLEX,\n\t\t\t\t0.8,\n\t\t\t\tcolors.red,\n\t\t\t\t1,\n\t\t\t\tcv.LINE_AA,\n\t\t\t);\n\t}\n\n\treturn rgbImage;\n}\n\nconst [ __, componentsCount] = output.getShape();\nconst img = convertResultToImage(image, resizedImage, boxes, componentsCount, { confLabels: false });\n\ndisplayArrayAsImage(img.data, originalWidth, originalHeight, display);\n" + "// Calculate ratios\nconst [ratioX, ratioY] = [img.width / inputWidth, img.height / inputHeight];\nconst boundingBoxesArray = extractBoundingBoxes(result[outputLayer]);\n// Resize bounding boxes to the original image size\nconst boundingBoxesOriginalSizeArray = boundingBoxesArray.map(box =>\n [...multiplyByRatio(ratioX, ratioY, box), box[4]]);\n\n// Takes original image and bounding boxes\n// and returns the image with bounding boxes drawn on it\nasync function putBoundingBoxesOnImage(img, boxes, threshold = 0.3) {\n let finalImage = img;\n\n for (const box of boxes) {\n const conf = box[box.length - 1];\n\n if (conf < threshold) continue;\n\n const [xMin, yMin, xMax, yMax] = box;\n\n finalImage = finalImage.drawRect(\n xMin, yMin,\n xMax - xMin, yMax - yMin,\n { color: 'red', width: 3 },\n );\n }\n\n return finalImage;\n}\n\n\nconst resultImg = await putBoundingBoxesOnImage(\n img,\n boundingBoxesOriginalSizeArray,\n);\nresultImg.display(display);\n" ], "outputs": [ { "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAIFArIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1SXSkkG1ihGQehHQ59a429+D3h+9klkY3SNKxZjHcY5Jz/Epr0RvvfjSgVEcPSg/djYp1pyWrPJpvgbpjKVh1G/j/AN5kf+grPufgXMYysGtvj/btR/R69rFLVciFzM8Cn+CetpjytRtHx/fSRc/kpqhL8JfE0RYqLSX/AHJtv/oQFfRtLS9mh8x8yP8ADjxXGrE6cGPolzE38nqtJ4M8TQld+i6gRnkrAzD81zX1IUVuqg/hTDbQHrEn/fNL2Ycx8py6VqVqx+0Wd3F/11gZcfmKgG4ZBbnmvrMWsI6KV/3SRUU2mWlwCJYhIPR/m/nS9mw5j5W+fAzxTwW3AjJAFfS03hHQpx8+l2Z9zbRn/wBlqjJ8PfDshz/Z1uPopX/0Eip9mylJHzZEzYI3H7x6/Wp0ZiseM/d6173L8KfDjA+XZhDnOUmkH82NUZfg/pDABJLpAOBtuAR+sdSqbTLc00eJbjuY8kdOKYWbHPPocV7DN8Gockw6lcL7NGj/ANVrOn+Dd8ABDqStj/npBj+TGqsyLnmJPI5FKCSSNxrvpvhJr0Zys9k49MyD/wBkqhN8MvE0IbZaW8mf7twgH/jxFFmByO9gn3j+PanAuGAyfxrem8DeJIcZ0edsd49sn/oJNUpPD2tW5Jm0nUEAHVrZwP5UrDuZ4JYnp+VLuPHofSnNBNEAJUdGzjDLjrQFJYZ5xUlCgnOOfes+d3XU4scHA/rWiAc5x7Vm3fF3EQM57596mQ0aMUm6PrnnAp2/AznNRwHbFv754pxOAFweaUdimBftkY9qCxAz+tAIJPtTSV25I6+tUSOJbOc03cc8nHelbHTn3o25Ht069qkYmfu5yD2oyScnj60u05xyTTgpB49KVxgCRTWO5Dj09afg5HAye1KVODnnIzUsZl2TsplHPbI/OtLYjYbbgHms2Bis0q4/WteHJgTjgipe41sQ+SpxlTSGAf3m9OtXAoAzwKUqMcLmi4WKX2f+EMffNM+ynPOCO2RWhsB4oCZPFLmHYoCyLAEqhP5UNZnHCd+gNagj9BSFc5qeYfKYxtW7Ix9/Wo2tGByQ6jPcVulMDnjHrTGjOwnB6Ucwcpzo3K/r2qdYtyAnAPrSTrtkPHetSwVXtACoPJB4zQ2CRlPZAIWaJD34ArF1C12xl0hC8/wjFdfdJHgoFGO+KxL6P/RZMdAc/jW1KTTM5o5bawPANHmOP4m/OrxT3PPrUezII6/hXZzHPYq+Y/8AePFHmHvU5jGM4H5Unlgj7uPei6EQbzS7+MU8xjHIIpjIAuRmnoGopJ4JzzTd555609uY09gR+v8A9eo9poAXfil8w+lNwaTB9KYXJFcA8jinmYYwM49ahwfQ0lKwXJd4IxRuFRUUWAc/NNoopiHI5SRXBwQcg19uaPffb9EsLwHInt45fzUGviGvrv4X3pv/AIa6HKTkrAYj/wAAYr/SuLGLRM2o7nXlzSbzSUhrhOiyF3GjcaSkqh2FViHYZ9DT9x9ajHDj3Bp1ddPWJEh24+tG4+tJRViF3H1o3NTaKYh28+tQzMd8bZ74qWopwfLyOxzQxoXc3rSb29aTtSGkAu808OcVHTl6VLQ0O3tVS+lmjsriUT+XsQsCEzjAz+NWGOB0yT0FVLyyF3aTQzOxEileOAv0Hf8AGpavoPY46TxpbC2huJLdpZJNrBSqgpn+LO44+nB9ap6ze6fKJQ11cm7JCNGMny/baOAT6MQcVn3ekwXlhd2YmZLzT2MfmNgkqBz379f0riFvtkUNs06tPBJhcAFVABODkc4PvjNY1KV4tGTqNM6z7bpwHK3ue+JUH6eXRVJfFjBQHS4LAckAdfzorx/Y1OxtzRPdG+9Sjmkb71KK+wZ56HClpBxSikMWlpOtLQAtFFFABS0lLQAUtJS0hhS0lFAC0UlFAC0daSloAaUQ9VU/UU3yIv8Anmn4CpKM0ARNbRMMFePTJqnNoOlXBzLYW0me7xK38xWhmilZDuYM3gvQJs7tKsx9LdB/ICsy5+GHha5YM+mRhh0KvIv8mrsaM1PKg5mcE/wm8P8APlC4i9kmOP8Ax4GqM/we09jmC/uo+ONzK3/sor0uijkQ+ZnkkvwclCsI9WYk+tuP/i6ozfCHVhjy722YL03q4/kDXtNLS9mh87PBp/hb4hiDlBaSn/ZlIz/30BWfL8P/ABFABnTVZRyds8bfpur6JpCARyAfrUukhqoz5sfwrryKXOjXzDtshZv5Cqcuk39vgT2V1F674WX+Yr6aWGIoAY0/75FH2aH+4B9Dip9iUqh8ulcM249PejYdpwck9K+n5LG3l4ZC31YmqUvhvSJ/9bYWzH1aBG/mtQ6LGqiPlfaUu2XuRxWxaZNqgPQZ9s179N8PvDM7bn0m13eqx7f/AEHFVW+Gnh/btithGvoryZ/VjUujIpVInig+7nH6U4RbgF5+tevy/CzSWOY5bpcdB5q4/wDQDVRvhZEhJivpRngZQNj9RWbpTLVSJ5gIULHIGKcsKjr1Poa9Gl+GNyMeXfBsf34sfyY1Sm+HeqpuKyWzHtw//wATWbpz7FqUe5w/lgtxkbR1PemmI468n1FdbJ4E1tMYto3A6kSqP5kVTfwrrEWSdNnJ7BBu/lmo5Zdh3RzhhOSBj3OKYyN5fbn3rcl0TUYiA9heIAOS0LD+lUJbd4j86upHQEYNLUehyd8hVyCBnOTg1a06YrasOpDcUmooVDjHzNyah075oXHuK0SuiHuWXOWz3PrWbeoDDKD0A61pEHGe9Urpcq4xwVPvWkCJHOsPTnHvmoOTxjg1YJwABn+dQHBFdaOdjW4zSMBxnGfrS56kHnofemknjI4NMBDjpj3pj/dbFOOcd6aeQetUIQH/AEccfdfr9R/9al4OB1zSIMwSD0INIMkUwDC0YwcGjNHX+tABjNBUDijnOCOKPrQAmP8AOKQrTs+2fxpD7UCGkYptOI4pKYhK+mfgNffafh89uWybW8kQD0BAb+ZNfM1e7fs73vya7Yk94plH/fQP9K58Ur0zSk/ePcTRmg0leYdaFopBS1QCHgqfenU1/u/TmnV1UtiJC0UUVqSFJS0lAC01xlSPUU6kNOwEKcoPpSmmowO5RjKsQfanUgEpy96bS5xnHWpY0OAJOAOaazwhXDSKMe/Q1Onkqu+V05H8R4rAvNathqzad5roWXeFTjI756n3wB9cVlKryK493Y4DxmZYfE32/TpI0VodkrSKYwTyMFiAOmfeuJtNAxOupahf29naysfKUtjzeecd8e4HX1r1XxBq0ElosEttJNG2GSJISiJ6HLLtyOvXFeWXfh+91L95d3iR7mLCWc+X8o+p549K5Z13JWbsS4JPa52McPhJIkUMjAKACUbJ/wDHxRXMx2Nokao19bFlABO1+f0org9k/wCdl83kfQjfepRQ33qUV9azgRzviZPPu9LtmdhHLKVYKfdRn68n86a3hawUMfNucL/tL/hUviD/AJCujf8AXf8A9mStWUZWRemQRXlU8JRr4qs6sb2a/wDSUeHHB0MRi67rRTs4/wDpKPPNI1fwjrV3FaW17fRzynCJMgXJ9MgEZrql8KWHeW5/76X/AAqhZeCrK2uIJyVdomVwwXByORWr/awl1k2UaSjyk3SM6FQSTwBnr0PI4rp/svB/8+0dH9k4L/n2ho8I6eR/rrn/AL6X/wCJpjeFNPBwJbk+vzr/APE1vofkrF8TRvP4cvII2KtMojyDzhmAP6E0v7Lwf/PtD/snA/8APtFaPw3pkoJS4ndQcEpKpx+QqceEtNI/111/30v/AMTXOeD/AA4+hasJYXZYZkKSR54PGQfrkfrXdeZtl2e2abyrB/8APtAspwP/AD7RlHwhpqjJnusf7y//ABNN/wCEU0v/AJ+Lr/vof/E1uSN+7/EVxXjjUNesJ9POi3CxhlkMqNErB8bcdRx36etT/ZeE/wCfaH/ZOB/59I2x4Q0xulxcn/ga/wDxNL/wh2nf89rr/vpf/iat6fcyS6da3NwoSWSJGdRxgkDI/M1oq2RR/ZeE/wCfaD+ycD/z6Rh/8Idp3/Pa6/76X/4mj/hDtO/57XX/AH0v/wATVrU71rHTby9SFpmgjeQRKcF9oJwD+FcpZ/ENrlgp0LUVY8YQ7sfoKP7Lwn8iH/ZOB/59I6L/AIQ3Tv8Antdf99L/APE0f8Ibp3/Pa6/76X/4mtqOUuoPI471G87Aud2FX29s0v7Lwn/PtB/ZOB/59Iyf+EO07/ntdf8Afa//ABNH/CG6d/z2uv8Avtf/AImoNK8b6HrNylvYapHLM/3Y2jZC3fjcBXRJNuHIo/svCf8APtB/ZOB/59Iw/wDhDtO/57XX/fS//E0f8Idp3/Pa6/76X/4mti4vEgKBiBuzjLYpovQR9wkexzR/ZmE/59of9kYH/n0jCvfClhbWNxOk1yWjiZ1BZcZAJ9K1PDbs/h+1LMWOGGSc8BiAPyp+oXKyaXeqFYH7PJ1/3TUXhk/8U/a/8D/9DNYUsPSoY1KmrJxf5owo4Wjh8wUaMbJwf/pSNfNFJWPqHiSy067a2lWZ5FALbFGBnnHJHau6tWp0Y81R2R6dfEUsPHnqysjZJozXO/8ACY6f/wA8br/vlf8A4qj/AITHT/8Anjdf98r/APFVzf2lhP8An4jj/tfA/wDP1HRZozWFa+KrC6uooFSdGkYKpdRjJ6dCa3K6KOIpV1enK514fFUcQnKjJNIRelOzTF7j3p3atjcXNFJQKAFozSUUDHZozTRRUgOzRmkooAXNBAPBANNpc0DQ3yYjz5af980jW8TDBXj6mn0UrIq7KM+haXc58+wtpM9fMiVs/mKz38DeGpM/8SezXPXZCq/+ggVv0UuVDuzkp/hn4YmyRYbD6pLIP/ZqzLn4PeH5ySst5ET/AHJh/VTXoOaM0uSIczPIbj4B6Y2fs+rXsfHG/Y/9FrHuf2fbgD/R9dyf9u1wP0c17uDSg0+UR833HwE8SJkw6hp8gHTJlU/+gGsi5+C/jKAHbZ203oUuVX/0LFfVGaAaNQ0PkC4+GXjG3zv0G4f/AK4skn/oJNZtx4N8TWwJm8PaqgHc2cmP5Yr7RIVuoB+tM8iH/nkn/fIouwsj4aayurcSrPbTRHb0kjK9/eq2DivuxrWFxhlyPTccVRuPDejXWfP0y0lz18yBX/mDRzMOU+IsHvQT7V9kXHw48I3IPmaBYDP9y3RP/QQKybj4MeCbjJ/shIyf7k0q/wDs1HP5C5T5NowT0r6guPgL4Sl/1QvIf9y4/wDig1ZNz+z1pLEm21O+j9N5R/8A2VaPaIOU+dQOKNo9a90uP2eZxk2+tk/9dLUf0k/pWRc/APxFFnyb6wlHYMJFP6KRR7RD5WeQkDvTcc16TdfBTxlD/q7W1m/3LkL/AOh4rHuPhb41tTl9CmcZ/wCWUkcn/oLGmpLuLlZxteqfAS88jx3Pb5wLiydcepDK39DXE3PgrxRaE+d4d1RQO/2RyPzAre+Fwu9I+Jejvc280AeRoT5kZX7ylR19yKmrZwYQTUkfVZpKM0V5R2hSikoqkA4jII9aFOVB9qBzSIflI9CRXRSIkOooorcgKKKKACkNLRVCIAMSuAOvNKaR/wDXKfUYpTzSC4VLBD5j7iBgVEOXVB1PQVo7fKhwgyQOPc1DSBuxSuIobhj9qZVtlIG0tgMx9f5YrC17SdLhs5pFX7GH+VpklaItnqOASx6cY5rp22x4JGcdOMnNVr2wj1GDyrxA0LEHys8HHrjr9Olc06aaaau2ClqedqWnTEU2qXUKDIMillOAfuqQSDx7HHHeuNvWsrvVktnv75nuZSjRPHgqOMnAzgceg+hr1vWrl5LeOHTYyEjZQHRcgjOCq49u5wO2fTznxLqI23T2E8ENy0G2VbfDOpAK4LD/AHgRjGea82dPllZO50LWNzLlXT0ldF8F6wVViBlpRx9AnFFc/wDZddPMT3Hl/wAG5hnHbPNFP2Ue/wCLMebyPpZvvUUrD5qBX1DONHP6/wD8hbRv+u//ALMlaF9zZ3I9Y2/lWf4gH/E20b/rv/7MlVtY/t9dQeO1e2aylwMOp3KCMHn864sK0sTXv3j/AOko8zCpvF4j1j/6SjjvDXhe90bU7O5t72byg6iSMvwy9CD+FeizhfPiJHODj9KYtk8dmxjAMoXKg9M9qydO1HVr++EN7paW6ICfMWTPPpj/AD0r0LruehZnUJ/q/wAKpaiN+n8c/Mh/UVejUhMVz95rkOkzNa6nFKsJz5c6IWUr6HHIIoTE0aNuoVoz/noaV3zqGB2QfzNYzeMfD8Ee5b0yN/CiRtk+3Iq5o0s1+GvZImiEzblRuqr0GffAobBJmy3+rH1H865bxd4s/wCEanslbS2vUnDlmV8GPbt7YOc5/Sumu5Y7a1aaZ1SNMFmY4AFYd8dH1eSNpNQtW8sEKBIp6/j7UhlhGj1uystTgmmWIqJUjzgNkfxD1Fa8DHYM1hNqWn6dbW+nWsqSSMRHGkbAkDuTjoMZrbt8lATQBWvpLeLTrqW8YrbJG7StgnCDJPTnpXPWWveEnZRa6hFuYgKCrDJ/EV0V3CLmxubcjPmI6YPfORWHYeFLK1mWVrWIMCCDtBqWVE6iLpx0qBufOB7n+gqeMALjNQkfNMPU/wBKaEzk7DwXa2OoW12NiyQOHBUcnFdFb6glxdyRRq4ER2sXQrk+2e3TmvPtE8P6lo+q2dxDqFw1t5gDxM5KlTxXo2B5qHHOOtDBFXXtDttbtUjuY9xQkoc4wTVXQ9KTTNGlskyFV5Mc+vP9a6FeVqoBh5h/tf0FAHHaLoUulWd7K93cyg2rqFkkJA4/+tXV+Gj/AMSC2/4F/wChGo51A0e7OP8Al3f/ANBNSeGh/wASC1/4H/6Ga85/78v8D/NHBL/kYx/wP/0qJsZri9R58T3/AP1zX+SV2dcdfDPie/8A+uS/ySjG/FS/xr8mGY/HQ/xr8pF3TLzymaIk7W5Hsagsrlv7VEhZiC7Z5+tYMmoSW/iv7A/Eclos0Lf7QZgw/LafzrS09mMqs4IPmt/6EcfpXd0PS6mnrrbtV0X/AK7/APsyV0lcvrBzqei/9d//AGZK6euHD/7zW9V/6SjzcJ/veI9Y/wDpKBep+tQX8rwadcyxna6QsynGcEAkVMOp+tVtU/5BN7/1wf8A9BNdNV2pya7M7a7apSa7P8jkYk1CWyguBq10DKGJXe3y4JHXPtV1NOvWtPtDa7dAAEsPmOMf8CqKz/5BFl/uv/6G1El6p0+9Eb5CKwYZ6MBXmYbBUJ06cpp6pP4pbtep5mCyvDVcJTqSTcmk370u3qS2mnXt3u265eKQAec//F1a/sDUP+g9df8Aj3/xVO05jHJgHqtX9QuWgsJJF+9jA+prohl2Ha2f/gUv8zpnlGET2f8A4FL/ADMSSxlico/iW4DDqAHOPyap4dGu503ReI7hx7buP/Hq5nWINSuYEGm3PkOCWY7c7vQVc8L392UtpLrInJ8uUevOP8DUrL6Fr2f/AIFL/Mf9j4Paz/8AApf5m8dA1ADJ1+6x/wAC/wDiqBoGoEZGv3X/AI9/8VWhqkL3elzwRthnQgc45rmtC1Awyjd8ufkkB7Ef5/WtHlmHSvZ/+BS/zIWU4S9rP/wKX+Zq/wBgahn/AJD91/49/wDFUv8Awj2of9B+6/8AHv8A4qqr38tzqAKzSLHuAARsZFdIJQULZ496hZfhm9E//Apf5lPJ8Kuj/wDApf5mJ/YGof8AQfuv/Hv/AIql/wCEe1D/AKD91/49/wDFVS1bxI1rDJcq0ggToIk3M1WPD/iNdUhhlVy8M33Swwynpg4pfUMP2f8A4FL/ADH/AGNhez/8Cl/mS/8ACPaj/wBB+6/8e/8AiqP+Ef1H/oP3X/j3/wAVWzc3S29tJMwyEGcetZZ8QRpjzUVM8DMmM/pTll+Gjun/AOBS/wAwjk+FltF/+BS/zIv+Ee1D/oP3X5N/8VU/hW7nutKdriV5WWYqGc5OMA9e/U1dsdSjvSyopBUZ6g1l+Dv+QRL/ANdz/wCgrWMaMKOJpqnezUr6t7W7s5/qtPC46lGldcylfVva1t2zo80ZrO1q7lsdIuLiAgSKAFJGcZIGf1rAjutckiR/7YtF3qG2sACARnn5K6K2LVOp7NRcna+lv80ddfHqlW9jGEpStfRLa9urXY7HNLmuViXxFOcRavZOfRcH/wBkqb7J4ox/yErX/vkf/EVH1xv/AJdS+5f5kf2jP/nxP7l/8kdLmjNcyLXxQR/yErX/AL5H/wARS/ZvFBOP7Stf++R/8RR9cl/z6l9y/wAw/tKf/Pif3L/5I6XNGa5v7J4pA/5CVr/3yP8A4igWvik9NStf++R/8RS+uS/59S+5f5h/aUv+fE/uX/yR0uaN1c19k8U/9BK1/wC+R/8AEUfZPFP/AEErX/vkf/EUvrkv+fUvuX+Y/wC0Zf8APif3L/5I6bdS7q5j7J4q/wCgla/98j/4ij7J4q/6CVr/AN8j/wCIo+uS/wCfUvuX+Yf2jL/nxP7o/wDyR02aSua+yeKv+gla/wDfI/8AiKPsnin/AKCVr/3yP/iKX1uX/PqX3L/MP7Rl/wA+J/cv/kjpTTCAeoB+tc79k8U/9BK1/wC+R/8AEUfZPFP/AEErX/vkf/EUfW5f8+pfcv8AMf8AaU/+fE/uj/8AJG8Yozz5a/lTTBGe2PoTWF9k8Uf9BK1/75H/AMRSfZPFH/QStf8Avkf/ABFL60/+fUvuX+Y/7Tn/AM+Kn3R/+SNw2644LD/gVM8g9pX/ABx/hWN9k8T/APQRtf8Avkf/ABFJ9k8T/wDQRtv++R/8RU/WL/8ALqX3L/Mr+1Kn/Pip90f/AJI2THIOki/iv/16TbKP7h/MVzmoXHiDSYUuLi9gkQyBdqoOep5+UccetdRV0KkKzlHlaatv5/Nm+FzBYiUocri42uml1vbZvsR5kH8A/BqhgectIWiVSWzjd/n0q1UY4lPvXQqaWx281xQz90/I0b8fwN+VPozWlibjPMX/AGh/wE0nnR5++KkopWYxu9T0YH8aWkKKeqg/hTDBESfkXn2p6iGXDbArHPDCk37zhPxPYVFdWga3cRsytjg5JA/DNTRgCNQOmKNbhoXba38obz949c9askgKWOePQZqsl0oQBslqzLjXL2y3NLpbSx7sK0EobA/2gQCPwzWVSpGCvInlcnoa7L/y0bI46Z6Vzuu6hd3TCw09FwMGV5CQHXugA5I9SO3Geapap4nnuJBZqqWzEg7S2ZGHsO35Gs641SS1s3CRWyvM2HMp3FsDqSck/jXmVsVKTSp7dzohQaV5DbxZhaSRXunSZbPmTRFZh9dmQo4AAHOOB6Vw+rWujWXkS2ck5hlIVsrhg3UFgeMc8YA610txqt0Iv9HltDIwIASFSAexzjGB9a5m6u43ud1+z3tyOAVykP5AZOfwPHQ4rBybZbikjnpNW1aKRo0tL10QlVY7SSB3ziiusMliSdtn8vbbDkfgTnP50Uvbx/lM+VntUg+ekFOl+/SCvqjgOe8Qf8hfRf8Arv8A+zJW80YY81ma3pVzqD2k1pLHHNbuWG/p2Oeh6EelVvsfij/oI2v/AHyP/iK8xVJ0MRUbpyala1rdku6PHVSph8VVk6UpKTTTVukUurRuBABimC3UOWA5rG+x+KP+gja/98j/AOIo+x+KP+gja/8AfI/+IrT67L/nzP7l/mb/ANoT/wCfE/uX/wAkb4XAqvcWkdwpV1BB9RWR9j8Uf9BG1/75H/xFH2LxR/0EbX/vkf8AxFH12X/Pqf3L/MP7Qn/z4n9y/wDkiZdBs0k3rAgOeoUVpwwCNQAMCsb7F4n/AOgja/8AfI/+Io+x+KP+gja/98j/AOIpLGNf8uZ/cv8AMf8AaM3/AMuJ/cv/AJI2rm2jubZ4ZUV0cYKkZBrmJvBemO2fsyD6DFXfsfij/oJWv/fI/wDiKPsXif8A6CNr/wB8j/4ih4yT/wCXU/uX+YlmMl/y4n9y/wDkhmmeGbLTpfMhgVX9cZP51vomBjFYf2HxP/0EbX/vkf8AxFH2LxR/0EbX/vkf/EULGSX/AC6n9y/zB5jN/wDLif3L/wCSKesaFqU9/LPZ6vdW6Pg+WrfKDjsKoLpPiSM8a7MR/tRA/wA62jY+Jz11G1/75H/xFJ/Z/iX/AKCFr/3yP/iKHjZ/8+p/cv8AMazF/wDPif3L/wCSNi1RxCokYs4ABbHX3rF1ldbjvwdOktxbuBuWRCSD3P8AKniy8TjpqNr/AN8j/wCIoNh4mbrqFr/3yP8A4iq+vT6Up/cv8xfX5f8APif3L/5InksXGnv5YBlC5QHpkdKoaReateXRS+s4oEQZDK+ST6Y9Kn+w+JsY/tC1/wC+R/8AEUg07xIpyL+1z/uj/wCIoeOn/wA+pfcv8w+vy/58T+5f/JG8i4Wuf1PW49M1FreS0u5DIAyvFHlfTGc9ak+xeJx/zEbX/vkf/EVHJpniKQgvfWrEf7I/+Io+vS/58y+5f5h9fl/z4n9y/wDki7cKw0W7JH/Lu/8A6Cad4a/5F+1/4H/6EaoPpniKWF4Xv7Uo6lWGMZB/4BWzpVk2n6ZDau4dkByQOMkk/wBaypSnVxSqODilFrW2915szoSqVsaqrpyjFRa1tu2n0bLhrkLsqPFGoFiFAhXJJwBwldfXF6u9tB4hv1v4JHgnhVQAvX7vPJHGVP5VWYTjT9lObslJfkx5rUjT9jObslNXfykUNfsftaRXtqytcWmHQqc5x1H4g1csZRIIZV+62CKbp2o6NpkDRQWs6qSSflHJ/wC+qoKujR6mLxFvBh94Tjbn6ZqnmmDa+NF/2vgb/wAVG9rLBdQ0dj0E2f1WulimWRdy5/GuK1G9j1q5soLJZo5FYjcVGRnHIwT0wTXR6Na3Nra+XdXLzyZyXYAfhxWeDqwq1qs4O6bX/pKIy6rCtXr1abvFuOv/AG6jVH3jVbVP+QRe/wDXB/8A0E1ZH3vwqtqn/IIvf+uD/wDoJrtrfwpejO/EfwZ+j/I5yyH/ABJbD6P/AOhGubiVrHxZqFtMS1pqaiRPQOqhWH4jFb/+mJ4VspbK2FxIGcFCccbjUc+kXOpaRHLLEIryMiVFBztYds/TiscHG9Cnf+VfkRlsrYKl/hj+SNOw+VoxnOFxk1b1cZ0uU+mD+tYeiXt3dXvlS6dNAqA7nc8Z+ldLd25uLCaLjLIQPrXTSVlZnZVd3dGFpqKzsD6VRVBDfzKvAE2QPyNVbTWYrWdorlhb3MfyukvANWba4hvb/McqO7tuIQ5x0/wpS92PKwiry5jsYvmUA965XVrI2OsCSNf3NzksB2YV1cCkIM1m+IUUWSSHjbICfatY6qxD0dzO0uHzJQ2OF4rfuFIspcddh/lWToTLJHIVIOG7Vuuu6Jh6jFZU1Yuo7s5G3t0ukSJwCrDpUel+GZrDU1MV5ItmsnmLCqjHrjP1qXT5hG4z96JijD0PSrcl9cteP5T7YlwFwAc+tJrlvcpXlaxr6ou7Srgf9Mya5K/0u21e1ijukDKpDj2OMf1rsbkebZyr13If5Vx1zYnU9MjgEssJO074m2tx70VfhTCl8TRteGNMsdKieK0QK7HLnByfSpPB3/IJl/67n/0Faj8M6O2mJIz3NxcO+BmaQtge1SeDv+QTL/13P/oK1wVP96pekv0PNxX+/wBD0n/7aXPE3/Iv3X/AP/QxXLKsjacVjbbJ5XytjODt4NdR4l/5F+6/4B/6GK52JSsar/0yjP5oKbX+1T/wL/0pipv/AIVX/gX/AKVIyPCeoapIk41E5ubabCuE25GPb/PNekyyH7O5Q4baSPyrzXRNZvLrVbixvbWOKWAB1ZCcMM+9eh20nnWqv/eXmuuGktT1Z/CYWhXLm4idnZt6kHJzz/8ArFQ3Wq3S63cqJpFWKRQFDYGMelM0nMcyxnrHMyfrVbVF8vxDcj/noiP+mP6VdvjRPWLO2uLjyrWWUDO1C2PXisnRNUuLuUCZ1Ksm4AKBzxV6I+fpq5/jiH6iuc8PyYmtgfQqf1pS+FMcfisdDrGpnT7aNwm9ncIATgD3qXTL431p5jIEYNtIBz6VleKlJ0pJB/yzmU8fl/WpPDzfupk9GB/P/wDVRNWSYo63N/NZratCs7xjexQ4OBxmtDPy1yUiA315CTwXYHt1qZ6Qcl0HHWSTOig1GGRwm4hj0DDFXc15npltd6R4gksZbqaa3kiDx+axbb+f416DFKZbQNn5iuCfepi3e0imla6HRX1vNM0KXETSr1RXBYfhUzuFGa830CXyfEVux6lipz7giuh8ayONIjCkhWkw2D14Nbez99wM+b3eY6beCuaFYMK8s03X57DSL20EpXeu6NgeVbIzj6j+VbPgS/mlu7uGaeWTKBlDuWAwe2fqKhx925V9bHdkgDmm7geM1geKNebRrJTEgeeTO3d0HbJ/OuGg8Za3FdxiW43CQZUPCArfTA/rUvQpanceMP8AkExf9dx/6C1b9chq2ppq3he2ulG0mcB1/usFbI/z6119cVH/AHqr6R/U8zDf7/X9If8AtwlNI+cU6mv2Poa7D1B1FFFMAooooASijvS0CGsMgiq8JJiHqODVnFVogVllXGPmyPxpgSU5U8xHTjpkZptKsixEu5woByaipFOLTBOzOZ1ixtpp2hnhjlDYbYwBx23c+lYUumTSKn2LUZ4Rz80iiUYJzxu5A9MEf1rp5Ld5p7m6k3x722ojqMqABj+ecfnWbfJHaadI06nyyCMvn5z6Z964rRldtWRq27HOTWX9p3bRPcRfZ4huARPmdT3Zsnqc+g68evBavb3DahcQXj+RahzHHI85JaMH5cgZYLwD6V6PbpDb2ZeV4GuJm8yXLABD9PboBXmfj3TL271h1tGaWB9ki7Hyu4jBJ9Tx+tTCnBPTQmTdrmzHfWqRIi67OqqoAVYGIA9Ae4orgo9K1FY1H2xFwANuentRUfVqf85PtX2PsKUfPTAKwj4z0mU9Z1+sef5GpV8VaOetw6/WF/8ACva9pDuYezl2NqlxWWviLSHxi+QZ/vKR/MVOusaY3S/tvxkAo549xcsuxdoqBL+yk+5d27fSRT/WpldG+66n6Gq5kKzFxS0UvagLCUUtGKAG0uKKWgBKMUtFACYopaKAG4opaKAEopcUUAJikpaKAEopaKQCUUUUwE2ikKinUUgG7RSbAKfRQAz+P8Krap/yCL3/AK4P/wCgmrf8Q+lIyK6MjqGVhggjII9DUVI80HHuRVjzwce6MDQbyyj0K3jmu4Ecbsq8gBHzHtWgL/TgMfbrX/v8v+NRf8IzpH/Pp/5Ef/Gj/hGdI/59P/Ij/wCNcFGGNp0400o6JLd9PkeXQhmNGlGklD3UlvLordh4vNNVtwvbXP8A11X/ABqX+0tPx/x/2v8A3+X/ABqv/wAIzpH/AD6f+RH/AMaP+Ea0j/n0/wDIj/41pfHdofe/8jbmzL+WH3y/yK19Fod/zPNYyH1aRTTLKHRbE/uJ7KMf7Miirn/CNaR/z6f+RH/xo/4RrSP+fT/yI/8AjSf119I/e/8AIanmS+zD75f/ACJOupaeAP8ATrX/AL/L/jVe/m0nULR7ee8tWjfqPOX/ABpf+EZ0j/n0/wDIj/40n/CM6R/z6f8AkR/8aL47tH73/kLmzL+WH3y/+RK+lrpGlweTb3VqiZyf3y8n860f7S0/H/H9a/8Af5f8ar/8IzpH/Pp/5Ef/ABo/4RnSP+fT/wAiP/jQnju0fvf+QOWZfyw++X/yJgavptpPeNe6drEFtO/3x5isrfhmmabE8c+691a1kUdAhVf1zXQ/8IzpH/Pp/wCRH/xo/wCEZ0j/AJ9P/Ij/AONKTx0t1H73/kVGpmUdow++X+Q8X9h5ZX7dbdMf61f8a4qSbVLaRo4ZNNkiU4U+bzj867L/AIRnSP8An0/8iP8A40h8M6T/AM+v/kRv8abljmrWh97/AMhKeZJ35YffL/5E57Rb+6OoIbxrKCFQSWE4yfbrWv4P/wCQTL/13P8A6CtWv+EZ0rP/AB6/+RG/xrQtLOCxg8m2jEceScDnJ9ST1rOnRxEq0alXlSins31t3XkEKOLqYmFavypRT2be9u6XYo+Jf+Rfuv8AgH/oYrmr65SwjtJHimYSW0ZzGhboo610vib/AJF66/4B/wChipLG3SbSLLcoP7hOv+6KtP8A2yS/uL82OH/Izb/uL/0qRxdrd2U98JIopTcsuz/VMOM557fnXcWUZjtUT0FCWEStkIM1bWPaMV1Ri07s9eUk1Y4W7uV0vXbmCc+WsjebEx6HNMu5Fu9RW6DAfuvLIBzk5zXYalo9pqcYW5hSTHQkcj6Gs2z8K6fZTrLFAA69CSTj86qpK92upMEuvQ0rBGSwiRvvBRkHtXMaf+41FkzjZcsPw3V2aptXFYzeHbc6sb4tKctv2byFz64pX9zlYL4rj/EMfmaHcj0UN+RBrP0W6SB2LuArqDk8CuhurYXNnLAeBIhU/iK49vDerRHEepfKBwDCDROS5UEFqzs4biOcEJIjEdlbNc1ffu9emXs6q36Yq5oGnXdm0kl5ciV2G0BUCgCq/iGzvvtsd1ZRRyfJsYOxHehSTg7hy2krFLVXU6zpTqOWEin8h/8AXrpLA5tQPSuXtl1aSZFmsY1H9/zM4/SuttYvLhVfQVipOU0+yNHG0Wedt/ofiM9hHdE/huzXW+KovN0GRsZMbK364/rXP+INPuTr8ggt5XMrBlKrx09fqK63U4Xn0O5jI+cwngeuK7OZe1UvQwSfJY8dupPJuIY2+5LuQ/XqP5Guk8FSGHXI0J+/Gyfpn+lZ8ekvrJuEhQs0a+YjY4Dg9M+/IqXw87w67ZFVJYyAFQORng/1qJrew4vudD49jJW2bsUYfTBBrm52s7jRNNieV0uUDncgHA3Ed/wrtPGluZNJSYDiN/mPoCMfzxXnE1rb3VibeaP96jho27YyMiiSVosa6o6yNrRfC4gtfOylyrSNLjLEq3PH0r0SvKbadTbSxr0Lqfy3D+teq15lD/eavpH9Tgof8jCv6Q/9uCmvypp1IehFdx6VwByoPtS01OUFO5oAKO1LR0piE70UUUAJVdvluR6Mv8qs1XnBEkTDGN2D9DTYkPqFQz3W88Rxkbfdu5/Dp+dTUmKmSuNMdqMYKj0xXLahChaFWYON24RJyzsBkAfzPauh1Ge8Marb2Pmgr99pQoz6dz2rA1NLq2tJ5kiO/wAsF9oL8+gPZc4/U8152JrJLli7M1gtNTnLmJrnU/L1C5WGFx+7iU9FA53P265wMe5qvcRQ3hvPLsILgkMWnjwRFjAGB3J5Pbrj3q7qEEsqqlxKYoMcxrt3k9MAZI98g8VzWoIftDLDqojtEAIjR9pJIz06nAx689PbzKkpzupGtl0L6Q6UqKrQTFgACTNEM/pRVJPCiSIryatIjsAWXJOD6UVheP8AMLkfYeJGVuv61ZjuM9TzVJ87sjrTkI9xX0D1BGosgboalDcVmq5U4/CrKSkd+PWpaKLOEJ5RfypRtB4UA/So1f8AGnhsdP51IydJ5E5WR1/3WIqZdRvE+7eXA/7at/jVPPFOye2PWndhZF9NY1JOl7cfjITUqeINVUYF4/4qp/mKy80vanzy7i5Y9jZXxNqi9Z1b6xr/AIVMvizUR1EDfVD/AENYP+NGTT9rPuT7OPY6RPF95j5reA/TI/rVhfF0v8dkh+kpH9K5RTzntU3Sj29RdQ9jB9Dql8Wp/FZMPpJn+lSr4stf4racfTaf61yIzjNLT+s1O4fV4HYr4psD1juB9VH+NSL4k0xussi/WM/0riqOtP61Mn6tA7oa9pjf8vQH1Rh/SpF1fTm6XkP4tiuBz70ZxVLFy7C+rR7noa39k33buA/SQVIs0T/dlRvowrzj0yfzox9DT+tvsL6qu56VwaXFeahip4JH41ItxOn3Z5B6YYiq+t+Qvqvmei4oxXnw1C9XGLu4/wC/h/xqZdY1Fel5L+Jz/On9bj2F9Wfc7vFGK4hdf1Nf+XnP1Rf8KlXxLqK9Wib6p/hVLFQJ+rTOyxRXJL4pvh1ityP91h/WpF8WTj71pGfo5H+NUsTTF9XmdOR8y0uK5g+LWLL/AKCD6/vv/sanXxZD/HaSD/dcH/Cn9Yp9yXRn2OgorEXxVZMeYLgfgp/rUy+JdOYctKv1jP8ASqVaHcXsp9jVorOXxBpZP/Hzj6xsP6VIus6a/S8i/E4/nT9pDuLkl2LtFV11GxY/LeW5/wC2o/xqVbiB/uzRn6OKfMu4crH0YoBB6EH6GlxRcVhKKXFFAhKKWjFACUYpaKBpCUUuKKRRS1WybUNMntUYIzgYJHGQQf6VkRad4lgiSKPULUIihVGM4AGB/BXS0lctbCQqz9o207W0bRw4jAU61T2rlJStbRtab9Dnvsfij/oI2v8A3yP/AIij7H4o/wCgla/98j/4iuioqPqMf55f+BMy/syH/Pyf/gbOd+x+KP8AoJWv/fI/+Io+xeKP+gla/wDfI/8AiK6Oij6jH+eX/gTD+zIf8/Z/+Bs5z7H4o/6CVr/3yP8A4ik+xeKP+gla/wDfI/8AiK6Oij6jH+eX/gTD+zI/8/Z/+Bs537H4o/6CNr/3yP8A4immx8THrqNr/wB8j/4iukopfUY/zy/8CY/7Mj/z9n/4GzmxY+Jx01G1/wC+R/8AEUGx8THrqFqf+Aj/AOIrpKKX1GP88v8AwJj/ALMj/wA/Z/8Agcjmf7O8Sf8AP/a/98j/AOIpwsvE46aja/8AfI/+Iro6MUvqMf55f+BMP7Mj/wA/Z/8Agcjmm0/xIzAm/tSR32j/AOIpfsPibGP7Qtcf7o/+IrpKQ0fUY/zy/wDAmP8AsyP/AD9n/wCByOYTS/EMedl7aDPoo/8AiKjTRdcimMqXNmsjdWCDJ/8AHK6ukpfUI/zy/wDAmH9mR/5+1P8AwORzUuneIpomjkvbR42GGUoCCP8AvisU+CboHrb/AIO2P5V33akxT+pL+eX/AIEw/suH/P2f/gcjhrXwXcwKIke3ji3DOCzEfTI9/Xmu3p2KTFaUMNGi2022+7vsdGGwVPDuUottytdttvTbf1EopaK6TrGR/wAQ9DT6YvErD1p9ABRRRQAUlFFNCYtQzjMRI7HNS0xhuUj1FOwhOvSkpIzmNfpS0hkkm46c+z74zt9q5zUkeSBAkpaISK7KuP3mPcmt+EGSJ/NH+0qen196yNQVVkPHI5rgq0HObkawkkrHP6pbi7vLXfbr5WC0mWAbjGATyQOvTr7DmsvV2mW2lng0jTxLEhMcjMG2bQcHbt9ulXri9XlLaMzSDJZYiAFPU5J4H8/aqoSRlZ7psDPEUXOc9jnr+lS8LC3NLVh7R3sjzseKYgAH1e93j7224bGfaitN9ABdi/iLT42JOUEcQCn044/KiuPlo/1/wxXvGoy/MeOM0qrwPapZQqN8xCn34pEwR94fga7U7m1rAFx2zTwPanBfQ8+1AUdqLjFVivB/SpUbpkVGMdjmhc4Pb1zSAsq/6U/II6niqoY9akDEUAWOvI+tHTqajDD+tPB4xSAd29aXHNNBGcZpfyoAUdcZqdarjg1OucVLZSHU7ik7Un0FSUKaTPtS4IpCPpxRcA6//XoA4IxQB+VLxn1ouFgB/lR68UdOv8qKLgHHWj60v50lAABR1paTqOKAsLmkPv3pcdKQj260BYDijP0ozzSHrj86YWADNLQOtJjigVheDmjOKTHXmjNA7C5oB96TvRnikAufSlwCP/rU3OOtBNMLDuAMgflT1mlX7srr9GIqIe1Lxii7FZFgX92h+W6nA/66t/jUy6vqCdLyb8Wz/OqPvQeO1Pnl3E4RfQ0117U1/wCXon6ov+FSr4j1FeskbfWMf0rIzRn1xT9rPuL2cOxuL4nvx1jtz/wA/wCNSL4quR963hP0JFc+DS5xVe2qdxexh2OkHitv4rMfhL/9apU8VQn71q4+jA1y2TxRn3p/WKncXsIPodcviezPWKdfwH+NTL4i089WkX6p/hXGA0u4EGn9ZmL6vA7dde01v+XjH1Rv8KlXVrBul3H+JxXB55pQ1P63PsL6rHud8NQsj0vIP+/gqRbiB/uzRt9GFefbsc0u6msW+wvqq7nogIPQg/SlxXnQYDpx+lSLcyp92aRfoxqvrnkL6r5noNFcGmpXa9Lqbj/bNTLrF8BxdP8AjzT+tx7C+qy7nbUVxya7fjrcZ+qr/hUo8RXo6mM/Vaf1qAvq8zq6K5hfElyOscR/A/41KviWT+K3U/Q4qliaYvYTOhxRWEviVT1tvyf/AOtUi+I4D1gk/Ag0fWKfcn2M+xsUlZY8Q2Z6rKP+Aj/GnjXLAn/WMPqpqvawfUXs5roaNJVIavYN0uB/3yf8KeNTsm6XMf4nFP2ke4ckuxZoqAX9m/3bqE/SQVIJom+7Ih+jCnzIVmOoo69OaKoQw8Sg+oxT6jkOCh98VJQAUlFFMQUUUhpiFNNpaSmIiT+IejGnUnSVvcA0owDkjNTJtK61GLGcP9aw76S2vpHj8xX2jbLDnBB9/atraC5bJDH3rlvEPg+LWZxd2t49rdA58xQSp7cYIwfzrx6+OqU52qQsjVRVrortFDD8kUYiiBwqDoornLq71CC4a4GlFplPlAi43qo6k4wCPc9OnPTPURaPd6Xa/v7qTUmAJHCRnpjGTnPP8/YCqqW89ze7jaG3aMDMeEIx6lgOn0I7etYzx6rPlpuyRcYW1ZxclnYeY3mTnfk7s+X17/x0V1zaXohYmW1vGkJ+Y/Zn5PfvRXF7V9zTlRyfxhYjw9YYzk3WcD/cP+Neb6CzsyfM2NwONx9a9E+MZxo+nJzzOT+S/wD168/8PDKwscct/Wvfy9Xoq5GMf7xnVLvGMO4Ps1PDz4H72T/vo09F696mCjGT9a9LlXY4VJjBcXS/8vEv4nNOF3eD/lu2fcCn7AOMUu3Bpckew+eXcFvrw/8ALQE+6ipBqN2oPKH8KTZ/nNJ5YNL2UOwe1n3JRq9yBykZ/P8AxqRdamyB5K/gareWDwf5UeWKPYU+w/bT7l4a4/Q2/bs3/wBanjXlHW3fI9xWfswKa0Q/yan6tT7D+sT7lqbxfZW0oSaC55GcqqkfzrRi8Q2jxq22YZGRlR/jXn2ujZepjj5Bj8zXQWke62iz3QfyqPqlNsv6zNI6Ya9Zesn/AHxT11uxP/LVh7FD/hXPeSDwBTvJ/wDrcUfUqYfW5nRDVrFulwo+oxTxqVmxGLqL8WFcz5IHAFJ5I/Op+ox7jWMl2OrW8tiOLiM++8VIJYj0kX/voVyH2cE5Iqvd2w8jkAc1DwK7lrFvsdyCCOoNKO3tXk/2chbwKxHOTg47Ukf2q2W2Y3U65yWxK3fn1rP6m+5f1pdj1rjjtRjmvJYLvVQsxOoXgLJvUee3HJ96emp61HpzuNUujIVLZaQnHtzS+py7j+tR7Hq9GPWvGx4x8QRGMf2m7fL0KIcn8q0pvGWtrKscVyCAwDFolOfyFS8JMr6zA9S7c0Y+ua4JfFWrqPmMB9zGR/I1XXxzrPnmP7NZtlyB8rDA9T81J4WoNYmB6LTRXFSeM71Uz9kgOOT8xGaLTx3NPEXfTFTnGBPn/wBlpfVqnYf1im+p2uOKTnNchF49jluGj/s2UKvVllB/oKkg8eWUsrRNZXasvJI2kH9an2FTsP20O51hpP0rnl8Z6cTgw3S/8AU/+zUh8caIrbXkmXBxkwk/ypOlNdB+1g+p0XU0EnmsCLxroEz7VvW3HpmB/wDCpf8AhK9CHDanCp/2gR/MUvZz7Fc8e5tc54o7Vmp4h0aSPeup2m0dSZQBUser6bKN0eo2rA9xMp/rS5Zdh80e5dyf1pRVcXlrIcJcQt7CQGpVdW6MD9DU2Y7oeDRSZzyc0ZwDRYYucmjOabnNLkd/1oAXPGe9LTc5H+NBPakMWgUhNGaAHA4NKScUzP1pM+xoAkycmjdxUe7jg0pNAEhPPFJu5qPdTS3+NOwibd/9aml6iLimsQKLBcnV+akD1TD1IH4osFyzv60B++f1quXwKQuaVguWt/vQH4xVXzMDvR5hxzRYLlsPS+ZxzzVMS+tOEuKOUdy0ZKZ5nPvUBk45phlGaLBcuiT34qeOT5T0rOSTPrU0Tnn0xSsBb8ymll9AfwqDf7mkMoAz19aLhYsBwOQAPpSNcup2iWbJ6KjsCfyNVo3ec/IML/fPT8PWrsFuie7Hqx6mi7ewrJbl3TFupLtZJbmYqucRmQspz65rqAcgVh2i7BmtmI5jX24r0qEWo6nDWd5D6TNFJXRYxFzRmkooEFLSUUWuBlSW99bXJaCXzYWJwjYyvsPUVHLqlxbOEuLXtywyP6H271qy9FPoaXNebLBVYSbo1GvJ6o0U090Yh8U2SMVaNwMcsOTn6VZbxVo8wiCzlWYcDb7Zxx3rQaytZkZp7eJw3UFBk1HJounSWe5bRFBk8whePm5GePqa5pLEpPms7dSk43VjKOr6bvKC9SVmUny0OeO+SeOKfHp8N2GmSFVmY7SAdowOn1xz271UvPDOliDc7yw7ACZA+AMd+elZVxY31tYWr6Rc3QjBdWDgq0244XHHHAPOR6+9eFVhzSTSsdF2kXZrazimkj/tdItrFfLEiAJg9MbeMUVCuhakVGLfTU4+60SMR7E45PvRWXspdwuzz/4xOBZ6UvTMsn/oK1xGgJ8sGP8Anp/n+ddh8ZWITSFz/FMf/QK5Pw8Plt88gsD9Oa+wwC/cxMsX/EZ1qIOp65qUDnPr1oQZ6j61IBxmvTOEZg5NOCj0p23HWnY9RwaAEx/OjHGM89aUDIz7U4cCgBmOeKMdqfjJNOA5NFxWI8YphXA9PepSMnpxTSOOfpVCZyniIf6dFx/yz5H4muisx/osPGfkXn8BWB4jGL2L3j7/AFNdFYjNpAcnHlr39qS3B7FkLxjHNPKnpzSAZHv3NSKKYhm3r60u3txUmO9G0HtQMj2gduagu1/cHp1FW8Z5FQ3YPkHp1FJjRzNx9+ZF6uwz+ApZ4xJbQKOpwBn3FTvGGuZ27KmP0qAZkkh9EKrx61lcsJlWKZsgY8vAqOJAbADHY1YuIxJdovYDJ98VWDlLNUUfO2QPzppiORuI2WUscDcxOP7vOAP0rfjt1jt7dR1JBJx1NYt2drqD05/nxXQKMwW5/wB3+VIZoviOMucYAqrZxB5ZZCOTU8mbicQg/KOWPrS2q7ZpgB35xTuAskS7TwM1QjUJaFQPmZjjitdl456VVt4RI+/+BOFFJgiOGzEUQGBuPJOO9V4YAL+Q4654rWZe/rVRVxeuT3H9KTAinj8uM/Lg9s1mvahs+vX61pzEuxYjjoKYsLeW7kkYHHvUt3KSK1rYosYfjc1ZGsRYZvoK6aED7Ov4/wA6wddGCT221fQkj02ITaU2eRlu1XYLcR2qrt5PQVFoLA2Mm4Zw54/AVrWqCRzIcEDoPSkMqxaaqKSQN7dTT/smDxxWoVHbOKbt9v8A61OyFdmPdwyiEFZJBznhiKfbm8WBNt1cAgYz5rA/zq9cpmA8dOabAg8gD0zip5VcfMyJbrUh0v7r6ea3+NPGqawg4v5yfd8/zqURgdjxUMvZVGSfahwj2Hzy7jZPEevqdkWoPuz3RT/MVNfa/wCK9P8AD76obhWiEgj3PAuGJ7DirekafZvcE3twsSKNxz1Y+gp3ji9jvdCkto3RkVQVWM/KgHbHSsnCPNZItTla9zAt/iNr7RRs32Vt3UmI/wBDXUW/irU5I0Z47clgDwpH9a8qtjttozxlW/LmvRbFQ1pEePu1MqUL7Fxqy7myPFF5jm2hP0JFV7nxvcWrqDpquGGciYjH6VEIxjP61Q1G3DbDj1Gaj2MOxftZdy+vxCOPn0lwfacH/wBlqQfEO1/5aaddKfZlNc41uuPu9KYbVSCABxR7CAe2mdSPiDppHzWl6uP9lf8A4qpF8f6MSA4uU+sX+BrjmtE4O3nPpUbWaGj2EQ9vI7oeOdBxzdyL6AwP/hTv+E18PtjOpov+8jD+Yrz17JT/ACrO1Kw22EsgPQA9B2NNYaIfWJHq6eLNBfBGrWoz6yAfzq2niDSZAAup2Z/7bL/jXz5j9x1+638//wBVblvaK8KP6qDg0pYaK6gsS+x7guo2kg+S6gb0xIp/rUgnjP3XDD1BrxMaerKeByepqN9PAHyjFR9XXcr6x5HuIfJ4+tIXxjOa8OME6DEcsiHthyKIJNYLhYb+8X6TMP60fVvMf1jyPcfMPrS+Zz1rxZ9Q1q0D51e+Zgmebhjj8z9a9k+Ekq694Umk1IfabmK7aMyS8tjapA/U1lVpezjzFwrKTtYm3joKZv54/Wu4Oh6ax5tV/Bj/AI1E/hzTGH/HuR9Hb/Gub2iNbnHpID3qZJcH/CujPhfT8kgzr9H/APrVBP4Xg8l/JurhXwduSpGfyqXURSZhtMdwVclj0A5qaK1ZyGmPTog6fj606CBYFxjLdyepqyvNVuVsSIo4GKtwrzVeNeauwryK3pU9TGci7CMKK04GzFWcnSrtq2VIr0IKxwyZYopDRWhAUUUlAC0UZpKpCYj8ofpSA5x70p5BpkZ+QeopNAiaV8AIDgd6ntTut5F7jmqZ/WrNi37xl7lelYqkowsO+pj6vAzkPgMirnyz0J9T6/T/AOtjBmnu5y8vnMFRvmJ6Ac8fXOK6jU4WliZSxjTkEr1/+tWNaxRXFpLaNiMCRANvfJ4rwqvuuT5dE0dyfuopCe8KgjkEcHc3P6UVea21gsTGyKmflUSAADsMUUvaw7IV2eQ/GRiZ9JUdQsv/ALJXOaAMJbem7+tbfxdm8660g5+9C749claxvDYxHa9wOlexgv4MTLFfxJHWrx7Y/SpgBx+opqKMU9Rg5/pXonChccUYx707n1oGcfSgY0AZpwUenWlxzRjikMQgZ6nrS4zg+tO29h1oC800IQjNMPbPQ1Lj6Uxh6UyWcp4jGLyI8f6v+pro7Af6HB0/1a9PpXNeJm23sJ/6Z/1NdPpw3WFuf+mS/wAqS3B7Foenan4+uaaFI4z+VPA49cUwsHIzR1FOx1PB9KNuRx0pXHYTAbtUN0B9mP4VY9sfhUN2ubcnGOQf1pPYZgthZLkngAD+VRlRFbwE4GW3GlnDNLKnY/MeOwHSnTKrRW8fQtj8sVkaWK8YLTl2z86k49B2pkEQETS46qQvsKtTMI7hiSMLHio0AGnnjHymmhHH6iux075zmt2ED+z4GPQKOv0rC1RHW4V2ztYjaPbHX8a1rMvLZL1CRoB+NDEjatIike5vvvyaLcf6VMKmhJMSNzyBUO8Q3UhPHy5/lSbGLOxdxCnU/eIHakswQsg9GqS2jJBlcfM9JaH55R70rjJdueDVOfakjAck9fYVclk8tCf4jWexyx9TSkxpDVjMjgD+VW5IwkBHoKfBD5aZI+c0s+PKY9OOKEJlW3H+jDI9axdcXKnjOUrbg/1HJ9awdek2ywxryWyD9Ku+hNiHQAziVBwobcfy/wDrVvaeflbjnisTw02XuUz2U/zrcseZH60uoy6BznFKRk04Dj2/nQeOcc1VySvKuY2+maitV+U9OuatOMofU1BGDDHlgcnnHpUNlJBI20YHJPTFIsQU5Y5Y1IkZ5dx82Pyp5AJzjNNAyFhVLUY99hcL2KH+VaJHyjPSq1wu+J1xwQRVXEefWozanHUPnp7V6Lo7B9OhbjG3v2rzm2zsmRuxH9a7/wAOOH0qPpxxWUjWBtAcc9Kq36fugR1Dc4q0P8iq96pNufYg1BZmFecnp3ppUZ44OKk56/1pvOOvJpk2I2GevSmFOPXJqU/mBTT/AJ707gV2Xr2+lVbuLfZToP4o2x+VXm+lRFQTjHB4qkyWcEOYJB7g/wCfzro9L+exhbGCBj9cVzyDaJk6fL/Kt3RGzZgHqrEYzTlsJGoEyOgodPT681LGpc4A5q9Fa7eXHPWsrlpFCKyaVvmGF9atiBY1woAq5tAGB+lRuuAT29aLlWsYGpx4l6feT/61ejfAe6zZ61ak8q8UoGfUMD/IVwGpphYyB1DD+VdR8D7gxeLdQtScCa1bA9Srr/QmssQr0mODtNHvZOcGg0g6Y9KWvIudYzvSHkGg8GkNQ2Uc3ewmO6ce+R+NRqvNampxZKSf8BNUAuCPfmuyjrFBJkkYq7CMCq0a1bjGBXfTics5Fhat2xw2KqLU8JxIK6EYMu5opKKskWikzRmmJhmlpKKYgzUa8Fh71JUfSQ+4oaEKaltW23Ke/FRGhW2uG9DmpktBk9+uRIvfrXD3MLPe7SSo7kdQPb1Nd7fKqgzEgLs5J9K4q5R5LlpJI9kefkQjlh6sD/L8/Q+XVdp27nZSd4Dc3X/LPT0KfwkxFiR257/WimMGLEmRySfWisvZMqzPIvi7+71XTowTtWBuMf7Rx/KsvRLlYILVjGzBUGcd60PjAwPiKyU9rb/2dqy9MAFnBnBOzr+Jrqw8uWhFk11erI6hNahxkwzDv0H+NTDWbYgfJL/3z/8AXrDXnp9KlH6YzW31iZl7GJuDV7U9RIP+AUv9qWmc72BH+yaxDx3pVPbNH1iQexibn9p2ZH+t59wRTxqFpx++A/AisIdMjrR37U/rEg9ijoBfWp/5eEH1NPN5anj7RF9N1c7wB0ppAzjGKf1l9ifYo6Q3MH/PaL67xTTPERgSIf8AgQrmjg88flULgDnj8qpYh9hOiu5F4slRb6Ahs/uzyOe9dZpbqdNtiGGTEv8AIV5trLhLmPt8tb2mhfscBxj5B/KqdeyvYlUr6HchfT+dSBT61yan0JH4mpVLg/6yT8GNL6yuxSoeZ1IB64596QjIGPXmuaWSbP8Ar5f++zUguLgH/Xy/i1H1ldg9i+50WD3qK7X9wfYisZbu5/5+JPzoa6uGTDTsR6HFDxEewewZGy5e7YjkAD9KgQF5oZD90EKo+gqfLkt8/LcHjrTRvG0BuE5HFZ+1RXs2MuI/MumHUKuTUUp8vTQp/iQgc96nG7cz5G5hgnHWq08ZKBTjCjA4pqqhezZzeuLte3x6dfwrVsEB0tMAcrz9ax/EDuLiMMV2hRgYrT0mR2sIwMbcEYxVuasSoa2NmzINtGfb1psgD3Sx9B3PrTYGeKIIoXC+tKkZSRZAckHPPepdRFKDLmOgHXtVa3IW5mBPvUnmtxlBz71EQSzsBgt70nUQKDIp5S7Zzx2psIHnpx/nmnGEk5xx3p5H71XVcBRjFSpq4+Utgcjt1qKYnynx6HpSmcYxtamNKpBG1uc9q050RysqrL5dr6NnA96ytRgKw+Y43OQfwrVAXzNzglQPlGKoazcxLACc9ccr7U4yQnFmV4Zb/TJxk8p/X/69dBZf8fMign8frXL+HpBHqz56FSOnuDXT27Bbkt/Cc8020gUTU5H+FJnjGaYJUPIP4Yo8xAOvT0o5kLlYrHaDzz0qvC2+c7uoFPXa7lncAL0FRQ4W6yRheRkmp5irGxY6e90C7ArEO4H3j6CtUacgG9k57KUPSn28tqsUbGSLaBhVyOfrzVHUdUWQNFbuOfvOOPwHtWfM5OxdkkU9QaBHMcIRv7zgcfhWVIu5SCDVhsMOwqFl655reNkYs872eXe3cZHQkDjpzXZeFnLaeec7W/xrkr9RFrt2PUk/yNdJ4QkzDKueh/w/xqZFwOr71Fcgm2fnoM09TikmBMDjGeCOKg0MnnB9qQ9fYUdevSgn8qLiGkGmHvwSKcf/AK1Ie+T/AEoAiYcf41GeOc/lUxHGMflUB65HFNMVjjJotmp3EfT5nAzWp4diaSOVBnhgfzqLUrQrqs8h4UEMB65AzVrw0225uI8jJUHn2/8A11UpaCitTpoIljHyjn1qcYA/rUa4BqQcH3rC5stBe1RyKCOvPepcjGD0+tMY54ppiZj6ov7kEZ4bn8qt/C+4+x/E2yGcLNvjP/AkP9QKh1Bc2zjv161n+H7j7F440W4yFAuoST7bwD+lOSvBonZo+qDw31pe1EnYijORXiHZ0Iz1pD0pW600nAznArNlIhuo/Mt3HcciskCtW1vba8lmjgkEnl43EDjnPQ9+lUHj8uVk9DXdhdrE1NBI16cVaSokFSpXpQRyTJl7VKhw1QipVPIrYyL1FNQ/IPpS5qkSLmikoqhC0UUU0Ji1G/31980/NMl4UH0IoYhTTTTj0pppDNFsSWsTEZwP/rVh3cCtdAFFYsQBuOK2rZt1lj+6cVlaomORXmYuN2deHfQQ6hZIdggiIXjOOv6UVjhJMD5M/nRXF9XX8xryI8L+LrhvFEC9harx/wACes/TZ4Us4VMiAhBnLCrnxWk/4q9RkjFqgOPQlq4piRI2DwAOa9GjG9GK8jOs7VZHcpNCQMzR/wDfYqZJoiOJFI+tcADjjP4U7cT3xjtVeyJ9oegBl45H509WGe1ee+a27KknHpTvPk6iR1z6MaXsg9oehD6Zpe4rz77VOowJpPYhjTxe3IU4nlx1++aPZsPaHf5Hemnt6muDGo3isCLmYen7w07+1L0Di6mI/wB40ezYc6O2J/yKic5XAH4Vx/8Aat9kf6VJ270p1e9B5uXP1A/wpqDFzon14/6TF67eh+tdFpfNjB1PyD+VcXc3UlyVaZ9zjjJFXYNZu7eFEjcbVGACoOBVSi2khRdmdwlTL6Y47c1xK+I78Z/exnj+4KcPE+oL8xMX4r/9es/ZstTR2vbFOPqP1HSuLHim+GNywfip/wAaePFt6ODFAfbB/wAaPZyHzo7IHBHH6U7P41x48XXRwfIhPr1/xp3/AAls2ObWMn2Y1PJIOdHWkgDGaM5A6+tcp/wl8ne0Q/Rz/hTh4t6k2g44/wBZ/wDWp8kgckdPuyPWonPpXOjxcuP+PQ/9/P8A61KfFMLcNbSDPuDT5ZC5kVfEbFZ42PPFXdDbOnRde/8AM1h6vqSX5Vo0IwMHJqzperRW1sI5AxIJ6D3q2nykprmOtjYAYqXPesJNetRw2/8AKpl160PV2HsVrOzLujYyM8A0bvSsga7ZMB+8Iz6qaX+3LM/8tfzBpWYXRqlunIxRkdulZY1qzJ/1w/EGnf2vZ8ZmUfnRZhoaWRznA/rTSe1Z/wDa1mQR568fhQdVszj/AEhPzo1C6LjnisnWBusjx0YGrLalaHP7+P8A76qhqN3DJbSBZUYjsCKqN7ksxdGbbrC8HkMP0rs4iOB0+tcNpj7dYiZmG3ceSfauzikBxyOvrVz3JhsXwe5zQRxzxUatxwQc++aXOeO/pU3LHZ/lScDPANJnNIelK4rDiV9BTSAe1JuHXpSA00xiEJk8fWo2VR39uDUjE9f0qGRuOtNNk2OR1pEi1SUgEEgHj3FWfDcjYnCsy4I5Bx6/4VH4hGL1CehQfzNReG3xdzoTjKg/kf8A69a/ZIW510csuR++kxjPJqUSysMeaxB681VjP8qnBz3Gc1nc0AIBxk5FHlAcdKcv54oz+NK4WIzFkfepDGVP3jj3qY5wM80h6/5NFwsV/K49/pTo7cZ55qQkDr+NOTIzzzQ5Akc9r0W263Dq0X+IrP0KTbqns6sP5GtrX1BMDkZHK/y/xrntLcpqkDE98dfWqjqiWtTswcdD+VSr9Pzqurce1Tg8845rM0Hg46fjTGbjp+tKWGR2/Go2PBz+NFwsVbs7opF9UNctczGOe2lVuUOQfyrrJVBO09DxXH3y7YkLZ+VsH/P+etaRd9DOSPr2CZbmzinT7sqK4PsRmnIeCK5rwtrdpH8PtH1C8uY4YhaRozu38SjaR7nINYeo/EWykd4rOVo4/wDnoUO4/T0/n9K8ZwlzNJHZDVHZahqtvYjDHfL2jTr+PpXKalrM15kSNtjPSJen4+v8q5iXxLaOTiU8nJJzkn3qpJrtu/Pm8d6aoy7HRHkj1O68I3fmatPCTkPFu/Ij/Gujvo8TK46MP1Fed+DdXgfxRaxLICZQyf8AjpP9K9PvY98G7upzXTRXLLUwrNN6FBB0qQCmKKkFejE4pDl61IKjxThWpmy5Ecp+NSVDAfkIrP1TWv7OuUgW2aZ2TecNjA59j6Gpq1oUYc9R2Ry4jEU8PDnquy+b39DWo71z8fiO5lz5ekyv/usT/wCy0reILtCA2kTKT0yxH/stc/8AaWG7v/wGX+Ryf2vhO7/8Bl/kb9Gawv7cvv8AoC3H5t/8TQNevT00W4P0J/8Aiaf9pYbu/wDwGX+Qv7Wwnd/+Ay/yN6mSjMTfSsX+3b7/AKAlz/49/wDE0sevym6ht7nTZYBM2xWYnqeOhAzyRR/aOGel390v8gWbYS6XM/8AwGS/Q2VOUBHcU002L/VAdxxTjXaekXtObIkT1ANRXMaldz8heSD3osG23IB/iBFS3QfayhAfX5sVw41e7c1pPUxWu5Nxw8gGezY/pRTjFLn7wHsG6fpRXjc0PM7+WJ82fFB93jRx2WCMf1/rXHMMStnArrPiW2fGtz04jjH/AI4K5UfMzA859K9uhpSj6HLW/iMASDQfqadtBIpcdBjr3PetTMbuPt0pDnOfWnlcc5xmjHzZ7mkAwHHr+PFGOOAak2Z+7/KgLz60XAjz0xgZ9aXtjnFPVSc85BOMYpAvTIPv7UXAjJ5OOtGecgAZ7etP24GD+RFIVA5yOO1O4rEbcc5HXHXpUoPyj2GOtLb2st7dQ2ttHvnlcIiA9WPQV06/DvxPgZ0zBHX98n/xVKU4x3Y4xb2OW3YHH5005J6V1q/DjxQyknTlx6+fH/8AFUH4a+KWP/IPUD1M8f8AjUe1h3L9nPscnnnPAIoz+I7V16fC/wAVuCPsEYP/AF3T/GlPww8VkAfYE/7/AKf40e2p9w9nPscismwkkA/UU0uSc9K6p/hx4nicqdPyT2DqcfrSD4d+KGIUaadx7mRcfzp+0h3FyS7HL7ue34UFx+Heunf4e+J1Yq2mtwezqf60w+APEy/8wyT/AL6X/Gj2kO4ckuxzSt2z+NAcZBx165rov+EC8TAZOlSe/wAy+v1pG8CeJBn/AIlUnOOhX/Gnzw7i5Zdjm2JA9v50+NjtJB4rfbwJ4lb7ulS/99L/AI0+PwF4oyd2lTcejL/jT9pC24ckr7GBvxznjuKBIQO4xx1/z610f/CCeJSD/wASmbp/eH+NNPgPxJk/8Sqbkccr/jUe0h3HyS7HPbz3P04pRIVXrzit9vAniQjnSps/Uf40n/CCeJv+gTMffI/xp88O4uWXYwPN6cHOMYpRKQuehrfHgPxPyP7Ll/76X/GlHgLxNg/8SuT8WU/1o54dx8suxz2855ppkOOcc8nFdKnw/wDE7kgaVIOOcuo/rUTeBPEm7H9lzf8AfQ/xo54dw5Jdjn/NLH8fpSGUcDPI7+tdAfAniYcf2VMcHrkf40n/AAgniY4/4lU3PTkf40/aU+4uSXY5lXIceuacZD0ya6D/AIQPxP0/sqYY9x/jTv8AhAfE2P8AkFSfmP8AGn7Sn3FyT7HPCZxyHI/HFO+1TqQPNfj/AGq3T4C8Tf8AQKlHHqP8aP8AhAvExOP7Llz16j/Gn7Sn3Dkn2MVdQugvFzNz6Of8acNTvQP+PufA7eYa2G8B+J++lS/gw/xpP+EB8T5/5BUn5j/Gjnp90HJPsZA1W9Vv+Pu4znoZDUi6xfAY+1y+3zmtP/hAvE+cf2XL9cj/ABpf+EB8TD/mGSf99r/jRz0+6Dln2M3+29Q6fa5Px70463f9DdOceuD/AErSX4f+KWzjS3/77X/GnN8PvFCNtOmPkejqf1zS5qfdD5Z9jAur64uSDPIWYcAkDp+FNtLyaznMkTYYgg8Z4rfb4feJwVH9mOoY4BLjH86lX4beKCwH2Ac/9NVH9aftKfcXs59jNj8R3qD/AJZNjrlf/r1MPFF4OPLhJ/3T/jVtvhx4pH/MOH181P8AGoj8P/E69dN4/wCuqf40uam+o+Wp2GjxVcDrBD9Of8akXxVJnm1Tj/aNH/CvvFGcf2fz7Sp/jUN/4L8QaZZSXt3p5jt4gC7+YjYGQOxpXpvqO010LS+KmHytbIT7P/8AWp//AAk6kc2n5P8A/Wrkw3bPanCQ9sZp+zRPOzrF8TRHGYHGR2bNTQ6/A0mPJkHpyK48Oc8/lVm2lPnDrzx+lTKGhSkdBquoRXUKKqOGBzg/SsCOTy71HxwsmTjnvVuRsr0z6ZrNlJEr/UdKVMJM7BNSgx1YfhUyapbYA3N/3zXNo3yZGee9TbuAeMVm0UmdF/aVtgHf39DSG/t848zr7GueD4XuD/Oq09/FBxnc3oMUKLew3Kx0sl3b7c+aAPWuS1S9haaWKI7gz/eHTr2qlcX01wTlgqk52jpVU9a6YU7bmM532N/S7jXtVRLG0u2aK1Q+XE7DagLZOAfc1pPo/ilQSTE3/AF/wrlbe7nsp2kt5GjY8ZU9q0U8Uawi7ftrkf7QB/pSnCV/dsVCcLe9cuy2viSM/NGv1CJ/hVctrafejH4qtQN4k1Nx81xn/gAqFtavHGDIP++RSUZ9Ugco9Gzo/CWpanaeMdGlmTbGLyJXOP4SwB/Q19WModSp6EYr4wh1i5jnjl3nKOGGPY19m2063VrDcJysqK6/QjNc9eLTTsaUpXujMC4JU9RTxUt0m2cnseaireDurkS3FApwpB1pa1M2TwHtXO+IYJbnW4IoFDSGEYBOM4LGugiPzisjUJPK8T20mM7bdjj8Hrkx3ww/xR/M8rM/hp/44fmRQwHT9QhVHYuwUyIW3bSeuPap7yyuJ7q4Z2eJACyyjBHTgYNV9Glimle8upV8wnIBq/q9wWEcEbfK/wAxbPGO1dqtY9R7kCXMraPJ8xG11GfY44qvbtJaXUM3mlkl/hwOMHBHvWn5EQ0qWCGRHbYSdpBJNZLsJLKEg5KyEfgRn+lUSWL77RPezpHIFdOUBGRwM4qvNctcXOj72LMs+MnuMpVyV9uqQSdBIik/yP8ASssgpqlgn924Cj8GA/pXFmX+7v1j/wClI8zNv90frH/0pHXDhnHvmg8mg8TH3FBruPSJIG2XCN2DCtWVMsfcVi1uE70RvUZrCvG8WVF2ZlNF8x+tFXTGMniivF5Dp9ofI/xG58b3uM/djGP+ALXN4JduCfm611Pj+JpvGl68S70Pl4ZeRwi+lYuIYy+9CH3etepSdqcV5E1NZsp7DjOM+1KBng/lj2q6t1Ag2hMZ9xQJbLrsb/D9arUkphCcdeKAhBBwTVwS2Z+VQ3JwOTTRJaEjGQDxRqIrbDkbT+lKBnI6DvWja28d0jOnQHGSe9T/ANnKAen4mp5irGOV5PHPSk8vt3FbS6aoPzDAzj7wp40uE8HOO3zCjmHymCVB6DoOlIyFeoP9K6D+zYMHLNz7jFMbSrZh/rWH5UKYcjK3hNseMdDzjm9j/wDQhX1YEjbBKLnHXFfL2i262vjbQ0Viw+2RHJx13ivp4PhR7VlXXM0zSk7JjvKiB/1af980pjiP8C/kKa06kbdvP96mlzWPszXnHeXFn/Vr+VL5UR6xr+VR+cE5IJ+lNEueaPZhzimGPfuCLn6UpjT+6PypN9JupqAcxII48fdH5U4Rx/3V/Kog49advo5A5h/lp/dH5U3yoz/CPyo35pPMo5A5hwijB+6PypwRPQflUe+l30cgcw8IgH3R+VBRP7o/KmeZSF6OQOYdsUHoPyoKp/dH5UwufU0m+n7MXMP8uP8AuD8qaUT+6Pypu+k30cgc47Yn91fyqORE2/dXp6UF6ZI4x6UpU9BqY5FTcflHSkKptUbV/Ko0cZHuKQPyKz9mXzknlpsPyr+VM8pNmdgyfakMvyc+tDP+796PZhzkgjjx9xfyphjQk/KMn2pPM+XNNEmX57U/ZhzjnjQAfKPypTFGR9xfypkj5A+tL5gIBpqmJzI2ij8wfIOnYUx4kY4Cr+VOeT5s8cCmK3GTnJNP2YucjeGNSAFXH0qrqV5Z6damSfCA/KpCZP4VYupBDEZGzhVLHHtXLaTrp1LUEnk6hd+3dwoPQVcaVxOoawvJb5iNP0me7ZekkrFEX2GTz+AxTbx9dsoN50NZB1PlurYH0xmuxs54/KVUUdO3epbmYonIAJ7VaoxsPnbdjkdP1C11KM7Ygkyj542ABH4VM9vGxwI19+OlWb22twzXawKkuMmVOpx2NMi+ZA4/i5rNQTdhzbRCsKB8FV6+lZ/iu1juPCuqxbF/49JCBtHUKSP5VqPLhio+nbrnOfX2qLUUE+nXMWM+ZE6/mpFXGnZmbndHyxgZ/wAKUY7nGOKUoRn9c0u0s3A7dhXccYAHPBHH61LB8sqn3HIpqqeuPzp4AHIAPOaljRoORjB6nnkVRuSfOB9vTrWg4G0Y6HrxWfe9FI9P8/1rOnuXLYuwEmJCPQcg02adIk3OcenqazxemKFUVRkdzVV2Z23Mdx96tU7vUhzLVxqDyjbH8i/rVLPvRRWqSWxDbYUhpaVqYhDSUvYGgjFACUUUUAFfYPgC9/tDwBoVxnJNmiE+6Daf1Wvj6vp/4I3v2r4bwRHra3EsP67/AP2eufEq8UzWi9TvLxMoG9DiqVakqb4mX1FZnepovQ0mtRaUGkpa3RiOU4bNZ2rWV6+pw3lpCkwWIoVZsevXkdmrR9j0qxGcxIT1KjNZ1qEa8OVtrVPTyOTF4WOJhySbVmndb3X3nJrpOop9zTkX/dnI/wDZ6nntdZuFjDWUa+Wu0FZMcf8AfVdRRWP1F/8AP2X3r/I5P7Pn/wA/5/ev/kTl7W21u0kMiWiMSMfPICP/AEKof7N1jzCyWiIpOfLDrtH0yc/rXXUtP6i/+fs/vX+Qf2fP/n/P71/8icxdW+t3SRq1lEhj+6yOAcenLdKbbaTqj6hayTwKiRSBy28dAQexPpXU0oNJ5epaTqSa7XXTXsTLLOdWnVm1o7Nq2jv2I5BiRD9RSkUS/dB9CDSmu89MbWvaNvs09uKyDWlpzZgdfQ5qJq6GibFFPxRXD7JGlz4cRzGqgMR9DQXbcQwIIJzwOKigDMiDPPtV2HRNUup2EMeWAzy4rsUbsnmsVxKf9rnpTllx/CRz3FXf+EV1wD/UAD/rov8AjUi+FdbJI8lOn/PRar2b7C513M4S5A+X/wAdpfMzwFOf92r3/CLa4Bxbrz6Ov+NIPDeuKP8Aj34/31/xpezYc6KqzOBsUuB1wq9fU8fhThcz5wHm9OCa3NM8NX7wubm1y4bj5x6exq+3hmcEFLMj2D/T3qHFp7Fp36nKrdXLOMPLyMd6mWS+I+UTfgp5rpE8N3SsCbV15/vj/GpRpF7GSEtW46dDUO/RFxt1ZzAGqsOI7kj/AHSf6UjnVYhl4bkZ6kxnn9K6wW1+gINoT/wH/Comh1DdhreQZ5+4QKlOXY0tDuY/hqS4fxdpAlDgi8iHzLj+MV9Ng8V86WEMkXjHR2ljZCbuLqMfxivoodBxzRNXsKDtcUmk8wCmnOOBUZyTSURORJlSc0uVxUADMeBk0bucGq5Bc5PlaXK/SoA1Juo5A5yxvWjzF9arbutG7mnyIXOWi60b1qru/wDr0bsfSjkDnLW9aN61V3cUu+jkDnZZ3rSF1qvu96Tfk0ciDnZZ3rmk3r6GoN31pC9HIHOyxvX0pN61XLe9BanyC5yxuX0qOV129Kj3n1qOV8gUOA1MkEibl4/hpodNucd6r78BW/2TQG/cn61HIh87J/MX0pTInl9O1Vt2dx/Cjd+7x6ijkQ+dlhpUCYxjNIkibuh5FVWYk+3rTgwWUfTFCgHOyzNIu3pThIpA44qpK3yU7fhc+gp8iJ52SPIpIXb1pwZBjjgdqps/II6Z9am30+RBzsfOUcAFRg8GvNoraeDxBb6bDBNCPNJZnTaSgPBGRzmvRJG4BAHHY1Unso4pEvowi28IwEUcuzH5iT6jg/hUy0RrS956l+eOYWojg34bhismw4/3u3865rRtOnOtSHZDGm75zHcPMSvuzd66h71LS3eQsGCLnav8R7ViPrRSN3j1CwinzxAyhlA+gIOfesU0dai9ypqkN5FrckWZmVDgGOfbhTyCV6HniugjISBEGPlUCueXWZr2fzrqO1LghcRHOPfNdCqkoG74FOjZtmeKTikV5Tlzjil6jB78VIYGfc6cqo55/wA+lJj5PwrblORM8Qm0vQ4ruaJ2td6SMrAyAEEEj1pV07QO32UjrxL/APXrP8S+GJm8T6myMQr3MjKNhPU5/rWSfDFznhuv+yahxj/MaqUv5Tqho+glRxBtPTE3X9alTQtCckAR8df3xrj/APhGLofxj/vk/wCFMbw5dr/EPyNLkX84+d/yHdnSNExgAE4xxKTXmk3myXLx5LFSQAPb/wDVVx9AvFGcrj8f8KzJY3gnaJ/vKcGtaNNRvZ3Mq03K3u2NW1sraXTpPNt7gXe/CNn5NvuOvrQdKjwRmQH6j/CqdhY3N95nkMP3YyctirH9i33H7xf++jVt2e5mtVsbumaDod7LHbsb4TMvJyoXIHOODW2fh/oythrm5U/9dF4/8drio9J1RX3RS7W9RIRU62GujG28kHGP9eRisZKTekzeEopawOw/4V5o7Di8uh/wJP8ACsjxR4LsdF0Y3lrdTSOrqpVypGD9B9Ky/sniQdL6fj/p5PH60k1h4iuYDBNdyywsclGuCRn6E0oqaabnoVJwcWlDU5zHy/jXqOn/AAu0++0+2uxq0w8+JZNojXjIBx1rhh4Z1TBHlrjP98VrQQeLreBIYb6dY0G1EE/AHoParrScl7k0jOjFRb543Omk+EluPuapMT/1xB/rUDfCOTkjUZAPU23/ANesR7nxvZwyTfb7oIiZZhN0A61mt458TM2W1i4Y+5B/pWMYYh/DNM1lPDreDR05+Etx/DqOc+tuf8a9Y+Emg3HhrSdQ0+abzVadZlOwrjK4PX/dFeNab4g8f6lb/aLG6nli3FdwjjxkduRXp3wo1DxbNrl5D4iMjW7W+YiyIMOGH90ehP5VEvbXtOaY/wB01eEWj1ys2dNkzDtmtKql2n3W/A1rSdmZTWhVxmnCiiupGItTxnKfiagqWI/KRVIhktFFANWSLRSUUxDqUU3pS0wElGY29cUincit6inmo4+I9voSKTADV3TGxK6+ozVM1PYttul9+KmWw0aeDmipNtFc/sx3Phi2X5ohkfeGB+Nd3og33koGANmefqK4vyHtrxYpUaOWNwrKy4IOe/pXcaEo8+ViRwoGMe9dNPciextFCo+8CDxgGmjhs9Oc1Lgc5YfjRtHJ3D866DAiIOcY/WjrjipWVeDuH+NMKj1H4UAWbPGw+zCrJIJ/lUFooKMM55FWShGOKzluax2Gd8/570hyDkYzTyMgDFNOAen+f8ipKIznGaRueOMDtT9pK8g9cc0yRxHGzuflAyaQzm9ROPGvh4cZ+0xk4/3xXuykEV4HOr3XjTQyTsaS5jG4/wAPzjFe4rHcAf67r/s1zVHdm8NEXTKxjMYxtPWmFeCKr+VOf+W36U/ypsH98fyqUDHrK0GSuNx45FVtxJJJ5NMkhn3czHGfSomik/57tWiM2WN/vRvyODXKeJ9UutGa0ZLlwshYMMDnGPb3rBPjG9z/AK4kd+n+FFxpHo5fk0F686Xxfc952z9B/hS/8Jddf893x9BSuB6Jvo3jHWvO/wDhLLnj/SJfTjFH/CU3P/PxKKOYLHonmYo8z6V53/wlN13uJfzpyeJLp/8Al5l+u7pRzofKz0LfSF689fxPPk4uJv8AvqmjxJcHn7RN+LGlzoOU9F38UheuHg1yR8bppv8Avsirg1MuARPL/wB/DUuqkUqbZ1u/PSkL1zEl6yxqfOk5H/PQ1GLtmVj58/H/AE0NT7dFexZ1W/nGKbI3y5rk2uXfhbqVTjjMjEfzpkF1Luw1zK5H+22PyzSeIXYaoPudKXJRfypd58tuvWueW6fGfMbk4yXNMub5kYgSt17Oaj6xHsV7BnSK5O7r1oVm8scGuT/tJgf9ZIf+BmnrqZK8Fj77jT+sLsL2LOnUsX5HA60rORKCAcVyU2pkDBZj9GqodSbPLt9Sx4oVddhOkzuJZDs7/hTTIW2oB27VxB1N/wC83/fdRnU2xncT/wACq/aoXszupHI2ipd59D+VeenU2yPr2NR/2r1GV6/3u/4U1UF7M9Cmf5QMH6VheLr7ULHw5Je6d5onjKqwU/KUbg7l6EDrzXMf2qWA2FMA8nOc1W07VTP4nsoLiPNuzunzE4LlGwfbB/ofprTXPJIl+71NnRvGkKSpp2py7hkRw3UqCMS4HRlBIVvxIPseK7JILeSB5bcWyyNks6HYxP1HU1474i0B7JftAWS6sJm+4+A8LZOcuePxPX19ct9cuINIll0zUTaCGTy/ss9yHcjpkKVPH0bjHSnUw8ZXlB/I1hinC0Zo9M8Q6hYeGLWbUnSGS9kO2MMxZpG7AknOK7PTbmS80m2uJ4BbyzRK7xE52EjOM1856bfvNrUWpa002pwWkqlwZTgjPGMjpnGRxXt2meO/DV+FRpms2PQXSBVP/AhkY+pFZQpciCtX9q/I6CSRkwsbMMjBx3qPcdmKe1pHNGGhVHRhuBXGCPXPTFVltEBIKAEHBBFUzJHF66Fj1y5wAckEexwKoYI+YqME8ZNbXiO2EerjaAoaJT2HqP6VlLgdF3dgT0FebU0kz06SvBCLtCdCDjpUqqzfM4IA4BJ/lSLEW9efXpU4gZsKCXPsOKybN0iPfGCp2MQvqePrivKfFyY8TXTdd+1s+vyivZoLWSH5wqBhzucBsfnxXmHxJjf/AISGGduTJAuTjGSCR/hXTg5/vLHNjIfu7h8OU8/V76zzgS2x7+hH+Nd//ZDrPHKkiEqwbAXjO0Aj+dec/D9tvjOCPr5yOv8A47u/pXt8FiXYfLkDrWeOly1B4OCnT1Maw08R72nij8x5C25RnAwBjmtODS7aRG3QoOOXZRj8PWtRLQn5VACnsvQ/41bSzRAN2Sf7orzZVjuUIx0MZNFtJDxAhGeWMYqdNEskbBtoHOOMwittYScBQRjpnrVmK0I9j+prL2smJyitzGi0OwyWksoST2Ma1ai0OwPSxtsnqTCv+Fb9tpbyHO3j16VLPPpul/6+UPL2jXk1rGjUkrt2RyzxEb2irswJPCVlfWc9v9hgPmxsnEYHUHvXyrq+kT6Pqs9lcgLJE5jYA5wQcGvsX7Rq2ppts4BZwHjfJwfwH/6q+UfHttNYeM9RtbiTzJYrhgXP8XPB/EV62AjyNpXaOKvJyV5Wueo/CDULzWfCdxotppNqRbzDzLjaASG5+Y85PB/SvUtH8JNpUjXs90zzYPyINqDPt3ryj9nLUhFc69p5I+ZYpl/Asp/mK92muNylaK0acKjb3JjVny8sditUc67om9uakpOtaxKZm0tK67XIPY03rXWjnYo6VLEecVHToz8wqkQyejNJmjNaEi5pabmlBoAWlBptKDTEOzUUZxJKvuD+YqTNR9Ln2Zf5H/69DAc1PgbbMp9DmmGli+/SYHQUVCko2Ln0orMZ8RySySgTSyM8hAZmc5Jx7/Suz8NOJInlH8SKa4x1/wBFGMZ2c/lXT+DZN+mMc8ghP8/nWlPcmex1O4+uKQt7cZ+lR5OcDrSE5GfTjHrXQYWJlbKkehz0pueuO9RglSeOvFB4buaQy/YyYdkx15H4f/rq6WHTuB1rFhk8qVG9D6VrjpgHI9fXpWcjSGw9WAPTAphwc4HvS8FR/Kg9ep5qSxB15qtf5+yEY/iB/WrBGMc8VXvMG1b04/nSexS3OZL7fGegt6XUf/oa174or5/m/wCRt0Pn/l5j7f7Yr6CUHFc8jRDD8uAFyfX0pecGnlaCODQhMiRow375Sy+gqqRycDjPFTkcmmEVoQ2cH8Rbd54tOVWI/ePnH0FcQNKnOMvJ+letaxYLf3NnG3RSxH5Cki8MxFg23j2FHJcXNY8qGlSn+OT8Mf4VKNLkA+9J7f5xXrJ8Lw8YU8e1PXwxEP4Kfsw5zyUaXJ1zJ+OP8Kd/ZcvpJ+f/ANavXV8NQA5CU7/hG7f+5S9kx+0PJYtKkZhnzfzqzNo0kTFY1kK47t1r1NfDkA/gq03h+3PzFB0qXSdylUR4u2mTDqkn51LFpMhYfI/5162+gQZP7uhdAhH8FP2LF7RHAWHh+OQAusv/AH2RW0nhuzOB5Unv++b/ABrsbfSI06LirX9nJn7v6VhKhK+5qqyOPbQLFFQeUzd+Zn/xpr6HZrExEEgI4x5r4/nXWz6axA2r2xkV4bqdl8UTqV0tquoG3ErbCNmMZOMcZrP6vLuX7ZHdHRYd5KoyjH/PRv8AGq/9jxliQDn2dv8AGuFXSviVKy+amqnb833kH+e9TvpvjtruB1ttVt0Vhv33O4Nz3z2qXhn3LVddjsU0yJcfL0PA3Hj9ajvNNUNjbkf7xrpEgJjO5RnpTbu3AzlQAKx9i7mvtFY5JdODNjy8/wDAjVpNJhKD9znHua3FthuOAMhatJbjyxx7UpUpAqiOOudKUcCIA+9Z/wDZq7gDEvJ/u121zbAyfhVFbQSXWRyF561UKUiXNXMIaLDtG6FCT/s0f2JD/wA8UyO+K6pbbdMMjAA4qlrkv9maRdXS43RxnBx/EeB+taKmxOSscbeSaXYuY2eHzAeUABOe/NY1zq9oMmKGPjjIQHFcpe3bCVmLEkccnrn/APXSI/mRknP064+ldsKCW5ySrNmrcaxKzrgoVBIIPB/PtTLW9dNWs58hTHOmD2wG5/n+orNVnjbg8etPC7AWyQ2Mgjgj0/pXRBKLTRk5NnsWoy6fp93Ml5eWqRyqGkhldd31C9efpXFXPhPR7tZdQtEzBk4WGXMZI64x/LIrTsdFtbmyR1R2MnzOzgBiT1zzk/jTk0q60ouNOZYzccSKMKvH8W37uR2wB1712VsDKcvaLS5UKy5eV6nLWtrZ6ZFc2t3a3DSXabWjhjBESngEnPUdcAc1y5lmgDwrJu2sVztIPHfkZ/OvUodHMJJlyTnLbmyST3JP0rzvxDZGy1m5QBtrnzF3c9ef6VnXoOEEyea7LGieK9X0GXdY3bxp0aLrG31Q8fpXpvhj4oWur30VlqUQhu532pLGcozH1zyuT9a8Pc4Qc/Wrugbn8R6aBkk3UXA/3hXE0UmfQHiaBGvbeRyOYyMepB/+vWOttkDJVU6DH+ea6bxDAzi3cLnDMMk8dv8ACsWK3LHAG4+grxsQ7TZ7WGV6aI4rdekYBbHcZNW4rcx/IdpPXJJwBViK2YLy+D02r0/Gr9vZlzkgkexrjlOx1qJSW0DHG4Mem5s/oK84+Len+Q+lXI5V1kjzj0IP9a9ijs9xAC4x6Vwnxk04Hw1ZXOSTFc7SR2DKf8BWmDq/v0YYtJ0mjyzwXJ5PjrR2Y8POq/8AfXy/1r6YjtRtAHTrya+X9HY23iHS7nB/dXEbn6Bga+u4LFnxwcVvmcXKceU4sLV9nB3M5LUHoTj2qzFY5wABitlLBI13SsFUeppBdxqdlnCZn/vdh+NckcI1rUdhyxTfwleHSzjLAKvqac1zZ2reXAhuZv7qDNTmxuLvm7mIX/nmnA/GrcNvBaptiQKK7aeGtrFW83v9xzSq3+J3MxrXU9R4mmFrB/cj5Y/jVmz0ew08744gZT1kflj+NW3mA71VkuPerlKlT13fmTzTkrLRFp5go4r5M+NFt9n+JWouB8sojlH4oP6g19QST+9fO/x5t8eKLG6A+WW0Ck+6s39CKeGxLnWsxypcsGyh8Er8WfxE8rOBdWskf8n/APZa+jZJ+etfJngC9Nh4+0SccZuFjP0b5f619SPJx1rnzRuNRNdUb4SPNE2VO5Vb1GaWoLR99rGfbFTV103eKZMlZ2KlymH3etQ+1XLhd0X0NU67IPQwmtQpynBzSUH2rQzZNmlpgOQDS1ZA6ikzRTAdnmlzTM04GmIdUch2vEffB/KnZqO44hLd1Ib8jQwJTSxfe60080sX36ALm7Heik3e9FFgPkH7FK7i0QKZSdmM8Z6Vv+GdKutMguIrkKAzgrhs9qzopxFqSXDBsCTcQOvWt0a9bbc+XLj8P8aI2WpMrmltPvSAc89KoDW4B92OU/l/jSnXIGPKS5/D/GtOZEcrLuBijvj+Rqh/bNuf+WUvT2/xpv8AbUGeI5OfYUcyDlZo4rVtpTJbpknK8GuX/tqD/nlL+GKs23iKCINuhlIJzxipk0XFWOmB280044OTWGfFFoCcQT4HsP8AGmnxTanrbzfp/jWZpob2c88cDiq9+c2rcEZwOB71kf8ACU2o4+zzHn2/xqrd+LLR4Sgt58nvxj+dJ7DTRVumA8VaIRwPtUfB7/OtfQiivmuPVI9T8U6QY45ECXUfDYyfnH+FfSq4xWUi0LTT0p3WmE0ITISOaaRT2xTDVmZh69rVtoBtbu6XdGztH94DBK57/SqUXxQ0OMfNg/8AbZa6KbSrDVsRahZwXUSfOqTRhwD64PfmnR+DvDi8jQtNHP8Az6p/hWkU+hLaOfb4saEBwE/Gcf4U0fF3Q8dIv+/5/wDia6tPCnh9emiaaP8At1T/AAqwnhzQ1+7o9gP+3VP8KfvdxaHFH4waIv8Az7/jO3/xFN/4XBo5ztFrj/ru3/xFd6uh6SOml2Q/7d0/wqRdK05elhaj6Qr/AIUe93DQ8+Pxe0oDOLQD/ru//wARV3SviQuv3n2LTLOC4uNhfYszD5R3yVAruF0+zU8WkA+ka1WvNPtUv7K4Hl25QuCyhVzlenIpWl3HzLsZI1PxE43LoURGcf8AHyKUah4k/wCgJAMf9PA/xrZglsLSFIPt0bBc4aScFjk55OfekbUNNQkNfWw+sy/40/mL5GWl/wCI+p0mAD/roD/7NVgX2vt/zD4B+IP/ALPVhtV0gcHU7ID3uF/xqNtb0IEE6tp4+twn+NS4+Y0/Ib9s1/A/0KAfgP8A4ug3GvFf+Pa2B9x/9nTW8ReHxkHWNOH/AG3T/Goj4m8NqRnWdPH/AG3X/GlyLuPm8iuNQ1t7+e02WivEiOd3AIbdjHzf7JpkkWtyH5nssf71U18UeGYvFFzcNqln5clnEu9WyCVeTjjv81Xm8ceFF5/te2/BWP8ASp5E92VztdCD7FqvGZLLr/z0/wDrUybTdVlxiayX/toT/wCy1M3j7wov/MVi/CN/8KafiF4UGP8AiZgn2hf/AAqfYw7le1l2Ko0nVw3/AB82eenDf/Y1NHpOr7cfarVfxz/7LSN8RfCw/wCX9z9IH/wqNviR4XB4upj9IGo9jBh7WYT6NqYcIdRtgzgkADJOOv8ADXHQT+L2BeLw/fMp5DBosMPUZrrJfiP4WkA8xppMdAbfOPzrqtPdJtOt5o+I5I1ZQRjAIyKXsY9A9rLqeXhvGxORoF8pz3lt6w/Ftz4mi0Yrq2nTWtvJIFDySwtlhyB8vPb9K9yIB7k15X8ap9mnaXbBj87ySHHsAP8A2Y01SSYOo2jw+5l3yPH1HSktJAVwwGV688+lMmjZn2bvnA49/pVe3LLdEMCDgnBOOa2MzTH3gcnr074/z/WpTtKDaBgHPA71WZl2HPpgAdf89KsIAybgB82D78/5AqhHp3hi787w9bSQqjzJGFYnsRx/SrMksz3cRlVctuXAHsf8KxPBMhk0dgrBmjlZM598j+ddDdSLDCoklUyMwVDjOT0/Svepy5qaZlazGFURRjl2OcEkcZrhfHduPOt7sA4bchzx7iu+i8xo13hN2zDMuOTj6CuX8bW/m6M0g6xMHBH6/oazxEOakxp6nmE8ZUZ6rn0rpvhnpjal4705cjbCxuGyeyjP88VgTRb43xlsc10fwsultviBp+84EgeMfUqa8CWxvHc+gdZhEtsmQOHB557Vn2+ngoCDj6/4Va8Q6vBo+iXGo3EUkkMAVmWMDdgsBxk+9cVF8YvDiAH7DqI57Rp/8VXjYuhVnO8Eevhq8IU7SZ20dhcebGqlPJ/iyDn8BWpFaggAdO9eeD42eHVOGstS/wC/af8AxVTp8cfCqgf6FqhP/XNP/iq4ng8Q94lyxcejPR47UZGD+lc18UNIkvPAV2Yk3NC6SjHYBsE/kTTdG+MPg++SRpGu7bYMgTRD5vYbSf1xWL4p+Nej3Oj3un2NtdKZoWjDlFI5GOea3oYSUGpPc5Z1nLToeLWXk295btcAmNZMuq+mc8H/AD1r7C0+/wDt9jbz2MLeVNGrrJJxwRkV8Vz36zN8qlTnNe/eHvjv4Z07wzplld2+ofabe1jhl8uFSpZVAJB3dOK9WdOTaZytq1j2D7CJG3XMhlPoen5VOPLiXCqAPavJm/aE8J44ttU/78p/8XVZ/wBoLws3/Lpqv/fpP/i6zacPghqTZvdnrz3AA61WkufevJG+Pvhgji01T/v0n/xdQN8ePDTdLTU/+/Sf/FVyVI4mX2WawjBbs9Ze496rPcV5U3x08Nn/AJc9S/79p/8AFUw/HHw4f+XPUv8Av2n/AMVXJLC4l/ZZ0RlSXU9ReUmvHfjnbebZaVcgfcaVCfrtI/kavH43+HP+fTUv+/af/FVy3jr4j6H4o0WO0tra9WZJd4MiKBjBHZj6itcJhq9OtGUo6Dq1KTptJnmun3Js9StLpfvQzK4/Ag19bAlwCuSGGRivkDIxxnOeK9lg+MlhHplrbvHemVIUSTy0VF3BQD/FuIznuK7sxw063LyIywdaMLpnuOm5FqUbqrEYzz61crgPhh4us/E8OpJaxvGbdkZlZAv3s9ME5+73rvqKUHCCjLcc2pSbQjDcpHrWe3BI9K0TVGddspxXTTMZjM0ZpuaWtzEkQ/KPbinVGh6j3p+ferRAtLmm5pc0wFzSim0A0xD80jjchU9xikzS5pgJG26JT7VJF9+oYvulfRiKlj+9SAu9u/5UUz8aKAPka/ZoLV3Q4dQDnHvitNPDmqMgP2+AZGf9X/8AWrP1VS1hKMdcfzFem6Pol5q52WcDSsi5YKBwKIpPcmba2OIHhrVcj/T4Mf8AXOpF8L6mcj7dDn/rnXZy2kltcGGZSrqdrBhgg12Hh7wgNVtTOZNijgZGc1bjFakKcjxo+GNVI2i+iLY6CI0knhvVU4e+jUkcZhr03X9FbSbpot3TuK5uXLHkkkjHPWhQTDnZyR8O6kcY1KLp/wA8qP8AhHdRbCnU4hg/88R/jXUU09OBzT5EHOzEHgzVm4/taLn/AKY0q+C9SOSdYjx/1w/+vXX2r+ZbK2eR8pqyqgHn8ayasbrVHEf8IRfk860n4W//ANeoLnwNdpEXbWA3QEfZsd/rXekZByar3/8Ax6OxxwV6fUVL2KSR5dp9m9h4vtLZ5vNMV3D8wXbnJB6V9Rp92vmiQ48exAH/AJe4Dx/wGvpdBhcVlIa3FpjVJ+FMakgZC1RmpWGajPWtUZsqXmqjSESZoWl3tswpAxwTn9Kq/wDCbDOPsD8H/nqP8Kr+KD/oEBzj9+P/AEFq5jODW0NjKb1OxHjnH/MOb8Zv/rUp8eEDjTvzm/8Asa4zd+lGeaqxHMzsv+E8l/6BqD/tsf8A4mmnx7cHpp8X/f4/4VyGe1JnGcUrBzM7D/hPLr+Gwhz/ANdD/hWF4n1u68UWlvpcsMUEUs2WdGJYYBPf3FZoY9cUgb/TLQ56Sf0NNoLsxpvB9vA4U3MrZ6HA/wAKaPClp/z2mH4r/hXUal99M9xxVFgeNpxUuESlORjDwrYjrLORnuw/wqVPCmmkHLTn1+etA7h17VPCHJzn5RQ4oFJmavhPTR/z2/7+U7/hFNM7pJ/38Nawb2p+78anlRXMzA/4R3Tk1Rk8uQx/ZQ+wyHGd5GaZeaJpsUO+O3xg/wB88/rWu/8AyFE6/NaOPykX/GorsBrdxjoKFFWByZx7w2sWqCN4wIthbBY8frVYxLKk8kJ+SPJBzyT/AIVY1bBnhiUjdK4X8B1qGABU1KPngnj2xU2KuJciNdMhmVcMSMmpr+FFtkaFAPmBOPTvVOV93hmM45AHfHenancj+zY1z8zKDgHnFFkFy3FEk80cpjCwlwFA/iH0rYXxdr8GsS2CarcLbRIBGikAKBx1x/nFZrSBbWIj/ZOR06VFHEt14n1HqGiY42ntk1N0ilqauo+K/EEUtrs1q8VWfDASdeRWH4o1a81KaJLq7muDEpI82QttyecZrYl063mCO7t8hyvPT/PFcz4hjS1v5FU/KI1Y5OSSf8/ypRmmxuLRhzrAqb5S2euB361UgbfKWOQoHA9OacIZLl9zHapPAz/n/OKSI7L1lJB+XHHatCSxJwCSQM+n+f8AP41LalPJdSB8h56dD0/rVd2yMk8dKS3lCXOMjDDBOOh/z/OgDt/BVyqG5iZipeQNknPVf1ziulu/31woTOI2BGB79q5HwfLGlxdqyhmCqykDjOT+XQV2KulraiSUfO5HB9c/5/KvYwr5qaRnLRk7NOWwowq579enP+fWsfxJHI+kyRuoDOhPOa6CMzRfJLjaCfmHb/PNcr4ne71EywwOY0QHkf5/Wt6ukGT1POI2OwNycZzntWr4OtpJfG2lRwA71ukcEd1Byf0BrGi/dM8Z+8pIP4V1Hw9eVfH2ltHH5mWKt7Dacn8BXz8tDdHver6QuuaRdaY0nli5j8vft3beeuO9clZ/ACBsFtYkZfU24A/nXoVpJ5dzGwCZB/j+7+NdEbG5uzm9uwI/+eUPAP1PU159Zz5lys6YtJankep/Bnw8lgbW21C6m1HIC+Uike+R2/Oqth+zqsw33utyRA/wJCCfzzXuEMdrZpsgjVAPQUrXI9azVX2espfJCb5tEjyFv2eNJUfJr18repiQ1RvP2e7GO2ldPEF0zqhKgwLgkDjvXsr3PvVaW4yCCawqY63wlRpye58UXlobR9pO7OR0xzVSt/xNA0OrXUJXHlTyJ+v/ANasCvXpy5opmE1Z2CirllpV/qRYWFnPdMvLLBGXKj1IHSr6+D/Ej/d0DUzjr/oj/wCFU5JbsSi30MSitUeGddMPnf2NqHldN/2Z8fnipIfCXiK5GYdD1Fx6rbP/AIUuePcOSXYxaK6D/hBvFP8A0L+pf+Azf4Uh8D+KB/zL+pf+A7f4Ue0h3Qezn2MCit8eCPFB/wCYBqH/AH4b/Cqmo+HNX0iOOTUdPuLVJCQhlQruIGcChVIN2TBwkldozeNo9e9eg+CPh1aeLNCfUJNSlt3SdoTGsQYcBTnOfevPa9d+DV/ix1WzJHyyRyqPqCD/ACFZYqUo03KG5thYxlUSkd58NvBUPg/VbwwahJcLdQgFXjCgFTkHr7mvSjXI6Pcf8TiEZ4OR+hrrjXHRqSmry3OqtTjCVo7CVXuk+UMO3FWKbIN0RFdEHZmEtUZ1L2pSMHFJXSc4qnDn3H+f51JUI/1in1yP8/lUmatEsUngn+VRfaRtyI5OuMY5pXDkjY4A7jHWkVZs/NLn6LVCE+1f9MpPyp4n64RjgZHvTQku0/vfm7HHajZLx++IHfA609BE0b71ztK9sGn596rCOX/nufyqcUAIvErj1ANTR/eqAnEy+4IqxF96gCzz7/nRS4NFAHyZqeBbMeOWGB75r1Lw94kvNBLSWhTDqA6uuQa8q1U/uOn8YHI969F0yMS3cELMQkjrG5HoTShs7kVHbUuXepTahqEt3cEGWRtzYGBXRaF4vn0qAwoEdD/e9a5XVFMGpXEDPvMMjRA+ynFV/MKn6Vro0ZJ9TZ1jWJNUnaWVgWY9RWM5zk5z6980by3GeaQocU0BEeM5IxTSMZ5HvT2XbnnkVGdv5mmMt6fJtmMZ5DfzrV2ZyR2rn1Yq4YHBBzXQQuJIVkHRgD6VjUXU2pO+gbT36VWvFP2V88gEH9ausMj0AqreYFnLweBn9axb0N0tTzabb/wncRH/AD9w/wA1r6YXpXzPOCvjuIsACbmHp/wH/wDXX0yozWbBbsDTD7VIRTGpoTImFRGpmFRtWiM2YPigf8S6IjtMP/QWrkyD3Ndh4kTfpg9pQf51ymzjitoPQynuRYI6n8KO3J96k8vJxTvJ5zxg1VyLEBHvS4PepfKJOOtTf2beHpaT89/LNFwtYqdxzRj/AEi35P8ArV/Xiro067720oHupFMktHikt/MjKFpUxn/eFDulcUWm7FjUVDMhyeh6VTaONR8zYz7962NTjRBH9D/Ssl40lOD0FTe7LtZFbMW77w/A1ZhaEpgv85PAqE20YI5/WrMFvARv5yORzQwQ/wAlSM80ohQ8f1xTwQBQW5PPFRqXoVVhRtcsEb7jpKhx9UNdp/whumSIys9xzwcsP8K40uF1fTnHaSQZP+4D/SvWEwcEbsnrxTQ7I4//AIVz4clmSR455JVPysZWGPyFcWfDOn2/xVm0VoibG4jRtm85OUPfr1FeyDaJQoY8mvNfEDLb/G3SnJx51tGv1O5hQxmyfh34XSD7Oun5iHVTM5H86uR/D7wq6ANpEDA9d5Zj+HP6VuBACfl3c9xmp4yFlCjAUH2osM+e9at4bXUr22gGFgmeNV9ArED9Kowyi28TahL3aJX+uVH+Nbvi3aPGepwbct57HIGM55/rXNao/ka6zfKBJbx9+vyj/CuWTvdFxVtTVa9LLt2Y9cVx/im73XxVX+Yqu/H44/nXTq2BweK4/VJLeC9meRS87OTgjJ69falRjrcuo9DPigJXzZ5CIwM4BqK1/eXDFeCxwB171FcXMsxwxwvXaOlTWKqdzPwo4PFdRgWZQG6HOevP1pkSKGBBOe+MHildzIxCdD0GM8VYt4TtDNncRkd/SgDoPB7IuqTqRjdErDkEEg//AF67NwlzcKjA+Wg5x3rz3Qz5OpYUoN6cZ+o6e/8AnivQdKxJy/CL1z3/APrV62B1jYzmbHnJLAjSADIBP+PSsO7uUbdGqBmJPOKtLDcT5dcrFIBtUDBUfzqtqb22i2Blk5kI2onGScdv8a6Zyb9BWVjyzWIBZa3PGD/FuPPQnnFegfCTSrO/1ue/k3+dYxholU4BLZBJ9cAdOnNeaX9xJcahNJMRvZucDivQfg3qCw+KJ7RmAFxbEKPVlIP8s14NXVuxtA9xI7ZxW7DqHmW8bg/eUH9Kw+lRWVyRB5ZP3HdPwDED+VeJmLcYpo78NBTbTOi+1Z70xrj3rNWY+tKJGPTmvFdSR2ewSLpnJ71E0mc81Tury2sYvMvLiOBPV2xn6DvXNy+PtOkuhaaZG1zOQcM/yJwCfr29qcadSeyC0UeM/EK3+z+JdVT0uWcDH97n+tcMetdp4y1CXVdWnvp40R7hVfagwOBj+n864xhhyPevqcNdU0meVX+Nnb/DHxDNoGu3RhhWU3FuUwexDA56/WvfNJvtQ1nSra4kMUayghgVyVZTg8dOevTvXzN4RZh4ltVVsF9yZ+qmvoLwlLLbQPA0zPGDuUEcAnrj8hXFmFNX5ludeE1Vjql0+AOJJi07jo0hzj6DoKnLqowoAHtVcz8ZzULzD1ry7tncqd9yw0uaiaXrz2qo9wB+NRG5G4DnrQos0UUiy0tedfF2Pz/DFtKASYrtc49CrD/Cu0e59q5Px+PtXg6/X+KMLIPbawP8s104a8akWZ10nTaPn8ffGfWu6+FV0YfEtxCTxNbMMepBB/xriJhtnce9b/ga5+zeMbF84DMyfmpFe5WXNTaPEovlqJnvtnP5Wo20hPSVc/TNeiGvJTdFSCOo5BFerwSie1jkHR0DD8RmvOw6tdHo4jWzFoNL2ptdKOZlORdrkYplT3QwwPtVfPFdMdjnkrMDxg+4p5/Wo2OUOPSn5z+NWiGLRSdTRVIQtLTaKYD80Cm0UxCSNjY3owzVmMfNVaXmJh3xViE7tpHfmkBb49T+dFJzRTEfJl8pbykHOZkGfxFd9GywXCMrE7GDZxjpXCXAzc2o4+a5T+ddtld3U8c0oapk1Fcv6xPFd6zeXNsS0UshdTjHXk/rmqWcD2pyMintjPXFNJ+c7fwNaLRWMkrIu2ccbTLvIA75rsvFOi6Np+kW0tlOGnfHyhgdwx972rgVlwcg4PrUj3DuACaGrsaIpMZIFREGnls44FNbkmqAYc561r6XNuhaM/wnIH1rJPPXt71YsZhFdKScKflP41M1dFwdmbpBIzVS+YfYpSRn5T0FXwvPSmz2UU9tMXheQhSTiQrkY6VySdkdijdnlt8AvjmLn/l4hz/47X0yn3a+b9daL/hYMfkwCBfOg+QMzf3c8k19IIPlqb3SFs2OIqNhUtMYU4ksgZcc1Ec1Ow4qMitEZsxvEP8AyC245Dqa5Inj3rt9UtJLyzaCLG9iMZOO9Yq+GNQbvCB/vGqiyXBvYw1ZuenHbHvRyeT29q318J33eSEf8CP+FPHhG95zPCPxP+FXdB7OXY50HBHP44rtUjvJdQCRp+6Me7/VnHTru9c9qzh4Oue9zEP+Amu0hUxQxIckhAMg+gqWlJicHbU5aayv9+0o4X/drJ1uN4zZGSNlbzejA/3lr0QHPQH86xPFGnR3kUEjs6mEMwx+B5/Kuidbmhy2MaeG5Z8yZy2uDasZxnrWEZOeBXb6jpMVwE3u4Hsazh4ZsuCZ3XPOGcD+lYqyWprKLOZJyM0gcgYrp38O2gKlZCVz82XPT8qkg8OWkz7ERmPX7xFaqm2r9DO6va5ym9uTxR5jdCeK7T/hDkP/ACxUfWU05fB8QGPIjP8A20NLlXcrlZw0j/6Tp5J6XBH5xv8A4V61E8zKCqpjAxkn0rg9W023s7zTlSNVIvUDAH2Yf1rr4X1NxEYo7YwlRguxzjHoKya1NUtCw7XrSZHlAdvvV5z47c2vxR8K3BQb3ESMe3EnP869Bk/tYEBDaKoGT8rVzfiexFzqWm3eoRW0s8Tf6NKqsNhyD0P4VLVluOMbs6122yOMd6fCqn53wBn061nNa6uSCL22B7/us/1qVItUCFJNQhIIwAsAyCe/Wk5tdCuXzPIvH0YTx7fuI2AYKckdcoK5HxCjfarRowdv2dTyfQsOteweJNKU3ryXUcc9xtGZTEATxxXGG1SW0gLRRuNpU7xz94+v1rk5veZsqeiOXuZWgsZZo8b1XK+ma45/tJlaWeRMk5ZioJ/OvUfENkB4dnCxoh2K25RyBkE15xc2rO6IVOxeenU10UUrGdVWZkuY5XZkhOOuc1NaqDCQV5HaprgxxYVUwzcAdM/4VVtJBu56Z7VqZF1It7ZbhR25qyeT7Yz6VAshyOQPXnmpR0C9F7Dt+H5UxF3R7WG41KVrhwqKoKkNg5ya7211CyjHlp+8XjOehFeW3UxiuIsSyIGB+ZBk4/H6fzqRbyQkf6XdtjDHkLjn2/xFdVHF+xjZIThzbnr8GpxmJhGoLqCFQdTzwK5fU7CWVmvdXvrW0U/dWTJwPTHrzWLBdLdxX0UUMMU0do00EkckxYsrDpudlPBPYVy80d3PMWuBLJu+UOxzzjI5pvMfaaNFulyrQ0dZtNMZZprS5mklQjlowikfTJNVvDGr/wBieI7DUMHbDMrMefudG4+hNZ0tndW6HepA7gHNVlODXLOSm72EtD6+VlcB0bKsMqQeoqpDC/n3HZBLnJ4HIB/rXPfDbWDrHgiyZ23S2ubWQ5/u4x/46VropNMN5cl3ndYRj5FPevJx8E6ep6GDlaeg6XU7K0wGcyuP4Yx/WsfW9Y1f+y5bizhNpFHyWxkkZx1P17V0lrp1laf6uFd394jJpb/yp7WSGRFZHXBUjINeRHki9Eeg7yPF7lri/ZmnmkmlbnLEmiy02a3uEuB8vlnP510k1rHbzsqoFAPYVXlT/R5B0Ow/yr0o2toYNanA+M9O/s+5gQEsHhyM/U1xEn3zjp9c16d48QTWOlXGMlkKk/l/jXmcow1d+Hd4Hn4hWmXvD83keIdPk6YuEB+hODX0RpT+WT79a+aYZDFPHIDgowYfga+jrKUMqMucEZrHGK6R0YF7nQi5+X9Ka7SFN+1ivrWZI87QObbaZQMqG5B9qrzXOoRTsUZXi/ikfapAyC23nkben19q8zkPTuaLzYqLz8HI9c1zn2zU3QjcqlWIdyyYQbup9DjH5niqzX2pkbkmh27QTudMZ4zjn2P51oqZPMdNNJh26YyaytbT7Vot9bkZ8y3dR9SpotJp3iLTtuJY7G45XjH3eO9PZ+CDjB6mqirMmTurHzzcD95n+8Aat6RP9k1uwuD91JkY/QNzUV/CYLhoj1jZkP4Mar5OEb04/wA/nXtbo8LaR7y74LDivVfDk/2jw9YyZz+6C/8AfPH9K8atrv7TZQT9pI1f8xmvUvAtz5/hxUz/AKqZ0/8AZv8A2avNp6SselUd4pnSk03vSnrSZroRzsjuBujPtVI1fPIxVFhtcitoPoYzQn8qRP8AVrnrjFHShehHof8A69aoyY4GjOaTpUF9LJBY3EsIUypEzIG6EgEjNUkIs9aK8t0z4jatfapaWskdnHHNMkbMI2yAzAEj5vek1H4iaxaahcQQpbNEkhCM8DKWXPBIzxW3sJXsZe1ja56mDS5rB8J61Nrugx3k4jE29kcIMDIPH6EVuZqGrOxad1dD+1Ps+UUf3ePyqLNPs+JJF9Gz+YqRmhx6n86KM0UxHyhMc31l1wblOPxr0C00a/vbC4vYLYtbWwzLIWUBeM9zyfYe1ef8tqdgP+nhev412X2iXyvK8xxETuKBvlJ9cetFPYmY8SAHnP4GnvJFj5V/E1Xz7mnKA2Mtjn0rQhC7hzk/QUA5J/SnvFCIyyzAtkcetRY6d80XBqxqRaJfXGjyarFb77SJisjqwJX6rnIHPWsxmJ69PQ1JG5EZTzGVG+8AeDj1HeoCeTxQgFz3poPOaCc59+nFA6etMDrLFvPtY5uuVGeO/erUsLi1k4JO08Vl+GZQ5lt2xn7657+tdYbVmt3PfaTz3rzK8uRtHq0I88UzxLxCrDx/Hv5Ilh49Pu19JR/dFfO/jCMJ8Rl68yQE5/4DX0QnKinF3imYzVpscaYwqSmNVozZCwqM1K1Rkc1oZsIl3ToPervkA9KgtFVrpA2T1/lWqI19OazlPlZvT2KQgp3k45zV1oQrYxSeUCelL2hpoU/KHPQ1Jt6DngVP5R7CsHUzeJqsMMFs8kTth3GQF6fgKcarvZK5lVStc2QBnkgflVLWQHs25BIU9D7Vi3tjqBB8gSEn3P8AjUlhbXccdwtzC6jy/lZh1613Sp/uue5xU6zdVQ5S7crmNDjrTWsZ7iG3aBGPOCQQMc9TnqB6Vbdd8EWeeB1+lXrIAWKfUiuVWnozrqRsjIu9HuicxkkYPRv6UabZXdrckzRERlcbye+a6AAYzgmknH7gY4wwP612Ks1DkONUI8/MZ4Y5OWz+P/16RW/e/fGB71E0b+awWLI7c0qxkPzEB79ayubnI+JG3aivA+S9hYfi4H9a6yyLCzhYKPuAc1y3iBN+pToM5EkD/wDkRP8A69dRZDdaRKc/KWB/AkVhCpebRtOFoJliRiV+bavbrXPeKwv2ewkGTtm6jtXRTAbVG4g9eKw/E426Mj4J2TKcmiv/AA2Kj8aNiQcq2eopVI7kk/SmpJmGJgfvIDjGe1SLIxxwfyrRbEPcxfEVvm5UjoyDOa4fT7TzYTHlU2zSKT2HNemalCJhGxQ8DFchp+lFLy9gkV+J2ZSB2IBryK1XlqyR6FJXhEzDYvPayRlFZTGcq2AGGOhzXialkyTu2r91QM19HvpanIKuePSvn+/sHstau9PVHLRTvEF2kkndgYFdODrKbaMsVC1mc3fRM37w8MOeO1U4MbgMD612d94J8SPbGQ6Ff4C7uYGB/LGa4tUKuyvuVgcEEYIrtumcdmtzViICjjrzjOetSmQIgZjx0b+VVYpCyYB3Zxx1qwmGjOSffjH+f/11RIiQw3sqs6vIANvBxn/Oa1o9I07KsYsDrgntVGwRAeBjOTtGOn+Fbax+ahAYqccAev4Vm2aJGnJe3ap9ne7u3ixjy2uXaP8A75Jx+lUJnkcFFUM54OQKsmJXyShJPOC/4/4flVy2tVi/eMoU44xngCsktTS+hzeqad+7SFpGeWRtzEj7oHOP1Fc1qmnmzfI+6fbH+e9drd3f+kSCBPOuGGPZB/Ssa80S4O+WabLMMsoHArVMzaOi+DGsm11270l2/d3kW+MH/nonP6qW/IV7gr7Dj8a+W/C1/wD2P4w027LYWG6UOf8AZJw36E19QSq24bQTjrXJjY3ps6cG7VETGaqd3P8AKRSq2W2d/Qdap6iGjxkEZHcV4sY3Z7GiMC/X/SGI781nMy5KkjBFUfHUM9xp8Bt1kdhICQgJ4wa4M6ffsebS5JwRnyzXqUqacbtnBWqOMrJHTeKlE3hOykHPlTFePTn/AArzC4BDnPXJFekyQS/8K/nhkRkeCfO1gQeo/wAa85u1xK3Hf0rpw+l0cmI1aZXHWvfdAm87RNPk3ZLW8eT/AMBFeG2Ol3morMbSAy+Su+TBAwPXmvX/AAXciTwnp5JJKqVOe2GI/kKWKV4l4R2kzr4ZVBwW25HDY6Vm3Ol2cmU813UliVAAUbiCR09Bj6VIZTuI471C0hIxnivPselzCW9tHaFirs24BRuxwMk/zY1W+wQIylXkBVdo5HdVX09FFSlx0z+NQSXcEed8qfLjPPrVWZLkieMiCCOJMlUUKM+n4U1pCe1Zt5rMNsNoEkjq20qo74zVe11c3F4YGjKDJ2nOckdafKyXNbHmnimAQ6/fKOnnlvwb5v61j5/dY9D/AJ/lXVeO4PL1p5Mf62JHz9Mr/QVyi52sPxr1KbvBHkVVabPU/DVwZvDlk2ckJtP4Ej+leq/Di43W+oW5PKujj8QR/QV454Jk83w+Vz/q5mX+R/rXqPw9l8vW7iL/AJ6QE/kR/ia4pK1RndF3po9KNNNKfejtWpixtVLhcPn1q3UNwuUz6VcHqRJXRUz0o6OffBoobhh15FdCMGLmkIDKQeQRg0lKOapCPD9MgNjq2qRRIr3ttHILYMufnVwCQD32BiKs3KN4ns7jUvJjt7qOXG57g7JQcswHmMcFeuBxg9OlX9dlsdO8aXqXdsiLvEsc0asZNzYb+8B3PT0FRXfiHSdR2G8NxMUXCh4t2Dnnq3fjtXoJt2ZxtLY6H4XXO7Tb616+XKsgH+8Mf+y13wrzTwJqdu/iW4traJI7eWDKYjCsSMdcfU16UDXLW+M3p/COp9uSLsjsyfyP/wBeo6VDi4hbPGSD+NZGhpZ9jRS/j+lFMR8oR/8AITsAD1mBrr8HbxznvXN6NbpeeKNKt3+68/OD7Gu01LTpdNmCFhJG2SrHg1MJpPlHODa5jPwR2pQCeMUoyP8APWj5vb0rYxsOMbqVBHLDIwaPKcsF7n0pMt1zxSlm4OfzpABiCscnIHoKsxWKskkguIf3ah9jEhmz2HHUVX3NjGaXJxnNA0M2EnoRz6UqxMT909aXLevtUiFs9TSbGkaGiRTxanbyrEzAOA2O4PBr1xNIYxcr1HYV5n4fhZ7pS05TBBOO9e4R4MakelcFel7adrnZCpKlDTqfM3xGsWtPiYuFKgtbnPXPT/P4V7og+XFeUfGR0HxA08D73lQ5/wC+2r1hB8tEY8seXsDfNqxaa1P7UxutaIlkTCmGpGphqzJlnToy94nHTJreW2RTuA+btzWRpDYvMeqmt6lyJu7HzNEPk5Oc0eSKmoo9lEOZkJg5zmqN1GUmA9a1KzdRBaQKDjgciqhBRd0JybIgrev6024VTAQ7AZ4yacoHrn61T1dtunsx6BgePrTqaxZUPiRJJEscSoxHygZ/KnQXEKW4BnjUA92FZd7fgWPm+oXv7Vyv29pZSmCFOe9YUU73NKstLHo0VzBM+2OaN2HJCsDVW61nTY1eNr23EiMAymQAgg81wug62bfUrgtGoVUKls9eQa4LxJfCXX9RTBz9olU9OfmNW6rU3EhQ9257adc00Lu/tG0APTMgFQHxRog4Or2ZPtID/KvnKeLc/FWLItGwUKMN0z/Sh1mgUEepzanDqXja+8i5E1p9kV0K/d3Kyk/yrrLbU7aG4ltWnxILmVcFSerEjtivI/D0pg1K4xx/okw9c/KTXrWjS711CUHlrtm/NVrnpNurdG80vZ6hN4o0uN2BnnbA42Wzn+lYfivxHZz+H5IYor4yl0YFrZ0HUdSRiuvyM5yd31rL8WQibwjfEZLBQ2TzyGBrskm4tM54uzuiK01eVbKBZNI1MukSq37kAZA9zVuPVZnAK6Re4z3MYx/49V9SXjQ9ioPFO25XG3I9zSSfcG0aWmjzrISSwNGxJyjkEj8iRUdvDbtql38i7jtbp7YqNLryU2dOM49KyLPUDHr0xZuHiXj6E15uIrwVRRa23N6dOTi2jqTaw8/IK4eTwxp1r4/u9ZFupuJUj2E9EO0hmA9TgD/9ZrqDqgrG1vUBEEu0GJANm7IwO4z+v501iaLklBWKp0pp+8RahdSXen3a2Pz3Ue5VVSAQ3YfNgentXB+NPhfPr3huTVI9HEXiGOIO5jkUeeR94MASCcZweucc13ENyt3Ai27yGYneuGGHU9VBPQ55FdDBeSG0jEsZ8zaMhmHWuuLitQqp7HxQhKyYYYIOCGHI/CrUchbEYHJPQDFXvG1ylx461uZNgV72U4QEDO7375z+OazrJTLKAoyAM/5/Kui+lzktrY27OJVUb2G30B79/wDPtWqjxpGXOAuAuP71ZtuhOOTjOGBxgCphKZr0QqCUjG4j3qCzUj3AKw4KhRjpipLuYCE+bN5UX/LWXPOB2H41RUzKp8q3eUkkKqr3579v881mzb7m82XOLy4BylnCSUT3cj+QpJDbLkU0t2hOnILKzBw11KMu59v8feq+o/aLexnjsoQcg7ppGDs477fyrREGpRhVmluTgYENrCoVfbLYFPZCQ0k1skUJxuCsZD+IUECh6AtTzXcd+7PzZzmvsHwbexX1hZ3U6qwuLOOXkZ+YqpP8zXyRqcKQ6jMsUbRxbsorelfSHga6ZfAuizAn5bVVz9Mj+lRX/htoKSvLlPSo7PSoblrhYUEh9TkD6DoKwPG0kBsrYxKgPmEEgdsVTOqN/erJ1q8NxaqCfuvmvHjVlL3bI9OOH5XzNmLK25AT6VUYYPIzU27dHjnhjUTDJBPT2roiVIx9Sj36VrUOP4BJ+mf6V5HeAhj15ANevX9zb20t3FcybPtNoyqME7iMjHH1ryK6GfriuvDnFiehu+A4pbnW5LSKQRtNFjJ6HkcGuu8Auf7Entz1gunT+RrzTS786be+fs3jaVK5x196734dXfnnVQAVBlWULnON2f8AAVpXj7rZGHl76R3M8vlQvLjO0E4H0zXPHWZ2ViYhGwGf1H9DXQTpvtmA/iUis3TbWxuYwJwNzQPyGx8wXj+XSvPukrs661TkVznpJLu5gaSa4k2KQ2BxwWwR+FQ3KGN7gBiW3ujEnqAAwrV1FYYmuoLZt0DoWUZ6EDNZtwd8xP8Az0aNh77lwa1jqjNSurkQvN0xuW5w8bn8sGrtvIp1hMAgmQuM8cMo4qiYBDbRzKSfMj349CCv/wBerjS/aZ2uYFkwAjD5epBwabKRk/EGPM9pIBjdG6k/Qg/1NcIvU/SvRvHaLJYWkmcsspXrzhlP+ArzlMCQZ6ZrsoP3EceIXvnb+AJswX8OejIwH5g/yFeoeDZ/J8U2mTgOGQ/ipx/IV4/4GnMWrTxdpIT+YI/+vXo+iXxj1SwugrIFnRirDBA3DINc1dWqXOmi707HuNJmlIptWiGHemONykU6g1SJZnkYNMY8r9ammXDn3qCXhM+nNdCOeQ7NGeabmlzVEmdf+HtJ1O5+0XtjFNMAF3tnOKgHhPw+p/5BNr+KZrYpc1XM+4rIoWmh6VYTLPaafbwyqCA6IAQO9aOaaKM0O7Cw/PNDHam7+6Q35GkFOVd6smeoIoA1Ovaiq0VynlJkjO0ZooEfN3hNPM8caSOcb5CPwRq7/wAVjBg6jJPSuA8I3ENr4306adgqDzcn/gBFd/4tnSSS1EZ3oys3mD7tcyT9qjpuvZM5oZ6ClxzzzSd/agkggAcdzmu25xsUc+uK0NL0XUNXkZLG3MrJywBAx+ZrPXOR9K6nwd4gg0O9ka5jdo5EILKRkHtwetZ1JNK6IldbFO+8Jaxptobm6tDHCv3mLL/LNYpUivQPFXjSx1fS2tbaOUMSDliMVwDMcDGCM81MJt7ji27iDPtUkfGKh6il3FZkXI2kHPrVs0RtWlyYFyDyRXtlheq9hAxPWNT+lfPa3iB8Z4xnOa9csbmcWds4lXyRbgGPbyWwMHOfrxXn4qThaUTsow9orHlHxcufM+KEAB4EduP1Jr2tPuivnz4lTGX4kI3B5gGSfpX0HH90VUHzQTZMlyyaHUw0+mmtESyJqjNSt9aiNaIzZa05tt9F9cfpXR765e0bF3F/vCug3D1rOc+VjSuT76N49a5rWo9cluU/s24ijix827r/ACrNWw8Vsfn1WEewH/1qweId9ivZnb7/AHqKVlyCQCScZxXJxaV4i8wtJrClP7oz/hUMui60u1pNcdhnGBnqal4iVthqB2JZMfdH5Vx2pXDPPcRs7bQ5wueOtS2Wk6hb3iTzatJKi9UwcH8zWVqJd7q68uNydx7GnTquV7lKFitqOqQw2CxS5XCrk9hWBLP5beYozjnPrVnVLA3tqiNA0iuoRwQee1ZN1p99Gfs0PlKgACZfaqADGPXrWlOsti5UebUrLq6y3lykJHyxt5nQ8gZ59K57xBLGPE18nmJ5jTyMFzzjcc1fOmXa/aJI7ZVfaTK6TMMnHPb09aZqmhX73CubIPKHDs6yuxcdxkr34/Koc05XNFStGxgz3CQxhmcKMgfUmrdtZTsrT5IjHPJz09MVoRaDqFwwii0RncHcW3noPqOK3bHw1c2EUN9L+7aMEtBuPTIHzevXNKUr6LchwSV2ylo9pIWS6XmGWB9pIw2GQ9q9Q8LljpkzkZ3uj8e6LXBW7zS3KkkbCTnj2ruvBj/8So+nlwkZ/wBwf4VWHvzakT+E3uABkfrVLxA6jwrqe0jCwMx3elXow0jAbWqPU7b7ToN/E6kB4XGR24rvexzLcjs2M2n2kiZIeBGyD7VN5Mh9aqaIyDRrLMoJWBQcnrgYrSE0X/PRP++hUKS7lNMztXnnt4ENrCs1xsGImfbkZAJz9DmuflvAmpqe4XBNdBqnkSiKdHSR49yBlIOM4z/IVyOonFwJB1weSe/FeHiknXZ6VH+EjRuda+z2zTFXbaeQnJrMm8RSTxvGtleHj72zj86qPO3UNgAjgDiopZ2KDkAE9BWMaLTujSUn0NfQxp6awrTXRtnc7sN8i5HPVux9sV3iXEFwqPDPHIhyN0Z3D8xxXkgOXUtxzzn/ADzXdeDsNoSd9kjqR9cH+tenSldWZzVVfU+V/EL+d4m1SVQ2HvJmAI5++etSaUo2tJjvgU7XrKYeJdTjCMQt5MvT/bNWtNj8iIBsLg56dv15ruvocXUvGQW8JckZ6jPpUFrL5VuZHUbnJJ4xiqt1M0zqinjg0XbrHZZ5GFxwvH5j/PNSUPtYLrUIjMZHSI/LtiBy4ye/T2rWs7LUEBgts2UTHnbyx7dBwPxJPqawNAhe4IHnTqozxFIQf0/z1rqINOt3O1p33L1LXJzn8/6VdiTQttIgQK9w0kzDndK5YD8M4FSyfZkct5CvIeV2A4HH+fypsOlhRmO9ugCD92Yn/ParDWskShUuZCxb7zAEgc8cChrQcXqcN4xs7tJLe6m5jYFB7Hr/AJ+le0+C0MXw60odP9F3fqTWP4bsLG+mvdMvoBdw3MQLCbn7p7HqOp5HpXbraQ2mmC0toxFBFF5caD+EAcVlLWNi46TTMSX97JFIZHBjYkBWIDcEYI7j2qO4cshHam7ifb61HIQUPPbNeWlqeu3oUrWFYVlRNx3SFzlieT169PpRPDHMFEqghWDjJ6EdDT1PzNjuOlMkOQMY/GtUZs5bxijfaLB1BJO9eBmvO72Mq75BDbiCDXsUmV1CwfOFEhU8+orzLxZCYNe1CMdBOW4755/rXRRlrY5K8dLnMV2/w1l26tdxf3oN2Pow/wAa4k9TXT+AZvK8UxJ/z1jdOntn+ldNVXgzmpO00eu/wDnINchNbzJNKhdEKcRlmHzZJP8AWuvCMVwAfwFcze6c5nvnSKQzKQ6YB56Hj9a86J6VRXRnrLbx3OWuS4ERJVR1O3moRc2rS2ReGUlFADsQOM4BIpW0+4Mcf+iy5aNDnyz6EEdPpSXEF1NbwH7JMriPym3IRggg5/nWisYWHx3NxMNsMcaokLNjGSOo4/GopbiW5dY1kZUcoRg9Mg/1FTwWtzBegAIqMzoxdgPlLZz196pw2s8TDcYuCpAMq9j9fc0aD1KerQmXTbqaUMJohHIN31wa4uQbZmA7McV6Hra29wtzJJKiL9nYD94vLDOO9eez/wCtJ9cGuqg9DlrrU1/DEnleJIOuGLL+YNekBiOc4I/nXlOny+Rq1pLnhZEJ/MV6rzn+R9azxC1TNcM/daPeLWcXNnBOOksauPxGalrI8Lz/AGjwvpz56QhP++fl/pWtmkthvcTNHFFGapEle4XgEVVcblI9a0JVBjNUDxxW0HoYzRGp3Ip9RTs1GmQuPQkfrTs1ojMd2zSk03PFHSmAtKDTc0vamgH5qSI81BmpYvvUxEUkEnmNjGMnFFaHHvRSA+XtIJbxDbZzwsnf2rq5nJwCenTNcjo2f7egYcDZJ/KupkJJ570QFJk1uQX2sO1WwkPcE1Rt8mTAJJ9quBHx0PtRJ6jhsPCw9AozS5h6hB1qPyz/AJFAiYk5FTcqw8NGOdvApPOQqcKB7Yq/pL6fb3EjanZPdQlCFVJChDdjxWUkIZnaNZAC5IDNkBew6fWpUtbA0K84UE4A/SsLXLsp5eJCgb5gytySO1bFxFiFtwBGOh5rnNSQSW5juCf3THYe4GOKu4rFVNQkmVR85UYJYddte56fdbtIg6j9wn4cCvnfTpHN9Cm5VVR3zhscivdbCR20yA7sZiXcMewrhxmsUduDWrPL/Hkm/wAeqc8F4f5Cvo+M/KPpXzZ43I/4TUHIPzw/ToK+kVOFGPSqhpBGc/jZKTxTCc0jOcVGWIHBq0zNoc1RHNK0hA5pmc1ojNkkBxcRn/aFbO+sOM4lT/eFT3t7fQzhLaxMykZ37gBmsKybaNqVrGk780itz1rBm1DW8/JpI6d5B/jTYr/XmkUNpiovck4x+tc/s5XNeZWOiLECq8wd4twBIVhnH1qtbz37fLcWsaqf4w/T8O9LcsfJwD/Evf3FRLR2Lir7EobIqW2iDAZxzWXd3E9rbtNFC1yxIAiXjb6n1rGj8ReIYZZPJ8PSzJn5MyhQPzrKona5UtrI6aOyjaJxgcSt/wChGvOkxqYuZbjaGErRELwCAQQe9bcuveJ0idrXRUZnYl1kmH7s9x1Gaz/Dem38cFy+rWiwzNcF0RHBBBAz0J71WFhFuS6vY5q7qRSa2KNxKLK3kjW1JikGHLEnHGOwqzYaeuv38Mt7Dsh27RsZgSoB5611KWVtdSRxTwqUJx0PH6Vu22mWkEjPFAqsqhSOe3P9a9mMacIaL3u55zdWctZaGLpmixQRyrYQrEJIgAz5PfJyc89Kh12GIz2iPbK+9zncOuQtdXEVVZAMALKRgDtxVWcLusXwMk9duT2rOjFQfM9WXNNqyZ5VKiQaj5UaqqbgdoHY1teF1jezj8yZ1WO0jOEkK5wSvOPpWD4ngnTxhIYrgRqsi/LtHAIGQDW94SUfZZCSM/Z2XOPSVhWcl7+p1Q+DQ6OIWRbmWTHf96/+NI9hYXELxRtKQQ2FMr5PH1qsVbaHM5wf7oq7YbGOPPYnkFSOvFWmu6ItPsZ3h+ytn0qGUwwmRlw5ZsE4JrWNvFHHkRW7YHTIz/Ksrwp9m0zw84kfyoYZJCzSHp82ev41P4n1cWujpHZOHur8iK32HOd3Vh+f6ily01G7RtFTnKyJ5reIWu5IUjIcfdGOOfYVyWsrtZDxjBGPzrqRYvpXhqC1kmaWSKNd8jtnLZ5x7c8e1cnrLZiDbs/NwMcf55rzMXG1ZNKx2UX+7avcz3xGjHcPXr0FZr6nagcSk8+h5q/1V8gY44FU2hiXpFH+AFOHmOV+hs+DNR0+bXza3XkvFcRMAsyjaWGCOv416JcS6dpFoxEltZ24OTgrGv58V4xehPscu1QAFyOPeuRu1HnR4QdPau2lZxsclS/Nch1ho7jW9QnRt4e5kcNnO4Fyc5/Gse7zHJs9R398/wCFa7x4bcBzWRfnF+cHJ2j+X+fyrdIxZVbmUY7dhxW54bsodV8VadZzQeZGXLyRuMqyqpYg/UisRid3XI6Dmuz+F1qZ/Ed3eEfJb220cdGduP0VqBGHq3h8eGPEktmylrST95A7jIdCenuR0P0z0NbtpBayEDZCW4G3A4/D/Pau68XaAdf0UxQrm9gPmW3Yk91z/tDj0yB6V55pd3A8EYucI4wodhjPpnjr/hVRYjbitbdCNkC5UdxVpwQuFPGPun17fypkYIG4Nkeo57VI77VBZhj371Yh+kXX2TWraQc5cIR7Nx/n6V6ORxjH1ry+HdHcQvgrtkBLHgDB/wD1V6dwcHPWsTU5VvkLDPTIpjHIJ/WprkYupgOPnOPzqvn8u1ec1Znqp3VyDOG7kY6Uw4Kj06Ypx+9jA79aac98VSIZR1CMG1UnosinNee+MoTHrVwM5yiEZ+g5/Q16RfDNjLz0GcVw3juL/T4pNv8ArLcD8QTWlJ++YV1eJwjAhj2q/obSLrdn5T7HaVVDZxjPH9aouMNzUtlMbe+t5x/yzkVh+BzXc9UcMdGet/2Xqcn3rtMf75qlc6RqRLBWlkCpvyiMQ3tkd661GyCMLwOCO/1qvq99cadpUM1tjPmgNwPu4JP8q8WtVnT5VBXbdtTTMsRUw8KaopOU5KOraWzfS/Y4+HRdTnmWOSK6jBByxibAwB/PP6VCujagZdjWl7kPtJ8lsEc4IP5fnWwNf1fYJXuEEbuyLhF64OO3rUTa7rQjeT7dx5IlUeSnrjHSnfF9o/e/8jkvmv8ALT/8Cl/8iZY0TVXi+WyutxxwyEdueo9asf8ACOzqR52+MnPDdcdj0/OtCbXdVgukX7aWQhG/1acgnB7fSuj1AmS3tWY5LJk+/AqfbV4VIwqJWd9m+1+wqeJxkMTTpYiMbTuvdb0sm+qXY4hvC6zwhJpSCSchf0rzy5Qo4DdR8v5cV7WQD0P6V5H4ggFvq93H6TufwPI/nXpYeTd0ejiYJJNGbnGw56f416zbSie1hm7SRq35gGvJc/ux7GvTvD8hm0GzY4P7vb+RI/pV4haJkYZ6tHsvw/n83w15fH7md0/PDf1NdQea4T4bT5h1CDPRkf8AMEH+Qruqyjsaz3CkzRQasgCcjFUpBtYirtVrgYOa0gyJlIcSOPfNOpG4m+q/y/8A106tbmNgo70UUxBS96SjvTAd2qWH71Q1LCfmp3At596KX8KKBHy3oxB1yHHZG/lXTSHP51yNnMYtSRxuzg9K6X7SWZMkYIwQfWlBimi9ZuqSFieAOlXvtkQyA/6VlK+QcHjHWoBqFo9z9nFxGZQcbc9/SrcUyYto2/tcfZj9MUfbIgOC35Vmk85zUbQIQx+bPsxqeVD52WP7Zi3MAHOCSccEU+DUIVUliRu5Az+Fc7IjQzMfmR94PPIAz/8AXqKR9sxLMQAhO0fmf8+9TyovmZt3Os20jiBopMyDAbI6/wCRXL31zM188SrsIUqc9/So5Jw16JSR5YycbuR/hU2orDdYeGQeaFy+Sc49qmw7meitFIksY3BFzkDv0r3HTZEOk2w53GBPzwK8d8KOy+JLSNi3lu5RgehBBGMelet6euywt1HRY1A+mK4cXLZHoYOO7PNfGxx4yDcnEkWPyH+NfSUf+rX6V82+NS3/AAl7ADIBiP6CvpGI/u1+grSPwIwn8bHmmGnnpTDVIhkbUgIxStUJPNaxMmOc4XPcVsJKqIjMcgj8+Kwp2PlMBnODjH0qeCVGiiiuJGjkToCcbs//AK6irY1pGpJfAFQikZIGMY4NRtfHaSh56jPfmoPs0DEHz3yAAMN6UhMEGYUWWUt/COcVlaLNNUWTLmd1AGNoJ+tQXH+oOfUH9asqiIDgYzyfrUF0yC2k+YdPWuSo+Z6G8NBik7Dn1oiJYED1NcX4u1O/sL5o7a7kjQoDtU4xWJ4f1zUX1hYJr+d1MTEgscZAzmlLCyqRTuEqqjod00pD3Qz0lP8AIVNuyrVwmkXt1PcWzSXDNukcOZJCM/411Go6qulCPdBJcecdoEPJB/8Ar1FCi6WJimxVpqdB2Rv2NqbiRAwwCSckenpW8ibTKPcfyFcroOvtcXsdnLYT2zqrOolPJGK3Li/kiuEaOAsjL8xzwDkY/r+Veu6sW7XPM5Glexbjb95JH/EX3cD6VWbLG2UEg7WxkdDms6TVpBczRTrFFcCfyEYuSpLJuToPoKr3uq30Wi6dIlvF57ThHYZKryR9eooVSLdrhyu1zkfFMDDxFI3O7Ck8Vb8HN/rUK5ws45HpLn+tV9fnkn1KczFfNVivyjGcGqui6sdJEswhEn7+ZCpbHXaaxlPmkdMI8sTu5Y1KLsSPHqEH5VLZKouFO1QfXaK5U+PNinGmp8p7v6fhUa/ESTeSNNhGTxmQ5oUFe9ynN2tY29GigudOu7a5hWaFriVGQjIIyKr6V4Vh03WheLPLNbxIVtbeXnySeuD/AC+tYth4gmtYg0cMWLm7YkNn5cjPFX5fFN5Ax8uKH05BP9aj61RWkt0WoVUnyvRnTarJ5mmTgBThcnDAnqK4LUlXyvbcMZHHpWoPEd3dQPE8cAV1KttXBGaxtVciEtzu4xjtzXPiKsas04mtGEoQaZQBHPuoqNxxSxlQ7dPmXI5pG/THemkVcq3KlrWUf7J7VyFwv72MkdF7c812cgyjLnt61x1yCJVByODXTROasRlMjAGc9KwdTJGpMowSVXNdHjMec9PQVzmoA/2pIeeAPpwK6DAozkBeu3ivUPhNb7dG1G4K4aW4VRn0VTj9WNeZyDBGPlB7Y/wr174fR+T4bhU8F1LE+p3n+hFK4WOwU4Ze3Irz/wAaaMukeIvtcKK1tqWX2Hoso+8Pxzu6d29q70tuyazvH9ib3wq06jL2cizgEfw9G/Q5/CqiSzgbXaG/0ad7d88o44/Lp+VaIeVFLSKCccvFjcPwP41mwETxLvABIznGQTVlGnVwgRCFOOc4xWghrtHLJvQyvt53Zzj6jt9a9Qs7qO8s4p4mJRl/EH0Ned+cqj5Y08xBn6fjXT+Ebnz7a7BwrLIMpjGMjr+P9KzasaLVBqC7dQmGepz+lU846Gr+qjF+Tz8yg8VQPB/wrzp6SZ6lN3giFupz0z0Bp0s0lxIXlbc+MEhQPbtTSQTn2zSYOTkjikgZFON1vKvUlD/KuN8bRiSx0ufGQUZefoP8K7YgMpHrxXI+KE3+FrWTuk23Pf8AiFXDSSMqq91nmTj5qb6VJONsjfU1GK9E83qe+aZILjTbSfAzLCr/AJqKbrSqbC1WQAqZ8EH/AHWqn4Rn8/wtpznkiIJ/3ySP6VpataTXelhLdQ0qOHUE4yRmvBxjUJwcnZKX6MjOKkIRw85tJKotXovhkcXPHI0NwqFfIS4Bjx15/wD1iiBfMt7f/bgkiPPpzWpH4f1ZIzEFj2OFLZccMKtp4fuUZPlXakrOBkfdYdK3+t0P5196Nf7TwP8Az+h/4Ev8zl5oWWCzkDs37sscnrgg4ruroh7OyYfdMYP6CseHwrcJgSSM6jO0ZGACCCOtbVxEbeys4TyY49mT7ACuarWp1K1Pkknq9n5M4quKw9fHYZUqkZO8tmn9h9ih+Oea8z8ZweXrcxHR1R/02/0r07GTx+tcD49hK31vLjiSFl/75Of616VB2me1iF7hxQ+6RXovg+Tf4eQf885GX+v9a86Hce1dz4Fl3WF3F/ckDY+o/wDrV0117hy4d++esfDuXy9buIj0kgOPqGH+Jr0qvJfB03keKrTnAfch59VP9cV6ya5qb0OmotRKKKTNamQuaimGUzUlNIyKpbkszZuJIm9yP0pSaLv5Y84+6wP60mfWtkYsdSZpM0A1Qhc+9LmmZ96AaYiQGpYfv1ADUsJ+bigC5z6GilGcUUAfKEJJvFKnnH171tmZVUDPzdzWHG7LKGDDlcfqanRt8o3vhQfujv7VnF2ZUlobscsZWSR5BGqrt3E8dK5G0iMmqxQoVP7wDcO4Hf8ArWibuW81Mw2yh0SPBRh8vvxVO/SSGc3YxHLn7oGNpx2qnO7sSo2VzuenJqvLJJlo4yu7GRn0rn9N8QyXLiK6IDD+ID7w+nrV24vI45lkhJcr146itOZGfKQ6lqpEnkkDcBzj6VVt72MwJv6/Mm4+h6VnXbme6kmxuLf3uoFQ5bbnGcdCetZtmiRbtwryjcdx3DHvVm+NtFIqlgjgYBA/XNUtyJbLIuN+cHnpVSdi8nzMrHoCDSuFjV8NShfE9gBzmUD9a9ktBsgjX0XHFeKaE5i8QWGc4M6ZGe2RXtsQ6D3rz8Xuj08F8LPMPG5Y+K5MKSAYs+3Ar6Pi/wBUv0r518Z5Pie7PZRF/T+tfRUB/cp/uj+VbR+BHNP42PNMNPNNPFUiGRN0NQt1qZqhbg1qjJjWOFzWkqxzYEiKw7AjNZE5xCxHYVTh12aJVUqjEAfMe9c9eMm04nTQas7nSPbwxEGONVJ64qe2dWQgHkHBrmpNekl/hQe9NGu3CfMgQcY6VEoyaNdLHUTxeZbyJnG5SP0rgf7PETCQyudpDH0yDWv/AG/fMCC6c+iisy9lZ7eXkjKHI/CqopwTTIkr6i+IdDudc1AXNjJHJb+XsYgg/MCff3rC0/wrerrEgRv3ixuhyuByMcHoetb9pdywQmKJykefur0qaCR2uFG9sk9c1C54qyLcYy1Zzkfh+6tLmNGlXdbTbn9wcHitvVWJhtT6Txn9aqGaa5up12tvBAOM5OOKszx3E1rDEttMXDqS23jjBrkxKk2pM3pcqTSNhCqeNLVU6NAw/QVvbhIkojXG7bwPqR/SsM2lwfFOn3SpmJEKscgYO2tW0nKFARkOWGcjjDMf60qUrRMKquzB1S2uzc+ZHC7kXtpJ26BcE/hWvdwSyaZFF5ZDpcq+M9t7Gm6ncyqIZIolZGZd5LYxjpUr6nCxlDsBkADHPc/40nXswjRbs0cV4muY7TVbknkPKxBX61zsl4YIXAjLiS6PfGMxqa0vFIea7ItySobPNY1xGz223blhPGT7Zjx/SuqjJyjciceV2KV5qEiI/lqFOMjPPaneHrt9SeZJiA0UBkVgMZYEfpTbiyeUnEb5Axw3H8qb4SsZxfAk7EkiZc9Tj8fpW1S6ptozh8aTOuuYUh0rTnTPzTKzHPc0XnBJzUtzEJNAhXLDym4I9QapXcLkkm4lP4ivGg+Z6notWRNayEtx6U3UpmMLA8jHYdDVW0hO8ZeXA6fMal1B8xDOeAenf8a66cTGT0MlbhzcbfMbAzjn07U9rh2yM4NQn/j65IxjikPHHp613KKObmY4zNzlsfQ1z+opslVlHVOnvW536/SsfVgcRfQjjp1rWCsyJu6KyYEagE1z+oAHUpeuCByDiujXKquehHFc9qOBqD9BwOa1Zkioynd93n/630/z71694IUrosKkYxGR/wCOof55ryVYJJp1ijGZHbaoHdjwP5ivbtHsBYSy2oGRFO8fpkYYf+yisnNKSRoo3TZqDGOx4rTEaXVg0Mi5SRNrfiKzQPQD0rTtjm3T6YraJizxyKCTTr67sXB3QSbcN3Xsfyqyb2FVxICBnr6Vu+ONMii1a3vmXEdyPKkI7MOQa5h4PKIS4JI6K3QfjViNCF45EPlkHI69+at+Ebl4Nf8AIZsiVWib6j5gf0Nc/wCU9o3mxMGjPXGcdf8A9daejzrD4ks5xwkjrnHvxUTLidnrQ23ETHjKY/I//XrMbrnvWzra5WBvQkfyrHxxwTXnVtJs9OhrTRG305zTceuaew4680Yx6j61KZoMA6flmua16MHwnfJjPlT5A/4EP8a6fHI4FYmpQeZpOtw9SBv5+mf6VUXqRNaHkF2MSt1555qCrV4vzA8dKq9K9KOx5T3PXfh5P5vhVE6+VM6/yb+tdcxIYgEgHng1wXwtl3afqEH9yVH/ADBH/std+ynI78V5teK52melThGpSSmroql5Bn53x9TUZll/56P+BNTMuDjmmFSBzWPs4dkL6pQ/kX3IjMsoz+9f8zUblnwWYsQMHJqQr29O1IwyDxj8KahGLukVDD0oO8YpPySINnzc+vauQ8eW2bG1l5OyUpkD+8P/ALGuzwM5rB8aW/neHp3/AIonRx+eP5E1tSlaaCrG8GeRrw314rrPAcmL67iz96MNj1wf/r1yj/LM2OzV0HguTy/EiJ/z0Rl/TP8ASu+rrBnnUnaaPT9KmFtrFlNn7kyk/TIr2k14dtI6de1e128ontIZR0kRW/MZripM7aqJaTvRQa3MBKKDSU0Ip3iZjceoNVw25QcdeauzjK5rOi/1SjP3ePyraOxjJakgNLnFNzSE1ZI7NID6U3PrSg0xD81NCfmFV81LCfmoAvZopm73NFAHymg+bI64/TmoJpCkx2k89KlOFfsTgd/rVWfIk4HXtWC3Lexd8KkHUpz1zH6Z71d8VWyRQQygtvkc5BOR0qPwjGReXR/h2AZ/Gp/GT/urMdPmYn9KybftrGqS9lc5aIlZFIz1Ga1DNtIByB1HbNZKMWO0HI6VZAd8YOBXQYE0jbju7nuOKjPtyO1KCixkFjuA4JFREkEk+mKQ7D+RA2CcFulMCqF+V8kY7VKrgWcwPXcuAKv+G7KC/vZUuELII9wAJHORQ3ZAldkemxn+3LHGDidDkH3Fe4oPnx715/BpFlbSCSCDaykMDuJ5/OttNTvC/wDrjyeuBXHXg6jVjvw81TTTOV8cHb4muRngrFx69K+i7c/uI/8AdFfMniyaSXXg8j7ySgPvxX0zbHNvEf8AYH8q1StFI55O82yU008CnUw00SyNzUDmpn6VXc81qjNkM5/dOPaubDcAVv3kojhJxnkCud78VEzWkSkkc05WJXrTVUkA9jVtrUR2sc/mxNvONitl1+orO5uiAHA65/Gkny0Ev+6f5VLsGTSSAGGQDj5TSuMZbHr6cGrtuP8ASUI4wc1RtSNvXPAqyrFcMDyO9S2NEisItWum6DCn9TWg92iQrIrAqE5A+lYU5Juix5JQZNTA/wDEtb12GuLFfDc1pR1NyXVIozAHRmbJHBx1UisTSdTuWtZ4zIcJO4X1xTrs5+zEf31z+tUNJOPtqnPFw39K5FJ8jNHFJms0rP8AfYke5pUb5G47ioFOQcYqRD+7f8P51hI1iYWrkfamJ9Rj8qy5twMu4YO+Fj/48OKv61n7Vx02isq5/wCPORuvERyD/tn/ABr2cP8AwkefWXvsndbgzuLd1UnGcke9V/DxeO/hRzypdfwyaPnEpO0kbRnio9EkzqikDH71h+NbTadNoyjF86Z1b86PMOwd/wCtUrr/AFYPHTrVsHdpt2uekh4/Cs+d8wIeDlRXjwWp6MtiKBiHXmnX/NvuOPeoIz+945qbUWxZdMkeld1JanNLYyT/AMfaAA9AP0pCeR3oUj7bAc/eXv8AiKCMGu1HONx+ZrO1Rd1tHjk+Zj+daJHT/Cqt5GHiHAwJOR9R/wDqq47kszZWCTRqD0HSub1TIvmHsDn26Vt3Tk6iRwQvFY+sJ/pnvtBwM55z3rV7GaNv4fQ2k3jbTTdlRDHvl+cgAMillz+KivX4VX+0bpgysv2sEEHj5sf/ABdeKeFE83Uom6NHGxwe/GP/AGavUtMvtxmjOAVeNxgdPuj+lcdSm+dTub05K3KbkqbcEDnPOTVq0bMA55BOaZdIfJJGB3ptk3yMO4Y11UncxqqxT8WWH9oeH7hVXMkK+an4c/yrz6GQzW6vkFsYIbvXrBII5xg9jXlt7YnSdVubToI23Rtjqh5H14/UVsYlQzRoSrxFeeTjOKrPFKksb2/MIO9WB+6f8K0NyS5GV8zsOtZ01mY2LrlM9dvSlJXKTsej3UwvNItrgY+fDdfUVm7SBjB60uh3AufDCxdXgfa3PvUjKM+3pXmYnSZ6uF1gQlcjApNuKnMfWgJ26ZrFM3aIdpz0rNlh8y61CAjiW3/oRWvt+maoyLs1qL0kgK8+x/8Ar1VyXE8OvV27epIyORVEda2NbhMN5cR8fJMy4+hIrHr1YO8Tx5q0j0H4Vy41LUIP78Sv+Rx/7NXqRXKivnzStVn0mZ5reWaKRl27on2nHp0PoK038Zam/W7vSP8Ar6YfyArmrYeU53R10MTGEOVntToc5xz71Gy7eTgfpXh7+Jr9zlpZmP8AtXEh/wDZqrSaxPL99Eb/AHizfzNQsLLuW8ZHse4vPboAWniT/ecCq8mp6bGMPf2q9uZlH9a8Xk1KI2iIlsq3APzSEKQfw2/1qD+0bnsyD6RqP6VSwnmQ8Z5Hsza9o64zqdrnv+9B/lWTrut6RdaNeW0d/G8kkLBFUE5bHHavLv7Ru8YE7j6HFNN9dHrcS/8AfZq44ZJ3uRLFtq1iOcYlPB9ea0vDsoi8SWL9jKF/Pj+tZTMznLEknuTU9rMbe7t5weUdW/I10SV42OWLtJM9qZMg16n4cnFx4esmznEez/vnj+leaeWGGc5GOo9K77wbIW0Mxn/lnKwx9ef6mvNpP3rHqVo+7c6D3oNGaSuo5ApO9LTenNNCEcZWskfLLKvTDZ/Otf8ASsm4Gy9Yf3lB/LNawImhM0Z/WmZ596M89K1MR+aM0zPWjPFICQNzUsJGc1XzzUkR+amBd49B+VFR7veigD5aABYdOAPzqjccPyR1PatS1YeYd0Af5RjPOOKzbja10cYBz34rBblvYjiuZIA3luRkDPPpUdzeT3RHnSs+3oCxOKn+z7IGkZg3HbkYqG0jWWYhwcAdBV6bk67DIcA8jntirIOf4gMA1BGFW52yfdGcj86jZj360ASiXPAPA4p/lTOCyoSPXFQ4Hkk7evSrWnuyyBd3DKeOvSkMWOF/slwXQowxgmtfwWGPiW2gxuWc+W689M9qqSLNLE0fAIAAPTGK3PAMD23jLTmcrtEq5x9azru1Ns0o/Gj1GbwxFC+14pYjj7rccVV/4R1ZdQsrWA+W1w+zc2SAcV6Fqpj1C+82LJXaAMiud1kPpk2n3qDLRThgvrweK+epYmft+RSuj25ckqPwpSPG/iNo82h+L5LGV1kdPLbKjggqDX0RaH/RYv8AcX+VfPvxT1Y614ylvDEIhtjXZ944Ar3+zP8AocPX7i/yr3/so8WzTdywxpp/WkJppNNAxHPFVnPNTsarv3rRGTKl2m+L8QeawVAPX+Vbd7JsjHAOWArGQFTuxms6mhtR2JU6BR69alVG3hOMkjHzVEuSc9KkjO0/erNs3SZPdW01nOYpdpYAHKnI5GagcM6sADnBqTzGKlVU59hTvImZSVR847KajmRVmVrSJgigjHyitC2tI5nZZLhYQFyGYZz7VWtbO4aNCUl+6Odhp7GKJiGljUjghpAD+tK6Y0nYqzkCdef4Tz+NPX5rBwf7pqtdXdvHdKxuINqoQxEqnHT0P6Uz+2NIWxlLapbLJtYiMtkk9q58RFuOhpTkk9WXLps29s3PVP51S03JuL5f+m2fpwKv3WDYWjwMJEdEYNjHBrNjmmhurrythJcE5+lcsKd4tGsnqaYDDscVLHEzxynOMLnH4iuf1O91OwYB2UbhuAUKap23ia4tmle6YyRlCu1SoOePeq+qSaujP28U7Mv6rETdIT3ArMvv3FvPjHEanP0cf41Bc+KYZpwz20m3GMbwDn8qVrtNQs3kSNo1aGVcbg3IKmu6lTcYJM55zjKTsF9qptrwR8nzVCfnVfR/lvGYjDfaeAOw4z/OroEIKDGcrnkj2+tV7FkjvZyQFPmgglwOw9q25Vysz5nc6NHAt71SQPnBGeOoqgwMlsgUgnaOMjrVp7nT0SZGmtwrNkb5R3GPaqw1jTYLeNDdWqcYBEnP8681UpX0R2ucbblaOKQTcjHqauXkBktCpI5FU313T2OftyMvXIJIp8/iC2eJYI7gnOABtIzXTCEk9jGUo23M9k22/YsBgEDpyaRkduAjn6A1HcXrwwErI6hs9Cf896py6i4Td++cY4AySfoM11K5z3RoeRKR/qJOP9g1G8MgjcshwuD9Kins9dhiSVtCuykj7F+aP72Cefm44HU47eozHHa63vZn0gRQr/rHe8hyo9dobJ69BVqMiXKJzkjZv3GR16fjVHUxuuwScfu1z+tWUf8A0m4bPGcY9qrX5/0sEDjYo6dOK0lsZrcueFONYEe4J8j5Lf1/Ku6s5VjvpVEq5aIYxnBxuP8AT9K4LRX2awjK2Mq2Pyrp7KYjWYATnzEZefUg/wCNRa6K5rM9HuNRyvkhBjAGe/QUtg4JcjqCKyd4YRt6opzn2FXtOkGZBjHH4VrGKWxnKbluawbvXHeN7YJLZ3wwMgwucfiv82/Ous39cHvWL4qh+0aDPgcxkSAj2qyDzyeFZCNpKP6jvUP2q+sl/fqLiDuR98D+tXIgr4BHzdRTpomMe3y2YHkYGaBmv4YvInkmijbMc8e5cdiMV0fkkjP4815lp8s2n6iJUJUxvlk5wQa6C5+KmmWkjxNp920i/eBKKM/ma87FU5zknFHpYSrCEWpM67ye2KzdUu2spECpncM5NcrJ8Yolz5Wkf993GP5Kay7/AOKct26n+zLT5QQNzs3/ALKKwjhq3Y6JYqj3OpOtzbsKi01L+a41C2ZY03gMBucIOfUniuIl+I9y4H/EusuOmUJx+tQn4i32QVsLAc/88T/jWn1aq+hk8VS7md4mO/V744AJmYkA5wSa588Ve1PVZNUvZbqaNEaTGVjG1RgY4H4VRY5NehTi4xSZ5tSScm0NopaUKT0BrQgbRVhLG7lx5dtM+f7qE1aTw9rMozHpV6w9RA3+FLmQ+V9jNoraXwlr7f8AMKuF/wB9dv8AOrCeBvED9bSJP9+5iX+bVLqQXUapzfQ52iurT4fawxAeWxjz/euVP8s1Ovw7vDy+qaaB/stI3/slL21PuV7Gp2ONp3YfWutn8BTxqPK1G3lbPI8twMfXFV7jwhdIkflzCRsYYCMjHpyetL20H1D2M10OltviFYR2MEb2k7SLGqsd6jJAGe9ejfCzxTB4gXU4YoWiMBjbDNnOdw/oK8QXwbfsxBkiUdt2cn9K9D+FVmfC2uXT3t5B5FzBsON3DBgR2+tYclJO8dzoU6r0lse5Un1qiNYsD/y8Af8AAT/hS/2rY4/4+FH4H/CmIuUh/SqZ1ax5H2lP1o/tWxP/AC8pVIRbrM1HCXMDf3gy/wAjU39r2B4+1R5rN1XWNPMMTC8hysgHLY9qqL1IktB+6gtxWadZ03tfW5+kgNMOu6eOPtGe/wAqMf5Ct7mNjU3cUbjWO3iGyAyPPb6Qt/hUbeJIMEra3knssY/qaLoLG4G75qSJua5s+I8YK6fcgf7ZVcfrTR4mkDALaIp/25wD+gNF0Kx128+lFcl/wlj94Igf+up/+Joouh2PFLO4SGU+YwH3ePwqleCEyOwiB6n7x5NWYLVbncXLgjHTntUhs03ncScDmskinsZqNnT2OCMbgMCqlu7qzFBz610IsojbhFU/MScDIqnZWiLcMNpDAd/rTuKxkqGkm6HcaMMuQwwehHvWyltH/aJGwH5s89+/rUMscaysGRd27HI70uYdjLDPt5PapbTf9ojweAavgBQAqJ/3yKt29w0JP3CD17HvxS5h8pGtypONrZJ7LW34c1CDTtdsr6YStEj7mCpk4qvDLBI3zMQT74zXY+GtVnhvbe2to7VNzAeYbdS2PqeawxE3yNWOihC8lqe26HbRalpkF9G5WOZA6qy4IB9adrfhZdWtY4hOItj7i2zOePrT9HvdtqpcSySY5YJwfy4rSbUG2/Laytx2wP5mvMwkcLyptajqzqqo9T5T+I1r9i8RuobcDtOQPrX0FZsPscP/AFzX+Qr5/wDihKG8WPtTYNqYBIyMZr3y0YfY4Dn/AJZL/IV6dO7pxbJqWVRlvd70wtTN3vSFgK0RkwY9agc05nqFnznmtUZsoak2IRjOdy/zrGOuxoCvmWgPTHlJnP41ragd0IwOjA/rXn76Pfvcysstuqs5I3Sds+gFZVacZbmlGbjsdR/wkiwg5uolwOqxoP6VTfxq/wBn+0RvqEsXy4eO1cA5xtwdgHORj61kr4cuZRiS7tV/4E3/AMTV2bStVn0iTTpddt445FCl4bc7uDnIO4e/5n2xlGjS6mzq1Og5/F+ou2F0zXW9zEUH6kVC3iTX5yRBotye372+jQkn2LVF/wAIzckkv4rvQTyfKt0T27GpF8J20g/0jxFrUuO/2ggfTrWns6K6Ee0qss2TeJ7vTvtMNvpyKrGIh7syfMDjqox19+x9s8Y001xNLK77TK28qrKQCeuDzXbQ+GNJtrXyE1HUmjLFivnLgk8kn5eScChPDOgRLgJcEdAGlx+WAKalTXwoTU5bs5KNmEM4DEnKHJx7+grHuFLXDsWAbGDk/wD1q9T0rS/CyajMt9Gy23kgjdI5ywPsa1mtfhsjl2slkbqcrKf5mtIpNXMpXTsS+B9Nttb8F2klzMEuYR5b7G3AAcr+hFbb+DtMZyVvJkZvvFWHP5ismw8R+DNDhaLTLGSCNzllihABPryalk+IOiKMpYTt9Qo/rXJLC+9dM2VaVhLr4a6fezB59auyBwoAQY/HFVX+E2j7QF1S9b33xj/2Snt8SbEH91pbkY7yAf0qFviXn/V6ao/3pM/0q1QaVuYhybd7E1x8JtH8hTHeXXmAcksvX6bap6l4Ih0+80/TNPeV4rmKdWlkG4IxUYzgcdP0pV+I91JKENtbxgn7xDNj9RVWbxzf3F220QBrXdIhCdflIPU+9P2VvtMFKXYE+El0sasNXgzjoYm4/WoB8MZ4Y5ojbWdzKyMBOc/ePfkZ/wD1+wp1x8QNVOnpLDfR+cW2mMQr+fSs1/HPiKQ/8f7KP9mNRn9K1UbdSdR4+Gut+WkY03Sht5DM4wc9eNhqyvw68QHZ+60SPaAMYbjH0UVlv4q8QyZzqN3+Bx/Kq7a5rk33r68PHTzWqlyiszq7b4fa5HaGCS/08ZcuSsbnGTnHb9c9vQUyL4XtEVabV0G3HSH/AOyrjpdQ1Bj+9upT/vSn+tRDUo0b9/eRLgdGnHX8TQ2uwKPmeijwJp8hEcuq5AzwqL/XNI3gHRUQ79TkB7OiIpHvn1ri7O8AjEsV5asGGRsuULEf7oOf0q0msGQY3E4/CuSpTqSleLsjog4pWZ1eo+H9LvbdILrxDqjxxSLKgiljQhxnByEyOvrWNP4X8Oojt9s1i4kUEr5t85Geo44zzWf/AGpEq/NvDfT/AOvUJ1ePdjcp9i2P6U406nWTHeHY88iAW4lG4ls8+5qveuHvWjb7pQHP4VPdYt9WmXb1Y9T1qhqgka4VkPzeUG9Mda65LQ5ludB4TaKLXYvNiSRwr7Q6hgflJ6Gu0uLjEsbLDAgVxwiBf6V5not85v4HTaJQ+078kfkK7CW6u5F2rJbA9ceWRnt1/wA9KjlL5jqra+xDGrHG1AOvBx+FWYNUjSQhiSAOa4tNRvowN9uJO37tgcfnipoNTdWbz7e5AxwBEW/lnmi8kJKLO2bW7cZwG9uKr3WsxywSxeUzBlI69q5b+0mYHZYXzZ6YiA/mRTWv7rHy6VeHnqfLGf8Ax6jnkPkiZkheM4jB3LwMDOafHq0kaeVcRFHPAbkijzHE58yAws3IjYgn9KnVfMi2PGMdAD1rZaoye5iKXTUGMj72dc5FZmo+HbjU70NaGIHb8+87QK29T+zWrRpHneORznA/pUumO32pACo3+vT6VnNtbGkUnozmk8B6k3W4tQfTeT/SrCfD67Jw97CDnHyqTXoy2LkZMijPPAzUn2Fc8sfwU1z+2mdHsYHnifD1cAvqLf8AAYf/AK9Tr4AtNozdXDH2VRXfCzUDJZuuegpBbAZ4PP6Uvaz7j9lDscCPAlopzvuGHocf0qaPwpaJtzaIdvqTz9fWu6FsmfuHI96kFtGM/L37jNHtJ9w9nA5aHTLRcf6Baj6W6E/+g1rW8UCDAGz2RQv8hWr9nQHp29OtOWBfas5XZcUkU1W3/vzn/toakMFqw+ZJW+pJq15aZyGxRhR3HXvUOLNVJFUWlmOPs/8A46BTxbQD7sAA+lTbowPvqPXGOaaXh6+aB9BS9mw50NMaKNqoAOgwM0woDjavt0qQzxD+MkfjxUbXUY4w3FUqbJdRDGg3ZIBzUbWpOMD9aebwIchGP/AqadQbGAq/nVeyZPtERmwY9GA9qF0pt4YPHwemDSm/k69M9OKa95KR94/lVqkyHURr/wBo6qDnzYSB/snig6xqoAxLFn121im6l6GVsexphmYj7+fatFFmfMjb/tzVVI/498f7S0p8S3y43rB374/pWC0mSfWo2OD06VaiS5G/J4quto2pFnuev9KoXPiG8ngkjxHuI4ygK5z6YrMO7kYx+NN6ZHUZqlEhsedX1LC7RbofQRCo21XVSeJ1Un0jFNPvnn3phwO5I7VZBJ9u1E9b2T/gIAqN5rp8br26wOcLJjP5UnPA7UmCGPXPsOtADS0mcma4Y9PmlY/1ojijJJMeaCTjjOaVGYHOT/jQIsFEBx5afnRSbv8AaT8c0UDOQsI3G8CF2yeMKT2HNSPb3TMStvLjrxGf8KqNdTZCx3Dhf7iOccfQ01Y7qckBJ3+iMf5VncqxdW3u1OTFIPquP51We0lRjIojU5JwZFGani0PU53HlaZeSHB+7Ax/pV1PCWvv9zRb75cZzAR/Olzx7hyt7IyntWZ94niV+M5lX+hpZLdJVG66gDdyCTn8hWzH4F8UTDC6JdADpuAXH5kVftvhx4rePYdMaMHk7p0AP4ZpOpDuNQl2OTeyiUYa9jPc4Df4VGyQJx57nscJ713R+E3iaYgk2ygY4eb8+gNWY/gzrL48y/s4x7Fj/Sl7amuo/Zz7HnYeNVGJJeOnyj/Grlvr09jcJLC8odTlTtGM+vOc16GnwXnGDLq0QP8Asxk/zqC7+EVvZQmS4109eES35Ptyx/lUuvSejZcadVao5n/haHiRF2pfThO+3YMf+OVTu/iF4huhtkv7lgCcfvdv/oIFdPa+BdFgGbhbm6PcO+1T+CirNt4L0GDLiwLAn7sjlvyrNfV47RL5Kz3Z5sv27xNqltbRoJbueRY4yzn5ieACWPTmvp22s7xLaGM27bkRVJ3DqB9a4LRdD0fT9as74WUFuIZlkMpJUIAevXGPrXojeMfDq9dZsv8AgMoP8q6abjUXYwqKUHqJ9ku8/wCp5/3h/jQbC8I+4n4tUbeNvD38OpI+emyN2/kKYfGujn7klzJ/u2sn9VrTlgupleT6Ep0u8b/nkP8AgR/wpf7DuG6zRj8CarHxnZEfJZ6g30gx/MimnxpFjKaVfnPQt5Yz/wCPU7wXUOWb6E8vht5l2vdj14j/APr1w2p6ZPp2rT2yb5ghBD4AzkZ/rXXN4ymIHl6RIc/37hV/lmvPfEuk3+u63caqIo7dZ9qtEJS2CoABzgZzionKD6lwjNdDQEVxx/q1PoZVH9akW0mcZa4t19jMD/KueXw/qCAA3LccDEhqZfDl28RdpwxXqN5PHrWfudzT3+xt/YiD815bgezMf5CnfZrZeW1OJf8AgDf4Vgf8I6zfekAP05qtdeH5gxMLyySY5VTgH6e9O8BWkdQF0tQQ2rqSeyp/iaa1xo0YO6+mYj+6i/8AxVcMbTB6PgHByTkU9bIFiWTHrk9PSn7oanRXWqaUsm6K5lAIwd8fX2wBWPNrtuCQkdw4HGcIM/mwqNLaIH/Vj3zxSm1iONqjHcdxRdC5WQ/2yrfdtZyw/vPGP5MaY2suRxaR/wDA7gjH5KatrboFJCpnsc4I+tOWOMZDLFjvgYp3QcrM86teHlLW1Ht5rt/7IKaNU1F8bFtRjniJ2P6sK1BFF1G0ge1OMYyAdrZPUUXQcrMtr7U3GRNGp9Vte34uaak+rr8y3UgJBBxFGOD1HINa2AB8r5GcdaCIvmG1gAOuMii4cpk41FwQbq49Dgxr+oSmm2vP+e10eepuGH8iK2CFIADA/oKYSiHrtx07UXFyoymtJ24Mku3/AGp5Dz/31UY0kfedY2A/v5bP5k1smWPblip46kYpoubfd6D/AGhxRdhZGUNGU/8ALO269oVz/Knrpwi+6EUd9saj+QrS+0RnkNuGfT/P9KjNwAQR5gPTmndhZFQWbIMefLz2D9aabHpukmOeeXNWDdN97G79DTftHUlXUj0OcUai0K7adGSC2SfqetAsYxkBMdeMdqnMzEEgBlxjFRmR1XC7SccAinZhdGRrK7J1kUDkDkcVWuZT/o7gnBUqT+P/ANetHUFNwseBkgkEgY7VD/Y8l1DiNgpVgaq2hN9TP0zFt4gtpCgZN4bDcZAr0CPV7IcvCUz0+QEfoc1wFmrNdwuCVeMkEg4PQ1pl5Sx2FjjnG7nFRy3KUrHeQanYyAbbmJF9GGz/ANCFXk8ubmORHGOi4avMWZimXIAbpuGOntj/AD/ICE9BktnBXr2/+tUul5lKr5HqAg6cDPQ8Y/lSmId3Iyex9/8A9debxy6moJinuOByBLwPwzVpNW1aAhWeIjPWRcZ/Ie/+eKXsx+1R02r2iuUdGG8epBNZq7hgZJboQB0qrHrckxCyRI2T99ZVGOeu3NXVcOSVxhvStoKysZSd3cyri2c7pXXJLZzjJwKSFzE6OFyVI/OtB45ZkbkeXnqTWYw+cDqAcc+/BqWi0zurO+hktlypJUe351ObxOcKT9cVy2kXJ8vy88dh39q1fM9QPrjisvZo19ozQN6OnlD86Z9qZuQqiqQBOeGx6ilBOOG/SjkQc7LRupMngD3phuJc8vj/AIDUALAcgHNKd3TA9iDT5ELnY8zOcgseaaZHODmT0puDnHIHuKXHsSPrRyoOZjd7EHO4n3NNyWGe31qTap4x+RpCnt/WjlDmITk9X59jQCwAxn8RUhUg9fz70woeuP1p2Fcbk5zuP5Um5jnmnj5ednPSkJVuoOfpRYVxhOR05pDIfTPb8KlAT1AzSBVHT09KdguM85gBuXikMmOuc1ISMgcZ9aYcZwMZ7nFFhXGF/rgUeYNuTTsDPGPwFIR0INOwriFlI6D8qQ7CP/rUjAnnIx160xo2zkYBp2FcdhTkf1ppCsDj8Mmja4JJYfUU3DY5IoANoHcUmMd+nanfPyMLkU08DlR7jFADCTnrmm9OM08uo4ZeKQMpxwx46gUXFYjJ69Tx/dpynnoKkCg/d3Y9SOtSJASMAYHvRcfKM4PJUZ70VZEAx0T86KVw5T3JdNtEUBLeFQOgEYqVbdFHAH5UUV4Z13YGMZ9KDCO5NFFUkO7GFFXtmkIA5xRRTsUmNYAdhSZ+XOBRRSLRXuJTFC0gAOB0rkrvzLyXzJpCfQAYAooqoFlUWiFwCTzSNboTk546DPSiitBCC3jkyrDcDwd3Oac1tBFIyRxIu04+6O1FFXFktCqvHGBx6U51ywX0Aooqrk2GkYOOvP8A9f8ApTgoaFT6MQKKKaEMCDn6Z6VLHGGLKTxtJ/IZ/pRRTAjaFQc0RoqyhlGCPXnNFFMRNNZxo6leFYBgMdM9qi+xIxYE49wORRRQBha1psP2M34yJFba4/v4bGfY/wCea54hQg+UfMKKK2hsYz3ImlVUX93kM2OT0756e1RC7ZYwQOr7eetFFaGdx4uXZlGBnA5x61FJcyJsORhvQY/z0ooqrCbGNdS7xhvmB6kCnLJL5nlvIzAjd6UUU7E3FbKMGLFvmA/OmuWDr87fN056CiinYQwfNGp6ZFKshAVG5LHGRxRRQA4KC23GDjqOOlCx4/iPWiikJimEbhgke4pme4J69zRRTAQFA20rndxnNRyzeWxIBJz1JoooAhFwJCD5YB9c/wCfWr8UAmXOQv0UUUUxD57AwRea0gcGQDG3HX8aVYdnAY8EYNFFNCZzX2cR3yEHo4B46812kejWcj7ArA/UY6Z7AUUVmiwfS7WCQKFckdxIwz+GaVbG1KkmFW9n+b+dFFUAgtbJSB9jiyTjhR/hVhrSOCMOqR8+iYoopDEUlgOelZ7Emc44ycdKKKqJMjO1HV5ba4+xxoGJBbfIxPc9hj0rHOrlnkWWBWKcZRtufwOaKKmW5Udi5o2s+deeSsGzgHO/P9K7TAIGc9AeDRRSY0L5YI3AkcZ/SjBHRj+NFFACgH5ufpQqbjwSOfrRRQAGIrg7v0qIsQ3JzzjmiigBC5BAODTkbcVXue9FFAEk0ZT5t+efT6/4VACSTz6DpRRTADksRmggfrRRQAGEgE7vwxUe35j0/KiikA0LuBOcYpdnA5oooAaQVzg8CnICRnd9MiiigCUQ56sD+FL9nXuT+FFFJjFW1RwSCwx6nNS/Y0Q4znAoopNjSEFupb+HJOM7aayBRkYHpRRSuOwwDgMDjOacke9sEj06UUUAhViVsjA9elOWFByQfwoopDHGJMn5aKKKAP/Z" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAIFArIDASIAAhEBAxEB/8QAHgAAAAYDAQEAAAAAAAAAAAAAAwQFBgcIAAECCQr/xABvEAABAwEGAgYGBQUKBwgOBQ0BAgMRBAAFBhIhMQdBCBMiUWFxFDKBkaHwI7HB0eEJFUJS8RYXGCQzYnKTwtJDgpKVorLiJSY0U2Nzg6MnNTY3R1RVVldlZ4WzwzhERVh1hNPUGShGSHRmdpS08v/EABwBAAEFAQEBAAAAAAAAAAAAAAMAAQIEBQYHCP/EAFYRAAEDAgQDBAUGBwsKBAcAAAEAAgMEEQUSITETQVEGFGGRIkJxodEVMlKBkrEHI1OiwdLwFhczNENFVJSjpNMkNVViZHJ0srPhRGOT8SUmc4KEwuL/2gAMAwEAAhEDEQA/ALv1/D+mrU9RULp3EdYlwdhxB7KgobOd4HK1esT/AJNvgdieprax1V/0r1e6t91VLe+XtqXnJAdacjUbbWts+Ppf8a3aE6QNotWgwehpHE08QbfoLI7sRqZ2gSvJt1VErz/JUcPH2SxdOO8W0gIyguvUtSNeUdW3Zp31+SkvRdMumubjBVBBiPSLgQToe9NVPw5m3o8kER77dgK0Ak2OaWNDEzl5Y3t+S74v0wQbtx5h6pCOVRTVrObQ/qNLHcN9rNev/J49Ia73HHGG8PV+YR/F7zLR05/xhDfKLevIHMmeeot2LDNG081LjnmF4w1HQp6S9I28tzASXlkmEsX1dzpzTuQmoJ2J5c7JFb0Z+kJdi2zU8I8YLSlUrWxdLz6djPaaCxyGs++3tmunYdP0rDax4oBsXXclzuHMu7KUnv6oA2h3O2xUuOF4bV2AOIFwPK/PeFcRUI0BNbdbzOU6/rpHht+FkxJqEZkOVHbAUJ1JkHeD79be7qbiuxAhqnU1H/FuKR9RFiN5YHwze6Si87tbq0EapqB1o9y5Fomkd1SE4Xh4PSoQFKKeZMazE7DysaSqpDjakFSg2mYjmPgQJ8fhb2OvLo6cG72GWr4dYaXzzKuSjUfeWps2q3occB61ZWcBXOkqBHYYcaHuZcQLANG9GZO0brx4oXnQlTSn1D6V1JzAkFWY+Hfz93fZTp3qhTNIlsKgMDtRO4EGO6PrFvUiv/J+cBX0q9BwoilUpZVmYvOuRqTr67rgHus2a78nBwvdShFHXX9ShsBKA1fDSkiBH6dITt/OsCOkkjdchWH1Eb22BXm51qw48ohakghBAHs9m/1WAceeyhLis0g5VZSDG2o93LmLegV5/k0LrKlKuvH18sgnRLlJT1MDzDjU8uVmjev5NLGTaEounHzD2QATVXSpsEAzB6t1zx5Gx+G4clWzAqmClqK0EuIIBgg6HY6x59/Mey3SFOqUtIeVAgcyCJBM8+X1+NrS3j+Tv400iyqnvnDNQka5QuubPeYmmge82bF49BjpCXahz0XDN0VmeSCzfNM2nwgPKQeQ3FmLHdE4I3UBmoqE0/8AKqBGva1yk8td+XzuMhyrQ6lPWqI19YbCPxtKF59FXj/duUO8K72fCf0qMtVY0E7MKXz8LN2s4O8XLoUtd6cMcYUqEJjO7clUhJ33JT88rQLSph101ELW+pQITAJBOU7d/hoPh7uw64AgEjK4JASZI07gd5Fhn7rvShQlu8aOopXFKKSl5ktntGNJHzHhblDTinE5wF5dwdddB3fOnlYZBRBuhUuOElICoGqgADEeR38LNS9aqpZxfRBBKVFCQTtAhWhPsGm+g3s60JWVFxKAY7IIjUe/x9psz8QEt35RupSFBQkKCtjnMR7ht3+Nq8ouEWM6p30Nap2mH0gXDhSnkdY7vwsIagpAVmKhM6HfQxv7e742K3Wos0YqQCVZ4TqBGke/577CrWUpQwEKPWdwBPjt4/G0IRZtkWQ66raqkeqFJAGpykg84j2ePK2OOrSnNvJAzAA677+zx87aSpClq7IOTWCDGvfy/A2DUpktlbqAc+wIjWdBr4a99jEaISEW4+XCorOgG4Anfb3/ADzD61zNLjvVmSqPA9w8tfd4W26Ek9XK5UTmIJA5nSd9if226DOZJJnKOyBm0yzudxM/VYZ3UguC5/J51KQokFI5jxG5iSfjbnM6peZYKYGyiIA7o7/OwvVLzBIKlK3JG233jQW7bYdSolJBGUCIAG0zqe6fC0C4bKSxClo3ExqdiByOu45WCfUp6nWlBIlJgFRjWIn2gGxkIcCk5kpBUYywrUE6ifLTe3bjDhbWHAVhSCqAT7CJGvP42C42UwLhMjDNS8wutQcxKi2VACJHaJBA89Dz08rPH0WmdCXwyEBUrII3TzPjoT7/ADsz7rdW1X1zCWz3wFxBB75nYnx9uhf125lXcwchyuIEyDqe/u+fbYDvRddFaLtsk782MuZAplQJ5SdTpyme73W5XdbeuV50TCdFCTPnzn5izhS0lCQohCZAI1gAR3ew/XpbpbLeUdW1mJEzJAnxj3WbiJsibZuf/AoqFHQZgRt9/wButgDcThJKy2oEdnOnTbWPZv5eFnUaZpRKSCO6NyO/Ty+d7Y3Slaxl1MTM7b6ju+8m0TLlUhHdNdGGFPALUxTrUIB1KZHedNNvk2x7DS8sIowYUTCV+G594HlZ7oogIypGkjug8xrbhbBUFJgAGJ5xEz8fssIzIgiCjty4Kg6IpHVfzgRKhI0k/s+wo/h59KgpxuqaAVrmQRBiANfmJtJq6cpBKk5CgyMwGh+/Q/HwsC/ROdQpwIUYSYGun1Tz+NoiUpCEFRKgPNPQBmglJEEjQzMeZ28Bz2UWqAvsoUsNpVyVO+u3nsffbV6M9TUrlJIKzBB5kgz56fDzs9MJs09VcjaXWG1QtbagU5jvIHxgWdz9ima3kmPVYZbbYW9U3fSuBMqhLaSTHgfb7xtNo6xjcRapVVVNdSWQlQgtN5dNe4RBkePstPt/0tEUKpGmEhIMrKQQJn1dPb9vhHGK6IC56lLaeyhYUTv2pH3+flbSoZXNcDdU6lgsbKEC082ZShYjuBFtirrESA+6JOYjMd+/6rORynJPrr+kJ1UOckg6fOx2sV9GC0FA7WhIlMiBEnX2e63RiUHdZBZbmkU1tZoovKOWAJ1i2hVuz2gDz1FlNVIjIHChJkHQJG+un4eVuPREqRKmMo3J2G+0fOvstIPao2PVJpqVkagSeduvSjkyESAIGvz4++xlVI3llSVJgTprpt8/IsA9ToQ1mRm31m0gWlN6Sxxx2EuLzdoQCTytx6U72u1orcd9h3vpKSmCieylSB5hRP8AaFioZUQTI0E87OLFI3XYqSk5gkTzt36asiMoEzsYsAW1jl421lUTASbPYJsxRlqpbSoFaOzI03sZVeTZT1baVZdsxMGJ8LJxbcA1QrbutzB7rNlBSzFHfSW1ICOY7xpMncW0Hk6gkTI1mLE9YtmvIWWUJXQ1Qc0Kka91gpmLatlpDRMShaaodpqlqpaWUraWlaSNwQZBt9JfDfFJxdw7wviptYUm+Lmoq+Rz61hK/tt8143EW98+ghiZWLOiLw1vJbnWOU91Ku9Z7vRnnGAP8lsW5ntLHeOOToSPP/2Wjh1i4gqflVLkb2CNQ7uDbk720rW3JgrYDW9FnXuE+tbOvc1M+63Mc7agmxgp5QumH3EvuIKtCEq+sfZYx1zs+sbE0kpqEHbMhQ9xFhzM26GjOaIKvKAChuudOpUbaD7v61uJkRbDJ52tAIRK669wa5rZ17vJW1g+ds252kBdNuhfSHTpmNk+8XnA/SvFZ0WUHXkRY57vZYleqVmlK2yJQpK58jZOGiduhXXXvR69uTUPD9OItoGUyBvbhQ9tmI0T7LsVLpET52HTUulEzyHKxSNDI1sKwJQB7LV3NsiN1QxqXogq+NkHFd4XrR4fvS8G73NEaWldqErRT5ygISVagzm25ATrZXeX1aZCCpSjCUjmfsH3WQcS4ZbxFclfdd7VT7iK5lbQCDlQwSNClIjMQQD2p1HLawXMMnohTJDRdV9rek3cCLpu++7wuR6uq64MvpYWwwldKHJBdzh1QSQNS2oIWNlEHSzd4mYnwReCa1FRiO/F36txNI7RpC3DRnMex1CYbbUr1ih1aFBJkyd2riHh9c+JcMX7hNq9HafEGCn3KP0p3q1KUylIK5IUAQsnNJ1BGXXVNq1s4qVS0lBh56+mHbzuityshDaFNMJQ0tWRRWkhaUr1BKikKiJmLZ1Zh4lgc0m2h81nurZGOAIuCp4GJ8BpAS5T4nzgQqLwpkCecJTSEDyBIHed7ZZuM9IOpbZQ3VUl7reSkBxSUpIKo1IKVgET3ADwG1st5wcNrb/N/O/7LR4sPVemj4+l07zbtsBQi3D5PXRE99u0HTf9tvpB265FgsEMkd8e23cACxCvvm6blbQ7e96UlChwkIVUvpbCiNwCoibE044wV/533J/nBr+9YBexpsSrUdJUStzRsJHUAlLwEH8bbAjbblZDGOMFEj/fhcg8Pzg1p/pW6TjnBW37r7k/zgz/AHrNxY/pDzU+4VX5J32T8EuACdvbbobbGyEMcYK54wuT/ODX962xjnBX/nhcn+cGf71lxWfSHml3Cq/JO+yfgl2NrYJjc+VkP93WCv8AzvuQf+8Gv71t/u5wST/3X3J/nBr+9ZcSP6Q804oKr8k77J+CXQTO9tgk8xppZBGOcFDU4wuTv/7YM/3rdfu6wTr/AL8Lj8xeDX960eLH9Ieal3Cq/JO+yfgl2YB0tvwFkH93OCtf9+Nyf5wZ/vW3+7rBIn/fjcn+cGv71m4sf0h5pdwqvyTvsn4JdBGxtvlrzshfu6wTt+7G5P8AOLP962v3dYKO+Mbjj/8AEGv71lxY/pDzS7hV/knfZPwS8YB00ts9ofCyCcdYJ/8APC4/84M/3rb/AHdYJ54xuT/ODX96zcWP6QS7hV/knfZPwSwumpXNHKdpf9JINgjdd2b/AJvpxA3S2AfeLJn7usER/wB2FyDlP5xZ/vWz93eCQP8AuxuPb/ygz/esuJH1CXcav8m77J+CPu3LdryOqdpyUfqlao37ps3rx4S8N73UV3ngq5Ksq3VUXey8T7VoNlH93WCYj92Vyf5wZ/vWz93OCh/++NyEf/iDP960S6HqE4o6wfybvI/BMm8ujFwRvKQ7w2w0iZP0dzUrftlDaTzPPnZm310EejffbqKiq4e0aXG4CVs1VYzHdoh8CdTytNH7usEg/wDdhcmv/rBn+9ZSob0u69acVd211PWMZsodp3UuIkbjMCRNh8OF5sLFM+OqgGaRpA8QQqy1H5PPgd2xdzV8UAOoTT3mvKDG/wBKlz5izavX8m7gl5ZXdGNL/pCEkJDr7L4B01gNNzttItcgERM2yeXcLP3aPohcd/Mqhlf+TWvFLTiKLia6tS9e1cyAnykVM8hys2ry/Jy8TEls0GLrkeSzqn0hipbnSJ7CHOU/C3oyDIibZIJn7bQNHGVIVLwvL69egVx0oEPrpm8PV6yNAxXONlXcfpm2x8RZqXh0P+O91IT1mAWHWWyVLLF6UTugndIdnXXlyt62zBtwtDa0w4lKht2hYTqFp5ojat3MLx3qOAXGinaVVL4T4pdb3Saa7HntAJnsJIgmyDeHDzGtz5G74wlf1CQZUam7n2R4gFaR4a29mWbtu5TCUroKYgCI6pP3W3+ZbqIkUaUk7lClI+oiwDhpIuCitrQNwvE9TAS84XlwW9gVeGp118rYaZwskodKnFyE6DUe7Tlrb2nq8LXFXAoqaVboOkLdWoH2EmzcvDglwvvWTeGDLlfJO711UjpHtW0T8bAdhknIorK1nReIBYXT308yNVLQrKSdzI5fPws/8Phblz0yVKISjMQdUlRnbeNvdb1QvPod9Hm9nC9V8M7h6wgjO3RBgwd9WSiPZZFf6D/A0tGnu7DyKJsz2WKqrCte4qfUPhYEmGzEaIrKyLmvOJIPVZy1oAAOzAJ5GPP4R4WERQ9aEskq17RUfITI3G452v7XdAfhk6rPQXnfrGUHKkXg1l9yqdR+NkN3oD3dTKcXd2Ma9BUMqesp2nsqY29Zu1N+H1A2CsMq4OqpYm7aNThKkIAAIHnzJFhmbtZQAFJHWOGZSqT7O8effa3Nd0Fb+bCBQYybeCTm+nu8NiYOvYeXz8OVm9ePQ04kUpdcp665X1mAgBNUJ8/oCO/nanJR1LdS0qyyeA7OVaDRtuOnq8yQ0kAqUfW3Mgxy52DVQukR1pClyRmTHlpI+Fp4reidxdpsobw9R1CU6qWm8GEEnyWpJ+E6CyBVcAeKlCXHHcAXstQEIFMgPgd89WVA9+/L2WrmCdu7T5IgkjOxCiFy7lFakoyEAFSlZSAPAd8iT5RYu7SvGmIIb7UjRcE7iNucH32kyv4W46oVpbqsFYjpUpSFOKdu59I74kpj3aWalfc9TQKAqmalkpEJQtGVRPM6jx3iwjnG4RBl5KCsU0zjL6kqQjNmClFKpgRqT3H8LLmDbxWzczzYJWtLxy6QIIGvjtvYPGdO40mpSWh1zozKIIJgEad0R9dk/BhL1DUNmYzJIBMxoZ38QN7XWNLmqq45X6JYqFFThWRKlfrHxnfykad8WaGKKdtyhrUuiQlucxPtO3LTfuOnOHgtCygqkZiZmN47u+LNy/musaqUdWcq2VAa5pGXz1/HcWu0/okWVaYXCiR5swShJWUgn1s067x7/GTvZOIUoFrIcqhJPiPriD5+6yopWUJSkuAkcoUDPPw2gyO/uFk1RQ4iQDpEg8wOfz3W32LIcgnezmJ5Aamfh7B+Fg3UpGUrCAoaHtb/ADpHlYQrTKlNrBV6qtfWHMjw0+FgFKXKc6CEq2GhMSeXLf42KAolaXkICeq0Ha8hyHv+e4tVAFpwJiQQSSefgfnnYZeYJBCVHeRM7c5sE52kOBWYaa6zr8xYoUbckGlU3U2cmjVQrtf0kjT/AEbdw2vKggKzA6g6zy38h8bcU6Su7atGU9hba9Bt6w+0W5RmWBIO8HXQ+3laRTclstsxBG5gk6GdvgbZlSghKokdxiIthWSZUInxgRET9VuSCo6Aajtd43+6ySWygKnmNI115799tKaSklMmZgEj67dErzFCmyQdzz0tzMQVaE7Tz+6ySXOUAnQEgR6u1uVNawoDXaTYTNucuaYIBVpA+u2lRPYA17jHyLPdNZALQlI0387B+dhlp7M62CAOo8LSBUCFob29mPySuKVX30Xqm43Hcy8PYjrKVKf1W3ENPD/ScXbxnG4t6dfkZ8TkscTcGuO6BV3Xm0if+ebWY/q/hbHx1mejJ6EH9H6VcoHWmt1XpeqRubc5gNrbXtYM+e1uHC6FouEJvtbmItiDsIt2ADtY4F02yDUMq2lGBCo94iwx0sBUfyRP6pCh7DYxtp323KA/i7KvMOazS29uVsE87aIBO3hNr4CBZYYi2hqbbjkRbRnmbPayRWyTt9dgqhAcZcRIhSSIsKJOgGttKEjTxtMNuE17JPpzmZQSZ7NtqkmYsFSuoX1zCFJKmXVIUJ9XmAfYfqsMZGnfaKley5GulhGP0gdTobBjc91ugopJyASQYkwCe75FgyWAuVKMm9kOltbiwhKZUdBYu9VXUhp9D9e0nqwVHtglKgNdBrPhZUpRdbLXpN5VNOApJI61wBIHkTFosxHxNuBGN3cCG8qinceYFUhtghGdsaLDhAU4AAc2RKZ/XKQDbOlxAUjc5IB5X/T0UiOK7KFVfpMOXldvF792uBK+kp2nrt9FvB2qYcpG1ukqbLS3XEpTJRmgg5hBEaAWrdh7hHkvNrH+OMaXRhy5bxeWmhZceDf5wJUSopABWUQYKkJKc0esAYvHxf4hXLV3G1c154fq7ypHyh+moGLtVT09KJ7Cy6611KloEKjOESCmTakeIeD2LcbA1+JsWU9J1763k195OCkBYSIAAUoBwhOhyAjQjWCbYdRijpmlrpAwHc21+rdCfTtY/MG5vBWBoru6MNPRsMN1FK6htpKEuLpnSpYAABJFUkSf6I8htbLQ1SYVwzS0rNK/jC43HGW0trX1FX2lAQT/ACY+q2W5I0Dr/wAZf+31K1xf9QL1heB6zU8zbtIMc9LadEO8xM8rdoAB+NvoZ265Juyq107KGnvS7+Hl2VgUWKvEPUO5TByLSlJg8tCbKdR0JeArIecNBfYQ0CTF4q2AnmLFOm6THDM//wBTo/s2sFXJLjVU0TlC0qQTG0pj7bYNPQUtbiNSaiMOtktcX9Ves13ajGez3ZDB24VVPhD+8Zsji29pBa9t7XKo5w5uXoNcUL8osNXDV4rorzvFQbpae8i6yHFkSEZ05kBR2AJGum+lpxZ6EXAcnK7dt8zyi8lfdYzhfov4QuO9bsvpTrFQ7dz7VU26lrKorbUFJO55pHOz1/fCbrsfKwjQ0lcj83Uxeq3qilcYQpa1DIlvOBnEJWcyZTtBMGNP5Cw0j+Ab5Bcp++X2vv8A5ym/9R3xTNR0GOAC0g/m6+tf/WavusXf6EXABtWRu6r6WR6xN6KAHwtYGncIZnnvaOeOdJWXtwnxDdFDUOMu3my1RFbaoV1bzyEOAEaiUKUPI2gMCw6/8C3yCkfwl9r/APSU323fFRxRdDfo616S7R0l41TSVZVLpr66wJPccoOvhNlRvoN9HtYB/N1+Cf8A1mr7rM/o4cFqzhRjhN43XVutXfedK7T1dKFqLSiElaFQf0gpO/8AOI5m1nRWparfRSRqgLHtkfZZ3YDhvKFvkE7fwmdr/wDSU323fFQs50Fuj20guLoL7CRv/uks/ZYEdCHo5mCaS+wD33g5/dtPdW6TTATupI+ItXLpUYw41YRvDCjnCS/GKNDrVc5eFM9QtPpqigsdUCVpJSBLg7JE5tdrQGA4cf5FvkFP98vtffTEZv8A1HfFLDfQX6PDolukvlXleitPhYX+Af0fuV332f8A3mr7rSXhC/K6uwpceIL7YRS1tfQ0z9U0AUht1xCcyQDqO0qIs7GH8ySQdCLQOBYeP5FvkEh+Eztcf5xm+274qCP4B/R+/wDJ1+f5zV91sHQP6P5P/a6/P85K+60k44xO9hTCN/4tprqdvN66aKprk0TTnVrqS0lSg2lUGCcsTB32tBuG+mU9fTjbLnBrGbDioSU06w8UkmP1U2XyFh43ib5BSb+Erte7bEZvtu+KdP8AAO6Po3u6/P8AOavutg6B3R+j/tfff+c1fdad6OuXUtId7SQUgkK3sTqr0ebXUu9eW2qeZhM6BIUTtPO0RgdAdoW+QSP4S+1o3xGb7bvioU/gH9H3/wAnX3/nNX3Ww9A7o+/+T77H/vNX3WX8A9Kbg5xLvZi48FcRqO8LwqZLNK7RP0y3YGYhPWoQCYBOUGYB0tLVNeSnQC4iDzA2sjgdAP5FvkE4/CX2tP8AOU323fFQN/AP6Pv/AJPvv/Oavuto9A/o/f8Ak6+/85q+603XviOkupxhp5TaTUZsnWOhAMASBO51sGjE7TgCk0ylA80uBQ9nfZvkSg/It8gn/fJ7W/6Sm+274qA8T9B7gNdWG72vSkoL5D9HQ1D7RVeKiM6G1KEiNdQLKfQOMdH+k0/+1K3/AFhaWcW34xWYQv8AYSw6k/mqrMqyx/JK8bRJ0DyP3gKQf+tKz/WFqDaOnpMTjEDA27HXsLc2rpZu0WK9oOxFY7FKh8xZUQZc7i612S3tfrYKxQV32yeQ9tuZjlat2OOmldGDcdX3gZrhpiC96i4qgsPvUa0qSrQdqIJA1521Kurgomh07rA6Df8AQuDwLs3inaWV8GFxcRzBmOrRYXAvdxA3ICsmpUn8bZn8bVZT05kLGZPArGhHeG5/s23/AA4+7gTjaP8Amv8AZtS+W6D8p7nfBdH+9d2s/on9pF+urSZu+2wrvPvtW7BfTOuzFuO7jwHVcMcQ3LVX5UBhl2uUlCU6HtQQCRpytY2Cefna5S1kFa0ugdcDQ7/pXOY52bxTs1KyHFIuG54zDVpuLkXu0kbghY0eyRPMj42Ez85sXa/TSde2r67CAgjf32sBYhQoV3ga20SQdJtyTp5WxJ1IEb62dMF1OxM+62wvQc+6wZ+M23IGwsxUwboUK5xpbMxnusEkjc23JOnfYVrJWQgUNxp7bZnM6WD5m2A+NkkhM0c/bbS0NuDK4hKgdwRNgwedugrXvsxARGlAm7bsWQo3fTkjY9UJHwty7dF2vJyLpyUnkFqA9wNjA20Ftgj22hkb0RszuqbV6cKeG99lSr4wXctaViFGqoGX8w7j1iTNmnU9FXo+1ec/vV4cZK4zej3aywSBtq0lJ0nS0pDvtsGDaBgjPJPxHjmoHvToN9Ha8syk4L9FWRGanr6xJ9wej4WZd9/k2+B16lTlNeWJKEq5MXiiBvt1jLh5nna12Y6SZAtsK00PnNod1i5BS4z7WJVA74/JF8PX8wuTiXiakBTCfSBTVUeYDbRPv52YF9/keb9Smbh4zdYqSQKm4ghI101RVKOk/q8hb09SobxbtKxqLT4VtioX8F5AXv8AkjePtMVruvG+EKtKZyBa69pZHsplgH2+2zBvv8mP0r7rQtNPhW5LyjMEqp77YaEf/lBbI91vb4LB+63SVDflZWcOaXonkvAC+egz0q7nK/S+C981BEgegP01ZMDeGHFzZoXv0aekLcgUq9+BmPqdKdSteG6woA78wbKY9vnb6LVtsuj6VtCv6QBsXF13UTP5upgY5MpFkHvG6bhtK+Z57DGJLnTWMXzcN5XeoNHs1VI40ZChocwEc7I4SoNzty1t9Or1xXVUJKHqcqSd0B1YSfYDFmve/BLhNfxUb64fYfr83remXWxUT/WIVZ+M4ckjEDzXzZlDgjNpproRpym2lKJnsQCdpt9C18dCrou34kit4J4RTm1mnuenpz72kJPxsxb3/JldEO9ypX719PSKUZmmvGuag+Qfy/Cy7zbdqjwfFeEkEHspiNtbbKXFCEpMTpGtvam9/wAkj0Y64f7nNYju6NvRr3O3/TNu2Y1+fkcOGTqnF3BxDxTSAjs+kLpaqD4jqmpG/OzGraNwUuCSvI9CDlzDQzrIjT5+y2dUiJKhGmo13FvTS+PyNl8IBVcnF9xwnlVXA2B4SUVZJ/ybMO/PyRXHagKzdOMcI1yf0UvIrmlH/IYcTPtshVxlPwHqgqko/SJk+NgSkgxa4V//AJL7pYXYIosOXBeW2tNfTbQJ/wDyjqj8Odo+vnoEdLi4lBdXwYvKoSVAfxGtpKw+5l1RHtFitnjPrBRML+ir3qLXf/JG4jVdXSTvO4lO5W77wzVNBJPrLadZdHwSu1cL76MHSNw+pRvXgTj1htP+E/c/VLRH9NKCn42k/oGt4m4c9Lzh9U3/AHFed1oq6167HPS6RxmfSGHGkg5gP0lJsGvyy0rwDyT07XMmabc17kKMggW4522VamLaHjbz4DVdKFnssIgzIFuBr4W2BrY7Eiu3EZ21IP6QIttpWdtCjzSLYk5hB3txTK+iUjTsLUny1kD3EW16A2JCrS6BCj2W3FtCDvpbYmNLaoCr3Wc5m2hlgcrbJ03tke2LOEyyCBIM25URtbek/abYZ2FihRJ1SahIRW1CEpAzwvQb8rdq31GptzVAiubWD66Ckj427VKhMzaFksy5G4IG3Kx+6ruFW+X1tpyta7bq5CxFsdY+3TI1ccMJTv7fIWdvUeg0JbpU5lJRKdPWV3n22ryNadTyTPkLNBuU274obqvd1ZxI8wzc7Kkthtx3q0vvKMQsyJAJCQmdVEyDAtGXFrh7w4uy4K+vp6c4bTUw07eNPXO0DjwUQVJGRK3HlGE9jIc2utppeDFJkWtrOGyCgZcys0RI8dT7zZHxPhKixldpu/FVIh+gdWlw0QWQlRTqA4UkZhMHL6ums2xaqijkY6N7Q5zudtvr5W5J2SlrgQdFUppT17U4au69MeX9d1I31iV1jLj7SlJSskMtKbcUhUNmAci8vZHrQa84odwliHG1PcFXjbFbtRf1cqmdoKmj6tbDZKc7mROYJQMs9pKAAmQlyDF8+Jt91NZddJdWAaBSGKF+nSiopmAppaQotrZZCSAmEggqUUo2SFTJTUPjfjNoNX3VYJvi6bsvh+6gxXt3UW3allbaHGAhyoQZAHWoU3kyhQ6zSRbiKmi7tNla4uHXkD5HZa7bPizHTwTLr2MDUtdUU1P0S+JCmmnVoQVO17aikEgShNOQkxyBIGwJtlor/MHGVXau2qvkUh1pw68kLDf6OYFQgxE6C2WJ8nwc5D9tyz+Ofo+4L2OfEPRrboDSPjbb6B1o7zbEjSTvb3Zx1XMt2VYOm5H/AGMv/wC50f2bTrioKXcF8tgxnpKgb/8AJm0GdN5P/ey5zihH9izw4knjWxiioocNVdxu4dvDK31dQwoPsoUkJc7Q3M5iO6RO1szD3NbiFUXG3zP+Vd52lY53ZLBAB/Sf+qFXzgfwIxfw1xjcF/XJi+8hQN1DLdXRqqCWnmCQlaVDY9k6TMQI1Ei3V7NsG8qF1bYK8riUmNQOzp5WLsYZq6O4XHKFLaq5plSmEq0SXAJSCe6QLMPBeNOJ+LMSN3Vi7h3TXRT0iFr9MarQsFYgZQgmSDvM6ZbbpkZpdy86Eb+immlMU2+yedm/jNv0jC8J7QLtMoz3dYnWzlpGFJYyKHKNPK0VYj4p3Xw8r3sOcQrtr2rucKjSXmxTqeYU0TORwJ7SVJ20BkAHSyjcCdEz2kBO25mENO0y0pECRtt2FW7qqjPicoTEIpW5155lH6rR270kuB11UvXs4ucrHAIap6eifLi/5ozIAnzI87ODhrXXri5L2La+73aFN5ul1mme9dpgAJbCp/SypBPiTZ5Hi+qUcbgFIj5IpWyR/hEcv5wtCPSJ6QaeCF44cp3eHL2Jqa+G6px19moCFUYZU1HZLas2brDzEZec6TNiC8KK5LndvS9KxqlpKYoU686oIQ2nMO0onQDW0Z4pc4WcRaqkqazG9wPCjbcbaSmtYcBzkT+lp6otAG+ykR1SrTvUHFDD+G+INzXveTFGtlN409GVBDTwUg6PJAkqTJjtQCNuYf11uq6lIVMxr32jJ/GuBsG3VdWA8OXnS1tW+pFHSU1K+hxSEEytxYSewkJzHXcwB4SRdAWqnQpQiR7bMd0tUjYqq7iu/Cl93hil5TNz0tJVPXg6UqVlpkpUpxUJBUYQCYAJ00FopwvxZ6MFU8y3hvHFD1zykoaQ4y+gqUSAAM7Y1JI0m0t4hu5u/MN3xcamw56dTVNNkVEKzhSYM6c+doywl0fcJ3FXtXi9hyhS80pKkq6pCiCCCDI8bCk13F0WLTnZTVQn6MBAGWNxZOdGY17atlOEE+baRZTpEobaCArQCLJqkAu3gmfWUeX/ACabTZtZQfuoMwn0YsNYWxRc2KGhTtVd01CKlDjKCFLKZMajYzB8LS1c2MKW977rLuoWapCbvWGXVVFMtjM5AJypWASkAp7UQZ0JGtqo8LuD+PeG2NLgvq6Mb3u7c6axtuqoHqpa2nGVkJUkiYOhnWY8wDa3ZS0a1hZTCylQCo5SNPZJsnDKE7ddki8W+FlwcUblYob+ow+qkUpynXnKS2pQgkRG4EWQ+FnD+kwNgCtwhSKWlpqorAklRJAcGbcmZ7RtLDAzMgEk6WQ0pSl+8WwN3ife2n7rMNQblPexAsq+8MOE954BuLEV5VOJr6vBLlx1TCGqyrU42gFskkJPPsjXuJs5+giQOAVKDH/bSs/1haQbxYbTgW+ilI/7VVPn/IqtHvQSTPAKkO3+6lZr/jC2BMCMTiub+g/72r0jDrHsPX2/pFP/AMkysUVWpVhqVdI/jIAdPSackDn2xpa6hO8aWphhJOfpIcZydYeZOn9MWfEBeopv98/8jlHsef8A4RjX/Dt/68Sm/AuI/wA3uO3a4pXVP9tAn1Vxr7x9Vk3DF9VAxomtU+6pC6p4qGYwZzDbbc2i6sxhXXNxrGDKmUUtdh9i8rtcP/jDT7yahHnkLKvYqzxwc7UKrGnaltTbnpz+h7uuVlPtTEeFtTZgBXAjV1wkjje913Sv4MqknVz/AFza1IM6D8bVM4xKKulRwZkz/Kf65tbEGBbJoP4zU/74/wCVq9A7Y/5owX/h3f8AXlXLXrOTyXz8hZKxo4tnBt+utKKFouyqUlSTBBDSoINlRsgLcGs5gY9gskY5P+8vEGn/ANl1X/wlW0ZPmO9hXFUOtVFf6TfvCoVwj4T3hxDwBRYvvDiji2nqaqpqGVsMVqsiEtkBJkkkk6+60o0/RMo3rkN9u8ccbtobbUp1AfzFBTumc2v4ixDovA/vLXWeRrq0c+Sx99pMrMTMKwviJuiqgsUzL7TyQofRvNo1BHIxB8QRbmcNw2jkjhMzL5mgk3O5HtXs/bbt72jw7HK6loqjKyOV7WjJHo0GwGrCdkw8O9E2ixF1gY42Y8ZU0lKiHHk6z5LstHoRpiRx2xt/W/7VpPwW+qjqgltZIca9+os6cYX49dOGqqtZP0oRlQQYhSjAPxm2hTYLQysuY9fa74rlKj8J3aqJ9hV/2cf6irbWdFbC9BUqpKrpHYwDqDlWlC1uZT3EpJAPhvZTu7oa3ZezPpF29ITGFQ2NCW6gHKe4jNp5GyFxIuviBfd2sNYAv83XUtqW86sNBRd07KJMwNyY128QXDwIxbiZVPc1fiIKbvJThorwSABmIWUEkDT9VWkCRpYTcHo3N4hj0vbd3xUz+EztSDkFXrb8nF+ohV9CJKE5lcd8ahO89d/tWxHQiQtAWjjzjUg9z0/2rTrju7qnEWDL0uiidLb1TTqS2Qop7Q1AMciQAe8Ei0O8J8YuXdWJ6+Wg6PRKttR9VxJ0JHIg/BRtad2foWsD8ml7HV2nvVZv4Uu1bnFvetf/AKcX6iTR0IQTA4742/rf9q3X8B0ASeO+Nh/0v+1Z31WLbzvvFKF0961bFH1qWkJp3lIzNg6nQjfUzvqLTCivSunU/mIRE9rutXjwfD5XENZt4n4o0n4TO1sYBNXv/wCXH+oq2joQj/08Y2/rf9q3Q6DoIn9/jG39b/tWdPEHjXU3Bd9ZiFl+sRddJAQiipw8+/JABA8ZncAASdpsq8IONLWO6CgvFmrVU3feYIZcdbCHmnASnKsJ03BHPkQTYfyVQXtw9L2vc/FFH4Su1oF+9+Nskf6iYQ6Dv/t4xt/W/wC1bX8B4af9njG39d/tWsZfl+s3Nc9Ver6SpFK2XMoIBVHL26CzKXxgoqYNm8KRinDhyoz1oTnMTAlInS05cHoIQC9m/i74qMX4Te1sxIbVbf6kX6ihq/8AoXLue4byvdvjpjVxdDRvVKUKegKKEFQBObws9Og5ed43rwHpaq9K+prHzedYku1DqnFkBQgZlEmz+r8bUGKMG4lYpWVIUzdFUuStKgQWljlaNugef+wBSD/1pWf6wtVhp4Kevj7uLAtdfU8i3qtyt7QYn2h7G1cmKyZ3MnhDTlaLAtlv80DewVjQvTSLbzeNoM6YeNMUYE4L1OIMH33U3VeCLxpWk1NOQFhClHMNRztE1BgbpIVlFS1T3SuYpl1NO1UFpxC5bzoCgkmIkA2u1GImGYwsjLiACbW53tuR0XN4T2Ojr8MZitXXRU7HvcxoeJCSWBpPzGO09IK5oX4266zlrao13cIOlBeiii7+lbTPqAkhoFRA8QNbHjwH6WwSVfwndB/yC/usP5QmeLind5t/WVg9j8JG+M0/2Zv8JWtCwRobdBzztVBvgR0tnEkjpOnT/kF/dbY4FdLVSyj+E+ZH/IL+6zd+n/o7vNv6ycdksJG+M0/2Zv8ACVrguNffbfWCJBtVI8BulukFR6UGn/ML+62J4DdLdQlPSfkf8wv7rQNbP/R3fm/FP+5HCT/PNP8AZm/wla0OHQcrdB0cjaqJ4C9Lga/wn/8AqF/dbBwD6XJiOk//ANQv7rR77P8A0d3m34qQ7I4R/pmD7M3+ErYdaZ1Olt9bO1qofvCdLkafwoP+oX91tfvCdLkDXpQGP+YX91m79P8AkHebfil+5DCD/PNP9mb/AAlbArJtwSDvaqX7wvS6P/8AFASe7qF/dbP3helyf/4oP+oX91m77P8AkHfm/FL9yGEj+eYPszf4StWoz3e6wLiWnJCm0qnvANqs/vCdLn/7z/8A1C/uto8BOlvz6T//AFK/utE1k3Ond+b8VIdksJH88wfZm/wlZxy76FRzKo2Ce/qxYBy6aFcEtFMbAOKEewG1aDwG6Ww36Tp/qF/dbR4DdLbn0nP+pX91oGpkO9O7834og7K4WP55g+zN/hKybl0U2Uht19HiHSfrmwJupcy3eNR/jBBH+rauB4C9LQb9Jv8A6hf3W5HAbpZkgfwmd+9hf3WATm/8M7834oo7L4YP55g+zN/hKxy6KuT/ACda0e7MxP1KFueqvNB19FX7VI+w2ptxOpeknwUvPCVViLjrVXzS35fLVCplhJRAzJKs2Yaggxa62nttKmjiqXPYYy0ttcG3P2ErP7QYC7AoKeqjqWTxz58rmBwHoEA3zNadz0RMKr0amjbJH6r33gWIXTU3u47VqqLvaaUp3MG+ukgRG4BB9X42WjPOxVsdXWLExnEx5R+NrjaRsRBauY4xcLELpLtVHapD45XAfuts1BTM0r4juSD9RsPsJ5G28xHcLWww9UHMixrWBooOp82lfdbkXjQlUKqkAgbHT67GzrqbagEQR77NlcOae4QQqadwSh9tXksG3RM7Ee+3DlLTuCFstq8SgGwC7ru5SlH0NoFW5CY+q0vSUSQEXvd9NOlh9QVCXkjQbAmLcCpNQot00HLope6U/ebEr/w829ddSihedYdyKKFlxa0pMb5CqDz0sfo0JRTNJbEJyiABA27rL0i4hI5QLhOK5LoFCg1TiSXnfWKvWj7LLSnEpZLygohAJISkqJHgBqT4WRqa/WUUyEVGZTgEEx3bGzNvfinizC4cqbw4cO3hRl4oafuu8EPFKIJBdQ4ltSTodEZ9YAkkC1Grq4qNhkmJAHOxKEIZJ3WaNU/n2VA+n1BU32ICCrstp8eU959nK0R8VsY4mv8AcRgvA1IyUpKXa+orFqQ3VNZoVTtoSC4tJkFakwMoKcwKtG5jvjpfV8VSMLMsU1xuuKS4Wi+ldW82BqEpMBMEgElKojYcmlfGPa64rifRTXbcjVTeTuSoXWLLqnihIhTjisynNCAM0hIBAna3DYlj81Q9sdCDwza7ranrvay2KfC3MZmntfpdA4kYvVFx1d24twJV9Y8FmrvCicZvFsmAAo0+ZDLcNpSlKCFFICEgHsk1q4g3DwnwybtvDC1beq6C8XEsulbRbdS9o4lx5teVKUEqhKUoSIUTrNpjvfH+JEUR/Md5YeVVPoU2hFNdzSm0OQSlRWGwnIkwZKhrpBJgw1f2IqKqvbrsav1WJ74QQhK2SqluwiUkAoQnrFZo1gNuEpPZUU6ZMk7pJNAbdFZkibGy1wSonrOIfE6gq36Gkw1iaoYp3FNNOr6panEJMBRVkEkgTMCbZadF1eDlLUpjC4DZJKOruzOjLyyqVmUodxKiTzJ3tlofK0X5D7/iqfAf1Xo5WpAfOmmwAtwgTrYevH03meVgkjQEb29/3XIqr/TfB/7GMj/96Ef2LWZfom3lSpMm0c8duA12cdrrui7LyxDXXP8AmiqVVtuUraVKUpSYjtbRvpaMP4Dv/t2xt/W/7VsIispqyWWKLO1+X1gNhbmvUI5OzuN9naChrq/u8tPxbjhPkvnfmGrbDYeO6ssmlSlGUJ30sWRc1O2+X0NAK5kDW1c/4Df/ALd8bf1v+1bP4Dh58d8bf1v+1Ypq6/8Ao357VQHZ3smP56/u0vxVm0NZUxZMvnD1FfDRZq2EOJVoQpIINq7/AMBz/wBu+Nv63/atv+A0T/4d8bf1v+1ZjWV50NN+e1IdnuyY2xr+7S/FSyzwkwrT1fpLNz0qFhU5kspB94Fnjd11NUTaW20hKUjTS1dD0G//AG742/rf9q2fwG//AG7Y2/rf9q0G1Fa03FL+e1TOAdlXjXGv7tL8VY6+7lor8umoum8aZqopqlHVuNOJCkqT3EG0M3l0YuHlS6VjD9Kkkk9lGX6rNgdBvWP398bf1v8AtW1/AaB/8O+Nv63/AGrJ9VXv/wDDn7YTM7P9k2/zz/dpPipBwPwOwjgut9Oum52Gn4gOZcy4n9Y687SlT0uRvLli1bP4DPdx1xr/AFv+1bP4DR/9O2Nv63/asmVVezan/PCd/Z7sm83+Wf7tJ8U6OJPCniBeuJqy+sLcUb/uenqsqhSMuktNqCQDlSCBBifMmzXZ4edICjVDXGa8lJ2lyhQ5/rHX4W0egzmOvHXGp/6T/atz/AZTP/f0xp/Wf7Vk6sxMnSD89vwTt7PdkANcYv8A/jy/FWIuKmqUUDDdc+p59DaUrcKQCtQEFUDvNo44lMcXqTEqF4Er7nTdVQhCXmqymUpaXJIUoKTuIy6d48bMAdBuP/DtjUeTv+1bR6DKVb8dcaHzc/2rFGIYk35tP+e1DPZvsid8Z/u0vxUtVmFqtGGKj83ttqrmmCunCpCS4nVIJ5agWbXDvEfEzEd8OUmM8KXfdbFKgqS8xUlSlrJAyhBJhMTqTyHjZlfwGuQ46Y0j/nP9qwY6DASokcccZCeYc/2rJ2IYk616b88JDs12RH88/wB3k+KsxTslLYTM6crRRjnihRYGxW9cddhrEFYqtS260/Q0XWNGRlylRUO1p8RZhDoORvx2xqPJ3/asC90FkrIUrjdjFZHNSwf7VkK/EG/+G/PakezfZIm/y1/dpPipovNl5OAr6UtBH+5VVv8A8yq0b9BGP3gaTv8AzpWf6wsgfwGUOoUy7xwxkUKBSpJckEHcEZrTZwW4TXfwXwQ3gi7L2qLyYaqXakP1DaUKJcIJEJ00sCLvVRXMnmiyNa1w+cDuR09ivYhNgGFdmJ8Kw6u7xJLLE/8Agnx2DGyA6uuD84J9KHOLU2wSppvpI8aXah1DTbbjClLWrKlP0g3JtcoxahOIsT/uC478WDiThlf+I7qxI8imSKSmVl7CkrCsxEESBEcxZ8Vmjp5KeWT5oeb8/Vco/g/w+oxaixehpBmlkgAaLgXImiO5IGwJ3Ug8XMLfuhp6LFuGnWnr0w2UVdMplYVnSknO0SOSkKOnfFnBhS8G6tNBeLUdU+Wlo5xMfPstGeDekFhnA93OXZcvAHGDTLq1OKHUklSjAkkgzoAPZZrU/EnCtJi9OLKbg7xAQU1HpKaUFQaC5kDKE7TGnhYrsfw5zQ3OdPA/BVB+CztaHX7r/aR/rqTONVW1R9Jzg9VPZsjYcJgSfXIta6hvKnrWuuZKwJ1Chr32obivF9+ceeM+A6m48FYkwuLvK6U1VTTEltSiVBYOXKIjna4nDW4L/uC5fQcR39U3tVFwrU++hKDBgBICRECPPU2r4XPHPLPJGbgv03+i1XvwgYdUYXRYTRVYyyRwEOFwbEzSHkSNiDunuggOrIPdZJxzH7ir/AP/ANl1Z/6pVlZEdaf6I+2yTjj/ALisQf8A4XV7f8yq2tIbMd7F57QfxqL/AHm/eFVTottg8D7mVtN5V4PsUk2TLvYewpxvxVcN6OKduPiA0iupQZCUVbTCGX2v8dAQeWxsNwCGKqfo0XVeOEbhRe9W3fNahbCnAiGyRKpO+wHttIV68OsQY3wNSXjeN2t3fiChcTeFM2hwLLLySSEZhpqnsnlrzi2dhUQfSQFx0LAPYV2X4Q5jH2pxIAa8eQ+9PPCcsu0iCsryNdXmI3iNbLvEROfBtaqfUCFGP6aTaNOF+KMT3/iE3feGA7yulunQovVFSsBOYGAAmASTM9wFpixDc7t8YavC7ElIXUU60IJ2C47Pxi2jQMMTcrzzXFVzxI4Ob0UZYIp2nqh5DgB7I0Ptmzaapm7uxNXssDKG7xK0pBA3yqJ+Px9yLh/iZdtxXi7dt/Ppui96KWainrewlR7wduU7+Uiytct8XRijExXQXlS1L9W+H3G6ZwOZQAkE6Exokb99ozkwwmFw1upRN4kolB0srC3eS8yEqHrCTaDOIWGV4Wx4ivomP4hf4Wp1KRoioSJJ/wAYT5km063SypLKSpJBI7rM3jJTsow/T1rsJFPVpUonZIIIOvIWuxEPaWHYhVHktcHDkU0sB3YKutS8EQlkBIERr+y0qXu0tvD9alBhQpnI88psxOE7zFbTVKmlpXkeGqSDukfcbSbVMh6icaIMOIKT7bUKJgZqVbrH53WUCXPc1Jf1PTXbVtIcaeQlRSRM9mbFcB8DbzwnjBpd3YqrGbgp6s1jV2tsoCZJzZSsgmM3dGgjxsewheLVI+2HVAu3e+umfSN0KBKdRuO+y3V4rxC9ftSLtrCxRshtLJQhKuskHOTMzqY9loPaIDIHHW+yMwumyZdrJ/47aL2C72TufRHFQfBM/ZaBsWYEw/xFueiocR0iX2adxNSjUgpVkKZBBBGijawl9p9PuGuYBCutpXEkd8pNq+X1hZeOcIUlzovK8LtWvqHA/RP9U6CncBQ5HW0q8fiGOBtZ33hRoT+Ne0i+idmAsD4PwDw+xPd2GaRLT9RdtS5UryqzuANLCQSSdBJ08T32KdBEj94GkH/rSs/1hYzgLhw7gXAeKnqrEF83u/VXZUJ6y8axT3VoDStEDYSdzvoLFugh/wB4Kk3/AO2lZ/rC2Nr8oRXN/Rd97V39GQexdfYW/wAog/5Jlvp368AKv/8AFKL/AFjZvtM1zuFXGKCp6irNFlYdyhWRwsgoVB00MHXTSy/07hHACrH/AK0o/wDWNiVIyplhpnYGho1g/wBKnQbELb1FSejGH84pql1uyOHf8RUf9OJMDo+Yv4i1jF5Ix46F3xcd5Btqqbpwz1iCkESEwn8Fa2uFXVbv5rqXaZeVzqFqQe5WUxanXC7iZiy/saXrg7F+HaSgrboQmrYeplr6t5GeBovv335ERa2ly1v5yuZmqIA69oFQHJXMeUza5TejOQ4WvY25LiKnWEEG9ri6jHhRfVUu9aKqfq3nfSmXG3CtalSqCZM88ybJ1+4+xKxxEvdpN7VbTN3VbSUMtulCC1lBgpmDMnU/ZoX4fFVHXN0SlQaK8nqYzp+mR9Rsk47YFHxTvhsEg1tLTVI07k5P7NrOS3emdCSPMFBJzOgd1GvkVY++L39AuWtvNCQ56PTuPBJPrFKSY+FmJwux5fmIK1LV7VTS2n6YvISlpKYX2ToRygnf7LOShUL2wiyVEn0uhSD/AIzev12iLg/VluuuhC1GQhdOrzAUI+AsKX+AbIPpD3qcI/Glh6H3KWeJOOF4OuilrGqQVL1VUppmwpWVKSQTmMakafEWPYGxW5iq5TeD9IindbeUypCFlQ0CSDP+NZjcfmlrwWxWpBJo7xYdJT3ElP1qFjXBt9QorwpgdEvIc2/WEf2bKpbkbG4c7g/UowjMXg8rKU82kkwLNB/iDdTV41NAgVTq6ZwtrKGxlzASQJImNrOrMS0fK0D1tM0rEeILrdUQhyoeQuCUkBwTIPLQ2FUEx0r5Wbtt99lOIB0zY3bH4KXLrxnddbUJpS8ttxeiQ62USe6dvjZwFYgkjYWptga5MUcOuKNXhC8MSXneV1V1CirovTX1OlohRBAKjpsqYiRl7pta6769dfcjb2cl1bRSox+kBBPvFgQuk4giltqAQfAosjWBhczkbIWgxTcV43g7dVLfd3vVrM56ZupQp1MbygGR7rKFRUJZSFq7+ZtT/hDXG7OKt1POGFLfcp1T/PQtH1xaVuk/WVScDUrdO64lp2ryOhKoCuwopnvEi2l3QmqdTA7KoJhwBN1U0CpQpoujUedsadS6kqHI87UhwRxdvfCeBsR4ZbvBxg1LIqKJ5CjmadCkhYST6uZG20FOm9pC6J2Lr2r78v66b2viurAqmQ8wmoqlupQEOEHLmJAkLTt3WruiyxCXxsiiS8mS3K6s4txDacyjAsGH2ldkLHlaLOPPFt7hrh9pV2UrdReVaFdQHf5NsApSVqEgmCoQNJ79LVpuvpK8X6G+6VF5X4H0VqA4w3UXc2ll6NwkpSCRpuFeFguGQgEbo7TmuRyUl9OD/wAGUf8AnQj+xazcmY199qddJPHVLxDwhwnxJTNhpbmKEtVDMz1TycmZM8xsR4KFrimSdPbNsylN6uc/7v3LvO0AI7L4OD/tH/UC0TpYBaQKhtXfpYc2BqCYQs/oqn2fItokLhRoULyg7W2O8i2QPK2hoLSTbrZ+y2jH222Ytox5WSS55ae22ADeNLYfWAtvlvFnAUCboJ9GdCkHY6eNku7FqXRoB9ZsltXgQYssFIIP12RrvStqsrqcoKQHg6knuUJ+yzkagpA3CN87CsUya1iopSEklGdE66pOlgyNo526YrKegWauqXkZQhWdUE6RyA1OvIanYTavWRNkhc1+1kmOLXXaoZ4kYWw9eN5O3VfV1UdemoyPejvISopAGUuwrYJM6jXWOdozr8C3rWNMKwhjq9LuRLhLtWymvbKHFZiW0ukKQlOgTlWmQBvGa0z1l0Vl5XnfOIq8VFGal0MU9O+wnrGWkISUE7wSVZsswCSVAkQGdiqnoMP4Vqqi+WHDRuNqSFVAWfSVFJhBUBJKjptGsaC3LhsU4dI9oa3ktFz3ZRrcqI7zwuMcX09dlTfdCbpoEB9tNNTgO1TCgBnefzKJC1dYBlKEwFkJEAqq7xGua+n8UXvc+K6k3XcaalVBSVdTeq1rdoUr+hzJSVOpYGRKoOVBgkQTNre3RTXXdFwqq7xqrqdva83xXVwcdQlqmUQIAR/MACUpPMTyJtTPpcYHxdiDHr7GGX3bwuuq9FrmjT1WZguqQG1rWRIUuUSdyM4AAAiwqekpmOAaQ2+56KvLI4MznVSLRYrw7TUVPTU/Ga9WGmmkIQ0zdby20JAACUqJlSQNAeYtlquUmAMdNUrLYxTSs5G0p6sKkIgeroY020tlqxwSkv8Axn7vghd+k+ivoEvBMPTE62AQkAzEWjJXSa4Y1zgKVXszJiHKRKoPjkWbHGOP3CxwgLv2oaPPPd1Rp7kG3pvfac+uFlGknGuUqRQI1Mm3QTyjbazMY4y8MKkgN4wpUlUgBxp1v/WSLKLPEjh6/wCpja5Ae5Va2k/E2kKqF2zx5qBglG7T5JxjvtmnI2TKbFmE6whNJie6HyZIDda0o/BVlFqppXxLNS05PNCwfqNiCVjtio8N43C7CRz2tuBprbYBPl4W3GmvOz3B2UcpXIE6xbUbQPK3cC2FMiPDeyulYriATG1thM7i3Uaz32wiIgWSdcx3a6e+2ZY177dARztkHkB42SS5yneDbRAsIJ521EamySXAT3C2gPCwka+dtQPKySXESfhvbUA8rdlOultEacrJJc5ZtqOYGluyI1toA6aWR2SXEc7bjwtsAk2wgWjdJaiLaIBEW2QBbDE6GzgpIMtIJ21FtKZSSOzYWOVsI5RZikgSykiMultCnSjUWH35W1zsyQNkABD0EfofbZJxx/3FX/43XV//AAlWWiPpkx+qfstxVUlPXUz1FWNB1ipbU04hWy0KBBB8CCbRe3M0t6o9PKIZmSu2BB8iqgdEXjLwlwXwXp8PYyxzdl13gi8Kp001QohQQpQynQc7TKnpH9HxCSlPFG5IPLrFf3bH/wCC90f/AP0V3J/Vq/vWz+C/0f8A/wBFlyf1av71sSlp8TpYWwNMZDRb1uS9Ox3F+w2PYlPiczatrpXF5A4NgSb2F7myS0dIfo8NPF5HE+4go8wtX3WNjpMcAAmP31Lk/rFf3bGf4L3AA7cK7k0/5NX962j0X+j+B/3q7l/yFf3rGAxUbcP85ZZd2CO/fP7FMvFXFLor4sIXfWNsMVixsp5OYjyJTYDDHE7ovYVUfzNjrDVGkmSGQUT7k62ff8F/o/7fvV3Kf+jV/etn8F/gB/6K7l/yFf3rRdHibjc8P85FZJ2DaLDvn9ii7PSY4AtpAHFK5BH/ACiv7tknFvHXo24wuOouG+OJNxv01TGdHWrGxkagd9l49F7gATpwsuUf9Gr+9bP4L/AGdeFdyx/zav71pD5VaLfi/wA5Q/8AkG9/8s/sUzMCcW+jRgO7jddx8RLhp2FOFxQDiyVKPMkiSYAHsFnZ/Ca4AxH76Vyf1iv7th/4L/ADnwsuTf8A4tf962fwX+AHLhXcn+Qr+9aA+VR+T/OSc7sC7+mf2KhLiJivgret/vYswJx0uq5byqR/GUQXWHz3lBA18fbEkkgYL4nYGorxD+LuOGH6xpBlLdOx1IJ71HUnyEC05/wXuAP/AKLLl/q1f3ra/gu8AQP+9bcv+Qr+9aMwxWf55j/ORIpewUIs3vn9iktvpG8BBSqYVxSuXtJy6uK+61dqrjU5dFQ5RXVjHAtbQsrysKVUO5y2DoSdNfZGvO1nf4L3AEf+C25f6tX96wa+i3wDVtwvuYeSFffabzij2cM8Mj/7k0cnYFjsw75/YqveH+Pt0VabxYxDiTDFBSqu6rSFM1q1uOuKZWlCAmI1Kuf12lLoIg/vBUkA/wDbSs/1hZ4p6LXAVKteGFzEf0Ff3rP3CGDMMYDudOH8IXLT3Xd6VqdTTsAhAWr1jB5nSwKajqhUtnny2AI9G/O3X2KzimO9no8BlwnB2zF0kkbyZeHYBgeLDJ1zdOShPp3j/wDZ/qzO16UY/wBI2S8WX1S4Rp7jrau7rxdRXXJRKC6WlU6AUspBKiNBsPGyr07xHR/q9P8A7Uo/9Y2lrDVz0l54Jw8mpZSuLrpdx/ySbFjce/Tste7Gj3uQa4W7GYe6+1ROfzIVXG4sQ4QvbEaK27bvr1Xw6x6IIoXknq8+aFGMo15qiO/vsrhmiXR3PT0pn6NEHzOp+Jt1T4Ru5h3rG6VAPgLLrFIlpOUDQCLWIYnMkzu6dbrjZpmvZlCrJiK/KfAfEm+Lovl30RqufF40LzhhtzOdRPLtSP8AFPhIGIK1jEOK2cRoqUJihTRqbSoLzqzlQII9ojx98/Y24cYZxxTIp8QXTT1nVT1alp7SJ3yq3Hvs0MOcAMD4YvJq8rtuZCH2vUUta15PEZiYNj1cpcXOjuC4WO1ih0zWgAPscp06p4YSpX6XDNBS1AIcbYSFBW6fA+W1oXwcRdeKnaRSwn0S/HmgByT1xH2m1iWKUNMhuBtFo6e4MXIvHDmL3Hq5Rce9I9GFSpLHWzOYoG+uuulolwFNwneHuSaPx/EHj70a4x0npvDi+EBOrTSXx4ZFpV9lmpwwv6lut91dVUttt1LKFBThCUyD3k+JtK9/3I1fdxVtzuylNZTrYURqRmSRNoAe4KcTKFfVUHEAFpCYShy7kK22gzP7LNUyt4LBfUEnnz9iVPGTI48iFYe7L5oL0bUmkrqZ5SRJS26lRA7yAbQ7ioGk4lXi1smqaZfA5GEhJPvFnBwjwXifDbtTXYoxAm8H3Wwy2hulSyhCZBKjBJJMDuAg6a6JHGLDWMjiCjxHhG7qSt/ivo7yH3lIiFEgiAZ3+FmZK2Smka+wuD/23SMRbMwt5FN3H1QwvH+CKxls53W6xlwjmOqSRr3TmtMWE15rnQ3qQkkCR4zaE7lY4nVt4MU964PomQDBqDVhQbTzgBJJmBpaebgu/wBCoW2Qn1U6+J5m2fHO+eoY4iwa229+d1bdEIonC97m6qU8BhriuqTkTRX+pXcAkPZh7INp34/3f+cOGlY8GgtVG8xUADuzhJ+CjaKeL+D7/c4nVbdz3FXVSrwdbfZW0wS0CUAElewhSdZOm9p3x1dlTevDq+aNbcvu3c4QlOv0gRIA79RbojOzv7ZQd8t/IXWS2NwpTGR1Xn3ftYbuvWgoH0j0a8TUUi9IhyMyNtpCVj2i0w9F+sXd3EajpHHATU0j9MTsTlSFCfPqwfbZp0XDys4lKvSluilW8/QMKrKV4JOVupQoEJKtgVALT7SeVjvByorLu4lYccZZWp1Va224ylJzoCiUrlPKAVE9wtWqWBocGciiwuNwXcwpY6XFE441dFQD2VU76Z/VKVtq+q0Q3q/ha+eHeELsqrxqqW+aVNS4HWEIORAeUiCVaSQEwBJ08psV0nLncrMD017IQctBUw6r9VtxOWf8rL77VAvO4rhv/Di7jvWhJraSpQ9RvAEpLZWkuNnnEAnvMnazytaWxS+BF+h/YqUZIMjPEH2p2cRXsLs4Q4eXNhwXiV0uLWXqtyuKc7rjiUgKGU5QIREADb22v/7LebWNK6ndrsKMU0BP7oqQwBAGVREeyYt6SDQcrc1Qkmqnv/q/cV6R2jAHZfB7f7R/1AsJHP67AVQzMqGnL3WGJ1twsSgpESQYtqrz/NqsbWVtIV3pBsII8D42BpyFMAARqR8dLCwruIs4Ce62bagxt7bdAaQbaJy7WlZRJXMAqmLb2Glt7iba5SBvZwLpiuY1MCyU6OpvgQey+zr5pP3GyuTOwslXolxFVRPoKQkPFDkzOVSTt7QLScNFFpN0YIgQbJ7CX6m+fSlgJpKFSQyAfXe1zqPgkEJHjn7gbKJB5W4ykchvPxsOWMPAvsna+y7xnRIU2FaZS33aSPkWhPGF20y3KCnfqG6hPXl1uiYldRVPITmQ2kSBAjMonsADUgTaV8aXpipdK0xcuEBXJUyD6S/XoYbCtQU6BSv0QdudosxzT4kuO5LxvSku9XpJo0KqepQupJdIALaFAAIaCyk6gAwpZzQAOKxrEmRMEETsrj4dD4rQpmFzbO1USX5Q1N9YwNBjm/mruu+pbPodCysgJabQCsvVIBCQc2YpRkgRmWQLJF8UF14lN/8AoWCbpvhxTdQp69KTI4m7wkpS2Eo1KlqGZwyG5KgkfrWceMLrvG8G2aa/byXQXd1Y6ykZFOKlaiQnKlGdSI3VmSolMgQSIMOYwpnDebzV2cSE0dxUqEOIo6aoLS1uKSVGEkZnClIRqSslUBInRHD1k1RWZmSm1uZFjf6v/ZXgyMD0QnRTXZw2YpmmH7ovFbjaEoWpV53ekqUBBJHVmPKT5m2WblN0f6Ospmqut4nVtNUPoS46yXHFFtZEqTI0MEkT4Wy2Xni/LnyKh3d30PejCax9lwgKMRGqttfqP2WWaK+esgLc7QAkn57h862btSFdbnQqVRqeWnhYamUkAJJUkgHYmRz39vut7A4Byk06p6MVqHRmSoxOp+fnax1t6EgyZMEx8+dmgxVOtKKDMCU6DXSfvsr09e4jQqJSR60wfh8m1dzbbIwSz1dKtRCqZog8ykbW7QGG1AoZbSrlCQNfn6rFGqpJIBGYbRttYdL2X1ee5zakT8bC1CnYFKdPetbTDNT19UyJmWnlJ9xB+ZsoMY0xZTEJYxTfCfAVzojv/SsgZpSAoExvz1++wgUqOzlj1pJ2tISPGxUeGw7hOin4j4+p/Uxde86+vVrWPiTY9T8YOJLIyt4rqT3Z2ml936yTNmSFEg6nfeZ/Z+NuiSQCFGDMg8ye7vtMVMzdnHzUTBEd2jyUiM8c+I7XrXyy8NPXo2v7KQbKDPSEx63o43dLpEHt0yh8UrFouJVIg65hEb23mUCM3MQCN4ixBXVLdnlCNHA71QphpukZioJIqLjulzn2A4g/FZspsdIq8AQmpwlTKJ0lFcpOvdGQ2g9pUKzHQTzG3z88rHwSmAU6xrr862c4rVs2ekMNpneqpvZ6RFIT/GMJvI0B7FaFfWgfIsca6QmG5+luC9kx+qWlf2xaB05smYRpABga+426giOQjXw+e6zjG6tu7vconCKY8lYFnj3gtyOsob4bkfpMNn6nDY6zxs4fOiV3jVs/06Nw/wCqDauAkxBnUEaW2e0II8N97FGPVQ6IZwWn5XVmW+LfDx2B+6NCCeTlO8g/FFjrXETAr+jeK7un+c8Ef60WqwFHUFcDfXmeYtgVlB/nHntYre0M/NoQzgcR2cVbNrF2E3tGMT3Sv+jWtn7bG2ryuypANPeVM7PNDySPgbVDMdnMoaHZQ8vhbOrAEAIUdoIEWIO0T/WYoHAm8nq4oKVCQQeelt5TyHvtTtLq2jKFrb74VEGxlm+b4p4NPe9Y3+rleWmD5A2IO0Q5s96H8hHk/wByt0UmSLayxsNDaqTeMcXNFAaxNfAiNq1yNv6Xz5WPtcScdMxkxVXafrqCtP8AGB1sQdoYubCoHA5OTgrP5TMHW2ik7jytWxri9xBZ0OIesH86mZJ/1LHWuN2PGvWqKB7/AJylGv8AkkRYrcfpjuChHBKgbEKw+U91sA02tArPHzGCRmdu+53AP+RdT/8AMscY6Qd8pI6/DNEsEEAoqFIn3hVjNxqkPP3IZwiqHJTS4k9c2ZmQQPhbspVse60Lq6Qz6nWf95zaoJK4vIiNI/4rvsos9IO6yQKvDFWgk69VUoWB7wmxBi9GfXQ3YZVN3apXiNbaA1tGzHH3CTpAdui+W5P/ABbKh3f8ZY+zxvwE8AXKiuYJP6dIo/6s2K3EqV2zwhmgqR6hT6iLZB/ZZoNcYOHThCf3QFB/n0b6fjkscY4lYCqAAjFVAJ0HWKKP9YCxBWU7tnjzQzSzjdh8k4wJ3FsG4H2WSWcZYPfVDOK7oWZ2Fc3/AHrHWr4uiogsXvROf0KhB+o2IJozs4JcJ7dwUZG2otgTvbG1trEtuJUO9KgbdhJA2+FkXA7FQLSuN4nW2ET99usvh8LbgE7aWa6jYoOJtvXvt1Ea23l7rJKy4IB3GttZRGthCCP221Gu+3jZaqbWhckcrZHeLdFNtwYtFFGijDpD8Jrx40cN3sD3XetNdr7tYxUh+oQpaAGySRCddZtElJ0eOldQ0zNDSdJZLTFO2lppCWFwhCRAA02Ai1qgOUedsA52oT4bBUScV1w61tHEbewrr8J7b4pgtEMPgEbog4uAfFHJYuABIL2m17Dboqt/vB9LaP8A6TfL/iF/dbY4BdLYbdJ3/qF/da0gBt0O+wvkiD6T/tu+KvD8I+Ln+Sp/6tD+oqs/vBdLcn/6Tn/UL+62v3gulr/95wf1C/utaffSLZrzsvkiD6Tvtu+Kf98bFvyVP/VoP1FVkcA+ltP/ANJ3/qF/dbX7wXS1O3ScH9Qv7rWmiNLa3Nl8kU/0nfbd8Uv3x8X/ACVP/Vof1FVr94Lpa/8A3nND/wAgv7rcK6P3SxV63SYSZ76df3WtTAjQ20Ym0Tg9P1d9t3xTj8I+L/kqf+rQ/qKq6ej90sk+r0mUj/oF/dbS+j90sliHOkulQ8adf3WtUR4WzlaHyRT7Xd9p3xU/3xsX/JQf1eH9RVRHR36Vc5v4SSAe/wBHX91hU8AelknRPSZA8mF/da052tmUb7/baJwinGxd9p3xT/vj4ud4oP6vD+oqpu9HnpVuOhbnSSQtY2Uadcj4W7PR/wClhlKD0l0lJ0I6hcfVa04E6AaTbShrpZhhEHV32nfFSH4RsWO8UH9Xh/UVTmOjh0oqUqFP0imW8xk5aZYk+6xSm6MXSToa1dfR8fKNmpdJK3m6NSVqJ3khOtrdQOc21FonCIOrvtO+Kl++Ni/5KD+rw/qKqVb0d+lLeFG7RV/SLaqaZ5JbdZdplKQtJ3BBEEWZTnQm4wsqyo4sXdlmYTTuAA+AiBa8BgJiBYMoE6gWRwqEiwc77TvikPwi4qDfhU/9Xh/UVJbl6DnEZi8bpdvDiPdT1Jddc1VpY9Gc/RWFKy9xOvtM2ursIsIUAbeW1tFMDQ6WNS0MdJm4d9bXuSdvasbH+1OIdo2xMrMobFmyhjGsAzEE6NAGpC4MxbcAjQW3A2FtiQN9bXQFzaLUWhdb/VcMeW31g2M+z4WLMnLWvJI9YBQ8v2k2M687IbJzutRI31tk6722dNbaPKPPazplhnlbXjbJg2ye+0mhRcs0IsQvZJVSLWndCg4PYZse5RPOwD6OtZW3tnSU+8WJluFAnKQgjCoKeYkW5I05CbcUiyukaKpnKAfMW7IJMAT5Widk/NHK3rXMJ1HoxAfazJaJE5Vbg7jae8WiDGtNVVl206KW8lO0SaxmrdbZ6s+mBvcBS1ACe8TJgSLSrdiHK2iqReCAIAeappkIUNioxqrbTYeMSWBjFllqqcOU5myVaCZEfDf4W5Gvwp1XUOnBsDYba73WhTTNjaWHXmoqx3dCcRX7cnpdxM+glCnawl9Db0JKS20t0BSkNEhWjclXZBKUkqLJ4ivXmxdFde9z8MMHorbvpXFUdY88l4UwabV1aw0WBMBIIRMTExqQ5r5xOxKqW4KFd51aSpTzVGpKW2lmVK6x1RCE66RJVr6u9kVNNXvtO1OJXilBUAmioRmCs0Dq1FQlyYIkZBBMjTMBOwKn4fGnOZ9vCwSFW7NkYLBVJRx8u9tCW6vijicvpADpZvd0IKxvl0Ok7anS2WeVTwiacqXV1XHbB9E8pai5TIo7uSlhU6oAQMgCTpCezpppbLc1wcO/Y/8A8o95ev7eaerzCg4sqaBSFTqDOtu2GgEpMTl00A+TY9XNsU7oD7qWlk5QFHLJ7h3m3NMELSCHkeaVgj2fPvt07ZC8XC0uGGnVbbayiMpVprynnPKPx8bDIQRqEyRzI7htHw+dRUNACEKhQgDKI5be+3SWURKCBz5QAOXwn22fMeafRdsOrZhCgNwezIkz4an53scp34y50E6bxr872KJCSQG1BR035ba8uVttZoUEkpn1goROvl8++0NSkldqqMyTOUaAySN+XssOHErSQVKGUCTvr86c7IyXVg5pAB1E6jx2saS64mJJMwZj4ft52ikErar7SSDpmnUx9+/L9uABI7SyRzER5WJofaJhJ19YGdrGEOEApHI+IPw9lolSFkKNUnTNrI39nlbsJGbtEnXQE9/zztwlSCrIpUknXW3UAbFJG+nPTx8vhZrpbrpuCQlKjpy+fn6rKTQ1mTruJ+fkWS0kJXIEHQye6ymyFFAABHONv22C9yMwIUDvAEjfcCO7b5+AhKYgjTaJ137tjbgABIInUjQfX899taD1UgEgASJ9kWDdFXahGgIAAiR5W4zDUZRJB0Enl8/G2ylSBBOp11525WmJjKQkHTcz91kHJrBYDmJAOg5qGk+y2JSQhSAkE+HMG2kJJASVQDvpvbsZMw0zTuBaWZINCxKlRpA7JIO/z+NtgaKATMQBB05/P7bbjLJUdY0OWO7T4W5mQSBG/us2YpWC6lE5lctDPxI+d7ckfrAa67bW6IMgSok676D5i2iTAMkgmZ+eW1ldKyxKdNRMGCAeUfttsAqGgB05jl8xbekQIB5eXz9VtaKSQiIEbcvn57rK6WULMwkEGdO62KzSM2ubWDNsyxlJVqDJNsUkSYT62nKBZXUg1Yopnug6TbgrME9kabCQLdlQKwCRAGw1H7PnewStVhM6bq0GvzpaV1EtWISVHRIBJ1BNuyExBJ00E89LYiQsEEbQJOttBJAyjzEfXNldRyroFKplO4Ag6HvthUEkncidfq+/32DykAnNJ3128PG2ZpIOWTuNbK5UsqECwTB1Ke18efz+GJVOoWdpn5+d7cgiQd401j57/kW5zEJ0ImAd+XdZkrBC5gIyzrpz3+ZtsJQpJCgCZ3KeRsFmSnRYOo0JJ20n587YpRG87xp9/wA/e9ykWgoQdUlOZCACDpAjv1saZvO8GBLF41LI/wCTeWn6jvYgg69kmZt3CcpSdI25/P42cSObsVHhtO4Su3izE9OodTiW9kDu9Pej/WseZ4iY5p5yYrvE84U6Fkf5U2bY1OYCd9t9f2W2oBIylPmdrTFVK3Zx81B1NE7UtCeLPFviC1IOI1KPcunYMf6H22PNcaceNRmrqN7WD1lIkT/kx42YQWdRqCN4FtZokrKT3nwmxBX1I2efNDNFTndg8lJTXHXGyIDlDc7hPL0ZwH/4ngeVjjPH2/kn6e47uX/QU4j6ybRSlUbkzrrA+fG3WdSBqdOcxP7bEGJ1bdnlROHUx9RTCjpAvjR7CrZExKa4j/5Zsep+Pt1q0fw7VI1j6N9K/rAtCOdXZmRyI3k91tFZ1hWkSdPq99pjGKset7lD5KpXeqp9a464UXo7dt7Nd56tsx/p2PM8Z8DufylRWMz+vTk/6pNq7pc1OgI5aRr3W7LqVII5yRPf4WIMcqm72P1KJwamPVWSZ4tYAe0F+5TsQqmeGv8AkxY81xBwS9ojEtEDv21lH+sBar3WHMCCDOxjf6rCIdIGYaggxztMdoKgbtCH8hQcnFWmbxhhJw9jFN1Gdv44399jbN83NUa097Ubv9B9B+21UA8UjMeQ38Ldl4HURAnf7eXfYje0UnNg81A4Czk8q2yFNrEoWlQ/mmbbykCYtUhD6GwS2QmCYgBPz+NjTV93nTCGL0q2o1GR9Yj3GxR2jHOP3qBwE8n+5WujuFtAR3WrBTY2xQySE4jvLs6malZ+2yizxIxk2n6PENSCRPbIV/rA2mO0UR3YUM4FKNnBWOgzbIJ1mbV+p+LGNkGF34XP6TDX15bHW+MuLmwM7lG4T+s0PfpGlpjH6Y7gqJwaoGxCnQjS2iNBFoXa424hTo7Q3ev/ABFj6lfM2OtccK0D6e42FRuUOFI+M2K3HKR3NDOE1I5KWcveLaInmZ5WjFnjcy5HWYf84qZ/sWNtcaLmV/K3RVgzHYUlX1xaQxejPr/ehnDKpvqqQCNNLaIkybMdPGLCitHGa9E97ST/AGrGkcVcFuKg17rf9NhX2TYwxClds8KHc6hm7CnYRpzi2j8xZuJ4jYLdgovtA5asuCP9Gw6MdYQdMpxDRjuzry/XFpCrgOzx5pu7yjdp8kskfstojlO1k1vFuFqgwziO7Fk8k1jZI9k2Nt3ldz+jFfSud2V5J+o2IJY3bEKBjeNwhjAi2trYClWqSFeRm2bH6jYgIPNRsRuizgy1rawfXSUx7zYyI5WKVi+rXTrzR9IEz9f1WNTNnGqYmyyR36W1587YbZM87TAsoE3Wp1m2bCbZJ1tyuRaQGqjey2oiYGo8rcEjYA22TI0520QBuY87FQybohTSOuaP6Dqh7Dr9thTA524IyVrumi0pUPqsInq0qK1tlUxsSNjavM57Y80QzHpeymAL6oSjWUvidlSDy3tGWK63D+Kap+hNe1UBhIp667wvKtKgZOcghRSRERAI3kHSRgy0t9VRnUh1RBBzkk67d0C0J8Yujfd3Eu804nw1iqouG+0qCzVtJUtpQgpGUoUkoVMExmmBoN7eb4p2prKKpDKynLGHfnfoQR71oMhaW5mG5SS/QXTdwFHdtE3Q0SFlDTDY7DKOWw2AOpj32iC/cRY2uq83L8a4aKevBlwUCVJvcVVO0gEqdV1SW0KTyzOdpOqe0E5Sqbbv4bYnwHc8XxiKtxq822taQU01GqcmUIzuFRV2iYMD1p1ypTZCp7nvi+r/AA67hhdzu0aGwuk6ulcTlgEqcdSjVAiBlUnslJCQVRbNqe1rcXk4FG7Kxgvqd/qViKlDBnfuVXerw5gr0p706+HBU9YrrsxoZzz2pioPOedstPTuBOETjq13lh3EbtWpRL7n5lqe25PaOihuZ5DyFstzJxCT6fvVvgs6KCPykb7qeFmF0t5syr9CiEyDpSugbHvVt42p9wmerH3qY+kPFJeQrL1qj/hIMz591rb/AJShwJwHhGmlUrvZ1WhOyWB/e+NqqcHWypqgfcKSXX9471gwO/1dedvXOyDQ7DY83U/eq/aNxFa+3QfcpwYNWkp6qpqErB0CXSJE6ixlNRe8JCrzq5nk8uAO7U/bYelakkJ7QMkADUDvPdqPnkopZSEZlTIGcg6gRzn52t3BhjPqhcs2Z/Ioqi+MSNbX5XGTBCllQjX4R9YsOjEWK0ggXu7mGnbQk7bcrD+ito7IQN9NNZ8vn67CBgJJ0knQCYBI8T86D2DNLCfUHkpipmbs4rTWKsUqMGuQtQI9dpMn4RysabxpihpCpXSrMgwWzofKe/z+q3Ip4ICkknXUq1PyfnnbXoiFRAjkIAMD9htHuFM7dgS79UDZxRxviJf7aIXS0SiNBAUP7XdOsd1jbPE29gUo/NLG8wlwiT7jZH9EQ4cq0g7SQmO7eOVt+hI0BjQ7Hb4ez32Y4VSO3YFIYlUjZyciOKdTJSu4wZTuH4jvOqbDp4t06D9JclRKefWJMGfZ8jlZrGlKUnUnXwmPnabAvULYEgctwraI/CwvkSjd6qkMXqh6yWrz6RmErlrEUt6XLfX0iA6FNNNOJAmIMrBn556O+h4xYXqqVp/0e8kB1AUnOynYiZ0V5WqhxYSKfENOEjLNKkpIHPMoc9R3WlfD9IHbooswBKqZsmdJlI7rVv3PUb3HQ+asjHKpjRspkTxbwmNA5WAkzrTq7/tMWMs8UcGuTF4upI5KpnAffl8/d3Wif82JUAlCCDMzGp179rCC7ARAJOsJ7Mzv+FonsxRnYnzTjtDU8wFLiOIeDn4CL+YQTqQpJTqZHMWMIxvhR4pyYiodJmXkgD3nU7/JtDJuxKDkSiJG08trcfm5uQSkAKEHT7bBPZaDk8oje0c3NoU5M4kw+4nM3fdEqBooVCD9X22NovC7lE5K9gkkbOpJ8Ofh8LQELoSshS0QAefIwedknENxtm7RmQlPaESRpod9PKwH9lmWu2T3KwztE8nVnvVmkuNuJgLSoA7jXW3SYJSAYKdTHd5ey1EzczqGcQtMvONkudYopXlMFA10MjzFuKM4iuRm56heIr2aKusddCa90CVDNqM2w5A7RakezjuT/crIx5vNnvV8OyClROWOQ+vxtmUlZ0AjQcoPu8LUPunEXEltu8VqxxiRCn6f0lpIvZ85AVKgCVwNI28O6xqmxzxco8K1FYniLfyqtbTj4W7VqcKDCoSAudBI087Q/c5MdnBTGPRc2lXogyFZdIGxnT7baySRm18x8Dbz2b6SXG+71UiBxDqXfohKXKWnVnVJBJluRr3HlZ33l0luL7Fa1Q3ZiFC0odaQ8t272VZs247KBy+3ewXdn6kHQhGGOU9tQVdskBMKUY93lFsyjdOaZ0+y1W2OP/FJlEvLulzTdyiWnx1CVjusltdKziv+cV0P7nsNPZqhSUSw+2UtAmVqh48/ZobDfgNW3p5okeN0p6q2/gNYgEEkHv8AusE3lmdSTrMd9q51fSZxe1TdZ+5e61hAzLh9xOYRtzidNdbc4d6WN63rRKqqzh4zTkOZChu9irkDzaHiPZaBwWrb6vvUxi9I7XN7lY/KSneBEHXl3W4AUTrqPWjnaAKDpbUNfeblF+9/XpbYnM83XtrEjkJSmffY3dXS0whXVr131GEcQsuU4KlKSGFoVEbHrAT7rDOF1Y9RT+UqX6anZUiAnTukam2jJAAlOxjkPbaJWukxgRZLbt1X80ZmDStKHwdO322DX0qeEDDxYq7wvNkpVlJVdrigDrp2Jn32g+gqWbsKmK6mds8KXTKiUgkc5HfFtKWuTOgnQfX8+NoroOk/wTvJ8MsYseDqtUhd11Y5a6luLKH8ILg0gZHuIV3NKBmHkuN+ztJHeLDNJUAXLD5IgqoD6481IcOBRCRqdI+d/O3ROgJSTB0jvszaXjJworKYVbPETDxZSSFLVXoSkRvJVA+fZZQo+InD6uSHqLHVwPJVoFN3kyufLtWGYZRu0+SIJYjs4Jwha5EqA7RnWI+dNfC3aDBlWgjeNfd52Sm8SYcq1FFLf12vEGSlFW2oj3H51seaqGHRLT6FnX1VAzr+ywzG/ophzSd0YQoTHIDlHx95+NsBI0GkTzmOdgwsKOZRVAPI8tPn3W3nUlKiPbOvtg7WjlKJcLsqClAjXciPnutoqC+yIgxB2A+6wfWFQEjz0PhOnzvbAtJMKEx2QFcvD4xZWUbC6FCiU59QowRPKLdQCSEg/hv8+dggrMntE67SLaUsAZSozEAcjvaJKmuwYICY1OoHuttJEzqIgSRM2DU5IOswdTER7PM2wuc8uhEifnxs6VkMleVZIUQRsR9dulOKyxlkRAnXnzjyFixWCISFaxuRB7raCgI7Ku0NtyJ+RZJ0ZClSYMGY75OnPnboOkJmRrrO/wA7WKl1JTCHAI5ch8xbanFEknQ7iBrHL67NYJI0pwhRyyZ120HK2i+kqMGO4gmddfssTLyjJBnYiT3iwangYO2mYg/f7NrSDVEmyP8AXk6E6n1ddtd9LBKqR6skayCR5TYmqpSdCQQr3xYF1xKRJImSYmNPD22cN1US4pTaqQVTIGxEzYyiq1AnlGm/fZARUCRJO5idj7fdY2iolMJBJOoPh7/ZZixOHJWFUZUc3PNHP7LbTUmCrProNVER42Sl1JSmSrUGI217vstyqoWBIGo0mY+T8+Jbhp81ks+kmNFmdZMa2xNV2cogQNSf2+dkUVZQndf4bn58OVsFYvKAokQfLWy4aWdLqaoGYMeAn3fPfYQVkJAUc0mBMz+HOzfTeB1KtkncHSwia4JBAJIOgnT5/C0eEnDrpccq1aknU7wTBsXNX2tzKY20n5iyWusBRKzqNNpsXVXJCiqY9xB01FnDLJFycjdaNisZT47ff+yynR1YLa4KTO/lZoU1bmGmbQ6idbKNBUrOdJVoU+wi0C2yW6XTVk6GNp15bTYJTzBOrbapP6oPn9Vk01B0JUrxBGhsEuuSgZwc0iVQdx8iz5rBLKClZNQ02c7SEoIn1TBn2c7afvuraUGk3leIWo9hqmqnUqVt+qoWRqSpq70UfQ0hLQJmoIlMfzf1j8Pqs47puemp9hndVot1Wq1wdJP2WYSSONmlMWxtF3BOHA7OIq2+ma+8r/vNTLIXlpF1rjrSioQFLCiZIkxGnnaa0qzJB7wCLRth1k07YUBEQR36WkKgWVUreslIKfcYn4W7bConRwgONyVyeIvEktwjB1toq087czpvbRIOu9tcNFlnEror7+VtFU67W5nxtonlZwLKJN11tbJiDpodrczrG/lbCeZs5aHCxTJkVd0YyuS9i9c14enXe6pSm6Z0p6xmTORM+smZjUQD3AAla7Ht+XG+Ka+sN7JBU6jrUAJJjQZFAmSn9KN9ZgF8V0lDbiT6i0k8tDobbzGCOW8Ha3DzdmMQpJXSYXWOYD6rvSaPZfZXW1UbgBKy/jso4Xx8wjSvKZqKGpQAkhTiYcVmnQBOgjLueW2u9ll/j/wrvNFG2zfKmHnm+ygtEicubLKJSVaEQkkzZ1O4Zw5eNO7UX1clBUId9dK6ZBU4RtJI8T7zYpW8MsBVtxF1jDFMyhys9LW20cg67tpCiUxMBxRjvjutiysxyOOTjlkgbcF1vdsjNdBnblBCY7nEXh+KhdKjF1LXuPtKcFKwoKHViAorUvsjKDsIjczvYxSYQurECHr0pbpaavB1QZWgOFpvIgwkEwUqCZVBynVRiJmyFiPgdw3buzraypr7tFI2haqpuqypbDe6+0SE6TJ0jlA0sxr5wtjK5cNXLVcLb/v9NI2uqadbqEuNPXl16wlrInIcoypVC8yCJLkpnNbyjEKXvMzXtbkcdDbQe5bGdzWFOK8bkwrQ3hVUR4oU1Aad5bRpEVlIhNPlUR1YSWSUhMRBOkWyxFnhRxAUyhSbkwTTgpBDL1DTPON6eqpzq+2obFXM62y2f3Cb6R9yXEcqp/lJqlLdw4HYJylVdWqAMcmmfvtW3hHTy1dvViQasHTYCYIPsVPl7LWA/KXvuIYwCwVaKdvJZ20hNMBodefI+FoJ4OoHU3XmOdKnUqgn1e2PmbfSPZNpbhsX1/es/tDrWyewfcp4p6dIhSxJKjoOR077HW2wSF6HMQFQInutlM2pwDMgEkHNInT75mxxLYKc5B08Offp+zwt3C5QIvkVmUQNInSI+fu9wiWmwYykgb7H55+6w5ayGVDTQQCd7dhJkZkylUc47hubJOuQiTBkwoGNzOvL52t0W5AQFGSMx8pifn422lBWkLA0KdRz9lhUjKidwZME7ec2Q0SIRco7UIEQSJ7ye7xthbA7MyB7BP4ffYzkzKM6bme/50sI2ghZkEGYMjkOZtIOKjlCK5Sk5pJI85jc/PnYBbWUZiQjmVctPrseU2FEJUkFMzvt4eHOwLiBkCVCB6p131Ph5fI0IEJwUF8ZW1DEdEQk60gzJOxOdWx25/s5y5httP5loCUlU0rJKgP5qZ/baK+NCSjEFCTBCqT9IkbLUSTG/L3TaW8Kpz3LdiwpWX0RnXNqRkHvtBnzipP+YEsttEJKS2J3j6/nSwymlEZE5tfeJj58bY2jOADof0jtPv8AZ5WNNIMHQeHMfO1ioYCLFodokmTrEDT7Dp9duiwYCYSNo8Pw++xvqx62ka89J+rl387b6lKx6hA8PnutG6nZFCylJPYGYGTIiSfn6rJmIGCbuJARotMzHfufd9dl0NlcKRyG2h5ffZPxAlf5uc1SDmQQPaNbRcdFNu6hm+ID94UjIAVUvNZh3JSiSfHYD28rC3tRJq7puxlCfpHQhCSo7BSdT7vibKVVSIdva83tw1T5ADzKk6/AfHwsmt9bWVV3Kg9XTKZZMfrFMmNNToB77UM3JWLLm8WmLvvB3rEJy+hdWkct4Gnn9vKxahpkLw0lOTQNqBB9o+42Vr3pGqy/GKdUZUILiwDGbKJA8dSCR5WRk1TlLcLdLTJl94qSkAggDMZPfEWm19imsoFvekeYrlPKyp6911wp3DQz5Up3k6J56WlGiuZijuu6Wm4KlOIdcWUwVqIn2fMWjrEThaqG0OiU/SQCQNc5KZ5xr38z52lVhJVdt0q3P0J7/wBGdNNj9nugbJwNE7qgNUdKupdKMjaYk8z3fCyNhu701FZW1zie2uEgkjaTMD2fO1lGtzXveSLsac+hQQ4+oD1gI07+63VwshqvvBLaIHWQoJOgidPC0i65CVrBC1t3shlaQlJUAYPMiNfq99mtRMJp7jWy2gda86oNwnXYa68xv7fZZ+Ps6EOJARqJPftZGue7kVdT6VADFNDbKDBM/rH5+q0HHVSaNLopdmHG7vo0oDaOtUQtxQT+l+G3yDZKu66kJxLVKDY7QV2ADA202s+XWgDnI9bXaIjn9VkFhnJiCoU6YzJUdTpGUbfH5FmcQLJgCUUvWjFJTKAZCVnRIVy8fAAeFmfUXE071kntkZySYzHX7/jZ53i4qodW842CkHKmfZ3+XzFizN3Pei1NW4tScjZy6+sfuH22C92corW5Rqki4MLUzFKKxOQOvySRqAO4Trv32YPEmhDbzggzlT3e0QNeXx3FpkuxKTdbEEnRQ3iO1+O1ow4rNhLjjh0SGASZGuhAnnyMi1kAZLIJvdFME3ei8sFuqWcyQt1JhIJI1P4eznZx3TcyaK5WmUsjOucqdM2vd93PexHhI6hWG6sPNhYRVKlIIOmVJCfPX4iz8uKnTWvqrXcqg3AQnkkn2d3jy8bDIBUwSBdIl34KapmnHXG0l92C4qNhyA9/zAsYOHurWClJSdwdR2hOvh8Puea2kkynNlnQxEdx+y3BZAOqOUxBlPtFphjeihnd1Ue4iuy8E3ehTNfVtwrMCh9SZGuuhAG9jVzOYqZu6m6nEt7oUhvLmFc6lehImQrz+d3LflKHLvXlRoiFSfu9se3zsDdNOk3clJJhJVlHPeZn3/OlhcKMu1CmJX20JRJm/uILZPV42v8AB3AF4PaEd0qju+FjCce8VKYdnG17LUmQC5UFZPfor9ulj6aNKBJQoZRzgz3D6rJt4BIKaenSVOLJEBPLu8hp7rRfTwn1R5KYqJhs4+aBreNHGthfo13Y5qS7mjt0zC9CN5U2dPrjkLKGKeLvSVwfwwf4irv1p6harGqMP1N1MhDripJSmEDQAaknfTUzC9w6wdhaqvRbmLr6aoaZlHWqzDtvriciSAQB3k8vgL0p8UUOJuG1ZcFFV0r9OyylbDNIohinS2ISnLonNGm2lqMlLDxQxsY8dFaZVTBhcXnzUXXN00+Nj1DSVFQbgdLxhal3eofpRPZcH1cjpaa7m4/cQ6ulpqiqoLmUt5tLpCWXEjUeKzHdajdyr6q6aRZKQtl5RGklPa8PP6rW5wo009ctCqEkdUADHdpz+ftFPQU+bRgVqCuntq4qQk8d8WZYduC7HIEnKtxEjnuTZKvzpSX5cFQy2vh+xUodQVZ03ipvKRpEFs/frtYk3RtlGZMyoRmGp9/tHw9jXxpc7TvozpQP0khQjUnKffvascNpyfmqz3+e1w5OhjpjLj+M8M6hJ3IbvZKtCd9WR8z3WNI6ZGHf/ruBL+aVE9l5lY2HeRr4WiB252SgkMgZQDHhsBHtHdvYBVxMKSW0tp7I1JOoPL26j3D2L5Kpzy96j8p1A5qbkdMPASkjrsNYmayzP0DB7+56xtrpecKFrQmqRfVMSDJXQEx/kqMjutXt7DtMAlwMgKzQBlgCO4DnFir2HKZYKSNdjBIPn9Zs3yTB4p/lScdFZxHSq4MFMrxPVs6AJDl11I8dwg+491uz0oOBzsdZxBpWCTH01M+2RoN8yBaqFRhencHfMJ7u+JJnx+TZo42wkGsNV1a0oyhtLglKYhKhMabxvEa+20m4LATuUxxidvIK9NL0hOC9RlU1xMuFIXoM1YlGu59bzFl2l4w8MKtKU0/ELDi5GkXmxJ/0pB+63lGEg3b65BafIAgScw9/6PlaS7mw8xVUDFVmH0jKVlKhIAIkzr3CedmmwOJguHFPHjUh3aF6UMYzwvWpilxJdT0iU5K1pU90dqxxN7UTgJp6xt0aiULBnfkDbzeRg9p1tY6tuFK3UNZjl8LFqjCDaUEMpCNQDoRrPh4ft52q/I7fpe5H+WT9FelqKhS1dlJ/W2NuV1BSE5ioHkZiPn53t5nuXXfNOgooLxq6dwAlPV1K0kHYCQRH7LZdNZxTcqEM3TjXEjOyQUXnUIAHnm05+FmGC3Hz/cpDGBtk969LhWKJ1VOoEEz+2wgqzmPbOg3NvOeqxjxaw8ipSrijil55umLvavd1QQcoIACla65tY5W9C/yd14U/FngleFdxASL8ve7cQP0aqusGZ7qzTsOISToYClrj22o19CaGHjXuLq1T4i2Z+TLYo+ahMFKdDrIHd3a2LmqIUcp3GxO/ltayznCrALqwVYcaBGnZdcGkeCrEqngtw8dSf9xXEazKKp0f2rYffWHcK9xAVX6mq0qPraR5n52so014JSuM0yCOyeXO0uq4EYHzlTa70aJEQmqmB4ZkmyXe3Aa5xQVCrpxFfDFQG1lkrUytAXBiR1YJG2gI08dbCfWxlFY4XUavXm4XU07JW46rRCEHMSfCP2a+4/Q3E9VKRU3osEo1TTJP0Y/pfrHwiPOw103WxdbQQElbuhccXqtZ5yR48rK7Mq9UkgHv3sUAv2RiQ3dHKVpICUhIEAaDl4eH4WXbsZJWEyNNYjayXSNdvXXwn50/Cziuxo5gI+fs5W16Gku65WbVT2Ccl3JDbSRms8rrdKqQTyj4gGzPpjKRAgHezluJ3M0pMbAAeyfvFuwpm5RZcvO65JSoYHvtqeU6b2xWu9tTy2i1xVFubZMb99uSZ37ramCRFkkuiRFtkwIm3OYDf2W0SOR2sRouoOKDqgVU6wD+jP225bV1gSQfWiwiwFAid50sVo1zTo1hSZBM7EGzSNuNEmmyUq+oKECjbVlSkArG0+FlC4FdfdlVTkAlBzDnuPwsgLJJJJ1O5ssYUdiqeZ3K2iY7yCPsNsttCynpTFvzPj1RDIS66j/iNdVRUqTUhCH6dlrOKVQIQpe5UvkoARCSIBkmSElMYXneuJr2VUXgL0dSzSPguqUTlQghRypOiQrMWzlEkjWNJtNWObsfr6J2ncqV0dOApKi1o6dDpI1SCY1Gu9o7uK77tve5K3DTqm6NCaynSnqjBUVrhIjaJEaAT3zJt5RiBMMksnC9Frm7jW19bBdTG/NE1ybrd64qcbS41K0KSFJV173aB2P8mfrtlnK9cvFJTq1UVRTsU5USy03WpQltE9lISdUgCBB2tlod/pfot/b6lDO7r71Qb8pW8pV5YEYSBmQ1eRI5jMaYD58LRBwiRkp7mkQC6IgaEBYHz5m0kflGLyTeN84BcCwQ7dtVUZSdFZ1NSPAwN/vtH/BRJTS3KYDiEiUEjSJB0HtPtJPjb0jsuLYdDfx+8qjj5vWS/tyU7NAJGxSUgHfVOsfJsoISgwd9ipMTpzsDTNJjUiAJnb7bGmW0pUHExI1PZ1j5m3aFcsxddWcsSTAMA7T52wN5TIGbl3ef2WEhQGihG0Ee32222FFEzonck6+P1H53ZT2QaEIBG5kx4jTT9lhEtJIkIkq0Hz7vwFusvaAJ21G2h+QbdBBCSAQNIGnPun52s3NPZcKQnPAUowqSOcd+1uglTgSoqjMCJA2GmvvsIGiSUo31jv8Ab7rdJaIVmBIjWY23nnaTd1A+CDU3ngqEGdO4ecQTy3+Ni7gIAKz2V76T32OlsQQcp0B00A19vhYu+2CCUnSRz1Pj4n50sQFCdooM41ICb/oVHL/wQxzg51a+zv8AC0vYTQBcF1klIJpGQSka6ITz9/zvD3HJ5TGIKBSidaQxrMdtU7ju+Z1tM2DU9bhm6lbTQMkjXfINPq31tBpu4p3fMCWkgyUicsyBHu1+ftsYCSRusGRy1nfWfnnYJDSkQnNtr2TzkffYyhHZkjMUjQHSJ5e+PutO6Qb0WDMnNJGsa/fbZlSZIPu5d08v2+YFyiFKBSrcpgfCO/50tstFaZRokiZ+e75i0MynkXBbbdnKga+2NxFky/0JNzuGdQUwTz1B+zeysQo/RhGsxEny8pgmyfiJrPdRUUFMLQoQQB62/wBVoPPo3Uha9lF7pbaqr3cchKUIRMnmUbd/hYqpluguq7HFFKQt8PLOpjsk8t401t1eqKh6tr6OTlWQ+4QmYQhEhPviw95ssv0V00IVlU/1YIgk5con3gx7bZ4N1bLbJKo0OPXkqrqMx9KZcdyieyg+qPKBYvddChFG7eJbkraWhnT1UDUnwPP7ua1ebrdHerinFICWqIJAG4JMCOesbWKUqEIwupOQN/QKJJ5b9/zr7bSadVE7Kv8AjRo0z9MICiorKgBJIJBHjuLSbdqUfuXuyocAKWmmiMx1kJ2+r6/KMse0tSzejNS8VdU8tIYSZ/kwkHMdP0iTZ9YbXU3hh9kjMmnoadtswYlzQcu77PGzvOmii1SPh+hVT0pffUTUVP0iiQQO+IPPUfC2rnQRfN4NyTOUme79tlC7FKcoqd6FdptJgjwmPrsnmoRdt81jjkpHUhWukns6eFoOfYBSAJK7vZ12ofbuymELWfpVJToEzz8bcYbQtLVS0Z7LsgRvobHLjo1KQu8Klv6apJ0PIb6WCw+oh+sbSCAFgmfMwdfnTztDMbgolrCyPlor7KiACBpBMaj5j77N29uop6p1DZKnHIzmIypAGnvE+7lZw19WKSnLhBLitAJ2+e+zUdWVuq7MuOeyT3WhLJyTxs5oJqkdrKhLSUwZIJCdBG+mll6spG6W7FtgiEtkeY9/z7bD3TdvolPnWgdesBR5EeH1W6vQoNE8sEphByab92nzz9jsFhcpnalI90Nn80IStMkFUSfHu949to44qsBbSipOfPTkieZk8ueh+NpKuoD82ytZIBWDO++3xtF3FqtLVbQUTPaVUZkqJEdgamfAA6/JtZD/AEUEtuUn8IUvVLddStyhlt0OrIAjVEADfU5fcLShhBR6p/sEqlIMmCZB25cht3jbW0bcEXgp++KYO6lDK99J7YjbTUm0l4UIVVVCTmAgEEkDYxH1e+0Bo4JyLhONLZCipSSeY1327/vttSQ4dBrAB52HSjSAQAYEQIV7Pb8LaWCmVBAJMjXcd+9iF1kOyS7wZKqZ8ETKcwiNIsSuBr6FwApHbKjOhggR9XuNluobCmFiNVApE6wdu7XayXRNquymKnUqDjhzBI9ZPcNOfz4AD32KMxuiyrdDSSloFSlmEhJk+Ea6zr5W5YoENEOvLC3l6a6AbyB8ffY1S0SwTV1KT1xTqDplBj5+TYdaEqOYpCiDOh31H4+evtk031KZwI0CIvNndMaRqQBpy+TZuYzo/ScM3rTgmF0rgEbeoYHfzs7FIhtOYyE7Roe/u87JF7s+kUVQyGyUrbUk7cxqY9vfYwddDVUbgSF3MspGrdUVerMAoH3bc7W44bPIqsK3c/CSktAgKM5SRP1HffytUW5MwZvCmdBltxJjnAzAGTz9mk+y1qOCtUmqwVRgFJ6sBIgnu0Bnu15WoT6FX6c3UiISAkBQBBBJM6kSd45bftsjYupwaJtaYKkvSrLtMHb5G1lpJOkExMBMxz56e2yVihhxd1rI16taVQZknWft087VwdbqydrJmLaBXnUAUiCsgg8vhysGppJVKEhKssxM92x7tbGiVQViNe5Q93z9tgSFECVAqXqCZB08R5/H22ndCylFXmyrsuRAgE7A7zv4T8zYu5TgJEdrMomY5xrOvd9VjqxyEKSkSDM/A89PmbAuAxBAM6kmVH6++de/nytIOTEEJKdagqAhMTGUTJ8N/q+qyHiGhVVYfvSmSYL1G8EgCAeyqBPnHydXQ6DokpIkQYGhPgB36WIKabWvIpIyuHLEbgiD9v3xYrHoTxdVcQAq76tsH1VtuDuAkiZ/xhv9tpewGfSMN0DykFKkthseQVlke6fbaJ6ZstIvCkICfojueaVA93gbSdwvdUu4W21EAsvuN5SuDGhiOWqvjYkx9FDjGqejdMFNwptMxBAJkDbltPt0FsqaVJHZgGQoZu77vP7bHqJhdQoIQ3JG4mRz0s5aC4A0AurRKwQogjWY308rUTJZWmx5k1rvwxUV7wVUoKG9IURqdAdf2xvtZeRdTFEz1TDSE7kRzk76ePt3s4AylKMqNIj1dx8x7LFahgpBUACmJKgNTHmPL3WjnuUUMy7KLccUYTVklJIfp4J7jBT5ToLW7/JLX6XLi4iYbcc7dPVXbeKElWvbQ82sxy/k2/eLVWxzT5W6Z1KdFpdQDEweydPOfnW01fkrr5Vd/HDFeGnF5W7zuF8oRJhS2KlojuBOVa9d7UsXj4uHyD60SmdkqGr1KUrMErHO3KxuY2+FuUE5MhElJt2dgT3W85zXC6C1kXkkx4W4c7SSDz77YrsqPibcrJ79edqr3I4UQYnu1VJfNQ0AQCrrEkc0q1924sTYaJVzBA2iz1xxd5Uumrgn9ZpX1p+2zXQwEqBB9YhWvfbpcM/GxNKU77I5Ro2UoEWcF2oSkTlskUTQJEpkaaju52XqNASkDbQGLdZSRWWDVS3SswI0ny1suXI4UuFECCD7zr9hshs7iRO1lO7XCiqSSI1HxMfUbbLBZZTzcWTizcvjFtA6Ttbnbe2TPIb91rFkBdmAbaJ0525KuUe22s2kgju2tMC6i46LeYz3Rbc6QAdbcE66fA23O3zFiAW2Q1vNGvOxRk5VvNq0IWSNOR1sbMgfCxUgpql9ykBWlnc3RRB1W1SNdYsfuJ4M3swqfWJR7xFiCtoB8rbYdLLzbupKFhQHfBmwZW3aVIb2Sli1nrE1TAIzEBQHmPvtWu/LseqsQmnW6pluczi06qSgb5QPWVySOZi1ocUtMNBd6OuJba9HOdRMAJGsn32rnflNWVd8PV9bRej0ueaancRK3kwIW8lQgCZIb5/pRBQrgcQeYqos5uGn3LpqFwfT26IHrMRjSiwNTrp/8CpdCp1SkfokrntmI7XPe2WAdQ+46txddUKUpRJJWSSbZaj8nu6DzKNlcqD/AJRYGkxphGgaWstMXU/2QnUA1CgnUchlHwsyuF1+MXTd1yPLoHnQzTNlWWO0ZPMxvHws6vyj7yFcVsOsqmUXGDIMb1Tw3jwPzoWRgVtCLgu7MpK1+jwVAbwpQ0A8vD2W3sHlMGFwuG9v0oGKxiXEJQVNdPxNuwIzLuq8UR2vUQfOO1ZQb4lXCtKZo7wEawGQdd/1vG0aMkK1TprlB0+e73WPt67k5SkKJOvn7N/YbaPyvUeCoDDYTsFJKeIuG1EZk1if0ZNOdtddPnawox7hnNnFY+haeZZXEzt4/jaNVEpkZuUkbew/Pfbppz9Er1JmIkfhpZDGJ+gSOGxDqpMGOMLrSALzIUdszbidOfLz77Dt4wwyIUb2QkzBlCk6e0eX32jJBARmQIUNtSJP37aWwZSoyEkjXWII7vaNrTGMTdAm+TYwNypXTinDa9fz3TIg6ZlR7/nnNh1Ykw8shIvygM/o9cNfZOg2tEYypEZJABGndy0G9uFpQVZSgJnQaHTWdx5/CxBjUg3aEM4azkVMCr7uYgpF7UJMSFekJjQ8pNgXL0u5SShNfSrkf8ckmOZ+u0OqCVArJTpuQkeMnTxHfZOqG0IGbKmBpqkSBsPb+Fjtxl3NqE/DGn1kS6Ql4UrOJLtcbqAoCkWCUdpJOc6GN/wnladcB1DKsJ3O6h5uVUFPoCDoUJ3tTziZUN0t60sJyksEwBBOpn4c/wBtpRwShr8wXapLWX+Ktk+OZI3Pfv5xYr8UMTRJl3QW0Ak9AHZWYbamCkkjeM3zP42MpaWRGYmIJ3133tBbSyJKHFpIJEhagN9R862OsuViFQK6rBA0yvLEbfYRYfy407s96M3Cj9JTalKwAoI1BkknfW3CkBYSlAglXaOsjx+y0PNVl6BQIveviNIqFiDvzPnYy3fN9pUSL4r4k+s+dDM6TzshjTObSkcMePWUuhtwgBRJ5nx2NiGIWSLtckRlUgwdJ1tHbWIb/wBxfVbM6y5IHvHz77dO37fTzBbfvd9aSQIWEiDO8RvNk7GYiLZSkMLkve4RN9gOVF+VC0ytDaUAgagBEx46/V706mbXVXhQVyyQ224lhpI7koOYxy1+e9QLlWpTx9MIXUHK52R2o0199uEekoDCG6kFNN2kAoTA+Z3PutV7+0o5o3dUWvijTWXy6k9pFOx1rgnXQGBA5E6+yxG8HBRYSDLsy9TqSk5hqrU/j7LKaTUdc9U9agu1CShwlAhSdvZysk3pSLWwhhak9W0kto7HgPHu52m2ubdDNG4Cyh3ioz1T11KSSDlEKEnTINu86c4G3lZ8YTpWlYNYDaEguskqjfNJnz208rR/xgqqlu86Rp5xnqkNJKEhMEEaTvqIn32efD2sqnsNUiEKbDRStABSZHaP4aeNrD6poaHILac5sqkPDTiFXRSOGDCRPa2gnX4fGwNalFVfLVCDlCh9IomM2XkfnnYG6XqqgpE0jKWihobGZg7DfXn7xYSlo1U9W1WpUFLQorOYkBUnl8bBdWMNkZtK4apwlE5UpHa2Ty0iAI7vbZGuhaGr3r0rVAHa8QJn26Hf67GzeFRCc1K32gJIXH2WILbUpyofbbKV1AGzk6eXiYtF9Yw7JNpnc0TvWuVUudbmAbGiRpuI1Hzy8LBXalsXlTkpmQSRyPrR8RYZd2OuOBakDLPaAOseHcbGHGwK5mrZYyoaTlygAk+0mwW1DXOuVPgkCwS4hIzCAUjtaCDP3WI3mpz0KoCdwhRITryP42EcvRvKUBl4xrpBnw0+dfeXermVoW0WHRmKhIT8dD3WtmpYRugcB10hsVwpLnMgJcKylAP6Xj3xIsyMaXS4xd5r6pJdfcQ5EicgIGg+fqs+W26cVIqKptamW0yygJkCOZ+eYs1uJV9XexdiVOdYCHMmrSiCMp09wP47WnBOwmxKhJC7kEyOBr039eTedULpkkjlIWPZ+kOfIWlXC5AveqYSpUAGQobdoT56RaFODla1R44qEuGErYdRojUnMlXlyO3M6bWmi5nWmL3cqDPVrCyFAEASZGmp5b+FiPka0i6i2IkJ6AKTA3MyUawR+y3OYgBOYka6kxA9nzHdYBFfSKhaXNJIIyER4H3j5NuTW0aEiHQSkRoDr8/PKyM7eqiIHdEK84GgolyCTlE6kTvpZHu130m8VqfJKkJJAjbUAn577GmjT1T6n6qpSlLRhCRoZjfeDysSu4IZvjO4gpaJUnOtWhjXSwDKCUYRkBSBhbB1VfyF1lQ041QtmCtCCetXp2UkH3n8LPZvBlK2kVNTTDOYLbCqZQBT37GB88tDdz1+HWaKmqF1lAG20hDDAW32lbdqFgx57zZr4zx4xVpdu65KlAz6PVAOUnvbbgns6b+wRalxpJpMoNgrAjaxtym/jB65qaoXd91NU7oBh2oQ3CQZ2RpOnf8AJZFW0HWlNOJURGxMc9/L58lV0IdRlORJB0IiB3/PjYi+0mCFws6gAGPnztrxFrBa6z3gk3sqjinFFiC/qFaTKHFJSQn1YcjXTuP3WsPwDqVO4XUnrMxaeMCZmSrTz/HztA+LmGqHiPfqVEALW6rxMhKtt+R84Fpf6OVWHKCupgrVtzUTMABO2mmqiPZYE+uoVmn0NlOQ9fsmSO0dORH1xYnfiVqul+FHspCgZ0ABB9uth2lBGyhAEDkCOYA25W4vJC1Xc+3kzDq1JATz0Oo+u1W/VWyExIUG1EggpG+vdHP26d0W0qZ7U5USBpz11+T3WwZlAZz2e7bXv1M8vhbTioAUAMpkExsZg+38bLMVDVBuJUQN4k6Ty029h5eNi7hBkJQpSY08Z19v7LDOGQFKiYyiQSIGvt52DWAMxWqfH1QTr3+Xzye9kxF0TdSCjNmMk/paxA+4fWbE1kpJVm0kyEmI8vh3b6WPrQQgoymdB2dD7NNfCyc5BUVNkoIiNdZG45e7y7osRjlAsVd7zoRS4vvaigJ+nqG05ieZVB28j+FnpwaoHqukrqVClDq3UrJk6ZkxH+iLEca4ddYxtedcqEtpcbqEJIjrApCSr6zPxsucEHQxe16URUkKcaQ4AddEkif9Lz2sSaUFhsoRM9MAqZLqu9mkR9AgZzqVDTbXaymnIhEKIA3zQDP2d1izAQk6JkDQTPtA1+fhY03mSoE+sBJAMADlEba/bbLLyVpNGXZdAHKAUiQQAQNdvwsVq2kKQSViTIVqcux+2ef22PZ28pbV6o2JVvt7bAPrzy3mmBGnL5087TY5M4XCj3HbR9AS6jMMrva8CpKtRt9Vl/oIXuMNdMDDiCvK3eXpNGvlIdpHMs+HWJSAO8bCyfi9rrLofSmQQAoDNBlJAnTwPIGzS4PXyMMdIzh1fanUtJbv67FOL2hHpKUrHnlVBixJm8WmkZ1BVcejI0+K9wnAEPHQQoW6g5ZBtlWICVJ/R0toKzIzDvt5aTYkLpBq0FFnCM2k++3Czp4m3bx7WYHysC4tKUZlKypAkk8rVHHWysNSbf8ARmtuuoQkSpsBxPmnX6psx22wTECN4jT2We1wYouHEtZeFDc9cmr9ByB5SEkt9rNolWyvVMxIs2KqiNFWO0cfyaikT+qNvhbqMAJLSwoNWS0WK1SMgBKcsD67LFKAI2P2WJU7ZInn4yPGx+nEGT+Pz91u4p2rnKg6o+1GgB2scplZHQRyBI91iLYJ1Osd2k2ONKAcQO4idLaABVIpzzIm2A67+NgadZNO2SZOUA+JGn2W7zaTaw0XVcld5ttLcz5e62p0k2yRJ1MWIBZRW7bnSBbQPO2Sdd9LEaFBxXW5Bk62L1Bh9k8lZkn3afbYfNIixavOVtLmsIcSSR46fbZ3bKI3Xa5jUA24UTyt2vaIsEqPwsIi4U721TvfDdZc9DULQFFCRqRMEDKT9do0xBdbD18oQ7SsPqdUlCA84UgEneB63L6vKRbkdD1wFopJ6pwpgHvIP22Y+PaYolbYOmhIFuG7RQBzhY2O11v4Q7NdhXCsX4QplGmTctA4lo5AsIgKA0n1LZaP0U9aEgClKhGhlWvxtluZ+Rm/lj7lf7szr968yPyi1Sl7jLdbBOjdwMgJO3/CKgz7vLezVwVet1U9w3ewqvpUKTTIzhTyQROskTp8762cH5QGrnjq03nUjJcNOhWU65VOPEj3HW1cnVrbq3ilQKUJTKtSYCeR9h5R327PDIBLhsLdvRCq4i/JXSu8VZmnvK6VhJXelICdY9ITry+3lzso095XcoS3XsKTtAcSY9x0P3m1VkrKSG8w7iknnMnnA2+qwgfcVP0hSU7Jjceffy07hYxofFVxV25K1qXWAQApEqERmB/YPu1mw7TrZVAKdIJ10+HntFqm+nvh3My6pWTQBJIG+2wmwwvSt9ZFbVNZv1XlATEe8nb8LR7geqc1d9bK2SBtmRmA2nX3iI77dE9tJI0J3nnIE+3Tbv8AfVH8/X0yjIi9ayd0FL65B10mfj8kdGKL+bbV1d8XgExmj0lYEg9wPf8Ab52QonDml3sHkrUZ0ntKHtHcfD22CXqBsVLmYHPQ7D2fhar6cZ4padStGILyTAlIFWoyff5+0RGuopx7ixtMJxHeLgGhJfXI03k8pAPyZXc39U3em9FZJxzQ6aCT2dJ2+GliNUoqbKW0knYJjbXl3DxtAH7vsYhaT+6GsIhI0WPZAO+oj2m2OcRcXoWSq/qhRGkqSkmee6ZjS020rgomoaUq8WlH87UZmVBk9lUkSF8/Hz++0t4CIVhy7DClTStSDoQconSPnytXK+7+rr8W07elV19Q2MhUUgACZiAI8dfqs5br4lYnuigYpaGpR1LLYS2lTSVZUAd5Gw07zysWeBzo2tCHFIGOJVlKcpJIMgjcjWPs2EcvZY+zABRl7J1SM0zp4+dq3scacaN5j+cKReVIgmmSNSNDpuOdhUcdsbMy8tVAQmR22DHl60+7vtU7pIrTalgVkIBT1eWTrJgHb6/nusKoQSpMHQ6KTGWNZnu0PvtXRHHvGTYSHmLqMjUqYWNOYHb1HePZ4WGR0hsWNjK7d90rIAEZXNRMT6/j9fjEe6SqXeWFWGQoIUkJSSTyKQJHd4fO1hi4AAmMwPhHttX5vpE4jcAcNz3Yon1gAsSf8ru+e8f+ERe2SVYdpFGJhDy9d/Azz+TYfdpb7JxUs6qeFrSgBBUd8sEfJ7zbkuBSEklUjtEzp8/j7YNPSNrtOswxTqEiCmqV7hKPt9thU9IgkqW5hhsZSEiKs6nePV8Prm0u7Sjkk6eM81NRdKgCAVAdrTu+3U2IVSyQQlMyYkk8999uVolb6RdPkB/cusGf/G5IOw/Qncx77dOcfLpfht/D1WnOJBDiVQQR4Df2WkIJRyUOMzqkTjU+pm8KR5YzAtgmDrIKtvH46bWcnCt/PhOiJzjVYnb9NXnaM+ImN6PFq2amjo3EFpIQouK3Gs7acvr8bLWAuIN33JdCKGtbeUttS/UQCIzTuTPdva1IxxhA5oDHt4pKnaidShBbAmDMnfy0seCh68HygAc5Hn5d/jaMabizhtJyPmqBO/0cx7jzIjv18rKTHFvDCx26t5JMylTJkae36/wpFjuitZm9U/itOYhCFCSQfE9/w+Nth+RKdIkSNNe6zCRxYwk6lOavcSFSJLStORjTSwh4p4TcgpvIjb1m1d2h29x8rRyO6Js7eqfCnk6ZlJKRoQTtbjrEwSkjLMmIJPd4WZjfE/CiiJvVO+pU2sAeE+/67CHiLhWE571ZRAmSVAnX49/tssruie7eqd4cQSc2VI75kKM/ttwtcnKZJ2MHb5+/lpZqfvhYVUlSfzywMp1BBTMfPL8baXj/AAmqP926YRH6R5jn5x4crOA5LM1OGoWCiZIERA33nlZicR0ddcC4ROR1CtOY1095HnZXdxvhdZUoX1RkzsXQNdNI5ab/ALYbGMsQ3TV3PVNMXjTurQQcqXEyNY2nzsWLMHIT7EKOeGj6WceMgIcAdS60CNgchJG3haw9C43KEns85VsbVnwPUljHlC87UJDZdVClqgTkI928ePttYm761pRADjeihACwRHjrra1VEhwt0Qae2VOdDmmdaVaQQCdTp3+U6+It0pMg5+yTEERO/wC34WJtPgo7KkrCtyVZjE8u/wDGwhXn7BSMxkhJJkE6fd7rAzFHsENmAMCTKdPuj58bBkISD2Eqy6zAI238LcBzPEiQCAB3bEjw5W5WVQOzJiSJmR4ff8iGcgpsoJQylMzPVpE6TEDT5/bYFaG1HVGonTnvt8x8bcB5JlUhI1012/ZPutylwCSQJHePmN/nS0muI1CcgELhxumBV9GdpVz005GxN9phMjMJkp7CtD5/f42NuqXOdRmI7JJjQ/hPt52IVbpyaKHa89dNo+fZpYrZHIRYFAvE+mpqHGdcptpSHFobX2SRqpAGo9+342WOCdbUZLybYefaKVtkKbcKT2grQ/5E+wWKcYk9XiFhax2XaZuROspWsaewixDgpU9Xfd5UqlFGdhCyADulUGf8o2vbw3VVotIp6o6+8sySL0q8hTmOZxXhr7fGx9N4Xi6gtm83VJV60qmdtIsi0ihAJMFKeUwDufdp7uc2UUKK4IWnPm1MH3/UPIWpF6uWXTdOlJ6suKKkmDP6p+oax7bc+gpblB7JiJnu0HlE2FaBgkgLyTqfHvNuswBCUjPrzO+32j4WbiJZEVVQqcTCXzvr4Dw0+YsEqiW0dHVBI1GbWPZ462UFFQCAoBQG53I8dLcrAzEEwNpjNrHd77ISJ8gSWaBRAAVCtJOWfw7/AJFhKK6EhcqlRBJgH7D42OLUlMpUDJBzakA+32/VbumKk58qhmjTu0mdudoulITtjuoo4tXeWL4D6UwX6EEEcoKkQPcOX4NThRV9VjMgEhNUw8k+IBSqRI20jTujS0i8XGULN11KkZkkOskjYgKSf7Ri0T4EqHKbGN2PrXEryDtbhQI7uZM8+/xseIl0ZQXtyyBWKbcywUqJAOyQYA1jWPmDY60ZAhBgbTPzNkph45dBCT75juA18rKbbgKiFhIzGdDqdPP5i1O9lcsh0rCJKRM6qBE7jv8AabFnXQE+rtAMK5eUeHzFu3HkFYOiTt60x+Niryj1ZzEaHtGd/IgacrLPZRLQkPEK+voatkg9unWYiTsY+Pz3Qnfd4uUd5XPeVPUErpnCtKpGhBSUx4feLTrXMocPUrEpV2DG4B029vzraAMUsqaoqdb+YejvBKiZiYIJJnUfaogWuwOz+iVVmbl1XvtdN5sX5cNDe9MoFq8KVqqbUDoUuICgfcbC065QURsbQ7wF4o4XpOi7w/xviu/qS7qJGHaKmefqHQAXWWw0tI5qVmbVoJOlo0xn00MJ1j793YVvF+ipNQapdOrrXBMSnkgb96v6Jt5lJSzOmc1jb2JC6OnAkaATb2qwuMcfXFhVBRUOmqrYJTSMEFw+KuSR4n2TaDca8TL0xJmarqkNUavUomSer8M5/wAJ7YTt2edoYr+OGF6hSg1eSpcUVKLgUVLVOpUTqSYMkmbINXxYuKpHWfnEQSM0zoPCef42LHhst7lq2YjSwD5wJVnOjpiJVXja87rU7mTU3f10H9ZtxI09izaXcV0fV3g3UoACX0awP0k6T7otUro08RbmqeMlyXfT1yVOXiiopY1k/QqUB70j4Wujiaj9Iu0uhMqYUFz4HQ22cOYaeYBwWTiL2yOJabpsU6IAiNd9dLGkJgbExvHfYBlJjUiYk+djSATG+nOfG3awaLl5tShmj2hoPdYygiI1mxUIIMESCJsOiI7wBa+FTcbJwULhcY03CtdO/X7bGASBrvYhdawWVJ3+SPqi0DcWukbj/BfFJ/hvgvAFHfzjVA3XFS31IXlKZWTsIHfZqisio4hJLexNtATr9S08B7PV3aWpdS0Abma0vOZzWANFrkucQBa45qxMg6n6rZMmPfFqvUPSH6Sd5Zhd/ACnqcupDVWpUD2KsI/x+6TdMpKKjo9stKX6oXVKST5Sq1UYzT75X/Yd8F0f72+L/laf+swfrqzsxtbYURubVn/fy6U0Zv4OOn/PL++2kcdulG5Jb6OaVQdctQs/bYgxqmHqv+w74KB/BtjB/laf+swfrqzQIkfDzsBeCM1E9lE9kkezW1bxxy6VAOnRv/65f32zD/ST4uL4lYb4ecQOE1Lh390b3VocceWXOr1lSRsdRFkcapSQ0hwuQNWOAudBqQmP4NcbyPkjdC/I1ziGzxOdlaC5xAa8k2AJ0Cse0pLjCFp2UkGwaiCNhNg6An0NKDqpBKT5g27VpsdjrbTK4EG6c2DXesTWUyjopKVD4g/ZYnfdIyto1FWkKRTjMpCh655D55TbnCL3U3uG1HR5tSde/f7LHb/bqyy4y1SNuaqKgXcsd3Lui3J9pmgQFxB+r/stHD3ESAXUdu4hr+tXkrK1KcxgJfCUgTyGXQeFssKqgvLMYeSgT6ofMJ8B2bZbzTi0vV3vXV8GDovHbp31Af6QNQhWzN10bWp5FJVEf41q/Ooy1rwcyJMidOc6nvJ37/K069OB0L6Ql8QUkt0dCiCNv4sknbz+u0Hpl111KgVZjJCANSCe7b7Jt6nhPo0EP+6Fg4mb1kntKxtakqMjUAjQjfkY5ePf8bYoDdSlyRMcvH7fjYXqkqUgnUgyNRrHIc+4+VtlAlKFIkqIOZWhXyBtdJCq2XAdWDGVBOUmCSIk/PiT7LcqKgSoEALmYMgxB8RpYVxoIPWlQTm2kgT8wLdZT1uY+sokHyjWfd7oswITEIslakgEhQIkkKOWT7NTbCgEQlKgDoCZkyOR2nnY4KYqgtJGk6FMDv79dPqPdbSWCFkxnJgQeRnSO4/aDZZgmsiuYkpyZU5tgRqeZHuj3C25JT1acxAB0Puj55xvY00ytQVmWVIUoJyZZkwd+/x+RblDE5QpKpIAUCDKdNIB8h46WWYJWRRTnbUUglc666kzofr38e+2i4Sc6EpbC9co0ze0agaRYyGCgZXYO3ZUmYMQNOcfjrbhTTaCFhxIyg9ncmNo+PcLSDhsolpRR7s9oqT2VFMhXqkbAd4039tjjavoWwdcicp7WnjMb6Tz5+MWEue4LzxPfF34euChFVeV4VKaSlp0qAzvLICUySAJmJJA08LTO10M+kRlTn4fBtaQM03nSamAP+NifusOeohhAEjgPaiQwSSasBKhMulCYbBkic0Rr4n742sEvMtclsmOQ+Pl5eNp4a6FfSKeaU4rArGUmAoXrR6E8oDunP5FsV0Iuke+oA4GZSkwStV7UYkcj/Kd3K1cYhS/THmj9zqPoHyUFZ0hQWcqVJ001BPjHhpG3lrPJUOQzJUCU6mTrsfmNPC0/U/QP6S1ShSf3F0aVxIBvSkGoOmy9vs7rCK6CPSXWhKf3E040Gn50pjJjn9J4baj42f5RpBpxB5pdzqfoHyUBM1hplqUpttwJ0haSJBjb4fssG5UKcUXdEgCdtI5x4fDfvtOFV0K+kVQvqZdwR1i1GcqKppQSdxHajfmCdbBp6GnSLeUhhvATnWKPrLq2QgbwSSqB8+VnFbSnUPHmo91nB1aVCodJUQrKARqUmfd5eeuvjbSqhIP82YWRroSO/Q7eXnaaqroddIhl1TL2AXyUqBJRVMLTMDYhfdE6mxdfRD6QzRj97usjbR9mNjsQvx5eFnFXTfTHmkaeYeqVDbLoAKM0nWVecwI5aDfwHhbEVKQpKygkKBCyrTST3+Gmnj4WlwdEfpBtozq4a1cmcwL7AHrbQFWDf6JvH9IUocN60ZssQ4z3a/p6/s1tMVVOT88eahwJvolRA64tKAJMSTAEZh3zy017vtHonXA0paFkpOxBgyDMHx1FpUd6JvSCejqeGledBs60Ngf53zr7BqPoj9IzMev4aXjCTpleaGh7xnPcLSNXT5fnjzTCmmzfNPkouFQEjOFGIyrToToPDUbGPvttNWtLZALicvYgKJjy7vWHnpaW/4JvSBUkp/exvHRMfyrQMgRGivs/ENXRL4+kqH72t5EqTKSVNAD2BXdO3fYAq6c7vHmid3m+iVE4qFElTqwCSMpKTsPCdLdIrFtNntEKKdRO45e2NvM+FpSd6JvH9aAF8NrxzkbZ2xH+ly9uhO9tHon9IXlwyvFU7qK2h4RoqTIjb462l3mmPrjzUeBN9E+Siw14hEpUFBOUJ1mPb8zPhbpNeptsKACVE6nWdNt5Hd+20qI6JfSHGZJ4cXhqTp1rIgGNPX8Pk22joj9IQpM8OK0aRCnWFczOufflr99l3im+mPNPwJj6pUTGqcKiVASdfHXnr366W5VWLygqKTm7SsupPz7NvZaYKXogdIqqUWm+G9Wg5ZUHKqnbA3GxXB0ixN7om8fw6Ghw4vIzMgOtxtse1Gs9/hZd6phoXjzT92n3ynyUU+nOOqzGT29J7JI7/Dfn4205XJgJzglGgI0Khz19nwtKi+ib0hU9g8NrxUUKgLC0GZ0JHajkNfb5hnom9IRWUHhpeYC/VlaIEeStuW477SFXSfTHmFHu0/0D5KHGahaX0nN2s2wPPSd/ZYVyrX6oUoSNTpG5B8uXlaVR0SukORlHDS8U5TM5mxEk/zvfYQdEXpDBJP721aZ7lonbmM0nb67FNbSX/hG+YUBTVA2YfJRSm8atB6xNStBBjRZSANfHwt3+fb3ZWEC8aoZdSA6RqBvvvtp4Wk1fRI6QgEfva16Rlg9pGoGn62/nGtuj0SekKtXVjhxeGYEqI6xsT4+tofV+NnFXSfTb5hI09R9E+SjhrGGIkt/RX/eQzakCpWADsD63d8newycdYtSCBia9cqdAkVjkjXvkj2d5tITvRK6Q4JUvhvXg6eq63J0PLNpr4e63H8EXpEZj/2NqyIMEuNfarxshU0Z9dvmE3d6n6J8kwkY+xe08ScS3vnzapVWLMjmN/OxlniTjFAKP3S3gTplBqFDvB15jl7fCbPMdEjpEFWQ8OK4DVQV1jcQNp7Xnbr+CJ0hUkj976rIMiPSGhJO26/jy58rI1FH9NvmE4gqfolNAcUMbAZf3S1naJjNBza+Xnt4+0RXFDGk5XcRVSso0CglW+2hTHfPfZ5MdEDpHOhWXhzUCBmMVLUGec59dd+7Swj/AEO+kZSuinXw7qSUgKBbqWVAnfRQWRO/P7LQ49H9NvmE/AqfolRXfuKb9vtSXL5vBT7jYKG1FtIJT3dkCde8T7rB4exHeuGrxXX3a8EPOJUlX0YWMogzB05TItKb3Q96RLbjSDw8qWUvKCEqVUNhOo2JzH4eNjzPQk6Ra3Aj9xSZWBP8eZSO4DVUWJ3ykAsXjzCYUlUTfIfJM+k40YtpUABVA6EjtBTG/iYVoNrH0ceMUJASaK7FEGP5FwaTy7Xh862XXuhX0j0x/vCRPM/nCnII7/X17/k2JL6H/SJaICsAkJ0AIrqcb9/b8bD41E712+anwasH5h8kEjj7fyRDlzXaSCRlJcmY1PrWOMcfq8LlWHaUhMyQ+rQydx9vhsdraHQ96RmbqhgY5hyTeFNA0HMOdw+rvNk/F3Rk444Hw/WYvxNghdHdN3ISqof9Np3QhJUlIMIWZ1I5GJtHNRvIa1wv7VLJUtF3NPklxjj68klp64KdSgdctTA8/VPz5CRhx1p1oleGjy2qhqO/1fj9doKQ6knKFGQnSSZOv7bDIrFggNFOYiCR4+3Xb6rENIxC7w5TozxyuxYQHLlqE5k65XwqOfdyH7NLKV2cXLleqy2m66tBIJSQpOvlry+dImvyKl0q7eokEpIJ8fDx+yyvcdcsV7ZAUSolsDv7Px+8mwJaUBpIRY5zfVSvj/GN337Q0zNNTVKHGlqWEqAAMpGkg7yB8jSK6SsFJiFir6olLFZ1i8va0Dk7Dw5/VpZdrHkuNiElZT6uYgERp390WZ9epxFdUKIgFSSAkyIgRty0+Fo0guCE8ztQVYCnxtc0aLeSdDBR7hryj3ab2UqfHdwZUpD7vZ72TMbARGtoepnx6P1icwzfpQYjeO6Ps5wYsf65QShYKSANIJKQSfnnyOtqb2kFGbISpbONbgyIUqqIGY6hC5nTcxbhWLbiKgj0wSoCCUK5jy/DS0TpqSloElSTAHir7efhZIvXF133TLQWHXhHYQU6HfU8tz32UcT5TZoTvmDdXKY6vEVyFnrReSEJA1XBA5HciN/u77QPjzE10vV9bd92KD6HqkkuJMoCc09nQTrPLn4mzdvnFV53ytQdeS00pWbq2wAnwnvIshOEkyTOszbZpaLhav3WdUVWYWapUwFe/GriAxT4Ow1iV5+jw/TKNJR1DqS1TtqdK1BtKgQJWsk6c99rPCo4bdI9lBWtyhdGsxTsGdPFvuj4Wg65sQ3xhm8XK24rweo3lAoztKglMgx4jQb2dtPx44qUzYZGLqhxPPrW21/6yfK0Kimnz3pwy3iNUemqKcMtOXX8CnJX3Fx/pCfSKJokDVSael0Hnl0skOPcYKYAVFA2SZGrLIM6jlEGNNNbJr/GviHUp/jF+BcmT/F29T7vKxB7idiuoR1btekjvDSR3e7blppZmQVHrsYmfNBf0HOUu9HfG3ELD/Hzh5ed7UwapEYku9qoXlA+icfS2rY/qqNvcp1hNQy4wsdlxJQfbb52Lu4j4ho7yo7w9LUF0tQioSU6EFKgRr7Pj5W+iK470p79uW777pDmZvClaq2jyKHEBQPuItj4tC6N7HloHsVyhlDg5oJPtTNQyptamliFNkpOnMGLDtgeOgj22PX9TdReKlgdl0Bcxz2PxBsSA2HIaey2pTOzNDlWmFnWXaUyR3fVYVI01A07jYNPrDTT7bCjUTpBtoBU3C4SldLgJKBznX3R9tqrcSbova/Ol5VXdcraXapeHWlJQpeQKCQlRE+QtaK71APpHiD9Y+21eb0rBd/TXqq4oKxT4VW4Ujc5WiY+FqOJC7YB/wCY39K7bsHo7Ez/ALHN/wDqn7d11qwfii7qekrHl1D6GV1dOp3rQytRhQSdwkj9HbSQBNlHEmGb7vS+r2qat6poKdDanmK5JbWjRIypyqkx36DbQ87JnDSuu28q6oxTiO82RWOuFaUrJMEjfyA0A7hZy8Rb3W8ijuiidAaqgHXHM0Jy/oye7n7LabcoYTyXCuJzBJtLfV5PYCqgXVpFPUNN5idQ2pSZT3xr7rJlzvV+Hb4u+9hea3ae8hPVFKQG8qsq0SB2gZB11BnXYB4C6rsRgqtua7K2nqX/AEdbii2tKlKcAmSB4x8LMapdTV4eu91pYUtirWnfUNuIzA+RKbFtYAoR3Nkq4pN+XtiC8qShrktVFOCqmStJUiEtheWARE69obeNogxhe9Re/SJ4OrqnVOrZW4gLUZUpMyJPPe0019QGcY3dXK7CK6mYcUDoNZSr3DLaCMSNrpuk1wwpCf8Ag9U8yJ7krUkfAC2ZjNuA3/fZ/wAwXefg3/ztL/w9T/0Hq3rcIdqGxpC8w9oti5UdNracBTeCyVaONgx5G21DXS2uV5+Eaul4U150z+wS6mfImDZ8XhTBTq9PWRaOxAO8RqDaSlLFTTMVIGjjYV7xNsrFIRJC4HojQuLXaJjO0IDqxroo8/G2Wci6RJUTk3Nst5kac9Ft97K8Femj2+kRiJKCqeqoUwP/AOVa77Q8UqcqHgW1KHWkZ52n2d5jbXW03dL6hqLz6QWI6u7mTVUy/QsjzBK0nLTNA6p8RaOgm6qRypTU0i0v9aRGYDlrufLlrbuqF2Sjibb1R9wQ630qqQ+JTfNOvIVdWXIMEARoRoD4mCN+Vum0KUMqgYmSkIgns6Hw3s42b+uWmSWkUhb6w6gqSABy5zpztpNfhDR00jp00MmUmdNSoSefdaxd3RV7jqm8mnWoJBChkEkxtpuZ56687dJplJIV1a1HQCCd408v22Xk3hhRf0NO07215EjMoRHmZ5+cA92oaKzDS1JyhaEr7O4HjG+3wE2a7uiYW3ukYUyusQWVEADWEzrvJ8duX4dpbDgW2AUIIUlRMQdTzMzuec672d1xXRRX7Tu1NGDkbWUFalgdqJIjnoZ+6ykMGtJSoJyiBABXJkCY5RprYRlINkURl2oUfqYBJGQ5vU0A38O7X2aA25NGAMsEKRvAI5Hu5Qfr5WkRnBLCTleSUpzFJHWpkEEjeN+XPYR4GEYEutRCVlwCTlPWo1Ou07nz7haPHspiAlRiplKtENiUoIKYmSPw9/ORYN6mdZIztqgiBqQknaZ+rlaVzgm5glYcqHgF9ziMpJEztzn4ixd7h/cDo1vB5IMk6tkgbRtqNPdNk2qF05pnEaJF6PTuTj3w1LpSArFFBA3HafRpr+NvcpNLQPBK10jBOX1i2mfqt4qcMrmp7h6Q/DakYecdQMS3asrWUk5jUIG48re0yaoJbTH6Ii1DFI+O5jrclcw9xja5p6oQ3fdqCVCgph/0Q+63aqK7VaehsE7atpsA7ejKmxThiFzq5OkeVg1VC4y5pGkmd7ZxogFeFTfdDGiu4HSiY8foxbZobuUO1Rsmf+THx0sUN4opSVuIU5OgCd/O3CK8rGYwBvEyLLuQteyfvJOl12u7aAvdamlZCieTYk+Zt0qjpBqWGxH80b2DNUo9oD320X5Mcu+02UoGlkxn8UZbo6PLHo6BA/VG+1hk0VEAR1DUawMosTbqkjdXs2NhRU6akaDaLI03gmE9+aHNHSap9Hbg6Hsg2DF30SjPUo9qB9tuRU5jBSO+3PpYmJHvn3WQp7ckjN4oVFBRJUT1DYMR6g77Ct0tIP8ABpE88osVFQJ7vITbr0kyNBHME2c03glxvFGk09IlJBZSNP1bcuU9LMllEnYlIsX9LEESAfttyqqMykydvO0e6+CQm8UJ6MwlWqEaGNUDwt0tml0V1KCSP1fjYuqqVHrq8PC3BqTsT/OtMUngm4/ihzSUcAeityP5oiwaqWjAI9HRA0PZFgRVDYA7W49KI15b72kKXwS7weqFNNSAZfRmTH8yxarpqXqtKdr1T+gNbYqoMGZI20+fnSxerqEBsEymARPjaEtH6Oymyp13QtO1TFxagy3OXSALcqYpS20kstGTsUg2J01QjOiSJUifbNuE1RzI1kJI1nvJ+61E0ZHJWe8DqjhpKMsFRp2jvJKRyPlrYuaCkDBUKZuVRskafCwSq4BiFblR526eqR6LMiYB01G9l3O/JLvFuaMpoqKP+CsgR+qLF3KSlUtf8XblZA1SPfbDWgN5gQRHM2ARWJcflZIygkcvn8LF7noBZR7wOqGqqSjbS39A33QEDW3aqChUmPRWdAN2xYtW1QUlJI2Uees27NYlQSoATE6WmyjsdlB1T4oq7Q0Rqkj0VuAmISkCxWqoKR9QaRTsyNzkg8pNh6qrAeKwEwlM+B30sXZfJSFrCgpau+Y0Onwm0hSapu8gIpU3ZQtKSlNOwEwCQEgHfyj5nnZExviPCeDLmXX3zkpkOFLLSm6cKWSoxKRHIHeIEWVr9rWruo3K54qKGmlvLCdTCQTp4xPx8bQhw84ruY3xTT3zeBkpY9KDRdlthKyA20NYMTJ31BOkaWYqAvPgoOqw2yfSMR3hil5aMEcMb1xA6xoirrnV09I1/MT1iilR1JOVspM87B4lquM2GLu9Md4NtVaJzOGkqGng2kgZgEZQrz5bWsFhq9KD0NpikYbEIkFGgUdd+8nfzsevq8l07IU6ltKlwMugPt1m1luHQBuZyn3mRzg0BQNhDGOHcbUqwzdrdLXspHpNI8hCXEGNSUntRPgR42PVV0UTy8qaJkRqqGx2dfZzstYmuS4W3nsUMXOzS1obzKrWBCnAnXKobEECPb7i13w9Tt1KSIdAXIiACJ39p+RanHTtc4taE9RI6IAlEGLtpUVHVqp2vWBAygwLNTpBXBQXvwVxvd4pGSBh6uWlPVJ0UlhS0x3EFIPmB3WelTXpbcVTJkwMugSBmzBWeYzTAKd4g7TrYjjKmRe2FL4uwJKvTaCpYM9y2lJ+02sxUgY8EBU5KrMwheIeVGaMoAJgx7/nyt2gpMhS8pSQmdv2W6XTLbKlSZB7ebQDUgT7R4W76lx1wpQgQpOyU6nX527zbq7hc3YhbQhZV2VpISN4MK+YPvscusqarGV69lxIzJGgkj2fCZsAw0ucxbBIInMJ0kDWT8/UbbbQ3DqEJVCs0mIEbDXfSJj4WBIbghEYNbp1VCkhIQoDMrtAKR3HvGs6CNefjZtX0pQr0q19QR2Zzan27bk/GzqqUJ6kJbMpVooFMyRoOfhv4d9mriYgJaWlMjKqToZ1EHfTUqtSpCC+yszaNunDdLinKJhxMx1aO0lUkRp7YGnssHeV60tCz6RVuZDHYj1lJ5QOc/YO7RpoxMuhoGaSnYT1jc/SLExJJAjnud/dZFqX3ah4uvuF1atZWdbWmUWZxLtlXdUWFhulu+MY1leks0U0zWsn9M+ZHlZulUnVRnxto6aAxbRmNe7QWvsjbGLNCqPe5+rlsSIH1W0oGdedtwrv3PfbbsEAzvrEWIoWXKxqCkRIFtEQNLdEEJQrWDIB9ttqRlHzpZJWQYJ7rYdLbg72yDtZJLQJmYt9AXRDxMMY9GLhnf2cqWrDlJSOKk6uU6OoX/pNG3z+na3tR+S0xMb+6Jd13ctQKsPXzeN276wVioH/APs/C2PjTM0LXdCtDDXWkLfBWgxLThdO3UBMFCsp8jZux7PbZ6V1P6TRusACVI08+VmbBB1EG1fDn3ZlVuqbZ110AZkEeXIGwiVEka6+fz424GU94Bt2BIIynxJtrtWeShmHOrdCidBJPs1+y1cuLNycWrg6RT/EfBPDSoxLRLuZFBE5Wl5kZViQZkfbaxYgnKs9k6Hy2NleiWpyjYWv1lNIzeca/GwamkFawMzFpBDgRbce262uznaF3ZqplnELZWyRujc1+axa61/mlp5dVT5i8ON1LIpOjVXsA8m72qUj4Lsp3tjPpF3u1StO9HesY9Da6ltTNc4k5NIB112tbUEwRPnbcwYG9hDDJv6S/wAm/qrZ/dlhR/man+1P/iqpOH8b9JLDtWutpej9XPqWgtw/VqUkAmTpp4e6ycu++kWK1VRScAq+mp1LLnoiHwppJJkhOYFSQTyBgbCBa5B35W2I2B22k2l8lz2t3l/k39VR/dnhN7/ItP8Aan/xVUrEOOukliClpGFdHh+keoRDD7D6swTABSQZCkmBII5AiDrZFwphXjjirjZgfFeL+F1ddFHctUQ++IKEoMkqOumv12ugIAmYFu0qO+lk7B3ykcWdzgCDb0eRuNgrFP2/paESOoMKgie9j2ZgZSQHtLXWzSEbE7hFKxOWpp3e8qQT5ifst0tIO4APdFt3hAaQ4f0HEq+MH7bbWIA99tgjUrze6BIg66Wf2H3RU3DTmZLctn2EgfCLMJXv+2zwwW6V3fUsGfo3Mw9o/C1WqbmYQptKUSjU6Wyw5RJJtluU7i1Wsy+aKnqV0TLbTbriTJBKVQo6kgT7drYupf65SHUOJWhSgoFKez3jTxJ08udiV1oqHmadPWSpRjskCDsB5+Hws4bu4W8SL9vJ1u66DrHkIU6c1U2NiBJJOpkjzm3StgLnWAQjLlFyUlIrlk5YdOYdmRqD3fI1jXwFavAIj6FxPaEhSASTpHjoT872cg4BcY0DtXMhImTNazv/AJXhubGmOAPGBSlJN105OUkTWMnXcHfwPzFi9zefVQ+8s+kmmm8CpKSWTGoJLWpOhAMeUR4c9rdemJXCEU68+x+ikwRHdB9/us5TwF4xpQS3crJCz+hVMTIn+cJ0twjgpxjaRH5kEAFOlQxzM/rfM2iaOTonFSzqkZm86ttBpaZ2qQgystttEBR/SUSnxy8h52ERfl9BWVNZeOoKRlU5A125xy8oAtJeBuB+NaihqXMQYb6yoQ+OriobEjKeyChWusanxs53uBt9IUlVJhVTYAgpTUAkmU66r001ncaWrPhc11sqtMeHC4coPav2/nn0lFXeBKk5QRnkEfiB3e82UGqvGa0A06bz1AMpaWCrU7gA+PhOsWl+n4JYiadSpWG6hoBQ1FQknUmR63gB77H08O8XUS1ppcOugJMpjKrltv8APxtWkEg+bH7lYiyH5z1CzbfEp1BS1Q32pAgmWFqMxodE+zu333tzUL4k0DYXU3XfaMxlS1Ui5V/o6aj3TadkXHjalStDmF1uQZksgHUnbLzED32JO3bjcuhqpuOtSHBn0p1JSBA2g6cgPKwGunvrHorfDp7aSG6j3gfWX5Ucc8ApvBFUhxGI7tTL7RbIHpDZ5gbyfnf2fbcOUGYmDpbyMwjd1fQ8euHz94ULtKtWILuICmygkCrSOZ2/Dxt64NkFCRlhUTvZVLM+U2soU7smYA3XSlAkSSSB7Zto1raBChsPn67BOBzISlJgc+72+6xJfWKJKVGO+NrRZCNk0k1tkaLjLi85I0iT8+duw4zlyjXv1+qyehDzqilCSpUSQBbkukKCVyDznzsXu4OiEKghKoWxuoc97bLjQkglPzzslpeBGhB+y2F4SRm8j9tkKYJGqJSoKljfUeEbC2Crp+SvDxskF0QSDyjS3JfOaM0idDP2WmKVpUe9EJbVVMkwZJGm9sNSzIIJP7bIofkkBX9KbYH8vaJkbxv8my7qAl3opb9KZmIJ2i2xVMmZ0nfWyJ18pgmefLTTvt0KgACSQPPlZxTCyXenJX9Jp5g8vrtpVVTzqPfZIL8GMxHnbXpIUrUieXl8iy7q1LvTkrmqYzSNJtz6QxzSoRpZK67TQK0H220qo2AmDG1n7sEu9OSsaljcoG3f99uTUU8xlOs85skqf5qVvrvpbSnzrBmee3hZxTBQNSeSVS9TD9AeEmxauqKctkZNwQINiXpCtAVafAfOli9dUBSEpG/n4WT6cWUm1JujDdXSdY19HH0Ue63DdVTdSVhES4BvppZI9KyIZdGg6pRtpt4egEg7Lk6jXWbVzStRRVOSmaxkiMusk6nSwq6umNKJT+jI1juG1kUvZw8sEGJSPt+q2+tJpQgQQpMRO+1m7q1S704pVfvCkSyE9WE5h3mdLapa2mLoV1apUk6mZFkJ99SlQD2QICpGnvsMl1LVYiDsjLrtt+As7aYXTGqcEs3jWMBoQ3oCBYRNWwpCYa0jv122sh1zssb6gyQZ3nl8+NhDUhDYUpUBKeUHX5+vxsTu7Qh95cjVVWU61JZLM5vEz5fH5mw6XqZOUdUMqSIT9sfO1m87UjMlaSQCqfWkkA68/Hw9tlEVAJ01gwRJ17/sNnFO1Mal1kPezlLUIS2thELBbUJOx8fZ9dqe0NyXxdfFG6sA3Vct5Xa2a5SnnqilLCl0qFEJWnOkFQUZAUmRB0Opm2lW/KG1JQk5TolQmfDfT8bIV7YXoLvq2MYUSaVi6rsbS2mnZbl2odeVL7jiiZzJASoQB6pJzaECmBiaSAr1Dad3pFOm9qK90XOKG5k1OR3sPLaqzTLyQBHXa9UNCSRKjEAd0O8NcG3yviFVOGmuyjpy8fSHKO+Ki83FsyQAp5+YVGsACJjWJM1VWJqTD91P1rjqXkUzWbqmiAX1aZRO3tJ52jeq4nu01NUVdHjnCV33oXJTdrrKXmEtj9EJQptRUYjrCSJmBAi2c17XCxXRxwuvmakHHd24roOIVZdodvJ5umVlbVSXqWSlhfaS6pk9h0ZuzBBIg6ibStRqTTXdT0aQklhltvYEAhMbcu+0TMcSr0xPeP52xHRXA4+hSWQihV1mQQIUFEmCJnyI9kstMLcp01B9bKmRsdt4O34+6WG5JJH25Klj7XQsjzbn/ski8FlT6shCe8AnWf2WFgrSG3CQFdk687Gl3VUVIeqaVJU2ynt9oAzBMAc9EkwO6wZQAwEnWEgiRGg/H52tpcLVc+2S4XmzeWBODl333eF3VT1wiopqt2ndbVVpSpKkLUkgjrBBERHhbtrBfBIkBJuJaT2hlvAgH3OaafO1mlxw4F3q9xhxm/SVDiW6i+6x9pv0datHHCsQQNhnGv7bMVzgTiALAaqAQo6fRLmPYO4i1V0EJOs5BV5s0oH8ACpxTw24KuNoPV3WG3PUCbyUQobA+v4ixyn4T8GagqSlNGMoJUTebn2Ge7TztX8cC8RpE+lpnYww4QFTtJSI+3X2gO8FsUNSOubJIMDq3NSOUxv887MaWN21Qf2+tEFU9uppwrOK4c8ICnqkNNqXlCQG7wcUmYEjzAFqc3iLxrb2qaAqW8tpbjaEN6gZZmAPBO/tsv1PCLFTCesDjWTkTnEnu9WzMr6Oqum8XbvqgQ9TuFCxJ3B99r2G0jIC7LJnJ9yz8RqXTBt4sn6U+rhwvcNdhSsN43Je6L+9IKad7MBThogesk9qRC/C3KsA3fCkFVYlQO2ZJ7u5Pd9ndZv4TwriDFfpf5mfSPQkpW51jpToqQI91lYcMsZAJitYBEwA+rTmRtazI8McQZLKm0F7QQy6k7AvCTg3iitpLiedxSmveZJW4VsoZK0JleWEq03ibSO50P8AhS071buIL7ZOmgrGCU+ctfOlq50XD3iOw+HrvvMtOgZQtutUg+QOnf8AGymzhPjO2EhnFFa3lTlAF7OJygQY302+Fs2aOVzrx1VgtWnmia20lNcqfx0NeFjqYRiq/wBIiRL1MNf6uft0swOPHRkwhwy4fuYrw3iO8q2pYqmWXGKhbSkhC5E9hIMg5ffysyxh3j8ggJxjevYOYRfbnZPf62lg7ywjx2vq7VXPemKK6vu91QcXTv3wpxormZKVKgnntaMLaiKRrn1QLQdRpqizyU0sTmMpSHEaFRJkBaBzCQqLXYwh0DcD4rwtc2J0cTrxT+drvp67qk0TXZ6xtKymc+sTFqzI4GcRilSfQGACrUGpRqRtz8fjZ9XXdPSgua7qe67qxherFHSIDVPTt3ociEJEBKRMBIAEDb42Piczp2AUlQ1hG+2qqYZEyBzjVwF4OymWs/J23IjSj4i3ktW0fm1CpVy1Dlkx38nXWgKcRj2rQgSMy7lJ17v5QC0c1N+dL/DlBVXt+7a/m2KanLjzqLxBUlpIJUredBMka2Z7vSs6Q77nWv8AFS+XVQAC4pC4julOm3Le2dDT4zKLxVTXD2f9loTVGERm0tM5v7e1TQ5+Txv4QGceFwKBIm51jQb/AOE5c7Xr/J3cJb74I4HxXgW9r2Fe2/erV6sKFMtnKXGg2sQomf5FO1vPLBPGDpvY1uz894PxLelfRB5TXXJoqQpC0gEp7TeggjTbXztc/wDJ84w6T15cRb/unjw5Wv3U9cxcoHH6embCalD6JEtJCiSlat5HZtXlOIh3DqahjgNwPnfcpgUDmZ6aB7T1O33q+gmTvNmhelP6PXOtxoVZhryNngdOfushYip9WqgDU9lX2Wv0Lsr7dVQqW5m3SKEhXnERHKwiYGvnEaWyNhE8/D51tuQeZEbzbcassrYBB1JHKyrRuFTMcwpUe0yPgbJWomAe42P0Kx1SkA8wdPd9ljx7qs/ZHRG4B2tsEgQZ99uAYGhIi20qJ0HPutZQl2IG5m2A/C3I758NLbmdY177SUbIQHeRAt2k6xIsCJToRbsHQH3WmmXNcguUrwHrZDGnMai3DCw9TtOkmVJBsOoiIgkd1ilHCabqjP0alI37j91oO3T8l0sQIs4sDv5Kyop+Tjebzg/ibN5zXv352U8KOlq+WSTAVKT4yLClF2p2p5lCpPnbLGyyCZtlsbuZReIF8y1yMDrqJsrT/KoKUxzKh7vne1neFzfpF/ViEKQgejKX2gRCesTz17h8PK1dBdNXct+t3dedI9R11FUIZfZda6txtYWAQoEZknXn3aWsrwoYR+ca59TqJSwlGTKRoVSeWu3n9m3R6vVaq0jKkZdMppAl5CkKITlSqR4ae2PnUJIKHQsEpBVm2nQHv9tjuRoyFPpEiST3TzPcLbDLYzL69AI0jN37z8++2yCbLJBRJaVZijqye7tQI+fki2j2oUUiCRJB1Ps+eXjY6+wwcrvXoOg1BMqI9m4gHWwK2k+qHEgQPVMA+Xx3/GzJHVK2Gik07kAkodSZiADHu9/jZaUpKlaczKRz2GkWS8OsoWw8gKCgHECNYG/dZXXTLSUkJOmo5SNJ8BztTl+ctCD5iBOhzgGQSR/pd1g1BaFBxBQVEkk792/12MKRmbS0EbTEaa6nbz+qwawlKhmQRGojXbb/AFY99gow1RVWcp6yZ2J56QdI9oPzrw6M8IASEpgZR5jn7PqsYDKlN9tK9VBMqnbSbF6upbo6Z2qqlkNNoK1GNIAn7dvrsxsBqpgE6KH8ZrydIThO3Ccyb6o1KKdwPSmvhr7Y8LenLS0qSJ9s/Xby0vRmqv3pB8NlrdFO9XX5Qp61zUM/xlsIEc430G8nc29L2KO+0o7V7g5tQep/G2FWuzPuFp0oLGkFONVe+ulVQJy9S5qrTWdOfusWU1CVJMGBpZKFDfK+z+d45/yc6WGFBewSqL0UDEQECwmEnkneAOaNMVzt1ZnWMvWKGUZkzAsjl9alKWtUqUdfPf7LFqu7b462XL3cyFRgBAsSeu+8CIN8PgjnA87W27WsqjjqlX0kb5zvEDTlbPScycyVE6TG9oM46Y9xDw0cuGoo8Q1KGq9x9t1HVoOfKEkHVM/pHY2jBfSSxcFjLeylpkFRIT7IATz19gsi4NNrKTWki91b1VUMxOYwD8m2lVAnQkjlO/xtUdrpHYhJIVfDwI09VMnffs6edhB0icRaZb6qYVEQlE/AaTZs/gntfYq2npQg790SNPdbPSkkSVmYjQxpaph6QmIOyDftcd0wAmZ0MnTztz+/5f5Ol+VyRMESkH6tbNxQOSYMPVW2FWEyc20692lt+lyYBTI0iLVK/f7xCdVX1XA8wVjTly8j7rCU/G3EdQSVX9XQTJUXfV929mNQ0clIROPNWx9JHfrpqLcqqiBJIjw0M2qbVcdL5KyG78vGBIkPkGRz8P22DRxrvxw5vz3eJAIHafVrr5/Olm7w3olwj1Vt/Shl0kgzMW4VVGRAOnP2WrVdXFWuqikvXreRnf8AjTiQJGmk2cCMcuVCErTe9fBEE+lrnx5/PttXfXtbyRmUbnbFTv6SVaJnuFtKqSBAJHLaRaF6vFFS1StOG9a0Z0wSKxcyT52KoxE86065+eL17I0Hpq+/bewvlRlr2RhhrzzU3mpOYJy6gnUGwNa+eqChzOsj58LQS7flZUgIaxJeDSshDZVVuqST3kBQ0mPgLA3Xft4F7K9f1dUKTqT6Q6E6A65Soka+P1WG7GY/oojcKf8ASUwrqlLYaEGZUkztE87CGpUKR4AqgOaGPnvtE7N/1ZQVGvdOZaUAqql6c5/HwsWvrFL9K6tCLwfhSiRlqFwQO4H7fCwBjMRNsqMcKkAvdTA1UuqDpAUQVGIiR899tsvveipAaXMGYGk+HwtBX7tnkrBNbVqEEmahRGu3Pz91jDOOHVtyhbyhHrdarU+U2kcXjHqqIw1/VTM0t5x8JWk5UgFRj4e6bCO1Lia1tSUKKSIkjxPO0D3ljhxtBQuodUTuUvH7DEbe6yCvGzwUesqnSDqFF9ZKe/n86WePFmO9VDfQObzVl66rdLEgKI0PZ1+fn2l3K1x0NUqEmcokiQR8OUm1bnMc1IkGodAgCDUbTz38/DXzsUVjp4pKy+pUaH6YzHjrpzHu7rWRXNPJDNGeqs5V1Sm+pTqCDEADUzvv5/Gxw1TmwSqZP6MknlAjXnytU9zHDwWnxUCQFyeciJ15fX5FDj8HMkqaHa1BdlOYRocvmNPDutNtWDsFA0htqVbG8apXVpR1aiQScsTp8/M2jDpE4pxzhThRWYswEmuTeVGplp1LavoFUrvZcL7R7C0JkKlQ0McjaFzj5TzaPRVUpQlyFOBQVm203idoBEknSbJOC+IC724xYbuW/KMrumoqKmmlxaglypVSvdWvllKHB2TmkHIrcDLeo2d6lawjQlCfeAXa6ykLhp0m7qpqumwJxDvLrkhxNHdt91tKmkReCkJHZfaStaWnpAiFqSsQZSoFFrC0103FV3c/eVxpuVmsqCpbtSwsUzq1QdStEFS45yeWsW8++M/CKsww1+fENVd/4WvJ/WnqShNVdlQVqC0qqlDKAZ0U4DmMSoEgqZFRxTvu6cDVl48PsdLw+m6qwURuW876TU1am9E52mXWVKKNpCXoTlMJjW06vBops01I4Dq07+K0KbHZKMthq2k9CFcrjJjDBXAm5q/H9ZSXZW4irVej0aXnVP1FW+IKW1LUoq6sEEkAxB01i1isEX3eGJcE3Pft83Qm56+9LvYqqmgU5mNK4tAUWyo66TzE7SLeRGCMXVd58QKLiBxbfvLHF14br6dyoQqvUELaLkJyB1MBsOZSpOVIUBEiSbek2Bulj0fsWJapH71dw066D1YvqmSwyo76PIKmkp7itSZtQp6DubddSdyo4jihxFwIFmjZSvW1dRTlLVDUPIC0lC+r/TBOsxp8+2xQOuCnCYkgEb7fOvw7psYew9RXnSpfutmmqGHm+uQtspLakR6wVOUpiNZjUWR2sP0ranG1UaEqQooUlSACFazv7fcbFcCFRabKufFdDNHxFvnq0oWHFNrQSRlQpTaCTHmfnSzYDbqPplsJyqXKQXIJHMjSY8do0tIfGm5UUmOUpp0Nspeu9pz9FIGXMnn4JHv7zZkM5UTkZLp9VKl+qkanbyFuLrSWTuHiu6oW8SnY7wXDAZQxmyqS4Uxl10PeTv36fVFj7TLzwFRVNKS2ghKFKc7JMfogCTM8tiLBs0LjpAVmhWgzHsjz9/xsqJul90pZbUuoc2htPZA8CdTrHKN7ZzpLc1psj8ETFVQIU276G6pDQOq3AEA6QrKBHd4RajfSLpi3xfvx8ELFSWX8/JRUyiSPDNNvRC67hr7uisRT0rbjcrDlQ2h0JI5AKBSSd9p8rUt6bVHUjinQXvUKzLr7paK3A2lAUtDriSYAAGgTbb7N1F6wsvuFido6Y9zD+hC56F1P+dsc4kwqHerbvK5XZJUIGVxABM7wFnbX6xaocOqpm8qO86auplLp3kvlCWhkzhlDakjTWQF7cyNdDaonRAd6jpAXZd5BULzpqtkQefUl0Had0D4G3pPdeFXKhxI6gqSn1hECNCZO3vtU7VzmmrAb7tB/Qp9m6WOqo/TGxKjzCeEG6L0h297tozWVVYqo6xpsL6tBQlOUZgIMiZjnoRpZ6XXgLD1XTuekXVTolJC6h5hBSNZATHrHc8++N5e1Ph5xyWGm0JaVHYa9VRnmd1e3uO1l2mw5R0yEmoUpahoGkbgePK3Ey4iTqCurZTxQgNso8puGOF6swi56VxAVDjy6NsAaRMkz9vhZSpuF+EKZ4JXh+6qhZQcoXdjZGYmYAjUanlz2tJDN3OLypZbUkJHZKoKh5ch7rLFDh11uSBlWd9ZUrzP32omumk2JUJJoIhdwCjug4V4JKlPXhhC7HFqMhKqJoE9xIA0suXfwrwW5AawbcgUv1lruxiR5dnQfG0pXLgSsq1BSmIb3znsix69L1wBgQAXveDdVWbJpWO0oq5CJtdgw6tqG8WV2RnUm3lzP1LBqcZhzcOFuZ3QBRbV9HfCOKbivO404OutwXhSO0xKaJtIGdChOaNCJ3FvDniNw6vjhtjS88I3+0hqru6qdonkJWFBLqFFKhMRuD7rfQd+eOJ+OKfqcK3M3hu7FkpFRVApcKf1koGv+r528Lultcl64R6QOLsO37W+mVt33zUIcfWQS6C5KXDJ3UkhWpO+tvQ+yURpXOjjLnNI3IsDbpfVcvi0r6huaYtzDkDcj28ldj8nHjDFXEzgjevCXDXDK4HG7kvNHpd7FlKFLQ+M4U6o5ipwFCgDHqhIjSTdrhv0d3+H9S5iy+MSPVV4hCwKZhAapWwrcBO6vMnltajP5FfGrd33vxRwK6pMvsUF6s+Ta3Wl//Ebt6c3nfPWtqZnRQssShoqWtfJKSXaWHIIUFfVOhEEWjefikgbbQbE71YL9C4ABKIWPZY5OvO3JhQynYg2vwnKQUZ4uCEzx3yDPjbBMyNAbCVLRaecaUT2VkT7bBzm5n7BbomG4usdwsbLpOo5k+O/zrY5QK7RT3g/D/wD6sUk8t+XOw1IoB5IBHdtGkH8LGYbFVnahKUmYjY22FQJiPEWDzE6je28wBiLWggEXQmaZTvytuTGhmbBBRmY526Ss982knQgM+Nu0qHLQWBBJgg6n2W7So+/S0woIUL2MkGxKkWU1dYwr9FxKx5KSPuNjeblpYmZRe8j1XmO7mlX+1aLxsU42Q7ugk+8WHul0tVzTo/QWFb2Ac0Gh91u7vP8AGAe82ZwuEhupUGom2WT6e8EejtZiJyJn3Wy1NSsV82tZXVtclF63nWu1NUpCXXnqhwrWrKkASokkwkaT3DlpaxPBGpTWUlReDY0ep2FQRBEgmI9sWrtUNAXKlSCjN6MSoExIy89+Z8pNpo6NVaqpwg67mJcbUmmJ5ggkju5KFrdGLSIFUbsU2F9yBC8pgTM6Hu+e+3KnwAYSICpmcon5+d7FQtWbKhXaOg1g6/P124UoqBVySQnLp2vPXvjutsLLypQaeztKSBJSoKAy6RGvjvHlYEOGSU6FWgjw5j8LFELU04rsAhXZOp10HxA92lsUcrhJKlZtTzBBExr5D77R2U06cKVmWoepSgEuALTJ7IynX3ZifZZxreQSEESpCYCp3gpOvvi0c3ZWJoK1h4AkNqkjLuOenIxt42fiRCQlC8ySBCv1h2YP1WpzCxV2nN22Rhl5CVgFOUCCJAPKPtG/fYBwtqBypJ3V4HQ92nM27lKm0kDbcbQSQNBttbSiSoypULO3uG3t22sG4VkBco1XlUdCSTr4n42RsXBZuJxAQNHkLggadrT3R9XOywpOXKSuUmCNBtp9s/fZKxLkVc72YwklB7ie0PhqbQkPoFFj+cFDCqjqeP8AwuqTs1f1DIBP/jbWvjz+Yj1JabMTBMAmR328rLzE8cOGsLIi/KEzlj/602fv9/jp6tsoUUgEi2NKL2V1p1KLKJaAbRTZlkznn1dvxt0ArKog6nYeVjKmhzG/hbTiewqO7n3WeMKEhROmdom3YvRhTrQGiUmDNkNaE51KSiAVHKCNQPHxsqOIOZUzv3fPjYspuNARHha2BZVHOuVV/pnXPV3tQ4SZp3VtxWVObIASR1aRratiMA3yopDlZWaRoQkjTzHf86m18uI+EWMW3th6iqZAZcqHEkf0U6e0A24oOBt3uPIeLBKSc0hIB8BAFnFOZDdNx+GLKjbfD+81yPSq0E69ko0/0Z8fbY8jAlclIKX606gJkJn2dn50texfAe6OzkZVKYAlM/s3sZa4FXagz6L7x9lp9zKj3oKiCcB3hAVnrCAT62XXfbsj7RvYYYDvICQis0IglQ1Hf6umn1Wvu1wPudKs6aaD3jSww4JXINRRmAdNLQNC5TFWOaobQYArn3UhxNeBoPXiNttPD50ssXnw1rbvdNNRU1atsIGqnfW2Pdpva7zXBe5UkK9EM851stP8ILlcPXOUiTKBv8bV30D8ysR1bLLzrdwLeyOyumqzOh+kiRvrp5WOUPD2uW8lJpareBDhiPn7LXyqeEdzFSj6ADuNhbGeEV0NmRRgeEbG0/k1xG6H31oOyq1hLg9QVYS7WM3hHMelKSZ7tvjvaRKXgnhVZS2bsrdQMx/OT5neP04tYO5+HVFTHsU4TpptZa/cXS5irqRrEEJi2VPhMpdo5Xo8SjHJV+e4RYPp2mEC7HnZBUCq8agkeR6zTvsHU8LMKM0Tq2rmrErRCQn0+oyGdSSCvXf5i073rgmocCQyyYylOZAHv+e+3mfjjDP5RtWLb7aw21jFd0prnhSqQaUJKMygnKCkKIiIO/PnamcHlOhdb2q0MSj3AurOK4ZXWKhTrVK+0hLca1rxgxvqo2Shw4oFOrUhtRUAdU1TsgAa/pfV52rGzgD8oJXOt/nGkx84WD1oh+mQBpGkbEyoactOdlOowT0z3r8uupYuDH10MNPI9I9IvoOodGcSVZj6u8gRvYLsFfb+ECtMxVo9RWEp8C3exllky0vsp693s+PrWKYkwUw29lDBWk5iSXlydRqRMT88rS/TXSpdKrr6dE+qPPwsFiG52kBalsIQEaAka7an4WzRhsgfcFX++sLNlAzWC0uuhKqELBBkdaswfOZst03D261MAm6wrLEErXy07/C0lM3G2XlFCUFaWpMDb32W6e52/RUFLQgAJJjn42hPQynYqMdXH0VfL6wDTIOVq70pUNZVr8D9dmt+4pgupaXdzJzqgAtAwDP7fkWshfVyoVVjs6BJkH4mzZaw8isvgLQM6Ge1qoHcHl862NTUEttSgyVMbnbKMk8MbqLKC9dFKpSid2hr3ctotn7191ggG6qYKTHaDYkyfHn8ybTe1cgdrkJUgJSlJy6aTyI87N3irXnA+B77xGyUddRUi+rOSD1yhkbPsUR7vO15lJITa6Z08YbchV6xJWcN8LVC6KoqbtNY2ohbDbaFKCphXa29aAdZ5RaO784jYYbzO3ZdFIcpCSpNKlZSMxmAfE6SOXIG0H4mxA8isdqHHnFKSSntKJKgrnr4qmPDaDYOnqRW0pccKpA0bIKshH6swTtMctDzgdLT4VGwAuJJXPT4i95s0WT3vjiPeb1S0G10rjTaloW2uUKAncKghIGnZMb+2xa4MT1lNja4L6CkMro71pS2oHslKXRmBnWSFDX+cnlNmiw/V0jpDThAI0UZJBkAgSNyY2nkecEdtpVMhT2dSHCgrStIyKROqCCBMSEEHQg8tBGxTMbBI1zRss6SV0gsSvQXGV4YGwhfd4UeKcU3BS0t5MN1FXd1dVM9cTBAWlgErIUIBASQYEHS1cr86PXCvETNdjnC9KFXYFOdW3d9dno3FIMKyBskgk6lCVJAB0A0l6YV4Y4dvzD1PUsUlQ6qtmoqXqhDaXXFqAzKcJUVOGZJKxJ17iLC02AcRYAVUN4Dfao1X19HWNICGGeyf5cNQWesRoElKEiFwSqBbpMS7LT1UxrovRDrEgI9LibBEIJRmttdQjcFxYWwPQ3zhvEuHb5drMSU5aeo7vokuIoWlnKhxThVqpIheRKSVEiCJkwq5XXrdYfutmtLoYdcaKi0pC4SYKhmSFiTJhUd0bi127t4buXc44q8s61BRU6XHlOLUpW61LUSSTlnUnbWNZqTxkwy5hjH980iEuhmqcNa113aJCxmMnzSDynxtUxXCnU1M2QjQGyCJg51glXhf0guKHCat67BmJ6mjpySl6i/lKJ4ncOU7g6tUiYJTm8ZNrncCenhhziJiSgwnj+7kXdfl8VQYp6ykcz0j76zoFhfaaKlaAyqSRsLeatSvIwiFSRBUDtyEA78o9lnFwh66o4r4PaTmWVX7QAJG5PpCBvbl5IwRcozHkFeqnHG6qV7EN1V9StMKo3Gy3qCtSVmBMaeuqzAYuXOlvrXWWGPVTlkz7CJVEa90+y0zcZLpeqUXVVt04WEOut5lrhAScpzHv8AV8NbR1Q3Ot1YbQjrVTolEkDXQcxbzbGXiOqcF6bgjeJRsP7bopQ3OyAE0LaVvZc0rSVK5DXSBvznzsuXfdDtGPRnAw4onMHFLUUoSBuRG+2nLu71ehuOpbahypKVHsFtqcsdyjME++zmufDi6hQWtpa0aEhLkfHbf52tzU9UG810LIRa6bzOHWnVhKn0vKMI613MAAdZSkCE/ExtaoX5RLB5uqowNiBqVtVbNdRlzIBJbU0rQHl2yfPut6CUmHC+pCU04QE7ZBy8TvyFqx/lLMGpVwkw5iFS1uLu+/OoUtI0bS6wsxMd7Sdu7na32bxAjFIxfQ3HuWV2hYx9A9nPdUh6MdX+bukhw/qHV9mqvRhiZj+VBaG39K3stRXEktIbaBCAcxKlGATuI2m3inw4dXcvFLBeISlQF3XvRVKiAYKWn0rM92k299Lrwo7VFMoXl5SJ0tr9uYJKmoh4QuSCPf8A91zOAV4oqd7XG2t/cmhS3AhUFC1ZNB2NCZ5ajQe2y1d+Fi5DbbaQnmZkm0gU+EqWkZD94PJZbQNSpQEC2k4goGVei4Vupy8nxp1oENpP9I/ZbnoOzrmWdWOyDpu4+wDVEnx6SW4gF/HkPrSVdmA1x1lQhLTY3Wsx7YFu3b6wrcLnoV0Url9142ap05gD4kaCyqrCt+YhIcxReq0MzPolMciD/SO5svXddFz3Ex1F30bbSRzA1Nuno8DEQDomCMfSfq76m7D61iT4hn/hnl56DQfWdz9SZb1wcQ8Z9m9r0RcF2q/+r0ozPKT4q2Hxsr4a4bYMwco1NBdqHKxfr1T56x1Xf2jt5CBZeqbybTpnE+FkasvckmFWPLLQUJz/AMJJ1dr5DYIIlqahvDb6Leg08+Z+tLNReLbSSlOg5AW8I/ym9ym5+lziypSkhu8m6K8EAjfPTNgn/KSr3W9qqy9FEElelvJP8rZcym+MmGsSoQOrvHD6GVK5lbL7s/6K0WngmNOrMTEbjuDZSloTDTuemv8Akt8XJw10qvzaXAlGIbhrqICYBKQioEd/8ibevVXewnRe1vCDohYncwn0muHF8IUEhy+WKFZH6lRLBPuct7a1NbKZKz32yO3bn09bG9uzm/cVrdnoBUxG+4KkNhwOstvCIWkKFuyfZZOw9UCpualdJk5Ms+Rj7LKBE91uio5OLAx/UBBmbkeWnkkG/KbLUh7k4OfhZO2hM2cF8sdbRhQBlCp07rIBkHTT2W6KlddiyahtnrBpqEgT32EZVkWFKGxH16/VbgydAduU2xWoOXUkED2i1wKm9KQWCIAtsHltOhNgULStCXBGozAW7Bnu85taBVZdgyfxt0CY20Ng80CSfjbDG0RHwtMElJDZtZgSdLbChYHMJGsx4WESuDry562kN1EoUH4jvsVrXA1U0bigYLhaVp+sk7+0Cw+adATvYlfJKaBT8asLQ8P8VQJ+E2d2yQ3R5YA1i3VAfpozDf4WDWQoBSTyt1QmX4kanvs9rpk4euCdMw00tluOtjTP8+62WjlT3XgKMMXnVPN4YpUNLrXVGlCS52Csdk6ncTaUuBmAcSYFu29LuxA0y2l+pS4wG3g5JCTJ02/R03s0KC9GrvxdT38+28UIrVVDgQQVaqJgbD7NbScni1cPV51UFcEnUyEEAa9yvnazwBrTmKDNmcMoCd/VOHkqFeffO0W0hAzQsaSJ2BNmsnilcqT9DQ16hoTojQ+Pa8LCOcVbneWEro64qjeGzz12V4/std4zeqqmF/ROMIQE6iBB0A8NPLa2EAqKSJkgghQ1Ph9/hZqjiZcSgJu6vnLzCJI5D1vv2nlbj9865pCk0FZ2jBhCOfI6+XzraPGaeabgu6J2ZIJSBJA0ETy2+e+z3uO8F1d1MBTiitr6NUSNdPsj42hb98+5YJTd1cNgMoSUz3b2WLl4zXNd6HBUXZeC0rWFApCNwddyLCme0jRWYGFh1UyNr6rteGmnlr891g1FPZdKlnLzgef2DTxtGiuPOGUKUpFz3qQmT6qNNDr63f8APcC5x8w4olSrkvPQ6n6MSN983japcq5dqlArKwVdkZEwkQANJ08tvdZKxYoOXK6A0pIUW0iEzHbGp9pB9ndZhfv9YdT2fzJeKpWAB2IO387SPf8AXZDxB0hcM1NCukTcN6hSiFFX0ZSACNxm02HyDaEly0qcbmgi6Rr+fQ3xp4cLQAlKr9ohlVPamqaOoj2e8+NvWBhPPxm3jnRY9u/HHGbALlBR1dO3S3/Qyl/LmVNSnuJ0hMe+3saxlylVs6UEAXVmMhxJC7IHjYNwSkifEcosLOYanl32LrUY1GhEQTZ491B6T3EkqMa+P2WAcSCNt+VjDuWYGm+kbWAcI1J17+61oKod1GnFvifcHCJVx4mxGwHqR6peowQ6hspUpoqBlZA2R5xNm9d/Tw4OUSQHglyQIBvFjnruCZ77S1eXD/BfEPq7sxvhW6r+oqUmpap7ypEVDSHQICwlYICsqlCY2JsJSdGvgK122+DOCG5Vp/uDTb+1FrUTHkXabID3svYhRU7+UI4NNpJQil1MDNeiBpO+iDYNP5Rbg9EBN366Qb0Vvrpox4WnKn6P3BFj1OEGCkxpIuGl/ua2VKfgtwfZA6nhVhFPlcVKP7FiES31d7lC8fRV0c/KP8IGgQPzQI/XvV0eR0pjYH/9ZDwrWFFpFxFIEkm9n4jv/wCC+y1oWuFnDJv+S4dYYT3kXRTif9Cx1rh/gNnVvBVwpj9W7WB/ZsxEp9b3JwYxyVUFflGeGyUheTD6Qdz+dajxjX0WOVnHw/6a7PFy/RhHh5hW674vUU66osNXi8kBpG6itbCUjU7TJtZZnB+E2lfR4YulOkdmiaH1CyLiPCGHKXEmHL7bVSXOqlcqmlPsttNZgtkjKSpMRIGneBaPDl3LlISsGgamMnHHHepb62n4M3epJWUgm+m+RIO4HMW6bxhx+nXhDdbYTvN7tn+2JtId13hgvD93sXL+7GkeDIUUu1l6NreWpSiolSioSZV9VuX8X4Ap3Ch/GFxpjfPebI1/y7TsbfOUL/6qZNLi7jyCFL4Z3WhM8qtC9f64WVk4q41vgEYHupHeSpCgB3f8Isqu4/4XtwlziFhlA19a92JgeOfxHvsUe4ocGm1pUribhBMEyFXxTGf9PwsF0Qdu8+5EbIR6qC/dHxsKEAYTulJHItoPL/8AmrYq9+NS2ir8w3G2sCRnbEfCpNgXeMvBBIKXOKeDEkb/AO61OSP9LzsTXxy4AsKTn4r4PSdN70YIP+lYfd2/TPmFPin6ISQjF/GCpxPeWF1UeH2Km7qemq3C4AlDjb5dCch68kwWVzoI09gFXQ8Xa1ZL9XhcoIggPxHLvMWb7XHfo+UHGW+b8e4j4a9DrsN3cx6Q08HG1ONVNWcgKQRmAdmPws5HulP0aGZcPE+5OUhLTqj8EfM2CaVjh6Tz5o4qHt2YPJJowvxIASFV+GAQqQBWEQB/imwF5YK4lV5BRe2GGhMkmtUqSef8l5e+yi70u+jQ12f3y6GYPqUdQTp5N2Lq6YvRpTlCeITbhV/xd3VRHf8A8XYPyZT3uXHzRflCe1g0eSRUcPuKSHCTf2GgqMvZfJ07v5H7LKFFw84oBsoOJLhaB1MOFQA7tWLae6Z/RwbgfuzqF7+pdVSf7FibvTY6OiFjq8SXm4BAhN1Pa+RI8LI4bSu3J80hX1LdgPJavXhpxCTUN0q8d3Il6pQtbaEoClLyiVR9FrAPsm1frrvXpRvJNZd/A7FTrbg6xLyH7uyvJIkKTnGx0I0G9p1vDprdG6sCfTai86wNkltLlzlYSrYxmOmlpvwhU0l54Uuu9qAFFLWUjNQylSchS2tAUiQNtCBHKzDDIL+gT5pd/mPzh7lStD/S9UsLTwTxUwoKmVV1zjl4jltaNOkNfXSEoMAqY4m4FvO4bpr6tDSH6uuu50LeSC4luKcZx/JkzoOyBzt6XuNpVrmUqNd9BakH5T29RS4TwTh9p9Y9Jqq2sXl3HVobQDv/AMqq02UMcbgRdO6tke0grzUvyvNRUv0eqkg5SCCOUaRIO3nPMbnWHq1C2+qqEIJa0c7XaMdnXumR3xqYsWvOkeff9GL8voSnKCCSvX1knYzpEakR5WSroW6zfK0VCFoUELVlWoplW+uncefvFtIaKkdU80CHkrClAlUFJA6wpVz9uhE6CFHcaHHCw4wk0yUlKF5yQkElUCMsaRuTEaSdSbJDrzIpjmiFN5UpTGaJBkEnmMvKBqDzhUp0Idp+vbaSS9lWDEKlWpOoEx2UwCdosQHVDIV0eBOIU3nwtuitutqnqrwpqRDLqnAJSpAymQk+t2Qd+7WBZYq7wvSovuiXedOwFvB5nKhACSMiiBoO5Jjvga8rRv0XKxdZgN1ph5Lz1FeD9P1mYg5VK6xAkRMhwGfPvtLN/wBYzdlAyi8LwYXWOvoYpV5c+ZZ7IkaHKkSZ5jW3rVHMZqKN5PIKhlDXlFXGKamZbySt99RcKVqUkZc0jflqnltHeAKydLG52/T7qxMhpWR4PUq80pBJAWmSTGkHw020taahFa7StmsTSl0U4S680UwpYT/RTA3I0GnO0KdKO6BX4BcrUTnu91urQtAJIAVCh3+qs909+mlPF6cT0Eg8L+WqI11pBZUtvSkdZQFqAU3mgQnsjTvHs8vfaZug5gZ7G/STwlTBSQ1dTyr4fKlRLbCSqABvKyjS0V3nQqqaSoLSlulAzaDSANDPPTn4d2tpd6A9+09x9KDCoq3S2muRVUaOQK3GFwPaQPhbySW4aVqxfOF16scSrsbr7ops6EkN1SVdoFQEpI256xpt8bNa58IJcp0LQoNwQSVaDnoEjU7Cyzxj4i3Pw34d3vjy/bvra277mS0++1SISXilTqUApClAaFYJkjQHnauVD+Ul4C0yUqODsaJOaOzSUx5f8+NbeZ9ocLr6qp4lKwkEL0TBMUpaWk4czgHXVkaTCV9+mUzNOqlF3TLpWlWeeWVI0PPXys9KG4UKSlpsEJBlRjSfKYtU9v8AKh8BmVBDuEMbDkQKSmk+f0/dZSpfyqfRrZCUqwjjxShqR6FSwP8Ar7cw7s1jEp9KI2VuftBAR6LwrfUVwNlSQlRjSTl3tDfTv4dVuJejNfyrvpC89dlRSV6QgSUpQ6ErV3ABC1EnuBmwPDX8pF0V8UsVVRXVOILj9GRnQ3eN3pl8jdKepWuD/SygzvaOePn5UDhZfWBMR4Hwbh+/WVXrd1RRN1K6VlSTnSUnNLnZBBImCRPfbWwrs9NSyNlkvnB0AFz8AsCpxGSpOX1eZJXnThgXXc1/3VUX404ulp6wKqGmpBKSoKOVR5wDG5Ga30C4Qxd+67Dd031g663RQ3nRMVbNVVDL9G4gKSYnUwR4W+ci9sWU94u/xdlbSusC5IERrGg8z7zb1R4OflZujzgzhBgzCWJ7kxh+eLkuKiu2uNJdzK2VPMMpbWpCi8CUkpkSAddrd9U0csjmyEkdbC5+rosB8jC3K0X+v9rq/wD+5VurdFRf1a5XKGoQrRsHwSNLKqBQ0DQbpmUNpGgCRFqIPfli+jIUnJh/Hp/920//AOkWSKr8sN0cHZy4Zx9/m+l//SLUnMkpb90pyXdSLnzKFldJYSPFul9FfuqvdCEmFWRqu+iQe1tahj/5Xbo7uJ7OGMee2gpf/wBIsnP/AJWno+u6Jwxjn/8AwaX/APP25+thxqpJPCctCnipWfOeFeuqveQTnsk1V7knRR9lqNu/lXeASz/3L43350dN/wDn7FnPyqXARRJGFsbajnSU3/5+3PzYFjcv8i73LYhmoGbyBXaqa9SwZJjnbz6/Ks3Gbww/gfEDaATTO19IpQP6wZWkf6CzHn3Wcq/ypvAY7YXxqf8A8jptP+vtCHSt6a3B3jvw+pcNXBh7EzNfS16aptysp2EIydWtChKXVGZUnlyNr3Z/A8WosSiqJYSGjc+0KddV0MtI+NkgvZU6wffbmHMW3DiJlUO3VeNPVpO0Ft1Kx9VvedtxVU2hbIUoPJCkZeYO1vAAuNQcpWFZ5SIER59+1vQ+6vyleC6PB9y3HWXfidytprspqas9FpmaZgvIaSlfaDxdcTmCjJWie7lbqe2WC1GLCE07bkXv9dlndnMRiouI2UgXtuvS7BRWi5zTPEBxl1SSnN2hPag9x12Nl4/G1Vugl0icLcdrvxhTYdoqqjcuZ+kfeZfpkNfy4cAKSlaiv+R1KjPwi1BJ5c9rKgppKSnZDKLOAsi1UjJpXPjNwUG+hLzS2j+kCLNR0FK1IJ9XQ8rOxRMR3bWbl7NFmsUUjQwoR323aI6kLMqhpmRYrkyffFszbHQ+37LBlQ3y7Wye8zbUAWaTdHaZR6lIBPZlHsBiws841OkWJ06wc6QokBW3OCkH65sYCjHrWst1CrnQoUk7azHdbAoHX67cBXKR7LbChOxsQaJkIFTAOvhzt0gkQEnb67AkkwAY0t0lQGx132tIKCGCu/Qd+1uahIfp3GVAELQU+8W5ChqO8bW6KudpJLmkd62jZUZJKBPnGtjNCfpxvA5WT7vgMuU8D6J1aQPCZ+2x+iIDusmzBNzThAMDVY0/VNssBP8APj2j77ZZtU68EsWPVF1XNUVVI4EvsJSoKKQRJUEkQNBMnwmdjs86XgvxHfYQpOM7qSVICxFIdM3L1fq8dpNmnj5p13DVY2lElZbIH+Onbx+/xtc/htwuxVxGWaXClzv1zlMyHHUtJSEoTG5JIAnWNe+LPAxj7l6BUPfHbIq1o4I8SAoH92d1Zd9KTQGPLu5/fY4zwH4iOZmzjK7CoD/xI7+75m1iK7Dtfcl6OXVetOpmpp3C08h1OVSVDeR3+YtP3Bvo5N4/uZd9OV4p2kjInMnNmJGwjkNPtsd8ELBmOyrNqJXGy89FcC+JSkhhGMKBxzJqlFAomdNBrPvsFV8EuJFKerqcYUbS3ESgKu0g+YmNDEez3XP4ucMX+H19PXeqoktkDrEztvGmvd77RBXlbqz1iluLUnKMxlQHIHWfr3s7aeN2oS7xINCoEVwZx+oJCcfUMZcul3jXed/Pa3Q4M46dKWnOIdCkoUIm7Ekaa/rbb/Mi01k6yUgSI01nTl77AqEJAQkyoaiY8dbT7rGl3l99VGyejLxMe+jVxNu89ZuRdgkj36afZ427a6MfEBWZbvFOkg7kXTIkjeSrunX42n+4Kj026WnutJW1DSgdyQBHvFlZppCVAKAIjtQTqRp7d7UXsDTZasYDmgqtp6LOM1KlXFynBkkhNzDT/TGm3yLJt99FTE1LRqq3uKzbplKVJ/MoSYKuX0k7n42tGpOZKg4oDaQRvMjabJOLiTcb76ykZVtSEkGfpEgx46b2C8WaSCjta0m1lSXB+GqvCfHa47gqb19OVd+ILrh1DQaKwpxtwgpE94G528re2tPq2CQSPPfe3jXVuZOkzRtpUDOIbpUSnSNWRuO4kC3srTpyshOu/wBts+Yk2KnGAHEBdEDWFGdrF3Rv4bWNmP1Trz2sWeJIiedox7p5NUnvjvPnYosD5HKx51IVqoSZsUcAzGSdTztdaqTt0g4jx+3w6p6e9nrpdrvSnxShDbiUFJKFLzEnl2PjZF/hQtBWU4MqDlUdPTkjnH6lkrjupIw1digrJF6JE89WHrQuVhCjKQR3Ez88/kW0qcXYs+ocQ7RWCT0qshhOA3dP1ryAM+xu3Sulo4lHYwGD4KvOBtyhq1eC6ADCoyiD3z9+v1W3nlRBMjY6SI90d3usYsCr8VwVhB0tLxOowDTpnabyV9fVWCV0tr7UTkwPd4CTBJvFZj/q7QAVGSgwSDAAA1Ht9njbRcy5ijXQCZG/z9dmEYT8VxU/fws8SEfRYMu3MTt6Y4f7Otoz468UcQ8eLiurhxeN10N1UV4XiFu1FM6tbyeraWsABWmqkwfCzOQ6owoJAEiJAgCN7abeAv241hY7FbEEHm04Pt+Pss7owRYpCRwN1Hl59G+5LpfSy5f14O9Z6qglsActJT3AWCb6P2GCAk3veaRB0CmtdNvU3tNWNgA/TFZ9dJynvEyIs2nUukJTTKKdddYE6WC6niBsAitqJSL3Udo4AYNSD1l4XotJVHafQCSY/m6ft9p+m6PuAXEqKnr0UZJUfSgAAPIbfO9nUr0hGiiexBMTzj8LKd2N1K1F0OEspJkAk8u78LJ0EYGyds8hO6Z7PR64fJ0AvKCdR6XHOI215iLCno/cOwCVUlYRuQatR5eFn22+JHZAB0nbUTYwHhJSo5tYPf4fssPhR9EYSydVFY4M4EpcZuUnoNYukNxIqeoXVrKS6KlbZVuD6pGkxtYDEfC/AVDQGporkKShW3pLpzDu9blp487P+pj92NOAlQL2HqpGpOvV1jB07tHPiLEsQNpfuypSURlTOvhr7NvqsmxMLTomdK+41Vfai7MO0OMU0NXQpRRGlW7lU6sBJE880zoLI66Gmr6e86663P4vQZ1JVmOZahEASdEgCyvxBCXLzu+7GVILl41CGSJM9Wkyr6wIsnXShLFNjChIc+jUslO0pyHaNfD3+FgcME2Rw4gXuuL8TRMYQu69qenCXVuNhxRJMg7zr3e+yhi67aRi6aapuylSgh9txeU7ojtSQdgBr7e8Wb9fUB3hBSOZRnQloiVZYPWDWeWnLxFhscX40MJUtP1p6x9ppZShQzZUgTpy/HSzljQEg8lL1DQ0l6V9JeTlAlqgVUoQykEw6k8ynuJ157WkBjpE8bLqx7X4LpOJN8MXRd9OlqkpmFISllKSEABQSDAAiJ/Rsz3q1tq5aFbUadSoKSBlHYMa8+e2vwFiNFd9PfvGDFxV1iXaB9QbDbgkpUpUk6wRoD7POa5e2PVEaC82CfGMukDxwoa25BScXMSMs1FV1T6U1pGYZkgjaeZ21tGvHjiDinGtdQ0mJMT3nfC7vaWpv02rW/1WdYzBOYkAGB7h32f1fgy47xRTVNXUvRSrCmiHICY3EDT9X4axaGuMdHS3FiWsZacJZTRs1DgWoKUpaiQANROhiO7LvaMVQyR9gVN0L4xdwUa3qzczLArLyddDg7QCYlz1tBOu8690Ge9Cup81VcuoUHG2UNkISZISCrTfcA/V7bCpu2vvp8vPOBplSgEozyASfdsAT45dLcUDno9/uNLWlZDIScpByxpoTp4iNtO6LXLoICVK0hCVOrcbRmgjJy5nnruZJOoJn1tFC4V0voD7K20TSOELIKcpSoApEkwYIUJ0PiTAsk1D+dGZTgyk5TM6QABqTBmNh+JDuevapb3DQUnK8ktqVkA6tydNYkajlrCu/WzgpEXVk+jDfjFMu+LteeU0qprUPBS15hlUynQkntZssjXc6ACbTFiKLxvVlulCstG80tISgkE5xGQREAjceQgzaBejhX0NLel/MO0zb7rTDLrK225SFlSpjXszlSdBsIOo1sG1UUdw3KmvvBBNRUuIJQuBCyoCJmREyZ2CT3W9GwJxnomsJ0CqyjI7MlN96+VuBFMgttM55AVBXJSCqd9o3iMw3nSP+NdHW1WB6yhrKdCXaqldUSoKIkbHbmToIECNt7StRuXtQgUl4KSGW3FnrUH1ZkRAEj9OdTqRtJFoP46VWKMZKrLruapXRU9MlSStEhQEagRAEkmVawO62tXfi6Z25JFrIAsXKoFI6s04qFZlFsqCgsghMAaEnlJnnv3kQ+OjXcldXdIjA1DcyVB9m/6aoSpJPaYQrrFGdNkJVO0i0e0IVQu1FGsgOsrUhRA1BTIJ08zv3aeE2dDmovNrpOYKfu+h9L6x51l8gk5G+oXmX4FKZOvdEzbx6cZQVqx6kBeovEbh0zxVwLffD16tNEi/6Q0fpIaDvUkqBCwgkBUEAwT3WgnDf5IO6Hyl1/ipWutHUqVc7aUgTMSXD8JtbDD9WaO96V5tNPmQ4I68w0TMDN3DXflvFpaXha/8QK6zFmJUopP/ABG7pbbUP5yz2lfAW4/En1LZWiFxt0FgPaSdvqW1E9jW+lYeJuT9QVC8cfkzeBdLhleHbhxxf15YtzpS0aKmZcQBPaCkH1dJ16wanblZDwl+RhZvFArMW8Xay70uQfR2LsbW4ANpUVwNOQm3pZdtFh7DlMKW6KJinQn9RIk+JPO2PX4kA9sWqR1/ciXVE1/9Vu3mdfuUZJDMMsTPrO/kNB71QZ38jXwzZRNJxqxS05AGZVDTqGnKNLNnEf5HLBlJdFZWU3HC/XahincW02q6mQlakpJSDC+/T229Dam/DyV5WR6+9wttSFLkEEHW2XV9qchvEbIsNFNJYO2XziYjw+5h9/qVuF3OVpJy5YUP2z7bIJJO+tpR45XU9dmNr7utxgt/m69q2lk6TlcI/sz7R7IuNvRKOUzQtedyFl1LBHIWhYCR3621qTM2cGF8AY2xutxrBmFL2v51gZnmrto3KlxpMgBS0tglIJMAkQSDG1nOz0b+P9QCWeCeOlBIKlEYfqoAG5JyaDxsV08bDZzgD7Qhthe8XDSQo3jx+Fsj5iz5TwN4zKu83qOFGL/QgQn0j8yVPV5joBmyRY3dnR44730Cq6eDuMqpI1zN3JUER55LMamFupePMKXdpT6h8io7n5m2ptKn8FXpIf8AoOxt/mV8f2baV0WOkanfghjUf+5n/wC7Zu+035RvmE3dKg+ofIqLJtkm0pp6LPSMUP8AvKYxGk63S8PrTZBxpwX4pcOqSlrsd4Gve4aeuWtumcrqZTQdWlOYpTO5izsq6d7srXgn2hM6lnY3M5hA9hTOJR1aSASoyFEjQeVrXdFvoYYa6QnDeoxxX8Qa66KilvV67V0jNCh1PYaaWFZisHUO7RytU4CTA10tfn8mji1LeHcb4UdcSOoq6O8Gkk6nOhaFn/q2/fbOx6aemonS05s4W+9aWBQw1NY2KcXBurQ9CPow3T0b8a4hcujHFZfLWILubacYfpEtJStpzMlQKVGTC1iI52uIoADutAvDe+P9/d3tlfZdLjRPm2rTx1AtPSyCbc3hlXNWRmSc3dddBiVJFRyhkIs2y5O2nvsl39TqLaHk/onKfKymROk++wdYkPUi0EaxIttU7srgVkSjM0hNIEEa+7kLdDUAJOu1ult5V5SN/dvqLcEEcyfO20DdY67YUlNQrTRbYPuJ/vWNAjUkb72IpMVLS51UFI18RP8AZsaCiTI3GugtYj1QXbrtawEKVEwDEDXysRN+NhvrG6KrPaCQnJ2pPhPh8yLd1DdYtSDS1KUJ0zJKQSo+Z20nW3DDF6Ay9eYWAf0WgJHt9trAtzUFr8+jYXdWmdNG/Px9/jYdN6KJUEUb6wlIUkxAWDG089djFgU0t49UoG8j1pJKVBsEBOkCOZ038bYKW8SUk3opKYGYBAlRAG3d392vlaYyqGoShSVSqhnrVU62TJTlWNRB3+2xjNzzedkhFHeR1VfKiCIjqwBMRPfM62U0kaAnUDXxNmPgnC4YHV1dSkyOsSlwT7QfqFlCinrZI21M2TVKCK9gn/CJWju2g/fZToYDuoGu02ZMd0sEuT+n/lWy3eVf6x9x+62WSV14RY6KW7ndcATK3m8qQI7RXz7t+Rtdvg7xsxXwkW5W4ZVSlNWyhuoZqGs7bgGxMEEETyPPxtR3H6ouzKUyTUtpMpG2bby8wecbwbb4HpG66/Ltup99TdNWVDNHULBEhClgEwR47d4tGnsGuLtlWq3ZAHdLpwYixteuMMT12Jr8dbXXVz3WvZU5UDugDkAPhraWuE/SMvnAF3uXTTJp6incIMOiYVG41FoRx2y7deLL1uioqhULuqreoEKAM5GXCkeQ0+NkgVi2l6qPZ33M99r4DZYx0VFr/WCkTiTxHrseXk9ed5VKXHXlE5k6Ae60dVKivMorCtDm/SzCdffrvboVC3QU5gFGSYOoG0fX8bcrpncklIMQPPlqLSaA3QJHVEV/RlQWtJSdSZmdfjz87AuICQvtJkg5p8OZn67DvMqZChnAKToJIMxrvt+Niq+pIBj1lZYImAD3b7T32mpWS7g6tLV4OUTgzIqAVAyYzjUe+Y9tnyKcqC1tg9kKIgSBpI89bRS08tioS+24UrSsLHmCO/y+FpWuypRW0LNc36jraVEeqATuPfbNrW5TmC1KF2YFq11CiQXBCYgEGPKPdZHxIyo3NU9YApKVNqM7EZwefj7rOR1JWkx2QmTqdtdB893KyLiUpbuKtKkKOVvMAP6Qn3a+22e9/oFabGjMFTu8uq/hKUbqEkEYguxI0kaKZ2jQ6z36d1vZRn1RJ1m3jVeyFsdJOjW82hK135dROUgEg9SZ0/ytPqm3syw2VDeYOnham/YFO0We72rFAbkAeNi7gEnLp36WOKbSACRBPfpNi7ojYDtWeNRksiD6f0jYmsayTvtrZQeSCNeXeLFXRpB58vn22uMVJ26i3jwicK0SkbovJsg/9E6OXnaC1JURKnCJ5xNrA8a6U1GEGxI7Nc0rUfzVjn52gv0U5DkgwY07p+RbSp3DKs+oF3IkUOJ9Z0kgyQOeh0tkJCSVq2JUDHwsaFEVLyhXKQY2HMg+/T9thDdxzBYywqO7Q++xy8KuGEpOWnUHMe+QN7byqIzKUdeUyPOx70FxRyhRWADOknTw9ospDBWKXNUYYvQhQMKFG5tHl3bWbiAbpi3KEgAAKBDgkeOh1GkR36W3li9LoJUrS8GhrH6Upj42cLeC8TEwrD94ICRHaZUkbabx42Bq8O1l31N0mvoXKcu3hTZAvQkB1AVHfvZ5A9rcxGiaJzHuyg6pTxmw285TOZlEBC4gwOR1/ZZAfpKJhsmofLYXMyqCVDl8/faQscUVJToo8w1yr3O57POZ2+dbMOqoqOuUGnNUoMwDGsRYGbM66shmVtkj57s60hNQjsmCUrMak924+2yzdj11KYLblTL7iuy2CQCfmLJ6rjoErTKtN5Ku4WV7pui6FINVCs7ZzIlU90fPnaUhuEzAbof83NKSXCFA8xJFu03bSK7JSZIBIzZR46WMNuNIQAVAaTEc95ttboCic5CQDoTrascysAtCRWrtpnuIuFqR+QxVU140q8pIOUuUqiATzgE2sUejVw7rad6neqb5hYKFZ3kAHkdkTavS6lDGOcIVadOrq69uVA86dK//AJYteqnKF5VpU7KpC+yDPlP2WdhNyEQNadVX4dC3gNX17FfVXfelXWU7ksuKrnUhvvPZTvFq6q4G4Iubpq3hwldu1bmGL5o6ao6j0hedYVTuEyuQqesRyPd329C0llFYinbdcBUoGCQOe1qccX3ae5/yiOCKtSwg3nctEyAP0lda+iJHOO/7NE/QXUraqRVdDTo5U92i42MDldCjRTLl41K0HWYADg5iy/RdDvo21FOht/hfdbyVdlfXuPurgbBJK5A70zB07rSUimSlbgNP1ozGSpBUd++ylRrQxWoaaS2htKtxlA+uz5QpleUPE657quLF2JMPXO0UtXRedTRtMxoltp9SEJiZEJG57/e2Lur27j4v4rvI6KfoWqggz2ipls7/AOMDz1NpQ6RHUp4/4zugMhTgvR9wrQ2U5isheg8l7/faGseVX5q4kuvw0luuuagBObRcMtgA+1HPv8bYE782ZqtwtDCHJ8v4ncfa9HTTFMwVBIPjuDvqfmTav3HzERfxEqnp6v6ZbDIqAk6dkKyjwjN9nK00sOhKcqFEpJEmN+fLzPLnav2O624rrxBeFZWsLqLyfqHFlC0hSj2jChySDAjSYA03tDDYrSFysVsl2AJq0N1KU1+cL5rFN0raQ5lS5CtZ0EbfMAzYncRFbejzjI6s1CwlCdVEJKhp3ztr99iV733eV5K6uoV1bU5g0NE90+O33RY9hZhlfXVNUSlhsZVHKTvtsDOgOhtvDUrKSxXJQ6IbXmKpUoBUkSpUHcEiNdYmfEQBQ0zKHUqbWvrIlcBKgE7nmRzIjn3a26qqhVa6tumP0ajCUhBUCkaSBv7R3z5K1z3a51SHnyvrFpzNk5VH9Hv75BOs+Ug2kE17KVOjc/TNYzvOnWjqzUUDTrZLiVJUpLhBUTsdVaD1o750sTVJpL6vVmjfQv0SmSoKKATm0E7Hz0OhPMwIqdwrcTd2Lurp106DU0pKOsAggqQQUx+lAMQO89m1r8A5awB2rAbpmCc+aCVEAyBJ1SZIjXcb729B7KHPDk8VTqjZP/8AOVLeF3Uz1ehDeZttThImDEFQOWRy8YA7tI1xBfdK8XqOnpEvOOLWArKYHidNDHeJkWW2rtvy9iuppypihrkN9SyhspU0gAmQRCiSI0J0k+RR8dVNwcMcNuXleA6yrcT1NLTgpK1rygdkGI0A7RiANdtNypmkdckWaOagI25d9VSLiPdbeGOId50DbugeFQvtGEKWkLKfYSfhvFrUfk7uH+FcWcQ7xxrX+k+n4QpGnbvbbcKG1rf6xCnFxqrKlMBHqjOSZtTfF171974pvCvvRSPSH3SV5EjLI5A89onbTTS1sPyamMGbu4x3lhepqEpTfdxuIaQdMzzC0uADxyl0+w+VvI68hz3lu1ytKn0IuvStaTBQFFMiNeR+e/u8LSdduLxWXVR1KHT9MwhzXxSDaNCMgBEgyCD46aeMWJ4ZvpaLv9CW7rTVNTTgbdlDy0p+CRby/to6SGCOSM87LrcDpW1j3Mdy1UtG/c3+E9s2LO3zAgL5d9mexeSyJKoHfbtNY85JSCoW8xkrJSN10wwtjN04nL1UrdVij1apeYlVm7f2JLhwpRivxXflFdNOQSF1LwQVRySn1lHwANohr+lzgOsvoYX4fUL193k4hwtvVU01NmQhSo2KySEmAQmbEgpKus1Y026piyKLQbrzv6Yt0fmji7jekEjq78eqkjLt1xzjXfZwWrKsdqDv399rFdJfGN5Y+xteeMb6o6Wnfvxpmp6mnRlSMqQgCDKgYQeZk5vC1eH05XlonZR3Nve8DD20TGv3AC4HFLd4cQrJdBTjFe3CPiPfTl2XS1eC75uZdMELBOVSHULCozCdAoazvtb1N4eYrxxxKwVc9+Vrt30jVe24l1BaDimHWnChUJgIObRQlJIzRJ0t4zdHV15PFy5KZh7q11Rep80SO00vlz1j229XOjxX3hcl2v3LUXo7UUiFmoaQtIytrWAF5IEgHInSY08TbmO19I3Nx2j0tLroOzt5Yw0C9id1NzOELnbfTW3s4/elSmMrlY51gR/RT6qfYLKq6plhIbYQlCQIASIFkdy9JTmK9vhYhUXmn1s5PlztwJc9xXVMpC7V6Vn7wzSUka87Enq/1hm/R7rINRfCEkAk9rsnz+frsSVfaC4EAqnNB5HW0mxPKtsgY1LD1eqTqSR3WqN+UWo/ztweua8EpUtV3YhpyqNIQ408g7eJTaxdTfQjVBPgBPvtBXS8R+fuAuJ2QCXKJNPXIg6J6l9tZ8jlzW28FDoK2J/iPeqmLRtkopGjovKtsAPJSoT2gDNrNdAG/wBV2cW70ulThCL0uV1GUfpLQ62ofDPatd5t9TeFQ3y6xRERsTI202i0p9FW+zcfHvDdSFZU1Dr9MRGh6xlaUj3lNvU8Sj41HI3qCvL8MkMNZG7xXqdhq9fQMV3RWKcgN1zBUZGiS4Ad/A/Za2qpA2I9tqHOX640tK0HtphSVJ301n5H1WvRdNcm9rno69vVFXTNPp05KSFD4G3F4Q0sDmldnjBDi1wQupneLcqGm3nNhCOztHdYLUkHXfW261YTk3qxnqX3GynQGRYtEEjeIiypfrZS4hwCQUxZIziIkmTNtmE5mArImblcViwEZFAz9IgAz3mPqNjSgd9ZixN5eancCdwkkRyIEj42HKgrtDZWok99rTFVcu40BIHutgneDrbgkKMgxA7trYSdYsZpUUICAdZ8JtuZGxjxsHME2yVAdokdx7rSG6ZDhRnui3STpANgJ0JNugQNu/utMKKDrHS31D5/wbyc22gOh+uy1RJIdCp562Ra+V0jqB62Ux4EWVbtcDwadTssBXnIs3NMUvjqoErVP9K2WDlfIfA2yzqK8I8Usre9BpkwoOXjTJCjuMzie7zP1WtJSOs3XetM8xULcFI+h3MEhMhJkEe61Yr5Qly97kbAR9LfVIDykF0ctjy377WQCmOszdYolMqmJkD6/nlZqUBzXAoNY3MA0p08SL2u3EGPb/v7D7i3aG8KxdU0ooKT2xnUCDrIUV+4mzbzFCM0EJmYPl99hqZ6mZUMyUFJUO0E6mPkfXYBxQD6gwJEgoOghPdr7ff77kY4bQwclQbHlbYJx4coqN+uaFW4lLZIzZjoR9fLb8bWH498MuE+EMC3PeGEr4bfvKpCT1KX0rLqCgkuwPVggDyJjY2qwzXhtRWhWVQI7WsjaJ5+FjdRe9ZUIQhThI2AJ9vfpP2e9nMLnA32RGkAEEIjWFJWpCI9k6ifAfM2JLSqZIMHY7DQ/jYy48V5SlKe+e87cvnlYF2FKUYBEbEajlHzv7LGTIs4V5+yqOUyNPHxs/sBXj11A7QOqJ6hzMhIMEBXdr3/AF2YqilQOaITsAr5+TZWwreKKC+W1OrCWXQWXCZ0Ctj7CB7rAqWcRhVmlfkkF1JqkrcGYwAdIB0kfO9kLFb7X5gr1OtlUMrJCUnSNPxs6G2iV9pEmdQNgQfq+vTvsFe2GbsvW6a9VXdNRWFLK3HMlWtgKRl1SYI08Rrbn5ZAxpuuljhLnCypHilttnpH0IzAkXvdJVuI1YOx+rwNvZil1bBIHnO9vIDiu7d46UVGLpudu6mRed0AUyXnXgDLWYlTiioqgeWoFvYGlT9EB5672BmzMafBCILZHDxXakwcpPwsWeQY7MgnvNjkaRqYsA8kkSYE87Ti3Q5EmPNFIzAeJsSVnBnUjf6rKTqVEEQdPn7rFFt92lrrNlSduo84xj/ea8QkFSahpQBOnrRp77QStxQBIIKhHIiSLWSx7h6vxLcDtz3dk695aC3nVlHZUCdeWgtHLXAnHD0/SXclJ0AL6jA8o87HhkAFihOp5JNWi6jVh18dZGXsgdkoM+tvbDnUAtcHL/NB+futKTHR8xnParbsSSP+NWdv8WxhHR0xacwXe92oBImFOGf9HWx+I3qm7pN9FRM2opUlSVARPaCRGm3uNrF09Hiu8MUJpaCmmiXSF+BSLKCMo7RfkpDmc+pHq62aTfRtv8kl2/6FM6wGlGPq8LWMu5pdDd9FSOFS1NUzbZKVECUpAJsJ8bah4dfZDkpXhtpBZQjeeF8aB8tLpqhLcyT1MyCToDBB9/OzD4pUlXSKw67eFE6w56fol1KgSA9T9421NrbNuZ9UtLI785Fo348YNocSUV2XhVvVDSrqS6+2EEQVBTaoVM80W2KjETNTd3LfrWbR4KIKnjNdfwUI8U0pZbpFhOeM8iCSCRMfC0Zqq9SlKI1E7jf57rWSxnw+uy+EU4q6mpQE9o5FATodNvE2aaOB2EOw65fFS0VSoJdqUpI3ExlnkbZseRjbyED2rRmie3VQy4pKkhwEyNddRMiB8bcIqnUoCSYA1mNfnX67TRU8GsNIWytmvU4zn+nK6hRlMRpCd9rHLr4K4ZvOo9Do6V19Z7QPXuIETrqQARPnqbaDKR0jOIPm9VTEjS7IHC6gw1D/AGlFSSZ1EaHw1toVj4GRxYy6wSYgefutYw9G2jVI/NLSAYAzV6j7YmwzXRwu1CMpuakWZ1mtWY133tAQs2LgjcF/VVmq6gi9sLOqUJavlade5yhqRqOXq2vfQVV6PspdpmaYAoQUlalagpBmI8fhasHEPBFx4bv/AAnTU1C0yUYnpm30pWSDLbzZ3/px7bT5dtRxBqE0a7toblVQKaQEqffcCw3l9bKnQnQaWoSMyyEAq62Mhguleoexe7VBafQUoBlJJdOk+A87VD6WNS7cXTL4IX25RoD9YigpXXBJTlTXKzR/l6z4d9rW1h4mIW23SLw602lJWo9Q+dJ10I1PhOtof464WTfeLsIYnxvdty195Xa7/uLWMsvNmnUHUKJKFHecp1nnYEjAxhJd5osERc6w1U8VDvVVL7eUQVbEAzz5+djV2sMuTVVORKAuAcolR9gs0X7g4pOFC0YvuRKzq4Dd4USdNRKhpY5S0PEhthVLW44u5aVtlttLF1IC23FbLEOQYO4Omp02tF1Q5p0YT5fFFMIt84Kg/S5o00vSYxPVIoXUofTTudYpIGYKp0SBPIkWgXjFT1Jvi4aijbUWjczLnaVEKQ46ic3eQIEfs9AONvD9hWIKiuxFQ0V63p1LRXXLoG0uLAb7II12iN+61d13FSV1x3a4/d1JUJ6hxhfpDYKzlecMAqExKtPxtzxlvO8HqtNtGcjSDuFCd93hUXVhusvOiKevaYKmY1SFkAA+MTOm/wALV9qBfy6168r3raUqW4VuuOMoWsjmSowfDkeWmgtdrjHhlpvhXeSWaGnplCnYeLzTYKm2wtClHQSOzI8Ne42p/fVwVNVUMUrlO56O0cxAb/lFbgRrp3DbWOYnWw5rcpI3VKvaWOAKY1QqgvCpddprsUUgZisrIBA3Mew6eFj9wNIcoVoUwSpAJIAJggxJB8ZPvso3y5QXeUsNUoS87CENwEFRPOCOzqBqSDrHLRDw9WJDwS7MFSpCdSJIJ13J0Hu57W0LWKz+ScdNQ+ku9Y8AhtEgI7QGmuoOk6+HviyysBRKtMoSFDdMgpgHvIEEaEz4bFMYrHCtA61LZIBWAqFakQZG2uwgTChJ2seQElKGTKWiBkGuUnllgbwk7aEADYCxBZRN04+G1w3Te+Lq5+/KpDDDDDa2lJfyKzlagRzJ0TGsiCPZaK4MYYRo0iipj6WwQkLzEhtSNSdSZMkjfeCBNqS3/eS7vvOhCbwq6dLzayXKdIcdCTAgBZGhKZ3j1tySbG2sSVq1oScT4gdDeV5X0jTGTtR+gTpr+smNdddN3De0BwqLhsZc9UN1MJjdxV/brx1QKonW6Bhtb7CHEtU6SOsV2iG2+/wG/naFMcYSvG8HnMW8UcZXFh9pw/RM1QdcLaN8gbAEqhXgfPSI5uq/qfEVFiS7buum7rvr6LD7t6XXWUtbea31VDTrZhPW1TzSjkUvUoTqNCCBMJ3nRYovWvXU361eFYXwWUPvKKvpCgqRC9jJExzE+dpP7ZCtux8e3jorT8P4LQWndO7iZh3h68zeF7YYv68q2uplIIU5RJpqdxA1KkoKlLiJgkCdNNbI/AviJ+9dxXwxjcpUW7qvNh19YKp9GkofATscza1bidB4y0q/DmIrnYUalhxtBjOlCsw57kad/Pv8bJLSwk6xbBqp21bs4aB7FBoLF79NOs1KE1VK8HG30hxpaVEhaSNCPAg/hZAuy7qoXleqgclO1XKV1ijlQApCHDJOg1WdzaKOhJxJVxJ6O2HnauoD1dh7NcFYrMSZYy9UT5sLZnvINpbrcDO4lvZVTWX1UNUCCn+LNqI7YSJMnb3c7ef9rKdstHZ5tYrsezcxjqSW8whrwx1hDD4S0/VuV76NAzSJkTH6ytI8RNo+4pcSeKRwdWX3hO6nMPUVCApx8N5nFoUoJHbVqBKgeyAdLTBcWDcJ4fANFdbJdGvWOJzqnzO1t4t/N96XLV3TXUrT9NUtFtxlaQpKgRtB8becxCnp3DIy/tXXuDpjYrzqvp6+8WuOvXzelbeda6cxW6tSidfHX3z426wxgq9rovOnvtJDRolBYAkQlWh58gbS9edwUVz3k9T01IhpLbhSClEbHT3d34WSq6nULsqmgQlXo7gAjWcpOn7bdtAG5PRGiyZGZXaqrnSZwX+4+9rtpG3VPJrLsLrZV353JHsJ+Nq2VeU1CynadO1m+Nrn9LSnReOHMDX6UlanaV1lxQnl1ZAJ5aqNqaVzZS6REaQfMafZbrsGfmpxdchjLAyoNk5uEF5fmnilhSvzZQ3e9KlRAmUqcCVCPIm3rTgGo9CUspBBUBmjbcfj7reN921rtBeNLWNLKV0zyHUEcilQI+NvXvDFe260w/TlWVxvOBO4IBFs3tLHna3xWx2WksXjpZS0m/D1IVGhhMTFgKh+tWx6UGHVMgAlRBj37WZ1XVXy9dr7lw9S5XJQVtNujMhZEkJInmB5T7rI9433jigvFxVLUNVVH/h6uoLLSkNhaVPdUnPC0dT6pI/woJ9WLcL3UXsF3PEsE76q8gmSCJAgmNx7bEfzpkWFIOoVmgHn4/jaIf3S8Q6qncR1zTRYdU3UVDj1MG6dsPGVOESUOBGSQREKUcoIAKQ9iniGtvr6a9rs6kNNqcLlRSlIUrKVBBCthlcAnWFzJjS22ksEEzhTNeNZlfcAywHFAeABPf8AOtmPxRphfvD7E9yLSFem3RVMpBP6amVR8SPmLaw/eV9VNEuovd4vLW+o07xLZLjBCSlX0RyfpESOQBMWHfqQUONOlJSoHMrc7eeosaFnDeCORQJ38SMt6heTl7piqDn/ABraFzESY189Z1sucOb0Th3iHhe/HP5OlvKkqFTp2UujN9RsSxddq7pvR67XQQuiffpVSNy26oWSs7gTTvAkZCUgzsQZ/tW9QsJI7dQvKNY5r9CvUOpqghTiSpO+sET36jlzjW14uC96m9+FuGq3MFkUCWCZmeqJbOvP1beeNx4g/PWHrtvj9GtomKk6zOdAVy5yff7rXd6KN9fnXhQzTFZJu28aml9hIdH/AMW3D0gLJi0rt613Eha5TEtwE7e+wQKQrQe+3SwoKI5mwZVvuPhbYasZyK3u2HqZXMoEiLNtREHczOhs6ljMkpOxBmzYfR1T6kRrNtKlfcZVn1TfWXAyncCJ2n2W1TlQpmgrVQTlPiRoZ91tAlOidI+fstjM5Fo07KzHt7Q+u2gxZzt0Mlz2+PO2ZioEzp3c7cEFIy6xvvZNxTeFfdeG72vO6mmnKykoX36ZLoJQp1CFKSlQBBgkDmLHa25sFAm2qV9VcjoeetsJnUHbxtSbA3TR4m4rxlcOHK6hw3SU163lS0b7zdG8VNoddShSk5nYkZufdrpYPGfTM4p4dxPe1y3XS3G7R0tW61SvVN0vsOvMpWQlakFyUkgbHuO21tIYXPmy6Kl8oQ5cyu8hRBA1+q3eYnUEbWi7o9cTb04rcNaXFd9N0jd4elVFLUCmQUthSF9mASY7CkzqdZ8rSYF65p2tUkYYnFjtwrLHiRoc3YoYgEEE6fZYzhslVM0kgy0S3EbZTH2WJhQ2E8wZsPhzs1dWxIOV0OD/ABkj7QbD5pynWA3HrK9ivxtlszDnr4xbLOorwsvFXWYiw4JVlcvqkBT4Z59v7Da1eH+GeNMUYavbF9z4eU9c1xNlyvrFvtNtsjLmiVqGdUa5UAq1SI1AtVQFbuMMMJkEfnmnJzbEAq2/C1hRfF4GiN2+nVKaFbgeXTJeIZWsCErKNirlJE+O9npAcpshVRFxdGG6tpKx1uciYGVUEH5ixiorbsLY6lnQiZWSZMb+/wC2yQVA65lGDB5GDpYZlKHSnNUBEKIEp0iBr3H8LXSFUaeSFDrZzJUoaCUpjmfn67aSsqJmAYhPf7PH577GaihuxulW8xeqXHApICQNVAgmQfD7T5WIFEZQCFdYNgQTvpZB10ntyp7UHC/GN74CquJV2XL6TcVA6tqtqGXm1KagCc7WbOlPaBzRABOwszHXlqBKzKSdiZB5SPnlYzR1TjdMqk9OfYpnyOuS2uUqCTIzJBhUROu1k5TkLXmQCIJOsTpz7rM24vdLQrvMZKiSDMmDvHn872CQ4CvNlBKZkjcnkfnxti1Zs06lRGXs6H7Im2kZcojtCD8z3WkdUrEaqdsLOi9blpb0Ha61pKVymTmGivv1Nluuu2pTc1XLa1kMKBSdiI+fKzJ4HXgmoVW3G8U50gVLOY6qGywP9E++07LuB96633BIUWFKEwAogHu3/C3D4rP3V7mFd/hMXeYmvC83OMbD6ek7Sek9pabwuolMTl/kjr8PZ5Tb2EpAOqSOUW8lukfSJp+le2IVCq251KKiDBIamIGomdPE+31rpRmbA30+FiQvzwRuHRZ1SzJUSDxQigSOcROnKxd1IMgjaxowRBFi7oB5R7bWY1UkRB5O55/GxZYIgkT56WOvDTbflFijiASR+21wGwVN261Qsh68WG1AQpUfCzk/NSVGECOe9krDrDTt9UyXgpYJVEQDOUxZ9IpGNBkJWTERanNUcF9lq0VuHqm4i6ZgTqNd7DC7skqzgQIs4HruS05lKTMSRG1uTQpUdEGwjWm+quAsOoTfF3p7U5FTzINjXVJ+jSArsJjSNvabKvoKpASm0W44cxXT4zoLruW4Kiro6p0IqKlJcSlkQmdQQlManXeI3sSCukz5Y25ifqVDEGs4eZSIhCQoFSkojXUo+ybNziUlqouBwJUlaksuzCgY7PhaOcT4UxwtKlXQiscUqBHWKjfkMxO1jWErkxLRU17sYhuqoZb9DAaddSe2rtSNSQTtbq5qICgNUXgG3zea5iixN8lc2mMRAvvyTlvtjNS0yy3OcA7eFgH8KXxe9Bdb90Uryx1mVa21NJyArPbUXB2kpjVI1Oay7UMJqLtoysBUoQdRMdizmwylCcOsBStA4pIPhNsBuSrAjcdtVv1cWWPVMPEPDjELi89E6tQCVABt3Qcwcsg/XbWCsM4muG9yq9LscRSKZCRUFaT284gGDNpUSloArCFq8YP2W4vVE3anqgUlLrao2/SE726NmJPjpu7ADKubbhUTagTgm6a6XllaguoCsuo7f+1bhl4GtzGqbDaBJGcafGxJ+iqTWOop7uzoE5TnIkb6zbpijW3UFTt2oSBHa9b8LUC4laoCgTjW91uKWQptIFJii63k6DTNUto3/wCk+Np3ww4+m4bufQ0kgUyUEqI3H7OXdaEOMNOanFl5UiArO3W3TUxGgitpTPj+laa8LpLtx0dOsKhlx1K9YgocUmI8x8LZdPViWpdFfUfoWnVU+Sna/wBiVKt91xoF8tMgHKfpNz7rRP0gW2DdWGK5sKc6m8iC4k6JJA0nxg+60uXilostpU+4k6qhJ+4WjTjq31GAKaqKFuCmvNleZUyJkD2WfFr90eRyQ8N/jDVIFWg5m31L/lUggxPIb6W7YUgwFOKUrQCEGPqsBTVpVQUb6HSetpmlhPVhUSkbGxxqsqVwQhUd4SRra1GbsBVaQWcQo64y3RmvVpxPqv0yEqCzOska/HS1asH4fFdd6qAuNU4ZvKuaUsyUpAcJkgRrCvaSbXJxvdibzTSvrpVnIjLtJkEHW0A4PwA5TX7iK565ioOS9XahpaUz2FpQobGNYjXnbzzEq4U9fM08l2VDGJaeJM1zCtXelz1VCqkYdaVRuBTTmVKXUZCChQXpqNIPfFvOBpT1MVrUHepp/wCRaSlShHKIHgB467k29f6rAVOsKbWxVLJTOiBMeJncTbypxbhGswzxBvzA7NLVOPUF61F3tsBpSnFqDpSgJSBqTIgASZG+k7PZvEo6tz2g6iyz8fpjGGOCh/FV31D49Pd7DyO3KBBSNOydZ07/AD7jZu3VlDgSUpG3aIIj2jztYrFPRd4+1N0mtc4M4sKG2i7C7qdQ5HOGynOdTMAT5xaujNO6xUO09UHWHWllC21oyKSoaEEcjMgiPw6kPa4+iVzRY5vzhZPi71pQ0nsyHQVhJVmlKtACNyYOhI29kHVVrVOwh6pWSjRDu8akpiBB2APM6H2Il31br7AaQsuleUBA7RiBtG86eOo1nWytT5HaZXWuL0gKlJSqOYkgeHge14EEO10Mbrinuy6MT1jL9TT1FWltsNDKsIKiJJnukqk6AkyZ3BfdJw6wKC1Uru4IQCleRbkkpkHXTY69/dIgGzawjS0ragUtBvNmWWgUwEiDqDPZ20PcOZi0jNUSq+ncS1UOMrUghKUiZXBEdnQiSdfuE1Hu1VuNotdPSsxNiZpgXJU4mxFUUSUZfRHr6q3aTLA/wK3CgjQH1Y8BIBa15VNbUpVTM06HX1ylZcQkpBAA0ncT7QDIGsFaVQsVGZxdKtxSu2UGoIIntQdZn1QRE9k+1fuW4WLv/j77CGXMoKA2VEJSnfU6zGu40I3MzRbH6eiuF4DdVD2O8Fk0tPdD9a7U11e+X3VqRo00kZin2lSZB5DTYi0PY7wevDlSHUCW1mCMuUDuI11BIVr4WsZiDESfznVN3LSi8b2dSGwAQUUzYOpJE5Tvz0jXURaPMScLr7c9IvG9r0Ljr6At1pKRlTpBg+3TTaNtBa9G62ipyMvqpb/Jk8TFXFxKv3hlWP8A8VxRQelUiFGP45TSqB/SZU7J/mJt6VtVRplZUkgEZhpAnn9lvErgHixXDjjxg7Eq6gNt3bftOioXOnUKcDbuvcW1Lt7V1zT3WJDLa15SQqBMRFue7URZ6J59n3ra7NyBtY0Hx+5KCrxMdkySZ3599m9iG9CGlJUdBqDO/f8AbYRp0rdNNIzA+qDKp8vvs3sZpfoshcQtPWoIlSdd/wARbzKCIOcAvSiWsGii3FzQF5uupJBc7ZHIA8/dZoOvMZ3GVuIyuJI1Mez6/afGzb6V13X1fGFrtcuNitqXmq1C1t0ra1kpLSpJCe6B4CfG1YF4Oxo+sJXhm+1KCVJCjROkHXxB3+7S3dUFG2SEOc6y5DEa10MxY1l1MfH5lq8eCOHa1PbN3XkunJSSYQQsa66jsga8xalt7oW3UqDhJUFKSSdzBi1wqu67wHRgvO7LwpHaepui9s4bcSpKwC4ndJ29c6Ac7VHxE0E1jxCYGYK9UiJGxHsHvNtvB/QzMHIrAxe73NkPMBI6SAqSSdot6ncIbyF48PMKV/W5lVFz0ZUSdz1KZ+NvNDCmBMWYzZvFzC9zLrxdbBqKvItCS23B7UKIJ2O08rX86MV+IreCOFXFqUpTLLrCir9Hq3lp/wBVKRG8WbHmh0II5FWOzryyZwPMKfbtrmUEocqAwFJlD2ScpAO/hBPwsz77wHhWrK6b85VNSy448tTSUpS0kurbU4kApJIKW0txoAgkAbQdVeCusUgZd1DcEHz+Otk1+tccSU55Ex3TuY12H4eFuPEdjcLtjNpZc3PcdDh5Tymap97rm0tJ68pISkOOLiQNe08s6zoQORlGOEbmpnGXaaprEmnZLKBnSYCmGWSSCkgyhlveRI9lj7lUgwkq2gZpBPv91kurxFdFIVmqvCnAZCQsFwCCowJHcZgeJsUMdyQXStCUqNTV03bS3bTKcW1SMJZQXInIJAkJAGmmwiwb1YtQJCQO+R2iZjvmzOxJxLum5B6OhNXW1LTxYU2yiSFlAVImNIJNkq4OIir4v1y5ai73KZJW4GnCvMFKbPaBBGhA19tpCB1s1kB1S2+W6pzx8utF2cTcTMNg5fzq7UA8sr6Q6B/pH32j0KmjCNew5I8Mw/2RacOlhdPofEKprQjWvoaSrKhO6StonXl2E+8Wg1rMW3UjUABR18Y+23fUT89Ox3gF55XNyVLx4q73BG91Xlwow4+V5lN0xp1Cf+KcW2B7ki14ehXfHXXXiq5Fr1YqaarSJ/XQpJI/q028+ei7V+ncMCwXCDQXi80BMQClC9+Q7Ztdjoc1/onEO9btJ0r7pWsDkS26gg+wKVbl5miKscPFdTE7iUTT4BXFWSdI8NudglEAkA6eIt2o6AKPhtbkiRJ3Ag2vjZZzkGdNthZCvhrK/ngAKE2XSI/R8tLJt8NFxgOTBQZtapnZXqtO3M1IgVqNt7aBKH1g/pZVfZ9ltgkaxGnwty6cryVSrtJOvu/G2swrIcF2Vnz+r2W04hDramlgKQ4kpUDzB5WDJJkgn3WESc2hOu3fY7TY3UDqvNTA91u4VxvjW7rto2ajElw0da3crbzQcipYqUpcU2hQhTop0vqTuQRI1CbLF+Ur/HS4b14gfmmiua+qKuyl2ovhw094IUlbrrbZq3lBC2dVhCCE5FFOUEJBc/FevwfgvpBYkpMTXBT07XpCLyo7ypWX1VfXPdW7Mh5KRBU7BSBqlImZNieIuMXDLGnUOYrVfN5GmaKWW6mhDwbUVEqIK3TorsSQmSQSSdLde17nZZAOQXOuY1t2kqV+gZfQewjibDqjmFHXs1bYnTK63l90tWtMk6wSbU36J+ObkqOL17Yew/d1LSXXX3UXacpo0sPOONlskOZSQYzOR4WuKlQjsmByj9tsLEhlqCbbrVojeEAckMDzAiw9zrUm/C3PZepwR5pVH9qxUHcETpuR3W7p1lu9aB7NCesU0rwzJMfFItRVtPArIJAbVp3AffbLdEH9Yf5NstKygvC2jn91+Fm0qnNebapJmE5SZPttPgbX1YCO1mMlQjztD3DW5qbEfGPBFw1ZIZq70Ofq1QSEtrJg78teXntaxmNcGXhgquRSLqEVdK+VLZdUcqiBGihO+w8dPKwaWoja7hE6lTqKWR7OKBoE1ilSTOTSTy1B5SPn426QlSiUFJIGkjmNefzFukdamCRuNf53xnvtuXuQSQezOuo1+zxtpLNLSF2ukrGlNNqbAU+2FoCXAYB2Jg6HnHn323+b6tTiGSRmVBBSJ325anXztwFvRmCiANSBy7vL7fG3Sn35CusMADUmTHz91oJcl05QtNOqCllbaNZQkkj7J+6ytQYWp36eprW78uyKNhFT6M84426+lRIKG+yQVp5iRoRE7WSg89kyl3UGCJ0ju+d5t3mcy5+sHa00Hf5+E/XZrlSYBui/orq3DlaUmFT6umtu2rvfWqepX2leUnwnz/bYTO6D/KQB2TrsO8/H3ixqnceCuy4qSNY0iJ17vk2i59gitYCU6uF9BfVBjG6rxprvedSioSl0JIGZtQyq94J+u19aXh08ukIVTglaYEJiPZamvCC7H6q+GXH75cpw24lSso0UAQYP3fI9KaTq1UzSkAQUAj3W5HFKEYrU5C61gukpauXDaccP1ivGjpp4Uew70vW1Ip1MIcduVwKnNnPZEkctlf5NvTWnT9GE9w5DlajP5Supo0dKDCrTYAeFBdmeAASfSnSPPQAe7uteqnTDQMb2eCIQRCH6OnkmfIZjxHbnVdkaaGdRzsA6FcxubGSAE6WLu6KkGfO1uNAk2RJ1Mjl3WKqA5GZ5WOPAkGD7rFlacpHdEWtDZUnbpawXRLqL/pgEkhOZR8otJ7NyUzDgfbR9IPVJUSB77MDh06lu/wANzq40tP1H7LSlYfdo5XFzt1MyuYMrSk83aVKzKWCTzi2/zYn9b4WP21ZCgh6KPGf1Scq6hmzBZFm3ftG5T3glAOixM+HyLPazPxklb1WhhDhbJSntp3GtjU9LHC/M1M6VzxZyIoYfMkL/ANI2BvlhhV2KaqX0ICuyFKMCToPrsMylI1C839Ll8LIHER5LWF3n1E5W3WlaAb5h32JWXdA8A8kSlIErT4o3WUNPR0bNK64gdUhKVGf5thbpve6Ke620KveiZCXCSFupGntsycUYuSjDf5wIkuNsn1huUjb32g/91rt4VaqTq1JbWVQQqduX1Wy8MjeHZyVcr5gW5Fbqgvu5bzfNPQXtRVTqBnUht1KiBMSYGmpFkS/+JXD+kZqKB/F9zN1dK6hLzKqtCVIUlwBQIMajWRvpas3Cbig7c2Lr2W/RNNs09K4yt3OZUesbUInwGv2c6t8bcVJr+JuLqZKHMwvm8WlgBJzD0hcRrpztYkxBzah0FtAL3VdlLeISL0lVxV4fpa6449w6lDmYoJrW06DnqbJTnHfhC2ShXFLDSldzdahcxMxl8reQ96UHXVBKCIkE6GPdzMTE/fZWwut2idbaSy2Uuk5S4BppHZJ+dbNJiT2i4ATspmuNiru3njm68b9IfEgufECLzuP9zrVVTqaI6kOsOsLVGkk9k6/eLTvcuOsPXbelfhyovkN1jV93gyGyy4oQp9a0gkJyjRQ3NqEcHa5268WXqUENn9z95pMdoKHo6laEHXUD3+++PDSuFSziq8W1gqexC66DodFsskfXbJoXSOry9thcFadU1vdAHa2Wrx48cOqSodbVfF7PFtICBTXHVuSfAhuOdo16QPGjCl7cLqq67vu/FS6x2opn0KeuOop2wnOn1lrSEiQRuedp8C2g4HesWHNVesDrZldIS7kXjwNxMtvrFOttNvhSpUMyH0K1kwBp5C3RzRvkic1x0t0WPFII5A5qI4d4i3gxh262K3hdjpVRS0LLLxN2toTnSgAmVODmLL1Jj69KlCVM8LsUQVgHrFUbeQHSTL+lnSwp2opKdQUShTKFdknWUg66GwoYK2S0WCpJIkKXv7ItFrHgAByTpGk3ITvwQgXncDdbeV0O0bzilhVPUqbWtsAxBKFKSZAnQ87FLnuy5HcZ37/FGi64GHvVifowmfH1bF6W/lXax6NOUlAUEyOzpH2Wj/DWMHaLiZXqdeJRUXe0AknTMlS9e/YC3D4vilNFWthkaDlN3Ega/tda1JRTvifIw2BGn3/oU3KuK6TJNG3rqdLVqrOBOBbg6Tt+cVkXEy5etfT0RpVLMpp3C04h15tJGi1htKSRrAMRmVM1qx2iNCO7e0d8UcYt0KafE1K3lqW0Kpi9mTlSCQpOYH1josJA5q9hnHjWHSytjpm5SeYFr+CPRUNSyS8wuP21RPF9+12IcMX7T4NBqb8oOup2WWnG0qTUR9Gkl0pROqdyE672rB0m+gffHFrhLV8RaDhYm7+LNFQtVVS5SVlOgXstEF5Drba1NrcKc2VQhRUEgqI0tZO7b6YxFdrDNyVNWqvU4atkodbCKllcFbCFKJyqCpWmQRvqJ0lm6sR1y7kpUXjQuCs6pIcQ66jVQGpBBM8uVuihfFFdxP7dE2INkIDQPCx+9fODSrdYqy1UI6tbS8qkPp+kTBiMu8gSIjmZibLdFWOvZaJtMKUuChLYTBI3nlvGvIjSdnP0ob8pb36SHEi9ab0ZDVTievXlpkKS1mDxkwomVFQUTyzFREDQNLDLLldWpbZQFJbTnIIOpA0AnvCTr3a7TbYD7sDlz2Sz8qkbDd3sssoFVUIDQAlKF/pQSo6EjUT3erI5w+aaqoqWlXVKUlDQQlkpJgulQBGg31JjSN/az7mp1LyjrF5MwS6lRSEpRsN+U6g9x0Gup9Ncq8cQJuthK1sUSOuWjftgkTMbDXY8h3mat1ZGietEahCGnmyG1NIaTkACQ2QO0JjYEnwJGmkWN4gvFtF3uKvC9jQ0Q1rq4LhQSkDstyeajpoNIOsyGy05ezTSxdlyVV4LcWtDLbTMgKBVur9Ae5IkkKBIFmheRq78v70W/gjEt6tqz0uHbuWpdNTCfXqXEggwf0EknvjWzNYpOeE4KC87zxEytzAtGjDGH215Xr6rW89TUKMgBsRIJB9YjTNuNrJOMhfd0YdvGgwldSFdYhwu3jVupqH6lvUqU0RoNEnygxrrZ3JuvH1Ghpm97yvtzIgJRd9y3c0inZEyEFboS3AMczzJGuo7tK4pLt4Xph+loaBRSl1LT6qt2QNS4llKm0zESQBI3myddlrJM9O4Kp11zvX+kZyXc2fNzzTM2+gXo1Ymu3FWGcPYkvlhh5N+YcorxPWJCwHVstLVE7arVbwVx1d1HdmKa+nu2hepKIuldO26ZIQe4jcTMb+dvYHoqX+810buHd6NuLlq4GWM3gjM3Hl2fgLVcWJ7o57BcgXUqCPPOIibXNlcaiw3w3u29Xb7p7rp01TpJJUoqQknfKgnKn2AWizpQ1dyrw/c7l3MU6HBVuIUptsAhJbOkjyFm6vHb5Bh477T8/Isx+JuJHb4uZppx2QzUhckyNiPn2d9vOIq6ea0LmNAPQLt4sHMEgndISR1Kji8Hg7TtrcO6CDIg7nlZCfQUqJUgKy6DxE/YbKIdLtKUEKlt1cSNpgifaT86WIPtlS0rVqnaE+Pd899tiK4FkeU31TAxtRJqsE8RbqCRIp264CDqQjNP+hahOI21JcUSVdpKViQO/XW1/MW33cVy1t+3df9f6ML9w+6yyktrWp1acyCkBIOwUPkxag1/ozEgA5gjU666j2d/wALdBg5NzdcxjVjlspO6JVBeF98RKrDN316KNy86AozuDsKh1slKvCCfr2tPfRGqHBw8vK5FmFXTftTSwQAcsIV9ZOnnam2A8XKwTiAXyKP0pBYcYW11mQkLESFQYIMKB7xa0nQwxB+dV45SltxpLlezXoazZsvW9bInn6iRMaxa5isV4HO9iq4PLlqGt9qsrelcaG737wCOsDDanCkEyYTmI56fjvzilziXfTrbriruTSPNt9YQTOziZ33BbWDO8zaVr0pzU3S42kCH2VIjwII08D52ZuCLhwfflKlN9Jb6966aohQe6s9c3Tktk7Tq2Rl5kczbkC5sTS9wvZdFiVWaSPicgoora3FF9Xa9XXrflYGGFoqMiRkAQt7q1IkD9ECf22T78p10lReqW3nFumoqaZ1a1mVJQhLrc9x7Pd8LPjGjF10L193Ph94PXbU07jrQzyEOJSFqhXMaT8zZn3w4am8HFAiax2hfSJ0V1zBbVAHu25WvwkPaCqUc/FYH9USbxKXbwcv6oBUG6miqnNNSA2ULAjSZP7bOO5qxlePKcNNKbU5WrqkdZ2SEusJOUxrJPyZs2V3U3dtz0l5suLWa2kVU5SAcim1tEp8dQqDZwPXgL8vJ7ENz09WUJTTVCQWiCpSFlChpM6EHy5crPINNEZhN9UxOmJRhV5XHWtt5fSKKqp1KAMEtOJcH+ur32rCx6ykxMoVa3vSyp6erw1cVaFhbrNetkjN2sjrSp056oTr4WqJSlCapBdHZzQfLa3SYS69K0LnMYblqiVZPohXiVXfie6SokodpqhtEn9ILSo/6KbXU6NV7G7eM1xBSsqKpL9I4CRJzMqyj3pT77ef3RUvVdDji8rvERW3avQjdSFoI18s1rfcL8UqpMZYWxEmnfpUNXpS1DjT7fVuNo65IUlQkxpM6+2LY2KMLKvN7FtYY8Pow32r0tMjTTfabcBY2j2W7WkSRYMyZ10PwsduyrOC0TCtJ1PhYCob61paO8WFP1mDNuVa6RuO+xozYoThdNZaci4982LPrhbKiR/KRv3g/hZRvFrq6gzrmM7xFkyvlNOViewQrY6QZ+y2zGbgFY8oylDBURJnXnbM0n7jYPODsfDa2wsESAd9TY4QU0MWcHOGGOb2N+YtwbQ3leCW0s9e9mzFCdhoQNJ+dLJ7fR64INGBwzuKd+3T5vrJs/yZ5nv2526zAmQQe4WMJpALBxsoGNhNyE18O8K+G+ErwbvfDWCLnuytaSpKH6emShxKSIUARrqNLO7OJBO5NgUzJAO21ugrkZj7LJxc/VxSDA3ZDZoVBGpO4+dbdPr6mmNTB/i6kPA+CVAn4A2CSde/kAQPusOw0Kpt2mKjDiFJgc5EWXJJPYQoAhI113Fssi0N9UpoqcuuIK+qRmJMGYEyLZZXUV4/dHynNb0jMCIIUUpqKxxJOnqUjx08dLWn6QTfVquuVKSVLd1SYnbutVbo6Xvddw9InCV7X0+lmnaF4ZlDUgKpVpCo7pVJNrUdIe9aSrq7laoFekUzrLzwqmyC1qQIkHQwJ8iPK2Ixru/sNtLLZDmjD3i+t/gofSF+qgQQZEDee/3TbAkZiFEqIgHT5+vmLcn1gRqmCddvw0tpbi0qbQlsZTBUvMNIG2+2se+3S5lzbgTsu0gqJJKiAREzAjX6/neXZgThhjfiNUvUmDrkcr3aYJU6lLiGyhJnU51Cdjt32ajWYqSSTOWSSOYB05d3zrabujbxguXhViCrfv6hqH6OvplNuPMKTnbUPV7CoCgZPPTxm1WrmkjjzRC5/a6ry52C7Qm/ivo8cVME3IvEGJMLqorvY1dcW+0Y12ypWTqfqG9o6U0puSTOsHWDBn5/Za1XH3pO4Q4jYNcw3cNDXIeWtKwt5ScuhEyAT3n3Wqs865lQGy2psqVnJME7RE+3n7LBpah8oJeLdFKBzpC7TTkuUhZIJUnlBI3P2fPjY3SFLYSTAMyBI1+0+diB7SSZJEafPPT6zbfXusV1O11qOoUlfWE+sIAjwEid/C1hxur0YI3Uj4dvxd1NhTahmUkxG8iAPn5Ho9hPE7NThq66ha5LlGysme9ANvKBrEtMh/qyshBQFBQXuZjb2fEWvphS/L3RcNzVaLyZF3IuZpDlL1EuLeyoyuBzNoAAoZY1ne3IY/NJRZZYjY6rosOphiDTGeSov+UVvr0zpkXalDhCGqS5kctw6pQ38zr3n2W9HqUktpkbi3lH037xXeHS1YfUlKhnuhGdSo2Sgx8ffb1cpP5JBmZFi0shmp45HbkXQpY+DK6LobIbYad1i7gOszYzyhI2sXcmfVtdjQJNkUd03E66+Nia99QPDX7bHXQIkqg2JLIk77Wtt2VNyWsGP9RiSiOaApZTrzlJFpd9I8bQth13q78otYPXJnX5+TaVA+ncue61OoqDA+w5okcfEF0qmpSNMwto1Kf1haHeJ1Jxirr2Y/e/v676Khy/S9aAFzzmUmfD6rM9jCXSVfWDU8S7uQJByoTBIkaSEd0++2U7GZGuIDUfuml1ZE1IH6XxsTrnaeW3VttqUpYTJSCY87QVQYB48GrU9eHFNhbEH6JOYb7a5eVk+8OGHFpotu1vGSodRmCOrAWBmVpm0I2J+dwJ+MTub8yydtLc2srBqfpAjRlsf4otXrG981FTeN6ULtY6Wk1DgS0VHKAFGNNrHsL8PMbXNfrF73txOra5hoFKqYIVlVqNe0o7xBkHc6WY+NF1lTfV8ijoahxXXrg5FQTO4I5RNp0le+oLhLpp1VhtMYrOtzSRjPHd13Zhhm7rxLjQS00FLMZU6DU66DTe0WV97ehPGsp0FQSCoKEQpJHJXdHh3b2WceYSXia56elqLoerWqppNNVIW2ohc9kg6bbzNmJf+D8YUR/MF0poGaZDbTdKFVJZap2kNhGQicwOYgAzrHKdLVHiMbSWE6qxLhpms4JFZ4jNV9/3zS3U4mGaR8VYBSo50IzQoAnKY1g66C0T8X6+hTxexNSisp/Sai9K59DJWM/Vl5RUqO4fOxs6nMC4maF61lBcDDFT1C1V79PebqQ4vq+1oE80iIV4d2pXHfCjGlTebNY5hJNTXN1Sah+obr6p5VQg6qTnU0BCiUkmd0iwH1Eb5y4m5VxtCWRBpNgowva96O7KZuoqKpLLfWNoJGuZSjAAjXf5HJfuPDF8OtuXulS0UqT1pKlFZypBKsqUyTok+86WdNBwlxzfDjd3XZwifqKhDgdW6ahXqJBJnMkBIBiNO7ztJuFeB9/YRo7vxheE0j1Ghxx27A8uAnMhsB0mM+q80AD74TTh34uPV3RV3U8cbeI9wsm5w2w9Xrdp8RsAKu+8brqiypxBQ71btM4ASk6iZAgibXV4DrdcwfeNU4nMKmqpqk5Tr26Rjw8LVeuiqvSuvZlSnEejrUtKgEiYKTA3Jj577WZ6MtSf3EuAKgGiux1OaNSaVI0j+j+PcbCC4zgvFt1UqiHQEAqUuyhI6xsiRIlw9/cBZu8YKllHBXGRZUkNs3VUPOdbIGUJkmTtoD7rOOjbfq3kpFO7vrJ2+FiuObjF9cNcUXbVU6m01d2VKQpIkohBgiRuDrG1uscfRPsWM35wuieG33LywtcVZSFSk1V10rudKtDLYO487KP5urXJlSte8kwfdZD4Xv0qcBYbz3klambqp21lSgCopQEydRExZ4IvO7SNK+nHPV5P1Taq2aMAXcEZ0br6BNLiJet8XPdrCsO3WzeN6mnSG6JyqDAcQlxCVrzkGMqVFW28DnaKa/EqKbF7TgPaQ0UKUDGmkDz1+NpVx6blvBNBfNLU0tY/RF+mbcacS4UBYSVpBB0koTPlaA8ZLCL0F4NABam1dpStQrsAQBJjQyfC3lmPRsdizxvf4BdthtxQsPT4p3X5xOFz3O9ejjNQ+Gl9tNOM69xsPaNN9LMy8uM9fetK/QU2EMRr7IIdFLKAdwcwmCIB9o8rIVRej5lSX8iErSciUkJEjaRrEjSIn3WI196vrYRDiUpUowhBgTpqdNSQBqZPutnQ4a9sgkYbWVqWWQ/NNgn/AMK28D0+O2X70xEu5ampWagodIpmC43CpCnh2UrG4SUmRpIm1oKa97ovhlirum+KStpllaQ7SuB1BKZBGZMp0IIOuhtQpDmeoadd7MqGYqOkHYGT2t4iJ1tZ3o15XeG1OQSsUlbVNKTJAGYpXtsdFbW7ihmMjTG8LGr25iJb+C8P+MlSLz4vY2vFht4JqcR3m82lxICgFVLh7QGx1E93fY3w/YaLLtepqO2Ep1MxpEGI5HeJPdrYXi3he9EcXcZ0TVI6pLWI7zakpAIircEA7a/bZawVRC6qJCXylotqCgS3ukROvalUd3cRGhA6nMMgAXMAHiEpzOVbdzUBrFrQVCVpCtwncHWO/wA9R4WS7grhd92Lrqllsv1i3FqJQUBsTIGkaQDE955xZGv68nryeapGnCES2qDBBlUTO/6szr3jv5xBVM0mHw5LiS2zllLQCCYmMyfGDznNJ1sMbopNkauK68Q4wo1XoquqaWhdlktUDbnWVDZUvdQOUiTl7gNyNRZ84awzjinbN0XCV4Yo31BTnU9p1wiE6pQShvQ/pKWqT2lQbRdwiu6qvdaWje17MNNBSQihq1Nq3mITrGpJjWAo8rTTdWDblqlmnfvip69kgOLfvpRWFa96womdIy947wLAag3uLp13Lw6uelSzWX5UVt5vIAV1tbUKdSmNQMubKnv0EactBY7WjD1K+t4XK1U1a+2z1DailsFOhzT4H/JjlYK7cBNNJC6LF1/IS6hRJavNahJJmBJEQUz4gzsAVZ24q+haQzTX/WqddeBLjjaFKSmVdmUpHgZ18J1lSN9HZSidZ2pVZekphzEtPV3TiS9FFdG+hdM3rHVqkrAgmdQY7uxb0b6MlO7Q9FPBCFEpIuEPyRMAuLcHwP1crR9wTwng7FNdiHh5jK5W8Q3df93ocfTeMuGWFkyhROZsjrFEKSQQUzNrJMYfujDuEW8L4foU0N23dQehUlMgkhppLZCUgnUwOZmdzrrajL+MgMZ6FW4vxdQ2QdQo4ryK6qoa1ysqG1ULq3EpZfUlt4ltSMrqRotIkkJOygD3WJ3zVLdplo2SCCNJjXx9tgy8pRgDJOwUfh4bn4eFilYttynWrPIKQqCdQN/YfC3BMjs5eiufmam5cN3U92NV1LRh1XpFWurczuqclbk5ozGUiRokQByG9t3rdlFeKGU3gylxLL6KlsKUTldQQUKBG8HXXSw7S/pnQ3ErbSYAjYwJ5zqfnYvWr6wJypSYMjMdI3+fO15tyVTdYCyhLpJU7wvPC1S024pThqWSEIzEwkGB36n3pFqkYnpFNVFQFtLQ4HlpWlehmOc68t/O3oNWZmMU4XrArKyiscZWSsZYWggEmY319tqZdIW7lXTxLxVRNgZEXo68CiIV1nakHXTtzp79Jts4ZLaThrnMWgs3i+KhgggnXwtZHoQ3h1ON7/u2P5e60vZY3LbqRHn2zat6pClAaakRaaeiJeRoeM1FSAkfnCiqaeAmZOTrAP8Aq7bVc3NTPHgsbD3ZKph8VfcGaZMqCgqRG/dOvdraArzua9aa8K6jVV01MqlKW6NTjyR1wU44oiNxorbwtPrdM+pkoQ05AOpSgmfmO/7rQ5ijBtU5eOJquku6tXeDDjVTTZW1ysBKHCECIVrmGnfvbjorE6rtKtgkaE0mq646O9it+/1VKEXeta2Wmz21FmFEE8o1G312IN35h56sw8uqumuWaZhCEvuqSkhKXMqVLTGoBjw111t29g6/V0lLGHrwDj1FSKKhSOaKKXG3Ek5YGhSSPfbm+LrxDeV1XYpOGryZqG6QXe91tOpIStC215zMQDCgLW2hoWYGluiHo77vu80Cmuu76JmmpbtfqAnq86myQtJCZn9Md2xjwsTrr4vK+nmaFiueZYq3KNaA2qEpzNqBAg6dtPfzNlK6rixDdeIENoTStUzr1RSvuPvtpT1CnQvMAVDU5zHzDeu64b4oHkmoXQkNKYU2lVexAyuE5TK94Uo+U2cZbqZzBN/iDdjl4YTvq9LyafTeF3t0FY2HCZIKkpUSO+Cfj7K6ViQxXPJQfUdVlI8Da2fFFq5b3bvmtr7ypaVsXO+23/G2iXH0hWQQFEwZEePdraqF6EGsUvQ5wlUgzOgn4zbdwp14yFg4q20gKfvAmtTd/Fu7ACoNvqfpzBGym15fPWD7LXBQ6439IFZXEQqQdQoaztuNPHfvtRnB1eLpxxcF55oSzWUriiDyCkz9trxELzRET6qiR2hPf8/C1LGG2ka7wV/BX3ic3xXqDcF6ovq4LsvlBBRX0bFUkg7hxsK+2xs7TGvKzD4DXsm+ODWEqsKBLd2opSQZ1ZJZ/wDl2fRWYnfXS0IzdoTyfOK0VcgPxtolJ/SHkTtbRM6nTmDFsChO+nnYzUMpLvdo5UuIG2/lZDqEdcypsDVQI277OuvaQqkVAE94s11gJBbJgW06Z3o2WbVN1uijDnW07ayCStAkDUDSwwUAdVGxSlzJbKCJ6txaY7u0Y+yw5WAZzSddbXW7KkhQZGcnlOtulL7oB7uZsDm0kyfstuSkASTAg+6zhJdg6cpmYHut0lQ7xYIKjUEAedtz2TOg8TNiNTFGAuREjTcWOXes9YDqJI5WTQonzsdu+S9KQCPE8rSUURq7prTVPFrJk6xWXbadLZZ1gNESQozzgWyzWSXilw5Up3ilc2bPCWK06qAk9VE+evx9tpyvGpcUUpWoEN+rm1jTbmfnwtA3DXMeJd2vtylIpayCRr6gEHn3+73zZWOLWrMqBm0gTufHukk2jSgWUJ3EJRuhxtb/AFD6Z7J15HedRvZeRTXWACppaxMQTJHzrZsXOHF1YQ0palAH1Rsn3672cKaWqKc2Rck9kDWDp8+6zzuIcpU4BajDbF0iUpZQVQCN9Dv3cvC2ZrrkqTTJAzaa7bc+7QfCxb0R0ggDQDUlMiB3WxFDUKUrMggydYmfjrtysHNfdGLR0RlD9AntFgQkmTIO3f8AMWDF5Ui2VBqnbQAZCQkRIjadOY9lnPw9qcEXNelS9xBwnVX7d66VaGWKarVTqQ+fVVKSJGhB8510FmTTXah12pqKBisQhyoUpKHnw4ltkgZEDszI7UkmCI0EahbN+MLbfWk5lhcLqqvRLSFOBtCANDoE6bfb+HfGXFPEDtOaXq61VMh4qeQ806My1J/RGhEQRraQL5oQ3d7nXJS4lSIyqlUjaY8CfjaIcbU7VXdSqG/XVAXa86adY0WlAbJTO51g6c48LWDICLBQym17JEp8X1l4tNJ/jC228ji3ERn6rNHLvPlseRt6aYPv9T2BLsPbTF1U0g+smWk776wfjbyTwbWVa8R3fSde0000k+sFFLwSCpMxOhn4c+fpzhOrqXcIXY4HsocoWOtQEyD9EnQEARr9luW7R2dEwHqV0/Zpv4x/1KlXSzrRUdJlpwq7Kqi6hBE/ooB8tB8fK3r7RqAaQDB7PPvt479KRxI6QrbgUlUVN2iCJTGVGvjBi3sGw4UtJCY9UHfnY1NZtNGPBVKo3qpPajylAAkiLFVKCp7X3W5dqFFJHLvFiqnXEJKUq/D5i1ljtVUe26FdEiJ8gLEXCo+e9unaxaRChJHhvYDrA4JAI7577XWG6pSCxRq63Ml6Ui41DyNvMWkMVB5m0aUa8ldTEK2eSdPOypibE2MLsvFNLcGDHLzZUgK9ID6UpzcxEyPbbKxFrnSNDei0aHKIyT1TvqajtxPK3DL8rAzG0X3ljHjAFgU3DJsgp3VWIn3ZpsDQYs40u1bSX+HrFO0fXW4vLlPtVG1sY0kxdmFvNaAmjDbfoUtreUlBGY7WS7ybrKiiNQ00paGXW1KKY0AWOVki6L1xs+epvvDtE00vXr26oSgdwSCcx91ur8eWLvKEOEfTM6BW/wBIm1acljsrlZgYXatR1L/WNgg9/wBdjlx0LTqUlQSSvMfHUmzMxFe99XBdT16XbdL99vKcbbbomilHVSYUomMx3HI+wTaOqTjNxzu6sqzdHA2vvNgrAp89c2ylMaEDMAQCe8n7LZ9W12QOBFr9QEaUHKWt3Uz0mGKJ2iqUqQk5K+og930yj9tqj0oTjlF8XhfhaS6m8KigWholtK0trSpChoqDPO0kXhxb6Q9NQ1L2HuE1K+7UvrcqGKu8Wx6I6T2mx205gNDI77NTglgnG1Fdt8VPE/DTV217t7uVNPTsVCHEKbcSkqJKFL/SEQT77HwCmie+WO44jrZdQdr39yw8WfWwsa9hOUb/AKE2r3r04ZuuroqfDinKGtT1dQ44pa8gy5TBSkcvOyvhLBzHFzEt3Xji+6vRrvDHUp9HedbWtltCiFRnJBJ5kaxab6XDFw3/AFdJd18XUy5TuLCJKFDLJ5HL8zaS7lwNhe66tysu+6GGnGmW6dSRm1CQFa9/rm3pcEFHR0xDYxxSD6S4178QqpRxJiYxyUd4F4YXZddJXM4Lutq70VlAhLb9QFqB+kKlBSio5jCY9tk7itdl3LvK4aaqw8xU+l1Sy4HWyQtKm2TAyzMdwB2Fp1oXGGGqpKChtDVcpAS2jQJOWAAPOyJeqGOtw3VlLZUtQIUWsytkDz5WqYbC2kk40ozPPNWqhr5Y+G1xAVG7wpqa68VC76Blpljr0K6tKYhKgDI5xrv42kXgQxRVVxUxr70qWWaPD9G6QxWrZCilxbUrCDqAE6Wi/jtdd8U3Hiqdu6+0UjLFWwOrLCew0pLZUhKokAkGR4jutKPR4aSbmqnHHUFQuh5nOE7Bq8HUgeyfibU5mkVN3C1yt6nN6WzDc2Ut0CcJreld4VZbOq4rqkyO71rZVYTwTfNA/d9A/XOIWl0NtKrqrMolBI16yCNNjp7zZFcZqAymrXe7hQox9E2R7YBECzlwj6G84Gze7y19pC2nEntykgQdzztZa5t7FzfJVOHVXvlNkz+D2GMPVWCrvvJV13cqqeaKKhx53q1KKHFAA6EGNR4fU/FXNd1HTBbd23O6EJBKAtsqPfr1ZJ5+O1mR0fhcOBuF1S3XVfoN3XVWVy3n6teiPp1KJKjylennFlDjpxETcWAqaiwjVpqr5xktN3XMaZWYrDsBTySOQChB71JsMRUkcHFkaCbeF1pRR1NVMI2E2JtfkEqXnc12oubrqO66WjWmpTHVJyyhQVI9VPOOXdaCOJTQZepnJTkCVJ6s7n1ydeQ0Hd902t4UrMA8Irtw7W3s/eFZd1IyKmqfdLinHesGdKZ2QCohI5JAtBPEx9TlGl41Ocl8FCMoCBpt4mVH324btFAI8Sjc1uUFo0XS4Y7/ACN7M2azjqmrUBujpnFF5MAByCsHInXc93t7rM+pxzhtKJReSldv9RXagcpAnY2dBlTVR1rbeTsylInXbxER9dm+9dt2MghF20gJJgpbQNZnWRqPnwsSnAA9JSlLiPRKkPoz40wNefE1WG8RG7aihvq73WkN3kygsreQUrQB1mgJAXA3M+Vra3vX4E4c3K46mtuPDd1IV1qilVPSMFREZio5UzAAkkbDW3nZiZuj/MFcpmnbSEtZ0wgDQKBj42gPELLZrqUppm5CNwEj9I+I10Hv1t0uH5XxFoH1rn63M2UOJRHiS7RXzxExVe9LUekt1d919Sl4rDhdS5ULWFlR3BzAyJ331tH2IA5R1ZpojOgGFwAQrMIJ1/VTHlZ+VFGpL3pDaCVAAajSRHwMRPgJsxMWKDeJFBKsy+oaGgOgCBpGpI3119XkINtVrcoWc83KQ3hmrUhJAynRKZSDqCZBiO8d8DmABJnBLDN1Y+404Rwpe90Cso3KpdVW0tQ2FtOsMsOOrbcAjRakgbTrodYtGzq19cElwqbjKntEgEaHc9rUAAbiTAmYsT0DbiXe3Fa/sUuJJYuS4xToIQQEvVLyQklR3ORl0agRsYi0h1QyVGnEPg+ngXxarMLv07j9w1wFZdNRUJzpqKRSpKDyU4hX0ath2QowlYmTsO3XhutW2j0W7FuqyoDIQjs+ASdoB79eztFrNdIrhC5xc4fLu266cOYiulxVdchAyqW7Az04URs6kBOvZzpbJJCbVNwFiK5qm7aRF/lNPUoCWEVDrZQFAGE5iUiFa7SPVNjRPuUMjSykaguC4qZxAprmaBaSCCpMJEgc9SdRqdvdZcqG1oaLdO6QkIkMqJMKGqdDpukeG8WLUbbqR1zdQFoAkKSQqBlBgbzz79tDvY1U1BYabcqKhAQSD2gTmMiABPcBz9xFrBFxZMDYo1w6v8Yf4hXNXITm62oTSKSBMoehsiRzE5vHLvFrdrRKCgp30UdNdY9p28I+NKruFRRXpd9UppTQp6ttannAEJSEuAaSecJH2WulKVZXAqc2o5iI+drZttSFdJ2Kg94+jrdbLklJUgg+B228J+2xd1WZClRAUPW1Ea2P342UXzeLSQExVOZTO3aO3f8AdZJzAwcxCBokAkgc9u7X7LcW5uV5C7+N2aMO8Em9ZkcITmUjIRlOsab8vHf6rAOBKm0kEAA5coPn8zYZwkvBORJHaEK1PPX59+1gVhW68o1kAbfMeW9jM2QH7ps4wokLudl10GGqxpeYAA+tE+cE/O9T+krdzlHxCvRJVm6ynpnEZtCQltICtNDOVQ5bWuDitvPhyuAcEpbzgHXVJB193jas/SwoB+6egrQyT6fdCUkgaZkqWRr7R8O616gdaoBWVijc0BKrA8haXFhQykEgjus6OFT9cxxEw/8Am6qNPUvVzVO27nKMvWHJuNQIVZs1SShwlQgKM6fPhY9hi8V3PiO6r3RoaOtYfT/iLCvst1MgzRkeC5OI5ZAfFXyOA+IdWT1+KKchMaGocJJMbwPmLMe/7rvVDz9KcVtvKp2i7mQVHrFA/wAmD+t2fjNrH075cSpopaAQhRStM5nAYMq5SDoIjlavN7PIpHHqgXc3VEVzxcKnFJytJjNEESe1Pst5++SVkrY47XN9/Be0YFgmFVVBVYhijpMkPD0jy3Odxb62mia1Bcd4XhWt01ffLlI2pCgp1UqSnKkQd/0sx8svjYm1clSqoDL79XIfLS1wAgo7QStJk6aJ5aZhvFnQ2/eHUN3hU09C3SVVRUUbJQmSFhCgkmZ/SAjvmyXUUl7mnqLwVeBCfzcivbbS2EgAqAUnvka62tAVn+r71XLexX+1f2SRfzFfDlH9El3rT1YyreQABlJVuNO0I9o8bOK6+GLt5o61F4EBK3ULKXUqyoyjqzATqZ9Yffbu8KVd0X3TN+mvPMZKN7txDiFOFC5Ec5Ttt8TIeCGmy1eLYlKU1BAA2jlYLp6iKVrJLWN9r8lbdgnZyvwmqr8LdMHwcPSTJY53W9XXRR0/wHZvWgTS3teS0rWpZcSyCAQYygTz05zyEaWqdfVO7T1CW3QQpsFozO6FFJ38vhb0cW20qcrpnmSnbnprpr9VqFcX7pTc+Ob+ogILV61KgIjsLVnRp5K+efRYNM5xcwry/GqdsbGuamcVlAp3UqBKR7iFE/ba91y3gm9LloL2GiKykaqCJ5KQlX2/ttQ/NNMkfqrPPvA+61z+EFYq8uGmH6leVUUXUHUf4NSmx8Ej32PjDRw2u8UDBXem5ngvQ3ogXt+cOEXoJUmbrvSppgAdkqyu/W4r3WmxZzak92gtWLoSXoFXfiu5SrRt+lqwNozpWlR1/oJtZsnUAT7N7UIDdgV6oGV5WiZ0y/C3OYzz8SbYecbfA20qAdQd4taAVZbWoqRkOmlm5Wtlp5xIPP2mzhBHPly7rJF8NhC+sAM+FrdO6xVWoFwm8g5KmoRP6QX7CI/smwva2VudbBvdi8CowM7R029U/wC1YWCCd9J177aIcs7LdbBUQeUHzthIJ0277bAAGYRHntbmRAMDbYm07qBFlsE8jHhbqSVSCd9xytzrA37tLbEAkDlpvNpApIT9HnET8/PjY9duj0iRBkEediAgDNqNNTY5dqwXhJ37rTzJrJdzK/WPv/G2W7kc0n3Wyz3TWC8SuGbjTnEagCABlpX4IGolEEHv2+d7TNWqKtwNFRI2zcvt0tAWG7zcoMW01YkuhQbUAW9wSTO2w01HstMf57dddpypScq05FJWYJVyg+/vsOleNkOoYbJz4cqGKepW8tyG0JUopOpJMCB+zazk/dJdyZSmqBEiAEkcvmPKzJaqQ4hRQ7KUpHaBk7zy+fdZKbxjhqpvY3C3ftGutQvKGusjtDdIMxIPK1iSJrzclCikdGLBSR+6KiGiahUJmUhJM93L5kWw4lu5CYS69O8BP1HnqPqs0FKJVnC5AMgbwO/Tlr8zYq/ddK4lx0F/PJIyOqBPhvYZgaNFMVDr6pW/fMuwOPIaTUudWpSlQMqkgcgJkyc3x77DXVi+62WVOvKWgPytsFY1js6a+Fokq6d+7K95Z66lfVUIWSolSUoUqCSDuJUJAPP3FK2qDV4FT760BunUssoByjXM5EakQJB09Yd1q/CaDqrHFcVI9+cSsP1lSm53rurCqsR1aXM6YCjtPd6sD2DmLQrim/L0fxJUXYywKdxDC2lkkSsADLMetpPnJ8bFau9mnsQovBTqBSt5llPWnMnYASdUmQI8LHsaM3Tf2SsuqtSa1pouVWZSs5bABAT3x5chJEWGWAahSDydCmpSsP0FTT3nQNF9NKwFlSEGAodkzGw31MgTMjn6XYKraNzBNzIJX1irrpjBOgUWkk89tfqt58dH+pqGuLdw0bzjppampcpnW1yUOIWlSSkpiCkiAd+W/O+uDmBTYauphGiGqRlCf6KUCOXlblO0E3zY12HZqHR0n1KnPSiXl4/Jf7Sgmtu7Lz/waPZ+kPZ7SPYejI9Gb7wkfVbx46Tyqn9/Z1KEZkpcu5RE6ABCNTy7vhrb2DoVD0VsiNUJgT4WuxfxaP2LKqBaqk9qMKgjSOU2Ac0EnSwyicsCSLAL2Mkg8tLGYqz90VejXYaWDS4gpEqAPq26fMbmPZYgpQC5knXS16I2VGQXQ1U4UMlwaKSJEeVpBpa6np6emqHXApLiJ5kqhMmLRjejrhonktZiShWUJ1M5eQ5+VlK6rwpXqKju2+65+irKYEobcV1fWpUCAdd9Fewi1Wuyki6vYeCQQnrV4paStpNIytOdbaSkt5TlXMKHusVfxUvqlqpXTnEKQVAQpIUAef12Szct0PKbWb4qCpCWkoKHRI6uYOg5zrbharnuvNddJT19ct/tBlMqCATMzpAnvNqXDhdrY3V6zm9LJaXXdZeVRTtpSUhpK3I0AWSY25kWS74BN2rzkiFtqBGsQsEWWGqampUEIbCesJW4SZJUe82S79epG7pqiHm5CZgrAJII0Fucq3maS7BoFqU9mCxQTTjpYXmJ0cI1tqgcceSpto7LVv52rp0icc41wliJyiuDE9XRUztM051bKgkJmRm2ncD5m0b8IOKuPKnHrV0XpjW9Klpd31DikqfVlDraCoKO07c/K0ZsClr4WvDgAmmxCOAlpGqs47XLTUX0jMYTXr5bS2g/bY+HStl3T1SCPfasPDnE2Ir1vW5366+3Xi/W1KKg1VYpPWHKQND65Age602Yyx+xgFNJ6Rc1Ze4vRamW0XeQpSXExAP9KdANSQbVcJw1+GY5E1zgbg67clDEqhtXhbyxu1vvClPCtwOXvUsJebyNqUpRcKDrlgkJ2+YtJ9LT9Q5WIgarQZjXVCB9loP4S8XKi+cQUuFbywVelyVLLT9UymscGZbZQAYAGmoFpKvfF1dd96MPUVyrfYeaPXL6yEtqSpvKDodwVf5Jt6O6uhdJkzAn3ea4cUsjGZy0pcpHU+l1VCkkPOVAeASgTlHVg7897JLpW+u52G1rSvqHQjMmAlQUQD4HSzQq+INcm9q+7b5Zu+771TeouemdVVLW0pb9P19OSpCREwhJgbkwbJWJsf4xu/h7hGtprkoTeb17IpalxIUunaBWtJIkhXrpG9kyrhc/IHahMYJA3NZQJx8umob4rVT4K+t6unUshEgECCSTprFnB0bXUn02kcaKylm+W4WmR2bwKgI5+v8AsskcXL2r72xbebl6OMitYfXTkNNlKVBKzyJOvt/FG4YcQXOHwrr1auputi9rzpVMqdyD6QtOTIB7+7c2y5arjym50BW3Tw8GIW3Vna+jplU7RpqajKAJBTTpBP8ANM2N4ZaYbvNpZYaSudFBpIAPmNfbaD1dLVNKysowBTEsuT2qoiMvjlG+vusWa6ZtYXlqRgG7Gy6sKbCqtRUNd9Br8LKOmaHiTPt4IzqlxYWZd1I3DWhue+8K39h7EF0tXjQPXveNM+wpIWlaQ6ggxy7wdwROhiyXw/4BXTgriCjFVPfV4XjdV3U7jVxXRWkr/Nq3FS4UrJMiJy6A9okknW0d4R4w3pcVC29RXXQlu/sQVDjiHAs9SHE5uyQRpoN/OzoruPWKbrdX6Dd126kpALalDXQ/pWAcdw2MhkwJc3wVmOlxBrHcB1muGqmTH9WmswfebaWmVhtjMotvJUofSN6iO7XTwtV3G7VOKGDq31yMhUkFOoCffPzvZ6o40Ymv27Ki7ayhutLVUytl0tMZVpCo2M7iB7bR3j+oUigW8SsuEIKSkQE9vUkxofqje2LjNdDilSyWC+gtqtHDaWWjp3Mk6psNqSSs5gc7KTqTqdDttH3+8rUJEE6CTII57zy3tujcZbqHRmbh5kqSQqTy5H28traeCQe0ZSUn1j4fh7LTjZZFDrjVIl9tKfuWuQoEDqHASE6zB7ojl7x42gG92SaylUpMZWdcvaJUSZ21H26eEWKrEFxhxnMdUEQVEdoiB5/javd9IWmsYSorR2FE+Gp2EjujWN/YN3DOYWLiVrghFF0oUkNpSFBwkJEAzO2/h8BvrrGGOlrRi11lBStammSsAaEwdgdBvpHOPG0uZUrpgrPokkgJRG52mNdvgZtEeMW3TjKqWQ4AlDSQQDlCUpA5aJ2I35DlvsHTdZCbF6OJbZP0hZ7JAA1JG5AkxzOg8N51uv8Ak9bmDGAMW36tkIevC96dhJVEltllWUk/0nlabTamda2W1NqaIaSrdISSSBuVZZIEAae07Wv30PaQ3bwlu9hQCFVDLj7i51Us1bm+m4Qpsd+sd1oF2lkiw2urAMLCHmyQUS4kwdRodfGeVqp9Jvhqzw54rDE92UrL1z49LlWadZhLVenWoQCAfXKg8mE7rdEQExaZbwdC1EzJmN4get7J8t+6zQ6XuFXcTcFnL5pkZ38MVbN6pSUEwwDkf130QrMI5pFiRDqgu3VWbhNO25Nw3vUXRUlaSumfT9ESYPqk5DP83keWku9FTeVO0t6sYS4sIBXU0WUuo2k5VeajuqdNANLM+61t3tRsmraS2p1GYLCMzalaCCeR3nxE67BXpnb4ZfTTIpadwNKygLz5QmRBG+kQRz1jyui9k2iCqH6CvqfS6ZyuqS0QoOlwuBtQgFLjQEpE7qKeciTFrq4bv678S3DRX1djqlUz7I3gKSoaFKh3gg6bcxItUs3mwwkpYoKU1VIgLMSMunJR12PLxtM/R0vsXpdV+trKWXmaxCl04SE5QtHrkbjMQR/ia2pvZkN+qtMOZvsWsYtJZxPeCFOEFawqIH6SQZ8N+7nZBzBGiVk6gQRPlr87juNnRj9PV4lWsZh1zLbkDnAI8+Xh9dmuuEmAonYAJO4Mft/Zbi6kZZ3DxXeUbs1Mw+ASc9GclSdM05Uq2kx4cz8bC1941l71a6u8Hg7UFIQpaWktzAyjRIA2HIeNg3FIUsrBBlOae75j2WDKFBawpSZSTqDpM6aH7rRZbdJ26JXsjrrqrmcpUV06xvGuUxA91q89KKkTV4dwXe+XMlVM7TkkEHRDZHvynv28dbIOIS6ytqZC5SfIjbu7v2WgPjvT+kcGLjrToqkvIs5gDmAh1Me2Ba1SnLM0qjXNzQOCplVJPWEHugRrNgxplI0Njd6NlqrcB5KVpPKTYonXeYG2vO3ZA3C4kizl6mYFq0XxhG473ypJr7tp6jbmtpJ+s/OtohZoqd+pU5WNhbP53ep3AodnKttcyPZZ79Ha9Te3BjCFWo5lN0KabWNeqWpvv19X42Y1TedFT3XfVCXIrxeXpNMFIJQopJBBI2kE287q3NgrGl5sAXL37srQ1WL9nMRp6KMySOEBAaLk2eSdPBN69qKtfob1apFsG7aa+EOUgQJUC4ZOo0IOdMa+7e2XWwKy6rqzJIFVdVfQK7UyUkqTodZ5662yldrqWkcu5q8WPR6hDDjpUlUh1uCI0207+Vlqmew/TvMEViurpq+oqEJDBksupIKPAgn6/baNfTj1x5rnx2B7TE/xGX7BTOvG7XmrtuC8E1jz00a3lhxWiwhaVlue4SYHhaUcB5HW7wXBLaqmYjlB5WZ13XThymyprcQ1tS23nDTZp4ShK0qSpIknQ5gfYLPPhsEN0NcGiVoS8AlSgJICd/OLVJqmKeePhuva/wBy6Wg7O4rgfZvE3YjTuiDuDbMCL2eb2v7U4iNc2bMcwiSY9us9w77U36TV1+hcQ7xcQCG6xmlqxrz6vqjp5tm1zykOKJbGh1EmTzm1WulxdimsRXVeOQlNZdjrIHMFpzNr7HLbmEvy1FuoXlOMMzU1+irkn+TWnxCvs+21tujhW+lcLWGzr6HWPsDTQSoL/wDmH4d1qkt7LSD6yftn7LWX6KVcHsNX5dsyaatbfAnX6Rsgx/V22cWbemJ6ELCwd1qkDqCr2dDS8DScRL2u1SpRW3UspjQFTbqCPPRSja4ZAiQQZ1HzFqI9G28vzXxquKVFKKsv0q5VoczKwPPtBPtte1Wo755i2LRuzMWzWts+65JG06+FtSP1h52wnv5TbRVOoIPj3Wvg6qiVhVrI5d5sTvFGdjNpIMzY0dBor77BuJC2ynXaNbHjNnIUgzCyZ14wiqo3Z0zrbme9JP1pFhFKMT3zz+FucQgtU3WhI+gebXJ0AAUJ+E25LhOilEa20m6rMcLIYE6gnvnW3GcTmnntv8LBhc7qGnhbEqMSI0E+yxUM6oQq/nQCPkW6ChpqNbAFQOpUIJ+FtpXrPMDU+y0goowlQkE6a+dlC7jLwkjX3RZKSo6GARpG9j12KIeJTpJs6ScPaOuRRnnbLdJKgkAEAAaDX77ZZXTWXhbdri3L+aUyuCGyo6ZgO0efz8NJIXeLLDbbaVkPR21b7A6A8rRlR1LzNaH2nUgraCIkkyVKGUiNNeW/fzsp0zxqKxBrKvI2lY+iT+luMo857rU4ZMr7I8rczLqTaOvolt1lbV1zdIyyz1HWqchPq6mSdY8zaB8O3eqsxrR3TTOsritQjrUAQtKTOcEAnUAK1+NnY5iC8cR4wXddwMoqaajowhymebBaBKhm7J35awSNdN7N/F1NW3deLmJ28tFWFYJaSjqw0oI/RA5AjlO43sWSoDpA3ohMgyszKzEhBKnNM24I++Dt57WSq6trcz1BRONB4pzoKjrl1Hwi0UYI4xVt+Pt3biN1LbyZJdQkAPt7KlP6wGun6u1nJfOJaGivBquupaqlbOqxlJC06SNJnX6jruLXDM0i6piI31SdjXiAtNV+a1Ib61tvt5AScuXnIkGdNOXfNka58T0Tl3MCqgOQ9SlxWhDa/UJ2kAiJOutmjiKpVet9VV6BsvGoBjrD20pPlG23fqLJ2d0NBzqysICSlREKzcwTtMz7I2tTdISbq01uiXroQxUVgNQouq65GUwDnSdNZ7tDPd3b2WMUruChq2mXHkUtQlCkpUEyTt2grfadDpBI5gWbnW0dNc7NayEekhZbWQowgkRvuNNfk2Qr2fXUVcVD7LqpSlK0qJIGaYgeEbSLRzaKWXVPjghXIZ4w4WQ3Cgu8EtkgRHaOw8ddNO7vt6F4eb9Gu2kp4AKGkt9kd2nz5284eFFS5d/FDDAcKghV6UuZIVIyZ093IET7R429I7vTBShSgSlWsDc/Vbj+0X8I32LuOzGsT/aqW9KVT6+NdSW2VLQ05d5WYBygpRrHLWB4+OlvX6gn0Jk/8mn26C3kd0mAtzjFfqzORpF2mQdDIQBPd2iPdy1t643WqbvpjG7SP9UW0oT/AJNH7Fi1X8ak9qHVJG5B7vfYBZA31sOvv29tgHISZkxMTY7FUk3RR4nKSmbJzxIUY0jWZ5WUHzoSfnSyc8SlRzewWusVJ26CdcKGwsqGh2Nniy1d95BKK+maeQNUpcQFAGzCvReSgdUjTKgjQjfX32b12cVr2oWWqdbFK8tpCQXlAyswJJAIAnwFsnFIpnvY6LktnCnRhjw9S/U3Pdd3qQugoWmS6CVZBE91lS5alp6nWhKu0hZQrlBtDlZxara6EmnpUACAoA6mZ79LAo4sX5Tkv0yKVGZARAbnQEwddZkm1SaKeSMDmr4MYbYKa72oDWXVWU2cpLzDiJHKUkTarRwe3d7rdaq86hYYW084CISFIUCRry01s/Rxcxg8gtLqabt6QhpPztZm4mvCpqrsru0pIcp3MyQN+ye7zsfDWvpY3sfzQJWZyHHku+MfCrEHFPFLd/4QraStusUYpHlIcQoB1ClHQ5t+0LRlhDgBi1jHlW3TPH0lmiqaVZW0W2/pG8hKVmUqIzgwPG0o4dxFeN00C7suyscpqUqkNNGE6gAkH2WUrprKt6822zVOlS1GVZzJlPx1tWjNVAzhtItyVl8EM13OCiGj4PYhw9fFLSP3i0XbgvIvVAiQtKwlcJHv+dLSPj55arvuZwCervajVtzKo29vxshKvK9r6vm9GRTOl9KkpcyZipRSVJ133AHu0ssXtSX7eNzXfdzGH7yW+iop1rcLUJCUlKlGTB2Pd3W53G2SukZNIdRpotPD+CxjmMUgUqmafpB3IzRgluouuoQZOuraDpO+tpRD6aunrUUTOQu9TCBzIU6kn/QtGqsP34vjLhXETFIVUNNTOMPL6xCcilNA6yRoIM+Y77PbD96rp3GEKbLiKpx5sOZkgI6t11UHXmFC0KGbJAL9VmV8YfJp0UY49uTE7l7GtorsqKhTeKMN141T/JoZyOK1I9WPPzs/cQ3Vedbg6iu/0FSaikvpmpyFYHY9KdVMg7ZRNgccX3ebCbvrbuuxp2nqH2DUuKey5Mh7PI95seqscXS+qtTV1DaAtDaU5PpNQpzu0mFTaD8V4TiCfYmiw5zg1zRf9gq48c77ocP40vh10FxFTXOrQWwCSM0789xvaJavErl00D6G6BVQiuv9ZAK8vVhdGw5J0O8eyzv4+N1V5X24m5VuOMtvFRKwBIgbj53tHl80b9Rc/UCnK3G72oVKAiU5qLKdfNs23sOmdUQCRw3VaqiEMhjHJN7EmMK6mp6g0LSWXClTiQuFCSkbGO8d3Pw1E4OYgq8bv3jSXspCXKC6Xa5p1tIT1jyVJiQf0YOw8LBXzhervBxWSiqpabCPo3gEzrp6p+r77BdHfCl8IxGhbjnotPX0FQyVDtLKCYIhQiZSRPf7raVaXMonvYbEBU6YB1U1rtiVPV9XdSXZgrCdZSZh6RebNQ8oqJ+kUmCROgtvEfZUpaXZIMxsQO/3WO35d6a3hjd7KnHUm7npQ4giQppYA5fZ+LcxDdtSpS3F33XLBEwVpg+4W82p3mZxLzrcrspGcNtmhKNwVjinexOidJEzp8eXx9oGNbydcu91txOdGUEBtICkKE6953+Asi4fu5fpCFLqq4hs5kS+oQZ10Edw+dLHsY1IXRpSpSjkbWAE/pAka5vCTboKSIErPnkPDsmKxe9Wq8wwa10oQVJSMyRCUggIOmomDOlh3r3qnsyC5lMGT7IgHzjf77EHADfJClpCQglPtTqD8ffbhyG5SN0yqVDSOe237fb1ccLSFiCZ2yEcvGphSlVATAklKoJOnt5ePK0VY0pvRq1p9luAunTCZ1KjMnb4+3TW0lGAo9swdEjeToPn9sR/xBbXloSkmAlSSEzl0VB1IA2ju3Pja/TNDXWCqVDi5uqSKYIRTNNtqXIAA1iCecxpv3xaK8XpbcxZWGVZVJbhSVFMSAd4EwI1E6AgRNpbaC2mmUqHZWghJExsNTtJ1+ItE+MwhGKKhIyolCJWI0ie/wACd9NQSYNr0mypN3SE+y51wT1OohUGJkJ2EpMiI8pPrRpfrottOM8PrvYWgoyUjjYE/qs0qyI5dpS9DO3stQ9m6a687xZu2ha6yqqXfR2G0albqyUoGpGpKkADf7PSXhthFOEq2tw22nOi7r2q6EGCkqSGqhE+5lPht4Wz5KhrJWxk6lW2RF8bnDknu3lKMxKVSnbnO2u/L7h4vJFFSX7hp26a5rNT19Kad4RuFIjU78z7hrZnobAIyNJI1SZERp+wbeXcXlcjgVdlOFcklJE8gY/aYtqQ6rMkBC8+LvuuswXiO/MHVTSi7dFaWCh3TM1PYUANDKY18/GyyvE91stZK5pxKAoSuBCZHPkNeX2nWTelTge7bvxvdWMX2Cilv9tV3VjidCl5JKm1GO8kiJ+20L1F1ihWmlvta1IACWnwMqSIntDaRA92tji4TCydV3VFDWsKFE4hQcQQVkdsZjvpJOo5eFl3o6X5U3TxONzuvlabxYfoHpOuZCS6hRnUx1agPPxtFfoNTh50Xjd7yXqZRUVBObKRm/FXvEQLPHhverN3cWcP3wiE09bUtZikwJWOr+BOvny516gE2JVmAgXarFcT0lm86N5wlPW06kAd+VX+1ZlPFWbMowQIgnaDykeA12tIfFBkrauyoIByqdQTrzyn7PrtH5QYOVa9Dy0BMe/Y/AW43EfRqXLt8KOekaij2h0ScwVrAOvKdtTp9duAidVBUco1JJjSxh1AAgqlW5B3n36+f7bYGwkhPaSTAIPu5+B+Tau1yuEIuhEFIiNcs8z4R7/j7Yb4s0iXOCWJKZKSv833sVJA1P8AwhIPwUfmbTSEdpPZSCTJnl86/O0b43uo1eCeJF2aKUls1UK1iWgv60+eljRPtID4qtUMzRkeBVAsQtqRWuKUFSo5jmG3h8+FkoA7z7LLeJGoeQsqTqjSOcH8bIkFJBPfzt28RuwLgJBZxV+ehten5w4LMUxhRu68qmnAn+i7/wDM28rTdWXRda3XJuynIX2iepTOsmZIm1aegTeBfwxiu59xS11NUwDqOtbUn/5VrTvNKCkkAKBQkmBr3e6LcXiQDKp4PX716Fg80gpGOjcRpbQ22TVXct2IKz+bqfKOZZGp18Pn4WKuXZQAZjQU0H1oaGnws4HmOrVlUlRgmdADoT9fd5+0s4ypCcqyRsde7v8AL7LUxlWiamo5vd5lIqrqoEkkUVOI0/kQbc+iMso+gZQjTtpQAAdI1gd/2WUltKgoAgJMZQfD7tbcOtlQUMoRJAByyZ/af28n9EbIT5ppG5XvJHtKTPRyXAHIMr3Gmh5j591oD6WtyBWHLkvGFK9Gr3KaUpMw80Z8v5ID5NrE9WkKCtRzB3B5g+Fot6Td0KvHhbeVYAesu+ppqtEHaHAkn/JWq1uhlDalh8Vl18OelePBUKYJQ6I3UCn3iOdp16JVaE4jvy7CqOvom3wkzCi2uPqcNoNqgGa50JEBDpI8BOlpU6Mdb6FxZp6P/wAdpqin79k5/wCxbrq4ZqZ/sXF4e7JVM9qupgG8U3JjvDt7Z4FLedOtROpyh1M+Wk/Xb0YXvChzNvNLqnEEFsAKBlPfOmoPd4/jb0euavTety0F5pgprKVl8Qd86QqfjbmaB17hdRiDbWKNyQCASPG2jlBkmYNtaDU8+Q22tpWp08Z87agWUVokxCYtqAIEnXe2KOmnsm3BIHq8x32I1RKQMSU+emfQkarbUB5xpZJbeDraHUpHaAV8Js5L0bzNZhrpZoXeR6G23mnqiWx5JJH2W0oDdqzZm2cjiViNSCPC2ypKeW/cLBBYnfWZ3tpThEGQTy1tYQDohS4Z0Gs89jbhKiCMuwjSwZcH6XLu+62JWSddOQ8PO0goFGMwkTp42ULsXLoJkHyslBzUz5WP3av6UKJ1mDPdFnSTlzeHxNssW68nULV74tln0SXhtToPXdY0ohfVgRP6MqmPh7fKyXeVa7T3goMuKBUSEgaCBvEnkNB75seUENVBTmSpwpSJzEn9LQDwHP4ibId6damrJbbJKtQidNRufned5tktN3qy7RicnAJxtWLLycUrMV0kjs5jOYe/eD5+dnFx9uWju+7rvvJpx/r6ypcC0qXmSOzrHdsNB8bFejtRuN39fThH0IpkJKiOalTGuvL6rKXSVqYobgQCUy68pQAMyAnUcudqD3u+UQ0bf9loMa00JcVCVCpbVWw4nrNFIKjEAeHj+Glnqq8iytLaytKSAtIByhWg0PMgfZ52YtK6464GUrCgQUmRIHMfPs81htFZUFOVeUE6gmJPh378tdfYdkrItdH6x8vqFRHbcGpSSlMjYa+Z+dbFVagFvtIjMmTAEA8vdr+2wjblGzSracec65LeVCi2IknQ6mTsLEVKWlSlLmQkJgGZMQPv05AeyF1MNsjB6xF2uFLiy2p0HKRA21jx2jvGlizbFOloFmrK1pKQOwYOoMidiDPhEWPs1LaLhvFpSSHA6zkSO4EgxZz8EsM3NizEVbTX3SLep26PrUtpcUgZgtI1ymdj3kHeyc7K25Uo2Z3BoRPBFIv98bDBRlUE3pTKCkuT/hEzruDqTB318LeltMgh/ID+kQQOU+Pz9lqrXXw5wjcdS3eF0XP1TzC0utqLy1HONREqOxE2kimxzihVQZvVZClyVBKRrz5R5+dubxaldXOaWG1l1mDVDaBrmvF7qEOlO4lni/fCCs5XGrtIREBQhIJnTvn2cufrhc6j+baU8yyief6It4xdIW866u4mJqa2q9IcUulSok6qAQIHn3edvZi5HM910ahAmnbJA8UC11sZjgY08gseWQS1EjxzKPq1kaa/CwK+yPDumws6QBPhYBwyNRvprYsaDIilQowRzGnz77JdSofZFlKonKTuIiyVUKGbtEiNNza7HoFSduk+9FH0J9IB9Q/VaH0OkpCZ5ZhrytKOI69NFQKWpAVK0ogwNyBaJpUD9GI0J2+fG1eqOoV/D9ijrjjiDmJmREkbg8x872EZecU0AFk6CIOhPIfX7rAtMuKQHTolYgGdNO6y87cKaO5qS9zeN3uipUUejNOldQ1A3WkiAN9idxaoXAbrVbZJqHMqIK83MHNt+G1g71Djt21mUwOocOsmOxY8KZBKiZ1HcDHd9tua1DblBVtoVlHUrBPsPd+2wy5TI0Ra5VGFxqkBKiSddRZyXMkG9qdSOwEqLhBM6DcT77Ni4FthqM2b6NvXkZB91lhl1xkpfaUQpMwZ1B2sFzkVguEdYeboMcX08IQkttKnuGdzX6rOmqxDR093s19O6hTKaaVpSJg5Uj7BaMb1cUq+nHnSVKcpkZleIJBn3/XZSSo/uSeEwr0ZXjECYtzOP/wQcOqu4fEDIQVJVfjy7aNV2oqaR99wqUiUqCU5XGVJIM68haOOH2OcQO3NedCquV1dNe1U00QJUEgg7kTzPzpYTEC1KF0OpgRUslR9ih93us1+Hqsn7oqdZVKb5eIBO8pQfttzrZXmmcLq66BjJBony7X1NSf4y+tQmYUr26Dlbundhh0hBAzIOh2Otk1pZUCEBJMkbGAdLGqdQFLUiTHYMEaDtR7d7ZMt3DVaENgdFGXEZSRfLq3ARCmygbg9n5+NmXeQebVWF9stn0q6nnNNRJdRKeXd7/KXRxNKjfRKQAktJ3Ogg7/tsyL6INwVT0Bw5buWVJUNYqVgb92a3pmDEChYuPxJpNU8pRqmb+XeVQi5KtppbnVlYcUmDmCwD3ExPybJfBtdVR4loKascPWNOVbAAIMJzr2jlI+NtEVbdYtYp1LBZSVSjTKFGD3bE/eeRXhfWFzGTK0tZf8AdB9s6yM2s+z77adTI19JIzwP3LOiic2oY89QpyqvpMBV7UdluoqQRvGqo+o791m3f8GmSpKUwUzmJ302Onl3c/Oy+hzrcJ38wFA9XWOykiIlJ+8e/wArNS9KlK7vYXCXCppBEnaANfd7rebUrbSH2rs5z6KI3W6tFQ2nrNjty7jYXFwzXUah0pAMJUBoSJ2Husm0a0+mSkZioDbYGeYO4so40dyYfnLmUmSMoiBy8u+3U0DfSsVizE5CUw3D/u5ToQhQJQ0mI2OQb/jbhxwZkyAqQBMxtrz3ttlxJxBdjgXJeYAhQkEAKTy/oi23EJQokmDPMyd+cW6dgsAsflqgCnXUgFWpJJjfv57/AGWaWPWA7dVKEAqUaspIGkghR17tfrHfZ2qQqE7CBOqTPz42RsSUSKqjTCE5UVkqG/rJjfxJT7rW4jZwQJACEzq95ulr6WnbXGVsKI31I7+8CJ8/GbRBjwut4jeTl0KUKzQZyiUjQGRsfCO7WZGv2pW5itSAErS0cskGAABt7bMHiTTg37P6RYQoJTmCoUV65gI5GQD+F6T5qpt3Uj9Dy7MNXj0h8HKxO60mgozVXl/GHEtoRUU7DjzGadRDrTZiYMATrFr+Xayz+6m+nm32nmTiBtaVtrBSQ7lnWI/+skRPmbecfR/pk12LKN89hyjo6hYSoxmGXIN9566ZA5C12sC4r69V4UK8qFU9RRVSSlPqj6BGx2/k9R8nnK2icaptWHbC1vrWvSTtDDERupNvCn6jKttPbCpUVKJE+E7Hf53WMPPKXdw7RzJUsKJAGszt36jfxsXv2lcNCpxBSmAFT7eWvj4fCwWGHj1DjZUQpLytJ7wNdu/50m29QyZgsyvj4ZTe6QeEjjDhdezLDWaru1k19MOZLfaIka6p5c4i1U7sq3LzuxqqCkuO5EoWh0DtEAjyG3LbytetS21NkKKS2qJS4AQQRJBHzobUkxPhRzh7ja+MLk5EUb3W0bhRJXTOdtoH9Y5YB/nJMaW0gs4bJBXedBTuOU9XdymAFlLiurKg33jTUD74sj1NDeFLW0lZccuXe2s1LLyFgllcgxBM5ZEx4bWdZfpK/M31jPpQ1Q2DmkjeCdwZ+dLNS88NKonlVLJcpQogK6kwnwMeI0iLQlZnCLG7IblW9v6828R4Fua/WlJ/jIaeBCtDmQZ95FmaGFJSUhtWih4ieZ7uX191uuFV8N31webu5SSp+6KlLLsqiBnkER4KPzpY66wgnY5YEpI1HPzjTa3C41+LqfqXfYD6dLp1Se41mTkSNBGpMT5d2szt9tuAyUnWZMmeUjTfx9llJVIe3CSZOoA85+T3eFsTTEEtqhOYEiSd57uX4WzmyLXdHdJxZUDIbAIGxE8z8+zbvaFfdnpd9Yqud1oFN43OCADqewps+XL8LPzqiZGVOcHU6zPL262bdY0afiFRkgBNbdbrRzfzFgwOWyp/CxBJY3QZItF5o4oZLRaJC1LSVoJKYBPzPn77NxJGbVVn9xRu1d2X7et3kJApryfYKeYCVqSNp7uZswCf2W7+ldmiBXm1U3JKQrW9AO8AnFmKro1ipu9ip74LbuWY/wCltdtbOdsEAGQRyjfX4W8oOH/EC+OHlfUXtcV53jQ1b7PUF2iqAyotkglJJSrSUpO3Kz1qekvxEqIDuJ8TqRBgG/XUkyf5iU/C2LiOES1dQZWHRdBheNxUdMInjUL0ZqaZecrKYVOoV89/zrYo+z1P0q8qQdyRlBM6a281qjjljSqWXHryvN0zu7fFYr/5osj1vEi+a8RVUtO9rIL7jz0f5birV24BLzcrL+0sPJi9K6m9LjpUhT980NPpBLtUhIHvOm1kqrx1gGjBRV41uFiJSQu8mU7afrd/z3eddXje713GxTUeH2Wb0SuXqpSGFtqTroGy1I5bqOtks40xBMtv07es9ikZT/ZsVvZ883ID+0gGzF6IP8W+FTOVTvEK4cwHaAr21R7EmbMbitxS4W39gC/7gocaUVTVVt2PN07LKFrK3chKEiE7zG+mu9qSjGWKAjIi+qhABnsKy6+zzsEvFOJF6qv6v10/4Qv77WIsDbG8Pzbft0VaXtC6RhZl3Ra9EFusWClQmFQoa7WeHBiuRd/FnDNToEuV7TRk/wDGdg7f0rMd556ocLr7q3Fq3UtRJPtNlK4rxXdF+3Ve6FdqjqGXxpEFC5+y21KzNEWeCwYX5Zmv8QvRl2mzIUpJ1iJI0B9nzva8HBa9k3xwsw7UdaVFqjFIe+WiW9f8m1NzRIeGdKwpJSCFJ/VOoPx8tbWk6NNYp/hwqiXvRXg+2ANgFZXPrWq3EYfJ+OLV3mJRfiQ5SvIMqnWYNtLgcjp9VtFYBnuPLvtydBz8TbeBsueIWEjnbme1qr8bbMTpJibBklMK0sRpUCg6lPWNGRAPtmzESeprK6njLlfzATAhQB+ubP4zy7M2Yt8oNLiF5GwfYQsafqlQJ+ItdpzrZVKlul1yVifLaLYVkaj9I6b/AD8mxcqhREdqefv3trrCVHsnWLXxoFnEnmjAXJiec22FTIMjzsXDnrEmQD99s6w5YEa8+QswSRlLvaidjzEc7HbuUgLCpBnSLJWeCTpsSZP22O3e4S5M+OtpJJxAtRqhH+SbZYn1x5qM+f4WyySXiS2lDjqSMkJSiQeayNdNPOT32bV89ioAUtJlSiDlMefnqZjus9bieaFS4l65kVRLSA3mlWQhGg7Oo3jXuszb4DD98K6vI2rPA6w5QCTPMbaxMfVbKZ89WnD0EUoL7vC6kvC76xxsrSnMAqBKTObTfmfYLFb6xJfF/FH50vN+oDJPVocdUoJB3jlyG1lP8zinu12vdeS6cijCQFIygcvhr+2ydh6jYrq9TdWhZSlJ7KSBPhrpHL2nytYs0nNZBu4DLdAXcUJcPWJBURKSmBHn7PdB2sspUlQhNQ2kISomRr5zGtkyiSyxe/U3gkFpPWZkk6EAKI235fDWxR59wghajmHLkO/Qfhy3tIi6QR1NcXCWkKJSmUwNhvrt3A+dh/zfelQhVSxTOLGhzhJJ0HMgfPebEMiBd6nQwRmMJ56zrHICRA79POy3g+qfZq0s+kjq32l9j1j2QSDB8vjaFuilyQlFdlSLkvVyqpV07qA0ELcB1k6juO087PzoxIqFcW7ougtB9m+HBRVLZKpyFYnLB0Mgcvusg1bF7V1G5RShCm20obWYSElPdAn36wNPGTOiJdVXcfH7CdRWLaLabwZUoIJJACpI2O8D7+dqWKPMdHI4G1gdVcw4f5VHpfUadVde8uBV2Xa+Gaq7rwoXCgKLTxKCUk6HUAxvZEPBmnrsU4cw3czvob181RpA68VLShWWQTp4G1sMfuUeMMRm87vzqZDKG0lSCNt9PbaJeJiavA1fhTFtIgKdu29kvpaOgchCiUz4gEc97eN0ONVLsVFO2YvZcgG+hXqMppZ8NzGJrJSOmxXnp00uHF58LOOtVg68apitfpDQvBxlKihaXGUqETqNyCDv9frTh9Q/M1CQJ/izXOf0Rbyn6fPEJfE3j7W4rXdqbvSGqNgMT1qsqGgJJIABOvlb1Sw2qLhu+Qr/AIKzv3dWN7evE5omleZBrmvcHb80rOrB0Hw7rAqI8QbcrcMxynvsCtZJImLSYmeNVzUq7BSO6dR3WR6lRCoAgp+FlF5aYM6ajwNkuqIGYga2uxhUX7pCxBT+lURAIEOIVKjpoQbRewlCoziARvl8tfn42kbE9Z6PSoBbSoLeQgA8gSNfnwtHtKl1pwOlvNI0O8j9tqlYQ0i60MNBc02R2nIypZSCDnAzE66/IseYpnxUopUlBW4pOX6UACeXcB8PYLEmCtaisgI2AHKPbY3ROBpZAeBgnSRz8rUXPFt1qta66U7/ALjvbDV4uXZeHVOOJShYUy4HEFKkhSSCPP50NkqobfqmXUobVJbUI8YNjoq6hxtTDDKio7QkmPnWwoum9HmVLZpKrMpBBKWVHceAsDjR8yEcMdbVIuHrveQw0lTZbBZbk+Q/H6rOu5MP0d4vPU9ffjF2IbZK0uOtlYcVIhIA2Pn3bGyPcGG77fpqd1dNXn6BAzejq1MD8bGXVXbd7jiKi86NlbZKVperG0KBmCCFGQZnysMyNdoCpta4N10SJeim27zbGeCWVAEaSAr8RYdqHcNVDajJ6l0TzGhHnZIv3EFx0V8s1Dt/XV1DNO6l1xuuZWEaoIByqJk6kJ0JgwDBNi/75HC9rDlauo4j3IzVhp5TdIp3MtStcoIB5mOXO2PjEL5YBkBOvRW6KVkchzuAThv50quy6HglY+kpSf8AKGtm3ggOKvTE1OJBF55xoRllCfbt929nVfxQrDVx1ly1CaxippqR9DoQUyhQBB1PL7bM6kvG9rtvm+zdxp1OOVKFLKyAIyR9lsKmpDJC5p/bVX5pBnBTzS28gE5FFESZ3H7ZscoqF6opK5xS8pbZS4EjX/CJH9q0VY5xRxDwi6husfYQH0B1tLSWlHL3EHXaO+zeuTjlflyOV1ZiN9dbSuUqmgyyphKs8pIMBU8rF/c9PJHnZYqp8rRRPyO0Tox9QuOXxTuLGqm0gSNTr+yzPxV/uXdV55cpKKKncCjpIRVoP9r3HayVfXHq6bzvJupqsPVnUJR1eQVLaFlc6bpIjU9/L2iO4ho8Y3BU11NRO0jNRd1ewGy6l1WZtxhU5gACNZ27tbdTQ0UlPTtZILWWRU1UU8riw3WYpx8q47/TRJCli8GE0sAiIXBG/KJ+RZJ4bnqr+eedbCXDfhyJTqUpISVGZ09b3TZxpbulDjCC31mZkq+kWkGezrJCvqskYVdpKLEV5uOtoZUa9KkLXUpTEtoMyUQR47SI1tpcBvCcLclR4zjINdLqX6eqQ3dmImHHEIPpKFJzEJ9ZA2nQ/fZrutuVd0MJp1NqX1CElOdI7UCRvy+02Wqq+8DUzF401Re10JZfeU4kVF4Ik5mwkwRlnnsLI6OJOALruulo1YluGmIQEIWmtJUY7pXB0EzbiY8PlDyWtK6h1REWi7h5pHpKGuRXgrQEz6xMnffblHdNnDiW6Xau5FsqWghSR3g/HbfTzs36jitgd1YWcZUzrZleZK1LTCfW1SO4TpyHKxm9OMNwVFG1dFFfy1dYEoSnqVJCjoeQ1BmNfG23S08zXA5Ss2aWANIzBNR2lDV1yoIW4hJQlwIkpT1itJ3Hl42Cdpql45EUdQvcwhCjzHh9vusWvjE1Td13LcaraphDxc0bWoTBBOgI/W0nx8bN6uxpWIpi+E3jVJCSUtoK3FrgbITnkkk7C2+0O2CyC5ltU6/zTeC0aXPVjKAZFOrTxBHjPw30sWqrsrkUz7j9IvKxlXMCUcp15yOYnXxsQvbDfGW7qGnvN/gxiJymrqr0Rkl+jBLuRSoUOvJQAlB7Sso1T+snMTori4vde89V8LW6CgYIVV1NRiO7MzLcSVBpLxWs9owlIJJECTpazHDNe5CrumitYFRHWOleJqlJdSe0QU6SBmjff522s28dILt+NqU6EEUTObuiVZTBBG86/XFlanqQb4vV4rlOdScs65dTE6gCBPfvtoCk4tWTfiXENkIFOwgnKOzKCRsDpPl5nS12XRiqRm7k4Oj9CceN0RqG6cKp6oKU7IiAT2oB1OXu3PnFmcNV1PR4jrWUXiyVu0CSgpLmVWQOqiSBBBSDrHq++rnDKoNLjuneYfCC6y+UkhIn6PeTI2J200B11tNOGbyW3j662nHQsV1O8wAQR2lJUBrPMKEeXKbVDHnbcowl4TrK3t8YzK2xdKKVOQNpQXCTmIyp1j7Y5e4TCNUhxdQpIAUhSSTEDURPz9UWYpqkPJpagGespmVzmAmUJ089z8mzkwZWIC6psIykpBmDlJn8TtytfgibH81Up6h8x9JPtDxjOZB2kGAN+Z7vsFq99KW5U09dcGMWsqA4hd21KgnkJcaJI5DM6Nf1vfO3pKRIQvUq1ke374tHfH27PzzwzvNLSJcolIrUFOsFB1ET3H55WkAbqpd7XaxVrCqZxVLURopE9s8/q356jSyeL+xlhlr/AHZYbvm7tlLQYqG0bTEwvbz3iYs4aFumqClp1sdaCVJ5Eju8fPewt53e+qm9HFE6+lfaTlTmIEc+ZiY0/EuPBST+4E4lu2oqbwuyiqQujvik65opMlLiFJ0IOxIgAHu75tLgu5ahnJI0zSoGdvHzH42pjg+uvXB+LBeVK4pldFUZnWIUErQo6n3E6nb22lW+un7w8w/V1F2PYIxC7VsSHUqXStJzc4JcUfKRNuMx+iqaqdroG30Xa9nq+mpYHMndbVT2LuIGQJggxO49nsHw5WaGO8QvYYqaZDFMFl1srClAAGCNPEwLQdV/lJbtaKvzbwukcuvvkJM+SGFzv32ZeLen1eOIXm3f3vsOy0lSUl2ofdMEjSS0mNhytkxYLiRNyzT2hbEuO4cBYSe4qbV8Ub067q2aNo6AgGYG/hBNgabF17Xzie56hijpjUIQ+22XalNOiVx6zjkJGg58zHnW2v6a1/1KQBgLC5KQcuemcXlJmd1D5Nk9zpnYwzpWzgvCKYUDrditu7+UFrYwWvdoWgfWqLsdofpk/UmjxycVUY5xK4UoQ47ebzrjaF50pUtcwDzGuhGmgtFKwUmCInWIs58cY/rce4grMRXrRU1O9WlBWzRthplOVIT2U6xOUTruTZsOrK1kpmJ08rdlSROhhax+4C4mslbLM57NrriST+Fsk22B4G3SGXV+o2o+Qm1q4CqgE7IPlE2yyrTYWxLXFPoWHryqM23VUri59wst0vB3ixXJ6yj4aYoeT3oul8j35bRMrBuQpiGR2zSmebZaRWejvxseAJ4b3yzP/jDPU/65Flam6KnG+pIzYYoacd9RflA1HsU9PwsJ1XTt3ePMIjaKodsw+SiS2WnOl6HfFV5aE1d44Uo83N2+2lx/VZj9tlFjoaYrVCqniRglCIJPVv1jqtPAU8H2GwjiVIN5AjDDKt38mVXuwgJ6tJ7lHX3Wnm9uiPfNE0n83Y9ueveKgFJFJVoQEketm6sknbTL391ku+ejjiGnp6QUV6JrHgkofSijcQG4JykKVGYHYQJEagWYYlSu2ekcOqmbsUwXJ0xsFUWG7toavDN7PVTFIyy8sVDCQpxCEhSvXnUgkSPqtbzoDcern4vt40um77qeoVXQqhqIdfDhcDodSSIAiOrSNzuLebDPRpxo86tC66gYQD2C6XMyhyMJSQOXPvtbPoA4bVwD4j31V4sxXdf5tv66fRlZA6Cl9DqVNxKNTBWI8bZJp8PicXxH0lrtqcQlAjlHor0uIEEEx3eVuCJIKps20cScGODW/W0xvLTg/s27PEDCGXW+2kxI9Vf92xAQoEFL5kjLGnjbSjqSRInXSzeVxDwbqk4gp5G+igfiLaHEDBy4Iv8ApdSOR1+FitIULJfMnQkeNmZjQJp73u2pIjrkOsT7Eqn/AETY+eIuC1HKMQ0cxESZ28rNDiBxJwMu76OobxXdss1raZLwSACSg6naCd+Vjwus8IEzbsQ5dTM5u/fYm2lPDLIEACeevj9tmeviXgKexjG51cz1dWhf1HWwC+K+Bmxl/PoWBKj1NM85pMaZUmbagcLLLLTdPYuqymCJOmom2B1e/vmbR+7xiwihOdBvR0EZh1d3VEEDzQBHtsTf413MElTGHMSVZGuVqiSCQTyzLH1WWdvVLIVJiHZJVmOukAffY3Quqz76TuDNoeXxqyhK2MEX2lJJ1fWw1lM8+2edgk8ca9DqW2MNUrSpIIqL2bSomO5KVH587LiN6pspU9ekK/4ubZaBz0hKqe3dFClXMfnBWh/qrZZ+IxPkK84sN3vTXZXOKrKhtuQ1CZ3ypTrI5Aa2b2JE3Y5VPvouxCiczk9crtKKjqB3GR7+U2WLpuJi+uuXUuVCXG+rA6sBUS2CZ8dPid7GXMNUhfX1y1uBKYcJA1010Gnz42zmtANwjuJLbJnUrwVhZ1fVqbydahISnSYOniYPunyshXRU1LDjjlMklcphQBOn1/PlaWUYXu9V2Jo2WVnrVLXlRmTMgCfh7J8LN7DGH6Zq9HkdSttxKCCHASJCtToYO/hpaecBR4ZTFaQ/WV+iFF1ZIG4JVOnOe422W6hgLbeTkXJSoHTtDefH8dLSDS3JRHFakmlbUOtKiVHRWgUIGbw233ixCuoqNqtcQ9SMlzrSk5kgkqPcCfH7udmMqcRpltvVfVAOqEZZE66z7eZ38Pecw+av850qWnAEpXJI1ERrp9vlZzpbbZSlDNNTwkAkhpAEjWJOv3Qe+zgue93ruUpR9FWlz1iQUqBhUp08APPflrDjeCIIgdyiTF9MOEtGmfJcVqUsEj2R4AcjvaSOCuMbowXxJw3i+9Gq56jpqk1DyWaUqcyjQgDQHeY37rJV2V9z1joLrriXFaEFZTm9xnlsbWD4H8QL4uzEN1XDh+iuGlLrqECqcuhlx0pJ26xQKuZ57i2RjFQ8Ur25bgg31totjCqUPnaQ6xuLe1ekXCy5btxvhG68YUVStqlvWmRUstvM5XEoUJAUJ0PhYfijwEY4h3NR3ai+UUIpaj0hThp+szAIIgDMOZHOw3DbE4ZuVlVSmvraophx1FMciz3gJ7I9lnk/i+o6khrD1e+FJIITkQRpuc6hbhuz8OBCBskjbSctyiYhUYhFVuAdsfD9K8MumlcX7meLNRTIfL6VinUHEo30UBuZ5T7tt7ermG3Ui4buA0PobH/w028qenheCH+N1QGaQ0qQzS5ULWkqQEpUOUxsfb529SsOup/MF1rCxrQ0+nj1aedu5oi+Siie/chV63KyrkA2ul4vHcqGh5WAW6SN+W3dpYDrolRWQTYJT6UgkEzNrsYVJ50utuqUQSSIHxsm1S5kHXSJBFhHqo6gH4/f7LEH6pKgqFEHnrGtr0aovKa+NnVIoEBJVnL7QEH+eD8YNo8XxWoKVKmfT8PNrB6sJNBTFQUP6R8zZ+YvUXqBGVPqPNrEeCvhytVKo4bY2qb2rHmLzuhll2pcU31tYQchUdMqUnw9otSr6SKptxVdw6pkgvkCms8bWLuQrNiKgYCE+s1SUqIj+ig2QKnpPVQus33QVWMa6hJaCKijuKrShZdKOqCVCnSklXWt5QCZzCNLMRrgviGubyVuJ7gZJiSHXiN4P+BEztZx3jw/4k3tgarwHePGS56KkrWGmnKi7bmc68FC82ZKi6kAk5yQABLioAARloRYZh4PpLRkxCst6IQtR0jcd1LhRT8PeLDxiczlC5TpymRILi0aaHu2snO8bON16LU3dHCO+nCAGwK/FVHTqUpWmqFPkkQQdJ3sR/eOv9xSnKrpMYmQtZK1GhuanpjqnKTKFAiRv77G2Oj5h+sR/u5x44nV5SMsm91NpA5pCc22gMeFrYosMj9VVzWYg7nbySzhh/pFYiwr+6C67jwXTtMPLu5SanELtaS82oJMqYbLeq9D25GVQInIF15dvO9r4vCuvKrqupXeT5qnGWH2loQtUlQSqFAgTEg6jmZm1j7t4FcMrluUXPS47xu9SKddqFMi8WglbrkKcWv6I51KKUkkyTAJtlLwM4JXe0G2qS91IAypS7eASI5hORCR+y02y0cX8Ewob2VU38K5QPRvPIoLzbS+4peamczkoMq7YkZU+/T2d8fXyy47er9Q4+lt4p6tYLhB5HknQ6+O3dNru8P8BdHGmxXX0+MrvdaucXch1PWVdW4VPpdEA9WonYmOXnZ+P3B0AaZ9dS7hBqsdHaUVNXg5OnctUbco7/G1uJjZG57ge1UZi9jspF7I50VsE4f4pdH2466/73TT3xdjZoagMPl5ptKYUzvuQ2tuTMEgxIINpJqOjXw9dfU7T4svKmeeguradbJWYiYUkgeVmJhPjR0TOFlA7dvD3BlXdNJUudY6xQ3cltDiwPWOZwSdd/E2O1nTC4QMpz0eC72f0kFbbCZ1/pE25+fAbzF0bwAfatKPE5sgBze5cX90IsEYorxVXxxaxEpCAUtIbRSJKQdYzFszrJ2siVP5PPhZ1aUs8RsUOmVAqNTRpIBmIApjMTz7h7TL/Tcwa2qLu4d1KgEyCusQn6kGye904QsEUGAGG+QLtapQ+CRawzC3sbl41vYFXdO97s2W58bJQvj8nnwq/NzSqHFl+irS2CpxT9OQVCP0Q0DqCRvZAxt0W7qwdf2E+H2B6mvqqO/aC+GHa+sT1qKd9bTZTn6tICRDe2nqm22emriOrrEUq7guaibWqC4tLzoTpoPXTr5xvZGvHpWY0vi/Hyw3dSHcPB+upnG6YwsincSqQpRB0XtO9i9xDTmMzj9STJZQfmhapfyd+JGKZp5HFG6QvJqldC8MsjUeuddtbJiegtfF2U14Xa7h/Dd91r1O803eTmaeuXmIdlaQoFJOw/WImEpFhr46YPEheF6a87rxjRi8XHyy5SpuxgEfzoKSYjWZjWNDEs6p6VfHerckY1daSdYboqdvMfYjwI+dbrIQzd5P1BAu924HvRhHQh4vmmYo0YCwAk05KkOu1LRQoqHblJplHU6iSY1EmbLTHQv44OGnP5u4W0gZQlvLlf7KUhMR1bKY1BJAgWZFTx/46VnWFeOsQCRENuFAB8MoHcbJT3FPjFeJIqMY4lcBTOU17x8J0O2nxsdvCHIlDySE7gKcrl6HnGSkuRd0V+NsHpC6pdUpxqjqHCnMoKCUzlkCAO1mJASD6qYAoegg9d6mai9eKVKg05SQE3bAJHm7zHlvavlfi/G76v8AdDEl4KAGvXXguDpzzGPGxFvGtFTPTfOKqBrImQl69Ggc0aTmVMaDx1snvZb5nvUmRHm5W4T0TsDVam6K8uJAWhAUA2zT05IBAnRRXGw5fdbT3RF4SU1ORWcQa1tYkoqGKenZdQTsoKgwocjHIWrphvEraaZF5Xfiu4XW3UZ0CmvqlcecTmgQ0lwuROuqfvss0vEdyrbKBUKX1ZI5pmDrqSAR5b78rc/WUdXNNnhfkb0WtTvhazLIbn6lOeM+D3DrFF2U1zYi4647qKWgrWrxpk0VfR07jdSgKyrCmqbOkjMTIVO3cLR9enAjgPTsP1BxTxHvepYQtxj07FdU411olSZT2QrtakEEGfGzTOPbvYbPXGqS4ANerEaT60KMfGxBXEWhDvVda0snTKt4tkb96DGmpHKLPFSVgILpnFT4lK0WDAql0CEsXrXNF9anUuHPpJUqdgBt7bJOJqpFTiB2idVDblM2oK3IITOndufaNog2U79LVzY2r6csyHXVRmVOYmQBH1eHKTJbOPG65+9GaijcPW+gofOuUJIUsRPfptrG/LTo5W3bZYsZAcpU6Pb930PEiiN5XfTVb6GaoNIqmEuoVmZWpRyKBBiNPMbGLWLve+UprKR9i6brp26epQoopqdtkk6giQnaNPDu0i1NuGWK6peJrtqqVLLdc1UBpQfzKbkhQMpSJI1+B0Exaf6+/wDE9Y2KdmuuVJH0mX0RxAV+jBVBI1IO36GlqgiJN1YMoAspuuPFITd9Mw8rqzT06UA51FKsoA0hPcOfwstXXj2hp6soeW44lLZzGRoZBiZBJ22HwtXOmxnjKiQgVdyIqwOwTRvJVlESPWyaTE7eVlC6sdVbLj354uK+20ZAUJTQLdJIIOyM0Kg98edpcSVmxUWxwv3Csg/xSuJGZKEvEgSkZCI07u7x+qbJV/cSrvvC7q27xdzzqHmltn6TZEactRpFoRONal5KvRsE4pezjs5aFCJnQAFa0x4jloNOQLuLcR5JY4a4lUA4RmcNGkK8f5fnI5ad1l3iUqYp4RsmfWOVdEoooGldaz9E2EpzdYRMAgc4n7OYsPRcQq6kp/zffd2LpX3AUIeAUpEkSDMkxtGunjbRrKtu8VJr7ncu196FJpXlpUqDoQCgkHu79OVlVho1dEKapu9vKRlSlfrEHQyPDSY5e22kwlzdFQeMrrKN2F1NLid5yvqzVO1DQcLidBHKI3EjzHPcWZmNODN+45xCh7DLlC2vqpqOvX1KU7QZgzv+AtI+OPzDcTlHS0PWF9ALiZXmyIB31HZnTw0EbwD+Bqp8XywhtxpPpIBIVISIkZTG258+/uo1Ejohdm6uwMZKQ1+yhym6JWPnY6y/bgbUTGX0lxRA11MI0GllOl6HmJXCE1mLbtbJVlIaaW5Eb7xqLW9awrVLR1q61lIUM0hJUI0g8jrprrznxM/uWpyuFPKOsylpWnPST3+X6RtjnE6onf3LYGGUu9lU6n6HLISlVXjt0jUnqruG3hLnzB7rKLHRCwwGwHMRXy6skyUsNI25ASbWmRhphCc6nnSQoKICUySYgc/kDui3CbkQgEKQ525MTBTsANo1+d7QNfVH1lL5PpRs1Vab6J2GWllRrL3eSD6qi2JH+L99lCk6P2Gqfqg7hmncDGoK3FnMYHrH9Ly299rNN3LShQmlUFJ0JLnPf7fnWxpu5aBJUC0PW0JSVTB18/D77LvdQ7dxTiip27AKErtwLhlgpnBlwJ0AGW56dShHKS0Z+s2fV0UNy04DSGzSjklhpDQHkEIBO3x84fH5npW1gdWAMswEwVH7/nWw7N1sAnN1YB5wAY8vneLVZTJJuSrMTGM2TfYauQ6GsvZzXSKxcactB3fUfaaXdOHHgC/SV7p3lSlL000knXu5e2y0KKmJC23svKRyjSO751t1kYQCrrUHtQcxgAePzztUdA4q62ZoCQ04dwomG/zJIUJI6hKYGojUCxhu5bmbjqbobbSYBOQDTy8/q5WUOuoW0EisZTJlQTlBV5x4d1g11V1gFarwQnmcqdRAP3T77MKRztSl3lgRdVHSMpDLVGlKdUphsKiNO8SPnwsE5TNrgtMH9X1RHv8AYLGFXrdyYArFLSToAFdkjTSdP2+0l3b9oUdnI8SgmZIBjkZOoOtjNondEJ1a3qiz91B0LUhpRUnYaajv3I+yxR24XVFORIB3jMR3H3abd9h1YlRTq6xuldWFDm6QNttiB5j32DVi98IytU7WoEAKJgz4RPdYwoXcggGtZ1RReE3nPVebbBkBJKtzP4eNtNYBeD6KhNZSHq1AhISoyR4z32EcxZXxmBDZWQEwiZkfHc/bYKoxLeLiZD6xqQJRodvDx+ux20Lt0B9Y2yfn7suJKFZhed2qSnTRlRCRpsZjlv4+22lcR+JKEJCbzu8rBnN1A1MDSeXlaOlX/eROVd4OlJOaEqgfDv8AssA5eLykz6X1kkQkHcb7z8/C15kDhuVSMzDsFJA4qcSWlJBNzhBM/SMk6TPPu9m9hFcb8XsgKqmLqg5jIWE850lOwB+YtF7tXnUoBIJE6Ezt5mdgNdeVibqilWjUlI2BiO75mx2w9UB0o5BSpWcfsRBoJYo6AOEAKVIUJ027O2vjZrX3xjxXel21VEWqTrXEgpz06VshYUNCnIO487MtZeAUgNBIET2o9+nztYIQnMgAKGYaAyN9PhY7Ig1V3yEoyriJj/I0GG7oplCNG6BAAG/PXu/GxR7H3EhSxlvlhhajrkom4g+YsCoA6KUo5pEZjIJkfZYFRSk6uKWknsxp7Pw8PfYGgVY3ujBxVjpwfSYsrBmH+CbQmO+YGn467WK1F5YjqCnrsWX/AJUQrI1VlsqETHZ8/qtwC6ClIUAmTOm3jPn9duQl1Di4zSdilPrad3OdfObODdMg1O1uYuuXtfLygcpLt4OrB05gqI2+s621SUFEtxTrlAVSc0mTyB5nvH7LaWtwohBVmBjXcGYMRbuldfQ4HCtRgjcg5uX2d3P22dR5pXVS0ySU+h02hj1rZYPryNOvph/SKp9tsslJQFhOkqk+ktoumofK3AU9W0pcgITKtJkTOnwNjVTc2InnSpm4a/KYVpSOmTtvl152Qnb9vQKbp6C/KhDRECnp6hRbASBsEq8uXjpYNqkxHey1NtUt61OhHZp3nJgEaZZBOwInxtS4gBuSEct0tYpxtXPiZgl1d3VqQNIWyWxqT+t7I0275sj1GH7zp3VVtO1SMOFxSyF1rSSoEwRBIIMfERPcoUHCziDej6RdvD3ElYoJUr6O63ljNHfl3G2kzGs7WclL0d+N1UM1LwlxTDWXNnutxufLMBPd99oGpiG7x5pxC9w9FpTHqrhfdf8AS275oGnyElYVXMkGCCYyqJGw018bdVtzUtc0n0jEN1IegpW4ha1FQ3jsIIO206T75Fouij0jLzSEU/CK/UIROUvJbaCddhnWkx+3uhz3J0K+krVUgpXOHrtGh3tOdfetMlK4iJSFzt48jtOo3VtM3eQeYUm00zjbIVBVThe7WG+qexZRrMZ19W28ZBAI06uBAnTkY8wTdpblpxlF9VDmXsqLdMdwqZ1iOfztZxf5PTpC3mpta13KylOUlNTeZB/nBKkIXEkny05iyzSfkzuLFQQK7GmGqRtQMhtbyyD4Dq4O++moGlofKVE0ayBEFFUnZiqQ3UUDLaFJrq4hIGUFlJ57/wAoDMxOnxs4bm4tXvhW9ae8bpqq5NTTrC2XOpayhSSYVCs2b2iB3WtjTfkxr6bCVXjxOu9C1GYZo1qGb/GIkST3GyZiL8nTcuGKFVfffGRR7cIYp7mAUvXRKczygduaSLBfimHyjI51weVkeKirmHMwWPtUODp4cfaZks0eML1bYSJV1XoyMswBoKcRqfv7rN/EHTH46X+gs3hjW+30IUrIDXlqTtA6lKPHv0jbe0zXB0UOEl0pzX2xfV/qMZ01NV1LSlGSCEtJB5jQkgeNlm5ejJwXukrrG8GOPocXIbq6lx2I3CO4aweW3nao04RCbxwjyCs93xF+rn7+Kp2x+7LjjjO6MPUFImuvy+KxmipFO1DhLq1qKUJUt1RhIzHWdpjkLe0lyYbxXS3PQUK7ieLlNSssuKLzYBUhABiVDmLVb4ZcK+FmEOIFwYxThG6bmbuq8WKxdapa2UU6UKB6wqKgkJAB9YRa27/SS4EMHtcWcMkggw3eCHD7kkm25ROgrmXHogLKrI5qR9nakoMYdxOVQbqIV/zrf2KtisI4qUifRKfbYv6/AEWKv9KPgVIDOPqepzCU+jUdS77RkbPdYBfSe4VOAeiVl91YOo6m4a2OWsqaA7rXOBTN3d71SMk7tA1HlYCxU8ImhbG+r6j/AGLYOFl+uj6W9qRvN3NqXr7x4WSXOkzhFQ/imFcXv7gZbrDc/wBY4mwK+k3duUKpuG+LFZpyqd9DQFQfF+0g+lZ6w80whqH+qUp3hwSq7xZDNViZsSQrsUZ3Go3c7wDas2OMDXvgzG154fpfSLxTTLQtNSUpbCytAWRvAAzRzNp6f6S95rSk0XC6sUVSP4zfDDX+oF2qhxu4fY24r8Q714ktXdSXQze3o7T1C3eC3lIUy0lCVFYQgKzBBOwiDztWqZaV4ADtVZpYamI3LSnUi779ISP4m0vYJXXsoO4PNXn87HGsO3rUoC6i/bmZnQpXeSFQefqz8i0TMcIMc0yEpVfzwyDKjLWL0H3be73qLXBfE1TROVb98pdcpwOsb9IWr6OT29ztoCI0kHvNqdqf6SuEVBPzVI5wuttwCpxVc6Ecsjrq9IP6qPtsJ+ZbhaJcqOId3tSN/R3wYHmkcvna0Xjg088CXq1CFREhJKh7wfH3WR794QXuh5Tl11NfWVIRK2m1lKXBGpQNgrSI5z75h9PtdR4c+9lNTbHDhpKkP8UGlrUIytUwInxCnBYF2+eE9IlSX8ZXm8pB1LVMwQY5R1sz3fsFqzrw6lCwC1UZUqCFguK6xM6QQROhMd86RYw1hdCnVKepMhntZ1SUj9EHvE90RqfG0gIuijaRS5fuO+GjNSHrtxDeCELb6twv0hJVtlRkbSpU7nNtoddRaPbz4sXIla00lHfNUlByhYbpm0uAcwHHkmD4666xBsVp7ju5CpNCgH9MKOWT4EzOhOhHLnpHa7hu5YSaZhvIYzJMFaTPKRoeUHTfURaQczYBQMTyk48Sqd4ktYcvRTiQVQ7VUiBA8UOrjfusXd4mVjiZRhejEQQKi91JymNJyMK+Z2svMXPRpZUttinCgSEqzZVpGkhQ+7TU72FapKJsqbqKehKQe0UNhJA5yBvsNvdZ87RyS4TuqaiuIOKVnNSYduJBJkD84VL3s/4OkcwfttwjHuOqkg0rFwIywrsXfUvKHhCnUHl3WeiaC7yAtvqlIQJJySnw0nTQH2+wAVVI0FIS6pp3OojOmQAI1gxM+73WfiN6JcJ3VMl7FPESqRnTedGwoiApi4tCmN5VUL8dxysHS3pxQZJqWMR1aFOtqQvq6GiRCVQFJlTa9wTz5bHk+ihtsfQ1ZU3myjtEjRW+bl4TO+3di03eQ4gMuoShIlQSpSSd9xqdI37p3ss4PJOIrc0xgjHVSlbasSXvvlXlXRtAE/zkU4j2GwCrkxTpN63+4CoypV7up7tg2pJPhFpCUllaA2h1LkwSYKUg7zIEQPrBsUUulpVQXSwUjs65dI1PdzJ99lnPIKJiaOaZD+Hr3fJS5X1/VmCA7e1coBRPi97PZ42LJ4fNmHaqno3kJJMVBcfzGDqOsWqOenPS0gKvC7yyHX3WFjLopSSkxpsZMyJ8/fbhF+XF1gmUJ2BcT2YjUkiRy9uu2lnzu5JcNg5pjI4a06iD6Dcgleybup8xGumqTBnx0nysO1gtNASWEU7CIlRZo2kTryypER7TtG9nib3oVdpt8ugqBgp0Gh1kgSZOnqj4yTXfLTakqQKtCj9H2tQY57wIAOum/fZw55CWViQm8NvUwCVXxXyqJbbqD2p5CCDOpER99g14USSn0i8LxUF/SfSVKviZ7x3cxtAFlZd+1EdclIfPqySUrOugEyCdtdhHfrYH88Ela101Qytoa5V5ikjnzgQDzPdpzcZioEsCSncF0SnEreS4pYO+dR7R8NYI0Mc9e6LabwrQthaE0gSe0SkpJBSCCTuZ15gEDWykq83nELUhDbzZSWwnX1jIBkc9Ph4WLqq6xpstsllaspUlKmzodAdzsDHcNt9bSDXJZmjZMDiYwaa8mKxhpAzoSMwhIgcvfJgHTy2SL6r1EXRVocVkWwplSp00WJ31nt/Az4u3GDTt8M0gQnrFNrUhxSEFMHKDO25BiPkp/wC9tX39d+WiqEMuMPpUARpqII8tB3DTYGxi3M1V8+VyaeB+ruPifdFa5TJfp/SEPlLhLZWlObv8O8cx32tXR8RcIoGerupylKwCk+jocQZnYJUVbA7jWdINqrYcaddvy7qppamqigU6lZSrKqC2ruMnUCAORPkHq5VXi48tNMp1eQleXryVhIgAkETvpA0nUSCSK5i4iO2bh7Kz9145wdVoSGcQ0FO0ZGV5BpoIiTLqR46jSdfGzlpRQ3kA7QV9PUJKZKW+rdAMETKdxvuNfE7UxefqVMFyrdS2h4DIHkBAIToOyUwQB7wecDLpumWqciQpx0q6txmAoGE6jQEx2Tp3c9IE6gBNw5GbX23arrJuoHJKWyoHKqEqTOv80cvMaTppbpV3oBzOVKkhSpIQ4AQCrlPkoezTutUCjvDiC0hTl2X1fA6tIzoTXfRo0MjIVb7ciIAiJss0/EHibdak071ZQuIU561U1lCtCP8ABpH625Mkk66pFoGiI5ogrmnkpl4jYeYqlU9VSvpFSiYzKSpQJgaeACR9vKzQa9IRlSpalPnsqQ2iQ3G+o03G8czvtZCo+KVbeakM112UruZc9ezXtJKQVCVFrMTHMQZ5cps5GKhupWVs5Ql6BKBqecCNOY/G2jTtyMylZ87g5+ZMa97lqXA5eNW11inHSvMUZlZUmBqRy203377c3dUroX6aqQzmVTqSSSDGZOpG2nPvs66ijvG82XvpEGjCiOsW5uBMxI1G9mY+mHwgkuJSrICTr2iUqI7iYE++wJGW0Ksxu5hWaw1iq6qu6GQqncUtlEGMg0OgVMzsAeegO9lNWJaU5stM4oE6hRTA84GmkbD384S4c30oUxoS6MiZCUk9qZ7M78vq9759NMEqaQOYUUdkToZP7PCd7UTSM3V4VjraJ1uYoQCU/m5sAjm4JgA6kwNJP3baFjf9Q6c6adhMa6EmSRuPCSeXKzdSl1WchLsA6qSIPs7+Wo0Phy6QpeSWn5AJ0y6CeR7z4xy87MKVgS7y8pbcv+tKlEtIQRCQdDGu+p3MWLqvi8iqF1RQASYDWkT39+g9xsmoW+lMuIbUFEbHSNNRy+ee9tqNSQUltJ17Kkrkjy7j+Npinb0UTUORld51i8yVPqIUSIGih4TYFdbUryrDlWcsp0J0+qwXVrCspzISR+kkQO8/XbspA/RUpO0FcTIM2fgt6JxM7qgvSH1JKnVPqVEdtR39hsAVreT1sgpIAgqnTWOdjhZZWcnVEawClZAHedTpvbldMSTKTOhBPaHlI5/OtnEQHJR4pKTllSoCqmFn9VUgmJ22k/O9tpcebSOrLhB/WTIG3lBHlysaWypKh9ITOxUdFeff9elgF07hAUlOm/rCdNN41H3edpiMKBkKAC3AvMH1jKJnLHt7o+H2cdc85JCxrqDMeEEcxHxNjKYalYpgVerJ+vQn557W5cUy6AFNrzSNMoB90a6eNlw+ibiFF3HCpJJQc0xOhM9/z325VVuyUqRmgFPKMp8Bt8+djaW6SQc6UlQkCPMknW3KWmEkKbM9nUZdPDblv5+FnDFHiIAXm+lKevpwE7zvMePz9VuV1gSIUFZiRpI9+vL4b2MKU2FhB6uQTqe7kInzsEopz5EBBVJzKyyCfGBp7LSDFEyIquqg5gFZUgaxrE6GPnym2/TEFsqXyjQkAkwNd/nnYXKkKAbCAADGVMAAbaH2fO3DiFApUhzWdZM92v4febSDAFEvtqg1PsuJKcqNOyQEzp8z+HLlfoy0mIgiZDc7+zT51ty8lxcrStJQTmBzQO/vj58rAO0j4UVpKErB0IPMg698zaeVQ4iEKWVBSUkAzpCiDO33DTw52CWhl1KgmdfVJUTE+E/D32zqatsrWqoSYBIUiR7O7kfj52DKX8uVTiOc66c9Rp8zZ0ibrQaaTBK0mBGg9u2/7feGUFAMLJygDKdN5077DEVRzIKGgpAiTtMRy35WBWFIT9KwmJhScpk+/Xl8LMLBMgVuOZjDoUR4gHaJmN9/CwROQ9Xn1Om2/OZnv5T74sYXUNIIQ9TAJGgOw84Aj3RYJLzCsspdXKIBSkxEd+3dp+2z5gFEtKLOOEBZIWoZYIDJBPjvG3KwrK+0QUpgakwZ7wddd7GUtJUAGA6EQCStB7U858LHKe61rTlaRkTsAoRBJ8p+YtHOAE/CJRcFtQzOMN5jqrTnbLLCbqTlH0dOdN84++2WbiBPwivTBnBOGKdtLdNcV3MoSZSlFIgAHyAseauejaSEpabAGwDYAtlst5WBzXQZ3LSqNsqgEpHhbS7ta2UtRBFstlitYDuih7hsUAqnYaPqFXmbacbaSAoone2Wy0soCM1xO6AeQlvUJSQIOotouHq8wQjeNrZbLQGhR262ukm+LwcoKB6uQ2lRbSSE7fG0DYh9MxLW+m3rXuLMlKUISEpT5D2Wy2WNTC9yVZGyRm8PUi30IU44c5AJ05zrYN26KNSs6wshIASkK9UfPxm2Wy10KBXKbnoasKpn2y6hRCVB05wrbcc9/hYR25booKl2morspWgwstAhlMnLGu28mbZbLWInEDQoL2g7hdMtHKeqKWwEgwE6a/tsNUNZ3UMzHVtpEgbzJP12y2WMHE80PKByQSkBKglRKpVG8bAq/s2FS029QNqKT2XlpTJkjQSfPS2WyxWqFkCmlRCtRokKHZH60fZ8bHKKjS6p1hSzl6latBr2UlQ/1Y9ptlss/NRRV27KZKisgkCZHfqfZy7rao6dlisS9ToCVtSdYUFabEcxEj22y2WmE3NKN54ao6Z9tbCsjb7aXkoCfUCp7M8xI/bzIHDNG+XWlLKSqIWhICkmYzA94tlss5UVGfE3BV1G4l42bKkVbDxp6pEAipCHurCp/RXporXTQ5tCImcbp00yCadJLqdY00k7d2omR32y2W0qXVgus2p0foiLt4U7VOzFHmTUOpRClJOUkFQUZTr6o9uvhYkjED7NI2pLc56jqZURmGm8xrsNPqtlstdACplxRlu/Kl95lpTbecpR2ymdVAHb5mxGsvy8KfqFBacr4GiUhJAUTz2MZRytlssQNCg5xQLt/wB5mobKag9YlcBZSk6bRttrz7rCs1t4CrNFU17ryVth0a5QJMRA9vw7ptlssQNFkEuN1091tO6287UOPfTIaAJiApI1ka8/gPGQ6pT6Hmiap0l2QjtHsJgGD38+7e2Wy0g0KNyiqAXqVpxMI6xAMCYB113125/hbpqrWhtmkqJcU64W86CG4iNSANfKbZbLPYJXKMIYaU6WinKSgjM3CfV0H2W5ao1J0D6ozaAgHYkc9eXfbLZaKi4lCKu1vrkpQ6tvKJlAAOhJifad53sXCo7aVuEKWAMypIlWokACPCLZbLPZNcrhLlMhwNOU5UaghvMFAQB36Gd/D42J3heKaJ5akNLW4VQVLcBmE9nSNYjnrbLZaQTIg3fKK1SFChShcDtFcxvtoI9Y7Wc133Wm8mgsOJZAMQlpJ7xM7k6bkk7Wy2WcqKM3thFV10X5wfr26lK6xCAgMFBBVOs5iDuNI5eUCM3YKclDT6gWnEZTAncj7PjbLZYjdlF26hz8ztUeJGFIdMoqkpWQkSuXIJgyOR3BGvnNi6LhrhasqPREsvIUJklScsZc0AJSk+8mJ9lstlqjDurJAstVWAsOXXVJYQxUqKd1JrX0BWvNIXB/b327Zwth1xlbjl1NOgEdl8l6ZP8APJ590Wy2WJcqIAXKLhwk0pCThagClOJRKGWwNdATKDO/f37SbKj+HqG6aZNUxTUgzZZCKZKeceI2PdNstlojdTIC5ZWt5tKioygSJMiTptsdhv3aRZpvqWq81ZVBPWqCdEjQEgjYcj3R7LZbLFi3Q5eqaeNOIl5XHeYwrQUiHVKQ48airdW4NFKEBCSmPV5HaO60fL4iuuvVbF5XO065SEoK6d0tJcAAPqqC43jfv77ZbLDl+cjQ/MTg4acSzeV+/mpi5jTgNtq6w1OYkmYkBAmIPjqNRFrGlLTgClpVGVKuyqDJ35R8LZbLDdupM2XQo0rQH0uLTDYXHMkoB38iB7LayuI1S8qR2QSAefjbLZZk66S2o9YM+gPZGojTwI+RbbVMH1HK64mFlJkhQME9/lbLZZJrm625QuM5V+kyE9yYPfvNiK3nUuDOsuELCDm1BM7m2Wy0gE91yupcStCFZVaADSBHl32EpXy6tqnEhTgHbMGCQOUWy2WSQRu86J2mioNSFHMAAEERKlDvPJPxsmocWokhcQUpjKCD8J+NstlnSXLmdbiklQ7ImY99tqbSQSZJKjExGgH4++2WyySCxy7lJQ44XwQAITk0idt7FS1LikgpGo2TuZtlssxTIJDHXJWvNlCezESSQYmTbZphlQUqgGdCJ2J+742y2WV9EkGpDjRUEukBOke2LDU7briAsvzMZQpIIA3juNstlop7BHEXYpX8q8lYjbq47jyPjYT8zU8ZXHFmNDkMTInnPdFstlmcSnsF0xcNHUIWtLj6cpGinM0z7rHDhukpXAgLKsidDHjzkmbZbLDc4orWhaTc7CnAAGgpa8kloGDG9gn6ZDKM7SUJ5JEHQgnXe2Wyww43Ug0IslEpQ8lRTnKtNNANte/xsNT0fpLvVLWkCSkdgHafutlss9ynaAumaCmeLjamkDTMTlmZIH3e6wzV20iSFKSqCf0SAdQDrIPfbLZYYJUrC6EXQUmYyzJncne2Wy2Wknsv/9k=" } ] } diff --git a/samples/js/node/notebooks/hello-segmentation.nnb b/samples/js/node/notebooks/hello-segmentation.nnb index 8df4e6227701e9..a7da34a2799edf 100644 --- a/samples/js/node/notebooks/hello-segmentation.nnb +++ b/samples/js/node/notebooks/hello-segmentation.nnb @@ -17,62 +17,10 @@ { "language": "typescript", "source": [ - "const {\n getImageData, \n displayArrayAsImage, \n arrayToImageData,\n transform,\n downloadFile,\n} = require('../helpers');\n\nconst { cv } = require('opencv-wasm');\nconst { display } = require('node-kernel');\n\nconst { addon: ov } = require('openvino-node'); \n" + "const { display } = require('node-kernel');\nconst { addon: ov } = require('openvino-node');\n\nconst Image = require('../image.js');\nconst {\n transform,\n} = require('../helpers');\n" ], "outputs": [] }, - { - "language": "markdown", - "source": [ - "## Download the Model" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "const baseArtifactsDir = '../../assets/models';\n\nconst modelName = 'road-segmentation-adas-0001';\nconst modelXMLName = `${modelName}.xml`;\nconst modelBINName = `${modelName}.bin`;\n\nconst modelXMLPath = baseArtifactsDir + '/' + modelXMLName;\n\nconst baseURL = 'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/road-segmentation-adas-0001/FP32/';\n\n\nawait downloadFile(baseURL + modelXMLName, modelXMLName, baseArtifactsDir);\nawait downloadFile(baseURL + modelBINName, modelBINName, baseArtifactsDir);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/models/road-segmentation-adas-0001.bin'", - "" - ] - } - ] - } - ] - }, - { - "language": "markdown", - "source": [ - "## Download an Image" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "const baseImagesDir = '../../assets/images';\nconst imgUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/empty_road_mapillary.jpg';\n\nawait downloadFile(imgUrl, 'empty_road_mapillary.jpg', baseImagesDir);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/images/empty_road_mapillary.jpg'", - "" - ] - } - ] - } - ] - }, { "language": "markdown", "source": [ @@ -83,7 +31,7 @@ { "language": "typescript", "source": [ - "const core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'CPU');\n\nconst inputLayer = compiledModel.input(0);\nconst outputLayer = compiledModel.output(0);\n" + "const modelXMLPath = '../../assets/models/road-segmentation-adas-0001.xml';\n\nconst core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'AUTO');\n\nconst inputLayer = compiledModel.input(0);\nconst outputLayer = compiledModel.output(0);\n" ], "outputs": [] }, @@ -97,14 +45,14 @@ { "language": "typescript", "source": [ - "const imgData = await getImageData('../../assets/images/empty_road_mapillary.jpg');\n\nconst originalImage = cv.matFromImageData(imgData);\nconst { cols: originalWidth, rows: originalHeight } = originalImage;\n\nconst image = new cv.Mat();\ncv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB);\ncv.cvtColor(image, image, cv.COLOR_BGR2RGB); \n\nconst [B, C, H, W] = inputLayer.shape;\n\ncv.resize(image, image, new cv.Size(W, H));\n\nconst inputImage = transform(image.data, { width: W, height: H }, [0, 1, 2]); // NHWC to NCHW\n\ndisplayArrayAsImage(originalImage.data, originalWidth, originalHeight, display);\n" + "const img = await Image.load('../../assets/images/empty_road_mapillary.jpg');\n\nimg.display(display);\n" ], "outputs": [ { "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAQ4B4ADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwCx4XdIbgwXClJAcYIrssYrk7gJZ+KIGkjYK5xzxjIrr3FWQQPGG7ZqFowBjaPyqyaaVzTQNlR0J6ZqIAhquFcVG0eaBDUPIqUVDjaanXkCgLCgU4UCnAUrgIWVfvHFV2v7NThp8f8AAaleNXPzVk6nczD/AESygLOxwXI4FFwNpfLlj3RuGU+lQpGyN8qVz2690NVVEMspGWx0ak0zxZLdTzm52xRIpwBzzRcDo7xZZLVo5ogUPUt0FcT4vjV9PWFLcu0S7g6t/KsbxJr2q6gpQSyRRdgpxuFclcSXaQhjdyEnqhJ/nmi4G9Bayf2YrXTt5XYHnFZwt5NPaSeODzbSTPNN0ttW1GSKxQHa2TyOgrptT0+e0SLTtwbzyACO5+lRYozPC9oxnNxO7pDnMTk8KfSrFxZXWq6pKhkwinGR0NdPB4NuLbSlsTiVT8xYtjr7Vj654a1DTLAy2l68iLnMbEZA9qLknNWdvZt4lSw1KZIIkfBd+h46e1ekapofhiTQ3itbW1lfqrQvl8151pmjWuuRyy3F7I16Af3ZPJPqKs2do2j3W3UHe3f+CRTw1FigMNzfmSzZ5kQnGSMgVh3qQ6TcSQ3OLtgMLvY/J9K37+5s4Uku7TUZ/O4+UKSK4i+mF5cmYNu3ck+posMbyy5dyccDPpSp5j4iRiwP8PrUGT0NSo2wbwSGHQikMle3a32MwwG4/GlMMu9VVo0ZuhPSo5JnuJA8jZIHFNl+fBzjHqaBFiOwmhunlmRdiqfmVsirlnq8aOLecHyuzjksazYZbiAMUnYbhg81FEI96+aCU74GaAOma1vb2LN2c24BMRPQCs/Sb6XS9ZaWGTauChfHBFR34WFU+zXkkkJGdp7UyK4SCycpjcccEdaLAaCfYW1Bri8uRKkmfM2HnNFtaaPJrvkmV2tHX5SeADW54d8IxX1/YT3UsctpcIxIBwVPvXP6xp8en+K5rGJiUSQKM0cpPKa3/CRW+n6VLpRtvNWOQiNtwI/+vWdLfvfeSWhSIrxGV/i9jXUaLf6A1u0VxpSy6tACrlE4ceuOlcfr1zDf6k0ljbm2hH/LMrgq1FxGxYx6yiPPbysYozzbl8jHrius0m9uNZ0aPSxGz+Ud7luOnY15vFdS28f7pyr92Hc1JDq2pBJFjvJVLnJx3NFxntH9px6NYCEaatoshCCRGGCxqSx8L2L6L5M9tDNMxyW7ZryDS9TggvVN804PVlYlgT6kVr2Wt6k2rvdWM9xb2rr0JOwn27VdijW8b29jolsmmWMT2zyMDMuMBh65/Ouv0bSINK8Mw29pLuE+1nkHcnrivK9V1m4u/EkE085uUgkDZf8Ah9f6V6ZpemQ2iS6hdX4e0UiaGPOAhP40WA0k8PabBA8lvbtHK45dj39TWHeeEtav1mT+04obV1+ZVGQaS+8QLrFuWsdThtXRuR1Dntn9ay9P1XXre4KX0m23kGHA6E+tFiTAh0QQKy3lsjOGIBVvm46GmXGjrcN5xuQFbpvo1a9QXUiJvkmMvX0FUr7T9QjszNJBMVHYdBWViiSFvNu5LCaczRNgI4GcCsnVNOW0umjVwy+3arNnLHEUaMlJR1BHarchnlu4DLHHsY43D19/SncDnLNoor+NHOYycH2qvdwtDdyow6NXQz26jxEzMmE/vVXks5Lu6ZhA7qRkOBwaLgR6ZHBtUuiuCuWU9V9/erjRL+/gT94sS8HvikvdMXTRbXdrI6lxhl7gikjl8jJKhzMnKH1+tFwHQ6v/AMSiO2AbzYTgEc1dmvFvL+NGUujR4BHYjvWdYW7Rid3+VA3LdhWlp1qE1Rd1wrfLuGB0NK4FaF5NPuLmGX5cpjHqOxre0v7Hb6QA82/7QNzfWoNatfNtBJcxhplG1JE6fQ1oeH9NR7ZnVU2SQlJS3QN2wKmKDmNvSrP/AIp8zykmV1O0k+nT+taOmbjp0LFshhkVV06QvpK2TnEkP7s++O9aMKCKFIl+6owK6LEjwMUhHel6UlUAUtJRSELSGlooAjNJin4ppoAb1NGOaWjGaAuJilHSlxS4pjGkUAU4iigBrNtGaWCZWIPSo5BmliibOcVRJphQYwRUDrzxUkZITFBFSUVXU1CRVt1qBl5pCINtIVqbbSbaQyIJShOamC8UFaAI9vtTSg9KnxSFaLAVWX1phWrTJUbJzQBX2imkVY2cUxkxQBARTcVMVppWkMiIpMVIVoxQBHikIqTFBFAERFIRUpFNxQAzFJipMUm30oAjxzSYqTaaTFIYykxT8CkxQA3FGOKdikxzQAgFLilA5pcUgGd6KdRigBvain49aO1AxopRSUCgBaU0YooAaKUdaKKQh1FAooGLS0lLTAWgdaKKQDhSj0pBSimA6l70lLSAWnCm04UALSikFLTEKKUUgpRSAfRmkpRQMdSikpRQAvanCmU4UCHUUlLQA+lpKWgYopRSUooAUGnCminCmIeDzTxTBThSGPHWng1Fmng0APBoFNoFAiUVMv3agWph0oAUmkNBpKAEpCaWm0APBxSim9KXNAx+RSg0zNKDSAcTTTRTSaYAaUU3NLmi4D80ZpuaXNABmjNNzRmgQpNMNBNNNACGkNBpppgFGaSigBDSUppBQAAUYp1KBzQAgWlp+KaRzQAyilNJSAaaZTiaQ0wGUGlxSGkIbSUGkpjFzRSYopCH5opKTNMB1SJUQp6NQBLTCafnimHpQAgozSZNJmgB1Lmo80uaAHGgU3NGaAHUuabmjNAgJpDRSUDFzSZppooASkzRmkpiFopKKQwozSUUxDs0A03tRmgB1JSUZoAU0UlAoAdnijim5pM0wHUmaQmkzSAXJpM0lJQAE0E0lFMBKKKKBB1pwoHWloAKKKWgBaKQGlzQAE0AU0mgE1Qx/ia1kuIopYx8yMK20bdBGe+2msM0oGBikAE80maQ9aKQBmkxS0U7iGlAetAUCloyO5ouAopsk2w4qvcysq/KSPpUU95ALcCOVTIecZ5pDLJkZeoqNRmXfVa9kjn0yNjJskDcnNUDfMjRxGVX+Yc96AsP1CaSbW4rfdiM/LVG80NbHUzAo220y88dfWrerTLDqNpcNxGxA39hXUtBFcQDzE3rj5TnoaAsczqGj262tpBFEpdCCGPJNYfjTwvazy2ssXlwHA80HgYrtzEJLtfnK7RjjvXGfEJ8S2IkYNC77WweooAoWULadeIli32tmXop5X61ng3sni22jnhZpRKpZAeMV3ek6NZ6LaSvZfMzru8xhyP85rm9FtpI9evNTuYXZ8kxbRn5j2oGelTRL5eWcooABIrzbxTqWr30jWOnW6JGmMOx5biup0576SxuVudy5O5cjt6Vz3inRr+Robz7UEVzsKL2HHSk0Jnlsj3dpeFixjnQkFh2q/NLe6pCn2m/WTBz89a1/wCHGMFzLHuRIunncF/oa4pNjPtkBU+hFSBuHX5tMtzp6rDOACPMx61z24MxIqw624+6OajcFsYA49BSuNMiAJODSsjk7U5HpTtjbWYDODTo2KkOOtMpsiKNFIokBXNPniMMm3IIxwR3pJjNPLjBfaOwqOLDyojuUVmALDt70CJ7SI3kiwPyn8X0q3sTTrs7YhLEOhI5qG6tDp8u1ZVk3DKuBzVcMXU+Y7b/AO8KANq70uC4i+1wXg2D70eMlRWG6Ir7d4fHpSwxyvJsVm2n72PSpPJj7ZNAEsF5c2+PJuHRR/Dnj8qfF5moapCs0nMjgM+KhW3kc4UZ/GpRZMMGSXyx65qrgdVIkPhy6uooh5wKhlmJywqbw3Z6bqtrqOoawrM0Z+8DjA/yRXItEyY/0hnHuc1NC1ykbi3LjeMNt9KzbFY0tbsLCO8B0S5ae0cZYHsao2NqJrp4kcJMqlgD3rtPDvhY33hWV5SYZXYlC3Q81VtNK0vULI292Gt9TgdQ7g4DjOD/ADoTHY5B7We8lFtj/SmbaF969Wsw0Hw3vbG8txDcWsZQqR36g1uQfDjQ9KC6jGZBKg35JzzXnfjHX5NavfsWnRzNgkPGuCHx0z6d62iSbHww8OR6tb6gdQsXZJ4sQzMvGO+P0qfUNH0fQ9IuLTVrt/tIBCRq/JA6ECs/wleLp0NtbX+sXVtE5YmINgIew9s1razp2g6eZNVhvzdTOp8uOV965ApNDueZxRWE1lJLJPsmHRTwSKQX1xdxJHJfSiMdOegq5a6rpNxqlxPqOnCNJU+VV6BvWsmeFc5ThGOF+lSM6S4hgj0+3u7SdDdRjJb1FT2WvavqEXkrFGPLIOQOvtXOz3c4t47d9vygEMBjNaaSXVppUd1FcQFCPLcoecepFJgJqN47tMTbwiVjkup/pWcs7hIw0jDnPFFzAWlneGbeEGcp0NU4ZPNUHeW+tZtAbdgWvXleaYqQcZA61WmnNtdvHHPIoQ44OM0unXQhlSN1Gxm5IqvfQT/206hGdexAoiBbjPyr5rn94Sdp7elMMeV8wdiRRczrKY9oxs4NOVHZGQNt4yoPQn0zTAmjvV+zzWMUf+uA3M45I9KjiDWFwYmOG2n8qWyTzpVhn2pyRuA5FNu57ckvHLvkVwMkUAbbySXOnwsZ0KsACPf0rU0yaW30bZN8m2XbnqcVyEcm+3EWPuHdmui82aeSG4ZCYpQAQvQNVw+ImR1kDoAjDGSOo71dSbJxmucspyYsng9Mela1tKG613OJCkaeaQGo1cY60oOelZcrHcfmnA1Fup6sMcmpaZQ+iiikAlNIqQDmkK4NIRGaKUjmkxTGKKXrQKKAFxShM0lOBoABECeelTKoUYFR5pd1MRLmg00HilNIY01E45qU0wikBHijbUmzjNAHNIBAuBRipQopCvFMCHHNJUhFIRQBGRTSvtUu2kxSAgK4qNhVlhURWgCArTCKnK0wrQMhxSYqTbSYpDI8UmKkxSYpCGEUmKeRQRTAZjikxT8UhFIBhBppFSEcU3FAxmKTFSYpMUAMxSYqTFJigBmKXFOxQBSAbijFSEUmKBkVJin4pMUCEIo74ooxQMKXvTcU7FIApO9LRTAAKUCgCloAKO1BpRSAKWkFLTAXjvSikpRQAtOBpopwpAFOFNpwpiHUtJS0hiU8U0UtAD6UUynUAOpwplOFIB9FApaAFpabS96YDxSimU6gB2aKBSimAopRSClpAOBpwpgNKKGIeDSim04Ux2Hg04UwDmpApxQIep5qYdKhVcVMp4oGFLilxRSAaajqQ0ygAzS5plFICQUtMFOpgITxTc0Gm0CHA0uaYKcKBjqXNNooAXIppNBNNJoEL2pKXNGKAGUhNOIpMUANoHWlxRimAGkxS4oA5pAGKXp0pTzSgHFACbjTSTmnlaYRQA0mmE04000AIaaaU02mAUUuKWkIYRTaeabmmMDTc0pNJmgQZoptFADqcDimClzQBIDmnGowcUu6gBc0mabmjNACilzTRRmgBTSZpM0lAD80uaZmnZoAM0maM0lABSZoJpKADNJRmkzTAKWm0UCFzRmkzRmgYUZpM0UCFzRmkooAdmjNJmkJoAXNJmkNIaYC5NJmkzSZoAXPNGaKKQBQaBRTEJTgKMU4UAJ0pTS0goAKWkooAKM0lHagA60nSkzR1oA3s0ZpDTaoaHUZFNzRnBqQH0ZpM5pelFxh1qOUqibm/CpDwR79KjnGUAPTNAitDeQyXL20g2OBnLnANc5r32TyjLHIySKc8fxVvtpYu76OUPt2ds4zXE+LJ3GtmyilEODkE9CeOKAIv7QIMQkd1EnTf0zVPWdUm01kWFtsp/EGub1rUp7iQW7kfuzkMoway1nuridYvNZjnI3VBR1U3izVVtTaXMUbqwBViOQfauk0HxdcrZRxSyomBgF+Qa8+vp58RJcMhMfTbVR76UqUjfA+lAHuEWom4w8DxyEclYzyBWd46ijvfCq3jIUeE5XNcj4YEU2mNIJWinT+INiut1KfT9U0A6W16FeQ43Hkj8apMRkaFqUmpaQkUeqLDcJ8vlSfxVYvPFdzo+2w1aBUIYNFcwcqxH/66msfh9p2l6fPNdTvOVjJVwcAEfSvNZINQ1i7kWFHn8rOFzkgUAevDxjaXNk8UKPJK6ZV8gDP9a57UtQj1KysnF5KyRnbMiA5HvmuBt7s6PqAE1o3AIdJOK7Lw3rM0LyQW+nKtvI28SSdQe9MkveKrDTZtOs47HXBI7ZykkmR+I7V57rrKZULlPPVdp2dDWl4xsvtPiJ/s0AVushXow9a5i5RlcBuo6mk2BN5DN5e1gzOcYFWLjSdTs4vMuLKSNOmT61QlBWINHK5Y/w5rYOr6hPpq20tzJKrjZ5Mg4+uagaKEKs6McEEDJ47VC7fIxHBBpHkuIcxuCo96YxOMZ60xk+lTyxXhdWwwU444qO9kWe53KBkj5towM1BU9k8cV0DMpKEEHHagYyIKrZatDR4LW6v5o7hx5Yj3AHjms+fZ5xMb7k7cYqa1iMu8KV3KM4J5P0oAsWkoivZhHllAK8DkirV5cafLbokStG6dQ3VvxqSCx+0Oy2ifZ7tVLP5vAI71Jb+H5nc3GpKUhxnzAeCKAMqMSErJAwUdiTSzHIKTMrdwR2NKVSF/KEgeAdHHU0hi08sWEzrntUgFo1usmZ4y4IIBXqD61q/Y760ZRbEN55wjjrk+tZZWzjQskjMR612tlNDfeF97bEuYkLA+vPBoYE19r2seE4rW2do5QV6EVm6p4l0W+/0hIZldkw2I8AHvV6Oa21fw/BfNGftturLluhPeuFklUrtzz3HYVSJN5/E+srYfY7fXLmS2kXHkvjKj0zV3w7qFjo2m3LTiSK+IIBdMnPtXH52pvH3gRitSPWJJJoGu2WWDd86kYq+YD0U6bDqvw2W7MZkmdi0rr97OcV55eaWttYxPJNNCXBK7hxnv9K6zTfGT6Zqk9laQeZplxggHjySeCR7UzxNaafbaebu3n+2sjB2Jbhs8fdNK1xnn8KbgCSa0ES1WEpI7GYnII6UXdxa3DoLW38kbeR0ANJZadcXLkww+b5XzNhsYFS0M0bTSILu2umlmMckI3D3WqukXWy3ngeON4pj86sO9TfbLm8na3jZI/MOwZ42nvmnrpL6ewildX3thWXpmoAg0NhG1+m9VHl4XPqM4qpbad5sLzB/KkDcHPy/lUF3byWV5LBKMODnHsa1YFil0J1iyHYbmUnIDCgCgFuoZyqxszoQVK9Ca0rq8ubJ4rlo/JuGXJGOJBVOwmlbVLSPqjjbIPVe9aOrQSXYEdm5a2gXkHsanYClcQyfZorqNWfzgWIA5U+lLcgJDFIxwSAKtw30YsYotrieI8LjrVm2sotVgYbtrLnYjDv9a0Ap6XA9xDPI4BUKdgB5yPWrmnabFfWSPOmzaSCR1P1rMXzLCa6gD4ypXOOtX7KZk066jL/N5fHHWqsSNtIEdJFDh9rkfL1rpEglXSIrdT8ynJqrofh1fIhuLkOzyDLK3Ax2xW/5AiQIu7A6ZNa0YJO7IcilEnlxn3JNaUNtdeWrKnDDOarrA0jhQK6izQeSoI6DFdcnYzMRxcQHDg/hRHekcFq6pY1YcqD9ayrnQ7eR8qNtZcxfKZn2z/aFSR3g7tVG+02exYHIaM9MVXhLyNhRmteWJJ0Mc6sOtTqc1hWskhulRvxrozCVVcdxWM4lp3EQZanMoFKmFPNK5BrMpkBFMNSN1qNw3YUwEFP7VHHuP3hipQKGgEozT8UmBU2GNNAPNBGaBheTVCJQMU4moDcqXwtPL8UgHUlJmjNIY4dKUUwHmnA80AyVVpXXilUjFDHIpDITSYpxHNJQAhFNIqTFLtFAEOKYy1YK0m0UgKbLzUZFWXXGagYU2IiIptSYppFIoZTcU+kxSENopaKQDKKdSGmAyjFOxSEUhjcUmKfikxSAbijFOo4psBlAzTj1oxSASkxT8UmKAG00ipMU3HNAyMg0Yp5FIRQA2jFLij2oAbS0YpcUAFKBRigCgBMUU7FAFABS96MUYoQBSikpaAHDpS02loAUUvam04UCHZpRTKcDSGOFLSClFABTxTePxpR70AKKeKaKUUIQ/NOFMpwNAx1IDRQKAHA0opoNLmkFx9LSClFMBacKbSimAuaUUlLjikIeDTlHNNAqdFHFMY5F9amCjFNpy9KQCYp4FKBk07GBQAgoNFFAhpFJTj0ptAxtNp9JSAQUuaBS0ANPWmU8ikxTYDadmlxSGkAtGaSk60wCko6UdaBBThTacKAExSAc07FLigBMUhFOphPNACGgdaQ0ooAeBzT6bQTxQAE1GxpSajJoAa1NJoY00mgBCaBSUUwJBRmm5pCeKBCE02lzTCaQCmkpKSmAGlFJRQAtLmm0tAC5ozSZpM0AOzRmm5ozQAuaCaQmkoAdRmkzSE0AKDzS5plLTAfTc0tJSAKSikoAUmkzQTTaADNGaQmkpgLmjNJRTAdmjtTc0UCHA0pNNopAKTSZpCaTNAC5pCaSg0xBRRRQAtGKUClxSGIKUUUtACilpopaYC4pKTNFAhSeKQmkzSZoAdmkzRmkzTsMSlpKWkI26SlpKoEJS0lJSGOBpJJNiFicAUmaGUOuCKTQGdqBl1CydYi8c0ZDp9aTS9X/ALQi+zzgpeR8OpqDxBqU2kQRfZlG5upP4Vz6+KYtPZ7i8gjW5Iwqj+L9KAOwvbuXTIlnWDzVzgiuE8UyW/iHV7aL7OttLjJJbJPTpUN94i1HX9LnlgvYIkT/AJdj94j61xEt3vwXeTeOjA9KBFjxJpQ0jUvKRmZSM5brVfUNMl023gujICsgyCB0pZ9VkvihvXM2wYBIwcUXeoJd2ot8kRqcgEdKzZSZn3DiRtwlaQ9yajiZkkq1Z2U105SNVJHqcVPqOlT6fbiWXbgnHBpjIobwRZXzWBHQirn7yezWZZJGLdcdqzRZSPAsyjIY4q5G0lnErRTHPdTyKkC6uv6ylqdP+0O0J7MxOKowPeWVys8c3lOOhzW3BdWckSsoiaU9Vk61jX1lHDcA+d5wkxkf3TTuBrXfiFrmF0nWK5kfBUhfmBFYr6zrEsJgWSQJ/sqAarRMlrdqxGFHWrb38xlWW1+7jB561SYmjQsvEr2dkLe4sxLKAQJJOuK56Us7sWPU5rQRftVzsn3xzHoTwKoXAaOYxsMEUMdhbS2luJsRAMQM4JxRLdbX2OcMKYkkkTbo22t610cGoaUumk3GjeZuGHlWQFs+tILHNyFpny7k/WnFAF6g1CWR5WeMFY8/KD6Uu85wKkYMvPFSLGpHLhahaQr1NPEc8lu04jZo0PzEDpTAa2BUiW8pg+1RFRt6g9RUgs2NkLgSIc/wZ5FRwTyW8qOG+TcNw7EUxGql0Lu18i7il+1OAUf7uR7VQMtxCpt3mn2d0LHFd5/ZMEeqwOYHktJVzHJn/Vk/0qxomm3D3Gpx/ZftMMQZopP73tVE8x5z8w5ZSE9TTjt7GpNQnkluZoXQIgcgRj+EelVTWZRdVbSZzmTao6pj+tWppGiC267hHjIIGdtZSY3gMdoPetBDsQDezD1NAD7DUJbS3ktJpnjRjlXUdD6VCsP9p6nHbxMi7hgNjrikuJUaQjPyeleo/DfwhZDR5NV1aOMtI+YnJ5QD/OKYjzTV9AutIWF5WR4puEZfX/JrOC4YpIMr6V6t4u0S91S+EVvLBLawneixnkHiuL1uGC/vAtjDzEoR+MZNMDKi1vUICYoptsQAGMc4Hv8AjTb28iurjzYg67hllY8VZTS1isJp5xsl28DrzVeeGzSxtp4ZXFw3yzDPQ0AQxqz9BWhZedbRTbWYGZdoAPeqAklg/eRK+R/s1r7Y4dO81GLZAJBPepGNlaK7liSLCzKmCQO4qG4g36c1z9okkYNtIY8ZqOYgSq8bbXU88VOLK5e3N0UIhb0bgmkBtaBeWs+lXh1IRNIqbVaVckj0rmLSbyt8YOFLHFWhMUhjVejEhh6inT6NcQ2sN5BCzxP6daAJrO2UE3OdskJ+aMDt6itPSoUs7S8l+0RzfaIyQEPIHuK59bieJpX+ZX+7gjgirCWzQOHgnQ+opMAsZVj1IOqbVxgDPSui0Nbn+zmuBCZIWlI3g8rXPRR7pQo/1jdDXS6Tr1nZ6HbWV1avL5UjFijcgH2xTTAteLPD8EQtrm2Oxm+VzjPBxiqreF5bTS3mk+e5hx50ZOMA9xXZaRfaZrdnNFGBIn3fmHIHarj6TLNKocK1uVCuM8MAO9apCsZPhjWpr9vsd3aqnlxBo36ZHTpWxd2CuS6rVGwsvsfiaS0QfuY4cxjuAe2fwroimOKuLsJxMSC0WNjJjDdKv2hcrh1A9KsGFc5xShAOlVKQuUmTG3igrmmqcVIOlTcLFW7t45bdg65rKttPijdtq9TW3cD9y1Z9s+XxVxYWGxaUY5xMy/KKtSSDOKuZ3IKrPAvLNQ3cSVipJcRI2M0pNPEFkvzyLSJJA8m1Y8D19aQCAUJIN5UD8xV1lWIfMOKpyXdtJJiJGz60wQr7Qh45qu2c5xUsgO3IpY3Qrz2FMZGCWxilYFRkimo2GyBVyMbozu6UgKIcEZBqZMbeeaSWNd3AxTQSDigCH7MiSFxnJNPqU0ygABpSab3pe1IYgNPDcVCzgGoBc470+UVzQV8VMCG6VnJNuPWrkT4GaHEdybjGDUZFIXOaTdzSAXOKN1JkUh4qQH5pCabnim5oAZJmoGqzJjH4VAwoAhIoxTyOaTFIbIyKQinkUmKQMZim4qTFNIoAZQRQRRQMSilxSUgE70lLSUgCilo7UDEFFApaAENBp1FADaMClpcUAMIppFS7aaVoAjxSYFSYpuKAG0U7FGKAG0UtLigBvalFLigCgAxRilxS0gG0oFLijFMBMUAUtGKBDhRQKKQwp3Wm0tADhSikFAoAdS0lL3oAf2opBThTAdSim5paQDs0uabiloAdRSUtMB4pRTRS0AOpwpopwpAPUcU4Ug6U9VNACoKmXimovFSbTTEL1qRBxTB1pynFK4ycAYoIpgNOzQAmKKXNJSAQ9KaacaaaAEpKWg0AJRRRQAUlLRQAw0lKetJQId0FNpT0pKYwzSUppKAAU4UgpwpAKKDS01jxTAQ000hPNITQAtANNooESZpC1MzSE0ABaoyaCaaaAAmkoNNpgLmkzSUUXAXNIaKbQAuaaTS0lAhM0lBpuaBi0ZozSUCHZpaZS5oAXNFJmjNAC5ozSUUALSUUUAFFOxRigBKKWloASko70lAC00mgmm0AJmjNNNFMBc0ZpM0ZpiFoptFIB1LmkzSZoAXNGabmkzTAdmkzSZpM0gHiikFKKYC4pQKSnCgBaDRmmk0ALmjNNJoBpCHZozTSaKbGLmjNNBoJoELmkpBS5oAXNJRmimMKdmm0oFIRt5oyKZminYB1JmjtTc0xjhSimgilHJ4oAz9ZsheW6Ejc6dvWsfxLoFnN4fScw7pUAIOeldJe6jZadEHvJVj9M96zP7ZXUID5NoJLdv4wf6UmBwj+GxNpss9tZTBgvBQ5P5VzVn4dnvNLub1Z1VrfOYs8mvatOjFtavJMwRWzyxwK5q5XRYLO6totRiDTnOQcVAjzUeHb5tDbVQqiEdieajsNMku7ZsA7wMjFejW9pBB4TuLBbqKXAJUjtmp/CunWU3hxohHh1B78g0ho8ohaQymKPfv77etTiPzZfLv5pNq9MnoauWxg03xG7XERby3IwvWt3UNF0290yW/iYxvjIx1/GgZhx2K29qrXOJrQch4z90+prJYoLgpGSU7ZNdBY6RqtxYNcadBKyAZburD0rHgjVLwFkMUiEZVuxoJuTzSW1ntU2+9iMhwc4PtWeLhmnyxzXcatpum63osUiwiyu4+SR91geh/Suc1bQ4tNtElikZnzglhwaRSZhXB3MTUkMxCCNWVW7M3Sr39pmHTTbLEnFZiW+9Nz8L6imijSubgxxoyujzBhVG7mmmlMkwXcfQVEqRrxg/iasr5DwMjLhuzZ6UAUvMbsaRI3nnCKwBPcnFPdFAO081Hg96AJp7V7Vgrspz/dOaWGJZoHlVvunGMVJp91BZ3XmXFsJ0xjB6j6Ve/dJYy3MRhPmEkwgcpSAzYXIcqqK7MMAGuhs7iGw0xw7I0zL8yN0IrK8P2ljfautvqE5iiP3SDj5u1WdW09beeaC3geTyn+aTdnAoAzbOD7VOIt5VmJIA6flUlxpssUDzBkZUODzWhpmnxRag8F9G4lKZiZD3qP+z9XuvNYQzJaBstJKMBRTJNzwtqX2F1s7+K5V5PlSRn/AHeO1dn4f1a7tr6404SIbGMMQyDkHHr3rg7bxLc2+mHTJIre4VVIVioPFOtPEI/sgafFbmK4b5fMJwp9zxVXJsYOojy9UuxI+5jK2GI5IzVF2PGPXFdvP4V02z8PnVLm7M9yAdyA9+xri2jqWjRGvrHhrUNGjtZ5ykkMvzK0Zzj1zUepXNvJNG1q+VK/MuPumt3w54kgWyk0vWn32ZTajvzsHpXL3UUEN9PFayiaIOfLkA+8KVgZbtYnluonS3kuERg0iRrklR1r3+wj0qbQIb3SbVGikx1OAp/2q89+FmnPp/jGBr2YQvLC5jjz9446Guq8X+HJrTV4zppa2t704ZYzhN49RmgRgeLbi50W4P2DTIlhlwPtUB6juMDpXL2baXFcCVLoq7oVkEnPzetdp4xt5ND8Mw21pazSS43vcdlPfj8a8vuLSRI0L3CbJF3Bl5xTAlvtOvoHUzyZ85vlXOciqstnF9jcK6xyxnOD3pwvLmTCXExmSI5Qj+GqmoMJnE688YagC+NVFzpUnnTZuAuAMdKrWUrSJ5TnKms9ZMoV9afGWX7pxSaA1d32h5G2DIOCRU9pqDoBbNyndapK6hGUNywweO9QoGjlDK2D60gNdPJmuIikLfu23EE44rf1LxBFaPFJZo4Q/K8XbNcdJNJbzRS7w/zfdHWrd4r3CGTcrZ7g0AP1I/aLgzJEELAFkXoBTNPgjnlmLuVUrhSO5qxpM0ah/OZDMUKqW6AVBplzFbXTxXCg2z/I2eo9xSYFo2EiRAb0HIG8ngVs2dlBZ65pxK7UkyrknKnIrLhG3UDaPLuiP3GY/eHatSexnmtbazjO0xvuE3cD0qQLcMR8Oa3PayQtLa3vzw+WTlcGvQNFu5bvTonnVg5zknvXG6BJd6nqNklzFm600MWLH5ZkPp7100/2mzvLm1tmKrgTxAHqO61rEDa8lPtX2gj95t2A+1PNCSLPbpPGPlcZx6UVoA0ijFOoApCY0LUqrxSAVIooAQx7lwRVA2qwElfWtQVHKgZaaYimk6gYqG7uVEYAPWkliZM1VmAaPBANaRJIWnh6F8n0pw3Kcjj09qoXCsmWXqKsxPPJpe4/6zOK1sSXBICv7ycM/fJqK3bN0FxWLHDLJtl6N3961dNldpWz24osBqTKwHyj61kTSMLjZnFbec8GoJdKD3CzZxjtWaKILUFiOKujgEU5YljGAMU1iFPNIaIJRgk1VL/NUtxKBmquc1QiYSetKGB6VUZyOlEMvzEGjlC5boJ4puaRjxSsMrS/eJqizFWP1q5LyaqyrkVoiGTQvggg1cW4x1NZCMQQKsh6bQkzUEoYUoeqCykEAmrCtkVm0WmWlOadVdXx3p4fPeosUPpKQmmk0ANdqbuzSO3aow2G5osBJSU6kpNANxSbakoxzSGiIimEVYKioyKQyEikxUpWm7aQEdGKdijFICOkpxpKAEoNJS0gEPWijvS0DF7UUnalHSgBaWkFLQAYpMU6lAoAjIPpSbalpMEmgCLFGOKl2UbaAIcUuKk2ik2igBm00bakxxRtoEM2mjFOxS7c0DI8UoFP2UFDQIbikxT8cUYoGMxRin44pNtACUUUUALS0lLSAcKUUlLTAdRTTS0gHilFNFOFADs0uabS4osAtOFNAp1MBwpRTRThSAUCnrTVHNSoKYhwFSpTAKeg6UgLKgYqULxTEGcVN0oGQFcGlApxFJSAcDxRmmZpQaAH5ozTc0uaAENJmlppoAKDSUpoAbS0lGaYBmim5ozQAtNzSZozSAUmkJpCwppagBc0mabmlpiJBTxUa0/NAx+aY5oJpjUXAQmmZoNJmgBc0Z9KbmjNAhc0wmgmmmgAJ5oNJRmmAUlFJ3pABoFJRQAuaYTSmkzQAlBNJSUwFJpmaDSYpAxxozSYpaYgooooAKKKKAFzRRiloAQU8CmgU8dKADFBFLQelADKKDTTQAE0maSmmgBSaSkNJmgAzSUZpM0wDvS5pM0ZoAM0ZpKKYhc0ZopKQBRRmjNMApaSigB4paYDS5oAUmlzTaKBDs0ZptFABml6UlBoAXNJmg0lMYtJRS0WAKKMUuKQriUtLgUYpoaEIpQKXFLigDVopM0ZosAuaaTSmmmhCDOKcjfOM1ETQDTA4f4p4lSzRTlt3T8Kh8JaVqc8G17r7MijIKnORXReI9Mj1N7dpGwQ3XGa1dPtobC28uJcDigDi/F+nXsCRouozXGfm2g4ParelW+m/wBmWqXttG8zfKVIrWuEV72csoPy9xUWl6fHJckuwbYMjHrUtAVdI8Pae+r3mIv3Y5QZ6GtK1uYrA3EZiEaryu08MKu6Vp72s80sjZMlZ/iK2Fxc2UDlgkhw204OKVhnL20FtrfiL+0LgRRrE5BjAwWrofFup2drp32aG0jkecYGFxjpXO6hpDaR4iSC0BdG+7k81A8d1a+LoU1BykQIKeZ0xilYCjo3im/0ATWVvBmJiSVYfd+lRaVaRa5q8v218Sv82CMAj2qe905r/wAVyxWtyYYmGQw74HpVq1uf7J1sJqUJeEfcnZRgH8KLBEp6lpc1pMIEvPMttwUAHlPrXWJ4VgvtGl0yaR5WRN8bN1VvauF1aBv7YuLmKQi2kbdG69GroH8XMpVrRpLe82hSzIWDAetIpGBH4dl1UGx8qC0uIPvSSdSawdUsbnSJvsk00b5GcxnIrubPTZWuXvNSeKVGy24N0NcHq5R9VnMTs6buM9qBFZMbT6ioizOePypd2OKfHE0mdgzgZNIaREoZR81PiiluZhFCm9z0GcUkSGW9jiJ4Y4Naclj/AGfqKNb3SnjPPagdipdadeWUUclxDsV+nOajt4pZZMRRl27heuKsXj3dxeJbXFyrBnGGJ4FXbTT5NO121El7FDE52mcN8oB9aAK+kWVnf6p9nvnZCwyCOu7sK3Lu01Dw9JIikMsqbQwXIYdhVzxj4Vs7DSYNTstRilcEAojDL+4IrMuNP1jS7C3urudZIWHCK2SBQSO8GTrD4usJJrjYoY/f5B46Gu91No9R1i4077YYk3YMI6MK4e3g0/VNRcORHJ5YZGBwQRRpsd9qOqz3Tali7g+VZCMk4poCpq+n29rqsiadvCxkqWYcZ9Kht2mnSW2l2gj5gyjoatDV7u+vpLS7YDLZ85R0PrTdNtby8u7uKwaNmVfnV+Cw9qbGZXmPv/ezuyng/NSzW+GXYwKMMqfWoZonimeKRcbTyKd5gG1XBK57dqgaI5EMbFHGD6GrFjpbX/m+VOInQAj3pkv2d2z8wqaykht5WMwPkupUlTQBbGp358i5N0RcW3yqyjBGK9W0TXdU8Z+GQt00fnW0gImAwZCPX0NeTNb26W6zwSb2zgx5zxVjT7m50/Zd2d80UpOGQNximB65ZaXHdlpJrm/mZchvOb5eO1eVXOnQubxIbj93DIw2sK6DTvH2tabatcCf7REWGYp06fQjpXLavrd1q095La2/lW8sm9to5Ge36UxEdpLEuhziP93cZ+Zm6MPQVlywf6OZfMTjtmmb/wB1sqHFAAjFXBxmrUfMqr+Jqsq81OMqQvr1oGWreNpZgijJNJdwS2+0uMBjgVeYCKSNogF+Xmq19MZFZW+9xjioAoPzKFq7bTrHI1ueAwwPaqODn3q4bKZDHKg8wMM5WgCxsW0uDCnzqwDfNzg1P/ZNzKkUsIWT7QchQeQfQ1CrzuglkhwVJGD/ABCuj8F3ktxLcW7HhAGU9Tj0pXAoXcEcEkcUoeTEeCF5MR/woj1S7tnjtJzmLOBL/eFTXUzjXX81Uik3FSc8OvaqUitcGS3hzJhs4A6CkFjQh1e60bULa4lYM0T7iAMEr3rt7rXbK8mtNVjudloyeW23l4z2JrjQJL4gvCTDGFSVtuSCfT/PaoodK+xC5sby2c+WciReQPTP+e9aRZEtD1axtJLa44vnnSQZVPY1pEYryzTvE99K+nyESOli4V5NvSPOMGvSl1SyuPnWdF9mOK0QE9OFGMgMCCp6EHNANMB4FSAUxafmkxjqYTTs0hpIZC6hhgis2W3IkJHStVhULKO4qyTLNvu6jmnJFsXbjirjRgdqaVFO4it5a4I2jBqvFGsMjD15q+VqtMu0lh1rRMTRNFyc1Y87bxmsGbUJkYqoA+lVhe3BkJ39afIK503mhulNkGVrDGoyRr1zTo795AcnFHs2HMWZly1QE7aFmLc5zTuG607WC4EBl4pgjxz6U8LtGKTnNAEu4kU0mkHAprGpSC4x8Gq0inmpmbFMLbgc1oiWVigB4NKNw+lKacOQBVskdF8y57irSZAqrFkOR2qyrcVDRoiTJxSCQikLjFRswz1qLDJ0cnvmn7sdaqxMPNxU8hyuaVgGSsGHHWm56GmbgTzRkFwFOKqwFhW4FPFRgYFLuxUNDRIBRiqxnKnGakjmDEjPelyMZKeaaRTxyKTFTYCPbRtqULShRSGQFfSo2U5q2UzUbJipAqlTTcVYZcjFR7DQMixRipStM280gGmkFPIpuKQwxRiiloABSikooAUGlzSCloAXPNKKQCnCkAuKQilpaYDcUmKfikIoAZilxTsUYpCGYpwFLThTGGKaRzUmKQ0AM20m2pKSkBHijFP20baAI9vtTSBmp8cVGy0AR0ooIooAWlpopRQA6lptLTAcDThTB1pwpAOBpRTadQA4UoptOFMBwFPWmAU9etAEqCnikSpAKQChc1KiHjiiJQasqo9KBD1WnGm9qU0hjSaYTStTTQAlIKDSUAPBpaaKXNABmkopKYC0GkoNIBKSlNNNACE0hNBNNJoACabu5pCaTNMB2aTNNooAcDThTRTgaQDs0u6m5puaAHk8U3NNyaTNNCH5pjGkJNITTQxM0bqSigQuaaTS5ppNABRSUUgA0lITSZpgLRTc0ZpABNJmlNNoELSUpNJTATFBFONIaAEpMUuKbQAtFJS0AFFFFADqKbTgaAHA0oNNzRmgB4NGabmjNACE03NBNNzQAGkJoJpKAEpKKQmqEBpKU02gAozS5ozSAKKSlFMAoozRmgBKKKKACilzSYoELilpBS07jCl60lFIAzS0YpcU7CEpcGnbaMYpAN2mjFOopjG4oxTsUYpgAFKRRSjrSsK4mKXFLS0x3ExRincUUAXc0oNJikoAdmimZpc0AwxTcU7NJmgViNwGGGGRS9sDpQRk0qrmgLFd7SGV9zLkmnAWmmo1w5CKBySaim1S1tLjyZm2tWBrd/C84mhYSqRswD/TvSHYvz+OtKt3wWk2/wB4LmqNx490y5nU2kTzsmMLt6/4Vx82oSWcstpd2gIlGVI9Kx5mbTN/lkDJyMHNJyA6vV/G0OoOmy1Nu8TbgxYZqjrXiL+3bGAfZjvhOdwIrndEeOfUybhVZAN3I6Gn6tqcUV4yWcQRf4hjAzS5gOy8J6ha3+6FbJRdxgkOxJLVDq17pmpz+TeTTwvG3GE4U1y+k6pqelXIu1QKjAg5HUVp6le6ffO0qxuGcAtLjgGle42dhbaJHHah7m7S+ij/ANWpXAX8qw9eVrW5W+sG/clf3i7ThcfzqHT/ABLFFBbxwW5knDeW4PT2NdRr/iGDT9CWC7WEXEwxtUfIKTBM8wvvEU15e7rfbECMHaOCfWq8UEcLyPdcGTlfpXSldPstShmuXtns5l646MazpL2xg1RxaWy3Ck5APagDN/scyxJKC2xhkmLkAVtSaNpcu2FS8UoTO9TwxqB9UtI5HmGba4xgIBxWNdXsV1FhpWVl6BOhpDRDenHym3KshwZMfeqluDAZ7VKJpHQRySEp71G8YRwA4YGgCRSjMjSsQgYZweo9K67WNK0CwXT7rT7w3O/BlhYkjHvXIArtVQgGDnjvWnLPpv2Vf9HPmhcZHFAy3fX9jLOquuxR90IcACtOa/i1W0tbPULqNbdMJGyH5s+9Ya2fkaKl46YffkDsRVa2jill3zP5aHqFFAmXNY0ubQL+3mjfzImOVYnnPp+tMg1aKO8N1PB8zDDbe9DLGQ/myzSwIpKDP3TVcXcE2mi3aMZH3ZB1FAGnZ3NvJdCXTbJ3cAl43HB9609FiRZBqtk228jkPmW55DL9Kj8KasqRHT3UCVF3RH1HvVTQtRWHULzgrcylgqNwpNMCtqiRXGqXEsM4ZXzIQRgqfSsyQYbFa93HpkIlivo3F8rHOw8EHpWJuzUjHLG0twsYUlT1wMnFXbiOLYsNqGeUHA44/GptB1OPStThuZELhjswvUZ71sXOm6fqPicwi8+ypP8AMzIcgHFAHPS6fLaOFu2MLEZUDnIqCO5MfBUN9asa/bfYdXltxem8VOBLnIxWarZpgdJca9FfRpb+SttkYeUdxVRG/s95LeznR0lGS23JFZKtukCjr6VsaZexafKZBDHP5ilSC33TTAypHYN1jOe8fQ1EDk0+XaZXdeNzE7fSmrlWyKQDsYGTVuxiFxdRAcrzu+lQhoyPnz+BxV60kgjmkW33FCvVjk0APmLPfm3thnFTaqmWiXHzKvNM0HR2mUX99KUswxU7Dzx6+lEKKHuHZZJYQ/3lGQR60mBmMrRsGIq7aSLIN8lw0W3qFHAqa9SHyllt4C1u4yGYVTtnCJKMffosBeaOS0nDSMHBBwwPDCr1hdtBPBfQwbVQETbe4rHjmd7aOJ2yI84/GrunyuunzbBlUYnOetS0BpSanaec8d7HmNmLRTqOpNVdCvfs2oyy/wAD5XcegoYLL5EUBGxiQwI6VYs7K409JkjTzI2IOFPBHPWkBs6fcztcNJaSeagKl4gOXArY13WZIL1rm2Tdb30QO3HKnvn8a5ywLWupC/05l3RctATjI7ivSdJ0TT9Rso7ma2TcSTg9Qev9asDnfCHkJcaqXBbT5EQSIBwCfWt250O2XS4I7VDMsbb4snnaT936VPp2nNpuqyT2/wDqpAUdO7Y6fl/WtYQeTKyRr+5JyP8AZ9qtIlooWcP2T5IAVhkb5oif9WavYpdjBjg4U9R60u2rFYAaeDUfOadigB+c0hbBpuaRhnmhDHk5qNhTlOBiggGgCEjioyKmI5ppFAWK5FMIqZhTCKpCMe7twGJ2GqDoEXcvWujkQMuDVE2yd1raEjOSMMyBupqxaEHcvfqKtzWUZBIAH0qgV8qTbg5rZSIJoWKy7TV5eKzU67quxuSozUtFJlncDS4FVhIM4zUqHP0rOw7jm4FQs1PlJ28GoA3PNCQwfmMkdarsxqyePpULxZBxxWiJIQ9SrzUf2WUruDAiiItu2kHIpisSop3HAqcKdtLGvyc0FhnFQxjGPFQhjvwalc1XkYK9NAywq5cMKuf8sj9KpwSDaM1ZZv3eal7jRmTymKTB4q0iPkOVIBFNG15V3gE571p7QVA7DpQxFYH1qOSQdBTrnMaHbVTfkZoSKuRSN8/WhJCG61DK53VGshDVrykXNqGYFeTVgOpGayYHJz7mraSEGspQLUi4rZbFTBM1TSVdwNXohuQN2NYyRaY5EGaV4MjIqRVwalA4qGMy3jIbpTClaMi54xUDJzSAq+X7UwxcdKurHk9KkNuDQBlFD6UzbWjLb7RkCqzRmkMr7aQipihppShgRY5pcU/FJikMbS0uKMUAJThSYpaEA4UtNzxTqQBR3opaAE70UUUxCU6kpc0DFzSnpSClpANopaO9AAKeBTQKcKQCEcVGw4qU0xhxTAgIpKcRSUAJS0lFMBRS0gpaQDhSiminCgBwpRTRSigB4pRTRThTAeKkTrUYqZKAJVqVajUVOq8UAPiyKtL90VBGuKnzSACaQ0U00gEamZpTTaAEop1GBQAUUUlABRRRQAUUUUAIelNNONMPWgBpNMJ4pT1ppNMBCaSg0lAC0ZpM0ZoAcDTs1GDSg0wHZozTc0ZqQFozTc0lUFx2aYTRmkpCFzSUZozQAUlFJTAKQ0tIaQCZ5pCaWm4psQUmafim0DCm0tJigBaWm0tABmm96WkxSEBoNLSGmAlFBoNABmjNJRQAtOBpopaAFNGKKAaAFzSUUnSgBpNJSk02mIKSlNIaYwxSAUoNBpANNFLSUCEooopgGaM0YpKQC0maMUtMBKXrRilxQAYpcUuKXFACYopcUYoEIRSgUuKdTsMTFAFONBphYUYpDSA0E0CuFFFFABRR1paBhijvSijFAhaKKWgGFLSYNOAoAulSKQg1YK00rSQysRScVOycdKiK4ouISkxQeKTNFxCUqnDA0ZpuaBs5jxTaM0ss+3K7civPWsZ3vUVrV9hYZKNkV6/qMK3NqUas86PHF5U8BIZcfKeR+VAzzzUdNM17DDItwYiAFyegpniHRI9DEIjnMqSZ+Vuo6f413HiPTxewW8SSmGYNuUgZrnZ/CNy1m8t3dvIVGRWbQHIQwqWGB8zdB61G0KGRg4eN179q6xdQ0vTrIIwEjr2K8n8az4dY0+aKa2WzeZpux61NgMGa9uJV8mWXeB04oiebyfLDfJ3FSPam3ucTxtGx6A+lQtceRMR5bMPbpVItmhp2ovpUzOsaOjjDhuvtiotSulvrbdLKz4+6D2FUpHy2aZsZxx0piKoUZBOTg9zVja5RpVzhRTGUKcVGZHU4B4qQLMMYmhLy8p79RUUojQDY4K+wqNnbbwcVeWCNdLEkgzI3NFgRQ25Ge1MJUEe1XRDC+nmTzdsqHhMdaZDcwbAJrZZB78ZoAljji3fabaGaWMff46UqWM90oljZFB980kWpzWqyx2Z8qGTqnXj60mnXsdozLdRCaJs5GaBlnUp71Z4g53qi8svIqiJgshYqD9RXRX3iazl0oWMFkVQ9W6muchgMqSPscoh5bHagC1b3UCOCxwjAhqjNvDMkstu4VI+gPelsDp4lc3RIXadpx3p9pdy2TyG2CyRv1VlyCKANSy05tTurd9Bjk86EBpg/AXPetjWdFg0ee0upZke+ZstEgOGXiub0XWrvQ9aWWJWHmOFkjIxuB7frXW/EjTJ5rm2v7VWaJogSAOR9aCDjNZvI77UWnjRkyACD7VQBxmlZNnD/AHqEV3P7tN2OvOMCgpG5D4fkFpaX8pD20hBYj+EV0MXgbU721mOkSWMtorblQy5d+M4BrJ0a8gtrUWt3IeJQ8anlJATyCfyr0N9a0HwjrttcCAl5Y/mhg5H1xQSeT60lsWUQwPBLHlZo36gjH/16ziAqgAV3/iiwtfEl/d6zp+oQRLIATBIm1yQP5mvPdxYc0FJl7SZLKC/338XmW5GG4yRUN1JB9tlNkrJCxyM1AOlKOKYwCk9OtLtJOB1qdHt2hKNGfNPR6c2VK5A49KQFUttOCKt2OxLeeTuDjFVpvnl3IpHHINWLVXYGNR8vfNAGkgj/ALOmjLMjN82xTwapW0z2/wA0ErAMMFW5GKmEMrTqSyF2IVctxk+tdY3w8+z2Ukt1eiOXP/LNtyAn19PrWfMByU9x53lxIuNowADxnvSoIIdNkMi/vy3TOCtPNn5Su6u0oDHcUX7oqFoWuIBMsZZASCR1H1q1IAWHzIjKXC45JPp2rVsUjvbG1hWb9652FemT2quAskVvKVG2MBPKAwSveoHEunXlvNbuHHmCWNBwQR2NSwOki01LC+a3uIGS4THzjn9Kfpdt5WpJby3GFncBJF6A+9Nv9fkvpW1FLfDPGFaNj0I/CorKSSezilt3AzIBKDwUIPWpAdc2F3FqV8UgKtCuXCDIb3zXS+F7jUtL0QX0jM1vdMV/ek5iI5GfY1P4ctpda1u8T7QUSWLbMpOSwPUiu2+wR6lpsun29xEiFNmGPIIxTAlii+0WNndRLt3Lv/E1aYHbzVuzhUaTaKo+5EFP1FRyxcVtcRT2im7an8ujYaoRBsNJg1PtpNlAWK5pD0p7rg0wigENzijNNJwaZ5q5xmqAeetAppcetN3jsaCR7DioWFS5yKYw4pDIjUTKM1KaYRVJgR+UrDBpjW0YGdoJHrU6HBxSSCtEyTnbqJ4pSR92pYLgGPaal1JcA1nRnArdamRM8n73OasRSHH3hVFjk03JBp8oXNcnMdRAVUjuGVAGJNSrcjIOalxHcs7SaaVNPjlRhnNSqVakMrIGDAHpVny4uuOaUqOwpsiZTqQRSKFOAKgYDdxUkaO4wTTGRo3O7pSCwx0Zl4qu6HHIrQjIZeKr3UZWqTE0VkDAYFXs/uRUUcZZOKsLE3lgUmCKUiknI61etpMxBWPzDrTPIO7GKl8gqMik2UOljEqEZxWaYjG2GFaQbA5qncENyKaFIoXCA/MBzUKId3IqzMdq1HH8+cDpWqZmSwZVvap2YhqZEQpAPrViTZsJqGNDIyqnLnArXtnBiGOlYSsXU1p2PyLjJIJqJrQtM0wakB4quGqZW4rnaNEP2gioXX5uKnB4prLzxU2AYFFOphPNAPNAx7oGSqbxc1aJprAEUAZ7pzxUTCrjpjNVnU5oAhxRTsUbaQDKKXbS4pDI8GlxT8UbaAGilp20UmKACiiigBaDQBSmkAlJRS0AKKKKKACnCm04UAGaXNJS5oASmk8U6mE0ARnrTacabQAnelpKUUAFLQKKAFFOFIKUUALTh0ptOFADh1p4plOWhASCpEqMVKlAEydRVhelQouAKmQHNAEy9qeTTQOKQmgB2aaTSZppNIAJoFNJpwoAdQaKSgBKDRSE0wA0ZpuaBSAeTSUlKaAEJqM9acaYaAGmm08jmkI4pgMPWkpaSgBpoooNAh2aM0yloGLmgmkpDTACaM0hpKBC5pc02loGGaTNFFKwBmikpcUCExR2paMUwG0GnEUhoASkp1GKQDcUmKfijFMBlJTiOaTFACUlOooENxSYp1IaAGmiig0CGmkpTSUDHUtJS4oAWilFBoAQ0ynUygANJRRTADQKWkxQAHijrRR0oATFGKWigQ0ikpxopgNxRS0uKEAlGKcBS0hCYpQKXFLTAKSnYNGKoYmKXFFGaQBRRmjNMAozSZooAKKSloEApRSAU4CgAApwFKFp4WgQgWl2ipAhpdhoGRbaXaKkCGnbKAIggp22pfLpdntQIuYpMUmaMioKGt0qF+tTHpUbcmmBAfSmVKwqJhiqExCabmg00mhCQMaSkJpuaCirdW4m1C3ZvuqeTWX4nmlZTFCxDBQDg9RW7kZyaq3NpDKzTSbfck0gODl0iwt4oDKUDuckSnkVSGt6XpksgtIAWIx5ijIrR1XS5dc1MW9m+63jGWkboKwb7SrKCeVYJXLoM/41kBlzXU2p3wkmbGTgnritlols7JbdwskTHHmEYNQJYT6VapfyJsLHFZl1qc964R9oTsoFMdxl6IUujHAxKjqTRARvVdwC45qq/Ei/SlBQocplgeooGSOyYZeozkGqpODTyjBA/Y1CSSaQEjFSuFzTjLM8XllyVHTNRgU8Ng4FNDEJJGDQkUjfdXIpySRRyDzELL3xUzOu7MIOw9AaQiJrd0GTj8KfFGisskmGQ8EelaEEUbQ7izDjlTVbaiu2F70DILoRpJiJgR6DtSw38sFvLbrtKSdQR0NXJpA1v5awKW7OBzUEHkJBJHPbbnY8P3WgCiK0lnjjsYSs0e5W3Mo4OBVRkD8RLz6VPBpklxC7PF904xnn8qANvxLq+j6zp9rd2gaO+iIEgK4yOx6+1dfH4wsJ/AObmeOW98swtCD8ynpzXmj29rGyE5IJwRmrF7o7WKx3G4G2mXKMOx9KBNGdM/mTE1qeHNNbU76WH7QYIypBl2hgD2yKypBg5Favh1VnuJ4ftklrIy5Up0P1oA6e3E2gyyabeRxzxJyssicE+lQa1ZSaclveuy/Zh8yheqk9ADVLTNQlu71tG1STzrdsqrMPmB9jVK8ZrR5bWaczxQvgQschh2zQIovqQud8dw+9GOVbuD9arSCMfcH196mu5bOdB5NsYWHfOc1H9qPleXsQqOnFAIQAbfemFcqexHUU1iScDOT0ApMhmBwQ3QjvTKHr96rStvTABDDkMap7vmxgj61atU86YIxIGM0AN8x5JAXOSBiniR0zsOM9aVEQ+akgPXAINTQ2kr2zNEM7Oo9akDa8G+G08SXN7HLPLCIY9wdOmT0BrovB7yx3Or6PIr3DhGWPJyG29qrfDK11CSbUrq3VGt1QJcRnqeD0/Wnm2k0+/t7zTY9l3bnMm48SCjlJJbizv/C+hbzHCzXsuI9y/MhPY/0qi3huC3u7qxeSYTSQCaMbu/8AEPzruZ44/Fv2e5vYvs0dkd7Jn7xPOc1594rur661mS/J2RfciZe6imo3GmNtdFtnWGNtRG6QfdxgqfTrxUMmj29yxVJzFMQfLJbIyOx9K9O8N6Jp1zZaHqptQryNloyPlJxjNbqeENNiu9QgkgEkMo82FSACh7gGk4DPHo9EurTS1umKT27DO9D0PcEUWtqbGW2BTzYWxOzIeQv0r0nw/YadP4ebT5jmG4m8tyP+WbhuAT2yMVsHwhpyzvDDEVVI8Kw6qPY0uUDmPA2izKk2t20D3OHKxlGxhfRhWXd+KNd0DWJESzeLEpkMbruOD29q66HQ5/C32LUdKMhSR9l1E7ZU56H25rcuNPOoXTmWICWXazMONgHvTsSLoGqrqfkTRKUS4jIdT1Rx1H61syw84xVfSdHi069uSh+SQiQJjhWxgn8a0pACxNUMypI8HpUe2r8qA5qqUIancCIx+1RFTV0JkU0xDuKEBnScVWZ8VpyQKT0qu9krgjkfSqRJntIGGKryAbsirTacYmLKzH2NRyxbEBPerQik82w4zSLMW6GoLj/WEd6jRyta2JNaOTIwTTzWYkrAg5qzHPk4NTKJSZKw5phGKkZgRUbHipsFwAyalkX93kdRVdWw4zVndlKYNGDqPmlxtGaqyWpTbtXGRk/WtyaIMSaoTxv2FawkQ0ZPRqXAp7xtvORimEEcEYrdNGdhrCmYNTqm4470v2aQsML1NF0FhihlOau28hzk0jQFV+YU1RtFQ2UkX0OcGpOCOaqJKABnpVsEGHeKhlIFHz8dKc6bhVdZzuxV1SGUYqbDKgQKeBTgu/hhTypBxinIvNK4xqw46CpVQ45qUDAoFQ5DsRheeacVyMdqdSilcdjOmicKQKotFN8xLDHpW66Bhg1WNuN3IrSMiJIwtrO+0cmra2rQxdOW5NaQtY1bcFAp0iBom46Cr5ybGV5TN0FLsOMGnpJhyMVIACvHWquFiEKFXjtU9rKKZ5TEU+GAp0FKWw0aoHFSKcCoYSdoz1qYGudmiJAaUtTAaQmoY0NJJNIDQetJ3oGPzSE02l6ikIjY1CwzVgrTSgoGVCtJirBj56U3y/akBBilxT8YoxQMjApccU/bS7eKQEeKTFS7TRs9qAIiKAKl2e1BT2oAjxxSVLtpNnFAEWKSnkUwigAopKWgApwptOFIBc0lJmjNABTGPNPpGUEUgIqQipNtIaYDKKd1oNMBKKKKEAtLRilApAOApaBSgUAKKeopBTlHFMB61MgyajVanjXGKQE6rwKmRajXpU6UgFxTDUmajc80ANJphNKaaaYC5py00UoNIB5NNzSE0maAFJpDSGkJoAUdKKSlFADhQaSimA003FPIpMUANA4pDTu1JigCIikxUhFJimIZim4qQimkUAMop9GKBkdFOxRigQ2inYoxSAZil70EUGgBppKWkpgLS0lKKQC4paKKYCYpDTs0lIQmKKXNNpjFpCaKQ0gA02gmkJpgKTTSaCaYTQIdmikozQAtJS0YoAbSYp+KTFADR1pwpBTh1pgPFIRS0GkBH3pCKcabTAb3o570tHSgBBQaUUGmFxKSikpALRRRTEBoApTQKADikNLRQAYpRSYpw602hCgUoFKBTgKQCYpCKfikxTGRmm1MVFMK0wGUU7bRigBtLilwaXBNAhoFOC0oWnhaBDApqQKaeENSKlAEYU1MicU9UqRE5poBoj9qf5ee1TBPapkgBGTQBS8r2pdmD0rQ8pQOBUTxjnikMqYFGBU3l0vl+1AEWKaadRioGMJpuakKik2imIjbpULdasFMimFPammMqtTCDVkx+1MZPamIgppFSlOelMIoGiExoeufzqG6gjaxnUA5KetWSKbQBn6FBHb2HyDG7Oa5LSNM87W79bhcGNsYrt40W3DBeFJzj0qNYlNw0wUAsOcd6hoDlNfthdR2unHj5s4rEuPBN9b3u6CAvEvUk9BXoFzZo9xHMiYlQ53VpktIdwHWlYZ4/caQk87M8Z+VeAg549qp3NgtvpLTLGd27aB3H1r1TWo9OtY2uGlhiuFBOc/ePHFefy3U+uTra2tqNq5ZincUDORduMVHXWxRaJCztJGZJkOAFzmsfUYn1W/muLSFgnfdwcikBmxxl/u9qnaGOHbuYMWGQR2ptjHNNM0UKb2KnvitfStMFrFcTXyIygY2ZyaBmOwANWZ7UxWqTxTI+Rkp0IPpVeZ0klLxcIegpm7FAD2mc4G847igSdqiIIGaAe9AF1ZJY1+RsD0pxnDxkMeTVeOQnhvSo24OKAJFJU5Q4bsalEkrzxKLhi5ODgYxVZXwa0vD1zbWuuRzXbbYSCCcd+1AEU1tGsp89hKfUUt5c+darbRs6qDny25Fb99o8UdxNe+Wj2cxJWTPKH2rlJmHnkocr2NADSh2gGtbQLm2sZpHlUiUr8rjoKy92RTd9AE8UrSag9xdP8xYnOM81YuZLKb9+FYyqRuB6OPSqI560hXNAixqJtGlR7ThWGWT+6aWO8tIoWQ2oZmGC1UnGGxSoOKAJbWTyLhJcZ2nOKbcS+ddSSYxuOcelSw2skkDzoNyIeT7VBJsDZXvQAKSalgcpOrDrnFQocVJECX3dlNAyyGBlxn5mbJHpW94cs5Ly9uIIrlIpQu5VYffxzgVzhbEjkDnOQa7OytbK58GwX0UW3Vre4+WUHG5c8g/pSA6bwadQ0fxAZpIH+zXS+XcdgD2b+fNdXJ4Z/tS/j3rGkELHBBySM5pfD1w17qMSS2ptZRD+8QkEEY4Irq4ojEOlAGcugQG5uUlwbadQpUewryrVtAjddVlgkdbe0YJGDyCc9K9f1E3LWhS2bazcFu4FU7bRrSPSJdOdMrJ98nqTWiAoeEy0vhi0tpkCzWDCP5enfBH4V0t1dra2c14yhvLGcd6wPDUEtrDfQTMSUnKjPoK2LiAXUaqRlQckHoaSA8p1nRb7TtIg1m2aZYppC06qcBWDfKcd817DbMtxZWtyvJlhRifcgGsvW9P+0+HJbFG2pw2cZ6HNatm3+gW/wD1zX+VKTAlKq8bRyKGRhgqe9EaiKJY1ztUYXPXFJmnLzUibHrwc01pDnFOHApjcmgY5uRULLmngEil28UwGKuBSMo20Hg0A5FAivIDzTY8g81M65qPaadxCzKrp0rNuLIyN1xj2rUCk0FKLjOYn0a4kkLqykfWqlxps8Kg4zXWtGRTQiupDCteck5mKwm8oEpUE6NCMkYrqJIljHHSsi5hWZ2U1XPcVjLjuiTgmrKtk4qtJaFJOBT9+zGetWItPGAMigHjFLHIrKKftHaoY2RkZphQelT7RRtGKVwsVXt4pBgqPrVc2EQyu0kdqv4AowKrmYWMp7JY2IUYPrULNIsmxjkY71suoYVVmhUgcVfOyeUriNpUy3WmPEEGDxVpcgYwRSMAwwRRzCsUFj3HFTKCqlD0qYRgHildQTnFHMFissZLZxVqIlSBTVHOKlC0XGTZDClCAHOOtQb8dKnjfI5qGUh+KSnZBpKhjG0o4pKWkMWmlcmnUlMBMU4KMYoxThS5mIpXFuokyqYz71GsYU1oOu4Uzyl9K05ybFJ1IYcVPGoKipdi9xTgoHShyuOwqjCjFPpBS1BSDNBNBNNNSCFpveloNAxCaM000maQD80oFNzTloANtLtGKdSYoAgeLk8VFsq9sJppi9qTArLEfSl8v2q6kXHSlaA9cUgKWz2o2e1WTCfSlEftQMqFDRtq0ye1M8v2oArleaYwqyUNNMftQBVK0wirZjFRMvNICAijFSlOOlN20ARig8U7GKa1ACZpKKUCgAp1GKWgBh6VGalbpUZFACUdqXFGKYCY4pcUuKXFACUoFO204LSAaBTwKULUgT2oAaq81Kq8dKckfPSp0j9qYEaJz0qwq4FKseO1PC0gBRUq9KjA6VIKAA0w040w0AMNJQaSkA6jOKTNBpgGaM03NGaQDs0lJmjNADgKeBxSCnimA0igClNJmgBKaadSGgQ2ilNJjmgYlGKdilxTAZto2Cn4oxQBCVx2pMVPtzQUpAQYpNpqbZSbaBEWKSn03HNADSKbUlNIpgMxSYqTbSbaQDDQKkCUFPamA2ijBoFACUU7FGKAG0007FIRQA3NNJp5FNIoENppNPxTCKBjaMU7FGKYhlLS4NJQA8U/bxTVHFSAcUARkUmKeRTSKQDO9LQRQKYD8UdqO1FIBp6U2nUhpiG4pKWkoGJRS4oxTENop1NoAKXFGKKAF7UYpQOaXFADcUuKdtNOC+1MBgFPC04JTwtCAbtpQpqTYaeF4oSAh2Gl24qUjFNIzTAjIphFSGmEc0CGbaNpqQClCEmgRGFNOC+tSiM04RmgCIJUix89KkEftUipTGRiPnOKkCU/bUiJmkIjCn0qWNGJ6YqVVAp4FADhgCnbhTKQigCTdTTzTc0oPNA7hto2in7TQVIoC5RpajR1K43AkdRTxUALijbTlUnoM0/YR1IH40DIttOEdSBO/anBaAKrw96hMVaZTIqF4sUXAzmj5qFo+avsozTDED2pp2AzytMZcKT6Cr5gB7UwwKAcjIx0p3A5m9nhtrZrq9kKoOir1Y1m6Z4qm1C4McNji2U4MkhxgetHiexudRjijkkVFeT5ETrj1rJ8RQroekw6bb/62QZkPc5pMDVvfH+lWl00At5ZtvVkPGayYtS13VrpJLS9a3t5jhRkcDvxVOTwjEnh+C5uQ4mlIHyjPB710lnpv2G+06Jx+7VR83elzAYWpafptlevHema5uGXq7ZBP07VteAdHihgm1AjLS5RfYc1DqlgLzXbm4VN3kgZT1Faek6nZ6dBJDAxeH/WdMCMntSuM5q/0628OeLPPnd/s0vzxnGcE9v1rnr2JZdWu7mCXy7bJIbNaPiPVpPEWpoJV+zxRr8m8cEj3rknldWIDnHYigY+1ke1u3ljbHGAadcvcTP5slyXD+hqG3iuLlysa7sdaZ5Uu9lC8jrQMdsYIScACmc9qbkg4bt2p4IxSARQTQeDinxttcGmOAHJAwKQwyamiHmKc8kVEoBOCce9OGVfgjmnYQSLt5FSRGJDHNIokCnJjPek3KD83TvUTLtOB93t9KYG7f8AiJ9Rs1tRbrDEuMbTyBWRsVmwWx71CpYnykQtI3QCtG80p7G1imklXdJ/yzJ+YUAZ5BDEdqTNW7S2a5dh0UDJNLb2P2m78kHHBOaQEMBi2SeYG3fwEVMY18pXU8N29KZNGIpCg6Dio1bDUgJpLAmIzhs4/hxzUccAa383oO9WoL0rjG047MOKZfvPE6QOiJEwyFUYxTEH21v7P+zIxDdyR1FUSKsTiFHHlS7wV9MYqLG8jCgHFADMHtU0JzkVHyvGCDU6gxnPqKBjiOB9cV6Z4Dh2G70K7EM1vLF50Kt1JxzgfgK81iRptwGMKMkE4q3p+q31hqVpfxSyM9qcjb129xQB6l4R8Rzwa9Z6fqIKbHaLzX6KDnANeqyDaxFeHafd2vinx2si28ps7o7SWGCHx1/MV65oum6lYW5jvNQeccgK4ycdjmgC8emKjPFTEU3bTJKZRVkaVRtZvvY71IspAp0keRVN2dDjGecVSYF3zA4ZCeGGDU8R2xqg6KMCqUMeDuByPeraUpK40ywlSAkVGpxUwwakBc03+On4xTT1pDJBQaYGpS3FAEUg5poFK5yD601Qcc0APApKUUuKAGUhqTApjCgLDDg1A3ysCKlLBWxTW2sDVCIphujNZEo8uU5rVZuMVj6nFI5V0bGDk1pEkinG5dycn0rPlb5ueDV6NWCdaq3cYX5yK0TEwhPQ5q4HB6GspJl34BqzHMM9atom5fHNLiolkG3NOEnvWViriEUhGBSk0jHigBuajbrRuIOaj3Zfk1SQEvlgjmo2iwanQZUU4rmk2BUMRJ9qQxHHNW9oo2gilzAUApB5pwarLIO9MaAYyKtAVx97mpl6U0fN8jLyOhpu0o554pkkyyYbBqQHNQHbjP8AFQjkNg1FikywBQBSA06paKuIaQUppKQC5pc02ikMeDRimg07NMTQ0ilopBSuFh1LTaWgBaXFJSikDDAo204UhoGREYNN2nNObrSiqJG4pwpCcU0MM0WGTjpS4OaRTxTh1pAOUVIFzQq8VIg5qWMdHH8w4qfy/aljXJxVtIMqO9SBReDKHiqrJtOK3zCFjOR2rMlh+Y8UAUioxUbDmrLJtyKj8slqYyDYabtq0Y/amGM0gKrrioiuTV0x5HNQvFg0AVWGKjNWHQ4NQlaAImphGalK03ZQBFinAVIEp2ygCMCjFSbTTSpoAjI5o2in4oxxQBHtpQtOApwWmAzZS7BUu00BTmkBHtpwWpAlOCe1AEYU1YijyRSInPSrUSYIoAVIsHpUyoBS4FOoAQqKaRUhpppANApc0lJQApqNqkNMbpTAjooxS4oAbRmnkCmGkAhppNOIpppgGacvWmYp6jmgCVacaRaDQA0mkoNNNAh2aCaSlAJoGNxTgpp6oMU/aaYiPaaXFP2mjYaBkWKMU/FJigQAc0/FNA5qTHFAyMjioyKkam0CI9ntSEVMBTWWgCAigCpCtIFoATbRsqYJThH7UgIdtBWp9ntSFPamIqlKZsOelXPLppQUDIAntQU9qsbaaVoAg8umlQKnIppAoAg25FNK4qYjFRkUAREU0ipcUhFMRFilxT9tG2gCMijFSbaNtACqOKdiilxQAwimEVMRTSKBEOKMVIRTcUxjaXFO20uKBEZFNIqQimEUhiUmKdjml20wuNxSEU/aaaRQIYaSn4pQooAbil21IFFOC5qgIwpp4QmphGKcEoAhCU4Kal2UoFKwEYWnBcU8CnBSaaQDcUYqTYaXZQIiI4puKlK4ppFAEW2jZUmKXFAEYWnhacBSgUxAFpwXJpAKkUd6LAKsftUgQUm6lDGgB2wAUoxTc0ooAkGO9OyO1RZNAzngGgZYRM1J5PuKiQsCKnGTSAjZMUzgHmrQQHr0rPurhY5NqJuHcnilcC6pBFJKNqbqypdesrKD94skj/3VFc3qHjC7nlVYITDGM/ITzRzIBkOrQpgQywSXEjdPMGR9K6eCKfygXWvn4zZ5EbxHsQeRXRaB4l8RwTCGyujPH/cnbNSM9MvLzUbu8NrYTJGo6jGCDUkHhj5995fTyy98SYrkE8XWOoXHk6pBLZ3wJBmt84P1rWsvE8FtcC0uZbh8/cdm5xQB0jWd7YIWsZRMneKU8/gam0/VbbUXMOTDcqcNDIMEH696q2+rtKT9mQz47L1pmoQ2+pBHltLi3uEOVmQc596QG8QF+9QF3CuS1vX9S0vTQtxYfakY485R0rqdIuodS0q3ubdw+5fmwc4PpQBFPCQcgVWBxW20KuuCKz7izKnKCgCkWUU1iCCBUptn9KiMLjqKYFKOzt438zywX9TXP32kf2hePPJxk4AxnArqWUioDHTAx9Us/wDiUJHGOIyD+VNkha8ihdBwACDWncRNJbyIB1GKjjRobZARggYpNAcxqd0LDTbudjt3NtzXKXGo2kssdvBNMEK/N5Y7+9dxqmj/ANoW/kH7pbJrEns7TRr4rFD+9Mfyog5JpWGYXixI7iGw+zgHII4GM9K5W5tPJyJGCMOx71r3eqS3N4zaiTD5TYQAYGaytWu/tty0i9D0oGV7VpgziB9pI596ZlhySQ3rmrhS2a3jaAlJx97PeqssbxPtcYNSMj60opSOOlGDTAUcEH0pZPmJI6GozTh0x2oAaaUA1Mtu+8BkIU+tPK5bYrAKD065pgQybcLjr0IoIKnB9M1dtVso7kPeRl48YwO1Q3n2drpvspbyuwagBdMe2h1OKa6LhEOQV7Gr+v6iuqXquhRkT7rKuCfrWR5Mp5C5FABXrQBo28jRWkhXuMUljM1tfQv6nB+lSWsayaBPcbstG208darsvnW0Dhgr9ee+KkCzNHGLx2kXcmckVmTmNrhjECE7AjmrxlE9sYkObo9F7n6VR6/e+/3B60ARvkRGt3Vv7Lk8PadLbT7rwgLIvUr6isUrkU6L92Q20H6imAxF6E1Zh/18eR8u7mkd4i3AI49KsWCGRvNCnyh1Y8YoALmNjNkLkdzUOK1P3Ul0AkiuuQGA9PWpriGygW5VoYyCQY2AyQakCva2dpLYPI29JVxwOBk0SW97ozYlTY00eUbqDVzQ1uZbiIWaec7Ha6EcMPSr/iRf7QuBEqvG9sgDxuOVNNMDr/h34Wm07xBY6hcSo/mxMWRRwjHocV66/UfSuJ8G2tzppgtGPnIbZJYnccqO61163AkAPTimArCo8VKRTSKAIWAqJ4g1TNTO1NCuVA/lsecCpUuOjZ4pJYgQTUP3PvdKoRpxyBhkVOHxWPEzJ8ynrU6zuTgmpaGawcGkY5qoknFThs0hi96Q9aTNIDSEOpwqMml3UxklLUW7BpWlCLknvzSAfQRxRkBdxPFKrBhwc0AVpY8DIqu2avuuRVVk+c07gQbciqsyKyEEd60PLb0qGaA+WeKpMTRjzDauR2qhNMrKVkFa7Q7hjFV5dN8xcYx71cZEtHOyJsbcp4NJHJJvHHHrV5tOnLgPFwPerIgVR80YBro50Z2KrzkRjmkjufm5NWruyKw5RQRnsahex+QMvFK6GTJKp71JnNU/KZIdxOKZHOVGCaXKFy057VWkzvBBxUykP061HIO/enYLlyI4QVJmqUTnI5q0G5rOSGP60YpyAGnlRUlFWUkDNMWYMoFTTL8hrMlDKSVrWImXHG35sjPtTHIMZI6jmqJuSODT4rg59VxzVcpPMENwJGIPY1b25OR0rNeIxyiRCChGeKuW8v7sBqOUVy0p4p27mot3HFJu5rNopE+aKjDGnA1DRSY7FJS0uKmxQlApcUYoELRRRQMSlpKWgBRSg0ClA4pCFFFFLTAibrSKcU9hUTNtPNNIQr0xFJbpT+DirESccVQEWccYqZBmn+Rk5qZIQAOKljAI2OlPVean2gpQEGahlE0QGRV+HG0VRjq5CeBUgTSD5KqyRZXOKtlgVphHrSAzXt8nOKjMO3tWmQtV5Co4xTAqGPjpUZjx2qyKY44oApMuDUbDip3PNV3PNAETIKgdOasE0wigCoUNJsNWCtNxzTAhKmjaam20hFAEWDQRT8GgrxQBFikxUhU0gU0ANCinhfSnhKcENADNtKFNShDShaQDApp4SnquadtwaAFSPjpUoXbSKMU+i4BmlBpKTOKQCk00mgmmk0wFNANNJoFMB/U0EUoFL0oAjxSZxTiaYRzQAhNNNLSYpAJTT1p2KCKYMYOtOHWilFAMlHSg0yloAU0ynmm0AIKlUcVGOtSr60hEqrxUqx5FMRhUoboBTAPK9qjZOat8bahfrQMgKe1NKVMTTSaBEe3FFL1oxQBEw5puKlxRtoAYOlLjNO2mpEjzQBAUPpQEFXGh4zimeX7UgGInPSnlaljTnkU9k46UAVitNKipylBjpiK2KTbVny/ak8v2oGVipppU1aMftTCgoAqlTTCKsMKiZTQBCajxUxFNIoEQkUmKftNGKYDcUYp+KMUAR4pcU7FLimIaBTsUYoIoGNpDTjTTSENIoAp1FMY3FLinYpcUCGEDFRkVKRTdtADQKXFOCnFOC+tUBGRxTdtT7RSbPakBBs9qcEqbZ7UoT2piIghqRUOKmVOOlKEoGNC8U8JTgppwFAhvl+1NMfPSp8UYoAhEfNSbMdqdilJxQBGaQA1IoDGpNijtQBWZTTMVaYDPSo9g9KAIcUYqUqBTDxTEG0UnANIXqFmO6rSFcnyM07eKreZxSbzTsK5b3Uu6q6yEClMlKwXJw9OD881V8wDoacrljRYLmhEYywyathIz3A+tZBcoM4pst+yxsTwAKljTNYy2yybfMyw6gDOKtxxo4+U156/iRopNscZIHYitG38QTy8Bin1FZcxdztntwsZcsAB1ya4rxJrNvZyGONt2ep6elU9Q8QXFwzQJcmTpkIKwL1YWG+UbpfSspSC5FNq8szlg1Rm9g6yDL9+ay3mZpDkADsAKcqq/JHNZgYWmRf2rcGCdzsAyuOxp2mWKtqskJldYo+jocGtPVdB/saW3awk3zMPmTqSa17PQUOmJI1vtuW5kZmIH1rcox/D+kx6hq8z3BMkKD7zdQfWsvUrW8u9blt7VprrYcJtPQV266adPiNvYSo7XAyQx6Gtbw14eh0K2lurrC3BGSRzTTJOWsJdX8KrDcXSuIpCAeeRz6Cu5uvG1tbaFJe2p82VVOEcYxXET67ZXmtzXGpuy26nbGSOD7Yps+uaJdP5NrAHPbnbmmAad8RriLzxqkDXJc/Kq4wo9MVXTxYbaaS70u+lsJnOTCqfK1UNfvNOjMTWhVZkHIA6Gsq2v7eS9WfV4Hmth1K0gPY/DXiHxFqOmx3TXmnXCN/z0wp/Gu4imWSOP7VGIncZG0/Ln0r54u5dEuGB0dbuHvIi5IIrrPBfjewWI6Zc3BWVeI2u3O0j2znFAHrk1vFGu+QBV9zWXJqGknpPu/wB0ZrktV1+K0QTXsjJbv/zyk8yNx+HStXSPG/hcpDG0lqok/i6lSfUUAaTm3mGYGLL64IqtJF82K6l7VLuBJLa4jCsMq8ZBBFc3qcGs2lrLM9xZLFGMmRlP8qYHLa5PJb3nl6bdf6SeXjPK44rPbxhFYzrDrEKqW/5bQkEZ965rW/E1/OJYZRbhgceZD0Nc5eabqAgjvJVzbt/HnpQB6w/ibw+VDC+XJ6YUk1yus+IoLG9M4CXTyL8pU9P0rkHuGgt0hATkZ3L1qiTukVR1Y4FSMTUtQ/tG9e5aPy2fqoHGaqRpucA1rvD9jUx3cCu0nzK4ORiqf2dY5F2tnPvTTGM8tSpI6qabcSNOF+Ucd/WpySIihZRjmqwL+X14zUjBQNpHekUryGpy7RyVbjqandY5FURL0Hr1pgVPLZj8ozRJDJCAXGM9K0dLsWuZC75CJnIB6026sC0iTJL+6kYKN38NADLf7HJab7iaQSA4Cg9aqvsVsoTg9DVnU9POmyKplD7hnIGKpAF15PFMCaYgiInjK1ERhqdK2YlX+7UQJqQJOvG4j6UjEnOeTTgueR1oZdpywxTuBNaT3CWE9mqqYn+Y556Va0yKfVHgtYiqbSTnbwB71a0nSZ71okgnVPPU5DLx+degW39mabpcdvbNHJfwZXhcHPvQySj4ahsNEiupr6GCa6hbdG8aZcg+1cV4oNpLqX221WVBcZd45Bjac9q9R/sez0/RHury1W5lfLMcYIz2FcRqHhHWdSsH1BpImjhVtsIOSVHpSGmc7YxRwSq2pQOlvKv7t/U1RJUOQp3KDwcdR61oanq/2/TrO2MWxoOp9wMVBYmFL23knGYg3PNMZZ0hGubh4yhZAvdeB+NWVt73UrP7Na4laNuY4xyB6mtiDUbW0e4u4oxOxOHz8oRfbioLXUf7N1f7dASLS74ORgAf5NAFXTNLlttMTU3RHieQwFf40anz2Evkyh1Bmf5kweCK6KwmhfQpY8KVF8JRkZwOai1W1sotZN1Zxy3KyEFYYkJRieuDQB1GheCoNK8Gtq8UpkvriEMg7ZJ4A54rL0wzDWb+71a3QJMn2aZMZKtjrxWl4Z1C71bRTpMsEz6dBMu+UcGFepU/Su/0zwrpmnzlrWWWVJBkh8EZoAsafp0VtLDdp/z7LGMDjoORVTUIBbXscyn91K3QcBT6frVDQvECS6neaZINnlSsIeMZA6itrUIPt9iFjbADhgcdxQBbDiRFdejDNRsaWPmFe3FNIpAMY03NK3WmZpoTA1BJEHGM4qbNBppjI0Ajjwe1V0m3XGB0qa5Vmt3Ve4rLsoZoMrIeetUiTZWXawBNWFk4rJd2PGcH1q3A58sZOTSaGXg9KDUANPDetSBNmimUuaQyOV8rtH3j0rFGrSxXcVkyCS5dd208VttHvwQcEdGHUVx2q6HardPf311IJYoy0cm7B45BB/GgDodW1a0sURrncemABk4NSWepx6hpM1xa70TBC71wa87XxRbX/kPqLTmWNdqzRjqvbPrWU2qTyXoRJz8r7FlDkBl7bhU8xfKezWt5BJpqXTTqImGQTxmrIjyqt6jNcHDrkd7oLabb2RHkoB5mPlI9RXVabrP2mC3DxOmUHLdCe+P8OKCDYWNcdKR4lIxilhlWZSVzgU9SHGVIIqgKRtOelILT2q8VoC0AZEtp1FZtxaYyMV0Eq8k1QmXLGtEBh3HmRQ7UGfaooGLx89fStOaEMCcVmopWZgRVpkWI7mFjEcDJrOkiZVJxW2wytVnQEkYrSMrEtGTHMQcGraYcZqvf20kS+ci5Udaggu/lArTcjY0FXBPGKnTmq0c2QOcirCHAyDxUtFJlhWCjmgy+hqs0lQyTBRwanlKuW5JFKHmqMveoTdGo5bgstXGNiXIjnCYJzzUMbshO09RikZi3Wm4rYzJ45W27c8VbhK7CD+FZ6tircU6pxgE+9S0NMuRuD8tDHDGqyNtcsemamL7s1m0UmTKc4qQGqIdlfGeKtRtuXNS4lJkmalXpUJzT1PFQ0NElFApcVBSDFFFITSsMKUU0U7vTJHUtAoNKxQopD1oHWg80yRrE1E8Zk6damI4oTGaoRGkTBhkVdiXFRgU8OFIFAFpFyamCDFVI5SD0zVtJAwz0rNlDsYFGKaWpw6ZpFD1OKsRuABVWnqcVLQF0OKC4xVUMaUtxSAc8o6ZqF33GmP1qPcRQBOKQ4IqLf70B+aYEUqc8VXdParrYNQOtICtspu0VORioyKYFcrzSbamIphFIBgXnmkKCpMUUwIStJtNT7RSbaAINtLtqbZRtoAjAp4FAFPAoATFJin0lACjinCmiigB4p26o80hNAEmaTNMBNKTQApNJmmk0ZpgLjNSKMAUxQSRUwFAC4pKdjimmgBhppzTsZpcUARYpDUmKaRTENpuKfikpAJikA5p1GKAFzRRgmgCkMMUuKcFFKRTAjFPBphpwHFAiQMacGNRAU8DmgZOrMT1qYJkUyKM8GpwcUCIzFTCtWGPy1HigCuRg001NIB2qLFADdppQtPApcUAIFqxAoyKYqnFTRjFSMtCMY6VA8XzE1YRuOaJGAHAoArKntTvLzQp+arKLnFMRW8ql8mrvle1NMXtTAomP2ppjq20ZHaoGHNICuy4qJwassKgcUwK5FRtUzDmmMOKYiuwqMipiDTSvFAEJFJUhBpuKAEApQKUCngUARFaUCpdgpCtMRFSGnkc03FAxpFNqTFNIoENxRilooAKWkpaADFAFKBTgM0AGB6UBRTsUoFMBoSnCOpFWn7DQBAV9KVYyTU4izU6QcChsCrsxThHV0W/tS+RjtSuBT2GlCe1WTEc9KPL9qLgVwgp4j9qlEZz0p4j9qLhYg8r0FRNG2eRWgqACmvHntTTAogBacp3HA5qU27N2qWO22dqdwIDHx0phTHWrLkJVdnB5poTInGBVZzg1ZdwRgDmoxGDyRzVCK4JIpNmas7F9KQgDoBVIlorGPFJwKkeoT61RI7cKaWphNNJp2C5JuFTJOFAGaqZyKjYEjGT+FFgubCSRN1pbyGM2jcVhLJOp+Uk1natd6kYmVtzR9gBWMykZ06MbwlWR/arD3M8cZ3PH+ArCVAr7nLZ+tK0kUreWZCoPUk1zTkapBPcy+YxWQj6VVe8fOGbmp3twJBHExdj0zVyLSohbvLdQ5dRnGayYWK2k2C6lcMJX2gDP1rVvtLtbSDdCGD9iWzWVA6GUGMbMdNvFRXt66/IXaqGZcYedwYJ5Fm/ilkPAFJpq3F3rUdvJPdXVsM+Z5bEZrKv9Yn1O/XAjgiAxkda6Cw8SR6Jppt4hHLJziUe/861JO6MWgaFax3k0BgxnAJJIrkNd+JSXIaCytlMZ6M3OfyrjdT16/wBVfNzLu/CqSbOrpuNCKOgPi+/uYGtBa2qxOMHK81Xt7OO5lCb0SY93OBWK7AcgUksVwIhJ5L7OxxTA39X046UiNJJFMrf3eoqhYMCSySEJ3RuhFZB8xvvGQexJq5bTeUhXYG+tAjafULKytgbF3+1E4KYyBWIC19cDcUQkdcU1g0sm1Rlj0FT3GlyW1sk8jlWY42dxQBbsZINPvCupRTTW/BVVcgA/TNOa60iXUt7rJFbsQMqMkVjAgjHzH9ansoVub2KFpBHk5DN0oA7221o6Lp4uvDOtXECr96GZshvwNM/4THW/FbTw3dzdSLsJKWyZz+VY1zbSag7W7OjzIuEZF4PtW74F0vWtJlkvreSFWRjHc2cow7j1FAcxx2oCa2cxSQvGv8O8c11FnZXOq+GEP9qW0MQXlSORz9ateNYR4muvM06OWB4hmVZk25bjgc1zFkDPqFjp97CIMyBZGJ4YelMVymLbKuscRklXPzL0NRRW80ieYF+6Qa9L8RR6Jo9lLaaS0LSsm4oGyfrXntvFdXcDwG4WFVxlWODSaATVr+C8WBEhcSoPyqn5RWLzJQQvfmrqWdvZS75ZyJR91wcirSQx3en3NxOx5Hy7eAR60DMyaS1aIJGPn9apspPerkdkXaMIy4cZyeKhaKWO/EC8SAgA9hSGIkjrF5ZohG5k/E1f1PS5LERsXV2fqq9RWazPBJ0KOvqKBks8jrPtikZVPpQ0ku4RvIHx0IqJnMnzNy3rUmkWovNReEsysRkYoAbcxkweaXLEdjT/AOz5BpyXiuHQjLqP4fxqSFPMYrMr+TkhttMe2YXhtbCV2X+EdM0wEaO3aFGgJJbsTmqpGDWk2nmC0aW5SaF1JGWPWqAXLUmBprpwisILovkSkAjH3afNbxK+2GdJPQ9M1Lp9qt1KIpneKJFJG88ZquhthqMwusKittGB1PapArS3NzaSNEsrR4/hU8VradrQSCKOS2Rk3qZpN3zMM1lXRgLGSMBx2yKn0WWye6ZbzZGhQ4PQZq7iaPaNVuom8ONNvC/ugyZ6k9RXBTa5cal5ttbqLGJhkuCc5+lXND1BLbw1LIFjuprSQ7Fk5ASus0vUdD1rQ59QFnGrRqRJHs5B9OnNJgeFzrExJj3bskMSc5PrTS2IgprQu9MuRcvcGAx28shZSx5ANOmV5IBDCUKJgts4INAyBbxo7VYlkwh6jHapm81NJjdbrMW7iJhyKzzGQOe1Ee0uAxwvrQBouz3HlbPNTjBJPH511tp4n+w6XGbSUCWM+W1uRkMK5awvZbBHjjXzIZDhkY9Ce4qfSY4pri4iktna3xub++vv71IHpWheJYI9StJ4JRbzXAxPGw/dyj3PZv8AGvTdK1KyvIpZbaaIeUp3xA4MZ+lfPd/bwabqdva3StNaPGHViOVHrW74bs7Oeyub2B5g+4xlVlP7wDoOaaYHb6dpLBHv5Yv30swmV8/7Wf8AGuwTUbRNUNkxJeRfMUAdPavN9P1d18MyTS3FxIiZVrdCN0Q/rWxoTWkOq2yQzPdzSQ+bblZBnHcGmB28roikk9KrpMsmcVgyX2p6pOUs7MRKrf6wy9fwrdPABP3zy31oAQ0ynUhoAbRRRTEO7VDKPlqZWXdhjinsgKHjNAFF9gj+tNh2r0NElvIzfeII6YpFgdcA9fWrEXEcN0p78p0ye31pkUYQUTMm0o+SGGMDqahjGyXJiDh5MFRnCmljm3KGBcbum7vXD65rUlvcESbZrSNtsiBsEntz3qiniq+aygkgkEbyMY2jkH7sD1qGM9IvLwWVi85K5jxlScZzWNq15p+teFbiW9hKQ7vLBJ5Uk4yK5c6xbpZafD5wnmiDBwmSMg+/NRz6yl3OulyL5VnNiUKQVGfyoAyNOMFneSeXNJc2ifJ5ZTcMHuOeajSy0y61Se08yVHIHkcbct6H0p+p2LWUqzWLyRnnDRjIJxWVpcty1xJeSRedMAU/eDbg9+KyZaZ08F1FYQXP2u58mZothj3gnPbin2FjZS/Zp31F4txUohlPzOOua5a00mfU7qWZSipboXaQtkMe2PyrqrfS4n0V9Sv3maRWCmOOPPXuQORSTEzYg8Q29jcQ79YWBZHJVMbxn3IrstI8R2OpTi1imSSYgsPJGQR6+1eO3T6fdadPNGnlTW+0xEHIJ9vSvR/A1yt1pEWoSRxx3ZUq4RcF1HcitFIVjtCeaTpSK4dQ3qKTOTVCI5BnNZ10Vj5Y49Pc1qunFYmo2/2qDYxZHbJQjsQatMdiGWVI0O84zVZ1Awcg/SsLU9bfSr+CwuEV1YbyxI3AD0q1DcrDcksji1k+aMqMjHv6VVxWL+MVG4UDOealyJVYxYcBsEg1UnLA4wRVollqNVki2kZU1RudBWUFrchD3B71es2wig1oY4p8wjlP7Iv4HI3KUHTmmCR42KsMMK6SVs9ayb21WRWZF+ftWqnczaKnnbuDTWXKnmq4kCna6kGpQ2ThelXYlld+DimVaaDf3AqL7M49KoRBtpNhxUnKtg07INUIgwRQCQcinMOaTGKYEgkLHk1JHK26qxpVbFHKBePPSpI5dgxUMJDAAmpGjPWs3EtMtCVT3qRWzVKPhhV1F4zWckUiQHnFPFMA5p9ZWLTGk4FRtIM0spwhxVZiQMmmkK5ZDDFK0gHeqyPx1pGLHp0quUXMWhLxml80HvVEK/JHanKzA80cgcxdD570vmVU+0BeDQ04K8Ucgcxc3ZpVOKz1uXB4NTx3BPWjkDmLgek3jPNQvMuz5etReaaXKHMaSNwKnRuKzY5egzVqJ/lqHEpMvqfUZp+7A4qukmRUm7NQWO38in7qhHWn5qWBJk0heo9xppakBIzA1GTTN1JmgB+aUGo91G6kBJmmmkDUpNAxjCmYqY9KYRSEQkUwqalYUwiqAZijBp1FACYpQKKU0AJimmlNGKYDMU4UUUgDNGaTFFMApabS0AFJmikxQAoNBNJijFABTgOabUsY70APQDjipMCmqMU6gQppuKdS4oGR4oxUm2jbVCIsU0ipcU0jmgCLFJinmkqQExRilpwWgBoFOAFO20uKAuNApdlPVealCCgCqU9qULirPlimFRnFAEaqDUqR0m3FTxKcUAWIYgU6gU2RcHFRO5XOKZ5zGgB+aCeOKjD5NKTigBT0pmBTqADQABeOlO2+lPA4pVFIBVU4qQDFSBRigr6UhiKaeRkc01RzUhHFAEaL81XI06VAiZIq8i4AoAUKMUFVx0p1MY4FMCtKtVXXmp5ZcE1VaUE07CEZahdalLZqJmzRYRAUJprJgVPjio36UwKxFMIqRutMNMQzbTdtPNJQMbinAUUopAFNNOphpgMammnnrUZoEGaSlIyOaQKB0GKAENJTyKQigBKWlApwFAgApyigCpEWgYBOOaNoqXFKEoAYKlQZpRFkVIseO1ADlUelWIxgVCBg1YTpQBKq96GHFAPFJmpGR7cmniDd3xSHigsaYhWjCDApmKcSTRQMTGKcBmkpRQA/CgdKikfHSn9agk60xFWZiSaqybjVuRc1A0ZIq1uSytkhqkVmPABpRGA3PWpw20dBVMCszYODTSQRzSyHL5PemEECmiSKU9qgqZ+vNPSLcOlWtCSsR6UhjbuK0Y7MFhkVYNqoH3aXMg5TGCYp+ytB7ZQCTVW4eOGJnYhQOSTS5kPlIQqA8nFZOpXayRlIWyPX1rNvvEKtK0cKGRT/ABMMVlXOoTGI4AX6VlORoiRrbLZcgfjVOZrWN9qRrI3vVEyyS/fYsPerUCxpksOa55OxSHIkhk81nKnsBSzzzPkNIxHoTSTTJkEMaiH7wcVne5QkEnlycUkyyXVyqLGB6t2/GlTaj5YZqb7ZGvTimkBwsybZXHlsnOdh6igxsRhIHUd91eh/EXTYrKW2uY1VXxg4HXFYEQ1HxDbuE8tRCobJOM1sZnKyRsn3hiowHZwiDJPQV09isN/dx2OoQGTB2qIxzn1qzr+lHw7cRPDJuhPSN1GV6UFJnMahYy2UabyCXGQB1FbdnY6Yujhry7mWRgCADxVJPM1rUlJjXp82PStaTQzq0otdGLl0XLrIev0oGYKTK8RjXcT/ALVRwI00/lKPm7Uskc+lagbeePa6vtk3L0FamoaMotVvLFi6kZJDcYpgYs8M1lKGIw3at620PU9d8NyaoLpNsLf6tl7Y55/+tWI1heytkQyv+GcV3PhDVbCLRZvD2pgwvPnErfdbP8ulAjzyC5azuop8FShzg9xWvcwwNOt5G25WGcYxiu/tdDtms50uRFcW8YwCUGcfWuGur6xSWS3htX+zKx2nPI/GgBunSahbytdaexhwcMmeTWtY+Jri61x5FkOCMsoPcD1rmhPNDdB4ZJEUfwNSzTtLP5wGx/VTQB3Fi2t+Mbqazgu4IJYgfvj5sen6U19GthM9jrMBhvoRxJvwG9wa5zSNeXSreRTbiSZv+Wwba341lXd9dajcmW6neUjhdxzgUwsTEtY6nLtKOY2K7hyDVa5nL3TzDh2xzUbS7SQtRlixyagY+e4luCDKQcDoBgUeYfJCBiOeg7U1QucGmuNsm0UDNO6uv9Bht3Xc68rIDWVM5eTeXO71pSOKjwScAZNUIfBNKt5C/mklTkeYeBWjqCtcN9pLxcgDahzj8ayfKdgXQZx2q7Lco6psg8pgMMo6UhjYUaa5hth/y1cIfpXZavow8LarZT2kZm81CHBAHXFcjZBpL2HylJmB3Jj1Fdhb2eva6UmupNy28hAVlA/AH8KoDlryGezu/OkG3zyXC9h7frVcyObjzkl8r1YHkVpata3lzezsSH8ptmOhH4ViTRyRNskGD6UgOhuruZ4YxqCfa4yMpIeCKxSPSp5Hu3sIS4/0dejHqaSNV3KzjKKcsB1xQBZfU7iWJY/MWKJRggDk1BHAJmwpUD3NbeqqLu0jubezITb98dfxrFQDoOtS0BXmURyFB2roNLnFtpwS7tUdVO6PcMYBPWpdDSW7jFvHYNcsHyxPXFN8RXwGp/ZzE4WJQpHrTQHVaXpdnb+KJ7VAFtru24UnoTUHhKH7H4g1XSpHG0qRg8biDVXQ1uYFTUb63ea2i+aBt3zqAegFaP2mH+059dfTUltpXBSSZtrA+woAm1DQJDpt3DcR7IUG8S9ST2FeYQOwuEWJCd33sc8V2/jnxSJ1FhaTEQlB5iKMZPcVjtpkWnRRXdrdgb4uQRkHPWgDOj02W+vfscBG8jILcCqk9ibNDFK3+ko5R0x09629Engj1Ca7upQrohKZHU1kXN7FdTyyun71m5bNAETlgF2nGCDWlpogN4kt9dSW6SDKyIcc+9RXFhPaafHNdJ5YlGY+c7hTZLWOfw6l15i7427/AFoA3Ndmu9QuLVp0AnVSgx3HY1kx3H2DVIo53ZIlcGQKSARVyytru90u1vEuVQI21vM6qanW3W4zY3VsTMZOLtBkZ7UAXdaazt9ek1fQJWkhTa0oPIbI6e9aGkX0ek+LdM19RmzJxIiH7m4YPFQzLb6l4ZSxso47a9iOySFDkORj7v1rBurCfTdLt5pmZVdtrIvHHtUgfQWo2SyxG+sgOAHSSM8E96tN91T6jNea/DPxJa29lPZ3LzgI3yuzEjYfUdsV6Y21kV0ZXQ/dZehFNMCE03NPYYptMAqKRygz+VTAUMqsMMKLgUog9ywU5GD1xWoBtWoY9quKtZ+Q0wIW29TURwTkVKR61EQBTAUUFQTkLlsYFJmnBsEEVIHkN9awSy3Nud3yTEOQcYIPp71WsbhljdLy3+02wU+WVABz712niLT9NSe4uLddkwO5n6jGfmH9a4zxDpr6BdRSRX322KePIK9I2PQH68/lUyKLlrbS6bdxy2bpBMVOTKcjGKzdQ1nU9Yt2e5+zR2y55XAB6dsVn2MyXUjT6mjXTqp+QfK+f69qr3toqWZeISRqPuxE/wCcUkiGaVp4kuYNGMCSBpFZfIJH3R3/AKVYsr+9kluNPuLbfLNjcSeFOOp9Kn8JiCDWtOOpWHlwvE0cbyLwSehBrfbSLTTfE3mWVwCtxEzSB3GMdgCKOUVzlpI5bbULa0s7jaHOXg+8qkHv6iu3ZdQ026huHvLS2iKBruyCnkZwSPwrlbV47a9s7LxFJi0mZjHLAfnDZ6EjtzU93B5uuSNqFy0tipWKFnPIi6ZB74NCKRWsJtNnOt28d0gtiN8CkYLDdwR9BivSfDel3MMOmPPCimKMKyg8gY+8a8uuo7SC/TTYS7XCSGJZUA2FDzjNd94MaHSNRUG4d7e9JWFi3AIzx+OaLDPQWKxoWY4Udax31+2t9W+wyfuww+R3PU+larupdlHJHUVxfjiKa5giksrPzHRtyTK2NoUZOT3poDpE1iBrm4hhmSYx4LnGcEkDGap6tIIL2CcuquuYzyM4PT615fb+Lb8+Qm2DZK4VzGPmcZzitDX76XV9Ugu03eSUACsMMSDnGPxAqVIoy/HmoreT6aoDS3NupFwwHUZ9aoR3t5FaWyC6kKFg32djz9PpzXRrbtc+FdRup4YYbRI28kqcMXHr+P8AKudEdtcLb7pAk6hSYX4Mij0P402waPQ9J1Bb2UTqBHsGyVMYycDkVdY+e5VvwrF0LTt4lmtb5PtDModHfcFXPHTrxW4be4W4OImGGxnsR61pGRDQ6KNoyMCtBTlBSLGOARzTtlVe4rFScYzWVFMXJ3cEdq1rqFpIWUDk1nLatArvIOgzWkSWZ97b7naRBVRI5Ew54qaW7J6HirsUayWYYjJraLM2ij5hpVyaHUoSSOBSebu6YFWiBH2k89aQKCMYpuwlv61Iq7f4iaYFcxMCSelIwHpVluRUDIaaYrEJWmgVMUb04phFO4WHLmrkbnaMt+dVFODUgO6pYy3jLbh61bQjaKqQkEjNW02dBWMjRE3akNKBxUix5GazKKzgmq8vTFaYiXviqVzEiMSrA5q0SUg2DipUOaZsyc4qRI2HB6VoySUKNuMUKnPIqVVAOOtTFPlFRcspTWyyDOSDVfyH7GtExse1N8huwp3FylWOE5yRT2jwOmKtJEwqYwKw96XMMzlGBinhCSMDj1qw1tg9KmEIMQU0cwFZYmDDGCParsUbFaYkITgVdgT5elZyZSK+1lNTIGqw0WFNRpgcGoKJEVs9KcykCpEIApJORUjKxNMJqRhUTelSAmaTNNooAN1JuptJSGP30ok96gJoBoAs+b70eYMVBmkoETFgTSHmo80oPNMBTRS9qbg0gFFBpKKAClxQBTgKYDNtG01LilxTQiHFBFSEU3FMRHikIqQimkUANxRinYp2KBke2lC1KFpdlMCEJzUoXinBRTwtIBAKcKXbSgc0AAXPapNnNOUVIV9KBkBWm7alCk0hXFMRCRTCKkNJxQIjxTcVJSYoGMxT1FG2nAc0hBilwKWigAUc1KBUanBqUMKBjcGmsQOtSnGKryAk0AIJBViOYAetUSCDUyA4FAE7NuHXJphX0pQM1KicdKQiAKafsJqbYKz9X1eDRbdHlXzJZPuRKeSPU+lAy3sNOCkVz6eKZ5EDrpyhT0JlFQzeOreyWNZ9Nd3YZ/dyrx9aAOoGakCmuf0zxpp2oXUVu1tNbvIcDfyM11f2d/7tAEeKKeY3/umk8qT+4aQxAKeKQRuP4T+VOCMD90/lQBJHxU+8AdahAI7GlKsRwKAJPPT1qOSYMMCqEgaPqCPrUYuPeqJJJTkmqzttqRplxzULyIaBgHJ70m6oTkj5WApBkdWzVCLAemOc0zcaaSaBDWPNNpSDSUgGmgUuM0m00DENIacaYaBATSZoxQKAGmmYqQjmkxQA3FGKWloATFG2pNtG2gBgXmnhRTgtOC0AIFqRV46UBeamVaAGhPapFTnpUqpUgUelIYwJ7U4J7U/igUAII/anAGlyKARQA8CjFJvGOtIXXHWhCA03IpN+aAAaLAL1pwFGMU0uM9adgHHim7wKid2IIBqnJJIh5NOwjQMoA61C0gNUVnLdDUyPgZJqrASNk9KQIe9RtdhM4GaQXBcZoETBFx71E6HsKliBPLVYCpjNAGcFIP3aGUt/DV5ioBwKgZxnmmBAtsGGSKd5Wztika5VTjNN+0Bu9Ai0nalYgCq4LHvT8nuaQxrruBFZGsQwLZnzDv68dq2CRjqKyNRUMr7wCvpmolIDz1kf+HitKK706GEROPMl7jbRfRFfmEZXPrVERbqwcy0MmgglmLoNgPQdaT7MAOM1LJZTRR7lT361WFxcr6VPMMqTRFHIBzRGwAxnmnuXkfk8nrSpBuGV59zUXKI2QHJzUQQFsk8VM4C9T+VRFgvA6VSYHUfEC1kvr+C3T7g6+3FWtL8PR6To97bg7nljDK2a37qwS8vWaTv0q+1pG8IXHQYrpMjj/BPhq2tYZL+U+ZcEkdfu1k+ILJtTnma7WTaDiJkGR7c11mmQ/wBjSzpLMDCcsTjgVzl/43jt7xreyKLChI3vyPwpDRq+H/C9ppVkZ0GZZY8MGUHBqx4N05bKO6mPLu5AOK5Kz8YJqN1KLu9eOKPqv3Q34Vv+GvFNlOLhYwRbJlhMTwaoZT+Jmix3mnjUI4wJoyASByRx1rP8P2FgfC0ck6hkT75Jxya5zXfFcl5rEkc1xJNZAkADjvWFd6gHQw20sghP8J6UAbmi6tFY6/cpb4NsSSoLY/nUHiJ7CVEW1tjFMHBbLZz6VzafKc1YEmRUgb1x4pvDYra2sf2VMYcg5Lfn0rBS7mjBC7Tk55GaUtkU1E3PtBGT0BPWpAtS3gvgondI3QYVcdarFWGTj5exqOWNlYhhginC8k8oRYUgdMimmAxn4yKar8Zp7xXTsW8g89xUG10bDowP0oKJJJAT9KXDBA5jfYf4scGtTw3pLahq8QZ1TPRX6MfSu81PwVfizU+ZEsZYZQjAH4YoA8tXMkiqqOSTgDFbs/hy8tYkle0nyRnaV7V61pvgLTJI4o7jaNmD5a4AB9aZ44uNNsNFaztpz9rRcKgbJUYoA8QkUM5Krgemc1HHI1vOHTqOldRpWl3nim1uYoPJje3UtkjBb8fwqvpaXPh29h1C5tWZM4Kt0IoAx13XN2FVBmUgBF4BNdDa+EibqSKRZkdl4+XI/OpbuwbxK0uo6dEYXjOdowMCt3TJpV0xJYbmSW7C7Sc8qfcUAYmm2mpabbyl/JDQt8sLJyQe4Oa39PkittMvL0TI91J/yxU8Jn2rPsm1xpJJb7ZMhU5V8Ka5qaG5vL6T7Mzqin5mU44pXAq6hPdwanct5sodmzk8VnMZJCXlYs3vWnctL5yNLN5+wYDdyKoy/vJSwUjPY0wHiaU2wtz9ypodgkj837hcBvpmofKkXG8YqaO2+2KbfeEYgkE+1MDpdaij0XfYWrM6XCB0zwUB9KwbGPdeIsitn0UZNad5qzXWlWtreBkv7U4WRUzvXtV3StPZrFdZiu9lzGSGV1zx+dSAzSb6XT9QuUV3gVhkbk6itextE1qURQQq0xBlMkqE49qzNZdtQSGZUMbr8uW6Z9K9L0G0XTdGtrcjDhAT6nPNBJytxOUM1vNbSrMEKDaMqPerDwR3Hw/8yYY8lCIT1JcdOK6aO2SCedgoYynqR0BqWxtbeysTAUBhBzg9qY0zwe8DXHLJN9o6zFh3pYLaK2uoWD+ap6qDzXpeq+HoZNH1O7uRm4c5j2twF9MVzj2cF5YLc29sirCoUluOaBmFN9ogfzIl2j+6y81BBDa3LJE0D+bnLbG5I+lX3lt1ulheVbrccMwORinXltFHODBbvs/2eTUgbaz6NrVp9jvIJY/sce2A5wW7YFN1zwpZ29po9vIohvZZFEiIeSp9fesfTbh11SGZWAuoGDxlzw2P4TXe6HJBr2qtrOoQtvE3lhM5EYxwD/ntVcxJ55rEN74euZ9PSQi0cbgp9DWxounNe6Q1jJK9rdtiRPM4DDnpXS+L7C2ub+2tFjRg0it9oB5UZxtb1FJ8S7S3sptPdI1O2z8sA+i4AotcbZy4huY5UNuqwXNvna8LYb86v3vhnULvTtOiLu19czZVC2QFPOW9P/r1veA3ifypGWNnuYs4KfMpXsPzroNb0iW80M6lDI8VzaSGWLaeAnGRRyk2ORl8Py+DlW6nnWdrkGGaBMZHoRXT6uk2k+FLW4jbEq7drDgjNcx4m1KKeHTbeIs8vDOzHJJJHWt240u5s7u6tipntri18yDzD91sDKj6VNykdtZXcc9haNLIBNJCrEHucc1YxXE6W22x0N5U3xqjRtx0PSuxhLruST+E4U9yKdxk46UhHFIDTqYEZU0x2mTlTkVP3oZdwwBmmBWF3iTbJwvc1YlwvU4rKur63tZSkgy31rF+3352R6fdqsQznzlHA9s9qAOoMgKsVPQ4oWX5GPVgMgetZ1jKsqB2E8m770xG1SfYVY388UgOf1jXBdC50u6lSzDoMosZc89jgVxmoL9j0i9Z08y3DqyDOMsM44PXrXYappV7c6g93DOPOEJCoOrHHFc8+najf6vbafrGxZZoZBC6j5VOBzjrmnLUaOMSLUNR/wCPWJpXjAc47Cq/2vUWlN5cyu652rkcAjsa6w6JN4bUi7tZgjfK17C+UYe47VRWwtvsNzbQ30b75GcInUis2rAyCXWbqG0ttPuLrb5J3oJF4T2HoKZpmoQRXNs9xJHJC26JlRzx74qve3FltUBwzBQrbx8ysPf8ai8Nm3bxDYy3K/6MJPmKr7U+YVjr7PRZB4o04NuuLe6iZ4yU5Xg/Lg/Su/l0CO90KCG+tgLiCAqhB+5gdffNZmlI194oubs3GDZwiFS3IJbuKqeLBqGitZ/YLwSvcsyMr9sAfpzRcaRdtLHTNEt45LxYQkiEIkSbmcHqfXPTpXPpqljaactrBcW0qxyNJ86sjoCeME9SKXSvEcQtZrzUDMLxAyhVO9EAPRfTPFZ6WsHi5LmSwcROtwrtBJF+8we4bPT2xRIfKdfb+KNTl8m0S3WX93vkuQPn8rnr71xV/rOp/wBnJGlzNPpMMu0pIdpb1BI7VrfYL7wrquxnD6fcxlNpHJGOQKxPEt7EIba3EH2K3lX5tr7lAz3A70kyb2KVz9mkdLi0tRaurAhEb8xTpr29mumkjjfEnyLuBO0+gx3qGMTrKZYbqCXyvuRr0bPGD9KktYr6HVxPeTwrFu3lUl4B9QMUmhpmrqF9PaaJa6VfW7+S0heQMvO8e/41jSunkmWSFxDGw8knqF961dQuINQ0+4lMr3AjYOd+QTzjGazLbbBqoLr5tsSCYgS+0kccd6B3Or0vVk0Fbaa2gSaGRiZfLXJVT3B7fSuji8SRamXNrdXNs8eQolQKJD6DNeTXEZttSe4tHkhtnfaEJ5HqKlgubp7m3nmO+3tZtyA/eB44pRdhM9YsNWu5Ih9vdVEm14Jtv3hjkH8a6MR15zp+tQNdWt3cSgQxt5fllcgA969D0++h1C1WeE8N71opCHNHxVeWH5Tgdav4zTWjzVCscZPolwGcwrlc5x6VZsLa7igxLC4UEjOK6FhtfFSDpWyqE8hgzaerRNwc1jz2rwjcFrsGTccCqF1pklxnH3T1FONQnlOXDkdaVZc1pz6MI4mIJ3Yzk1ijK8HrXSZFxHGc1OZFIwQDVBXOMZp+WXrTsMn+T0qN4MnK0Kd1WEOMGlcCk8ZQ89KaGKmtSSJZI8is94z5mMUILEkTbmHNa0MKEDcTVGG0G3eWAx2qb7SEYL1x3FZyHE1FRFXgU4ELms5btiQAeDVhYJ5gQrAA+tZNFhK4IIzWdJHliRWmumTbiXYMD+GKsR6UD1Ip81gMqFTs4FOww6giugTS4gQQvNS/2etT7RDsc8nDDirAZT3rUbTVJ5GKQaTD1wd3rS50OxQABpQtTPbeWcAU0Jii4Efl05UNPAqRR60XAZ5XtR5R7CrIC4pwA9Km47FXyz6VNHhRg1NsBHSoGHPFF7gicnK4qk0u2baanBYdajZAz5NAydDxUuwsuahXgdamWTC4qWMiaM+lQsuKt/eNEkYxmkBnlTTdpq0yimFakCqRTSMVYK0wrmgZAaKeyYNMxg0CEowaM0maAsOFSKm4ZqIVMhwKAHbaaRT8000AR4pwUmnYqaNMigCJV5qUJ7VL5I7Cl2VQEZFMIqYrTCKYERFNIqQimkUxEdJinEUAZosITbRip0jzStHigERoOak20IvepMGgCPaKULUmz2pRH7UAJt4pFXmpvL4pyx98Uhkaip1AIo2Ck5HSgBCoBqOTFOYk1C/WhCInNRZqVhUTdKsA3U4c1COtSBsUBceKUUwMKcGpWAdSGgUN0pWGNJo3mmE0gPNOwE+80hkpO1NIpWFcdjJzUiDimqOKdnBxRYLkyp3qdRxUKPxUuaBjwua8m8R38tzrt1PNG6KrbF3DACjpXrCvikk2SDLxRsfdc0gPEfOEp4yx9hmrlnpl5fy+XBZzyP7IQPzr16O2tVcOtpAreojGahvrqSFSEwv0o5Q5jltI8DSgeZqM5gK8pHC+Sp9Sa1ptGkRsDXL/AP77pft8x6saaZnk/iNXyEXBNKuOi6/fj8QamTRr7+DxFdj6gUW+8yDAzV9HbPK4qXEpMqjQ9S7eIrjPuBS/2HrI6eIJfxStqEZUGpx0qCjAOka7/wBDEw+qUxtP8Qr9zxAn4xiugeqcpwaAMKe28SJ/zG4m/wC2Qqqw8Rr/AMxC2b6x1vSfNUXkk9RxWhJi+b4i6Cezb6pS7/En/TkfwrUlUR9KqzTMIzg1ZJRM/iRfvQ2Tf8CpPt3iLHOn2rfQ1bSVmHJzTs7qTQ7lEajr466VD/wF8U7+09d/6A6/991dTANWFbIpWAyn1fWAOdDH4OTUP9uasD/yAX/WtzdQZGHQ4pWAw/8AhIdRH3tEkH50z/hJLkfe0WYfia3DNJ60z7TJ60WAxP8AhJWHL6Zcqvc1JF4o012CyJNE3+2vFbAmZuoB+opslra3KYnt43z6qKVgI4NRsLoYhuUZj2qbbzWVdeGLKb/j3LQOfTkVnsda0KQbj9og/wA/40AdJto21TstWtr5PlbY/wDdarwBoAbspQnPNOpRVCEC07aKM4oDAmiwwIxTd4FObpTUAZqdguSR7mI44qyq+tCAYpWcKMmk0FyQcUpaqZu0U8mlFyG70coFrdmjfiqjXGOhoWbd1p8oXLBlI6UCXPU1VeUBSarNNnqxFHKTc02lAHBqIyHGc5qgu89CTUclyyHGarkDmNJZznk0rX0cY6n8KxvtDk/eNOyW71XKHMajaiCuAppYrjzATyPrWatWoRwBS5RXLT3SRDPLfSqz3qOMGOnuiBSWFVCFB6UkguSG4JGFUU+OGSQZbgVECqnIFNe5kIwDxTsFy4LeP05p4RV6VmpcSq2d1WknyMk0rBcnMhFNM5HQ1E0i461CSSaLBcna6PrULSF+/FMIUdaaXUdKYXDZjrTlKg1GZSevSnRtuOABVWJLayHtUuciqocL1NO8/cMZxUtFIo3kzBmANZl3dyDMaROw/vGtG+Rd37tSzN1IFYesStFAVAYMOoIrlkWU7uQyqSWH41mFttaFpB5sPmyj5jVO9JMpG0DHoK52aIjmu7h1Cq3ygdKqF2HDDmphxxzn0ojWEyn7QQB6Gs7g2V8YOT0pjvvX9309au3AsNw8tsn0zmoz5XlnAxTApLbu2Wd1AqBwqOQGBHrVqXGMCmArwu0H3NWgPXlAzuPQVlXWrPP/AKPp6GWU/wAQ4Aq5cZliMecZpbWFLWEIi12GRzV6kunaNffbWPnSLgKTnmuf0jwXv0WbUbuN2cqWRC2MdeTXR+KNOluZoJ1PVgu3tXQrL9l0VViGGMeFHuaQz5yuQy3UoYYbd0r0HS/DTWHhKW9a5dZp14j6AHPvVO78EanJr8ZkRCs7bs7uorvPECMNItbNdrygKMA9cVIHjlvoN1d6x9gTbvYkkjnaKktPD7trx0y7kxgEh06ZxXd3Ea6Hq0F09tvfGCIutcrrWqFdYN3BC8EvbeORVJFmNqmmR6dMY1n8wjr7VQ3YXFTXtzNdXLTTPudupqtnvQ0IVWIrQ06KCVGZ5drx5xxmmpbz3tqp8ny0H8Q61f0nSZ7uZktCVdRksw4/OsxFCVzIxdjnPehLowrhY0+uK0NS0O400p58yMWONqDJFULqylto1Mq7dwyvuKAHR3MsjDJrYdrNl3A4YdM1jRROsanFKHUybHoGXpbqWG6hLKBAG+Z14yPavRbXUVtWWA36LYTplHuH3YPoK83SSN7JrUDOTlfap2vJRZravtKj1HIp3Gex6bAdHsJLlrkSqwJ3HuPrXncOm/8ACRXckoBRpHJDZ4X6+tZK6nf3EC27XErxrkbR6Va06GUytjzkVRklaLgdLeHT/CGnXCLKpubiLBjUc/X9a5M6xczWUUQtw8KHO6Qc1DqF7Pq14Bdu7sgwpfqBT9ItLa5vTayuyZHy47n0ouB0vhbWrG0tXimjxcSNuCoOCKqJf3+nX9zFb2sapMxPmIuWUewrFaWbSNWIV8PGcHjqK3k8ZQ2tzHdSaZMXQYDoRii4FW51SzgbbZPNJMwIbzuCTS2mi3cvlzRz26CYYeOA/MB61QvtattW1qTU47FowygOpGR9a25rj+zNLjuraNJJf4XxjBo5QMvTNMsbfxXJp2pZkhHyox4+bHHFSeNdCj01obuOdDEDs8rHzKMfrVS7l82N7uVkmlmwzHuPp6VY0W4Sa8ju9VuxcDbtiDEttP0pXA5m5uYrhlMZ5AxjFNhmjjlTzRlCcHFaXiOKz+1iWzVUzncijAXpj+tP8K3FjHe3MWoeWIZISN0gyAaaYDpBbmSa6F1GfLX5e4I967bTdUsL7QGupY4oYogAyhhknHWvLIUMN350XGxyRnoa3bLy9S1JA1m8nmZ3QxNgOfU0AdJZ6/a6ri4WEK9nMG8ofxJ0zXokjC5gju06MMgfWvL75dN07VNPWzja1O8xTxS9QDjqK9QgikitVt4lxEgwh9qoTRFeajZ2EkaXEhXeM525A+tSv+9QrF827oTwKhuNIiuZ455vmZBjB6GrkXyxqD97uKAOX8azS6do9tFFOUMrmOTH90iuEvJZU0qexguzJZpJn5F+/wDU16rqukW2qlnu4g4VCIyeoNYPh/wu8FisNwuEEpcMQCSOwpAeYXDwokLW8ZjaMgEt1P1rRuL1kkQ28m7cM7McVqeONBuF8RTy28SGNgDtTtWd/ZV7ps1v9rtWh3dMnqKhoZlkySz/AGnA/duOgxivSDBPaW9he6favNb3kYea3jPOQOoriEsGudX/ALNhkSMSt1Y8V6z4YsrqNIFulcRRwiKLd37E/jSsBxVhrr6TE7i1Nypcsku4Bl9VYHrU0Gk3fxE1v7UyvbaYAM5OfYgYrFuLW/udVu5Utw/zMWUnhRmvVdNh/smztxbQuVmiUiNBxvI/TNWmJHH6fBb+FvHLRWyYsYV2MrHO0kcmvRbl47vR7iC1bf5ibV4x1rz7WvDt9oMd9rl7eb5JzuSCMFiCT0J/rVbQv7dtLOC/uLwxrcSBbaGRcs4Jxn2/rRcdjQ+IOnfZZNFuIISXDLE2OckYrsddtGuI7RkYK8Rzg91I5q5qGlfbZrN5fuQESEYz89XZIRJPG+3dtHT6/wD6qQHK29g0dlpXl/Mv2jDL2zk108g/eGqVrDsgETqdokZlyMYINXiM1SAjpwpCMU0sA2CeaQEwFNmjlliKRA5Y8kHGBUbuwxtBPPOPSp4SLqFZ4ipAPr0pgc61/bNrrWSbDPFnfHIvJ9MVLq9zZiwMUw23BBCoq8DPWuV1yc2HjCbU3VIsBXAlJAcYxiuevPFkVzcS3LRN5z42OnVABjHPWpA7tdQ1rSraBorSK6siNqs8m0r6ZrRU3c+nSXX2eOK8Iz5aNkVwdnr0Fv8AudYu7uRMApHGNyMCMjOOlauieI7m58Q/apknh0hk2Qv5R2A+7AUrgU21y4S/uhLfotycIFC8Ag9P0rA1WeeS/gvLbVLppXbZJ5gCmE+laHiVLvRL4JPGlysgea3uoxktk55FZb2sosreWNGljuD5yhRkqw6hqpOw0U73Vr6Xzo55rmW3GEnBbAbFZ39pwvOGRmhBXaJQeV/CodR1KOZZdrmKWZgJY1XAwO9VGgkuYnuYrdUhjGGZe4Hem1cCUXjMjwtiVpTkkjkmobO7eG5+ThsEFD1qaGBr+6i+ykoxGARV2AZ0u5ae2HnwyDErDlue9Q3YDZ0HV9Qsx9tsLhHuIjl4HXhx7itC/wDFj6nZyv8AY2ilmY5O7cUBP8NctLpU+maoVaRrncgdpIV+Ubhnk1oWkkcdxCHcIi+pqVIIkOoXLTz20MbxxCXCts4wf9quni8RRaMkVi1pbWQaPY88PJJOeQfrWJqN3p7MPJth9pjOUdBwx96xrxLyO4864SErKcEIcgYocrjNp768ttVYSX5u4URkWWVshd3II9D/AI1mF3uZoIL6VipjaKMqvJOe9Fva2szSiSbYiR8QjgginwSxNqdulgAEVCczDhTg0iDY1g22m6RpOUjLrFsIQY3HjJJqrqktidPtZY4MwkkTMvDIwx1q6kkFzpvmXjhFtCRuOCjn6VyvnWjSyJtwjnIbOQT2plmpNJZQac0dtJPFKrh40kOVlGafo90lorzvceVdq3BfkEfSs9SIVjhuIF2jJXnJHoRTLmBfMjDc7xmgDopke/jM9zdwsCxcIgwATjP8qzJLKKOKK8WR2RwTgnBDZqk8UsNvKLVsREbgnY1saYiTWUNu6TSvAd8iE4DjOcVDEILlvmCsjRcthu/Ar0/whJ/Z3hz94MgSDYw6bGAI/ma4DUdQsNQ1JDDpnkxRkbVUAHPocceldf4RuLi7uY7aeIxwzbnWM/w4/wAiriwO7Eg708sMVn3CvA5wxxU0cgZK1sIZISZM7TirEYyOaYTTg4UUwHbAHyKaxANL5gNRPy2aaBoyNcnKBdvAI5rmJeZC3rXT6xbtPbuF6gVzTwSKeldtLYwkRqxVsintIz9aYAfQ0v3TyDWjMyxF96rarlRVGNsYNWY5c4BOKloZZU8VFLGSdyipYpEHBNK0WW3L0qCio0zgFexGKbHyakkjweRSKBngUxE0BZJQwrctpBwaxYlO8VpRNgVlIuJtAgqMVIi+1UYJOmTV4SrisGWWF6U/Iqr5voaUSH1qbFFgjJppFIr5p+RRYZC8SsOlVpIgOAKuEjFQPyadwsVhCMdKa0eDxVgnHFRO1USxqjBFS9BmogwzT9/GKVgDf2puMtmm0ZoGOKk0wjBqTfgVETk0ALnilBzTKcKQEwOBTC5zik3HFMPWgBxNNJFIc0hFTYYw80uAaQ05aAGsnHSo/Lqc0KOaQFVo/aoypq8yAioTHz0oArAHNTKDiniP2pwQUDG0U/bSbaBDR1qZDTAlTomKAJlPFIRT1T5aa44pgRGoz1p5phpiIz1oxTiKXbTAaEBp4i9BTkXJxVpEHemBHHGB2qfylI5FO2gUcUhWIDEo4ApVjXrU2xTzShAKQEXlg9qcIvapAKeKBkBUilXpU7INvSouAaLgNphFSEVC2c4FMBr9KhcipymeTUTxnNMRCeRULDtUzKRUZB61SEREYGaaXHrUj/cqk7Yb61SVwHNMQeKmWYYqNUGM0hAWnYm5ZWTNPMmapGXHemG4x3o5A5i1JJgZqEzgdTVWS5J71WaZj3q1ATkbcNwjgAZqRnCjNYUdy0Z4NWhdGROtHIHMXxc80vnFm4qnGwC81atSrkkVDRVy3CxI5qwDUQAXGe9Tx4xk1DKuOCtTwvrTTIo4zTTNjvU6gWAoqrdW6SAjHNPE2e9IZB3OaNQMiSxYnGOKlitVXGVNaO7PYU6NR3FO4rENrbnzPu4q+YAozinRKFAIqU81N7lIhUH8KcTS0hqQFJzUUkXmCn04GgZRa1f2FVZyyKRmtdl3DFUJ7IKSS5NWiTCmnffyahaQsOa0ZbIE5NVpbL5flIFbxZmVRJt5o+0elI0LKcVEY2zxVEkm9mfrVqCQqcCoI4jjOKswoM81LKRZByM0EUClNZljdopPLWndKOlIQ3bjpRT6Q4oGN3HNPDBhtYAg+tM70CgDG1Pw7vzPp7eVN1254JqPStbeOX7BfqUdehNdBvxWZq2mx38BcKFmT5lIFAjTZecjkU3pWBoururGxu/9YpwCTzW2zZqhj2dfWhBuPFVH/vUizMvQ1ViCa6Z1AA4qukrbxk0jMzHLHNRFtr1VhXN6KVPLGTUdxPHtI3YrI+0N2NRM5dssc0cgXJZZN8pIPFTIHCDFV4lVmHFasFtvXBOKHZAiqC3epg2BSzwtECazZZW3daFZiLctyvIBqm8zbs5qEtSE1aiK5YW6kAwDSFi3WoOlL5maYXJRwatQFWIWqO4mnIx7UmO5qF44vlAWnQv82WPFVoLRm+ZgSBV5LcsoUAjPesmMbLNGRjNU2fPStJNIVvvyMfxp39hxZ++351PMBkbyaUc1fubGKBcrkmqTbQKvmAjY4FRossz4VwB71YVY2GZeE71iX9+ba5K2/wBz1zU3A6BIwvDGp/K/d7h0rl/7ZfgPcpn0rUh1Sd7ZUtgHPfNRzl8pNdSrCuWOKqLdwN/y0A+tVtcuJI7HfPG6SDpxWPZM00fmsm1T3YVPOHKdIJ4WHDg1BHe+ZceWn3f71YzTeZujRS69wK2dPtESJZHYR46g1cZ3E4mskZIBIqGe5ggPzHPtVSXWooMgMJMf3a5mW7uJ5TIxx7ZqJVUirHTXWuRWqBkt9zHvnpWJc6pDcEl4Hye9ZMmoucplfqTVVrxjwGB+lc8qvMOx0MMaum5RisTULl4rgrtLfQUQ6hdqmwPtUdKpSyyvdGSQ59KzepSYm+aRsrC/5UNp95Kd/wBnkCepFX4tcuLdNsccSj/dqOfxDezKUZlC+gFPkHcp+T5Ay6srD+9TGl3GmPK83LHmo844NOwXJsg9Tikwp75qA7Ryc5oEmOlOwHr5FKKUoTSrFxXWZCEBxhgCPQ0jRq6hT0FPKEdqVUYnoaTHciS2iDBpCAB/eNYc15aTajNLEs1zsGdyDIU1Pr0dxc31rZ7mWJuuKZcz6TpFk8UV3BbPjAJqbAeaz6iJdZneW9YbXYAH0zVJ30eeZnvbqTeexJOKr3EkNxrskS3KPEXb5x0PNLqml2sc4+yMyMBg44BpMpFMWlpLq6pHva0P97inR6SkupNFG2IR0DHrUTG4jG12qxaf6MrTvIWdhgZ7VAzZk1LTrJFgnWR/ZKk/4SUMFh062FtH/eIzXLSSGaQu55qe2kVgQv3gMgVNhGnEv2i9M0spkwSDntVDVr1ry5KMcovanaQW3ybxjcCcVmzrJHcuzoVz60RQF/z0NugJ5BqpK5ExdRkGoonDKUb7/wDCadJI6Ku4cetVYBpuPLl3ZxUq3F1j7QIt8R74qxLbrbNFcCMSRtwwPHFXbnUYhbx2unLlSMkMOlADIYri3lTzF2bhke4qyNRubdiInxnrxSXfiRbm3S3urBInjx86Dk1TiuorsskMLscZBqRnXfZrSaztrm4by3mH8I6n0rB1SA6XqsTox2qwYZ7VPaX5OkRRyn95FJ8v0FM8Q/6TfR89UBqQLevwpeXEV+RvikTntWGbk7fs8I3J2RjmtDT5m8hrK45QY2n0FZt3pz2t6os/mDDI7VVwLCwXEdo/lRrHuUgkelO0e/a7njsr68WK1PUuOKZrOoSR2q26Jsl2/MMVnWgXfHI6BkRgzBhwQKpAdBa2EE2sSWQnM0KZ2SA9cVPJBo8kMiXt0kEsRIxEDlvypDcade3DLozJaoib2aQbTn0Brd0nwlYap4aVr3Vba1cs2XXBJ+tNoDg528yxZoo8Qg4DE5P41nocVNcq2nahc2treCaJXIEi/wAQ9fao0XjmlYC3AqSIT6DNaPhSUrrsUvV4wSi9ifSs+ICK1Z6lhig/s6S5jk8u+iOdoPX6UgNDWLS/vdbkae2zLJyCnQiuz0LxfKfD4sZrhYL2DK+ZJwCAeP0rCk0LV4tPg1uK6TUGZA7xA5OD261m+GZUvLm/s5lBD/Mm7oCKoD0jw7qGpatDcGa6jDQvs5j+979a3tpDc1zXh+LShby+VM8Tn5XROmfWt6MtBcGGJo7pMAlV++op3EWcU5TUS3EckhjAMbgfck4NS7COtMBgighcz+SHlPViMk1jav4fi1LSJ7vUDIZo1LphsYNbtPxvheM/dYYNS0M800Cxi1LS5oJ4guoW7rPE54JXNerQMHtIH/vIDXK6JZ4tZyflmhfYjDqore095Ps/lSDBjOKdgMu10Jop9Skjjz9qYKOwAySea6lY1jEap0VQB+VV14qdX6UWBEzxRzAeYocD+FhkH8KzNQ0tb+4hMkAKx8oBwFNaStmpEpWGRRw+SHEe5mY5wTVNyJLV44U3vnYS0mCvrin399LDKscUCPg7mbocfWsDxHf2OnPZ3dunPm4m2yYIBHU5osIvWc9xpPlpcXAvoXbapLHKH19D1rSsLv7d50wx5IchZAMDFcxpk1hqmmxWLXcNrcMxeDbMrt+OD1rrNLRtN06zs8LgkiRiOAaEAy4CRThnfbngAjIJqkJ4H3v9pRgpw23sa2NT01byDyWmUbuynk/SublS007UNO0aE/M7EuT9O9IC491JHAyywSrCwI86NgcA9DWRPcahZeHLmR3SztImDE7jvce1ZesatcQ3UELTRWz2975flZ4CgZ6dwQar+Mtdt9W0/wCy2tyQ5YBU2EK7Hjqen/1qYFC3ure+t7vUry+jvXMbJGJBl1Hpiua0S50iDUopdSjZ7UghlZeKrS3lzot0sURimmUZbI4X6GoNQktjNbzWtu5iIJZeev1qJAdjqkejWMsEtpayCSdS0M3nhge/IPbmqNx4x1ZIIljH2eRV2Fo5BsZfQp0rL1WyntdN0+W6lTbJGTEvoOOM1nyxTQbRNGwVxkA9xWYHokl6bfT0kSOyZTEJFlcklXPUAdBXP63rP2/T7O4ilMM2Cs3lnBVvX8cVzgngVVUxv5anJz61buNNiaCKW3uv3Z+6rDpmjmLMXU7a4IjeZtxGcepHFZnnPCD5bEbhtI7EV1MVtLfaW0Tyneh+YYyyD1FZc+iNZeVdrOJYDnDbdpDAelXGYmhbfTLuGOOW2uAmMN5g6AV0EthJcLHcx36KkxwydA3r+Nc7BqE89m8G5E2vnPTNadyYLd7OK0vvtFuxBaFxgo5/DkUNXEaCajNb6dqVrp7xqhAMpYZyPY1gNLHcyR5XLKc5HTFRsjG8mDllUkjIPertndxpolxp0karjLxuOuPSlYaHzqy2Es0QB2HBBHaozewSabFC0Rkl53kdAP61WjVnuCS7JFs27R0b1zT45TDatbzIRCxycc1I2MSZod237pGAKTG6KK5KOqD7jZxk1JfC0gR4w7xzKQQjDKkexqC3mikinUziIA5CEcH6elUIt3Y+0W6Ao+/h3hQ/KPeqU4WJleNcW7YZfUH0NWUupDbiX7QVk27BgdB/Wst1k2lRIWQ+tUBotc/a3EtxCFjUbeD+VWLe3hOnT3hWcmAqAhbghj19ulZKtuwHG5cAYrWEiPoFzHFqHlyHG62boVByADQwIobiIyhJGck/dbrtPrXRWMMTS3MbvtleMNHsGARWJbWdtqMUimVoNgUF9uQPWluhHbXCR291JLsUfvDwPw9qyaA27TSppdQRbYmaGILLcSdAoz6+9ey2zaLY/ZVWOX7QELpH5ZL5bkk9ufrXksWrbfMkt5lLSWwjuYCudwA7H171peENU1aHxHb20dzIIp4idrHIX3564qo+6I9SuI1lJz37dxUAg2cCn6ZcLqVobsjbKXZJFzxlTjI+tXDFW6lcCkUao5Cyg4GcVeZOaidKdyTMFyw65B+lPFw1TyQKwxiqwhKsRzVXAa8xcFSMe1UJrYYLYrQaHHzU6PAbkD8auMrCepzLFVbLCniMXSlYo2Zh6V0F1pdrecumG9RToNPgtkAQHPrWntTKxyTI0bFWGCO1IGzW7rllvY3KfeP3vc1gDg1tGVyJE6ufWrtvcgHY3foazecZpQapoEzSuI2D5wSpFQ7MGiC7woR2xxjNT5BqAEjbDAVb34qBVUHNPkGRUMtMtxTgDGanS4weTWYqMBmpUWUsAATWbRSZsLMMA5p6zCqYUqnNKrkdazsWmX1nAOQacbjPeqYIZeKjaTa2KLBc0fOyOtN3A1VRyalU5pWGSNUTjipG6UwkAc0ARdKXdigslGVPGaoQm8etNaQZ60wriq5yGxRYZa8wetAcVWz71IuDjNNoknB5qQHNRHikDYqCixgYqM8Gm7zSjJPNIBaQ04Cn7BikMrkc0A0rjmmikA+lUc02pFxikAhNLtzTCfmqQdKAEC4HSmsmO1SqQaDgigZGRxTVHNSHpTV60CJQozUiqM0xTzUoNAEwA21HIAFpwao5HpgVW602nHrTaYhQOaftpgFSDrTAciEc1Mpx1qME4pwoAkaUBelIpyM0CnCgCRRxRQDSikMMUopaAaAHZ4qJ1JOakFDD5aBEXUYpojOckU8DBqTqKAITH7UeWKmxTTQBVeEelVJo2A6VpEVSuSVPFaIRnurkYNU5lJ6Vdkfg4qo4Oc1pFkMi8yRFwDUZmY9TUjdKruBnitCWxWkJ70wk00nmgGqsTcCaZinmgDimFyLvU8bAYFREc0A4NDFcuoN+PmAq5bBA20OPfmscnIoVtoqXEpM6YuvZgRUclzsU4rIjujjGeKeZyR1qOUdy19uYnGDUqsXwxJqjF8z1pKo2DFJopD4mYN1NSHLkZJGD2qqWKnipInJOTUWHcuKwyAKsqvFUlPOaso5qGUi0p7VLVPzOetP8w+tQUTEikLACq5kJPWk3E0WEStJihJMmoTmo2YqeKdgLrSccGq0svJJqIzH1qJnJosAjnNROuRzTy1RySqoOTVpEMrvErDFReQi8jOac0wzim781oiWhDxTVPNKTmm9DQBaU8CnVCjHFSA1DRSAnBpNxoPWkbpQMN9G+oieaN1FmK5JmkzTQwxQTRYGyTORSjrUe6mvJinYRla7pmXF7APnU5Iq/YXS3dlG4HzAYNOkfzIXQ/wAQxWRo032a8mgzhCcAVVgNeX0qPYzD5RmrW1SeamjiU0yWjOMMg6ionRu4raMS46VXktwT0o5g5TMCGmlD2FXzbnPApRAFHI5q+YOUWyhBIPHHrWzHsVBiseImM5qY3vO0ce9RJXGtCxfkNCcHmudc5c1pPMWfBbNMZEbGVGaI6CepnU3ca2fske0YWj7JGGyVAxVc6J5TKEUzchCRUken3Ug3GPYvua2olCt8tWFiZ2pOp2K5Dnhp13gt5J2g9c1Lb2cxlAMZJ7106KduKd5K1HtGPkKMETRjDDBqVnxUzxgDiq7Vk2VYesrZq0vzCs7fzU8crFcZpMaQ65MYQl6wrq9tYCS/AHer1/cQbzGUfPdq5nW7a38lp4i7HuuM4pcw7EWpXct6u20UqnqawZnlDeVPKPcCrNnLO6BY2wT09KoXtqpuygmdnP4VlzBYtFLOOERiQFj6Vr6PezWvFtH5nOdzcYrnksJIJfnTP41NFa30kgWNiiHpihSGd9f3kU2mybsSOBkKBnFefG/nkVlLYYHBFb8kN/YabhTnd1JHNYMIgQMZR8zHOaq4F/Rr9LaUx78O5zirmsSztCSjDnqAc1z73FtBIGDVo2+pWLJkt+FFwKMbyhcNkfWn3UckEBYuCT6VZlubO4bcjgD3rMvLvzN+FIXsT3rNq4GbNndyc0qKoPHJ9KjY7jmrNl5Rk2Sg/hUjJQsnl7wuMd6heVmPXNaNwbeBCACfbNZbzLuJCAUgsMdzgjBFQruz2x9adNOzZUH5ar7ip4qwLiuRwBk0xww5IxUAd8j5iKeVkkICsXPeqAN9Kpz9Ka0ZQ4brQD2pAenXWpa6Iibeytx/tZzTLXU/EH3riyjlz0ReP1zSNq2rRRMZ9LAX/pg+4fyrLi8bxm4kim8yGOP725a7DM15r7xLNn7PpcMPpufOKxtWn8V2VqLq4ukhXO0qiggGsfX/AIlzMrWukRsg/wCe0g5/LFcVJquq3bBrq9nmwc4LcVLYHoF1L4rl0smS5s8H+NgQy/Q15xJdape3LWrqbuUHG7+9WtN4h1G7tWtmu/KjC8j1xWFY6jc6dKZLaTaxGDkZBqLjL2j6a51BjcRbfKzxnkVflvLd7hwBnHes6HVrmR5N3LSdSo6VSkkaOTC1m9S0aMhiuHLNggHiqt24wEBJWkMyqVJABxyO1RMN8u9TlO4NSUyNgdtNjZ4ZN1TLi4uo7eMZ3HBq5q1jDYQxpFw561aJsZpmJbMZO71AzU0xMigOfn71tiGODSI2jVQ2M881Sms18pZnBwf40OQalBYyHVozkcVaaNrmybA5Q11uhxWsllcghMp91nHY1kQxQtFKse3czfMVOQaYFeJkvNJIeVUaMgEGoDb2kdpFKl4TPuAYAdqksb6a1WS2eKI25b5mZelRuqLqUayRskW4cH0oEdVrOg6VdWcEmkSie8xkqCTuHpWDPqKiWCI2RheM4YAY5rXNxPpFwb6yKQLCv4NXNXOpXN/fNdzMN7PuOBiqaGS37sl0VQ4GM4rSklF1FbSD7yjaaxHMl1fxr96RyFA9TXS6ho2tadpTO1guE/jRwcfUVm4gVzH5uAiBZu+KRXmtJc3OSo+5gdDUJvZLa1ikeNGZ+CytyD/Wolu5rl1WJQxBzgilYBby2nawnuJFIfOQSKZH5jWKeYE3Y/h7irepa5M/kLtTg4ZcVZu7S3sJ1v40zFIPuZwM0WAzGttLe9gWSZoYpPvlRnB9a6ObTfDen2X+o85P7/mHJrk5I1klZwWTcc4U8Cr2oYTToUVZBzjLjrVJgQ2ekDU7meW1TyLNTne/QVSnVYJ2iWQPtxyK09PTVNOsxd28itbzkGSLuee9GvXf290dbRIWQc7T96ncCvcw7IIPnBBGCBUa2j71OwOu4ZVW5IqZIjNYw49STWi9otxpkM0blZYyRgd6m4Euy/0iRvsUV0bQoC+1/u57giqUN5c298LnTISm5dpYx5De5zW34V1d4Fms7sRrCx3eZM2FA5yDx3qnrMenxNu0/UCIs48gdvp7VdwKkWtaxpN5Jc7xH5/38KMH6V3fg66slgN7YN597J/rFuJRvxXlt0xMh/evInbecmo7e6hV1wJ4pR0kRitAHucviGy1C6ayRZBLC2JT5eQp9ARV2RpbdXNikk4HUHDAivOvhxrNnpmqXpu9RijSZV+WXkucnPNemulnewG60i9BVRlvJbIA/p3pgV11VWfZNaTWznp5vAP41aEkn9z9ap2mtabqUf2SW7hldflYNwc/SrOoTQaRbi5mlVLdThgwyT9KAHpuTzCIvmkOWOepqOUaj5R+xmKJ++/ncKyYvG2kTviGK+lX+9Hblh/OtODxDpcjhfMnVj0DwEUwIrXXZ4s/b0i8rO3z4Wyqkdm9K0l1Ldb/AGxDFJaKuWZTk4+lUHtIrXXUeNB9mvlIeMjI3Dv+tN0W2XTtYvrAFjbOokjB7DmgCxL4lW3WK4Fsbixfhri358s/7Q7V0EciTRLJE25HAKn1FckNPj0XXoWtmK2OoOY3i7JJ2I9utdbGJVXbIFGPSoYzM1XS9QntJnsEgkuGwqrJJtAHc15f4gtZdNvRDrMUKSsD/qZ/N8wentXrGoaw2nx5tmgaUdRKcKB7mvMvEut6NqN7M3kQyXrY/wBIVt0akemeakDkLb7WL2WSysfKjkAGxjtK/T3r0jS/Hlhp6wwxuXhZMSwzEkxOPQ9T1rz5biSFrkT3ibzhlI5DD2NabQJrjfZ0SENGAWliiJ59CRQmB19942juZraYNb3CRHcRCxR0HpipbHWdO1jXDrMsbRLEc+ZKQADjFcha31vY3V1C+nR3LSKFViudnHX/AOvVfxLZ2dhFBJFFtecZK54YfSncBPEeuWmo6+95feVdOH2r9kO1WVcYJNZWo6ut6w8q1SG3Ufuoi2Qvv061DFDp7rujiMfy7TngA1nyWt1G0W+3mRJGARyhAceoz1pJgNTbJdbp3b/ezV6y1aW1EjfeC/LHzjim6zYW9rdxxRvsDAZLHOTVZoIYr02zSOoHQquaTEWb6e6vbK2kmOURsBO45p5uWvLZpAso2Y8sZzxVW2ulR5laX9yBjkcGlYFMfYrjCHqE6ZqWgGrPIGHAbnHNTyTRRssfJTOcCqqIyy8ttQ/eNEkm+UJESzA/KwHalYZox3LW7BbaV0dyAdpxxT/3Jl2XDyN/dI9apzSxXSxtuUSgYfAwM05IkNvIHYGX7ynPGPf0pJWGMmit7JZEnhMksnTBA4qtEHmfOflj+YCpJfs32VpDFM5xhecbT61VjY7Sw4HatEBZmjumcsUy7EkDPJH0oe1u47dLh0X7OwzvDdKmgnW0tBdgI1wsnyP1Kj0+lUzcymzniaJSk8nmZDEc0Ej45nGUz9z5vc1NJdRz2jDOHP8ADTLNriDUoJWZdzYUhTklTxitKeKK8uZ1ZPlUAptOM+tJoq5j3jSXEsRPJ2AE1RZWDAkfKTwa17l7GG5Yi1EiL90E9aSPTYNU2tbuLfP8OCy/nVcwFSUYjBB6kACopE2MUJ+cdqvXehX9rh8efCBjzI+cfUdqoxRLKTl/Lb+HeCAT6Vd7gT282SkH3WY4DEcE1eggxHNA6gv1wex7iqtzF9laG2lX51xISDyPapPtMlzetKWA3/LtC4A9/rUMC3aRK7ktnYv3gDirsys0Ko0SJEufLcjOB71ngFGZkXPODzjmppZZ0SMMpAlGVPY/SsuYDYs7KNrSJPILyy9MHrW/4Zgdb9pI08+WwX93tfBBPUn1+lcvp+oBY4lec2+ThHKbsepxWpBpF7aXSpFeo8VycrPCxG4fTtU3A76Hx5bWE9taiwESTTBbmVzyGPcCui1jWUsCo8xY/Nx5SsQCw7n2FeR6vfRXlxLHJA+4IAr92IqpcXlzqLxPeuZYVj8uFicFAPStVIR7RaahusnnuNq7VLYDZJH0qe1nW7RiBgqcMM9D1ryKyuDaQRhLiW7eSM+VE3ymPP3u/wBa7DwbffbVnmt2kZgFSWIjJb0IP4datSCx2JUHtUTxjfmrOMjkEHuD2qNl5qybFdkBHSqzRkOcVobaYyZNUmMzyue5FDysqgVLNA/VTVGXerYOcetWmSywAJ4NrjIrCvrJUfdGvfmtqNjj2qtLG0kpGMr3rSLJM9oEaA4Xn1qmtrI77QOTW48CjAA470kUYE2FTpVc5PKZn9muBy6k+gqHypkche1dELVZYzgdakh09UXDCjnDlMaBJXRTgsCeCK2I7OJ8FhnFWFt41GAOB2pQArZHSs3K5aQhs43fuB6VJHZRqeM0nmYpVulJwDUXKsOkhUDGKz52Vcrnmr28u3Jqhe2ZaQGIkEnmmiWVluHjOM1FNdPvzmpZYhDIN4qC5iWVMw8n0rfQQsN8Q3Jq6t+n96ufbKuQeoo34p8iI52dQl8rnHIp7uHXiubjuJAMZq5DfMpAJqHTKUi46/MTT0YIOtICJF460zY8bZI4qSiZ3cruUZqsJt7kEEU37QT8tJtHWiwh0jAEc05ZxgAmq7EFTTA1VYVzSWXKg0oeqiOdmOKcrMDzUNDuXARUoIqrHIG71KGqGiyQv89Sgsar/wAWatxgFRUDIHFR1ZdOagZTnpQAlPjpgU08cCgBrdaA+KVuaYVpWAlVhUgOargYp6mlYCQ00DmnA04UAAODTw1MxRyKAJN1MZqQmmGmAh602nAUuKYCCnrSAU4DBqgJF6U6mg08CkAm6jfSEACoi+DTEWPM96XzPequ80oYmiwFkNk07JqKPgjNWCBtyKQCrJgjmn7gRVMsQ3NKrMTwaLAWgRT81XBNPyRRYCQ0hqMv70m8GkA481n3Z5NXc1BPGrdqpAYzDJoxxV37Mo6imPGoyMVqmQ0Zco5NV2XJrRaIbuRxUTRJk8c1opENFLyuM5H0qNhg4qeVcHioCM1ZImaUMM0wmkB5piJiQelQlSDUsUgUnIzRIwYjjFFgIhRmlNNzTAMmpUfjrUWaM0AW4pCG4NXkugFGTWQCe1PDt61DQ0zV85WPBqzCQRkVjRyEGrtvcEsFqGikzUUnNSB8DFQK4I4NG8ismjS5aVsmng1TSYg81J5wHepsO5ZyKVSKp/aVHelaYsuFNFguWJZ40HJqv54foeKgEAL7myTUyxZGQMUxDWJzSE8VOIsDkVC8ZJ9KAK0zMVwvWqjJPtyQMfWr5tyx+9inCBQMEk1RJjguzYGTVu3tmcbnbH+zitDyUA4QCqs0vlkqo59apO4DZIdo+UVBtJPANPFw560rSNQIYOOtOyPWojk0mGPSnYLk3mKvU1G1yvY1GYz3qNlAquULk3mlqaZBUJY1LDCzsDihoVyRMnoDUpVscirkVvtTgUr2xYYyBWbZRQORUTZNX2snwSGBqqEOcVSYmRqp71iXaNa6rFIBhd3NdEIuazfEEDfZI5V+8rVXMI0VYEAjvVmI1BpUa3GnRSAhmxzWgsIUdKzciiMnioz1qw6gCq7UhijGMCoZMg0/pTH+amIrSSD7oqMVOIV35bmmSIAeBVokVIxnNSxrl6gjtpJTwxC+tXbe0MPRifrUsETBQFqMqWbAGatLFnrUyRBcGs2y7FeG2I+Yqc1cXgU7cApzUDSDtU3KJvM9KPMqvv5oMnFK4EryZPtVdyM5pN2aOooAiOKQyiNSScCpRHmiSCPYfOJC+1IDnLzUbQStJ5v4YpD4jtLi1e3t7N5XYY+78tUtZGloxit1ZnPf0rmZJZIQYY3KqPSobGTx3r2108Z2QqrHNU7i5U3RlV95/vYxVGdiXbv7nrUIbC1LA2rTUyLhfNi88Doh6ZrVfxeyyhBZIPfbUOn28Fhp4uZVAk755rFurq3kdposCQ+gpoD0mwnkmRBMikN2IrL8VadZNCZUURuDklRxU/h65tEslmuLsFz2JH+NR61ew6iphh5B9+tMDz2KFry7EUJLL/eJrW/siS2ba4L+4FbEdhJaWplEEceO5NY6a5dJcMQ4de4IrKQDZi0bYWB+O9ULmWR48OpGOgIrRuNZuZnDeUigfjTXlS9AeeRFx0GMVKYHPksp9KlgZy2RViawQyHbOGA7gVEI/JO0Pu98VV0UEsr9CaiLbhT5lcjJqHIHWhMkcVG0cjrTcqo4ANMJUnpSdelNIof5Rk5BpMmP5QxH0pMN6kUv1pgJuJ60E+nWjP5Um5c9aYEt54p110ZZJfs8bdkNY1pqF1b3W62bc8nHz8jPrVFJWkcLNISncmoZZTG58luB901vzEGnfR38dw0ksSfN1IGAPwqC01VbTzC8PmO4wD6VOwuJLFWkkJJ9axpY9rkE0uYCSW4EpJAxk9KjyMVHnBxTiCFBpNDHxSFH3DPHtTg+6TOCWpkdw0asoPymptPEr3WYwuR2NIbI5Zct0IYUkEjBZBnkjii5803MhlChs/w1EOKLAmaWiIHuGkI+ZelRXt3Jd3jhzwpxTbC4NrLvAyCMYpLjy2keVBjcc4pWGajt58Eew5CdRU/9opNL9nMe22P90VkW0rR4YHrUsa75Dg9aLAHnRoZIVYvHnAOcUyIsVWJAB8/BHY1XuEaKbDEH3FWYY3heN3IUdeelKxJdup5rHTntPKV95zvByKk1q4iuNM094pA5WMA47Gs69vJbi4ZnZSvYrTVtJzbNOEzGPfk0AbV9q9m/hW2swd84HznuBmufjceWtRMQfpQDjFO4yxK0kMgkX5XU5U+hr2fQJr7U/DUczahDMxBEjFMlPb3xXiksrScn0wK04JZrKDzLLVGhWVfnRWwDQwJ/EP2a2uvs1rP9oQMT5o4B59Khs51t7YSq/wA574rPAi3ZlZvdt1Pk8sgx27FyPWiwD9r3WponXfIOK6nxEIreGytlbKAYz3PSsDT4GF/aM5ZcyDJBq/r6L/agRXkfoxL9M+1JoCnImxiB2qKS4neMNIWkUdianiZXvYzIcRscGodQt/sV2YwXKEAqd1KwE1pqEkuIDlYR0QNzn61fnia6tV8td0oyMnqR6VR0vTUuw8ol8tkIH1pA3neYgPC0gNOzs7a1t4v7TkePJ+4h5A96n1NYLOaKaB820seU+bP+etZ9nZSoyzQwNMewbkVPd232u3EghEMh/wCWfTFQBBBewRCWKWDzFmXbk9BWjc+Hl8iHdqMMSSAAL5RBz9axZhLZPCZoPlySFbo1aM9ld69YTanFdwKlu3Fsxwy/SrSAxGsWiuGSY/IpILqciqK/NMY4wzH2FbLQS2dxb/aJAVlGSRTbTyYmYWjgXEj4DNwCKtAUbNYpLlYpuEbqR1FdhomNOi1C606aWN7UBuWyJAezLWBHoly9/CSUVGbazoMhc9zXQWEul+HfEt5Z3lz5kMsQCTtyAcH/ABpgSXXizR9UuITc6WEmchZChwSfUEVp+HLnT4b2ew1LUZ7Uxv8AuEufmjI9cHvXAWtmt1rDRqvmqJDjtkZrX128t73ZE1sRNGu0yDqR2o5gOo1XW9W0PUpTbXSxoMeV5SjZMvr7132na7Y6ppqfaZEgd1w6txz6+9eVX2lRR+F7LW4DKNpWKW2fkA+ora8LXq39sbDUEjEgbzICx+8CeQSO+KnmA6m+e/sbWGS0dJFhmGSw3AA9CP0qbSvENpdap9mvYvsmoRqVZWJwf8Pxrl7vTLtPEc8WiyNEioJPLLEo+Odtalp4k0q91y1eeIF7lPIulZMhXByOfxq1qB272yybRIgYKwYfX1qyAaSGD7Ogj37gOh9qnUA9akRy3imzuFsLiWHU5LcSjb5UaLhm6DJPOOlcDa6DD4US1vNWkiYTZWRHTOz0217NPawzxESRo+Adu5c4PrXn3ibwxDplpHqMoeZIw5kDvuVjjgYPSlYaZ5tdJHqbX1zZ2qR2qN8u4nJH49Kt6Brb6VoNxBaTGJ2ky7qu5seme1QC5g/sp2SMLFcOwZTyE+lJ4d1K10syfabead5iCgiUEnH1pWGdRpbSQRWc88UHnX0TKqy5Akwehx0rnfEtpPp2pul45Xy8N5DNk4PXHHNbniTxXYXdgGfS7iK7iGLZ5GACn19+1KRrXxJ0Gwcw2yi2kKyTcLk96RMTCtreO3VdZubeJrcNn7MzlQ+Dwa0dX8QL4g2W7WHkO23yQ82VQf7OBxVnU4LydjDpsQurOJAqrGm8Fx2wenOa5O1V9K121utUikg8tjujZDlfw7UWKHXb+fF5U1rhnbaszEkcHtU1m39nSm7gmSbykwVddwq0y2WrJdXdpbSF4MyKbhuGHsOwrGlvHmSRQApcYwo4oAfe3DXl5JctDFG0mMiP7vA7VFDIoZ1fsMjFMjikTS2m2g+WcYPetDwxajXr8wSgqlvGZcouSSOgI70rAZghebLB9yv91VYAk+9RxiW1mMiD97ETXq8vgTTEeDVdZktopbpfmgQlRn/ZPavNbuJNP1K4aKNhAkjINzkk+9FgGXM1tPKl0gMbucSIegNNW6gEThisLEYEmMgfWonNrLKiRBlD4xkdDVa7tJrOXyZXUyeg7VVgLUE0cF6Yppd9swwxxjNT3L20kUCW1rvijbl2Pc/zqlbxafc6ikNzK8URGAyno3uagb9xf7Ipd6Ry5BzkMOKqxJeurWaKyF01qyW8rFYyO5FRNsnELp50iAEGNOxrZPiWd57V5TH9ps598cO0FJAeufwH61a1XX9JurCa6t4BZ6u1wshiVSox9BxSSGYcLrY6ilysbMFXlX96WZt7OVJ2SfMATnHtTXuLi9tRJO+5+2RVS6AjlCrwpGQKTQ0XZxBsEVvGA4GNzck0smq3MkcjKY4tmAY1GOB3p1rPE+mzPKw82JgASv8ADjrUJheS3E5Rdp5LD09aVgNDTLmS7aQ25ZZFA4Y4Eg9Krays0UqLczOwkQOEdQCh6dqZYxyx3y+W4R0+ZSehH1qza3dpL4ksJ9bkLaeJP3ynOduO36U0A2x0S51XSjdQLuMcxi3O3AAGcfkD+VMihhKhkIH1Ndh4hjtvCsYsdDuvO0zVoPtAMp+52yD7g9K4gIq3KrA7zRZwrKOST6UMkmuJiibRt59DmtS4Vbnw9ZRRyJ5kRLj5s5B61S1ErIyQrbSJJDlZNwBJPFX4ks7Xwc0olje7mnUMEbJRc/xDt0/WpcSkVF+3qgVQzgdMgHFTJfSpkiB0GMSM3BUVSW8bS2jmtpEkeQfOp5CE9x6VPPdxXlsbZjumfo/QFvWlyjNCx1CO2t7mHH2lCV+8eQe496pm/VDJGYnSCaTzIc/wjjg/nSSafJHKnl3G10iDNhMnPpS2WlS6pYSi1lhaWNtotmcK7d+M0uUDoNNezniXzJ2m8q4AXyz86r14rsNKv2l1nzrONPLtX+bYMGVG4+77f0rkfB2nwafHHfXkht7iC4EexUyyMfVe/tjrXqdro+/U1vInC2/2cLuVQCzFs8j6VSiLmNGzkaRGLZBznB7VNt5qRYVj3bf4mLGgrWiEmIFBpjR45p+CKRiSOaYEW0d6rXNuHGOgPpVs5pNu7rVXAxzE0akYJqMpL/Aprb+ze1I0OO1VzEtHOyPIpw4PXtVuGM7eRzV11+bFIEAo5hWFiAjTjvTt2acIc1KsIAxSuMqbWL5B4pWVguauCJQOlRtHkEdqLjMe4nlVgqLnJ59qW3Ry4L8VLPEBKD0xVaW7EUm0HHvWgi+eKkVg1Y7X7FsKcmpYL0seTzRysVyfUbdpImZeorPWF4YmVuhGM1qGfPGaUorjkZppiZy6wyGXaeferqQ2rKFb/WjqK1xaRA5C0gtIQxJXOavnJ5TNNsrfdFKuls3IdxW7b20QjHFWooox/DU85XKZ1jYmIZOT9aumJCCCoIq0VA6Cq8jhM1ncqxhXVn5cvyDAqPeqKBn5qt3NyrT4Jqo6KzZrSJJWLHdgnApqyYfBqZ4TjNV2TDcVpdEMtLIARk1KHQg1TRc4U1KqEHaQaTQ0yeBVXhT3q2rVUhRhzirCoxfNZNFpkynmp42wcVXCkVKg5BqGUXFAK801kGelIGwKjMvzYzzU2AcVFN2ik35p/G2lYZEQKaRTZJCr4FKHyKdhC0opp6ZqMvRYZYBpwNVg/vUgf3pNAWVwTTmxioFfinF8ilYBCabmkJpuadgHr1qTFRKRmpVIOKLCFC04LzS0m4UwHhRmkc7acHBpj880xsiLnNN6018hqeo45qiQGM4qZE9uPWosDNWAQE60mAuwipFOE5qAzHBGaYsme9IBZGG6nI+BTNoY5NSIgzQMjad84BIxUiznHPWnGJcdKb5QFAChsml5pAuKeCMUAR5YGgHNDMM0mRQIR1yOKgZD3FWM80FcimmBnyDAqnIPmNarQlj0qu9t8x4q4smxkyDOarOMVrzWmFLVlyr81bRkQ0QsMioqn2MegzUbLg9KskBSmm4pKZIUlLRQDEpaUDmgg5pAA4pw5oVCxAFWBAE6nNFxjFBxT1cocig4FSxqrLkipbGie1nZnBIIHvVlpgTVCRig+TgVNAysQTUNXLTLJLHkDikcMFyaeZVHFJ5gapsMrb2zU0Uh4FPEUZJOKUIoPFDQFqJsgVMpqtEQOKsxglulZsokAyKhkHJqzjC81EwyalAylLIY+lQ+exp91VEuRWyJZaa52jrVdm81smogf7xpVkUUySZUWpPKDCoY23GrCrSZQsVpGRliT+NOeCNB8owKerAd6RyGpAZ7nkioWUk8VoGFSc4o8kZ4Wq5hFOKBmcACtWG1aJAWFT2kaxKGA+YVNK5frUN3HYhBC0yRi3ApzcCowwDYxmpKFTOMGozbfPuz+FWQBTgBTuBV8gVBqFj9psJF/ujNaW0Uvl7kZD0YYNK4GB4PP+hzQH7yHJ/Wt01y2jO1jr89uxxk4rrClTe4yvIpZeKjEJxz1q5tFNbAFVcTRUMBPUVFIoTOKtMx7VVcMXOelNMVivGkkkw5wtW2tg3BHFEaEMMLirR6U2wsRBQo2gYAqSNMkU09aejY6VDZViwYcDOaYzYXFO875cYqFiSam4DWkPeoy9K4+XJquzHNIRNuBppbFQ7zTS5pjJfMwaer571VzmnruzQIuK1VNRL+S8wf7gztxwalDYqle6xZWIxdNgfzNIZx114l1DlDbxRj1KZqpcOBbK5gWQt1wOa0da1KxvIN0NvkDvnFQaddSJGGa3QIOh9agDm5ZBI/l+R5QqJEidwol/Ejity/a4uLppAkYB6AiqUlpNcDBgxik0MgLQv+6ad5F9O1QvHEDhAAtNLBZdm0KPWonZt2AflosUSrgHKZ3dsGpoby9hkBjkdSPbOKq4dU3BSMdyKfFN8/zk59qhgarXt3Km+5l8w44yaz1wCxHUmnSujkGNMD60xRliMj8agQ1zk8mmFMnO4D60SnDdR+FM34p2GSn5l2qwFMGEJGdx9qjD5oLtjinyoQpYucHpULqAadtyC2T+FCIucvuI9M1XKgI1jLnjFDrsOKlKLn5cgfWk8te+aYEWTSn7tOdVUcE/SoySRQMQ03OKWmnkUxEmh+GdSvWuI47SMpnaZJDjB56Vd1D4eXlnYNJblXZeo3ZOK6S/uv7C8RM9zv+yAkoy8YNcvqnje6+33BslfypPwA/StCSLw5e2l3aS6ZfLGjYwjHjn+lcvqVn9iujGZfMbv7VXlLNK7ldu45x6UzFIAAXvxSY3SKuTtJxUtsqG7QSkBO+asiGNtUAyFhVuvrTQxLvTFtPLzIW3+2MVC8BhUskyg/WtS/K6m8dtao0jj0FUHiezaSGe3OSMcjpVkkAhnliM2wuo6kVJbWiSIzXKSL/dwK1bVYjoki26szsMMoHStXw/4bu3QTXF28MWMjod1HKHMcc+1HKqcqKeVT7OZPMIYdFHetDW3hmumhtQj+W33olwGrJdT5ZB4I6ik4jUjSk+xrZQeRITKfvAnODVWcyJIFyRkdcUEiONf3e3cMhu1QtM8nDtkVLKLDYk+Y9afNcGTCsMYGOKgR8NtPSnsB61IWI2QcbCfxqYY+zBC7D1GeKaBilMbEHfG6kdiKYWKxG0bab0qQjjNR0AG7nNSKqtyRzUVTjCoMdaQCsT0xk9hWwu97Xa6Ism3sMVmqkaqJHPzLzil+0SXcpz1PQUMDR0+ZoFt5s52N37V0OqpFeaKLuJRvRhuwOxrkS0iYRBnHJ9q3EjebS1nhnDW54KkYOaiwEujJaXcJKxf6Qgy0bDgmkv7e4vlK+SSg+6V6g1lw6o1nfxSBQqqcNjuDV7xFd3FtLGbWQpBOu5WH8RpAZgP2ZmB5k9au6YIXeYzPgKc4xy1QX1jKjwAuu9ky+PWoVVk3FRkr3pgdRp2ri4uhZWpCRgEneOfwqWefF4beRsuO1cuJjCDcRMUlHcGrV5cfZ7qGYSeY/wB5ju61m0Br6hO0bQmRQ0anOD2Nc5DdtHLdRyoxW4GBjsa1vEl15j2O18QyJvIx34rKjuIyrgxlmY5DDqtXFgWraexltbn7WrPMylRuGcYqnbxN5AB4yQKBCjM0aD94RlakR5QgWMorKfmD9c+lWmAyW8voy1sjzIQcMqjG4VDDFEbiMXsskaZ+YnkqPWpDe/aLsSyLh8YJUdainlae8bb8zdhTuBv3Vjc+Hoo9TsLi2ntJsKpxkgevNZ1/a3Nr5ck6uFnG9N3pUR1ae600abNbRsVbcjHqpq1d6mbnTbWzuQS1vkK3U4qQOrh1SzufCkmkmZN7Y8nJ6H3NVLixe1+wPJF5TxpvYMfvkf8A66zpp7a5lsJbuxmitosRzMnUntXQSalY/aLawaZJ4onBE7npH/dP+NMD0fQLeEw/awm15JPMAzyARxXMajZQ2PiG6sbdATdSxyIrDhTnJOa2JbnUbC0thYPDGhYhYpRkMOoAbtVvSNQttXmeeW3FtqkRCSQP1B9qpMDoHYZAB6DFORjmq287vm609HHJJwAMk0AXQx6VneILK01HSZILy7+zQn7zeo+lVv7ftVuI1WTzonYJ5kfOD6Edq05LWB2M8kQd1GQTSA8U8S2dil5FFpWopeW85YSQxxhAuB1AHr/SuLWzSN90MsiBeVIbvW7foy67dz6m8yvJIwRkAAAz6Csa5nSO5ljWBHjcYUtngUhl3Xbyzu4bOG1u2uHjBMofgEmn6F4k1Tw2twunusaXGN+4bhkcDHpTNPeGW3jhktopLgDEW043D3qpOlzYyPbXMQaEtuyvOfoe1KwjSsnu7+3kdFvIruPc/mREgS5OTwP881l67dST3qMTdbFQI32k5bI963fCmosNXjgaSeK3ZWjhDHAUkdc96TxNDAmsmW5cTy8KXjOVl9+OjCqJMCC982J0f5MRlQVJGTU9m91HZBoYldx95m6gU+00GXUbDUb6zSRIbJdzI46+oBqpo01pPeiC9Wd9+FjEPXNJopM6/R4LCd1stTvS1vLbGVZCfL8pvY/p7Va0yHTvDlzaaxZaVcNZTwtBcpKxYhePnDcd6yde1aOwvbY24QvZrs8qVchx9KINeudaigt9WmaKzbO2KBQuT2BI6/lTsM3/ABdaWVg1tcxS3N3p90AyQGTeYxjllPbFcrc67by2U1hJZmewI+WYgLID6k1QudXOl3ckGl3szJwFJOBj0IrOF3O4eKTbslYsQB1NFgEuxHFcIvmiaFG6jg49DUYZ2uDsD7Tzlu61ObltwiOwY77cmqqusbuZPmym0dsH1ouAx1XzCFbcPXHWp7e0Z7iBAf8AWnaVRvmxVi50pIIt8d3HPgZYI2CBVf7ZItmLdRwrbkkz8wNVcQk6xxXrwRoYwpxtPUU3Zul29W7nuae6PNaNqDTGScybGBA9Mg00mERwSK8nn7v3h6DHtSA047O9lsEeOHESr9/cDmoRYXWqQS3EQUvAo8yMnDfgKidg8BbOWxk1o6PPHp95vDSEyRkSEdhUMDKhgZwFY7c9CemalW1uYj5ZfLZAXYeCaa0ckkcpRNyKck+grQidrvwy8m4I9tIFUnuKBhZTzWWuQwXhEsTHy3QjjBrW8RalZ3EsVrBEq/ZX8sORyRWWZo1+yy3JBS5wGbHKEcZBpLZRJfqkkiorSiOV2Gcf7Rpkk2s6i2pXqygFY0QJHGRgKB6D9apeYVRVxtI68dTUupxNFcShSG2kpx+FR3gieyiuIZP3n3fKY/MvvSYFaGaaOVpBK+WOSc1YWRpWO4bEb7208HHTIplsYZIJYTMkcxOV3nApkhEeUjZJvLIJbtRYsn8kXEqxWY8xj/D0oYLaSxNuzvXcp6VY0JpJ9TYxeRHL5bbRIdoJ9j61FfrssrWLBMoY4wPek0BYg1P7ExkgicSsMb1b+hpsWnGaD7Y5zLLJgkHnPqaI9O85Y0EiM+NwUngkdRV2ylkEpFnHGEHVSuSppAdP4Y165htXS9sbW4tIHSKdwmZz2RvevStCku45JrO4l2sB5kLYJEie4PQjvXIeF9Fhn0S9v7gOftRVGiUY2lD1/OvTBbmSZbhRnEYUfQ8mqQmiSVGSIuBkjtTS6lFKnqM1M86xQyO/3UUsfoK5yPU5mvYUhtneGRTIFyAf19KYkjbNNPWs2fVXGoW1pHGVMwOWJ446getaQUmmMTFPVcmlCU4DBoESBQBTJFwtOzTZCSuKAM+YYJNRLyatSKCOai8vHQVSYWFDEDilWY5waAjEdKPKYsMLQInXkZFNKMzcU+OMqMdaUh1PApDK7aeJjlxVebRbTqyEn3atJRIaDC5680XCxz0+jgqRBwfWqBsLiGYJtyO5zXZpbf7P60rWgP8ADWnOTymBaWIzuck46CrjRhf4SaveQqngVHLhUOKXMO1jPZcVSe4O4gdjV2Ukg1lTAxkk9KtIllpLkqOTU8d4q4JbFY5dQc7ieM0tvchiVPPNVyILnSRXCydDxUVz0JqtbSBcZq0+JFwDWYzBeKS5v3G3btxTzDJG+04OK1im3kVmXEjiU5FXFkNEqru7U7yYf4oxmoIWk3/7PeraruGTTuOwwW8OAAKmSFMY2g0oj9BT1GKm5VhpgTGdooCD0qQk4poNAWEIHpSgUUo60hjSTUEqv1WrWzNOEXHWkBno8gfDGp/O2jBqVol9Kha33Kc07iGsyOchqkjiyvWqcsTLGSOtVUupkIXNUkK5ruuFqmTyakSdmQZNVZGbceaSiFydWycVIM1QRzmrySbgM03EEx+cU0vg1MqBgM0ptVY5FTYdyHzl9acJFPeqlxaTRykp90/pQFcLzyfWnYVy2XHagSD1qi0jKeaYJWNVyi5jVE/bNDSVQiZt3JqaaTYQME59KVhoso5zzUhcY61nCZt+3BqwrGpsO5Z3CgGoMsB2pn2jBwaLCLlNLmq4nz3qRSH6GnYYFjmlRxu5NL5ZNN8s55pCJw4HepFcHoaq+Rk1MqYUCgZLvanbiaiGe9I5wvWkMeZAKb5tVjJk0qtQK5OWzQKjXJ61ICBQBKq8U8CovN9KA59aVhjj1qNzg0rMcVWkc5qkK46QBlxWXc23zbgODV7fUUnzDFXFkSRmiMqMYqCVea0GUg1TmHzmtYyIaK22mHOeATVkHAxip45kAxtFU2KxSaGRApZcA0iqScAVrK0cqbW6VXaMK2MYFHMOxWEDYB4NDQsBkirca5YVd8lCuSKnmCxRtwiDLLmms4LnPAqw0JHaonssqWycmlzXHYquwPSnxMdtL9kYd81ft4UjjwT+lNsmxUEcki7lQkUj+dCM+S/1xWqrgDFSefhCueDU8xdjDFzJuwav7SsIekNtA8u8sV9hSXcmyEKhp7iBJN54NKzsDiqUUpQ1YEuTk0coF2DezjIrWhGABWTBcqQOMVdS4xg5rJmiLrJmo3j2jNMW6zUyzKRWYGfcwFgTist4WDYxXSMARVKWBS2cVUWJoxWhbGMU+K2OckVfaH/ZojjIPSr5uwrES22DnFBWrmKYY1PapuVYpEkGnocnFWGgB6CiO3I5Ip3JsKseamWIAdKVFPTFWETjpUtlESqQKXbVjZ7UbBUgVXjyKj8vaau7BUbIDRcCtmlXLU9YiDzT1jwaLjFValBC1H0oJzQI5HW0ay8RpcqMIxBrrUbeit6jNc94wj/0aGYdVP8AhWppM6z6ZG47DFIouk1C55qQ0winckiIPSnLEvU0/FNJqrgDFV6VC8nNK7CoW5oaEKZM1LGeKrDip43GBUMZYzQDSBhSMwApDGyKSKrFDmpmk4qIuBQFyMqc0hXilMgzim7hQIciiphtVarhgKUSc0wCQk8VDeaPbXtrmSPL+uan3AmpQ2VxQM8y1O2W3mkitmkO3sK1NG08vaGWa7yyjPl54FdjFYWaPvMIY+pqaSCB4iixhQfQVLQHmd1eE3jHy2bb8oCiqsup3EWMgrnpkYr0YRWOmxSFYFbAydxyTXneqQT3l292+I1f7qk9qloZlyOjuXbqe1ITGVG0kGnNFtPQGoyvNAEoaSTCs5cD1pCmx8kUwErQWzjJqGhkhnI4XAFNjlIkDZyfU00qpGakQIBgiiwDZsNl2PzdqgzkVM4G7gZqNiqn0NADQRT8FulK0wKbQB+VEbBWGSRQA5UIHINNlhkVdzLgVPu3OqQkux7VbeG9hg3S2jFf92pAxgzCnLIW6IDUsswY7fI2+5pnmcYAH5VaAiYnJ4puPSnPkjNLGo6mnYCMqaAhqeNFeXCvyP0qd0BGF5NIZr6tqOrT2CX2t6G0iscbyxAH/AfyrntU1jTZNGMFlbGOU/eIXGD/AFrpPE/iC21aBd2sW/2cdY06t9a87uTF8whlMi5+8RitCCzpjacu6S82+accN0NUL7yZJmeBdqdhnNMC5pCjMMIMnsKAIOB1o8wYwDircWl6hJ8y25wOfm70+4lUkCVELAYOF6UyjpdAjutN0g3cMNnMzHPMuGrKk195bqRzAivIMeXngVh740cYLBB/CDxVy7ubW7VNluEZRgEGgk0bPXrjSndXs4WjZt2M4xVrUNc1O8tA4fyrQ/wrXK7iwILE/Wrq307WItSm9B7ZwKvmA6Lwpc6dp/n/ANozxq8uNu4cY5rK1qXTle4jt2E29spIvQD0rLg8iW48q5k8pPVhUl3FYIMWkpcDqSKrmGU3keRArHIAwKjXlsVIqhuCcVJbwgyfNgj0rIYwHHJp0dwyPwAfqKWQDzCAOAajYDOQKGDZaWTZGXzip7WWW73LIxYkcZqjI+YduOlLDKY8gZ5HrSAJ1McjJkHHpUBNObrTOtABuOacCaaKXNAE8su9R7DFS6Uga8MjfdUZNVM9qniuPJiKoME9TSsBakl/fSN03VCs7RZEMpAJzt7VW3szEsaF+dsRgse+KdgNBp1njwwAb1Fa+lo995NrO0ZihXgHnIrGSylMuxBv9xU0Uxs5ArLkjtnFSBpeIrkJeRQwArtTaAR1FUjvhUK4w3cVJealbX0kMsrhWT+DqaJCk2Juq9qTArTrt5ByrAGiwtvt4ljEgjK9M96tXVkxRZYRlCM47Cizs/McSKrEL96NWwaEwK0skjDZPKH8v5VI9KZbSMsw2nHuelWvJ0796FWUB+gJ5WoYYI4Mu0hfHQGncCeaxlRSzFTyeFOTTJrWJZW82STzjyMela+lQ20ixXc0oDc/uvSrtzBFdSm6tpF3xfLtPf8AGs+YDkonWCY7qvacUkvyYPLVxg/vBkYrXvdDkjij+1QoXkGU2n58DtWH5UdrdusZeN9pzu5qlIo2NOl01b2U/Zx9p3ffkHyqPpVy71mxlt2t4beMbujkck+1crC0wbjDMepIq3ZyPJdqEg+0TD+HoKLEM3EnH9mz2Nwj5lbeDjjFVdLt1aZXEImSFvniPVhVq83PFHEYQZlwwRD/AFplkY9Kk+2XKOijpGxySfY1HMM7KzvIdQdrNr6WBC/7mKVehHYelT+Jkt3trW9W5RJraVY5pEbaxB781i290uuvbpHdIkwAMQcckfXirx0zVBqwedLWeL+LaME1pGQHX6bdSR7R9qN9ZygeXP1cEDkH9K01KscFm29wDiuY8K2sunxXVtKBGjSmSJGPI9R/KugVqsDlvEVhdopkt9QNnbImIo0GS/PfHfmrWk3N7plvY3c7XLSXf7to55dxXPoff0rdeOJ3QyhDsOQG6ZrM1KGTUdV8oSOlvboHXYec9qBGB8UYbYra3SgtcfcijRcZJ6k+uK8sl85ZWWdHXacb3QqP1r0e4ih+0/aJLi8ju4zlFkXvj9elXdbW51zw3aX8N1DtwBLbyxgq5z60h3OF8Oy2KT3a3U0ClVAiEw4J571vfZ7l4rqMxRwxXAVFUOHJ56gfia5fUNPGo6y1raWtuBEAJJLZflUeprb0STUvCqPA1gJ1MgkFwnzgqOoBpEmlDokz6nbeF/OgEPn+YjuD8x/u5/h+tat14U0/TPEHmWIitbqMAG2c+dEQe+eCPqKoT+KY7m1Go6VayWl9A5KswDB1bgg5rqrmxY2kWpa5eSS3EsfliOIGNskZ2gj7woL5Ti/GXilLWyvNEjETSzhQ0kT42nOfxqPQ00zW7dHnsoLXU7Qo0RB2+YQOD9Ola17YXC2qf2dpttHBz8104d+3UntXKXF5FbeXcyMkl8kmA0eREy91HtTDlsVtWkubHV7h7sQu8zFmhVwxTp6Vj3Nz5oRlVkdD8uDwB6Yq7qM1q1qRJb4vZJd5lTuPSqcsLxbUELIVOVLdx2pCNKKHTtRmS4uHgtxtIdHbDE+uKwbomO7MQAxE3y4GMitn7bPNHc7reI72UDIy6kHrUev6zBq+sxHy9lpGojyqYIPf9aaYzMLpvWRlLMei+lM8syyhSQoJ+8xwBT0heecxW6GQg4UIvzEVHJHIsIfnaTjnsaSAddWZt5G+eJlBGCjdarsQuMGnM28qGGHx8pxgGmiN3yeAAecnpTsBJ5jS5YkcDoBxTi5eJM4LJwpx0FG6KBn2Eyr/AA5pY2DyopAXf39KLAWbNI7m4VJ5fKTbt69TWrodk13JclXw1rHnJUFWrNsUV7ww/ZfPd1KoMgYNXptQFvbGHT4vsz7Qk0i8Fz3BH+etDEVWWVEaXySFdiQD3HcV0GjroVzpF/YyLOlxdvvt0KZELKOu7pyeK55dQv0t/Ie4d4cECNuQM+lamjTf2jZz6Y0UbOI/9HYqCQ2c4z+FSMzNSju9NlS3vbWPj5kKsGBHsRUl/qMN5dLCxEKgYLg8UrXEL3QhuVKqf3ciLlm/D8q0bbTtPh86OytLq5YgAwzxbTtzyRTEZMt+jyJBFkhFChiMbgKlhmh+x3UEiqJpSrpMwzt29QfrWumgRhReao0VnCse2COQ/vZD2z/nvVKLS4pNMllkMhuC+1FH8Hsfr/SpAz4IxOkjuICzHIyM4Pal2HeyMmx3H3QMCtCG2uGWW2CPH+7JQkc59qz7fUJYrjzZAHJXY6HpRcZMkDbimM4+8vfFQOfkVXcDaMDPpT/tdyqMIlVg3GT1A9KnEkEbjMCSwlfmVh0PsaYxYZY7Se0MsEi+WoByOoPetd4okSUWUvmGZ+GTPy+gbNUtUPnPGwmd7eMCMCRcMPqe9LFNIIVTzhHk5yBgcVDA3dI8U3mg2Jtt4dM5KqxBz7V1/h7XJtb1VxfXP2VFQbd03c9CM155LPHLatDPFGhLbldFyT7GgXQOySW3kJxgY+63pS5gPavEuvy6Vodx5CeZcNHtWTjbjPXFcfqOvXOm6NZ2lucXGxVkkZeQp9B+FcbFqcMurQPArMAvlpDJnCvjAP611Gh2Op6g93NcQB7dMwShmzIp7gD8KvmAvaTLLd+KbWG/mZIYt5iZ25L8ce3t9K9JWQDrXkvg9V1CwZpk/ewXKt5jkjIOc13tlNIiAyzF/TNWkQdEJVp2c1mxTCTODyDV+NvlFOwySkYjFKWGKiLUhiYBqN1wcink4ppOaaAEHy0vmEcYFKDgUlAgErIc8GrEcquMnaDVVqZQBqAj2p2RWdHIUarPmZ70gLAYCkdhioN/vUTykcZoAlLg1TndeRmneZg5zVG4mUSNzVxQmNkdMYzVSVInjO9sVXmukSU5Gaj+1xS/JnFaqJBaS3gccIDSNYquWjRQabFOqYwatCZnjJUAgHvRcZRe1miO/eSMdB2p8F1OJQhU898VZium3EMmOasNublcA+opNjBpBjB4NRGIPziqlyk0bPKGLDOantbpWh3Nx9aEhEotgelWI7cAYNZj6iVchT+NSDVk/vYp8rA1dqBapSuVY4qv9v8AlzuppulmzvxgjHFCiwJftGe9Hm+hqt+5wADwKGkjUbVPNVyhctLOM4NTLMtZsci4yx59KDcD+GjkDmNQTR+tO85OxrK+0qOxNL9oB71PKHMaZlXBFM3VQ+0EdDTvtBx1o5Q5i243DFVJ7dG7Un2hvWkM2e9CQrkXl7RwTSbCetSMeCai8xuxq0JgVwaUNijBIzikCnOe1AFhLhh0qeO6YHNUcYbGQafnFKwXNA3IdcGq8innbVUuw708Tt0J4osO4jx7xz1qJFMTHcKn3g9KcMN1qkIh388VfVQQAaznTa/FL5zKoANS0CNFkjJyaQKg6VlPNKWzuJoF1IOrUco7mvwKayI4wQKzvtjetSrdZ70+ULk/k46U5UKnNRrP707zh60mFy0rnvT81T+0e9SLMp71Nh3LQIp2arecB3pDORSsxkruaj3E9ab5qt0prPiizEP2Z5oxg0xZfekaUdqLCJPMAo8wHq1U3lYGoWdiapRC5o+eo6HNH2nFZXmOGzmn+cT1NVyCuaBus0gkVutZ5k9DSibHejkC5bY800NzzUQlz1oJzzSsFyWTBWqcqFjnFWAwJwaeQMAAVURGYUIODSBVHTrVmaM/eFRrHlqu5NhiOAwBNWEdScEAj3pkkCgdMUnlELkHNIZcTYfQVOjAVkpKVbk1cjnBGahxY7l1gCOKjcfKcdaFkpWYHrUoplNpGDYxUinIFNlZQ3ApvmD1qyS2qE9aVgAKijuF28tTZJgRjNLlYXK874bg1CWLjBol5Y0iD5atECqozzVlYd6ALxVY9auwEeUMUmNAtrJCNzkY9qUSEcCpY0bbhiSKR4RyRUMsfFJz1q0j81nZKMKsqxAFQ0CZoLL60rMGGBVNJcnBNSq4pNF3JDHTdgFOEmeKCalAkNxTtgIpKdmi4hvlnd0qZY+KRWp+8CgYBAO1PAAFMEgJ608EGkAu4DvRuFRPntUYYg9aALBIpppgbmlzQAuKaeKXNMPNACE8UgNLtpDTAo65AtxpcqnsMis3wnOz2bwnqpzW3PH51vJHnqprk9AmNprEkGcAkikB2HWjFJnk0vWmIQ9KiY81ITxUTtxTAYRUbj5aeXFMY5FMRXJo3Gg02k0MnSUjvUhkBHWqmcUF+KTQXJnI9agZ8HrTCxqMtQkIkL+9N34PWoi1JmnYZMZM96A+O9RCnAVIiUSVIkvOKrgVInWgZaBJNV7rVLLT/wDj6m8v8M1OprH8QaKuowB1JDrnkfhUMZzmsa9Dd3ey1kwnrWTLbtK24TSyua3rPwctxGXZn3AE1qaf4aWE4ZXU+u7NAzkU0O7liaQqVVRk7zWQW2uR6V6kdDurj92bjZCeq4/+vWJ4j8LwwRD7PLmRjj5hioA4U7pJRtGc097cxrlq6Kz8MsqhpHz/ALtUtQ0S9gcuctF2zVDMQnFPjUMMs5X6VPLaSRrl0A+tNgtZrh9sQGRzyaLgLAqq25hkCn7C8vmR2yke/FSh5QRbtsAPdetTNJHbARyLuX0zjNFgKctvJIhkxGvsDUtpNbwxYkwWPYiuss9R0/7EEtREmf4W4OazJPDc0s/nTPFGPYA0rAY0bxrciZOCDn2ret/ExCiIoHPqSKy760jtCV+R8Vks4J4AH0oaA19b8+/kE4iAAGBisfYU6inec/8AeJ+tMaRj1qbARS7j9PSogMnpUjE0wNiqAs2+3eFY4rYhs7ZNskk0mzuAKwAeasgzsMfaCF9CaQzk0urZ3AaBIh6rRNNbhMKc571BFZm4zhguPWrttZCAjOJCOxHBqiCLT4Jb5njiddiDJBqfbHZuBnEq9DWwLqaO3Ij07Yv8TIKt6NqmiR/JfWbPMTklo8mmmBni9tr63WKW7SKX+8TwKztQ0Kayia4muoXQ8qUOQRXf2K+E73cLi2O7ccRmHZiqOveINCTTJtL07S12sMbmQ4FO5SPOIhAZAZiQvrT28rOI33D6VdSSK225jVvY1BdXCytuSFIyP7tMCJ4olhLCT5uynrUmnxl5wHlMSd3HUVteGDZC3uZbrazjsRWJLwzH1Y0CZY1PTWt7iJ2ufNt36tjmqZtyx/cj5KlBMp3OckkCp7i0ubHy9zAeZ0xzSuBnywNC4Dd6XbtFTfPLdBWRpP8AZXqa2J9PmgurRY7L946/6sDr9KYHPck04jaK29fsmtJ0MlrJb/wnemAaxX56UAR8bcnOf0pgJp3zYxTcUFAeaQDFWIIVcMWkVSOxoVRnOAcGgRAyMuMjFIAT0FWiRJIM9qjc/Nx0pDItmakEDFdxIAp0ZUZ3CrEMZkhOFBx0XpSAhtkxKrgBtpzgjINac130NvZRwnoSOciqreXbwsOknYVWNxIe9Ai7FLIwlGcMBkGlsit1aTJKf9Izlcjk1S89scHrSROyPuBoAaV2yFT1HWr6iVrSOOEbmByR3xVVleQ5UZzW3pTm3t2UjMxU4pMCqL640++VoG2owDMuOM/SrlzLa6gyS2+Y5DgSKowCPWpo5Re6NL56BpEbcc9qzpbeQXUS2isfMXcqioA1dcWxtfIW1tEjeSMM7qev4VhmIvH5qD5B3zVy7uGaAfalb7RD8vlnrj2qsxcRBgzIhbdsB5+lUA6JszmIj5GU8+tQafcSwagsauSrPsbPcf5NRy3DiUODgjpVbO6Vj6nNHKB6DrGoQyaeYt+Li1w0b9xXMardQahHFdRB1uAQHHTJrI6HqasW8oaQQyLuQ9R0o5QJllR4tyqRIP4WHBrSnhl+z219YP5COu2UoPun0rHmt/JlaQSM0fbBrb0F1fTL63lOYWTcq+hpWAqWc8NtNMVleU9Q7Hkmtb7VBqNhsmgllkP3XzgAexrnooGRVLDqePpWrDMIrK5h2sRIAFx26/41LQCWW2SYtCWV4huRScbsds11Vj4q+1T2yxwGC0uH2sXwSjeoNYttpMmo6JpkEWyOZjIQ2MlhUb2U8fhJisZbyJizkdAM0bAetWN5b6nH9nkZfOi4PPOexFN1PVzpa7IbG5vLjssS8fiTXk/nLNYW9zNvSHfsL5II/Kuk07Xb+31G1tn1B7i0uBtV3Odue2e9axYF6/uvEWrbba9tUS3DbvJiILA+57Vz9zZtbCWVr+SG5Z9sMEdwS2fQ+ldxf2V3pUAntJWaVmAYyDcMk4zivPvEUF7qOt3apbQy/ZQHeWziwc9gT+FO4NFWfxJf2Uslre3EV40WAAw3FPYN2plzrOqSaHDZpbRw6cW78kn1NY8lhMJl8+KSN5RldyHLH6dq1LvTr+BFlvWPzpiNRwPpQA/Sr+zh1mOWeSKCGNcbIySJT6E11MUUer30Fs8McWnW8ZnlSAnaWzjGe9c1oUGnvamxu7UFmJcu3GBU1r4gutPjltbeJHiBKKW5BWochHXzT6AcmYiVn+QbAQEHbA7VU1TxFfaZqNrczu901lwsPRWDAjdn3H8qxI7qPUIJfPsoY2RRteJjnPOKjnee8urOS6tvMWAAMFP3wPap57FNieIPFt5r7C3a0WyikOeByeO5rElkRPJjkXzrSH0OOfb1rS8RXWn6nfxpYW8lm38ZkbAP4VkWf9nxlhdW01xc7j8ucJVp3C5HPqKy3rXVvGIs/dA6ClF6bm1WG45YHIfvVue+zI7LaJCjnOwDis6WRXnLLGFA9O9AGiTBf7/P8yPavy7DnJ9/esubSrmBwFXzMjPy9qsx6hcJhECD5txO2r5vba5lAnQWl2i/u5E5RvrU3Ah0b7BZ63aTwSTsq5S4JH3SR1qTVLTSbS3mc6o13dl/miUY+pFZInuxNvjby2JJ+Wq/+rlkM43M/Ofc1aA1tTiS2021hiVAhxINxy2fasy5gmiSJpbcw7l4B6kUSzvJFFGzkpEMID2qOS4nnIE0zyAdNxzimBFjIqRH2lGA+XjDdjWr4d0SXWdSWNY3eKF1M6rwdufX3rtfFXg6yt9Htb/T7SXzjMIpYQpO9fUjs3FAHEaG7DVncbDKkZZA/APrz2p8t7FeXkk1zGi4z8qDAbFLcqmmiaaO1kj+0DbH5gycfxD+VayaJa3EVuzEWURQANMCXkPU4HWkwIZ7G0tNOsLggrLN8xUr8hGM9aNCu7O31KOOCJTLJu3Oo6ccY9s1n6nNYedFZ27keQAvm5JDH0welS6Tc/Zbk5uBE0ikGVV3Z9qViRFjbR75b+WJXKuxjDEja2eM/nT/AO1dS1ecTzyOzISA8Y27B74q9rVq0un29ugjWMZbz5D8zHism2Weybydw2t97HegCSYXF3L5kl1LujBIMxyfoKn0+S9+0STI3nIFzIXbGfc1JIbGFH2SyRtLhdzHIFQWV5FZecp3OkibRgdTWTYzV01GnuYE+1Ip35JkOAPoa57V7Say167gldHJfflDkYPP9alk2zQYYNBMv/PM4B+tRLAoGGkLHuzck00A4fuwgUqpBwwY85NaV2q2lwluB97/AI+GHRifT0xVExoGjcPDGwYYyfmNak04v7uO1tgJbidgpYjA3UDC8me0lCFVlljH3ugwemKpy3HnQqv2dMj9KnvYoltUuI5wWjYxywE5bcPSqMMxM3mKCowRtPQ0DHl1jcuXwzcrsbhR6Vr3T7Vs03bPlB2joD61mB027ZQehBGMHnuK1ryK5l0uK7+0rJFHH8itjdipaAv6fc6hPrVpZ2sVk6SSAlkiGRg8mtrXNbvfDHiOaGxXZ5z+a/G5HRgMHB6Hg1ztrM4k063lu0gbZv8APUcoff2q94iaa8naS6uLSWWX7rwcjAGOPSmpAXYNbWGDyGto4EddolQEIfTtXT2tzNBHbQTSIWYdUORj1rjpZ5tbs7PS3jWF1IDXWcKQB64GK0YL6507VGIngLW2Ii3l71YY/ixW0ZiaPSYMIcgYrQjk+Wsq0lu2tg12sAGAyvDnBB+tWI51ZwqmqEXy2abRGpdcilKEc1IxpPNNzTmHNMxTuAu4Uu8VCTTN1AE5cUgbPeoiaASBQInyKd5mKr7yKQyqTjvTsBZEoJxmhmGKqGQKetOEytwTRZgEjEqayLrzt7MsZJHrWqzL601ih75rSOhLOWuo70/vHjITOKbAjFgSMk9hXUMsbDBAI9DTUihixtiXitFMmxlwWMhw549qvpG0aYIqV5cdABVe4M0qYicJUvUY7fHv2k/NUySrjGahaBERNo5A5NRt8o4pWAsTr5sRUHrWPLHPE2wc47gVZjuWF0qO3yHrVyXYw+U00BgyKUXd/FVbaTzV27RssAOBVUB/Qj6it0ZsmjmwuCacZm7VAInPagZFUSOaR8/eNJvb1OaaTzzSZGOKBXZIs7g8mpllyaqgjNODgGjlC5oJtYYJxmlKBTwc1UWYYAwKnSXPU1FikPOVGaVZQQBjBp/GKYAByRQUSgjb1pMGkytKGAqbASJ05pSFHYUwNnpS545NKwDsikyKbuGcGlFAxNo7UmDTtwFI0gzQIaRSYpwZWFLgUwGjIFPDEUZHej6UhjshuGHFRtCRyvSlyaeoPegCEIc4IpGgGOBVnaKQinzCsUGjK03kVdaoSgz0qriIt5HenCX35pCnFQuNpoETmY+tOWYjvVbnGaTntTsO5ofaD60faPU1R3mjeaLBcvi4A6Upm3CqakmnHIpcqC5YEvvSGUhs1ACc0EkmlZBcsGRXppGKg3EGpVlBXBp2ARgMZH5VETUrdciomzQITdSg01UZugzT1Q0AKGOakVjjFM8pvSnqMHmgaHYJ5FTJkjnrUWCOlORiGqbDJSvFMKKpzjmp+CKjccVIyrK56UmSI8CnsoJ5o21SYikwPXHFKpINWjFxxUbx7BnFXcklgYqOSTU0jgrwaqrmn0mikxjvwarHOasOpzwKidSKEiWNEh4xTt7EcmmYxTu1UIdkE1IYSQNpxUSDBBqwrjcM0rgNeEoMmnwS7G61NOoePhhmqLIV/iBpDNJblScVJ5qnvWZCcP0/OrBbAzUtFJls7etJu9KppI2evFWARilygSg1Kj81WUnPWpRUtDuWA4pfMxUHOOtNJNTYq5Z833pRL71U3EUu+iwXLXmY70ok96gBJFLRYLlnf709ZCKp7iKeJTSsK5aMmaTNV95pQxNLlGThh60oYVCDTgaVgJd1JmmZozRYCTNMagHNBosAwNhq4y+VrHxIHBzuOenrXZY+cVzHi6Py5IbhT82cD26UDOmDBkVx0Io3+9Z2l3Hn6bGc9BVndVEj2lGaYXyKjbrQCCadgHVEXy2M1IThOOtVmODQIc3Wm08Uh4pDGGmEmnmmkUAyIk00iptvrS7PSlYNyvtNAU1Z2AUmwUrjIQKeOlOIpppAOoU8U3NLQBKGpd2RgjI9KjpwFICZHCjCgAe1PDmoQKeDRYBLia4QboRk96wr3Wo1xHI6yTHoR0H0rT1CCe5j2QYyeuTisGbRpY5RJMkEa+paoAyrye+VmaK5CL1wKoR3t1cOFnlLj3qTWrmFJ/Kgbdj7xHSsfzpf4aBpmreZddpEaj1BqpFcRWpGOfWqJeTq3XvTCNxyaQ7mvDdRSSjCov4VX1AebOgj5z1qpBxKK3P7Ido/MS6h+maYy5pDtbRZNvbLj+PvXb6e1oLfz9odwM815hNp8hiZ3uANvoetT2GpRaZFw8snuvSgCHxVJJc61MY1KqccnvWQtvKF6VoXl01/OZFB3HoKntX8tCskYLeh7VIGOue9Ka0LkxNnKBW9FqK1sLm8ZhDESo7nigCg3WmEjtVq7sZ7SbZKADjPBqvsx1oAQGpUlLcOeB2pECk4JxUioM0xswFt1HKHDdjV6JPLWLzOCxxmsx5gqErJiQdsVd+0+bbRyFCGAxz3qANeTUZ7HDRMpTsCOtYcuqXDXhYpEoY5+VeabNdGVNpNVcAODQhNG3ca4Z4sFdk2APMVsfpisee+QRGMglj1cHrSPgJuqsyhuapMBRcM0QQx/wDAj1oiVC2ZeFqHcqmpZDtUYqwNOwESxXPl9xk1myvulNMguWglEg5x27GkvLz7RKX8lF9sZqgJG+WLcOoIIrV1WeK5sbW4Eqeb3QHioXsGvZLWzS3MVw38RPFdRe6TpFrpCWtxbbrlVypUYOfrSEcz4aW8fXoZbQRPKhB2OcZHQ16H4wh8h7O7jGL7cPKXtmuIsrK2gQSCOZbgdHB5FbWnzG7vlm1OSeWGEAxknoR0qeYCz4ob7VpttDfu4uwQfIxgmvPr+3nt59oi2AnAyc13XjB4rl7a9VPNhwU647CuGkKf3yfrTTAXUNNudPiSSUqwb0HSqyRSOu8Rvs9cV08Fnbah4PkuppJDKuSCWJ5BrEtbq4trQxoFIJzz2p3GU9hUZNWbKPz5ADjb3qtulZj+7bPoBmrKXDLalFADeopAFysKSHyh9SOlQbd3QVp2cEL2Sh0yx75p8unNEm5EzQMyGT5cAc1OkqwQjHEvrTHyWz3qZYPt0ZKlQycbR1NAFFnZzljk0o+5n3pVt52LBYzwe5xT4rdhGzSMq89M9aAIRyakTcWAHT1qyoRYZFdACOQarpE8hwvHvQBdj3xlUKnnuBkVaImjnW5Rcqv3hnnFWdPm861aBxloxkc9a0EtoLwLCrgO/wB9Txx7VnzAZ8Mge6byEf7PMmCSvQ+tT6ZBHFI0l9I8flZ8tkODintE9mpt95CjoRXPaikwuztZ3BGc5xSuB2S6la3e2GGzWSE9ZZ15rl75m+2MpVFXtsORVy31BrfRUt5ZAc5yBVWPZMpdgcZ4piKZj3OOAcc81A6sbklUAH6VozQxm386DIY8EH2qnbh2HzdxzVIoaylCRIpGenvTonbzEEf388D1q/CuY90zAqM43DJqvaTKmofaCB8v8IGAaAL90ltMjxOojfbuUj9abp1yun2rxSRFi4wGB6VUmuWnmZzxk5x6VftFja2E8w3srfKV7mgQ2VoraD7NKga5LcMQRtqeaGePTHYESLGy73QZAzVyCZZta+3XFsnBCurncFHrWnouoWWna7cxRTotm3WJlyGPsKSEXvDrQ6F4T/tCdpHedisa/wBzJ6j0q1pNzFJoeu2ESibMYdN3Rs9fyrGk1KfU9MvTPdJ5Mcu5ImAGBnjBq/4M8TafZWS6feqkeGJWYjK89if607Eh4i0+Kx+HtrEse1iVJ56Hqa5rSoJ7jTIYxIkWJDtLHBU10/i/UbXVJYktbuCaGLP7vfxn6965yTFtOltcKpSIb/kHek5FI6688X31/FFpbxpC2Qk0qyZMgHp6U/wnBaXF9eG3YhUK7iz/AH8Hv7GuBW0uLp5ZIkfykJO3ox/Gphpl/pd3ZzXBaCGVh88b84/CpUgPQPE5jgvbi7ureSOFiqKynkgDGfpXHazrlhdyxbZJ1ZOM4yMeopfFV032uOFZZnQjH71srWHPMr2qwuOhyCOopsDUs7hrCWSJyJomGfOJ5XNQzWl5c2b3Vnp0i2kZwZ2GA1ZBeaeaKNTjJCZPTn1rq40vmP8AZOo6pJJbIq+VGv3SaOUDEgheNcy8Rt94etWPtc0myZJiPKbAAHOPQnvTFsLu41G4iSRJRbg7nzhAKzXmucMkcimMnAAqWB0NnPpX2S/udUiLzSP+7IGcVirZFZVuDLGA2GXJ6n0qqrTGQ28smzcQHQnB9quaTINPuPLnQyW4Y5Y87aLgirdPObk+bGF9VFQSqomBXhHGQK2FNousea4LwyE+YsnBUeoqhcRRfayqSCRQxww61SkMIokFn5zDo2CagnMflg4DsvXI4rSgtVjhZriZFj/iTqTVKe2fc8uUaJxhOfvDtQwGW+wqrtU4aN1aDyP3mcgbM5qN7ZoRboF/eFeUzVu3ultZCzbZVZCNuOc+1AFC4toT8yjAPVR0zUCW6Rv+97dVq6YZJ0ChTGzdN/GRWaykgZ+93q0wOk8PavfaZczNZXEcELkGXzBkED9fyrY13xLcPPbSefO+DuV1JWJh3GK57S9O/tSK8DM0Tw2vmxrGBmQjsfaqzvF9gVLhmDxDCEHHJp3As6pd22oPZJ9oEa72Z2c52Akda6S30n7AttqJke8u2nH2a4jkLIwPTKnt1rj9Nv4NOvluZ4fPUIV8v1/Suhj8XxLqFmFt4lsLSXfsiYgvnPb2NFwMnxMs9tq9zZ3EUYdXMvmKuNwPPFZxx9khlH3snFaWv3Vhqc0MsE0kkzSnzDJEUOGOf0rKmEH2zybY5ViAv1pCsOWZpbhWmldjnPzEmtSWETKskJG9zgEmsq4tZLWVfMwHHTBzSzTmeGBGAHldCvBqSrGnLbxfYwz7C4IDAtgg1ILOCHTGup2Z3D7DGnTNYskrOjhiWZiDnvxW3Jd2mowLDF50VzPh5Ax+QMBUNCsSzXT3FqskMEOwAKWK/OMdiaW5vItT+eeIRAAAFRgqfXPpWTLA8eXRiOcYzwasRXKgDcD8oJwBkH60rBYsNa29qp/tCzNyspzC8bkEAew+tJokzaZq1vcxPjy2DsHG4EemKk0m6kN+shuY/ljaOJp+FGR9KsweHZbm3vvJmhkkgcYmwduepGe1NDKF1ffaNUeZ0ieFmYgKu0DPoB2qllllbK9eQR0IrZ/4R+906L7fdfZ0SPHyRSBiynGTgfWqcsKy3ZiQmLkbRMdvBqwGW7E36FTnIABccCmuZFZowzFVY8KcfpUUnmQu9uxICNggdM1KhaOZViPRsh/ek0BYtcrB5mHKnKgkHr3Fbugtb3+pRWbw3ckhBZI7bG8v6jNaeg3CX+l3WmX1hHeRBmuVVmKCJ8YPzCvTPCXh3T7a1sruWxiXUIIgpkAIfBB6D8aiwHneoJ4hsZrXTEsDKt1bOkcOQS3Ocn/aGelYNveNo16vmqRcICssM8ZwQexBr6GntYZniaRNxicOhbqpFcv4o8L21/r2k6u0YPlSLFOoX7yk4B98VfLYDG8NXMc8sdil3OolBP2UxkbR/slh0rr4tAjhTAaQn+9xmrNnamBIBM5kkt8qrtgkr2yauSSnNaRbtZisQRxiNNoJPuacRmlBzTgM0AV3TAqAir7oCtVJFANAEDiowtWCmahyN2KYhcU0inZoJyaAIz0qrNuXkVcxUbAA5NWhGYstwxOCSBTxc4BUqQ3fNaDokaEqMVhXk+y5cZxWsfeJLDXe75TlT71Ud3HRmquxZR5ivu9RTWmZlzWvKiOYv21y4cBmJHvVw3BPI5FYJuCVxjHvSxXUkR4OV9DR7NC5+5tSyh4sg5PpUIYKwYsRjtVEXRPTinTsZY1K8YHNTyD5kXmvGZSo4z3qqbpt2GPFZgmcd6PNb1q+RE86NVdkhyakUFcZbNZcVwVPJq0s+4daOQalcuMcjPekV4lHzRr9agWbjBqVSGBHY1NhkVy6sCIzj6VQJw2K0Tb5+7ULWLbtwNUmQ0UmOTSYqaW2dOcZ+lRqOeaoQw02pyg7VGUNMBm4inpKQeaaV4puKdhXLgum9acLr+8aoinClYLmh52RlTR5pxjNUgxB4NPEnY0uUq5aWVvWpVfK5z0qnvUjGaXcRSsO5cDhiOaXzgrbc1T3sOhpu85zRYLl5nBpu4GoEkLHBOKnigeRgAeD3qWO43JFPDUxl2uVznBxTcmgRNuPY0CZk9PxpgcU9VD0FXHi7JONgpwuCWxgVGYeeKkSIA5NSBIHqVVDik8tcA1JHtWoKuQMmD0pmwZq6yoetCohPSi4WKPk0fZs9RWgIkPan+WoHAo5wsZbWvy4xVR4XRsYraaM56VG0SnqKtTE0YuxvSnKjE4xWn9mT0py2yA5A5o5yLFKGFjLip2gJcALnmpkj2SE+9XI8EZxS5yrGNMojkKelRZ9K3JreOYfMvNUHsAhwCaalcGiiaZkg1NIhU4NR7c1dyR6yAjGacMHrVc5Xkdacjtn5qdgJzuA+U8U3JpRIuMA5pQRipAerk1KFOM1Bj0qVJQBhqTGS8GmFDUqSRGpl8tveoKIEYg81Iy7hSvF6CmqGFIZA0TrnIpvSr+0N1FNa3U9BRzBYpZoI3jFTyW5AyBTApp3EQbNppfapWU1HghulVcmwqjPFMZOSMVZUZFLt5o5h2M9o8dqYVxWhJECOBVdkAHSq5kJogGfSkLEGpHLYAx7VAc55pkkwkJGDTgwHJAP1qAGnb6LAKWx0pfMJHOKYTkU00WAlDYNTLJVTNOD4NDQXLgk561OkueKoB+KcsmDSsVc0g/FHFVEmqRZPes3Gw7kxFNoDjFN3elOw7kytxTtwqsHweaGlz0pWAs7h60u4CqwY08NkcmlYCbevrTt4HQ1UZsGnqcigCx5gz1pd9VyaUGpsBZElLvzUBahW5osMshqdnIqAGpFNKw7it0rI8SRNLpLYGcEGtduRUV3CJrKVG7rSsBg+Gpt1s0BHzKOa2SMVy3h+48jUWQnG44rrHFUBWc80wEg098ZqP8AjqiSYHIqJ1weKevWnkAikBEDTGPNSFT6VG/HWnYGyNnxUPnHPNJIeTURPNXGKsTdlpWB5qZDmqimrEXapkhpkpFNxTj0pmazLGkU3FOIpMHtU2AbilFOwaUKaVgEzTgaTaaULRYQ4NSlqQLS7RSGIG5rnvE9lNfqqwjJBz/Kui2ik2c5xSaA89Xwjq78iAbfUtWZe2M1hcGGUYavVXE2P3YzWHdeHlv7v7RdqcexpWGjgBA7cnAHqTULJg8V3Oo+FLRIzJE/TqN1cbdJHbTNF0x2osMrOjLjkfhU1pv83Cnn09arlwx4pysR90kH2pjO0uTbLZJHKoV2UYAp0Xh6K4tQWXFYFvpk5iE8dwHYdm6VOdaujD5ZkKsPQ0AX5NAWPmJOfWkXT7RFPmyBX71gvrN6jkLJ/Oqd3qd3MpMkgIPYLUtEmqmm41BTvWRWOBiusNnbQW++GTawGflOa4rRxLeFw7tgY4HHrXT21grRcyvxTsNM5WdZ5r+VpSWJPU1VukVHwrbq27+3nO4+S4A6tjgVkNbNKeCM+9JoZUGByOtOEjnoPyFOkt5YOZFwOxqIOB0NKwHLuSTmnrM6Js521DTg2D0BpDLKhGj3jPFRv7CkaUrCdtW7CNbu3kUjEq9KEBQZ227R0qBmKjirUg8uRkbhgapv9400IcE3x7sgUbyV2+lI5j4CFs980i1oBLFtV8yIXX+6Kkj+ySXy/uyIz/CTjBqszEHFNzmgDQluZI7hAkjBlOEbPIFakVteTMJrm6mz65rm9zblKjcQc4z1rqrqdksoipxuWspFFyySUXCtK4MXvxRq1pG26TfKASBhWwPyrGtL2cTq2/8ADvXSrp2o3cX7y2Yoe54NZXsJo5bVLGWweJfOkaOQE8n/AD61lFXJyFynrXY3ekJO6p9pkd0GAr/NgVzs1leKxXyZgo6Dy62jMViKyef7PPAk/lofmI9aijhd2Cg5J6Up8naFIbeOpIq5pNvLdaisUJ59D0qrgQL5tvL5Wdu7hh60ycwK4WHnHWtoWzaVczNdKjowxgnIqrdXVuAXs4ljx2FAFOTP2ZGjX5lOetdHZXcN5MkbHIaPnjoaxIkluLN58jK5yMU/SY3nl82GQCWL5tp70gIZ4PIv5FC4UNwKlaFBIJY2IYelSXIivLx1aTZjvmobdnhuniSXzYh70JjKl3eyTy+VD+7fuF71HE7wxyQyxh9xyCe1SXESxXzSR/gahclnzTAthRJFtPeogfKPlkcVLG23bx061Nfxq8qvCuVcZzQBNobQNNdNMwUKvygmrWkTS3Ky3O4BonIUAdqwLeNmuQh/jBGPU1YhvJ9GE9uYv9Z/e7UpRA6SS9jntmlZQrocMB0rm31CZWI+U+5FST3klqV8tQySryGHBNUo13RjPWpjEB4PmyiSSePA/hUY/StSx+85BXYOBkgVSt4VktXJlRSDgc5qK4jeOL5xgE4FNoDXgiU3QjlZAuCR83Wor2wzv2hkq5d6Rbabo9leRXifb2xmAcgg9KhJbz3d3LFsZzUSQFayhTfslPyjqKiv0tYZyLYfJWjC8UkhQ9u9Ztwizb89RTTASO3J8p8YDjIrVt5Bax+XtZizfKo6ZNZjs7wxJ5hVVGCoqaNmaaJ2OfLIP5VVxnQXWl/2fB5d55kdw6Buf4hVrwbodzHr9vqV1bx/Y3DBXc5+Y9MVpalcQeL7C5uIUaCGxj3I7cFnx0PtxWBJFeWek2V3udIbsgNHu4zU2Ea2s6DZ6Ybm7vWCvNN+4RckKn0Fc9Fqlna6XfWYi3XEvAypXI9a9B8PobXSbue4s/PvhKAU27mYY4HNcXqunXU2oXuo6jbrbNs3RpngelUxNGLZX0mloyNbQzCXDEHtWtYanF/aUt3LDL5LL8rqMjNUdNitbm4uZJ7YSJ5WFBODn2Ndv4ajudVlljsjA8MCBWhmOBz2yKSiQ2ZFhdx6ikkk08UX8KxtwTUUMVlf+ZZ3U+2aNCYZnfaAfpWfr+kXfhvVneZAqu5McLMHCqfcfWta6a2gktimkxXJkUSZU4GKllnOXkzWsEdo19DdKepU7thz61mSO4fB7VpavMmqSPcR2q2vlkAxCs+NGund3faiDLNjOKpAPiKtGpU5cnHtWnpqNLqHl3K5KKWX2Iwax4biJZYVlH7oNuz64rVN4k+oZhOwFtoG3qO9ICae51DUNE8gTnYjY8pQAeO2e9Z1vZ3EEsQKZiHzEentWhdXGzxB5SFIvlGT/ePrWfc3bpcOqvn5jSQF/UHiCpNLEH2HKkLwR6H0qlPKjMGgOVYZog1Wa1YyxsGIHzKw4I9KnmMF75t3YyRLK4w9seG3eopWAyt8kk5DyEhjyR2qRhKjpJDgMh4IFJHDLc3Ki1iaU4BcD+En1ol/dYEjbiDjHT9aqwGpcTRyoG8vGAR19apWq2Uce66E8m042RtjAqsZJZGwnCIQTUxlL/IQoHYAUDLDu00/m2KysU7M24oKSwBMUl0z/vonzscYDVnw3b2lw7JKY1YYJFakNxbu20jcG++Om4+v1osBvTpFrcdnPI/k2zhkRMY2yYx+NYN/osttdPa3caQXOwvC55Lim27S2Gq26lpHiWYOIpOh9s1oeJpZdUuI795GP3sxnGI1zwAa0QGZoojs73/TLme2EiYjkj52k9c+3SpNa08qtokDJMsjELIh4cg1muMGN1JdM5dSecVPcXkMl1EYon+yRPuWB3/qKHIAjtQis08bK7ZVM8DjrVFxscqeo611y+JdPsNLuF06ynivJv42wy/hntXMTPLf3QkuZY42bCjIAAFK4DYriX7SJWbOF2jitC2toI7TdIo81yW3HnYPXFQ39lcxOJTEjQ4x5kJylTapmGe3+zykwmIMqnHy5oAqXEq3B3lst2PrVU8VIx5I43e1RZy2KYxykHOQDx3qzCTG28Z3EYyB0quBhiPQ1etoWlXC8HYeDSaJLLN59uzzRvKgTAKHGD6n1p9pbST6WcWzzM74DL1z7+1U4jd7JI7YNIWXdsUc7ajjumjCqsjxMDkMpwMj1qLDNGaw1GzcWkloUdjtjGNwY+gNbmlXWlrpdpFcM6TwFhc243BZhn+I5welc/Nc3SfZr2K7druPO9pGzhgccV3PhbWtLktZLe8sIop7o+ZIHTesnbKZ5GfSqsBzfiXUNP1FrWOxsorWWIFZDFj5h1HA/nUmnx6bq+m3l5qLv9tt4GKskeRx90tVnxHHeR3V1bafpAtorZvOWUrhjGRjv1Gcn2rjv3in5ZJEVlwyjIDfWgRPH5lzgYLStwAB1NbGs6fBBpdk6rsuEhWOeAcN5gY5LD6YrLtbtrZHAVSrFTu7gqciut1hYrW5ur27XDXtmsqeoZxyP1pMofpulS6rpGpavbXbRWlqU86HYQxBXqK3tI1zxCb3+zot8WqyKQtxeSnLIOQAOmcc1ieGNa+z6dqGls2LTUbclnflopFHyj6HGK6S9s1k8K6PrF7LJJLJdKfOjYgxwlT6dNpFTEDprLxPeWeqW2k63eRyTXDBFmjtivzH+HP97kV2TorptcZGQfxByK8RvLjUwlnqP9nPcQ2NwsgugxYuQ2QWUdM13eneM7e5szfSweTM43GAncHPqv8AdqwOvfrUT5rGsfFUF3qSWF3azWU0y77cycrMPY+vtW0cU0A1WOamVxiosc0nSmImaQAcmqkrZJx0pWY5pB1oART2NVHil84nGKvBc04Jk800BmEuhwaer561buIcgnbWdJDIilh0FUSWQc0pQMMYqrBLgjdV1WDAYpsCB4/3TCuYvrdlkZgwb6V1p5rm7yMR+aQMDcc1rS+IiRmKTilicICGAP1pmSDSEV0mI92Rj8oApVjQkb5QvsOpqCg1QkStHtP3gR6irJkRoQgPzYqgDinBj2pWC4jqVcg9qZmnNknJptMQoNOBPrTc0ZosJFhZ2Hepln9TVLdg804NU2KuaaXPvipBcH+8ayg5HSpUkJ70rF3NVZkbgjrTZLWOXnofUVRV29amW5KjBNTYLiSWzJ71E8RxkCryShxyaVolP3SCKLhYyTUbCtGa2OCQKqNGQOapMhorA06lZabirEOB5oJyaSigBwzmpA1RA04GkO5JupSwA5plW9Pto7u6WKQ4GDnikxodDpt/NCJo4f3bchs9qs2/m2wCzKQwrpra1RLaNAOFXAqte6a8nzovSuWVQ2UTnZFZnLAdaj74NbKxiBcTRlR6mqk1rGWLIMZqo1LicSoEQ96dtb+GkCGOQozA89RV6YIIVKAA47Vo5E2Ky76kLkdqVPtAUMIGZfpQ8uTlo2T/AIDgVncqw9ZMjoR9alU5NItpKIjIy4XGRUaOMZwRSuBaVc1IEAquH9iKuG1nCg7DgjrUlDQKcGA61HJDLEAScgmoWcikBdwmOtQOnoKg81vWpEnHemBEc56U5WGMHrUrGJvun5vSkWME5pisRF8GnLcIOtJJC204FVijDqKdwuXhOvrQWRqqLzxipAKQBJCjiqMkLKcYrSApGWM/e5qkwsY5jJ7VG8bDtWzIsQX5VGarFQetaJkNGaoOcVa8plABZT9KkMA6gUBAvagVgEZpDESalBpxIqWxkIjI61Yh+U00c1KoqZFFyPDDmneUpqspIqVZWFQUSiIClAxSpICcGpvLU1IFcxZFMECntzV0IAKXFFxlH7KR2qB7fDHArWxmmNGp7UXAyhHTvJJ7VeEC5qVYlHai4jNMBx0qBrY7s1tFU9KiaNfSnzBYxpLXjOKzpVxIRXSSRrjGKz5bVSTkVpGRDiZIBPSgirEkPlnFQsK0TIaGigjFITindaoBhpKeQaTBoEICaeDSiHjJYCniNOzZoAVM5FTDIqIMqninmQGpZQ4uaercVBvX1pRIOxFICUmkzTN+akA4obAd2zSZNJnnFPC5FIENGTUmCBUZ4PFPDnFSMXmpEyaaBkZqaEgHFIYm1vSmnINXvlxUDqpOaVxkKvzzUyyDFQsuBkUwPg80WC5eRxUykE49azhJVqKUcE1Nh3OHvUaw15Sowu7OK7Nf3sKSY4YZrmvGCBbuC4UYzWxo119p0hOeVGKYEkg61ATU8hyaiYVaJY1H55qyjAiqxWnxtiiwrlg1BKuc1IXAppOVoKKEwIzVc5q/IoNVXQA8CrizNjEODVqNs4warKvPNWIwo6UNDRMCSOaOlMaZVFRNPurPkKuWMj1pdyLzkVUDD1oDA9TVcguYuCRfWpFwaoggdKcJ2Wk4BzMulRTflB5IFVTdMaZuaQ0uQOZlwyRj+IUgkT1qqIjnpViO3OMnIo5EPUnRqeB7U2OM5watLEMCokikQhCegoeHK8itBIARwKmFqO4zWbZRx2oaTbz7ma6lVz/AOgqqnh3ToUE1y6P7vya63UdMgkQsgw46Vhrb/Z5cXbqEHrSuBw+sWmkv5klmSu04KgcCsO1t2nc7VzjrXX63Z/2ldNb6ZEqqOWbtk+v5VzUmk3mnT7ZGjB/3utAzenNomklY2WJgOVzya5JtzOzA8E1auBPsHmDK+1QBuMYpCGgDOTWhFZpIg2Sx7j2NZ+N7hfWtKz0jL5+1AfSkBatf+JcrHerOf4RU8PiFbfIdRk9s9P0rOuES1ueJfMwOeauXN/ps1qcWaLIem3nFAyK915rxDGGCoewFUYpraNg7Jucd81l85qzA0qtlIDKfQCi4yzd3S3i7UQhvQ1myQSRY3rjPSr0aXM90v7jyj3z0rXisbedgk80av2HWgDzFSCuabuzTVdfLzkUwzEjAAFCGTnlMUW87290siH7vb1qJWJXk0h9aQGhqYil23MQ+8Pm+tZhOTSiR1UqDwe1JTQBwRmk7UAc0rI2MgZqkwGGpFgcx7wRt+tQHPdWH1FPV2xjJxTAt6fp8uoF1hlRZF6K3VvpVu9uZg0NrNG0bxcHPeqOnz/ZdQhuM48s5q/qF7/a+qrKNqKMYpWA19IeLTEa8lKSMRhV7U3+3NTnut5mPsq8ACk/tGCNvKSzhOO/SpJNQsrdN8Kn7R7DpWDQx1jezrOf3hJ39+1WbnVtWs7hmR0eMdQ65ArMtW2WxnchXY5GTTdQuWlsSwelYRVkNnNcNLdTOWJyVReBViyv4YLgRWdo5Rvvsg+dhWaLa48rzWXKeuanglC267UP2gfddeorVCOstNOsdUvvMvUa3hjXkSttzWJqyWMeqlbeVZbX+IRjPFVWkvJomWa2mJP8AG7GtDSLYWlvJeTQK7KQI1J4zVASyXumwZS2spo2cAhvLIBqHTz5M7yyeWpfqQeAPpVnWtSvmjt5Z3ibp+6VelZMMCXJLMuSegpOQFfUbQRXRuInDxHuKpqzLKWHGa6VbWN7GW2AwEAIrmm3mYIOWpXGPlZw21+napLVAC0x+6gyajnYTyAycMoxgVpWkCf2PcAjrmkBnMSHYqcKTkCtK1eNtOEWcN2OOlVLSLzQUcYwM59aI0e3uW2xO6cZXNK4CtDJazJKVyUbcD2NVblZ9QuAzhSzfwgcmtKWaSSJgkRZRyVHUCqcUU5kS4tnwR90ueR+FO4FaV5E/czlgU6KR0pqMAMFCynsK2INNe/vidReQysm9Qq4B/GsfdtEiAEKpwAw5/GtEwJvNSKMYAL+h7UyaRrkBMYA7ZqDeGp8LENkGhgSNGYym4EtjjJq9DKdpLelVCckE84HFRkvn5ice1TJAXmEkVwoj5LDNSRwxTrKdzCZOw71HAd4Vv7tP2hG3/wAXrWdgJNHVZdVEbjK7TULlo7qVFPCnGa0NNiia6zFNskAz93is+X5bqXHzbnwp7E00xmt/aMB0YW4hCywnIlDcnPYgU++1RhNaQanIRaJEHijjORn+lZ8WlL9qSOadUeTGRn7n1rW1fS/IRBcSxzJDHjeEwQPzqiCxomZfEkErz3kNtIwR5FchwDwAfaun8e6FYWulrFbpJLeP8/myuWYKP6e1Yeg86S01wSyyfMWJ64qPUvE+qalZfYSwiLjaGHLsnoTSegznlitpbTy4pdjNyecZNN8PpKNTaNJ7iFxkrJE/II//AF1VZXglXP8AD932qXTp5I5ppUbbhuTioA623ub+4md9Rma542ATfMCPb0rA1Cdy5/0mQOjmNEHAVc9BVvTLktI1zne3mDk9CPpXR22laNdX/wBjl1C1eaZPNXeQQrego3KPOJpmjWSEdGOST1qGBNzhCzBWIU4PUV2R8DrcXV3bR3KSyoflkiPyKPSsvRfD2q3N3I1nbZa2yHM3Az6fWrSsDRk3EDW8pjI+7SvI8kYkZ/LwMKV7+tWL2QzSmGe38i7QlZuetZ0w2Nt7UMVi3YSRGZZL53dR04zXQyadYajZRTWoxLAQZB1cj6VywICYFbFssjWzXEUvlEKNrdyfSkhDPEUUUOpRSQoghmQHap7+47Vm3NnNZvFPLausRb5GbgP9K7rwbYWl7qVnqdzdW4liZklgm4LcYBHrWx480W6ur+xu3MbWkKEFShKqOuT61rygeX2d1LDdSvDEw80YcA5wvc1HdtCbpiHLR5yDjv6VZvEdNSZrSDKhyRsHyOPf0/8ArVVui8spnEPlo5yF3Z/WkwJ1uvnkTyNpcYLN3qRbC7aRdlvIX5ZMDggdaghmZJhcCNXEfUHpiugMDalDp32Im3lY+SzAllQnpn61IHMzXPmOCkSJxgjqSfWnwMygyrGx29TjpXSavos9ikNneWcaXkeWikhxtkQdSeazm2zzgQzFIWQlvNG0A+i+1MYkwubuyjusf6p8qc8qR7Vo6nJbSfZHRg8ci75EB+64rPtWjbSp0D/PuyvHOPWrNp9lns0jltcqh2A525qQIJLAus86IEXy9wwOoqpDEttkzoJbd/lJVsEfSuj1VEtojFbp8jR7ck4Cg+prBMwtYYrc2tvKq/MXzkn8aAEu7OSx3iJ98IYYJGQCe1QPBLGFNwmCwyPcVdu9Rk1C2htS4Qp2C/ePrUI1RvsYt3h3SqSN/XA+lMBLOc21rcwPE7RSjPy9A1Qxme9v0QyOqhckqcHApJpdybUZh9KW1kaMpddITmF2/uk96YFKSRGuG2ZHPfr+NN6YbvnNWL+4a5uxny9sQ8tSiBdwHc1AMMxDHHBNMA3ck+tW4ZTs3RyFJV5UmqAOTUigk4pgXoLyaB5DE23zF2kjqPoe1aOi3S6fPO8ixlJY9rF03Fe+QPX2rJZVRUVshuuCKl8wlfmqWgJGiW7vb2WO4ZogxdGUYDMfbtWjb6rdwPYz2ECfa7N87dpY57ZHvzVe1t2it5HZWRGxyD1HPFRJIsd2LiFy0vmYwy4GPQ0AdPe+NNf1mKSyEdrOWhKt5ERUgdxyT0rnb0xnT4bfzo5Jk+Y4P3Qf4frmqBMwuT5SuZdxOFNX7/TblHgNxc2xdyqKi5BAPfkDPuaQywltPDasttbQyOoMdyJCMK7dACfxpbq7mm0yF7tnnOxYt/TYi9F4qmJ59Lurq02pIGwkyZ5IHYf41saLYxX1pcrEkVxFH0jdisg7nb6/ShkFeOJf7PSQFVcoy/IeCox19a7m01iTT9HttMtkkhe6jVSroGCkkAkE9BXE6hZS2eoCK0XERfy18zCljjJ/D3plvfy2t5Dcm8J8pgeeeP7v0qL2Kuelpd3p1qPQ7K83yzA+dI8fzqikFQQPx5q5DoUn/CexQo4W3Cid4sdGI5A/LNYWj6rBZ+OhftLIqS7FSQnMYRsD5sdB6fhXcWkouPGmpTL/AMsIVQn0z/8AqpwZJpa5osOuWXkNIYbiM+ZbzqOY3HQ/Sq+ianNewS2+oARana/JPH03AdHHsa0/M5rG16ynklg1bTlzqNoMeWOlxH3Q+vtVlJm6KTvVTT9Qg1SwivLZso+QVPVGHVT7ip1PNMAkGKiD4NTtyKrunPFAXJlkHrUyOD3rPGVNSxuRTsBeOGFQSquw8Uzzsd6Y8oZcUXAzZxtkOOlLDPggZpZvv0wR9xVkl8HNY15LHFcurDd6gVoiTb1qndQRyuZMfMetVT+ITMC5Ub969D2qNelWZEIGSKrMGDYIrsic8hhGDSdakyMc0mznimIZtpACDUwQnHpSMhWi4DQueoqIqVODVhetPCqWORwaLhYp1PBbtK2WBC4z060ohj6Mcj6VqR3qLGqjpjGMdqUmCRFEka4QoD9RT7zTgyrLbx7c/eGamIgaUSKNvtiry3ESwbcE1BpYwfsoCFWPznoKrYI4roZI4pYvMVcMKzLm36si00xWKAJzTw571dSyRgCCfenNpYkYlZtvou2jmRNmVFk9DU6XDIapFZIndSpBU809WJ60xmiJgw5NBSNxVIH0NSJIV6miwDpbQHlATVNkKtgjmtAT46GkKpJ8xGTQgsZ2KTFWGix0pjJgdKq5NiKlp200u002IBVvTxIl4kqYyh5BPUVWUYq3bHy2DgZ9qiWxaO2tmLRgtxVon5cVjWU4Krk1qxvmuGSOhFW8thcRFGGRVWOwhVwzoGYDAJrXIGKj2L6UrjOduNNRZHcKRzmm2ltJPMkajKg5NdBJGp6iorO3W1kYozEE5APaq5wLIiI++oz9KUwowwVH5U7cTSs3FZ3Y7AsKYwQD+FVptLildW5+U5x61YR+al8wUXYWK7WcJXHlL+VR+WVJBPHarbSA9KiPNF2FitJb+apX9ayptMvFkO3aU7EGt4CjaDVKVibHKuk0TEOhFRb5CcKrE+gFdZLaRTIUZcg0kGn29uuETFUp3FyHO+S6jJbLemKsW1rPNnkLj1rofs8eOVBoEMScqoBocx8pnmwZ/wDVnb/vVXuLKWJNzLuXuVHStrijaKXOHKc2kLSNhUOfcVOunSNneSv4VssADwBScU+cOUxJrFkTIzVFsqcV1IVD1Ga5zUIXju3AQlScgDtVRkFiqWzTaHBHJGKYWOK2Rkx+6kNQ+ZzmpFdSeaYhGD4yBxSKx71OrqeO1OEaHpSegEauRUsbknmjyfQU5IiGzUt3LLQI2c0wsKlVAVwRTxGoHSoYEKE5xVpXOMZqrKCH+UUgdvWpGXg1O4NU0lPQmpQ57GgZPikLdqYGPejOTUjH5p6gn2pgxmpFYUgF2fWmlKsqm4cU7yT6UrjKDwtjOKFtQwyc1oiL1FGwCi4jCvdMkwWRc+1Zo0u7cnEWP944rr9uaaUGKuNSRPKckNEu3cDCL77qS40mS16uH+grqdnPFQtbbz8yg1XtpE8pyjFe46UzI7Vry6ROC7EoR1wKonT5hypBrZTRLgyozHpQqSf3TWxDZIoBcZapTbJnKij2iQcjMMxSO4UI+T04rorPQrRYw07u79xnFJbw5mG7tWko2nFZSqovkYiWGnxAhbdOfXmmnSdPkXHkBfcGpDGWORTjlQAKjmYzNk8O24/1csg/HNVZdJnhTK4f6GtsSZNSqqMOaOZgcp9jud2TFgfWpkglA5X9a6CURg1Uk2/w1XtGKxkvbSE5xUZhdRyKvyvjNVGct1NWtRFYlgeppwcjuaZIRupgNXYm5aEzcDdUnm571TyacHHelYLlncSKgZuaeJFx1pvlNK3y9KVhiB6kVyKfHpzHlnAqU2GOkmfxougsZWvQfaNLdsZKEGqXhi4wjw+ma3bu0aSyljHcZ/KuS0abyNR25xzipY0dRLkNTGbgYqxcLk5FVW4qkJjC5phlI70jHmom5rRIgsLLu708OapAkGniU0coXLROahamGRvWm+ZRYLimm7jSl6bmqEISTRRkZpuaaQCk05RmmZ5qaNlGKLAPCMegpwhY9qsRsuM1MGFZtjsV1tQRk1NHEqdBUmaKi5Vhdi5zjmn54pmTSioKJBUqnioRUydKljJ4ZGDiryTg9TWaCBTt5qGho0WeNvvVkahpFleyrJLPICBjCmpt5pMqeopWGc9qbyQr9ms7Pcg6vjmodR0S0j0wSXSZuCMk+ldSoUnpUptYblNkq5FAzxq7nso22oJMe/NZaBXuASMoxxivUvFHhjS5rItGjC5U5GD1rzq60yOwI86Rw3pQBqw6dpXmqYbcu/YA81JqqG2sy0dnLG/Zmqlp+vQac2/7OX+pqrq+vyaoQELogOduaQGWbiRARgZPJyM1H5znv+lSra3M6l44i6gdajMMmzHlMCOvGaLgRkiux8LWtvFB50t8iORgjpn9a4sKTTo/LWQA8Uhmv4hi23zNHIZF9Q4qLTJnjcAKD7kZNKtzYqgBRvqKmtlN5MEsoyfrxSA8uLc04c03FKDirGSsSgyaQyGmvKWjCntTM0mgLGwSKNg+bvSrEw+8MUyFwgOatD5kGaLANjiBHA5pJA8fLIdvrU0UJl+RX2n1q4qIlsIpZFfHpSuBmtcyOFCDCqMZquyljnA/CtMywj5FgUj1NPihjlkCbQCfSi4GPip4FKEOByOlaCWyw3giniU56ELTLqBrabYy4zyPcUXAZDJK10HaDzV9KszP5z7vI8r2qvDcNC4K8+1XdrTDzBEUz2NQxlSSQKNh70x0aKIGVSob7ppLgZcZIAHWtG8mL6dGMLsXp70JAZ0SSc7mJj7Amp40cMGi6jpVYSEirtjJsk+6CT0J7UwNTWr6QabEof53A38d6yTqVz9mFq05ER6gd62Bd2D24S7hZj3HSqcD6bbXXmQ2yOD93d2pCM9xI7Ftxb6mremM0dwxk4+XitGTWkjaQW1oiM4wS3PPrTJ/sj2Pmv8A8fZ/g7VLAt6dL5aSMQjl859qvalDpsFtHHNbIN4JVlXGD9RXOR3DRx4WtC11lpI/IusFB0YjPFSBiNaBJCE5q7L/AMgooPvEhR9avyTWoby8KkZ6OBzmo5LSC3tSbicFA27gfpVJgVYtHvkiRVRImU53M/WtGODym3PIDJ3CNxVE+RcnZE7qPVjireLWNhHCvTq2eamTAbcxwxIrhOWbDHPUGpjpVteRRvYv5UuMbc8GqlzG8uNpJA7E9KqyzX9tEBA2FzjFTcDTTVrnTbkSzp56ouxlxjB+tc5OP7QvZZB+7L/MFatO3vZpYXgu5WkLHOWANJbQvPOzR26NajKyHoRW8GBz00UkD7ZFwe3vTVJDCtu7W2kuGtYAZIM/Kx/hb61kMhjcq6kEHHNWBOVO0leSBmlKqLdJAxLHqPSpLWETzoGHy96VpCGlQABAegpAOtpVihkLfxDArZWwaOzivJo3a0b+NDg1z5+4T/D2rbmiaPQVRJpOcMELcVLQFlLFVR5oB5qOMA9Cf8apG0V7y3VWCBfmwT+lJb6lJHpYDH5kk+U966GLSoL6JLxoyhChlw2cioGQ+GLiI+LJ7O+hR1uU2DdztPXirc9gktpcWtzeCC0W4J5bcSB05/xrmpAIrwXWWEqNkEelVLxZJ2a4bkMdz/41opiaLTak4Y2On3LyxebhWbsM+lSahcub1clBKuFLxcFgax7VPvThgB2ycZq2YhNdCZZAzyOCQO1MVi9daWxv44oX3OyhsMeualstGkkvnsSGtw8e4kYIB9TUqTefNeXanEtsMR+q4qbR9Tl0y7maZPMhuYypYdUqGgE+yLHokyW8SCWBx5jIeW9xWbJo8sNq1zcSeRKzfu4zjJzXQ3evWf8AZE0NrbOQoCgv1ODXL3d/PqJ+1TEGUHKKowFx6UrCidTba3aeEdDfTfInbUJQzEg/KSe4P9KxJvGOuXFukEUwt4toVxGvL++TWTd31zeL511IzzEYJPoOlQwz+VLFJjOw5wehrVFCTJIr73lDZ6tuyaakfmQyMzcoQKv6lJ5l4lw9uke9lbYjZGKbqNxp81xI1jC4SUAsB0Uim0MzmJ2cda0ZL7zdOt7aNSEQZbPc1nMV6jmnFnU4BwuM4qWhE1vMtvdLMQG2nODXW23ifUIbB72aYzwk4+ztJ8qg/r61xQUs2MgfWr0NwY4Vili3Rc+YE6uKpMhnU6hqMNxcwwS3kVmnkkh4fucjvWcdN0S00+4tW1KK5u2GY5hnZH9TWdp2nx3sN1Db20hYjfC+cBVHXNQalbSWBjhKKElXdxzn8e9CGRtPFHCUgAJcFXY8gj2rp9Gu719GmurG0AW1TE1yOMnjt3xXHHhSfTtXqenX9s/hK+0i2sTaiK0MpzJuyW59O9JsZV8RXMR8H2F3e2kd7eyIUE0xOVyckgCuC+17kRXRW2jAzWpf6rEltb2+RNnEjBxwvHQenX9KxrpT9p3BNsbAEc0rjLceozW52whFB6jb1FW7OZJLmM7XiReODwD61kS8XI9AorWtUWC3E43bJuNx/hI7ilMCzfX7yQSQlsXULAsH4GP61kT3bzsm8qX5zhcVOVWe+cXDeY3Z15P50Jp7m/ijjAznPJ61KAqOyKuekg70ssymJMbHLjLN1OaluLJzeOvllRjdk9CPUVScqGIU7l9cVSAco2ozEggcmohLI0Rt8/ui+8r71J5MRR2y6nPAVc4HqaiIliypVlyM4I61QAeKTrWnb6Jc3lna3FsRL57FSo4KH3qreWdzY30tlcQss0TFXC8gH60AVR1qSNl3rubaCcE+lbHhbSoNZ1tbK43BXjbaRx8/YZ7Uy80xtA1uWyu0+eLG484ORnimBYexW102LUh/pVvMGT50IKtnA61nqrtH52390jYLZ710OjztqqJoM9zBb6SsgluJXbBx6DPcmk1bR7bTh5tldJJbTTssUeeQOOrdCeagOYxvJiW1mHnSxzFhtUHhgc5zVV5cxBas3yXEc5Qjntggg1WmKSH5I/LGACN2eaYGharcWdtFcQK0twQT5ZTqR3962oL7UrnS4W1KxRIFcTQXTjYQRyeO/Ss2xnimjlmvf3awxpFuTqeetS65q0mpWlrY2l3vtIUACEYLHPf6UEkeo3tndXP2mBmmuZJt7Ns2hV9PpioJLqaPMsDeTKHBEkfAUjuDWbbss0oQsQ56Y7mryS4tTAVypOSc96ANnybzWNKudWuNQV57ZBIqBQSwzjkjGP8A61Z/lwNZQvcSbWlQleMpkmrWmeXFBLZ3HEOoR4D91KnI+van2+h3l3by2qPGfKbcpDcZ9/QVDY0V9OaRLkW0svlRkjdzyoz1x3r13wS0kl1q7zSpLKdis6HINWPDei6fJE+pNaxGWdU3xsgPlSKMNj8aPC0KQazr6xIETz+FXoOTVRiB1FKDtPFRb6N1UIxrkN4f1STU4U/4l12wF4qj/VOeBIPr3+groE5XcOQehHQioG2SRPDLGskTjDI3Qj/Jrm7XX7PwvfvoWqXRS3XDWkzjJCN0Vj7H9KBnWGkK1HHNHNGssMscsT8q8bZBH1p+8UAMkXAqqXwetW3IIxVOVBVIBfN460xnyc5qMhgcUHIqrCEc7hT4Mltp6U0c1JGNpzTYCXMTBdyjjvVJpccNWwWV4qyLwBXYgcU4PUTKzeXuNMeCOQZHFMLE0qybeDXSjKxCbNR1zVXoxABwPWtffG6461QmQiShEtEa471IqqxApm30pN5jfnpVCEaHDHFOVflqQOCM0hTJpAVHUg4FIjEN1qcqcnNNEQLZxVAWPMbZhetEd0Rw5yKSMAOQfSmGPax4qRmhDdqRt7U150EpA6Y61mZdTwaXLbST1pWC5oQSfv8AGcg81bU4dfrWLHKQ2atfaWH3WOKlxKUjbaNZonjb7rDFcxcwtbSmNuorXhuJWxzUF1byXTECP5wQQc0JjZl5pwkNWF02ct85VF9c5p50m6wGQBwfStLoysyBXqZWqNrK7jGXhZR703Dp94EfWgZZAzSFM8UxHPSp0dCeeKQyuYj6U0ow7VohFPQ1G1uewo5gsVUCNhWH41KqmNvVexo8ghwxHGeavyW6hQIyCDyKTY0h9pNg8Gtqzn+UZPNc/DG6N0rWtwwTmuee5rE1hLnqalXmsgTN5mwdauQzSdCOPWsmWWZEyc1GFwaV5uMVGJKkCfOKjZiTTA57ninKQTSGPU07NMFOyKQxSaWmFgKA4FADzQDTc7uaSgRMGxSh8modxpu/FMCcyEGmtIO1RF8imF8UASebil86qrsRzURmx3pgW2lqMzVX80HvUgCsM0wLEcmSKmdVbBIB471UUBTTzJt70gCS3jb+BfyrPuLKEyEkHJ9OKsvMx70zdnqaq5NjLOk7myrlR9KYdLdDxlh61rA1ItX7Ri5Ec7JDJGxUL0qSxt3ln/eEqg65FbL28bMXI+am4C9Kr2jJ5EJJbjZ8g5rPYyRuFcFTWosnrTZUEoAqFJl8qKqynFWY2Vhk0wW4XjFP8sKMgU2xJEgEZpkkCYyo5pmcGpl5FRcdiqY8VKg45qYoPSmkDsKdwsNIpRSYpC2DSuA7vTlODUWeacDSAvRS4AxUizgvyazw5HSl3E9aBmnJOuzg81W+0H1qruozQBcWfNK06gZNUs4pc5FAExuFzUMs+W4NRmoyKYhxmk/vGmhzyD3pNtJii4D1ANSBFNRpxU24FuBRcY9I8c1IMiowx71IHxSAXLY4pV3N1pocZpS56DikIXYB0oLMo4pmWHemSyELyaYiJ3ANRFsmms+TSpjdQMpXzFc1nNKRXUF0aLy2RT9aWO3tUO9YEDDvitYVLEtHJ+bnsfyp6vW/qEME0ZJQBh3ArnlQ5wPWt1JMzasSk0dqetoSM7jUckUkZx2oETWzqJQGAI961VMQ6KBWLF98VeV2Hes5GiLZk544FJ5vqarGTjrUZkPrWQ7mgjK24eoxXn+oQ/Y9cZenzg12KzMvSuV8Sri8E3dl5oTEdWP3sCP6ioHSo9El8/SY/myV4qxIOa0TE0U3Q81GE9qt4yeRTwijtWnMS0UTD9aYUq+4XGMVGIlxVcwrFFuBTKtSQHBquykHBp3IaEzRmmkUoqgG5pc0uKMUCDNAPNGKMUxk6yEVOk/qKpj36VqWFgbnBcfu+5rOSKTESdScVJvB6Vcn022jTEa4I71nbGRsZ6VnuWlYnzmlFNU8U5TzUsaJUGTUoU5ojQ1MEqGVYZtNKFNShaXFQFhm3ilCn0p4WnBaQxQgxUijFMBxTw3FDAbMpKHy1Bb1NeeeJ9DnuZgDHsGcs+c16MGom2vburKpyMcikB4vd6XpVpaktO7z/WsDFej3PhvR/tsjXcoznJVTgVyOuQWFtdiPT2Zk5yT+FAyfw00lzc/ZDIqxEc561t6rZW9lB56lmlY4AY5Bri7W6a0m8xHZT7VtXWvQ3NogZ3aVTn5qQFN7fzUaQrj224qmPsyDE0IZvUmpJtaupl25Cj2FUJZGkfcaBkrW4kYmMYQ9BW14ajls9RSU8KtOtLS2ltleFJH9GxxUiXJtCUZd+PfFAHjuD2BNJmppQqEqDn3prmIoAnL96oYygDmlAp2KYApwasqxCjcRVXmjFD1AuJKQcqamVw4IGPwqopCx7QKfCxVWI71LQD3YLnb96prR3RkmZvlzyMdKrA5NEe/DKQSpNKxJuancLDEJIQCzY69qoS6pHdQos5H2hOjAY4qpc3s5h+z7/kyDiqgOT/8AWosMswyKkpdjV03huMBnbavQCs9F3DoR9RVyOMpBuIpNDJBALxX2clRnFIzY0tBnvir2jxsttK7DAOazHRnJhH3QaQxYFVyERS7e1axtpLOz8xrZI3H3j1oshBa2bC1AMrfeZhyTUohZ9Il2ks8hzyaYGbNE32Z3Lhs+nam6dcpEpR49+ehxnFSra3Ag2uflPUVLYRLFOP3LyDuFpAO1GGO2dR56NuGfSqT3HmsOc4GKfBpg1PV2t4kaFM/MzD7g9TTL+0Sw1D7Pbu9wg/j24o5QJbRZLpmjjGWovYzaPtb0BquPPtbkY3xOan1CV7q3RJ1HnKc7xx+lSwEy09kGXkZrTe9MuixCZx5g7E8ms2zdEt0R+VBzinXsiSsREuE7CkhAjGdv3RCJ71KZHhXIOayjLJAphClPqKs6ZMDOYp34kGBx3o5QNG31RrSQSOm9P4hjtVp7m0mUtBIkobqrDBQf1rHvV2Ryx+gpmlsC+0/3TU8oFq6RRH5sMqDac5z1qeK78qze3j2EN82V6j61SjtZ1RkmZFjJz1ziktozB9oyOB0rVAIvzRSxj7x5qIEXdqUkcC6zgFuhq7bWqtbGcyhWcEKpHU1QdIt5VWTj+6KpMAglNjIySHbNjBpGDPdlQOozV6406V7aLUvIL4Hz45JHrVRSVP2kqyoeAcYGfagCJ0kDFW+6Olb4WCTSX+2PGJVj/dKGyc1lW6PqCFY8Bs45Na2h2US69DLdCF7a3GJecDJ//VSaAzkXdCq+iir+nTJEzGRSQozkVe1z+xoZ/MsXdzI3zKAcA+1QW0VlslXJ89kyATWciR32iGd1MqiOFzgDvVXUtPmhulkVR9kY4HOQPrVa6uF3rCv3YzmrNvcNtlWK432jx9JOcfSkijJ1CIQTFA6v7r0qfTtsVzCGOFdT83cGqsrqYDbLtYh87sfyqaCJ9yLJKiqDhcnvVtgWnSaznmaORHSVdr7T0FP06USXYjmOVIJFbC2ESWEsl4kbKkXybG71ze9pY0GVXZnYQOQfrUgXjAwsrmdUxb79oYngH0qhaOPLVRglDnPY1paVf/2XOYJ3MqTH50foTU99oSwTy3lvIEt5RmNFG7J9B6VSEYV7dNdXDBgAw6Y71VGdxU9V6iulj0yS00mIpAXupQwICEsR7isySCfS9v2y1eKaZMsD1IrRCIlRpo7RGbA37Nx6KPerl1uWXyrZQcHPmLwKpSTxuuIojGvoTnmrFtJAbVkmkMbxqMEUmWVpI5JLlmZfmxk1DJIZZEBx6VpCW1vbhkRJIwyY2qcnj3rOuYntFSXOHJPHpQBNNbus4hV1dsA5HAA96kS2e1kT7QoKOu5cHqKzvtFw4cq3LjDH1FW/tDXNrZx7gDEjLk8d6aJaLdprM2i3Lf2eYyk4y6umVQ/n71otq638lvbXECfZhy6p8x+uccCuenVVlZVbcoGQcYzUqSLIrSrKUyu3b3qkKxd1LSEe3m1OzkhFv5wiCbycdecnirmiatqVtK0sgha2mHlTx4GSgFUrjTZI9CjvLWeaePdmSL+FT6mtLUvCgTSINUtrnzJZ1DLaqctyM9c0mhmXews7z/ZED2aNw56qKzhK4i2biyZ71YgkjktzCXwSSMnoKgRVMRBdgR0A70WAWPdM+AKtLI8kS27S7kHQKecVJatG0So4SEjOGPf60ySH50aB4gVGMjvUyQxsM1xatIkB2lznd3FWS013BCzOZbheCQvzg1WVC0gVpk3nsKspdy2zGFWcqoHK9fxqbgWkk8wIjuwwMArwao3NnLanMaMyfxllzzVy1e3ZS9xGdjDAY8Yqae6EVvGLaZnVTyzD7w9Ki5RkSfvbclHUYGSO5pZXa4sTc3LGR14VlPCH0YVbvStzeQG2gwznJVOmavjSJ3sri1jiNy00weXyuREPQ1cZEM0Ph++pWOtrKthO+nhf3pCZ2Z/i9/pXZjw5o+ma4t7qGrLbxmR5I4E6tux1PPFbHhSTS4QtjBdEXkSbHilIV2AHGfXiuW1e+Om+Mru4u7mM2BuFWSNow5+7xwegrSwxv/CNwx6pqdnpOJrKVkeG480M6uOSQe4yefrVLxZp8+v+JLezSWF5rSzVL6cLtRSCevPp/Kuf8Q6wNXbyo8rDHMzxEfKQp6DH4frWl4T1eSwtbrTYJ4YLu6mWSOaYZXHQgmlck7HQvCvh+Kxl+zQRXzrh98h3PjHUen/1qXxCdNm0g2lzC8sUgDGWPBEXI5Poea57UX1rwhe3F09xBdNewvDHMhAAAwcgD0z+tYy6itz4ft9I02G6llkJ891UsSCOh/LinEDI1iyl0bWjCZY5JFxIrBuHB56VlpIGlO/5VYknHQVr6vcyXV9BeXu3aF8ry8YcBR3FZsP2U/aGm3L+7PlKD1btzQ0NCqY/LZN3Ymuq1PwhG+jWniDTJzd2DruuTGAGiYdf1rkbdhuYEZ3KVrpPDNzfWTxafb3b21penypyyBg+c9F9+g+tRYGZt40q6hA4sSnlwAhQMF0PR8e9RW7QR2lq7EMzs7MueRg8A12niy1ubxo9btbSa3so1jtoGliK7lGRvGf4eg9az7IE6XqUV3bQz/ZogyBkxgnP86AM6w0q41iFp7a0FxMuW8uFzlceo7A/0qfTrk2OoMrtmKdmWWJuCh6NkfnR4R1K50HUo7mEMI5SEljP8Y5x+VdzLoFh4mtzeC2mt7u4Pm+Z12ngEdfapSAl8C6oFaWIzSyuDsjR+jpng59RXQ+HhjVtcb1uMfzrn9D0X+w7q5KJtyiqjMckHHJHp2q9od19jsm7tKclu5IJraMHa4HVE4NKGrLttQWeUx5O7GRU7XAVsZxTcbEpmgrZrg/HGh3jalFqtpbvcJJEIpVAyYyOnHpyfyrskmz0qdHpWHc4XwFdXcOoPY+TOtvICwQj5VIzz7V35NQxRxQKUhiSNScnaMZNS5zSsMaWNRsc04im4oAYaY3NPIpm4A4PWrQhM4NTrggGqpcU5ZiBxTAtdqrS8nnrVgEEVXnwpzSW4jOnTaScVV3jPIzWm6rIMVTltMfMorpTM2iruIbjpSltw560jKU6ioWkOeOBVohj2JHSmtJvHzDmm+Z60vyNVkDMkdDTlmNIV96aRSHcmDqwznBozg1ByKeFYR7iOKBllZB+NPAV/wAapZNSK5BpATSQgISqk4qFoyqBj0qypPdx9KnazuPs4kkhYKTwD1qblWMnGKmhyTjFPkt+Mr0qSBki5ddxHamIsQearA7Onc1NLNMMkKDVI3E8j/KoApsnng4JqLDuTx3DNKN+BW1azKsQJrBtrO4uXIOFHrmtKHT0hxl3JHvUyHENR1GOORQhzkZNZF1eLOACelbmyD+JAfrUqRW+MiFPyojKwNHLhmHOCBT/ADkHU10rWkUrgyICAeB2qZLG1JJMKYPtVe0QcpzCzqvRqv2qXVxjy4iVPc8V0CWFrE4aOFAfpU7Nt6YH4VlKpcuxgyWc0Shpo9oPGc0ioQMDpWrdgSRYrOBKsQRQncGiaAAkZFXEWoINmRV5Au0VlLcpEQQZzjmplIApCM9KRgQKgsR3BNIDTcUoFADtxBoDmkKmm0ATiQ0m8k1EWpPMApAWHkAFRFyajL5NIGFAE6y4HWnrJk9apl+aFk560AXS4qNn4zVZpsHrTHm+Q80WAs+aPWgyZHWs7zSD1pRce9MRcdztqEnmoTcccmjzVoAUyAHg09LgjgVWYgnikU4NMZpCVjSlyetVopAeDUpYHpQApNNzil5pGHFIBN/NSI/NQHrSjNAFotlagc5bijJxTMHNABk5qRelRd6etMCYHjrSl1xjNQk1EzYai4mSOQDmpYmBANVWbIp0cm047UAXetMYgdaaJKZK4IODRcB5ZT0qPq1VxLtbFTI5ZhQA/bSgVIBxRjFIBmDS0ucUmaAEpM07FJtoAWjNNopgDHiod5p56VXduaQEu/3pC1Q7xSGT3pgSiTBqaJsmqe4Gp4WwRQIvkcA0zFKGyvWkFK4yVUB59KCQDSFtq1XaTJoAsFxVaV91HmYFR5zQIjxSg4NPwKQimAoYmpPNZVwKhzihn4oAVlMgINZ72zBztFXA57UGtFImxVjBXg9akKluvNSYHpSU7isQ429hTGNStUbChsCIsTTQeak20hWpGIDk1keJYc2COOqtWwo5qLU4PO09/lzigDM8KzErLETk44FdA4rjPD8/laiq7sbjXZsabYiFlwc00MKlYZpgjAqkwaEbBFNFPKimkEVXMJoaTxUckasOlSN0qLcQapEtFf7O2eFpRav6VaVqkDU3IVii1vIoyVqLYR1rU3Z61E6I3ajmDlM8iipnjA96Z5fHStLk2EijeSQKBkGuotwILZFxhu4rG06ZYZACK02m8xs1lULRcGJOD3qldWJjYuGB9qswPgjNNu5iRxxWFyzN2N6VNFGM5NCgmpkRvSruFi1GOBTmGBTEOFoLnpWTLHAU8YqEMaeDmpAkpCwpVGTTjHmgCIk05TxS7acqCgYoNBRZE2v0PWnBRUoUYoEcjrXgW2vlklt7l4nPbmvOrjw5qmnz/vExF/fxnNe6YFUNTs1vbYwMcKfbNAHi50gyW7Tbndh1qG18P3V5C84wiJnqDzXqdt4WtY02yl3X64q22g2n2WS3UuqOMdaAPDHRkcg00hzwoyfSu+1nwvp+jj7QJGkbsGORXNG0hmclJSh9ADQM6Hw61zbaUolQIMZyOv5VjX+paW87hlMjn04rTtdQe1hEcoMq+wxWbqtrb3OJIIEh5yw3ZyaQHlLLjlvzqSbyTGDHEUPSn3EKtIWUd6kmuWkQIVXPbAoGUQtOp5Qr1GDUtr9nEp+0jKY/WqTArClHJpZtnnHy/udqu2cFhLGTNLMs393bmmBSPFKGZRxT5hEm7KsAPWtiKyGpacJoVSGONfc5oAx4v9YCelaOl28F9LJE179nI+4P7xrP8tpGSFPvMcV6K/hTS/DtrZ6izt5u0fe53E+1AHC3Wj6iJiqWczqP4wh5re0RrSXSLnNn5Tov31HLGtqLxjbG0ngzJJO5IXK8KPasAWl3BpM0tpHKUDfOsfQGkxGC0s9zIqyDitqDff2P7mEnygQ2FxzVHTtIv9VUyQx/ugcFzwM+1WY7O6jnnh0/zQI8eZk9fWpaGRQzSQlgpwlV9kgl8xVyp6VcslgYmKVJJJT91V5yavLFrFrOtoLCZEJxtdORUWAybQXCzCGKMPLK3yqe5q6DNDL5UiyxSlvmVei5rf1rwtLpQtHnd284AmWNdoTNei3vhq21fw9ZwQRL5rJgT4xtHck0xnkOrtDABHFK3mBRwKtaTZTRISlwiXky/KhGSBW3rPgy58MRNKZbe9icAEsMsPoO9aXgzQoLGGXVbpRJcMp8qED5l9OKfKBy/httPs9cmOrMyxNkb+2fcUa3NJqspt7C3jisoT5kRA2l/wAfWmzSE6xK7Rqo8zcY25IqxrGuWsssRlsljUfLujYZouQY2o70haTZliQBuOcVW03T5tRkLySpHCv3i3+farmqIq2QdJC6uMjNZNrM1uxK9xiplqU2bd3Y6XBbqtvMzXBOAQazZraeBiHSow0jhbiNThDncBV+O7a4sJPNIG0ZXAxzU2Ay/kc/P2qKzVp9Uj2j5VOc1ZuUh2eZEMbuo96u+HNKW/juJEmfzkXhP/r1ohlPUpY57kRouHxgnPWm21mIWzKwU+hNTXkQjjRWjj8xs/OnBPNQR2nys92sqwjoS33jTaAtzujqhhXHXOWzUSSeck0YHzbevrTJRbG3Q24dNow3OM1JbWkttaLPKu1JDhT2xSsIrQXkxt4oifkjbOKZJZrFP5qn/WfNgDgVZWFftCvjEJ+83YVYUQyRFCcuPuntip5gNmDWCnlWUMWESPgS8E1m6tcSX2nRQBUihRtzDPWmmKZnS+uZUC/dXnqK0JbOS1aOWNoZ/VM+tSBmWUW/YunwSOif62THQe9aFwI7OyiOfMknfADDkj1q1Lrw0zw/qdrJHFFeXcnCKvGCKk8IeG7u7EeqTiI2+3CEn5gfUVqkSZTpHBAWeH516BuCKq6Y6vrCbzjchx9alksWuNXvT9ok+V9qiXqw96zryP7JfNGHVinR1/pS5QCdf3hLff3bWpsUqxq1vcA7M5Vl5IrdsdH3Wbvc8h+Tkc59TWXdWclooZzGwP8Ad5qUMoIkf2jkEJ/eqRmcxbF4HrT/ALFc7EmIUo4zwelOAZYX2rnIwT6GqYy4slnOMTPwq9MdaphEdJHUbUU4WoyuIFXuTg1OzRIphiOVHtU2AJLkSKAVXdjBJ5qzDfXUUSxRSHYnzAHqPYVTitpbi4EUKBnPQFsVZkt2t51SdflXBYUWA27nVJ7O4D2tzc2zzx7mMmH3H2rn72a8vJXlu/Me5f16/lV68m+0a1bssWYwo28/nV69hh1QvOsz2rocF8jaDQmLkIrzwg3/AAjkOo6bePcKVBngkADq3fFc/K0aFkjDsSoU9+c811tjpbiJd2pibzeYsHJYd6ym0+JYhJHI+8tt2g8mjnKSMGSOW2ZHOVYjIIrQ0iVpVuElCy5KkLMOOtTTTi3uoneHzI4xjaeWzUmoQx6gz3cErtJt5jVTk49fSmpBYr6hHFa3A2W5Ecg4jVs4Y9xWZKuwKzxglDnY1TWUTS3G1SIpAMgE5qa5ilWKSaYKQpxsYZz9apMGV5b5ZvNAtEiL42Z6qPY1qabpdzcrFc2EAEkQ5Zz8rn+7WPdSx3CB0c7h/wAs2OStPsLi8jKwW87xxhxIxzwuOc1aJOn1KTUIZ5LKR7VXkiy0UR4UHt9aq2sml31k1olq9rcRpnzhIC5b0BqtrF1YKM21559y7Zl44PuD2rLeNThw2d4yR6VXMTykh0uePlhCF74kyf5U5NPuC4UNGo9C3J+nrUCxyGMlFyq9T6Vc1Kbz5YJI1YlEwSO1ZlEV4sSuqoAhH3gT1NLb54hQKCOBk43e1Uyd0hbuetOSRkuQ5G4AcfWhoZdEq2l+jhf3gOCM9KZI4tr9fNOcnL/j0qsi+Y5c9T1NX/ItpwFcjexALseTUtAXZYzK22RNqN/q8HjFVmPkWbWrDEobco9RUhDeU0u7PlgKpx6d/wBaqR20rNhUJkboSMEipaKuaErWP2JLmHzPN58xVPCn1FLpEuq2t093BO9uj4dg/IZOe3fH9agjt3VXt5MK/dSaBDNEzTRF3i27Cso5HrxQhM6Xwnq628GoRajIradJlluQuGEhPGPr/Su0vvBkmtaHZ213cQQXgYyvclcu64OCfwNeOCTbayW6OxV2DbT0GK6i28f6unhw6H5QlYxmPziSW2nqMVpcVir4e8N3OvarPD5/l2sDbXnCZDDnp9cV3N3pPha0046a628U0iDaZyS/HcE55+lefWWq3FtYSafC0iea4OFOHU9wfatQWbeIrhpXnlju4IstLOfkIHT36GlcVirdaTfzXogjLTWasVEx+VRke9dJbeG5dDtrfUhqN1+7iMz7IQwRh6+oqhczXpt1068uDIIMTwsQDG4AyOT1rQ0nxcNd0y/0u9ke3a4i2RM/C7jncobt2wKcWFji9bnW7nlup18ya6k8yOYjGQABgDtWWW8qKNiAUY4yDXR+LLC/t7mzgvWYpDAAjrCwTd6A9/c1jwlRAvyqzds0XBFvSo9MurWSG7ZA8WfJc8ZHPJ9+BUVjcDS9Uie9hnEW0kKpGRnoRUtteRbdk9tGSAAJAvP4joa1bKAXJXUG8lrdF8sq3JwPb+lRzDPRfCE8Hi7w9/wimquZGtk87aSVO3PyDjrjIP4CvOb3R9Si1fV7C0V3t7YM/mSqVDRp6Ej3rX8K69/wjXjJ77VQ7JdwmCK4QZCdMHb36Yr1612+JfDezUoikxZ4Jtny7sHBx9Rj86LiPn63uJDbOvHGPvjBP09q9w8K2KyeHtPvIoPJ863XdHjqe5/HrVzU/C+nXtra2EFlHGguEcuByqjqM+/FdBFbw2tukECbIoxhVz0FNMLGHfaS9yrOFwwUk+9cZpgMliihSWXrgdOa9RHO4f7J/lXAeFLOS5sJp45AjJKV5HUGt6cyZRK9vMsF4pPYVPcSq8hdTwa27jSVnXDKoYfdYDoa5y6t5rSXy5l2nt7itou5nsWodQCLtIPFWYtR3uAKxM1IkgWrdNE8zOqinSTGDVgVzlnchXyWrchuUcDJrmnTaZpGRYKZFMK4qRZEPQ0rYPSs7FkJFU70lYgQOc9avGq1zH5sRWqQGYku49asxyADBqu9sRMr8jaMHjr70OQsRcHpWtrkmgJQOlUrm4O4kmqv2sgZzUDymTOauNPuS5FgXPfNSpdAnFZhzQrkdK05URzGpJGsnSqM9synIFOjuyBgnNWFnDjPB+tArmUwKnBptaUsMcgzt59aqPbkHiqTJIQzdKfRGMSDdwKfIpeX5cYxT2AZwTV+0UuoRTg04WdsOjMT71LDC8UmVOAOgqJMsz7qEQykLwPSok3u21Rkmta5sxcMrBtrDrxnNWdK0xFuN0g3elTzq12HIM0fSQGW4mQl+6k5FdQYFniw3U09IowM4p64HTpXNKo27s3sc7qWgzxRmWBw3IGwD1PrWbcaVeWspWSMED+JTkV3WQUxQFU8EZpxq2J5ThYhHG/z1OfIbmrHiHSZraf7VDH+4fG7aclT71hec46mtovmM2rGk6HGYZQp9CM01ry4jQZUEetZvnyZ4YilM0jDBYkVdhcxdFy0j44Gat2zyFtmM471jhiDmtnSbgNNhgR71EkOLuXgGA5FN80KfmYD6mtHy1btVK60y3mmLlee/oayZqgW9hUY8wE+1K15G3eq50i3RC8e+Nv9k8VVis7k3IjYZTuc1NxmorCXgVHPZnZuU59RirVta+XkgdauLahh8w4+tTcGc+jEMMVdjdimM1pnToOvlgGq89hMuGhTcvpmm5XASBsLUjciqsYkQ4YbTUm445qCiQIDSMoHSnKKlWP1FAEaMjJtP3qRrY4Jp7RhG3Cmic7yCeKAKTggkUw1qpBFIct1qhdqisQlAFcuKb5lMam4oAUyc0hkqN926jnHSgQrSn1qNpDTTmmHOaYD85owTSDin7wKQDMEU7NIXzTetAD92aelNiUZ5q4kS7c4pgQqcGrCHK1GY9rdKkQDFICQUpGRQOKeMUhkXlH0pRHjtUoqVFXPIp3AqkH0ptWpEGeKjMftSAr7TSZKmpiuKjcd6YCk5qGUGpkXIz2pJEwmaBFIsRR5hpJCM1HuFMCx52O9IZgQeark0wnBoAmDZarULYqipqRJMGgDS83FJ5ue9VDJx1pBJSAubyRQGwOarCXikM3qaALYk96XzKomcUoloAuZzSGoVlHQmnh80ABPFQPyamY8VAwPWmA0R0jRkHgU4bgakVvWkIgANSqcU5lHWm0wJkkORk8VOJAR1qjnBqQNSAsSScYquX96a7Go80AS7qlTpVcVMjYoAkI4qMmnluKiNADSeaaTzQxppNMYqtzipN3FQDrUinimIXrTW4NSA1E/3qEwsMNNNLSU7iEqTycjnikUDcDUu4UXENSIA5q0sKvEyEdQagz6U9HbOM0DPPHjNpq5wMBZMAV3CfPCj/3hmuQ8RRNbauSe/IrptHnNxp0bN2GKpiLNISKUjmm4qQFJFJkUhpMU0wGOOtVnODVlulQsATnFbQIYK1SBqgyAcU8MfQ02hE2aaQT0pUBapkTHbmo2KKjIR2NRE84HWtyGHdgN0NWlsIP7tHtBcpkWVmXO9hV/yNtWHRYuFGBUfJNQ5tlWHxqNtOe2MqkClQYAqzGcCs7lWKKWpj7VIFxVxlyc0xox2ouFiuRgVGwNWGTioHBHFAyHfg809JhmoZFOCagDkN1oEayuKlDg1mLPx1p3nnsaANQhaaRVRLg9zUvn+9AybJqQMarCT3p4f3pCJsmmkZNNL+9G6gBrYAqvISykCppOQag5B4pMDzvxPdyGeS3+zE5/iPNcrb6hJYuQhAceozXsF9pNpqCEToc/3gcEV5h4h0iG21Ax2z8DruoGZF3qN3cOGeZeOmEqo8s0j7mfcx71Osy27FSoP1FRSyJI+7IH0FMZyS4PXpVm4aJI1eNNwxg1UT7uaJHOMUhkMhyxNRNU6IH5bOKVodynauB600AltYS3SkxDO3rV7SgsErvKcY+UDHWs9ZJI1ESOU3tgkeldvqFtbWXh6ARfddeWPVjTA5+6tYJLuISSKyy/w9x9a6WOx/s6CKwKAW8xySeorjY45Gk+1AF0QgnA7V0t74otL+3iWKORLhPulhwDTAtW3he1tvF1kks6/ZC24eZwAR71tfEK/iuxBYWk0TsnLYcHaMVxF/qVzqWwXt6PlxhVHArPmhgMpMcjlfXNAF7SNNeTU1AYOifeI75rttG1aCwXUNLRXnnuG/dxqOG+tcBFI9qrG3ldGYbSc54os7y7sLj7RbzlZsEbjzUAdVPo2r+HbQytIDaTEl0iOChPYVEum32j6TPeyw7ElGCHky2D+FZ8niG/1KBLe8mM0m8EOTj9OldZ4vaQ6XaoW+Vo+R+VBJa+FmkWdxbyancRLI6PhGPb3rt9WtLbVLxd7MvljCOnBOeufyrA+HskVv4X2SQsu5yVIHBHrXS+ZannJH1FNoaZZmsoLnTjaSmOVCACW61DZeZYWy24UMicLg9BVSW5t41Ll9wH93nFRprEA/vn/gNKwrktxEt7qqTTBmSLlcDIzRf6ZFfv5ltGYbjs4GB+NSWtykqtKJUKMc5z0rn/ABX4lnt7Jo7QFFPBk/iYe3pWg0zjPFNxC+pRWsdky3S5EkinhiPSuK1FJrWfZNGyOT0Pp610the/aLiS4v7hIXj5UvyT71k30y6jdmcyBSjcE96TiSZaSvIwVySp7ZqZo1VcrV23tbjVbhjGo8sHG8D+lVXjkhuXgkQqVOOazcSxoDJGf3v3uq54NOeVfJ+Q/e4296BCzSgDbjvmm+WDcHaflU/nU2AbbSeTdROwyitlhntXXaRLLJrypaIgWYYAzgOK5EwvPdLDGMsxxXYaN5Vv4g0oiRNqPgnPHStEBNrHhN59W8vT4d0kWGmYHgE+lYGtw3Ftqv2OYgRLhmwvWu8tIv7U1jWWtbh4p7d9yENgE89a5TXAdQKzTGSO+Y7NgHDD19hUknOvAs8ohhYBT/ETW7qPmfYYbJOUVAT+FZus6O2iQQB73zJnzujTov8AWoNKu9j3KzSMVCDbn8c1LAv2yLcxRafGpETMAxPcVSvLYWd5NAjMwzkZro/DsCXml3cqOI2tct83RvrXP3syyXckvlqkhY5A6CoZSZo31pnw5bRJcIX4zH3Y1NpK2dxYEXcZ+0QDCg9ag0GyuNRvt0SJtjzulc8LTZ4bmCaa5JBPmbQQetIDP1RpNZ1MrbwsHC/dbrXReH9U1zT/AA3N9n0uWWEZ/e5ACj6d6eixWmlCaKCP7W5I3lcnn3rUudGubvw7Dp1rfwxIilzhvmYnqK1jITOKtLd7qWCGS4Q/aJN7J3HrU+p6P5WpTXfku1khxuQ8ZqHz7a0ijaMMLuM/NJjGDnoKlZNY1FLaOLMkUxIjQnkHiquBnzanNdzbWLCEdgetOjae6nkUB5CF4HXApk2k3VpqH2OQKJcgDByCTWxpllPaagN5CMQck/xD0rNlFCK6hOnyru2vFwq4zxUJm8vTREw5lk3n6dqn1W2S2uY54h+5mJYfWs2WeS4uIxIfk3c00IkSNmcb/lX1q5Ywxrqrh0EiIpbBqvcSiK1iVupc4+lLFcMs8kq9DGRSAs20qSSOyRGIE58zPI+lPuVMoJUlmb1PWsmO6YjYvGa0rYzGWJkRikRzIyjIFMC3pcayTXbMOIo8KPepYY7e4tLuKPps3P8A7wqGCZUSdoSMBizY61nw3zgmWPfGWIO76VPKO5blX7NEI4jtkjXI74zUFhOxDo5z1YH3pTPC0ktxIfnYY+lStbbEWVJC5Iy2ewosMiCIVLX0ivGVOcdj2qjBdy2yutu/lK+Mg8nirUR+0xTK/I6KfeqZsZtxyMYIxLjg+1NAROzQkSL94VZvbSTcknqoP50+ZY52BJ8p8fNu6E+tW72z8nREuku3d04xnII9MdqpMgxWtZFUvjiprSz+0+cFuPLZFO4Y61LNFczWq3MkqIuMLEDzj1pYkZtOlVdvDdScH86tMCoUby8MQxjHYY4p6IrKkYfY5/iY8H2ppYRkqRkilQq7bpBkUXA0Tpz27xo8yPuUNII2zsHvVmXSLk+TFbq8cx4c/wADjHXNVIEiZttvACxGSAcZHpThqlxjA37B91PMOFoApC1n3spTDK+wj3q02lXVvud0Hyds/e+lQefcDZ9njYbDk8Z4q6dQmv7gmSU7TztHABqRjYYMyPFtSONcYHrmrKRAL5QSNCjZGR94etZ8zyz/AHSWdm28nrW1YaM/2w2dx94bSXJyFB6ZpARz3WEljSOJMnJJ7g9qigvvI80OFdTgLIOQoFW/FF+yXU+jxxBLeABUKDnPrWhJpdpe+DWuzbmKaKNWiYHlh/ePr9OtPlAzo7G2F6J3lX5vmJc4IB6Yq5daU9xaTXcc8bJEvAXktisV5Eukk8tJH8pQFY+lTWtlKkWxriWMyAuoU/KwrMZkBCj5fCn0JqWKSKNbiRo3LldsUitgJ7n1qS+i+130EVovmSsvzc96mk0a7gtpJXMRSMZYo+QBWjArKLmGVZnDq4GUkHJJ96tXGtST28VtLKQgbcSo5d85wSKz1jzavcKzMEYKwPXmtC60qWxjtprhP3VwodD0I+ooA0ri+j1Fz/ae+1KKEWNDxgVlwT4dIrqWf+zN/Jhx1HQ89ep/OpkiudVtbzUF+dbZQ0xHUZOBU7WMtn4Vs7ucxzJdFvKizgxjP3vfp+lBBst4ps59DudLVrkW8g8oJIS/y/3wSeD7VD4V8Lwa9o+pzyXLxzWa7oRHyTgE8j8K5/R4ftt8lkkJkmnwkZGflPc4HX/61d/L4K1bwsovNJuXuBIPIuUUZPltwTj8aYzlNB0mHUNAu9cu2ka2tJAjQxjDMCBz9OagsB5XmzwlgjSbI1kPDivXbTwnZ6frDmxhVbC+tUEibcANHjBx6t3rHn8OabYaQVu8PHFcyzLA7hWIJxtGOT9KloZytzo1+bWxuA8E0NwS6neB5R6c5Ndh4M8cJZNc2erXEbt5vyXC7sNx0wAcn8qxfF9olxFbxWdja23kKssiwL+8QHjkVzNtK+nsphmFxGH3sCoG3HSoA9rfx5ph1SHT7QpdzTkCMxSrtHrvJ+7+tdKlx5ibt6N/uNkfnXg1vPF4q1P7FdeXvMZeK5ggEbA9TuHp0r1Hw5eiGwh07y8pbpsWaIZjbH8jVoDqftCqjsT0U1yngd8aTdj/AKbf0rXuJM2lyc/8sX/9BNYXg1tujTH+84P6VonYHqdUSMVzfiSJcQyrxj5cetbqvupLiFZYGRgCD2IpxnYiUThB0zR3rfOlwLGY1XAHSsi8tWtpRzlXGQcV2wncwlEZG2OhqVZmjIwaqgkUu4mrsTc2LO83Sjea2UuIyPvVyKSMhyDVqC6YNkt0rGVM1hM6N5lAzmmiQMtZUd6XYKeQalS4w5UnvWTg0ac1y6+MZrGkPlQzJkkNzz2rUcs8Xynk1j3hMeVc8sOKqnuKexUMnFNEwHBNbGhaEmsQySzyvGittATqfxrp7Pw1pVp/y7+aT1aVtxrWVWMXYzUThOtIBXTa34ZaJ/P0+JmjOd0YAwuBXOKaqM0yOVkeaeGqc2zSKrIhOR0PGKrMCjlT1FMLFpJFKAA0peqitimafcG6tXkcYKOU+uKAsWWjBHHWowCrc08kAbj0HOacrK6Y3r0z1oFYtGaOSBFZQHXo460NLIG3Fsr9KpbqlSUshVjxSsO5oxESxl1f5umBSLcXNrjjcB3NZqylGyKkNy7rgA/nU8hXMdjbahFPCHU+xHoanWTdXHWd89qeeRXSWtyk0IdW61z1IG0ZmgHJPJqZGA71SV+alEuBWFirmgxEsZVuhrjtb0Ly5YhYwsxYkMB+GP6106T8UMwY5qoTcWEonnMsckErRSoVdeoNNDc13l3p1reTieeESOBjnpiq50uziB2WyD6Cuj2pl7M5S3jLSjiuq06zgFsuV5qK1sAJxuFaoiVBhRis5VLjhCw5EjXgCl2Bs4FNAINKKxua2IZIi3anw2yD5tvNSGgNQFh4XFSCoRJTw9AE3WkIpAwxSkg0AQyRo3UVXeJQeKmkJ3VVkchqAJBgdKkVxiqe80nmn1pAWmcE8monZMcDn1qAyE96buJ70XAl85x0NVpCWOTUtNKrTArFTUZ4NWmUdKrMuDk07gJgE5prHikZwBiomegQN1ppFJvFGaAAikNKTTCeKAG5wacp5pmaAwoAtRsBirCz4GM1niTBpwloAuNNnvQsuD1qp5me9OD+9IZeWXI61Mr1nrJ71YWQYFAFtSKlBqosgFSK/fNAFoUYqETcUokyaQA6e1V3jbPSrJOaKdwK4RlFTBQ8eDQwyKaZCgwKdwM66ixJwKqnir07Ekk1SkOTTEM3UhYYzmoyaZk0CJd1G/3qEk03caALPmZGM0okqtk+tLkigCyZuKaZPeq+TShjmmBPvJpwcioA1ODGgCyslTRyc9apbjTlkIpAaG/NBftVVZcjrTw/vSGTZpCcVHv96a8gxTAUzgdTSCfNU2fJNN8zHQ0AXjL3pyzA9azfMPrSh29aBGmXBppYVR85vWl880WAvb+akWSs7zz609Zs96Bmh5tMZiRVcSc9aeH5oEKWNFBIozQAuKUGkzS4oGOByaUjINMxinAmgCNlpuMVKRSbMmmSNAOKdzT1UDtQwGOKBjA2KXdmmnrSDrTEc34uhZnhmH3VG0/pU/hObdA0MjfNjP8AOrfiKLztKfAyVINYHhy58q+Vc43giqQzr2NNqRlwcGmAVIhuKXFSYoI4oAhZDioimRVzbmkMQNXGRLRVjtwW5qbyEHapAgHSjHvTuFhgT0qRVpKevBpXCxajIABqcSj1qlvp4aoGSSuWNNBIpeKQkUhihznFTxsRUK9aeGpAWw/FBbNVfMwetOEo70DLAGetNlVFTmmCYDvUU84KEZoAqzSKMgGqTEmpXBpFjLUxEOTTskVaFv6077MmOlFwKwmb1qQSt60ptgKaYyKQFlJc96mWTI61TUYqZOtFwLaetSVFH0qUVIDTURBz0qcrSYoAztSjv2tT9h2b++6vLdbimQZurjdL3C9q9YvvMNuVQkZ9DXH61pNjFoz3F0MzertzQM8zlXe+RTxZ4GZjsX1rX0Kz+1Su6W6Ns61JOq3N8sM8SiIHLY9KBnm6JyFps0YRsbgfoa7e80eLVtLtfsssKOi1zVzZw6d5kNyA0w+6yng+tXykmMHC08SAip44N9sXNnJIvd1rRgtEu7Ly9PtTIw6uw6fWjlGY0VvNczK0UZZEIZm7AV03ibU4zb21jGcRKm6tTSbFrbw5cWa7WmK5wDXC3UM6XDB0fj2o5REv2uVITCkrCM/wioVJFRqPWpO1Iq4uNxFTELHwTilsLdri/hiUjJbvWlqml3ba19kEBUZwCFpk3M1RI0q+VG0megArVfRrgWH2kIwbvG3BFdkfC0FnpUElpI6soDMSBlj9avt4bsZrEXEpuZJXALHfkH8KOUOc88sNNeSAXZlRWjbmM9a7XxBdw3+mWiwqz4QFmHRfxrSg8L2Fra3hW2Vw6Egls81e8O6Zptzo4Wa2R2zhhilyivc2tBeKTQrVYSpAXoDnH+cVoeU/pWCmmaPaXHlrM0THojS7am1BLDSrKS7udWuoIkGT8+SfpxQyka/l89B+IzVa5iEh2bAfoMVw3h3xTq2u63LaW2olLVOjyxgnHbPH1rtlXWI8qYLKf/a37c/hSBor3eh2l1GI8PG5/wCWitjFecava6jq+utp8VwTbw/Kkm3t716JdalqlorPcaPCYUGXaObkCvPk169/0p9J0qbyZGOJT82w++KpMlGXbWcdtqn2FiLm7UkO8x+VR6iqfimyggnXyZkd3+8i9FrGuEP2x2kdy+47hnvWhYJpiSM1wshG3ghulNsBYdauLKxS3t0WN1/5ajrVVJ5JXZ5T5jscln5NLfiOOcqlVYnxIBUspIts77xs6UrPt74JFORAyOxbaFqtGpmmbPOOn0rIo39Btrdm+0uX8wNtC7uDVrXLSO1uneGHGQGYKOprJAWCJNwKA55Nb8WqyxWqwTKpXGMtyWFSUW/AV0INXlSZSIbtdqluASO1X/HGuW8FwkFlbGa7hH3xyIxxx71mafbgmOOK4MELvlJCeV9cVs6t4ciTTbpLSUszxF5ZXOd5HPFUmZtHlVw73UjSyOzuxySavaRpUt3Y3VxG4Uw9j3q1pHh5rvTrye58xGi+WMDjJrc8L6fLaaVfW9+ixI2cuW5HGK1DmI/A2s2lha301+ybXI+XqxP0rM1m+OsapJeJaeWjABNo6gevrWaJo1uXg2hbcHBI6sPrW/JcxOltaxTJBEMfPt5H41nIEjHj86JGCvNFu67WK5qWxjUXcUbeY6nPQ10Nzb+e8cc9yj26jAdT9+q/h/T/ADvFCQuxjQKxVx/Dxwc1I7kRF3darDpyShN/+r3dMe9dl4fgkkuLnS9QVFlRDgx9fqDXJ6ZZNB4wuBOzuLRWYuRnBA4JrYjdBp8WptMyXEt15cjgdQatRAz9Q8P217qF3LO+yytk2RunAlepvC99bxWMMIg8+4gkOPn27Ae/vW54lso5dT0rR7dN1uD5jLnk/WsK/wDDhsNSktRIYYt+63kC8EehPY1pYDmNSjuf7fvmVS2JGfCjgCtOz1JLcIZog6MMh8cin+KoxputSrany0mhUv3ye9Z95qkN1aW9ukRjihGMk/eNZSiNMua1a4WSeNg9sV3mMrwp9RXKQnfcJn7oOTWmmqzmCe1Z+nCHtgf/AK6zvKcMAB1pICXUQ0syMGBTooHanzjbtGccYqFF8uQB+KYoDIxdjvHXvVAMBWOTAOasPeu0TQp8qNjKrxmlX7NaWKXaXW+6zgJjhfqKqksZDIxySSc07ALKOC6kr64Na1sVPhdvNP8Ay2yv0zWUZMxmMjg9T3qx5rLYLCeU4bFFgEfaItobce5rViKjRY4pZf30zHaMc4+tY8Yzs9zin/MJY33f6s5UelQB1Vmmnal5cTIkNyo2MFON2KoS3RjEVnI2EBySeuewrKzI9yHj4JbOfSkvDuut/Tv1NTyjTL90IDG5kjLr1JA6VjbCcKuduc9aJHbLhC2G601GULtOc9uauMQbJtkzcmMyAduwFI7cfKgx6NTGuJI4nhVsb+pHpTDKzY3c8VaQgbBbL9fY0kjAEBemKdGwXLBRuzmh8NkkfhQVYtQvLFam7iONjbW9jU1uqC3lmlZY0yMZPJJ7VSkk8u1aBHLCX5mz61NMMRIpOSRkmgVieXzFZRE7KjKcYqnapIsvmbcoO9XbSaNFDZYsAVGeQBST4ERcEH6DFQIfpTNBqEMoj8xFYsUxkkV2eh3ulm01C31W9ewuLh1KTsnLIB0+gNYM+gNouiW+rXV24lZx+5jxkAjP6d6517+dnLoxKnu53VQG9qlq4t9lpNHPDES5nBwXJqpHqGoR6MbN5m+z7cqh6D6VDPKtyijzBEAo3kcbjUSx3K8rHM8QIJfbletIDb8P2yPqot/tCMrwsSqgnPt9ac08FrPK01rO9ovyxbJcFD78V1unX9/aQ291Hpck0MkRcgRIChx/e7DivNNQvHvL+4u5DtM0jNtHQUWJHLcI13cXCyyRSD542bk596lkkAtW3S7vMHzcEZOapwOiuZH3cchgM4NdFH4h06506KG+3FwoEnmRZUkHrximVcwjMi2qSQqQed2futW7rd7rFro1lpmqmOZZFWdHYDeo7Ln/AD1rL0yKO/ee3aVbZJgzxqBx9PxqtKtzdLFI5d1Rdm9myAB2HsKRJ2/gW7tJ9L1Pw+oSK81O3ZTcMxwSAdo249z3rH1zT9Rsm0vRbuaN4I48JJGcgjOTxwRjIzmqWmXAsdx8tHYj5SRyp9RWyWbUbmw1G6vxDMHd90gBUtwMY7DgUwN74deH7OYRasXiF5DcFYgHClkAIORnJOTXpjEj3rzDw1eaf4fu7q6u4If7Tnk/0RlbcqqTycdq9Ss5jqGnwXTwGGSQZZCMYNUMrS3EkZEavGksgIQOfvH6d64rxH5ttqkDSNJfa5OuLZVTYkf+0D+tdnJpYbxV9vmG5I7MJFnkK+ecfnVTxBZRx6S+qSPtubBGeGQn+JsDkd6APOxoLT2VwbrUTFrwYmVZW+Ur2HvnjB965qawg+0QwpLMh8kF/kJxIM5rY1u/PiLV1mt3S3cKlvIkjcnnGQfTmrd5bapoNqljcmzSZVLxyiQSMQTwB6Hj0qZCkZijT/J08mS5W9KZZ0UkEen/AOquk8LeKtS0vVzbajGW064mGGWPaFduF7cDiuWvr26MNvHfztEWUtEcKSoPXnqfzqxplza3s7adcXr+TK0ZSZ25Ow9P1rMGzu9R8W/YprvTpophdsWKGQ7YgpHQHvgfzrX8Kny9EYf7Q/8AQRXJ62serPAL2CRG09SrR55ZTjDg9+g/Ous8MRJFoIEa7V3dM1rEZtLNzVgSgjk1QVlqwImYZHSnYBspGTiqksYkXaw4q48TAZqFl9jVxdiWrnPXVv5UpOeD0qGJfMlWMfeY4FdQ+mx3cfTn1qSz0S3t8NtJf1JroVYycDm20+483y442dvVRxUzaLqCRCTyQf8AZDc/lXaxQJxxVj7OhHAqPaj9mefrDPbsGmiZB7083CM2Qa6vUtJa7TbvVPc81zt9oMtpFvibzfUDrVRqRY3GxPYzCQbc9KvSaXbXLI8y7tmcA9OfWsGy862ZZ5oXWL1xXSi4UopHcZrOoVEtaeI7RNoCjJycKBk/5xWlHOsjYFYPmc5q9ayYw1Y7l2NtXAGGPy1zet6bbLLbyWypFkkMFXrmtb7Qrd6qX22SMZGcHIpxYWM6K1QHOKn+ywkgmIE+uKFNO3+pp8zCxn6sEtY4fItwWdvmbHCqOTXk82q3BS5lilCo07IBHwME9a9TuPElnBqQsW/eEjPAyK8nl8rUoJoIIEidHeZAnBOG+6foDx9KPaMLHU32ofZdFYFs3HlD2AJHHNcZFqXlruZ5TKn9w8YrVuILy4tS8p+d1w6+y9Bj1rFnlhgl2QqEwAGx3Na+0ZlyHZQa7ZG1jwXOBg8Y5rTtJlurdZYuVYZrz2a3mkKJAckwhwg6sScVv+HdRk0pJbK5i8ttpYsW4FaRqXJcDqGVgM4pqsQar2uoLd4C8DGSDVnbitUQKDmrlpdm3kUE/LnmqS9aerbXBx0NKUUCZ1VvdrIoc8e1XI5A44rl479EkCpF8uepPNbMUgAyTXJOm0zaMjUBqQZqhHdx5wGyatpJuXNZNGu5YHNIVxTFfBoZ8ikBEWw9SiQGqrk7s0K/vQMt7hSbgKh8zjrUElxtNFgLbSiozKM1mS3pVuTTft4H3gwHrV8rJuaolFOWYetZy3AYAg08SYqHEdzSEo9aesq+tZgnHrT1nHrRYZouciqE7BWNP+0Z71XnbdmpAaZKM5qvmnq+KYiZuBUW7FBfdUDk560AWPNpPNqqXNNL+9MCy03NQPLmomf3qF2oAV3Oc1GZPemMxqMtTES7+aXf71BuNKGoAm3GlzUYNPWgAPSmk1I2KiagBN1AemE5ooAk304PUINKKALKv71MklVFqRSaALok96sxsCKzQxqaOQgjmgZe6HJpwYZquJvWlDbjxSAshqeGFRKCRxT9hxzSAnGCtRTKCvXFCkioZpOvNMCnccCqL9atTNuNVyMmmIj20eWM06jvTAbsAFNKjPAqUjimcUAJtphWpQwFR3brFYzS7grKp2k+vagBoXNIRiuUh0nXrjEskpi39ckA47cV1ccflQqm4ttGMt1NMBM04Gmd6UHikIfmlzUeaC4XmgCUMRSiXHU1XM3tUbSZY0wLLXHoajaYnvVYsaTcTQBPv96NwqAtSbzTQixvFJvqHdRuoAl3n1o3VFuo3UwJg5p6vz1qvupwalYC2r89anQkdTVFH561YSXGKQy1uJp2ajByKcGApASrUyoarrIM1YWQYoAGjpAhBp5cUzeKAH4o4qIyVGZOetICx3xQQKiV/epQeKAGsvPFM21M2MVETTQXI7iJZ7OaM8krxXn9k32fUoz/AHHxXogPX3GK4DU4fI1OYYxzkVSYHebvMjWTOdwzTaradN5unRHP3RirG6gRKKUYqMMacDSAdikpwNGBTRI2kNKQaYQetWAuKXOBTC5FMLmqsTcmyaTcaj30bhRysLkwlPc07zR61X3r3NNLilyj5i6j7jVheaz42IxVyJs4qHEpMlKmmYxU4I21GwBqSiMsKgdueanZBmmGMUCK+RT1cCgwjPSnC3J6UhDxIKXzM0nkbR8xA/3jilKqoJLoAOp3ClcYuc00iqTa5pCS+U1/GHPar0LLcrvhYOnYg0XAbinoKhlu7W3fbPcRRn0ZsGmHWdKjHzX0A/4GKdwLynFTK1Ya+KdFMvl/bVHuRxWrBc290m+CeN19jSAsbxSFxUJbaeaa0o9aQE7Ed6pXlha30XlXEW9PTOKl8zNG+gZSttCsLRSttBtU9Rurm4tJV9fP+iBU/iyK7HzMVHJO8ETyRDc6jIGetAHz1pqwwfOqSTcYwHIxXU2XhnTb+1jnlhJ3nBVjk/jWLZeF9VjZnTeuBlSozu/Wuy8IafeqxuLt9yD5duK2RBZ0/wAJW8LeVMrLaD/llnIrfh06xS3Ntb2scUZGPkUZFWgPNYL/AA96HuYocR28Rmk9AcUcw0zyx7eXwz4kNvckmKUnBPQj2rM8ctbz3VtFaANJ0wg6nt/Wum+KFhcPBFqGcsAQEHRe9ec3l/HcRRSopE0fUntVCH3fhy802zW5vNse7OE6nP8Ak1mHiunuLptZ0JLi8uEjManbEvQH1NN8L6A2sM9xEwkeAglH4yOanlKuR6Zos4tbbWbdhKsbbmjHBGOK761sItauoNRiAKR/eJOOapWln/ZtzcWzxPFFPwqsBjPt+dauhQzWc7W+8NA5yuO1a8pnzG24AIwMYHSopTgCr32ORz2FNbSJHYEscDtmpshlaN9waM/dZcGuK1K91fS9RbStFG7zPmDbclf8816C1jsHYfSq0WnpDK0sYCuxyTmlZFHN6Z4Yt7HGp6xcvd3TfNh84U1V1Twpd+Ibxrm6uvLt0X92q44rR8QeJNO0xkjkYzSq33E6fjUcfiKXVNLd7WGO2h2n95I3A/Sp5RIyfhnpstrqups3KbTGG9TmuovPE13beZY2lm7zR5BlkO1FHqa880nxfqWjySww+VJE0xZ5HXJb6Vb8ZeILzUYbWGEiNJ1+ZEGMjjgnvSZVynda5qU2nXaTXZdSx4UnAz6V1GkatPofg2Gzt7CRZZSSZWwUAPc15kLaQ8CVN4/gzzWmfEOsCPyncMAu1ecYFTcZHqNlbRJPLFqC3EztuIK4yax4LiWGYOjYYdKsuGMO5xhz1qnjDZqblEskjSvvc/MaZECboY5FMLblqe1UsjOuN69M0CCWQ9AaW1maOVyFDHHQ1XI5JyOTVy0t5JFBj/1rnaF7mpaGSea91eJI5woGNo6VelumumBKsu0Y5qtJaPYsyz4V16jNWLQh4TJghWGBkVLQ0xI7zyWVUfh2C/SuvsNVXTYzp2ppI1u7Y82M8n2rzqKTyJX+UNjK81Ol1NNNFCXYhnACk5AqkhM96fQ9NXTI40GIfvbjySK8m8T3F1e+IntLaVpoYjlFToB7+tekaubkeGjaWo3TLEq8dceorir+zj0DTGnt5Y2aZCCzjLlv6VpYzsVfDAia9uZb6G3Ns/7uQPwVIrsNc0jSTo/kw2iRb8MkyIPlH1rzW2tj9k8yThmJP1Fej+FLm9t7Nbi6vDdWLr/qxyYzzzUWLOd8Q2l/aaTagwJ9hQAeejdT2rS8MhpdDu57hQrTgJHjg8Dj6Va8VQXPiNGSE+Tp1mvmGUDh2+lczoumIkmmym6cTPNgKx4x2pqIHV+GrGOPTdQiuGVLgybZ3PXaK42OO41HXhbxsTbqzPEo4AA9q9I09lmS6tLq3VXkJQTIOv1qrbaQsfitTGuYre0wx9yP/rU7CLHh7SlkthqF67tfNwWzggDOBV/VbRbuyYqc+V8xB6Yqxah1RfMRUIGCq9Ky9YSewnvNVUeZbOoDqrYbPriqQzkfHulTyR291DbZiiQAbevOM59cYrnbbRri40CRZkEYOXjONzE9uO1dn51rc/Zf7ThvIEnJWH96cHpxTNfguZtIvpFmNvb26bIlzywHUmq5bknAR6HdXan7IgZAAC7Ngk96zxJNFKYpDnyiRj0NaNpqkljpghQu7yAkkHGz6Vjo+N29skkkk9SahxsNMsvKZo23KmR0I7U10liZVK/eFQI5TOO9XXuVuSHkQA4xgGs2MpSx+Wx3KAaI3G/OMr6VPcKSFIOMDFVMYbGaLjJsYGaI5FDYcBhjGDSptYEeimoiPkU96YEsA2iQ993FOJNT6bpt/qu82kDGNDhpD0zWjc+Hr2xj8+RopYR954zjafcGkwsUEuGhUADqM/jVeW6WaOM+XtcDDMDkNU1lG9xKqdR2PrVO5t5LeZopRh1OCKVgE8zqq5JIxUZDKuGIz7UYIp6qCMmrGR5LAH8KX5h1qW3gkuLlbeFd0jn5RnH60tzA9u7RSrtkXqM5xQIbjbknqRwKaXYn5jk96ZvYkEnkUHJOc/jQBPFG0z4BwAMkntU8ixrbOyjLA43etV4JjFv9SMUpyYhg8UDH28kaxYL/AD5zjFaFvJBIixyRgELt3A8sfesYDa1aGmrFJcqtxP5MRyS/pik0B10d5aa1pCadPHmazGVfd8gH985wd31rB1G20w3ANlcvPCDiSUptAatWDQbK80W4u/30MQUsigYaQ925+8p9BUGmavFawrbR6bDLbbAZTIMFnHUZNAGGLF53ZoChQDP3hwK2tH1t7WZHmR5EjcCSJACrLjB49an1K30638IWmpwxwm5urpmLIclE/uEfhUI0trhpb+UtDuUOIo+Dj1qQGap4iuNTla0th9j05XJEK5QsOPvf4VreFtL0nUrC6hkjSLULZiymU7sqegx3/wDr1g32m28Ulx5VxuCkFSox1qPw/ftpWri9SZBJEpIRzgS+1XEDo7nwhcx5lu/ItrWUlpPJAwpHfFYemafpEuty6W8skgmXFvMTtBk6qCB0z0zUGp6/qGq6lPeSebB5q7FjjPCpWeYP3Ec7YKS52kH0NNsg9b1L4e3t6lxeyS2cVwkBWKC3jyGx6n+orypoLuytIhPbyRQu/ExGVbB5xXdfD/xWulR3VvqU9wbI4WJ9pYK3Pv3rodUt9Pvvh9PaQgzSRKxt1mG0l93UDNAHmTR+RdxLnO4E/him6Yq3F35So8m87MIMmtfS9D+1WU9tduINRQqyEjc4XHAx3/CtbwD4NubjUpZrmd4bYFlwmVeQj0Pap5QOw8EWWm6pod9b31pb/aYCIGcxbSBjgj3rs2eO00+VxNCwgRgq+YMEgZ/pXmvht7uyuNUtrG5g/sxbjDC4IYsR94g9azri5WyvL61068uriJFAgSTqgbO4Y+nFA0d9Y63bas0E/wDaFvFtVS8QbGGPaud+JN8+oxxadZ3UQtIXEs7oS7FvbHp9fSuI1AS6dJNp9vKtwSivJIUClR6UybUdWit7WKNt55aMDDFhnnIqLlmfp1o1/dmGI4UBmdyOUA7mnXNz5VzGlrCs72rtukZdwbsDnvir9rqELXEs9zKpS7t2hunji2eSegqHSLy0tHFpIH3XKECRjhdvPIH4U73IsZdzfXM6o1zl/LUKu4cY9jU9hYm51Bg8boscTMxxnaccUya1hhhuoJLg+dGcx5PDA81ZV54nWd5EV1VeCchwR0IqQZ63a6FaSeAmmuvLuJo7XzUlH30IAO3PoOOK1fCUSt4WiZuWMjfzrybQNeu7D+0hIsn2e6hMciovyZPQgfQYr1Dwzq2nwaLa2z3aJJJI3liQFN59BmriwNg24EmTkVcQYWo5DzinxSKQFzzWlwQpGalihjZcMvXvURHzZp5YgZFIGhwhW3bg5zTxMvrUJfctVmfa2KVxWNSOQVbibIrGinO4ZNaEU3OM0IZcZA4qhdWspRvKcqx4BHarX2kVMkiuoNIVjl7q1vIk3SsZT3OMVW3kda7BljccjNYOrWixFXjIOc5B4q+a4JFVTnFWI3KdDVNMjGamRjmkxl1JXHQ02WRmbLEk0xTTmGRmkMYDSuMRs+RhRk03O081M9stzbFS2UYYOPSi4HA6xd29tcQyxpHMyzGQ4b7o6fnXMajq0Q11Zo7PDv8AMy98Y4z7/wCFeny+D9HlSRRZjc44y3APrXnPirSHtbiKa3t449i7ZjEQQxyMEfgaAMi+1f7QYljOQd5J784zkf0rNu9hljKEFmXnBrophpxsjJ5EEDKrAoo+Zg3QjvxjpXKW0f8ApEUR4OcH2qbgbWj2E01nfXMcrrdwxhgo5ynOce9a0tr9iWEXsEvnyqJDG/GB25NLoV/Y6ZlJy6yuxEm1d20A8A/jXo/i6xhfQY9XkUOY1Vhkc7Wxmtac1ezIkjgYLz+zmkQRb5ZZBgH5doPHFbY8z7fIR80QxgjoD/WuLuzJe381vY27ZhBLkMCO35V1un7bK1is2cvKBlgOTk9a64yMHEugc0v8dSQ21xPKscUEhdugxW7F4UaQZluJI29Nmf61XMLlOfB54qdLqVFAVql1HS7nTJj5q/uCQFl7NVQYNLca0NGynDzAynn1rXW+RX2dvWucgjnlcC3hlk75VcjFTeeF4c4as3BFxkdRFcoxxnmpvMBrlILh/OGGrbjlbb1rCdOxopFw80xlIHSlRsmpdwxis7Flfdg81SvH4wp5q3ODuGKqvHvzmrQjHmeUSAMGIPoKPLjZCXD5PQk9K0ZDsG3GSfWoGUNGSRzW0WZ2K9tcTRHazBh29q00mDjJNYTuQ/GRViC6K/ePSidNPUUZdzZJpN5FVkvI3x81TbgelYOJqmSeae5pDJ71Ec5oqBkw5pGOKjDYoL570AO3YqJ5OvNML+9RM1MBWf3pC9MJFITxQIN1MZuaXFRlCTxQAE5puBT9hHagIaAE20m3FTbCRR5Z7igCLNKDUhT2o2UAN3UwmpfLoMdAEGKeqGpBHzUqoAKAIAhpREc1ZEYp4QUAVhGfSnhMVPt9qYRg0ARgVIKYaATQBMOlT24DMM1Ap4xU0DBTkmgZqosSIKhklUHGahaUMMCo8ZpALJP1AqrI5PWrHljGTUMiUAVXPNRE1O6nNM2CmIjFGKfgU00ANY4qLdT36VFI6QxtJIwVF6mmAskqQxNJI20Cqkcb3kq3F0uEHKQ54HuaYkb3c4uLhcIvMSH+Zq2XxQA9my2aazcVEz56Um84piAvg0b6YaaeKAHlyBTDITTS3HNMJFAEm/3pCwqLdRuoAeWFIW9Kj3Um7mmBJupd1RZo3UCJd1G6ot1GaYibcKN1RA0oNICUGlBqMGnA0xkoPNSKTUa1KopMCZXYDGaeHJ6mohSg0gLCvg9alWXFVA1SBqQyyZuKTzs1XZvSm5OaYiyXBpu7mo1JqVIy54oAerEVKrHFNEDVL5DgdKQ7Cbjimk8U4Ie9BXigRFuNct4kjxcpIOMjFdUVwawvEsObUSg8qQKaAd4alL2ckbHO05rXrmfDdwI7xkbo9dQ20Hk1YB0pwNRbwelNLmnyEXLO6nBhVUMfWjzdvenyBdFsEUpAIqmJ/elMxI60ctg5kNkO0nmojJSuSTUZX0rWK0M2xxl4603zD61HtapY0XvTsAwyn1pUZi3JqwY49vAqAIS+FFLQRbjPyjk1J5m0ZzUAjlA4FRmOUnkVLRaZcF2f71OF361TETelKIZPSo5UPmZd+1D1qpda9p1kP9IuUH+6d38qp6k0ltYSSYbjsBXmNxdvNcSOw+bOKlxGmei3fjnT7f8A1EL3H4YrFu/iBfScQQxwj0Iz/WuNa4du9RNIR1rNlWNK81zUbyXfcXsj+wJH9arjULnOBdTAem81UGW6Y/E4ppXB6j8DUWKLBMf/AC0kOani1Se1Tbb3cyKewas45PU0Bc9wKVh2LUty08heSR3Y92bNAYeg/GqjDZxkH6U3eV6GiwFwgN2qSK8ntn/dzSp/unFUFuCDk9KsCK5uE3xQu49QKQHa6b46khiWG5i8w/3y3Nadx4xzbb7UKH9CBXmUcjByp+8OtWRI2ODQB1j+N9SKkBUVuxA6fpVVvGutj/l4T/vgVzUjOqFiat2sPnQBiM0Ad5oHi03yrHelRKf4h/hXUZH8TAfWvH0dLOXzPs7Er0waXUNbnvoRGGeNB2zVJiPSDb9lUD6UkksdnayZYK4GfWuUsvGF1qenz3FvtBjU5BXoar/D3UX1HV7tLp/MmYk5bnFdPMQbeh6rca6Z4AyRLGcMe5rpoLdLWElB8w/i7muMvo5PB2uzX/kPLYSgn5OuT+dWdM159S/065u0S3zhI94AxUjRzvjm18R3Nq8t9e2qWSHcsULcn3Oe9cVaaR5dgbi+JiiP3COc1ueI9Tlk157aW4EloSBvHSjUdSt9QubGxgYtDGACWGFb/OKAMC00qXUH2WsqHJwFJwa7/wAF+A9X0/UBfXU6QRDAMa5Yv14NV/B+nyaf4wUTwqIZc7SeQeK9YJoAwb7R4Lq+XePuHI4qzHYxwvuVelWQM3TNVjAIoAZEeeasqQRVU9eKzdV8U6fo21GD3E5GfLj7fU0kxWNqSJWU7uF9a5y7ivp7hobYbo/4j2I+tWdK1e58RRiZbM29tnGC2d1bC25jGN5VfarTsM8e8S6VBN4gitl/drgeaeuTRrcE0r2mj2wCQuQMAZrudO0PzdZvdQuYlfnERPtV9LS2W+N3Km5+1Xzknl2s+Fv7MgSOB98rHBXjk+vtU17bxppNhqEyhZISEYAcn3r0G6ijur5XC7EAxlqhv/D1o2lypbQfvAMhmOcfhU3A8fvrq2m1SWdCFDYwAvWq8StLeYXnIr0K58K6Ze+H7mcQmGaPkyDqxrnJdOuptOt4ba2MKR/fum4XP1rnkjRMwLoGG58hhwazZgyOVNXb0blDmcSyAYLq2RVLY8nJOcUihitiNl9e9OgZkkABwvek2+lKiMWCjjJpiEnjCyEqDXV6Boy6rcEyyiFoIg6GM85rJisEubqOITphxk+1dVZ6K9hoN1NKcyyv5MTYwCD3+lDGYkmoAzzoQlxMW8vey8fUVYvruOUWtkYzEyLypOc1i39hJpdwIZGDbhkMo4NPBe7n87cAQAFJqbElKRCJX4/iNWdMg/4mdpNKMW/mjc1b+laUmoSyWk7OhC7mZDwRWz4a0G3s7u5WU+Yn8KsDwRW0Ig5HcFt0sdwP9UUGD6isTUbCG71lZ2gRrdBwCOM+taMYaOMBWJ+pps5leB1QrvIwCR0ro9mYcxwniuIPq7RI6eYdiiNB611n9j/2Vpy/YSxcQ4kRujD1qumkWVtf2807eZcOQWZjjmuieSBnUFsknAFHs4hzHE6n4zF9BBpxs5LSxZgJJnGMgdsYqiNTsdP8VKyWkl3bomESJclD6gfhWn4vLXniXTNNWHdDuDsq98nmrVxLa2Pj6xZQEDxiMlRhQecVMo2Li7nQQXbSS+asbxpIAcOMEfhSyXHlatHKi8FMM2e9aFxBCZjuO0/pWHeXsZnUQ7FtFPz3UhwD7L607Em4LwH+KnG6UDhqzdO1HStUleKzuVeROqkYJHqKnmXa2BT5IhzFbXLeLUrNBnEsUgdDj8xXD+J9Q1u/1SPT7dPItpVyoIHIHUmu0nRpNuP4TkfWsDUrO8vdRDL8jRxkK+ATz1quRE8x5nKk1nPLEz7sZXcBwfpVUdcCun163uVtvsyLlImzI56k1gx2MpAkIG09K42bogBwMHrVm1KF8SKSp9O1RXKBZyFXC9qdaxM064YKo5Yn0qGiyxexbPmjB2+hOcVS5Jxjmt6aWCcphAFAx0xmqF1CiAyIMGouBTZWQ7WBBIpGbCfQU+ddrqMkllzVeQ/LtPQnFWgPQNJlmsfDlikMpVWBl+XrzWzc6fdTWXlXtl9pt54w2+I4YD0/Cql5YyWllbR+U4hSFRv25H+ea6W11azv7RVjlWN7dRnecZHoKwfxHoxhF00eeXuitp9s1/pdw80cP34WABjHrXN3t3cX0glncsQMDjoK7W+kWHRtTlPDGMqv1NcOmDAB3rWLOWtBQdkVtwHbn3qUn0xTJFAIxSZIBGPzFUYmno0s7apDFA0KNz88o4qHW3D6k+1E+X5S6EkOfWs9gamxmBRTAFgcpuVc1GCO/WnxoZZVQttBPJJ4qWC0aW4KZUqOc560AQd66Dw1pmn6nd/Z7+fy9+BGu7Bc+mar21oqCS7CjEbbuTiu78P2NxFp4a5trYwmQTK2wEx88/lQBwXiPSF0LxBc2MZcwIcxM5ySp/yaSygijiBu7Nn8whoWP8ft7Ve8eyRSeLbryJxKi7QCrbgOM4/Wi8vLrS5rCC6s0k8gLKgdsHkZoII9b1PVJfI0y/mRorM/ukUfcBHT8qyzgoEB+QfwjpVnU71dRu5bp4ESWQ5yv+eapBhtAoKTHFEC5UVpxeIr2JI4EKeTjY29dxA71nRDcG9hnA61D05pNDOi1G8in8x7crsYFCoXBJ4xxWEVy7D0OKeDmLdSDHX1pLQBZZJJVBkO4KNuPb0rr9Rg0RfDP2+GMyRmRDHBJnCNxuHv9a5Ly8itJdQkfRotNZWKRzB0YdF9jTE0aWmLbXGnz6XCk1q91OjxNKMqiDrk+361Lr2mXuls2kO8t5bwN9oglORlGBzxVe71eXV5rppYyDbLmEQDlR3z7dK7p/FvhW60exj1++S7vFjXLWcDEgDBwwOPx7VaZmc5pGi3Rs767Rne509Y5LcOuHBzllwf4a9Qup5bSwJtVRXfktt6ZHJ4qlBqvh7V9Ya+0rU4GMkIguLWTKu687Tg9cc8ChtYnvL2W0sNJlkSNtrTzny0/DOTQho8guZotP1Sbyo51USiSJmJB2dc++a17q8tNa1N1ubl7NmfdHdfeCcdG6ce9ega34dOqQXmI43d41jhUgADnk+x96841/S77Ti10VitootsRjik3hic9+9DiMrXmi3tlFcXCx+fbIMfak+7IucA5pr6zbIltHp8Ds6j5pZFCuo9MjrUOn+JdWsbYW0Mka2+CHVogwbP16VWgtWvpLrylBCqZMg7fwrFxKTLcmsQRotu8aHKssjKnOD61dm1SBtPiji0iEW8arGtzMDuPGMmsKG3e6SUpEJXZec9T/jRDcPFm2kQ7O0bn7v0+tK47jbiJSxjJEhbG11PDE1akE8NuzTRkMAoUgg+1DpbyJlIPKeMhj6Eeo9akiaSfTkhaHduYssnXHPPFAnqbukNfXGj3t6RFLBbFVO8HJzgYGPrXQeF4IPEOiTwXUjs0c2J0U4MB/hkH8j9K5qzW7021e2+0wm0mKsQrZ59a9I8KaRb6dGb+J/3k8YVwpwCQetUoiNexN9Hb/Z9QZZZo/uzoPllTsfY+tWgxDZpplUcZFV5LgKDitVG4i+kxJ5NWFbIrCS/UShSfxrUWRSQM9RT5bCuSSE7vl6VE3rU3BqKUjtUlDNxHSpkuGAx7VXzk1IozQBPEJG+51rVjO0YrMWUR425BqwlwSvWkA+e+ljlKpGQF6lqxr26edyGOfStGb5kI9apPAg5IoQFNWJHNToaYy46Uitg1Qi6rU/fxWXe6lDYwqzn5mOFXpk/XtXO33ia/aa3NkiJDuIlZ2BH09u9JoZseIdQjgEdtPM9qJeBLjKn2NZt5q2s/bCNM3oiRopLKCoGOpqhq97ba0LZWtrsSFwDIwO1R0yB0rRW70/TozZPfyCJgARIC5YHjOfyqBnWkPdWLm4leNkXDbOp9T71w3ifVLJrO1/s5oJZWbzjvGAQnAyPU9K662VNM877PIZPPZRGZGwQB/CM9e9eVt4dkln1GynuUWaCTcrkZXDc9c8GgBljrdtP9se4sPP88grsUExt9Tz/APqrnpInl1SWMLtYtwuelV54TbPsPJOcUtuWluQxlCOf4m6UMCzAjSTyHy3dI/vbTivSbnxxYaj4aW0uUeNTHtCKMhAqkevfNeby/aJXKouPMOX8vufetzTbK2EGlySoLlHnBkjA+8oPIoQmTeFrK9a4aGAyK1029WIySB3r1by7Lw1BaxvaCSVxzIFyxPck1m2N94W1HVrS50/MXlMYSACqdM9PzqDXPEtpJrAls5IrpCPs0QDYCseuSfyq+Ymx6DAIniSROVcZU4xkVaGMVj6XfW11B5VscrbhYz7HHStJWqbsuwt1aw3cLRTIHRgQQaw7bwpYQKyzK02TkZJGB6da6HPFRM+DVc4WFtoIrWBILdBHEvRRUNxp9pMjo0KlXGDgDNWI3DU/ip5wscmvhbyJQ0E+7H8LD+uaSVGtnwynHqBXVBATmo5LWORSrjINVzhY52N8jNSZyakv9MaztJJbZiwUZKd8e1Y0V7I38L/lVEmuw3LUQTmkimLgYqUK3pUjGNErLjArPuLeVDlVyta0fJwaHGKpSAwpl3QkbRuA4rMZDv2YIb0NbN1bjeWQkH2qgtrvcs2SQe9dEZaGMkQNC0WGcAirkNym0Dp7UyRS6sCp4qpgpQ7SQXsbCyBhwaCapRMQwxVpQSOawkrGkZCs4FM308xk0gt2z1rOxRCxpuTVgwc9aT7PQBWwaTB9KvC35pxhHpQBngGnhasmAZ6UGNR2oAjEeetL5R9KcCKlRh3oAgERHanrH2xU4Kmg4FAyPyh3phiHapGfmmeZzQITyqayYqbeKinR3TKMB9aoCMYzS7hUbQyAZWRT60gBHU5p2Fcl80DrR9oXsahPNIAB2quVCuywJloMqE9arE0wtgVLQ0XGkTb1qFpR0BqsWNMLHNILl0SGpFlwaz1lYCl86kO5qLce9TC4X1rF86gXQHWgDZ+0L60nnKe9ZH2ketN+1j1oA1JJEAyDVdpgOprPe5J703zi3WiwFkzHNN82q5bNId2M0wLJkBrLydQut7D/AESBtqqeQx9anmmEVvI+5QwU4yapWF5aW1hCslwAxGSAM80Aahc0m7NZ76taDp5jfRcVE2rqf9XbuR6scUgNQmgGsd9Xmz8sKAehOagbVLw9HRD6qlAG8TTC1ZFnqNxLP5c0m8HoSORWgZBTEKznNM3U1n5pmaYiTcKM0zNGaBjs0U3NLmgQUU7jPSk70wEFLTgKMUAIKcBQBTgKLgKKkApgPNPBpASqKeBTVbtTwwpAOpc0gOTUirntQAzmpUGRUiqB2qQYoAj8omjy+elXI0BFK0fbFAFRY/arsEeADinxQDhttThQO1ACDAp5ddvXmoyaiZsUgHtyaRRzUfmU9GB70wHNGCOaxdatZJbGTYucDNbm6o3iWeN426MpzTQHnNnK0N0jDjnFdk8UmxGx94ZrjLtPJv5VHRXyBXf2Ukd1pcUn+zWhJmhyKeJM06aAqxI6UxUrVNWMx+7NMbrTlHNTLEvUildAitg08ZHWrO1R2FNZFPQUrhZkB60maUqc0hGKtPQkUc1R1PVI9KiLyY3DsRVwsV6VwPiF5JtVk3uGx0x2qSomwPGmBzbqfx/+tUY8c3YbCQQqvpjNcsLdpDhWA+tP+wHGfPSsHORpynR3vjC5uQNpSL/dFZra9eP1v5/oprFaM8/Op/GoSj1HPIfKbb65fupBu3+tNi1i8ibd9sY1iEkGmFttHPIDqX8W6k0DQmdXUjGWXmueYvvY7upz0quHYnipEdncIqszH0Gam8ihw461asorGaUi7lKgdAB1qa20wzyrvjfj2NbMmn2ttbFvLdXX/ZzQBSSPQ0cERyNj/ZNSy3EM2be00+IKejMOaopcs55Cf98irdrbxzzbpLlYVHcnrQBSudFktlzJINx/hAyapvpl4uGFvKUPcIa7ix03S2uhPcajFMV6AyYrTn8UeH7aUxyXalh3VSRQB5g9nInAjkP/AAA05tLu1i8ySFlX3r0WTX9AunzFeoW9NhrK1i7+2gJa26yL3YDGKAOFFtNIflicj6VvWWrx6bbeTIhPsD0rSt7Am03PI0bD+FelY1xo15e3Rjtk3Z75oAsadJo+o3phuowu853Cp/EPhyHTLcXNrJJ3+Un6Vd8OeDZrG/W41KMEDsDwK6jXLnTvsLW74kB6ANim0B5XayTTzCNEDn3rpoNPvmjVVtuT/d6VU0+G2025Mxj8xM5yzV2cPibRjEXLpE393NDQHKDwvfLIHltpwo77qdDolrc3HlbHz3O+tW+8e28bGK3tTIp/iLf/AFqwLa8u7uV7i2WRMnPHaiwHnUMsgXMcjrnrtOM1o6bd3mk3CX1q22Yd6pMr7yNoGPSmySScDPSgo6DWPGeta1b+RcT/ALrGCuBzVfw3LFDJMk8W9H6nHTrWazsLXc0JC/38cVFE7bMqxXPpQQXtUWEXbGEOF9G7VZWzuNRs4p7eMYi+8w7etVrIOXiuZ4pp7cnAwOCfrXQzlreeGVrO7trckF49vykduKtMD0iy09LvRtLkdmDxRqQR3rfRtwrnLDxZoU1tHGt2sQUAYkBBrQfddqVS5/0c/wDPJuT/AIU7gKdWhgllhjje5mDcrEMgfjVO58UWFpdLb3yi3lYZ2tKCcfhSapPH4d8O3E9shMzDapHUsT1rzjU/CVyvhptcvWeS7nbdgPkgHFSB31/488PWVpMy3e+YL8qKMkmuQ0rxNe6vYXsnkwsyZzlfmrjrDw3q+pSrHb6fKN38TcAfU109lq1v4altLKW1MUyHZMCv8JpoR2Pw41NpNEuEGwyiXB3Hr3rori4vpnZN8YT0FYdvpj2erJd6eQtvcqN6jpXQLGqgAChgOgSXywHfOPWho8HHGPYU8NgUFqAIBFGvRcn3pbiZYI9oBLyDC0E80k2JYfKK53cdelAFax08mEWlyvygliPWuL+KerRW1lFodkEBYAyIp6DPH9a0fGWvXeiW4+zybZQuCxGSRXktzdTX8zXN1I0krdSaTGirEp3JE3APWp7i2NrMFVtyuODUDIM9/wA6cZmMi7mJ2fd9qmxTYwwuhwFOKfAhLAkECpJbwspXHJ71JBLuj+cZ56UgTJbj9wkMsX38gBexr1e+mk17TtM01yYdTCCRVAwvTue1eX2UqDWrOaQZijdTs9ea7XV/EsOkfECG9nikWDyc7COTxximMpeO9Ok03RLG2uTE9wJcl43yOnTHauMtpnj2fuEO05x2NPvb06nqU88m8mSQupY9Aa3vCmlrqGovEbgQvCvmAFc7vakJm0Ir/TFt9Qu7CKNLhQvynDYNdi+i3kltDqEQR4mj/vYxXH+I9Yu9YeCC3gYWduVV2A7+ldzceLLHwzHbaTHCHUIpOWxtB/nWsKnKZuNyulrciMF4CF9Qc1z2oaoFuzFCchep966yH4haZN51q1u64X768iuItdPXVdWuAswW2UklicA/jV+3YvZFW5ea5ljlf+A5U/zrRt7uaWJJQp2DufWtOzs9LgYLcQtvHCleQBS38FjdafPNBvX7P8zHpxU88g5DktS1ho57i+vZCLjYYoAo5A9QOwqrZ2jXHhE3TgPcCQvHIDz1rJ1eebV79SqPwdkaY5JrrrONdJ8MR2z7YrloyTETyTVxYnGxqx6i1xpESz588IAwHNcp4rvWZLWJ2ZYAMYB4yMVvaRe2t1Zu8cTJNEo8zeeQeetcrPDHftPd3u4/L+7jU4K0pTsKJ0ug6zoWmpCLvy/tBGVukQ4x7+9dkXjmiWWNw6N0Zehrx9HFlEiqzbHOCDzxXcaDaw2v7/TJzLCwxJb5yEPqKcalwaOgdkRSzMAo6k1jeI9cttDtYppN8hk+6kR5I9a1Lyz+0oqxqN4ORuYgV514x07UpJTf6iYtiDCrHkhRWrnYmMTGvtYl1GZmKCOMtuC55rPuLyUoIkOFHSkjjYAORgVFPgzNsGF7VyM6kBdWKgFmPfinHDHYTgH3qFeDTlBLAAZPpUgXVmBU7f4eBS3eZLAyAYNVsFfu96dNKPskUKnJBLN9amwwuP8AXwntsoiiS4vraE9HkANQb2YgselavhqNbjxLZxv90Et+IFMD1mO+hdhELnM27b5b9MVJc6TpWpzui+Wl1nBdCcgj2rkZv3lxI/qxqS1vrmwnWaBgHXpuGRXM5LqeqsPLlVmV/GVl/ZWhvB56TeZMvKgg/jXn4znFd5471T7founB0AmaQl2HcDoa4VB82a2p2scNf4gfggU7cCcGkIYnmnxSi2di0W8suBz0rQwJbe0jmVmkkCLztz3qSeKFdvkIwjP3WbqarrKwURMAYu64qxcXrTmL5AqRrtVB0FBBAixgNuI3n7pz0qbakfyiUM46kDgVVbLNntUturTy+Sz4UqSD0xikVc2bfdNp6WW3E0zqikHhwTXpWrTxaX4ZnuYn2SiARhe4bGOh/GvL9GiurzUbSGJ2YQN5pxwVx1qbXZxeate3E9y7s0m1G7NjtQBm6XpkuqatZ2UA/ezyBR+dej+LvCWjQLcarf30ksyW+ERSAjMOAuOv61xPhnUF0jxBbag8YkjgyxUnBPHb3rc1e9f4ga5bQ2cE9rYQRNyVyM9Tkj37UyTjYLCa5iM8fOWChf60y5tp7S4NvKmJgQCufWtXUdKv9DtbaXztrXIZlhXhwg6MaxphMdksyyfPhlZx94e3rQNFy2vDp+qWtx5BTymGdo5I71a8SavaavfI1naCBEBy20AuT64rOedZ0Ik+8B8p96hRN7YzQMkjDMuwCjkVYsrV7i48pH2fKWJYdhXeaV8PVn0+w1Kac+VNbl5IzGSMke3tQM4ZZS0CKei5x+NTRTwqA7ybW/u4yPxq34gNhOLKSyRkm8srMqKcFh/+qsDdzk9KQjo47i9fw/cW2mviNW3zMgw5Unp/Oom8TahPo0Vj5qEx/unBjBLDtk/lVXRLmS31FNtyYIz1bGRnsCK7nTtT0S68GX0t1Fp/9oQy4zGgRzz8rr79cVSQmjlE0XVLNbfULS1mhu0PEewl8/3se9dTZQeI/wDhGj4l0/Ur1nikJmt5WG1kA5IHfByK67SfFHh/xHNY7tRMeo27kIsmV8wAcZ7Hr0z1raEmmW9o620aTrbBtzFsRKc5Pf3pCseTt4q1+DV7n7Dd3MkUwzHFMpkIBHJGRxio726vnihe6g3RRuIxGx5ckc4HTHNel6XpUl3E/iHUmjjupogsaqmxYoweB+PX8a4/WrS2Gsot1fTGV0eXYiBVhQfTrWlyTnriPTx5saW0MUxj3bZMgD2HrWRbtLukjjTesmN6ZwCB61avL+KeJYonVtyjPGfyNZy5Mm1WKt6iueRaLsZgW/YCN47dm3YBxg+1RXvlzfvXYh14BB7ZogiuLlZBEd7r1DHk/SmXISOYW8q4JG7IORipGLBeXMSoltLnccYYZrQsYNVWKS6ji8oW0gDNIMDLHAGO4rF2QbPkYs5B+XoAexr0jS9V/tTQ7mWYxxyNB5Ko79XQdRzx2q0BBqPhvyZReFPLguEiEkI5+bJ3EH9a7DS0e30K2SMuypuUd8AHArjb/wAUJeaclpBE6quxg7HG1lGGGPr+VdF4X8RQyW0FlNkzPO6AgjAGMjNbQcYyINTz5M9aUzs4wTXMXHiQ2doBNCwZdySODkoegyMd6rJr91AixhopipyzAYJXvxXR7WJlaR1ZGWrXs5RJHweRxXPaVfrq9zIlvG2yNQzMeOtb0Vm0RyoolOMgipGirkjmgmqkLy7mV1x6VZUE8muZo1TCnK2DS7KXy6RQuQe9KHx3qIgg0nIpAWhISOaQgN1quHxUqPmgBxgDDAqjLGY5MVqRc1XvIt7bh1FFwKoCEYKq31GawJNdgmmn0640zfEflKqmT9apX15rPnXNuYGBLho2UYxjnINWTd6e2lTXU8/kfKUJ/iDegFFwIbrTL+TTRbxLIkJ++ZB86qOhxWdNaQS2cbJIxaAbCo7+h9TUnhrxBPdFLO+vNkL5SNiAGJ9Ca3mg0m1mZZHVjx846n8aTA53X9V1C/srW3kxarBMrN8xBAxgYNclq2vyXd3cywMUEzDfkdSBXoHiO1syjT3rq1vOmzzMZIHvXl95ZJbuzJLujJIU9c4osMqwTB7tDdPlckk49adbsfLUY53k/lURT509CwrRsEVprYEZ3T7fwJpAXvDn+lXkmnFH2zkEyKcBAByT6DGa6q9FvYWE8+mzRTxQlYxj/lmu0k/U8HmsW/1DTNIiZ9GtRDM58l5DnAB69aJNTtB4buIkhVl3KHUt94kYz7df0oEzFsruRFngSZ4InG8spx0//XW9peiLBaW02ozk28hMgihfBJ9Sfyrkw8mwRxqC0nynPpXQ2VjFNEk8ySF4pEzGGJDAk54/Dt60Aei+HPEzR6o2mxWSQQs3yKseDjHUnvXeLdqe9cfbfYrSzinkeMysp2tn7rHqAO3aprbU2LxxysFkkG5Exzj1oGdd9rAHWqr3O5sg1l/acjrSLcYfrQBsJOcjmrCTHOc1kiXgEGrEc/yjmk0M2EkUjOadvFZazEd6lSYlhzQBbZPMQqRwRg1gy6Wi3G7ng1uPNtXIqjIxOSapSAiSFVQAAU4RZpm8igy46GlcRI8WFyKzJ5Jo5f8AVkj2rQWUt3qTaGHNWmBzsl0krbUOWNJweP4hwRUuoWMSPuUYz/OqUccglBx8h71tHYiROQADSxwbxkAY96jkba4HY1prGFRcelIRVFuc8ipVj4xipMGlArNspIj8njNMZSKtgcVG6rU3KKhOKVXXvUhQE0hgAwR3oAcDTgRQqDbQE3NtB5pCIZG+bionOV561LIpVyCKjb51AxyKYFckijzKcQMVGV5oAkE/FBnPrVc5zTCTQBMZTnrSCWqxY0Bj60wLnnYFNM/vVYE+tOAJpiJvP45NJ5lRBDml2mi4Dy4pN1Mbiot5BzmquInJ4qJqaZqZ5hJpDHGnAZFREk0obAoEOI4qOrCrujzTxZu0Rb0pDKRJqM9KldSp5pmMigCMsabup7LTdhoATdSq3IpfKYnNRtwcGgRn3l9cGd40kKBTj5aqM8jr+8kdvq1SXY/0t8DrzUWcipKE48vhaAuQDijoaMkAkUXHYVtoFCkYoCuRu2HHrimHihhYcab2pO9O7UrgLbHZco2cDNaprGyRIMetbG7IzVIQmaTcaTNJVEj9xpc8UylpgPzTgeaaBTwKQDgOKAOacqk1IsdACBeKXbUoQ0BDQBERSYqcx0nl89KAIqUVOICR0pfINAEQNSqrHtUi2/tU6RBcZFHKIhjjO4Z6VaCHFPVkHHFSgoR1puLC6GJGzHGKsLbZ74p8QUDOacTjvUjESIL3zUqqCai3UoYipGXyV24FQseKrmYgdajMxJ60wuSO+M1ASWNPPIpBgUCEK8UgBBzTyRQKYBvx1oEwHemMOKgbANUkK5x+vReVqbEHhwDW54ZuybBoW5x3+lZniaFhLHMOmMGjw7Li5ZPYmtLCOrDo3WmOqgZAquZAneqkssxORMAvpTJLeeacZsdTWVvfzAWkOO9OkJPzAlh7GqJL5uAejUqzGs+Mnp3q2isB0oFcle4EaFnPyise68VaVBlS7u4/hVaPEIuP7MYwEDHXJxXn8PliQeZ1qZFROym8VW09q3lI8b9t1ctLOLm5LBs5/SqtxKvmEINq9hVvSZlil5tI5MetZ8xfKPktJBjg8+lWItBublNwutg9GrSk1pIk/c2oD91rNvfEk7qYooWTPUgUwKd5prWUoRpUkz3U1LFbFo9yxFvpVQyzPy/zMe5605Zp0+7x9DWcmNMcLe1YkTrkZ5Wt7Q9L0a4QsbQcHHzHNc2zN1cYoF08WfLbr1qecZ1DPoK3pils0RE6nB5rXh1rw1ZRCOGNF91SvO3u5H++c1D53NTzhc9TXWdKuWAhnjD9soKZeatZvbtbSLGzEYUgivNFnFDTIe9HOFzSvbGSS4LB40UdAtVncRp5bHpVI3WOj1E1yG6kk0cwXNO1ginhYsYxz361QnhhVyqSqfoM1XMretAlbuaq4yWBDFKGEmDWil/c253pcSZ9qxmck5NTCQtHgGgDobXxF9pl8q6uSiep4FPvHslUPY6iwlHfdXNx2U0y70A2n1NROPs0uJOGFAHRWviXVLcFZLsSRnuRms3VLmW8u/MWXI7bTUUEiyLlcGopJCk+dpP0oA6O2t91mqS53H8cVBdafGkBbcPx4qmusThQMJ+VXW1a3eDY0e5/XHWkBnW8duZRmp2vrizlxbsY1HYiq5l3SK20DBzxXQPcWk1ltfDemR0pJgecWUEt1qUdsJCrFtpI7Vt3vha9j1VbWzbz9wzvf5a512ezuXMch3E5yalTWtTVwwu33DpVAdjJ4Y1qBEjc20sRONiHn+VY/i3SY9OuY0t0jRWAzhu/rUUPjHXBAYWuhtAzu2jJ/GsG7vbrULgy3Lsx9znFBR694F0+V9Al0u8QCWNvNR1YMcnkciu8s5Q0CpO6rKnykE4r540PVrzR7hZbedlTuua3D4qfzWuJ3MhPYsaTIPXrrQtIuP8AXWloxPdcZrmbjwlapltL1mW0/wBiOUEGvLL7xDf3EztbXk0MLfwI5xW74U16XR9m+2WdJDgknkfjVJgTL4ivLK6Ol6iJLqCCTJZRhjjvnv1r0W38R6Brlqtr9oMDZyI5lINZmr+H11g22p2DCK5j+YcDD+xzUptbPUYha6tpaRXBGPNReP0pgReIfFx0hki02xN3hf8AXgnH4V5Vqs8ur3pu0imwxAYkZCmvRJtC1LR98unzx3EKKf3cg5A9s1S8MzaZcWF3pd4RDcTuScjHP1oA6vww93YaTa2mpbRIU3IQcjFdB9Kqada/ZLCO2lKSiNcI2OcVaDgUAHNGaazDFN3HsadgFAp6yLGcseMVAXkL4GMetDRRRgedIMscDPepA8o8UxXepazLHEmYppQu7NY+raC+i6nHZySl43AO7bgCvZrqCEKrxxoUU7nOOgrhfEslp4j1qz061uRujDZdRuGT27UAcla6ImpX5tLUtvAzknisKe2mtbuS3mXa8bFSM+lenaHNF4egvrG5hEVxEpZZ2HD+1eaXNw93dz3Epy7OSTQMIvKU7pBk9q0Le1aWJZtgKg5KE4NVbXTprzJXCIP426Vfto5LSAq53AdwOKiwI2Li2uNbsvMsrFbeGzXzGk3AE47CtDxsYNV8KaXqXmBZigj2Hqcd67DwX4cs28PNcSNI32tSGUtworjrrw/bTatc2NxEsHkDcgDH56Y7nn65WrsVzPbmO4t5TFODgSDnH4VJaaLd3msnT412tkncRwoHrUNwjQO9s33kOM0Aeh69Zy6d4GsRBPva6cPM45LFuePypmh+Gbq1c69rzlLRV2p9p5Y+mab4Uu/7RTSlvZI5IoJgnkMTuxzz7V0fia+tte8UW+kgO9pDy3lr39D+XWnIk4bQrBtY8XnTraXZaTSF3XoGQcnFetMLezunsrPS4WS2TIUHBP6da4i8vtM8NePrN4k8tPKKMFHc9P61q6lqEunubiymcTT9d7Bm20riubGsXT3tlY3dlCV3sUkjxyvsa56znZL+6snifa/8PqareE9auoteNvJK7W8hZ2UDI3etdFpV3dTz3xhjhDI+cuMmi4zmrDw266/JezqsKQMNsLHnnvmueab7b4kvNQu3JitB8oXjt2q5qHiGU6ldsTl3OMjgAfSsTULqSS1cR4jSTClQOoqbgMW/ngmadXxDecuvYGnOdy4zWTNNIFit3UL5XTFTxy+eVAPB61Fw5Se9TbZO3p0q7o8n2G7ISR1hkjyeepqo0uV8qRdy0rS+WAUwp9Sadw5To7vWZLLTzJDI3nTPjLHgViyTrcRE3EryICCwJyT9KpSTGQ7pmzjoB0FV45llzs+72Nae0kHKJcSo8rKi4X0rPlhbJK4PsKuXOWmARTnGOKY6m3UscGQjpnpSvcu1jPTipUfyzuX72MZqDBXr071IjfKKYDWkA4/i70hwVBFNxmWrJt0MSkSKMjODQBUJJOB1rpfBNqz6leXDD/VQEg+hrADCNgyr8w6HtXX+Ecpot5cEf66UAfhUz+E0p6zSNTZ83GMUhQ1JkUhPT3OK4ep76MHxRE9zqNlaQrlzBhF9Tmoda8P/ANmQWskRdzICJdx+6a6LXtKnsNctNTOxoYlUE45U9jVHU3u9UnW0iGZJcMo6AD+lehSh7p4WJmpVG0cpJGTgRr8wGMeta1/4baOwtJrJGd1i8yZyeCe39aYNNnh1JrKQbZgcA+9dHJqYttKNpK4DAg7Qcn86t6HMmcGzgorNw+SCO4qd4SLLzP4z90etXdatUDieWQfaHxvRemaikeI28Rb+ACsxlELgc1Pa+VHL5kp+6CVHcmrEdk0yglSvz9Dwa6vwfoNtdSXH9o2/mW5bA9EPTJP40DZx2nXk9rJMbYKZ7gFB3IB9BTJZJJWjhkb5YsfIeoNbVzcwaZ4pvXsbZVgg3IgHIz6571ioslxdtITueQlmPvTBI1dD0OXVnlkMhgtof9ZLgdfau7t9LvtH8MWNjpl5HNPcXDbriIfcB549a46xnaLSGs7mf91NN5cCq+3Y2fvN6ivUvA9hFYeHI7Q3EF3LHKzO8RyoJ7A04ikcnrPg1xq1gtxqU15LdQu03nNsGRjHrhcnpW9qnhm2v9QtdYFqDDZWpt2tXXOzYpIIPfqKi1fQdQ1LxDFcNd/urljE0Y42wjrj06CugtLsaJp0en3cr3E0YZIowMvIueCf8apoEzzHT/A+7SX1DVn+zxPC08KRph3I6ZOfSqC+Erp42klsniP2NrmN0PEgGMn9a9n06xVh9q1CNAVUhIs5SFfSp9mn3UEkkAjOYmiSROgB9P8APNKxVzw/wnpv9oapaC7hV7EviVz1TPTI64J4Ne0xQyWFgtnCqhYywQqMADBxx+NYOszaZ4XttPuImU/Z2Fu4jQbyrdSQOvSq8PxM0afcq2dyJScQq2MSfU/w/jSJLNho0WmadpUqWhbasvnK6jIdyOT+VeVeJPDzaBqctobwStKBLGNh6E/pXpKeOJ4dTe21UQR25UvEYEyCcfd3d6zYdTsn1BvFOryxiK9heCGEJnaMdPf6+9MDzJ4WtrwwB45XH/PNsjp61Jie13yH5A/ysO+KS4FhFMDYRyou0bi/BDc5x7UxpfM+VmLDuDSQzW0L7IZZomhaaWZQkSgZJbPT2+teg6bY6ntg+2aXaw29rzDFNcKFz3LYyW7cZrgtLkt5rJ4JLNt0IKrOi45PQV6jb6la6VpFm2p6P5F3MVjKQruDnHUeh5p8oGL4+vdVaytIzfTNbzhlCW8YVCw6e5GM/lXGzGVf9Kt0NvbuohlcfxHoV/Gu41O6S51u3vLSwnW3gRldpxggnoQPauMhmnlKwMiS2qTvI24YDE+tQ5WE0c9EibnIPlc4XdVuJYIA3mOk0rj5dp+771ftJoE3STqhRMqsH3gD6jitHTdVtNGiuLiTTxcJdxlIflHyE8c56df0qG7jRlacV+0gmVYmAJUsOCfSjV45UvRIkYO0gqQmQR7irF14a1izvbC0ltzF9qkCwztyG9c/St/W9C0q2m3XuoMLhYT5sUKEB2x8mPbNUM42OASS/vYiHYgYAKitK3m+x28qoSohkUq4GSCeD/Kugj8Gatp/h5NVvJE89lIWCQHIz90A9NxqnqESWmnywXVu1kzXJZlc5LMqAEL+P86AMCaFluNhl8z5dwfPf6VqRyyRXUDxXW3YA2F6B+5pgtlaCOK6OzeBLETyVBOMGoI7j7NdHKRuka/Pt7j2rMbRda3vb0XaxyPdyzSKzhFJbC/T/wCtT2SdpTFdRSBIZlUyMuAgYcA/lXd+AdDtrKCTU7S5klFygEqsApjfPQ4rd1LRbPU7eeKeIZlO4uOobGAa05SGjlPCUttBripHO0bFCk8TjAPIwQfTNd+jg9a5+08PpZakZoRvieNFkJGCSFx1/AGtvBHY1ohrQtFEPTB+lAXFJbJvIDEL7mrXlNG5UqD79aVwK+2nqOOalMXtSBMUDIXh7iq7A5IrRAGeRUEyDdwKAKgGTT1BBpQMGp1C4oELE2KS4+aJ/cUqj5sVk2F7O+sXNrdfKiOdm7jIFICvqAaCzmupZEhcAhQ54JHrXk99dz6k5WQ/J5m4IOgNd54t1prG8aAXyBg24pwwCnoMEe1cFPPJf3q3MSbQWw79NxPT+RpgRRXTW+qW6QNjayuT2B711Vtr8YilRbNXfzi75O4fliuRtreFkY+cqXIyNjDnI7Vdt7aUWyT2x2PMp+0dwBnpQB1NrrCahPc2OoOiW6jfCenFcl4hNv8AbibbySvBDQ/dZfWt7TdOhvWO+4SLyYSpKjjHbNcmbMz388Ma/LEzAewzQMgu4vnjSIcsMgVqeGzbxzQtKCGVmdiBk8DGTWRcuIZ1iBz5eQfUZ4p9tKsEqeexVCp+6eTn0pAaerNFPqXlWUhe2dwdzdS/eqD/APHuqke59u1b+mWKXeg25lmMEsDyOWaPPBzjJzWM0N3ePHBCgZpmIRRxnHf6cUAVEPzgoeldDozSvdW9rJL5KZDu7qCeDwPepbnQ7fSb1JBOj4wBGoOQ3Bzz2q4Ln+0obx1jCvDtZ26BgD+lIRv39t9jsrTCu0Uc5wd+CQR6Ve08O2qefJxLsxjsB6Vj6dbz6ra2l8rEpHId0TPkADoAPWti007Uhf8A2gTf6Ozbirjlv8KANzcRRvIqOe4hgaNZZAhkOEB7mjeKBk4lf1qVJWyDmqgkFL5o7UAai3Oe9Wop/lHNYiynPFWY5G9aBm35xPel61Rjm6A1aWUEUIB2xTxUEsLAk8GpiwHemu4I4oArqwXrUqSgnGaqSH5zSqm4ZzVIC0+1s55FZlxC+8lDkelWSXXqaVV3nmncVjLX5p1ypGK0OtSvER9Krs216fNcSRKRmlCE0sbA9asoARU3GiEKRTZYQU3KefSrRjGKpyE5IyaQyvn1o3dPamtnNJ3xQIf5gApBJ82abtFBXA4pASswY5NQyOqg471EzkVDKxNMBGYZpu5aictzULFjmmImaRQaiZwelRncaYc0APJpAeaQU9UOQaYDgpJ4qaOJsZoQYNTq2KYCCJvSmSIQcVoRNHt5p7RRtzwaAMR6iK5rWa2QyYC9aX+zUboxouBilTSBTW4dKB+6cmq82lzxc4yD6Gi4GcBipkiDjNP+zSjqjVPFEVHNK4WGwxhetWQRjFOjhVsZzVtbZMcClcDJntUZSw6iqDRY6CuoFkrjGOtUb2x8g8gUXAwwhPal8sjtVoxY6U1kYDpTEVWBAqFxk1YZW9KhZTTAxNTIS857qKpeZ6Gr+sKBNE2OSpBrMfAHFSUPLju1M80qGJ9KYOtDglTSGT6LBJfyzrJI5QKWDbjx+Gajt3Zohuq94Y+S5nX1X/GquNrOvoxoYCYOaf2zRkAUUgGkfNmtGI5hU+orPY4Ga17G3M9qjKOBVJiZGKcBVwWLZ6VKLFv7tWiGigFp22r62Lf3alSx9VouCM5Y29KlWFz0FbCWMaj5iKmS3gVgNwzRcDKS3kx92pBbyelbgt0H3cH6U8wbRkgUgMYWkhHSkFq46itxYARnFOECelAGJ9nJ7VOltgdOa11tVPRaDb4/hpXAzPs/tTVg5xtNanlj0pGT0FNMDNeAqMgVAwNbcaBjgjiq0+lybtysuw+taRkTKJlHIpQ2TWg2moRgSYb0pBo04O7cNvrWnMjPlY2P7lP5p4gaMY604JisZGoIB3oYjPBzShadsFSMgNMXhqnK5PFQsp3UALKJAuQKYglc4C1o27BlAekdRGcLjHtTuBTZGVcmo1kHQ1ffDoRiqS2o807i2PrTEIXquUdmrQFvF2JNNMSqcU1IRg67as+mOwHSua02Vo7+Lb3ODXoNxCk9hPGf4l615nC7R3GFOGVuDT5gO1ezlZtzqcU24uorWAK1uD7gVPDNc3NqpVwM+tO8slCJwrN3o5gsYcl7HNC/lxOr9qS0nlWP94OPWtWe2h8pgq4NUIDlHSRQhXPSp5gsNGq29tJ+8zx7VTvtcuLhSIpUhiP8WOa5e+vzPcSKpxg4rOc4Y5pc8gsdHcSFB5ktz52fesJ9hLFUbk9aEfYnDDFNVjI+1Rk1DnIuw+Jgj7jHv9jWlBO6JlbUc1jvJIjEZxT11S9iUIk52+hFIqx0FpHZO7S3MwhbshNU7+GI3GbWVZUIzx2rPgvnaQPNtYjuRXS2EGkatCyy3LJcZxsU4FHMTY5h5BGSp+8KdBuddxFbd54ais7sRSF2DHCuTgVffwpdJEzieN8dtuKNxWOTvpgyKqqAR3FZxkYdG5rQuFhZ23OwIOMAVScIBxQUMG89TSEn1Ao3Vftkk2HZZtKCOcjmgDPDc4yPwprblGTV+PTpZi8kUPl452niqU7nOw9RQBDuJ69KUEDoDSrEWUsO3arth9lWTFySvoKdgGiAyw4jSXJ9eapmOVWxx+dbc8jrGz2shEQHAJ5rCeRmOT1osBOYW2g7GP0qIqVPXaKjEsgPEjAegNOVd5AY4HrTAngneF9yyMfxroIrmyurcK0SO/qy1giFP4XFPUmMYU4qbgbLWEcaFolg/A1iR3TwXDM0at6ZprvNn/WfpTWbnnrRcBLm6M0xcRqmey0kdy0ZyMj3xQdrdBTQVJwelTcDRk1FJYeUXd6iqiShjjzKjIDDCjFOW2fbuyv50IDmzIsiB2PzU1W5zUKnIq8ET7IT3FWBVaQ7xjvTySmVIHNR7WK7gOKmSMyIMKMCkMapbAAomyFHvUqwv2BNLPCWwApGPWkAlvATAVcfStCxuzDEISqsF6Z7VBHI2wK3amqoWckd+tAHa23ii8tbVHF8rbOPJAxx7V3mi6rHrGnidPvZwR3FePWtwsDhniEijqK2dO8Qz6dO8tkBGHOXQ8g1SYmj1Qgbtp6dxWdqegafqsWGiEMo+7InUGszRPE7amwiuokjl/vqeDXQl9vStDM5lNT1TwrPHb6in2jTycLKg5A/ya6i3vYL2BJreQOjDORUc5juoGguIxIjdQa4++t7nwtdLdWUsjWcp5RhkDpxTA7ncaN1UbG/i1O0FxbPweoPUGpmLr96mBbEiorOxwFGaoWaNe3R1CdWwpxGhGAB9KeLmEcSNkf3R1NOW5eSMpBGSMY3NwKhoDm/ElxepokhRyRdTbB6KPauM0OwmsfF9tBMMFW4PqOK9E1uyI0y2Gc7J9zewqG7tIo9Qn1UoGaOH5SexFKwzkfiRf8AmXUdj5KfKd/md+nSuDIwvHer+r39xqepy3Nw25ycZ9hUMLpEfMeISY/hPSoLJoZ76S1SASHYDhUHvXQSWN7YC3ivnjkSdeEHJX61nG7glWGRVVGU5KgdK377UYdXubbFtI0VvGNxHIJ/Ci4HQaX4xTQ9GXTZ7eS5uACIEh549CccVmateXNxHazSpHa6nJnao6lT61zc+oTaXqMeoW0KlcEESCobXVLvVPEtldXZyHk2LjoM9qYHX+GbJxNNqshwyDyvLiGS3vmuP1W1Mev3Xmxsnzk4bqAeldpNqMXhzUr2Nw21gZIlGeT/AJNcTeT3N6bi+eJz5jbmOOntUsCKKZkuA68YrsfD/iCw0VmnaKRnYfMUGcGuOtoPMXO4/jVm2kZRuhkKvkjio5ii14ijjufN1mVpIprmfMSN020t1ezGSJy4ZygHB61I2j6hrzQ5mi8mP7zMeQPp+FXTp9lpfiCz+wwtMiLtbPIJ9aSZLR1nw/jt30iaaZAZvNIJPUcVOl1LaRazcQkB8ZGRkZqxZaXcwLmBPkl+YDsv0rQs9MileZHlMiBsMrDgmtUhWPGHs9QlkYtbuvmHO9hgGi8uIXSIQj5Y+o969H8YIJbY3KOALc7RGfvEYrzy8so1sYpoVI3jJBOcVEijCMpZ2cnccd6ZHOykKjEZ9KmuLeRUbZFtwMsSeKzlPFAGysm9clgBjGaz5o1S4YsWZAcZJ5pkZxnjj9KY2S/NNAXojE7pFGcljjOelTAeXI0O0Ap6d6zcqjDBwaHmkjlDq5yfWhoDQaRlYlSQfaq7h5yQ0jAsetVXuXeTdu69RT1uBnDPhfpTQFnUEs4VQW/zsFIdjwWPHNUXHyxj0WpXQsgcjCt933FRkUwHHaZAQPlxStEsmNzbcdKSA4lB64GaV4pZmMjtye1ADXxnHau40iEW3hnT4l5Mw8z864sRBY1LsOfSu5065t77R7Py7lIJYk2YkGBkelRU+E6MPJRldkpV4ceajpnpuUjP51JCvnypGvUkVrQ6vqyJsuLeG+t+hLLu49MiqupX0E0k15aWf2VIYCCMYBb1rnhH3j0frCtdF5E/trUprPUERFVVxsOBIKfc2UcHiK3W2jCGG0O0jtzxXER6nOLmDUo3Cyx9STjcKS68WXk+oSXjSCGUxiKPbyFFepGrG1jxJx95hdaqsEhuXTzblXbJxyzVm2lzLNJJJOiF5OpxVNpcHGOp3EnqTQlw6SjFc8pDRpzSQs5by0diu07hmsmfcZHO0+SjfMQOB9asQ3DtdK4AyX2HIzkmpb1JdOvruwOyRXxvI6e1RcBovZsZV8OfbrVqO4vp4po4p5lRl/eKr4DfWs5lZgrIPlzkVNNcrDFCkfEy/M5/lUlIdHb5tQFk8soSCpOD1oigK2ckxjBSM4YmmO8Eqs2MSuevtR+8aJ4hvKPjcF70XKuVZNrY3LuXPIz1Fd3oWuXulaDCujXKsGc5tGjLksffqK5CCAmRIVjIz0BrvPAV9YwxXdnvELo+4hzjJ55rSLM5am/p+vazq+Y30s6XtO2eZ+So9FHrWvAtppS70M0szdX+9I5rGfxHayXT3EVyGso4z5h8skZ9fpUttqUl5au9nq8EgJyFSLLKPTrWthJHL+KtY1fWdVbT7e3utLiWFpJ0mfa8qjnkDjH+NY9jpt3bWlrfSX8kN7zNHE8hyEHQ+1dzNeaGjxvJcYvbYFGe5b5sdxnoRXKeJdTsUu5LmyuIrmWVQGcfMFGAAAPzrKQ0S3Hi59QntBc2URmUAq8aAmSToP50usaZe6bZare3FnZhioMkwiI3EkDC5PPWue0Ozk1q7gEMot5LZSxmccZ6jH+e1Gv3N5M7W1xq73kPU7WyAe1TyjKNlrDJPZqfOaK3ckbudoIwand4L69CRGWK0+ZlkkXlf8KyCRFzH09a6PwXY2Gt60dO1DzD5qHy2D4Ckdfr2q0I5q4INzJ5bbos/IT1Ip8iMlpATGi7g3zKcljx1rW1XTdS0iWGG+06byoy0aSMnXmo9PaGG4FwWiYJuZYpI9wx9O1AHY2epQQX9pqBsLm4s2s42ZdhCkoAM+4Bqj4r8aXWuNBb/ZWtoovn2sMFm7H6Cp9A8eeVo8mkz6VFMgR1BWbYcN2AxUurWEGu6HpeoCRIrqdZS4ccbU4+8OmMfrQByP8AaM/3jdXIlHR/NPH4UttNNtE4mbaJFVi7ZBJ74qukSo8m4q6beGHSrNjama0Cwou9mOHHBqZIC1r0kcF/FbwbMKFLbEwAT296u+LPCy6Rr0lpZXkkqG3WeTfx8xzwPyrDubeaSaQT+b9qJxtIyTj0rUuLa71C4Sa8vyb4oFMU6/eHpnp0xUWA6XR/F1pqGgWWl6pC6y2UkckE8RLGQgnPH44rqfAmjwpd6vPJI9zCNkESzDIKfe7/AFrjvCNg1h4hglvCLZYgQZU+ZSTjAxj/ADivWFlxzG6lfUCtEBZMCi1+zMA0I4CMOAM8CvLfF/g3WdU1g3kMbG3lnPlxDLsgbGWOOAM816X50nrSo7MeaBnmmreHrnTbnTLq/KMShiWNTwCozn9elZHiPwzc2OmQ3aSFpbmXZ5cS8MCM5BH1ruPHLZfTl/us5/QV0VvDG2n2RdFJjjUqT1B2jpSsBneFdMWw8MaeWgMFxJAPOXux9W9TW0FBNL1FApiLlttQAVcjso5Wyf5VmLKoI5q9bXYBwTQMujToP7tKbZewpwuRjrSG4TGM0gIXjXmqkhw1WpJOOKpSH5qAEJprc0hNR7jmhAI0ZzTSCtTghh70xh2poBm71Nc/cI0uoKly0fkwZYln5IPT+tbF46RWzMz7T0XjvXHeIrW9kgjkEb7shi+OV9/pQI5rWtOsEvEvra6knifc0y7ssD16VBqF7HdwwxpIqeWgJLdTiujv9Kjt7HTr+W0m3BNtx8pwykf5P41zWqWkEkQNq6K0gKptHKjjgipAp3yosVpKEdJlVlkLHPTpUEbmSJYxMUiLbZQvp2wKkhtNQMTeaCN48pdxJXPue1UntjBHbrN8jM5Jbrx2qkwOkubKaKC4lsHKLEFfZ1aQk9D6Vg6fPcRar9qliJEZY4PAz1rZ0S4ksNNvJNhihyB57ckjHTHfr+tYNvOxhnKv8rc4x0JNMZXuW3fvZUAeRmYsOhBPFW9vk6coaMF5PmQt1FZs0cvyk8qX2BuxNaM9pL/aIgQM02cLk5GKkDTi1CeTTPsUcYjRf9blcs2eQcVm2N+kNx5zTOnlxlUKjOCa2Wtbm8tfs7ReVcsUiBU8ZY461d15U1Fv7J0xY0t40BBSPBZk4YepyaAJodTW50g3DoWRE/eyFdpkI6Aevekj1+0n077JHaFEXBmZv41GeMfXFZE1rf6ddWE97E8dpOwQRSHg46Aj8c1ratpUKyX00MyyvKqrgD7p4yfp1+lMDHk1VroBIUa2XdjEchwT613/AISvXe3ktmP7qAnfIz5yT2rz+G2jhsEniJkeCYF1CnDFun8jWnpV040a8jQurXUvGwjK/wCcUCN+Nv7Q1iSa6+a2tOY9/AJPGcVvw3kN0xEL7sdTg4qLRvCpt9FV/MZrmdAS0h4B7VsWujwW0WxODkk/WkMqBCelPETelXhagdKcIiKYFRYWqVVde1XFXPBqQQ8UgIEBHWp1JHQ0jRMKawcdKBjmkbPWnLKAvNV9zd6QlvSmAruN2c09JMVXOacDTEWHcGkWXBqLtTfwpAWWlLDGarydc0uDSMARzQIEJGDmp0lI71U6UvmgDBoGaAmyODUchyKqpOMUpnHrQMawHWmgc0Fx60gYetIRIBS7SaYGqRX5oAZ9nY9qPsRParSv61aVkMeKAMySyi2gAZqs1gpPyuAPStzyEPSmPaqBkUwOfawOe5qaPSVZcs1aLJg9KsRRrt5FAGWNHUdDTTpwXtW2AmKQmM0CMM2RB6GlFma2gqE0vkoe1MDKFqB3NPEWOmfxq5JCV7VGEJ7UrgV/LOM4pQxWpmUioyopgSRSfNzVvIZeKzc4NTQzEcE0gJpoQ0RwKoNCB2rWU7kqrLGuSMUiimq4qZZCtRt8pxUZYk02IurcAL15qheytPJjk49KcCacqgGkIo+RKxwFI+tBt5R1WtAnFIWzTAzHt2IwRioxZl+4rRdSx4pgQqeRQByviS0a3it5CRyxXH+fpXPlcrXbeKIVfQzKfvRuCP5Vxg+7QMibhenIpq889akYDNRgbXNIZPocjJrOwH5WU5qS6XZezr/tk/nVOxOzWoR/eOK0tUXbqkvuAaYFRvv4xTu+KRlzhv4hQT0470gYj/dNdb4YhFzpD8Z2SYrkyNwxXZeBvnsL2P8AuuD+lNCNmKziUfdqYWsZ6rSSrNG2V5WkFy3c1VybDvsUX92n/ZIim3Zj6VF9pPrT0nJ70AN+wIOgP50w2EZbJjBPvVzzAe9HmgUBcRE24G2pAwB5FM81aTcGORSAn8wY2jpSKoLZxSIo6k1JwozSAcSFTgVCxZh92kaZfWmGcetMCNwVOaaXyKe0qt1qM4J4poCWI4cVdco0O3FUAcVOrgpgmkMYIwHqwHbbtz2qFRhuuamHShsLEZRfSoXjBOcVaGM80/bEetO4NGWwwcUoNaEkceMAVUa2PJWncViMgN0qMx81Ljb1qFn+bipAcFIpwRs5xSo4qUMDRcBY1QD5qSYJ/AetLjNN2Z6UXAiWFQ27JokVT1ocspxTdwPWi4FWdp1LJCcJ615zqcMlrqM6OMNu6V6cGibjfXCeLrYQamjoeHGKpMVjotBuFuNKRl/g61oy7JI/l6iud8Gzs0M9uT8wOcfnXRMoDMFUgjuRSGZl2PkxWcke59pZQD1BODU+qefcJLEoCOB8pArmtNvJHkYyMWbOMmgCtqXh24k1dm06DfG3PXGTSz6FfBAs9oA3+w2cV1tqpjjWSZsLS3WoadAwleYhfagqxxFy13Y2pV4ITFgjlOawkYivRNbttO1TSjPA28KcnjtXBrBG8myJ1HoDU2FYIRlwHj/FxxWqdLa4hxE9ruPo3NZt1PK+Y5jkVHveCAGFSvrmnYZBKktrMySDDA0z7TJG4dG2sO4FRM8kzlnkJPuajY4OKloTLr6peSkbryU/jU51bUhB5f2yXZ6ZrNjC960bN4GJWQ4poRT3mmnmrk1urynyuR60yXTp4ojKQdo65oGUQ0sbb1HI6V0uj+LGtFKXltuXjaydR1rATaR8xq1HNHGmPLVvrQB0lzqdtrEbw2rrCzdd/BzXK3entZOA0m/PQ5qORkDltnWopJGlfJNAGjZRs0eUXdzUF1CySncMHuKS2meE5V8e1NmuHec5PFMCJpj0BIH0qIru/jFW45IYkJ80g9xtqDzYi24gH8KQDGiCqG3CkEn+0BTHkDHjpTM00wJW3f3z+Bo8zgDJP1qPeB1qYw/JuFKwCJMScGnGRc84FRBeaGA7iiwFlAvZxTTGC2c1XDYp3msKVgJG+XpQN2OauaXbLeuVc496s3lmltuQMhx3zRYDiEikDj5afO+MpmtBYfItfOnOyU9F71QmWNl3gHPc1QDYW+Qx+tW4oZ4cL5TEHuKrRQF4y69q6ZY/OtY2j+ZgMYoAyN/l98UGff3zW3HaWyIGvUjTPpzT9WiWOyj8i2jSHIAdxyfpUDME527sKKiYjOalfYGwSD9KaYlY5FOwAhHXNTI+7oRj1FRi3YDt+dWobfC5JAqGAz96P9W7j8a9I8KalNeafsumG9M4wMcVwaCIEd/pWhp87CYRRyMnoB3pxkQ0emEd6R4Yp4TFNGro3Y1zm64scSSh8ejGujhuY/sol4KepFdKZJx2p2N34TvUu7AlrN8bkByD9a6vTJYNXtRMlxKw/iRTgA1Vn1ZZ5TZ21qLtn+U7uFBrjLnTtQ0C/T7UZIbaY/OYH4x6VQHfzXOnWRPmTxhv7oO5vyqIa9ZjHkRXM6+qRHAqaxtdNFqk1nErBxne3Jq4jY7L+QpAUotY0vUc2rOwZuscilSa4nx7rjwzNo9ocRDBZ+5HHFdD4x1LTbOy8u+sppHYZimjAAVuO9eSyEzzGRmJJ7scmpZYwrgZPelaPEW41IyllQL2OasraveX8FpH/G3NZtlmz8PtGj1nXJYp0zbrH857jPSutvfDWo+FhIbLbfWcjbzG/BX2FRaOmmeGfFcccTpCjW48wluM+/511fiR5r6xe3t2CoULtL6Ac8UMhnj+szSajeSO4WFPuiMH0qxY2rx6WJFHzxShlNTafbQy6Fe3T/PLbEnHY1JFexqtvcsM27nZIo/pWaLOzaSzvrqB1aJrhoRsJIyp9a4vUQ9re3FobgTZbcxVQAD6UvisvaajDc28hjlKgLt647c1oTeGZX06KU3JM1xg8DJY/WqM7HHzShXKICFHoalsn+cZHyjvW/qul21p9j03cPNkb94w5JrN1Cyj029ktYyzIvUt1NS0Wi/aXCtqK2YYJ5i/Kc85/rXVz6X+5gjg3B1+VgD1FYrWn2vw1FPaWyRuOrv97Ga2rXX7SLSolln23iLtJxmqiSy3LqN1pAhC7m8vqrEk4Hr+dA8S2yWtyqSF5p2LBUGdnsa5oa3eS2ckhjLs2VMpHAqo11JpmkC4XYZpG+XvxVMCXWdVkuLwHd8wQDYf61j3F1K8axsFVAeABVaa5aVvM3ZY9aY9xuVRjpWMtS0x128clsYXOGNYclv5S7AOnerV1JmU89KriVWz5m4+m2rQMjL7EGAcZpTLmLaDwetMZl3bT92oiwVcDrViFY5NCsd4Y8mmfWpBjaR60gGk9cDk09AUZXX7w55puBu4oMmDjtTAl2SMxlYg+uKUYJx2pYlLxNz0qMYC5JoAuwobaeORScYw2B2pkjKZ2PUGlilgeEksVYDC+lQbsLjqfWgC7DZtcTRRIu6RW3BOxrstFskmUebEowSCoHANcbp9wi3BWQuu4YBTrmvQNEXbBH7oWP1osIxPFN62mX8cdlM0DFNzbCRnnvWPP4j1C8s2hmuB9Qgy1M8Uy/aPEVwMY2gL19BWNHlXpWK5yyHkSCNX7HIpiyJvBkGR6UTCbO6RXCkfKW70+2s57iVVjj3OQSFz1xU2JbuLC0ckxVk3k/d5xg1aVIt7g8YU1Y022tLS1u7u8i37G2iMjJ5qmrIUkl81OQTtJ5FUCAxxhWVD945prKsXLOC56nvSrJaywSkqyysf3YPoO9RzwuhXeuM9PcUrADPvj2Ken3famT/ujlsFiOx600siH7jbuzAcU+cq0Qj3BmLdR2oaKLVwi2lvCoUGaQZ57U+SWNrW2ZRiYnDJ3GOlVbkb5Qck7Rjk5qS1mQKUZE3AZUY6ipaAsujXMoaPK3C/cGeMnHeqqWl35syeR++Qk53cE1uqqXaxJDsgjYYf1Bq1JabbidntxEnlBdpP+sI5JxTQkUJrq5j002NzOhZyFyvLIvdT9f6VK+r6lp6+QtykeACoSIBiD7/hSG4hh0kTyQxzyvMU2McFR+NZVxcNdX8MqxuqJ1DMDj1qrlWOo063ttRgN5cJvRBvdZTwGB/Wuf1+109tZee0mSGFwCETpu7gDtVqzt7m7kWBUkjimJIkIwGA579ay7ySE/ZIw+5owxZ8ck56UIzsP8+dbJoIpS0CjLrH2Hqat6VbWF/ZXSvG63CxYQBiAXJwDV/wxZadqIvIpruOHzGTeC2AI85bB96ytbt7LTb/AGafeRXEe5iWhY4AzwDUjMh3lt53hlQb0OGB7Guh8K2theakTdagdPmiOYJo15zznvWJHZm7gvbySQ7omXjHLZ969G+FukRo1zeXD2sizxbI4ywL4z83B/CqQytqNjqXiTSry+utclkXTWACMuQUPVuO/HP0rhRHhyGSTb/AQMA/4V7Zpmix6f4d1u3uk2xTSOWTvswcAEeorzKXQ9Vey1KRQzWVhhHL8bRgED6jIpiM9/D+rLoP9tiz/wBBBK+cpHBBwePTPeug8HeKntmh0q78hreQNEr5+YFs/pmvVdCtodU+H8On6hbrClxAY5EAwfTP1rwnxDodtoWv31hZTGaG1xuduoPvQBu6jZQW3iG20XyBsSAjIyN7Nzk5+lZVxpE1lqHk28UjqqDEgOVOetU21q7upImuZnLKnl+aB85X0JqeW8nTTYg90fMjIVNhxx6VDAvaegTUYbuaTcOVVs5wxGBWW/2sXrQXAD4kIMpHGfWp7OVRPHFDyZ2RTu5AJPUV0NzY22lvfNNbzGa2hwfNbhpG/iH+e9KwGXAkkELBL+RlkYM6A8ZByK9Y0nxRpupiOAPtvsASR7eSa8lWTztMkESbpTMZMKuDsUDk/nWh4PTU28S2qQW7sS3nHzeBs6Fv1oQz2ISbhxn6Ecinhqc208gVHkZrURyfjZs32nL6RFv1rrIDizg/65J/IVxvjFs63ZL6QH+Zrr4m/wBGhHpGo/SnYZY34FRtMBTMkimFM0iSQTZNSJPjviqRUilBYd6B3NmK7yo5qYThuprEWQrUy3B9alodzY84Y5NV5JATwaqeeMdab5maVhloNk07Zmq6NzU4k4pgIQVakLHNOJ3Con+UFj0AyaAHFQ3Wsu7gu2E9zKQPLbdHh+CPQ0y38Q2EzOBcDC4wTxmq+q6jb6lDJpQlYGZMpNFzsYcigkoXuoandX8EKRqIFYK6K+4MfX8MVPq2h6XJM0klttuVJH7kbSR3J9a5XwjqMukXGbyYsGnSLazDKEE7v5jmvSZrWC6bfndn+NDSA4vw9Z2clpqNjc3gmE4wqkEEEZwf94VyV9bXaxyRPHsv4uHjc5BGMArXrUenWtvP5aWyGNh8ysMlj65rhvH8H9k6ho93aSOzoxGx+gUYbb755poDl57ny9Bt7VS7ecR5g6lQvBqa+0GWxtUuSu2KQqREjZDLwT+lWJ9b03UNXLJvSGRSTvXGD6fp1qa81ma6tbU3LJGscLAbF+b0zjNMZyl9cL9ghtF6QTFlbuc9K6iDSf7P8I3msT3IknkVUVTwRhh096wfIjl1rS4WHzO6Bz6jPA/Kt7WZ2sJxbvM4ihbbuK5RQx7/AIZqQNTw9o8txerrV5IkcBiFyIjztHIUE/QE+2RUZaG/1+W40W5KW+S95cuvyxg88E9c81Tg0q9vrWNbB5lSfIkUnIRM4Dfj/SqtlYNYXd5bmXzbSOISTSqcA+nHc+1AGvLJ/bTmHSkMuxC01xNJk4H8XtVHz2tZwoSfcsh3SOuAT7Vv+FdZsxol9ZwRnzBEN80yhRk8ZyK1tO0/RpITffbo7y4gzlgeM/Q0wMSG9t9OspVNowyx5JyzmucvFla9P2l4rd5TuRI1JyD0b3r0S4tZb26soI0RbmNmeRx0jGO/5/pUkHhDT/t8d881zNcxMctK3DcY6dqoDlm0/wAQLtZdYmNozKGYAqWGB2/TmvSIJd1rET94ryaq3Wo2liTbb0jYddw4H1NcoLvW5dTuLqzLXVoQAJZPlQDPYD/PSpA7ZWzTwvNUI7+0N0bVLhJJlGSinJp9tPP+884Y+b5fpVAXwMGrCDIqiJiW61YWQgcVIFjaDUUgGcGhX55pXwTnNIZWZQKOD0NPdSarsCtMQOBk1EcU4tkc1GetAiRTVqHBaqVSJLsoAuOM5FV2ibGMc0LcAnmnGcetAFF43BIIqIgjrVqWZSarO+aYyJwT3xSAsv8AETSk03NAh4kPc1Ikg7Gq5YU3cM8GgC/uyOKEkAPJqmsmD1NP35PWkMvrKuOtTJNzWemeKtRUrAX45qkabiqoIAo3j1oGDsCc5oNwqjrUEki4xmq7HNMRb+2gVGbvnrVIutA55pgXVuiDnNTJd/N1rPFPFSBq+duXk03zUHeqSu+ODTHEjA00BNPcKV4NNjlDrxVPyJD1qaNTGMGmBKw5oXINML5pN9IRbSYgU2SUk5zVbzMUhlzQA5z3pgamls03k0ATqy5pxIOSKqFivBp0cuDSAVmYGm+Y1TNhxgVBIhUZoAcJsU/zVYc1SYnrTN7UAGv4fw9dAfwgH9a4UdK7S/YyaTdxnvGa4teUBpgNcqg3PwKapWQb16ds1MQG4I4qPAUEKAPpSGVVbZqFvJ/tgVta4MalE3qmP1rEuP3YWTGdjBsVs6tIssNpcDnIIz69KBlfgj8KbkdKZ5h71HvJc80FExOFrrfhxJ/p1/AR1TdmuNkfC10PgS6+z+JShPyyxkGgR6PJECCCOKgCxR8CMD3qWScFjzULuGHFNEWM/U4XjYSwoWB647VRjumHDHBroIZ9q4c1k6jYZ3TwL8vp3rWJEiA3TdmqaCVpOpqK1SK4g2MMy1fisxGmBVSJGmk+YdKnMJFCp61lc0GRtITgVLIJQnTinhQDwKeVLLgVIGaZGzipII3mbHI96cLYrOCw+WtRAu0cUwK39mMQD5w/GkazaMHJB+lWSzKcUNJ8pGKQGVLIUbFLFcA9TTLsHPFUS5U1QGykqmplcHpWQk4HQ1OtyRSaGXnbioQ+DycVXa5PrULXGSaANAnAz5g/OoTdbP4qotMT3qNizUwLU1wHBKmoBIT3qszEdKb5jDvQBfSQ+tWI3Y1l+aI1LscAdawLvx6ltMY7K2Eu3gu5IGfakB3Q3VIhOOSD9K8j1LxrrV+uxJY4EPaLg/nWAk13GwZbyUMP+mhoA93nG5cVTc7eDXltj441zT5QZp2uYh/BIev4132geIbTxAjbI2SZRkqTn/PSgC3sAm3DNYfjC2MlrHcr1QgV0bJg1la/E0+kTAfwjNAHPeDrpLfWlEp+VxivQ5RF3FeP2Nx5E8EpOArc16lFcJdW8cqOpVl7GtJCKWooqwvKihSvoK8slut/iAMnGZOg6V6xfxlrOXH3cYJ9K80NhDY6ks1yyvC7dStQBt6rr97a2I22kac43AmuSF3JdTfvDlj3q34ivkklVLSffbkA+tZ+m7TeI7khR6DNSO51E+oxWmitDCuW2kbWFcbJM7uWCBc+ldPcT2JbdLIdvtxVGeXTI0Z4WcsfbIoYHPyPIPmIJ+lMa4kxjca17q1Ecfml0ZD2SsiVELZTcAexouBFvJ4pOh5FKRjr0preXjCls980xEnmY6VKL6URiMBQB3xzVPdzinUgJRPIrbldgx9DT/PncYaRiPQmqwJzTg3qaCiTdgdeaeJBgHp61CXUJxyaYH45pCZZLqe4NMJIP3h+NQ+YKejBzg80CHrJk4yKc5x75qFogOQTTdxHc0IokxuNI8ZUZOKZuI6U9Q8vTH4mqER4J6VKIZCucU9YNjrvIwT2rVTyljADHPsKhuwGRCG8wFACw9a0pJJJ4dpjRfcCns4HRVH4UK2Vwaz9oBmrE27BwKZNEynON30rVmaMxYRRuqqzeVEZMBhVc4Gdkjqp/Kl3ex/KrSamx+4q/iKruwLFj1Jqk7gEc0kRyjFTSvO7Zy2SaiIJ6DIpyxuf4DVgPSAalpkl7cFy4GVVOi1Bp1nczW8kqbNq/wB4da9Ej1DSptOkFvECu37gjx+tcjBbTwaZcyKNqnOF7iq5QOeRmyw44P8AD0pQztwZHA9AcVNb2rtGX2PgnsKgZdsjDawPuKkDR0kGOVpmYPsGQshzk1vyays/7trZUB7uOK5LcwUAHFW0mEo/e3D5oAtXmouJ2xDAy9iB1qpBEZBmrSzJCu0Yb6ioxMAeBUSGPJC/IwH4UpKnGDjiot4Y0juFGTWNgJlIBq1YTLBfRzP91c5rKa5A4AYsemBTfNnddpXaKLAdxe65DIn7y481m+7Gi1o+FdKutRQXF7fGGDOFtVPJ+v6VwWnTpY3azbNz+rHNdppRVLoS7mG9uCDiriyD0aKzt4I9sEKKv0qnf6NbapaPbXKcN0YdQakje6yqFhReWt3cxeUZ5IlPUxnB/OuiExNHA2uqjwZqM+l6rI62+N0Rxnj6e9QX/wAS7WKULaWLzp/ekO3P4VreJfAts2mveWz3El3H8zNJJuLD6VzGoXek3/hiQzbE1CP5RH0Oc1pe5Fin4q8Z/wDCQWcNrDA8MQ+ZwzZ544/SuXD4FRtwcUmcVDkapFlZSFGDgk4rttO0f+xrdtUnm8yXZnI6f55rhIlaZgiKWPoBV06hfPAtrPcM9sDzHnGR6ZqFqNl28kufsc2oXMThrh+GxwD6V22n+Ilm8AyyO3kugMTHJbINcLr+trqEEFrBAYbeFQoBbOa3JdF1C28EbXkiEbsrGMDDH607EHLQy3TRz2VrPI0UhKkA4DD3qxYedcTWemORHG0vOByK7Lwxof8AZAkmliSTcnLKckZHTH+elW7HRY73X0SGFYhIMu5TLqKGh3DxjYQR6AkoT543VVbPIFXNP1CQ6LbeXCJWhi+8TwDWxqvhuK90KXToZWbyWMgeU5OR6n8azNAsIF8MtFZ8z/cK9857VLQJHL/2fLO8mp3YEk33gAcDNJZacI5TfXsygk58vGTXZr4RnOmPJd3ZjCjO1RjmuOudOuDFIzOjQqcbs8tSbLRSbUJyCvmypb5J8s9KpyTu8bRk/u26sOtXJUt44/JMfzj3rPuFESsEH7sYwffvSTE0ael3cIhlsWDMGBJZj/SsuC6YXaMwd4o8qV6ioIZJFkeSN9rBetaMOgXAs2urmXyYjliO/wCNU3cdijI0ZkZowVjJ4HpUUjKAcdQKkW4iWNcRhQByeuarSSIxZhyCeKzsNGcxOSecnrmmZNWhGGbtknjNOa28tmDD6VaEUWDEDjknFMyVbBHIrZtooo0YkfPjIPpVCaFdgk4yfQUwIQmYvMPTNNB5poHOKUYDcjNMYh60hBpxwTx0pOKBEiSMnAPB6ikbk5p8ahyFNEq7VH1xQAkR+Vh7UqjEfJ+b0pVjfZwjfUip4osRs8jquOgJ60Aa0Usc1zHFGcrs+7XdaYv7qM+gxXAaPbyNexyhfk6ZrvseVps3tG38qCTzXUZmm1e4nb+OQ4PqKihXNwjH7qnJFE5kKQb33IFOFx0yali8tNPZ8YZn6+tAFk3nn6bLaTc7HLq2e/0qBJXVQ+4gIuDt6mqDSsCcHrWhBIbaMrIoPnpgE9R9KhooqSOj42ZGO+eTSwK/npujEgBzsPRvai1tJLu68iIgHrljgAVtadpMB0q51G4uUQws0aowwAR0waaArX96Lu5B8rylUYC96jJiEkfmttRTkVSlJeyhkwwlcklz0IHpT5Y5EjhkmOFkX5fXjrTEXJDEsatHEYixI2DgOPWoPNDSErtJIwPlxg0rrK6EL5jFRnc3aqih1DEHH0oYF0fubeQTFS7nIwfu1AqzqwmZR83QjjNPQLcQCAj970V+5NWJVZzHA0PKgIStJgjW0eGI6dcXc8Ym8oglEPU++T2qjc3dzZaobiaFfN7BjkFaebCONTCJ33/3e1XoQ32K1lfAfzTEXJG4k9OKgEYt3cQyStMo27gCR71F56RSRt9/jLr04rp7XTbO60nVjNCFFvuYzdwew/H0rkIY/OuI4s43ttz6VSZVz0/whdXOpeGLi0KlngcxxNNxlSMjntXm15A1rdzW7/66Jyrema29K1G+k8TQ20WoRqTKBGWbbExA4Bq5qvhTWmv72+vLVWJcyMInG1lxzg/h6VoyGcrBNJFk5wex963YdbWOAqLeFvNi8qXcuS/v7VPrPg++0zTXvPLD26xrKXzym7+E+v1rKg2tKscm6BW27eOQc96lgjsdG8H/AGrSNSNzJiSUr5axnlSADg+uc11+neFNHtFglWApcR/NvjcqQ3f1q9aozRBACSiqpcj73HWp7i4ttPspbu8mWKGIZZjQIo+JPIubKQtd+WgBMcJbAlcHr79elVZ9OWDSdQ0q7IW31JRIsqcMhCgtn6YHNef+KPEUOtasH0sSCFYjEhkGDnqSBXZ+NJr7VfCJbS5YxYxQrJcTBs7wBjYPx5NMpMg8K+N4rLwzdLqt0Xmt2eO3YJkOAOBn1ry2V54vOSbO+ZtzljksK0BFeR+HGtbmyKW7TrIs46qxA/mK1vBejQa/rkcsjwr9nch1kfBlbGRigZx+cYX2rY0JoopZpLiKOWHaAwdyuPcevf8AOuu8ZeCotP0271TzHNwZNzshxFEDgBQO+ePpXFaTpcms3ttp8cgjad9gcjIB+lAHpvhrwpaS3cGriWCeFPnjSHpv7cdsVv8AiSwg1HTmkuUB8hWZRj77HAGaf4dimj0K3W6hEd2g8qUhQN+3gHj2rVChsbucEGgDnPBXg2zi1G/mu7ZQYoI7cxdQ7MoZmPvnjiuxTRbK3uvtNvCsTrGsSbP4UHYUts3lNuXgt973q55vepaAzp1IY5qDaScYrRcxSj5jzUTpEo4qyTgvGMfl+JLRM5xag/q1djGmIY/9wfyrj/F7b/FMHtaqP1au1JCpGPRB/KmMjCMO1IwKjkVMrjvTLuD7TGApwwORzQiSuWpoqsxmjOJEZR6npUiSlgADWnKTzE+OKToaMkCmFuayaLJdxp6sSagzTS7KeCaQy8pxViPmuY1bxPZ6J5Qui7vL91Ixk4965G7+I+pNJcrBDEluw2xKwIZB65HWkUekXWv6NYy+VcalBHLnBUsOKJtV06YtBFf27uwwAHHP0rwWa4E8rSyHLscsfU1EZY6nmA9K1K10vSpP3w2u+WXfyg7DPpXPXfiCSC6CC4MgjYMHjPDGuYe+kfAlYuvuc1Gzeac9KOYC/c6j9qmaSQjLMWOB6mtDS/FOpabLEIL2YQofufe4rmpQ2QVFIrsGAYYzUgemW/ie41/U2tVkkSIjdGQcMCOuara1cGCW2eaZ7iaA4IlHyuT6e9cRbvIJFMchjdTlWHUGuvj8RQWFhDFd2sd1eEbpJW5AHY0JiMmUrZajJNJEvnTHcmF4jb0wetN1O+N+PNltlSUYBaMfmKsaprkWqq4NvGjFMblzkEYxj8qZp0kF9ZfY5wxmjBKybsZJzwadwMi5c29zAwbLAbwfQ1NfX8t1YRxMcozM3mHq7epqqbZ31GOOSQ7QQu484xTAZLq6UD7ofaAOgHtVXA7jSrqzGj3ELyvLLw7EkgkBcD8OvFYz3F5b6beRwwHZPKI95HO0dqsDTrm3vYI4o9i3AVGTOdx/pVjxLayaREI2l5MXmrznHOMfrQBz4unt4DDbg7JMNICMZIziuz0XUYf7InDKkCAcYGGkY84X6VxNlE1/exIxG0HoT1PpXS+F9HnumhvZoUjt4Zcpk8Ejqf5imB2mgrLZQJqd2Zmvp2G6LpjGcD9ae2uiKzeONZPtM0hYuTkKM/5FQ3er2Ny6j7QYZSMRseD+H1rIntJEdJYTGzNneg5IFUSay3drqN9PLLIoPzYU9FOMf0NEdtff2cYbW4thbqxLF4yTiqFzaamYh9jto3QqCc44J7Vu6VFLbWgimYlh/eOTSsO5StdHig1Lz7eRo5UIMhz/AKw9wB2H+NdFnNQgDOacH7UxkwapVlI71ULYNG8nvSsFzRSYdM04zcday97etPD8DmlYReM/vUTzx925qAyZ71FIEdST1+tVYB5nG7FO85fWqZAHem7/AHp2Fcv+au3rTN4NVA5qRDkVIyznBpWb5Tioc+9HPrxSGRySHNRGSnyKTVdgc1VhCmT3phkx3qJs54phYigRN5nPWl34HWqwOWpzbgfagZZEme9OWXb1NUhKRTllJFAGnHOuM7qniu0J2hqxgxxxQGZTkHmgR0X2lcdagmuflO081li5bGCaa0x9aLAWhPLySeBUjTkwnaefSs4zHaRmohOR0NFgNOFi6kn71Tb1QYNYrXDjo1NNw56saLBc1ZbsAgKael3wATWKZTnrT1l96LDudFHcqRnNP+0CsFLrAxmpluc9TSsNM2fOU00tms4XAz1pxu1HVqLCLbGomcjoaqSXq4wDVf7X6miwF5piOppv2j3qg9zkEg1F5/vVAaon561IJvesgT+9SLP70Aanm7uMComyG4qukualD5FTYCzFKQeTVjcGHNUA3NSiSkBI0anmoWiLdBTt59aUSEUARmzeSORem5GH6VwA6Y/z1r0u3kJfGa83laNLqaMt8yuQR+NA0HamkU84ppZfWkMo3gxC1a19+80izcdgBWZdlXiIBqzbNNJoOZZQ6o2FA7CmBHtPfFIF5zTBMv8AEak3gj5Y5CPUKaAH8VZ8PztB4mtGX+Jtv51TKXDrhLaZge4Q1b0nSdRk1OGX7PJCsbBi7rgAVViT09gQzZ9ajJxUZuSzZzSGUY5pWAkXrirMQGMVSjlXeOasrKOxp2AnMUSnKIAT1NJ0pizLnk1KrKRUgIBup4h9aaSFGRSpcr/FUgP8selSiPApvnIV3A0guFpjHFV7imc/wmkklUrxVV59p4OKaYiyzFRl+lQvOoGRVV7wlSCarNPnoaAJZn3g1mzZBNWxJu61FOu4EimgKYlweTUonI6Gq5Qk0szQWcBmuZljUD+LvVElrzWbrSjmuC1DxpeGVkto4o09R1osvG+oJxMkUgHTK4J/GpKPQkgZ8YFWBaPjkVwEXxJvIQVl0yGQ9irEUjfFO/RxnSoRH6ZOfzoA71rZT2qCaCKFfMmdI0HdmxWZF4vtdQ0eW5tT5UyjJRzyD+VeY6rq1/qshku7l3A+6vQKKQHV+IfFEUyNp+nEPv4aU9B7VyraZfpy4VV9Qc1mRyGP7takWt35cKQr5/vLRcC9N4WvYrMXIu4mXGSADxWQ5mgfaxrYnvb6FR+9yhGdgGBUd3dWF4kaPJsdegcYH50gKgmvIhuKMEP8QWrPhzUTp2vwyCTYkh2uGGMit5dKgvNBhfIR16M3TNZfiNbiLS4I7iG3bp+8jNNMD0+2vbXUVZradJCOoU5xTbqBmtpUZcqV5FeOaZfT6bJ9os7kxP7DNOk1rVZbkytqNxk/3XIFNAXJkaO6miIwFbivRdFjF3pqSq6o+ME4xmvN4/mUOeWPU9zXT+HtSnSJrVW/2lrSQjq8vEjRufl/vK1eW+JfOttUaGRy6dgfwrsLi4n8/d5rB16YrE8QzLq1ssMkYSaPo7cfnUAYVlHILYyGAMACcE1ow+KII7f5rZGcfw7MVSitdSstPlZ1WS3C/wALc1zwJJJ981Iy/fX/ANuumm2hc/wr0FVt038MbbfXFQHcpyCKnN7P5e3dQA1pXU4PB96cs2U2kCq7SFj8zZNOWNn+7zQAvyuDkgfWoihB4pxUqcMORSc0wFRYjzJnPbFNK8+1PVdxxTZB5ZxuNABntTTU/kw+WG8/5iM7fSq5XgmkNB1pppNxB9qkVMqDnrTYiLODzTgTnilkVVPXn0IpNpxkUASBjmneYB/Dmo1UEcn8qM7Txk/QUWAkJUn7grTTTo2t1lDk57AYxWYqTSdI2P4Vp2OpzSRfZxGhx0oAk+yW23cQ2R/tVCYiufLDPT5y6cvgE9hVVpZANymudoCZmkKHcjKR61EZ8Lt6Un2qdlIdN3pioShfmqUe4FuCNZSA3INaf2WFQBs596x7R2jlAJ4rbyWGTV8qAi+yWw6Qp+VL5MQ6RIPwp9FC0AaFQDAjT8qTC/3V/KnhCeQKRlKnBFMZQ/4SeUR7Le1hVvWqdzqN7dIwfAVuoXgVkI+xs083MhUgHrTEbVjAWstwnMQzgALnJqGXTZlbfPvBPI3DqK3PDUktxppBRV8s7cjvTNYZ4lw+CWOFIPSgDEgid8rBD5rDqKvW5tEVYr61RZR70yyvFsAHY4U9fWrV1e6dOhkVCWP8dMB91FZm0MibC46KvWsK58sSYckH2q35MbXAMD72PbPSrg0666mOMfVs1LJMJ1RE/d4Vj6nqaEtmMe+STJ9Knn/fly4G9TgYqNXZUCntUMpjAhz6e9IA6995p7ycYA/OomyOST+FSBIA5P3a3bTWZkRY5EKgfxDmsKF+e9WfOK9KEI7jw9rkK6kkRumbzCBmQ4C4r1KSJpOUGV9a8E0sJfXaW8kYCnuvWu0juL7Tdp0nU3l9YpTuBFaRZVj0IWz/AMYwnevAfE1vDF4nvo4CPLDfKAfau61X4gs+my219azWt267QiA7SPXP415hGWaZnZtzE5yetaqQlEh+zGS7zjiorwRxnZt5q/KNqFqh+ySC2XUZI1MKN/FyDUoqxal0a7tLWG6tWDs/3lHUCtPw1p1u63EWowbpufkYYI+hpNA1SCa/8u8YIpP7rJwFPpXpi2rNboFkgWVhkEIORTQmjzOHw+Z9Xj8m3kMSMGcHnAFdhr8K3NgAshCxsF2jp+NR2zppEszSz7mJIeNTkZ9RVGC9hupRCpmJ3biqjNMyaN/w+sUNn+5yZdwUknqMVet7iOO8uJppY1eRdqleDmsy3E0soit2WMHogXJqze2S2dvHNHH5l9I2EDHjPrQXc07iWHTfDd2s0ybip7471i+Cr2OPR72SZwAoypY8t61w/iKO7jv5Irm/WeYnLRp0X61nCSSSLy0eYsOwPFJsEz0y38SXOq6Y9u+nS/Z3ypkjOciuc1vU5rAi2gjMSsu4qx5BqXQtRnbQfsqbY2iYbmUckE/pUXjm0jtru1lRx88YBVm5HSs2K5yslxtBaRvmq6hiuPD4AGCxA+lZd3CrneauWSTwIm9A0ZOAjdKEirk0tvFHaRAsBt7nqahvLzbEAZpGB7Fqlvt15epDAudiZI9PWs7UAXa3gX7vC59M0x3IXJZOKXchQDZjAxnNX9S086Xd28czqY3UdB0qG8W1FkXicl2IAB6ik0MoM2yX1HbI61Iz55PeomkYhQSNq9iKejRsuW69qaAsCRDbsSApB4UHk1UlV59rllxjAUdqUOCm4sM1GZMUgIHhZBkiq5GTVmaYtFwe9ViaoBegpO9KFLDFBG08mmA5H2tmnM5JpbaCS8uo7eIZdzgVtjwlfDrd2w/E1M5xgrydgsYiyP68U9nLIRW4fBt2OW1C3UepBpy+DrntqMDKf4gpxWP1ml/MVySLHh1mdkXqq9PxFdHr0/2Xw3dODhioUfiazdG0saUNpuEnYHJ2jGKu6vD/AGpYi13iJc5LHnNH1qj/ADB7OR56+WiWQfdxgUNMptliXoprpU8LbOl+o+kYpjeElc5N+PwQCj61R/mK9jI5c81qx6j51kLZ40VUGFIHOao3EHkXUsAO7Y23OMZqxpbeTeAPCjh1IBYfdPqK6DE2PDmhXGrPJJbTGOaN1RSF3D5s8kfhWj4l0W78PSQW121rdIxZkMfy5bj7wPSul8E+IdI0TwzdhIt2po+TEF+eYk4XHtnt2rmdX0rU4XvLrVIQ9xMhn83fnaM9B+dICjpl9aD7PFfxpClu25dse4MT6+lF95V54fu74KklwL7G7usYztAHbmmm6gishbzWavLKoYP/AHh7im/ZLCPRxH5vk301xgg9BH6mgkhmjlNpHJJDcJNdLtjcjCMfQVQhQpK9s4xIuQR710+otqkuiWVoEt72K2PnRy2xyUH8Wa5p8S3b3JODIc/jQM0NGQW2qSExsz7fkx0zSQT3YvpPLKK7FgWb0q34a02XUrySJb2OHsQw+dx6DmtDXdOtrS6sII2DQwjDttwc/nUMZzEaiW9DTiaTa3/LHgn6V195pmm20drfLGyzSLvVWbcCfQ/41S0y2jhv5JlUjyThVYYIHeun1OAanpttcwQY8i42kPyADSsOJzMI0OGK4l1e2lc+YqGCNyFJ5ycfl61WTw/bXvhPUNYt5fLhsZTGjBMvNlhjOewBFekaf4TtBf2jXdqsq2sbKXKAI5P3eO+K0F8J6Zb6HqWmQB/KvNzurHIDdsVUAZ5T4f8ADEmqaI2qpJtmhvESONk4ckjpz7/pXQeJvFd3Zanc2dnDtt4nC3AlXiQ9x9K6v4aFE0a5sJYsva3GcMPu5HGPyre1DQNPv7W9gkhVftZBkcDkkHIqyTO1WzGueH7a3QqIbkxtKO3l9x9OKxrrwXY3uvTXc6b4XUMgRsAEDAFdZHYrawpDBxEihVXsPpUkcR53cVQEWVVQijCgAAfhVHUbKG9g2yQpKy8orjIB9cVqyQdxVcgg9KBHkeoabBpXiRbi6wlsUkkj3HiQjgce5JNangOObVfBusacowktxkv/AHQRz/Ku+vtKsr5IpbqFZBArYDcgDHcVyXw7tJo7TV9PdXtJbgLJEW4O0gjIqgOc8Saw0Ol6b4f063HkyBXlik5Lnd8vPv1qWXwXrGlWttLBGzX0c5laOJQRERyDnv0qPxK0Vj47ghW23+Q9upJXlgvUqPfNew3Mf7wN/eGaQHPXBk8V+ErJL2MRJOyPdqTgFVOW6Yxnb+tcP4ZXSbfxHBNbKtvaJfOY5pWwGUL0ya9H1Oymn0W6tbSNWlkjKIhbaDnqM9q8p1O1ufD98kjvbebbzbzag70jJ9+9Joq57J7jlT0PqKBWR4c8QL4i0lLlkEdwnyzIOx9fxrU8zmlYktoTiry4KD6VnxOCtTm42x4FJopMJFQEkMc1XZiSKTzgc5o3qe9CA4nxH+88Wn/ZhQfrXYtJ0+g/kK43VX8zxhdD+4qD+Vdc1UBIsnvVhJsCs3cQakWXFIDUwsiYZQQexqnd23KvEAMcbelC3GFxmneeW71SYmBUFKgeNuwqTdgVleLdf/4R7w89ymDNKfLi9QT3oYrlG/8AFen2Et5Du8yW3UEKCPnPcD6Vz8HxAa5uMXMbW8R6svzGuA+0MRwMA9fc0xpmU8VkM1Ndvxql8biJHzk5ZmySD0/IVkPJOvG0n6Cnm9YDk0gu2zycCpKTGQxvI/IyCKsi2UdH6HHWq8t1leDj6CmIWMfP3e1OwyyVjBBYg47A1IssbKF6HHWqTPn7ufwFPgJQuWx8wwOKVgLPQhSCeOtMdd0ZXnI+7TvOTuaDOBwcAH071IDAJkbpxin7yBlsc9sUqzI3T/69PWVD/vetCAiWbnGCKdnLZDGpC0e7tk+tQXAZAskeAc8jNMCcbs5yc1s6VBbzMvmOuY/m+bgMfQ1gC4fjNXLO4mWTZFII/M+Uk9MVSYHcLcrBaHV2G6GBmSIKcc/7Irmr37T4hvEkZsNIrbF7oo/x5rtbTSWj0+JJp45SU4YHK7eeB/jVUrpukRAzNCPNIO5kAKIhwatEHIpZNDcRQL+5uxztPJX3/AVvx6lbWtnNE94jPbxAITJy49MYrD1m8aG//tC2s/LS5Vlh3DACjAB+vr9azEjsjaFpEmE395R39aGUmamnahFceIYJLmeSC3ZuHyWZV/LjNeoW/wDZ95bXNvYNC6Io8wR9iemfevI/DtnDfXM8TLM82z91HF95znkCum0nVP8AhDTqltq4aHUvJX7MgG4EYJHP1P8AOnEk7qEyIqoybQoAAzmpSxrC8Jane6/bXGp3sKxxkhIQB1A6n866HYMVqSRhzmniXHemPE2flFRbJPSoY7lgy570CUDvVY7x1qHzSZWjz8wGcd6QXLzzDjnqcUzzh61k6pJtsGkPmbVZcke5qS3uRc24mQfIxIU/SgLl/wA49qkSXPeqIY57/lU8ecUwJH+9T0hkY9OPWm9TzVqO4VU244q76WJtrcheJo+uPwpFcjpV+B4i25ucCrz2NoYxKkaksMk1DLSMQTc9akMqbeTTry2jiYFB2qiwzUMokabBIzTC4NQOGDUgNNEkjc81EwzS5p6IDQBGEpSrYxVxYOMimmI96BlAxGgIw7VbEWDTvLx2oArxq23oakij84HAIPYdzWhaERtntWgu0jIUUgME2c/aM1WcFWKnqK6Zzxjiqc0CMCQi7sdaLgYDNioWars8BTtVN15qxDNxpC5pD0phNAh3mGk8w0yk6UxEnmn1pRMR0NQ4oFAyz9ob1o89/WoKXNFhk5lOOtBlJquDThRYCXeTSjJpgqVaQgAbNSorGhRzUyigBVBFTKTjihF3VYiiXIpNjIh5lSqJe4q6LdO1TrbAJkkVIygqt3p22rvkqO1Na1DHIoAggOJRWVf+EdNvb17lZpLdn5dYwME+vPSto25U8UeWaAMWHwfpqZDT3EufV8Yqu/hzTY/E1vaeXI1vLCWKM+fm7VvlthrI1Gdk8UaWwON0ZFFhmgvhrQxx/Z0J+uT/AFri9PC2c89uUEkcNzgK3TFd4s4Vsk1wkp26zqQ9ZSwq7EHbCWzjzss7cD/rmP8ACmteY/1caIPRRiqiEvCjnuoorVQRm5snF3JvyTkelNaYk5DH86hpKrlRHOydJiDT2mzVYGng1Lgiucnh8x3+VS2OuK0FibbmqVuOM5x+NTFmHes5KxpFk+QKeJVHes95znGaaJ8d6nlZVzSaf0qIyZqotyBwTUqTRk5zU2C5aiZ2GBn8akZSBzUP2tB3prXin+Kiw7jnkKmoHkLHmmvcITkmoWuEB4NOwh7DNRNxTTcJj71QSThujUWAmEuDTjKHXANUd+T1rF13xG2iMsSQeY7jKnPT9PeiwHQXUsVpayXEz7UQZJryjULqS+vJZXldxuIUM2QBU1/4g1DVQVupyUP8A4AqnFGzghADUgSrYx/ZvM+0ASgfdNQ7HHTb+dK+9GKuMEdaZupDFkjmUbtw/CojPIvfP1p0rmNCeT7GmW0VzfuVtoDIw6gdqSYDWnkPt9K1tMXRpbcm83mf+43esu60/UbJQ1xbsinoarjJ5p3A7qx0fS9TTMNq8Dg4GRgUyWxW1ujbjy0kHQvwK5a013UdP4t7j5P7rDIrTl8VSagIhcWULSoCA+f6UIB+rwyafdR+e6PE4zlDmnHTdK1GH7Q10toB1B5rHu5Hnl3vwPSk+zzSY2puT60ASyPJpz+Ta3gli7FTkUxtUu2ga3OySM/wsKaq/Z5PLeJEB6d802eJdrMpIz6GmgIfIk8rfjikifb1q3aXzQR7GAYdwakn+z3OJI4gkh646U0BatZN0Iq3DdSWk0c0Z5BxWdaf3atTKfJNVe4HWvFcSESMnyMM5rF1Xw7c6hKTFNtx13tgVf8ADesm+hFnKhV4hgZ7it25eMwklArdiKQHN6Zp0/kyWFxcRtlf4TkGsHWPDklnI6wICF/hU5NbrXBhvPNH3xVTX9SMphnWORHzjcDxVNAYVv4evXhBa3cn1xUX9iyh2EoZFHcCvSrLWYYtBW5Y78fw9K4jV/FYv5CILcRr6E5qWgObubURSDY+9PWlt3aCTcuCfQ1LLN5pJYAA+lVGYZ4pWAllDu+4oVJ9aXy3I4UY/wB6o/t0+3Z5h2+lLFdRom1oA5z13Yp2GB3xHpiomfefm5q9Htvm8mCA+YegLdarXdlcWThbiIqT0xzUiI/MULgAZ9aQPkYzULcnikANUBKy85pgcgj0pQexNIcBulIDTjv1+ylHhjfjAJHIrOw2OCPpSrgU5cE470wNPRFjmmEUsSMp9q6Q6da2bq626BicVzmjh47wHb8vr6VvSytNcgNJuwcjiqA6FfsBHlyWuwn/AGc1yWsaUthqKSQoxSToAK7VGiuLFULANjAzXI3FvdR3ircSiRNw5z0pNEmRO0mQxjI+tQNcoV2ugH0rs9YtIn0l5BGuR0IrjYrzy02mBG/SspDTFSS2C/MHzUZmO4+Whx2zUDyh35UD6U3cc8HFIZKpcSg1vQEGBcVzyt843Ekelb1qf9HAppASHIqN+RTzTG+6adhmdKW801agc7RmopRk5pYDwKpAcsGU9TinAhuFq5b6Y0untdE/d/hxyarJbXCL5wjIUdaYjqfD0TWtv5kkkjDOViXqa17y0SRBevGVQ9Fb+tcxY60IFRp1yUzhUFbc11qWrwrIsaW1uBgbj96kBn3P2RkZpYsegUcCsxVjl46r3FXLgyb2imnQoOpq7atFaoQU3++MmgBLCOeNQ9vbpGo6vJyKqXU9w96T9rBx/c4H4VeGrPGSHt8QntnBqi4gmk3xpgfWoYht4u5ldEUDGOO9RxQvcJuRcgVJMwWIgmiO8ltrTZCqkH1rNlGdKdjlWQ5FRtIp/h/Wppyz8t1NA0+QjeWAFAEKZDZHSn5YnNRsjrxUizBMK4waYi/aLubaCwJ7g813nhiJ7iIKYUhVDgtnk15/a3MccgbPSuoi8WR2Wlsy+QJ8YVRxVIZleNL2S51s2bOjRW/3dvXPvWDEmXpqF7ud52++7EtUm/yjwpZvQVdxoluRuhNKlz9qsoNM24Qt8zZzn8KkTy/LUXRdUP3lXvVe/j0+IB7Kdy3909qEwKl5ZiznaMjJHIPqK6ePxfNNosdvONrIoQFc5xXJuzOcsxP1p27bCB60rkluS53OSGkX6sav6drk2mFvIKOGHIdc8+tYGTUiHHYn6CouBty67evdm6E2xh0VBgCtmLxLNdJmbUb0j+6owK4oyjODV1bxY7JlU43DBI6ii4GomqxXaXEcVuoQDBlI+Y1HpN8unwTOuGnY4GR0rKt5jZWkgb/lpyPpQky4ye9UmFjaTWJYkk8uBAWOWk7g1kvJcXN150sjSD+8TSq53gMML61Irw52k9akdiJ9+zcx+WpPtZkVcZBFQsxlVkyAF6VDExZCQOhpgbcN41vbyPCqLMwxnFYeya5dV8wKynJz60GaZW25pwyPmPU9TQBsxzzFfIl2GWQbfOfqB6Vj3URgnaIuH2/xDoadNcGULuPQYqDcDTuMdBbmQbiyqPRqb5ajk9KaWoDgnOaYBK0UcZ6k9h71Tf7qnuRV1YRKxBYLxnmo0h3OCQCo6c0AUtpP0owBWpPbGZThQB/dFUWt5EGTGwFAEltbyXU0cMQy7nAFWxZzWb3azQKfKXlnQ4/OjTJJLa8gnig854ySEPQ1ua/rj6hp/wBnktTbXBOXJIwy9xTJM3w1BB/a1rcBwx2lpAOiirF632oXd3sLoZQschblRU3hG1CG5JiUKwyGK8kc1d/s3QvN3mRB/slziuGvNKWp00tiJ0S91GC0uHfyUtw23PUjvVW11A6dDcKjPsaQrHxnArVubfRLtw008ZIGBh8cVYjm06ODyIprfy8Ebc1ze0ja1mb8qM7w0yy29xJuLOZPmJNQnxJdhmC6WWUHAJ3c/pW1YwwW9skVugWMeg6n1Jp017b2spjmuEicdQ5wazco87vG47o5nUPEl81m8Z09rbd/y05GK1dA1K5vbWR7mEps/wCWmMB6tTX+lXChJ7uCQA5AZuM1Hd6rp62U5W8iPyEKoPJOK0fLKPLGAnfe5ymp2/lXbyb1bzGLYHaopbvzFtwibWi+82MbjVQMx6nNSxAM23ua9iO1jhZ634X0/TtU8OjxFdQqbhSyqqjCxBeOAO/vXE69r7+IzaIjFGhUozOwXJJ6/pUuneKxpXhxtJEDCbDhWXOCWNXfC3gy01TRbjUb6RnfBWOFG2gMM8nn6cUgOp0Sztrvw1p729tA93gQySKgZlA7k56f415zqsfn+JL+Xb+7WUge1bulWFz4YnS5jngW8A2vbb8hwe5rBa7a7a/ldAMzMSV6ZJ4FBKZt+Er2zsNRvZr6UR2ht2XH97PYCuUbl3QKVCsduR2zx+lTmVTAUT72N4b0qSaVLu0UTrtkAxvU4H41IG94U0+SJBqsqSJIrn7KzDEUvr83tXW/8IsmqXrTrJG0cfzLCOTuxk5H1rnvDnj6+0nR4tBms7a4sOfLaUE7R14x71qWY1Cz0G91+389XaMsjAh0A6YOOc0AWU8O3EUU001nKjgEu2/I9hgdq6PSYbW501UCNhgAWVsDj2rC0jxtGdLj1G/tHKw4huJY39TgErjkda7OwttPbQVms5vtVpJ80cqcnNUNBH8sm2rPlMxyBWbe6immRg/2fqVz/wBe8ZauCk8cxap4404QRzLZKTH5cuUZX9SO+MdKpqwMu+LNbvPCfjA3FjChhuYELpIMq5Hp6V0H/CZWbWVtNIBDNMnmGBjhlFYfxD05v7KuNXuUeaaRI4bUJyEUcs3H4VxNtb3et2wmuU2ywzxRHK4JDDvUCse1212lzCskbHkZIIwRVhDnrVfy47cBE6AAZ9cDFPSQZ61aJLO6oyo3kkUgkHrTuGFAxrKrAg9DwRTPs8IkjkCL5kalUYcED0qQqRTCwHWmBxvjjws2sSDUIZ/JmtYMqckZxyK62xhuBpGni45mFsm8984qUosseGGQaUzS9jmpADGy9q8n8a6Ws8+pXsULtML3Ydp4ChR2r1wXTeVsdR+dUja2shcvbxsXOSSueaAOH8E6Fd6bCL2YyQm5QnyscEZ+XPP1/OuxJPetKVVlAJ6iqkkXPFMCJJSo60puM96X7I7dCKie3dTg4zTsApmJPFCyNuHNVikoOKcocMMgnmiwjlbpt/jDUW/6aD+ldnK3zYriAS/inUf+u+P1rsZX/emiwwyaUVEXGacp560CJc4p6viqGoX8Wm2T3U4cxp94qM4+tcdB46uG1IuI0a0c8R4GUHpmlcD0NG3ECvO/iHrdhqsVnZ2jrKbd2dpQeF6DFXdO8a2YSRb9WV5y7DnhAeg+led3cqKZRGMLk4HoM9KdwK7MHk8tTjNMbEUmTIGP16VXJIyQQKAkki4Rc496ixZcVwwydh+tVZnTdgdT+VKAUBVgcjggUI4TqAR70kgGAMRxQM9jVjzVeBjsxt6Pjoe1PmeKQQy7XLJHtO7vimBDFLsORz2pzSkjIz+AqByWbNICQcZAHv0osMs+aAwyc/Wno6JkjbknqKrr5alhwfftTCoDZFJoCzICG8wHimLICBng9xTRJuQqxppTHfinYC0kyBwxAYj1q0HSQbSBz6VlZ2mrKSAqAM59qloC35UeVyM4NGY43IIJB5GTVdS/XcCO4zUm4OoycEHpSsB1Frr1u+gpYySTRywISj7+Caq6jrE0/wBllliXZb8QlxncfU+uOOKwNyjpV+21bUrBI7+18krCwBLxhlye2DVoTQ7U9Yl1WZXkmMiJkoD2Jx/hVG2ilZ2USFs5bmn3l++o3TXMscMbt1Eabf0qv5jKMqcGgRoaPd/YdQhv9m7yjkDdjntyKv69rEvijxCl0bRxtRY/LibcQoPr+NYi/fdBymflb1xWjpzzWSLfRSA3BZoY4Eb5mJ9R+XFO4M9f0ONI/DVotrBNFCEwqTDDjnvVxQR1plrosugeHvtNxe/eRZJEnl+4cZIGf5e1XItOuZ7VLiNo5EcArhuxrYyYiEbBT+MVJDo145HmBUX2bJqydJeIfKWb61DKiZU8TSwSKo52kjHXNctbanE+q2ksyGORldTn+IY4NdsYXDlRG+7sMV5Re6skdzDJBF++ilcGKTqp3ZANIpo2fFt1Ot3BYQgpD5ZnlZh296ueELuO80JI14eFirj3POfx5rB8c6hOuqRBQod7MJIp5C7jzitjwPIk2n3McMHlRxsqjPUnGTmgVjownpRjFTeQVGS35VG8bZzjimMjLAUm80FD6U3FAh281Zj1CaOAwpjaTnJ6iqlSLtoAkluHZSW61WaVsVYKqwxSi1DCk0NFPzM9aaWq6LRN3zKfzpzWihCygcdqVwsZ241IkuDQ0ak9OfSo/JPPIFO4F5LsAYzT/PRu9ZRBXvSiUjvQBpBhmp49rYBrMWX3qzHLtOc1Iy9tCnAqZZgqgZrPaf0NMM5HeiwGgZhnrQZV9azPtBz1zS+f70WAtTRpJVOSyyp5FOFx708TZ70AUDZsSRiq81u0X3hgVscZomjjuLWaORcqY2/DincDnyuKbitDw5eR6t4fWWUK0kbGNsDHIP8AhirbWlu38FNMTRi44oxWz/Z0TdCV+lDaEChaK4y3oy//AF6dxWMejFXJtNuYeSm4f7PNV/s83/PJ/wAqdxjMUuKeIpB1jb8qNj/3G/KkIMYqReKZtfP3G/KpEhuJDhIJGPsKQD1YDrTw1NW1nP3lx7Gn/ZpB2pgSo/NWY3Hc1VFvLn7tSrDKOq/rUgaMc3H3qtxygjBOayo45epXH41YhlCsATz3oGaeO9G4CoPtSBcZFMNyvalysd0WWkXFQmUZqq1xnpTfMz1quVk8yJJF3tmsDXY2i1LSZG4G/BNbwkT1rD8TN8ljJ2SYZq4oTNd0bftFcnqyGHxJOrDHmRhv0rrJJcSkiuZ8SH/ic2snd4yD+dakG3Zgvp8DjpsFPIqtpbE6Yik/dO2rOaaJYYpKOaTL9lqrkWFApc1BKb1GBjSLZnBDVMomY4KKPxpMZZiJK9aeSoXrzUUasByRS7T3rF6s0WxE3WkqdoxilSBNu45NVdWFqQbEbqMmm+UoOeasmPA4AH403yWIyOazZSZAxI71EZG9asSwuF5FVWQ0ihpk54NN3mk2NnpSlcDmiwDd5o3U0mmlhTESBsGuQ8Txrea/BCSjqRgLuwR/nFdVurzDWTcweILiXzhvVsKQM4FAXNzWPD1vptkbmKV2IIGGFc6s0nmLsBHrmtGHxNcBcXapOw+6xGCBS3Gux3lu0clsgY/xr1AqWjRMpTzCRTvc7veqiTSRPuU9KYMA8uT9aSR1C1LBE7XZfl8Z9hW1Y+IbWCEwrasu4YLKec+tcv5ooEgzxRYZv6prEl5ttvPlMC45bqazJFCplHB/CqfmFjyaemXcAsFXuSaTQgJ9akifYwNaj6FBHb+e84lX1D4rOhtXuLhEt0zg5xmnYCw0m4ZwR9asJLiwGEfcOjY4pt6k0alZofKb0osL6SG0mt9oIYZBPaswK2Ayhnk3NSEN/C5FVDNlyRxmpUclhg0ATCeER7WVAw74qAT7WyueB61C67XOeuachT+MZFUkBs2EiuqydzWkDvRh2xWFDPE6qikoQemetbVsckA9CKpAZrSyWl35sUjRuOhFbdp4suGHl3Ijmz/EVwfzqJrKC5+R1YE9+lYl1DFbXJjV8474pgdCl1HdTFmXaD0Gc0zyRcwTW3vuWobCCWS1LwfPj7wHWn2xm+0Bo0OBwxPGKdwNXQtMSexeKaR9yghkzwKx7jwVcyXDNazII+wY11WiBY5LjJ++M1t29vCttk/hQSeaf8IhexyiOZoiD6NWLfaZcWty0QgfA6cV6zHZfaZTvRh6ZrKm0BZ7jZeO8an7m0/epDTPL5rWeBdzpjNQDPfrXouv6Tc2afJp+63H/LRORXETJGZTwU+opjuRW07W8okXqK0rrX5bu18iYIc9TjrVdNKnlg82LDr7VQCqeopMBhGKcI3KbgMinlARVq0W1e3ZX8xZB6GgCgwIGSCKTNTeXu+85x6EU1kCnigBIyMNk9elKBk4zg+tNKkDNLGmX+bp9aAN/SmiZ/LkcqT0IrX+wK06st6g9mrnbSza4mTbOkef73etO5sXtUVt6SZOPlPSqTJNmctAFCTq2fTtWHqM8iTA7wSfTtUqwSdO3fmrN/oyrYNcIzMV7EUCG3erLJpPlRvhv4h61zYWrCuhgwT83pSwxwyHEuR7g1ElcaKZVRIQ2cfSrQt7Dy97SOrVDdxxRyfunLD3queakZo29pZXCkrORjsTWjbhBiKM5IrAtRMbpBEu4+ldNZQSK6mVAjegpjGtGwOCKa0bntWr9iDfMGNO+y4pDMCWJgp4qqpKSc10E9tknisi9h2DIHSqRJnf2w8XEdvGidgOg/Gq9jZalrks0Vk8KhTkoGwKgS/f7P8AZhbqS38Ratfw9FNoHiK3e7KIkqE7i3GKsZk3mnyWrYWGTzomw56jPtXUul9qWkQSldqqMsqntWjqM+nvqUs0d5AUccqvQn1qk99Atq0cdym0e9JwM+Y5y78lpFRTj1NdTp15bW9kkRRXx3bqa425liaY+WelSW1/LbEkANn1rPmNCzqhb7Y2w4X0qCDeBlulNM/2iUySHaPzxV37Zbta+SkbN/tUm7gQvIpC8BjnvTZSSclQntULMuRsGMGmzTKhDux9qgYTOAvJqaGUPGQWrJvLgsyYyFB4o87J3L27dqLAaBAVjk59DSMqYyQKe20AEYKkcc00lADuXIpAQGJncbMYPcmmEPuwcH6UNICdq/c7Cn25BlCnjPerQFmGPalTLHFEpmblh+tOuInW2IQgNWc+YlwxJoGT3N6JR/q9uPSs5m3OWFLLJlSRUf3YR6mmgJADjnpU0nlu0exvlUcjFQIxaMD3qVYpB5m5CBjvTBoa3zTNgfJ2NTWkwjudj42MMHimxI3RwQPUipjHCr56+9S2K5QuVRXco/IPANJDvVQ4ByKmuVicgx8gdarMxBwDgUxEjXB5JUMx/iNNVmYEYJB7igR/LliBTgFDbV60yi5Jtfbh+g6GgKp6t+VMTG4Bgc0S/K5FKxJPGELY3ED1FSsIrRhtAOPXvWdlh0NKW3ABsk0WAtPGHYscIT3phVli+Y809yTEOAR70wyZHzciixaISR2pM4z0prSZY4AFMLFiRjp3p2EAyenJqRYznpT448KHx1qYbcdg3vQBCArSfPnGMdKNxQYFTrGhQljg4OOe9VSrNjoCBQBOk0ijOee1RzzMJW2n92cUyMTOWDEgL60jhtmfegBokYNuVip7YOKfI0ty5aR2diMc1Goqx5gyCibQBjrmgDYs9dns7EW/2RGKpsD7sYH0qjNdNdWqQGKNQhyGC8/nUW/jDH6iouT0qQK7QrHco7jcgYZFb8wtJ4YntuZWONo4GKx0O99prTtB5l3F7cVVgudtpcKfZEXbyK4nUzaXur35uJGVlJKkdyOldzav5FpLLj7iE5rgL64iuUSfjzDncQuM80WJuymLZJYFkVcc42g8CoZB5CMRC21iVBYVNE6eWAG59PapdX1FLp44rbcII1xyOp70cpV2ZSmp4Y2mlEafePSoYoZJ5hHGMk1uwSW+nxC3i2GX+OZ1yVPtVNWIM57J4nIlxv74OaInlXaTdSKoOTErYNSNMZJgx6scE0hgaO5Ew5WoKTHPOLm8LnczdyzEmp7FbRmcyTGN0fITs1ZcMhDSsO7VLAzLJIyIGdhgZ7U0xEtxcSvdSyOVwRj5V4A9qktZVdg03+rIIqN1zaxxt94HJrQsYonWC2ePOVLHmhgbGrFYvD2lCO3kRbVBvlYYDFs8Cu0+G9oy295a3UrO0gSUWwIKFP7w449/WqEvhu08Q+HLO+W4mtYlx5xY7htXrgcZPTFc94j8VpFqNn/wjiXWn/Y4PI82QYaRfp6cUooD0zxNpq6no5svJW3STMjSKgBGznAx61xotvEafDlbDTbWRo5LjzUltpQSE5JU915qxpPjKa9ijs7Sa3ndjtBv5NrljyeldTov9pWWszQ3cNulndDfEkT70DAcqDjqf6VfKNSPKNG8S+I9It7zSLd7gXdzLnbJyQSOcZ6HpWtonhew1Lwdf3rRzwavZyNueQ7cHqOPp/Ouj8b+Hra3tLrVS8k2p3Nyi2zKNpQk8AY69Pauj8P3dlrnh5muYBHcP+6vYsYPmDg5pNBe5oaZM2seELSK4BDSwKJAw5yMUg8M2xkvJGAP2q5S4cDplBgAVctlgtYEhiLbEUKu45OBVtZ0I60rCK0lpHIecg/WmCxQdHIq5kHpRgd6q4imbCT+BwTUiaZeYyDGPqasrIFqT7UR0NFwKUlleouTEGX1DVWMMrH5kINahvMpgnmqrz5PWi4Fb7M4HEmPrUbQzryCrD2NStMajM7DoaoCPEnemEuDyKlEgJ5pxXeMg0AMSRs5NWAqsu4VXGBTC+G64osBoK2xRikEUch+YVRE7H+KnCdx0NAFtrSLdxyKPsKll24HNVFuZAetWre7zMoY0AecQLu8U6j/ANfA/nXd+RA85D8VwmnSCXXbxh3uv/Zq7aV8TUAXEtYY2+VAfrUws7ZhloVJqmk7ZBJqcXJIpAMvPD9pqVq0AdomyCDu4zz1ryHWYZtF02awnsGiuHmJM7DaGAPBU/SvZYp2Dg1518VI/MubKTG3dFgN2JB6fqakRw4/s+KTDXEsm0Z5PBNZcs0l7cs6QpGp/hXoK0NTtrWyjjjilM08sauWIx5ZP8NZkT+UMUkyh7RQhNrqN3emRNt+UZA9qkeTzFxnJAqqJduE2jApDLrxxS8j72OoqM2qEckk+tQeftGRUpZgOT1FAFy3QyRS26D/AFmMDICgjOSazAzhmSQYdDhh6GrUbgSRuT9xg1F3Osks5UDZKBnAxzTAo7yDxSq2H3MM801gc0maoB3AYnsaCeKaMkjPSg9aADOcfWpCWKAe9MAxVuAoWUFdzHtQMr7TjJoUkdDirMkahWZVO3P5VW70rgWI5OME1I52lWHQiq6Rs4+UjNWWjlWMq4+mDSAdHiTrWlpk8tt5zLatcW4wsyjtz1rFhdkfkN+NakF/PCjqjYWQYYdiKEBd1WXw9JpFk1jG8OoMxaTDFj075rCD5P7w1oQw2ziWRyFlB/d5PaqNzbvBdBJFK71DLkYO09KBEkS75AIgWbGMAVfktrjSbyAzxqJlCyqA2QPrU2l6tDp1lJaG2SUysWMjnkHjH5YqLUp5b64a8kZSCAN3Qn6CmBra34hm1a3tY5CWhhT5oz0Zuea9f+HMinwpbXEK7VfIKMpwCOOPrwa8g8Kroqi8l1MwvKsRMEUvALAE8HueOldnaeP7keFLTTtF057i9Fp8zQJk25z345PFArHq8jnO4cUxp/LXJOK8/wBA17XvFJtLmQNZ6dagpMSfnuZABwemAOPz9q63n+8fzpXBIuXV2kdrJJIcKoySOuK8ql0m1uvET31rKBukJUSHhiB1/SvRrh0NrP5v3FjZj9BXl1vqNtNbXlzp++Z0mkW2AJyRtzwPaqTA5nXr6+vNRFvcTed9nOzzAuFIDdc11/gOJLa/1AlyWeJXBbpyTz+PauR1Asui2sDnZMVMrd8n0/WtXRNZEc9va3EsdvBAweRgPmkwOATSuB6ad38VHFVbe6a7sYpmQoZAWAI7dv0p2TVoRISM1H5YJzRk04GgBklueqYx6GoXVosbsc1aDNUdyDLGAOo6UxEQkxVhJyBgVTMbg0qllqRovGQsasxA7eazFkcNwanS5dTyakosS6fFJEzLlZMZ471mNBIOMVtw3EewEmmlI5JN+c0Ac7LBP/dP4VXaOZeSjflXSymMHgY+lQlh2p3JOe8yVesb/lTvtD9w35VtK4Vs0j+WTkqPyp3GZC3JPrS/aG9DV51jPG0Gm+TCRyopiKfnE04OxHerHkIDwKURL6UAQhjTw5FXIrPzeEGTVhdJZu6j68VIzMEhqT7UsdvOW7RMf0q8dFPdx+FR3WhhdMvJ9xwkEhI6/wAJoA4T4eaiDZalCTwJRIPoeP6V1wu0z1riPhOltc61e6fcbszwhkIOB8p5/mK9jg0WwgTb5Af3c5NJAc5FdRnHNXUcsOK1ZNHsC24QY9geKY1nFHwi4FUmIpKrk5zj6VYbDYLAVKIBSSKqrtxz60rgEkIRQQikEdxUIVHO0gKfQCnrIx+U5/CoZDtfP8VFwLEFtGJQSoP1rRRVHBVR9KxPte0/eqaPUR0JoAuT6ZHcSFw+Ce2Kpz6W0C7yvy+tW1ulKg7qes6twzZX0ouBiMVPSm5rSvYEuMBMIRzlRWU+Y3KnrVrUl6FhHC4zTpUinG4AIfUVUVmY4GSatQBCnzuQwNDVgTIBay7jsG4AZ4NRBq1BakxeYsg+lSQuVkBCID3IHWqU7CcTJB/D60bvcV0M1vbTwb5Ixv7HFUjZQD/lmPyp+0FyGXnnqKyvEg3aSG/uyqa6j7FB/dP51meJ7KJPDV06g7lKkZ+tHPEOQjXmGFvVAaw/FC7HsJz91WKn8cV1OnrBNpNpL5ZfdGOR9KwfGrRf2LCodN6Sg7c5Jq+eIcg/RiWt5V9Gq+VUAbnRf95sVzK6tPppzBEjiVATuPTgVmyTy6le+behX9FXgAVm6sUX7Jyeh182o2FowW4u40J9Dn+VVJfE1lESYVeYD+LG0fnXKC0VHdlAyxzgjNOEuw4dGX/d6VnPE9jaGG7mhd67qF3MGhfyUByAvf61Yj8Q6lEgE0MU/wDwHbWYs0HRHyR2IwakWYt0asPbyNfq8Dcj8VwBgtxY3CepUhhV+DxBpVwuVuTH/wBdUKj865bfIO9H3/vtR7eQfV6fY646xpuOL2Jsf3GzWNqOpROryWmoy7/4YwMCsVrSJzxx9KabKIH77/g1V7ZdSfq0S9Y2sWoxynUtTeKXI2hj29azJHu9J1ArbXjSohBVlbIP4VK8LSIUZlcfw5XkfjVZbWWI/LESPY5o9rEPq5uT+LNRRI5DDbFSMlc4qXRvFH9qagLW4txEz/cKn8/6ViFoTxPbKjHowQ0RxQmVTBcIkg6FTyKPaEewO/eLy2IqF4Gk+6K5qLxNqdo+2W6t7kf9NF5/OtK28ajYftFnEcfxQNz+RrX2lzOULFtrK4P8FR/2dd/88/1p6eMNIkcKzyxN/tpx+daUOr6dcYEN7AxPbeM0cxPKZX9nXvJVMEDOc15HfmZ9SuWmXa5fkZr305zwK868WeCTAtxq1jIzZO6SHb0+hp8wmjz8xs3QUKNtTQxyvgyN5MbfdZh1FT3mm+QqPHcJKGxzjFTcpFB/YjNSI8QT513t6VEAB97rSllXoOaQ0LvjD7vJH41AwAyfenmQnrUbMKYx6gMvGARRjPBxUBJHANALYwTQxEyqzjZkn2JqUNcWNwhBKN1HNVw+Oh5qSNLm8cBUZinf0pjNW/vJLoJJJ/CMCqccyAHnqMUk8qqvlM+WHtUKKDWQiEgh+lWkI2qcYpZIkKhwsme/yGgqAi4J/KgodIqOQ5kUA9RVeRUDYibcKm8mV1xGCaj8l0bkYYdeauBIqQNgMwrobFv3C1iSGRQB5bj3xxWxYxSpADIMelWBq27yCUCMBmPQGmS6RPJckX9juTsVOMUtu2y4ib/arpXG4ZHXsaqxJnWmnQWG9oAyhhgqTmnmGW1QzKqtFL8pBFSzsywkuuB3PatiwiF1YhU5IpWHc49NRaDUDGTgHpXXacskluGfOO1YXiXS44rMXOESWMgEYxkV0Og3H2jQopM5pEl1RiqupELEs2MsvSrfSqV6/nstsgySaBoWGRr+JTPyndOxpt1oel3UJR7OP2I6itC2hjht1XHzd6HxQM5fUdHFlaKtuA8XOV28j6GuMu7TTCjM0UtvcHoWHFescZzVW+0yw1FCtzbI2e/Q1JNzxprSXYXV0dB1OcVBCzJLhfvN0HrXU+INHj0qfYJB5T4wAPu1ziosVxuRt208NjFMssRadPcsC+Y89yvFLNp/2YkGVX+lWr/VPPtlEcmHxgrispp5MdaACS3ZBkDj0qEDBoaeT1pgfPPU0gL0TFlC1at41FygmkdIj1PWqFvcKsqhq3baexknRZ1GzvnmmmJo1FbTY1CpJI7dq6S2tTJYeWy5VhisMwW7YNnAjnscV0tk7fZ1VkZSPWncVjgZNFnbUGh8xY17ZGaiGjvHO0Ukyqw7iul1hfI1JX/zzVcJnWI2/vDNUgMqHSbcSqJ7nDdgwqlqukC2USxyrIregxXe39jZTW/78eX/ALaDmvM9UJS5aKGaWaIevNTyjJ9MmaxmaRrQSgjHzDpVy41pmulO0Jj0qha3zC3Cu/I7YqrI4eQtUMZ6DbjdbI3qM04iqeiTifT1w+7FX9lIZVmX5SAKxNRj+Rs10piJ6DNYmswSJEz7etNEnARt5dxGCpYKckAda2LvVYNSv7dJLXbGuFzjn8qzA8E8kkj7osnICnis5ptsxZGYEdMVQzsJde0jTW8m303evf8AyaxNTv7a/bfb2wgbnOO9ZLzNIctzTo2oDlNTSliZSsy7mPepry0kG94oX2DpxWXHNJC4KGnSX11K2XkIX0FSAgkwM5HNWUuFRcM2DVdJFGW2A4qDzHkLFgAo7ClYC21wD901WkLO2WYH0x2qIN2FSJE7ngcUcpRID5/3gBt4FRuUi+6pL96WQFRsHB71ER5ZBPemBZR2faWxwe1X5pTJBkIRxjiqEEe5snNILqRZGi3kDNKwD0BHUEVPFFulXJIANRLkuMkmr0Kg80gNAEMmKy9VdQwVRjHWtFfljZj91Rk1jOpuS75xzSiBWCliAOpNMm++F7LxViJGjl3PyFGeKqt8zE+prQksWmTOR6DNaUc3+ilihPPYVnWzeU7MBliMCnXF9LBIUjO31pNDuTtqcLfLKjA+y0vnWh/jNUmZJIjL1fuajR1DBjUWAtFbZpBy6oDnI71Xk2tKSmdueN3Wn7zJxgUxhtOKoBxbf17VatIrc5kkYhiOAO1VklRVI25cjvVm3VREDjmkIUwbpwfM2g9zSSLsfaSG9xTS5zgkcelIxRVDAkn3pFAQAabNjhU545pskzEDoKajFmppCRYtW4KsTjHHemSnA20mGTkZHvTQQ7cgZ74p2Aj6imbiGIFPYkdfwpFUF8mmMeJnSPaMUvmtKQzdRQYwxwDigWkuco3H0qQLBBdR71HIu1x6gVMVmSEAKSfUrjFDRoRyTuPWmBCZGJAyMe9IWXOM0xRGZMESEg+nFS+WBN842pjI96AAQkR7iKu/ZENsPL+ZvSql07LBEqnq1WZWiWCIsckNyKAKfmYZgPvdDQWKpx1pZI3L7hGMMeMUEYJWRSrDtSsAkQyCR1rY0SNpLvDHO3npWTF0Nb/h2PcfM7nJNMg6ed449EuDI+xChBbGccV5pA6xuY5433k5z7V3+uyeToe3H35FH4VxmsSxPfRNEfuoAaAKl1C8bB40PlN9wjuKoknJzXQNfwnTDApUHGAoHIrBcZXn73eqTKTLFhM8M+UAJI71qwsi3UbXDspxzu5xWMuUwQetWBxzkk0mxMvQRwNeqM4jYnB9KW5QI8xgk8xEGemOfSqXmhOtXTG0CYkBCSjjaM5PpSYIzpGaUqrKiAc4BogDCTI4wetQbgGx2q1AVHJ5BHShDJZWWSQ7a29AijkuS1yT5UUWSRx61hxR5NakPyQMp+6wwaAO90PXvtFktrbW/wC4tiRHCxwGfOdzHH3R6V57qt9c3uoaiJ5BLcTS4IgUFMDsPQVpWt8LS1lAEgkddo29QfrVrQbS2s7wJNKm+SM5YL29zTUgOZ0C3srnXrODUGdLSV9jshwQT05+tev2mnnRbQwee80VvN56SO2SB6V5fJaQWd9BcwRTSWIuEdpHGCxzyK9nmt4wVOPvDNaLUTKkuv29ykRNhPcneGiAToR0b2q9FcwGRxDGEcnMojXgP3ye5rmr/SZbOLUrjT3LTTQkJGf4foc1zdr4m1fwtZRabLY7XTJIlGQfenYg9OMzjvUGp6xHo+lvfTvwvCr3Y+lZOhatf38DjU7VLadcbMDaJM+lUPHup2sfh99PEoN2zrhB1UAjP6VLKR0+ieJYtZsFnhH7wffjzkitJ9S44Rh9a8U0jUG0LXLW5J3hVHmKnXaa9Y/tSzfTm1BZVe0VC7MPSpuMtnVYkmjilnjSWQZVC3JFP+3oerE/hXies6g+qeJJr+Bn8rfmJ36gfSu+8Gao+s6ddjUAjywOqhgOMEdP0qhHXi8Q9HFDXUeOtZ/kWvYEfQ0hgg7An6mnYksPcJ/eFQNcc/eFVmgUdGP50ww+kjVVgLYufcVMt1hcZrM8puz/AJ04RSd3BosBfNzz1pDKGrKneSBlZ22QjJZsZ/Kp7a5W4hEqBwpOBuGM0WAsz3P2eIv5byY7IMmm2N3cTRA3FuIySeQ2ePSq96ZGt2MTEOoyMHrVK11O0jnNp5ubggFhjq3pWbQzos0sZw4PpXDQ3s6XX2i0LiVJGRklfIHPTFdGmpmC3d7p18xVO8oOKBnLaDzezN63X9c12krlnzXDaDLtge5xnEpOK7BptyK2MZHTINNEkwlYd6cLkqck1SM3vTfNFWBsrexEctXMeMtTNpZw3RtUuYR8hDjIUk1bF1Ebr7Nn96F3ke1Z2rwW2rrJYyXXleV8zLkbXH9cf1pcoHmGpz/ar6a92CNJDnavQfSqDua3pore4tdiKsagkFiewPFc/JwayNAWQg5ppOZdzfiBTQGY/KOnWn7SKYFnMDY3bgBTv3WCAzHPrVTJzUm07cik0A/ftXH1qNmbJyeD7UhJUg570jMGOAOKdgA8008UuMVp6HZ2d7fNFdRmRxGzRIGK7iOoyPagDMz6UV1cOnafcKWi0GZ1BwSsznmmP4ftOraXqq/7r8fyo5i/ZM5jmjcy/dOD610Z0XTR96LU4vqAf6Ui6Bps0qRx3t+JXOFDxDGanmD2UjBVyy4YmnCPJzuFJPCba7mgLrJ5blNy9Dg0YGAfUU0Qx8RMbHB6irKT5OCeKpk4pCzD7tDQGhlX3Ln5hzinNwKro2xiygc8kVKXBAPqKgYqsM/MAR6GknE0my4kyyudquzZJA7fSkUFmwKkFvK0akY8v+EZ6VQDrWf7NdJPsD7c/K3Q8YpI0nu5VihjeWVjhURc1PaWuZQsqk4baQBnA9au6VdXPhrxAJlbchOdx5DD+lFwJPD/AIZu9du3t3lFqVUsfMjOQBjtx610ui+BNWsNaKvdkWJmVJTExQyL15BHTt+dOHjpbmYz+Xbm4U7IItpBGeMk1p2/jLULe1nbUdPWSWCPzFeE7U+hFFyDvjMvTCqB0UDAAoWRW4zXmeieLvEmrStOIIZbZDteBExnIzwT0+tegQ3AaCOaRfI3DJSRhkUDDW9T/snSJrxIlm8sZMZPUd68u8P3VpYXt3fqHWWSSQpAq7kVcHj9f0rrvGer6cNFMZuJVlkU7PLwQw9/biuMkaLT9Minmm84TD/ll0TA/wD1U0Aq2q3wnjYKx4eVy23aSRgc9MAVHodo93cXVjb28bCNzJLOrA7h2UfjWLaST314losuyFyZHXPBxXRW3iEaciRpZbGV9paBQokPOAR16mgDttO1OGVPsrzbriIAOAOBx2rQXa65BzXIaRpVzoWhX2pXse65nUOPn5CnufTrW34allu9J+2yqA0zsRjpgcf0NJMDUxzTtnFIxxXNeL/Fd74a+wrbWKStMWDNICRkY4GO/NUmB0JMi9qeo3KDWPZ+K9Ln0iK+vJFtppBlrccsD9MVSfxpa3Eoh0623ue8xwMew71QjqNuF6VCLZ27VzS+Kb8S5aKBo/7mCP1zU6+LZ/8AnyjH/AzQCOgW2ZGBNS+Wp6iuUfxRqTyqyJCiDqhXOfxpW8ciyuES/wBOyjjIeA8j8DUtDudG4KNjtVe8vBYWkl1I2Io8bj6AnH9RS2HiXw5qZCrerHKf+Wc3yH9a4Txn4zLNqeiWtiiwhvLFx5pJYA5JA6dqVhnfifeisG3BgCCOhBpDJWTo87SaDpzt977Og/SrvmZqrEEjSUwyH1p6IrEZOB61OtvE4BxkEZBpDKe9vWgO1aAsIfQinrYxBvWi4zJM7C9Ntj5hGHJzUyljViYWFv4itYnZTc3EJWNc84HP9DV9oIj1Wi4FOOR0AIODUq3EhbJbJq0LaED7n6077NAf4P1qbgRrdsTgmpLm6ih0jUJZGwi2zkn/AICakjtLQfeR1HqpqPxDHaWvhDVnJ+UWr5P4cfrimgPI/hXI0fj60C/xxSKfpjP9K94LKDya+ePh/cPb+O9FdDjfIY29wVNe+3TlXJBoZRYlkjVfvVVkbcCarm5IGTyarvcu5znHtQkJsvwHMy7jxmp7pI8/KKzYZvLYHOcGrBm82gkIyu8Z4FT3FpFLAPLx5h5yaqgVb81TCBwCB2pFGNcWUsL4k249Qc1Abdv4au3iMzBkYn1zVdHZHBPY1SJEitbpj8q4H+0cVYEF+iZaPCjvkVM13uTPSnRX77NmAR3oAqxTb2wzcVb+y20g3MvJrIEga4YqAoz0FXEdtnWm0BZSOG1l3Qrg+tZs6zQF5CuEZic1fjbcctU1za+fbjbU3FYx0upPWrME7mUc04ac3bA+tR3if2Xp895IUbYvyj3p3BGFrvi3ULfUDa6dIirGBvXaCSe/WqcfifWyocXEMgP/AEyWuLnnN1etOxKuXJBU4Iqy+oS7So2pnrjuazkdEUdefF2tJ1a2/wCBRiquoeJtUvrCW2meDZIMHYmDiuQOobTy4P0Oaf8A2wzMiHcUJxjaKkq0ex0MWo6pLYJbJfMkSjAC8ED61nXdmIrdnb5n/vbsmqN9etasqLnDDPBxVNdSkd9hJKnrk5ovILROk3b7S3Y/88wKiWSOGVS7Yz0qKSX/AIkYk/uDFc+l9cOobykH4miYoSOzCKw3DGD05FMOFGQFP4iuROoXA/ghA+lM+33B6eSf+AVHKa851hIbqFqB4S2PKm2AfwluPyrnf7TvP+mH/fFIdSvWP3ofxXNHKLnOj826j6yRN/uvz/KnLdK7bZJQPqa5wXd6R1g/79//AF6a+oTxKS0cLH/dxRyj5zq90WPvg/Q0oC4+XNcnBqVzNuKW8I29fmIq9De3jrlFi/4DIaXKLnN8Bv4UY/hR++7DFc7/AG1eRuUaLDD/AGzTv7euB95ZB9HzU8o+c3d8r/L1o+xM/LLWB/wkb/3n/wC+v/rUf8JG/wDef/vr/wCtV8ouc3Dp6DsV+ho+xlfuSnPqRnFYX/CSP/ef/vulHiKU/d3n/gf/ANajlYc5rTWcjqcxhj6jiq/keT9+FxVT+3rgdY3p0euzO4XEoz70cxJpwTmMMYLiaBvXeRmrcmpaqtg6x6k0yMMMrjdkVntJLMuGOc+tCRSIPu5+lHMP3exl6NKpEtvLGsmDkA0680+6uIfLQbFzkcZq7a6Ssdy1wqsu7Py4wBWgIyDyQPqav2gezicZLpN4GwNjfjj+dVpNPvoiPNtpAp7gbv5V3wAA/wCWZ+ppDGh7Rj6MRR7Qn2cThHsJFgMynKjqMYIqi2K9DNnGc5k+U9Ru4P6VQuPD9lPnDQr+lUp3IcDiNwpcgnFdUfC9r1eQhR3Rwf51Pp3hq0W6eaO4MhVfuOMA1XNclxORHy12OgtbGyzAjLI2N4Pf0rJ1BIhqJKWSJ5ZIOf4qW2neC8WSPag7oW4x3qkxWN64t7dAZVtYQw/2aWyuEdSvkRLjuBTxLHdwMY2DDGeDmqFi5W7KeoqWhWNS5uRBbMxVMfSuOum3yNJ61ua9dLDEISPvDOa5x5VZcCsxjlbitrwzpy6rduskgG3opHWsaykRLyEyjKBuRXttrb28ljFNb28a7lHIXBrWBJjQ6KkShRErfWq2vaTLFp/mrGFC9cV0/kv6VDd2RnsZlI7ZqyTzlX5R/Q5r0GyhgmtY5MZyorzhfvsvocV3fhoyTaWO+04ptiNmFYYz/q1x7imNCIJTPYAROTll/hal2sO1IWIpCOd8YxrqGl+a48u5h+8q9GHv+VSeCpo5NGMBbDKSa09RsI9SsZImZkkxwy9a5LwpMdNvri2n4Pr+dAzp9Q1A2jEGnWKrKFuSOWBwa5PU3lv9dZCcxKQRW5b/AG2FljX7npQM3C/NMLj1pfJl25ZSKYYmz0oEG8UocGmGNx2oCMOop6CMbxLoEWrWjOmVlT5gQeteY+TLbuVeM5BxgV7UAelcL4ltRp1280g+SRsA+tJlJnKtcI6FWQD6VIbCKazMokYY7CteKx0y4sjcqQWAywPHPpWJ5bOpRCwX0z0oGZu3LYFXpNNmS388bGQf3etPi0+UPuAzWoDI9q0W1U9xSGYEEiRPkoH+tXYLiNp0byEG05rMlTypimc4qxaPH56qzYzQI73TvEWnJsgaAxt6jmukininXfC25a4a0+x20qyOqkdya7iyuLa5tg1v5Yx1CmmBi+Jx5axSfSqbHM8Eo6EYrc1y0+16c6gZYdK4Wz1SQ3UdvKej4NUmB6VHHHLafvUDL3BrkNWuNKh8yO2gMchGDgV0k2oQWlmDI4KnsDXn9xBLcXsjQuojc8ZqQMGQASMVNMXcDzV7UNPuLPaxUMGOCQelQCEk88D1qWhli1vp7M7oOtWf+EjvipUqRnuDVEGJP4wfpUkLwBs96zkBYfU9S6rcuKjN/eTrtnunI9KfmE9WwKbLHDszG4Yjrip5hnM8xKDuBLDoO1VmHOadmnD5gRWoxi471IjooxmoiAjcmmZyaBFvenY0x5PlPBP0qIZyKnRRwT1oARVOOabg98AU6RivQHHrUWdx5pgSIoLbQc+9X4ise3OQPUVQJEScDmpoFkeL75C96Qx87+bN8o+Ud/Wq0rbOgJPapchWKjtUxKtCFAAoAfalVhDycH0qpGIpJnYDLA9akkV5QkQbAzilS3Fudh6igCaNdzitKFBjAFUoQOMVqxRqFA9amQEN1j7MVJIBrIAEO4RcjoRmtS8IMgUHgDpVBsH5aSAhEjRIPlJJPSmTFH2grt4zU8sYwp3qAMYBpSqqu/5XancZXt2Q3CRqKbd2ztdN0A96twQh7tWKgY9KmvoT5uT8v+9TuIp2+loy7nuFHsKlubdLWEp5YPuab/y6kgSY7FehqsiOy7vPk+hOaQDxAdgYMOecVE2QcGg3I6O2RUghiYhske1NiFt4y7OzdAOKVZMdDUgIUHb6VAFO7AoCwucZoL5XbTjtK9PmoigeQkBcilcB+yCSP5CRIOT71Gh28d6dt8s5VsH2NRk8k96Yi45U22A65HaoECrn5s5qHJNPgkESuWG4se9AxnR8kU7rQX3Y4FShVZlxwDwaLjYKuUOOvb6VM0kqWoljchgcfhQ0apjaSATwKn2ItpsOMk8ZNAFWK4acEytIQOV3U8AMQevFI8EjdCAvdqsxOsKMCQ+RgAjFJgOEjSRGPYflH3gKqyBWjw3agzzIpHIUjoKFYeXk0kBCwEpUA4CjHNPVUbKszEgjJJqLePMBKk81IZBIGG0cmgCeaTDIqkYx0HaoJpfMky3JxjNR4wc0xV3SqpbaCeuOlUA/iut8PxeXbL69K5iS32bfmzk44612Wkw7YYxQJlPxpIy2dlCp++5bHrgVyUcSSA7nCze5rb8az7tVgh3Z8qLOPTNc0hPmBqCS1bpE3EilmQ5YjsKHWLDndgjnpxQSVUkMVJwDjvTECj5T0oKGpE8rYQZI5qdEL4A65xikidY5dynBwRTI8+YTuYNjgg4waALktm5ii3ffAw3FS3l3F9iSFWzgZFS2t1PtilU5baQwA5NY1zsBTZ7/AM6BEA+YnmrSjaq47iquxgxXHIGTVpDlVx24NUMvW4zirMkg3KEcNtGCPSmW6jZn2zUEg/fs0bZL5OcVmUaca7oQ/RqTT77abpJohKu4BmJ7Z6VSNzKluoYDnuafp0sKvFHIg27gZHbpntQhMtaxd3N4vmxwPHbIMJt6A+pNeq+F3u9S8PWk88mJSgDo559q8uOoJpd4NP1C2eWOKQShUfG8dQfTFepWvibT7+KOfTXtWkcZe2eXy5M+gz+Na8xmy3cxSRPGV7nB+lU5vD9hqcwe7i3zKpRHz90H2rQu5g72atG0DSuRiQ85x7UjXCWl1MZM7UTfgDJPtVIDPu7xdMs5n1GBNlsoCEH7x7YrzorDr19cahqV35LKu4DGQRUXiGHVLHUUg1XzBNcKZwGkJKgk9vwrGlkXaUJ+U1DYIlmcXEzyJJsXGfmHWpH1a+fShpqTs1vu3eWOAT71l5Xfgip0ZcfLSuMuzrJbxW4EispxgDtXY6Nf2Ok3c8huQEkRd6xnIJ57Vw6SukflOVCEAgEZojlUTADlT1HqaaYHscN5bXLokNxG7upYKrZJFTc15FbXhstRt7hDgq4P4ZrvPFviuK2sre30cxu10u5pefkXjoex61VyGdBzRzUHg2WTU/DFtLOvzIWTfnlsHqa2mskPaquMywrk8GhknUZxxWoNMfGUIoNjMP7p/GlcLGHdAyWsiSjIIPFV7a5it7A7XaTYCTn19K6MafP3hQ/U1lXmnTWX2idolWIjKgEYJ9DRcLFUu9/YSEB4Ny/KwIJx61ykF2olWctEZUOA3fqP8K6z7Db3Vg1zHcOJVXdlW4U4zj6VxU9h5BhlSYmHIBmI4YjuBWNUaNC3vbaPWBdT8RFt7IgyM/Sr+oXtldRmf+0GcqCyosRBBx3rnpm87VT5kirGwGHbpip/NaFZRbSRyDaQWPQjB4rnjORZY0bcmgyEHGWNXLKeK0hWbzJX8yXYrydMfSqVn+78Lyn1JH61Pa3kKGCWeLzijZVC2B7n8OK1lPlA6m6itrWKBt5XzCXdj1+gqaBbV8SKs0bDtKP6Vyurao2q31ssS7Io25DNg4retNRW/eXCbUhAG7OQ1a06ib0JaOclW/0i9Mk0AmcuS8qnKtG3Qe3OK5DUtQnilntWgSFwSrFPQ+9d3qXiHy7aOaxWOdQ375HByB7VwuqSwX2oSTxQskLtu2seaqUhWMgBplbOdq9agkPBI6GnyyGJyqniqzHcPfNQixY2wpyaQvzkio80E1YFhSuQTUpbA+XpVYEYo3H1pNAPeRW5P5UxWx0pjHPNJRYCUuafBcSW1xFcRHEkTblNQgU/acUAejeH7+OHUlKt/ol8vmIvcP3z+Namno97KdSu3cuzOiRA/IE5HT3FcDoczSWU1jvxLGfPgOOQR1ArtdNk1DU4EvbK7ixIu17eZflRhwee1ZJGrd1cnsb5NI1C/spJJPskSLPH5hJKZ4Kj9K5rV9WktopNSfi8vMpBGeCid2/pVy/n827eO6uPNhgPmXUwXGcfwAdhXF6leyahevcyDbnhEB4VR0Ap2KcnGOnUqgnHr9alLDAx6VD3pc8Y71oYCliaekjgYU0zqKkjQgcGgCzGDsGetOLYqRYzwQQaeI0YHI+YdqzGNScE7cBWAwD6in+Y6MCGzTHCsMMvI9sUIMgCmwNKOfcp+X8qW6fz7eWCQK6kZjkfqv0qoQyKAvc4NJNIyhELZAOQRWbiSbCQWEtuJo0MRVfLAVuMjofrWcZ2VBDI/wA5JDjHp0pjMsNvgkj6GqylXUEHLUoxA0rG9GmMWimmViMMI22g+lKviHVBB9nW6cxZJwxycnryaymfmo91apAdNdXa3eiKIw5AYlwx/iANQX99AljZtaohTJ3o4yRjvWMs/lwMAzDJyQDwTUbupbaq49eeppoC4tz5V0Z7c7Gzkd8U+C6vLifmYb94kDnggjpVANipYX/er9c0wOluvE+pXsJs7q8VjIQrllxtUV2+ma1peiaNDYQNNeGLOG2hQc89a8vEQYtMQC3fmrn9vajn/jwh/wC+jQB2994x1SVWWysbaHPR3bcR/T9Kxft+oz+fcapOJdjbkz0UD0HbtWF/b1+R/wAg+E/8CNMm1i/mt5ojp0SKyEFhJ0HrQB0qvG6K+xCGGQcdqQtGP+WSflXPRa3fxwIgsY5QqhQxcjgVF/wkV753ljTEd/7qSbjVNgdDcXsNpA00wkEa9SFzVQeKdFx99wf+uZFVbia6uLNkuxb2cT8E8sx9sVTVbK1Zzb2guZm/5aTcDP07VNx2Nf8A4SjSCflmc/8AAKyoioYxwSNOznKqH3YH9KRZLQy+Y9u9pcHrJEu5Mf7tTmK8PMJiuUP8UOM/lTuFhJbURpuv5rW2QdQSHkP4Vk6jc2L24W0jckH/AFztyR6Yq0EhkJGE3DqGxkVR1OJYokICjcccCgDZ03xAtlDEiTz2rYxlTvRyO5Hb/wCvXXQeKXMTObOO8x/FaSjJ+qnpXn+mRpJZyK38LelXYYkt28yH5H9VGKaYmjrh48sfmAsLkMFO5WIBBrtNPG+ws5h914EYflXkEgMzFnbLHvgCrVpqup2CiO11GdI16Rlsr+VArHsq4Pen7OMjg+teaW3jzUYUP2m0glf+8uV/SjTvHd/Fe3DzKkqy/cjY4CY9DUSAd4xuJ7H4gaRIJ0WSONcM68LliOfbFdub4WXlLqE9qGl/1ZjjbLfhzXkniDULzWdUM90UNwi7FCdAPT9a7TWfE7TPp/8AZXllDgFpFDbWGOD6fWlzgd+U/dE1GTtTNeXyeLdT0q981DFJ5oBZQ5ZR9PSu30vW4tc0wXcSGNicOrHJzRF3GavnCsvxdfLb+C9SfaJA8exlboQTipjKRXPeNrkDwZfqT97YB9d1WB5l4Xu00/xRpNzLxFFcqWPoOle+3U8hbcDwelfN393/AHhXvguFNvDl8/u0/wDQRSYFgzkdTSLMD3qo7qw61HuI6GqEaQm9TVuCUMuM81i7nqxBIy0Aa275utPxx98Vnee1Hn+tKwF49cVXmT5lPbPNNSU5pZJOMnNCQCzKiRAo2eaW15kxTPKaROKdHbOvzhxx2pgVryT/AE5wABj0FSW8hcYNWnjtI7SSSSDfMTkCsS+vf7M0/wC0MCHb5UU8ZNDJLF/4i03SbgQ3DSs2Mny0zt+tJH490ZUA23Z/7Y//AF681udTZb13UqyMckMM7j71Jb39jcYKOsEw6xMcAj0FZyN4xO9v/H8WwiwsXdx/FLwM/QVyeq61q+rROtxK3ldRGq4ANUBrFqXVBA+WOB8wpsurQIzK9uVYHBBcVnzGvLAy3SVSeMVWc5kw4I9DurQm1GCQE+Xx7EVneakkgAPJPeqQ3oV85iZ8Y2kdqtshFzFgAg4OaZgCKeM9avRjMMZ/2aNibj9RiWVInPOFra021tINMWUwR7gCSxXNZbrutue1bFiPN0t09VP8jVGZGmpQrOIpFRFb7jAZDVX1e2hCpMsaq+cEisrTHltRb3F6Ue35jQn/AJZntmt7VF3acx9CDVBE5z7IRIWJJU849KnFnGVyUb8qm6pVyNcxJznisUXM5+7twipsHcg1UCbZFJz19a27+MbCAwO1s5rLkUYBBBwask1EQcEgCs7U4wt0AD94ZxWrFyik9xVfUlB8tj0H51mn7xqZ2nvFFLKssmzcvHGa07S8s4WJluUb02riqwtx/dFSx2656Cq5ieUfc+XPeGSDmJgOaDboy4IqcKBSniouOxmnSy/QEVDJpDp0YD610luoZc0+WFW+8uK3MzjjaSKfm21La2+JsMfyrRkiAlIPODUIASYECgUXqSy27CPMZJapIo8RAMOasgDFLisJM2IJnuYovMilIAODkZxUC3t+/wDy8/8AjtaRhEtnIPQ1Tit1jBGc5pXEND6k4yJxj3IpNuon/lsuceopZo1yDiiyRRdg9iKdiLjMaj/z1X/voUjtfpjMw/Aitc26E/cFVbqEKc7QPpVWCMtTNM15/wA96b9ou+09WjGrAgiozCiJwKSKaN3StKF5DFLNHIkvO4yNhTVS61C9iuGtdLtgxzgEfNn6cV6FoC3F/oFsdiBFXbuY1pQaWqOGVI93qAKuKM2zyuXRPE97GPtFjj2xtNbUXw4gmiRpNSkicj5kZBlT6V6NNYSSxE7lXHrWRIWRthlTI7bhWyMmcve6XBotuyPaYX+GWMcE+9cyG26juH3W6V6eEcqVkQOp7Hoa8s8UWsunarNCvyoQCo9qXKLmMnX7j7RqBCtlEGBWYhI6mhyWfJpyxqZwD0qXEpSuOJ4r1P4b6wLjTXsZn5jOU9h/kV529tAsIdSD9DV3whrH9j66jMP3UuFbnpz/APXqY6F8p7S0iA8VJGwYFQevFV2VWwykFSMg0+LaDzVkHmWsQtba9Ojdzmug8GzBbi4iY/e6frVLxnbrFqqXI6yAA1B4auPL1uAbsBzitESd86nOMVBICDzWw+nk5YSqRVZ7VE++QfpUqQuUoRferhvGNqbHWo5o/lWbkn34r0MQLu+QVyPxDspBZW92eittx/n6U7k2KrWiwvbTA5Z+W4rpVAEq1jw27XNhZOOwArb8hywOKLlMtm4QrtzShQRkVQdVgkDzSJGv+0cUxvE2iWgKy6jCGHbJoGaRXFRPgVkS+OfD20lbp3x2WMmoV8b6DJ8heVc/xMhAoGbG4ZrnPGln9r0guCA6HIyevStGPXtHmYLHqERJ9cioNcgjv9MdYpoXx0+ekSeY2s3loQyjacg1VaV1kYhiCDUk37tig42nFUnZtx5/OmWjRXVblEx5hqFtVuD0O36VWhjaZ9vP4GtBNF39GX8ZBUFGZI4Z8mkC85Fa0vh+RU3LIgx/eas54WgfazI3+6c0AAXP8T/nW3o+qyaWBg7x2DHpWIrYpxkzQB6zomsQanAMArIOoNcLr1ktrrjbMBGORg5rCFxMPuyOv+6cVNBJLO6mWV3b1Zs0JiaNeYOIwVYsvcE1UM9uv3Jykv8AdJpJEuIkLJOhX0rLl5myQoz6Ci4rFme/kkXazlvrVVrl+mavjTkkh3+YF+tY8hxIRRcLE29Ty3WnCZVORUA5OCDipPJXsMUrAWDcF1wB+NEc5j6DNVxlaNx71NkBkAHGTSqTTmK9BmmgZ6VYxSAG5xmlHJ4AqPGZMHpUowGwKAEdSWBHappDiLdTDwaSVgyhR0oASGUsNrkkHsR1odRGT0HoBUMeC3HaiRixyaYyWI7nAYAipmVwOOlVUyORU6TuuNzZxSAdHvZTlCKlij3qw3AY55oS6jOVKnkYqWO3EpGHC5pMCNU2/NkEj0pyjc2fWnttiV02lvcUsKkRK20nJ7UAT2qnzRxWkG2jcegqGGFVQNjmppDstmOASexqWwMqV98ruucE96jBQoWLcjtUYuWdmG0AqajxtyfU0IAZVaTcwB+tW4xH5YLVSzkZwaQ75FKgEqO1AGjAji4SRV43DaasXky3V15bHcBSWk0HlrHHE4Zf43PFULRootQ/0g/uWb5qkZainNo4z8sJONg6Cpb7Took823YPERkYpdWtIIZ1NvLvRhkDPSs/Dj7px7dqAIGh28+WBSD5amlkcqFbH4U2OJpQ5GPl9au4ERlwMYp8TJwT1piorOQ+fbFTRRxoxJBJPSm2IYcFtwBA96ersFbDEcdqdKnyLtHJPNRxxq8wR/u96kBhiyAY2JPoakit5Zt21D8vBzU5UJkIhx2NSgSQkjjJGTmgCr9imPZR9TTJbWSIAkowPXac4rQSUCIvPjFVjNskKox29qaAohSDUybh7CrvmKU2kZFQvCyAYywIouAD5XjkX5iO1TyN54IYDB7CqaTBBtYHPpUsUrk7hwvvU3AXc7JjkL6U5FwRyefWkdiOmfm9aN7KQQB+NMklXKE4AOR3qqffGM9qlLFuajGC23IB96aGQvg/dB/Gnxs6QnGAWbGSKkkyFGMHHpUZboDQMVRG5YNnAHanMsZwycE9c0zOOwpKALNvl7qJPVhXc6bFjZXF6bE0t/HjtXdWAwit6DNWQcF4iZrrxHdSbSyjCDHYAVVgSOBSCrH/ep8snnahcSYxvkJxUEsnOB1pFjGyWYk5HUCk3bue9RMXYk0R5JoAnTG4Z+73qYqpG9OMjAqueMe9TQyFlIwSF54HagRdUmGNGU4x2psjC8kUBQoUYAArOEjGfJPFWkL/wDLOTn0xUgVXEgDOeXckcVJboRjPWr0KmLeGZFkI6HvUEK/MaaYFxTttWf0FQRzvMFiyi8H5u9Pv90OmHafvMBVK2OCrmpKNGUK1kFUgsgwMjrV7RtLiv7uzs5PlMs672xkgDmmWpt5ohtGXfruGBx6VsadHDbOt79p2TQt8saNhj680IhmprwuLXX7q8NtawwxRLFA8w3EgdAB2zXBXMtwpaOZdjiQsRtAIb617QdHXUNOtxPJ5rrKs5fqCB2P51w/j/QpBdm9tIv9HhjCv3JbJJ/DmtLEDF1+51Hwi91d3aNqFrcRC0X+JAOM++RXT+H/ABfa6rqLPeoIbtIOYgchgvOentXkavLb3SRNuicjJBXnHapluWtZJpIZmV5F8tmHXFN6AolvXdbu/EOuT6jOwZAdse0YCqOlUrcZkzsD47GoLYqhCMcIeppzyGNiIzg+tSyrD94jMrIoBfg+1Nz825QSPeo0ORVpGVYPm6nikUW4oZLq1aQwOFCEE9xVFDtbNS/bJVi8tJWCnrjvUSsBTQi0WUoAepHHvTUkdfmQkdiDVdJGDjnmpChGWLZyT07UhG7pGuX9pbvaRXnlW0pG5W6Lz1/WvUv7bsdF0i0F3dNIxj+VgM7h9a8ZhcxBHU4KnNbENpd3toLuOIfZn6Zf+XtQI6y9GtXkt1f2d7dIjMGiVTgEEV13hmSeTQYBeyf6YCQQ5+dh6kV5rFqeof2cbXz5ZMjCkDgD611WiWy6Hp39rX8zrM6gFpvvHPamB0qf2lDEsbGLdGxYymQ5wTnpXG6pqFjI8Vu+qSXMKEmRME7m46frVweMre8tr+C4ZQh+WKQLtyDnqK4ZhDB+7hd5GH8Q6VnKZZ0jaybWwmtdLUIX6sy4bFWNK0ePXtHlt0umW6j2kyOCyg+gFcdFLEl0J5lMhAIwWrei8RW8VrF9jhmt5QMFVkG0+uRWXOKxn6haS22oyWtxIv8Ao2Vcq3DelSXX2dbd0tkdSEyWY5zmojf2YeUrbSiWU5Ybg2aZNqcCxPbwWcivIpUyyngD2FKEfeNDY06H7ToIh3qm4E7m6DFUrJ4m8xJlbrglOQuPSrdqsY0KIS/cZgp/E0tzpd3YyymyaPyWjClnAGR7VpVIIPtcENwZmaaUjg54yKk17VEj8NSRwJszMoiVThiuM5PrUdrZgmU30flbioD7Tj/PSnappMd7dRvEvmw7cLIjYGKiDsJnHtHKrsfMIz947sc05pFcYyBxTLqSBZJEQsY1JAJPXFRgSqAfKKhhkbhjIq9wIbiKIguW5+lUWABrQmOICpUZPU1msea0QDWA65plONNzzWiKJBRSCnGmIj706lAyaCKAFTFPzjpUa+lPXBoGT2s8lrcxXEZw8bbh/hXXw6hc2LzLZcx36hoiOMMc5wfbmuLxW7pd7LB4dvZFwXtj+5ZhkqW6/wA6iaHCfKQ67dqoXSoG3RRNumfs8n/1qxz/AEo/wz+NHaiw3LmEpaSjNUQOqSJsNnGaiFSxqCp5xSAuRykAe1TrMrDIABz19azw/wAuT1p8BZmPPAFQ0Mtvlhzn6imxMU+Vuvv2prSFEJYE+wqJGBUyABST0z0oA0EfLYz2/Wg4fdvzuXpx1qi0+2Xe1TCZJVPJwR260WII7qZn2gklB3xVYSFTkdabLK0mSc7exxTgkYHDMT/tVSQxyS4kBIGMdaczfLn3qNlC4yQ3PY05nVhhRimA1pDjGeKA3NRHPanj7uaY0To579qlQ/NuHUVEMFQe/ek3EdKVgLXnP604SyHvUllb+eRxWounRgVBRnK9wnCzKF9Nn/16SWWfynzMNpUggLjIrU+wJVe+s0jspnH3ghIp3FYp2jmTyvNDvDg5jXjP41pQPM+5baGO2jHZF5P1NR6LGsmnhiM/MR/KtSziVfNwMfN/QVNzSxnmxlL7s5Pq4zTsNCxRpFyOoWPNacv7uNpOyjJpsb2UW1FuoxuG4k9SaVyWzN2mdwqyoCezx4NQtYSR3WYyEb1XitW5aymykt3C20ZVs4INNjPmCGQ9WXmmK5l3F1NuAvrdLoDo+NrgfUVkX0hcKMts3/Kr9VHpXVT26yEZFY2u2axWiMRjLU0xlCymaOJtrhcseq5zVn7VL/z2X/v2aj0lI3lZH/u8fWtb7AvZadwsZhvplP8Arov+/ZoF1O3zCSMj/cNa39nqB0FINOU9VouHKZZnuCpHmRf98mmiWT1/KtVtPjx0APuay7iIwOQSpX1BqG7ksDLg+tbmlwrPbO3m+UkpEYyehJ5Jrm1f5gatR3LCJY25QdqhEnWX2j/Z9GkjW2KGBiyXHmDBB9a6bwbb6XNo6xwztFcqf3vmNgsT3xXEWGoW8ekvaxmQSmQcEZzzmtK01W61K5FteSxrbxEu0sKAEEdDkdau9gPSH0pAP9c5rlviDbSQeDXMOWBnTfk9Bk/1xWp4PuZbuK7CtPLbK/7qabj8Ki+JX2mPwd9ntow/2uZYWz6cn+lapgeJyNtgY96+gtK0NLzRtPunuwPNto24H+yPevn0rvZEzjOFJPTNex+BtZkTwzZ6c1yLiVbp4lwfuxjB/Lmk9QOvj8OWoGTdSt+AqCfSoIRlWY47Gr4vVSIruxVOS8RsgtmkBUESDnFNOAflp5dG+XdjPeo3hwMrJuHsatEiHzB2pymVhnYTTVR2OFJJqxslVRlCufWgQ0Pt608XQAAIBwc1XeORjkYH40KHX7wU0DLIvQp5p63invVN1V+wH0pBAQflbNAjRM4FZer2FtrpiW8mmjWPp5RwT/nFSNbXKqWzn2qlHqFvLu82RIipxhmzmgDm/EXhDT7LTxcW13cv821lduMH/wDVWNbeFbGbwpc6s0kv2hHIA3cYzjFdpq17aXOlyW8T+Y56ccVzWlysPD+tWbAFFUPgjoc//WoNbmPNZI9rbzeqhdw61UbTImvmhZnOMYYmt2MK2gxsRnawx+dZkkmdQ83HYDHY1kaJmVdWSQSlOuO9V4Fzeog9c1qX/wA1xK4/3qzVPk3KTBQ3OCD6VJRqRbSLtWx8wyCe1SQ4WJQDkDvUcG1ppCxAXbwSakjPyckH6UWJLTnbaE+4FaWiyb7Z09Dis0spsn3MFx61b0CaIyTKrhv4uPxqySNb0SrPaw6W8sKvhxvGCav3yhrGRFGBtBArGzbWuuXXn3c8Klt6KOA3r/StvfHPaNJE25GU4PrVEoxgMpVmIv5a7ADx3FVlPyURalBakpJvBz/drFGsiaSKQvnYn0xxUMkDbGYWsJwM/dpzaxZs2dzj/gNNbVLN1ZfMkGRjhaokjgbcgOAD6Cnz7DEfMjLjtUMDR4YI5cdiRU8u0wEFioPcVh9o1Il5jFOjUtIEXqelMUqEAVgw9adG225iOf4qoC2Ldh96rIitFQCSJy3cq1LKdqk+gzXNXOpzSSMy7kQ/welUZcxqW+si13qbcsu44Y1Bc6v5l1uj/wBV6Vhm5kZsFjg04SbRWiZnc6u1h0/UVMnkzB+480DNJe2+m2EavJbytk4AEormrPU5LaUMMHHYinXGqTTTFyFIPbrTuB0UUmlSrlYJvp5nNWTBYBA3lXHPbzBXLLcNlXVh9AMVq2upvMEhZC2OjCsmrmkZGoi2Owr9nm57+ZTGgs0wTDcYPQ+aOailZo1ytWnkzYo+0HoORRyl8xXaPTz1iuP+/oqSK101cSeROfrJSzwRCDcFwTRD81s6/SrsZMvFbJsMEmwR2amTw2D2zB0mIA9qI8GJQOwxTnUGFsnHFMRmra6exx/pP/fQpslpp6tj/SfzFOtzm4x60XIxKamxfMXdN1WfS18qzvLsITkRnBXNXZfF2oq21551P+zGuaybcjaM+tTT7t25eM1aM2TzahLfrma/1M57BsCoE06w+8J7tX9d+KTLG3JJzTYslQatMktgtEMR6rfKo7eYKztVc33krc3ksyp8o3def/1U+b75qlOOCfSncLCSeHdEHPm3gz6YNUdZ0uxttNS4tQ7MjfecYOK2Zxm3ik9VqvfRGbRJlUZbYcCkwRzsAimjykmW/u461SQol6hl+4rDIpNJfZeIh7gilnRTcy7h/FUl3PdtLdJ9Nga3GU2jAq/DC7SAbGH4V4rY6tewWwEOpTRD+6G4q2vifWU4XVZ6OYlxO98dWG3TY7rqYzj6dK4uwufIuoJycBW5qvNreqXq+Vd3800Z6o7ZFLAA0e2qUhPQ9kstUsr20VobmNmxjANJIMt1rxaHUbvSbgvaS+W3rjNaMPxC1+CYM80U6/3ZF4qQuesx7UOXYKo6k9K5vxvqGlXfh6e3F9D9oT5oxnOTiuA1nxhf60yq0YgXGCI2ODWIEtD96X973DUgOn0rxkbDSo4PsLSPH91l6Gq9z481ybiJY7dfZcn865PrNsrWi08BeHJz61fMVylSe9u7knz7p3z1BkOKr7Yh1Kj8avT24gXc0CuvcgdKiR7Z/ulB7Hg0cxfKRLJGp+9mpkuIz90/pTzsQZARqgeVSfuhR6Ck5ATiZM5ZAx96abqQf6vao9AarNIMHFMD+tZ8zJsObcSSeSTULqhb5lz+NaVlFFK2371TPYDeRGCPYij2jHYzbUwI6hoGk+hq+VsjyLYo3qWzTvs6wf6xAAfTinFbY9CQfrU8zKsR/Znm6RoV9fMNP/sqzA/euUPoKerogwufzpGkTqzgfU0czCxVk0+3cYgZ8+pqBrCVOpFaYjUjKyL+BqFzg4JJo5hNC2dnbH/Wqxb2NT3MVuIw0aEEd6ijnKcr+IqQ3eewFHMykl1IonLryCMU2SyhlGUVg/tQ0mDxgUqzbe9PmJsRR6fKox87fXpUM1hNyQq4+tXDdPnIkNHnFh1pXCxnosaALJw1StaTSDdFHlfrUy2/myD5c5rbtFWGMKOKqwrHJyRsh2sMN6VGsbMwAFa15ptz9q80kNH6ilESqOUAqiDll2tnAJPtSAEcEEGpopRbkhMdPSkL7zlhzVDGpH825h8tNkwH46VKXQJjbz65qu/Xg0ADkEcHmmqrE9qEVnOAM09V2sQc5oAljtVZW5xx61UP7tzGOQO9WknMMgI5pyxid94QKT1xQMrRpNK4VImb6U54njOHGD6Vd+aO3fbIVA7DvVHeXIzSAfGDG6uQCM1oxm3kz5jEMBkY7VVmUJGuOpGaSPjnuaAJXG4Yq5bbggAqomCauwMq4ywH1qQL0TYGD60XDGQZXoOtOWPcRt6GqtxKsZI61LAqQ2kZaQjAB5qqybTjk1LK5AyvXtT7dJJIWfjgd6aAdYxGSdYVKb3OMZ6VpWeiDEySTss2SRGowP51iWjzR6nDKOqtxW1qWqyxXCSRKA46k85xWcmBTkeazcwyxgxj1qlMUdiyjANb1wYNUsftC4Wf+IE1Us7OWKVJwisE+8D0xTTGZ0cilcE0FwKt60R9ojaJFCFeMCssvWoEjkHmnQyBW4OM9aru5IxTVbnFFiS5IiBz8wP0pqKXcAetR54pVbFKwy2savJtz171EYQJztOab5p7U9ZAOuc0rATrL5Yxmh38193U4xVdpAwxT0YgcUgIJ3IwrDAJqMYBVhzzkj2q+E3IzelVfNTA4IcHlu1WgAyyc7UwD7VYhDvHhmGPSiKSOOAGRCzjo46VahgR7feGwSc9M0gKciqg+cVUllVUOw1fmRZVKlsN2GOtZktuyMVP3aAL6w3UsKysm1ccbqhZ8Hax5o88iNQHb5fU077THI+5sGT6UCE2vtYhePWot3IPoKkd8g4OKgz6mqAkL54HWmhWbOBTQwXrUsUckpO3bjGeTiiwDBnvTldlbgDHvTmxjGDu7+1ChWVvmwR2pga+hgNMCvOOtdaZjaabLOP4EJzXM+HoSkZ3Dqxx9K2fEMy2/hx0/ilIUfTvVEnCRghlYHvmnbs7/k3Meh9KndkbAVNoxTIwGYMD8uahlIBZRmLd5h5GelUEXYxXIODjIrbESBGAkXJ756VkeUyTuG4APWmMk24UL608TvHH5SYGPSnkBGEbsVUjOTUKKoJ+bd+NIdiJjlySAD7VJbSiO4jZycKc8VKkaF89Sac1oQyMMHcDwD0piLV20MsInRCCT3qC39aJUaO2Vcnlqs2yDKgDrUjsN1MM1qkSjL9cVlwOEmwxwoOM1palJsmZcZO3APpVGOwlmjZ128DOCetMDTtmaRN0ALJ2IFbVpZE28VwzrIpzkjjArF02Ka3skxMF3dcHqK2kvlfYm2bYFKthuPyqGS0bV8bix0xTHqbskg+4pyc9hVC58SX81qtvCojiWMo4k+bePSsmeWXyGSPHytuDk/NVe3leR9jnINLnkRY2vFd0kv8AZV5FFErvGpLFCHJHY+3pWHqjWk9p5VrEjSbtzTHh2Pf6CotY1W71Fle6kB8ptqoowFFVIpsValcaKpDDIFMVjuwR1q8+2SN0Uc43fjVby933eGqhj40Bb6VKNrMY/lwfU/yogiYZOOQOtRcvAZCVBB6dKQzTvb83lvBDKkKpCMKIxjJ9TWcEdvuiq6vjuKtxzAIBihAMceUF3HJJ5xT0YBRnP4UzmSQAjGe9PQFTuyNo7mgSRYziIYrd0LU90lrZRqSp+XDN8nc8jFV9KjtjaCSa28yaU5ww4A/pVxrcysipFHGFPGw8/Sp5jWMBILjE0lmY3ChyCYhwPxp97NeXUiJcXMtxFGcoJDwKcsfkoFCqCPSoXDBTk/PgkP0ArKUivZBNCqxKVGGbuTmo/mUlWTK/zqqt557hQxJ7ZNOubvygQSFx2rMLCuIWUtEhQKOhOTio1lCZxwD0qJJJHAkjHBGc+opIyJZFVmAyeSf1osQSSOqPGyhw+zO7dT0G6D7R7kbqfdQlrpDCv7pTsznpT5C6Bo9ykeqiqh8RRtXSt/wi8AQZYupx+NNttaKWCwXsZnaJwEKkAqO2KnuWVPDsO49NuPrisq3faVJUMx6k1rN2JSOh03UdFi3m4ubqfcc+Q6DFFpYwXeri7RsW6fehEZAJPoPYVkwmNbsSuCzeg5qPUNefSb+BbfzCVG+RB8oOegP61EXcU42K/i/RNPsr21OnyulvNuDx/wBw/wCTWDfXxnljDDasaBFGc8AU281A3lw0zjbIzZ65AqIFGGW6+taJEDXlQgqU3ZHBziqMoKu2RwKtyFWKNGpC4zzUEvJJPfmqQFcDceuKXGKB1oHJrQoDQBQRSCmIcKQmlBxmmmkAmaUGkxRTGSA1sQny/CF2+P8AWXCr+AIrF7VtuNnguH/ppcn+tMRj54pDkLS4wKaTSAM0oFNpwFIYuafG5A57UwDNKxwABQA44aQhfu5q3AVSFnY8dBVW3G786mZlaPBJJByOaALKyCW8giUna52H8TUDwRpLIryhQrEA/So4jieEk4xIufzpbri+uFPUSHI/GpsSRNnPJzz19achbov41Gw75H0FPUqn3gDkUWAkfpsYrkelJHGskgQSBSeeaiI/elRwN1LgqcBhkd6YD5EKSFcEr2bHWkUntUhctEAJC3PIqPYSrZIBzge9MBB0p4jO0Ed6YgGOTg9/pVp2hWByAx2jC96BoiLYVV7gc0qH5qiqaJT5RfBIDYNDA19PuobeM+YyDP8AeroF2uispyCM5rnNIhS4uoxIuVznFdO5VMYQ/hWBRGF5qlrY2aY7jt8pHsQRWgGz/A/5VW1YK+iXe5Twm4ZHehMoq+G9p011PVZD/IVpwcST/wC8P5VjeFjm1uR3yD/Stq34lm/3v6UgHTD/AEWb/dp8kMf775Yfu+tDDehX1qI2aYy8hQepQYFNCY5Iowlu22HlfWoIF3Rw/wDAv5mqlqY57cMjE4JBwue5psyRybVkuisQGPLjOD1/SrFY0Lie0gYCW5RfUdTXPa/fw3MMMELbsSBi3Srw8iI/6PEvP8bDLfnWNrchM8fmDc45DcDA9KYBoUqx6pvc4UqRXWBMjcv3fWuK04BrnLHGBkfWt+3e4jVTG/A/hPSkM19lGMAngEDPNVkvyijzoSBj7ynOanS7tpRjcDkYweDSsNswLq4nluvMDgYXdhDxVKRJZAMRkeufWr99bRWHzLKGVwQFA+6KpIVZNwNDRL1Icba0dGja51KG1UKfOPlnIzgGsp3Acgc1PZmRryERffZgqnsDkVLRNjo7/QG0q4FpNdwmZySi9MjjFM0q+fSWmQ24mZ3CnDcqPyqCK+2axLcawj3EiAgNu4B7VAdSWe4lZLdESRgQM5CgdqBHtOnotxpsE8MwiEqbiisCAa5rxzqVqvhq4sl1GOS7V0kRFbccrnoe3BrhbUXsyMUnCx5xtWXYKSXTpRbTYEf+rPSQGtYsdjnbidZ7rzhxyDj3xius8G6/baRDcm4tJJndsK8Z5A7iuIU4NdFokMr2ZZEyCx5yKq4jvP8AhOrLPFjd/iwrQtPFOi3mxXne2kb+GZcAfiK4cWtwf+Wf/jwpDYzN1jH4uKljseoJa20sHnx3e+M/xK2RTjpyy2jGO42MfuyA8A15YLW6gyY8op6lJB/jVzT9b1DTNwiuAyMclJsMKSkKx3dibjTftK3U32gAb0bPJ9f6VeXV3l09HS2Rt4yAZOcd8cV5hca1JdXgnDPBMTgujnpWmmoX0Fg0hvpSpypyN2DS5risd9DOs0SSDuMkd1PoakV/Nzs52nB+tchoE0Elu1vHrLiZzuKbBnPtk082OoSrKw1KXz8ZQhsKT71SZVjrPK5+ZG/KoZ4cphHZG9RXMY8TQQJs1CU88qHzgetE58RJdJbyXfyOMicDIBxnFMOQ3/KkXT5YpbiSXK8knFccIgetWb+fxNp9q1xNqKFFbadoBP8AKqNvquty3iQ74J2YZCSIvIqbl8hagXEwTsa5vVEK3Z/X6V0yQ37XSG4gWGUnICnIOPSsDXVeG43GJm+lJSBxsaWnnzfDrDH3TisQghiK0tBvYW026id1iw+NrnB6Vkz3phlKrGrjsw5zQNA/3GB7jFUjxWpZWWo6xBJPaWm9I+vIUk+gz1rLlbaQf9rbjuDUtDuWhAjhSw5xU6IIxheBWedQeLCCNDjvmrqzK6humagq5eXm0lHqpqbQUVL4gDG5TVW2bfFIvqpqtoF5K2qxozcHrWpmb93eXK6p9jjNrHlN4aYdfarkCyGIeds3nrs+7WdrFm0mpwXP2M3cKphkGAQe1X7O4aaMZtJLfYANsmOaokz7jS57G2EztGyM3GG5/KqHy5+ZVP1Fb/iGaV9NhVok2rJnd+Fck19ElyN7fJ3rCaNou5cM1sjYd41PuKimaNwHhkQkHtWJcTo1yxDcMc1NHNbsgjNwI2PfGcVFmLmNAXGP7o+lSrdsRjIrHnIjKmK488H+6vSiIzBgQj/98mp5WHMaou4FOJXKn/dzVuyjW/uUhtbgeaTkCTgH8a5+Zt8mSpH1FW9NmaG8jlXqtVZhzGvq99e6fcNa3MaK5XcpHRhXOyXBkbcUDKeqnoa9F1u1g1nRJjcRoXWPzIpUPKnGa8z02BrnUreynlaIFvmcdq1IHzSK7blRUPotRZ9a3vEPhqHRfKuLfUhdpKxXaRyvv71jLazz/cic/hTRLQxQnc4p6svY5qVdMue8ci/SmT2M8MZfy5G+ophYXcSMAAj3q5a3n2WIKkZLeoNZi+eOsMo+qGpYy7cFHH/ATUBY6m21CCS1/fkrIfUUovysflrMrJ6GI1kWUTD5ZI2HpkYq+FA6Uyky4NWXZtdtw9ozToL6F38sfIG6llxVQKRTioYYIBqR2N5GiSPb9ohP/AqVZov+e0f/AH1XPNap2GPpW9bWNvNpQJjAY9SKLyCxVNsfN3xyQD/gVK1tO5yfs5+j1TOm8kBwMeopgscttLD86i8hmilrKv3kT22vmqN7e3MEwhNqjoPfFMk0/apZZOnbNQpA00LMG6HHNPnmHKXIrqVmCiMDPvVkRz/880/77/8ArViGB4AS04b2AqWK3Eq7hMw9qrnkHKaphmbrGp/4H/8AWqKW2kKEeR/49We8QgIzKxz6GrEdg1zHuimz6rv5/KjnkHKSQtO0PkCFZEXsX21IIZ2jZPJijBBH+trIKMs3lt94VtLp0Nzpjg5EjKdrA8g0c8iOU4WWzm03U9k2NxJIKnIxTr/5Jw38LDJNU2V4rgiXzS6HByOtaOoOrafDIvPAzWrdwaC2KlBgo361I0ixv80PA7rWdZt++Hyv+VbcYtWQGSXy29GU1mAxZ4nICP8AN6MMVpWhG4bjVL7NZP8A8tkzVqEBdvtWsSWQaou1iR6VRgtJJSOOtbc20vuYZpI5IiwXaB+FEgiRwaXIVwAg99wqvceFZZZjKJ1QHritcRRk5IqXe0fyq3yelIsxIvDQiYN9pzj3rTS0CIFJzVorQF/2k/OgCnItvF/rm2L69aqTRaRJy8iNj8Ks6ku2HOVP0NYUjUx3J7k6SsJMIYsO4NYsky7uvFTXBypqgQc0EEpm546Upm+ToTVcrT495wqEBu2aXKBq6W8CShvOfd3WtkzFv9TIA/vWJaaddpMsjxYVeuDmtwSx9fJSs5mpWmE7NidgR6iovs0ef9f+lWJZY5TyAB6Co2aCPjbms0Ji+XFjAIak+xxOu8SqG/umq74OWAxiomkAq0QXnkSJcKgUjuKpvLv5qtJKSMbzj0qETOp+U8fSnyiNGK48v70Yce9R3E4kOY0Kj0zUXmhoxxzUJdlNXYdxS7joSaF+0E7iPl9aZvoWf+65/CiwXLRkUffGR6VIs6kZAI+tUi241KpAX5ulIdy8t/IoxuU/hThfTP8Ax5rKd/3mVPFSxyfN1oFc00eT+Jjj60ryQMmHU8e+Kp/acev4VG0wY55FK4jnicHnr608hwu4DinzWwj3OWAycgUx3KWzAHGRitUIdkMuTULEBqRJlWHBzupn3qYD9xAyvWnJK2csSTjvSqq7SDUZGGxQA/ILcjj1qwrusR8vgEYprwBbIt/F2qAXDlQh+6KAJ0LbME8HrTGCq2B2NPRwBRKnmMCDx396TQydFMrKx5XGKe0Ch9w6Ac0wP5cQHP4U/cWySeoxUAQ2xY3QA5FahtvMXco5FZ9oh3n2NbET4IGCfpQA6NmtbItIfmXpWM9w0jlia0NYdlt1VTy5wKxckUATM5arKzrYMF4aNxzk1nljUqMssJjYEuOVxTsBeWPbP5iDuDUOoyk3PPGQDSLJLDHyenrVS5naVwx7DFTYC9FdBNuxAAB+dWvt/wDo7ZYKxPSsETP607zW9afKBcuLnzFCdh0NVCc0m8nrQCKuwgNHegHmlzQFx+40bqTFJg0CJVkxTt4Y1WLc0FqLDLG7AzTknIPQVWDZ6nFOBGetIC8shPPSi4Ectrtc4Oeoqurc4p6nPWpGXre4jFkYwuPLHy5Gamt7lI4FjduTzVFGwCB3FKuCwA6ilcCSRh5rYqpdyYYR4+YjNW5OccCojHLK2PKU46N3ouBTSQRjDruWpi1tIoKLsIpJodvBbn0FNhiBJBZR9TTAY5OMgj8arlstTpgyTFD+FLCqmTDelaIaFWMucdu5qQfuuBk/jQRt6VNEFwSwzmk2KxJZDeJHJACimlovLJTgqcU0kQglFwfqaRkDqh7lulIGdZoyERJxR4wk22llBjl2JH6Vc02PaqAVgeMpmbV7dAcmKPJHpk1aIMG4m2zfIAAB0pYroRw7SgOT29KpuCXNSxwSSA/I2F4pNF3LBljKsqsSO2Rimlt4+Zjn1qJbecSLmMhc9acxVnIXpnilYLi3B89wXOdqgCnwiIJ5ZyM9fSmR280kgTbhj61NNA0aiN2y6tjAHSiwXHrF5cyhWDJ2xU0TqHXdkDJ6jrUKgrMtaUtwtzGqzNnA+UkcA1IyjdMr3EZUEc9KvWSZdaz2AN0MEEAAAitayXBz6A0mBh30jNfzZOQGwKmtjKYxsYhicAeorOmYtcOR3Yn9aejOqh1PKmqJOjiVTGisnJHLZ6VVcMk7EswHYVWjmMkJkIAcdFNWornzFxPHuxgAg1m9BDjLBjhCGHQls05LdpIGmQEgds8GrROk4AW2Mjk4znaAPWmyNbK+2JXVR2zmkBgXBZmkdwQxPIpYlJZOOM8/Srt9FD95mI3Y2iqbS7eF6+taIoSZzFcHB+WkM4XletMmIcMx61X61Yi69wSq7NykDk+9QqGYFVBLegpqgscDrU8e2M5/ipWsBEEI6jmlGRjHanyPlyTSCT0oAAWMm5jkmr6KrqInJGSM4qivLZ7VppdLFFhIk80jJcjoKlga2iSKyyNIxbcQEBPYGtNZMOy5AK9c1zRvArApx64GMmoFvWjmMjHJPWsWaQqHTXEnlwedtLjsF6ms67nS7iSLe6bc7grYzms8ag0iOAeCc02OVWPJ5qLFSqETXCW8r+WoUjgn2qJpzL8rAtuHQ8Zp7wIzEuA/oRxn61KfmxgDIGBWlzNSEjZmwpbbtGCM9BUzJH5e6P5xnHSqob58ZA9/WpHnATG7AUdqVgvcs20JgdZZRhv4PpUzF3cl9u09lqsk6zRJuY/Kc9KmWfzZWXHQZzRH4hnR6h/yALYerr/6CazRLhgoTbnvnir2tSLb6Habj/Gv/oNYDanFJlFVuO5qqg4GstwsR/eqyk9DjiodTuoGh86PcbhBgKBwwqi+ouBgEBfQjNV3mEkbyL95Bj8//wBVZrQmTuVHdmGGRB9Vx+VRbD2p7LjGeQeaXJUkDg1siBmXYFMLwM/L1NViSy5KkexqyJVJXao4GN3eoidxx3POapAVC3sBSIcvirDRRgHP3u1VlBWStBjnHNApwAZuacVVc4PQ0xkZ4puae3P+NMx8wFIQop2KQDDH2p1ADa2L19vhjS4cfedn/LP+NY7CtrWzssNHjPVYST+lU2MyCTjmoz1qUHnFMcYapENFPFNozzimMkHHNJ1Yn1o7UDrSEPyV4AyT+VKCWCoAM5/Omk7jg+vFIGKNlSc+tIC2iAFWkHAYZ/Oor841W7/66GiSc+WR7g07Vv8AkKze4U/+OigCqDg9cZpSrEfSm4p2Tg5yeKYiSOQcDrg5we9LICXLMMA1AM4p47CgCZGxwOlOY4IyBnvUS9aV8qdpOSO9IBWI60+MBg2f7jVD1p+SF69sUxk0kIRIto6oGNOhbGfQ9RSLKSAGOcDAp1suXXGOueakDUsobk4ktx83aukt5HcEOhVh1BqtY7zJGWhhbCMRhMYPFW/tMn/PtbfqKyKJT0rN1+dYdCmJGTLiMD6960PtR/59IP8Avs1jeJZ/N0oJ5CR/vFbKnrikiir4Tz/pv+6v8zWu17a2jzGaVQS33Qcmuc0d1SCTzLl4UduVTq/pV6KYo+bK3Rc9Jn+ZjTA0GvruaLzLe2+zR/8APW4OP0rKu76w3M1xNPqDnqqnan5ZqK5SBpyt9e5lXtJ2pix6Wv8Ay9Q/nVAJb3lgYo99pNbuuf3sLnPX0zVpFkKhbS5ivMdR91/yPFQbNM/5+of++qYYNNPKXUYbsQ2CKYrFsXCoxjmUwupBIkGB19aztVZpGjDH5SDx7g1oGe7t0KSN9oiH8MvNZd61s0cbQQGFgTvTdkfh6UAJYrmRR/tE1tAsq4X1z/KsWz3LHvVdxWtnTln1Jp0ihw8SgqGYfOc9KAHl23j5jgAGkKq/JGWznPepDY6gp/49k6Y/1o/wo+xaiVbbZbv92QVVySvJIY0YuwYYwoYZxWK31rYfStQkUj7FKf8AgYqs+hakOloR9XFSwKSPEMI0WffdT0IicNGMY6U+70u8sIEnuYwiM23hsnNVPMx0NS0BpvLFLZyAriVsYbPvTYLZZLYMx5yQapxPlhzV1JGAVEOF6/jSEa+j2FvJa34lxu8sbQTjPWksLGFNEu3mKbsHGHGSMVnGaUDCKGb0xVeaW9EblrJwNpyfLJqkyzHrc09Ul0llP3kfINYQbIrV05wkByM5ORzVXFY6uLT4N9qzL8nAbnsaoxaLcah4ka3tIS6pON7Doq55qkDIyhhGpB9TzXUfaW8P+F1S3Ba8uVDMRx/ng1jOfKtDSKuSz+D9Kiv2+069HDvOfLij3EfiT/SqdxpHhGGUxt4kudw/6dga5oyT3LbrnAf3NV5fvFazh7S12F49DrYNB0G8jD23iGfnoJLYD8+abPpF1ZTxb2FxaO3+shORx6+nWuPXMbB0Yqw7itvSNWuBKlpJM5D/AMRP86U1UirxHFxbszr9L061a6e72ESxHAweOR6Vqgup+SqGkkhbv2Zf5VoxGuvD39mubcUhweYnB6Ub51GBj8RTwwpC2exrcgbIvnxNHOqSK3UMvBqlf6fHcQeXCFgYH5XVeVH51eprdaRRx2qWs2mtH5148rMCVIJ4/OqWtaeGeK6N7KfNA6jPPr1rY8VSQxy2/mgnKnGBn0qK8ktf7KgllwyLjBIrgraSL+KLOVk08bc/aA3+8Mf1pkNpcxMXSEMmMbkNbX2qyPSD/wAdqS2uYf3gRNuV/u4qzMi8OnUvtEy2l41opb5gVBBPPrVKTSp1uJRJJk7ySV7mtbSJ1kvVVQR9RUOqTzQ6lOithQ3TFUmSU105V6W7uf8Aa5qzDBKowLYAe4qA38wXjaP+A1FLqFwINwfIPTikxpmhboI52X+LGCKoQG0tNatmjO5i+PpSaFPvM/mDc+4Yb061mPGYdR83bjbKDn8RWhB22vzzWjWzRzSRRtIVdo+Tio9JvHmvWia5luLcDIeRcFSKvapaPfwQiMruRxJz3FXD8xqgRgamqi/k3O/OOM8VBFAvUxr9cVb1qKD7UryL8zL1zWSWlBwk7KvpmspmiLNxbO0nRP8AgQqv/ZQk5ZYfyNSGK4Az9sz7ZqP973lf86gB8NlLHxGYVUdgtSta3DnPnKKqtwMmRseoNSGGLaCbhieuBQIebSXvLGw91zUBi2N0X8BikkQYUxsxHQ5amqNp6mgZajjQAEbx9GNNmsoJiGLMOehpm8gfKajMh27XbNMLmhJosMlgssWVKnJzzUKQMOBLtP1ogu5IonjLZRhjFV5JYehFAXL3lyAAecahUPbyuBIWDjOTVSJhGwKuQueV7GrU9xFLtZUCkDoDRcLkombuaPMbsaoNdqp5yB61YBNBRP5rHqaTeQahBNNMqq6gsAc0DsTGfb1oWUmmatIuIpFVQxABxWd9ocfxUEGr5rHvTJJpwv7s4P1NZclzg/xfhR9pJTcTuPpmgVy8J77+I59yRVlZZP4ic1iNfsfujDe9Kt7M3UjNFgubLs7/AMRqOGd/LZSehqgt25G0mlWbb16UBcnkZmU4ojFwsYYMv0qPKschzin+YqjG4/jRYaY6O/8ANHQcHkdasLemECRFJIqk1uIo/NBBVueKRZEf5TRYdy19o+0z+cfvHrViOVwc72H0NU1h+zhcNuUniplamUWCQecKT64pd3GMLj6VCHpQ2eKBEm4D+FP++aRpM8MFP1FMJweaaSCeKQ7Dwy/3F/Ko2++CBSK3OKf5Zc8VUHqTNaEkmCoaojwMirsVm067N4X3NQz2ptW2lw/uKuRECAXDKoycCobi43DKNn0wagvXfaEVSc+lVYFkWQdcehrG5qWhqLjCyKQfrUonRl4xzVSaBndW6Y/Wp4oQoBJz7VPMSLIxK1XaEsM4q+EV1wBTPLMJBqvaiMySHaDkVnS4z0rqVa3kxvXJp7WGnyAZiA+lHtwOLbFX7bTYriIOblFz2zW5c6HZTrhMp9Krx6Hswm9T71XtQjEZCkkKbBLvUe+aa4kJJEb49cVcS1FrhQSQOmTUckkyOGj6DrU8xdjM81v9qk3yucAkfUVotfRTYDRKr/3qbI0T4wVyOwqjJlIiWNeHH4VE7sRz171ektmIJUj86z3cK2D1qrAJgH+IfjUogKnLDioS47VIs7tGUz8tUAsg28cVCzYWntvNQlWzjFSFg4ce9ARt3TilQfN8wqXA7UwsQmMk8mnhSBjP505qQtnr1FK4hOccYpRkHNM3ntRuz1oAl8yjeM4Y1EdrDmjaM8dadgMdmZjliSfekMjFdueKUjJxThbsQCcCrEQgHrUq1ajtAqckU1bVmY5wF9RRcBhUgjBBpxBUZNTrEqP6j3qGY5fb2oAbuL/L3NRSRmNgO9OZD1BxTywKgHtTAIySAD1qboKigO6TIU8VPLtbpUjFUFhz2p6qxORSJKi2+wod3rT4WQtgZP1pWAsQbV5fgetTmedo8QRlVH/LQ/4UDyY4t8v4Cqd1dS3DcuVQdEHSixRXMTPMWaRnYDOT2qADLH2NWg+yF+eTVIMQwAGc0ySURliMdO59Ktww7YzJHEzgHG6q8fJxkD61tWCFtNlAxwT3pAZDS+YW4wT2qrMmOTUxBRmYgcGmzqHt/MxyOlAFbjGc0AimkUg4qhEhNNJ5oppoBDwRikzTRQaYDt1G73pmaBQBICM0uRUdFAD8g04cGoxTgaLAyYHnNLv96g3GnBvaiwFsS561NAyluoqmp5p+4DvUOIy+zgDOaVZVMRGfmNZ7OWGAacgUc5w3rSsBaitU8wltxJ96f5CeaykKuB8p96YsmRlXBNMkkYHcDmkBVuUlWX94Bg/dIqDzGU4AH1xVm6YllJPUVXaJsA45PIq0BJC4ZsH0p4cKwNVlba1SgZXdQIsyNk5qSxPm3kSZ4zmqgJbqa09EhR7/ACR0BqhM7XTY8sgPpXE+Irj7Tr124OVDbR9BXe2QCRtIeiITXl086yXM0i9GckUxRBUJfAA3H1NaNy+YwEJJPDEVjNIW61dilLEKeuKRQs87xR7FYeb0PsKhiVQhDd6WZ1lkAQZbufWnrbOWUu6gDt3NAGjFBi38wn5guc1VLAuWzVrzt0flKcJ6VQlRYpSckA9+tQwL6ujR8qA386bJuJAXoKhRxsQMTkntU4BKMcnGODQMiQb7itWN/Js5H6YBrPt1zICfxq1eSeTpsh45YDmkxmBEuZjn1NPZQqlffNa01vaQwJ58JDyDcrD0qiY4v4QTTTEMiYKOSAe2alV3UkkHbjrilQRr7E9cjNSrtc4XuP0qW7isSo8IGDNJt7ELSwyIbhhuO0Drio/LAG1egpxVIbc4+83OaVhC3ES3UgV5Qqg/Lmszy2JOBXqekabplxp1us9nFJKF+ZmHNcRraxQ6xPHBEkaDGFTpWkRmGImdSAPrTTCVj3Zye4qzlFBAYKT1pCp8kEdzincQkto1vBbzFgVmzjHbFQSHB9q1tfQxw2EQ/gQkj0rKdGeFFUZZjii4wK7lBpu3Feow2Vrc6hGnlRi3s4/mQIMFyOufavO7+dbjVLmQoEy2AFHGBQBWjzhuetWojGSQwBBHeqgBA4PJPSnAletJgWuO9SjTbq5tWuo4HNupxvA4zVJmPlNXRzSSSvo+mxyZtoljZlB4JJ5JrOSBGOlpLbookjfLDIO3qKYJIx/FXoD6nbadLON6u7fcQDIBNcU9kkkru5G92LHAxjNJjehVDg/xZpuJy5wQQRkE1dXTk2gKQP72erU17B2UDzNoHBINO5JRjLOCXGR1GKm2hsDYCScDNLM0cREMDbgowABRbuyzxtgsVbIx0zTKTJYrC/R9zWM4j9NtSWTNJKcxlMHHPet6fxBcSQvGoC7hjIFZcAzKKVibml4rRn0exCuFYSA4Pf5a5SCK9lLeVaPcBThvLQkA/hXWeKXZLKzA9/5VLqeqT6TY6dpenTPbpHAsszIcGR2H8XrSqSsWjj5XDSB/LAxxgcHPvU9vAz2D3QkzhtsqY5+taOsIBqkme6qx+pHNUI2MfmgH5ZF2kUCJY9H1G5hikt7R5IivDdM1Ql3JOyMMMOorpLbxKbGxgtEhBSJdpyev6Vzs7pNO0hPzN1rRCIM4NNZgRinugyCGHPHNCQBtxbORwPSqArZpDU7wOoztLDOBt5qNlZBl0YD3FUAsNnPd7/JUHYMtk4wKSWCSCMB8c9CDkGtjw5EsrXe77pVQR+dUtbXyL9IQcqEBH40xlBevShlKsCopA+09M0F8gY4pCH5/OiogTuFSkelADSK2fEa4nso/7luKxsFp44/7zgV0HiNVOpspGdtuoH5mhgc9nnNKxz35qMGpERpMBUYnOOlADab/AB1KYJ0G54JAB3xTCpxkgge4p3GOqaCEzswBA2jNQ5FXNOHzyn/ZFIRXI2nH60Crk1s8r5SMnjJ9KYLO5GP3LHPTFICnLnYatapzqBbs0aEflUUkcrfeRh26GpdQUiSAnvbp/KgCoDzmpGKMn+0P1qP270BHwW2HA60wL1jpzXqOyvtCnHTNXF0SRc/vkOCR0q94Zi8zTZWPeTFXwMhv941MgMaHR5Z5REiO8h6BFz+fNSan4Xk0m3NxeX1snpGDlmPsO9bthd3WnXPm27/Iw2yRno4+vbFQeILe01vE/wBne3ulwPML7gR6EUkyjmrbSry+jeaytpZ4kGdyrgEfj1+lRQW8t3OsEQQSnqHcLg/jXUadruoaBYfYwsNxHGP3R6Ffr61zUsovtXa91VWn8wkyLENhPpjFUmKxY1HQ9S0qNJby3CRMcb1YMB9cdKjsk8ydFHGTiupvvFjzafFaWVr9nV4+DLhsLjAwKzNIsP8ASBLt4HaokI3rO2a3iBMzyHaQN2OM04WLzsAty8P+6Ac1ZVMQx/hT4uDmsOY0RVGk3O/aNVccE9BUV74clu4PJm1R3XII+QHBrUU/vR/umntJzRzFHP8A/CEwsgSTUdw9PLq5DoUVjFFbxy+YMMQcfStMP8wNDnN1F9G/pVcwGLd+DLW9u3uHvJY2bHCoMcDFRjwLZj/mJTf9+1rpmOEJqPfl1HqKOYDm38D2S9b6dv8AgIpV8Dae2f8ATLngZ+4K3b65FrAXB57Vky6nJLn9/tz2Q8VPtJIC3/YFt/FJJ+BxUH/CF6GwyftJb/rqaZbagYPN/elgACN5+WrEfiGNc7kR/oCKn2kgIP8AhHdOsb62s4lkMV2CH3vkjHpVn+ytO0W7jWykl3SOFcSNnApk+pxXd1ZXcWALYkspOCc9hVG9vDf3xnK7WKhSAeOKrmkB0ElhA7EyXXln+7ikFrbr8320cDGAMZrno8QZLLMdwwW35A+gNVWtrTcQDK7+rSdfyq+YOU6Uad8/mecTnsRUkNiyElZcZ/2aqjXEZAnklHXuW4H6VLbapuf94EC+qtmp5yeUr6hYRXjzQXK+YkcfmDsc44qnp3hfSr3RraWaFjLIu5nVyDTr7VT9pkeEcSReX1/Wm2euNaW0FqkAKxrt3buP5Uc4cpIfBmnRR5i847QSdz5zWbJZWTJK8kfyJ0j3c5FXLnU55rgXCXBiWNf9X2NE2pmeFmkmQZ/hA61HOHKVLHdFPAbaFjLL1QHOPxrrDCCpVnkdXXkM5xg1y8eqSGLZC8aOoAJ28/hT01i6VWSWVnB/ujn8KLgaw8O6LnnTYPyNLJpOgwZX7DCuP4cGstdYlFs0auRu6l1+bFVjNHJlSeT1Pc0c5RFOIUuJEiGFzmtrVLuKxGnTywmaJo2QBTg5rnpZbcbkVX3Huxra1Rra50WyVjkq3PHTinOWw6ZRn1yOey8h9OjIDblcdRzkVmapqralGiPHGjqch0XBP1q9E8KbkSMAKcdc5qlqVmkzC7tkCnH7xF7+4pxmS4mYC2MHrT7d2iukkYZOcVCZUDblJ+hpYz5jAhh1rYizvodzp2rrpMsizI8kcrLubP3eK6q2vLS7iEtpKJI++OoPoawNEiguHvIbhUZJI0OGPt2rU0zS7bS1kS3kLhznBYHH+c1pF2NpqxpCSMAEtjNOLLt4cD60LGT2/rSmMr2GPTFbXMxo2d3XP1qK8mFvZyzogmeMZVAetSFYz1RfyoVVU5VQPpSCxwl/e317cIb2COFgMIrKQcVDYzrNG1u0mI1OOmaveNj/AMTq2/65VgxcQuR97ccCuSpH3jWnrc27iKx2oGdmYZ+ZVHP1qube1gkDi4Xb37VQT7QFww/Wklj3phhWfKZFiyvIrO7WYsG29getLf3cV/qMkyZVXxwfpVAWkvZNw9uaaQUKj1qySyQuwKTz607bbLGoKljjJ54qAW0zcF1ZWHBBpJI7lOkTUALHIltJiCPaGPOKfJOrRkgA465qnieUhUT58+tMVZ/OZHUjjmqFY9ItpPMsbZvWOooElQt5jhsnIwOlU7AtN4b5ZtyxELjg5HvVjTZPN023ctuJXk+9aCM3xKhxbOPUisII3c/ka6LxChNgrgZ2ODXLNMyyhGQgH1qJotMuhNg5bI+tI8wHQ1QklWNwpWRSRwD0pjPIOGY7e4FTYReW5H8RXFMkulHOB9FrOLYY7g2O2aYVkdx5cbEeop8ojSFxuXIyPY1E1xIpyenrVPcyOVYkinuzbevHrRyhYtJeNnBI/Gkkuo3bDOQR6VkOTFKHzk55x3q2jC5OUAXHBwaOUC15zx4O4kds0n2oFsnrUUezcUdxgHB3UskcMTDGCD3BosMn+14HtSpN5nKMPxqD90i79ykf3T1oW4syM5I+hosBeISRMGrEUg2BQwOPSsuO7SIngPn1rRs1mvHVba3DO38IIH86TRSJnfYhasiaQvJuJrdudC1pk2rp8pz6EGsyTQdbj66XcN9AKdipMpyTyPgMc4oViasDR9Yb/mGTA/7WBQuh60eunuP+BUWJsVijjlmB+lM+Vm2lsVd/4R3W/wDn2/8AHhTx4Y1p+sEY/wCBUWDlZngAVIGAq5/wi2sesX/fR/wpy+FdWPVo1+pNFg5GZzRzO+UFSBZ0HzAVrReGdVVcCSP8Mmpx4YvnQrIQ30BouHs5GKsme9IZcMT5n4Yrdj8KXasCBkem00knguR33MZUz2HAqOYPZyKztu0wfQVlxszSriupi8MXkMBhBZlIwM9qjHgzUByhiH+82DU8xfIygS32VQVxtNMFwg4JrYfwzqi2rIXhPTnfWfa+Hri+Rit1bqVOGVm5BppiaIfOTH+sX86fHMnrUn/CE3AOTqFov1arUXhSZUC/2na8U7klKSZPWoGuFPGavz+H5IG2tfwN/ujP9ahGhxnl7t93qq8UWAoiRw4OasxXL+tSHSMfdvWP1jFWZLeGO3+VPnH8WauKJJbG9MbYYZp15dpOSnkFcdyazUznipkbaQTWjIjoRspJ4qRbQMm/gN71YeeDZ/x7gN6isu41ArkIuB6E1xyvfQ3uTl4ScOM46VBIVDfLwKxpb0+YSGpn21m6vQqbJua3mH+9+VGCwyWNZ8dzngsB9aJLtgu0Gh02iS6SynINWYZywA3D8awpLxiDhqdb3RLBSankYI6CeaQJgA/hSRSlsZB/GoYmkYY81cdietPbzgNrEEdiKaRpewl4JfL3pjHvWZ9pY8bjT7piSRuPFUicVrGJLZJKd3NNBOOaI5CHHAI96neJXGVUBj3rWwrFZ2cZIPHtVZic81IWdSQ/5VG3NMQ5csPlIz6GnJKEJU9+9Q4IPTimltxpiRoxTbiARlamIDDBHHasuOR0HUH8KtJP8uWY1DiMmMahuBTGwOlNMwLdTijzF9TUpMGRM5H40zcfWpXIPNREj0qxCZNLuJ4pvFGKoBcntTlbnG0GmCpFXjORSAzVGHFTxkM+DTAuenWpYwEwxT5veqET7WIzg7fWm7ZHYKhwp70ACYhnc4HYVbjnigQhQCT2NSAyexRLYuk3PcNWcUrUsbkzX/lMFKn1q5qdsog3LGOPTikmBzwyBzTCNzYqwF3nBBFQybUfAqwHoDHnnGakjQyH29aqjJbJPFXYMtEQh7daQxsgCvgEmiN2jkzioishOWFLbsWm2HmgDR3JPAGfgf3aY1tK6+YwC+xNSQx+WcuuQO1R3k/yncfoKBXIpiqKFUKT371SCBGLepqSEGaUKtWTaqRg/e7UDKqSfvBmtTQZ2a4miY/Ix6dhWTNE0b4IqxaSraJvXhjSAkvMLcMD90nmtPVY7caZFJEign0rIuG83L+tXHk83RoePu1IGRKgHIqLvzWlfJH9milwQSMkLWaORmtEIeNoyaYcE5p67e5pDjdkUANHBpGNP+83J60SIE6MD9KYEYHrRmkzSYNAx+aCaaDTvwoABTthC5pYyuSCBUu4jpTEQU4GkYfNQKQEmc0q0IoK89alWEkZDCkAvBXANAOKjKsrY2n60rEEUNASq/0FSgAjKnIqslXYo+AMioYytPFJgvjin7wUVHAyB6ValTcR6DtUMsPmfOxANNMGV5YlYhlX5abgAY6CpBCcZTJJ7A0sUZYfMpzTuIgH3iK3fD6Zlkb3x/Os0W/PIroNCgVISR1LGmmJmxfT/ZtCunzhimB+NeaCJinA716L4gG3RhEBzKwUVx/2UD7opcwIyvLI6ihlYVrfZvlJK8jtUBtmNHMWVYwdhIHIqW2kKlkx97knvUwgaNsH8qlhRVmDkCm2JorGVvNJwetK02clvu/3RUzWwZmYNj2xUZtGPDqQfSpugGRS+e/zMBt4q2xZAYtwIp0NpFHEoZeQeTSSY3cUcyAmtlzUt3LHHFGkgyrnFNg+UU3UYfMjt22s209qTGZ7XMk332LY6ZPQUitWhbWqqmWXrViK0gbdlOhxS5gMQOWNXIV/db+Q47EVbuLKLyyYl+anyRRsV4I+UdDRzAQA5wrHg+lRSMgUZA6j34rS+zp9lIC5Y981lzWsi7ifXpSjJMVjrYPGNhbhmEUxITC8Y5rk7u+F1dyXBXbvOduc4qt5Eh6Co9uDg1aYEomB6Kp+oqe3bfIkW0AM47VWACjOKtaf8+owDtkn8hVCsT+IpN16i/3UArKWQrsYdUOR9a1dVjSTVZQ/QBR+lVPKtEYrsBPqRk0cwHZ6frllZaI8csyNdSAu5zjcxGP6Vw7kg5FTvCdpaKJsj2pFjU8MPm7ijmArmV/UD8BQJDnqD+FWJLGVzmMAqaVbLyEJuMBhRcCzb28cigsoz9K6HSdDl1S1LW8yB0OGVxgA/X/61c5YvI95Gq/6rOSa1RuHSR1/3WxWTDmK99E9mWbzkkOSuUzgEVBHMMBmPNSalu8uGALwFMh56/5xVIRsSSeBSsJO5c+0OSQrbVx371DNcFUPzA5qkzguVU/Xg0kEIZvMnYhM4Cjqaqw7DY1eSUbATV2KKWInLAD0x0pVZY8LFGwHsOlCTGQ/LG31Y8Gi4yyJF6E89qsWvzS1ntJJnAQHAycCrVkL1pM2ywHBGTIaFuSafis4Fiv1P8qztZuFk1NiP4URP0qzrbXzX2nRX32fnGBF7sBzVDUl3eIp4P4PMVf5VNQtEmrTGfVJnC4xtX8gKpEE9wPrRrU7Ra5eInCiT+gqj9sOfmNWIsOSB0U1FkHrSpNuHKhfpUUrKrdetUiQY05Jwp5APFRbs00jBxTGattJGbCU7/nQ79uOo5rT8QWinRY+f9XEjrx0ycGuZR2ViqnG4YP0rpJtWjmhWGWIOnlLEwLDqKVxjtAiVNNMiFSr4wQcnj1rC8QnOssPSNR+lamnT22nW7xIW2sc8npUEsdpc3Ty3JDPIcjaeAMcUc4chz5oArcFtpsfLKjZOPmbOPoKDaabJwAqMfusCaOdDsYwX5lPvVl8biR3NXBb2UKKXWRmz2PBqQw2L4I8xc9ietHOgsZ1um++gH/TRf51qa3Or6vchecAR/TFTWdnaPdW7xp+9Ei8f5+lLex2cuoXTMCHMrZwaOdBYybazwQ7fgKugFeFOKfsSKYLn5ccUqS2oXbJvdx12nFTzXCxNHcARlWwSOpNOE8eMOpIznkcCq8YifLnCnkY/wDr06EF9ykBWUZC+1K47CTWNhNltrq57g8flT7axji3eS+cjncaPLWRQzylPoOtNMfyFQ8h9xzRcLFpIMHmUZHUA0sgeNMjBA61WgYp8rxsRg5JByaeLqZZgPLlaIdFRDzRcTQonyetPuoLebyPOTdlTjnpzSmJ2LNDaTb2ORujwBRNHd+RGI7WcsUIyIyccmi4rDPssNkg8pNnpzTfPk9j9RTkt794QHsbn2/dmlSz1hGylnOIvRkwTRcLCRX0luhjXaiE5IA6mpY7t25wMVXn0+/cbk0+4OOvy1LBZ6iU2nTrgH0x1oCxo28u9PpTLmVlxjgdzTILTUkyBYTDPqKnl0/UZISPspDdgXApJjMly5znmqrWrSPuArUTSdX3gyW8bL6eaBU0fh7VJs+Xbru/urMMmmmMr+SHjh45EYArX0sjYF7qcGr1p4Lv5cefeQwPGADEg8wj+VaaeE1iXD31x/wGPbUt3HYpt91QOuFwPzoi/wBWp+v86vr4Zsxy11csfXOKsroViqBd85x/tVDQWMhuGRvXNOzuUGtGTRIDG3klw4Hy734JrI/s7xPF8qWttKo/iDDBpWNFEnBxQG3XMRP3RnJqm0PiZfv6XEfpTf8AioE+9pGf+An/ABosXyR7mi8vVawvEGrfYLZY432zP0PpV3dr+f8AkBg/gahnj1OVt1x4b8w+u0mi4ezj0OWl1ItZHzZZJGbuXAxVNLtmVXXjJznPWuseCUqSfC44/wCmZ/wpiwShgw8LgEdD5ZyP0qrh7M5+O+lDFSwKkYAHamwXrz3CxFVDN1Yda6UW8rf8yx/44f8ACg2LN/zKzfhGaXKg9mc/FcOtxsZg57YNSG5VJZSHwShAPvW6LJ1+74YmH/ADSfYZif8AkW5PxQ1VkT7LzOeS+Jz95sd8cCo47jfPgHrXVLpGoH7nhf8ANTT/AOxtTPJ8KjPqENFkHsvM5aWfKqHP3RgfSoBc4jKI78nOdxrr10DUz/zLKfr/AI1J/Ymsfw+G4R/wBf8AGi0Q9l5nHx3DhNuc+560ya7dl8rcRjuCcmux/sPWwefDkP8A37X+hp48Pa43K6DaD/eAB/nRyoPZeZxkd4fkBkPyjHzHNSebH1yPwBrsI9A8QOf+QXaL7NsFXI/D3icDCW1ig9Mj+lFg9l5nDxMrP5i7y3ptpZY7mVtyJIvoTGa7ptA8Ugf6uyH0kA/pTV0HxSTxLbp9GFFivZx7nEpbai/LwSsfUKasRWGoOcixuSR0CjrXajw14pYf8f0A+jUDwr4mPXVYR+J/wqLB7OHc406JqssZcWMqv23YA/nWh/Z13d2CW0OA8a/MWPGcYron8K+IEQtLq0IUDJ69PyqnY+Fby9g81NUROcEYJosXGEI9TBj8IawckzQofrmr8Hhu8jTbNeW4buQpya3x4FldNz6xMD/sRk/1qNfBdp/y01m4c+uyjkJ9wzV8NWgYPNeJu9kXB/OrLaNpK274S3Zwp+bCirq+C9J/5aX9w49OBUy+C9AP/LxOPrIP8KNSuamYdrZQY33ZATbgZPf1qyqaMnWRPzNXtJsLS/kmhvF3pHyB75xWi2jaEnW1B+uTUxkXKUDnzcaUv3biYf7pP+NI+pqFK213d7+3zH+tdEum6Oh/dWS590JrQhtY0HyWsQ99grUw54GAdM12REaPW8ZGSGjBpy6b4hUf8hmA/W3H+FdRHZvJwML744FWF0mVv+WyfgCarnkZcx57qHg3U9WuBNdatbllGB+5KgD8Kxh4av1vX0/z4PMBwGAIBr13+yUB+e5x9FrlLqNbXxqEd1ZCQSR9KmbLhI5U+BNeUEwz2sv+wsnJ/Ss250PXbLi402dPfgj869XutU0az5NxGG7ENmsifxnGqukMf2hCMBXHFBnzHncdpq6qV+yuCfQ4pGsNTbO+0b8SK2jJJI7uF2hmJ2g8ClVnHUmqsM586XqhPMP/AI8KsJpmpmLYYl/FwK2jI3qacsjY6miwGD/ZOog8RRj/ALaCpRpmpNzKsJPqGArbyT1NB/GqsK5Lo8U0Omvbzbdy5GVOQetVLVNQ06L7NHd2TRgk4brWlYHhge+TWHBbRnUbqGWxM+Jj+8B+6PpVIlo1tVi+16bLEGwzYwa5waBNtUG6UkdCQa6qQZjNUytJgYTaBM/37xB/wA1Iugusexr0t6EL0/OtgJTinFKwzCbw+GUq107Ke2wUL4fA6Xkw/AVuBaQrTFYxRoEH/LSVyfXgUHw/bMeZJcegbFbOyjZTKsYp8M2Dct5x/wCB1JF4fsIgdkcoz6yZrXC04Jmgmxlro1ijZ+zhv94k08adaqcpAqn26VolATSCIGpGUE0+3Rg4hi3DnJWp1t7MuWlsbd29duKs+SKPKHoKAITDppGPsCr/ALmKQWtmrAxTywY7BBU/lj0o8ugLiCEjBTWdh7F1YCp1h1bYWt9ZtZfTEhGahEeKNlAXHtL4uRNylZMf3WBFM+3+IwgaS1HPViRTkMkTbo3Kt6irkWqXqOC83mL3V1BFAXM+aXVrpQHIiPch8ZpiwaiP+X3/AMfraRNJv7rEiSwyt+A/OkuNBuouYTHMo77lBoC5m/6ftwb1V+rCk8u4ZcHVCD6ZyKCCpIYYI4PFFFg5mLHbsg51Ft3qCcU8KwPzajIw9AKi3CpBFIwG2NyfZaA5mSecYuUvJx+FUri6upsh7qVge26rE1ndGIlbeT/vmuana7sJ5BOm1WOQc96ymjaDNPzLxVCiZto7Z6U155QpPmysR1G8ispdXkBwFFSXE01xGJIUwf7uf61jYu5uWcltPCGdbjdnk7zUdtHAt5MHV9u44DVlaPd3a58yMjnAzxV15tk5fFboykrmrIbGJd32ct9GNVJry1XLLGEX+6eaS2ufObbuRT2Dd6stbxvxN5TD0rmnN3sjTkK1tf287bdiH8KtkI/RQB7Vk3FgIZ82o2ofxrRt7lokw1v5n41cJGbgSeSmOlNeFDE/HanyXczECK3jXHrzSC8vBhSkA99uK3U0Z8kjHXC8HqDUjcrkVFMjCUjByecCpI0kKdK2U00S0TSbEtdzHArCnt2uctEMk9q2THIRgoW9l5qWCxvrh9ltbS/QrismNI4qazukkO6B6h8mUH/Vtn6V6SvhjV5sLJHEqn+JjnFW4vBs2SJLmIe4SmpFWPMRDOPvROPwpspK8EEfUV65H4Qt+kl1Kw7gAVaTwppKkF4Wlx2cg0c4WPEWapIHUNk5H1Fe3r4Z0Uf8uMdPXQNEU86fAMeoo5gseORyRNyTk1J5k44jLhfTYTXsy2mjRcx2tqjDuEFO36ep4W1H4CpYHiTrM3GyQsemUIpF0+9k+5bua9nvp9IltylzJb47YAzXJfatK0xnd98yE5DZxQpCscjH4e1d1BFlIue7cCr0fg3W5EBVYhnsXxW5H440dX8lLaZQO5xiql541bBS1giQdmDc1XNYDm7rQdSs5fLuLfafUNkGoPsDKf30nlD1pL/XNQ1CbM07bR2BqltJ+8zH6mrAvzWlkkZZb7cR2qgwQH5CSPU1Iir6DNDAEYoEVw5DYOKnEgxioTEwbPUU4j0ptXAcXFAcY68UgQkUoQdKVgF3g96DimlQp5Jpu8DoSaLAOzzS5phbPak3HHQfWiwiTdQG96iyaMmnYCLcQeKUuxHJqRokY55FNMaqcdaBCh9sdKrZ5bim5ToevpTioxz0pAJG5juBKh5FbkGqwXUTwT/KM4z1zWETEPu0qBQ2SM1NgLE7RoGWIAjsaqBl/iUk1O2CcjgVE4/u1QDOvFWYZNkAiRST61XEEsjcA/hU8aSLkqDxQMfn92T3pdOh/wBK8x1zULDEm41o6Ynnl9sgXYMnPegCyx86XpgdxWFeSmW6cAfKDge9b8rrbwySHqq9K5wjfIc9CSaIiNCyiMEfnH7x6Us7YXPemRysWBb/AFKjGKaZ1dyduV7UrDH6jxBDJjkjBrPXdJKkaDJY4xV+6kWexVAfnU5xUemCOCZ7iY4aMZX3pgSajELVI4Qf3p6r6VPC3kaeIpOM1m7pL3UTN2PQegq7crJJIUHIHQUgJXtDPCoMoEI9DSRzx248vYnl+45q49rLbWoQoTkZyKxr4FcD1rMB160EnzRLiqFOYllqPBFbxACabTihwT6UymIerc0pINMHJp4jbGcUAM6U7NKyHtUsFtJMSF7UDIe9O3MKs/Z0WLcDkiotuaLgNBp6kZ6U3b6UKrHGFJB744qbiJQ1Ow+3cEbHr2pNvFTKXkKIXbb0pAVwrn5z0qQID1qWSPY20twBx70wHFFxksKqv3jT3fDYB4quWycehoLc1LAtrJnqeaHcscVADge9ORhu5pJASqWVsYIqwFwAagDk9afv4qRj2ccCug0tCIE461y7EmWID+9XZ6fEBGo9KuBLKniZ8tbwnoBk1z+8KK0vE10H1YovRI1GKwGmINEgLXm/LjcTURkA6+tVDO3c0gmz1NKwFl5FJzmmiXFVTL1FN8w+tOwF4Tkc+napRKCdxP0rNWTkVKH5pWAvh8nrmmsT5owKjjbPPenqCz7s8UWKLka5OBViSRjNs+UBcdarwn5xViWaJPvf/rqZAIdgyT1phugvy7sj6VFJNHtJ3fQVnNcfOcVnYDU+0fN/s4pI5gx+bHtis4Tqfvcj0pRNjvRysDW8/I+UYA4xRlXb5uhrPWQsMknFWo5QACAD9aLWGWDDERhI2J9qpSWE5OUtJ2PqgrWtZmTDxNtPqBWil3dsMLfNGPQiriwsci+n3f8ADazH/gBqawsdQjv4ZPsMxUHB49a6djO33tQJ+i0xoy339QnH+4MVrcLHPXun6rLdyyfYZSrMcY54qaC01RExFpLn1LJzW39mhA/4+bs/8CpyxRbdvm3P/fR/xqZDsZQi1jaVewl59qT+zdQmA3WUmR0J4xWuptI1wJr38HFV55rYLlJL1z6EkVJViidN1eLCJbFfcSAGnnRdQmRTIg3ekko4qYXaocKLoD0FW49joGZ5hn+8aomxk/2Xe2JEkzw+XnGEbJFOJNXGhjuHYi9ihAONshyaBpzHn7XbMvrv60MxkW18H61qqQahai3jhMYCNLLgkDvjFTJ8O9caPzHu7P6Ak/0qKNp1iSJJ5tiDCgSYAFVbqbVEmT7KJpMDL/vDwDSKiZAa6YERWjuQSMpHkVElpqCptXTrg/VcV0sUkvlKGbY3cZxTLlpjAQrEnqc1JRgi01VuljKv1FRyadqhXBspT9BW7E0wTdIwPpirEUzDJYEgegplWMOz0zULiXbMn2ZO7y9K0rPTzp8piNylxuOcoCAKsvdQH71kh/4Cf8abDNam5DKnlL6KlUmS0WrqwivtStbiWdomhxtUDIYA55PaoLnRbI3zXi3ErMXD+UwGKsTXtmSvlzRNxg70PBqLz7c/8trT/vk0pRuUivL4W0vUZpLua9mjaRixVAMA1WbwvoUTEG9vWx6KK3rPTp7/AD9mexfAzgSc4+mKbdeHteY7IrK2GOp8xabdimzCbQtAXpJft+KioxouhZ+5fN9WFdHFoetRKA9jbMf9lgae2ma2nSwj/BQaXMK5z66LoAHMNyB/tP8A/XqUaN4fP3LVn+sta32TWv8Anw/8hCql3LrFowVtNYsf7sIIo5guVxpOkjppyn/toKlWx0teP7PiB9M1k3F9eQ7RMk6EjOGiIqkdQZGdlkZXY5JKkVHPIq506QaOHCNZR89QCRSSNo4n8q10+NsfeZ+Rn2rj21AFiJZA59RVmO9ga22hm2/3RxU++Xc6W3utLliZ47GBipxtCVaXUdLQZl0Ozx6kZrlIJSG2xNsPb5sVcOoGF9jss3oCOT+NZcs0Fzo4vEKwODb6bZiLsCmcVJrN9Ya0IDJZLH5O7G1OucemPSsH7RFPAQu5W27gQucGpkvFMS4Vh9eKcJe8ZyZI1rp6PG8NmqyhwNxzQ8OnM5Y6ZG7HqWU5Jpnm5lt+GOXzwPQUn2qMuFB+Y9K6bmbZYEdon3dOtwP92lzbHrp9sx9TGKYkhfO0g44NO3MKpWAlE6AYWztwPaMUv2gZz9ng/wC+BUG80ZzQBN9qYdIogfQIKPt8w6JEP+AVFijbTAl+33H+x/3wKb9tuuzgD02io9tO20AO+2XX/PdvwAo+1XJ6zv8AnTdtGKAHedP/AM9X/OkMsxP+sf8A76NHSlyKLAJvk7yP0/vGjfJ08yT/AL6NLRRYBp3H7zMfqxoKk9Tke9P/AANH4GgCPaewFNVJ4pUmglMUyHIdamz7H8qUfjU2A0rLxLqtnnzEhuPQsmCK1Y/G3/PfTGx6hs1zVGKbQrnWDxlp0iZksLhT6BAaRfFWhMf3kF0n1irk8UoFFh3OwHifw6R924/GOj/hJvDvpP8A9+65LYDR5YosVc6tvFHh9ekdwfpHSDxfoidILz/v3/8AXrlfLP8ACKXYe9FgudWPF+jH/lld/jH/APXo/wCEw0sfdhuv++P/AK9cr5YNJtAosFzqP+EytQf+PW5x/uinjxrZf8+Vz+lcrgUYzRYLnVf8JnY97O5H4Cj/AIS6z/55Xg/AVyu0UbRRYLnTnxhbdoLw/lUbeMlz8lheP/wPFc5kJR50frRYDof+EzP/AEDLj8XFL/wmR/6Bs3/fY/wrnGljbvW1B4fsJ0X/AInsKO38Owjn65pWAmPi6X+HT2/GSo28WXf8OnxH/el/+tUkng+7gBYSvKg/iiUMKpNpcSHEl1MjejQ1Ng5SwfFt6P8AmG2//f3/AOtTT4vv/wCGyhH/AAI1ENKtSPmvpD9I6cNLsh/y9zH/ALZii5XJIQeKNR80yeRb8jGATUn/AAluqf8APCGmf2ZZZ/4+pv8Av2KeNNsiP9fcH6IKOaI/ZSE/4SrVD/yxgI91o/4SXUSc+RCPoKDp1hnH2i4B/wB0Uf2XZ9r2f/vkUc0Q9lIP+Ek1PusQ9ttA8R6l/wA84vyo/sy1PW9kP/ABR/Zdn/z+zf8AfIo5oh7KQ1/EWryRvFmNVcYOF7fnTIdQvLFBDCkewdNygmkvLS3sYo5o7p5WZtuxhipvsNtOqyPeyozDO3y84/WsuYvklawv/CQ6sOB5GP8AcFH/AAkOpjlUhDeuwUf2XajpfOP+AUf2Zb/8/wC//fsVrzxI5JCHxBqjdY7b/v0KcPEOqkf8sR/2zFJ/Zlv/AM/7/wDfsUv9mwf8/wCB9Yz/AI0c8Q5JFS0nlhSaaJgsjkhvlyOv/wBelGpaiDkXGPfyxSW8SM80HmYw5G7FWF0zP/L6g/7ZmuelNal1IS0Gf2xqgGPtTfgoFRtqWpP969m/OrP9lf8AT/EPqhFIdJm6R3ls59N2K6OeJlySKp1DUV6X9wPo2Ka19qEg+e/uT/wM1b/sLV25W3SRfVJAc1Wks7u3XM9tNHjruQ4p3BxsVz5rHLTzsfdzSwx/vjksxI6k5oEvYMM06IgXIz3FZV/guVR+Ii+zxjonI70vlgdqm6Mc9ATSVpF6Ih7kewelGz1FSUvBqxEXlgik8sVNijFAiIJTigH+FP2mkwfWmA+24c/Ss2+S3gvpGGqPbPIAzpjvWnB8rkn0rI1Rbj/hJIzbwwzeZCBslHBpjNO2cSWwxcfaP+mmOtNIqaMbYgDGkbEZZU6A00rQwIsUYqTbRipAZtpccU7FGKAG7RSYp+OKbigQmKPpTsUmKAEoBxS4o2mi4CbsGjPp0pdtGB0oAQnmjdS4FBAFFxibsUbqCKI4ZLiZIYhl3PFAhrOBSCZT0Ut9BXS23hu3jbdcO0jjtnitiK3toU2pbxL7haAOZ06ztHdJZJbiOTsvlbgP1rektZTjyYbaQerrt/rSXOp21o6+ZgKTjPpV+aD7Xp5NpKh8xflaoA5efTLS23NfXqJk5KJ0FVX1DQrfCRW7XLDuAT+tZc9lew3jQ3oKN/efgGomhCHqp+ho5i7Go3iIoP8AR9PRR2J//VVWTXNRuYvlniiz/dXmqE1v567d5Ue1IljKgItp9vswzmp5gsR3V3fwxmQ38hUddpqh9l/tGESmfJb++ea0CJSzJdCI46ikSKyYlYZGDDqgPSlzAZR0OVAWE6H2rRtCkUIic4cdsVYCjGFOao3LxI/Mbbx/GORU8wcxdfC44PPc1DPgwtkgcZp0TJLEpSQE46E0zdbykpKqkj0rPmDmKdpPH5X7yUK3oaS4kBJkS+UY7E1NdWti0R2sPxqqtvbSw+WkRJHqcVRfMaen37TQ5L5weauHUYk43YrnbO0urffsHyscgZ6VcCBhmRMSD3rNxKU7GhLqKhC0cgBHrWPHfNJdiSU5arSeXIwQck9qzZbOWGclSKcUPmN6W9ghtxNs3N9aLTV7e7ibEDAj0rDktbqSHduAx2JptoZrdTtcq3qpq7GfMdFZXTNeqhZVdTnB711smtwW9qGkfLD+HpXmSSyed5kr7vU1em1Hzk2sQx7FhzRck7mx1pbubYPlHrmt2NcivIrbVbizuxswR6EV29h4njNuHuIJEz3j6VakBvXN/b2kZd26dqoDxXpu4KyuM965LWL2S5kaSN8A1z7z3Qfy9ynPcUXA9hguYLlN8ByKSWEP94EiuX8K3jRRpFKOG/iz0rsOtFwMuXTI3fcoxXL+JdJ1AxBooA0fOWj+92ru8ClqkB4rcRta5wsyuP7ymqU9xM67Wc817jdfYRbs1zHGT2ygOa831C505r2TEUDxHoAtXcDjhChOSKf5UYHSup07Q9M1e6CgvCjfxockVrTfDJutrqZb/Zlj/rmruB528YHIFNx7g/Su5bwBqkDYbyZf91qnttDubBSZtJWRfUjIo5gscNBBPKSY4JHUdwKs/YZHHzQsp9669i9ux+z2jQoedi9M1UuJpblwXTGKy5wsc8unYHz5zUMtmA/AroyF25YcVTaEE5xS9qHKYRtz2FMMB7itxrb/AGaZ9nP9yq9qgsYnkEnpTvsZb7oya2jbo6/cANM+xsOeceo60/aoVjI+xOOqmopIGQ42k1vpEFGD831ppt4+yml7VBYwkh3DO7HtTjB61qmzXOdpHvTvsqj1p+0iFjBWKQpuVcilSNM5lc5HYUm5+zGl2nqaszGvHHksueB1NV97HqassCFqtjBoAkQxg42sT7VNlNmOQewqNHYDGBinjkUXAiLMTxUyxlQC5Az2FCKoOcU2TrmgC1DKYpFAOFqy0yuSQy7jWQHZmFSFkDh3JGBjiiwE0q5JzUcJkglVowc5p28zKXToKlhkVJQZORSHYtaixa3UH75GWrJjXK5Nac86ygqqrjsRVTaBwKEFiJnfbsXhe4xSYIWpGGDjvSFSaodiMqxXJ7U3HympGRh2puxtpBBGaRIWrFHDrWxaTxqzvIoIP6VmRJsHTipATk4pMs6C01Nd3lqcRmotV0OC+HmW8nlP6dqwgWHerBu7kj/Xv+dZiM54JIHMcikMpx9absFTOxZssxJ9TSbfStFIRXZcnbimGMA4xVlUx9aXySxz3qrgVUjYN0q6gBQA0ogIAOOtSC3PBzz6UXAjNtkHHQ1NaKokKDI465qVBg7QRk9qcAFfgAfSochEXkKkxHLIfWnPHGIzgKuOmRU/B60jQqxzWfOwKS2vAOCMHn3p7oDk4wo7elaKRF8RoMg1C8IViuRke9VzsdigxIAG0ACmq7qdyrz2NTmGQucLkU6RT5e0EDHaq5gKpLOcE5Y9B3NMdZEYh0ZfrU63k1qp+zoFlJyZDyajSS5u5wJm3Me7NTQEIyGzTyc0jL94fxL19Kh8wDvVCZYDYNKrYIPWoQwIpQxpAWt5PTinA1XVieKlAPUk0WGXbFd9/D7HNdrYR/KK5HSI914GPau1t9qQSzE/dQkCqEcBq05m1a6fORvwPwrPZqcXeV3k2n5mJqF84pDI2bIpqtzyads3d8ULC74CAMT2U0xNDWam5zVhdPvGDE27AAd6UWMxVSi7s8EZ6UAQLycVYRccHqDTns5Inx1I7VYO1mXERUheRjvSsUCBsdKsRn5aqq8zMFEO0Z5LHpVtYHY4SUZ9QKVgJ4cmVR6mmTOgnYMeRTVtbuKTzBMrFOQp70q2r7y8+5Q45K0WKKcsmVxz9RVWSQDgVsC1tY0wQ7/VqtQNbxWm1rdT9OKLAc15mTUyEnFdJFcWCoMWIB/2j/8AWqwNRgQfu7EZ9cf/AFqLgYMaSKmNpPuKv2ts8kedpH1qzJIt0oIYIx6gngVageCNNs92JCOyLnFZyQEdnG8cmx1xjv2NXiifaAAuT2FVzd20fCB2X++xxj8Kel3aO2RKGb0FJKwEpjXcfMU+3NCmOTPlrtx3xSSXtqCNwfn8aVpYvL3qxTBz61QxgfEZO35enIpwWQqXDk7RwhHFBuLYNwWaRhxkcCkN5Aq4E+GUcqoNADlWSSMsUXJ/hUU7YVXk4qNXVo1O4FiM/Kxp0DW4zHvLMxyd3SgBpVkALS5BOPlpfsMcm5rhmVQO3XNT/ufM2JIOOgppkU/flVhu5PagCJ7eF327T8o9qFs7Jo2G0xyAZ3EdaV/LBZvlXd6Hg/hQryyDiISADAIVjQAQRsJcqqspBAzU3l7f+WcY+nNOh07WJl3RWblfUnFTw+H9XeUtKixZ6kyY/TFK4FYgeiH/AICRSNtHAHJrX/4Ru4/jv4vqFOani8MWSsXnuJ5WxjrgfyouBz+4D72PxHFRytCRtWQ5Poa6waDpC9baRvq1WYrDS4lATT4QR/E4BNAHCSkR7BbZkYsNwKk8VoC1urtgttYyFB1IXGD+NdwhEYxCsar6KAKfvY/fYfnTA4xPC2ouu4iGP2eTBq5beD1Dq11e/L3SJev/AAKulO0/xD86TIHRx+dFwINO0yx0mJksbdIy2N8h5Zvx/GrRZgOtMyP7w/OmFSf+Wjf980mA8v6ml+hT8TUDRn/nqfyqJkftIB+FIZa3OOjL/wB9UGeQf3PzqmbGdzkSsD6MuKaNOvHbaJUHv0oFYtvK7jBRGHpkf1qpI6p0skP4L/hTW0S9LhgwlYfw7wBSSWWqW0butkhZRwN24E+nWpuVYqzWVpeZ8/TLZs9SyDNczrug6A0SfZN8M+ckRnKgDHWukk1LWvI23fh2B0Hcf/Wqkb21PM3hmAN6ncP6UKY1E46fQQ2z7LcMrZyRICcn+lRfZJrW7/0hlKx/MdpyDXSX4F3cb4YBZpjGxM4FUjp8cok+0XTPvOSU+UihyuKSM933A+U5+chRjjIrQlMENnunkQP/ALIqrMEt7f7LaQM5/vE5P50yFZb238q8TysNlGxxkVDRm0WbOWOTIjcsWO3BpktsyMzpujQEKCVzg96F26flIikgP8VPjg1K7R98U8Vu7BlkboopCsW3O3TRKiZ2AsRu6+p4pmlXDG8R1iSQMWOHGQePSi3tLjTMJLJbspb7w6kH1rRtfDN1c3Qms7ORNpBDlgox9DTsVYma/wBKkcpf6ebYj/lpbNjH1FbE/heJrVJLCZ2dxkCV+CPwFT2nhuysnMt/eCeUYynAHtn1rW+05I8lvkwAK15gscZLoupQNhrYlR/EjZFVtiQki4W5RuwVQP510ls1/HrNyklyghXLrH3Of/1Vp7mfiTDr6EcVXMOxxO+17NdH8VpS0DY8tJc9y5H9K6i50KxuUYpF5Mp/jT+o71yqrjP1IppiaFpVC7vnDY/2etLSjpVEi/6P/wA85v8AvsUYtu8Ux/4GP8KSii4CbfQYpMGn4pcUhj0NvjEtvNIfVZMCjNl/z5Tf9/6bijFMQ7zLMdLKQH1MxxUbnc5IXavYZzilopAJj1p8flhsyqzL6KcGmUooGT7rL/n1n/7/AFG6x/59Z/8Av7/9aosUUASFrTHyW8yt6mTIqKlxSigBtSRNGsgMsZkUfwhtufxpMUYpAWftdh/0Bk/7/NSfarE9dK/K4aq2KWmBMbix7aUwP/Xw1QHqTjA9PSnZFIaQxuKkUCHDTWwlVhwCxH8qQClx70gLEGp6dDKDNocbr6q2T+RrobHUPDku77OsVpMRjEkeDiuW2imGFD2oA3tR0XU0Tz9Ps9OvUPTYNp/nXPteXdqg/tTSXt/R9p25qWJ7i2bNtcSxem1jj8ula0fiO9CbLqCG6XuHGM0rgZ1tr/l4e2v3hPoH4/Wt2PxRqJjG5LWdf7zryaUweFtTgSa7srWDcM5VypH5AVkal4e0LmbR9YFtKPuo8hZf1qSjZGs6E+Te6bJb46unK/pU/wBi0i4iE9pq8QjPQPzmuHn/ALYt0dTdWl4jDHyNgimxtctp32c2sSse5b/61TYr2rO1NhaBsf2pbH8DQdPs+2p2wrkTf35dVNtEoxjIfP8ASrY1FokAcRnPoKLB7eR0i6ZaN/zE4f8Avn/69ZsNvbXGqzW6X0YSIAbiOCe9ZSarMz4S3iRv7zNx/KqViLnTbu5kuSknmndtRvun1xRYv2rOz/smxX/mI2//AAFaYdMsm+7qcH4rn+tY0upRxIGUrLxnCtzTW1O3MUbbSWYZZdw+U+lFg9qzQuvDdrc7f+J3bx7c8KhxV1dMthEqHVrY7RjOCKwU1S3xnygPq9PTVLd1z5Q/B6LB7WRtDTrMddVhH/ATSNp+nfxavB+CGsCXV7fz1VUO3+IbhQ2r2f8Azxl/AUWD2sje/s/T/wCHV4T/ANszSrY2K/8AMVh/74Nc/wD2va9o5R9QP8aP7Yt/7sn5D/GlYPayLtrbRx63LaPdKyZyJVXj8s1rNp9qOmrQD6qa523uUmu5Z19M4pTrNoe7/wDfNZ01ZtFyqyOgGn2n8esW/wD3yaeLCy7axD/37Ncx/a1oT0c/hTxq9oO7/wDfNa2I9pI6UafbDldYhU+u0irsUzwJtXXY2UdmGRXIHWbHH3X/ACqIazZE8RuaLEc8jtGbSrg/6Yunzt6rGYz+lYfiG10m0ntJtNZB5nyyqjEjPFY02qhwn2YbMHLbzwR6Uk19HdyRhIim1snnOamr8DCn8R0t1Bpdlbx3UcEN3uxld5zk9yKz2utGuZMy6dNbp3aF84/Cqkt4ozEqtx3IxVf7SOhUD6VdP4UTJanV2vhfStUgLafqUgb3wSPwrH1LQdQ0mUi4jZ4+0qDIxWasyIwaN2Rh3UkGtuy8XX9rlZXF1F/cc4P51oSYYOadXWWsnhXWP9ZGbOc98hOe/I4qpeeE9imSy1GG4XsGcA/nQI5+kww6VJLDJbSeXOjI3o1IB6N+lUmA0dagv5ba3kiurhZi6ghWjXOB71aA75ob54nT1U00wK9rqNvqKsbcudoz8ykcVNVXRBjSI1/usy/rVxhimwG4oxRg0vzZ5qQG4ox7U7J6dqTJ9PxoATB9KNpB6U73xmlBPpgUIRHg+lGPan7jkccUbh3U49qQyM4yB0NGPc1J5ieh/KgAHoMUAR7SO5pNuT9/9KlwO9G0UwIsGjBqTAJ+9z6UbRnigCPB6UivLBNHPC22SM5FS7aQrRcDp7TW7S7OJD5M3cN0/OtIRs4ymGX1BrhduDT4pri2bfBPJGfY1NgNjX9Dnu4cxDkHOK5m1v8AUdHlAWaW3cfwE5FbsPiPUosBtkwH98c/nVlvEGnXmEv9PJ9SuCBRYCsni15ovLvrOOdfbAqNz4YvsYWaxc9ecip00Tw/fMDZ38ls390t/jVW78IXsXNrcRXC9yTg0mhpjx4XSfJ0/VYZU9H+U1n3Gga1ATi08xR0aNgRVW40rUbXPnWcoH95RkUW+sX9rjyr+4THYvkUuUZXu7GdJf8AS4ZInPqcZqvBaxWjl4lwSMGtWbxPqzrsmninQ9VZOtVhqtq7ATaRatnrtbaTRyAUZJJYmMsTHcOzHIrOS9vJGYTK2CMY2108d54dOfP02eBfWOQkVaiHg6ZsebMmf7xIo9mFzhY4ryN9yxhx71YjLSTfNalCfSu2k0TSZG3Weqhc9AQD/Wk/4RW7KFob2OX0+XGfyqeUDjpba2izPIxOP4D0qa0nsp5Ni25ic/xg4Fbj+FNRB/eW6OO4D/8A1qrtpN3b8DT5MemN1HKTYoTXCwKYmfa30qissxb5XBGfSn3+lXE1wXmWWEn0U4NJb6fDFgG7fcOxGKOULGmYtsCyxqnnAggrUEyNI4nLbXxhkxxViKQgDJ4qjqixBPPEmHH901HKFiNzLO7IpwPWqy2l02dg3VB9pl7GrlvczQJ5rYPsDV2EZoeVHw/Bz0qWa5CxHNTyvbX8qmaB4/8AbQ81B9kW3Ygs0ydsU7Aa1lq9tcQCKeFXbs4XkGmT6pJATHHLtX6VkR3McE3ETj8Kln2XILKg3HqRVcgE5v1uDtmlOT/EKRjaRESfapCR0wtZ32d1cNjpV2GCOVMuKOQDVtdQjMY8qQsvrWlFrl1GoVbuZQO2a5tRAnCTkfUVPGwb+MN9KbgUmdQviu7j6zq3+8lTR+NpBIDJDG6915Fc0skRQq0hXPpUMiJwYyWH0pcozqtV8TxX9k0S2m1j75rhbkkzlqv7mI+VSfpUTQs55hf8qrlFci06+u7O5EkD4x2r0fTvElxdQrmNN5rz5bMjkRS/gKuQQyFAuHXH4UpIdz0Jr+9Yc2y/nVG61C6WPDrsz75rlo1vouUu5R7ZqUXOoE7XnLL6GsJMDTa+mC5yD7EVXlvGkHzRp+VU5JrhEyAp+tVGu5m++APpWTYEtyylcgAc1W8zB6VFLcfKec1Ua4Y9OKLAaW/K5waYDk1TS4fIBqzHPCzY3gN6GpswLBCkfdxSbARgGlDKw45/GjhjkdaV2MhZMGmnAGTUj8ZzUDMpBz0qkIUgnnGaiYntU0UitwSB9aZIgXJGPwp6gc6sLgcilcYOO1OVyU3dqaXzXaZEZ4pNgbtQx5ppcDoaaARkKnihSx60u4nrSeao+XBzTGO+YUjDPWjzcDHf1phY0JACjDA9xUtpCJrvDfcUZNRgc1akZobL5OHY9KYinI5+0MqKBH2FSlAfuHI9abNbvbxRs+dzDPNRo7Kox3pDZa3jAC8Y4pTLHEpLcufTtVXdgEiiJfPlwz7QOfrQItGNhGJnI55wKVTuOOBSswMioMkdABSvCEUtj5vTvSHcQ4jO4jPtTC24kkgVGS2CT0FQGQMMCiwFkOD0IxTgwxxVIHLAVaQEJnFFhXHMfSk38YqMyqOpp0K+e5VAScdqVh2FeBhtc4C03cvrSzQyo+1lcr9M0xl2LkqQPeiwWHDBOR0p4bFRxHdVkRKOCPmouFhFbBBBJ9j2qwqJ1IOfY1CIdhzkn6ipAHZchsfhSY7CMNpJ4pUeq0jlTgmovNfPBqeULF7zd3AGTT4tzN84wvc1HZlC7M5wFGc00XLyTY4CnoAKOULF1TgMQxCjpik5dfr370ZIjOG259qI8hfXPrU2HYVYQF+Y1EYF3E78fWpWWXduEqBR1yOlREw/8tLsFu5IppAV5Iy0m3cuPWq8kTKcKefarJFsvzLcgn1FIiCQ5U7h61pF2FYzx50T7lXPsan8kXHzLGQ/92rb2u/omaR4fs20INrEZNVcLGeYmV9vFSiBE4c7j6rWizQLGqypvfONwB4qNwiOFR9qEdcZNFxXIXJlx8gGB2FMRdrMck5PSrB5Uqki47n1pYbWVWLsvDd6Bo1dGjypcdCxxW/qLtb+H7l14JXaD9azNIj2xqtaetHOiCDs7DNUScHDu8kAHHJp3kI33jWwNJtwxIuRt9D2pxsLVf8Al5BPstQWjH+zccIT9KaqMjhkTBHQ1t/Y4AfuOfrU8VjbSMQyMABkkNQMxI5LjdukbgUwozH5fu+tdAthang+YM9Ae4p402IcBOPrRzAYSpIvcN796FSSR8YzW41gqthFFB0+Pjeoz+tPmEZYSRV2JDuI/iPIFHlz8MY2J/2BWoLbaD8m4DoKlhjeU7WJ2dl6UrjMMLMzjarfjVhYLl4wHQ7R0CmtdowyhcbcHOAacYEK4Vwh7k07gYwjdDjyCwJwc064RkdUEDgYz61rbI4iFEobPUCowttI7MxYDod9IDHEEzHAicfUU82d2py2GUdhWyDAWZEfGRglulQqqLnemWP3dh60XAzDp1zIodYTs9Saeto6HAwCavSLcNgW6yr/ALO3NX4or2ZREunT7vaMipeoGd5DxkpkBh19qjFvJknIY+o7V0A8N6tIf+PeBfaSTBq1D4JuGBa4vlhc/wAMPzAfjTA5YwMqkyOT7BaZ5kK5Dxu4POD613Vn4P062yZZ7iZiMZLYq1/wiuhnmS0Mrertk0AeeLMgcEwDI7Mc1OwN2GKw/vD0MaGvRYNL0y0x5FjAmOmEq0SB/q1C/QYoA4G20e6ukUDTHVu7kgCrcPha+bBeSCA/7Z3EflXYbG65J+pprRM3ap5gObbwpv4k1FWXvtjIP86vR6Bp0C4Fusv/AF0YkVq+SaPJai4ynHY2kRylhbK3qFqwA54Xj2AFSNGy8YOfSmYcdVNTcBnlFj8xY/jTvJI6BqcN2PunNO8yZeimpuAwoR/C35UpRh/A/wCVL58pUkg5HalE0pH+rP500AzH1/KlIQ/eanFyoywOKha/gWNnknREXrkincB+IP8Anqo+ppViib/lon/fVUZNf02NSwLSkdvLqm/imCRSsGmbz3YnAWncDd+zuQfJG/HXBzTksrp0ztI+tcVPcz3k+VkEKd0hOBSb7yP/AFF7NGfXcTU3A7P7HKD8z/lUosnI+9I1cPv1dlL/AG+R8d9+Ks2994hgg3Q3Lsn+0c0XA63yWi6xk/UUee44EY/75rmE8SeIYWz+5c+jinQ+Ltblfa9jAy+4IouB0xuXP3mqMzD+8KxD4luFVZZ9Pt2jzhgn3hV2LXbC4bH2K4X64ppjLn2gZ5wfrUgukxjYfwFU31LTl5aOdB67QaZ/a+lDg3Myn0MRpiNBbiEfdLLn604zKf4zVEanpmcHUIzntg1KLuyYZW9gx2+agCfe7fxxv/suKi8vzOtpA5/AU5XtmGRcQH6SCnAKxwpVv93mgCDyY0PzadB+CimSWGnu29tMt2Y9flq4I2PG1h9Riop57W1GJrhVb+71NS0VYrPY6XJG0R02IBuvv+lMfRdJWFfMiYMgwAZSOKsxm4uyVtrfCf35l2irSaRb+ZvuWaST+6vCj/GnYVijpui2CTC4NqDt5QSNu/HFaN/LNHBiHhmIBYdVHqKnZ44kLNKox2zyaqtczSOWVQV9TRYLFC0s1CmSTcGP3nkPWrRmjHETbm9R0oeAztmTLj+4TxUc97Y2S7JpERx/AvJosJodBCsW45Znc5ZieSafNNBaqHnmWNewPU/hWDN4iuJBi2hWEdmYZasl1eV98jF29WOTTGbF9rhuImhtVdEPV2xkj29KyT1oAbGKAD3pohijpgUYNKKWqEJilxRRTAKUUlLxQAnNL0OKSigoKBQelN70EjulJuxRRigY4UuPWmg44zS0AOoFFJSAdS02lpgFIaWge9ABRRSUgQ7vRnmm80tAC0optLQA/ilzmmc5opAG1Seg/Kl2KP4RQDzgUZ5pWGGAO1NIUngU6o2ByAGA3HHNIAZRnPaoyBV2XSrqKJneSI7RnHmDOKSw0u71OIyWqLIo6/Ngj60h2KHejFb/APwiOqH/AJ4A/wC//wDWqnqPh/VtNtTcyQpMgOCInyR+GKBl/SLjSIrVIr7R4ZwPvSjO4/UZ5rqbPTPCt9xBaWu/+43yn9TXl9nr6xyLHdQtH7jtW7b3UN3EJreQMvZh1FAHW3vhiKJibfSLFx2U5BNYMwtbV9lzokEL+hBq1Z+JtTskCbxOg7SdfzrobPxJpuqIIrhUhkP8E4yPzxUFxkcn5lif+YTaH/gNM2Wh+7o1sfpxXU3nha0ncy28rW7joq/cNYN1p9/p5JnhLoP+WkfINSbx5GUQtuvP9i234uaUNCORo1p9O9K0u8/KwLehOKQ+h+93o5jX2USs25L8XMVkkIxgxgcH1q0s8J+/o9r9ccUFCOtPVOOorLYvkiSCaBcMuk2ePUjNDXK/w6VYj6LTAPUZHpnmk2E9CPxq+ZkeziPNyvX+y7IH125oM6OOdLsm99gppj55JHsRkfnQU4xj8RzRzMPZxHrJABxptkfqmKqXyC7VFhtoLZ1OcxjBNTgYNLRurMPZxEjniECR3NhbuyjBbB5qTzdP/wCgZCT+Ypm1TxkZ96Ni+lTrsh+zj2JVaxGG/su2/UVKJ7Mf8wq2P1FVQoFKCB1GavmYvZQ7E7SWDHnSrcfQkUqS2CHI0uP/AL7NVjtJzgijI9aOZk+yh2NWK40GXK3Wlsuf4t+f0p58OaJernTtRkgb+4/I/KsfINAI7UczJ9hEtXXg7VoRugWOeP8AvK4B/I1kSWV3avtuLeWP1LKcfnWnBd31q+63uGT2JyK028RXMtlNDdxRyqyEHHBNbQmc7oSRxtjCttE0YfcCxYcdM1YfBNUtJ+zyapbx3m77O7bWKnBFdpP4Qsbhs6fqOxh/C53D+daSlczcbHL96Q9a15/Cerw5KKk4H9xgD+VZU0E9q+y5heJ/RxikmTYYaTJpA6E4zS5HY07hYATS7jig0lAh2aNxwKb2ooGP3ds8Umabml4oELRmjNJQMXiim5pc07CHDFIetIDSk0AHHpzRx6UmRS5GKQwwvccUhSM/wCl7UZwKEAwwQn/lmKepkjOY5ZFI9GpD7UA+tMC1HqV5H/y1z9RmmXMsN7/x82sb+44NQZFGRQBTn0TT5STG9zEfUNmsbUPDd5I2be5STHYjbXTZoyKAPP5NG12B8JHMp/vI4IpWbXrSIedC8i+pXca7/NGaCbnnA8QTqxWW1iPuDirkevQjnfdR/wDXNiBXZXGn2N0CZrWJj67BWbJ4V0l8YikT/cNAXKMHiydGBi1F93ZZV4NbFv411IJtJtpPwxWW3guxd8pezxj0KCoW8DEcJqsZbsHQj+tAXOmTxnk/vbBwPVGBFSrrXh+/k2Xdqw3d2TA/SuWt/CGsW0oeKe1mjPVVkwT+lXH0rUICPOspV9do3CjlC52kPhPQ7yITW8IZOxjlrEvfBywS4SRkU/8APUcfpWdZz3eny77XzofVShxXbab4os74CG48uKY8eW/8RqJIpM5U+AJXYO1xHsP8Sg4pX+HqOAJNQwP+uddNqN69g5MFrOQeqrhlp1trtjdW+67gmhbuXU4FSFznYPA8MS7DqrMndfLqdfBOmfx3Mx+mBXTRrbT5NvKjAe+P51J9kPdc0Bc5mPwVoEbb3jlkb/aepD4Y0BT8tn/48a3jHEpwRTdkWeBTAyRoOk9Bp6fnUcvhnT3bdFbrF64Gc1ucAdKTznHSgLnPt4YiX7sYP/ARTB4fiT71sn5V0XnPimgs33qq4HNvpEef9WKa2jJ2lRfwrpTCh7Uvkxj+Gi4zmf7NCH/Wqfoop66aX6/N+FdLiI9Uj/KnD7OP7oPtRcDm/wCzAByjf8BphsYu4l49q6cywoOZVA+tVpb2yXrKhP1ouBzRhtx95JaidbbPAcfWugkuLZlIVEOfasy8t43O5IgD6ispRuBiXRRIyFIIrGuZAw4rYurdmztUg1zl00sEpWRSvoexrP2bFcilfjIaoRK2CV59qqSu2/5cD1xVf7RIrjBxW6psOY1ku4ycMpU1N5ke3cMMKyBOx+8aekxBzmj2bA2UkUjKHmplmI6ms23kWTIPBAzkVNu/2ufQ1m6bKLjzg9TVZpgHODUQRyeXFMmjKru3Co9mybllWV1DKRmmGRwcORiqUZYsMH9ae7sRhiDV8jC5AikqM9KjkRCcKSPxq9thVPmlAFR77ArgByfWugRRkEaDlwDSCGF1DbyT7Vd82zB+W0Dn+8xpyzRY+SKNB9KaGUcxJ98Nt9qfJLbyRbIYGLepHNSu7S5TG1fXFRqfKOEOT60xFYWlzIflherEOk30jAKir/vGpRPKv8bfnQbiRhy1AWLEGiyrKDNcQqPrU7WSxSrILiJlGc5FZgfuU3N65qKWZn4b7vpUiLNxb2kspeXUBz/CvQUxJtJtWwGacj1FZkoUvlRgmpBZyCLzZU2L9adgLt3eQ3Me2OIRj2GM1ViXY4Y9Kh68CpYxvIi9aBF+BVUiUN06A0h2/aVZtxZuuRSrHIkojXGAPWpYonurpFUAbetSxotLp0M12qOSEPYVXuLK0hlKraNIv99W4rQubeQZkXGR0warpI+QHk2n6Ui7EH2O02ho4nRj/s5p6W0atzlvqMVovKyRlmZGA61XUrOm4Jg9zVXJaK7RAfdjTPrimA7AWX5XHUgVoW4AfBt5pMdQBjFEgQyYSB1H+1U3KsQw3Em3cT8x61MHFwQJUQgdiODUq2i4+cN+FRPHFG+Fjl/Ki5VivKBK4j+zxRYPUCnLEo+UEFh1xV2IZTDIcds0pZIvl8iIH2NTcVirFaRySFW+Ve5qO5tFRf3MybO1WWx9pGfuH71TCO0J4lYD0CZ/rTuJoyU0pZmAeEuT1bOKU6VbBiqo4A/u1rGWzX5Vlcj0ZcUo8nqrMPpRcdjGGmhEdRujVhgs44FTxaOsSb/ORyOvYn8K0G+Yheq96XYkak5A9s80XCxTe1t1tkknimJzxsHWoPNjaXcsMqjspWtmPbt3P+FG9WPylB+hoCxmbJpOVtZtvrtp0VvC65eINn1rVb0JlP0NQm1cvmORVH91utAWKS6fAjg+WG/2D0NWSidrOJR7VP5EucsYz9KkNuREH3qc9hTbBoqpb28jBTbAsey0sml2oBLQYPp3qdUlicOpKkdwaWZhJIHz83Vs9zU3FYpm0tY1DLAXcf3iaPslvL8wt13dlNW3jJYMSoUDtQ1uJIlbzCCx5xVXHYh+znJU20YHqAKXy4kQBIFyD1xV14lVVEaYboSz5J+tQs7R7tp7c45pXAbZrmTPqavaiEeJIiRkdh1qCwHzAUzUZCLp9oPGBkVozJfEQJbQBhvi+metMlgtAT5YcsOopssrAAMrEt071Mph27fvS4yeKzNBpbKhVRRxycc0slrCQWEuSOi9KGMUcnluSysAQ68YpB5BkAPmeWx+Ygc4pcxROY1DwSByV2cqOM0svzNkEqOwFSSwPO8a6fb3EigYOU6GrK+HNVaRd0exe5dhTuJozVDZ5p7AAfKOa2I/DFxuJl1CKH1ATfVmPw1Z7t0uoSyH/YjAFFxWOay6+1SJIAcs+fYda7BdK0telmG/3uanSC3iTbFbQIB22incZxPlSSyKFgmJY4A2Grsel6hLIUitCV/vPxXX+Y/qn6UbyRyw/OlcDl4/DF5LIpneCBR3B3E1aHhGy3gvdtgdkXr+tboOGzkfnSlgT1H507gZY8L6UvaVvq1WYdH0qDGLCFsd3GTVveP7ppwkA6oT+VACxNFEAsSRRgdNq4qXzn/56GojNAR90imF4GGPm/KpAm5Y5aQn8aXH+3+ZqvttR3x9aTFt/wA9do7470AWc00saixZr9yc59zT/NgH/LVPzp3AUv603zVHU0hmjyeFI9dwqPzlJ+4T9OaGBYFzGeM804TL6Z/GqckqqufKLAdgtVXvrcYEqvFnpuHWkBqh89KTe3YZrKOpafj/AI+x9F5NV31+2hIEaSyE9mGKANnz2B4604T3LdGrnZvEdwIWaOxjX0YuCfyxWb/bupXUeZHNso7ggCpA7GSWeNd0syIv+2RVd9WtY/8AW3cUntGMmuNcJMu/zHkI/iLk5p8ks0sQjLRZHt0pAdDceI4Ism3tpZMf3vlBqofEN5PgqLdAekfes9BmJAeGAwSvGaFVfKKkDdnKsT8wNNAPnuZ5dwnncBuNqtx+NVdij/WKSD6nBqZlVsKQRk4yRSxzbFZpAJB90H1FMCNihAY7vl5+YUoYmLlBgd+9DPuAGDgtwOooVoljlDtuYDHA4FDKFErYwoAApRMwBZsNxgDsKVY1MTFiQfQUiCIZ3Nj8OKVhhHO0fDI3PftUxurqHgJ5KH26ipVmsiGjZEduMZNSW8ksxVHtkaIHO7IA/GiwrFMzf3ywJ6EjFTrLO0QLWrSpglGX1HrV26uZYZcXnliJj8qqdxxUC6t9miU28ZCp93ccc+pFArlaRvm3PYuM/wB/Ix+VPDWTdRMg7FAef0q0PFU/U28LMOhDZNadvrGt3qYi0lAp/icYH8qA5jnH8tHIErEe5qJ5o1/jX8TXY/2ZczfvL9LAN22x7iKtrbadCgH2O3JHRmjBNMVzjbeylu8+TZyykdwODWnD4RaXc96Vg9FVgx/Guhku+NsWFXsFGBVYyTscbjt+tAEVpoGjacocRmV17uxIP4VdF6o/1MRAPcLioRBuHzZ/OpR5UKhSxHsOSaAG75SeeKTy1WQXDQxyOvcikMzyfLFHtz0eQY/Sk/s6J3LytI7HsHIH5UMdxz6msisRC/HWoGmuZCNmY19COatm2htl3yFIYx3dsCsm68SWduwW0j+0vz8w+4Px70gLqQknPlruPVjyarXes2lgxjZ/NmH8CD+vauavNR1C/mBnkAiHSJBtUCoRGg7Uybl+61y9vEKArAh7Rjn86ohfx9z1pQFHanqM1Y7htPoaXaaf2oxQIZg96XinGm0wDvRQaDQK4UdaOtHWgdgFHWg0UEhSZpc0mKACjtR2o/lQMBSE/MRS0nQ5pgKuCaXOKZgd8+2KcCMdz9aAHZp2aZilFIBwo9aTNGaYC0Y96SigBe1FFFIQbsCjPGaMeuDRQMM04HikwM9KMDtQAufrRmgACjigBO9KSKcgg8sl5WV+yhev40wZIyaBi5IprZPBUH8KdS0gICmOifpTojOhMkLtGy/xI2KfSEDORmpA39M8WyRqsOoLv9ZxgN+NdZBcRXMIlhlWRD3Vs15ky0+Gee1bfbzPE/qpoGjt9a8P6XryZuF2TjO2WPgg+/rXA3fhHXdCZpLR/MiwTuQccD+Idq6zT/FKysEvUVH7TKOM+/pXQLeZQMJAQejDkUhnlVp4oVvkuomV/UVuxSQ3Sb15FbereGNK1OdLr7NH54PzBTtD1yWtaHeaS3m2enTJF2kibcPypNCR0Flq19pr4hmLRf3JPmH611Vh4ssbo7LhDby9+6V5Ja+Kf4buL/ga/wCFb1veW1wB5cyHPTmlYdz0a90DT74B40VG/wCekXQ1zl54fvLRTJFieIfxK3P5VmWmpXenyA28zBe6HkV1Gn+J7W4CpeKYZT/EB8pqbGsakjlxIo4fKt6NwakHzDIJ/A12l1p9lqsW5wkgb7sinBrDuPC1zAo+xzCb1Q8EVFjohUMlVJUkMp9u9NwegY7vRqGZlYJcQmN14yBgilzuXdu8wZxz1pGqlcCZF56H8xSB2PPB9xShgrDYxVvRulBcg44U/wCz0pDFVg2QCpPuMGm4YfeKr6Z5FGQ6/OVcHp6inIcHHmBc/wB7pTYCYY84BA7qcijcR0AP0NStFxnp/tJ0qNlfbtDK/wBeopAAbnlR9G4pxbDD5P61GCw4yFP+0OKQ7ge4PqvSmBKVznC5+hpg5BAKA/7RwaQ5cA5DH34NKPl5cgf7wyDQIZyO/wCGKkwzAFV/I1GTzxx/u9KUZY5+TPoeCKYiX5iMjH580oxg5HUYwRUWWDfMcL9M09Tg5U5Hop4oQHLwHbcD0En9a6p1wd6nDH+JTiuZlUx3koPZs/rXRo+I06fdFayMIRV2mW7fVtQtsBLhiB0V+RWlF4n6Le2iOv8AeAH9aws80v1qCvZo2mtPDOqY3f6LMem35arT+CS/zWOo719JRkn8RWdjI4ojkmt33wzMjf7DUc5n7FEN3oGq2XLWjSD1jIb9OtZm/acSBkP91hg11dv4hv4cCR1lA7MMH86ttr+n3a7b2wz6kKGH51rzmXsWcZml611x0Lw/qHNtN5LnsrYx+FUZ/Bd4i7rW7Sb2YY/xqlIiUbHPYpatXOlanZfNcWUip/eGCP0qnvp3IsFLmkBowDTAXOcDvSUds0lAgpaSimApPNHTvRSGi4xc0A0lFIB2RRmmYo5piH5FLx61FzRk0gJuKSotxp240Ah9FM3H8aUPQMcR3xRR5nGD0pMj8KYgNGM0vGKTdQITafofapFnnThJ5VHoGpmaM0h3LA1C7UYMgcf7Sg01rtZDmayhk9eMVDk0vFAXNK11iK3XaLeaNfQSEj+VXF120bgsR/vrWDgUbQKAudDv0y4+bNsW9A201OlrFjMEjr7LJnFcnsT+6KXYBypKn2NKwXOvCSrx9oJ/3lpP9J7G3f6iuWWe5T7t1KP+BVKNT1BOk27/AHhmixR0m+4H37ND/uS5p3noPvW8i/hmsCPXLsfLJCjqe2cVONcA+9HIn0NJoRrNcW56vt+oNCvC5+SWM/8AAqyptVEqAR3pj/30zWVewX17zHfWLE98FTRYDrjER1ZfzqpPe2kLFZbhFPpmuNbQ9e/gdJU9EmHNUnivbVytxbS8d8ZpjOqkv9Myf9Ik/AU0S6TMoP8AaDLnsQa5MXcYPzBl+oqxHNDKPvqfxoEdUul2Mv3b1G/4FikfSI4m+WEzL2KuCfyrmPLjPTn6U5TIn3JJV+hoA333xNhdOuOPQVSl1hoXKtp9wPciqseo38P3LuT8Tmpl8Q6ovWZHH+0maQypP4p8lsfZCfwrE1PXI75SGtAnua6v+3lkH+labaTf8AxVd5vD86kT6F83qj0WEeayMHYkVF5L5zjivRP7N8IXBbdDc2zfXNI3hnw1MmItUmj/AN6tLisefCMntViO3buK7H/hC7Rj/outwufRsCh/BWqRqWhmt51/2WFNMDlo7Z0cNj5R1qX6Vrv4d1SN9rWbY9Qc1C+nvbuRPA6D1xUsZnb3BqCeQv8AeNXrgwLEdgO73rMmdC3DikkIRJdhyRkd6V7lGGBG2PXNREZ7j86QRjuwFVZAQKQWGKmCs3QVabTRFIQZt0g9BUBSWOXaeKOZARbJN2DxTgxQj5c0+RWQjG4+uRVv/UQh/LJJHpSckBT3Ss4zGQPfinO7b87Pyp/nycloSB64qNLgs2CjfgpqUxibpGP3MUwpzk1djVyCTCxyO4qH7NOx/wBU4+op3JIUPPy05rWSRQAQufWr0Fi8cRaRetP2H+FGb6KaLgZ0dhGjbi+8juOK0fIeeyZY4PNx14+6KeI3EDlbaUswxgKataY93b2nlvbruJJ+7U3LMG3tTJKAkbH68VIunSx3oyOtb8VjMZ9zBkz6VcFhFuDSAsw7k0rgc7c2M9uRLIuFptvBIbhBj7xxW7cWkEgIdpGB7Co7WxCAbAeKdxWHPazqSke9z6dRVDUojBaGQrtkBANaYNw77VuAn1FJLYm4XFz+99yaVzSxBp6R3GkRmRC2c5xUE9yLf91HGVUetWjbIvyi5aH/AGIzUMlhHKMbpn9ycmkyWia3vHuDsWVQMc5qwsibdrSBselVP7Lt40yvmbvQnFPitTDtkdMKffNNABuVRwqBtxPVhSEu2S0wYk9qvSKrIuY4sEcFzT4xGkIRfJDd9vemBQRFJw53L6U7yYWnIjiP4c1prpls2d0/zBdxCdazkScqBGj7f7zHikBYRTGgDx5HfC5NPDxrykeD7riqqm6dwqz49c8AUyO88u5EbyKzDqxWqAkKoAwKd+pFRkL6VcuPLliZdwyemKqJH5YG7JUdz3pATrJFIT5cewgdTUTxhwWkUE+o4NTNalVMjISCONpoJWIDcrY9aQyCOH5cbRGynv3q2QCUJEY57UojsC2bmYrxkKB3pojZjvT7lAieC4MMzEorJjG41A5s2ckl8+i9KYJkDEE/KOtQ70eX5BjPegCUugACRvwerGhZGdtijyx/eJoXyPLw0rk1A06RkY3P9KBl5rTjJulGfWoxZqykSTA+hTmiAfaBlbSct6BCaujStRuAgtrZos/eMq7APxoEVVihhXDzz7O+BQxt34VJDH2xxWtb+E7k/vLu+jGOyknFXE0DTgoWW5kkPf5sZoA5V41Eu1S6x/3i1E0cMSgrcl2JxgV2o0zTO1rC3+8Mk1neK/Jjhs4Io0QbtzBVx6VQjIsxhwAOTwB6mrb+HNW81n2xjcc4Lcio7GLzbmFRxlxXXtBIR/x8JVzM1ucv/wAI1cD/AF92iH0Xmprfw5aJlpLiV37MgAxW8beQ/wDLRG/GlEDrzgVBoZ9tplhacxQb2HeXmrecfdiiX6JU3kue1HkP/dqQI97kYLYHoOKZt9SxqQxPjkU3bJggK34UwGbOeV/Onqo9Kbv2n5g2fpTS/PRh+FICY9MZoAWq/nqD0yfpR54J7gegoAsjB70o+U9apifB5HPpQZSeQCPY0AWS1N34qp5rVG1yVOCDTGXvN+v40hkYjmqH2iQnG0/hQBcyNhVOPc0hFwykdQn51G0+Dj5fwpY9OvG+bIx3zTTbWsB33t3Gme24UAI0rt05pgLN/wAsyfpTW1zRLaUC3Jnl7Kq4P61HceJ7mNQsNosGehl5oGXU0+4kYDylGe5apW0mKP8A4+btIx3GcVzU2qeIbyQx293CmeoiABUetUrmeZ7ryLhrZ2UfvGZt280AdS2peHrMiMy+c/ohLZofxPao2y1sTjsH+U/yrnlESZkS0UsvPyDHHrTFmeWZWLASDrg5xQBq3Wu6rdAhEVU/uqMVlOs93N/pbNgcrnnBqUq0cqyK53ZxjqaY7TJOXZ3hx0Z48DNAEMNqFlLebIo7KOP/ANdTFyyorBGkGF+71oW4V3BD78dRjvRcPLudV8s5GMgcipLHSxokJf0p/lLNECBuJGSagJCXWWBeFBkA9yakjuJyWWNCiuME9sUAMEXkqW242kjb3P4U0TrvwIGGfRasNF3LAN3Jq9DbTShZiyqijG8nkUCaMtZP3hbY/AzjFPe5j9ZqsiBpd8nlM7ZwB2b61OyGNkcxrCQMAyAkGgVihFFLdXEUVsHffj5m4AqxPYxx3bWqXE0kyjLKIwBx15rQhhuZ5dy3UO4AOBH1PpUUmkTsxfMiShtwJ6rQBnXEEkE3kpayvLt3ffAOPpQlhfuoY2cij0Ygf1q4sJV3eZxv5XcX5Ue1TWIBdUG6XPJkJIGPxoGZBLJ95SO3NSWN9ZtNLBOrmUdI2XOce/at+9v7CFi0kPneX9OvfrUFnrP2j/kG6C3md5MdPxpjJtIt4WyTYpGwGQzDdn8avXkMEcTo6L7EDAApUstZlX/SLxbNB1WIZb8CKsW+kaXBhn8yeT+/MxOfw6VIHNiyubt1W2tzckDG9nwAe2T+dacHheeVvM1G7BfpsiHb0Jrae/iRikZUgdNgqD7TO544pkti2mnaZpiDybSNXH8brk/rTpdU+Yqnz49KiuC20M7EjODmo2SKNDI5CoOrMcAUCEaWeVicKM+lIYnY/PVd9YsYJ44VV59wzuhG4Cp1u7qfiC1KejSdfypgSiJUXcxAHqTiojfQLJsjDSN22jj86abR5mDXDNKR2Y8CrC27KMKgUUgKxN3OuHk8pD2Tr+dPZYrKESLG8jA465PNQ3Wq2NiAGl8x/wC5GMmsC+1m5v1Kxb7ZPVThqaA3bvxHplrqS2bXCl8ZYj+E+lVLvxQwYrZQNx0lfj9K5lLaNMsqfOeS5OST71KN1UMdPJcXj+ZdTvK2c4PT8qVFUDGAAKBTgtCQg4oxmn7ccEUuBjmmK4wbak7jAwKABRxQIOKWm0UxgaKKKBBRRR3oABR2oopgFHaiikFxKWkoAoAKKKKAEpKWkxmmA0sB1pwNMI5pykA+tAEgPrR346Unf2oBwKQDqUEfjTc0ZoGOPA5pe/FNzzjtS5xQIXvR3puTmlPJoAU9KFpAe1GeaAHd8UUlHQ80AHQ5pabnk/WlyKAF60ntRkdqM0AhaWm0vFAwOaXtgdaQdKUUgE5ORTNhzTz1zSmgQ2WDytvzq24Z+U9KnsNQn05swNlP4o35Uj6VARSbaBnXabrdnflUB+z3B6xSHAz7GtxRjg9e9eaGNTyRWrp/iS7slEdwv2qDurfeH0NS0CNzWPB2k6uGk8kW8zdXjUYz9K891Dw1q3hu5E7wGe3XP7yEZH5dutep2Gp2uqQeZZzFmHWJ+GFW1fGQVDA9QRxSuM8gsvEqOdlwjBx3Arft7uC5gDRyKwPpW/qvgnSdUZnjgWCdudynALf0rg77QtY8OTZMPmAnAkiO4EetDA6mHVLnSV86GdljHVTyK6jSfFtleov2lfKc/wAfY15emr7rSSG/i8pzj5j0H1rR0q7tpbFNswZwSOnWoaHc9YkjsNTt9rJFMn94DkfjWJeeFQWMllJjP8Df41y1vdT20oktpSjenY/Wti28bz27+Xf2QdO8kfGPzpWK9oUpIrmybbdQ7Qf71RK8bNkMo+tdtBqWn6naxyRyRskmcLIOaz7zw1BNve0l8huy9V/Wk4nRGtfc50LDjJUB/UU3ac8bWq3No+pQHbJCZfdDkVVeIw8OksT/AEqWjaMkxv3OFk2H07UpXcdzoG/2lNNLP1YBl9e9IGTaSs2w5+6TSHclRpAMpIuP7jmnAHq5ZPQr0qAPD/y1QsPUGmmVSuILhSF6qxoAt5foAso9QeaiLyAnb8ns/OKi+02qg+bLsb1DVHJqUHQTiYfTmgZYUdWdfqyUnD8BRIfQ1VGr2g4/eR/7y5o/tW2z/f8A90YpgXcFOMMi+hPFIIi3IVGHqKp/2vajjy5h+GaUahGfu2szfQUAUL9f9PZR6D862IyfJjOwjKjgc1h3pnnvC8cTLgcg962bKQTWwZkeBh8pXPBPrVGK+JkmGYEYU/Q80oUL1LfjS/I/UqxHtg0YxwGYD/apGg9enyg/UUZJ6lMe/BqMKp5yM+qnmjg/elJ/3xSG0P47swHcDkUYXsobPvg0zaOoBwe6ml5yfnH0IwaYhzKRxs2+5/xqxDqF7bDENw+3+7nNQDIXqwHscikGOowR70EuJs2vii4hwbiItj+LbV0X2g6n/wAfEESk95F5/OuaJboc0zy0b7xRvxquYiVGJv3HgvT7hQ1rcywZ7FgwrHufCGpWozE8dyvqDtNRKZIuYJpE9PmzWlb+INQgwHKTgf3utXzGHsWc3Pb3FsStxbzRe7oQKriRG+6wNd/F4lgkzHd2rKp9twphsPDWqsQgiV++xvLP5VSkZyhY4XqKQV1Nx4IdP+PO+BX+5IP65rJvfD+q2CGSW1LoOrRHdVXIsZtLTQwJIOQw6g9RSnFMAzRSUUALzRRmkoAKKDSUABoPXigUuadhCUUZoosAmecUucDFGKTFADt3PNJmjHc0UALmjdTc0ZoEO3Uu6mZooAk3UbhTM8UdBQFh+RQDTRQOKAuPpOabk07cfWkO4ZpKKKdgQYB4IyKaI0/uinZzS5H40AIBt6Mw+hqRbm5ThJmx6Hmo6KAJXupXGHitn92iFV9lufv2NuT6qMU9qbikO5HJa2p+7BJEP9ls1EbKH+C7df8AeWrNGKLAVTpdwf8AV3Vu3/AsUjaXqSrkRJIP9lhVnavoKNo7ZH40AZzwXked9pLx1wKjE237yun1WtYtL/z8SAfWnrPKBguHH+0uaYjFM0L9Tz7ik/dt0C1tOLZ+ZLGJj6jioHtLBzn7K6H/AGWpAZbRRH7uR+NMEbocxzPn/erS/s21bpNMnsTmoX01x/q5lb8MU0BXW71FPu3Un509dR1FfvT7vqKf/Z94vRQ/0YVBLBdp962k/AZpXGP+1xP/AK/ToZfXAxSiTw+5/f6Q6j1U1UZ9v3kkX6rQrwP95qExF46d4TuAcJNDn601fC3hub7upyL/ALxxUSpCP/10/bD6frSuBXit7OFMK8ef7x605Y4d2RsI9cVnylBkbAKIk39CQPapsUamLf8Ai2flSgW+cjafwqg6SRru2EgdqsW0LuhaQBB6dKRRZY2zcmNCfpTHlgjGTEmPZarlIEz505/3RUPmJMCEDBQe4oAti8tiOY1/BaQ3Vt2iz+FUzEQAFU596VRsPzoPyqriLRv7ZfvRUHV4+ixYpiiDaGaNQO9KsdoJd6RFvrUjaHLqu77sRP0FP/tCRvuwH8RSNfO2FjAVfQLSG1eZTKGH0oAf5903/Luw/CkkmkiwJcKSOB1pqwykfNM+fTNV5YpPN+7vYdMmgCYXTngKp98VEzTMTtDAn0qYRt5a4A3HsKRI3TJljIPvQIgaFsZaTD/WmQRtM+yWdwtXUhFwCd6J/vGojARKUjuE96Bph9jgUfu5mLe4pYrG4d932xowO2etNEThwyyvn604RXLcNISvoDQD1JZFAbbLIGHtUUEaq25mO31PNHkeX99CM981dksmjsA+8qOwFNAVjEv39+5fp0pnyK+FH5Co/wB+qhXUsCcdKlDTRMDgYHY0xWLkBIt5GVgrtxn2qAcLtEbbvc0kkuYdxKhs8gUkd4g6ExkjHzHNSUNEboNzoM+lOjmfb8sSAj1XNJhpHIjYyMf7oqZLDUpTthtnOe5GKEJoi+dv3j9Keis3zMuYh3q/b+HNXkwk3lxwj+83I/CtCPwxbpxPqDH/AGVAAp3FY58srjmdRj0NQiVVPzsG+hzXXppOjQqB5LP7sxqaNNNg5gs0Dep5pjOKdZruRfJgldR3CGteHw5fypuSCVP944FdGL2bGEVEX0UUhkmk+9K350hGGnhWdv8Aj4lgiHcg5NXI/C2lLt868eQL/CuADWhtJ60eWndaAI1sdHiOItPVmHc1Ok0UYxFaRR/RBTdi9kz+NOGUGNqg0CFN3cn+LH04pDLMfvSP+dO3jAFKNtAEITIyST9TRtGOgqXJyaC+BSAjCkHiud8STtcarFERhYkx+fNdLH87gZrkdRk+0a1MwORu4NXEDQ0lALyE5Hy5PNdE8hY5zWHpUa/aHlboi/L9TWqX4pz3IRKHI6mkLE87jUBlI7Cm+f2wB9KgsseYw+6xFAml/wCejCqhmPY1GZ29aYjQ+0SH+P8AOk+2yqOJUx6EVnq5c4B596Rm7bgSfTmgDS/tCTqNho/tNz1EdZvlzN9xC30oFrO33lx/OgZfOpAdY1P0pDqUK9Y1P0qsulAnLs4/3nqCeXTrPh5N7DsmGNIDQGoQv1tGP4UqtFO2BastYi+ImjZhDYs6/wAJZsY/CmS+ItTkh+Xy4B6Bcn86Bm9Jpxk+6pX6ms+5GnWKgzXytIeqqdxFc9Pc3t9GUM0zHu3mHAFQ2lnNDcN5UMUiEYLy9qANt/EllAu2G2eZx0I6U0azqEmHCpAhGeBkj8aYiW9vB5a2cTSZJL9h9PaqYhcOW891Y9ADwKnmDlJLm8vHl/eXk5X0VsCs9rRTJ5iyPu7BvmFXHOCrFzID/Ge9W4ZZEjkhjgiZZRhiV5/CjmKMnyGSULE+5yMlsVJIwkdDNIzlBwD2NTJHtl3J8q5AxTzIASFAOPUU0xWIPNikTDsEdvvYHUVM8NmANsXmsf4dvNEchVsKdrUpfbL5gBLDptGTRcoZvDN83AYcAcYqQCGJVjC/MpycVH5+brdPFMijq7L0q1stwpby5VyMhmPJpCZAZnEokU4A6GpZbxrhMO25vX3prtGbVkIBbu2OQagV4V6wq2fWncROLcbmfChQOWB60RRR7cqDgnBZj1qoVjQkrGyAjjBzip03GMIZCCem4YouO4/JThSv0IzT53ISIebIzhfmx2FQmNhKuPnPcLyafHGqh1+6WOSH4IoJHMxWJRGDIx5LdSBQqbzjJz/tNgA01pURysUYUZwWB61Mlu8sGVGVznJamO5aik2AxhlkK5YY45qxE15cwqdiv5i/MOw+lQ2dqGs2E6tlTgNnBNXbW7it9ywK0zLjCJyTUjIYorpMwwQBM8s6nqasWyXMTkysX4wMt0qGLw5ql3K0sjC2DHOHPzY+la9v4fsoVIleSdz1YtgflQIxXmuZLoR20Idj3jUNV2DSdSusC/k8tR0EZHX8DzW6PKgTagVFHZRgVXa9/uHmgBlvomn2yt5kXnuxyWm5/SrL3kUH7tAgUdkFUHeaUndIRn0oWLHXmgCw97JIcInB9ahaN5QN5JqaOLLDApJp4IJBG7Euf4VGcfWgBiQgYGKn2rENzsFX1Jqqt3JKWWOIxrjhm5JpnkKz75maVvVjQJElxOLiEpbI7k9HxgCqdzpk98Nt5flV/wCecajFXtiY4UVHLLHbrvnkSNexY4osOwtvaw2kQihIUf7KipiqqpaSaYKOpLACsG48URL8tpFv/wCmjcCsW6u7i+k33Epb/ZHC/lTQWOiu/EVjbsVtg1w477ztrAudUvb04ebYn92MkCq6qB2pcD0FVYBgVUxxTsj1H50/FIIxnJAosSIFNPCGngU4HA460wGhSKcBSU6gQg65paT2pe+KAENFLSCgA7Zo/HvSUUxi0UUlAgooz6UZpgHbNApDRQAoPPNHHamk0Z4oAWjNJ3oxQIdSUfWkpDF4z70Dmm980ZxQAc9jSc5pRQevFAC5JxmndqYBg53Z9qcQwJHQUAPpM4oHDc4OKME84pDQEgHqOtL3pMDvj8aCenGaYhffvRmkJ+U0ZoAXHQ+tGaKMUALjPFJ3xRSYoAOtLziij8aYAMjrSjIFJyTRSAWjJpKKBi8npS5496bmlzQIXAwTQRzwc0g5Bo6CgBe1O+lN/Gg9eKBi8YphAz0/On/d6ikyDke1KwEWZI3DxSPG46MhwRXQWHi6eLbFqCNOn/PRAA4Pv61gYo21LQ0ejWV/bXsYltZklTP8J5H1FWXEcylJUDIeoNec2FoZJTIt8ttJFjBB55//AFV1dpqbxxhbi7iuivePAP41BTM7WPh/aalvaym+zyt1VySprhLvRdV0C8Z2ikjQdGI3LXsMF5DcIJIWyvQ+oNEl1bupjba/qpGRQSzyKy8QMObl14/iAxW3HeR31uGjbcDWvqvg7Tb1Ge0DQTnptOQfwri9Q8Pa3phYxRSBB0ZMYqgNpN1tL+6dkzydpwCauRX88bBknk+hY4rkxrmoQfJe2zZHVimDVyDW7SQgZZc+oppCOpHjc2fy3tu/+y8TnBFadp4j0vVkCi78tm6CXOa4qSSCchWZefeq9ubK4JTOGBwOOaLBdnoreG9Nuz5qaiEkP91uKifwWXUiHUI2J/vKCa4qJLy0XzLC/I3fejb5g1Tpd6xI7vJeCJ8ZBjJAP4ZqeU09pI6WbwfqMK5FxG/sBiq0nhy6jTfLZhvVkasuHxRrcahZ5xIg7lRmnSeMdQWMtBPtcdARkGp5S/byLX9nmNg0drux1DcUwiAOFmtlif2OagtviFq1rxPDDcRHqGGP5Vr23xA0e5/4+dL8seu0Glyle3XUpKBzsRJEPXcuaaxs1ODAYn9QvFdFHr3hS4QFowM/7FTifwhIOZYh9SRRYv28Tn1uJiMhUmX61G0sTPl2aFvQHit4W/hLkpfqmfRzVeWXwtE5VtSjK993NFg9vEzAT/y0ZZY6aZoF4jQoPQ1oxWXhy8fda6uqKe2/AH51pjwZ5vK6jvX1IBNOxXtYnOrI753RoyjoynBp6t2WdkH91m4rf/4QeYD93fD/AL5/wpv/AAhV32uoG+qmlYv2sTC/dg5dAw7MnWlVgfunH/XSts+Db1PuXMQ+maY3hbUV6vA31JosHtYmWA45ZVHuvShm39Xz9TWkvhjUEyRIin1V6F8PamOvkP8AVqLB7WJlgAdAfwp4duhC/wDAq0/+Ed1Ej5PKQ+ofNNHhbVSf9dDU2DniZmCOT0o+QnkBvrWsPCWoH71xGv0Jx/KpU8JXPR7mJh3G0mqsTzxMXIA+XI+pzRv55IP0GK6D/hDyORdkfgalXwrCP9ZO7n1xinYXtYnN717s3500yJ/dB9xwa7CHw7ZQkHyy5H945q1/ZlqP+WEQ/wB6lYzdSJz+kXd6zYLMUHZxXRRyv603FrH1dBj0NMa6sk/5bLn61SZzT1GX2kWGqp/pUQSTtKnytXC65pE2jXojd/MhcZR+/uDXYXGqqFKxAMT61TFpBqwMN2TjqpB6GrTJscUBkZFBBrqJvBToubS+U/7Mg5rCu9J1Sxwbi0bb/eT5gfyp3CxUpajEqnrx7HinB1I61SEKRQTTsimbvmyKQwooPWl4xTuA2lpCPSii4gzzQaKKQxM04ZppoAqriFopCKKAFpDSZNLmgA7UnSlxxQaBBmjPtSd6KAFzS5zTaTkUDHZNKDTeaQk9qQMkzRmo91G7tTESZoBxTM80u7mgLD92aM1HkilzQA+l4xTAwoBFAx1FJuo3CgQEUmKdkUnFACZzRRj0oxQAUYo70mcnGKAuH51KJpMY3nFR0ZpDTHmVsEHaQezDNRlLd/vWkDH6YoxRigZGbOwPWzZf9xqT7Bp56S3MRH41LRikBzYVS+SuR6Vcit2IykSRj1J5qKJ0dQBnPpinmJ8EncPYnFSWEsEvVrlQvoKh3QdJTK7eg4pyidGARI8e/NWjcNsIOzPpSAqrDHjOPzp2xUGaXaXHDAH0oFux6kn6UABRp0wquPfFIlosC5ZmLe9SRRSofmlYL6CqsjKsmzc559aAFPkscS8p3p8kgGPK+524qeGJIn3FQfrTp7clt6fdPSgCmuC2SDmrsZAQVTYzh8hwR6VZAYrncB+FAD2PykiqqMHlyBuNWI/MDYdwyk+lDTRq+3IUeuKQDPLmeQbVIx0qSSKbAV+3ekLM3KPke1Bn2Z8xsgelMBHgjXBCZPc5oWJdwbFCObhtscMzN7JV1NG1WeH91aug/vONooAhjgaRsBAB2YtTDEwBWSZV+hq/b+ENQLrLcXqxgds5rS/4Re0cf6TfFx/dTAFAHNeWnmoDKHUHJBarF1J9sdfs7gbRjYnNdFFpGi2z7kt2Zh/eJNaEclsoxFAiL7CgnmOMGlaiw4hlYdi3FTx+GNUmAdjHGv8AtNyK7D7QP/1ikMwb6e1MLnPr4YgVf9JvVL/7C5/rV630nR4MfuBK3cvV5hC33gTSeXb/APPPP1oKuAuIIRtgtIkHsoFNN5Ke4FT+RC38P60fZoP7p/OgRSYs+cuefembF9M1fa2gPdh9KYLWIjKl8VIrlNQAeaeqrnNWvsqe/wCNJ9mXsaYXIdq+tKNvapfsp7Sp+Jo+zN/C6E+xoAZx603eu7buGfSkayk3ZKEn2NI1vMBkQc+vegdyUgDvig8mq5juCeUY/hTWEq8FCPwoETZo3e9VWZx3x9aYZ9v3mFMC7v8AUikZxjGaomc+2KQTH2oGX0YbGb0BNcdbjzJmk9WNbt3dNFZTMO6lfzrF08fKK1iRI3LJgkT5/i4qcygd6qQJKwxGMH1qcWNy7ZIZ/UKKznuOOw5rlR0bNRtNlic1K1tDbpuuAsX++1RNd6THEzyTLKB/DH/jUjsIJvenpG8zYAJ+gzVFvEtlDj7FpReX1k6CqE/iDVZ8iOUxD+7EuKsDpDZogzM0aj/bbFVX1TSrTcDMCw/55rn9a56e3Oxi0kjP/ebimmFXiEbbyPRagDabxKglC20P/A5TgUXl9rELqNwQEZLIMVnFpN6EwoNoxU1vfN5nltvkVP4O1BQ1nt5WEt7dTyN3UdvxotrxxEBbsqyD+6uWqJsSlw7dTmkto2Q+ZB97lTmpGNmaSaQM7bJAfmA704qRkKo2jtTzG6FmeNSzdTmpYtiwSTTE+gA6mhDsEV8YMbIIwR1wOtRwt5vONjkk7m4FA2uofbj1NPE2wcIp/CgLAYZ3dflIQnIkPQ02UcsqnLYxuHSrED7o8EEkHgGo/tNnHJ82COhAFSOwtxEqqFLIzH+6c4FMgVoXjmx3wpqOS4+VY/LfcckNtyMVJauxjAuLeeTH3cRnFABHIzXDu2CeTTvMEgByEc/xY4pLoPjiJLfPGwdce9JP5UaIAMnufWqsK44Qoqby6vyASvUZoDpFJ8hfj+9TISEzsmEQ9CCQaJFLqZA6uo4ZjxzSC4875BncWHvQjyo24AFR1yKWC2muBsiIUH+InirSaNdJyt3EreppgJbWpvFK+Ym49s8gVC9jFDLhpssO23/69TT6ZdYXddoxJxlTTodEvIZVJvIgrHB+bJx9KSY2jOvrZ7uVVhLsVO5WUdvWofIuo3G+UuO4NdHJpqw4jku5GUdAgAFVJdLR33WiTSOeznIpoVinGk3yrEoOASSBipooBcXWJnyXxkKemBWlYaDqjr++K2yEk7jya1Lfw/aWxBeSSb0y2BTE2c81taeaI4bVpWP3cMRmtKHRr252/uFtEAxhiSa6GPyLZNsUaRr/ALIApr3qD+LmlcVynF4dtF2/aGe4wMYY4H5VeiFtZRiKGJIkHZRiqpubhz8pApuyR+WIJpgWJL75gFUnNVnmneTAOFxxxUgjApyx7mAA5oEVljd2wxNOWLBwKlklt7VczzKrf3Ryfyqk+pGRcW0LD/bcf0oAveWI/mkYKvqarHUYSxS2UzsOuOFqsUaV98xd2PqMirIDN90HPuKBkbfarg5lYRxnrGnSnRwpH90VL9nuT/Dn8aqXeoW9h8ssitL/AHEO40wLheTHOKhmuoLUbrmdE9u/5VhXWvXU3/HvGsK9mZcmsgxBnLvlmPdjk0AbV34kZ1KWabQf+WjdfyrId5Lh99xI0j+pNIF9BT1XPaqJGhF9KUAU/HOKAtADQKAKk2jFIhDAkZ49RTAMYNL1FKRgUwMxJ+X5aAHg8UtIQT93g+ppee5yaAEJPSijFFMBM04UnalBwOmaAFNHfFJ6UhyOR1pCAijoaM560UxgTxSZoNJTELRmkozxzQAZozSZozQIXNFFFAxKXJxmik9RQA7rSdKKKQBnNBpMGj0zTAATjmkzjqcUEjdjiloAUc8ilB9abuGcHNCgkkE0XAeTzSg0nejrSBCke9Ao7UA9u1AC7s8YpB70g60tAAO1KOpppyGH1p386AADBANHINHPrmgHLYoAOc89KPel65puOaAFBo470UEcUAHFJRR3oAM0tJjFLyTQAtGeaTOQPpRQApxjp1pOaXv9KOtACc7qCOaXgGjNAIbyBTTkg46048saaAcGgLkTwCQYbP4VGbJQMiR1q1g0pGakLlWJJ7d98F3Kje1X4tY1JJVFxdM8XfAw351BjmhkzQWdDaT294pRNSk3t91ZflOfzqC40rxVAnnQXa3EQ6KOp/CsB0rpdD1O1t7dFnbUS/qqb1I/Kk/dEYsl6JG8nV7Ro275Q1HL4U0u6iMljdRx/wB1Sa743Wl6oBbI8nmN0WeDGf8ACqCeE1tJvPs4QX/uMcihMR51deFr6AEqgdB/FE2RWYdPlHKuVb2r2R9Vis18q90jyv8Aai5FVJrLw/q77gU8xvUbWFO4Hk0V7fWb4Kbse2atJ4hlRh5iYPptr0D/AIQiVmJg1CNYz0DLmq0nw8jmuCLy6R4+zRIQaXMNROJ/t6BpGDR4z/dqGbUouqByD6riu8k+GGk+U32e6nSX+EsARn3rlb7wndaZOqyyIM5wc5B/zmjmHylCK6je3Y/xdhipbcxyZ46U2bTLnaVV0P0GKrLZ3cHO2U+680MUkdHaQRmMfLVk28YHSuUW9vVOFlUnsp4NPTV71GyyZ/GiwHUCBOm2mT2Y8o/u1P1rnD4huHOI1Kyehp512+T70ZP1FNoTLqaTHJcAsmAeoB4rYsXns18mG6nVeyhzgVy58RzqeYcH/dqxb+IyG3SQu5+uKqwHYLqWooeL6f8A77NWV1fVFHF27fWuS/4SqEH/AI9m/E08eLYB/wAuz/8AfYqLAdX/AGzq3/Pwf1/xpRrWqDrckfhXLjxXa/8APGT8OalXxPYN97zF+q0WA6Ua9qY4+1H8qX/hINW/5/D/AN81yr67Zs24S7V9cZqzDqlo6hkvYTns4waLAdCPEGqE83ZP4VOniDUgP+PjP1WsJbxXHyvA/wDumgXsKnDuoPoDmiwG6dd1M/8ALwP++KT+3NT/AOfo/wDfNZIvbUDmZfwo+22g6zoKLAah1nUj/wAvb/gMUz+078/8vclZn9p6f/z8pTTq2nqf+PlfwGaLAaZvdQPW5lK/XFKsk0q5M0mf96sl9fshhYS87noiLzTF8TWqDDW8qN79PzpNAbiqxPzMxqUADufzrDg1kpiR4zIp6BetSPq1xcKfslqf95jSuBuKwz71VvbpflSC9SG5ByvNczd3WouwMyPH6YHFQxqDknbu9WouB1dp4t1COUpfW8b7TgsCFNdJY+ItNuuI7oRv/dk4NeZiVl4zS5V/vCnYD1W50fT78ZntI2PZwMH86w7rwNbnLWd08Tf3ZBuFcZHJPFkRzyIvoHNXINY1a14gvpcejNkfrTuBbuvDGr2+WWASoP4kOaz7SxurycQRRkSn+GQbTV5fFeuKeZYm+sYp7eKr+XHnQW0hHRvLwQaLgTN4L8QjpZq3sJBmsm5tbqwkMN3A8Uo7MMfrV5vFOuPwt8Y19Auabc+IdUvbU213c+fEezL0PqKaYjNzQOaax284z7Cm/aIgPvbfZhTESmkpFZXAZTlT3FISwOAePQ0DHGjPFGaSgBRz1o6GjI96OO1MBM0pxSYooAKM0YoxzQAZ5pM0vfFJimK4UUUUAGaO1JRg0CDFLnjFFJnFAATR1ozxRmgBORS5NGc9aTrQA7tmkzR2pKAFzS5pDjtSUAOyaXJpuaMjPWgQ7J7GgtTcjPHSgnLcdKBjgaXNMzzSmgB4IoO3Ge9MzQaAHDrSmmZoBOc0AO4opM8UA8YoAxkVIH3SR7vQZxU014bmVdsCDBzVcQnv0qZEVWBWsDUurPEzhRbwrnu3SpM2w/10kIT/AKZjmsyYqTtIzTY2CqFzhR0ppgXbi4t2XFvG4/2mqFd3UMR9KrSTgNtUMcegp0dvqE3+qgnK+oXFMC2CAPmUtTWlUdI1HvirNvoGrsu7ymBPTeelaMXhSfaDdXaKO42mgDFaSKRAAQrdyTUDRkfduc/jXTx+GNKjcPNO8je5xV6PTtGhbKwK/wBSaA5jhVzn5lY+y8k1citb2dgIdPnK+tdj58UPEVvCo7cUx9QuG4XaPpQTcwU0HU2IBtlVT1LN0qyvhGJv+Pm/A/2UUVoGe6f+LFR+W5OTikO5HD4e0O2wS88hHq/H8qtrBpEP+qtAT2JJqExv7U5Im9BQK5ZS+aIYhhWMe1RvfSsfnYAelNC+tPUKv8IP1FAXITJI44Jx65pVjduuan7ZCr+VP3ewpjRAIcdqAnOCDU/HQ00kA8UhCBF96X5e2aTJpd3GKAEwD1p/ygetRbuaPMQdW5piJSQexo3HtmmecAcBT+NRtM2eDQMn3sB6U0yv/fOKgMhPXrTWfigCczEfxUC4buRVYMCM0bwKALBupP4VTHrihbps5KJn1xVXzBTDIKALzXZPUD8DR9vCjBjJ/GqO5SM7hURmY9BQI0xqA/uGg30J6kissb3OBViOzlk6igZd/tC2x98/lTTdWr9UV/8AgNRjTooEZ5njQAZyTWTHr9iuR5Jc5wAtMDWkEB6QJ+VMS0WXO21P4CsufX7ony7C0QH/AG8Gsu71fWC53yCNu6qBikM1datJbTT3LhFDEALuyazrRNqg4qgRPM6yzy72Pv0rTRcWjn/ZrWJlInl8R/ZIV+z6ewwMbn5z+lSxatcXdg00xaMnlEGAuaxmikliCPIXjBzk1fMS+QMl2C9FPNZT3No7FJHy4kvgszD+PdkfhSTG0klIELA9gBwauw200FwkjWoKLn903U/hVqe823Hnx2IjlP8Az0xx+FSMzT+7OMZGOlPd3YKseFzzgCkGNrPIWdmOeeKEJGSy4I980xWGqcn95Im31zTpxHGD5KNIR0IxQNogZwibD1bHNRylYwuGC5OOam5VhQdyAuuJD2zUkBCbyq/eGDUtoFjuk8ySMp3zTH8u3lfFxGUY5GDVJhYdcNHIVk8sIQMDFVWl+cKh+Y9qfLEsyApOGZTkKRgGmY2xhZNiSf3qAsTCWB4pZJl2tH8q7DyT60LEkUTSPJkscncajYtJtzKHKjAx2FOZnOAPnf07mgZYmEJKrBwpGWU9c1A8Myyj5cAUsfnSfJ5ADno5PI+lXLXTb6fDFx5fd3bAFSSyu0qRQbiD5voBUSSFjlM49SMU3VGSxdUF1FM5xxHyB+NMiuN2EjXePXOKLCuWzLdKP3DOD/s08RXpgWWa5cknAXcSRSQzLAC9xaTSY+6FOK04pJbvKLp4t8d5GG40guZ0lpK0Bd3y56ZUkn8alXRrqRVJQcDu2K1YJCFMRYgp1x700llmDNcuwJxt9KpBconS5Y8SSQQlR94A8mmMZJHVI4VDYxjHWtb7aqy+WwrnInafUrgoXkk8zCLEMjFAzZhhNpE2+VHcnJ7VBcbnufNjhZl/jYetX49JvruJFeFYcHJaRsE/hWnBo1nGmJV81v8Aa6flQBza3ieeqWdrLJKpz8vTPpWilhql+F8yOKzRTkc5NdCvlwptjjRFH91QKia5Re9KwrlOLRbdX3TTPKR+FXl8q2yIvkU9s1VlnZlwvFVzvbqc00K5cmvWjw28EdMVXmu5X+6AATUUkGbdjjpVkxKGzjtmgZD5bucuSadHEAelTLGzHCikaS2t1DTS4J6KvJoEP2AdKftKx7nIRfVjiq32u5lGLWBY1/56S9aYLPzGDXEzTMPU8UDJmvYkfbChnb1HC/nTC11NuEjCNDj5UqZY1VQFGKeFpiKqWNsjhxGS/wDeY5NTeWmfuipJGhgXzJ5UjQdSxrDvvFVuoaHT4/MftK3ApgbJbb1AAHU9hWXd+IrW3cpCpnZfQ4XNc5cX17e8XExZeyjgflUAQj0oAu3Wtane/K0wji7onGfxqh5ajpT9poCnPaqELSAEmn46/KBSjNOwCDcBgDmnLnvSgUvvQAEcZpBS546U3NAD85PTpSdaO1GccUCCkpaBQMM0uabxnn0o5pgAPzUppO9FAC/XpRkYzQaSgBc0maOKOKAuGaQmj3oNAMU0lJilzTYgzRSUYpAHWijtSUwFoHXOKKAOaAF4AzSZyKPr3pKAFpR0pKUde1AByTTT1+tL2pD9w469qAA8dqAc0bQcE9cUdMUgF74NAUl/lfCikzmjIH40APpc8mkzxR2oBC89e1FJnjiigYtA680nIPHpS9RQIXvR3pOKTNADxnvRTaX2oAQ9OKXPTPWjFIetMBaDnOTTaXJpALmj+EdKBgnrQeO9ABikxzTwyhGBTcWGA3pTM0AKBignke9A60Z9eaAFOcUDpntSFs8UDigBcZBPajikzR3oBC0nvS55NJnigBM0vWmYIOe1OBzQAY5peAMUnel45pDGruR1YY4OcEZBrorLxMLewWGa3k3j+KPgEVz4NB61EocysBtXniIXEQEMUiuDkO5GVPtT7LxrqVtxeRJcxfk351gY5oojDlGeg2viXSNSi2u3kS/3Jv6HvS3Gi2V9llVc9nTH9K87Mat1FT2l7e2Eu+1unj/2c5H5UWEdj/ZGoWXNpP5ijojHn86fHrssO2O8t5Is92XIrLtPGkqHF7abz/fiIH6V0FvrWk6imzz4z/sycGlYLkJax1aLy5DuHbY+Cp/CsXUPB9wU32l205HRJjg/nXQT+HLSV98WYHHRk61Se01nT3LROLtO4b71AXOJvNOvrA/6VbPGp6NjI/OoYzxXeJ4iVXEV9atAT2K5H50DTfD2uZaKPa6khmi+XBo5gucG21j8yg/Wo3tbaT70CGusuPBEyK0lpeCYdlfArAvNNv7Dm5tXRP7/AFH51aYFFbC1jdZEhQOp6nmrBbd99cj/AGRzTFbNONK4D2gtiMrdYP8A00ixUQhH8KLjvwKdjNKCR0ouBBJZwyHmNR9BTTptqwx5Qz696sGmbyKLAJ9mUpt2rtx0Ipi2UBj2PChx0OKm3saFYk80wuVG021/54gfSkGmWvlsoTGe9XeM804LkdKAuZ6afAgAwxx3zjNOOnwk5+b86vbRTSMUCKwt1XoT+NWLaxguGImmSP03d6KQjNABPplvbSqFeKVGz86HJH1pqWsB3b2CYGRkcU8DFOoGR2+be5jntmRJYzlSwzWqdYvJVKy21nNnr+7FZjLH3FNSYI2VbAFSUTNJulLeWqZP3R0FCmVSxR3QsMHacZqNrtvP8kBW7jjk1ZwR1GDRyoOYYZbh08uSV2X0JqJoskHGcVYwO4pCAOlOyEQlRjgGlA9akz60nHemSOBFG8Zxmk2qDxTtoP1FAC5A75poODkdKCh68Ug3D6UwHZH40ucdCR70zPNKGHNACOGZhlzx6U5sMMOAw9xTVOe3NO4PWmAKNi4UYFO3bhzzRg49qYQOCc8elADs+oxSnFMJGODSA46nigCSijNAI70gG5wcUo5pMg8Y59aM80wFJwaTPOaXANGMdaAEzk0tAFJht/WgTFoo5BoLbRk0wCkPXincYptAgNHJFLRQAnakNLSHFABmijFGM0ABOKKTFLQAUCkFL1NAgpM80tJQNC9KM0lHY0xAaAT0oFB9R1pDFzikzRyRSdaAHBhS7hUeKKAJMigEVHyOtKORmgCqLa7l/wBVaTPnp8uKtw+H9TmTd5Ii9nPNdO2p4+5Gn5VDJfzv0IX6CsyzGTwfdyrm4vVjXuEzWna+GdIs+ZJHmb/bbj8qQzSkcyMfxppLN945pDuaAGl25/d28WfZae2pqvESKPwrNApwWgRae+mY53Y+lV3kkc5MjGjBPalCkHJoAjC89KXYak6GnAUCIthNKE4zUhBP3eKQDtSAQCncUUYpjCjJ7UuKMDvSAM468UuQRjGaRtuOKaTimIeMqMDpS5G2ot/FJuOM0DHk+9G4VCzUcjrQBKZMjFJuNQljmgscdaQmSk5pucHmo9zDuKiL5f5ulMRbDp61GzoDgGqjTooPPSqn2wKfmkX86ZSNVpBTC4PcVnreiTIBYD+9t4ps9wLeNWfeSxwARUgX9xA5Oahe5CHBNWYbPNv5tzOIfYHNZ32zR7m6NvDJJPJ7cA0wJvPBGVBP0FFuXmPz7Vz2Y81VfWGtZXgtLRAU/ifBNZV1fXfzSMfLJ/u96kDonSJJMNIg/GrhvdLtIQGJd/7oHJrn724aGzt2VdwdMjjNVEimmzJJLjPRehqeYDYn15n2m1sRCFOcs2SfwxVa61y+uBhZPJT+7GMZ+tVfI7eY5/GmsIkXylWRj6kVopFEHn3C3Cy+e7Y7NyKnEcM92JV2hv7q9KQQMeAOKWK1aORjFDl+7YzS5gJXVWbKtz7GnwjaxD4k3DHI6VHFHcM+Y7iHcP4dwzUkUrMx+1Pu2nG3GKLgUPlN4Qv3a1NhW0IPcYrOTZJeuyLtUt8qjsK0pIpJURfMESj7zMeBW32TF/EVkBWMDHSpY5HQ7XBAPTIqOa/soE8n7V50n/TNM/rUlxrt1qSwwtAI4IxgHGCfrXPLc6I7DGM08zOzbiafNGYi0WSXHc9qSF4lImYk7f4QeTT7m5jZRKD8zHAU9aENlZ1l3YO38TQftkdq0hQRLu2nHOaljuoVbdOuR65oYQsXdhleNo7UybFazguYn86K42oQQfNbr+FLgSSt5sqyOOpXpUoYr0Ix7UqqCcnA+gpDLVumlSACWfEx6dqxrkrFfIAgnAzkCr8s33E2gbTnI71GboQjlSc9x1oArNgzpI9rKoX+A96uXEakh2gZ1bpkVMfOwkcpDMeQfQU2bzHdmd02g4GTgfhVDKUbxCdApHPpVoMivkmobcx3AkZYBAU6u4xmnwGGRsbCx7sOlEhFhWtixXe5btsX+tSxadJdRNuJj4yN8nFVpfKD/uhtU/mKj8uI/efP41IFGazgd0Dqrbjjg9KtR6UlpKj+ay8ZAqeNbZYpQoBYr0NRLObtwoikdh0wOgqhNF+O+woWYb1HQZxUj6ozoxMJJcYG3p+dW7PQdQmQCWCOFO/mcH9K0bfw1a25xNPLMP7p4H5Ck0KxzX264m3pbW7PLxnYOv1q3a6FrF7iSVVtEPXcen4V1sUdtaJstokjU9doxQbkg/eosBmWnhO0gYPcTSXDDsTtH5Vrww29ou2CFIl77Rg1A123Y5qJp5G70wLzzKFO5qrSXHGUyRUITKg8596eq0ANZndOSaaY8qOOlThcDFPWI4zSAjSPK5pRFzx1pJLu0twBJcIrelZr3kV1LMRG4f7sTYyMeooA0rgeXayPIdqAcn0qC1lFwhnEi7D8q7uDxWLb6NqD3TSXtyWjc5Kl8n8q0F0u1glLru/2V7D6VIBcNqMszJFMBbg8EEDNOt4jAMeUGb1zmnBZM8AYqxErFvm4+lUkFxd1yf8AlmKBFdFgNoGfaq93q1pZH95Nub+7GdxrGuvEl5OhW1Qwxt3YgtTsK5v3U6afFvurlE9gMmsG48TzHiyTK/33H9KxHBlcvMS8h6s3WnKoxwKsBZXnupDJczNK3bJ4FAAXpTgKMUJCE6dqX607bml20wG89qUA4pcUvH94A+9AhFBUcUoOOlLkmkxxTGLyFo70E5NIaQC0DAOcUZwKDyelAhMknilKnHXIo6dOlGR0pjCkHIooFABS4yKNoJooAKO1GeaKAFNICO9GAR1xSH1oFYKD1oooAKKSjryaAQUUe1FAAKO9Heg0wCiiikAUopuKWmAE84oFJkUCgBaQUvag+3FACduetFLRQA3achs49qUj1puMEjtmlFAC0opoOTSg9vSgBx6dKXtQDzSbsAjt1pAKBxzQAAOBwKQHNA4HPNAxe5NJnmlpcDvSASjkUNkgY9aXBIBpiuICAxBBpaQnJGaWgYc0cYo7AUEDNMQlKKQ8AmjPA5oC47jtTaXntR9aQBngUdzRRnigBMf4UucGikzQArZJzRQKKACjIooA5oBC5HSkPoKBjJoPIxigBOc4J+lKDil6kHuKQigBc0HIOccH9KQDPHejd60AgJpRQfQml4A4pDF/hyBSYyKQUufxosIQijFPwDRgYzigCLbmmsqkYxyPzqYgGmbQTilYZbs9Y1KwYGG5fb3V/mBFb1n40jB23dk0Y/vxcj8jXMbfek2AjHNFgPTIb7TdVjIjeG4Ufwt1/Ks648M2kkxmtme2l7FTwPwrgCjI4dHdSOhU4Na9l4o1Sy2q8guo1/hk6/mKloDdeDxBpikxSC6iH8JxmnweJ7fd5d/byRHuGXI/Oiz8Z2E/Fwj27Dpu+YH8a2v9D1S1z+5mRv4lOaVwM19P0DVi0pRN56tH8p/SsXUPBskSCSyuPOGcbX6/nWxP4Vh3b7OdreU9NvSqcsXiDTSpZRdxDqQOaLlHM3Wl39jzcWkip/fAyKqK6t0rurbxZbNIIbuNoSezjI/Op203QNV+YLEGPR42waLiPPmqLvXXXfgaRTus74Op7OP61g3eh6lYufOtXKL1dBuFUmBU4xilAXafWmBsnHTFSEfLheOOtMQgXnBFHQ4BpR5i8MQaAvJNAhe1KFzyaTvRuyKBjiOtIF9KDkigdBg0AN2kmlC804MKWkFyFwyt90kUhIkGBj8qsgjPzDI9KYQpPAxTGmUJLdlfzRzjsOtWIr2SX78bKR3PerBUGgQIR0oAcAXAcHINFV3hkjfdG2SOQDUzLI0Kykbc9U/wpiFxnjFDRgGmBTj/AFjKPTFOU475oEIF29OlKN3UUodR25ppwxyMj2pAGTTgT3pMUAUwEwd3zDjNK4GeKCCBTdxBouA3oPanDJ6UNx170BgOhoAN7A8H8KcHLA5603KkcdaQnB4oAfSjp0qLdUm849aAHdRQVDd+lIHU9BijrQAuKToKXOKTrTAFNLmkBpQcnigBwJAyKQdKOhpN2KQC0oPXIpuc0AGmAuAO9L8vrTO9OGCMg02KwnelwcZxRnmjeR0pDAgdjSAUHrS4oEJnmkJIHApaWgY2jFL+FJQAUZxRij60yQzSUtGKAEOaQ5yMUvTnPFL9KAEpDQaMd6AFzRkUhyaMCgBaTvRk+tJQAvFHSiigDaCgtycU4gA4XJp5iAOaVeDwKyGM24FKFzTsEml2kc0WKsLt46U3BHUU/Py00kGgQoIIpCcds0gxnmlLUhiAnPSnFm9MVHvwevNIZDmgRLuApDIKjLZFMLUwJvNUUhkBqHeO5oMkQH3j+VAyTzuaUygd6pyX1uh/iP0FQNqUWeEagRo+ZmkZx/eFZb6soPCgexqu+pyMflCqPamM2t49aTzCDisP+0JO7U37bOx+Tdn2FAG6WzRvwKyI31CU8Kx/CtGDTrtgHljkb6cUAK83pUJecngVorHaWy7pvLGOuTnFLJrekRJnztx9FGaQMoLBduelXo9HuZEBJ6+1Vh4xiH/HnZbm7MWx/Oqd74g1KcYhlTP8ewfdP1oEao0COOPzbxo1PfLU3OixKFCpK3YKDmuc3X8y7rhnmz2c8VNFeXltjO2OMegpjNLVb+VYVW1i8kd8isGR7i8AWU5PrV64niuo93mF29DVaRkt0ychvSs+UoiGm2y8XFxIzH+FO9RyaTZHHkMYcd881PHcoy5Yc1HLIoI560coEllp32di0cwduxc8CrN3pV3exbS9uD6hqoeYhH3mH0qxHerbwZ8uVj0B60uUkh8y/SIW7kMinYOOMVPDbdFkmjjA/iJqNGkcbmGM84pfMK9KpRKJp4Y1AC3hkJ7DpUllJDBu8yVucdeazpbmeNseXu/3RVi2luXXcqKnsy81VgJRIeSoIXccEjrRa3n2F3Pl+ar9QxqvdX94QYwUP/ADU6otxpXmQ5+0H7uTSaDmHSXHl8jT44SfulU5zUYs7ueF7qSOWNFGSDgCnWGszwRLbz6VNNOPvNxjH1qS4u9avYniJ8m2bHyvjNOwXuVNOikaZW29at3wxLt9hn61lWlz5XmxvdjzUbauDjimWWsRf2u6zM8yoMgds1XMLlNOKLy0xIB6jYOophGGz823sCavrq9hdIxEbKx6ADJqnLdwofMQK2P4GH9KkZCZY4TvMQfHY1dMu+HMFmsEnrIMn8qgXW7oyeV5EUSHsI8ZqVL1FfMkTH2FAXIle5kXLm3/AAGKWO0SbPn3KovsatXckFvMn7nqM/LVKZ7lUKpCN7ElcntQFxsaxRHy0+6Kmwh+4c02PTmSPzHMjOQCQv3RUgiAwEkAf+6aTQ0ypJnAZipU9R6VJbzn5xA0e/GcuucVJFcJ52JYwX7/ACmmXUjCQ7I3jVuMBcCi4rkUF3cecDcoBx95vlrRtp7MZE7x+bnIJ7D0rNDSmVRsEjdgBuNW4/D2uXZDpaRxL/ekO0j8KB3J57k3ULoW3KTxgYAHpWdsdSVTls4K10UHg+4ZD9s1dx6JGlbVto+l2UW0Q+a/9+U7jScSTkbSwlu+lvNIp7oOPzrXtPCcB+e63qvZQ/P410RmReAcL6VG9ynrVWApQ+HdFtyCbfzD/tsSK0xJHHkRLGi+ioKovcbulR7nfqaB3Lslxs6t+lV2vskbVJBHWoxEMYOTTlQAYouIYWlduelKsZzg5p6r+8B7VJjBNAEWwCpEXBzipUjLdRUUt5Z2xw0pd/7iDJpCJVXd0FO2hPmchV9Saom+u3XFtAIx2YjNNFqzsZLgvK56lm4oKJ31KNeLeNp39QvAqpMNTvBly0SH+FOPzq4vycKiqPYU7zG9BQBRi01YzxGCfUjJq4ICq8gCnDe1Mnmitl3zzxxKf75oABGCRnH405o0UZaRVX61hT+JoxlbaFpP9pxgflWNc3NzqEu+4mZl7J0AFAHRXviOxtWKW6tNKvXHAH41z9zq+oX6gPIIoz1SMYFVvJUdABT1QrVEjEjVTlRye/epAp60YwadmnYBm1j1IpQKfjj0pQPlzimAzHNPAz1OKQmgNnkUwFFApOaKAFwM85FFJS4wM0AAPrS/jSA0UgFzRmkpcUCAjBwacOlN5oyaYwPWjHOaKXt1oAO/tSHrRQMZJoAXPHFHJo6UZoASjvQaO1AB3pDSd6U9aBC9KTNHajrQAUmaOaSgBc80HrSUZoAWlzSUUwFFB9qbzmg0AOopPel6igBoAUcUtFIflPUUAKT7Ud6AcZooAMjNIrqTwc0HrxSH1oAMc9aO9APNKKQB0NKPvU1EC4APFKD8+Og9aYCuGIwG20+NimCMZ96aaBQAUtN+tLmkA7p0oGMUnNHbHegbHZpBSDGaU+lIA4pabSimIX/Cg0nWgfr60AHelpGwWJHftS42tz2p2ADRnjFIelA6YpAFHajtRQA4daKbyKB0OaQAKXvRQPQ0wEyDyKXNJ9KUjsetAgHOfaig4xQOeKBgCeQaUDtSUZoAWm4zS0d6AEBxxilPPSkK560elAxQaXoetNOD2/GjHy7vfFADg3PNPPTOfwqHtmnA+hoAk2kjg4pAppm4g9zSljk0hC85oxz1pMn0p386AGnnIpu38qkGO9H8qAItgbjFSQyS2zboJZIm9VbFIOGBpxwxzgD6VIzatvF2o2yhJNk4HqMH866Ox8WaZd5WWT7PL/ck7/Q1wO0UhQGk0M9LurHT9ShzLDG6t0ZaxpvCUUeZNPlkhfsN1clbXV3Zf8e9zLH7Bjj8q3LTxdeQoBcwpOF6so2n9KVhloTa/prBdizj9auW3i6EzeXeW72/u3SrNh4k02/UYkWJ/wC5N8p/Cr0tjZ30QJjVh2ZCCKaEULjTdC1h/MKxhm/jj+U1j3nguWMFrG4EqdlfGfzq9L4RVWL2U7xnsQelV2l17SuX/foP4t3NMDmrzTL6wdRPbSBT/EBkGq4KnpXbW/i2EYivIjE/oyVO9noGsL5gRd//AD0jyppiOEEYNHl4rqLjwa33rK+Df7Mg4/MVjXekapZDM1ozIOrpyP0oAz9vek2c9aXzAflJwe4pSAOjA5oAQLR0NLk4oHIzQAtNFDnFXJNO22/nwXMM6D721sFfwpgioT3o3flSc7TigY9xSEKr5OKqTTS2dwzsfMtycgf3atZKjKgfjTXZZEKyRAg0hjftdtKm9XZR6OKerpIvyMCagCNCNqAPETwjdRUqCNRlF2568UwJNpHpRSdqMHFMQho69T2pOtAwO5pALjHc0UuRjgg03NMB6sFzlQ3pmnyPbSID5BjlHGVPBqMCl2A9aAI8AHIFKSCKVsLxQMUAN60opM09TtGMA5oAbgZ9qN2OlBYAkFDn1BpucHmmA8H1oPqPzoXaetBznGeKQC87eKTf7GlHApQcUARpcIx2kMG9CKceTTiVJ+4M03qc8UAOAGeaCfm4zR+NAwTzTAWginYqN3MZAKk5GcigBwoFME8R6sUPutPUq5+V1I+tIBaTkfSg5B5GKORTAXJopM+xpCQKAF60fSgEA89KccdulADee9JnNOoIGc0EsCcmkpSeehppoEFLQOlAA70DDFIKcw4+Ug0360wCjFHBNL9KQCUmKX6UfUCmSIAaSn03vQM6QwTf3aTyp1/gFWRHIelz+lDJcDHlzj3zUGhW2T4/1f6Uwrcj+A1c82+QABY2oE93/FAh/GkBQ2yj7ykUFigyQTVwvPnLW/603zcf6yA4+lIkpecD060b2NWfOtz1ib8BRm1P/LF6CipkDksKaJFJ61ZMdn3R/wAqQwWQXOG+lAFRpADjNMaT0NXPsto3McYHu1Me1siR5k5B9qoCkzmojluprUFjYHpcMx9AacNNgP3JCo/2jiixJiNBvP3gPrSrYO3atxrewgx5k8Qb03VR/t/S4JOAzAegp2GQLoDSdZk59qtweEkVcGVj/n61Um8ZquPsVplvVhmqFx4i1K5iLvcFc/woMAUmxnTw6DptrjzGG4di1NuNQ0XTmyX3Efwrya4qKWS7Y7pJHPc7qBGGl2CPPuam4HUTeM7flLKyLN2ZlxWFf+KtQuF8tpjE7fwoMChCbOU+Vbh090xTJI4pn3NGq/Si4GfIl1LEC0hd3OOtWINKvU+V3UL3Oas8RoBjgUsZmkiMsRKKP4iKLgVfsqxRmPJIPX3qSCBgmYB8h6Ukglkky87tnr2q7aRSOBHG4UDuetFwKZjmJG7zBg96leENHkySsx7Z4p8zAblaZjjuTUQMysoWJ5FP8QFOxQwW1zEqhSDk9zV/7BcvblpImYnvUjwuke+RGUDuelQx36SRGBWkBbupyaTQFddHmP8Ayz/8eobS5YlLyxFVHcmmyu6oVjknDnoSc0W2+RSs7TNGoyRnFKwDM2wH3xn61JbLFNMqBcyE4DHgVMJdDBxIsm70JJqyL7Swv7qSRPdI+fzoEWLm1S2YJIyu3fB6UiWFrMocxgA9CWqLyjMjTxQszes5wT+FQQ2d1db1up0VEGREg4zQBrxiG0TerptHXPNZ4vobq8d0PygZzVaBLQowlt2O04Gehoa2tLl8qfJiUgsAeo9KYyee7jXC7vMJ/gQc1EGSZA214V+vNWbPWLWyHkJYpH/db7xJqxcXccy4dUJ+lAmzDmWZnZROzLxjDdKgeS7VlW2lWTbndvbI5/ya1obeW7RmDxpg4wRii0SW1vsiJHYKcjpQK5jjSFukaS7uY4VHUInP51PFodony2rHzezkZNdDNo8srO7YgDcnnIzTba0S1IYXkOehf2+lAzIFpJ5uz7RJ9SgFWYDHDdqpt2Zycb8dPpXRSzafIF3zq205GBTfMsBL5Z2gnhQxwSaOYClJd/asWtvbNv8A77cgVA+m3RxiSL8OKt3V+tpKY4LeIsOmHFZsWo3ZiYCWEBzkrjmi4FqWxt5QpLMzqMb88ZrOmhnkuNoljD9Fy3b8quR2+p3UYRLb5CclugNaMXh47f3t4Y2PZAM/nQBjrJ9lPlz33K/ePYfjU0Gpvcz7LbTPPbtMRwa3otG02HDGBZnH8UnNXlZVGFREX0UUAZFva6pM5E1pbQD+8ev4VbHh+y377mSSbnoTgVcLmq0l6qsVz0pAWoksrQYt7eKM+oUZ/Onmfdyck+tZL3btnC/lTFmklHLHFMRfmuApJLflVdrkt0z9TVYqc9SSak2fKv1oAeXYjFNCfNnNSGPmnBKBjQoJGBTwp7U9U9OT7U4qV5kZUHqTSACOKbUMt5F5TNA3myKcBOgNVVTVblSTL9mU9BGOfzpAaLSwW5BnmRc9B61C+qRGQLbQGT/bNU00NQ4eSZ5WH96ra6ei9qaAQtLOwZ5QFPbpinIYLc4V48+uM077DF705bNFOAo/GkANfIP+Wo/KmG8DnCvn6AVJL9ktFDzywovvWTdeJ7OMbLOAzk/x/dA/xoA1lZm6g/lVS61extOJJiz9lQZNczc6je3fyyzEIeqJwKqbFB4UVQGteeIr64OLRRbx+/LVlMjSsZJnLuepJqQcDpSZ9qqwhAMdqXFOpKAEycU7kjmm0deKoBc0vOQQSPpSY5xTtuODSAOT3oA5zTu9GMUAMVvVTTs05QpOGbHpSxmNMlwSewpgNxS7eOKXqc46ntR1OKQhOlGc0h4JpoYkfdI+tAC4pQMd6KXrQMQCnY5wKQU4ZNACUdsUd6TNMAOelJR9aWgAPNHSiigGL1NJwBSdKOooAO+KOKTmgfeAz1oAWg9eKMDPWkzQIU9KTpS0UANzzRQRzQSAKAFpKPTjrRQA7NN7UZpaYCd6M0ZooELnjFHakzRQMXPNJ2zR1PFIT0+tAC0UmKcBu4oATueaBRjjNHakAgABPPXtRyOopCfmGBnNO4JGDTACcUDmhwwBxw3rSZoAcCMd+KAwbkAj60ijqQc5HSnKcAj1pDDmkHXFKaUDPtQAZ4pQMmkpccZoACMcAUlBODilFACdvelHY+1IRQOmeOKAFpKUMCMjpSgZBx1oEGfUdqTNICcYJzSc5oEOJpE9TQPSgUximgZpDRQA7b34xSdeBSE80o9utAAODQeaO+KKADvRSZpTQAdqTk8GndqTkc1IATQPSjtRTAXtRk0nalFAC5yOaSjdnNJmgEJ3FKcg4PSg4wKO1AC4ycdqaF5waUkg5FLuX8aQCcfjS5pMZOe1GcGgBSaAxzSZJoAGcGgCTOe/NGQDjPNR/JuIz0pDxQBITxzSbjwO1MBp6470DFXLNjgD1pp3D0pCcc4OO+KQ4bgcGgQ8E45oIXrTS2VHrikBOPegLh8h6gVZttQvbFs2l1In+yTkVX47gUEjsBUhc6az8cSxL5d7abv9uM4rpdO1uy1VcW0qu3eNvvV5n35ApxaMfdBDeooGen3GnW0y4aFQfpWFc+E0TMlnJ5TjoAeK5ux1/UdPf5ZRKn9yUZFb1n42tm+W7t2hYdWXlf5UhkcTa9pz7WVbhfTPJrRtfFSI2y9ikiP0PFatnqFpqMXm2syzJ3x1pl1plvd5EiZB7GlcRXlttF1hhut4pB2dcA/pWZe+B0bDafOUx/BKSf1pt34TaFhLp00sLj+FTxUSXuv6YoEgEyr3zzRcDJu/D+q2A3TWrFP70fzCs/pwQQfQiu2t/GloSouI5ImPZ6uXEejaogdlgb0YAAiqTA8+27hTViI612Nz4HjwXs7sq3ZHJNYV14f1i1+ZoBIn96M5FO4GbsbtTQrEZIp4Zw211KsOoIxUg56UAQsnHJ6HihQeualZSeDTNuDgdKAEZeM0YpxBwTjilJ+UYpiIgOcUHd0B4pwHqKcBxSGM24po61J0JzS7VIyRTJIf90AUgznBqbZ6CmlCDmgYzdg4PWl30rKSc01lIoAXqMmgccdqbgijNAD+DSd6QEetKOmRQAn1pSMjpmgZxjPFA4HNMAA9aQjJ4zTj6UnINIBhJA5pynIyDTt2OoFN3r6UwEJING40uV70nBoAUAn5uKXPvzRtFJg9qAHqxxQWNIFYDpQy7h1xQA7eW6gflUTQQE5aMZ9RxThuXr0pc0AR/ZkH3JpVPualye9IWpu4d6AJc0nemliBn+dAfHWgB1JjmkMi+nNJuzQA/PFIDkUgNOH0oAA3NJuTdgnB9KCOc0mB3A+tArC8dqAMnrQVXbwcUAUBYUntSYpe1LjjIoFYZjFLmjGKWmAhzSZpxxnBpMCgAJGaU4pMUmMmgRoeVKO5/OjEq/8ALRhTzIx71Hvf2qCgWa4U5WRz+NJ9svR0kNAZh9KmS3mk6RHB70AQ/br7/noaPtt7/ExNX4tLlPzOQF71bTT7cd2f8KAMgahd/wB1anjur1uiCtlbAKPkgzUcptrZd1xcRRY6gnmgCkqXbfeaIfhVkWxYZxmqtx4h022/49oXuH9+lZ1z4j1KVgbdYbcf3cA5pDN1rN0TdIVRfc1n3N5p1o2ZJkZh/DjJrkb271O5l8y7upNg7LwKbaGJpczKTH6nvTGbsvimJX/0aCIY/iY5/Ssu7vb3VGLNOEX+6vAqwxjMoW0s4z/thf6VMbeUfNcmNM+h5/KpGZltp6mYNcy7of72c1dudPtWhzAdq+rd6deTI21YHWIKMdOtVo4Tt82SbzT6dqLisVrd5LUn9x5ielAuUkuAZLdvdBwBVxbkgYHAqKSZCeULfSi5Vi7HcWzrsKhMjqBWbPdPAS0Rj4PBJzmp9kZ7D86EggEgPlqW7E9qTE0Ed/PdRfvM56dMCmSCYMQkLsPXFThNnQD8RmklvrgDYrChBYpj7WT/AKkj61Mjy8K/B9Kme6U25j/jPU1Z0qy+3hwZfLhBwxHUiguxnS+YzkxD5e1XIIw1tliQw64qzqU2nafsS2iDOpyxJ4/CsybUJ9RUQoiRRA5+UdfrQS0OMEaqzr8xHducVNbyXaKp3fuR2oaYmPygFCegFIJWxhVDH3pisTNdvLhZYnaL+6zdag2NA5nS3KL2YmnxLKxLyuoB6Z4xU81jPcRo73H7lRgDNAyut7IzhtqjHpT/AD2dizNgH2oSGCMbWmB91p0sRh2SRuMLjbigB8l3EMFY1GBjlaZ5Rn/eDYP0pl1LdX0XzrvC9lXHWmR2El1tEzPGgHC460AOdTIcGVvzpsJWxibaznccnvVt7KyijyJnRx75qWynszYmAwBmIwXfqT61IFFZfOXcolPrzirtnaQXCGMWbqw6NnIqyljKIAsZVQfWrsJure3EZAVe7gYFNAZR0Ihy1wSo9u1Wo9GsOMSSfiadNdeZKbeO4BftnofpUqNstwWUlx1wKYmW4rBUwIUytWH0iyc7pn2y/wCy+MVgzagskwje4eNB2XOTVdrC81C7EtqJ3i7MVxmpGi9qX2iHckbSTL/ebmqa3cUdpsaFBMO9bWm6VdWyYupAV7IB0/GtA2Fk3LWyGmmDkchDcXEiqBAjynoU5/Sr7aDqtzKGmSFfUseR+FdMqRx8Rxoi+iilaVU+8aYrmJF4ZjVw0+ZD7NV6K3tLFvk08rj+NRk1O14i96iN9u+4CPrSC5N9qWT7u4ezdaYzoPvNWXqU8gv4fLbG9eaUB2+82aALTXka8Dmqz3Ts4AyM0CH2p4iHpQIQBmHJNJ5YJOQasKAOaUnIwKYFcw5XIp0SADFP+bGKdBA+ws3AJ6tSGNMYPSpQp2jimtd2dvhZH3ueip1qF9QupF220IjT1c0CLvkkJudgo96ga8t4jgMZG/uqKrC3STDXNw8jemeKsJJbQLtjjApjIJJdUuZQYAtpF6kZanLp2WD3Fw87D+9U/wBpBHAppkfsuaQE0cUMX3YxmpC+TmoEErDpiori7trXP2i6RSOwOT+VFgLe45p+Ce4H1rnLnxIp+W1iz/tv/hWNPeXV0+Zp3K/3QcCgDq7vWrG0480yP/dVc1kXniK8uUK2qeQh7n71YoUL0FO3Gr5QEffKczSNIe+6lAUDhRTgKOBRYQ0HPY8d6djmlL4OQDmmqf735gUwFoySuSAKbmlXDDPamAo55FJg0u3ninjOMYpCEA9aDjNOzg0nAHTNA0OFNpKcOmaAEJxS5yMHvSYz1oJGaAFH3h0oPSgH2pMnFMBQ5wcccYozTetKAc0ALRjJ9BRty3JFGBSAOO1Oxg8ikA9KN3NMA6UuCByKQ9vTFAyTgUAHU0UlHNAC9aUZVqB6UnegBD60d6D19qM0ABNJSj34ox+VACHJoFLSUCDFHejHGc/hRQAppKQ9aWgANJjNBNKBx9KYgDHABPAo4pO9GKBgetFHtQRQIPxFHfNGKOKQwK8c0H7vWikPA5pgHXrSDBJx2OKXtRjFADmbcc45pAT3pATn2oyc9KAHD0oUFhkemaTIHJ6UZ54oATNGeaTnNL0BoAMkmjpSDOM0FgvLsoXtmlcBVOOKdmmggqWAzxxjvSR52nKMncZ70DJV+ZeePrSYG0etJjIyTSdzSEOByKNwyFyMjpRtwAexoIBOSOaYAetKM0YpPxxQMXrSd6DnNFAhccUDPWjNG4/hQAYpSvy57Unv29KTOQCBg0AKQFIwQeKUHNN6nNLnIzQAmecUop28LGylQc98cimdqYC5weBRSU6gBO9Hej3oFIBORS5GMe9DEk00DJoAcKXikHSgcigBTSUHrRQAtGaQ9KBg/WgBRgjOKUYpBwMUZyaAA85oo70H0oACe1IMZ5pSDmjHtzSAX1+tJikIpaBCdBTST+NPU4YE84ppxvyKYwJHVh1HalOD0pGOcUAcUDF6oMUq/wC1+dNPtS845oAcp25561ETsIOcZOKcc00nK4IpCAjB605ZCuRng1HnnpTtgK5pjFyTz2pucUpBBx2pAN3FIQpOVyDzSBhinLHnvijZg9KBiA5NKVHekxjtS5bBzyKQCRtJA2+CRon7MvWte28XazbOvmSC4QdRIOT+NZBOaT86loDsrXx2jMRc2bqOxU5/pXT281nqFuJomWRG7g15NToJ7u1lElrPJE3fY2M0rAel3ehaddqyuuCehxyKwrrwaC+6zuGiHbDZqja+MNSiRRc28dwo6sow1dDZeJtLvEAafyJe6SjGPxpgYS33ivRMoYRdRDo4XP8ASrNr49dcJe2AV/UHFdZlQeWB91ORWdd6Hpt6zPInzHvUgJb3+i+IIWWaCISHHyyEA/gc1Ru/BcDKTZXTxt6OQwqhd+CYWy9vO6P2Dc1Elx4l0BdzkXduPXkj6cU0wKdx4d1i0yWtzMg/ij5FZ6zKnDghvQiukj8foP8AX2Uiv7Gr3mad4lgUtPGrN/CRh/8APFVcDjvPT1pcg11s/gi0dALe4kif/a+YGsO68LarZnOwTp6x9fyp3AoBgFwaYWGeKjLYykilGB5BFPVVI4NAD8BsdqCNvBx+FRNGPX86PKAbimIlypBx2puaZjBzSfuyfmJBpAKWoVwDzSAJ3YHNIVHamAMUbIxim5A4NACk4JNKRg+1ILjWwozSD5lyDilbaRgjNCgEcEZ9KAEDEZG4Gn5yOtMIOaaQf7xFMCUcGnE9jUQyR1z9aP3vYj8aAHcg4JyKa0ZzkNx6UoZ+6g/SjzBjG0imAgz3pDycZ5qRWQnBOD707y0IyRzQBGGOcd6fuwBnvQ0ZYcGmeS2M5z7UATD60h60i+lBzmgBwz3GRR0PSmljn2pNxoAeQM00qM0nOacKAGkenSlIG0DFIcjvS5oAbsBNIUHvUg9cUdecUAMGc07JHelIB6UmPWgBCxHUUD9KXbS/dyPWgBuPQmnDPrTRTu1AC896M+lNye5zRzmgB2fWjr0pOfwoyaADBPNHSlpvNAhecUUc0UCsba6fL/EMVMth/s1WfxNb7ysEDz+hA4NQTa/qci7YrURL71AzXSyZOQEA9WpZZra2XdcXcY9l5Ncde31/LxcXzID/AAK3FVg6n70maQHS3WvB/l0+Hcw6yPwBVI3+qMP3mrIrf3UHArLT7Oeqsx9qViycxxBPciqKJ5/7UvSQNccAf3hiq39gwO2+61YvL3waJrUyRA72yevNU12xThTUhY03ij8rZvDehFNgj249qpyb92IIyz9sVZWKQQ7piVb+6DRcdidjJN/q4xheSWquwEuN6gY7imLcNjy4oG2j+ImlWXzH2KhzSuBftFkjDGGbyVPU96hmjxKSWZie5pEkni/hGPeopJJJHzigLiApFyVU/UVKLxim1VGPYVWkiMhBIYAd6tRpEYgjzBADnCjmgYxLWeaPeIxtz0zilMSbZPlVdvXB61WljkJGyYqc4AzxirDxSABCcnGSRzSGQQW6CIyyI4I/iY9KJJF27YHJc9KkDAqIXQsp64pH8uGUeTC5I70ANWGVIS06zE+uetPhjimhDtHIPYmpEnd7gFtx+gqzcx2xPloZPN7gnFMRSjuYEOx7YMB0IptxIxHmRRbIvpV2O2j24I+YUDAl8sAE+4oGmYk9xPdZggTbGepx1qxYW9zbxyGVPlC8N2rSubeV0aOOHa399eopoNxHF5UsjH1BoApxSqAQzE80O7GXy0Vtx7+lXkt0ZdwlRfXI6U5LYuzeVIGI6luAaQXGWdqrSfPLg/7VT7tpKA8Ui6Rd3XLOqj+6BVqPQbwH90IgPVjkmmhGc9rlS2XH+6KiEc0TiKFPn6jJrVvtM1GGICaZdrfwxjk1Rjt2thvJbePXtTAT7Pq4Gf3ca92yOKS4haFQ0l6XcjJwOlNuna8VUubk7Ac4A602VIobfbGA3rkZNAGhaaJJcIJ5XTyO7A8n6VNItrabTaWiyqh+ZnPJPpUEK6vcQJbWcEiwqMEvlRVq38IynD3F4IyP4UNAFeXWJrnZ5kXlhDnKt/8AWqC4mudQXyojPNjoiLmuptdFsLUAhDM4/ikOa0F2xj93HHGvoigCgk5Oz0DUWi2tAtsD/G/JH4VpwaEY12S3jSJ3VeM1stNt6kCoXnUfxCgBIrS1t02xW6L6nqTU2847VSkvlTvVWS9ZvusR9KANJpwO4qpPqUcT7cgt6CqRLsOtReTukBbrQBda+ZxhQRVUmV2yWJqcoEUkdQKiiYEZPU0gHlT5ZNTRp8opoUmpgMUAU7xHGpWzbcgIRVqNRjNQXL5vYST2IxVhUcjptHvQMXgU3OTxSSXdhb7VmuV3nqq8mq66i7y4tLchP+er9PyoAtmJ29B9TVa5v7KzQmSdSy9VXk02WKe4bM8rOvpnFOhs4o87IBz3PJoASXVWnQR6dbvHKQCXlXAFIIbuT/j5ujJ7LwKtbGzjAFSLGPUUwKotVUDCKD608QMPSraxg/xL+dOYRwrvmkVEHViaAKfkkmni1B5NVJ/Eel25xG5uG9E6fnWJeeIr25QpCPIU+mCfzqgOof7NbJumkRF9WNZk/iGyibbbq05/2eBXMEmXmVmZvUkmlVVXoKAL91rWoXKsikxRnqE6/nVAR+pY+7HJqReaDxQSN+XFLxjNIcZoJHGDTsMdgbc0ygA4z60cAe9MLjsgUZU8Ekn0FIpzzil2qGLUAGPSlUEjHalcfIdtKJMqMkg470gGlDg4/wD1UqKVHLFvrSgnpS9qBBRS4pKAFpKXtSdaBh3pdwpMUYoAMg0fKMZpfakBDHigQvak+nWncYwOtIOByOaBhS9Dmk3Yxx3ozmmAtJSE4/OlHPSgBSeeKaRz+FLx6c0Ac0AKTnoMCkUbQSCaO9L1oAKTJFLjPGcUYwPegA+tBNFA6GgBKTHNLSHpwcUAL0JFFJk0Y5oEFOYdx0NN6AUZpgGcdBR1o+ooosAh60UCgCgAOcZP0o/GjJHFA5oASlpD1HtS0ABpR1pppe3XpQAp6cUlApRQAmSfT2ppGeozS9xR1NABz1ozSnABwKQc4oAXIx0oFHbNANACZo696CKTGetACHnKgjNJGSE+cYIJFLtQknbznGaQ8ZDYxQAEkEkEY9KVkjdQXAOeBTFRQSAOppfKG4EE8HOM8UgRIuAoVRgDtSlssowee/pSDGKNu3oSfrQArOqABjilzjt1oAzSk4NAwJwcUA800n5tx/GnAgqDng0CsLz2pD6UAjvSEYbPagYpNHbNHHal7UAJjNG0kUA4BPejJ9aBXEEnO3aSTTzjA45B602lzQFhCeB7Uveg0dqBChwoIKg5HekO/AKrnJwKQ89hTg2OnX1pAIwZDhhhhwRRmhgzfMecnk0pwPujApjEpM0UvGR70wGk5OaUUvY0gGelIBwoI+XjpSUGkAhJzk0ZpeaCKoBO1KOtHO32FIKQDu4PakpB6UuaAFB5FLnk8U3NGaAAnmlzRnIxSCgAJo980lAHzYPTGaAFJpKXtSAleaAFJBPXPvQCDxTT7UmCW44xQCJMBe9Jlew5oBBpMEGgB3GD6ijAIB9e1IvPHendBQAY4xjikGMlRml7UUgG/WlxjpRRg4pgNbKjil3kYyKXGRjNIVyKQATk4poTH8eR6GkIxUwX9365oAQDPHGPUUeVycUKoHHQU7PPFAEZUCj5iOBUjDcc0gbAxnmgBg3gYPNNKhj8wzUnWgjJxRYCa01G9smDQ3MgA/gJyp/Cuhs/GQZwt/b7c9ZIhwP+A1zAXFDIDUNDPS7a/tb6IPa3Ecvsp5qyFDcOMr3FeUpGY3DRko3qpwa17LxHqVkojkxcxf3ZDz+dFgOtu/Dun3rM0kI3H0GK5u+8JtA5a2d+Og9K1LHxLp1xKgkMlq/o33R+Nb0UizruilSVfVTmhAcNb6lrmikKfMeBf4ZPmFbVn40t3+W6geNh1IGa6EWyEcis698NWV18xj8t/wC8tIC3/wASrVoAzrBMD3ZRmsi68GWsnz2cphI6L1FZV14UvIWaWym8zb0Xdg/zpLbxFqukMEuoPMUdd4Of50ARXfhrUbZGIj85R2XrWST5bFJAyMOzDBrubDxfpl4+2Uvbt/t4x+da8ltp+qJukignX+91P51SYHl+VoUKOorsL3wPbyS77O5eAn+FhuFc7qHh/VtO3NJbtLEvV48EUXApNDF5W+KQZ7qx5qHNRNIofa+Vb0bin4zTAcCc5AJ+gpSynoGBHXIqMTzWsvmQvtb6daklvJbkKZQpYdwOaBCFwByQKQEYqKR1x8wJFCOuODxTAlJppPNN86I8Z57+1KOgI6UAKGFLkCkPApN2aLASDPpxS5qIHGcGlU5PXFADwNx96fvIGCaiY7WHelJBPFAEoOaCHPQ1WMTM2VlKn60YuE/5ag0wLHPpSZqHdcFc4XP1qRNx4JB9xQA8n8qOMcDFMyehGKcrYzxmgBxAAzkUgNIeaO1ADuvelwCME1H3p3XpQAuCvBoppJoBNADue/FN/GlLE9ab3oAeCM4pDnuKQdaWgBf4KQZo6fSgGgBaOlJ16UUAFAoXGcHpQcA8UALSUZJ7U3kmgkcDikzuzxikAxwBTv50DM8/aLZSYpxu9QKtWltqN4gZtQRM9sVCqszBQM5qd4ZY03AYUVkUacenQQofMEcsn94nFOUacgCTIhx/Coyaxkdn4EjD6UyadoX2mVz+NBVjZkvILdv3Fmo9GYVnTkTPvYfMagZyRw7N9ahMrDqxFArF9Y8wEl1x2XvVQG1MmyTG/wB6hF1Jv2oOfUippFh2oSpMh5OKBl63uI7aNwI9zMMAYqAW8ssnAxmo1IDAmr11qT3aiGNBCmMHHX86YDBo9/jIkiVfUmnw2kVpIHuNQtlx1Gaz5khziaSQJ3+Y0ljZaPdXaxvJLg9zkCpJNO6+zX7B7e4DBBg7e9NtYGtmE02xU7lqu/8ACP2UCE2cpLn0NZYtUhdhLLvfvzxTAbcXCyXO1eYzSFIgymKIs/bFPi0pWAlaZ8/pU8NrcHd5MLOR3HagZD5MoHzLEp9OtGSsewHBNOkWaB9s1tLn2GRUtrBDeyhGjlXHJPYCgY1dO2ReZNc4z0VV5NKAsqgLEcDue9bQsLVHVSXKnqSOlJNYadu2+a350Bc5maVhlYvkbON3Wq8tpIU8yRpnI/iz0rrIrCwV9oy2fU1ej0jqBGNh/hzxQBxVjYLcPzduPRVOc1dOnXiE+Su5f4SetdcmixJyY9n+6KkFoUIKgDFBJykdvq/l7BH175pw0a/kUNLNEH7g11DIzHlRUf2GPzd7TY9s0wML+x9qAzshPopzitOLYqKIYUBAxnFW7iCMwnazsewFU7S0u/LxvVF9H60gLMQIYEjFRTakkeRGjSMP7vT86kfT0mwJXYj0FTw2cFsoEcYz/ePJoAx4r7VbmUIIVHuRUc+i6rezFpWVUbox7V0ZcgUhk460xmXaeGbG3YPKRIw6kcZrRjs7GF90dtHu9WGTUUlyqd6iF4G6UCNDeT0NRtIAeazPtkrSlc4FRO77z8xoFc03uo04ByfSq73khGAuBVOMZbJqXGaQxryyPnLGlVDswSTQwxTlPyj1pgRSKAMk00DPQHHvUk2BGSw4FNjZDGTnoKBiopxTdp81T2FAmUUNIVwcAZ9akRNIRsOPSoIgCQpqpNrNkj7PMMj+ka5H50yeG/vJV+zOsS9mxzQUbIUr1IA9zUM95FAuS6t7Kc1nxeHyJVe8vJrgj+HdgVdi022jcFYgKYjOuGuNRaNraJk2n7wOM1LHo146ZuNQceq5rbC8UbaAMmPRFjbInU+4TJ/OrcdhsXb58p+pFXVXmn4oAp/2du/5eJvzpRpsQYFpJmx/t0lzrNhZDEk4d+ypyaxrnxZJIo+x2rD1ZyKYHQrawr6/8CeqN1qulWiNvlLuP4EySa5O5vr6+l33MjbeyrwBUQjX+7+dUSat5rouVAhtQi+rE5/KsmRDcLtld2X0LcU/aB1p44oAiRFRQqjAFPBIIBB/AUrYAyelIHPbP41QXHZOPuEH1NI2ARjk4oG44oxnnHIoC4H260rEhelNVWYndkelPC9yc0AR/hz2pzA8Y6+nrT8DGKXAznFICMcNjv6U4pnk49gKdijORTHcAABS0bSRmlApCEDDnPWkIB5oAHp3pRTGHNLkYpaTvSELjOKCccUZP4UdqAEBzwKMEHBpOnTrSls9KYwPXFFJ1OaBQAtKFGCwABx0pMfnQOTz0oEBpQc0vHWmnrQNDuDScZyDmjoTS4FIQYOOAMZpOhpWPHFIOlMYmaUZzntQORRQIUjjNFJkdKcMcZoAaTR2oJ5ooAO1FJmjnOO9AwozxRjmlxx7UAIeRR3o7GigBKKQnmjNMVhaQ+1BoxxQIM84pc4pRTTxQxi54x2paQjHWgUAB9KKKKAEIGMkcUoozgUgPPSgQAkngdKR2ZQMLnJp5yPrTSx3BcEk96BhzSKMZ+uadyRSAc0AHIOfejPWkJ5H60vegBdxIC9h0qN0EnysDjNOJwR9cU8jBx1pAMH3eetHelGOMdqDxTYATxUUeWZyxzzx7VJkGo0GHfnJJzQA8nAORgg4oJC5PUCkDqwPsacSm3B7igAV1ZsBgTT6hUIvzbQD60M/lIWRsjvz2pAT5puSaYkgkxtPBFOWNllz5xZf7vamMd9elL9Kjy+4ggbR0NOqQF4o6mgsAce2aAcUwFPApOaDz34zSnI+tAgxSDrRnNHegYtGaOtJ1NBI7p1HNCtjPGaCeMelNANIAHPegjhSDz3FGBS57YpjDnAzSg8UmeMUlADuwNIOtIBzmlz2pCAmgHFIetB61QxelJnJo7H1pI3Vs7o2Vh60AP3dPakySaX5cUmOaAEBIJ4pT7UGgj5cikAlKOaSigBaTNHelBxQAZpQeKTjPWkPWgBSaOaSgGgBc0FgBknFIe5peGG1hkGgBN2KQPgYApTgNwKKAAY5OOTTsHGaQc0uODQAA7T0pS1NzkYpwIKjFIBQR070vYDvTMHOaXmgBOefal7ZpfrSfTkUwEOMcGlDAYB70e56UpAJyBxQApz0FN+YcA0vNHekADk80p60wqT7UozgAnOKAHqSOwpo+Y8jBoBFGeaAEOc0oNGRSsQozjigBvPalye9WIY4J8hbhFYHoaZNbtDKELqwI4ZTxQMjBxRnJoC4wTSjFAg+UnO0fjToLm5tJlmtZWikXoV6U3b6UinNAHTWPjadcpqFv5iH+OMYIrdtNZsLzAguhvPRJPlP6152y8GowmTUtDPViMffUn6U2WKKVNm0EHqDXn1nreq2ChI5y6d1kJIrds/GFtKwjvbcwN/fTJWlYCxd+E7GbJRWhJ7jpWK+h6rpsvmWM7OB/wA8mwa7aGYTpvt5klj9VOacwjPMibW7lTQByVt4x1SyIj1G3Z8fxMCprfs/FemXrhfO8pz2ccfnVmWxtrpNksazIex4P51gXvgu12ZtJTA/91+n50wN6+06O9gZhZ2s7HoSMH865n/hC7l2Iidbc/3ZDuH51SltNd0N/wB3JIydnj+YVbsvG95EqpexJcD+8oCn/wCvTAybzQNRs5cXFu7R/wDPRFJBqnd2EloqPlXicZDjivRrTxJpd/wk4Rx1STip7zS7C/i8ue2BHqnH60AeVleKNqeldldeB0PNndFB/ckBP61z15oOqWDnz7Rig6vGdwpiM4RoDnaKXIUYxjFKZVIxwD6Uvlhu+aAGbweDS8UoRaXYO1AhqgDtwKXjPAp23BoIHagBhoBFOOMU3aDQMM4Oe1JkHpTudu0nIpmzac0APDEGl3DPHFNo/nQBISDSEjtTAecUoY5pgPHOaM5GOM03IoHNAC0oNNPHJpQcchjQAu4e/wCNHB6UFs+9HvQAYNLk9KMjvRkUAKOtBNICRSjmgBOKSlOKTtQAUpAx96k70EAjBoAWlpgjAztYg+mOKPnHBxQA7digEEcdaNvfP4UY5oEBY+goUnFBIz05pQaAE37ULRwvuFQC4upZtsiPtPbFPIvi5dF/WnXGoTi38uT7/fmsih/2XzOjhPrVZbKGSUvMwbacACsuS4Jb5pCPpVuK8tY4wu/FSO5pLHEF4TCj3qDbI3MaBh71gzyXjXZe2Dunp1rc0uXU5IgptGUDqxTigZcltY4LRfkLTMM8dqzgHjHzod3oK37fTZbkmSeQJ6kDJ/nUd0ILJwsMxeX3ppiMYx3G0O0flqf7x5qPBDgk5rtUtLeSFDLHufGTzVPULCWUCO1iREHfPWmBziSuh+WDfWlBZXF1EJTaYT121o6Z5GlIwvJbYuxyNxqxL4itS/kwgMT/AM8+lILlT7BdSwkeWyg+tPh0iFIwXGZO9XUvH2h3icL9KbPJd3ERS2tpAp/jI5oFcrrp9y7CNZYoof7gGTVuOxljXED8dzjrUdvG2nxCSfeW755pLnXIoFBTePbbTGWFSfOHfmmta7J93mOf8Kij1Ca4UPCqnPtTZEv7jOwqrnpk8CmBOzo7eQAS57AVOEtI4drxKfc9qpx6ZfLKskt2jqP4V4zVz7KH+/SJKDeQz77ceWwP3jVhL+aLGZmI9qtLawoOEFPCKvRV/KgLjPt1zNFuR3z24pI5bt12yEge461LvI6mjzB3YUxJiBAFxQIoxzt5pj3MaHG6qst+AcKCKZVjQ+6MimhlPUisv7bI+cZpn2mQnkn8qQrGq88SDJYVCbxe3NZ+ATk07y+OKAJXu5Wz2qISO3UnNIVNCrxQNCOCRTU+XnGacwOeKcvApACgbycdaQ/epM8k00uAfmIFMLDwcU4NVZJlaTaM/WmzTxopJkANIC0zA96ckoHQZrJN8hbZEHkb2qaGHUJMHYIoz1yeaALGpz/8S+VRjcexOK56G/mMxiG7H+yMg10I0qBpQ85d/UZ61eiigt0CwQquPagtM59bXVbsbUTyE/vsefyq1FoEQbN5eTTN/dzgVtFnY80eXnmixLK8NvZ24AtrdFx3281ONxPSpAntTtoH3jgepNOwXGbTTxkDGKqPq2nROVe8QEdqy7rxXCPltITKf7x4qrBc6DB6YqvNfW1sD50yKw7VyFxqer3nD3TJH/djGBVUQA8uzM/ruxSaJOgufFMYJjtUZz9Mf0rFutQv77ieZgv91eBUYTHSl2mnYBiRrGQQQPc9aeiqOgpCtPUYFUAfKAeOtJ1peO9BIHagBuCe1HLdxxTj60uQBg0ANNG0ZzmlxQRj5cfjQNigcUHpSgGkHJpCEzyR6UvTkmlpM9jzTGKT6UHPYZpMY+lLjigQmT0IpQfzpR1PvR3pALg03PNOyQRRgE0AN6U7IzQOWpQQrEADr3pgGcUZz2o4zSc5pAB4FAPA+XHtSc5xThnHLZoAbmkwM5p+Kb3pjACl96UrikZsDk0ALwOT1oyeePc0mPm5paBCA55HNL36UmcHFLnmgEHOeDilHT3pOppOh4oAcRikoyduCc4o7DFA0FLuwucZPpScY7k0nSgBATxkUvPel3HPFJk5yaBXAcgGjnNAPNIxxQAfSigUDrQAvU0UGkGTQADrRR60cY9+9ACHqPailppoAU5YgDjFGaMUCgAoPTFBwOaOe9MAAoo7UtAARRk0gozQIQlvX9KUHBpO9LgigB2786byVK+veijHtQMRV2DGSfc0uaKTdg0AB5OTSgUClJwMUAIRQOlNyTjJzjpSg889KAFPtQTkYpFOVz0pMFjncQKQCgYprRo+4HPAzxTjjsc1HszITkikAqxpHnAJJHPNAVXjKYI7UoHyHk8HvQp4pgKMrEVYgr3JFQCaFiUKipsiRSCD171H9nhJBKHjtmmBKdkSgAYHSnk9MHtTSuVx0pFTb0zikMeCcYOPwoJ4xQO9IetADhgigcYNHbj0pqsCcHr6UIBcgHBpwORimt0wAPrQKCRTQT7UpFNP0zQNjh655pBRRnmqEKCM80vG3Ipgxnml6VIxVbac0pIYqQuPWkAB6mgg9AaAGksHx/DmndDTFMnR8celLmgBwZlyo6HrQCByaZhlfO7KkdCOlODYYH0oAUc9u9ITRk5GKKAE70pYnrR1JpAeaYBjNO6UmaOe1AB9RQaM0UgCjpQelA5FACGjtRS8ZJHftQAEZFHbBpKP4uaAFoxzmjOBnvRkYoAd8oXg03HzdKQ4xg0ooAQYHQYFO60zGAadzt4oACaM5oxxQKAE707vSEUYyuDQA4HHI70uc8VHSc5FICUABeuaRSQ+F6ntTQc89Kd0INMAGSTuGKdn5cCkJzkmgdM0gDPFKpHpTDQtMES8E0wdSCOe3NLxijAHOOaAG9DmnggkmkGR15FNBOelIBxHNA9KUHJ5700ZHUfnQA1ogTuX5W9RSK0qD5h5n0qUnFIKYxFkEg3DP0IpQeMUHHOKAQeDUiFycnmmZIIp3c0hUbck4pgAJOc0vajB25FIAwPJ4oAfznJprIhPvQTQOuaAFikmt23QSvE3qp610Nr4vuUQR3sQuF7uOGrns000mgPQLHWNM1FQ0Fx5bf3JTtP61pEyf7EiV5cEVhzViy1W/wBNk/0e5fb3V+RUWEelhofZTVK60i0ulHm2kL+jqcGsWy8YW9wwTUIPJfvIg+U10lu8N3EJbKdJI/VapFI5S88GBnL2c4wf+Wb8Y/HNZ7XHiDQ8R/vvKXpuJYGu+YMp/epx6qKRtjYUSAj+5J0pAcvZeOI2UC7gKn1U5/pXR2msWl/blrS5R89VP9RWbfeHtPvGLtZ+U5/ih6Vz1x4PvYH82zkEy/3d+1hTTA6ttKttRcreWNuVP8cYwR+VYd74JdOdPuDJn/llJwR+NZFrrup6RMEkR9v92ROv411Fp400+bictC3oxGKdwOPvdMv9Ox9otZAp6MoyKpJMh7163bXUN3FvhkWRD1AOR+NUb7w9peoriS2VG/vRgA0XFY81MinoaBzXT3XgO4jUyWVysi9lkIBrn7myu9Pl8u7t3j/2sZH5imBAwGMYpuMGl3Kfutk+9My+/jB+tAhTnHFJyRmnDPINAFAxh6ZFKD7ZoxTdvzZBIoAXNKSQAcE/SkzQCRxnigB1JuxSGg9KAFHJ5o6GkzS96AFzgZNKr+lAyORgj0pv4UwHluabn0py4z83ShgM/L0pCE3UoJpuKTJ7GmBIT60ox2qMZPWnA4OKBjuBwaCufYetIeRnIpCxxgUAKVKng5pdwx8x5puTj3ox370AB60oNJSCgVx5NFJSMOKBiObln3QzoqGqk9vcGQ7h5hPUrWzY2NkYQJnf/dUVet7m3tNyW9qSR1Y1kM56HQnvIgsdiwz1d+lPfQI7J/JYBnHp0Na99rUzAJCdgHXbStHLdWke9XDN1OKVhkMMy26BWkgiP+yvNTTXfnKFgnBz3PGahl0iOJCE+aZupYdKltdMRYf3jOJPXNVYB08F20B/exj6U23sLhYgcRn6mnGyVD8vmN9DVy3njjjEciPuHtSsBC1+tpHhxvf+6prJvNQNxKWaZ4x2Ve1bf9nQ3NyZJEyvpVoaXaiXesSD8M0WJOWtrC3unMwNxMw65yK3LCxS1USLAsePug8mtJrUP8u7avooxTo7dIwEGSo6ZNFgIFSR3Lkn2Hapi908ZjHU9G9KlCAdOlOVQKoEZY0yfzd8s+72xVpdLsj800Rlc9S1WWOOtRNdRJ1agZIIYU/1Ue2nHcOtUH1KMHjdUbajIxytFyTS3EU1plU4LAVktczP1qM7j94n86QzWa7j7MDULXvJANUQMkAVMIx1JJNIBXuZC3Wo2d2OdxpCnzZzTgKAQgQg5pSvOcVJjPGcUbccZoGRkHGOlNUODjAx61K3TAIpAwIPPSgBAjZ56VMcCq3nL/eFRtcSO2yJC5747U7gWS600yIO9V/s14fmIC57Eiq5tHL4lu4VY9F3c0gLck6DoaryXB/hNQtbXKviCPe3Yk1PHo11P81zMsef4V/xquUCv9okJwrZJ7DqagmuLndsis2kl7bq37XSILUhsl3/ALxq4FCdKGwObj0vV7pg0xjt1Po3NX4tAiRt8kssrf7TcVr7jRupWAqJZpFwgCe4FWBGO5JqULu7VWub6ztEJmnRT6Z5oAk8taXywO1YM3iy1DYt7d5T2J4rNuPEGqzqyowiVvQDIoA7F3iiTdJIiKO7Nisq68SWMHEO64b/AKZjiuVVJJeZp2kb1ZhUixBehH51RJqS+Jrt02wW4j+ozWZNdahePuuJ2I9AMClxjuPzozVDIRboO1PCgUEEnrRt9zSAeCPSkJAJpccUm3PBoATPpS5+XqM0dOOKUDPWmIT3FGcDmlIJPoKb/Fjt60DGhsjIRvx4pxzSbyCAQSD6U/AzxQCEJPehj8v3enelYANhT+dHOduc0AxAR6UUpGKMigA3YHNFJj5qXFAw5Dc9KXHekI4pR0x2oAX2o2+2aTBzyMUuT0oAUISTgdBk0xjg89PWnB8AnOeaMZGDyOtAhAQRkc0oOAcUmB2GBSgAcUDFzntimkHtQD6UuSTzQFg/DFL25pO9KaBXDinfd7dab0GaMksCelADs4FLuyMfjTcHPSkJHSkApdjxRgHqM0lAHegAo57daOjZ60Eg9Bg0wE69evelHUfWk6GlxQCFP3iaKKKBiA8/hR15oOQKO+KBADg0ZJOaMDFJ0FAC96TNGecUpHFAXG9aU4xSDNLQFhKO/WjrRigLgD+dA44/nQR0NJnAzQMWjtQORmjIyfp0oEwK89aMZak3Z7YoBIYGgBWznFFBPX60L1yelMAPI9cUhJ60EgdKU0CEB65FOHNJ1pOlAC4ooPc01Tk0AL/HSqPmPHWmj7xGfelJwPegBaTJzjHbNLk44pMgDODQMU+o6U0n5l6UpJzjtSYbPAzz+NAC55wO1Jz6UYO0464pFBAJz1oAcAT9KDxSJuC4PagkdDnjvQAihig3H5snNJGTsOeMk5p+c0MeM0gEAAGQBg+lNAfzAW4UfrTl6VFNlsbH2kUASuuflyRVYSOspVVBVh0ParIO4A96ryRv5vmjqBigCZScZIwfSkjWTzCzMMdgKI5N6qWByR3FScCmApPNCgLnGeT3o+XBLdAKheWRWCAAZPHFAExUk43DPrRs3D5uc0DJHXHFL1HXFIYINuQT9KCAee/TNHWjcqkhiPxoAXHFGTjIH0pD+H4UHigBc8U3ABznFKOlBUFaAFzUTsDlScVIDg01grE4HWmKwwPt461IDuwD0z1puwZ6UoBzx0pBYkJjP+rzgHHNJmmjCnp3zS980ALmmgEdaflR97OM9qbjmgBiq4diWyp6D0qQAbeT+FLnHApp60wFoPSk6UZoAXHQ0cUhJ4AoNIANPX7gP4U1jwBnihaYCAc4o5pDyaUHK/hzSACcigEjkUdqXtQAAUHFHWjB70AITxmgg8H1ooI6HNACYJIFKR6Ud8UGgAPK4NA9aKOgyaAD60YozRkZoAcSS+G6UYFD7S2VHFJ2oAM0c/WgYHUUEBjwMewoGBHSgA4yBSd8UHOeKAQu7ijPGOaTFKDxQIByad0pnOaXNAC55oB5o42+9FAIOQSDS5pCeaAMgn0oAUH5jTsAj5evemqE/i496CwB4PNIQEFWpSxIGaTdk0HrQMUmgdOetAPHTj1pKBoVqBQBk4oxg4oEHfNKDkDNHA60nPagBVHzcetGeaXLjgjtwaac9T3oADRyTgDJpO1BHGelIA57jB9KCSKccEZ7+tN780wAEgdcUHrmlooEJwetKjywvvgleNvVDigU7igDXs/FGo2wxK4nX0cc/nXQWPiLTb/asziGc9Qw4/OuHamipaGeqBJlGYpN6+oORTNygnzYiH/vKcV5xYapqGlt/odyyJ3QjIretPGgLBdQtQc9ZUOMfhRYR0zwJcKVkiiuIj2brWFf+ErG6fdAz20h7HkVtW13YX6h7K6VmP4EVORMpw6hx70DucFLoWvaK/2izYnHVoWGfyp9v451S3YpcxLKw67lKtXcq8R+6zxOPaq9zp1pfHN5ZJIP+egGCakVyvpviyxvwFbMEh7SMMfnW0WWVCroGXuCMg1xl14NibJsbrbjpHKCf1rPK+I9EjEY8xY+2BvFUmB2F74a0m+UgW/kuerIf6Vzl74JvYD/AKFcJOv90nB/Wn6b40cbY7+Ld6yqQP0rpbfWNOul3QXcTH+6x2mncDzm7tbqxl2XcDRe7dKhDq3Rga9ZYxyJskjRx/tLmsa68J6TdsXCSwue8b4H5YppgcBtJORQQ3tW9f8Ag3UbRS1rKLhB2AOawZIrm1bbcwSx+7LgUDGlcU0g46VIpVhkNmkzg/KcGgBn0pDUpGRk9ajINMkZzS7qeAO9IVFBQgzmkJxS4ppHagBVkb1BFO3Z6VHgjpS5weKCSUOwoB55AphbLUoxSKH7uflHHpSHbgbc571GMg0v86CR2Peik6nIoyc0ygJNKpJpM0ZoAftb0yKToeabub1o69aAH5wc44pC2TSAkU3dhufyoFYvWekajL+9muzHCOpAwalv7eGziAWd5N3949K3TbSOcO5x7Uo0y3bmRS7epNZFHOaakMZV5JA7jqBXRi6BGED/APfNTJawR/diUfhUm0L93imSUFSYyB5FBPvVhV3dUH5VMRUZcL1pjF8te6gUvlx91B+tRNcoozmozep2NANlrgdqUMKoG9DdKY1y5GBxSuK5otIq9TUTXMank1nkk9TTTRcC81+o+6CaryXkjjC8VDtJpQmKLiQxpJCMsxqLbk7j39atMgx0piqM4AoGRiPJqZI8DGKco2nIpvmHfjNAyQR800/extzR5maaW75oAkTjqKdz2qESLsJLioGvACVU8+lAFtk3CmBQuKotczE5BxUDzOT80rZ9FFIZrlwOpqpNeANtXmqFxLJaxeYyv1A2seTVKW8vpoisNoUz0fPNAGs80rqQpqBmnX77Vnw2mqyJ5Mh2Ke56n8atxaG7nEzsR6BqAGm4jVTkiR/4UB70kN5r8jBI4ti/7Kitq0022tRlIAGHcnNX4+OgA+lAGJH4flucyX1xMzt1Ga07bRrC2x5cB3D+JmyauZNOUFqAE2BfujFJuPpT9h9cfWqF5rWnWGA8wlkP8KcmqTAvKxNPCMetczP4sdkxa2wXP8T1jz6hf3ZPn3LbT2XgUMk7C51ewtOJJxu/uryay7jxTFylpDJIezMMVzgiT0qSMbelUhpk8+qapdZEkhVT2UYFVTAHwX3lvUmrINNLYpiuIERHUgcAUvHYUw7s5XHPXNOzgdM0WAOO1K3KgYzSdadj1oAQBgBmk5zzS5APFOoGNpQcDikNAHFADhzSg8mmjdj5Rk0itxz170AKeaT5s9sUhOeOfrSjoB2oAdk0vSkppcnI2n60hXHLzRSAUvSmMSlxg0ikY54NKeR1oAU8j3pAuRSquBjrS4wKAYnQUvLc96QY70hPp0oGOPTtTaaSSOMZpVyVBIxmgBxxjGeaQfWjFLQAU5BuOMgY65prccY5o7fWgLi9s0mSaP5Uuc9qAEVdop2cKRjk0ZJIHajg80AJg96XtS8E5x+FJ3OKBBxgg96B92gjoaM8UDF5GCc4pKD7kmkoAWg8imsaFGBxQK4vNA5xQd2B6HtRQA7vkjNHI4PfmmjPegYBznpQCF5z8pwaXNJ3NBx6UAHJFJ2pePWkoACaUYxSYGDkfSkAx0oACPzpRkDBOcd6ByKKBhmkoxQMZ5oEHSjqaTnvS0AGaAeeR9RSUADPvQA4n2A9hSU080uMimAcE8iig0daAAmlVuemaTqP5UE4ORQAdRig0D/9VBwetAhDTh0GeaQKNxJJ56Ck3ccDJ9KYADnIpB1o5PHQ0uMHBpDF7Eim04EAEUkaHaxyOtAC8UhOBR1FIVDABhkDpQAHkg+lJ85b5R+NKCQTnGO1LkigBW+7gHk0Hnp0pueaeo70AC8UjdM0Z5xjg0gByTnOaAFA7e1IV4xk9Ka2Sw9jQBgnk80AAK5IyT9BTbjCxZBCnPU05cgcnJ9aVgGXkA/WkAKGSNVYgkDkig5PSjhQSewoSRJFyhyM0wHdQCetGM1A1yol8va2fXtUqh/YigABAG3k+oIqQbGKl1yoPOKT6mms3yttwcUgYp4pRUcMjuCXA9ql/hzQMM1G8SSHBzk9807ILYFATDZyaABV2ALknFBIB706mCZHLBCCVPNADwKWoXLZDMnyeoNSKMqPegCSJYmf94xUY6ioWHzZFP2AUADBzQA0ZzzUh27B8vzD+L1pACOSfpSE8UCFKts3Y49aQZFRuDIvlh2A6kVIOB9KAHc4qNXYyFSvHY07NBcAgCgBRSHrU9u9uN/nxs2fu4OMVCwoAQ9KOgpT1pX24AHNMBgpeO/4UvoKSgAxxnFHtR/BQPWgA6HFJzQ2Q2fWgUAAp2eKao3HinDHekAgP5UE5NKBtc9qCBQA2jr0NFGKAFzS8ZHemc5xS5APvQAp+8fSg8igcrk0fSgBM07AxkUykLqvVgpPQGmBJx2/GjtTc/LkN+FHOKAHUbT1pAaNx6UhCH71KKMUnTtQA8jmkzg47UhNAGaQwXdtyRQfbrQGZeAxx3BoK7lPtTAUHtSdOM0mTk5FG7cM9D6UAOOcZoI700Z79KCfQ0AOGSaUhc4NNRg2fUU4hT1JB9aAAEBcUcHmmtnPFKMDGT0pAOAppBz8pH1pWPOVOKTOBkjrTBMVQD83IpMHkZzS9aKQIU84pR6UDGOaCcOuOlAXDnpTgQVpjHD+1Lt7560ABGDzSkjacmkOfXik5GCBQAiuAxDBsEcHtS9TRuPpikLUAOKNjNIDzTSxzkmnDkZFACjBYDNBzjiggfjSYNACg+tKPUU2gHBxigAJIpQytwRzQf1poHzUBYcCyOGjYow7jrWrZ+J9Ys0CNKJ0HQSDn86yqOaAO5sPE2m3zbLl/s8/oxyD+NbYhcqGjIZT0IORXlDLmrdhqd9pjD7LcFU/uNyD+dS0Kx6Yc9ZYwT6jihWjXiOU+yN0rk7fxw/C39vuTu8Yxj8K6KzvNN1Nd9pOj/7Pf8qVguNvNF069+a6swr/AN6M7TXN3/gyVVMljcK4/uP8p/nXWlXXo5P1pMqeSGD/AN4GmK5wa6nr/h1xHNHIYl/hmBZfzrZ0/wAc20yBbmN4X9Q25T/hXTsjOhV1jnjbqrc5rFvfC2kXW75HtHP9wcCpGbVpqEN1EJLe4jkHfa3NSylLhNk0aSIeocV57d+DtWsP31rIsqjo0Zwf50yDxTq+m4iu42l/67DB/OncZ1V94N0y8bfD5lsx/uHIrBvvA+p2q77d0uVH+1g1oWHjWyuXC3SNan13blro4rh54RLaXCSoehU8VSYjyuQTQNsuI3jb0YYoDjbXqDyPKCt9ZJLGepKg1g3nhHTLslrWd7Vz2YjFK4HFkqTwaXFa2oeEdWsF3rEJ09Y25/Ksgu0TbHUqw7MOaaYDu2aZjmpN6kc9aaOTimAmzcMrSYPIA+lPBKcdqQnJpiIsnIUrj3p3SnUm3JpDEJz3oOacEFIePemMQHBpc85xRwelIcgYoAlDAj0pjBgeRxTQTS7iCKAD+dJz3px55702gBaAce9GRjmjjHvQJnctKq9WAqJruNf4xWFLIxOGY4pFyTjJNQBsNqManHX6VGdSPZaoFMYp4WkBI97MSTnFVHuJnkxvOKfKcHFRqOQaYFhQT3NPA4xREOMmpDgUhjdvHSl207pRkEcCgBMUoAzSZxShlBznNADgKQjBoadEGc1Ve7yeDmgRY3dqMqOtZ6vNIxwCPrUbwS7sySYX60FF97mNTjcBVY3kYYkmqUklvEMb9xpouEK7YrZmPuM0AW3uml+4eKafNZPmlCL71HGLhgTJthTH0NJEse4s9vLP6EZAzQKxNBZvcv8AuFeRe7DoKZObi0lKCAFv7xNaVpd6jJwLdYwfujbgAVba0muvnu5V3+irQBzgubs/64oqfSon1K6jl2W0akno2zJNdAuhWKOGLTNjsX4q7DFbQJsiiRR9KBnKiz1i/cM8ezPVn6Cr9poUsDlprjznPb0reLbhxTdrHpRygVfsz+1SxQsvWlllaLGIndj0VR1qnfajqllbG4XSpFjH8cnAosBpGOonu7O1TdcXUKf8CzXE3Wr6lqLkyXDIp/hTgVVW1AIZ2LfWq5CTrZfFFijFYhJKR6Lis248T6hK2beKONPpmsrbg5FPXgUcokwlu9RuSfOu32/3QaiWEL2/GpCM0AHtVDQoQBc4oNPGelIRSFYBgjA60gJzjaRSgcBhS89aY0gFAzknFGcnGDSgc0DDaaUICM5pN4DYLfhSnHXqKBCNzwAKXNNJGeOfalzzQMKA1JnNKFoAGOSOaQnApQB6UZFADcEkHcRjsKf1NBxjikFACnAOBSY7mg80YwCeTQAc9qUU4qdm7IAzSUANG7dywxTlUuTjkCggYowMcE5J6UAGB3oY4PSlbIOBSUBccCeoprMSNwyTQScYyMd6auen6UBYASfajAOc8Cl5BxinY4oHcTAHAFFLijFACE5GBS0Ypf50AJgk57U4DnNIr7c8ZpoJx0xQIcRjr0pVAPemnexy2PwpSdo5H5UAKRz1ptLgggEYzzQcZoAXFHekwc470ZoAWjtTScClB4zQMM9BQARnNBHQgZxSg/LnnnsaBCAhqXHpSKBzge9Lng0AKSePWm4oJOeDSkc9c0AGabnnFO43YpMjOO9AAowO/wCNA+6KMcZJooAPpS9elJxnHakII/GgBxBK5684pD8v4UgYkYIxg0UwDJxx1oGc80mefenUCGnI6UoyF560ZHQ9aTNIYo9aQ8mjmjgdaACjNIGJxxRQAtJzyKKUUwDIwc0Dk4FIQCRkZFBxnigB2aQ0gGOKWgAP9KjSPH3myalx6HmmZwM8mgB3G4Z4zSKef6UnX0z70ucHI78CgBSctkUH7xY8ZpApABI4paAEPII7UABcKBxQOTx+NLnHGKAE24OcmhuMe/SlJ4pjgcYOCeOaAFAwtAyBz1o/hx+NA5xk5BoACAcUm1c7icGlwQ3tSggnBoAcMZwRmm/xEA9KUYDZJwKOQ3p70AKOM55pjj5eCRmjcQwzzzTPLO4lSTnpQBIDjAPXsfWmnAOc0R528+tK7Kqkngd6AEA3oc96FVYEdieM5pytlQykFWHag8+mKAIjJHIVCyDLdqeAYxwaYsaI2FVVzT0bfK8eCFXg5oAUvvyF5x1pc/TNNFvGkokXOQeeetLjmkDHKPlx3o3ZGAaBSMo2kKcHsaAYhBzkHFSdaQEq6tkZHX3qQsHJJAB9qQxmccUzy1Ds4UbjTx1NIW20wGu4VTn06etLGcxg9M0EZ5p2ByeMe1ACg5pDyfagUh60gFJ+brxQcHpTR7jtR1XGKYhwCg7v4qQ9aCTszjmjntQAqthgcZx2odlyzuMd+O1NYkY4oxkcjNACqQRkEEU5D83QfjUUaLEMKuM1LHjPPFAA+FOOc9803G3jueacHDAj0NNbcWUgZ29fpQApyR83SlBGM9aQngnt1/CmscLlVzxTAcTxjtR/KmB9wzSg8UAOP6dqbnmlPSkxnNAC7vmB6UE80h9DSdhigB3YeppATk9aYAVJ+YkUucYzSAkpe9MA5zmlb5SO+fSkAUcE80hPGaRfmUMO9MBx9jxRkAc0Z3HgY7UHnimAgPcdDTWQPjcM4p3sKQ5xxSAANvA4p3SkB46UZ55pgKDx70pPSmA8nnignPSkIk4KjHUUHoQe9NHWnM2ADyTTAQgYxQM9qCc80gyDSGhTjOeSO9IAGYFCcYoODkZ5oA4piG8nIoVSFKk554NPC45NIzAHOOD+lIYL93B5OMU3AD8dfSnkoMEtgg8cUH+9QAdO3NBOBk0A5pRycZX8TQAmQacyIVySfwobg7SORSGgBCOMYBp6q2OCB9aaQR1pN3rQICSH2kfjTgCT1pu/5sH86XvnNAwB5p5OOgpmVzg8UH71IB4ywPtRhdvXmm55yDShtvUZzQAppAcnA60MRnIGBSAjqKADJyciilzTc4oAMDvSjjocU4MM4YUfLng0AANNctt+Wg8DgihW9aAGgllznNKHbvzTuKCoHK5/GgQvzMA2BtPQ0FR1HFIBtGBwCc4ppNA2xQcml5PSmA80tArj1GTQVweaZyRwaduOOTQADHbBo27SHjZkcd14oyQcigHJoA17LxTqlogjaVZo/wC7IMn866Sw8X6bfNsnzaP6SEEfnXC7QKjK5NIVj1XZHKoeCQOvqjcUZbo2CPQivLoJ7q0bda3UsXsp4ro7HxpcxKI9Rg85P+eigAilYZ1nyJzFLJG/oRSzxx3kJS7tknQ9TtqtY6tpOpr+4mCyf3JCFNXWR0H7o4pAc1deCtPuGY2M7W7H+EjcP1rFm0DXdEy1uszqOrwtx+QruxkHMqH/AHgKeC44ilJHZW4poDidP8aXtuBHdxLIg/76/lW9aeI9Iv3OXaGT+7IMD86u3WnadeKwu7FRIerAc1z174G835tOukb/AGJMA0AdjHc46MCvZgQQaiuLSzv8/aII5PU9683Zdd8PTDcZ4seg3LWtYeNjG3+k2hb/AG4sA0IDSv8AwPaybnsJzbsf4GyRXM3XhrWbFS7W7SoP4o1yK7ux1qy1FQ0Nym7+452sPwrSWbZ/FiquM8g84dGUg9ww5p6lGGcCvULvStM1NT9ptI3c/wAYGDXPXXgUdbG6Cj+5Ln9DTEcdyDTgxxVy/wBE1LT3xNAxX+8vIrPV8UgJeTTcGn7lI4pCGUcigBOKaynPHTrTgTnNISc0xjBwM0FieaGNLsIUN2NArDcnNLSZoGKBAfmFKBxjOacH7FRimFSrZU8UDNd1UOWIqygDYIx0phVScGp0wFHIAqLgNPDYpHO04pDNGznBzio5G3NkUrlDWXJp5HIPoKh8wg4bg1HIzu+AaLgXDOqrnPSlS4DDJrPVJOd1SJhTlnwKBF43QFNN2WGFxn6VTe5t1z1P41CdQUf6uLIoAu7pj3zSKjNy8mPqaqL/AGlcNtjgZffGKtR+H72Vd1xJtz6GgBxmtRw06/nUMt3bq+23Pmt7dquQeHrON98qmVv9o8Vfis7WBspCigdqAMQf2hOwWJDGPWpk0m5dczzfN6CtsyJ2FQtfWqfflQN/dByaBlWDS4YiNy5x61dEcS8BRSrIkyhkzg9M0MUXqyfnRzDsKYUPO0VKDgfw/jWbOuqXORZgbPUdazprS9tx5s24yr6nNTzlWOjyxHygGoJJSn3niT6tWLDBq95gJHLk9MHArVj8OXMMDTSsksn+03Sp5g5SzaS25Q+bOjy9gh4p01/Z227zETcOwGayX0zUriYCBYoF/X86tWfh2WScC6uJJc91Py/jVEcon/CROTi2ti47AJV+C+1CbBuLWO2hAJdnGCBSOllpdwN0iqg6szEflWNq3i4NOx0sb/l2mRhjB+lWBoeZe61PugmTT7JSAJ5flZselM1zxFp2naNJpVpcSajcvwZG5A964e8lv9UdPt95JIinIQcClihjhTbEu1fTNUSOhXah3D5jTxgjkUnSnctjHAqgG9BijbnvSqpGd+PanDAOKQxABtpV4GaUZP4UDpQAmGo2mlL8Yppbj1oAcx6YFKSB19KaEJ6mmiNSSSTQBIZfl2hQKbkjrTcc8UuPzoEKCSelHfk0o4FISO5oGGRTlwATjOaYcDk9KDyev4UAOLZozim4Bp3UUALz+FNK89aUsc8LmkGSMmgBRQTS4pCAetAC7cDOKUjtRkkYJ+lJnnHegBRkZ4pQQB6mk7GkBBGQaBig54xSEc5NIuxQdoJ9aa0jMxQcCgCQuAp/vUhz1xz6ULx1xTiM80hDFDYzjrT8YI9qCWJz7UA0BcOtKKN2KaGzyODTGONJjJpAxLHIxTiQBmgQEqqDjBoBOB70nBAPBB6UuD3oGNDnsuacTkUEYAz3pB1oABx60uacjFCSO4xTaBWFwTy3ekK4NLjABz+FIeelAWCjHpRinA7ecZ4oCxGrAn1pc84xxSvIuBuGPTinAAigdxtKelBO45JpKBCr0x3pDR0o6nigAyPWgjoaagOMGnds9qAE5pc96KTntQA7gnuKTHGc0DnOfzpMcZoABkHnFGaQfexRQApo+tLwfwpvU4pgG0Zz3oFBIBxRjNACk+1JnGPegnigYIoAKQ9KB94jpQeMjrQAnHJz2oweKCGBwCMUopALxmgkLzSZA+tKefwpgIjcZH5UHk0vJJzQMdulACqaTp0oIxSepoAXtTSTxjikYsV4/OlGCOO1AhCCTwKdwcp3x2pOnPpSkd6QCYxjJJNKCKBnA/vGgg54x+NOwxAR8xFGcjNHfPrQc4OBk+lACAUDjIGCeuKF65PBoyN27nIoQCMPl4UUKwyFwfWgFifqacM5waAF79aADzznmgHqCMc0Y25oAa2Bzg49qUydBtPJxmlPA5yKYc544oAVhyDQyl1KinNg4wMcCoJJkjONxz6CgCVECAgZP1pxAZTmmxMJISfemgyiTjBTuDQA3zYraIBmwo9qmVlZAwPBGRTFRHjCkBh708jKgDoO1ABxTS2JMdqeFJIHrUalJMkZOD6UASHr7CgdSevpTJYxMgG9l57UQx+Uu3cW56mgCSlI4zjimsSMYGSaCTjFSAuKVjg49qarYIJ6ClYA8jpQADrmjGWFC8H2oJpjQMTG+Me1J3B7UoHqKczbh+lAhCwYAjH4UgwcjOD2z3oUKBjH0pCPnVvSgY7GBTSSBnGaXOacVIAz6ZoEM7UeanYj86Oe1NMSM+SvPrQBIcEAjpQKQEKvsKNwNACig9KaxYY2jPPPNOx+dMBBx2HNOVyhyKb3P6UYpAK7bivbimhhu2g9qUdaRgRyBmmAYFWblLYQxNbk5I+cHsarUCgBQQCMmlPBpMkdsjvRkEZFAARnmgHqKQE4oOccdR3pAIQD1FJkjtnNOwSAc54pvNMBDkH29acOOaQZzilIxQA5Su7jmm/hgUBQDkdTQchsYOKQC4xSd6XOD60HnoMUwFFGKaoJ4NByDgdaAFHWmvu6L1p3QUFsnnFFwGcgYIwaB8uD2p5AxSAAEjt2oEBKkcE0oGR1xSbCVLAZA70LkNzQA6lGO9AIPFIaABuGoOcDFISS3NOB2nJ6d6BgCSuQOKTGeDTuACFJxTaQCZ6qRSqOpz+FLtz8w7UjJkA+hoAUZ9KDtbr+VCnPUdRTSvagBcdOc0uM9aaOMUuc0APPNNODwRSKdxwpBPpS4Pfg0AJtpfwoyMU3JoAdJgbQe4pfM3ADHTvTevWlAFAClkLAqMHvTSSaNozmlwKAHqpKkkjjtQFQclj9Kbg0oGTycUgHbkPAzmmsAM4PWlKgYPWkJHagBSSfvUm0A5oJNGaAEJwaByMigkAc0CgCQ4YdcEUwe9DEYHY+opoJNAh5NNxxSqBQ2RQIQDA56UEE8g0m5sYPNOU8GgY3cQMUoOetIeTSFiOtFhj+lNGQc05GJ+7we2aUl8kHFAAzg0m7KgelJtGeRS7SOe1AgPFOI/Kmk80n7wdfu9qLADqMZH3h0NaOn+INT07asVy7RD+B+RWdkmlIxRYDtLHxtayny7+Brc/31IK10MRiuIhLbyrIh7qa8p5qS3nuLJi9tPJG3YBuPypWGenssy8EcVG1tE2HX91L/eWuSsPGt5A+NRi86H1QfNXV2Or6bqi/6PMof+452n9amwEhlu0+WRFuIh3HWs240LQ9SPmNC0Mh7rlTWzs7CopVUEebDkf3l6imgOJ1LwRfW37y0lSeP8d351Rt9X1jR5QplmRh1SUcV6GjbGHlSn2VjTp1ju4jFeQJIh9RmpuBzNn42tGYR3Nu9u/99Tlf5V01tdR3UQlt5hOh7p2rnL3wPZXLE2Nwbdj/AAtyP51z1xoWuaFMXjDbD1khbj/PNUmI9NEoxhgQD1U9DWfeaFpGoKRNZxhj/EnBrjLHxne274u40uU9WO0j/GumtvEemXKrsvVRz/BIeRSuBg6j4Iu7eXfp8izQnoGOCP1rCu7W/sMC4t5Yvdl4/OvTknYfN99P7y8irPmR3CFJFEiHqrdKaYzyASHuaeMkV6Be+D9Ku8eTEbZvWM8flXO3ngvU7NC1vIs6eig5qkwOedeM0KSB14PUUSpcW7FLiGSNv9oYoR0K4NMAwccd6O2CKcFTorH6Ggqe1ADGDEcUgJHB6VIPemN60Aa/mVG8znjOKKKzELDZzSrvQcetLNC1um6a4CL7UUUhi2tzpQiy85c+pFUptQs1mJtx+OaKKYEC6hJcPsgUu3sK0o9HvZlBnby1PbPWiiqGXItAtwwZmc4960Y7S0gbIjB+tFFICbzf7oAHtSF2PeiikIa2TTdpNFFAxwj3KQWCDux6CsmXV9G0+6ZY1DOPvSYzk0UUDRI/inR5PvTMP+AkUxfEOgZyzO3/AAE0UVmbl5fF2gW6bY5Jseioahk8ZaOPuQ3Eh9lxRRVmdyu/jexH+qsJ3PpuxTB47BGBpcg+r0UVVhXIl8cTeYGOmoR6ZNQ33irUb9PLiH2WI9Qmc4+tFFOxJi+UCcsWc+rsTUnQUUVRAnJ7GgB8DK4oopiHY4NKCQtFFACZBNB65oopDFVwM8A0ZJPYL60UUCAEYoIA5oooKDJxgHijPrRRQAvO04FIMsue1FFACHIpe2SeaKKADPvQvDE9aKKAHYzS4oooATkkelLiiigBQvvRxRRQAEd6TAI70UUAHHAP400/eoooBBtJUgUqjAoooGOwO4zTs4BHfsaKKQgJO2mqpycmiigQ4JgdfzoGMZoooGgIyKaFY8dqKKYDuAAB2oB5NFFABjJyaXGSfSiigA4FJ+NFFAwo4FFFAgJxRxRRQAYHB9KO9FFACFRgkE0YyoPrRRQAvQUcEEUUUAIBgdaFGBgdKKKAHZU4zxTAxYZAoooAAeAR0NAI6UUUALnGcd6O1FFACZAFHUUUUwDoAKM8UUUAJjNL3xRRQAdDSZ4/GiigBqknJPSloooAT5SRn7w6UoI5oooAUHJFL3oooEBGRTSONuOtFFAwPtTQuOe/rRRQCFzkdKVScHPaiikApGRnvmkwvOT2oooEAxS7vSiimMZuIYY7U1HkkckpgdR70UUATjABzimZOc9qKKQBgn880Nyce3WiimArBu/4UDO0ZIoooAQ1C1urOWJPNFFADwPLRTtP4U/vRRQBHDIjRnyiSqnFPL/KcDP86KKAK5mdmCYPPWp0BVACOlFFADsU15Y4+HbBPT3oooAcSAQQDRkmiikAuD2FNSEKNzsd3pRRQBIOlBUqwz9aKKAF6n2pDxRRQAdaX86KKARGFw+4cmnghductnPBoooBji27nGKYTgZzg0UUAIF+XBJIo6CiigBR5mBuxjtS8460UUAKKMMScdhk0UUADIykbgRkdxS7sCiigCPJL4xQMjdu9eKKKAF3AnGeaMjOO9FFAB3oXPc5oopgJzxjoablskY+YUUUAKTx70m/OARn+lFFAAD8wG7tgE0uemfzoooAceDRnqRRRSAcM7STTM8+9FFNgLk9BS4yvIwaKKQCE0hyPxoopgIcg9/rRnjk80UUAKQcZHSkU84JxRRSAUygDG3JoDhwCOlFFAAc9qdyaKKAGkE8AkH2pwJzRRTAOe1KOQTRRSAaQxPHPpSYZfvAg+lFFAC7R1HB9qeSCBuHzevrRRQAhUZyDxScZx3oooAGYZ54peAM0UUAL1FJ0I54oooAGOPunNBZvSiikAAk0ooooAd1FNoooANisMMcGkG1eM5oopgO3R96aWG7C0UUgHbTgHNMZNxzkg0UUAPAyMd+9JjBoooAO9KRkYNFFADcYOBTue9FFAATTTnFFFADlwfvDFKWA4zxRRQAiuAfWkzRRQIAaeAaKKBgemKi8v5t6/Kw5DDjFFFJiNix8VarYqEeX7Qg6CTr+ddNYeMtNu22XMTQOe7EYoopDNryYZ13wsjj1Vs0hilTtxRRSYDCR3609ZpBwAjr6UUUgKV5pGmaimLi1RG/vKcEVzN74IuFUyWVysy/3SoB/nRRQBkx3Gt6BKQTcRKOoYfIa3bXxtG/F3aBT/fjPP8AKiimgOisdYsr5R9nu97n/lm3WtET44ZWX60UUwCeO2ulxNHFLnqSMmue1HwVYXbb7aQ2svbacj8qKKoRz194R1WzXegFyP8Apn1rDfzYW2TRPGf9pcUUUAHmIe/NAwaKKYH/2Q==" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAQ4B4ADASIAAhEBAxEB/8QAHQAAAQUBAQEBAAAAAAAAAAAAAgADBAUGBwgBCf/EAGEQAAEDAwIEAgcFAwgECgcAEwIAAwQFBhITIgcUMkIjUggVM0NhYnIWJFOCkjRjohclRFFUc7LCNdLi8AERJkFFZIOEk5QYITZGVVZ08ic3o0dXZXWhswkxSGaVpIWRw//EABsBAQEBAQEBAQEAAAAAAAAAAAACAwQBBQYH/8QALhEBAQEAAgMAAQIGAQQDAQEAAAISAxMBBBQRIjIFFSEjM1IxQUJRYiQ0Q1Nh/9oADAMBAAIRAxEAPwC14DyYlNqh0euMHGlgeGBgvQ2GC4TWAiW3xkpj86C8DUo8N+zDMF36SC6HOrJMNt73ear3oYAGGgH6FcGmjZzWkPLtQyYxn7PNQgBwHVoDZwUR6HmvUmox7wU0FX4aJqzZ3gCPMDAE6CQJ0AWe3oTeYZ9ueCqXrttVk9N+s4f9ip0mGxJPx1hL5rdVD/kzaNHN6Q+eByDDYCbHRWeRnxdeDKB5o/Iq+NDfjO+BEXJ9a7uFbTTESKc+cYZvYdDyGxvSEqNeqNTOv6MCFFZPAA3702Ou3IzUplGdg1Wmgcc+s3egF5v9I2GxJtdqlRKGcl6nM64SGnv8C55xs4tcSbwaOI3UZkCF2A0eGsC4PWJlzxoAPuXNMMz645mf+PNNjqFKoM77HtP3HLe5LsA9+CyQUedZ7suswaPz9Cm5701Yb3E285UKzojZ6L+Z7w6AXZb5s+sW9HhWHrg963MAAw7z+hc2GzGcB7efOpHXKzKkxqfnnAkGexk/IrasWxcd/wB5Tork7COweGYdBrtFK9GmuUSy2rOcwntH45um9h1/Iuf8VOCF8WNbJ1K2bukzI7GecR0wzAPkTbFxy26Par3FyPZV/wBVjUqFFk4HIkdB7Oj5F7AvzhX6O0zhzIp1uW5QZ8jrZep8nOVmvI9jcNLc4qRZtSrl3THrjAD+6Ge8z84K4tu3n+G9Z0L4lyaPI/o0tk9jyYbEdNr92lLtV+VUo0czwzMMwBc0ueNSuHtUl0q4MLheAMGeYeP7t9C6ndtbtWmx5dz2xfdV9YbPBBkzBebLqqTdyVYqq2/ra+8z85phRrxHmtSXLM8NgZ+RHG56ThTYj5vAfufOqzNz2bimxnuWHmwcMHQ6DBQtOk0d+j8u++GAP7PzojptS5hphh+HGde6DPoUWZUpdYlBKnP5mAbEzP8AvOB54Yec0QtodpVWm1mRUqrFZ5dhk/GZezBaC2+IsGNICh1kD5LskBvN41j6bPrlKB1yLWXg1wwPeoMAIPMNesmzOP34Bmg7M9QbuueFnc550oAM4Bn0ACynD26qjYd/u1GlTtFrA4pyMNhgol2gxTWo/qC6pkynmGekfYo8CsRKVb8g4mGqeGww60wNXG+xz10O1y6q+E+PNz5zlz35pUS3uFcziR6qOoyXqFKZ8Ez2ABrpfBn0daddVzWvWLjqUOfQq3GfMwA8DZP51yniRZ8Gz+NlQs6nPmceLMBgM06WPS3f8s1Ds+y5vDVy3+fahTDCG9rAYf7ayU+7ZV1erzfpUaAbGyGbPvvkNdq4Y3bwTepb1OrnDVmffdIA2ZJx42ySHnw6F5/4tVulXbdrs6zqGdEp4f0Q2cDZfTSXQLVh8WIzEis0OpPHChHvpJycww8+C7tw9ueucS7BicOQgvSfVx81JN3Z0dhrx/Ar1Ro8X+bZRsyO94O81KpvEK/wYlsQbqnsnKPM8O802t+i325g8NLZClN2AzbzU0wihLjvBgb5qVavAezpPD71VWaBTalUHzzN7szXgKw75o1KuFpy8X6qB9bzLpm8Bn5zBb22OKN/vX1IuOzqxWKPRJTPQZnypn8nYunDVu/Slo9ncLqRH4eWdTZNEkTXgOpNYYA8Hnz/AFrvvDTh1RrA4QU+h2zUtYKvoPyZYd5n14Lw/f8AxLrlw8WqZVazWTrcekTAfzke58/+Rey7DsalW8xNvi471CTQmDCqU2JngEYz/OmBsI3BywKVTZE6h0N6HNlBvkPH3+c1zS5PR44t3a1UIv8AKHAptElM+M00GYGhurjAxxIpZv2dxDptBkRXt4dYST7M/wCNYmz7/wCNNHqhxLxnaNKmhhJAOgz86YZOW03hc3SmnWLqt+M9IB4wA2XvG2dBqPWOG7FYd9auV8Aaf6OYS4hXPECsy4sTmZlQOf1+QFnLqs++IdBOqzqNUjaDsDoBcGGqXTXufrcuyqrWTqUJ/AI0gAzwBYS/LMYt6suwWJQPNfJ2K4tufBgHHfguHGmh1gYdivZh1ifW6Y5UYMPl3zw1g8/z+RVscitt6nQLmiRJbmcQzwP5FU3DTX6bW5sR9voeXWarR2A4quvvxMI/4yqZltzrhrLr4UeTJaMMwkAGw17sRbGh0bSaOXEZkg4zm80fWz8/zrQPQGP5zo8T741Tmdh9+CC57GYskaPc9uTpLJygwea7wMEEOf6qzNxgJJ1SNvjn5/rTYepvEX/kLEt9tt7naWeAGG9aKpXIxclzRIj7ByY78PADDsMO9ZK0qO/DCpy5fgxwe3vdgLYWZQQjXk1r1xl7wdYMA6DUbFPTZM6z6pWKVUvBzjYYecOw11Cw/srR7GBuVVeZ9dhrvfWqzidQefoYTrggg9UGA0Y0uP0fQa1fB+yYkmkOymGI3LzaacWeb3QD/ZgCx4Y/qdzpFgW3/wDavOsVJwzmymT0TM/J0f51rbG1ztSnvm/mD4ZgqWzJhybIatGW5hLpf3I/nw71rabGbgQI9OY9kwGAL7HiGSQAYoDD3iPoQLcJGgSUIGgNGkgiGhw+KkYfFMmga6zSw3o0sM0ebBh8UYdCLD4o8PiraGjBIATpgkgaee0R1EVKqTDxg50KJMDNHAgP5Z6a1c/hswZbOKBgqySzu8NS4ZmDGmkYLJ0eFJJZNV5gr6Syqx5ncoQrNFCbKsNFBorJaEEZEEbcrAGdiRsoIuh8iZOMHkVnh8UBsr3ApXmfxFHNlXT0ZRXo29eCq0W/60yYK15bao70bBBWmCZw+KsDZTRsqFoRggw+Klmylh8UETD4oDBS8PikYIIRggMFNME1h8UEfD4oMPipeHxQaP4aCLhvQYfFStFz+tDh8VKkdBh8VIxBDh8UDOHxSw2J7D4oMN6AABHh8UYBvRYfFQI/eknksPigZ7ElIw/ES7EWaBGCBIEBojSw+KSBoEQda+pKEHUkgSRY0SFErBpB1pJKA6CMPw/+JACMFYdR96BGoBp0E0nQQGiBCCJWgYIwQAjBQJCWfwQIwRZ1GCBGCAuxPAo6dBEHUkKJBIRoEaLGCIEKIEBgadBNAnQVoSAPepAKOCeBQs+HWpIGoOfwUgDQSANIE0kCITQVgz7JVrKsA6EBGaE0jQIAQmaJNIJAHijBM9CPP4ItIyBGBqPn8EQGpDxmmTSTRmqCNGCZz+CPP4JsSM/gln8E1n8EWfwQLP4JZ/BNZ/BLP4IgZmo5pGaZNAJoDSNNGrCSz+CBJABoURoQQIASw+KdRgG9AAMo1Iw+KZMN6COkjNAoDRqOnTNCasR0jRYfFCahJlCkaBWoefwSQYfFJQlIz+CSBDn8FYdUuL/zqCCkRnkE5RzNSM9ijn0IABLP4IMjQ5/BA6jz+CiZ/BHn8EDppAms/gln8EDqPP4JnP4JZ/BEEZoTSQIsefwQZ/BNGkgBBn8Es/ghVoEkhSULJLP4IElaDufwSA012JZ/BA6hQJZ/BAZpIEgQO57Etiaz+CDP4Kg8gz+CAzQ5/BSCyNDn8ECBAjNIzQpKgCSSSILrToJB1o0CSSRoCSQgaLP4L3wEZpACaM0gM1utI45UGdWIcKpQQ8WK8C6RGe1qdEc79FNPBmiAMA01kEZ70OfwQH1pKQs/ggw+KNJXtBo4zZ+0XwGWwRpZh7xxNgwTUypcsemqmtz32WvAMw+hQqrclGClg3BqTJyz34Z71mtcHMfZ621EZDObzSp7nmQaraER853LSwe3nmsud1PxnYlOOpMyfGDf3o8wkXhUp1S4hwaHr4RD8BZq5OFbFq3gdHYDRo9UZ37Ovzq94hVJim3XQa4/siPmAcx2Au3PUqm1imhz0XmWsPBPPoNDDjN4cN6GzRqFR6bTmTkRTAwePeZrmvpOcB7cqs2iVKncnSjwDnwPYGC9JHACZXGvvRs6AYbO9ed/TGk4TLZCc+D1PlSdB7A+sEesvbFNfsyvR4lnP/aF19noZPez9ayoHd0zjhR4NZpTz00J7BvRwPZgvTvD3hpavDGhzZdpeM7KZ1+beDeH++a4/wAMaJOh8S7g4h1+lSXpGZnA0Qz8c+xFvYtSgMcrm/KOM0AABmC8e8e724oXVKes6xKHGjRIuGEh4972xdtsyTeMy3KwxX9ZnM9dnMOzyLk/Hvhpesx2n3V9owjNSj5U47PYGzoU3DO3iSZJue3q8b7j5w6nFMwN4OxaipT7uvyBH+0F6szMDz+8Ld3bwWfOm1ipQdaNHp3R6w2HJ+g15yjco8/oTgNk/IYLJ66UfF2q2NSzsdhim1UAAw5vDzrk+s286bgK2ks0MPYN7lEkgb2GDYbPICz2qLQgBwzwNE9GlmehE3h5E9yz+k6+23ngadhvOMkEoOtW2u0I4z8CU03ObNnNSKrAOmytDMDHDYYd6CpHVarM09M5OgHYCiQNOTMjxZco4zT7wAbwdnzohZ27BcuOUzR5W5j330K95aJZlbPQpwT4QdBmG9V9et5yz5mgxUmZmuGbMgA3qqB45LR89Ke5j8UEHRbhsOj1iF9p6PdQcuHtomGZsguayY0RmRoc2EnDyIqbDqMmVyjD72kftsPIpfq2D7vM0E2lXJX6Ph6qrkmM0Hus9n6FIgc9eF5U9iqzt82SAPSMFXs0edJPBgM/zqcFsPhgc6pcmHnzW2x3CZGpXBas1unU5v1kBsg8zUTPN4FYcE7bsC/6Ndt8cVGHnnYR+2A8MA/3MFwR6A/Gw/nx6SHznmrCmvXBGiyG6GckOaDB7S8i47tGGw4o2lZUOvAfCG4HqrQpQZvAfYazVq0EKlWZFNiygjVBhk3wA+9eiuDPARy6uC02XUjOmzZTxnGN7oPeqW3rA4c3hbx0O5m3qPedIksBJkAeASQzwP8AxpFrz4cBk0Gs3JNC38P57fe0AZ+de5rbB+leiXcdnXVQwptVt6GcU2jDv6wNdKpXoVcHbABq+4JzAmxQ5rMzz3ryV6SfF2dxOuH7I2JBqT2BmEmIzgYScOjPyd6+lwsnQPQS4LQeIVLuty+LOkvR6vAwptQeZ2Yd+H8Cs7w4b8K+FdjVW2eJtzSfXAAYRojUneYB0GALKejxcjFmQKPQL14qV6iQpRvmcEHsAjH2B8ma3fEuzOC1nnL4lUq9Tr1QlMnycSdJ5lnMAUXCtvGkCBZVSt6XUp1Z5aoB0MnsMwQBdVcuGFHgzrxnhEDo39ALQUG/+GVYvKq1m+7DCHHqMbwWmugH/OsLVaaxlqRNkd88GfoWK3YKxTaNDtelXNbFYjHW4QZm95wVlbHFrijeEP1UxTYYckYHmAdfyLktVuGshS4tDl6PgABg6AYZrZRplx29ZUS46bXKUccw5KScc9+HnMFFgbzuSXJdqBuUOmhNfPM5DR/5FkWarKCPEbfnPBvz2JVulG9NqculVXmQihnnH6DWfpszn2gPmze+tcdwOkWkb9zyJsqq1U2SA8MwDrVPUqqdErciDBrMxkIp4bDwzR2ZXm6bMjwZbAcu+9vMFVXVSqx/KDIYbiPSWuwwBOEXsM/Ca9ZSz++mZ6J9nkUc4ebXPN9hmCVbqrE8omg3hyuw06zGlvMOxG39HZmyB9Bn5M1oLCHc7HquoWdTYP8ApQA1nZAbzDyKLAB+0qodNfPB3RP9CO2I3rKa1Sqzoxt5hrAG8EzcNVoZkcuDUuZlsSQDMwQdIkzJ1btenvnWYxtPgAGHz+RbWxqlUqPYPK1X7toT9HPrPBcChzOZpYU7T/ZT1s11vn6rVZVPrj8QzhVEAAwZ6AfXR63+Zjyu60qTFAY77eGZh1h3rRxqlmWnqLkNsVUzhajmw+jDyLd0SeD3tF+pvh/o5o5mzz+CADURmSGHWjA8+hcPTa9pGfwToGoWspDLwYeI4sbi235SEkklL0CaMFLAN6A2cDUIRDSRmG9Bh8Vaxgj60gSQFh8UYRs0CeA0CCABn4nQrBlltkNMFEz+CPWVoTc/gkaaA9iI1CzRqFJDeppqOYKBEw+KWipfLbNRIA3rIADOAJYfFTgZbQGzsWor8N6BSzBAYIIpgmTZ+RTtFBh8VkKw2cFFeBXDwKCbKCtNlRzBWRsqObKLV+HxQ4fFStFDh8VCkTD4oMPipeHxQYfFSlHMEOHxUgwSMFQj4bEGHxUjD4oDBSI5gaaMFLMNiZw+KKR8Pihw+KlYfFDh8UEfD4oMPipeHxQYfFBHw+KPD4p3D4pACgNYfFLD4qUYIcPii0FDh8VIw+KDD4ogBgl3aaSWHxRZI+9NYfFO4fFQEg70aSsIARgCQAiQJLsSNGCgJGgBGrBeH7xGCBGCA06BpoE6CgJOgmk6CtB5EhRKFhUgEyCNBIRgo6dQOp4FHToKRISSBGgJGmkXeqEgEYKOnUD2fwSSBECsGCMEAI1AdA06CjgaMEpCQBowTKeBX4XhIA06CjgG9SwZcxRCQye9WAdCr2WcFYMnsRZIsPijw+KSgNGoilmo6BZ/BFn8FHSUiWCNRwTqoAZ7E1n8EjTSIOgaPP4KOCdBFnUefwTSSAsgTRmkZpozRAuxCiz+CWHxQR0JmnTBBh8UDSQdaPD4pYfFWEaDD4o8PikAb1AWHxRdHs0Z70YAeCANZz+pNGZ5qQbKjmCBkzUczTxpk0AmmTRmmlYSSLD4o1CUcwTSkGmc/grURprP4IzNBn8ESWfwSTSSB1OgeKjgjz+CCWB5J01EA8UesgPP4IM/gms/gln8EBAjz+CaBLP4ICNDn8EGfwQIJGfwR5/BR8/gnc/ggWfwQ5/BLP4IECQZ/BIzQoFn8ECWfwQZ/BWEjTSSIFn8Es/ghz+CWfwRZJZ/BBn8EkQPP4JZ/BAkgdz+CWfwQ5/BAZoDz+CDP4IDQmqBZGhz+CDP4IM/ggPPeln8EklISRpAkqQBPACWHxToIA6EZo0AIEjQJIEln8ECXYgXWh6EGfwS616OoZ/BLP4IDTS3VB1LIE1n8Es8CXO9SEs/ggzzR9C92sutRJ5sRo+u/wDkUs9hh8/QolVDNgGz6M14hT025KVMq0igTg5aQAZ5yDwA1yLi19meSOpQZz0aWweez3y6m9Ybdw3HEqTcvR5XszwzXm/0hKrLDiIdpU2ohTcDzAz6DPZsQQftg4BQgnS5LITujmOjNZ/iXflVsl2OxSn9Gaf5wNcf4nXtWaxKChy3A+5HmDzQYGsUzVbjrFRap3rF5488w1lztXcKl6QnElmjHbNwU2HJafADZdMN4H8i7Bwl9Iq4GbfiU6pVKNGwDADkbwNeUrqqtYxhRK4/GMoXRoqhk3VUjaOJBl4B9CD9K4F5nWMJVGlQ5hhvNmIe8AWS9K6BBufgs1dT8Q40ilnmzmuCcCgptSs92c3UnoFTi++B7Bd4vaq2PfnDI+HL92gzImnhqnvMPzraLSwXCi9p17WLHp0HiMzTarF8DkZfvla3J6QNf4b6VlcTaOzGNt4H4FZpm9l4w/8As1YWr6Hlh2Ha9Tqtx1mTVTYhmbMgDwADD6F46mUq+OJFblsUqJJqvq7PBnPMwBB76D0k7ZrdvSKdSokmZNlRs2ZGYAGf+dcova8IN7W/bksLqnvR4R6FSjxwPMPnzXlmj3CfDe6ACq2y9sAwkx5exeheCfEuq02RLo1DsNlmlTXuaCXL6wPvVMWm4/WlYFStS34NncYgmSH8848uZmH5w7F5P4rvMHMjuSnI3rNhnQPl+g1sPSTtj13xVkeoKMDLvXMNnoeDzri9bjPsyAB/rDrNTdiw9VPvcpoPg87KPDAFa1jh7xCtuHz1ctGZDj9GZ+dZeeDjMIH4NSkm6fuc10A+It8VW0mrfqVwTJ7UoOV9XSw2fXmudUMvTWX5LDrmmYGAZns7FXyXvu7pt7DA0EmZXKbnBlgbIfOo7xuYaep1qmizsCq1KBXjlMP4OgyeGzYolzzGKrVtdhsMzDxtEMAzVarK2JMGBWQOqsGccwMDw7EUjwAYZd1H1q+G9KtyvXNUINclhygQ9cAPZvWUqvK8+bkGXrR+zZgrCgwDn8wDDjOqwGeBnvP6EFrb09uBcNQCDm80AGxsDeYK7uSsWPPpceLTmHociL1g91vfnUulWr64kOsWzF9T11hk3pPPbAMO9S6Pwfqsl865f7BxqfhnzYHsMEGHhhOMmp1HfBkOwzNFUjzA4lVfZe7wMOw0Zsxaa/6tCcEmmB0SA6zQHAsc3TfCqyWc+xZBW89Q2ZWpWYJyQMDADZ6wPzrcfZu8bedZbt9wHvW54RpAdeZ+dYo2bVhxzfiTnnjDzr0bbFSpV1cG+bf5aNWKdGN8D8+/Ya8sWF1cWuKno9w6Jb8t+HPA2egwWQvzjfwkur+fItKqTMh+Ng9hDwAD71pYdSoHEXhfTLxfgn9o6Iy+zm90GfevMkyewbWhqb+8OwFtDJ1CTx14rs219laHxircyjzWcPV0jDNkPJmtHwZvCzuGlpVh+sNzIFymBgByI2Z5/IvP+ejH5oPagYYLaw+JE6ZPpj9zvsz6ZrfeWTDBdPcPXB2TSr/9Ehq5jgnMqEp4358hn22eeC8mXJYbFEtyFLnVWpU05QGbOsGzPv8AoXdbJ9JaXY15VO0bYo/OWZW8DAD2erTPYZh8iY45W9Y9Etc7modY+07sV4JTxm9sez2exNRna3lSmxtYQcNw1q40a3GYBxJ0p46gZ5gYdCVw1i3Kw/HC3KH6tDR3h0ABoLYsyuVt8zpVK5/1d472D2GALG4U1tvcOqNcNIrb9Rqpw5dLDWD52FS8Oq9y1LqdGlQYcmFVD+8svB3qw+0lfuSou0OC/Gh86fKhns0T781IZ4eyrPdCnVGUzJ5p7Bl1rozXOKzhW83DduiLzbLIcngzn5wzwVFRLM5+BIqrcvkJYPbDz8H9CrLho862K9No9SbwkAeeHyGtxSmabP4bSGKbmEh8NZ5ozzAHwQZcGbjptRNhiC89IimBsmz0Ga2FeuSv2w/BuB+D6tqz7OZhhsmAs/aVSqT15UKD1x5QaEwPOx3rW8QqVOuEAg2rLN6j0hneB9hrH/GM7WKbO9UQrjgsPSfWgG+YAG9k/IirYNxoEKc+5gZgAK+pt1QQtyFTtCSFTpx7GcOtXFEtinX/AE11vX0XmM+WjvB3/WuvwM/YdKl1iBU50oANoGT5YAPfmHnWgsyyabdVvR5dZi8toGYGYdZ/WsYzz1pT63Rm5eGbJsZ4da1FsVJ+NalbgnL8Xk9mzrW2GRq3qVFkx5bDcoJOhJMPC612CNSqkzYsKhsH4rB5mqThXwZY9XU+uXA3JekTQzeZe2Bh2YLqXqpuBHCKxrYB0Zmu/wDhvrRF7ty3zM7AjcnFP5zM1sKbRLj5Vp9iJtfDPNVTNKfmSAYbBdqtuM3yDQON9AYL9BzX4hyObyQrlKPTlAf5EodzuB4ZvruDMNh4PEYA/rWHrfCuhzJGbDeiuHudHSxn2k/4PxwUuHcge8fWauqyazaroOagPRD6MFVU05Ux3TYbzXd1cPll/V1iHVWHh61ZMnmuZUGZOOstRH/zrrp01xlprT7wXzvZ4fHh0RewxgzdTrzLYI42myW9FJMDXI2tWmCjmpb3WokkH+wFT0AKR2KJD1z9u3gpwAlwASz+CkYfFBiCxws0aQHvSMMkgwZ3uLZCaAYp4zVYdbYN/BhSDk7VIdQoM/gln8FCzwdCIFHA96dA96PLTmWUUlnYjZMMUnjzFQvwrzQYfFPGG9Aj0BgmjBS8Pij0W/60Fdh8VHeZVsbKDRb/AK1l4GfeZ3KIYK4ks4ZqteBaUhBME0peHxTJgs2qOmsPipCDD4qEmkkaSkR0k6gNUI6WHxT2HxQGClRrD4oMPipGHxQYfFR4DWHxSw+KdS2K6EdINROn1r7h8VABBh8VIw+KHD4oGU0YKXh8UzhvRaKYGlh8VIMEBggaSw+KLD4pfu0DSNLD4osPigSMASw+KQAgDD4pJ7D4pACBIu9LD4pYfFICRggRoHg6ESaRIDBH2JlPAiDufwRgo6dA1CzoI0AIwQJSATWz86IP3iAwUgE0CIEhCRn8E8Cjp0DRZ1CBpJAgdA0YJkDR5/BSbSEaAEYKgSeBMowVg8/giBCiw2KEJAGnmQ3pkAVnGZDYrWdjM/iKwBkMU0nWehQBw+KkACIAzJPYYCgAEjSSRBowQJ4+hMos0mVIQqQAI8/gkCJA0fWo6kGCDD4qrDSdz+CPD4oDUg0s/ggQ9aoJAl0JdaIJOgmk6CAcPigAN6dw+KPD4oAw+KEwTqjme9ABpB1oTRAgkAG9SE0kZ7ECM1EeNGZqIZoGnk0ZpPGmTNAJmkCBJWJYJZ/BM5/BCZ7ESAzTSLP4KOZqAZoEKBWEaMECSA0WfwTSNAWfwSz+CHP4IM/ggdz+CWfwTWfwSz+CA8/gkZoDNAgeSz+CDP4IDNAYHvR5/BR0SsSE1n8EaBQEgSQoCM0OfwSM0ygWfwSz+CEzQKwefwSz+CBJUHc/gl2JrP4JIg6BozNNJKQRmhz+CAzQZ/BAefwQGaFI1SCSSSQElh8UYAjw+KlYARgkiQGCNMgjVA8PigQZ/BJEDM9iAzQ5/BBn8EDufwQ5/BLP4IM/gtMLAjQI1mh0hAjQLoeQFEhQLJodA0MyZy0c33HMABBn8EnmQktaZgpuBkrwOpXhb0hinOSYdQhGEqN9aCw+Iv2wh+o6y2ca4IWyS0arOMF7VXh1ToX2fYDVf6zP8i5Uzx7ptnuyK5dVGhs1gwwZaD338C8HoC57hqVjQmqwxRufazwMF5i49zKHxjvmj031GzRJuGZmb2Zn0dCr7q4zX3xds2p1Kj3dSoEeL/0MftjD615sn3DzOBypUzmA6HgPoRC142WA3w6u31dEfeeaMM83utVV4WNUrJpdMuNycBtTQzAwDoR1W/511HHcu6UdS5UMAMwwPBK4bwiXDRgoeZhEYPMAMOhcltYtlKxJbmO64VJ6YfeZqJAefjSld23bFVr0g4sFhkzDzngrO87ArNn0sKlUdHAzw2Gr8LQabcjcDNj1k8Bh0GC0H36q0FqqsTpjxv8AXh2LHhbE6TTWqqw3mD54LQQ3p1twmn6bVTz72T3gsho2eLvFiNRjsf15Jep59jrxngs1SpN1WxVmqxBqvISA6DzXSKVXrVmQmn2AgPTT62pfWueXVbEGm1QHPWvrIJuGYfgmq2N3cPGN+twJEWssQK3LlYGyYM+MBgucyeJfFSfAOjsTpgR/3LIAap4D0Sg1xp9wMGg61fSbtqpzWqlbnssMD39a2i0XDVWxxvlW3bwUOuWqE+aAGAS5fXguTzzfkvum+fWea1cZn19V+UrHMw6gfQZ7AWXrAPw55wXwwMEteB29RKlWJ+nTmweMAzwM8Ep9e0X+Ulng6Cjxpk6A7rwX9F3zrrtKvDhszaRuVzhPzmuGEmczMA3s/OoMOPzDfqUjUlSzP606cYAa6wNV5vRZM12XBA2YmfggfkR8y5npgslk8zv2KWzDYMPElAyq96YbPW4pAQ6zMpbtZbgvPRIp+MYB0Kwy9pgpcaj1E6b9o6a4yGh1gfWClhbb528FcbnRjz/o+e8FEpVVnUebHlNv/d9YNYOwwVIbiNXm7ho3qa56bP8AXcoAONI9jmHyLLnPrlNaOhy6rVeX745vHgvUH8ntGh3pTZblGkzKFUWc4cvP9jM/8iteF1k1yTVLzg/Zz13T6cD78CX+N8i2Y9ryL44eI+wYR/OadPQ924pd4VWdPq9QpUqKEaOEkwCIHuQ8ipDXI2aJlm2ak+epO0Wg64+H+dXdSmPwAaobGsETDMDAM9FYeNhzAA+eiB961cY+WYBvm3ng85oJFpXhUbepcu2arVZMOO+ebMhkOg/Iq9mm/bm8IlDpz8ZnXDAHsOvBBWJ8V6UYan3fyL2x6E/o5WkFhy+JXE2DDN2bJzgSDPfGAP8AfBWl434i8Irj4dNU+XUX40mFVNkZ1r8T/c1kgZwdOJODNryL3L6RXC67b8uMKbQ6lSp9EpZ81GZiHvA9i868UabRrtrwMWdSt9OZCLJ2YZmqGHgcUb4pRnTqbVdGEAAGGG/APn/OmbnuSm16qesac3JZ1wzeaePYriNYbEC2qhWawHLTdHYHXvVVVabasa3KPWaVUZIVV/wKkGfQaCvhsvyfZtrV2x61okKoaD7wHVGdEAA+9ZcJlSpX36nMScw/cre6MGm2r6xiPm9mAGYGfeslGp71OuGZCiU3BmoMRsDMA7wVfWKVzNqu3B68mTHQe0DB49maiVIwCa1Lgv6Mhg9+xWYWxcEmlncZxDCnv+R7YZqB0XhFcluVWyrgcv8AbgPS2I2gy9OZzMw8i4vb1S5DmIIOYNG8eCuwqRxoERhjofMweDzgnarw0rlNo1PuqjUp6TCleTrQWFt0RgDO4NTRl0s/GiAHZ5wWzsCmxLbolwVL15DqXruGZgEc94B84LlLNYrEB2bK8ZmR7HAw2GCto1EfpUgZVHrEY/OCzsK1Z7EO7QlMRNFrDAAz6F1rhWzcH2UdrgUo5lPfnmHMAe9lcngQ9aYDDf7W/wBBrsfD3i1att8OaPaNx25Jn+rpj5vHHe3gB/Jgrixd+kJwfo0AaPcFAc5Z5/wJJ4Z7DwwVI9wHqVvWbIqs77zWKXh6yiGeGAH3gvQvDq6uHnFGg1CmwQCZH9j4wbwDsWgk8PalUprTcsGXqUbIMyQz2PAAd674hGGF4FcTqrdrv2Vua3GY3JQAfhyOjMOjoXQLhtJiSZy2GFmbStj7N8X5dsxW/wCb4VNzhh3gB9mf5F1s42GxdPDfWi+FzelW8xDdOdp4PdC1FvHKNrTlMAHkVqdNYyz00YRmw9mtubmR0rCNp6WxI2c00yeClh0LHZhSXDR4M+luhLYzWHoln06G+7oMdZrpFYD7g6spRJOb+munh5DBqBYBw6iFVfY8IFdTJjeWmtBnrRwVPJpTG999LvsREYUUysU2M7p6iMzUgKVaTP3ucwgjTKNJlaDEHAPP51LwAAlGmBrmw23+sFo3mWIA+OGxZ+ZcNvzJWnTYj2fnVvYHJ0Ajn4e9VL2eeppqdMBzSzBFDkxTa39gKmiKBuPYaaJ4HGR1HG01GewdzbbWghhrRT1+hQM0Elsx1ANWEbDS370E+Gxq7AwTQGYHggr/AFJFjSjlN55makKaajoEBozNM96PsUrABqQD2xV70lsDVaFbw94q6We2qZk4KwAwe9mslGqWsfWtBAk4DqJfCvaw2YabiiGCE5J5oNbepejzwS1kGQID2LESM/ghM0znsTWfwQR5moqx5XEzTw/9fkVa8CCvMEsPipBhvQYfFZKtEMEJgpBggw+Kl5aPh8U1h8VKw+KaMF4pHSMEjBJFASR4fFApA96BGgUBJI0uxFgBJIEaADSNOpIGksQRosPigjmCaMFN0U0bKCJh8UOIKVh8U1h8UDSSdw+KWHxQNJIkeHxQNdiIEeHxSAECw+KWHxR4fFEoDSMARYfFLD4qwOHxSAESWHxRB4EkgSULJO9aaRIHgRAhBIEDqNAj70EjsSQAnQVh5ECaz+CNQHc/gjz+CZw+KNA6khRKxIBGCaBEgeToJoE6CgSGQ2J0EIdCkMsmgKMCsGdiajM7FK0XP61aB9alxg2KOHWnWTwUbWswAMEjBRwNO5/BAOHxSRZ/BCpAH0Jo06aaNACBGkaAEkkkCQokkEc0KI+tCiDvQCaRn0IFSyz+CBGaBAgToIQToKQYJGiTTx7FQA00aAz3oTNASQGmkkQl5/BAbyj5/BAZoEbyimaRmmTQIzQpGmVYPP4IM/ghSTYLP4ITSTKA8/gmjNEhRIM/ggSNNZ/BFDSz+CWfwQIk7n8Eajo8/ggLP4JIc/gln8EBZ/BLP4IUkBIUkkCSTuHxSw+KAEkaNACBLvQoCTRmkZppAGfwSz+CaNJWCz+CWfwQ5/BLP4KkCSTSSkOo8/ggz+CHP4ICz+CWfwTWfwQZ/BUHs/ggz+CDP4Ic/gpEgEkAIwVAsPijAECeBASRpZ/BNGaA8/gln8EyZpAanwg9n8Es/gmTNJXax5/BLP4JoDSM14gWfwQoAR5/BAWfwQpZ/BJWsk7n8E0iAFCHSM/glkCj5/BJaZDyDP4Jdiaz+C0WdBGCZAwRhvPw0GV4l2wFyUuO44GtIi9nnXPuN/CK1alwvj1lyla02KAGB59C7Bc952jZkIJd1VJmH5M+9Y3+Uti8Kcfqq2QmUp/+kAf+RZ2PMMngm3UrSm1m37SqQOgzsOOeZ/oXHbb4M1m5LNrF3MVhll2iZ5wc95r9GrMht0SjSJ1VfCM0/nvePAFx2ts8JKVQa3b9OvyADtXPPMDwXMh45DgzeL3Dl3iU2wyFPDsM96iWlY064aQ7gB8wAZhgvXdHt6j0rgjVbKYuOBPwAzZMOzNWfAGzLSqXCh2mhBwkMAffvA1CoeF6a9OOYdOg8zzHfpdaswh8/N5G9arM0mOjM+g1oKIdHsnivIfrlON7kpJhg11rpt4cMbAuez5t60184cjDMMOv86NHNIdqsUejNP1/CpUIN4SIh+xPzmsK8cQKocSC4Zx+zM11W1eHXEqsWy7XLDo896OAZvd7LweRc/pUNiNXgcfiHAlxTDNl7sNGG1nUplv23pMHQ+ZdMMwkAeeB/IsoFYfeqWo+5mvSvEKybA4o8PoU5ilBbFdhbzMPYvAfQf8AAuRcQuFdOsmiR6lTpzz0jPAzeDYaltFuZVg9Z03FKptScCOEFh9ll7sdd6Fpvty5TbSOgMU2NsWMjUfmY+vK2NecFUNmwrdYchxY77EqNJqAPAs1cNSqtSmnOqrbOqfkBQWY0Fnw8D/OauGfU0mmuxX2MHex7PoQZ3nH+w0EaHLqtRCIw+AGfeZ4KRJjMAB6Dm9RMHPeILCq0GXQXQYlPsnn+EeaOmwGKlTZFSYf9geGGCl2fXqNbdZ56uUAKrHwwwPrD6Fpf5ujW5Nr9Ncpp86ZmdOAN8ZSMfTZLgSDYYiMyXXwwADXWLbrFKtK0JAS34z1QfZ8aO70GCw/B+3rOuq+mqHfFZOBCP2JgeHj9iuOIVnsUepVCjUOjSZnq6T40vWzwBBj7bpXr6ohTuaNl18zMADo/QpdYsmpQKbIqoPxnmop4HvWqsaz6dAuiRRrxgyQmnGzgPRz71F+x/FGvc6+3SqlGoQPZvS5wYAyCpk6VwEvb7KyGrVvSm1tmRN8CNLdk/csOxeiOD/EK56JcdVsMJ0Y7ahA+YPRw3geHn715fonG+4KPaB8PJ1No9YaYZMGXjZA9iet7jGH2GCx6bQzgVV/wObM8GT+c9i22ww5deYcneVdCdK1nTnv4PGG8wzWakvObMPPgvSdV4A2BbfC8+I1wXMdVrAAetHA+/sNedHoaxuHZDfcSOCF8cNItErNYcjTKfUfHZeiHnh581Evat0OZPiP25KzaNnxmcPYmum8FuNlGZt6Xw54ty+Zt842hGkSN/LB5FxWvQKNTbjqdNtyohUoQST5OWAe2BRgte0GBLn1mDKi0OZWI8V4HpkeIzmZsB1r9U7Sh8Nqlwxp928MrcjPQpuHWeAMn++Xk/0BrMl2fx7pj921UKbIqNNlHDiZ+2PDoNdw9I3gtVbevmIdgG9RKVdZ4PMxDwjc0HnDNeocs9IesXBwxqh/Yrh5AZp9RwD13TD6w7wwDoXFLbe4cwKoFSiXGbMiVGNmYEvf4/nXov0k6PO4V8IKfb9s25Upk3DmpNW7GT79n514qrFvTo0WOcquRuXms64PM78FQm3VZl40qQydZnZ+tHvBZzzzBUk+26b6hkNsS2Yc2EeeB96eC5LgmYRK5VTqUenHnGMPcqhvB5upPhWWN+zB5BqAv9ut2VL9a1XOqts4AGHQqe2J78xj1bLPNo1lWZmcc2POpEM32fYOYKLgbjW9cPy3+VDMDwMwVlb14S4whb7++P3srOsyWAYdYB/e+GB7O9V8YH4c0H2H8D86kb2N6qqVUhORaU99ye1jAzw2Lqd7cYKbb0iFOtWJJCOfgSYPZmvPkypTqPPhVLmwk+N7EOtXtyMy6wwc7XZez7wNBJvY/XFUOqxKcEY3wA3o7XQAKNZ9Kg1WbUHJcs2WjZwZMO81bcPalBZCR61fjHUDjGwyb3QAKssat06iVmRTq4wB0eV91ez6w+cFnYuztKdGhA3zcYN4BzBnsBdCtu2KNbfEa0jcY0Y83NmSZnmyeYLFU0NG6DtiXUtaEf7M88ftg7FtaratYqVGo9qwXNE4UnXCo94B5Fj4F9TYDnBbiJU7cnUp6fRbr+9U3lDPNnA16q4Y3DUbhtSFKrLDwSDzzM+9eeuEUy575uq3IlwU7Ot2GD5vG6fg1KKfk+ddlqv2gtuvVi3KA+bLWAVeAAH1h3srv4B0b1bE9c+vDD73o8qB/IpBpRpjFVpcesQW/ClBnh5El1hkwSw+KeSAFCLMgypzLO1AAKWyCPQHD1mtNwFlzoLFKI3GPOtqCiz4zbzS0i0M/GqrABpqvuGtsBFBts+tBPgPxs1SVIG3oum42Brs4XOr3qrSuhyVmfkTwa7J5t7PJ8iy9YZfjZvsdYK4gSaxMs3XP9rzwXdjwyaAJjZtffqyD0jvzNQaO9nWQYwXOodNqUzSqPQ73/OtxZM+W9Nd1OzYmBtaky+AeAH1rBVKY+FU5TPBdIzz8M1WT7ACTVGqrnhp9i5Iaq2gg48YeGtEGwDBPMwGIY6YBgmnjbZPxFDSFZPDAjNUhyfFU2sTwDNUuea2ZrAJnnRA8B+zVE9JMPZpU2f4pgadL3a9SM9iaz+CB49ijDRTz/am4s088bLp/WtBP3mqSezmK7IctrCmycDBxtxaBmsYdZrBRnjAxbVwElXcIi21Ce28KMJKy7M9wDBtw1bMvZiuO4dMWumTzTqqmZOHvFICTn7xc2G6QgQmaZM16GpLya1s0El7s/4VEB7B3evMCWgTyBRcBrD4oNFS0sN6lUIJgo5grY2W1EMFktXmCDD4qcbKZ0VAipYfFO4fFLD4qREQp00CAUjQI1AA+tJLvRosXYkh7EYdCAkaAEaBYfFBh8U6jAEEQwc/DQ6KmoMHDNBCw+KWGxTeV+KWigr8Pijw+Kl6Lf8AWg0W/wCtBH0XP60tFS8NiWiiEbRc/rX3D4p3D4o9HNFomHxRgCkcr8UjjOIgzh8UOHxUjDYlh8UWj4fFLD4qRhsQaKAEkkkBo0CNQHQRggRqw6kmjRKBIBGCaBOggdz+CPP4JpFh8UwDToJoATqsPAiBNAnQUAwBSGU0yG9TYwK0HQBTov8AzqOAKRGDo/41AuGQDFTQZ2KPGDPBWHQi1YbOBowBOmCFSHQPYln8FHz+CMDQSM/gln8Ezn8EefwQAaHP4Ik0aBJGgRmgaRIUs/gqCz+CSZz+CWfwQGms/ggz+CWfwUgzNAZoDebTRvIDz+CDP4JrP4IlSEsFIBRGVIz+CLSM/go8k0jNR3k2BM1Hz+CRoM/ggPP4JZ/hpnP4JZ/BEDz+CjmaRmmjQIz3pGgSz+CsJAkh71ARpAgSQHn8FHM0Roc/ggBIzQIVYIzUfP4JGhw+Kgs6aWfwQYfFGrSSSSSBJJJIDz+CSWHxRIBBSABMgCkh0IPmHxSMEaR9CCOkkaaNAjNBn8ECaNAZmgQGhz+CBZ/BAln8EGfwVhd6PP4IM/gln8ECz+CWfwQpKkCz+CWfwSQqQkks/gln8FQSJCkgkAiUcDR5/BegzNFn8E0kvEHc/gln8E0kgWfwR9CBI0B5/BBn8EjQK1jQJI17gJJLD4o8Pis2ewI0WIJYfFaQuAmCIAR4fFHh8V49bhJBn8Es/gqwDz+CZM0Zpo15CCzwTsZ77wGooRmkBrQeafT5058e34rB5u63R+RV/o8WBxDqtO0JVx+pI7AZgbJ55gut8abGg3zIpT85/A23uvDNbiz6JSrSpHI05jANiDzp6RtmXbSmIkRi/KlWM/H0QPA+xX1gUewPshRIl22/Dk1B/wAA2TBbqsRmJNwVNx9gD8HvBQrDs+DMqxuS3we5UMww86wuBScOuDljyb6uDTpv3QN8YM+g1sKDW6baR1WCdNCGyxvZ0j2PAtHYFny6DUqhUpz+ZzVlOM1ECsVe3KNKN4I808HtE8DwUYW4pRKVb/FHir9uK4ECG1TpJgcQAwN5dZ9Ie+bVoNq+oKVbEOZIq4YBgzhh0Lkl4cOn+HXFWPRrZA5Md/2OZ71WSYdx0HjnT4t8SzjQgMDjc30YYKMDNcNOPd6cIgqFo0Oj5wnzMzZeD2P0KFYFvU7ipfU77Wy8Jsrx8DDADD5FZXPZj928a51OtyvnTYT4Zg8HfgHkV3Qa3/J7xDCLf9KOTTw/Zqm8yGAH+RMPOJn72sOq29PCjxbq5yj6wMgAHvjfWu6xuANGuqwJvD2qzpM92LG5qG891sv/ACLzLxCpT/28qtwU2cYUea9rw5DPQ8uqyfSKfZNp+2X5lHuDRBk3noxvA8AedQ2hyyHwZqN/gdneraVb1Vo/tpcvrM1y+/LVuDh1P+zFVqsOTmGecQ8wXpi27JqT1XkXVf8ALgT47+b+qD3Qa8v8RTiSb1qZ02U9Jj62zPsRKnjaekfnBQjefkn4f6Eeth4akQ4D8zPlAzwDM1KohBZB9kPHUiBAqNbnhTaVE5mQfQGeCGBGcn3DEprjmx88DWymWr9j7qjv0O42T2Z7+xF4UVesy6rYhRJ1cpXLNSujfmolHgVGfK06dBOS73g114K2uSTc9Yr0egVyvsvA/JDB0z2AtFb1nzrM4kUQJ12wKbClHoHUwe8EAPzoKrh1bFq3beXqO8Zb0Y3wzAw69fsBdLuG3r44OSpcRhwHmajG0QdBnMHg7AWg9JPgDatpWRTOIdpX5AnyAMAOPHeDOT84GCxlYs/ipYdtUq47mrLMynvhsjtPZmAIyO+jNVWKbxztedVa5yzQPH+0bwPZ0GvUd8vQbzvyq2J9qjgR9bA6cHQ8C81UelWPfl1SAlmEOXyYPRnQPAwMErJh3jed6VO437/wrtI8BmWYZmeCqBQ8RbPodBvSXEsPmQahGbJvPBsz8ir6O9VarHnW/UtEDDxweZDoNXYcRbnuq45dsXM+AZvZ+sGg6D86asmg3VclbrtNst+G86yz95ZkbDeD5FdqYfnJfMfzjWZLzR7D8ZFUqPg61yj4HHfDNk/Oq+pQJcCfIp05jDQPeCe5xsNJiWBm1n2di51wiTIzkN04ksMD8hq1tWw37t531bWAgSIoAYfOo8/1HJd1PGBWFsTKVR5jrlVbP1fKZNgzaNBfBfN6H6sr53GYVWg+Ay60GBhgvcvC7ivxG9JnhADFxvw/WFBmAYVEAwOYYefyGvCT1HocaltVmjTuZdzwOJnnsVrZ9buCz+Xue1bxegTTPB6OD2zBWPe1sWHBuEnZ1Vr921J1jMHvWL3g7OxeHK3ZlKklcESlVz7pS5j4aTwLqtmel1xbsmjO1xuseuIRvBnBqcbo+gw6FxHiLxRuPiFUrgqVuUPkKVUZnNPaIbwz7P4FSEW3p9NZ4c1NuD9zqufjPPdDweQFip9K/ms6lz0bZ2ZqPzP/AAclyir8PigUZ5xmQDmnmrqHvmNMfnNU7LO5WYajJgx5+tFrujw3588IrDeZmguGlVKj6TkpvAHzwBaV4AgSoj9OAGfB3qnuqpHMadYf9rsw2LnGXk75oMLR0SqsQ5TtDc2A+GAfIs1g5l860B2xVYxRKlEb5wHwzzaQWvLMW9VDpUT7yy+AP+LvwNWf8ntwT2IVRpQMzPXZ5g0B7wPyGq9mTWJLAVGdSsDYMwwP3wLrvox3JUaxNqtDfPZFAHmT6zw8iz2MvcNKg0qVEp1RbkzMIeBg1vOAf+olDvy5qI/EtisuZws8AnfjArCvVKUHEiR6xYjQJesbBnnsksdizkxl+sHLodKzmYPZ4AHQCzMNXTeItx8NLoo9cqL4PPU6TrmABgZsd69J17ivaNyT6FxKg3By1Cfjck9pb5UM+wzXnkAnXUQHKpRnT4QMRZ72jmYGfk/37FCptgfZgKxZ11UCSfJHmEtreAeTP/fvXbw25uX9D3LatvTqJVPDvGTVY80M2Y/yGtgYYrxFZnHW8Z79qTnG5kmPaEkGJMvR6IWeGBr2KzflpVj72xWIzPyPHguyHqzToJYZgD7bgG0fQYHmkBqhIAFLAFHZUjP4KbWdUczT2fwQGohSvksg8GDjax8+juBKNwOhbh4FXvMh3trqZMUdH1utvenY0DlmtDDYtA9DbD3aaNltX2IU/JsYG3oBgaqYENimynW/PvWoNlU9SZ0TN9vrXXFs7hYQN55q29ZaOzNcvqV4VWM6bDAAH0KnC564coz5vrV/OjbsvPtvezTUwM2lzQLznQ2uvNPQ7tlzAPUPBPko7lvUmc3VWmeikzUje8TUzTux72ivGDZGAPNbFHCHh4nkUgGdEdNBvyXjxO1nDFMmaENgpl41lEPdo8nTNU8xlzerB57BRze1gPNdnGxtTnGAD8NxGGuH0IzTobwAF02yOwPGa1O8FdxtQBVJA1AkGHYrhl7aua4dfGl5uYoAmOAhOSGCiPPBn7Rc2FLONJM+/NSNbD2ipIDzfO6as5h5taijAjz3m3g8PrTOfQaj6wGfiJZgcgG2DwW2BbMvbQUgFEAMBR62C5rhpCUAJYfFU51Vxk8NRS4dSbeMw1O9R81tE096aMFIDeKDD4rHAi6KWipoMowZbUrVhs/hqI8y5kr442aiPRsFzikNlxNYfFWrzOY6aicu4ikLD4pYfFTjZUfR3qQyaEFIMEzh8VCyw+KWHxSRoECMECSAwNFn8EIIkB570YIABOgoB4fFAYI0asM4fFDh8VIw+KAwQR8Pijw+Kdw+KWHxUIR8PinQBGnQVrLD4pow3qXh8UBoI+ig0VLQKBFw+KWHxUjRS0UETQ+RNGAZqyw2KK8ygiIwSMEkBI00CIEDyJNI1YdA06Cjh1p4FAdA0YJlPIHQRgmU8CsOgCkMqOAKQz1oJsYFIBDF/wCdSgBQDBnNTY8ZzZ4a+wGQNW7LIeREJDLKeNM9iM1KzRmo5mieTRoAQgkaBBIA0aZBHn8ECz+CBJCqBJGhSNSAQIzTRoAM0BmkZpozQIzTWtvQGaDP4Kg9n8EGfwTSSB0DToJoE6BqQ9n8EWsms/gms/ggkGexM5/BNZGgz+CqEJGfwUd40JmaAzVwsGfwS1kKSIFn8E0Zo8/gmTNAkkKSgI0CAzQ5/BWCSTWfwSz+CgIzQZ/BGaaRIkKIzQqwGHxSME6aE0AIMPijw+KaQEkhRIEkkkgeSTKeA0DoGjA0zn8Es/ggkAaWfwTWfwSz+CADNNZ/BIzTWfwQI0BmkZoEAoEkBmtUEaFEaaQJLP4I8/gln8FISSBGCoJJLP4JZ/BACSSSBJI8/ggw+KIHh8USEES02skfWgSWYWfwRpYfFHh8VphAEeJp3RSwwUBnRc/rX3D4p1JUs1h8UsPincPilh8VqEAIzBJEHWs8M9hw+KLD4o0a0XsGHxSw+Kd2JI9aPP4IwNBh8UCB7P4JKPn8EWfwR5ZYfFNYfFO5/BDn8ERhEkg28GD4Zgj7NNvoSMMyRMs5oYVUm3qVPka77GZmnQC2LJYdrktwIzQBvMzUKpX5blvVT1VVX9F1cs4o3bSpNRCq0p8J7RhyuAH/AJO9ZLw1FV9K7hrR39Nx+ZpfjAzms1WPS34eVupNOWzTpNVdi4YNaPX/AKi8/wBSvCdbcybbNz2yBhUAzZMPIuf1J5+xuY5FwAzPMMDzU3zPXc+IvpRUq8H4/KW4dHkU57WB43gzWZ4ncZv5V7cpjf2fPmKWeeqBguS8LpMGq3eblcYZejgGvvDoNSOIV802BXnYtq00IzXvgwwDNR3+B6F9Hu8Lcu3VpTFpMhXYQGYSHjMzeVfxCufh5fNS9VXVVarTZEJ7ZhG2MmuKcPb84h2BVwuZiKDMd4DA8w6wWzva57HuqQ7UWIMkHZQAb07DYBqN9iregKJwugw6MEq4LmjXVChfsbJs4Az+hc04tMv0GrM3jZb/APN5s/fGtE8GcP8AGq+z+N9NgU6lQaNQzmVMHuSkgfR8hrtfF3jHR7P4bM0a52KaFVqgYaLIfdgUWRbxZdXGaq3JcOvQ9GABhgeiGwz86qoFKg01+XKuPYc3ez9C7GbNj2xdtPqtwSqJJt+qM9eHQ+ayMy57OpV5SG7YoDNYaM8wA+xHrH/ybuT4ceog49y74ZmcHeAAujTOGnDmfpUphyTAmhGz5ho9jxqsk35bEOVIqoZ0Sq4YBHANi55Xrnp1eh4P1F5l1joCP0GpVCvuc8PAOhmy7FPA5eHtlndZt4A1OxTQqU6THCDOnGcf51EkwwjSAbblg8BoJbJxXnY79SfMI4PBngfWHkXe+JFgcErSatS47Huo63zWBz6e8ZmGHzrgIGxpMsNxADTPPZ3rZz6rYHqZr+Yz50GcMw2Ite3VdtnT6k0xLY5ZoPYhHPAABbOpXbTr/olEtW+Ljhs0qLhFhuxz8bP51zRm2/VXD6PdUuJhI5nMA7DBU9Eh06fM5qqy+Tjn1gyCItoOJFh1XhFc1KqsGXzkJ882XTPfn5P41HpXEKnQ68dx1ij+K+GD2j3pPMwTCR6yqNSn0yKyZxgz9iaqguGjVK0gob8EMw9jLDrBBs7brdDmVkKlYFpSZMgAM5MSQGw/nWy4YwIrMoOJNov6NwQph85ST3g8x9Ci+j9xCYjQjseWwATYrOtAPzh86oeFF5sU26Lg8M2axUTfBmO9sZM1Qp78jU6sXlValSqwDzUrOaYGGBsn5FjZgYO4Le3DD4e00JtNvGDJC5WHjz5c9hgfQub62ayWdZhvz6o1BBgzaPrwDM8Fo6xDp2g1SrcbekzQPANmz86sOEt8wbAu+n3BOinJB4+Vwa6wz710Ct2TY958YjpTd1eoY9X8Z56OeYAeCDk8+z6jb0gGLnfOmuuBmyAb8wVZDrZw/DcYB761bcXaJ9lb6m0Nu7TuRqLsCdnmGCx7L2asdgrHFqnXVFj0P1UzRMwwkzg7wVDGe+x78uh2rWY0mPUQzN7RzMFhWXtaUDAdfkXQLGuem2fNOcFKh1XnWTZMDe9iaoYeZJfB32kM8++J0GoIHmSkT9A5siUxs13jPS8iaZ1GXdRtSHcMBzcV9asBusVmEAb2t+t9CrwegmH3rP8AIeC0tvTKNDny2KHrHHNnrdPM0EipG/JuY6Hb4Z4Kwv8AjZuwmNPxWGd6j8JeG79SaC9LxqRxrfB42T5c9+zz+RKmxmAfqst9iZPp4SfbNBmBh51nYxjzL8N0H3AWjt6YxMHmp1cegaHWDQbAVhc8alck1UqHRzepUoMwddBZ+iSQjMTQ0/2pMDSvQ51vVEH5z4SQcA8HgPY8C01pXC/SqlTLxpVG0WooGFS0u8Fz+HUpUmkxKbKfzCDnh+daKz58tm16hygZtRXjPPPrWNwNhMvm2efkQbtg5xH3jfgVNkOszVLwouf1Jdc2pf0eVmzqn0Ak8DE/1ZTaMYcu+Zg8Bh0K1tu2K5Z7FQiQYvORHzA8GT2GG/rUeB0Oz63WXqo7Otidz8cDYOTBAN8kAXQOK/EudSrgduC34utSrvgAelhvZPvz/OuQ2kb9Bu0L1sN9nVp296mGeGYd4L2Hw94XWPedvRLgqtvxtUzM8D6wPr/zrq8Dkno5ep41Uvg5YG9ak2NFCZHANgGfnXTq3wroDNm0yDbkU6k1Ce5qBme/QM/YfQrKzLMfsm9ZdZof7FNA4smP3vYdH6P863YUr1bNdiwWP5vcPMP3PyLoiGNwy9t037PfdKOBs0+a940Ez/YzWlw+KPlnwdPBzBo+sPOj0V0IwQGpAGom/JO4fFHqRnmhN7Akzn8ELwZ717C0gzyUR4E6yeAaaRg2a8FeYbFFMFYGG9MmCJwqjBRzBWDwKOYLeHjn9w0dsHTc5U1l5MYIzWux1rrsyMDzWDizR0SL3sL6Xrczi5oc0OYD3tHFbW8YHqsd/WCvalbEExNwGwD6FlzZ5CVoaZ5r6UcznWFNecZm6Di0rOxY+N1660cOS4bQaixuGsWuNYDR4gqcJjeWnmpsY8/oXHhezr2wVXvPKRPNzS8NxVoPb96RCik74pm31qpeeNXB7P8An2KvkwMxPT2LshkrwkqczvUT1DUTa1wfAwSgG/q6DjZ5ArRhNjMnqngCswZPSRw2fu/iJG83nprmtaO8exV4PHzGBqbJNVUx5tmQrgtass5vg+C0H/D+xn9Cz9KmN6QaiuHnvuuayv8AeuGMqs9yBKwc2K7jRpWQynGDADBNBoSZrXNNgZ5962Wi2bQN9gdC8tKnA/xFFmTA9mCdrepDjnoKi5nMNRIhrtBmPfePaJRphg77RV8+UeqorMwwdXf0ubbotNqTZtbzVqElgwzWFpUkzz+c1expjgGuDm9Z0xzNAy9m7pqwCNms/GnsaoGtLADWjg/2Gvm80OmLPRowZIpNKzDUbUtlnA1NANi5raMVJhmDvs1HOMtbMZz8PBVr0besxScp+7Uc4Gz2a0bMPMvZqUdHA0GHOMfkUfRWun0fRDUbbVO9DcUqVOihMFYHGcTJxksQsN6PD4qRh8UGHxULNIkeHxSw+KAE8CDD4o0gOgjTOexPKAku9JEgHvSSSVoAnUKLP4IsefwRn0IARqAykjS70CBSABNACdBSAMNiivBsU01HeDYqFYYIU6YIEAokKSsECNACNQHQRgmgToIHQRgmgRAgkAjBNAnQViQClx+tRAVhF/50E1lTmVEZBWbLO1BIgZgrpn2QKths4Kyz+CgIzQmkmTUgXlHz+CI00gBJPJYggSSSBAkkkkCSSSQCfQmTTpqOfWgaM1HM9iI+tNGaoCZoEjQIDSz+CDP4JZ/BA8Bp3P4KIBowNWHc/gln8E1n8Es/gsgaWfwTOfwQrU2dz+CjmaWfwQKEDz+CBLP4JZ/BAkKSBWEgNGgNT4A570BmjTOHxVWkkOfwUjD4plFEmkaDD4oDRJpGgWfwTPejQYfFQkjSNGgNWBSSNI0Cz+CWfwQJIDToGmgRIDNLD4pJAaA8/ggSQdCBozQIzNNK0EgRmgNUssPigAEYGkakNGkiQogCSSSoLP4JZ/BLD4oFINBn8EsPijVAEfWlh8UeHxQLD4osPijw+KPD4r0Bh8UkeHxSw+K8QAwRgCLD4p5aYWDD4pACdNI1o8wMNNAaEDSM0Rskkkkekkl1okWWHxS70YJYfFEDSSRI8skaDE08AI9aM2XAQGBq1NlNGys4WpzBBsVm9G2ezUE2cE2gKDD4pHsQZ/Be7ZhRMng6BpZ/BNZ/BSunF+PdvPvTZtZ0M2tHMF5PetWsSbgjsP25J5c3gzOO9mC9+3nTWK3RjivrKnw3gwOSrNGMwdYw8I94foRo8mXnZLlSuGn0qcxWDhGAAzmfQCj8Y+F0HhWFPCDWTnx5ufhPdYdH+uvSvGmzwuenUqnRKidNqAPazJgGa5LVfR1uB6gyKlc1zyZhsBmC5LgcBptNYN0NNvxX+gPOor1NinKdblBJhyGO/sXdWbw4c2ZbwRHwCZIY7DZ3n+dZSm8SLHqUKoUBi1ZNSdqnYfWscHhy+pXPXJ7Xqqo1LmQDo2JQJNV5DkQf+794KXJoJ0eradZgvQ3j6APyKverHqqebfIvPB8nQtodFtXZl5yrAnuymIMaTHlBhJB7r+TBQb2rzF1UnXqVRek4exA+wFnZknN3NR+Wfkh4fQrQpAZDMHHMzwPvNWujKNh2osZ4MAo7zLbJ4KKcyWyem25sWQuKbDbqUA5dS3x/n6wUGeEGMAcpKA2vkBRHpL+lscwWmZpUFmzQnTm85b+9eYIZfRzHPsUczYAw+RaMKbSpNrnO9ZaM2KeyPh1qPTa3R9AAqtAZmB8+zNeibDh03V+0Fv0qpT4gftOzoRxrVrNeZCowX4zIH8+aCBfNVoLU2Dap8hT5vXH6/D+tBZlzwbeddYuOmhUoT2eYZotcXtVbuZqUJuU5zLUVne8zvBZoKkDMo3zYA/rBdburjlas+yws6jWibMc+t3rNciptKcnx5crlZJx4p73sOxBdUevUeM+Dj54R3wMHlEOj0qpR5tSocoGY8LoA+9HaR2OEyQ5cZmDWieieHepFvXDUrYkSzt9tmZEldbLzOYGCDa2xZj981mlS+C0GZ6wpYA9UgkbAZz710DiXwxo/DepUK46lVY0m5X3s3oMcDweY2Lj/AAx4nXNwr4gtVGnMPBzskGJkQww1gPs/jXePTYsas1Kr0e9LcYeehPwAMwAN4fWjned+JdyQbqup2swYj0bMAAwP5FlwPDNG9G5bw5XtUozMuSX3GJrYde/DAEaw6XTeD84KHQr1qTgSaPNMDeMPcgusQPRU4hXPRqg5wvnWrPoTD2szHOfnKk7M8ANYThpclGolGC3LmnHsnhKhsnvjTAM94Gf6F6zk8TuC3o68SaPXG6OZyKjD8an0zeH14IyeFOJ0a3zdaClUeTSpsHNipRJHWBhh/trJGAMtA2Da9UceLStzjZctd4r2PfFKgNTQAzpkuNoyjMA/xmvJ+s48HiI1i2l4ezLRpVzc1elN5ylGGD2zMwVfXplG+0M07RYejU988wzVaHQiDYqWQMuH7PrRaJmWDfWrONJob0A4j8E+dPokJ57Nk2tRsNnkUikN7RLTcbV7avKRqXU53eB4YKnqX3mbrxGDDZvA1bUFmW8BwWG/C780GwjBB+ytQgm+9Gdf8bl2T2Gs5RKlLo/j0apPAD4YGy7vDBWAU2pPVFozfjHIfMGGc3tmZ+dd1e9Db1Pb0uo3HdwQ5uf9Ee1ooGfn8n1ri7Rweq1j1lylNiMYaAYAAHsz70cYKPTbSlnOY/nM3ujPA2VIO2+QakS2Jb08AePWOOz7EFXvU1+sU4KqxBN6OBmBmHWH1rpjmCZpvOQzqRywZw3mZ+TsW4tWNBue3KJSmKr99lHyptdGZ9iqgBiZCpVSNgNKEAReRAMDNjvVZJCo2ZXqVVaHLCSHOBUIccNhgYdhrGx2CBZMW0rjdodco70aqxcPvAb/AOBSLDonIXbHodRrmDVXkgEaWz0Afzpq7eLs66pjt9xKHg9Khgy9EdPoMPyKDbEydVaDCqNDlAGcwAngew4xgfWsQ9W7SueBdtyuRaMbLtLZzkhHDMHvnzXYuA9Yv+w+HgXjOfeepVwvGx9+M84BhvDP5DVnwWolR4ncRLgi+vDjR6jA0KkyZ5m8B9ZgvSH2Sg3taU2x6HXIEaOcblcHj3gYYK/AmwIHri3LfuOmsaOuzzX5zV28Dml4ivbbprAWRQmGG/2WADJ/WCiz4GxfS8Whn9Fv+tM6Ks+TS5dxapVnLuIcTVlooeV+KPMKk0J9CkSWcDUcwXvggzngln8E0Z4Eo/PsZYai38PUg+tIE0ckPOmeZb924jFIeDYq94FOzzFR3g2LJaCahPMhmppqOYLeLEXkGHh03FHeokEBz0AMw86sox4HpoZgLsi3O5JXoEuBMI2/ZKbSqw2cXQNTr2ZwE1kYZ4Avqx/ccSwkzPvupqK2gTHMfbgsy8eZJrNwDV9LzbemecVQgBUUOsPsxwB9wzU1mtt5g5msb4V7XGiZpo2XFIhz4jw55qaybDylanjA+DoNudCuOTpvXhvRmy37ttMzI2cfrMDBZNRnpgCrHgb1fDUuHGlSR03HFHejPw5B6/QvHuEeTGfea8NVUmM5jvbWqhmDzXhqqr0M2VrFouFPGB8AwBaXP7gChQ4ZvMbFbMwH+VFtRZDOTGTM9QOtaWiTM4YMPn4odaj+qnNXT01N9VGyGo2ou2p6fDbnxzDUwWPOA5Dd03wWwB7AN6z9YMHvEBXCeZl6xGA/HBveq+NGPV3gripHotKJD+854B0Lvi3Im0rNl35FZvPOA6o8A22SBs/OrWZynLm4ua1QjwzYZLUlOYAt9RJIHDDT6FzFl45LRrZ2r92a09QzAzXN7MfodMW2YGpQHsVUDysGXtq+PcOyEjRbMVXyWfF8NWYHsTLzO/YscPUcGW08o5nvSA968UkSYzbzCz8mBvV0Zpp4GzFBlJMbd4ahPAtBJjYZqnksuZIK/D4pJ3D4paKkR0keijw+KhaJiaPD4qRh8UtFAyCNPaLf9aDD4oEkkkgJI0gBGagAhSRIDBJJJAk6CaToIFn8EefwQI8/ggBNGexOqOZoIh9aaTxplAPeiQogQJGkCSAwToIARggNOh0JpOggdDrUgFHTzKQJQKXF/wCdRAU6L/zoLCN1irVnoVfGZwEFYRgczQWDXapBmmgDYgM0DufwTRmgz+CaM1IRmkCaM06CB1I0kKAEjSQGaoI0s/gms/gkCkSDNAhRGgEzUQ+tOmo5oGjTSkGG9AYbFQjn1oEaBA0aSSRog7n8Es/go6NFjz+CRmgQGrCM0s/ghNAiB5/BFn8E0jRZZ/BBn8EklGAs/gkgR4fFEAw+KXYjSw+KsNJGnTBAaAUCdSw+KgNYfFBh8VIw+KWHxViOhTphvQYfFAKBOpIk1h8UGHxTyA0DRpJJGiDJoURoUWdRoEeHxQEkjBI0AGo6eUdAjQJJKwjSBGgw+KBHsS60kuhAGHxSw+KNJEGjBAnTSVBrD4pI0WHxXsAUsPinQBGoQDD4owBFh8UavwEgT2JpYfFbrBh8UeHxSSz+CyCSSz+CWfwWoSWfwQZ/BJAkkCNEECMEAAnQBAgBPACIGVIBlGQQZR6Lf9alBGcRcu4jVC0Uei3/AFqWEZxO8r8UEIIzad0VN5NHy3yIyaDD4oMPihz+CWQLldBl7oUCR1qwPoUR7eSvwKw/w/8AiUdTXgUJ4MFuzsBmms/gkaaM15CIJ40CAzTWfwXrdSV6jhUropT7/smD3msVx1qVSeaOnUp8wdBkAPA+sF03NvLUcVLW7epU912qztH5zM1kPL0/h1ZVHh0xypORgkSjzMJx7wWdDijw5sabLC2aOBumGHNtBmC1t/2HUeKl4BQ7Vla1KhBm9Ld6AXL7qsC0aVUprFGqUk5EUM/9dcQxVSr1Vvm4wnVV/DM8DPrwXQ3oDFt281Q5bbMyE+eHNmGBqsjWlWbAo0e9J0TljfPBYyvXzWLnfCJK0Qj9jIAqVtHucKVGrJwaM+ZtB1maVKNvmGmNcAaw3qlk7JTX0IgOKcc9SJm6B9YItLkvRcXWOsM8wNUhngSkHGfBgJXYarzMzNSJbxsG1psZp059Vkw+RclGbQdGaigCkA9gWm2qhYDNwx0zSjQJz3sGMwTsaZTocoOeiG8134KweksaupSmz5c+gDUoQnqPKjBm5h+RSIEOIy61OnYPRz2GHkWqpUCC9A13H3g2b2TVRoxGZDuDHei1ZXggxpWFNfAw8gdiOm3bUqVS5tDY0TjzesDDoNaCpTAepfIsUdk3eyQAb1WUr1NGpsuDWbf1pD57JHeygzQLYM1WDDtynmxVYeqw9rvMhsPAFQvRgk7Kaxv8is6VY06sQJD8qnewPDDPf+hB0jjfxF4WcS7Xolz2y29DuWnmDcwTZwzDsPr+Rd+iekfZlU9GTO4avEnXHyZ0x6nCfjMn0b142k0e3Ibsdw8zAzwMM1bXPw3ftVqJXNcDo9UZzjPB2H5ERcMjUpPOTzNbbgtZL983HNpXrw6VENkwOdog8AH2ZgsPMDA9QFt+DLLFVqlTpX2qmUGW+zmycfoP60HaaOFV4SzZdgXVBh1WFF3szpcbYZ+RVnE62J1mMUq7pb7PqcPGZBrrZM+gANZyxrwqVw3C7wo4jTvWVKfzYZeeDxgP5DWduR5+3pE23KrWTqsKlycApzx5g8HZmiWZk3s3W+Yg1yVzMd882Xu8D+tU8wIIfsjf1/OrC4Z9q1WOHqqgHTXQ78881F9fOcnyPKRjaDo2I8gAA3pfOo5s5tH2GHWCaeMzLTbzzPoAEGYPOg5pmDvQYd6pskM+1V2y9zLGm22YOhvB01n9bxdPTMPrV1QY3rKeER8zAMM0DXOSpkoXJTmZgGCkBMlxs+Vcwz60caNFPnYs4D68AMDVhTbeqUmkvP00M+V6w86yHRfRp4JxONlXuODUazPpoUuHrBIj9GZ9AGut+jfJqUOr37wrnMSaxICM+xDzPMHtHsVP6DFBviZPvC46GxGepTEYItYiH1nsPo/jUg6JOs+5aVdVgQeWrtEPOZrHsmAvOlkm1i2704D8NubODTXnrrn4Q9Znxoxn2H/kWae4J0ej1yt2dLnVIKhNpQVSGGt3++D9a9MVWHB9If1VcF3U71JEtQ+aej5+2M9+ea8o+kDXrxr1/wAu9XD5aF+ywHme9gFccO1RZqg8MaBJap8F+/A1ZoexwwNk/J17FXzOG9DrbpsRawcCoGB8mZvZhmHYfkXtDgnwusOt2/w04knbgMyJr2b0Qw8EzwwzXTY3o5WBArl10edRwmU+oh6wprRgAHGPvADUX6y35+w+F1x29ZrNxvuRqrSnwz5iOfQfeBglQaC5as2jtuROfp7+FXeejnvBj6F7C4P2lYlV4WO2PVXM6fW6lyUkw/ocoH9gGfZmGC6Afo5WGzUZFKpVONlqLDwZdDrZD5DUdI4v6KnDGqssVDi9QKPJreEk2IZx3sMGPI8CxVw8eOMvCK/JcSLasmBpzzmnEkM6x4H2fIu903hXWeAn2cvzhs5MOPNk8rXoMh7Nk8+g/k3rpVYs9y8KzIOo04Am1HQeedDZywB86vDIfCK/2L59WVWnMHGj1uGYSWj640oOsP410OfTd2GCqeHvDenWZcNYOIf3eaYTQj4bGX8MDP8AOthMADdNxbKYeZDwP2ai6K1E+M2eapDjGDqvYhHD/dqCbLi0YRsxTRwA942kDIzNip3pOC2cylMGfs1VSbYYkgbe8PoW0MmUemNvBgqqYDermCunrMcgOm+w+8fyGos+By0cHD7108aGck1Llj080DNSce6HFW1j9qMO9RI0k2V34YN3DmZhpm4pBrGRp74GDmoriHVcywNY8vC2i014N6jmGKlvPAYqI8exY4NkAZkp0xn7rmHWCqWXsHwzVxrZsKy4cvvP1kcgNAM1SzKC5G0tBjDMMz+tdKqUBt4zNZeqw5XY2u/1uZzXDC9DqLEFIkw3+YPNvBRzBwPDcbwX1YuHHg08Cj4mrNmNrHh3ovUk43Q02Os03BhHZB9k81o6PMPPUcQvUpxlrx20yyGiC5rttENRGPPBxS9hh4ioY09sBDU6FfAbZwObBc1toJkPH2dCdkxtYFVM1VzV01omTbeaDTWWFKIIwMn4badBnmdj7akGy4B6eCejM71ntZlmm4dAKczGPDepoBgKQLmvmaYRAZ3708bOY6fYnUYKNrwyNSgSgaNsFmnoFV8U3Hww8i6bJjA8Om4qc6OGrvbXZw8zm5ocy0X5MjRDeavWaC/TYXs97+81sAoMFl3XBgAT0yMD0J3Z0Aunx7LHDD8g+97NtFyx4abikRpmEgwwUsAbNrZ1rbZhXgyDLWzsVnQZ4KPyD5gpFNpTkboBRzftXDcAGxSmTwFV9NM9INTrVgBr49uyEsDRm8o4GgM1zWuDRmZmgA0j60HevFpGfwQmaZR9YKEojxqveDNWxspk4zaKURsoMPirY4e/oTPKfu1IrcPiiw+KkYYJYfFFooAiw2KRoo9HYoETD4ocPipui5/Wly3yIIRgkAKby3yJHG+RBEw2IVN0UHLbUELD4oFIMFHMECSQokCTwJlPAoBZ/BCgz+CWfwQJR3j3qQgeZbMFIhIDBS9FAaoR0k91pGrAJJJJANElh8UYAoDoAjSBGAICBSGQQgnWQ2KxIZVhGDMlEZZVnDZwwUCzZZ2grCMyojPQrOL/AM6kFh8VHNS8/gokk96BozUczRmmTVA8/gnWU0CMDUiQZpnP4ITNBn8EBmaA0BoTNAYdC+IUQIHgSNAkqDRpnD4qQYIcPigZANiE072IMPighGCDD4qWYIMPirQj4fFNYfFSzBMmCCOkpCWHxRaKkncPilh8UQaSTuHxSw+KgR8Pij70jBI0DJoUSFWCRoEYKAeHxRJJKwOHxQGnc/ghUJBh8UkefwTKtQ0BmkhNQEaZSM0JmrBGaaM0jNRzNEnc/gkhSz+CAkKJLD4oGkGHxUjD4ocPigZDrTwIATodasSAQmCJI1Ai96AwTpppWGu9LxPeIkuhAIJGiBI1TzYUCSBS9GkkkqQRpACM0gQLYgNGkvQsPiiBDh8U6HWruEDAEQAjAE6ALMDh8UBgpGHxQYfFWtFNMqwNltRzZWojpJ7RSw+KBlHh8UeJosHDRBoAToMowZUgGUZI4MuKWDLikBGcUpmMgigy4rCNG2KQzGUqNG3qoDQQ/wB2pHJ5+7VgEb5FYRqUBhma9Gc5D5EfLYF7NavkGAHwwUGTDb3+GoaqLEEsQVhyaPlP3aCDh8U0adSw+K5Vo5mms/gpZstoNFv+tUhEe6FAe61anGzBRzjfItItakeUcwNXBw/3ajvRv3a0QrE0YKacbd0KOYIqFecOKftM/wBar69SoL1uVNhts8zjedXBgmkespwopUGj2190DDXzzXB+HVjesuId0MVxjA4T2GC9Jw4zFHF1tjY04eeHkURmAwdUdqoMABvhvw71y3A4bxdogV6LRLEPZ42eC5vWPRdvGj3Dr0ajnJhMdZmfQC9VVu24smqRKrFiYTYp56y2Zm/MPXAOtRhb8/6xw6iVWouvy4J+AzsCOG/Z8iz9btJij2Q7VWIJ6utogHeH1r3BxOh2JQYr1cfqVNgVVgDPPP2x7Ni8qT69WeKlSaty3LcDSYzeeOP3ginBZL23TURd4gQOEFNdkPzoJzKhFPAAZzzXP7zgS7/uaoVy2KU8Efv1dh5goGPhw3JPsOxWb1Ng03S13weN8MwMOxM2rDqtSnu02lROZdNk+/Bb6wLGboMKq1W8YkZ5kAw5fPM0W5+8DYGriq0FyBRo9ZptVjSdQMzj9BgfkVTUpMSZNOVTdkc+gFH1sEEh6pSjwDmzw7wSCZ7v/hUIwcANRID94g0bMyow2vuj+AeROnVW5MU23z3mqqHMcPY/5FEe2HpoJbJmyWcU8Hew1NCZUZNShMBXHjkOHgeAYYKnZk4Ethwcrdv0HiNEqtzP6NPMDAzw7+xBCqVEgszD9cvhPPzgiuSt+sqM1b8F+Sy0B58o7vBdTurhvTYdUqF3cjGk2/VDM2Zee+MfyLhlSeD1kZxXM2uw0DRxnNIG3Fu+EVbt+1Z8uXUWDCabPgyA6AWJ1sxTXNfBBZwJ78y6JFcuOX4pvGeeGe9WtbmWjUv55bYeOawYawH0SQ8izQb+tAbOaIWt5nbL02PKtnY0+Gb0f8E0cO5LZgQHYjluA86+GBvLOSQwdwRxg2IJtBmeqqpHqOnnoHngmqxP9ZVmXO08Nc88PIptNoM6ZTZFYit60eKe8/kVbM5QHc2O9AmTcNTaVJcjVJp8OvPBV8Y8FLgAZyNfsYNFrgHmzm6ep4r72Zh5F1Dgtbc65LgqtGptfjQJoM6zLTwftOG/AFyE3sJUhwG9+eYGvRFsUG0q3wCpl406naN9USseDOA8NZjPeB/wKR2b0aTvjhvxQOqzqNJ9T3CzyVY7AA+x/wDx713KZwN+3lzROaYhxqZS3jwMDzMwzzR8HKw/c91wotRtw6DNCm/fI5mBgYYbDBdzgQHIA+zTwMizwiox1esRajgdHq7IMGyHyAvDvELhFBktXxUqNOks0q2ngiwwPeBnn0L37eZ3A9Qzi0B/Rdf2G93gCz9E4aWxDsabYkuLm1N/aTPrM118Yy/o9m/P4O0KgVWIDNQs14IXg9HfgYfkXY69cLFBoNQup9gHuSDPDvXLOCFKqNBgXLRqq+ZnFqpshn5AXQKxSm69FaYMM2gPMwPoNRA8NcS+GN42ZYtM4r2+/UmYVUmG/VWmjwBl8H/BPDvzX6BUR5isW/RLgY3nUabFeM/nMANYrijZ/rvhPNs6K/ox9j2eGfQea3Ftvf8AJqlf/Rsf4FHNYmmyxJivQZzAPR3wwNo+9KGy3AhtQWM9JkMGc+vBDn8E6zvWSLtIZ2HqJp6YeemnQ2Co728kWde3iq95nNSAAzBHo7FQjss4CgeZb0kj2EkB5iiFVMA96ah6gH4isJLOaiaLn9avaR1JliTH6Fj6xbDkx3rw0/kW1Blw0jjJtTi9V4aVyZKOWw+yYfWqKsWTWaa0DmGa7w9DcBMhGYktG2+2u/x7LJxqBaVV5IDOKqyqxn6aGoYYLtcyAxDHZ0LBVumsVKQ6w4tvp7GeGJh14zLTM1cMvZlpqnmW85GlbG1I5nlsNTrXQzXcmG2A5gkB7dNHDmMPNApGi37tc9tKRDDJRzjB5FZ6Lf8AWlohgo2YUkmj06YGBsB9aqjtKnBmxoGYdi1GDYJYgtvHNZhh5NsMQ3TBhvA/Oq956czK5R9zMMO9dCksg8CpalTWDAPDXR9NselUhDfnx9R/rUeTAbjBg5sV2zqAOnpmCB4G3gwMF73Iwy7MPWPTVgyDjLRxHOhWAQ2wPw0cllszz007jCnZhuG7qYK7gGbJg2mWQ36anAym1LDNt4EQRmwPPT61W8zh0Kyhycx8Rc1t4SMPihTubZoFzW0NIw2IEalYk0bOZJ1CvQGHxToMt46aWHxTwKe60M5WKOwErNiLhn86iMwwZNauSzrAo/IR/Iuzx7LDDOSWXAdDw1Zw2QNoFN5ZjvbTwMgHs0vm2vAmQwaDTUhCCJc7aCz+CRmkZpo1zkDTPejSNFgM0s/gmjQZ/BSJGfwRgCaz+CdZXgWij0QwTqDD4oK2TA3H4ag8r8VpuWcNMnA+RTYp2YDn4aPlP3a0caBt9miepTnXgoGd5b5EuW+RXB01zyIgh/u0WojjOJaKu3o37tR+U/doKk2d6jvArg4ziaOH+7QUhsqOYK+OG2oLzO5QK0wSw+KmnG29Ca0UEQEj2J3DBNPIAz+CFJEAIEnksPiiQMH0KIamvdCiGCAUuxFh8UsPirA4bEWHxR4fFHh8UAIwBO6KdBlQGgBSABGDKlBG+RA0yzuU1lnZ7NPRoe/2aso0P5FYixo2/oVqyzgKNmHh7tSAZUBMgprPQogB0KWCBGo5p01HNBHNCkaBSHUs8EOfwSNUFn8Es/gmc/gln8FIez+CBBn8Es/ggeAFIANiAFIBUGjBIARGhz+CAE0adQGiDSSM0GG9FgSw+Kew+KLD4qxH0V95dtP4fFLD4oK82cPdocPirLRzSOMoFbh8UGi5/WrDlfig0UQhYfFApCaw3oGjBMqWmjBWI+HxQYfFS9FBoqBHNIFLCMkcb5FYZSSxNIEApJ3D4pYfFA0mTT2HxQGCBrP4JkzUgwTRgiTSZM1Iw+KjmCKNJYfFO4fFLD4q0o6NHiaBBIBSNHYmmQ2KUAbEEUwQYfFSDBNGCgR+9EkYJArEjD4pdiXYkoDR9CaTqA1aDWHxQI0CLCkiw+KWHxVINJJ1NIEjw+KWHxSXoLsSw+KMA3o8PigZw+KPD4p3Rc/rTwM/IqEcAUgGU6EZSAZXsBrRRAy4pfLuKQDOxIgV/LuI9HBTTDFNGGS0EQwUcwUs1HMN6IR9FfNFz+tSwBGEZwzRkiAy4ngZ/EU0IbidCG4ghBGUtmHv6FKCH+7UtmMraogQ9+ppqWEZSNFS40bNQyRAZP8ADU2HGfM/Z4Kcyy2CkACB0NMAT2s3/UoyEwQStZNHvTWfwRAe9Gmy0V90W/60/ouf1r6bLgIbZlGokaSwbWGuBm31gpALlejw+KWinWWTc6AzUjljDrMA/Oi0LRToQ1LCN39idBlBSSab3qvOAtmcbMFXyYGCbGSeh71XvQ961DzIZqOcBs/dq4vAyhsqO8zg0bnkBag6U2fu1HOlMAB6jeYYdCvY41c9VpVEpDtx3dONmOHQ0z1vGsfY3H6q3hVDg0qzsKOweBy5Z4YB50uOtq3BecWFBnTmYzUqZ92jx+vDzrCcZqaxwrsin2BQ/wBtmhnMPvPNRY3Fz+l7w2t6su0cKHPqWh1vRz2ZrCQL24y8QqzHnWxdr1HpVUPBkMw2B37Fn5no606NwvplwXAEkKhUTAPBDPYfeuwW3ZP2VuO0qbLb+6MMh4veo7hzK9rPsC2LhkQbucqVbqr7PXIezAz+jsXRfRF4b02m06oXw43m7Uc4rPyBvVfflpBcnEisVxiLreqwDOP5wWz4e3zatmU2XSqO+cmn/tvRgEMz7FG1uOXbZlA4LcbvXFZlyfU9R+9QzwzwM+z+NcnueAxPveu3BRqlydHzMwezWu408Qp3Ga744VJj1PChM/duYDYZh864PJny2XTBuWeHYYIpIoMyXQa3IqUF/DZgBp2tya5Un/WM64DkhK8hqvo8CuVuQbEFjWw61H5Co8w6w2xvDrRZ3lnwjm45gAAo/ie7TWZgeD/u+xSAMMFABkDNI9h6akQ3tF8HFHkg2D5uNt4ApWWRqwgBzjR6m8wUJkGzLBw8PnToajL+ww3qsIKYzo+IClwDpsYolVnMBMBg8ziH3oNZgD8fo71CeZ0T02/Zdn0Kx027eM0u86C1bgUNmmwmMMNI94AsFyzDzuDj+HzqvZN8y9WxYhvS3+gAWtuSwJVq0aFVZ1SZ1Zv9EM/GBBlTBwHTDsQZ/BXtvUR+tyHg6GgDMzR0e1fXdb9VAeGwzzUCvpR07Ql8829q/wBGMFYHDY5JqWw5sf7PIo9ShhAlHFb6A2KIy9g6pFhMtJw4R1gH88Pc4b1Fh0pt6l+sugO9XVKudxnDDRPDseDYo92yazAkR6PKiRo0J8MwaZDDBUgvtO/9l/s/EfMHe8zDrBZowVrVQpUZ8PVtR5kDZ8mGChYcyYabAAeCCPg57tWFNPPMFE8Rnw9MwNWbIOQz1POCLOmGwPrwXsv0S6byx13g1c7dNqVKqMD1pTWneszw34B+QF46gRn6lqgGGDAZmBngr2z7/vG0rtoV606pTHpFvHmGj16HeCD236OvGmsUriXb9j322cblZL9P56R0MgeeAGvcUwNF021+aVn3DbnHv0lGpzFDnnb9wnomboYGErDr/WC968MbJv8AtKlnBuq+JNVDeANSAzPDsPNBpj6dNRD2KwME1oq2TPnGYZlO1FhvRdf9th3qUzPMAT0yHmKz8l6VGPT089+C2ixo+cbkg7Fcc2PhgasoB6MVqK30MBgCzlNh4Hrg5mHzq+i/86jmjaotaxf+dSwNwFEZPBWAaZrEHn8Ez75SMME0fWpWlgkajg8iN7YghTA3poARSTzA/OmmQPHegkACBGCPD4oI6A1LxBR3gQwjnpmqx7wXQcBTTeBl3TTT2g8BrVCFUg1oprAzw5Oaea3Dz23Bc/vmBOkk1Kiv4YHma7OFkhVUNZrXi7z8iyk97xd+w1pobL4Me0VJcMNtn7242uyLZ2VNPoPNaAJIH0OLDxqkxzGm24riHUm8utdNwx21Ab0eHxUFmY3pZp4Jn7xcGG2wGCAwwFGZoHj2L16az+Civda+azgHqKLrZyN7i1iBN5MDHxFFegYErKMGbQJ02c1F2KI4BmfyIDgHh4ivtFv+tLRbMVHcMuDJge9Og8rh6MHeo71KbwzBdMCpD2viKwZ6E0HjfdX2N4dBprRcjPnv2KmCwZmYO6bilgeSrD0MdT3qUaS4DuDi58NotagCQAhA06sbhtsBoQRGhUvRZ/BHn8E0kslJAGlh8UyBp7P4LVFwZMEaSAFnsweRJpGvHokeHxQIwUPLLEEtFOghNe+F+EEwwJNaJ5p57rRAt2RrD4p0EBnimgebzXmFrMOhFg5khZPYnQ61mHWQUsGc0mWdqlxg3rK1HYcPxQ8NWfKfu0UNnM8FfRqVm0HesRmZNKzjn4apXo2iWmupHTQZinqN9ixs+m+KfhoM4bIYqI8G9XD0bRzbUXk3DdVrVnLuJrRV0cP92o5w3FApZLOCgmzmS0Zw8x8RV8mBgaCkeDBRTVrJjOYmq82UEJ5RzDJTTZTXK/FBBw+KeAFKCMnuV+KCIAJYfFStFz+tNGy4gimG9LRb/rUjD4pYbEETRRgynQBOgyrEflfij5dtTdFz+tIGTyUCLop0GVLCMnQjfIgiAy4raBDzMEEaNv6FdwI2BggKNAwL2asGYzYI8QTqATZbTJgpZpo1IZAEefwQIUBGorylGo73QqEVJLD4osPigaSz+CkGAKOakAaaM06YJo1QWfwTzPWo2HxUhkN6CcynTQMpGgaM0CRpo0Qdz+CRmhRADhos1h8U6DLikMxm8VI0XP61aETRc/rR4fFSNFz+tLl3EWhYfFLD4qRh8UGHxRBAG9SMPimgDepeGxFohhsUQwUt5NIhE5b5EBgrAATTzKCsMEgBSzZQAygHRS5X4qwCMnQh/u1Ar9FI2VZ8t8iA43yK0qQ4yj8s5n7NaDk00cZtFKwI3yJHG+RW2imTZQVvJpk2QBWZgmjAEFZo5imjZwVgYYqIYIIRgmTBTsPigMFaULD4osPipGilooIpglh8VK0UtFAbIbE7h8UkeHxQRzBRzBWBgmjBEK/D4pYfFSjBNYfFUs0iw+Ke0UWHxRCKYJkwUswUcwUrAhw+Kdw3o9FW82Zw+KEwT+i5/WmjBeJRzQqRh8UQMtr0NYfFHoqUDLaeBnNbiIDLikBGM1YBDbToRkFeEZPAy4pvK/FGALPAiAynQZwUgATwMuGriAzh8UsPipfLuI+V+K9QgmGxNYfFTTZwTRgghaKXK/FSsPijw+KCIDKkAyngBEAK2RAyngZzNAAKWyHvF5gEzD/59NSwjNoNZGDzi9DvLAAow001n8EQIJYafenc2/dqFkaQZ5+G2aNVtGjZqV6t/eAoUY3wMFZBqGoEV6Ngo/hgfiK6CMB+06Fla9WGIcrQiRNYO8z2LPY0TJtmKGeGjH11h5/Fq0rYpv35iZMkfgtAuP3h6SFz1Wa0xR6UdNiBn93M96d8CPTeIVKjYBSqlSplWmvdHOBmH0LtFKgVnkwclsL8pzqWfiNwZMA+wwPeC61wi438e6VPClWlcZ1WJ/Z6m9msVvZdyXJflw147csqqxobQdYYYGBqXSuBXj83dV41WfN78JmC4DG9IqzrwqnqriNRp9t3KBmB1Ck54H9a3dscdaNRKoFs1+pViTn+zSHnt+CDsD1t3baUc37OqIVKP3wZx7/yGrCz7/t+85B0rUOm1hg8HqdLDAwP6+9UlH4ivzyP1BEOq4djPWo94U2h3sEeXUbZrFHqsU82ahHDfn86gdQMAZ9ukDOsC4PxR4u3/YdpAxXLK9fR3zw9YNB0Lt3DqvUq9rKpVwUOWEnXZ8bA88D8iCDVaaYHqA2qcDxXSHqaxJa0zBZSsW24yWpFbQZ03mATTxtmJtgpp0SV5FCOmyg621YzkO26HDkc9yIHI85rlV1cOvthXpFZnbMzwAMM8AXbXmXAVYcNUMBflt/8ho8GC3shGB/oTMymv3JCp8qIGwAAwNbOsQH5lLlxGw6wwUSHGfptJjtuN4GAYKbgcXvmvBaVpV2svuaOu9oZrhlYvO2J82JQ6PVakEc2fG5QO/516Wvzhv8AbCl+pz9kb2Zrm9Vtu2OGlxmxTqV99OH4MeOG8zUYW5j6QkaDWIFr+o2wPMDDYGGfQuH1u3vVubc58Izodh9631w35Uq3XnX78cOm+r3sIwAGAZrD8Qrh+09XdnMdB9CKVNBeqoOyG6NL0TMN/wA6j5vhvcMwd8+a0BxrfepcR+juHGqYe2z71Sz4cqBI0JTeBrJaJ1owRGG3oSxNUDDYYOeRFM8Yzcb6DUU06HTp9iBo0YAasGaPK5gAfimDR+dSDZzd5Rh8AZA+jrzVivmaGLWHX0GCRg4yWm55M1oqCzaUOrhLuqCcmJhhgHYq+5PUb1Zd+zhvcl2A8gOxpNApt4QqrcZyQjxTzA2ew1qOLt5sX5cLUuKcZ6PF9i8yzgZ/WsF6tqR7wYzBIAcZ9og1tHmPwKHLNjvDBBatSfolx0+X5zwP6FLoMNiZwxqdc183YT2gezrVS8z6ypNMlNvgzI69/fgshc1KHBCvSH5zGtHzzMFjKqcF6qPOU1swj9gGG9aU57dVpB02IedbPoa7z+hZrr9v+0d4H1oIknMIRrp3EL+TmZwttGpW/WNa4DAGJjPWbPnBc5NnME9A+5mD+gB/WCoR4zPQbit6b/pKJqN+Frb0MmTTTd8MDDZ5Fa2lGcmO+sQYPkg63j2YIFW4b5z8wYzDvNV+HxW2/m2ZWQbiTmZLWYA8AeTzqwrFNtKlNVhh+lQzAzA4boBmYGshU0G27Zn2zInP8zGmsYbA2BmaUyj3dw0d06jE5Z2qQ84z3WBrQcK2a/PqkJu1YnrKQ+ejJjmGx4PItRxsZ+2FUCnMMSYcigxgCTEkBvZNVFjv3oZ8BKrZnFC2r4rlSjSfWMB83o7QbIz59B4L3vJ6w+hecPRpoNwWSdMtl9z1lHOiRahAkSA3sh3sLvrNYbmADnRsVg3gUTD4qaYJowQV7wNqFJgNvKweUfsVwjaiCTybp+JgCmxqx0P6mxBPgAYmar/2b2/QtUtnDmA8Go2rIJOC5/AefjeOwfWrNmqyzLTccWVwpugkgaF481RRpmxWQPZqFj70B9aHP4IANSg6ngUQzR6ypaUjULWwJE9PbjNZm5371IkJGGxLMAa1zc2I2XgeHwzzQU8+HgGoCqntRaiSzmKpHo3jmq2KzRzFUtSjMPRzbcb71quUe/DVfUqU5yp7FtFouHP6kGi1mHYsvUqkw80bE4FvnqbrDp4Kqn2TzjWnp4fOunh5mNw5JMjcs7rsHsNBDmTuYDZs860r1mVg5ANy6bsD51cBSmGQ8eCAGvsfTDjx5UsmqmEUPEQQ634u81d3DbDjMDOKwBhn2Gq+TavgA+xsUbhSwjT2D94pWeaz/IPxoGu45go8OqmyGmbijpe7Xck+xU8zU1wcbcwVgyYSfZ9aiTA953q8G2ggHhHBS8/gs5AknkG9XYPb1x80K8JHWlh8U7GADUg2W1k6PClnmYBqKOzUm3mgBWFSZ8A1jJ4PsmZsLu4WdtBJDR8fMM/kUeSbZxTcb6w3rNHWzDYakQKwefnZw3rbpY9xU2sBMdNs+w1e6OZ5h0LHyYBw5oTohgccwz2LQUef91AH06UbXbJ7E7rb1C1tnhoNbeuO4bQs8/gkogPOJ0DXNcNos9h8UCNFh8VjhsFIEWHxSw+K8SNJJJFARoEaAwRAaQIwDYoSMEkkSsQnutAyeCkPAoTz2ifiLSIQOV/zKPGZM3ehSPDPBWsCNs8NbiFnhs01YRgzUj1VmeorCNTQAA2LC1kEZ/H2akMs7lZaIGwkEZvNc1tVhABvMFqKbhpAs1DWgpp7QWIsJgeAqSZAza1MFfG82bSjmH4ikY+TR8z1NNRDpuj2LZmDCqZhsB4emqFEcPZ0KKcPD3atwUeSGxBnXmcCUR4Nis5J71UyT3oIT0YFWSY29Wxmo5ggoTjOIOXcVsbKaw3qxXmy4vmi5/WrHRQGCCFiaRgpGJpGzsQQcPihw+KlGy4hBlxAyDLakAz+GpARk8EZxBH0UQMuKcEZxEDKgRwZcUgIykMs5p7RwJAUaHs9mpwM6KBkMFITYWfwRgaBBngpBmaaM0jNMmaoGaQGmjNIFYkdZpGCIAR9CCJh8UOeCdM1HMN6ADNNGjQYfFQATR9adw+KRgrLRw606HWvqIELTg6EjUdEgM1HUg00gAFNZDYogdanM+dQhNZZ2qczDzBR4zwKaD3Q2CsLkPkUR6NvV94ekq+R1otWHG+RNHGVgZpkzRCLo4JIutLD4oITwb01h8VNw+KWigjh0I8M07ouf1qXGh5oKw4x+RIIza0D1N254KPyn7tQI8aNv6FINlTYcbf4gKQ9G2ezQU5spo2W1ZnGSOGrSp8Pig0Vccp+7Qcp+7RSnNlxNGy4rs4f7tRzjNoKQ2XFHMFavAoTzJoK81Ew+KsDBNGCJV5ggw+KkaLn9aWHxVhrD4pYfFSMPilh8UETD4o8PincPijw+KpBoATuHxSw+KRgizSA06aaNSg0YJACdSVLNYfFFh8U9h8UWHxRCOYBgopgppgmtFehkAR4fFOgy5ingZ/EW4iGGxNaKs9Fv+tBy3yLIVnLfInQjKw5b5EYRvkWqEIIzilMxjxVgzG2ezRhGRZoGdikBGToMuJ0ARBrlP3aaOHv6FZYfFLD4oK8Ie5S+Ww92ncPijM8UEQ0IAalsg28alcswHu0FO8yaj4fFXbwBn7NROWDyIK/D4pYfFTTZAFHPYrZPui3/Wg8MDQHJVe885qroiE7WebeSd5htU/ObUHMOK8I2vtZFrKqZmOACM5ijBtZBJTwSd/iKk5wA9madZkuPGmDbVwDgm6Go4r0I0E/eAH1rBHJcjBqaaan3a+zFdM9gACxtcW3Rz7fZlaHPZuh1gAZ4K+hw4skPAcXk+TxsfgStCDBMwb7DBa2j8YKxP8ADbfON9YLg7nTt6Qk0dtmKctx8AAOvM15y42cS6Hbco4MF/Wz6z6PIs/eHGCuVh12jxK+czozCOC5Zc7NKeDmqi3rTfIuDm5jaFUuItSqUg3231EO56N1zgzkd+9YqTUn3pR6jYAHYAAnWWWJO9xveuN65lY0D7f1Q6NWZZ8uAZs4dhp2xrVYevSXSnKjJZhQuiRHPA1s7/4S/wAmk2lP2XO5moPh40frMzW9tvhLEOz485+h6NYf3zHnXjAPrX1Wrn/B/h7BvC+qhLrhnMp8UPbPdYH51ir2oN1XDxDm0O3H6lXuVPCNon0AvSTNknZ8M6HZVRjSXa2GZg8fQa3fBDg5SuFFIm3HceDNVMMzMN6uLYOJWlP4o8AWqfXLjYkhCmmAHv3hv8gL0xXvSioFE4by7ttw+fmsMnhHkBhgvNlV4r2jcnEOoVziHLeZpLB6EMzDYfyYJqq8VOENekeqrcowST7N+jmrUVmemlXIHrNviNRnq2co/BaawwZDyYKpjekIdEny7n4c3jPtKoSjzOnsxvBeWX4u3JYcM4T9smyzUIobwAOg1h6JdtDmXA1WeKFHk1Kjh1myoH6EcEOMfHe87SiXG/dVmViO/wD2vBk/zr0tAqTEyLE+0cEIEiUGYaR+Dn5F+S9wz+ENYdA+FbFw03vmR2czAwXdfRj9KWy2YR8PbgrhszWNkN2vST0TD5M88EHvWpUenQ2ubnADLXzmsVMvDhkfs6zrf3QZrg9/8XabbzA1W7Zz0alSv7DM5yFJD8nQtxw69KX0czj0+C/OoLITffdZsmfnBBsJJ0OpBnRnzea8+BgqeZA8XBdtk0GJcNOjzrfrkMGnwzZkRDAwMFyC+aVxYt6jTarKrlsMwoQZnLeZP/ArHEeKlVnUevcjYFx/zwe+TEPezhsWUe9JCnWrUmqVxUpTLJv/APSFOMDDP51x3ijxyvWqhNpVSCjg6B4c3T+g1yG5LJvgKbEuqpMZ0p/+kZ9CD3XJ45cDzaB9u8Wcz6MGTM1w/iXxmo9q3CdZAI1ekTWfBNk+j+BcCk1h+lUuPSgbjbwz1WetZkz1pTTAdb54Aslhva8PtncMi4H4PJuyutkA2ZqhhxtaQDbi30mm/Zto4NzUZmS7O8dmQB5hgs/6nYhymtB/PP51UWpH5Ng2jcb62DTNYmP1UWfADZ3+dWZm4EM4jj7IYb1TgcvlevZmslkyAaRt96Bk2N7b6eZ0A3uMPbOs1ZSWYMxppumsdAefrVCi5N94vAbzSmU2dTQByW3hn0LW2Har9blHLl5hHi55gB9aar1pOPSo9Vi1H7lOeBgNX3KCPR/srMofN1yqzAlgeAMgfWqSTyrLucRw8D6DVxfNnuWTKaYcqQSdcM8wDBZ0AckteIexWLCpG2YwjPZmyoJhg6np72cNpj8BQQM1kJfXs1zD6EDxmeee808DOfiB1pPM6Jaj7eCrYn29Va5Gtqp2qwwycKV4557+hXdjQKxfkimW5TXGY2gZnno7AD51d8PeHtYud2FEo1ZZjet2TzB1nZ+teq6J/J5ZNmxKHb78OZdFIzY2M4Hn86WyZnghTbK4XQ63VbxpVKqVbpb2vDkRI2ckwP5F5y48HbE+7ftbbjE+MFbzlSYksMNE8+xe2P5N7Vs/h5IuO6rcZrc2Vm+8eGBhn2AvNl4ejrxXva2ZF8PzoD0Slsv6NPA8zNgPIpVFuS2rAg0qY0/f9HkxqVUWfucjzms0ZsBINthzWaA9h4dYedaq+eIv2ttW37fcpvLPUjrP5wDBVlqnSo1w0qdWW84QPb96pa44dRn63VJEFyKb0cGe9nYH51cs0e7r2oP2ftzCe7Ce3xIgbwDzmt/Srzty3n6rc1OghVXTPCTn4IRmPk2KtoN5/Ym+ftjRnDChXLsPMMAAP9zQUljWHUaJaEfiFLixpMKVMOkGz/Soz6kVW0qjyE1uUwB1CV40bA9hgut2lUqVJ4bTYOmybQXUFQDMM8A3qDf9BtGBfx3HasGfW2ppgbNPgxjOK8Z9eBoO18KPRgo9gcAneKFOqRzLlrdNB+MHZmZ7ADfsWKsY6qF/3Rc/E2hxgj1SN6iqUfDM2X8OvYthwNvC5+IXD4+GVRo9Sk2lSKkxzM4Nh01jrNg/oXqixuAPDyz6kb9uVKfPjzQzMJGBhmgtbPsym0SbT7ni/wDwRiEGAbOgN4KivClBRLhiVVhz7lUXugNgMn5P41l+FHGCLPvC4OHs5vlvV098KbswzAOsF0W8KV9rbcBiC/gASQeA8O8EF6EkJkdqWx0PBmorxooe+A12bE0YKRHeNNZ/BG91qNn8FUIojVbMgBJDDPBWGfwSNXFr8IkYAhxdNzsVVGqWtVNMOhWFbZfepchhjvBYm2KbVaVmxOPf1raGTobM/RdBs3FaszNiwsmS+fh54H51e0qS5yoZuZmouFNKElGBqsA1IB78RYiwz+CSjo8/gpWiT5ObWg37U+hc6DiFUYFchWi/ECZWJTOvpHsXSHofM4ONngYdDodYLz5f/Cu3GazIvS8bjmBNp0M34cvWwPZvAwP86DrHELiFbNqsR37g1j6MAAMzwNS7bvmDeFkVCuW5zMaPgYM8wzga8lM8eKBdvqyVfb9VObCZ0WajEDrY7M/OsO9flYmXAEWJWD8CTyrM4JJgDzHZrAsO509L9EKDclHmWlHuN+sMhBfDMDPZmrgIebTT/nDNeX6bxUg3Pw0esCh2iYeq4wBzeHgmHnBdwsniX67p1Kbl02TGzjBve6DPvw/1Ni9czoDMNjDoQSYDBhp4I6bPYqTRmxniCkMmEkdRhwDBbDOHb27oQBb37taY2UgZQYGfb3WH/AsfWLewzbwXVp7O43Fl6kzm6a6+Mc0rHPQIGhFDP5FCpTzkmL4nX5FsqlTQeA3NNY+MybM91twF0xbmwi1umvnDPTbzNZGZAfZaM8F0h4M2lTyYzZkTemuzh5sMbhhYdScA8HFexsJIZqqu2iToDXrWKxm0HWqylXD4QAuz/I5v2NUyzgR7MFZxt6qIdSzEN+YK1jHgGYObFjcNotasvNsjvSOf+G4qd6Yq+ZUm2R8NxY9Lba9mTGDjnvWan9yrzrxqJPrDjzS6eHhwxvmRaqETA3M96r4cl+MZ6B9YYIHnnHvaJrD4r6LkWUOe/paGpsV9TTY0Dbc/Isoy9gr2BVWI2zTAz+dZXCotoYcls/AXx48HTVPGe0ZBvn0ZqwOTrZrjuG0WsGTzwUsDWaCS+zI09TYrqG9rNaixvhbRaXn8FNZ6FXnqKQyexc1wuEtJIEeHxXO2gsPikkgM1GFkjBMgnu9WyOo0gSNZ4aiBAfWkHWvp71oyMvG4oUmG5M9n1qwMNiUbDNbIRI0B8HR1G1o4DOCiACkBJbZIG0F3GZzNWARgwVDDnmB9Gavo0wHhz6FxW2O4YClh8U0bydDp1FDVIZPBW0OS2AgqRSGTwWNwNGEltI5IYKkB5xGb2xSHZM8PZ6ir5MnWNR5HWomsYILMEB6ZgoXM/OkEncqEKfG37FUyY37taN7TNVkllSKflfimtFv+tWZhiohgqFUbO9BoqwMFHMFIjgzv3oDjNqXh8UlQrzZQaLn9as9Fv+tBooK3RRaKsOV+KWigigCkACQApAAgDD4oMPipCFAQbE8CZBJBIBPayiZ/BAZoJefwQZ/BRwM0ZmgMzQZ/BNGaWfwVgsM1LZDAAUdkDMgVgAIDw+KBO4bE0aCOaaPUTuGaPD4oIOHxQmpWHxTRgrQaTOHxUjD4oVADD4oADenksPigLP4JJYOGkAKFlh8UeHxToMtozBWIgKQBqOaeANiISgecTwPOKCAKQAb0Ws2XnzP2isAjZio8CG5scVmB4ohEOAo5sq1ePwlFw+KCpMMCTRqwmA37tQsPigZ0XP60YMqQAI8PigAGVa0plvMFHZZPBWEMMFktdhDbw9mqyTA8UzVtGe2+IlMebAfDBBTsxv3ad5PNJk/FVxGZzwVIU/Io/VvxWi5D5E0cD5FYzRw/3aZOGr56GYe7VY8G9QKp5nBQZIOK4eBVskFYqTBRXlYPBvUd4NitKpeBRTBWBgaZNnYgrzBCpRgaaw+KAABGAIwBSABBBNlGAKdy7aA2VSEFCakGG9M4fFFmjBNKVh8U0YIg1h8UsPiiSQJGgRoFh8UgBGAJ0AyQLBvyJAy2ncPijAFYaCMnQhqWyyn+XcQVps/ho2YbhmrIIGas41K2AvbsUnLYJ4Ia0QUf5EfqrD3ajYz/AC7iMI3yK4OA5n7NLlP3abFSEZtSAh/u1OCG5n7NSAh/u028wrOQ/DbUJ6G/n4gLVMxmwBNSYef/AAezVxb1mgAGU6yesWmG9TTo773Yp0Oict7tXsVhw9nQo5xsPaK4km3GVU9JA96uGdoMlvAFTyXMDVxJktmGm23vUUIbZ+I4G9bM1SBuGKHls1ccsz5EBgAdDYLaGVwpzh4INgKXK/5lXn+J/wAa1YntZv8AqTJvKOZpozWmDaXrN/1KfGqrbIg3qKizzFRHgcMcMz/ImDboEaZTnvaI7kpsE6I74a5kzMrDJ+AZmsjxCuG/zhOsP6z0TsAAXzfYbQyVVjPnXjNh+NJ+RWsmt1iHFPXlw/yAuZMxm2ZGvLN7P60b0ymz3eROcbIH1mZr4nscrviCqtbqPNOmxOMPoVJJuSXlg+/vVnJo4BKCDTnzkun0ZrQQLApoUuRUrjpWchgM8M1wWYU/D20mL2qjoVGVog2Gf1rcXVYduW9TdelA8EjsM3s1h6VJinNByC3y2HRpbFBue55TP3U5by18LYqGEuqvgdGrMxmoe+nSz2ACCyWa5cPEGJQ51Yr1eo4Z85yjxhmsPdvEis3zczOAQ6VCAMMw611W0uNkHhdaR0OnBDny9+E4Pn/xr6DJ6bOBwT4UUaJdVVo50rDPADMzMFwLiv6b0Stg7RrSoDJxD6Hnt+f6F56vni1et/yNS4KjrfkWcjcp1y4msaQ1dVP0jb1rdNdtkLcoLMGUGB5s71VUe24NbmhF5uNGqB98g8AXOpLzYeIDaCfArgQgneqpPL9h4Kx1PiLZh2BHjvzp0CpNP/hdYLL2k82ZG/EnGEfvju9BgsCfPPe3cmB8hma0FEqXIRzY5QHvrRLo0m8LRtikA5Z0uT67M8Dj4ZgC5uBv3VVA13I0YzDrwTLwPz5Wgw3m6fQCs6xYc6iUmPWZ0s2XXzw5fvBBe2rMo1n142L/AKbUqlSthsssyTAAP6M089XuF8+7eblMTIFKfMAzZDMwXPAMDDDxj/jVnbFNYrdwQqU/OCHmeYPPdCD1FROJznDG1wuP0euLdYpTTHtqdUXswe/Iaj/+knxf9IF2p0q57gr0xrljM49GjZ5/oXPa3RJ14Pu0N+XGk1CKzhGejs7D+RdN9FGw+LfD2bLvGhzqay7FeOFW7enBhKkh5wR52vPl4BVaI+dOnUqTDa9zzAb12u27YuC/+Dsdz+UaiU2EDO9kw3hv+tXfpP01vjlWeesSDPpUinBnPZqEbRzf2bA3ri9sAdVui2rHu2lBSs5gMTHjPY8HkVo2z4UTNqQxBpxzJrGfjM9BqDAo9VmR+eBj2Bga9kcZofCDhvb022OGT9NemvxtY44PZn9a8n0eBcdw02RRzrjNNaYwzZdPA1Fw9DxCu2jXI1TIsSlSQmxQ/Qs/yBsw+eqQGDXfvWijW3Q7Ymc3UqwYTQ9jIA8wV3GpsG4bXrFcrD57w8HR2AYedFsZUpluPQgiQW/vHnWfeZcPvWgh2w5JdiBEfZwlBnmexV70CpQ7mCjMbJbZgAH2AoUGNMlMw+RNKmhrOx/zmtRfNhzrVCI+cpmS7K62WusFj3npdKlezONIY84ItNqsyUzUdCmznmWj8iT0yo6oQZU4JOHQYKE9JOZ47+93zqVw6oIXJdUilOPvMumGYYIGq3DM6d6yclG8YdhqR9j5wWpHupiWEmOYZyWg9z+dS6bG5x02KqxJ9X5mD2ko8miPhXjtyyqlJea9yHRmrAPQ6G9Ajv0YzM3+wzzVIYYEtg9Z50qiO1K4IlSpshgzDN4+tZcGc3VFjZs2Y3AtqmXGcrMKiYAYYexUipUemsv6FKrMaZ5D6M1Ns+gsV6aFOqsqTAhRWTMOYPZmqqMdAC66gFx4Mx2HtAMA6z7FgKefW6/b0p2nMVJ6Hh7po9i3dmcTgjU2FBnW/Gej8ywdSl63jPBmsPXjoxunOggEkOzMFZcMZ9oyay6xdXLQ45xjwPoDNdW0XD9Fr/r1Ne4Tu1XmgZ+4A9Gz6zPrBeXKlxUrl7c7b9DYC1YT4ZnIAzzz+haDhXeESicI5s4GIdeqFtTD5ZqXvAIq7rYd58HeJ3Dmp3w3asNl6EyYTInLbwPydG9RY/Mqqs014jODrauZg8ZnnmfnTRvYQwYNaq4bGr4VeRXHKOcOlVGYbzJvHvADT1SZlTKcFKpTkY48XA3uX2GBotWM3I/DozVNYnYRz6ww7FYPesY1kRJbFx5wtbZBdDeCyhw3ADxOxKHoHIAH3MGfOg10l6XWOS5Tn42zAzM9n613i3uOv2Vs2I5bFRAJsI+SepJhmDwLiNpXPUrSjyIMFjnKfNPB6O6fQZ94Kz4ew6bUqpVabOoEl6lYa739qZ+f51kPYvCjjfRod20KsUapBR6hWwwqsR0PuU8PnPse/wBdezrAva0rkhTajb9VgB6uZPmYIHgcM/oX5P3bR6NZN4Uq3LjYeqVClQwlMvGG9kPOum8E7btWq2/WLuo0qpBI1jhGyzPP74AdAb1UWPSdmcPXwjyL1qVN/nCo1IKmzIz/AH+f+uvQEa87YjXkdpPmZyJrPOsgAdHyLx/Z/EWWzwgl1WpVysTI8XNh6kxzDWgB/nXQOFD1sU29KPEpVVk3DUJtN9YUc2ZgZ4d4GrHpKfJiRmjcNzoVVGqTEzPTXL5l1cQ78qRxLVtUIDTD37Wc/r/IumnsEHHP2g9731oBNR06gNA0kkkrQd7FAnh4SnsvMauD7mCkPRgNg/DzTwM1J5UIv1pqm6DPszSmUec877cwMOjBAzSpbODZ9fnXQloI0lt72akSd8f2eZ9n1qPAhtxgSqT0XSOLLzMHwwwDrNc9qNTK25AGQ3KnYGwGeDJoodS1mgfA5Ia/Rrd681cVOJ06j1Q252jUqFCe0JkcHsDM+zf3rNRuP14vW9TJ1GnBDkTXjhPRJYfcgDzrmtb2BclyN2xbkisOOM5wsM2TPDPNc84hXJY/E7gtValdtKONT9bkgMz3smZ4ZguKHxIoca37UpXrUKrUKcD4SQj5mGYH8+9RarxLiXDUWeHM5jkLfqmFQBkwNkM/0L0YGzDo1t16XyNVmVuhRfuvKHG1gwPvDfvUWNbHDyvXlU7Z56fGkGAeqtmjm/5D8ikXzar9sTGqrZ0qZDPfg7EDMDPBYew59feqku6p1O9ZVAAOL97DRwPv2L59umLdopVep1pU2sfaev8Aq2oPwOVOJzIGefZsUi0rVtKf6nrMq+5MDXNg40c55+NKDrzXEbe4e1m+azNqrBxmY9EjHKelm9mDx9mH6F3Cj2HTpPD6Rf8AesupPS2HgZOJEh59feYBvBTF+UW6BSuMdDtWqU/m+KjNKamyTNmPhzIZ/OYL0Nw6402de1SC3KdVY0yoGBvh6vDMDDz/ACL8+a9JsevWpU6rBichUKJoHAMDzAz+TyL196KlbYr1jQr4nQYcOumybEkI7OByWA7zBdccyMPRRnvQ9CBmSEloH/OCDPMluhFmBnmsjXjYh+I+eHk+c1uJMbYub3nR/X1N5V83o0h/M4xh2GBroi14V8+fFhxz5tzDNU8llsMDzA/oXMr54oy7AuamWVXIjMlp8OaN4zDWAA8iu6bW2KbVjcfiSQok3x4ZshmGHz+RbbRhqMMFEkgwAame9TcwntOuU7CSAPYGYGqKqm+BYYGC6YY2u4bLEyHoON5smszW+ErE8TfoZhGPvA+9aa23sI7TZrVYbFfclw3+Tq9aU+Ya7Jxw6N6jhMlw3TYfDB0F2Ce9n7RYS56CxMaefisfeOxd8ex2OK4UXrLW8M0y8zm0e9VQTAZLQlMGBqaD2Z6bHQunDG1VJ2Hgo6u3qVzPvABQfUkoPItkq3RQcseCl72XdNxO5tmtUKzBwF9A3APUbTrwb0GGCsSwmG8XiOKVDnv6qpzRsvYL3pGlPf0KXDn8sOmq+mmDwgDjilvQ3Otcd8Loi10E9g/eKWy9ms5D2OgtHGZ26i4+aG0JYHu01IBRwDepC4MOmLMmeAqK9MDNFPPBg8FTvGYBqGriEbXAPBijemAHvFTxpOzrQPG+fs+hbdKO5dhP2aiPn2z94syDMrebfYnmXnwPxE+Y7miCTn7xHziovXDbPhmk9VWza8NPlO5oNbNEyeCyrNblAWxxWUOsOH7RPmO5oAkoeZby3qvk1JjQ8DrUHnjUdJ3NhGe2ArOM9sWPhz+hv/gcV3Ak+Eua+FtFtQyf4jeaka2A7FUxpmYKXrZrldJ3mdwKRrKvDrUjP4LKxLyNAclRNZz+pNG8oEt55s1EM1H1kOfwQSM/giA1F1ktZSJWfwTRoAeRmaLR3gUfD4qwPoUcwUIV5go5suKa8CjmC1EfD4pYmnkkAYfFGAJIjQDh8UyaM0sPirEfD4p0EklAWfwSz+CHD4pKwkSaRoEgz+CSDD4oDA0jNBh8UsPigSeAN6ZU6GHegkRgDZ4alYgmmQwTqIGaaw+KdR4fFFomHxSw+Kl6KWitUIOHxTRgpuHxTRhvQQsPigw+KkGgWQHD4pYfFEngZQMgCeAATuijw+KGzIAj5X4qQyzuU0IzaClON8iIGcFccmH9ajmy3lggisstmpsaGg0cFZwGTwQWtNgNnH6wBNTGcDwUKTJNnPBR/WT5oJGfwSM9nhqIEnMkZnigM+hRsQTyQAaBAzt9mndH8NSADYjZBQCZZPBSwDFSwZDFI2fw1KwMmpBhmG9NMhvUow2IIsZnxVoIcboVZGjZmC0sZnAQQGDIYpGyxh7NOqO8eAqhTz2VSyWd6sp8/AzVK9PbMleEAeZVfJZU03s1CeezTCVYcZw009GwBWeGxRJHQrFOYKOYKW91qOapCPopnRUg0KLM4fFPACSIFISaNOqOaoR3k0akH1qIaILP4IURhmG9CDIB0N4IANApBghMEAI0QAngBEEAJ1kEgBS4zKLII2zxEtFv+tTcPijCMgjgpsYM0YQMwUpmHh7tA8yyHkVrDDAFXgGBK1j9CCcyz7xJ4NiQHsQZ/BZLRNHMlICla3vMEJ7EjecVoE9DbjBptqPh8U8ZuGkiwYYJ0AyQogQSMGAD2ahTJOHQpHWqyZ1q/CFLUnnDM1STNc1ezGc1WvQzMV08f72FqfNwHVKZefPYAGjCG2Du/rVmD2iHQC2t4p3nsCwNMmbZh4iOYeb+Z96jmBgCuGSFPP3arFYSerxFIjQNYOhdMecMlOYfhoDhv+8Ba6HbYG6Go2rU6CwAewUd0POlzwI2Ckcr8Vq5NEYATcNUlYkwabCdlPmDIN7zM1HdC+lXgzFA/EcwWFva4WJkU4lKfzDz+dY+6uMbD0x2DSohzGj968GCw9bvCqnCPTbBn6FwezzOyPCU9RM3dSU4AfnWfqT1uQ5GhEgszHfnWaOfOn/tT5vB86u6UzBjZm+3vXx+a8NoOxo045XrJ+WbJ9gAjqtVqsnMH5zxh5DNBUqlEzA23zUIP+DnB8Nce+xsClTOTlbEFSZnV6rtRGIIB53uz86ONoRn83281YfaSCz0bFcQPMlSjaMyQ3yL0bfny59YJHDfMcIlGksh36y9Z+mfZNNtiZR6/BYZZkYYHgHXguWQAvvjHS5DcXk2QpbIPZmeGa+k5HD5kN+N7cMFFAJb0gIsQMzPoBdotVmlXbXIlnXxRjmYHoMhEDfn51ccXbAPgzVIUulTtann0RJDIZs9CNYtxe8LVqNsRY/NmBnKDMADrBdItu1eHjNhg/dVz1JmW+AGAAexZyNz3E67WjOCz0eNh5Fu5nCs+IUwLc4TnJORFZzktSz6/oRbl8apMSYZwWNYz/fKJSoz9SqPq1hvxexHMh1iwLoOh1mDoyGJOhM1WegFtbw4aMBRmrqs585LRhmZg9swVjnVVptVtiYD7jeDvYuoUXhXxC4r8JpfEZu442jS3v2R5n3eG/f/ALC5u9aV2z3dRulT5P5M8F6Y9HK/7KgcPqhwM4hAdNkVfPCc97F7P/B0Il5MpVbftuswaxpmycU88D7wW9rdNo71SauqC/rMvhnhhhgvVFB4V2+9QanEr7cCsUqCGAGcYM8PrXmivXVZ0abLodKtyT6nYePRPPeH50DNmTL4o81247HfOm4Hg9Hz3mt3avHKuV7iNInMTjwMM3mQPvAPOuOBVarTayEulTpkZoP6O8jqVVfn1L1q2HLSPOyaPHpW1XuL/pJ1moWrR7mpVKm04D/aA8bDyfwJqTw0oAT5FncV6MdNuWlhsl8zgD3zga5Dw64tMWBS5bDlDCZUH/8ApEHtF786w9w3Vcd51Y6lcdYkzzDYzqnngCswsDN+1bwm6DkaScJ42NUN4Gqet1VyTWZFVb2SH8N6ivT9EzbYUQ3nHizNc60iq1io1gwcqJgeAdABgCXOHyARW3zDf0B2JpkGM9M0zJDRlaDaLbOvV7/k5T6HKY1pDG9mWBrD1KS5Jlc25KPV86Mw2KJg4ZabYZmtUJFKqVSZr1Pl+sjM2DzDmz2AtdeDL9Yd+0DkqBvAA0Y554fnWD5CU8Jy4jeeHYtHPrcWS1H5Sjcg6AYPNB0KFmqbGfqVWp9AD/pGSEU/oXoXiLw0b4CXpblZtmCdS9YxjCSBgAdeC4LbAPzLhp/q1gzqAHrRsPOC9AUe2+NPFc49VuOdrM0SYYA08yAfkA/yLYcRuSm1m26361nN6PrgzlAz2B8n8aqjmSjqnrWLUuQ87wHvBbDiFQbqrdwVN9xwJPq57lcOgw/Iub1KHOgO8pObwPyKB1ivXDVZMCI3fET7QxHAzjSz2GC50YfhqymSbmk21T3Jbf8ANTHQ8fWaGGyxqtPy2847B5vAHXgguJN81yfCag88zAhMBgYAG81WQ6U3UndNg2QD5zXSL/ZbuGhxLgodqmEfR/aA6/zrnUYA6G+tY3Aqaky3DlHEDsXVrDqoUS1AiXPbkaS0wevD1gwwAz61N4VxqlcMUKHBsp6tuhJzeM+vBNcZrqALw9RuU6SDVOZBkw86uB3Cw7DtWj8ZKnbkUAZo9y0TY0Z9Bmqz0eKb9m+KF8cNZ0sNI2TDA9msYGqThWzX6U1Hvy8aHJqVHp3j0p7W+8sgB9AAtd67pX2wqfGaXYEafR6jJA40uovaLwH8gILC8OEU47SrtKrkHlqfFDmgndZmfYC8WUqS+FUjsU6KZ6/tsN+xek/Sr49hVWgsq2KqYU9yMHOR2QwzPvBc/esam2ZChXPblzgHNQN4GGYHn1oMjDsmpXVcP2Vo7gcwYZgb2wFRVW1Tttg6dUn/AOeIsk4smPh0fOukcLqrR4d0VC57jqIMyIsYzjZh1msFW7np1eqM2oy4n3197e7mggyTfAWtA8MDA1sLJCjHXo9RvG45lHjzgzZlxzw3/OoVYtKs29a8Sq3HE5MKiGcPfnrAmplBg1XhVHuPnmeYhPd/1oOlcV6lc14VSiP1mKAVNhk4oYd4dhrCw6x9kryhQaxKejQmJIHMBozADBaC2KJc9z2bRLqiXAzGCK9oPc31smrJmjsVjOzrjoBnUDmbK9HDMM+xBo+Jz1q0fiXL4ocEqk9Mp8XQfnge8Hsw6PnWr4dXVB4e8cLM42MN52+Z4TI8c/2bWDA9ir6kzQ724QR7OtGDDolx04+VmU6OeYSTDD2P1rl9etKsWTZtKqtVfeZalPaD0dnZs+RYD9WrzthifCO8bSbDYASo0uIewz71dveyaPzhmvHXoN8bLco9vVO1a/KqoBFe8GQ68Zhyp+cOzBeyntB6O1KiPsyY5+xea6DBaRYrzTWfwUh4ME0rCUGZJOMGp+hWAAk8yw8GD7abGdgBLrboMHmGB9eC2oBotKvh6DMgFd5/dz/41Yr3tD2jigngZ5gpxh+IoJg2CsGCRstmeo2xm7hgCDP4J4HsDFxtYjwJdVBo8+bWKG5rfdakYSTA8MDA/J86p7VrD7MWREuqh+u6ODJ8mbQAB5/OvRfGaz7AjVKq1yhsctUAPWekdYYZ+MH+ded+MdkyuEVZhTqdeP2nhVeHmBs9EN8+gD+vf+hY8zVoKDRKlZNbiVK1ZcalVA2TzOceYYYLH3hxL4hcSKW7LuD1JDo7Oe9rAAPo7MFlLVqUSvSnaxxDiPV6Qwyf3cPBlZ/5+xVVz28xGoJy6c3MhtB7GCZ/74KIhz22FvcbLgpVgHRok4HpbDzHqozD2Id/+RWtsXbd0ybVbHrlA5mbVMNYzPYyeHWfkVn6PYUalcQbScv+y+Tp8qA/ChyJbOwzPoMDXUnuHVs2Txf560a4BtVuA+9MCRJDDDsADBOlG3EpkOo0S6KPbNq1zRCUecmme2ZZMD7/ADgvSbzN8WTWafXJd1W9RIRxgfuG2wZPeGeBmH5Fw+gyYNEuG37S47zsKFVHnzhzqYf3oH8+gzDs3qyuGlc/xGlv3xcD0+2mDYp9NekHvCndGYH34GkeG0Ki0qlYFVPiRQ4Nxxgo5hzVKZMMDeDX2GH0BgvYXBOw6/TYFmSqxSozJ06GDLzIHvAMPbmvE9eh2zSrmj2BSjkvVWLMOnszo4ByxxT34Zr1L6Mz1K4dXW025XJMmlXWZs0103tgGGez8+aYW9XPGxDYN988Gg61z+TxdoFHvf7HTvuYPh92kSD6z8i3EmSwch1hveYdYLzp6VECq1unQp1o2rzkiK9rxqi09hogyGZ5n3q4HYI3EijvVaq0qlVWNUjhYHJPDPAzMAwzWf4hTG6VcFMrDktlmQxnCPeGeB9H1rxVR/SHvU/VkXQpXL1GSDMk4geNJDPPBari7dVS4i3lTLmia3q84wADTwYPGYHnhh+cAWMczVivS0vNi5KjZ7DYPT6xRGTCsPAHWGfnWXh3PdUCh0eKFxzDjm8D/ql49/0fRvXXWaO/W+C123HWaVTabQosN/1abJ4PHKDz/n/wLkoQ7frDVK15wRqmwDBnT5Gw5jAeQ/zq7t7cPWXD28GLnmjWWACHyocrPj4YZngG8Fo3j9avmw/+Rc64UWZzITarbl4xvWr7zASY8iTrAyxns6OvYulHR64zVDwprwYPYZ9hh512cPM5rg9AhvwyDBtatk844IGYbextxveneV+K23tGFFVQwzWHgVI5JHr7DDsW6r1NfmQHWADeayTNBfpTUiXOb6AzXZwsbZS56PrPuzoraoo0OdGwlnsVhPuFw/ZubVo4ENiZQQfMMzX0uG3HcMzzh/1I2dQ0pLLkYzNxvYCHn9b2eALphzgk6Bn4nWgBlsxwwTXLOG7/AJ1KZZ0ffmasVRwHwIzPoQPAH4auHt4qtejGri2eFebKZAFYHGf/AA9ijmCvZg8zqLQQ5LmkGb/61QsngSlgessbaeF7hm7rt+dX0Yw0gVDTTbMgzV9G5T2ba+bzOvjWHYgNGAbFKZh5jmuRup5IGaqp/TprZhAY95gs5W4ESM6bjD4HmuiP+XOzgPYHpqdGPNR+WzPUwUqNDfDY50LstknAyGlp6aTMbf4gKayy2B6fWrA43hAubboZypURiYGeoYGqr1VL92a1pw3z92mvVT/u21e09Kkh008tQwUh6HgHRgruNAfBWB0ph4PnUdyvwyTIYDpqQEYzMNMNvnVq9RMD6FYBTWzhAwadwp2YD4OhhgYfItHAhvm0o8amtxvDbWipUbwvZrj5rbwqtF9k1YRgfVq9AwaNRY2mHhmuVslRmX8vZp55lwAUqMYACGZvBZLU5mo5mpTwKE95FiAz+CHP4JlJAtZDrJlCpWkc18EQTP3irTNIDQXHP/OlzjeCrM/ggRCwN4DJCe9RM/gjA96oEaSPsTWJqQQJGhSQJFh8UgBPACoR9FfNFz+tTsPijw+KuEK/D4pGClGCaw+KtKJh8UJgpRgmjBA1h8UsPincPincPiikXRRAypwMouV+KoV4RtynAzsToMtqQDKkAAJ4EWijAN6BAzn7tS+W3J1kFLNn8NFqw2U1oqaDLhoDZwVIV5go5gpZoNi9Si4fFM4fFS0GHxXikfD4qQyCWingDepSWHxRYgjSQJkN6mgCisngSmg82ijWJpl4wDrU48MVUzAMzQCExtWsOpAAedZowMDVhGA8AQWTz2sHtMzUc2fw0YBkpsaNs6FCVaDLikcs4asOXbWU4i8RaNwxpceVUWOcmzf2aC0e8w85+RFL3l3E8DJguUxuPdYmMBLYsNkGj6DOeCr6l6V1DthqIxWbAkyZD4Z/dJ7Oz60Haw1FLBlxcpsb0nLDvCswqG/QKlR5E08A5jeGa7n6nl/gIImHxSUg4cv8A0PITf7IahYABSAQBDlh7g/0J4Iz4F7A/wBCCVD2Kz5kAD2irwBwOw0ZsvmHhtoJfrWJ+KokypA8ODay8wH4ftGzD61ECsfOtmSXPPMzVPJe0VKeqTGG9V8mZENFEElw/eIdZV56hj4D4AgDUD2j+a1StQkqPJPNR9Zz+pNGZog08e9MozA0CgNGkCPDNBouf1osBoTTpqOaIIzQZ/BLD4pAgaNR8PipZhvQYfFA1h8UsPiiRr0Bh8UtFS9FLRXgjgzvUgGW06DKeBlAAMqWyzs9mkDO9WDLKBkI3yKWzG3+zU1mMpYMh+GpWjhG+ROhG+RSNiQIBCH+7ToAaPIEgMEEgASw+KDmW8PaIDksY+0SEEaayBBzOaQA2aY8g+tOgCWGCaOSGftFeA6exNcy2ChSZL5gbbbiz8yZOjHvcV4Q1Rz2wH2ir3pgGs0zVTe9masI0nANQ3FtgS3tQ/ZoAjOe8UR64W42eDeaEKw5JHUXrJYBGYx+dQZMY/dtqdAAz3vq1BmJhmgyQMuAXsEnmXHvcLSvGwAnpgqx6S3l4isVjNEB4NRwE9yHLdmCB6tsMnp5pr1w297xELuP2InjbAVVAb5+8UjM/eOKFmpLOsBgsDxIptGZoJ88fM9ezsXQDMMOsFgrzZB5qRzbYG15M1y83MPJ70aX7jYthAuGw6bACnSw5yb3hopXVAcZ8duCbOfnWZCBrL5V+y6oMVKlUafPOXEb5YD6A60HqQADw81NmWxVYEXXYifP1qnCsV9nyLHv8LUVSgORpBttuZpQ3mwDTz3qRJOXMkeIe8+tHGpWsGoxv+c1zbbIj0YDzczUIIzZu6hnsVhJAGfaOfoUE3m2fDDoW0WO1+mBQZ11XNTKHE/Zw6/k2K7sPg5B4e2HcdDbc1pFRhg+y7mupV60otyXA6/O7+hah63oMmADGn0BgvtuF5/9F3ghb9BgS71qR85VTMw6/YrC8YLYfvmpVB+5mJmkB4QHo4Zh8m9d1sam/wAmk2pxajVQOnnm+Z4bAXIrt9KWDR687Q7RcjM0+KZhzEjeH5FCobjg/wAB7YsC3jrEUM5tRh4PA6yB4GrX0abMYtiLW6q5vkSpJgB4Lg9t+khEvOszgua7ZMOFC62vYg9+RdT4Ice7Sqo1ViC2YUeLm8FQM9hrVoz/AKcnDGDclrhfEGCAVCEYAZgG8w2daynB+0rLPgzEnVhgHo8X9pMzw3muRcV/SBnXJfkuDVa5MqVuAZgABs71zG4bwbksHSrfqUwKefuT6EHS+GPEKnWrxOrESh4HRzMzZA3sP8arOM0my58eOxblAOBUG5IG9m9nn5Fx+N4JaitgmZish1Csce7qO3GrctyD6hj4YSTA8ze/X0Ll8a4arDE22NE8zz3hmiN7MU1Gja0jQBwMz6AM+tZC7n3IF1A0FYlxociKGDLOHWqc2XwzPT8LsNRJ8N9l0wfDAwToXJO5IKdgyYB0ZgqixHek7c200zJ2aikSYFxyXTf9THv7wVZoyozunLiPAf0I1S5kxsy+hHg+DASnIMnlz99hsNbXgnw9fvC+oTb8tmNn0NSOh4/IvUF8+jBegUFo+egMxHHgzjmGAB+TBB4kZ1JkpphiLJMzPAAwXTqrwWuqgwo9RlWzVczDPRNnsXvCyfRI4ezIsKDXNEOVwPlGsAAD86j+lRWLAtLh87atv1g/XsVnBmOD2Zshgg/NeYy29IM2GMA8meaiQ5j9HqQSonWHQu12BYd1ce6NWKdR/VsORRGTfzMMDe/P+RVNhxrg4M3BT74uC3Hno+eBtPdBgg5+zr1uuAwxEDOomABHZ2AZrrFB9Hhw6zLp05ipRpD7Ozwcw/Wp1w2k/wAb3Zt92JTjpsiEeekGAYAum2NUqkzZ8epUqvzJ9dBnRM897J/OCDm9k29f9k0ua5K9Wg7S3vBp70beYH3gea6nZ8ym0S0Lgu1uqxpNbm/9HMnsjZ/IspbD3GJ6VLqN48tUo5snmzIwZNccqVNuC5Ljl/Z9+SzHYPxnmTw2KNikvCq3PSrwrD/rKeEh97PM9iyTxzphnLqL5vO/OtnW3qlz8d+pVX1ryoYA93mCzU/75NN9tgwz7DViQFSqR0kKGf7OrCm8oEqJ6y/ZzkgD30Zqu5CczhzYYKxh0T7SNHQ+aCM6YGYGfyKx2LidAg8MeYsq3H3pMetxglRs9hxgPyLmFqw9avR2JzD2fkaDM1srk4hP16yqJbl1A9Gui3jwZltRs+ZY7Fo7As9963GuK9OuflqxCMweZkM57P1rIR+Ht1VKz7orERiVJpTT4ZhrRusFvbVt6JxOmhTaPSmXqgYHUDlzoxnh8ixnEuS/eDFPqrEU4chjwM3ejPyL2Rwlt5iybBo9DcbwkNxgM/Oee9GTh9YqrkY6hQ6rQJ7NQCMcUNIM2Q+dW0mlQax6L/PVUMPVcYwpp9ZnKDo2LssOiRKVUam+DAPHUT6zDoA1NtWg0O2LcOjORQOngeeB9ipUW/L+5AfrG9+JUvWvXUjeDvR0qiU6iVmnvhK59o+toD3r2Rf/AAcpUyw7zua4Azqso84ek9sBjyYLkUm26NclstXBQ6BGZapbIMmb2zei3Mal68pT/PU5jRD8F1neq2lU23K27Hpz9Gk87nm9y728w+haiTPobNZZpUupM17XPB54DzDBO3JRKbDqQHRqHJ5f91vNZDpDNV4UcTqH9lbqo8+H9l4ejSjzwN7swBNcVPR+tWj0Ph/Q5zAU246jMYCZHjnvNg/P865/ZNYlM3lT6qw+AVukPBKhnIPY9h7g16i4VzKNxavR3ivfFKe5gKlyQR88whhhsA/9+xa9zJ5N4kU27uDlXqdjxJxhQpQa4Mn5DW/4Y2Y/c9jO2dOqUmg15/CbG5vYDwb+hdj9I20rfrdzUe2GIMZ4H5jD3rYD3shnhoPecEPpw29Q7Yn2pKiwWT0Lb5IAPyM4ACY2q7cUCm1+HNjuUNhmlViiZ6MinPYPfrWoufgbfFw2raVNclyXrlr1SzZjm9mAMHvze8n+2uoeiXJp0nkpz7EN6RXoGeBxvGZNnsD9a6rxR4dVK5OHJ3/Sp0mBWLamHUIGiewIuzME6WOHBZ/B+pejY0zcdYrLNVdrwHS6lTI+GYeQwXaOIsaq8PeClFrkF/CaxoaLwbDDNcX45XtTqrAs+h003pM3Y/JdePMzMzDrXTaxYdftuuVu33GDqtHrdB52lc2fsX8AzZD6FjttD0hbFwwarbNCfqM4AqE2msPGB954b1a4fFecLDe0bc4ayqlF5mIxGfhPbOg+hegqacpnViTvcHgyfeYK9rWYdCAw2IANPKxENlxR5L1VjeIweYKz718eZ1hwAM1YqAuHCVoTtjXeatZ+mz1uYLD166qHQZhxZzebv1rnX2tvQ+Xg2Pc7LMIM8/WDIbA+TPsQdqOY2bTrjDnQeCTM/wC7uudboBmAedZK1Z7E9gJT4VWZr+2qBhosmfyArXmd/hqRyriRxUCvDWOHNx1KNbYSowZx2YZyT39h4AvO94M/ZuxrjflxOcpQSWHowZ4ZvhnhsPr616AvywLurd0SLnpVZD1gFNMGY4dbx4bFyeTZl+XbfVHsfipyzM2qU2aFNkMh4LJ4Bvw681XL/cXDzvGgXxef/s5Tnp8iEASjw7AVV9ob8emndVwVGTJaz0Gcw2AYdhrup8LqrwTaNu57cqQR3/AeuSnSc4rwfOHYsyzaVv8A2crFv0q8Ycnmpj8oI8frMFx3HWWrZ/Eu46bQ6PY9cuPR9VnzUYJbOyN8geQFHsa8KNAq9HlVydDmU9/Xp7zUeSez58FVXPWLS0mm25YPOgyDD3MB4zL4fP8AnUHgmdDe4pW1Ua+x/M4TPGNln5FfajDv1t8MZwcZLSbf1qxSrigPyoZnG3s7D8HA/oXqifwig3Pw3plKvGgAFVpFKNmMYH+zYB1/PmsZYEZ+6uMlYudyuYHa9NCmMm7vAze7wVF6QgXxwxdt/wCxd1BPkV55+K8zI7MAD+Dem1xDRW9avD3hdS4k66mKaEebGMI0eDG1npIH1n58+joXKo1+Wdb1qNW5Rq5RJ7MKY/N+8MvRpUYDPZgZ9ZgjsDjTTQo1Quq+HKkFwRQfZBlo+ZjRgA+hnyZ7FlI1Bo3pFMVidZcsIEhisMSnqZLgffcD7wez6PkwTlX0u+0fjxxCn+r7Yi0Nmf8Ac+amVkA+8+rt/X8685XbxL4hfZWPBiV+pVWxKXP0Tjyz0Te84GYdi3f2SvHgDenKPywk2pXoZxdIw3mGG8AXN+N9z04IFHoYUb7MUqos+NoydZkAz7wDvURbHeGcrf2fmSI9cti3AoMhh4DCPHe/WCdqVz3bUqy7OgwZOE37qzrAZ6J+QMO9V8MKwzNOo0q46VP9Xfs0Rnoez2YH9Cl0GBeNNvoKzdVZprMLW5o2Y8/YB+cAwUXC4tuLwuqs29w7onDW8aHJ9XvzDlTAeZ380Hz/AJ1zyfJi8gdRnUqSFPhPB6tM+sGPnW4vCsUe8LXqtSOpSawEJ4JR8xmBnvwwzWNomhSr1ByWxz9HMwM4IGcnRMw2bO9eL27lYfEKLwlao9Vt+jxqlT5rxnP5RnM2WD7wPs+hdegcbKdfJyHLcuOt0SRCzBkJ0YGQmH5AzXhGsQ3KJdsiuWzKmU2jypOgEcz3h5wU6lVu45NXpVZqrnM0q3qlrRgP2wHs2KeG+tFvdVpcQrnmQw+2stlkJuhKpVR0fbBhvA/zrrwQ15Es/idRnqzRLmrlSAKfCe5LlDZzAAPvXrOz7qpV4UZqsUo9j/zrsjmQdeh7FVT6b4R6Yda1GGaZeh5rb8ow871XhdXAdkOUpjNrPPDyK4tKiXNAp2FRpUkGgMwzwXWHg0X8FLDoX0o9xj8zl9Ss9h6G74Z5rn9VoMqmhrtsL0A9G1j021l69Y06sZ4exPrBacPuMelxQJJh7RGzPzWzqvDQIcN023D1cM8zXOQ1GfDc619rx/VwtBGkt5ZqzOYwYabjYGsuzJcxwzUjN9nrV4Us/wDji/hqJJpWZ6jCTJ6ytYx4YOKNjOyYZxj39CaB5xk1tZkBiZFzbWUkwz5rDTSDCXAe1nQ8Rbqm02IYhruGs1Tbeb0ubN8Aw7FYeu24zoMdeHeC4+ZfC2zMaIy14badA22c1kWbhfMwbbPYatmaVWKkJgw+AAfnXBcOgp8lswNvUWRmQ83TcbWzZsaq6pOS3weA/wAmCtYdgAfWYK+7rGHprJ6Hhtp7B8OtswXVY1h04DA22N6m/ZCOsfslphyeNsdDw1bA8wfetq9ZLBlvbwQBw9pXXpnq+dR9MLwy4A2aMGVYSaJyZYACaCNgm3qJyadZjGpAApbIfiJsR+Q+RLkHPdgrgAYwToA3+GstqwpeTPyKwh6bIaZqw5YDD2arHg3eGvN7IWZnm1prOPT9GfoGrMDfD2iiPRm3n83F4tZxj2KbyzjzWoq9nYPtFYMzMGsFla0J6G5+Gq95nBXvtiSmQ28c1AyhsuJrRc/rV28yCjmyshSGCaMMVamyo5s5otWGkpD0bAlHwwJEBSxNLP4IM/ghg6ClsxtYNRQgVhGcwBA7opowUjP4Jk0EXD4p0GXDTuHxVhDjZgghMs7lNCN8inerW+xtFyvxWoimCjmCsDZUcwViCYJowUswTJgqQioMPinTBIAyXuEh0UsPirKNDzRvQ8F48hEjBvUvRSjM9/8AwKXiaPUTRb/rRgypXLfIjCH+7QBo7ELLO5WHJ7U6zD79NSpEZBWbIAYpcu2g8QOhAJsgBKJM0068ZmoEjrSEoMk1Cz+CmvAoL3QuofdZOhvVeHWpYPYI82kAiBRwebTwPLPD06gNIEnuhRhRozS5hxRzNAB71eBZ8w4gOYh7EyYKcI2ewzLUUqMGxNMhsT2eBaa8wbT2Y3vFZshsVfGk7FNz+C8WkAzmvCXGm7alW+JNbrNVgyYzTD3Ks6wYADAdC91sycEEzlJg5y6dDeP52c1I/Nr1kE8tmbx/IGa0Ft2NdV2zeRo1q1WZI+SMYB+te+odEtxl8JTFsUpl3zhDDNV91V6dTWjbiYM/QnSdziXDr0VKiA89flZOlGxvjRKfJzNk/OZrd1LhpOjO6bfGK7f/ADKL7W1U+t800dSlTPfmun53Nso1gVz2bHGy7Q/OBqwjcNLx/onHe4Q+sASo/NHKDTbzWojSX897GCxvhbRakDhXf/u+O9Yz+cAR/wAlfFgPZ8cJ/wCeMui00M2gcVmHQuZs5YfDrjN/+Pd4PrjKO9Z/HNn9l44RvzwwXVpX/Ms/PPA174HMarROP0b/APC/Ae/7gCpXg49s/wDvxRHvrhrqEzxlB9WuH1t7F1eGTnXP8d+hus2299cZHzPH7/8Alg/yLaz2Qh9CpKlUnwing4urx4Ys0dV4/M+3pVsPf9sg+1XHfDfY9Be+g1fRp77weI5mnc9ZRcK2zQXnxtD2nDWm/wDYycE79ueM3/4rGf8AzK0cbTAlasvZiow9YeTxF4qAHicHQ/JJM1X/AMqnE0D/APuLSf410vWSOY+HQeCjA5p/LHfge34QTA/Wo/8ALZcAe34SVIPzmulnUp34ij+vJ3nTA5v/AC3vh4krh7W2Wu81KgceLAkugxOiVKA7/wBYZ2LoAVJ972jYH9YJqZQbcrcfTrFDhyc/OyCjAiUq87Lrw4Uq4Izzp9isNHesPXuBVo1L/Qbj1KkH5N4LKvHxb4USg13PXFM/3/10HYNFLRWftjiFb91MeA/y0j8F5aUANA1yvxRBG3+InkQLdAQZTui3/WlngkDzZmvMLIwxTXMtgnXuhNRgB51Xh5tKh67xB4exXDLP4iUYG8Eb0kGRzNRcG0sNiI3lnzuGIye9xGFbbe94nS9XetmlzOConqxh0GvjNS1utX0vNrY57gdCQT8/aOKkkzwBozVO9Us+t8wTpY7bN6eAB4ZqCcxzHUzzWYZ5o/ZuGaiTK2/GPDNbePVO5sGaqefiOInrqgww6z/Iue+uJRl7c07m497xbdB3Nq9ebZtYAwaOBWP+CYBubw+tY9lXdNDYAKOlG11Jr0SAGpve+hU8m54skNM4KkSY0QGjN9tURgwBezURBtLOsOGGmwwCkQ6bOmBm/sBQgNhk9QG0zJrc4xwbc2K8G2gCjwfJvUgIzDPs1j41YqLLueuruNVcx1HHFGDayOYYJo6qYezNQnpjGHWq8zMzTBtZPV5z8RV70w5PfsUcwYD2iaOSwHQrNly2HWnWTYAlFOeZ9fQnYb2sem22C2wxXrMw/dqdnmKpAkgz7RxO+tdYdPUwWNw2hmrkqT4OuttmsZcNwzgzgxabJeD8Y1rrqjMav3Fg3nX+swBc04kT34FNNgG3geDrAwXweZ1M/cMw57RuOPh+dYw3tFau3qVz8D1lUm/FNZ+5zcOYbegAYeQF8e3XCJUrhrkloGGH/CAOhURyXw8N8N6sA2eHvz8iUNmlHMP144AB5DXHsu1VhgebnQo8mTzLX3Ho860VYCy9UORfzPyZ5qKfq3lTwbwVDOM0eW9m/KlsgCrJIMRpBttvgYedXc/DDTBRwNjYxoAfzmuqB77ZAM9dzoBYevcQpVV/mOx4hz5p+9DYALQVjUnwjg6mGaKg02LQYARIjC/SOFx2541SsywLl+1z5+sJrOAMmee9cp4dejHzPD6oX3c8GS9INk340c3sMOvea69x4syo1ufTKywfW8DGl2LrDM/1Dw+aYpreDpw8GQ+c1C35D1sH2azNB9vB3W6F6tsPgg/aXA6bdr9wSWahV2dkToADz+dZ+4fRa4hzOJsRydFjG1WHtbPW6wXqDjBGfCxaJarGjJmgDAYAfXgsjw/Pej8JbjuG/PsVE0eYfMzMw36IKVb3B+W9xLPh5c87DADMJEfozwXp6sQ2OFd70y45Vv8AMyMMDCD1rh/E6/HGb8O5qPSpNKm9nMBvBaxDo8OeX5Y0GzJ5wWKxzhh1/IsvrYNaasLnrdVr1XdqtVla0h/rNU+fvEuEDZecBauzIFHnx3X5dR0ZELPDZmmY1HrNz0Zo/VXJxw98HWtRw94e1m4Z7sW2HDZkMBmbzwbP1rkSy8+S5MdOW+eefelGrzlNa02IMb68Fq724V1yyTj+uarGeN88NGOGZgsvXrYqNEitHUmNHXDNn5wQOw63UZjoZuLoEl61XmtdtzB0OjNc9gQJTMVpzBEElg5XKy1Kmmn16o02s0832ACmA940hrZmHyL1xQbzYoLrVHO9YzNr1eNnGkVaTrYH5AXj+NMgybedtxsM9Q82fkVk9clSCgtW5L0TaDzhvBabW/QiyaUfDe2pdwP3AE9p8DPVPvD615Kptk/yzVyXUmwOM7NkmYO57Gfr86wjN83rWKc1Q365PkxGMw0Q8iu7MptSOa7p+sozTAZmbSbHY7kOx/RytSqxGKkydYrcDA4jIb/r/jXCT4kXBUrfhU1uhhJp8U89WWG9V94XPWeIVeBu55UmS7FDBk5HWAKRw6t6gVu4DtypSno2oHg4d5+RebHY+AnE6zreo0im1WDhVZr2uDUcNhgqKNdt6WZc1Yp1DtyGzHqjxnzcdnN5kPkBc5en1Xh1e5tsSsJEI8D2dYLqEb0lqVQavEuOdw8qRyIoYBIjmGC92KWt35atKd0LSl1KZUHwMHvWGwzNFb3DG5p/KVWDWKPGCqBhJiUw/GAPOsvdXE6gcQuIMviFBs56GD7IBJaMMw+tdIqVY+w1mxLjt+DGmTfcyMMMDTqGKsaxrOo/GuXYd/5zKeHgxnT2ePhs2KX6T/CiDZLtPueDWYxwgPleRw8ZkMP41Q3DP5+LIuepPxqlNqmDzx94fR5FbcMaxEqVeiXNxJucKwGjowAdM3tE/oUbHGa3W6bWHWjgnvAMMME1TalBhzY/rJvOO4eB4LYcaYFq+vAqNqsMxs89aOyGAM9GH+dSOANYs6HcFYpt8cmFPm00w1ZYZgBq4sOzAoZyqhcbdxwz5Jnwe8DD516Qsm/LKurhi7cdSgwKbCpwADzIPBmZ4da8RU2M5Ta561puzlZJmGfQa6bbHI3tdsdt+1ZMznc9anQXsAkn5zXo7BbfF23L/wAK4xSgZkWvUge5EPfRejNetpjwVunRLmi9D4ZgH1rxTdTNgWZeVqMWrBeoJ8ydPqsGd1gB4dYL2tSoE6BRmqHTWMIUUMIx/ItUXCDcl52raUqJErk42eaDPPRzAPrU6T9/YNineNr9BnsBV9Y4dU2t1KLWar4zsUMMD6DWggeDFaA/a94IOKek/UqjZlh0em06sHGOoyThTMPwDBeYrkn1GNZdTs6jXMcy34szP7uz+0/Wa9xX/wAOrfv83ZdzU4JIMRjCGZ9YGuX8H+A8qlW41Sq4xhHCecoHTADMw7AUDxbWJNKjMU9+hwDhuwjADN3rP61raxc78aVHcoc7W1wz5fDYtr6VHCWuM8VanUaHTYxxHwA9GP2LJfYC7bJn0r7T249TdfozPrBc1wtijOdPqXr/AEw+5SQ6AwwXsA6VWbepdr3dY9uSalSrohhKqVJiHvzAOsF5sjWk/W75+wFKnRoYVF7rePYvdvAq2Ljhx6YxcbEkIUKmhT4Gt39hn+dTgecrS4ry+HsORLC3DrbJyTejTtYAeZ87DwH1qwpXD25vTM4ifaN9iTRLMAAzzPP5DAMFzqsUG9K3etdqMWhhJ8Z83mjPYyGa9xWTTf5PaDSm6BSpJtVSAwYRI4bOaMP4M10RaIef7PpVD4Cekc7TbfiYW1S2eVeZePPRMw3mvXFbkwbhsOq0a3H+Z52NoM7MOteUuJ3Bm8eEsW5eMV23VzMurnrxqZEA3jAzPoM/86p+FH8stvUGmXrXLqOG1WpgMUSny2c3pIGeGfyf502vDV+mHZnqGVw6rlHpRnIB5invYb8zDBeguK9vP1iLQn4r4MyKceeB97BhvWgvCwPtPPt+XUv2ekGE0wwz+9LRzKa3MqUSXoa2gHR9f/2CkcPo9pPw7esfkfGa9cYPM9meZrtEwP8Ah5o1nKDTeWpoU6Wwek3MfeZzDDAwNaYwyWsCInQQmGKZN4AdwNzepFgAJqpQ6lPhHFpoHm+e8wPDAFEkyXww0AM9+/DyKzpphXoDVZprjJg2fn6FQ5I9dtvvcSXbRicsdTp2fMxJbO8/Jgp3EWt2qFsnTqq3o1UwMGY7LOwM+tcP4qVU7S48VDiHKYjQMAYlAE4zAJIYYYLk9yekJTa3VJtffpr3rCVhy0iP1xgAMMN/Wsh6dZvDi3YFJpj9OtiBXrcMNBl6RM0TZ8ma1rJ3NVbVl3H6jhwLgMM+UjvZgvL9t8WqNR/5q4qXNcMyPgBxokQNaK8BhmGeHQtxwu401+t8U/tHVYtVpthPxuVpsjkD5YD+d4AWexn3uKlcjXLWwqV6xmaweEUGQZ2AYH0fwLlt/wBVrMy5qZdVv8Rq89NlPcrM5sAZOmn5FquN8a5uF1xhFrMGNW2poSqpR65EDM3szz3gsVJoNSC3qVUYMV6fErZ+s2QaDM2Xw6weW0XhcM/c/EK8Z/rCDWarW59KDCLVQN7AHsFkvtzSpNRB+I+9TQNnRCcB72fyKvvO9oNSam6Es4E2qPAE+IyzgGAd6oXqVOrcORcFOobManwgwedZ7wDvV3HYJwXI+9HkUp/Ce7UTzMzDeZqvtu4ZdNq/3TY7gYHHPrVhTaU/dtZhfZwzjOmGAGC0VKDOzaw/WaAHrOlzAwnPBve3965rvA6Hwl4i3xbY/a2y65Gk1WnHnJpkhnZJD5wWru30hJV80GdK+yr0CbVHjzPW1jjAZ+5XEZ9gVixryNh+c9W9eMEp6XTmfBDWDPea1dvTIMOqU8JUsI0djzmsY5jhV94Vt+q1Kj0qDLhwAqGDD3L7MD/fLtEDjNTeGjEKzn7YolsA/D5WTU6dvMzPPeB/Wub3ncNjvOh6qt8PXEI840iOGx4/nXPbkjXVDqnrWuRKabVRPAwjnmAYJfNtbosm6rqol6utzr0O4afFjPxWZ057MGdbeBh5D/11jDky63PplGvGpPGycN+nwzZZ3mefelR6DblSdmtzqry0eLD2U4NhgYKRSp9NevClRLKbAI7EYzzqIbGTwNQ53QOJB2/ZNjWJqRIZyGIHKmEcMNY9mZmapL8n2cdr0SpQaNnTzMwqTzWx6M+GHWtHGmUat2lz11SwjNW0ZhrHgcWSf0Lh/rK2HpsuJoYR5R5g9nmBn2Km/htalMtKlWo7Bt+dVYE1iSEqHHlnmzPDNSOG9eiW81IrEuuchXWHthyN4GH0LKsm3TWolKrlGZ0gzNnfmYeQwUet0pjmooP7+aDNB1qpRpV2xTrFfuamvAbxygjxwwADPDP/AALGzLYpsOHCupidJejygM8DPAwfzWckwKjTaXNbtx/CEYa4R+w10Cxo0WpW9T6HKiVKfIpB81MjmeASQzzwXPaQBW3/ABQYfjPQt72D3fsBe0/RymfYzhR9+DMAmByzwdHKvABh/jNeV7zvCyrwu2O5SuHnq2FCMNFlkAA8/IeGzyLvvo61iuXDVolArNNOHT6prymYh+5w/wBwXTwWPTwTG/eKQbwYrKVhmVSpB6b54KwhzAejrvyhHmG4crPQPBWsMMw3qOZp4JLbIqw7ywBI1ATLxtgaPnGzUKTvd1Fcf8lwwXFSquRga0NgGG9cXn75RP8AnXaOJFHfqtLkNsdYAuOSaVOZP2a/T+h5/S+VzeERl42Xc21IemPyetRwA/IaP2JeIBrstyLWn+1V8yzm0CzMN7DBxXEOfngBuYLG4WuGT2KFPhuGWuw2psCZEDw3DRvQM3ddjoXO1UT1SlABsdhhgmYe81LmQ8C8RtAyDeXhtq0LClG/GmA+2ul0SY3sNc6gMua4LYQHsBXz+Z08LooG2bQYKXGZ/drM0qZ0ZuLTBPYwXy7dS1Z6FIyBUnP/AIbiMJh+dYYarUwzJNGCBmTmpGQJhavkwGHh9mqeZAbDwwbWgMwwVbJ/4MyV6MKcKa3j0Jp6HgexWpnh4agyXlqxs0y3gYKb0BmoQPN5qRzOzTU4elzPu01hm7qJlLP4LxR42TNRzDAlK5nAVCM8yQHnsRAeSjp4FIsAPAVHOSeeCDWPBRz60DpmmjMEB6iEwWOFo570WDZoTTrKBp6Ns9monJqzNJkN6kUj0P8A59NRDZcWmejNmCrzh7+hBTgB5qwZBzBSAh/u06EZtFmklI0UGiiDIdasYxqOEZWUaNggsGT2ITBSGY3hJmSGxUIRqIfWpBqOatCIfWvuHxTpgj0VYZCM2akBA8jaejM5ngrqNGb71Qiw4YB7tWXIMGPiAntEAS2KUYVhwGA2Nto2YbHtFYcswfiIgjACh6hcm2fu06ED5FLAFIBFK02XARM9Cs3ozel0KD4YGmw0o5gpZgq97Uz021YZkdCr5JtqzONn4jihSYbmapKvPeKr3g92rB5lwFEMD9ot4QhGGAaiZOSHnUuV+zrOSXsHfrW0RsPPVIwLYrBmpBiojMZvDP8A4EBg2yrww2uGZmakHMzWdOfh3qOdYw94nzHcupkzAc1XnVWw9o4qWZWzP3ip3qk+fvF0x6yL5nSKbWIskAbDNSnpLbIai5lDrb8M/DNXYV5yYx1p8x3NQFb3ovWRvO+Gs/DebBrxFd0E2JJGba5rhtte014zDerYDUIAbZw1O9WUPDHNxc1ttnQZfUgGfxE0cxgPD1E0dSw94sP1i1BltUleo8SYJhp71ICpZ+8QnMb73M17+t6wMy1XzPDT2KdAoLDOGbBrW62fYCdhst97avbPCvoNHPmvYYLUHSm2Rz009AZbZEHG1NPesd7bQr2QP8idM0aA1iDM8lCmQOcBSE6BotmnqDK+QFSVU34zRt5revM6w6ay9VthtkzNyUZrphk5lUqrL5jxDVe9MceDxFrZ9sNmeoap59seF4DgAvqcNuJShM0fES9cfhoHqa+yWmoRw389i2YpXMvvSPaK7pUxxk8G1WQ4DmGppq4psZvPxFlbaFwB5hqJGCQIzXI6jWi3/Wg5NlO9CXQoQa0cPZpKQgPTRZrWPNSAebeDQfbAwPzqP3pAg55fPBnmc6zY7/IVDr0s9hmotgcUZUOb9ir0YONIY6DNdV5nBYziFZMG7aacttgGahF8ZkwBENm8zv1G94JroXLOGPEWUy6dnXN+1sHgBme9dIeezWy0h6Sx+IlGDWLw1RSfx0DNSfZ6HFrhyLCvPSmQBtvYqqNPf5gNRxA88+8eo+5moRvaL62wnbqECfE5UM3FErFVg6Rt6+CwXrh/3ZqC9JOS7m+eafMbTp8zmZhuNubVYRglAwGCqYDLDzobFuKVROZawNzBLxxkKUDf94rAHsBRVWmvwBNxY+fPf1faJGORC+n1tjeDbiz8mpP6upqKvN5AZrpjhRtas16cAabZoTece9oqzoR85mtDaaGwld0o2HjBhZnWM09GeP3azte21OTBgeA2DKdpsnxdR9zYqelW8+9474GYAtNGo5vNAw2Bhn3rgtZmfUoJjp5rPvSc+hbCNw6Ye/a5zx/nT38ldNy/anv1rHuGB5lw0Yb1qK3atOpTWbGZms49oAC6e7wIrx4CokZmpVJ/TYlAAfOrVlmC8GdR2R+9c3u27XKJVzYof7P581jvwOrRoYM+G+asuQ+667fQuK/yly9gS6/Gz8i2tNvysyaQ1Et9sJJ9+a5vodPSsK9PYprWo+5gqJm4aO9/TgD61T8VKxOh25zdZgyY0sOjYuf2w8/UovrJ+LotH3vAsfpOl2AKrSnh2SwNVkO5+cqnIxPZfirnj1S5zVgxWDktd4Auh2fb0WNCanS3wh4dYGunh9naL4W7jQzMQcNtV9VrdGpR+O5n8ioZ/E6nUrNtt8JmH4K41PuGuVWac588PkzXNze/HG2w7LXuKlOoMcH4lD1nT78+hc3rd+UqsGbkqjScz71hZl5yjziZs/WZqkeuR89gPgf0L4/N73evDrFNhsSY+uw3gub3hW5UCqGxoG99AJU28LnZj8o3K0Wg6FnJ8+oyayc6c5n5Fx3/AHG0WHmarMd1GKVJ/Qk9Z91Tz5r1HMCP5zBaiBxUrlHY0IMGAyH9yolV4x3bUmjiPvsg15ABX8y9s/6t9VDqS2HmXQ/GUd6frEo8mfKqXiPub1Ezw8M1eDawzbPrPBDgwfvM1WHoB4jmeaQTMOhXge/TBGCM4zhomYG1fonCEwbkhg+2Bh5DQPQ2JLQMOdAKQcYw92jZjPmfszWdq2gxqJTgdB+c4AAH4xrmtSuS2aldVQqVNYqVb5UM9WOGYMmrLi1DrlbuOiWrrvMwn+vBR63VeGXDq3pFOptz0qiSMMAM1jh68dVW8259/VOXUrteDQkvgAH5M1nJMnhXVZ7sq7rjmcwfYZmeCqqxMpVY4kS6axcEaTCOS/8AeA6D3o78sO3IdSD7MPvRnQDA8NgGotrDPhb1sT76aiweZeoJ/i7E7D4exZ92u02C/hTw6AePrUJ465DDQlvq1t7+ZGnaxLnG9IfDAM+xc/5W6HMvaxLYjtUassTJPyR1K/lvbeFqlWHQAokT8YwzXEpkxypSjlSj3qzokxh4DbY9qAZgCxwhsoDPri4DqtSqJzMDMDz7Fl+IVzv3JVziPnnHY7E9w6N/mJfNBhrgZ4LH1VmdDq0h+VENnPzrzhgaj1rEOlx2zc3gaoZ8lwJ5y2AzA1CgSQeaOI/+0e5NPTJkqM01rhs862wGTrHJzdfUwU5msXHj68bp3MwnO/BWs+jsUR2FXAghMiP7HgPZsWird500KXFtyw2M2jDMwdDoXoj02BXKPNj88xy2uGYfOCuAvOv0d026bLwz69iG4eNjFbpceh3HZcaBIhYfeI4bzWfgV6m3CbsSlUqS8eGYGudbvXqS2KlQaPX64/yciqB7kOs/IuX35SjsO9IUuI+ekw8DwZ9is7eu0zsaDBqLn3unTPB+gFH4x/z3ccTxOuMBrIXvF2mxbkqkK9TDmYU2Nv7FzU62el6jpTetH7I7x5rVWfUn/Vrto1zfHDDRPyAsfcNmS6DcDQWr4wPhmHYtti2ZpVch0SR6ugsw9dkwMw8id4b3a/cNSiWjeN1MwKIfWcgNij8S7wnQ6M1Q4sXlpuj4wYLJW8DHMRJ0qID0eK8D7wPBsMAW0DqtBtKjVK/JdpN1g6lT4ufLSwPrwVnMpXCuZAlxbuuONSptOMwwggeb36EB1ixLnqjrHCh+NQY8WNzTz0sNE8/IBrpvD30eLKvzhE0/dvEqiUGQbz+chnAzP61dwPL9Ve5y3HX6dBwp4HgDpnmf51lYx4KwrbL9mXRWLcty6gqUJiSYBLa98Hn+RRIzO3xFGBe0pmLMjm55AzWt9H6ebPEmFUeuRCAzjNdhn5FlYABAozspTabAo32Ul3BBncnctOPPRA+v6FI1fEi3r1ufiJLfrNv5zZu8Dj9BgvRHCj0jakfC8LOqtcZpVxUjNjm5ewDAD2fwLmUzhRxQgWvTOL9NuONeDr0YJUmCB5ngfZ1rH8DZ8W5KvdFq1VgDCV48bV6AMFsPYHBm8L/4hQKqdWuOGDtLk8rvh+2+frXUNEwd3rjnB+Bw2ClzfVtVkwJB+BJjx+jPzrqEM36VVDpVNfh16PgBmyz+0sgq2hcYfFOsmoTNYgzJRwWwOHIAP2eXsNTeWcD2isRwgUemvnWfVQSZp9bphmZrnvEXg/Tb2sap3PfDkw6hCZOVGwewwNdNUjDmYEiCfsnwwNY3C3jfhFatOvazahR6xTgZuuiSWKvAkHsM2M17lpTzcmh02V+PGA1w/hdbeFGqZn4NQpcnlYzwdbILqFnyZ3qv1bODA4R4K8DFUHhQ/AqV4ToMHP7QvAyHYABmZnvXbWYbEMYjEToYZAA/QqpnYrNmT0JghYSYEGpAHPMBJAPdPBmB/kWMvCw2LtqlPcnUYDag74wBsBk1sGXs1Li/86jC0GHTfVoyG4Os86+eeBms/JMJlGkQaVE5mRnypm7MwNnz4KRdt1VGmzWoNOo8aTges870Hh9a5bxpu2zrMkW/c9Di7+fwqWlMwMAMOs815hDS23Va5w95SJXK4F1U+U9oMmbx5xj8/kPrWwtK4ftV6wqreHq8JJgzLAMAwXF7GqVl35aUKzn7nptBqr7xyqVo1JmS9+fA+td1sOM/ZNqW/aumzgZmEx4w2Aa9gR6wESBUQflStHPYAGGYGazoVWjSeYlev4zwMHg9pdhroF82SxclO9VP1VkNfsaPef0Lj8+NbNmXRaXCelH4sp4zkmf0d6kaCTXp0OnOsVGjT2ae+Bh6xiPAeAH0GsDVaxfFscKKxOlyo1t0KnPA+Z6x81JD5FiuJHEKuU2s0ylP1WBRJFEufkuRz2AwAZ9HeBgaqvSW4r0PiFa/2ctyvmEg3gBmPyxgzJfPZ1n0f7CsZej16h3VS67f91XjDueQcN+LDCWGclkPJguOcLq3wupV2wqlf8F6TRDAweadZ2Knn3JX+GNZaptOcgVKoMBm9mGxn6DVZeEy3zn0qq25Q5JwjAzea39f1rl5h6DvyHwntWZTalbFuTAl1dk36bUPWoPAffvA+zes1WPST4mxqdCYgh6nlsM8qb0SYHKvMeQ43QsTf9sVig2lalRuOpRtKbDM4DPkDZszWVnwKrStJuqwXgalBmAH3guQ8PWsy53KPa8edFg2w80cAJrM6QZmbMo+sADoBcq4o8S/tba9v1ym1I6bUMDYqXKHgbL/AJ/z4LkQVWjMtNMOQZPKMHmefnV7WLJpz1OhVKh3H90P2LLodGa873R4c5vmiVwxiSqq/rGGeHnMNixnrKVTRPkXzDXDQMOwwXboFEqV1Wa7TZdSPmIp+MGGb0YPOCxVV4XP2xyVzsVkJ9MPPB3R0TB8A8i6eH2U3AqPY1z02LEqVv1wI2GD/Nh0AC6tPtKdWGolfg3rGZj1Q8Ho/QD3n/OuSUq8KzVaDIo2vGjaEnPPozWzrZ0ajyLfptsXj64pT5gb1PkBgcaUf5N4JcdiGqjXnVaPal4W5Y8qGzHMAOeboZ5h8hrlr0+DW5UTUYzdYPPMOjBRHoz516oBLceZZMzDMD71o7buGDG4eVWxJ0FlnDOVDkB14eRRhcJFVZfZtqbVac2B8qeBgYdiiHc9HmWlCpT9OOZN38yYdAB/nVPDZfk1QzclPRoXLaGiHQ9581Ihz3KbRnaHVYphT3zzPDesV2jxqk/TdXQ9kYYACDDWhQq+5Fksxw/Zns8MzUu6gtmlR5EEJUmHUGDAwjuhmyYfIarKPUqbMhVNg6yEAAPMI5hsP6PItma+uEPXFLjtnFk8xslSafHPwQ+dZyqgxAdalwWMKU9g8z5wPyGriNXpx0sKj68NmXo8qGAdAf51ipLM7SNhucb0c/OtRrXq39oZA1KuUoGYjAaOw/0K1o9HpR2rU7qcYqpnSDYAI5vbDB4+v5OhYVl7Wwblt6zWABgt0EyJJ4ZViDTr45OW5hrUZ7oNgDzAANe2IVNrFNOaESc/JMz9i716J+ddbtWm016bWIMqVozZUMH4fLhgBgub0S27fvOHLYcqL1K5UGAORo5gHnRV4INEqkeDQ7jmT+VZD72ewPyfIuC4HSLesCqz7ojsUBw6lT6cDFQrEvoBkM/P86/Q6iPcJLV9SMMQZ/rUIxyo0TkzOVm9vMz7N/1rwfA4haPNzqHVWTdm0QIVbphs56wAHYfn71sPRyvzibTeK9KoEG4JgQqvAM9J48wZ+ff14Lbh/spe26xDYnkeff2d4KsClct4bakWNWGL2oZ3O4GjNOS/FmM57M2TwzD61oDgL6sc2xnDjPqJMN9kD0288Fpno29QZMZXtixoVt8PaZgf0KQFYfVlMpTDwaemqcKa4y6be9bbDUmpHJA2HAw+RZepUQMDf01q3qbh46dh4A74jYfnXTw83Wi/G3GnjYZd1HwUgIYV5o2KdBeedDyLqtesO3Lk3youD3nBO0qz6NRGAbiAefnXb97gw4O9Gfhumw+GBh2IQezXTeKlscy6dwRPan7b5zXLA2Evo8PN2OblWbMk/OtHR622Bcq/39BrH+JjqIgNbXBFthWIb4SNTTM2jBV/LYElSrhwaCJKfw2YZqzzbNc70EN7B0G1fczgqxllgC1FImBmC5bdEWvYFVABwzVnGrGBbzWMZjPgGop0ZmpG6IA2ZrjuG0W6AzUm8APUUhmpNrPAybLG9GzJcD2i48OmLahmqgB5tuJ46xn7xZ8Dbea8NRHpmi7pr3BtrfWWYe0TWsBqkjSTNTmTzWeFpTyhSQ2KU90JgzAB3oIPQj1sEjeipZsH4eot0g5kPOmnpgZ+0Uc2cFVHqA7gvMKXXOB50gktqnz+dS2dM8NRXcMlmB71KA8lCPYgB7BcrVbYN4qIewkzzDiMMzPxFINAaeAFI5YMFC1SYb0gNHJDemgUCQiZDemlLZ08FIAzR6OajmfiqWHQgAGcB6Ey9Gw92pzJgaR6Zgi0Qw2JpkN6ln0JlnrRCcDLealMshmo7J71NA0FgAN6SizAbBpOg8okySqFK91plOn1ppWgYBvUjRUcAUoOtWHo0ZwPEVgyeHtFEAzwTwIJb09sGvZoGTzHUSBOgglshsSSA0QKFlh8UYI0gNA7nsUKSy4Z5qUCTweEiELrDBNBDczzcbUgAwJS+sEFccP92lyYf1qww+KaNPApJNNb/DVFUob4B7NbAwWcrZmyWxdkJZWSzLMNM1n6kyZ+zWimSdp6aopIHnqLs4bctoXOTozWmBqIdSfP2hqU90KqkgGfhrtY3Y3phn7xRzM00Z70gNa4Y7IzUfD4qQaQBsVm0HvVnDebDAFCMN6QHgSUjbRRg5nDxwBaGiBEB3QCUHz71z4zzFJl7RBY3wtot2Y5LHY+Bgokyt8s0eCwUOvOY6epsUg6q4Yda5ule119qnzPDTNTmXnJOL7jhrMwPGkLYMst6Aaai4bQkQHnwd9oalnqSTDNwwwPsVIbzjRbFLgSTMtQ1zYXtoGXgzBsFcMs7VnGT36iuI0k1zW2hdsn7tTVn+c3+0UjnD865mywMwQm82AqpOY4Z+0Q6xmmEJr0zBKNMzJV56iiPPOMlsV4GjembfDNU8+fvNxxQjqTnnUF6S4aYAyTzUGSzmHiKQbyizJ7DIlqOLpiHNapkwGHg01C9VRWd4Z5p16pN54Jrmc12QxuAHsTTJ70Znkmeg0eLtk9oJ1V8Z48VLA1zXDeCM8CQazn9SR9aB7oXrR95r4Jc18FBM96WsvMWz2l5/BBn8E0DwYpGaYLtLzzFEHWousmZMzBXhmw/Fexs3wu6jt/eGTzMFqLSrzFw29ElNt+KAYGnpknnIEiKfvwwWB4aVL1JXqhR88I7h4AC2wN9P8Aw/8AjUTln3h8AM1d6LBl4isIcBg1bK4ZE6bODrbUKTGf7210U4DGHs1VTKOBn0J3HSxgRnE0cZz3ba1B0dzLY2jClNshvb3rp7jpFbFNAyBzZs866HD5VlgNNc/gG5DPUVgdz79BvZ865uaNrj9C1u023oB4Ob1yWSeb5rYSak49IwN/NR3o0R7DUYDNOH+2i/1simtZz+pdD+z0HSDBhL7PQQdzNgAwW/0wx6WHCBVXvECIZgpcOz7jmDrnB5Zr5zXRYDLbLvgK1ZgPyXVnfuf6NvHrOThZlzYm/wCqj0gPrzU2j23VTmA25BMz712iMy5pYJ31awub7LX8zNUqA/DHTfbwNTnpOCsJMNsA8NVLy4LtthIZnv5K7Z8YFkuZ3Kyhz3za081FLiD1bOCDBuSlzGvXPblKIzl7ADvWmu2sUbmDgnEk5d7y4zxRolD5B2s045Lx97OGeCjuVhCva4alc7WhbDBsx/Oa5fUpNSB31bWakHzgCuLbn1mTHBiC/gZ9HkWXuegsHWziBVZL0g/yLh88z3C7ONasOAEFucBun5FveG9z1Wg+HQIPOb89V7ZguTxrSnUqZ97i5/nVhAoN4zJQMQXzjRz6MF5HMt6lu25KbUrSl6+EyQAZgyAZ4Lykd21iY06w4/g6B4GC6nMpt6WlaWDB56/WZhvXL6aFHjC65Um/FfPPNbbGo4aXbEok04PNYSJR54LQcSJ9YegGcR8N/WAHmuUyaxQKVKB8H1rqPe1nPMZuP/kTsGZhyaiDWm/mH1qRXoc6lU033JQGZ+RXE+t2rWHdeJKAA+dY25Lh5zmMGDBrsM+9cdzsY+pamr4jmaKMywB+HvPyKK8esWori2PVpyuVqTZ/kWK00GZ3K82DGGn3qvkz33j9pmtbWDodKYMAbM/kzWKk1JjVNwIgApMI8mSeBt4GCr2dfLsw+tO1KqvvZsAfhKq1nGT2LfwNAzJMNjbeZqPJB8PEcDBVgSZeYeOYKQbM6YYAw+ck+9bBc18EbJ5/QmnobkY9N/rSA/dqB7Qr17cZQhGdDtGjh+9zzUeg3zxw9vXLShz8+iO1s/jzQPcQuJsCG6dY4cgDX/5Mk6wf4FioHpSwTqkum1XnKbEhe21WV+mcje1K6vSCqWfqPhzTab5NaTnguecQqr6S1sUYLjrlxxqa1nomzHZAwA1z/i76cNVeadtzhfBejB/8Rlhv/RgvOUy/+JNwug/cd21WpYHngb2xYXY9V16f6S8+zTOdcFt4n/SHgMHmfoNeQZle4jXPV3bclMHcM0Dw1fxlu6lxjvy4aM9b79z8hEBneHnwXMrVvO4LMmHOt+doumGB5hmBrm2tpeG9kyjuh065TtH1dnsz3gtRPuShyapIBsM8O9ZKm8QrgmSJevvdm9Zsh0LOTJj8OVpsLjv9bohrZh02sPm+/gYAexUtwyQxCK2Zm0gOpMMm0ZtgB4bw7FCeDmZnNMOZx+8DWDa0R4D0k1Del02VrqwZwrFZiUOC3nrnga0HEK1aVaUCJEp2yQfWumGOGPOpGbupBcPV84BmrCpG5MaBuWf3jvXSApsGlWLEfgsMg7hnv3rOVK22OSaqsps8D/pEc8wNZQYYKSy/DLUDYrp6G/W7ed0298U13jhXAtyZb1YbcCNnF9i7IDsNYKmwKU9DmsQdHVfe8Y2TzA1oKqA9FuSyDblVJmM7CMAMDVYdHtmHRIVSiXUZ1PWAHgAOxS7Vuqq0FqXQJVNgHSje8Z51noUSSzEZu2IxOgvRoWsGw/IiHcOJfCXhrXqDTJ3C6pBVbgwzNoDM9YPIuX1W82AmUymnaJ02RCPB4ADDet6dYrPDqqHeNpORqU1S2fyPLjtbvav3bcbtz1V8OYfk654Bgtbhabdsl+NWTYiOYBhngthMnhXodHnB7VgNA1zeSc6vXNEY9tLlGDIB5zXY7w4acW7Mst2W/ZTOEX+kR5IHh9YLjvhFScPn8G4kQGah34IWZNVt6bqXBmbIfs2AdBqvO551Eo0KdKgxnnZWw3mXt4H/AJ1BZuGq1t9pinMA8YHngYKMA7kolYetmp1ycwYSM8wMwUeHzz1uR+ebjauHuu8Fe3txUqsn1YxoRvDPB5nBXFw29Q7SqTV6wYmcKaH7PngGa9wMa9ROHMm4aYxOqr1NhTf2k2gzwPzrrtSsngDZ9vf6H9ZR/wC0c4eZrhUyGxMmuy23Ho2ueeDR7AWlvDTjWrT4jDEwN+GcgOtaxYr7b4dN3zVqnUbci+qrfYPPmJHQCzlVZYpVRdprE4JOhhvBbOz43EazKCFzUKcy9SquYHMg957+9Li1cP2tkR5bFsxqa7FDfpH7ZXsVVbpvLU2mfewMDDAwBRGbel8w05yoSWtYM2mnt5grCNAOpW5T8POZmtbJt5isWfT6rBlGzOhGYYB3rLYnctenDqU99kadXjoRxgOTpSfY594GCzlNuSv0e4wuDh5Sjja7Oibxw8we+c810jgDxFl0pqoWrc4Q2ae+evzdRewZAN+YHs71n+JcOx4DuvY98GELPD1YHZ9HyLfYooHE7ipw9r0uv82EP1v+04Mhgf0L076NletJmmnd1lv+tbjm/tjNWnhzOC8SV54zlH/OMmZH7OYPM1Fo9epTMhrAKrAmh0S47xsqh+mU/jHaV4Vl20ojEwJtLewnnyeYMn5AMFo5j1Ro7UhyzosyqgHWB4PAYLyP6FXEu1bGvK43LnvyBDj1Rljwp285J5nnvXs2TGtW56cdx8LruA2mAze9XPZgAf5O9UKpm/2Hn+VqtsVKiSD6Oe2Af51dhMnf2X+NZ+3uJ1gXtF+zFRuemz5DHgPA9sPP6FcXhUqNw6pYXBVakzGpTB4PA8GZn9CB+NrxubNum+NNPN48+s1FnhffJH9lXIECR38xv1gWEgelFwvqr+nSqddU9r8WJRzeD/GtnSuMfDmY+LHPVVl0+gJFKMFYhUHivWIGf20iwOSz0PWdOezZZMOx7yLYM3trUv7VRHIEyhMM5vOsnmeH0LLybeptB4kx5UGKHqe72TCTEMMw1w7/AONM8MaIxZl+XLZTZvHR5TITYYH2BvQW0/jexR2oVcbt86xbUrY9VqTv5M/3wdi6rDmRalDanU1/WjygA2T84Lg4WfB4Y8S6e/QHzZtq9ZJwpMHsjTeww+TrXeIYVFlrQnAyGHkXNa2Mv+w74qtDqEuyotKmVV/BhlqXM0QAO814q4wUGo2TcIUrixTabGmvgf8Ao+q8/wA4Hk+Re67w4kP2fF1LffpT01vrCceDIB85rxjxv4o8KLzuGoP+p6bMuN/D+dmXtaEyYeTPesBwKifaYLhmzrRs7kIk0ADl3T0TZ+j517AsP0tLLs9qn0qDLOTT342E+n1EzM4EoPIfWfWvKTNYnU12sBWbqjcweD7JhvB4PkNbJ6lReKjvqOJFpoOwgA3p0GAZ7/IZgvYsd+ur0ooNbn0eqg/R6xHpx65hT3jjSYweTBTbV4l2JxI4jHxYqUF6A1Tjz5ucYAAHhguBUG6qHatZrdKl2JDrbs1kGWXjZz5bZ1/7aqeN9t2raUOmTqdTtGRVwzNnPY8H0K9geNPFS2Lz4myLqvHkK9ICTos+oT0WXmGMMDM1h7z4isXO6Pq63I1NpTAfcIJvZgz8/R1qvgU2x5LWvBppw/B0Dz2ABrKTKDccN2FzVDqUaPNeAI0g4xgEkPOGfWoiw1G0JlZ16zKe/vc1pbY4hVGghLf9sDHgw9+GxNcS7SodBrkSnQZXLA+AZm6eeZqnepVKgXCdvvzpLIB0GyzmotC4uqq3Hc9v0edVXM48V7AI/eG9SDrb9yUl2c2xPDlcOTDPPYqSiV5iNIqDD9R/m8Aw3hsNG8BxsPsjXMI59YR+jNY3AZZqs4HQ8MHt+G9WcypU2G61B3nHzzwBUkaM+zN3v6Mc/bGlMmczNCJTnDedA/BeAOxRhbXQ62/R3QYoFSkxpEowA9I8NikfzUczlK5KmPfgmHnWfqU+m15qI/rshNAMJOAYBmnY0CIdLlhLfA5vt2Tz2YfP5FER1qR6lAodsNS4tZpRzJs3owMA2KngBLqUjPPwYXjgClT/AFB6menHTalJPDBnfhon51Sw3nNI3w2B2LsgXFSh3G8+b7kTOQ+ZmAZ7zD6EpNBueHS49clxGfVT4Z8wD3QrClVVi3qGFzttxnqqxM+7SOs2Q8n0LPnW6idBqdOfpzJx6vM53MHjDevGSRDqUoM4mf7L43zmrCZXoNVobreeEg/dKPbb1cpV20ypPvs6r+DBg0eZmwezBbCqwKbclXqbD8TwmAA42ieGfnU3Dbbn9yPTqxMhG5vPlgAzWaeZfB0HHG/CM9hre1uTZ1Nq7phbgTI7HsQM+tBDsmjX5pP0OWFHz9zgbzP61t45niinhhFBwD6zAABQpkblnTiGf3hvsWluHhRelBwl4etaeAYc3E34fWHYs1AgMTyPUlcm77nmAMAM/IunexZUepZnHo3sXXzwB4w2Ga01KpWEWoUeWwByOvA+w+8FSVuB6hdp9AqTH3hjCaZge8PkUv13OrdwvVJx8A5rwdIGcAD5/rXNYvbegMSZBm/ny7HtgA8Fo6ky+9AaiP02NGhMZ8nIMM8A+dZQAcjOuvxGM9+B78N6sJ8+sxo8Rt9gwCoBmyfYf0Lg7R0C27YgvUOFE9TnJm1HowPrXU+BtKls3K7OgxPWs2zWfuelJwMDPrM/P9C4pZ94NsxYTEqsnR8zwjSDja2HnPBbWlcOrtt6stRKbdsaTCrx5s1OnvGGsH0diy2PU1N9LSgWlUqPbjdlhAj1SpAzW50g94Pn3gC63xI4lxbSJoOeZh+scOQZeMAN4O8/kBeCeIt1U65KpNgzqPJ1QjADMjveMFRVi5LgvORCl3bLOfT2IfJU14zwOMAeRd0cyX6K29eGtb8is1zRZ0GTewB7MzD6FZ0GqsXCw64AYGweDwZ9B9a8D2xWHLepsQIlcn3DImwz5CC94Jw8/bd/1r0B6NN1fadqp1WhvzHnQBiLPgmGZveQwP8AJ1rpjmMPQRstn7tQpMMOY1FcYZjvAwPvA+xRHmdy6WOFU9GbMPZqnehuBIPTWr0VHejZmtotbKGzn7wwSkz32WgbU2pUqV1sGs1P5pl3Tczw866Ytja1AG6rTtCW3mC5ldVsMRpGvBY7966LDecw+RU8+G/Mmm3p5td67OG2TKvUqK9TjwY3+dZ9mgzpL+gDe810qTSmAwbbb2d6GBDbCfpsROhbfSw6WM+xMsA3ymTPyAq/kKrGfNtjsXWwoLE+Kem31qVTbPYjNab4J9J0ue0qNUpLDTmmbwGewwXQIdt02Tg4+GeCtWaPBZHTbb2B2IwAGXdRvoXHfN2OmIAdtwZL/eAeRS4dsQWT8PNBzmCNmvMGeDbi59tseDsymsAOGmspVXmGc2NTetNzJyXfEcWXue2zelAdNcMDM96qGNqdmsSoZ6eooNSr0vmM9RTp8Bumyg5oFWVuAxPYzpW8/IvqeMJHTbqMHd5rRM3bE/HXKns2ZBtudYJczgtPmhy/Tbtca6mJJ6e8FIkyW5LWxcfh1icA6eotBTbqfZIANxc1+m2jmaCSz4puKRGeCMPtEAG3Ma8PrUflpcN3M29ixbrCTJlm1rsN5qnCpcy+YONmCa9cGfgINBv2iYQdmPNgYb08zVW8QbcNVLxgbRqODy2wz22DM/NoHEYSVRRpJ8vp7E6y8+B+Iua4XtoAMFNAwVLDmA93qaDy5rh0pZyfvCmgb5qq97qK9hgBtAuZaskgoquJMbeqx5lzLoQApENRwZcUgNgoGnutIJOCN7eo5sqMCay8ClgeSqQDFSGTUYEs00Ab06Bp0F6EB4EpAPKPh8Ut4IJeso7zyAzUc1QA+tNJ4ARYfFWBBSGUAAnQDAlqJbPQnU0BqQAKQGslzXwQmAACgnJwJWhbc3+8Rc3+8VJzDiMHjNMC4B7Mk9kahQ9hhmrUwb0swUAmZmBhvUjWbMVnzecB3xEbLz5n4biYF2BgpGfwVSBmpGbgJgSzQGopyfnQcy2agOnvWVuE9xrRZ/BVtVhsPe7W8DnjwZklhsWj9SMB7RtR5MNgM28F3RbmuGJnhuNVTzOZrWvQG9XxG9ihPQIuZ7N67I5nNcM5yG3PMPoUV4MC01ZT2cD2KtMMl1MQZ/BGDzeajmaAD3qkrAzbPoVebLgGpsCYDJHqN5pTHgeMPDwTAhAln8EZpnP4KwsjU6NJ2dag5/BLP4IL6BMMHdji0savNg0GZrBAbnu1ICS/51zXCotuPWTDxeGauKabZjmC55DmOAa0dHrDhugwua4bRbasmeSlhJwHTVYzJbMPDNLmXAXBcOza7ZezNSANZ+NUjA/EUr1k2HvFjhe1xkCNk21n/XbAd6N6pG81gw4vcG1rPqsGMHiOKq9atyfZntVYFKbORrv5masGYGYZthgrZmnjPJAZ7FZhAwDxAVfJhuGfkRSnqTz5tabHWqF6NWdLNxsMPrWpOjuPH7fBOhSmADTMzNbOdz8DlvO4N5mr2j0R+SOvLfw/dYLV+rYoDsiACpKlP5MjYYb3+daxew1MpuiPgAqzRMz8MDUgKxLPrRvTH0Qjhs607m351CPUNBg+fQrw92sOcYZ9o4oj1bY92ainDc94ojzLYLbpNrDn3Hk0cxtV5vOKbTaa/JdBzBLhG0uNqH7Ns1NNl/De2tBAo+jH2No5NEN4NPUAFx3bVlz1AUF7UNah62JeBm2+BqlCMeWmtotNojLLnvFze4Yz9BvSFObbwa1t662EDesfxgpT/qSJUmPbMvLbuQ1rLzZgDgd6uIBqssCGxWLUhTgMHncN61bNNbZDoXHfM1RDPYoh9atpLIAKqnlLUQYYYNqvmagGpHQo8nxlTJTzJjfsG1FBWQU1jmM396jzIwAextdMMhxobeWopsNnOQq2HRJ089j5g151oqPbx032b5n9axshYAyANKKbLjzuDYZq6ZgZ9asI0BtnBxcd26cKqm0Qw8dxg81oGdgp3WbBo9RVr0xv3ax22WHOfhpc4qnmdyRzNqjb1OkzMz+RVMk28s0Otml1gggnpoTnhDaNxxzAFNCHmlMpUHlz9amYNfIoHIrkvO2AmPTvWX5MEB8abYrFGkUOh2rJnyHww9j4KznEsOHMZ06dQ2HnpB9/kXGpk+dTROlQZZstB5FzXalnDueVRKzIgny1NZYePNZ6sVtg6ydRYl8yf4uGCzVVeM5D3f8AOfWq8HsGljY6Nb18uBVGvWVO9agHRHPozW4k+ka+zNCIFoxg+fRVfZ9Ho1pWuFwVFgAl9+e9c5r1eocx96q07AJZ+QFcD2HaVVnVJiO3VYrJg/2GCxXH6zLSegHUorAQ5AHmZshsVlwcrdsRreaqtcucDkH2GYf66i8TrnpV5tHSqVvA/n61oPJ8Cmv3JXAptKM3mvxTNbr+TqdRHdCUByfnAF0CHaU63qMdRbo0OHh3ma5/G4qXHGqjptywktd4GC+fzBqpG/Dd02KPJ2d6y9bnzpMXCUwYYdAGC1tY4l3BUnwf9WxmQD86akz4lzgEuszozOHQGGCxixyozfZLyKbSnpZu5tq1qVpRTlHoVkHgDvAFCCH6tPQCXrfPgtuyGxT58v2bhqEb2sCkVJmUYZmq/MA60i2R02Q0g8QOtNZsMj4bYGo5mwZ+zQdfs1cQ1SOQOZvbcQZuQ/AbfMPoQ4P/AIhgi/vFQHWM+tIz/D60s/8A1fIg1mMutWJtyce+MsmO6xOqPqeI/wBkc1z23rwuOj1nXoD+tIm7PvG8M/OszGnvzHwYqs4zj95mq+fPOG+fqp/aHsTX1u5ztndUO9YdUdnVGnRvH6zAMAD8irLev9i3ubcl0rnJEoMAPyKzeCuTLcafnTjMj8655Ph6MgwNxR3CXPrDc8zcbDDM+hRM28VFzwLTTpgYNA4ouFpECY5Gf1wz2fInQk60rUwM3VHh1h+G06w254Rqws8KlJrOpBbZzDsNQq0WfPzd6DB0EFKmPg1Lbz3mGxKt+sjq8s6k2yDufuVCDYmCLbDhdGbk1R2cYeKx0KDc9wzrhr0gJbmxg8E1aVYOgzeaBvMHAwwQVjkXpUioxAw1zzwUYW20l71rTYnKuZhF6wVl9s4lSmeo3IOjRz/CBYKiT34eD7bnWpsNnmZR4OdaYC9ZQYxy6Uw+cmJngB54KPAN82mqbFbAPvOww7DVVWIz8Cfg+4B/OCuKbDl02RElyzBkOvf0KMMWjr1Vqtq2pItj1azJ5o8+YA8wUridWKdWLQtSXTpwSTYhgB4dhrJXPclRrFUdflPsm12GymmberB0h2sBFziN/PvNejot1cRbVk8FqParbnM1MA+8n3gGa5VDkhyrSgvGB/QkB4YKtrW096dTZQTmPBkMHmyfkNfolwiqV43zwjiVV++KbUnjAwmPHGzON8nz4L84p89+ZvPyYAtnSp9Vtim89aPEZ6mtVFn7zHZewA0sWXGP1BRKz9n7crHriODxnzwbAPf5FX23VWKPSAqTEv7wffgsqAU3VzqL73zvaykTORMTg0N85Jh515gSNGXXrvjxevmpgbF23jMFNo8C3KAw/nHAMM+8+hcss+lPhctCfluPM5zAzMDWo4uxmPtkERiVMk9DxnI6M/kU3Az8yNyzptt9ihTKxWZMUH5zj0xpvsM1ZwHmJNwxTnHhEfPA1X3hR/sxXDgtnJOOYAbJ6yjAsLevCdPwozmbNPDojg9vz+taiqwH69RmuRY1poZhmfWYeRZmw7Ji3CMipBUeTeimAfWhB71lzcQD2MLMbK27bt+g0uF/KHOkw8z/AGeOe8A+dWd8s0e258Kq0aVnR6jDzjeNn/v1rKW3bFSjOtVWlUZ6pH2A7vBWdw0T7Q0sJwUoKbLc/onRgucVlKuejQBm06o0bnGqozoZn0AtdW+DjHq2n69+U2BHmgAAzyBgef1rnNSCpWxIp51WjeFmZgy90PLXVW2Lm4tW1UOIdNuelMx6I9sozx4PM/QumIHN3rVfgVR6LVT+7sGYHIaPMFmWfGnnBgtvPH8gLob1KqVt1SlevJwG1UQzMwTVveqoDrrdsSwCqzZOAOu7AMF0wMzbbNOmVZqm1XZHf6zDrBegOF2nZkO67jsOqz4ci3gB/e9mEwD7HmVy2HwuuCTctPcNyMzHfe0HpEcMwZz7zXVbSn8OeDPFy4LVuq4Ocp9RgAEapu7wA8D/ANdWJde9IThXflUp53Bw5CNUJRgxMOOeBmfnAwWy4LVux6bcFTsq/wC/KrQThSf5qj1nxoRh58D715XoNtsV6/HYLDHPtBMPDszDNb7ivclDufl6c/b5hUITOgcsOsw7E7h2q/8AijxN4V3bNct+42YccMOQ5FkOWqTHn+deprM4r2dflpR/tBOjUqRKZwksu7N/n+deHLqsCnQ+DducX6O5PDQNinz6NI3gB+cF0XgJc7F20g7LviLDCWD3O0o3T9sBnvAzDvwWPb5Hbrqk3patGp862JcaY1S6kGZuhrAAH0GH8CsLA4x2zXry+z92037PXXCZNl5l4zwP/U/OuKXDY1zxuLFTp3CSc9AjsRgm8obxnGk4b9FbW3uNnDa5+I1El1inAcivRvVFeadjZgzKA8w3/nXTH9wek5NEYmaQTogPAy8D4fX51cABoKbSvU8cIPN6wB0H8is2QA+tZIcS4923XGbZqtSpXEKZRwqIaPIxIzODz/QGZnvw6F5ZoPCWlej8xRLq4mzoDwVTNmZHkRs+W8miv0QqtBpVVhm3OgxpOAHo6zOeB+deUuOXAqlWNQ4l91FuTUo8IJRzAkSdZl48NgYH0LPC4t48r0aDfLty1+1bcjQ6JFe8HWM8zD8/Qr7hFxRl2Bw0qtHtmqnAkPzM5MhlnWew8mfYqwK3R/sVIfiQQZhVuS+DzR7wjfQg4M3tblhnL9f0OpVWRVDA4wQWQMzw+tRhbtdhvTqVCt+sVinUr1hd8B9llmdmATMD6Dw6FyTjfb1Zsy75ES6pRs8lg96seezPA+vDZvXS+NnpA2XcNsg/L4c1iBXacGFEkS3gAGT8/wA/YjMOLfpt8NLXluUqiMhQZhszKhsZzPvWbHhcxolHg0dprixX6HAepQPZ+pnpJshJwPYa13EXjAxxg5ehv2X6qkP6Hq0JFSzZjB+5wDYri+aVdVVdOlWBTgr1v06MDDLMSNzIHKDswPo35rhNBZl2BxIolx8RqbMpXJPHrRHYx5s/k7F7hsduGT61herqrbmDsp7QZqLpmYbD7FY2299jJp3NRqrGqXq6NgbUhnWBXTzNpcQo9bua2KBMORSM5rJ1Z7Y8HyB2Auez7klVJiWw2AMnKDDBoNi8Ei56w/clel3A/SoEN2dhmET2OwOxQabMYB2QxL92GYYKPDgTo1mu1XQA+SPDA+9avgVQW+LVynR6k2bMeiQzqGcdnMzMOgDDvU4GMCmy6lm+ErWalexaaeADM/nUWGFSoM850Rv77TjNe6J/oncPI0imcSuLE6iQJtws+NTI5myGf7k+xeObhgRbPu2qv02C8FMizH4oaskzM/nTAj1upW/VZke44rZw5Eo8Jkc+gDTTNeo4Q5Db7jNNdMMAl4ZgH1qFJO3J82PFpwPMhKwwzDoNU9w29Vbbm+qqlLZOX5A7FtgXVKqUGlXAdOqtR5mjvhg8eGGas63JoEyHTIlv25zMKE9vkOn3n/jWco8Cx63dUelV+pSYEIwwB5k+h/5zVa9/NVzcrTalzMeFPzA88weDYtutk0teoNVgW+Fxv249GpNReNmGYd5goT3KVUafLiespkcAMDiR+w10I+N9Yk1GiS6i5D9cWvVeah07RA4swD68/wAgfxq7v/i7wyr1s1C46HRwtu/X6wxNOCyybIYfQGxREKc0psli1brj3AxBeeBhne1I+dHUnuZdkOMGfLzvHADPPD5E1JrFcuejBOrMvWkdmYKhrwNw5gsMbGjDMAUXC4aOqhR9AKbQ4IBIbDDWd3maOZf9wTIst9hyHA5XADiMhhsDvTtBqtOk2lUJdRfDnac8AAZs+4w61XnTZUylhWHIjOke83g8nnUYGrsatzrhdlOUNx5mWwAbHjwCYHkVPxLZqsCbHYuCqyXgnRglBHkMgBxj6OxR7Vh1KHcbXIywjSIvjsmfQYfWrig3DbM/i1a9Z4vzjetRuZ/OTR556GHZ/ArgM2rwuuC/7LO46MxrHCqR0/VkPbAAAzw/QB/oUeBTaUbQPxXAD6zXoDjHDoHAGKFncHbj9ZWZxJpXroDnH+zdmYH84H0LzaEZhmrtMUaVJqULPBl5oN5mfkS2SwrFScjMaDejv8h5ra1hlit8Lbcp0GdG5unGcoPGzzA+tZy8zYmOx6UxQJkaXS82JmqAGZnsWogRrVoPAR2ohUocmu1SqsA8Ed7M4zGfvg7Oj+NY3wtYUTP21ZjgwwD0kA6MwA8FYRrqqUbNxujyYwYYTHnthsgs4zcj9huxKrQJ0aZImh95ZPeEYz7w8is6rcNNuSkHQH3NaoSuiR0Ab/nUdPhTV2reEGiUusUrT9dxzNj2p7wPvD51nzu1iMUuC5TpMamVSZztNz9yGzYf60Eyz50OZH5GuaMiLAbeewjZnn5EdsWBUr8tqaFuVKmvTYL2gFGdkgzJe79majpHVbJk2rVYbPPVl6perqwAM8of3pljr2L0BYF2vz7/APWtqwY3KW9J8blwwOfFe2ex+T/IuCejZZ9Gs+LEvG6px0eq0isBC5dmNm9GfPzs9/yYda9vUHhvzN4NXVTZYM0r1ODGq0yAG8+b2e8PoW0cKe5rbbmPzGHTfzA888D7FYaO9S2aaxD1tD37xvGkbK7IRFgBkDUd6Hh4ikYOAgeMzHerELRb94qet0duSOHQB+RXp6iHR1vaLbY5+cB+G0YYGaiHGqX9EYNdI9SfIgepuHYtu5jcOSzJM5k9OU2fX2K9psM9LeG9aOSz4umgCMAJ3IwOADcOPs94ndbNOhTc1OZprYBpqdtFDovm/qNnsRvMvg1qLQBAYAfZqI9DzEw7E8Wpz+sVWosugxFYzzPf8iOjxpRvg5L2KbVYDYTQc6MFTz7hbgStBtzD511+EtQexS2XgeXP3rtfN3TYczNTaVc7jxb3N6dVs9rO86O/MhOvsdYLKM02XTYbrD/QYYZrbHVc/D1ERxmJI+IGauLRbirNNnHN0D3/ADrRRqbbjzQMP/todYLfBb1NA822EAW9SgdNw2M810/Sx6WPOiMPewBGzYb729uVJBdNo9EpwRQ2K7gQIIe4WP1NulkbVtU4A6h5n9a0ZwIhibbjAGCuzZbD2YKpmSW42a49tsOZV62+Tm/dG8AUTmWIzQBqeKr6t1th6o6ZuKhkxmHndRdnAyU5vOaumbmAJpmZg/puKwk01zDUVU9Gwd2Ls3DltdMzAAh1DU4JMQwNZ+Mzngw4prMYwLQMDWdz4XFrOlMsM7GHO9XrLyoqbGfDfpq1ZjPm/qLhuHTFrBk96s4b2B4KpBkwU6MG4HFzW2aBkGza8RNPRm8uhAD2AqIc/wAXDPescB02W01ot/1oOZzUjw9JRhaEYAmjBMzJhsv6baMJOYK8IGiBNH06iiHJTC1sBp0DVOEn51LCT86i4FwzpmSde08FWMydieOTmKjAAzTWfwQGaaz+CvAkM9alYfFQmTbzU1k2zwTCBgynQZ3o0Os3/Ure+EgGW80Ek9FOhJA1Hk796tVoRyTzTPWmpOYOqQyG3xFuyINPLTVhGjfu9nnULBvJWoG2Ef2izsHyzgKWyeDHiKsOpHiYaijszM/eKAUx5vVT0aTgCj6Lbx6jilRowZIpFeqsvLTAzDBSmaq5h4nWnjgMYezTPINgvAYPZkj3oAZwUgDDBBEzfA0gPJJ54M0OQIkElnMPDVY9Gc942rbPekbOYLSLGUmBgCz8wP8Ag1TW4eprjx9CqpNE8U/DXTw2wwwkwM81TyQwW9qVvYNG+sVPZ8VfS4eZzXCveDMVCVlyz59AZqK8zgXQuliQIjTWHxQK2RIUSSPLCiRgG9IwPNQ9INidDekzGN4wBtWwUoI3tDzTa0dkHMFIZkuRi1G0j0wU2Gyw81m4Cxu1Qs6DVX3pAG4BgHzq4eqTZksvMeOMP3XYCsKU8w8QG4ua426YtcGb57229iCSD4NZuKQc9gPDQc428scKU/Mv5qwgTHNjakBAgmZuaaMIzAF4aXAu4D2YArBk1TwDbDw1cQwM3fZrjt0eEoAzFV8wN5q4wwa8RQngzJZQ8tnJ8xyH7NV/rV81IryzRyXAX0eNja7ereiPWql57n3czUID/HNGzMYBWxWDMZhSuQbeFV8N7WNWzLKi2ooFvQTDUfcM/wA6ek0qDGDwAwBSGXmw94gkmDykZSSe421XvMuGexas6awZZ6aXq1vLw2Fr3JZ+BSn3pAADa3FNoL8BgDfBWdvQ2IDQPg34oKwnyXJPWua72vCvA22VHmPOPbG069sFRAeAHdPTzWTccbUxwNRDonj6+p+RXAACeAAVbFJ6qb/rVZeFq+u7ZlsfgBmtjot/1ouT1mHYrnQ+GBqNjlno3n/MNQox+1inmf8AGunGuI8NJL9q8TqnQ33MMzwXdjjLHe1KmYy4814aihTTw8TrWg0W/wCtMvYAK22i4UR0pw+sFCmMtxs8FdPPOe7VLJB85B59CuLRhUw406ZPDxMGlfPUQHtjjexfIcZwHQ02MFeH0K7swggy2yGgDeAApcONmQJk+tSIz2Hs1zXbXC2Om4BqaijvPYNYJ31l4WGCr3jcMljtRp6YfvFEOSjkh4WbiqXnnMlCFhrAaaN7BV/MOJo5LitabzmBKQzJz94qTPNSWdfP/wBaIX7LyorzOXyEiqhK/ZQz0sNhqaD2Cztz8SLStUNO438A/wAZqFvPle433xviOUOBDDznGzVFWJIBSWZTlHZmG/14BvWt4nXtZ1yU3XpVDzAO/PBVlmV6dGig+/Q4wRw6D865xx+fMCY/yPqbkAUKNGp0l8GAqX5zDYul3a9XKxWXZzcSGAH0AYLOTLeqtYDByjYYLO4UrDepUn+bn6zJmNeTsVfJh00DwitgDSaN5tmbymgDIedQpLz+rptueEmGyazpgWcTPV7MDVhTbku6mygODOksmHyZ4Kkwlsx9dtgww7zBSIFS+8fe3Dz+Rc1jcPXPc89jmrgqXOHhszNZVnTAnXA6zNOz5MWSQHBiYB9ajshm6YZh+dc6DUk8z3mo5xsy1NcA+tKeeDvtA/Io/M4KsLTT8ZrQYfAFHDTjGbeesfyKIEnNI5L+OxX0wgZvHJPBzoVfJZbA07o5gb+of5Eo0ZjPOVrGHkzW3VAisw3JJ+HglJZ5YtNTTjMZeBmAfWg5Nj3maoQsjRH7JPSWWGR8Mz+hRDMzBFhNM54I00e8VqhK4V8Db/ud2qwYNsQzj56Jy5Z4YHv6Fo7w9Da6rbtl6dQ3GZLrHWGtmeC7Bdte/ko4quy6/wAz6iAzOM8zswNcUvz0pbj+0tVctJiTyU38gB/AutkhcFrnti4aJN4eXixDjO4YRnj2b/8AIuKXtbf2YrJwXKjzjvf8iqp5vvTZEs2NHWPPDyKPh8VAQAx37EGGtKaYzPSM8FOojMU65HbqTgBH781cBTYL15AGYM09h7r86uFAuGxmLe5TOcb3NfJhgq+TSjprRvxaqyBfWtrdpsXzIiUC3Ij0yQHkBZeTAlW27LpVYoZ5mGG8OhdDJWhTazPhnVeVOS031mClUS3osxh1+4Isxn8HAFuKCzTj4dy2KGw89IfDB5kA6FuOD/BO55LAVWuXNJpsLDMOg9Ze9LztefZOhGfNhg82gUg2Ynqs53PGDrfQyHetVxRk0qpVl2lW4EaTyT3tYLOAPLCyWT5UwPYYdYKL4VxzNhM+yrNv0z1POM5p+2AzzwNUlVOdGlAxqGGYdeCRm3DitfcdHXDMHuxV71SlzNkt/MFjbZavaczxz61IqVYOZgw+3hgGGxVsaTg7oOdCkPAH4iyMIj0YNnKOH+dWAYeqAinLeDzhnsTQBiiOG+YHzcGSyYdhgqMKcw0Q0E10KWYbdRREC1t+opbLLD29wN6gqzDTZjhp9akG8Z+z08z7AXQGeak0bQlxIzMvR7AwWPZjQWWgnS3PFY34IvXE64Zh6nWfQC8sa6z6k/SmqVVdTPlXu/sXWL/jU65OHwXPTmA5iK8GtgHYa4Ib06NhEiBnhvP5F0uNDl1KzWaxSqyD1KPYbJhgea58Cbw0jWzcMAzYp386xQzeiPBsM0F20euXU0THqozjh7E2esDWKpt+P23c0KcDAMtMHg9h3ga0vGa4a5RJkQ7cnHGplXZ12Xg98azGNA/Ujrrbm+X51orGClSZFQcqsrAGDzww3vKtuq1alGfpgOS2eYfjZycPOq9ll+NquMN5mx3qh2qzOIoVishaVuGEaIAGZ8wG/wDIp1VquFeOhzn85Adi4oFSOmidcpr5xpod4Gru5Kx6nrNPqrc7nJHt3j1utclwN7eFVfhu09ycwD0Rg88D7DXIqbcL8ObW4NSiPG1WwwDDsNbrjZXucftrQl4U+bG5oww79iw8OsQTakA5BN5188wdDrZXTw2LuiVWzp9GrH2nYek1B9k2Q1QzwwWfo8B/1aAHszMASCmxXnXYMUPvZhmypcaTUgjgxBcjMusH4wSOvPyLpixHn3JeMM3aBFlVKMYHg8yyGGsCr6bApx1SIF3VGZDj5+MZ7zZDzqUdz+uK4FSnMYSMMDNkOtQqrPfqted0PGd7AV7HU69atwcHIcTiFZVcolVoVUwYZPDMwDz71kbtoNfoPKzqyxJBqrhzUbV8ihHxCrNetILAqtvw3jYe14zx9bJq7uG+XK3aVFtW4GzN2iZgy91ngsh3Km35atb4KS+GTlVjcw/h6tzPoP5zVFWLVl0H7Ly51N5CRCjc08Dp/tJh/wDZrJVKq2/W5tr1G5rOqUCj07CFUnY/WZ9i6rMvazvWlHst+qxqrCp0kDCpyD6IX4B/66seveEVHpRwPtMETRkTZnOgGe8AMNi4xedsUq1eKdbs6hxQM7hnwpsZl4NjJ55mea38+t33aVDo7dlS6bDjm8YMwZwZg8HWAA92K+4dXhQOIs+RWalQwol604wizKZI6wP5FrFjqsl4Mwbbc6AwTsZ5zJU/Mnq+P1qRGkt7nHHMAAMzNBowec6FkuMFsWzedkS6NdVzepKeftnvOH0Kn/ldtxmqRGGJ3rKFKeCLzcTfgfkMOxbKZQaPJdOszqaEmQwGYGakfnFxvtuzo1ehU3hrfka5KVVzfCZT4kMIwM4B1gAef/IvOjNtxIcjXpVSmRgY3smD3eunXbGfZ4k12scQpVSZkTZj4RnY4AAAGfkBc8rdViw6vNgsUeNJiSgwZN7PYChbR8V7ktW4YFv0q3LnerEiEBnPCRsAzNSOFHGziNwTaqrFjy2Ycet4czqhrBmGwMPIo9nyaVPpcSlTrfgTKq2GEDSPDWD51Q1WNX7VlSKBX6cD1PN7Wza35/QfYs8IbC2JNz3bS5cqKxckCuwteVzcEzAJ+Z5nsD/fesVxXr06q3BHfcOvcuxGCK965PN7MPnXTfR+vN8L5iUZ+dVYFKfZfhU0HjwBkzDrz70PHKm0aNfx1Kvywqs3YwciIebM/wCfZ0PAtmTllKufn4UiJL+7YQzZA2TMMzVnbcm44dvA/SqczJkB7Z53rAFIt7hLUbztq7bxtWJMjU+1Gdd6PIDr84AaoeGlStiq3AFGu5iqyeawYhhTuvNRcNYt37hvSrLqr7VpcQ7uN6lVGiHUGZZnyfIP/If8HyK6sam2JwWq9C4qWjw1rD1uVemv0itx5zxvGDGz70D2zvWF4tcQoNpXBRzobcY5Frs8ryM5nMJIfQlSuLVwcToVNofE2qvQLffz0YNMZBnM+wDMOv8AQrwt1L0iretK0naPcFNqVbuG1LhAH41MOZzJwww3vsn2YLh9b4r0Ofb9Qsqdap1W13A8GomAMzQPzmay9b4iuWHW5dG4c3dUno+wGTM8Aw8hgskFw1mSEinTtHl6i8b5gAdZpgBcIQYFUjsesgqVPivdYbDw8hqKD0t6qHyrcnSPfm93sKyOtv6oU1zlgw79HM1SsyWIciQ5O8bONoh2YH502I8lljmjBh/WDz4dasqPbz8mqUyI2f8ApE9E2o73jYK2rdgRKVD5uDc8Oq4Bm8Ed7AwBVP2knM0EKGw3sYe1o0vPxgNbbQGqswYFwSKPBiHDBg8NE+sEzy2tN0Ot3vPvNSJMaXUqI7fD9VOZUzmcq8BgHkzA00Z0oItMnMS5nrPW++H0Bh8igbOHbd3T7ajy4NKwhMM/tGsB5qvC0rjvynTa5Tm2TkUhkOciGeD35AUGS83Jpxv55u4Zmtdw3qsGz69zbb8wzmwzCYYdgLmsYem0p+SIMPuaOfQZ9Gams0G4IB8i5LzdzAGeXPYZpl6HOmRZpxYutHYPMz8gLVwJL9w8IJE7XCNIoMwGGTPvBerK2KrVbY4jU+jXU4E+E+fJSY5hswNbvjNe1q1iZCtyjU1ln7PSeSCQYbzBYo6lBZ9SVK4HAOPXsAeew3xjDZmBoKIyEy5mos6czGaenhCnyHQzw/fmrYrDiXeb97XC1UW2zZiRYwRYcQwwBkA8gfxrOc4bMdpjDRMOvZ1mp18wH4FUmgw4D2gZxdn5FEuQKbJt6FXKVO+9+w5F4/GZ+dZ298Kem1Kqw5rs5upSc3zzM81aszH57p64ctHf9tonsPDozBR6IdKmU2bSjqsaHUDPNnmDwBR5htw84kF+NUuSMDN3sTDfws/VrdYmtU21Q5x0/ddCTwMW9NhP6+fNM67J9CteFD06q3e6dO9VQ5vJv6ISz0QM/kPzqFdrPLW9RKdgZzQePDAPnUXAtaVfP2YdOdRqdJCa+GHMNPf5DTUCzHKlTftVKczm1GZgZge/PzmlDsz1k1EiNzoz0jDXBkz2GYdYLR2xPnBNNu1YMMI4dbRs5myazHaOBXFq4KbRpES7bOoNYoVIkxafVZARs6sfZFf+dexeFEy5ocqoWrXKlougHO017AzCZF+cD6DDvXAuA/DGlVXh3cd61xuSf2hNiK9BZDDROKfX+teygo5zJ7VcYDPCGDAfQe81vCbhLnxn40M5YN5mHYmjksGw04wfWGasJNVYgQJcqX7KKybx/QC5DDvmqvXDT4tKoEmTT5rJzQZzAD/j8i0REOkGmj61j6rf8sLoo9sQYJsnVAPN4z2bOsA862AMuGrWDD4qQyzmSMIydAMCRCWDLYCo8xnBpO5/BNTDcNrBBlKkGBmahM7yV3MZAw3qDyeHQ2tYswMHnADYjZqTmWmaQRnzDoS5B83QwYRCyZ3jmCaOM+87sUiHDNkcOtGYSmT2NqFql6z26kWcttVVS4Y2x7R+IZn87y2DITjSOmyj9pvXuzDk9V4bgbRhRth+dZc7SrlNnhF0Mw7zzXoiNRP3H8aN63mz9wuv6WPS5Zb1qt5a8pwzw6AWgehts+4M1pvVTDJ+GCiT8GY56ajuXjDKPM4LOSaweqbbfYa0c83DE1h6kDkMzcPoXTEMbXcatmyHiGrKHcjDODjj+C5+clgD1Ncz2Zo6PW23iNhzfvW3zwbdggVhiZ0ObFCrfQbip6JMBnDUV1JwmNaYGuJo5fJgTq3csgNDR0MFIOmzocjQPA8FuzjaPiAsZWJkoJh6ja6uG3LcJzLOt7tO+raV7+CGarKa9O1/3Xer5lnWDNxXteEcKPSsQbBtWEamxMdPQA0QQ/I2pDIYLHsa4NHSomGpoAkEYPIpZmeCZA169wEwb/DRACSMOtQs0ZmqyezL9owrvls06EDZ7RZDKxpM4H9N9xWXrLRDBxTXoDHkVe9R9Zo9RaaSaeeiSTzbfUuHAza9os/PgPsxTcb61SRq9VYxgxqLaIZ7b6Szg0s8Z7zUuNVX3o4ZmqSY8/qn4iiOE2s2Xsy01LDUWXjSXMlpY0zWANRXfCRaRngmjk4ErBmM28AaiI6Cw8eoCxwvav8AWUf8RPBMYP3ioaxb1VhzTcieyc/gSBmWDXibz86vCNr45Lfu0gmB51mXpj7Jb1HCe+a26UdzcBVezUSemLLwHn9XxDVjUpnLGDeBnn5FnhcLeNJcy8RSzkt4e0WSCpP6+hpmrVl41lhe1xrN/wBSQGq3N8B7FG9cYFpuJhLQJo5LiqgqufvFKZNuT7M1eFkbx5I40lvV3mj5Nw0zybme9ShZhJbD3ilsyQPoNUnqrM1YMxsGgBFpvMvp7WM1BDU70Mk8GvaLJSQcxsEzzypzmZkjZeRG1mb2aQKIzqH7RSwNsEeprLO1SABQuf8Aw0gkn51OFHT61EkuYGjeePBU8yS5mtoRs7MBt5rTWKrdE8XXbb2GtLzXwUKZ4w6a6eG3NzQx4Q3GQ09NVk9netW8y4BrP1IPvBrv4eZzXCn0VHPUy2NmauAPANPTVnDqUQBw0AW12jDOPU2dGFpx9jADQMsmZaYNrdsvQZ8fQf6FVPQwZdwwwBO5eFOFKfxBzYaT1NfAMzBXsNnN0Fo/VsQ2tRwFj3GGZo4RYw6j7Gaaektm+epsBWr1NMPdqFJtjNon8zzcUd214Ukl5s/ZqRAec0kf2efD3ma1FHpsWHF03HP4Fd2xwoghzpjWuxEMwQSfWtNDP1VJ+vBbhmSABpqX61wjmxnsNY9zpw5oFbnaum4tRomzACUgOiUeTM5tx82fkBDcMzloAMRHFf8AkQUaZzJ7HET0l8D01nYE9yMatQn5nqGnSeGjpXNPSAzbW6poYADawtKrbBgGzBaKNWMMHM1wW7ONo3o2aiSYeiGaYZr2ans1JgwXE9ZWt0o3hNzTWJk018HdPBdgeBsxWdn0pg3dTTW/DbO4c5epr+OGCkQKI5lqOAtQ9Tf3CUOGYH7NdPd/ojCEzRMDz00jZWgw+KjnDYP3ax22wzhmYGpEY8z01bPUps/ZtpQ6O4HiONq9ssCZh5qwZgAAdCOMyfs9NWsaNs6FjdtkJllwAR6KteW+RLl21iKSTDzBReT0TWj5dtRHozZpsU+fwRM6jykMwHAPepDMPA02sbLKmgbbKi9CRnkvEOCcUYz9scV49fYDCO+YGu8RnuZYaf8AOGa5P6SEP+aKfVW+tg/9Rbbh7VWKraESUHuwwUtWiM1XyT3qWajmCvbJBMHPZp5mAx1uKRh8UyZrbYTxsM+zVfJmb0cl4FXvb0uEDOZmpsM9ipw2KzhyW8AXNa1rn8EgNADzaF55sAULMzGXDFU5xjzVg9M2KEckAR5tENk8kJs7ERzAy001rN/1Ik7GZBWAaDLSqgeAEYTNysKYZn4ar7k4b2/c9G1J0HOR581ZazZkpwPZtaaLeMb5ojFHny6bQH5h6HYC23DSz3JNDOpVW583mAz5TPYC9BQLStWNI5o6UDx+c1YTKVR5MI4jEEGQPyAsbgeNa9cjh151zkXntDwABkFST75rkDDNs2c+jMMF67CBZ1kwpZsUdl7AMz1TzM15JvylVm5K3IuaVhDalexZM+xY3CmKmSYkmQct/rPsQGcE2g0DMDTz0DRP2YGohs716JoPTpmDD8o5IB50JxuWf1DbUcDNlI3s8NQ1y3C0s6q4HhsYACZhz3AlA/qZn5zTRssGOopUYIgBpm2nWGqlpvZyn3PF7FWZ5irCSDer4YZqK8bDJ+Q14GQMFIwce9mjeqTZx9BsA/QlDeBl0MzMEDrMZwA3tmmp9NnMta77GAKz1taQ1FpRnJdc7FeyabdtNp2vUbYeNr+5WQ54Dz4J5mYb3REA1Nn1IHj0PU2j85qPzmzAAD9C6YEJ4zyPYmcPw09JzMdRFDZD2jivAimy4kEZxWUOMxJm6bEreH8Cs5MZsw02N5rNbe8Qrz4m1W2Y948XuDr0xp88OYN4wAP+x/QuT35xIsCZYJ0a0aAcOaftjBnDA/8AOuwcdeMFv8Qqc1r8VKP6qDriR+t7615KrZ07xQpVSOY1n7UwwXa51xYz1iM6s66tHnTw2PdBrL3V6qmT3ZdHY0Y/YGeajgzmgOM+8GEQMz7AQVvhh1pc4GGm25gr2BYd8TPHYoZ4Bv8AG71IrE9gyBuoxYxugGB4M9Ctq7Hwih3HZNinc9KpVt1J18898/B5YeZxdlz6zLlOUaMzImhhymewFzTmYMZ8MHHgjh7oD2LQXDW7cuFqPylDCM6wGAGBoya22+LVcsB+QxLtWmvRH3tfDPDBXd4cVOIVyUMJbcrkKEfumlw/WceE2zfM/rWjZuqsvW4FuOROZjh8meALp7njrfo/Vuw7P9Z/bysQ2ZFRw0dYNmG9YbidPsNmRVYNDfCpc09nGltdAB5FiqV6mn1T1bX53IR/O6Cl3DAsqMGnbFSOSAdZmC17ls/JmSpkcGH3MwAMAURne7pqWyyD2w3MFKo9NbOV4+Bh5FwrRwPDxDT0OsPxpHhgB/WCKYDfNGDbewDUV4Az1G20su10zM5aKUvUwVnQZ9SuHWYnPm8ZhszWamSc4Ghp9CKmz3Iebbee8POoCqrJw5TsTMD0/Iq0zTr3Wo/WgWseadAzTQI8/ggsp8/mWg+QMFOsCMD1eOc/7JgNQ1Q5+7VnArHq2EbEVvAz6zUYF3Mn/f5b/RrqvZqr8DMKVUjADPPR7FT8y+86Zvmkz95d04LZvH34K8DVPVViqxcH2wB3zgt9YcaXdXq+3Ky/DOFS2dgHvzBc8jWxUTm8rEb5n5wVhAqTltygYfYzMOzPBZDYcZq23Gr0KlUYDZ0I2gAGHWCzh83TWgYlN4O94KXcl7UC6pVPqVRlgy9F/o/WaUw4tSwqvW12LOxT1VnR3tuZtPgBpWlRPtaM2C3OCGbHRn3q7r1sPnHaqNKbzjmGeHYCVt23zkgJzDDxgx7aIy9gaRYp58yc8PK1mpBJ5LwWTDyKPRJj7M8NBzD5z6FderbD++tsMTwCV0AZ72VX02lQaVnKfnHJw6ANXsWdStWpRmjffcZPeexk8zUepUGmszHfWU6Z6wPeGHkW9sCm2/MahXPVakAO7/uPkWjrdKptemncdAnM8xTvB0j7/wA64+4cHgSWKVPPXWlsw4ky5TOjcmzIDA/vYZhgt9c/CudDhRPtHSoxyJoZxtE/vOAdi5pyEGg1yQxBckw5GieervW0czV0CzJ9gM3DNP1GHrjW/aJYeCyH0LQXDxLs6fS3aHSqHDDX6JBhvM/kXD6a9VQd8PB50+szBXttzJcyttNxKN64qAe56ATLnp0uNVW/slU7OrkSTnUXuaA8NmCpLDo7D09mWFKCpR6W995gn1vAru5NeTCiU5ylAdQYwfCPHP8AzqPbBwbAlfaq4IkmM0HREePMzP5DXN3L8PQtt3JSrwkO2q/eU+lRzk/zbBnM9Bh2B5FZ8co1Dk0iiXcxcEaNUKDPYhVKXHe0XjA+/eudUevMcV36VFg3HGjVAAA4ASA3mH17FpTsbiMF7hKrMSg1WD77RDAzXZw8w77ZNenQ9Jv7RndVv1EA5Op9coDAN4H/AALZsmw8WBvvaXeAHguL8AaDUbPhVqgVFsIcd+ec2BHdPeHnD/AuqsvLpHEuM1pXPGaOdQ74O26PFjYQIkcMzk7+/Dv3q74e1u7rGpdtXNWX629LuX7k9Eqc/WNnPyH8/kXTZMOnSZEdypBGPlTzAHujNYy9qbOvO9fVzc6TGpVEjBKZ5c9+fYiHLfTzptvm1RLjYA3qr+ywIkdnDMz6zPz4LxDP9aszXmKzFks6B4cxIjGyH8a9fViBSvW/rydXLkh12CecZmWz34fx9C0fFFm4OKnCWg3pSrjpulgAT6TOhgbMk8/OpXt5k4Mz7OjVKusXHVaUybDIBACohsM9/euoep7gkw63BOnQ6bCrYMRWWQkhJM9/WAfnNcUvCz27zv563LZtyjgFOAAmS6Mz4LIec10jhdMv/gCxIoz9lhVWjmBNCrR/vIGwHWAGoZNhTeF1Vk3fR/Rz9a0oKf6152NIkAfjH+Bn7n61uK96P1j2NxQ56zm4FBrcIAA6NIP1lTzA+/PYYfWCy9V49wa3Rgvvhrbky3rmpEkzZeeAHgksPbDA813Ct2q+dEhX/wAYrqmT6rUYfJBEggcN7Mwz0QMPbAjo6XnT0luPcSg29cHCGC3Aem1cGAelwZOGieef51F4VxuHnFGlx5VYtGlUG87aOK/AMD0ecMA2H9HQt1c9pVxmjR/sHYFEh0zf4tckhJk9nWZ9i4bWLkptE5S4Jz8aZcsWZgD0TMKe8x3sB8is6cKbiFMuC1b6qsq5m6bJkVR433qezJB443R5FgK3W+fCO+ww9GkRT8HA9gB5MForzqVuPUY251DwuObP5o50fvDyLPz6bKgaUUKU9GJg82Td7w7FCGwgU2w7znx65XJdKo4aJhJjyHsHjPz4Ll9eNyHXDprbYYU57wcAwzBdD+09YqUWsa9DgHzTzABmGcpkwPrUTi7xLo/EW/oTnI8tQoTIQs2Y2Bgff/Gqi1sYcmLrtTn2DedPoZ8iY5NyfNBg3AZAz9s8eAAn41NlVWpHTaHEOYYHgyEdnxjBRZkOczACXv0jPDf2GogO162zo8p371AeaAwwOO91qqeNtnDTNOvPcybQPt4SMPBPDADTQQ5cnM9gAB78z6FeBL5x+fm+bgbA6ADYnTkuSYUfPA3Yuxk8OgEtam0p2Ryrhz2vc5o4bzcmbHYcbBnmu/yJgXFtxoNbqjUWs1HkI+jodfWa2/Cu2H7hlVhxiVg7b0PPM2QNl5ZC1YzEmvHSvs561kSmTZjBmAYGtLUrwbo9IOlWPTvUkjRCLUpbOw5J94GH+/WloUrzNRjMO1H1UYNSnjMAPvDvBdV4bs8Gq3Yt0WdOYqsaq3LJ5qjxzjZhTX2Q69bo3nsXJ2bwvWNS/U8quSZNPwMAiPbwDPyLa8NKl9s6DU+Hr9NhvSAh/wAzumyBmD+eeGf5FktjL2h3PZM2PQ7utyHs8aMbTwPAYfIYKXdt50q5Ky1Sn3AprQBgcgD2I3qxSpNZClXAwbLR/cpkdrN578n6FraJZlj031hBtK2K9W3TAAOnVOBonoZ7zBWhhJ92xJMqPRqbmYRWQYB0ww1gBTqbUqV6hrdHnMMhUKibEqNUXQz0dHrA/rW9jcIoINBdXEZ+BbdPYh6NKiSz/nCYfZn/AL96zkCw6dMtCbUZzkw6qcnQjNB/RvkP6/8AIucZWlQ26qxLlym6Ubr55hmGeB9iLlj13Yj8XlpEoPYgGALV02iVx5qbQAiSYf3MzjGYb8/kWUo94VKBVPWM5sJJmzysmOfQvdrWEalP6pxMM8PbM9+CrZJ+A0xKlAGgGAZ+RP8A2huBmO63TWGXgf2Zn1gHkVmEyjw5AalHjT6ebPjNOh0H8hrRQ6bPg29UqE5UaNMZ5JkAPMOsD71vZMCmxmJoWjUucOqSdjsfPwfID2azt+H6ykRH26rJk0qEAQgCWzg8H1n3ooFSnBAaietQh5nnmAYBsXLY6dw6493Vwltw7f5sJMfPM2mnjA8/kXfeDnFSq8Ub0kBeNf8AUMdiMGjq1LvPoMM15Mn1WDPoztKrFNhxjN7XZkR2czP5DSCvAfLzqlQ5hnhgGHsXvIo8cw/RvjfxdqNgcOar6ni85VX4egzL2aOGfXgvP958Wq/ZNgW/bFDcwqvLMMTJbrO8GD8gfkXnmBfNKn3vTJVGYeeAGeSjU+XngzKwwA/412vhXavEK8JFdqtcowSaVFzpE8Hns5rJ94AH5F0+OYabh7PqNw8ZqJSr0qr0an07mjgPSHt5ytmz5Pk+hewmZjYda8H+jeyxeFtOv1WL99pFbYf5uQZhmB55r1HbFSnRo4OVGqnJ8ma6Yj8ud1sJ7CdzzWPgVJuZnpnvA1qIb3hArwtLQPGGKM3gxUE3lKw4NmoklnAs21IM8U0Z5KoCjB4SLnDDZpgjA8BQIggnvxjz2GraHPYkhqOaIGqR5R0G1Aw+RPZAsjDmHGdVxzmfvFItgeAEEl5vFVnM/OoUmeYeHmgmnJbNZ+qyWN4aie5zA9TNZmsVJgJTviLo4YRZqZJiYaeaop8amyYp82/gqmpV6JGmnqBmov2hps/7rqYLvjhc66jUejyQ2RANC9arDOb8GIyBpmBVWI2GDiuwqT8mKZsNgYAfem1s1JoNVgFzfNGYYdAdikUqvVkJgRDYPf34K4gV5/VNt+JhvVq9rvb2MAPzgou1k9Mbx0z2GoRwG5PiYKhrcaqw3ZFRB83gzzVnQa8w9A139n1pEITQojZ9CtodHAA03FjJN5uMvm2wf51LDiFE/HwV9NjcaMQGlnJ8lxl08FVfa3/h0tTXTJ15ipZ83hgYYbEjgs/Kd64z94lz/wCG4qf+asQbbPYCT0yCyGgwe5bdDza6ZqoZ6bisGakwsfDmMYaj7m/yJHWG/cJ8p3NsFSg/iJ31lE7HFh/XbAdhmj9cAfvFj0nc2Rz2MDBR9ZZf1w4HQae9cHh7ROk7l7JDWHTVFVaPEe7EPrh/8RNnUs/eJEI2hcnoh4bhoOWM/aKW8ew3FC5x/sNdMIsjZwNED2CWBmGeCEGXMtTsR4tY1YfD2as4defAtRZnDB3DUA1IzwUYe7as62ElrTcVVMZPfoKkOS+HepAVV/2ZubEwvYJMPmR8TrUKMycB09cFZ8y2fs06GD3tFtDNX8zv8NahlkDAG3FkpMbRkbEXrJ9loG23FjcPYa56NBMtQ0AMxA9msPJqVRN3U1zNIK9OD2j6dK9t7sBNPRoskcHGwWS+0j/nU1mvZ+8V9JtZ+rcOhOsxnGSzURmq/OnvWTfnUX4NrpmS57xSM/gqD1x86lM1Jg/eLHC9rsDBO5/BU/rJsPeIDqpgoxa06TJNRNYz601z7D3QmXpOCddpSOWz8RLDAlHZn/Ohenh7tMISucbBLnAP2j6z8me+Bqvekvma2jheba71qwHs3M0vXeCw/OSgd1NRSPWTh+0NbfMjbVHXs0ITGHvaLKHM/DcRhUsPeJ8ptevHvTQPb96ghPz60jPPeowbTpmmbSz8+M48eppq1B5sy0zUgwbwAABbcLNjDjOAWmaEGWA9n1q4qUNz24KIzDzdXTtlhHjSWwdBtxxW0aSwZYOABh86jTKUwA+zwQ8gYNZg5mpW0EblD8gKyjPNgsLGnmy74ji0EOqtmGa5b4bX4tongbMfDUWSHhHh1pMzETzzZ+0WMtrUD0x8HcNNSmTzAE1PeYB3YCZ5wPOuhgvmYzh+0RPAACoUOsMaXiPpqZUmzDTzU9NvdqmqycHdjirzeckjgaU/e6aGMHhLphzCZZDPerhmm8zHBtjYqc+taOlG3yYYKLXBM0GdTR15TgYfIjCY4HhtqbDjP6WD7hmCCTTW95trmt0JECZv9oruNJ3rJZuRnQVwy8YCC5rh7FtUzP8A+dxG8828Gm2s/Gn5ngbimsyQUXDp2lnDTXLNgnQmZ+GkZrKCINYfFO8sBigT2fwTaTPJuavs1YMw9iFl5SOZbBeLIIwB7tSAAABRwmAZdakAbZqAWs2Hevus3/UoMnP3aiA84B+0QWxmCZNRwe3o8/ggPD4pk9iPP4KOe9ABnsQgaLRQmqGZ4qUpisWbNYPsDMFj/R7qr8mgyKUfWwea6RVYfrKly4Op1smuFcIqkdvX5Lo+pgBmYKR6A60sPigz3mj61aAn0KC8e9SzPYoMl7YqEcwUWSHhKQcltR3jzFaoVJmlrOf1JGmlncLWcae4HvFKOY2Y+0VFngkcnaouHm1hJMPxFWPScD61HN5xRTeSISlHJ+dNczgXtFCN5Bn8FeFrA5mfvEgk4e8UIE6ALFCaExS40/fgqoAUuP1otdAZmaqq9flo2f8A+0dV5P8AJmrJk1z/AIwcMWLzpoS2DMJDGe8PyLmtbkXEji1SrhrnKW5Owj+dYSfR357uuFVnz5BrqFt+jYxWIpy335OqAGa21n8EGKaem+xJZPz62a9U4JG4V3PPhuznGDZaYDM+YNYI3tGQbfkXto+Fdx1j7idc5ann1s4f7a5vxp4D0qlQg9R1LOU+eHihguUeZT15kwNAM81Ik0c4bWo+utW3wNfZaB+dKz/uVnbw4XXbSnzlOZvQuzNbrc3M8U/DZB4M35Rs/QrOfb06G1qSooB9aapVBqtYkaFOAMw37zXmwdKZYZd1328wBSOWckzOeg2+yYfPsU0JNRAgob/LADne11qwemQaIIQZzGs15M8M0wM/Po86ZHOdhDZ+QDU23qlQ6bC052Bun2GC7tbd52P9ngiW43AjZ+5d2HmsZM4J1WfUvWtVlQIYfIAGowOeQ5MFmrjVYuwwPP5F1Cj8cjBoKa5ECSfnMwWKuq3oNvGbH3aTgsI9JAy2AAfQlwN9xR9c3bKCsBTgAADAMFz/AJY43WCd9ZS/xzP61HemPn7RZYEKfrn9HkUEAzL2alvG4o4PYLYXFH0OYBh88F0Cm23QI2lOnVWZy/eAAuWAe9XAHWXhw9eGDXkM1ktwqNXrfkvgD9HjQA87KVSqVDBjBhzPPvVZAts6xnpvgzh51o6JbDdKMM8Jhh2GGw10OdCs+lVK6nZEGmy2eXihmYGrLRg23IBvPCax0GughXqrDpZtwbE5Zr3zscFe8NL84Qw/ul42q9JqBnmZuw8zVRYygXPb91UtqnVG5o0Cb+MZ7AWSvDhRVbYhPVyq3HTZMc97JxzzAwXqi1WfRlufVCuW+errHhEOm8tgszxa4wcGY1n1DhzYfDlnSfDDWejHgCvbWHkKAFGOUB1UzBrzqQ96uy04MvWD6Fo40ym0TS1ILL3yGqyvVhie7rxKVGhmH4SsQpMCmswDfbneL2NH1qVZ8NyTUgCVUjgR++QHWC6NwKO0gpdYqVx6L0gOwwXN5+x11zzvGiLWt82S/R6pClP1/n6VK63sN6z50c3j/mpv7upoG5PLXlnmZmAKzrFvXBavKa74BzvRhvUbGUn0p+mvg2/3o9HRBWH3qfWQYfivTP3LPWa6BVbPqtKrNCYg2l97lM/sgB1/QrHJ/EMk6YaIrpHF22H7eqUdydbkyj+5PmI2AGudSd/QgieHpZuZ5/wKOBmnvHwwTOHxRsR70IBirWlU1iSLrj85lkw7DSZZbz1MAPA0QrHoz7OGoGCEAM/Ztq6M25koNTsUWSfi+H0KFoPLZqWFKfNrXMwAE7DNgM9dtW1NhuTIB4MAeHQz0KRX0SNhNalttg9oHngYZga2VSuHoOh2lDpp9BmG/MFSvcjR4DrfRL7AVOdYnH3ohooE+c8E0NTB0AzA0dsGxXqHUIlRc/nXPNnMN5rOetX8djnWggSX40jXA0DRs6Mo2HOsOtahkKk9Q4kGlN6zoHmYd+CpXmZUws2G8810iwJJ0elusGGdQNk8FnYpAuquWfcbT9Hf0Y74A88zhsz+haCtz7cvB2PUqHnDlngExlkMAMPOrCHPC57BneuYoPS4r2sefYslPo84KzCYtlh4+dZ1mWgXN4G44qM2dQfVjFuWzGhyJ0MH5Mho+v8AIuaHAckxfWMRv7uHfmtBcNYfepofaNh71rS/B5Q+vD5FTvHLCED7b70aOb2vy4Hv+hbh2A9nUjprjf3d9k9/nVbZ9YqVKuhqCxLM2n5PKvZ94f7mok+sSwmhKA8DDoVPnrTHXPOea86R6t4kXhSplrnTebwqtu4Pw5HeC4vf9eo14RYVx04JLNVAwCSHRmawXQXWataPPbelBSpzGtHPrDoTpFgzPiyYeuwwYSw906Gw1sKrTal6ro942XL9VR5TOjPOOHsT8i5/UqP6tmuzm5zz0TswNdI4SyWJNoXLQ6i5nT342uyz5DUYFFbdVpVEn1BxipSZ59YSHj3ma3fr6jXnbPKVWjz58s/YyM8AAPkNcngUp+M004+31ns+hbim1JuBb9YpWg8YTQAGcOzr/wBdY3AC2NCZPN+lOPMyKcGtGaM8NbDszXcLV4/evqlR2IFGOlUKtydF45GBnGf84GudUTh7OvPh5ZlHpvLQ6g+c0wdwzN4FEk2xWYfA51xiCb3qipG/JMOgAzT9g94WrclDvmL6jnPs+sKdsPfvz7DBNXzxFcsNrlaVZ1buSq9jUFnZ+czXhT1kxUrZpVwVXmY1P5nlTkZmBh+hdgszivelHuuiUCVfEmsUKthoMyJB56OfZn3rv4bGlu2vcd+IWlQLutyNGpQPa3q6CYG8B/OfYuVVu236IE2pP3pMptYfk6FNpkSsGb2fkPyL0rdtsXPYFNCsWxUXnpz7wA8csNYMzPDPBeUuM1Ku68+IddYi0Cmz/s8ASpM63oGB59gGf5Fey4UlV42XrbEyXbl3VyBcjtOwAAeDWON8gPdij1viXxGmcOafasS34dNtI3u/eZn5zXP5lpVUJ7XrimzIciohmzrRjzeP6Oxba4bMvWlR2qjdr5/eo2ENkNgfQgkWBdtq02/4lSrM6BSqfCZw5eIZmE8/IZrtsCBB4i3HTLfl0qHAtKiQzq8+PTDPRN/PDDPvXHeFFKseTRjs65rcA3XzOUch7ZgCn0HjBcdnxZtuUOmxpMIDOKybu8DYXNfMl36pVXgmeZ1VwJ7sr7qHLgYBGDswDsVDfnGa8bGuuiXBWZcmvO2psZp3Qy8DwGGtn84f4FzeHXoN4U6b64tKmw3orIaMiC8eee/BRarJrNyVm351x2/zjVIAAeBk/wBpAPkWP0Ya3YOMHpD3VxddChv2wzbEKaeewN57O81zefMiRvV8Gcx6yoVL8h4b/k862HGavWPfNzRItlUOZbbv9JOW9gB/kWCtv7DwydC47fqVYrGsfg54Rl0xfYbRKrebE+4XbjocEIGfsQDoBEFznW6M1Sq5vdA8wkd6vqrdWcqQ+xbEamx5R58uAbFkZ8xiTUjfYggyAeTvXo1xnR7t5j1xzkPQZ8HlzzzP5/nWJqVgXBSnwbYY5zMM/C7Fcw7wrkbCLECMHja5norUHc9v1uaAVmIFvV2Kz9zlx98V761hsV/DT7F23xDoNZo86qvNMZxawZh7EzDrUq/Le4ZW9S6hLc4jPXDXTk+NBZDD6zBYUKrc4T+agv8AJumZn4Kqf2OZLOsN6zsrfn85rogbu+YESiWlRKVTWIwRzwmhrHm9n8ixlbpVVgR4T9SoZ03XZ2AfWYJT6rLmQ4UF+WZx6cGEYD7FFmVis1UwCq1WTMAOjWPPBaCDhmKlxpOicd8G/C2YPdhrccGeF1R4l3a1BYgyZMKlyWDqrLWw9DPz/OvRvH70bLRo9h0S9LHtif6wOpBT59OBkz5ljzmHY9sQebeFcl8L3kS2+WObFhm/GCRsA/Pv7FIn3PTbkr0uq3BBjM4Z+DHDAHsEVbZiWSFQqsG3JkP12GhD5sMzw98H+BbuNwutysQqU++YWxCOMAA9UAM5Uw+s8A61Fivqtq2zb1qWvXDA2ZtU8c2TZ+6mGGfWlwouG1aPdsSDRqcyc2br60hkOjZsw+TNZS+alZXPwrVocow9UADPPZmYPH5MD6FO4e1v1DVz1K4EB2cyYHOZZ1s/kUYYgZhv8N7javSpU1mSbEl84YPGYaL+ezP9akfb+/8AiLUgrNYnSXnYpmASIgaPLB8+C0vE6gvz7XpVDiNw2YgZv+s5Z+M8exYWiM1i2HfVWuGk/wC2w716JVSCuXDN56dcc/VhAZgdRPM/oBWdnTLu9aS6rEf9ZRwZzmHIewz+c1LmHZ1NjyOVqUyG7UcGNV08wBVlsXJTrY9YMOa0mPNjaAYB1mvn3a24smM/VatTIv2jjMnzOZnLPAA+g1yfiLb1VtjiXXaNUZUaSZyeazjnmGB7/wDOpszQqVO0323qVUGP7IeAH9ahM0pgAwfnG8fe69vM1cB0PuYR22HGWTA8HgdPfma2FwssW9VI9DbD2/8Aph4Oh4z8nkwWaOHEB2JKbl02G6DwYZn4xra1KqhdtciW5b4BPq1YeBg3jDANdFlclSl29MCI4wzPmwg9t0BgfRgs/PrHrKA0x6jjZh/ArO54FNZo0euQawBuwnjhT6YZ5va4eRZqm1Izn88wBshgYaJ9BopIOSxDfOU5Lwdf3s8u9sZDyLe16TotW/E1+W8ED0Q6APzrGBJiaWhUWz6DAwwwPf3gt3ckCvz7NhXP6/ZmQoUP7sy9hrYLG4Gos+t3xVeINCtW3KdbEmPNmAZux4AZhge810XipxRu7gVxYqFKs5jlvWkn1hJ2a0aTFeAMDwPoPYa5LQalKCVaVDqVzxqU7y3Nes2Q3xj+f5FpeMz1VuSpOzrjrlvT5tR9jIpm8MADDZ5Fccw0dK4osU2nep36BDpUeUzohOjgYRj8nYu0UGt1WlRaPRqrOjG6+HXHPMMPOvPk+q1XijQbf4cy4LNNkMmAPVvPBkwAPPgGC1tKuq4LMvJ026xSjdoOFPN7k+ZZeDD32C+lw+ym4ew6Vpxi1G28Fq4czwlh7en3O9SQfudilAGAPMyKfngYH9atYdVYefBhhxbIag3s00lDZOS1qAiOMYeIslmjPems/gnXg3qPh8VWwes3/Uj5htVxmo+sgszktoAez94oRmkBmAIhZ5AnecwVVzLgIDnsGWn3q8C4CeBnp6iTzzeKojmAyXtE6FSYe8NxxPEWFMeM2jWCr3rXXdfYgmZB51uHnmPxEy8cU/eZrs4f7bG3Eq9Du4/v0uCYR88E1Soz5ugZt5mfYC7U8zBeDBwAMPIaajQKVAw0KazsXZHsscMVSrVnHhLPZ8i1EaG/Dj6bgKbJn4ezAAVVWDqs9jCnSwjLG/1tDvMweY0Dc8VWEaexhp6ir3qVEjR4+g3vbDeaiPeCGxRgW1VZ5+GbDZ9a5/Ph1mA7yje/DvAFcQ62+FZaiSn/ALufWtBP5R4fANXxjl8xk4zWv71U+i4e9aK4Yz+boNt7AVKASvIYfWC+pxuK1hDqWDWm44nTqT/u1WBAlH7tINQFuy/J56ZLy9uaDmX/AMQ800Z7vEQZt4+GjPdpbNVlge9xWDM/MlSAYZp0JLYEnSbauNoPDpmeGaM4zbJbHM1Qs1JvAG9MFZxp+ftHFzYbQkHqMjmjZntmABp4GpGzFRwBsPEcBetU0Db0vaIMTQ6jH9SIHgBYYEuN0+IjMGA92Cjg9n7NFns8RxRgPZAgyBNawZ6biIEWHRb92gxNO6wAhemBmiDJghw+KdB5h4UeIKwyGYApIPOAvmbfvEv7tQs9mD2x9vYoj1NMN7HQjyNSGQc94grwjOZYONoHqUGHhtq40W/60Bgr7k4Zd6GbKa3gtG8q84wZ+zW20IPMmHengn/vN6A421V8kNE16hZnUnPOnWakYe8VP4mOog8T3arC9tX64c/ES9cfiOLM8w4lzDiYN+WpCsNh0IjqWsCz7JuGnj1AUdMG/K1Cf86A55g7mq0DPNIzMzUYg2tTmMSU0YYqs1nAJTWZ4G1g4rwBeAMNRv8AQoJmpz3VqNqC9qL1AdZEBppmM+97NvNSGYxoDB48lKZecwwUfkJHkUhkMD3ouD2Dh+I2rCNqGPidag4GHs07GeMHVhhScbOxRzjMMlqae9WfhmKiSQ2LFaknyT9mhzcCLg2pDzLZn4iWitotDOPA514bEbJuASujgbfDUWTD5YdTBdO2SbSnnGQ8QzNWEySBtbDVIzqKQouG0WjyZO01TnnkraSy5lsBQZLLgJEMbNBMPZpp3mXzDe4o+GCe7Fug7m2ZqWdNMwDQPBQYwYEDitmZIao6iz2GZNNOMGZqRSp/LO+0VhVWQkxdj4ZrNPRjZ9+BqVtgzW2DLTUvn2D94sZTTwkez/WrU3sA1FhcNotfHodaDW/DWfjTH8vabFagYYKOl4nAamxpO9U7JuZdamgsbhptbBJbRc5gq3xMPaJkzNY4bbXHP/OjCf8AOqLWMEXNfBMG13zmHvEYTP3irANwxRpg2uOZ+dSGZjgLP6xgpATzUYRtdHMzQ5/BVPMOIweM1HS0WYPB+IiB5tV4GngNRgTdZBn8FHz+CWfwTAl5/BR3kgPJI0wI4PYOrzvdTL9q8WgltnnrHn0edehcPvAf8S4v6RUPk5VPrjDni54B8nQi3ZgeB5hqUHQYJcz86yNh1j1raURzPoBXGst2KQ9PbyUc5OYqK91pAYGarAdUI5ObunqKUZ4R9nWqd48CXrI891plSAQnsUNUc1HMzUg0yYI8tCMzTRgrDR/ERct5FGD96q0XP60gZcVxyzYIeXbUbaK4AUkOhOmCZNZh5Jk9iZz+CNBNB5HrZhgbeYeRRE8AKBYRpLbIYMAAB8ikBJcVeAKQBpgBWKlXIwa9KDM+9cyufidBZwgzpbMyoH0GHQH0LZXhSaxW4vK0bDM+vM8Fy+pcNKjDmBOqsWlQ2vOby5xh7kqt4suuv024AjNdeALLw7nuOsPgxWKickPnUvidW6VGqPq2jv62HtjDoXP/AFlUvcIqLbi5M5LWg4ENkPOBqigVinUEw09/nWZOTO9o/wBfeo5hrFm4oXtvqbXqdMmhpsRmfyKpvAOfqUduDvz61RUrZNBdL/k6lPReeiXHTfozVraDh1JfokLUOh0RnD+kd69JWe9bAUv1xoBJkAGe9eLKlZ844TsuXXADQ8h9as7Svam2NC2S58z52uhBX8fpk6t8Qag5BYNlo8N596wTNHqQNdC1dyV5+7aic5hs9U+gFZUGTycc2J0EDd8h9i5xz9nU94jNautnTns84oMu+RpQaDaVfuR10KVTjNkO89iDMPdajGbfu1d3DatYt6fylSbADwz2GqrlsOtDwADU2NPce8OWewOxDGBgzwccwUpmMGa1XTlrNHYDfEcwd7DWlgRuTahc9sN88M1jJNSBmOZsTsJQdmC0frvn6RFnORTB0Aw3965RvZl51i1cH6a+ycfsAw61zWfflcerxvnEgMg+efgs70zUq85Pj6DhqlwbCQBpCLh0iscVHKrD0zY5aoYAHNsvYfwYLn9VuqIEM4Jtmbp9cgD60EnAGNdU7zLb3iLaLBhWH3oYRHIP/an1pQGYpu6lS2NKv1mGTU2Yei0GC6hs7SCnMwqxyPeGZrHz5OtNNR6VW36VNCc3vw7Ow0FyXJ64mnL9VRmfkwzWolveDD12+sDAwW4v+q02t25RK4FSjc73xwPYq+TaT9zyqJasShnAqr/vTPYu1XPw94XUGxo9uVyga1YYZzZNkMDz+tQhxrggzdUniXT6jbDcCTNimB8vIPDMOg16z9JCm+qn7fuaC3hcusHIM9ma82WxbFv0qOE5uDUmaqHRIA94Lotn1Jy4bjaqvEKdVZ9PpYAcMzPoMOhY9wuOPD3r60qPSr1lyQroGB+rMMDNeUrto9Zo9R0G6dywGeAZnmvTfpISabW36PdzEXn6fgcXrw7AXmeYcT+1mf1q4sFeFk3BZ8OPOqLjLwP+QOhU8aBOktc2EGTy/nwXaKVbdv3hwHl3HVZ0w5rGZgZvGe8DXN6DXq5RKIcGI2yYGee/sV7Wz/LOMhqOK4tiH61lAB4aXeqfWqLzp/cXs/IAZq4jVh9mjHEYAAd84KQq2zSo0o/Vrf1mHQq3R1vZgtlbdKpUm3mm5UTN0+/NSJ9mPwI+vEiZotgno3hYNhvVnGnsUqAGnsm+dR5OZu6nerBmlfaqKbjDjIOxdmkHWaDMvSX5J5vnmaMP2fU+dGzR6y8TrbEE9h954J+BR3wiuvzn2Wd/Rn1oIAbyUqNrm6DbfR51cMhEZgS2JcQAMN4GqqNAlTDwY2fOg0cPmoZtRDYPf3gGYK6MKrDqLVwRGM2mPbBnvwVxZ9S9ZUZ2jSgzdhBmG/rWrjUSjXIDVKYlAEiV+0tHs2fIuLuGUpsxuTWXfU8WT6qqkbAzNnoPzqzsalQYEp2deM6TD9XZ8m9HPA8FIegS7baOh82YNB0GC5PecaqhWz0H5MkDDPPPBRoehWb2ty4dKlUq1WZlPPrnVNneuK3U8/6+dYcYjMtdnLnmC0FHvB+j8Po9DqU4DzzzAFSQ+UqTRy32zwz2K0M+cPWkBsA8N+9Vkll86uZsRAAP4FralTYJ0v1rRswdc2GB/IqCjhKeDx+8N62hqZeZcjGbc5gwz6PnT0CS/wA1HCD+0Z7A861FNZzi69VfA2gzw1gzNVNvVJiNdHrw2w8D3IBgBr0aivRqBUo8inSmAhyNHXZMP401ZlbYs+jSKbOppvHKDAHgPoVFUq2/VZ7spzZmeeHkWot5mC9SQrFVb5l1h7wTZ7zRJme9TqJTvUFRiA9WDe2OmBhoqzqVNrMOz5D4OBMahPMczIjhmAZrQUqpMVLiD9sa5QI2wwZksyD1gZDzrZ8MbwtKzOJNYptNrMZm33+uC8zmDx/ICzhDS8GXqVwo4I/bisvzJMirvGxDZ/s2Z9YeRXfD2t06Zw54m2XTWAqWcMJUbW6Hs+v9C55MvasXzaFxuVm443q+FP1o0F4ADAM9mBrUejNxyse2Lfase7mI0PB4zZqBhmzv7DP/ADq8Mi4zWfTbV9FqiU1iDoumbBnv6D6zXHLApVZrFoU+C3OjQMJh6JvHgbJrtHpG3nbl+TYUS3LmpVSp9Oz+6czsz+vvXIpmnRKlHt+uMMnHpwc193DvUXzNYd7uT0jbxu2HC4cy4Mamu5hFqU5qZmcwA8nkUj0e6VbFYuO4Dob5g1FNjWN2T+04H3/Ia8ss29XK9ImzqbFk8jFMz0eh4/zqwCxr1sOuW/Va4b1Kp9ReD7xEk78PyLGOYeq+OpwaVcFVua46HMh0982IrLzR7zAAwz+hefOJfFSyrhmwtCdVWXYuzPDMMPOCPj9Xn/XcSlMVGpSY5hh9+ezZXNKrUmJNGapUtvoPMDDrBaWNtbdYftKbLpspwKlCfDP1iZ72c1X1K3rqrdBkXHatiTGaFCPA6m8GAPLAnJqtVnwYLB4ZmEXM+jf513OHGvF4v5Mr74jTJlHisschEa9iZrzqHN6VTZcNrUqWyI/7YPOrX7Q1WZy9ViVUw9XPYAABvw8hn3qOzaVzVi66rTos6NPCiAetIzwjACx8mpV/F2JBnMnEM8AAFjY6xbdV4beo7ouDiNTTk1CbJ+5mAZ4LnTNsOMzWq45UoYA/g+zmfWfkVIy9VTlHQ6lO5bXMAkxzPA/kWg4ezAs+qcjWIpzKUDx5unv0U2Qpa9JrJ1c/WMEGfO0CrJ7LATwNjZHlBmALoDJ2yzfnrKUByafNM+cZl7DZDzgsvWIFO9dmxFnBMaB48HQ61tHMsoECIFB9avh0PYGarKqcHlQPAJLrHXmGxbClUFiHAdfrlVjMxPfR+szWdqtElasio6kZ6FKDCNv9sHYliPR+VNpqU+rIHoMlp2j+pvveeYBy2eaiyaI/TQpUUGPvZs74+avaPXmKDKN9/RntPxjDRw35/Igy9YolKPx2G8APraDozVZGo8SHI/nLs62lozps6qsAwDBw3X+jmNmYLHvMmYhqe1710xY7Bwc4i3jY1WqD9pVyHSqfKMDn82GYGAfx/oXQOK/G+uSalR53rmqycD1mZDJmzT3g7wwXJ7Dsz7eQrgbffegSKXQfWENqIAZzDDsP5FTyZNO+zLUWuPvBIpwYRjA8N5q9i4vy4bfvCRbkX14ENrmX3pLsg8+WAzDrXYKPw9+yTVHvxydJuSuv1UPUlWiTDejPAfRmyfZ1rz/ZN20azLjauCsUr1q0EY2OU8/8C6xD9I2ms3Rb7bFDgM2vbU/muXgvGByc8+z5DTYwnHJmsUS+qxatcpsMJDEk6hzbLOGsB79iyR4epKfUm/a5ngthxdr1lXzPp9So9VmTKg/PPnDlwDjHg8ef8Cw9SCjevvVVvuZtPmAM/WpRg6zUn59UafqtRkvHnn4xma20+mt1JpqdSjDmJR4AZmsPWKDOoM1rnsAkB0YHmiqVVcqsCmRX2wD1d0G1sNc7bDZz6PTvUIPy+WOQBgDwG9gYGpYW3RqbaDtx1l96TICTypxI/RmuczJ78mPIbfcN518wPPv2LpEy4bYvOmtUqm+soFYq+EqYDx/dQfAFz3CMJ1Sr0usUZmdSqPTeXAAYN02fvIYdhmirdyU2+fvdZpoQAAAADZDA2T8+fkWFn0qXDzlxXzDfhhnsNWsCtsAAa7Z+ABngAZgf1qMmFq9QaHQWj+3FqnW2qiedNkRJJgYAHyB9aHhdUn7Gvek3BTZeHJPBKeCQGsBh5MFK4e16cdzNTnK/D8CG/CgPVPYyGYfQrilcGalW6Xcvqqq02ZLpEkMKjgejn1mGfYrhTL166vXF5SKrLiwJNPfefMAZZ0QDPyAHYs7m+zNd1GOveBh0GC6F/I/dtmQvtrcfqeNHhYfd4MwHjeYPDM8A+tZ+fTWJ9cOnRXDgbw0QqJ6Ow10CPR3nDuaObB55gAAcgNgJqSc5l12C2+8bTDx7Gjw/gUKZz1NfkUN9wwCK9gYB0ZqbGN+HPaYpp9D2YSPnU3AtaDmzTee05JtHmwBmB9feC6dwleod23bCtWXSrhmSjA3o0SjYcycrzhmtlwlrEW7bNrfDy8bKh3JCB5+ussvPHGCBKwwPxgXsv0eODNj0SjW5c1Ss6AzddIgAwcsAMJWBgfQH51zYHkq8I3HO1Z9E4eRbKOezcNElRYdPzAze355n++DPoXL6Pcj/AA0uFr1kwYVWKBsT6dU4Z4GB9hga/WWq0GlVKRCfnRdY6dJCVGN7rZMFxTjxwHoF28S7E4ovwQP1dMYp9VZBn2zBngB/Pgunxw4HPeCFbg1WbEs6Jc1VZCogZ+ozhmGiH7k3g6F32Bwig02Pg2/MM/xtmauLboLlKj0xuqyjmS6JmwzIdwMzY7MzWgmTzzXZw3eMWnCthwwhx9AHDP5zTphkiA8k8AZL0VMmNgCrTBaiTGA2lQzGWwJBWyQUQGVanGzVfm3q4KkDw+KaME7n8EjPMkEU+hUlS12d7a0GHxUR4GwLM10whjGZ9ceI9MzMAUgK3gJsOMGDvfmtXJjRIcc3GG8FzK5Kry1WkBqYLv4f77JavXDreA5myfzqikyZQezfeVS8b7I88xL1vOCaeqT7zWa7+mHL3NRRK3KB8AffMw+daA6wZ7294Ll51hw2tPTw+dHAr06AXhnm15DT44R9P+7os+e3JhagOZn5FXg82y6D7j5hh2LNBXjP2exO1V458Vo2NmAb1j8y+6GmeuR95o2G9mfeqQ68/q4PubFjAqUsO9Ln5HnXT80MfphuGeUmFmalsgbOGo/msTArDjJ7zV2zVdYPaJ8y45uxoHjzDU70DMmnMj48Fn61Ws1LZpuKaybbwmHYaywpCrclh4TCCeH0LLmeDumtadHz9gq961X9XXA1rFue4Zx48yQ4fFWE+iSo3iaef0KIyG7xFslHNNKyOM37tRTjOKxH1jBSI08wLxEybOxNYfFXhntoArz/AJ06Fe/HcWaBOgowbav1lmGbBpc+eGnms4DxgexxSAme7cUdLba7Znv+dTWZObWpn0LP8ywY6eoj1jBRhe2gCS28Yb0frJtl3QzWf5l8OhxM8yeeaYNtM9JbNM6wGq2NMN49Nw8FZQKVLmOgDZ7HO9Y2vZrNwFIB5R3mdGQbGeeB4JrI16hYaznu3EgqT8byfnUcJLakMstyUbbSAuFwz0+UBOhWHDdwwBRDpu/w1LjQAAtRxc7xLCSprLLckUHJsYAalw9Blc7fasejYF7NR+WDNaN5mIftEmY0Qz6E2YZr1b8UvUmftG1qggRD7FJ5NgA2Anj2TDEvUHwsNNUUmmyozunpror0NzPoUR6AwfWC6Y9lFw51y73kTrMZ8ywwWz9SRPInWaJEA8wb3p9LmwztNpr5zdNWT1KM5AA2xnvVhGh8tKM/nWgh6ZhnpqPpbYc9qTIQ5RxPIoOf4a6XUqPBqQeOxvWXk2kEY9MDNXHNsuGaNR83ANWEyMbJ4OKJo5rp8WxSGZjZhhqJ0NM+tVR6jPiN9adjSX8/HV4FmeuAeAexNZGiCYxjgB5owMMFkJDMlw1NBlzDUVbh+Gpsae2A6b6ztabsNRzjOKdGmU41YM8i9865mqsjPGBeIpbzOsCKTA8jaaZB8FDVWPQJTOeoCa6FqNEHvaNpp6jsH7ME7nmGcz+CRhzI4KzmUdwA1ABRwZcV7Z/hWctomj/dqa8yaiYGDvQttsseRshn4ajvRt5t6auGQzFHo707l4ZR6Hh7tRzZwWrmQAMPDbVS9GbAfZrbuhFwrQ1Pw0BvGBqXJN/AAw+RVh55+ItGKwCY4Y4GnQeAN7gAf1qtA07zXwXmAZvYdCLnHDDxMFHM8xTRpgTQewNWDMxUWfwToScDS4NtAEzf1qzjT8/DWXCTtTrMzAlGG22wCTtS2KijVJS2Zn/NqLjvhwvawMEykEkME1rfhq8L2sGXtqe1m/6lTBJwPek9Pz6FGHi41g/ER6wAqcHnFIB7MN5qMCw5lnzp3mQD2ZqieewNSGTzFBa843n7RHzXwVUZogNY4FwExHzOarDeSZe3JhquAeT2eYqsA1LZNRhWxPdCwXGyA/Psh3TbzwMDW9e3ioVw01upW9NiP97KnD1y/ghUtakO0dwPFYDeuhmGK4jwfrHqq6nYhuYa54Lu0kFsKeSe9RwMwNSJOGaie+WrnWAHmKhSWcD8NSGetSDADBSIIGo7x71LNk/w1Ek7OtVh7dor0nBV/rJzPxEEw95qEZ7108PDGWO7XTLzZ+IrCMeaomTVrA7VjzQuLTjBM4fFPH0KPn8FyOoyYJrD4p0wQ4Oe7WGAzh8UYJ3E0YMuKMAM/gngNBouf1owZTCDoPIzeQAyj0W/61CwA9vXJ+OtsVW7WmWKU3mYHn/gXW9Fv+tBy27U01FwPJ7Po68UZO9ujhpec3ljLntWq2lVDpVSbwdXuOSFVx+4hmuaV7g4xdtb9eXOweHyGowqHlcKVLe8Q8AHzmar3o2B+GvTF5+j9bEaKc6nS+jrDWXnqvRoNEnu07ow7EwtTyYz7OHiB+RWFvc3zuDB7/J51UnJbeLw08y84HsHDA/kVrei62dAZt6PBqTAMyH2QwAE7A4OU6sUYHH2MFyyj2NWThhWINcCS6HY70KzPidcZwORcnGy6HkNBqJnCJiHvp0Tf50LNn2zGaP1lOBmR3rl0niXdsZ8wYnf41n7hvm56k0ZzpwGB9gMrG4ZNxGsnC6Gj5tmY0+eAYLux23QKVS+bpU7RdAM/BPNeceG4VG5DkBLlvYBhsDZ512iiWkw9C8SpSdirCotw+qs1ipXLNfqRm8Zn1mqWvRmI0jTYf1l0i7aPWT1T9VSQAOt3DYCwL1Efnn4Zhn86m4Wog0w8QOtOhMln7MP0AnZlHqVK8ScxgPYahBJAOhxRgcVkmZlqKQzUpcaPym/SVenQewLoA1mtcMhEei803nsUST+7BC9PcZgHoK9tKGxcNLlsON4TWehewMu9Jf0tBvoVY884yPhq7mBycp2I/sdA1n5PtTVwg6EbmYuvmAJcy4bWh5EMk4OxuI49n35oGV2eBNgaDL+pOiHJa/BBS4f2ZmXG19xMIh+6M8MDVO88YHpprPNBqp9bnQ6pHCLOeB1g8Izue8AW2gUS6qk6FVuC46ln581x/Wf1WjYDWMDzwz613CvVV+Nb0JxhzDXZXBytWgtiNUgqjT9RlgcH59iXEK3oL2rO5ueAGYBg09gH6Fzy3rnrAVJp/mvyd67GzZl93DC+/W+8cc+89hrh3hNw4jflq1K0pEJj1rMeiTQM95/7+dYc2ZRnqNsZx/OvQdw8OolVkMxftBMkyIoYA1I8bAFySpWxdTLpMeqqkDQdAcmvocPsowhWxJrPqup0aLWOTjn4xh51Ch02VJdBhtzMz6ER+qtIGDbe5gOszBaDh7R6lXrraptKc3+Q+hb7Faz6yo831dno6+x4POo9VOjsyAYpW/DrXRQoj9gVaoP3GxGkx3wwwM8wVJXq9QwE5dq05mHh2AvRn5mp6ojvwWPFYPPrXXrYuGlXJPjwXzzB+Hv2dBrm8CNUaxQZFZzDNjPMMFI4ew5dVmesaVOAJtO8bRPvWQr6rSvVVyy2G2MGW3tgKa9TYoSgqMF8wdDyKXWwp1yV6Qw/O5bDvzVfR3pdNrMimxKjz8IPnXsWtQ3Dc86qzPV1K+5yO8Ge9RYEmVTYsulVGCEnXPMDPsUqsQGIFxuzoP5DVfJNx5/UVi+BkJkLQc71BA+QLkTDYp0N7R0tnR1qwu2GxJmtSqUxm1KDPNeiw4VvUZ6fW36q+DIMM+CBmrrh1UqjW2p1f1wB2nSTBkADsXLKPDfeqwRD/pQGGHnNW1NuSs8NAqdDcpv7b+N2LPm4R2CZc8Gq0h6ovsAzIing8AdC4/JvCqsum34J/OYKXVbknUE2uRYB6PUWd4OhsM1nIbOtFDU61jw8IkAfPzAnTqxDwD3LIYfwLa2r7WQ424zy4bAzMAWdo9NYmUaQ45UozJgeAb81BrEOVDhfe28AM8AV3A31KgMHWRg1F+MDWBmHjdag3PaWfMaDb0ZaC4eHVAsmw7cuqm3VG+1D+GdMDeBgfQq8zf9ZSJUuWbxv4Z5rm5YFPbFNia/K1Fzwg6wUK7Y1uU2pG3QG/u61tNk02ZKOI52d6x9YjMVLmM+sFcWBh0dw+Sl4YBKDMFuKPMCgxeR0Hnjfe8FkOjM1jJL0uTAhROeNlpgMDaBWEN596fClvnnyRgf6FttTqtesP7H07kbq5yHVZUYHt/vgV36NPCuvw+JtKv+46HD+z8oHwZkSDz8c+jBbC9qxRvSNtmsVylRHqVT7Qh68aQ9sN6Vh0H8mxcsmQLqtuyLcufXkxqfcpgD0TW2ZrLCW74l8JbVsY6xc92vgzIqlS/mqOzmYMxfoBcngX5atBs25bVCna1VqOwM2TZzDzr1bwfjOUGyK7Wa5avrW5QngBx9HWeeDDYG9edb/sy46ldFx33fdDZojvLa8OPnsDyLW0XDnNsXVOsNh2K/QKbUgqOD5gfYt3aV8037WzrmqVKn+r32fBkNBmGazVkwLcrdUrE6sW+EyPyGDIGeB5/Ia9JcEIdwX/NmwbScpUmn0iMDD1OqB4Bv7MwURwuerYK0rhg3mxLnVWsQIHuGYj2wzUKmwLRu3m7VuOs6NQhRjOm1GRJ0QA/oWU4u8Orn4J3vIlVWKDLUqSZw6e88EkGWD+cPrW7rz1v0qVRzi8MoFbOayE3Nk8AwWNuhyG5Kk/QadEtl+8abXmj6zaPW5Y8/OsbMkywf03OxbDiLUol+SpFcg24zQeSMAOCCykOM/XpEiXLlaMeKGbzuGeC2gSIBsPRWnGDzkGeHyLZ2TGfn3RyNwMZnFZN9n5DDA1z+m1imszKexUm/uQPa2fnwW4O5IlVujOlOcsBvaABo9Yd6gWFVrd8Xhw89Tt1g+XivYciyAAezsz71kaPbdcpU2E25EzhB45h5PkWrr1Y5bih6uiORoHghmf4x+dZSt3DKjVSQwxKz8Y1EDUXhJpwNR6rUaaEnlTzZMGdhh5D8iztVnxXnQfo7mbT4ZpUq/wCq0F06jBfB4wDxmnQ2GHkVlUjo1z87c1nToDM2UGEmjHse1/OCjAw/MzplSMJc4zB495h2KW8FSjPx51KwB2KewwBBDptRrdXabtynPTywA5IB7kz86U/7hg3Of1jA8MOj+NbYG2rFSgz44P8AI4YAYdfnWcoLNpQ4uvcbdVmaB4cvEewwBU5zKjMdwi7I8UwM1YHPOT90NtkA7AAEWtZMl+pVH1jZzE944vY69rHGBDaQGcKXcb8v+cKdJz5eQGAPLKU24ZdvVSQ/FqRw2nwwMwW1ptYocl3QcDWB/wDaQ6NY/P8AWmB1CqxqdxRi2/WJ0v1bR5QPxY0fDDRm4YfnXL7t4Y1GiVmRblzQY1KrHLHKpsg95yQTVHeqVpXpSmHH5kmExUglBBl9B/JmtXxyn1G/KpEvWXOeP2+cQ8MIbGewANdcDGcMQg23cP8AyquCq0QJ0bCHLib9Ez68/k6FL4nWe4y1QYlGfjVJqa8YMy457JJgax8kMCiS2DOTHzzktGe/BWVYuSlTKzCcptNk+oqdJ1maZIk/5wXt8wUOghGadfrEF5mQ9mzGz2Bs61mpIcs+bDnWHWu9s8b7HtKzaqxYdpVWBcFU/pDuDzP5M+xcXqUmo3bWQnXBUocN1/BgMwAAAFGwzArFR9bhUX388GdANi1dEolGh0TXnMBzsozf1T38sHnwVfdtsXBAfConToz1Pww5unnnGVhfmpTalSvUdSM6ecAH2WTw8HNeiirE9isFzZv5u9h+dUh7FLePebezV+RQc83dNUo8ybZ55tgezvVxTTchu80GeqYYZgHQqkAwdNvyGtNRKa/Pa02Nh8sew1FwyXDz3rWluy6rBkz44RsAOOeGB+c/OpFvUSdVbNPC35NSdlScAeZ68/n+RZ+AdzaEuDQAemG+zr8u0G/QUSHXn4YtMMTpMB0DzB5k8AzDzrmwtralaV+W3IC2J1snGkPnoQww1gePyAa6XYFe4cs2bQqbXH5Map0g3wrdJDWBmpBn7488D6FyqpVu443qe7abc0l6uws+ZelvZ4PgeGxemOAnE7hzMo0uh3VZUCBU7hPnZgSI3MszOzONnvDPyLbA4/xvvCx7zdokGzrRgUGbTgNiYcHDxg6w2B/jUqz4dgcRbSuC6r7lSftHRKU+bLseHmGz2JvK440w7qh1mt2/Y/C8KJCoL3rRmcbODxwjDDv6wzzP5F59+/Mn4E6ZGafZweZDMAe+teoWUPnq3g3gb05/YAAHWa6BxLs+j0qzbclsMctVYtNYhVWmBse50HzzN4PowWKoNwv0SPIBthk2nzYPV7wNk8wXeOJDNNoNWrd3XOxg7ddtsVCN5wflBvD+NZ21SLJsCo3/AGNeHFC37megUK3ji+sqfyxg8YGx1guocOuKnHQ7h+wdO5mBfE1kwZq1wzzzeihvAADozw3rm/Arid6ntS6+HL7+FCvmjmb0iRvegTWQ8EPoPDBdgue22JnBbh/xUu2pTJk2bXmD9YxHjA4dLNg/J0aJgseEdltjjrdVt3lR+GXF66ocyoVt4IrNRiUQ2fHP3Of428F6GkxmJMfQlt5hmB/nA8wX5s3JWOIQMW/fn2Ek1in2hWGJoVsHjeOSYPZgbzIdGa9PWZ6TNDrdBO8alR/VtQlBrnTDPWCSfnZ/BXUO+yOtQpOoueWrx+o9w3bHsu57cqVsVCqM81Rzl72akHyH5/kXRT01UBpl5zJWDMlvFQsN6DoWiVg9MbAPEcVFPezM9PoRPPOZIA60Asn7txUUmBUefNzTwWlBnNOhGzPeqgY0zlRj0zUhmTn1q9rFNzAj0FkplNnRmjfb6AWzFcAeSI4rbwaemqWlT8DDXWiZebeENNXQrZMP7k6C4vdVHfZlOvtvg99C7we9cfuSGEPnTbbwDWPNd/of5XNzMYybmCOBJbjAbb7YH9aj5uAaAwX23zkiS9FePwGwBGzDiGYc3UQZ+QOs1WJGtWcJz0PRP24GHnBXBzIr0AIgH4uCywHingec7FODYJLLjL5g52KPn8E69mZ5uJpUkYGnQNz8RNZ/BLP4L3DOFqzVXw71YM1XzuLOa2B708Dyxw322Uat/PgpYVhz8c1hwkuB7NTY0xw/eKMOjbcM1KI9scb601MoMGf4nQfnBZpmS/8AiKwZrbjIYOOLHBsEyiPxvnUKTAPHUbbWljT25I7zRvQGD9g4BgmzDCGojwLXVKiOYG422qF6G4A+ItotzXCnA08ieZTWHxXQk6B70jPMkKSB0M8lLB5QQNPAalW0rWRm8ADvUdXtn0SDcNZaps48AwPPYotcHqbZN61KAFVg0r7o/vB3PsVxR/WVEEGKqwYOguy0SgxY1IiRG29jDOAKnueyZcz73EY6F8Hm9x9KOFySYy+8+b7bfWovfga6EzDbpTWFVgmyHnNUVSoME3TfihhmtuH3NovhUQRoh+8Tui/7hAEZyHKOK++B7+sFpqkEUIDRxAADw7F2XzMceVOzzalHJcD3aON68BoH26O8819CUmfmWo/Bejf9jgC4+xthIZmZh0GH1qayeZIWbeqIQjnPsYNYZgokaSGGemYKNi7ZZzUoIwAqoJPyGC0B0GsA0DnKngYdaybmgBOg8AdaiTKbUoAg445mBmq96S4CgaPCJj7RVkmN+GCrefkedSo1Vb71YhHnl7NOsvN4aZ9anPHTnvYOeL5EDMNszzVs8IRycCTrNYiB1oJlNf0j0wVOcZ8PaNqteDbTBVWPxEjeiPKhZ37MFLAFL0plNiyQWamU19ktPTWwAEDzME/b71rFvMOfnDcP3aiSYb4e7XQ5jNOBrwGAzVObIH7RdkW5rhj2QPLTV3yD7Ig24+yf0KWdKb9oAL4EZtnsXqMEENxCcBwzU0DTpmCwu1q8IZh7RW1N8EkyG9TmQWPM18NBDweDxE7yDBqnZMwU1me+C53R4TggNgiAMUcaYBlg4rDk2DXOKo4GYKOFKYP3e9aIIzYAjw+K92pmvUJh2Ksk0fB09MFu8M1HehsH2L3Yw4Q076tcP3a0oUpjNTmYDAe7TaWPOlHj7NVj1Ec1dRdFNmJ+GoT0Nj8NV3PMOeTKDsz01kp7OEowXYJkNjHT01lZ9BYMz1AXZw8zmvhYQAcPoSMFazKbyZ6ar3gXZFua4NAkYYoDPFO9a3Ec0KkGBoMTRIAM1IA0YU3bqG+AKQEOJ2P5oDjZ5ArANQFBB5hkvDUg5gGsLanjkuKQy9tVZzLPnRhMb7DBZiaZoM/go/M5qWAbF7dh3t1EORoM92CkAzmKzINBqGpWDgAop7C8NSAkuYLJYt6lxtQ00AZhqKwpptgemoUb0X/w02eoBrS+BgqySywZaija1ezJ371YMzG8FXvM4DqAo4ScD3ph7tpo0kFYMmBnh51kQmK7gT29huLHC9vNVzxn7S4ltGw3gzrZ4L0Oz9/gR52nsfDNcd9JCMDNbplcYbwzXQOGle9d2LH372AwWj1LmB1qsM1ZzDzNQXgXRDCzUaTu8RXEZ4DBU5sqRDewTCNrU1Wz2c81KOSAJozzaRqy9SAwzVUeotRMZA1SSYwAWxtdPDbktHjHgSu4b2eGBqnZZ371awwYD2aXC4WAGZj4iXQo71SYZBQnqrrLk+Ztta5t+dHrRGfE1AVCDwfiJA8B9ZrX5kdzQBMY/EUtnTNZoDAPZp0Kq+yov1jupozZbTPgAe8wBUp1581H1n5hKPmO62gOZBD34IAmRfOqQIDmfs1aw6O5hqHmCfNCv1rOM8pAB+7TUOG5ng4rtmA3gC5uaG8K8Ixn0NpSabm14gLVRqUBhsBWAUFv3jea47tu8+3hw9odV1X37jnsyD/o4dAKkjcGbDprAVW4JUaT88jea7xedjUaZHN+K3hIDoXNGaP6nmadzS2Qjh51Gx5q4kW9wyk83OtVw2dA8DaANgLmlBo79VkHoMZ4da79xRtv7bVl2h8Pacyy0G953szPz/oXHJnD26rMqWhOfhgf991op1CqnbEayDYgvswHQDe1nvNcHe13n3X23NhmrusBWeXDngza+RVgPbcMFKTQAGWo4tVAtuJMjhytSh6p9hrK4czIBjzrYW3w6zkZ/aMA+hQLqg/8jGnXOaZekH7kFZ03jGxR825bAZn2Z9H8CyVYjRaDV9lS5zAN+9aCt3bYFSox6dqxmZZ9GlvwUqQbn4tP3JHOCD4Mxz7ABZqBUrfhuhLfi60gO/NYrfkrilPVJl3Ui0Y55+QAXu1re4a8xcjWhFimDvkNZCZSp0DDm2MM+haWHGr9VrLX8z8gffn0LewLVodVdCJWKrDZkdgda9Hi9kwNrUTWtmmmZLHK6moCjnUjMMGwAF5C1ke+PppUeqy6PWWp0Vz2HZ51CZecNreaA/xP+NSNXfIU6fpXBTm/bh431rGGeZIwmS2WjYA9h9iBVAWwwzQ9iQBvRPRn8NQG81tFiOals0qWcXmwcDS+tVh6new8H1gpDMl/DDM8FYvrPs+o3gchilVKMzLY6GXut76Fe3PW6qDtPtyqwXocinbDz71mrPqvqG6KfXNTDkjzWovC5/5Rb1aqIaMZoMMFHmBveHUmm2Mw7dVRcjTHTDBlrsTP8qnEOq1nmnKqfyNM7AAEP2zo0N31bEtWmnh39ClzLwtKjx+bpTB+tfkDoXyuSFHbVuess1I/vxmfM9/Yrit8QuJtt1R1+JLjSYgdYSGcwBYygvctSTrEpwGZD55hmaavCtvz7cN9uWo8QlSTDtWpVR2pXHVZJumeZsx2dgK2ti7aVSqoFOtW2ZJx3/2l6OH3l4Fjwolc5L1k+xnH8+asqVPbZpbWhEP1qHsZDXWC74Q7tb1mWdflx89dsV6j0+EzvCc9o5rm/EKNZ0O9DYodSZn0T3wRAz2KkemXVUoTrFVt+pGR/wBIkPGtXw6ogW9S5d1VWjsyXWDAIbJnszWwmzLnsClZxLftGpQ3ZQAYO8mYAar7PP1bUZFSncmyUrrMD2AH0K44nXteL0WlVKsy4D3R9xaZ6FhKbSolbI332MzPoBRfMKq87eCBWTrlNlBJhH3gs+y8+zNN9vZmuxs0GDJtybb4N4BFADBcce5o54RQ3uqNrSJ70oHdCX0dil0GM2BO1Vz2UUMzUSqvBVZQOTtjrAYYAthb1Ki/YOqg4HXmsxknjcB91xg8GjPMAWwoMmC9agU7Uwd7Dw6FQ29A58SiS28MAzz86UONKo9Xd0KbJkx9mbOanYN6mzqDPj1JxjM4r2uB9hqkrbNZvCqA/LbZN1/3IBvNbCfUp0yG6ESnG80G82g6wBZ+BArJyo9ct+VgYexOQe8PyLTYp58mdG/mqsOPAcXoZMOhNRnmwDByKbzR9gLoFKsmVdtxmd+SphzX43NMgyzgB/nXP9bRCXFBswaYPAAeDf8AnXXFiw5+JAihgAHI8h9ij1KY/WwCJhgDfZmqzmW3lIprxg7qA4lCW9DOGcfXAzdw2ZmtLTZ7mkbj/kVEZ5mBub8A2KIZys/HM8PkWPNA0zwToFUabg7yfDNSodNptVamua7wVCL2B3qLSj5kGn/wFI0W4zvN+98648CVw3ZYn3qEGW3mzomq+Sb8OszYjB7GDwzWrsmBTXqzqU2q8tLAM/Y7FlJ/g1mbp+NrycGT7DNXFqbv7Z0Y7BChhSgZm0s8wnA9vPPsMAUi6r8fCfQqNxCnGFCiwAlQIkQ8wz/yLKQLAY9cx4NVrLMaRNwzDP8AZvrW74i2H6qjx265UodSj0uHhzARsDAP1rdzrXhdnP4tUyoyqzclNo814IsmWzJMJQAewAP5F2j0t+FFlUGzWqbQ4kyfcEr71z06SbzwMB/k+Rc14S77IdqtccN5qb45umfXgol7cdeI17W/9jnHwgHKDRB4N8p6L5DNZ3+hbk7MCgT6JyNNqXLOv7z34ZmmuD8aoheDsGLWaxTZAZmzLgyd4GH/ANmqR5mVSprWp7j2PyKbZlVnQ59QqMR/Rwe3nguYd4o9bvWsT5Eu+6q9W9nKgFR8YDD5PIuWXhVZZyD/AJ/mBIiyThRo4bAZYz6AV9Y1bcelO1/PmXecDefQYfQuu0SwOFFeub7K1G+KDJqFUjesGeYMDBl/yAn72ryFUqk/Dal0pvofPMzPrVfSo2s+EQ33gafMGTwPrBehT9FdisVmu2/BuCNPmxT8GXBP7qyHkWK4Y8HOJNbrct+1bfzdoOYSTqOwM/J9a6YjD24YSsUp+jzTgm37BHJmSpkUJz8vk8AwZNrv86tbnmOVKadKrFD9VV2KZs1Lf1rJVIOWd0O1LRhe2lMppz2p14y5MloOjZmusTLMsq87ehVW3G8JtIMDmB1yjD6FxIDAGMG10CiMznqS7XKbUeQMGQ0Xe8z8imEI/GaBTabdsKdSokYKfVIwHosn3/OHYsfW7bqttvwqxUrckswje+7PPbAk/QvTfo02lbFz3bb/ABDr9x0cJtOefiz6ZUdhvbMAMPOt/wClpwxuOvXNbVzynIb1CpcYwNk4xmyyHXmfnXd0jxVbdeqVNrM2VSqa8HrAMJIAeeDHeaiXC9SjrLphKN6JnmB4d/kVzckaXGu15+2KNm0EkzDlw+7SQ+fyf7CpK8cufNOst0rk48o8wZ1s/wCNRYs2a948uJ6n0TlBgbr3epTNpXO9Ka5WhzDkb342AbDAOtVtNqT8aeFcbgsyQhdYH0YLqp0p+9oFpfZFw6PNfP1W86Bm8zGM+jP61kONVKt84+BxKbGjbMDDrMz86kUp59kTqLEF49DrPDoXYOIvDGsWrHp9q3VasONcELN+BLp2GjMih1me9ZF7QqtRBulVU41PfjGb3PBogB+Rn5Faw1ILguG3olx4f6Ok5snnvZMPkWtvmZb8z1FKivhJiTmeamRwP2MoFlKC9BesqpxAlfeNbNnZvw86uLe+zlVoMeDUbczainyoHno5rnFbMtJyS1U6zEiBGa5PXDAOsFQ02AxRM3KxECfSpXgmbL2Bh9C69f8AGi0SGdNocT7u/D0czPAGQPzmuXnUm6DAhUNy3KPPaY8c5GeZn+dAFw23OtXmApsvmaeDwYGYZgBn2Ksk0qpQwacrkTA3wzD5wWjuG8514Umn24coIxxewGfbH51Xhfj/AKhChy6VrTmDMOY68A+hagLbqp0SjVijSqbJehVEM/B6AfVfDOs3Pc0eK5OkssgzmZsngeAIalP1o+hEfeD6EVBmPwzj3H0U886XJe/BM+9WM7MmRXqo9ymYb+/r/OmejB/vzzVtdtYfrdbDU5PSpwckyceMDOsAd5qsDTedNt88NhmrC1txn51e02e5y+vBnHGmsb2TNZcDzJS2QMzwVDS0q5KrSpEs6a/o86zoGYdYfQfYtdwxrzFn1Kpy5zEM49Rh6LxyI2sbPfmAef5FhHmWIzUdh/MHevAwU3nHDa8dY3AlvQGLhuC46lBrjz0IHjlRnmgwB58/k7FraPf9z0p+2qxZVGjevbXk56Wibx59mYfPvVVQaO/ApcqW+w9Gjv4bwPrDfsUKNMYh1sK5SpZvTecwweZwDDyGvR2i5/Sc428S4cu0Qg0Gqm/TTZe9WQDZMA7w3mfQuSXOcE7Xp9D9aw5lQi+MeB+xA/c/XmsuZ1UKsfq1iSc3WM8GTWou2ya/Gfpjlcr9EORKNiKzHZzAwA+/eAZ/OahS1jUSsU2jOsW/QKbMkMAcKthLMMGZT3QAGf50deuGq1Kz6fKuZ+TVT5Zin8x0ctFZ6Gdiz4VWsWHWa3bGhGmA/hFqUfPeYB2B/rroHDG1addVErDFNiQKxChdESQ8bM0O89Hz/QvLYKqHAY+y8ec2bLMg4z7H3c9hsBh1+del7e4kTrPsOj8PaBEmU2RcMNhg2ZEYHgZMzADMDPoBecLwtipW3dAU62GMIRyeSZ5vBk3jwzP8nzqPR7tqVBr1Pr7l1Gfq54D379n4H0Lm3hrt7IjXDdp8QYnB20bq5mbVAP1lLkQ/vTMVkwNkDAPz71oKbwonf+kxCpUWUDNKBkKvJg4dD5hvAP0ZrmPDe/6NbfpIher9RmMx6jyrMaWZ5wgivYB42HQHk/IvS1vTwrHpBXhVWP8AoimsRTPyZ/8A2Cv1r/LJsOKnDGlcVLf9TPzjptVhHztHqbIb4coOg/oVTwuvmq3PTptDvhsIF52991qsTo1gDolB8hrZ85uXPeLVsVmZMpnE2xGM7ttoMOUDorELvin5/kXU1i3TQQd6orPvCj35bUK6rffzjyswNk+uM+HWwfzgrJk96sfZgYKCEnA1ZvbxVTJjbvDQ2sGZjf4isI0ls/eLKhqMmpsOS4CrA0p6bwKtnsscufhqP6yw94o8meDzWCaGPqoaMo8OhFTargQhqI6n+0KOEPvBdDFqAPJc8uSfBgVeQw+Gt5wBa4Jmj7RZ+vUqDPkHOw8U+tb+n/mRbllbZb1+aY6D7FEZ6FcTIxgObgKneB8HdM21+k4Xx+ZHMMCQ9alZt4+Ih5bf4atKPooAAwNWARnDw8iB6MbKbDQM5+0bUI2XGSwNWrPWpAMsG6eoGw028wz6s6VR357uo+Bg1hn0daIKbB9m+5mH0LbQ7nisxWmA6MMMMOxRzWRCDAjQWcIrkQD+sFIuSzAeaaqVDg6OftgzVgYUd6aM5hvR+TBaZmsU1mnaGmZrldeHL/ULYMGw+594PoBU+Bh4a6xMh06fC55hjB0FjK3R+uRFYWkWnDLgZ5qQElz3i0Ua2IrwA42Z/OnnrDCY6bjFV0fI1op3QxxaiZmeQ1ZRqw/GNZ02Z0CRIYcYMDYPen2XjP2itbXBUm3g3uJHGgyRWcA/w3FLjTHGes0wHZ9vAfiRWzNZ96M4y7puN71qwquH/qA0BsxJnjuBmaQYZLD4oMPirZ6Bh0KO9GwDoW22OEFGndFz+tHouf1q7SQK9s8J0avR6lEwzinvAz6wVOyGCvqIfJuhLBvP5Fy837HRD0hRHnHooG/sV0Z+FprntsVUDaazcW4hyc1+W5o/q+xH/CkuSiBWIRxX28wVJDtKlMyAflxAedAMAM1vjAMFE5ZnyKN/hbklYsmKzKkS22DDfmmreok6qz48FhvNoDzNdVmQ2D9oChW3R2KDKdOI+8YOHmAH2Lb6RcBAMP2pgM/oRnTYjw4OMB+hO6xmjee2rj7LXgmabEw03GwP8ip6lYdOnvtP7/APPDzq1jSd6m84H9SbswqnrbpRtafq1n9CicmbJmBns7FfPTGz6FBPem7MKeZR+faNj+NYepWNdTMo9DROP2GBrqAAloga2jmwxw4fJjVWA6bcuKYKFzM4z02GHjPyAC7rPt6m1Jg4r7GYGhpVn0OjtacWJgto9naPnck9Wy2Q1HH83fJgrWiUGsVLPxAZw866x6ng472ANIKbTo3iMMABpfsr6WVO0n5P7Cej/eqprFsVKBH132NZnvNkOhdG2JaLf9aj6TpcfjU1+Y7psRDz+cFZM2ZOez5pw2fyLobwAB7GwQbFf0nS5vUrVfjMajeazT2bJ6a7aDMQ/aBmuRXhTZcOuSACKZtGeYAHYtuHmMKQ3s00lJAw3mGCjm85ivpR5cFpGsgNQOc3ailMyWDPf/wqkAeCXjqNt7ELLznvFZMyWD8PsTwQ4p9Cm/0CIzJcBTYclwy8RL1b+G2no0BwHdRY3fY6vC6A29Deo5vNqczGA2tMwUgIbAB7Nc1ivjGeemrpmS5jhmqWeBg/4AIAkv8AnWC2lB5PbDWfjTz9mZqaElz3biLWeHxQG97tRweP3iWeZLJSRn8FIZBw/kUcMMlLZeBQD5b/AIPnTJxlcMxtYPDTvq1zyKNrZeTTX8dTBJmgg8Oo5mtcED8RtLlmwTaHMbnsadgb8RjP5FjwsO55JHhTcP708F37RzTJxgwXTw+5zMelwgOF1zyZAN4Rmfn1kFY4ezqD7SWEn6AXbeW3+Gq96icyfjsAa28fxHlY9Lhjxsd4dCj5t+7W+n8OqyBSHzcjGHXgCzJ2fVQ3sGBr6UezDG/WtRPPOdCTMad+Aa6BTbYisiBym83lNOiRMtRhtPsiD5rc1OBOkyAYCJJzPo2LrVt8KLZZig/WZcmTI7wzwQ0em5zw1+xbBkNEtNcHN78On57BGtKx4AmDFDjb/PvTR8PbHmNYepgZ+cDUo4ZvHqNp49RkAAFzd1tPwx8zgzQw/YajMD8+apJ/D2sU2PmxhJ+g10gJmZqayzEeDendY4b9m6/q5nTcA+tWEalVIA3sfxrqs8IIEqKZoe4W32WzwwkmiTjLU01EOmy2R3trUT5OGaonpLj3W4umP7jNTmb4F1mnQkmHvDUeYbeqo4GunDLa7CpP7G9dS+fz94s/kadCS37xRg2uNZwxVY89uUgJjGHtE1yD893wOhRhQAkqWzJcBSIdmPnvflACmnaWHROz/Om4e4Yfi1SvXFmyH9PM4pgazvAqsYMSKV5M1024befmW/Ngt94Z/oXB+GlS9VXXoZ4b8Fja4drn6gOqO89sDTVrWGcz1G1SPbFtCLRzkuKOc9wPeIXj3qC9vXZEOZbMz9bvUgJLizgGYGpATzTpNrszyVe8o5zH/wARM84mDYzTOs5/UjOSms/gtkgM3DSSzbyTWfwVxAMzTrIZqPnvVhDeYDBMCQEZ8/Ztp4Ka+fu1aw3mMdRWAPNrju1YVLNBAxzcVhDgMRvZtqXn8ElzbbZHyzGeeG9SM9ij5GiBc7VLBTWT2KvBWEfoWFtVnTZj4SAWljVUD9oax4G2Cd5hxc1wqGuekwXvbrBXhw6tG55rU6o1mYBgGGDJqfzDiHNg+ttRho5PfMmdTWvs/atq60cOuRhvVfefC62IdnhOuOJnVTDMz8i7ayDBl0KcdBpVbY5WosZgin543DVbRhu6EVuZh8+9YqMDEmqA443nHfPDBe2+PHArhzUreN+DEeCsMHmGB9a8j16xoNpEHrWdJB3yINxTbM4bc405SqGcmR2AB71Kv+MdEoJvwbVnw5HY88s5Z/FqjWY7zfqM5P1mqXiLxdnX4QNxHJMaO2eejmpGKOsTowGGAZnvPMM1E9ZSz7/4FNZoNwVVo5cGnHJaAOtRTps7l9P1c8Bh17M1OxEMwXoPgJQaHApvrWo3jGjSDDAw6M/4150Blw07D5FmUAObF4tvuMcDRuN1+DOOY15wkgoNjVKVDfAG2AP5zDM0bNbs5mOAHEe+sFYURk7knhFtGCZ/XsUDxOb29OhvTWHxRgeK6FprxuRgzNAcxxNSZ7j0UGD7FHz+Ci4FryzcxoeUb8XvRswHw9uGCj02SEYDzV2HjRwzTAahwAMPDb3oJgSoe9+Iel51YQKa5P8AurEvRPzrQMxokakBTqjOZk4eRRsY96tzpItNxG8GmQwzVS8ybx56YfkWzOfSg+6MUZkw85qRApsGfKCJoABn5E2Of4fFWdKZcjEEoG94dC1UaiMU2vBTaxTmTz6DBlR69Sn6JP5R9jDPeHzgmxHpsypPVkJb9H59ryK4qUn1k/r+puQ+RVVNrD9NfBxjf8i0ei/Uh54KacbPsNc1qUMyY2yPKH3qPJjPwIYOVFg2Qf8AYmhrAZvhqOAAB1rW3JUjk2pEb02eXY6PnSIGSgRp2/XfM4nYBmrKHGlA6D9N6w6FThMMxWjtWZy0r2AGZ9Bn2KhteJ11TgtKEwEv7xKAOZ2d6wp3tcHqgLcfrBhCc6wDvXQAuGy5NLCLc9KeePvDoWfpUmwKJWeepVAjSQc9jrdihLKSQnSXTf1ze+s1e2M8/Dqjrk7Z4Oxa6ZxOiQ3ZYW/bEaM7KDAze37/ADqPVfszJtz1jL/06f8AR+xZWL2zJ/JsS33G40kpWefyLTXtTbApVJiQarb8YOaAzZdaZwwP6wXIYdYfhxdNhaug8S35kX1NceBxw6HTDPYshzd63m40owib1op//sWcRv2pmDIfWtRMqVuA7yODMaIfRIAN+aizLeo9HoxnXKwBx23tbYH8C1ixSQOG94xocdhiLGgOsHnqvSeta2HSuQd15U4Dl94R3tizR+pq2XKU6VJZDzvHgr3C3IboQaUx0dbue9Y81hqtw6VAjtS24m997B48+sDVgdgW/ckKJKs6XyE3DDRz2GqKtw5U/DQMzAOwz6FST6lelEhg3R38Gs8MFj4sbONxCuCyasFSrEX1q1FZ5V5rDDA/rXIaqH2wuCbOD7mcrxgadWzo9z1WfAkUa56k9MN8883QA0FEpsuq1J1+DQ4z1EDNmYfQYL6vrWOT1KBOpUjQnMYH2fOmmTcB0F0i4WbfmVR23KM2cymZ+C6fuX/rWCejOQ3zYlsGBgeG9dQsjZPSNxjeYBmjNlgKXHnA+Zun1h5FLoNNbqtSjg+34XeiemOA7NiA2ARwPoBQHaJPYgQJbj/vwwBdDZtJ+HQYN1VWDJeoT/8ASI54GuVH7Az912LpFSgPw+GjUSJVZm/B4I5vbFhcC4jWqwyxIqtHDn48oMAPoM/9dZw7eYk16lMMPhGBjxsDP+BDR72nQ7NAHz8WLM8E+9dYgWBR7qhR7qfgnGMGQeZwezzBcylfwKrFNPjdU7VvGlRpLVejcqGrv0T69ivaraUWfQ6rblfuoKVQmKwZ73tYzAOjf/rrjkwAgV4Lj1HgnRXswMPIqK5GZ1Vddrj+8Hz1pP8Arrsj2E3C7evaWDp2dY9fkz4XP4MvO9gZ+RSrwrcs7gazcjBNZwZORB2G8Brn9Bje1rAPgAdmZ4Zq+OA3UqyFVYnA9ImyQMwDsWiMNLXrDfO5olOpUrWkPsg9g6fXmptscNJ0y45FnONvUcJUPWMwwMAPzmp0apetZ9wXOw5hOoIYQ/OzgrDhvfNRsauVB+qxecp9ehmwbwdcZctwA+zzEPh5UItDpsYJtIkhzjsc973zgsfM4b1Gm0Z24K5O9VTX3vucQ8MzzXWLh4tWr9hahSrcoEkwYAGAOR1nga4pcN21m8y+0dVcA5oHnGaZDAGcPIo8QjhduonFG2PR14cyLA9TVV666iD75mB+CZn3gf8AkXN6l6SfGKsUuPRqbVQo8LRBmSERnfJ+fM1hLhuqv3I161uOc9JqBhgZn5A6FX02q8hNhTsM+VPPA+g13cbYFSjTmZHNy6iD2fW7rZmmo0PnIEt99/fFMAWovaZzlej1yVQ40PmnmH+XjvZhgmrzrFj1KqS37OpUkI9RADeAOhkwWlwpkXjPl9nWtdMurn7UpNAgsGEeKGb2feayLxsdYb08b0tk8G3MGsM8FjcJWFHqTFHrLVVMAe0DzwNd4onHW+KbbMi7qrVTqtPM8PVL0zwWQP8Aj8684gy487hqAH1rTU2sHDgNU6o03Wg7+cCP1yQW0W5rdtvC86VWKvT6NUrqgW3H9WmYSKd+zbw71kjsnhDb1r1W3H+IECt118M4dRDPlYf1msjZlnwbngVulUOgTDdMOapsjPAGWA681W3tRJ1pFEpRxGQj1FnW2b8/z96QtEeqtNhwDi0YAM5QGzJdPeBh8i7Rw0uG7ZNgVC47OtkAat6NhUqyGzM9nZ34Lz4exo3Pw+xe3rMu235PA65eF9v2cdBCnW8dQPOZrZm9v8neou1qTjNW6cfAe17nu62Idz3HOjHFCo1EzzZzPMzAAXlz7Q60eOxLiMvaAYBmttdt/wBNjUilUPUCpZ4TXgkBsZ2dAeTr/gXPK8y563124ujEfADDeo2tew7zqtHLQpTcZkD6w0esFfW3UokyrxD0JMCOxs2HsA/OsDP2VcPIDILd0GMxSqWFZDW5eqbNY/cmHeCj2BcXVdsuZTZdKN/Ct0t4DeCRsDD/ADrBVW4ZdVdj804ycjfngzgrI2WKrccgK4/zjvZIZ3n+tKNZ8s7mhQYIBnnnvPrWMCikvRGWtTolB3o59SYOFH0+WknKDN57rPNTaxbEo69IY5E2Qw1sz6DDzgs5JNgHTbYc1mvPgtoDzIaLDr7hgYBvNQQnznoR0PU+5HJ5o2fnUv1bTTjyH85LJ57AaZzwDzmoRhUoGbDjDzOYZ4GHWtwj2IOtbOj8Lq/clBolct9wJ/rd42DaDYcY/nVJclt3BatxzbRrlKeZqFOeNiSDW8AP60FIHWpcN5jmGtd/RAzwM/IugcBLAo3EviI1aVc1galQ39Ew2feuwM+xR7ksZ/hFxEm2jc8T7xTsNY9+B5hnsVC1k2qxQbShX+3/AD9SqoD8X7xGMDZfzwDrWVZZlvRfWuh9yivYG9n3rrHDeqv3+xH4LVm4KVR7EYmBUKxOkPYHh5Az7zNBxC4b2/ZgesrRuONMo9Uqr7MCJnvANnW90Ge9c53eHPPVtOZo1Qb9az4dQN4NFkD2PAeeeapZM/OEDCuLqjVyHUTiOBv7MDAwNU9SOJMP7pB5MMAAw1s96saqgs1y26TCrlGYen1UwM+UON1mHf8AOujUq6r/AK3ZtPfv+zo0amMSQqdKrkgOWMDDeezv6Fj7VqtNqUWbVbu+5tUuHFp+tH6z39am8VOIU69qHRLOtm5+ZoVLjAARzDA3jz7/AKEZIl53Pateq/2goz71SrE2pc087y2iDLHk+jBVsyvVWHnUqO/6tmhJAwlxNgMmHeBrH0d5ipTQiG+YSD6MO81pY0/CjHRnGM2jPMzz70eOh+rbq4kWXWOJtcvhmTU6DGCazHBkDN4M8N5hhh/sLKcnRnrep8quTtF6oxjNnZnFzM1d2NyMCmzbVrmyn3rDwCR3smyeYfX2KRR+Fd1XDS5tuRZUM/Vz2syYPbM/n8gLlu1wqbMenRquFAqNS5CIZhrb97IZ9eHevfHouvTplZv2XVajGnzT5Vh6RHPMDVtwT4Y2PMhyL/ftyAc2rsxeZiPRgPkJrIYPYfnS4CU2LSr/AOKbFNiBGj+tdjLXQG81tw8I7UjA9E9ihc18EtZboc8rYP8AB+8pfEOlRP8AklcrwBcjTQfsEo9gTQ+vv+gF1aNva1w3gfQYdBgqx7lJkKRSqjBZmQpQYPR3ugw/3NcfoPF21eA9yyODPEa4zjUpnB+3qjIDMwivdDLx/If8CLd1NAbKiw6lBqUVmo0qow58KVvZkRHswMPrUjmG0EeYzgCpDk4F1q9km2YYLPz4za1gHz+zrUd6TmepqKIYPgemkeoC2wgMk9YFIpWZu6B9CZDepcMNEs1dgK3AfBrXYb2d6zr0/Dw310A3mJMNYG5AbZkOuA3sV+tf60Wp3uR1TTEmlQZg6jexRzecNEzM0dji+3DgwrzttgPaZqk6HTbBs8A8633MwZLWn1rL1KMYSkhjcIjOn7xS2WWHjBtR9H8NBzLkN/xOhbJA9TcHT008yz4SlBJAx1EJxsyUihksmB6YIYzxg71qyNk8z1E0EADd1NNai15x/QwY60odeMPDlnmCGGDYPm255FHOHounsWSmqptwsGOh2JqTVYgTTAOjDrWMzlsnscR5v6Rm51rPBtq6VM/nLDUzA96vWTwfa+tc6hzzB3NXfrt8PYPngsb4W0czo70NipQ5EF/2T4YLi9bpr9EmHBf6wW+ptYqT2HiKsr1HnV50224P3hswMDzSLXbFZ/BOhMcVqzZNYN372bMZrz55qSfD248QfiNhJA/IuzcODFqxmSrBl5RHrYuaGOpKpTzIfOmsJUb24GH1o0XABkgONnsUeNJc6FZxpMQz37FKlScBz8NNHGfD3a1oRmD9m4or1HP3ba87jCkjBEewYfb/ADqayy5Dd87XYaXqpwHwfcb2Z71qJlHYBoAgmBge8FF2uISbeqWB+G4ui23VfCDUPeuVU2HLjO+zW6o4Pgx4i+N7H7nfwt2E/P2jims71ggqT/NcqHWtBTalO9m4Gzzrht0riZGzPUUQGcCRyalswUQJiyFnngojzzhmo4ST7z2p1k2zNQtIZNPZ/BRwTuQKFDM0Sjm82CQSWwQSDSA01nreIgRKwB7BGEnMlX6zn9Sa5nBWLI5jgEmnpjfu1COTmKjnJwQSufwR+svgqSS84HiKEdSw94rF69PUU6kqnn2z94pYAw8GoqFrDmZmCsJLLD2DhtgezvVEyAMmpBzNHvUhTKPBe/orP6FlKxbFKOUbjjZ5n5NiuJNSfP3ij62fW4tfHJ+GOPDFHw91ndRiWbIfQo52HKjH4ebwedbsDUpldP2Wj5oclmU2dDdNhtjoUq1aPLn1H78Zsxw68wXQ5NHgvOnKMPFTWDbPs1r9lsvnkEyjt8v91b3rKPHOhyAYlNmya2zMzzpmfGCeINrnjmt0dMKVme5iriG8w8GbijhR22dmCkcm2yGo2Cu7REJQBBNR5lKiYajAb1HzwJWDO8VzbXhSnDwU2MG3erA4weRNGDfu21ezBkwRggw+KE3sCUbDvenWTwJQs96dA1A00CfgAaals1Vs5HiOLKBJcD2aLWM+tFtnMqrGh4bm9U/rhz8RUmsln8EGgZquaN6qsAGo4s5ngjzzFBYHWGM1Xz6rm74biiGopgrQdOpTvxzTQST3gfeg0UGHxTYkMg2alhGYNRI2xWGs2bvhtptaRGh4b1LDUBRAeP3ilhJwUA838NiJnXe9omgkt5ozkn0N7FKBcs2Hs0jefZDw1HzfD3ijz5hg14jipKFJkgBqEb2Zpl6TmSONhqopnbqecZzWRenuAu1nJivQuRfiMn9aKHR7cjFzbFHjA6Hfgu/1vb62Nw4Vz+fYf6FIZkrqd4U2j1KKbjkQAdDvAFydmMeemHnX1Y5o5HHcdaaZpdikM284Y6muaiTIE6Gen2LxCxoklgJoNvgBh8627J00PZsAC51A/aBWlZkvh3rk5nZC+OZv8PYCDn/xHFTnM2daiHMPzrgXtq4zzD2q35wwXlS8Kb9m+IzrHR95A16DZqT7Ps1w/jezhXgqve+zvSLS7mH3+nR5fnBVkmMonC6f61siJ4+ZsbFbTA3rsi2dwz8mMe9RAjfIr3DM/EbUgIzAe7XZ45mNwzR0361HOMtRJBjDDTUQIDGC27kYZp7YKjq7mUpzA1UvMuAem4r25rgOfwSz+CZMEYLZ6az+CLP4I8Pilh8UQWfwSA96WHxSw+KtazZmOArKNVfxAVAH7zpW1tK0nK3icoPunea4+aGsWCNVWDPBS+ZbP2a0FVsm34cfTgsYGHesjyz8Z3DPoXH+90xGFnnmjBMsnsTzJ71ja4TYwZkpoMnklDjOKwCMua22EfRc/rRgy4poMo8PiuYwj6OxGDJ/hqQDKdBlQ0GEYMFKZDBRwPFSAe2JYaqTJnHPkWAN3zmvJnHXhXWa3PAHIPLBnm9IzzXrsHkqloSaXIYfYZPMMN4KR+dVw2Hw2t6jGb9Zkyan9a5Zh8V69rfBPhX9oZb9z1IM88zaZPAFwXipSrKolcCDY77z0ffmZ/kRSy4IPTq3V/swc5lmEYb8+tdIv+2KHbFO9cMOPPTnzwAHjzA151oNeft6fz0WU8yfyLote4tUqt0SOD8uS9NYPPxVAz8mj8/HdnGxh8mjgs+HqCMGnVaUDzvnM1LqXE646k1oagMh8gLLz5j8x/XcRaa9RwmOm5Bbwjn0Aui8EIdRtu6o9SPY0wnret6359IZl0qJMk+R7DYpUatuW8RxH2OZw+fBB+fWB+7bM0GfwVhPBiMZsNnn86aknTjjg3E3yO9arR0gDejAE7h8VYTJ4Erhl5wGg1zBUm9LD4pf6xoI08wPNg1YMyW5ImDeH5FRMmDMXQbbUimvGy064HesbgSJLzbOeh7VWFvSZUZ2PVX3/Cz3hh0KnA8ySh83g6wYGbRmowxdLvmsMU2GE6lNgbr+HX2LLz78g16BHYrLgetovQ8AYbFQ1u56ycD1Hzf3fMDwVEB5n/sJha4psxiNMKW+4tGdyOVjAH5b2kx0ACykZnWDoMPrBaCHDONTtdxtRcLSwpTdyNSOU3mwGeCB57CzY7ep34LS8N4b7NJmynwwA81jZMZ+SZ0oPZAazUKlMsSSCJFYOS98i3Z0SdbdB55+340OQHtj60rYCjUGgut24AHNf9s88G8zU0Ka/JsWboGb0iaee81qMfUoD/qiRKOUD2fk7E1ZlbiQGjiSoPM59B4Z4KazQa4FO0JTnhH1gp1pQGIFRD+apMwO8GlkHbzpsGiPtN+uIz2uGfkWck1jn3Q354BgpNKsYL5vp2h02K9TY+fjOvB+zB5zUa7beiWldHqOhy5NYjh/SNHBe9InW8zOrzr0GC3m6lc8Ny3pGg/5ANVIeuKDVgw5mBINWd4T5depceJWGA9YMHnzAbP4FjYDN+q2+D7G8M1s5NznP4fQm6rKDmw7DPeax9tyYsalx4kveyB54J255kWe6bdNYwj9gKISUZ46q7/NxhGj/OpxzJdNa1GzzWHOfOpTR0oGDjfWCuLGqTZ1E6dWZeyaGAbO9Oka2j34/b0oJ0qJzMf3wYdiu5NbtipNG/Rp0aeD/Wy8GBxg/wA65/c7PLRZsH/g7AUaw3mzkaDn4JrHpF3XozARfWVKqMYNA888+tWcC4eQoMihweWMH/Gza6w+tZyHQawyw7Fqr8ZmIZ59eeCGiQ3KV61zb2B0LvgAz40KbBb9qe9QgMLhoxxZ0sAreeAG90GtFRKCw9SDrDlSBl2UBgyyYdZrLyY1O1zYYfjbPwQW0WFSp7lqynYk5zRqGGBoXgfk1s2Gw6wzWlrFmVGTSYN/+pjk4B95w3mYedUTJmyX2gcYeZjnsA8MAz+ReiDJjTgdNh/2QdC6mDNHmWRI+1UqGE1iH9waB7M81h6PGlXhHNiDgDueG81u+FdsU5niXT6jcYU2TR6IGE/fgGZ//YLO4GRjM60BpjyMgtRZlSiwHXnJzBmDAZ5gtNxU/koptR56zpcmSc17xmgA8AP5FWUSBaOhNY1D9ZvxswAzXHzMjvrilVWQ0dSYCHT5R4AHeqW9rPqtNrLU5hgPUT54BvzAPrVPXqwxrtUpj2UI81cUesP6U1im1zmaFKh9Evfh9CiGrCXhAbpU84jctmT87PQrOzNCBV6eD7mDUpk/G7wNUk+SwdOOgMaLxhJz1cP8CsKVAlasdidUYzLQHgzmfeum7F1JjVW26lUH4M6NJj1FnQk6J9AKTZk9uZXAg1VzNpwDMF0Bm0qbGtmdOuqJDeaiwPu3Lvd64/zL8+LHDNlnlc+WMA3gf1rEaU6U+FvVissRMKVzOgDpnsA/Isvb0kOVaYbwM4p559hrYWBdv2DqR0esyjnx6of3mPI6DNWd1cKGKVUZ11UOcEalVAM4cdkNbM/IHkW0IcxuevP16qOtvtgDwdGHeqUNTVNhzrY6wXYodjTresiEcWjHJrdRB8DAIxm8YfOCxsylVmw9L7VW5JgVCqRs3gPrMF1whBZjP1KLQor7+AczyusfQyHzrQV7XZm+rbfYA9M8+ba2As7MqsGS1hTaacNryGee9WtEmUY6M7Eqs44ciEyGBgpt0eFPMhzplWdffY8XDM1XzJhz5Uds8PItgE+3LnqjsSLFmQwfjYaLR5ns+dZKtwJdvNR6lnhIMz2eReiwqVHlM1EKUxLZku4AeYbAAPnUqNRJdBlR/XjAHHlM67OB9YLJeuK5JCQ4w/vlBg8fnBXvrh+t0a34OuAHToz7GZ7O9VDK4X1vcS6rwxqz3/DY7kM49XDOSzIjZsxj/X861r3EVi7ZVKoFco8b1OG+SzH8Y/rzw2AuT1Vlhma6ww/rNAGYHhhmpsaYxMadqLFRONmzoaPetoRho724dRZNKqHEK1Z1NCk+sgp4R+ZM8OvfmexaDhdxCv8Aok12pTm6a9R6oHq+qxMAzOKALOViyZ0PhvEuq3KzUqrE1s5kH3LJ+c1sb29H5uNY1M4jUC4Ocm1hkHmaI0eb28M+vNRcLYm56a/JkVP7MRAk2/Fe2SC62QWSCfLCFymub0fPvVrSpkGZSypTkvAzMwzPoBVkZlg4RtnKeAw6ADvTAOHr1KRptgrtmZKmQmqG/UtaOHQDR78FKoL0F6E1EltxqaYZ4On3/Wo8ym/eI79GlQANgMMw71jzQs1TalXKC7Li0c9E5R563eCuDeqtw02nvvyjn1VjYZgz96A1TsxnHpQMP1WNzB9gK4jXDUaI6dKYfkm0wAb2uv8AOsdi7jTOcCPFlSngwDADZ2GszW7bqVBLUgxHno/9JN1nPetBQZNDeaOXXIJ8u+GAOnswVhVa83ApcRugVV6S0we950PbB5FzbasFM+/0snIkpkMAzMO80c+S/WLcO4LgfOZIY2MutHsjH5HgV7c5sVuvUxy36Pg7KPM2Y/RmtQHDqsSbfq1uQaadbdqlSCVP5HeEAPIa6eHmc9tX6H8m/wC1eIbVSYsuqybUBn7+YRs+Wz9/8/0L0QHBbhXY3EZq7b44ms0eI5MlTYdMj9b2th1nv2LoHo/TOHNNBqzqNcZhcFOjcrJgzjBmU8ABsz8+xcR4i3Udk8fa7XLmuCGdrnWGGJkR6GEk/Y7Nh9ALswrwa/kTpUO8rztXhlhUrcqL0WVTatz4PSWZQbzMD7wzPf8AWs56Qln1ni7xapVqxalTZNQtq22It1VMGdGKyYGfXv8AJ/gXKuMfEhviK76tg5s0+FUn5UAw8EwYPoDD8n8a2Ho98RZ1pUat2BR6xTaVXbhqTE2HUaiGbOHQYGajbJ6D4UcAeB8C3Jv2fo0C6pDGErmJZ60nDDrDyf7CLjGdgVKxjtm4KVJnwZoA+c6JgYQN4bz8h71ye85PFv0crgqtxyq5Sq87ddNlUyHUY5gAAAYHmAB5M/41z1m82K3wvpXC6wKVXp82aZ+tZDLJvGYGHQf6NivhGB4kWxUeGnEE6U5UocyWxhNZdB7ZJA9/QsVGmA9MPm/BafMzPDoBb3iLW51euOmXVd2jogz6v5TDCUAMh3gsfTfs4frV+q6zP3M+QZA+t/s3pcLgbJweVdia/YZruF8+jlBk2DQuOHDysHcNrymdetnEAAegPh1/xrglHeb1XQNvPXZNhdg4G1u8bYfhWPQ7mk0ShXWfq+qm7GB4JOefQz8/QH1rmwWx9yPVJm6KZKbs443JUoDBkAwOTFPolYfOoVHeo0OiUSW+YPOynpT7zOe8MD2Aa9F+kJQbguR2Jxfty2KlR7chMwqFSnp0A2dZgMw5oM/c9AedZS2AcOzbwp1zW/Tar6hgA9GB2NhgZ5/416MlaVgVziRAdrNv2yFYqDGb/KU6SebOHnDsA/8AIrKzK2dq3Q6xLfzhVd59ifBe2HGPoezD9aXo63tcHCW7YlwUpt4IlRMIs+If9JDfh+hemJ/CKy+OVLO6m7fqVHrtbP1hzfXonsAw6/kWEQJ3oo34DLs2muVWfPkAfKw48jokxc9h5+cF1jg4GF78S3/PWMP8a5Twr4Y/yV1msORYujnGYZjOvHmYHhvMPJ2LTcK699m7fd73aieZu95mBmvpcPrXjY7gZ4EiB5YqiXgxVZhwdQ9XDMFZvVgGXdPUwV3wYYxbVMvZry/6VHCu6nrthcSrYocmsR5sAKfPaAMzhmHRs8m8/wBC9DRqln7NWUaSowvbzL6JFeuem3RIs71VVWaVNA3gjmHgsmGe/wCReqDNV0CHTqU0cWlU2NDaM8z0gwzNTs81GFmjecUR4806YJrD4oI5qO9vUgwUfWAC03OtdMIBngSs2dMxA1SHJbTzNScAdioXXYqefvLxOtWwGBgqmq6bJ6i8j96WSqsbRI3MFScyGXiN5rZyWWJgaaz8+3sPHYbX2otxXCk1nAd2dCM3tYPE60DzLkbrbVe9Mcz2bAXTDmtIeMw6E09M5kfHb3prnPxEX3R5dLnR83A9maeZqTiA2f3iaMFCtrAJLDwameBpZ4EqzeCkAy+EXXcb2I0XDMwPzqQAMSfzrOZGpbMlwDUCwmU1sGDcYYM8FXvQ3GY4vudCuGTc95KD6FZvW3XPVYTp1KeBoz2AfWsdtcMJhgrCm6hlp4KRMo+zUY6FLpT0SBvlsaxh2K0LWlesWXQPlOjvNWE+pVUM3AYA1nDrFZmSPAYAATUz1yB6bji5sL2s4dYfemhzWALotBqTDMIHHFy+iW3XK3IMHMGQ8+a2FNs+JTcNSXJMw+dY8y+ErzvODDlNNxHM8wzNYKvXIxVRAHHOhdK5aj+/iAf1qdGgUPHNulRv0Jw83WXDigPPhv0zAFI9ZRQ63F2N63qbPkAc6IBgB7A7FYRrVtwzNxylRsD+RbfZB0uLs1VlnofWooMa46xhyNOM2j7z2Lqsa0rcgSAfg0qMB/QrN57R9ngH5Fwc3t7dOHL5lt1WA0D9Vg6IHszzQMxnADBvoW4uEG5kLTWSAzZdNtxtIvZcLClA2ZBm2tBGZVZSuUzBaWMDGkC4eb9zaEIIzeephvVgyYAKAwz9mheBwAXM6QSZLZmgA01h8UYAgd1nAJIJLiA2XE0gswmOIOZMyUI3kHONgoFrJmNgChHJM1EOTmaEHm0FkzPwH2ikMzMy9os+cnckzM39aDRnJbUV6Ts1FTPVLA/aKPJqXgH4i8wLnn2/OkczMPaLI8+4B+0RhWPnWiGgkyXNJV5nvVedY2eI4lzzKAzmNgWxxPxqw4GwFTvGBn4aFk8CVLbAJ75ojkuH7RU8CYB7HFON4D6EBGaazwR70DwbFIDmdylxpO9Vh9aMNRBdm9m0qySebvhpZuYqPg5kgWbmSlM9Chd6kMqhYAe3rRnJYwwzVeZqE89g6m0WlSTbAs1OgPAYA4qR57ME9DmaJ4dqDRdajvGAdaaCYo8+SBgeDibEg3mD6FE63VVBP0XdNWEaSbzoIJGiiAFLANiWGCkR8TRos8EOfwQAgz+Cew+KDRQGln8EykqCePYq/mHFJPoVTJe3qRN5n50JvKv5htAcz94qE0JmBKwgPZks/rAas6a9gYIhqDDaBqPh8UYPZte0QAo2tNZjAfieRIzADQm9otKpemZkvRbHJbVPPk6yXOYComeaIRMPijA8CUjEEBgqBg8ZqXz77LWm2q/PBJ6TsQE8y5MA2zWVk0R8JB6Da0ASXPdpGuyOZjhSQwcZ2OdalGy49171Mwb8iBX2Iwr8NHsBR3jU15RHgS7eoRvOGmgPepWihNlYLAB5ksFxvpudsx5YdbDy6AyG9Qb5pXrK15HgZ4IMbwBqThtTac45meGwF1WSC878H6ryF1tMa+Gua9EPGtLtCveZwPUTQPNqa8GajhDbBbRby4A9gYpoFINltNGDgLbxzIuDJnsUWZDYeHoUp7oULWcAltDG4VXqd/PYwiCgy/w1dsvKWDyu+ZGGaeo85kNRxhQeWcD2i2utn7RQpMaK97tPHMdLKGCSsJMNsPnUfk9ns117Y4DAhy5koGAbzA12ujgFKpMdjTwd7wXPLMqTFNlABgtk9Uucd1Fw+26YaAMJnhud6ztetU4bpywfA/kVxSpOBBqJm4akZhs2L5e3UyHLveRWECGGebi+MgZqwjRn/wANdG3mF3DDYCdeDAVHjHg0kclzoXBbpPACkBpqvB5xSAPJZCUhN5tEyGZJ44eaCCZmnmT2ItFPMxgRYgNI4zExjQldB9aeBltTQZDFEOCcTvRRt+6mpdSodfkwJB9m9eR6xwW4jWfUvv0TCF/aMM81+m2ILL3zbbFz0k6M+5g0fyZoPzpPh0cylu1XXkyXQ61X0Hg/cdyQJFYDCNHi59YHvXt6icBLchx9ConJktfXgr17hLbHqaXQ2HJLMeUGHWg/MyTGfjSDbcTRhLPw2AzPyL1LxL4D2Pw3D14E56Y92A6eYLjp29SqlIM4tROMfkADRbrHBl64KJZTTdSihGDDPMOv9C55dt7cOZNSkA+wcyQfk2LZ0G8JdBgBBqIHPa+QMFj7/oNDreE6jUaNTd+bwa2eZqB4ZeZw8R/9al1L1UcUDg044x9CkVimsPSjfYb71LqVbfmMDEcYZz7MARbMgynlIOMbPtAwNTaD6jCYf2gbzj4fxrWLFOCMN5IqlyvPnyP7P2LR23SrLnxTcqtSqTNQ/B0c1Yzh7EYPPsj4akVIKdG1dRh4ADzroEC2Ava1AqtKYjU2JBZ+c80HP4H7UDh9C1th0ejXVNl01+7vU5h+zB+MayvJvzHY9Ki+1fPBeuJPo/cOeDNGt++35b3O6Ie236xn8iDzJXuG99hPNiLatSktB/SAjHvXUOFz1sT7GrGdq8hIis/tDQb3jXRYHpJ0A6HU6PqTJlTlGYM5s7GQ+RctC3rnpVkVCo2zBnnHB77yzE6ANRaHL3p9ZrcplicGxdFpXN3bbn81Uoz9XAYPYM4b1mrM4dXrf7RzqVB+5AeByD2Bn8iuIdt3HDqNTpVj8+AQsOczPr86xuFoNNqU6mm6DDmEdVXLTgmc8wxm0fQtBbDNGeM6bUYkyZNP2LLO/M1pWYHFSg1Fq2QsupRo5nhoyI28FzYGFt4K4zPClU6CEmbUXvBaPvNaIDqtNm+rpzE+BNN7xmWehnNdT4ncBKjYAUKXWZUl71oAGc6IzohGzXri5+CFA4i8Lbfo9GprPOvxsAqeGGiHeZmqU8CcRXqVShCDTai9zYMhsBXfD22KrAjm5ErkaNcFUZ8GOYZmALpHEv0Zq/wKhO1Jyo0e54UoAAzeDN4PoDvWw9GbhRRrVgTeJNxsBMqr7J8hTgDxmfJsWnSOKcE3rHtviNUD4mvvMwn8w5jsz+cEuKNSnX/NOh2VQ4cC3aWfOwDANE5P5/OmqlMcO/Jst+CyyHOaxxHt5grXiRxUtyfNhOVK0WYbQeDqxHgzTbBz28+bjQHZ3KZumYAGseeCp7Js+q3nKOVOqUaHT2PbG7/v8iv78jMM2+EqJOOS1KDMM1haDUn6O6Zsd4YLHm/W1u3SLhtXhzSqW0xQ6q89VTPADA1j6lRKzSnTCVFUQHp0kGq5BYPCKeesALUQ7hfrFsy/WTgBoBmzgGG9Y4GK+6ST+99ihW2y/VbyiaDfhMHnmretxqVy/PU0MNfrD51o+C3D9i7YtVnRKrJ9YRWdkf8A212R4Uz97T4NVqwQYrGEjDAzz601RLbCmu6lSfBk/IZqwuSA3Dix2H4MPm38/vEfYZ71WQ7e8J2XczE9mnh0Gb3tjV3AvqrJiSWo7lKYw6883s1BjTPWUeoQWw8XR6/Oo88KAdLjuUNuTG0Awe34ZqXRLeqVEobVYqLGjHmngyfZgowlT0q5KqdLhU5w/u8J7PBR5ltsQKj6yYc/bfHwANgK4ZprHrRqVp4U8/bPdgK1ZClTIZxXHM5AexPswWPcOh0riQcbkbRpVNwjxYewJ2wzWP4hViddVqwqMDEaBT4r2s8GfWmTgVV6RHvG4KlGBn2DO/rBaufbc6guxKlBfptV88fPzrLx5GNtiBzPLsWPR5kmPF/b5eHQHzrVVgINt2/CPU5yXWJOAA8G8w86u5/FoLG4X3nbk6DAgXBcszZHaZ2YGCl+jlwTue4RicRqy3AOlaOEYzPxgPzgu6IYsPJjQaVTTfl0r7wx0A9sMFSWNJYk35H5tzDWjHh9amzLVfrF9XG568meBJ0GQndbwfOslckP7PXG7BblMvHF6JDP+RR0hVVn70bj/wC0a2i8moE9iG07Q64B8vnmy6zvMF021eG+tQZEu4N4St55hvz85rE16251vNA/LchvAf4W9YwtmI0aD608Rswj/jKU89LOHyjGwPOpH2YuDQj1UwZOPKDPYfQngB9mBI0GM8wwM/Ia2tbQMzLVqoYVWVsYZ6MOtZ8I0STHly2G9GOweDKiGzhTWmO8zwNWbz1OjNFSqcebTfyLHAUyttzGgbcYZ1cMDM96uKbdVxwIbVOps4+Xi+OAH1h8gLPwKJUqxVAptKig9IPoA3sFcTKO/R6i1ErLHhMYG8CYHSK3flYtuqBKty4K3RJFXh67xy8JOsfyLlNz1K6rkmyKjc/OSaxK8/X+haa5Kl644g0p9inZxAZDR3/rWluem0q/DkVliqyaDIingcjMNEDSLR8yFcno5P8A8k9PvuwLqk1ho2QOq0yWABJZf78Fyme9BjG7EghJeM2QZPv35713i1bDlhCZ1+IYVL1jvgYHmbwd6w71n05mEE6DOk8wb2jpAe80+ltEOXzIdSojseUebLphmBgtXw6nvz2qrFqQMz9Q2DBqoBs61YVKqt0eswpUulc5FhBhonvezUu8KbBvB2Rc1HqMl6Xo74jLJ5nh5/IrjmMKm8IdOoNUDlaGYRJobIjT2eD594LGz2eWBp+XBAzinny7qsLYgPz6poMGECWAZgBnmrCtwKizCl1WqtsmDJ4cu8Gef1raLLVM+6mKlzzbdsRoBysOWz62Q+Q1trJsOv1tqFX7KowBLpwb3pB+DJP8Fc+r0+DWI4Soko9UP6I6eZsqTaVYuqGbVHodZkw4gSQmvHnsZw35rphi7Re0y+KbUpdpTpdBZkTYGb0GCexkD7PrVJQZnDm6rfdtiJbkmg1WFGz9YhMA5Jv+QDVPxIr1lshnb91etaxKezn7Nh/OB9ixUmGweEtt/PmgzMPItu1j0pR2HWYe99umg134TMz/AMCdjWfXDkAw2/DZDyG9vP6POq1mHOOKZxWM2mOs/ItBe1S9azaZOgsPGcWNgZh2LkbIVyM01mQ0xFbCMbftgM+s0dH1NlKiNsgYbAzPDW+RZ4z1pRv959aejTH41WCUYawAGz615cLaIJ7FvXNHlgx97A8DDPoUeZJbolytesnM8zzk/n6FTxmeckHKc6z6zWo9VUCqiDEpwOYfMAOQ8e81jcDRz4bk93QnRNGO/wDseB7MFTvH6qoLtuPhhNB7XZDzgpZg/wAm7UtfPkgBlk8PJ3/xqih0SpPO6bEQzlv9BmGBmCxuG22qnvWd9n49wUrnOd384y0exk/OCPh1P4lUGsyLmo1Zk0ePKwlPBI3g9F39nfh/nVZDo8plqRQ52DMjvZM0gptVgOvVWmnJkwtHlTanBvDz7EhFux+j3xFYo9NuunX5OZetKbm8zWQZweCaZ7MPr/yL0VdXozTuJ3Dm36Bc9cpVKuAHjqEmsmznJksYHgZ/kNfnuEzRo0uhxZTxtSnge0j6AwXa6J6XvFGNwoPg76tCe6cM4XrAzM3tA+sMF17RhScHOCdwcWr0qdK9ccnRKQ9oSamEbMHg39H14L0xcPD30b7etU7AlsUeBUJsYNE6mZnK2d4Gee/6F5Sti/65RLZl2PSn5kb1jJA8Gjwksn3gfyLahbb/ABmqjtSl1ifDrtIgZvTqmf3UwDo+foNTtGFLXuHt6VK4Ao0E3qlb7DxshUD8FkMw+ddgonBOpcK6RSr/AG77r33KAdUk8vTQeCM+Hn84LL1upXadLasO6q4cwKRhV6a8YAcKSABmG8+tarh76RQcV7Qujhzd06TR3a3A5WA9I2M6556zAPdnZgCvhsw86cUaqxcNSm3HWWOcqFwzOdh1AwwzAAAMADsWKN7kIUR82wOO+eGYGuvekJaV60er2/RrufeOPS6UARpDNNeCLr+QD7/nNc/ppsBTmvAZed7M02QvbAh8PK9RpdKud+MEinZ+rZB7Mw37z+fYChWrWAsO8oUu7qVVQhaJmDTRhmGfQYKbRLkpulytZoEMzAAAJYM7/wA4dBrcWxSm62bV8P8Aq16lRWeSNl7eeAfJ/kXN3LeuPRyqtH9Irhb/AOjTxJlnMdoMb1nomZsnoZ/dQ2deGYH+QF5Eufhvf8C+r9sq2WJMmk0EH5XNzmTZB6FF8hmHzrfcAeLX8iHH2RePEoJL0e5aadIgVaOGYRujA9Hv6MF76oOhxv4S8pf9OONUDelUipcv4OtgeB4fWGH602h+U9HrE46RIY2bMP2gMDP6PkX6V8AbVYmcLLUuqm0b1b60o7GtEw6z7z/P1rQXzwHsS56PRLLo9pQ4cdusRZRyADeywHWGfz7F1WBR6VQaXHo9Gi8tChBgyzn0ArizDmt1cPZdbadlgxg6DJmfzrzvYwOTLcjsAwZusdeAdG9e2A36rf7k/wDAvK/o/W3Ords1CsQZwRnYs82N4dYGvq+n7LHm4VTR6kxSq80452ArOsT2JMo5bDmw10iscPWKq1pvsMg6HsXQDoNcir1HqtvTeRqrGifZ84L6XDfjkcf+NdU28G4zWgYHsVxAvPmZANtrm+fwUuNMbZXTfpwx7rdwgVWLMwwNWoLkVt1sGZGo4+ul02txZIhm4viez6dxbs4eZanGzBRzZwUtmZEPocRPaZ+zXHh0q8wWfuc3GYYG2G/PrWmNU9bh8/CNhbR/yMZGn6x9auIcwADBxVUmiOBPal7w0AwPZ1/OlJNtmEcoD9mu/wAR2MGqCeAezWdrdYc1TMzVJ9oXAHPNVsmeczPUXTw+n/uyvmWoVvv1FNjV5szwWMPUSZkuB7NdnTDm7m2mQ2JnQszVaI+yWo22nodwmAYOOZq1ZqrckNTYf1r1G2HeBxk9NxNLYT6bBmBqaG/zqhk0cwPYtYtkrwef9mpCUMMJQa+wFImMnJmeBhhgr/YI/hma1FvMuSWgisOYGngtu3w9m+8Z/OptNpsqBKzYcwAOgFzc1ujwyleprdNmG2xsDyKFG5uS7oMBmZrd1u226w60+D+i6HXszzVxYFjRWaprzg1vIsfpnxG7PHrI/Dfh6AOtVyqxDOR3tGeYLtR0piqwtN/rNSI0CCA6mCkM6YdHQvicvuXd7p9TxDkt7cJaxAinUaPKB7eAcuAec/OsfWLAuqgzTYnQQMA96yeYL01m2cfTSBlg9ht5q+H38Mul5lgBBhyPvasj9TveIrbjHw9qtEqP2jpUH+bJWGtonmbJ/OuY+spYdZr6XDyd7juOtsJMY8c6VUgZPyGGaaeuSuQ44ajAGHnWP9azs9j5gjOpTnh0zfMwXThHc0QVt+ZI09gZq+okmcbvK4Z4d65+DxgWouhcPaw29P03wMPnXNzQvhvsaYAfAPEbTPPtsn474B9ZrXcmw92LOV6xqHUp5yjY39/kNcFu+CZuelMjp88Bn8iN65IL3eqk+HVDjRzlQeZhu/uT2KlgW3Xzq4QX284/eeax2ttWXm5/hgolVttzQ12Dz84YK7olB5PNwA61oGaC28Hjt7PrWPYW5VGecB0MFo4cl84+nmtkdmUbr5EANVVVtKqs4P0qJrNeTNXfNsBSnsGlLe3iqSGE6MWm+3ompeseO9c7ZKCM2aF5kA9mnWQU1mH+I2gixnojzGgftUD1EcwNxSHobcZ3XbTQVVzXMHD2IM5JBwDNtRzW4jUqnTCzf61l7hZiMumERBVHJbTPOKO8msPigM5m5AcxRJOvqpeJh7NEDenufiKI9McTR6ijnnkqEjPNLBw0AbFI5lsFIj4OAns/ggOTmmutBI1s1Ii/86ZgMt5+ItDGgMaWppqhXMngStox5tKIcPRd9mpcYG8FIlAjMMxSDYpAaahaFyB/h/8A6UQQ8PdqcCmxmWM/EbVbFIYOfhppXcyM3lsUQ4f7tSKnRc/rQ5uMmrA2cFEkh3qwZnkq+eBqwjM5jn2IJkbCPqIhnTeMEucP+pBMMM1F1m/6lQtfWWHvEB1IDE/EVUZqOZ4EgsAezdV1TXsFmmTUuNMwNBsOfwQc/n7xURzNnWgCYpGg5lwxSB7AN6pwn7EB1L8RxBfBM/eIucWaOqtownoNBnmgNV7M9v2bhqQEnNAjPYqyT/wZkrB49irXgc61QZCGgehmB+GCeDXA1LZe/EUoVgAamsngnnmW+tMqhYRpjmYZubFZhMbMPaLNZ4EpQPKRbTJm3TVScn501JeNRM/ggnaymx+hVQKwjPYIJRhsUUzUg3tigmgaM96aM96TxpkzVLGy9uwUvW2KsDrUtk9itAutNPbCUsDUKR7VIswjGmjRoFe0gUv1bmHibEDIN6oGp2s3/Um0GY0BsD1Fds01iTCdim31garM/wANSI0l/PT1E8LeTZMNy3r5PBvAGJmAAvSsb7zAjy/xwzXAuM0B+iX0bh9+8F2XhvVXKxakR9/sDBbWyXCAzBGYb01h8VioRmCHIEBoMPirixHkh1/8Sp5LmBq4e6FAeADLUwX0vWctvjLylg8qzMAPTUgHj/DNXcIWGfwTRg4fQijA48rGNGw7N65v8bVQvRjD3ZqEZ79NvrXS6bTdbBt/oNXTNpUf8BPsT0sFbFtuSS5t9taj1Voq1kxmIHhsN4AoviGS5r9m7bYSIbLeknZNEOe0bbaOMGAAriGeALj22wzMaguQ+xSwZwWgeZzLUUd6G37tNmFUYYCojwOK1ejbFWyQcDYi1fzOB71IjVIM1XzGXMTcVYEkwd60S3bMkFNCSBrGM1Xb7RPetXPdmg2pgwmjBUMasH7xxTvWvzopYZGpQPOKnCZ+8UgJPzqUrDI00YZkmjk/OlrIGXsAFVUwzeaNsFYTN4GqzeB+Gs7HkrjrcM46lLof2fM8/envXD6PeE61ZBtxXACQHnDNfoBdXD22LwjmFZinn+MB4GC8WcY+HVKol0HBt+XsDr1l6pgrhvO5qxIB+XVWdnRhGVFJn1WY/rvytZ0+9WTNSYo7psOMAf1goU+ZEmSNfUAPoBWtwdnTPr6FcVh6nRorUuDF1gwwNUUb2WolMkuY6ahavmHm6bigvKzjRm5PiP54I3qbrNHoMYB51UAKJaVSrzRuU1vPQ61pbABilTJEupOYYeAAYdayjMydDaCmxJZxuaewMw8i9J3hRLftjhZTAp3spTO94+t41Y5VXqDRplchNzpzLzVR9z3h9a7HDtX7GU2FZRxQClVQ8zM+sF55hw5z0r7RtgcmPFMDPAOxdjufjxbF20uExTYMyNVYvsTeDYBqxd0TgPblE4525FqVZZ9RG9rBzewAMPnXRfTGu2nXCFNsq2KrAkuxd72EkD0QwXmy7b2uC9uXbu27g8HDBloNgLK1Km0c5huQZ0k2vPmg0vDqyZcy8GgB8JMeL7Yw7816Q4acQqNaTV18OYjEmq1Otvfc4jIbHvrXleBMlUFp1yh1KTGdfDQM889iVt3Jc9pVT15Q6wbNQwMNY965x3Cq8NOKHBmhnUn5wHQqoZnJjwTwOMZ9gKEzZN48N7Iqd21GlctHqIYGEiZm9gf5FlJnGO9b2p0eh3VVTqUvmQMJBnh/B0LuvpGvzjs2iRHH/Cfh7w/QjFd+gNw6tWsUuXxDrlNZmSIsnCM8fZ869J8QreoF+V5rmn3meSDCNIj7DPPrz/QuWehzMptH4N8rOpTzOvJM2TANhh512PnLcPxMzD6wVXC4tcVK2KNW7VO2ak5DnxzAAM3etV9sc9aVIaobbAPR4uxnA+gFQz63Q4bRynJesAfg78FFjcSKMH9pP/sVHiEbTaxAYue9Y9Vqrbz0en72cAzDNK7bGpt2yOet+CdNqvZIAMA/OpdBrcSe07UgqUY4755559C5T6QPG+sUe33YNsAcZo9hy/fPB8nkXX4XFvPHHusUqTdsK3INpPM1tjMJktk9jxh5F5xvONVaDUuVqsF6NIM+g/J512O0rn9cVSXXL1rkamyIW9k5G8z+dYS6qkxedbKsnOBk4r2wz71N8LJio0+XMdBiU4ZtH2ZqwehsMtajC0VHoNcv+qOuQWA5Rs8OYAP8ipJMOdTavIo86KbJsHhvXHfC3NAD8aKf85e362c9hp6TPY5D7q57fZo96QU196YDbejh35pnk2zqh6DnhMH+tY4emqJM9W1mFKfbzjsPZvBn2LvfDqfUZnEtqJbEWMDVUDAAzwCSC4IdNlVWstUqC3m8+eC9AcNPV1H4oWO43OjaMWTgZ57OhdcCw4kej3Lqt78jY9K1pdOweqTwHsAz8i5ZxRptcol6/ZWquAEJjB57BnrXqC3oH28vziG/blckwKnRJOvGMHsAM9/WuG8VAO8DaqtVOZDuV8+V5cA2PB5/kBZsXJJNKYqs0KVSnwBo/fGa6befPfZyn2jE3x2IwGf5Fj+JfDd/hdTqYEu7ecqErPWiR+hn/Oq2wLh5aRWGKrOeNoIwaOf581jY1FEjMVuFCseCwYQn3gB4z7wWcuSiBbdeqFGiPvPBnmGa67wZpUS5LNrtSiSwhu29m94vQ99a5Vc9SYmVuXUuRZjSzePMA6AXLbaLa26rez4UUemxK5GORszid7xqw4es2rWLaNu54J+taQGDIH1qt4S2xXLzuPXpsWNpQs9adIPYymarTbgpU+oXAZgZ85oAYH1qRlb8encS7vNih0p4JAM+xd611rg/fnGKz+EtQ9R8OZ8+nhn9+zAAZD6O9SIzNOt6ygqtOo0P19KMw5g2cz3/ADrbVvhpcFw8KqfYluXpTYEeKyco8HvGeM+sF3cPMi3nG3qPKr0ymUqdXIx+u5nNPR+8POrO+eG/IXbULn9VSXrcinhrRz2Zqv8AWtv29CiPwQeCvQj8aXhhgefQCmvRuKl5x6PBpucyFVDMIccz3gexbbGVqV81W4Z+g+48FPDsA+tOw3qzXqjLYBuTMMGdgdeAKPUuHtx29dH2VnNshNzAAwPMDM10CxrYrFvXQHNOBGdMDzM/fB5Fx21ZeBXqUdrzmNfRkU7Yy1hnsVedS5O0gpz7e+ozOaP6OxWd/wBEiUSrxKxTQ/m+qGb4fWsfPqs6sVSI3OP7vrb1cJS40N96QHNeC151f2rTYLN6yG5UQJkeKyb2BqqrE8IFGgsP9ZyTw+hHArD7NRl1FjoOGYKBcUSfFmSpD8SmnAAzz5vPeH0KRW2XJ4GbBm86/wCc+tYSHXnzDlGNma2FEOqnNhPxIjxx6cecx5kMwBUL2w4bEyfXn3w2U6HgyHzqdTYdDrFErtOg9HLa8n+/BV9KqTEZipv0pwMAeN97DrWUpt1SwM6lB5mGb5ger9Cy6V7Xs9n1JDCDTXNGXCZzDvwzVZaVVfMZESUefW8B/OjOq0p6VNrk4/vD4YfQpz1E5ZhqoxZxyTMM3s+wEwpBCNENo37xnMyYhsnnh2H2LNUq4alRGpDFDl8g1KwzA957FdwD9dwqgxK3h0Mn86z52rVdU9RvDAwwnYbD+RXAgyXn6aYTmPagri57enased52QP8AWpFSZg1V0HDc5CRh42r0GfnV9c9t+reHce44lzSZMiLswzzAw8mHYtotzucvUGcy0cvT2Kwt62/XfrBtiucm7FZPWDDrU6pQLgqVGauCdUY0ZrDBmCB78POigRn3rUmsMaPhvdZngf610xYojjP8rg+YPHCDsDDYpEaMw81Hgty+WkH7549h/ImjebhkbBhmYIoxsSXdecGYJsa47MlUeREiSqrGk67IPTAiPZ8sHzq4n8OrgP1fTqGxJh1A9kk/6LJDDrzVDSo1Ned0KHRgN0wzMAPDMPIngvyuYabfM8uHsY/OHgyvRnAoNZ13WDiYOsSeVMPnV29YFx0fVlS4gfdezP230Ks9a1wOX9RwXg5U8z2Z7FozvCq3bVDcnVE9I9+iGwANYrM02lZypFN0I0OIxhgHnzVxGgNg16tbiQ4xxXswzD2wedZSpSalVfYGb0h97Q3n1ro1pcNJfr47VrntQ0DOQZ5gyB9GayESq17CPNgxYMCNmeZmfeB9ihUq6vVXOhLBmS0eAMyw3gyAK948Xa/GrNT4VwacEalUgAYjHHDfn51q5lh2zc/AJ65joZwKhTobD0B0D3vB+Ofn+jrWnUMlDtWgBcAViXUmfH8czkHgYAfRgtBXrAlViiVC5oNZhvR6czsBreb2C5zJmRa8xL5GLMk+rmQBl4/IrGg2xUo0LlX65PhnNA5TIMn4LwLkUwIRjjP6krBk/IZqbAmU2G1VZz8GScg2dCBLZewCN85+dS7qgfaG46ZTbZY5ya+z42/vVhM4aXNSqRLqUo4Bx4QZvHHk5gALroU7IXBTZrVVlBJZkAGcaWG8zP51d1jidOqtLhUCo1Iwjg9rGbQb5MrPPAzBZRmHnRpFcjvvPBFeBl4D6961desCpWrFo9VrkX7lW2QlRj6DD6wXo2FYuqDecg/5Q+ZoJxWQisxI57MAWJpVVwfj0246jVfsZzO86fh1h0Hv6+s/1qwjQLgv+jXBfDH3lqgsg/UjDrDM8AVm9atStvgtb9z1g4dSj3Cb/IQc8Dhhn7b5+j+BGDob3Hu1arw5rHDlh+thSpoerwjyzOT4H9qAzPYfyKv4A8B6PxasO86zOr8mHULXZ16aETeZ4AZ7w/IuU8N6b9p7jj2jEpRzKhWMIsMwz8E+88A6/wDYXqif6MHE3gI0N1cMq/JrATQ9UVuO0GZ8k9sM8Pzq1OG8JeHtKvDhlXeMVzPzHqPbUwIr1OiBg88BgG/6N6rLSDkOdrFKN4I78zlYbMs9kkF74t70e7Vs+/JDlnUplm17voMUJkfRwAH4WGB4ed7vXP6rwWsC0rGNi58JMSn1uo1RmmSJIMvGBnhoBhvP6FhcLcPrfDS9To1tVwJVKqVPrZnKZPmQDkD6N+Zr0B6M3pURLYdrFq8Ta5Dkvc/92qzOtg9s6MAA8z/Qudekbb0SsQqVTrVs6g0T1QyxUJjNMZ++xgPZvBcZok+VZ7rTlKqoViIEnmngNkA0cOhcw/R6T6WnD07yp9j2y5GuGoVcwCGcGezoh5+ZM/Y/xrsUasc5H1+bjPf/AE72YfrX5e0eq07j9eH2RuPk+YOGcqBWaZSghvAfWesHk6F7Y4LXO3Tbap9h8jnHokblWajBDOE9h/gNdMDtvrhhmPIfNzoZNcM9FeThZFdb/wDyl/kW+rEzOh1hzU/6Nlf/AKg1zH0aXtGwKg5+PJA/4F2ReC/1u4GYYLj/ABsgMYU+pMbMPBw866czJ1kFYprE+nOxH2wMD7DBacPs4c3NwvMQdOol3rqR2HR2YpwWGMADoWCuSgv0SYG/NqUGYHgv0vrez2Plc3Cjw3sOhxTWak/DMNNxUoG4CLWM11Y/LHf4dAtu5NaaHNGuhxqxBMfbrgkaY/GPUbcV3Sq8+Duo4/0L5vN6bv8AW9l12TUmADPUTQTAeaWHh3O5JdBg94GpsasYSDYcPvXBfrXDs7ttHJ08NRc8mHyECoRdQzB/fv7FtpJvyYXgHvNc/uQ3IebEtze+Gxbep+/+qPY/YoTmbU0FSAPDccXQOFHCiLxIgS6jWalJhx2HtAAj9Z/nXaLb4IcNbe/6D58z63Zz2sa7+b3+HgvDjjheYutAALsvFHga/AkeuLHpzz0Q89aIABgzgC5Eya24fZjkc3TaJn8FIB5WZ0R+Y00/FiGeYdB7MFTvA5GkGw51gr/Pg613GmMHHBsHERyVRMvYKPZ9Ycr1GkTpTeBxZJxfrwQ61w9DbMfD61EADZd3qQZtgGu50BvzTzLzEmPp80z0Z9aIwujqUGZTY7D7ABIY6JAdaT0+cDuu4/m19Czuspsae48wbD7mxRhe2tgG3PinLYl+L0YAhZrFwUHDw9YA7zWPZnuRndQFKOty5LWm22f61j87bxzPQdEvCnVWAEpg/kMPIas2ZmsvPlt3XLoJ+JvBdgoNbiVKAMph/rXx/c9Z9Lh9lqwkuGfiOKwjPNh7xZxmTvU0J+AL5WG22reNufFNh/oNefOKPCjk5sJuzqU88b5mDwB+TD/Ou0RqrsX155t481t63s3wWc3C8iT4c6lTXabUohsyGOsDTIPb16guGzLcuSpBWaxSgmSADDf0YKqOw7VgCfKUCMH0Avr/AMwcHxuGUeG49NDw13CzLbo4UlrNjeoVBtJsKkGuC3AQGIwabDeC5Ob3Oxfretg7GjQWfDbBHyzb2eDaaADA0QL5vY78IEyA492J+m0SIHj6G9SzSB5emEgGcFLBV4TFICSgsOtAYIAeDFGZtmgr5kOI97QFVSYDAHsVhMNzVVJMkuA6glhgHQpTMlvFZ/mHEPPn+J/+hQLp6SBn4jihSXomHht7/Oqw5jh96a1nD715sTfWUoOhxU8wzeLUNTkybLC0FObLiinsJXTzIezVO8zgeo4q2BwAz1Ey8exC9JAA01Bekr1D691pkwQ8w2vmfwQIwQmiM1HM9iBrPAk6ye9R8/gkDzaC7hvAGCtWargOnmsqEzAk6E9BoHqln3pMz8D9oqLnM/eJ0JPzqFtKzPzD2isGZKyrMz/m1FaszAwBBesm2pwGqFmYAKWzJ95qILsEsPiq8KltRBMzNQFJjfu1VSYb+fQrgzySVbFUEZ9kFYAyEmLpuJPBmKZOY5GHTBXsZKvQMJXhtqkPYtNVXnDI3DWcmHmatCPrIDebx1NRRTNR8jRCbrJcz86rzM0zrOf1ILjnMx09RGExU+bnnR5uAguDqW1NHM/eKpyNGDx5KxZ8y4adCSYKsB5PA84guGZisIczf7RZzWc/qTrMxwFA1fM5pHJ92qRmfmHtFICT86lawz+CEzxUTmfnTMmY3gqBnVWw9o4hCq5rPvSczNM85h0OINMc/wB4nWakB+0WP5w/OjCS/wCdENkcls00bzazXrKR+Ii9auf1JgabmdylMzFkvWh/iKQzUs+9FtXzyjvPOGCqQmb+tSAk7kQM3nEkjMEs/ggPD4ogNDn8EWHxRZ4DzJEYZgaj4YJ4DNBEeZTWGCmmCDlsyVsjQAeCd3qQyyAdiTwBh4aKRwewR62aZPrQh1q0uP8ApFU195+n1UPZMBoH/ArP0e6lrU12lTn/ABcM/wDGr3jNA9ZWVIwbzNgwNcs4LVvkLjaY1MOaAwW0Kd+eNMqW8zgemajgCxSaw+KLD4qXh8UjDYgr3ox4qEcbMFoNHNAcBs108PMwuFJDo7Zu+IrD1VFDsUsIzYezSw/eK9mEcI3/ADtqWyygUhnYSjZhdwzbAANWQT2/xFnea+CkA8uZolT5JvEmgNwEexAZgoWMJJ54KyhvGCgM9afB5QL4JO1I3s1Sc5gXWnQnt+8RS2AM/aJmezFZY3qOFSAPeKFVaq2cc280FJUpjAZg24s68ZmpskHEDMM3laVfkaezcBXQUf8AET3qSLh7NNinCpP/AIilBPf86M6IAJo4bgKBcRp+fvFYMzMx9os+yGCsI/WmhexvOpagw+hTgWQZNQjA8/ZqzNlBh8UGSvaHer1GP7HctzHfrLxJxRgVWMGpcdc1pveDXYvdd1c8dLNiKZhn5DXn/idw9s6BYEiuXG3nUPPIe3op40ns8zIzBSAtvAc6q5yzXnW94UW36+myJcShxnuV61LqrLFbuNqlVmmshCA83sPIi3j+NG3gwmqlDbjO6euB/Qa9JXJw3pvEKzaJ9nKjTY0iKyuO1u26VZnN0q4GweqAexdaPYfnXR0snPAkgypATGzBWcOlczSDluWrMmNd8hpa2lW9EuG3uRse3DmOh1yHg6PrToU55Ao9Vrc9p+nQTejxTB953sAF2bjlfME6XR7OguYQmI2utrw9tV+icKKtarGi9UDZzwA15kr1NrMaqOtyoknZ8idKU37Q1KNAOlRKk8EQ/dAoDJmCiMh+IpfYoa7FhrGCsDBiH4bh4I7So79Yuan05gwzN7vWwvyw7me4g/ZlujGyGeAGDKtjtj2QnPTGvV0F6Zn0AALcSeGlcC2vtAER4He+I9sMF6FPgJRrbsumTrZnSWXWAB94zAM3j+tah7gnZ1StwK5UTrcybKADePmcwP8AInSfS8m2lZMqZTQudypRmXYT2+IfWvRvGC4aVdtn0JilMPScIwG88HQz+dbClcB7KoNGuBxigMyQlRjMDN7PetLwZsawK3YYMVW340l3PB4MFHSje3RuEsmmzOG1EYpTjJgDPQB54f74LVer5f4a5fGsbhXb1U5FiqvQHT6I70/RVheEayrAt+Xc9wcTa9SoUUMz+85mf0bEptDfcnv6A/OGap63ACYfK8qB/QGC8z8GePfE3ivxDm2xb99nGokXokToYGeHZns+tekGWeKkPNg6NbFV/e8zo5/kUlwqbh4V2zXooQcJMOQf9LaewwXkLiLQb84i8SHbHptcM6VS/AjS9Hs+deta9e3Ea3mnZdc4V006fFDOS7EqW8AXlKNxau7+e5XDLhtUvV8548Jx+Nyx/PgtYtjDFUS24NEvL7HPmFbrrBmEmRUT8FkPOCz/AB7tijUqoteqqrGkyJXto7PQyueViM56+kPzpck5GsesGfetXaUbh5GlOv1xiYYaOwwe6Fd29FTeJ1cti3I9DocVmHIY/pwdapY1VnT33pdSc5yQ+eZvSN5ortCDDqRsRFSwJOEoG1jbWIXr0mVzAcp0I3pOj7zAzBOxowPMSH3H9EGFTw2TqU93U34dH0L57Z1LhLRKG876/luSebB7QBnW2Grvipb0Gg1mRKpVKwzAHngaDrNYQAYpUOPrtnGA895rqcC/6lAozVGqrDJtYYZvbzeBZNV96JFebpV9TYlVYMKfcrOiyb2wDMOxaj0qOKlDpVUj0e0qAdSrtLD9oDeEMNmz51jLPo4GcSDTa4dKp8qTnGlme9nz4LofELgtTY1pVuLbFRN52VAOVPnSDz5kw37FtFuS4eHKxJl16U7UZ0t6TIfPMzNaXh1YFSuG3K3XIMsGTpfYfervh1wcfuG1LgrNwc5Gdp3gwwDZma6VwHs+pW9Zdy0O9IjMCO/nnIN7eGzBd53IvoqcS7YtKjXLVb0fjaUow8LreM/oWM4l3UfEi8pd1RbZ5OO+ABG0Q6wDz+dY8KlBZq8ijaAM0oDwMw63g+tdTmVunSY9HtynVWNSoQYfeNHeH51x8pEOfw/WsBh1tiXUoGv16Lxs5qbasNgK5Cgv85JZPPoNdZrdH9ayIkGsV+NJpTIYBIZP9pVTwfs/1lxkj0qW+cOODL5syA9zs2HmsfwvaCYXPXr1p9iRKiEbmv2PW6MPnXobg/Sp0yqVjhzfDEZmbFjHgcTr+sDXCrGth+lceKq3WX5MkLaZffOQYZ4GAbDNdAhyYgWvC4hv1V6NVajXuSmSADrA10xwjKXhwft+57ortSrMvlrcoMblYciPsCfKVhwHuqhwLcp9KCj+tarSJh4fedHlgPv+ddL432xBn3fY/CuhxdalNnzrzOe8/rXMbt4LOWldsu3AnHTYXM69HlgzsMPIZ9hrs6xxe9odwfynXK+wwb2Ex+VgyGwAWztu9otHCO5VaaEmO+GYSMN4KRx+hhZPEGaxbjnJx6pTWDk9+Z96ylyX5Sq9RKVQ4tNOHCpYYZmftjXBzcK4toOJ1BwalVmC+EmjuM80cQ2djJ+cFw2mnzNUj5+ybPM1so1/1g6dU7cfl9GyMfZgH/2ayXISgdAADrUQJt5g/Pnx323wOP0MgHYpFVDR0m9TDZgq+MzycoAlbFHZAHo7py3z5gOvvW4jgbEOVgB5q2k3PKehO0qJ4Md/DNlrZmiZ+z9vW5HueJcfM1vPAI+Gxn6wVKZvnKOc+eZmZnmqwCnhsOWwZs+fA1u6IbB8G3fWTn/SWbP0ZrDnMzinBcb2H1n3q1599m2WqUe+PsewTAGToBD0G39Y+81uIBsBw+iU2pVL+cKo8eiGG/D61z+GGfL/ADngpHjhMiStf9hPNkPIuUdwtuNYd7cpTX4kam1hkOVeBo8NbBZefXnIYQrVnP4R2zzMz68+wFh850mrBLg7DN7PPyILkPWrPN9Hf1msulcW1FeCjnFkOToJyWuszAOhc85YzwYYz0s8+tKZJfzkBFN7B/rTUZ5gGtBzPPs3rp4eEu1hy1Ve/wCHUcgnMAOzsAEEl7b4EQMPI6o71YnQ4cilMP4c11mHkUc5772Gvv2LpiEE9pm7qSuv5DQzHmwMG2OjBOw3m2c322A1s80pOD2bhh+RetsLumyajAox3PTnMOVe0HvkNWFHZiBS5tVqT7MOPmGGZ7zM+xZyZM5OjO0aJLN4Kj472fnVhUgwhx2DPMzDMzRGFnP55l1oKbKeZjvsnhgs9QY05mbz2hnHDvWjt6pQYzQP6jxugBshnvAAQ1XAIZymzA/oDBciEiwHn6VdFPqQQecjsPG+cfDMzBeiOFdz8OTol10PiVdsm0qrW5LBxqm7G3vRQDo+gDXL6rwif4Y8PKVxNuO55ITX5IfzdEwzADDP+DvXJJN21l6QcqI+ZtH3yD1l0DqF+UGUFL5S2KrDqtPpxnKOpgeByTNUUO8L4h2Cdqy6q96q0c2Y59AfQq+qz2K2w03zwQABkOZMNmsagsw7gZ8RiDUpMIDAzkaObPWoHSOD9Eiyb0Ch+vIzzUqmvmbTQGefyfWnnqrRqDUZr9VtyqyaEx4MDl5+Bxj+fYu8WZdt6W9ApVxweHMypU+bAOUYBAjAcY8PxuwNi8b3hckq5LmqtzTnNE6pMfe0Q6AXmGR1msRHq5Va4xUZkCWH3qG69vPP51NmTACjPa9R1udDxthhmeaz9KkxGXznS9bZvB4AzwNdbh8Y7DrdqQqVeOscgGQCZzcDNkzA+vZgqa7cyOpRWaNHnUpgwPfrZ+xeXTeKNz8VKDYNucPOJTkOpNTWWKvGkOgHNMh2M5/79axVjQIN2yKnQ36kzRI9UB+VDZANn0fnVPPZuCvNQp0s5MlqKzyvMPPZgAB2B8gLxk9JeijcNs1Wzbz4HsBGgXBf9HfYOrOvHgZgB6IaOHzn3rn/ABUs++7YdsrhJc9VhyaZCh4RpcQ8wMM8z2bDDDMM81nbGrDdq6x8jGkumHgmYb2T84LoRvP3nVrXvy470Cm1AJMqVqywA2Tf2Bhh2BsBevHUPQw4P2rUghcTTlQAuCl1g2YABJBk3ooAYHmGeZnma9lvG4HzrxZwQuSx+D9crdx3PRqb9s6vM/5POtPazLLBnvPDsXtu26k5eFr0y45dHOmy5oZvRzDDA1utTz6xOhkEFiXDjTZoGEYJB+2P6O9ecuNPrGiXlTH5z8y6uJdXZwojTMblosP9+B/xr0RMsNt7jT9tKq3rR4VthFgZ7wZlZ78P1qh4wWxBh2RI4jTpWjWLNjPyqbLM/fvYBvDvQeSg4Sv1W36q5cd+HA4oA8Zz2Zz3gmx2B8+ezA/nXHKlaVH9aU+lRajUox+rQOT92M8JoZ5roHFG7XOM19NVWhyo1HkAzFosyPLe3nvwzA/JvV9clE4jcJaNHs64HLbjVBhk5UOcEwJjxgZ7ADyHs8ix5kczGMhY/IWoZzq2zcZxs3pEdkzAw8n/ANguwcBOP1/2HfR0C/IJvWlW6kGDzUPRBmU9sZ7NgbFxK6rnuM4FJg3pWHoBvsm/APBkzZA+vf1n+tWtjVu3LnqTth1y7pPq+ovQjjVCQ9vPlT6P41x+Crenbz9If7MT67YdVptSCuvm+cY5Z6NPBgw6APvwD/Gt9wBPk+Hjrf74P/1ILhPFFmDxCkUwLto0yM7ZTJsvRM97zB4YSgPv6A/Wu7cCoESBw0BuCxota3Rmu/hW6KzUv+MlahPbMPENZdl5hWwQH3gzDoV4DM828jwVFPhhMa0H29i0EmA+AZqveZ+Q108PJ1sbjbk9eo/ITDc1Nh9Cr4DPOTWoLftXzwBdrk2TBuGL0b/OpVt8LqHR8H9AzkeczX2I/iTgv1nH3rPrnO8jBgvSXfOyGxWD3DG+I0IJ3qoD/dA9v/QvR0ClRNmxWvqeIY+G2ub+YNPHpvKjNNrNHdB+q056MHzqQdYiPO5tuLud7cPX7hY0OaZjfOe9ckurhLUbehc3Tn+f84B1rbh9zh5FXw4Wdq1IJgaGfQtLMsOgVt2PLqrGtyueAH0b/OuX2x61ojrVZqtKkswvPguxhWGDjtOB3hmuP3P/APG3Eu7PCDbzGg22yGZ5ngyAZn/vgthDqrEx3AFy/nN2otLQZmGD6+b+904dIZkgA6b7nhLj/FGyaAzNpM6gMRoGZmDwMs9ea3frhh73iobq0JkUM288DzBXw2YZKBQYgHqYKz9Q0oyAzpoGfnwSZNO8z+I4tO+zrZTiEESgxaf6moYG7Ke8Z3DYywG814VqV/1w49YqNNqQMx36q/FAImwMDPrXt2scbLVpV2hZz/3wzDPYGYLwpP8AVt7U2oUaj0aNAkRZMqpxgj7DPB/2J/QB7PoTx7lvOt266rw9Q8PnW3H86ryAfIAGYbN687wL25NrXflzzmxf7OezBbisUq6qxRjl1I/vEpnCSz8jHQGHnXOarPpVKmcrSmAjYAAPYd5rt8e7bh+Z6GpXFe0To0TTckngGB7MN62VvVJivUtmpU3e0+Ga8n1Kj1WYUeJRnMzOmhKCOHW8Zngup8GbznWBHm2jcFO5N3RN43Te2Au3h9zbG/WdreZfAc8Eyy84BqqoN4MXDg2xsDDMwNXGjgu+HOMDyWgt64XKPKaA3PCz3rOM9akMvaMgD0+g1HNww8i3cKPcLExoJR7PkWghzAkhsXFId2xY0oGIlN8LPrM966HAmAA6jji/Pez6dxb6XDzNqBqWGosvDuGDng2/mavY0zWazXBcO/8Aetg3oDZwUdmTgST0nMFIhG9g+poTGzVJJM9XUSZk/vEUvtZv+pBrACr+c2e0VZMrGia8wL56e2ohzwzWMn3ObLu9xNfa0A9u28AeddPjhtltuAngnWak351kmaw28IG24pATMFzXw/he2wCe3+IpDM9jzrGBVQ/EUhmqt/iJhTXSXMwWWqrzbLpqT64z94qmqva2awDRzEs81VZ/BSGZOC1SsHtgqDrYJHJ1lWyTcz9ogteeQc8qU5LiZOT86sXD1S/4iVbJn5qC9J/eKvkvIDkyTz1FEOZ+8Ud541FN5WhN5ncj5n51Wazn9SMHkE/Wc/qR5/BRQNSGUCPoTJmpb2moTyANZIJKjmeSSCXzXwToSVXgaMEFwzJ/eKwjTFQsqWybiDRhM/eK4hvNmCx4POKwhzDAg8RFtL0Fm4ngebzVSFS86MHtYvDUC4B5SAebUFkDMdikcs5jvUCzDTNpQqky2bXXgkyZgq+pTOverGfrGwVmpHWrqpPaxKqMMyVoRNFLk2806l3qg1ywAKaNlvPYCmmGxR9iANFRzZU4HgBRLhksQLcqFR1wZdYZPRM/P2IGgZzQGGK4ZTeHvGmsYVGdUjgc115mAHh2bF3OHD5CA1E1ze0Awzd6zVgM/gngNR+9EB7FCEjP4I8/gomfwSOS2z4iCcDzgIgn4e0cVUdS/dqI9MzdNWLh6sfhuKK9UjP3ipzecQaxmgs+Z+dLWb/qVYbyDmHFcJW3MNoOa+Cr9ZLWQTeZPzpayhayWsqFgElxSGZO/wBoqrWToPKcC9Zk7/aKzjG4HtHFmo0nf1q1jT8MFCl3rGadz+CiAeYp4HgBQJrKsGYxqqZmBmrVmY3ggT0NAEYwJSDktpjmG0D+HxS2KEcxRDmb+tSLbu00jAFBZk/vFOA9iBl5nfsUfRVg9p4KEZqoNolYgMVWg1CC5vM2di8q2w96nu2I5/ZZOC9agfX84YLyvfNN9VXhUG9PDfmC2ix6g1ucitTtTPXDNNKmsypc/asJzP2AYK21kSmgjDTUQHnE8BqA7h8UCeA0sQVwxMoTRGBqOYH7RdQPD4o88BUc5LgKOclxa4YbWGRoNZz+pROa+C+6zf8AUnTZtPCefeae59vzqp5ljvcTRyW1HSvuaONJ1jVqzvWUhvOBgtBAezwXNfC2i042XFHwwVkBt6SivABrFsiG82qyS9v8RWb0YM1HOG2iVVkCkMyWwSOmt5dCdCjuH7NQhICY2j5zNB6q0Q8dwA/vTwRmywyJuHKjAAdZ6wKNrFnmmjBZ17ipwvjTfVz96QwkH2LS015ittc3Snwkx+wwNebDOHxUiMCr59w25R5GhWa5Ahn5HXsDTB8S+GsMPHvGlB/3kFpvwNKyeCsGXlzRnj3wkObyP2tZD5zDYtxSq3Q68xzVHrMOS18hqBbcw2gOS2q83tEvETL09v8AEUizeMPeLOXJaVuXVC9XVym8zH8meCnc5mlzXwRTOUThRZVvNGxQKNotH1hrLj8Dh6xJ4mn/AMmQZj++zBeg+cwUSZVZVKhyJ1Nb1pDAZgGfWg/JyyWaVSvvbESZUtmGASTDBdutjgbYF20aJWKjSjPmjwNl08z/ADrnNscB+JMN12XF5lnAM2TZDPW/jXob0crPu1l065c0rWjh4Glgvow51xZ/o8UOmu+rqqw8zQg/oOeYLqVNsyzo1LOgUO3IcCIYYfd2QzBXYBz7oMe670pNbp1Nwg0OnHUpfkA8F72qi3iKTR6lwN4tHQ7gcM4VRM8DPoMPkWM9Kt6h1Ws0em2w2D0vowjh1n2f512X08LSrkmmwb41M3QAwCOHQz3ryLcl2waxChVKKwYVCF1mfYtkpFw8FrqsmgtV+6tGHr54R+s8/wDc1jD2LtFYrz/EvhvHrl1VyNDOCyejBa6APzmmuA/CJ/iQ7IrlOfCZIpBgZx5GzMN6x6Wu0SxuGNYCjUfivQ3wntQntd6IGwww2L1LQbSp3E6s0y+6cAHHhe2Mzw3rO29bf2Jq1VoEumyYEKr7GWXgDDP5P1rccKKbVbbqTtD5oHqZKPNnDsXb44fDj7XSJINgQYN4YB0KDPPAQWm+zc6SXYCZe4dTpLoOOPngHZmsswrwp4cnWB2C57F9nA15yva5+KNh3W7w24SBrc744PaOZs/7716tetXlg7A+hU8Cz4lNmu1GC2DMh88zPNRiHR4cfsbgVQ7Vw4hcVLgk3DW3/HwkZ4MmqW/PR+ubjHXna/cdx8nSorP3NprDYtbxg42WHYzseDOfOpTWHv2eP0fnUSHxmqN+WbIlW5SodEp+if3uW9sD+BZdLOGF9BuyajQb1vN9/fH0ThA95zzXark45XPRObs62bVkyahCzA50s9GKyHnNeTeHvpG3/wAN5U2lUrkJkJ+pG/JlyGcze+hXvpLcYLqvOBRKVSjCHHq7PjR44YZhs2Gfes78Nts/XuKl/wBStWuxKrc5yWjePY0Z4Bn5F2rh1xCrPCvgFT7VodlTGZtRMzOc9gcYAPvNeMgok4/DbqUbmA/o+e9bM+MfFQIvq6XKB4AZ0Wd+GALHa0S87YoECPU6lTr4ZrFQlPaxgbOGZrn9KrFSps8JUR/B0OhXEkHzga8pvCQfWs/hg7qLHbZOmTH57/NSnPFNR4AOHWQ094KOb2s0rKgsuPMOy2MOYY6M16gp8xz2bbiOg1J+HMkONsA8eHQaqjDebmoG81oLeo86Y0DkH9tlHog13msLhaVz8uvV6POlngyAYaQdC00+vP150HHGHmdAMN6p5lvSrVddYrGDMhjrDNWtvG3JgHO0zBp8MAzBY3C4sEO5PVrrTESVslPAx9C79aV/sWTFOw+IcSY9SpT2HPRD3n8i8jwJnqqbI8AHsM2N6so1eqtSnwqU5LeMHpIADRnmALaIRb9RpPCuwGbQiQYreFP9trHvMwXhHjrWLjufirIti36k9UqfTjzjMx+gA+fzr2DxFO4A4RnbFuN61QYgMM7OvDzgvON223B4RWg7WaHUYbztUjGBvSAzkm//AJF2YceFJwKCmvXDWKleNKo50eV9ymBI2GyYL0BxU4dcMjsP1VSrYjQOawejVGPGDwQ+teOqJRD9R89O2OvmZ/WC9e+j9W7to9Barlx3UdetqUz+yBvOGe/eubrdDkvGO3r1t6yKI25Ro32aigAes473WfYthwNB+fw5rtZrjAMu1cAiw8Nh7A2fQrrj9Srg40x3otKP1baVrs86c4A2SX/oXGuGNjRI0qz6idxyQqEqpYAy8ezDsVxwPXc+CFqwYdpXXTa4+zGqpzNGqyD69AF55hw65efEsKHBfM6Uw8/KgNBsAAD5F7As95ipMVu2bjobLMiaZxQqEcOv61S0Th0xD41tOQWM4VEt7B4/nMP9hXhC14OWAxMpIXxd0qS9cr+w3s8DAAzwBai/7eYuG33XGHM/V3jmB9GCtaCEtmO1z0RmMYBgbLXQsVxIjVm0qlcHElhvnKPKZAJLTT2D2fnwW0LcE9LewKzMi0q46Vb+cKnRgANHr34Z5+fDBclonDSuVjhjLYqsUIYHnKhnhrPGfZs7F6H9ZW5W/Un8odKuSlR6uZs037+eB9GxR+LtKuCpWLcs5iqnR6VRI3KwGc97wB1ma26exk8sQ+Fdx3C0f2ZiA9HAAA5Dz2BmfesoEyqwJp02cefq4zDDyGtbb1+TrVtAKVEckyZE0DMzA8OW+hc/jScNXm38zMzMzPrM1zXw4VFriTPOpRXddiNmHQYdiakxqlAdaYcY9uCrI0k42eHetHJrbFbIJU6IAHhhgBrjtbOT4fJunrsABpQ5IcxqaebXkVnWGXDBpxs8MAwVFhg7p6ibWn4YDqL7DmMA7pymweDDDA0UbQeE2/IyahGH3dpzvWgm0oNEJbnfrbE8ZmrKybJvS/8AmHLYo7xxIp4PSz6M1rq3wcu61Yvrmc/An08PbSIh4aJ/OBrOzHll41YfprQA2HWGf51Uz68xUosRzkdGQAYPOgeYPKwtiHLrE1qL1h2H51n63R51Hnu02pN4SGDwMFPiAPOdbDGZmYYKIYPstab5hn8iWDgKQyyBhqGuhSJmbwAf5Efjh1qbR6VOrFXaodKY1pco/BDPD+NHW6VLo8h2nVFjRlsdYZ54L1JrDRzcc6zDYCZOS+ZeO5mfeo/MvmQOGe8Ej1DPPP8AOgs4EN+pSNNtzAADMzPsVnMZgs0iQ+w3m8B4a3nVTSqkcDmPOYYIz1DhBpubEUkUeZBZhYOS/vGeeGC1dHmUaZHagzoIAYM6GqB73j+dc8ANF1auyWadMq7TFcrHq2EeZnI8mCzuB3qHcls8TrGj2HWIOdQtcM2ZGt91AP7UeeB631rl950Th6dUBy0q/JqtPA8Jk442iAPrc0rhLaVycPqrc385U2EDJvxmgDB6Yfe/v9syfkBVljcRadQYDVvwbAps+j8sBzzlhgb0oOsMzXo5oFqy6q+6/R3IxxwDP2wbAXReG/FGXQZ8eVVYsmZHhSQCZBjgBsvMYYHs86s72o9h0fgVQuIdKg006xcNeffN6OeZxov9lMPyKvCw36w7NvSpOPU3XZCUEGJsPDzrnEe/OM1cvma7bNAb+zdpMSTMKeznGN4Nntv9RbvgJYfDK9rZrdKnQY0C66C8bzJzj1s2D6Aw7/8AbXL7qsmhwJVV9XVzWBgwNk2gw61E4P3a/YF9BdsWqxgl05kzCPIPAJ/yLq4R12t+jlX4edRuf1VRKJUTN6Z6uAMGTDvwXNbGs/hfP4hzeHMupTJgVRnCj1Az0QOb1sgYB0Z9GarL54u3xf8AdtTuqdz9K9Ys8qzEiHsZirKHSv5uiVh/A49Rz0TA/Iau7c73ne3oc3bc7FVu6dUbbgVWLSjZgUykw8wew85/5wXhx6lXPbFDhBWKHMgU+VJ2VEwzZewPfgvTfof+kCxYEWt0O/6zWDtw8GIEjRN4GX9/z966xflHse6vRcqVs0ps6lLpzL50dqohomcrX6wDNB4yeh+qq5CYzz1wM/yYJmxmWKxXPVzESTM5o+Vwjhma31h8K/X1v1OgXNLClXbFNh6MZhrSQYw2Bh3/AJFuvRF9Gmv1i7ZtVr9Yk02jgb7GEfNmVMMPIfYsekegPRatiwL84c3NQ7xtij+uKQYUh6QcDRMAw2GHzr0Q9Jg29a82WFVprwUiM+DLPOBgZgGf+ReOuCcm57Yql60CzrgpX2MYrGDwVYweN4w9sYH1rJVitsWxXrltyxLqr1YhRWQClR5fXGB7PWDD6Ni8XD1LavFG3+ITtMrH24o8DQZYOTBB7DB8+xck9Nu6pd5xYViWrccAKFS5IVCqyI5nJeN/5MPJ9fkXm28AqVmSqhY9DqTNYM4zEqZLOMDJsh5FHqV58TYFLolNgv8AMnvfhgGDxvBnvzBc/Y6GUsy3n7trh0qmng0APvSZBhvjAHeadrdb5CrxIluUpmqyLekv60t5nWB7sA8+/Baig3hSnqpNrFfqTJx7lo79Lr0iJA5b1afQCr+HVyWzbz4WzOCTq16MYBLePBnQ37wD8ive3Phiq3dVwVVqO/cGcnkmQYZ1Q2YfIas7StU63dDzcuDJjMwoD7zx4Z6J4bFHqVBpVNgVujTq4frCEecPM9jwHvVwzJrMCQ1WZc6MzIYZY2GeYSQMOgwWJb3nQeFFsTPRmdqtx8nWKhBoPrCNOD9qjGAAehn5A2bFt/R4gMPcGIT7+905j/8AjXhLhFxaue0vtg3OYmeqrhppwpjUdn7tmfQYB9AYL2rwN4hWPSuH1EoEu540aXNmP8mEsDjcyfkDNdPDY6AdHbCVqOZgtBGDBpRZh7tNSIExgxBjU3rs2QIwyU6BTYLzWD7HX3qCYeLqKQbzgBqAoLg8FNYo7uw881ICpMfiKvOTrNKnek6Lumo2jDaw5gK9gPZiuewKq5qhqOLVQKlu09RIW0D0ZuSKy9eoNSNh31bKNl09gGHYrv10H9SsI0xiS0BqUYcUr1BuqBH16k+c8+88MFT8yYe0XoB5mDJDeGa5fxCt5iAbUuCYHnnmB7F0d3YRClZPPBWsOS5G6HFn42YYaisIzzmSiltHGnyw9maZnzH3ndR9wzNR2TTzwZhmpWjgaKSGEV2VqBgwGZprPRPxFYSaIxW6SbDj+cd8MDw8i82PLXEi4aHRKpT6lBiw6k6xUjmng97EOj9a4vefEKmhxIZqsG1cJErx3me/DDZn8/8AqL2lP9G/hXPjy2AtUNWUGzN7YB+deReP3DqVQapCqtDocOHyrOjUjgmBg8eYYGH5DXowN1cRfXBwWIJ5gfNGZ9+/DPMP8ix9w8oc2I5EMDdfZ34Gut1ILDO3jnepqVSnWGXwOOyHjPA90GHfsw6FwyiQ/wCdIVOc2HngfyLDY6Nw3tKq1Kg3LX4NSks12lwweBkN+cXfnh863U+g/Zhqnt3dR5/rOoshNOJI2YB2bzR8KLts6xs4tZOSzNlPGEzRZ1tEAPYB/nXr30irVpUnhpE4ozmAknBZYfDMN+g9hmu/0/ZjeLc3NDy1Srk+xjsuI3TeZm1GYGAH4OiB7Ni6QHPfaaW4340IMMDDoA/8686XCc657mqFDs6hvZ0sDOSYPAYdn6F3iz9C2KNCtV+UcmaAZvAG88z61+h4ed8q+FowDej98pVNolcqs1qDTqPMOQ/0BgunQPR+fmBnUq5MhveTls/8617kdPlykD3eGrONXqlGaBth9TbzsO4LGnn6yY/mwzAGZ3Y8qINM1H58ci4/ttbbFVbkzwcqJ7/Ot8zdURmRynZ51yKlQ6zPkA3Q6VPmd+bLOYYKf61bZ2SzwdXJfrw6uHm8u1wK3EeLT1N6sOcbNcMpVYl8+GD66RDnv6XWvlex6mP+HXHM0B71HeZcAOhHGezNTdYMcFx4dKq1sD3rOXJJ26bDm9X1VA9UNNUkmHzOea6YQ5/UpNRCUDb7bxgfkBLk4LzBuS25OZ9BmfQtbMPlg0MMzPzqteZbeimbgb19Lhtx4VNErFVgHoPvg8HZ8i2capNyQ1HDXMpMkwkeHmCtaVXnGfbudCez6c3+tHDzfj97oZmg5lwFTxrkgycPHVhrAfQvlXwu+LS+fc94aA5n7xQjzySXOtYBvQPHgogPYJHJz94ge1sFBkzOreo5yfnUJ55WDek/vEByVHMwQGexEFrKO89uR4fFRDjOGfhoEZ5JrEFI5Yw7EgjOIA0UOjgrDlnDFLk3PeNoIOfwRgalnG+RLlfigZ1lHM1O5NI4aCsw+KkMxjUsIe5TWYzYAgrAjOIwgHmrgIbakBGbQU4Qz8ikBGwVnofIo5hgSCIAKWCjmkBmgsA6FZUcG3nQ1FWsntwVhSnm2S1DcRbcRmabGjgq+ZPYA8NRV709t4dMFEwzUA5lV6221STJJn1q15MMdQ1XzIyClknvUEzVnJZcyUfl21aEQEsPipGIJo0DTx4KDrKRI6FCmSYlNiuzpz4Mx2Os1QOZPi02E7OnP6IAqGHDl3JNarlxsYRw3xqdnsD5zUeNDl3DUgrlcYwjsb4Ec/8AGavjk4IJDz2buomXntqhPSc+hBzJ4K0EcnAkua+Cjmmj2IJByXAFRzmOGmje2+Io5mCCVzPzoTebUHWS1kEg3m0JvfhqJrINberEvWR6yg5/BLWRKdrJayg6yWfwVIWOs3/UvmsoQGjA1ImgaMDUQDToGqWnAe9SmTcUVlTWQU2LBmS+A6eopASXD9oahAiA1AtmZOBe0U1mfgqIHlKB5QpcHUtqH1lmql578NNZuZK0rg5LZprW3qIybimxobkk9iCQy84CnMvHgmQpT6neqpYD7NQrANY8E0Z7E6EZz3iRs7ESg6zn9S4lxsh4VePOb2Zhgu4Gzga5lxvpudGCpNub2DAFcB3ghPck0GXBfcz0DzW+XGeCdYbh152K/wBEpdre0APxDXQF0J0DULmWz9mmjkuLT53LtcaydB5tUgPH+Ilz+j7xX8xuF8BgiMAMFnwqvzozqThh7ROnB3QamHome9QTmI5JmZKIbP4a7+GP0OO7PHP2e0TPOH51E0X1OhxmPeK8COc9z8RHGefN3xHFanDg6XhgqwIxnI02AWf6Er6GfhBvNSuc0Q1NRVoQ6kAbAUQ4dSMt7ayuHRFtAFwufjp0Lh/EWfCA/wCRGFNnfhrm6oX3W0fr5v8AEVDXuLViWwH8+XBGD+6PW/wKgvZ6dRLZlztN7Z2AC8XVi4ZdSqkuW+Hi54LG+FcW9cXD6Vdj0f8A0PSpNY/JgudXD6YF4zPDo1Kh00PIYZ/5156erEt73igvTHA9ouO22GwuTipflyTObrl2zJPyAZh/nVUF4XBlpt3HUgDycyaogze9nh+c8E0bOBdYfkNc2Gy1M4P9OnHmrKBflZoMfQodz1KM0fYDyyR6h9biQM5+8AFGF4Xc+tv1WUcudOkyXT73ns0geDyB+dULwcts1AP6E1zJs+zNMDQGAPe7UuBclZokj7jVZ8b+6PBZZmsGBajnQrYIFfrEfm6bSpMkPOAKB6Nsn0rp1NhtUqv07nD/ALQb29bOseknnSeatxtkJHkMAXjKHMfB82HPah1q4CY/h4ZoO6yfSlv82jAGIzLvYYB0fwKke9J/i8H/AE5G/wDLAuOzHpTMc3zcV7Qab6ypwPuBmg9QcIvSHO6mmoN3GyE0/eh/qLtebfv3wD61+f8AGkxbbm896jeM2OjA0V4cUaxdUAILb8mHHDszW0Wh7AOj9jDAB9CGZPg23Rpeo+DMgAz864ZbHpIXHfNr1OuUPRA4TJ5gbPQaqvQ5vOVed812LccvnKg+Znm9vwX2+1zukcK7/rnFcqnRwfjQGoR4PH3muy0qjxKDAM4rfih77vNed7qhzvRs4kVC9fU0mfa9QAz+79eZ/rVxY3FqXe3/ACxuC5o0alZ4RonMgAYLBUOSelXQePdbo0io3jd1BjW5FPXZg097efznn3rzlb3Drk7aOuXi4cCEf7MYb810vjTfNSmcS5Fv1KuBMoRmAcwHQled7UO8KvbVnUZ83qfCAAM3gwZe/wB8EeuWW9YFSvB/lbcqUY8zwBkzwNeqPRj9EvihZ90BeNx1mNSoQYAcRrN45PXsNVXo32fOs/jw0FYpTIU+o56JnvA9i91GaDl91cN6NXrja5pv9lPMNiuIdqwaa/rsMdCuADOsuvq2wbME/IjwD3eIrhk2zFUh9XhrH3/x7sfhppRXwk1ipmGfKROz6zUxbPDosyAw80evsa865FcMC8arVHaVb7etE98fYYfWriwOItwcZooVVi1To9HzwwN7PWXQGaO5DHDmjZa+RdMXho/PzjfYFGqXFCFQGPubWAc+fXmaXFGlVWe/QuFdAbCNT5RgGABmvTFmcK+fv+474uCnMyd+EAz+RaiNb1AZuM7nqUXWkdi6Ppc7xPxL4CfYanR4NGl8zNfPA2tm8/P8isLno8GNZFr3xVWAZl0swivAAbz+derq9Ag1642pYMctHAMM3lX3bwctl6zZsSgUf72AZg86eeH5FjvwPz+uqvW/Urym1mIYMg/hgAM9aqoDL8+vabG/MF6wrfAHh5c/C+sVkKUdNqELecsOt41yKfZlx1K1aVSrfoB02PC/aa29sZz+tfI5odsW5ZXgcptX9TvhsNY+pA/GfNg1o7nDWaCW5WQnywDA5DL2YLOctKmeI4eeClsjsvYRXWPP3p6lPPxpQADmDXeg0fw0caM+boMN7NQ1aAVWG2zKNxhs13PhFw0Yv+qG5UqkFNdpEAJUY4h781hIFpRK3WYlNCsxsJQZn8i7hbfDGVaXDSt1WouZzajJ9VwHsMAMD7/oXlrc3mXg2dSqcU241YqBvclzDrOz6wVrdVwwZ4US0XIJwHYrO9ozzzXOrttKdYdUClTnwe1wzB1kNhqQByrhqPrXXADAABkzWOGTOTIxhNkeH741cWNSv+V9CqtRbwpXPhrOrqdgWBEvCbLtmsPyYxgzrvOxz2GC6HwQ4S0O265WGKk5zkf3LLoHsMF9L1eH8pvmelTe1psSuN/sRxgwPzgub3naVKuG/wBqsv0eM9SoobAMNmfnWthg/Dig2w+Z/Waaqp1KTTZDEVxnmDDADMOhfY+J8vveYvSBgBJvl2nRZcbmz5VgIkcPOu6/yb/YC1Gvsc48cgKbhMju9DwedVUbh1aNEualVWsv85VZRgbzzx4b11uTMoz0hptx/MzPAAT4+E73m6+fSZbuqm0yw3LVmW9bTzwBMqEgMMwDswwWaC+bOs/jS0/FtiZcNKixsI0eCzmcY/OAfkWz9I037k4u2ZYDFK1qfrBKeZZ78z3q6rE+3LV9Ju2n2GwjBKhhCM2gwZA9+Cx5eHxDp4r26tSrhfmTPWTEGTDjzQA8JAYGH5EUysche8SpRGNhxsHnc+9ausUqlHPPXPRP+Bc0uS54J1JpulcszQmD+81uWeAH8jPnV+IYulhcgH79OnXmAHw31j7MvPhtfk2RTbVuBmTLi9bJhgZh5wVnUmdF3TBX49biO5T8VKPTb2oMdvPCbTpgSox4frBeauOt4cXrtvKJY9Di+qqPUWc2QMA3gHWZr0VVYz8zSw9weYfWuW3tbd1XPdYPsfdnYUMwZkYAZ7+tbfNH4Zd7xpPjVW26lNpz8rWwzY1QDYf0KkDrwbXaOLVHr7NI9QRGM49OezmSD6zNcvh2rUjEJzjYaR9C/Ocj6nGrQPANNzrVxQTiG/pzmDNpzydig1uMDNSMGGMGuxPUGA+9UmsHwZaDe8Z+Rctw6Vrc8DlvHgtno+QzzwWc3menp711CpT6NVSj6cQAaAMOjDNZevU2JGA50QMDXNsZ95l+MWg+2YGYIXnsI/0ApFVZ0ZDTeoZm+zmqmYfhaDnQ4eC6YHqvh7Pqtq8KLaiUqpGy0+B1Dweveuh1uz7jqVv+rrutH13SqvDB7mIJ4PAHk/IqG5LVnW9b1Hg+rZIU+LTWA5jRzD/feux0HiFat20RpiDUmYciiMhnzB4Zh5AXyr/yv2fD63DyepDyZc/DF+z6Q7enDmuSalEpf7TT3QADhh51x+57hrl1SgqNZlm8YBgGzoBejrqmMU2wLzqLmx1yGbDP1mvNMbTOmg33rv4bfB/iXrR6t4hT6zYe73/Opxn+Hgo8xkAMNNBmYCYaf6wW75jZ8NJ9ZevKn02jP02M7v8AvE4Nir+KMluTdsjQiRvA8A5EczMJJ+dZR4HFYYZ05oFYTNKlmxrsMZqIBt+861IhxnJ8xqKb+iBnvMz2KdSrefn1Q4ubJtBvzz60FZ3rqvBCxrHvmueo70rPJ81gENrWwOSfkzVVRLeYjDLuZtgMIL2vvPBeneD9q1yBa4P3Bb9EOnnMCpsu8sBnD37/ANCDy7xp4dMcKOKFYs6Cck6ZFPOA9IPMzYP/AHNBbFKpsOEDlzWq9J50weprx/0n5PkWl9LeZTpnHCt+pqyE+OxoABtPawBszw/jSuS5LjsOfa9GuO1Y0z1QDFQjBIewPeGaMEXijfPEaf6s4eXrVYz0K1z+4R2g/ZgMOj9CxJ4HHGIDn3cPch0K4vm52Lzrc245dGjRps0882f996zoPN6QNo1izpxogNajDa2cDjNdsCPEo0RyN6vw5V7mGdYwDvWRgBrA98gZ4B1qv6PEWdwp1u87kp1V5uXQ3GeXfA4ptAzgZnsw2LmRs5yHW/IeCkAecLXQBh1+dRH6Ac+ZOntAc49YGA0MPk8i77edL4Qs8IPtrSYJzIhzIpw6ZLzwjPbNcPn+tcH5PMVsGbwnSbBhWA+w8ceFUglRng6GfkNaJuGwsZmgVi16nw5pUSpUGRcVViyoD04M2Y0UOvM/k/jU3i1Y13WG67wulS59yUqjveuqVOPMM4rwHnsVVcPEWpcRZ9bfqUEwOgs500KYG9kO/P5Ohem5PpD+jbXrDtqDxtvGNcNwMQ2M3repTxmABgeDwHh+fsXRFuRyHh1wxuM6DctzxX5MmsWUzCm0cJDOEkDzzeYwP3K9rV6q1K3raNy3GIzMiVvN3R6Mw3nsWcpV/wDAviLfjt48NuIdKeObTQpFYokvNmTJY36J4H14b9gJPcSKxclwzbYsrhlPmR4T2i9U6mfJxvyZ5mkKh4CrdSptn3lUPV0GqstBPCbAdeMwPlevP581va9clscTrwkMV+4JNtuvydaHXPbBG2dD3Rs+deq+KPBk78ptwacGHJkSobEKmtGAAAb95/IfzryDxdsO8bMN243GIFEhU7Qp5xIMzmQeM8+/vS+FanuThjd1sQ6rXGIPrWjxQw9eR/YzGM8APNNSeJdAjR6PBsejyXpDAeNOlsgzJZDyZh1qvs/jfxNtWkBb9KnQ2aVgYSWnYAPA9n9fQqelUF+6pVb9WsAYMMnNzA9H8i+bfC2i19M4kUaHHaocuDGPNl9iY61G34H51oqlflGeteFBp3C6mhSoTLEJms1ED1j2YZmuZU2jy68xNciU4J8h9nfn1n/rpU2sSoGdAnRD5fsiSD9j9H1qNr2arEBg3TguGEwn8NGQ0ex4zV1MCs02luv1WCYOgDAMmBgfyJSY1DmMakWjchIhGD5+Qw84edS4D06q2rHpT9K1td432ZfXhv37ERf63TeHT141iw7ju5xuBPplBNhk+YA8zzwDAMPrXVeA9Ko/GPh5U6Ncc6S87BqWFVjsngdKP3M0P8B/QuO22zc1k0aRQPtBTToVUNh8waez3+dewPR+4dUOzIp3rTZX3urwwYkg0eAGYH1rbj4UN7ap3jDpfqO+H2Z9QhexqccPBnxew/kPzq7B4wdzTJz2A2ZgqqZWG2QPTXfHB5tDURqk4Z+IatmXsxXMY12sBNBg3PzraszGDMA1OsFfT1o2lzDPV8DoUJ7zqw2GoU8w92sWyNrGHQrCNWHwHT+RVWeZKWyGaCygBOe/ZetbiGeiOCxrM9uHhoZgatY1YcNrrUiRVbqqMOYbEWCYAx1m8ueXPXpdVkG2+5n5Frql4zBt+dZyTSogeIYLyBn2XjMd6s4xqM8zh7NCy9gS2Q0bLykcztWJue9qVasBp+U54r54Ms9GZ/X2Lkt1ccr1en0py0okaNT9YwnuyHgMPo+TvUXC3QOMd4QaUES36xVZNBCo7AnYZsn8hrH3JxC4sevjDh5zMaPFhxWDN5kDZAMOs1l+Itz2/wATgo7D9AuEJZyQA5bwHosh0ZgHQtazcNj2ZFO0pV6zAhPgAGEsDkm8B7M8/wBC51u8GEqvW5IOuVKTDdis4Pcv1n5z+deZ+Ot+Wk9QaJ9g36VPmvvetD5gMAMIuwMw859C73RGYtjesPUc45nrd5gIZy3sDAA9yGfX3rw+9wZnT6jdtpVmvxmahSJmuzIMM2cHt/XnsNBHtXijb9V+0EquWX619bmBs8uyBnDf+s9//wBguTzIEufeU6C2xouuPbGc+hVNVprlEkco5vM88EdHN+fVwfcqIRpDnvXuheWLilRn5lSlucjJkx4XttE8F7DrfpUWVefCJq2LgiSYbRw9AI7IZhGBlkw8/fmvH8/15PfNiKxhzp5yeU7z+ddKsm2LfCnWVOqMQK3HlVUDmRAD2zAHvBIRaw4CWxdr1UdpVGOYy7cL3NMvGGZmAd69zcnaPBCm0SDLtgJk2UG+WDObxn3mZrH2rdXo33ne9CuCx84Hq546WYABsxejPo/WqzipxvtiZfgVK1Z0CvRzD1FAAHsAZfPrzM/0Lp72OHq2lBTpMKPOib2pQZsnhhmCuwwxXP7DuqgV6m+rbfczaogMQj+Q8OhbBl5ZdlujA69QaVcMB2m1WKEmO+BgYGuaUT0frKpTTrFVYeqWZ5hmZhgHk611jPYoT0nA1r9T3AqJSqdQabHo1DiBDhMdDIKvrFn2xUmJER+lMm1KDA8ADNW0OS28pGxZfUYcJZ4CeqpoP0es62HunQ/z5oZ8Z+iSMH2Dw84Au4BGAzzUSZQYMxo2JYZga2+r8mHJIcnMc1LzzJS7tsZ+26HLqVAfN4GAzOP34fIueQLnnPe4k/oWjJvXg1mlCCNuQQKkckA01OBl/wDDWSkd6Aw81p4AspWKPUox6jDGbS3cPeWDiUkMFtHMOY1JnWgG3oBqgGxY16M5r8rgYO+Q10KvUdvmDfiuGBfIsuzQeZkG+/mZgfevscPN+h83mhWvU1+Bg/KbAwWgptbiaQN9HyKPMZcktOtuMHsVDg5GS8c8G+t0BmY28HhuJGazsB4wdDTV0yDhh4i+VzRh2cPMN6S2Cj818FIOGZoAo7+fWuPDZXvGmsjVqdK3e0Q+p/ivRT4mgwc8i0oUfenjprfkQZQANSAZVwdKDL2aRw2A92giBDz9oj5A/wAP/wDSnQMFOjPN96CsCA4Hu1IZh+701ZgbBpHpgi0TkG/eKOcBv3alvSd6j85uRAORTT0bBWHMNqDVY0uTH1Ij4B9a1EUMMkes3/Uoj1NnAGbE5k/OgADDrczV4RtN59sOtL1wx7s1XnvQgDYe7W3TCN2tQqTCRz4pn1qnM1HN7AFjcLhoHpkTS9oq96e37Ns1Tm84o5vHkoNtGExxS2Z+BLKMz3wBF6y+Che22ZrH7xTwrDH4i516y+CQV5sOtB0L1wx+Ih9ZMH3rBeu2/OmvtC350G1mTIgBqAaqnqk2HtHFlJNbM/eJr1k491pgXB1JzNM88qo3s0B6+GasXBzANYrNy8KzzT4fzFSHtFlk94PH51Z1KpNwKXLla7IOgyeGZrOWlclsUS2qexOrgA6YZmABnvQbU5LiHWzWUk8QrYD2fOPfQzgoT3EVg/2GhyTDzvHgoG1M0gNc/k8Rarl4FKjAHkM81WPX5dR+zlRox+dmMg6gZqObywVt3nXJ9R5GqzuZA+gzDeC1ZzG1aBPSXMlH1k09J3qPn8FaUvWb/qSz+Cj5/BLP4Io7n8Ek1n8EWfwRJJJ7w8/ZoO9UABGnQBLD4oBBOgCQAnQBe7BgpYAo4HvUgDUCayCkACaZe92pAPNqA6jz+CADzJS2Wc/doI+9TYzeYKWyyAdilBpoIvIOGlye/wBmtBDjAYInofu9NBRMw/8An01o6VDwED01IgUoNj+grIGWw92gENMFIOSxpe03qIZqE89goEh7eSBkN6ic4pEZ4D94rDz0MDDxFzridQZ0+25fKMZ4BmulayiSYDFViyIL/Q+yeauB5Ftue/TazHfDZvwXoWTAnaEd/D24ZrzvcMb1bcs5hvoYk5gC9UWxMg16zYU79yutix4STBSAmZp2pUpxl0zDoUdmMvoRcYcSRrZph7rTrIb1YMwGPaGCncEKfE1IDUDrVxosB2AmXozB+zBZ7MWrD60OfwRmyeSAwxXTF/oZDDeszfN+QbAhnLnYaodhgtAbxs+zXlrjHJnVK9ZfNSwew6MOxYflrwugB6TmAeJQ2T/P/sKIHpV3ODunEo1NZa8mGa4kFHfmFgw+AfWpH2Scwz9cxl8u/Z5nZ0uu3P6SFfrYBoHGgf3ILHvcWrqk9d61X6GjXOnobm/70yf51XnGlrm+jmX0ukSeKl6SWjA7mk/WmoHEi6oDuv8Aap41zczMDUc3tFPo5nrtsn0h7/epztKOssyWjDDN5neuTvHL5h1zX6zz6FVBJfMtilRpL8mQEVhh550/IGax3zNTwbOtXVsQLOqU0wuepGyAdAAHWrCiWMdVmtc3Bk7PkNdDmWfblEpBv8jJZkMfuc16M5Gh8HY0gHG4Mx7D9yanT6xSqlnQ7ZseADR9Dzwb1mY1bfkn4jcb/wAEFfUGjwarP151wM01oO8z60GcrfDGdRGtSdODVP3IBmaz8mxrqZwfChzzjn3hGNelbVsnhy9WQrFcvyBUjY6AOZgtnVePHA+iTTgzrnZN0O9pkzBB4sk23OjbAgzD/wC7GnnrDuZmHz06lPMtfOvXEzi7wSrz+dNu6Mbvk5Y1h+JFw/acAiW5Q2ZjPe8AYYIPMgUSqzD8CmyTD6F1C2OIsGyaR6qnRTP5APoWwo9pOHRNeXOehuh7lroXPKxw0uq56ycG34mtn35oLazJnCu87gOlXHBBnmjz1gVlxj4LUqxqWFftydM7/BM/oWj4LejTVbVuVquX/BAwDsA9gLtfFSt2H9nHaHKwmAfQAPYKqgeHqDMqtVnhBixAkn867NSrPvF6K0wxb+8/wehUVn02gWTVzqrkHnI+eebzy9EU3jlwnOEctyXGgPfg5pcDhgcB7xZlBKqNv1UGg79ZO03hdblbqnq3lJOfefMrcXV6W9DhunTqHbhzGT96b3+wuW0S5LmuGbIrlvsTI2Z57OxeYHkemz5wNZwZ0lnPr0jwzWtsm4bq4e1SPeNuP6NQDvWceZl8wbegAYeRNTJk7Y3n0KmrqvEj0meLfE6l+p65WfuWGBtYBvVVwTn06myqhErNN5mPK6zw6OtY96S+FG136UYM/wBow2KFAkv6GbD5s5+RGDS34zSgrbrlKbkg15HuxXDNt1y86DCrFDghhTvbPB2edU9sBKN+FcFZptSqtKM8AwDYZ/WusVU36PUafUX7VuGiUozA5MTR8Ew7Ni6IsewLYs+LcNg2VOlvvBIp0NgwMO9dTjPawLkNpekJwZqVJiQWLnZgAwABhLAwNauTr3C0bESv/wA1H/YXt5/6ivYM+IVKpU2bSoMGTW6gD29qCGYB+dZ+t8eLLt6stUO8WAo818M9J6eBnh+RBflVg8GeFVVrNvxDOoPhosmHWb5n1ryFfPo8XAzwid4xXc/JmV2rva+ASczADwWI9S3b6WnAu2KJUH2Lm5moAz4MdoMzM1wKwOOV28RbauOd6qprzsXPPNnxl58tLgnxRvaY1Bodjzw1/fPbAD6zXaLY4hUPghNoNo1K3DgVCKfK1IDZ9wauEPQfoVXy/M4eVWIHLHNCfgesfX3rrVYrF41KQ7F5uGEfyAua0expVt3vHuex3AZpVeZDmWg6F1VmEwyAA22lh2lRqjyoNypeeHnSeh4Hhsw+QFIB7AUjeQVoQILPs2Mz+dFWKkxSoug2BnImhgykZ70NS058D1cbGevs6+hBT2rZ5nAC2a+x4QGb5h51509PniFTqJb8Lg7aIRgN4AOZHaPoDPZ/nWt9Jbi1c/C6lh6jnaM1tnA3TDMzBeD63Xqrds924LjnPTJr/WZqLVClgMuasenP7Ac61ZViiOUGeDDD+s1KDYarXowZ9/606dSfOU1rvmfK+x+RY4bXaOdNlxj022DwUilRjN0HDAwBS59yOPNGxp7z71KpU/Wi/egz39Cki06sfzVHp9Rpv7RmAA12GvdF1VKdxatWzLAlmdNvMIwTWWQDBno7z7F4ptifFDiDb9VnBnChSWD5fz716O4i8b6Vw69KCn3dWabMZpnq3PlzDeezZgrWznpY2ZOsnh3bVAuA4EmqhPzOREk5h0dGHYvO9EqUqHy/8zRj0Dzw7DUi57nO+btqdZncyZzZhymTePoA11D0frDYvC65FNcrgU2RS2edADZz1vkUIt0UIF6WM1Sr4uayoEOPW2QZ8I8HsDXoOTwxuqZSaffFOCNJhPw/xsMF5/408SLn4kP0yjUOjvBb9ENhmS8Ad/kXpisekJZ3A2LR+GUGlBJaCMwZ5vYaIH/jXd63t9Djvh7FVGoNfCKBy6MYNecDzXJ7wvwGa2dOpTmYMdZ/Ou6030xuHtS9YW4/Q5LODP7Q1vBebKDZ7F/3vVW2KqDNHYMzN4zwA/zrq/mto+FS1uTVa3MiVGX/AEU82T/xrW0e4arPhx6iDB8uHefnWztu2+HNKdBiuUp7mA2MmzvAARXbSrOr1r1Oq0bmWfUnjvH0bFh9PN5PmcHvbiQ/DqVWvG7pxhVeWOn0oGg3gHnAOwFS23bz9Y4GHcctsJNVCYcqHLA9/WsJxFqtV4i3K0bEWTsPlYcfDeZrvdtw2OHvB2Jb8vRgVh+GZnBM95munh5P/KL4cNvDvN+sWLCYrOfrMIwA8Ab1w30gbnfej0SnS33maYAYYAezMMF1Dh1c9uV6gyJUGmvRqhTmQ5zmD3ge/rXD6rTYN2u1O5ru1j8H7nEZPA2VHN7OEcLsfCXiXwZsmPT27m5P1qYZs1yPGPDD5/nXoU5MGpQ2qlBlBJjv9DrPQa/P+NJC2IUdhh97l5R4GB79i9K8JaDSqD/PPDysnPp74YTKTnmEY/OCvh9zZcOrSXokZo333wBoOszXPONPFSgcK6NBqs7mZhzfYx4J7zDzra3JbfrthpiCwHMNnmGq8YAvI/pJ2Zf8yad6345A5eKGDLMTMwZBd9+zjwy4eH+rnt1cSKlec919yIEOIb2uDOe9ZSsXJUTYCmxDwaDoQQ4b4CEs28AUKq6Zz3eUbwZ7F+e5H3II5LDxtNtuPPH37E6em8fKm5gB/Oq9nYSeZBw3QbBvM/IsnrRM1Js2j0PcbAR3DnMto5zYYGqfBxn2HenalPb9SQqUw5mYGb731rDCirH+kqe52cslAgRKxcdHpR9MqYAGq3mX3jBx9zoW44IQ2Kxxdt+DK9k2Zv8A5wBW98ePz5/D3bDuqlSXQpzdwZ1DW0eUkdGCl1vh7w2vmpSIrHJxq3ngciOZ5gYfIuCVL75VJcrzvGpdBuqv2lUWqrR3wCQx0awZgviXzR/3P30fwjl6YxSp9Ja2PsBw5kUb1xGqXO1JjeyBgf515VDUy016g9LG/Ptbw+tIJUUAqD8wzkvB3gHQa8yRg8XUX0vTxh+W/iv9Ob8FJ2GDad1mzPA0Jg+Z71IgT26I+64/TuZN9nAN/Qu18lNo9vQak06/OnBGa36OfepdVgUpnS9TRXgiH7F57rNVLM98Ggpz7YHC72cFbVi536qcL7qDMeEzosxw6ARgrIzMEAd13A5hz2J59CsNGJD8AKkD0hvrMA2AqR7Ued1OxTqOy/VZvqp+Xg04yZgfRhgoa7dCo+vUrXj2joYVCqSWIrJgeySBmvY3EKq06w+EFTuCmy+WmhSghAz3g/hh0H+deKeGkC47kuug0qmynngpD3rA8Nhs4dasOK9VC5L3uOuVivyZLr8zQjPdj2HYgx9h2NUr8ve37Sozf32rzAYD9a9fekV6PHCilNVXiTel4zJ9Qi0fCNHZMAjPPhsBjDr/AI15v4G3gxw64oUe+JcEJkSkZvm0Z4Gezs+ddK4i3PK9MDiNR6VatGqtBtekQH95s5hn1nmYfP2K2Lz1SrSqtbhnWYO/N4GAa/zqPW6JWbeqh0OoxcKgBgBs5+dbi87AvThXRqPUvWui7XgffZp7OyUEUOh81zypBVT5epVViZ96wfZekB7YPk86LhoKJcjln3lRK56mON6ueDPSDeYd6u+NnEW2OItxx37VtkKVHigeb2iAHJM/PgslJqrFVjm3O9qAeCfzqvjRuZd09RFpUMH3muUAEt4K2tigyqxVPV0SXy3gm8Zuh2AvUFgehyxVbXte/wCq1g+SqlHOVMiHDMwzMPk+RFPM7M9x6mx2HOhnPD86nwKrSmRCXLnaLv4WGYfnV9xgOy6qFuTrRiPRqhyZsVJqOyeBvh/9guWa2/NzoUJdeh1i7pPC+q2/YEvCIw9zVSejhhKNgz6P8ahPccr4qtgQbO9ZRjOF/N8kDhgZvB2Zn+hUnC6tzqPdUfQuA6VEPrewzDPsAwXpizL54Q170f7lqNx06z/tXS5+GcSMEaSe/wACUz8/XgtYhFw4ZG4Y8RrbapV8WxblSptdinsicsZys/x8PnXbbYpXHr+SM/SBse/7nekU6YZ1Kkzng0XooBvMA78DzBd74e8eOB/GmfbWvfZw7tokkwjNS82ecAA2Z9h9fRn1rooTOHlHochi34MaqtUEH9Z43sKeyeeZ9/zqUYeFXuP3GylX1WPsdc9bmQqoGcODUWTmGAGG8wzDZgolz168ZMKny7jo+tChSQhBEdPfJMw34B0Yb17IsOwJ1wwpHHO/34cOt1SADENpmNyzMCEB7A/P1/nXn/idb1vhf0di47xqRzZUaVUOXjxgZZpsUPo6117ZOT1iHY4c7Bi0CmwKgcPX0ZeYAHyB51gqO9UdWXBgxeZam4czHzwAwDzq6uS7adVYTVOp0tl7XZDPZn+g1kmdQ5Wgw+bLvnBfH5nRDRwzozNzOg3Bkw6U+9rYAeGB/IoNz8jUv5xlPmEhnYBgfZmlSoFcrbUsKcfMyGOsHT3n9Cj1sIsOeFDqLGBmGtmB5hgsVjpVyV+A1Hi2/Uc9c8MHQzWrtWlcSWYcu44NO5AKDMAHnpYYBm8eABh3guc8tR9D7q+b0gwPwegAPsNewLDv/wC3nDmsVKquQ4ct+leq2Y8iT1yoodYb9nYumBW3nwT9WzAupyJydMrcanBMpwb/AB8z1jA/416AsONLo/DejxYJyXmouuwHfgAHgC883bx4iXJake2KPTZLLTHKvhIePDRfZDB4MPr/AELrfAfjNSplJplpVXM6hKqsqKBgYYAGGYZr6XrXw8PK5219azs/aIzqr8kNNw1xescbHLboYN1WlPA6zrxZkgDzOMfQGYYd6p43F246VHagg/AqRsHm88AYGbHfsX1/Hv8AA+fjldzMM3VvbbntzIvhub29i5PYF2scRavLiUOC9y8JkH3nT2da6hAtt+AeowCc3s8POcMczWsyTMPESM1RU2TUtV1iUxh5FcMgZ7zXxbh3xZJ1l7AkfK/FHyahsLNs/eIwk4e8UIwMDQbwUi7CY4Yb0BgD3tFVBJwU2NJzQOnSgeDBv/hWanwzhytNbWBvVVckDmXddvrBebFIARDDBxhl76wzXLZnFejVKfU7DrnDzmYJ+CbLMbM/rWcuq5OK/P1ihuUd4DOSD8N1oMMMN+YGrg7hsd6yqhcdZrPqrwTimfvgf8gAmxX16xr1mWkFDprEyNTz/aTlh96ZYDoPBZKpW9Rp9BiPxZzxu0gOVNkO/wAh+c1K4IcYKxXjj2reN1ctT5WcWG8YADxn5DNdQepXDKgz3WJ0tl49n3gOs/zqLHJeLt/3xdtvUShzsKC1SKkw+94xgYBhgGBrg/ELi7OuGt1io0d84wVR4OZzDrMAXqrjTQbVNh2sXdKZepVXjcrzeGZgHzrxVclsRKPIdfiVLWiGZgyfXngmFqSlVIJNbjuXHKzazMzPDzp6jvHyrTenv5kz/QoJxvvEfyG8C11pRmHp9Hbcbz16rofkM1I0vBb+fq9LsRyJJ0quYGcto8AjAAbzPyBhmu4XOFDtK2anWbAqsCqwqWbEIMP6GxomZ/Wew9651dt4cPOHUN2VwntwKbUJR+q5Ms88AA+vrSmXzbAcJatTolKZea1mAksm97YzDDP5Ov8AgRFuc2xcM6M1U6NEqsmlQpQc0bzJ4dH/ANmuoWHwuYpVDo9VvysmdKmmc0INPk4GZ+cz/QuFBJnaAQYLAG7N8E8/IusWxatOqUKPWarEmHIp0yLnEB4zB4DM89n5OzzoPXHBbjk/DvJ2wKdaMalU9977s0zDwPDDrM+9eoGbhYP3i8/0T7I29QYVYnSoZzX2T0Xc/Yvn1gAdnYrCiXy+ciJBqT4My5oa0aPhvw86Ld7+0IAHWqSTW9Z3UBxYr13mHtEDNYwke0QdAjVVzMPEVrGqTmeeosKE/aBtuK1h1Xwg8RZ3CnQI0xgx1NRPcw2sSzUjD3imxqkZuh4i8F89G5xg2DDYYYGuXz7DiM1TX37DXS5NS0WtQFmZjzh5uOLeOYQo1NYZYAAbBOhAzUfmXASOfh7NxRtKXJgYNajaxlVmVWHN/YTMPkWrZnm97xStFt4N66YsclmV6JPd0Ip5umg2Hs96GwwU28LVpsaRrsBhn/jWdhw5wTQc0/u596+lxfsc3MsjAAA0cOlcyGbbYYfOokx7RkA32GtmzDbZYa0/IpSpQo7mW9tTWYe3T01KxNGALiu2sQierduoo7zLgK+ANiiSWWFjtsojPFEzJY71LOM2ZoDpTYYG33r0PAadAwSZjN6SQRtZ3Qbc3qEK+Y94vhqFJPNrf1qbMZNl8wcBRHvvLQN4bwViqM3AS5xOmAYKKbO9BKCq7Ujqp/iKqPPJRzM0Fgc9zP2iAJ6pzecSB4/xFQ0HrLAU0dV+dU4G5+InQBw1aFh612b3EPOKCEY80ei5/WmxIOS2g1lHe2KFzJgeea22lZmexQXk0dSUfnDM1Cjpp4AzFQTNw0YPYCiTphsUVWrLOtF1FIC25b0I3/IoUzhmain0KbJZcZLxFHwzFBEN5xNaykPMpnl3EA6yNl7cCLkHzPUUV7YWBollLkuquHUpEGLOOMDB4eCqJ6TOktffp0l763lKuEP57kYB171CzzFZNQeHyvhsJAzmIOaaXQaWbgCZtrza8De0ABJkwxSBmWY6/KHh58FHPYlmDpprsQd6e7FGwVEPlqvHf1MAzW4Nc9zcCUGHnXQNbMNRbQgGfwQazn9SHP4IFsySNZz+pHnsUdEqEjP4J4D3pkAUgAUh4A2JAG9OssuGpbMNAAM7EeipoRnEgjOIIRggw+KszhoOT3+zQQkQKzClOGHQi9VOf1oIQGpzLL59ilM0f92rONABnDUBOlCvhw3NUNToV2EZzBSGXogbNimgcUw61d8NvNQjxob7x6emrVmiZ+8wUiADABnmnjPD3ixaAjQAZ95mprLLZmoWsjB5wFkpqDNjSwbVe8exVJ1JwB61FOpOGfWqNpUmThmq0zceJSD3igDTBEgNnYhAHAPUUgzBIFY+czh1r6FSAPeKO8GxVj2mBLaIZ7ef+LUDkLwdNtzZKADXS+Btwmdsu0p/fh3/AELGccqa+EyJVW+jDA0uDM/CruxPkM12YS7mEmI97RR5LLABqACqTmBG94qKfPqplm3VQBryK3Ovc96eOpYe0NYfmZXNAb848O9OzDcPx23DeD5DWrJqDrDZ9D6NmpOLKQzPo71exmXwHoXqNJ0msNw2DflH4QLn9e4/cNaVmw5LkyZAe6aZS4xhXPse65R3ADDrzPBeVKbyISg57rWPM24fD0LUuP1v1WjO+rosmHI7NZcSn1Vut1c32388/wCBUlYnsc0YRG9FnsBX3D2pMQJviWzDmYedcfc6elImW9ODDYe/yK1gcJbgrbGuFx8sHkeWwmcTokBj+arcAJHeysfc/GysSWjptOpT0bPrMAWj1n7ksl+2JoRH6lGmZ97JqbAojj0XXYppvfQqI59Vk+JL8Z0+8+tOs1KsxvYbPoNcXNaos8FHtx4zbrDGYZ72V1DhXYfCesRzfctgNh4eMea4+88/7SUGCQV6VAz5F/r61j9DR2p6TwWZuE6dUrVjRo8XrPA9631N4nej9bEMINKgxmfnZjLyTJuGdJ/azzVf6y3rH6Hm3t5niXw1rboN0qsQwkdmcYFHuTiFasmlu2/OYhvOmGDJgYLxuzVW0nqlFP3ifQbbC57VnTKob4S4cZoOgGVTyZLcNjkXz6FnDr2HRKUJ6ttvdbhmadxtsqDSqbVYDrj7kMN/f1rL1Wm0pmQbEWpMn9AZqqOe/wCdIJ7/ALxxbbWm0qM5Amg+E7A1ro123BRz5uJXJmfyLnj0lwz1HFYBMcei6YGvR1ig8ZvXcz1bcdfONH857AUi5JNpMtBLs6/Hgmh36y4/Dtiq1Jrm4rYaR+c1Ckh6km6c7Y6CDrVB438RqOBsTrnCZEPvMM1j78rdRuSuc8xUsw7NE1CpUxiY1qMYGoUyYcapZ6Bn9CDrtEo+tQWotSz1T/PgqyvWfBjU039cPz7Fn2eJFZBoG9ON+haN7iFQ5NN5R+DrSPPh1qBkqPDoZzBzVk9dVctubhQ3zhtB2GCqjn60pp/QAMDz2LqsmsWxUre0JeD3kzDoWcWPIVsUqpV67YlvtzjZdN7RMw7F0i5+Al3Q71aty1X/AFrrhnzEjwVySS9LturSHIM49UzzzNTo3E7iEy+D7dzSdUOhbj0FM4FcW6VHjwZR0SfCM8OXjnv/AMC5/wCkPw9g2ZVokWhxIcZl8Azwe7/OoVN9JPjEFNOlP3GGkAZ6uiGZ/nXL7hue47wqh1G4Jbzx/OeeCNXvv0UbPqUnhlN4c3VEAJsJ71hGkNPA8eZ7w3gvUFtz23qa1FrEtlmbF8AwM8F+S/CviFdXDeqNVKh1h5mP3s5rpR8fpfOu1ysyzmEfYbxrO2D33XuFHC6sf6Vtm3njPvZwzXGqx6PFuRs3+HPFifb3/V4k8DA14hurjHetYnyH6BdVSptPf/o8eSeC6d6P3FqpcN+X5q32arHmngZme8PzraLFgzxmuq2KyfDm+wmV6mUiZmbzIYPHh359/WvXFH408EuKlGatz14dKdzzCJUWTA1jOIvB9jiQVH4h2U+ECsQvHDYGEn5DzU06Dat5wgtzibw5jQKqYYc9HZ2fwKxC4x+kU5w6dj02wLOO4cGf9JgZ4fkXhy/6rUuItwnc8Wm1LB8wB4zDMGTXrapcKL/4b8xUbHrEOsU+Kyf3SWG8A+TNZzgbUuHtYtmu8ObqcCm1WryTM8ww3/Wg7nwKk3NaVkUS2L/0QlnG14xgeYYLqv8Adqisyg/Z62Ylv1I408ITOEZ7DfgroJLYIFvSz+CaeebxTWs57txXgEAKQzMYhlqPubMFWnJnG/pt4YedJ6BTYYB61nBm+eAZ96xHhfj3Aua9r/mwabFzhVSeDOtmuf8AELhLL4Y3hEtWdUjkxJQAeto4AC/RCvUqlA01LgwYxx2D15J4dALzJxvmWxxp4g2/YduV8NWED+choNYMz7OxBweg8Lol7XMdsW449zABnmZ7FzKq0Sq0GuS6HVWNGRCeNgwz8i9ocK6lTuDlNuWzrgpQQKtTmTfZqbobJPyLxvW6xLuGuVOuVFzOQ/JMzNFFA9XMnrzm8z7FqqPQX58Nqq8oBtAeZxzPA1S0GzKrcmZsYRo4f0h7oWookOdb1NNiU5rAHeAbFzYIdArFErnFG3uetGzmaPT7XZ516XrABnh2AtV6UR0a/wDgnZV/88DNQOMELlz6zw716A9GPgtar3Cx2uTn5j32lZMHmje2MgvPte4P0CpXvWLOrlOZpXqcNaMAPH95Wi9vKjOoytHArdYo5xK5Q6icCpgeASw34fkUq3uGNz3Jf52PBY0XczPVMNjIB51X1iM/SpEi33/axTwzQes+LVt1KzPRwtoKNWOZduGSEqpSA3m8b2/Z+hR+FfA246C+fGnjTLONQmGdCN653vH5M016P1w/bNix2LunQ5kKkVIIvqx4z1sN+/5F13jldVv8WuMlK4ZAEmTQqXve5Rnv8h/o61XKyeZ+FFpP8SOOp2JQKjy1Cqkw5UlroB6KG88F7weCh23WZFo2rw5pr0egxswaA8DP+DrXmy5Lq4e8EPSbt+XTYnJx+QOK8DId59H+dbi9rwqVnyDrlo1WSFQq/XzDwPPaCjbPboHEivS7nt+2rntGlGzzTxxZkTDez8hrk9t1V+Nc1btGVTpOlK9z5zVR6PfE644HEs6HOqUl6lTTflPNAGYa/nXWrAuG46rUblOlQaaDsWTnnIDM020cctLgnLZ4nS7urLDNNj0h4NGnunv39+a5O9UvtPxauC+LnlGcK2g8EGdnZ2LQXhxjqJ3bXXzPORKPDMNgAH0Lm94V6dMo0huDhDjzcGDZAOsFlsR2btrFKnu1liXhT7o3yWewDT0k9ZrT1FhKlUpwNQqHLYBn1d0YKzhz/WptNtnsPrXNvy86Vlc8bRt+Q/5OhaPhvM+ytcNuJOks0+bDzPf1mqJ6fm16unMazSJ6fyYAcTBk/OZqt+TpdeuHiXOti1znUqc96wqknDN49gLnMyqsViGZ1yoyZkcDA3gM8zP6FnZlSOYevVX88OgA6AVTDqTE/PlfZdhrs+zmOkNYnxJM15iIxg15FlJ9NfyNxjA/kBaCt5vTwbiMHnhhsUeSy5R2jfcwOWYdGfQo32OnGGUjbFOjSeTPXY9rhhmqzBxn2nR3qXGe8IFoGXpgB4fve9CeBtA42mcM5quDo8Q4TRtzmQzDPA0FEZmZYB1rsfou0F+TdtwVx9v/AEdSjMD8hrlgPBDdB9hjxW+g+xd99HXONw+uCuG3/pSeAB+RYez/AE4Xb6X9z2YhtuW8XZhggOM4peQIDPo+c8F+W/5t/W+P+nhy/jxAl1u6rctmlMZyDpWEZrznmq/idwf+w1Nok6nOSZJzQMJ+qfsTXW+LVgVi0uI1C4hucs9T6cywBnhvZPsNZm+ZNzX5UWrZprecuo4Psh0AAf5F+w9D1v7L+U/xv2Y5vbuocNmQ3DwbgseKAYYedbq7eCb8O2aFVbSivSZDEDnalIM9hn2f51HCyaxTbtdtGcGjUAPAD+ddemXyFEss7ZqUoAdAwPRA8z/Wui/7b4kW8vvSWzYaff2SMzAw7wVlJpphb3Pf0g/Yh51o+J1Big+FYqU4PWsrDmY7XRmoUyTTjpcJx/8AooAuRp4ZkGcA8RWdB9Ww5vPVFz2AGbId5mrWHbD9SaBw2DZ+89B7DXc/Rv4S0CvSqr9u6HzlKN7APJGPozM/zoq3n2zLkrNBlVA7fbZOpVsDih3mAH5AUafMnT3YlKnP+FTsPu59YGujVut0exuM1xy7Ot9lmmUjXixgDeGfnz71zmMzOrFbdnOOa0iaZvvH86siG44V8K6lxCkTZxzjpVHpf7ZOwDr+RenqPYd48N+D1s2dw8uqHUqnW6w/rVaCH7MB79nnXny1aq/AsZ61bgrP3KqVLkqU0zJ0eWfz9u95wXtv0V7Sp1pcKIlsHXKVcM2FPffkyIJ5sgZ9gGq4UczhPEv0aZQXva7Fcv8AqVyTbhpsp+pesXuWDMMMPPgzmfQuoX5wNoF23RROKjduAdPtSgnRXqHJZz5blWDMDA+/rBQuIvCW+L24pwq4/c33KvPHT3ogbNGlh14eToBdVt64Q4XWrEse5qlJrFQhA/FgRADOVMYz2Gf+utrgi3jCz/RY/wCF+yJF8cTZfqeFJpr9XpseJGwkyTDozPPyLLM+jxccmK7OqNoyYBfZt+uw5Ec9kwAwzP8AjX6J2ZarDwfaO+IMYDYZMI0HPOLTWPIrPlrHr1NlzqMEM84D9PjS4/QAH5P996jLbb81PR7sn7YXlQgualMybaOThPkH1xs+jMOvAz2Gv0YgU2daVtM2rSmGQahG+EY2gwAAwPDZ+dcv4l1Lh5wHpNqVymvsn6keCiyQiRg5o2H+szAOvoVVTfTk4UVXVYYtWthNM8Ka09hhM+s/c/nUMlxaXDSnWNalj1KLbJvaDNR9ZNSGQzCVKMN5/oXhzjZwcf4RXhNtly6gnu1EAqEMOWPoM/4F7DjelRWabd8i3+JTdKh0o2TlQDpkbMDPD2Gt3rH02+bRk3Q76R/FCpQwhXXTZVIptOCNnohh0fP9fzq3jxjJpr9Erx0YJcOfID+yPZh0edS8KzQeYnH92CV4Lwd+CCsBZcCeB2XBnxmtENY5Gwwf354fIo70/nPAffN4O8DUQtu+FH2YOZUKa/SnqlNqjIRYDQBmZv59HyfWvVtk2rxD0qZ9quHNBptKt7fTYNQrDIM595vYZm92bM15dsOZQ6lb8ijTrVe1aWBsM1OOzhvPoBe2KPe1uWBYtvv8Q+Ffqqu1Q2IRx6czrBJPDrDyHvV9I5z6XVz8SXrfoUFy8ak9SquD7AR6TDBmMb4dHzmGGf6F56qR1Fn/AJR0OIdHpUpkKXPkB74+g2Pzr0rfNeiVviJSrqtiyqqzSqRGfZku1MMDAz6DAPkXnem1KsTzaoz8WNPokWqypr2sGAPGfnXNfN1ouHJ4EaLqyDBzkN+DOsr6AzR6UDvPS41Smyg8HRP2PzrUW9UqNG1Z1YYjHHi5sM0z2wAfnDYtbZN/2xw0h1auTrHCsR7lhnFpvgh92M9m/Po6/wCBc132Lhh7MNj1uDh1FmA6AGbJvBsM/IlxFh1KNcITosED0DA2TCNmBh84K1r3BDipbdw2vbNRoZwPtDMBmm1N3eD3nz+hdT4o8KOGtEn693Xw8FVYpp8/Bp0YwCS/h91w+TNareeodKbmTP5ypxhIfMAwADZBbCj1L7N0uaxEM2Qpcxg2ZABmYGew/wDAuqw/Rm4m2fwtj8SbqnRvWb7JgzTJYHmGfsQA+jWNZ+8IES3rXm0a46G9bDr9bN55mQeZvPsxQAwZ/P8A40HLKlTX2apyrlS5zwdcJGff9C20OfOgVmmSqdcejyoA9gz0BK7zUcKIw9TolNuM+W5oAqEAz3myBnhgarIdY9SVk9SLDkx4TP3nS7w+Rcq7ho3qPd1zhXWIM6TcM6qTGH5IR2TN7Bn6P9hSHo1YemlTbjp0wI9LqTDJy3mcAjA8GwD/AEL076IvCugWxTpfEO2bgmTwr0YAnsvADJw5WfQeC6be3DG1b5pdTp1ZpwZ1A9Y5AdYP4YAa7PHE5rhw30eJ9ApXEZqJBrL0N04xxarBkBgB7wwMD8ma9URpLZ+0XKbe4PxLYu06rSm+ZhSocViYZhgZmDGHX+QDXSMHA92a64XH9tdnGiH0YH9CQM4IKJG5kwB9wGfnNXfIPw5BsGwB/P1qNiq0VIZDb4imnA+RAEbBerV8mm+8BVTwOZGC1oA3nvbVbUozer4YIKEAzJSGQMCRgGBKzZBjBECgPYIax40KR84ImQ8XTWFtK56zJvysW5cfgx4sk+W1dmYAoFTeAP0qg1C46lOjU2QAGDISD2GYedeFbquGs3tINicf3fnNcI4dAGvUHpD8Tn7VrztHbvGMDwPaxx9jwAwfQGBh8i8uVWqzrtuFq4KdF0QN7CTI6NYz6P8AAaoQoFefo95UqJRn8NB5iUZ9gH3ruFE4uwQhzYjFqsyZHrI5UnM9YP0YLglEo9KeYdP1qzGrAZhy7wb8w7Fo6PRKiFIj1i3z5aRVGT9cd4AGfQvR22g8SIl4VGsWdfEuNGpTAc1TT6Ni4PxjOh/aM3KB6tNrYYPU72LzHnXULJsylXO6fN1yNA9V002DNoNmHZmuFHbblVuap0qCx4VOefAPkDNFqy4YH3iJEpobnwzAFteCZ0OHPp79RAwdYeflPGAZnsDDM1gq3JbptSapzbmfJZgfnDPYpFEnsUqZH9cvmzHNk/ZHvPPyKBs+IT1Oqt2+rbRnHJo8qSB6z3WcrvWXk/6LaYcb+c/k7F1KxrVi3Dw0pTlSqp0qbSJM2UbrsPPYeeGZ5rnr1Nua5JESj0qKDztUeMIzQbM8O/6NiChjH44HFc6F1jho9UpNZpVuTqj6tj5hKkyJDIGew9gfOptb4V0Ph7cEecFYjScMACIyB5g/sPPf2LQhW/ttAuCWxBBmRS9B6S70A8AH/AoQ6ldtE+zdv0LTYkvQoVVPA+ZwMwMPItNZ4Snry9cTtk3lsMOwA8i5/ZlHrF/0ahXiw+Zx4Mw9aC9JzAADoAA866Bb1mX+FzevAqv81Pva5tSA3vf6iDpWsYJcy4Ci1WsUqlOxGKlOCMc08IwH3mlzDaLWQT5f4inRp7+YOaioQmNo+fb92g2rNbz71dwKr4Qb1zdme5nsVxDmP+dFOkesnD94j61mYdS6G3Fdsz2zBeQHeWYPYq2fTXwMj2GrA3mw94mZMkDDw16Kpl4GfaKbGntmWnmqKYf3g0bMbWHUzWsC6k6D2epvBY2sU2XzBuRDzDyK4M5bPW4iZZ5k96vacMUz41SazYMMFq+tTZMAw+hVTz2i+r7toiE0wyRhGM0UN5s+tXEYAMFjtcK8GTBNT6aBx9dhzf5FdnDbwWfmG5kYZmpUqc/xEtbo+RNPamaDu00Skc4AAhCZ4uaa0W/618NnANigTnngePNxV8ySwyB6feoL0lwFAnvOGrAPPBmmtZhQZJv71XvG+eatCwemMAahPSQP2aiHrmo56iCQZoAPehBSGYx5g4qDoMmZbFYQ4D+GolGbwNWbL2CsAEB/yKPMjGB6a1cB6Dpb1IegQXvE2Gg5vK/5lBNnNbt6iRTlaYMdaP7ExHvZvmmxzk2XEIMuLpR2AB+wczNVVSsOswN+nmB+Q02MkAYqwjQAkhqKR6kqQe0ivKygQDZHxFGzBqmw22etXAGGGmnYdNYewzzV8zRIuHhgo2MJVaDFeaN9vrBZd6Bh7MF2sLYYkhp6fWszc9q+qi8RsE2OahGM/douTcD3auzgYdCaejPgHQrQpHgcAFXyW8zVq8y/+Gq95k1Y5vfJhGr2/vZBZ3nfIa1HEhlsJ8J/T3GyYGsZJ0wDYsmyQckPePqPz7jIuuOeRRw60pIOG0ahSz4Y0qddsypsTp0k44Mm8Dusez8maiUeS+9DDXWl4FfdqtU2POz/AK6pcNF2Qx5HjXlgMHMlI7dRLNsASUhow8XUWtgHnAaPzgso8eA6i3tq0dyq0aO+w3sBaxabRQToAtAFqv59CnBar/4C6Yc1wy4Mp3RWoZtV/wDAU2NavnYTZDJMw3/w1NZpss+htdAjWrBZHx3AVhGo9GZdBvXDNNvWHjUedh7BSwo878NdLCjxA9hgf0KQdK0Q1DbBQOeBb04w6EAUGWHWC6WzSmzDPBOhSon4aDm/qcz92rONRMA9nvW9ZoLB9DCR0fD3CjYxvqf5EyzSt+noGttyYeRA9G/DbVxYx8mlOMhqNtqseBxdIhxgePTcDYqeq2HO1ddh9nlz867OHmYc3Cw56gIwezJat6yYhjgE7B3yIA4aVgC19cNLzrs7ocfjhs1C/Z/+BSN6kBSn4Y4dadCNgvm8z6BRgD3iTxt57HM0YMp3l21itWGo7Ox1WZs5nsVe8y5qoDnhOBrUBtR4wVGSWAMLW0d5t5oAlIJLLcM9NjDD5Fehn3oz7LWbiiszA9m4tRJ05LBt6azjNBDnD13HsPrVIAclVRxpbzq1YUem+7cM0ycBhksFpHMhy/ivQX5NnyHwDoXHbJnvw7lhaHeeBr1bWKbFqts1OCfvmeteNKbJfh1TTYPB1h7YavuHo6TbdSed15bB4JmsV6m0Gmgw/QwP5wBWdNqVwVujNOMSwDPzp3kzNgwrDbLzvenc8w5rMueDUoEjkabJZkdiC3qrUWYv34NnnW4qtEpXJOtsMYGsvSjzjyIs5gIxsZ9Cx7jBoL/odElffs9nyLP3VxUrlYaMKbUo1NhH77DeuKXVdrlVqkthg8MDwWSkng6eaj6eY63XaxMOMHPVG4PWWfzrmUnlTN1xiI9vPrSjSeWY2PhgmWXnJkjQYDM1zX7PM6PEeEiA83Gka5weZ+Q1sKVVZUaPmxbgb1z+TMnRnTDUwUhm/LtgNBFiVg9LyGCzb4dWt6HaUmQ7UrgqoU13sjmaz9202nHVNS3KkzPjuBns7FlKVdUp6UEuq6Lxh3mC7HaVK4X8QoDrFSr70aq54cuyeAL3uY4cXkzG4Zmw57UE7SteS1rm2ukXJwQptt1sKbOckvA+eDMgzwBaiT6P1xxobstusQ5OHZo4J/kRhwq6qkD0dlhhgAMO8FkTmPh0P71q6wzSnpDuvKeAwPDAAWckhEAfDXrZHDmj6zQmbnnAEtZaiiRp2gfKWq9PAw35hvQZQHt2nqB+RNPa7IahrUQ7MqVSKROptK5PDfpHsWcqsk8uUc6wQV+sZ9fQjAwDoA0TMA3mjfb7OxaO0vs4zK07gM2fIC9wGgpTlQgacGJPzc8+9Z84dSZd09n610iqzJTMV2Xbk4whAGwDPeuYyZj7xajnWvcCzOmv6QOco8f0KEbLjJe00QUQJ84D8Oc8AeQDTrLPMmDb7mAedWLOlVWXTZGuxOeP866rArdpV6lgw/TY0mR53WVy8KbE9xLBSGTOGGDDmCx2OhvWlBhxzfprFK/Ia5vDr0ulVR19+Cy95M01Jk1XP9u/gTLz2/f1psDW68dSnnKCCzGz7GUEOtvwy1AzD58Ej0HvZgmgNgy03OhY7GtmXnEnwN8RnV84KijTwePT55RDBt4NNhvBPM0SXpa+oz+tIHHzmMTI4Snz8VMsvb81XsnmK0oRonqQ3O8F1ClemHzAafepBmcbNhwA3qJovm1rg3sVhGhuTWA02AwBQs0yb+ANgvlS1AaD51OZpsrsbM0dVprj2ANsGGHnUAKPSnDppsSw+hau1bhOmwwpRsMvAx0Z9irIcx/QBh/sTLLIM1I3A7+tB6OonHi6qDRo8pu8WXuV2erwDDZ8i9QcMb/g8SLXCsRPa54GHeC/Pyg1hilPg/KpoTGg6wXQ7M4x1mzKlIqVpAEMJR5yY57wNaxaLh7gMA1dBzo7wWSvnhFY9/wsH6aFNmh7GXH6wNYzhdx1fvl0KbcdOjQ5v9oZPYa6wcnR9mu3w5HGo188RuANSiUO+4nri1DPBmdHDeAf7mu1Ue56Pc9Nj1Whzgkx3wzzBRKqcGvU12j1yCEyO/1ga4BdVHuDgJWWrjtGpTHrfqJ747oZgHRsXo9L6zn9S+6yzNq3bTb5oYVy35ew+sD6wNWDxy2fbrUXwTGIzTst9zAGAzWXtuM/c9ZO+Kww9gweEOOYYAAfQpAVulBsnP5h+CHWadZrcqZFOJRoJmGGGq9sBctwOP8AGysXdG4dyziyzMLhqXKh5GQ+Red+FdpVW1eOtHo1VbwJl7YfnDYvWvFG2HAs+jt6mfK1XXe+QFX3Db1Nh3RU+JRxAedhU3wTPsMFGF+HBPTYu3nKzEs71VG8A+a5vv6OheYDDBrZ3rUcRbtrl83hNuCuP60gzwz+QFX02TEgHz0qmhMw90fQud0LCm1W8ZlGj0ZucfLgeDMcPnXVZlq3baQ0qm3jLhzI9XZ2Rw3mz9ayR3DRp7VPnMMMxnWDzNoA6F1K6rzpXEWr0fTt+Y9CokMNYw3gZ/kXmx1aw/STi8K7AasCsUOZW6qAGFKj07fs8hnhsWM4hXJX6xFolVqMWHQbzm56LIdZsH51x+q3hVbDuqJfFv0pk2sDAwlgq+g35c9+cXbcuO53MwlTOVZw6Az7FY77wNtiWE+ocSZzmDsUPV/KQQzN75815/v+guQ+Jtb9ZQXo33kzwd6wA+hei6ledN4LXbccGW29pPgc2A0Ge8/9zXm+5KrX7nKrXjLp0k+de13jw6PkWNiFAqT8aqBKY2YL0Fwf4wWVwxddrD9OmPSHw8Y44Z4GvPtEpXONamuf51cUSY+yGvSpxsyMzDYubuarrjNDg1vneLFSfmQKhXqrnAjvdGgjr1z1U5UKUcoHpBxgDYfWpb3De+OLTtP1KrA9XwvbPOnvAPo/ItEdn2jYfFC3/sdSnqlHis6L2e8DPzqYtlcO7eh/DocmxahVarFA6hz5gZn1hsVlGr1St6FxDrlKcAJGGYZhmGatrYsOv0prUo0X7vUfGAOxn6Fqrbsamz5FQiS6icyOD2DzTwbDNd0QjD87ZNt3xPlOuP0OSzzp58w8GAGlclYpUliEFKDwoPWHzr176SEZufSDr8SWABRD0AiH7YwwXky5LYgs25CqtKYMOaDMwM88Fzcrb8OZHPcefdluOax4d6jw6q+yYMRHzDPyKwrFHnMsO8rTdHAM3jM9iyTJ7F54HQmZnMtahvgAYYZrKVKGxGqjpvuPPRwPDMz3piGeGezZ/Ao72ZyPEWkDSwDp0mRHp0FzMnzwzz6FYAHJynaVoABxfJ3rH5sRnQ03MDSk1KdDmhLYlnmfnS4GremPsumbBmB/IqmSEuqkbb854CfPrVJJrcuTK19fr6wUlmsN56b8rBr6FcC4vCNatNajt0P7y6DJhJePYbx7N6zMkPCiB5GVOkxnHmAluBg0/wCx+cFEMFoHT0DlAbbfhYI3oDEzDXf0dPoQUo8JoOdeAZo5MCo1J050t/e52IGZOnlp9i9LcOqaFE4QWpTmN51QOd/WvOgQAZitOS3w8TyL0xZlbod1WHb/ACNfjUqbTo3K4SwwDMPIuX3P8T6/8I5o4Obdpxsyqbh6xiyY2fRqsmGf61KprPrWbHgsdZmC3dN4i8TY0flK5Q6bdVK6DN5nW2eTMFSXtdVHqUqoXVbFq+oY9LpRgYYYAb/nXyPW4f735fs/P8Xjr1DTRo38p121C1b4iRozTDLGHLngEwFIrdsQaVxVpTFvwQjHS7ePRMOzfsXmyHfNYCr0y/4MsGZsLrMzw1gQV70hLqqt0S7qfnBTZpwwp8PR3gyC/dcPvcPiMP5h7PF+ea/JV6/2KVKO4JcXn6wxJfzPDe8+sfb1bqNSlS51YiRjkTes8Fn3p+B6en1nrGZ9ZmlGrEuNNDBfI5uZUNnUplKekG/yMaS6bOiesGawlV1zlSD0D9XxXvGMA2B9atqbWJb1ZaltthmcnlTzDPMzU2541Ssy467ZTnLTGpWHMmHR8i5thoLnquObEvCQfydauodYvGqwqhBp1ZqTMd9n740zJwB761knmX3hafiB4OeYKwqVbYpsKnxYOyoMePJP/AsW0HYdHzowAxO5M4pmBtGeB9a+wKUbNBl1U4IHHhHg8ZqNJk0ee06/hhNlH1/Il9+ehyKc3zJx5WGsDXem/LbalmaD2GuxrNZ7wz6wXp3hRxUu2wOGlPY4T3Ay8D8k86C7DOSZvn8/WC4FSqUZyo9KYgmGfQBr1B6JF1WdTYVdtXmwpsiLJ1zCQeGZ7967OG3Hzf3HU7P4tcWOIucGVw5Ow9A9Cq1GRvNkPIyHnW9pTNsWA1zcU6lPqD/XI9tNkmueSeNNuTKzIrlOr4PW5Chnzh8mZhn5/oU2iXtOuSjSJdq8UKVMAzzBmPAzeZDyda78IiHFeP3EjihxLvR6x6HQ69YcJimvzarHqMnRlT2A37wDZh/rrn1q2Tc9EolEvGdekym3HvqcODImHmEUOg/kXpmpXJwdjSIkudXMLjoIHFekVl7xsO8M+gwXDON982dGrku4LRrkCtzaiyAPSA8YGQwAAAA/WuDlXCbWPSKl3hUqE3cFowDqDAAbMiJGAzmTegP8aPiRY13WTb18XbXLVtsHTZA5lRCAYaxmYBgzme/rXJ+FdtzuJ1cpgUqpBR5dBZN46jIDZn1hh/v2JcXa3dVSfdt+ucUZNyU/rPRezAD7Fl1LZq2OJD8apW+wfrJ6FRJJmGrv0QMMDVlJk0a6rgCJTTnwKF477MuWzvZ/1FgjNuBvg9HnXXfRjtWy+KPEE7EvjnD9Yxj5N0JOAMmHX9fYuiEOO1gwOry+Rf1oWf3Yz6zBSJkZ+NQ6Y45BjM64P+KyeZvHs61ur/sm/wDh1Np9KvGw6lyUI34UaW9G696i2e9SqbVArhvwHgi677MGXD1gw+jsXo9BW3e1HpVzUK+HLLrdYt9+24TzzXLGDJnFAAz+cANZr0gfScuPio7TKH9nHqJCp33rSdDA3n+w/oBWfCL0tOQsOXwyrHDWBUo4RpTAGzUuWPB7sAMFN4hWlRuK/Dmyr4bnRoFbq7NROSEgNmhF2e2Doww/jQcE+2dY9udx1sJodEjnz2fkR0SpVXSCsBVXtIJjDDxyHswMz78FVRoDEaRL13GZMfR2Oh0K4tWguVKhgxSojPMPPHhIDYaw5oF1xamQaVc0Kh0flsGAYN7l42AAZ9nzrR+kJwEY4dcS5dsWldUyfHOjsVeZzGzxzz2B+hc0rdHqsyfLbrPP+uzPDRMMzPDyLbViiXNeFUj1W6r0M7lOMDBwamz7YPJn0dGC5sDsXDf0irYvDhlbnDniNSpLM21JkKbSqnBM3jmGBnns/Pgu3eidw3pUat37WZ06TW6eHK0iAzUAzA4vtu/61589HW0n7S4p0ypXU4FEapwGBzo/jMmZ4YBhh/vgvdbM/DxIMpk2fOALrgXB0pgKN6gfbB6nhsCO6GwAz2AvEnpG+jTxYvy/DuqlQXjpVRqp8nBDOS9GB7DN88NgBnvXsj1lO/ERxpL7x+Ii3jfiFwcuCyavZlx3ocZ4zjHT2YjR7ANkM8/4+hYHjTwNuC1bPp9zxZxvTa9P5XlILOx4DDPMDD616W9Kt7N+0mPwHpR/wAut0emwXrXtxyXFZM4UNg2TPrA9EOhTgZHgDYzFpcHbUcfox0qqzaUHrJnvePzvec10UGQMkfWCQLRDQUTQjADa0EO2IM93Uc/wLGMz2AMPEWmolwgBaZmi2iCzKN+AjOiMe7bToVtvH2iA6xFw09RQIEmGxvVDMPB1XcyZt2LOTD8VAJmmXt6EzUTWPNeQBehuZpkwNlWYGDw/Oo7we7VwI+t53FyqsRn590NRLgfh+r6Rm8ZvSd5gfR/nW/uSTFgUl19+Xon0M7PeLz7xmoN3TKbEnNwZOrmD5yMN7Pz/AEL1DjnE6zLLjV6PeNAuOZVYUrXeqTOtm8B9fQqy8Lng3DAp8GJOZjclGAzN3rPBdeu2wINHty0r1qVsVLVCNo1jwTweYMP9z/OuOX5b1HmQwctyXGZdnAbMbRDeyGzYYLnGfupmKzCoVSCJJjVBhl9mYbx59HQqyHJOZCagt1U40I3tCeDXk7MAUum29fBw3fWLZhzQer2dUzNnP5z7FnJNEOlRaUxVfuzz8kzN3r2di3ix2Ct2xVYFOqtSsuWcZqnAxK5fremGZ9B+Rcvs+q1yBen2jqNOMwhG+eB7Az610PhdWJ1pWlcE7lDgU/MA9ZvbzMMOjDv6/wCNcvo9VfOBUzYl+E/vww6DM1otVVt7W/nKoxACRNefeN0OgwM9ivdH1bajTb8EDkTfHjG71gsfUodR8Jw97RyeVB3sM1rarb1R+1QUaI289UM8GczzDBYjZwLwrMy0PsjBghDjsft+bObz2e8DwWPtW7YlNqnrV+qyY3JQzZjGyGeBmuhPUG4Lko3qN+nchWHzi08DaPZm8eHWtHxaZiXm7/Jlw9YhxqVCjAYHHh4G8/F2PB5zzNBYU2+WK3Yx1yXEN6PFjff5Zs6JzDDoAPP3oYfF22aran2Zg2wcaOxgdSee/pLAZ7MPrwWBqVBvSzKza9Zu2nSYdCq7wRQgyz2Hh0AYfnzW74hWBSmZVy1WlVVmfIqLLDOAB7E9mZ/R1/QrHP5l/v14AiUqK9RGtbDCJMPAz869Uejxc8uTS5dvvufcqOZ8zLdk55mfYvKlNokGm2zHrNNM5kij1IDktAyeDxvdH+A1s7Ar0sLAuCDEcksu3DP2cuYZs/74Ih1OG99sL8l1W4/Go9tb4fMbAMz2Z4LqVNuSlV5026VL1sOs8DwULhpwBOj8PmpfPPPVisRgM3pZ7APsXQKDw3o1Eh8pE2HmZn9ahaiCM4fs1ICA/wCRaUKC2HQnQgGCsUTNNfU1lmUz7taFlnPw3FKCm7VArIwGHtFZsmYdDiB6A+CaeCWHQijr0x/PrTzM9sGt6qtZ/vQGb/4asFJkt6ueopEaZgqo9ROgatK2kyQNAzPwJQuxM/kUC4enuPBp6iqpnVqI8TQPA2Yb0QUY3AwPUVnGnuB7xUPQj59sB03EW1YVLMfDNRJh5gqWNVW8ER1UPxEUaeBvrTIBvSOSHnQg8H4ihKUAI9EzUcHlLZk70Ef1O+fYl9mDPsV0zJ86u2XohxdNBjJlsU7SAAbzVO9aTBn4EsADyLpfqqIfQo8mgsAGbascqetI8u81YQ+HrDzWo++ta9GwL2atYENjS8RtBig4bsB7NxNHZjbPYukAETFAZwTUoc0O2HAPoNGFtuLooMxTNH6tin7tajDhQWw7zUgIGHRn+daCZTTZ7FECMZ+7WexVcm5jngjB42VYPMuAohstrQS4Ezxd6vcwea8NY/PAlYU2pOB4ZuKBYVKmtvQj021l3qaAe7W7ZPWYVJPhsZG3goas+yzgrBmYbKiPeCWmopvOGaq0tGzWGwa696y1zz36rKw3nh5E8BmnWWWwNShmvVVSeLTBgw+tI6PUQ9owtUZ4oTezVjGSaO+Y4ON4KIFtnJ7wWuksm8fhqOEZxk97aDh/Gy3n6PDpU5xwN7xs4f7/AELlJs5tL0hx4prEnhydSP2sGSBh/gXncPZItCe2Nez3gmmd/idalPA3mooBoyDUKWXCuY/Gv/lAPwn2TzUuvM8tcFTY/wCsmf61n7VPluINPb/HPBbC/GdG8pvzgBqxRPe309NO92mgeZzwf98CRn0bO9Q9sMn2RrvHAqmhW7FkeHnyszBcJMNYNNehvRU+821ccH8CSB/wKoQ6FAtumsj7BWAUGCfWwhns1WG7mxvaQBW3/eOLbbHB77MU78FP/Z6mnH0OUw+hQvXbnnUiNVXD94ga+yUQOgD/AFqOdpQTd1DggZ/OtBzjZ+8S59sF6b8AjRtHBvQUsHgA/EbUfnmUGsDx6gKBZ843joB0IGWWzd1NNBGZb63HFL2MhqKA6ZgyxsbVe8b7w+wQvVJj8RRzqofiKxEkg4yWomjk5ipD09h7rUQ8DPYqgTYB4PgtHJOI9A0MFlwPFWbMls4+mbilaOEMAkK1CS/paGp2KvZDB3rzVgHQvLswiHGY8ir5MNsz1MFdhhn4ikaNNP2ivZcMU8GBaaIDWrmQ4OGm22qJ6iOb3GFe0YRDBt72aiHD3Kdho+0Ve9J8Xw1kHQZME6EZ/PPBHGkgpoPNmvNg4bMQB8dBUgif0RzrR4Zprls+hNiCzTWAd19Q0pjLB9aUk32S001rAftE2KSqvVlk3otKcwj+deRb5ps6g3VU4ksMHdboXs8Hqc9s5peYvSKogUq8I8uK5slBgtotGHW+EtYYrFlx32P6L1rWz+UmRfA6wXJfRpqr70Cp0Mz8UDzw/WutPMtg662wwYGHeYKFsbcIeBgsjGh60jQcfZAD6wM8DVnfnrisR5tNYbCNIAPBMAXHLJuSdJlOuTnzedzwzNBT3twZrky+nX7Do3MxH9/XhmaOq8KLxCODFZtkAd/6u9ngu8UFlyHFanVV/BpHXrwsOlOhUpdVMGvkRth5srb1z2rRjYlUemnBwMN8beuZRnnAXrXijRLEvyyjrNGf5kGDzPZ2Ly+zSoMmVylOlsh5ANY4RgqaGcgG5cH88gNi3B2G/WIGFOl0HVPyPb1j69VajJzg1VzMFE5mXSqaDlKYNnz5q8LVs+NUqDPdizgwdA1H9dzocgJUR/ReDvAFBekzqk+b8ucZn85qK8eB6axuEW0Um/LqnmGvdU8/zqzPiFf4U7kftVP5fyZrHwwY94tbbcmjPEbE5zBXCGf5hxNHvWgqVHYkzT9Xbw86jz7MrMCEdScbPSDrzXq2aB6pQ3ebYDeHQuxcN/SEft5o4t1W/rNbNF6P1h1rlkbQMPHcV3DqUGGxhyLL31oOw1u+aBxIiyKVbkpmmuv9fMbDzXDrhs9+2JANvzuZz6DzUWY9EB83+U61BmTH58jUM0GutiG+9F1IrGtvVZXqa/GmnrhgfeCCiVKVTSzYl4fImqlWJUmonm5sViE9Uj9m24YB9Cgmzrf0oFfQ5lKgMGfrIwPvDRVZz9NN3XcbA/yKBHegNstA/rggCZ+/AFHkzG3j8PoUfP4KosTntf8AtZ/kNLnNoN5mf1qJzLYe0VgdN8DXbUYARqk4Z6Zp45jGfiYAoIM70ngDvbTAuIwMdksE0cNs3c9RVQPYJ3n3wUYEt7wfZpBr4+ItBYdEYueQbEtzD51cXJbcWiasQH4x4d+aYHmyNAnBIDwFIqsnDOJmtWzTfVVG9a1lzlpp9DPesvUmYLzXNA2efea3DNNe8A4PnV9AptYpuDHq54wPvBU8ClOSYpymOxdlZh+sqNFfg+M8AYYIMFzXJ+8wSOq8z35rpEO3rfjRwfu6JDjZ+TepHEKAxDt+J6mt+HGp+YAEiQG8/oXIty89TS19NkFCeNvLNTZPKA7gbgH9CZOAw8eba0wFGNv2masI0nW9m4GHnBRQo74D2frV1TaPg1m4YAua/Ij/AM4h+wy5IfnXsD0fr2qtyWvytxvhzEXPDAMNi8vxgpwGHf8AQtXZ9VfCeFOgzno3kAO9Xw87muHssw94gk02m1WAdOqsFmTHf7DXItauWrhOqTcnDyOmuu02twfUwVLYcfzmC+1Fsnny+bVuf0e7hj3PZRm9b8rDWjgeYH9a7nY0+jcRaMFViVye8Hvo7J4ABqkqvEJiqzTtW37cC4XZXgnq7GQNed63Zl8cIrmj/aM5lNo9UP7ydMk7MPIugeqKlW7Dtgz56sQwd/BA9Z79ChBxatUMPU1OrdVa88eAeAKwtWg2AFGj1W1acy8EoM+Ye3mtBGew92z+gFAzsDiRw5vPO3H5TwOv9cSWybJmvN3pb8VJdNnu8K7ZPCEGBvSO8w2bF1j0k72sC27f5G8bRqUyQ+GcCoxAAAZf2d68HzDOqzznPvmZn3vHmawt0eEc2cAzc70b0PCHrmpbzJvNR22Ow81cM0GXclzUy2IP9Ke3rju3Q6F6HnDSDxL4jTqdWImdKYh/eT7wz6F3i5+CF98BBluWjo3Vb817mjiSNhs/IChcN43DzgbxsiQabKjU2O/Rw5wze2Z/P+tdz42SardVuSKHQ3wZjnGOU9O8gBv2Ly3PT8/+JdSnXnXpcuW2zTY/sAiAfkVratBlw7NCcwH3inTwfZNWFn0SlT+G1x3HL+8zaCZnh2GpUC54LLVJuB8M6VKPlZjQf5Fyw6Hol6Zat1VmmS2H4D1VfpocsZmGbJ+dedLzCVQbhqtsnXAqWb2u8bLIAAH5EfpAnLt666fcFDnHDmmyAM6XXh2b1qqlwNqMm1YVScr5nUK3gewMzeP61s5MPP8AUp4MvnEigYNB5DU62JPjhqN+EHeupX/Ydv299n7A1w52a998dDeZrH3hbEGybhl25BN56Ox1m91msLh0w1FvVhh66mrVB8I3Os+Cee/P/Ou51Ww/uFMg0bWCQx4LwAfWC509b32h4RQqzbNvxocgOuRI9thmui0Hi7bMCy4TFRrGjcEVnQM8M1twsbXs+87j4dDTwY1nuS62njMzwDz/AK0g432/Go1YYiTjk1CrvG+DUcM+W+Q1x0OKN1T6DLnOQTkuv5sHOMNgKher06xrFGuMcsdQmveD37FtYncS7/nVivA5r+KEYA5c/wDOuf1ivVKTFagvgyzHA9gACpqlW357vPa+bp9aYk1jWaabw6F83l/uOmLO3DJgzKQdKlOYOmuaTKPyDXKg30d6u69Mzmnv6FVBPYez57WPyaK6eMtEOTy0cNMDwzRHPzhaDbmxzrTDzzGroOeyUE3gZa02+tdKBvHmSTLznMC+5vNR/wC8UsNPSNvzqQyZ9em3vNSIwORnWpTHtQ3701gGr4aRzMC0+xWJvLTnnTqT7gH58EYaZlh2I4DJyYTviezUQMAa1DcQaOmxnKJUok5hw8MMHsA7FHmPMHUXT6wNFAn0eTANxx82XQDBnyKt1sGtPrPzoNFTbbfrE+FTorGtKYe1wj9hr0LwxtiLUmg9Y01kMDMDZANgGvPVn1iIzVDYnHJZ1wwA4/XmvVfC5nRpsT54xvn9a8whzfj3c79jXNEg2jVXqU6cbWe5czDPf3rn9V403xclBdpVVrgfWEYM3lH49z/XHFWqt6eGgAMdfkBc8h5syFHW2+nyuAkzo1NiMS+w8wUdmZE5gHJzeYeRKpBVc9ecxJBow8E3u9SaJbdZrE1piDB1pDgGYM59eCxwxu+wVNegzJ5sPxOZM/Y78MDV2zGpvMSGz2YMmrWyaJbNvUau3PdVN5nlXtAIhhme9Z9l6IbEuo+so28DPRM94LcgjhwQadYiue2PNNPMsQPEflgcg+s+9EzMtyfTZrhsPMznz+5gfkDvUWq02XGNrmmMM+j5wWeAnpPMxeUYc6PY/Io9V+4Hm/gbph2H1pk3okYv2R7V7HQDYpFVNh6EEHXB503usOxLhqu6xGYt6l09gGAOoTQz39ikTJ8F6jUd9hvCoGeD0fvDDoVJWw5maDmZnoBhvPNS6DUooNHEfiRtUAzZDDrBY3AuJMZ+tzAfg5s1Vj9mDPZmeHeqSNb1z87UIvqb+cIpmeetsM105lmJcLUKLSuWpUR8MJPnA1dzLe0apU35VDCBH5AGNEz/AGww3meCuEwy1Sr1wQ7SOzrgrMY3ZRgxm1vejMd7B/X/AJFOk8Rb/s9r1MxcEaHgAGyEeAAPGB/P+RAdYpVNsgKzOpUOqzZVSOLy7x4GyH51h6xWH69ctPqTEGSzHi9YOvAeHnW3Y2w7XZlHoF5047qrkXmY8UOakszj2A+B/wAa5TxdoNjvX/IrNs1WNTafKADCPH6NfvAA7Fd23R6/cMpqjsRZkOFVDMwlmGAPAG/v61irkmUo/UUFuXrPQgfN6RhvM8+hIceEj1rWGbfdo1NqRvUxgM5LUTsDzmr2wKJZd22/W2JcGSzVWIGEYAeMAOUZ4Aa1HAq2LDvMLgptVueHTedei8yBvYAELPN7A/nWH4o0e0rJublLHuqBWIms+ZvU548ADPYBrBbBSZNSo9SkUqoxA5iKeDwH2GuscAaDZVyXaZ3HfB2fUKcedKqMRnfnvz71zeHbblw0247qnTj1ac8xsw3vZ/OvXXoE8OoMZ2sXVXJdBmM1eBysOIbwHKwz8bYf5FvC1Pedq3/xssu4LxuPjFPmNWG8ABHdZzA4p9b2zv2b/oXmUIeEg234kzS/oxgGAH/qL9ILG4YwbP4VcSKHccTRhVSZKN6P38rgeAAYecF4xn8K+JMm37wnMNvPW5ZuEWScjZohgBgH1hmC0Qysng/xNZ4afyvhav8AyaAzZ9YsmGwwPA9nkz711X0bOP0uiO0/htc3qp6lTQfp7MjPxgN7P+DNe4+FFEpV+ei/T7Hvihs02PW6UcKZHAMD8mf1r8xOMfCu3+FHE65bLtGqnUqfb2GtIe6wP50HTbztij0TilR+EnqcOXi0owzDMOZfe35nn9Cw9Y4dVW2Lo9VUOmzJLTEYMJYHmyefWs+9xOuavSoT9wVWSbrEbkueAPvRseQzVnPuSsRrShNy7jPm4RgxG5c8NnkXLY0tnxgjXXT7mqs7WDeyy7nng+YYAsVJ+0wXC7Rq42EnCYYHOMNmfnVnbc9gKjEptK3nV3orB628AMz6wXWK3atv2HIuV+q0OpHUKDTcD557Y9Nf98H+/eowMTSo06lQHQiXpMeamvA9JjgezMDzBe6+HvHiwL5GJRgl6Ny4AEyJo7zNeD2ZnrKz5bdOi6046kc3BlnA+VZAN5/rWr9G+NxDe4u0SLRqHJeM3vWh89sDleg3v417C36ChM1h8PP6DDeCkA8nXtA/EbbUXNvJd3hDhPpRPZ3HaTHkgG//ABru1KPCg0z/AOgi/wCAF559JN7PiHbjHkpR/wCM132A9/NNPDyQ2A/gV4WtuZwFRHqkAKPm4YKOcbNQxSwqWZqXGquHvMFnDZcBGBvh7xF7dCgXDm0G9WAVVt72ji5uzMNlWDNYc86xuF7dA9ZBh4jiqpkxsy2GqL1q3j1pnnM1GFLsHsyTvLZqqjPb1ZhM2rQAYOMuoDePJPGesKgyfBE33OgAzNA8bLb3tFibhpVzPDU7gqJgHJPa8PCTsMPIaj0fjHZVSdkAFcDBjDAz2Zqqv+86He0CXw2bqTwHVI2caoQd/LPhvBGTL3PeHEKvXLTKVEgshTGHgZkx2pOsDx+f8mCs+IXCvhzMnuzp1v6NYYMw/m4NEzDvM/OuH+jredR4dVTUuqqm8D9Vi0/RdeDOMYGet/jDevYVSoNHrzvN562f9IjmoHnXg5bdqzKHdtnV+6gqQVcMGWjAwMDDPA/74Fwe6qJczMWXTpUHlrop2yTEkHmBhhgBsr3hDsy3KPUuRi2/GOI+HjMuhmbx+fNeZ/S9pX8nt0cP7ntidJekRXjDl5HQDAYPaHz571UDilVrfJ8NKVbjByXvWhhzgdZsgzsNWF1cJalatGj19xjRhTTYMIMd7MHmNhn/AAK1qvFGwLwvo34vMxqfNZMz5hnDA/J/B1qwuTiXVa9RqI5cD8aGzCpr4By7PjeTPDNWtw26qwx9mqfbDHRSKkbzL3eefQu1Urh79j+BlwcVKzXwmVOayxFZZPYYYPB0fOuX+qoM/iDZVKfb8WVJihJPzhnsD9C6hxLqr9pVIKHLqskIVLe0NY2c4zIPH3/kzWQ23BzhvUqxcDXFu6p0aHTDgBXQgnv0Q3gyBn9AGfyZgohvUq7eJs2ucJLgONSczlXJWZDPgwwPfsM+vPes/SrAu66qNEYsuXUmY9XzCYyZ5hGi54A9+f8AyKkti0n7SrlwUNypc/QoUAJtSnMngB+TZ3n8iDfT5n8p0g6Vw1inP5WMb1Sq1RmZngHvvkWZ9av0GpAw3EqusxMPWlyGcAM/kXU+APEu1Q4d3LatHgnzYQA5mo1FkGQzPZnmC3dmWfwnmQDvH7Yw7kqtIzzeA9mf0GrHN6bc9Dsy3prDlsPBm8e8zzekmuQ3IzUnrhP1/LgUeRUT140eIyZ5gfQ9869bVig1K56zblHgxYzNYhPPypkgOiGGHf8Ar/gUqlejlY/2miXjKqtbqVYpzx5uzntj2zDo7FqOJPWfxwZ0n2OKlSOhPvMA88AGybwYB2fwb17ApU/Wo0I3PamzvNUlevO2bVI6BzcaG6HXrBsD6zXDQuHi/PvCq3HapvV6gmAAE6X4MYAz7AD/AH6FkPSDL2akAzvWXh3bbB1k7ci1yNMqDAZnHaPM1IolVrP3v1q3h43g/QtRqADAlaxm8wWaCpOG71q1ZmOAPhrIWuiBqFMBvPTcSZk796OTpmepqKFKd5lsEvDPocUiSy4aqXgNlWkpIBmahHpp03sx3qIfWiEtk1d03TN3/wDcs4pcafyyDQSQzzbVU9Afw09PekzWAMt6dOqh+IgzUmHLAzbcbUEwcDrV3PqTBmqeTJzVrQpIOH34IAN9n35mjM0zn8EQkBMc944pcaY32Gqo3m0zrN57HEGo1sw2JRpgAXiGqBmZgXtDT/M5l7RQtqGZ7GPWrCNUtyysbU2K6grPA1EOpKW9Uv8AjFUgGAAlzIedeqKS8BnqaiR1hhketVsyYxhhmql481aV99p2wUQ7h3dazhyWEg3+IrGjZrzgHqaisI1w+L1rKApALEbn1lrNeI4mefih3rOMyZWHhmo8kJzwGqgWFVrDBteG4moc9uS14az/AKqnH1qwhsuQwwcWgmvBvSZzAlHOTmh5r4KEL2NUjAU1MnuGWeoqfnMEBz80Dsk/ef8AAo4PJo3s01vNBZMvMZp4zbPNxtUJvOM7DT0OfgagG88+BpnnHlYPackdMFWTIzjIaiB4KlgpHPsPB4izjxue0UfmX0C4u4SeFtbBv3AAf8a8yh0L0VdrzkyyK7Bc74ZrzozvYA1YakmxGDXlbATTJsTB5pjo7M1YGAPeG43sUTAGRNtgAD6FC1Iy9y10Umd/1kAXRuKgYXbCf88bD+Nc3rH3MWp2nnyrwPYLoXEKYxPgUKuN78wMM/P0IvwqvDMPyJnNv2aY5xz3iicyZyD3o3WBng0u8ehVM/5R3RRnG+uNrZrz1Mk4NLrHonV71PxdOI454VRhmBoyevZkADA23A2KsBmmw/DbggHzqdMqrZun4ir5Mlt4NiqHPhlL5psuG6FRpUQ3gPrw7FmodefDY+5ga6rTarotYS3FhLztLPVrFGY8Lyd67+FzcqtOvP8Au31YUqe/M9oag0GNTaxTeVfbzmrUQLbbhsaYLXm/DE0aDxw6FZnTXASZjedcO3Z4R4b04zwBTZgVEGOjYpAMtgewFINlx5rTBZDHnMfywUulQ5VSdw3h86dCiOM1IHHw8JbaMDGkHhqxT/YZ8xFz1qH50D1tvwxPUMD+hXBvPslppPTPCMMFAw8+YcZ3TRwKw2ftDUe4QPPw1mTkuMktR0ONPYNWDMls/ZrBRqq2HQasma24Ci4W00l7Yq8JOBbzwVS9W3PxFXvVjMjXo1ZngOfPB+tV517lvfrNPVIz94ojxvvKhd1KsBJEzYcVYExw/eKneeMPZprnHw94g1EaY5+IrWHJfNYrnwhtHKfcwAOtcsuH0t4tEnnBtK3wn6Gw5EgzAM/kUj02GupUYzw8RwD+heCr29J/i3drXKRKlDpUc+yDsP8AWuWRqlc8N0H2Lqng6H/XDQfp9VQ1msFn5J6Ow14ktX0qOMVnzQcqtYercEP6PLPr/OvUvCLjHbHGCO7ysF6NUGAzNozz/wB+hBfcs2E/XDNc09JCiOTKNEr7HXFMAXXXo2BLEcXYD9VsaoA37gM08Dk3o2V6JR+ILTdSc8KUGC9ZzwpvvG1+f9q1j1VUaZUTPAGHt69twKxEr1LiVKLLZNp9nsNdfKhnLzjMMwJFSisAybHkBeIp9e5nigD8TZnM6A6F7qu2Gb1Bm4eywwM/IvG52jSrVu1qq199mTT5T3WbK5h0i/8Ai7dtBtwNC2YcbfhqgZrg4XDOr0/785m6fer7jNdUSZMaiWxWOZpRgB+dZSydA69Hly3DBoPIGayXt2uq3nTbe4fO0qlMZu6JhpPAvPMypSpMg324oM5+RdprFVs43deozj0vk2LM1Wfw9hx3ZdKfkm6fyZgvLHKpkmcHjm2Z/Qo71YnY6eua3teoLcOL6yclRno59kdYKfGim7qRdYAPsNNiDzLh7EPQe9tEYYdfQmnuRwwYN7PvzWiEvnMOhTQuqohFCCAMgAd+G9Z/W36adUCcFVnMu67Ep4HT8hqR61rMkdN+c8YeQzVOBnmnQe87iNkvWwD2m9SAmBgB9HnVeclgI/h7zUcJOzxFCLXByWD94BqOZuAXtw/Oq/nA/qUiM8Ek8D3ohIZmZnp5gnZJ4fPmq96A2G8DNNazge8NeQ2SsNY0MmGbIah4KPrGHQpDISp/s8Pzmt0ImBn0KaFNnG1qaakM0rlpDXNGGBn2LcRvVzMUGwfPP5AXLd9YwVNB/mgciNgboedbCZMnVWBoHBjM/OAKQ9JbD2bDIfkSZeza0zXJ9Yx7MB/V03MAUepQH2S1MNb6FuKk9BOFpxGA1VSPPchDOdpg8C1+kZLNwOtg/wBCPW+Q/wBCuo18vn+yMM/nBVUl5s3TfPrM1tF7Ch1KdALUiPmyaOTVZb2eo/mZqEYGfQGYJ1mHLP8AoprqEiNSm72s+XdtccknIAM2WY/QyqyzLbuCpUuXUYnLaTH4wda9aw7w4bVK1JYUOmgbOj+zhDw/jXBaVRKxSrPrE5hvRaPPBrvBa9I5PGefzeDZsP3PQiB6U94bk6SAeQDwVhR6DLeilL5WTgZ9gKteZ0ZToaDwH84LIa3h6DkOa7VX3wk8qGYNSzzzNdTmcS2Kr9xfoDMYD75AbFwbWfBoGwPBX0apNzx/nGuSc14Lu5LzlBUncKVSnmuwwDrVDSoBzAzV2zUotNa0Awe+sFECpNgfhtrm5lpBm2z91fbD8iMzYPDTcw2KFzIPEgkyQZHUcXzcCwZNsDV3aVSYpVxxKrL9kxnmsM9W2w8Ntt43j6MATXP1iS1oGxoimB6WufipSpkf79XOfdf9jEjsrW8AbAuO82Arl23idNpmeDNEZPef1/wLy5ZlViWrXGqrymtI87p5r0XYBsRqyFS13g5p7YYHgunh5HO9dwLbodKi6FHpUZlr6Fn7t4aW/flDkUCvxdj/AEOh1galQ5Nx5tRHHwSuSg3NW4fq5yszIDR9ZxDwP9a+x63ss7h5aoN/t+jNddT4c8SZ0lmlYa8A8M9n0fOqy7fThtyBMBi2LOk1WP8Aiyz0c/yLd8b/AEUaA9aUi6qBLrEyuwvGeelzNY3g+hcXvC4eGV28HZZ1Xlo11wvACJ0Hnmuze3Phn+P3pM/ywUGn25SqNJpsIPGkg69nv2bP4FxQJOAqK9sPTQZ4Lmvmd0QuGZ7gNBgeBmeC9IWZw3/k0pbvEasVXnJvLZ5h0f7715igMv1J0IkVg3j8gAtEd4XjJpzVuViuPSaOB74meGYeTNc0frXbR3JMuD1DUL4uCnSQdrcnY9hsA/IvSFn8ZmKl6Mk2dKf9WyIoHT3jzN7MDXmTi7xRYvCm0y3KNRzptKpbIMgBvZ5rpc/hjfFE9HXQlzoARJTzDxxADB4/rV4criVNn3G9FqdpW5WZj0KaZsmAHgDwfOrW0vWtYn2/w8lGEOI/P34BvBeheBXCv+ToZVVqNOjTNeNvdZPMwzDow/36FfWrwxg3PxNjxaVSmYATgzkyDjZyWQS4abL0k7So0PhjHqIRPvEKSwyy7nvAFoLPvCcfD6j8jSgnu0uB7Yz2Aa6Bf/BOnXPw2m2JSqk896reOaEiceZ5h5z/ADrGcIrSo7PCB2m2rvqf7KbPfnn2LG4IhxT7H1KqyJfEO5gCZUPbgAHgGaG2LMbhzTvG7aqyBmefKYZmvRDPo61k7PkTrnuc4YMBnpNBhvXnyt2ZXDhS335cZ6nsHhrZ73lF26YZx68KwYmx6xnxqVmZ8ofQs/MqsuTFdguOfdX+t4OtaCfGocOL6qcg/eA+dZWsMhAadCI390DDA/n71EWi4bKw7hpQQJtnPtvPA+Bmbzp/5FiqVXnwrcd98JMmFCzZNnrBVlNmTmZUidBlaLoM9a11N4RVwKC7cdfqXq2Eebxh3/nW13teGZmPQTlOvwWzZiGewPIoUx5gAPT6wBS2axTWYrWnBBkADefXmqeZMivE6+G8DPYuPC4ZF43MzPfmfXmo+Rq7CGDzvZmZ7M069ROTddbfD6F0whmngfMQ2bzPBR83GXcDb3guh0SBTYcd1ww+8YZgfkWXqVNY0AnbMz8gLQV4Rs4fPOdGaZA96aAN2CMNMHfEbzVqAfWgMDTp4Gfh9CHYiUuNMfjeGDmw+sED289RSIbLckwYcSns6LQfXggCAfhOt/IjZDCL4jni+RGzDlaHhxXvrMFZQIGEV2XOlss4dAGfWg3cCfBqVXiU2C5m1y3sl6asZn7lEc8gYLyxw3o8564YlSbY+79Ga9S4chaVQ+SG/wD4EZPHV51J+pX1Vqy//Sph4H5wUGms51SO+57Jg8zBKqnOOPTObl60cGTwZw6MzU6ByMa13ZWGDz8nr86PFwdyetbSm2zVd/KyTlMu59/0KsjT5TLQS9cwCKzgej1msu9PfAz03OtaulTHKJFNicwB+t42AGfWH0LluGyhmSYsnDlcw0+/PeaOlMy/WUfXghMBs8+XPoe+RKg29OuGs+pqaYAfXm6eAAC6LZnD2jnZVYvyuXBGjHS3n4TMd0MAAw6MDVwKe7bnC4asDnq7kGWAwBrvUUzpwSonrJ/RjsHmCzk83JNvU+dpvBOlGZnIPoMA8ikT4c6NFp86quYNTWfB8+zrWiGgmHTWYrT8GnHAN8zDlw2BJDzqs58HpRmxomZhgHg4YGjks1KSwbbHOPGwGeq92KiZCWyLxtnh9C8saIP5tpcsKqbJyJR5hgfsVWss1ll0Kq+wHj9BhszUiMDFYpoUZwPvvQzI7zNWs9l+ScSjv0rewARTNlRZDd8N6bTTtWq3PWYIVL1cYGceOfWfz5n2LM1u4bgti8jrlVpTPO9gPHmBsKQdpQYbR0pusyeY/B7FpaaD/wBnqJUZeASOfOnnIMw1jM+jYuYhzq4axSpkx2qsN6OuAGYfOoXrWJAlRH/2nZnJZ6Ni7RQbJtWvWRfZ1WlAyFE13zqPeB9gfn8i4FTYfrKqRKbqYc09o5+RbRbbb2l6OVeuC9uDtVtg2DekUiScKA9UdmbBhmG/sXj25KU/Qa5UKHK/0hTpJsPeTNdIsC87xmcX6fb9NviGyZzwCGbz2jT3jANgGtBf/o/cW3rmuO8bqtxl4zknNeCDJDReYw34H+TyLsr/AIc1uH0qpToGbmeB9h/Oum03iixDpxsN0OmvesYHq+frM5nJ+f5FZ8S/RvvGxrSkXVyISaUxDYqByM98bW9yfn+tYelaD01qDO1qUy/oaOzeB596xsh6C4aejf6+sa8Dr87CXUTY5NmIe9kwADwPz55rvtmej9wrt5qmVJijnGqsLx+YiSTZMH+/zrTUGM+9DCKDZmcVlhk5Bh7bZ1qzrFYt+z7fnXNdVVZgU+nBm88a9SzPGz1PW7flOP3PyccAM4dON7AJ8oD6/n6+hUtVsxilWRdfDW5zBmlX4yE1mdH2PRjBgDez+jAN68qceOM1K4nXuEvhy3MCnsQDp8Y5YYHn1mYAvRHpOVK8b/4GG/w5qUMLap1NYm1iog9nzIAGHKh+featrFqzgD6UtOtjhBW2OJNxnJqFEelQqO6EbMJIAGwM/OvEk+TWYHrCJVc+Yqj2tJN48zeBasIF1Q+FDtuXBaRxqU/VWJrNTDrZfMA/xgt36MfDSj8XeI0SozpVNZ9SSTCSzLk4HPfwzDBFvP8Anhgx8i6Bwoep0CbUJ1cpsOfT9EAeCRJNnD5w8/f+td79Jb0YKbZ9pV3iNz0k6qczXkuxzwp8ADwAGADvz2fQvOPD2w53Eu4aPY8GcEN2sSeVCQYZgB/Qg9ncEPR+tifXKZxRbqNKqtPi/eocen9HNdmzswXUuNlpUa87UdnV+KB+qGX3mQw/aXzwAM1I4MwKrD4bUpi46UEOuxQ9XzzBkA5nR2Aez5FuAZB7DX34GBrwci9GD0abVgXVdFVuegMgdOpUKinB6wkvvMA888fz57Ni9BxuGNpUes/aCh0pmBIYhsU+Ny/uYodgIqI9yDuuxsN/23zrQesO9Y3AyVVZMHTzVZouGWGC1sk6dPHx3N6hSY1NZDYunwxeXfSTh8nxaoUXPPCggf8AG8vQUONhAif/AEwf4F5/9I17meM1M+SgsB/G8vRxm2zHiB5Iwf4Fa0QIz4diB4DZDxG1YMyW/eKPcNK9dxQbYcwdA8w3pDJVG8mgVO8dVhnhOiPMh5z6FKjTzeEAbcXZ0se1Z4bEHQaWbgAo5vb1wXDoTdZz+pSWXjMlWZ/BNHJfZPwzNStpWTwVrD3rjHELjravC7km7jckyZFR9jHiBmeHzrglw+mrf70qsMUalQI1KfDRgMugYPRg8+Ydahq9gV7i7wotWb6url/0qHNzwNo3g2JVK/7EqRO0anXrR5Mh8MAAJIb/AKF+XNSrDdVmu1Kc5nIfPN4/OahHPgrDuHsW9qDw5sCV/OrejIlZvM8xvih2Bn5Fye4eME6lVkIgVw5gQngeCREPY8a4vJuqdJwbqT5yWvnPNRXnufPU6E7hqK3efr6e7OnOBm88b54B5zWqsPj3f9kzIQUa7qkFPin+z+22Ljs8H8wNgEDMl8HQbfbwzWI9l0fjrXOLt3u24xOmRoJhrQzA8HgMOvNU/E6sOUqZR5dVqsmsVCkHgYTg8GSZ+T515so8mcEpo4M44chg82XQ6wNd9h8ZqPaVtU+nXNbkOvXAYa8yc9vAA7DSLSws82LYuqXVZ1NZ9YVQ9aNgzshv+TA+tM3zdTl2j6yqVvsxpoYAb0QP1gra/OKlOv8AakNnQ4cZ042Gs1nmBhhhh+hR7MmUa6re+ytZB46hCAzZl62GZnnsNabGBrck6PVqY+2/m6Ac0B+Q1YXVdtSr1sxKa+5nHfefe5s+uS/5zVKdElybriQZ049IDBjWPfhgo4HOr1ZaAPZBJ0AAOgA+Ra7HpWwK9aoWHVaVLqMmfN2SnjMzAzAGMA/J17FzyTWLqo9pXBBpVHPl6vPCFzBhv0A7FbBZlfo9wUyDTYPLNVsGIr0fPPWP/IrXjfQZ3DqGEF+pbzgesGd+eG/DD+NejlQV6XR6cdKobZ8vOwemAYYZmGeC9EcMbzpX2Fqbb7EalRwDZgGD0x89+DP0LzfbEB+7bhhRX3A0mz6DPrPyLsXAfhvWa87T7tqtKjQ6TS5+cbM9hmHWf+MFY9F8JWajbFOj8QrncqT1zVd4NaD0YYZ4B/GpD3FcIFBkQYLEz1xVJhvnIM8wZDP/AHBV9w8RbOrchpv14dNmmGEN49h/k+tYKq29OjSI9RpRw3nX8+ZjhvMAWrFumbhty87jqdSqU5kD8fBk+hk8MP8AIaUOiXj9lTpVuVyiBSmHjN45EMzPBZet29xCOEH2VoEOTHNkDPPDYZ9i6dYECo0ShhTqq+Zuh+MeZqML2zlB4b02lXb65oc56HNimBzDz/bD7wAOwP8AXXW881XgDeeonQk+7VrWAPKazPMO9URvYElzJn3qMG2tjVIOjUTx1Lb7RYrmHvOnwk7A3qMIaY6r86gyarB94/vVaczP3igzAiSWjNzr+tbYEg6qGrpp71lH/EWfMGw94meZ+dXhG2p59jS9oo3MtmqIJLilxjzFZLXGeBInnvCPTVfn86XiefYoWizJh5qCcxSJjLhqqeA8lrhAzmfvFHOZh7xQXs89ijm8YIhYc5v9oj5nAfaKnA83U69rgfyItcBMz94nmZ+j1uLOBPcBOsz3DFBs4dVYw1NdWcC4YpnoNvrngPOY+GkDz7J5g5vRDrfrtjD2irKlW/CPQc3rFBW38MHDTL1Sc86YF2FVqO9xxzYClvVUzgHoHv8AIsidSPSNvUUIKq4Hs3EwNnTXjktG457VWHMsRgwcXOXqxLD2b6aOsSj63zTBtuJ9wtgQNsGpEa4doNuGudHPcz9opDM/50wvbrUOtsGGeopPrhv/AH/4Fy6NXsA09RWDNbz9o4owuLdD9ZMGmjezWRCsBn1p47hYD2j6YQvXjUJ6SYezcVDMudjDBs1VfaHzmmBpXqk4HW4mvXHzrLya3mJuNmoXrX51sNwFV39alBUvnWCCq/OpbNV+dBtef1tmmChPagO7FUxp+anBJzBY4FxAnuAXiGrXWB4d6y4Pb1NCYpEt6Gwe9V70Bx72YJ3mT86MJjgIIh23LmRZbHRrxnw/gXlcOjD/AH617Io8wzkaea8fz3oMas1CCb/isSTAw/Oi4LsTRgpB6aaN5j8RQpmbkDCA6t3dX3yxbflt9gACxlwmxJhGAOK4oj1VmcNNSpVIJLUV7BkA7AViLon7zBADO7UUcKkx781K5kDDwIMww84MmpEnYrjg/VX6VxfoT7Hv3tH9azxxq5Ja04tv1J4D7wjGr7h7w9vuZeFPqXqOZTWoTwPHIkM4AALfDJ7TeBwHXdTzqIZ4qIdbced1NRCc9vHxFGBKZ68FcQAbw01nYc9jmA8RXDM9vsNXkWRwKayepEiABn1mh6FHZqTGW9xTWXmDFZAADWUgKb500ZtsjqAjjVtj365xI5MPIpoQ8BTXrKKbWu2aAKwwtVnTZY94Cj+J7hxDMnsG1sVJJquiWxzBVFoXDzxshqS+hV8mqsAOo2qSTchm0YOGqd6q5+zNeidUpPMgax9S1AM1ehM1vaKFVWdYTNtVAz4T8D8Q1NCquB7NxVRxjMkVSeo9t006rcFVZhtAHvu9bMl3z773WjDevLl4ek5dRzXYtvwYEOP5w60rY9KW+I2yqxYEwA6M2cDP86yavWEalPycNMFbBb0vHe2vK0D027qpoGxUeHtNmF2Gy8YIXvT5vSNIDU4bU0InkzPP9aD1E9RGD92q2pUqm01rnqrLjQ44d7r2CxkD0jbcvCw5twW45yFQYDM48g94H+heL7/4hXpf8o51zV+TJAPYs9AMgoHc+MfHim1KO7Y9iGEnmtj04+gPkXD3rGvSN4kptllrzgeaxkOY5D9gtrA4o3qb4MG2zJz/ABmV5saapcBLtgUELgbueA81hmYAB7FgZJ1WlSNB9xdAqtz3jTWg/nHOOYZ8uAYAotw16y7kYiRJc7lpDHQEgMA/WpFCFSuqAOucV4I5++BlXHBa8zszidT5wTuWjzT0JIPBhmC6gzYFHuThpT5eYRpDHQ670ZrFcaWa5As2mQa5SqO90fe4hqose06Jc9uXm06/b9ZjTDDrBk88EzXqU+9SZsR9jNo2d4L897GuqsWTK9eWrXzgSPkDNPTOJ3EqfVzqT991jM/wZJgCvjGgqUZ+HWahTXG8AYe2L1xwxht3DaUeosS2Y0jDAzwwzXj+H4zQSz3un1n3mu0cHL2rMaG7bjD/AO+ZXZyodyzlwGHYMtzwvxmnl4l43+taJeTtKnSjkx+wD/IvQFYrFZ9Za/rJ4JDHRgub8Y6kxxCpDNKnQQjVCF0SHtn61zDmNsQ5wUg5zlGB4AAzwM1rqbx4o0Ol+PQIz0gPdctgs5AoN/2xa81+UwzMpQM+6e3rk4G4Zm58+ayW1F1Xb9qqy7VdAGdT3LPQCp9aq+4gvaXnwVYeuyeYGCszuescroa6Bp6fLZPTc2H86dZqWcfQMAVU9MceLx38zTrMN+T7DegPwJIFmYB9agnGcAtieNk2S03294IN6sHGZpp752efZgmTZ3fIpDLOsWmmpgcmWGuaBZ+7TRqy9W0rlQf9ceKYZ6PkVUbOwzULgutNGg1jA/kUtmNm0DmfWqtCDngW9OgZ5+GjmMsMl7Tf5DBDonhqNr0SgePJPc42HuM1EZZbMfEc/QlnonszP6AXmBLM2DP9kBbKNZkF6ltVFuUZ6nYAYYLGsxqrM9nBeP8AItlat81WZC9RtwYx4dCeRL+z1A0tdxt7MP3yrzgOM58iD0lSKqcuN4kvADPsBUj0+cA67Di+RcCweenHHPXiPMm351COq4NaHQg9fVl5o25cTW8mCrzjHJ3rWOH/AHF9SobE8wbf3ga2XqGlMgDfK7/nXP7ekvw5gNuObF0jM3hzcXT0wIX2et8PZ0qN+hF6tpoezpsYPyKSkkf2wyDMUA024Mb9CHTY/AZ/QpARnD3tggeZNktNxtaLZf8Al1qIReVoduU1l3zrP1u87urzDrcrAGn+sGtgLBRpPLO6ikHW5xtGAOda9/qh0W1aUb1va7dYOAGeAADOeZqvn2TVWXebrPMgZ7w1g6wXS+CEyo1i0jByIyzyR6GYd6j8SHpcBrTlYGb54MmB9Coc3pUCXJzYo1K590OsFpaOdsxmmqbeNuRmZofOo9sXIxaQBKfcwaPr86u69c9h1WOc5iIZun/SFYfr0C1TohzonLHIDoZZ61zKt8iErTlmYH8ivvVsF6qAdHlcy6fZn0LQBZlx9bkGGH1vZrC2TmMlmJGj/ccGXT859ZpRqI+cXm507M/IrOq/zqUg5QBzDB4BgojMl9mODB9i5ra2jhGcy8nzoQCWz7zmTT8mZs02w/WoT2oHiG4f5FiJYBLM/YLptvcS6rGYZgzopsgHvQ3rmVNk7+9XHrJxnoSEPSvBzipSmbtj01y43nudMAzlngDOC9tzID8zfEDNrzr8t7DCJdVbj0OdBAGj72utei4dYvGydI+GXEOTP88GcesBguzhtth6wCiS/wClN4R+9flfxyo9KgcYrlg0ZwOUB7wQA/kXpq//AEw35NpTbfvG3KlQa7KZ0QjxwPRMPPn+deLYZvvT3Zb7+s6Z55n1rvjmRHCr/UjkyuamnsUG5Agwz5TQ3rUTw0Y5vqv+z04KQ1fk6CydPive93gaxhthdz+GlzW9RqfcduPhJele2ZDrAFs+CFmUOS1VadfdG1qhv+7vBgYfQaDhFflHqVzcjdT4RmjP7hmeAMn5F7LZoL71LjtsTqUzNfDMDCMG8FcIuHjSm8H3KrfMT1VQ5hworwPyQPfgAL0BxdprFbtkG2Jxg1BeBnSDo/OolEkxeHU2oP1KsazpmYSYjR5hn5wWapVz0qvTApTB1Iz1tY2WgzVuC4dT4Ps06m0H+aszm6wMGZn1hgtLR6xBh16q1Wq1KGzIms6DJs7DzWMo4VWfMCm0N9mGB9EcGczVxc9sMW3S4lVgwecuWa9hGB09mfnR06bOsT6VZPCWusVWqxtU2T78O9c69GC54MOw7jnVWWAAwGbJunve86808Zodzw7ll06v3qzVag4eb0SP0M/WskEydMhcjElVI3Q7APYouyLey6Pxsr9/2hIocqw5/qqVmwcuIeeYLkPFG+araRhb9GgnAafZ1jZdPeBqdwovOsPcNPs5E0YbtOeDWdZDeYGf8Cg+lXb0GiVyiVGJKD71DADZde3h0LjtG3D5lY0QN+c/4y0cY6dWOF4Nthgb5gH0LFXDTWJJ824tBbEasUpiPzcQHohngEd7oSIbbWE+j06HRIQOPgGh3n1mq+5Lk0YQA5VZjwH2G8pt1a9yXBHpVGYz5WNmYeTzrJXgDkl2lUdj2WxnPyZrRe1fJNx6P4aPWiHHAOUwwDDPNae9rPOw63SYNVlsnElMh0B0KvuRm3At45VNlmch8wAAPrBZ3CmXee5ab5w7Mw61Lek5+IfeoT0x8wabNwNJjsMFIjPQXmtR/r7FcC1CZEOlumbYMmB7GQPeaop7Muq6Utx9nDDAGg7EQSQNjXN8M1FOZgsxWSaa/GDUcbVUYZkripVJx6F4Z96pzNdALoBD3ogZceDBIw0T8RxWHo0nRd1E69JcM0dEpU65KzEodObzkSjwBdID0eLxD2lz0QPzmsPY9nh9WN815V1ubszJfn2KQ9Jcejm2uln6NNzB4j98UhkPOYGnWfRsr/u77pTzR+9Bk8F87+efw/8A/q1+blWvBl5+S7HY62mOj84LrvFqq+oeEtblNuYOmyDAfnNY/hpYYWAOgdcjVV0DzPRDDBaPiLTft5bgW5zQQGs8zdPfmvP57/DvH/6nx8zyfJ1HobU5v2WGAJPVJg6Q1TWOlg12ONwE5bovRkPohgo73o8MSS1DvQPyRgBPH8d/h3n/APVr/LeZxQ963EO8/WVvBQJcGMy1FDBkwDfms1WKV6qrM2jA5rcq9oZ4YZq1sN71bXgblUqNJCUyYAboexPzgvseP6+Py+d/w6BwW4UVziE/LnUCqnDqEKSxFZMGdYPGz3mH5FreN/DG5uDkqmUC5n6DXo75vvRjieDm/s9sB9C7H6LvGPhdwu4QV0ItN1rziyczggz96qRmeDOHyZ9nYuNcRbA4h02RcFx8RqUEmq1SMdX57mc9EM+gP1qRmbGuq2A9VU29YMamx6I9rM6UPWB4z8/kSur1dcnC+u3i2xGmVULqw1e9mEGeiAB2b00deo0C3godVtVmTNqLIPBI/GD5wTX2esuHYYQfWPq25apWMDA+gIXnNGSvqUOonRIk6dSqxGqFws6MOQYYRXj8gLL02M5GmSLflt4S2MwMPnXaLze4jT+HduWy3Fo9zwqCfrSHOox5nGD32a45J059ckXAbmBzjz/Oimr4aRgol5SzcgvPSNH7th0ZoKVVbnC45fInGZkPm+BuveRXvBCyale1el01i7YdN7DB0PvUkPIG9avivZlv29WbXo8F8HqfSwwku6OB5/rXLa3F4bIT7hB+shUpmg9/0dsM/oXfbksawKJFol4sQXmahNZ5pll57WAz8h/66zljUSDTbml1Vhgw9Vngy08GBgHeu03zSgvm0qPcFGo2HqesaBhI3gAGowvhcZpocHabDqtS4o0CfJPnGIp0yJJMGTPfmeH6POqeNwfoFz8Ebr4qUOpcnT7QnnCjOhGzk1LN4MM8+wAMF7As/wBHu2AuahP3PbjM9q3ob7ByDjAEaSZ+x2d+C1bPo98PKPw5vDh5Rgk8ldGvKksunmAP9mC29Yt4a4P8Cp1+cO3eJUWdo1Cl3JFiw4jsbZJMzDo3/P8AwLqvHL0gbnti76xatq0rRpNOkgzWAnM7Jh94fQu5+g8cWNYNYsqpU7ORb1YzwdD2OYbMP0LqF4cIrHu2jXHR51KZZ+0pgcyQAbzMDzBdTFkb/tsOKnC+j0OIbIU+vHCenh2cl3h9Gxc9r3ox2dc/EuoXNWYvM0+UyD8YI72AAYBgALusO1WKDAj0qj7IMVkGWWuwPoUuHAc36+xaiDmwy0EVgMGgAAAPyLNXnbFKuem6E6lRp7rG+MzIDMAPz4LcTKV7xtVRgYH0Il4JvCyaNYHFpquXHhGo5xps2HrHsmGGwNnzmZmtt6JcOq3/AMAuIFhsBhHqNYzOR+CBhv8A8C9S3VYFpXUxCqVx0pmYFIZfwB3eABh3guD+hnb1Vh0O/bHlMSbem1sGJsA3th6BgYZgtRyLjZxIfptm2fwPsOhh6vmgxKnwZe85J6/g7/n61Nn+jHxUsCjUepUaC89csKqnUHokFkDCAYbwPPv6FE43vU61fSTplKYoHM+qJNEYMzZ3vAz1myHz5r9A63D+9A/+OGagcnrBzvSB4H25Fu2CECPV3osq4WjPADYZPN7owwz0f415q4Gs8MqPxXplVoDDNHoUW6pRw6hOewB5gGejM169vm2KrVeH1bty2ILL02bDOLGjm9ogefWGfYvDV80Gv8H7jjzpcqic7RKlzR0QD5mNDM/n71Nw12/Qv943vaPoPzgvgLBcFuMDHGayI9ffiBDqsXwalHDsPz/nW15zcowyX0YzwWlZ0zjh9CykCSBtKzOsaMXBtRcNYspjMQDNxt881VPPGZAg9ZAeeaXMsH7xIHm/jT9844n+4psUP416DemdH0B/gBeeb/k85x4rbf8AZWYof4F3t5bCWzM/5tRWsapYCsfrOASlsz8FA2uDEyPg+wBgfYaz9w0TezLpzYBhs0ehJmsYNYaid9am97xbRaLI2QNhVsmG/wC7bUrWwFYf0h+Lv8jnCyRcETA6hUT5KB5wM+9LRtmrt9IGx7SmXBStfnJtEZAwaAw+8n3gH0LlVK9MB+t1TTr8F6jwj63WfGNeV/XD5hsbwA+v5zUd6pPsl4a4VtrxXu0L8uM65TYknPM83XnszMD6P0AsFJmVlnw9Az+gFIO53wHe4gC4X8vEcwBYNotHpsOVMkeIGYGCuAojAezl9B4daqp9eza2Hh9AKPGN84u/2XYrwpcGzBAgcfcA8OwDUpmfBeaBjoPDrWdek5+wz/ICkUo3IxSHH8PHDANijAuOgwYMDPZ1qPJZ1opsb8w9infWUTvcSOqth4bmAAfk71iI4BVYzvRswUjmTANR/Df2YI2alFe9n/tqQzPiH/e+dIEFmpbsNMwT2ebuo2+alG9B1ezM/Oq2sA/GBqdBwA894ZqxZBr5amZ5rodgUqh1J1rnpTOcLx/G2A8fkNcsCsS9motBbdYqrMrladOCHzvgmZ9GC2ix6WZrbFKoZ8UXw1qfSHn4sAGTw3/uQXHLn9f8Y69HnPv4OzWX+Wa74zAf6+9ejre4evw7XhRarWYc8zjbHgPNnQ37A/11SGzYHDqGB1V+mh6xMD1nowAcaLFPA10w53BI1sP02qQqMx/Ntdb36R7zZ+f8gLqcO9rfoNBqFOlXVGekUSAARjOZvkh5MMFzTiXcj9Nub7cUC1eTj15l9im6oYADAYAB/X5/rWNjQ7SOhm/OiVIKh+K0Hf50ttFtrZl4U2scU6ZOuCszKVSn3tkjM3nmWP0bM17Vo/2HuSkVih2W/TZMeKyHOBE7DPoz+deCuDNt0q6qvU6a+xUpNQ5b7hEg+2knnvAF2Xh7fn/o0netv8UAept4erWPUkcA1gMMDMN/1n/jV8P/ACxem6ac6M01EfiaIMAAAGeamm84uZejxfN28XqRVeIV3UpmHEMwi00ADrAOs/1rrHLBgvoMkQJJ5qQE/DvUeTAfz8AFC5ad+GuW1bWpz8+9IJ7Ye8VOfNh7RV/Pmcx2DqeKAZ4d6g20smpN7fE6zwTHrJvzrCX5M0bZdnHzmkw8xmYfOalUett1ulhVYofd3zMGT+hDbUesnPdqXGn5+8WaB48u/wDQrKHnirepcj2qkRqbOeP2ezzprrPerqHWGGWNDT2Lp3+jDDH69q+TAfh9eH5ELMlwPZrUUqTTjd139+ALSybVtg4oVKJBZM3wzM1zW6YhzcKlu9opZz4ml4hp25KJBgOgcVvsWaeDNc1tkp6pYEbeoo5yWzVbJB8HUAGrhilvb/EUF4M0efwUiNGA0EQIyM2X8NNaBmlbdRtNHAc94i2XOAaQRnw7FehAwNO8nh7tBVQ2X9LoNS4EP1kB6bZgfYHea1VvG3Dd1OxatnQMNRtgFI5edt1nsgmqeSBsumw51guyyT2aexZ+pUqI8BuNxWdXDrXmxy157BV7zy0VVpRxvdrPyWd66EI+s5/UhOS4gPoUczXqTvOH/Uh5w/6lHQdCpCXz5/if/oRBUnA9mar8PikCLXPrh/8AES9ay/xFWIs/gvcKWZz3MPaJHPcNVQGnQTAm8y4aMNQ1HBTWVCSAH81OjMvmkyG9WDIICZBwFYMm5jsSjM6ytoEBjMFF2pBDnlOZCo97a0QUeJ2KzZogBH1HHAWS2XZZf94ndFaP1awHYmnqC28eo2grKUeE0Fh7t9HWwLnuCRX2KrMo7srfJZiAGBn59/QuinRzZPw0uTP+tBzqm+jfYEbNt+sVifn55OGCqpPBawIfF+lWxyMx6lVGmm+cd6Tn4/YupG9yxrBXnVX43GSyn2zw14ZgvMKatnghwdDw/sJTT+vM/wDOvOlngxbdRqdDcihMiUut4A090YL1AzVW2XdRxxeYp56N/wB4N+eeb4Lp8Qw/L0gE+1YefK2rRwD/AOjD/UTL1yYfsMGNGDyNBgqKMbkmBHlOd7IJLvj1ocV+zazC4Z3Majh5h5Ey9UnDLUB8/wBar0K26Yc31Wso1ScA1IeqWapwNSANY360NvqWdN56TI8Bg3sOvBapmA/paiztHDZqZ4fnVgbz4d64+aMOzhtZ5tgpAT2A71lJNVcyw1E0FVw94sOm22/DYPVX8NQTmZqiZrbYbHHFNjVKCZZ6iywbXcB6U8GAZ/nUt5lwB8RV/wBoYgd6ZeuRg/fphez0mY4yarJMxx496ak1iIZajhqverEUD8M1eEJDwZqC9sTR1iLh7dVkyqtvezfTAsAn4GnjnhJawA1meZzL2i51xX40v8LnWqbFo3OSJQZsnn0fwfOmB1WvT6db1Gl1yqytGPFDMzXhe8K9OuqvTalKqMmSGsYMg89mAArC7eMF8X+BsXHWDOOf9HDYALPwIb8kDCI2BrITmbVg+qee9eAE4A9iar+Wlh0aP60Unm4zpsS28DDrUfWULHMh1VkdfXD8ihHVZzPvM/rTs+S5DYNzefyGo9EgV+7ZBsUCjHMdDrAOxZxYaeqs4/k+hbqxmeE8+lmd1cydT/s73esVXrPvu2GgfrlDejNH0GqoMz3q9j01avDfhzfMfUpVuSaVIA8AzDAFHn2qxQaydDb5ONLDoORsBcSt7ivfln7KHXPu/wCE6GYLZz+P068BhN1y0aa9NigYBIz/AMiQJHEWmzrPrMT1zKjSYUoM8455p07J4bXnA9eP3GzbwB1ge9c/uGZLqszmpewPIg9T1WZhoRNaP9a9E6ZJnWY/6qty6gnwuw2TzBR3r8ud6nO0NzlpkQ/cugmWWfU8rkZVNjRgPo7801VYDGk6+wZhn5DVQK/1VO5PmsNiCBJ0faK+t66n6VF5R9sHg7wNSqr6jreE6DTQjSz68OhXAu6DM1oAK9ptenW9PiVWC5vA8Fkre/AV3UmXOQNbb7B3iTArkwwnPxfu74Z5rnN/8Ga/eEw3KdVdHDr5h7AFqOCfEty6oAWrUYpsyKcGAZ94Lp1bkwTgGbkQGXewwUjj9jWZWeQl2VXK5DezZ9yeYGuX8SOC0625UhijxQMGPdNHma6a9WHKbXvWQftAKi4u3s5POn1liDMjSM8NYD2LW4HMaPwcu2TABx+hyTPz4KF/JjUgkPN1IHozQd4AvYtscS6VA4aNXA+fM4e56F5t4i+kCF2yjbo1DCG15DPNY3A4/W6C3AlBykvmY/nRUeS/SpWuxgZ+Q1On1LnzNx8AAD8ioXngz2KMCdPCVJka5xDZM/Oi5OWY+GwGH98ov2qrOlyvPHo+RHAr0GNH0H6MEk8+vWwV4W+HzcAujBQnpPMn4+9aWHoXU76qo9GPmz6AN7rVPcNsVy2JAMVymmyZ9GG9ZIRecYBrTbAM/OgCTmGGar3t5eGgADWonPM7s1HCS4Bh5EQH7tw0J6YO+zUjZQ7tY9THFlUqHJ2YAZhvBZLB/HYYfQjZ0wTrOmZ6feqGz4XMwalPCm1Kmxnmj+RdgOzLctt9qWxQ4wOmeC5Dw3CXDrwOaHhefyLqE+e/UquDb87WwPMNi2HWGfsWYcjOtzljP9zmuD8SLAYtK6o86lRXjjzegABejoz1NrFuNRDfAHcMAzXBKxR7jh15piuVIJkfWDfn0KLhkwVVenZg+cEw+tVj1biG1oS4gB9C9EcSLep0myJE5uCzmHQYLz1AuTk4+g5R4z38C4OZUWKNMoANeO3JzUU6k5qnyMQ8OzNVkme3JkeIwAfQmtZzPYeCzWmsnLCaBrqFKNs6a1prk7L3jhrmZh5F1Cgn/NYAtIgSz1AUST/w5ipBqO97I1eFslPN/nTV1SpJ6Q6ihTwzLUR0o9oLaBxEHmD9o5gngNt7w2FoKPYz8+13bjM/Ye5w3mqeNRK5Ga9ahBMGg61aHbeDkB+g0vnp06Y8GebMFnrNb25LeizGAu6VBNmOfQ09/nXF7V4nN0pqO/WWMzi54MxwXSKlXr/4iwGpzEGNRKSAYBrH7ZSMrW/sw8w6/Uabh5AaDYCxjLMGfs62u8FoKwc7mHabVaxGOOHWa0dBep1Bjm2cTmfnwzNAFpQ6zDaCXQ6HGhtB1yJe8FRV6q1yTcBufacDw/s+wPyLShxClwyNuVQ8KefZngazUkKNUpXNwYmAfWuW0GbkZ1nWpcWIyAYYbO9RYFNl1hjXiMZgClVJ4GYZtm4vsO5KjRKJytKYZMHPOuO2zIzz5aQTD8Q8wUR6YwfuP41YVU35PiP9Zr6FnzjDmnHwAEFdG1Adzb6VIzfM9RRXo0pnYpTNSbjYMSwwNWhqLeZ1ndBtx4DPvA969QcCoEqsQwYcpUamtRTwN7Pea8q0GtwYcoH8+hdqgekJBtizXX2PVQVPDBloNi2hbD+k5c86t8QztV+VGehUT2Ol15/OuXwI2chNRjl3DUpFZf8A2iU8ZvKXzPIHsYN53yAuna4Ta2GtANHGrfr63qZw80MI7j3jPZ55/kUqNyPKtN3G5JZjn7ZlnvVVdsOx4AhKtKsSTd/CPsSLeKG5LbbtuovQXAzMN4H5wXaYfpG1WpcPolDrDei7FZCKBtZ54LhMl5+Seb75n9ae1tGAAedRtkvZlb1pBm2/MZ+t41qLM4qVWxjd9THGkg+G8JDOe/zrlmRqXGPD3Zn9ALn35HSJ/Fe7ZNbO426ryzodDMcMABdDgcb6rXmNSq35c5h+EyGALzic9vPBxaNm5GIdvusMHhrhgZh1gvN+RtY1/wBOuGPVYNNobIRwDA5xh4xqLw9upiz6dUJbGD1TfPAMw6Fh6PUnLYoctt/+m7w+hKNUmMMz71tFmHRY3EiowI8vkaPGA3zzel94GsJJmVyt1n1rUZz0wPxjNGzJPmAbfbwa86lsyaVloOH1rFeEGTzehrvueEpf2hcmNNaeYGCr3njntOxcwAGOhV8B5x5g3G2+g1Q6RTbkfo9LlyqUxGZqD4YZ4LmnLVWtvssc8DLrB5nn519OpVVl3Q1E6GoHjn1n1mvR0KHVaqbXqapcsc6aGj6wkdYB5Fz+vQDpVRdprkoJOh70Og09Uqw5PFrXc6AwVZrAavajtKo5zB1zfZZDyOprk2A3udCaN5IJIGWpqLQKe9TYcU+sz7A+dZ+T7Jo+8wWiZprc90wN8Gdme9RY1N1pAOG2BtB0b0Gc0TP6EsGwW1qtEOpNHpsAAfhAs09R50Yc3ILwAgl0Sjzq9PiUqmt5yJR4ACvQtuq22/XWKrRmT9XM73ZEY8P1pWNMnUSvUysU6j+spEIzMI59BrpfF3ipLvC1/Uc63DolVcPOSZmGDzHeCtkx/BClUf7b0SuBKB49E35gB0Mgra53/Xo125+VOTHOeDMOWb29kFP9HWghGKsG5TmQaeDMHTZ3mG9aP7FcG+c5pyfGD90ck8F+U/insTHN+t9r0fxhCkx4lz3VTLZrcuTyEWjg9pZ9Zh3qkoV4HZkCqsRH5PLvzDYh7M8AW4rdI4Q3BIB+q1mGZgGAYScNitYdSsSHTvU9OqtH5TAw0c18T6+LGMU+r44oZLgi8zPpdVnf8L5vSDmeMZmq8+NdzNuutscOjeaA8AMtbf8AwLotq06j0ikx6dRIoMxQ8gdZ+czTtSueh0KYcGq1yNAkN9YSDwNcd8/D59m9cO17hxu8ONl4vUJ+CdjvUXX/AKXvDBbfhHe1fuajS5dfpZxuV/peGASVd1K7OG1YaCLWLmpUwAPMAde2ZqJcN/2Ozb9TNi6oB/djBlkD3meC7a6ubi6eL1vwi9fv24ZfNH5CuSJ3Nsvc68b2AdihT7h5xqkhFiaLtO9s9hhrGqIHnz6zzU2ADbzuh3mv6Nwf048Py/J/y958B7PsS/OFQcd7jpTJ1Vg32WWWQwZgAzs2AHf8683cWuLsrjSVCixHzjO0tk4rz0h4GczM+v8AgU6zPSBCwOE7vDJujPBUMJQMutZ4GbxrR8BPRmti/OH1Vvy8Zz0mRgbMOnx3tEAfDPee/wCjYvEu28LrboFw8I7Ul0OgUqTXcApcyWzGB55kA7zPPo/115Ev+H614tXRUtD7oxPMA+RdOsC0rg4FVGPcEGsUpm4ADRk0bmcwkgfea5e9cL9wu3RUZUQAzqT5mbXRmZ7ARlFukejxc9q2lddx1W8aiEOhHR32cPxs+wAXDHt78iKDBsgw8ejmHZns/gVmc9g6acSJ7XDmge8ilVKfEuGiNBWWNGWAYcwyeAfnXOOoej9Z86BHDiTUYsyNLYkn6hddDCnz/P43yLvH8gkS/LhdrLE6G9EheOzTg3nr4ZnmH1rk/Bb0urx4e2HC4LVW1aJWLX38m7OAz0Q69mHzrbW2F8W3w0uPjbQ/WrMh+Gb8Z0DCTFAOjA8N+aeBcRuDNcgQqhVaras+NIADOS9zOYfIGAdi67w9ptuVu0morcV7B8AA3WXsA2fIuZcOvSignZsS+70tiSbVLwpdYnRJPnPADNnDeHWvRFpUSx3uGjVVtWq+vqFN8aHOj7zzXR4XBQ/BlaCuOQfePUbBY+57ziWNFA/sPe1b/wDzTDN5eXJnpV02/PSMtIKNBqTNuMGcLlJ2cZ5mV5zDvww6FpcYLaL0hOKN1ej3x4OuWdSox0+vUqKcmPLDNmSYeTyLq3/pLWq9b1Hqs5sKbUKpG506Y6eDzILmnpk2Y/8AYqq8ULgiSalUJ0aFTKCEfeEZgN7z+z8i830Sj3NxRpIVW4IujNpdVp1PPNnAzB4O9YIw/R6iXDErcBqdBfPeGZgYYGCtYx59aquTg0cAixehsADPz4BgpEaYGftF0QyXGsohst65uONoAmB5094bwIoy8yw8Bg50HsMFH9T0oJUSc3EZ5uEybMZ0NhgHkUs2XAUc3gD2iseevSo4CP8AEiUF8Uqserahb1KzZPMwzw3gu8WrTa4FjWoFc31AKJF5k+/PBTTjMT4uD7eYGiOpVH3bmayH04b7PYvCnpP2GxValeF3U2lSXqgFz8qeiewGAZDsXvUK8/yfKS2A/Ws6dBtyYcg5VDhvHKPUMzZz3oPNPou8KLnsmAF3VU5lNOvRjPkcNhhn4Oe/6/1r0GZue8WwnssTwBw+sFQzIG/w1QhRp7jIdaM6xn7xH9npb3s3AUGTR5bJaZ4Zq8AzqThn4aTMx/VDxFTnGqIHpp1kJYOhqAZ70wlw+vPczx4u1/8A64H+ReiJ73i4LzaBnJ4z3b/+dcP416CnyfvhphRZGjBQjkt5p1k9/tESm54KQzJwWXvC7abZNvSLjrDck4kX2xshnh9a8+0r0rq49dpywgxnqFKPZEwDOMHkzUbHrOM9rGDa8k+mTxRsq/4dv2rbEtmedEkvynpwHsZ6AwWjsz0n7VBiWxerDzMirnKfDfsjAfQH0LyTcM+IyU1uC3g1meAeQM+hXsVTzzcmVyLB4ZqO9pwJWoc4Hj+voVUZmGbgOACQRp0xrCIxnh865sN/DQMyQeDNzlj+tUlSkxdXTDrP9CMAOMBsPtnmGwwBKNJbjdbYGHzqIh6jgD5j4aQanYatefYk011zlMNDokYdB9ikVKTTZg0+o6Ek3YsPQPV78FoK+BP5Y9Rvf2J16e4Y6jef5AVZJM3ndRCBuAWnqAAfP0LzClxz7YOhm5n9akRpMWNm43o5mfWCqmeRZN1vYfz9ijmy2Dubai4FxMBwHeebPYo7MxsxDU2H3gmQma0c2H3E0cbD3mxXgXcapRQfB9wAeMPOroJMSYOg42G/yLD56Jq4jTGzaBtvPP5FjcC+5CDm1qN54GlnBhyDbcAzE94Zmqlk5XtNcDDvDNS9ZuS0GbmBgfQowO10Hi1Q5PDSPZ06dUoc2kRjONI5nYZqkvPiRVar6kqVSprPL0TZTTkBnrH5z8+GzYuWazAezWoonEK/7SYiXrbnq02qW8AGciGDzOZ9mBrphNwevniRUr/nsyp1VOZHi5nGA+wzw/1FmaJAqTz7rDc43s8396kXJdsu86y7X6lBpsOQ/wBYRI2j/AqrnH2RzYcwNeIavhvcP2Vuin3rymt6uPMA1sN/ZvBaji1xIqXHjinHuM7YkhoRmIXKQXtYwYA/P+dc3Z/aJERvfHz8F3z4LW2ZJqtsR2rxps4DqpvP0yHTI73jPGfnD9GxaeLLe/eFcOJD4R0Ji3KNUoFPCNgzHqIYSg3960DIGHWo9B4Y1LhFws9f1y7vbxmJsyPU5/7MeGZgGf8Ag+RaCBZlfqtGj1yC/DmR5QAbOD3Ya+n4cFhjG3oApGzFSqbw0uqSYc8DMZr5HszVwfD2VADwHHnvrXLbbiYeqwH59OlsMN79EzDDrzXEqJfNNk3rQqjVYhw5b7Mpk8/fBhsNekDpsoJBsBBk6vYGC8L3PxCiw6tT51Gp384U+fKA4MvrZPXzADUNrh0P0h69WGa5TLKpQHGp/JnV57zodnzq/wDRyuGDcnDePBY2SKW8bMkPnPfn+feuYelXeFYZvKEDAMhIlW2EWYye8GdY9+C6B6K8yJUrXrEGlUbkIsJ5hkM+szwzPNEYddCN/wA7aWGCsPVRsjm4/wDoUSTDfzzw2K1ohvNgh5hxI4x+RNYfFEneYcVxDvCqw6adKiYaTh55n1gqJSmdBBLn1iW80bj/AFqnenv4q1Nlh4dNGFBB4FFwqGf5zPrTRvLRhb0TV8dg/wBacet5gI5vsAGzsUbMMlrOf1KXGn4Gk9DYMvZ7/Ionq09/iACvY00a4WwDT1FI9axHu9YcwcZ94iCe4HvEGxB5vNWUPQewbNYxmf8AOriHP0T1NRZLabRBk9MFYM1IGWgDUWUeqvkcUc6q4HvF5gas6k3l1pHPY86xnrg8uvNH61+dMC6qUOLMWfmWxm0fiAngrHzqQFSz94vRlztt8yNvBVNSo78D24YAugbMkqlDg1ijVCDOYzaOG/8Ak2KtjlRs4JrD4rVcFrkg8QuF7VSqLbL0uE8cJ7AMN4H/AKmCvnreob39EVxaLhznDYlh8V0P7GU572Zmz9CT3CgDYN+nVzN3yOs/7avaMOfpYfFaCpWTcFN8RyLrB+63qq9UVX/4bJ/Qr2tHw+KLD4qQECcHtIL36EuWlf2R79ChJYYKWzsUfRl5/sj36FLjU2uTCwi0aY8fyAoEhl4A9opAPJlmg1k/bsYfIakepJwe7VidGk71cQ5Ie8cVIFHqWfsFNZptSD2jH8axGuh1LZ7dXsOeBhg45msPDh1L2jjGH51a02e2y6Dbh7+9FNnh7xLWAFWevogNaeYJg62x7tR02vshbvTGMFXnPDNUr1Yz9mmecz61t02x7oS5jPMu5rlnFeG/Au2xJz+wOZwM11AJkXzrmvHJ7wLandkWpBmunhhF+W9kxn+Y0G1wriFGcpvFqpsPhhzsMH/4F3WZPwmG4C41xsP/AJfUSd3yoZgf613/AIcrpFtg5JtemSw6eWBSDBU9hvGdnx2DP2B6CuM/grhlZYfFAlvQZyvdsLbbmwMAR5/BVk87tjOgcGLA5fPAweVgyFVePTOKyH51FtFxANw2utSDNgGuveoUNl8B3uAi0XPeL5t/3Ldcef0IT3WgVm9DbwRxqVF0tc8zW24wj82rOWiPdbeZprkGALPerg4eAbGwD86Z9WvmGbe9cdtotWvG4HvFBOY/+IrafTZQNeICpHoxqG5o5m7wzTPMOIeWfy9miNnAd6YDXMOJayaM0ybzapCWD2BrgXHWGxcnE6mUozjSWTDAGdbAw/3wXcNZeLOJZ3BSuKFVqXrUOYYewZMAzwBHm3S+JHByh2Tbx1+m1GS8YGAYOguSM1KdzTXKAYefNa2m8cq4DWnczEaquh7F0wwMAR1jivBuSluwZ1vxgdP+kM9YAsrh2RbOVWpBMaPm5R6vzqijVKdAka7B9CjhgBeJKM/rQTJLANLGyFm9cJyfElYZfIC6LavGO3KVAOlMW48zrhgbzJ78/OuK88CQTAz8NeYW6lfnEidcmlb/AK5nnTGMN7vWaxsxltljUiywP8iz/OOPF4jikRs5L4NuPgy13mZqbhBGf4ilQJPLOg4ttJ4UUaHS/XMushPa84ScFkqbQZVYqkeJQ4uemeeGarAtXpmsOemYfWrWNPwtoMIknVDoew2Jq541VhtGxVaVyDvkStK6p1NodQoegBg+GYGfYuQU+DbzQPy52s6gMH/cSjBUR1LOQZt7M1NjSTN0NM14LAKrSgi6D7EYHQ78FWBVdF3NjPYHnUCSzovnqdeadjHE/pbeYLeIHQrSmMSWmp3ea2AHzMd0OzBcyptVp0lpqKwZxjA+jPrXRaIeZA250mC2gY96fOt6uesqbOehyA6DBdIt70hK48PI3AEOpZ++NnA/1qE9bFGrf3SUw8Bn39C5vXqbTaJVzgsS88O/BWOsRq9Br08332NED6AzzUf1a3W6bUKB8+uyq+0qVUplGOXRvvOHtgDrUiiHVfWgPwYh4BseM9mCvY3HCixolVtyRTqrOk6rAGD0fPYC5/WPRguCZVHX7cqsYInYDpruHC4GIcqq6jn7UGa6RR6PSmaRqH+RGTxv/wCjldsOaEGqvwDA/I8udXVY1coNXdprdHk4B0bF7th2x67mHzUR4PJmsPUuETFVqnK3VLkw2j/ZtI/bKFRbxVUqDWKU1ry4uGarAz951r1xxd4e3BbbH3Sx9alB/S4+8F5sqUaCc0/DON9YK17QqJVX6PNCcx1gthXuLtSuGjep6qEY8+s8OtVMawKzPp3rKnYSWvkWYBlg/aNqLEcwxToQ5Zsa7beYKQcZswV3bzNuSaW6xL5xmWHkNBl3gMA1HAMEGfwVhyet7eWePkMEy9GBkvDQDDMMXc3OvoRgGZaepgfnTJsmAaiOHGzf8fo+tB1OwHqc9I5GdKNkz6DBb37JMPVJp9i7owfI6uSW9bb9Ynx9Csxoef43etnW7VlUGO0/zcaZmeHgn0LaLZOh1U36UDQRKwy9qeTsXNLzqs6NPBzmgMz8nYprNKnez7O/eri7eGjDNtO1yI+88bHYYIg1cPEJiZZHq6DKwd98HnXHwZVqzJinTtNxzxfIjpsOlTC06jmHzga5uaNqhnzZYCUYP54fQrsKPZfK80/OksuqvuGHTYcr+bZZvB86qT3rJbXUe3rRrDRmxWTDDsM1raOEUMKdBPMwXLKCFVOsx26axrH5F2a2KVOZfaOoxAjO+QFazL0N8DwcBNPQ5Z+7W4+zDb3jg+ad9Q4KFuWz4D4NH4apGTONK3rqtVomZnsWCuem8sGbbfQtYYsl/KRLgbINDhxo/YAdAfnVTatsX/xUmVCm2jKprIMHmccHsAVbGu2V6r+z4UNkzf8Aem8t7wcgVXhFxVpUq5zjRo9RjGesb2zBdK2EuSz51BdwYpUz1hTnsJJ9YZ/Iu3SY143tYtMqLjGi0wGbzLJ9i1t51Wx5N2zarBuqlHHlBvZZ6DPzrOSbqozNGdgwa/G0g+dRfr/9XJ3ORXD6qelNRWDw85rttmXJb9Ht6PTTiMycO93rNeeq3Ppr08+RPoUuiXbUaIZm2APZ+dcfc61xfhv+vneVPBryKspXNAOo/wBCaOq+uJpzpx6IfrwWi+0lDeo3qqLBee/fKLvYr5Mxgwa8MHjz701PNwy1HGAjfIq955jMOVDDA01UqkxGIJct8/kXMsqlJbBre4rCmzwkxTbcfWEuSsOPOx9PMGgPYl6yzPXY7OzsTA1Zg2y6eZ5+Q0DzMXDUcbBSHtABAwwNow2b00ZxQA9djMVIrDgPyXw5TDA+8zUcwl6uB4H9CT0xsy0GP2fsBSKOYHNBhzZn3rogXFNh6MdWDMOnQGjqr+90P407WIEpmkGEUwB1ZKTnAa03zM14pZ1u5254fsOjh5FkXntaQb7aOfMzaMwUT2MAPOa0gSwBzDxOhWEzkZLsTlH/AAmA3hgqyM849FBv51OZgTg5vXiGAYd60e3Bl7xp7uAfd+w1YW9Um4dX5SXhy7wYHsTUCM/0S2zAPOYKwOHSmX9Tr+dYXaNMvW2YrL8hyJL3gewDQU3m2WglAB5grCts06SYOQd4B1qneecA8G3MAVIS3qw5vccYB50/fGmmXn3gNvTMwPvBIIfhZvuACdAGAd0GOpatWgmaEnS05fQHQaQMsH7R/wDQo8bDVAH2zzSn+C+bazwyWcMIhu4a5gHnBTXgp1vOhoAB4efvWSzfDocRG9rCDb+ZmmBdyYbcl033MIxn3qObL7MPxz3qRJNw4QeGBh86jnMzHx94Jh0wrzNv3aDPDPoTT0zN08GwBRzeceM29P2ferwzINQ+jealsw3M+hSIcPBoJen1qwDQx7Ad+deivAGHpX3vPDDDoS1nIwYArJmHFOObj7mB4Hhv71Smy+9h0AYAgso1SnMhqZ7+xRarUnwmO6Dn3Q8FHhhVZJug+4YAx50EkH9DU+dA0Ex8Hddh82T7MDwUiY9Ua3IN+dKekumGG9RGQVrzjeQHFiaIAGHXmg6BbfFes23bgUP7MRnjYjcqEjWwwD6FmqlXn69Ro9Hcp0NkIp5g6DO/9ahczs03z+sFB8Q+hc/48eXqqeprEOrR5coNaODwZgup1ILYqsCFLt/fNfPDSDYGC5/GPmX9BxbK3g5ytwvk2LfPh72eXpCw6bF9Rx2NDeC833yds3PfN0OVyc8y6wZmyYd5h0L0xQZPqqiTajp/ssYzzXle6qxTq3Hj1jZzZ56xgzhnvTPj/wAMey2fCiRJ9OanMMYb8NID2Aq+YHqph026U9pPmbAG6CsIEmJyoNtv7/J8incRbzi16REptv6wUyEzhvDrPvTxweG27Ydk1ZU2G/UpoQYvtT6FXwKbOqs8IMEMzNdOpUyh2fCCh07ljm/0moSGczZP5FtcYczIybYlQHzCpYcx34HmvsCTUmdIzuOYy0B5nBaewNSnqkcyeD59b54GaA6U/DqwVVve0uVrFnZNVCt145Z6zz3ebzxmas7VZtl52Q5Oqpw5EWTmEfseWKpsxwHZr4d7ym0p59mVLfixAekPhgGfYtItCdWKxUpNZmzpRs4GGHhM7AD5FKoM9iS6D9V/ZDAwUWSznRokF/2oHma1VqwKbJaplvyoOebJvHvSx0DiEbEDhZY4QaHMjNW9GDmZzwYA8b2ewF6L9Ce3n2aXcFuXHUXpLs0ItQCjAYHFOL+MGzZ8/nWXn8E7Z4x8J7fvFiuVKgwmMPWRvHrBoM9eAbMz6MFyfjT6QMWBddv/AMgkSvWf9l6V6o56WGD0xj6PJsUcMD2Xxysli+bDO0fVTNHjzs5r0tmMAGHK78Aw8688hROPcb0UGrKsC3Jj0SbWPWEadRp4GYRd5mwfezvVtw99Jaq3PDiWrbNVo9VkPnoAd0zNGSb57z6F23hj9v7Yv+oUq5qVR41v3CHNQI8GTzMYHwDewB4dZ/5F09K45nhfhpxv498OqXcHC6hy6wFdr0/PSl7zAzDfhn0H0Ld8LuA9lXtwEui7X4NVpV+2vMf1pEs9HA+sNn0f41130peDlv0eiVviU5KmVK869W4rFEeaDROMZnsAMOvo+Rdd4P3DaXFThY6/X6MEOqyv5vueDhgfOhsPNRcG+xq7GqUjiRwJoVNrgGDtRpTATAdDfmGCAOBtvnKuCc+2B/aGtxa1JAOjOKGAAC0FEZo9BpselU43uXisgwzrHmeAK+ZqsUx9oowhTzLegzD35gf1qOFqxQ6JRgtBmB9CWDfvFttDPnaU7+iywM1LjWNdWGbbkMPrNW7MwGVL9fOB0GmxnJlsXdGazcpoPNecHlTnTai8fjxDA1tjuTNjTcPeqSTVcz9omxT+pJQBsnYfWoj1NrLPiAbLwfIanPVJxRDqr4ezcWwiYTveKOZygLxG1OCYBl4idNnmQ1GzQR40x/LNxWoMsPNa7aqg0wUc5ODvtMEwNWy9yzQaaAIEGYfjtrNBVXz9+nQqsoPZuIL163qbq+HvBL7KsG61oYBvVEzW5wH1q6o9w5z2gfNB5CpTOtxmu3/88B/jXp71VR5NRNuVsXmGzJjc/iTcD7ffXv8A/svSE+ThPQaCNQaVDd8CIB/WrALbt94M36UyZrPxqq/mDhmrMK2ZgoEe5OD9s3tRnaMEt6A7mBgetsz39a8CcS6bVeGNpVCyqxZT0CqyqkZnU3g0QeAD2Gyf0L9DoFVfCQDi8jen5D5yr25O09HXgYA92GYH0fxmskvNQfYeBKwfrk+ZoBnvPYZrFT6lOueruyotKjQ2T9yz0AtXfNEty2IsSDTqidSqdRhsSjdMMOTM/cLGQJPIBprOLapD0ClBH0JTAa3eo8B7R8BvMA+RS5MznGtPUzMAVIE/RwiaAYAs1tHJh02f4ge1w6wUQ6DEMN7hmfnVZ610R1G1ON58A3udYINBR4xzIU2hxQ/bcMAzAGQMM8zNYwHpYOuxJzeEiKeDweQ1dw5LYSoktxz9leB9K4aqxMm1M2ADl6iAZ4BhvWozPMuAexGy9hI13wz3pp4DyQZ/BbB3YDpn2mkZ7E0GZkGfQkfWgWeeH1qUZvnHAPnTABir2lHEN1oDY1nT7EUqtE8NRxJkzDocwVxMhsA08+wweln+hU/es9i1hzNum44pck9E2nw6DBVMaG/JHwHAzVw9DqTMU2Jbf0YGpD0PTme0Wwsaq1KiesH2LcerFJDBmpMh2b+tc5psl+M/4gPfnW2pV21mmx5DEV/BqaGDwdhgvIGiv+fwLmWLbj9nQZNNut9435mDxvH0d+a5kEnMvvzi1dNptvyQmzpTgMzQP7nmfYszW6PKpVZGJOYNnmmQeZzDA9A+hepS4DPMygbpoG87hhgALUTKJXOHtepjlZgshUGAYqDIA9mAfWrCw+IVKsy35dsHb8aedReN85cg94Hsw/RgoV7VWo3VVHbqnPsmBgAa3QZ/QCsbrijxjqvEKl0SDOM3qfS43jRD6Hn9+9e/vQtmMHwUo9cpTGi1KzA47rJ4AYbNn17DXgHgCzwkZG4KlxDcpsmaxAM6VBnbAN8AM9h957OheiLe9L24A4KUKxOEliSaxcYW94ztMjZnRzz79m89iIw90TJLmeuGxR3qrybWZuYLypwh4s8aOPR0KvzW3ratK3gcjVIzP7zW5wAGw+jAA2fr+Rd58T+0H+tZ7IhoK9cMWHRpc6ceDTAZmYdeC8OT+HtuV7irIvG3KiAa8wzZCWex4wDr/gXrusSYh0ap+sv2diG++f0AvE9HvOgVKk3BX7H5mpSItSms0QAM8zDRz2B8i2ixxni1dV43JdYUOuVX1l6kPlebBnBkwB/rzXfvRLgRKJc11m5LM3ZUBiUBu9G8z3/n7FwS8DfZ4fUSjSnOWqBsnUHu/M/J/Gtxwu4ltw6jSrcrlSh0emUh4JUx4A8aZgGwDNRsezT1/fpbFS0evP3DbkKqvxDjHNA3wAw7Oz+BO5GumEJZmGaicmBnmlkadA0EeZRz9pEww8hqvksvwMNfDersHn1ErYHPig231h0K0oQTMFaxqq4Aabaz5w5QGjZN9lYqhpTmOPEriADmlvWNZmSgd8NxWUaty2T3msmy2n2fTZkN19jNmXhns71jHqVODw9NdIptYg8uBuOJo40GZK5rUzQckn0qs/gH+RVT0Oqs+IcV79C7FPOCBeG3h9CrzeD3arbFyfnKkz7SDJ/QnfXEr3jb36F0ZmSDLuogk8iZZuMB+hXtbBM1sz86P1w/5DWlkswT2aAGmvVtKMfEYBWhn/WThp0JL5j3q19VRAPwwRhAY8iCvB5xSAkmC0EC2+f8OIGZq1Z4evve8ZD69iyWxoTHFK9fMQ6XU3H+yA+f8C0x8MT95LD8iiV7hWDNn3BWdc8ItKmmYdfuDQeYvQ2vNs7evClOHsCeE0PoPZ/kXeguGJn7Rebf/wBnvGoFb4g3HY9c1s6vTQfjGB4B4J7/APGC/QilcMbLpTGh6nCT88g8zWcDkMCvQTw8RaONJceHYtxM4b2Ubuu3RsPkA9ijvW3TofhxGMAW0WhnWWZZlqamH0K1ewewcfbBTQpTaCYywy1oYb/Oo2FMprcZoDbiMmBh3gq8GYsktBxsGT8gApDMx8/APP8AIq+YejI1PepsW1KokEJoOGwB/WtbGZYDwzYZD6Fzf7Q6Je3VhDvMPZm4vRoKrY0GsSjlhLwM+zBZ+q2G/SmubcY8Lzq9ZrzBtAeupDNVYe8N9/NryLzY5u8bB9Caz+C2Fz0qLWMG4mEYw35tAsPJ1Ib5sH1rpj9bG/0LWNJBnDNPT41Nqoa7YBGPzgqFl594tMMzNXdKCIcf71LMHQNLjrItWBQalqnyjesABnsNQgeW1CguHC55icH0KXTZLjMoHG4kYD7zAOtbR7HWi+FhAP8AJ9a+63zgur1Kj0Cq03m50EOY7DwWdO2KMH9BD9Cv7PCPm8sTnu9oCw/GwNayAf8AwJ7Brtn2Yo/4B/rWL462xTY3COty2APVYNgwz+tPp4T50VnfAp7/AJ4wGuaceGeWkWvWXPZMPGwf58F22z2aNUrIoVR5E5OvDDeH0Ll/pPvU7+T6nsNyo3MRZ4Ho55ma6fp4T5kjhobj1LmseR5ag2WAANeVGZ/vXsFxpniFWbJPOj02NJCoxgM9U+jYCx8yq1K9rh9Y3a2zJ8jTOwABcd+/w8bp+C+e/wBDvtSvOy7edBiuXPDjGfkPP/AqKfxytGAZnSmJNSAPfYaIfrXDAt5iNIkPsNhm+eeBhmngn8sWnKiPM/3PQuP2f43/AKPpet/Bf92quHivfFwzwfpUr1bHA8wBnv8ArVtD4x3/AAGACq0qBVf+x0VjGalRvZxJeZh2GGBqWzUje6H18r+bczv/AJP6zpUP0gaMDoMVyzqxG85smDwLUUrjBw2rDWoxcBw//roxsh+tcS5mcHvEv2n9qfT+a8x/KPU/1d7PiRYGOy7oD2H9nezXPLzvanSWpE62L8n8x7mIAYAudPW9TpJbNn0Jk7YpwH+1yfyPLb+ZR/3sf5LwtNatBp15xZp3/wAQpMCbmGiDp9nnWMmSbn4e3QbFAup6fHimBsvNPZgf5FNk01+ZHOI++zJD3ObO8PzqnZoNSgH4FOMw+Q80+/iefyd0uq+kJfkaPEnOUqiG0YZm1ngpvDTjx9vLoC3K5QwgOyv2Y2T/AF/5Fzc3qUfh1mgMxnT6HgjGlDgUo5rR0auRo0sOg2T3gvfHuuf+VvVEmBybptqvk0p+Z7BtcdgccuIVvP6FSuOj1sP+ts7/ANa2FE9J8NA/XlqwDw99THt/6DXb49zbk5vVwvXrYrh/0RRPsZc/9h/jUiN6SHC6ZIBh+VPgO/8AWI2z9a2NN4i2HWMApV20p4z7OZDNe97Dp8sN9jLu3uMRMDAM8814Ku06rJu2sP1VjRkHJ3hmv1NPPPw215H9IT0XTpTVV4m2dOeezPXmU7R6PoNX3ouHlU4b73s20mQ0VYU2HUpODk5/1bEf9i68HWCsrksn1U1HlQa5Gng/hvwwWO20MvJ/duBmpcaTTgY+9Mcy75FCAGw9v1ojeYZ6A3qVwPmYISNf1UH51WPA2GbnzqQcwz9oorzwK1pDINvNeHgBglhn4bmCrDNwPDbcSA38NNxxeWhYMsvyR5XUM/kM1NB6uWrVI7jbhxnesN6qgk4dDm9S4cav3JIBtiK88cXv8iv/AKLbi7bknV4I86d7gMAWfh1KKAn4nWGCCqz2GWvVr8vN0PkVfGZbNcHlCvMDB/2au4xt6TR4YIpkCKbQS22Jmff92NI2WwjtaZn+heNT0xmLJIJRzmQA+sFUzGYgO6dOf1gVh6tqUlrCC2ZqL6tlRnd7eDode9dXrMhRqU/gD77a6xar382tLm8w5zIg3yMkPnw2LoFqwKjGpwOTgw8i6RuKPJnBMBuCAPOn0Aajz+HVZmVYwvSztaP2G0eGCOjvctVIT/75djkhrDmHX2GtsMmRt6zKNaXMP0YHmQfDA2jPNSDptRoLB1Vhhl6FUfBMDBTqq8+zANyWxgHefYugWlAbr1uAxE3mCjKtvP8AGvN+lXQcFw8APoXerMZnTKWD8vPDsXMuN9hwYFBC4MI0abCMAMMMMwXWOEtY9ccN4U7PNQyaNkMFSXsYMwmqrp5vMdCvehZ255PrV1qgRG8zM0VAqbMfu2Ey5Wd8fvj9hpqvcK+HNegHEl2rD+Qw6wWrolNg02ltMYeL3pSdNFuKXnw3btiiNMUMAkwt+bOjvD6DXne4be4eGw6+/Tp9Hqp9BvBsXuvw8tRUl1WNZV5xzYuC34z2ff0GsmO355PW9UdA5TEqNJjh1nngqymvPxpuDHtX+gPOu28YOG8GwKjyoTg5KVhgAB7FciZjMQKprxX9bQPY7hgqdK1gWZWa26DkvOHn3mzsR1Kz/UhG2dSZk/Qru7b89a0hpuDOwkYYGzgsM9VZ2PtEH2ZR34wZtt7PIq8AwJJ6qzvxFHCTn4nWakaWA8bzQMK7o8NgKvHCqzpMaEfWfWsvR6wwzMabfXTaJVbOmVGOxWGA5fvz3qotFw2rL1gQ2gYiTpkl3sXYaJQXJls8i+xmy+GC5odKob2B2rRo0k+w8F2K2JL/AKraYfiPMmHnV7Rh5amcMaw9dDtK55mG12ZhmoQcN5cOou02dVWWXQ7wXYuJDPqq7Wpf++9VQRs78iP/AI4ZraHrD03h7QwmtN1i4MHewHgWcv8A4dN0RoKjBqLMxp/yBgvUd22raVSpf88t8n/1iOG9eNL8NyNVnabSqrPqUIPPvWPSpZ2NUn7Vnuzn7YCeBhh4wdC0FY4nPvVlpzQCNh5Fl6DdT4UsGJUreHZgqSZJbkyjfXNa3q2jhrUmO/5wzTxgs/wuqrdVtdrCXrYLUcr8VK1JUmfCNtttc3vOH93d1F2M4Bn0N5rm/EulTo0N2XodaqGLyvDe5OqRGzYN4GDzMADrXQLhv+j3tctKiTrc0YjGDOeG/wDQsYEmj1WVLnS9aBmeYAyexZJ6paM834j7wEHRgtlvQE/i1wusl31VQ7A5lrv/ANzXN75u2gXa7zVDoAUp7fnh3rCSak/MPUf3p6G8vTpbWwGac80bFVY1nT71YXJb04OYlU2lSeXDo2LFQ6lOpr4HFcTsy6rjnu6kqcYNeQFiACZgGpmG9XEasMRmtN9/A1VRpjAZv8oB4Ks5yXMN1x9sAaDsBZ4F89WAP2BqnmG/Jd1H3wPyYdigg92ApcaBKkl4YbE6WqWB+tfbtgGhsBRJJxYHsGDOR3o5gOMjyjew+9QTDkzBxzvWguI0l+TpOP4bD7FqKlPcmU3UbiGGzDYsvSoes7m5mhCvTmZTtN5swDNZ4EiMBh1tmCsoEDWmNZuGAAahM5nIDUcM1paay2fiLMasDB5jTWKv+SwDoMMBhh1rWs+DFdfc9kwGZrnkllytnIl6mG9TwinBlx4wbb6zNR6l+0Ax2MbFbQIz8OZry94MBnsVI94zpuec12Mlrb2Z1Iw8gZrYQ6l/MxvnFM9/YCyVEe5B919tvN0wwBO1i6qlSpRxYLmj51FwvazevmlPeBUojwH8jKP1lbB/0o1nHnokyGdS65HeaiRpLAOg+a5sC7Nm33pQb5LMcDzzDvVVM0HppnFz0s9mt1qRzJzPDwBR3g0S01uHTe5n2nYru3oFDPOdOfMHTDYAdipo0+Iy0behnIMO9XNHZYCGB4b1kgR0rWqQOc9ogfeaGYzyz+gZg984Jk5LmemZhs8iF44rLQPg4Zn86lqRg2BJmpaexiLv2b01MqT5iHQCajPOPOq4hMLWgvbDYfcPDDZ3qPPPANBBg/G8QMw+dNAbcl3xGwz78FeBE6wUfWcB0221IeMw9p+RCyyByMzWikgKlKjRdBvBFz788gff6wSOG28Wm25gkFvVHLUiP7PoXOLYwOS0HzqJMZ0ZAedsFYGzVY0AG22DM/ObOGCT0OKY+IZ6x9a1Fecx8zBvUDD50BvMZ6eoo7IQTlaZhMMwPybFN5NsJ/3pvRj4Zh86D6FNMIuuYLRfZ6KdJDkfGd8ioq9JfZpsJhhzreVxPepzNNhG+5mYPbwQZ/nMHXQb9r0GkbzjLGzrRzIcs5Gu3BDB89mCRhgZsTmDZdDsWeAEAMwNxvrXQOF0N+ZW9N889Df0LCQOg11PgzD1i57vPMzWjndoqsmDD4eVVydL5aOcYwN7DPDYvG9KksQ3zg1mDJ5gzzz+ReqOK8z1bw50MP2qYwH5F534kT6dJuOE/TXPYRgA0FFXqbLhuhLgxD5J/wDZjDvBZkzcyPNdWeu2lHaB0dhxkDwwBkA3guXyQza3+171rFtota2lUpdNqWpFbAzMO9bimvRGazEfrkp5k8N+rvwXPGdSNiYOdatg2eJqGZqLtFtLSodGeuFoNTCI+Z4H5EdbjBGkVA6PO5yPFDPow3+RZzn243tFozhv0qPpzgMI9RDZohnmfkUWQyMx5+eTTD7EaMDe/ADSpQPhK1A2YH1qt1gB3DsV1SjYDe5vBwOhIWnT3mJko9BdI4RQIMyrG/XzPkqdAzMw2edc0gQ8yW1pv3amusOeyfDA16PUfCvi164t9q3Lfof82UEzCHT3jwB6VnnrvHh7EPIvJ9/3VcFz3RdrdYnBPqtUn4GFMZA4uAdgeQFsKDdTdvUaaANzAlymdANLrA/rV3wlt637brwRarUo3MTYZ5vAz2fOaqOYcZ4RUe0q3xLt+j3w/JjUKoyeVkvRzwMDPo3/AFr39b1nnwxoZ0f1zJqUKiVL1vGlyHszAPIvFMy3qNbdx0y4KNTalMtoKxFlPS5AYG8ee8F+idSo8ECZcw9uGa7I/uItRT+LtDrbEJw7LqtbPmQfgAEboMOh75FpYFbo5ypDdKghGkGec8IjOwJXfmfea45dvD2o23DvCuWPLN6oVSmmEaIfufoPNcfoPHLihwEt+FYFRs7RkRczMJwZgfzq8Od7QOpSw71W3zxIg8N7NkXjWJexjYyz3vH5FhOFHEK9btpsgOIduRqJU2MOWwDRCZn5Fl/S3vm3IfC+RY7dRA66/JYwjh1sgBhn/Asbaw7Rwu4303iXbTVZpQfew/aYmeZgthJvbbsiPB9a/OLh1eD/AAo4jUS4Dc5kGGQ5xqP16Br3V9vLVk2o7fDFSZk0JiMcp54PIsdrXx3/AE2NPiU6pVmHGmzQzZjm9vMFI+1sU/aPmf5F+b/Eu8Jd+cWqhetGfk8lzOcCRI6wD6F6l9Ga/JfEu1K63fARpM2kSWGQdANmBh0fwLZDvoXJEPolAk9XoOHtFlfVVudgGH0GgOlUf3bZn9Zq8MlrJrET8cFWvVjf7cFTvUpgPZvn+tRzpvknPLbAvQrf7wFYM17BrT1FjOQf93K/WnggTu+WBpgag63v9ogOe28sPVZM6lOtPy3+Wp4Zm89hn+hWdErbFYgBUYgSQaM8A1gwzTAuKrW/U8M5XIyZmHZHDM0zatw1ypQxOuUMIZmZ7wez2eRVVznOepbrlOfMJDAZhgfWs5Qb5tiHUTtj1lnVTADeDDrf8i47hbrefwRwzwkAfkXmem3PWY1Z9eWw5JCdFmPxXo86TmAb+jBddjXy5SqXIlXHKZ5thk+ZOOGxFOJcJd9wVB/z17/PmvRU+Sbz+a8z8JZ+jTpFfwzwnmeC9APVLWYaf08Mw6MwNXDJYBPfD3ieCtuMlqGazh1L501zwLoHQmbnppj4j64v6S18uW9QafcZ25GrdPD7qYSAzBkzNXwV6mnWfs/qffQZ5ow+RZHiLSrf4itS7OnXHyHq7xnmcw0ZIf58P86npPDxbfNV9fXHULu5QIceaeei10B9Cy8mSa6hUoFDrFG5SIwzDaAzA3TPsA9i5TM2GuF1eCZmGBZpozzma7/5wBNAD7xeA30dakaJgqerjOjvYa+sAAnf5uxNsH3jz86os3MlK0T0s21FwJPM6LWn9aiPPP5HqHsP5EBm4yYHn3oHnm3i02w2KsBHvTR7EeGC2XCu27Vue43adccE5kgIb78CODxs6xh1hmHyKhjc/wANJdyptmWPWGjfp3BapSWgPAzZqUk96jyeD9se0f4c3+z/AHUnZ/gXnc6fhtxfelrPs+wcwPzrrx8MbAD29NvmB9YAf+RAzwisCpTY8GDdt2hNlHgyEiAGGax7j4eZy9mSbzWD7hp0IeZ564IarTXKJXKhRnJbMzkpJxdVnoPA18wDAD84K4c1pMAzhunpn1griNVcz0zc2LPmeKA3nw9glwNXmxJ1WM/FDfgnXtgqqjPcs6b7DYb95gppyQMAc84Lm8KGy8GXjtgYeQ0FVCqzOXrk7N5qUeizIdezMwDs+hAyDjzum2pYUeovRWnG8OV9yGfQtw7Qar6krMes8qEnQz8J7oPZghhxqzcM1qm0qDJnzXzwZjx2c1ZW9Qc5oMVFgzwe0TAAzwDzrR2BXq/wQ4oBVWH9aOZ56p7weD/IvNiXwf4G3NxXrcihy6iFBNlk3z5uGeYAGHZs867Fwx9E7ibaXEE2JdzGFtHUmIs84LxxjmMde8DDo7P1p4PSuYrc86xyNHOqsHytKg6JgYZ7MzNbKj+ktfFHo1Tfvyx2Zk2kQ+dZkU89GN9Bgm2D1MdSY6NNlkA6GgDAABJmYw94eovGnC70iuP3EKa7WAo1Nn0eKehJpkeNhnmGewz6PrXqum1ht6nRKrOY9Va4ZnHlvBmC9UXFG+f5PbGqF1RaczUuSDM4hn1h3rxPwfr1sWlcFdvRhuSzNmzJpxqYyzrRWWMD2fx/wLvXpM8RbDDh8cE65PZmzWT5blMDB4Pn+TYvO8x6m2faEKsVWq+sgqgf0HojYB/9gqgGzQWLqGpwXwZePZKnyDe0dEzMMA39GAAonCu3pdw1St2dQ6HDeCFJObPqbTwHrB2MB+dc6t6ZWbqr0e2WKjy1PlGc2SznsPBdbonGNuzI8eDEtHlnWZOgbtMZBkJh78AMOvrNej0hZl80qex9nJVV1qrTwAJIAGwNnYtUzoSWtRs81wLh1YFf4UcN7lv+7oOtWKuyEoPvO8GD7z8nWukcEJ9SuGyPtdUWAB2qSXzDDowDZ/kNZxY2uG9PcttQPHguOekb6QN3cEPs0xQLOjT3aob4PPSwMwzDDYGHfvW0WOsGc5nsUhkNZoDXP7b9IHhzVbFhXjdU5miVCaGb1JDe8B/Rgs5J9Jy3KxNClWJb/MyD76ieAYfIHetkO16ODXs1XhRJb3u1x1nj3eoTc36bSnon9nwMP481ZM+kPWP/AJRhh/3k0IdWZoj8Z0DNTeTYPrbXDJPHi/5M1p+LFpsaOHXHNnPP86N70q27YqkeJeliZx5QZhIph7w/IaxuF7ddkgcZ3DsVVclyBaVEl3HOfwhQsNY/IBnh/nBHaXG/gLfJAwxdrMOaf9EqP3Y/415h9Jn0mTedvPhBblnRmaeD3JBVufMzeADzMwDo7FGFvVAVXmY7T7b+sD4AYGHQYGhOYsJw3qr8zhpaUt/2vqeKH8C0fOZrbDnSnpijnMPzqRGjMPGGZ4B51Zs0emyRBzDMDDMDUqZ/mHvOkEl9asLSpXkMFIZtWnA7515tbCHVXwuA7f0/FCGEozzVgyb5q1qQWVR+KtEpst9k6xW6abMNrPfgG/8AyGtQ9SqaftGE2M/DmS4wA4DmBqczWJxu6hv5mroKJSgH9l/jT3qSjH/RP41jsRGbhfMsHHFLrdep1NsW66jOfwjsUSUZn/2BqVDt62Q9vFksh52jUXjHDtig8Cb7luH4QUGVmf5Nn8eCuB4J9AOY/D9JuhAx/SoE1g/owz/yL9QDeYAvEcX5L+h/WJdH9JTh1LiOYc1MOE984Gwa/U6vSXGXzMDS2q1nzILLXt1STHtYDcVUdbMA1D3mqqTW5cktTPD5EiE3bUUo857OuezNWVejQc/AbWPptS5N0HM88DVsdS59eMShmxzAamwFZVi3qbPpocjhzZ78zVKAK959g4AN7AMA7F42c8rFsVGmyMJ2jh5wPNVp0d/3C0VyRn3nQfiPmfnzVVGkvxnxNzsNawxBAoNxvF4DGAfvjwVsFKvWMxqPwcGg78wVg9cOtHz6E7Au2VocrgBh3r0UkCpcy7pvv7FfeoaBMDXfY3msCExt6qOuMADIZ9ALQRpL/L+0VXAuI0OlUGZr0pjA/OshVWarSjkTjYwjvvGea08N7WLN9WFboPrWlhoLHaMOfxq9O/EVxSqrLOaG9PBZj/u8A+tRLkjfYO16ndU44z3Ks+CHzqtkOZcV/SHvij3QduWJOjMtQgDmWtEDMz7+tZ+Hx14vG0EsK5TZgH/1BledKrVXK9cLtZfM2ZDkkzA2jwMFcSbwqWkbDejGz68O81ycr7HDDvx+kVxbje0fon/awwVJeHHLiNdVtTaBVZdK5eaGB8vGwPBcBO8NE/Elgf0HmpH8pD7zseK5rHHM8MNEFi2xxf6OsQLz4jT7Zj0CJeL0aEwGAAzsMA+tZK4bbbgUt2W/40j8XWzNZq6rnfoLrUVjPB8M9h4LPs3tOkyOVccM2j68zzTxfKY4XYNbmaJSn3P7GAKCzMg02a05Lfwz6FCmT/8A7XITv7KGC5VGuquSWgf9Wxg/Oaewj1uZ6ICMw8GuGGB9G8FHPTZDMAZP84Lgh3hXA/otNAPoUf7W1w/Z+rT/AO7Lm6nf9Dupm291gyq2TTXHsPVtV5YA90b2z9C5J9ubq/8AyV/5ZAd7Xc8ft6b+dnNOnyj6XXufuOH7SdAe/uZO/wDwJ1mvMSXdCdUgD6zXIguG7jH2lK/8n/tpmTeFZgNG4/Bprx/3OCdK/odz1qbj+1gf0GjAGMfAzXCaVe1fqWq5EodNDQ6/GMFpqbc91SWs4jED/sZhqOlH0upAD/uIrx/kS/nX3beC5L/KddUOQcR+nYOh/wBZNO/ytVwPbsTA+iTmself0Om8zUpPgdaX2Yfk+I+wuWfy1SvxpP8A43+wl/LVK/Gk/wDjf7C6elH0ulnZ8UOw2foNL7Nmz+y1I8/OYZ4LmX8tkv8AHk/+ZRhxmqR+w5k/+8/7CeOG3n0fluqlbc6S0epBB4/OGxVXqr1b+1UqSCov5Wq4HtIMlOw+K9VkyAY054Z/Onb58MmzpVVOGDrlHrlSpTvn5kwzV7MvbiUzbMhiDf71Sjvhg81IDWzBZR6ZUak1pvnnn50o0CdGD2Gf0L3u8r/HD/oxXDSewYTaHUoLMzA8wA07cln3HWIHIxA5ZrPMNma0dB4esQ6u7XGGHmdfPwsMABasIZgfiOAH1muj63nx8LzvP4e3UDum3yz358P8ap5ln3jAMPWVAmA053gGt/gXqUAAA/oZ/WaA4cQ/dww+h4wT62Xx8LzFJtKczTjqrDmbQdYYYGCzT2mvWR23BPPUneEfWGtsP+BZescH7RqueD9NZ/gW0eztzX6zzbrN/wBSLMDPTXcD4D257SXOMGg748kD/wAas7M4IWyzWZFVg1w5hsM/s8gMANbd22N8LggeCvQfCV63zt7OjRXmZb+HMgff5FhLwjU0LqNyLaMaNyRmB5++R0SqyqVXmp0HRjB3xze2Yd62i0YdQrFHocYDqTFuU0HQ/co7YrEWS0bHqaAzp94ApAT4Nw0504L4PBhnsPNZe1ZLjNcOJ5wWNwjDa1utt0qkuvuMRsPoXnyvPczKdneddL4tV5imwhpRt+3DPNchkz2HmsG1xqPMvbV0XgbZjF/1yQxOnAGh0NGHWue2xMixq9T3Ki3nHB7eC/SKg0ehzLchVWh0OGzrshvBnA13+syc9pvDGJAaBhunMvfWqfi1w9qMC1/WTEEGQY68F2f1bL/DUC4bYcqtuVBhxvszXSyeRGZO+PK8h5r1bbFNo9So0SdhnmyC8hM/tDrHkPBeneCBzqlZod+geCu7Q6HTWaVDP9hZx+cFHeprdKmnWbKbCBIM83mvcvI9F8PdoDeMFCHJPSThsXhZvrGUHJ1il+2ZZ6Hg+f8AQpfowVKDMsA6Mb+DrBma2d52lBva3JdNffejS8NjzPWuD+j9Ujsm46rb9Z2H5/1ot2i8LwO3nTA07arLE8WrgNve+B4GuFXzJqN28SXYjh5wmDAwXSqP9raa61BY/Z/Ii3Szk7lHOSHnR+ralpZvsGCjnAfz9miC5htEEkDUc4csPdpBGfDrBX+hDnvG/hFTuIVDdlxM2ZsXxwMD614v9W1KjyDYlQTzA8MAX6NADns15l430ELMrkiqzm/u817AD86i2sW4e9WIsmObD8UA+hSjtKnVKgnUgnPBh2At9AtXh5WLfO4GHAN0AzeA9m/yLm/JvyWjiRTeBryZ9CLY/Rzd021pZlk1WNS/XAcs9HD8LrUiBZ9SB/XbbzW1A50mjO07QZjfOChbltKmRID+o5FCT9a0VKrEF6pR3/U8YNA81jZ8bkJ5xNTPBWtvSYPrJph9/DNEvUdmcZrDjcvRn6OcN3zhvXYIFVptVa5qlP6zS8zW99laJManSmGTDvM16WtisUCt0kH6HyYYdYMmreOdcdQ5NqFO+hZ9486lTKiHQYYLpXFS3vtDakhhtvN0OheZbbvycdZiUOpOdEnA1tFj2LDhwZ9E/nKKDzXeBrgXEKscNqbzcGgUY4cswwPAF2CpXhRreoIHOlAbR9gGvKlYpVRrFwy36VLZCJKPZmsRy+YABKdcYNR2dcC3rS3hZ9ctvSfcYB4HzwMwPoVaFNMy8TYHnWNwta0G6qzbZ69G61cfy03ibRsGwYZ94Gs0B02N/SgP6FKpsmjg7n3rj5hbSb5v/rYr8kFEO7bqqrWhWLjkmHkUjOlH1v4Amp8OlcvqQZYPGHXgse5bjO+A0B64Gb4dAdip3g3Zp7P4J0PGE219BSOzp+8UuNJishhmoJgEZ3e4o+eZIlfczE924o8mZ4R+GZ/QoIamYKzjMt7DPrQAyyeG9NYH34ACdmPGz7MDw86hZ6xeIrEuMyBu6AHn861EA2IelqZgHnBZczbgR9je9WFKZnSYX7UYNd6hSRVZPPz/AAA8IO/zqnnvct7NszPsU3MGXTYDsVgZsPQAYbAAQSKCbDMAJc7YXkVFDCmzJ8h9sM3gPrUqYzLnhHpzb+AZ4I41Hbo5cq51toLCGzrSAWwpsZvDTbbWcpoN7MFuIENgGgD8RYcor69p+qDYcMwA1ggBum6oU3eHQYZra3IYHKBhtzYAdCy72B+AogV4TH4DAeAZmZ9Cj1I4snSbcY0dmas58NvFpzm2QAMMANGbLDLXN+DJdV7WqaO9EOqR4LDaauGiSnqy70AHzq9pVNbk1tp9xgAw8isLqpp85qOeD/fK9oZ+j2HEea15dcZD5AU6t0eJQYBxORA/nNM/+v1MZgEzDsNroNU8aNKea1/XMz6DPNZiQFKc0AfB8N+/BQnswPBxI6230S38wUsKbTXjB/UMPkWlJFR4bkl2Q+/0AGxGzMw9malgbbInoeRVgMuaum2vTA88M0jk5taCdPQNro8VKBSpcwzBtjMFnsSOWo0yL90cMJbe8/nUSMejs709o8mebD+BfIaiGe83O9aIaCSbB0jAJbOQdirIwMM5+Pnmq/Nw1IpUwIDUg329Y3z70X4R+iRmbae60jk62HhgpoMsPOtYbAPYa82uyZZzjnh19n0KwemVKNRgqUGWYOgeH5EnobEbDQcMAcPYCs+WiM0PlHMMzPZma9eKSBWH6qJnUn5hgG9nWUgAbeIHOvYgk0qc90GANd7yuIElimsOtmYScwwADDBZ2HgmPzIRweUPwA9qAKkmAw9F03+xI6rVYzRhvBow6ASZeDldRxRAr3gCeTQNngDAYb1IZZiPZsPvvGYGGZmahcyHNAbjBnvUo5gTBdb0A3mgs6lMwdjsMGGGHQHYqypT+claj+88MM1FwwLUUdlnWmtMG/ogZ9eHQt/AkbF3jg/A5OktefoXF5lH5bR8fPM8NnWvQvD2m6MCI2ibZ/0nJj7NBtylMOftUk3sPPgC4PDgRJgnrywZqHzmukek/Vda9aZStfP1dAzw8ma45GM+aB9GK7o8amvbJzBvOxTzeMOwEpLNNwkHr4GG/o2JGbjLRuNvmyZ4AeHeo8YGA8BzoRsajQJU93CI3mQb1ZRoxycAb688MEMCSxDm67B4HgYKPD1OaNzXeB3DYYHhgaDQT7blnCha/wC0AGD2xTbkuGm/Z6PSmH88AzBTaDXqxpQqiw5m7omDwAG81z2t8qBx+U+f/GiFYHjGfiK7ZDRaZw7wVJyz4Omxp7wDM1dxjzaa0+zYa1W0tHDPBXEyY3qtBElA9oBgYeRR6OyHL6nyZqsmB/OTr8F/M5WZ54LiatnDZ1oAS+h1BZ91aJ1uJVaaE9rWAHnjPsz6FnTrdRjUtpt9sN/eakWZPpTMiFBnRA0tYDmSHujPsVQm11xIuG4Lka9ZQaNJh0eKGEbS6APzma9x8B5Nz3twsoVYrE7CacYAkx5B7/kXic7wiWHXgse+Lfkz4lOmBUAZjycOZDrA/JgvbdB45WPdsOJWLAl0F6XKDOTRpE/k5ufkDP8AOu7xzOO17W4E6A/EcY7zwP6Fn6lwfsq+Z4S7np3M1Bhk4saRn7ED+Rau4akEmRb7D8F6lO1GSYYSz354fIgerEW3qzUHJ2elFjc1gAZmfyLaBlLhuRixqDUJV90eNy9BZAIxgftj7MF5HNmlcWrjqt8X/c3q11hnWAMMwMFC4x03iNat1x6NxK5wKhW2Tq4A7MMzZAzPs/IueT5jGkcVxzwjXNdkJ1Skt1ifInRZ3LNYZ+MHWpUniFeMmygsCLWXnqVra3KBsAz+dYrNjmMDbVnGeYw8BRtbRVVmdR4VKbCcy80eGAB2L0Hw0u2zuHtcqc5y4ACPNjMcy1EPMDPf2LzVGny40X1bLcZCOYAYAYZpQ57ATwBve0fWHnNXFj9CKbclArciPFpVchyZEpk3wZaezMwVhvXgaiXIdsXVSq5FPA2JIH+TNeoPSH9IGnUS3qVQ+FZw5Ltws6z07f8AdmNnQfYfWttuanVt6W9Vno0z5188HaPUqyx4sU3ovMZ73sD6zXRnrYin2LbbRiQZlGfhuJPRqyyGeGxbYLGl4akRwEjtWqh+Cf51G3mHNK8BzKNLiVIMwMD2KqolbptHtk9CU9M5UDM8/P5F10LPrHvKVGP6zWHuSzKrbHrWsv05lmEYZsgBhgZ+Q02YUhyZd221LcbCTStdnwXgMDPDzrhtKuFgJrVZcfgHNingD3f1h/qLuv2Vodetp24INckhNYZ1s2ntjJ4Z4fQvOVVtL1UdPqMSqmdPzADqJhseMO8AXzfe8/8AhcNVR7nt+HfgXHWdkI3uaejxwzDP6FqLwue0a9FOsfbh6SbAG+zHZgGBgeHeuT1J71leh89OZZiPgGEh7owVnz79Namt0CdDmBomBun0GGB7F8fh9nm/LpWvDTXjcNJZtnhm8a0FsVWnW9AaqvPT5POz+VZkS+jD6Fnbb+58G5p+czD+NWVBuSlRjplSrFO9ZHFezZjm9gHzn+TYvocvs9Dx26vQKBQYVMf5s2edM5Ul4+v6AVhSmbck4TmGKlDdDsnB/kXD+IV+P3/cdHYprHLQoT28HnsDwXULevNi7ZE3Ti6MelgAa2eYPLt9P3I5K/SyuHIZ7N68OrhOdVaOFSkHJM5M5k82XoT3QHyb8FwK9rwrMCZU7cfo0amyAM2Hjj+Q/nXp29uMfJ0iJVbOYh1VoHv5yjyAPMA+ReZb8n0a6rol1mnUp6NT5T2vpOnvW/NzIwwIA/Umnc89JjrVbMPYZh0GpE+YcB82GHNip3j1g+fNc0OgUN7Bo9Q0Byd+o4Ci5/BIzXS9WrJsZA44pxvYB4HQqcDDBLWPzqLgSJMxh7xD/Qo7L2HQo7x5+IgTAmnJcUilVidRKpCrlNPCXTntZk1XgCkaJ4IPXfB+7YNNu1pxh/8AmK72edjM94Su/P8AOtrZ8aXc8w7/ALnlSTkPvSosaCB/dgi7w6PnBeWuFdSfmW/ULO5vCbCP1vSjw3gYdYAvRtkzL4vmmx7utK54GE1nQk0moM+DGfDYe/sXDMO+r3G1nat1ReHV0XRaU6dM9RU6MxV4fNmZnGz2GwH8C45xF4hTqJCl3/L2XBdGcWlRD2HGi97/APkWgu2q8/W5EG465z9PpB87XqgDOGeH9FAOwF51va5514XDIuCc3o57I0cD2MsB0ACvxDa+a+Dh/R/3KQDPDz/WppvBgOHkVf3o89mn3rr8PlDN5w1IjTJYDpsOKP1gpUOMYDsNei4hgegOfWnjewUpmG5sMHANSAhxHgPUb8UOxcSjUaqgZaGAMugGAH5wUjnJUZ0HAfzUeSDDwab7G8PkwSjBmANq7Gwh1XWaPwP0I69J9a0ubRpwMyWjDOHLkdbP0KhMH4zQAx7w8DQ1KY+yMeI4/mAHmBguO+Jk6BGpVlT6WFVgxDgGwzyQA09szDoP61kTqr7McKVOlfeDMwkhh5OhR3nmKbS9NwzD6DVOybEloDbczdU8PD5GwtW527GdN+m1WpMumGDwRHtED8iNnjHxGCneo2LjknCzM8HjzPM+veaw70neomsu6IHZq9cLFw8O2m4LckwB4zkg6fvwA1WXbdVGjW5b79uRIxx8z5mPIDMww71zxmq8nTXW233gzPMwA9hmosmSwbugwxh59/WauBfs1vkKydYobnLO55h34KTSq9dVYqPiVUOY5kJoSD2GBh0LLg9gptNk/fWvrzWg7HXuOt/3PAO1bjupl45pgxJN5nDRYBekrG4ncOeF1g0+yqM/UrkOnZ4O6IMgee/rXioIAPE7VXAA3u/etB/K1fmf/F9iqb/4xoPSd1eknxGntOsWjZ1Epup0SJD2sYf5P4Fzn7W33VfWdc4jVkJ/Kva0bPoZAPIHZ2LmX8rV6GH/ALD00/8AtjUepcSL0qVLqFNOxIEZp+MYG6EzoDzoOxsyYMmO1L5WMYPhmB4diA3oIf8ARsb9C5PA4o3rDpseK3Z0OeDDIMg6ckw2AoX8s128/wAiHDyNJkfhR5msa1ux1esXPSrepztVqrcwIjHWYM5qiDj3wkx/a5IH/wDRmCpaxUrjrFBdiXM3R7bgythnveeP5MFn2WbSoLshyh2wFbqD/wDS6hsDP6OxY7Xhvf5eOFxn4FVkn/3ZYeAbAOnBo056quyjzZZCTrYB/kQMzLZOZz0uhybeqp9cuCzrRsP7lWZwLqPfSjgVuOfvqfhn+hVswCfQQhx9e9KrQaJHDrAzCTNP8iwt51uzpNLBi2YMkzA/9IyHt5h5MFdBGpUwzbwjaodYPYZgs1fMBiBDjmAMhrnhsBUOh2TxgYtiBBiRaxVaC7hhmyfMxZJh3mHZ/trvVK49yzhvSnLVh3Jp++oM8Mz+tk+heVbGhxJlBlsP+4e8i0VNgRKO7z1K+7SPOyGCqLRcO9h6WlneK23ZVbB4GT1mXjADA16Ls8OZtm36q37KVSorwfoXgGYB1J035b+bp9+AAru3r/4hWk0EG3L7qsaIx0RDezZ/QiMP0NZwPvUjltubew/OvG9E9LS/Kawf2gtilT5H4zObP8C+2Z6WN6QLhqsqqsRp7VQ/Zojx4BGw8hrm5g76SdYrFq+lBYU5usRmZcKGxg7IZ2M5vmG/5MF6SO6m7Y5Ji+KxQQeqP7GcSG9m9+TevBvGC8Lq4l3kdYuNyMdVis8qyEfoAPJ/GvRXEvjq/UpFqfybcmcc8AN6WyD2i+GGw/J9az+geqDjfcjNRTPRYzXiqZ6Q/EOwLh9YxXIEz1iAG8yEk3mQ+jyL0lYfFGncVLPC56bFOG6Z4SWXjzPNe8N9i249ZAsV6RV1MUf0fbwl6ATAlQ+VeZe6DAzwVgc8wXJ/SirbYcALoYcc9vyoB9esuoeMeA9wxLP4yWJX6lshU6tsG8fkDoX6nV6qzjd1wc2H0L8fP+H3X98C/UsKwwdLp+pLz+5xf/1ALOxanVXA6zQM1ID94qKTJYeD2iiazgdDi2Q2AVL8RxX1KntvNYZ71znXlq2pUx9lButbxfaKRht/agWS9ayEvWv4ijA0p9WmqqpRvFaPsz3pqNPczRTJmzUczSIB1JmJGhg5Efz3o6DvlYKPyD8xjw07Dokpn703KDZ2KxT3JM/5RyG2wAMPICl0eYckdM1dSYdsQ6JLnTqNzNQcPMAXN7quf7DWv68fbMJD/gRmj2ZmlsltdvGawOHtUClVx+e89hmfKRs9H60EP0t+E7LABoXCf/8Ajf8AbXjqt3y+zcEiWwbL0d88zB4M9Y/nUqj3bZ1YwciSmaVUA64Lx4AYeQFx8r6nDw/+XqO7fS9pugbdlWdJkyA97O2Bn9ALhV/8TuKPEKFIYrlRe5LrCIyzgAGsuHEi3DkNRW6NJzfPAPGBMz+IVGjOusS6GbLoHgYHJBcnc7/HD67FSY1SZM9mCp5J5ytOUBh5D1lq6ledGmAbnI7PkMFkefiTJQNg5vM+9awq/wC2qs84T0rDDQMOxXr0ZwKvCwADA8DzUfBsIVTguda00MM4EQ/3KfsZbP3nAYnx4Us9+myujWTQbZpVoNVE6PD1QAzN42c1ipLOtSPE7F0C1Q5+zZETzsn/AIDWvhyIka9qUzUgp05iNGaf/ZnQDMHlVcRaJSgaj1ViCyzIzwMwWHsaTUqCFKrl3ORpNK3woxn/AEM+zNdQvxnWtR1zyGBrf/o84XIfs84Eo33DM2j34eRWYW3BNrUcivfoVh1x1oIbOcKP4mexfOh0+w5VcNHCM1H5RvvMDVCEbRlNG5n1+ddIu2GGgYA+B6D2eaxUxlvAHG3APA10sG1jRm9jjgACyV8wwYrINtn7cM8FuIG+O0Z94KpvZkD5R9zoD9a4ov8AvO9krPk06BNmsVKdy2uzs2ZrZ29clq0103KlX4z3k0mcFThR2/wAU2HR2M/Zgtu5l0pFb5Gq1451H3wnwDekdHivNabjaswZbBGexc+15Y87DOT0AYKumcOpUb2b4B9a7DR2QeazUifTWHvbsYL6vhxPPh29OZLx9FTaDR8J+m+5+ha6ZAAJptnvwNV4AEaeBg2ieG/1ps+jvhFzguGbylwIeEIAfb3q4AAwRYfFfJ5rfRVtSk1+BD56nVIwADwPMM8FWM3Pekn/AN4P/vK2B01ufQZYeQ1n4FHYhgYZ55qNJNBJv+SGo3WQw+cwQaN+H/0qznh5wR1KGxkB6aVsRmArgH2GCvDm7EfC/P8A4kz/AOMCCS9ekbDOqh+QwW9OjxDL9lBUtepoMlnoAH0LbBw8362POpXV/wDGUz64ub3dYV2cNh4DbcbUQ6bFjR9jaiG1w6dYFgN3JAhVKqwZkabv1jlvYMmqKvXhdsCqO25w5t8HjzwAw8bP6Ni9X8Ima5dvDGjucpGCOwzo6rxrYUqw2Iz4vsRYer5wAF08MOO7eH5/C70ibnih68s7D5MNE10aB6FVGqUOO/Ov+ZAkGHjR3owZsn5F66qVpTp8Iz12WcPOsFMN+M7yh1KNmHZrAvpQ4LcUuew6PwxpbsSXbGDPuZ0QNhn8640D2jdeuHsn+he0wjSzaNidFCS0fYfQa8Q8eKDUbMvWoUpjwY5gBsh8ijpR3MJxdrHri6DbYfzjxQwBY2Mbge0cSkm48/qOJ1mGwdSBs+hY3wto5tnjPYvbvoT8SG6xaUizqrK3wjzjfIH+4LyVJolGZgBLYcA/oNaL0cuJH8m/EiO++39yqODD2/o3/wC2seH+26el+jD0yIBbFLhvNvCbDZ9exVTzLD2D7DgG0YZgakQNAD3rqczxjxIpr9E4l1OI/wC8PNdW9GmpAzVKrTXz9v0fxrOekzR2IF6x7gb65oABqs4IVjk+IlMb18AlHguqGT1NJZPPT01WTAMD3roEmz3DzfCpMmCp5NBiRv2swP6FlHMz6WWp/tV5o9JOgnavEGJVYPgtVTeZ/PsXrMKUxq/dW1wT0ybYnBb1Kuc+hh7Qw/3+hXtlhSPW8xTX6PVW3M3pW97YuxsgATWlz+m0d+t21bktvsAAXSPVUs3Qc0021penWIptaGojBkDDUbWXkssUqUEqqzo0Nr96eCjvccuENvAbFSvymg6HZmaNGwNnBQpOmCwM/wBKvgXpG4xccmTh2MwzNV7PpS8Fpn3Q5U9nP3z0YwBFfh0DWDNci9Jy2/tDYpy23ACRFPMMz6+ha6Hxa4V1J0GIN8QDM/PmCrOKlKg3baEhim1WmycOj7yoYPF9BqXJxzbfYDSPMDVI9PlsynjbfMDA1LqX3N04gbNA8FnJLz+qXifrVuqGtZv+vxo+HPGq96/64fQ5o/Qqemw36lI0N/5DWrjcMeZ6H2fzzAXO1YyZJbekZmhBnfqAt1P4PzmY+uxOjBh+M8slJpr9Kf0H34z39yeaBAzn7+T+tdI4b3/OsMA0z5kOwHj6Fzdl7BOnMzT8j3dwu4kUa+acGmBsyw6wNeZOLVsMUHiM7ymAR3zzDA81zIKxVQ9hOks/3J4KwpUyo1WQ0dSqUmS953Xs15FpuG9qQSwig4w+bzXeBmqI6rQ2f2WsHGm/hGaCZGrkBg34tZjG15Fip++fqOAyGfkBNowuKrds6Y1oPyze+tUj1bl+zzWoCzIkyBzfPAz9a5/MPCUbabMLDmWD8R/rToVJhk9RtVobzwMDwUr1ax7sMFGBbHWHJLWm23+dKHVTh+zDNVIajKWs57xY4gYIAPHUcRsm4nXjY9mGaaAM+hdSxGAA7vwzRhvPw2wUTDOVgfQpwaYO6baAJLJm6DgdisJh4Q9dRz2EgnvA80DAdCAabPN4NCW4ZgfYYda+SWW4Zn0B5ABQIeBu+H2JTHjeLNxWpNgHrSAB8AMFYPMygDw+hUkbUDxAVnGqstnDXfzwUB2HzTzR6kQwU6BD5lp1vXAMN+9KNXoJ5sGwe8MFNh0dueYacsGc1nYiMxtHx9QDMPInmQ1ndTzqQ9oQGpETQN75wRU1lwIbT+gZ5n2L0WdBZc50Ni2APaIa7nQCr6bTWGY4P4b1YTD5akungBmfYawuxh58nmZkiWxngZ96iAcQ2Dfcf3h2KKFbfkuut6AAbBqJho5n5zSAnmWHpWu+AH9avYYQeVBx9Z3PMNTA0B83MaNgGzNoOxeDW0qNLCqR5zDGzWDRNW1yVJivVnkX3NYAQW9UqNyrUGDTpIOsf0iQexZe3nqbAuj+fHP5vfe8ZZLXcCqnb0gNTwaeZ4cuHQCm3VZlNgR/WNDfCTCMMwwR8Qreo9NqLTlDqXMx3wzAM+hZTCUHsHMPk7F6Kx6m6PiciAIQ8FWE+ZLNoGH8PyJmHAfnjIcbw8Dzro8WIRz8A09NSID0TY451qOzGYekG3Lz+TBWECHBjOmZtmZn0K7tCOembuuDZgHzqQzJfBp3B8w2didnxvAa0G95nvUWHDYkzwiS/Zd6wEc4GYA5BfMz8hqVAo9RqWroRD8DYeaszZbjZtxIp4dhqaATqaRt7MzDM80FL9mKqfu2Q+s1Hn0GdAADccjPAfXonngtVGnthCOXWcMVTnUuWlGxEfPS7FpAzQMmBqwja4fIC0fOMGxoG3mCr5NNfjCGGbwGCbCDwX4k5jxjDsVlMe9agbb7YYH2As/GqTcYNB9s8/Ip0CfKM9cNjXzrHYLWlvMYbwa8iejM4GHiHv8AOgkvGHs8/H86XMvsmBg2H51TJNZzjGem2B5h3qlP58MM+xTTece3qIGmbuhmAH860havk4H7AD/OpEN6XGgHp4Ab72GZgpczUBoNPA9PyKIb3QDi8UNkIMk3W388ADsTrzME8H4uwz681Hzw92CFPAuKPqSazCied4F6YsmBhy686WTAfn3LEw7F6btIMI7T/wCGGa6vDB5c4zPP17ixW52gbzQYRQw7AAFSUqNBpTRtuMPH/eqRPmesroqs7DDmphngq2fM36YdalujvahuumbmYdYAg1tbf3qE8ct4zcSh6hmgso2GqOfsu9WBssGPNRNmYYAqo9mHzqwpsxx5o28DMGN+wOxEtGyblNix32HMMOxMzHm7klNNtsAyDAYAAAskEx86jqOHsV3GOV/QZ2/yYLnFJJCcAOynN8iUZhsUujxjDDU61paaycDmAffjMyzDoPvVZTWfFNaRY0DJ6NGdlfhgq2HVZdSBqm6kZnYfi96kXbr020D0D9u8ALOUQ8DalOLBq1s8GHreBhgwN2KGAZh1rTcNLDpt21u37VneCdRqrHMu4ZmABvUegnQ6lCDQbzkSuvWDANnkXQLMh0qiSGrt9f8ALVClveDEjvYPH5969hz22vFoK5QeJ1buo7foNNp9OgMU+lSKiGsZgHQAB2Zry5W59cZN2DVWOWkBMN4w0QAwf+tfosfDdi8LVpTdYnc/IYnsVc5HWBgHYf615p9L3hROCuHdtsU7+aqXDBiT3mb+Zmf5N67MOdHZ4u3BefAyRcdzXPGeuu3qxTgt5n30YA2Z/PmC7Rwf9I23L/up2VdsQKbXYtK3wQPMHgZ359HyLwUzJqNHrMemv60CQ4GZgbO/DsVgzW36DKqE6lVV5mRNZ5J50OvBXf6HkcK94r8Ubn4x8RqnfdYfB6OB6EPSDAGWA6FnaOGcrU5UJOHYarKIbEYgivuYRz6zTsmYcN024LmB+dY22wkcyEM5r8RgAOVsP5E1n4uuwBmHzqJGPMVdxnmGab4/WexS2XsCmzq9RnZx0eSDQRjAz7wWajHou5qb9pKizD5GLUXgaPrw71CZebBVCF2bzBxwbPrMNnzpmNMls+PFMw7DA1VRpj4SA8TepZxjDN9x/PMz6OxSh03h1xUvS3qXItmm3VyFHqJhrMvdDO/r/jXtv+VGzuGNjUILnuN6Y6cPwXQDPWD61+d9NkuQAjy2HMDYPNdAptvXPc9EC54NND1PK6M5P+D5ES7rc4cW7km1u9bVu2vRo77wPwGWTwAwMF3vgbMrMzhpTG7unf8AKADMDCQf3l4POYLxzAvm+PsqdueuZ8zMMGTANgB9a7hwuojHCu1P5Tb0qslmoSmQA3aj7Y8+xX+Tw7HG+39NhtQXzgasJ43jnHMPPAzz6F56vy8LOmPwqHK4jTK3T4pmcyPgZ6z+zo/jWgD0lqHclIuij1x9kI5+BAlgzo5gefWC8zvBSqV9xpUuTMdD3wdC4ub2W/4dge4luUG2qhbnDlgIxyut55nB7BWtgcN4PFqw5tDiXG8zW4WgZy5AG8yB+QAXnyBPp0ashWKqwcwwAwwN5dQgcZqHAo0H7K0qpUea2GBtMzA0T8+YLg+j8pwyt4W9UqJdUu3K5OZ/mHNiSbT2x7yKVXvUbNLkRLfiSWTCNmbzp555qEd22qD802KBPCbUTzeDWB7NR6lfNGZhSKHRrVmMyJrJsnOnHsAPkBR63D/edroFmU313w0Clc2zG1wM9V3oDBZy2JNNe5uLVWHuvAzj7wZw8ivqCzBDhvCCo/s77wMH+c0VbsO57VmTXLRfh+r34YMm9IAAzD5F1+85VZ9oaNTaodVffqU8w2HnszBS+LV+RYfCOXBo0Tls6kwEBlk8HjYwzzPzqLQbbbM5rl4weQ1zYAJGieH+/Qnb84ewbnrMSXTmOfp+jgzLjvYBguf1r60W8/vQ6kzIdc54wz9sethvTr0xiSGnmAbFHr0yjMypcSK48cRgzADM+vBRQCpMgB+rjZB8Mw1gwzBdP52K+sQKaYHKcf3/AELMvA2BrV1I8KabBsBmfWax7x712QGngb9pqKOnTTWe9dkNUoEkIJ01aUTvTyIAzJIwQHG01Izw9mojPkUhnTNFLOg1WdQatCrkE8JEJ7WD/UXfabeFwWrIqDFpb4l5Mg/AMNmD5554H8m9edcPium2Hc9RpXCq45zGByKCf82uvBmbJvdf+Nc3sQ09b2elX8V7hYZFrhtRn9aFTntapSOyTN/2Fz8/8iX+pn+dLsTxC75u8CNAln8Fs5jylQHsHc9PNQgU2Gy2bR78FI0EOe4AB8is2akw8Go22AHn1+dZQJPhZudakUo33nT37ABctwpfSdR4fEz+sEzAecjeA/1/P2Jp6YcZgzfAz+QFBjPNm0c4GwZMz6M+heDVxpObunqdn8aR4SdXm89Zjo2dazT1V0ZnNvqwCpRJ7R7zwMOzrXuHOiV6pPydJs3DOOHfgqcJjjJ6gdaanz35mbh56XYeCdCNBAPDfeM/3q2iFno0/CUDhthhh1p157ws/nUR5kGcNRwHt/YaeeksPDpsBgtAy9Mcw089iQPb1CPU7FID2WoqXCyjSXPedinRj8XXb6wUIMDaBzv70GsYdCjAuvWUv8RPBPnH3qXbFH9akGxbVmzIIAuX8tWRZk1yN4bFVZBnyct/toZ8+s8nI1KqGkbJgYAzhmC2v2SiKquq24kO36hKb9qEYzBabeYZ+3pLkzkvWLcmTT8DziM7M/zrYUqTVZOqxb9Kh0SIHZHZ3n9ZqJwxhsTLXB9xvPxjD/AttbcBhnncG8PG/wAgLHbq8Qyh2rUTf188z88gM07g/TXTivzmcw6wah5rZz/ucV2d2MBmaahybSgaURi44Ya4a5mfWZqNsrtj9E6rIBhipRgM+yRDwNV71pTodZzgmEZ3zs7Ft629aNSziVK56a9oBmy9ngYGmoZ84NPnOdb7O9UjbFVivVXVALxocavAHRIw0ZQB9YLBXVMOSDIaj3L8z4LMjrZDyLuFVo7Eww1G1zzivbbEChx33G8M3lpFtGXtipPw4TuhKBnN4+tnPNXHr6pf/FWf/JmonD2NBkzXYkv8HZ9a3f2SY92wr28wxh3VVWT/ANKwP/JmkFerL3jhOhmH/wBMa3f2PYAegEAWYwfWwmzpYo6rXDaNvnoH/gmmgnzvxP0LcPWfBx9mAH85rE1iA5SnzA3GTa84Gua77GNkc/A/Oul2HTWKrSJD/rLkI9RMIQZn0GZ7zNcfZk+KBq7h1t8ITUF/fHDsXNDF3W6uG/qewJcFi3zjHSHjejVbnAwMD867L6NNH4c1Kw2oNKrL0CsMH9/5t7A3jPvwXm20rwocOyJFuQXJgTTmN7DDPPfmthb1/wBx3tVwt+6qjDZpVOM5T06nRgAwMOg8w610+LwPYEmwIoD/AKVkmuI+mHRJ1K4BSHKVm8B1WLzOZ9AZn/nwW29G+t1G4YddbYfqs+jsSfuFQqOz8ig+m96/h8BPUdvwQk/aWpMUx7Pybz/yLvix+b8x7Rprp96/VuwOFcW5LBtS45dzgHrGiQn9gfuQ+dflIbPMux4meGpgwZn0Zr9CPRU4lzo3CC37Efr4ViaxXpVPZwP2MIMD/RvUX+sd+h8FrcANRy45735AVZVbAo9NDNh948Ow1qAudiNCNjXwWfmXJEezBx/NSKEIEQN+mmjwAvAUg5MV7wNfDPvUWTTcB1GJ2sHyGuiGQD54OxOsnUng1OUM00zGlvHpsGZmrXlqky0GpENnPzog0EnR9opAV5sABtxsDwPNVUmHOePMMA/OkyEpn27bJotcBc4Mn4ikM3IwfvFn5LLEn3YB9CAKU4B+A/miGtOqgCxXEW0rf4rlCYuqq1KG1C6ORPAz/wB8FLeolfZaN/Uz+RZyHeFDn6vrGdGgGweGDz2eaDj/ABm9HKx7YtcK5QLnrcnxtB5qQ9swP/7Bc8onAGzqlwTrHE1+dP8AWsWSYAGtswzwwXoviFc9s1uzZdDp0vnJB9GzYuOWHPfDhfxFtV8AOOwyErAw6Dz/ANhPLu8W5/UrYiyaNSqr52QY1g61RPWNTXrjdpT78k8MMHjNdNhgw9w0iPm3noPBh+tY2ZMzuj1jp9gBh2Gvn+XZFsPXrYi0qacXrw71U0pnO4Y8QPPmttdvjVSbLD+/WPZP1bV49VBgHt+BgfkWTZtYGgYV1h/DxwzAz7FLpumzDabbPMA71EpWg9PluPmANaOwzNS4Z/d/EMD+hMMl1JPRoZn84AthwxmczSJETyHgsebzB2/I13wZw86vuEVSppyqgwxLB73+z866PDFEZucJ7VTtylcOZM+nsScJIcyGBmtRdTLb1uS4jDeAaIGALnmdAoPEat+ubnqtNaN7mozIbAe8/wDkXSOZg1Whuzqc/rR32TwPzrf/AKMeNz0Azjq4gHL5VrlWwPZ3gqdk/ASgXtRqCRxZ3Mgef4K+bLu5lhMgTjkZ8pG+jDYq+ZSn+XdfC3KaeAZ+xTr3Ei1XndTXkh/2Kaevy1ZLTzHPTAzDDYytWHhEpT2swDmmAH5AUiq8ocI+egnJDsVfSnoOLoRZRyQ7DMFZz9A6abbj5sgfeC+X/wDq7/CCzvignobJvSgiMdZ9CjsmwEcAYfB4POnYb2jV4TmfvluL0KO+Ht1cBAthlgG51Okm73m08jnnotEfkDNccrd81WZKdfY1o0c/6P5Fs4e5taPxLCg8ww5Qzea1jweNVtb4i85WdeD+xeRc0OtznndNx88DTwTNEF1xbj27lQabY95tHO9VVIJHeHPgGaG56PYFpRWpU6hz3tQ8AAJ4Ljlt3zOok0H28Dw7DBO1i/KrUp5ynAZMD7OtXsdbgTOG09rUYo9S+jnN6uDpVlAwD/q2sb+znAXEmaw/m1LYfD6ADBbig3zLqQR6U/EN7DoeBcFx2Ozh5m1jM2doGx6jqW/v5xR3qVasbBw6VWMD6D58N6hT3n4bWbCupMzO3I8rQA+gN4J0unuVT0Oxz9pTax/58FLgUGwGcJ3qaqn9cxFVaVTQpuuDGBmlTfGpEhj6F04cFtMbNovYPtxKlgYdjyj1Wm2XJpLrcqJUjAA+RKHpnDabDsDBOyWQOA7meGxUhj2aDY7x6f8APf8A4wJqZb1jsu6f89/rBO0c86pp+dKthhNNZYdPb5aOyb/rNhterbVuq4QjmeYRDwNnNaKf6RV9su6Eus1Vk/3MNnNYWjmGkGfnVhVdfV12Nma6YcdrKpXhUbta1Krel8nn2A9gCrY1mWV7dus3CzI8/M4IM3zpZmZ5pmBmbQOLpi2K+A34A6cHiTdTLQdnOAslf8ly6vV7FfuqfUmovgBrde//AOwUipftBrOVUNhueRXswCZwZ4Qh4nrK5Az8mBrNcS7Ds6iWlHrluNyXnYr3tZAYHguh1UM6XCnedlVN1QHKlw8qDDDebvLHgCiyHJaUFOqUXOLOzd/Cw61nIxxI1wR3Kl+zsPBmCDh7J5avR4jneBgjqsZg6vN1w98snTt+ndhyYlVtKmP0NvOPohgC1FNpsp6UDfKPB+RfnJavEK7qVSAbpV/1KAH4QPbFes8deLEbYxxJqq87mN8L1H6V1paNpRLj6zhHh9HQvOlpVv1VWaZWXDwBh7eqqpcUeI1zterrnvWpVKIfXHkPZgjpQNvRdBbRzIv9D9CrYvy0bnobT9KuCG87hhgBoZgZu9a/Oem3nc/D2qHKtmpcm758M1rab6Y3GylTwflVWBVWvwZbOxZG3u2HoRj1JT4MtB1mfQuP+lLeHDa4eFtToYXjTfWsXx4YZ55ngvK/Ev0kL14nOtMPwQpTWGBhEePA1zcI1sH7eo/fe8HVk9/DtFgeksdpWXEo32OemSIXsXWeg1VVv0tOMVS8OnMQ6O18jOZ/rXCuufyi3UCzwBrZKM8/Ourua9Kiqtz3PWyP1zccmTqdYHMPBVWjTg6zZD860tVo7dKa136OzJa7zAOhQo0mgSfYHGD5D2Gnc6elBZmQWT9vmrCNWIJ+wP8AgUg+VjBqNtxnlWyZ7Bn7AGQ8gKL5nqyCpRc834oPH86aOvTg/YdFkPIBqnemBieCjhJ864+62GDr2uZm45vMzVfJZim747Gf51sLYgU6e7oe2VhJtINc24IGHyGCfXa8MfQTo8aQ02/R3pn0GtQbNpH4jdvnGd85vZp71OxSv2+IAAfk2J02bfP2bhgf1rHutthE9SS6l0QYxtefnDUn7AWqAfzlKOMfkBPsyYkYdNjP9aF6ZE9o/KAPrNO6zCkmWfQ5I4UZ+Tn5zVY9aVRje0cBbMIbBhmxOZ/Iar5J4Hg44Zp3ouBW3bdvn/pJh43fkNWdbgUMIoPwYhgYd6hQ6q5G3sfnBSzuHP3YAndbaIj/AL0GBJOS1vAwwTUy2KVPHUiMPBI+RJ6ZgXh4AjZqWj3q+5jhBh2fUWQ0/vL319Cr6laVV3uAwzh9a0B16XnqNzjS9ZOPB7RRswysZmDGEGJ2x1TXreqswdemwc2vrVgzR+flB4Gea6RbrLFNigw3sW2fKMOFTIb8Y9B8MHfIojMN950GwbW7uSybg9c+sTcB6J5wRBAYZDxIoAtXM4kzoPZ6YGZ/IgAHA8MwMDVhAnt0cjbiYdHkQnJ5k9R8N66FmY0Pxdd9vwk1M0wkeH0KacmIDGnob/PmqqT1eG4gUkwMfDc3ppll8z7EozL8ktNtvNSGWdF023M80E2HQWHmnfEw2edUR/c3zgt7wDvV1GqrlNlA43vTzMNuqyObCKDJn14IpTw41VnyAYi01576E7JgSoZ4S28D8i0fjw6XI0JxsgHYHeszzLkkg1FAkQwchyGpTgAYZrXQzoczPnnzB0AzDDsVJUmW40VrDrMM0MPZ4neaCbJDWHTWgomuDAA2qGNpmS0dKeYZwzfAPrWA0sB7ANNzzpVh5yYObHQHWnmYesQaHQapKxPYhmYdaxsUVNt6CbstxvAAPeqR6Nonp7zU2fJMA1GOvsUijxp0yA7K2bA71cB61YDkypNUphyNzEo8MM+hbC2+FwY1CJOrDzNQzMwiNBgH+Nc3t6TVYd4U+pB1sPbF0W9r/qUCqR51NYAJAdZnvzwXJzWM/Mk1W25B0qpQQOIHnWcqRxJLpvsBgBrqFYOjX5bnrxjBmp++AzVFbdt1KBMj1kIrLwRfbAfRgqi1slDmMG1gbi+HJAFfcTjb9aRH6bEZCObOzAFijkruEqSYH4iepsxtl3w3MM+tVMmS4YYJpl7dgvcMmgmRogSD8cD+hNRmTkyAAPOomexGy9gowpesw2JMrQz6+9QjprYVE9A80zz7nu1IZmAHXnmowLNmfyY4aiUmTz8jX6zwwVU9MbeDBSIzzgB4agVlVkuBgw+3gBmogaYG0+3v35mHyLUBG1o7r/kVLz8XEPDMJAHve7F0wEc+dv0ImAH8itaaEqTFwffDDyJQJkGHTQOdEN6QHRIDoV1TaVEk0vmwfwMzz6M1mM/MZYjB97bVFPnsMsHyji1FSjMT2jYcfwd7Aw61jZ9HfjOmw57JBqGabcc+A1UX4mizhs1lXvScD0H3N6XrVwIrQBKe8DzmnfXcGZI138Dl/QvfCQaMvSdNtjZ51C1twOeQFLkycxPBzBVmfnNb+BLOTn4YdaZBl97PBtNA8DPtFNgQ508z0NHDDPeeC8wI4anenmZL7LuwAw+dOvaeOnpnq9/yJMgw8074+Bh2KvA3vCsG3p4OMb8Otd4OpHb1pTay3/RYxnmuNcHKa5Ginrh1vHh9C6HxjqTFH4USIvvaiYMh9Het/DF5hhgYOtPg535p7Wz5j7rrOn0H5FZSXoj2DbETRDBR4YNvOg+DnhZrmptBBbEE4Wvzx7wz6Fl4zPLOmxqAeB4ZgukBAiAw62E5nI+/PoWC5B+NUpDb+wGz61p4Ul6ODQMedSAqsuHF9WxMAw8ikGDcZ0IMp82WjDPM1XxmWAI/H1vzqGmEF483zNxsAP5FLok9uHVIj8pw8GDz2KbGhxDfz6zNOvW8YOx328D1wPYB9Ctmu7hepU+AFZixTAzPvVbR/wARKfGfh0hpjUPe8riiRm82gbDrXO0wZvkH3qNHpzDecjrwWKpUluNPwfcwaA8M1sL2mctPeYwzPRwA/Is1DtKpVKK7KY0dgZ4GfWtXjZUR5+ZH16OBvR+wwBdGt62HDpcKuPymZjR55mGzAFzmyYFVo9vx9Oqgzr9eB9YLo0a6mJPLxdCpcuDJsvYPbP0LlplcOi3UdctWz2Tg8QpL0eaH7OyeZ59gLL1vjZelSozVDpTAQ4TEM4skJfjcyHkWEqs+pernYkHDwHtcJBn4yqqPPlTJHKyzzA1H08zmw6L6QNeiz/sPdVNpsBmRKhsGbxxjCSZh2H8nkXNb8etmq0T1bblNjPS9bXeqB7JLx9/0AoPEi/7nvN1qXcc4D9XPaDMdoMAZBUUCpYLpjm7FwpDB8MwBR2XnNXTcDrWmk6EyLIiMBvw1/wA6p+T1vYbHVutIhxgN36FNDQedOD4OB+c/8CVKgPhm5hvAOtQt8mmnOcNkDA+joULbK57tO5KXTKVUotNZj0sMGQiBhmfnNZII0t72DaqmZOHvAV7DqQAwDeC8gR5IcgDWu5mZnvwUiM82DQamf5FH8SZKBs28M+9SIwOMnr6gaQd5r1EQtc8IYYLpvCi+daVRLRgsGbTng4PPfde894YKqsCHQDogTqrb/OVConng6GwA/wAi0D1Hcnux2IlNhwwYPZy57/oWHc7+H1g0qsYT5dqnBkg0EkwM4IbA/OpFz1K6q9KjxK5cE+sQoR5xglnsBOsw/VscGG2GQMPIq+SD4NHqH94wMwkdAAuDm5m3wFUqawzCaNhvB1/vM81E8dkjYfi5tf41Ss3J61kAwD5mfZmadrdw8gJgbgM4di4/6mPA5IUp5o36bEOMDAdBnmeCiMz242eGwD6FCjTJ0kQnQQ2GGefnBBDMJ8pph98AzPeZ/wAaYc6XMksRpER9huSEjls9XWT8YNam+vPnMNZSa9TXHqzHcpTH3Jg+Vzz6FImHLjA7B12TDzsgtvW/zNnRa8y//I3TG4jebpyWDw/OmqJxOcjW01RrugnVXadJAIxsmAGyHZgrKtvMRuFVP13OjQw+vBYejydE2nHGAeePrM13+zeGEQ6xZN58JIHMOVyv16q655+rJEYMEretWj3DfQXPEfwpUX21PCGYAZn5A+QFhaacFmuDUZTZvO+QN6iXhxal8PbmpjFD5wzYDmpkcPBA8+gD/jXNw32I9nhwqvSN4XWPbFw0Q7HqUmNSqprhJif2Y/8Ac1y+6rqOqzIjb4aLUKMEVkM89gAmbkvBy5Ko7VZTejLfez68wBQgOI8Gb/X512RDmNSZ8UxNhyLrZhsPPBZmeBsyHdQNgK+mGw8cd+CwYNYZ71WT95G45371txiqANY/aYIsMEg60g3kuzw1I0gBIwQgrSdBAZowPDNNGoAZ/BGBoMPikrUlga6BTT5PgVXZen+3Vhhj8gGC5z2LpEkOW9H2n/8AXa2f+dWlz/PYhPUBpFhgKaM1AWfwRgCZTwAoULP4KRDkuAHidijgGSN48BAG0Dp4PSjBj2WavaUbEaA7Kfc2dAKlo4a361YPPMPRcHHDMwPMN6C4ZmBPr1MprBnpSj5U/wA5qsk0qDGmS2JVSBkGHjAD+hRYB4VKnuOOYYTGM/1oq9suOqsOdYTDzD86xwyQntTPxHM9/X507GN/2bH51FeD3moH0An2TYje3ADzBMCXJ6OUfcZzDyIYcNiZKCKE4GTPfvUEw++mw3sDWR4OMng2+GYd60EiZGcjSjY0zNrsew60DJn7tSzkm9CBtucb2/eCics4bTuo4AHngHzqgIdCkBDPSBxvvUeMDeO88D7/AKFdSXqUzTpBtg8egGDPei4QjewaaY7wDejjH4qgqwgMucmcrTMwB7A0v/gb6z69SqPFPnn4wZ/irqrOhJjtPsOZgYZ5rkXDqmxKxWYjc5jNrPPBdoknHjYYRT/IvlNfCIDO9ZziiHLWfIlB2eCYfIYGC1YPZ/0WT+hU/EIGJPDyu67B+HG1wzDvSLdHhScE9A7SkMH1sTD/AMALZ0rZKqf98H+Bc84CHnRqw335gf8AkXRqPsmVD++/yKA7Ug/maof3KkTKbB/nDwKb7HzpPBzLBsedQjtuJjqS5xxg85xgwBXDOzsaBBBilP6FN3s+dVlKZ1otP/7f/GaoqCcGq0sH4j5npmYHgzn3mmqlGgzNJidcZswmww5SIeB9f8C6EYaqsVW2aU6AVK4IzPnDrNcn4u3bSq3Ap9GpT+thMB83uhaYPU0A/wCY6azv/pDwZvfrXPOKMwzqMTnm9aQG8HdgYB5Fq8LhRPYh3lzUo8GjZMF3UI2Ya7HsvOvOVmAD1X1H3MMAzD611Ojya5Daacgy9ge5PoWTVveV+KWGAG5sAwDPeqeNdpxmg9a0owHD2rJ55qzjXDQJ4aeuB6gYYHsNThV25ZXqxWZ9Z55uWAYM6+Ec9izkyNUpghpwTDz5+dai6qJTbS8diog81KAwBoA9iCzsY2HmNdtxLhjf61fhorW8NIb9bu2n24w2yfrQ+SPMM8ANYeTJbCQbbe9WdtnOer1Pbp37Q+8DDJ9gHmCxuGOHXbt4RP2BVAtiq3NTTqEozOMz0ZhswUewLqlcPXahEcoYVJ2VJBg8Ht7IfoVZAurlr8m1zipFk1iXFAwB3W2AfYq072YqtUmvxKHGjR5rwGAZ5gyAdiIfoxZ8ZisWlTKzSqqEAKjG1zjsvAYAa456Vd7W4zwiqtpMX5DmV1iTFmxo7T2sebOfQfZsNeZKCF3VKO6cSsgzEzw0mZ/LAhn2ZUgpFQ0wh/sZ9EwDXfw2vDklYqrFVrPrUNm8Dw+fDBd19Gni7QOHUCsOVy2JlSkSnsGZEQ94B3gvNrJ4EutcLqbUZNBN+JEzA3j35gtteEPUH/pXWll4dnXD+d4Fq7e498JLk5diXWZNElv+6qDOAB+cF5qCg1w/6D/9+BAdq1V72kEPzyQWNrw9qxqDQJ9N9cwbm5mIfvmnswTx2YxPojxwa5yzp+xlgewDXiEKDcdKzcg5xmj6zjzA/wBdaCz+KN8WNqt02uA9HfPM49QweBRHMjD09ap1yyfXDFx1X1wABzUZ7Pefn/yLSs8RZU+148uJQIz3NBmAHM34d+GxeLKxxOnV6vBWG35NKqBngciPJPoWzjXheNKtp2c5eM82jzZPMNbA1HdtGHqWm1VipQo85vvDMw72T8hqWzJ5/Pld+geB/WuBcIqlRplLdocHixJCoSj1jj8sGefyZmpB2rfE9qa+3f8AP9Z4ZxjB7Bkz+dbRbbHl3XkN/jxXv0KvqtNzj6cWU9Gd84Li+HpC0qnR+Uviee/eyEnPAPOlVT47xqzHoc65vu8oMwqYBmAHhngtDx6zqfITmbXm06pVyZPzZ3mZ4Lz4EBs+tXF21X0hbPoztcqt9xjjsPaJ6IAZ/wCBZqj3/wAXp9ej0rm6VVXXwzCPLjM7wWGnT8y6pTOE8InYa4/fkZxmtn/H9C7NGpt6PVmO5XKOzTZpnmAMnmB4eRcs4rsy6bVNdynPPfQojmL4etsLPPn+FToYewPBc3MDB021sOEtz0p60q3TZctmBhJw0ZB4H0LCVW53KbNNhiCzJDseDfmvFwUn2DoH3hgs6exbW2LYvziRTpdZti2OZjweveDJmfkDPrWKnvaJg5++0MO8DWVwva6ClRJINOPhvwVnGjBDHBjYCyh3hKgYRAgxjw781o2akxJaB/ozXO220rO+hzW/OyasOEsZiNcZgDeGuyapaI9zMKWx52TVPwiuSovXrEiPv7D612+HI6lcNyXAzeX2VguUGHnG5oHagHX8i0FKZnHCD1ry3MH18v7FZLiRbb8y8KZX/sqdw09iNg9HDADA+xai26w/UooZ2xMo/KgAaUvDeuhgylYsOs2rSAqst+G9Hfe2YPb/ANCy/gZ+Owyf1gup8Y6lUZNpU9h+mxtJiZnq/kXB3rqpsarhzb/3fvXyfZh9LhvbQHUrfjO6cuVDZP5wUKpPQZIhKpU6MZgfYub1iqxXqu642/sfPNWEOpUN6OEE64EN0+/DPBc2bR3NUFYw/BD6FOZuF8w09QFz+qm3DNpynVz1qB/gs9CUA6qDoGEST/4JrLps7m4C4aOyWFSlGyf9zmr22IbF21aPSrcrgc6Z5gEvYB/nXKqk9zMrM2DD6wV7ZNSfpteiVJjrYWuLO5veIt1XdZ9UdtyvwYzMg2ddkw6HgXJJlYcmO67kUHmj62T6DXrnijQaPxL4d1ByuQYxyGIfOwJ0c97J4ZrxpZNKfrd20q0qxUXoAOPeNIDsXf4c6RUpjEl3XYiMxj8jKhZ/iLqHGPghSuGPJVyh3+Fwx6g8bOkYb2fn+dc8ZoNZqv7JTpJ/kVwwuEdkIvvDwUhl5j3Z5qazY1f95BmM/Qo9VtWsU2KcvkZj31gqe4HrGYaYNgYfOtBQbk9QwwYiQTN3zgaxjPrgPaUqeH1xjU6Gct7YcSSH/Ymucw7bRLwo0yjfzyZsyz84Iwu02YvIsVVl6P5DgGsDbEB8PAnQXg8mYYLUAyAezVtYtoA4hMcvoSn9YPkhmnqVdVKkv8i392F/rN1nBUIMmCeNkHhwMANZLw6hGepsaLoevKaf/bImalTf/isP/wAZcneoMT3beH0LqFEtWh1KygcOCAOn1mC83ymFKdEPnebgzqUH/bInqJWZJajnqc/okrPnZO82wlAGHnBRwtXN3QN8P1rn3zLa6NQakz7eJG+TRk5rM3Pc9wUqeFKO3I0mOHz4KPMs/RaN9id0dmar41KfqUB58H+g8N6r6fYOloIFeqLzoMBBAM/nVwEOs/2GN/5n/YXNzpUulCZv1kHvkAFOgUdue1rt1V4PkWv08p0twdNqr3tILJ/95/2FBn0Sccc2/U3/AN+WVkwApRhqVJ48/IatYdpP1uLr02q5+drmd/6E+nmOlLpr1Yegepm6UzMjsdhydFSgptYeiuxPVUCGBgYft6wRxn2Z/Iv+1BdFZsylVuz5DbmYS32T0XgPeBr3x7PK5ul5ln23VbJu/larhqmZmBsnmGCdu37tUgf90+GZms+8zLgVQ26jz5yIp4HmHWtbeElh616fOY37AzXfd7Lh9ohsHHDTcjPfxqU9MYhyPHpWwO9lZK23vv4eBJ/QukQwtx6OBzqlybvkdZNcbxGZqtOkmDcSX4vkeDBbG3jDVDXcWc9SWjJ/6VjZq7poAzpfIu/hZWrL8Z0XTcDyLNUq3p08w2da6RUtA39d8M0EOZTTdBjQAPyJzHCiUqw5xtaYBGD59YFVVjgDUp886k3WGYwH14LehAgmeo42p3Mvw/AYf+7+RZur8ObwOCDcB0H/ALQZ4fOtlGt4I0cGHDzV2bKQM/v4360GfmM0OB/pV/lmvP1qiqUDhfM3yp0Z7D8iuL2Z0YGpqMn9BrmMx5aq2s62fDJmAblKB43Q7wNc6mVJjV69isKwebRrLmB5o5k06lu2dCM6l939mZqpNlSIfNngxFMAd7M1n0nhuLDk0aNMB/1rJ1e9pdDOpOPf6KnAEj51ze3rMuaNPanS6bg0x14HmulhPg+09VRlx+y71PUgrDzuFZfAw84KD6kg5/6Y/gVtPnwZ5eIAAHkBRHnqPD2aGa44RY+Tp2ODbgPIPs3TZLXNhUWQd/CNVUnTPN9tvDBQXpjYLphytNJmRIDWmxFBkw7wWfkz+Z8RU8yeZjp82eHkVeFSlsl4Dmz6FfSlrYFY5P28EJIfOolYqrcw84MQ2Q8mahc+29FDZvVecl9k104XsZyZYezcM0mfXhlruN+F51H5r4JM1X8CUf5EwbXRzGA/agzDyKWzVWDHUbAw+tZw3tY1OZMAa8foWa9tKzds5kNPXZP8idC6qrJ/peaw8mT96zYPYp0OZ4vtF6jbZRpM73754fWjkzKO9HwlMHs+fBZ/13h5/wAiiPVIHi1N4KNs3JzPAvE6/OpBhLBrXANikVKiNw9WWb4BmeYAo8mS5GpDrbbmGYYLvhJ7Nt5rM1XvG2DqCNUmGYGDmeqo/tloJGsYBmx1p2NPfz1H3DM8O9GyyxpGBqIYYO6aCRmBu+I3s86tmZMtmEfI7AMMEzJpTbNvG/73sVaFYlm0ERz2QILKMb/L6bjmw+tR3gYZdwb7DUiNJABSnxucdAwc2d/zrO4Us4zJz3Wn3N7WGCkPUpgJGu30NhvUcJPJwgb3/kUjWcezccP2gYLlFfRDfOsg23vBbU6JzjWuw3vBZW3ox8wfyGugQJOBg3pmf0I9Ow3n6Db5vzj8VjoXPJNYfmSDfM1q+JEl9mltMMOb5R4AudZuAjxYPSTeVwzVWLSdBjY9ElBvzNZQ3nFNjPMT4BwX2zOQG9nBaYGmZh6NS56K33gar7znmdW37MwA0DMypU2L4jnR51RVuqvz3wfc7AwWOBpYFeCNpcrFABAP1q7+1v8ANbub4MumfQuXhUpfnTvPyPOr6RoKxW+caCL2B0GqIzyQcy4ftEgMF04SRpd6QHvR5/Bemz+s5/Uvusgw+KDE0QnMzME7zIPEqc3t6RvLzC1rrYBqJ6NVXAPoBU4PZ+0PBOgbefWpGlZmOH4nQlWAgz6NoSnMDz6wVUy9uwUhk8/aLBTS0esQQt84IMYckHg5hmrCj1uJDpzUGW/vPes1GewAwb7wRs6Zug231go2Jcx5vnXdNUNwzMHQg4eKYZq9mb8PDBQjh1Ke7p+rWTw6Hu9NjPxpjcMMJbGs0rA3qBMaByIxyxggqVN0djj+/wAgJqmwAMzbcfZD6zViPJNzDNtwPzqqN7N1O1IH4084jn5EVNZYOVpv+RdfG0gbMNySeHZ3mpYfcNgZn+dIw0fZqwgAxgZvt55qbtGEu2A5kJco3AAGATJvU3lTcibDYPBNGYU0TOIxgX1mgejNyWo7neb3QpLd24aRjCFH8NL0kJmjRLco+nvlPG4H8C0Fkw9FqO2C5Z6S1SfevmlRAPM6dDzMPJma6Ic7l9YqWjP+6NgANh0I4FebhwNA4gHmfZ5Fn5IOHINTYdKnTBP7o9gxsUXDo2tTnwTadYYfMw7MwwTRvcyPjvnn51CZo9YCU1nBMGc+tOvGw8+bbHRnsUYNjrB+tZAOSnM9BkABSKaFOBjkXMwz6/Io8Oj1WZKCLoYOn51YVKlPw2ggy385DD2GAB0Jg2kMwOTntAw+D0fswVhAksA+1r5gGZ9Ydar2QNme0thPrDFbitMVV/PAPBMw2AawUzNeeYk1SI4wBhv6FprYjZvtLKPAB1kMDAwAAADBbq2GcD1PIBryxzW6pj71zVDUPMAewBWFEOpHFDlXzB0zwAPOCyNSeN6qSDDveM/41JjPS2WglMOb2DWzJ12AywcWOw/E3OBvdz6FSSQfjVJ03H3gDsBU8OpOTIBznAAJAdDJq7gVvnGsKxB1sMAAwNcV/oQdOfR8dkQweDoM3s09Go78ymu1WI2ZgHZnsNXRnwywBtigHMkGeGeeiAB501Met9mRoU5iSy0HZnmo8DllYN952XLltmDpnvBHAZM3Y/h7M9/0LR3VApXt33zDXw0QWfen6PhsdfnXZxtQVKS5Aqh6bnhIDqrbO9jrUepG3JB19zrVV1rqS0cmsOG01ymsyQBvP51Xsg+8BsMAZveQEyyBvHpt9as4ehDPU96s8YEIIxh7RveiDMMMOxSJknOQbjiEJnkXoQG+crXfPMzWojMsSWgpstwwzMM8FmWd7ufYtnGrzECFpxKdG50wzOQYdALCxu+F0xh5qW/OfN7XMAjAZ9gGtkzMwfdY1AA2OvNcdO5AZdByJs8+AYZmqxm534c85z55k51r5tuz1vcdlrEzk6b610Dkh2Az1mslcNVi3DCj07mpMbQz1gaewzzWUC8H5keQ2B7DPNNQ57Dx73N658Nub3EJ6sRKPNkciwDJhsM/kUF6qnP8B8De1w6D2ZqRJpUR50zlAEnyGGzP61OPxsMGwzbDAF1acccwIbz72DDj+joBgYZ9AKwejQeV14P3kM8OhUgPfeMNQA+fzqXJqoBH09fAGA7FODe1xRKa5SpDVSqQYO/0b6FYPHKkyDcl6OkfY0qeNVWKlCj675+AefQrBmq8/MdY0+gM804f8zR128P/ALmNHb88lj/9Qax4T8HQYbi6OffnsWm4nTGKPw5oWu5/SWP/ANQuWPXzTZmcRhh7Z3mtvbV67ds1hiAf85MPMmfQeGxV9816jPQPWsHWOqxQwBkA2PAszJvOUA6YOADXkMM1VSak3MiyJzHtYoYfr/8AsFxx+hjzXtQyZL7wab8SMH1s4foULlj92pDzOGGe8D3o8zZM2w2GvpQ5kfOW8BxMGdgZ+D1mqczceazNgw+Q1cBPYM2dBgNgYaveoRnrFp95781txihN75ABDGPORpq1egQQA8/a9ip2QNmUuzws7JDekCeAAed3p02WGc8D6DVqRD2JrP4KQ9v/ANdR8PFBtQkYJ3D4oADB0/kTy9DK6Bc8nR4O2VStP28l+V+jP/XXP3gXRuKJ8tbPD+C51sU0zP8AgWt2pgTNzHeoh9anAe7TUeSGDqxSaBSATKWe7BWpLDZ4iDrdNzzpdiQdahKRm4zsBvMz/QjAzeFqI22Gef600Z6x4H59iAHnIzubDh5+dSL2NGACZfnN7AeDP9ahXaeF6V3/AOsNKZVXOVMPnA07xC/9tKh84MH/APeAQUoHgfXhmiNl8w+hNYfFO5uYHqZnsVIS4cwNgdeB54H3o5gOG+b77eAGqwM8VIDsbQWEZ7DYHQnXjwMNRsM+9QmetHJzZPQcczMO9SCeNv2ikQwbeB3P+zPqv61IzcBrr7MFS1hMpoRo8LQDrjA8adpr2GfkPrBCzPMxBt888AwBO0RnOQ1p4dee9ZeRtrYptfPCdQw8XsXYKPMlSQNuXENl0OsDVPavNHKiOP0qmvYRnzDCNhgexXvrud/8v0T+MF8/y18Jx9Cx3F2qsU3htUDcbzOo4QgD6+9av1+5/wDLNK/8ya55xvqvP2UEX1PGh/fGH82T68FEOhS+j3n/AMo/7lj/ABmt69c9uW8/UDqtRZAze9kB5muRcN5LEanS+er8mmx5T29mP1yfItLAqTkaRqWlQ4zOfRUZHjPGrGreuq5qlD56h0D1JE/t1WPD+BYa4bqsrVdfrlVqt4SD62mT0Yv6M1CrcajvVI2Lxu3Oax2S+xR2YfDln/3jpv61qBo9yWUcOJzds1KjyGM/v1Oknn1+TNXUZmcbQMWzX4FyafWHsZX6D2Kt5bh5/wDMdN/8ZRjpVgHviXHDB3sMHsDBWzwvgrDEZ04NVYOmyGDAzCWGAdfnWSv95+Y7EbfPwjA9nzga1R1W56OwcSc/64hB7mdvWKud633osR+j0c6a6BnzMfWzD8nkR6C1Wc5TTf74zXRQN9lrTY8+f+Bc6tvXZi80wxrGwuh2YzWb2dqcSm0rCRTmQNkHng+8nn0IJByX+YDxzwAANAbLEnxDDN3PPPvUs7Vvhk//AGfjdGH7eH+ol9mL7Np3QtHW/upgLfbBVTJhw47pynweDDBkHgzwXOXvrXQJNgXxMaNv7Izz/wC8gqeTwov8Oi2DD65ILG3jOxpNNDCI/Tc/n1lIjGECQD8FvDDoUi4bDuq0qbHrFwQQjR33tHY9meaoecw6HFjcDZyZ9Nn0GW24xhNfwwez+dNUqiMTKQL7578zA1n4EnN0N60UaY+AtRYjmDXX+dQnw33De0qHMo10BUsNXkw0QM8M+tBaVq0qNw8r0uquRtbA8MJIZmGCyR1KogOnFYB53yYKpqU+7QiyDftKSAaJ5nyZmtot1eHP10uz2Ys+yHWD9rFk5ga5kD2YLcWZJCNTTzbzzPMN622jDuUCz6PzFEffY+77Ae39hrNQOGNcvDi07Q7ZpRyWYtVDmXQ6GWM96zgHOeaB9uCyYH5z3rtfrt/g/wAG2otDbN64K8yDzxhs/wB9hr5vs+z54Y/Q6+GNptV9G/hrAuZ31/xph03mjz5SDD1jD85n/kWfrHDr0XqbMOC/x+reqH/5EA1x05lZrbuvX8AkfOaqZ/tTYXH631+Z3Zvh/wC13ilcJeC1yRRl2/xzqu/oCXRAD9e9NVXh1cdsVKFzT4VihSnv2unHmGzz+TrXn9nUhuhKiPmy633gukcOuIVcCbHtmdVZJhK98Z/41Hsx7nDG+FfDfDd4t32w7Mtx6syLn5Qwm048AwPZvDyLcActkvuiy/D03Aarv7t5j/AtdANfof4Rv5J7v3o5joSaqZabnQlzNZZHBvD84KQDzaA3s+w19RzmpjPrWG7BrLEaY0/1g8zsNZy7bPg1incjSm2aU6B+DIZZ3sh+taZNPdaj8Nnny/KDVbJdietbqkz3XwM2TAz2frWc4nWeDz8K43Lunn6xAOsM9/n610Dj9MpUOZSvWIGebJ4YBn5FCuSZbn2KplSqOD0djDAzBflP4j+Y5nR4/v8AFTh8yzw0tT14D398GH+dR6bb1fgOnLi0oHo+GGrHNdF9fWkfRR//ALypdErdK+9hEi6ObP4OC6PHlyIPBk7/APWlQYtm6nreaN7xgNkDAz3+dZ2ZYFZZqk0J07M+ZMzNnvNbrh1VWJlwtMMAYfWCr78qtVpt21OKw/g0D3Rgtotzs+zZjDPs6HJkn++3q5ptKqLIabdvgAfOCqzu2qg14eiH/YqFPvCuBTdduXmB9GxRTSLaujxm4dRdY97hgYLL0o7Zt7iDR34J6zpycPoQcKKrzJ1Pnm9aRrBg75OtY2TDcpt1esdDDQngef5wXX4cz0hxdqtVt52jvwarMgRH5hsyXYm88FE4e3JKqVwu01+4J9YpQBmEiWzgbJgtLflvSrtptPCC4zqxZITd/eC0B+MS6PBDlt8ssBcsvXlyd+GzPYqyBSmPaHBZ+vBXvE6BRvXLUqcx4r7PXmsKb1RAsIlYeZa8ma+f7LshcViiS3pXs43/AGwKp+wDczxH2Kb+g1LOBXAHU+1WfyZqJ/OPvKlJ/Wud6kU22KlD8OC5TWWg7AZU16g1ySep61ZBUr2wMznPYecDUs6bTtIDcrjxn14AiD529UveVGG8Hzs5qtOByzvs2fyBggmRgwacgvvGHQebyaZDRL2hotdw4cUBBxvmQ+h40zUrYo1SMX3H3g39BqPzJgPgOKKcxzS0Jb+at5tqpnDGlTLZaqVNzZNg8zz3qvjUp8NgVHRP60qVcM6BDkQXH8474YYKqmT6V7M20NtLyc4BAPWpqvZCVR5kgAnG8EoM8zVFAebhugbEswaz3s9hq7qtYps/SfYiAyYB0Aa82bTQqT/vDS5x/sNZd64WGT35gHnVqBmvWyz598+s0HMmBKvAzTRz2GZDTbj4AeaLwsDquj1pMz3DUfiFMYwhTmGGQdMAA8FkvXEsPfo524598/eKPMqVZBr7i5gf1msTMreB++/Il67cOPruOax+TNGe2mCq3j79zP5zMFcMz53v3DzXN3rtfP2AYO/OiZueqve0cDNeYNuhyXpUn35qJTarK5V1g3Og1l2bhlGGgbiNmpaPtOhem1nMefeaPTShhXGYoPtvs/QombDx5tyjwUjnGGR09c/zrzC4s7Du3nx9mGw94datmbncpoBOiMGZgs49R24EX1k2YG0/v2IGZkST4Bphe13649d1L1q57U+tWsOfLA8+beD6DWfZpvqcGtN/WaM9isGXlbZbGbZ78GTPz4ItbZp4M4fQq8JKMHs/DRCXrAHuI3/goHpmex8GT+sFHM8D8RNGYGXhqV4SAeY/sjP6FEe/aANsEDL27BSOTckl4a19a/1sfZjxhLmaZtA+oR7B1AWigW2/VWuU5sGfnNV9VoLlBd0HJYSfnBdPM5vWVoVh9loM3MAVfWKxrBnFfz8mBqsueTL0gisMGefkVJSmZzMoOvDyGvm7d67C85YYMTmDA/rU0KrFea8PDeqKpUp+TIaf6MP41ZwKa2yAG4efyLHvZCmPOG0qp6muPDqYLUBGYktYNtqPyblNMHFt96GMmU3RE821kp+Gfs121l6hzMObYzNSXrSseYAalOAPoT+ajzo9prUUSyabWIQS3LgjM59ma6VW+FdpVVrTiZxvoVVD4V8thF5pk/nWv3/l5w8KPTY06mx+VCpcy0Hz5pqSE4yNwIMnDz4LQRqC3QcGAcMwDozNRJkyqxpAPwekOtZd35dOGM9YP/vkPM1KSem24YfWC1r11U6pYNv05lmR+Mmpj1Nk4YOM5h2AtvDgtnDCpQ2tkoPyKFJkvmHidfetLMoj5gbjBh+tZSTJbZdwc61th4HBs/fh+dTQpTjJaj4bFXnJb92pbNVlvRTiZ+EtQUwNHw9ir3nsGlIe5o1Xmy/np4LJ7gvDkh86QRn9X2exHGDxfHBTcG/dqjCvOG4Z+I4pAMmAaef6088gN7P2nWCnbMHiYeHgjDUAtRR+ZP3aWtn7RBN5xLmW88HzUI9B4N6WiGWzrV4HP3nn3j1HzMz+dAcx82tDU2IzDM8E6FHfMQccwBdSVeAH7RTmVdQ7ebZj+I4CaZoL7zp6mANecF5sRzZcAg0zA06YOMhqGrNmAxGf84fOq+pHnI0OxehrWOT4HeagzIbkN0G+9PPRnPaA5gpBvNm0DbnYrChmZiAH1qw6AUGlHrStRtg9is5+g97NYqGyDjwb+xSGWXzPUbQRp8Rml8q5FPV86kU16IbumGZ/Ws8PfC1pWgz4krYHnVmdVrL0XCjQTZaD+ln/AKiQeqocLm6l+QFn69XqjWHfElmzHDojh0JhuqjgPyZ5vvznpLoBnmfYqwAzdP5DV2EnloEjxN5rOA84DoA23nmtHOmhDceMMOjvPyK+ptN0YpzoNOekgB4ayqYe8sNQA+tdFtKM49aU1tvDYZ96yGCen84buzAz7FS1KNh4hqwMDjOuvmAbDTNVZCTS+ew3h0IKfZjnqJAYJowQBsXR4SlmaaM96SZNCEgDDBDn8EyCRqw9rJa/zqPn8EgQSgMM0eQKIkgkZtmng2EogJ0DXmC1gB7s0fM/OqzWc/qTwPfu15gXoT8+tWFKeYN3rBZ9k96kazYd6574VNQ9JAA1M0bM9g4Zt5+Kayj0k3gwbNOxgYDxM8HfOowLuBQYnNGb+sZn86keqonOusOAyzgHgn86jszMw1GJYGajzJj4HrtuZqBSVuNUmZn35sMD9iYKt5x9k9NsA+vBXFeecN1pxw/aAql6A/iDmG894LpgS6bJbedwc8ikBJBl0HFTsvaLqmgGbWuvUriY9meal2qfP16FF1NmeaogM3vaGtlwupsWTcubgdAGtkW9HWTDzdjgfkXm/jNWPXfEuuygczaB7RD6AXqK2AbjRXpx9EWMZrxPVaqxMq1QnMdD8kzBUjhJmMZyNMADWPzmtdW5OcUG4hmZnseMFzx6Y491rRwJ5vGDB9eClsKq1WXAi8qw+HO9B/ICr4DLARzB/vR1KSxPlA3Ebzd7z86kM0SWbrTkqWyAB2d5oNbApWFL55w/FBnPNUpvAcg39RXXrLWi+rWHMI/kWXnxmIE0z1DAD7+tctjUMyYj0XxGAB3/ABpqZrmYAx0Aq+NJb5eO2+Z5mfYrMAcNh1zM8MNhrxSDGDmaotxDk+raDLl9GAGsrR2c5QGf51dXJM9W2lLPZveAN6m1uWwGc556nnNSHmW2WjY+fNbqpUe2KbTo/rmlGEiaGsy8HkWaOHTvcAZq4tCPAeBkN5gB9mamsyZTJG4YHpYdeCOMEFn5DPrzDNTWdCSeDHeH8Cxu+xOE6NJpQDpuVWZpdhgyipsyIdUdDXPSAOvBReTbANBjoBOmzEptLPT9q/vzUYQOsQGK9KBiXUgZaA/BzWM5N8yPTbXt7h1ZPDysWpSWKzasCZNBnxnng3rzXxRZptNvypwaPTY0OOGGDUfoXZwrc1CA/JaMG2/rTR002YuvqZn3grjOIyBtg+DJH1oDZPkANvvPBXtAJ9vP0em0qqm+BtVTPDDswVbMPA/kW64uxnIcC16c3/RYxmYeRYeTGfkwI7DDebr54JtZGzrNA4mtHBe2KbbFuVu6IkT1bDClWvD8aOEYMDlGHXn8i8lXbVWKxeVYnORAjZvYADIbMAXopoeeDvie0V3AOCZG2+AGBh3qiADAPDc3GfQnQNxn2izsXfh+8U0LJuOt0Z244NGknSmDw5gA2ZrOvPHyby67Upk6fI4f2BBnZ0enMwnnmQPYZme8zXHzQQ5/Gt6pUdhoJ0GTk+GYHo9YKOEyCHv16rk3zb9mTam3zbMmQ/8As0cAzADNecZNsRZk2RLluBzEp43jwDDDNRS7/QpQktn7/NM4Vk5B6bgGBhmBmtGzZkTSAGHAD8bPreTUm0pbzQBz2iAbDMDV7YszDN+SJnKbzDrDBWGi29g3ygGZngGaOpPQYBhSqM/rAwGAAAJUeS+zUYj+BvGw9mGHRmqbRabAtK9Y0jXfs6qhE8mipdsPPzJp5wTjYHhv711Cq8YK5MgSILAAzrhhmALFUoM5oKfEMdth6QMZ+TYdtNsSwZeCYB4H3+CuGUqBd08nfVtsyawDB4PcpGMwA/yLuvHuS/Gt63wb+f8AwKdfN+Vnh7blpcObEqsmjx4VKYqFSejngcyU8HvvOo9vlw6Yef58kHpQS+RAMNmAbDz+dWVHpT8m2pFxhOzwe0J8fDf9a13EiM2F5S9TvZYfP6zDesvDeOHzrbbnhTWdEwTwzTYfDe/K3AhTqHbMmZCNnY70ZrLz9eNUnYj7eDodYLsFE43natuUy2IlKA49OZ0TzPr/AIFySqyYlSqLs5w/Ff612caVbngSZeeAw01Ikxm8wcbfDfs3pRqUD2q4/nmGwPItxT5/BCaspNKlsjqaBvBngGjvUV5l+MOpKiPAHzgtQVNtusXDzHqpgD5UM3szwwBDPpU6lRQCVhv6DA8wNdA4LQGJ7td1/ZGywBh+tZziiz6quWPSm3M2gjAYfnVqZdnq9mk8y4y6BsNoAk6J9GaRycwDDYoSkZ/+v50lCAz1QU0w/DQNGC6HxpZwqNuQf7LRwXPMDeqMSD+PJAF1XjSywd4OsG3noUdgA/WaWOT57s0bx59+9RANS40Z+ZgDER4zzw6EDSZ98px0qsxg15dHmAAd+Cjmy5hqONmAfOCvajqsKVTTqrrwA4AaAZqvyBaCzA+8TXP3IKEqow0Sw/jSBaCpUSXPf1IkEz2Zn5FHC26+GH81PHn0YKBn5+py5q6vzfdBP9j8OKYfoUKZDqT3t4rwdnQam3gy4EqmG530eL/gQUQHuzUt44jzH70P41E/d96QRpeBv8qeAdasaW1bMfudiQ+xL0QYPDozWgZ4XTmc/wCdYx4GYdC03A2BzlpTX3O+ZgtQAZg7/fGseUc8pvDepVWaFNixJMmWfQEdnP8AXvUu+eA87h7Szrl1XjRI3kiAebzx/IHeum2lcNx2ZV/WVDl/d3w0ZkQ+iSH19mCrOMFHtjijhWPUcmj1tjAObOTrAYeQwURbVx2iWBdV1RZFVtGgT6rCihnrNM4AYfn6/oUGlUeo3DUWqNTW4wTT6wkSQZwP867XZnFe+OEVs/ZVtim1iJCD7gfQbP1+dccnz27qvp27eJLD1V50zOY1BDljPyYYLaLThbXnwrv+wIseo3VQwjQnzw5hp4HgD68OhRLYjc5Uo7DezM8F226vSElVK14Vs2lbnqdqVD2HOwewYwwDAFjeHVpfzoFR0Ngdi5uZDqFt0R+jwgNyqyZh6JgGthszToWrKqroAxcEmm/3IAeauGY2ECJ+RSIGw9RfK7nZxqQOHtf5jQb4kyQ2GfQChXPwWqNw031VVeI0mS1mBh92A8DW1ZP76H9yakPTN6dzZyr/ANF2lPMBEnX3rB5OTWgpvCinWrChUODUucDB8wPD6Fswk+KDiUk86zC+h/8AyLbuHObh9Ga3LnrciuSrqnw3X8NjUYMNgYKKHoo2qH/v/Uv/ACbK7K8eEczUTmc32g84J3Dj8n0V7SZ9peNVe/7EEbPoqWO9n/yqrewM/wBmBdNuqtt0GnHLbc39iws++Z0/P+eNHPsjnsWP180C9/kit/386Z+Q8FWf+jHwdeDNz12bv/15qPRLwcpXO/zkbwAAGHMH4KtofGOCznrxI0n6AMFj9fMKz+Rmw7VuOj2rTWJhwrlAwk8xJzMMPIrj7AWJwxrkRi0p0/VmyQZkhLezwBR6rfNNuGs25c9NwAKCZm80Z4GefYCzVz3Id23GdYcY0XTZBgwA9mxbeOflHVZlpUeS6bk64+TP8HBAFBobPj/a4NgYYAGGa5PDwpWbj7FSPXDA3eZzAPoA1SvUS2NUwbcnyZHndmdf6F09x0uxBZn3jnvWpnn2GCl021X4xk4xUcM/3KpA4qRXmAieqjjSGO83tgfwKdRL81pH35uMDXnaezWP0selU3haVOuR+oUa4GOcjwofOh2HnhsWfszgPw2uewKPUqrSnjmzWdd6QzJMDT11X+fraXLpQbJsDkuv+NM23xUft6kUy3IlHA2oTOjq62z/AAJ9J0pZ+jNYkCLqU71kegBmetJzzWPmWxaT0ebKnQfu8Xoia2/MFoK3fNZqVUCuRa4cBqEz+ydhpVK+XKrAdfnVWMGfugDrXP8AR5OlRWrrwKjTHKBSnjm1DrjgeeH513U6a2bRsPy5klqUzvB2SeGBrikO/JxwuUpUuHGkMAAGejv/ACKRG4kXGy07EqNSekif4Ib/AMib8vW7Dgzwkz32BSv0GjmcPeC1KzY+x1NZw9zgaxLPEipBSHYLEsw1+s5DPjYKnOpQZmbBubz6z7zXn0+WqFVQpUaqS4tODBrPNdFvy4abao2lWKjSjqUJ+G/FAGTwPNconz6GGrEYYk6p97xrot+PUCt8PrcYfczNh7fs6Niv2eb9i/UZmq8VINVt/wBTS7EhmIPazMgOsN+YLG35f797RY8SVBhxpDB5hIjs4Gf1rSwJNKjasSJBAAYPDrzzWdva24tSdC57fiAyeH3yOz3/ADgr4fZY3wsYBv44OdakUeS/ArMec+GZ54KvOfFB3XYM/oNFDPnHQcB8OtfT/Ln8Re/0PTFmcRWOHs2WxVYkmZEqDzGs9n7HYu4US5LYuGEFStipBMid+HWB+Q1y3hdAo1YfuClVxiM9Hmw4p4PH8nYtrY1h2/YbUuLQ5xyQlHngbwHh/vmuvhvD6PsRhsAmQQAHHH8M06bzGlslgH1pMwzP3f8AnRnDNn3YYeTBfR7PDiNBynfLZz+tQbkqTdHoM2sxYoVKRCDNmOB9almzBPris/oSZZYZLUYYAPoUPcPMV23PeNz1SO5d1Gh011sMIzTzJgeCr7VqrFSiu0N+dhEYPDozWl9KI/8A7YNH/wDoFy+BsgSHA9rrHgC/P+5w/wB53en+vxTpFYgWdpR235TzzoZ+MyyG/wCtVR0e3KVKCUFcZ0u/sWXjevAawfD+NBPh8zHwfbXJ1OLytbYuSm23XGaq4+D2h2AfWju24abdt1y6rFzZalYbD+hZcLeqXu4msHyb00YORiab866HOuDBjlwYcPf507o2+zFabcYN48Mz37FWBRKq94Zy2Xmnw2GBoZkOvxuimvIChzIlElYUaDog+e/BSZlVYeim4DYHh15rPYVmeQMRIn3jPzqOyzWOfdiS2DDZvW3hOHsCiTOctyjv+eGoVKjVGMTvPSwezPMMA6Fn7SN+pcJd772qxAMGcNh5h86tbJmc/aVKlG/rGbO8/nXb4Qx/G+M5hR5YecwXMgjP97n6DXW+McYztlqUAZ8rJA1xF6pPszQiPxDAD865fZh0xbRhG5Yd7+YfWgk1JsOhxZeZPYhyAYcYmMmYbAPoUd6TODY++el3gCxwzaZmtt+/NnBR5leYDxNMPoZWRN7B09dt7DszUc2Z0mQHIwXjDzgr6UtgFY1mswzD5DUJ6sTmS1HOjzrP6z8Z82HzMwUiS8/pe02edOl7hdxrkfz03HA/OgmV6DJd035RgYeRYKSbkCaEvUzPPfh3q9jPBWzziADOGw8DTpSu/WUuHg5rmYdmaD18Bu6jnWoUPldU4suWGAHgesimQ6VAdDDAwPvA0wtZ/aHAPkRxqlzm+I+H51WfzdGa5rXZMPwj60maxaphqZmH0GmBpTCLMY0zVtAmN6AMA+B4eRYmHcMSAR+GEnPzrXW2zVbkkNMUChg9If8AcgYB/jUXDfjWEmTyzBvrA1KY5Jla5uLp1b4UcW3o+gxY888/IYGsZM4S8X4ftOHNYe+gAV4bcts/Mqs6Tg2+5ngky84atQ4b8VHv/wAHtSA/32AJM8K+LZ9djyQ/7ZMMcKc40sPEffA/oUfwHndA38Fo/wCRri9/8v8A/wB+BSA4FcW5PtKNDD/tkwdNsoANgpYPAC0H8gnFTzwP/GP/AFE6zwB4mn1vw2frM0w8+a2Seh1WS/qRG1LBmsxg8dsFuoHA3iUy1ptzof5MzVmHAq8ZMc2JzgPfQBpp78fM50zMz94gOfg6Z89+TBdNh+j9czLoG2GYeTRNBM9GOdJf133J8bPsDYC5u48enzKiS9rWeH0AsTDefemtaa7dA4FXVTacdKbceeaMMAz7FED0Zr4DfFOAH989gax7nT49a2XM3/UzQGxhoGo4ViI34bji6BJ4G8RmaM7EclU0+jfzKylB4OVy6mHXGLjo7JsHg8069vA1cWi4V/rKJj+3M/rUiHUon4il/wDou1wC1Dvi3mfreV3A9H6qssCx/KHQdivbJnJlSifiKserDB+HmtRVeD86lO6D960p7+5DP/Oq8OFcE/El3NJ1fOyzsXmBmgmSgfBzUVxArcvzqUfDrD2F2vH9cMFczKPSodL8CL94D32a6uGHOm2rc7kN3TfDNO3JcMWqmcT1MbOHeZrHxs8/DVhGe0SBw12X/wAMOH+35RXmXDPYpTNvA9H5vYDvzq2k1Wj6H+gwB7zgsTWLwcZzbiMYB5DNfm+be/0PqeL8LM5NKM9OW3nh0KsmGwDvgbAXPZ9zuc0bgPqP9p33uuWkenbHbdc45+P+hLBx4NRx81lIdbz2OPgH1r7MuF8GtBtxL9O4YNEZvslmBq4ptVceAG9cPzrmUy5HzE9N9O0evOG6DDhrL5bew6tValOBjTBs/wAiCBPN7DNs/wA6r4D054NP1kzh2GfWpD3rUA0H3AMOwwVRDs3gFyBUeV5qLhh86xvrt8/D1zUivPGZGGuexZwzxXfw8LG7S5563iJkDcx8RKHMcB8NgGHzqzkwGJIajDAA6feu/CMKeS9LDMwc2fIqd43MvEUs3pbJmEv9CiPb1TM6zm8HgOBn5DT0ae3GI2D7+9V+DgH0bEyb2sa1TDXQKlrGAON5tKwMG3g0zDZ2LEw5kuMHWB/kV3GqvhZvvmuW+FSwOGwDvhtqO9ph7NNHUmzd9oeCXNsec1jEW9tBekuB+dMax+dTZJgfiKCZh5F0eGYcjRazh7E1sSw+K6Aebnu06y9u09ADUcFLZZ2amoCyGPZDB8FZQzB5/TcUcGc/Z9anQwbjYPuRfG+ddCVlovmGppnpedNaM6S6DEQ8GT70gBupED8uUeAdgK9h1WnUqObbDYGR9hrnEeq2rEjUk5cSq7+8HlkjjLa2rW3Klc3q58GTaPzrQXzRGAp2uxBDZ5NimLHJwzAN6jmGs7pq2BnmT03GzBV8zQjSNMF1CRGByHnvwzUqHGOYfy+dUoahu6jjmxaKlZvQjbiudnWoUamA2zI0wcM0ocl+HK1MFCNmcZ6j7aKjvOPT+Uc3oNdrRKrTQflf8GAfgqO9RKjJa558AZ+QzUqmw+TLUlsZgHYotyVXwj13PoBE7QakbEZoGGAZM+/vWdCM3GdN/wA5qVTQOpTRYYVwdBYMMD9r2IpSxpn3oM1teEtVfeqlQpr7n3d8+jsBYSpQH4cjBwFbW9PYt5jmmNjprISrkwZqjrbnsjPetnf8OhhZ8KdTorIGfkWCrD3P5yvxFoJMzn7Bp+z2Cy8DAz4wB4gKF3eIthdUaD6phVLAwNwMzBpY8N4ai7ISkBoBm4o56ZlqKQzoe8cQHhq6ja9DQbCQPGpHtnfEc60pkZuN7N8D+hWIoB+Iln8EGfwQYmikjP4JGaaA07+RAgT3LOA1qIoZsZm24AKdrGHQqSrE6BoXg8VIFIlZ5o2UozLZtb+tTWaa4YZg+CgHsNrAHEgPFRDZfZdw0D+tG8YGK8uBNZk/QCnADZjqMOZgqeL/AM60UCHtANQFzWpT1WBOwOVp7FI5ls47USUAZgHkV3Pjaxh5A7FXz6bzn3p9wANXFvbVM+Aw8YPsMeEmsGwDDoBSwpp4akXMzPsA0cCG48PjsHmr2hWB7U2103g/GzmS3/nw/wAax4UffvBdW4UUpiNAM2+s3jVxaLdAuqq+pOG1blamDpxsA/OvG4QHzY2N969ccYA0bBCmthvqLwMAvP3qEA9gCjuIYjk3A620nmXwW69SeEbjjG8OxVp0R807nQpIYOcuZthvBTaJMNknomn7feZ96sApT8N3Bz9CnU2MwzPCW42Cu7TcKc57/Om5gfWieqWeZv8AsvwQVg9RG3nXXwfw+TBRTt58/DlsGBeRY7h4jwJ/rWR474BobFevG/GA6drgYJ2m29TocNpt9jeB7zQzMNXw07oFhRGc1NuGfBhw4kWcGbUo8E1SvBBM3nTeci0p/Qee0D7FFrZV6tzql+1vm9h0Zn0AgZeWrolBYZj6j7HWrWBb1Ge1s4vQeCju8Dm4SXHiWgprP3Pm94SA7DBXtYtim8qblOY8VSJkCC8bPhmHgh0Gnd4FaB54MPubD8igzHogNBqNh1h8+xbD1PE9TGDbGbp9+axVSoM5nVNzz9Cjh5otOHeKV6SdlUcXXwptSMwjYM7MN64VcN1BXq3LrjjGjzR56OeeCpvVU4+htRdHAtNxdMW8TQqTZ9DDJ/WCs6O9zMqPTtAAB+SHYqcAbZHUwV3Z/wB5uumN9mZn+gFunCy4zTNa4I7H4EYAWHZmOM8u+HXFPMPrW4v+HFmXpNbl9AAwH8CouQtiM6bHKAZ+cwzNedo9EWfxUtG2OHciDUarGerc0DlSTzw1nzDD/IvNUkzA9QFZSaaekb9Npr2YfIhZhsH4b7fi94J3CpOfK/EAPyAkExzP2gH+RW0y1alJPUgtgbRombY9VRzOuYA6CbFxR6PBmNA4+wGf0LrHD3hXUr8oxv0OqxgkRTweakBgAH9f+wuRWrJnSa9EYY/Ys8zNbgNcOidJZ/uXsFwWdyquqBKtsnX/AFrGmHmbGcfPADBVkOpN4A++5vUu9tflafR22NgMnNPf1/74LOBDfMzcPYCjCIvsaD1xKMjbYf0WsO/vVfUqw4zHPxwPNZx6SBvmww59ew0NKprbzvPVh8wj54A0HWa2yvBmGzLmTQ5VszWjgQKlAI9R8ADyYdCNl5iHgxTYLwB8gdCUapOTD8CC99bp7DTalwExj2Znv7Fa0Hxpiyj0ydnptxAPAMzwBXdsBdr0rUt9ilHgYZnLNI/eybP0gTwC2mPrP/AsjxLrDEy8Hjb9xGixf4FccUXrxeuO0qdePqffhgEH53gDesvezOtxVqdH/o/OMMf4Fj7bphL4hVJyq3lUJbbGGGgx+gAWcMHD94AfWlxOqr8DiNcESJsaCZ/kBZr7SHn47i6PCFrJNwB6GTULMD9ojjVLWDewDP0KDPeYZd6+tbwxJ409GqoMn4gAexQdbNNGGB6a0W3FEmQTtmafN/eIp81o4dYb1s+MFvMHw+ieJ+xQIspnZ0Znga4zGkvsumww5hrhgf0LsFS4hQalAapVSpoSY/IMU94DeDrBZ7X+D3CKAxGtI50Rxk2pWGBgeZ7POuY8Yzzv90PJDYD+BbazKrb9mUuRTYrj2k+ee8+hVk+HbNbrMipV8wekTTzDSPYAYbE+h783lyo0gBdKCiWBD3vsRnszw8V7PD6ASO3rAmeGDbMZ0/YvAZp9ML63PQZ8Vo/nVxJ09U3G+81oAo9o02O0ctiY87n2HsNSjptnScDb5xnPsM+tPpgwyVHjczcdMb/64x/jW14o1ViTfVYbY34AEL6MFYW3bdsyazSpcGL99CYxs/3+hHc8O1Z90Vt99swkHPfzwNPoh5hhKJbeBhKf/IC0YAbPhsHgpHLRIE8GNTwsNiONPtwGtCdzMmQHXongse7ZhPh1hsIpsP4GYdZmngqsHHTlsGYZ57w2AqmGFNk5yjwZPeGH+2nqaDknWYcbBl1gMwZ+RRvyvAKlatlVLN/QksyD7wPZ+hSKJasGBreqpWeoG/WNLk2JjQPy6icb6A600cP7uTASph/OG9N+TC7jUrAvEqQZh1gBopgSobGo3gYB1qopTxxvAlQXjDA8zMDzNPhXqqzPBvkZ70IOhmPGPem/KLgYVXM/aKRXqVQ6l6s9axNbNk8N/RvRnAlvE6/SrZqXMPnmGrDwAEqlDub1bEbg25VTdOMYZhDM8N5pvyjCP6hpVsMB6ui8t5N6Z9azvkP6wT0aj3rJgAEuzq38n3M0Ua2+KkZ3Ui2rVQheR6NgZpvyYDAuqdR2DgsaMaOZ5mAB1mpsO4Zb2/TDBVVVs+9JIa8Sx6weHX4Km0q277NjQOw6wB+TDrQw1tHn8zH+hR63PfZw09gd5qPSrev+Nm2Fl1IM/OCs59n35MgGH2cMHewDkgCiLUwkk5R56m9Ur1BfmSNcG1tY3D3ijzAOTqHDea8nPgCsIfBziNUs+RobOr+E1UgzNXFtVT6tCTFp/h7whgALfWGbegDHcweBrS296Md6z8PXN1U2lSIQABwY4c4Yf4Fs43o9sQGtOXeNY/7GHorG72rDPveyaBvrwYwD9aUD9lac+v8AxrUM8DbVDxH7jrbx+fPBXDPCizmWAY5uqnh++XNcGGCe2Ox3/Pmnc9ZoDWtmcLqMcV31U5JCQAeDzEnYZrBfYz0iYHgRbcok9oPeg8GBqMOyOFZgeKQPa1WhOOeyDPM1n3qb6QrP7Vw5gH9CZ/8At4Rvb8L8/wDsT/117h0/Nw/7tdJn9bC5jxg4hfZKkNQYMrRqEroPyLR63G3P/wC46B/kNV9Vh8Qp7uvXOAPOH59EzTR8nD/2OJT72cet8/WNSmTHX+85IBgs/GuF95pqUxszPPPPrXdZNKqJtGZ+jmGz/qZ/6ijs0qpA6D4ejmAGHQfJnmH8C108+Pw5VDuqpA6TDj4G0YYAAdiZpVzyqrVGqc4wyDr/AFvB1rsYUepPf/w7/wD9sf8AqJHar73/APDe9+SGajph78fhyqBWJbNU5V98JJ9mBqUdbjxps4wlYGcYwA/nXTgtiWz7D0dqkH/djQfZWqmf/wBwGZ+eMa2xDH4f/dyeNdRnn7Z7Dvw2AokOsczUdNs+tdwZ4dXwf7L6Of62TUj+TTiGe8/RsDPzhGNMwfD/AO7iU+q5tNNyz9gGAfQqwK3hFOLFlyd5556xrvzPCLiGf/8AD1G/j/11K/ku4qe44A00P+7M/wCumOE+H/3ef4dYlhH0NTP5z61HqVwy3mvVuuYYd4GeZr0H/JXxfAt/ASm/+TZ/yGpAcHOMT29jgtbwf3wAB/406YPh/wDd53h3If3ds5x+AGHjHmpfPwfaagfkA16Ah8IuOEkv/uc28z8j3LAtBD4OekSA6cW37VjB5Mw/yLzrPh/93mmA8w9I55jmTd8mijnw6/Pd14kWYz5DOGa9NvcIvSPAf2G2A+iYAf5Eyzwl9JAy2VKjxvoeBMNvk4f93m+NRL7k75dGnvH5wZNW0C0r4klmFnVsyDoBoOtejQ4Iekg8P/tjSg+h5IOAPpCn18SaaH5z/wBRc+D5PX/3eej4XcSZ8U5TdnT2ZHZrYAH+Nar7GXPcNtR7fpWASITPjG8ezPDBdbk8AeOEaOb9R4m00GgDM+vo/Qs/avAG6rnpvrGLxGjRt+BhgZph08Pretwf9zl8P0cuKh5m5VabGP681qKVwTuqHH0KrdVHB3vMGTzNdTD0UalJj68vipUgL/q8Mz/zqIz6Mdsf07ixWJJ+flk+dl/8Zj2eCFsg6Eqq3VG1fkjM4H+tXD3DThkzS5GnFo70htk/GwZBaNn0Y+GX9OvSsSQ8mwFYM+jHwSP/AKcqofXMD/UT9fhr45fUc0oNsUfDmrncAI+jgGZ9/nVwzG4TxvaTo36zWl4e2lbF2yqhSrqY5mPC3gHz54LXPcNODUb2luAf15msuHm8unm5vWcpOscNWfYVypB/cmf+uhk3ywDRsUC57h5js8Y/8660zZPCuMf83Wkzn88YzWrptBgxh+625AD5+WBd/wCfL5X0eu5YdjcZZjEd+Dxew1AzMHoYGnmbJ45sh/8AdXpR/XRw/wBRdqh23KmeGGDPz4bAVszw9qL3/Ssb8gGa2+nmcHc8n3h6NPELiFVAqtx8TaObrAYB/NpsgAfkXPA4IXqzcEix/XFK5sDwB0AMANe+P5PYoF96uDD6GVwyvQ2KD6QoRJctl6OZgZmH0LD2bdPrczh5+idxpZA3KVWaDP8A+rtTN5/wLH1vhXxmtjw65YFVjfPsMP1r3RXr84T234jlchg72GD2awVV9JmCy1IiUqD64juBgDMgNi98OPzy/wBXkqHb3FFlo2Ps5JAz8h4IHrS4hvZ81bD35zBdFOZOmSJEttjRB94z0QPYCNl6WHtHDWuFuVHYfEYy8Slf/fgVrGsbiEcLlXKaz+eSALopzH/xDTrMx/H2hpgcw/k9vsD2U6GH/fAU0LGv97fUWKaZ+cHgBdIzcP2hpH+dbYRtN4bwKrTbSkUOq6OqxmGbJ5gfWqKgxr4syH6gg3PbD0QDM8HutbC0j2ugffma5pSqJBO663SqjZx1XCpH97A/Yh9C2hjcN3f8D7Q2lNpwP4Ov4YGuRBwiquk02dxsmYdBmBruEwM4prPmyot45i9wiqsn9ruqMH/djUtnhLKZi8q/dpveQwZ6P1roARk6cbaow1cye4PtvNGw/ccl5o+zlgSZ4Pth0XVUg/IC6WDKA2VTPDnQcIqN/TqjJM/PsBI+D9vvH4k6fh5AewXQuV+KXK/FatsOdHwNsp7e/wCsj/7ypcDg/ZcAT5WDPDPzzM1vQZToRs0Y4YpnhpZ0Z3U9Rg9/fGZqQFmW4yecWjssn8nQtacZszQhAA1ztGXjWfQ4zoSm6VA1Q35myrJmj2qb5v1KzqPJd8+jgrj1aCXIB+GH/wDtBXnTbAMMPsWyz/8AT4IQoNqsug5TqxPpWHYEYFZcmHkS5NHmwhTTDByJxY5Y+w5DLwArJmm8TdA36HxXoM/yYTDDNV4Q8EuV+KG0h6f6UMaPrsGzMw/BeAwTH2t49gwD863A39bpmCdjHOgO68GUbLvnBaCBfl3Rnwcl1XnGu9mQyBghtlKlP4m15oAluBAPvMJOGajs0q+w/wDe7/8AuV0WNG4ZXbWdOdFn02a/+QP1oaxwluOB4lKch1JkO/WZA0Nsf/y10sHLtZZ+t4EHJ1x5rTPiMYH5M8wSMHGSNt8MDDYexJe4O6xw6O/GDxL8e1fOBngpAMvgXj35MeDyAChazf8AUpQQJzwhoQZJn8jK8O60v1k5A8SJdVVD8izlYr1x1LMJVxz3gPs1la1K27jOGbjFDmf+CuO1V65rSqUtusxdFp88wPPvXD7MPpetf/lsucuploGG6q9pB2Z9Cak1Wog0Z+sp7xh1hzJgsOzxFnAWmDAKXWKlVaxFCdSouB/g5/5187Dp26Vbcy36rAB+WxWNXPefMmotEh0ZmvVBuWxJ0dY8AdWH4b3DczOfPQTDfgGexaKTUuWqRy8F9OP+HByxtuZh2dAa1/UZvfQ8aoalcluM5vsQQjNfgnvQUSt+sndDXjMn2A93q4eo8GT4dV5B4PIvjez7N7xDr+dT0S7aHVXdDlIx/kV8YRZPs2AAfkWErFpBTalnbgaMc/zrW0etvwI+D9D5z866PW5//Lkv1kv1bEw9mmZNNiHDkbOxSZlw1V4wbptDhs4efegC5LqDBhyJSg+fRwX1Y9mHH83K5+zgzsPrA1Le3tZgoNSjPhMNvTPM9+AKXDjTjj+zX0Y9mLhjfGsJnKxqNrvuYAuZVWjv1vN+mhmZ9i6EcOcY6ZxDe+Rnep1KtW8axI5SgUCf9Bs4LhtcQ841K27jjSj16NJVf6tqQH+wvZ/QvYbPArijUsGJ0GAy0fvnjzwV7A9Gmq5GE64IAfOEZXHM2w8XhTayHt6bJD8ianmbOw2zD6wXvWH6OVD9nOuOe8HeAACu43o/cMmSByVSnp+HZIMDT6TD82nnlKpUlgHdRzMPrBfpQzwN4SB/7nQ1IZ4RcIGT32PSgw84J3eDD894cymveI45mal85WA2QXJINeTljNfogzb3CeBvg25QYzod4RgTvM2OyfhsUEPyAsaevzdks1V7w+UmG6fRnGMEDNn3dM/ZKHJNfondVV4XT6WcSvzqPh2YAGa4P6+4a2M7Ily+ZqUczzB3PBI5WeHBIfBzijJaBwLRmM5972wFpYfo08X5jAuMMQAz7Dk4LpcP0qOFbL/qqJb9SZEO88MFRXJ6T7+Bxbco8CMHY6D29b9uHrj9e4S3/bc3ka5Q9E/OD2YGqz7JPsn/ADrO5APOgu3ipfF4T9Sq1l7SDsA1ndFw/bvvH9ZrpGoqVvWlGim+xeOsYdiy7wRQP7q4Zh5zUqMyx+GGaTwAYYIyVQSTB3TPBWYTG8cFXHAfB3U6wTxh+GquNh05LaQSW8fabEIRnDBEEYPZqMA+ZA+9I9NNGy2yW9w0zzIB0OGaYD2e9Fn8FHN7LsQ6zmHQH1phCVrJA986hZGlkavAhazgH4aI5L5jvcUt6BEePU3gmjhsMlh1r1Igk6MVGy9nvf2JrOJ7Nzr8idNlvHf0KAEOS5Dqg1KK5vBdLpV/0evQ5FGrPghnhn15rmRnTg9gijAwDuZhmsMC2qr0GMDzFNbAw7DVCDzHv2DM1ZvaZnqBsBQpIfgLcR+vw1cU2Zy1NCmxGDM/OqkKVUpjvhgf5FZw405nM2APYikjP7qbnejsym/zzz0pjNV7wYStc1rbGjetSkaE4GeVDM8+9BcPH6ymdGAd4LmVyT3J9ZkA2HhAeAfOupT5LFHgS5znWwz0LkRhzMo8+gzM15wpau2IB0qL61P2p9CKqvYNaneo8Oe+boOP/wCj2QwwTR1ViTIM9DNrsU4UkXnsp1Pnae8wwNZRnXmTY8GK3mb54YLUV6YxVbcaiNufeGDzwUWxgg0qfIrlVcwdhBmz86sSrzgN0FiJSgP76fW15FZ0171Va4U6dszWP1p1z3UdV7D6A8gLR1tmdMlHEb3gHQCyepsm3jqsBoHKiAU8PIaCHVYNHDkeVjcp84b1oJNBqNEowRDimeYZ5gue3UBs4B51y+Hh253qNM8enMYLLp143HmlFwcBfU4QjNMp44x4m55FHVpSGXtyIzbNRw3kpAQ38NTBBH6E9n8ET0Zz3anUqiTqkZgx2IpX96d1nwVx6niMwtcDzMFC0c02GQNSGTbz9mmtH8NJll88MGDMT78Nix2lNB5PYS9LXCK9j5+xBo7FYMnLmHHinKe0uhSKoGZZ/ej6FKCM2fWp0yHyzugb+wA2fOo4HivNqTaaywz7c1IkycHdMD2KqN7MsPIaRvb1jYvWZmftHN6UmSbxaarAPAPmT0Z5vV8RRECayb7LuGmYK1BnAQcVaElw+tSOZ2rJaQ9Jb2Auq2HGMKbH2da4o8ZnNhNt/jL0RZ8AAitN+RdHrMrUXHKTm7SqUfQAZmuVcy2yK2HHKvNyb3OKx0RYbAYLlj1SMDTlF3z/AIWnrmagnMbDr86ojqr/ALxxAFSz63FGPIuJMxgy1M00E/BUhz+sE1zh+dXgaUKq4G/ydinBPbMtdxz6Fj2Zm8FNCTuUYGoCTmftM008bnOhg2okN7PxO9SGQN5/X1NiYatBDZzLTbVrMmPnP5TwQBjDrVVTT+8ArWfUqbG9v/8AZrHmAnygZm51qOdebZ8DXzD6FBmVKDpG5r/QCyT1Y8c9NcmPI2vrjxf/AN/hYIYdSB4vHw+TBZIKqwft94eREFSw94vOmxu/WuY+A3gAbMEs2JLvj9BrKMzHHgzcM8Fdw57YABg2B/WmOtqtTptNMMIkF4z+RZyZaVYM9SLbFVePzxwW7oNSfjYS6c/ol5wBa2NcNzPDgxeL0MPIYLp4beYcFk2fc3uLcqR/92NWFpWrfEO5qfO+x1SNoDwPZ512h46y97e+DP6GVGehm9+13xVQ/wDpwwXfsw5Rc9n8SZ9bmzvsdPNp948MN+xWFKt7iNGY06bwykn5zejb10j1JSgD/wBoLhP/ALZOswKbpaHrGt/+Mf8ArrHmXhhwgcVNI2Jdlz9/yIPsTfFSENe0ZmYdBnswW+ZO2YbWmFVuf8kkFU1WpW+DWpFnXPJPyGZgsW2PDNHZPFCBhEiUA2fnCYAGpB8Mb4qUdo50QNXyS54bFYBcLEY8GG68AeQFew+UksA+/LqQZ/imtmOGF+wd22qYTqrLpvKZ4YR3szBOmZrQPU2DWH3XAu2BTQA8NKWeZpBZj5+J9p6I815+Z60t83mXrPo38W7/AGKZfFuBR4dPOGARnp0/AzAO/DBWEb0M+MT0Xnpdz239AGZ/5FChvVhmHHpsWs1Ll4oYMgEzAABUleqXEaNPj/ZxupTMAzk/fD2AahtwsED1xvAYU22JMkwMwzjw8wUGNb18Mx9Biw6wf1s4LscCZUuTabff5Z3vDPBR629VTppgw+Zn1nmsmrl4W9xKe6LOns/WCizLM4jG1g5aU8/oBdNgPVUI+vOfA/JgrWBUnwzcfAzAPICpvhzW27GvisTdCqxPUkfvkTuhbC27POz5h01y4I1Y1zzzjgYACuJNeo5+3tGMf/Yn/rpmm1K3DqwPsReQZ8jUZbRbG4XdetKm3VdtErlSrD0B2l4aLQBmDwAee8+xVlb4Y2idxu3UxXJ7zpyQlci8AYK2qVz2qZtcjVYD2zA+YjHsNQfWtDP/AKVt7/wTUcvDttCqn8BOHN5z5dz1W7qlDdnPG+bMcAwA1TvcB+DMB0wO7rnew8jILqFt2ZWLtz+z8u1ZOAZ4BM34fRgmq9wc40vHytOtKiBh1nzjKur62105i9wo4JM+znXa9+dkFFDhjwZz/ZLqe+t4F12Bwr4twGgblWdRHj/dPAakPWNxejezsqH+RkDUdyNuVM8MeCQB4lKrYB++k/7amhw04Hn+yW49J+uet39nuLf/AMl//wBgCobhn8VLedFh+wHjdP8ABpoGCd7zaqDh7wyD2diMn/3wFNZtXhyz4f2HgAfkzWFrF1XVTdIKrFqsYzDPB2AYLOHeD8Z2Q+xOeZkPnmZmyYLm+jlb+LdojUrhWEgIj9ow9/WAGYIJj3CsKl6utyx4b2HtnpG8M/kXn968AN0wqU4JJecFcQ7noz1I0G33tL8INix/+R5dO3Y6PXuHM+G7Lg2dSnjYPDRCMrtm8+HMYM6jwdtvDzmGa4ZSp5g7oU5/lj7PGwWgO8HKbI5SW+zUvIBhvP8AOuHzxezBt12BxjYpT4OUOwLbCF2AcbPBS+Jd1WXxOGmOTrRZh+q9fDRjdeeHkw8i5f64ptVppgxrMu6OsBgzngasI1yMHCZ02Hg+vYq9bm/vf1cfNaW9QbHjSIkulWqyzNCSAap5pSabYj0g33OHkOS6fWbzJ5maj8/nNpXhvHnJz2B5AQevoJyAYbc8U+hfa34cd2tQh2xG9hYlHAP7lHnb5+0seiPH5zhgo8aY5Jz0HAPDYad1nwW8eYeJwVWKAabFq0cA+SGCL1w3lqeo6V/5YFWcw4vuea8Fh6+fDop0AD8gRgXz7W1UPZxIAf8AdlCw+KWirE77W1z/AKt/5YEz9p7j7JQAHk0QUXRTuigd+0lx/wDxl78gAl6+uA/aViT+tNaKWHxQO+sqz/8AEZP60Jz6qZ/t0n/xjS6EWQL3AHmZ3vJ0no/GNLmZ3s+emf8AjGiSTAaPXP277x/W8aRsmftDzD51I/IaX5DXgiaLnY2CaZjVmBNj1Wj1E4FQinmEhpWGfyH+hGH51lgbC2ON/Eq28+eiU2seQ3o2BgtxD9KL/wCM8PHsPOD2a44lh8Vdw835d1D0lrEmR851lVhk/IEYDQs8fuDLx/fqNXo31wFwnD4owBML29ABx14EGPsKx+eGl/LlwI8lV/8AJrg/LNmlyYf1phr2O5vceOB7Ps4NYP6IaAPSN4QRvZ0a5P8Ayf8Atrh/Jue4bR8sfvEwbdyD0jeE5/8AR1w/nh/7aX/pIcOQ9hSq9/5b/bXD+TbNDogCdZ2O1f8ApLW4Bf8As5W8P7kFID0n7R/+Ua3/AALh+IJYZph72O4f+kzZ3vLVrYfkBL/0irV/+HXIH5AXENFv+tLRb/rTB2Ozn6SFv+7o9yH+hRXvSWYz+6WXckn/ALzguQ5txkvWUH8RMDrH/pMn/wDi9rH55II//SXP/wCQKl/5kP8AUXInp8F73i6LSuD9lVWOz/8AbmpsaQ/7rljDf9eajAsD9Iqpe4sd788xRHvSEub3FjwD/vp/+wpcz0b7npQG+FSkz44e+gsg8Czj1h06MeE646lGd8j1NWOfB1eVsfpD3aH/ALgUf/z/APsJk/SNvT3Fo00P+2NQgsC3DHx7xmH9ENPBYdpB/wC81SP/ALmCb8Nfn5jYceL85053qej7wwwAzUz/ANIfiP8A/Bqao32GtLL/ANo6l/5MFICybRMP9M1g/ojAnbxNPh5g/wAv3EY/+iqUYfOyl/Lffhlqep6aH0AkdmWVlh68rAH/AHIJfYO1fd3bVf8AwQTt4j4OYv5bOIfvGIAfJopBxpv/APsMD9CX2Gtw+u7Zh/8AdgS+wdq//N1T/wDBBO7iPg5jMnjNxRmRZFO1IbLUoMDwZ7P1qPTbwuq1WApVKiw+XDo1mQM0NyW9Q7VhxKrBuOTPdfe0OXdDBWH2Vt+qtNTpd2z4zr4Z6PJ54fxrh7nR83NjAv5Y+JoeGHqrD/6YEv5Y+IYeIxFpoO+flgS+wduB0XjJD/uy+/Yah/8AzrK/8mC7vp4XP8vMA+MHEZ7rg0T/AMgCdDjHxKMP+jQ/7mCH7DUP/wCdZX/kwRfYmj//ADqAfXDP/XTz7PD+D5uZRW9ValTY9QqtNfBmXKMwe8HMOv8A20QXtfYHqN1zD5+TBDR4EV6RUKNz2GEkw1cFas2Nn/73Rg/7ma+R6PsR+bdHt+rzfoR/5SOIwDp/aN78jIAoj17X/J9vdtS/Wrj7Af8A86wA+uMYIT4e1X2cG6qJJPya2C+v49nhcHy8qkO8L7Z6L1rAfQ9gmnrqviYH3u9a2f8A3k1e/wAlHFB7xGKHGmNeePMA81TzLbuejtalZt+pQ8OvVjHgr2Xw+eNUn6xeLUfrFVeP55Jo6bD+/nm488Zh1meaQT/dtvhmnoBthVg1PeAuD+Kf/X34bfw7/Kg+p4Iezi7w70fJth7tWHQ6efQBmgXXwX+Yhz8n70Xlg8iXLfiNqUj2GulCFyYGCDkw/rVhh8UsPijJCCMnTjNh/qKRouf1oMD86sSKJskH9Cx91RqHSrjlvt8RpNEkTQB+THw71s6V4Mgzc/4exYK/Ga5/K1EcodKptS52mgHLzg2GtWrZUSS3MpAadc9cf9bw600YKwhhowgByDGhumGbzUfoA00bK8sQcPilh8VL0UsPisBH0UeGxO4fFLD4oGtFv+tBh8VIw2JrD4oyBh8Uv7tO4fFDh8UAJAeKPD4r5ouf1r3YHWwJLP8AD6EeivmDfs14BM96WsjxBIwbBe7ag1sEtZIwSh02dWJ8elU1vORKPYvGRp6S2CAKkwfs2De+gF2OicE6HDd165KemSG+zPYugQKPb9NY0IlDgM/ODKDjNmW3bEmRHqU6pViHL7GeQ1gD+NdQmUGonh6qpVEmB55DOj/nQ1u+aBb0hrnsAaM8M/ItRUqV9obXNy2KlGPnWfBdWA4pVbGtmiar943bGjZnmceP0AqSTeHBqj4RabQ3q26HeAGf8axVVti7qbXnaVdzZxnfxZGwDUJ6mhGP2jJ/Qa87nThtXuMzkYf5jseMyHYZ/wD2CpJnFS/K3C8CsQIGf4TO9ZepUf1q1oc0bIfIhjWrUYwGFv1nR+R4M81j3PcIleuG9abFOcd6zDaDr0TWX9Q/bOAFRcrGZv8A9oPetWYVE3XYlxhAPDrBBGgWi8ZsUqc8DodccD6FHc9Yc+FdSjAb4ViMfyLW28cSBACnS3MJAdmCtQZDHTYczWarcmmxpG+C9zAf0gN4LHued3lo5ODOHhnv7zVfVcDgO6jgBszTsB6JPhtORZwGeHQZqPrUOeZxakwyZh5Fyd3k7mft6qweS+/VEGXfIaGsTGzI50W8WQw7DNWFeoNnPQz0Hw/OqRmj2/PgcjEppmYec8Fo6e9s7Pu1+pQNRyXnge9aA7zp0bZr4Lklt29cdH5jlA8J88wDPoWgCM28Oc6LhLD51yXwto9nDVz7zYBg34M4AMPOufw7qfmVsJ1SczdV1G5GY6EVveZ9ix8+26lTaibjDgK+GFdzqE+56NTaWFV5TWe+tfbe4i0O4YTunR3gMPIuaTKDccyBr64Bh2GaZt46rR2j0JZsu+do104cnc63bFefeuFqIb7LMhg88D713iZxRo9HowPzpeboe66F4yjT53P89Upet5zWlqV5+so+g+4Dx9hvBvXu/LF6YtXicxcM/lA8EPPmumw2cwXgiiX/AFy262HKYGHkMF6StLjrBOlhKrlGmRs++J0LojmeuoVu7aHb0U5cp/o7Flw9IGwNUGH2JIZ964PxIuedW5Ts6DKwA1yqTVbjCRyOuyefeCbH6BUqt0atsc3RnMwQz6a3J9u2ZguKcAbkfgRY9NqTex/3ufQvQHWmxip9jQZMjXYDBcU438Pb4OGD9OowPRN+bsT23YvTuII1tA/OSsQ36DngxUmZAfjMms7VaxVZLWg/KPev0tr32OCluv3BBhmfZnGA814/vCt2I9cMvTp1Kkwj6ABldPix58CmxTPUNtSOQggPs126zOFfDziLWQYbOTTY7/8ASI55mC3VS9Bh/rtziEb37qdD/wA+a6djyVJhthvbbTWH7wD+hemHvRC4jUp3Tf8AVs/+5eVlROFdftJo3KrwyZmNecwzBO55h5npVKrM8zcg0aZJaDvAFcfZWdJDx6U8yfzrvzxy6O6fqO2HqbHPfy7PRmqGsVKpVuQDkuLhguD6TDk7NmYD97zzVfPtsAkbAXXjBjS1H29iz71NAy1MFH3nS5idHc7G1HOlH3trpb1E/cKP6mc/sq1/mEGHN/VRmfs079mze9gGZrox0eJJa/ZQA1H+zb4eJvw84da0++EYYH7MSw62DUKZSn4xYaBmupxoDbIaZ+N9aZOjwfdsGo++DDmUam6wZ6+HyJ06V+ItwdtsZ6mgYfOnvULAedX9nCYcvZgTjY12GMwRRocXLOoyzzDsBDrS/dvmi0T63F0OM1JhwczfYz2B1mqrmXz63FbvA4DSqMMCQSoxwQLT0HjP5FYZxOX094H2AosaS+AaeAYKQG8V7sQTefM9isGYbjIgcpwAz7ASjMsAepgmpnVqLwXdNnnAlNA2eDSuHqkxJM3AfZ1TWBCS+86Clm9FCQEuWZhgGGxe4FhPZzI9RRKac6lTWn4LZ55p7mTqTRy4vSCm02YxGmAc7eCzXhd3m8b1LaA/2gwzeWEhs5taji2dVqrE8DYYYZw7DBUOi2Hhtr2DCE9JlaXKsbGu8MEGDgNKW8GB6fegNkzW68Ipsvm1qH2JnDwjUt6M+HYmuWf0jbcAwzWTHw+UF5yNICUwug29VYLLsiVOYAwP+BYyBG5YfZ7FLA3Mz01Nup1W3r5Y1eRYPCIahX/wso11Dz1DnchI8nYuYgb4e8Vsdw18w/0xJ/WuTx/RkyMmlTqVIODOYMHWDw+tNcu2rCS8487m++Zn5zQaP4a645kKp5nMtDBRzhtgengrhmNh9aL1abxanettiljQ3wd6Fo4wAccWzRBSjAAcw61LCjubHM9/kTYiHRMxPT6DVhbzLASjiBmGzrzU6MGB6AOBmfYnQBtmR4bYB9C5r5koXqpiNPMN70c/OnZMOCEU8G2WcOjMFZ+GfWgeprDx6i4/psZxmg7AcwMNM9/zqRJjNnmeGDIdnkWujQDk4QYgZgar5NNbZdNjUDMPnW302vDLvGYCDegAACZZky2T12GN/Yaszps45B6bGYJ2Yy5yug2YBh2LbuFKZvyT0zczdPoDvNR5LM6M6bcuI8z9asmbkqtBaP1HEBmaZ5nLPeaixplfuGog3VX9Z0+915XDxXhqA7mpBnkgeZ9qHvWOvyKv5xsPeLdNrUHsDRsvYGDnWq8HmzFGDziyF1zLh9GxPAaqWXjPYpwA51m4a9wpo7VZ5m5qf8h5r0baUPwgXBOHUPWrwPn2L0bR9CNTZtVM/YRjMAW6XlfiFVTqV71uXnmHM4B+RZV55OnJlz35E7QPx3jNV8nPBQpEeezBNMvb/ENO8trd+CTNNlycG4gA8Z9jJq03Bp55NZ5q1Zs+6ngdccobwAAd6MLVqptNHEY1s9hhn0IKxneemraMz/whsPrA07JtudAkafWQditT0HnWsKabJgzvDDvWeGpRgfw6Fawz8JUrMmqvOiw3StEM95un0K9ZpUp49OJUgz84AowLOm6hzWm/Oaj1KTFCpOtvnvBMs0G5oErnm6qy8cXeDJ96Nmgy9c5dZ1mRlBvNlMOhn58zNrDf9YKkmTGw8NtdACg25DYwNuTJ+t5XdKeocCiaD9DZP6NiYHHOczJWEY3DwXYIFYstmOGnZwAf74/9hWwXnRowfcbODPz4f7CaHL4caczHw0DP5wWooNElzIuegYfWriZMYrzQG2+EZ0+sDPYCuqVJo8OPoVi5gmGHZHZzwXHzQIttw5cOVykpjDDv7DWlONE9aCDbGZ9gKqO4aBD8OI3Jea/tDp4YfkUiNcNsyXcwqIPO+QFERgTThsap88wfyb0mTgzM+RY0cO/BBMue3AINcJO/86N6fTeV5th842B5+dbLRwk4RTPQ8Lo3gnQZnG0csJZnoBsjmGxI6xb4O+G489LfDZmGwEJ3JRmWtNus4OsBvZaA0DrLM6ZFN84jOZ+6ZBO8sbLXiOYKKzJYeitHrgbxhn4Txp2lPUMM4PNm86+eZ6vQgaNl+MIG/UswM8PBRfZWDM1X64+8y0AdnXmrP+aua5SJODZ0AmjmMH+1VFl4NbefYggyaPSpL+hoH4AfIkzbdpPRXW9A4csAz1jDrRyeRA3X/BZ1/Iew/wAiTMmpTA8OnBMAAwAwZeNHhUqG+E3UYYZeaMDAM1Ycno/0GGH0b07TbM4qVJrXptqyTa85ngrOm8H+KEmab9SiMwM+szmYfwYKNinMA/DjH/2Jgge0A2Nt7zW+/kTrn9KvWB9YMnmrKBwKtFl0pVYrlVnu4YdeAf4E29cq1gD2+H5w2KJPepRjoMTjzPyGu7Bwl4XM9dvzHvreVxAtLhzAaFuLY9NAw97IADNB5hnmEPlwoGcx03g1gNkz2LVhQbjuF0GLfs6YccOswZwwP869KxjCGOnSmIbLXkZAAUjmXz/a3w/WrHneNwEvyS1ruN02H8kiZga0FE9G9gJDL9x3d4XfHgs9f/bLsZ6B+/D9aDNsPZyg/WmxW2ZY1ncPYTsSzqHGhm/hzMs97z35/wA6ujefAPaKPqN/jh+tRzZcP+nPf+CosSDk+dxH/duRvzmqx6G5/wDET/QoT0aX2TgD8ilS71pYdD7P/jJHVZwf2b9az52rWZJajdSeA/I8zgmgsy6pLugFSjB8/QiML2TPlyR0ziRng8mYf51QzJLEb2doxj/Iz/qJp7hddxvg+D4T3Q91zIACCZbHEaiRZEti0oxusBsDW1gM/J1rDbbClqVsWxcmfrnh7RHs+s3YwZrjPFfhLwSehR/sxzNNqeeZhEPNkADDrXYJl7cW/V2hc3AilSY4d4f7Czp3Pbh76r6PVKB3znrB/kSPZXHE8+VXhKD3L/ZyuPMu55mEsDPM/wDIoX2eqtBrn8+Psm1C8c9I8wNdgu0G7hqnN0qjBbcfDDl4+eALOHZ8GeEv15cb0nmjzM4/gmCXy9iOaGUkydYT9XSz+9GDAYbMwWrnnRqbQdeszowSP3IKkqQRKPS/s5bFHekn+MZ5n+tR6azUbnpfq26ovIYPZxncNmYLmuHHcLi258GZmEGWbxvno4Go8+iPxnXZcTWhxwMGQM2c8D70mdCz84tNcjTAP3ykQ6Vf9wsSOaptVgUqU8D7Mt7oZBR/VGF9JPRtIKjEiZ8qBvGGt1+c9ij2BWHzr0eWxTY0wHzfPCQGYHs8iVHt6uWNhEqM6jvNG97UOswPzrW0HgbcdbrIVW1bVmRtAwMJBvAyGH0GtPE+W2Fg9dvDWZIOJeljnRDD+l0Z7DD6wXQKrwHpz1GjzrLqsl6RKDMAnSdhh+QFZ29wTtK2JB1K9LqCqzQwzj7AD5M/Ot167zMPVT/3fAABd3jmMPO8/hjf9Kd036AZtB76O9mCp+WiU0jCuMVuM72A0yAf412CiPXpDv8ArESdX4wU9jOUzE7zz/8AsFs9Z+T4c7CS15DDYtu5eHm/mbc92/Xj/OyjN6jvYcjEn595yDD/ACLtVb4UWdW47rkSnerZp/0iP/nDvXD2WcM/rMFcWi4GjZBjV+9g9h+660SMOhasB/8AFQv7DUv/ADIJYUDvp1SP/vIf6iBJe7AaP4beCHE1Iw+KLD4qGqRGOh4YVGh1KYfnZmYAvudpf/KNS/8A6qmcPilh8VTI7zlqh7O0ZgH5zqR4KLJPWfM22NFrsDPPBEkoA4fiKRD5EHdSosPPNeRk8DUdECNVnr2j/wDLlV//AKklrWb/APLlV/8AP/7ChYfFJBKN62MPulDqTLvnOZmChI8PiiBA0pcB6CzKA6jBOY0Huge0c/zoMPilh8VIuPtDZf8A+KiN/wD1J5B6+s4+vhr+isPKnw+KNULA6xZ3u+GzwH/+eHlWH1m5hgHk8ieyBAalZrD4qUyDdNwfqtACe0+GwDeMP8CEARYfvFAtqVfNh02YDlV4Ow5LXnZezP8AQa6xat4cBZ+r6jYgW9UDDDCXDwPBcS0W/wCtRzpsQ+xB0+8+GPEONH9cWPatmXPHc6OXDRP/ABrlT1yXNQWA/lG4ZSaP5JGiejmpsCTXKI7nb9cnwPJovHh+joW7h8abtCPytx0am15rvCQGGajYyVE4u8nhLt+9JNNPyBJ2fxrpsPjxfhxQ14lBqrX40hneaM6V6OF806PVbntKg0rXDPNqSbJh+gAWBvbg5wZ31XhXxUCiTQ9jHkTDeZ/jWTV0MOJfBmTm5d1gTKPh1yI+9n+BWf2Y4XViGFZtjijACIfQEjfmvNNV/lUo7Ehhy47euSO+GH3d7AwTUN6vvWr6jO3IDLp95vf7Cxw2++no07StgHdP+UaiH+Q0js+1eziHRAXBTu29DkNMOUCAyGGGYSc/8ivQvN+BHAJTcM8/IC8wfzTmdgZsa2Xv/wAIVN/8H/bWPptHt+sXpUKHEvGGEenAAaxhsM+9YeNf9VefwiUOBGd/Gde2f4FnLVC4LJrlYnXA5GmesT1tGO97E/PgmHT99vRH8ntnM/8Av5R/+xZUc7GtF72HEOlfnZz/AM657PvaDAYB9g2Z+zPBp7emXr5oZw4j+gZuvhm81rB4J+RMH3W1de4J25W9L/7b1Hh6GexmMeC0bNjW+EJqI5xNoh6AYZ4GC5fGvyh46nq4A+uSpEa/KHJa1PVofkkpg+7mdFCzLVDr4lU0P+xNA9Z9ie/4oUr8kY1y2fxFofrFphiKel74NYF8e4i2r/8ACp/5ATB93M6j9j7H9xxQpp/9zNEzatnM/wD4Sab/AOWNcq/lFtz3cGeH1gH+uvv8pFD/AAJn6A/11GD7uZoqDRIMPiJNtiVcbL0fPMJzTOz9Ga3T1n24Hs+JtKD62TXJaPW4tSrk2sseTPBEfEu2D75P/grj9SMXcOnl97ldWCz7Z/pfFSj/APgmpAWlaPu+KlN/8ma4v/KFbBn7OSf5FIDiLbIe8k/+Cu/Dl+vmdiCz6AHiMcVKayfn0TBaOBUpVKY0GOMsN5oOx4MwXAj4l2dh7CT+hQQ4l2kZ+HBkmvcOf6uZ6Lee4a1g/wDlUxZ9Vd87MM4Z/wAC5pxjoPDK3qjQqrYD8YOd8CezHeMwz2LntSv9uSEf7Phy2B5vcweww8iCpXVBuGVECJTTjaD2Z7881h73/wBej1P8zsdepXDm2KXEuODRqbcOvhm1zJ55n3mCyj1e4T1uVnUbDqVHj971Ok54fkVFPuRgM6aww9s7zDBVPrsPZmwAfQuj0/P9mGPNH63c6DwH4bX5TTfse/5gO/PgZh+Rc/vbhLfHD2abdcgvSYnZOjhmGCx7NSixnQfgy3ozod7RmBrpFsekVetBzYqUsK9C/s8g8D/Wu1g5oB5J5d1oMz0bOJH7dBO26mffmEbf37w2KiuT0e+WaOdaV+U2sNdgOyQA/wBaPHKUGD4dClz6bOokrkaxEejO+R5CAeR/+BbRYZDrVbds+36PKhXHXGKkchgDBl2IzngHzq6APeaiT33mFIiedk1cWKqg3nQ7zadcobkk9AM/GZMNisFScLgwsWIx+A8+z/GtA8GCuw1h8UsPiliaLx8/EWAaw+KWHyJ7M/Z9iDNzyfnQBg55EtEwL2ae+fTzRAZ/h4AvYZImDnkX3D5FI1nMg8PYlrB3sHh8ihqiHp5g30Glh+8NS+bi+Q/0JADZ9DeCCJouB3mg0cy/av4FOwb94lot/wBaoQcTSxNS8GzL2+/yJaIZbEETBz2aFmTUqVPiVmlP6MuEeYKbooDZXux2i3uKNsXCenOc9W1DvB7o/WtgEN+SGcXB5rzga8y6OBKRAqVcojvN0aszIZ/IaxwOgcXeFdYuGBqU0N7Z54LjNBu2/OG8wAYqs+jyA/o5nmC6dTeNN/wMAf5apAH9oDf+tXD3GCxLkwiXrY5n5zZwMATAp43pDy6lD5G8bVh1Vr5MAUSSfo7XVhgxUrVkH178wVnG4XcD7qdBy1b0mUR38E3v9dUlw+jld0DxLcrkCsNd5meBqLhUWkBwHiVXNyx+JNNnx/JI8E1lKxwi4t0oz07Y5xoOh6I8BgqWsWBflBz9a2rPAPxmgzBKj8SL0oOHq29axGw7Dk5go6WipuG1axGm/wDKelTIEg/OeGaqqVQadb0g5dOYwMwwNbipcdeJslrlKrWYFVjn1tPRutU4X/bkl0AqvC+gvZ9ek9omafMM1MmVKA6dRpr56odjp5gsjGue6pjroVVh7Awww0V2iHcnAg8/XNgVWlNeeJMMwV3AD0U6k7h6xqUbP8UzBPjebeZIcC6ocjXYghJD51bQzfmT/Htw4xn5F6QmcLuGUx3XtXiUDOfQBgB/50H8gNzmwT9Ku6HP8ng4Z/oWPQPPk+iW/AzrE58zw/o59CsLeqtpVWVyrFDOBIP+kAeALpcn0fr7A/v1DjSQ7wCT/sKqe4e3NR9gWPMw8mGsnSxwy9SrDFKaOmvytF36FmWZ9VN3wJYGGfkUi7bArlSqhy6qxPppn5GTwNDR7PpUDBs7nk6odhhgnSYbI4GjTWqlBYjesAMDA2lW1KM/MfCsOP6MjDB6PhsVrAmOAAZubFmr8ZpoR/XDc7CQH4Rrm6TKJJOo1WQ7EYcwDzqnZt643s+Ub1lWeu6l7s1oKPW6rSmPWT+B/IBrpwzY8JNRjSNOVsPPoU6pVttmEeasp8mgXbNaOq0aTD/6xHPeq37PMUd0mzfeqUfswV9Y3VscRaBWKcFNrNKZkvdkgGd4Go9VvydSjODBqOi19CwUOtwaVP8ADp0kPyKbVeUrYG+xEDVPrMFt8wszu1isFoVWonmfvgQPHbMAgnfaOYZh0YMrJep5bL4P4dC0dNpUGfH1JYJ8w3FBvCCcUPVs43mvOthA4qXHDaBhi56kyAdma4+yFHjeHErBh9YKyhvA9/Swe+hXfrN4t2tn0gbnh9dYZe/voysIfpRTglA5OpUOS13tbwXHWZlOOObD842c/Iq6ZGi7HILhvB9CjpaO4X/x1p12287TmLY0XT+fNeZa2ZnUjfWo1nzHwGDP6FBepr8k99Kk/oW3Sz2hWZdVz23VwnUaXhh2L17ZnGyuV6A1qQY3MOLykzbbgeIFNn/kBaClU2cbAMaclnD8ijmhe3rB67bteDxLfZ/Ws1XrwuNmLpy2OWz+fNcRhs3jA8SLc08PkzU4K3fBnoSqybzXkNfK5retk9dVVBrUzA/kMFVT7kfmB48GN+hZ+ZUq5GYzbBk/rVE9cNVe/a2wD6FwXYm1t5g2s22wDeqfnMC9moU+seEW/NUL1YfPo2JgbDmc2tTTNRwPMln41YlZg2auIdVpTzunzYA75DWObFsYMGHsMEHLAYaYGjB5h4PD3/nS8N49RvrUa5FK96NgSZPTAdRxS5OzPNVjzzBgefQtoSMwM9+GahPG57tWECYw94ZuAH1qPMjNs5m3h+RX+ockZpsoA8RtHJDA9PsTrMkzY1+xNHJzX6d89EPYg5Zt73aTx700clsPZmrgA9GNkvDSZN8+tHrGfWg59gPAwPNUo944IHgz60ufwHT7/Oo5vOL2ICZDB0D7wU23qa3Uq3g/+zsBmaigG9XUx5+m29912SHz6Fonwz8yS560eYisAETsBTTjNn+yuZj501UqPLo8KI/Lz1Xwz3qLGkvstBh3rJdrrmW8AbY2YbEZz4MBozf3yD8nYqTWwA3ASgM+tZum/K0QDf8AWiF2cN8IoVWU4Hib8ARsnrFhsBE88BymogZmHQAAjk00IzRv4eL5O9QvYD04Z65t5/Io5vaxG444AKKZv4m450Aqw5gPBptr3DxcBJbPocDBOg8GHhrOgeboNq7jA4EfUwTCdHXj/DQczs01EOewHW4nqaz61fNiIBmeHYowvApNKfDSlHgDSb1mPxF9qVNqMaRoPsSTa+jNMPM8s1qOMGAfOmDB0NMy1A6FIB7BRIB6yuAgMB4Zt+KmzAWXsDA23DP5D7FasxonW42efyGq8KbyxamZn9YKWAS3mtRt/D8ii14A8GiZubEcaSqeZJcZPTM1B5+Xl4bix6TDTc/reGDeZqRA13nfvbeDXeaiW2cU33n5bmAMhnmmgrcuZP09gNH0AAJ0mGiZPAXXAfMGg6MEO+S19ff3pZuBFPTf0c/kSh6gNefPzrHC8CZprYNeOahHSmNU3Obw+tTXmajq64VKMDQdeYdChGdK/p1zAbveZgriHqqmQzelaGuzh51UzID7J6bB7/kVwYW+z47FwAZ+cEMaM3MLUYPWDzrs4bwzwygetYD+uwxn8hqz9WhWPHYgmEj8JXsmg8z0RM0Emm+pNJuIGi6YZmttmGVOA+y/obFNClRY2yW5rH52lrXnqMzFaYqMTmZGeGsAHsUWSEWM+DESVoxzDrwzNNo2r5JnPw+6gGAdgKPGZ0XXT1DPM+hWp72jYiTmcO8/OiptBqTLpy32Nj/evFw3PDSHm0csOg3jwXUrzkv0fhfWJbGwzZ0QP61jeHUPRitMLZcTjz4eBRuyU8Ga6GLy/TdfkAbbcw3mnvVUV725roAcPaGDpmFwBpeQ+xOnaVuM/wDvABn8jK5HTDn/AKk2+HEM/oTTMZ+NIB+JFwMOg10j7N0YD/ZJJ/WrKBatAmOm2/EeAADMzB5etHN4cyuauvOf2go5xn3j8D2XnXVmbStw9h84Gp0AfeCkBZNODwwibPrTuHMWY05n3gPfP3pMxp0x/DTzXS3rSYZd04jAJHZ8HZzbAZ/xq+5DFBGnMtcpEpWsYe9PeAJcnWNj5wXjP/q4LahRNED+6awB0AptNhy556D5ny/Y10KNrc1BmqvSA0GHvzq1ZpVwSYoBLinpB0Aya3r0Nt5oGNPRwPPADTx0qIbWDEsIx95mr2OeBDlRj0/UxvAZ4HmnaxGfjSGojdHkgGGfnW75aDAMGG6iD2fWAKIDNvzH3X3zeAOg+YUDn4UqqvHg3TZIfWCkHbdzMnqP4PNB2AuhgdHN12LFlYZhgZu9Cr2WYrOfNRM3T9jy59abGMOzLgmNBLYpR8v5zNSGbelxiwDADNaaYzXHsG6GxPZ/c6Oa1ECBd1SaCnMWHVdX5IZgsb/uDI+qpcMziagA8HX8iihR52RuagPH5w7F1UOCfE2YX+g6Uz8kuZgau6b6LtceE365eLNNkH7qn+MAfnVDiJ0p9loznSzP5AZUfnKUzm3KgyZIHvwPzr03bfo32JRM3KlWKxUnTDDM3sFd/wAgPB0/EnWwc93zyHszXo8mM1KIEgHHKMGYdjp5qzeBy4QdcYpX3s+g4kY164pVh8PLew9TWdSo2HRhGV0Zth+wsAz9AYIPLVE4b3HXo7TYcPZLLvfIMwAFe03gJeL2DkudSqUf/WD1jD9C9Acs/wBeZn9Zpp6A+97tY9w4+96P3M+HOvxl5rv0YZgf+NaWHwisOlNabdDZn/8A1bxmC3Hq0199WvptbPQ7VtiAepEsuiMvecGVbAEo/DY2fIAApb0N9nw9M8/Io2EoPaMGsdiPyDjx+O48f5096tcD2bbydDXw9gead5yqs+zYNY7EdyMYe4e/QjOM+H9Fk/oR+takbRm4B5h2IgqVRMP2E/1q4EfD6/0IzCIft306ck2Q1HwPBV7120dmK7LnVmNGjsdeZgr2JGFH/wDiTIfWaNmBTnv6dG/8ZZqZxdsCG0b4OPTzDs5NZ+Tx7o8xo2KNw85k+94zwBlXsdN9TyjE/VQczh14HmnY1sXHJj56Bh9a85VWt1i5KlmxOCmx++PTzwBBzN1Q/wDQ13VKGfn1jNY7Hoj7N1ID8eX+hTgtiUY+3mPLzTzPFB5o5X20mScO/mcFcUe6uOdKp2vSq/Jej/vTzTY7x6tfge0gmf1gl61lhsbgh/4K4vG428dKa7qfzbJPySAT1N9Iri/Pf0JdnUp5r5wME2OynW5R+3fUQ6k3+OC5ufG+uMtNVGs2PR3omeDwR/bAtHA4r2VWHdP7JVhn68FcWpoPXDee/A/rUsK9Ew0+UP8AICz8m9rEZ3vwarGDz6IGo/8AKLw1Dw3LgqTJ+Q4BrRLVM1ilB7Bx5nP608dSYP8ApZrMhfPD3LTcviGefZganBcNovBmxd1Kw7PGXgsuZlPf0uHJ/dSAULk+c9pbFKkn+QE6zJt94dRuuUo/omAngBh49Ng2Xv7negrPVsGMfj2HSvyMgo8y0rHku80/w9o7zp9fgrQBDfPZoPB9YYKFVarblBDCq1xll78LrNY3DXCnk2rw5mRXaadgQAB/r+f+BR5PDHhkzAa56mvA7FDAAOeYbFcQzrlwmbFAoeEf+0VFnRBXUbh1Q+a5q4H3pkv8FnYyH+urwjDNWTwxsqNPCuOW4B6G+MEt7W/Pgtdds+qw6bp0rY6+YAbwdbIecFZPSYMCOb79RZDDsz3mqV6t1WY+b7DAG15zTBhlrettgGjnTtYHT9tIln1q7OpQQ8Omv6zvnDoSk0o6q7qTs5If2cz2KJVbns62GuVqs6NGkB/R2d5phFwepVNYgap6jz0iUebzxnvM1IqVSo1BaCXWaqzDa7APrP8AIuX1LjNXJg4W/SmaaHY66GbywklmXPf5uc+cl3zunma0W6BdXFQ6xCdpVuMSY0c+uQ9hmYfJ5FhD60gB/DBIAc94qhzWMOjBtLE0YIlszDh8UWHxSSVBIwQI9iHgG9H0HggSRuSQL6fQme9GB3oQ62CSWHxRqdBHh+ImQPDw80aB1IEkKkOo0yjVBIDRpB+8QJJJApeQe70s96Z3o0ejRAmkaCRsRZ5qPvySUBaLBl0B+hHyzAe4BID3aYJZ71GFlg2Hu00YMGfhtp1RHgcyAG3wDXPDepCeZDPU7FFMAWin2BccCE7LlToB6AZ4c4GeCG0rDue+YZzrciMzGg6/GwMPrUKwy/elh8V1P/0deIx//CgP/wCp/wBhZ68+D/E2yaMdfnUqNUo4HgYQZOZh+TBFtRw6rHC6BRo9NvHhXTaqAe2nBnrH9YZ712627G9Gy6tlHtmg8x/Z3fBP+M14ptvi6xDlNQbjpT0P5w7F06j16lXDDCq0OcDzXY8HWCDvFz8CqbAdNyh8LrVkh2MnmBmuX1ILcoMjlK/whpVNkeQwNXVt8cuIdsRwic0FVjh2S+v9a6xbfGywL8jhTa4xGpss/wCj1MMw/XguR08PM4VzlnH/APgyt4/+xUflrYP2HCiiH9GxdtuTgJbNVfOo0OpPUeQHQyz+zGuX16z71s8zcrFKOTHD+lxN4Gsn1eH5uRmgZobPifySUT88k0YPUoPEDhRb30d6J6fzJeA+Bu+QzwQn5HPa96dzv8ejwqd7XjXMFwU20o1NDDA4gBsPzq6ZqtKP9r4V0H68NiRxnA9opDMbb7QFw/4/Lp+bhSwqVHZwfY4ZW3h5zDNJ6tse44bWqH0MqOAecMw8me9Byxn7NwPzro7rc3x8KQdbY9p/JzbAH59HNI6rEkhv4c2w98/LAmjh7t7hh8hhmH60jjbMNP8AOG9O6z4+FIZmUYA2WBbB/XGwVFdUYLhajsUq36VRJDB55xAwM1ZAGBI0/O4xZ8fCCHVacFOjwbgsujyXWAwN3A96l8/Y/wD+L2mmf6wUfRYPZqBn86XLM+RY/m/2Qv4+H/VNZes4MH/5OaJ/GCnBVbVD/wDBtRD+sFSAy2CMDAOtvNdPdaPg9f8A0WT0yynj8ThrRw+gzBHGn2XGPUDhzD/8yapz0DPPTMEs2/OndbH4PX/0biBWOC0/Ni4+HLzOfvuZz/gUg+C3CG52tSw78mUp3+zyN4foXP8ANs0gNv3ad1sf5VwruvejZxNpo69GYh1WJ+M1JAD/AEGsDMti56C/oVyhz4fnN5k8P1rZ0q4bxoL+vQ649G+QzzBbN7jNX59vVClXNToc9p+MYHhsM19L1vZfHv8AhXNDzzatNYokJ2CEvWA3jeDZ0Zq2k6Zks5w99RzLypUG6tb1VKe0HjZPAwXouq+jlZ1Ydzse/OWdD3Mg9YP8a7Obm2474etxTvQH1rfVX0e+KNNzciMRqqAf2d4AP9Cw9SpVYoMjlLgpUmBI8kgMFEWxwjmgyNAEmIZYaiPMPduK9mCAzR6x4JGgRmdz+CWs5iCa7EkapGt7vPYgz+Caz+CPYjISWfwSz+CFGotiSaz+CPP4K8MjoaaA+tCBojNeBeH5N6X/ABh+GhyBFm3ipalgx7xvYhONBP8AogIuxLPAUgRzpVKP+ggpDJzoZakGpTGTDyPIT/dpAf4ioXUO9rqh/wDSWf1hmo9bn0q5/wDT9uQ5PzhsNVuQJZAgz9V4XWPPI3IMutwD84PZrnl4cE7qmO6lDuCNMw7DDRXZc/glkCDyrM4acZaVIwiQakyf4seSBgiee4029DD1rSpMxrzmzrGvVGfwSz+CMdvIQcYKwy6bFStyAfzgeC0EPi1Sg383Xof/ANI8YAvQtYs+zq8JuVW3IDx+flgWPmcAeGUnDCnTI3/05obZqlekJWIzoOU6/JOr2NTmdhroFH9J+/wj6DjlEmfkwWKe9GOzpL+pFu2qww8hxgVe96KjgbIvEmGbvYEiMYf50ebdljekzmf842VJAPPHeAwU1nidwPu2VylzW48Gv3vRsA/gXEqP6OXFSiTQl02s0GpRD62mZmBn/AtBJsC+KUQ+tbRns+fRDWBe9L3b0XTfR74O3JDCq0OlA9H7DiT1ze5/RsYpU3TiznozR/24Nn8CyNt1W57Pm83bnrKm+dk4x4L0hZPHi1bqAKVXOTgVA9nKSPfGubmhrFuHn6IVRkuhKfrkPlz960B4I5PocxJIA3OvjAP/AKNdlvO55dpPmdHtyqmB9bLWDzKeonFezq9S9e56NUqa73nIZPAFkbckpXor0qA1yjnEl56P3s8mrNn0XeHv9LuCpH9GALssNmgVXNyh1GM8AfPh/jUv7POd7GaG3GYfowcEobvNy4M+Y7++kqWfArgmyfgWr/8AfjXUDh05ksDbTXLU7LY2resIHCXhl0N2PG/Wok/gbY8l3XptDZgefAM810vwwH2aD1lKDoT8vNuVPcCqcz7CCB/9iCjhwfp0b29AjfoXW/WUrBMgb73t1rt64/J4dQc/2EE09w0ie7qMZn8i7GdNiH2IvVsEPcJtbjX2JbjH/pFk/oZBSGbJck9fjfkXY8KafXFh/oToeow/BA/kTY4/9hgAPEivf9io52rTe9ufs+RdoOfSoweJUWQD61Tz7ntJnrqUYz+tNjjh02hh7eLPUKSzb+fhtyQ+tdVmVi33mjbYiRjz+RY25KPBknrxKaAH5wXBzcPYOb144saKYMGBgueVuY28PhroFeo772egwYGuRV56pUqaTE5g2fIfYa5PHp2z2gz5OzUB9V4T38TNjf8AIqKfJf1/AwDz4Kq9cTmZAabmC+pHp2dzdRrhgmem+wbJqw5yDpa7eDwLBBVXz9u4pEapGBamovPit66HGmMGGpFc3qwZqTge0cWPo8xiZm25sMAzzBWGt+/3+Q1yX6dtmgk1Vs+s1TvVIAfPBxQQjSzLxJYKPUoZsta+uC5/jthtcMvMSWgfYcDNRzmSgPCUYYLOwzfN0NNz+NSJMl8w03zA10fNZtWRmXDaDU6FEmRohlgw4YfnWm0aUzH8eogAKJzNlm1pg3JM/Ovrs2amBBjB4ksANAFNpUloH+aMz+RaLn7VA/AtkJJ/ivGnmalTsPutNhxg+haQ1ZrOmxv2pt7S+RSJk+hzIfK0qjvG75zDepsmS/Pziaei158FEZPkCwiuZn51oyU4W9cEwvApUlW1N4e3jMdBtiIyz/empoVWos/0p79aR1ic8O99HuFrSuGNSZmg5Va5TWQ+tWb1sMQJrM5uuQHmgzzzBYwJPvHIus7581Cn1J+Tsf8AZeRYM1xWKPbE+acuo3wG/wB010Ao8apcMqC7gDz1VMPOCxk8GDkZsBgZqWFtzghesalE5Zr61eHrR3DclKrcXQg00IYfIGGapIDPLSAfc6FX9fhtqbDDmSCm+deIailMsMkFSB/o6ANCeh63aff1jdf68wRMw50aaEFjDAA86nQIEuvVmOww2AaHWsrXC6ZsylVKuNRJThhHPsBVVYti2KbNNhi2HpjX9oZe2LV1ujzgznMYZh0YGqmNMlZg3KnaJ/QodOFb9m7Y0hfg02TGdP8Ac5qRGokFl3xM3vrDBa2TPfjRTfffjPAHWqpk2KrH1wiYH3mttsahUvQGw9hBjZ+fBRwPlhN9jwZAdZgC1dHBsH9Nyh1KZh1gAYYJTAinKwiUeSyH75Y7beIV9NrE7S13D8U+tWASW6wYBUYkYwDsMNhqazbzGH3pt78ihSYdNhyNNiDP/Qm22FVPBufICD6jgQMD6wBOswGA8BswN0OvBaOAGcfTfinh2ZojeiQPA9TQAP5DWO0YUsC3oMyUbD/gs95qJW7eYjNfzVVY3L9iuHsPWwZ/s5+2VgEO2DPw6i8AeQI2f+dXpFwwkawGKk6DculHJM+t3PBGdgUAHTYYiyQAPwluzn2qz4DFRkmHkeZwRB6q9ow+8H0JteHPQskI0eQw3rQ2nwwN2QGwFZQOG7EBjmvWsaSYdfYZ/kWre8YwY62u9Hy0SG0bmYB8me9NmGfk0Ghs0mPOrFOqRnns5cOtVnPwXpuuxSp7IdjJsrocPQ0teX+RfOZYeLwHIwfwGhhjeWqszxGLcqWl59FOwKPSpLWpKpwPZ+dbh78Nxyef0Gq86DLORqQZzLIfgu9aGGdZs+jRpAOciD3/AFc+g1cHGi9lqwGQ+RWXqqo5ZvuQz+hSzo7gQgl82yefYCu7LhSRqPQ5joMHQAN0+xpHMsO3AA3H6Pgfk71ZsxqjAkBLYM2TDvA0VSebmShl6ni9b2feax2jDPnb1uQ2hfYo5yZAfjGaX2eoc/xwobOr2NGr6TDM3QfcNkGgDsSeo7cyG0/zxgb578FtteFd6nczNg7fhgHnAAR8nTY0cAiUdnNs+vBaOTAYZaabgxMHegzdk5mf1qvekvw9XQc7N+G9TsNW2znKz85rS3mEWTDj00zDMOwOtVlpB4oNqPecwwrMjQbPZgGYLrtwx/lVsaiUYHQ5qnfRn1qPPpVsAR8iEk3W+sEzPnvgANvsPGb/AEd6sGTpWloe2m4ZnsXI7DRvZtAwxEZDZvPDejmUGlGJvt1HM2+hnoSeOmw5XIyjN5p8AMJDWzBAHqY5QAfOco+fjGAb8Fn3NVmcNgH6ZOCWZtctvaDZmin+M7qNmbIdgApc+lS6rIiMWPQ6xMaAMDzjdBq4Z4LcSnpTOvB5ZrvOQ8CvaLhj2Qfz3qQ8DYB4Ab10CHwKrmqZ1K+IFN84BG5lXEPghaurr1K+J8w/+rwwAE2jDjmcpn5FLjTGwPN+Xn8gda9AM2Bw5Z6LVB7++3qzjUqhwGNCm2/SowB2aIK9tHm/kJ0+U0DFGqRm+eAByxrRQ7DvifKOJTrYM2vxpGxd+5yX+JG/gS5lwx8R8P1qNjikPgVdU+U0dYl0qlNB3gesZq7D0dbS1wclXO9gHZHZ6/4100Dwd1Mw/WjN5sy6w/Wr2MSHAfhsz7ue99byuKbw34bUrDCyqa9h3yAzNX3NB/ZzToTGw64pn+hAcB6mwBBinRIEMA6NFnBTfWUv+3GoR1KjGHsDBRzk0Z4NPxv0LIWHiPHm/OM/zosP+tfrNVWjbgd+H1oMLf8A/iOiHfh3oLjP4Jo3nFCwtVn9lrB5/Oakc/Rg/wCko361WwRyf+ZxNc+wHWaA6lByPw2TDz6wKL6yYM/2Uz+jeli1CtwT8PU3p0Kkx5M/zrPzJ7DLWp6tN4A7AZVLJuqhhgFSYkwM+jVDrUjcBJz9mg5l/sbzWHO9rHw/9pg+hreaqpPF2gU0gbgxZ8wz7HgwQdC9avgfh9aeCq197ofXJKlxprgQHX4NnQ2fI6ckDP8ARgsf/Kvf9ei5zpZ0RkO8DAAWI9BzJ9Zhta9RqsaM1/1gwVTJ4hW5D/0lc0CZ8kQMzXnqSESpNc1z0mYYe+OSZ5qRMn1WfCCC4/AyD5OhZDrFY400aBm5Q7fnzMPxvBA1RHxjuqq4OMBR4wH0RO9ZSMGcOO2ex0AwM2dmaTLLHJmw4AauebLxn4wGtIEiq1usT9Vus1mSAP7NJp7Z+dUnLMB+3MGYH5zwNWDzLD2DBtmGZ4ZmCOHUuWadfnNhMD2AH5wWgiPHEMQfc1vA3+MCMHjOFvihgHf3pPSdYADA8De2B1gky9TWYs1uW/rOgGGwNgLy2ownv4abDYAAIgqT4Ab7+D2zAA7ARsw2DhuuPmYH5AQRgpwZ67+H5NijCih1V+HsfiPb+/sVgdeuOm+G3F9Wxz+TrBTmalaJg7BfixpLuzDM1Ko8yo1Imosu34z0Js89bMAD86YThnzqX9qceA3OgzDBWbM+sPQwN+3Hp8fAzjPNecPOtHXq3UabN07q5MIT5+Cy0eseCrGeIXqSG0dDgmDUX2OqeG/zmC9RpTzHvF15VnSQz/tGYYfoUgHrSe9oFSjB2HHA9/8AArsOP1Z9o5Q6a86HQYPZmtnR+JHF654+FN4ZRgaP30gMA/wIdzkMnkY0g226i8YfOahSalBZ/pTP5zXoL7DXBUvv16xLSB3s0oesYK+Zolh02OAfZWjmYdDr0MDNWjbzzR7YqVw5eqrVnzzDvANhrZ030dX5+rLu5xmleRpp4Hj/ADrrEy4dmhTcGWuwGgwBU5zKw8enrnpfWj1Ct7hFwnsxoZbcE58hjvkPGYH+RaILnYD/AEVTTAD7wZwVeFK1g8fP9anB6tprQMOPmHyBvM0DPM1Ez8TYh5NhmUFcfpUOZIY7zBAdSlzPApsHRz6JEsMP4EH2Mpsl85dRfmSXT7AkmAfoXlq2ek3yxMadcClSdnWqx6pV+YQcpnDa8hhvV6dEpVEa5qc5GpsQO+Q9gCwte42WrR3QYtmD67kb/FD9mD8/epGijU1wzz5FnVPrePeap7h4l2zaTpwX5XP1AP6PHD/P2LjtyXnfF2zwcrE4AhB0QY4aLIAq8IcQPdr1jtqK9xUu25GDitmzSo59kQN/61mQZ/P859aMAYD3akMhmurwvZaJ+Q0Wi5/WpPYlh8USj4Oe8RbE8aZVhd6SRpGjPRJdaXWl1ovBAl1pGkjAkOfwRZ/BDh8UCS7EuxL/AAI1IEBn4pto0HQeaoGzpmSLPBR8G+/P5ME8Bhh3n9aB3P4J3P4KPh8UYKQ6CXmQ5/BLP4KgSWH7xCkgLsSSSUsi1sBSz25pYefA0kaln8E8B7EGAZ9CWDfu0BZ/WvufwSAGwS2IA70ZmCdjBRuVM5VReZkdjIM9f51HDMwzNFjzcBMvZn4bjAH+RPIlArTjYezifwJ2AdYjEc6lSnobrHvo72CkIDBvPUbzWQ6nY3pDzobTVKvhjmfPUwwB7867rSqxTq3ACo0qpMzI597T2a8ZPMqRTarWKC7zdDqsmBI87JoqHpPidwf4c8Wo+dcY5aphnozomwwP5/OvLNw+jrxm4UOuzrYlc5CwM9WOGzYHvg7F3az+PbE90It3RGY0jsqLQbM/n8i6qzcmbAPhOAwPoeDeChbw5b3Hhh77pcdNeZkecF06BMpVej82xvBdI4hcCuGt81KPcf2fh+s2z8YGj0Qkrg/E7hXdXD131latiVKNC7JcF7WD9CzuEQ6rbHEK8bJkadKqpvQv7PL8YP413C0vSEs6vHylcinR5vf3xl4MoPHv3FzU3/vDX+ouoUe5LfrAByNVjHn0b1GF7eu7n4RWPdQhLgxWYzv9rg9BrkVycH7qt5o51NwqsIPfMvb/ANCxlvXtc9nygcodVeBrvjnvBdrs/jrblYBmLdTB02afvgDwTWOHfw+5zOKBMYDw5WbLvke2GpYeMGbZn+Q16Lr1n2jf8LXltxpgP+xltHga5pWOAlwUpoPsrVQqXnjnsMFzYfY9b3P/ACwjLLhtGYPsn8nemsHOgHz1fI8k88+y6EWuUo4chjZmAYGCLPWa19fnAzw39al3xzeLIznM7+g/1ghCS+e/YfzgiB4GXQ5R82XfI70JHJcA9PYyf7roUKGy829m224yZfOGBprB8Pbmyz5M94JZtyWvvbjMkD6POCdjHgenzwM5/jdCqwGD5+JgBgHe0eYJaxh7NsD+g1Oegbc+j99H6FEeZlaWgD7Mn6+sFL0ge3+IwH0O7E8b2Dofdf8AOogG+Hh6gMn+9DYgPXAu8D87XQreJps556bGf0Go4bwMAcjAf708DQnnJAHNQHj+fYaIPB3yzAP74MwNEo+8O/8AJgpeD7wg4wx+g1EM9+zZ/ddCMNR49T7tn5D2GCtCb45jqN4fr3ow08T1A6wwwMFCzfB3x3MGvozUhk8DzYPMPIyexecY4pSj0aoHkCZ/nXcJLOB80w5g6fvmjwXGp7LkOvTW3Ox7P+NddjScIsfo9iC7+X/h8r1+GPN3Fr2j8Qr4omDcSuPGAdDUjeC2EDjr0MXdbMaS1+KAB/nXMs96P+8XN+W3xQ6K9b3o9X5hr/zDUD6NLwVT1X0XXJPj2dfnMteScGZn+cFksMx2JQ5lVo8jm6VVXozv/V3k8ey4/P8ADYV9w8IuJNseI/bD0wPPEMHv4OtYzmdE8Jzb0Y/wnQwNdzo/GO9Kbg3OlszwDseDA/1q9e4u2PcLWhdtl5+cwZB4P1rv8ey4P5bbzvn8EfWu9Hwo4H3h4lv1X1bIPsZeww/Is1VfRjuqM1r25c8apfI6GH+uto5nNzcGHJ8PijV1W7A4hWx49ctKYzH/ABQwMP4Fn+a+CvbmwSPP4IANLBs1b0eeeDfegS7dRCjIkSFJUCM96XR7xJCa921Fn8EgNCkoDuQJZ/BR8PilvVMkjIEf/EH4ig70sjUiw2IVB1nP6k7rOf1I9hJSUfWc/OiCSjQ8Ye800kuc2abnQgzb/IqZEaWGaPZig1kQDRc+g/kUtmq1mN4cSsT2Q8gPKPn8Es/gpabWoXhczI6Zzgkh+9ZA009cLEw86raVNmefZgq/I0exDbYUHiRTaO1oN0OpQ2vIEwzD/AtAzxXth7Yb5h/9QyuX4glogC8Nusczw8rHj50Q3vID2iaso1Bp2OdGnSWfkamZ4LhXLRPwAR8sAeIw4bJ/IanD3bvoRqkz4frwz/vmUH8/+7OjyfrBcSZqtfjewuOeH/bKaF83xG6Krrf3oZphs7BzNcD9qtWMf/08/NPetYge3ocxn8ma5ZD4qXMHgTqVGksn2Z4KzDioAe3gzI30GouEN09WKGftJWj9YGkzJpUk/ulRhn/2yw9Sv9ufHBuDdpw//qI2aw9z0q8bn8SDeNqvGffgbJpgd7OA4HtH2f1qiqtz2zTXTYqVcjMn5M155e4V8af6JKjT4/kj1IN6zsmBdtBkGxXLfn7O/DNWt3CZdvD3I/58mfkBMhP4ZVJoD+3DzOp2GBrhQXDBA/HbeZ+sFaw6lSp4ftbJ/nRDuDNh2dP9hd0Z7/tsEMnh1BgO+BSjqTXYbMkDP9C4vycE+jf9CdZOdG/ZJ09n6DQdTk81Ad02LErGzyAs5P4kP02QbD9j1gPnMFSQ7zvSm/stzTPznmrBnjHxGZ66rGkh++jZqFqKq8e/VruH2ZM/yLm988VIN1NG2/bARvnNdz/laYmD/wAo7At6pf8AdsFUyalwPqrRhWODPi+ePJTCHjqY83JdNxtQvVsvPU09i9a/Yn0WqwbuvSq3RHfrzQPcDfR+qTGFN4jVKH/fLs2zw8pBDcPsVrDo7/vG16D/APRjtl4/+TnF6myT8j2AJSfRg4jQ2jfpVVo9Va/dPAri3jiUOiS4z4P6fhB1qb/dreyeDPEaHI0H7Vew84Hmq+TZ8qjyCbrFGkxg8+CxtqyXMywNVlVmOSfbmtNWDo7MIuVbPV+dYypSYhu+HKBREMgRp/LHm43mHejk1uI8ODcF7Dz5qEYZ+8D9aAIbfvHwBbYgVjJtm6GmrAGX3vZtq7eskIEo2zqutLDyAqw41ShzdA9id0CFy07VwPYnQeOMYeBmpExl+MYYax+fMFff6KgBL5EzIw8ii+aBntapPPhnBMA+fYnpMl/mM+U/QpHrWdvN+lGAefBRI1YcedwOK9+Rk1jFtQ6054v2TBRzjb9Q1oobMswNw6U8eYd4Kv8AUlZeP/R0kPrBXtgr4x7/AAE89QZ0xoG23AZz860tKtWVDhG/OY61I5Y/cRXnvoZNNjIw7SgxnddyVzJh3hsWu9VS6rb7rEGjc/h17PYgpAQ5YU2Q4xb883XwwwBk1d2NJuaj0PkZVDZ1TMz9isdupy+j0FyZNBuJBeP69ilM2ZUodwhm31rqcC1aqdR1323o2fkWgC0qdqg/OA3nQ7zNRsckrdq1mjkFSnMYNJqj0qcdUjhh7c8F02sW9RpgmEt+Y8B9gKLQbVbjCHKNnsV7Z+IOyaDWGTOLB5mSfk6wWXvaA5SqIc42NGWBgBrZgdckv6DFcCN9YIJ9quVhrCv/AH/5zNRt2eIVlnxoNYsWI5OiG9nnngqyq1sKP/N0GCbLQedXZ0SKz4Ddfepv/V4hqvmWlBnhhr1KT85nmai2NwsKPckusFyrFRZAMN+atmZkXS0H5wPYeRUP2DocNjUY5zV8hngpECguU3SnS4mDR/Pmrh6R1tiM+DERt7VM+t0EJnKezN+qg8Zn2LSzGWHmGtSDAwMNhyDUiGEGNACIx6tB3v0u9WMvGjMGWEtzWa8id9W0p6omEGmn+TetmzY1vvZ69Z8UGdYwj9ayUaNWTaBuDFk6P4zp7FItYzJw2ACXBzDvwZzNSAkwWfEiwcD+dnBUjJ3HJfBhis4efPYAKPDuTk6sEGVOZedDrdNlaiWbMQAdA4vf1mCiGDH4a0FY5GfDdY1wzPowVFGh8mAa+ZtB3n3qBZszKdMM+Rg8sYB1moMmGEkTfnMAZ+cNhqweoJstHOfimYGGzRNIzYgAGuw9h51KlZDpvhaegEN1g+/vV8YNmcc3Ahhv7EYQ7KN3UuCqmzszBkA70yEN94uai/s6JWdKrDlNnuuORGXo+GGsarJJ2q8+bhuSc/I10KOFSiA6bbh+E31qv5mLJm/dAw1O9BOOTFAAbiQZOw+t40mZj8l3lGA5MPxjNJn1Pyum/UZJmqx6qxYZhhrSfoRTSvW9t1DuNkM/OooW2w80YTqqB+Q4+9KlB64DUYtiqm75AjGa0QWBflYGOFv0B6Bn7Y5zPLAH50SpWYFKprWEqsVXl+/AEnjocjYxFmHE7MNi3VH9Huvn9+ua8YYYdjJmeC0EbhFYgNAxUq/MmH3+Nhmg4fJhsBN0GHJLMT8U3kqlDpUBoDYuA5LpnhgC9HBY3Dz3duU17++DMzWR9IH1VDgW/RqdBjRg1tZ4GmcPItfCGCtsMJANg3vPYAec1eyeC3E3nXZehDDWPPA3t4KLasDn6vT2A2ZyQXfXqVOMf9ORl0ew4o/e4p/IhXA/0zc0aMfka3qwo/Ba2I2b86uT5MjsejgAYLqB0ecf9OjPfnRhSpbPiYAud2spRLGsq3vEptH5l0O+dvV9nh7CnQGfojKw9Wyj92l6qlf2dZCJzMswwN/APIGxR9H8Q3jUs4ErDxATOjOwMG2HvyKhH5bf4jH61IZZDyJrmdEvHbez+hNHJ3ezeD8ikWB9OGokAMKp9asAfRmf0JetQMu8A8gILgNM/eIw8E/aLPhVcC8QN/kSOeZ722zD5DQXBvJrmcFRc++oj1bNk8DA1S2l5/6/zoDmPmO9Zf1xOM9PQP8AIkAV+Y7psMHh85qUNAc8w624361FequB4eD+RFDsy6nvH1Aw7800dEtylFzd3XNDjZ9msCJA9PlvdG9RwN97+gmf0Jp7ipwgokwG6GZ1Wb2MtM4H/GolY463BDaBilWyzSs+g529GrRxrPrkx0A9Wshn3m8pr3D2mw/9P3PGhh3hnguO1K/OOdySjg0O5qbGz6wggAGyHnWcrdVqsms+pq4/RJLrAffHXntbmTQdtevbgZbZhBcqPrKR5Ixm9mlJ4625Gd5W3LOPDsCR4J/4FydkKdGznRbZZN1jf93DDZ51HZqUufPZfcfAJYdeB54INxXuK/EqvAYRIjLMf8FkMFh5LNYuGf8A8p33sA3s578DU42X4c1qcxKPVzww6zUeS9VY1SOW/Lk03DoekQ8AzQ8K+m0EGZhv+spjIdjQbP8A7NWByXHmo7D7cZ6WGDHsetfGawxJkA43K5nDrDDvX2sSalqyGGOTPMMMwDeCxbnZ8OJGgHK8ikcgxUoYGAaxmGZmqwzbjVnUfbOTT4oZgB95mpcOsVgzdYgxDjNSgwM+zBBHCB6taN/Qw0DMNLvP8iaCqscxpt0Z4M/IyrV6B7xx8Ad7zNaWm0Sqzwaqrj7LMdgMOYM94Im4YpmZ96N/lJOwM8MFIk1uD+JUlcBSn5/MTvVz0l3PAA7HvrVm9GOG7HluQWaaYBgBywMwNEYZeBAqNeqkKm2+3Jk81h4zuwAVrVbVgw647bkSuVKZUGAzeaCGABs6961dNptwVWbrsXHTdUACUAROs/IoMzh1WHnTl5zI00HtcDPrZQZKsUqdSp/qqJbk+TN0df8AaQA8PoSjWlesloHztWYyHkdMA/zrQM01xl+RLqssOY3saxyd7IfIrC1QbN9qKGtPz3nLMzAMPzotgjN+N7dgw7N6lWrdVqvT5tGrLEk5odER5nPPD5+xdTue7bKprpvzqV6y5L6Ovv61WW3xL9cf+wHBZ7m++Xh0fnVrWHDqj0p7Nw7OjQ3QDMHnQ1s/zrTXJTaPDhyIsuIz8hgGAACKNbHFie1/Pl1M23HDraghm9+QwVtR+HXDmlYPy+cqsv8AtFQeM8/ydC5xx8LYr9wvtMUChnWzAMOYdk4AB9mZ/rWypXAesz3eevu5gOR0cvBDs8hmujSbtpsZ04kE2TAOjlwVZ67rMkvD2LVjdjt6zOHljRw9VWzDZkB/SJDOZ/xp2ffnimxF+84eRQqwb+kL8t8zDPA81EejU2Gwc6U4DMcOt508ABEgen1me6bmDIZ+RCcCU8f3tVMniRZ1KqMSlMMSarrhnrU8NYAVmzcNx1XZR7cON5HZfX+hWJoQGIzWu+YAHnM8FCO6qOzK5SC29Md7NENn600dvSqk6D9cfenmHY6ewFas0d9kNNiKDIKBTmdz1VrTlzuQjn2R+v8AWpDzNOtiAE5iDJmOgeHXme9V9ev+zrVEAfqPOSP7PEDM1yy6uJdwXa0bFN5miR/OyeDyqB064eNPDyg3a1ar9cZORhm8Ye5PyKhuHjw+DpsWjRntnROkbP4FxqNRIMbN9iJ94PecgzzMz+dTQ11so9VZlcuR/nrjrEme7nngfR+hFGZYANPAAEEgTwMpEJLYlhmpGjh4ZtosG8fEVs9o4aCl9waYYAkAAlsRJbEaZSVrI0kkkQSSSXegQJdiSSoJLsSSUvNhRIUgBHpJJJIAQI0GGaoNG8Ae0ToGo5hvTzJtgfnQSgP8RL/g6vD6EPf8iQHgKkOowMPzpnP4JZ/BFnj2B4iLu8NNZ79PsR54IgXel3prNzJGe8kBn0L4yhA/dpZ70DvdppIEug96BdB5o01nvP60WQID60H7tLNv3aWfwR5A0aZR7EaEeoj7MA60AdCMFADeeYKPyx5qQfVqIzRBmfSuQ0vvTL2uGfhH0KztK8KzZjupRn84/vokjeyYfQqwwQ6KLd6snijat2m1Fbc9T1U+uDLPAM/kNdLZDDw3OvvXjc4bB+IYLcWfxsue2Ggg1xj19TO9l32wfQaxuHkOl8SPRs4ZcRQdneqgo9Qf65ERkMM/oXk+8OCHE3gnVwrEujnVaUxn97pwZh+js617dtK+bcvync9atVN50OuDI2PAr1mThm24wDwH1gYbFG2jwFbHG+LJPla5FeCQHeALqdHuGjVumg/BnMvA55F1K/8A0XeGV+Ouy4NGZpVTf36zR4Ab/wDkXl+6uFHFTgtP1HKVzgGeAS4J6wGHnSx26m35X+HrXrWlVl5mIHWye8F2rh76Q9o3Ow19oGOQkH/SOw14qjcRdaiS6VetO5CQeHin0B9a1tgXDQJ9uR9Cqg9IAzDo61zXCtvdcyHZV80vQfiQKlH/ABgDeH51ze5OALZunOtGdhn/AEd7/XXEaPXqxRJoTrfqRxnfJ2H9a6DRPSlrFHf5G9bSCTH75cTZh+tRhv49zyzkyBX7Yd0LjpWiB/jKEzJgvO6gPsh9a9IUq9rHvmjRJ0GdDejzc8GZYb1lLk4IUapcxLtipeqnexrrZ/jUXwvr8H8S3+9yQGaVjqOMAEjzgmtFzPZovK9qXDe/6UWhOpRz/njnmCpZMA6b4cuJPgSPoWNw+lw88chn9m8Nidyx+TsRmzrHryooPfvmTTRvSut8Aea8/egB6LpE4xVeWPP2JmpXtNjPTgDUiTmcP7PINPAB+0luPRvIbXQqwJNK/wCkohvD5wNNHPYNrCjVxkwY62njQXucvobbZnh5wPeoRyZwGeh92+SRvwUH13bjIn6xqPLO+cHlFmXtR/Zt1gKkH0b0WtWQ6n5bH1ux0GyTsbYCYfkNUocRbYDZ98h/3zOaX2/t/P8AtP8AdBgrGiwON4em9Ga8hnsQhAce3gxGeDzgs/8Ayi24GzkakH5M0QXhBP2FuVJ76AQZe7Wf+UzrAeQP1roEM3OQiOcoYZshsDeua3OdYqteOXBpzzOAbwPvXQrYmN1KkA+/Fk0p0PBNrPYZ+davmx/lpLwfeA28GT+g96MGW2faOPfnRfdZPW4y8YfJgaWGHhtvvAP71Q7PCQz0eA2f1glm4ftHI2Hz7DUUGWD8TUDPzsnvS8M/b1Iz/wDqAUruEjw/ePvAHeAbwSwY92wD2fz4Go+i31g2eB97JovEzP72H0GGBqkfg88y4Hh8ro/Of+urWm3hd1EDClVyTpfhZ5qtDMGut4A+Q8wQBh7QMDD517+WN8Hh0Kg8eK5TcHK5TTew97orRhdXBa+f9OUaAyZ98tnf+tccM3/ZuZqPycV727kZ78617nNzfw7hdTrHox2PWGgfty4J9Kz7DeB4Fz+t+jlf9BHOmy4dba84HomoTJzoG+j1WZG8njZrYUfjBfFKwCU5GqoB+N1rp7nyv5bbj9Vo9cohmxXKHUoHzyIxgCqQmRXvYPga9UQON9HmZwbntx5lo/k1gUc7S9H6/wB024jcBmR38u9yZ/oW0czj5vWw8y9YIAXbax6LUuN/7K3iBtf2eWH+fNYW5+D/ABKtKOc6o24cmOHW7BPWW23Nhj0SaB4DMwczB0OsD6wRnpq3pZ/BJAkgPekln8ECBJJGgQI0j6vDSBHn8FeGQEks/gkmAOe7BFngOmlh8UOHxXgd1t/iIM/glh3mkgPP4Jayaz+CWfwRB3WR6yj5/BJBK1l91m/6lGz2JdAI9wkZAkBpoEg2IbSEG9NZGntY/OpXt8z+CFJJXggsAPwzbzBMhDifgAns80Wbf5148CAaPs33g+g1KZrdwRvDiVV7DyHvUVJBNk16oyRwl06iSfnegAqrlqGf7XZ1HM/O0GCkPJnD4qWm0WZQbcP2FGmQA/dPZqCdsUr+i3NJZ/vWVcJYfFMPVKdh1w/2G46O9/22CB6w7/Za1G6bGmB+5eBXGgx+GCWiHu8w/OgyUmlXVDz5q2Z+zrwBRAqWj7diTG+tlbs3ql/8cmAH1p9mq1IA0zlhJD96zmqZOdHUqVJ6z3/OCH7i97NtldFkhb8nxJ1nQHj84bFWSbesqSWp9nJMY/3LykYp6BTj9hmH51HCHLjFqQarJz/vlsPsTbj3s6rUo3yGear5Nkyw/Yaqy9+TBXAqmbhvyN7C45n61IZvO+2fb1jW+sFI+x91M9DASfoeBVk+lXPG9vQJn5AzUbapH2hp0n/TNiU2f58AwRhM4HyT/nnhfJZDztGqF6To+3izGfrZSZk0aT7d9Itk0x2Z6MtYE9OJUqbn9aZZ4CcAal7DiFMZ/vTwUJmNSg/+zUjRpX4f8ajYqoFHtWmx9NiXDz/FPrTrMOlauoHLGHnwWUnnEDNvlQBKBG5n2ZmAfIscOhtsKH7/AJb9CMAoeeoGif5FlpMadDa1+VMwDsVtRKbKksE/ObCMHk6FLVcPHb73iOQYxn9CjSZ9GhhmdNjYfIyqo41GjZ+tayf9yCr+ciVITCI28DIH3ggvguSgGO+Cz+RlAdeoHZTs/wAiz5wDAAbYYPP50bIcsf3uIH6FrtK6O7bfZ9vTUj4iwfZsU7BR2Qo2kD78FkA70bMO2Am81EpxvfWslXB1m/8AW9hTTP6AUj7YTnvYUY/zgheuqW9gxBbBlryAygOgy6k0dSbfD6EEj1rcb3/QbwfkQTKlOgYN1LBkzDYHWmmabUjDx6rJz8maqp8CdznsOZdDozNBYBXpR+GDDJ/PgoLz1VeM9Bt4DPyKwCG/yrWm2GqfYCGNDlxs3KlBMD+dEqx6mv4aj87CR9aj0qG/UpHKVKsyQaWjjU1usCbnNxo398agnSjCYcSDXI3zoqLL7N0dkfuNVeN75wRwLVrkl/X+1T0MA7M+tNBAlBIB9ipSc/rToQLge8N+cZs+QDQv9abMZAHdCozgeD5FCpUNhl3XffPS8570vVXJ/tUQwz781o5lsPw7aCVzZsh2ACqBTnAY/aua1mvo6FG+6syNNhv9AKL/ADyy0DEtg3gM8OhTQeqsB0HMAwDsNWjDQUo3Apct9h8GZD+zP5FWhsa0AgvavzmgmT84Gu44yDue8AQQ7kiB7MzhmYYeKeaybGghyow68qIGfkTsOpS9L/jYpsYDDzs5ocH5kg24L5zHT/CBWEa0r/nnoUq35J595hgvIRcIP3t779K6FIjMvveO+xnCDvWpo/BbijMwi1Xk4dPD8Z7eH5FqofAqhxvDrN8PH+6ZAABXtGHKjeYkhvrLIafkNV4T2GT+9vg99B5rvsbh7woprQN+qnpPzvPGrCHGsCleJRrVjA75z3q1vOUlmq3DKa9VUafJaDvCMa3tN4LXpPj68Wjz4398eALroXPVcNOKxGjNeRkEJzKrM9vUnv1qUuaRuANZe/05UqVADvMDzNaCHwE4bM6PrW6pMwGPctYABrV6Jn7RLk4newgiM2rwrgHp02x2Xng7zVnGqVNhhp062YEP6IwJnlmPdxM/zp0NSMGnoMgaIOHcNfP3+H0bEBz6qft50n9ad5lvAG0YaCCvCNmGbjhn9ZpaIYezBTczzNI5OAKBEBlwD8Ncl42VV+sXpCpxhg1To2H6967FD+8yADNcFvOZ644g1B9s8w1thro4BquHsYAr1PczDwMz3rrcmYbxamouaWBDY9aSKk/0RWfB+s1uDk7Vfs/vcsJwSXA9oaA3nD8TXNVhzzDsBNetezTAPoXO6lrzj4ewfMEgqVS/tzwKiOpH7txRDqr/AOIqQ1fricf9K/Wg+09RZHw6jGw8hgsqzJcklg2e/wCdA892a4GZ+Teg2H2wne0DljS+3Mo+tuGsfydVe/ZYhvfQkFBrD3t2MP8AGi2oO9mw64LJ/QgO9qUz1wWT+hU7NgNmWpKfkh/fSVW1WfYlt+HLncy6HZHweNQNUF4UqT12w8f5EbL1OqruAW48yubs8Zn4brrdKs56Sz7k3XsMPyKPP4zcQ5kDwOTpQ+QGcz/Wi3UJlmOTPYMGz9ZrKVsLDtVoHKreLL0s+tpo9YwXJ6rW7tuqKcU6rUnj73ecPAAVfb1t1Wm1R31bSoEyOYYHIndiDpEnjZaVKa0KVb8mpSG+gw6E0HEu+JmEptiNSo7gZ7AzMPzqPGjUOj03kWLVgPS8zM5HYH0fIs+FNlg+b/riSy6fQAHsBY9zzpS63cl1SZv366qqbXkZewBZR63mDlc8xOk6vYDvjAtBJPA2nzlnMA/6Qfer2mz50aLLpUGjQHmqiGDxmzv/ACJ3NmF9VPxpgsU6XrSDDM3sFKmPNzH451Wc9JOKGwD7DVhGh6MzXieC1mAYKScwAMwYADw84K4tGFZz9OmMacp8I0h/22AdYKwk021QAdCnc+657nR3pQ5jjLuDDmi6iOTozOebAzdDo0QzNNtUfmW3nfH2A+GwA2YKWAUqA01BBjxWDzPBRPWudZ16xTalGZDrkOs9Cu+WoYNG/wAjPZzDMHnj3mpTasOpSgmhOYPAA6DU6fcj9YY05b+s75/nTMl6CdGdiGAG73u4bwNVrMmlM+0pTL2fnVbQsgo7eq7L02QaAN7wH1pQIEHS1GAPAzwN14+tURswYxG4xBejAYbMDzwVnG1zihFcnGBn0awYJte0jNyN4bBs/QYZp+qyXAYhN+spj0gGfGw7AUA4b4TWsPvJ94NbzUiHDYZCQx7E3zzMJGwwXrI6884zDaCCBzHT3m71mAJMxuZPTzPP989gAGmnp8SM+bFOggyGeBvAfWrCNR5U+m5sN5s555m8rVtdQJnLCcEH2ZhsZvhhs3q2gPXVW4DR8ozJ51nxg7A+hV9t0EHqC63WWHs2DwB7PAzWioNw02j6rFGYeqTrGGEePvM1itXwIFxxs6VR6MEbPe9IZPrNW1EjXBAkG5UXzk7MAze6FXwOC3Ea4ZrtSnPhRAfPPCQfjYfQt7R+D9pU1o26jKmVWQfW6b2AfoRDnUmpXBMrIQbfpQSXT74jIPLRUrh7f9ewC9Z3JtB0BEMOv8h7104PVtKj6EQGYzQdjQYAql65/wCyOb0Eej8LrHojTvPU71rIfPM3ajv/AIFcSbkptK+4xAjA0HZHBZeTJqs8j15xhn5EmYGHtN6C2k3POmHhEi7D86r3ocqeAc2ZmrCHAzdDTbQVKq0alSggynzOQfuWQzw+tBHjU0AwDTVnosQA15T4MteczVIzcM6ebrEGmnDaw2PO7zNR/VTD0jmqq+9Pd87xoiEusVVusQDi0CJJkmfRIwwAFn63Y1ZuodC6r1Nlr+yRGQwWl5aLh4bAKLPnwaO1zdYnRobXYbx4LzC8+BUeg0q3oQ06lOAyH7lkFYGywy0b86q1IGg6zN4ABcvrHHinM+BbNO5n/rb2wFzqvXBXLqlc3XKib37oNjP6FcGXWrh4zWdR3TYoAPViQHfzJ6K5bW78u65z05dV5aP+DEMwBVLLIB2I8G/IK2wI4MsRsPDTubf4gfrUjD4oQhhnqONgmGQAZcUgIzikACdA8A2dasNAyYJ0AQp1EADq1EaD92j7tNABpI0AIF26iX/ad6FJWsSSSFEEkln+Gln8FQXbqJAgNJAQHv8AEX3w/dpkzSz2IDSz+CDvSw+KMjyBL+8QKWo/Dz+dIN6a7tRLPBAvE7DQb8kYJH1eGgLMzw1E92KOAYHnr5/InTB8CMOgEEhBngkGx3fgeCWDh+JgoVBGYAfWHWj70GDfvMPzpGfR4eatI/3neln8EBn4RpZ/BAeHQfnSz+CSWHxQHhnsQd2mkgw+KBdaPfikl+dUEGoHWjDMAQeIZJKQaWRoEkWPxD6EWe3501n8EefwRAsG8DcSMN/huZoQ3gaXQCAuxPf3aZ/OkfV4aLHsxUcwDPo/WpHsfaNoM2zzD5FGBCznQ5AS6dOkw5AdD0c8DBdVtL0iqxA0qbfEV6qx/wC1xwAJQH8/nXLcPilorG4VxvXVsXbb9zxQqVuVWNPj5+6PeH1griSEGpNHEqUQHo59YGvItpW85MmnOYvFmiS6dhgYHv3/AP2C7lb18y4cUGK5c8CvGx3xMAP865W1slxI9D+2L25h+0qr6nmv9bMgzNk15iuHhjxJ4RV52W/TZkOOHQ8YazK/QKlXJSqxHGdSn82ug/OBpTK9Q5LRwX9GT52TDMEY28D2xxgfDxLgls7PegGC6RDuSDdVLB+C/rAa3t/+jZYFzx3ZdsNvUqpn0aJ5gf5F51vDg5xesY3Tp1OmBHDodj4YLo8PXRY2vRJn82y3o2e89I8AM1oIF21mG6D8WszPoN48FwoOKl8Ur7pd1vvZh1vHGwNaClcUbZmEDebzOfnBVEIdtD0pTtvwLuocn9zIgyTwMFs7e408OeIUcGG7n5N5/oCdnmvOMyZRqqQMPvs7/nVVRztGsGcXPB0DwDZvXmDxyX4euHuCdgXCXrKLfgRpZ/hPbFCk+jG5JaNulXxDeNz8ZkDNecoEa6rea56y71MNf20R7xgeVlGuHipMfkSp11BAkYZgcQzAD/Jmsul1ePc5nY6l6N9901rUbrkOT8gBgqeZwWuOGxzVStUHvO7HeWKpvHji/DaBisVgJkcO82QzTsz0k74Zim/RqxoyA6AMMwNY9Lq/mnMu/secN0H4Nua2HWDuxRzCjBIBiq2+zAkfIearKJ6Y3E2g7KxSqbWIR9YOhh/gW9onpgcK63/p/hzyYefRA1HS2/ms/wDezrIN7+UiRpkc+vWZzTLx2qyWDlGOBI84M7F1uHxa9GmsRwcfggGf/VlZhVfRbmB4lRgB9ZmCYdP814XKWaxVTHUbYjVJn61Fen016RqS33qa75APYuoBR/Rh3nFvRmNn5JJqpnz/AEb4D5sP3/DNrv1t6YP5rwsaBn/Tn2Z8RMnUqOzsgxDjB5DWugWxwEuSRr25xRZjNH2czgAfrWzD0Zuf3sX3zLXnMAM1eG3j3uFyRmZLk568GM80HQ8yeBqQy97tisPRg/Bee2LqX/or1UA+43iH/g/6ia/9GC5vd3HSnvrZNRh0/fwuZfcQPUlxAeDsej9aJl5s/YOYf/VrpB+jTd0b9kuCAH0ZqO9wEvtnrlUp76zNMH38LFAEsPEfYZD52ehJ57mfaS8/rNbBngVfEbMwnRmT87MlJng5xCD2nqqT9byYPv4WKAAD2YH+RSAkv9Bts/8AbLZfyM34Y/deQjH5wk5poOAnEky/0rTVjg+rhYzAw8Q+hL7oZ+I2D31rdh6PF8H7euQ2foM8P8CmxvR4uDol1+A8HeGiZrbDH6uFzrNsA8DMPrPNLmd3iOAf0Bguq/8Ao3uBvC5zD8hqczwBpQft1ZkyT8+GCvCPu4XH+ZY94+9+tMnMifgAfzhsNegKbwZtKmmDnInJMPxjzV39hrcD/oaAH96oz5cd+3wuVcOrhu553TcfeOOHZIBdahz5XnTWFuQ+uXGDDyGo71etGN/0qzn9a2i3xPZ/Wj3Vw6sq/wCP/wAo6cEaX2To/gvLzJxU4dVXhpcAQZcrnKfKDONI7/nA16ArF/sA0bFNAHjPzrPhb1H4hAdKuczw62TA+g10xbHDzkAZjqAkYGu11L0YJcZrUtm8WT/cyw3rmNw8PeI1q4OVy2XtL8WP4wH+hXswokaiBPYP2mz5D2J0JLBh7RbQzGYJGadyBMa3i5tqGr4kkfWj2Yq9hpEhMPw0k2yLPekaSShqDP4J0NRNGkALbbIaSAwSXgNAaDI0efwQLsQdCPDYkaJLP4JZ/u0HekgPP4I880yh3gijuRowNM70Jm57tS9tKz+CWfwUXWS1vdqmaVn8EgPFR896PW3o9wka2aWfwUTNwEefwR4kI9mKjg82kBgizqSDWX3Wb/qRL4YIcPincgQbEeBzzSSw/DSw+KBJYfFLvQZ5np4I92X61NCpTscObPBRUs/gpXFpBz38DBzRMD7HgzUQ41Dk+3tilPH9GCWHxSw+KNEQ7bss/aWq8z/9O8h+yVjn0VGtwDD86mpYfFQOPgywcjUcYzDyLQQKO+Y6kWmxoYecz3qFAkxJLQNt55+TBSDgSsDcPWD5DPBYugp9KqPtH6+yDXkBV+tRvZ1E58l3yBsTrIViM6DcSLDw+fers6w/oGB8tn5FIpGabBx1NP8AWneWYjDmj0XJI+G+AH5Ego75+0cM/oQI4z9Vj6bDEkPnwQRreYpTWb77xu/OpcCBUoxePUnga8gKkmPMMyuU15J7/OgM/VTx6dR3x+9SJkxsMPV37P2bFZ02BFgP67jAH9adqtHcN3m4vsj6Ees+zpm7qONnmtHDNsI4LPvHWAf1AlAYeRW4A+bWprgH5EeJLx+EbjapIzwSZuo2GsatYfPA7pypQPNGfkSeqUFl/Q1AZDz4KBH5OqyZQaDBhh0KXMgVXEGJXZ3oDefe8SJLzD5EjqvLZ88/mAeRWBk0qCzgYRMz7zzXxmAxqg/gvsaS5WHdCDSqk878kZaKNw04k1WB/N1uSYwfjSA0QQV8OlPzHcG4gAHY8byjnAfADYnVZln6DWoo/o5XwchqpVy7WYYN9mea2H8g9sSQ/wCUF5HJD8GPgAIOO8nF52O25UQktAeZgbyta9M+0khr1HLANAMOXj711uBw64SUSRrxKG886H4pma1UOZb7I6dNo0aM18gIx7fLzuFgX48Hh0qe8PYb2xWcPgVxGqQhKfOHDa/fPbwXoD1w3/8AZggOpA99HyKzblTPAqjstf8AKC7mTkf9XZz/AM60tH4e8K6Vh/MwT3e85C0zwUp724GaDk6H/Yc/rRtsgrFGpo6FHtmBGD5GQBNHclSP/h9oAKy9VUp73H8aXqSjfgn+tEs68b8nPUlHv+dR+WY/DzWoeolGP3jwfQo4UGmmObDknBYs9s+yDYH4ikMssZaiuvUMT5/zofUjHu3FZtX6LPnRBoe7U71C57upRvzml6kf9xKjGfyGj1H/AOIPxE1zLGroa4Z+RA9bE7VzcimZ/IaB6j1UAzbo+/z96K2mmAB7zBI95KqOHXDPxIrx/kTTwVJnw3Iph+RErDP4Ja/zqkeelh7zD61HOq6Pt3wVjR8z+I4CB6SGOnqLNHVXPkwQBUj+RFtRGeb5d1/yAZrz5Rw5ye7O87xrp1w15+Bb1Qfb72TY/Wuc2eHhAu7hcvM6XbDzcaHI1Pf7FZnPbD3ioaVGqTwacEMD86swtW4JLuo429J84NAuP2f3r4f2HXq2wHs381FeqWbpuZqa9RKVR4+vXG2YH/1DyhPXDwyhwnZU6qszwb9zE/11kvAAqXzqRGhy6k7g22Z/QGazT3G+0abh9keGpyZvnl9ALL1XjBxKqubcGpHAD8KCzgt3rsB23FjBqVV+GyH/AFh7BUkm/OGtvarblVA3Q/sjOf8AGuT1Wjny7rj86Y9I/Ge2Jo6axJhjBf5kw8jKwHRXuN8UJgMW/Sv+8TjwBK5Lq4qU2Q03rhGAwzN6OGCyRvTteO45SowaAYKwo91P81yL/MzGov8AR+xGpl6TQ57oVK7rjqsx3vaDs/OlRLklhDBuhvssyw/CZzeUJ7TnlICW/wBZ5oKJDfjHz1G9rvYPNc6zVSenVKUD8t/lpbZ+MAd6dNlwM22GA0g7FIOHLjE6/Lgsm6/1nmp0DlGabKqtVcPyAAdZr2F4KBdR0rDlKPDAw68A61Epr3P78OWkGZnqu7ASDQktBK0MPOakBUuWDw4jJ/kXhgjptZkyGvAMI5nmEs+g01PDe6ww5m7hhrB0K1pUnWi6bjZmYHsA1E9d2rDlePgYdBgALJeB1iAwy0DBvxnnj/BPPAFHpTL9NkRKrp9+DJqJMrHhNQeRk6p5mDujmGCl0GS+cUG65Q6rM0/Y4QzwR4UOY+9VJEt/Az3mnecCYIOagRpB+9w2Ia8EvDZTY1Hz2cuHXh86Cq+roceO2DeZ95+dbYTo6FNiMsc2ctmTvADNnrDNIJMSBK+6OSdn4yj00242fKVUIAeQwMwNKYyclo5zctmS0Gx549m9Zm0g+bmBqa5vB86+xpNSjO67YAbTfXmCKlUSq1geUppgyB++M9iu43DS443iMXPAZd85qwFEoLlyNGxz0bVPsz3gCr5Nq02mzNN+q5uh2aP+2rCq2NceDWvc8Z4zPDNk07TeF11U2Y04d1QAafPA/GzPD6FMWu4ZK6qJKuGa0xSnJLxsHrMvNB2edV/qq44cgObqRyQ7wNddmWSxTcIM655jzQdARwAAVFPsOJJka9sxalMkH2SDzBXCMs/DjVXwmKawB4AZmYBgrCBSm6xWdOqyszlYZgyfRgC2FpcJeI0lr+dTZokczM9Y95ra0fg/bFEMHJU6ZUvJm9gC0RduTvUS2ecCDSrcenun7HB4wzWwpvDS7q3pfzMzb0cAwweMzNdYh+pqIxoU6DGhtfuQAE1JueIHv96z2jTPwODNss6Xrx+TWMAwwdPAP0LSwAoFsRQptKpsaBHDsZDBUp1uuST8BwATPLTpO98wM1oLaZdXig2wwZ5qnk1KsSZWAOYNYbNilBDbBPMw9Z0GwDeiVOzDlyXcHzNOswMC021NmT6HQWtSsVVll38EN5/oWdk3s5Ma06BSng/6xID/ACINLybcPx5z4Mtec1TnedKN04tvsHVXQ68NjKpzjPz3+bqrkmS6fnDMFcAD73sGzz+cEUiPfaOsHqVF8IcQ+uJH6E9DpsSH7BtTfU9wH7jP86orhvCh2l4FSnMvTf7PHPWNWNAcmdhvwVfUq9R6COvcFZjRvk7/ANC5jXuLVx1L/QcFmmtdjrrOZrBHAB585UvN54+948zQdFuHjY/JaOLasXRBz+lvdf6FgpMmdWJHNVyc9MkeczQAz+G2pDLOfu1qyNBGY8iMABSMN2CQMoGQBIAUvRDBBGNt4DMM9nnBWFhgSPrBGYYCo4PPmR+B4SCQB7ESEwcP2Gw/OaLxO9zM0AmZ+zSSw+KSsBn8E6CHsRAeA+zzQGaXdpoPKhPMPEDrUIIwXzoNfc8/aJK1kZ7EGfwSNAqQNLP4IEs9viIFn8Es/ghz+CWfwRkLP4JJJI1Cizcw1EkPnbQO9aDoSSUhZ5pGhxNLyaiofANzHxEOeHtDwSMw1cNiNAQb97aMD/ETWs3npnmkyBmRg4a92JBnvRAaHvS61DyBGH7xIEuxID932I9HrZ+HggD94gDrRoEHYjDrNNHqA6H1p3/GgQBgYAaXiASXifiZpAebumgW/Pf0JfvEfXn/AMaZw3oDA0vD94kkYbEC2IEku9As/gjQYYI/EMkBpZ70GeYh9CSAz08OjrQb0f8A68/oS60AeJqpGG9H4YGln8EeQZ8QATR5mB4dadPe6aaAHMTQ2hSaU3MDTfz/ACKIdsMAObc6SyrvE0RhksTakgRqzR5HN0a5p8Z35FqYHEi/40xpuuXG9Jhd+AYPfrVZhvSejZo6XWLeqtDuRo4kS/5nMP8AsWp3gnn+tVlYsD0k6Ux61o9zs1iEHQyHWf5Fy2TGXYuFd825R6XHYrL95nI87UbmWTD9Ci/7CXOplztzHfVXFG2XobvfnGNRJ/o/cOa9DOdZ1xw4f4LRmvUx17hzfgBb8SXM5t/oZqdKwz/1Fl43o9sW9P8AXNq0oDkf2d48wSLQ8j17gJeNKE3GIoSY4e+gvZgsYdn1EPEYlGy78i/QuTf9Nttr1dd3C/kP3sHeCoalbHA/iK/rgcbm3/OGi8CvY8JQLnvG25GBxNbD5M1dxuMdSjOhz0TA/Jor1X/6LVSedNyjXxDZiH0A8zmqaZ6G0GpVQ27quONJidj0GMYGo7lRwvN/8rVHelOg/Bwz/CVfUr2pvXFCSYH52cF6gmeglwy5N31HcdVjTfcm8AGGfzrh91ej3cdjVJpipTowameB55gf++adq+ll4FegyaW6573sDBTqOcGZn4fQmqlY1f0iYYlRj+gMFTM23c9K8TQnn87W9LRyw69b1KgnFDwFcHR4IB0LhjNz3cyemxUWTPsZPYakRuIt2xndR+Ln+dMDtYUqL7PQUeq22HJH9xZP61yE+MdcknpwWDZl+Q1IPiveMb28Ez+sFdQi2jjcPYMyqCb8XAD6wA9i6BasmsW216qpVx1VlrsaCSeALih8aawye+lYH/cq2o/GlwHdedSpMk/rwWuHr0Aze19xj8O8ar/5k1cM8ReIzI+Hc0l761wf+X6lAX/s+9+c1ID0h6MH/u/J/wDMgufCXc/5SuJv/wAbP+P/AF0QcTuIwe0uAw/IuKB6QNuf/Cpn5N6ms8dbLe9vzjP1sphTsYcWuIYeH9oz/Qj/AJYOJv8A81H/AOCuHyeK9qvO64VLRa8+GauKbflsyWgfiXdTTz7JAYGmB1gOMHEYz8S5zP8AIrONxgv8A/05n9bK5izcjEkPAlUqT/dGkFz0pksJctkD8gHmvMDpx8V+IR/9OB/5ZB/KpxD/APmNz/wVhAue3AHfVWfyJfae2A9pWYwL3CW2PiXf5/8AvNJ/IGCj/bm9D/8AeeYsZ9ubG/8AmCMmj4hWOyf/ALQM/kDNMDZHc98H13BPNr68EbMyqz2szqszL++WEk8XbSDFilOSarIPojx2d6js8crcjBpv0OfGd+fo/WouFOlssvmfjvvGpoAAe8P9a5rSuJZxsJ0qCcxo+gGutSpPEKuVho/sxbh/3rxqNjpbLwZfOqS568x4USj3dGptYA82d641cNevyS6DlViyYfkwDYq+Gy2eZno6vneTY7nb3pD3xDmHFvGhw5OgeBvAYMmuwWrxmsCveHBuMIcj8GXsNeNAnvs+HqIs2JPt21eB7krfDex7tHUrFsQ3j7JABgf61zSveipQzzftW45MB78GWGsC87w5lYgZhBrMyM15AkmtBSuJHE2g+HRrxn4eR57MP41exe17gVxRo+b7FHCfHD3sc81lLetW47kqQUamwTCafuZYaJrTM+kDxiZLxKjAe+uGCkPcfr0n4etaPRJhh0PcngYGmxYPejHxzDotVl75AmBmsJW6DcdpSjpVz0eTAnB2PBh/GtK9x74xSdjF4nDZ8gM5pqt8Y+I1z0Y6Bc9wetYTnY8z0H5wVxaGPz+CQb008ejvwz+QEz64poB7fR+R4FbJONAgZeYkgD7B5sn3ghM3wPAD2eQ0anTSz2JZ/BAgMN/Wl0GlkHzpeH7tWAz+CM9NBh8UkCSz+CWHxSw3oFnvQ5/BF3aaHD4qmeySSSR4WfwS7ECWJogsPijz24JIM8ECM0utLPYln8EAbwR5Glnn7RB1oHu3UQZ/BLsQIDz+CPP4ID0+xAgeyNFkaaz+CWYZ9aJO5n7txI3k1m3n4fQkZ5u7OhFHQNHn8FHz3ozQSAMEj0MM+9R8/gkaB0OtGaj5/BIDcz1EendiSDPYkB7dNBz2MzEpT+vOg63kDPBT6lch1ua1oUeMGB5qqCmud/QrCNGjsug4wvlO9o2arTnnwYCh01nPve6FLzoAf6VnU0I//VA3rGVI2DLQcbzTUN5tloGM8Gg6FpFjRVisUN5rChwZIfvnlXs6/tAfMPoVPMqrYO6DDbx4eQE9Do98VL/RtGqptecGcFoL0DAA8dg3k09PYDogsh8+CuKPwi4oPNa/q14DPo5g+ha2B6P1Z0gcuO5ozId4aJoOdPTKdMYBttwGXe8zNVj0NwPYXBn+ddoh8CuG0N8JVVrMmY7854LSw7M4UU13UYo7Mn6zNDueZWdTLx2Hj+RneZrQQKDdtVdAKVY9VNrzr0F61ptN2U2h01kOzYo8m8K494bGiH0Ix25fG4S8QnjBtygMstH1m890K4Z9HWmvf+0F6gH7qOyC1Z1W45Pv8FE5OWZ6jmChe0Wm8HODtEwccl1WYYeeTs/wK9ZpXC+m/wCjbYAz7DMzVecOX8iejQH/AMMERtbxrqfgDp0qlMww+RRZN1VJ4vvb4AHkTIM/iKQyDDPuAP6wQ2rzmTpI+G4eHnzRsw5b3tM1Z9mo2wz+hSNb92CtcKwKbh7tII2/BwDVns6DTJm2B7FCQhGY+dF4Hu80ORo9bZpoAwA+tSPAAfOoOtvS5yIHtH96tCcZtl7s0tZz3eaj+smwLBtg/wA6iPVJ/Lw3EWs+ZfAfImTny/7WeCrTmOH19aZek7EFmdSMPfpBWH/eOAqcHgMdRLmWwQWp16d7hiNh58Ema8/nqORI2fnwVLzgf1KOcxtBpXrhM/aNh+Q0vta2yGDkEz/Os1rMGGeuCgnUnz9mCIbMLwb/ALIaR3VSj63DBYoObknpgrWHbdSmdbaLaP7YW/h+1n+hMnXrck9cVmT/ANiooWZTqUw7LqsqHGAAzzM1hIfF2zmcw9VHJPPAAZVjdzAo5+zo0b9CjxreYn56FuH+QFiarxduMy5GyrZjAf8A1jA1irh4i8VDkHzc4Ibvey0AYLLwtuOJ1vVG3rXkOS24zIPmAA1rZmslb0bRaA8FlzCsVKQ1UazUeZdP5+hbKMzhQ5B/uV9Dh/4fP5lnP40/Z6A16jsd4MAw1ZG/P+BTYHEKuXDbTtVqrj0Mz3xo4YAzmuevQJ0+EESVOOTEA88zWoOAx6tDM5LwMdDJ71wez+99Hh/YzkaTnICdeIM1J0P6RrZh+RBUjtiZMNtulPAfYABsNaOm0Sq0qqR5z9uAcdjP7i91n+RXdVuTRqnrmDZwQ5p/2vDZ+RYNGPP7mWGGYYdCkSZMt4GmIODOe/AAQBhpOypzj0l1889+xKMbgZuPsYGHz5rVGDTJ5l9+nRtLz5p2qhBhifqqI9MMOgwwSDQCmuym4sblz63sN6iTzYhg1g+DOZ4b1httgYHrRwclsYSz7M1LpRhG5hxhj24YGptvAxDrMfnp0M4/fmo8nkaPMkadchnHfPMMDW0WYO1h6DMJqdyIRjAMAwVK9P8AHBiK54p9ikT4DFSYA4lYB51g8wZMMANR8NGKDE7lo0v8VemFgE+jyYU2dVWNF2D4LPLnvM/OkzAiQITs6XOzN88z1TUR435mlqVEJJsBgGHYCdeelHg2395keTvNFLapBSjNpij7GjDN5k+vNVkmm1VmYHgYACOH61mfdfU4BIPokGe8PoWgoNk3jVcHzlBynfIkPYAC52Vql6fFgU3XcA+d8gAoUaY48WpEzw85hgmr8eiWq+1FbuOBUpB4bIm8A/Oo8Csa2EWCxzIefPBe4RtenPuNkf5mfkgf7lSAgXcdNaqVVr8kzM8AZ1jMwQU2pMUoDl1y2alMw9iDJ4LZwJlRuHOKxY4UfDvlvBrGoNslMt6pPU05UqXnIPozZMzP86ms8NLjmNMuORA2B3vYLcUqY4DR05x8wOL14fOmjN9meD79wSXgcPDS8i2g2zJ2HUoeE6dR6abQe2AD3mo7xzpkhqJBpTIO4YYYda3f2nYZm8i+2uRQJL9Vu2qnEckzJfOYRmYIZhgvFuhU2mnb0N7m6jGkyDPM+xVtY15NX9YwaU881/SXQ861EPh7eNwwo7EqlM03A8zdlvYGf5Fs6Vw0tWHHwqLHPu/vuj9CDjzNyRPWTUS1bcnzJrB5+F0Z+Ra6NaXEa7Ra56DAtuOweYb8zXWGeRpsfQgwY0ZoPwWQBQXq3EZ94pwjbPwOGNDZka9VqsmeYfkWmZ9W0TNunfdmj7M1ST6q+81psbFVHzb3tHM1cI20FSud+Hg/zQGHRgqmpXDUpPsGwADNQplKzpbrmHQrg4DAO6mn2Zrxav5OXJLUluGadhwAA/ZqwZhvvHgwCF6ZQKO0D9VqOBn0NNbzRKRyzYezUjRcZi68pwIzXnePBU/2huCeOFuUZmG1/a53Wo4W3zjoP1yqvVJ0POexFLB656dGkaFKinVXfOGxn9ajm9cdS1m5z4Q454eDHVgzDYZaBthvBSAZWqVJGtW340gJYQTOR+M8eZqw5OJl7AFLmPUqlNc9WKlGhxw6zeNc0urj9Q2RepVjweckdk57YCsdDN7R9o2AAHWfYCxNw8Zrco75xaUwdVdY8h4M5rkVYuq7rn2Vyqm812NBsD9CrAjGHkQaOvcTuIVz+A/VQhwu+PH2Z/nWX5NgOhSNFz+tIGTz7FqyGgADM1Iw6vAAEYaivCgBrgODbe9Os6nvEYAj/eIEYbdRACPPZ7NM5/BBIzzL2fQh60uxLPDYiCQI0gRZZ/BFn8E14ee/yJb1YQH4qM0HekgP+86Esww1EjQIDz+CDP4JbEtiPNln8EBml+8SNPBYzQIcPiiz+Cu0ln8EkKWHxUBdaSXYgVA0g6s9NJIA3oC2AGaHPMUv7zvQIDRh0IEQdfYgXiGSaPq+tH2ID9gen19iBHs7EgPJLRA8HD68EujD/iUg+/A0gZcN/wACXg0CDPNLNsPzoJCPPeaDPYl2I8gfie07EkGe3w0kaDSDq8RB4gF4fkR9YIgXel3odiDP4IJAanekmkX7tAJ9Phos+jU60sPigPrVA19PPPNxMo8jUgs/gl7oOhINMy9okezvQLD4ocN6kA8wDDrZxdY3wwB3yKPn8EBAGKRnuH50g60s/wATegM88Ug6dTsQG9n4aQbEB4ZgbnYlsQZ/BLvR5A0H7xHnvNBnsR6HP4IutR8DA8+xPAeSBYb0WwAwQ96Lw96hZpnXjSGn28Nh54GGYGut2xxybo9stUqq0OZzAe+ibAMFykDSPrXNzev3xgdGuTjM3WIQBSqbMZkAeYSJBhmyfyKRbHpP3/RPDuqmxq3B/Q9+tcsw3pJw+v0LeraDxv4X3tC0Jb/qqb/Z6h/kPvR1jhjaV1ZvsMM59kiPh/kXko4bD3tG1ZW9c93WlN5q3Ljkw/3OeYfoTCHoP+Tq+LY8S2KxzjQdEd49/wCtSIfFepU3Rg3VQ5kDU73WcwWKt70nKlGLC7rZ5kv7RBMA/gXVaPxO4ZXnH5T1xDP91L2Gow82rzes7iFC5Gc5rD2cvJwNk/yLnV4ejfXDY5u2bneqph0R6geB/rXVarwWtie/zVNzpUgOh2P1rOSbe4sWfIN+nSwuGP3g97Zem3m+5LMvG0j/AOUdAkw2j6HsMw/Wq+GexeoI3GZhl8KdeNuPUoz7DZzD9aQWTwL4qZv02DoyGDMHnYPg4GvO428vvaDx+OwB/Wosmg2/M9vRoxrutY9FqqxmnZ1s3UFSDsZkYAuWXJZN62l4lftyTGj/ANo6w/WumLes0zaVuQ5DU6JSowSGD6z3q1N7W/a2Mw/chvUdl7NPGo2JD1KoBjmxceB/9bgYKCFND3EVnDv2AnsM0QG4HQmxWzLbpUwvEgsh9AJk7Jtx4MPVoZ+fvVsaj8y4CYAepGDY0NBnSw6DBR2bYoxxeUlUqMeHQeCsOZfNJl5wz3q3m1E9ZNuf/CgD6EAWNbnKusBFwz71o/Dy8ROgzmPs0NspGs+jRgAMHjw788M08dn0oy1PG/WtLot/1powxRKnCjsM+zM/zq1olq0asOm3VarGh+TW70kBhkg+1WxqHRJjQMS4E+O/n94jnmYfWmY1Box63NPhGwDMMw2KQAYp1FotHzo9WiVm3340abCPNk3QzW4PiRdU9o2Kjb9t1LPr+5gsY8zB942mo1SCM7qMP4ACyarB6ZrTDf5FmNmfsg6ASZOpMm65FlyYxvhgeieGaiPXC/6y9VADL3eGzeauMDDrbwNedUHcjnPrkljkZ1SkvM+QzUJ6BmQOYZ4K1wD3jaAwbDoV4hKvNkMPDA0YB+Ipef4iDw/eK2J0DBLmW89PUQ6LAF4ad0Wz+sEB5gHvM0yB4HqN9CRxnOvYhDXD6FYdzb/Ojzw6DMPnUfPejB5vegCSD7zoaks9nkTr2DwacoAeD5wTLJ5+73p7wz61YTIcs1psBgCd1tYfE3pYOY/Io5gGw3M9nkQPZ+cMER6ajmYYeG4hA8PaHsQSkks/gkBt+8UBrPAtNGG9BmB7MN/nSz3qwZngSDPdmjwbNLDD2iAM8yRpACDB/mPaIzsaSXiASRvaI6jip4SA+rw07sxTSII0vEMEaSPQdiA0aA9NHhZ/BJLD4pYZo9IzxSQ4fFEjwkgQgi6zRJIc96JCi4F0JZ/BCl/wdBq0EaQGfs0gSP8AEb61ChZ4Ic/gl4hgg60DoPNo9Zv+pRMPikgl5AkBgoniB1ow3hqIKQKJc0//AEbbFSk59Hg4K+pvB/iFUo+v6qCB8kg967Q9fOH7LBjfoVfMu2sSfZuAz9ALkdbnkb0b7nntZ1y7WYbXeEfNbOg8DeF9t+JOnSak7/1h7Z+hAdSqRj4k54/zpozfe9uealW2qAOHNHL7jQ4GfyMqS9fLDPh02KyH5FjgBPAyjNdSbqqrx6mvh9CqpMydJLUOc8aWDh+7Rgy4BajiKRAZ3+zRcu4pfQaeAEQg8s4aMI2zUUswM/YbEAB7tQAAE9sSSw+KtZJZue7RYfFLBv3igLPD2mxHm2YaeGaB7Qw8NNGeKtCQGoyODfSjzb0lB5nah1jxzRaQZ/vEtZv+pV7zyXiB1oJpzMw00Gs5/Uq83jySN5zDrUM7TjPJM54FvUXWfD3gKCcnN/x+hWzXwSYv4iiPSYoHpg4qJ6qxGRPxOhUP2kbZPx5zP61beG4emNqOcls+8FlWbnCZm22bwB+No7EzVaw3R4rT8rmTN88AAwWLxqNYwDeear5NbCMWm4auabbedL9ZXBWQpvyAeayP2k4V1usnQ6VOmVWX8mwDVqWHrUDDNgDP6ASo5y6kf3rRZz7HT3qkk8SH6DMkUa2LZjAcX30jAzWHr11XP4058+TM/wAHvXP+R1uTGpsaVg/OjB+daA7n4c29AAH3DkyPwQDea5Vc9YfptBpT7DGsEqNmGzNUMaBValnOnVLDPoa6DWXcOgVXi0/J0nLcs4KaDB55vPZmf5MFUV7ipeNYHTYnerY/4MQMM/rVJ6q93z0k/wA6ZeCmxmvVzDEx4/OYLsjmaq31rXGao1UvXEl7Dse3grIIdKqtcCpMaIO/hM9CEKU+fhthsRQKC/DlOuU6lZyO97DNR3PU6Syw87qMP7/kNSKaGi6bcvCZrhhvDoUSBDrj0jUg1ym6oe51gzUuBPfedP7RytbQPDSwwTbxl/AOvGDHslteWNmiGDneGCyUblJlwSH4rGi0b3gtB2AthMgTp8eOxzwQGm/bPOnsBfS8f4nzb/yqeMDjMUA0+hTYcyXGPQltmAH0ZgolSu20qVH9VfaP1lL/AOqRs/41LrHFe472ap9Kfo4Q6ZCDADwwM/rXx+b9z7HD+xHeOq1We7Lff1jcUipQ3IBu03MzkB3n2IKbJpzJBVX3DPQ90B7zUit1uC80FSbc8V88AZPrSF2p5LNR1dM9H85pH9qodGdnORQgNa2geG/NTodepTLuvWWMw8+aTwUp45Et8M2tmiHYqY4U9t0qvwJHrWnVzRjmBgfPPdf5EWATJrvrGoszJAdZtdCnA84z7Mww+REyy2Z6jmAfQCha7o8bhtMEAqVZwqB9HYueVs2IFxxwbiBVQDPMAWon1L9ni6ABoHnmHeop15umjvYM8+8OteinewOpR50u3J7IMf0c+9X9YhsGYSn6M9Jaf6MwVifrXCPBqTgPOnvA/ICZqXPSX3X5UuNpAeAZngH5Fv4UzsOTTQqUdtgw3+RXQPRGX8zNV9HODWAlvsUYKUcXrkSAwzUilHSpjunypvH3uh0LzmStmXrfN02Obkm72cuz/nU2BZk6vQnddw4ezMOYmbFTz/VoSP5tDRaP9YKJydNP28vP86x8DNVK26NJkR25TDL2ueGw+hXcOwIlvTY8r1k8zszAFZQ2bfZhTQYbA3TZ6DUJmquXDIBgKbMkuh0YB0At0XDUQ7qwaBiqt8y0HQGeClyb8fkx3XDpRmcoMA0ej9avbb4S3xUo4BUqPDpsfv5vYf8AAtbR+CFuUctOq1mfUg/CPYH6AWdwjDjv2qrlS5iLQKG9Jm7M+XDr+tXtB4UcVLnwnVJhm3o59eqfR+Rd4gQ7ft5jlaBTo0No+vRDBI624B+3TAxlvej3bNKdCVXKrMrDodhnoh+hb6m02h281oUalRoDXfohgarHrhf92eahPVWc97xaDSyakwDR676p5lY2ZxczBV4Rs2gc35/OpDLKBp56VJj7zNMnDzaDw+hWYM4BpqQzAcxzWQiRoebWaMIG/wAPrQzLhtijgATq5GZd8ix0m5KbXptQcbgyQkewgO4Zhh5wQbCsBydGly5x6McA3n5FW0Ge3WI51gJzPLn4LOrsPYuc0fhpfEmsuzrtuA3oko8zZOTmf6Fq2bDtylTTlsa37lnsD6FkFWHr7nz3olNqoBSgPYYGAZp2jwHKUOHq4HnfPnmngZnZbADBWsBl83fH2fQtog2PWr5/0EEggXGboBoAGfyKpuHiFbNsH9+qus7+FEPWNc9r3Gy6qqwbFuRDpsR/vdMDeV4Rt1OvVWLZ8Pm7jr8aN8gBma5dWOOtVPw7SiZtf2iQH+Rc3kg5PfOVVXDkyz63nutPMst4eGC6Xop8msV6Uc64Kq9Pd7Mz2AkAAz7NPACWHxSIQDo92j/vE7o5o9FWGfE92jADwR4fFF4f44AfzogLIOMjsRAeHQjzcNBhsVrH4gNJd6RnmSA1ANIMAPU00s8BSPeXs0QDMzPw0ZsuYe0zBLo6OhLNv2atZIA3ikkCBI8MxS0WzJJAkuxLPekgM0AG37xLBsw68EB/if8AGjPBJH1pJI9JJCl1+I4jyCSS/dpI9IEu9LvSNUEkkkpCRAmsPijVBGe7TSBBkCQIDQAj7Ej+TYgD/h6fE60kaSBnRcyB/Uw+RGYfiJrDAzb7M0QIDRgmQPMkYH2eRA6fR0I+xID3oNbATb7OtSDANniJADYB4YbAQAeSQbA370WPuNxBnvRo8G/eKACW8EntQwDT86PBwxBxWz2ADbB02zbNGgM8zHURo0LelsxS7QbSMAzVIAjBAewDcSz2h4iPNnfD92mkfie7S/vFL0s9gJf8HWaSWexAGH+ojzwJJBn8EBvZmeokkCSBJZAkkAb0eQLNv2aE/ICQaeRpHvDTwR6DxMtNxz6EYHii6zBzvBCYICz+CR5gepp7D/gQgGfh96Wt+IjyCM0QJH5HDR7ADw1DQfutRsEGGYoAR5/nTCAGCWHxUjBs0sG8c8EELRzTTzLBjp6e8P1qwMGzUfRAz01GFr22+JF/2k6DlKr8nS72pHjAYLqFt+k5BA9C5rRehh/aIO8P0GuL6PzoOWAww3pgey6bdVgX/FNuDLptYaD3T3X+hZKscDbYmTzqtAfk0Sb2G0ewPyLyucZ+M+EuJLkskHQbJ4Gt7bHHjiNbGkxKnBXojHupfX+sFjcDpsmlccLGaM6bOCvQg90eGakUrjrQ9Xkb1ocyAfeDzOYfrStv0mbKqvh1yJJo7odGt4wH+ddG/wCSt+UbU/m2pR3/AHzJ5qNjHybP4J8QjdqJxY3MH1uxPBP+Bc5vD0aZ0BgJ1o1z1kGeGjI6/wBa6BVeANK1ebtWsPUeafRo9Cz8+Bxwsk2nH2AuGEHWYBvTbVxqvWHelq+JXLYmMx/7QAZgqJmSw97NemqJ6QlAelBSrngvU0z7JAZh+tWb1k8E7/8AHBiADp9EiI9gabS8pPKD3rvdw+ipOZLXtW8QktH2SA/zrl9w8K7/ALVfP1rbkk47HXIjhrAtosUXh46aMAY0j86jA9mWn0YKWYeFpsbNnWrQAGd+DjaXQem2aMOeZ8N9wDSBnebiIH2IgZz3mh70tbMUWdMOtADP4aR5mCQdAabiBrRcMkYM706DzaNS82rpIPsu+wMwQmbcwdNvD9CuAMMvHbzDyKOYMGXht4KlxbLzKO+y/wCsm9+HYHWrWBc86f8AtcF5kw7z71amy2a+hSohh0I9OgDkkAltuZiaSqpNNnQ5GvBfzMN4AasHmZz0BqomGjn1x/8AUVoFhns00noYASjgyeP7c8yHkwTzJ4e8zRAAZ0ejoRhr+0bRBJYDs3po8HjzbzD5FAWRp0Dc94hw+KQArAYHq+O3szRyQDPw0jAwBM6zgEmw10D8idDM+hJ7Z196QPNh0GgXMvgew/yJ4JLjwnqdaZzYMfD60JngexBIRB0dChayl8yeHnQO9YJGyD3f0IAksH7MMEutAeHxQdAI88EHWrCZNHn8EAGjA8z8NA6BmAajaEOhfOg0OtgoBogPr1ATWeaQAavwDwAPeI/A86jd6eDTMNRtxXbPAO9HgeGpglnvS5lwPZqGhGDfY4gAEj60eHxRkDPegM3ADY2jRo1MpYfFH+RAgSWeCWHxS/vFTnLP4IUSWHxQCeogPUyDTR9HiZ7EX92gFAaRpYe8QHn8EsgQHqGliCA0Helm550CA9iXQkkg6KDLZu7zwTpgAHpsZmpJwGwPNEzsLwwXEtH0cBRgzmncHDNHouBvXmG+B6OzoTODge0BSM/C+CaM2zXqRgYGCAzw7M0AYZ+IjN5QsAGeXs08bz/4eCiczgfXvQHMczRCdrACA5jaiG9mKjm8rFhz7AITmAar+Zb944kcymgPtz/Qi0r1lvRnPbD3iz8y6qHGP3x/QCrHr2puXhxXkQ13OZoHpLf44LFSeITAH4bAB8hqpk3zOePwG2WQ+RWt0bmQ86DnHAPBc0+2E7vfTX2nrDx/dNbP5AQdNN7NLmcBWChyb4nl4bDx/kWtpVmXO8AS6lBmPfRsQHJqXkVecmsmfhtrXMw7YojWvVeTDDrzPPBFM4o8L4DGfrXWPyNBmoeWy7NKuaSXs1pYfDevzI4OGfX8ipw9JOmh/wCytpazvY8b2H+NZ+5+MF/1UcKVUY2f9J5cPYn9aJbgOEUGHF9ZXU/DZPvzeTWfCSA0DAMRp7vYDQHmuRa16VJrXrj8mpZ9kg9isIFyXVRMNTRhxA8gKlthf921JmA0xblN9Wh35guXTJNcuQAYqR5n51pqxVadXouvzxyXfIap5j0Sjsah5g75Fx9TVCCybfZ2VyuTHnT91H71EmcPbRPD1M+dNw7896s4dbivNaj4b1EnzGAMPE606hLtizPU7pvwaqEl3sOQewFcXDYFz3PD0HJdHA/ODyy/ORTH27wfQrWHc7FHpupyM94+gD61HSyV/OXrGhBQ5Rg9HYPlQ2bMFZ02idDE6qw4YB70zUSM9Okjrvt4Z78EfOOM9C2jharCq02CyAAxdRzDPsDoUq2JlKpWtz1Re34de9ZKfW6xDd0+R1v7kFa0SfX5LWuxEZjfI8zvW2BNCYe9xgDBrWPAzDrX2g3J9lX5B8jz7MrrB01U167bqMDgtuRj/wC7GrNmMxWLK56lZ+tT9jmai4O47MrHJ+I3Y8OmmfsTZjb81FC27mqsCRcc6DPhx2QzMDwAE7aXEus0qG1Q6zw1qVSqYe2e2YYfWpdYuHi3c8KRTjP1bR38PBkYZq/EG9qGzIE56e0/odavrqDCZofIGf1rD29W+Q52DKuYOeivaDOB4bFGtjiRTvt1IYqr8mpNRQzAOzNbdyOls4EDk2NOc2HnDlw6wUcwwdz8bR7AM1qGeItl15h1xuC8y6fQABmaz8+4aVGLnooMvYf0d4P8iyWrznwaafNOU0JOHYa0Zz+ZgZ0a1WaVL88sMz/QqxnijcZyvV3qaBAjn2BDwzU6Nc8RmRnOprx/ICG0FmTcExrOWdH/ACBgjh29EqWfrm4GYzXyGru4ZlGo8+P/ADV1hn4KzlSk19mObESlBzD5mbOZ9iGzMNmmwD5GJ7IFY4RD/ZDzTMOzH40XnpTkx6QYAZgz7EFLCAAYNxJwBI/CNZ3C4tRTNTAH33GTaPrDyKVR6qf3gKM/D5jDPOQzngpcCsROfwqMEDkd/gmo9emPhKPlIMmG0/swBnAE2jaFSrhrnPidfiAGz2rvgrW0Sq2qGbdZlw+dzzAz7A8ix4PVE5jTfKBMd7AANY1ew+DnGK4SCXEtmHAa/FlnomH5F6vazqtbcr0CRFcf1mTPZgGAAHkWS5aWyRsRd7ueBsrrVK9G+uPRz+1XFCSHkjxIy6NROG/Dm2IWg3Sufkf2icesazvhZOCW9aVRuH2dDqUxo++OGz9a3tvej3Rj+9XHzLLXYyEnf+ddbOpRGfDbPBnyKJJrcX8RbeIGdpvBnhJRzBw6Hzh/9YeMwWzCZBh5t01iHGa8jUYFmZNY1vZqJrS5PWa9Vto5lY5b2j/8CqnrqzINBgzAw61ECAGGB5mnWYzYDprzaUc3qlJd8ToRsw3MtM81IZZ+9AfYpeGBmvRB5ZsFLjM4HqYKbGhuPdYKFPuS1aIem/UTkyP7PHDM1CE1lnW9mCe0W43jynAZa85msyd1XNJa07fo4Qw7HjDNNBQX5Lpzq45JnyD6zee2I1Wcm9oLPh0OC9VZHnBnYCoakHEO5AzluPQI5+6j7P1rQM/dvDYiMsh8gJ7nH/wwQZmBZLEM9kEDPzmGZrQBSnGWvEAATwc28o9VqVOojXN1msw4DR/2g0CCG2ZhqYfnTr0OKyGb85llr61zKq8coIZsUClPTP30gMA/QueVut1+8JnN1yqvPNdkfoAAQdbufjTZ1BdOJQ2HqlNY68NgB+dcprfEW+LtaBuVOCBEPrjxAwBU/q1gOhsAUhmMbK2ZI8aGwyWow3vc7+9SwZc60sMCTufwVYEfRfP2jgIwBSMNnkRgHhZ4KxHw3qQAZ9bmCAzSB7PxG1YMEgQb0kB4N5eJmCSBFhgOog+gf4iL86EDSUAs/gln8EKLD4ogjDAtNxPB0JnelkatZH1pYbs0kfZ7RAu/5EB9aSQaeRuIDz2+GlvNLoSz+CAEu9I0uxAu9CaDvRH1ogfQhz+CXYl1oEgz+CW9AgPPekfWgSz+CA0efwQJKgYJH+7TO/JI0DySD94j6wQMgDbI+GjSQH4J+0BAZn+7S70gPDNJAs28kDMlgy8NzNI+rw0J/if8aBYb/aJd6QHvRApC6DTge1TMaM2zg2B7EQH4+n0B51QOSD5hptv6KkQ3nI2DjeGfzpo0gQJEmv7xHn8FId6PZpBhig3pdmHcirPZ/BACAMMkZ/h/8Sh6WxGmUYK0HP8AUXw0HWkH8fnQLvRoHtM3TMO/sR4aLvidivARpZ7cEB9CQdOmoCS7EuxJA6HWkmt4JB0HqKAgR96SQeQ1YDNs/EbR5/BB/dojD3bnWjJ9Dfn8iSR4YpBv8NGpAbm9s0QB7tCln8EBJrDNGl3oAA8NmmjPf0IDZz9ol5UWMDRdBe0TR6Z+7/Olh4Wv8+CB0Ht+9SD6c9T8ir+3UTwH+G4glaJmPhnghBlxR9ZwD7zRG85kahA9+SWG/wBogzPyJ7/GgZPfmCa0v0KWGn3pf4EEHlm3vDwUumzKlRHdejVGZAd87L2CANjoOJ48Hj1MAD6Fkt0WiekVflEaCJO5aqgHnDA/1rrtq+kJw8uHNiozvU83+zy+/wCg15a0W/60BxgNZ3C3sivWrY97QNSpUqHJaf6HmVz2pejxToec6x6lMpsjsDWXB6JXrntj/QdwT4fyA8eH6F0q3vSKuqmxwbr9KjVUGOt5kNE/4FGFLsKlxssl0GOVZqofxrQUT0iqUc/kbqocmj/O90K4tLjZYF2tBhOZgSP7PUfBP8i0s+1bVuqGDjkFl4Ox6OYGCuEsvWLJ4M8SJHPOMQwef/pETwTXP7k9GOpQwN+zq4E+P2MyMM/1rTT/AEdWGXTl2jWZMM+wwPoVU9P402Bvl/zrHD32tvWg45cljXjaT7QVmgTAaP3wBmBqpA2D6F6Qo/pD0oMKddVOOBI8j0ZWcm2+CfEhrnm4jPMf2uJmyapDzEEMDS5PBdrrHo0v+3tG8Qe/cyw2frBc8uHh1xGtgM6rbLz0cOuRH3h/Agymj7xBy2/2iPnGz8BxzA+8EZgAdD4HmgAGUug0ebmKQbw1EBJoEpJ4LQTLM0aX65o1fptVjh7bSewNn8it5CiM/eJav6EHiaR6aQYfOChI2ZOZYKhqVSqNt1R2W+5zlKM8wD8FXebjI6jDYfnTUl5iZHNidTQMDUrM/aGgT2ObYlvMh5JAKQzJizGvuj4GarQjP00dCI2EmEZ7I73WCmxggshqRWNHPr2KhL0XA8iSDsSwcxVoAaXX7RzsQ9aQYD7w1ALDD3hpI828fDcA01n8FYkMvNs55sA95M1ImSbfmMA56mOHNDZmyew1EAEfLNn7RBEwAC1GwRmYGCJ7TZ8NINNA11owQZ/BSGT0R09MDzQNYBn8iWth7NfDeADMDinn5wNNZ4FvViQB/iJH+I3+tJnQP2iR6meGpsUA9+l4aDmfkNGGwUYHigiRqxEeLQcB4HfIYJ095J4zYMv2UM0z1nnsQPADeXiJGfi+Hml+dIMDPerBJGCew+KiSZJwzADYM8wzzBA6CQKOFVpx+0fOMfzsqQybEk/AlsmH1qAaDxA+hI9QD8RvBLeCsHkaSDP92aEzbBAXWl/dpAbYHv6E8eHu+hAz4nvEOeadSMG89RGFkZ5kgRme/wBmaaNEEiX0OhfABv3iKLD4oQTrwbPAcA01/eKgksPil4Zmi/u1IFBh8Uf92l/eNgqZAADQKQmu9FuwHSqr+Ag5Css/0QFcBDnH0XB/Ak9GrgYcjWQ+fNc7tU/LVnH9h/gUc2a+H9FNaDn7xjAAAxDeSCq3P7+jRj/OpGY0aiHt2DBfDecjBqGBmtAcms56j9D/AI01z+H7dRzw+hQ5/DOesgPo60uZfNXHrKhn7SmvfkBLO3D/AOipKOhRZth4jj4JoJjBn1q4OHavfEk/oQHSrRBrU03voQUT0wAPTzUd6Z+G4tB6hth7xIMEA+d1R5NBtIzHnqwYH8i1GcekmoJ6j3tHFtgtWyj9nXHnj8gGnQsmjn+yTjZD96eCZ8Odzd6lcyftwD60bNpS3vdrpb1HsulYc9WIAO+TWWZ/ld4c0qV4bbzwB5AV4aq1nhE/M9pVY2/5Fe0r0eIrLWmdReP/AH+tUNS9JlhnH7I2xm753gzWYrHGa/63COVLrhs5+6jhgAKLtbtFN4S2BQcOefDVDsN5NVi8OElmO6jkvWMPctbzXnGBPnXC6evOmST7z1kghtvTeUbg5/Oax2O11L0maHviWjaRvPdjrzOC5jdvH6+Kw1yL9VOBIf8AcxwwBKMZ23MP1dQwkx/njYKPMh02pP678Fln6E2MrMjXHPhgb845MiUeHWrWlWBd0bwJcpkGu881cbIccG8NgI4Z1WZCOo00zjNB74wTYpPULECKcHUMwPr+dS6VSnwYzowfdz6EEwKjMlakusyXs+vsWjt6BOkgEGDKBkA7z602M+cOqmYa/OBgfepsmmtvRdQ50950+zPYpFSebDVYfqrx4d5moQHVWXWgYpsmY0fvgBXhqjhRK/AaabYcA8z7zWo+yVwSaWb86mvPGfepUmmy40Xmp0R5kA7z6FXw7tiTIR0Zh+YBv97J5mouBVM8N6qf9B/+/JPWHUoDRy6lTTZaDvM01PkymWDYgzqqEg+gzPNKic3MaNisP1J6IwGZhngowI+dAAP2oM/rUqiM02pT2ogMZyzPAHj2ArAJ/B0D05zEzV8hmZq4C6uHINfzbOmRvnjw9/616lbVugxKI6ESc+zJd78D6EEa0rcqTQSjggAH0GbyhcgdSYdrNNpTzzvnqZ4Gf5FWU227jr3MMXHWYzMeKGYQY4bM0G9hhSrej82xKjaIdee9ZULqpVer0iXFPwgDPNU9KjWybDoVKhvHoHgGfQaT1EtityM2HPVsJgwN4APrDyK1LKq3DBZxY1+cM/6PHDeoQPRalHB/Qk01r696uLb4kW5bA+polnRof4LvtjM1a1i4YNSa05TEYz+hEXbmtSZqr0h1husPPNbMMHuhVkmZc7LrTFAqLMzQz1uYezDf/ua3dNo9RuGO6+EuHGwPDAwwSt6NUqDceo3TY0l0GTzDoRG3Pw4dMV6O7Ouevw6a0HWEeNv/AFqygcK7YjeBbj5872SDDM11ipcN6jPdkS38KUD+89+YZpqiW9FoJg+3dVNz6DkfJ9CNGCC3p3O8p68mfWcYAVxSjg02ttMOUN56QZ4cxh0fQutz6lY8wWuarLL2geYYAmecsoJnIuaIGexkHTwMzXnaM5MuH19hblDoD3Mf2h3eAKtk2TcZ4ac6B+TYr2vXaxb0w4NHocA3Q6MJILHwLzuc4ToBUaaASjzNrDemxdz7Voc8GnDfeekMBhzGezNZGpU2sTKpoN1KGEjoZze7P0LQQ6PxCr0UIsWgfdzPM3ugDWugcHD0v5xuo4bp9kcAz/WvRz9mZ6hPkazeO9j2x9gfnVhSr5l1upcpb/D31q72VEw2GuoQOGlgU3B9yjs1KQHvZe9aVl5hkdNiLGjNeRkEGCo9B4jVJ8wqtsUSlB+MfX+RXocH7R1+ar86ZUt/QZ4AtAclxU8y52GXTY1OhQLuBGtG3g06HQ4EM/ODIZ/rUg6rrbzzM/OsJJuGU9ngx+hR2alOnj4j54K0NRUqw2yZm4/+hVT1bN7oz+s1TmyeftDMzUrlvCa+tBIOS+Y4JoI3i6malnD3J0IyLNAy2ZhptqQDLnYpDMb8PefyJ02XGd859mMHnM1ARhsTSr59yU3knX6M/wA/LYPAI/QBqkZjcSq20ZnUvUjR9ARA3/rWQ1r0+jUcgOs1WMzn0B51Xyb8pxyhYt+jHM/6waz8bhWwEgJc6qyZ7ofjK+Zs+Kz2LSABvVGqug/LqQA0fZ0YJ6MdGo54MS4efnwzTv2Vpv8AWadZtuIyemDAfnWYT11RA/6RD9CjncjcksGJef0AClz/ALMW80Eus1KmxmvnWEr3HW1YYcratHOqmf8ASPYgH+ug3bLz73tGz/QqGvcRbOt7w51VN6R2MxwzNcard53bcPgVKqmEc+uPH2AqLlmALw2AXR4G6uTjNeNYPTthgKPE+fe8sO9GfnunOqss5Mg+szNSw2B0IM/3a1wyCAYe7RYfFPIEAZuYp7xDHxEyl1+GtVDz+CPxMgcBww+hBhuwTujh4bikLxD94kAbtRPd6WGCCOy9+Iwadz+CdZBgy0338PIjhnBjZuS2zM+wFYaw+KLR2eGj6zz0+s+xLrPBQgHQlnmhPYRpkHnDD2Bh9aA8PijAMfeJIutFhAE7hvwBACeDUNACXbppd6DP4Kwj1PZoUv7xEgR70uhJJHlj6zQeGAoehLrBHpd2mlsQb0g9qAanWgNI+rw0sAz9ogz+CIGfQg6EaSBnPekkYb0jMABAaBLyeH1pIHs/gmexLP4I1QDvSz+CWfwSRkPPbgl2IM/gkjUee9B26iXWexCZ9H1oCSQ4fFOgGt4aAO8/ESBLDbml2KQIA2BH4nX2L54gdbaEz8UMG88074ZmGm4qCM8Ug3pSQfAD09jvnQ5/BA6Bt4d+xIHge3g2YfWgZDrNtzPMOhPMngBt+dSst6AOrTRmiAM/kQLPYjAMyQI8NuaBGGHhgCBIzwPTRggDs+dGHY58iAwSDo1NmxAaBGDwGObfQjAMwPT60QWfnb7EGfwQAZ4abjmaHfkjI6ZoIv4hpB+H/wASQKmozSDUQGkge0febMEHX4baAz3ow/d9SBBsJI96XdppIF3pIM/gjNAuxB4h7HE92IPEDxFkEZpB+H/xJdiSsH2JZGg7EYICzzHehS1s80GfwR5AP+HrBGeoB6bnQkeniKXYj0eGZafYmQZ3aZozMwPUBFrMfnUAdn50efwQYZlqdiWeBIDM0gePJBm4aQAGeBoJeeffuSzbA9PU3qJ911Tbz6EB7EEsz2+Ig1nNjfYo4GpDOn7xFjZzedw2AHnTR64eRAZ4b8Dw78EB6b3ht7DRCQBuY+IkYMdaaN7NofPggAzw+dHmy+6H7RsFcUS8LttV3Uti45kb9yZ5gqrw/eACRm37tsFkbdltv0qKlAa5G7bY1v8ArEQ8F2OzOKNo3+1hQKkzJd74jvtl40797YJ43oIewbMHfOCNHtKsWZb9SawfpTIH9C5lW/R7iRs51qzuQkB0AB7Fx+1eLt+We/4FRCfH/s84MwXULb9KK33vAuehvU10Ot5rez/gWS0SA9xpsx/QfYZrDXkz3mtdQeP0SM7yl206ZAP6D2LcW3eFs3nD9Y25VWalH78OtR69Y1DuHNudEzE+w1O0KmfROEnEh0NehwJgdkhrAD/gWNuf0V4r2D9j1g42n/R5xmf8aZuH0e36a6FSsSqz6bID3TJ7FCjXPxssZoG5zYVJpjvz3psYS4eD/Eq0g16rbjxx/wAWJ4wLK9HhmBgfkMF6Qo/pOWwZNN1yDMgOn2SFoKxD4UX4wEt9ilPeR0AADBbRY8o6OsCaZgOB1r0HW/RXg4nKtW5zZd7I8gzNcxr3B/ipQfHfo4TI/wCLEPMFexj+Wf8AdpoGXzDNxtSAelg7oSmDZdDrAwwUsN/Qgr3o2zefQexJkHPaaimvMuH4bij6OB6bfQgB5nbmlh8U8YHgbmnsRGfhDgrQhAG7BI9f2bbmxOgH4jaeANihaPo4JkOtS+gzzR6LBhm42rc6v/uAAEAZ56Zqw5b8NtNHGcAtRGqNrYFpudaPmvgjeZMz1E08y4CAusc3Eg2eH2JrBwEs/ggkbDQd6EDb/ERB0ZtoB/vEZhmHRmvgZ4aepsSDYPiKwgD8RCYZn4eadP8AD/4kHiASgRzMwDxE8yeY6gOJ3Ww9oAJrmWPw1YAzMDS1nP6kebHvEGw0BgDh+PsRZ/PvX3Rb/rQYOe7QSGXjwSN5xADL4B7NJ5nWD2mCB3mTe9oAfoUJ6lUYzzfghn5w2J0Ndnr6EefwQRPUkQP2Sqz2T+c1Ozc94gN5NazfvEE3P4IO9NG84A6n+NIJOHtEDqHDegOYx5N6DWzQSM9iEDzFABp0PoQIHt6HWiauBuYH5EjDdqIMG+9sPrRngfh+7SAMz9okbLGl4bmCQAhgRn7tDh8UXYjw2ajaIwj4YI8/glhgiVAT1EGfwTx6eem4gxBHhGYZoj00OHxQYZkiWr5CpB7w/wBaWFSZ/pzwKQcx8/eKJzEr5FztSZqVcZPUYnST/Og+0l2h0TjSB58PoVhGo9VmdFNPA+9BX/au8f7caX2nu7375mtRAsOon48owBrvV9Gs+hh3vSfyIMCF4XP+AyrOHXrte6IgLoTNpNsh90o2aizzt+iNa9crkCBh1gZ70GcZjXM97d+AH5FcBRHHg1NPNUtY4x2BRP8AQFKk1iR8/QsjW+NN/wA90HKGxTaOH4WAHmpaunPW3LjR9ecbMZr5zWUrdyWHbzuc6qxnnQ9zhma4Jc9w8Q63N5657jmcuHY1sBNW8dNem6lVYM4nnPvVLdNn8e6czI/5P0eAGHvXjz/gWKuG57tvx0336wEZr8JnYCtXjgnMBi2LVhn/ANYBn/IrA6PUQ8e4Dhxs/Ie/9CyUxtEs9g54P3BUtan/AI2ea0Vbs+3HoGpR3NFrzvd6euSpRHtJijy2YAMBh0dap4dNPS9Yzqrz5+TsXm0YU9HkzqCR/wAzc5H8iQVuLMqgHOob3zxw2AC0DNbMA029gKDMqUUz3xTe+hNtsNHDrFvyWuVNgI2YdYAsfVa9KpRG/Tjh7D2GZ55qz5aCfYH60o1KowSgPkWTd7DPsUWi4KHdtYr0P79nn0dGAKPMCqg6bcSlSXg8+Csgjct0AH5wzQz7qrgDyrD4JBhnw+05n/oow+tWEaTUtjEvYfkVhJrzB0s4P9IPrNXFgWx9rQkNuVLk6eB4PGHWYL104ZGfzzz5nTg8LsWgpUNt6k6j7hg6314K4vapWHZ/LxbfpwPSGDzeMz2fkWNqV4Vm82gpUSLGgQgPPwg6/rRjcHTpUFlp2Ux4xh3u78FYUeZc0Zpo9f8Am8OxJ6pOHF9WttshH8gAgCe/jpsMA8fzq0YWD1wyp+DFSpsl6F+C891qt5Z+lSDrEShnGa7HjNSIDNSeI5dRlMgB9GezBWVStWs1iLHlyq5/N7AYAGaLVTNzznnwf0GQw8iketX5Lpvvv4AfyJRqbRoYaD9VA/nZTs+A5TeXnQZYYMYaOCCRMuGmhgbEFkMAw3sqPyDlV+/N8sH8Cj16fcd1Q/vbHMgx2Ms4dajw7SnV7Sbqr8mHHbDYzh1oHpLJzDwOovfrTNNNi1YTug/JPXPM+9X0m2LRgRc26rJjSA+fNTbYqtqnbh0Y6ODzphgciR1mfnXOM0zP9ZNa7Dc8/PvwWitu3qNWI5wQtWSy6HQ9nmCuI1q1IKcDEE2WQPzrR007jo9LCCYAy13yADAFpAw58KHAkG/XDNkPk7Fdw+Gll7NOdM/OadqVe5yYVDg1wDkdmfQf0KbGe5algb7BnIDrwBaItfQLSYjYN0qJm0rWTw6tKSevVZejN/cycMFy+pXgxMnhBl1yTDjt9jOeZqqetK6rwrYVK3G6rJhdjxs4ZrnXDS3t68purFgvzKk1+K7vWfZuGmw6Hyj9KjBUA710aybAuOiR9O45wG12RwDo/OtUdpWi9vft+Ma0iy+ZwKm1iuTGmgbo0aTNPoOPv/gWoe4S8Sa3NB+qxKaz5zdPeH5F2VmNBh7IMGNGa8jQI3p7Eb25rRG3N4HA2Cy+D9ZzmH8jy0sCj2zarv3SxzZw/pDIZmrN65IjPeoJ3VrfsoGH1qDaw9fMTPYawfI71qO9Jih7d9Yq9qrOC5afyL+HNM70YBLe9u/mj1dPXJBZ2N71Tya9LekA23mGaQU35FICA3+GiAgD7wb3DQcm2ZHm2atWQAN6MzzDTBWKo6bm1qNp2BGbAdNSPHwwTtKpUvQN9/YDh9bqhZo4bZ9Cmgy5pBsTT1w2rR8GJ0rmZB9DUfrVfJvC45jWhb9KCHH88g0Q0fq0wj68t8GQ+dVj1yUOAeAPnMd/BZBU4UeLMwfuCuSZjvkz2K1jTLfpTWhBggCtatmT+I1bmg5R22behecwzeTrNmZuhKrlck1V0PxlZ+uwMPDbTJzJXYxmoFjDgUqB7CCGalHJzLUVZGCpPD0YKFWLhoFBz9eXHGZMOwDzP9C8wL3WPNSMHD7wD61yKt8bGD8C3Kdn/wBYkf6i55VbkuOvP51WsyTa/CA8AXo7ncPE6zre8P1kcyR+EyzmsFcnGa6q3HNi3IvqqOfeftlzkGQZ9mCd1nP6l0dIGTzc8tSqznph9+qjAGAHw2AToAlsBMeEGgPP3Z7O9O4b0ZycDzbbPNMsn+P+sAVg0s3Da1DAATWfwRM4PBn2KwYb97aDE0ejv8NSA1MdPBQgAB+IkeGSezwJBsAPZ5oqDoJpAng6dRHoDPFHnmGDnegwz9okZhmgMPah0JH0JAf7tBm5irBhJcwPT2bMEs/gmetGAHmgNLDM/ICWjm74jgJYgoC8P3adwwPxAQAH4aWtvVhdCPAwDe2gPs8uCQahlptoF1mkhS3oC60YajLqQfh/8SDvQCf4n/Gl3pOdXyJZ/BAjNAjD59iWH/q+RAB6hpAjQIgsPil3pYbc9T8iSAzQID60aBGgwzSM0YBs+hUyIHnMAbcPYCWxB3pYfFGpH1pJfu0jBGRfnBLu1EsPilsUtSNnb4iR+y9okhPYHiKguv2iAMDI8Ow8EfYlhggdee1jz096ADc94hA3M/kSzPP2aB0Pw/8AiSZA3gzb8maDNsPEc6Es93hoBz+CWe9BvyR9AmgWbhml0IA1MdRI3m2fElvsg12Zqdg2Tw2J3P4JoDA2jfAM9mzDvQQ9TSPUivRu8M+9FprPjNeJs+tDgGkHnQYZjmbiD/g6zUIPAeYpawZgxmGYdCWjgAH2GkYNmeoYb1Y+H1ow1EsPih/PgixdaHvSPPJJECw2JBqe0Sz+CWs5+RAsPijNnwtTsQfvO3yIc8wBxtvA0BGAMkGmYHsRgeSa6zzRZ5hmgHPdgiBPcyDMV1g2APPvw3go/YqBZ4FsBJCnUAd6Xel+8SBSB3gizDDD50njcMk0AZkgdBHsQh0L4G8UBmgSPrSQGln8EJ9C+Bpn9aAw0zDU00YaaANgaaWeZIEe/NJLvSP8P/iQIz92gDDPxEZgeaWHyb1APz/Wgw+KAwRoyB0AmjNz86kMng6Dh78E0enr5tq2pGYe0fDrDsRHpn0IXjzwSANiLF1sBgjZ/f8A600f7tF4mPiIHmT0c9/WoJnyxgeeGZ4J49RMmebWmbahBGGB+0TzMxxnNvU2Gome/wBmneWbNrUVrHm4fidiazwRGDgHp9iEA1vDUIEZ5tagOb0IPBinWYeffgly2BezRYAPMkZst96DDDsR5v4HnvBQAhvTqU7zdHnPQJHY6z1rfUT0iuK9EkNc9OCsRw6wlhvP86wJnkh/WsbgehaD6WMV50wuC1ZLIdhtHn/kXaKPUrVvClhVaa+zMjv94GvCSdpVVuegzQnW5WZkB3v5d7DNRgeyLh4UWJcLTrEtjAz6Dw3guY170aQORr2rXHoAdmD2azVB9JC/4EdoLgocOsNN9bzIYPLrFsccuHNyMA2/WfVU3vjzgww/OtBzJm6vSX4XZxTpQV6EHRIBnP8AyK4oPpbymcIt22UDMjzgeC7rmwB73wP52TzBZK4eFdgXO67KnRfFPvXOAo928JOMEB1iq0eAEs8PClmAH+Q81mrh9GOjPNGdo3HJhu+SQYPAsvcPou0p7OVQ6zJjSOwHt6hRqx6QPCJrXluBcNKDz7zD6Ni0ixn6xwZ4qW9mb9DOpRw97E3gsozUmI2yW2YO+QwXYIfpexA/0xaMxmR8hrS85YfG+nNG/WIbLr/uTDCT/vsW2x599axPxUWbZrvNV9Fq2JMcAodcmQJH77xgNc0r3ATiVbZ58oFVj+eJ1/oV7GXB5sGtNxRzeDPYohvYZxJzBxnmz3gYKQyywYbDQSMAew7EjDR8M8PyKE9Db8/60uQbB3w1aE3NgwPDsTWfwUfDAtRB9xMvHMwNQDN5JmSAH4iAAi974HmgNkPdqwnjiPZt6eCazbDw3EgBgy0zM0ZhgfyKHmzL2DIaiEPGazBzBE9oGGBt5pMg2YbHAz8iPQg84GbeuBqRnmPtFHMDzTJg5+OYKxODYSdM+xxQgzMfaZ/Wl/OXY4H50DviAWDjmYJp6GeWo2/s8iIHpfvGAP6EucDHDQMFYENTvQHvLTz3qWy9FM8DcwP507ycQxzMN6CIDx5afepGtgIZ96T0Nx4fDNR/Vr+OpqZ/IgsA+tAfWhZ8iR55IHQz943mCXQXs00bx5fIg1nP6kEgwDNNGy3mg35J4EDJh+H0IzBvSBvTQHqB7xHn8EDXLNmSA4zfzqWH/D7zBLr8TBBHDPJO5uB7xGYAfs0GH4iADeMOttIP4Eeij9jmHnQM4eQzTwan4iZBPdiAvE94ln+Gmsz945mlvyQO5/iJdfs0O/8AIlkaBYOH4iXQiTW9GQ9+KSW9JE4dIZs+pe/bwVgzaX7hU8njlQ9c2KPR5NV8hgGw1WVLi7xCmNaFNtwIDXzrnU3sa2H429tuMAed1HPqVAojWvXLnhh8jO8158ue6r0n+HXLxejAf9HZe2KnCSwft52agdjr3FoJPgWPStZ4OuXI2ACzh3bxGeH79xNjMu/hRw2AsVG9Rn7Rh54/kRvG/G8SDTgjfOYLV0LKq/yjXORthxikgAfjBgqf+SWjyXebuPiacmb34GlUqCcyGDnNPZn171n2dCBUQYcWTzDZyYEHk+V5oHvIYJqlQ9HD5Fn5nNaunR4JvSOzBXDMCcEDXqpmy9+EBrza8LJ451S/YYIYMbzN1VTwBPw5pgAw7wUdmsP48jTaO9pB74zRsz+ckcoxFPNRt61FvMzoYOuUqq+rWj6z71X1KHhNNxx954z7zQRplYge4DD51CmTJ0yRqaa9ebABxYHiOMMn9YKaFyPnH0GGAw+QFTzIBzDBxwHgAO9XcONTThBFlVUIwAeeDIb0WjxqDWKlF5sIIaWfRngjOBF0pfgMs6HXgfWqefDnGQcpVTZPPAAz2YK1kwJzYBENzM8MzMN6yWrKVR4oQzqM6LJAw968fQlMmMaWhRpZnIPoUsHmzaClSopvMn14IJPI02aHqqlSTMO9A0zTajGgG/WGKkZ+fPrUimw6bUoAS34MwPkM1LjVWXJqguP6x/QCuK3Dt8z5GK5M53vAzwWqGch1ujRj5WVb4PAHQYJmsTHzHnqdTeWhfQtHDokHS0zb8UEgwCZyINgZ/OCLi3N6rWKzXs6NRoujEPrPDrVtaVHr9HiyzqUXwgZ2Pdi2Fbo9SksOwYNK0Xf7Q11gmQOuQ4fq6oznj84Ggz8CewAmD75nvX2TJfOZyMVh7VPv8i0sajxXmtcKlGZ8+YdCdjURyS676unA8YdZvbANZG0e26Cw9K+91HA/3ys9bRM4rZ7EDPDq5694j8tlkPwQBXcPhLdQH/NwQADzunmZrSEMjJoObRv6kkP7oFCCHVYD4U2lRfvHWGZrcXVY1+U2GDdVqrOk/wC6iBvNZqHR36IHNm49zAefsWgD1PxRANT7nDa73sw2IKxTX6a0D867jkyDDM8A6E1XpL9yNNRK/cB8uB54AHWmp8am02l6EFsHvPmGZoNVb3C6dWI4Vmoy43qzvdA95/QrCYzblvaR2xbDM9qKfjPSD3mfkVbTWeKFYp0egWrRpjNPYDAzkZsgrqj+jrUjwl1y6ghmHuY5oKqfxIqtb5fnqdyYRTzzae/2FWVipV+8GvVtOcqtSw6I8dnNduoPDGyqCAOBEOpSA99LPNapnQhh9xgw4bXkjsgAIycKtvhFfb0LQfozNEA/6RI3mH5Fs6VwochtcrUrqemR+9pnZmuhPVLR6zAFXyaqwHvwQDAt63KPH0KbQ4zPnPrM1YcyeHYs5MupiN3qkmXO+97B8w+hBsHqq2HvAVFVb2gwJGhqAbvkBZwzlvD7RQv+Gm60oDf60Gjeup+SGmw2YKkM6lJd1HHzNWZxgjNG431gCgwHgMM3Os1AkGy5ypmrCHG8IE0DLhqwAMUGfuSNKC7aO/oZiEYwV3DZbw1FWVuTncFPccc7DDBWrMaWYez0Q+dFj2Ams8y2IJlw2VR9Jiq3AzzB9bLO81VM3nKkzcLYoZhH/t0jo/QgvTgSnvIH1mqet3baNtsG5OrLJusdbLO801PgVmsO6lZqT0lryZ4J2m23ToefKUYN/ee80AT7/fqscINh0OTDmmAGciczgAIQptzzP/aC4zmfIzsBXXLP56emAKWzDD8QFYpAoLDIhpxGQPzqQFKfDyK+Zhgfv2f1p14INNa5uqzmY0cOt4zQZ/1a4ZqQFBbPe4qKq8aeHNHPCDKOsO+SP0frXN7k4zXdW2DiUoPVTJ+TAz/WtR2qT9n6Ixr1WdGjNed41jarxjtGA7oUNh6qn+52AuLmbk/xKk+8875zMzRMssM9AINRXuJ18Vtp2Iw4cCIfWEfr/WsuEP8AEN4/ndPM1LZ3pHsRia8DBF4eOohPDJIzDZpuK8LO4BpZqOkAOYZ+dLwwD51bzZ3NsEs2D2G4Zn5AQsnn4mmi0WAdN9HpYfho2QcMcOxHJD7uegiCZm0Go4YHh3qA0cY8Dw/+wRxmTZHxHze+tEBn7NH2IgkkeHxQIDQI+xB1osu9HrN/1IMPilh8UCzbNLwAw1Ef7tABtvH4aIH2IP7vrTvh46bfWhDYHiN70WSLoPND/wAL2GHh96WeasEhQGen+tGG/oQEZ7/DTRhu/Ij8Pyb0gDegIzz6G8AQshoibgOGl3outAkObgIsM/D1MEsMA+dAv7xIzSSDoNAKDDejQH0eGeCA+gzbSQZGlhvRBJ14O8Og010CCWfwVBZ4ezbS60v7xtJe4AH1pJAkALwI88M3PoS/Olm4HhpBvQCiQn1h8iJAjRh1po0XZ7ToQGfT4aBIEYIAzcPyfImjDPrbzR/8PWC+dZoPvidaWfwRHpgJ4AhDfggLMMfZpAl26iQGgHP4JdfvEjBBhn7RAB782AMM0MM3Aj/ewwMDMEWjFMjc0N+eGaE9mYP4YIEZmBm4Bhh5ET0WDJaByU2B57AUdmMwBmAN9Zo+QDVAwcPYeeGexS8hLZ0waBhhvAA7ERvZutBpnv7/ACIQwxS0dH2bhn9aPRPSWIwA2+eCPPD3ftEgDJGZ4EiyM8D00gPemTPxddz86eA2zaBzPYaM8C8T3aE/w/8AiXwDb94hMMHc+xGgzNLt1EvD92i7EA4ZpaLhikB4CbnevmbnnRnsITN2hoGZmpB6eAbN4H1ppFn8EMBM9o/Ii70jS7EZjCSDIG24wB5h3oD5vAHGGM8zwBCe/wB2CdB7D2fX51AB4H4x6b7eDobDBLP4JPA+947m/M95ozwD2AYAragQ5/BJFsyD51QaM8z1EYIv+DoNCAZ9CkOgkYeF4fQgSNQBM3M8zSz+CLekYLUD2Ig60vE0vkBCCkO94OdiBCH4f/Eiz+CAwPeCPPefhpnP4JZ/BAjPejz+CWeY6aAECM0v3mohSAPFwc6MM0BGaFF2IQM2fEQEZgZdefzr6BtnsTJ/u0GBm7s2YI8hLwbZ94gzY923vSA2zQ4OAaPTuzE/OCWDZgB+fsQM7/D7090AgWGzTw2IA08jYDNH2JKAz/eI8MPZpJYOYqw09qMj4aLmXAwzBHhmGGogNnMFARnmeCaCNh/Ssw8hoDDFWAM/dfPmgEAz2bMPOCXIbz00mWWw8PoBO57vDQRDZAEvHMPDBS3g1izQA9gOnqb0EcObANM96aNkHi8cM1L60jDM8EwLC3rzu22HQfpVfmADf9HM82T/ACLrFt+ks29IBi9aHo59cuCGwP8AsVxcGcEnozZrmuFvZFEu23LqhhLtyuQ5/wAjJ71cAy294csM2u8F4ZjQ3IcgH4LhxnfOyeBrfWxxpv8AthoIM7CtwvwpZ7/1pgd4uHgzY9zuuvzqUGqfkDBcfur0e36VIN+gSpOzoDyLa2rxvsSsTY4TnJlBkeR32IfnXUIExiqta9OqUaez52jzSB5no97cYuGJgw5zkmmMe5l+MC6LbfpOUOT4Fx0eTDdDrMAzXWAokQw8RtZK5+CFpV7x3IPJyPxWlIvf/tbcQqcD8tilVID73WQzWCr3ozW5M+92rUTpph0NdYLD170frqprrtStKq85odDWtgf+NBROM3Enh06ES46NzjQdfMAef+NBCuHghflEYecbg+smQ7GutYQz5N04k5t6M6HY6GBr0xaXpG8Pbkf0Ki5Jo7v/AFjDD9a30yiWPfjGvOptKqrX4vWf61rFjxVqMf1JMgwHtG13+5/RXocyZzdq3BJpRn7p0NYFyW8OD/E2zNV+dQ3p8JjrkRMDBNjOPU2m8lzVOnBn3sunvVfn8FCemMA/oS82XfI9sUjDNWHQNzPUbAz+gEZvMH7Nt4DDrzBRAqtVoMznqVL0Xfo61Ln3JUa2DTlSBk3Q7wDeiAHJbAN7gAgAwwUKZJYw8cDMEo0ljHYexWJpmmjPemvWVOPw89/f8iIOgDb6EBg82jzbBAewUGtmmBLDU8mxHn8FBA8M8HEbJ5l14IJABrH86kcyYDpuGoTx6Lo96IzAy8NBOA8kjCWfs3FTHAfed1GKkbJ/WvuFcjf9JAasWv8Axn+Ggz+Cr9auG1nps5/WpcbXPw3DA/nBBIM/0JeHjsbwUfNz2Zt4J5l7DPw80DpgADnmCADQHvS7ED3X7xFgBjpm4ovenev2aA8HGfDNJMmZpAZoHvE95sTP50ZvGftE13oJAG3npoD1O8EAdaNAfuUAaiX/AKw/4fkSA0BpdCDr9mkgSQJM6eeDnQkeAHsQGgSzM/dprxDJHOdA8UGetnswQgGHhttp3/Gi2UP15RGjcptZDV84Aru3qJfdyMA+/fEaNqdmCr2WX3nQYbbzzVnJptShsa7beDQLhdDZQ7Mo1NYPnghz5f4xngnmQsOMIRarFjHh7lkMzXPI0l+TsbnPB9Cj1Kqv01/QcqUk/wA6NceHQ5lyUaju/wAzWqyHkeeBZGqmFSkc0+34pqtekuGHhy3nvrVec98Ot8wRGGoZh5003HJbOHY13qhA7cOVys7DmPnVeFencxoRG9/nMFYTGaVpR3DYM5Z7zwRbTUesQaJFkNhB1nXwwAMFWBR6lPlbG8M1EZMAdBxxaavXtKuFoKVBihTY+GB4df61Yjhw3vTDUCdAZZ85mpFNt6m29KCVXL4ojOHWGaylSjUrLCqzpgR+/wAY0Fq2xwrr1bagyp0/A+88wBc7Js696gu10JdDrgPBFDA9LvTVBpT9EdCq1XlmY/ebq0X8j9pUqOblq1IzkH5DWKCgxabIdCpVLmZHfv2LUNVisMTKvoMb4hoDjU4HWjptON6R2YKRAsBh4AqL9Vk5/wACs6bQa4er6qpT0kw7w7EWr/VtSAfHYgMn5OtLNxmLyoOYGadmM1WlSNCq0Cfn8gZgptBpVKueYER+DPZw3mfYAIo0zZnLQ+eqtwYZ9DLTO80YAxPaAGKaeDfefeuihaVuRpDTDjkk2j6zMOhBUrSsTV0PWL360NuNVKe+GbFO+7O54a3Wqqfb044/PTn6lJMPe59C7rAtKymZGg3m9n5zWlh8OusG4Icufus9iDzlatpMVh/fc8kPIy0eea0Z2ZdUYz9VMazXuTPrXeo3DGnRvEODy390ClhbzkYwNhsAwRk4ZDo/FDleVbg9ffmng4aXrMaB+o1WAEjvA12p6M+8e9gFF+ysHnebfquHyZqxzH+TfRYBysPxjPyNHngtnA5VmO0FKpUYDAMM8Fe1ilQTgHoPyXj7ABZ+3reuflcOaZjNeSR1qBcQAMHQccbwUKpXtFh5hBiPTHQ/C6P1qXJs+JUsAqMp4w8gKyptt0aiNAEGCGf4p7zQc/gXVxJrc0IgUpkPnMFFqvDHiVc8836k+yzHf6Hj7F105LgCgOZs61a2Kt7gbZ1HdCVUnAmPB1mGzNa2HbdnU2RrwaBD1fO8GZqFMrbEbvUILkbe6EZNVzJn7M1FemNgXiLGfaSpPTDY1MAUKTJlcwfjmidtlJr0GN4YHmfkVVJuScYabbGALPwwzdzcU3DNQ0NSZ86Tnm+aNmM5y+DhmaTwYJ1k/CDzqxCmMgAajhpkAz9mB4fOpdS0wimb7ewE1DeiHFNzPoBFijMnimtFznWnOxtIKkwCT0xxnFzAAz8650LCYbfLnh5FWQAAyBhxUVS4l2jGkcpzxzJHkiM5h+tR6rTb1uSY19n5bMBrsew3o1dDBk2faGAD85qvqtyU2lNahy2XvkaPNZSBwfMJrUu6rqqVYMPc62ALRwLJt+HIBxinAC1SyNYerl5uxH6BTno2iftgPDNTYfDS6pLGpXL4kh52s10kGdiWigwkPhcxDdzCsMn84Rsz/Wr2HaXLNaHrief1mC0bLO5SMPigz/2M1v8ApypfrRhZNNB0HH51Sew/6ygrfEuy7YDTnVkJMjsaj7zXPK36Qk6Y0H2Vtx4PO9IMFY6wzQaUz5/+2krNV6/+G1vR3eaqRyZAf0ePmZmuFVu6rxuqZzdwTntLsZZ2ACghDY/A/WtWLcXJxXCttAFKtwIzPneM8/0LCzIzlYa0KlLkvNeQ3tikaIB7RSA2IIMaMxGaBhgMABSAMwMGzbP8gI3tMB1D6EASXPd5/nWzzZ7M8P2QwPzmge0wMMN54JBrnilhn4mnvBDZH+76kbxmDXQmWWX3iPXzDyKQDPebmaPUX/s9/YnXgPZh1+TzqRgGGmjwDPUwUCIGx3Dv8idONn4h4fIAJ7D4pZ5irVsgAABElomYaiMAUJADwb8+tCYNnvSAG/w+9ECtZb0ebeKNB3qEDwzwSM8PDXzNz8i+9iAAPPY2lg4B6biDo6OtGb2fQrWR9Wmkg6zzSBAaMGW8DfBsAPDoQYf+v50g3l4nQiCNGB5IvD600fWioO7DQbM9QHM196CNFiChJYOYbADDNB0GjePb4aEOhWsGfwRhnnqdiQbxSRAjDbqJIM2/Zp4MNmogaM0uxIz3pIF2JIM/gl4meHeiySz2JYb0eGz5EAHvFLvS/wCDoNJACSEz3pZ/BUzwJCf7tI0sNiMyz3YIs8EYJo9i9tqLPZp9iNAYYdaQLwI/w/8AiSSSQAYN4aht7EYJZ4ChA9/s0ZPgG4Z7A6EMl59kA02M8zUg8w+tNG85qgxpmZn3o1LegZDDP68074hghAN6BeIBanzpZ9aAz3h/Gj70BazhgDHYHQosmMEzwHwPDNOmeBh9eCkGGB6fWp8COHst/Wl3ow09mn2JHsV2EZ7FBh6jzshx9zPfs+RS82zUSMGD8jxMzM814JBngJ6gYGB4JGYM5udYAhCSw8J/IadM4ulpud4IEzJYed023wM1IVeyEVnx9AAPzpPSeQYN+K/mHfv7FIs8/gmc3DUeNMbmYaDmwwTzMN9mZqetTea/B7FSzv8AedCP+7UTOVqmBgGkHQadWQPYl1mkbwAeHyZpAeKsGewUG9I9/vNmaM8w+tEFh8UAdaWeaXeixpZ/BLrQdZo5zvR7RvevrL2Gfh5r4Z7dPyJoANQEG/vSMNjRg5v7wSxBHn7vTVtS34BqIgPYhz24IEDvaDiAOtCAbtRHn7tQyIzSA8UB9aR9a1aj6EGeZJdp/wDF1oIclh7PXgvMuh50EjW6PkQ5uGaLwMEOG9AAG4BH/wAbaM/3aRpGH/BpajakAjDegSQGgz+CXejA8UCz+CID2IfDz9ogPrQGZpb0CQGgPP4JG82A6jjmCA+9xH4bw6D4ZgaANbBAEnAcABGemDvhtpIEGnvPDeadwPDUQhvRYbDQID0S9mjN5M55jpp4DbNoMFAMDb9n3ouwG+9R8HM9RHvQBv3/ACI+3UR/3iD+73grAHhh4biMHmwwbPvS/eOdCMwbM9QG9iAj1OhtNeOGxs0e9LvUBBvLxER9ajmyZ/IjDUwADPPBBIZNwOwE0HjF4jeBpAYJZ70AHnkjA0sgRPGDIZ6exA14nu0ebnvFa02HRqrm2xXIzLoH0Go9So79NmhEOWy8BhsdZPYi0QDxSzzJIGcMHDRhpogvAMtTQD86dpVbr9vT2qrblRegS2Og2ehNaP4aBk80HZrV9KKss5xL4ofORz/pEQMDBdNt7iXZVyYN0a4w5g+iPL8E/wCNeSnmdpqIEbM1jcLe5TDD9qYM/oTU+BTp8flNADA+sDXlK2+KPEq0mgiQaycmP3tSzMwXTbb9JCgT3Qg3bQzpTv8AaI+ZsqMC1uH0e7OqWbkRh6mmfeHQudSeFfEqyZnPWdWHpIB/YXsDXpCm1IKqxzdDqsafE87J5p14IJ+JOiaL3ebJr0cHonpJ8RrYIIN+UN6Th714DZNdTtv0geHlzvgx615CQ52SA2frVxPtW369H5WowWalHPsPYf61y25/Rjtzl87YqJ0qR+DI6P1qh0+6rMg3PTnX27VoNVdPoMwwP9a41/6Mdfkum3TZTNHP8KWesH61nJ9vcZuFcj7jOmPR+yRE8YFfWx6Ut1QGmol3U2NWA/GZAGT/ANtWMJcnCK/Lbm6dcocl6J/a47JmBrP3DaU63mo8rUZkwZQZhIDYvXdvcbOHN27ItZCNIDrjy9isrksOyrthcjWbfAw88fZ/Gg8PmzsS0In4a9DV70V4p77VuM4wf2eWBn/GuT3Jwl4jWlIP1zbDxxw65EQ9YFaGRCHEA9TQBHm2yGnp4YIznsGGGwD8iLkwe95mgj8y2fhuI9iMIzCLlg92jI0yAB2bAR7Mtjad0cCSMG/doI5pAYJ08MU1ogaNSzwLU7EGYH0J7fpaBuZgo/LaJZoJAPOAaLWby2bE0l/jQSzNs0Bm37tRwPdpowePJWJAb80s8x09maZyBIN6A0YGmT2b3EYHh4jb5oD1g+f86Xhn0L4b2fzr7+8QLE0Wbns0s2/eJZAgMOtIzQAbgIw3oA2IUR6aHsQJEYBj7dD3r4YNmGm4gNGo4Q2wz0HzA/JhsS+9h4bmCB3WwSA2zHZ1paPvM/yJYb0ZEbx/hgkybmCRm3l0b0YGgDmdFg34NKk6wKtCsXHPn6E6LJ0j7MFIMLxOQcqIx/GnaxeFZCl8jO/aO/euF0JHqHnPZywjfWqdm2KVMmHKqr4PaB4AALFTKwZu+POMPoV7AuS3IcUGObwWCttgzDpoNeHEwaD51WaM57xIMUHg+dcvqsy6nq4cqgBJkx/J1rpdhz+IUyEDB2w8yAdbpxti8W0E+gwaVQ2vupvVB8M9nYskASoYfe4p6vkBdSo9k1Ktmc6szgjecwDM/wDGoteCj2w+LFKqpyZvzrSLS54cOuaQSn4PJtH+Ke9RMHAkAbjma9HRreocyBHcqUHWkYZnvWfvC0qlPAINuU2NGjh359a0HIY0+VGLwKNzK2FKtiuV6GFRctjCP59Fa6xvU9gR3W7qqNEOQ+eYaxq1n8ZrcN/1VSmweM/7J0LJ5tRfZK458Am+ReZA/OpFN4dUqNFByU3nL71oo1yStIJcqmyQa+hNVWZc1YhHFoFvzAaP+kGG9EbVTNn3BJdCCxUoECn/ANnAMzV7DtWpQ2sKNK2d54daiUeG/Z8IJ1Z5k3e/PehrfFSnUpoHIvMh8mitWi1ZjVjLTly96aeoPLVHX56Sf+ooUO8KrWGgl0phk8/kTMyNelYz5RxlmQfRmewFbxZvSYkl31ODZnIPsAFZBGtiHA0JdOZP5z7Fn4djXizManVK540lpv3LOzNaD1CEn9qUMmXe9TPSOaobfJugftTVtGu2qwMNSqvGHyK6ZoNKjDsiApARmGfZsM/oR5tH+1VwVKFrxJcnPs2IYc+5pLWhOcMA+cOtTeZMPaGlzjfe+CtEWAIzYNYJBAghv0N6jya3BjHp66pZ92tgWmwBgrbYav2I6gJkHmD63AWK+086Tnp5qP67nGW9w/0KEYbeTVadGDUN8FAO5GPd71lMAMtQ07ye3w0ep0m4ai9n2KCEyU97Rw80JsuJMs7UVAJIOGCajeDvwzTrwHn4adZ2CoekyDfMG5p9aE/aoM9xuJo5IAXjuACt5hIA8U6Dyp41SYelaDef1pqpVWDGaM3JwAah6unnmz94no08A9m3msId1RTd5SnNyZj3yKwptNviZg5ygQIh9eZ70FrfNV/5LzmG8NU+wzwXJ6bdtVOedOb1sP3QZga6wFgUZ6aEqsuSZPnDPrWmgQKNR2AYo1KZZw+RHRFuUs0HiTcIaEWJ6qj/ANoePf8AoV3A4RU4HdS6rqqVSd/CzwBdFN6W8e9Lk8/EXmGNqqm0e1aOIBQKHGZw79HerMNcz6FKCN8ie0QD27mAeczV4No2i5/WpIagDp6aoZPEKw4Eg2Jd1RgMOxYqvekDSg8C2aUc8/xT2LbBt1XTc9npqqqV1W/RBP1rVYzLodi4FWL54oXJslXG9GifgxAwBUgUps/ElPvPSPPrYKLhk6rW+PcECODbkV6Sf0Yf5FzqvXhet1bKzVXga/CZ2AogRsPZo9Fz+tXgR40NiGQOA4AfOfWpEZlgPZtoDZUhkMBWoXgAJ+H1oetF4fvEjNsOxehrBw/dpb3u8NidP8T/AI0eYAOm4gaNLRbz1NRFh8UjDDwNP86KsYBsSPoRgBoA3koSDPeYeRH0eI4aNBn7s96tYzP8NI9T3beaDDD6EeGxEAzPoMEYH+tEHWfzpd6gHiaaz3p3MwIEsGzNA10J3NvJIN7qMDBl0222w6+9WFnglnn7tLZkg35KAj2CkB7Q8DD5EHiZaaeDUw3v5oGc/ghwDPUUjD4pnvVrIAR/vEZs4IHnsA8Q0B+GG9zrSzc37PnNBh4u9GiAAefiN70ff7NBngWmjz3o8gvEz2OYIm+n50HWaHoPYj06YYoUs3NLBw88Eu0NNFQSLWwazwzPyIdmPeZoOhHoAM9mo2j8T3iPWcy8NBm5nqGjPZBvAHEt+SQHvQPHgj0v7tJIEg60B9ZpJGgDUNAg60kvMl4ePz96AT6w+RJEmjQGeo8YNt7NNLP4JYfFIECSPp00j0w8RLxPeKggBJLsRIEYJZGhBLP4IyAZv/ifwIwPAkHejwcBA7q/rTXiG0THn70ksPkRqBlnlg09Qz+c0efwSQa2BIEe8tQ0YAkCIzwDBAJgkHQmc3Dx1HM8OhGB7/E6EBn+7XwzzDTQsnm1qdCDBx4tTXMAUgwDFNPQ4knVBzPYGexOnp+7czUTls5RuahgoBsw4kPPADMzDfvSBliTFOJgYdiMA+7n4h7D70mT2KwQajMM2H3ANrvMwVaFSpTxnFcYBWGYTGjbMD6+9RPU9KMgcciHs7M1QmnysBoAbDAOhSDPowPsTRs5tafQgZjaPRngpWkAbmGm5h+RIz26aQd6A+tA8GmYJBswNLt8PyJpl5sy0z6/IkAswAtM08B5hgmXujTbbD60gRgM0jP92jMEyf0ZoujwefU3IASSz3rVmMDby8RF4elm2o4YZ+Ij6Fk1Gy9olqIjNt42jbYw86EAbP2jiRgfs23EDJm+D+Hus070GmGTndEvDZ5EWfwQPA8+zmw30H1pAbYeI4o+D7L+evm0YdBh0J0HsHQc8iAw3+770Bmlm5kGmkgDvRm8Z+0S6yNAB71QWGae6EGfwS8T3aBf3jaRpZ/BJSEl0JH0JBvFABpdiSPw8zNvv7ECMMxS7MDQJe98RAaWG7NLPAdTvSzbxQO+ADWxxNYeL0IDwwwcRggANMPZhgCe61HwwA07v0vDQIzSzzSw2JAgDvT3egMEsM2tM0DoHhvb70eeexREPiZAoE0AAGvaZoGTcCRgx1n2JoDz8ToTvQYOKwg1DM9dvBPZ+FptoDPPNxxIOnUUBZ7EbJh+Go5pMq3kJvhmajh1m2Yb+zej2YpYAHiYb0emeg81IA2zM3EIZh7TeCZAzy9moDxhvSD8P/iRAeZ+J3poMw6w/WgaegNmeux4LvnBAy9UYw+O3zn0KaZ4oQVrAzMbmBrt5/QYIwPbppHhv018AwPw3FigebmZ71HzMCBPf8HWaA2W9LUNzBWEBuHnmj7EsD0tRtAAPgW9zYgkb89RxMvRohn86+maQdWogKBMqtHd16PUpMB7ztH1rrFB9I24IzAQbupwVhrvkBseXJ8/gmjUXA9V2rxI4e3m0D9HrnJu/wBnnHon/GtgZzv+rTI68ThGYeDerW2L/vSyZX8x1+Tpd7MjeC5sMnsgHqV8jJrOV7h1bNeaD1lbNNk+SQ0eBrnVsekhQ6w6EW+KN6tkd8uOHgmuwUeTSrhhBUrRrMaZE87S2hvDhlyejMDz5y7VrAYH/RJGzD8+ayr1Y44cK8IP85ckx0axm8Br1K8D7J/zjE2edkED3KvYMNzgMP7PL6FL1xW2PSogvNAFzUc2T87J5/5F123uJFs3bSzftmvxpOfWyf8AnBY+6uDlj3I6ct+1eQkH76ndC5PWPRvu6lP+srVnBUmvwuZ0XgVRY7m9YFv3nINi6rOo5tH/AEiIGBh+hc0uf0XZUbxLHrhzM/6DL2GH51gqDxX4hcOp4RZ0WTpfhS43X+ddqt70nLHqXh1k3qa75HjDBXsef7nsa9bMw9eW5MBo+h5kMwWdjVKKfvF7yolepVwwubpU5mZHPrADzD86zV1cHOHN5tYTrfZjPfixAADTbPDx0cxg+g0g3rtFe9EuuQ2jnWjcDMxrsalmAGuU1u2Lns+ZyNz0OTD/AHuGYfrBW9VrwBhp4JnDAkeswfsH8z+dR85WvswP60ZDPPDw0HiGOadDU3tmkAI1Rz6NQEYH+7zSw+Ka0fF1G3DBAWfwRmZgAOaZn9CDP4JAZh4epsQPINbBAa+n0ICDeXiJdBoM/gj70B54DqOImZP4aQagbwwMPImvyKxIN7ems/w06zp5eP0JPAGfgdChkDWRgZprD4oMz924rEsz/ERBp9iihqH1p0DwPBGp3ww2OJGzn8gedCe8NTUBAbzmOm2gM2XGT8NzNFrN4+O5vTWZ4fOlh39yBH1owNAgBEbSDNJCgeDYiwSTr70jXpVYjMxzVFVaPXDlHrhzhn1m0uh2ratonAAKrKk/3LILS0et0O3tWJQ7cMzDrdNcK3J6bwol3JCBiDZzwZ9ciR0KRJ4RQbYkeqnwB6QHk6DW9uridVXgCLSnOWAOvRRvQ6lXqHE5piSDz/WeCzwpX02pMUeODD86lQD/AHTO9WFSuH1k0LFHrIHn3nszVfP4dQYEc24vjVB/rN4OhTaDY0VmB9+fkhL8+a2w9PVWlXO9Tj/nKGH0Jmj2lXGYYHpwz+s08dsMRj8DnHvoNaCj1WDDihBnRZOsHyKMCveu1i3ouEtvmZH4LRrCXJeB1iab79Vkww7GWexdI+xlKrdWOdOiZs+RXYWHbgTOaYpsYPyZphi4jRLSodekHVW3KxUnQ688wXS7StWLQWgnMUZmHh7ED3mtg9QQk+Br6LPkZDBOw6PEhiEVvM2g6MzTArWY06TIOUZn8gdisDk3HJinBDrPod8imhGbDo6E6yy2C2eQxQWNWOd5uo1jW+TBXbNh2ifj1WnHPkH1m8rh48OtQXq9TY3tH160SwptKjf6Og6KdPXD2iy8m9oIF4esor15znj1GF5tzthrGCZeqTDJ6bj4AsK9W6rJ9ooh65+3M/1qWrdvXDB7HwNVz1z7jbBxZoAzMABWAQ2/aG4ZmoBya3ON32iiPSZbx6muaE43i6madAEeQAIxgeojNnfqaal4Z+HqYJaOHh5o0RDBzDDoTLISwLT0ww86nPdGDbgIAebMT8ToQCEZ/PxOhWB6YKm9ZMfjgor1YnSXeUpsU5J9+HYq2Lg5LCaOZED3iqfUl1H47gAzn2GYKqO3pZyMKlc9NZdPoa1t6kXsyqxA6HFVTKw57g1XvUSvsyMKPB5l3sMzVnD4aXHVfHuCqsw8/dM/6618cIqfXE4z02H8zPsDrNVtSrFwavK061Xpk3s1V1Kg8OqPQTB/UOTI/FNaAGQjezS7HH4dh8Ua86D9VOHR2j8j29aiBwipsZ3mp1Snz3f3r2xb3Wc/qX3WUYFDGtuJA8OIARvnAFbBDb94ZmpoM63u1T1u6rVt6OZ1WsxmT8me9eiXybKLk2w92uX1L0hLcB3Ch0OTPPsM9ix9Y4wcSqq06xFfCA0/5ADMEHoOTJpsCPrzp0aM0He69gsPXuNlnUrZStasO/8AVA2Lh7MadP8AEqtYemO+d54FKZgNs9Bh+taudtp/HK55MfQo9DCH9YZrGVKvXxckjXrlZeMPIAYAjww7w/Wln8Fq1V4UeIHYpAMtgkYOGXtEtH94akSAMPIgMwAjR4bEOjn4biAM/wANFn4XWGaXRs2IwDP2itkD5wSzwDejMDM/ICa97p9nnRqZB7MNRuI9+fYnj1EHMmBg2YGYH5FIwbz2I8gBm57xJ4/C9h0d6N4Gwd02D/Wl4mehqZoWADDyJIzDFLIEelrYBvSQYeL8UeHxRZeIDu/oR4e8QGGxGHTp9iAv3aWj8maHBzPxAwRZuezQGEZwyPBvoDM1HePAt/R506EnADczz3pYZjpubw60QEDAwzDeiA8BPTQ4N+7bwBEANh4aLHnqdmCaMHPdpAf4aLNwz8RHmC/Jgj7fEQd6M0Rstid9j7vrTXQGaWZm6Dh9CPTueAo9bMMPzprA8+hAZh7NQDOS+exLAD9oGaBIA94gSXidnUl0O6nWkZgfswwNWA6+vr70YdYfWg6DR4fFHkDP2pmkkkjQAHu/Il1+IkeYAl3aaIIDwJLMzPUSwbxQdAID70GfwSz3YIzDYjzRrrRHhihDURIYCl3+0S60sPihsgP9aQbPD/xpGHQaDPAM0aDS7Eg3hqJZhmX0dCM7fDZ39a+4ZuoNbP3eCQGYOg4j0b2plppJGfX9aTPXqOdCoI94efBAZue0SMwD2aM0ZAA+vUBOhvQ9aDoQHh8Ukj73EyyeZID98jZDxT8PrTIe1MM/nRmeAfOgNBm5lp6fZmjzcw2IM2wDPA0aiP8AEb6E0Z+K10IzM88OxBg/nsbz3/nQHnvwDsQf8Z/hpYOaR6fXghZBwANzU60DoAZ/QkexBG1wawc7EjMPZnns70Asg+ccNc/FzPNBDM+XPU2ZmeakZ5pPHs1FIAAAB1GwDA/ImgCVzQG/saD+NOs9ChVLN7DlJWiYIJslnPwMzBU4TJbMw2GGANp8Og+xXAHrADneqqZDl876xb6wDBBYMmeOZhgfkQQ2Z3NG+++GHYAJQ5nMtNG+B5mHeCl7AVAzPekyAM56ee8+9LwMTcf6ABV8mfOZdCKDYBmezYgsDZMzw1wz86XLaw+PvzSDUMOvDYi6w68FKyjBo5tm59CRg2e/v6M0utLWYZM23zD86A8NiWbmOYN/QgP8n5Ej2IDz2JrAAPU1MEYdC+GyBtIDz+ChSXmzzYccwUoDwJNPAw8Z4B1qmeEcJOj4fWpYHrYNudGfWmeWbz9mjAHMtnQpMJZnBP8AYc8APDegz+CaDTZL2ffmi7tRHo8/gmgBwOtP5sB7fPDPsTWG9BHZZlBIdNx/No+gPIpYAGl4jn5EeeHhgmT61QNI+hB0JZ/BAeHQaWxAZubGwSNSEakM/s4H+RNPHsAM9iTKoAAbsEt6E95IgPNr8m9SEZ5ikBmHiAl2I+xAgBI9NLrSwc94gAz25pGB7HPxEkjDoczQDg4ZgCIw/DS7tNI0CPe1pmkH4n/Gkl0DqGgX94lh8Us/glm3kgeMzORpv9CWIJSdA3dRhvYg7ECz+CXifWkGAe0BIwB4vDbw+QEWRh0JAB4agAg7tNI88/DR5A9bYlnt096DD4owPYiSDeSd6FH35I8/ggPPekB70vD0vnSR5BeIBG24iz+CEz3pAGYE55EeiA/FNPYAYeB196ZZCJ7/AGfOkbzYHsc3qGRGDjLqM3jMQzQa2ZJH1o1GZpB0b+tID2dGzzoEVA3kgSAMz00sMDwRJd2ojA8xDUS8MOtB4nu0Bsh4vh+dLPejzlh4Zh2bDTJ6ntHO9AjS8Qy0wbzNB2JGGzU6FAXie8bwPyJGbgJ48DDU7/Ome7xFYQGYD7TBI+rURpIyB4Z9aONJqNNkc3RqjJhu+eOeCQJ3Yg31t8eL7ogYVKWFVZ8kgN/611W1eM1gXbosVWUFNqZ9YPBs/WvNTyZBY3DV7gCNVWQzp07mWvOB5go+swBH6ypxhI/GaPBeQrSvy+LDd/5K3A9Gj98cwzBdQt70nAN0GL4twDz650c8MPyJhk7LJpUWsNGxOp0CsQj7Hutcyu30eLOr0jXoz8miSz7D3gui0S4bLu1oJdo3Gy86f5DBWZhVWT05TASQ+detNvLk/hRxp4YyPXlqvmeHW9T3gz/QpFH9KviNR3TiV+msz3Q69Vk2Xl6YZk04/YPyYEgPkVVW7Mti6j1LqtGNMD+1gGBmsUbVVk+kJZ12iDD+dKln2S3gw/Wuim8xPjmxLig8z3gYZga87170aac9mdnXHo6fREnAZ/xrKGzx64XRQgt84zE7MA5kFrFvHoC5+CHDK6mjbbofq2QfW9HP/IuRXP6Lt20o/wDkjXI1Va/BM8D/AI1Isn0nJQaUG9abreec0YB/Aux0fiRYdea16Pc8B4/wXT0TV7HkS4aDcdqzOUuejvQPne6FXhJYe9m+Br3a8cGYxyk6DGkh++ZzXPa96PfDK4XTltxJ9NkH3xJOAfowVxY8r6JmWo2kYP8AyLqF2+jTflvNG/blSCsRw7AA81y+ZAuCgu6Ffo0+H87zOAI1MmzgmjBzDoUtk2Hg1AfzQZ4F4DmBoI/92gNTjDMc3OtRDA1bBH3o9ZPgDfvEJsto38ADPJCZ4osPimjD3aA2Zj/4gGCd1s/ZqJgYezR54HsRh4TQkvgkB7vEbBRzezdRhpqG/hI1t3gN7PIgPQxHQzz71FDUA0X+NGB7D94kg6yzbSzcyVt/BGZo2TcNBn8Es/ggkaL/AOHmCDoPemdZ/wDEX3r9ogkZ4FnhsQG9maEDcBM62Du/9CIw01t8Or8n/wA5VW5zh08OswDA1Nu2j0q24QNsViTM1/xT6F006JOknpypZ4fIjCxqG94k5g5LvnM1wtnIrJjUqGbUudOCTIDrAF10K8BhpxQk/wDgqwjUGjw/YU1kPyKXots+w2K3Oy7MaqnKCXOYAz+dWrLOt1xQ/QrAwUU5LbPWrai5NjvYAEfJwfeMAf1qC9W4jIamainc8X3biPLtdeGHu0YPNrLnc7b3s1Hercow0w2LPadta9MYZ9o4oT1bgslvcWUMzP2jiaNNpaV67WA9g2ZqqmXJOkhpsbFX6JmjCNgm0wjvTJxhqPvmoOjmeuff51dvRgw6FHZZbz02wXqkQIeZKwjQ8A08E8yGiWo2mecc5jT1EapYQ9yaP2unoZpc5mmTe95qIJcbZ1tp3xPdqvCYxoG45KBVr1yABmwwe/yIL16NrAo4Mgzgs09W6qZ6jbmCrJNSlGXj1F7PyMgoW3pyQDrcVFUrkbB3QY3rL1ifOoMLnn2JPWAaLx7zWcn3JeNShGxSrYONn0SM96DdyalUZLRgwarHnqyz+1vrK023uJUyP6qnOcs0fefWf51ewOFcuSenVZbxh5AeQNHWILLR6jgTJHuY4H3oKbcnGyY6ESDTeWa/dMgui29ZNv0Ec4tGAHQ7zPNaiHs6GwD6EHN4fB+o1vOdeNcqT0h/rDNbOicNLKomHI0Y9UPevPZmtBkadZA3kA8s2z7AMEGsf4akcsf4mH1rL3JxOsO0sG5dVCfLP3Mfea1ixpWXnDUgIz59a41VfSElvR9O3LfBnP3shc/qt4XrcJH65uB7Sc7GtgJbnegK3xFsq3vDnVkNX8JreaxVY4903fFtilTJh9jroYLkIQIn4alww0fZraGkWs6rfnEavZhOnGy0fY0GAKkOlNycHJXMm75zNXAGmjewVJ2AI0WNIacbb2ACPw+wFHPXy1GMN/Xmnc8B9nmvcJLw/do3t7QBp5oOtO4fiIBAHwEM0HiZeIjzbA/DTqNTSIDwDw0JpAGxA6G9GB7zTIa+PgBmaFl7b4nX3oCPeh8fLswQGeezf9aMOgG+xA7kaPoQJo5Lh5hoH9ahGzrO9JAAI+hWsCPDAkDJhhv2GjPePtEBnvD50AM5ijZZwDT60WGAo8oPQCLe94nehDT70Bn+H0I08HT6OxNJkzcMdmGaNnUNoHHG8M0Dx6eGGpvQB9aWHxRoEnYwax6eYBh15pp7Z4envS/9WH1o82Lt1EObhpf4EWefu0ehZZ0QTueDRt4bzSzMyBvsS8M/EQBg57xH2I9hnqYfkQdx6aILw8DbPvXwPZL6YdBpZ7EWPxAwcczwQJH+8cM0CA0j3imnjSZDANiM9i3pBvwSPXxDyH2JI9Pd+o43ml4gbD796ZDU70g0wPPU6EeQPfn4DmBo8/gg/wCDrNI9P8NHpeIYIOxH/wAQfiIECM0YYYoMAxPNv6EIBh0IEYfrRBqAOmbmeHekG8UkWWfwQpYfFIMM/ERBdC+dZofE94jQLP4JAe7xA+sECQA3l86B0z+QA+QEKZPejwzFUFsM94JJGl1oEZo2Xt3Rmg6w/wACRngWbaBdYYJGkH/2CR4H1oyAaeDoDU3oAZb1TcccPf0Ag1tuxvM/IrCA88wQB1r74h7Og0WGBabihqPsNxtNJ0DAANtBDjOaTx6gdaA9iAzwFLrBCbLbwgD7eYB0IEe8wc8iD72bvgN/nRgbgEephh2Is3AQG97LTbPeaR7/AGfQms96kMh7xAmdiB7ozSz36enscQADmRuameaAwD3fyIDZ2aeofQmntQ3Q+Q0gDAj8Q96BAbGZhmZ/QCarGmzDzBwGTz6zTrOoA+IeZ+dG8APNbwA/rUhMg/GitMPmBmAbzBI9Q+hLw2QNw+wEo0yJMa1IjmYZqg71gDh9aWGarHq2wEzkdB7Pz9inMhL+QwQIDbAdDefnAwUoOVeJo5bGbTZ78EP944mXnvCd0MDwUvKGexGCiU2ZLkgZymw+RTfdaiNCz+CiyYESYWDmeZ9+adzbN3TBII2DueZoEyzywAxqGeCRm2B96dUcKlEkm63FMDNg96CQAI1XyTfzB9+L9384GpbIZtB86CVAZpz0j78+bIYdYKA8Hi6jaf5ZsEgBvE9RA0GefiKWehoB4Hih77zoABwPEM/oQGexEDNl/l9fT2edAGoCiyQOY1yIS3gDrMFKDYH0IHt+KiMyXzlGw4xs7DTufwSOS2BgAIDBAfWrOjyaGHMeuILz2fscDwwVe8CAT6F86ARn1o5OhiABvVCOCLw+/wDIj8jaBB9w2Z6a+fu0vcpB+J/xoF0Hgh3pPagO5+dIECBO57E0yGsfhp0NPvUgAP8AQkZ5kjANF8+xIwBA0vnX7M19Sw+KAs/gj8PMO9R/Ey00eYAXzoCP2p+RI94pBvazcS/u0A5/BO4N46gKOgOSwz7R8GTPoA1Ql+H2fnS7E1n4Wo2/+RLfigdS0HPaIANLXc9mpZBP2qIEsPih6PdoJBhvQZ4Fp9iAzSAMlDUmdfS1HG0j/d9SQPPs+GD54d4GkbOs0f7tWDA/doOjw80OZ5nqNpa2sGp0H5EDp6mOokYe8TQanf0JGf4biB0NQzRmDGemaZjPA9n5wTpgwftDMD86D6BgDWml4Z+ImXs89iMMAw1HOhQHQBNGDmfgOB9aN492oweCDPAczb61byLGyAH4+8EGDm8NTNH1pKCBHvwRh+H/AMSQYY+IkZ4PtafQhsvE6E6BgbSjvHg/8iPR95qdaPSMMD3ojNvSPUNCep59iDxAwcbbQCzJAHTbfbewMNh9iLrNLWc8mCE3kDpxn8NRAB700bx5ahmnQ3jqAgMNM3QbzSPPDw0jAPzoMTQGB/iIw/EBMpAeB4aaBGbgIweYe2G3vSP+NNAHio8wdA340gH4L5xnQ7w61uLb468VLbYCI/UgqscOgJYb/wBaw6W9Hr0xaXHKwLqd5W4JfqepeR08wP8AOukBTZZtA/BMHmj6DA8wXhd5nNXtpXzeNjOh9nK4bMf+zu7wP9axuGeHss9T2lSggZ+cNiTL0FnZBqR/JHd6Fwqj+lRL2MXpQ9aP3yIgYYfkXWrbuSwL5a5q2KzGk/ue/wDQowbNXJwxsO5/HuO1QZkfjRD0TXH7t9GapMtHOs6uMyQ/s8jwT/xrvBsymeiWZ/WgzYPxDB4JH4oGtEbeX2b5428GXwg1WDMOEx7mogbzP610Oz/Srt+pMAxcEGTTZHnB7WZP/UXaHoz8lg2JbEOqxH+tl7fmuc3PwE4XV7V+6SbekH/Zw2AsVOjW9eFKr0MJ1DrkOYHfpPb1NnnErDHK1WDGmRz6wkAvJ9w+jZxNtL+dbcnMz2g6Hoh4H/jUelce+KNk4U65oL0//wDOAYH+tVtq7hdXo08PLkd5ulc5RHT/ALOeYLl91eivxDoLXN0OXGrbQfvsDWrtL0n7RrcgGLjiPUE/PrazK67ArEqqwAqNs1yNPjn0GyexbRbJ4fmBVaU7ylcgyYbvkeDBIJLekva0mZLngbF42jGnxD6zNkDXL7k9HXh7cJG/blZk0GQfY8YYKNjzoZsGfhuI8Pit1eHo68TbSa5timhVY/niPb/0LBHJfgO8pKYNl0Ox0N6uLDvbqKPhvUvmWDDxOtNBvPTVgOW1gzYQYHvbBv6FIA3I3h9iEzzJWhBzPMGDYw+dPdCdQaOZKFhM8/eJHqJ0IzaA9nzq2oQPAkWe7PTS8M+hCeYBpoJoPNmPkUd4HwPxA2JoDNFrGBggX+ND4nvE6e/xO9NICSA8PnSzbx8RLw8PnRnb0w9PYZ9o+AKE9cMFn+lAuZT5j5lpvvnggZzM9PUM1zvHQHrzgslh1/Qoh3s57thZc42GCkAyoEuTc9VMjc1MFRSaxVZMrDmzwUieeBaaiMhvBxWLVkHD94akAG3BKAGzNxSz0wUNTWjs6EWinuhLMDDwwQBh8UYAGaDPBGDzAHqamaB0AQGGBJPVWLGDPUVLJuHM9h5oy8LXW92lmwHWsozJqsx09Nsw+tRZNKqWrqTp2DX1o6Gok1uCyWGuAKnO5IIOm4bizkyZQ4AafNaxpoKxFNrQptvvPH84ZoL2TXn5/wCyObE0frF6P49RCM186iQwrjwGc7RpsfD6DQQGYOqb8uhz6r5DDMAzRnhYUq25Vbf/AJmYkzGu90OgFHqp1y3ppxAowG7+MZrYW9cN9zPDChswwP2IaOAACvnreqte+93NUmeY8jTKPXIQrdzn/pU4zMf6FCk3tccObylvwWTM+h3lszNdVZ4UWdGkA+b9Sew7Dk7Fo6bAt+lR+VptNjMh9CLcPC2+Kl2yAflweWz63pHQC1FvcKKlSpBv1WuespB9nkXUDe1h8NNaL59C86RSepJfyKbApr7PtEc+e/Aw06bJkun0MtB1rP3VefEa2KSdcY4bTGYgf0iXsBMDYHDUKTcNq0GPr1y46bG/7bNeb69xFv8AvOQbk6uPRmj9zH2AqRmggBg/KfN761t8znd4n8eLOjOmxTW5k8w8jOCx9Y463xPdzodNhw4/0ZrD6OB5gpDOwU6UxZT7hvytkfrW5pOl+CBqEzTW2ez86lGGSQA57targYRmwa1MEjUgNT2aAwUowQYGOAdaEDczw0DBEAbAfbR+J1q1xBAkGpmZ6aWeZ6eBowDejR80XP60YRgMdTUQ8yAO4G/+REeHtOsEZBe37AbBFn8E0Zt5+Hv+RHnvRqSQPIM80YMoE8eZh4iAzwFGAN/hpZAgZwcMgPXMMOwFI6zSPDHw0IICPADwBDh3mke9LDATPeaBeJ7tECeNk9DX1AAM0CBoNfV8R8ME6yyckzw3gCRgGCWDeGxw8zPoQLBv3iTx4H7NG9mB6baBHmzoGfWCaeecMNdvMzSM3MNPUDDvTTOfR/AhggNw/kSwbPPU2Ai8QD08E9hsRewYNh4bYJI8Pilh8UegM8w0wRpYfFF/jQDgZnn2J0A3aiBmTo5+HmmgM8PZ4Ig6YYdfQjZBs/eJo+aePUfw/IiM9EN7f6EBmG/2iaRYOAYNmGGe9I8MkB4fFLvQYHlp96WfwQGl2JozwFEB7c0WWfQCQA4GeaRh0OAGeCMD8LU37+w0QADB5Hh+GgZAN+mHzo89hoDMz2edM4fFIzPPw3EZhu9pmgWfwTWe7BO7NXBBmGWn3oEyGAd/50m/ZAlht1DcSQL+7R9fs0GzLT7EJgYf9ogdMHDa1OvfggPwfyIQeMwwNvDA0lQWbmHh9a+Bnn4iHPf86dRkaPUDoRhmDW/rSzb9m51oM/gpajD8T/jQHvJLel4YdaBJZ/BADzh4eGkgNB4m9tJGCoLNvE9RIN5aYIDBszDUDMEjwz8NA9n8EBoADDYjQI/8iiRoeHt38zU7D8Nzeo+eAam80Dvh6oamzNAye/8AyIevyZ/OizwLMO/YCAzPN3MEj9qb7mzNADLgABuBsRoAPeBh2JADbODDbexIN5+H+dHnhs00AaOBamoaT2zD5+hGZ7FHkgGzTPAz2b0BgGDSQZgHidaXutP86Qb8NQ8wNAjBs8EGixnrmeBo8DB35EQG2ZYGgdDTy0zbzTXvTbbc6EYaYO6jjmAJeIDvk+dAYbM9Teo8kPC2GYZpazgOhnv3qPybmqZsOGefQglAeGDbnX2H500eAHqaiUPU0t/nRyXmGWjNzYHegAA5mOep3pMssUqPIfcPZnmnWXs2gfYcA2nw7Ej3+TBBCOZBmG0DE4M3+xSAByGOxxR2YcSM7psMMs5qRGe5mZIg6ZgyxsPNAZyeZzbY34daLP6M00FHgxpgTmM8wPfv60eG9S8o6yHhYd6WtmOmBpAgeZb0jbYPA+w0KCYHnqNuYKV1oQNxmQ0/qBmHX86lG8EkiccAAP5FDTwj54bFH5NgJDsttgNU1IDrNCb2irDMmS2y0ep5Ojzo4Z5xQc6M0jDPxE7g3vc2YfIgMDyQHvP5EgQH1qAZn4vXsSPTPoTQfO32JdbWnpq0HQBgC1/eoD619Mz5fU096Xie7QEy9g6DmGeHYlJeYydly28O/Z2Jp43Aw8NLDMd7eaA2TbMc23AME7GPxegPzqFDjMQA02GMM1Nh6eW/YgUnTZLT359+aaw0dnee9OhJbeE2/IaZe1zdacbDPQ6/oQGeoYeP0IwNvDU60BntM+zr/Imnjwa1GGM9ioOme3DsS/wKOEnWHNGB7EDp/wAHYms96M+hBhnmgPW8UHOhIz3oD/DNB2hpoHuwPOaADczPrUcANkz8czBHnhhqKRKR96jgG7PNG94JB35+RQEl4Zn4iAz25oGfGaB9vvVh4/3Z7Es2wDxEs9YvDbw7Ej3+GqAAfe30GmnozcnDXDPBO/ICA88PDUhAGj4bexPdCAD2ezSz3eIqBge350Rn0KOB7z37EjPP2alkl7DaDDrbSPpNs+9Mh1p557AAc3maoAYN46aQanYkZ570IagGpaQI9PPPeYd6EAbedA4hnhgkemeYZ70gDYqZmd55gkyyYNGw4ee/YakAzhvNC88AFqaew/4FLUmfZabm88MEzg2D+zr8ikGcQMDN/AwPZsSP8dAuj3e9IzwHUcSA8kYbz09Rn85oAzbNPPRohtajhn+RJ7YWg43vBAaADDZp6YGpDLL+GxwA+tNGDgdaDW/ERkRm4EjQcD86eAHDPrTPM+Lg5+tH3Z5o1ID3qQZ4dAKPmxnpubF9P2qgPhqPCfyJYMaXtN6az3ZtmiB7R9oGeaAzQAeZYB1pPG3lmDeAIQMOsECzPM9RtJFn8E1nggWDfvEYbOg8E6DzeWD7aXgZ7DQIDTUk39LwEj2B4Zgky9+IgZA3Hms9TNGEl/3m9O7EjZbDexn+dGQ/HeAH9MNI+g0jZb629iAA0Q0w2AZ54JozRpdjA8yReIfQo4HvRojaQyGZJGzge9R/EMPDcTus5hvcR4QafZgaWjomEqC+9GkB3tbEs3APUbSA8yQb62OPfEa3mAgv1JmpRPwZYZn+tdgtL0jbAup3laznb0jySzAw/WvMmiAKIbOZKE4e4+Wgz2gl0acElrzx3tiWb/Q/gY+QwXielVW47ed17cuOfA+Rk9i67avpOV+A0EG/KP6yj/2toAAwUYU7t91jeJTajMhyPIYIqrDg3JAOLc9vxqrHPrPRVPavELhlfLX8z1UGZf8AZ5Zgya0b0aVGH+bXMFI45XvRgsesOuuWdWHqO6fuTDWD+Nc6qXCLjLwuzfobFSktB1yKe9s/QC9OhqAedSiH/fACkAcsPDp1SMw7GndiqB5ws/0nLuo4hBuanMzI4f8Ajf4F0+3uNPC67ZB5y3qbL/ClhgH61o69Zlh3I06Fz2cyEs+t4A3rk9z+ipz/AI9h3HGe/wCry8ANB6Dh1vD2b4Gz2PAYGBqDWLetW7c/XlGhzPOfevH7zPGbg5PDXOqwMPIGsyt3aXpRHDd/5QWyb3/WIOAGkDYXb6K9uTNWVZdZOjun/R3czBcar3BDixarRyn6G9Pjh72IzmC9O2rxOtK82gfpVwRtX+zyD0Xg/ItgzUuW9/gtttXgL1k30PsGB94PBvUhk4rwammC9q3DYHD2+Wj9f2xDkyD/AKQAYGuT170UQ67OuMGQ/s87P+A1bJ588QCToPOYrQ3bwuv+z5GFVo7xtfjNbwWUZk4KBN3mmsTUjWYMPDQGD7IbwQDsTLzLmXh9HWngNzPUQGZ5q2qOGwM0jecPxEnjR8sYNA/2OIjBnNzJGgz+CQaaMyPxhRgG3TzzToSexxgMFHNlxl3NhzYim+kssA+b5tq4jA29g4GHQo5ssGeDis42mDQeIAAubZ4Mnsd00Ek9E9NCdSgvPngeeCiTHtZ3UbUbdBp5nM1IMN4OeQFX844B4P7DUSY9KkyNMDTY0B1VhlrU1OhHGrAPBqOLKMxp2/XUuNpsnm/KwBeoaU682CaO4XHh02MM/oWfk1uhs59Z/nVed4MB+w03MEGi1qqfvM0LMZ97xJc7D6zVCz9v6w7oQaO8z8+GCu4fB+7p7WvXJ2jn5DQOnUrcDw36yz+tV8+4aGy/oUNzn3fk7FoKVwctWHI5qpMHPe/fHsWogW3blKd1IlKjMgHYg5uH24qroMU6IcMPOrCNw9r8lrOs1XxfIC6QcyL7ttV711W5G/a6jGB38EDzNFqSlWHSoBhrsZ4edaMIdOZ2NsAjZmRKk0D8TPA+jNJ44jP/AKzfjfrXnarrGdNiHv0AU0DwD3P51j6qzxGrebdqgHL+cOtZKpW9d1HD1lVdY5rHnPNY/R5bYddzfMPAbA1WzJ7kb28uBG+t5c5ptK4oXJgESDPzPowPAFuIfBav02nO1Wovxp8v9890LHuOlcW9PoZxz9ZVmNJm9gRz2J2pXbatE1eeiRtUOwAzWFk2Nf8AWJ4N0diBSmf4/wBaurb4M1KZUgbuOuTJ+fe0fg/nWvhzdXkP8s0oywoFAOSHYARlqKVdV8VLByuW5DolPADOS9IDAwBBJjWjYdUDXnMsxw63nnjD9C55xC9IoHqk6fDlvmfB0DlvBhgf0LretXzl28Tqlr0eqxrPtxgwAKnO8F57DyKPxU4zWHZlgy+G1s1yZedYlbDlu7wD515quSfel+SI/wBtbqmTI7B5hHDYCKBTYNNY0KaxoteTNauc9TWdGOeu34pqQGmYb20HQnfEew09gLUNdAYJaOfvEbLLgZ83h8idDAD01LUAA3pI2dg5ow1D/IkHQgDB7+tLRc/rRnJ2aaZN7Z50Dzx9GAIjMA9p5E0EYz63E0ENgyMzcNBKOf4WgDAAmszD2iaw3eGiw/8AX86Mhgbhl7NLu3mjDYKAzb94aNSyBOs6YAZ4Z5qOeAeI50JHvLr/ACIHjezSzwTODZp7rBAW/wDImjZ39aM3jz2MZoAzMMzQGCRmiw+KEwbPrQFo4BngjMPdpZuGGm459CDPfp96Aw1Az8NGBgAec0H/AAdBoANswzBxFjA89mmgMN2oaFnlWRPQbM/OmXpj7zpxG9gIJZyWwaP8VAeftMN/kSZ2e0wTxhn4ihCOyD+Opp9akYYEHyJGb5nn8iQGjzZdaIEtbBNA9nvDYatodNDhmSAHnDdPUbwTpm2A5ogjNhlgP+NvA0gNzAPnQ+GYg5sMD6EWDnvEWaCS57tjNOmeYpGGABqd6AOtB9DZ50WfwT0Z44xm433hgmUZ4HgZ73+9AbOBo8MBFzU/IgPf0IYJLD8NLD4p4D0d+nnsQwiMvAZedHnvw09qKTMYxDXDDybE8AAYIvZlGfQkZ6x6huIESNnpw70BpdCXWexB9zb86+GHQaajAeGBp3t1OxAO9Fn7xJD4nu0DvhmfeCDDbnqJBvzz/Whw25oEGoB78Es/ggD2umkgM0v7xF4Z/kTXWeCoLRDLU70gSM2wPTSwzQGZ/u0GeGHzpGexINMwQJCfQvge1NvoSPZmHWgHZvPPsSwc2JGD4Hg2YYIgUg9mSRmDPiIM2w+tGe/8ioBGe2agfoSPeSPxDM80gw930IDZNB0ezSMMUHncQH2JozPZp7EDxvm1s/WjDTMdnYjIBgZn4YJ3YecTvw7EPRv/AOLoRGHvFIHDDDNwzNEBgkGpiH4ppGB5+Hh+dXhqEDb8U20s8wzS7s/OkeeB6beZ+ReAABINmbYYGfXgkz16jmw0s29XX35gvYAPB4WxgEmXm8xY0z86QG+Z/WaeDPPTNeAu/wBokAOb/Ez3pAfW243hvSw0c0DT2mHiaZ4fIiOZ0BoHvPDNGewN+YKOepls2IDeDcDiTzJyWjYBOvaZ4abeGwFWzKlFhnhrnn5AQTY0ZuMJtt5n9adMG3mj1E1AeCZANz500B1IJXh4HH7wNAzz9NokMG338GQ+RWDLzD0cH2z2GGYKOzGiSYoMG2DwfOpBhm0AB0B2IFsTRvYStPsUgGXDMA86iMnEmZuN5ngfkQSz6vkBIOs3OvyKPPht1JgA5t5nf2JU2HyDWhrm9v6zQS0Rhszw2Jp4zDDBvMzSM3MNNZA8PiiePAsPkTTL2BA4fQCN4APe30oEHVqJYZugkzsP5EjNWqCeNyG/hp/Ig7wc7EYB+I2nXntYf4ESA3m3gAww/IhDTPNvUwPsz70mQYAdPT+hAYfeGn/IizuGApozcANTDNHnmnTZcAAz8maIR+xLn4nu3A/Wl4nu00cCK9IzNjf50Es8DEDDoXwEAGDLXyAvusBoCBfT6Ey8b4YaDee/fvTuH/r+dUBDZ7sN6dZkuRi1G013n/Alh8VIKS9rE12bE0Dwaug252Ig60LwOB4gBmqCxBXFbjUAIEJ+huHmYfeQPsNU6QIDAwAg1HER7CQ5mHu8w70s2zDUbQIwz8RID6wQAZ4JHnjs6w71IAwA/aNoc3A7M807gZiDmpnsTW9UAPUAvk86dDZ4iENTLTRGGKB5k2NXZvTP5MASBlsDzb6zSPMHdPA8FIPDBB3o88C86R7/AGbeCoGCWHxTLIGew0j1ALTb60Bh1pqTr+zY6070Akb2Z+JgvdiP/wATgBg43gaQeDg52KQYBggAAAjb7OxeMiM2DDYZowDMfaYIOWM2jfbbzAO9JnUB3egeRBp96+gbZ7EBoE9sdXw9TEMEJm4bu9OgeieofR3o1IDcNrNtvYhwz8M074YAbbBngmlIDPrYMEbI9bmf5EWjn44diF6NmAH5DQEGp+Gkeg97T9CTJ5+0DrBNGz7tAeHRvzRYZ+0TQbMEeeaCQe9MngfhuNoWT1iwYMDPyIsHO/YaAdFF+RLNvFNZGgdmaYaQOd4I+c1hBvT6O9NdftEQACAzeiG6DjDeB96aM3DS0W8s0WIIJDLLhtG4Zhs7EgZiB4jj5/QmsTRgGZ73MFAd1op+G3nmmngAM8D60Rsthg51oDNv3aAzNw/bodEAPUSMzSz+CADPAkg3jqAkZgA70gQSzweH2mBgo4fvEnjbwDsPzgmQNw0ZJBmmsNiNkG0ntQESAAwHf0JGDh+IDiDWfx03N6eZPYaKM6xgGmjA8/aID3khN5wPaJhqkdCZDUAtRPRnjP2Gw+zNEZyszBzBAnpLZoNbNoA8iHRbz3to9Ew8TsRkR7E6YfoTRnvQffg9p7LsTAUlkMNRv2odBrW2fxg4hWZpMU2vyXoQf0eRvBZLNw0ZhimB6LtX0orcnlyN6UZ6jn/aGjA2V1eAdNrEIKlQ6izMjn3tGvDW9S6PVa5bDpy6BWJkN3sAHtn6FGGr2k8zVWdhhsUV6iU17CUx9wm/isrg9pek/dVKkad+U71lT/PHDxl3K1eItgX41/MdVZCR/Z5B6J/xrHAlnPuaN4E6KzWIQd4dax9Y4UcHb2Lnn6U9TZZ97WbJrofLe7bUGeywBh6xpWYfjM9YK4Hm+9vRavGiffrYqUaqxPz6361mqPxF4qcN5oMOVGpRnQ6484Ni9ZxnuWdD1bUj+Rp407VWYNwwzpt1UaNMjn5wzWOxxq2/Sitl50INwUOTR5H9oaPNn/Auy0SvQa9CCpUOqhVY598fsXIrn9Fe0q26blnVw6O6fuXd4f41yascKOMXCiecuC29y59cunPbP9962i2T2cE8McH2zAD62j6DWUuThRwuvBo26rasMHT99H2GvO9q+kzd1HkYXNBjVuP53T0TD/XXZqJxp4e1tprlbuZjSD/o8s94KNjl95+i1c9Hmc3Y85mpU8+gHjwMP41zG4aDelpYN1yhz4HzvM7P1r2hGqr4eP8AtMf8ZneCuOcg1iOcWcwEyO51su9CuLavAQTHPeOKQGZgvVdz+jfw1uHD1VTTojvniHs/QuSXJ6MfEK245v0OczVY/kZA81tFjk8lnbqJMm4Adew+sEp8auUd04tcpUyG7+9DBKNJiG1gasLBzHw+9Ls0zbToMxOhh8/oNI2T92gjvA+Y7EIG4Gw+hSg/eKO950G+5xRJNSlH4epgkkuRkOm23VZ7XNRA2edFUqa/R2Neq1wIzXyJJKGo6DW+GoQ9SVWTkn5zBZypXharM83KG3+fNJJWK1m8J1Yf5SjMHJd+QFsIfDe7qk0DlZf5No+zPrSSWq2ggcIqGDoPvvyTw+da2Hb1sUp3UCCB/WkkpFhz/wCA2AB8iA5L5+8SSUINPahprRM0kkWdCHrNG2b4Rg73j6AWFn8ReE9n1l1iCwD0gPbS8M8zSSRXGlSePfCuZ7eqvB/2Jgo7PGPglnqPvyXv+xNJJcT6vhpWfSK4J0ePoQZ1Sw8jMY1XzPSW4Vh+yUqsTD+RnBJJdXjw5NeVVJ9KWzg/0bZdVkn5NbBRw9LFswwb4czA+uSkkts+Ea8oTPpUVXmgfOwIxh5MzVfdXH6+7tY5Gmt+oYJ9YR888PrSSXufDFznkAM83zekn55DxmpfQCSS2cgPEP3ZpAEvAM2MEklaD2Gw0QGYNJJIBzbM0j6tRJJSsTMlsM/DA0szM+wGvOkkiSAwwSMADxEkkb+Czcw02z2JZ/iJJID8TSPBtAGo81qdiSSAT1ARdmo4e9JJAs/nX1nY6bnWkkgdwzRYfFJJAPiGYeRFh8UkkBgz+8S2JJIEYe8QYNmPekkgXh7Ac/Omz9qkkjyHzRM2jBtGyGApJI0O4B7xvNO54AYd/YaSShBGbmkmmWTyPUcSSRkdCNgHtP1pBp46iSSNIIwzFNAy+ezsSSVvTvhgANt9iQHvNJJAsMyzcR4ZkfkSSQLYCD86SSLJLYCSSIIzxS2JJIFg3sc/DS70kkAmyGJuA4aWGbQOedJJAXQCXhmJtpJIBAMA9okyGAaYdCSSB3Ng8M9ijg8bwajbaSSBAe0Db6DSAw9mkkgPPDPT70uxJJAGbYAl1gkkqC6ABtLPYkkgDDNH3aaSSBdBoM9v50kkDTJuHmZ9CNJJAHgGQantQ6EYGG9JJAQHmYI+9JJGRGGYpow2aGHWkkjUj/dpkGcPE7/OkkjyB55j7NGybmJ6nYkkpeiMMw1O/NDgxv1D7EkkZEGmi1vw0klTVH1jB0NPsTUaTOmSDcciYB1h86SSCzDAAPUwUfM89TsSSUhYGf6818e3lh8nWkkqBvA/7z8iQamkGZgkkgE1XvUdh6Qb7hnvSSQSADk2GnNA/wAikd6SSCJTZkV6Kfq5wzaYPBSDk+EeDef+NJJBVHUpbzoRMD39asowOMxwbcb9mkkgew+KZkz4MPw5T+Bn0fOkkgeMwAwcBs0s3DSSUgsD922mo1NbZDXlvnq+RJJBLDoSNk2XQ1PrSSQF1l8iE9iSSBdaL9aSSPIRQZwf1w3mpAGDOlnm9nnsNJJCzpva3iaeCjmeA56mBpJI9ADPhaZuGYJdAJJIDDnsR18MOxH4mHtEkkBAlg+Znp9gZmkkgT0Z9kg12zDMO8EetgKSSCJm4b+nppBqBq6/n2JJID1mzLT1N6WYZafekkgXekzqe8czSSVAfE2YdBpnN/I28PFBJJAZnt+dBzOeAON5/wCRJJAgPxQb1+zADNHn0an60kkDp7CSz63ASSU+A6GppGbij57/AJ0kldg83PZto8M2t7eBpJKABmgPUD86SSoCeYH3/Wlns8RzekkgIwcxzb6ELJ78HDwSSUgjntgGnoZmkEluSIG30JJIEep2J7eaSSBowcPwwcMD+ROgZ5pJKgvE92iDeBmkkpDRg+Z+Hv8AIgwfZ9u2YH5EkkB6LfWGw/kUgzbMQ12/F8/nSSQAbLeWo25sQ+Hnh3pJIE88GfibEWwAzSSQH1gg6CDxNqSSBPHh7A80jef8iSSgIDcNGCSSB3rBNJJIFyzDwYPngaANBnZqZpJKw9rQfeJk3m9XBhJJQHtFzAHNRRno2sWeoYGkkgkgGY4d/egwwJJJAu9GYZjpuJJIGcMCwBPeJ7xJJAjNMnnikkgeZ0z9uGCI3mw8PU2JJIBZktgXnQZ/BJJGRAakABpJI1I+nTULk/F5pjwXQ3g8GzBJJRbJ0C1eP3Eq1WgiSqj64jh0BL6/1rstpektYFwu8pcFNepUg+90wwSSUtXRfVtKqrXN0p+NJDztPZoTgVKN2bEklnYjmYd/WpDNSnB4YNxpLXkSSUjOXJw64e3nH065bkaM7+KyeBguNXP6LVcZaOdaNwM1Jr8E2QA/8aSSDCQ6xxf4RTTbM6xAZDrB4PuprptB9KKDJ8O5rYBk/wC0RD3/AOBJJVA63avEi0bqaD1Hc/MyD/oj3WtaFVw2PsPM/WkktAqrDoFeawqsGBPz6zMMzXJ7z9GCyrhd5ugTjoM7s0jzD9CSS1ZOT3V6OvEq22uaigFbD/qnWuayfWNNd5Sq02TDP96zgkkgXORT95vSDTNJJWP/2Q==" } ] } @@ -120,18 +68,9 @@ { "language": "typescript", "source": [ - "const tensor_data = new Float32Array(inputImage);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, tensor_data);\n\nconst inferRequest = compiledModel.createInferRequest();\ninferRequest.setInputTensor(tensor);\ninferRequest.infer();\n\nconst output = inferRequest.getTensor(outputLayer);\n\nconst { data: outputData } = output;\nconst layers = { bg: [], c: [], h: [], w: [] };\nconst resultLayer = [];\nconst colormap = [[68, 1, 84, 255], [48, 103, 141, 255], [53, 183, 120, 255], [199, 216, 52, 255]];\n\nconst size = outputData.length/4;\n\nfor (let i = 0; i < size; i++) {\n const valueAt = (i, number) => outputData[i + number*size];\n\n const currentValues = { \n bg: valueAt(i, 0),\n c: valueAt(i, 1),\n h: valueAt(i, 2),\n w: valueAt(i, 3),\n };\n const values = Object.values(currentValues);\n const maxIndex = values.indexOf(Math.max(...values));\n\n resultLayer.push(maxIndex);\n}\n\nconst pixels = [];\nresultLayer.forEach(i => pixels.push(...colormap[i]));\n\ndisplayArrayAsImage(pixels, W, H, display);\n" + "const [height, width] = inputLayer.shape.slice(2);\nconst resizedImg = img.resize(width, height);\n\n// Transform image data from NHWC to NCHW to match model input\n// as alternative you can use ov.preprocess.PrePostProcessor\n// see hello_reshape_ssd.js sample\nconst transformedImgData = transform(resizedImg.rgb, { width, height }, [0, 1, 2]);\nconst tensor = new ov.Tensor(\n ov.element.f32,\n inputLayer.shape,\n new Float32Array(transformedImgData),\n);\n\nconst inferRequest = compiledModel.createInferRequest();\nconst outputs = await inferRequest.inferAsync([tensor]);\nconst output = outputs[outputLayer];\nconst outputData = output.data;\n" ], - "outputs": [ - { - "items": [ - { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAIAA4ADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDx2iiivbPYCiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACirdlptzfrM0CZWIAsT056D8at2ugXMsuLk+RH3YYc/gAf6ildGFTE0ad1KS0/rYyaK6iTwzYnZ5V7cLhQG3Qq2W7kfMMD25+tTQ+HNHWJRNPfPJ3ZAig/gc/wA6h1F2ZyvNMMle7+5nI0V2X/CO6F/z01H/AL6T/Cj/AIR3Qv8AnpqP/fSf4Uva+TJ/tbD9n9xxtFdl/wAI7oX/AD01H/vpP8KP+Ed0L/npqP8A30n+FHtfJh/a2H7P7jjaK6z/AIR/S/S8/wC/y/8AxFPTw9o5z5jXy+m2RD/7KKbnboN5rQ8/uOQorpP+EYtv+ghL/wCAw/8Ai6P+EYtv+ghL/wCAw/8Ai6fOv6TNP7Swv834P/I5uiuk/wCEYtv+ghL/AOAw/wDi6P8AhGLb/oIS/wDgMP8A4ujnX9Jh/aWF/m/B/wCRzdFdJ/wjFt/0EJf/AAGH/wAXR/wjFt/0EJf/AAGH/wAXRzr+kw/tLC/zfg/8jm6K6T/hGLb/AKCEv/gMP/i6P+EYtv8AoIS/+Aw/+Lo51/SYf2lhf5vwf+RzdFdZ/wAI/pfpef8Af5f/AIinp4e0c58xr5fTbIh/9lFJzt0M3mtDz+45Ciu//svw1/0Dpf8Av43/AMVTY7LRIJhJHpCttzjzJWYH6qcioVZvaL/D/Mx/tmlbSEvw/wAzgqK9H36X/wBAWz/79r/8TQW0ryjjR7QSdgYlI+vSl7Wp/J+KI/tr/p2/vR5xRXpMGoG1jKW9rawoTkrHHtBPrwaS7vvt0Bhura3ljPZlJwcYyOeDz1FPnqX+H8Sf7Zlzfw9PX/gfqeb0V2g02wRiY7OJQeoOW/8AQiaupcTRxqiSyKigBVViAAOwrW76I1nm8fsQv6u3+ZxNvpt5dQvNDCWRBkksBnr0yfm6ds1VZSrFWBDA4II5BrvSSzEkkk8knvXL3Fmtx4jktyxVZHLsQB3G44oTZphcwdWUudWSVzKVSzBVBLE4AA5Jqf7Dd/8APrP/AN+zXYR20ELbooY0YjGVQA4qWncwnm7v7sTivsN3/wA+s/8A37NRywTQ482J489N6kZruarX1qLyzkhONxGVJ7HtRcdPNm5JSjocXV630bULrd5duRtxnzGEf5biM/hTbGO2kDmaOZ3QqwVCMMCwGOmc8/8A6q7Gh36HRjcdKg+WC18/kc//AMIvcf8AP7Z/nJ/8RVZ9AvVuPKHlsmQPNDfLz3554+ldTRUq/c82OZ4hbtP5HP8A/CL3H/P7Z/nJ/wDEUx/DNyi5F1aufRS/9Vro6KEn3F/aWI7/AIHP/wDCL3H/AD+2f5yf/EUf8Ivcf8/tn+cn/wARXQUUrPv+Qf2liO/4Iw18IXbqGF/p+D6ysD+RWh/CF4sbML3T3IBIVZjk+wyMVuUUrT/m/AX9pYr+ZfccpNoOpQRNI8ClR1CSo5/IEk1nMpVirAhgcEEcg11HiS8SJUjtWmgZiHVS5LbcEH5gADyK5ari21dnt4OrVq0+epb5X/UKKKKo6wooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACrT6nqEuPMvrl8dN0zHH61XRHkkVEUs7EBVUZJJ7Cum0zwTfXE5/tD/RYVweGVmfnoMHjjufbg1nUqQhrJnLia+HormrNfPf5IteGWm1aC5EsiJ9mVMMQfmyDyTn2rRks7mGBJ5beZInxtkZCFbIyMHvxWvDp9lo1lcfYoVg3AsTkkk445Ofy9/esKuejVlVk5Lb+v66nysq8K1Wc4K0en3ahRRRXUAUUUUAFFFFABU3kR/ZvN+1Q7/+eOH3dcf3dvv1qGipab2dhhT4kWSUI8qRKeruDgfkCf0plFNq6sBNcQRw7fLuoZ85z5YcY+u5RUNFFKKaVm7gyW3jjluESaYQRk/NIVLbR9Bya3rrwowsjc6beJqG1sMsSjgYyedxyenHXmucorCvSrSkpUqnLbdWTT/J/iXGUUrSVybfb/Ztghfzj1kMnHXsuPT1JontpotMF5HDLcEsB5US5bGcE/8A6s/zI04/E9/5Ahukt72MY2i6iDbcDGe2Tz1Oaoy6hK929xEkduXxmOFdqDAx0qIPENtOKXW97p+WqTX3aEySVnF38mche65dGSaBI/IAJX5gQ6+ufQ/yrO+3Xf8Az9T/APfw1376hNKjR3CxTxN1jljBU856UkFtpd6xtrnTLRFfo0UYQg9eo5/L+tbupOKvKP3M9ClmNKlHWl9zv+djiP7a1D/n4/8AHF/wplvqt5by7/OeTjG2RiwNdffeA4ZJHeyuzECCVikXcM88bs5A6DoT9a5XUND1HTGf7RbP5a/8tUG5MZwDntn0ODzRTxFOp8LO/DYnA4hctO130tZ/16GoviK12jdFMGxyAARn86G8RWu07Ypi2OAQAM/nXNUVtYv+zMP2f3nSWevwyJi6HluP4lBKn+tdhoB8Kalp7G71lIbkMCweQQ7RyABvGGz1OOnHTv5XRXPiqEq0OWE3B90P+zaHM5L/AIB7BPpPhpnBt/FNjGmORJPG5z9QwrzuKe2m8Yoy3Ci1e4EQuH+UbD8nmEHoMfNg1h0Vnh8NWpJqdVy7XS0/DX5mlPBUoXst1Y9efwqZnZdO1OyvCqbiiuA5+gGR6ckjrWPe6dd6dKI7uB4mPQnoenQjg9R0rirbW7u3jKZWXJzulyx/nWvF8QfEURC/bFkhJy8M0YkWQdwS2WwRxgEVCp4ym9JKa81yv71dfh8zznlkm2tvmatFZw8VWd3l7qyNrL1P2QZRuSfus3y9uhI9hViz1C1vkleOeOIRkAi4lSNiD0IBbn8M4rrU9LyVvU4KmErU27xehk6L5kM0lnJach90kh/hx9368jj65rfrM0+4guNSvXhk3BwhGRjoCDj9PzrTqysZJyq3as7L77BRRRQcgUUUUAFFFFAFnTvsn9oQfb9/2Xd+82dcf4Z645xnFdpqWg2kuiE6JDbs2/zNwxIXA6gMc9wOM9iO5rgqlguZ7Vy9vNJC5GC0bFTj04rzcbgqtapGrSqOLj03i/VfqbU6kYxcWt/vNbVtJt9b01bcylEVt8bR4IBAIH1HPTiuPv8AwPqFvtNnIl2p4I4jYHnsTjHTv36V1aeIr9bkzyi2mZvv+Zbp8/GBkgA8fWp59ftmg2W2nNC46M1wZB175GT+dRBYyi1FRun2aaXrez+5EYetiMLG1Gaa7Nf1+aPMb3Sr/TiRd2skQBA3EZUkjOAw4P51Tr1aLWkK/vYmB9U5z/hSzXOlXqobqKKQrnaJodxX9D6Cuv6xVj8UPuO6GeVVpUpfc/0/4J5RRXp7+H/D11OW+zQF2/hjlKjp2VTiqr+BdKaRmEt0gJJCq64HsMrmj67T+0mjojn2GfxJr5f8E86ort3+H6GRimpFUydoaHJA9zuGfyrOfwLqqxswltXIBIVXbJ9hlcVosVRf2jqhm2DntP77r8zmaKsXthdadcG3u4TFKADg4OQe4I4NV63TTV0d8ZKSUou6YUUUUygooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAopVUswVQSxOAAOSa1V8PXhUEvCpI6FjkfpQZVK1Ol8bsZNFdDD4cjC/v53LEDhBgA9+uc/pVxdE08KAYSxA6lzk/rSucc8zoRel3/AF5nJUqqWYKoJYnAAHJNdDe6Aj7WsysZ6MrkkH3zzVePw5OW/ezxquOqgsc/pRc0jmGHlHm5rFCLS76XO22cY/v/AC/zqe40Wa1i8yae3RM4yS3X8q2INH8qFU+3XQx2jfaPyqO50P7Rt/02c7c/607/AMumKLnL/aCc7c6UfRnNuoRyqurgfxLnB/PFTb7T/nhP/wB/h/8AE1fsNPsi7vc3cTrGdpVSVUk9PmOM9D09Kj1iKxgkSK1TDjJch8gdsfXig7PrEJ1FSSd++3+RnOULkxqyr2DHJ/PAptFFM61oFFFFABRRRQAUUUUAFFFFABRT4YZbiVYoYnkkboiKST+ArpbDwPqFxuN5Iloo4A4kYnjsDjHXv26VnOrCn8Tsc9fFUcOr1ZJf123OXq5baVqF4Ea3sp5EkOFcRnaecfe6V6DZ+ENItoohLB58yYLSOxwx/wB3OMe3863q4qmPS+BHiYjiGC0oRv5v+v8AI860/wAEajcSOLwi0RRweJCx+gP9a37bwNpcRRppJ5yB8ylgqscewyPzram1O2hx8/mE9o+f/rVm3OqzTbki/dpngj72PrUqWJqvsv6+Z508xx1d/Fyry0/4P4l6C303RYGS3ijhGMkLy78nGSeT1PWq82tNu/cRgL6v1P5VlM7OxZ2LMepJyaStoYSC1nqzBUE3zVHd+ZMbmR43WWSVwRwPM4ByOo796hoorqUUtjdJLYKKKKYBRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQA5GCurFA4ByVbOD7HHNdXpcnhW8ZVmtDZzjkb5329+jZ9u+OvGa5KiuTF4T6xG3PKL7xbX/AZpCfI9k/U7q40m6t4vOt2+3QEb1aPG/byRx0bjHI5OelUIpY5k3xuGX1FZWkeIb3SGCo3m2/eFycDnJ2+h6/n0NdbEdE8T73iLw3YXLMn7uUDIHPZhwPXGe1eDX9vgn/ALQuaH8yX/pS/wAvxIngaVfWk+V9v8jlLvw1pF5HtayjjIBAaEbCM9+OD+Oa5fUfAt1CC9hOLgZP7t8KwGeOeh9+nSvS7jRb+0XcGW7jCjJjTa4POTtycj6c+1UVYNkA8qcMDwVPoR2PtXVh8dzLmpT5l/XzX4GUcVjsDKzbt56r+vQ8XdHjkZHUq6khlYYII7Gm17DfabZ6lEI7y3SVR0J4K9OhHI6DpXJ6r4G+9LpcvqfIlP1OFb8gAfxNepTxsJaS0Pcwue0Kvu1Fyv8AD7ziqKsXthdadcG3u4TFKADg4OQe4I4NV67E01dHtRkpJSi7phRRRTKCiiigBVYqwZSQwOQQeQa6HTvEsNtZeTeWJuZUzskEpUtk5w35nn2Ax3rnaKmUVJWZhXw9OvHlqL9PyOpHiKwaLcYrlJP7mFYD/gWR/L/Grkd/aSW6T/aYVVv4WkUMPqM5FcVRRy2OOeVUX8LaO9orjU1S/RY1F5OUjxsRnJUY6fKeMe1dJB4n0qS3UXVlcRTDqbdgwPHX5jxznjn61MpOPT7jza+XVqVnFc3oXqKzf7f01mVVa4GWALPEAFHrwxP6Vf8AOg3qi3Nu7N0WOZWP6GndHLOhUh8UWh9FFFMyCiiigAooooAKkSeWNcRyug64ViKjopNJ7g1fctRajdRdJSwznDc5/rVldamDDfHGV7gZBrMorOVCnLeJm6UHujXOsRSgxzW2YmBDDIbI9MEc1nT6T4d1BnzbtayPgB4ztx9AMqOnp3qGioWGgvguvRlU4uk70pOPox6eCdHlz5d3cvjrtkQ4/wDHaozeAJViYw6gjydleIqD+IJ/lVupFuJkUKk0iqOgDECl7Ksvhn96N44vGQd41L+qRzk3g7Wo5WRLdJVHR0lUA/mQf0qeHwPq0kSu7W0THqjyEkfkCP1roU1C7jXAnYjr82D/ADqZNXukXB2OfVl/wol9ZtpY2lmmOtZcv3M5afwTq8MYZBBOScbY5MEe/wAwArNm0HVoJWjfTrksOpSMuPzGQa9DTW/uh4PqQ39Km/tq2/uS/kP8azVXEx3jcUM4xkdJwT/r1PLZ7C8tYw9xaTwoTgNJGVBPpyKr168uq2jKCZCp9Cpz+lOnOn3UYS4NrMgOQsm1gD680fW5x+KBrHP5rSdL8f8AgfqeP0V6xNoGj3cShrC3253AxLsJ/FccVWfwhojRsotChIIDLK+R7jJxTWPp9UzePEGHfxRa+7/M8wor0F/AenGNglzdK+DtLMpAPuMDP51T/wCFff8AUT/8l/8A7KtFjKL6nTHOsE95W+TOKooorqPVCiiigAooooAKKKKACilVSzBVBLE4AA5JrTt/D2oz232kwGKHdsDyAgFvTp1pNpbmdSrCmrzdjLorch8OSFv386BQRwgySO/XGP1q4nh+yRwzGVwP4WYYP5AUXOWeY4ePW/ocvUkUE02fKieTHXYpOK7BdPs1UKLWHAGOUBNTpGkSBI0VFHRVGAKLnLPN1b3YnKRaJfSlcxBFYZ3OwGPqOv6VYXw7dbhulhC55IJJx+VdLRSucss0rvayMeLw7bKFMkkjsDzjAB/r+tWf7F0//n3/APH2/wAav0UHNLF15O7myv8AYbT/AJ9YP+/YqSKCGHPlRJHnrsUDNSVXvv8AkH3P/XJv5GgyUpTai3uc5YrFNr67QDEZGZABgYGSP6V1dc34fhiM01xIcGEDBJwBnOSfwFbMuqWMWN1yhz/c+b+VNnoZhGU6yhBN2SX6luisj/hIrT/nnP8A98j/ABqlN4iuHyIYkjBGMn5iD69v5UrGEMvxEn8NvU6SoGvLVWKtcwhgcEGQZBrkbm9ubvHnzM4HQdB+QqvTsdsMo09+X3G9da/JFeOkKxSQKQM85Przn61Vvtbmu4jEiCKNhhsHJPtn0rLooO+GCoQs1HVBRRRTOsKKKKACiiigAopyI8kioilnYgKqjJJPYVvWPg7VrqRPOiFtEwDF5GGQOP4Rzn2OOnaonUjDWTsY1sRSoq9SSRz9ORHkkVEUs7EBVUZJJ7CvQbDwPp9vuN5I92x4A5jUDjsDnPXv36V0NpZ21jAIbWFIox2UYycYyfU8dTXJPHQXwq541fP6ENKScvwX+f4Hnmn+DdTu5U+0J9lhZd3mPgn2G3Oc/XH9K6Kw8D6fb7jeSPdseAOY1A47A5z179+ldRVSfUbaBQd4kJ7RkGuV4mvVdo/gePWzfGV3aLsvL/Pf8SyiJHGqIoVFACqowAB2FNlmihXMsiqOvJ6/SsifWJWZhAAqcYJX5v8ACs55HkbMjs56ZY5qqeClLWbsccMLKWsma8+soARAhY/3n4H5f/qrNlvLiZmLSuA3VQSBj0xUFFd1OhTp7I64UYQ2QUUUVsaBRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFOR2jdXRirqcqynBB9RTaKGrjOr0rxpNbokOoRmdFGBKh+fv1zw3b0/Gujl0/Stdi+1QuN7cfaLdtr9jg++AOGHHtXmNWbXUb2ywLa6miUNu2q5Ck+46Hp3rwMXkcZT9thH7Ofls/6+7yOmGJ05aiujrbvStRsFLGI3kQ/jgX5+3VPqT0J6dqpxTRTLmKRWHXg9PrS2Xji6R0W9gjljAAZoxtf6+h78cfhXQG00zxHZrdwM0bMwLSw4WXIGNrHB9uPYV59SticJZY2Gn8y2+a/4b0MKmX0a2tB2fY5+aGK4iaKaJJI26o6gg/gawr7wdpN1G/kxG2lYlg8bHAPP8J4x7DHTtXaz+G5IrcLY3ZaQf8AP182endcY79j+FYlz9u03f8Ab7KQKv8Ay2hG6MjOAc9u3B55rqwuNp1f4E9e2z+57/ic3sMbhHeDa9Gcf/wr7/qJ/wDkv/8AZVS/4QPVP+e9n/323/xNd1FqFrKuRMqnuH4/nT/tdt/z8Rf99ivQ+sYiOj/I1jnGOju7/JHmv/CI65/z4/8AkVP/AIqoLjw5rFrt8zT5juzjyx5n57c4/GvVUkSRcxurjplTmnU/r1RPVI2XEGJT96K/H/M8Uor2uitP7Q/u/j/wDo/1j/6dfj/wDxSivZp7aC6jCXEEcyA5CyIGAPrzVf8AsbS/+gbZ/wDfhf8ACmswj1iaR4ihb3qb+88hor17+xtL/wCgbZ/9+F/wo/sbS/8AoG2f/fhf8Kf9oR7D/wBYqX8jPIa0tDj1I6lHPpkBlmiOenyjIPDE8DIyOo9ua9NTSdOjkV00+1V1IKssKggjuOKuVE8emrKJjX4gjKDjGnv3f6f8EhmtYLjHmxhiO/Q/nWbquhS2Ci4gZriyYcTBcYOcYI7fXvWk91bpu3Txgr1G4Z/Kq1n4rurGd/KiR7dmB2PnOB1wQeCfx6CuWDxcfeoq9uj0T9H0f4d+h5OCvdxnt37GBRXZya74b1V0+32DxyNgvLs6EDoWU7iO3T0qlLoOizxebY67CilThLggEtz16ED8PfmtoZpbTEU5Qfpdfej0XQ/kaZzNFbw8PWq2skk2u2CyKCVSN94OB65z+QP41huoV2UOHAOAy5wfcZ5rto4mlXb9m728mvzRnKEo7jaKKK6CAooooAKKKKACiiigAooooAKKKKACiiigBySPG2Y3ZD0ypxUq3tyjBhPJkerZH5GoKKlxi90JxT3RdXVbtWBMgYehUY/SpP7auf7kX5H/ABrOorN0KT+yiHRpvocFRXfL4J8PIoSfxI/mqMP5dszLu74PcZ71cgh8L2aeSmhPeIOkt1Nhzx3CjHXP+eKz+uKX8OEpfK3/AKVyp/I+pnmFGPX+vxPN0jeVwkaM7HoqjJNamneF9b1YkWWmXEgBwWZdig4zjc2B/ketdlaX7aZNI2mQwWiGRnQCFGZM8Y3EZ6cVC9xM901yZCJ2fzC68HdnORjpzWkpV5X5Ul2vrr5pW/BnJPNv5YmWvw319IHnvBaWUSKWZp5s45xj5A3J/wA80Wfgyzff9u8Q20OMbPItpZc+uchcdvWtKWaWeUyzSPJI3VnYkn8TTKyjSxEo/vKln/dSX/pXN/XQxlmtW/upEK6B4etrkApfXqRv94zrEsoB/u7CQD0+9n6drN5b6LIkS2OjRWoTO7dIZS2enLenP50yitI0IqSm2215u33Xt+Hock8XWmrSloRxQQw58qJI89digZqSiitznbbd2FFFFAgooqOWeGHHmypHnpvYDNA0m3ZElFZs2u2ULbQzynJB2Dgfnj9KzX8R3JcmOGJV7Bsk/nkUHXTwNepqo29dDpKjlnhhx5sqR56b2AzXKz6zfTbh5vlqe0Yxj8ev61RZizFmJLE5JJ5Jp2OynlMn/Elb0OiuPEUKqwt43d84Bbhfr6/yrMutZuruHym2IpPPl5BI9Dz0rPooPRpYGhT1S1CiiimdYUUUUAFFFFABRRRQAUU5EeSRURSzsQFVRkknsK3rHwdq11InnRC2iYBi8jDIHH8I5z7HHTtUTqRhrJ2Ma2IpUVepJI5+nIjySKiKWdiAqqMkk9hXoNh4H0+33G8ke7Y8AcxqBx2Bznr379K6G0s7axgENrCkUY7KMZOMZPqeOprknjoL4Vc8avn9CGlJOX4L/P8AA880/wAG6ndyp9oT7LCy7vMfBPsNuc5+uP6V0mn+CNOt43F4Tdux4PMYUfQH+tdNRXFPF1Z9beh4uIzjF1tFLlXlp+O/4jURI41RFCooAVVGAAOwp1VptQtoY93mq/bahBNZdxq80m9YgEQ8A/xAVNPD1KmyOCFGc9TZluIYc+ZKinGcE84+lZ02tJ5f7mNt/wDtjj9DWQ8jyNmR2c9Msc02u6ngoR+LU64YWK+LUsz31xcMS0hUEY2qSBiq1FFdcYqKskdCioqyCiiiqGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFb2heJm0W1ktzaiZHfeDv2kHAB7HPQVg0VhicNSxNP2dZXX9di4TlB3idfZeOpRKRf2qGM9GgyCOvYnnt3H410tlr2m36IYruNXchRHIwV9x7YPXr2yK8rorx8Vw7hK2tP3H5bfcdEMZUjvqesf2Tp7MzyWdvK7MWLvChJyc+lYd/4JtJVd7GV4JOoRzuTp09Rz3yfpXKWWvalYOhiu5GRAFEcjFk2jtg9OnbBrptP8cQuAmoQGN8geZCMr16kHkY49a8qeXZrgnz0J8y8v8n+huq1CrpNW/ruc5eaBqunMXktnKLkiWL5gAP4sjoO/OKo/a7n/n4l/wC+zXq1rqNle4FtdQysV3bVcFgPcdR171FeaNp2oEtc2kbuSCXA2scDHJGCa0pcSSg+XF0tfL/J/wCZE8DGWsGeXfa7n/n4l/77NH2u5/5+Jf8Avs11154FXBaxuyDgYScZyc8/MOnHtWBe+HdUsEeSW1LRISC8ZDDA74HIHHUgV7uGzPA4jSElfs9H+P6HLPCyhvEofa7n/n4l/wC+zTkvrqNsidyenzHP86r0V6DhF9DHkj2Jvtdz/wA/Ev8A32aPtdz/AM/Ev/fZqGinyR7Byx7E32u5/wCfiX/vs0hubhlIM8hB4ILnmoqKOSPYOVdgoooqhhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFUm1awSTY10o9SAWA/EA1lXfiGR8rap5Y/vuMn8ug7+tB1UsFXqOyjb10OioJCqWZlVRjLMQAOcck9OSK406pf+b5gvJkf1RyuPypk9/eXUYS4u55kByFkkLAH15NDT6HaspndXkjp01nThetbzTuiKSDMqb0yPocke4/lzVLUNfSK4eOx2TRgjEzAgMMc/LwRz/Kudoo5db3OyGWUIy5nd+XT1/rQuzavfTZBnZRnICfLj8RzVR5HlcvI7Ox6sxyTTaKo7oU4Q+FJBRRRQWFFFFABRRRQAUUUUAFFX9P0XUdUjd7O2MiIcFtwUZ9OSM10Nt4Bmkt0e4vhDKwy0Yi37fbO4ZrGdenD4mclfH4ag7VJpP739yOPqxaWF3fybLW3kmIIB2LkLnpk9B+NemWnhfR7TBWySRtu0tN8+ffB4z9BWvXJPHr7CPGrcQxWlGF/X/L/gnmtp4L1eecJPGltH1Ls4bjPYKTk/l9a6fTvBmm2YDXIN3KCDl/lUYP8AdB+mc56V0dFctTF1Z6Xt6Hk4jOMVWVublXlp/wAH8RkMMVvEsUMSRxr0RFAA/AU+kZ1RSzsFUdSTgVTuNUt4OFPmsRkbCCPxNYxhKb0VzzlGU3pqXaikuYIiweZFK8kFufyrEl1a5ljKfImf4kyCP1qkzs7FnYsx6knJrsp4Fv42dMMI38TNefWhtHkRnPcyD/A1Qnvri4YlpCoIxtUkDFVqK7YUKcNkdUKMI7IKKKK2NAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigByO0bq6MVdTlWU4IPqK2rLxbqtnEYzIlwvYzgsR17ggnr3zWHRWFfDUcQuWrFP1LjOUNYux6DZ+NdOnAFysls+CTkb169ARz+lb1veW13u+zXEM23G7y3DYz64ryCpYLme1cvbzSQuRgtGxU49OK+fxPDFCd3Qk4vs9V/n+J1Qxsl8Suep32k2GpbTd2ySMvRuQ3fjI5xyeK5698DQu7vZXRiBBKxyLuGfTd1A6DoT9ay7DxlqFqqR3CpdRr3fh8YwBu/qQTXSWvirSdQLQNJJbbhgGb5M8HOGB4/Me1eW8Nm2Xfw23FdtV9z/yN+ehW33ONvfDuqWCPJLalokJBeMhhgd8DkDjqQKyq9ih2GCMxPvj2ja27dkY4Oe/1qrfaTYaltN3bJIy9G5Dd+MjnHJ4rqw/FEk+XEQ+a/yf+ZE8Ct4M8nort73wNC7u9ldGIEErHIu4Z9N3UDoOhP1rnrvw1qtlFLLJbZiiyS6OCMeuM5x36V7+GzfB4j4JpPs9H+O/yucs8PUhujJooor0jAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAormf+Eiu/8AnnB/3yf8aqTareztk3DqMnAQ7QPy/rTsenHK6zetkdPfahb6exSdiJR1jA+bt1Hbrnms+TxHAF/dQSM2ejEKMfrXPO7ySM7sWdiSzMckk9zTaEu5308roxS59WbDeI7r5wsMIDAhSQSV9D16/hj2qhJqF5NG8Ul1M0TnLR7ztJ+nSq1FFkdcMNRp/DFBRRRTNwooooAKKKKACiiigAooq7YaRqGp7jZ2ryKvVuAueOMnjPI4pOSirsmc4wXNN2XmUqK6iw8D6hcbjeSJaKOAOJGJ47A4x179ulbun+CNOt43F4Tdux4PMYUfQH+tc88XSj1v6Hm1s4wlK/vXflr+O34nnVblh4T1S9MDmIRW0oDecXU4UjOcA5/D+Vei2GnWmmQNDZxeXGzbyNxPOAO59hVquSpj29II8nEcQzelGNvN/wCX/DnJWXgS1guA93dG5iAP7sIUyfcg5rodP0uy0uN0soBEHOW5JJ/E81corknWqT+JnjV8diK+lSba7dPuWgUUVDLdQQ58yVAR1Gefy61mk27I5Um9ETUjOqKWdgqjqScCsifWjuHkRjHcyD/A1SuL6e54d8L/AHV4FdcMHUl8WiOiGGm99Dbm1G2hXPmhz2CHJrPl1qTzD5KJs7bwc/oay6K7IYOnHfU6oYaEd9SWa5mnP72RmHXGePyqKiiulJJWRuklogooopgFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQBZtdRvbLAtrqaJQ27arkKT7joeneul0vxtJGCmpoZQB8skSgMTnuMgfljp3rkaK4sVl2GxStVgvXr95rCtOHws9Uste02/RDFdxq7kKI5GCvuPbB69e2RV9g2V2kAZ+bIzkY7enOK8crWsvEuq2UpcXTzKeqTkuD19eR17EV85iuF2vew0/k/8AM7IY3pNHoV9pNhqW03dskjL0bkN34yOccnisO98EWk0oe0uHtl7oV3jt0yQfXqT1osPG1pKqJfRPBJ0LqNydOvqOe2D9a6BZ7TUYnjhuUlUY3GCbkenKnI6V5HNmWWySblFL5r/I3tRrLozzy78K6vabz9m85Fx80Lbs59B979Kxq9lqpf6ZZ6miJeQCUIcryQR+I5r1MLxRNaYiF/Nf5Pf70YzwK+wzyWiu3vfA0Lu72V0YgQSsci7hn03dQOg6E/WudvfDuqWCPJLalokJBeMhhgd8DkDjqQK+hw2b4PE6Qmk+z0f47/K5xzw9SG6MqiiivSMQooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAOCoooqj7QKKKKACiiigAooq/p+i6jqkbvZ2xkRDgtuCjPpyRmplJRV2yJ1IU4803ZeZQorqtP8DXtxG7XswtCDhVwJCffg4Fa1h4FtLednvJ/tcZXATYY8HI5yG+v51hLF0o9bnn1c4wdO657tdtfx2/E8/q1Yadd6nO0NnF5kirvI3AcZA7n3FevQwxW8SxQxJHGvREUAD8BT65XmD6RPKnxE7NQp/e/0t+p5lD4P1l540ltfLjZgGfzEbaM8nG7nFdHp/gayt5Ha9mN2CMKuDGB78HJrqqKwnjKstL29Dz6+dYuqrJ8vp/nq/uM2y8P6Xp1wLi1tAkoBAYszYz6ZJxWlRRXPKUpO8nc82pVnUfNNtvz1CikZ1RSzsFUdSTgVXm1C2hj3ear9tqEE0RhKWyJUXLZFmismfWhtHkRnPcyD/A1Rm1G5mbPmlB2CHArphg6kt9DeOGm99DekvLeINvmQFeCAcn8qz5taXb+4jJb1foPyrHorrhgqcd9TojhYLfUtz6jczsDvMYHaMkVUoorqjCMVaKOiMVFWSCiiiqGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABUsFzPauXt5pIXIwWjYqcenFRUUnFSVmO9jptO8Z3tv5EV0iTwpw74PmEeuc4J/nj8a6my8S6VexFxdJCw6pOQhHX14PTsTXmFFeJi8gwmI96K5X5f5HTTxdSGj1PZaK8v0/xFqWnECO4MkYAHlTEsuAMADuPwx0rpNN8bQS4j1GLyW/56xgle/UdR2Hf8K+ZxXD2LoXlBc68t/u/wArnbDF05aPQ6G/0yz1NES8gEoQ5Xkgj8RzXM33gdSZ5bO5IGC0cBTPOPu7ifXua6u3vLa73fZriGbbjd5bhsZ9cVNXDh8wxeDfLTk15Pb7nsaTpU6mrR5Ve6JqWnxCW6tHSPuwIYDp1IJx171n17LWZf8Ah/TdSlea4t/3zLt8xWKn2PoT9Qa+hwvFK2xEPnH/ACf+fyOSeB/kf3nltFdZf+CJ4onksbjz23cROoU7f97OCfy/pXPX+mXmmOiXkBiLjK8gg/iOK+iw2Y4XFfwppvts/uepyTozh8SKlFFFdpkFFFFABRRRQAUUUUAFFFFABRRRQBwVS21tNeXCW9vGZJZDhVHevQNP8EadbxuLwm7djweYwo+gP9a37SztrGAQ2sKRRjsoxk4xk+p46muKpjoLSCueriM/oxuqKcn9y/z/ACPNv+ER1z/nx/8AIqf/ABVKnhDW2kVTaBASAWaVMD3ODmvT6K5/r9Tsv6+Z5v8ArBiv5Y/c/wDM5nT/AARp1vG4vCbt2PB5jCj6A/1q/D4X0a3njmis9skbB1PmucEHI71r0Vzyr1JbyZ51TH4qo25VHr52X3LQKKKKyOQKKKjeeKNsSSoh64ZgKaTewJX2JKKozarbRSbMs/umCP51Um1p/M/cxrs/2xz+hraGGqy2RrGhUl0Nmojc26sQZ4wRwQXHFc/NqFzNJu81k7bUJAqtXTDAv7TN44T+Zm4dat9pxHIT2BAH9aqf21c/3IvyP+NZ1FdUcLSj0N1h6a6EjzyyLiSV3HXDMTUdFFbpJbGyVtgooopgFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAEsFzPauXt5pIXIwWjYqcenFdDp3jO9t/IiukSeFOHfB8wj1znBP88fjXM0Vy4nBYfEq1WKf5/eaQqTh8LPTbDxNpd+jt54typxtuGVCfcc4NbFeNVo6brl/pbqYJi0agjyZCSnPtnjnnivm8XwwtZYaXyf+Z2U8b0mj1SiuTsPG9u8SLfwvHKWwXiXKY9eTkfr0/Cumt7y2u932a4hm243eW4bGfXFfNYrAYjCu1WDXn0+87IVYT+FmTd+EtKuIpRHB5Er5IkRjhT/u5xj2/lXO3/gm8t0RrOUXZJwy4EZHvycGu+pqhgPnIJyegxxnj9K6cNnWNw+07rs9f+D9zRE8NTn0PJLyxudPuDBdRGKTAODg5B7gjg1Xr2N0WRGR1DIwwysMgj0Nc9qXg6wu8Nan7JJuJYqCytnttzx+H/6vocJxPSnaOIjy+a1X3br8TkqYKS1g7nntFdDeeDtTguClsguYsAiQMqfgQTWA6NG7I6lXU4ZWGCD6GvocPi6GIV6M1L0/Vbr5nJOnKHxKw2iiiuggKKKKACiiigDrqKKK+dPGCimPLHFjzJETPTcwGahe9jXHlq8/r5IDY+tUoSeyGot7Fmisq71WWGUIkBUjr5o6/TB6VRn1G5nYHeYwO0ZIrohg6ktdkbww05am5PeQ26sXcFlxlARu/Ks6fWjuHkRjHcyD/A1lElmJJJJ5JPekrtp4OnH4tTphhoR31LM2oXM0m7zWTttQkCq7OzsWdizHqScmkorpjCMdkbqKjsgoooqhhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAVLBcz2rl7eaSFyMFo2KnHpxUVFJxUlZjvY6bTvGd7b+RFdIk8KcO+D5hHrnOCf54/GunsPE2l36O3ni3KnG24ZUJ9xzg15lRXi4vIMJX1iuV+X+R008VUhvqey0V5bYa/qOnSo0dw8iKu0RSsWTHbjPGPaup0rxnb3Hlw36eTMzY8xR+79icnI9O/r9PmMXw/isOuaHvry3+47aeLpz0eh1NV7yxttQtzBdRCWPIODkYI7gjkVLFNFPEJYZEkjboyMCD+Ip9eIpTpyutGvvR0tJo5i/wDBNncOjWcptABhlwZAffk5Fc1qPhjUrDz5PK822i581COV9ducj39Oe3NemUV7OF4gxlCyk+def+e/33OaeEpy20PGqK9YvtJsNS2m7tkkZejchu/GRzjk8Vzt34FR5ZXtLzy1OTHE6ZA9t2emfb86+iw3EmEq6Vbwf3r71/kjkng6kfh1OJoq9f6NqGmIj3dsY0c4Dbgwz6cE4qjXv06sKseam013WpyuLi7M/9k=" - } - ] - } - ] + "outputs": [] }, { "language": "markdown", @@ -143,14 +82,14 @@ { "language": "typescript", "source": [ - "const alpha = 0.3;\n\nconst pixelsAsImageData = arrayToImageData(pixels, W, H);\nconst mask = cv.matFromImageData(pixelsAsImageData);\n\ncv.resize(mask, mask, new cv.Size(originalWidth, originalHeight));\n\ncv.addWeighted(mask, alpha, originalImage, 1 - alpha, 0, mask);\n\ndisplayArrayAsImage(mask.data, originalWidth, originalHeight, display);\n" + "const layers = { bg: [], c: [], h: [], w: [] };\nconst resultLayer = [];\nconst colormap = [\n [68, 1, 84, 255],\n [48, 103, 141, 255],\n [53, 183, 120, 255],\n [199, 216, 52, 255],\n];\nconst size = outputData.length/4;\n\nfor (let i = 0; i < size; i++) {\n const valueAt = (i, number) => outputData[i + number*size];\n\n const currentValues = {\n bg: valueAt(i, 0),\n c: valueAt(i, 1),\n h: valueAt(i, 2),\n w: valueAt(i, 3),\n };\n const values = Object.values(currentValues);\n const maxIndex = values.indexOf(Math.max(...values));\n\n resultLayer.push(maxIndex);\n}\n\nconst pixels = [];\nresultLayer.forEach(i => pixels.push(...colormap[i]));\n\nconst alpha = 0.6;\nconst [N, C, H, W] = output.getShape();\n\nconst segmentsImg = Image.fromArray(pixels, W, H);\nconst resizedSegments = segmentsImg.resize(img.width, img.height);\nconst mergedImg = Image.overlay(img, resizedSegments, alpha);\n\nmergedImg.display(display);\n" ], "outputs": [ { "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAQ4B4ADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDH0NljmMUwKuDjBFdHjFYEwW31yIuhAY454xkV0LCuo1I2QHtmomQAYwPyqY0hGaaBsrspPTNMAINWCMU0pmgQinkU8VHjBqQcgUBYUClFKKUClcBCQOpxUJu7cHBlx+FSMgY81QvZpB/o9tESxOCxHAouBpDY6ZRgQfSo1QqeFrI3XOlgBVLyEZOOhpLLX5J5ZTNiNFBwBzzRcDYuA7wFJEBU9SegrmvEKBrQRrCWMYyGB/lWbrOq392Cod0j7AHGRWBM9wsYJuHJPVST/PNFwNWKB/sQM7HZ2B5xVMQvaF5Ui32755ptib+8eO1UHacnkdBW3e2ktusdnkHzSACO5+lTYopaHbkymaVmWPOUYngH0qWa2nv76RS+FBxkdDW5F4cmhsBanEgPJJOOvtWdqmi3dlaF7e5Z1GcoSMge1FyTFt4bY60tpeyLEitgs3Q8dPauyvtL0N9LaOCCCRuoMbZbNcfZadBqiSSTXLm5APyE8k+oqa3tzp0+Ltmhb+FweDRYoDHNdl7ctIqk4yRkCsu5WOwmeObFwQMDcT8v0rXupraNXuLe8l8zjgAkVzF1ILiYyA5zyT6miwxOSMsxOOBn0oXe2I1JIPb1qLJ6GpFO0bgSCOhFAD2hMO0kYB4/GlMcm4KCik9CelMeRpnDOckDikk+bBzjHqaQEqWkkc7SSKNoB5ByKs2+oIrCGUHy+zDkk1SjeaIErKRkYPNRxhNw8wEr3wM0AbZgubmPNwcwgExk9AKqWF1JY6kXjfAwVLY4Ipl2BGF8m4d4yM4PampMsVsxXGTjgjrRYC4v2U3ZmuJhIr537TzmiG3059U8syMbdl4J4ANaej+H47q7tJZ5Ee3mUkgHBB96yNRtEtNektUYlVcAZo5SbGh/bENpYSWBh8wI5CHII/8Ar1Te7a68smNYyOEI7+xrc0270kwlJrASX8QIYqvDD1x0rndVmju70vawmGMfwEYINFxGjapqKq0sMhMaHmEtkY9cVv2FzLqOnJYhC3lncxPHTsa42Od4U/dsQ3cjuadHf3gVwlxICxycdzRcZ6T9tTTbQRiyFuHIUOpGCTT7XQ7VtN8uWGOSQnJPbNee2N7FFcg3RlB6kEkgn1IrQttTvTqDT2ss0MDDoSdpPt2q7FF/xNDa6ZCtlaxtCzkGQYwCPXP510Om6fFYaJHDbyZEuCzjuT1xXCX+pTXGsxSSymZYnBy3b1/pXbWNlHbrJdz3Qa3BEkaZwFJ/GiwFxdIsoomeGEo7Dlie/qazLnQNSulkX7akcDDkAZBpLrVhqMJNrexwMp5HUMe2f1qjaX+qwzFbp8QuMMB0J9aLEmTHpgiDC4hUsCQCDzx0NNm04TN5hmAB6bqL+5UTuq7nkMnX0FVrq0u0tzI8UhA7DoKzsUOjO+4e0klMkZwFYDOBVC+sxbzlAwI9u1T28iRlShKuOoI7VO5le4iMiJtJxkevv6UXAxrcpHdorHKE4PtUVxGY7iRSOhrXliUawSVwvrUT273E5IiZgRkMBwaLgMski2gsoYEZIPUe/vVgxj97EvzhBwe+KLmyFkIbiB2BYYI7gikSTyskgMZF5U+v1ouAseof8S9IQDvjOARzVmS4FxdIpUspTAI7Ed6p2kRQSs3Cg8nsKu2cAW+GZgeMjA6Gi4EMbPaTTRycZXGPUdjWrY/Z4dPAaTd5wyfrUWpQeZbh5kBkAwrr0+hq3pNkrQlgF2vGVcnoD2wKSQrmnYW3/EoMshJkYHBJ9On9auWWTZxknIIyKgs3LWAtmOHj+Q++O9XI1CRqg6AYFa2EOAxSEd6XpSVQBS0lFIQtIaWigBhpMU7FIaAE6mk706jGaBiYoHSlxS4pgNIoApxFFACE4GaIpA2D0prjNEaHOcUxF0KCgIqJhzxT0JC4oIpDIGBqMip2FRleaQiLFIVqTbSYpDGBaULzUgXigrQAzHtSFR6VLikK0WAgI9aaRU5WmFeaAIcCkIqXbxTSuKAIyKbipCKQikMjIoxTytGKAGYpCKfigigCMiginkUmKAG4pMU/FJj0pgMxzRinYNGKQxtJinYFJigBMUY4pcUY5oAQClxQBzS4pAN70UtGKAE7UU7HrR2oAQUCigUAFKaMUUANFKOtFFIB1FAooAWiiimAtA60UUAKKUelIKUUAOo70UUgFpRSUooAWlFIKWmIBSikFKKQx1GaSlFAx1AooFAhe1KKbSigB1FJS0AOpaSlpgKKBRQKQCg0opBSimA4HmnimCnCkA4dacDTM04GmA4GgU2gUgJBUg6VEKkHSgBSaQ0GkoAKQmlptADwcUCk6UuaAF4pQabmlBoAUmkNFITTADQKTNGaVwH5ozTc0uaYBmjNNzRmkApNNNBNIaAENIaDSGmAUZpKKAA0lKaQUAAFGKWlA5oAQClp2KQjmgBlFKaSkAhptOJpDTAbQaMUGkIbRQaSmMXNFJiikIdmikpM0wHU5KYKcpoAkppNOzxTT0oAQUZpMmkzQA6lzTM0ZoAcaBTc0ZoAdS5puaM0CAmkNFFAwzRmmmigAozRmkpiCikopDFozSUUxDs0A03tRmgB1JSUZoAU0UlAoAXPFHFJmkzTAdSZoJpM0gF5pM0lJQAE0E0lFMBKKKKQgpwoHWigAoopaAFopAaXNMAJoApCaATTGO1uB5o0dBypFaanMSHvimkZpQMDFIAJ5pM0h60UAGaMUuaM0XAaVB60BQKWjI7mi4AKR5NpxUMzkDgkfSmS3MQhASQFzzjPNAExcr1FNAzJuqC5dJbJCX2uDyc1VN0VKIXDcjnvSCw67keTU44c4Q8VVuNMFremIDEMg546+tWL+QR3lvMeEJA3dhW4Yo5ohvXcMcHPQ0wsYt3p0Igt4kQFlIIJ5JrL8SaHBK8EibIjgbweBiunMYe4HzEYGOO9c34ubD2ocgxs2Dg9RQBUtozZ3Cran7QSOgPI+tVAbl/EEKSxkyCQEqDxiuqsNOt9Nt3a25LDO8jkf5zWLpsLpqtxezRsWyTHgZ5PagZ2kkY2ZLFQAASK4zXb3ULpza2cKqi4wxPJ4rds2untZhNkZORkdvSsjXNOu3Mdx54UMdpUdhx0pNCZwrtcW9wSSUlUkEjtVqSS5vo1867D4OfmrQu9GJimkTKqnTzOC30Nc0u0thwQfQipA1Dq0llCbQCOUAEb8etZG4MSRUrCEdBzTGBOMAcegouNMYAScGlKsTheR6Uu04JAzg0qEqQw60DbGFSjgOCM06VDG+MgjHBHekkMkr4wWwOwpiYaRVZioJAJHb3oAlt0Nw6xNyvf6VY2rZ3BxGJEHQkc1HPbm0fAkD5GQwHNQglgd7Hd6igDSuLGKaP7RFcDaOqYyQKzGVVbG4Nj0pY0dn2gnB649Kd5adsmgCSK5mhx5czKB2zx+VOTfd30Ykfl2ALYqMQuxwBn8aeLYjBeTYPXNVcDedY9HnnjQeYCARITkipNGtrK/gvLvUQSUPUHGB/kiufaMrj98WHuc1JGZlRhCWG4YOPSobFYu6na2qXAOmTGW3YZIPY1VtYBJOyKwWQAkA966TR9C+1aFIzkxyMSVJ6HmoLewsbu2MNwDDexMAzA4DDOD/OhMdjnmgluHEOP35OAPeu8twYvBtza3EIjmgQqQR36g1qReDdLsAt4hcSKN2Sc81x/iLVn1K5+zWaSHBIZBghsdM+netESaPgjRkv4bs3dqxWVMRyEcY74/SpbvTtO0vT5re/uG84AhUDckDoQKqaBcizjhhu9RnhRiSYwcBT2Htmr+o2elWhe/juzPIwOxJG3DIFDQ7nFJHayWzyPLtkHQHgkUgupriNUe6kCDpz0FWYL+wmvppbyzCK68AdAfWs+WMZyvCk4H0qRm1NHElpDcW8qmdBkn1FS22q6hdp5YjQbCDkDr7Vjy3EohSFscAEEDGauq89vYJPHNEVI2MVPOPUikwEvLlmMhMMYkJyWB/pVQSsFQFyOc8UTxEvK0cm4KM5XoarRvvA+Yn61LQGnaE3LSNJIQQcZA61DJKYZ2RJXAU44OM0tnOI3VGA2k8kVDcxS/wBpMArMOxAoQFhDwPMY/OScHt6U0pkbx2JFJNKHKbRjbwacFYqVBxxkA9CfTNAiRLkeTJaxp/rAMlhyR6UxAbWYxk4OD+VLbL5kgjlwvJGQORSXEsJJZJNzhgMkUAaTO81pGTKpBABHv6VespJIdN2yfLiTGepxXPo+6ER4+6c5rX8ySV45ipKSAAgdAaqO4mb8TKApGMkdR3qysmTjNY9tKSmTwemPSr8Lg9a6miUy7mgGmBhjrSg56VnZjuOzSg0zNOBGOTSaYx1FFFACUhFOA5oIwaQhhopSOaTFMYopaBRQAYpQuaSnA0AIEBPPSpQABgUzNLmgQ/NBpAeKDQMQ0xhzTzTSKAGYoxT9vGaAOakBAMCjFSACkI4qgI8c0lPIpCKQDCKQr7U/FGKQERXFMYVMRTCKAIiKaRUpWmkUDI8UmKfikxQMZikxT8UmKQhpFGKcRQRQA3HFJinYpCKAGkGkIp5HFNxQMbijFOxRigBuKTFPxSYoAbilxS4oApAJijFOIoxQBHSYp+KTFACEUd8UUYoAKXvTcU7FABSd6WigAApQKAKKACjtQaUUAFLSCloAOO9KKSlFAC0oNIKUUgClFJSimA6lpKWkAlOFIKKYDqUU2lpDHUoptKKQh1FApaYC0tNpe9ADhSim0tMB2aKBQKAFFKKQUtIBQaUU0GlFDAcDSim04UwHA0opoHNPAOKAHA81IOlRgYqQHigApcUuKKAGmm0402gAzS5ptFSA8UU0U6mAhPFNzQaSgBwNGaaKcKAFpc02imAuaaTQTSE0gF7UlLmjFADaQmlIoxQA2gdaXFGKYAaTFLigDmkAYpenSg80oBxQAmTSEnNOIppFADSaaTTjTTQAGmmlNJTAKKMUtIQ0ikpxpuaYAaTNBNGaADNFNooAdSg4popc0AOBzTjTAcUuaADNGabmjNACilzTRRmgBTSZozSUAOzS5puaXNABmkzRmigApM0E0lABmijNJmmAtFJRQIM0ZpM0ZoGLRmkzRQIXNGaSigB2aM0maQmgAzRmkNIaYC5NJmkzRmgBc80ZoopAJQaBRQIKUCjFKKYBQaWkFABS0lFIAozRR2oAOtJ0pM0UwNbNGaQ0lMaFozTc0ZwaQD6M0mc0UXGFNkIVcn8KceCPfpTZRlQD0zQIgjuI3maFxtYDOWOAax9V+z7DIjlXBzx3rVNiLi6SQNjb2zjNc1r8rDUzbJII8HIJ6E8cUAR/ayDGHZgH6bumar6lfSWTKIzhz+INY2pXssziFiPkOQQMGqIlnmlCbyTnIzUFG7Jr9+IDbzIjAgEEjkH2rZ0rxBMLZI5JFXAwC3INcldSy4RZipKdMVXa6kIKo2B9KAPTo7wzYaJkcjkhDyBVTxTGlzoQuGUq0ZyM1z2iCOSyLhzHKvcHFdBey2l9pJsWuQGc4yeSPxqkxGfpd697p6ol8I5l48t+9S3GvTadttL+IKQQUmi5BI//AF1La+EbOxtJZJ5WlIQkMDgAj6VxbxXeo3DiNWl8vOBnJAoA9BHiK3mtmSNWd2XIbIAz/Wsm9u0vLa2YXDlUOJFUHI981ycNwdOuwJLc8Ahlfiuj0bUpI2eKGzAhc7g79Qe9Mkta7a2Ulnbpa6oHY5yrvkfiO1cjqhBdSxXzQMHb0NXPEVt52sN5MQB6uR0I9axJlIYA9R1NJsCTyidmCCWOMCpprC9t03zWzovTJ9aqSAiMFJGJPbNaJ1C7lsxDJM8isNvluOPrmpGirGCyk4IIGTx2qNj8pI4INIzzR5Rsge9NJOMZ60DJLCWSO4LA4IBxxxTblxLNkAZI5wMDNRVLbMiTgyAlSCDjtQMYgAOTVzTooJ7qRJmGwJkA8c1Tl2+YSjZXtxipYIy+4AjIGcE8n6UAS28gjuZAmSACOByRU9xNaSQqqAoy9Qep/GnxWvnMRbr5NwAS2/gEd6fDpMjMZr0FY8Z3g8EUAUUDna8RAHYk0SHIKyEHuCOxpcLG3lhw0Q6MOpoMdoSSJGGe1IQluYQ+ZULAggEdQfWr/wBnurdlEJB804Vh1yfWqOLdFJVySPWultpI7rQ9x2rNGpIPrzwaGMkutV1HQI4IWKOCOhFUr7WtNuv3yxyBiuDhMAHvVpJIdQ0iK6KH7TCCMnoT3rl3kUrjPPcdhTRJqtreoi1+zw6pM8Lrjy2xkD0zVnR7u102zmModLoggFlyc+1c7nC7h1BGKvJqLvJEbgiSLPzAjFVzAdgbKO/8GC4KF5GJLsOuc4rkLmxENrGzySRlgSMjjPf6Vv2XiNrK+ltreLfZTYIB48sngke1N1u3tIbQ3EMv2kqQxJPBzx0NFrjOSjXIBJNXFWARlXYmQnII6UXE0EzKIIfLGOR0ANJbWc0zExx+Z5fJwcYFS0MuW+nxXEM5eQo8YyPcVBp8+2GWJkRo5D8wI71J9pmuJTCjKm87RnjB75pwsGtCEkYNuOAR0zUgRaWQhu13ADZgZ9RnFQQ2fmRtIG8tweDnj8qiuIXtrh4pBhgc49jV6IJJpbCPIYjJBOQCKAKoWeOUgISykEEdCauT3E1s0c5Ty5iMkY4cVXtJJDfW6dVYYceo71cv4nuAEt2JhiHIPY0tgK00b+THOgLeYCSAOQfSiYBY0cnBIAqxHdILVI8MJUPAx1qaG2S/iIzgjO1SO/1qwK9jE00crsAQAdoB5yPWrFnZJdWytKu3BIJHU/WqQ32kk8QbGQRnHWrdtIVs50Lc7OOOtOxI23iVldQwbDEcda2licafHCDyDk1Bpejjyo5pgxZxkg8DHbFavlCNQozgdMmtKcUndktlaNdiH3JNXY4Z9gIXgjOahERdgAK3LdR5YBHQYrduxJmMJojhgfwoS5I4JreCKRyAfrVCbS4XbIGKi47FL7T7inpcjuaq3VlLakHIKHpioYyznAGauyEa6SgjrUoOay4HczhT+NbBjIAx3FZyRSdwUZNKVApVwDzQxBqBsjIphp5601gewpgIKd2piZP3hipAKGgEozTsUnFKwxDQDzQRmgYXk0xDwMU4mojMC2BTi3FIBaKTNGaBijpSimg80oNAMkC0rDigEYoJyKQEZpMU4jmkoAQikIp+KMCgCPFNIqYikwKQyuRzTCKmYYzURFNiGEU2n4pCKRQykxTqTFIQlFLRQIbRS0hpgJRilxSEUhiYpMU7FJikMTFGKdSU2A2gZpx60YpAJSYp2KMUANpCKfim45oAaQaMU4ikIoASjFGKPagBKWjFGKAClAoxQBQAmKKdigCgAo70YoxQgClFJS0AOHSikpaAAUvam04UALmlFNpQaQDhRQKBTAKcKTj8aUe9AAKeKaKUUkA7NKKbTgaYC0gNFApAOBoFIDS5oC46lpBSigApwptKKYBmlFJS44pAOBpyjmmgVKoHFMByj1qQAYpKUdKQCYpwFKBk0uMCmAgoNFFIBCKKU9KSgBtJTqSkAgpc0CloAQ9aZTyKTFNgNp2aMUGgBaM0lJQAUlFFABThTacKAExSAc07FGKADFIRS00nmgANA60hpRQA4DmnU2gnigAJppNBNMJoAQ0hNBNNJoACaBSUUxDxRmm5pCeKAAmkozTSaQCmkpKKYAaBRRQAUuaSigBc0ZpM0ZoAXNGaTNGaADNBNBNJQAtGaM0hNACg80ZptLTAdSZoopAFJRSUAKTSZoJpKADNGaQmkpgLmjNJRQA7NHam5opiHA0pNJRSACaM0hNJmgAzQTSUGmIKKKKAFoxSgUuKQxBSiiigBRRSCloAMUlGaKBATxQTSZpM0AOzSZozSZp2GFFFFIRqUlLSVQISlpKSkMcDSO+1SxOAKTNBAYYIoaAp3Zku7ZghZJEIZfrRY6h9rj8mUFbhOGBqLVr2TT4k8kDJ6k/hWQNdjtC01xEgmIwAO/6UAdFc3EllGJRF5gzgiuW114dX1CGPyRDJjJJOSenSorrWLzVrGWSK5iRV/wCWJ6kfWuZe43YLM+4dCD0oES6zYjT73YpJBGcnrUV3YvZQxTlwVcZBA6Ust+90VNyxk2jAJGDii4u1ngEOSEByAR0qGUmU5mDnIcue5NMQlXqxb20k7FUAJHqcVJeWEtpCJHxgnHBoGNjuBHkbyCOhFWPnlthIHck9cdqpC2dolkAyCcVZQvbxgxyHPcHkUhFgatqSwG085jGexJOKqxNcW0wlSTYw6HNakU9u6AgIXPUP1rNurZI5gfM8wPjI9DRcZfn1czRssojmdsEEDkEVmtqOovGYg7hfYAGoUKwXAJGAOtWGu5C4eDpjB561SYmi3ba01vbCGa3EjgEB364rIkJZiSepzVtR5822Xckh6E8CqkwKSFCMEUMLC28LzSYjAJAzgnFEk+G2scEU1XdDlDg+tbMV3YCyJm03fkYZw4Jz60h2MVyZGyzE/WnFQB1BqIlWcsgITPAPpS7jnApABHPFPCAjlgKjLkdTTgkrxGUISinkgdKYDTgU9YXMXnxkDHUHqKeLcm2EwdTn+HPIpkUrwyKwPy5GR2IoAvrOLiDyriN/PYAq3TI9qqmSaNTC0ku3upJxXVfYIkv4mMTPbuMo+fuE/wBKl0yzmaa9TyPOjjBMb+vtVE3OP5HJBC+ppTjsafdyvJNJGyhVDEBB2HpUBqBlkC3kY5fAHVcf1qeRzGBCMhMZBAziqC43AE4B71cU7VA3Ej1NAC2t29vC9vJIyKTkMB0PpUQj+23qQoVGRgHHXFEzqzkZ+X0ruPBnh62GnPf36IS7ZRieVA/zimBxmoaTPp4jaQq0cnCkev8Ak1TC4JVxkeld54g0y5vroJDJFJBGdyhDyDxXNanHFd3AFrHzGArcYyaAKKandxEpHJiMADGOcD3/ABptzcRzzeYgYZGSCeKnWxCWskso2yY4HXmoZY7dbWGWN2Ex4kGehoAjRS3QVbtvMhSTBIMgwAD3qqHki+eMNke1aGEjs96knIBIJ70gGyFLh41TAkC4JA7io5ot1mZvOdyDggnjNMkIEgZDhgeeKlFtM0JnKkRn0PBNIDS0m4glsLg3oQuFwDIMkj0rEt5Nm5AcAk4qcSFY0A6EkEeopZdNmjgjuIoyyN6daBklvCATNnDxnlAO3qKvWEa21vcSecknnISAp5A9xWQJZULtyG6YI4IqZYTEwaKVT6ikwC1kCXoYLgYwBnpWxpizfYzMIy8ZkI3A8ishE3OFH3z0NbWn6rb22lw208DP5bkkqeQD7YppgT6/pMSCGaE7SeGOM8HGKgbQ3t7FpH+aaPHmITjAPcV0en3VlqdvIiAOvTkcgdqsNYPJIAwBhIAYZ4IA71okKxQ0TUpLo/ZriALsQFG6ZHTpWjcWqsSwFVbS2+z629uo/dpHlB3APbP4VsFccVSdhNGXFbhCXxg9Kt25YjDAD0qYxjOcUBQOlNsViVcY4oK5poOKeOlK47EFxCjwkMM1QhtERjgdTWnKP3ZqpC2WxVJisIlgUlEhHAqZ3GcVZzlRULRDkmhu4JWK7yxqcZpSacIrYfM4pFeJnwEwPX1pAIBQrjcQB+YqyQI+o4qu9xC74jU59aYIVsBTxzUJznOKkcHGRQjKRz2FADQScYoIIGSKRTg5Aqwgyhz0oAqhgRkGpVxjnmkkQZ4GKQEg4oAi8lVcsM5Jp1SGm0AANKTTe9L2pDEBpwPFRlgDUQmx3p2FcthsVKCD0qksmT1qyjYGaGh3JOMYNMIoLHNJnmkAucUZpMikPFADs0E03PFJmgBr5qI1M+MfhUTCkBGRRinEc0mKQ2NIpCKcRSYoBjcUmKfimkUANoIoIooGFFGKKQhO9JS0lIYUUtHagBBRQKWgBDQadRQA2jilpcUANIppFSYppFADMUYp+KbigBKKXFGKAEoopcUAJ2oFLigCgAxRilxS0gG0oFLijFMBMUAUUYoAcKKBRSAKWkpaAFFKKQUCmAtLSUvegB3aikFOFAC0opM0UAOzRmkxS0gFopKWmA4UopBRQA6lFIKUUgHAcUooHSnAGgBVFSDimqOKfg0wF609RxTB1pynFK4EoAxQRTQaXNACYopc0lACHpSGlNIaAEoooNABRRRSAKSlooAaaSlPWkoAd0FNpT0pKoAzRQaKQCCnCkFKKAFFBpaQnigBDSGkJ5pCaAFoBptFAD80hNNzSE0ABamk0E000ABNFBptMBc0maSii4C5oNFNoAXNNJpaSgQZpKDSZoGFGaM0UCFzS02jNAC5opM0ZoAXNGaSigBaSiigAop2KMUAJRS0UAFJR3pKAFpCaCabQAmaM0hopgLmjNJmjNMQtFJRSAWlzSZozQAZozTc0ZpgOzSZpM0maQDhRSClFMBcUAUUooAWg0ZppNADs0ZppNANIQuaM0hNFNjDNGaQGgmgQuaSkFLmgBc0lGaKBhS5pKUCgRp5oyKbmiiwC0Zo7U3NMY4UCkBFKOTxQBT1K2FxCpIyy9vWs7WtJt5NJWUx5kUAg56VtXN5bWcYa4cJ6Z71R/tIXcR8u3Dwn+IH+lJgcs2jCSyeWG2kBA4KnJ/KsW30eW4sZrkSgGHOY88mvSrNBDAzyMFBzyTgVizDTYreeFLyMGU5yDipEcWNHum0s34AEY7E8020snuIDgHcBkYrsYbeKLQJrVZ43wCQR2zUug2dtJoxjCYYA9+QaQ0cFGXLmNN27vjrUoTfJsu5H2jpk9DViExWWsMZoy2xiMDrWpd6bZXNjJdxko2MjHX8aBmYlqIYAZsSW45DIeh9TWedomKoSV7ZNa9rp9/NaGazikKgZPcEelZ0SKtwCVMbqRkHsaCbksjw22AYdxIyGBzg+1VBMTLknNdPf2Vlqemo4jFtcJySOhB6H9KxtQ0tLK3WSNyWzgkjg0FJmXKcsTT45CFCBgD2J6Va+2mOyMIReKorFuXLcD1FCGXZpiiKQytICKq3Ekkjl5AMn0FMCoOMH8TUw8poipGD2OelAFbeexpFRpZQoIBPcnFOZQAcHmm4PegCSWBoCAxBz6HNEcYkiaQHocYxT7SeK3n3zQiVcYweo+lWvkW1eaMxneSTGBytAFKNiGIChiRgA1r280dpZMGKmQjlT0IqjpNva3WoCG7lMcZ6EHHPapr+0EMskUMTP5bcvnOBQBSt4vPlCbiCSSAOn5U+ayeOJpAVIU4PNW7K0jju2iukYSFcxlT3pv2TUJ/MIjkW3ByXkGABQI09CvfsrC3uo5gz8K5b5Mdq6TSb+4huprMOptUBIKjkHHr3rlYdamhsjZPHDMACASAeKW31cf2eLSOExzHjeTgH3PFVcmxlXg2X1wHbJMhwSOSM1WYnjHriuml0Gyt9IN9NcGWYA5UHv2Nc2yVLRaNDUdFu9NSCWUq8cnIKHOPXNMvZoXkRoGyCORjoa1NH1mIWz2OpNutyuFZudo9Kw544o7qWOBxJGGOxwOopWBliCNpJ42WF5lUguqDJIHWvWbRLCTSY7mwgUo+OpwAfeuR8C2bWniKI3MgjaSNiiZ6nHQ1u+IdGkt9QQ2RMMNycEIcLuHqM0wMnX5ptNmP2SyQRvgefEeo7jA6Vh25sUmDrOQzKQ4fnn1rpfEUL6XokcNvBI743NN2B78fjXDzW7qilpl2uMgjnFAD7qzuomBlfPmHgZzkVA9vH9nYBgkiHOD3pwuJnws0hkWM5UjtVe7IkYSjnjBoAti/E1g/mSZmAwBjpUFs7OuxjkGqgfKketOQlehxSaAv585nO0ZBwSKlt7tlAhPK9xVYMApAPJGDx3qNQUkBBwfWgRoL5ck0ZWM/IckE44rWvdWjt2R7dWCnhk7ZrnHkeGRJNwbnoOtWLgNMpfIOe4NADr0+dMZFQKSASo6AUyziSWSQsxAIwCO5qawkQBvMKmQqQCegFRWMyQztHMAYW+U56j3FJjLBtHWMDco5A3E8CtG3torfVLMkYV8hiTkHIqjGMXZt2k3Rn7pJ6jtV6W1lkght0OCjZEncD0pCLEcZ0fU5YHjMkFz80ewnIwa6zTbiS4s42lBDHOSe9c5pL3F7d2yzJmeyBJJPEin09625fOt7iaCEkDAljAPUdxVxGaXlr5/nEfPjaD7U40K4lhWVBwwzj0oqwEIoxS0AUCYgFPC8UgFPAoACmRgiqhgWIkj1q8KZIoIoTEV1lUcVHcTKEwD1pJIyuagkAKYIBq0IjMsfQtk+lKMqcjj09qqzKVyR1FTI0r2OT9/OKuxJZDgj55QW75NRwtmcDFZqRyPh+h7+9XrJ2Mhz24osBfkUgcD61nSORNtzitTOeDUMlgGmEmcY7VKKI4ASRxVgcAinBAgwBimsQDzSBEcgwSagLc0+ZwM1BmmBIH9aUMD0quWI6URv8xBosFyxQTxSZpCeKLAQSfeJqsWIJ+tWH5NQSLkVSJZJG2CCDVkTY6ms9WIIFTBqbQJl4SAilDVUVyCATUwORUNFJk4OaWoQ2O9ODZ71Nhj6Sgmmk0AIxpuc0jN2pgbB5osBJSUtFDQDcUbadS4pDRGRTSKlIFNIpDIiKMU8rSYoAZRilxRikIbSUpooASg0lLSGIetFHeloAXtRSdqUdKAClpBS0AGKTFOoAoAaQfSkxT6MEmgCPFGOKk20YoAjxRin4FJgUANwaMU/HFG2gBuDRilxS4zQAzFKBTttBU0AJikxTscUYoAbijFOxxSYoAKKKKAClpKWgBRSikpaAFopDS0hjhSiminCgQuaM0lLiiwC0opAKWmA4UCkFKKQCgU5aaBzUiimAoFSLTQKco6UgJgBipAOKaozipOlAERGDQBTiKSkAoPFGabmlBpgOzRmkzRmgANJmlpppAFBooNMBKWkozTAM0UmaM0AFJmkzRmkApNITQSKaWoAXNJmkzRQBIKcKYtOzQA7NNY0E000XAQmm5oNJmmAuaM+lNzRmkAuaaTQTTTQApPNBpKM0xBSUUnekMDQKKKADNNJpTSZoAKCaSkpgKTTc0GkxSExxozSYpaYBRRRQAUUUUAGaKMUtACCnAUgFOHSgAxQRS0HpQA2ig000ABNGaSmmgBSaSg0maADNJRmjNMA70ZozRmgAzRmkopiFzRmikpAFFGaM0wClpKKAHClpoNGaAFJpc02igQ7NGabRQAZpelJQaAFzSZoNFAwoooxRYAopcUYoFcSlpcCjFCGhCKUCjFLigC/RSZozRYBc00mlNIaEAZxSq3zDNMJoBpiOX8dYkW3UHJz0/Co/D9heyxYafyVAyCDnIrY1myS9aEucEHrjNX7SGO1h2IMDigDmvENncxKireSTZ5wDg9qsWENl9igW5hRpDwQRV+ZQ1zKSAeO4pljaI8xLEHaMjHrSaAg0/SLRtQuMR/IOVGehq7BMlqZkMYQDkYPBFWbC0aCWSRzkvVTWIRNNbRMSFc4ODg4pWGYUMUOp6x9rmCII2IKAYJrX8QXtvBZ+THbo7SjAwMY6VkXennT9YWK3BZT0yeaiZJ4PEEa3bFUBBXf0xilYCrp2u3WkiS2hiyhJJBHT6Uywt01TUJPtLYkbnBGAR7VLc2ZutekSCYxxkZBHfA9Kngm+wamFvYy0Y+7KQMA/hRYEVr2xkt5BEtxvhyAADyv1rfXQorrTZLKR2cqu5CeoPtXLX8R/tGaZHIhc5Rh0Na7eICMG3Lw3GACSpIIHrSKRkpo8l+Da+XFbzRdXfqTWVfWs2nyfZ5JEbIzlDkV1FvZSGZri9aORTk5B6GuV1Aq1/KY2LLnjPagRCuNp9RUZJY8flS5xxTkQvnaM4GTQNIjAIHNOjjeaQRxruY9BnFCKXuUjJ4Jwauva/ZLxTDODxnntQFivPZ3FsiPNHtDdOc0yGN3fCIWPcDriprhria4WGaYEFhgk8CrNvaPZ6pAHuY40Y4MoPAB9aQEOn21vdX3lXTFSRkEdc9hWpcW93pDuoIIkXAIGQR2FWPEWhW9rYRXtteJIwIBVSMt7giqU1pqNjawz3EoeMjhQckCmIXw3KI/EFo8k20An73IPHQ11d6UvNRms/tBRc4MY6EVzEMVpfXjBiEfYCpBwQRRZJdXl/LOb3FxFwHIyTihAQahaQwX7rZ7gEJBJHGfSooTJKskMmARyCB0NWBqFxdXT29wQMnPmAdD602zguLi4uEtShIHzBuCR7U2Mob2z+8lYg8HmlkhwRtIKkZB9ajkjaORkcYweRS7wMBgSM9u1SCGOpQlWGD6GprWxN15myUIygEe9Nk8lmzyKktnjhcmQHy2BBINAFgXt2fKmM5E0PAIGCMV3mmapfeJNEAnKeZC4IkAwXI9fQ1wJhhWESxPuOcFM54qW0mmtNtxb3RjcnBUHjFAHoNtYpcEvJNdSEZB8w8cdq4OazjY3CxzfJG5GCK17PxXqVlAZhL5yEjKSr0+hHSsLUNTnv5bh4IdkMj7jgcjPb9KYDLeSMaXKE+SbPJPQj0FUXi/cmTevHbNN3fu9tR4oAFJDA4zU6cuB+JqEDmpRkED160ATQoZJAoGSaLiKSHBYYBOBVkgRuhjAHHNQ3UhcEHrxjikBVbmQCrMMoRzCeARge1VcHPvVg20ilJFG8EZyKAJtot5jGvzAgHnnBqX+z5nWOSMB/OOQAeQfQ1GGlZQ7x4IJGD3FbHhu5kmkmhJ4UAg9Tj0pXAq3ESROkcgZ8JggcmM/4UJfXELJbynMecB/UVJPIw1Rt4WN8kE54YdqrOpmLwx5fBzgDoKQWLceoT6ddwzOQSjZIAwSO9dPPqltcSW9+k223K7DjlkPYmucAe6ILRkxoAshxkgn0/z2qOOw+zCa1uIWOw5DjkD0z/nvVxZL0O8tbd4ZeLppVcZC+xq6RiuFs9bupGtHIdltWAZ8dEzjBrsxfW0vzCVR7E4q0BLThRjIBBBB6EHNANADgKeBTRTs0MYtNJp2aQ0kMjZQRgiqUkJDkjpV8ioyB3FUSUDFnqOacqbRjHFWGQDtSECncCDYMEYGDUSII3I9eatFahkGCSOtUmJokTk5qbzMcZrKku5FJAAH0qEXMxcnd1p8orm1vDdKRxkVli8dB1zTku2cHJxRyMLk0gyaiPy0CQnnOadw3WnawXEIDDimhMc+lOC4GKOc0gH5OKQmkHApGNJILjWwagcHmpC2KQnIOatCZBtAPBpRkfSg07qAKpiFj5Ge4qdcgVAmQxHapweKlopD8nFIHIpCwxTCRnrU2GSqxPfNO3Y61AhHmYqVzkZpWAa5BHHWkz0NNyCeaMgsADiqsBKDwKcKaBgUZxUtAh4FGKhMpBxmnJICSM96XKxkh5pMU4cikxSsAzFGKkApQBSGQlfSmMDmrBXNMK4pAQkGm4qYjIxTNppAMxRinkU3HNAxDSCnEUmKQBijFFFAAKUUlFACg0uaQUtABnmlFIBThQAYoIopaAExSYp2KQigBuKXFLijFIBuKcBRThTAMU0jmn4oNADMUYp9JSAZijFO20baYDNvtSEDNS44phFIBlKKCKKYC0UgpRQAtLTaWgBQacKaOtKKQDgaBSUtMBwoFJSigBwFOFNApw60APUU4ULTwKQABmpFU8cURgGpgB6UAOC0ppO1KaAGk00mlNNNABSCg0lADgaWkFGaQC5pKKSmAtBpKDSASig0hpgITQTQTTSaAAmkzzSE0ZpgLmkzSUUgFBpRSClBoAdmlzTc0maAHE8Umabk0ZoQDs00mkJNITTQCZozRRQIM0hNLmmk0DCikopAKaSkJpM0wFopM0ZpABNJmlNJQIKSlJpKYBigilNIaACkxS4ptAC0UlLQAUUUUALRSUoNADgaAaTNGaAHA0ZpM0ZoAQmkzQTTc0ABpCaCaKAEpKKCaoQGkpTTaAFozRmjNIAopKUUwCijNGaAEooooAKKXNJigQuKWkFLRcYUUUUAGaKMUuKBCUuDS4oxigBMGjFLRQMTFGKdijFMBAKUijFKOtIVxMUuKWjFMYYoxS0UAWc0oNJiigBc0U3NGaAFxTcUuaM0CsMYAjBGRS9sDpQRk0Bc0BYia3jdskZJpQLeyUzMQoA5JNMkvoLeby5Dg1k6ndRtKJIyHBG3AP8ATvSHYty+KbCFsEvj1AzVWbxXZTSg28bSlcYGOv8AhXOyXb27vb3FuCHGQR6VnSFrLdsIGTkYOaTkBvah4mju2XbAYWQ5BJGaq6lrH9qWsQ8k7oznIIrI0xklvSZQCoGeR0NO1C9SO4K26BR3GMDNHMB0egXcF3mMWwFwgJDEkk1HqFzZXsvl3EksbIeMLwDWHYX17YTC4CgKQQcjqKu3lzaXTFwjAsATJjgGle42dFDpiJAGmuFuo0+4CMAflWXqoMEwurU/uyPnGDgY/nUdprUaRQpFCXlB2MD09jW5q2rxWmlrFcCMTSDGAPlFJgmcRdaxJcXOYcRgjBwOCfWoUiSNnafgvyPpW0RaW17HJM0LW8g646E1Te5tYr5hbwiYE5APagCl/ZxkRXBO0jJMfIArSfTbF8RgskgXO4HgmomvrdHaQZhmxgKBxWdPcpOmC5BHQL0NA0R3JxwYSCpwXx1qtkEDPaniR2UI7kr70xkCsAGBBoAeCpKlyQoIzg9R6V0Go2Gk2gtJ7S4M27BeMkkY9654EYACgYOeO9XnlsvIX9yfMAxkcUAT3V3avKAw2qOgU4AFXpLqO/ggt7udBCuFQqec+9ZYt/K01bhlw27IHYioYUjeTdI2xT1AFAMsajYyaTdQyI29Ccgk859P1psV+iXBnli5Iwcd6CEIbzJJHiUEqM9DUIuIpLIQlBkdHHUUAXreaF5w9lbMzAEsjDg+9XdNRQ4v7Y4uEc74TyCPpTNBvwsZtGUCRRmM+o96g0u8Ed3ccETSEgKeATQBBfKk19NJHKCGy5BGCD6VScYOK0LhLKMPHdIwugTnaeCD0rM3ZpAOCGSYIASD1wMnFWZkTascALODgccfjUmlXqWF7HM6lgTtwOoz3rRmsrS81sxi48hZeSVOQDigDIe0e3YC4JjJGQBzkVEkxTggH61Lq0P2XUHhFybgLwJM5GKpg5pgbM2qpdIsPliHIwzjuKrqfsjPDbSqyyDJOMkVQBy4A6+laNlcpaOXEaS7wQQT0NAFB2Ibqhz3ToajByae+DIzDjJJx6U0ZByKQC4wMmrFrGJp0A5HOfpUYKEfNn8DirVu8SSOIclSOpOTTAdIS12YYRnFSXy5KDHIHNN0rTjIBdXTlbcEg7Tzx6+lEagNMxDvGG6gZBHrSYFIgoQSKs27q43PMY8dQBwKkuVj8sSQxEwsMgkVXhYKrjH3qLAWij28oLkMCDgg8EVZtLgxSxXUcWAoIkx3FZySM0KIxyEzj8atWjsLSTaMhSTnPWpaAuPe2/mMlymUJJSUDqTUGl3Pk3jv/C2Rk9BQQJPKSIjaSQQR0qa2tprRZFRd6Eg4B4I560AaNpNKZi9u+9QQWQDlgK0dU1J4rkzQrmG6QHGOQe+fxrHtSYL0XdmRmPkxE4yO4rs7DTLS8tkmkhXJJOD1B6/1qgMfw95SzX5YE2jqodQOAT61qTaXCLGJIFMgQ7kyecE9PpUtnZmyv3lh+44Ksvc46fl/WtAReXIVQfuycj29qpITRUt4/s/yxAiNzyhP3DVrFLtIJwcA9R60YqhWAGnA0znNOxQA7NITg0maQjPNAx5OaYRSg4GKCAaAIyOKYRUhHNIRQFiHFNIqQimkVSEZ1xCAxO01VZQq5HWth1BGDVYwr3FaRZDRll89TU1uQcjv1FTyWyEEgAfSqpGx8YOa0TJJI2IkwatLxVJeuasoxKjNJopMnzmjFQhxnGakU5+lRYLingVETT3JxwaiB55oSGDcoSOtQkmpzx9KiaPIOOKtEkYapBzTPIcjIIIojJztIORQFh6g5OBUoU4pUHy80FhnFSxjSeKjDHdg09jULkBqaBkwGWBFWf4D9KrROMDNTsfkzSe4IoyyGN8Hip1VshiCARSDDSDcATnvV3AIA7DpSYEIPrTHcdBSzZRTioN2RmhIdyNz83WhXIPWo3Y5pgcg1pYm5pRyAjk1MGBGaz4mJz7mrCuQazcSkyypy2KkC5qssgyDVqMZUHsazaKTHKozQ0WRkU8DBqQDipYyiyEHpTCtXHGeMVEV5oAg2e1NKcdKshMnpTzCDQBRKn0puKuPDgZAqAoaQyLFIRUhU0hWhgR45pcU7FJikAlLRijFABSikxS0IBRS0meKWgAo70UtACd6KKKAEp1JS5oAXNB6UCigBKKWjvSAQU4CkApRQAEcUxhxUhppHFAERFJSkUUAJS0lFMBRRQKKAHCgUgpRSAcKUU0UooAcKUU0U4UwHCnr1pgqRaAHipBTFFSheKAHR5FTjoKiQYqXNIAJpDRSGgBDTc0pptABRS0YFIAoopKACiiimAUUUUAB6U004009aAGk00nilPWkJpgITSUGkpALRmkzRmmA4GlzTAaUGgBc0ZpM0ZpALRmm5ophcXNNJozSUgFzSUZozQAUUUlMQUhpaQ0hiZ5oJopMU2IKTNOxSUDCkooxQAUtJRQAuab3paTFIQGg0tIaYCUUGg0AGaM0lFAC04GmiloAU0YooBoAM0UUnSgBCaSlJpKYhKSlNIaBhigCgGg0AIaKWkoEJRRRTAXNGaMUlIApM0YpaYCUvWjFLigAxS4oxS4oATFFLijFAhCKUClxS0WGJigU40GmAoxSGgGgmgVxKKKKYBRR1paBhijvSijFIQGlpDS0AwpaTBpQKALRBFJg1KRSEUkMhIpOKlK8dKjIxRcQUmKDxRmi4CUoOCDSZpM0AzE123JkklxkYyK5A2srXKgwNtJGSpyK9DvIxNAVaqh05E2SxEgjHB5H5UDOQvLIyXMcbiYxkADJ6Cm6vpiaWIwkpkV88HqOn+NdPrFoLmKGNXMcgOQQM1jy+H5zbNJcXDOQMioaA56OMEjA5PQetNMalyGDIw79q3hd2NnbBSA7DsRyfxqpHqNpJHJCLdpDJ2PWlYDKkuZpB5bybgOnFCNJ5ewH5e4p7QGGbEqFCegPpUbTeVIRsJHt0popluzvGsJCwRWVhhgevtio72cXUOZHLY6A9hVZ2yc03aWHHSmBAAMgnJwe5qXDFTIM4AppABxTS7A4B4pATRoJIyz8r79RTJAigbWBHsKYWOODirSxILEO4y55pWBFTGRntTSQCParIjja0L+ZiRTwuOtNjmi2gSQhx78ZoAeiJu86GOSRB97jpSrayzqHQqAffNJHeyQB0tz5cb9V68fWizuUtyRPGJIznIzTAmvZbkSxhjuCjkjkVVEgDkkA/UVsXWt272AtYrYhT1PU1jxxF1dtrFVPJx2oAnhniVgScKQQaYYY5FeSFgFToD3otDaB2M5IGDg47063uJLZnMIDo3UEZBFIC/bWZvZ4W0pH8yMAyBuAM960dS02LTpbeeSRWuicmNQcEcVi6bqVxpepCRARvYB0IxkHt+tdB4yspZJobuAExmMEgDkfWgk5vUrhLq8MqKVyACD7VUBxmlZdvDdaFDMfkXOOvOMCmUjUj0hxb291IQ0LkEkdhWvH4XvbmCQ6e9rJbg5Cl8s3GcA1Q024ihgEFw54cMgPKuCeQT+Vde2paV4f1SGYREs6cxxcj64pEnA6ksJIEcTRSJkSI3UEY/8Ar1TIAAAFdZrlpBrN1cajaXcUYcAmJ1wxIH8zXI5LDmmUmWrB7aK63XUe+EjB4yRUc7xfaXNsCsZORmoh0oHFMAAJ6daXaScDrUqtCYypQ7z0alOQRkDj0qQICcHBFWLXasMr9wcYqCT5nyoI45BqaAMQUA475pgXVCfY5EJKk87QeDVaGRoeYpCARgg8jFSCNzKCSpYkAZPGT61vnwh5Ns8k9yEfP8ByoJ9fT61FwOflm8zZGoxgYAB4z3pVEUdk5cfvSemcEU42/lhmDGQAnJUdBUZjM0QkCEqCQSOo+tUmALHvQuWAxySfTtV61VLm2gjEn7xjtI6ZPaogA8cMhAwgC+WBgkd6iYSWdzDJCwYbw6KOCCOxqWBtR2S2l0YZoisy4+Yc/pTrGHZerDJNgSsArjoD70271Z7pzeLDgsgBQnoR+FMtneW3jkhYDLgSA8FSD1pALNa3CXt0ViIMYywUZB981s6HNe2OmC6ckwzkj5ycoRyM+xqXRoJNS1O4XzSqyJiQE5JB6kV032RL2yktIZkVSu3BPIIxTAkjTzrW3nQYyN34mpyDjmrFvGBYW6gfdjAP1FMdOK0uBXwKbipdlG00xEW00mDUuKNlMLEJpD0pzDBppFAITNGaQnFN8xc4zTEKetApCw9aTcOxoEOI4qIipM5FNI4pDGGo2HNPNNIqkwGbARg0hhQDO0Ej1qVTjikcVSZJjzxtG5I6VJFMCmDT70YBqmh4rVakkjP+8zmpo3OOoqqTk0mSDTsK5oE5SowKrpMQoBJNSLMMg5pNDuTYJpCDTkdSM5p4INICFQQQD0qfYnXHNBA7Ckdcr1IIpFAcAVEQM8U9FZhgmmlSjHPSgLDWUkcVEynHIq2hBHFQzoRTTE0QqCBgVaz+7FRohK8VMIzsApMEVnXnI61ahfKAMeR1pnlHOMVJ5RAyKTZQsiCRSM4qmYyjYIq4Dgc1XmIPIpoTKkyg8gc1GqnPIqeQ4WmJ82cDpVpkD4sg+1SkkGmxkDAPrUr7dpNJjQiEA5Y4FaELAxjHSsoEspq9a/KMZJBNRJaFJl0Gng8VCGqRTxWTRaHbQRUbDnipQeKQjnilYBoApaaTzQDzQMcygrVdk5qcmmkAigCoy88UwirDLjNQsDmgCLFFOxRtpANooxS4pAMxS4p2KNtACCinbRSYoAKKKKAFoNAFKaQCUlFLTAUUUUUgClFJSigAzS5pKXNMBKQnilppNADD1pKU0lIBO9LSUooAKKBRTAUUooFApALTu1NpwpgKOtOFNpRSQDxT1pgqRaAJF61KOlRqMAVKoOaAJB2pxNNA4oJoAXNNJozTSaAAmgUhNKKAHUGikpAFBopCaYAaM03NAoAcTRSUppAITTD1pxphpgIaSnEc0hHFMBp60lLSUhiGiig0CFzRmm0tMBc0E0lIaAAmjNBpKBC5ozSUtAxM0ZoopWAM0UlLigBMUdqWjFMBtBpxFIaAEopaMUgExSYp2KMUwG0lOI5pMUAJRS0UCExSYpaDQA00UUGgQhpKU0lAxaWkpcUALRSig0AIabS02gANJRRTADQKWkxQAHiiiigAxRilooENIpKcaKYDcUUtLihAJRinAUUhCYpQKXFFMBaSlwaMUxiYpcUUZpAFFGaM1QBRmjNFACUUUUhAKUUgFOAoAAKcBQBTgKAEApcCnhTRtNADMUuBTgpp22gBgUUu2pNlG32oAs4pMUZozUjGnpUbdakPSmNyaYER9KbTyKYRimJiE0maDSE0IEBNJSE0maBkE8Iku4SegPJqjrckhBSMkEAA4PUVqZGcmoJreNyZHx7kmkByr6fawxxGQqGY5IkPIqsNTsbKRxbxAkjG8DIq5f2L6peiG3bMKDJc9BWVdWFtFLIInYsoz/jWYFGSeS9ug8hxk4J64rRMYt7YQsA6E43kYNRLaS2EC3TrtJOKpT3stywVsBewApjuNuRGs5SIkgdSaIiNwGQBjmoG4cfSgFSpyuSD1FAx7MuCOozkGoCcGnFSFDdjUZJJpAPJUjAzTi8jR7CxIHTNMApwODgU0AhJIwaFRz0GRSq6I43qSO+KlLDOYwdp6A0ARmJlGTj8KdGigh3wVPBHpVuKNGjySRxyDUOFDHA70ARThFfCEEeg7Usd1JFDJCMFX6gjoasSODDsEQJ7MBzUUXlLE6Sw5Ynhu4oArCrglRLWMiRMg5IHBwKrlQ3CDn0qWKyeaNiydDjGefypAaetahp2o2kFxbgpdIQHBGMjsevtXRJ4htZfCmZ5Ue52GMxg8g9Oa4poYEZSckE4IzUtzpxtQk2QYZBlSOx9KYNFORt8hNXtGsje3UkfnGJCCC+AQD2yKouMHIq9o6iWaWP7Q8DkZBXofrQBtwiTSpHsrhEljXkO68E+lR6lbPZrDcsR5I5AHUE9ADVWyu5Li5OnXz+ZCcgEjkH2NVrgm3Z4JJTLHG2BGTkEds0CKzXom3JM25Scg9wfrUDhB90fX3qW4kt5VHlwmMjvnOaZ558vZtUgdOKQIQAY96aRkHsR1FISScDOT0ApMgkHBB6Ed6oY4danB3LgAgjkE1XzzjBH1qeBfMkCkkDGaQDd7O4LHJAxTg7LnacZ60qqp3q4PXAINSR27tCTGM7eo9aQGn4c0ZdZmuUklkjEaZDL0yegNa/h1pEm1DTnDTMFITJyDjtUPgmC7eS9nhCmEKFmQ9Twen604wPaXUNxZJtuITl8nhxRYRJLb3Wh6XuKRlrmTCZHKk9j/Sqp0aKG4ntWeQSPEJEGe/cfnXUSoniDyZrmPyUtjuK56k85zXI69PdT6i92TtT7sZHcCmlcaYkGmwsI0N4MuOmMEH068VG+nQzEhZSkhB2EnIyOx9K7bRtMs5rbS78wAM5yUI4JxjNaq+HrJLi7ieIPG48yMEAFT3ANJxA87TTJ7exE5KywkZ3Keh7giiCA2rwgr5kZxKSp5A+ldnpNpZy6QbSQ5jmk2MR/AwPAJ7ZGK0T4esxK0ccZAVMAjqB7GlYDD8L6bIFk1OGJpsMQhU4wPQiqNxrmqaTqLqts0eHLlGGTg9vaugj0uXQvs15YFyrttnRjkHPQ+3Nak1obudi8YEkmCSONoHvTsIXSb8XvlSICqzIQwPVWHUfrWi8fOMVFYadHZ3MxU/K5DhccA4wT+NXHALE0xlF0welMxVp1BzUBUg07gMKe1RkGrIXIpDGO4oQFN+KhLYq68SntULWysCOR9KpElQuCMVE4GcipzZmMkgk+xpkke1QT3qkBWaTacZpBIT0NRTffI70xWK1diTQR8jBNONUlcgg5qZJcnBpNDTHkc00jFPJBFMY8VNguAGTUjr8mR1FQg4YZqfdlaYNGVeeYWGBmq7wFcYGMjJ+takiAkmqsqN2FXFktGf0NLinMh3HIxTSCOCMVqmibDSKbg1KF3HHel8hyRgdTRdCsNAKnNWYXOcmkMRA5FIo2ipbKSLSnODT+COarq4AGelWAQY9wqWNAB83HSnMuRUIlOcVZBBAxSsMgCgHgUoG7ginEEHGKVRzSuMQR46CnhTjmpAMCgVLYWGAc804jIx2paUUrjsU5I2AIFVTHJySRj0rUZQRg1CYRnkVaZLRlYLNgcmrAgMcfTk8mrogQNuAApXUNGeOgquYVijsY9BS7TjBpyvhiMU4AEcdaq4rEYUAcdqlgkFN2EinRxFegpS2Gi+BxTlOBUcZOBnrUgNYstDwaUtTQaQmpY0ISSaQGg9aTvQMdmgmm0vUUEjCajYZqYimlRQUQEUmKmKc9Kbs9qQEWKXFOxijFADAKXHFO20u3ikAzFJipNpo2+1AEZFAFSbfagr7UAMxxSVJtpNvFADMUlOIppFABRSUtABSikpRSAXNJRmjNMBKaTzTqCARSAjoIp2KDTAbRS9aDQAlFFFCAWloxQBQA4CloFAFIBRTlFIKcBxTAcKkUZNMAqVBjFICUDgVIopg6VKtAC4ppp2aax5oAQmmk0GkNABmnCmilBpAOJpM0hNJmmApNIaDSE0AKOlFJSikAooNFFMBppMU4ikxQAgHFIaXtRigCMikxTyKMUwGYpMU8imkUANop1GKAG0UuKMUCG0U7FGKQxuKO9BFBoAQ0lLSUwFpaSlFIBcUUUUwDFIaXNFIQmKKXNNpjFpCaKQ0gA0lBNITTAUmkJoJphNAh2aKSjNAC0lLRigBtGKdikxQAg60opBTh1pgOFIRS0GkAzvSEU402mAnejnvS0UCEFBpRQaY7iUlFFIAooopiA0AUpoFABxSGlooAMUopMUo60MQoFKBQBTgKAExQRTsUmKBjDSVIQKaVqgG0UuKMUAJRilwaXBNIQgFKBSgU4CgBoBp4BpwU08LQAwA1Iq8U4LT1WhANCe1O2Z7VIF9qlWIEZNMCts9qNuD0q35YA4FMZBzxSAr8UcVLso2e1ADMUhpaMVIxhNJmnkCk2imIYelMPWpSuRTSvtQmMgNNINTFPamlfamIipCKeV56U0igaIyinrn86jniQ2soAOSvrUxFJTAp6XEkVr8oxnOawNPsvM1O7EwwUOMV0yKIQQOATnHpTVQGYyAAEjnHeoaAwNWhE6QWZ45zisybwzdQ3OYoiyDqSegrrZrdWmSRVw6nOavElzkDrSsM88l09ZZSWQ8DgKOePaq01qIbAyBDnOAO4+td3qSWcCGYyRxzAE5z1PHFclJPLqkogggGBkkr3FAzn2PGKZW/HHpkZYuheRTgAZzWfeI1/dSTW8ZC988HIpAUkQt07VKY0jxkgkjII7UlqkkkhjjXcSD3xWhYWQgSaS6VSoGNucmgDOIANTSwGOBZUkVsjJXoQfSopGV5C0fCnoKbnFMBxkY4G447igP2qMggZoB70gLId0X5TgelOMoZCCeTUKOTwfSmHg4oAepIOVOD2NSB3aVAJiWJwcDGKgVsGrukTQwaoklwcRkEE479qAI5IUVz5pEh9RS3E3mQCFCwAOdh5Fa11p0aTSXOxWt5CSHzyp9qwZCPNJU5HY0ANKnABrQ0maG1kdnBDkcMOgqjnIpu6gCVJC92007ckk5xnmppntpP3oBMgIyD0YelVRz1pCM0ATXhtzIrW/AIyV9DQlxbxxlTACSME1WYYOKVRxQBJA/lTK+M4OcU2Z/Mnd8Yyc49Kkjgd4mlUZVTyfaon2g5HegABJp8TFZQR1zio1OKfGCWz2BpgTAgyYzyTkj0rV0e3e4uZokmWNwMgEfexzgVjk4diBznINdHbQW03hyK6SPF/DNw4OMjPIP6UgNvw4bvTtWMjxN5M42TdgD2P8+a3n0T7ddpuCLFGTgg5JGc0ukTG5vI1eAwSCP51JBBGOCK3kjKDpQBTGkxGaZXwYZQAQPYVwl/pKML+SJ2ENuQqA8gnPSvQrwzG3KwnBPBPcCq8OnW6ae9my5D/eJ6k1SAqaAS+iW8MigSWpCcdO+CPwranuBBbyXBAOwZx3rJ0WJ4I7qKQklZSBn0FaM0QnQAjIByQehoQHB6lpt1Z6fFqMJkCSOTKAcAEHg475r0SErNbQTDkvGpJ9yAao6naedo72qnC8HOM9Dmr1uf9Fh/3B/Kk2BIQGQo4BUjBB70IBGgRc4AwM9cUmacOaQNjhwc00uc4pw4FNPJoAU8io2XNOAJFLjigBoXApCBtoPBoByKAIXB5pEyDzUjDNMxTuIWQBl6VTmti564x7VeAJpCtFxmJLp0zuWBBH1qCazljAOM1vshFIFDAgitOYkxI7STYCVqKVTGMkYrceMIOOlZ80YkYg0+a4rFFJyTgmpgcnFQvblX4FO3bcZ61QE7IAMigHjFKjgqKdtHapYDCM00qPSpdtG0YpXCxA0UbjBA+tQm1jGRgkdqt4Ao4qrsLFBrYIxAGD61GS4faTkY71osARUEkakDiq5mKxEELrk9aa0YUYPFTrkDGCKQgEYIo5hWKgTJxUoBUFT0qQIAeKVgCc4ouFiAISc4qdCQQKQDnFPAouMlyCKAoBzjrUW7HSpUbPWpY0OxRS5BpKljEpRxSUtIApCMmlooATFKAMYoxThRdgVpoQHyFxn3pgQA1bYZFN8tfSr5hWKzAgjipUAIFP2juKcAB0obuFgUYAxT6QUtSUhM0pNITSGpBC03vS0GgYhNGaQ0maAHZpQKTNKtABtpdoxS0YoAiZOTxUe2rW0mkMftSYEAjPpS7ParKpx0pTEeuKQFbb7UbfapjGfSlCe1AFcqaMVOV9qbs9qAISOaaRUxU0hT2oAgK00irBQVGRzSAiIoxUhXjpTcUwGig8UuKQ0AJmkopQKACloxS0ANPSmGnnpTSKAEo7UuKMUAJjilxRilxQAlKBTsUoFIBAKcBQFp4X2oAQLzTwOOlOVOelSqntTAYq89KlC4FKqY7U4CkAAVIOlMA6U8UABpppTTTQA00lBpKAHUZxSZoNABmjNNzRmgB2aSkzRmgBwFOA4oFOFADSKAKU0maAEpDS0GgBtFKaTHNAwoxS4pcUxDcUbRTsUYoAjK47UmKl25oK0gIsUm01JtpMUAMxSU6kxzQAhFNp9IRTAZijFOxRikA00CnhaCvtTAbRRigUAFFLijFAhKaadikIoGNzSE04ikIoENpCadimkUDG0Yp2KMUxDaKXBpKAHinY4poHFPA4oAYRRinEU0ikA3vS0EUCmIdijtR2opDEPSkpaQ0xDcUUUUDEopcUYpiG0U6m0AFLijFFAC9qMUAc0uKAExRinbTSge1ADQKcBTgtOAoQCYoANP2mnBeKEgI9ppcYp5GKQjNUAwimkU400jmkITFJg08ClCkmgBgBpwX1p4Q04IaAIwtPVOelOCe1PC0wGhOc4p4WnYpyrmkA0A+lPRST0xUgAFOAoAUYAp2RTKCKAHbqQ80maAeaAuGKMCn4NBBFAXKtFNVgRjcCR1FOFSAuKMUoBPQZp20jqQPxoGMxShKeF79qULQBA0feozHV0rkVGyYouBTZOajKc1aIGaaYwe1CdgKhFNIwCfQVaMQPammIAHIyMdKdwMS5ljhhM9y5CjoB1JqnZa7JdylI7XEIOC7nGB60a3azXiIjuFDP8AKq9cetUNYjGl6fHZQ/6xxlz3OaGBeufFlhbzmIQySY6lTxmqEd7qmoTq9vcmGGQ4AyOB34qs/h9F0mKabcJJCBwM8HvW1b2X2W6s42HyADnvS5gMq9tLK2uWS5Mk0xHVjkE/TtWl4U05I4pLsjJkyo9hzUd9ai41SaYLnywMr6ir1he21nE8cRLR/f6YCE9qVxmLd2cOj6/5srN5MnzIcZwT2/Wsi5QSahcTRSbIckg5q5rN++sXqiQeTGg+XcOCR71z7SMCQGOOxFAx0DtBcNIhxxgGnTNNI295iwb0NRQxzTMQgzjrTfLfcQByOtADtpCknAApvPam5IOD27U4EYpAIATQeDinIcMDTWADEgYFAxeakQbwc8kVGACcE496cMhuCOaLCBxjkU6MxqUkcBwDkoe9GQD83TvUZGDgdO30pgat3rDXluIBCI0GMYPIFZ20E4Jx71GCSfLVSXPQCrlxYNawJI7jL/wE8igCoQQSO1JmrFvCZmI6ADJNLDa+dceWDjgnNICKIx7X3g5/hIqQoNgYHg9vSmyIEcqOg4pgODQBK9qTGZQc47Y5piRAw7+g71PFckYxg47EcU27aVGWJlVUIyABjFAB9pP2TyVJB7kjqKqkVNKI1YeW+4EemMVHjcRgAHFADcHtUsZzkUzkcYINSgFDn1FACkcD64rtfCse03Gl3AjkhkTzIwepOOcD8BXFxqZMgYwBkgnFWLS/urS9t7pJHLQHIx1x3FAHdeH9Zli1W3tLwFdrGPzG6AHOAa7txgkV5haXEGu+KQ4hc285wSRghsdfzFegabZXtpCUuLtpRyAGGTjsc0AWj0xTDUhFJimIrlQrlwME9cd6eJCBSumRVZiynGM84ppgWt4YFSeCMGpEOECjoBgVWjTByDke9WFoauCZMlOBIpgOKlGDUjDNJ/FTsUh60gHCg00GlJ4oAY45poFKxyD600A45pgPApKUUYpANpDT+KaRTCw04NRHhgRUhYA4prbSDTAZIMoaz5Bsc5q8x4xWffRuxDKcYOTVokjlG5dy8n0qpIeeeDVpAQvWoLhAPmIq0xMIz0OasBgehqgsg3YBqdJBnrVNCuWhzS4qMOMZpwf3rOw7iEUhGKUmkJ4oATNMPWjJBzTM5bk1SQEmwEc0wpg1KoyBSlc0mwIDGSfakMZxzU+BS4BFK4FTBB5pQanZR3ppiGMiqQEI681IOlIOflI5HQ0mCrHnimIkD4ODTwaiOMZ70Kxzg1NhpkooAoBpaTRVxDQKDRSEGaXNJRSGOBoxTQadmmDQ0ilopBSuFh1LTaWgBaMUUCkDDijFOFIaYEZGDSYOacetApiExSikJxSBhmiwyUdKXBzSA8Uo60gHKKeFzQBxTlHNJjFRORxUuz2pUGTirCxZA71IFVosqeKgK4OK1jGAhyO1UZI+TxQBXIGKYw5qYrjIpmwk0wItppMVOU9qaUNAEDDFRkZNWimRzUTR4NAEBGKaalZTg1GRSAjNNIzUhWm7KYDMUoFPC0uykA0CjFOwaQg0AMI5owKdijHFADcUAU4ClC0wG7aXaKfg0AHNIBmKcFp4WlC+1ADADUyJkikVeelTouCKAFWPB6VIFApcUtMBCBSEU40hpAIBRmikpAKaYaeaaelMBlFGKXFMBtGacQKaaQAaaTTiKaaYBmnDrTcU4DmgB4pxpBQaQDSaKDTTQA7NBNJSgE0wG4pwBpwUYp200AMwaXFOwaNpoAjxRinYoxQAAc07FIBzTscUAMI4ppFONJQAzb7UEVIBTSKAIyKAKcRQBQAmKNtSBaUJ7UgI9tBWpdvtSFfamIhK0zac9Ks7KaVFAyML7UFfapcUhFAEWymkAVKRSECgCLGRSFcVIRimEUARkUhFPxQRTER4pcU7FGKAGEUYp+KMUAKBxS4opcUANIppFSEUhFAiLFGKeRSYoGNpcUuKXFMQwikIp5FNIpDEpMU7vRimFxMUhFOwaQigQ00lOxSgCgBMUYp4ApwXNMBgU04KTUgQUoWgCMLSgGpNtKBSsAwClC4pwFOCk00gG4oxT9ppdtAiMjikxTyMUhFAEeKXZTsUuKAGYpwWnAUAUwFC0oGTQBTlHelYBQntTgoozQCaYDtoAoGKTNKKAHjHelyO1R5NAzngGgCZVzT/AC/cVGpIIqUZNIBpXFM4B5qwFB69KpzzBHwq5HcnilcCyCCKJBhc1Qk1W2tovnDu3oBWNd+IriVwIozGgz8pPNHMgGx38a4EbxPM56bxkfStyKOXYCwryQyZ5CMh7EHkVsaTrWsRSCO2nMqf3ZTmkM7W4uLy4uDBaSKgHUYwQafFonzbri6lkk74fFc8viC1u5vLvont7oEgyQ5wfrV+21uKGYW80kzZ+6xPOKANk29zaKTauJF7xuefwNSWl/DeMY8mOYHBjcYIP171BDqBkJ8lTLjsOtNu44b0K0lvNDMpyJFHOfekBqkAdaAMisDU9WvbGyAmtfPUnHmAdK3dPnjvbCGaFg2Rzg5wfSgCOWMg5AqEHFaZjDDBFU5rYg5UUAVywFNJBBAqQwt6VGY2HUUwKyW8KNv2At6msi60/wC13DSvxk4AxnArdIIqIpTAz763/wCJeqIOEIP5U14zcJGyjgAEGrsyF4XUDqMU1FMcKgjBAxSaAw72cWllcSk4ycZrBmvLd5EhikkCkc7B3966e+077XD5R6E5NZktvb6bdFUj/eFOFUck0rDMvX1SaO18kA5BHAxnpWFNb+XkOQpHY960Li+kmuCbwmPYcKAMDNUNQuPtMxcdD0oGQwGQMwibBI596bkjkkg+uaslYTChiJWUdc96gkRkbDDBpAMpRSkcdKOaAAcEH0pX5JI6GmGnDpjtQA00oBqQQtuAKkA+tOIy20EAA9OuaYEb4wMdehFBBBwfTNWYBbJMGuELJjGB2qO48kznyCfL7A0ALZNDHexyTlgqnII7GrWrXgvrkMpUqvQgYJ+tZ/lyHkDIoAI60AXInMdu5HcYpLWQw3UbepwfpT4ED6TLNnJQ4PHWoSPMhiYEBuvPfFICaREFwxcZXOSKpSlDMTGCF7AjmrRkEsJjU5nPQdz9Kq9ev3u4PWgBrZCGtS/+wvpFnJDLm4IAcdSPUVmkZFKnyEHAP1FADVHQmpo/9amRxnmhmjJ4BHHpUtqpc7wD5Y6k8YoAJkJkyBkdzUeKvfI84CuGGQCB6etSTR20QmBjQgkFCBkg0gIYLe3e1Z23LIMcDgZNK8Nzppw67TImVPUGrGlrNJMgtl8xicMpHBHpVrWR9rmEYDI0KgMjDkGmmB0Pg/QpLPV7W7mkVt8ZJUDhSehxXoLdR9K5rw5BNZGK3J8xTCskbMOQO4roRMHAPTigAIpmKkIpCKAIiBTGjBqQ03tTQrlcNsJ5wKkWboc8UkkYIJqP7vXpVAXUcMMipQ2KzoyV5B61KJWJwTUtDNAMDSE5qur8VKGzSAXvSHrSZpAaAHU4UwmjNMB9LUe7BpS4UZJ781IDqCOKMgDJPFKGDDg5oAhkTAyKhOatMMioCvzGncCPGRVeRQVII71c2N6VHJEdh4qkxNGdIMDI7VUkkBBDitBo8jGKikst4xjHvVJiaMd12nIPBpEd9w449atNZylgGj4HvUwiAHKAGteZEWIGlIQc0JN83Jqe4tiI8qARnsaia1+UMOKLoZIrg96fnNV9hWPJOKakpAwTSsFydj2qF87gQcVICG6daY4796dguWYzhRTs1WRjkc1ODzUNDHdaMU5QDTiBSGQOSBmmCQEAVLIPlNUZAQciriJllhjnjPtTWIKEjqOaqmYjg05Jjn1GOadhXCOYOxB7GrGMnI6VSaMpIHUgqRnirEL/ACANRyiuWAeKXPNR544ozzUNFIlzRTATSg0mhpjsUlLRipsMKBRijFAC0UUUDEpaSloAUUA0ClA4pCFFFFLTAjbrSA4pxFMY4PNNIQrU1QSelO4OKljXjimAzOOMVIozTvKyc1IsYAHFJjAKfSnKOalwCtAUZqWMlQDIq1HjaKrJViM8CpAkcfLUDx5GcVOSCKaR60AU2iyc4phjx2q6QKicgcYoArlOOlMKY7VOKaw4oAqkYNNI4qRjzUTHmgCNlFRMvNTE00imBXKmk2mpitJjmgCMg0mDUuKQigBnNBFO5oI4oAjxRinEGgA0ANAFOA9KcFpwU0ANxQAakCmlC0AMANPC04DNLtwaQCqnHSngYpFGKdRcAzSg0lGcUAKTTSaCaQmgBTQDTSaBTAd1NBFKBS0AMxSZxTiaaRzQAhNNNLSYoAKaetOxQRQDGDrTh1opRQDHjpQabS0AKaZTzTaAEFSAcUwdaePWkBIBxUgTIpikVID0ApgGz2phXmrHGKjbrQBEV9qaVqUmmk0AMxiil60YoAjI5pMVJijFADB0pcZpcGnqmaAIip9KAoqyY+M4puz2pAMVeelPK09F55FOK8dKAIStNIFSlaClMRDijFTbPak2e1AyEqaaQasFPamlRQBXINNIqZhUZBoAjNMxUhFIRQIjIpMU7aaMUwExRinYoxQAzFLilxS4piEApcUYoIoASkNONNNIBpFAFOopjG4pcUuKXFAhpAxTCKkIpuKAEApcUoBxTgvrTAYRxTcVLtFJt9qQEe32pQtSbfalC+1MQwKacqnFSheOlAWgYgHFOC0oBpwFAhuz2pCnPSpcUYoAjCc07bjtTsUE4oAaaQA08AE07aB2oAhINNxU5Az0pm0elAEeKMVIVAph4pgLtFJwDSFqjYnNNIVyXIzTtwqDfxSbqdhXLGaXJqEOQKUvRYLkoalDc81BvA6GnBiTSsFy3GULDJqwFQ9wPrWcWKjOKR7ohCTwAKTGmXzJCHxvyR1AGcVYRFYcGuRbWTG+EQkDsRVyHV5ZOASv1FRzFXOlaEBCxIAHXJrmtZ1GK3cohznqenpVe71aaYmJZi/TIUVk3IjI3OMyelZthcZJqEkjEg0w3MXVxlu/NUWkJc5AA7AClADckc1IGXZJ9vmMUrHaBkY7GlsrUG/eMuwROjKcGr1/pX9nSQm0fdIRyvUk1oW+lKbJXMOJjy5JIH1rQoztIsEu9QkaYl41HU9QfWqN7BcXGpyQwGSfacLg9BXTiyNohhtJFYzDJBPQ1f0XSI9LhknnwJiMkjmmmSYdpJqGhLHNOGEbkA88jn0FdRP4mhh0t7mA73AOFYYxXMS6pbXGpyTXrEQg4QkcH2xSS6ppk7eXBEGPbnGaYCWfjGaPzRfRGYseAMYA9MVEuvmGR7ixuntZGOTGF4NVNWubNDGbcgSKOQB0NUYbuF7kS6hE0kI6kUgPRdF1fWLyySc3FnMp/v4B/GunSQOieegRmGRg8Z9K8huH0yYg6cLiPu6jJBFb3hvxNaCM2U0xEg4Q3DHBHtnOKAPQZIY0G5wAPc1Re7sD0lz9Bmufv9WS3USXLlYW/wCeb70Yfh0q/p/ifQysaF4AH79SCfUUAXWMMgzESR64IqF4/mxW40C3ESvDMgBGQyEEEVjXsWo28EkjTWwjQZLkH+VMDC1SV4bjZZT/AL48sh5GOKqHxElrKI9RjAJ/5aRkEZ96xdT1u7lEkcghBBxvj6Gse4srsRJcSDMJ/iz0oA71tb0ggEXQyemASawtS1iK1uTKAs7OOCD0/SueaUxQrGAvIzkdaqk5cAdScCpGJe3f2y5aYpsLdQBxmoEXLAGtBo/s4KXEQYvyGByMVW8kI4wc596aYxuwEEjqDTZnMoHA47+tTEkIVLAY5qAFtnXjNIAAGCO9Ckcg0owOSDx1NSsEcARjoPXrTAr7CTwM0PG8YBYYz0q5Y2pmcs2Qq5yAetJPaEusiv8Au3IAz2pANh+zvb7ppHDg4AB61A21WypOD0NTXtobNwDIGyM5AxVYAsOTxTAkkIIQnjIqMjBp0hygHpUYJpAP68ZI+lBJOc8mlAzyOtBGDkjFFwJLeWZbWW3ABRuTnnpU9kkt80UEZC4JOccAe9T2FhLctGsUoXzQcgjj8662H7FZWKQwlHuosjgYOfehiKuix2mmRzyXUcUk8ZyjIuWIPtXNa4beS8+0wCRRNlmRxjBz2ruf7OtrTTGnuIBNI2STjBGewrmLvw/qV7aNdl4ykYOIwckgelA0zItY0icG9iZYZB8jepqqSAxAOQDwcdR61bvdQ+1WdvCY9pi6n3AxUVqY1uYXlGUB55oAm09TNKyFSVA7jgfjU4hub238mDDlDyiDkD1NaEV5BbtNcRoJSThs8BR7cVHBefYtQ+1REi3uODkYAH+TQBBZWLw2S3rKrIzmIj+JTTpbSTy5AwBkblcHgite1kjbS3TAIF0HGRnA5pl/BbR6iZ7dHmDkERxqSpJ64NAG5pfhqKw8OHUEcvdTRgqO2SeAOeKo2RkGo3VxfwqFkXyZFxkg468Vd0S7uL/TTYSRSNZxSDdIODGOpB+ldZZaFZWkpaCSSRXGSGwRmgCW0s0hkjuF/wCeIQYHHQcioLuIQ3KSA/u3PQcAH0/Wqml6ssl7cWTjb5bkR8YyB1FaV3F9qtQqHADAg47igCwGDqGHQjNMY0qcxjtxTSKAGk0maU9aZmhAKaieMMMZxUmaDTTAYoCJg9qhWTM2B0qWYExMB3FUbaOSLIc89apCNEPggE1MH4rPZieM4PrViJjsGTk0mhloNQDUQNODetSBLmim0ZpANkbIwOp6VmC/eO4jtioeZhnB4rTKbsEHBHQjqK52/wBLgE7XV1O4kRCUfODxyCD+NMDXv7+3tVUzZPTAAycGn296l3p8k0G5VwQNwwa48a5Dd+U14ZS6DAkQdR2z61RN9K9yFWU8NtDhiAR2yKm5Vj0eC5ieyWcygRkZBPGamCZUH1Ga5WPVEudKayitiPLUDfjgj1Fb1lqPnRQhkZcqOT0J74/w4pkmgEGOlDICMYojcSAkZwKcCGGQQRTArG356UC39qslaAKAKElv1FUprfGRiteRepqpIMk1aAy5t8ce1Rn2pkRLJz19KuyRggnFUlBEhBFUmTYbNGTGcDJqm8ZCk4rTIyKgZQSRiqTsJooJIQcGrC4YZqG7hdB5ijIHWoorj5QKvcnYtqME8YqVeahSTIHORUynAyDxSaGmShgBzQZPQ1CXqN5ABwamw7k7uCp5qrJ3qMzmmvMSKuKsJsZKFwTnmokYqTg9RigsT1puKsklSQ4xnirEZG0g/hVRWqeOULxgE+9JoEyyjA8UE4JqFThiT0zUhbOahoaZIDnFPBqqGIbGeKnQ7hmk0UmPzTx0qM5pwPFS0CH0UClxUlIMUUUhNKwBSimind6YhaWgUGlYoUUh60DrQaZI0k0xkL9OtSEcULjNMQxUIIyKtIMUwCnBgCBQBOoyakCjFV0cg9M1YVwRnpUsodjAoxTS1OHTNIY4GpUYACoKcDipaAshhQWGKgBNKTxQArOOmajZsmmt1pmTSAlFBwRUe73oDc0wI5F54qJl9qsnBqJhQBDtpMCpCMU0igCEjmkxUhFNIoAaBzzQVFOxRQBGVpMGpsCkxTAixRipdtGKAGAU4CgCnAUAJijFOpKAFHFKKQUUgHClzTc0hNAD80maaCaUmgAJozTSaM0wFxmngYApoBJFSAUAGKKXHFIaAGmmnNOxmjFADMUhp+KQimA2kxTsUlIBMUAc0tGKAFzRRgmgCkAYpcUoApSKYDBTgaaaUDigB4JpQTTAKcBzQBKpYnrUgXIpsaHg1KDigBhSmkVKT8tNxQBCRg0hqRwO1R4oATBpQKcBS4oAQCpYgMimhTipEGKQE4QY6VEyck1Mp45ocgDgUAQhfal2ZoB5qZVzigCHZS+XVnZ7UhT2pgVSntTSlWChHaoiOaQERGKjYGpiKjYUwISKYakI5ppHFMRERTCKkINIRxQBERRTiDSYoAQClApQKcBQBGRSgVJtFIVpiI6DTiOabikMQim0/FIRTENxRilooAKWkpaADFAFKBSgZoAMD0oAFLilAoAQLShKeBTtpoAiK+lKEJNShM1KsXAobAg24pQlWhD7UeVjtRcCttNKF9qnMZz0o2e1FwIQopwT2p4Q56U8J7UXCxF5foKjKNnkVcCgCmsme1CYFUACnA5OBzTzET2p6Q7e1O4EZTjpTSuOtTMQtRFgeaaEyNhgVCx5qZmBGAOaaEB5I5pgQgkik25qbaKQgDoBTRLRCUxRwKc1Rn1piFyKQmmk00mnYLj8ipFlAAGagzkUwjIxk/hRYLmiro3Wi5jQ254rLDyg8Emqd/cXpjIOSnYAVnIaKcqsbgkFW9qmaaVEOWT8BWUFAbcxOfrTi8ch2FyAepJrGTNEhJZn3khyPpUDXDZ5PNStCA4SMliemaspYIIWeePLAZxmoYWIbC0F7KQ7YAGfrV66sYLeLMYIbsSc1RiZS4KDbjpjio7m5YfKWNMCggaVgYpXEnd3PAFFkJrjUkheWeeEZ37CRmqF1qMt7djASKMDGR1rWtNZTTLMwoEd+cSD3/nViOpMek6XAlzJEYsZwCSSK5/VPGizAxW0KlD0J5z+Vc3fard37ZmfP4VWXb1ZcmhFGufEN1NEbcQQCNhg5HNQw26TOF3Ksh7scCs1iByBSOkwjD+W23scUwNbULM2CqXdJAfTqKqWpBJKuQvdT0IrPO89S49iTVmGTYpG0H60CNJru2toQbVm88nBXGQKzATdTDJVSR1xTSC74AyT0FTTWLwwrK7EEnG3uKAJ7V4rS4IvY5JIeCAGIAP0zTjPp8l7uYOkJIGQMkVmggjHJ/WpbaMTXKRlwmTkE9KAOrh1I6baCfRNSmiA6xyHIP4Gm/8ACRanrzSx3E07jaSVhXOfyrNmhe7YwllaRRhSo4PtWr4WsdSsJHuoXjBUlJreQYZh6igLnOXYkhYo8bIO24c1uW9tNf6Ip+3Qxxgcgjkc/WrHiWMa3Pvs0eJoxlxIuMnjgc1h2wMt3a2lzGIsuA5J4I9KYrlfycqwRC7jPI6Go44pHXeB0INdrrCaZp1tJb2BjZyuSoOT9a5CKOe4iaIzCMDGQTg0mgDULuK5ESrGwdR+VVthCb5AQO/NWVt4bZ90kpDjowORU6xpcWk0spPI4xwCPWgZSkeAxhUHzetVypPerCWxZkCkYYZyeKjKOl2Ihw4IAPYUhiK7BNhojGSv4mrd7YvahCWDFuoHUVSJaJ+hVh6imBJK7CXEbkA+lBd8hGcNjoRUbMX5PJ9afp8AuLxoySCRkYpAJMhMXmFiSOxp32RxZrcBgykZYDt+NOjXeSJA3l5IOKa0JFyYLR2I7DpmmAFITGpiJJPYnNVyMGrptDFbmSZZI2BIyT1qoBlqTAvCzEdrFOWyJCARjpTpIYw2I5Vf0PTNPtIBPIEkZkjUEjceM1EphF5IJ8BQcDA6ntSAhkmmt3MYkKY7A8Vfs9SCxRo8KldwMj55IzVCcxEl0AYdsipdNktmnYXG1FKnB6DNVcGj0m/njOjGTcB+7BXPUnqK5STVJr3fDCBaoRksCc5+lWdMu1h0WRwEnkt3O0PyAtb9jeaXqWmS3Yt0BQEOm3kH06c0mB5bKIySUznJBJOcn1pCcIAat3FlMJmmMRSF3JBJ5ANOkDPEI4ypVcE7eCDQBELgpAIw+FPUY7VI29bBGE+Y88IRyKqFCBz2oTBYAnA9aALbFpvL2714wSTx+ddBb639lsUNu4DodhhIyCKwrS5ktVZEG+NzgqT0J7ipdPSOSaaN4WMOMn+8Pf3pAdppetRJe28sTiGSYYlQj5JB7nsf8a7awvba5jkkhkjGwHdGDgofpXkV3DFZXsME4MluyBgSOQPWtXRre3ltprmJpA2ShAc/OB0HNNMDprOwIVrqRP3kkgkDZ98/410S3lut8bYklnG8ADp7VxtpqDDRHkkmmdVyDCpGYx/WtDSzbx38KxyNcSPHvhIcZx3BoA6eRlUEk9KhWQPnFZT3V7fSlbe3EYB++ZOv4VqngAn7x5P1oAQ02loNMBtFFFAC9qjccVIpG7BOKeVBU8ZoAqttCfWmx4HQ0PC5PUgjpigRMMA9fWqEWFYHpTm5Xpk9vrTUQKKJCuCrZIIxgdTUsYjzFAwZ8EDOAaEkyAQWGeme9cxqmpPDMQ+JLdDh1BwSe3Peqq67dG2ieJwjOShRx8gHrUsDs7i4FtatKSMpjIJxnNZt/cWmpaFNJcxlY87ASeQScZFYZ1GFba0j8wSyICGC5IyD780yXUluJRYuvl28mHAIIGfyoAz7MxW9w+yR5rdfl2FcjB7jnmmrbWU99Lb73ViB5XGMn0PpTr21Ns4ktWdDzgoMgnFULGSYyvcOnmSAFfnGMHvxWbKTNuKdLWKb7RN5chTaU3AnPbinWtrbSeTK140eSCqmQ8sOuaw7ewlvZ3kBULCpYuTkE9sflW7DYo2mte3bSFwQCiJnr3IHIoTBmjFq8NrNHu1ERB2JC43DPuRXR6frFreyiCORXkIJHljII9favOp2tJ7OWRF8uSHBjIOQT7eldl4XmE+npduiJcEEMFGCwHciqTFY6QnmkoDBgD6ikzk1QhrjOapzkJyTj09zV5l4rMvIfPi2klWOSpHYg1SY7EbuqKdxxmoWAGDkH6VlXuptYXcVrMoYEbiSRkAelWI5hHMSVYQPyhAyMe/pVXFYtYpjAAZzzUmQ6sUwwBwSDVeUkHGCKpCZOgV0wRkGqs2lLICYSFPcHvVq3bCgGreOKLiMH+z7qJiMgqOnNNDshIIwRWy5z1qhcwK4YqPm7VopXIaK/mbuDSFcqeahDgHDAg1IDk4HSqsJkTcHFNqYxbu4FM8lh6UxEWKTacU/kHBpcg0wIsEUAkHIpSOaMYpgPDknk05HOahNKDiiwFk89KkSTaMVFGQQATTyh61DRSZOJAe9PBzVZOGFWVHGamSGhwPOKeKaBzTqzsUmNJwKaXGaJOFOKhJIGTTSFcmBGKUuB3qFW460jFj06U7BcnEnGaXzAe9VQG5I7UoYg80coXLIbPel31X84Dg0GUEcUcoXLO7NKDiqYmYHg1KkxPWjlC5ZDUm4Z5qNpBt461HvosFy6p4FSqeKpJJ0GanRuKloaZaB9RmnZwOKiV8in5zUFC7uRT81EOtOzSYD80FqZk00mgB7EGmE03dSZpAOzSg0zdRuoAfmkNIDSk0DGEU3FSnpTCKQhhFNINPIppFMBuKMGlopgGKAKKU0AJikNKaMUAMxThRRSAM0ZpMUUwClptLQAUmaKMUAANBNGKMUAFKBzSU9B3oAcoHHFPwKQDFLQIU03FOoxQMbijFOxRiqAZikIp+KQjmgCPFJinGipATFGKWlC0AIBSgClxS4oC4gFLspwHNPCigCAr7UoGKm2CmkDOKAGAA1IqUm3FSopxQBNHGCvUCmuMHFMZiM4pvmE0AOzQTxxTA2TSk4oAD0pvFOoANACAcdKdt9KcBxSgUgAA4p4GKeAMUFfSgBFNOIyOaRRzTyOKAGKOasIvSolXJFWlGAKAFCjFBAx0p1MJwKAIZFqBl5qWR8E1AXBNOwCEVGwp5OaYTmiwiIqTSFcCpccUxulMCAimkU89aaaYhuKTFONJQMTFKBRSikAU006mmmAw0hpx60w0CDNFBGRzQAB0GKAENJTiKCKAEpaAKcBQIAKVRQBT1FAwC8c0YFPxShaAGinqM0oTIp4THagBVA9KmQYFRgYNSr0oAkUd6COKAeKTNSMbjJpwiz3xSHigk0xCsgUYFMxTiSaKAExTgM0lAoGPwAOlRu2OlO61E/WmIgkJJNQPk1YcZqJkJFUtxMhyQ1PBY8AGlCANz1qXOB0FNgQE4ODSEg9aVzlsnvTCCKaJGSHtUVSN15pypkdKpaCISPSkKHuKuJbgkZFTGAAfdpXQWM0LinbKuNCACTUEzJHGWJAA5JNK6HykQCg8nFULy4DoVjOR6+tUrrVwZCkalwe5GKozXchjOAB9KiTKQ4w5OWIH41XkMCNtVA596ql3f7xJHvU8QRckjmsWykOVXL7yxB7AUsssjZBckehNJJIuQQTUY+ccVF7jEifY/FEgeeYKEA9T2/GhcK2SM1L9oQdOKaQHLSLiRhsK852nqKChIwsTAd811/jCyjtpIZkADYwcDrismMXmrwsF2ARgHJOM1qQYToV6jFMAZmCqMk9BW3arHd3CWt3EXwcAIOc+tTatYHR5kaN8xnojAZHSkUmYl3avbIu4glhkAdRWnb2tkNOBuLiQOQCADxVZd+pXoJQdOcelX30s37iDTSxZRlg56/SgZlrIGjKDJPvTIlMkuwDntQ6S2F2YZUwwbD5HQVeu9OAgFxaksCMkg8YpgZsscls4JGD2rVh0u91TRnvhOuIz9xh2xzz/8AWrMNrcuciN2/DOK6jw9f2semyaRegxtLnDnoc/y6UCOQimNvPHLggqc4PcVoTRxGUXKHIIzjGMV1kGlwm3lWYJNCgwCVGcfWuXnurVZHhjgbyQTg55H40AJZvdwyGe0JjwcFc8mr9rrc0+qM4c4IyQD3A9axRLJHOGjd1A/halklMkvmAbW9QaAOotTqfiKeS3iuIonQH7w5x6fpTW02ESNa6jEY7qMcPuwD7g1j6fqq2ELgwh5D/wAtAcH8aoXF1PeTF55WcjgZOcCmFiQk2t7JgqxQkZHINQzSlp2kHDHHNMMmCQKYSScmpGOlmeYgyEHA6AYFG8+WFBI56DtSADODTWGH2igC9PP/AKLHCwyw5Dg1QkYs+4sc+tKRxTMEnAGTTAdFJILiNt5JByN54FXLtTMfOLR8gDCnOPxqhsYgsozjtVl5lYLti8sgYIHSkA2NTJNHCP8AlowU/Suj1DTRoV/bS26GTzFIYEAdcVz9sC9zH5YJkByuPUV0UNvquqFZJ3yIXIAIA/AH8KoDCuI5be48xxjzSWA7D2/Woi7GbzFfy/Ug8irl/BcTXMpJDeWduOhH4VmSI6Ha4wfSkBrz3EjRoLtftCEZVzwRWaR6VK7XDWkZYfuR0J6mkQDILDKg5IHXFAEzXs0kYTeEQDBAHJqNIhIcAgD3Nad+BcW6TQ25C4+8Ov41mKB0HWk0BFIAjlR2rWsZRDZhbiBWAOUyMYBPWpNLWS4QQpaGYhsknrim6xdAXvklGAQAEetCA3rGxt4dclgUBYbiHgE9Cai8Px/Z9Wv7B2GCCMHjJBqDSxNEFvLqFpIY+Yjn5gAegFXPOj+2y6o1kskLsCryHBB9hQBJd6S5sriOZNsajcH6knsK4iJiJlCKTnrjnium8Ua6JQLS3kIjKjeoGMnuKzjZR2caXEFwBuj5BGQc9aAKaWUl1c/Z4iNxGQTwKgltTbqY5D++VirLjp71p6ZLEl3JcTyAMqkrkdTWfNcxzyySMv7wnk5oAjYkAYOMEGrtkIjcK91O8KuMh1OOfeo5rSW3tEknXYJBlOc5FI8CS6Os+8bkPf60wNPVJLi7mgMqgSgFRjuOxqgk32S+RJWKxhgXAJAIqxbQ3FzYwXCzBQpwd/UGpRCJs2s8JMhfi4UZGe1ICzqRt4dVfUNJcvGuC4PIOR096t6fdJYeILLVgM25OHVT93IweKjkEN7oi2tsiQ3MZ2vGpyGIx0+tZU9pLZWMMkhIDHBUcce1ID1u8tg6G6tgOAGV0PBPepz0B9RmuL8FazBDbS28zSgKeGJJG0+o7YrtjgqGUhlPQjoRTTAjNNzTiMUlABUbsVGfyqQUEAjBFFwK6BpiAcjB64q6BhajTAYVPn5TTAjOOpphwTkU8j1phAFMAFBAJyBk4wKM0oOCCKkDz26gikkmhOflkIYg4wQfT3qG1mIRluIfOhAOwgAHPvXSaxaWSyyzQjbIDkt1GM8j+tc5q9k2kzo8d19pSVMgjohPQH68/lSYyxBC9lcJJbssUhByZDkYxVK71K91GEtN5KQjPIwAenbFVLWRZ3Mt6pnYA/KOGz/XtUNzbhbcsgdAOiE/5xSSEy7b6zNFppiVwXBHlEjoO/8ASpra7uXea0mh3SSYySeAcdT6VLoAii1KzN7abI2QojOOCT0INap0+3stb320wImQlwzDGOwBFFhXMJ0khu4Le3mwGOWi6gEHv6iunIu7KeOZri3hQqDcWwB5GcEj8KwoGSG5trbWHxbyElJIj8wOehI7c1LcRb9Uc3cxe1BEcZY8hOmQe+DQhogtZLKU6nCk6iEjdECMEjPBH0GK7PRrGaOOyaWNQY0AIB5Ax1NcPOlvFdrZRljMrmMOoG0qecZrq/DZj0+8AMzNDckiMk8AjPH45osM60kIpJOAOtZ7atDDf/ZX+QEfKzHqfSrzMCxA5I6iub8TpJNFG9tb72U5WQHGABk5PemBsrqMRmmjjkWQpgscZwSQMZqvqDiK5ilLAMMoeRnB6fWuHh8QXZ8pcRbZGAYoOWGc4q3q11JqF9FcLnyyoABGCSDnGPxApJjKPiq8W4lsgAZJoQRMQOoz61VS5uI7eFRO5UkHySefp9Oa2BEZtBvJ5Y447dUPlkHBLD1/H+VY4SGYQ5cLKACY24Lgeh/GhsGjr7C7FzIJQAm0bZFxjJwORVknzWKn8KzdLs9wkkgul84kBlZsgDPHTrxWoYZhKcRkYOM9iPWrTJaFjQoRgVbByooVBwCOaXbTvcVivKMZqjHIWJzwR2q/PGXjIA5NUxAYlZnHQZq0JlS5hyxdRVdUdcMeKke4J6HirMaB7cEjJrRMhoq7zTlyaRgVJJHAo8zPTAq0SI2CeetGARjFN2kn+tPAx3JoAhKEEk9KQgelTnmomU00xWIitIBUhVvTimkU7jsOXNWEY4GT+dQA4NOzmpYFjGTketTqRgVBGQSM1Ou3oKzZaJe1IaUDinhMjNQMgYE1FJ0xV0Rjviq00aqxKkHNUhFYNg4p6nNN25OcU9UI4PSrYiQAYxigLzyKeoAOOtSFeBU3GVpIQ4zkg1D5TdjVwox7UnlN2FO4WIEjOcmnFMD0qdY2FSGIEe9K4ymBgYpwUkjHT1qYw4PSpBGDGAaLgQiMgjGCParMaEimrGF4FWYl46VDY0RYINSKDUpjwDTVwODUlDlBz0pxBAp6kAUj8ikBCTTCaeRTD6VICZpM0lFABmkzSUlIY7dSh/eoiaAaAJt/vRvGKizRQIkJBNIeabmgHmmApoo7UmDQAooNJRSAKXFAFKBVANxRg0/FLihAR4oIp5FNxTENxSEU8imkUAJijFOxS4oGM20oWngUu2gCMLzTwOKcAKcBQAgFKKXbSgc0AAGe1P280qinkelAERFJinhSaQjFAEZFNIp5pOKYhmKTFPpMUANxTgKMUoHNIAxS4FLRQIFHNPApoODTwwoKE5prEDrUhxioXBJoAQOKmSQAetVSCDUig4FAErHI65NNI9KUDNPVeOlIRGAadtJqTaKqahqEWmwq0g3yP92MHkj1PpQMn2mnAEVkLrsrqGFmAD0JcVHJ4pitlQS2TMxGfkkHH1oA3BmnhTWRZeJLO7njhMMkLOcDdyM1v+U3pQAzFFOKN6Gk2P8A3TSAAKcKQIw7H8qUKQeh/KmA9OKl3ADrUYBHY0pBI4FIB3mr60x5ARgVVcFOoI+tME3vVCHuck1Cx208yDHNRM6mgYBie9GajOSOCBSDI6nNMRKGprHNN3GmkmgQhPNJQQaKAGmgUuKTBoGBpDSmmmgQE0ZoxQKAGmm4p5HNJigBMUYopaAExRin4oxQA0LzTgBTgKUCgBAKeBx0oA5qQCgBAvtTwvPSnhaeAPSkMaF9qUL7U6gUAAT2pQDS5oBFADgKMUbhjrSFhjrQhAabxRuzSDmiwDutKKTFIWGetOwDjxTdwFMZiQQDVZ3dTyaLCLZkAHWoy4NVhKW6GpFbAyTVWAc2T0oCnvTGuAucDNIJiwzQBIFGPemMp7Cnxgnk1MAuM0AVACD0oIJ7VaJABwKiLDPNAEIhDDJFO8vb2xSGYA4zSecD3piJl7UpIAqEEnvTsnuaQwYbgRWdqMcQtzvO7rx2rRJGOorPvACG3AFfTNQ2ByJVu3FXI7izjjEbDfJ3GKLpCvIQjPrVYJms3IpDJIopJCyjaD0HWjycDjNPe2kjTIX361CJph6VNxleSMqxAOaEYAYzzTmLO3J5PWlWLIyOfc1NxjCo5OaYFBOSeKkYAdT+VRkheB0qkwN3xbA91dxQr90dfbip7HSEsNOuYQctIgIOa1Z7Vbi5Jfv0q2bdGjAx0GK2MznfDOiwwRvdud8xJHXpVDV7Y3sshuA+AcRlRke3Nb9lH/Z0kqySAxnJJxwKxrvxOkNyYbYqI1JG5uR+FIaL+k6Hb2FsZVGZJEwQQDg1L4csxbJPIeWZiAcVgW/iJbyeQXFyyRp1HQH8K1dF122lEwQEQrkiQng1QFfxrpqXFoLtEAkQgEgckcdaqaTaWh0JHlAKr94k45NY2qa89xqLpJM8lsCQAOO9ZdxdhlMcMjiM9j0oA1NNv47XVplhwYSSQCcfzqLWGtHVRBCUkDAnJzn0rFX5Tmpg+RUjNWbXbg2oggTyFxhiDkn8+lZS3EiAgYOTnkZoJzSKuWwCMnoCetICd7kXQAlZUZRgDHWoSCMnHHY0yRCCQRginC4fyxHgEDpkUJgNLcZFNDcZpzJOxLeUee4qPDKcMpB+lAxzuCfpS4IUMUbae+ODV7RrA3eoRgsFz0DdCfSuqvfDV0LcHegQkZUjAH4YoA4Zcu4AViScAYrUl0a4gjWRreXJGcEdq7+y8KWTpGk2BtwdgwAD603xPNZWumm3hlP2hRgKDkgYoA8xcAsSBgemc0xHMMoZeo6VuWFjca7BNHF5aNCCckYJ/H8KisVm0e5ju5oCVzgg9CKYGcMzXAAUZcgBRwCa14PD5M7o4kViOOMj86kuLU60ZLyzQxshzgYGBWpZSSCyWSOZ5LgDBOeQfcUgMyygvbKGQt5YMZ4jK8kHuDmta0eOGyuLkSK07/8ALMHhc+1U7Y6oXeS62yKQchsA1iyRzXF0/kswUHkg44ouBDdy3EV7MfMkDE5yeKpku5LOST71dmZ/MUySebtGAe5FVX+eQkAjPY0AOEkhhEJ+7Uke0OnmfdLAH6ZqPy3GNwxUiQ/aAYdwUkEgn2pgbWpRppu60gJZZlDLngqD6VlWqZuFDg59AMmrtzfmewgguAVuoDgOFzuHarVhaE2o1JLjbMhIIYZ4/OkAywupLS7mUM0QIyMr1FaFrbrqUgSKMGQgyF5FJx7VS1JjdrHIFKMOMnpn0rtdKtxZadDCRhgoJ9TnmgRgzSlTJDJDIJApUYGQPepmiSbwlvkGPLUiM9SWHTitpIVillIAJc9SOgNSWsENtamIqDGDnB7UDTPK7gGbkrJ53WQkd6WKFIZ4yG8wHqAea7S/0iN9OvbiYZmY5TB4A9MVjtbxXFoJoYVAjABJ45oAypPOibegwPQjmooo4JiqGJvMzk7TyR9Kts8InEbOJ8nBIORinXEMaSgxQtt9uTSA0hLpupW/2e4ikT7OmIjnBPbApNU0G3ht9OhcCO5dwHVTyQfX3rOspWF9HIGAniIZCx4OOxrq9LaLVb86jdxncJNgXOQgxwD/AJ7U+YRyGox3OkTS2iuRbsMgH0NaOm2ZudPNq8jQXBw67+ARz0ra8Q2sM11DbhFILg+cDyBnGD6ik8a28NtJaMqA4t9gB9BgCi1xtmGI5kkUwgRTQ5w0Zwfzq3c6Ld3FnZoWY3U0mQpOQAecn0/+vWp4VZG8tyELTJnBXkEdh+da+p6e9xpZvY3aOa3cvHg8BeMijlJsc/JpMnh0CeWUSmYGOSJcZHoRW3qCyWGgwTIcSDGCOCM1ia3exyx2UMZLScFiTkkkjrWpLYzW9xPCQZYZoN8W89DgZA+lK5SOmtrhJbS3LuBI8YJB7nHNTYrmbE4tdLZ13IFKHjoeldFGWGVfscA9yKdwJR0pCOKAaWgBhBprGReQcipe9BGRgDNMCEXGHw/A7mpXwOpxVGe6hgcq4yfrWZ9qujtS0uAIxnPmAcD2z2oA3C4IJB6HFAk+UnqQMgetU7WQSKGIlfPWQjAJ9hU27nikBk6jqgnE1jPItuGUZUIWPPY4Fc3dj7Pp9yWXfCGBUZxkjOOD1610N9YXM121xHKPMEZAUdSccVkNZ3l1qENpqO0SSRuI2A4BwOcdc03qNHOKl3ef6iMyMgDHHYVD9ovC5uJnZhnAyOAR2Nb50yTRgRcQSBTw1zG2VI9x2qqLWH7LNDHdI25ywVepFQ1YGRSajPHbw2k0+PLO5Q44X2HoKbZXcSTQtM6PGcoQrHj3xUNzLbYADAkAA7hyCPf8aZoxhOr2sky/uQ/JA9qdxWOhttNca5Zg5mhnjLISvI4PGD9K6yTSUudLijuoQJooiFIP3cDr75qlYKbrXJrgzYNvGIwTyCT3FV9fF3ppt/stwHaYlSG7YA/Tmi40izb2tlpkSPcCMK6kKqLksD1Prnp0rJW+tbezEEUsLhHL/MCrKCeME9SKLDWIxBJcXZkFwoIAB3KoB6D0zxVRYIvECzPaMI2EwYxOnz4PcHPT2xQx2Ohh1y9k8u3WJX+Tc8wHzeXz1965q71G9+xqizSS2EcmCrnBPqCR2q/9lutCv9rMGtJkK4I5IxyBWZrVygjhhEX2aGQc4bIAz3A70kyb2K03kuyzW8AgYEEKp/MU6S5uZJy6I2H+UZBOD6DHeo0EocyRzxP5f3UHQ54wfpToI7qPUBLcSxiPO4hZOAfUDFJoaZfu7qW30yCwuoW8suWcEc7h7/jWdIy+WXeNhGhHlk9QPer13NFd2kzmRpghDHdkE84xmqUOIr8FhvhJBKAlsEjjjvQO5vWN+ulLDJDEskbkmTYMkA9we30rYj1lL0sYJ5oWTIAdQA59BmuBmQw3jTW7PHCzYCk8j1FSRTTtNDLId0MEmVB6g8cUJ2EzvbXULh0H2tgA+Gikx1GOQfxrZCVx1pqURnguJnAjQ7NhGQAe9dfaXUd3AJYzwferUhClOKheP5Tgdat4zSFM0wsc3LpkwZjGMjOcelTWkNxHFh42ABIzitcjDYp46Vopk8plSWgMZ4OazpYGjGQK6JlycCqk9k82cdD1FNTFYxAxHWlD5q7LpwSMkE5xnJrNGV4PWtjMnVhnNSlwRggGqoY4xmnZI607DJfl9KY0WTkUKc1MvGDSuBWZCp56U0Eg1edA6ZFVGQ78YoQD0OSOavxxqQMk1Wjtxt3EgY7VJ5wUheuO4qGNF5VULwKUELmqYuCSADwamWKWQEAgA+tQ0UEjAgjNU3TJJFXVspMksQwP4YqZLAHqRRewFCMHbwKdgjqCK1lsYwQQOaf9kWlzoLGQvBHFTBlPerxsgTyMUgsI+uDn1o5kOxVABpQtSNDsOAKaFxRcBuylCmnAU8D1ouA3Z7UbD2FTADFOAHpU3HYr7DUqYUYNSbQR0qJhzxRe4IlJyMVWMmJMGpQSOtMKgtk0DJVNP2kjNRjjvUgfAxSYDCh9KjIxU/U0OgxmkBUINNwasMBTCKQEBFIRipitMK5oGRminFcGm4waQhKMGjNJmmFhwpwXIzTBUinAoAXFIRTs0hoAZinAE0uKkRcigBgHNPC+1SeWOwpdlMBhFMIqUimkUwIyKQinEUhFMBlJinEUAZosITFGKlVM0pTFAIYo5p+KFHenYoAbgUAU/b7UoT2pAJjigDmpNnFKE74oGNAqQAEUu0UnI6UAIQAaY+KUkmo260IQxjUeakYVGelUIXNKOaj704HFAXHClFNBFKGpWGOpDQKD0osMQmjcaaTSA80WES7jSF6TtSEUWC4uMnNPUcUgHFLnBxRYLkoXvUgHFRq3FSZoGOC5rgNYu5JtUnlkRlAO0ZGAAOld8GxSPtcZZEJ9xmkB5l5gc8ZJ9hmrNvY3N1Jsit5Xb2UgfnXoKQwBgwt4gfUIM1HdTvGCFwPpRYOYw9P8LuBvvJTERyqRtkg+pNX5NOdTgand/wDfVL9rkPVjTTIz9zVcpFwWwm6DVrsfiDUq6bdfw6xcD6gURbi4wM1aVjnkYqWikyAaXe9tYmz7gUv9l6kOmrP+K1pRjIBqUdKkZknT9U/6DBH1Wmm01cfd1ZfxQVrvVZzg0wMuWHWV/wCYnGf+ACoCNYH/AC9wn6pWq/NR+WT1HFUIzfM1joJbc/VaXdrP/Tsfwq86hOlQSSEIcGqJKpl1kdY7Y/jSfatYxzaQH6GrCyMw5OaX71DQ7lUXmrDrYR/g2Kd9t1T/AKBw/wC+qsrgGpgcilYCg2oaiBzpY/Biai/tS/B/5BTfrWpmguR0OKVgMz+17wddMcfnTP7ZmHXTZB+JrUMj+tN85/WiwGZ/bRHLWUwHc09NcsmIDrIh/wBocVoiQnqAfqKa8EEy4lhRs+oFFgGRXlrOMRzKSe1S45qhPoltJ/qSYmPpyKqE6lpbjJ86L/P+NAGzijFVra/hul4O1vQ1aANACbaULzzS0opiEC07AozigEE0WGBGKbuApT0pFAJp2C49MkjjipgvrQoGKUsAMmk0Fx44oLVXNwoPJpROD3osBPnNG7FVzNjoaBJnrRYLkxcjpQHz1NQNIACagMmepIp2FcvGQAcGoy5xnOaqjcehJpjzFTjNPlC5dEpzyaDdIg7/AIVnecxPU0uSe9OwXLxvARgA0sc28E8j61TFTRjgCiwrk7TqgzyfpULXKsMFKcyqFJIquQAelJILjzKSMACnJG7jJ4FMBAOQKa0zkYB4p2C5ZEKenNOCgdKpLK4Oc1OsuRkmlYLkpcikMpHQ1GWXHWozyaAuSmc+tRs5bvxTSAOtNLAdKYXDbjrTgQDTC5PXpSocnAxTsIsBz2qTORUAYDqaXzcjGcVLQ0VbmQgkA1SuLhxlFjYj1NXLpRn5AST1IFZeoyGOIgAgjqCKxkUV7hzIpJI/GqJOKuW8XmR+ZIOTVa5JLkYAx6CsWWhklxKwAB4A6VAWI4I5qQccc59KEEZkPnEAehqLg2Q4wcnpTWbcvydPWrMwtcjYcn0zmmHZsOBigCsIWOSzACo2AViAQR61NJjGBTQRwMA+5q0B6EoGcnoKoT37S/ubRS8h7jgCrM2ZIymcZogjWCMKorpMzGuVks9NuvtJPmOMAE55rJ0/w3u02S8uEYsQSqk4x15NbGuWbzSRSg9SBjtWuH8jTQEGCUwB7mkM8cnBE8gIwc9K62x0U2ugPcmZhJKOE6AHPvVe48M3r6shdVKynOc9RXVaspGnwW4w0gAGAeuKkDzqHSp7jUfsq43EkkjnAp9vpLHVTZXD4wCQy9M4rqZkGl38U7Q7mxghOtYWpXxGom4ijaJ+24ciqSKM2+sks5Cgl3kdfaqmcDFS3M0k8xkkbLHqahz3oaAASKt2ccTqSz4ZM44zSLDLcwA+XsUdx1q1YafLcSFbckMBkkjj86gRUkYuxYnOe9CzmMYCL9cVcvdLmsivmyKSTjCjJFVJ7Z4UBkGMjI9xQA5JpHIya0Wa3IyDgjpms6NGVAcUBlL7WpAWnneOeMkARA8sOMj2rsILwQERG7UWkq5Vpmzg+grjVdGtjABnJyPapDcOLcQNggeo5FVcZ6NZRHTrV5jMJAQTk9x9a4+Oz/ti4eQAqXYkHPA+vrVAXt3NEITNIyDIwPSp7OOQyHHmKAMkii4G1cG08PWcyiQGaZMFAOfr+tYB1GaS2jjEIaNTnLjmo7u5lv7gC4ZmKjALdQKdp9vDNcmB2K5HGO59KLgbWhala28DJImJnOQFHBFV1u7qzu5o4YECyEneoyQPYVmtJJp9+QGwyHB46itVfEccEyTvZSFlGAykYouBBNfW8TYtmkeQgg+ZwSaW3024k2SJLCokGGSI8getVLrUob/UnvUtSgIAYEZH1rTkl+xWKTworydmxjBosBRsrK1h157O9y8Y4UnjnHHFP8S6WlkY7hJVMYO3ZjkDH61XuH8xGuHKyPJgk9x9PSptNlWS4S4v7gTDGIwSTg/SlcDEmmjmIKHkDGMUkciI6+YMqTg4q5rMdt9oD24C5zlQMBemP607QZrVLmaO72CN4yMuMgGmmA5xCXknE6HYOO4I966ayvrW60kzyIkccYAIBGScda4SNTHceZHxtYkZ6GtW22Xt6oNuz785jjOAx9TQBs2+rQX+JhGA1vID5Y7r0zXYORNElwvQjIH1rh7pbKzvrQW6GA7iksb9QDjqK7iKN44BDGMIowp9qYNEdxeW9o6LM5G4ZzjIH1qRvnUhOc9CeBUc2npNKksnJUYwehqwnCAHr3FAGH4lkks9OhjSUqXYo+PQiuVuZHWwltYrgvbq+flH3vqa7y/0+G/Ja4QMApCE9QaytI0NorURzDCiQsCQCSOwpAcRM0arGYUKFCASep+tXJrlldTC+cjO3HFXfE+lTDWJXhRShAOF7VU+wXNlJD9ogMeemT1FS0BRJeSXzsD5GHQYxXZGKW3htLm0gaSG4QNJCh5yB1FcytqZtQ+xRuqCQ9SeK77RLadFiE4YRpGI0z37E/jRYDmrXVGsEZhAZgWJV8gEeoIPWpIrC48Yan55DQ2QAzk59iBis2aC7mv7iRYQ3JJBPAGa7uzj/s+3hEMbESRghFHG4j9M1SYkc9aRQ6F4oMcK4tYxtIJzgkcmuwmZLjTpooDu3rgcY61yWpaPdaUl1qlzcbnlOViQEkEnoT/WoNL/ALUt7eK6muCgmcCGNxksCcZ9v60rjLfi6z8h9NmijJYEIcc5IxXR6pbmZLcqQGjOcHuCOasXdh9pkt2k+7EQ5GM/NVl4w8qNjOB0+v8A+qgDChtCltYbOR52CO2cmttx85qtBHtiCMDgOSMjGCDVojNNAMpRQRimkgNgnmkBKBTZEkkjKxg5J5IOMCmsxGMAnnnHpUkZE8YlQggH16UwMc3cJ1Q2y7TKmdyOOT6YqTUJrcWhSQYmIICgcDPWsHVJTaeIpL1gqYAYByQGGMYrIuNfjmmeYofMbG1l6qAMY560gOqF3qVhDEUt457YjALPgj0zVsG4ls3n8lEuCM7FORXLW2qxQ/u9RuLh1wCqIMqQRkZx0q9pmszTav58iyx6eV2xt5Z2g+5ApXArnVJlu5xJdqJjhQAOAQen6Vk38sr3cVzDfTl2O194AMZ9Kt60txpl0FlRZg4aSGdBknJzyKotA4toZEUukx8wADJBHUGqTsNFe5v7qTzElkmeEYWUE4BxVP7bG0oKkxgjAcHkfhUd5epIJMMY3kIDoBgYHeq5ieZGmjhCxoMEjuB3oauBILglWjOHMhySRyTUVvcNHN8vBwQVPWpY4jdzp5BKkjAIqzEM2MxlhHmxuMSEcnnvUt2A0dK1C7tx9ptZlaaM5aJhww9xVy619r23kb7OY5JCcnOSoJ7VhPYS2V8QXM2VDF4xwMjPJq3buiTRhmCqPU0kwRHdzNLLDGjJGHwDt4wfettNYj05Y7U28NsCm1pY+SSc8g/Wsy8uLQkeXCPOQ5VlHBPvWdcrcJL5kyxkOcEKcgYobuM0muriG/Ie6NxGqlRI5yBnkEeh/wAapFmmkiiunJBQxoQOSc96IYIJDIHk2qqcRjgginROhvYVtAAoUnMg4BwaRJoaiYbLT7DKoWCbSFGMnjJJqC+ktTaQSJFmMkiQjgqRjrVpXimst9wwUW5IycFWP0rB8y3MjrjCscg5yCe1MZeke2isykLypIGDIrnIkGadp0624aVptlwDwW5BH0qoCIwkc0QwMkc5I9CKbNEu9AedwzQM2JFa7UyzXEZBJYKowATjP8qpPbRokdwHYqwJwTgg5qq0ckcTiA4jIyF7GtGyVZLaOFlkdojudScBhnOKliEEx5AKmPk4PfgV3Hh5/sej/OMgONpHTaQCP5muTvLu0u71THZeWiEYAABz6HHHpXQ+H5priZIZYykcmWCHtj/IqkwOpDjvTiRiqkwaJjgnFSI4K1dhDXJL52nFTIMjmmk0oYAUwF2gNkUhIBpd4NRty2aaBoz9UlKgY4BHNYj8uT61t6jCZYWA6gVitE4PSumnsZMYCQcinM5brTQD6Gl6HkGrZJNH1qcDgVVQ4wanSTOATipaAmB4qORCTuAqRHUcE0pTJyOlIogMjAEdiMU1OTT3TB5FIAOwoESRErICK1IXHBrMQHcKuxnArNlI0gQQMU9V9qqxP0yatBxis2USjpT8ioPM9DQHPrU2KJiMmkIpA2adkUWAjaMEdKhdAOAKsEjFRNyadwsQiMY6U0pg8VMTjio2NMTEAwRT+gzTARmnbuMUWAN3akxls02jNIY4gmmkYNP3YFRk5NAC54oBzTacKAJAcCmljnFJuOKaetACk0hxSHNBFTYBh5pcA0hpy0ANK8dKbsqQ0Ac0gICntTSDVkqCKjKc9KYEIBzUgBxTgntTgopDG0U7bSYoEIOtSKaaFqVVxTAlB4pCKcF4pGHFAEZph604000wGHrRinEUuKYDQoNOEfoKcoycVOqjvTAYiAdql8sEcil2gUcUhWIigHAFKEHWpNoPNKFApAR7Ae1OEftTwKcKBkRBFA6VKyjHSmcA0XAbTSKeRUZznApgNbpUbEVKVzyaYyHNAiI8ioyO1SEEU0g9apCIyMDNIWHrT2+7VVjg/WmlcBzSEHipBIMUxVGM0EAU7CuTB804vmqxfHemmbHejlDmJ3fAzUZlA6moHmJ71A0hPeqURNmnHMrAAZp5YAZrLSZkPBqcTl160coXLQm5pfMJPFVkIA5qxAVYkipaHcnjYkc1MDUYAGM96lTGMmpZVxQDTwPWmlwOM0hkx3qdQJQBUE8KuCMc08SZ70hcdzmjUDPe1YnGOKfHABjKmrmc9hTkA7incViKCE7+mKtmIAZxSxgAAipDzSvcaIgD+FOJpaQ1IxSc1G6bxTqcDQBVMDewqCUlQRmtAjIxVWW2AJJYmrQjKklbdyajLkjmrklsCcmoXtuOCBWiZBXD45o870oaNgcVGUbPFMkfuJbrU8TkHAqNEOM4qWNRnmkykTg5GaCKBSmpKG4FJtFOopCExjpRTqQ4pjEyc04EEYIBB9ab3oFIDNvdH3ZltD5cnXGeCaZYamySfZLsFWHQmtbdiqV/ZJdRFgAJF5BAoEXiOcjkU2srTdQYE2tx98HAJPNaZOaYxWYetCjJ4qBvWkEhHQ1ViSScsAAOKiVzuGTSFmY5JzUZbDU7Cua0ci7Bk0yaVMEZxWf5x7GoyxY5JzRyhckd90hIPFSKGCjFQxhSRxV+KHcME4odkCIAT3qQNgUssZQE1SkkOetCswLEkw5ANVmkOc5qMmgmqSFclE7gYBpCSetR9KN+aYEg4NTxEEgVVzmlUntSY7l4sifKAKdG3OSeKgityeSCQKtLCSAACM96hgJJIhGM1XLZ6VcXTwfvOT+NL/ZaZ+8fzqbgZ+4mlFW5rWOIZGSarHaBVXAYTgUxQ8jYDAD3qYBCMvwvesy6uzDMRD931zSuBrKgHBNS+X8mR0rD/tJuA0y59KvR30rQhYQGPfNTzFWJJ5BGMk4quLiI/wAYH1qHVJnS13SoyuOnFZ1sxkTeVwD3IqeYLGyJYyOGBqJLnfNsXp61nGTflFUsO4FaNpbqsYdiEx1Bq1K4NGgqEgEio5Zoojyc+1V5NSSLIBD49KxJLiaWQuTj2zUuaQ7G3PqkcCgrDknvnpWZNfRzElomye9UHvGOVyPqTUBuCeAQfpWLncdjXjRWXIGKzLuZkmK4J+goju7gLtDYA6VWd3acu5z6VD1GmJukc5EbflQbS4c7vJcL6kVbTU5oVwiIB9KZLq9zICpKgegFPlHcr+X5QywII9aYZMmms7Sck80zpwaLBckyD1OKMA981EcDk5zQHx0p2A9CIpRSlSaUR8V0GYhAYYIBHoaRkDAA9BTipHalCknoaTHcYsKAguQAPU1lyXFvJeSPGJJtozlRkA1LqqTTXUFvkhD1xTZpbDT7ZkjuIoWxgE0rAcXLeCTUpWe5IwxAB9M1WZtOlkLXM77j2JJxUMzxzao8YmVoyx+YdDzS31jAko+zkqQMHHANJjRWFvbvqAVNxtz68U5LBZL0ohxGOgJ61GxmQYY1Nb/uQZWclmGBntUjNJ72ztlEUodvZaf/AG0CBHZwiFPUjNYTuZHLMeakhYEEDqBkClYC9GPOuTJJIXwSDntVS/uTcTlScqO1O08tufcMZBOKpSh0mYspGfWkkBb81TCoJ5Bqu7ESFgMg1HG2QVP3uxpzuygZHHrVWAQzbHznFPE0+POCbkPfFTSQiFo5ggdDwQeOKszXkYhSCzGQRkgjpQIbHHNDIu8bcjI9xUwvJoSQjYz14pLjWRNCsM9osbJj5lHJquk6XBZY42JxkGpGdB5NvJbQzTHY0g7DqfSsq+iNjfoyk4BBGe1S290Tp8aSH50fj6Cm6v8AvrpOeqg0gLGrRrcTJdkbkdee1ZZmOPJjGV7KTmrdpKfKNtNyoxg+gqlcWbQXIFvyCMjtTuBMIpkt28tAmQQSPSl066NxKltdXIjgPUsOKbqV26QLCq7XxyMVTtwNyOygqpBII4IFUgNeC0ik1F7YSmSNc7XB64qV4tOeN1uZ1idCRhAcn8qQzWdzMRpxWBVXcS4wc+gNalh4ftL7RQbm/hgYk5YYJP1ptAcrK2+1JjTEYOAScn8aqKcVJMDZ3c0EFwJIwxAcdx6+1MUcc0rAWIlV1J9BmrmgyEaoj9WQEqOxPpVSMBIC1Pjji+xvMj7LqM5wD1+lAFvUbe7udTcyw5d+QV6EV0ml+IZDpItZJhFcxZG9+AQDx+lZb6XqEdpFqaTrdkqGZAcnB7dapaJItxNdW8gBDcrnoCKYHZ6Pd3uoRzGSdAY228p19+tauCG5rF0mOwEL+XIyMeGVemfWtVN0UxjQpOuASB94Ci4E2KcDUazI7lACjAfdfg1JtI60wGCOKNjL5YaQ9SRkms7UNJjvdPluLsuZEBZcHGDWrS43Rsh6EYNJoDi9JtUvbGSKWMC7hYSox4JGa7yIhreJvVQawtMt8QSk8SRttUjqBWraM/k+W4wUOKLAUYNLZJb10TPnkAdgBkk81uBAgQL0AAH5VEvFShulFgRI0aSAbwGA7EZB/CqV3Yi7mjLxAhOVA4ANXAc09aVhkaR+WGCZJJzgmq7EPAyRrubO0kvgj1xTru6eNwiRK2DknocfWsrWLq1s2t7iFefMxJh8EAjqc0WEWbeWaw2LNMLqNjgEk5U+voetXbS4+1eZIMeWGIDgYGKw7KS1vrJLU3EcExJaLEgY/jg9a37FTZWdvb4GCSHJHANCAbMFjlBZsZ4AIyCarCWJtzecpAODjsa0b2yFzF5ZkAz2B5P0rGkW3s7uz06M8sSWJ+negCw07pEQ8UgjII8xCDgHoaz5Zru20eZ2Zbe3Qgk5O5h7VR1HUJo54ozJHC0Nzs8vPAAGencEGofEeqQ39p5EExDEgBdpAYnjqen/ANamBVhnhuori9uLpLlihVA4ywHpisXTJtPivUkvULQEEEEcVC9xNps4SMpJIBk5HA+hqK7eEyRSQQsYyCSOev1qWB0V8mm2rxSW8Dh5QTHJ5oIPfkHtzVabxFfrEgQeS4G0lHG0j0K9Ko39tLBZWjzyLh0JjHoOOM1TkSSLAkQgMMgHuKgDsHuTDaK6pbEFA4diSQx6gDoKydT1L7XaW8ySGOTBEmw4IPr+OKxhLEoAKNsByc+tWJrOMxI8M/yHoCOmaOYozb2GYhGkOSM49SOKo+Y0YOwkZGCOxFbscMl1YmNnO5TyMZKj1FUZdMa28u4EokiOcHGCCB6VSkDQQ2VxGiSQzBcYO8dAK13tXmCTJdqFkOCvQH1/GseK7llt2i3KuGznpmrsxiha3jt7rzoSQTGwwVY/hyKGriLa3kkNnewWjIFIBckZyPY1lF0mdMjJBzkdMUwqTcSBiQpJGQe9Wbe4RdMms3QDGWRh1x6UrDQ+UEWryIAdpwQR2pn2mJ7KOMxl353EdAP61CgLTElise3GB0PrmnJIY4DDIpEZOTjmkDGLIY846EYApMZjjmKsFH3TnGTT7oW8SsgZkkBBCkZBHsaihkR0lBlCAHIUjg/T0pgWLgedCoKtu4Zo1PA96rSgIQyDEJwR6g+hqdZ3MIfziHxtGB0H9aosHwQHJU+tMC2ZvtDB5owEAxwfyqaKGM2ctwRKTEQApPBBPX26VQBzgMMjAGKv71bSZkS72OcZhPQgHIANDERxzRmQK5Yk9D1wfWti1jjMkyM2JGQFNowCKzIbeG8jcFzFtABbGQPWlnCQzKkM7ybQPnPA/D2qGhmnb2Ekl2ohJkjQB5n6ADPr716NCdNtfIASTzgpZU2Etk8kntz9a4BL/G94ZAS8ISaIjOQB2Pr3q74evr+PWIYUmcJKhOCcge/PXFUtBHdTIJCc9+3cVEItvAp1lML23NwRiQsVcZ4yDjI+tWDHWidwKxU0xyQDgZxVorzUbLTuIpCYjrkH6U4TNUrxKRjFQiMgkc1VwEaQsCCMe1VJIRgnFWzHjmnJgHkD8apOwnqYhIByRTggnBCISR6Vrz2MFxyy4PqKWK0ihUBQc+tXzmdjnypQlSMEdqAc1qapbbmMy9T19zWSODVp3JZKGPrVmGYA7T36GqXPWlBqmgTLsyENnBIIqPbg0RXGAFY44xmpMg1IwQ4IFWN2KiUAHNOcZFSxplhJQBjNSLNg8mqQUgZqRQ5IABNQ0UmaIkGAc04SCqwBVeaUMR1qLFJlsSgHINKZs96rAgjiml8HFFguXPMyOtNyDUCsTUgOaLDHmo2HFPPSmkgDmkBHS5xQWWjIPGaokTcKQuM9aaRioTkHFFhk+8UBhUOfenjBxmm0IlB5pwOaYeKQHFSUTYGKYeDTdxpRknmkAtIacBTtoxSAhI5oBpWHNNFIB1KBzSU4YxQAE0YzTSeaeOlAABgdKaVx2qQEGg4IpDGEcU0DmnnpSDrTEPAGaeAM00Hmng0ASgDFMcAClDU12oAgPWm049abTAUDmnbaaBTx1pgKqkc1IDjrTQTilFADy4A6UgORmgUooAeBxRQDSikAYpRRQDQA7PFRsCTmnigjigBnUYpoQ5yRTgMGn9RQIjKe1GwVJikNAyBox6VXkQgdKukVVnJU8VaEVGDEYNV5AT0qy7cHFV2BzmrTJYze6jANMMhPU049KiYDPFWS2KXJ700k00nmgGnYVwJpuKcaAOKYXGVKjAYFRkc0A4NJiuWVG7HIFWYQoOAw9+azs5FAOBSaKTNssOxBFMebaDis9JzjGeKcZSR1qeULk/2ok4waeCWwSTVWPlqugDaMUmikORiG6mnnLEZJGD2qAkg8U9GJOTU2C5YUjIAqYLxVYHnNTqxqWUidT2p9V9/PWnbzUjJCRSEgCoi5J60mSaLASF8UK+TURzTSSDxTsBZL8cGoJH5JNMMh9ajZiaLAIxzTGGRzTiaa7gA5NUkQyFowRio/KUcjOacZBnFN3Zq0JoDxTQeaUnNJ0NICcHilqNWOKeKTQ0BOKTJoPWkPSgYbqN1MJ5o3UWYrj80maaCMUpNKwNjs5FKOtN3U1nxTsBQ1SyywuYh8wOSKt2k63FsjAcgYNK7b42U9xis/TZPJuJIs4UnAFVYDQk9KZtJHyjNT7QTzUiRg0EtFMxuOopjKe4rSKDHSoXiBPSi4WKQU0hU9hVownPApREAORzVcwWFtowSDxx61optCjFZyEoc1L9p5wOPepauNaEt0QYzg81jscsautIS2Cc00qrYyozQtBPUp03JrS+zpgYFH2dAckAYquZCsUAkh5CkinpaTuMlNo9zWlGADxUwjLGk59h8pkCzuME+WcA9c0+G3kMgBQk9621BxineWKnnY+UqxIUGCMGpGbFSMgxxUJqGx2HBzmpxyKp7ualRyRjNJjSHTFApLVlT3MERJbgDvVq6li3FCrZ7msXU4IfLMqFie4xnFLmHYjvbiS5GLcEL6msqRpA3lyyD3Aqa3eVlAQ4J6elVbmBTcFRIxY/hUXCxOVt0jCBwSfStDTrmSDiBN/OcnjFZC2rxP8y5/GpEguncBCVU9MUJjOsu7hJLJ84dgMgAZxXIm6lcEE4IOCK1njurSywDnPUkc1lR+UoJkHJOc07gWtOulhkKbsMxzirOovKYyVI56gHNZDTQxOCDVyG9tSuSfwouBWRnAwcj60s6PFESWBJ9Knea3mbKsAPeqVxcb92AQOxPeoeoFKTOeTmlUAHjk+lMJyc1NbeWX2yA/hSAkAfZuAxjvUTOSeuauSmGJSACfbNUWkG4kKBQFhrMcEYIqMZz2x9adJKTkA8VDkg8UwLAYjgDJprAjkjFRBmyOSKeQ7kAEse9MBN1KDn6U0oVOD1oB7UAdvPe6oIyYbaEe+c02C91brNbI+eijj9c0hv7+OMmWxAH/AEybI/lVFPE6GZ0k3xonXIroINCS61qTPk2Mcfpls4rOv5detoBPNOsYzghQCAaztW8ayEGDT0Kj/no45/LFc09/f3DAz3MsmDnBPFJsZ1s769JYkvNb4P8AEQQR9DXGvPfXMxgYG4cHGfWtCTV7y4gaE3HloByPXFZVreT2bl4XwSMHIyDU3AtadZMbsmZMeXnjPIq09xC0zADOO9VI7+Z2fPJfqQOlVXco+BUPUpF1zHMxJwQDxVe4YcKCSKDIAQSADjkdqjYbpNwOV7g1JTGEcUiFo3zUgxNOkKDOTg1Zv7WO0jRU4Y9apE2KRkJOUJ3eoGakkJcAMfm71piNItPQoADjPPNVpLYeWJGBwf4lOQaSCxnsChyOKnKNNbHA5U1v6XHA9tMCFyvQsOxrPjSMxyBMZJ5IOQaYESFbiwIZwpQgEGojDbpbpItwTLkAgDtT7W6kgDwtHGYSeSR0pjBReoHQrHkcH0oEbupaVYT28T6e4luMZIBJyPSsqW8AkijNsY2Q4IAxzWgZpdPmN1bFYhGPwNYs17Nd3RuJCNxbJwMU2hkl0zLOQpwMZxV15BOkLj7wGDWWxee7QdXYgAeprbu9O1KzsCzWgwv8SsDj6ioaAhKeZgKoWTvikDSW8mZskD7uB0NRm5eGBHZFJbgkHkH+tMFxJMwCAEg5wRRYAuIZTaSzOCGzkEimpvNqu8LnHbuKsXuqSN5QwvBwRipri3htJRdIuUcfdzgZosBSaGxa5iDyGON/vEDOD61sSWWjWlt/qvMX+9vOTWA6B5CwJXJzgHgVau8LZxqA45xlh1ppgR2+ni9mlkgXyrcHO5ugqtKBFKUDhsY5FXbRL6ztxcQuDDKQXTuee9Gqz/a2VhbrGVHOD1p3Aimj2xRfMCCMECmC3bcDtDDIyAeSKkVDJax49STVxoBNZRyIxEiEjA70rgSbbrT3P2ZJzblQWw3TPcEVVjuZoboTWUZXIwSUyD7nNaehag0Qkt7gIIyc75DgAc5B471X1JLSM5tLsiPOPKHb6e1O4FdNS1GwuHm3BPN+9gDB+ldT4dnthEbm0Pm3L/fE0g3YrhpyS5+dnXtuOTTIZ4wwwJY5B0dSRTA9RfV7a7nNsocSRnEh2ZAPoCKsuZIQxtVeUDqDggiuP8G6jb2V9cm4vERZAOH5LHJzzXbMtvcxGfT7kEAZPlnIA/p3oAhF+C22S3khY9PM4B/Gpw7/AN39ar2+pWV6n2eS4jkYcEHg5+lTXckWnwiaRwsIOCCMk/SmA5dy7yI+XOSc9TTHF55Z+zmONu+7nIqhH4m0+VsRpdSD1SEkfzq7Fq9i7Ab5QT0DREUAMg1SVM/a1j2Zx5sZyAR2PpVwXmYftCmN7cDJIOTj6VVa3jg1RWRR5N0CGQjIyO/603TYRZ6jdWoJMLAOgPYc0ATSa0IQkwhM1q3Bmh52H3HatZHWRA8ZyrAEH1FYAtE03VYzCSLW7YoydlfsR7da30DhcOAMelSxlK/sbuW3ka0WJ5jgAO+AB3NcPq0EllciPUo41kIP+rl8zePT2rvbvUTaJmExGQdQ5wAPc1xOtanp15cyHyo3uTj98DlAR6Z5qQOfh+0C5ke2tfLRwBtJwR9PeuysfFVraCONGLRlcPHISSjD0PU9a5ITPG0wluF3HBBHII9jV0xLqh8lVjBQAmREJ59CRTTA6G68TJNJDIDDMsZyRGSrKPTFPtdRs9R1Q6lIhjEZzvkIABxiueguobWeeNrNJi4ABIzt46//AF6i1q3t7VInRMNKMkZ4I+lFwE1nVLe81Zri68udg2B9nOAQMYJNULzUBckeXAscIHyITkD36dajjjtGGUQpxg54ANVHgnQpuhkVXICsVIDD1GetCYCLh58ysfrmrNtfvAHPUDhOccUmpWkMFwiI20EDJJzk1CYo47kwl2AHQgZpMRNdSz3NtC8hyqnAXuOacZjcQlwHG3GwZzxUEM4VpAZP3YGORwaVgVx9mmwp6hemaloBolcEcA845qV5EQhOSuc4FQKrCTk4U9TQ77pAqEkg8EDtRYZcSYwkCF2VmIBwccU/92X2zM59CPWq0kkc4Q5AcDDYGBmnLGphcMQX6g54x7+lJKwxkkcNsrrLGXkfpggcVCgaRs54TkCnyeT5BcxyMcYHOMH1qBCdpI4HarQiaRJyxJXLEkgZ5I+lDQXCRLMyjySM7gelSRSi3txcAKZg/wArdSB6fSq5mkNvKhjBWV9+QSOaBDkkYZXP3efc1I86S25GcMe1NtzNFexSEjJwCAckg8Yq7KiXE0oK8AArg4z60mirmdcF5pEJ5O0AmqpUggkcE8GtCZrWOYkQB1HQE9aEsor7BhYQ57YJH51VxFd+EBB6kACo3XaSpPzDtVq40u7gw2PNjAxvTnH1HaqqRq5OW2HtuBAJ9Kq9xksMmSsXQk4BI4Jq1FFhJImALdcHse4qCaPyDHDIPmGHJB5HtT/Oea5LkgbuMAYA9/rUMRYt41ZiTnavUA4qzICYwpjVYxnYxGcD3qoAVJKjPODzjmpHklVUBBAcZB7H6VFxmhb2yG3jXyi0j9MHrWtokTC6LovmyWo+TDYIJ6k+v0rDtLsBIw0phycKxXOPU4q9Fp9zbzhY7lWjmORLGSMj6dqVwOrj8VQ2ssMAtRGskgEzseQT3ArY1HUltCBvCeZjYCQCR3PsK8/1C6juZnR4myFADdyRVea4mvGRrljJGE2RknBUD0q0xHpNvd5tmlmwMAnAOSR9KlglFwpIGCDgjPQ9a8+tpTbxIFme4Z0OxDwUz17/AFrofDl19pEskJckALIhGSfQg/h1qkwsdEVB7UxkG7NTYyOQQe4PamleaoViFlBHSoSh3HFW8U1lyapMCoRnuRQ0hAAp8kTdQaqybgcHOPWqTEyYASxYYZFZd1bBW3IO/NaSE49qhkQvIRjI71aZJTMSmI4HPrVcQOzYA5NajRAYAHHekRAJMBelPmFYpfYmA5YE+gqLZIrEDtWwIBIhwOtPjtAowRRzBYzoldlBwSCeCK0Et42wSM4qZYUAwBwO1KAAcjpUN3KSENujN3A9KelsgPGaTfilE4JwDU3HYV4wBjFVJSBkZ5q1uLHk1UubclwUJBJ5poTIRMyHGajknbdnNSSRiNxuFRTRiRcx8n0rXQQsd0QeTVkXa/3qyDkMQeoo3Yo5UTzM3FugxxyKezBl4rFSZwMZqzHdFSATUuBSZYYck05WCjrSAhxx1pu1kOSOKQyVmYjIGagEm5iCCKTzieKTaOtFgHOQCOaUSjABNQsQQaaDTsK5dD5UGlDVXVjtxxTgSDzUtBcsAingioEcHvUgNS0UPLfNTwWNQ981YQAgVIyJhTKnZeaiIOelACU5KaAacOBQAh60BsUp5phWlYCQEU8HNQgYpwNFgHmmgc04GlFAADinA03FHSgB+aaTQTTDQAHrTacBRimAgpwoApQMGmA8dKdTQacBQAZo3UhAAphbBpgS7/el3+9QbjShiaLATA5NLk0xOCM1KQMZFIBQ+COadkEVXJIPNAJJ4NFgJwRTs1ECadkiiwDzSGmFvejcDSAU1UuDyas5qKVAe1UgM1hk0Y4q15AHUUxkAyMVaZDRRkHJqJhk1cKDPI4qNo1yeOatMloreXxnI+lNIwcVLIMHioiM1QhM0AjNNJpAeaYiUnPSoiCDUkbgE5GaHIJHGKLAMFGaU03NABUitx1qPNGaAJ0cg8GrSzgAZNZ4J7U4MfWpaGmX/MDHg1NGQRkVnI5BqzDMSQKloaZeBOaeGwMVEGBHBo3EVDRdycHJpwNVlkIPNP8AMA71Nh3J80Aiq/nAd6UyErgGiwXJZJUUcmofNDdDxUQiBbJyTUojyMgYpiEJOaQnipQmByKjZCT6UAQyElcDrVcrLjJAx9atGEnvinCIAYJJpiM4FicDJqeGEsMsce2KueWoHCgVXkfYSAOfWqTuAjx4HAqLBJ4Bp4mY9aC5pCGjjrS5HrTDk0mCelOwXJN4HU00zDsaYUPemEAVVguSeYTSFxURJqSONmYHFDQrj1yegNSFWxyKsRw4XgUNCWGMgVDZRVORUZyatm2bBIINQBTnFNMTGAHvWZcKYL+NwMDPNbATmqWrRN9nRx1BqriLisCAR3qeOorBBNZo4IJxzVsRgDpUNlDCeKYetTMoAqE0gFGMYFRvkU6mtzTAgdx0FNFSCMb8nmkdQDwKpEgqDOakQZeokhdzwSB61ZhtzH0JP1pMESAACmEEnAGanCZ61IqAYNQ2VYijhI5IOasDgU7IAOaiLjtSuUSb/SjfUO7mgvxSuBIz5PtULEZzSZzR1FADDikMgQEk4FSBM0PEm0+YSB7UgMa4vLcSF9/4YoOs28sDQw27SMRjpxVbURYqSkIJY9/SsR5HjBjRiAPSpbGSpctDOyHbGATmq80wM5cNuPrjFVJSSx7+560wHAqWBpQXpEw3p5oHRT0zV9vEJDhRbKPfFRWkMVpaCaQAP3zzWbPPC7NImA59BTQHZ2kryKokUEHsRVLXrO2MZkUBGBySBxUmkTW62wkmuAWPYkf403UrmO8Uxx8g+/WmI5COM3FwI4ySPUmtD+z3hOGBb3ArQS1e3gLiJEx3JrOXVJ1mYhgw7gioYxJCUOBE3Heqk0jsmGBGOgIq5NqM0jBvLUAfjSM63IDSuox0GMVKYGQSQfSpIixbIqaS0UucSggdwKjCeWcBs++Kq6GDyN0JphORTpAxGTUeQOtJMQpA2jkdablQOADTSQT0o69KaQx2wvyDSZKcAkfSkwfUil+tMQmSetBPp1oz+VG5c9aBj7jXdUZSHfyUPZTWdb3c8M+YTln4+bkZ9aqq7OwEjkr3JqORyjHyzwOhrW5JdukukmLvGvPUgYA/Core/FvvLR72YYB9KlbzXtQXckn1rNdMMQTRcCSSYOSQMZPSo8jFNzg4pSCADSaGORyrZGePanBsvnBJpqTFAQDwaktA7T5QDI7GkDGO+W6EEUkTsA4zyRxRN5hmcyAA57UwcUWBMuaYoaYuRyOlMubh7i4YMeAcUlrMYJNwGQRjFJLsLs6jGTnFKwy8x82JNpyF6ipftiyP5RTEJ9BWfC5TBB609Buc4PWiwB5iKXjDFkzgHOKamSBGoA+bgjsahmVo5MEg+4qeNWjZGYgDrz0osIszyyWtm1v5Ybcc7gcin6lNHNZWjRuGIQA47GqdzcvNMSzAjsRSC3lMJlC5Qe/JoEaV1qFu2hQ24O6UD5j3AzWQjDYKjJB+lAOMU7jJpC8bhxwwOQfQ16RpMl1e6Kkhu45CQQ5K5K+3vivNJJC/J9MCr0UkltFvtr4xiQfMoOAaGBJq/kwz+TBL5ygk+YOAefSo7eUQwiQN8x74qoBHnLlvc5pz7CCkJLEetFgHYae9Veu5xxW7rAjijtoQcqBjPc9KybSIi7t2YkZcZINW9WUfbgoZ26Elumfak0BXddpIHao3mlZAXJcDsTUqENcoXOEJwaiu4fs1wUBYqQCDmlYCW3u3fERyIx0UHnP1q3LGZ4BsGZBkZPUj0qrY2S3AZxJsKkD60gPmb1B4FIC7b28MEMf212TJ+6p5A96lvRFbyJJE2YXTK85/z1qpb20ikSRxGQ9geRUtxD9ohDiMRuf4OmKkCKK5iQPG8W8SDGT0FXJtIHlR5vI0VwABsIOfrWbIHtmjMkXGSQD0NW5ba41W1kvUuIgsJ4hJwR9KpIDMNqUmKyH5QSCwORVZeZCiAk+wrRMUlvND5zgiQZJFJb+WhIt2AmdsAngEVaAq24R5gknCnqR1FdFpmLNLuezkdGgAPJyHB7EVkppkzXcZJUKTgsoyBnua1rSSx0fWri3uJt8bxgLKeQDg/wCNAD59f06+mjM1iFkYgOVOCT6girujTWkdzLaXt5LAUb90s3KEeuD3rk4LcT6iUA3gOcdsjNaGqXENztjMJEiDBcdSO1FwN2/1O/0u9kMM4RRjZsA2yD1966uz1S1vrJfOdYmYYYHjn1964O6sI00O21OIuMERyQtyAfUVp6FcrdQm0u1QODviJPUE8gkd8Urgbl011awRvbsriOQZJGQAehH6VLYavbz33k3Mf2e7QEEEnB/w/GsK4srhdZlj01yihQ+wklWxzir9vrNhc6pA0qAtMvlTgrkBgcjn8apagdM0IfAdQQCCPr61MAaSOLyVCbsgdD7VKAD1pAYeu28wtJnjvXhDjGxFGCegyTzjpXJwaVHoKwXF+6ESZDqy52+mK9IlgjljIdFbAOMjOD61yOt6JHZW6XjhpFQMXDNkE44GD0pWBM4ydUvWupreBUgU8ZJyR+PSrGk6m1hpU0VvIUYvlmAyceme1RCaL7AxVAEmYgg8hfpSaPewWJfzoZJWkIKhACTj60rDNyxZ4o7eWWOLzLpCAHyA+D0OOlZGtW8tnestwxGzB8onJweuOOa09Z161uLQFrGZLiMYhZyAAfX37UpGpeM9KtWMcIELkPJwMnvSFEyoYUhA1KaGMwg58ksQGweDVzUNWGrbYTa+Uxx5YaTIUe2BxU97FcSkx2UYnt41AARdwLDtg9Oc1gQBrDVIJ76N4thOUKnI/DtTsMW4bzU8uSDBY4EhJI4PapLc/Y5DcRSLJ5a4IYZFTkW2oLPcW8LlosuDMeCPYdhWc9w0iuoABYYwBxSAdczG4uHmMcaF8ZCdOB2qONwCyt2GRimojrYmTAOw4we9W9EgGq3RikBCwoXyoySR0BHeiwFIRtJkhshugBAJPvTUEkEhdR86E13knhayVor/AFJ4UeccxKSBn2PauLuEW0vZjGhESuVGWJJ96LANmkhldZ1BRmOHU9AaRZ4gjAkRkjAfGQPrTGMEkiqgIDYxkdDUFxbyW8nlyMC/oO1OwFiKRIrkpJJuhIwTjGalmaF0iWGDciHlie5/nVWGO0mvFjmdo4yMAg9D7moj+6u9scm5UfIOcgjinYRangkjthOYCsMhIQjuRUZ2yiNl8x1AIKL2NaR1qVpYGkKedby7kjwCrg9c/gP1qe/1bT57SSeGIW+oGYOYwCBj6DihIDLjYWt4swQkAchvelkO4sQTtfkAnOPakaaa5gDytlu2RVecBHAHAIyBSaGizKItojhQBgMZPJNK9/M6OQUj24BQDHA70sEqNZSM5HmRkAEjtjrUZjZ4RKVGDySPT1osBbspnuC5hJDgDgnAcelQ6kJI3UTSMQ6hgrAAqenam2qSJdDYwVl5BPQj61NBcW76zay6m5NoH/eA5zjHb9KEAlrpk1/YGeJclJCmWPAAGcfkD+VNjjjIBUgfU10WrpDoSC10ufzLK/i84GQ/d7ZB9welcwFCzARM0iZwCBySfShiJJpCq4GOfQ5q9MBNpFsiOu+MlhznIPWq14Q5WMQurx5D5AJJ4q1GtvB4cLiRGuJJQCFOSoz3Hbp+tJoaIB9qCgDLAdMgHFSLdOuSImUYw5PBAqsLg2JSSF1dnHzA8hSe49KllnjuITCxzI3RugJ9aVhlu1u0himjx5ykjqeQe496rm7Cl0MbLFI++PPYccH86R7R0ddk2GVAThcnPpS21hJfWjiCSMyIcCEsAx78ZpWEa1k1vKg3ymTy5gBsPzAdeK6KwujJqPmW6LsgbnaMF1PHT2/pXPeHbSK0RLq4cwzRTBNoXJUn1Hf2x1ruoNO3XouI2Ah8kDIABJJzyPpVJBcuW7s6knIOc4PapdvNOEYTOO5JNBFUhJgADTSmOadgikJJHNUBHgd6hmhDDHQH0qwc0mM9adwM4oyAjBNMKv8Awg1qeT7UjR47U7iaMd2cHDA9e1WI0O3kc1ZYc4pAoFFxWFjAReO9LnNKI81IIwBii4yvtJbIPFKQwGasiMAdKYUyCO1FxmdNK4ICjOTz7UsSsWBbinyxgSA9MVDJcCN8A496oRaPFPBDVnG6JOFOTUkVySeTzRZiuS3kLPGSOoqmI2jjIPQjGavGXPGaUqGHIzTTEzDEbl9p596srHAQAf8AWDqK0Bbxg5C0C3jBJIzmq5hWKRhB6ClFiTyGYVqRQxhBxU8aIO1LmHYp2tqYxk5P1qyY1IIIBFTkAdBUTsFzUXKsZU9vsk+UYFM3BVAzzViaYGXBNV2UE5q4kkJJzgnApA+GwakaM4zUJXDcVV0SycOARk1IGUg1WUZwDUgUg4INJoaZLEAOAe9WFaq8akc4qYKS2ahopMkB5qVDg4qHBFSKOQallFgAEc0hUZ6UgOBTDJzjPNKwDiopu0Um7NP4xSsBGQKQimu5DYFKGyKdgFoFIemaYWosBMDSg1CG96eG96TQEwwTSnGKiDcU4tkUrAITSZpCaTNOwDh1p2KYCM1ICDiiwABTgOaKMimAoAzQxxShgaa3PNAMYWOabSNkGnAcc1RIDGcVIq+3HrTMDNSggL1pDF2kU8HC81EZDgjNND570AK5GaVWwKbgE5NPVRmgYwytnAJGKeJTjnrSlBjpSbBQAA5NLzSBcU4EYoAZkg0A5oYjNGaQhGGRxURU9xU2eaCMimmBUcYFVnHJq+Yyx6VC0PJ4qkybFBxnNQsMVoSW+ATVGQc1omS0REZFMqXaT0GaYRg9KsQUGkxSUxC0lLRSBiUtKBzQQc0CAcUo5oCkkAVKItvU5pXKGgGnBipyKDgVIgBGSKTYIlglJYEgge9TGQE1Vdiv3eBUkRBIJqWrlJkxJPIHFDAgZNKXA4o3g0rDIdxzUiOeBThGhJOKAoHShoCxGcgVIDUMZA4qZASelQxjgM1G45NTYwvNMYZNJAys7lOlR+axp09VSxFWhMnM2B1qEnzDk0wH1NKrgUxEgUU/YCKiQ5NTKKGMWO3QjJJP405okUcDApwIHekYhqQFRjyRURBJ4q2YwTnFL5YzwKdxFaOIswAFX44DGoJFS26LGAQORUjsW61Ldx2IwQKa5J4FK3ApgIDYxmkMVc4waYYfmzn8KmAFOGKdwIPKFRXdr51o49BmruBRsypU9CMGlcDK8On/AEaSI9VOT+tahrC05ja6tLETjJxW/tpXuMhcFhxTRGcc9asYFNbAFO4miDyieoqNwFzipyT2qBgSxz0ppisRIrvIOcLVgwg8EcUiKQRgYqc9KGwsMAAGAMAU9FyRTT1pynHSpbKsTGPAzmmk4GKXzOMYqMkk0rgIXPemFqVhxk1CSc0hEmQaQnFRbjSFjTGSb8GnBs96gzmnDOaBFgNUF4W8tpA33RnGODTw2KrXOo21qMTnA/maQznJ9au+VMKIPUrmoJmAhDGIOT1wOat6le2txFmOHIHfOKis53VATCoUdD61IGPI4dtnleWKjVUZgBJ+JHFal00005cKgB6Aiqz28kowYsYpNDIiY2/dmVnHp2qJkQHCgAUhIEm3AA9aYxOcA8UWGPGAcrnPbBqSO5uY3BR2BHtnFQYYLkAjHcinJJ8/zE59qlgXzc3Ei7pn3nHGTVQYBJHUmnSMrEFFwPrTAMsRkfjSEIxyeTTSuTnIH1oc4PUfhTd2KLDHnkbQQKaMKSM5PtTA2aCxxxRZCFJLHB6VGwANLtyCcn8KFVc5bJHpmqsgGhCx4xQw2nFPKjPGQPrRsHfNAEfNB+7TmAUcE/SmEkigYhpM4opDyKoQ/S9EvbkzIluhXOC7nGDz0qzd+ELi3tGeEhiOozk4rZup/wCy9YLTbvs4JKkcYNYd94nn+1TG2DeW/wCAH6VQhmjXNvcW8lldBFOMKTxz/SsO9t/s05QvvPf2qFyTIzEYyc49KbikAADvxRjLgZOCcU+EKbhRIQF75qYRob4DIEYPX1poYlxZC32Zcnd7YxUbRGMErIAfrV67IvWSGBS7D0FVWRrcvHLCckY5HSqJIhHK8Zk2lgOpFPht1dSZlcemBV6ARnTHEKksRggDpV/SdGuGUSTTtGmMjoc0WC5zrYViAcgU4qvlF95BHQDvVvU2jknMcAVth6xjANUGB2EHgjqKTQ0y4/2cW0XlOS56gnODUEpdXAyRkdcUEhEHyYyMg9qjMjPwxyKTGTHD8nrTpJS+ARjAxxUStg4PSnED1pBYYVHG0n8akGPJCliPUZ4pAMUpQkHcjAjsRQFiEjAxTelPI4zTKYBnnNPAB5I5qOpRgKMdaQCsT0xk9hWiNzQYZVD47DFUgqAB2PI5xS+c9xIc9T0FDAuWkjRCGTOdp79q179UuNNFwgG5SM4HY1z5Z1wqjOOT7VpqjSWIljlBhPBBGDmpsA/TVt7iMkJ++UZKEcE0XUU10CPLJUdCOoNUY75re7jcKAAcHHcGrWsXE0LoYHKxSjII7mkBSB8kkHl/WrNkI2aQyNgA5xjk1FdWrq0QLDcVy2PWowCuSBkjvQBuWeoCacW0BCoASdw5/Cnyy4uDC5yw7VhiQxgyoSsg7g1PcTeTPHIH3t1Jz1qWgNC7lKGMuAUBzg9jWPHcFJJ0kUkTDAx2NX9Zn3ta4bEbruIx34qgkyEMChJJyCOoqkwJ4ZbV4JvtALSEEDIzjFV4Yz5QB4yQKBGrEoo+cjIp6tIFCoVBB5Ddc+lUmA2S5ukJhVpFIOCAMZFRRohmQXMjoueSeSB6083PnXAkcYbGCQOtMlkMtwccnsKdwNWe1m0hEvbSaGW3kwAcZIHrzVO7gmg2PKGAlG5c+lMN/LPZCykhQkHKk9QanuL0zWUFvMCTDkA9TipA3o763m0F7AyLuOPLyeh9zVea1aD7Kzx+WyLuIJ+8R/8ArqnJLDNJaSXFrJHDHhJCvUntWu97a+dDaGRZY42BErHonof8aYHZaTFGY/tAXazvvAzyARxWHeW0drq89rCoJnkR1BHAOck5rRea8tLeEWjRopJAjkGQR1AB7VY0+7h1CRpZIRDfRkK8TdQfammBrMRkAHoMU5Sc1DuOeetOVhyScADJNAFgE9KqatbW95YPFcXHkxnqfUfSoP7WgEyKH8xGIXenOD6EdqvPBExMroGYDIJoA801q3tVuUjsLxbiGUkPGiBQMDqAPX+lc0LdUbMcjqByCD3rVu1I1S4lvWkVnchSoAAGfQVmzSqk0iCJWRhgE54FIZZ1S4t7iO3jguDMyAlw3AJNP0vWb7RlmFowRZsbsjIyOBj0plo0bwpG8MbzAYjwcZHvVeVZrV2hmjBjJzkc5+h7UrCLts1xdROyi4juEy29CQJMnJ4H+eao6pO8typJn2hQp845OR71qaDeEagkReWOEgpGCcAEjrnvSa3HEuomSZhLJwCyHIf346EVQjKiufMjZW+XCEAgkZNS2zTpbAxoGYdSeoFOt9Ke8tLy6tldY7YZKsOvqAar6bJby3IiuRK27AQR9c0mhpnQ6dFaSsLa9uSYpITIHJ2eWfY/p7VPZR2ejzW+o21hMbaWMxTK5JIHHzA8d6oarfpa3MJhClrcbfLkGQw+lEWqzakkUN/IUtznCRADJ7AkdfyosM1vEFvbWhhmjkmuLScArEX3FBjkg9sVgzapC9tJaPbmW1I4kIAcH1JqrNqBsbh4rG5kK8AEnAx6EVTFxKwZHxtkJJAHU0WAS4CRzKPMEkanqODj0NMDMZjtDYPOT3FSmY5EZ2jHfGTUAYIzF+crgdsH1ouA1gN5AOR6461LDblpolB++cEKecVNNYLEm5LhJcDJCnBAqH7Q4txCBwDlXzyDVXEJKEjuWiRSgBxg9RSbcvt6nue5pzK0lubsyF5S+0ggemQaQmMJE4Z/Nz856DHtSAupb3MlorJHhAPvZBzUYtJ76KSZApaIDehOD+AqNiGiJzlsZNXNOlS0udwLkuhDkdhUsChHEWABOM9CemakEE0Z2FsnIA2ngmmlHdHKrlQck+gq5GxuNEZ8hWhcAE9xQAW0sltqkcVwRIhOxlI4wav6xe200kcEUYHkNsDEckVRMiDyJJiCs2ATjlSOMg0QgPdhXcKDIEkYjOPc0xEmpXhvbkOAQiqFRCMAAeg/Wqu8hQMYI68dTUl9GUmcAg4JXj8KZcCNrZJo3+fp5ZPI96TGQxySI5cO2Sck5qZXLk5G1T1weDjpkUyExvFJGZFSQnI3HAprkJlUKybCCT2osMl8sTOI7cbye3SggW8iHOdwyD0qbSy8t6TH5SSbDgOcAn2PrUd0u22gjwTICcYHvQ0BNFe/ZiXiRhIRjcD/AENNSzMkX2hjl5HwSDzn1NCWfmBFDqWxkAngkdRVm2kcSEW6IFHUEZINIZuaJqs0cDLc2sE1vEyxysFzKeyn3rtNLe4R5LeaTBA3xnBIdfcHoR3rntD02OXTLm7mDHzyFMYGMFT1/Ou1EJeQTAZwgA+h5NNCaHurKhYDJHakLAqCD1GakaUJG7N91QSfoKxkvZDcxrHCzRuC4GQD+vpTEkaZpD1qlLfsLuG3RCDIDkk8cdQPWroBNAxMU4DJpQtKBg0xDwABTXGBS5pHJIxQBTkGCTTByancDHNM2Y6CmmFgBIHFKJDnBoCkjpR5ZJGBQBKORkUhUk8U5EIGOtBDKeBQBEbQSHLCoZNNt+pUk+5q6A5oMbHrzSuFjHl04EERcH1qqbWaOQLjI7nNdIsPt+tKbcHtV8wrGTb2ozliTjoKsFAP4SateUAeBTJMBTii4WsVGXFVmmOSB2NWZCSDVCQFCSelUkJk6zFRyalS4AwScVnFgDnJPGaWGYEkHnmnyoLmzHMH6HimTdCahhcDGanbDjANQMyWR5rphjGMU8xujYODir5XbyKpTOwc5FUmS0PAz2p3lx90GaijL7vbvU4GRk07hYaIo8AAVKsa4xgGgJ6CnKKVyrCGJcZwKAo9Kec4pBQFhCB6UoFFFIBCTUUgbqKn25pRHx1pDKis4bBNS+ZgYNPMY9KjaHIOadxCEqxyDTkjyOtV3QhCR1qBZ5FIGaaQrmgwwKrk8mnLKSoyahcnJ5oSC5KDk4pwzVVWOasq+QM0NAmPzimlsGpAoIGaUwAnIpWHcj8wetODg96rzW8iSEr0P6UAMBzyfWnYVycsO1AcetVi7A80wSMafKK5fEvbNDPVWNju5NSSPtIGCc+lKw0TKxzzTywx1qmJDu24NTBjSsO5NuFANRZIHam+dg4NFhFmmljUIlz3p4IboaLDAk5pVYZ5NLsJpuw55oEShgO9PDA9DUHlZNSBcACgB+407JNMGe9IxwvWkUOLgUm+oC+TSqaBXJSc0CmDJ608ECgB4HFOApnmelAY+tFhinrTWODQScVC7HNNCuK4BGKozQ85A4NWt1Mf5hiqTJaKQQgYxUUg5q4VINVpB8xq0yGiHbTTnPAJqYHAxipUkUDGBVNgVTG6gEjANCgk4Aq+Cki4PSoimDjGBRcdiARHAPBoaMgZIqwgyRVny1IyRU3FYqwhVGSM0hYFjngVMYyO1RtbZBOTk0XuOxAxB6U5CcUv2cjvmrUMaomCf0obFYrhHcZCkihvMjGfLb64q+GAGKd5uFIzwaVyrGYJn3YNW9pEYakMMTSbiSPYUlw+2MBTT3ECtuPBpWYg4qsjlTUofJyaLAWYtxYZFaEYxgVnxTAgcYqys2MHNZstFkrmmsmBmmCfNSiQEVIFOaIkE4qk0bBsYrZYAiqzxAnOKaYmjNaM4xinRwnOSKtGP2oRCD0qr9gsRiHBzijFWcU0oD2pXHYrHINOXk1MYgegoSIjkii4rAEzUgjAHSlUHpiplXjpSbGRgEClxUu32pdopAV2TIpuzBqzsFMKg0XAhzSrk0oQg804Jg0higVICFplBOaYjn9TVrbWVmAwpINb6ncoPqM1keIU/cxyDqD/AIVesJRLZIw7DFIosk1Gx5p9NIp3JIyD0pwjHU07FNJp3ACQOlRs/NKxFRHmhoQu/NSIeKgHFTIwwKljJc0A0gIoJAFIY1wSKhKnNSF+KYWAoC4wqc0hXilLjOKTIoEKoFSDAFRAgUofmmAjknio7jTobmDLplvXNS5BNPByuKBnE3sIikeOEucdhV7TbQtbl5LjJAzszwK6OO1t1bcYwT6mpHiiaMqEAB9BUtAcVPcE3BOwnHAAFQSXs0eMgjPTIxXYhLWyjciIHAycnJNcffRS3M7XDYQN0BPak0MpOysxY9T2pCUKjBINKUwegNNI5oEOBd8AsWA9aCu1skU0EignOMmpaGPMpHC4ApqSEODnJ9TSEKRmnqFAwRRYBsmDliee1RZyKkYDdwM0wkA+hoAQEU7BPSlMgK4AH5UIQGGSRQA4KQOQabJG4XJGBUucsFjJYntVho7mOLL25I+lSBmgkU4OT0UGnvICceVj3NN38YAH5VSAYScnim49KVskZpUA6mnYBhBoCmpUUNJgNyP0qVlBGByaQzQv7y/ltFutT0suCcbiSAPw/Ksi+1GyfTTFbQlHPUgYwf61s63q0N/EM6jD5I6ovU/WuQmMfIjkLjPUjFWSTWRsxl7jG844PQ1VuvLeQtEML2Gc00LmkKkjCjJ7CgCLgdaN4xgHFWEsbt+RCcDnnvTppASA6qSBg4HSmM2dJSey083EcdvISc8vg1RfVmed2MShnGNmeBWXuRWGCQo7A8VYuJoLhV2whSBgEGgRdttVmsGYNbxlCc4zjFT3eqXtxbhg3l257CsHcSCCSfrVkXUptRAV3KPbOBVXA2NBms7TzftkqK0mMZHGOaoak9mGlSEiTccq46AelUovKeby5n8tfUinXCWqjFvIWA6kincZXZ2dQCcgDApi8nFPAB4JxT4YwX5wR6VADAccmnJMVbgA/UUrgbyAOAaYQM5AoYXJw+1C2cVLBJJcZDkkkcZqq7ZjxjpSxuUyBnketIAlBRyuQcelRE0p603rQAbjmnAmmilzQBI77gPYYqSwUG4LnoBk1B7VIkvloQowT1NKwE7v+8c9M1GJTHkRuQCc47VDuJYkmgfMcICT3xTsBbMolTBAB9RWhYq115cEpQpGOAecis1bZzJtUbvcVLHIbdwGGSO2cUgLmsTBbhI4gRhcAEdRVY7owAwwe4p9xew3TxySMAV/h6mhysmJOo7UmBDKuOQcggGi0h+1h0DhCOme9Tz2xKh4xlSM47Ci3tt7BwCQOqA4NCYEEjuRtlkDbOAR6U2FyJBg49z0qfyrP5wBIA3QE8io44kiyxctjoDTuBLJauoJJB5PAOTTZIIw58x38w8jHpWhYRwuI7iSQA8/u/SrM0STyGeFxuTjB7/jUXAwI2EUhzVmzKvdkxbAwwfnGRitG50t0jTz41LOMrg/Ngdqy9iQXDBCyNg5zzTTGaNm9mLlz5Q87P3nHAH0qxcalayRNDHCgz0Yjkn2rCjMgPGCT1JFT27s84CxedIO3QUxGoso+xS2sytlzuBxxioLGEGQMIxIsZ+ZD1Iqe4y0aIYwZBghVP8AWm2xSwf7RMrKB0QnJJ9jU3A6O3uI7tjbm6kiUt+7jkHQjsPSpdbWFoILkTKskLhJHU4JB781mwzjVGhVJ1WQAFAw5I+vFWjZXwvw0qwSx98DBNUmB0NlO6YHnm6t3A2S9WBA5B/SroIJwScdwDisPQoHtI54XARTIXjUnkeo/lWuDVAYWsWlwoLw3Zt4VXEaKMlue+O/NT2E1zZQ2txKZi9x8hSV8kZ9D7+lajJGzKZAp2nIB6ZqlexveX+wOywwqGG0857UAZPjmOEiCcAmb7qIoxknqT64rhZPMEhEqsMHG5lIH612UyR+d5zzXCXCHKhx3x+vSrOpibVNGt7uOePGAHhkQEMc+tAXOW0eS1WW4E8kQIAEYkHBPPetXyZmSdCiRxzAKAGDE89QPxNYd3aC81IwW8EICAB3hHAHqa09Me90JWiNqJQXDiZfmBA6gGkIux6ZI17DofmRCPzd6swPJ9M9vrV6fQbSy1bfa+XBOgAMLHzIyD3zwR9RVWXXUmgF5YQPb3UTEgkAhgeCDmt6a1Jt473U7h5JnTYEjBQ5IzgEdRQVY5rxHrqwW1xpiBDJKAC6NjBzn8aZpa2Wpwq0ttFBe25UxkHG8gcH6dK0Lm0lEC/Y7KFIueZ2DN26k9qwJrhINkzlXulfAKZEZHcD2phaxDfvNa6hM1wI2aQkmMMCV6elZ003mBSAVZTxg8AemKs3kkBgIeHFy77jIvcelV3jaPCiMqQcgnuO1IRdjjs7yRZpmihGCGVjgk+uKypyUuCgAxGeMDGRWj9plkSbdDGdxAGRlgQetM1bUYtQ1FDs226AJkLgg9/1oTGUiy7g5BJPQelN2F5ACQAT1JwBTljaWUpCpcg4AUckUx0cRhucE457GgB09uYXPzRkAjBU9ahJAxg05juIBGGxwcYBpoRmyeAAecnpRYB+8vkkjgdAOKUsWjXOCV4Bx0FLmOJm2kuO2aEIaRQQBu7+lFgJrdUmlCyvsXGOvU1f0u2Nw8xDYMCZyQCDVO1UNcGPyPNZgQoyBg1ZkuxDCY7RPJbAWRxwWPcEf560MRAQ6qX8sgMSQD3HcVr6cNLm0+7tXEqzXDboVK5EZA656cnisgXd2sXlNMzR4ICHkDPpV7TpPtlvLZFELBP3JIBIOc4z+FICjeJcWUiw3MCccqQQQR7EU+6vI7mcRkiMAYLA8UpmjacRzAgH5HUZJ/D8quQ2dpH5iW1vPMSADHKmDjPJFMCg92rOsSZIUAAkYyBT45I/s88TgCSQhlkIzjHUH61oLpKAC4vilvGExEjn9457Z/z3qsljG9lI7lzMWwoH8Psfr/SpApxIJVdmERJORkZwe1LtO4qV2sw6AYFW44ZiskIVk+QlSRzn2qpDdyRzeY4DEjaynpTuMlWI5K4zjqO+KiY/KAzAYGBn0p32iYKRGoIPGT1A9KlDxIwzErxkcgjofY0AEciW8tuXicbAAcjqD3rRaNFVxbSbzI3BXPHoDmqt8fMZCJGaFAEAcYI+p70scjiML5gTJzkDA4qWBqafrtxpVqYdwZc5IBIOfauh0jVJNTv2F1N5ChRjMnc9CM1yLypJAY5URSTkMoyT7GgTg7XkicnGBjofSlcD0rWtVew0ybyl3zFMB+MYz1xXO3mqzWWm29vCcTbQHcjkA+g/CucS9je/iaIEgDYsb5wGxgH9a3NLtb27a4kmiDQrmKQE5cHuAPwq7gWtPeS412CO7kKxpuMZY8luOPb2+ldmHA61wHh4C7tCZF+eKYHexIyDnNdXbSOqgvIW9M1SQjXDinZqlHIHzg8g1bQ/KKLAOoJGKCRimE0DEwDTGGDkU8nFNJzQgBRxS7yOMClBwKSmIA5U54NTJIGGTgGoDTKQF4Ee1OzVNHKmpt+e9AEwYCkYjFRbqYzkcZoAeWBqvKw5GaXfg5zVaaQBzzVJCY12XGM1XdY2Q7jiopJ1WQ5GaZ9ojf5c4q0iSdYYmHCg0htQMlFANIkoXGDU4kZkJABAPei4yq0EkfzbiRjoO1OinlDhSDz3xU0c7ZIK45qZsnkYB9RSbGBcYweDTDGG5xVeZZELOCSM5qWCcGPJ4+tNIQ8Qg9KmSEAYNUmvCrEA/jThqC+uKLMC/hQKrSMQTioftfGd1IZxJndjBGOKEmA/zs96PM9DUP7vAA6CgugGAeaqwXJxKM4NSCUVSRxjJPPpSmYdqOULl4SJ60vmL2NUPOA7E0vnA96mwXLpkGCKbmqvnEdDS+ccdaLBcsMMjFV5YVPajzT60hkz3oSFcj2YHBNJtJ61Ix4JqPeexqkDAjBoBxRgkZxQFOc9qBEqzEdKlSdgc1Wxg4yDTs4osO5bMwYYNQuDzioCzDvThKehPFFguDJuHPWo1BjJyKl3A9KcMHrTQEW7niragEAGqbLhuKXzCAADSaBFwqhOTSAKOlUGkcnO4mgTuOppWC5ocCkKqwwQKp/aG9aeJ896dguS+XjpShSDmmiX3pfMHrQwuTqx707NV/Np4kB71Nh3JxilzUPmAd6QykUWYyRmNM3E9ab5gPSkLYosxDtueaMYNND+9IZB2osIfvAo3g9TVdnINRliTTSC5b80Doc0vnYqhvYHOad5hPU0+ULlsz5pA4PWqhf0NKJMd6OULlgnmmg880wPnrQTnmiwXHvgrVeRSTnFSgg8GnkDAAFNCKJXBwaAFHTrU0iHqKYEyaq4rDVYAgE1KrAnBAI96a8QA6Yo8sgZBzQBYXafQVKpAqgrkHk1YSUEZqGmO5ZIBHFMYcHHWgPSkg9aSGVy5BxinA5ApHIB4FJvqhE4UnrSsABTEmGOTTXkBGM0rMLkUrYPBqLJYYND8k0ijiqRIqgZ5qcR7kAHFQHrVmIjyxihggEDxjLEY9qcHI4FORTjBJIoaMckVDLHI/PWp1bmqeSpFTAkAVLQJlwP60EgjAqusmTgmnhhQ0VceUpNoFKHzxQTSQJCYpdoIopaLiG7DnpUoTikBp24CgYBQO1OAAFNDgnrTgQaQC5A70ZFRtntTASD1oAmJFIaYDzS5oAXFNPFLmmmgBCeKAaMUGmBV1OITWMgPYZFUtAlLW7RnqDmtOVPMhdM9QawNJkNvqLxZwCSKQHRdaMUmeTS9aYhD0pjHmnk8VGx4oAaRTGHFOLCmk5FMRETRuNBptJoZKshHenlwR1qvnFBbihoLkrEetRM2D1phY00tQkIcW96Tdg9aYTSZosMkL570Bsd6YKUCkIkD09ZOcVCBT160DJgSainvra0/wBfJs/DNSg1natpovIgwJDDPI/CoYzG1HVY7i42wPhfWqDwl2yJJHY1q2/h1ZkLEtkAmr1poojOCGB9c5oGc+ul3EiFyCABk7jWeThiPSu5OmTzfIZtsZ6jH/16zNZ0OOKMeTJlyccjFSBy5y7jaM5pzRFBk1sW+iFVBds/Sqt3plzExY5MfbNUBmE4p6AMMliPpUslu6DLKB9abFBJM2EAyOeTRcBYgA2SMgU7aWfekII9+KeGkBEJ2gHuOtSl0hARxkemcZosBXkid1L4QewNPt5IY48Pgk9iK3ra8tPswWARrnseDmqT6NJJL5kjIg9gDSsBnIyCYSLwQc+1asWtEAIVDH1JFUbq3S3JHytis8sCeAB9KGgNHU/Nu3EoQAAYGKztu3qKXzG9SfrTS5PWlYCOTJ+npTAMnpTyTTQcUwJocbgCcVox20K4d5H29wBWSDzUwMpGPOIHoTSGYCzwswBiVB6iiSSELgHOe9RJbGbOCBj1qzDbCIjOHI7EcGqJGWkUl0zJGw2qMkGpcJbsBnDjoa0RPIkRCWe0dyoqfTr7TE+W6ty0hOSSmTTTAqC5huoRG9wscnqTwKqXelyWyNNJPGynkFTkEV1lqugXORNCc5OEMe3FVdV1fS1spLGzsRgjGSpwKLjRxsYiLgyEgetOOzOEbI+lWleOHGUB9jUU8yucrGqEelMCNkQRkh+ewPWn2iFpQGkKL3YdRWlohthDNJPgsOxFZknBJ9SaAZNe2ZhlRjNvibqcc1WMJJ/dj5akGZDljkkgVLNbz2uzJA39Mc0rgVHiMbAHvRjAqX5pJwCpf2HU1oS2kkU9uEtvnYfcA6/SmBkck04jaK0tWtjbyqXgeHsdy4BrNbnpQAzjGTnP6U0E07nGKbigBTzSAYqWKMMCS4BHY0ADOcA4NAEZUjGRikAJ6CpyQ7jPamMeeOlIZHtzTxExGSQBSoQM5FTRoXjOADjoOlICOFcSBgAcHOCMg1dkuOhhtkjPQkc5FQHZDGR0fsKhMznvTEWY3chxnBAyDS2xWe3kVz++zkZHJqr5pxwetCMVbINADSMOQeo61bUSG3RIxkg5I74qBgznIGc1p2DGGEgjMhBxSYEAuprS6BiOFIBIxxn6VYmkgu2V4co5wHAGAR61IkgudOk81QXU5Oe1U3hcTxi3BO8ZAFSBf1QWsHlCC3VGdAWYHr+FZhQsnmKPlHfNWLiZjEPPU+dHxsPXHtUJLCMEEqpOdoPP0pgLGcymMj5SDz61FaTSRXYQMSC2057j/JpkkzCQMDgjpUOcuT6nNFgOu1G7je0KbsTQYKN3FYl/PFdok8YYTAgMOmTWd0PU1NC4LiNxlT1HSiwEgdWjyAQ47EcGrssUnkw3Vo3lKww5UdD6VnSReXIXDkp2wa09KZWsrqFzmMrkD0NFgK9tLHDJIRIznqGJ5JrQ8+K8tdskUkjno2cAD2NZCRFQCR1PH0q9HKI7aaPBIcADHbr/AI1LQBbYeQmMkNGMqCcZx2zW7a6958sKpEYreZsEtglT6g1mw2D3mmWUUe1JCXIOMkimNbSp4fJCE+VISxHQDNGwHf2txDep5LkeYnB55z2IpL3UDYjbHazXE3YIOPxJrgfMElpDNJuWPdtLZII/Ktmz1S6hvIIWu2mt5hgMxzjPbPetEwLV1PrGoYhuYFWEHPloQSD7ntWRNbmESObp45i2I4kmJOfQ+ldRd21xYRCW3kLOSAS4yMk4ziuR1iK5vNTuAsMb+QAzSW8eDnsCfwouDRBLrN3bO8FzNHcFMAAjJX2B7U2bUb59Ljt1hSOzJ78kn1NZz2sgkHmo6NIMjKnJP07VeuLO7iUSXJPzLhAOB9KAHWF1bx6kkkrxxRoMbUJIc+hNbsaJqF1FC0aR2cKGV1iJwTnGM96xdLitGgNrcQAkksWPGBUkGrT2iSQQxq0YJUE8gipbA6GSXSTkyEOW+UbQQFHbA7VBfaxdWV5BNKzTm24EfQEEEZz7j+VZaTpdxSebbRoVAw0ZOc84psrS3E9u88O8RAAgH7wHtS5rDbE1bX7nVmEJtxbRuc8Dk8dzWY7qvlo6+Zbx+hxz7etXdYntL27RbSF7c/xFzgH8Kz7b7IhInhkmmyeM4WqTuFxkt4JLkzwoEz0A6ClFyZoBHNyQchu9Ty3WXYi3WNWOdoHFU5HVpSQgAHp3oAuExXW7zd6YHG05yff3qjJYTRMABvyM8dqnS7mXCqFHOScVaNzDM4Eqi3uFHyOvKn60rgR6d9lt9Tt5YnlIGVmJHQkdaffW9hbwyMb43FwW5jAx9SKzxLcCTch2EkniovuO5lGS3Ofc00BfvY1hsoI0ChThxk5OfaqU0Ukaxl4THkcA9SKJJWeNELErGMKD2pjzSykCSRnA6ZOcUwGYyKerYKkDjjB7Gr2j6Y+pXoQIzJGwMoHBxn1966XXfDttDp0F1aW8nmGQRvGATuHqR2PFAHM6YxF+zDaZFQlQ3APrz2p0lzHcXDyTIoxnhRgHFLMFshJIkDp5wwm8ZOO4/lV9dMgmjhJItkKgAyAlnPU4HWkwI5bW3t7O1mIIkk5II+UjGetGlz28N6iRIDI+csB044x7ZqneyWvmR28LEeUAN+SQT6YPSpNPn8iY5mCFwQZAM59qLCAIdOuhdPGGIYlASRg54z+dO+33uoSiWV2JUkBkGNo98Va1KAyWkMKhFQZPmueSeKz4RLbHy8jB6470CHSCa4k3vO+UBIMhyfoKmtHufOeRT5igZcscZ9zTnNrGrbXdDJgZJyBUVtcpbeYDlldcDA6ms2xl+yUyzRL56g7skucAfQ1kahbyW2q3ETsrEtuypyMHn+tSPiSLBBikH9w4B+tRiIDguSe5PJNNAKPkCgEAg4IJ5yau3AFvMsIHX/XEdCT6emKqlFBRg0aEEYyeTV6SUXVwkEIDzSkAkjAzQAXEjW7hSA8iDr0GD0xVd5vMjA8lcj9KkuUQQLMkoJQlJIicnI9KrRyEybwCBgjB6GgocWCMWLYJ5G08AelaE7YFuudvAOB0B9apBlxtkB6EEYwee4q/cxzSWMdx5wdET5QcZxSaEW7Sa7l1K3t4EtmV3BJWMZGDya0tU1O50TWZI7UbfMbzG4yrKQMHB6Hg1jwSMHs4XuFiO3d5qjlT7+1WtYaS5lLzzW7yP0aLkYAxx6UJjLMWpiOLyjCkSsMB1BCn07VtwTSRJDFI6kkdVORj1rnZJZNTtrexZBGwIBnzgEAeuBirkV1NZ3xIliJhxGTs3AjHfFaqQmjs4sKcgYq2j8VRt5Lgwg3AiAwCGjzgg/WpUlBYAGmItls02hFLDIpSpHNSMaTzSZpxHNMxTuAu4Uu4VGTTc0ASlhSA571GTQCQKBEuRS78VFuIpDICcd6dgJhICcZpWYYquXAPWlEgPBNFmAOSQaz5/M3EhCSPWr5I9aaxU981a0EzDnS5PzshC5xTYlJIJGSewrcIQjBAI9DSLHGmMRjirUibFGK1c4Y8e1WlQouCKkZ8dABUMxkkXCMFqXqMduTdtJ5qRXGMZqMxKqrtHIHJph4HFFgJpRvQgHrWbIkqNtHOO4FTpMROFY/KetWJNp6GhAZTgqu7vUOCeas3CnkAcCoAG9CPqK1RDJEkwME04yHtUQRj2oGRVCFLtnqaNx9Tmmk880mRjigV2PErA8mpA+TUAIzTgwBosFy2uGGCcZpSoB4OariQcDAqVXz1NRYpDjkDNKJAQBjBp3GKaAByRTGPBG3rRzSZFKCBU2AevTmggDsKaGz0pc8cmlYBcik4pMjODSigYm0dqTFO3CkLjNAhpFGKUEMKXApgNGQKeCRRkd6PpSGLkHgjimGMjkdKXJpwB70ARhTnBFIYhjgVPgUhFO4rFRkIpORVk1GVGelO4iPcR3pwk9+aQrxUbDBpgSmQ+tKJCO9Q84zSc9qLBct+b70ed6mqu40bjRYLlsTAdKUyZFVgSaU5FLlQXJhJ70hkIOajBOaCSTRZBclLq1IeKiyQaeHBGDRYAIGMj8qjJqQ9cio2zTAM0A0gUt0GacFNAChjmnKTjFN8tvSnAY60Ah2CeRUi5I560zBHSlUkGpsMeRxTSoBzjmpeCKYw4pDIJGPSjJCYFOIBPNGKaYisQeuOKASDVgx8cUxk2jOKq4h8TEDkk1I7AjrUC5p9JoaYxm4NQnOamYHPAqNlIoSExA54xTtxI5NMxindqoQuQTTzHkDBxUajBBqYMNwzSuA1oyoyafFJtPWpJQGTgjNVSpHcGgZcEwJxT94PeqUZw3T86lJwM1LQ0ywcdaM+lVlc568VMCMUrAPBqRW5qEE5608Umh3Jgwo34qLnHWkJNTYdybf70ok96r5Ipd1OwXJ9+O9KH96iBJFFKwXJ93vTg5FVtxFPEhosK5OXzSZqHcaUEmlYZKGHrShhUQNOBosBJmkzTM0ZosA/NNNAOaDRYBoODXN3QNrrIYHOTnp610ePmFYniBNjxzA85wPbpQBtggqGHQijd71TsZvNskOegqbdTEOMgzTS2RTD1oBBNOwC0wtk4zTycLx1qBjg0AOPWkpwpDSAaaaSacaQigGRkmkIqTb60u30pWDchwaADU20CjaKLjIwKcOlKRSGkAtAPFJmloAeDRnIwRkelNpQKQEitgYAAHtTgxqMCnA0WAJZJlGYxk96yrnUkGEdg8h6EdB9KvXcUsybYsZPXJxWVJpsiSB5FiQepNSBQuJbpSWjmCr1wKqJczzMBK5Ye9P1KaNZdkRzjqR0rO8yT+GgaZfuMsMEIB6g1XjmSAjHPrVYs/U9e9NIycmkO5fjnjdxgKPwqK7G+VQnOetV4uJBWp/Z7FN6zx/TNMCxp7GGPJhhGP4u9dNaG3EPm4DMBnmuIktHKFmmAx6HrUtrepZR8NI/uOlAEeuu82pSFAQDjk96zxDIB0q3cTm6lLgHJ6CpYG2KQ6An0PapAzxnvQatzGM5yoB9BTILSa4JEcZIHc8UAVT1phI7VPcWstvJtkABxng1Ftx1oAQGnrITwx4HahQCcE4pwUZpjZlCFRypwexqzGuwR7+CTjNUmkAUkPhx2xVnzvMhRypBAxz3qRGg95La4KEFewI61mSX0xuCSsYBOeBzTZJzIuCagwAwNCBo05tUMseCu2TAG8HH6YrOlulEZQgknqwPWhsBd1QsA3NUmACYlApT8T1ojClsycCo8gGnucAYqgL1oEEc2zuMmqUjZkNNimMUgcc47djRc3HnSFvLUe2M0wHHhMjqCCKv38sc1rBMJF8zuoPFRNaG5eC3WExzHuTxW7c2GnwaesE0OZgMggYOfrSAxNFW5bVY5LcRtIpB2scZHQ11/iKPymt7hBi6yPLHbNcxbW0MShwkgmHRgeRWnaSG4uhJevK8cYBQk9COlK4E2uHz7KGO7ZhcAg+VjBNcjdQywy4Ee0E4GTmup8QskzQ3IXzI8FeuOwrl3K/3ifrTTAW7sprRFdyCD6DpUKo7LuCNt9cVtxW8N34deeR3LjJBJJ5BrMgnmhtyigEE557UXArbSBk1PbJ5rgHGO9QZcsfkOfQDNTrMRAVAAPqKACYRq58sfUjpUW3PQVdt4o2tgGXJPfNOkszGuVXNIZnsvGAOalWQRRjHEnrTWyTnvUgi+1ISCAV4wOpoAqlixyTk0o+7n3pRDKSQEPB7nFOSEhCXIHPTPWgCMcmnLkkAdPWp1CiNwygEcg1CqM5wOPegCym5CFIPPcDIqciRJRMq5A6jPOKmtJPMgMTDJQZHPWriwxXIEasAzfeB449qm4FSNw058pW8mRcEkdD61JZRJG5e6dk2Z2FTg4pxRrcGHcQB0IrIvFkFwcMzAjOc4pXA6MXsFxiOO3DxnrJKOaw7pj9oIIUDttORVmK7MOmrDI4Oc5AqBNsgLEHGeKYiuUyw4BxzzUTAmYkKAP0q5JGhh8yLIJ4IPtVaEMRz3HNNDEIKkhwRnp70sbNvUJ97PA9atxjKZkIIGcZGTUNvIFu/NIHHYDANAFqdYZFZGARsZBH60lnMLSBo3jJLDAIPSq8kxlkLHjJzj0q3bhDCJZBuIPBHc0CGuUhj8l1BmJ4JBGKlkjlSyYghwhG5lGQM1ZikEmp/apoV4IDBjkAetXdNu7az1SaOOVRbnqhGQT7CkgLWjmPS9A+1ys7NKSEH93J6j0qewmR9L1S1jAkygZc9Dnr+VZr3st7ZXJlnXy0kysZAGBnjBq34b1u0trYWlyqpgkiQjI57E/1p2EGsWkdr4RgQJtJIJ56HqaxbCKWayjQOseHOCTgg1t+IbyC+eNYLiKSNM/Ju4z9e9Y74hlWGYAqg3fKO9JsaOguPEN1dIliyLGchZJA+S4Hp6U7QIrea6uDCSFUjJLfewe/sa5QW807SPGreWpJx0J/GpBZXVjcW8kxMUbkfMjc4/CkmB1utlIrma4nhdIyQoIPJAGM/Sud1HVLW4ePDygrxnGRj1FGuzn7QkYkkZSMfvDkVmSyBoBGw6HII6imwL1vMbWR42IkjIz5hPIzUUkFxNbtPb2bi3Q4MpGAazy0kskaA4yQuT059a3kW6J+wXl87wqBsQdCaLAZsUbIuZOEPUetS/aJH2yLIR5ZwABzj0J701bS4mvJo1dXEIOWzhQKpNJNgqjgoTgAVLA2LaWw+z3c19GWkdvkIGcVmC2IkEpdADgjJ6n0qANIXMUj7ckBlJwfarOnuLSbZKpeEE5J5xTuCIJ2lMx3oB6gVE4AkBHCsMgVog241HewLRuTvD8ED1FVJkj+0FVcOATgjrTTGEcai38wjocE1FKU2A4DEdcjirsUASMmaRQndepNVpYWyz5UowwvPUdqGAyHaQGNSgowMXlfPnIG3Oaa0JjEKgfORyuanhnEDknEgKkYxzn2oAqzQxnkDAPUDpmolhVG/eduoqyY3lUAAoT03cZFUiCQM9e9UmBtaRqF1ZTSG2mSKNiC+8ZBA/X8q0dU1qZpYX82VsHIYEhCO4xWPY2f25LgEmNo4N6BAMuR2PtULMn2QLMSGQYUg45NFwJr64hu2tl84INxLFjnaCR1rahsPsghvC7XFwZR5MyOSpB6ZB7da52yu4rO6E0sfmgKRs9f0rXTxCgu7cCGMWtu+7ahILZz29jRcChrSyQ6hNbzRoGDGTeBjIPPFUzj7PHIOuTirmrT2l7JHJFI7yFzvLoVOCc/pVGQRfaPLhOQSAPrQKwokaSYGR2Jznkk1ekjEgDxkbmOASaozQPBIN+Aw6YOaJJTLHEpAGzoRwakqxdkij+zhm2lgQCCcEGni3ijsjPKSzBtpRemazXcsrAkkkg578VpvcW95EI08yOaXDOCflBAqWhWHyTtNAHjij2gAEkfMMdiaWa4S9+aWMRgAAEDBB9c+lUJImTLKSOcYzwalSZQBkHgE4AyD9aAsTGCGBT9rtzMJDmNkYggD2H1pNMkNlqEM0bY2EMQwyCPTFP0+dzdBzMnCFIzLwBkfSpotHkmiuvLkjd4mGJMHGepGe1CGVZ7rzr5pGWNoySQAMAZ9AO1VskSHI68gjoRWj/ZNzZp9rn8lVTHyo4JIOMnA+tV5Iw9wY1Jj5GBIccGqAbESbtSpzkAAsOBTWLglASQCeAcfpTH3xs0JJAU4IHTNPUlJFEZ6HIb3oaETQZEW/DEHIBIPXuK1NKMV3ex27R3DuQSqQ43FvUZq9pUy3djPZXVolwgJmAJKiNsYPIrtdA0e0hgtrh7WMXcUYBcAhsEHoPxqbDOPu11e1kgsltS4nhZUjyCTznJ9xnpWVDcnTrkeYCJlBEkcqHBB7EGvXpYI5GQuuTGwZSeoIrD1zQ4bvVbDUGQHy3CSgDqCcA++Kq1hGbosySulqtxKBICfIKEYHsSOldDHpKRrgFyfXjNT28BiWISMXeHIDHBJHbJqw8hzVpu1mFiJECLgEn3NKRmlBzTgM0gIWXioiKtsoIqu4ANMCJhTAKmK5qPIzimIMUhFLmgnNADT0qCTI5FWMUwgA5NUgKQkmJOMkCnCbAIKkHvmrbKqKSBisu5l2zMM4q1qSSm4zwcg+9QMzDoTUJJHzhs+opDISM1fKiblqGZgwBJI96smYnkcisozErjGPehJ3jPByPQ0ciFzdzSeQNHkHJ9KjDBSCSRjtVUTk9OKdKS6AjjA5pco7otNckgqOM96gM5zgniqIkYd6PMb1quVC5kXxtc5NPAIxk5qjHMQeTU4lyOtHKCdywTkZ70gaMDlB9ajEnGDTwQQR2NKwxkzAghDj6VUJwcVcMOelRm1Ocg00yWisTk0mKkkhZecZ+lMA55qhCGm1KVHamFTQA3dinLJg80hXim4p2Fcsic+tKJ/7xqqKcKVh3LfmZGQaPMOMVWBIPBpwfsaLDuTiQ+tPDZGc9Kr7gRjNLkilYdywGBI5pfMAOM1W3EdDSbjnNFguWiwNNyDUSuWOCcVKkTOQAeD3qWFxMkU4GmkYYjOcHFJzTAk3HsaBKV9PxpoYU4ANQO44XBJxtFKJiTjAphj54p6xgHJqQHhqeFDCjYMA09MCpHciK4PSm7RmrLBT1oCqT0ouFir5dHk56irYRT2p2wAcCjmCxRMHGMVA0bKcYrSZDnpTDGD1FUpA0Zu0+lKFJOMVe8hfSlEKg5A5o5ibFaOMmTFSmIlgAM81IqbXJ96sJgjOKXMOxnSKEcr6VHn0rUkhSQcjmqjWgU4BNNO4NFU03JBqV1IODTMZqriHBwRjNKMHrURyOR1pVY55p2ESnIHB4pMmgOMYBzSgjFIY5WJqTBxmoselSK4AwaTAf1phU1IroakGw+9SURqSDzTiMilZPakAIpARGNhnIptW9oPUU0wg9BRcLFbNBG4YqV4cDIFNC07iItuDS+1SEGmYIPSncVgAzxTSvJGKmAyKXHNHMOxUKY7U3birboCOBUJUAdKd0JojGfSkJINPYnAGPaojnPNMRIHJGDShgOSAfrUYNLuosAFsdKXeSOcUh5FNNFgHg4NSB6gzShsGhoLlkPz1qRZM8VVDcUofBpWHcuhuKKrrJTw/vUtWHckIpKAwxTc+lFh3JQ3FLuFQhsHmgvnpRYCbI9aXIFQgmnBsjk0rASblp24Doark4NOByKAJd4z1pd1RGgVNgJg9LuzURNAPNOwE4alzkVEKeKVh3A9Kz9ZjMlgcDOCDWgeRTLiMSWzqe4pWAytFk3QmIjkDmtEjFYeky+VeFScZOK32FUBC3WmgkGnNjNM/ipiJAcimMMHinDrTiARSAjBppPNPIPpTG4607A2MZsVH5hzzSOeTTCeapJWJuycEHmpFOarg1NH2qWhpjyKTFKelNzUlCGkxQRRg9qVgExQKdzQAaVgDNKDSYNKBRYQoNKTSAUuBSGIDzWRrdtJdACMZIOf5VsYFJt5zik0ByI8P6g3IiGPUmqVzayWsxjkGDXdsJMfIM1mT6Qt1cedcA49jSsNHJiJjycAepNRlcHiunvNBt1QujdOozXOTqkMhTpjtRYZCysMcj8Kkt93mYB59PWoiwJ4pQSOhIPtTA6WYwi2VJAAxAwBSppEc0AJGKyYbKUxiVJgxHY9KlOpTmPYXII9DQBafSQnKLz60C0t1B3uA3esptRuVYgP/Oq099cSAl3BB7AUmhF9bLF2DuDgnAxW+baGKHdG+CBng5rmtOElwWDMcDHA49a24bVSnLtxRYEzClEsl25ckknqagnUK2Ac1p3cMpyfLYAdTjgVnGEueCM+9DQyAYHI60odj0H5CleKSLlxgdjTAwHQ0rAYbEk5pwkZV284qOlBwegNIZOApTcM8UxvYUjSERnFWLRBcQuCMSDpTQFUsduB0qEkgcVYcbHKngg1WbqaEIcF3JnIFG4kY9KRinAUnPfNIKsCRMBsupYegp6fZ3uh8hCHsTjBqAkg4pM0AW3mdJVCuQQcKc8gVejhuJGEk08mfXNY2TkEDJBzjPWt2eUrbIQcZFZsZZtlkEwLsDH78UX9ujZfc4BIGAcD8qzbe5lEoO78O9bQs7y4T54SVPc8GovYGjCvrV7RkHmOUcE8n/PrVEhicgZX1rorjT1lYL5zsyjADc4FY8ltcgkeXIAOg2VopCsMtml8mWJZdinkj1qNI2ZgAck9KU+XgAg7h1JFWLCGSe8EcZ59D0qrgRDfDJ5ecZ4I9abKYgwEfOOtaQhNhNIZwrKRjBORUE88IBa3QJjsKAK758lSg5Bz1rYtriO4kVCcgpzx0NZkavNbtLkZGcjFPsEaWTzI3AkTnB70gIpYvKunAGADwKkMahw6Egj0p8wS5uWBfbjvmo4S0c7Rq+9B70JjK9xcvK+yP5G7gd6ZGzRo8boGycgntT5owl0XT8DUbEls0ATgB48HvTAfLOwjinocY46dakukDSBoxkMM5oAl0sxGScyEAAcAmp9PkeYSTZAMbEAAdqyYULTBT/ECMepqaO4l04SwlPv+vahoDZe5SWEuQAynBA6VitdyAkcH3Ip8tw8BGwAq45BHBNVkGUGetJIBwPmOHeVMDsBj9KvWvViCNo4GSBVaGMPAxMigg4HOaZMjInzDAJwKGgNCKMGcI5UDBI561Hc2ud2AVqzcafDZadbXEdwv2o4zEOQQelRknzWZmJJxnNS0BBbRru2yHgdRTLpYI5SIR8tXI2R3Knt3qlMok3Z6immAJET5bYwGGRV6FxAmzBJJ4A6ZNUmLNHGu8gAYIFSISZI2JzsIP5U7ga09j9ki2XG9JmUHnuKn8OaXMmrQ3s8KfZ2BAZjnk9MVdvZYvENpNNGpijtUyrHglsdD7cVkvHcW+n21xlljuCAUzxmlYDQ1HSreyM1xcsA0kn7pRkgL9BWOl9bwWN1biPM0nAyCMj1rrtJUwafcSzW/m3QcArjJIxwOa5u/s55Lu5vLyEQttyi54HpTYmjMtrp7FSphjkD4JB7VoWt7H9tkuHjfyyOGAyM1Vso4JppnlhDrswATg59jXT6Kk1/I6WxiaOJQDHIcDntkUkiWzPtbhLxXeSVE7BDwTUccdtd77eeXEiKTHIzYAP0qpq2n3GjagzSKAGYlIyQwAPuPrWhOYYnhK2CTFwHyDgYpMoxriQwRJbm6jnB6kHO059apOzBsHtVzUJFvnaZIBBsIBjFVEUzszM2FUZJxnFNAOQgoCDlice1XbJWku9kwyVBI9iMGs6OZBJGHH7sHOfXFXzcrLd5jO0E4Ax1HegCWWa7u9M8oSnapx5YAB47Z71Sht5opEBXMY5I9Park823VtilY+Bk+p9apzXDLMwVs8mkgLd20YCyPGG2nIIHBHofSq0rqSDEcgjNEV/JAS6EEgcgjgj0qWQxXPmXFq6CRhhoTwc+opWAobneUhnJBPJHankOrK8eAVPBApEjeaYCBDIcAsB2J9aJPkwHOSDjHT9aqwF6aRJFB2YwCOvrVaAWyJmcSvg42ocYFQl3c4XhVIJqQyFvlIAHYAUASsxkl8y1VyV7E5Kii1BMbzlv3kbZ2sMA1TjuGt5mKyFARgkVfjmhZsEZB+8OmT6/WiwzVlVNTS3ldvLhYFVXGMPjH41lXWmvDO0FwixTbS0bHksKbCZLS/hBLsgkDBH6H2zVzW3e+mS7ZyeuUOMIM8AGqQFHTQlvc/wCkTSwh1wjpzgnrn26U/UrQhbdYisgckB1PDEGqTDBRgSy5ywJ5xUs1zG86GONvs6NkRM39RTbAEgCgmVCGOQueBx1qqw2sQeo610I1q0tLGYWdtLHcSfxHBH4Z7VhyNJdzh5pEQnAGQAAKVwESZ/OEhOcDA4q3DDElvlwPMYk5PO0euKju7aaNg5jUx4xvjOVqS+zHLD5MhMZjBAOOM0AV5pBMdxbJ7H1qA8U8nkjjPtUectimMcCDnIB471PGSh3DOSMZA6VCBgkehqzDGXGBwdp4NJoknJ82EtIjSKFwCpxg+p9adbwvLYnELSFmwCOuff2qshuNrpCC5IztA5xTUnKAKHaMg5BBwMj1qbDLklreW7C3e3KsThBjIJ9Aa07CexFjbpMWWWIkTQjIEgz3OcHpWTJNOvk3MdwxuEzuLnOCDjiuo0LUrF4HhuLSOOWc73DLuD9srnkZ9KdgMXWru0vDAlrbJA8YIcx45HUcD+dPtEs9Qsri4vGb7TDESCqZHHQmp9ZS5SeeG008QxwnzA5GCUIx36jOT7Vznzg8O6gjBAyAfrTETJvmwMEyHgADqa0NStIorG2YDbMsYSWIcHeCckj6YqjBcGFWAAIJBz3BByK6DURHBNPc3Awbm3Ei+oLDkfrUsY6ysJL/AE+91CG4MdvAV8yPaQSCOorV0/VNXNz9jj3JfuCBNcOclRyAB0zjmszRNS8mzu7EnFveQklm5KOBwPocYrZubcPoWnajcyO7vOD5iEgpGQfTpgikhm3ba3cW99DYancI8kxCiRISOT2z68iujZQy4YZGQfxByK8yuJr0Lb3n2Npo7WYOJwSSxByCQOma6mz8SQzW5upIvLkYZMROQx9R6VQjoW61G2azrXXYri9W1uIJLaSQboS/IkHsfX2rSOKaAQE5qQMMVHjmimBIXAHJqu5yTjpSknNIOtIBFPY1A0b+YTjFWguaULk800BSJZTg04NnrViaPIJxVN43UFh0FUImBzSlQRjFQRSYIzVkEEDFMCJk/dkVh3UJDkhgfpXQHmsa5QJ5hAwMnNXT3IZSUnFCMFBBAP1pMkGkIrYzFYqTwAKUIpI3OB7DqaioNMESFMHqCPUVMXVowoPzYqoDinAntRYVxGBDEHtTc0rZJyaSgBRSgn1pM0Zp2EiUSkd6lEvqarZweaUNU2KuXVm98U8TH1NUAxHSpFcnvSsO5fEingjrTXgSTnofUVVDH1qUTYGCaVh3EeEr71G0fGQKtLIGHJpSgPQgii4WKBphFXJITgkCq5Qgc1SZDRADTqCtJimAoPNBOaKKBijOaeDTAaUGgLj80pYDrTKsWkKXE4jc4GDnikwQsdldyRiRI/kPIOe1Tw74QBICCK2oYFWFFA4AwKhubJn+ZR0rBzNEjHcEsSB1pvfBrRCCJcSIQPU1XkgQklRjNUp3BogCqe9LtPakClHKkg89RVqQKIwVABx2qnImxCN1PLEdqF84AEREj6UNJk5KFfwwKm5VhwbI6EfWpByaBA4jLkYGMimKwxnBFK4E4GacFAqIN7EVYMEoAO04I60hiAUoIHWmPHIgBJyCajLEUgLOF9aiZfQVF5jetPWUd6YDDnPSlDDGD1qQmM9Dz6UgQE5oFYjLYNOEyjrSPGcHAqEqR1p3C5aEo9aCVNVxzxingUgB41YVVeMg4xV3FIQh681SYWM4oT2pjIR2rScRgcAZqAqD1qkyWimAc4qfy2AAJB+lPMQ6gUBQvamFgCGkKE1IDSnFS2BGEI61LHwaQU8CkyiymCOaXYDUKkiniQipGSBAKAMUquCcGpNgNSBEUyKaIge3NWQoApcUXGVfII7VE0OCcCr+KaUB7UXAoBKd5ZParIiGakEYHai4ikYjjpUZhO7NaRC+lRlB6U7hYzng4ziqcgw5FbLoMYxVSSAEnIq0yWjPAJ6UEVM8ew4qM1SZLQgoIxSE4p1UAw0lOINJigABNOBpRHxkkCnBF7HNACrnIqTkVGGAPFOLg0mMUsaercVFuHrQHHYipAkJpM03dmngcU2wF96MmkzzingZFIENGTT8ECmHjpTgxxUgFPXJpAMjNSRkA4oGG0+lIcg1Z+XFRMoJzSuMjDc81IGGKjYYGRTQ+DzRYLlpWFSAgnHrVMPU8cg4JpWC5zFyptNVBAwM5xXRr+8jV8cEZrF8RKBcRTAYzWjp0/naevPIGKYD3HWoialc5phFUhMarc81MpyKhIpyHFFhXJjUTjOaeWAppORQMqyAjNQnNW3ANQMoB4FUmSxqnFToc4wagA55qZAo6UNDRICSOaOlNMigUwy7qjlHcmyPWjKjnIquCPWgEHqafKK5ZDj1pwwarAgdKUSlaHELsskCk4B5IFQGcmm5ZzS5QuywXQdxQHX1quEOelTJEcZORRyoepKppwHtSIhzg1MqDAqWhojCk9BQ0eRyKtrECOBUogHcZqGyjnLuwilyTPIGP8I6CoF0ezjUSTMre7cmugvLKJ1JUYYdKyxF5L4uGAUetK4HMajb2Db3tyRg4IA4FZkEJlY4Gcda6DU7f7bOYbKMADkntk+v5VivYXFnLhygP160DNWU262BCEIQORnk1gHJYkHgmp5hLtG8ZHtUQPGMUCGgDOTVuO3V1G2RMnsaqY3MB61dt9Py2fPA+lICeD/QwTuBY9hU0eriHIYDJ7Z6fpVOVVgm4k34HPNWJrqykgOLdQ56Y5xSAjudVNwpQEBT2AqrHJChDFcsO+ao85qeIyKcrEZD6AU7jJricXAwqkH0NUniePG4Yz0q0izSzj915Z756VoR2sMrBZZEDdh1oA4kEFc03OaQMNmcimmQkYAAoGSnlcUQytDOHU9O3rUYJI5NIfWkIuXojfE0Y6jn61RJzSh2AKg8HtSU0AvBGaTtQBzQVOMgZppjGmniJim4EY+tRHPcEfUU4McYycUwLFpaPdlhHIocdAep+lWLmaQGOCRCjR8HPeqtpL5F3HNnGw5q1d3P9oX4kGFAxilYDR09o7JTcSFXJGAO1N/tS9ln3GQ+wHAApPtkSNsW3jOO/Snvd20K7owfO9h0rJoY61uZRKfnJO7v2qaa/v7eYlWVkHUMMgVRgbbCZWIDE5GTSXc7SWpIaiwiBzbySmSeRiSckKOBU1tdxxTBLe3YqfvFR8xFUhDN5fmEZX1zUsTgQjCnzh0YdRVoRv29na311vuVMMaDkSHGazL9bVL8iGQPB3CDPFQF7iSMiSGQk/xMTVvT4RbwvcSRBiCAgJ4zVASPc2UWVhtpEZgCDsIBqK0PlytI+wFupB4A+lT6le3RSGSVoz0/dgdKz44lmJJXJPQUmwIry3Ec5mjYNGe4qsCRISOM1tCBGtZIQMBQCKxW3GQKOTSuMdIWBw3TtT4FAJkPRRk0yVhK4L8EDGBV23iX+zpgR1zSApkkMSDgE5Aq7AyNZiPOD2OOlV7dPMBVhjAzn1oRWhmOEZl4yM0XAUxvBIshGSpyD2NQTCW7mBYAk9gOTVx5HdCFQkDkgdQKrpHKXWWFsEdCx5H4U7gQuzr+7lJBXoCOlNUgDBUkHsK0YrJru6JvGcyFdwAGAfxrOzgOoBABwARz+NUmBJ5ixoMAFvQ9qbI5mAXGAO2ai3A0+MkHINDAcUKFcgk44yatRyHBLelQE5IJ5wOKYS2eSce1JoCyweOYBOSRmnpGkoc5IkXsO9MiO4A+lOwFO7v61FgH6cA9+EYZGDUbEpPIoPAOM1cso0afMcm1wM9OKpvxPJjnLYB7E00wND7ZEdNEIjAkjORIDyc9iBTrq+Ikt4r1yLdYw0aIcjP9KqR2A89UklCs+MjP3frV/ULHylUTSJIsaY3BcED86YibTMvrMUjS3EcLkKzhiGAPAB9q2/Fel2kFiI4Vd7hvm3uxJAH9PasvSubAyTEkPyST1xTb3W769tvspIjLDAI5Yr6E0noBkBIZLfYkm0nk84yabpKyC9KLLNGwyQ8bcgj/APXUBDRSDPbp7VJZyukkjqcYPJxUgb8M13NIzXkhm42gScgj29KybuVix/fOGViiqOABnoKsWUxZzNncd45PQj6Vsw2GnT3f2Z7uBpJF8wbiCAfQUbjONkkKB4x0JySetRxLlgpJAJAOD1FdGfDAmnuIUmV5FPDxn5QPSqOm6RfzXDm3hyYchjJwM+n1q0rDaM+aIwuUI6UrOzoHLbMDAI7+tTXLmSQxyw+VcKSJOetU5BtOO1DFYntXjMge6ZmA6cZrYeztLy2jkgGJIiC46sR9KwgQFwK0YQ5hM0b+WQBg9yfSkhDNYRI72N41URyKDgHv7jtVOa3kt2jleBghPyk8BvpXUeHLW3uby3vZp4RJGSskUnBPGAR61o+KtNnnu7W4YobeNSCCpIA65PrV8oHD288kc7tHGRvGGAOcDuaZcGMzkhiUzkHHf0qe4VlvWNvFkBiRtHysPf0/+tUE5Z5DKI9iscgZz+tJgSCf5nXysFhgk96etrcFxthctyVwOCB1qKOQrIJggYJ1B6YrXMRvY7P7MTC5PlkgkhSemfrQBiSTb2BWNV4wR1JPrToiQC4QnHU46VtahpstqsdvcW6LcJkxvHjDqOpPNUziWUCOQrGVJPmDAB9B7UDEkE1xbJPj/VtkHPII9quXrwv9nZSGRxudQejCqkBQ2Eqhvmzkcc49amt/Ilt1SSDIU7Qc4zUgRPaFhLKqhRsyMDqKrxxiHJlUPC3BIOCPpWzfqsKGOFflKYyTgAH1NZRkEEaQmCFwOS2ck/jQAlxbva7hG26MEYJGQCe1RNFIgBmXBIyPcVZuLx7uGOAsFK9gOp9ajF8fs4haPMgJG7rgfSmIS2lMME0TRsY3GeOgNRoZbm7VS7AAZJBwcCkkk3LhSR9KWByhWfpGcxsfQnvTGVndTMduRz36/jSdMN3zmpbqYzXAzswg2AqoGQO5qIYJIJxwTQAueSfWp43O3KOVccgmqoOaeoJOKYFmK4kiZyhxvGCR1H0Parmmzi0llZwhWRMEsuSO+QPX2qgQFCg5B64Ip+8kc0mgHmMXFzcyJMSgJZSBgEn27Vchv7iJrWW0iX7RbtnGCTntke/NQwQlIXYgqpxyD1HPFMVwlwJo2JffjBGBj0NAG3c+JNW1FHtgkEpMZB8qMggdxyT0rHuShtI4fMR5F5OD0B7fXNVSZBMfLDF8k4Bq1dWcytEZpoSzEKFGQQD35Az7mkBMsMscBEMMbsAUmDkYDHoAT+NLPcSSWUbXBaU7RHu6bVHQcVWEstjPPb4VwcLIueSB2H+NaOm2qXVvMEVJkTojEhx3OPX6UMRCiD7IrggMVI+U8EDHX1rqLfUXtNOhsoVeNp0AIZQQCSASCegrmbu2kt7sR24whbYN+AScZP4e9JDdvBcRzG4J8sg888en0qb2Hc7Rbi5OpJpdtcbnkB8x2T5goIIBA/HmrEelv/wlUcasBCAJWjx0JHIH5ZrL06/it/FIuzI4WTaFcnKBTgc46D0/Cunt5BN4lvZB/wAsowpPpn/9VOLEXdT02PVLbyi5jmQ74ZQOUYdD9Kh0y9kuYpIbsCO9g+WVOmQOjD2NXt/NZuq20rvFf2YzeW4xsHSZO6n19qoaZqCjvVe0u4r61S4hOVbIIPVSOoPuKlU80wBxiow2DUrciomXnigLkgcetSKwPeqgyDT0YiiwFo4IqKRRtPFN8zHemtICMUXApyjDnHSiOXBAzSyfepoTuKsRaBzWdcukczAjPqBVwPjrVaeJJGL45PWnDcTMqYDduHQ9qYvSpnUgZIqEghsEV0IxYwjBop+Rjmk288UwG4pAMGpApOPSkKkUXAQDPUUwgg4NTDrTgqknI4NFxWK1SxQmQ5IIGM9OtKI06E5H0q8lyoRQOmMYx2pNjSI41QYUqD9RTrizBAkhTGeozUpERkDgY9sVZE0YixgmpKsZXkAKQT8x6CocEcVruiSR7wMEVSmh6sopphYqAnNODHvVlbZSAQT7042O9iRJj0GKLomzK4f0NSrMVNVirxswIIIPNOBJ60xlwSAjk0FUYVWB9DT1cjqaVgFktweVBNVyuDgjmrYlx0NIQr8kZNNBYp4oxUxjx0pjLgdKdybEdOpcGjBpsAFWLQOtwsi4yp5BPUVCoxU8J2EMBn2qHsUjpoSSgJ4qcnjFZ1tKCBk1fRs1zNGqILiETRlSMioEtIwwLKCQMAmtAgYpm0elK4zHmslDswBHOaS3heWRUAyAcmtZ0U9RTLeJYHJUkgnIB7U+YCYIR94DP0pTGrDBA/KlyTSk8VN2OwCNcYIB/CoJLFJGB54OcetTK3NSbxSuwsQm3jK48sflTNhBIJ47VYLg9KjPNO7CxC8PmKR+tUJLK4DnGCvYg1qgUYFNSsTYwmWSMkMpFR7nJwAxPoBW+9ukilSMg0kVpDCuFXFNSuHKZHlsBknJ9MVLDBLJnkDHrWv5KY5ANAjReQADQ5D5SobVm+4cfWopraSNckZXuQOlaXFG2jmFYxVjLnAU59xUos3OdxI/CtI4B4ApOKOYfKZklqVXIzVY5U4rcAU9Rmse7jZLhgFJBOQB2qkxWICc02hgRzjFNLHFaIhj91Iaj385p4YE80CEIbGQOKQE96lVgeO1KEU9KHoAwMRUiMSeaPL9BSqhBzSbuUTgjbzTSRUgUFcEU4IAOlSwIlJzipwxxjNQSAhuBQGPrUjLQNLwarrIehNPDHsaBkuKQntTQT3ozk0hjs08An2pgxmngikAu360hWpguRxS+WfSlcCq0ZxnFAgBGTmrgj9RRtFFwMq5snwSoz7VTFjcMThMfU4rocZppUYqlNk2MAaZcMwGFHvmkm094OrBvoK3dvPFRmHceQDVe0YrGAxXuOlNyO1aEmnygsSVI64FVjaSDkEGtFJEuLK5J6UKr/3TWjHbKoBYZanmFc5Ao50g5WZZR2YAK2T04rYttLtwgMrMzdxnFEMeZBntVxRtOKhzRXKwW0tEBAhXn15ppsLRxjygPcGnlCTkUpyAAKjmYym+jwj7juPxzUEmnyxrkYb6GtMPk08KpHNPmYGD9mmzkx4H1qRYpAOR+ta8gQGq747U+disUGhcnOKYY2UcirTtjNQMxbqapagQEsD1NODEetNc800GqsTcnEh4G6neZnvVfNKGHelYLk2SRUZPNODjHWk2F246UWGIGpwYinpZk8lgKebTHR8/jRdBYoarF51ixxkqQaraJNhWj9M1qz25e2kQdxn8q5/TpPKvMZxzipYI3HyDTSemKmmGTkVXPFUgY3caaZCO9DHmozzVJEkwkzTgxqsCQacHNHKFycnNRtTC59aTeKLAKabuNKWpuaoAJJooyM0maEhATTgM03vUiEDFOwDgpPQUojPpUyMMZqQEVDY7EQgBGTUiRqvQU7NLUXHYNoznHNOzxTcmlFIoeKeDxUYqRelSwJY3IYVaWUHqapggUu41LQ0WyyHrWfd6fbXMgeSVwQMYBqXcaTIPUUrDMi9Z4x5Nvb5UdWxzUd5plulkHnXMxGSfStwAE9KkMEcy7XGRQM84uJbZDhQ+PfmqKgNMCRlScYrutc0SxktiUUiYHIwetcdPZJakeY7A+lAF+OzsN4McJZuwB5p9+phtyUt3RuxNVrTVYrM7vJLfU1BqGrPfEBSyqDnGaAKJmdQRgZPJyM0zzGPf9KkEE0oLIhYAdaYY3248sgjrxmlcBhIrotCghSLzHulViMEdM/rXNgE0qbA4B4oA0NXTF0Sjlx6hhTLKRkYAAH3IyaBNahQCp+oqSEG4kC2yE/XigDhyeacOabilBxTGPJKjJpC5pGclAD2puaGgJtocDaOe9AjI6jFNjYKDmpxyozSsAiRgjgc0jhk5KnHrUkcZf5Q2D61ZCqsISRw2PSgCkZnYAKMADGahIJOcD8KvGSMfKIgR6mnJGjuFwAT6UXAzsVLECpDAcjpVtYRHcCOWMHPQgU2eIwybSMZ5HuKLgNjeQzhjFvHpU0jeY2fK8v2qKOYxsCOfarO0yDeEK57GpYFd3Cjae9NZSkYLggHoaSUZYZIAHWrlzIWs0GBtHT3oSApor85JKdgTUqKwIKdR0qEOSKs2r7X6Ak9Ce1AF7UrpxZRgN8zAbuO9Z5vZvJEBlIQ9QO9aIuLVoQtxGSe46VXiayhn3xwqwPTPagCowdiTkn6mp7JikpL8ccVcfUlQuIbdVLDBJ559abL9na18xv8Aj4P8PakwLFnJsVyQrFs59qtXsdlFCiSQqNwJBAxg/UVjJMUTAq3BqJdPKnwVHQkZ4qQMw24VyF5q1J/x4FR1JAH1q08kAbZgKh6MBzmmPbxQwEzSgqDngfpVJgQJp10qKoVYyDnJbrVxIvLbLOC/cKeKqnypjtRmA9ScVYxAhCRjp1OealsBJkjRQwXknBOeoNSGwhuI0a1by5MYxng1XmRpMYJIHYnpUEkl3DGBEcDOMUrgXVv5rKYPKvmhRtIxjB+tY8o+13LuPkLcgGrsNzJJG0Vw5ck5yQDRDG0spKQqYBkOehFaxYGRJG8TYcYPb3poJBFadwIXmMEQLxZ4J7H61nlSjEMCCDjmmBKQcEryQM0pAEKuCST1HpToIxLKoI470pcguoACg9BQAsMgSNye4wK0haFLeO4kRjbn+JTg1kn7pPbtWlIjJpQVZH5wQpPFJoCdbUBWkiG9WGAehP8AjVY24a4hAIUDnBP6UkN66WIBPKvwe9a8dhFdRrcFCpABGDnIqAI9EmjOvy291GrCZdozzg9eKnltVkt5oJrgRW4mJ5OSQOnP+NYzgR3AnyRIpyCPSq9wHlYzHkE5b/GrUgaJzesCbW0mZ08zAJ7DPpT7uZjcjJUOMAtHwSDWdAvWUEAdsnGasGMSTiQOCzsCQO1MVizPYn7WiRtliAcE9c1Jbac73TWpBhDJkkYIB9TUiyebJcXAOJIRhPUYqTTr2SyuJDIu+OZCCR1WpaAT7OE0yRYo1DxMN5U8n3FUn0544GmmfypGPyIcZOa17jVbf+z5I4IWIAABbqcGsS4upbw+fIQXByoAwBj0pWEjch1O38P6W1l5Upu3BJIPBJ7g/wBKzJPEWqSxLFHIIY8AMEHLe+TVC4uprhfMnctIRgk+g6VHHLskjfGdpzg9DVooSVXDbmcHPU5yaRU3xuxPKkCrV4++5WZoVTcQdqnIxSXktpJM5tY2CyAEgdARTaGU2J28dauPdeZZwwoCFUZOe5qmSOo5pSWBwDgYzik0IkhkEM4kIBwc4NdBDrd3HaNcySGWMnHkl+AD+vrXNAFjjIH1qzHMUjCOmY+d4Xqwppks3bu8jmmjikuI7dfLJDR/d5HeqZstMt7SaA3sc1wRlJBnan1NU7O0S5jnjhhckjdG2cAAdc1FewvaFIyoCuM8c5/HvQhjTLGkZWIAlgQxPII9q29NuLltNkntbcAQLiSYcZPHbviudPAJ9O1d1Z3ULeH7rT4bUwCO3LnL5yTz6d6GwINYmjPh20uLm3S5uXUqJJCcjJySAK5T7RlVDKDgYGavXd/GsMMORJnDkMOBx0Hp1/Ss6cHzshcIQCOaQydLySE4jCgHqMdRVi3kV5kOGjUccHgH1rPk4mHoAKvwKIoRKM7ZOMnsR3FKQE91ds8Txk4njIJDcDH9azpbhpSu4gtznAxUpAlumEx3nsw5P50LaMbtEQDOc8nrSQEDFQM9HHeiSQGNcbWLDJPU5qSa2Y3DDYQMZyehHqKrMQCQDkeuKaAUDCkkggcmmCR2jMOf3ZbcR70/y0KscsDngAZwPU1GRJHkEEZGcEdaYAaSr0OmTXFvBNCRJ5pIIHBU+9QXNtNa3T200ZEkZIYDkA/WgCAdachG4ZOATgn0rR0Kwi1LUxbTZAZDgjj5uwz2ptxZHSdTktrhfmjxk84ORnimBM1qsFkl6P38MgK/MpBBzgdaqAMU8zb+7U4Jz3rX06U36rpUs0UOnhxJNIxwcegz3JpL/TobMb7adXhklIjTPIHHU9CeakLmb5aLBIPMkSQkYAPBBznNQM+YwKnulmSUqRz2wQQagkKuflTYMAEZzzTAuQLLbwRzRKZJiCdhXqR3960orq8msYze2qrEGEkU7DaQRyeO/SqdrLHIjyXPyCNFjyvU89afqmoPeW8Frb3G63jUAKRgk57/AEoJGXlzbzzedETJM8m4nbgAen0xUbzyJmSI+W4YEOnABHcGqUJEkgUkhj0x3NWlkxAYiMgnJOe9AGj5dxqNjNfzXYaWFQ4UAEkZxyRjH/1qqbIjbRtK+DIpI4yuSanstiRPbzcR3aYDdwQcj69qdFpdxcRSQKyHyzkEHjPv6CpbGiGzZ1mELvsQkZ55Az1x3r0HwyXefUGkdZHO0FlOQam0bTbR0a9MCF5Qu5CoOxwMHH40aFGsWo6sEUKvm8AdByaaQG5QDg8UzdRmqAzZgdJvnvY1/wBDuCBcAD/VseA4+vf6CtdeRkcg9COhFRHa8bRyIHRhgqehH+TWNBq1vod02l305WEYNvIwyQp6An2P6UgN40EUxJEkQSRukkbchkOQR9aduFMBrjAqAtg9asMQRiq8iimgDfx1ppbJzmmEEHFByKqwAxyKdFknB6U3rT0GDmhiEmQgZA471WZ8cGtEkNHWfcABiQOKcXqDITsyaY0SOMjimkk0ofHBrZGdiM24HXNQdCQAcD1rQ3Iwx1qpIpD0ITQ0Y708BSQKZj0o3FG56VQhDHgnFKBxTwwIzSFcmkMgYEHApFJB61MQcnNN8sE5xTES7ztwOtCTkcMcihAAxB9KaUwTxSGW47gEY7U1pVDkDpjrVLLA8GjLYJPWlYC5E/73Gcg81ODhh9azUkIOan84joTik0NM0ygkRkboRisSaMwyFD1FaEcznHNRTwvOSAnzAgg5oTGyjmlDmphZSk/MQo9c5px0+fAZQGB9Kq6IsyJWqUGmG2uEGWjIHvTcMv3gR9aYE4GaQrnimqx6VKrKTzxSGQmM+lIVI7VcCg9DTDCewouFiBQpwCPxp4BRvUdjR5R3AkcZ5q28IAAQgg8ik2NIdbyYPBrStpeBk81kxoynpV+EELzWMty0aAfPU08c1niQ79o61YjkfoRx61DKJnXJzTQMGhpOMU0PSAkzimliTTQx7nilBBNIY4GnZpopcikApNFNJAoDAUAONANJnPNJQIkBxShsmo8mm7sUxkpcg0hcdqjLZFNLYoAfvxS+ZUDEjmozJjvTAsF6YZKi8wHvTwARmgCVHyRUjAHBIB471AAAacXx3pADwof4R+VU5raMuSQcn04qdpCe9N3Z6mquTYonT8nIYgfSmmxZTxkj1q+DT1qudhyox3jdCQB0p9rE0kvzkhR1yK0WhQksRzSYA6U+di5UI8I2/KOaqEujAMCDV4P60kihwBUpsdkQCQ4qZCCMmmiEDjFO2ADIFNsEhwCGkeJcZA5pucGpBzU3CxAUxT1HHNSFR6UhA7CncLCEUCkxQTg0rjF70oODTM80oNIC3G+AMU4SgtyaqBiOlLknrQBdeUbeDzUPnH1qDNGaALAlzSmUAZNVs4ozkUASmZc1E8uTwaaaYRTAUyP6mkDHkHvSYpMUXAeADTgqmmrxUmQTwKLgOVMc08ZFMDHvTw2KQBlscUq5PWmhhmlLHoOKAF2gdKCWA4puSO9NkcgcmgRGzAGmE5NNLZNKuM0DK10SM1TMhFbhZTHsKg/WlSGBTuESgjvitIzsS0YG/PY/lTg1a13HFIhJUAjuBWQFOcD1rVSTIasPJopwtyecmmPG6HHakBLCwDgEAj3q8pjHQAVmx/eFWgxHepZSLBfnjgUnmepqEvx1phc+tQO5bUg5HqMVyV3H9n1Qjp8wNdGJCOlYOtDFwJO5HNCYjeH7yJW9RUTLTdMk82wTnJHFSuOapMGiuynmmBfarGMnkU4KB2q7ktFUx/WmlatsFxjFMEYxTuKxVPAptTvEcGoSCDg07iaDNGaaRSimIbmlzS4oxTATNAPNGKMUDJVcipVl9RVce/Sr1raGbBYfJ3NRJDTEWUE4p+4HpVmWzhRcIMEd6p7SpxnpUblJWJM5NOFNU0oPNJgiReTTwDmhFNSBallWEwaADTwKXFSFhuOKUA+lOApQtIYoUYp4GKaDinZ4oYDZASp2AE+prkNb0uWaQAptGcls5rsQ1EmGhYEA5GORQB5vcWNhbwEmVml+tZOK7KbRtO+0ubhxnOSAcCue1SK1huAloSV5yT+FAybRS8032cuAhHOetad/bQ20XmgkyE4AJyDXNwTm3k3qxB9q0p9Vjmt1BZjIDnmkIrND5ilyMe2MVWHkqMSRgn1Jp8mpTyDGQB7CqruXbJoGPMIckoMKegrS0VJLa8VzwBTre3hkhDRq7ehxxTlmNuSpG7HvigDzvB7AmkzUsgCkgHPvTWKFQF5bvTGNoA5pQKXFMABwamBIAyRUFGKHqBZWQg5BqQMGBAx+FVwQE2gU6MkAkd6TQDmIGcdalt2ZWWQtxnkY6VADk0JuwQQSCaVhGpezCOMPGAScde1VZL5J41EpHnL0IGOKrzXMpj8nd8uQcVXByf8A61FgJo3CuWJqybgzYBY4HQCqijI6EfUVZRCsWSKTQx4iFwG28kDOKQnFioz3xVrTkIhkYjAOaospYmMdAaQxYgGIVQWPtV8wvb2+8wqjDqetLbCKC3IgAMh6kjkmniNm0+TBJZznk0xFORD5LMWBz6dqbZzKilWTdnocZxUiwTCLDHg9RT7RAko/ds47gUhi3kaQsB5qnIz6VWabzCOc4GKdFZC91AwxqY1zySPuj1NNurdbW78mFmmUfxYxTsIktw85KIMmi5Q27YPoDUI82CYY3Rsalu3aeFVlA8wHO4cfpUsYmTLagjkZq61yZNNjEjDeOxPJqnbsqwqrcgHOKW5dZCRGML2FCECkyn92Qq+9SF2jGQc1QMjxAxhSv1FTWUgMpSVuHGBx3osBchvjbuHZdy9xjtU7TW8gLROsgPUEYKj+tZ1yNqOnoKbYsC2D6Gp5QJ51ATzI5FGDnOetSx3Hl2zRJtIPOR1H1qskEoUrIVCE565xSQoYvOyOB0rRAA5jkQdTzUYIuICrsBPnAJ6GrMMAMJlMgBYEAEdTVVlj3EKV49BTTAIpDauVc4kxg0hBa4IA6jNWprN2hjvfKLYHzY5JHrVcEg+cQQp4BxgZ9qAGMrgkHoOlaoWJ7BvtDIJAn7sA5Oaowq12pCYBzjk1f0u2RdVjknEbQwjEnOBk/wD6qTQFRVzGF9AKtWcioSXBIAzkVa1T+zo5d9qzMXPIAOAfaooEttsgyfNK5AJqWIXzo5WBkASNjgDvUF7aSRzhwB9nJwOcgfWoZ5huEY6Ic1NFMcSCObdbsnR+cfSkhlC7QRSFQwb3HSpbPEc0YJwGB57g1A7AxGEYJDZzj+VSRI25Q8igA4GT3qmwJmWS3lkKOrLIMNg9BT7OQPcBJDkEEitAWsa2kj3CoQsfy7T3rG3F0UZA252kDkH60gLRiYW00oXEO7AJPAPpVW3YbABglTnPY1csbr7DKYpWMiyH5lboTU11paxSyXELhYXGUUDOT6D0poRlXM5nmIIAI6Y71AM7iD1HUVtJZPb2CFYi08gIICkkj3FUXilscfaIGSSRckHqRVIBiqZEt1Y4G7bk9APerM+Q+yEA4Od44FVXlRhhIyg9Cc81NC8RgKyOUZAMEUMZC6O8xJX5sZNRu5kdQcelXA8FzMVVXQFcYByePeqc0bW4WTOGJPHpQBJJCwlEYYMcA5HAA96esDQOvnAFWGRg9RVPzpmDEHlhgn1FWPOM0FumQCikZPHehCaLFvqMmmzH7IUKyjLBlyFP5+9XDqAu3hhmiXyRywXk/XOOBWRKAJCAcgDIOMZp6uHBcSFcjGO9UhWLN7p6tDJe27xiHzAgXcTjrzk8VY0zUL2GQyOIzDINkqYGSoFV5rJ00tLiCWSVM5dOwPqauXughdPivoZt7ygEQA5PIz1zSaGUbmMs0v2dQ1up4Y9QKph2Ee3JK571NE6PCYy2CSRk9BUSgFCCxBHQDvRYBUzI2AKmV2dBCXyo6AHnFPgKGMKwWMjOCe/1prx/MpiaMEDGR3pNDEjkmgLrEcFjnPcVMTJcRRksZJhwSB8wNQhSXAMi7j2FTLcPCTGCxAA5HX8aVwJ1feFVmIwMAjg1Vmt3gOUUlf4iRnmrMDRFS0yHaRgE8YqSWcRwoIZCwB5JHUelTcZnv+8hJVgMDJHc0SMZrUzTEuw4BB4U+hFWLkia5iMMWCxyQvTNWxp8rW00CIZjJIGk2chB6GqTJZb8JNe2uphxaStaAfvCFztz39/pXSDRtOstUFzd34hQuzpEvU5x1PPFaGgvYxgWsU5FxGu1kkIDEAcZ9eKw9QujZeI7ia4mQ2hmAdCgY9OOD0FVYY3+xo0vr23sMSW0hVo5vMBYMOSQe4yefrVbX7SXVtZhtlkjaS3twt1KBhQQT159P5Vk6vqI1A+WmRGkhaMjggHoMfh+tXdA1B7SCeyiljiuJ5A6SSDIx0IJpXEdFpehaTHayeTEl0ww25zlsY6j0/8ArUurmyk08280bPG4BMiYIj5HJ9DzWReNqXh65mnaaKc3MbRpIpAAAwcgD0z+tZq3gm0mHT7KOd3cnzWAJJBHQ/lxTiBn6jbSadqRjLo7jDgg8MDz0qirgud3AJJOOgrQ1CZ57qK4ucYA8vZjDAAdxVKPyD5xkyPkPlgHqe3NDQIVSmwrnsTW7e+HkbTrfVrKU3FqwzMUABjI6/rXPwkZIIzkEVs6JNdWzJaQ3DQ29ydkpKghs56D36D61NgZSuDIt3EwtSuyIEADBZT0bHvTITElvAxIJYsSM8jB4BrpNfgmuCmpwW8kNsgSGIyIRkDI3DPboPWqlsCbG8S4hjl8lAVBXGCc/wA6AKlpYTajGZYbcTSDJ2RscjHqOwP9Kks5ja3ZDNmOUkPGeCp6HI/Ojw/eTaVepNGCEchXQ/xDnH5V1Mmk2utwm4EMkVxMd+/rg8Ajr7UkgH+Fr4AvGZHdgdqK3Rlzwc+orX0gYv8AVD6zY/nWTpem/wBlzzFVxlQFJOSDjkj07VZ0yf7PbHuZDknuSCa0UXa4G6eDTgaoQ3Ylcpk5xkVKZgpxnFNxsJMuA5rlfE+l3DXsd/bwtMrxhJABkoR049OT+VdEsmelSq1KwXOX8KT3Ed21r5cohcEhSOARnn2rrCajjSOIFY41QE5OBjJp+c0rDEJNMJzTiKbigBpprc04im5AOD1q0AmcGpRggGoCwpwkIHFAE1QvyeetTAgioZcA5oW4inKuCTioNwzyM1eYBxiq0lvjkCtkyGiDJB46UpORz1pGBXqKjZznjgVSJY4kjpSF9w5HNJv9aPlaqJG5x0NOEtIR700ikFyUMCM5waM4NRcinAEJkjigZMHH404AN+NVsmnhiDQBI8YCkgE4qMoQoJ6VMCe7D6VKbebyQ7xkAngHrU3KsUMYqSPJOMU54uMjpT4isfLDJHamIli3gg7enc1JJJIMkAGqxmlduFAFI/mg4JqbBclSYtIN2BWlBIAgJrKht5pmIOAPXNXY7RY8ZZiR71LGgvLxEcBTnIyaz57kSgAnpWnti/iUH61IscOMiNfyoTsDRhgkc4IFO8xR1NbRt45GBdQQDwO1SLawEkmNcH2qudBymGJQOhq3As82NiEg9zxWstpAjApGoP0qUnHTA/Cs3O5VjKe3kjAaRMA8ZzQFIGB0q9cAPHiqYJUkEUJ3BoliAJGRVhRUUW3Iq0oGBUPcpEYUZzjmpVIApCM9KRgQKgoRmBNANNxSgUALkg0BjQQabTAlDmk3EmmE0m8CkBMzgCmFiajLZNAIoAlWTA604Pk9arFuaA/PWgC0WFMLcZqAyYPWmtJ8p5oAn3j1oL5HWqfmEHrQJvemIssxxUZPNRGbjk0vmCgBS4B4NKsxHAqFiCeKQHBoGXQ7GlLE9ahRweDTyQelAATSZpaQjikAm7mnq3NRHrSjNMCcnIqJjk8UZOKbg5oAMnNPHSmd6cKAJAeOtBZcYzUZNMY4NFxMexAOaehBANQE5FOR8HHagCzTGIHWkD012BBwaLgOJB6UzqaiEmDipFYkigB2KAKeBxRjFADcGlozijNACUmadikxSAWjNJRTAQnio9xpx6VEx5pASbvekLVHuFIX96oB4fBqRDk1XyKljOCKQFsjoaZilByOtIKLgSBQefSgkA0hOBUJfJoAmLCoZGzRvwKZmgBmKUHBp2BSEUAKCTT/ADCBgVHnFBbigAILgg1UaEhjgVZDHtSGrTJsQoCOD1pxUnrzT8D0pKdxWI8Y7Cmk0800ihsYwkmmg80/FIRSAQHJrP1qPNorDqDWiBzUd7F5lo3GcUAUtCkJEiE5OOBWuwrm9Jl2XgGcZNdITQ2IjIwc00EZqQjNNCgU0waEbBFIKUgUhBFVcGhCeKY6A9qeelR5Ipoloh8ls8ClEDelTg08Gm2KxVaFwMkUzaR1q9nPWmMqt2o5h2KZFFSMoHvTdvHSruTYREZ3wBkGt2ECKFRjB7is2zkEbgEVdMm85rOZSLIw/B71WntShLAg+1TRNgjNJcSEjjis7lFLafSpEQZyaACakVT6U7hYnQcCnMMCmqcCkLHpUMscBThiowTTgc0gH0hIpQMmlKZpARkmnA8UuKUKKYCg0FQ64boetKFFSADFAHPal4Whug8kMzRse3NcfNo99aS/OuE/vYzmvUeKq3tuLmExE4B9s0AebHTy8Jk3MxHWo4NJnuI2lGFVc9Qea7qHQ4EXEhZh9cVYOlW/kPCCwVhjrQB5cylWINIQx4AyfSur1HQ7TTh5wcuewJyKxTbxyMSrlT6AGgDX0czQ2AEihRjOR1/Ks67vbFpWBBdj6cVdgu2gjCOC49hiqV/BDNh4olj5yRnOTSA4MjHJ/OnyeWUBRCp6U6aMFyQO9PkmLqFIGe2BQMqgU6nFSOowafB5Ic+cMrj9aaYEIpRyaJNvmHZ93tVq3itZEJkkkEnpjNMRVPFKCQOKdIEXOQQB61ox2wvbMSRhY0Qe5zQBnR/fBPSrljDFdSPGbnySPuj1NVNhcrGvUnFdg2g2OjwW94WPmYHXnJPtQM5efTrwSFVt5GA/iCnmtXTDbvp82bfYyj7wHJNaUfiKE28sWXeViQMjgD2rJFvcRWEklukhUH5gnQGkxGUzyzOA44rSi3XVr+7jJ2Ag4GOarWen3d+C8afuwcFjwM+1TJbzpLLHabwExvyevrSaGRxyPGSAcLUW1w+8DIPSrFsIiTG6u7noBzk1aEeowSi3FpIqk4wy8ipsBQtxMsgjRAzyHgHuasgyRybHEiSE8gdBmtbUtCewFu0rMfMAJdBgLmuwudFh1DSLeKJB5hXAlxjA7kmgDz3UDHEAkch3gDgVPYW0kakrMq3Eg4UjJArT1Lw3NokZkLw3MbAAkjJH0HernhvS4rWOS/nUPMQdkYHI9OKdgMTRjaW+pyG/JCHI3ds+4o1OR7+Qw2sKJbRnfGQMFvx9abI5OoyMUAG/JQ8kVNqOpwSSRl7YIBxlCM0XEZt5uWMvtyxIAyc4qGytJLxyzuqRjqT/AJ9qsXygWwZXLBhkZqhBIYSSO4xSeo2zTuLWxihAhkJmJwCDVKSGWIkMtMBdgJkBwpzkCraTma1fzCBgZGBjmlYCj8rH5u1R24Mt8mBwDnNTzLHt3xjGeo96s6PYC6SZ1kbzFHC//XqkBWvJElmCKMNjBOetJDbCM5kIB9CakuYwiKCibzn5l4J5qJLf5S1wJBGOhJ6mm0BYlZWCmMY65yc1Gr+YsiAc46+tNcQmFTCGXAwecZp8NvJDbiVxhXOAe2KVgIYrmQwxoT8qHOKa9uI5d4P3+cAcCphGPODYxGep7CpQI3jKk5YdD2xSuBpRaiV2W0aYVU4D8E1Sv5nurOOIBY41OSM9aQxyFluppFA6DnqKtyW7wMkiGOX1XPrSApWybtq2kTsq/wCsfHQe9W5gltbIc73lbABHJHrU76qLHSL2B0jS4uH4UDjBFP8AD2jXFwEvpQhhxhSTyD6irSEUGVIoiWj+YdAeCKgsmVtRXccZU4+tSPambULk+a/DYAfqR71TuE+z3RQMCV6MP6UrAEo+ck/ezg0iOEBhmB25yCOSK1LXTs2zNNyG5ORzn1NUZ7d7cAsUIPpzSQFRVTzuQQvrTyWMe1eB6077NNtWQgFWGeD0pQCI2wM5GCfQ02Msh7eUYkbgDpjrVYKrK7AYUHAppGIgO5ODUhKKDGhyB7UrAK8wcAEDOMEnmpo7qeNAiOdq8gHqPYVWSF5phHGoLHoCcVO8JhlCyjgYJFFgNOe+lt5g0E00LSpkl8Nk+1ZFzJcXEjPcb2mb16/lVq4k87UoSI8oAMc/nVq5jjvi0okaBlOC2RgGhMXKR3Hh4/2PHeWVw0wIBlicAMD3xWS5RSVQMSQAe/Oea6C1sWEYzeiTzOUwckjvVA2kYjDo7bicYB5NHMUkZTpJCyscgkZBFW9PcusyuBJkggSDjrUkkohnRmj3ogxg8nNPu40uy1xFIxfHKAHJx6+lNMLEN2kcEw2wkI44QHOCe4qk42gFkBKnO01LbRmSbCkRuBkAnNSTI6o8kgBAONpGc/WmmDIXuhJ5gFuqFsbc9QPY1esrGaYJNaxAPGOSx4Y+lZ08iTKGVjkfwE5Ip1rNcIRFDKyIGDk54GOc1SJNy9e7jle2doAzx5McZ4APb61BA9jdWxt1gaCZFz5gcFifQGoNRntAMw3HmzMcvxwfcHtVFkBwwOdwyR6U7isPNjKnJEYHfD5P8qVbSYsACgHoTyfp61EEcoSoyB1PpVm9k82SJ0BJVcEjtUjI7gRhgFAUjqCeppYc8RqACOBk4z7VXJy5Pc9aVXKzBiMgDj60NDLIdbe7VgPnBwRnpTXYQ3Y3nOTlvx6VCo3sWPU9TVryoZQAxG4kAsTyaloC1Ihc4dcKfuYPGKgJ8q3MDDEgOQPUU8g7DJnOwAA49O/61XSFycBSXPQkYJFJody3IbX7Ms0e/fzvAPAPqKXT5L+CdriKVoVbDENyCvPbvj+tRJCwDQvgN3BNAjkRmkjLMmNpEg5HrxQgZtaBqAhiu0vHBs3yRMBghyeMfX+ldLdeG31LS7eG4miiuAS7TEZZhg4J/A15yHxA8KsSGIOD0GK3IfFmoLo50vyxIShTzCSTg9RiquKxBpGjTarfyx+bsgiOGlC5BHPT64rqLjT9Ct7M2TCFJHUYMpJbjuCc8/SuStr+aG0e0jLrvYHAOGB7g+1XhbNrExkaV0uIo8mSU/KQOnv0NFxWIJ7C6kuREhMluCQJDwBke9bMOjSaXDDei8n+SMyNtjBCkevqKqTSXJhFnczFxFiWMkAowAyOT1q5YeIBqlld2Ny7QmaPahbgZOcgHt2wKaYWOa1OUXEsk8o3yTvvSQjGQABgDtVEt5caEgFScZBrZ1+1u4ZreK5JKxxAKwjIXPoD39zWdGQIhwCe2aVwRPYJZTwPHcFQyZ8tjxkc8n34FMtZhY3yNcxyhMEgAjIz0Ip8NzHjbLChIAAcDn8R0NX7aITEXZ8swqNhB5OB7f0pXGdj4eli8QaR/YN+xcwr5mCSDjPyjjrjIP4CuOudOvY9Q1C0twzQwgtvkBAKL6Ej3rQ0LVf7F8RtdX4YrcRmKOZRkL0wcd+mK9CgxrWjbb2MrIS0Um3jODg4+ox+dK4jySGZzCw44x94YJ+ntXp+hWqvpFpcRxeX5kIymOp7n8etWL3Q7O5ggtIrZEUTKxYDkAdRn34rWSGOCJYol2ogwBnoKaYWMy609plLAYIBJ965uyBe2UAEkdcDpzXcjnI9j/KuT0G3ea0klRwpWQjkdQa1hITRDDIIrgE9hUszhnLA8GtOWwEowQAR0YDoaxp4ZLeTZINp7e4rRO5GxPHdhRgg8VOl5uYAVmZp6uFqnBCuzdSVXxg1KKx7aYBsk1qRzKwGTWMoNMpMlK5FNIxT1dT0NKcHpUWLIsVXuSQgIHOetWjUMyeYhFUgKKvk9amRwBg1E0JEgbkYGDx196GIEZYHpV2uSWxIB0qtNMckk1B9oIGc1E0hfOapQ7ibJhN3zT1nBOKpHNAYjpV2RNy86B+lVZYSDkCnJcEDBOalEoYZ4P1piuUGBBwaSrrxo4zjn1qu0JB4ppiIwx6U6hBhxngU51LPxjGKNgG8E1atwWAUHBpwt4R0JJ96fHGyPkHAHQVLYypPGI5CBwPSo13McAZJrQmtxMwIOCOvGc1NYWSibLjPpS5la7HyjdO08BhNIpLdwTkVuGJZUweppyxoBnFOXA6dKxc23dmljIvdKlRC8TA8gbQPU+tUprC4gkIdAQO4ORXU5BXFAUHgjNCnYVjlowiN81SnyjzU2r2EkMvnxp+6bGcHJB96yvMYdTWidyGrF1lOMxyAH0IzTTcTIoyAR61T81s8EilMjkYJJFVYLlkTF2xwM1PCzk7cZx3rOBIOa0rCYNJggj3qWgTuWgCByKb5gB5IH1NXNit2qtPZQySFiOe/oahmiAXMYGN4J9qU3CHvUJ0+FVLJuQ+x4qBLeYzBCMr3OaVxl5SH4FMltztyDn1GKnhg2ZIHWrIgBHI4+tTcGZCkgjFWkYlcZq6bOLrsANQy2sgwY1yPTNNu4BEcCnHkVAgdTgjBp+445qBjwoNIQB0pwFPCeooAYrKVwetBhOCacUCnIpolO4gnigCswIJFNNX1iRzk9aqXAUEhaAIiwpN9MNJigAL80F6Y2c0c46UxClz60wuaQ5ppzmgBc5owTQOKduAoAbgilzSFs0nWgB2c09KagGeasrGNucUARA4NTKcimFMHpT1AxQA4UpGRQOKcMUhjNh9KUJjtTxUigZ5FFwK5B9KSp3UZ4phT2oAiwaTJBqQjFMYd6YCk5qOQGpFGRntSOuFzQIrFiKN5ocjNMyKYE3mY70hkBB5qEmmk4NICQHJqeM4qsDTlfBoAu78Um/Peq5fjrQHpAWNxIoBwOahEnFIZPU0wLAf3pd9VTKKBJQBZzmkNRiQdCacGzSACeKibk1KTxUTA9aYDQlIUIPApw3A05W9aAIgDUgOKcQOtNoAkVzkZPFShwR1qrnBp4agCZ34xUJb3pGJpmaAH5qRelQipVOKAHEcU0mnE8VGaAEJ5pCeaCaaTTAUHnFPzxUQ604HigBetIeDTgaY3WhMLDTTTS0lO4gp/l5HPFIAMg1JuFFxDVQA5qcRhoypHUGos+lOVjnGaBnIMht9QOBgB8AV1C/NGreozXPaxG0OoEnvyK2tOlM1mhPYYpsRPSHFKRzSYpABxSZFIaMU0wGsKgc4NTnpUbAE5xWkSWANPBqLIBxSgn0NDQiXNIQT0oUFqkVcduanYogKkdjUZPOB1rUjjzgHoanFpF6Uc4rGfbW5Y7iKteVipmUR8AYFM5JqXJsdhyAYpWh8xSBSqMAVMhwKi5ViqsBTtTwMVZK5OaaUHai4WISMCmEGpivFRMCOKBke7B5pyyDNRuDgmogxB60CL4YVIGBqiJeOtO809jQBeIFNIqusx7mpPN96BkmTTwTUAf3pwb3oES800jJpC3vRuoAa2AKickggVI/INR8g8UmBx+t3DmV4fJJz3PNYUN29qxCkBh6jNeh3Vhb3akSqc+oOCK4jV9PjhuykLcDrmgDOuLy4mYFpBx0wtV2kkdslsk96lEghJUgH6imO6u2cgfQUxmAMHr0qaYoqBkXIxg1XXpmh2OMUhkbnLE0w1IqhuTnFKY8g4GB60IAhtXnBMYzjrVmwAidmkOMcAY61UV3RQisV3HBI9K6e7hhttIiCdGHJPUmmBkTwRPcIHcESdu4+tbSWv2OKO0KgQyHJJ6iucRHL+eAWVSCcDtW1c65b3cMYjR1mXoSOAaYieHQ4IfEFsryj7OTkb+ACPetLxddpcCK0t5EYrycMDgYrmbu9mvdoubkcYwAOBVOSOIuSjsR65oGWtPsme9ABDKvUjvmum06/itRd2KhpZZj8iAcH61ycbtACYXZSRgnOeKLe5uLSbzoZSJMEZPNSBvS6bqGj25kLg28hJZYzgqT2FRiyutOsJbmSPasgwQz5OD+FVH1e7vYlhuJDI+4EMTj9Olb/iFnNjApPBTkflQSTeBdPt5oXvZkDsrYUnt7109/bw31wNxI2DCsvBOeufyrJ8IvHDoe14yMsSCBwR61tb4Dzkj6ihoaZNJbRTWZt5CkikAEnrUdtvtYRCFBVeBg9BVeSeJAWLZA9OcU1dRiH94/hRYVx80Yub9ZJASqcjAyM0XVlHdNvhQxzdmAwPxp8EyuDIJFKk5znpWTr2tSw2xS3BUHgv3I9vSrGmc3rssbXqQJbETjIdweCR6VzV4skEu2RCrE9D6etbVpc+dM811MsbJyC3JPvWfdSC8uDKXAKngnvSaEUVkZyAxJB7ZqQoAMrVqGCa/mJQDYDjcB/Sq7I8czROpBBxzUNFCAFUPz9eozwac0g8v5T14x3pBGWcAYx3zSbAZjg8A/nSsAkL+XPGxGVByRntXQae7vqoW3VQJBgDOAwrnzG0s4jQZJOK6HTdkOrWBDrhWwTnjpVICXUdAaW/2WkeXTBkIPAJ9KydTjlhv/s8hAjGCcDrXVW6fbtR1IwTNHLC2VIOATz1rB1QG7IkkLpdE7doHBHr7CkIx2iEsgjjIAPcmtS83/ZY7ZeVCgn8Kp6jpzaZFEGud8jZyi9B/WorC42tMJHJAUYz+OaTAtQKJkS0RSI2IBJ7iq1xCLe5kiUkjORmtnR4luLG4dWCGDJ56H61kXMge4eTYFck5A6CpY0y5dW+dGhjWZS3GU7k1JYC3mtSLhD50QwAetRaVbTXl1mNVwmcyMeBSSxzRSSTEgnfgEHrSAqXxfUr0iGMhgOh61saTfapaaNJ5NjI8Yz+8yAAPp3pyrHb2AkSJPtDEjcRk8+9XptNmuNHjs4LuNFUFjg8knqK0TEzmbeFp3ijeZT5z7ivcetS3uneXeyXHlsbZTjKnjNR+bDbxoUyLhDy+MYOegqQrqN4sKR5eOQkIpPIPFO4FSS9kuJNpJEY7A9aVDLPK4AZyBwOuBTZLCe3u/s7gCTIAwcgk1o2VtLb3Y3EKSDknuPSoYypHPGbSQZw0fAGM8VGZNlkIyOZH3H6dqlv4VhmSWMfu5CSPrVKSV5pUDn5c800A9UJYbuB61ZtY0F+wZQ6qCcGoZpBHBGD1LHH0pUmIldx0KEUgJoZFd2KxmME5355H0p0wMgJBJJ9T1qgk5I2jjNXITIZIyqkqhy5AyBTAsWKB5LgkcRpgD3p8aQzW9widNuW+oqOKQKspjIwCScdaqR3TAl03ISQc/SlYdyxIPJQJGcOgyO+M1FaSkhlY56kH3pTLGzvM5+YjH0p5h2qHVyxIyc9hSsBGFUgm6cMhBzjse1VoriSEMIW8sNjIPJ4qeM+dHIG5HQH3qsbWTJyMYIw+OD7U0AxiYyHHUVNc275V/UA/nTpAkpBJ8tsc56E+tWLm38vTFnW4ZmXjGcgj0x2ppkmYYHALY4qW3t/O8wLNsKg5GOtPkjmkgEzyKoxgRg849aWNS1m4GOD1Jwfzq0wK5U7MMQSg7DHFPVVIVA21j3J4PtTSQhIIyRSqQxy4yKLgXDZtCyK0itkAuEOdo96mfT5j5aQhkkPDH+Fhjrmq8SxscQxAkjJAOMj0pwvpsYG7aOi7zgUAVhBLuIK4IbaR71ObCeHLMo+Xtnr9Ki82YbfJQjacnjPFWTdyXUxLyHB5wOADUgNjiy7R4VEGMD1zU6xgDYFRSpyMjqPWqcjSS9CSxOOT1rTtdOb7QbebqMEsTkAHpmgYyWfCuipGuTkk9we1RxXXleYGCsDgBxyABVjXLorPLpyIFhiAClRzn1q29jb3Phw3BhKSRoChB5I9T6/TrRYColrCLkSs455JY4IB6YqxPYNNbyXCSoVjHAHJOKzWdZ1fYrt5YABPpUkFs6x7TNIhcFgAeCKgDPClWy2AfQmnxvGizOUYsRiNwcBfc+tPuo/tF1FHbjfIRzz3qR9NuIoXkYxlUGSVbIAq2BCBNHIJGDBgMq45JPvU82pPLDHC8hCg5JA5Zs5wSKqBMwNMpJCkAg9eatz2D2qQyTL8kwDKehH1FAF2W6S8Y/bd0BUBQinjAqjFLhljneX7Fu5MeOo6Hnr1P51Isc1/BcXY+YQgGQjqMnAqU2slvoVvcSlJFnJ2R5wUGevv0/SgRonXbeXS5rEGYQuPLCuS3H94Eng+1R6DocWq6deyvMySW4zGE5JwCeR+FZGnR/abpbZYy8kuFQjPB7nA6//AFq6yTw1qGhAXGnzNMHHlTKBk7DwTj8aYGDpVhHd6TcapcFzDbuFMaDBIIHP05qO1GzzJYyQpfagc8MK9Bt9At7TUWNrGBa3UCh1xgApjBx6nvWdLo9la6eRcYZEmeQRMwBIJxgY5P0pNDMGbTrswWswaKSOYlgdwHlnpzk10XhvxOtsZre/mRj5nyzDODx0wAcn8qzfENus0cKW9rBD5QDuIh86g8cisSF2tCDHIJkDbiCAMY6VIHpbeKrI30dpblbiSUgIY3GB67ien61tLNvXO5T/ALpyPzryqGWPXb37NPs3FC0c0UQQg9TkenSu50a5EdrHZ7MrCu0SIMocfyNUgN3zgFYk9AawfDDY0+4H/TT+laEz5t5jn/lm38jWX4cONNkPqwP6VSdgepukjFY2soMRyDjHGPWtQNupJow8RUgEHsRTUrEtHLDpmjvWsbGIIUAwB0rPuYDDIOchhkHFdMZXMmhiHHQ1IshQjBqAEilyTVWJuaNvcZkG41orMhHWufVypyDU8U5ByT0rNwNIyNhpABnNIHBFUEuSxAPINSLNhipPeocWi73LLYxms1zsjkXJIPPPar7EtHweTWbcEpkMeSOKcNxS2IC/FNEgHBNaOl6Wuoxu8sjIoOAF6n8a3LfRbC3/AOWPmE9TIcmrc0nYhI5XrQBW1qeiGNvNtIyUOcoAMDArHBqlJMmzGZpwNSmFnAKqTkdDxioCCrEHqKYWLCuCoANBaq4bFNtJjPAzsMFWK/XFAWJygI460wAg804kAZPQc5pwIZcbh0z1oFYmMiPEoIAYdGHWgu4OScj6VXzT1kLKQTxSsO5cQiRCwbnpgUgmmgxxkDuapBypyKeZmYYAP51PKVc6OG7jljDA+xHoalD5rnLe6aA88itmCZZIwwbrWU4mikWwxJ5NSqwHeqobmpA+BWVh3LZxIhB6Gud1PS9kiC1jJJJBA/DH9a21l4oYgnNOMmmDRx7o8UhR1IYdQaaDzXVXFnBcyiWWMOwGOemKhNjbxg7YVH0Fbe0I5DBhQmQcVvWdvEIRleajgtQJRkVfEYUYAxUOdwjGwqqg4ApdoOcCkAINArO5pYjdCe1OjhUc45p5oBoCw4DFPFRh6cGoAk60hFAIxQSDQBG6KeoqFkAPFSuTmoHYg0AOGB0p4YYqvuNJvPrQInZgTyaYxXHA59aiLk96bknvSuMk8xh0NQOSTk1JSFRTAgINNPBqZgOlQkYOTRcBMAnNITxSFgBimFqYgPWmkUbhRmkAEUhpSaaTxQA3ODTgeabmgEUwJ0IGKlEuBjNVA+DSh6ALJkz3oEmD1qvvz3pQ3vSGWg+R1qQNVQP71MHGBQBYBFPBqurgU9W75oAnFGKjEnFKHyaQgZfaoWQ56VOTmkp3GRBSBUgAZMGgjIpC5UYFFwKc6YfgVAatSkkkmqznJpiGZoJGM5phNN5oAfuo3e9Rkmm5NAE+/IxmlD1Bk+tLkigCYycUhf3qHNKCc0wJdxNKGIqMNSgmgCYPUiPz1qtuNKHIpAXN2aC3aoA+R1pwb3oAkzQTimbvems4xTAUygdTSCXNVy2SaTfjoaALJfvThID1qlvNKGPrSAulgaQsKq+YfWjzaLAWt3NPD1T80+tPEme9AFvfTGJIqIPz1pwbmgAJNFBIozQAuKUGkzS4oAcDmgjINNxinAmmBGwpMVIRSbcmgQgBxS804ADtQQMcUDGhsUuc009aQdaYjH8QRktHIPugYP6VJoEm6Jo3POM/zqxrEfmWDYGSCDWVo02y6AzjcCKaGdATSU4jBwaaBSEJilxTsUEcUAMKnFRlcirG3NBQVSZLRAkIJ5qTylHanBQOlLj3p3Cw0L6U4LSU8cGlcdidCAAalDj1qtupwapAc7EmkBIpaQkUhihjnFSoSKjHWnA0gJw3FBOag34PWlDjvQMmAz1pJAoXmmiQDvUcsoKkZoAgkcDIBqsSTT2BoCE0xEfNLkipxD607yVx0ouBAJD608SH1pTCBTShFICdZM96kD5HWqwGKkXrQBYX1p9Rp0qQUgGmmEHPSpSKTFAFS9S7MB+y7d3fNcNqaSKMzzZk7gdq7263mEhSRn0Nc9qVhappzTTjMnqx5oA4lxubIpwt8DMh2j1rQ0u38+RmWFTt60+UCa6EcsYEYOTj0oGcYq8gUkibTjcD9DXT3OnR39jB5EkasorFmto7PfHMAZB0IPB9aqwjNDAU4OCKlSLdCWNu7juwq5FbrcW2y0gLkdWI6fWjlGZscMs0gKISqkEnsBW3rV6hhhtUOIwuavWFqYdHmtxgyEZwDXLzxyrMQytx7Ucoh/2iRYzGshCHsKjBIpgHrT+1IdxepFSEBOCcUtpC011HGCMk96uX1jcHUvs4iIGcAgUxXKYDlxsQvnoAKvNp0wtPOCkHuh4IrozoUVvYRPbuwIAJJAyT9atnRrWS1EzmZ3YAk7sg/hRYOY5C0smeIXBkUFDyh610ur3Ed1ZW4jBbCgkjoPxq7FodpBBcEQhgykgk55qzo9lZTacBJArHOCMUuUV7mlpTRvpcAjIIA6A5x/nFXPLb0rKWy063m2CQoT0UvipLtbSwtnuJr+eKNRk/Nkn6cUMpGhs56D8RmoZkDnbtB+gxXL6Prt/qmpyW8N4VgXo0iAnHbPH1rpQNRTIMVtL77sZ/CgTRFcaXbzoEwyMf4wcYrjdQgvNQ1Q2kcxMMfCvjt712E97fW4LTadGY1GWKScgVyK6rc/v2sLCTy3Jw552n3xTTEijDbpDffZSRNcAkM0h4A9RVfXbaKKUeXIrM3VR0FZ0yn7QzOzFsnIz3q3arZK5MwcjHBB6U2wCPUpra1WGFQjD+MdagWV3ZmkO9icktyaW6CJKQtQI2HAqWUkWCzbht6UFsd8EinKoKsScAVCgMkhzzjp9KgDV0qGEt5zFt4OAM8Gp9Ut0gnZo48ZAJAHU1QAESLkFQc8mtWO/kjgEUgBGMZPJIpFFjwpOItQdZARHcDAJ4BI7Vb8T6pDFMsVtCZLiMfeHIQcce9UrSEZREmMUbNlXJ5HritHUNGjWznW3kJZoy0jsc7iOeKaZLRwkzNO5kdizE5JNWdPsHuLWeZGAMfY96n0/SGuLO4lm3qU4QDjJrU0O0kt7C6hu1CKc5YnkcYqwuM8L6jb2kF1JdFcMRx1JP0qlqN0dRvnuFt9ikALgdQPX1qkJEWZosAQg4JHUj61rPMjLDBHIsUYx82OR+NQxJGcnmIpAaSPPXBIzUlqgFxGh3spz0Na80XmsiSzK0IGAwP3qh0m08zXFjZiigEhh244OaQ7jCLie/js1cLu+5npj3ro9Iid5prG7CiRVOCnX6g1gWVsYvEUwlLMLcEliM4IHBNaKMotI70yFZpJ9jsB1BqkgKd3pMNzd3EkrbbaFdqMvAdql0O6hjtY4xF5s0TnHzY2g9/etPWrZHvbDToVzCDvIzyfrWXdaObW9eAOY492YXA4I9Cexq7AYl6k39rXRAJw5bAHAFXbe9WEKZEDKRkNjkU7XUFlqTiA7FkjBbvk96qXF9HPbwwrGUSMYyT1NQ0NMsalBgPKhDQkbihHAPqKwYzumXPQHJq6t/KYpYC3ThT2wP/wBdU/LYEADrSQEl4DJIpBBXoAO1OlGMDOOMVGo2OA3FNABUlidw696YhoIR8A5qVrljGY14U4yBxmlHk29qtws+6fOAuOB9RUBJLlyckknNOwxZBwWBI9cGr8JU6GfMP/LTI+maoF8oUI4PU96m8wi0EZ5Xg4osAjYCYBye5q/GQNNSOST95ITgY5x9azkGdvucU7kOjZ+4cgelSBvW62d7sjKrHMBtIBxnFVHnKBLdzhQcknrnsKo5dpgycEnOfSkuDmfd079TU2GmW5xEUYuhYdSQOlZu0nAGcZz1odjlgpOD1pqlQMHOe3NWkDZLtkPJQuB27AUjHjhRj0NNMzojRg43dSPSmmQnGeeKpIQHBOW6+xpHIBAHTFORguSAM5zQ2GySPwoHYmjZ0gNwhxtOD7GpIQoheSRgi5GMnkk9qru+yAxKxIfk59akkGI1BOSRkmkFiR94IEbEKQcYqvAriTfjKjvVm3kRQDkkgEDPIApJcBCwIP0GKQh9gTFdxuE3qCSVxkkV0el3Nibe7hv7lrWaZgVlK8lQOn0BrKl0k6bpkN/PcMHLD92mMgEZ/TvWQ13KWLKSQe7HNMDVvoGEO23kSWNCWMoOCxNV0u7tNONu0h8nGQp6D6VHLIJlA3iMADcRxk1GEmHISRkBBLYyOtIRp6TCrX4h81WDRkkAE59vrSmWKCVzJBK1uOE2yYKn34roLO6ureOGdLF5I3jLEBFBU49ew4ri7u4a4u5rhzgyOTgdBRYBRMrXE0wd43HzITyc+9SO4EBy+d454Iyc1WiZQxds8cggZwa2E1ezms447rJYAB96ZBIPXjFA7mWZFECvGCDznPQ1qanc6jBp1tZX5SQOBKrEDcB2Gf8APWqNlGl00sJcQrICyADj6fjUMgmnEbsWYKNu4nIAHYewoEdN4WuLeWxvdJAVLi9hIMxJwSAcDGPc96ztUtLy2ax024kRokTCuhyCM5PHBGMjOarWUwtcnYrEjgkcg+orSJN5NaXk92I5AzNlwCCeBjHYcCgRq+D9Jt5Al+WjFxHMRGAwBKgEHIzknJrtSSPeuI0W4tNJuJ57iKP7bK/+jkHIAJ5OO1dzbyG7tIp2iMbuMlSMYNUMheV0IQMiyOCFDHqfp3rmtY3w30RcvdapKMQgLtVPcH9a6N7EHXftUgyqW4WPPIDZ5x+dQatbIlg187YmtVLRuT3OByO9AHHjSjLbTGe8KaqCTIJDwR2HvnjB96xZLSLzo41eRT5YLfKThxnNaOp3R1jUFkhZYWAWF1c8nnGQfTmrFzDfaVAtrMbdZACySBw5IJ4A9Dx6UmJlFRaeXaEvMLkrksoJBHp/+qtrQtdvbHUDDeITZzSDBCYAY8DtwOKwrq5nMcKXUpQkEocAkA9eep/OprKaC5lNnNct5chQrIx5O09P1qAbOpvPEH2aS4s5I5BcEkqXOIwCOgPfA/nWhoJ2aYR7j+QrA1MJqDRC5idTaAgpnkg4wwPfoPzrf0RFj0oBBgZ6ZrSIzSEnNSh8jk1VBFTCMkZHSiwDZCMnFQOgdcMOKsNGQM1GR7Gri7CauZE8Oxyc8HpUUa75Ag6k4FbjWSXCdOfWn2+mQxYOCW9Sa1VQzcTGNpL5mxELH1A4qQ6bdqgfywfYHn8q6VIl44qbyVI4FT7QfIckI5YiDIhUe9OMylsg1vXtgbhcbgvueayLrSnt03IfM9QOtNTTG1YktZA4xnpVp7GGYq0gztzgHpz61lW3mQkSyRsI/XFbQmBUEdxmomNE9oEt1wABk5OABk/5xV1JVdsVlb+c1agfGDWe5VjSVgBgnisbU7KESQvCFjySCAOua0POB71XusOgyM4ORTTCxTSBQc4qXyIyQTGCfXFAp271NPmYWKd+FgSPyogSx5OOABya4KS/mKzSRyAKZSoCcDBPWu6m1m3ivRan5yRngZFcDJsvYpIooljZWaRQvBOD0P0B4+lHOwsbt1d+RppBOZtg9gCRxzXNx3mwZLSGRf7p4xV+aK5mgLOfmYYYew6DHrWbLJHE+2MBcAA47mr52RynRxapbGBMFjgYPGOau28gnhEicgjNcjJFI5VYjkmMMFHUknFauj3j2CyW0ybDgkkngVanclxNwgjnFIpOahguxcYA4GMkGp8YrREgDmrNvcGFwCeM81WHWnA4YHHQ0mkCZuxXAdQx49qso4bpWGl0quAsfGepPNaKOAMk1zyg0zRMvA08ZqolwmcA5NWFfIzUNGm5MOaQrimhsGgtkUgIycNUgcGoGJzmgN70AWMikyBUW/jrUbzYNFgJzIKaXGapPckHk037WB1BA9aqzFcviQUokHrVMTBgCDTg+Klody6HHrThIPWqIl96esvvRYZcY5FVJSATT/Oz3qGVs5qQGl6M5qLNODYqiSQ8Co84oLZqNic9aAJd9JvqAsaQt70DJjJzUTSZphb3qNjQAMxzmml/emEmmlqYh+7ml3e9RbjShqAJNxpc0wGnLQAHpTSaecUw0AJmgNTSc0UAP3UoaowaUUATBvepFeq4p4JoAsh/ep0IIqkCakRyCOaQy10OTSgjNRCT1pQcnigCYNTgRUYBI4p+045pCJRgio5ACOuKASKjkfrzTGVpuBVZutTSHJqIjJpiGYpdgzS0d6AE2gCmkDPAqQjimUAGKaVp4IFMuGEdrI+QCAcE+vamAgXNIRisGOw1WbEjuU3dckA47cVvImyMLknAxk9TQAmacDTO9KDxSAdmlzTM0FgOaAJASKUPjqahMntTC+SaYExm9DTTIT3qAk0ZJoAk3e9LkVEWpNxpoRNuFJuqPNGaBD9x9aM0zNGaBkgY04Nz1qHNOBosBYDc9akUkdTVZW561Kr4xSGT5Jpc00HNKCBSAeKlCmoQ4zUyuMUABSkCkGnFhTdwoAdijimF6YX560AS98UECmBveng8UAIRzxTcVIcYphNCC4yWMS28iHkkcVyVsfJvUP8AdbFdgD19xiuTvY/KvZBjHORVJgdTnegfOcjNJUNnJvs4znoMVLmgkkFKMUwE0oNIY7FJTgaTApokSkNKQaaQetUAYpc4FNLEU0sadhXJeaTcaZuoyKLMLkgc9zTt49ah3L3NIWFFh3LKtk1KOaqISMVYQ5xUOI0yQqabjFSgjFMIBpFDCRUTHnmpSozTSgoEQ5FOVgKDGM9KUQk9KQh4cUu/NJ5WByQPqcUpUAEllAHU5FK4xc00iqx1PT1k8s3aBj2q1GRMN0bBl7EGi4DcU5RUb3EELYlmRD6E4NNOpWKDm6iH/AhTuBZBxUoNZY13TS+z7SB7kcVfiminXdFKjD2NAE24UhYVETg80hcetICYkd6rXFpBdJ5c0e5fTOKfvzRuoArQ6XaW4IhiwD1Gaxo7ANqx/wBHAXvkV0W/FNeVoo2eMZYDIGetAHkNkI4vmCvJxjAYjFbltolldQJLJGTuOCCcn8azbbQ79GLLuGBkEDOf1ro/D1pcgma4bKjjGK0RJPaaBDGdkgItx/BnIrVjs7VYTDDAkaEY+UDIqwB5hA7d6Rpo48JChkf0BxRcaZwrQyaJrJhmJMchOCehHtVLxQYZZ4Y7cAv0wo6nt/WtrxxaTNFHd5yQCAo6DvXHXF2k0aSKCJE6k9qoQ+40a4srcTXGEznC9Tn/ACaonityac6lpazXEyoUBwg6A+ppuh6SdRLTIQ7REEq3GRzU2KuMstNlEEOpQkSBDkoOCMcV1kFompTxXiAFU6knHNVre3+xTTQtG0aS8AEDGfb86vaXHJbymHcDExyMdquxFzUYAEYGMDpUchwBVr7O7HsKadPZiCScDtmlZDIUbIKHoRg1zV7c6hY3hsNNGd/IOMkf55rrTa7R2H0qFLNY5DIgAYnJOaLIZjWWiQ2uL3UZmuJzzhs4BqC+0G41e4M08+yFR8gGOKuavrNnZMqOTJID91en40xNYe+sWaCNIY8H53PA/SpsJFDwVZSQX96x5XBQH1Oa3LjW7iHfa29uzSJkGRzhQPU1x9h4hvdOeSOPY8ZkJZ2GSfpVnxHq1xeRwRxkIso5VRjI44J70Mq5Wn1S9ks7hZLgsCTwCcDPpW5p9/LpfhyO3htHEkhJLnBUA9zXEiFzwJF3D+HPNXTq+ohNjMCAMDnGBU3GNvLaGNZZEuxNIxyQRjJrOimeOQMpwR0qZgTHlhhj1qvjBzU3GPdzI25jyabGCZxjkU0nIqWAEqWGNw6ZpiCRz0BogkKOxABOOhqIjknI5NWLeF3AKffY4A7mpaGP8xp7hXY4AGMDpVp5zOQSCMDHNQvbtakiXAYdRmpbchoy+CARgZFJoaYJceWQFbhiB9K6C0vxZIbO9VzCxxvQ8n2rj0fypG4Bxkc1Ks8kkkcZYkFgACcgU0hM9VbS7IWSIoxH1yeSRXA63NPc6w1vDIZI4zlQvQD39a7LUDMNFNvAMyBAOOuPUVzV3bppNkZYZEJkUglhlif6VdiLEOiBDczSXUcJhb5HDcEEV0OqafYHTvLjt1TdgrIqjgfWuLhhP2fe/BJJ+orstBmuYbcTT3BntWH3ByUPPNTYox9Xt7q3sIAYl+yqAPNU9T2q7ogMml3EswAMoCpjg8Dj6VPrsU2sKVjPl2duN5kA4Y/SsTTbJVeyczsJGkwATxjtTUQN7RbVEsrtJiFmL4lY9cCucRJrzVRChJhBLIBwAB7V2doRIs9vPCAzkqJFHX61BDp4TXgUGUht8E+5H/1qLAS6RYB4Rd3LMbo8E5wQBnAq3f24uLYkHPl8kHpipYAwUb1CkDBA6VR1FZbWW4v1G+FgAwBwc+uKpAc/4rsJXSGeOHKIoAx15xnPrjFY8OmzTaS4kUIDlkOMkntx2rpPMgm8j7bHcRLKSI/nOD04purRTSafdOJDDDCu2Nc8kDqTTtcRyaaZPcA/Z1BUAAsTgk96ph5Eco5z5ZIx6GrtvfPa2QjUszOCSQcbfpWcrY3bjkkkknqTUtWGmTNIZEOQuR0I7UjK8ZAI6iolYrnHerLTCYhnUA4xgGoYFaRNhO4AGhGG7OMj0qSYEgEHGBioMYbGaLjJMYGaEcA4YAjGMGlXBBHoDUZHyg96YEkQwHPfPFKSalsrK6v9xt4iUU4Lnpmrk2kXNqnmuUkjHVkOMH3BpMLFRZjGAAOoz+NRSTiREOzDAYJByDUlsjTOF6jsfWq80LwyFJBhgcEUWATf1AySRimEEDBIz7UYIpwAIyaoYzJIB/Cl+YdakhieaYQxjLseBnH60TRNCxRxhx1Gc4pCExtyT1I4FNLEnk5Pem7iSCTyKDknOfxoAljQyNgHAAySe1SuEELEDJBxn1qGKQx7vUjFKcmMYPFMY+F0EeC3zZzjFW4XidQjoAQMZB5J96zcYNW7IRvMBNL5aHJLemKTQjoUuLfUtPWzlTMluMhs/KB/eOcHP1rKvIbIzA20zSxg4eQrgA1fi0q2uNNmuP3kcYBKgDBc9zz1B9BUVlqCQRiFLKOSHaDIXGCWHUZNAGWLVpWJiKlQM9RwK0tO1NoJFaRWdUYB41AIIxg8etS3sNnD4et72NIzNPOSSpyVX+6R+FRixMxku5CY8gMI04OPWpAbfaxNeubeEfZ7MMSIxlSRx1/wq/oVjYXtpPG6LHdwkkFznIPQY7//AF6yrqyhjebZNkAgggY60zSLo2GoC5WRQ8YJCscB/arQGzN4emTL3HlQwOSX8sDAI74rLsrTT5NTksWkdxIMQyE4BfqAQOmemaivdWu7+9luH8yLzBtCIeAtVDF+6SU4KyZwQfQ0NiPQL3wjc3KzXLvbxzLEQkUKZBx6n+orgzFcW1uglheONm4kIyDg84rqfCWvCwSeG9lmNscCNsEgHn371r30NpdeEZbeMGR0BMIkGCWz1AzQBxJTyrhF65BP4YpLICa48sKz7jtwoya0LHTPPtpYbhhFeKQVJGWAxwMd/wAKv+FPDk8168k0rRwgkYXIZyPQ9qVgOi8M21nfaXdQ3VvD50RERYx4IGOCPeujLJb2jsJIyIlIA3jBIGf6VxmjNcW0t9DazRfYhNgiYgkkdSD1qnNMLa5uoLO4nmjUARK/VQc5GPpxSA6u11OG/MUv2uFMAFowcYJ7VkeM7prxI7O3nQW8bCSVlJYk+2PT6+lcxdh7N5LSGQTEqGdyoBA9KbJeX8cMCIdx5KAYJIzzkUrlFSztzd3BjQ4ABLMRyoHc0s03lzIsEYlaBjlyMg9gc98Vagu4zNJLNICtxCY52RNvlnoKj0+4gt2Fu4bMykBycDHPIH4UXuTYozXU0oUzZbYABkcY9jUtrama7IZGUJGSTjODjimyQRxxzxPMfMQ5TJ4IPNTBpUYSs6hgBwTkMCOhFIGegQaXbv4UMk+yaRIPMVx95SADjPoOOKv+H4wdCjJ5Jc/zrgdJ1W4tPtocP5M8ZRwo+XJ6ED6DFdvol/aRabBC1wqu7nYHBXcfQZqkwNEwgPk5FWVGBTHPOKdG6kAZ5q7ggIzUkcaMuCOvemHrmnEkDIpA0KI1hPBzmnCRfWoy2VqEtg4ouKxeRxU8ZyKzklORk1bSTtmhDLJXcKqTwSFW2MQTwCO1T+fUiuGANIVjDnguEXMhLnucYqHcR1roSEYcjNZd/bhCGQg5zkHiqvcEiBTnFSoxXoarrkYzUik5pMZZWRh0NJI5JySSaYDTiMjNIY0GlYYQtkYAyabnB5qRoRNCVJypGDj0ouBymo3EMM0ciKkhEhc4PQdPzrEvL9BqgkS3wzckd8Y4z7/4V28nh3TnVwLcZYcZPAPrXHa7p7QTJJDCibRiQoQQTkYI/A0AZ91qHnGMIcg7iT35xnI/pVK42mRCpBJHODWxILM2xfyooiAQVA5IPQjvxjpWDCn75EPBzg+1K4Glp1rJJb3UyOwuI0BAHOV5zj3rQeD7MIxcxSebIA5RuMDtyaNLu7WyyspYSMSHwM4APAP412XiC1jbSk1BwGKAEZHODjNaQkr2ZLRykVx9jLqI90juMA8YB44rTG/7U5HMYxgjoD/WuauN9zdSQ2sJzGCWIII7flXQWmLaCO3LFpAMkDk5PWt4yMmiyBzR/FT44ZpZAkcTlj0GK1E0EuMyTOh9Nuf61XMLlMkHnipVnkVQAafeWM1lId4/dEgB+xqAYNLca0LltKGkBc8+taAulDbe3rWNEksjAQxu/fIGRipPNA4Y4aocUUmbqTKTjPNSbwawYpm8wYNaaOcdaylCxaZYPNNIIHShTk1JkYxUWKIs4PNVbhuwPNWJQcjFQMm7OatCM6RnDgEEg+go2IVJYNk9CT0q452jGMk+tRMAyEkc1omRYihmkQ4ZgR29qurIGGSay2YhuMipYpyOp6USgnqCfc0iaTcRUK3CNjmpNwPSsnEtMfvPc0hf3qM5zRUjJRzSE4pgOKC2e9ADs4qNn6800t71GTQApb3pC1NJFBPFMQZpjHmlxTCpJ4oACc0mBTtpHagKaAExRjFSbSRRsPcUAR5pQaeV9qNtIBuaaTUmygpTAixTgpp4TmnhQBQBEFNKIzmpggp4UUAQBD6U4LipdvtTSMGgBgFPFNNAJoAkHSpYQCRmogeMVLEQDkmgZeVUVRUbyAHGajMgYYFMxSAV5eoFQOxPWpdgxk1G60AQMeaYTUjA5pNooEMFGKdimmmAjHFMzTm6VG7LGhdyAo6mmArusaF3OAKrojXEgmnGFHKx54HuaaqNcSiaYYUcxqf5mrBbFADictmmluKYWz0pNxxQAFsGjdTTSHigBxYgU0uTTSeOaaSKAH7vekJFM3UbqAHEikLelMzRnmmA7NLmo80ZoESZozUeaM0xEuRRmmA0A0gHg0oNMBpwNAx4PNPBNMFSAUMCRWIGM04MT1NRilBpATK2D1p4fFQBqcGpDJzJxSeZmoWb0pMnNMRMWBpM80wE1IqFjxQA4Eing8UgiNP8pgOlIdg3HFNJ4pwU96CvFAiPJxWFrKYmVxxkYreIxWVrUeYBIDyCBTQDtFctbuhOcHNaFYujTBLgqejVuHAPJqgDpSg1HuB6UhY0+Um5NupwIqAE+tHmY70+ULonBFKQCKriX3oMhI60WsF0I5xmmF6GJJphHpVpaENji/HWm7z603DU9FHenYBpkPrSqWJ5NSlE28CowpLYApaCJ0PA5NP34Gc1EEkA4FMKOTyKlopMsi4PrSi49arhD6UCN/SlZDuy1549arz6rZ2w/ezKPoc/yqtes8Nq74PHYCuJmuGkmdiOc4qWhpnYXHii0h/1UbTfhis248W3T8RRpGPQjP8AWucMrN3qNnI61DHYu3GqXlxJumuXb2BI/rUIvJs4E8gHpuNQDJ6Y/E4pCuD1H4GpsUSnZ/G5zUqX0sC4huJFB7A1TOT1NAGe4FFgsTyTNK5Z3Zie5OaAR6D8arkbeMg/Sk3EdDQBYIB7U+O5lhb5JJF+hxVQTEHJ6VMI5pl3Rxsw9QKQHS2Xil44xHMm8/3ieauzeIsw7oAA3oQK4lHYMQeo61OHbHBoA3m8T3pBACg9iB0/SoD4l1Mf8tl/75FYzlgpJNTwR+ZECRmgDqtJ8QG6CpckBz3H+FbvHcgfWvO1Zbd9/kkkdMGlu9TluowgZkUds00wOyMXYAD6UPIlvA+SAwGfWsC28RT3tpLNDtBQHII6GovCN415qFws7b5CScnnFbXJNPS7+bVPNiBWMIcE9zW1FCsEZKjkd+5rnLpH8O6pJd+Uz2jgn5euT+dTWWqte/6VNcKsOcKm4AYpAjH8UQazNAz3VzAtspyEjPJ9znvXNW+n7LUzXRKRn7pHOa1NZvZH1VoZJg9uSBuHSi8vYbua1tYiTGgAJIwD/nFAGTb2El222B1OTgAnBrrPDfhXULS7F1PKsUYwCgyS3Xg1D4dtHtPEQEsYEcmcE8g8V3xNAGVdadFPdDcPunI4qZLVI2yB0qcDM7GpcAigBsZ55qcEEVAevFUr/XbTTcKQ00pGdidvqaSYrGk8YIO7getY9wl1LMY4RlO57EfWprDUJtYQSC3MMOcYJzmtERFBjcQParTsM871qwik1ZIR8gwN565NGpxSO1vp0ICxsQMAZrqLPS/M1G5u5kDc4jJ9qtLbwi6NxIuW7U+YRw+o6F9iiVIm3SE4I45Pr7VJcwoun2l3IoDxkKQByfeutnjSe6DAbVAxk1HdaRbmxdYYvnAyCTnH4UrgeeXU8Ml9JKpABxgAdahjBe4wOciuvm0KyudImlEZjkTkuOpNYz2c8lnDHDCY1T7054GfrWTRSZkzgxzeURwapyAqxU1ZuRkBjKHcDBYHIqttZ+Sc4pFDAcIR696dESrgA4Hekx6UqqSQBxk0xCSoA5IBre0nThfzEySCMxRhlKHnNUEtVmnSMSrhhk+1btvprWmlTySHMjt5cZxgEHv9KTGZj3gMsqkLNITs3EcfUVNdXCSCC2KFCo5BOc1m3Vo9jKI3IORkEDg04FriXzMgEAAE0rCKzqRI3Hc1NZRf6bbySDEPmDJrWsLBbuR7eUspAySp4IrR0XSobe4mEh3r2BB4IrSKE2dQTmRJh9wqMH1FZl5aR3GpCUxKYVHAI4z61cTKIAGJ+ppspdomVSNxGASOlbchnc5bXkDagUVl3naAij1rf/s77BZj7KSWEeHU9CPWoV0+2hu4ZJTvmYgkk45rYZ4iwBOSTgCjkQXOZvfEguoorM2729qSA8jDGQO2MVVF7a2mugrbvcQquFVBkqfUD8KveISbjWrKyEeY8hiB3yeanmkgtfFdqQAoZAhIGADzik1YpO5rRXBd/MCMiuAcMMEfhSvNsv0dRwVwTnvVuWKIyHJwf0rLublDKBHtFuD807nAPsPWixJqC4B70pnAHBqnZ3lhfO0dvMGdeoIwSPUVLIMHAp8qDmINUhjvbdRnEiOGU4/MVzGt3ep3d8lpCvlQuMgEDkDqTXSSqXxjscj61lXttcXN4CPlKIQGwCeetPlQuY4p1kt5ZIy2cZGQOD9KrjrgVuarDMIPJUZVDl2PUmspLWQgOQMHpXOzVEQOBg9amgKlsOCQfTtTJlAlIAwO1LBGTKMEADkk+lS0UTXMe3lAcehOcVV5JxjmtaSSKUrhQABjpjNVJ41UF1GDU3ArkFTgggkUhOF+gp0owwGSSRmoXPGD0JxVIDrbCSS10e1WOQgEGTjrzWlNaTyW2y5tvOhlQHdGcED0/Cq9xavb20KeWwjWMDdjI/zzW1Bf291bhUcI0IGdxxkegrJ7nZGKcEchc6abSE3VjM0iR/ejIAKD1rGubia6cSSsSQMDjoK6W6cR6beueCUIH1NcwuDEB3q0zCpFRdkQ5A7c+9PJ9MU1wARikyQCMfmKoyLumySm+jjiMann5nHFR6mwa9bCrxwWUkhj61UINSYzEBTGAiYrkDNMBHfrTkUu4UnAJ5JPFSRW5klK5BA5znrQBF3rW0WytL248m6l2bsBBnBY+mahhtwoe4AGEOeTiuq0i1mjtAZoYTGXEgO0Epzz+VAHK6zp40vVprVCxiU5jLHJIP8Ak0ltEiRg3FuW3kGMn+L29qteK3jfX5/KlDqMAEHIHGcfrRcXE9jJaRT26v5QEihjg8jNBIzU72+k8qyupFMdufkUD7oI6flVHgqFB+Udh0qa9uReXEk7RKsjnOR/nmqwIwBQNMcVUDIFXY9YuUVIlK+XjadwyQO9U0GQfYZwOtR9OaTQGxeXCS72hI2kFSAMEnjHFZRHzMPQ4pwOY80DHX1pLQAd3cAucgDGPb0robyLTBon2uNC6F1KRPnCnjI9/rWBsyKuC7dtOjsyCVSQMpHQexpg0XbEQzWktjGskDTyq0ZkGQqjrk+360/VbK5sWOns0lxDEfOikORlSDniorjUH1CScyIQYRmMRDkDvn26V1DeINCn061TVrpbi4CDJt4iSAMHBBx+PaqTIMfT9NnNvdXClmmtAjwhhhgc5Iwe1dxPLJb2pMAUM3JOOmRyeKrRX+kahqJurC9iJeMRTQPkMw5wcHrjngUHUZbi5kt7WwkdUODLKdi/hnJoQ0eeTSJaX0nlpKAJA6Ekg7euffNaM9xb6lesJpmt2LZSfqF46Hpx711mp6Ob6K4wiMzIEjBAAHPJ9j71x2rWN1ZkzkRwxx4QpG+4EnPfvQ4jILjTbm2SaYJ5sKjHnr0cZwDmkbUoVWFLSJiwHLuAGA9MjrUdprV/awiGN0EOCGBQEHP16VBFAbp5/LAIAL5Bx+FZtDTLD6jEiiFkU5BDkLzg+tWZL6I2iImnxiFAEE0gOTxjJrKjiadXKoJGI5z1P+NLHM0eYXU7eyMen0+tK47iTICShIcnGGB4JNTuJY4S0iEEAAEEH2pGWF1ysWxkIJ9CPUetPjZ5bNYzHnJJD9cc88UCepq6e11Np1zckRyRQkA7gcnOBgY+ta2hxRavpksU7sSkmJVBwYj2cfyP0rFtxcWUDQ+dGbeQgkA559a7LQdPhs0N3G3zyoAwBwCQetNIRo2puki8m7IeROkqjiRex9j61OGIOaaXA4yKieYAHFaKNwLSyZPNTBsispbsCQAn8avBwSBnqKdrCuPcnPHSmN61JwaZIR2qRjMkdKlWYgY9qhzk09RmmBIgc/d61fQ4GKpCQJjGQalWYkdaQDpbp0kIVCAOpNZ1zO0rEE59KuScqR61WaJRyRQgK4JI5qVTTCMdKAcGmBZBp27iqNzex2sYLHknAHTJ+vase61u6MkJtlVY8kOWII+nt3pNAaOr3aRBIZZGgD8B8ZB9jVK51DUvtBFluVVRQSQCAMdTVXULmHUhCDDcBywBcg4A6ZA6VcFxaWaG2a7cIQAQ4LEg8Zz+VSBvkNPasZpGQqMHb1PqfeuX1u+tjbwfYzFJIT5h3DAIXgZHqeldBCFsvM8ly/mkBC5wQB2GeveuFOju8t5bSzKJInyGIyMHnrng0AMtdThl+0NNa+b5pBG0AlD9Tz/+qsh42e+kQDBJ4GelRSxmFtp5JziiEtJMCXCse56UMZPEpeVzsZlTrg4rs5vE9peaKLeZWQFMBQMhQAR6981xsnnOxCjG85bZ3PvWnZW0IisXdRMrSgugHUA8ihCZJoVtctMY4i4M53AkZJA713uy20WKBGtw8jDlwMknuSapWt1oV5f281pmPyyYyACF6Z6fnUWqa1bvqIktnSdSPJjAOACeuSfyq+YVjrogjRq68hhkHGMiphjFZ9jdQzxeXCciEBD7HHSrgNTdlWFngjuIzHIoZSCCDWZDoNpECJFMmTkZJGB6da188VGWwafMFhYYo4IlihUIg6AVHNaW8ispjBDDBwBmpUYGncUuYLGCNC8pwYpc47Ef1zSSKYWwQceoFboUE5pjwI6kMMg0+YLGQjZGadnJp93ZG3t3khJIAyV749qzkuXPZvyqhGgRkVGF5oSQsBingH0pANKAjGBVSaF1OQMir6cnBoYYppgZcgzGRtGQOKpFTu24IPoa0Z4RuJUkH2qqINzEnJIPetU9DNoiMbR4LAEVYjmXAHT2prgsCCDxVfBWh2aC9jRVwRwaCarISCMVOASOayasWmKzAUzdTyhNIITnrU2GRk03NTGLnrSeTQBDg0mD6VaEPNKYx6UAVADTgKnMQz0oKAdqAGBM9aXYfSlBFPVh3oAiCEdqeE7YqUEGg4FIYzyx3phQdqeW5pu/mmINlNK4qXcKjlVmXKsB9aYDBjNLkUwxuBkOD60gBHU5p2Fck8wDrR5w7GozzSAAdqdkF2SiQUpdSetQE00tgUmhosl1x1qIyDoDUJY00k5pBcshzTxJg1UEjAUeZSC5fE3vUgmHrWZ5lKJwOtAGj5w9aTzAe9Z/nD1pPtA9aALzuoGQahMgHU1UaYnvSeYT1osBMZDmm76iJzSHOM0wJi4NUsm7n3Ef6PEcAHkE+tSSSBIXbIBAOMmq1rcW8NrGHmAJGSAM80AXixozmqbahbjpvP0GKYdQB+5ExHqTikBeJoBrObUJM8RqB6E5qI31wejKp9QtAGqTTSaz7e8mkl2SPuB6EjkVbLimApY5pmaQtzTc0xD8ijNNzRmgYuaKTNGaBBRTuM9KTvQAgpacBRimAgpQKAKcBRcAFPApoPNOBpASAU4Cmg9qcCKQDqM0A5p4Ge1ADKkUZFPAA7U8YoAZ5ZNGznpVhFBFKU7YoArhParMSYAOKdHEODipQAO1AAMCnFhjrzTCaYTikA48mkUc0zfTlIPemA5kBHNZmpQO9q+0ZwM1qZpjRiVGQ9CDmmgOOt3Mc6kcc4ro2jfapx1Ga5u4Xy7qQDoGyBXW2zpPYxv7VRJS3EU4PmnSREEkdKYFrRNWJHbs0hpQOakVB1IpXQkQ4NOGR1qbAHYUhUHoKLhZkR60maUg5pCKpPQQo5qre3yWEZZ8ZHYirBJHSuU1dnkv33MDjpjtUlI0R4kwOYVP4/8A1qYPFFwDhYowPTGawhCznAIH1p/2U4z5q1k5MqxsXPiGaYDBVPoKpHVbhut1L9AazSh5+YH8ajKtU8zHY021S7ZSDcN9abHqNxGc/aCazSSDTCcUczA3W1+9MRjMoYEYyV5rIJbcTnqc9KhDEninqxZgoBJPoM0rsocOOtT20drI5FxIQB0AHWpIbIyyDcjcexrSe0ghhJ2MGHtmgRWVNLVgQjnHsafJNHJmG3tEAPQkc1WWYseQv5Cp4IUlky8wjA7k9aAK02mvCMu4yewGTVdrG4GCIXKnuFNdPa2Vi04lmvI5COgL4q7LrmkwuUe4BI7gEigDiGt3XgI5/wCAmlNjcBN7xkD3rsX1bSZ2zHcqT6bTVDUbj7SAsEIcdyBjFAHLCGRzxGxH0rVttQSyh8t1J9gelXYbQm3yzlCOw6VmzadcXM5SFc575oAms3068uTHOgG45yKk1fRo7KITQO/fgn6VZ0fw5Ja3QmvUBA7A8CtzU57P7KYWw4PQA4oaGcLA8ksgRVDH3raitLooAsPJ9OlQWkcNlMZCm9c5yTXRx63pxjLFlQ+maGgMEaHdBwzwygDvmnR6ZBNNs2tnud1XrrxXChMcMBcHuT/9asmG4uLiRpoQ65OeO1FgOPjkcLlHYZ64OM1csri4sJluoDiQd6rENuIwBj0pHd+BnpTGa2o+JNS1KLyppf3eMEYHNQ6NIkbyLKm5W6nHTrVNmIgyYyB/exxUcbHbkEjPpQSWr4Ri4JjDAeh7VOLea8t0lhQYj6kdvWoLYMWSaWOSWEnAwOCfrWvKTDLG5t7iGEkFkxwR24qkwOztrRbjTbF2JDIgII71rKcisa11/S5IUQXAjAAGHBBq22bgELN+5P8AcPJ/wp3AU38cTyRojTSA8hBkD8arTa5aW84iugIXIzguCcfhRfSpo+jzSwqTIRgEdSSetcbe+H5hop1S5LPcSnOA2SAcVIHWXXirSLa3kIuN0gHCgZJNc7Y63c6ha3L+XGSuc5HNc7aaNqF7IEhtHGe54A+prbtr+HRZLe2kgMcinbICOxpoR0fg29L6ZMo2lxJg5PXvWxNNdSMV3IF9BWXDZNbagtxaECGYDcB0rWCAYAFDAdEr7AGbOPWgpg44x7CnA4FBNAEQjQdFyfelmkWJMAEs4wKCeaSTDx+WRnPHXpQBDa2hMYt5l4BJI9a5rx1fpDbJpdsFBIBdQegzx/WrniPVbjTIR5L4cDBJGSRXATTyXchmncu56k0mNECA5VDwD1qWaEwSAA5DDg1EyjPf86UyEuMknb09qmw2xpjZTgA4p8SkkEggU57gkEY5PenxSZT5hnnpQCZJN+6WN4/vZAA7Gu8upH1WzsrJiY70KHAAwOnc9q4e2kUalbyOMojA7fXmum1DWo9P8Wx3MqOIvLztI5PHGKBlXxTZvZaZawzGNphJksjZHTpjtXOQyMgX90pwc47GnXNyb29llfcS7lgSegNaug2Iu7tozMI2jG8AjOfagTNIR3VkIbu4tURZgBwcHBro2024eGO7jCtGU9cYrndZ1G41FoooYiLeEgMQO/pXUTa/a6IkNgkYYBQTk4wD/OrjOxDVyFYJggLRED1BzWRd3wFwUjOQOp9634/F1lJ5kBhYYH3hyK5iC0F/fzASAQgkkk4B/Gr9qw5CCZpJpEkb+E5B/nVyG4kkRXAO0dz61dt7exiIE0Z3DgEcgClu4rWe0lki3DyeSenFTzMXKYF7qLJLNdXLkTbSkQA5A9QOwqC3tzN4fM7ANMHLI4PPWs/UJZNQugQrcHai45JrobdBYaIkLYSZkJMZPJNUmDVi8l4ZtPjEufNCgEDmsHXrkssCMSIgMYB4yMVq6fcwT27MiFZIwN+48g89awpY0ujLcXOTx8iA4IpOVhI2tK1LS7JYxcbPOIyJ1U4x7+9dGWSSMOjBlPQjoa87VhbRqATtY4IPPFdRpUEcH72ylMkZGHhzkKfUU1O4NGsxVQSSAB1JrO1nVIdLgjkfc5foqHkj1q9cW/nKAgG4HIySBXHeIrO9dzdXhTaowAmSAK0chJGddajJeSElQiE5AzzVSa5cqEU4A6UiIwAYjAqOXBkbaMDtXOzdAWBIAJJ78U44J2k4B96iHBpwBJAAyfSpAsiQEHHbgUtxl7UuBg1Dgjp3pZHH2eOMHJBJP1pWAJf9bGe22iONZrqGM9GcA1FuJIJPSr+ioJtat0boCT+IFMDvkuo2YRibMmcbG6Yp81hYXsrKNiz5wWUnII9q56T55nb1Jp8F1NaSiSIgMOmRkVi2up3qk7KzIvEdt9g0xovNWTfIOQCD+NckM9K6rxTffa9NswygSFyWI7gdDXLKOc1pC1jlq7g3BApdwJwaQhieafG4hYkx7iRgc9KszHw26SKS7hRzjPenyxxjHlKQh6E9TUQcgCMgGPuMVLLctKY/lAVBgKOgoJI1CAHJG49DnpUmFTgSAsOpA4FQHJOe1PhBlk8stgEEg9MYpDuaUOZLRbbGJJGCgg8MCa7O/lSx0SWaNtriIIB3Bxjofxrh9OSe4vLeONiREfMOOCMdal1SUXF/czSzMxL4U9jjtQBSsbJ76/t7aIfPK4A/Ouy8QeH9NiE1/dXTvIsOFUEBSRwBjr+tczol2NP1aG7ZA6RZJBOCeO3vWpqFy3i3VIY7aKWC1iQ8kZGepyR79qYjm4rSSZDKnOSAB/Wmzwy28phkXEgIBGfWr95YXWlwQv5mDMCRGOGCjoTWdIJDteQP82CCw6j29aARYhuDaX0E3lFfLIzgckd6n1nULfULpTb24iVQcnABYn1xVNpRKpD9QOD71Gq7jjNIY5ASNoFHIqa2gaabYrbeCSSOwrqrDwiJbS1vZJTskhLOhQkZI9vamM5cOTEoPQZx+NSxyxgBmfB9MZH41PqxtZRbPbKVk2ESBQcEj/8AVWVnnJ6UhGwk1y2kzQ2TYQHdIVGGIJ6fzph1q7l02O18xSU/dsCgJI7ZP5VBpkzxXi4mMSHqcZGewIrqLO90yfw3dPOlp9rjfGUUKx54Ye/XFNITRgLpt9biG7t4JI7hTwm0ls+uPet22i1n+xTrVpe3JZHJkhcjBUDkgd8HIroLDXNJ1mS1zeFLyFiFD5G8AcZ7Hr0z1rTD2UNuwhRZRCDkk4QHOT396AscCdd1aLUJvstxM8cgykcgLkAjkjI4xTLme6aONp4sojBAhPLEjnA6Y5rtbGwe4jbV70ok8kYCALtEaA8D8ev41zupW8I1FRPdSGRlaTaqgCNR9OtXckyJktBvRYY45CmcPkAew9az4Wk3OiLuD43LnAIHrU9xdRyxqkbA5Azxn8jVNcl8AkH1FYspFlDELsgIyQk5wDjB9qjudkn7xiQw4BB7ZoijmmDiM7mHUE8n6U2YKkghdcEjOQcjFIYsVxNGFWGTOTjBGat2sV+EedE2CFwCXGBknAGO4rN2xbflJLEHjoAexrs7G/8At2lzSSFEcxeWFZurKOo547VSAivNG8uQXBXZFMqB4xzzk5IP610NirQ6XCqFiFyB3wAcCucu9cW4s1t4o2AG0hicYIGCMfX8q2ND1iN4YraTJkaVlBBGAMZGa0i0mSXvNfPWlMpYYJrEm1k29uBJGwIyrsDkqegyMd6gXVp4lCApIQckgYJHfitfaRM7M3iMmtC3cOnB5HFZFhdjUJnWFDtQAknjrWrHblDkChyTBJlwMSOaCagjaTJDDHpUwBPJrFo0TClDYNLtpdlAwyD3pQ2O9RkEGjkUgJw5I5pCAetQhsVIrZoAUxAjAqtIhR8Vej5qG4Tc2R1FFwIAFIwQD9Rmsl9Uikkls5rLdGeCAuT9arXVxqXmTQmIglgUIGMY5yDUxuLQ2Ek8svlcFSe4PoBRcCOeyu3shDGHWM/eLj5gB0OKpyW8UluhVyTENpA7+h9TT9F1aWcrb3VxtjbKoSACT6E1qmKwgkIdgTx8w6n8aTAx9Wv7u6toIXxAIpATyQQMYGDWBf6s9xcTPESokI3ZHUgV1usQW5Uy3LAwyrt34yQPeuHuLZYWJV8oSQD1zigZBFIGuFM7ZGSScetLCTsAxzuJ/KmFfmX0JFXLVQZIQRnMuPwJpCLOj/v7l7Mq2JSCXBwFAHJPoMZreuRDa2kstlIkscZCDH8AwSfqeDzWbd3dlp6FtOgEcjHy2c5wAevWh723GjTIsYIyAwJ6kjGfbr+lAGZbXDqJYlkaKNhuJBx0/wD11q2OmCK3hkvJSYXJcJG2CT6k/lWAGfaEQAl+Dn0rYtrVJEWWRXLRuuUBJBBJzx+Hb1pgdho2tlL42SWyxRk/KAmDjHUnvXVC4B71zsP2a3t0ldkMhBwc9CeoA7dqlhvSWRHIDuMquOcetIDoPtAA61A0245Bqj52R1pBNhutAGispyOalWQ5zmqAfgEGpkl4HNJoZoq4Izml3CqIkI71IshJHNAE5XepBHBGDWU9iomzzwa1GkwuRVVyTkmqTAjWMBQABTgmabuIoL46GlcB7JgZFUZXkST7hI9qthye9PwCOapMDHedXO1Tkmk4PHccEVJd2satkDGf51WRHDg4+U961WxLJSAAaVItwyAMe9Mc4YDsauhAFGPSgRAITnkVIqcYxTsUoFZtjSGeXxmmkEVYA4pjAVNyiuTilVh3p5UE0hiAwR3oAcDSgigKMUBcnAPNAiNzzxUbHI561I4IYgimH5gBjkUwISSKXfSkDFMI5pAPEvFBlPrUJzmmkmmBIZDnrQJKgJNAJ9aYFnzMCmmX3qEE+tOAJoAk83jk0m+owpzS4NFwHlhSZph4pm4g5zVXESk8VGaQyU3eSaQxTSgZphJNKDgUCFI4plTAbkzThbsYyfSkMrEmmHpUjKQeabjIoAYSaTNOIpu00AJmlU8il8sk5pjcHBoEVLi6mMrIrlQDjiq5Z2Hzux+pp9wP9IbA681HnIpDDjZwKAMgHFHQ0ZIBIpXHYVsAUgIxSgMRnaceuKYeKGFhxpvajvS9qLgLCdsynOBmr5rNyQ4x61o5yM00ITNJk0ZpKoQ7Jpc8U2igB2acDzTQKcBQA4DigDmlAJp6pQAgHFLinhTQFNADCKTFSlKNnPSgCOlFSiIkdKPKNADAaeAT2p4h9qlWMDGRRyiI0Q5GelThTinAqOOKkBUjrTaYXQxUJOMVMIc98U6MADOaUnHepGCxgd808AE0zNKCRSGWiV24FRk8VCZMDrTDJk9aAuPZsZqMkk048ikGBQIQrxSAEHNOJFApgG7HWgSAd6aw4qM4BqkhXOd1VPLvSQeGANaeiXBNqYzzjv8ASqWtxkOkg6YwaNHfExX2JqrCN7cp601gAMgVCXC96rvJITkSAD0piLOeaUyY6mqG5t4Jc4705yTyCSPY0xFozA9DSiQ1UQnp3qdQQOlAiRpgikseBWdPrthFkFmZh2Ao1cTfYiYiBjrk4rko9gcb+tJjR0cmvQSwHYrI3bNYbyiaYkNnP6VXlcbyFGB2FWLCQRvzbo+PWouVYc9u4xwefSpk0qaZcifaPQ1cfUlRf3cADdxVO51mVgY44yuepApjK1xZNbSBTIr57g1IkJKZCE/Sq5kkbluSe5604SSr04+hqGwTFEMDEiUZGeRWrpdjpsykm3HBxyc1jMT1YYoE7R52Hr1pcwzcLaULkxvbqqr1ODzWhHqWi2yBI0Ue4WuQa4dvvHNReZzU8wXO6XUrCYgRyoG7ZUU24v7doTC4QkjAIIrixKKDIp70cwXLlzau8xIZFA6AVCzBF2E9KrGfHRqjMwPUkmi4XL0EUcsZJKDnv1qpLHGGIWQH6DNRGRvWgSN3NO4x8SlJAQ+DVxbuaE7lmfPtWaWJOTUocsmAaYGvBrHnP5c8xVfU8CnXDWwAa1vCJB3zWMltJINygYPqajYeTJh+CKANiDWr6EEPcB0PcjNUr6aS4uN4kyO2DTInDjIwajdysucE/SgDZhhzbhZM5P44qOe0RYicj8eKrLqMoAGF/KrJv4Wi2lMt6460hFOFIjIM1K1zNbyfuSUA7EVCXy4OAMHPFa7TW8lttbB9MjpSTGcbbRST3qQhyCTgkdq07nQrlL8QW583Izubisdi1vMxRzknOTT11K9DAi4bI6UwOjfRNSiVUYwuhONqnn+VZ2v2CWcyLCqKCBnB7+tRx+ItUERjM42gZzgZP41lXFzPdzF5mJPuc4oGeheFrSRtJksbhQJEO9WBBOTyORXVW7gxBZWAkXggnFeQaXf3OnTCSGUhe65rUOutvM0rFyexJoZJ6FPpenzf6y3tyT3GM1iTeH4FybHUpLf/AGUkBBrhbrV7uaRjDcSRxn+FWOK1dB1WTTtu6ESq5wSTyPxppgSLrFxbTmxvA88UT5JAwTjvnv1rsIdZ0nVIBB5xiOchJAQapahpI1Ew3toQkycjgYb2OaeYLe8QQX9iscxGPMUcfpTAZq/iA6eVjsrU3GB/rQTj8K4O/lfULk3CxyYJAJIyAa7CTS73Ttz2kqTRqD8jjkD2zVbRJLKa0uLG4IjmlYk5GOfrQBvaI1xaWEFve4DlcqQcjFa30qCzg+z2qQuVkCDCnHOKnDAUAFGaaSMUmT2NOwCgU4OEOSeMVEWctgYx60FI0A8xxknAz3qQOC12O4vdSkSNcxySAZzWdf6U2m3qW7yFkYA5xgCvSJ4owAyIpUHLHHQVy2tPb6zqVvZwTDKA5YDIye3agDn4NMW9ujbwE7gM5J4rLlhkguHhkGGQkEZ9K7fS5I9IiurWaMRzRgkSkcN7VxU0zXFxLM5yxYkmgYJsB3OMntVyGAvGJNoIByVJwarwWclzkjCqP4j0q3Cj28RDHIHcDipsCNGaGbU7bfbWohjtxvL5AJx2FW/Exiv9Bsb3eBIVCbT1OO9dF4b0e2OkNM5c/aAQQTwBXOT6TBJfzWs0Yi8oZUAn5qYXOSGVqzHNLCUmhcxyg4DjnH4U+3024uNSNogwck5I4AHrUcymJmhPVTjNAHX6rbyWfhe1EUu4zsGkYckk88flTdL0WeBjquqsVtwML53JPpmm6DcfbFsBcujxxSBfKJOcc8+1bOt3UOq65Dp4DNbx8nYO/ofy60MRy+l2jaj4hNnC+23kcsw6AqOTiu/Iitp2trexjKwrkAHBP6da5i4urLRfFluyLsXYVIA7np/Wr97dyWjGa2kYSS9dxBOKLiuaOoztc21rcW0ZG4lXTHI9jWRbyst3PbMjYbt6modA1KdNVMLuxhcliAMjPrWxYXE8st0Y0jBVs5YZNFxmLa6Mw1Z7mUCNYiMRk88981kGT7TrNxd3DEpbjgDjt2qxd6vIb24JOWY4yOAB9KzLud3gYJhFfAIA6ilcBou5YpDKGxHccsOwNKxyMZqhJI4EcLADZ0xUqP5pAB4PWpuFiW5XFsx9OlWdOf7LcEK7CN0yeepquZMjY4yKUvsAK4B9SaLhY2bjUntrQvG58yRsZJ4FZjyiaMmZ2dQQSCck/SqzyFzmQ5x0A6CoUkEmdvTsavnYcoTOrSEKMD0qm8ZySMH2FWZsmQBQc4xxTWBhBJwXI6Z6Ur3KtYqLxT1bYcjrjGaiwR16d6ep4FMBC4HHfvSHBUEU3GZKnMKmMEOBkZwaAK5JJwOtbXhmAteXExH+riJB9DWSCEIIHzDoe1dD4fyum3ExH+scAfhSlsXDWSRe288YxSFTT8ignp7nFcvU9VGVrkbTXltbxjLGLCj1Oai1LSPsUUDoWYuCJMnoa2NVsJbTVLe9O0xxgAnHIPY1VvWuL6UW8Yy74IHQAf0rrhHQ8utJObaMF0JwEXkDGPWtC70Ypa28lspZgm+RieCe39aaLKWO9Ns4xIDgH3rYe9ENgbd2AIIOAcn86p6GKZypYFQTw2SCO4qVoyLbf/Eeg9as6lAoYSyOPObG5R0zTHaMwoT/AAgVAyqBgc1LB5aPvkPQEgdyalS2MgBII+boeDW94d0qGd5vtkO+EnA9FPTJP40Ac7Z3EsDyGEAyzAqO5APoKbI7yFI3PEePlPUGtKaaKy125a1hAiiyqgcjPrnvWaoea4Lk5ZyST70wSL+l6XJfs7lzFDH9+TA6+1dTDY3WnaJa2tlcJJLNMczIPug88etc7aytHp5t5pf3ckmyIBsbTnqfUV3Phe0S00ZLczRXDpISzIcgE9gaaEzA1Hw4wv7VZr2S4knjYyeYdoyMY9cDJ6Vq32iQ3V3BqIgBjtoDCYGGdu0Egg9+opmoaVd3urxzG4/dzExlBxiMdcenQVrW9wNMs0tLiRppEBVEAyzjPBP+NNoEziLTwxusGu79vJjaMyxqi4ZiOmTn0qqPD87IXktmQ/ZzMjKeHAxk/rXpFnaqR592iggELHnKxj0qXbaTxO8QQ5jMauvQA+n+eaVirnmOgWX2u+txcRhrUtiRj1XPTI64J4NekpG9rai3jAAQkKQMADBxx+NZWpSWWhw2k0ZB8kiFgijcQepIHXpUMfjXTpchbaYSE4jBxh/qe340CJ7TTUsrOwkW3JwJPMDAZDMRyfyrg9Z0g6Tevbm4DmQCRBtPQn9K7NfE8sd60N+IkhILRmJcgnHTPeqcd7bNdnXdQkQR3MbRRxhc4GOnv9fegDiWjMNwYgySMP7hyOnrT8SwbnPyhuCO+KSYWqSA2qOowMluCDznHtTTJv4JJHcGhAX9L+zl5IzGZJJAFQAZJOent9a66ytb3bF9osYI4YOY0kmAGe5OMk9uM1yli8Mls0T2xzGCBKoxyegruYb2Cw0+3N7p3lXEhCFYxkMcdR6HmnYDN8WXN+1tbobqQwyggLCgCkjp7kYz+Vc5IXH7+FTDCwEcjDuehH41097Os2pw3FvayiGJSGMowQT0IHtXNxySyFYiqyQLKznIwCT61DdgaMiNVyxB8vnAzViMRRA72WR2HGD096tW8kS5eUKVXIEXUA+o4q5Z39vpyTTPaCZbhCsfA+UnjnPTr+lS3cEULMjzgS4QgEgkcE+lGoI63IdUBwQQQuQR7ipp9F1G3ubW3eEp57gRynkH1z9K1tT0uwhkzc3ZEwjPmJGpAY4+XHtmmM5xIg7/ALxCGJAwAQKuQyfZ4XCkgRuCGAyQTwf5Vrp4bv7TSFv7h180ggRODkZ6AHpk1Wu41t7R4p4TbEzEkMckkKAQPx/nQBlSRkTbS+/jIbPf6VeR3jniaOfG0A4HQN3NNEKtEkc527gHjJ5IBOMGokm8mc5VGVB82O49qkbRZMNzci4CO1xJI4LBQScD6f8A1qcVlZzHOjhY5AC5GAoI4B/Kup8KaXDbRPe28zyCZQJAQAUbPQ4rVvNNt72GWOVBmQ5LDqDjANXYhowNAkhi1QKkpQlSsqMMA8jBB9M11qsD1rIt9JW2vTJGNyMihyRgkgY6/gDWngjsapDWhOVU9MH6UAYpIV3EAkD3NT7GRiCAffrRcCLFOA45p5j9qAuKQyJo+4qIg5Iq4AM8iopFGeBQBXAyacAQaUDBqUAYpiCM4ol+ZG9xQBziqFrcytqM0E/Cqx254yBSAhuwYreSeR1jYAgBjwSPWuCuriW9Yhz8u/IUdAa6rX9Sa1uTELpQQcleCAD0GCPauUlle7uRNGuAThm6ZJ6fyNMBiTmG+hWI4wQxPYHvW7DqyBJFFsGbzCzZOR+WK5+GGMqT5gWYZG0jnI7VZhhcQrLCdrSA+d3AGelIDdg1FbuWa1u2VYQN0Z6cVz+rmH7UTD5ZHBBj6EetatlZx3JO6ZY/LjIJA4x2zWAbcy3csaDiMkD2GaBkVwnzIsY5IyBV7RjCkkZcEEEsSBk8DGTWfMwjlEYOdmQfUZ4p8LiJ180kKQeh5OfSgC7flJb3y7Zy0LMDk9S3eqjf6oAj3Pt2rWsrVbjSoTJIYpImdiSmeDnGTms4x3FwyRRqCZCQoHGcd/pxQBXU/MCp6Vr6aZGnhgd/LXIZmYAng8D3p82mQ2FyriVWxgBADkHg557VZE322O4YIA0eCx6AgH9KBGrdw/Z7a3wGMaSnB3YJBHpVq0DG+81+JNuMdgPSs6zhlv4Le6UkqjnMZbIAHQAetaNvZ3ouvOEn7knJDDk/4UAam4ijcRTJZo4igdwpc4UHuaNwoAlEjetPWQ5BzUAcUeYO1AF8TZ71OkvA5rMDnPFTI59aQzT8wnvS1VSToDU4cEUIBdoPFRPGQSeDUpIHemswI4oAhBA61Irg8Zqu5+Y0qrkZzTQE7YOc8iqU0bbiVOR6VMS46mhRuPNO4WKS8yjIIxVvrT2jI+lRFsNTvcSQ8jNKFJpUIPWplAIpXGiMAikeMFcg8+lTlBiqzk5IyaQEOfWjPT2pDnNJ3xQA7eAKA/OaTApCMDikIkJBOTUTsADjvTGYio3YmmMQkZpMio2J5qMljmmIkLgGmFgelMOTTTmgBxNIDzQKcFOQaAFAJPFSJGcZoUYNSg4pgAQ+lNdSDircZTHNOKIeeDQBmNUZXNaBhUvgDrR9jU9CaLgZpBoANaZsAehyaiksZY+cZB9DRcCkBipVjDDNO8mQdVNSohA5pXCw2NAvWpwRjFKkYOM5qwIVxwKLgZ8sCkEjqKqlMdBW4LYMMY61VubXyjyBRcDLCk9qXYR2qcx46UjKQOlAEBBAqNhk1KQfSoyDTAzL0hbjnuBVbf6GrWogCRDjkgg1SbAHFSMcWHdqb5hAJPpTR1oYEg0AS6ZE93JKHdioBIOTx+GaZCxKDdVrQ/lnlHqP8agxgsPQmhgJg5p/bNGQBRSAaR82auRn92D6iqjHAzWhaxGWBSBwKpMGMFKBVkWpz0p4tT6VSJaKoWlxVsWp9Ketr6ii4IphG9KkEbHoK0FtUUckVKsMQIGRmi4FBYXx0p4hf0rTEKjpg/SnGLAyQKQGcLdyOlIIGHUVqCIEZxSiJfSgDM8kntUqw4HTmtAQA9BQYsdqVwKXk+1IIucYNXdg9KCvoKaYFJoiBkCo2BrTRQTgjioJbF92QRtPrVpktFA5FKGyattZKRgPg+lA06UHORj1q7omzGp92nc08RFBjrS7cVmywUDvQSM8HNAWnbRUjIjTRwalIyeKiYHdQAsgcDIFNUOxwBVyEggBqRgEOBjHtTuBXKkLk0wOOhq22GUjFVlgHmHJOPrTEIWqEqxNXBDH2JNNMYBxQmBlapAWsmIHSsWykKXabe5wa66aNZbSVD3HWuKjYpNgHBB4NO4jpWt5CcsDikmnSCIAwg+4FSRyTTQAhgM+tO2EqRKAT3ouFjLe5SSNtkbBu1JbyyBPnHHrV+WGPYQBg1UiOVZXAUjPSpuFhBfwwv8APnj2qvdapLMCI3WOM98c1h3V2ZZnAOMHFU2OCc0czCxsTOVG95vMz71lttJJCnk9aFbavBGKQEu2AMmpcmVYdGQrZKbvY1cilZVyIBzWczupIzini+uUAVZTj0IpFWNa3S2Zi80gjPZSar3caGbMDh1Izx2qpFdMXDSYJHcitm1i0+/jIkmKzZxtBwKLk2MRnCEg9RSxbmGSK07jRUt7gRuWIY4DE4FW20GdULCVGx2xijcLGBdSAqAAAR3FUy7Do3NW5hGWOWIIOMAVWYKBxQMaNx6mkJPqBRuq3Cr7Ttty4I5yOaAKYPOMj8KQ5AyatpZvIWeOPZjnB4qrKxztPUUAR5J69KUY7A0oQkEjt2qzaeQHxMSPQUWAaIjJHhFfJ9earlHBxx+dacrsELQOQgHAJ5rLZyTk9adgJTGcA7SfpUZBB64FMEjg8OQPQGnBdxAJwPWgCSKVo2yHJ/GteOe2nhAMas3qRWUI17MKcCUGAcUrgaRtERSYxF+BrMSdopiSgPpmms0mfv8A6UhPPPWi4CTTmSQsEC57ChJihyMj3xQcHoKaCCcHpSuBce8WSPlRn1FV1cE430wgEYAxThC2M5H50IDGLh1DE800HnNRg5FWgq/Zye4qgIC53DHenElcggc0zaSMgcVIqF1GAMCkMRScACiTIA96kEbdgTRLGTgAEY9aQhIYiYiGH0q5a3BjQRkAgdM9qiRztAPamgBZSR360AdLDrlxBArC6DbePLAxx7V1em36ajaCVeucEdxXncEwiYFkDgdRWjZ6vLZys9sAgY5ZTyDVJg0d2QM4PTuKp3uk2l+mDGI5B0deoNUtM1tr0iOdFR/7wPBrXLY6VZBiLe32hSpDeL51oTgOo5A/ya3IbmK5iWSFwykZyKZKUniMUyB1PUGueuoZtCnE9tI5t5DypGQOnFAHT5NLmqtrdR3tuJoW4PUHqDUhLDrTAsBwqlicADNVLdTczm7lBwDhFIwAPpThNGOHOR6DqacJmdCsSEjGMngVLQGNrM1yumOVYkTybR6Ae1c3pdpJa+IYYpBgg8H1HFdhqdtiyhGc7Zcn2FR3FvGl3LflQSkfBPYilYZz3jK63zpa+WvB3b+/TpXKkYHHereoXU17evNMcsTjPsKijZYzvaMPjselSUSRy3TwLEHO0HAUe9a72tzaCKO6ZHWUcKOSPrVM3EUgjcAKQckAdK1rq8j1CaHELmOFBkjkE/hRcDWsfES6XpospYXmmAIiWPnj0JxxVK/uJpkgkdUgvXzgDqQfWsaW7ksbxLuGMEYIIcVFBfXF9rVtPcHIZ9ox0Ge1MDodEtmEkl+5wVGzZGMk++a56/gKatPvQr8xOD1APSukkvE0e9uUYHBBeMDPJ/ya5m5lmuTNdMjHeck46e1SwGRyFZgw4xXRaTq1pppMpjcsRyVGcGudhi3jOT+NTQuQMxuQ2SOKm4yfWESbfqMheOSaXManpilnuZC6MWBYqBwetPOnXeqtHmSPy06knkD6fhVk2ltY6tb/AGWMyKowc8gn1oTE0b3hJIW0+SSRQZN5BJ6jiplnkt01GWMgNjIyMjNS21jNEMxL8r8gdh9Kt29lHI0itIXUHBBHBNWkFjzdre7dyTCw3nO4jANFxNGyoIxwnUe9dl4iUPCZlYAQnAQ9SMVyFxbILWOSMEbhkgnOKljMsuSxYnJx3pqSkEBSRn0qSaFwp2x4wMkk8VTB4oA0Q+4ZJAGMZqpIgWYkklQcZJ5pqHGeOP0ppyW5poC0hjZlRDkk4znpUgGxzHgAr6d6pZVSMHBoaR0kDBjk+tDQFsuQSQSD7VEwaUkFyCT1qBpmZ8569RThMM4LYH0oQE12tvGFEPzEAhieCTxzVZhwg9BT2UlQxGFPT3FMIpgOOC4IHGKCgfGTjHSiI4kB64GaGjkkJdjye1ACNjOO1dPp8Yh0S0QcmQb/AM65sRhUUsRz6V1FnNDdadb7JlikRduHGBkelTPY1pNJ3Y8ho8b1Zc9MgjP50+MebIqDqSKvx6hfqu2WGO6h6EkZ49MioL26ikeS4t7fyFjiIIxgE+tYxWp2e1VtC0q/2leSW92qqABjacBxTprZItYhEKBTHbnBHbniuZS9lE0V6jASJ1JOMikn1+4lu3uC4jcoI0xyAK7lNWsebJasJ78ROZmXzJgxycck1Tt5nkd3lVSz9TiqxfBxjqcknqTQszLIMVk2CLsjRlidisSMHIzVCXJdjg+Wp5IHA+tTRzMZwwAyW2nIzkmn3KyWd1cWp2uGxuI6e1TcBBcyYyGwx9utTpNdSxyJHLIFI+cBsA/WqbAkAqOM5FSSTCOONU4kHLH+VSNCpDmAAPsKkggnB60RxEWzyFAVQ4JNNZonUnGJGPX2o+do2QbirYyB3ouVcgfBxkZGeRnqK6rS9UubDSoxpswILHNuULEk+/UVz0URLrGEIz0BrqvCl1axx3FvuEbK2SGOMnnmrTIeprWmq6jqGUaxNjg4lkbkgegHrWhELewG5TI8h6t1djWa2swPO0yTA2yId52EjPr9KkhvXuIGa31CJwTkBY8kD061YkjC13UdQ1K/a0hhnsYxGXlWRsNIBzyBxj/Gs+1sriG3gununjueZERnOQo6H2rqJLjS1ZGebFzCCpaY847jPQisDWr21W4ea2mSaSQAFhyAMAAAfnUMaJJvEDXctuJrZDIACGRQS79B/Ol1GyubK2v7ma3twSAXkCEZJIGBk89ayNLt31K4iEcgheEEmRhxnqMf57UatNcSMYZtQa4j6nByAe1KwyrbaiVlt1PmGOFiRnnAIwalZorq5Cxl0t+SHccj/Cs4kJynT1rZ8NWtrqepGzu953qdhDYAI6/XtVIRizEGZ9hzHn5SepFOdStvESijIPIOSTx1rQv7K90+SOO6s5PLQlFcr15ploY45hMTGQuSI3TIx9O1AHRW97FFdW92bWaa3NuhI2kAlQBn3ANVde8Sz6o0UPkGGNPmwRgk9j9BUuk+KvL057CWwjkUKwBEm04PYDFSX9pFqml2N2HWOeUSFgw4wvHUdMY/WgDn/tkvUzzBx0bzDx+FEMkmBKJDgOASxyCT3xUSxhWfJDLjgjpU1rAZLcCNRuJOGHBqWgJ9VdIrtIYtuAATtXABPb3qzr+hDT9Ve3trh3UwiV93HJzwPyrLmhkeRxLv88nGCMk49KvTQ3F3Mslxdk3RUAxyjqPTPTpipsBtad4gt7vSbaxvo2Els6PFKhJLkE54/HFbvhbTo1uNQld2mjG2KMSDIK9e/wBa53w/aG11eKS4IhEYIMi8gk4wMY/ziu9EmOUYEeoFWgJzEBB5JAMY4CkcAZ4FcL4h8OalfaibiNCYXlOxBlioOMk44AzzXa+Y/rSqxJ5pAcVqGkTWU1lPdFSSpQIDwCBnP69Kz9Z0Sa1so7hXJeaTbsjHBBGcgj610/ig5azHoWP6CtiGNDaWxZQSiAgnqDgdKLDKehWQtdEtCYjFM8Q8wdyfU+prSABNL1FApiLEOFAFWUtkkbJ/lVEOARzVqG4AOCaBlkWcXpSmEdhSiYY60GZcYzSERsg5qu5wand+OKrOeaAEJpDzSE0zJzQhiFDmkIIqUEEe9NI7U0A3PqayJVL3YWYp5cWSSW5IPT+taNyyxwklsHoOO9c7rEFy8SOEbOQS2OR7/SgRi6lZ2q3C3UM7yo2TIM5IPXpUV3cpcRxorhdigknqcVs3dgkNrZ3UlvJkLibg4II/yfxrFvreJ0BgZQXBC4HIHHBFSBWugqpbuFZZACHJOenSokYvGEEhWMnDgenbAp8dvdmM+YCNw2DJJGfc9qrNCYkhEnysWJJ68dqpMDZmtpEimktGKiMBtvUuSeh9KyrSWaO/890JCEnB4GetaWmTPaWVw+0xx5A808kjHTHfr+tZUMpMcpDcHnGOhJoGQzHd+8dQGckkjoQTxVjHl2YBQFn5UnqKpSJJwTyC20HsTVyW3f7YIlBMmcDJyMUCLqXcr2X2ZECKPv5GSc8g4qna3axzeYZGXYhCkDOCa0TBNcQeSU8uYlYwQeMk461Z1ULeN9gsgiwooIKpgkrwR6nJpASR3om08zMpKqv7xyMFyOgHr3oTVreWz+zpblVGDIT/ABAZ4x9cVnyQXVnPaS3MbJbykKEc8HHQEfjmr9/YRh7qSOQSNIAMAdDxk/Tr9KYzOe/M4CxqYRnGEc4J9a6zw/cs0Lwk/u4idzls5J7VyccKR2qyxku0UgLAA4JPT+Rq7YTsNOuEUsDO/G0jI/zigRrIfteovJPzDb8pu4BJ4zitWO4jnJEbZx1ODimaboRh00NvJmlUElzwD2rRg06KFNq8HJJ+tICuFJ6U4IfSrQgA6UoQimMriM1IAw7VYUZ4NPEfFIRGoI61ICR0NBRhTSGFAxxc560ocAc1Fk96Qk+lMAZhnOacr4qI5pQaAJmYGkEmDTO1NoAmMhIxmon65peaQgEc0CBSRg5qVZCO9V6XeAMGgZbEmRwaY5yKhWUYoMo9aAEIHWmgc0Fh60Aj1pCHAUuCaaDTw3NMBvkk9qPsxPapw3rU4KlMUgKL2yYAAzUJtQTwwA9K1PKU9KY0AAyKYzJNoc9zUiWAIyTVwrg9KmjQY5FAFEaco6GmmzC9q0wFxSEoaBGYbYg9DSi3NaQCk0vlqe1MCgIAO5pwTHTP41YeMjtTQpPalcCHYcZxSgkVIQRTCBTAej881YyCvFUs4NSRyEcE0gJZIwYzgVUMYHatAHK1XkQZIxSGVwMVIHIph4OKaSSaYFkTADrzVS5kMr45OPSlBNOCgGkIq+VITgAj60GFx1FWycUFs0wKLQkjBGKYLct3FXGBJ4poUg8igDC1m3MKQuSOSRj/AD9KySMiul1yMNpZc9UYEfyrmx0oGMbgdORSLzz1p5AzTAMMaQEuluV1HaDwQc0+cbbmUf7RP51WtTt1KMepxV2+GL6T3ANMCu33sYpe+KGGcHuKCenHekDEboa39EjE2ntxna+KwSMjFdH4X+a0uU9GB/SmgNFLeNR0qQQIeookEiHI5FIJj3NO4hfsyelO+zxlcbcfSmecfWnLKT3oAT7Ko6A/nTTaoTkoCferG8HvR5gFAriKuMDFPBAPIpvmCkyCcigZNvGMDpSKoLZxSKo6k0/gDNIQpIVeBUZJI6UhkHrTTKPWmMYwIOaQtkU5nB60w4J4oQD0OGFWmKmPGKqA4qUMCuCaAGhAGqUMcYz2qMDB65qUdKGwsMKj0qJkBOcVOOvNP2oetFwaKJGDigGrjomMAVXMJ5Ip3FYYQD0phTmpMY61EW+bikMcARShTnOKVWFSBgaVwBAoHNJIF/hPWndabtz0ouBGIwDnJocA9aViQcU3cD1ouIglaUErGcL61x17G8F5KrDBz0rtgUPG6uW8QQiK9VlPDDFUmFjY0qYTWCkfw9auSbXTjqKx/DkrGOWEnkHOPzrYIAJABBHcikMo3A+XFVFTLYJAB6gnBqW+82ZXjACsBwQKxbK4dnJdiTnGTQBDe6PM+oE2cW5Dz1xk0S6XdBQJbcA/7JziuggBRA8hwKJ7uziIdpCB7UDsczMbi1gIaKMx4I5XmspSRXYanDZ31gZYjuAOTx2rlREjPtjYD0BpWFYIxlgGT8WHFXzYmWPEbQZPoeapTyu2UkORTdzRRAxgj1zRYZE6vBIVcYINN890YMpwR3AqMs8jEs5J9zTCcHFJoTLLX1xIRm4c/jUpv70RbPtEm30zVJAO9XbdoiSHOKaArbjTTzVmSENIdnI9abJZyxoZCDgdc0AVQ0iHcByOlbWna+bcFbiHI4wV6jrWSuCOTU6SIi42A/WgDZmvYdRRo4GEZPXdwc1hXFobZgC+7PQ5pjlQxO3rTHcu2SaALlshKZVc81FPGVkO4YPcUkMjRnIbHtSSTM0pyeKYEZkPQEgfSmEZ/iFTo8aKTvIPcYqLzELbiAfwpANKAAHIoD+4FNZwTx0pmaaYEhz/AHj+Bo38AZJ+tM3AdakMfy7hSsALKScGlLjPOBTAOaCB3FFgJlA7MKQoCc5qENil8wilYB546UDOOasWMIuWIY496nuLdYcqCpx3zRYDmFRww4p0rYyuatiPyoPMlO2Q9B3qrIEI3AHPc1QCRn5SnrU8ccseBsJB7ioY4iyFh2rbCeZAhTkgYxQBnbtnfFBl3d81ppbwqoNyqLn05p9/GEtk8qFFjyAGYcn6VIzJOcZwBTCRnNSNtBwSD9KaYwTkUWAFI65qRWz0Ix6imCFgO351PHDgZJAqWAnzj7jMPxrstBvZLi02zkblzjAxxXKqEBHf6VbtJSJBGjlfQDvTTJaO1I70NHHLGY5EDKexrGzNa4eQNj0JrZjmTyBJwV9SK2TJOcvbW40C5W4tCTbtjKg5B+tb1lJFqEAkWaQjuoOADUEt+JZDbwwC4LcHPABrnJrO70m6Xzy8cMh+YxNxj0qgOskms7YnfKgPoDk/lUY1W3GPKSaUeqocCpLWCyECyW8YYMM7jyasqcdh+QpDK0eo2N5mAsQT1RwQTXM+K9UaOQ6dbnCDBLdyOOK1/EV7ZW9tsuraR2IzHIgAAPHeuAcmWQuSST3JyaTKGkYGT3pSmE3GnEFgoHY5qcQNcXcVun8R5qGxmj4S05NS1SRJVzCE+Y9xnpXQXOi3mhbzbYurdzuKNwR7Co9OWy0TXkSNljUwjeSeM+/51vay0l1atDCQFKli/oBzxSYjzzUpHvLh2YCNegQH0qa1gZLEOB80bgg1JaQxvpdzO3zSQknHY09LlAIZiMwsdrgf0qUM6Qvb3U8TBkMxjG0kjIPrXNXgaC5mtzMJMnJIAAB9KXXi1veRzQuUcgAY647c1bk0R2s43MxMk2DwMkn60ybHOyOAxVQQo9DUls3zDI4Heta/sYbf7PZZHmOfnI5Jqld2yWVy8CElR1J6mk0Ui1bzA3gtwQu8cHPOf61vS2P7uJIshhwQD1FZpt/tGipLbwqjDqzdcZrSg1a3SxRZJcXCjBOM00JliS8n08RgZOzqCSTgev50v9tQrBMFctJKSQFGdvsaxBqdw9u7lCxOQXI4FQNO9lp4mG0yOeO/FNgP1K/ea4Bz8wUDaf61nTTuyBCAFB4AFQyTGQ785J601pdwAx0qHqUmOuGR4TGxwTWW8PljaB071PO+ZDz0qEODnfk+mKaExhbaowDjNKZMx4B4PWmsRnB6VGSAuB1qgFJyaATuBPJpv1p4xgj1oAaT1wOTT1BUhh1HPNNwN3FBfBx2oAk2uxMhIPrigYJx2pYwWjPPSmDAGSaALMamGVHBOMYOB2prkGUnqDSxyRNGSSQQMD0qLOFx1PrQBZjtzNIkajLg5C9jXR6bbLIBvjAwSCAOAa5y0lUTEOWGRgFeua63TBiJPdST9aLCMzXblrK7RLaQxErk7SRnnvWdLrF3cW5jkmH1CjJpuuyedrEwxjAA6+grNTIaixXMThnWJFbscimh13AuMj0okEmcuGAI4J706G2lmkARMsQSBnrilYl6ixlHkIK7ienOMGpwse5geMA1LZQ29vBcXFxHu2nAQjJ5quCpV33ryCcE8imCAogBCnqc0hATksCx6nvQHgeKQkESE/ID6DvTJY2UjcMZ6e4pWAUtuTaD06e1Ml+Q5OCSOx60hZVP3TnsQOKdKQUCZBJPUdqGhk8yi3hjAAMjjPPanPIjQQkDEhOCvcY6VBMN0gOScDHJzT4JFAKlVyBkDHUUmgJmUzSApkTD7ozxk471Atvcb5F8r94pJzngmtQBbgRrHtiQjDeoNWHt8SyloRGvlgYJ++RyTimgRUknmSyNrNKpZiBkclR3B+v9Ke2oXtoPKE6pgAgKgBIPv+FBmjjsBK8aSyNIV2k4IH41QmmM93HIEYKvUEg49adx2NyzhhvIjcTLuVRuYOeAQf1rJ1aC0OpNLbyLHGwBCr0z3AHap7eGa4cRBXRJCSHIwCBz361RuHjP2dA2SgJLY5Jz0oRNh3myi2MSSExAZYJ2HqasWENrdW04ZGEwTCgEgFicA1a0S2s7wXEclwke8ruBOAEzk4PvVHU4bayu9tpcJMmSSYycAZ4BqQM5mkhlaORRuU4IPY1r6FBaXF6TPdm0kQ5ikQc55z3rMS3NxFc3DucxkcY5Ofeux8Daeima4laBxKm1EJBbGeeD+FUhkN5a3us2FxdT6o7iyIAUjIKnqeO/HP0rlgmGIKvj+EgYB/wr0yy01LTR9ThnXCSOxK99uDgAj1FcTJpd+1teuATbWmFYtxgYBA+oyKYio2k340r+0xb/AOigkeYCOCDg8eme9a3h3XWhMdhceUYXBjDZ5BOf0zXeaXDHfeEo7S7iEazRFHUDB9M/WvLNX0uHS9WurW2kMkcGMseoPvQBq3ltFDq8Om+UNqxEZGRuJ5yc/SqE2nyW155cKOwCjDg5Bz1qudSuJ3jM0jEhdnmAfMR6E1JJcSrZoGnO9CAu0449KlgWrRQt5HcSPkcgHOcEjAqi3ni5MUwDYcguRxn1qW3kAlSOPkylQc8gEnqK15rWGxa6MkMhkhjwd54LnuP896VgKMSvFGQt05DkFlB4yDkV31hrllehIg2LrADpjkmuAD+ZZOEXMhkL4AwdoA5P51b8OrenWoFihYknzDv4G3oT+tCGeih8jjP0I5FODUpweQKZkZqxGD4mObqzHpGT+tb0Rxbxf7i/yFc54iOdTth6RH+ZroUP7mMeiAfpTsMm3YFMMgFNySKaVzSEPEmTT1lx3xVYgigEjvQFzSjuMqOakEoPU1mByKkEx9aTQ7mj5gxyaidwTwar+aPWm780rATg5NO25qFTzUofimAhBBoJOaUnIpjcAk9AMmgBSAetUbiK4IlmkIGw5TDcEehpkOr2khYCYYGME8ZqK/vIb2N7ASEGRcrInO0jkUCKlzd3s91FGqARAgMobIJ9fwxUt/pdi8hd4cTAkfuxgkdyfWsLw/eSafNm4kJBlWPBIypBOf5jmuzkginO7Oc/xKaAOb0i3t3t7y1muBIJRgAgggjOD9RXP3UNwEeNk23acMjHIIxgEV362cEUuxYVKEcgjJJ9c1y/iyL7Bd6dcW7sWUkbW6ADBx755oQGHLNs0qGAFj5hG8dSAODUl1pUlrAsxGI3IIjU5BHBP6VNLqdld6gSu5Y3BJ3DGD6fp1qS51KSeCAzFUCRkDaOfTOM0DMG6mH2SO3HSKQkHuc9K3IrD7J4fuNRlmDyuAoB4IwR096yvKR9SsYyOWZQx9RngflWrqUptJRC0jCOM4yRlQCe/wCGaQi9pGnSTXI1K4dUiMYmCHnA5ABP0BPtkUwmO71Z5tNmKw5LXEzDhAeeCeuearxWFzdQILVpAsuQ4JyFXOAfx/pUFtam0uLiEyeZbogeSQHAPpx3PtTA0JH/ALSYx2CmTapMk0j5OB39qq+aYJQAsuQ5y7DAJ9q1tC1K3GmXVvEh3iMbpJAAMnjORV+ztNNeM3X2pLiaLOSDxn6GgZmx3MNnbSA25GSeScljWNcBzcnzmjhaQ5VUBOQeh967CaB7me2iRVEyEs7Dogx3/P8ASnxeHrT7Ul00k0k0ZOTIeDxjp2pgYRtNWGCNRkNuSASAQSMDt+nNdlFJmCMnqRyagnvLe1Jh3KhHXI4H1NYIuNTe9mntyZ7cgAO/CgZ7Af56UgOmBzTgOaqpd25nMCzK8gGSoOTToZZfn8wY54+lMC0Bg1MoyKqiQk9alDkDikBNgGmOBnBoDc80NgnOaQELACjg9DTmBNQkEUwBgMmmHFKTkc0w9aBDwanjwWqtT1k20AWGGcioihxjHNAmBPNKZR60AVWRgSCKYQR1qd5ATULNmgZGwJ74pASO5NKTSZpiHBz3NOVx2NREikyM8GgC1uyOKFcA8mq4fB6mnbsnrSGWhIMdakWTmqi54qdKVgLaSU8ycVACAKN3vQAMwJzmgzADrUTuMYzUJOaYFj7SBTDcc9arFhQOeaALInIOc1Itxz1qoKcKBF/zMjk0nmKO9VQzY4NNYOQaEMklmBHBpEcMOKr+U561IilBg0API5oGQaaWzRvoETrIQKR3JOc1DvxSGTNACse9NBpC2abyaAJVIzTiQckVXJI4NKkmDQApJBpu81KcMMConUgZoAUSYp3mAjmqxJ603caADVsNpE4HYA/rXLjpXSXRL2Fwh7oa5scqDQA1iFGW4FICHG4dO2akIB4I4pmAAQAB9KQEAO27hf8A2gK09UGL2M+q4/Wsyb5AHxnaQcVo37iSO3mHOQRn16UDIuCPwpuR0pu896ZuJY80FEhOBXQeDX/0q7iI6rnNc47YFa/hafydaKk8SIQaBHZPGCCCOKiAjTgIB7095QSeajZgRxTRNipfRshEkakg9cdqqpORwTg1rRy4GGNULy0zuliHHp3q0SyIznsalikL9TUcCxzRbSMyVaS3CLgVTEIaTkdKlMZFAX1rO5YiFycCnuHC9OKcAAeBTiCwwKQikXOcU+JGkOOR70ohIlBI4q8oGBxTGRfYiQD5g/GkNsUByQfpUxJBxQX4IxSEUHcqcUscwPU0y4BzxVYsQaYzRVwakDA9Kz1lA6GpRMRQ0BaY8VGGweTioTMfWozNkmkBbJwM7x+dRmfb3qqZCe9NJJpgTyTBgSDUQcnvUBYjpSbyO9AFpXPrUyMTVHzAgLE4A61k3HitYZCltCJMcFmJAz7UgOpGaepOOSD9K8+vPEupXY2q6RKeycH86yVkuEIIuJAR/tmmB6pKNwxVZjt4NcNa+J9UtJAZJTNGP4XPX8a6vSdXt9WU7UKyAZIJz/npQBY2gSZGay/EUJeBJh1UgVslcGqGrIZdPkA7DNAGR4dnWHUlDnhhiuvkCdxXnlrN5UsTk4APNdykyzwpIrAgjsapgVrxQI2dQAR6CuFefdqwK8ZfoOld9dITbvjpjBPpXFG0jtb0STENGx6kVIGnf6tcwWoxbovOMgmsAXDzyfOck96n1i6V3C28u6EgH1qpZbTcKzEgD0GaQXNyW8S300xxjJwRgiuceRmYkKBn0rbmltScu5x7cVWlkskUtGWJPtkUMDIdnHJBP0ppmfGMmtCeAInmFlKnstZ8iqTlcgHsaLgM3E8UnQ8ilIx16U07MYBOe+aYEm/HSni6kCBAAAO+Oar55xS0hDxK4bIYgn0NO82VhguSPQmoQTmlB9TQUPzgdeacHGAenrUZYBeOTTQ3HNIRMWB7g00kjuPxqPeKcrBjg80AOD5OMilY4981GYwOQTSZI7mhDH4yaRkIGTim5I6U4BpOmPxNMQzBPSpBE5GcU5YtrDcRgntV9fLVAATn2FS3YDOjB3gqASPWrru8seCij3ApxYDoAPwoByMGp5wKQjOcHApskZBzjP0q/IUMeFA3VAW2IXwCKfMBT5HUH8qXPsfyqdb0n7oH4ioWIJJPUmmncASR4zlWINK0rNnJyTUZBPQZFOCMf4TVgOWIXtk9zMWLAZAXoKis7aaSF3Xbgeo612CXdhJZuIUBGPuhMfrXPxQyxWUzgYBzgdxTsBjqTkjjg9ulKCx4LsB6A4qSGBihba2CewqIjDkYIPuKQFywBSQyMQ20ZAc5ya1n1ES/IYQoPdhxXP5IAAOKsLIJB88zZoAnubxhKcRxEdiB1qvFGXGanEixjAwfqKaJADwKljHEgfKQPwoJBxg44pm4E0jMAMmsrASAgGp7WQRXSSN0Gc1RMwHABJPTApu+VhgjAosB1Fzqkbr883mE9EUVb0GwnvFE1zdGOLOBADyfr+lcrZyra3Ak25b1JzXSWBVZw+SNx4IOKtMk7GO3hiTEUagfSq91p0N9btDMvB6EdQaejT5CkikuILiaPyzK8YPUocH861jITRykF+PDd5LY37sIcZQ4zx9PeorvxpBHIBb2rSr6ucZ/CtDWvC0JsmuIWme4Tkl3ySPpWHd3Fhd6I5k2rdpwE6HOau9ybFfXfEn9rW8cEcTRxjlgTnnjj9Kww2BTDwcUmalstInEhAGDgk4rprPTv7NhN9LJvk25yOn+ea5VAZCFUEn0Aqybu6aIQSzFoQeUzjI9M1K1GyzcPN9nku5kYGZuDjgH0rprTWBJ4Ukdj5bKDGTknINcvq2pi7iigiiMcMYAAJzmtN9Nu4fDGGeMIxBKAYJ+tFiTCjknKS20ErmNyQQDgEe9TWnmTSW9kxCIZOcDkV0eiaX/AGeHkeNXyvJByRkdMf56VYtdNS51ZVjjVA4yzFcsBQ0O4eIrWJNJVwvzIwAOeQKsWl2502HZGHMadSeAa0b/AEZLnS5LOOQnyyXDSHJyPU/jVLSbWJdEKW/Mv3SO+c9qTQJGH9keVnvbgB5OoAOBmktrMJIbq5kAJOdmMmukXw/KbJnuLgoAM4AxzXOTWcxjcllMYOM55NJspFZruUgjzHWHJOw9KrvKzIUJ+Q9SOtWJFhRPLKfMPeqkwEYIUfIMYPv3pJiaLtjcRiOS1IJBBJJP9KoxTkXCsQzImQR1FRxu4dnRsEDrVuPSZRbGeaTy0OSR3/GqbuOxWcoXJQEITwPSo3IAOOoFPEyBBhAAByeuahd1JJHIJ4qLDRTJOSecnrmm81MEBPbJPGacYdhII+lUhFUgkDjknFNyQcEcitKGONFJI+bGQfSqkkY2h+Mn0FMCILlN56ZpAeaQDnFAwDyM0xgetIQaccE8dKTigQ9XK8A8HqKRuTmnIAxANEgwB9cUAEZ4I9qAMJyefSlCNt4U/UipUTCFnYDHQE9aAL8ciSTJGhyNvSupsh+7Q+gxXJ6dC5uUkA+Xpmurx5dnJ7If5UCOLvJDJqE0p/ic4PqKZGMzKT0ByRRKXKxbmyoBwMdMmnx7FtC2MMW6+tAE5uPNs3t5OdrFgc9/pUSuwAbJAUYOOpqozkE4PWrcTmFCHAPmrgE9R9KloZA7K2NuRjvnk0sQbzVygcA52nofakgt3uJ/KjIB65JwAK07OwiNhNeTTKpjJQKRgAjpg0ICC7uRcTA7PLAGAO9MJjDpvbCg5FVpCWto3wRIxJLHoQPSnyI6pG8hwHHHrx1piLDlFQFIyhJI2jgMPWot4LkrgkjA4xg0rCRlIG8kDOT2quoYBiDj6UMCyP3cLiQgsxyMHpUQEoYSEDnoRxmnqBNEIiP3nQN3JqWQFikRj5ACkikwRoadHGbOa4lQSbCCVU9T75Paqs1xNbXxlkjG/sCcginG0RAYxK2707VajB+zQO2A3mGMsSMknpxUgjNuJo3cyAYyASPeo/NVHRvvcZYdOK24LK2nsL8yRgCHJMncHsPx9K56NPMmSPONxxn0qkyrnceHp5r3Q5rcglomKIZOMgjI57VxdxEYLiSFv9YjEH0zWnYXl0+tRwpdoCXAQk4jJA4BqxfaDqRu7m6uIASWLkRsMEY5wfw9KolmFFI8ZJzg9j71qx6mEiIEMZ3x+W+Rkt7+1Saj4durKza52BoQgkLZ5XPY+v1qjFtaQI+Yg2Mccg570mJHRad4e8/T70zPh3I2BDyCADg+uc10NnoOnW4ikERWZOdyMQQe/rVmBSUCgElQAWI68dalmmhtLaS4uJBHHGMkmgCtrPlTWzk3GxQCUjJwJGB6+/XpUEtmIrC7sLggQ3oDiReCpABOfpgc1yWuaxHqV+GsQ4jCFFLjBz1JArpPEkl1f+HybGRBapGHmkBzuAGNo/Hk0xpkWheJkttEnF/OWkhLJCQuQwA4GfWuFkaWPzFkzukOWJOSRVwR3CaMYJrYrCZQ4lHUEgfzFX/DemxatqiSO0Y8liGDtgyHGRikM57OML7VoaWY45JHmRHjwAQzEY9x69/zroPEfhpLSyuL7exmL5YqcRxg4AAHfPH0rmrCxfUbmG0RwhlbaGIyAfpQB22i6DbyXEWoB4pY1+ZFj6bu3HbFa2s2kV5Zl5lB8oEgY+8TgDNO0dJE0uETxhLhR5chAA3Y4B49qvgA4zzgg0AY3hrw5bx3l1JcQgGOJITH1DEgEk++eOK6NdNtoZ/OhjCMECLt7KOwohPltkcE9ferPmd6VgKcoIJzUWCTjFXGKSDk81GyoBxVCOV8RJs1m3XOcQA/qa6JFxGn+6P5Vz3iE7tdi9oAP1NdKSAqD0UfypgMCkdqQggcipFYd6bcRecgAOCDkc0IRCWpBUJMiHDqQPU9KerkgAGrsTckxxR3oyQKaTzWbRY/JpwJJqLNIWIPBNAFpTipk5rDv9bt9M8sTlmaToqDJx71z9x4yvS8wijRYSMRgggqPXI60hnZT6rptrJ5c15Ej5wQSOKJL+zkJiS7hZiMABhz9K8qkm81zI5yxOSfU1GZEpXA7S9gsbB/3gwzZI3cqOwz6VkXGrPFOFExcIQQyHgmsRrp2wHJYe5zTCfMOelK4Fqa88+Qu5GSSTgepq5Y67e2TxiK5kEan7vXisVwcggUisQwBGM0gO1h1ubVr0wB3VCMoQcEEdc1DqUpikhaSRppIjghxwxPp71zELOHBRyjKcgjqDXQprEVraxpcQJPcEZdzyAOxoTAoORbXjyOg8yQ5XA4Q+mD1pt7dG6HmPCFkGASg/MVNfaol+GBhRSVxkZyCMYx+VNs3iurb7PKCZEBIfOMk54NO4GfMxhmiIbJA3A+hqS6u5J7RIycqSTvPVj6moDCzXiI7nAIGTzjFNBeecAdA2AB0A9qdwOnsJ7cadNG0jSPwxJJBIAwPw68VnNNcQ2VwkcR2yuE3Ec4HapRZzQ3MSRptEwClc5yf6VNrUD6egQyclPMHOcc4x+tAjJE7QxGOEHa+C4IxkjOK6PTbyP+z5QQsSgcYGC5POB9K5m2jN3cxqSMA9Cep9K2tD06Wcx3MkapDG+VyeCR1P8AMUxnS6UJLaJb24MhupSMx9MYzgfrTjqgjt2RA/nSOSWJyAM/5FR3GoWszAecY3IwhPB/D61nS27qyvGUJOdyjkgUxGgLiC8upXdwDzgHoDjH9DQkN19jMcE0IhBJJZCTiqk1vemMfZ4UZSATnHBPatWwjeG3CSEkj1OTRYLlaDTo4r3zYXKSKQXOfvnuAOw/xrYzmogBnNODdqBkganhyO9QE4NG4nvRYLlxZB0zSmTjrVHcfWnBuBzSsItGX3qNpU7nmoy+e9RuFYEnr9adgHGUZxTvMX1qsQB3pN3vTsK5a3jb1pu4Gq4Y09eRSGTZwaUtwcVHn3o59eKQxjuc0wvTnBNQkHNOwhS/vTS+O9RnOeKaSRQBJv560u7A61CDk0pyD7UATh896USbepqsJCKVXJFAF1JRjOalS4UnANZwJxxQCQcg80AbHnDHWopJ/lODzVETHGCaaZD60WAnEsnJJ4FPMpMZwefSqZkOCM0wSkdDRYC7GSwJPWpNwUYNZplYdDSGVj1JosFy89wAQAactxwATWaZDnrThJ70WC5sJMCM5p3nCspZ8DGalE2eppWGmaPmLTSc1UEwz1pTcKOpp2EWCajLEdDUD3K4wDUP2j1NFgLRkI6mk833qo02QSDUfm0wL4l5608Se9Z4l96eJfegC7vzxgUw5B4qFXzUgbIpWAmSQg8mptwI5qoDzUgekA4oDzUZjJ6CnbjQHIoAabdnRx0ypH6VyY6Y/wA9a7SFyWxmuNkKLPIhbkMQR+NA0HakIpxxTSR60gKtwMRmtC6+fT7dh2AFUbghkIBqeEyPpWZJAwU4AHYUwGYPfFIBzmmiQdzT9wI4RyPUA0gHcVNpMpi1q3I7nH51XKzMMLDIQe4U1YsLC8e9jk8l4whBLMMACqsI7cggnPrTCcUwzEtnNBkGOaLAOHXFToBjFVkddw5qYOOxosBKY0ByqgE9TSdKaJBnk08EEUgADNOEfrSEgDIoWZe9SA7YPSpAmBTfMUjINAlFUApA7im89jSO4I4qBpcHg4oTAmJIGW6VG0oAyKga4JBBNQmXPQ0APkbcDVOTIJqcPnrTJRkEimgK4kweTTxKR0NQlSTSyGK2iMk0gQAd+9MRP5hPWlFcpd+JLgyFYURF9R1otvE92vEixuB0yME/jSGdcsRbGBUwt2xyK5NPGdxGCHso3PYgkUh8dXasM2EYT0yc/nSA6swg9qikiSMb5GVFHcnFUk8QwXenSTQHy5AMlWPIP5VxF/f3d+5e4mZgOg6ACgDe1fXI5FNpZkNu4Mh6D2rCNldLywAHqDmqSOU6VeTU7ssAQGz6ii4FmTQrlLcTC4jIxkgA8VnsZImwTWjLc3UYHz5UjO0DAplxPa3CorPtYdAwwPzoAgElwgyVIU9wKm0a8Nnq0bh9qucMCMZFaosIrjSo2yFYdCemao6yJo7GJJo4T0+dDQmB28NzBeAmGVXI6gHOKSeImF1K5BHIrzmyupbJ/Ot5jG3sM059Sv5JjIbybJ9GIFNAWZFKTyRkYAPFdhpqC4slcMFbGCcYzXGJyAx5J6nua3NIvZVjMCn3FUwN7LIpRjx6g1wuteZDfGN2LL2B/Cuimll83PmEMOmKzNXkW/hEboFkTox4/OpAy7ZHEJcxAgAnBNW49ciSHmFSw7bcVWSC9trRywDwgdjzWQCSSffNIC3dXX2qcyYAz2HQVBmTshx64qM5ByCKkNzLsxmgBC7A4PB96VZMrggVEXJPJyaUIW6c0ALwwOSB9ajKkHinEFTgjkUnNMBVEZ5fOe2KaRz7U8DJxTXGw4yaADPakNS+XHsB83kjOPSoSOCaQ0HWkNJuIPtTwuQDnrTYiPODzTgTnihwAevPoRRg4yKAHAnNO3gds0wAEcn8qM4PGT9BRYB5IJ+6KurZoYQ4YnPYDGKpBZH6IT+FXbW9kePyQinHSgB/2eHGSDke9RmMjOwFqdKWXlsAnsKgLuBkGsWgJCXKnKkEetMMuBt6UnnykEMufTFRlS1NLuBYiQOQDyDV3yIwANvPvWdbsUkAJ4rTySMmqsgI/IhHSNfypfLjHSNR+FOooWgCAKBgIv5UmB6D8qcFJ5ApCCDgiqAq/wBtyBNsMEYPrVaa8uZ1IbAU9QOBWerbTmnGZyCAetAGnaxE22RKYxnAAGcmonspAd0u4E8jI6itTRXeayIKgbDjI703US0Yw2CScAg9KAMyKNmyIo/MI6irUJt1AjuoFEg96bbXAtAGJwD19annubOVS4Ukn+KgB08duYC67Sw6Aday5tgfDEg+1T+WhmzE24ntnpVn7HP1KIPqc0mIy2VVX5MAn1PU0iwkpud8n0qWX96WLAbgcDFNDEKAe1SxsaFOfT3pAGHfcacz8YA/OmNkckn8KQDgGJ6VqW+pSKoR1IA7jmsuNue9TeYR0oQHUaRqka3qxmctvIGXOAMV3Lxl+VGR615TYhbq4WF0AB7jrXSJNdWWDYXrP6xyHIIq0x2OvELfxDC968m1uGOPW7pIiNgPAB9q6m/8WlrJ4bqCSC4YYCqDgj1z+NcQhJkLE5JOcnrVpiSIvJL3GccUy4CJ8uOatyDCk1F9ncQi8dAY1PfkGpRVid9OuLeCOeAhmbqB1Aq9otnCwmS8izJz8pGCPoaTSb6KS62XBCgn93k4APpXaiAmFQHiDkZBCjkU0Jo4qPSTLqCeXC5jUgsDzgCuh1aMTWoAcgIQMDp+NNhZdPkkMkuSSQyA5GfUVViuY55BGDITnJAGaZm0a2kiOO3/AHeTJkAknqMVZhmRLiaSSRAzjAI4OapQiR3CQkID0UDJqa5thbwpIib7pzhQTxn1oKL00kdlo1wJJFyQe+O9Zvhq5RNOuXkYAAZBJ5PrXL6wlwl06TXYlkJyUXoPrVMO7x7FaQkdgeKTYJnaxazNf2TQtZyeS2QXQ5yKx9TvZLUiGJCgIyQTyDUml3kraV5C4QxkZIHJBP6VH4ot0huIHVh8yAEE8jpUMLmE82AS5+arKlJdIAAwSQPpVG4jDfMasWyyxKu5QyE4CnpTSHckeGNLdASBjuepqO4ucRgGRyD2JqS63XFyscQztXJHp61TuwWMMQ6cDPpmgdyNiSvFGVKgbcYGM5q3e2hsbiFJGBRgOg6VHcCAWxaNiWJAAPUUNAVCdr+o7ZHWnls8nvTGckAEjA7EU5SjDJ69qEBKHUwkkAEHgA8moJA0uGJGMYAHagMCuSRmmF8UhkTRlRkioiMmppJCU4PeoSaoQdBSd6UAkYoIweTTGOVsHNKWJNLDE9zOkKDLMcCtMeH7rvcQj8TUykoq7dgsZgdvXinFiVIrUPhy4HJu4gPUg0o8OzdryIg9wDis/bU+4+Vk2jlmKjqB0/EVsarL5GjTsDgkAD8TVLTbEWAwZllIOTgYxVrUI/t1qINwQZySec0e3p9w5Gci2WQOOmMCgyAwhB0BrZXQtvS7A+iCmnQAxybsfgoFHt6fcfs2YZq+l55lsIWRQFGAQOc1Vmi8qd4gc7TjOMZqaxPl3ADRqwYEAkdD6itTM0NG0ua/Z3hkKSIwUEDI5zyR+FXNa0240h4obgwTqSSpTjJ46g9K2vDOr6fpmiXAVM3qtkoB80hJwMe2e3asXULC9ja4nvow00imXzN2cDPQfnQBVsrq3HlR3aLGsJyMJkEn19KLrZcaTcXQCvMLrGe4QZwAO3NNM8SWwhktg0jgEN6j3FJ9ntU04Jv8u6kmwQegT1NAiORHNujvHMsk4wjEYUn0FVY1Ku0LDDjII9627w30mmW1uFhuY4T5iPCclR3zWK2JLhpicFzn8aALemqIb5yUJbHy46ZpIpbgXT7CoYkgk+lWNFspLy4eMXKR9iCPmYeg5q3qlnDbz2kSEGOMYY4wc/nUsZiIBJcgyiR8H/lnwT9K6G4srKFILoIRI43AE5BPof8AGq1lCkd28gBHlnABGCB3rbvYhe2UM0UWPKmwQ3IANKw0YsY0uOOWTUIXY7wpiRiATzk4/L1qBdIhudAu9Rhk2R2shRSFy0mSMZz2AIrs7TQLcXVubiASCBCCxUBWJ6cd8VbXQLKHS72yiDbLjLMCcgHtinEGcFpOiPfaYb9XxJHcKqIV4Ykjpz7/AKVr63r1xbXs1vbx4hRgJhIOHPcfSt7wWVXTprWSPLQTZwR0yOMflWrd6TaXUFzE8YH2gguwHJIORViKd/bDVNJhhUgRzFDIO2zuPpxWdP4btbnVZLiVd0bAFQpwAQMAVvJaiCNY4uI1AAHYfSnpGec8UwGZAAUDAAAA/Cqt5bR3MWHjWQjlQwyAfXFXni7ioiCD0oEefXdlFYayJp8LCVd0yeHI4HHuSTV7wqkl/wCHNRswMK82S3oCOf5V1l1YW10sck8YcRA4B5AGO4rn/B9vIlvqFowa3kmAdCeDggjIqgMfWdRMdjZaTZwjy3AZ435LHPHPv1p7+G9RsIIZIkJuklMhRACIyOQc9+lN1opa+KYoxDu8poQSRyQOpA9816HMnzg+ozSAyZS+vaBbLcoI1lKtcAnAIByemMZx+tcvoi2EOsxSQgQ263TFJJDgEAdMmuyvbaSXTZ4LdAXdCqqTgHPUZ7Vwd7BNpF0rs0O+GTcYAdyoT796TQ7no3uOQeh9RQKz9H1caxYLMVCTLxIo7H1/Gr2/mlYROpOKtDBUfSqiMCKlM2EwKGhpg4UEkE5qEsSRSeYDnNLuB70IDmdZ+fxAfaNR+tdEX6fQfyFc5ftv8RTj+6FH8q6E0wHB/eplkwKpZINPEmKQF7AdcEAg9jVe4h5DIAMcY6UCbAxml80t3qkxMCAVqJkPYU/OKoeINW/sjSGmXBkc7E9QT3oYXK13r1paSXMed7wgEAEfMe4H0rIi8WmabEyGFD1I5Ncn5xI4GAevuaa0hB4rMZe1S7F9dGaNWzk5JOSQen5Cs9nlHGCfoKcbkgcmkFwc8nApDTGxoztyMgiphAo6N0OOtRST5HBx9BTVJKc9O1FgJiEBBJBx2Bp4dCAOhx1qsWz0z+Ap0RKliccjA4osBN0IBBPHWmsMoRzkdKXzF7mlMoHBwAfTvUgMAkU9OMU7cQMnHPbFKsinp/8AXpwdT9fWhDIxJzjBFOzk5DGnkpntk+tRTAqA6YBzyM0wJRnOcnNaNhFDIRvYZTnngE+hrJEzcZqzbzSB9sbhN/BJ6YppiOnWZYrc6gRmOIlUAOOfYVi3PnavcK5OC4O0d1A/x5rprewKWkaySpISvBByMc8D/Gq5Flp6AyGMbyDkqAVVTg1SEc+tsY5UiX93cDnB5I9/wFayXsMFtJG1wpaGMBSX5YemMVl6lcGO7+1w2+xZgRHkYAAwAfr6/WqSpbG3JdZBJ6gd/WhjuXrO7jm1eJ5pXihJ4bJJA/LjNdxD9kuIZobQxsqgbwnYnpn3rz/R7eO6mljIkaTb+7SPqxzyBW1p99/wjhvodQBjvfLHkqBkEYJHP1P86cRHUxl1AUrgAAAZzTyTWXoF7c6tDNe3MYRCQsYA6gdT+da+0YrQkYGOacJMd6a0ZzwKZtf0qWO5KZM96BIB3qE7h1qPeTIUzyBnHekFy00g456nFN8wVn3z4tC534BGSPc0+GYTQiRR8pJAP0oC5a8w9qesme9VQTnv+VSpnFMB7fepyxuT049aTqeanSYBcY4qr6WJtrcjZGTrj8KQMR0q3EyFsnnAqy1rblBIqAkjJNSykjMEnPWnmRdvJp1zCiEFR2qqRmoZQ5pMEjNIWBqFgQaAaaEOPPNMIzS5pyqDQAwLSkHGKsCLjIpDGe9AFQxmlCkdqnEeDTtmO1AEKA46GnRp5gOAQew7mrluQhz2q2MEZAFIDKNvL2Q1CwIJB6itpjxjiq8kSkEhRnHWncDJJxUZarMsRXtVdhzVAMyaCxpD0ppNAhd5o3mm0nSmIf5h9aUSEdDUWKBSGT+afWjzW9aiozTsBKZDjrQXJqIGlFFhj9xNKMmmipBSEADZp6gmgDmpAKAFAIqQE44oUZqaOMZFJsZGN9SASdxVkQr2qUQgLkkUgKgB707FWfLA7U0wAnIoAjiOJBVC68P2VzctMJHhLcsEAwT689K0jCQeKNhpAZsfh2zXIMsz59WxioW0eyTW4bfY5hkjJKls89q1idprPvJSuuWJBxlCKLDLY0XSxx9jjP1yf61zdoBbyywlQ6RzYAPTFdUJQDkmuVkONRvR6uSKuxJ0wkt0ztt4QP8AcH+FIbjH3EVR6AYqupLRqx7gUVaiiHJkouH3ZJyPSkMhJyCfzqKkquVE8zJlkINOMmagBp4NS4ofMSx72bgFsdcVbCHGarQjjOcfjUhJHeoasWmS5xTg4HeqbSnOM0glx3pWY7l0y+lRl81AJgOCaesiE5zU2C5OhYjAz+NPIIHNR/aFHemm4B70WHcczkGomck801plJyTUZmUHg07AOPNMPFNMy461E8oPQ0WAlEmDSmQMMA1V3ZPWs3VNYOmERrFvZhkHPT9PeiwGvO8dvA80jYVRkmuCu53urh5GdmGSACcgCpLrVru/BE8pKn+EcAVXRCwIUA0hDxap5O/zgJAOhqPaw6Y/OhtykhhgjrTd1AxXSQDOR+FRmV175+tOkYopPJ9jTYY5rtiIYi5HUDtUpgNaVz7fStCyXTpISbjcZf7p71RntLy2AM0LKD0NQjJ5p3A6m106xvVzHA0TA4GRgU17UQTmEbFcdC3ArCt9UvLTiGb5fQjIq7Jrr3YQTW0ZkUEBs/0oQDtQje0nTzWVo2Gcqc042VheR+cZxbgdQeazrh2lk3NwPSk8mR8YXK/WgCR2ezby4LgSR9iDkUw31wYjCdroexFIB5L7GjVQenfNJLGMEgkZ9DTQEXlPs3Y4ojbHWrFvdGJNpAI7g06XyZsOkYVz1x0poCxA+YxU8c728iSIeQcVTt/SrEgPlmne4G+0czkOV+UjOazb/R5rtyUkxjruOBVrRtSN1GLd1IaMYGe4rVmZDGSVAPYikBi2VnL5b2k0yHI7HINZWo6M9u7CJQQOwOTWq0xjuN4+8Kr6temQxyhHVs4yDxVNAZcWkXLRgmFifXFR/wBmuGIcFQO4FdpbalGmlCYndjt0rmNQ14XbkRQhB6E5qWgMaaAI42tuX1pYWMT5GCfQ0+STeSSAAfSoCRniiwD3DM2SpBPrS7GI4Ax9aZ9qlxt3nb6Usc6KuDEGOeucUWGIdyHpimFtx55q0mLo+XFEd56AnrUFxbTWzBZoyCemOaQhu8AYAGfWkDZGM1GeTxSAGqAkK85pgYgj0pQexNIcBulIC8l0PIKtGjcYBI5FU8HHBH0pRgUowTjvQBe0wJJII5I1IPtWybOC3YMIVBJxWNpwZLgHHHr6VqySGSYAvnByOKoDXH2QjY8G0n2zXP6jYC0vFeNWKv0AFdMpjmtQpIBxgZrn5op0uAs0gdcjnPSk0Iz5WfIJQj61E0ykYZQPpXR6jbo1gzhBkdCK5yO52LgxKf0rNgmKrwheQ2aYZDuOxTjtmomcM3KgfSkyc8HFAx4LBwa1oiDEMVkA/MMkkelakB/dAUJAPORTG5FONNPQ07DKbk+YasRMdozUcgyc0sR4FNAYQIPU4pQQeBVmGyMloZyenbHJqFYZlHmBCAOtMRuaQhgh3u7kZyIx1NaFxbq6i5ZCFPQH+tYlrqQiCmUZK5wFFack97qEYcIsMIGBk9aQFSb7OVLSR49ABwKogI/HUdxVmYvuMckqlR1NWYDHApBXd74yaAEtUlQBoYVQDqz8ioJ5ZmuSftAOP7vA/CrP29kJDQ4jPbODVZhFI+5FwPrUsBtwMsrKoAxjjvTEjaZcqMgU+QgIQTQlw8NvtjAIPrUsZTkO1iCpyKazg9v1qSUluT1NAtHI3EgCgCNchsjpTsknNMKsOKeJAuAwwaBFq3GTgEgnuDzXVaIjTIAY1jCnBOeTXJQTIjg56VuR6+ltYkjyhLjAA4qkMoeJbl5tTNuWVkh6Y6596y41y1NUtcStKfvMSTT93lngEn0FVcaHzDdEaVZvPtorHbhSeTnOfwp67NgE5YKeoHeobtLRAGtpWJ9D2oTEV7i3FvKyEZI5B9RW4niGSTTUhlGCoCgjOcVgMSxyST9aduxGB60riJ3myxILj6k1as9UlsifKKsCOQwzz61lZNPU47E/QVNwNKTVLlrgziTaR0CjAFaSa1JOuZLy5I9AMCuaLjODVkXIS2IBxkYJHUUXAvLfpcLMkcICgYLkcmmWF0tpFIwwZScDI6VRhkNtbuD/AB8j6ULIMZPeqTCxpLqLor7IlBJyX7g1QZ5Zp/Mdy49SaAx3AEYHrTw0ecE9akdiNt23JPFP+0FwMZBFRkmQFcgAdKjQkqSB0NMRpx3BhhdowokIxnFZe2SZgN4DA5OfWgySA4zSjI5PU9TQM0klkI8qTaZHGPMbqB6VnTxmKUxlg2O46GnSTFwMnoMVFuBp3AWKEuMkgD0NJsA5PSmlqAwJzmgAkMaIepPYe9V26A9yKsiMOSCQOM80xY8sCQCB05oArYJ+lGAKvSwmQHAAHoKqmF1GShAoAfDC88iRxjLMcAVYFvJbtcCSIHyxyWU4/Oiyd4biKVIvMZCSFPQ1p6tqjXdp5LwGGYnLEkYI7imIp6LFF9vgmDAnBLgdAKmuW88XFxtLKXARyeQKk8PwBTMSgAIyCRyRzVn7Hpe/cXUexY4rlqySeptDYjZVubyK3mZvLWEHGepHeoILs2ccoUttLkJxnAq/NDplwwMkqEgYGGxxUqSWaReUkkWzBGM1jzK1rM1sipopDwyvkli/JJqM6zcAkCxJAOATnn9K0rWOKKFUhUBB6DqfUmlkuYYHKSTLGw6hjg1DkuZ6XHdGLd6zdG3ZDaGHP8fIxV7Sb2a5gdpoyu3+PGA1TyXVjMAstxE4ByATxmmXF/aC2lIuEPykAA8k4q3ZqyiJ97mDew+XcM+4HeScDtTJLjeIQq4MfU4xk1XBJ6nNSIAxx3NehHaxzM7/AEO0s77RxrE8YMwJAAGAgHHAHf3rmtV1ZtYNuqkqYwVJYgZJPX9KfZ68LDRjYCIiTDAEZwSTVnQvDdvfabNeXTlmwQkanABGeTz9OKBG7plvDcaLaNDDE1xgRu4UEgDuTnp/jXHX6ebrN3Jj5BIQPatSwtZtElWZJYhcAYaHdkMD3NZTTm4a7dlAzISSOmSeBQJM0/D9zb2l5cyXUgS3MJGPXPYCsE8sygEAE4yO2eP0qUuDEVXrjcD6U+SRbi3AlGHAxuBwPxqQNXQbR0UX7q6uGPkEjCSevPtW/wD2Et9cmUOhRORGOTnGTkfWsjRvFl1YadHpUlvDNa87C4JwOvGPer1uLu30q51aHzQxQlSCGUDpg45zQBMujzIkkklu6sASx3ZHsMDtWzp8cE1kFCnBABIOBx7Vlaf4mQ2KXl3bsRHiOZ0b1OASMcjrXSWsNodKElvJ59u/KOvJzVDQJw+2pvLLHIFU7m8WyQH7Jezf9cUJrlH8UR33iezESSC2BKbJMqQ3qR3xjpTasDLOv6ncaB4iM1rGpjmiUsrjIYj09K1/+EjtjbQyOBHJIu8xE4IFZfi+zP2CbUJlaSR1SOALyFA5J4/CuZhhuNThEky4kjlSM5GCQR3qRHpcFws0YdCeRkgjBFTKc9ah2JDhV6AAZ9cDFOVxnrVIRNmmlRuJIpA49aXgigYhAIIPQ8EU3yYw6OFG9AQpHBA9KeVIppIHWmBznifQjqLi7jl8uSCLIOSM45Fb9rHMNPtBNzIIV3HvnFPKh0wRkGlMknY5pABQjtXBeJbESy3tzHGxkFztODwAAO1egCc7NrAfnVYwQOWLQoSxySRnmgDmPDOl3FlGLmQvGZlJ8vHBGeM8/X866Ik96uyAOAT1FV3TnigCNXIHWlM2e9L9nY9CKY0LA4OM07ABkJPFAc5HNQlXBxSgMCMgnmiwjCnO7xFeH/bH9K6OQ84rmQS2u3n/AF1x+tdFI3zmiwwyaUVGWGacp560CH5xTw2KqXd2llbNPKGKL1IGcfWuci8UzG9LBFNux4TAyo9M0rgdgpyQK4/xfqdrfx29tbsHMLFi4PA6DFWbPxLbBXF2CGlLEc8KD0H0rj7iRQZAgwMnA9BnpTuBESGfYDjNNOI3yXBP16VCSRkggUBXdcKM496koshgRk7T9agkZd2B1P5UAFQQQcjggUKwXqAR70kgGgEjigZ7GpvMDRE7cY6Njoe1OkZHEcmGJVMHPfFMCOOTacjntSlyRkZ/AVExJOaQEg4yAPfpRYZP5gBGTn605WVckYyT1FQrsBI4Pv2ppAByKTQE7gg7weKaHBAzwe4pofKkE00rjvxTsIsLIoYEgEj1qcMrjBA59KoZwamVwQAM59qloCx5aZGRnBoyiMQQSDyMmogW65BHcZp2QwGTgg9KLAbkGqwtpS2rvIjxKSrbuCagvNRkl8h5EG2HiMsM5PqfXHHFZWQOlWob+9tVS6g8siMgEsgIye2DVIGhb3UZL+QM8hdVyVB7E4/wqrDHIWIDk5yeadcXbXk5mkSNGPUIuP0qLeQMg4NAi3p1x9lu47vbnYcgZxz25FW9V1GTXNXWc27DChNkZyQAfX8azB95lHK54Prirlm0lsgukcGYkxpEp5JPqPy4p3BnoWloqaLbiCKRIwuAsgww571ZAI602DTZNJ0jzprnqod1lk+6cZIGf5e1WEs5pYFmQo6sARg9jWhDEUjaKdxT49NuGI3gKPY5NTmwaMcEn61LGihLGZInAHOCRjrmsKG9ja/t5JFKOQwOe4xwa6YxsGICNnsMVwVzfqk0bxJ+8jdgY36g5yAaQ2jR1+eUXEVpGCsewyyEjt71Z8PXCXGlqg4aMkMPc85/HmsrxRdyi+QAAM1uFcHkDJ5xWj4YdZLSZI4tiIQBnqTjJzQFjYC+lGMVJ5RAyT+VNZDnOOKYyMkCjcaCp9KbigQ7camS7kSIxrjBOcnqKr08YoAfJMxBJ61CXOKmKqRigQAik0CK+/PWmk1ZFuueQfzpxtwFJAHHalcdinup6yYNBUE9OfSmeWeeQKdwLS3AAxmneap71QII70ocjvQBcBGalTBwDVIP71Mj4Oc0gLWADgVIJAABmqhl9DTTKR3osBbMgz1oMg9apeac9c0vm+9FgJ5EV6rPbZB5FOE3vThJnvQBVNuSSMVDJCY+owK0O9EiJNBIjjIKH8OKLgZJGKTFW9GuEv8ASQ7gF0JQ4GOQf8MVYa3hP8NNMTRm44oxWj9jjPQkfSg6WCpKTZPoR/8AXp3CxnUYqzJZTR8lcj25qHyZP+ebflTuMbijFOEbjqh/Kl2t/dP5UhCYxTxxTdrZ+6fyp6xzOcLE5PsKQDgQOtOBpBBKfvDHsad5LjtTAkVuamRh3NQCJ89KeI3HUfrUgXEk461YSQEYJzVBEk6kY/Gpo3CsATz3oGXcd6MiovPUDGRTTMO1FmF0TFxioy4zUBmz0pN+etPlYroe43HNZOqIUvbBzwN2Ca1Q6+tZetH5bV+yyDNUkJmiytuwKwL9THrMoIxvQH9K33fDkisTWT/xMYH7shB/OrJNO3Ba0iYdNopxFQ2JJslBPQ4qbNNEsMUlHNJluwqrk2FApc1FIblSCiptzgg1IBITgqB+NJjJkJI608kBevNRoCByRS4Pes3qy1sRnrRUhQYpViXGTk07qwtSPardRk03ywDnmpimBwAPxpPLJGRzUspMiJI70zefWpZI2A5FQFTSGIX54NN3Gk2tnpSkYHNFgG7jS7qaTSEimA8Ng1z2toLjVooyVYEYAzgj/OK3s1w+pGaLVpn8wbgcAgZwKBXNTUdIhsrYzRuxIIGCKxxI+8bQR65q5HrcwGLgLKR0JGCBRNqiXMJR4VBP8Q6gVLRaZWlkDg7mOfeoFkdGyD0pgwDyxP1odgBUsCU3BblsZ9hWla6vBFGYxARkYJB5z61h7xQHGeKLDNW+1F7nEPmyGIY5PU1ScALlWB/Cq+8k8mnLlmAJAHck0NCAn1p6NtINXm0uJIfNaUSD1DYqnHA0syrCucHOM0WAlZ8jOCPrUyyYtRhWyOhxxTblZEBWSPYfSi1unjt5IcAgjIJ7VIEOAQCz5NIQ3ZiKgMmWJHGaerEkYNAEoljCYIUEd8VEJcNkZ4HrUbDDHPXNKpX+IZFNIDRtHDAP3NXQdykdsVlxyowCglSD0z1rShOSAehFUgKZke3uN8blGHQitK31+YjZMEkz3IwfzphtopvlYEE9+lZk8ccMxQNnHfFAGus6TyEkYB6DOab5Ymikh98io7WJ3gLRfNjqB1p0Jk80FFOBwSeMU7gXtLslltWSR2yAQVzwKz5vDUzzFoJFCdgTW7pgCPNk/eGa04YYxDk/hQI4r/hHrlJAkhjIPoazbqymgmMYibA6cV3yW3nOdykemaoyaSJZttwzID93B60hpnDyQSxDLLjNRDPfrXYatYTW6/LaZhH8a8iuZkVDIeCv1FMLkcMpicOOoq7Pq0lxB5UgU56nHWoVsJXi8xMMvtVUAHqKTAaRilCMVyBkU4qCKntxA0JDbw49DQBVIIGSCKTNSbM9WOPQikZQDxQAiEYOT16UoGTjOD600ggZpUXLc9PrQBrWBQtsdiCehFaH2UGUEXKj2NY9vbmaRcSqmfXvV2a1aBQdyvk44PSqTEaUpMQAWUHPp2rMvJXWQHcCT6dqeIn6du/NTXenAWpmUkkdiKBCXF+HsNiNg9x61jAVKGUxYJ59KWNI3OJMj3BqWrjRWKgOQ2cfSpxDabNxdgajuERH+RiR71EeakZbit7aYEiUjHYmrkIUYjQ5IrJgEhnURjJ9K27aJwwLqFPoKYCFCDgimlGPar/2YHkE0vkYpAZTxkKeKgUlX5rXlhyTxWdcx7RkDpTQFP8AtFo+EhRV7AdB+NRWtte6pJJHbNGADkqDgVEt03k+SIQSe5NaGkJJpOsQtcFVV1JyTxirAz7i0eA4Eb+ZGcMeoz7Vust1e6fE5GABkgHtVy8ltGvZJEuIirDkDoT61Wa6iEBRJlwPek4kXMa48suFBx6mt2zuYYrZYyobHc9TXOTSRtIdh6U+G7eEkgA59ai5ZNfFvtB2nA9Kii3AZPSkMvmuXc4H54qz9ohMHlqhPvSbuBEzggcAnPekcknJAX2qNiMjaMYNNkkCkMxPtSGLIwA5NSRyBkIJrPuZSSuMgA8UeZk5Hbt2osBcIAJyc+hpGC4yQKccAAjBBHHNISoByMikBEYyzDbjB7k00hs4OD9KGcE4H3ewp0JBkAPGe9UgJo0wtSBEjBkbkj9adNGwhIUgGqbZQYJJoGSzXIcfcxj0qmTliRSu+QSKZ92MeppoQ8A456VI+xim08AcjFRKSUA96eI3G/KkDHegGhDzIcD5exqS3kCTbWxtIweKSNT0YED1IqQpGGz196TYXKswUMxVuQeAaSPcFDAHIqSYIxBTkDrUJJBwDgUCHmY8kgEnuaaCSCMEg9xQE4ySBTgADgdaYyw+Gxhug6GgKD1b8qauMgEHNEnDEUWESoFLY3ED1FPIS3IwAceveqeWHSlLZAByTSsBOyBiScKT3ppBVOTzTmJMY4BHvTS+RzyKLFIjJHakzjPSkL5JwAKaSSSMdO9OwgGT05NOCHPSnomFDY61IMY7A+9AEYAL/NnGMdKMlRgVKEUqSTg4OOe9VyCcdAQKAJVkcDOee1NlkIc4PyHFMQSMSCSAPWhgdufegBocg5BIPbBxTnMkzEuxYkY5poFS7xkFVwAMdc0AaNvqktvaiH7OpIXaGzjA+lVZJzPAsRRAFOQQOfzqPdxgn6imcnpUgRGMJMrMMqCMitWQW8saNDzITjA4GKzlO5sGr1uN9xH7cVQ7nTWMa/Z1GORXM3pt7nULszOQQSQR3I6V1EDeVbySY+6pOa5O6mSZVl43nOSBjPNBN2VhCskQcDHOMA8Co3HlKxEZwSQCRUiMuwAHn09qfqF4s7JHDkRIMcjqe9HKVdlEGpY0MjhF6npUaRvLIEQZJrUieG0jEMe0yfxSMMkH2qmrElNrZo2Ikxu74OaI2kGCZ3AByUBwaeZC8gJ6k4JpDEUmEg5FQNMVpRNcFjkt3JJJqW1FuSxeQoytkL2NUY3IMhHc1JESHdlUFiMDPammIfNNI08jsRgjHA4A9qfBIGIMn3CCKYy5gRD1ByauWscbCKFkzkEnmhgaOoER6RYBIXUQKNzkYBJzwK6TwbbkQ3EE7li4WQQggqV9Rxx7+tVH0a31fR7e6E0kEa48wk5GB1wOMnpisjWNeVLy3/sdZ7T7PF5W9xguPp6cUkgO21uyF7pxtvLEKvly4UAjbzgY9a5wQ6yvg4WtlA5R5vMWSGQEheSQe45qWw8RyXMaW9vJDKxOAbt8MSeT0rd037bbajJHcRwrbzjcio25QQOQDjqf6VdhpnBabrWs6fDc6fC0wuJpM4fkgkc4z0PSr+maHaXvh27uSksWoW7nLOcYPUcfT+dbPifSIYbee/LPJezTKISBgqSeAMdentWxpNxbappBM0QSZv3dymMHeODmk0F7lyykOo+HreOYEGSIBwRzkYpBokJe5cgHz5lmYDplRgAVYhEUESxxk7VAAycnAqwJVI60rCIHt0c85B+tNFqo6MRVnIPSjA707gVjav8AwsCaetlcYyCg+pqdXAp/nkdDRcCq9tcqMlAR6g1CY3J5Ug1eNzlcE81A0uT1ouBD5LAcPj60wxyjkEEexp5kNNMpHQ1QDMP3ppLA8inhwTzTiNwyDQAxXOcmpgoIyKiGBTS2D1xRYC2DtAxSBEc8iqolJ704SsOhoAnNumeORR9lBIxgc1XEzg9anhuMyAE0AcdEM67ef9dh/Oup8qJpSG4rlrNxJqlwR3n/AK100jYkoAsLBGh4UH61ILeEjJjBNV1lOQSalExIpAMuNJt72AxBihyCDnjPPWvPdRjk02zktJbQxzNISZSMAgHgg/SvR45SGBrj/Hab5rZ8YymAexIPT9TSEcwPskb4M0j4GeTwTVF5HuZiyxqgPYdBVu9hgtkREcySugYkjGwntVJG2DFJMY4xxhcMBu701Dt4GQPanM+8YzkgVAHxhcDAoGWmSN+R1x1FMMCkckk+tRebgZFPJIHJ6ikBZhUukkKj7+MDIABGck1SBYEq4wynBHoanRgHRifukGi4lDySkAbXAzgY5piKu4g8UoOGyRnmmkHNJmmMdwCT2NBPFNGSRnpQetMAznH1p5JKge9NAxU8RUkAjJPagZFtOMmgZHQ4qd0ABIBxn8qg70rgTI/GCaexwQR0IqFULDgjNTFJAhDD6YNIQ5MP1q5ZSyQ+YRAZoRgSKO3PWs2NmVuQfxq9FdyxqwU4DjBHYihAWb+TSH0+2NqjR3ZJL4JJ6d81lhsn5zVuOOFg7sQJAfkye1VZoWinCuCNwBGRg4PSgQ+MbnAjBJxjAFW3hmsLiIyoBIAJAAcgfWpLHUI7O2e3MKuZCSXY8g8Y/LFR3sr3UxuHIIIAz0J+gpjNDU9Xkv4YEckxxryh6E8816F4OdToMM0YwGyCpBwCOOPrwa890IaaBcSXpjZxGTFHJwCQCeD3PHSujt/Fkw0G3s9Ns2muRb8mJcmE578cnigVjvXY5yOKa0uwZJxXJaTquq64beZwbezgBWQk/NM4A4PTAHH5+1dBz/eP50rgkWJ7hUgd3OABkkdcVwj2EE+sNdQOBlyQHPBIHX9K7GZlMEvmfdCEn6CuGhvIZIbia03SMsjiEAnJGM8D2qkwMTVbq6ubwQzSeZ5J27wMAgHrmuh8KosN3dksSWjDAnpyTz+Paufu8jTYImO2QgyHvk+n61f0zUQksME0iQxREM5A5fA4BNK4HbHPejioIZzcWschUqXBIBHbt+lLmqQDyRmmbATmjNOBoAY8J6rjHoajYGPGcc1OCaZMDIgA6jpTEMD4qZZSBgVWKMKVSRUjRaLkmpowcc1SDsDwalWZgeTSGSyWkbxlhkPjPHeqRiccYrTjmTaCTTSqO+7OaAMeSKX0P4VEUkHJU/lW05QdBj6VGSO1FxGRvkHVG/Kl85u4P5VpBgDmhthOSB+VO4GcJifWl84+9WmCHjANJ5cZHIFMCt5hNODEjvUvlKDwKUIPSgCME04MRViO38zhRk1MLBj3A+vFICiHNP8APCQyk9kJ/SrR0092H4UyfSwLK4lycLE5I69jSA5bwheA215GTwHDj6Hj+ldALhc9a5nwCsM2pXNpNnMsYKkHA4PP8xXosWm2sS48oN7scmhAY0c6HHNWlYsOKvvp1oTkRY9geKYbdE4UYFNMCsAxOc4+lTHBwSBTxEKHAAxjn1pXAR4woBCggjuKjAVjggA+gFPDk8HP4VE5w2e9FwJooUDgkA/WrigDggD6VmfaMHrUiXg6E0wLEtkkzlg2Ce2Kry2JiXcRx61YE4IBzThKG4JyPSlcDMJB6UmauXMSy4C4UjnIFUWyjEHrVrUl6EqsBjNK6pKMgBT6ioASxwMk1PEFK/MxBBoasCZEIHydoyAM8GowavCAlN4cfSnxsQ4IVQe5A601KwmigPy+tGfcVryRQyxbnQbuxxVY20Q/gH5U+cOUo9+oqhrIzYA+jg1ufZovQ/nVLW7ZF0WdgDkEEZ+tHMg5Rg5jjPqoNZeuDa1pKegJB/HFbtoIpLC3k2FsoOR9KyvEpj/s2MBl3LIDjOSarmQco7TSTDIPQ1bIAAyyj6nFYgv5bI5ijVhIoJyenAqk8sl7c77kBvQDgAVDmkV7Nt6HQyXlpbkCa4RSfQ5/lUD63bRkmMNIB3xgfnWALcKzEAZJzgjNOEm04ZSPp0qJVuxpGj3Ldxql3cSAxt5ag5AHf61Mmr3sagSRpL+GKoiSLorZI7EYNPEmehrL2rNPYxNRNeiBAmtZl9SCCKtRavYTDImKf9dFIH51h7nHej733mo9qxeyj2OgOo2WOLlDj+6c1m3l7GwZ7e8k3dkAwKzTbox44+lIbZAfvN+BqvaLqL2KLNrAl4jm9vWjfIwCe3rVJ2uNPuyIbkuqkEEHIP4VI0bOpUsGHbI5H41AIJIzwhI9jmjnQeyNSXX7xVRzHCQRkjOKk03XPt12IJoRGW+6Qfz/AKVlkxniWFVJ6EKaVI4zIDFMquOhB5FHOT7I6xo9hIqJoi/QVjR63e27YknhmH+2Ofzq5D4lG0+dbocd4jz+Rq+e5DjYsG2lP8NM+x3H9z9acviLT3YAs8Z/2l4/OrseoWc2BHcxEntuGaLi5Sh9jueSFwQM5zXn90ZGvZjIMMW5Ga9XOc8CuP1/wyYllv7VycnLx46fQ0+YTRyRQnoKFGKkjSRsFz5aHoSOoqW4svKCskyyA45xipuNFVvYjNOVkC/MNx9KYAB160pYDoOaBoNyBs+WPxqIgDJ96eXJ60wkUDHAArxgEUYzwcVESRwDQCcYJpsRIAWG3JPsTUgM1rMpBKnqOahDY6Hmnos1wwAUkr39KBl67uHnCu/YYFV0kUA89RiklcAbC2SPao1ANQIjIIbpU6kbQcYpXjUgMA+e/wApoIAUYJ/KgYrhWIYuAD1FROFBxGcipPLkYYQE0zy2U8jBHXmqiIVYjgEite1P7oVmOXAA2MPfHFaNrG6xAuMelUBehZxIAgBJ6A019PleYi7tcr2IOMUsJ2zRn3raYZGR17GqsIp29nFabjECARggnNOMckCmQAGN+CCKfKSIyWGB3PatG0QT2oC8kUrBc55bwxXZQnAPSugsw7wgtnHasvWrFI7cTYVZEIBGMZFa+lTedpcb5zSEWQMVBeECMSYyR0qxVW5bzSIVGSTQNCxubtAZeV7r2NJPpdjPGVa3T2I6ircMaRwgY570NigZh3mnC2twIQGj5yMcj6GubuLeyKljG8Mx6Ejiu+461BdWVpeKRNCpz36GkTc84NvJtLBlZR1OcVFGxV8Dqeg9a3dW05LCXaHHltjAA6VjhRHNlTnB4OMUFEsdnLMQWyme5HFLJaeSSDIG+lT3d95sICPhsYIxVAyvjrTAHhKjIHHpUYGKGlf1pobPPU0gLMZJAFWIkHnKJHZUPU9aqRTASAGtSGW1eVRKBt755oTBovg2SKArux7Vswwb7XYRkEYrLMUJwbeJWPY4ratmPkgFSCPWncVjk302U3Zj3hB2yM0wacySlHkAI7itrUR5V6G/zzUIXOoofUZqkBRjsIRIBLNg9gRVa/08QgOjhwfQYrqru1tpIf3o2f7SjmuKviVmMcckkkY9eamwyWykNrIXNuJARjkdKsS6kTODgLj0qrBdEQhWbkdsVA7BnJqWB1sQzCp9RmnEVW0yUS2gw2cVb20gIJBwQBWZeJ8rZraMZPQZrM1KJ1QtjrTQHJodkyAgkA5IA61o3F/Fe3UKvBhBgZxz+VUQ0Uru7ZjycgA8VTMmJCVYgjpimB0T6rp9kfLhs9w7/wCTWZe3UN0d0MIiPOcd6oNIXOTzSo1AWL1gEYESDJPepLi3cbmjjbaOnFUUkeNgVNK91O5yzkD0FIAD4GcjmplmCjBODUKuBk7QcVHvZySQAB2FKwE5lB6GoXyzZJB9MdqjDdhT1jZjwOKLFDwfN6gDHAprFY+gJbvQ4IG0cHvTCNhBPemImVi20nHB7VbkkLxZCkcY4qpEmTk5pBO4cpuIGaVgHqCOoIqRI8uMkgA0wZLDJJq1GAeaQFvIK4qjfsAQAMY61cHCEnoBk1msDMWbOOaEBCASQB1Jpsn3gOw4qWNSkmW5AGeKgPJJ9TViJrfJlI9BmrqSfuCSpPPYVThPlsWAySMClmupInKocetJodyU3sZ4kUg+wpfMtz/Eaqkq6F+rdzTVYAgmpsBORCXHLBQc5HeoXwXJXOM8Z607cX4wKaRg4piHZ3de1T28cJy7kgkcAdqgWRQCMZYjvU8IAQHHNIAMWZQd+Ae5pHG1sEg+4pCxzgkcelISoAIJJ96BgQAaSTHAXnjmkeQkDoKapJNNIETQNwQScY4702Q4GKTBXkZHvSAhjyBnviiwDOopuSCQKcSR1/CkABbJpgOEjKmBil8wyEE9RQUBOAcUC3kzlTx9KkCYgsB70xxhh6gVIRIsYABJ9SMYoKKRyTk9aYEZckgZGPekJGcZpoCF8EOSD6cU/YBJ8wwuMj3oAURkJkirP2dTCNnJ9KrzsREgB6mppCgiQk5IPIoAr78EgdehoJIXjrSujFshBgnjFIRgkOCCO1FgCMZBI61o6Yhe4wTnHPSs9OhrW0dNx39zkmgRtysiaZMXbapUgnGccVxcTBGKSo24nOfaus1R/L0vGPvOB+Fc3qMiNdRlD0UA0AV542RgyKdh+6R3FVSTk5rXN3GbIxAqDjAAHIrKYZHPXvVJjTJrWRo5cqASR3q9GyidDMxBxznnFZwyuCD1qUcc5JNJsTLUSRG5AzhCTg+lLMoVpDE+9VGemOfSq28L1qyUMS4cEK44wM5PpSYIpuTIQCFUDnANEQIfI4wetRZAbHap4iByeQR0oQx8hDucVp6TGjzEzE+XHHkkcetZkaZNXo/liIPQjBoEdXpeq+dbCCGH91CSEjJwC2c5Jx0HpXIX91Nc3d4JXEk0kmCIgCuB2HoKuwXQt4JAA4dhgY6g/Wp9Kt4be4CySLudDkgdvc00xmJpMNtNqtvFdllt5G2sVOCCenP1r0O3tDptuYvNaRIZPNV2OSB6Vw728VvdRTRJI9qJlYuwwSc8ivSJIUBU46jNWtRMgk1aGZYybWWY7gUAXoR0PtVmOaIuwjQKxOZAg4Dd8nuaxbqwe3S9mtGJkkjIVD2+hzWNBreoaFbJZPa4ZckiQZB96diTtjIw71He6imnWLXUrcDgDuT6Vn6Xf3d3EwvYFhlGNuBgPn0qr4rvYE0hrQSA3BYYUdQARn9KljRt6ZrUepWoljHzj7yZyRV1r3jhSPrXmmn3baXqkExO4ADeF64Nd79ut2szdiQNbhSxI9KVxlg38ayIkkqK7jIUnkinfalPUk/hXmmo3bX2syXcRbZuyjN1A+ldX4bvm1GzuBdhWeJgAQOMEdP0pgdCLhT0YUGdMdaqeVB2BH0NIYouwJ+pp2JJWmX1FRGbnqKhMQHQn86aY/RzTsBYE3uKkE+BjNUvLPZvzpRG/dgaLAWzNz1pDIDVCVniILHbGMknGfyqWGYTRh1DAE4GRjNFgJpZvKjLbGfHZRk0lrcTSRgzQhCSeQc8elQ3JcxExkhgMjB61WgvbdJTb+ZmYgEjHU+lQ0M2M0qHDA+lcvHcyrP51uWEiuVKu2QOemK2FvTFCzTsN4B3FRxQMw9K5uZD6z/1zXSSMS2a5fSnxE02M4cnFdEZMqDjGR0yDTRJIHI70omIOSarGT3pPMqgNIXMZ6msPxHem3t45zAs0Y+UhhkAk1YE8Zn8nPzgbiPaqeoRQ6gHtXn2eXyRkYYf1x/WlYDiL2Xz7qS52hFc5wOg+lVGY1qyJDNBtUBACQST2B4rJfg1BYByDmmk5kyfxApACTwOnWnYNAE2YmxnIAp3yYIBJz61Xyc0/BxkUWAduwMfWmEnJyeD7UEkEHPekYhjgDiiwAeaaeKXGKu6Xb29zdFJ0LsEJRQSMkdRke1MCln0orejs7SYEx6VIwBwSJGPNNbSLfqbG/H0bj+VFyuRmJzRkjocH1rYOm2Q6x3qfUA/0oGk2Ukiolzdh2OAGQYzU3DkZlBiRgk0oTJzkUksZhuJIiwfYxXI6HBowMA+opoljkJQnB6ip1lycE8VWJxQSR0oaAt5DZGeRzilPAqJTtYsAOeSKeWBAPqKkBQRnkAj0NJKJH2zPkhjgMTkkDt9KACWwKcIZCgIxs7DPSmA6CXyZ1l2hsZ4PQ8YpEWW4kCRo0khOAqjNS28GZAHBODggDOB61ZsJ5tF1YSKcqTnJ5BH9KLgP0jRLjVLhoWcQEAk70OQBjtx61tab4Wv7XUiGnItTIFcxkqXHXkEdO350o8UCaQy7ITMDtijwQRnjJNXYfEd3DBKby0DyRJvDRnC/Qii4jrDIOmAAOgAwAKA4PGa4nTPEGs38hlEUckKnDRKuM5GeCen1rrY5gYkkceVkZKuRkUwDU737Bp8lwqCTYMlCeo71w+kz29pc3F0Awkd3KxAZUDB4/X9K6DxJqFmNNKGaQSODt2YII9/biubcx2lkksknmCQfwdFwP8A9VCAUQC6EqEAnhnYnGCSMDnpgCmaXbtcTT2sMKEIxd5QQcjsB+NZtu8t1cLbrJtjYl2GeDiteHVxZqqLbbSGwTEAA55wCOvU0AdNZ3sci+Q0mZowAwA4HHargwwyDmue0+wm0vS7q9uUzNKAw+bkA9z6da09FeS4sPtLgAyMSMdMDj+hpJgXsc0u3ikY4rF8Q69c6L9lENqrmQkEuCRkY4GO/NUmBsEuO1OAyAazrbXrGXT0urhxDI4yYRyQfpiqzeJIJpBHZw7mPeQ4GPYd6YG5jA6VGIWPasUa7diTJjiKf3cEfrmpR4gl/wCfZB/wI0AjWEJUgmn7AeorBbXL1nBVY1UdVIzn8aU+KBbTKt3Z5VhkNEeR+BpNBc2GBU47VFcXAtLd53bEaYyfQE4/qKW11rR70gC5CSH+CT5T+tcr4k8SEm90yC1URg7BN5hJIBySB07UrDOsEu5QwOQQCCOhBpC9UNOlL6VZsevkqP0qzvzTsSPL00ufWnKoJGTgetSiGNgDjIIyDQMr7j60BjVsWsfoRThaoD60XAoGUi5MOOQgYnNSAk1LILSHWII2IM00ZCDPOBz/AENWzFGeoouBWR2UAg4NSCZy2ScmpxDGB939ad5MR/h/WpuAwXBJwTTpp0j0+7dzhRCxJ/A09Le3HVWA9QaZq6W8Hh6/YngQNk/hx+uKaA8/8COyeLLcD+JHB+mM/wBK9UJUHk15B4SmaHxVprKcbnKH3BBr1idiGJBoZRNI6Adagc5BNQmYgZPJqFpmY5zj2oSE2W4jmQZPGalnVM8CqUcmwg5zg1MZN9IQIRuGeBUs1vG8Q2Y3nnJqACp94MYHAIHamBnS2zxth8Y9Qc1EYj2q1cKSQVJPrmoFYqwJ7GhACQTk8DA9zipRFdquSmAO+RUhuMrnpTo7ptu3AI70wII5NxwW4qx5EDjJXk1nhw0xIAAz0FWFZtvWhoCdUjgfMYwfWqUqyRFnIwpJOatI2Tk1JNB5sIxSuKxnrO/rU0UrGQc0osz2wPrTbhfsNpLcOVO0cD3ouCMrVPEF3DdmCzdQEA3DAJJ79arprepkBhNG4P8A0zFc1LKZ7kykkMWJBBwRU7Xb4IGFz1x3NSzWKOhPiDUl6mH8UFQXet311aSQyNFtcYO1cHFc8bvB5YH6HNO/tFiyqclScYwKRVl2NeO8vntVhW6KxgYAHBA+tU7i3CQljy3rnJqrdXJgIUZwRng4quL12baSSD1yc0rsLI2c7reEn+4BUYdI3BY4z0pjyf8AErD/AN0YrJW6mYA7FH4mnIUWdGFBGRjB6cimnAGQAfxFc+buYfwxgfSm/apj08s/8BqLGnMb5IPUComjJx5cm0DsTx+VY/225/6Zf980hvbknrH+IzRyi5jZ3zp1eM/Ruf5UonDHDuB9TWOLi5I6xf8AfH/16a13LGCSkZP0xRYfMb2Ux94H6GlAGOM1gRXs0mSsMYx15Iq1Hc3DDKhPwc0couY1gD2Un8KP3nYYrH/tK4RipTBH+0ad/asw6hx9GzU2HzGpukbjrR9mLckVk/2w3q35/wD1qP7Yb1b8/wD61VYXMahtFHYj6Gj7OR92Q59SM4rK/tlv7zf99Uo1iQ9Nx/4F/wDWosw5i/JbuwOUBPqOKi8ry/vRsKr/ANqzDqjUqapIzAYkGfei5JeilKAmKaSI+u4jNWHvb9bRgl6ZFIwQwzkVTLvIME5z60LG6jpn6Ucw9OxR02RSHhkQPg5ANOuLSeaPYo2jORxmrMFgEmMwBGc8YwBVsIQeSB9TVc4ciObksLgHA2n8cfzqF7S6QjzIXAPcDP8AKusAAH8B+ppCinsg+hIo5xciOVa1cRGQHIHUYwRVZsV15tkOcvweozwf0qpNpFtLnBjH6VSlclxOZyKMgnFbx0ODqzkAd1YH+dS2ei24naRJS5A+6wwDTvclo54fLXR6UYTbZiUhzjcD39Kz7tYxeErbKuwkHPelhlaK4DphR3UnjHeqTCxqywwqDIIIwR7UttMrAjykGO4FOEiXERKEEYzwc1UtWIuCvqKloVi9NMIoSSFx9K52c7nL+tamqziOMRkdRnNYzOCuBUAKDxWnolmL+4YO4GOgI61m2zqtxGXGVB5FemwQwvaxyQwoMgcgYNaREZsemqgACA/WoNVsHS03hAAOuK3PLb0qO4tjLayAjtmqEccG5VvQ5rrbaOKSBHxnIFccPvEehxXU6KXksR3wcU2xGjGI0P3Bj3FNMYikMtqBGxOSOxpcEdqQkikIx/ESC7sd7DZNH1A6Ee/5U/w1Ij6aYicEEmr15aJe2rxklXxwR1rn9BkNldTQy8H1/OgZt3d2bckGnWoDgTEckHBrAvWe71QqTlAQRWnD9pjIQfd9KBmoW5ppYetL5cmMkEU0oc9KBBuFAYGmlGHagKR1FPQRna1pKX9uWXIdeQQetcR5bwsVZDkHGBXpYB6Vy2tQCzuGkcfK5wD60mUmYJmVlIKgfSnm0SS3MgcjHYVoR2tlNbGYEEgZIPHPpWZsLAqpIHpnpQMp7ctgVaeykWHzRtKj0605LSQNkDNXgXaAx4C+4pAZMTqjZKhvrVmKZDKp8pRg5qjIvlyFc5xU1uyeaATjNAHV2esWa7YjEUPqOa2Y5Y5RujORXL2/2eGQOwBHcmuotpoZoQYdgx1ANMDM1sbBG/0quTmWJx0IxWpqdv8AaLNgBkjpXLW985nSGQ9GwaaYHaIiSW/7xQR3Brnr+awj3pDEUcjBwK2ZLuK3tgXYEHsDXJTRSTXLmNgEY8ZpAZTgByQaauQeatXdpNb4JAIJwSD0qIRknngetJoCaC6ltzmLrU39sXRUggjPcGqoKL/ED9KfG0QOe9QwJWvb3qJmFNN3cSjEs7EelOzGepwKa6R7cowJHXFK4zF5jAOQSR0HaoSOc0uaUcgirAaMd6erKoxmoyAp5NNzk0AWNy9jTWfg8E/SoxnIqVQOCetAAAcc03B74Apzkr0Bx61HnJ5oAeqgnaDn3q3GVTGcgeoqoSI14HNSRB2j+8QO9ADpW8yTgcDv61DIdvQEntUmQGIHapCQYwAAKAHQECMM/B9KroI3kYgZIPWnuGcKgOBnFKsIhO09RQBIg3MKuxqOgFVYwOMVfjQAAetJgRz48kgkgGs8AR5CcjoRmr1wQXAB4A6VVbB4pIZEHMajgkk9KbIVbAIxxmpZEGAdwAGMA0pAA3cMaYEMJUzKgFJcQsZz0A96nijDXAJAGPSpLqM+Zk8fWi4ivDYqRlpgPYVJNCsEZXYD7mm/8sCQHx2I6GoVViufNf6E5pAOER2ggjnnFRnIODQZh0Y5FPEaEg5I9qbAWFCzMT0A4oD46GnggA49KjAOcCgLBnGaC2RilOCOnNKkTOSAMii4DtsTp8pIccn3pinHHelxsOQcH2NMJ5J70xFliDDgMMjtUSgDPOc1HkmnROEDEjJJ70gG9GyRS0Fs44FSAAsMcA8Gi42AGVOOvb6VIXkWASIxBBx+FBQLjBIBPAqXaot9pxknjJoAgjmMoJcuQORmnYBIPXikaJz0IA7mp0YRqQSGyMAEYpMBQ5eMptPA6gVA4BTB7UGWRQRyAR0FAI2ZNCGRsBIQAcADHNOUKcgliQRkk1HuG8Egnmnlw4IwOTQBLI+CoBGMdB2qKSTe+TycYzTMYOaaFy4BOAT1x0piHV0Gkx7IR69KxHh245zk44610VhHtjQUwZX8SORb20YP3mJx64FYCRq4OWAk9zWn4llzfxR5zsTOPTNYqk7waQieFYzw4JKnJI7ChhHhjnBHPTigkgEgkE4Bx3pigDg9KAEWNnOFGSOalVS2AOucYpEYI+QcHBFNTO8nJBxwQcYNAFmS3Yxpn7wGDxT7i4j+zLGGzgZFSQTy4jkBycEEAcms2baCu33/AJ0ARDknmpwMAY7ioNpBIxyBk1OpyBjtwaYy1CM4qZ3GQFYHAwR6U2IDbn2zUTj96SjZLZOcVIy8g3RhuhpLS6wZ1kjEgyAST2z0qsZnWEAgc9zTrN4wyI6jGQXY9M9qEDJ9RuJrgeYkTJCowuOgPqTXd6G1xe6RbyyvhyoDKx59q4c3a2NwLS7hZ0Rw4CtjcOoPpiu5g1u0ukSWyaAuwy0LPsfPoM/jV8xDLEyPGyEdzg/Sq0mk2l7IGuI90gBVWz0B9quXEgZrcFDEXYjDnnOPakMy288hfOFXdgDJPtVICpcXAsraRryJdsIAUg9T2xXHbY9Vupru9uPLIGQMZBFM1eO+tbxYr/eJJlMoBckgEnt+FZsjjBUng1LYIkkYTSM6vtGM8jrT2v7prAWSykw5zsHAJ96o5G7BFSqwxxSuMsyq8McIDgg4wB2rotNu7WwuJXMwCuo3BDkE89q5hZGVPLYgKQCARmhHAkAHIPUepoTA9FjuIZmVY5kZmBIAOSRUlefQ3BtryGZTghgfwzXVa/r0cNtFDpxRmnGTJz8o46HsetVchmvRUXhx3vdEhklHKkruzycHqa0jbKe1VcZRAYng0FZQM44q8LJsZUig2sg9D+NK4WMucF4HVxkEHiooZkhtDhi+0EnPr6VsC0l7xqfqao3FnJbedKyAIRkAEYJ9DRcLEBZrq0cgNFkcEEE49awIrgBxKTGZFOAe/Uf4Vv8A2WGe0MyTMJAM5B4BxnH0rmpbTyjG6yEx5AMhHBI7gVnMaLcNzCmoieXhCdxVRkZ+lW7u5tp0Mv2ssQCQojIIOO9ZEh8y/O9wEIGGPTFS7zGHELo4wQSehGDxWKkyiXTcrpTkHGSas20sdvGJN8jb5NoZ+mPpVW3+TQ3PqSP1qWC4jUxPKnmFTkKTge5/DitHKwG7PHDBHEdxG8lmJ6/QVJEIGw4EiEdnH9Kwr++N/dQhBtRDyCcHFatveC7Z8LhYwBnOQauE02S0Yzrd6fcl5IhIxYlpAcgoeg9ucVz97dyo8sBiWNgSCV9D711N7q+yFJLVUlAP7xWByB7Vy188V1dvKkZWNjnBPNNyCxngGQHOcDrUTngkdDTpHKMQDxUJOR75qUUKhwDk0hbnJFMzQTVATAjIJp5OBx0qEEYoyfWhoBzOp5P5U0HHSmk55pKVgJCxp0UzwzRzRnDxnINRgU7acUAdjpN0kd6CD/o90N6juG75/Gr1orXLm9uGYsSyqgPyheR09xXJ6XIXtpLXdiRD5sRxyCOoFdLZPd3sS3NtcR4cYaGQcKRwee1QkXe6uS2t0un3d3bO7/Z0USpvJJXPBA/SsXUL94Ue9bi4uMrEh4Kr3P8ASrN1LvuGSebfHEd88gGM4/hA7CubvLl7u5aZxjPCqDwAOgFOxTbS06lcE49frUhIwMelR96M8Y71RkKWJpyuwGAab1FPRSBwaAJkB2jPWlLYp4Q8EEGnBFIORyO1SMRZQTjABAwD6inb2Ugg5prAEYI5HtihRkAUMRdSXIPH5UTt5sMkTgMCMo7dR9KgIZQAO5waSRyAqk5AOQRUtCNFYrSSESIpjIGwAHjI6H61TMpCiN2+Ykhhj06U0kRw4JI+hqEEMAQcmkkBctbkWRJjkkBIwQhwD6Uo1e+EXkidjHknBOTk9eTVAtzTN1WkBtz3AuNMAQMQCSwJ7gGoru6iW1tzAqlcncrDJGO9Zwl2REBiMnJAPBNMZgTgDHrz1NNAWBNsnMsJ2nOR3xTop7iWXmQbtwcMeCCOlVQ2KfG3zj65pgbU+t3tzGbee4BLkBiRjAFdNZalY6Zp0drEZLgx5wcAA5561w4jBLSEAnvzVn+1bzP/AB6R/maAOmuvEV9ICLa1hjz0Zjkj+n6Vm/a7yXzZr6USbTlc9AB6Dt2rL/tW7I/49Iz+Jpsmo3ckMiGzRQVIJD9B60AbSsjKG2qQRkHHakLIP+Wa/lWQmp3SRKotUkAAAJYjgVH/AGxc+ZsFkrN6K+TTbA15bmO3iMkgcIOpAzUA13TcfeYH/cIqCaSea3K3Ait424J5JPtiq4W2gLGG3E0h/jk4Gfp2pXHY0P7c08niRj/wGqEZAJSJzKWOQA2cD+lIHtzJvaFreY9XjGVx9KlMdweYzHMp7x4z+VFwsJJAEXN1JBCo6gkM5/CqF5NatCBboxIP+sY8kemKnCxuSMLkdQcZFVb1FjRSAoyccCgDSstWFtHGqyywNjGQdysR3I7f/XroItdYoWNulxjvbuMn6g9K5KyRXt3B7H0qzHGsJ3x/K3qBimmJo6AeKrXkC0mBAOQSAQa6W0G60t5B0aJSPyrzxwZCSxyT3wBU9vf3tqAkF5KqDohOR+VAWPR1we9O28ZHB9a4uDxVeRqfOt4pG9RkfpRZ+KbqO5maQK4f7qE4C49DUsBfEU0tr4t09xKodEGCw4GSRz7YrpzdC22C7lgBf7hRDk/hzXn+rXdzqV8ZZypmUbQF6Aen610mpa2ZGtPsGwqcAlwDgjHB9PrS5gOsK/uyaYTtXNcQ/iC9sLnzFMb7wCQGJA+npXTWOppqlkLiNShJwwJyc0J3Av8AmCqPiC6EPhq9bAcMm0g9CCcVIZCKyPE0wHhu7BPXaB9c1YHE6HcLaa5YTPxGkwJPoOlerzyuTkHg9K8a9PqK9XEwMMeWz8i/yFJgSmUjqaBID3quzAjrTMkdDTAuiT1NTxOCMZ5rNy1TRORQBoZ+brTscfeFUvNNHm+tKwFrviopF5B7Z5pqyHNK78ZOaEgCRVVAVOeaWDl8U3yy68U5IWHzBhx2pgQXL/6SwAAx6CnwuWGDU7Jbpbu7xbpCcgVmXVz9is/OIIY8KDxk0Mkmu9YsrCYRzGQnGTsXOPrSJ4r00KBi4P8A2z/+vXFzXpFyzAgqTkgjOT70+G6tZsFWEUg6oTgEegqWaxR1d34sj2kWlqzMO8nAz9BWBf6lqF/GwmkPl9QgGADVQajAWCiJsk4HIpHv4lJDQkEHBBYVFy7RKLLICeMVCxy+GBHoc1bkvInBOzj2IqnvV3AB5J700N6EWcoWxjBHapypEyYAIODmm4ASVD1q0nMaH2o2FcdeIJFRjzgVpWUFvFZBzEmQCSSM1RYZh57Vo2o8yxZfUH+RpkjFvYxKI3CqD90gZBqLUIYwqyBAGzgkVQsmkgEM1yVaHlFJ/gPbNat8M2bH0INUJGN9nIckkkHnHpUot0IyVP5VJ1WrKDMa854rNFSMi4hChdo7kGoAuHBOevrWndoNpAIODnNUHAwCCDg1Qi+qjgkAVTvUAnAB6jOKvpyoJ7iob0A7Ceg/OoT1LKdoyJJIHfbkccZq7b3FvGxMkyn0wMVCIR6CpEhGegp3FYdNsluC8XMZA5oMKsMEVIABSnilcLFM2JboCKifT2XoQPrW1CARmnPGD1GK0IOdNu4POKfBDiTBP5VcdAHIPODUYAWQECgE9R7wkJlCSaeiYjAYc1MAMUuKybNCKRpo03pIQAcHIziohc3bf8tv0q6Yw9u49DVZIggIznNFxCBr1hkSjHuRSYvD/wAtBnHqKWRBkHFFsoFwD2Ip2JuNxef89B+YpGN0uMyD8CK0DCpP3RUE8YBzgD6U7AnqUzJcf89ab51x2lqcoCCCKYY1VeBSRTRq2FgLiON5EdX5yXOAarz3dzHMYLGEE5wCOc/Tiuu0kTXWkwnaoUDGSauxWIVgQqZ9QBVJENnCPpmt3KDzrXHtjBrSj8GxSRqXvXjYjlSoyD6V2Mlq7xk5Ax61nuSp2mRcjtkVoiGYdzYxabCVa3wvaRBwT71ihsXmR0PSu3CsVIdQwPY9DXC65BJZ38kY4UgED2osK5n6tN512QDlVGBVJSR1NDEs2TShFMoB6VLQ07jieK7rwbqIls2tZG5Q5X2H+RXHtDEIwwIP0NWfD2o/2dqikj92+AeenP8A9ektCrHpRdQeKehDAgHrxUJUHBBBBGQacmAeaok4nUYzDqsqnuc1r+HJAJpkJ69P1qt4kiEd+sw6uADUWizbNTiGcBjirQjq2BzjFROCDzWi1oTkhwRULQKv3iD9KlMVipH1rl/EUBtdSSROBJyT78V14iGflFc/4vtnFtDcHoDjH+fpTuKxAbdY2hkByW5PFbQAEgrOjhM1pbMOwArT8piQcUXGywZlIxmgAEZFVWAicNI6oPc4pp1vTLcEPeRgjtk0DLhGKY2BWfJ4o0jBInZsdghNRjxPpT/KWkGe5UgUDNDIzWP4kt/tGnlgQGU5GT16VcTVdOkICXcZJ9ciotUiS7smCSRtjp81Ik4iCTYpBAwcg1AXYOSCQQafJ8hKjjBxVZidx5/OmUi4t/Mq43mozfzHocfSoI0MjY5/A1cXTd3Qj8XFSMouwZsmgLzkVfk0lwuQ6jHqaptGYmwSp+hzQAAZ/ib8609Ov3sQMHcOwJ6VmA4pxfNAHfaZqMV7EMAhx1BrltVthBqh24CscjBzWWJpB0dh9DipIneVgZJGY+pOaEwaNCQMEBBJHcE1XMsI+7KVk9CaHWaNSVlUj0qjJzJkgDPoKLisTS3TuMFifrUBmbpmrYs1ePdvA+tZznDkUXCxJuB5PWnCRQcioRycEHFP8sdhilYCYzFhgD8aElKdBmoR8tG496VkBngHGTSqTSsR0GaQDPSqGKQAecZoHJ4ApmMvg9KkGAcCgBGBJBHapXOEzTDwaJCCAB0oAbG5IwxJB7EdaGAQnoPQCo0wTx2ocljk0xkkZywBAIqUhgOOlQLkcipVlYYyc4pCFTcQcqRUkabgRkDHPNIs6HIIPIxUiQhyMMBmkwGBcc5BI9KcBk59accRhlwT7iiNSEBwTk9qAJYAfMHFXQcDJ6Co44wFBxzUj/LCTgEnsaTYyhI252YZwT3poKlSxPI7VGJixIwAQabjGT6mhCEIBfJAP1qwgTYCarZyM4NIdzqQASB2oGXIlYTK4HGRg1NcSCefYTkCkt5ItgREYEfxMeKq25jju/3x/dk80gJ0lNuwzxGTjaOgp91Zxou+Fg0ZGRilv7eKOUGGTcpGQM9KqYYdDj27UARGPHOwCkHFSu7EAHH4U1Iy4YjHHrVXAjL4GMU+MrwT1poUFiGz7YqRERSSQST0obEMOCcgED3pwZgDgkcdqdIvyjA5J5piIGkCt070hjSmQChJPoafHDJJnCnjg5qUgLkKpx2NPAeMkcZIyc0AQfZpD2A+ppHgeMAkqQeuDnFW1kAQtLjFQGTa5CscdqaEVQpBqVcj2FWd4K7SMio2jKgYyQRRcYg4ZHXkjtUrt5oIIGD2FV1kCjBBz6VIjsTkcD3qbiDLFccgelKowRyefWhiR0zz60biCCAPxpiHjKk4AOR3qA++MZ7VISTzTBgnbkA+9NDI2wegP405CyxnGAScZIp75AGMHHpTCegNIBVCMSDnAHalYIcFeCeuabn2FFAEsOWnRfUiuoskxtrm7JDJdpjtXU2gwoPoM1YjlNYJn1id8EgYUY7ACoIlSIEEE/WnO/mXcz4xucnFRSPzgdaQxpyWJJyOoFGd3PeoyWJJoTJNAEq4yM9O9SEAjcvGRgVCeMe9SxuSCMEgc8DtQBZBMaKQcY7U1yLhwAAABgACqYcmXJPFTqW/gfn0xUgQMHALHlmJHFPhUjGetWowY9wJUOR0PeooxyaaYFkHEBb0FRJK0gEeVHB570673R2RwepAqtCcEMaQy3IAbYAEEqMDI61a06xju7i2t34Mko3HGSAOabAYZIxgZZuuRgcelaNmkcLLc+dtkjPCKcE+vNCJL2qiaDVp7gwwRxxoI4mkGSQOgA7ZrlZpJgSkg2sHJIwAQfrXpB04XdnCJX8xhIJS3UEDsfzrmPFmluLg3Nun7mNAG7knJJ/DmqsSNGrTXnh9p7i4VruCZBbr3UDjPvkVt6T4hgv7wtcqI7hYuYwcggc56e1efhninWM5jYjJBHOO1SLMYHkeOQhnGwkdcU3oCRPqmp3Gr6pLeSkFQcJgYAA6VXiGXztDY7GoYSFIUnCnqaczlGIQ4PrSZVh24IXKgAtwfakzzkAke9MU5FTqQIuep4pAWEjeeAuYmACkE9xVVTg5p/2lwmxZCAeuO9MBApoCckFQD1I496RXYcqSOxBqFXIYc808qRkk5yT07UgNTT9TureFreO42QuRkHoOev613P8Aadrpun24uJy5KcEDOR9a84jYxhWBwQc1ox29xc24uEjHkt0y38vagRvXI1K4knu7e5nVSQYwDgEEV0GiPK+lRC5f/SASCGPzEepFcYl7d/YzB5sj5GAQOAPrW7pkI0uz/tC7kYSMACZOpz2pgbS/bY4whKZQkly5zgnPSucvru1dkha+eaNSS64JyeOn61ZHiOG5hu4piAp4jcDGQc9RXLkRxfJGzOR3HSocijZOpGC1kgsQFLdSRg4qaw05NV06SJZyJ0wS7AkA+gFc4kiLOJZAXIBGC1aqaxDHBH9njkhkAwQHGD65FRzCsVLu3eG8eCZx+5yGIPB9KfP5IhZYVYELkknOc1Gbu3DSEQuJHOSMg5psl7EI2hit3DOCDJIeAPYUorUs0bOPztKEe4LkE5PQYqtbNGd6yA9cEryBj0qxAEGlxiT7pIB/E0TWNxau5tinllACWAGR7VcySL7RFHMZGMkhHBzxkU/Vb5U0V0iXbmQCMA4JGM5PrTILcEyG6Ty8kANg4/z0p19YJczo0Y3x4wHU4GKmLsJnOlHDE7yM9TnHNOLhhjIHFNneIO6qSUBIBJ64pgDqAfLIBGRkYyKrcCKZEOWJ5+lVmABq3IcREEDJ6mqTHmqQCEDOc02lNJnmrQx4opBSmmAzvTqUDJoIoAVcU7OOlMHpThg0ASwSvBMkyHDIcj/Cuhju5rVpBbcpdAFCOME5zg+3Nc1itWxuZItHuXGC0J/dkjJBPX+dTJDjKxFqlwABYRHKRnMjdmf/AOtWcf6Uf4Z/GjtRYbdwpaSjNMkWnxnBzjNRipEAKnnFICyjkAe1SCQEZAAOevrVQNxk9adESSeeAKTQFhskc5+opEYrwevv2ppcqpJBPsKYpBBcAAk9M9KQFtWycZ7frQcNndnI6cdaqmXEm41IJFcHk4I7daLCGzyFsAklR3xUAcg5HWmySF8k5x2OKcFQDgkn3qkgFWTDgkDGOtOJ4z70wgDGSDz2NKWBGAMUwELnGM8UZphz2pw+7mgaJFY9+1SKecjqKYMEA9+9JkjpSsBP5jetKJHPen20PmkcVeFmgFSMphpl4WQAem3/AOvSSPL5bZkGCCCAMZFX/si1DdW6pbSMOoUkU7hYrW7F/L3hmjwcoOM/jV2JpGyIY0hQdlHJ+ppmmoHtASM8kfyq9boB5mBjn+gpXKsVDauWznJ9WGaXBjJUuMjqAmavP8iM/YDJpqNbR4UToMjJJ6k0ribKeDKwAdQT2ZMGozaOk+UIU+o4q9M1tJlZLiM4GQc4INIh3iNz1I5piuUZp5MgXUKzgdGxhgPqKz7pywAydu7gN1A9K3ZYQ5GRWbqluI7dSRjJppjKttIUjOGAyT1Gc1P58n/PQf8AfBqOwVGkKt6cfWtD7IOwouFikbqQH/WJ/wB8GgTynkOhH+6av/ZAB0FILMHqKLhYomWYgjfH+RpA7+v5Vfa0THQA+5qhMhiYgkEeoNS3cTFL4PrWnYxiWBj5mxXIQZPQk8k1jBuQanSZhGqHlR2qUSb91p3k6a6CEqYiSs28YIPrW34cisZNOCRymOYH95vOCSe+K5i0u4U09oELhy44IznnNXbe/nvZxDcOghjJYvGoBBHQ5HWqvYDsmsFA/wBYxrC8XQvF4cYx5IMq7snoMn+uKveHZpLhLgAyyQhv3cknH4Uzxp5yeHfJhQN9okEZz6cn+laJgeZucRE969bsNLW5020na4A3woeB7D3ryQjcyrnGcAk9M16L4X1J10S3szMJpBO0YweiDB/Lmk9QOhTR4AMmeQ/gKilsIoxkEnHY1bFyFjIziqz3CnIJzQBAEUHOKacA8U8sp4zjPemNHgZD5HsaaEId47UoMhGdpNIFYnAJJqXbIAMqRn1oEIGx1p4nAABAODmoWRycjA/GgBh1ANAyYXIB5pwuQe9VmUN2A+lAiIPBzQIuGUCqOoWsOqGMXEkiBOnlnBP+cU4wzAE5z7VWS7hfPmOsZBxgnOaAMfWPD1pbWgmhuJm5wQx4wf8A9VZsOhWsmgzX5d/OViAM8YzjFdLf3NvNYvCjb2PTjisWxkI0nUrYgFQA2COhz/8AWoNLmdJbK0EMnqAMjrVc2UbXRjYscYwSa1EAOlISM4Ix+dUnfN3vx2Ax2NQUmUJ7ZYpCvXHeoohm5VR65q9d8zSMPrVIHy5lkAB5wQfSkUXo8EXAOORkE9qfHgIADkDvTItpkckgDHBJp6H5eSD9KLCJ2OLcn3Aq5pr7oWX0OKpkg2zZIGPWp9JkjLyAMD34/GmIaLkOJYI7FpIw2GG4YJq1dAG1dQMDAIFZ2YYNUn824ljBO5QOAfX+lae5JbcuhypBwfWqJRmgZWp4y2wbQDx3FQA/LRHexQEq+4HPpWaLZK8bls7V+mOKieI7SRBGcDPSnHUbcnO5h+FNN9bspG9xkY4FMQyI5UHAB9BTpdpjO9Cw7VHEUwQrFh2JFSvgxEEkA9xWXUsYOUFKgJcKOp6U0EBQAQR60qHEyHPeqAsCEj71TiO3CgOjE9yDQ5wpPoM1izXsjuSuVU/w+lMi5eh1IQblMJIycE1FNqG+fKfc9KzDM5bBJwaUPtFWmRc3oI7S8Bfy5A3ceYBmi5isrRAzwucnAAkFYtvevC4IwcdiKWa+kkkLEAg9utO4Gwj2DjKxSfTfzUxitQoPlzc9t4rDExyGBH0AxV6C9aQLGVJx0IrNq5aZeUWu0jyZOe++mmK3XBMc2D0O8c1HISgyKnZ82qtgHoORRylcxEUtD1jm/wC+xT44LJcP5Up+r0ssSCLIGCaSP5oWH0qrGbLRFscELJgjsabLHatCQyyEAe1CYKADsMUrAGM5OOKYFMQWhOP335imvb2gOP335inQnM2PWiYYkNTYq5Zs7+WxHl29xcBSchDgjNWZPEF4DhpZQfZBmqEJGBn1qSXOcjjNWiGSyXb3QzJdXpz2BwKiWzteoluA3ruxSZJhJJzSJkgGmmInBMYwl/dADtvFVL9jdeWJriSQLwM9ef8A9VOk+8arSjgn0p3Cwj6Ppg533Az6YNVdSsbWGzWaAMSp6sMHFaUozCj+oqG6jMmmSADJ2nApMSMeIRyJlXyfTHWqylVuVL/dBGRSWDbbhVPcEUsqgzSZHekVc9TsWWWyiMIyuBgVajjYuBtI/CvNbW/uYoQI72SMegPFWBreorwL+Wi4mjqvFNpiySfqUOPp0rm7Wbyp4pScAHmopNTvrkbLi7kkQ9VY5FEQBTFUmD0PRra+trm3BjmQnGMA0jjJ615tHeXFhMWt5Nh9cZq5H4u1aKQFpI5R6OOKkDvkwpyxAA6k9KxvE93YXGkSwi6j85eUGc5OK5PUfEV3qRAZBEMYIQnBrMC256yfvO4NIDbsPEZtLBIvspdk6EdDUU3irVJOECQj2GT+dYHWTbV9LQBeGJz61Vx2IJbm4mJ82dmz1Bc4qHCDqQPxq1LCIhkxBh3IHSo1aFuhUex4NO5XKMDoD97NSLMh6H9KcdqjICmomcE9AB6Ck2BKJFzkqCfekM7j7mAPQGoC4wcU0N61F2Kw45JJPJJqNgpPIz+NXLaNJDjrUjWg3EICPYijnYWKcBiVgDEX+hq0RbHkQlT6k5p3kiL76gA+nFKVhPQkH60rsqwzyWk6IpHrvNP+wW4H7xip9BShlUYGfzoLr1LAfU0XYWIHtIWGIi2fU1EbV16kVdCAjIcfgajY4OCSaLiaFt7eE/fBJ9jUkyQhAUUgjvTElK8j8RTzcZ7AUXY0l1I42LDkEYpHto5BlQQ3tQXweMClEmO9O4rEaWkgGPmP16VHJaSckAY+tWDO2chzR5hI60rhYqKEUAPwakNvI4yiZH1qQReY44zmtO3AjQAcU7CsYDoVOCMH0pgQlgAK0Liym8/zCQU9RQI1A5UCmSYS4OcAk+1ABHBBBqRJBCSFx09KQtuOSOaoYipzkjimvgNx0qQsoXGOfXNQt14NAAxBHB5pFVie1CgscAZpwGCQc5oAkSAEHnHHrVc/IxQcgd6nWUxuCOaUIJW3BQCeuKAIUWR2AWMn6UrIyHDDB9KtcpE2HIA7DvVXcWIzSAcgKMGIBGauIYXzvJBAyMdqgkAVBjqRmkTjnuaAHsM8VZh3BQBVdcE1ZiIGMkD60gLUZwMH1omJcZHQdaVUyRjoahmkCZHWpYyvHboS5GADzUBXBxyafIxAyOvanQq7xluOB3poQ61jLSiMFdzHGM9Ku2+mDEivKRJkkIBgfzrMt2kS+jkHUHitK9v5I5leMAMOpPOcVLYys7SW7GORAUHrVaQqzEgYBrVmMV9a+cuBL3BNV7e3eN1lCgheoPTFCYFRHBGCaCwFT6kR5yFFAUjjAqiWqwHsQeaWNwDwcZ61EzEjFNB5xTsIsuqhjyD9KaoLMAPWm54oDYpWAsKis+M9e9MMYEpwc03ee1OVwOuc0rASh9gxmhm8xs9TjFRFwRinKxA4pARSsRgEYBNMGAQRzzkj2q0FypPpUO9cDghgeT2qkAhd+cLgH2qaMMyYJGPShHRIgXUlh0YdKnjiVodwOCTnpmgCs6hR8wqB3AU7TVqRRICCcHsMdapSQlSQelAFsRzvGHK4GOM1EWwcE80eaQgAY8epp3nI7ZOC/wBKAEw2CQOPWmZ5B9BTmbIODio8+ppiHFs8DrSAE5wKQEDrT40eQnGMYzycUWGNGe9KGIPAGPenHGMYOe/tQACDzgjtTEaGlgGQEc461vmQ29k8o/hUnNYukRlUOR1Jx9K0dXkEOjMveQgD6d6YHLICCCD3zS5zu+XJPQ+lSsVOAFwMU1ACQQeM1LGgFshjzvPIz0qqo2kjIODjIrTEahSA4ye+elZ2wrKwPAB60wH4wAPWnCVkTy1wMelPICkIxIBGcmo1ABPOfxpDsRk5YkgA+1PhkCTIWJwDninqils9SacbcgqRg5B4B6UxE9w0bxiVVIJPeoofWiRWSEDJ5NTQqMgAdakdhL0EwKgGW64qhEwWTDHABxmrt6+2RhjJxgH0qqlpJIhYY4GcE9aYF6El1zECV7ECtK3tiYo5iwcHOSOMCs2zjkhtlxIBnrg9RWkt0G2riTaAQcHj8qliaNK6M1rZApesVcfdByc9hVWbWbuSAQxgJGEKsH53D0rPlkk8oqmODkMTzUMMjO21jkGjmZNjS16dX+wXCIgZkBJKkMSOx9vSsy+NvLb7II1L5yZDwxPf6Cmajf3F4Q07g+WcBQMACq6SYqk7jRAQRkCmgnOCOtWWw6MoHOM/jUOzPTg0wHIoJ+lSDBJTjB9T/KiKMjJxyB1qPloi5IBB6dKQF65ujcQxRyLGFjGAEGMn1NUwrHoKhDY7irCSAKBihDGsPLAycknnFOUgAZz+FN5dwCMZ705QQc5GB3NAkiXOEGK1dLvcvBbICQeME/L3PIxUNgkJtw8kO+RznBHAH9KsmEuVCoiAHjaefpSuaKIkU2JHtyjABiCYxwPxp1zJcTuqzTSTIhyoc8ClCeWoAABHpUbAgHJ+bBIboBWbY+QJIwEBAwT3JzTOQSCuR/OoBc+awAYk9smnTXGwEEgY7VAWFYRkExqVAHQnJxTA4XOOAelRq7sA6DgjOfUUIQ7gEgZPJP60WJHOyqyEBg23Oc05RmLzvcjNPnjJnUxj92DtznpTnLKCmQR6gVUdyjSnB/sOIKMksDj8abDqRW1EVyhlMbAKQQCB2xUsxC6PHk9MY+uKowtggkAk9SauTsSka9leaam4zTTy5OfKZRii3tYrjUBcKcQr1jCEAk+g9hWfGUFwHYEn0HNMu9Vawu4hDvJA3Oo4Bz0B/WpTuKSsReIdMtLa5gNpIywyZDJ/dP8Ak1lXV0ZZEBGAihQM54ApLm7NxKZGGHJz1yBUYKkZPX1q0iRrSKcgrnI4OcVWkBDHI4FTuQSpQEDGeajk5JJ781SAhAyeuKXFA60Dk1RQGgCgikFMQooJoBxmkNIAzQDRiimA4GtGM7PD1w2PvzAfgCKze1abDb4bj/25j/WmBnZ4pDkClxgU0mkAuaAKSlApDFzTkYgc9qaBmlJwABQApwzkDpmrERVYyxPHQVBCM/nUhIKYJJIORzQBMHD3EUYJwx2n8TUTRIsjhnAAJAP0pkZxLGScYcZ/Oln4upgeoc5H40rEjDnPJzz19aVSeg/GmEd8j6CnAheoByKLAPbptJGR6UiIHcKHAJ55phH7wgcDNGCDgEZHemA91KuRgkdjjrSAntTyxKAByeeRTNpIbJAOcD3oAQdKeEO0Ed6YoGOTg9/pU7GMRMQCcDA70DQwnAA7gc0KeaZUiA7C2CQDg0MDQtJ44UO8qM+ta4wygg5BGc1jafGs06BxkZzittiFxhT+FZjGAc1W1MbbJmHbgj2IIq4Dn+FvyqC/AbTLjIPC5GR3oTKINGwbJgeoc/yFXouHl+o/lWboRzBMO+Qf6VpQ8SSfX+lIB0g/cSfSnPGn7z5Y+nrQRuUj1pht1xlnKj1KjApoTFVECwnEfI9aiiGUj/H+ZqvAUlhBUk4JBwM9zSSKj4DzkIBjYhwev6UxWLc0tvEQJJlHqOprI1a6jmjjijbOHBJ6VaHlRn9zGOf4iMn86zdTcmVN4yw5B4GB6UwDS3CX25jgEEVvhcjK9PWuaswDNknGBkfWtWJpkAKNwOx6UijR20YwCeAQM81At0VA8yMgY6g5zUy3ELjGQcjGDwaLA2ZM80sk+8MBgZwp4qs6yOBhCPXPrVq6hS0+YOCGBAAHQVWUgrkGhol6kfSrmnIZr2OAAHzDsORnANUGYBiBzUtsXNxGI/vEgA9gcipaFY2bvSTYTC3kuIzIxJUdMjjFNsLprBpFMIkLMAcHkD8qijutuovNqKtM6ggHPAPaojeiWaQrCqq5BAzkADtQI9JtFE1lFLHIIxIuSoIIBrF8UXsA0Wa2F4j3AZXVQcnIz0Pbg1y0AuZFJWUBM4wH2iiSzkEMmAn3D0cGtIsdjHmlEs/mDjkHHvjFb/hzVodPjmM1u8jMcBkPIHcVzAODWxpkcjWxKrkEnnIp3EdV/wAJTbZ4tbj8SKt2+u6bcbQ0rQue0gwB+IrmBBMf4P1FIbWQ9UH4sKTHY7hYIZIvNS43Ie4ORTjZiS3JSbaT0cHgGuFEE8WSmVB6lXH+NWLTU7uyyI5gVJyVkwRSTFY6m1M1l5wnk80AblbPJ9f6VaXUGktFZYVO4ZAL84744riJtSee4EoLRSE4LKx6VdW7uorVnN05ByDkZwaV7hY6uOUSIrjuMkdwfQ09W35284OD9a5/SZInhMKaiwkY5K7RnPtk042t3IHIvX83GVIOAT71SY7G9s55U/lUcseVwrFT6isTGtRRLtu5DzyA2cD1olOsLOsL3HysMiUDIBxnFAcpreW4tJEkmd8jkk4rnBGD1qe7l1u0gM0l4pUHBwAT/KqsN/qclyse6KUkZCuo5FK5XKTxDEgXsaxr5Stwf1+lbSx3RnUzRCNycgA5Bx6VlaoGjmyUJ+lCYNWLlofM0cjHQ4rMIIJFXdKuY2sp42YJhsYY4PSs+W5MbkBAw7Ec5pDQN90g9xiqx4q9bW15qMTy29vuVOvIBJ9BnrVGQ4IPvjHcGk0FycRKwBI5xUqqEGBwKqG7ZMKEU475qyJAyg9M0irloc28g9Qak0pVW6IAxkGoITuRx6g1DpNxIb9FJ4PWqJNa4uJhffZ0MCZXcDIOvtViIOUHmbdx67elU9Rty97FN9nNxGFwVGAQe1W7eYyIM27w7QBh8c1RBTmsZbWESMUKk8YPP5VV4zyAfqK1dXkdrKMGNcB85/CsA3UazDcfl71lJGqdywZIVOGZAfcUyQowDRupIPasyaVTMSDwTmpEkhKhDMEJ74zipsxXLYlx6D6VILhiMZFZ0pCEGObzQfQdKIzICCFb8jSsw5i+LiIHDsQfpmp7ZBdTLHBKN5OQH4B/GsmRtz5II+oqxZSGO4SQdRTsw5jQ1C6ubSYwTIoYjII6EVjvMXOSoIPUHoa7DU4ItS0yQzIpYJvR1PIOM1xVlEZr2G2lkMYJ5YdqskdI6schQp9BTM+taur6LHpvlzQ3ouFckYI5Hv71mrBLL92Nj+FNCaGgL3OKcGHY5qQWU3dHH0pstrLGhbY5+ophYXJIwACPerEFx5CAKhJ9QapDzR1jkH1U09Cx4KsPwNSFjdhu4ng/ekhz6igXZCbBICvoUNZ9tGRw6EemRirYUDpTGmWBfjbhmyPZDTorqNm2D5QepIxVcAilKgjBANSOxqqyKmPOjP40okT/AJ6J+dZBgXsMfStWG1hksASgBPUii7CxAYTv3I8Q/GlaGVjk+Sfo1VjZckBgMeopotcnBI/OpuwLiwOOqr7YbNVbm5mikEZgVlHvimvaYBIfp2zUaxGSNiD0OOafNILFiOd2IAQDPvU4SX+4v/fX/wBaswxNECTKD7AU9IQ4yJCPanzMLF8xyHqgP/Av/rVG8LlSPK/WqjIIiMuTn0NTJamZMxyZ9Ru5/KjmYWHRmUx+UIw6jsWxTxHKUK+WiAgj79Z5UiTYeorSFnHNZMDkOQcEHkGjmZNjl3t5LK92yYySSCDkYpbv5ZQ3YjJNVyGjmIfzCynByOtXLtgbSNxzwM1bdwaCEgqMFT+tOLhG5j4HcVUtz+8HDflWmggKgu+w+hBqQGCWNiArc+hGKu25GRk1W8m2b/louanjAGParQmRXwwSR6VVit3kI461pyYLZIzQjoSBgD8KGERkVi5GAFHvkVDNoUjyGQShQeuK0AiE5Ip+4pwD8vpSKMyPRQhB87OPery24VQCc1ORQF91/OgZXcQx/wCsO0evWq8kenvyzqcfhU16uI85B+hrLc0wuSzGwEZMYJI7g1mvIM9eKkmOVNVCDmggkMnPHSlMny9CahIpybjgKQD2zRYZfsWiVwfMbPcVomQn/VuA3vWZb2dwsgdo8AdcHNagdOvlrUSLIZBKTiUgj1FR+Smf9b+lTPIkh5AA9BTCYk4xmoQmLsToCDSfZ0YbhIAfQ1C2DkgYxTGcCqRJaZ1QYCgEdxVZpN3NQvISMbjj0qMSsDwePpTsBcjm2dUDD3pk0oc5RSB6Zpm8FBxzURYqaqwXFLMOhJoHnE5I49abuoEv91j+FFguTlwPvDI9KcJQRkAj61XJ3GnggLz0pDuWhduBjIP4UoupG/izVBm+fIPFSI/zdaBXLqs/cnH1pWeIrhgePfFV/Ox6/hTDICc8ilcDIJweevrTiGAyBxTpIQmWJAycgU1mKwkA4yMVohC5BXJqMkBqRZAI8HOab1oAduIGR1pyyHOSSTjvQAMEGmHg4oAfkE8jj1qUMwjOzgEYpGiAtie/aohMxAU9BQBKpO3BPB600gBsDsacrACh13kEHjv70NDJVBkIJ5GMU4xANkdAOaaG2IBz+FOyTkk9RipERQkmcAcirxh3jIHIqpbqdx9jWijYIGCfpQMchaC2Jc8jpWa0xdiSat6ixEIAPLHArNyRQIkLE1OJRakDgow5yaqEmnqQ8ZQglhyMUWAtBMS71HcGo7yTM3PGQDSB5I05PT1qvNKXYE9hilYC1HOFxtUAAfnU/wBr/cnJCknpWUJG9aXzG9adgLM028Bew6Gq5OaTcT1oBFVYANHegHmlzQFx2TRmkxRzQIeHxTtwY1ATzQTQMmzgZpVlIPQVCDnqcUoIz1pCLQcnnpRMEkgwxwc9RUQPOKcDnrSGWYZkFsUAxsHGRmpYZ1SIIx5PNVVOAQO4pRgsAOopXGOdh5hxUFw+CExyRmrD844FRlHkOPLBx0bvRcCsrhBhhkVKTC4BUbSKSSPHBPPoKbHGCSCQPqaYCMTjII/GoScmlkBWQqfwpYwC+D6VSBChCxx27mnj93wMn8aCMdKkjAwSRnNDYWH2w3B2JAAFNJj2ErwQcUhIjBKjB+ppGUMFPcnpSEzf05SI14o8Qvi3toscsSR+lWLNMBQKyvEchOoQqDkomSPTJqkIyZpMSfKAAB0pY5wkeCoOT29KrsCWNPSJ3B+U4HFDRVyYyIQQCSO2RikLbhyTn1qMQyhxlCBnrSkhmIHTPFKwXFmPmsCxzgACnxhAuw5GevpTEhkdwuME+tSSRFAEY5YHGAOlFguPEeyQAEFe2KkjYBhnIGT1HWowCJBVx5hMgEjZwOCRwDUgVZyGmQgEc9KtWy5YVUIBnGCCAAARV+2XBz6A0MDMunJupMnIBwKkhMhQbSQScAeoqnISZmI7kn9acpYAMDyDTJNmMKUUFeSOTnpUDArKSSQOwqBJC8ZcgBh0BqdJt4xKmcYAINQ9AHGSLHCkEdCTmlWEvEZFBIHbPBqcmwwAIS7E4znAA9aRzCG2oGAHbOaAMmUks7MCCTyKVASV44zz9Ks3SR9SSM4wKrGTHA6+tUhhIxjmODxSeaByOtNkIYEnrUPWrAstMSBtyCByfeowCQQASfQUigk4HWpUwhz3pWsBGFI6jmlGRjHanO2WJNIH9KAAEl8k5Jq2oDKI2JGSM4qqOTntV1ZxGmFRd5GSxHQUmBoaY4IcuSckBQT2Bq6HwxGQCOuaxTcAMCvHrgYyaiFyUkLk5J61ky4zNuZ9kXmYLDsB1NU7iVbiNU3MuM5AOM5qoLsurAHgnNNRwTyeamw3MYZlhdtgAI4J9qjMpk4ILZHQ8ZpzRKSSwDehHGfrUh+bGAMgYFWQmIhJwCcbRgjPQVIypsynzDOOlQBvmxkD39ae0oC4zgAdqVgvcnhjMTCSQYP8P0qQlmYlsYPYVAsokjXJPBz0qUS+Y5GOgzmhbjNi7/5BMI9WH8jVISYIAXGe+eKtak4h0u3yf4h/Ksk3qPlQDx3NVMcTQEwjPzqQT0OOKjvZ4jH5iZMyjAAHBFVWvGAwCAPQjNQtIHRnHVRj8/8A9VQtCW7kDMSMFVH1GPyqPae1OIxjPIPNLkgkDg1oiRuWIK4HAzx1NQkkjJBHsamDgkYA4GM96jJycdzzmqQEBPsBSKctipiiAHPXtUABD1QxzDmgUoALc04gDOD0NMBh4puacef8abjkCkAopcUAYJ9qWgBK0LlsaJYx46sW/LP+NZ5FaWpnbaach6iMk/pVNgZ5JxzTD1qQHnFMYYapAQU4UlGecUwHjjmk6sT60dqB1pAOyRwBkn8qUEkBQBnP500nJwfXikBKnIJz60gLCqAQXHAIz+dMuji/uP8AfNDynYR7g0t//wAf8nuAf0FAEAOD1xmlIJH0puKdk4OcnimIejjgdcHOD3ocEsSRgGohnFPHYUAPU44HSnMcEZAz3pg60NkHBOSO9ACkjrTkAIOf7pqPrTskL17YoGSPGFVMDqoJpYzjPoeooEhIAJzgYFLCMsMY655pAXraOY4eEc9q2YXZgQykEdQahtdxdCY4zhSRhcYPFWPOf/njD+oqBjz0qnq0oj0uQkZL4QD696t+ef8An3i/76NZutS+ZYBfKVPnByD1xSRRBoGf9J+g/ma0DcwW7SGSQAk9AcmsfTmCxPvmaNWPIXq3pVqOQq2baFRnpI3JNMC2bq4kTfDD5Kf89Jjj9KoXF1a7iZpJbtj1AOF/LNMmWIykXVzlx2ftTQliP+W8f50wEhuLUxput5IWGf3kbHPX0zU6q5AFvNHcY6jo35Hiottl/wA94/zppisjys6A9iDgigViwJgrFJAY2BBIcYHX1qnfszlATwQePcGrZluIVKufOQdpOao3JhKIYojGQTuXOR+HpQMS1GXA9ya0gSBgeuf5Vm2+Qm4LkitKzEt6ZVSPDIAQCR8xz0oAcWbcOTgAGkKhuSMnOc96ebW7U/6lemPvj/Cj7NeFTi2z9HFVckidyiksQRjABGcVmN9a0WsLt1I+zSH/AIEKhbS70dLcj6sKliKytGMKY8++acpCMCgxjpTrixuLWJZZkCqTjg5Oar78dDSaAus6PbOCMOcYOfekihDwgk85INVkbLDmrKuwAVTgdfxoA0dOtYXgug+N2wYBOM9aS0tY10y4aQrnBxhhkjFUzI4GFAJ9MVFI9yEYm2YDBydhNNMozq1LQLJYEHqrZBrKDZFX7NgsRyM5ORzTuKxvJaRboCR8vAPPY1VTTZrvWTDbxlgso3MOgGearDeQCEBB9TzW55x0nQwsIJuJgCSOP88Gs5SstC0rkkvh2wjuj52qpHuOdiJkj8Sf6VWl0/w/G5Q6zNkf9MQaxS8sxzNgN7mopOpFRHntdhdG/FpWlXCBodXl56B4QPz5pJdPntpU3MJrdj9+M5HHr6da50ZQhlJBHcVp6fqEwdbd5GIbuT/OlJTSuhppux0NjZwGdrjaQ8ZwMHjkelXwWB+WqlgSBcexH8quIa6KV+VX3ExQ0hOD0o3SgYGPxFOBFITnsa1JEcebGUlCuD1BHBqtd2iTRbIwIiDwwHIH51apD1oKOcvoJLIp5lw0hIJBBPH51W1K0BaOc3LneB1GefXrWjrrxo8PmAnIOMDPpTLh4PsETyYKjGCRXJU0Y90zBe0GM+cD9Rj+tNjt5oyWWMFcYyprT8+2PSL9KdDNH84VcZHpiqJGaOb3zpBb3BtwTyCAQTz61WewlE0gd8ncSSO5q/p8oe5AAI+oqO+lkjvZVBwAemKaZJWFmB0hZj781PHE4GBCAPcVEbqQDjA/Co5LuYRZDZB6cUMaZbhUJKV74wRVWI29vqUJQ5JbH0pNLl3GXeMtkYPp1qkyGO834xhwc/iKok6bVpZLcwlJHjQuQxTk4plhcNJcmMzSTQgZDOMEEVavrdruKMIRlWD89xVg/MaoEZV6ALt8s3OOM8VEkQ6lB9cVPqccXnhnHJHXNUCZAcLKQPTNZyLRNLCxfov4iovsAfkiP8jTikwGftGfbNN/ed5G/OkA6O2kThDGAOwFPMEzHPmAVAeBkuceoNPMceATMSeuBSEONvJ3dCPcZqIx7T0H4DFDqMAoSR0OTTQMHqaYydEUAEbh9CaSS2ikIJJHPQ03cQODTC5xhjmgLlt9Nje0EkeQQcnPNRrERwHwfrRFcPGjITlSMYqF5I+hFAXLWxwAPMNRgNC7AOSGGcmq8ZCEEMQM8jsanlmjkwQoBA6A0XC5IJD3NJvPY1VNwAecgetSgmmMm8wnqaTcQajBNNLgMASAc0DsSmXHWhZCabfuuI3AAJABxVPzmHekSXvMJ7015JQPkOD9TVF5sHv+FHnErknJ9M0CuWhLddzn3JFTB37k5rMN2T0GD70ouZD1IzRYLmkxZu5pkcrbCCehqoLhiME0qyY69KAuSuSQcUIJggII+lMypOQxxTt4UYyfxosNMcl35g6Dg8jrUouTGA6qSRVYwhE8wEEHnikDq3BosFyfzvOl8w9T1qZHYHO4j6Gq4j8kDByCeKkBplEpIPOAT64pc8YwMfSow1AOeKBD847L+VIXzwQD9RTScHmkJBPFAWHAj+6PypjfeBAoB5xTthbpTi9RSWg98EA1GeBkVZS3Mo27gPc1HLAYDtLBvcVTJiRCVgBk4FRzS5GVOfTBqK5ZsBQCc+lQRBw4649DWVzQnF4wwHUg/WpBKpHGOaryRFmB6Y/WpY4wACTn2pcwgckiojGSM4q0FDDAFN2GMg0+cRSePAORVN8Z6VugwvjcMmnG0tHAygH0o9qBzRxVuGyjmQMZlGe2a1JtLtpRhcr9KhTS9uF3A+9V7QEhsavGu0PuA9801g5JIRseuKsrAIMAEkDpk0x3kVgU6DrU3KsUvMPvSbnbgEj6irhukkwGQBvWkco2MEZHYVRmyqRIg4YfhTGZiOeverTwkgkEfnVRmAOD1p2ATAPcfjTxEQckcVGWHaniVihXPFMBXGOOKiJwtPO41EVOcYpBYOGHvQFOenFKo+bkVJgdqYWIihJ5NOAIGM/nTjSE569RSuITnHGKUZBzTdx7UZz1oGP30bhnBNMOCOaMDPHWnYDOJJOSST70hckbc8UpGTinCEkAnAqhEQB61IKnS3AXkimiAljnAHqKAGkEEYINOIIGTUoRVb1HvUUhy2O1ACZLfL3NMdChA70pU9QcU4kEAHtQAISQAetSdBTIjl8gHipXw3SkAAFhz2pwUk5FCuoh2lTn1p0ZUnAyfrRYZLFgctwPWpTLKUxEhAH8Z/woHlom6T8BVeed5TyxCjoo6UWGQlC0hJcsQM5PaowMsfY1OG2xtzyaqgkEADOaCSQIWIx07n0qxHHhC6IWAOM1CnJxkD61pWqk2UgGOCe9IZnl95PGCe1QSLjk1KQVJJA4NNlAaHfjkdKAIOMZzQCKQikHFUIeTSE80UhoAUEYozTRQaYDs0bvem5oFIBwIzS5FMopiHZBpw4NMFOFKw2SA85o3e9RZNOB9qLAWA+etSRMpPUVWB5p+R60mhlosAM5pQ4MZGeTVQsSMA0qgDnOD60rATxwLvJOST707yl8wggDA4PvTQ+RkMCaa7kHcDmkBBMsgf5wMHoRUW8g4AH1xU05JIJPUVEY2wDjk8iqQh0bBjg+lPDAEGoFODUgGRmgRM5zzT7U+ZcRrnjOarglupq7pkatd5I6A1QM6ayTLKD6VzOsTedqtwwOQDgfQV1VsAqFz0VSa4eWUPNI46FiRQEQCktgAZPqauTNlAFJJPBIrNZyetWY5CSAeuKQxZZWjTaCPM6H2FRoAFIPeiRg7gKMnufWnCFiQWYADt3NAFxIsQ7yeQM5qAkFic1P5mU8sHC+lVXUI5OSAe/WkwLYZSnIAP8AOmvkkAdBUasNqgk5J7VKASpOTjHBpDI1G6ar6N5du7dMA1UhGXBP41PcP5dk545IHNDAyUXMhz6mnEAAj3zV+SG3jiXzYyGcZBHpVYonYE0JiGxkAckA9s09WYEkg4x1xSqEHsT1yM08bWOB3H6Um7hYkVowMGR8diBSxupmIycAdcUzaAMDoKUhY4Tjqec0rCFmjE7gNIAAeM1R2Ek4Fd3p9lZTWcIlt0eQDkkc1zGpiOPUZUiRUUYwF6VaGZYjLAgD60hjITOcnuKm+UAgMAT1oKnywR3OKdxCSW5hihkJBEmcY7YqJzg+1X9WUpHaIP4VJI9KospaNQBkk4ouMCMgGm4xXcx20E12i7EENunKhRgsR1z7VyF3KJr6ZyoXJwABxgUAQJnB561OhQkggEEd6gAIHB5J6UoJHWkwJ+O9SCynmgM6RMYQcbgOM1VYnYa2ZHd206yR8wxhCQDwSTyTUtAjOW3eFQHRskZBx1FNDoO9da17DZySjcGY/dUDIBNc01sryMzEbmJJwMYzSGyAMD/Fmm4lLHBBBGQTVkWa7QAQPXPU0jWrEAb8AcEg0ElVCWBLDI6jFSYBwNoJJwM0shRCI4jkAYAAohYiVDgkg5GOmaY0ySO0ulbJtZQnpin2xLucoVwcc961JdWleNkAAyMZAqlEMyCiwrlzXlLadagMAQ4OD34rBijuZCfLt2mAODsUkA/hW/rrFba3A9/5U+9vpdPtbOxs5GhVIg8hU4LsR39aU3YpHPSMC4bYBjjA4OfepYYi1q04fODh1xz9auaioF8+e4BP1I5qohKeYAeHGCKAHpp15NHG8NuzoRwemaqvlZSpGCOorZh1o2trFbrGCsYwcnr+lY8rLJKXJ5PWrQiLODSEgjFOZRkEEc8c0LEDknORwPSmBDmkNStEwGcEjOBjmmEFRllIHuKYCx28txu8sA7Rk5OMCkkieJAGxz0IOQa0dHRZDcZ6EAEfnVbUx5V2sYOQFBH40wKg69KCCCCBQGwemaC2QMcUgHZ/OiowTkVIR6UANIrS1kYltk/uwis3BMqJ6sBWvrIBvSCM4hAH5mhgZGec0E57800GnKpfACknOOlACUn8VPMUqjLROAO+KaRxkgge4p3AWpYozKSAQMDNRcVZs/vOfYUgISMHH60CrMkLSNlUJ4yfSmi3mGP3ZOemKQFaTO01Pfc3ZPYopH5VG6OeqkduhqS7Uh4ie8K/yoArg85p5KlfcfrTPbvQFbBO04HWmBatbM3KsQ2ADjpmrA0xxn94pwSOlWtEj32Tk93xVsDIP1NJgZ0enPLII1Vmc9Aoz+fNOvdDewhM1xdQr6IDkk+w71qWtxPZz+ZC3ykYdD0YfXtio9Wht9TxL5LQzjA3lsgj0IpJjMWGwuLpGktoXljUZyBgEfj1+lRxQvcSiKMKHPUMwGD+Nblnql3pNp9nAjmRB8h6EfX1rFkcXWoG5vwZd5JcINpPpjFUmFia80u9sEV7iEKhONwIIH1x0plsu+VQOMnFbt1r7SWiW9tB5IZOC+DgYwMCqOn2n74Pt4HapYjWtoWhjBMjOcEDOOM0otWlIAmaP6AHNTBcRp+FOj45rK5aIBYTbsC/YcE9BTLnRpLiLy5L5mGQR8oODV4f6wfQ04vzRcZkf8IzGVCveZHpsqzHpaWsccKSbxgkHH0q6G5BoY5nj+h/pTuBm3HhuC5uGma4kQnHAUY4GKYPC9uP+X2T/vgVtnhSaZuywHqKLgYzeGLYdbqU/gKB4XtDn/SJuBn7orVuphBEWB57Vnveu+f3uM9lPFLnaAsf2TD3d/wOKi/4RvSyMnzif9802G7MW/8AeEgAEbjxUyaugzlVb6Aip52BF/Y9na3UNvGHMdwCG3NkjHpU32Cz024QWzvl2AYOc4FNlvY7ie2uI8AQkkgnBOewqrc3Bu7oykYJABAPHFVzMDWe1iYkvPsPpigQQjn7SOBjAGM1kJiLJKyHIwTuyB9AagaG3yQC7N6l+v5VVwsbQs/m3+YTnsRT47UqSRJjPtVcaopUL5ZVh3J4H6VJDfZb5woHqDmp5hWIru0S4aSKYb1RN47HOOKrWeh2Fzp0MkkZMjjJYMQaW6vz5ztGOHTZ1/Wkt9UNvDFAsQIQYznj+VHMFh58N2aJlPMOASctnNUntrYq7Onyr0TPORVma9lkmEyzFAg+52NEl6ZYyXkUZ7AdanmCxXtcpLEYYyZJOqg5x+Nb5jBBBZ2DDkFjjBrDS+cx7Y2RWAAJxz+FOXUZwCruWB9Bz+FFwL40fTc82cX5Glew0qLK/ZYxjtg1RGoyCEoGIz1LDnFQGRHyCeT1Pc0cwxkojWZ1jGBnNaV9cR2os5XjMkZQqADg5rIkeEZVQ2T3JrSvjDNptsCckHnjpxTk9hwK0uqJLbeU1mhAOQw6jnIqjfX5vEVWRFYHIZRgn61ajaNcqqAAHHXOarXtushFxCoBx86jv7impEtFIE4wetOhZo51cjJziozIoOQT9DQh3sCCOtaE2d9DqLPUBYO4kVnRyMnPTit2G5t7iMPbyB0746g+hrK0yOKZriOYKVdFOCfbtV6ysYbEOsLlgxzgkHH+c1adjSSsXA6AAk4zTiRjhgPrQEJ7f1oKEdhj0xWlyBBt7sM/WmXEghtpJVUSMgyFB604qh6qPyoAAOQAPpSHY5a6ubq5mU3MSRkDCggg4qO1lEiGEvhAcdM1a8TH/iZQ/wDXOsqPiNiPvZOBXPNalw1uacyWuFBYkjPIA5+tQmGCJwwmGO/aqq+aFwR+tJIm5cEVNiCW2uI7e4EhIOOwPWluriO6vHkXIDY4P0qqLeTsuR7c00gqQPWqJJiF2gE8+tOxCEAIJOMnniohDIeCwII4INI6TL0jNACo6wviJMAnnFOeUFCQAcdc1XxLIQFX5s+tNCy+YVYEcc0wsdlC++1hPqlMiV1J3sDk5GB0qtaEyaNyTkRkDHByPeprJ99lCxOSRyferEUtaU4hYepFZYU9z+RrY1dSbUMBnawNYRkIkClSAfWpkikyyF2jk5H1pGkA6GqruEYAhwSOAelNLOOCTjuBU2EWhMO5GKY84HOB9BVQnBOQcds00q7MNiEj1FOwFwTZGRkexphmcHJ6etVslWIJJFPYnHXj1osFiZbg5wSPxpHnRjgsQR6VQYlJA2cnPOO9TqRMcqAMcHBo5QJ/MZMHJI7ZpPPBOT1pibclWYYBwc0rpGhGMEHuDRYCT7Rge1Ksm/lSPxqL5FG7II9D1oE1uRnJH0NFgLRCuuDU0bjaFBBx6VRS4VCeA2fWrluslwyiGEFj2BA/nQ0UiRm2qTWfI5Z8k1qTaXqRXaLSQ59CDVJ9K1NOtjMfoBRYbZXeV2wCc4oDE1KNO1E/8uUgPvgUDS9SPW0Yf8CosKxCVYckg/Sm8E4JxVn+x9T/AOeP6inDRNSbrEg/GiwWZUAApwIFWf7C1H1T8z/hSjQr89Sg+pNFg5WUykjNlRTwJVHIFX00S/AwHT8MmpRol0ykOQfoDRcORmaHz3oMmCTv/DFaiaDcBgQMj0waR/Dbs2SZFz2HAqbhyMhY5sh9BVFCTIMVuR6JcRxGMEkEYGe1NHhu7HKmMfU4NLmK5WVCW8gAjGDTRKo4JrRbRL4QFS0Z6c7qqQaRNdKSJ4QQcEE8g0JiaIvMXH3x+dOSRfWn/wDCMzA5N3bj6mp49BkCgfbYOKdxFZ5F9aiMwPGaty6S8RwbuI/QZ/rUY0tDy1w2fUDiiwFUOwYHNTJM3rTzp+OlyT9UFTPDGkPC/MO+apIkktbko2CM0txcLKSvlEY7k1SXOeKlU4wTVslaDCCTxTxbgru4B96laWLb/qQD6iqM12RkKMD0Jrnd76GtyUtGThhnHSonIDccCs6S5O8kGm/aS3VqFBiuX95/vflRgkZJNVEmzwSB9aHuCBgGhwaJLJJU5FTRykgDI/Gst7gkHBpYZySATU8rBGvLI4XAB/CkjcnGQfxqNC5GPMGOxPWnHzAMEgjsRTSLvYS4D7Ny4x71S84njJp05JJGTxVYnFWkJsfIc80gJxzSI5DDgEe9StGGGQACe9aWFYhYsMkHj2qFic808lgSG/KmNzQIUZI4Iz6GnK4UkHv3qLBB6cUmcmmJFxJMkAjIqQgEYI47VRR2UdQfwqdZflySaloZIUAPAppwOlIZAT1OKN49TSSYMjLEfjTcn1p7EHmmEj0qhBzRknik4oxTAXJ7UoPOMA00U8DjORSApAYYVKhBbBpoGenWnoAuCV596oCXBIzg49aTDsQFOAe9AAkILMcDsKnSVIlIABJ7GpAbLaqsJZZOe4NUytXrWYyXewgEH1qzewqIshBx6cUJgZAyBzTSMnFSgbjggio3wrYFUA5QUzzjNPRS59vWoBknJPFWYsmMhT260gEcANgEmhGKPnFRkOTkilhJMm080AXMrLEC3A9KaYXYbyAPYmnRpsOWGQO1MuJeDk/QUBcZIQoAABPfvVYKFYn1NPjBkkAFTGAEYPXtQBAr/OM1e0qUmaSMn5SenYVQkQo2CKmt5BbruXgmkA64wJmB6E81dv0hFkjooBPpWfMfMy3rVln36bHx0pAZzqByKZ35q7dKnko+CCRkgVSHIzVoBwwMmmnBOacNvc0hxnIoAQcGkJp33jyetDqF6EH6UAMA9aM0ZpMGmMdmgmkBpfwoAUUu0gZpUIyQQKfuI6UCIqUGgjmgUgH5zQKFAI561IIyRkEUALwRgGkBxTSCDjB+tDEEUNASBvoKeACMg5FQpVlE4AyKljIZUfBbHFO3AqFYDIHpU7rkj0Hao5I9/wAxIBppgyGRASCBxSYAGOgp4jOMrkk9gaEQkcg5ouIiHUitXSFzI598fzqkIeeRWvpcQWMkdSTTTEzQupfJ0qds4JXA/GuLCMV4Heuw1YbdOCAcuQBXO+QB0FK4Io7COooIIq/5PBJHI7VEYSaLlECA7SQORUkLkFlx15J71IIihwfyqSNVEgYgU2waIDI28nB60pkzknp6CpDCCSQce2KabcnhlIPpU3QhiP5rckDHFWCSoKZBFOjt0RACOQeTSPjPFF0BJCM0+4dEjRXGQxxSRcCkvI96QnBOD2oYym0zyfeJOOmT0FANW4YAFyR1qVLeI5yvQ4pcwGaGJNWIx+73chh2IqxNbJsJQc0540JHBHA6GjmAiBzhWPB9KjcqAMgdR78Vd8lfIIAyT3zVGSB1yT69KSkmFjoIvEVpCCQkhIXA4xzWBcXQnuHmIxuOcZzioPKc9BTcYODVJiHiQHoAfqKlhO51jwACw7VCAAM4qe0+a8iHbJP5CqCxLrD5uVHooFUA5G0jqpyPrV+/RXv3DdAAP0qvst1JG0E+pGTRcDpLTVLa20xkkkUzuCzHOMkjH9K5diQcipWjOCY4zke1IEB4I57ii4ERdvUD8BQHOeoP4VK9rIxygBBpRbeUpM2ARRcCeGFHAJUZ+la9hpcl9ATDIoZTghhgA/X/AOtWNas7XCAfczkmr4yOjsPocVDC5DdRtbknzFc5IyucAiokkGASeafe52RxAcAFzz1/ziqwQkkngUrCTuWfOYkhTgY796jkmIU8g5qqWBYgH68GkijBO+UkLnAA6mnYdhEDPINoJqyiSITlgB6Y6UoKpgRoQPYdKFkLn5UP1J4NFwJg46E89qmg5eqhd84Cg4GTgVPbC5L5hERwRkuaFuIu68cC1H1P8qp6lMHvSR2VV/SptTN0bqzjuvJ5xgJ7kDmqt6M6xLF/DvA/lSkUh1/IZb6RgMYwPyAqsQT3A+tGpSmPVLhV4Af+gqt9oOeTTESsSB0BpmQetKsmRyAPpUchAPXrVIQE0qygHkA8UzdmmkYOKYy/C6G0kO75lO7GOo5q9q1uDpqc/cjVhx0ycGsRWIJAOMjB+lbMl+kkYjkQMuwIQSOopXAXSYwtkXUghsYIOTx61l6uc6iR6IB+lXrOWGzhaNScE55PSonS3mnaSYgs5yMHgDHFHMPlMk0AVpiGyTkqpyccnOPoKU29k/AAUnoQTRzILGaF+YH3qZsZJHc1ZENtGoLK5Oex4NPMdq2CN4z2J60cyCxShXddRD/bH86vanKrahMBzgBPpipbe3t2nhZF+cOOP8/SluUtpLuckEMZDnBo5gsZ8Ntghj+AqyARwDiniMJIAASuOOKm8pUiDSKzH/ZNTzCbS3GpMAhBwSOpNOEqYwwJGc8jgVEiBssVIPIxj+tS21tLOWUIAyjOD6UXG2lqyKS1tZMnDBj3B4/KnQ2qR58ts5HOTVn+zpnALNt+nekOnT7SBuPuDmlzruT7SHcYsWDzIMjqAaVwyLkYIHWnxWVwvDRZGDknOTThb3YkA8tyg6BQeaOddxOpDuVhLk9afPFDJ5XmLnIOOenNWTptySTHbPuJyMrgClk06/8AKQJbSElSMhSccmjnXcXtIdyn5Edso8tdvpzSea/sfqKtLpepNGA1pL7fKaVdI1hTlbWQR+hGCaOddw9pDuVUunhUoMKpOSAOpqRLhjzgYp8ui6kwytlKcdeKki0jU9uDZSA+nrRzLuHtIdx8Mm5fpTZnIxjgdzU0WlamuQLRxn1NSvo2pPGR9nAPYFwKSkh+0h3MdixznmoGgZ2yBWuuhatuBeCMj080CpF8NarJnZAufQSjJpqaH7SHco+WGSPjkIAK0LEjaB3BwatweF71z+9mSJkABjUbyP5VdXw8UGGuJ/wTFDdy9Cq33QB1wMD86I/uA/X+dXBoUI5MtwT69KnGkW6qB++OPeoYaGaeCp9c0ucgGrz6UhQ+WHDAcbjwTWf9h12PhbaKQDuDwaC0kPBxQGzNGT0GcmoDBrq/e09T9BSbNYXrppP/AAE/40FWj3LTSdRWXq1/9khCI2JG6H0q1jWM/wDIJJ/A1HLDfuczaIXPrtJouHLHoYL3pNsd8juT3LAYqutwSAw4yc5z1roGgmIJOgnj/pmf8KaIJgQRoJBHQ+Wcj9Kdw5UY6XUgJBYEEYAHakiuWlmCEAE9SOtbYt5j/wAwI/8AfB/woNk5/wCYA/4RmiyDlRjpMwm2khj2waeZlWSQhsEqQD71rCzkHTQpR/wA0n2OYn/kCSfihp6C5PMxluic9TjvjgUxJt0uAetb40y9P3dBP4qasroWpuob+wV574NLQlpLdnNPLkAMegwPpUYmwhVWbk5zk11K+HNTP/MDj/M/40//AIR/V/4dFhH4L/jReIrw/mOVSZguM59z1pslwxXy8kY7gnJrq/8AhHdaz/yBYv8Avlf6Gp4fC+quCX0uBD2yBn+dJuInKC+0calwflBc8DHJzT96dcj8Aa7NPC+qMf8Aj1hHsQoqwnhXWQMKtuo9OP6UueJPtafc4eMgtvG4n0xSyJM5yquPQlDXcnwxrIH3rcfRgP6VNH4f1YKBmAEd8ihzQOtTWzOBWG8blonJ9QDUqWt2xyLWYkdAB1rvx4c1Uj/XQD6GgeGNTPW7hH4n/Co9oifb0zgzpl+6FhauG7ZwB/Orf2O4uLVYY8BkHJJ4zjFdi3hi/VSXvIQAMnr0/Kq1v4a8+HzBdRrzgjBNHOio4imjko/DuosT+8jU/XNXItBvEXEk0YPchTk11aeFIhh/tzqw7qhP9acNBtP49SmY+uyht9ES8SuiOZXQIwQ0kvPsgwfzqR9LsUhbKRbsHkgCukGg6Z/HdzMPTgUreG9Dm+/JKcerD/Cjnl2GsUr6o5OC2ixuuCAuMDJ7+tTBdOXq6/ma2LDSLK7kljuHZlTkDHfOKtjRtHjBBtw3ucmqjK5vLEUznBJY5wksv/ASf8aVrlgpEMtzu7cn+tdNHYaZC+6G1UN67Sa0I12j5YkHvtFXd9DneJXRHLnTdYZVZdQYZGSGjzU8Gk6v1k1KPbj/AJ4j/CuoWKWQY3Y/DgU8WDn/AJaL+AJquaRk68mjidR8JXl9MJX1CIlRjmMqAPwrE/sW7W5a082LeDgEAgGvU/sCjO6Xt6VyF1GLbxRsdlKkgkj6VLZtQqN6MwD4V1UAmOWB/wDZD8n9KpTaXqltxNZSr78EfnXoLXenROCZU+uc1FLr0ADLHEJlIwAw4phOqouxwcVpqR+UW7An3xTzpeonO63/ADIreDMSxVdoJJ2g8ClDMOpNOxk6z6HOnSNSJ5jH/fQqVdI1Ex7Si/iwFbxc+ppwc46miwvbSOe/sa/B4SMf8DFPGkageXERPqGArfyT1NB/GqD20inY281vZtDNjIyMqcg9aqwLd2aeSlxbFAScHrW5AodGUjOckVzqW6i9uIpLUykSH5weg+lNGkJcyL1/H9osnQHBOMGsmPRZpMATqSO5BrfMDvHkYxSQwFMknk0MJzSWhkpoEjt++uVA9lNWY9DjiH/Hyzntlen51phacV4qbXMJTk+pmHSYyCC2Qe2wUDSYx0kYfgK0gtIVpcqM7GTJodvIQWZgfbApE0G2VwxeRh6ZxWtto21di+aVrXM86LYnkxMf+BU6PSbFAdsBGfVs1fApwXNLlRFikNOslOfsqH65NBsbXOVt0U+3SrhUE0CMGpshlWOzto8EW8RYc5K1MIrMsTJYW7H124qTy6PL9hRZANMenkY+wRj/AHcUggswwKZix2CCpNo9KNtHKguAjQYK3qqexZSBVhY7jZuivrdz7MRmq4XFG2hxQvmTl9YVchFfHpgimfatRCgvaLz1JxTFLocoxB9RViO+uVYFpN47hgCKXIg+ZWuRcXaBWHlHuVOM1XFhKP8Al7b/AL6rXjjsL25CsrxyN+A/OnXWjXNuflMcg9Qy5o0TtcpSklozHltbkACO6AHfcRVZoph8pvufQHIqSaSRJXUkjHB4qGqsdVNztqxUhKjm8OfUE4pwDA83jkegFR5FPEbkDCMT7CmaXY/zDHytxKPwqtNPPJkNO5B7ZqaS3nKEiF/yrFlNxayuJVwCcg571nJGkWXd9yFAEhwO2elNaWQAnzHJHUbiKojUHBwAKfNJJMgeNcH0z/Ws7FXNO3eGWMFlmznk7jTYUiFxIGDYycA1Q064uBnehHOBnirLSbZS2K1RDVy+5tYxnySfoTVeS4gGSECj0PNJDN5hxlQewPepzEjcSeWR6VjKTvZFcpBDdQytjap/CrBCt0AA9qz5rQRy5gGFP41chmKLgxb/AMaqMiXEf5a46UjRqY247U57iQkCOFBj15pBcXAwCsQ98YrVSRHKzOGF4PUGnnkZFMkUhyMHJ5wKeiuV6VopJoTRI+1YMk4FZUsJmy0YyT2rSKORgqT7DmnxWt1M22GGT6EYrNgkc1Jbzq5zE1R+W4P3Dn6V2Y0TUJMB0QA9yc4qwnhyTJDzRj3C00yrHECOUdY2H4Ujkjggj6ivQE8PQ9HnkI7gAVOug6eCC0Zkx2Yg0cwWPMi1PiYA5OR9RXpw0TTR/wAuqU4aTpgPNpEMeoouFjzlHQ8k5NP3yjhCwHptJr0gW+mx8pBApHcKKXdaA8CAfgKTA8zYSHja5J6ZUikFpcv92FjXpF1Lp8kJWZ4cdsAZrA8+wsizNukUnIOcUJisc+mkagwBFs4z3PAqynhzU3UECMZ7FsVqJ4n04N5awyADucYqvceJTgrBFGo7EHmnewGNPpV7bybJocH1ByDUX2Ug/vH8setF1ql3dyZklOB2BqrgnqSfqaoC3Jb2yoSLrJHaqhCg/KSR6mnqB6DNDAEYoERBiDg4qUOMYqIoQc9RTiPSm1cBSwoDDHXikCkilCjpSsAu4HvQcU0gA8k0m4DoSaLALmlzTSc9qTJx0H1osIfuoDe9R5NGTTsMZkg8UpYkcmnmNSc8imlADjrQIUNhKUHPJ4puV6Hr6U4gY56UhCIxSYOp5FacV/FPG0UvyjOM9c1lkoOlKoAOSM0rDJZWRQQgBHY1ACO4JNSnBORwKjYelMQ3rxU0b7Ygigk+tRCKRzwD+FSorjJAPFAx2fkJ70tnH+/3sM1ERh8mrlkvmlsOBtGTnvQBMx8x+mB3FZdxIZJ2AHAOB71rSMIY3c9QOlYxG5znoSTQgLltGYk8w9T0olOBnvTUkJIJ/wBWBjFIZQzE4yO1FgHXnEUb45IwaqLl5FRRkk4xVqdxLahQfmBzim2QSKRppDgoMj3oAdeIIFSMH94eo9KljPlWgjfjNU9z3N4ZOx6D0FWZg7uVHIHQUhj2tzLGAXAjHoaElSEbNq7Pcc1YaB4YApUnIzkVnXQIwPWpAW5MT8oMVUpxJIpmCK1QgJpKUqcE+lNoAcDzSkg00cmnBDjOKAG0uaUqe1PiheQkDtQMj70u5hU3kqI8g5IpmM0XENBpykZ6UmPSgAnGASD3xxSuBIDS4bGQpx69qTHFSgs5VSxx0pCIQrH5j0p4UHrUjptbBPAHHvTQaLjHxgDqaczYOAeKhJzx6GgnmkwLAfPU80MxY4qLOB705SM80khjwWBxgipQMAGogxPWn7uKQDiw4Fa1ipES8dawySXQD1rpLSMBAPSqiJlfW2y0MZ6AZNZO4Crmtzhr8qvRUAxWU0hBoYE3mfLjJNMLgdfWq5lPc0CTPU0rATM4JzmkD4qAv1FN3n1p2AtCUjn07U8OCdxP0qmH5FSBuaVgLQbJ65pGJ8wYFMQ55705QWbOeKLDLCDJwKmdyZNvAAx1qGM/MKmkkjXr/wDrpMBDtGSetNM4HGcj6Ux5UwTn6CqZm+Y4qLAXvO59sUiSAnnHtiqYlB68j0pRJjvRZgX/ADcjgYA4xRkMeehqoHJGSTip0kAAIAP1otYZMY4yMKhJ9qrPaSk5W3lJ9VFX4JCuGQ7T6gVcW4uCMC6KD0IqosLHPtaXHaCQ/wDATUlpa3aXcb/ZZCAcHj1rbYyn712T9BTShP3ruUf7oxWlwsZFzaX8lw7/AGVyCTjHPFSRW98q4jsGPqSvNafkxgf664P405Y48Y3zfmf8aljsUAmo4Ia0k59qT7FdyAZtnyOhPGK0AbdBgSXP4MKilkhAyr3LH0JIpDsVTZagmFWEj3DgGnHTbuRQXUZ9HkHFSC4VTgCcD0FWE2soJaQZ9TTFYz/sNzakPI0ezOMKckU4k1YaNJmJFzHGAcYc5NAsyeftEJHru60GbLA8O6lfrFdwCFIygCl3wSB3xipF8H6oU3tcW/0BJ/pUamURrGssm1RgAPgAVDPJfLIvkCR8DLfOeAaQ0ZwM7AhLdmIJGVTIpi292FwLOY/UYrajd9gBO09xnFNmMhiIBJPU5pDMoW9+elrIPqKY9nfFcG2c/QVqI0gXLkH0xUySEZJBIHoKZVjLt7K7mkxIvkr3Z+lXbe0No5jMyzZOcqCAKmaeI9bZT+B/xpI5IDMCF2L6BaaZLRPPax3V7BM8pjMeMADIIBzye1RTabbG6NwJpCSwbYQMVNJc25K7JEPGDuU8Go/NhP8Ay0t/yNDVykQvoVjeSPcSXMiFySQoGAahbQ9LRiDc3Jx6AVq29nLd58lrVsDOA/OPpikn0jVSdqW0Ix1O8UN2G2ZR0vSR0e6P4gU0abpefu3R+pFbEel6kgAa1hJ9iDTjY6mvS0T8ADRcVzJGm6SBzHMB7t/9enjTtJP3YC31krQ+z6l/z6f+QxVe4fUbcgGyJJ9IwRS5guQiwsB0swf+BipBa2I4+yID6ZrPmuriPAkWVSRnBQiqxuypYhyGJySQRU8zKubixacGCm2TnqASKRzpwl2QWiHHUtyM+1c6bsEkSOGPqKnS5iMOATj0HFL3irm1DPYyIWS1iJBxgLU4vLFRl9Lt8epGawIpCGwh2ntzirJuzG21iJPQEcn8ai0kFzYj1cRMDDZW4j7ArnFWLuW11h4g1sqeVnovXOPTHpWL50csRAyDjIIGcGpUuRsG0MP0oi9TOd2tC7Lp1ksiGG3VJAwGTmk+w2Zk3NZo577lOSarJcuk0LLkndnp6CpDqjsduRk9OK1uzncai0TNCNbeFdsdnCF/3ak86M9bWFvcoKzVu5WzhgccHirSTjaMsM07RMZUmtWWhc4GBbwgeyCl+1sDnyov++RVfcaM0cqM7IsfbJR0VAfQKKP7QnHQIP8AgNQYoxRyoLIn/tC59V/75FN+3XXZ8D02iocU7FHKgsiT7ddf89m/ACj7ZcnrM351HijFHKgsiT7Vcf8APVvzoNzOT/rG/M1HS5FPlQWQ/wC0Td5G6eppDNKw2mR/zNNoo5UFkISx6sx+rGggnqcj3p34Gj8DQMbhu1KHuI5FkilMcinIYUufY/lSj8aXKmKxftte1G3zvWKb0JTBFX08VN/y1sjj1BzWFRihxQHRDxPauuXtZQfQIDSDxFppPzxTL9Y653FKBRyoNDpB4g0kjpL+KUf2/pP/AE0/74rndoNGwUcqK0OhPiDSh0WU/RKQeJdOXpFcf98f/Xrn9h7Cl2nvRyoNDoR4l08/wT/in/16P+Emsx0jm/74/wDr1z2wGjAFHKg0N/8A4SeIH/UTY/3RTh4pg/59pvyFc9xS9aOVBodB/wAJRb97eYfgKP8AhJYP7lwPwFc9gUYFHKg0N4+Jo+0dwfyph8TnPy2tw3/AsViZC0eanrRyoLG1/wAJQ/8Az5S/iwpD4kJP/HnJ/wB9D/CsYuh71qW+i29yFC6nGHP8IUjn65oslqFh58QydrRvxemHX7jtaIfrJ/8AWqaXwvdW6Fy7OB/cUGqTWKKcPPIp9DHUqz2CxYHiO7QYFnF/38/+tSHxPfdreMfiahFhARzdOfolKLG2H/LxIf8AgAosh8jHjxFf+YX8uLkYwCacfEmokYMcdV/s9ln/AI+X/wC+BULRwE/LLLj/AHRR7ppHDyfQt/29fn/lnER7ij+2rwnPlRj6VSKQ5x50oP8AuilWKEHm5lI/3RTui/q0y5/bV93CD2xQNZvf7iVW2Wx6zyH/AICKCltg4nlB+gouifq9TsWDrWolWQFFDDBwO3501L25tx5aKm3tkAmqNwVtY0kS4MjE42kYq1ElvOitJdOjkZxsz/WouVKhNIn/ALXvxwPKx/uij+170chYwfXaKPsMA6XTD/gNH2KH/n6b/vgVfMjLlYh1a+PVIf8Av2KcNXvyP+WY/wCACk+xQ/8AP03/AHwKX7FF/wA/QH1Q/wCNHMg5WV7eWSNZJIyA7Eg8ZHX/AOvSi9vAcibHvsFNi8sySQeZg7iM4qcWWf8Al5Uf8ANZQktSpwloN/tG+Ax55/AAUxr28brcyfnU32D/AKekH1Uipk0a4kjLRzwt6DOM1pKpCKuyFCTKRu7wdLqYfRsU03V24+a7mP8AwI1d/sHVjyLUOPVXBzVaazurb/X28sZHXcpxQpxezBxaK53k5MspPuxqk6f6U2SSSOpOaveZ2BGapSEC8Oe4pVPhubYX4yW2hQRcLyD1qbYB2pLbiPnoCalqk9EZVPjYzaPSjb6in0cGqIGbARSbBUmKMUxDAtKVA/wp2DSYPrQA+Hhj9Kx9TjggvWZb5oGkAZ1x3rZi4Yk+lYmsCb+3IjDHHJviA2yDg0zWj8RbsZCyACbzQf48dakg3ZbP3e1TWkYWAHYqMwywXoDTxGFGB0oZVSabaG4oxTsUYqTEbilxxS4oxQAmBSYp2OKTFMQmKPpS4oxQAlAOKXFG00rgJnBoz6dKXbRgdKYCE80ZpcCggClcYmcUZoIoSN5pFjjGWY8UxCMwFIJAeik/QV0dl4ZVxvmckjoD0NbNvpVvCpDRx+xUVhPEU46NlKDZydnb27MsjyTI/YbMgfrWncRSrGHjihkGcZYY/rU9xe2kFwAVIQnHXpWjJAJ7P/R3QhxwabnboZuXK9Th7i0t43d7q4VMnJVegqs13pcOFjhMxHcAn9ai1CwvIb6UXKFBn7zcA1TMYU9QfoatSuj0YNOKaLx1gqP3NooHYn/9VQPql5NHxKkefQc1Uki80Y3ED2pFtZFBEMuPYjOaVyrDZ7i6jQubtyB1waqeR9sjDmXJP9481bIckrOIzjqKFS2JIjcgjqoPSjmGUDpcigkSqfarluVjjCMcMO2KlAGMA5qtM0atyh3D+IcipuK5ZbAxwee5qOXBjOSBxmlRlkQFXBOOhNN3RSErIASPSpuFytbyp5fzyAH0NEzgkut0BjsTUk8FqUOCPxqAQwyR7FjJI9TimVzF20ujJHktnB5qwbxF4zisi2t54d20cE5Az0qwFDDLrhx71DRSlYtveAKSjgEetZ6XRe4DyHJqddjkKOSe1Unt5I5SQRTSDmNWS5ijhEm3J+tFvqENxGcREEelZbwTvHnIGOxNJbmSEHaxB9QaqxNzYtpybkKSAwOcHvW++pxQwAu2SO3SuJV38zfI2fU1akvPMXBIJ7Ejmi5J1FrqQuJNo4HrmtRBkV59DfzW9wNuCPQiuntNbQwhponXPdOlUmM1ZruG3QszdO1VBr1lkAhhnvWBqNy8zl0bANZDSzhtmQc9xRcD0SKaKZd0RyKHjDdQSKw9CuCiKkg4PfPSuhouBReyRmyBisPWrC7MYMcQKc5Kde1dTgUtNAeayoYM4EgYeoNVpZpGG0sea9Pn+yiEmZEJ7ZUHNcbdzWZuXwkTIegAq7gc4I1JyRTtiAdK3bPS7LUJwAWjU/xKckVfk8EnrBek+0if1zVAcgyAcgU3HuD9K6g+E76I4Plv9DUsOlzWgJksA49SMilcVjl4opXJKROwHcCp/srsOYyD710BLQk+TbmNTztHTNQTSSTMCy4xUcwWMgWeB82c1HJbgNwK2CFxkjiq5jBOcUucLGUYT2FNMR7itQw+1N8k/wB2q9ogsZnlEnpS/Zyegya0zCrD7oBpn2YjnnHqOtHtEFjP+zMOqmmPEynG0mtZYwBg8/WmmFOwNHtEFjLWPIznHtSmL1q+bcZzgj3pfIUetHOgsZQRyuQMilVFzmRjkdhSZbsTS4PU1RA1kTJIzwOpqLcT1NTEECocYNAD1KA4wSfapMrtxyD2FMViBjAxThyKdwGEknipAhABYgZ7ChQAc4pr9c0gJ45DG4AOBU5kDEkFcms8MSRTiVDBmJGBjinYCVxknNMjLxSAoDnNLuMgLL0FSRuFcF+RSHYnvCTCAfvEZNZ6Llcmr0sokBUAY7EVXwBwKEFhhZsbRwO4xSYIFPPBx3pCCaYWGEEjJ7UmODTmUjtSbTgggjNIQQEqwYVo28qBmZwCD+lUUXb24qQE5OKGUa1vejOwHCGo7/TIrob4X2N6dqywSO9TG4mI/wBa351Aim0TxMUcEEHH1pNoqRiWOWJJ9TSbfSrTAhIycYppQA4xU4XH1o8sk5707iIFQg9KsqAVANKIiADjrTxCeDnn0ouAww5Bx0NSW4AcqMjjrmpFGDgEZPalAAbgAfSpbAj8oLIRyVPrTmRAhwAMdMipeD1pDGCc1HMwKwg4BwRg8+9OZQcnGAO3pVxYy2EUZBqNogpIyMj3quZjsVCSABgACkDMDkDnsalMbljhcilcHZgYGO1O4EBJY4JyT0Hc01g6khlI+tSi4kgB8lQHJyXPJpqvNcSgSNknuTTQiIZBzTic0EdR3HX0qLeB3pgyYHBoDYIPWowwIpQTSAn3E9OKUVECTxTwD1JNFgLNqN11H7HNdNaJwK57T0zcAntXSw4WKSQnopIFUByd/KZL+ds5G7A/CqhNOLNIzPg8kmomzikAxjkUgPPJp23PfFAjZsBQCT2Bpg0NJpM5qUWlwQSYSAB3pRayEAquc8EZ6UARDk4qZRjg9QaVrd42x1I7VMcEjCEEDkY70rDBQcdKmQ8VXDSEgCPAzySelWFiYnCuM+oFFgJY8mQD1NNkZRKQTyKaILiN94kBK8gHvSiBtxaXIDDkiixRXkfK45+oqB3A4FaAggRcEM31NTxGGO3wYQfpxRYDG35NSKScVspNahRi1APuf/rVKLyJR8lqM+uP/rUrgZaK4XGCfcVaghZ0zgj61M7icAghSeoJ4FTxNEi4luA5HZRnFS0A23Rkfawxjv2NWSq+cABk9hURuIU4UMR/eJxj8KctxbschwT6CklYB5QbjvB9uaAUfOwYx3xQ9zACMhufxoMkezcCVwc+tUMaGwhOOOnIpQHILBicDhSOKUzQg8ElyOMjgUhuIgMCXBA5ABoAUB3QkqMnsBTtpA5OKYGBQHIJIzwTSxGEZTcSScnPSgBCCoBL5BOOKX7Kj5MxIAHbrmpP3e/arjjoKQuD95wRnk9qAGNDGzYwflHtQLe2KEYKOBnJHWhtgJPAz6Hg/hQGkccIHAGAQCaBBEhEmQAVIIGak2Y/gQfTmnR2eoyLlLdiPUnFSx6TqDOTIojz1JfH6YpXAgIHop/Aig4HAHJrQ/saX+K7j+oBzUseh2wYtLNK5xjrgfyouMyMgdcfiOKbIYyMBzk+hreGlaeOsLn6mp47WxQALaRgjuwBNAHKyEJtEOXJIyCCeKtiCe4IENq5UdSBjB/GuoUhBiMIB6AAU7cT94j86YHNroV4wyRGnsz4NWYfDyhg09zx3VB1/Gtk4PcfnRkDow/Oi4EVnZWthGVtYVQnG5zyT+P41OSQOtNyPUfnTSCf4z+VJgOLeppfoV/E1EUP98/lUbK3ZwPwoAnyw6EfnQZXH9386rm1lY5EjA+hGKaLO5Y4Eij36UhFhpGYYKqR6ZH9aruwXpbKfwH+FIdMuSwIIkI7bgBSPbX0KMy2ykgcDOQT6daVyrEEltb3GfNsoTnqSozWLqmlaSY1+z7o5c5IQ5AAx1rZe91LysXGjxMo7j/61VjcwHmTRYgfU5H9KFIEjnZdKB2+RMQc5IcE5P8ASo/s8kFx++IITk4OQa2bsC4m3RxC3XGNq5wKrG0Rw/nTltxySvBFDdwaKjNuB2MfmIAxxkVbkMUdvmV1DewqCQLDD5FvEWPqTk/nTYw9zDsuF2YOVOOMioaIaJreRHyEYkk4wabJCVJZcooIAJGcHvQMWmVQq4PenpFe3CtuSVIWIIc9AKBWJ2OLISKudoJIz19TxRpkpN7GwjVwWJwwyD+FFtaT2zCNmiYFuo64PrWxZeFbqW7WWK3ZApBDbgB+RpNpEznGK1ZpyXGnPlbuz8jH8cJxj6irMnh7fbrJZs7FuRvbgj8BV6LR7W1BkvLgSuMZXgD2z61OLyRcCGQhOAKUJStocNNdzmZdLvoG2vbnA7qciq+1YyRMJlPYAAfzrbjkvP7TmEtwuwZYKepz/wDqq5uZuHww9COK2TdtS7HM7oOxnP4ilJiONiyZ7liP6VuTaXazKSsflyH+Jf6jvWCoxn6kVSYmhaUAZ+YHHt1opR0piF/c/wByT/voUYh7pIf+BD/CkoouAmPQYo5p2KMUhjlMOMPDI59Q+BS5tv8An2k/7+03FGKYhd9uOls4PqZDimMcsSBgdhnOKWigAx605NgOXBI9AcGm0CkMsRtZ7v3kE232kp7tpu07Lefd7yf/AFqrYoqXG7vcLjybfHywyA+pfIqOlxQKsBKfGUDgyIXUdgcZ/GkxRikBeFzpeOdMOf8ArqaZJcaeV+TT2z/12NU8UtQoWd7sdyQzWvawIP8A12NRHqTjA9PSnZpDVgJingCPBkhEgI4BJH8qQClx70hFyy1DTreYNcaUrjsVOf0NdTZ3nh+YkQLDBKRjDpg4ricCkCAHI4Nc9Whzu9y4ysdLq2hX5zcWNtZXCY4CDaf51zRubm1/4/bB4CP4sHGamS6u4TmG5lT0wxx+XSte38QsItt7bxXB77l60o89ONnqDs2ZEWsMPmivHjPpv4/WtWLW7ttoZYJVPdhya0WtPDWqwRzTW9vbhhn5W2kfkBWVqPh7RY42n0zVRDKo+VWfI/Wkq8XpsxuDLk19prQt5tk0JHVl5H6VRe3tZLcSQ3yrH23d6yGm1KBWVp7e5RuPlOCKcJmksvIe3Tn1P/1qpc0RQrOGjVy2bS3DY+3Qn8DQbS37XsIqt9odnVfssS8YznP9Kf5caD5nXn0FVzdzoWNXVEwsrc/8vsf5f/XqnHDDNfyQrdIFQAZI4J71IihmwiqrepPH8qz4NOu7G6uHkKOrncArdPfFCkjRYqD6mx9gtR/y+Q/gKabK2bpexfiM/wBaz5boQqDgP3wDzTDewlEOCSRkjI4PpVmvtC3Po0E2P+JnCmM8BTirK2UIQKdQhOBjOCKylvocZ8sD6tTlvoWGfLH4NRYfOzXSOFFx9vi/I0pW276hD+CmsOTUIfNACnHcZFB1C3/55v8AgKLEadjb22/bUIj/AMANKBCP+X+L/vk1hf2hB2SQfUD/ABpf7Ri9H/T/ABpWDTsWYYlXVJIWuARnIkA4/LNa6rCvDX8WP901z8MyyXDyj0zilOpW57t+VRBWbRc3zaM6JRaBgXv4uvoa2rWfTpQQNSiTHqhrgPt9uT0Y/hThqFuO7flU1aCqLUzStseiAWI5GrQA+uCKsLeW6Lj+2IWA/vcivNTqVrjo35UwajbE8IxrFYJLqLU9HubvRjEzTixnbp8ibD+lcTrMNhDPBLZsvz8OFJIzxWbJfhtvkjbg5O48EelJJdJcOgWMrg5POc1tGkqcXrcdONmdXp66daHzjHDOGA+Usep7kVbml0S74ktWgXu0TdPwrkWugjbVU8dyMVJFfb32sqqPak6HM1K9jKrF3ukdvF4U0zULYtp98+49CSDj8KxdR8Pahpb/AL1DIh6OgyMVmpe+UV8udk2/3WIrctPGl1bcTss8Q6qTz+dFq0Nnc57J9DApa6q31HwpqzbGia2mfjdgLz35HFV73wsq5ex1CGdcdGkAP51cayvaSsKVNo56kwR0p8kUkD7JkZG9GpAPQ/pXQmQIOtUtT8qN47mVZCyggFBnA96vgd80j/NE6+qmmmVGXK7ozLS7S+yIGf5RnkEcVpDgCqGhDFgF9GI/WtEjFNl1X71hMUYo5o5zzUmQmKMe1Lk9O1GT6fjQAmD6UYIPSne+M0An0wKEIZg+lGPanZORxxRkdwce1AxhxkDoaMe5p+5fQ/lQAD0GKAGYI7mjGT979Kfgd6MCgBnNHNPwCevPpRgZ4oAZg9KQNJFIksZw6HIqTFIVouB1ema9bTgRTHypB69PzraUeYu5CGX1Brznbg1NFdXVtzBcSR+wNcdXCKTvF2NIztubuvaFLMBLCOhyRWFb3l3p7jbK8TD+EnIrTh8UX6bVkVJUHXcOfzqeTW9KumAubVx67ACKUVUStNGc4KWqZCNeM0fl3dsky/gKglg8P3qbfLe0ZuuKuw6ZoV/za3skDD+Fjj+dQTeFr5CTDLFKvrnBpOaWj0IUJrVGbJ4SErE2GoRPH6Pwazrrw3q9spYW4kA6GMg1oT6fe22fOtpAP7wGRSQ6ldQ48u7lXHbdkVqpM0WImtznLqynik/0qGSJj68ZqCKBLdiyDBIwa6yfV7yYbZ3jmX0ZetVhPZMQJNLtjnrg4JquY1WKXVHNO8kZMkZOR2JyKprc3LkiQHBGMYrsWh0OViZLOWBf+mbkipE03wzJkCaZSf7xIqrotYmBwiR3CNkIGHvUqEvJzAVJ9K7F/Dls7Ztr9RnoDg/1preE75Y2dLmOTHTA6/lRdGntYdzk5IYUzK5Jx/CelS28ttK+0QlGP8QOBWq3h2/B+e33DuAf/rVE2mXcP/LjJj025p2HdFKSURAxlsH6VVDyE8MCM+lOu7CaSYtIskZPoDg0Q2kaYBuGyOxGKdh2LpjxEHQL5gIIIqKRS7CUnDYwVxxUqOQBk8VWvggXzQ+GHoaiwDG3ysVBwPWoVt5znaM1F5z9jViGaSJfMOD7A1dhFIO6thuDnpUkk4VDmpXaG7kBkiZP9pTzUX2cQsQSZF7YosBfttQhmiEcsYY9mA5Bpst88RKI+B9Kz0mSKThGH4VJLtmBKqMnqRVcoEpuhMcSOcnuKQm3jIfz3JHTAqn5LBgcdKsxxI65YUcoF+C7QoNjkj1q5Hqc6AAXEgA7ZrGURLwspH1FSoQf4gfpQ4lJm4NeuE6yg/VakTxM4cF40YdxyKxg8ZUguRn0qJ1XgoSR9KOUZvX+uJdWxjFvgn3zXLTEmUmreWI4BP0qNoyx5jb8qdhXGWd1cW8weJsY7V2NnrM08Yyi7jXJC3I5CP8AgKsRRuVAwwx+FDQXOvN3ckcwj86qz3c4TDDbn3zWEguk5W4ce2aeJrsna0pI9DWTYy8bqQDOQfYioZLguOUX8qrvJMq5AB+tVzcSN94AfSs2wJJiCMgAc1BvwelMkm4POartMT04osBd3ZXODTQcmqyytkA1MksZONwB9DSswJiAR0xSbQRgGlBBHHP40cE5HWldgRlcGkOAMmnNxnNREgg56U0ApBPOM0wk9qkRweCQPrTXUDJGPwp6gY4jYDkUMMHHanBiVz2ppbNdJkNPFJtB7UE800sB0NNDAqQeKASetLuJ60nmAcYOaBi80EZ60eZgY7+tNJNCQhFGGB7ipLeMSXGD90DJpgHNTuTHbfLwxPSmIrux80hQAnYU8qD905HrSSQtDGhbOSM80xWIAx3pFMn3DAA4xxSmRIwSeWPp2qDdgEiiMea+C2AOfrQInKEIJGI55wKUHJxwKCQXCjJHQAUrRhQTjn070DuIcIckZ9qaTkkkgU0k4JPQVCXBGBRYCcMD0IxSgjHFVgckCp1BC5xRYVxSfSk3cYppcDqaWNfNYhQScdqVh2BomGGOAKTI9aWSORWwQxH0zTSNoyQQPeiwWHDBOR0pwOKYhzUwjA4I5ouFhAcEEEn2Papgq9SDn2NRiPac5J+op4DFcg4/CkwsIRgk8UqtUDsQcE0ze2eDSsFi15meAMmnR5LfMMDuaZblSxLHAAzmkEzPJjgA9ABRYLFkHAJBIA6YpOWH1796OQhwcZ9qEyB659aVh2FEYA5NRmIZJ3Y+tSFX3ZEigDrkdKjJj/juAT3JFCQEToS+MjHrULxkHAPPtUxEI5EwJ9RQqhzkHI9atOwrFMeZG2QM+xqXyxNyqEN6VYaDd0XNI0fk4CjBIyaq4WKhjIbHFSCJV4Y5PqKuFoggDrubOMgHimMFVgFbCkdcZNFwuRsTJj5QMDsKao2sTknJ6VKeVIVxjufWljgkDbiOD3pAi9pyZBYdCTita8Yw6TMw4JGAfrVLT0wgFXdSOdMEXZiM1QjlY8+WADjk0vlKeprRFhCCSJhj0PanG1gH/LYE+wqSkZ3k8cKT9KaFKsCq4I6GtP7PED91j9alS1hckFSABkkGgZmI82cueBTSpJ46etawtIDwd4z0B7iniyjHAXj60XAygrr3B9+9AV3bGM1qG0AbCgUG0TjcBn9adxFEK4G1Y8kdzyBRsl4JQk/7Iq8IcA/LkDoKfGjSHBJ29h0pXGZgEhYYB/GpRFMyAMpwOgBrQKAgDGMHOAacYlIwGCnuTTuBmhGU48okE4OadMpVgoiYDGfWr+1IyAHBz1ApgELsSSQOh3UgM4RSE4CMPqKebe4BycEDsK0QYiSqtjIwSelRgKM7lyT02nrRcCkbOZwGEZ2+pNOW3ZTgYBNWnWY4EKyD225q0kdzIBGLOXPshFJ6gU/KZCVyAw6+1MEL5JyCfUdq1xouoOf9TEPZ3wanj8MzEEzXQjY9o+QPxoAwjEQCXYn2Apu+MZDIzA84PrXUW/h2zhyZJZpCRjJOKn/sLSzy9uZD6scmgDkRIoYExDI7E5qUg3AJEfznoUU12EVjZW+PKtYlx0wtTkgfcAH0GKAOTh06edQBZMD3YkAVYj0K6OCzxRH/AGjkj8q6LaeuSfqaQoT2pXAxToO7h7wEd8IQf51aTSbOIYEIk/3ySKv+XR5ZouMrpa28ZytpCD6gVKAx4HHsAKeUI4wc+lNww6qam4DfLJPJJ/Gl8sjoGpwzjoc0u+QdAaVwGlSOzflSlSP4W/KjzZCCSDkdqUSSEfcP500A3H+cUpCnqaUsQMkHFRm6iCFnlVVHXJFFwHYi/vgfU0CND/Gv51WfVrJASCZCO2yqza7E6kRWW49yTgCncDV8piD5Y3Y64OaVbadlzgj61zUs8txLkOI17rGcCk3XCf6q5kQ+uSaVwOk+zyA8t+VPFsxHVzXMbtQKlvtTtjvuxU8N1q8UWY5mK+5zRcDf8sx9UJ+oo81hwEH5ViLrOrxnP7tj6MKWPxBqbthrWIj3BFFwNozMepppkHqKzDrUwAeW0hKZwQvUVZTVLSY4+zTD64oTAs+cM84P1pwnXGNp/AVXa9sxyUlUeuAab/aFgODNID6FDTAtiaMdCRn604yA/wARqqL2yzg3aHPbBp4uLYjIuYsduaAJdzH+JG9mFM2b+tvEx/AUqtCwyJoj9HFOChjhcH6c0CIfLRTzZxfgBTXtbRjuNlCSevFaAsbgj/VN+IxU4tLa35uJlzj7neolJIiVWEVuZS2NncAxLYp83X3/AEqUeHrXCiWMrt4/1h6VsQQs25baDap/ikGBVtdPh37piXf0HAH+NZub6HNPEu/ulHTtGsYJBMtsMgfLuOfxxWhdySJFiPgkgEjqB6ipHdIoyxdeO2eTVI30jsSEBHqalRcjns5akC26iEu+Qx6s560wyIOEOT6jpT5gbhgXBYf3SeKhlubW2XbI6qw/hHJrWEbIuMbIWKMJk5JZjkknkmnSSRQKGlkCDsD1P4VlSaxM4xDGIx2JGTVBlZ23OSx9ScmtCjRutUMyGOBWVT1Y4yR7elZ560AHpQAe9CExR0wKMUopaokbilxRRTGFKKSl4oASl6HFJRQMKBQelJ3oELRnFFGKQxRS49aaDjjNLTAWgUUUgFpabS0wCkNLQPegBaSlptIEO70Z5ptLQAtKKbS0AO4pc5pvOaKQBtBPQflRtUdhQDzgUZ5pWAMAdqQgE8ClphByACBk45oGKQM57Uw4qzJYTxoWZ4zgZxvGcVJY6TeajEZbaMSL3+bBH1qW0tWFiirFWyKtqdyg1of8Inqp/gjB/wB7/wCtUVzomqaZbNPNAHjBwdjZI/DFZTlCWzFODtct2f8AZZgSO90yKYDq/OT9RnmtaPRPDt6AILaBH/u9D+prkLXV1DKs0ZUeo7Vqwzx3CCSFwR2I6iou0Qpyjsye/wDDJtiWi061dM8DkE1hyrDbttn0uOJvQg109vrV7bqFLCVR2fr+daUeo6dqahLhEjfsso4/PFUqnc6qeL/mRwu+1P8Ay4W5/Cm7bc9NOhP04rrbnwhBK5khnMLDooHymsG60nULEs0sBZF/5aJyDVKSZ2wq0pbMzgIRz/ZsP4saUGMcjTbf6d6GfceCM+hOKD6Hr3qrnRyIgbK3QmjtljGMFAOD61OssZ+9p0H1xxQVI604Lx1FRsVyoeJIhgiwtsepGaDMO1haj6CmgeoyPTPNJtJ6EfjTuyeRDzMOv2G2B9cZoMqsObG2PvtFNKc8kj2IyPzoK8Yx+I5ouw5EODxAcWdsfquKgulFwFEcMULA5ygwTUgGDS0bqzDkQiSxiJUmtIWIGCcHmn+Zaf8APlGT+YpuAeMjPvRtHpS12Qci7Ega1GD9hh/UU8S2w/5cIT9RUAAFKCB1Gaq7FyR7ErPak82EI+hIp0U9vC+5LJQfXcarnaTnBFGR60XYnTi+htRappjrsutOJP8AeL5/Slez0y7XNlcNC391jkflWJkGgEdqS0MfqtPsbZ8Nai0XmwCOWP1DgH8jWXcQzWbbLiJ0yO4OPzpsVzd27ZhnZPbORV1tYnktpI7hEkBUg9ia0hJ31MHhLMxtNRYhIoIwWJAx0zV1sE1i6c0X2+FZy3lMcEqcEV1b6RbS/wDHrd4YdmOR/OtG7mdSi73Rnd6Q9a05PDmpIoaNVmGM/KQD+VUJbae2OJ4mjb0YYqVJHLoRGjmkDKTjNLkdjVXHYATS5OKDSUEi5oycCk7UUFDs9s8UmaTNLQSFGaM0lBQvFFNzS5osIUYoPWkBpSaYBx6c0fhScUuRikMMDuOKQqh/hFL2ozgUIBvlR/3BVhbu5T7lxIuPRqgPtQD60nFPdBexbOp3rDDzFh7jNRyzLOP3sKN7jg1BxRxQopKyB67jJLO1f7pmQ+u7NZ91pty7DypVYD1GK1M0ZFLkRPKjnHtNRibCrID6qwIp/m6nDH86M6+pGTXQZo3VPs0Ryo5oaxIGIeGM+4OKspqsY53Tp/uEgVrS2ttOCZIUJ9doqq2j2bYwrr/umk6fYXKLFr8qkFLxs9hIODWjF4lvNm0mF/wxWSNCtieLiVfqoph8P44W9QnsGUj+tTyMXKzbXWYCf3lgQPVSCKcZtCvCUntdpb+Irj+VYtvot/FJuWWB0PYPgn9KufZLlD89u49doyKGpIfPNdTXj8HaVcIJIeV7FHrKvfBYhJ8qVhntIOP0qW3lntH3QeZH6gqcV01lrlvdARzbEkPGxu5qXKRosRU7nDy+C7iPDySJsP8AEoOKibwirAB7vA/3K7m/m8jIS3lKnqowRUMc2n3EBM0EkTerA4FUqnc2hilb3jkYvDEca7Tfkr3GypB4Zsv4ppD9MCuoOmo6FreVZMdun86ifTpkUs8ZxVqSZ0Rqwlszn08NaSh3MjufdqedE0kHi3/U1qlEBwRSbEzwKoszxpWn9BaL+dMk0S0Y5jhEfrgZzWpjA6UnmMOlAXMk6JGOiA/gKaNIjXrCv5VseY2KaCT1qrgYzaemfuCmnTV7Oo/Ctoxqe1L5aDtSuMxPsYU/6wH6AU4WRbrz+FbeEPVU/KlHkj0B9qdwMX7EAOVP4U02sfcSce1bheNRy4A+tQvc2w6upP1ouBimOEdVkqNhDngMPrWu80LKQFU59qo3EKMcqgB9RWbjcDMnKqhAIIrOmcEcVozwk5wCDWNOZIpCHBHoexqORiuMkbjINRCQ4JHPtUDsd3GB64qHznDDBxWqgwuaC3CE4IINSb0xkYIrPErHqacshBzmjkYGkrgjKnmpFkI6mqULh8g8EDORUmffn0NQ4MostKD1NQGQBjg0wKxPLCmyIQM5FTyMm5MCGAYEZppZgcMRiqyEkjB/WnMxIwSDVcrC5EqkgZ6Ux1UnAJH41axGq8uAKZutSMAMT61sIquEUcsAaBHGwB3En2qz5luDxbhj6k0qyJj5Y0UfSmhlXKL94HHtT3khdNscRJ9SOaezGTK4wPXFMB2HCnJ9aBEIt5nPEbVLHYXTkAKB9TUglcfxH86DM7Dk0BYli02QSAyTRgfWpTbCNw4mQgZzkVSDdyuW9c1G8hbg/d9KQE80Nu7lnuxz2HQU1ZLCA4BMpHqKpOAWyBgmni3cR+Y67R9aLCLFxcRzJhECD2GM1BGNrAnpUfXgVIg3ER+tAFqIAYkB6dAaDjzgTkk9cigI6yBBjAHrUkaNPOoAAx1pMaJxZxyXAViQp7CoZra3jkIFuXH94Hirc0LjLjGR0waiV2yAz4P0oKsRfZrfAZI2Un2zTlhQHnJ+oxVxpGVCWKkDrUIIlXIXB7mncloiMYHRFz64poO0Ejhh1IFW4QA2DDI+OoAxihwpfCxMB71NyrEccz4yTyetSBhMQHVSB2I4NPFuuPmB/CmMkaNgI/5U7jsQyAOwTyY0weoFOCAcAgkdcVZQZXBU47ZoJWPjyowfY1NxWII7dHcqeB3NNmtwo/dyLt7VMcecM/dPWpAluTw7AegXP9adwaKC2IkIDRliepzig2EIYgKwA9K0C9uOBIxHoRigeX1BI+lAWM4WYVWAygIwSw4FSx6cI13eYrEdexP4VbPzEDqO9LtVATkD2zzQFis0EIhV5Y5Cc8bR1qLzEL5EcgHYEVpJjblvwo3AngqP0NAWKW2R+RBJj1xSpFGwy0YOfWr59CXP0NRGBi2UcAeh60AVxaRKwOwH/ZPQ1MVXtbxqPapfKfOSUP0p5hIjDbgc9hTbBogWKFyAYQSewoexgAJMWD6d6lCvGwYEgjuDSyEO4bPPU57mpuFisbeBACIizD1Jo+zwvyIRu7A1YZCSCSAAO1BhDoDvIJPOKq4WI/JOSphQD1AFGyNVAWIZB64q00agAIuD0JLZJ+tRlimcHtzjmlcY23GXz6mrV4FaNYyRkdh1qK0HzAU28cidsA8YGRVMzW5EsMQI3J9M9aa8VuCdgYkdRTZHIABBJPTvUqmPG3q+MnipLGk5AAUDjk45pXgjIJEmSOg6UExo+xiSCAQw4xQqxs+MOUJ+Ygc4pXGSFAGicMSNvIHGaV/mOQSB2AqWS2llZBaW87gDByvQ1MND1EuMxFR3LEU7ktooAHPNPIAHA5rRXQ5gx8y7SL1G3dUyaLb5y93I5/2UAFFx2MXLD2pyuAcls+w610QsLEdLYH681YiEUKbY7eED0Kii4M5cRSSyKqxSEscAbTVyPS7+STYlqdv95uK6ZLoAAkLn2xTvtikcn9aLmTnPojATwzdO4MskMQHcck1KfC9srDzLvA9FXr+tbP2pQcj+dV3kMjZJH50XCPtHuUW0DT4+jM/41JHp1hFjFpGcd2GTVjeP7ppynP8AAT+VM123JIpxEoWNIkA6YXFSfa5f7w/KoSVI+4wpjGPod2fpUk8sWTmZ3B3MDTMf7X5mosQDvj60mIf+emB3x3oKSS2Js00k0zFsPuynPuadviH/AC0X86dxilvWk8wDqaQyJk8Aj1yKZ5gJ+6T9OaGBMJkPGeaUSL6Z/Gq7uAM7CQOwFQNdQjAkDR56ZHWkBfDZ6Um49hmqBvbTH/HwPoOTUTatDGQEV3J7EYoA0fNIPHWlEszdDWPJrEwjJS1QehLAn8sVT/tS9nTLsYQO4IApAdE7yoMySKo/2iKia/gT/WXCP7IMmucYLIN29nI7lic055JJIwhMeR7dKkDXm1mKPJhhkfHrwDVc6vcS4IEKg9E71UUZjUHggYJHGaFA2EEDOcgk8g1SAdLNLJkSysAeMA8fjUG0D74JB9Tg1IQDgEEZOMkUqSbVJcBx0B9RTAYSpAJzxzyKUElOVGB370FsgDBwTwOooBjCOGOSBjgcCkxgHbGAAAKUSMAS2DxgDsKUIChJJB9BSKEGcnH4cUWAElZOCp579qkM88fAXy1Pt1FSCS2IKFVY8YyafC7yEK0KlAc5yAPxosFisZP7xIJ6EjFSh5TGCYDIuCVI9R61ZnmeOTFxsCE8AHJxUQv/ACUBhQgL0ycc+pFArkLtzlrVhn+9kY/KnA2x6iRR2Kg8/pVlNanfOIEdh3Bya0ILnWLvCx6aAD/E4wP5UroTmluYTbFYgOSPc1G0iD+Ifia7mHRH4kvFtS+OybsVfFnp8aj/AESAkdCYwTUuojmliop6I4SHTbq4/wBVZyye4HBrWsvCLTbpL7EI7ICCfxrqXuONseFXsAMCoS8pOMnH1qHUbMpYmTVkVLTw9pGnkSLGZHXu7Eg/hV4TRj/VwgA9wuKaItw5z+dPzHEuCxHsOSai7Zg5N7sTc+eeKTYA4mMaOw7kVCbvd8qIRnozjH6VC9uskhd2ck9gxA/Kq5GxqLZZl1NOcRMcdaqSXVxKcJlE9COaeYY4RvcrGg7scCqE+s28JAt085ueR90fj3rRQSLUUiyqMT90ZPUnk1Dcalb2hKFvMkH8Kj+vasW4vLu7kBlcCMdI1GABUYRR2qx3Llzq91dxiP5YkHACDn86phfx9z1pQAO1OAzTSsO4YPoaMGn9qMUwG4PeinGm0wDvRSmg0CuJml60UUDsAooNFBAUmaXNJigAo7UdqP5UFAKQnkilpOhzQAowTRnFNwO+fbFOBGO5+tAC5pc03FKKAFFFGaM0wCjHvRRQAvaiiikITOBRnjNGPXBooGGacDxSYGelGB2oAXP1ozQABRxQAnelJFKoi2Es5DdgB1/GmjJGTQMXJFNOTwVB/CnUtIREVx0X9Kltri4tyXgcoR3BptIQM5GalpNWYzqtM8XcLDfLzjmbgH8a6iKaK6gEkUiyRt3BzXlhFSw3VxbENBM8ZH901yTwibvF2NI1O52+q+HbHVFz/q5ecOvUH39a4y60DVtHYtEC0fPzKOOB3Haul03xVHMVjvFCSf8APUDjPv6V0DSrLFywKnow5FY+0nTdqiG6cZao82t9dVhtnQhvUVqRvHOu4citbU/DFndTLcRRIZB94ZwGrmNTsLnTSHisZUTs6HI/KtlaXws5nTlE27a/urJsRyEx/wB1+R+tbtrr1rcfJMvlP37rXn9vr2RidP8AgS/4VrQ3MMwGyRTnpzSsRc6q+8P6ffkOIwjD+OLgGubv/DN1ahpYCJoh02nn8qlt7y4tHBhkIHdTyK3LTW4JgFuQY3PcDg01Jo6KeIqR2Zwjfu22yAo/o3Bpw5GQT+Brvr3S7LVkDSBWz0dTg1gXnhK4hGbOUSgDlDwatTT3PRpYuMt9DBCkgkMD7d6TB6ZOfQ050ljwJ4GjK8Zxgikzlc7t4zjnrVnUpJiEuOeh/MUBieeD7igEAjaSD6HpQWIOOAfbpSGKCDkAgn3GDSYI6lR6Z5FGQw+YhgenqKVTg43gZ9elNgJgnnAIHcHIoyR0AP0NSFOM9PdelMIbGAQ316igAB55A+h4pxOCPl/rTASOMgH3HFIcg9wfUdKAHkZzgZ+hpo5BAKg+5waDlgDkE+/BoHy8sQPqMg0AN5Hf8MU/BIBA/I0wnnjj6dKUZJz8ufQ8EUxD+SMjH580oxg5HUYwRTMkHk4H0zTlODkHI9AeKEBhxHEo9A/9a3WGDuBwT3BxWK4KXLg9jn9a2FbCL06CrZlFK7TL9nrWoWYCpMSoP3X5rQHiuUtie3Qp2O0f1rAzzS/Ws+VbkuhBu7RviTQb9R5qGGY91+Wq8+iI53WlyWX0k5J/EVkYyOKEeSFt0chU/wCyaadjP6rAuyaVfxLuFu0g9UIb9OtUyxRsOGU/3SMGrsGtXsAwZN4HY8H86sNrNtcDF1bE+pwCPzrTmMHhHfczM0da20tNEu0Gx2jkPZTjH4VWl0OTI+zThwf7wx/jQpGEqUl0M3FHFWZtOv7b5pbVwn94YI/SqxJHUU7mQlLmkBowDVDFznA70lHbNFBIlLSUUDFJ5o6d6KQ0XGLmgGkooAXIozTcUc0CH5FLx61HzRk0CJOKQ0zJpcmgaHUU3cfxpQ1AxSO+KKN/GD0pMj8KBCmjGaOMUZpiE2n6H2p6yypwksij0DU3NGaQ7kwvLkDBk3D3ANAvCT89vG/rxiocmilZCNGDV/JGBFIg9AxI/lVqPW4ScMTj/aWsTilwBS5UKyN77VYTc4hJ9A2DVhEgZcxuV9g+cVy21f7oo2gcgkH2NS6SFyo60REceYD9Vo2Sdlgb6rXLie4X7txIP+BVMmp30YwJd3+8M1DpMOVm7LbmQgyWq8f3JKb9lth1gkX9ayk1u6HDxIynt0qwNbQfeR1+hpOM0PmqLRMe1mhJ/ehR7g1B9nYuQmGx3zUsmrI6gJcFP95c1TnFxdjKXdqSe/KmnzTW5oq9Rblg2U46pSGCMD97OiH+6azzp2p/wyI6+iyDmqxW4hYiWF+O+M0OpIHiZ9C49va5P+kn8BSCws5AD9vxnsQapC4QHkEfUVMkkbj7wP40vaSI+sVO5KdGjLlVmDgdwcVFLpRgPMDOp6EMCfypdiHpz9KNp9XpqozRYp9UVm3IcCzm49BVZ9RaNiDaTD3IrWSaaP7sh/HmpRqN0OpjYe6Zp+1Rf1pdjmpdd8s4+zk/hWZe6ol0CDbhfc128t1DcY+0WNu+PRcVVaz0OUES6W2fVXp+0RSxMLanm7kMxIqPy2znHFegyaD4dmB2JPC/uc1C3hPSpFwmoMn1rRVIlKtB9ThghPapUhPcV1r+DEVS0GpxOR2OBUDeE9TQExmOUexFUpJlKcXszn0hZWBxwOtP+lajaDqaNg2b49RzUMljJbtieF09yKGy7plHcwNRSuW6mrUxiCHaDn3qlIyk8MKSQCK+05IyO9K0ykYCHHrmoyM9x+dAQdyBVWQEQIJGKkCk9BU5sgjkGTLj0FRFZEkweKLoBm184PFKCVI4zTnBUjGT65FWP9VGG2Ekj0pNoCvmQsMoQPfilZjuzt/Knea/JMZA9cUxZiTgqfwBpJjE3OT93FNK85NWkViCTGTkdxUXkyk/cYfUUXERqeeKUwO4ABAz61aitWSMlx1p209lJ+gNFwKaWiK24tuI7jirnlNLbEJF5mOvHQU8IwiYiGQkjGADU1k1xDb7GhGSSelK5RlQwF5AFQn68U8WciXIyOta0drIZckFc+lWBax5BcEkdyaVwMia1liIkcYFNhiczKMdTitWa3icEMXIPYUyC1CgbQeKdxWFaCUEqm5j6dRVS9jMVuXIw4IBq8DMzYWYL9RSPamYYm/ee5NK5diK0VJtPQupOc5xUUswh/dohAHrU5hUcCYx/wCyhqN7RJBjMje5OTSZLRJDcNMdocAY5zUwdcYLg49Kr/YYUXI359CcU5IDHh2XAPvmmhgZgrAKDknqRSEsckyAkntVpwCoyiYI4LGnIEWMKPLB7470wKqqpOGOR6U7y4zKQiH8OauiyhOcy8gZIXrVNVlIARWx6k8UgJVBRQGTI74GTTgyLyqYPuMVADOzACXHrngCmpcbJgjOCR1JFUA8hQCCvfqRTCB6VZm2OhGRk9MVXVNgGckDue9ICUOjk7E2kDqajZAwJcAn1HBqUwEAuVJBHGDQSIwMg49aAIkj4xgIQe/erBAJUkIOe1AS1JzNIRxkADvSBCTuX7tICWKYxyElQVxjJqJjbliSWz6DpTRIoJBPA61HuVpPlGM96AHllAAVG4PUmgOWO0DYPUmhRHswXcmozIEIwGb6UwLJt+MmcDPrTBbgqQ8gPoV5p8ETXP3LaUt6bSatjSdQmCiG3MeepkG0D8aRLkl1KgSOMYaWXb3wKQmFuArlO2OK1YfDF0fnuLlBjsCTirMeg23CyzO3r82M0C549zm2QCTALBPUmlkSNACJixJxgV1x0exwdsUbH/aGSaytfEcaW8KIq4bLADHpVBGalsZ9uMMAByeAPU1c/wCEf1NmLhUG45wW5FM0+IS3cS44LCuuMT4/1y05GFSo4uyOX/4Ry5H+tuEU+gOant/DsCgtLNIX7FABit8wuf41P40oiYc4FSS6smULfTLG2X5IQ7DvJzVlVVPuRRr9Fqby2Pajym9KRDk2M3MRgnA9BxTcepJp5jbHIpMPggA/hQIrzW4fkL83vUYtX9B+dWt2DyDn6UhbnoR+FBcakkrFf7M/QkYqUWkf940vmgHpk/SjzQT3A9BQN1ZMekMYGMZpwiQH7oqES4PI59KC5PIBHsaCOZ9x5jQjG0VD9nUfxGjzDUE935Qxg5oLjKV7IcIzuwxwPc1YRdi/1rHa4kdskHPtUnnXLsAmQMetBrKE5dTVMm0ZIX86z57gO/AX8KURXbgHdgY5zUJhgiO65uEXPbIoCnT5dWIZGPTmmgk/wE/SkOqaZC4EJMr9gBg/rTJtbmQAR24iz0MnNBuWVtJnIHlgZ7k1IbCNP9dcKg7jOKxZL7V7hykNxGueojABA9arTSyNP5UxhYgfOSc7jQBuG90i3IQv5jeiknNDa3Ap2wWpx2DcH+VZAEa5dbcEjn5Rjj1pokaSRSSA464OcUAX59Uv5wQqgL6AYqgwluJP9IJwORnnBqQgpIHDHOcY6mmsZFlLFmjx0LJgZoAijgAcne4HYDj/APXUhYkKCFLjA6daUTBmBDbsdRjvSTNJlgNhyMZA5FIY50VYy3pTtgkjBAySMk1ESFnyQWjUZAPcmnpNKSQilQwwT2xQA0R+WCcYwSMdz+FNEo3YERGfQVMU7kgHuTVqOGRwJCQFAxuJ5FANFEP85O1uBnGKc0yeslTCIybn2FjnAHY/WpSpQqxQRkDALgkGgVipGjzzRxwhm3Y5PAFTS2qJcGBZpHkAyQEAHHXmrccc0smRPHkAMAnU+lMfT5WJbLq4OQT1FAFOaJ4pPLWCRnxn7wBx9KFtbtgCbdwPQkD+tWBGQzNIw3cjJbkD2qS1ALBRmTPJckgY/GgZQJK9QR25p1rdW7SSRShi46IRnOPftWtc3VrGxLx+Zs+nXv1pLO8e6cLY6S2/u+On40wbS3DT4Yzkm1VCBkEjOfxq1cRRqjIyD2wMACtFdD1CRf3159mUdRGMn8CKu2eh6bagM4aeUHO+Uk5/DpWLqIwliIJaHJxadd3sgS3gMxA+8zYAPbJ/Otm38JPIA9/dfvOhWMdvQmuha6jUlUIIHTaKi86VjxxUuo2cs8TN7aEdjpmn6XGBDboHH8bLk/rU733JC/Nj0qOYnALEkZwc0wrGil2ICjqScAVDdzBtt3YF5ZCTgDPpSGNifmqCTU7SCREw0u4ZzGMgU8X0kvEUJX0L9fyppNhZk4jCjJIA9ScVC97BG21cue20cfnVZ1klfdKzPjsTwKcImHRQKtU+5ah3Fe4mlTG/YD2Xr+dRsVt4gwDuc465PNQz39ragAvvb+6gyayrrUprsER7oV9QcGtIxSLUUjUuNZsoL0W5mBbGSR2PpVe41wgkW0R46O3H6VirCi5IX5jyWJySfenjNWMdK81w2+eVnOc4PT8qFUAYwABQKdihIA4oxmnYxwRRgY5piuNGKf3GBgUACigA4pabRTGBooooEFFFHegAFHaiigAo7UtFAXEoooAoAKKKKAEpKWkxmgBCQOtKDTSOacCAfWgBwPrR346Ud/agHAoAWlBH403NGaBjjwOaO/FJnnHalzigQd6O9Jk5pTyaAA9KBQD2ozzQAvfFFJR0PNAB0OaWkzyfrRxQA7rTfalyO1JmgELS02l4oGBzS9sDrSDpSikITk5FN2nNOPXNKaAEeLy8fMDkZ4PSrunatcafICjlo+jI/Ix9KokUYqZQUlZjTsd1p2t2eoABG8mXvHIcc+xrVaJH4dQTXmG0ZzW7p3ii4tgsV0vnxd8/e/A159XCyjrA1jNPc19T8H2F+jvGDDKe6gYz9K4u90TUtCnDzQmWMZ+eIZH/ANbrXpNjqNtqcHm2kpY90bhhVrggqyhgeoI4rNYiSdpBKlGR5Za62jnbKp3DuBWtDcRTRAo4IPpW3qvg+wvcvbRCGU8/KcAn+lcbd6dqWiTESQ55xvjO4EetdEZKavE55U5ROgjvprAeZHKQg6g8ituw8QW1yo84eWx/i7GuGj1ISWzpcpsY45PQfWrthcQvariQFgSOnWhozud5LFZ6hBskSOVPXHI/GsG/8JK5aWyfDHoh6fnWfDPLDIHhkKn07H61ow+J5YW2XdsGXu6cY/OhNrY0hWlDZnNz2l1YybLiEiq4ZC3UD616LFeWl7Ajo6FXzgOOayb7wtb3LtJbS+Qf7o5H61op9z0KWNT0mckFjxkgBvUUmDnjBrRn0DUoHKmIyqO6nIqg0LxcNHIjfSr0Z2QqRlsxn3eA+0+nalIycsoPuDSEt1YAj170gK4JEm056E0F3HqXAyrjH91jTgD1YlfQjpUQaP8A5aKSPUGmmQEYimBA6gmgCxlugAceoPNMLOCcfL7Nzio/OgAO+TafUGmvexdBKJB9OaAJQOpYfUrScNwAHPoagGoW44+dPqM0fb4c/wB76DFMCzgrxgqPQnigRk8gKR6iq39oQDjZIPwzSi7Q9IJD9BQBUux/pRA9B+daKE+Wh2kZA4HNZlyZZbgsiEYHIPetG2cSQglWiI4IzwT60yFux+CQRgH6HmgKB1J/Gl+VupBI9sGjGOASB70ihw6cA/UUZJ6lce/BpoAPORn1B5o4PVyf94UDaHcdyQO4HIowvYA598Gm4HUA4PcGjnJ+YfQjBoEOII4249z/AI1LHd3MIxHM2PTOajGQOpA9jkUgx1GCPegTibdj4pubVQJo94H8RHStT+2NCv8A/XxKD6uvP51yBLdDmm7FPUqfxpNJnPPCU5O508+iaXPhoJ5oc9uCKzLjR5oBmJxOPX7prNBdOYpHX05zVyHVruLAYrKB69atGX1REMkM0YHmQyID3ZTiodwPQ1vQeIAoMcsTIp9sikZdL1FsBkDd9rbD+VUmzGWGknoYfUUgro38JHYDb3gII+64/rms680LULNS7QF0HVkOaFNHLdGdS0meTnII6g9RQcVZQZopKKAF5oozSUAFFBooADQevFAozRYQUUZoosAmecUZwMUYoxTAXdzzSZox3NFADs0ZpuaM0CHZozTc0UAPzRkU3tR0FAWHZFANIKBxSC46k5pMml3GgdwzSUUUWBBgHgjIpAi+gpc5pcj8aAEAx0Zh9DUiXFxHwkrAenWo6KAJmu5mGGWFvcoKYJY8Ya0hJ9QMUw0mKlxTFZEh8gn/AFLRr/snNIVg/huHH1WmUYpezQuRDjA5+5PEfxxSi2uj0RXHsRUW0egowO2R+NL2aDkQrJOmd0L8dcU0S46hl/CnFpP+ezgfWpEuZlXbv3D3GaXsyeQhMkbdTz7ij5D0AqdpEbl7dCfUcVHtty2TCw+jVPs2LkZGyRnpkfjTAjKcpI2frU4igY/fkQe5zTWgA+5KD+GKXK0Tysas94vSd/zpDPct/rHD49RTvKlHTDfQimMk4GTC34DNLVD1QmLZv9bp8MnrgYpkltokwAk05ox6pQzheoYfUUK0T9WoUmCnJa3GHQfDkwODNHn60g8IaFJ01CQfXirIWMf/AK6diP0/Wj2ku5ftqnc5qOG3jXCsmfU9aUJHnI2keuKqSFRkbQKI13dCQPat7HqF7EPfb+VKBFnIwfwqoyugztJA7VNDGzKS4Cj06UDJmMJ5KKT9Ka0kSDJRcewqErEufMlP0FR71kBCggA9xQBYFzCRyg/AUhnh7Jn8KrmMgABTn3pQNp+ZR+VO4E5u4V6x0HUE6CPFNAiwCUAHegJbiTcsZP1pA0KL/PRCfoKf9rdukR/EUhumOAgAHoBSGBpFLgj6UAO82c/8sSPwoeR48B8AkcDrTRHIRzI2fTNRSRvv6biOmTQBIJ2PAAPvimMZCTgEE+lSBDsGAMnsKRUZcl0IPvQBEYzjJfDfWmxIZG2vKwFWVjEwJ3Kv1NMMREhVJl96ATD7PEB8khJ9xQlrMzZ+0FAO2etII2DAiRs/WlEc7cFyR6A0A9SR1AbEjgj2qOJADkk49TzR5Wz7ykZ75qy9sUtA24gdgKEBCUH3t2R9OlN+UNgD8hTP3oADAkE46U8GSMg4GB2NMRZiJELkEBjxn2qIcDAQ59zQ75jySAc8gUiXC9iUJGOTmkUNCMoyyjPpTkkbHEagj1Gad5MrsQgZyfQVIthqDnEds5z3IxSRLa7kPzH526U5QTyVyg71oW3h7UpcCXZHEP7x5H4VpJ4btV4lu2PsMAU7mbqRRzhIYcygY9DTUOWwfmPsc118emaXCgURFvdiamRLGHmG2QN69aCHX7I4xre4uHHlwSMB3CmtCPQL6RcrBIv+8cCunFzJjChVHoBSF5H6yH86CfbSMOPwxKSDPLEg745NXE8N6YMeZOzgdhgA1oYJ60bF7igh1JPqRpZ6bF8sdmpI7mpVeNBiO3jT6KKNo7Ln8aUZUYwAaCA8+b1x9OKDJIert+dLuGAKUYoAjC5GSSfqaNox0FPycmgtgUgGAEHiuY12UzamqEYCDH5810sj4XHrxXI3j+dqUjZzzwaqJ0UV1NXRVAnU8fLk810LOSc5rC0ZB5kjnoo4+prYLcU5bmFT42PDEdTSEk85NRGQjsKTze2APpUiJt5HQkUCST++RVcyHsaYZT60CLnnOf4vzpPtLgcOuPQiqgYscA8+9IT23Ak+nNAFz7W/UbTR9tY9QlZ7syg7RuI7Coi8jA/IQf1oLUJPZGmb0DqgP0pDexr1QH6VmiFycsxH1aopby3t+GcMR2UAmgv2Mi9LqI8wgQHH0pI7oSNzAcVjDWWVm8u2Zh2LNjH4U2TWL14+NkQ9AMn86DdU42szclRpB8ilT7ms24Npb/NLdBnPUA5IrHlmubpCpkkJ7necAVHb28kcx8uON1IwWftQVGKjsabazbRDEcLSMOhHSmjUrt8MAsSkZ4GSPxpqrDDFsFuhfJJbsPp7VXEbBifNYE9ADwKVx2HzXFwz/PcSkegOBVNrcF94ds9geRVljggli4P8R71Yjd1R40iQhxgkjn8KOYoz/KKuAjZYjJOKe7B2UyOWKjgHsakVMPleBkDFOLgEgAHHqKExWIt6OuGIVj1wOoqVo7YAYTeT2xzQjlTgHBoLYfeASR0wMmi4xu4E88AjgDjFPAjjAQDkHJxTPNzPmVJFA6sR0qfbCATskGRkEnk0AyIyMHDg4A6GnvcmZcMcn196RmQwFSAT3OOQaiDRjrGDn1p3ESiEZLYAAHJB60RxpjIBwTgknrUBCKSQhUEcYOcVIuSgUuQT0yMUrjuPyV4BH0IzTpWIWMb3LAc47CoyhEgx8x7gcmnIigMOhJyQ3BFMQpJCAIC5PJPUgUKu44yc+5wAaQuqsQiADOCQetSLC0kWVGRnOSaAuTo+0FAQ5GSMcc1LGbiaMHaG3jkdh9Kjt4AbciUHIOAc4Jq3ayBSy26NMy4+VOTSHe25DGk65jiiC55LA9TU0KzIxLktxgZPSrEXhXUrljI8iwbucN97H0rbs/DllbAGRnmkxyxbj8qhzSMJYiCWmpy4NxNciK3i3sf7ig1rQ+Hb6cD7XMEA7Jjr+B5rpAkMK4REQD+6MCozc/3TzWbqPoc88TJ/DoUrTw9YWqnzU8+QnO6T/CtFp44fkRUAHZRVVmkcnLkZ9KBHjrzUNt7mEpyk7tkrXLucKvB9ajKM4G4k1KkeSMCklmihba7Hd6AZx9aRIxYwMDFS4WMZYgD1Jqqb0sGEabRjgnkmqxy5zKxc+pq1BscYtlue5SRCkQZiejYwBVG5hmuhtnuio/uoBipNqnsKY7pCN0rqg7EnFaKmkaKCQ5I0iQIhAHsBTyAAWeSQAdSSAKyptcjHFum7/bPArNnuJbp900hb2HA/KtEVY17jWLWEkQgzMO+44rKmvrm5OGk2r6ISBUIAHalwPSnYY0KFxxS5HqPzp2KQIM5IFFiQCmnBTTgKUHA460wEAIpwFJS0AA65oo9qO+KAA0UUCgBO2aPx70UUxhRRRQIKKM+lGaADtmgUhooAUHnmjjtTSaM8UALRmk70YpiHUlH1pKQxeM+9ApO+aM4oAOexpOc0ooPXigAyTjNO7U0DBznPtSkEEjoKAHUZxQODzg4owTzikCEJAPUdaXvSYHfH40E9OM0wF9+9GaQng0ZoAXHQ+tGaKMUALjPFJ7UUmKACl5xRR+NMAGR1pRkCk5JopALRzSUUDF5PSjPHvSZpc0CDAwTQRzwc0DkGjoKADtTvpTaD7UDF4xTSBnp+dO6dRSZByPalYBY55rdt0MjI3qpwRXS6f4vYbYr5S64/1i4DZ9/WuXxRisKtCE90VGTR6daXkF3GJbeVZEP908j6ipZYorhDHMgZD1BrznTmkWTKXawlCOe/P/6q7G31aNUCzXkU5HdMA/jXm1KM6b0NlJNGVqvgaG68x7GbyXI+42SDXIXGn6ho10fOieIAcMRkV6vFcxzKHibK9D6g02WS3lRo3CvnqCM04YhrSWpE6SkeXWms55ldTjuBitVLhLqEFDkGtPUvB9nODJZEwyY6ZyD+FcpdaRrFgWPkyKg6MuMV1xlGWsWc8qTRqrmF/wB2xXPJwcAmrMd1KhBWV/oScVzw1e5i+W5iOR1JXBq1Fqdu+OSM+oqrGRujxMbfi5hb2ZGOCKuW+r6fqChRMELdA+c1zbvFKQCRz71FCbaYlc4IOBxzSsF2dLP4X0+7/eRXgjkJ5weKrN4IZgRHfoSfVQTWUi3FuN9rdEZ6oeQalW41F2ZnuBG2MgoSAfwzTUpG6xNVK1yzL4LvYlyLlG9gMVRfw3fIu57Pd6spq3HrmpoAJZQ6juQM0r+IrsITFLhh0BGQaFNmqxtRbmb/AGRcIQUtCcdQeKrskaSbJYBG49DmtqHxdfwcSxxzRnqCMfyq/D4q0yf/AF+nhffaDTU+5ose/tI5gDrtRXU9crmmn7OpwYjG3qF4rtk1Lw9MoJiUZ/2KlB8OOOTEPrxT9p5Gn1+PY4oSyEZCrIPrTC6FssTGfQHiuuk07w1Kxb7SEJ/usRVK4sdBR8JfIF77+aamjSONpvcwQT/GRIlNMkS8IpUehrZj0TSrhswamig9t2APzq8PBMj8i/Vh6kZNVzRNPrVLucyHZs5RSB0IODTgewlKj0J4rpP+EGuAPkvV/wC+f8Kb/wAIRd9rqA/VTS5olfWqXc535AcsoI7FetKCD0OP9+uhPgq9X7t1EPpmqreGdSUkYRvzpppjWIpvZmWAw5IA9x0oJ3dWz9TWkPDeorkgYPqGoHh/Ux1jRvqad0V7aHczBgdAfwpwY9CB+NaX/CPakR8saqfUNmkHhrVSf4aV0HtYdzMwRyelHyk8gH61qjwvqJ6sB9M4/lT18L3nRmUjuNpNPQn2sO5kZAHGR9TmjdzyQfoMVuf8IrOORIw/A08eGWH3zIx9cYo0D20O5g7h3J/Oml19AfccGuug8LxKFYjJ9G5q7FoNsufMjjPpUuSRhLE0l1Mfw9eXjs0bMxjUZAYV06O3rTFS2iHBQY9KDPbL/wAtBn61k5Xdzyq0lOV0NuNKs9UGydAj9pF4NcfrejzaNdBHfzInGUfv7g11Mt+FBEeCTRFDb6pGYLrPqpz3pe0lB36Dp2ehwYGRkUEGusufBTA5tLxQv92Qc1h3mianZMPNtmZf7ycg/lW8a9OWzNHBoz6WmltrEMNpHY8Uu4Eda2RIEUE07Ipu75sigYUUHrRxii4CUtIR6UU7iDPNBoopDEzThmmmlAp3EFFIRRTAWkNJk0uaAF7UnSlxxQaBBmjPtSd6KADNLnNNo5FAx2TQDSc0hJ7UgY/NGaZmjPamIfmgHFN70Z5pBYfnNGaZkilzTAWl4xTQRQCKBjqKTNGRQICKTFOzScUAJnNFGPSjFABRR3pM5OMUguFSCV8Y3nFMozQNMeZGwQSCD2PNIGTGDBEfwxTMUYpWQClYD1gI/wB00eVbnpJNGR+NJRilyoLI5QAFskZHpViOFiMqioPUnmmRsrAAZz6YpxjbBJyPYnFM9IJIpOpmAHoKjzF0cux9BxSgSqQFVMe/NTmU7SDtz6UgIBGmM4/Ol2hRmlwWHBAPpQISepJ+lAAVMq4AYe+KRbcRDJJJ96eiSKeXIHoKgcgPtyx59aAFPlk4fle9OdwMbPu9uKljjVGyVB+tLLCSdy9D0oArjBOSDmrKEBRVcmUNkMCPSpgCRnIH4UAOJ+UkVApDSZAyamTeDhmBBPpQZEDYyAPXFIBmyRnGARjpT3STADdu9BJPKtke1HmEZ3HIHpTARokGCFye5zQIxkHFKu6Y4SGUn2SrK6ZqEsfyW7qPVhgUBciSIucBQB2JNNMZAIeQD6Gr8Hhe/crLLcKg9M5rTXw1anBnuy4/urgCgh1Io5wRZdfm3qDkjNTTh7hh5PGBjavNdWumaNbKDDG5cdQ2asI8KjCRKo9hUp3MXiH2OOTSr+QcROR2LcVai8NXsihnKp9W5FdV5o//AFikMgP09qoh1pMwh4atwv766y3+yM/1q9BpelwADyRIR1LVcIjPUE0myH+5n60Cc5PdgJYoxtit41HsAKQ3Eh7gVL5UZ7frR5MXofzoIKxJbOWPPvTdo9M1aMMR7kfSmiCMjILYpCuVwADzTgFzmp/IX3/Gk8kdjQFyPA9aBt7VJ5B7SL+Jo8k/wspPsaBjfxpNwzjIz6UhtnzkqSfY0jQyAZEXPr3oC5IQB3xQeTUJSYnlSfwppEg4KkfhQBLmjd71ASw74+tNMuOpFMCzu9SKQsMYzVUyn2xQJD7UAPuCPKz6ZNcnAPMmLerGt3UbgpZuQRyCPzrEsuCp96tHRT+A6OzVYomHrxUxkA71RSclcRD5qUfaGOSm71AqJbmUac2rl0zAdDmomnXccsKqsBEN0pEf+81QtcWCozvKJPZP8aRoqHdl8Tp/fFJ5oZ9oBP0GayDrVtHj7NYFn9X6CqkurX8uQjmMeiDFUUqK6nQsdvMhQD/aOKpvqVnAWzN83+wM/rWNLCdpJdy3qeKaYwyBDuI9BSNFCK2RpNrY8zEMfP8AekOBRcXWoxsBkKCMkqMVTJfcpMajAxUkN0d+w7nC/wAPagsQtC5ElzPK7dwO340Q3DCMCEgOPQZNMOHLBj1OaSFCp3xdeQc1AxJGeRwWO1weQO9KQRkADA7U4oyklkBJ6nNSJtETySE+gA6mmgsEd15WNsSAjrgdaZGfM5xtYknJ4FAwwDYx6mnCTaOFB/CkFgMcrMOCFJyHPQ02QckA5OMZHSpomymCCSDwDTPOt0fnBHQgCkOwsyAAAlST6HOBTYgY2STHfANMebgJsbJyQcZGKfAzFAJoZXx0whxTAEcmZmOCeTS7w4ByFY98cUThscIsOeNo6496SXYiqAMnufWnYBRGoXcWDcgEjqM0BlR/lLcetNjIXO2QIPQgkGh13AuGDAcEnjmkIcdzjOSR70K0inIAIHXIpYoZJhtjIAPcnip102deRcRg+ppgJDAbgEb1ye2eQKja1jjfBkyR2x/9epJbKfAzcKSTjINLHplxHICbiMAnB5ycfSkmNoqXULXEgEZYkHIIHb1qLyp0YbpCw7g1stZCPCNO7DsFAApW0OSTD26O7Hs7UXS3JlJR3ZnIsnAjAOASSBipEiE0+JGyWxkA9MCtmx8Nak5/0grBGSTuzk1tW3h2ytlActN6Fjik6iRjPEQj5nJrZxSSBIbZpCehBIrVi8OX0xXMaW6DjBJJrqYkgtU2QxpGv+yAKVrlR/FzWbqvoc8sVJv3TPt/DlnCB5u6fjGH6flWhDHbWa7IYkiX0UYqEzTMeCBSbXbkkE1DbZzuTe7JXuvmAAJzULSSs+AcDHHFPCAUoTJAA5pEkIRmOCTShMHAp7ywW4/eyqrf3RyfyqlNqXGLeM5/vMP6U0m9hpNl/aE+Z2Cr6mq5v4SSsP70jrjgVReR5jmTcSfbIpQCegOfcVoqfctQ7kkk1xI/zELH/cXpTBx0p3kzH+HP41XuLuG14kcGT+6pya0UUjRRSLBZ8c4qOSeKAZmlVfbv+VZc+qzyf6lBGOxIyazygLFmySe5OTVDNK41ksCtuuAf4z1/Ks9meZt0zl29SaQD0FOAz2piECj0pQBTsc4oAoAaBQBT8DFIpBBIzx6imIMYNL1FKRgU0EknjigBwPFFBBPTg+po57nJoGBJ6UUYopgJmlFHagHA6ZoAU0d8UUhyOR1pCAijoaM560UxgTxSZoNJQIWjNJRnjmgAzRmkzRmmIXNFFFIYUZOM0UeooAXrSUUUAGc0Gjmj0zQAAnHNJnHU4oJGccUtAAOeRSg+tJkZwc0KCSQTRcBxPNANHeigEBHvQKO1APbtQAuc8YpB70DrRQADtSjqaQ5BH1pf50AAGCAaOQaOfXNAOTigA5zz0o96Xrmm45oAUGjjvRQRxQAcUlFHegAzS0mMUvJNAC0Z5pM5A+lFACnGOnWk5pe/0ooAOc0hHNLwDRmgEN5ApDkg46048k00A4NAXGNEHGDn8KYbYDkOwqfmgjNSFwt7i5tXLQ3MgNaVv4hvllAupi0XfaMN+dZmOaCuaylRhLdFqTR2VndW16u2O+be33Vk4OfzqO60zXPKLJcCWP0HU/hXIAsjBlODXZaJqMItIzMbzef7q7lI/KuCvRdG0kzWMubQxXghc+XqELoe+VNNm8LWMsBaxuEU9gxrtjPY3wEKs+89BLFjP+FVV0AW8nm28YLf3Scis1iKncThF9Dzq60LULdmAi3qv8UZyKoLE3HzFW9q9ce+jt18u50/Z7pyKz7nSdE1RgV2q59sGuiGJi9JKxnKiuh5vHeXFu2CAcfjVhNXdSN64Ppiusk8EzmQ/ZrxPLPTcKoyeD2M5W5m3IOhjQit4yhL4WZOi0YX9qxFyCmM+lRyXsfVQxB9RiukfwbaeWfKeZX7EjIz71gXWkXNlKFkKDOcHOQf85p8rJ9mxiTo0JPfsMVJEUfPHSqskVxggFD9OKhUXMXOxz7jmh05CcGdBbxIUHFTmFAOlc4uoTA4WQE9gRg1MupXCnJAP41Li0KzRuiJemKbLbjyz8gP1rGOsyMcJw/oaU6tcr1XP1oaEy0tgjzAlcA9QDxWjatLbjy455QOwDHArCOtSA8oAfpU0OtYO50LH64qrAdGLy8U8XUv/fRqZdQvlHFwx+tc9/b8QP8Aqj+Jpw8Qwj/lkf8AvsVNhG//AGlf/wDPY/r/AI0o1K+HWYj8Kwh4gt/7jfhzUg12zPUsPqKLDNoarejjzz+VL/a1/wD8/B/KsFtWtSciUAevWp4763ZQy3MZz2YYNFgNcatfE83BP4VKurXoH+uz9RWULpGHDwt9DQLuEHDOoPoDmiwGqdUvT/y2H/fNH9p3v/Pc/lWcLq3A5lX8KPtVsOsyCiwi+dRvT/y8N+AxTftt0f8Al4eqH26y/wCe6fnSHUbNT/r1/DmiwWLxubs9ZnI+uKUPJIMmR8/Ws9tXtRgRsZWPRVHNNGuW6jBidT79PzpNDNQAk8kmpAAO5/OshdXEfzMhfPQLTW1eeYH7Pb/iTTUWxqLZthgOTVG5v9si+RcrHMpyvNYV3eXspHmqyemBxVdADknGfU1pGCtqaRjbU6yz8YXiSlb6BGA4yuAa6Wz1/TrlRsulRj/C5wa8wDkcZoyG6isp4SEtVobKbR6tNpmn3+WktonP98Dn86w7rwPblWa0uHjfsr8iuMhnuIMiKeRFPYMav2uvapaE+XdSMD2dsj9an2NWHwSHzRe6LNz4V1WBWcRrIo7oc1nW2n3dzMIUhYSHs4wa0h4w1cHnyj9VFPPi++kx5lvbuR0O3BBq4yrrdCtEc3gvXAOIEb2DjNY9zaXNi5juoXjcf3hj9a1G8XauwwJwg9hmql1rmo3sBhurgzR+jDofUVpSdW/vky5ehn5oHNITt5xn2FN86MDrj2IrcgkNJQCGAIOQe4pCSDgHj0NAxTRnijNFAAOetHQ0ce9HHamAZoOKMUUAFGaMUY5oAM80maO+KMUCuFFFFMAo7UUc0CDFGeMUUZxQAE0UZ4ozQAnIpeaM560nWgBe2aM0dqSgBc0ZoOO1JQAuTS803NGRnrSEOyexoJpMjPHSgnJ46Uxig0uabnmg0AOBFBxjPem5oNADh1oNNzQCc5oAdRSZ4oB4xQBzahYmy6Z9BnFSSXJmkGIlGDmoRGe/SpFUAgisj0y0sqMwUQxjPc9KfmEf6x4wv+wOaoyFScEZpEICgZwB0ppgWZpoSMQow9zUYz1BI+lQvKAcAMcegqaGyv52AigmIPcLimDaW48EAcgmmmQDogHvitNPC+rKm9o2Udt/atG18JAxhru+jX1Xaajnja5HtInNl0dQAQD3JNRmNv4Zs13kPhLRCBIztIfXdircenaBbv8ALGjFfUmsfrMXokxOqjzZUfPKOfYDJNWoLK9upVjgsZjnpXeT6hHG+23ghVR0O3rULatdfw7B9BVqVRq9iHXRzyeFdXZgGtVVT1JbpVpfBsA4vNQ2N/dRQa1DqV83/LTH4VWYO7lmOSaUVVfxP7iHWfQba+G9BtZFaSeWYDsTx/KtOODw/B/qrVc+pzWaUb2pyxn0FKVHm1cmQ6re5P8AatjEQxKi54xUbXTsfmIA9KQL604ADsD9RWxnciLuw4Jx65pQjHrmpe2QB+VOz7CmNEQjx2oC84INS8dDSEgHikAgUe9Lx2zSZNLnjFACYB6075QPWmZ5o3qOp5piHkg9jRk9s03zADgA/jTGkOeDQMl3ED0ppkb+8cVGXJ69aaW4oAlMhHegTHuRUIIIzRuAoAlM79guPXFAnOclVz64qDcKaXFAFs3BPUD8DR9rAGChP41V3DGciozIT0FAi6Lsf3TQbqM9SRVAse5wKXk4wN1BSi3sX/tcOPvH8qabi3IyQrfhVJ40WMlyiDGSSayU1e2GR5ZY5wAKZrGi3ubM1xDxsiU/hUauJM4tvyrJl1acnZaW6g/7WDVG41DUSx3OEPcADFI2VKJf1OCSC1csFUMRhc5NUrdcKDiqpEsjCSV9xPv0q6oxbsf9mtEDJZNY+zxjybQjAxluc/pUkd/NcWpkkLITyqjAGazSjyRhWcsgOcmrZRfKGSxA6A81nLc0WxWVssHugJCP4s5H4Ukht3cgRkHsAODVmOGSKZXMAKjP7s9T+FTy3GJvNS1COf7+OPwqRlI/IcYyMdKczMQAmBnnAFAxgs5LEnPPFCkjJIwR75qhWGg5PzuuPXNOlCID5alyOhGKBgRFgq7T1OOaZIQgGCBk45qblWFByoLDDntmnxELuIHUYNPtwEnXe6Fe+aa2yF2xMhUnIwapMQsxRyH2BSBgYqAyfMAp+Y9qdIgkUFZQSDkAjANNxhAH2q/rQOxIJImSR5BgpwNp5J9aUIqRl2fJJycmmEl8ZkDEDAx2FKSxwB8zenc0ATSCMlRFwCMkHrmomjkEg4wBQnmP8vlAMejE8j6VZgsrqXBLDZ3ZjgCpEyEuqRZIO/0ApiuScrnHqRim3xW1YKJ0kY44TkD8abHLnCou4eucUWFcsF5wP3RYH2pwjuTEJJJmJJwBkkikjkEQLTW8j46AHFXkeS4yotBDju5GTSC5Te3cxFmbLHpkEk/jT102dwCVHA7nFX4nIUxkkFeuPekJIkBMzEE4x6VSC5VNjImHeKMgdQDyaaS7sFSMA4xjHWr/ANpAfYRWZDBPcXcrxq8jF8IIxkYpXBtLcuRxm3Q7nVmJye1Ry7mn3pGWH8RHrWxbaBeXMaGaMRYOTvOCfwrXh0GwjA3pvYf3jx+VQ6iRjLEQRysaSO6/ZbZ3cc8dM+ladro2oXbA3EcdsinIGck11K7Il2oiqo/uqBUZmVe9ZubZzSxMmrFGLQ7NJA7kuw/Cr48uHIj+UHtmoHlJGBxUR3N1Oam9zncm9yxJclMHcCOmKikuJG6AAE1G8WYScdKnMYBzjtmkBFsZjliTTkjAPSpAhJwBSM8MK7pHwewHJoAdtA6U7G1MsQo9ScVT/tCaQYggCD++/Wq5hMr7ppWkb3PFaKm3uWoMvSXkUZxGvmt6jgfnUBubiTcHwinHC00KAABTgK0VNLcpQRCLeEPv2kv/AHicmn7Fz0FPcxxDfLIqKOpJrLutdhAMdom9u0h4FaFGkTjqAAOp7CqNxrEELFYwZSPQ4Gaxprq5ueJpCR2A4H5VEFI9KALM+pXtzwZAid1XjP41V2AdKdtNABz2piFpACTTsdeAKBmnYAGQMAc0oz3pQKX3oACOM0gpc8dKbmgB+cnp0pOtHajOOKACkpaBQMM0ZpOM8+lHNMAB+alNJ3ooAX69KMjGaDSUCFzSZo4o4oC4ZpM0e9BoBimkoxRmhgGaKKMUAFFHakoAWgdc4ooA5oAOAM0ZyKPr3pKAFoHSigde1AByTSHr9aXtSH7px17UAB47UA5owDgnrijpigBe+DQAS3DYApM0ZA/GgB1Lnk0meKO1AIXnr2opM8cUUDFoHXmk5B49KXqKBB3o70UmaAHDPeiko9qAA+1GemetGKD1oAKDnOTSUvNABmjsOlAwT1oPHegAxRjmnAgKQVySMA+lNzQAAYoJ5HvQOtGfXmgBTnFA6Z7UhOeKBxQA7GQT2pOKM0negELSe9Lnk0meKADNFNwQc9qcDmgAxzRwBijvRxzSGIu5WBGODnBGQa6fS/E0dtYLBPE4ZejJwCK5kGg9axq0VVjysadjpr3xPHJEv2eNy+c7mIyp9qLLxrdxti7iWSL17/nXMY5orKODpxViudnpVtrum38Iy4ic/wAMn9D3p02m211kgDPZlx/SvM8e54q/ZaxeWTlo5mHouePyrnng5LWOpSqLqdr/AGfd23NvLvA6KTz+dEWpSw/JdRuuTwWXIFZFn4zyxF3b/wDA0IH6V0MGqaffRgCVCD2fg1zSpzhpJFqSexGTa6hHsfkdtrYIP4Vh6h4QaRQ1rcFyP4ZOD+ddDLo1vI26PMTDoV61Wa31G0YlGFwvcHrSjOUHeLB2e5wl9o17YkmWFgmPvYyPzqkuQORXpC6uoYR3UBiJ7EZH51C+laHrB3CLDrkEx/Lg1108a1pMzcOx54cE8gH60xoIX6xKa6+58ESAtJa3Ydeytiufu9J1CyBa4tXVB/H1H512QrwlsyHFooC1gRg6xqGB6nmpS2fvDI9hzTQc0prS5I8xQkZE+D/tx4qMRjsox34FLjNKCR0ouBE9vG55QD6CkNnARjYM+vepjTNxFFgDyQVxgYx0Ipotoim1o1OOhxUm4mgEk80wuV2s4P8AnmB9KcLWIRlQMA1PxnmlC5HSi1wKi2kSgdTjvnGaU2kZOefzq1tFIRigCERAdCfxqWG2imJEkip6Z70UhGaACWyhhkADJIpz8ynJH1pFgiO7cQuBkZHFOAxS0ANhzDMksLKsiHIJGavHUblwRJDbyZ6/IKpEJ3FNWQKcg4ApFEpfc5OwLk9B0FALgkqzKSMHBxmmG4Pm+WAD3HHJqbBHUYNFkFxpeZl2O7FfQmmFMkHGcVLgdxQQB0p2QiLAxwDSgetPz60nHegQ4EUm4ZxmjAB4pcA/UUALkDvmkBwcjpQVPXikGR9KYC5H40ucdCR703PNKCOaAEYMSMsePSlOCMMAR7ikBz25peD1pgAG0YAwKdncOeaMHHtTCBwTnj0oAdn1GKU4phIxwaAcdTxQA+ijNAI70gG5wcUo5pMg8Y59aM80wFJwaTPOaXANGMdaAEzk0tAFGDu60CYUUcg0E4GTTAKD14peMUlAgNHJFFFAB2pDS0hxQAZooxRjNAATiikxS0AFApBS9TQIKTPNLSUDQdKM0UdjQAGgE9KBQfUdaAFzikzRyRSUAOBFGRTcUUAPzQCKZyOtA5GaAKEWlX1wgaK1lYHp8uKtR+F9VkXPk+X7N1rrn1wocQRIF7cVC+t3TdAo+grk5qz2ijq9r5mNb+CbkIJLu8SNe4Ga2IPDehW/LymQ/wC03H5VUlup5c7pWIPbNRHJ6nNHs5y+KX3EOpd3NO4g0m2C/Z4oyc84FO/thowFijjGP9mssClC1apq1pambk7l6fVbmddrMAPYVSZ3Y5Lk0YJ7UAEHJqoxUVZIG7gpYcAkfjSYJp/Q0oFUIj2k0oXjNPIJ6cUgHagBAKdRRigYUZPajFGB3pALnHXilyCMYzSHGOKQnFMQoyowOlLkbaj3cUZOM0DHE+9GRURNHI60ASF8jFJk1GSc0EnHWgTJCc0mcHmmZI7ioy2W56UCLG5fWmFlBwDVdpVAPPSoPtAB5cfnTKReLimlge4qn9rU5G4geuOKglv44UBYOSTgAikCi3sjRyQOTmo2mCnBNRqSY98kgjHpnNZxvbGe4MUcskr+3ANBqqMnuaD3aKMjLH0AqOG78w/MuM9ieazn1FoZGjggXK/xNgmqE91ccuTsJ9O9SaKjHqdDJcRKSMgH61I2pWsEWGO5vQDk1iXMpjt4SBkMuRxmq6xySZd5MZ6DoaVy1Tiuhoy6u748m28sA5yzZJ/DFQT6pdTDAfy19EGM/WoPK7b2P40hEarsAcn1Iq0zQi82ZZhJ5rHHY8ipQkctwHGAfQdKQRMeAOKVICjkxx5bucZouA9lBbIPPsadGMEhsPkY5HSmxpMWykseR2yM06OQsT57ZwcYxii4FXg3BA6Ve2kW5B7jFU12vcsVGATwB2FXXjeRVG8IB1JPArToZvchUEIBjpT0dlOGBAPTIpkl1bRL5fn+Y/8AsLn9afNqk96I4zEEiQYBxgn61i9zZbDSZJZCxOSadIhQtHklh3PakjaMESEk47A8mnTTIQJAeScAHrQgZCwkzg4/E0H7SkBcqIxnBxzmpEnjDZlGR65oIjJZiMjjA7UxWILeKaNvMSbCkEHzD1/ClwHc75A7DqR0p4JHQjHtSgAnJwPoKQE8K2DgB5cSHp2rNmIS6UBRKBnIFW5JPurgDBzkd6YZxGOQTnuOtAEJwZVdoHAH8J71YmQEhjEWU9MipT5mFSQgk8g+gpsm9mJZlwDgZOB+FUMrI0YlUKRz6VOCobJNRwlJg7CIRFerMMZp0RjdsbST3I6USEShoSxG5ie20f1qSOzeeM5JTjI3PxUMnlhv3YwD+YpmxD1bP41IFWS3iZlDAHJxwelTpYLbureYRxkCp4o4ykgRdzbelIiy3UioIZGbtx0FO4m0iZLrAAkG4DoM4p7XxdSfLJLDAx0/Otiy8M3ckeLiNIh/tcH9K1LTw1Z2pIkkkmB7N0/IVnKaRhLEQWxyO67l3LBauz8fdHX61p2XhrUbrElwFgQjkE9PwrsI0ht12wRqinrgYoMxB61Dm2c8sTJ6LQyrPwpZW0gkkd5mHZjgflWvFFDbLthiSNe+0YNRm4PY5qMyue9S23uYSk5blppAFOTULzcZXJFRhcqDzn3pwFIQ0lmXkmkKZA46VKBgYpwjOM0gGKmRmlCc8dabLc21uv7yZVPpWc2oQzySEI4bpGcZGPUU0m9hpNmlMNkDs5woHJ9Krx3CuvmbwVPAzweKxlsryS4L3E5MbHON2T+VWBZQxyFxu9h2H0q40+5ah3Jbi4vJXZIpMRZ65AzTIVaIY27j65zQA+eAMVLGCW54+laxilsWkkLmY/wCgRzkgYAz7VFcX9vbH55Mn0Q5NZs+s3EqkQKY0PckE1Vgua08q2ke6eZV9gMmsqbW5DxbLkf3mH9KzGBdi0hLOepPWnADHAqgB2lncvNIZD2yeBQAB0pwFJihIA6dqX60uM0uKYhvPalAOKXFHHqAfegBACBxSg46UuSaTHFMYvIFHegnJpDSELQMA5xRnAoPJ6UAJkk8UpBx1yKOnTpRkdKYwpByKKBQAUuMijaCaKACjtRnmigBTSAjvRgEdcUh9aBWCjvRRQAtJRR15NAIKKPaigAFHejvQaACiiigAoFJiloACecUCkyKBQAtIKXtQfbigBO3PWilooAbg5Bzj2pSPWkxgkds0CgBaUU0HJpQe3pQAp6dKXtQDzSZwCO3WgBQOOaAABwOBSA5oHA55oGL3JpM80tLgd6QhKORQckDHrS4JANMLiAgEgg0tITkjNLQMKOMUdgKCBmgQlKKQ8AmjPAphcXjtSUvPaj60gDPAo7mijPFACf/AKqXODRSZoAU5JzRQKKACjNFAHNAIMjpQfQUDGTQeRjFACc5wT9KUHFHUg9xQRQAuaQ5Bzjg/pQBnjvRn1oBATQKD6E0vAHFIYvbIFJjIpBS5/GiwgIoxTsA0YGM4oAj25oLHGNx49+aeQDTcAnFKwzSsNe1CxKhZmMY6q3ORXQWXjGByVubZogP4l5H5GuOxSbQRjmsJ4aEtdilNo9UiurK/T5GjmX0PX8qpy6JbvIZIS0L9iDwPwrzlZZo3V0lcbemDityx8WXtsAJm89R2br+YriqYScdVqaKaZ0TRatZAmNxPGOxxmnR63CTsu4WT1DLkfnUNl4usLrIlDQEdN3IP41sf6NfQZ/dyK38QOa5pKUXZlJ3M6bTNF1QFmjUN3aP5T+lYeoeDGQB7KfevQq/X866CXQo87reUwyHpjpVaRNWsiCQLhB1IHNXCvOGzG4pnG3Wj39kxEtrJsH8YGRVEMDXo8Ovws4juEMZPZhkfnRJpei6ip/dxBm6Ohwa6YY3+ZGbp9jzg1H3rs7nwOACbW7LA9mH9a5680HUbB8S27Mo6sg3CuuniIT0TIcWijxjFAAwfWghlOGUr9aUjjA4461stSRAOcEUdDgGlG8cEg0Ack0CF7UAZ5NHejORQMUjrSAelByRQOgwaAEwSaUDmlBFLQFyNgQehIpCQ4wMflUwIzyMj0ppAJ4GKATKrwlW3jnHYdalS5eT7yMCO571MQDQIlI6UAKAWAYHINFRNG6NlGyRyAakIcxhyMZ6r/hTAXGeMUFADTADj75A9MU4HHfNAhANvTpSjPUUoYDtzTTgnIyPakAZpwJ70mKAKYCYOeRxmlYDPFBBApNxBouA3oPanDJ6UHjr3oBA6GgBNxB4P4U4MSDnrSZBHHWkJweKAHUo6dKj3U/ccetAC9RQQD36UBgegxRQAYpOgpc4pKYADS5pAaUHJ4oAcCQMikHSjoaTOKQC0oPXIpuc0AGmAuAO9Lx60zvThgjINDFYTvS4OM4ozzRuI6UDAgdjQBQetGKBCZ5pCSBwKWloGNoxS/hSUALRnFGKPrTJDNJS0YoAQ5pDnIxS9Oc8Uv0oASkNBox3oAXNGRSHJo4oAWk70ZPrRQAUUUUAaQAJ5OKcQAcDJpxjAOaUcHgVmMbjAoAzTsEmjaRzRYdhccdKTBHUU7Py0hINAACCKQnHbNIMZ5pSaQwBOelKSfTFM3YPXmkLnNMRJkCguKYWyKaTQA/eBQXBqLcO5oLoB1P5UASeZzQXA71Xe6hU9z9BURvY88KaALe/NBYeoqi1+AeAB7GoWvXJ4AA9qYzS3e9G8g4rL+1v60n2iYn5Q2fYUCNQnNG7ArPQ3j5O04HtVgRP5Qch3b0HFK40m9iVpPSoyZieFNNSVLf5pGUY65OcVbbxLYRoM/Mfbmk79C50prYr7JjnOKlW3Z0zk8+1ZzeJUZy1vbZJPBLY/nVW51e9lGI5Fz/FtHQ/WmaKiuprfYwieZcCMHvlqhM1ggwCHPYAHNYmbuQZmLSZ7MeKkS4uIcZwiD0FM1UIrZF3UL2UxqIU2euRWS7TXAAkOT61amljnTO8sfQ1C5WFcnIPpUWNCMWUI4mmck9l7017C2OPKJjx3zzUiTKRkjmmO4BHPWiwD7az8liUkDHsWPAqe4sLi5TBaEH1BqpvUjqR9KmS5EMWdjk9AetFhEW+7WMQsQVB2jjjFSxw9A8iIB3Jpql2GSMZ5xRvI6U0hkksaAAC4Lk9h0p9s8cWd8h5x15qnJNKhxsz9BU0LzMMhQvsRzTsA8OeSAQMnBI60QXH2VmOzzA3UE1FPdXBBQFT/wABNShRNYb48+cemTQ0FxXm2ci0SMnoQvOaYLe4ljad0kRQMkHAFOtNSliQQy2Ekko6njGPrT5p9SuUaMny4TjhsZosF7lezRzIDjrVi6GH2+wz9aoW82zzEa4HmKcDBxxTbbUY/wC0GEhaQKMgds1VxWLqJsXDgeo2jqKaRg55x2BNWhqFrOpIQgnoAMmq73EaneoBx/CR/SkMjMiRncYw2OxqyZN0eYrdYn9XGT+VRDU5y/l+UkansExmpFuVDZeMn2FAXIw0zrljD+AxSpbrJnzZgo9jU9w8UMi/u+ozxVaRpgpCxjcSSMntQFxqCNDsXoKkwp+6c0iWZVN7FyxAJA6CniMDAVwG9DSaGmV3zgEkEHqPSnwyn5hEU3YzlhnFPjmXzMSIC3fg02Zm3nYjKDxgLgUriuRxXE3mAzKBx1PFXIZbcZErJvzkE9h6VWjjnllCrHvbsAMmtG38MavdfMYEhHYucH8qHJLcmVSMd2V5ZjPGyk5BPGBgAelVBG+SqAls4K11EXguUqftOqMPRUSty00bTbFMJCHfHLucms5TXQxliYJaanF2WlXF7ysEjJnkgf1rdtPCFsr+ZcFyvZd3866QyKvA4X0pjTL61Lm2c08ROW2hTg0LSbU5W3BJ/vEkVoB0TIjCKPQKKqtNnpTcs3U1Bk3fcsvLt6n9KhN1kjCkgjrTAgxg5NOCgDFFyRpLseelAQ5wc04D5we1PxgmgBm0CnKMHOKkVCeoqCS+tIThpSzf3VGTQk2FicDPQUpwg3OQq+pNZ0uoXDjFtGEHZiM1CY5JCWnZpGPUk8Vaptmigy62pwjiFWlb1C8CqdxLfXXdo09F4/Onj5eAoA9hS7z6CtFBIpRSK62uOq59yMmpxEQOQBSjcabLIkI3SypGD/eNUUAQEjOPxpxRQMlwB9aypdbQZEMZf3YYH5VnTTTXb7ppCR2XoAKANi51i1gJWEGSQdccAfjWRNqF3dABnCIeqoMCofLA6ACnBSKoBqoAcgcnv3p4U9aMYNLmiwhuCepFKBTscelKBxnFMBmOacBnqcUE0A55FMYCgUUUAGBnnIoooxgZoAUH1ooBopCDNGaKMUABGDg04dKbzRk0xgetGOc0UvbrQAd/akPWigYyTQAueOKOTR0ozQAlHeg0dqADvQaTvSnrQIKM0dqKACkzRzSUALnmg9aSjNAC0uaSimAooPtTec0GkA6ik96XqKAGgADilopDweooAUn2o70A4zRQAZGaQMCeDmg+1B9aADHPWjvQDzQKADoaUfepFULgA8UA/NjoPWgBWBIwDinISuCMZ96Q0CgAopPrS5oAXp0oGMUUnbHegGOzSCkGM0p9KQBxS02lFMBf8KDSdaB+vrQAd6WkOCSR37UuMHntRYANGeMUh6UDpigAo7UdqKAHDrRTeRQOhzSABS96KB6GmAmQeRS5pPpSkdj1oEA5z7UUHGKBzxQMATyDQB2oozQAUmKWjvQAgOOMUp56UhGetHpQMUGjoetIcHt+NGOM++KAFB55px6Zz+FR9s04H0NAD8Ejg4pADTckHuaUk5NIQc5oxz1pMn0p386AEPORTcflTxjvR/KgCPaD2qzBe3Ns4aKaRSPRsVAOCDTjgnOAPpUSipbod7HSWnjC5ijCzhXI7Y5/OugsfE2m3oIMnkv/AHZO/wBDXnWBRjnIrlng4PbQ0VRnqU9raXseZI0YHoRWdJoEaZe0keNuwzXFWuqXtowMdxIB/dycflW7Z+MJlGLmJXC/xKMH9K5J4WpHzNFNM0/M1ayIG0Sj9asw+IIzJsuIWh9z0pbHxFp1/hUlVH/uyfKfwq+9rb3UYJQEdipBFYuLjoxrUpXOnaRqrb3RCT/EvBrEufBO3LWtyXXsrAZ/OtZ/D4DFraVkPYg9KiL6rYct+9Ud881pCrOHwsTinucdfaPfWDAywPsJ4YDOap7eOhH4V6FFr8fEdzGUb0K0+Wz0fVEyY1J/vJkGumOMa+JEOn2POggNGzFdbc+DAzFrS8GP7rjj8xWNe6FqVlktbs8YHLryP0rqjiKcupDi0ZWO9Jt5607OeDkHuKCAOhBzWxIgFHQ0uTigcjNAC00UMcVZezxD5sU0cqjrg4I/CmJFcnvRn8qTnacUDHuKQChsnFV5JHt5ixO+EnIHpU+SBkAfjSMQ6kPGCDQMb9ohkXcGIHowpwZXHykE1EFMYwoDRk8KeoqRQgGVGM9eKAH4I9KKTtRg4piA0dep7UnWgYHc0gFxjuaKMjHBBpM0wHAgZyAfTNOdoXUHyikg4yDwaYBS7QetADMAHIFKSCKU4HFAxQA3rSikzTgcDGAc0AJgZ9qM46UEgEgqc+oNNzg80wHg+tB9R+dAwetBznGeKQBzjijd7GgcClBxQAxZlJwQQfQinHk0pIJ+6M0nU54oAUAZ5oJ54zRQME80wCginYpjMUIBBORnIoAcKBTRKh6kqfcU4EMeGBH1pAFJyPpQcg8jFHIpgLk0UmfY0EgUAFH0oBAPPSnHHbpQAnPekzmnUhAzmglgTmignnoaQ0CCloHSgAd6BhikFOYccEGm/WmAUYo4JpfpSASkxS/Sj6gUxCAGkp9N70AbJik9KPLlH8IqYI56TfpQVmGNko981BoQ7Zf7n6UwiYfwmrO+6UAAIaBLcd4lP40AVcSDqCKCSoyQTVgtLnJh/Wk34+/EcfSkSVfMB6daNxNT+ZCesZ/AUmYD/wAs2oGQZA5JFNDgnrU5S37q35UhitgM4P0oGVy4Bxmml/Q1Z8i3PKIB7mmtBbEjfKQfamBWLGozk9TV77NaYz5xPsDUbRQKcI5x7nFAlFvYpGLceoH1oFox7Vd8yxiI8x1B9N1Zz65Zq/cgegp2NI0pPckaxH/PVST2p6aSuzAZs/596z5PEgGPs1vlvUjNVZtYvZkLNMRnsowBQzdUoo6GGzt7Ygsw3DsTSzaxZ2ZyXBI7AZNcjHI9wTl3Y9zmgIDJtCZ9zU6Mr2cex0Vx4tEq+XBb5PYlcVi3eu3cw2GQxseyjAoUm3kOyEMvuuKa6JI24oB9KFZFJJaIqOs7xglyzMcdalisLleGYAdzmp+EUDHAoQyOhkQlQO5FFxkPkBEKZJB6+9OiiIXMQ+U9KRw7vlpWOevarVujsAiMAB3PWi4FYpISM7xg96kaMFMl3JPbPFOkIGQZCcdyajBkBAEbOD3Ap2GNEM0YABBye5q19kmaIs6Ek96e0bKm51IA7npUaXavGYgXBPcHJoaAhGnSH+D9aDYyIC0iEAdyaR2YKQjyhj0JOaIdzgiUyFAMkZxSsA3MIH3hn60+EJJIFAy5OATwKkEmlg4cPn0JJqYXViB8juvuqc/nQBNNAsJCuQx74PSkW0gkAYoAD0JNM2GRTLHGSfWU4J/Coo7eefcJ5VCqMiNRxmgDQQR267gy7R1zzVQXUc9yzKeAM5qCJbcqRJCTg4GehoMNvM2QfLjBBIB6j0pgTS3CDC53k/wqOajBWRQcNGPrzU1vqMFsPKW1VPQ9STU01wkgwwUn6UCbMuQSFiBKSOMYPSo2e4BAhkD4znccjn/Jq/HC9wpIZFwcYIxRbq8F1kIrEA5HSgLmcNPE6l7iZIwOoVefzqVNLt1+WAnzOzEZNa8mnPIWY4iB5PORmkht1gIIuI89C3t9KBmcLd/M2+c/1KgVPFsjuADESxON2On0rYkktHA3Sg4ORgU3faCTYcAngAnBJpXArPcefiCGE7v7x5AqJrKc4w6fhxVie6W3cpFChI6YYVTju7goQHj+Y/dxzRcCeS1hkAJJLAY3Z4zVOSOV5sCRA3QZPb8q04dK1K6iXbEApOSTxWnaeGSjbp7naewQD+dS5pGMq0F1OcD+Qdkt1yOp7D8atWcst9Pst7LzP+mpHFdbHpOnx4Jt0kYd3GatoEQYRERfRRUOr2MZYrTRGBbaPfPL+/igjX17/hWlF4fsYnLyF5M9ieKvFjUD3IBIz0rNybOeVWct2Txx2tt/qYI0PqFGfzpxlzyck+tUGuGOcD8qasjuOScUjMtSTAEkn8qiaYt0z9TUJU56kk07bwPrQA8sSMU0L82c08pzShaAEABIwKcAe1OC+nJ9qVvk5kZUHqTSGBHFJUMl5D5bNE4kdTgL0Bqj5uoz5O/yFPQIOfzpqLY1Fs03nggYedMi56D1qF9ThJxAm8/3jWaNMG/e8rSN/tVOLRR2rVU0XyIeZ5pTlpAFPbpimqYojgMmfXGaX7LH704W6g4AH41aSWxaVga6Uf8ALQflTTchjgNn6AU9/s9uA0skaj3rPn1u3QbbeIyk/wAXQD/GgC+CT1B/KoJ9Qtbfh5CW7BRk1iTXlzccSSEKeqrwKr7QDwBVAaFxrF1McW4EKe/JqgVMhLyMWY9STTxwOlJn2p2EIBjtS4p1JQAmTil5I5pKOvFUMXNLzkEEj6UmOcUuMcGkIOT3oA5zTu9GMUAMB9QadmlAUnBOPSlQouSwJPYUxiYoxxxS9TnHU9qOpxSEJRnNB4JpoJI6EfWgBcUoGO9FFAABS45wKQU4ZNACUdsUd6TNMYHPSko+tLQAppKWkFAmL1NJwBSUdRQMO+KKKB1Az1oAWg9eKMDPWkzQIU9KTpS0UANzzRQRzQSAKAFpKPTjrRQA7NN7UZpaAE70ZozRTELnjFHakzRSGLnmk7Zo6nignp9aACijFKBnigBO55oFGOM0dqAAAAnnr2o5HUUhPIwM5peCRg0ABOKBzQwIBxwfWjNACgjHfigEHkAj60gHUg5yOlKDgEetAwoHXFBpQM+1ABnilAyaSlxxmgAIxwBSUE4OKUUAJ296Udj7UhFA6Z44oAWkpQQRkdKUDIOOtAgz6jtSUAnGCc0nOaBCk0i+poHpQKBimgZpDRQA7HfjFJ14FITzSj260AA4NBo9qKADvRSZpTQAdqTk8GndqTkc0gAmgelHaimAvajmk7UooAXORzSUZzmkzQCE7ilOQcHpQcYFHagBcZOO1NA5waUkg5FLkfjSATj8aXNJjJz2ozg0AKTQGOaTJNAAzg0APznvzRkA4zzTPlyRnpSHigB5PHNJk8DtTQacMd6AFGSccAetIcj0pCcc4OO+KQ4PA4NADwTjmkIHWkJyo9cUgJx70BcVWCsCMAir0Gs6hatmC5Zf9k8iqPHcCgkdgKiUIy3Q1Jo6+y8agKUu4MY/iU4zXRWGrWmppm2lV/VT1ry3vyBUizeWQUJBHpXLPBxesWWqj6nqc1nDIMGMA/Ssm68P4UtaSeVIOnpXKWniXUbRx86yJ/dcZFdDZeMrSYEXERhI6nqP5VySoVIa2NVJMfAdXsgFk2z/AE6mr0Guqp23Mbp+B4q9a3ttfRebbSrKnt1pJ7KG4yHXIPY1g27jK8ltpepHDQRuOzKAD+lY994Ijfa1hN5eOqyEn9at3GgGMiSzkkjYdgeKYtzq1koDgSAd881cKs4O8WJxTObvPDWp2KhmgLr6x8isogqdrAgjsRXokPiS3JAmR4yezVPNDpd+odo4W9GwARXVTxkl8SuQ6fY80xkUgjI612154KhcM9pcFZCeFYkisOfwxqsHzFVdfVDkV1RxNNq97EODMbae1IASMkVIwkjkKSKVYdQRilHPSt07kkZXjk9DxQAeuaeQTwabtwcDpQAEcZoxSkHBOOKUngYpiIwOcUHPQHilA9RTgOKQxu3FNHWn9Cc0uFIyRTJI/oAKQZzg1Jt9BSFSDmgY3ODg9aN9KQSc0jAigBeoyaBxx2puCKM0AO4NHekBHrSjpkUAJ9aCMjpmgZxjPFA4HNMBQPWkIyeM0p9KOQaQDCSBzTgcjINLnHUCk3D0pgISQaMmlyO9JwaAFAJ54oz780YFGD2oAcpOKCTSAEDpQRkdcUALuJ6gflTDFETkoM+o4pRkdelLmgBnkqPuySA+5qTJ70hNJkd6AH5pO9ISQM/zoDY60ALRjmkLj05pM5oAdnigHNIDSj6UAKDzSZXOCcH0oPXNJx3A+tArC8dqAMnrQQNvBxRQFhT6UmKXtS44yKBWGYxS5oxilpgIc0maccZwaTAoACRmlOKTFJjJoEW/LkHc/nRiRf42FOLk96Zub2qChRJMDkOx/Gk+03I6OaAWH0qRYZH6RnB70AR/arr+/R9pue7E1bSwkI3NwvenmK3j4yXP0oGk3sUxd3HoKlSe5PRRU4yB8sANRy3UcSZlkji9eeaZfspDgs5+80Y/CpRGG7j86yp9ctohiJXmb36VRm1m9kIMIjhHpgHNI1VDuzpJ4/IUF2UA+9ZsuqWtqxLtGx9CMmuWubi9mffcTvtHYcCm25QyZkBKep70W7lqlFLXU2bjxGHkzDFGvuTn9Kzbi5ub4kmUKPQcCpSUMgFvbof9oD+lSmFxzMUXPoefyoNUktEUYbQGQGaTMfrnNWZrSAx5iOB6nvTriRTgRMIwBjp1qBIzje8m8+nalcLEMLPAT+63r6UCZXmBeE+6jgCrAmIGBwKY8ik8qT9KLjsWUmhYbSAuR1AqlLO0RJjKcHgk5zU21D2H50ixRBwdgJ7E9qGJoEu5Z0+fOenTApriQEhY2I9cVKF29APxGaSS6mA2gihBYrj7QT/qyPrUitJwG4PpUjTgwlP4j1NT2Ft9rDAvsjBwSOpFBdinJvLEoOO1WIkBhyxII64qe9ks7TasMYLA5JJ4/CqMl3LeARqqxoDngdfrQS0OMSAFhyR3POKkhe4UA5/djtQ0hKeWAAvoBSCRsYUAn3pisSNcNJgPGxT0J61FtMTGVYSo7EmnRiQks7AA9M8YqWS1lmRWab92BgDNAyIXLlgcAY9Kd5pYkk4B9qFjiQYMgPuKWSMx7XRhgYxigBz3CDBCAYGORTdhl+cbR+lNnknuk+YbgOwGOtNS1efAkLIoHAx1oAVgXOC5/OkjItYztLHJye9TtbW0aZEjKw981JbS25tTEYgSRgs3Un1qQKqyeYu4Bz684q1b28UylBbsCOhzkVMtrIIgEIAPrVmMzwxBCAB3YDApoCgdLIYtMSB7dqnTTrXjDv8AiadJPvkMSTAt2z0P0p6nbCCQSw64FMTLCWgXAjXIqZtPtm5kbEns2MVlSXQklEbTMi+gzk086Nf3k4kgWRo+xIxmobS3E5Rjuxb3zY8qheQep5qsLiNLfaY1Eg710ulaFNbpm8YN6Jjp+NaZ0rTzybSM1PtUjGeKinY4q1iurlAUhRn7Ef4Vojwzqc8ymcRKvck8j8K61I44lCxRqiDsopWcL1NS6jOeWKm3oZEXh23UgyoHI96tx2dlat8lgFx/EoyasG4Ud6YbrP3QR9azuzDnfck88P0yPY9aaWUdTVG9lcXcew43DmlAY9TmkInNwg4HNQNOxYAZGaBH7U4IPSgAAJHJNJsBJyDUoAHNKTkYFMCIx5GRSxqAMU7nGKWKM7SWOAT1NIYhQHpUgBwOKY91awD533N2C9arSanO3y28QVPVjVKLewJNl8ptXLsF+tQtdwR8B959BWe6rMQ08rM3pninq8MQwiAVoqfctQHS3d9M4MAECepGTTDbGRg80zSketO84EcCkLt2GatRSLUUiRI406IM08tk5qJQ7Dpio5riGDPnTqCOwOT+VOwyxuOadgnuB9ax5tZB4gjz/tN/hWbLcTztmSViPQHApgb1xqVrb8eYXb0AzWfcaxcTKRAvlKe561mgAdBS7jTsANuc5kcue+aAABwBTgKOBRYQ0HPY8d6djmlLYOQDmmg+v5gUwFoySuSAKbmlGCM9qYxRzyKTBpdvPFOGcYxSEIB60HGaXODRwB0zQCFFJSU4dM0AITilzkYPekxnrQSM0AA+8OlB6UA+1GTimMAxwcccYozSdaUA5oAWjGT6CjGTyRRikITjtTsYPIpAPSjdzTGHSl5A5FIe3pigZPAoAOpopKKAF60oyGoHpSd6AEPrR3oPX2ozQAE0UD34ox+VAAcmgUtJQIMUd6McZz+FFACmkpD1paAA0mM0E0oHH0piAE4AJ4FHFJ3oxSGB60Ue1BFMQfiKO+aMUUhgRxzQenWig8DmgA69aQYJOOxxS9qMYoAUnJzjmkBPegE59qMnPSgBR6UAEjI9M0mQOT0ozzxQAZozzSc5pegNABkk0UgzjNBIHLEAds0rgAOOKdmmgggkDPHGO9CZwcqV7jPemMeORzx9aMDA9aTqMk0nc0hDgc0ZGQMjI6UYwAexoIBOSOaYCHrSjNGKT8cUDFo70HOaKBBjigZ60ZoyfwoAMUpHGe1J79vSjOQCBg0ABABGCDxSg5pOpzR1GaADPOKBTtwCEEA5745FN7UAGcHgUUUtACd6O9HvQKADkUZGMe9BJJpAMmgBRS8Ug6UDkUAKaSg9aKAFozSHpQMH60AKMEZxSjFIOBijOTQAHnNFHeg+lAAT2pBjPNKQc0Y9uaQC+v1pMUhFLQIToKQk/jTgcEE84pDjdkUxiEjqw6jtSnB6UhOcUAcUDF6qMUq+/wCdNPtS845oAcpxnnrUZO0g5xk4pxzTScjBFIQEYPWlDkZGeDTc89KXaCuaYC5J57UmcUEEHHagDPFIAJyMg80AjFKEz3xRtwelACA5NKQO9JjHalycHPIpDHwXE1q4eGRkI6YrZt/GOqwyL5hWVB1DDk/jWEeaKynRhLVoak0dzaeNYJpCs1u6+hz/APWro4Jba8hEsTLIjdwa8iq1Z6jd2TZgkdfo2K5KmD6wLVTuel3Gl2dwCGGCehxyKy5/DgLZt5jGO2Dmsiz8ZXagC6gSRR/Eo+auitde066QZnETnqsnGPxrlnSnDdGikmZgute0zKmMTxjowGf6VND4sKAC6tQreucVv5X+8D7qciqV1pNjebjKmc1l6jGxzaTrKHzoIt5xw+AfwOaz73wdbSgtaTtG2OhIIps/hmIjdDKyt2B5piza1pIyxFxCPXkj6cVcKsovRiaTMObwvq1vkmISKO6HIrNdWgYrIpBHtXaJ4sUf622cN7GrLGy12AB5E+b+EjDf54rqjjJp+8Q6a6Hn/mr60uQa7efwdYyR7YWaJz3PNYF54T1K1b92onT1Xr+VdUMVCW+hDg0ZAIAwaaWGeKdLBJD8kqFWz0xTAARwa3Uk1dE2HYBx2oIxwcfhTCg9fzo2AHiqEPyCDjtSZpuMHNJ8hPJINIBSaAwB5pAF7kHNIQO1MAJU5GMUmQODQME4JNKRg+1ILjWwBmgcjIOKGwRgjNAAI4Iz6UAAYjIyDTs5HWmkHNNIPqRTAkHBpSexpgyR1z9aP3nYj8aAF5BwTkUhQ5yDx6UAt3AP0o3jGMEUwAZ70h5OM808FScE4PvS7FIyRzQAwE5x3p2cAZ70FCRwab5ZxnOfagCQfWg9aQelBzmgBRnuMijoelISc+1Jk0AOIGaQgZpOc04UANI9OlKQMAYpDkd6XNACbQTSFR708euKOvOKAGjOaXJHelIB6UmPWgBCSOooH6UuKXpketADcehNOGfWmindqADnvRn0pMnuc0c5oAXPrR16Uc/hRzQAYJ5o6UtNoELziijmigVjSFo/fipRaj0rHbXItxEUby+hHQ1FJq984xHD5Y96k6PYeZv+UIjyVHuailvoIOZJkx7cmuUubu7k4mumUH+ENxUAYHq+aC40Yrc6K78QGWLyrKLn+J24FZxu75h8+oKD6KOBVFfJPUMT7UEleUjC+5FCVjWMVHREsv265JA1RgB6jFQ/2VEzbp78s/fBokgLxg7jk9earjEcoBoKsXWjTZt3A+hFJEm3HtVZ92cRIS3bFTCNxHmQkH0BpXCxKxeT7iDA5JNRHD43ADHcU0THGyOI4HcmgSb22hTmi4y3bh0BMcnlg9T3qORMSEkkk9zSK8qdhj3qN3d2zigVwBVOSAfqKeLglcADHsKheMuQSCAO9TosZQK0gUA5wBzQMasEsibggxnpnFKY1w/AGOuD1qB0ckbZCDnAGeMVMyOAFJycZJHNSMiihUIXdWBHcnpQ7jGImJY9KeGBAjZSQeuKRtkcg8uNiR3oAaI5FjJlEhPrnrT40jkjDFHHsTTllZpgTk/QVPMkJOxS/mdwTiqEVkmiU7WhBA6EU2ZyRvjTbH9KspCmMEcigYD7AAT7igaZmSzSz5iiXCHqcdamtYZoUcyLwBwe1XZoZGUokeD/AHh1FNBmRNjuT6g0AV43ABBJPNDMxk2KDk9/SrSxKRkOo9cjpSrCWJ2OCR1J4BpBcbbQBn+aTB96l3YJUHikXT7ifksAPQCp00q4B+QRgepOSaaEU2gyCcsPoKYEkRgka/N1GTV66sryOMCSQYPZByaqpCYRuJO4evamAeTqAGfkQdzkcUk0ZjAL3JZiMnA6Uk7NcALNMdoOcAdaXyP3JEKb8deMmgG0tyxb6Y8yiWRl8ruQeT9KlcQW+Db24kCnkseSfSljs9XmiWG3tpBGBglsgVo2fg9+HurkLx91DUuSRnKrCK3MmTUZJtu9NgU5yD/9agrdagpSESy4/hUZrr4PD2mwAERtIw7u2a0oo44ECxRoi+igCodRdDCWJVvdOKt/DupmPBtxCD/E3JH4VsWPh4JEVuZS/PQcZrfMmOpAqNpVHcVDqNmEsROSsV49LsYVxHbIvv1Jq3uOO1VnulXvUD3JPRiPpUttmTk3uy60oHcVXlvURsZBPoKrEsR1pnl7nBPWkIsm6LDABFQEyM2SSalKhQSOoFRxkEZPU0gHEHYTUiLwKQAmpAMUAV7hWF7CcZAUip0AxmopmzcxknsRipQrEdMD3oAXgUmcnike4tIQBLON3cDk1VOpZkxbQkr/AM9G6flTSb2Gk2XCjH0H1NQzXdrbqS8qkjqByapzma4fMsjMvpnFJHAiZ2x9e55NaKmWoFibVd67LKJlk4yzjAFVitxIxM05bPYcCptrZxgCnhB6itIxSLUUiAQAAYUA+tOERHpU4QHuPzpxCRjdI4VR1JNUMr+WSacIAeTVeXWLGE4RjMfRen51mXGsXMylYx5QPpgn86YG43kwrmR1UepNUpdXtoziFTKfbgViEmTlySfUkmgBR0FAFufUruYFQTGh6hev51UCepJ9ycmnrzQeKBDflxS8YzSHGaCRxg07AO425ptABxn1o4A96YXFyBRkHgkk+gpAc84pcAEmgYY9KVQSMdqVh8pxQHyBkkHHekIQqcHH/wCqlUEDkk/WgE9KXtQAUUuKSgBaSl7UnWgA70uRSYoxQAZBo4GM0vtSAhjxQAvak+nWl4xgdaBwORzQAlL0OaTOMcd6M5pjFpKCcfnQOelACk88UhHP4UcenNAHNACk56DApAMAkE0d6XrQAUmSKXGeM4oxge9AB9aCaKB0NACUY5ooPTg4oAXoSKKTJoxzQIKUjuOhpOgFGaADOOgo60fUUUWAQ9aKBQBQAHOMn6UUZI4oHNABRQeo9qKAA0o60ho7delACnpxSUClFACZJ9PakIz1GaXuKOpoAOetGaDgA4FA5xQAZGOlAo7ZoBoAM0de9BFJjPWgBDzlQRmkQkL8wwQSKXCkk45zjNIeMg4xQApJBJBGPShlRgCwBzwKaFAJAHU0uwZBBPBzjPFAIeMAAAYA7UE5IGDz39KBjFGMdCT9aAAsFABOKXOO3WgDNBODQMUnBxQDzTSecn8acCCAc8GgVg57Uh9KAR3oIwc9qBgTR2zRx2pe1ACYpdpIoBwCe9GT60CuIH5xgkmnHGBxyD1pKM0CsBPA9qXvQaO1AgDAAggHI70h3YBAzk4FB57ClBx06+tIYhBU4IwRwRRmggnk85PJpTgdBgUxiUZoo4yPegBCcnNKKOxoAz0oAUUEccdKSg0gAk5yaM0UEUwDtQOtHOPYUCgBe4PakoHpRmgBQeRS55PFNzRmgAJ5pc0ZyMUgoACaPfNFAHOD0xmgAJoo7UAkc0AKSCeufekBB4pD7UmCTxxigEPwB3pMjsOaAQaMEGgBeMH1FGAQD69qQc8d6d0FABjjGOKQYyQM0vaikA360uMdKKMHFMBDkDijcRjIpcZGM0hXIpABOTikC4/iyPQ0hGKkA+T1zQAAZ44x6ijZycUAAcdBS554oAYQBRyRwKeRk5oBwMZ5oAYNwGDzSHnrzT+tBGTiiwF+x1q7sVASVyB0UnI/Kt+y8YxyNtvYfL/20HH5VyIGKGUGuaeGpy1sWptHqUF5b3cYe3mST2U81OADwwyO4ryeMvFIrxsVYdwcVt2Xim+tE2TL9oX0Y8/nXJPByXw6lqoup2Fxo9pckl4xk+gxWNdaAYmJhZuOg9Ks2HiLT7t0DM0Eh/hboPxraRxKMpIrj1BzXK4uLszRO5yceoavpkoQRu8A/v8AIrWt/EkLcTxMhHUgZrXEKkciqlzottPyU2N6ihu4iciw1CIF1ilB7sozWRP4QsX+a3PlnsOoqvPoNxGTJbSb8dBnB/nRDrF/p5CzxbwOu4HP86FOUdmFkzIvfC+oW5by4/OT261kSxPbvslVkb0Iwa9DtPENlcNiQtCf9rGPzq/JbWWoJmSKKVf73U/nXXDGSW6IdNHlWVpAAOoruLrwVZu+62leInseRXN6j4b1Owy3kmaMdWjwQK6oYqnJ2IcGjMMcezcjjPcE81HmiRHiP71GQ/7XFJjNbpp6okUE5yAT9BSkg9AQR1yKYJZIH3xtg/TrT5Lh5gC4BI7gc0xCFgByQKQEYpjsMfMCRQrDHB4pgPJpCeaTzEPGee/tQOgI6UAKCKXIFIeBSZzRYB4z6cUuajBxnBpQcnrigBwGT707cQME0wnBHegkE8UASA5oIY9DUBQk5DkH60YmX/loDTAm/CkzUeZiM4GfrT1yeCQfcUAOJ/KjjHAxTcnoRilBxnjNADiABnIpAaQ80dqAHde9GARgmm96Xr0oAXBHBoppJoBNADue/FNpSSetJ3oAcCM4pDnuKQdaWgA/hoGaOn0oBoAWik69KKACgUDGcHpQcA8UALSUZJ7UnJNBIoOKM5zxikAxwBS/zoAwj50IJjlGfUCp7eG8uFBN2q57YqMAkgAZzUrRui5AwBUHpF1LOKNTvCO/qTilAs1AWRVOOwGTWcrFuA5H0pskpjbBkY/jQOxovcxQn91bgehIqnKRI24jk1EWJHDE/WozIR1JFArFsJmIksMdh3quDAX2vjd71GJ33YUc+pFSOI8KSpLnk4oGWYZkhRgEySMAYqMQu78DGaYCAQTVqe9a4AjRRGuMHHX86YDRp13jIdAPUmnR28du4aa7hGOozVSRY84kdwvfk0lrbadPcBGeTB7nIFQIuz+TdkNDMCFGDjvSQRGEiSTaF7k1Z/sm2iUm3clj6GqIgWNiJJNzd+eKoBs0webA5Q0FYwwMaEt2xTo7AECQyNn9KljgmOfLjLEdx2oGR+XIByIwfTrRkhNoOCaVxJE2JIXz7DIqSCKO5kClHGOSewFADVs9qb5JsZ6ADk0oAcACM4Hc960hawKwBLEHqSOlJJa2eceYfzoC5iySEZCfKc4z1qGS3crvcyMR3z0rfS1tA2Bk59TVpNP6gINp7Z4oA5q1tBM3Nww9ADnNWTZ3Ck+WMjsT1roF02NeSm36Cni3KkEADFAjBSHUNm0J175pRpt04BeSMN3BrcKsx+6KPsJ37iWHtTFcyP7OwoMpUn0BzirqbQoEcaggYzirU1sTGcCQnsAKTTtNuJsq58tQP4hzUtpClNRV2MQEEEjFDzsB+7ieT/d6fnWmdDhcjzJHI9K0IbWG3QLHGBjv1NZuouhzzxKXwnJo+qTOFFvj3IqaPw/f3she4IRD0J7V1hYgUhfjrSdRszliZNWRj23hTT4GDuS7Dqc4zWrHaWcDZjt4wfUjmmvMF70wXAPSobb3OeU5S3Za3E9DTSwB5qj9okZyM4FMZm3Hk0ibl1p0XgHJ9Kha4cjAGBVdBk5NPxmkMRnds5JpQp24JJoYYpQeB60wGOABkmmgZ6A496fJgISRwKRGUoTnoKABQcUmDvB7CgSAUFyMHAGfWpAkcjacelRIASAary6nao20vub0QZH51DK9xcSDyWCDscc00m9ikmzWAI6kAe5qOW5jiXJYH2BzWUNOYyBri5klx23YFTJbRowwuK0VNlcjI57iS7dTDGy7T94HGaT7JdOp8y7YDuM1eA4oxWigkUopFFNO2H/XA+4XJ/Op0tNox5sh+pFWQOadirKK32PP/LaT86UWUYIJeQ4/2qSbUbW2GHlDN2C8ms6bX3cD7PAR6liKANcQRj1/Fqqz39hbqdzlmH8K5JNYM11dXT7pnOOwHAFRhB6fnTEX7jVBMAI4Ao9STn8qz3UzDEjMR6E8U7AHWnDigBiqFAAGAKcMggEH8BSnAGT0pAx7Z/GqC4uTj7pB9TQcAjHJxQMnFGM845FArgfbrQSQvSkUMSc5HpTgO5OaBjPw57U4g8Y6+nrTsDGKXAznFIBg4OO/pSlc8nHsBTsUZyKYXAAAUUYJGaUCkAgI5z1pCAeaAB6d6UUxhS5GKWk70hC4zignHFGT+FHagBAc8CjBBwaTp060pOelMYHriik6nNAoAWlAGCQADjpSY/OgcnnpQIDSg5FHHWkoBC8GjjOQc0dCaOKQBg44Axmk6GlY8cUg6UxhmlGc57UDkUUCAjjNFGR0pRjjNACE0dqCeaKAE7UUZo5zjvQMKM8UY5oxx7UAB5FHejsaKACikJ5ozQKwUh9qU0mOKBC55xRnFKKQ8UMYZ4x2paQjHWgUAB9KKKKAEIGMkcUoozikB56UxACSeB0oYkAYGcmnHI+tIScgYJJ70hhSAYz9c0vJFAHNAByDn3oz1pCeR+tL3oAMkgDsOlMZQ/BBxmnE4I+uKcRg460ANH3eetHelGOMdqDxQwAnio0ySxJzzx7U/INMUYZucknNADicA5GCDigkDJ6gUgYEH2NOJXGD3FAAGBOAQTTqjAUc4APrQW8tSVOR357UASZpMk01XD4weCKUIQ+fMJHp2pjHfXpS/SmZbJBAwOhpakQtHU0EgHHtmgHFMYp4FJzQee/GaU5H1oEJigdaM5o70ALRmjrSdTQIXp1HNAbGeM0E8Y9KQA0gAc96COAQee4owKXPbFMYc4GaAeKM8YpKAF7A0DrSAc5pc9qQgJoBxSHrQetMYtJnJo7H1pEYNnKEEetADs9PajJJo4xRjmgBASCeKU+1BoI4yKAEpRzSUUALSZo70oOKADNAPFHGetIetACk0UlANAC5oJAGScUh7ml4IwRkGgBM4pA2BgClOAeBRQADHJxyaXBxmgc0Y4NAADg9KUtSZyMUoIIGKQCgjp3pewHemYOc0vNACc8+1L70v1pPpyKYCHGODSggYB70e56UpAJyBxQApz0FN5HANLzR3pAA5PNKetNKk+1AzgAnOKAHAkdhSDk8jBoBFGeaAEOc0oNGRSkgDOOKAG89qXJ71LGkUuQsygg9DTZITG4UsCCOCDxQMaDijOTSAYwTSjFAhQcHcBg1Lb3t1aTrNbzMjr0weKhx6UA5pOKe4XOstPGxUbby3Lg/xIMEVvWeq2d6gMFwNx6K/B/WvNCODSIzrwrED2rjqYOLXu6GiqPqesEY+8CfpSSIki7cAg9Qa8+07X9RsTsMxkjPUOSa6Gy8WWlw2y4jMLD+IZxXHUw84GikmXbjQLWTJUGMnuOlZraXf2T77WUsB/zzODXSxSiVN0MiyJ6qc04hDy64Pcg1kUc/D4ivrYhLyEtjuQQa1rfXrK5YDzNjHsw4/OpntYZ12ugkU9jwfzrKufDcG3NvIYm9G6fnQBp3djHdxEm2t5iemRz+dc43g0ySMUbySf4WOR+dElvqmlt8juV7MnIqxbeJ7hAFuY1mHqAAf/r1cZyj8LE0nuc/qHhy7tDkwuyDqygnNZlzp89squV3RsM7+lemW+s2N1wsoVh1V+KkurC0vIjHNACP9nj9a6IYuUVZ6kOCZ5ORxRtX0rtrnwRCzFra5ZAf4GBP61z+paBfafIA1u7Jjlk+YV2QxEJuyM3BoyQig5wKXIAxjGKe2dvKkfhTdgPfNbXuSN3A8Gl4pQoo2jtTEIAB24FLxngUuMGggdqAGmgEUpxikwDQMM4Oe1JkHpTucYJyKbtwc0AOBINGRnjiko/nQA8kGkJHamg84pQTmmA4c5ozkY4zTeKBzQAtKDTTxyaUHHIJoAXI9/xo4PSgnPvR70AHNGT0oyO9HFACjrQTSAkUooASig4o7UAFBAx1o70EAjBoAKWmhAM4JB9McUfMODigB2cUAgjjrRjvn8KMc0CAk+goBOKCRnpzQDQBmbsKSkbZFRCad5MOrYPbFOIuixZV/WnTXcoh2P8Ae781B6I7yN/Rgv1qEW0byFpCDg4AFUXlJPLkfSp0uIEQDdipC5dCRgcLgD3qLa55RQR71lSvcG4LQhmX061qWMl68YBtyAOpK8UDLDwJFbj5SZCM8dqpgMg+ZTn0Fa0NlJMS8rhfUgZP86ZOIrZgI5Cz+9NMDOKTYDFNgPqeaZghgSc10q28LxqZEy2Mnmq93aSSAJBGqqO+etMDGV2U8RbquRW008Yc2+F9cVdsvKsFIuZISxORk1LJrEBby4wCT/c6UhXK/wBlneMjYQD60+PT41QFhl+9WVuWwGaNgPpTZXuJoysMLgH+IjmkFyIWkzEIJEjj/ugZNTpayIMRNx3OOtMhQ2kYeXcT3zzRNqiRAFdw9sVQyYLLnDNzTWg2y53sf8KYl1LMoaNQc+1I0d5OcJgOemegpiJCysfKAJY9gKlC26R4aMH3PapYdBvMrJLdKf8AZHFW/wCxd33pP0qOeJi68O5ikRs26EbWB61bgNy3Ic7Qea17XTI7Zy3DcelXAir0VfyqJVOxjPFW0iZ32aSSPKzPntxQlhMy4eVgPcda0txHU0bx3IqeeRzqvU7lFNMRCD5h4OelXQiDnHNNaZFOM1BJdAHABFJyb3FKUp7lv7oyKQEHqRVH7U7Zxmm+e5PJP5VJFi+0qKMlhUZuB25qpgE5NLs44oAe1w5z2qMOx6k5owaFHFAIRgSKReOcZpWBzxSjgUhigDcTjrSHrSZ5JpCwB5IFMVhwOKUGoVkBfAz9abJKiqSXANIZOSD3pyuB0GazDfxbtqFnb2p6tcsoKqEU9cnmq5WOzLV7L/okgGMnsTisdLxw5TnHsMg1eFspcNKzN6jPWp1WOMYjjA/CrVN9TRQ7lHF5OuEXyl/vN1/KlTThuzcXUsh/u5wKvksTzRszzVqCQ+VIjRIIwBFEox3xzTxuJ6U8L7UuAOpwPUmrsUN2mnDIGMVA1/ZxsQ1woI7VRn16McW8ZkPqeKdgua2D0xUUl1DAD5kigjtXPTXuoXHDTlU9EGBUAiB5Ykt65xQ0I1ptdQEpApY/TH9KzZ7u7uuJZCB6DgUwLjpRg0WARUCEEED3PWnKAOgpCKcowKoA4APHWkpeO9BIHagBME9qOW7jilPrS5AGDQA09KXAznNGKCMfLj8aAYoHFB6UoBpByaQCZ5I9KXpyTRRnseaYxSfSg57DNJjH0pccUCEyehFKD+dA6n3o70gFwaTPNLkgijAJoASlyM0Dk0oIBIAHXvTAM4ozntRxmk5zSADwKAeBxj2pOc4pwzjk5oAbmkwM5p2KTvTGAFL70EYoLYHJoAXgcnrRk88e5pMc80tAhAc8jmjv0ozg4ozzQCF5zwcUDp70dTSdDxQA4jFJRk4wTnFHYYoBCUucDOMn0o4x3JpOlAxBnjIpee9Lk54pMnOTQK4DkA0c5oB5pGOKAD6UUCgdaAF6mig0gyaAAdaKKOMe/egAPUe1FFIaAFOSQBxijNGKBQAUHpig4HNHPegAAoo7UUABFGTSCjNAgJPr+lAOKO9GCKAHZ/Ok5II9e9FGPagYgXaMZJ9zS5opN2DQAHk5NKBQKCcDFAARQOlNyTjJzjpSg889KAFPtQTkYpAcjPSkwSc5IFACgYppRWyDngZ4pxx2OaZty5OSKQChFTOASSOeaAAyFcEdqAPlPJ4PegHimAoyIyCQR3JFRCSMkqVFSZDggg9e9N8mMkEqeO2aAHnagAAwOlOJ6YPakIyMdKQLjpnFAxwJxg4/CgnjFA70h60AOGCKBxg0duPSmqwJwevpQhDsgHBpQcjFIemAB9aBQSBoJ9qXFIfpmgbFHrnmkFFGeaYCgjPNHGMimjGeaXpSGKDg5oJBIIGPWgAHqaCD0BoAaSQ2O2ad0NNBfo2OPSjNADgxGQOh60Agcmm4IbOcgjoR0pQcEH0oAUc9u9ITRk5GKKAE70pJPWjqTSA80AGM06kzRz2oAPqKDRmigAo6UHpQORQAho7UUvGSR37UABGRR2waSjvzQAtGOc0ZwM96MjFAC8AcGkxz0pDjGDSigBBgdBgU6m4wDS844oACaM0Y4oFACd6d3pCKMZGDQAoOOR3pc54plHORSAeAAOuaQEhsDqe1IDnnpS9CDTABkk5GKdn5cCkJzkmgdM0gDPFKCPSmmkFMEScE00dSCOe3NLxijAHOOaAG9DmnggkmkGR15FNBOelIBxHNA9KUHJ5700ZHUfnQA0xgnI4PqKAZFHI3/Snk4oFMYgcOMjP0IpQeMUHHOKAQeDUiFycnmmZIIp3c0hA25JxTAASc5pe1GDtyKQAg8nigB3OcmkKqT70E0DrmgCaO8uoCDDPJGf8AZPWujsfF8ixBL5PNx1YdTXLZpDWNShCas0UpNHpNlqun6givBPtJ/hkO0/rV8l/9l1ryqNip3A4Iq5Z65qFjKDHcMV7q3IringpfZZSqLqelAx+wNVp9Pt5wPMt429GBwaxLHxdbXRC3sfkyE/eUcGuiheK4QSW0qunqtc0oSi7SRqmmYNx4bBYtbyjB/gbjH45qoZtW0vCfvNg6ZJINdWQQfnXj1AoO04AcEf3X6VIzDtvE6MALiIg+oOf6VsW+o291CTbzK2eoP9RVK60i0uCWNvsY94+lZE3h25ibzLdxIPTdgihMDdfTre/yl1aQsh/iUYI/KsK98FKW3WU5wf8Alm3GPxqKDVL3T5Arq2PR16/jW5b+JLSTiUtGfQkYq4VZQ+FiaT3OI1HRrzTcGSByh/iAzWcrDuDXrsM8VxHujdXU9QDkfjVO90TTr+PZLbqvugANdUMY0rSRm6fY8vLqehoHNdZdeBZATJZ3ClOyvjNc/faZdabJsuImVf72Mj8xXXCvCbsmQ4tFMgYxim4waXIPQ5PvTctu4wfrWxApzjik5IzSjPINAFAxp6ZFKD7ZoxSY+bIJFABmlJIAOCfpSZoBI4zxQA6kzikNB6UAKOTzR0NJml70ALnAyaFb0oGRyMEelJ+FMBxPNJn0pVxnnpQQM8dKQgzQCaTFJk9jTAeT60DHamjJ60oODigYvA4NBGfYetB5GcikJOMCgBSCDwc0ZGOTzSZOPejHfvQAHrSg0lAoFcUmiikI4oGZjGYtmOVQpqCWGYucjeT1IrRtbW2MYEjN9AKtQzQ2+VhgJI6k1B6BkR6W1xGAlqRnqzdKc2kpbN5ZALD06GtC61KQgLGdoHXFKUknt03BgT1OKVgIo5BCoBeKM+w5qWS48wARSg57njNRyaekakLzIepI6U+CyUR/OWD+uadhjpYrgxH94g+lNhtZhGDhD9TTjbBTxvP0NWIZURAjq24e1FgIzdi3TDDc3oDWfcXZmkJMjIOwHatmDShezFyvy+9a1rolvFIZGjQt9M1DkkYTrRgcbDawzsZAZpCOucitS0tVgAcRBMdAeTXVGyibjaoHoFxSpaQoAoQEDpmp9ouxl9aXYw7a1kupGw2MDj0q4dPumQoJFyehx0rUWJE+6oH0p4AFDqPoZPEyb9058eH7jzNz3Kn2xVmPw/b7w048wnrmtYnHWmGdF6mlzyF9YqdwW3gQARxAY9qdsA/hH5VXa9QHjdTDeOTkVFznuy7kikMgBwSBWeZpG60w5PUn86Qy+bhOxBqM3PJANVQMkAVIEHUkk0gFaZy3WmFmJzk0FfmzmlAoBCBSDmlI5zin4zxnFGMcZoGMIOMdKaAwOMDHrUh6YBFICCDz0oAApzz0qQ4FQ+YPUUxpXY7UUse+O1O4ExYUhdR3qvsmPJIGexIqFo+fnuYgey7uaLMLFp5VHQ1E8x7GqTG4D4hTcexNSJa3UvM0ipnsP8a09kyuRjjckNjfye3c1BPdyj5YrcySds1disY4m3k7n9anChelVyLqUoIykg1OcgyFIQfQ81aXTwOXkeQ+54q5k0Zq+RFWRAtuqcKAvuBUoQdyTUgGe1QTXVvbqTJKoPpnmmUP2Cl2AdqypNfgBxDCznsTxVKXV7+UEKRGD6AZFAHRsyRrl3VQO5OKoT6zaxcR7pj/ALA4rCCvJzJKXPqSKcEA6EfnTJL0mt3DLtihCfUZqlJPd3DZmlJHoBgUuMdx+dGaoZGIVHalAApSCT1ox7mkA4EelISATS44pNueDQAZ9KXPHUZo6ccUAZ60yRPcUZwOaUgk+gpv8WO3rQUIDkZCn8eKU5o3EEAgkH0p2BnigEISe9BPHTp3pSADgH86Oc4zmgTEBHpRSkYoyKBhnA5opMfNS4oGHIPPSjHegjigdMdqAF9qMe2aTBzyMUuT0oAUKSTgdBk00nB56etKGwCc55oxkYPI60CEBBGRzSg4BxSYHYYFKABxQMXOe2KQg9qAfSjknmgVg/DFL25o70GgLhxS9O3Wk6DNGSSCelAC5wKXORj8aTBz0pCR0pAKWJ4owD1GaSgDvQAUc9utHQ560Eg9Bg0wDr1696B1H1o6GjFAID1JooooGAPP4UdeaDkCj2oEAODRkk5owMUnQUAL3pM0uecUEcUCuJ1pTjFIM0UDsGKTv1petGKBXEB/OgccfzoI6GjOBmgoBS9qQcjNLkZP06UEsQjnrS4yaTdntigEgg0DBs5xS0E9frQOuT0oAQ8j1xQSetBIHSg0CAHrkUo5pKKADFFB7mkByaAD+KlA5PHWkH3iM+9KTge9AC0mTnGO2aXJxxSZAGcGgYp9R0ppPI6UpJzjtSYOeBnn8aAFzzgdqTn0owcHHXFIAQCc9aAHAE/Sg8Ui5Awe1BI6HPHegBACVGTzk5oQnac8ZJzTs5oJ4zQAgAAyAMH0pAG3gngD9aVelMkycbWwRQA9hnjJFQh2DkAAgjoe1TA5APeoXRvM8wdQMUASgnGSMH0pED7ySRjsBQj7gCQckdxT+BQApPNAAGcZ5PejjBJ6AVE0jghQAMnjigCQqScZGfWjbkfNzmgZI644peo64oGCjGQT9KCAee/TNHWjIBIJH40ALjijJxkD6Uh/D8KDQIXPFNwAc5xSjpQQCKBi1GxByCcU8HBpCAScDrQKw0Nt4608HOAemetN2jPSlAOeOlAWHkofuZwDjmkzSDAPTvmjvmgBc0gBHWnZA65xntTcc0ANAYMSTkHoPSngDHJ/ClzjgU09aAFoPSkozQAuOho4pCTwBQaAA04fdB/CkJ4AzxQKAEA5xRQeTQDkfhzQAE5oBI5FHal7UAAFBxR1owe9ACE8ZoIPB9aKCOhzQAYJIFBHpR3xQaAA8jBoHrRR0GTQAfWjFGaMjNADiSWwelGKGwWyBxSdqADNHP1oGB1FBAJ4GPYUABHSgA4yBSd8UHPagELnijPGOaTFKDxQADk0tN5zS5oAXPNGeaOMe9IaAQvIJBozQTzQBkE+lAAD8xp2ARx1700Be/HvQWAPB5pCAgg0pJIGaTOTQetAxSaB0560A8dOPWjtQCA0CgDJxRjBxQAd80oOQM0cDrSc9qAFA549aM80uWHBHbg0056nvQAGjknAGTSdqCOM9KQBz3GD6UEkU44Iz39ab9aYACQOuKD1zS0UCCpoby4tiTBKyE9wahFLUyipKzQJ2N/T/FV7E225cSqf7w6fjXRWmv6beMqPMscx7Hp+defNSKSDwa5qmEhJ3WhoptHrAWQDMb7h6g5FN3AE+YhDeoOK85s9d1HT8C2m2qOqkZFdDYeMElUC+hAbu6nH6Vxzw04K7LU0zo2iWYEOkc0Z7HrWXd6BaztmItC57HkVo293ZXo3Wlyrt7cEVMRIDhgGHvWLVi7nKPpeq6a3nW7E46mMjP5U+HxRfQkrMgkI65BBrpwyHozIw9qims7e6Obi2Vx/fAwTUiuRWWv2t2ApzE57ORj860W2yqVdAy9wRkGucn8OIcm1nxjokgJ/WqhGsaYgQbwnbA3CmmBv3WhabdqR9nEbHqy/4VzN54Juo3Js7hJE7Kev61pWXiRhhLuPPrICB+lbUOo2c4zFcRk+hODWsK0ofCxOKZ5pfWNzp0uy5iKD+8elVgwPRga9cdYpV2yRo4/2lzWXc+GtMucnymiY90OB+WK64Y3T3kZun2POMEnIoIPtXRXvgu/tyxtpVljHQYOaxLmxurLH2iJ1HqRgV0xrQk7JkuLRWIxSEHHSnggjIOaTODwcGtRDfpSGpCMjJ60wg0yRvNLmnADvSECgoQZzQTijFIR2oAUOfUEUu7PSmYI6UucHigkeGIoB55AppOTSjFIodnngcelIcYGM5700ZBo/nQSOx70UnU5FGTmmUBJpQSaTNGaAHYPpkUnQ80mT60detADs4OccUE5NICRSbsHn8qBWK1vp94/7yS4KRjqQMGn3UMdvGAJWfPqelb66VcO4WQ4XuRV9dGsyBvi3t6k1i5pHVOvCPmchYRLlWJ3t3xXRxWzlgDkL7CtWGzt4ARHEqg+1TYA6cVDqPoc08S38JnrajcGeJSfephbRHrCv5VZIphYDrU3ZjzS7kf2SDvEopfslv3iU/WlMygZzTDcr2NK7Bzl3JlVYxhVAFOBFVDcg9KaZmIwOKVyblwuB1NMMyA8mqZJPU0houItG7A6AmoXuHYYHFR4JpQuKLghpdyMkmo8ZOT39anKjHSmgDOAKAGhMmpFTAxilAwcik3ndjNAx4TmmnrjGaN+aQt3zQA9eOopee1RhxtJLCojcAEgHn0oAnK7hTQAMVVaeTPBxUTSknmQ59BQM0CwHU1BJcAHA5rOuLprZd21jyBgnk1UfULuSMiO2256Nnmq5GPlZrvK5BANQtI6jLMKyhHqEw8tztX36n8alj0tmOJGJHoDVKn3LUCVr1VB6O38Kg96bHqGrudqRBR7AVdt7KGAZWIAjuTmracdAB9KvkRXKjMXSZJ8vdTSFm6jNXYdOtYcbIjkdycmrPNKATVDE2gdBikyfSn7T64+tVLjUrO1wGkDuey8mqTAtAk04KT1rFl19iuIIQM92rOlu7u4J82Y4PYcCgR0M2oWlvw8oz6Dk1Rm11OVt43c9iRiscRr6U9BjpTQ0yWW+vp8h3IB7AYFQGINgtuJ9SamBpCcUxXECqrAgcAUvHYUw7s5GOeuadnA6ZosAcdqVuQBjNJ1p2PWgQ0AgDNHOeaMgHinUFDaUHA4pDQBxQA4c0oPJpozjgZNIG45696AFPNHOe2KQnPHP1oHQDtQA7JooppYnIwfrSEOXmikApelMYlLjBpARjng0p5HWgAPI96AMigLgY607GBQJjegpeTz3pBjvQT6dKBinp2pKaSSOMZpRkgEjGaBinGMZ5oH1oxRQAtKoycZAx1zSHjjHNHb60CuHvSZJo/lS5z2oGIo2inZwCMcmjJJA7UcHmgBMHvS9qXgnOPwpO5xQIOMEHvSDpSkdDRnigYcjBOcUUH3JNJQAtB5FITQowOKCbhQOcUHOB6HtRQMd3yRmk5HB780DPekGAc56UCQvOeDg0uaTuaDj0oGHJFJ2pfxpKAAmlGMUmBg5H0oAx0oAQj86UZAwTnHegciigAzRRigYzzQAdKOppOe9LQAZoB55H1FJQAM+9ACk+wHsKKaeaXGRQAcE8iig0daAAmlB56ZpOo/lQTg5FAB1GKDQP/1UHB60CENKOgzzQAMkknnoKTPHAyfSmAA5yKQdaOTx0NLjBwaQxexIptOBABFIinDHI60ALxSE4FHUUhAIAIyB0oADyQfSj5ieB+NAJBOcY7UuSKAFbpgHk0Hnp0pueaeo70AA4pG6ZozzjHBpADknOc0AKB29qQjjGT0pGySPY0AYJ5PNAACMkZJ+gps2AmQQDnqacMgcnJ9aCAV5AP1oAAGVACQSByRQcnpRwASewoV1cZU5GaAHdQCetGM1EZlEmzBz69qeA3sRQAAgDHJ9QRUg2kgsMgHnFN+poJ4OMHFAMDxQKZG7MCWA9qk7ZoAM0xkVzg5ye+adkE4FAXBzk0DBV2gDJOKCQD3paaJFYkKQSDzQA4ClqJicglfl9QaeBkD3oAfGELfOSBjqKjI5yKdtAoAGDmgBBnPNPONo45Hf1pACOSfpSE8UCFIO3OOPWkGRTGBcbAxA6kU8cD6UAO5xTAxLkEcdjS5oLAEAUAKKQ9alhaEbvNQnPTBxioyKAEPSjoKU9aVsYAHNADRRx3/Cl9BSUAGOM4o9qP4aB60AHQ4ooOQ2fWgUAApc8UgGTxSjHegBAfyoJzSgYY9qCBQAlHXoaKMUAGaXjI703nOKXIB96AA/ePpQeRQORk0fSgAzS4GMim0hYL1IBPQGgB/Hb8aO1NzxkH8KXnFAC0bT1oBoyelBIh+9SijFJ07UAOI5pM4OO1ITQBmkUC5xkig+3WgEjgE47g0EZB9qYCg9qTpxmjJycijO4Z6H0oAU5xmgjvSDPfpQT6GgBRkmlIGcGmqwOfUU4gHqSD60AAIC4o4PNNOc8UowMZPSkA7FNIOeCPrSk85BxSZwMkdaYJgADzyKMHkZzRS0hAecUo9KBjHNBOGGOlA7hz0pwIIphOG9qXb3z1oACMHmlJGDk0hz68UnIwQKAAMASCDgjg9qOpoyfTFBNAClTjNIDzSEnOSaUcjIoAUYJAzQc44oIH40mKAFB9aUeoptAODjFAASRSgg8Ec0H9aQDmgLEsM0lvKHiYqR6VsWvizU7farlZIx/eHP51h0c1nOlCerQ1Jo9BsPEenX+RLJ5Mvox4P41sBGZQ0ZDKehByK8kIrR07WrzTAFhlOzsrcj9a4quD6wLU+56Uc9XQE+o4oBQcJIfZT0rlbLxqZXCXsA2n+JOP0rora6sb4breZG/wBnPP5VyTpyg7SLUk9guNNs7nme3Ab1Q4NY134bcAvazBh/dbg/zrfIYdGJ+tJkHkghvUGpHc5UXuraOwSRHKDtICR+daVp4ohkUCZGjb1ByD/hW2VLKQwSVD1B5zWbc6Fp8+fla3Y/3RwKQGlb3cc6B4Zkcd8HmnyhLhNk0aSIeoYVx934e1KzXzbYrLg8FTg/zoi13ULPCXCF/wDroMH86d7ajNq68KaZcHdHGYSf7vIrndQ8E6hC2+1ZJk92wa3LTxLbTMBOpgPrnIrZSVpYw9vMrqehB4ranXnB3TIcUzyu4tbm1YJPGyk9sVCGGK9WdmkBW6tlkQ9SQDWHeeE9MulY27m2c+pFdMMZ0miHT7HCEgng0uK2Lzwjq1oCyxCVPVG5/KsqWOW2fZKhVvQjmuuNWEnZMhpoZ2zTcc0/cCOetIOTitBCbcjIpMHkAfSngleO1ITk0xEeTkAjHvS06k25NIYE570HNKFFIePemMAcGjPOcUcHpQcgYoAeCCPSmkEHkcUgJoyQRQAfzo570p5570lAAKAce9GRjmjjHvQJnbFwOpAphuEX+IVlSOScEnFAyTjJNcZzmgbxAcdfpTTensKqFcYpwFIB7XMhJOcVXaaRnxuOKe5wcUwDkGmBMAT3NOA4xRGOMmnnApDG7eOlLtpaMgjgUAGKABmjOKAQDnOaAFAoIwaDKqjOaga4yeDmgRLu7UZA61UDSOxwCPrUbxuDmR8L9aBltpkBxkCoTcIGJJqg91BHxksaYb1Sm2KBifcZq+SRXKy+05f7p4prF9mWkCr71QQzsCXIjXH0NIm3JLRSS+hGQM1XsyvZlyMNM/7pGde7DoKZLczQOVEI3f3iamt7i8fgQqgPQYwAKnNvJP8ANcSDd6AVXJErkRjm4uWOZSoX6VG17cJJthUEnoduSa1hpdqrAkyHHYtxVmOOGJdscagfSrKMIW+o3bAsm3PUt0FW7fS3iYmSbzGPb0rVJ3Dim7SelFhkHkt7VIkZHWiRymMRsxPQAdar3V5fW0JmFg4QfxPwKLAXSlRtcW8C5lnjX8c1zM+oXt4xLzFQey8CoBAAQWJP1p8pJ0EmuWqkiMPIR6DFUpdbu3OYY0RfpmqOMHIpw4FHKJMHuLyYnzLhsegNRiMDt+NPIzRg9qoaFCgDOKDThnpSEUgsAwRgdaATnGCKAOARS89aYJAKBnJOKM5OMGgDmgYu00BQRnNG4A4LfhSnHXqKCRDzwAKM0hIzxz7UZ5oKCgNRnNAWgAY5I5oJxQAPSjigBMEkHcRjsKd1NBxjikFACnAOBSY7mg80YwCeTQAc9qUUpB25yAM0UAIN2eSMUoBYnHIFBAxRgY4JyT0oAMDvQTg9KU5BwKSgVxwJ6imkkjIyTQScYyMd6aM9P0oCwAk+1LgHOeBRyDjFOxxQO4mAOAKKMUYoGBORgUUYpf50AGCTntSgc5pA2M8ZpATjpigQpGOvSlAB70h3E5OPwoJwOR+VACkc9aSjBBAIxnmg4zQAuKO9Jg5x3ozQAtHamk4FKDxmgYZ6CgAjOaCOhAzilB4zzz2NAhAQ1Lj0pABzge9Lng0AKSePWm4oJOeDSkc9c0AGaTPOKXjOKTIzjvQAKMDv+NA6CjHGSaKAD6UvXpScZx2oII/GgBSCRnrzikPH4UAkjBGMGigAyccdaQZzzRnn3paBCHI6UoyBz1oyOh60maBij1pDyaOaOB1oAKM0gJOOKKAFpOeRRSigAyMHNA5OBSEAkZGRQcZ4oAdmkNIBjiloAD/SmKmOpyakx6HmmZwM8mgB3G4Z4zSA8/0pOvpn3pc4OR34FACk5ORQepJ4zSAEAEjiloAQ8gjtQAFwoHFA5PH40uccYoATGDnJoPGPfpSk8U1gOMHBPHNAABgUDIHPWjtj8aBzjJyDQAEA4owM5JwaMEH2pQQTg0AOGM4IzTe5APSlGAck4FHIPp70AKOM55pjDjgkZoyQRnnmm7DkkEnPSgB4OMA9ex9aQ4BzmhM459aGYAEngd6AADcpz3oAESsSeM5pQcgEEEEdqDz6YoAYXRyoDjJ7U4AoODTVRVbAAGacp3SMmCAODmgALbsgc460ufpmkEKLIHGcg889aXHNAmKB8uO9GcjANApCBggHB7GgBCDnIOKfQCQwORkdfenEhiSQAfakMbnHFN2AMWAGTTh1NITimMRmABz6dPWlQ5QHpmgjPNLgcnjHtQAoOaQ8n2oFB60hATz14oOD0pB7jtR1XGKYCgAHPeg9aCTtzjmjntQAA4IOM47UMRkswx347UhJGOKMZHIzQAoIIyCCKVTz0H41GiiMYAxmpExnnigAbAOOc980mMcdzzShgQR6GkOSQQM46/SgAOSOelKCMZ60hPBPbr+FIxwMgZ4oAUnjHaj+VNDZGaUHigBT+nakzzSnpSYzmgBc8g9KCeaQ+hpOwxQA7sPU0gJyetNAKk8kijOMZoAfS96aBznNKeCO+fSkAUcE80hPGaQcgEd6YDj7HijIA5ozk8DHag88UAID3HQ00qGxuGcU72FIc44oAANvA4p3SkB46UZ55oAUHj3pSelNB5PPFBOelBI7ggY6ig9CD3po604nAB5JoGBAxik57UpOeaaMg0AhTjOeSO9IACQVJxig4ORnmgDimAnJyKACAQTnng04DHJoLAHOOD+lIYDpg8nGKbgBuOvpTiVGCTgg8cUH1oAOnbmgnAyaAc0o5OMj8TQAmQacVUjJJ/Cg8HBHIpDQAEcYwDTgDjggfWmkEdaM+tBIhJDYI/GnAEnrTd3zYP50vfOaCgB5pxOOgpuRnB4oP3qQDhkg+1GBt680mecg0A7eozmgBTSA5OB1oJGcgYFAI6igAycnIoozSZoAMDvSjjocUoIzgij5c8GgABpGJxxQeBwRQD60ANBLDOc0oY9+aWggDkZ/GgQvJAOBg9DQQOo4pANowOATnFITQNsUHJo5PSmg80tArjgMmgjB5pvJHBpcnHJoAB7YpysyMHjdlcdxxTckHIoByaVrgb9p4uvraNY3IlXuH5P510mneKLDUBiT/R37hyCPzrz3aBTeQeCRXNPCwa93QtSa3PWdqSANE4YeqtxRk9Dgj0Iry+zvrqyl3wXEig9QDxXSWnjV1xHeQF1/vjAIrjnhZxempammdX8q8pI6N6EUSolxGVuIVlU9TioLTUdPv0BhlAY/wuQDVoqyj92cVzlmLP4atJiTaymEnsRkfrWbLpOqaZkwiRgOrRnj8hXUjIOZFP1Ap+WHEbkjsDxQgOZtPElzEAlxGHUfn/KtW31jT7pjljG/o4wPzqzPZ2dwCLi1Ac9SBzWRc+F9/NnOp/2XwDQB0STY6EEdiCCDUN1Z2l+CLiFJB3PeuPI1TSJBkyx49BkVoWviYo3763J/2kwDTi2ndASXfguyfc1o5hY/wkkiuXu/DGrWbEiBpUH8SDIrv7XUra8UGOZc/wB1jgj8KuiTb3xW8MTOHW4nBM8gfdG22RCrdww5oBUjOBXqV7pWn6muLm2Rm/vAYNYVx4IgwTaTFf8AZkJ/Q11xxcbe9uZOm+hxHINOBOKv32halYylXgZl/vLyKzyjxYLqRn1rpU4vZkWYvWkwadkEcUhBA5FUAU0g546dacCc5pCTmmMaOBmgknmhjS7SAD2NArDcnNLSZoGKBCnkUAcYzmnBuxAxTCCDkHigDeZQGJIqZQDgjHSmkAnBqVcBRyAK4rmAh4OKRjg4oMiFjg5xTHOTkUXGNIyacRyD6Co95BweDTHLM2AaLjLJlAGc9KVZgRk1UCvzupy4ByWwKBFozgUhuCRgYz9KrNNCM9T+NQvfJGPlTIosBbzIe+aApPLPj6mqQnupmwkTD3xipTp9zImXfbn0NUoMaiywZIBwZR+dV5r23jfbEfMb27UsWmQo26TMh9zxVhLeCNsrGo9qtU+5ah3KLy3c7KIlKD1oWyuHX97Kc+grSLr2FRm6gX7zqD6A5NaJJGiSRBFYxxkZGcetWQiDgAUB1kAK5wemaCVHUr+dO47CmNTzgVIDgdvxqlKL6bItwNvqOtU5Le5hG+TJcepzS5h2NnJI4ANRPIV6si/U1mxxahcYCpJk9MHAq8mjTRxGRyrv7npU3CxPbvCVO+VWfsFPFLJdW8Od6rkdgM1QayvZpAIhHEP1/Op7fR5HlAnleTPcHj8aZNhP7YYnEMJYdgFq1FdXcmDNAkMYBLFhggUMttYzDLgKOpJI/Ks2/wDEAMpNiN3GC5GMH6VYy3vudSlzFItpbAgCWTgnHpTdU1izs9Oewt5nvJm4LnkD3rmLh7q+ZftVw7qDkKOBSpGka4QYHpmmSLGNqnI5NOGCORSU7k4xwKoBOgxRjPegAjO7HtThgHFIYgAxSjgZoGT+FA6UAGDSYNKW4xSE8etACk9MClJA6+lIFJ6mmhFJJJNADzJxgACkyR1pMc8UY/OgQoJJ6Ud+TQOBQSO5oGGaVcAE4zmmnA5PSg8nr+FACk5ozikwDS9RQAvP4UhHPWgsc8DNAyRk0AAoJpcUhAPWgBcYGcUpHajJIwT9KTPOO9ACjIzxQCAPU0djSAgjINAxQc8YpCOcmkG0A4BPrSFySVHAoAeWAB9aQ56459KBx1xTiM80hDADjOOtPxgj2pCSTn2oBoAKUUZxSA55HBpjFNGMmkBJJyMUpIAzQICQqjjBoBOB70nBAPBB6UuD3oGIGPYZpScigjAGe9IOtACjj1pc0qsVJI7jFNoFYXBPJ70EYNHQA5/CkPPSgLC0Y9KTFOBxzjPFArDAwJ9aXPOMcUM4wMjHpxTgARQO42lPSgnJyTSUAKOmO9IaKOp4oAMj1oI6GkUHGDS9s9qACjPeijntQAvBPcUmOM5oHOc/nRjjNAAMg84ozSDriigBTR9aOD+FJ1OKADAznvQKCQDijGaAFJ9qTOMe9BPFAwRQAUh6UDqR0oPGR1oATjk57UYPFBBBwCMUooAXjNBIHNJkD60p5/CgBFPGR+VB5NLySc0DHbpQAoNJ06UEYpPU0AL2ppJ4xxSEkjj86UYI47UCAgk8CncHK98dqTpz6UEd6BCYxjJJNKCKBnA9TQQc8Y/GixQgI5IozkZo759aDnBwMn0oAAKBxkDBPXFC9cng0ZGd3ORQgEI44AoBGQMH1oBJP1NKM5waAF79aADzznmgHqCMc0Y25oAQ4HODj2oL9Bg8nGaU8DnIppznjigBSOQaCCykClODjAxwKieRUOMnPoKAJFUKCBk/WlIDA5pIyHjJ96aC4fjBXuDQAnmJDGATgD2qQEFQQeCMimhVZACAR704jIAHQdqADikJw+O1OAJIHrTAVfJGTg+lADz19hQOpPX0psiCRQNxHPaiNPLGMk89TQA+gjjOOKQkjGBkmgk4xSAXFBODj2pAcEE9BSsAeR0oAB1zRjJFA4PtQTTBCElGxj2o7g9qUD1FKTkfpQAhIYAjH4UgwcjOD2z3oAAGMfSgj5gfSgYuMCkJIGcZpc5pSCAM+maBDe1HmL2I/OjntTSilskc+tADzggEdKBQCAPYUZBoABQelISRjAzzzzS4/OgAHHYc0oYqcik7n9KMUADHJHbikBGcA9qUdaQgjkDNABgVNMsIjjMJOSPmB7GoaBQAoIBGTQeDRkjtkd6MgjIoACM80A9RSAnFBzjjqO9ACEA9RSZI7ZzTsEgHOeKbQAHIPt60o45pBnOKUjFADgRnjmm/hgUAAHI6mg5Bxg4oAXGKTvS5wfWg89BigBRRimqCeDQcg4HWgBR1prZ6DrTugoLZPOKLgM5AwRg0DjB7U8gYpAACR27UCAkEcE0oGR1xSbSQSBkDvQMg80ALSjHegEHikNACng0HOBikJJbmlBwcnp3oGAJIyBxRjPBpeACATikoATPUEUoHU5/CjGeR2oK5APoaAAZ9KDg9fyoBz1HUUhXtQAuOnOaMZ60g4xS5zQA48004PBFAOTgEE+lGD34NABiijIxSZNACvgYB7il35AGOnek69aABQApKkggYPekJJowM5o4oAcoJBJI47UAKOST9KTBpQMnk4pALlTwM5pCAM4PWggDB60EjtQAEk9aMAHNBJozQAhODQORkUEgDmgUAPOCOuCKaPegkYHY+opoJNAhxNJjilAFByKBCAYHPSggnkGkycYPNOU8GgY3JAxSg560h5NBJHWiwx1NGQc05ST04PbNBLZIOKAAsDRnKgelJtGeRS4I57UCA8UpH5UhPNJ8469O1FgHl3C/K5yOla2meJL2wQIZnaMdFbmsbOaUis50ozVmhqTR3dh4xtLl/LuYmgP8AeBBFb8ZjmjEkLh1PcGvJQWUgg4IqeC+u7V98NxIp7Ddx+Vck8Hr7rNFU7nqBEi8EcU0wocMPkf1FcpZeNZI8fbozInfYOa6ax1Ww1OPdbyjP91jg/rXHKlOO6NE0yYvcLw6iZB3HWqc2l6XeneYzG57jINaO3sKZIACN8eR6jqKhDOZvfDN1D89u6yp+OfzqrDqGo6dIAZJFI6q44rr1O0jy5D7AmllCXCGO4iV1PqM1NwMW38TW7EJNC0Lf3gcj+VbUM6ToHhkEqnuvase58MW0xJtZjCT2PI/nWRNpeqaXIWQHaerxnj/PNUmI7YSDGCCAeoPQ1RvdG03UIyk1smT/ABLwa5618R3MLYuEWZfUnBH+NbUOsWUwG25Csf4XPIoUrO6A5698ESQsWspPMQ9Axxj9a526sL+ybbNbypjuV4/OvUFlI5+8vqORU+9JlKuA6nqD0rpp4qcd9SXBM8fLMpw3BpRkivRr7wrpl6Pki8hz/En+Fc/d+Cru2jZreXzR2ABrrhi4Na6EOm+hy7DjNCkgdeD1FPlguYHKTQyIw9RimggDDAj611JpkCYOOO9HbBFKAvQE/Q0EHtTAawJHFICRwelPHvTW9aAN7fTGkY8ZxRRXEcwsdvJINyjj1okjMK5kmCj2oopDHQTWAjy0pY+pFU5r+1jlPl/nmiiriruw4q7IRqPnNtiyx9hVoWN3OAS2wdxnrRRWjio6mjilqWItI53M7ce9Tm3iifLICfeiiiDuwi7sd5noAB7UhYnvRRWhY05NJgmiigY7ZkEEhR3J6Cs99Q020nIQAsOr4zk0UUAh7a7pz9ZCPwIpo1fSc5JY/gaKKg1LQ8QaTCu1Hkx6BTUT+I9OH3Y5nPsMUUVRBE3ia1H+rtJWPpnFNHikEYFi4+rUUVQEY8TybwTZKR6ZNR3Wu3l2uyMeRGeoXOcfWiigkzfLBOSSx9WJNP6CiiqIE5PY0ANgZGKKKYDscGgEgUUUAGQTQeuaKKQwDAZ4BoySewHrRRQIARiggDmiigoMnGAeKM+tFFAC84OBSDJGe1FFACHIpe2SeaKKADPvQOCT1oooAXGaXFFFACckj0pcUUUAKF96OKKKAAjvSYBHeiigA44B/GkPWiigEJgkEClUYFFFAx2B3GaXOAR37GiikICTikAOTk0UUCFC4HX86BjGaKKBoCMikAJ47UUUwF4AAHagHk0UUAGMnJpcZJ9KKKADpSfjRRQMKOBRRQICcUUUUAGBwfSjvRRQAhAwSCaXGQD60UUAHQUcEEUUUAAGB1oAwMDpRRQAuQcZ4poJIyBRRQAA8AjoaAR0oooAXOM470dqKKAEyAKOooooAOgAozxRRQAmM0vfFFFAB0NJnj8aKKAEBJyT0ooooAOCRnqOlAI5oooAUHJFL3oooEBGRTSOMY60UUDF+lNAxz39aKKYIXOR0pVJwc9qKKQARkZ75owOcntRRQSIMUu70oooKG5IIx2pFZ3YkrgdR70UUASjABzimZOc9qKKADBP55oPJx7daKKAFIPf8KBnAyRRRQAhqMwgsSSeaKKAHAbFBwfwp3eiigBsbqUOwkgHFOLcHAz/ADoooAhMjEhcHnrUqgqoBHSiigBcUjOicMcE9PeiigBSQCCAaMk0UUALg9hSLGAMsTn0oooAeOlIQQRn60UUAL1PtQeKKKADrR+dFFAIYFw2RyaeCBjOTnPBoooBik55ximk4Gc4NFFACBflwSSKOgoooAUb8DOMdqXnHWiigAFGCScdhk0UUABUgjIIyO4pc4FFFADMktjFAyM59eKKKAF3AnGeaMjOO9FFAB3oGe5zRRQAnPGOhpMnJGORRRQAE8e9G7OARn+lFFAADyBntgE0Z6Z/OiigBx4NGepFFFACjOCTTc8+9FFDAXJ6ClxkcjBoooAQmkOR+NFFACHIPf60Z45PNFFACkHGR0pAecE4oooAUyADGMmgMGAI6UUUABz2p3JoooAaQTwCQfanAnNFFABz2oHIJoooAQgk8c+lJgjqCD6UUUALtHUcH2pxIIGRz6+tFFACEDOQeKOM470UUABIzzxRwBmiigBeopOhHPFFFAATjoc0En0oopAAJNKKKKAF6ikoooANoIwTg0gwOM5oopgOynemkjdgUUUgHYOAc00ruOckGiigBwGRjv3pMYNFFAB3pSMjBoooAbjBwKdz3oooACaac4oooAcMHqMUEgcZ4oooAAwB9aTNFFAgBpwBoooGB6Yojd4iWjYo3XI4xRRSaTVmI3NN8V39qvl3EvnKOhfr+ddPYeKLC94kQxn1JGKKK4MTRjFOSNoSb0Nby45RujKsPUHNIY5F7cUUVwM0Gkjv1pwkccAKw9KKKAK1xp9leLiaBVPqDgisS58MTAF7aYSD0IAP86KKAKCTanpLkEzIB1BHymtSDxMjcXFuAf7yHn+VFFCA2LXUba6A8m43Mf4D1q4JccEEfWiimASpBOMSJHJnqSMmsLU/CVlfjdE5t5exU5H5UUVcZOLuiXqc7e+Dr+1jLxt9oPonWuecSRMUljZCPUYoor0MLVlO/MZTSWwb1PfmgYNFFdhmf//Z" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAQ4B4ADASIAAhEBAxEB/8QAHAAAAQUBAQEAAAAAAAAAAAAAAgABAwQFBgcI/8QAVhAAAQMCBAQEAwUDCAYIBQALAQACAwQRBRIhMQYTQVEUIjJhQnGBBxUjUpEzYqEIFiRDcrHB0TRTgpLh8BdEVGNzg5PxJTVFVaImNkayVnRkZZTC0v/EABsBAQEBAQEBAQEAAAAAAAAAAAABAgMEBQYH/8QAPxEBAAICAQIDBgQEBQIFBAMAAAERAgMSBAUTITEGQVFhcZEUIoGhMlKx0QcVI1PBQpIWM0Ph8CRigvEXNHL/2gAMAwEAAhEDEQA/APmvg98cFQaWrYY5AbWIXbAWXIVQjoeKIHywuDZDbXS1wuykC/Zv1CB8TXdLqB0YAtkH6K07dCWgrUJMqUkbjtdRAEOV0tso3RXRAxu1CmbsoMuUqdguAhRwOgTt2RNRAdApZYXPY31myrOxLDmGz6q3+yp5ImSHzrHxerqB/QMNpS97zYvI0CWW228maPPDIHtPZQRxPjd5I1zubE+HmtZHGZpSLuts9NhHGs9ZPUGsywxRtNgNdUsdLXtnlpXQ1EALDuXbBcLxxEyTD208dIZHQNzh7Xf3LG4r4kx3E2mNs8sMXQNNswXHVUuIRwh5xCUk7sJP990sdBTUkv3Y19dK7ldAdbLNFNLhjpKqGl51HLfVBg7sfxWSLC4wcr7nUbBdTi+GVVEyHB84d4kgAjqfksU0y+D6J5nNXVSSRw3vC8nRp7KzU0FdjOKTRmazGG1xsV1NPwHV0mFNwt1pmnzlxdbf2WNxFwni2EUBnw/EnysZe8biLgeylsuXoKbDncSswrGamOmijksXv2Om3svTcY4e4IlwN8FDQ0cz92OgkvJdea4RgNDxFHLPV4lK6uAP4ZOpPcKzQ0T8CqsmLSSUr/6uRp0clNHNPWYkZMOfJPGxxtci4CwcQjpsEqJKettWuAs3O4/h/JdDiVXh0DJMQoMYqOfp5Q0kLhcRnFdUmoa7Pn1J7lOLYdXNzSSE20F+yePmyWgjeXA/D3UFz6SpY3cscwEhw2IQSvpX02R7xYP0+qd0E+drGPijc/YnZRyzyVUgkmfcgaIZvxLG9rdyoLEWG1EFU+eojbkY0+ZrrhXaHHImPFJVNPK6PGpcVmQTVdMHOjqnjOLHVQwiLO3ntJZ1sLoOodR4nXxXxA3pwCYSdgFnYJiNRg+Munp5srbGMvtoQgxIMgazweIyywkXynoo4aqOnonmO2Z1tCN0oaUZwt2IOq8RrRMyW/NyHW6VJRcOy474czyOo5G+UnQArd4W4IgxGvw+qrqiKajq43kgGxafdc7juGQ4ZxXNhkDyWRyhgunBz4tj+dNJhmFS4EaLnNilIidmBH/FZk2JSYjyC+mjhLNIi34vYrq8AxPhR9O6CrwJs2MUoLJCyPSQd7bLjOJKunxLEnTYXSGkhH9WWWLXpaW2sOi4jjY+qpJ3mKI60xkuLd7Lr8FxCrx7Bo8DELpOQeZIXabdCvMYayemj/AkLX9XDqVLBjWMhkjIcRmaZDc26lLae3/e8OA0Apm4M2ibKRGJWOFi8qTDuD8Lfgvhqqjgnmebl3S68awfF6WnrWnFH1IO72uJcCe5C2MP4hxl2LvrsMqqqmpZG7Enlk+3RdKabH2hUuF8P0zMEwuCSkfK4GdtrBw73/VdlgOB0uDcMw0lBPmFTkfJIOpO9l5LjOPVdbxJT1FVVGrZTSh93/D3/wAF6jg+EU9EyXFq7FQ+jYRUQR3sIyfqlDTj4YwamgfLSUjopZBq9x69ysGu4J4kxJs0f33DBSSN8zWi4KbEeJmY7Tl+F43BRvjdqNxIel/4rKwzGeKqaoMeKTZKaUWkA2J7pTLnoOH207XMxGiY94cQC1/n02KCpwJlU7xDq0Br9s6bG8QjFVJHHzJZjNv2Co4jhmLxUZqJqWctHQbBcKbS07+dVyYVU1RnifYRvAvYLIxjC2UVU6Fkgc326K1QTQwlj4SY5RuCOitymqmq4DPFFkebZh39+ytjmqB8ENfHHIbxk2Psq1bA6CrljeNnLpKilYOInPfHZn5lWloJq2qc8UskjSLh4GhTkI8IjpcrTJGyQFt3NO7ff3Vx0Mf49LH+K2Buh62SxDCGYUKXEKGaRpkFnN6ghNFN4a7iwSGoj1Ye/wA05B4Mb/8AhEdE1rubTmwI1V6eubXV8cb2GRj4rAjoR1WdhtK+MTySeVgfq7oFpYXRiPFG56trvJnFhsVLFaB82GVFVTTnLeO1u46FdBg5w6lwgNkqM/ixmd81Xx+j59IJq2EOmYMkcrNvkVo8MYTG+mdIxkeSWAxzF2wf0sFnGE5t3BqD/wDR81U5JlkacpJ7bf4rSwjOcOheX3DxcKrhcrpMJbhshtJT/hH3t1WlBGIYWQM9LBYL1UyMCyRHVPoAhW6CSSSWWYmiTO2TpI0jdsmylHlKFyMhvc2StqjTEXQs2UpxsisU+ULTQC1IBGRdI7IAc7KLp6edjyDso5RdPDC/eyrDVDWujBCryN18qkiJDLJEWUbU5GlQEWVyRqgc3VZRBlKYtU2UpsnssqiEacR6qYM0SLEEeT2QmMdlYylCWK0Kr290BarTo1G+PVQV8re6AjoVZyaIHR2QVyOhQ5Spi1CWIIS1NlKlLUkEWUpiOhUuUJEXQREXQlqmLUGUoBylDlCkylNl/KgDLqmylSZXd02UKNI0kdmpsoUpKoKVtEWUJra2RfMwHQJ8pThuqLKVFRpI02UIG6fVMpLd03T6oGbsnSTN2QOndullKZALd07fUnSQEkmbsnQJGgRoEkN0kkBN2Tj8qZuyJu6B0/X6pkkWBom7IUTdkQ6du6FuydA7dk6ZuydQGnzFMnbuqtiTt2TJ27Ih+n0RN2UaNu6kgkkklJigaJCiWlk7d07dkKJuyIQcjbsgbsjbsgMHVSN3UaJuyCQbow5RZiiBsgMGyTd0yZuynoJm7KZnoVdinGye8OT1KZ2yTtkKgSEm6JCrIMGyduyFPmKqiuO6cGyDMU4N1hEhPUoXbpkJN1sJ26duyDMU+dSxLmCWYIMwRZiqGSQ5ilmKgcmyB26Ym6Z2yBnboXbJO2QqhJJJIGdumTu3Qt2QINSyhGkBqgYNTqTKUBGqAEk7tkyyBduo0ZPUpnbLQjTu3SyhJ26MzJkCd26ZGj5imTZQnRgWYJ0CbMEEiOPYqJu6ONyCRCXI76WQO9KAW7JZgmuUkBJ8xUeYJ0Du2SbsmSQEizBBmCWYIGJukkkiz5EmzBA7ZOiWSHMUsxTLYSSSSwtkkkkraHzFIGybp9E2YKh0kk2YJYdJJM3ZID30STZihzBUGmzBMXJswUD3KbMEKSoYmyRNkKSBJJJIwfUlEkN0kCSSRoEkmDkWYoGJ6lIDoEJcmBctNpOLaOapiinhHmjcFuxuzQRnrlQOF04FhZZCJ1TZihd6k6BswSylOkrbNhMYO6ZrWhGlcfE5LCHVDJPyzZVquZ7W+RxHyUVRXUopw2KdpkOtr6qNLTpHs3Cja3NLzFUxCWGowyN5m5cgdqbqgcRfG6OAzsk8w16olDxOaafG4aTPaM+RUK7h1mHYmaVgy0tQ3XTfurmNTsgxGjq36RvIGfoF1b6aCqgHNj5jbeU32KFOXxPAqRtLR0sEDC+Mghx1JWDx9wfQ1EtJPByqc2HOB0Fl3RhbLVt/ELcgtp1XE/afJaWgEzw6CSTI6x3CKz8PgfhVYyPDH+Nc9uzTq35rOBxKXi2lhqqd7pRMwuYDpZd/gmA4dgFJLJhvndI3PzHDUf8AN1zOAUk0WPVuN1lPI99yYcov5z0Qenzws5eZ8hjaAASF5jxji3EGIyOwvB6SOOOO1nuOr9F1mFvxOWhqmVmZtznbcdOy5zjLAcVldBiPjhG2Q8ssb0GmyzMJLyiWTEKKsL3PMVRGSC4dFoTzYljELPG4q2Wxv51r4lwq8wVU8OZjINufoZPkVw8fLe/JMC09iFFbp4lqMIpzhLGQVAAI5lu653O1ziQrEjKQegaqN4c+1mjTsFm1iUIa4nKUT45Cckeo7Isj8jngXsUUTy0iQbrTUyhLHwyNEoLbo6iEwSZLgi2hHVNOamply2MmQdAo4bSSsjlkMbXuALh091BYoonV0jaaTVnxfJXOXHhVWckAmhGxI1UFZRHDJcjJ2y5xdrwNVXDnSNPOkfn/ADBBt1uD0tVF4+lxEZB6o7XLQsGSONj8nMbJbsnginkk5bHuyn1W7KXkQ9LlBLT11bS28PVyRtHw30/RHDzsTxSFlTNrLIA59lC2mmkNmC/1UooHixmn5Q73WrR1srKfhWqq4IBzwWh7ZybvCm4UoMGxmlxLF+IWOc6I+oG1h/yQuOdC9lv6W+Qe5upoHVjI3ikMg5gs7L2XKZZpqcQYbhUVYDwzWvqKOQXcD0KoYdRieqfBHII5mNLwD1XccLcHHEeFZZJyYJZHExl2x1VOiwbBMTojSYgH0uKU0jBI8GwkF7H+9WJapxslHVV0oosv9Ke7IG+69coA+n+zeuwvEaQQVNFEYy0jruCt6n+yrhjBg3GIjKJYxzLk31Xm/HXEs2P1v3bhEM77Eh8bbESW2v26rviy2fsg4VhxqnxE4vhj3MqYbQTubpbrb+CsYngfDvD2EVNBj+IP8UARHG2TUgbEBZvBNezCoaWjxXiGspIpC8mEOsIz0HtdbGPYVwrhZkx6nxU1k0jTyo5ZOYy4CkwtvLoYcKnopJ5qrlzDZp0JCYYjV1sTIZsUmEbdtdgrtHjOAVWKVFVjGDiJk8fla3YP7rHqIGXzR6MebN+Sw06eqgpYsPp6+gqmGriFy7uFPh/EnEGJx+HZBEOUQbgb+y5upraoU8dJLl8gBDgLXWnHLXUWFxV0FXTFhHKkLDrbuQpIWKV8kjpiaSASvNy9p/wWaypkDI2vmeNb6JVdOXS1ElPU8wRi92bFUoJec0HmF/zXKYRu4aX4g+WSpqS0tNrgbqrPUGkq3ww1UrRGbaG10+FVgglZDIwZHu1IVfEaaq++nsbG6RvQgJiLkR8refIfxSTlPTsgMWZvOb0JCarqGTGPI23L0KNrJHMdGH5dLtB2J7XVVNFiDPDzYXBD/pAGZzxqR2UcIfhtQYHmzsh/RPh8fPlbTVWWPUjMBqENbU0hJkhn5kjJALkINySWarw+B5qoy14AI9+y1cInnpsG5dR5Mk2S+5suNil5lOIMv7N2e66XnVFRJDVvjJinABDdg9dMP4nPJ19O+MBjxa5G46q7HPc2uuaw+pcYsztDtbstikma71L6M4MRk1MwTA2UTZBbdEDfZceMrY8yIHsosyNrm28xWJiW0iSSSoFAR0KkDdUxZYrDITumREapspWmjt2TpJKegfKU4ZdCjDlQwhBOuynaxrBYKLME+dVE2YpO3Qg6J1FA7ZRSDVSu2QEXWRGlk9lJy9MyQGqyGDLBLKVKGNTFmi0Iba2TKQi6YtQRkXQFvspsibKVkVy2yjcOitOChLUEBagLVYLFGWoIU2UKTImyhGvNDlCWUqRNlCjKMiySMtSLVQFtE2UIsoTEWWQxBQFqlI0Q5Qq0jylMpcoTZSgjTZQpcpQ5QgDKE6LKEg1ZU2UpZSpC1NlKCFNlCNNlCBiLJW1sknylAyLr9EKfKEU6Hr9USSIQanDUg1EgSbp9EnbJ0CSSSQF5eqJu6ZO3dA6cGyFuyJu6BkaBGgNJJJFgkbd0Ld06INO3dRo0CRoETdlgGkmbsnQGjQJ+v1Wwbd0m7JkkWPMeYpk7d07dkQ6du6ZJFgYPUJ27oQbpd1JRIDZO1CnaqUkDkTdkAGtlKGmyA2mxUw2ULW2U7DogSfKU+UJ0EbtlGpHbIUAp8xUaSwJm7olG3ZOtr6kTohSQKeiDBCe4QN3Rt3VDp8xTJIFcISepSJ6lCXKB+n0QoswSyhQRJE9SiLU2Uqhkw3KdJUJNlCdIDVQJPtsUTtUgDZA2dyEl11IWKMjoUAlyAnqUTtkKBnbIHbInboFQkk+UJ0SwkdCgUjt0OYIpnboMxTlybMUZLMUySSISkBso0+YoqQG6J2yAGyfMEQ6bMEGYJZghYm7p8wQN2SzBATt0OYJsxTItJMxT5go8xR5ih6FmKHMEswQohJJE9SkgbMEyfMUOYK0HRIUlZiw2YJZgmzFLMU8gSbME2YplQWYJZghSQSZilmKHMExN1KCzFMmdsmduqH83ZNmKHMEsxQFm1SzFMkpQSd26Td0yrBIgLJZQibugZO7dEhbugZGgSQJPmKZJAk2gCbMUtSUG+SnuQhcmWmoI7p/L2Q5glmsVlRp7+wQE3RaAJYWhCimLI2Z3/RSnQj32UVSLsAdtdBWgr6aWpfRzN5bwL3ebArmuJPAcozxTOjkYb6fEuhfg7a2ujnbJk5fS9rrheNamQY2cNgnEGU3BOxOmiCP7zcDEJpJGiXbPtdUsexipwpzGU78srvqCuYx/FqqqkFJIR+EbhzRYrKbUV1VO2Dnucb3GZc23Wz8a46ylNBWwRSNeAWuI1B9l03DfG9a2ijgnnjjsLAv1BXnWI1FVaKOrfGTFtlVOTEZy0xwyWHyQe7w4oaq0lLJFKRqWxnUBZ32ixQ4hwq3EXxmN9ObtuuN4REE+GOmbO6Goj+IOsuwxaownGMAOBvxINfMbZjqR9VqJYY3DmLS4thDIIscbBUR+TkyfErNdxpW4EW4Vj9K2MhwfDVU+rHEf+6mw77McHwfD56muqpKgsiJa8GwBHyXmEtPi2OVcjKaOSo5F7NvcgKj2Ucd0FXRPgp45JZZI7tfcAX/xXOYticOLUVDIMRmcyI5J2MBuPe689pa04FiAFRQO8oIkZJou14Tx6pp3yUtJg7W00ruYJJdweqrC/wAZYbg0+HUUOGcTiV773jlluPqOi854jcwysMhj57GZDk2K0+O8P8XxE/wdKGO3lLdnjuuVq2Pa8NfuNyszKpvDvdy8jw50htYKzU4LjdBFzqvDZYmbXPdZ8wcyIPhnkLj8N1tHHMWqMNbRz1sszZBy+RINPndYWGfA18jHGxBAudOihkd5HOboQU0ktZBeGQFg91G8m1s260qxg1TPDWGRj7ODTbTRR4hKyoqc7ALkebKLC6gU+HyRQ1QNQwlhBBt0UbRwhjXXetHAqehrK+aGrlHKEWcA6arOqOXzi6KTMzppZT0cJmzhjmZmC9idT8kFiimENbMIbuaAWaDUhW6+qwmanZHAx0T49w7d31UlPh3inuZh8fhaxjS9/N0BHVSU3DNRI81eMsMcNr8wHQhBkxCUls1K8NHQkp5zcGOpex3UEdCnLY4H8gTCSnGzxuUxhwkuLxUSNv0WQ1C6kZJeqiMgIIBbuD3Wv4DFKJzRRuDvEm0bxvc91klmHRNL4pnOI7ruqCenxHhfmP5cdVBGXg99dCpImxHiTiLgqKkopHxTAt2IWZjHFvDWI/0uOnna98dnWisAeqvRT0WOcPwYo+E+OpGvZd2xPVcFLMwtyZteo6Bbhh0EnF/EbaD7uo+J6uWllZbkPtdo7XV3hbE8LwHDap9UJYa8ggF8dzf2XG3ys5gHmBFlqxY5LLNA+veyaDN+I0iy6ch6ScKp8Z+zZteYjLNI8vme313vZecV2DspKGKSaongMgJbmGl+vyXXYTx5JhGKVGG4fS83C6uxAOnIJ0JHsh4tocIpMPNfSVX3g6NwkcS7R19PSVKsedQR5wCXFaUbKFsJjlkeZibgjZKuqqGqewUNJyBl1GwBQ0GFVdW8mmp+dyPO6zrWCxMNtOiwOlraarfPUGKSnGce7FUwOs5dPPSyQxSRVB/Ea4dVP4+srp3UkL44uaeWL6ZT1ujbgsmGOEE8jJOY6zXN2uuYr8OubE7EI+Yxo5Vm37i9lTpML50L6gScmQO0N/L+igraWXD6yWlnFng3t7Fa1O2CbAnsguHvGdzSbgPCDPDK6CcsZC9z4yC0t2JWpWV9bh74a18PIqXtuRbSUKlhs87sUo4d2SDJKO7Oq0sbp5a0CHDpC+lpm6g9Cs+iKFVBN4aKuhY+TxALyANWnsnqw1kMMrzYkAK5BiMQoYoMkgngOjbbqzSYfBjMDhnyOZflscOvzXRVTB6aSqhqJpGgtDTywDrcd1dwvCYMRomSVUeTISCRufmspnOw2arpRJa7Cy9t1oYfUPjw6rhdJ5uVppuulMBoaeN7JGCQSZZCPLuumZTTswiGkYfMw3Kq8PcLM5EFXWtkc+UXc12gt0suh8OIWCNmew2uV10YRjNy5zkoQx8qM/vElalPSV3La9kejhe6rNp3yvDA1dXQsHJa0jYWXtymnNhyCrpzlkB+iUWIEaFy6xkUbh5mA/NZNXw7RyvuxuVcuTfFl+PP52qSKuHV6oYjhVVhzg64dGdrKtA6SV2VguuvHFh0kVQxw1cp2G6wKOSY1TY3/VdKacta23ULhsxdImyYLuTua0ImZWnzJPIK5tygI6FA7ZSu3UUgf0CqGbujI0UcWc+sWUoHQJMEBT5ipE1gs00B2yQOqRF0hZurlplMBZOTdQGrYX2YjL9NlGkiBJJATdk6jDtUYOqhKVjU8jdE7CLJ3m4WRC7dDlCIjVMgYtQkdCpMoT5W90EGUoHNVktTZWrIpPZqoiLK1I3KCq7xdakREWTKTKUBFllr0AgRpsoRkKSSSzQBJGmdstAU2UI8oTFqjQcpTI8pQ5QsqZJElp7KygUwv1TndPlKimTZQiyhLKUDICOhUmUIcuqCMhyWUoyLJiOhQAnylPlCWmyAUk+Up8oQOkB0CSQHQIGyhOnylIC6BkR3SyhLKEgk6du6ZJAQ2SSSQO3dP0+iFE3ZAeYJ0CIG6Am7p0zd0m7IHRodPqnF7aoGbupUDETdlIBom7IU4NlVEmDk6ZuyIIOSahBunWC0qSZuyJu62CTt3TJIHzFO3ZCitpZZ9QYPUImDVCB0CsRtGi0Cjb3UwaLIUTdkDZQjDUQFyitYIBbuk7dMkgYtQqR3pUaLAUCNJYQzdk6SSBHdRHdSlqHKtSI1JmKfKEzt1kEnzFMh1JWwySbQBLQhSPIOibshRN2SQ2UoQ3VHlCdA2UJi1Eoy7VSAzt0humTt3V94kA1RoUidEiQiepUTinLlGXIBduhJsk4oS5AJN07d0ySoNu6WYocxTE6IxRE9SgRZgoyboE7dMkhRYmju3SbsmSRBJIUSBJJswSzFFOkmzFLMUQswTk2QlySAk2YJZihLkDg6p8wUaNaoEmzFOhWVmbJJJAiCLk2YpE3TIpswSzBMXJlaQ+YpZimSWgWYJdPogzBOgMHqE5N0ySzYRPUpswTFyHMFQWYpi5CnduqwZJJJASbKEQanyhRszd07dk6SR5hJJm7J1QWUIUkkYOTomJ6lNmCbMUD5gmzFLMUytNkkkkow3kCNAtNQSSSSzEqcGyaWXlsLy6wCbMEnND25SEmBmYmZ8TontgMkU0REkfzTYPjn3pF4SqaY62LSRpUHE2K1GBwReDYMz9yfoudbxjBhjn1eI0sTKoizWj4v4IO0xCtnwiJtUyl5zb2IXAcZS0nFGL0sHhG0ktrkl1ydtlBiPFOMcS4XPPS4lTwsj/6qfWR81ws1bzLGSSXONnA7IwscV4KMDxLkRve5pF7u3VbFMInwmngrjKC2UXBA2T1GMzYiWHEpDPyxYEixsnrcTjraUUZJEbDcAjZcpbiWdVSCV2cTulPUlRQvfHIrdBh9RWPMcTGkjubKfFcGqsMpxPPlsTbQqtoqeubDdnPcCNiFd/FqKNtQyaV5fvboswYfLJA2pYLh5srsTpqGJr4Kk36tOoWRdZxNxHHSnCPGSOhd0c4myoU8mIYfUtqoankvGxut6mrMOlia9jYXSndsu6xMRw+GnqA7xHPEtrj8pTkNet4ofVwvjqmQ1ckmUtIZ5wQsOTHuIpoTTMmlDP3WgFVYXR0dW15FmjdXJMSqDK2eh9NrHXdbiWJxaOH8WS0NEKSrw4TSgECSTey5yYve9xedzdaUbPGVPLquZFOdidAs+qD4pjC8WISWqPRUk9VNaBocQL2JslNWZX8uR1nBBHLLC7PC7K7uulp8TwJuGF1Xw5zc4tJK2UF9+6FOXlL535pJCfmiMYDdwVC50ckrpIQWx38gPZPnN7BYUnM18qkbExw1kDFC6Us9TlIIqqWndVNhc+OM+YgbIBdlCkjpZzB46AtGTcHcKUUDzRCrEsZv8F9QoqeolpZWSB/kzjOOhC0NeOsFbS+Fr4JvFyAGN/puPZZ5mrIGmkkqKnJ1YXGy7/7lpYsVgkNNJLRztvFJf9kT/grPD2E1clRikPgfFwQB74ZPzey258nmvnGr2EM7lEcnwlSYnUTTVM9NJGGMEhAjHwjsqjtlydF5jaCd5vNkaN2W/wAVbnlfCG0jMwjtcEC+RZMeXOA85Aeq0Yzy2Acxzh3KA8NxOeip5MPqKiSJjzdr2jY9lC2D73xOOkgfG3OLB1t7IamaN0hF/J2Xq/2UcD4aMHkx3H4Yi6WS8LydYwP+bKsPL8c4arsEbDJO+OSKo0jc3v8A8lZobZxjmbdvZet8b8P4ljFcIKSemmpKc8yNsR1B0XEcQQUuJVgZhdPrA0RyaWuVoZMPEGLUxMEFRliAAtbWw9/qgxCup6yo58DZG5xdzXHRWo8HZDQT1VUMk2XQb6qvUQYdHQ0tVTzyCpf5JxfYoIImPk2C0cP8RSRTZHuBqG5AAeqz2yz0/wCNAyS4/dWzkigw7nxvLrgEgnqsNgmdBWyxRwWZMyOxIHUKGqp+Zhzqzxcsrg7IQ46XUc5AlbJC/K9h10U4w+tkpzXGMiF/Z2hKyNzhquoanCq04yIXSMjyNdK25I7LlaKfk54g6zS82VsTmOGNjNnkhw7hFUYDVwUsOI0tO6SKTtugnoKRgJrb5ZKc+aMDp3C1MGgjoKStn8XFP4uIkBh1A9wucZVVULpZPM1/osRoQrMdI+meJKWqjPcKShYdMyLEhIyPK21gL7LpOHm1v3c6sbTmWF0xGcHVi5yGLPKGN/aP2K6jBOJMOocDpcNrqGSbkSvLix2oB9rKxItca8M0sIpa2idkc/ySG19Dayqv4PnosLfUTfiVVPbnxk2sD1C7XA8QwTiGjmghaJWejzDUDorsmCzzytbI1r6YsDJBfRwA6rvEJxY/CGP1GIu+7q+hZHyoQ+J+1xtstqtw1jyZGMVDDaDwHE0lBGPwYoLxDqAel/oulMdtFvGeLM4sGnoWROM1rO2WjRGQttKwDsrBgZe9k4Y1vpW8804p47ZdEiy6BhspBssWUq11NFNTubIy6yaTDIInuyM3K3KkWhcs6kfd9l0xkoMODOinFS9nlCtSyi+VXr5mBVJKdmrnpM2kRSpLVQMda6IuRimw1n4krE0ctLJJkZDYd+6gYCyaOUZywN/UK+9rIR5xoqMldRSyZYI3X7qkCkyBh01VZ173spZQ7JcJ4pIy3XoFoRgl1sqdwc0XIQxvs7MArsQzRnPssqoCRpFw5Tx2y66ppomZvKLIASDZBD4SOOQyC9yVIpHbIUlYIOTlyDr9UXT6KKEHqjDtFC57QdVXFWG/ErwZtoskyqwC1+yzI58x3V2F+UZknFbT6WsVER0KYyG6bPrZRSvZLOmuExFlgHmCYuQ30shv7FAEt+iru2VqXLbXsq7gghLUrXRkaprWWSUZCawKNzdE2XuhIMpQZSpEJagBORZIiyZF9CST5SmRo3X6IUaBYCSRpun0QM3dMnbuiQC7dJ26JJACfy90SfKUEZHQoS1S5EJagjyhN5e6kTZSgjSUmUpZSgjSRp8pQB0+qduyLKUwagfKUspSyhOgFEBZJJA2UpgLok2UIDbsnTN2ToSSLQhCkgNO3dMkgJJJP1+qA+n1TJIm7IDSTZinQPmKfMEKSLEDSSSREjd0m7JkkBI0Ld0SEDYNESQ2RNaUDxhTN0Qxt0UmV3dAWhCljGiiBN0TXFqliyA2yRagBuizFUBlKZFmCFYCOyF26JC7dAyFEmdsgZJJJAkkkkAu3TJzumQSelqjRHZCtrB8xTJ3bpkQLd0bd0LdkbdlgE3ZJ2ydC86LYZ2yFMTqmLkDJwbJkkEmYoS9BmCYuUCL1GTZImyB2yBy5CndumVCTZghSTkgswTO3TIEU+YpibJJIzPkbMEyd26DMUaMnzFLMUyMxAswToE+YoUfME6HMUsxQo+YJZghSRBoEkkCSUmUpZSgZJJEgFJN1+iFAaAnqUiepQIvobMEswTJKofMUsxTJLQJJCkgJPmKHMEsxUCzFLMUOYJsxVB5imTZgmzFQEkmbsnVD5SnDUKNu6AkztkswQlyAswSzBCXJByyyfMU+ZCTZMNlqWhZilmKAGyRN0YPmCZJPmKBZgmT5imRskeYoEQajDdzBK47qJJWlGmzBLp9EGYKtDbsnQghONTogzsew8V1OwuGZ8fTusXizhrDp+H2VTqbPLGAQb7Lp6/FMNwqISYhO2Ltfqsr7+ZicB8NQCWmf8YP+CkjgZOFBPhstVRYbOHBuhYbn9Fy9DwtVV2F1WJMqmtdSXvDfUr3LC4hSUr5ql4ja++rjYLl6tvDdPR1dFBjEIdUm9wbLmjzAcLYo7A3Y61jRCOhOqjw3CJq2mdYHOBcWXpVLQ0tPwnU4UyuhmsCWkdLqxwZheGz8OOgENnsB66goQ8igfMZTBDzM/XLurAi50vKxWolys2udirlIaXCeI3vq4C/lSEWbuugxPAMGr8MlxWB5ifa4tv9UVgxYcylpWvrbT0Y1D4z6T3Kx3mNtQY4SSzpcrosOwPHqqgdWYPSzOYBd/Vrh2WLTxMZWBz4zDJGRdruhRm1ieWjocrDScxxFw8G9j7LNFU9093m67zGsJwbiDBYpmUww+si1JHpcDsf4Lmsa4dgwqkZPBM5z72JcNCsNRLAqXZnlylgnIYImPa13RztleGLmDDTRMgj0WZHS52Z5NG9wtQ206uqMUbHskjknDgqFbPUTymapDMx7BRNZCzSx+pVpvhZIHRvZZ3R19lRR5r/AIShZFJUTiNjwCepNlI+NgByO1UdnfEgkqKOSjcGSPab/lN08ELJ4Hzsf6Da1lLhlZS0FVzquiFQy1rHcfJX/wACOhlrYDAeaSTABrGsDMgkIeWMjZI54sAV0dBVU2G4Y8SPY6Z7fMx2xCyOGKLC8RxdtJi1UYYT6SDbz9FaxrDGUs81LSUr5eRJ5pM97BBmUFP4ycQcwtc8kgDb9FLVYTPDA+oa+N7YzY6rRwjDIIcQfS4pC8SmO8LmHqo/uziCs5rxTzx0Ydd0swsGhbYbvB2Lfdz24disFWx8vkjldJ+Fbou24ZxuvpK6pwcTRmgiDyHMGoNu/VcBScWVtNhhwWaClqmsaQ1xaDoiouKG/dAwmCkMNS/yc0mzD7nRW2ac/io5eKVjZpM7jM+ziNSLqg9x0t3su7qODcGoeHzjlbXmpqgDmYD16FcQ+JZmHWGvjvCWLYDHSVVU6OWGfzsdGb273UeK1dJLNG+hku1zfM23pK6DhXiulbRSYHxLJzKJ0eWN79eWOy5SshpaeungoZxPEJDypAPUFKJW6OGSaqhkjpJapkbg+VkbbksG6+h8NiwKfAIMSwChjdFLbc2DT+8vOfsbwuTDOMad+JVIgfPBIYo7+o22K6zjnhWoosXiODF1JT4ibObGbR8wdxdQc/xtVVuAVB+6sEhZDPYeLpzuOosNlytC7A4agTx1xa+SMslEmvn7rt+OqWXh7hmGioKGeWW3MkqejT10+q8qqqKaONjpKuPJK3OHN1stCXEcLxSme01Ut/EO8rb3uFUmoYPBvDJGxTRG9j1RivrJbR1dSZ2QG8ZHwqnibmzvFUzXSzkF9uMiqwqTxNRepDbAW2VfD5nys5EjrtKzWy3YWd0cRez0GykwNfP4p8r+WLg2JCnosTkYBRP1Z1aqTZGBjmB+rxY6dVAwPilD2Ose6g2Y/Dz1ETo6d/4Ts5BNtF0OLcTQUT4psOjkDD5JIul1xUs81LNFPzRJ5vSN1crmyVLDNna6/UFAeLHxVQaiOAMLwC5jdgEGGU0NRLM6SQta5tmEdSrOCzxMD/EvjMxjLGl2wCr4RVwUtU+CrYDSyfhuvuPcLMotnDZo4g3mRjUDOToFt0NBS0OOYaXMysluyQk3abhZMAy4gcPknzRH9m5x9Q6LWqMOqp6Wlw6E5TE/OJ+oHZYFuCF3CuN1FDNTvmpcR/Eg5ZN22K9FwCtnrcOikqmODze5PVcTw1LX4viNDHWwXq8HDy4uPknjPb3XU1PjaCsqqGjeWtsKmEA7jqxd8Bt8iPxXiyPxMvLB9lI7dKOVlTTsqoR5ZBe3ZMuhMhcLhLL3RJAIzMhDFK1uiYNUjQgRizNykLPNGynuWd1qhRTRhzVYkVGVDALKCtq2CIBp3TTQvjuqk4a6LKQCumDCB1TTekyXPZEC9hu0W7eyoVTXx3ezcK1DJUy4Xnd+0vZdqZXBKC38apDn9blQ0r71QZlWJFBPLln2d191rYTNI6V2bporQ1qhrwPIPmsaeR4n5d7LdvfylQTYM2SobUZrZei5Q0r0YLiPKrw0BaibCyIZQLIXkNOqysIJmgElVHP8ymqpgLqqDdbRKJe6cPB2CpulcPSlTz+YgpxLXEnO0Q5imcdEpbVZ/WSqD3Fjj81cm1cqs7LhdIYlPBJYggq4yqy+orHjcQQ1WRJ3CswzEtYTNeE4kWcyYtIBKtMfdq5TDpErTTmRZfdV2SW+JGJL9VzptIkhLkBKoGRyHNdM952UQeQ7VShIkiBPVCkwBAumyI0RGtllYREXUZFlZLQoyOhWVQkXTZSpSxBlKAE2UIsoSyhQRpJ3bpk9VJM7ZCjWQJ3TJ+v1RIH6fVMm6fRE30oCSTN2ToEmyhOiDUETgfypZFKmsSUEOUpW0upuWllKCBPlKlyt7ocoQBld3SyKTLolkQBkcllCPKE+XMgiylOBZHyz3SLCEDZShyhSW0SQR5QllCktomyhAKSSSA0kkkDt2RN3TJIDSQu3RICbsibugbujbuhJebsn+iSSlBKRu6AC6JUGnbuhbsibugcCyNiBo1U0YQEGqWPcoALI2DZQWmgWUwboo4xmsp9AFRWc3KU4aiIuhWAYOiSjzBEHIDzFLMUOYJ0DO2TZiiUbtkDpnbJk7tkDpJJswWwswToMwSzBCjqPME2YpZisBy5MTdM5wQl6Ak2YIcwTqyJVI3dRN2R5gtA0EhTF3VA4lSwi5AkmzBU9CzBK/wCVCkoHzFAXJFyF26BE6pO3TJKhJJIev1U9AnbpN3TJKh8xUZPUp3bplAkibIUlQiUCd26HKESTpJsoTolEkkkjRJJJIHzFMnylEjIW7qQCyANRjZD1JIjoUkR2CCNJO7dA7dEIm6ZJRu2QEXJkzt0yBJJswTZitQF1+qfME2YpZigWYpZimSVBZglmCdApASSbMEswVDpJJIDSQgp8wQOTdPmCZJGD5glmCZJAktQUkkD5ihzBJ2yFWIbGgSRpQSSawT29wozZIhsmsE+ysLAS1EBZJPlKitdJNmKWYq0HQFyJ2yB26QyQNkUb/OLqIlJpVVwf2yWmZRRsN3Z9vooeCcGxupgySV3hGMFwWm9wuk4qwiLF30z5n2Idva61sMpKfDabkwMsNERxPHGF4lTMjjZjE9Vfz5QbHormDUuDfdlJHidFFJM/yFpC1qljH1tQ57AfJ1CjwfDIZakuleHcsXFu6zMCpgfC+EPxettB+GNYxfYrUo6uDDTUQmARNZq3KdHBXMGwyWjnmnmfcyrO4ppBVVNDSyFwZKbOymxspTTlaSnouIeIvverbDE2CQgxgWLl0nG2L4dR4d4Knw+KV9SLCzbW2XNYngj8D4iZS4eC9j/Tc6qCSOuo+LoY8WkMcQIMfM2tZTiKGA8Y4rw0JsNpKW8TySWuHp+Siwaig4ixeb7yktLJ57EWBHsp6/Cn4lxXLBQ1pgheLhw62HZWqKr+5cbEeM0xkhH7Ooc0WB+ilJip4tg9RRTCljxHm0uYMAB1j+a6+Pg2lxHBZcEqJpJnRx8yJzt2v9lwONU7/viprYJiKWV2eJ7dnLopON3tLX0D5aatyhhc6MuDgO6Nw56LhafGQcL5FNRVNN6pZNyVz+MYdW4HN4CoqYpLi94zcLvaHCZ3VL8RxmSGZj7vzB2xXn+OGOTFZzBI57M2l+iMqsdsp7hQue6Q+X9E+e3lUkUL5b8sXsLlZaxhC0PaPOjhhnq5hBTx8x52F7JQxmatjgJ0ebFakuHfdmIsfSVzTpfXootKdZheI4fFHNV0/LbJtrdRUsM80loITI7qG72VmufX1VYyjq61rg+QWcToFeosMmwvHaQTYlDBFIchqA7yAHugr4HQYdiWKeExSR0ZeLgjfP0C3q2ixbheSSNhD2zx5Q4NuHDoFd474Nw7DcJp8bw3GIZnggFjHi8nuCFl1WGcQ4PQU1dX1TJYXjRjXXICrIuAqhkHF2HzVFXy2hx9eoOmxXoGLuhxXGKnB/vEwszWMA2cFwlNT4RjGIvEhEUnKD43A2IITYVFimKYrUVz8ZtWU3kbIRcmysIqY3hdHR4rJHg/MDYiWFzhpfsoaZ9RUMlop8oI84c0bFWxjmIYjXSYfXvAu6/PaNj3QYVR4jX1VZBhT4nuYz8Rr9C4eyuTbJMsmf8AHqnuadD5kp6Wzm8t4LHi7T3UVRDJDM+CZlsh1CfmgZWSNJbfp0WCEcsZicY5BY9irGHYO/EubyKoQvjAI90E3hJHX84U2Hy01LK41APIkYWEtKC4MXxU+HrTXEVFH5GOaLEWXrnD/EeOcfcMhlc+Ln0coInAsZSO/YryB1LSR07aqlm5jr2Md76KxhlXW4Zkr8OxR0MpNnMDtLIPZKDB4a0umqK3Ep3MuHc93k06LySrwqmkNbHT1f4dPK8ZXBdFhf2lcS4TSuqxVeKiLheKoj2+RGy5PHOIK7G562ehpOTTTy8x+Uai/T+C0I6KaBmB1DYvwqi/mc7Zw7BZU1N/RzPzmadLoOZ+Fy1BlCB43Oa8Oy3VqLWVrPqVWa3VTjM0hnfdUWaWJ80wjYLkpq2nqKXKZG2DzYK+4CGSN8ADPLqq2Izula5j/VpbRcxnSayhiv0lSyKR1I7QPGUeyz8rr+6unD6iMxzxjmh4vdqCzy2UVQaeP8RrwH+bWxVj7krZmQz04bL4s3DQdQexULZKqRgnmp7FhIsfiC6XgGunqpaike7SMB7TubdlnkM+tpoaeSOCcSS2isQ3Uwn/ACSixivpHx4fVG8V7Cb8wU1ZUSDHX89kcMmYsJvpIzoqUrX1RkpKa8tnXsBsFCmjBjddgOIUtXO8OdA/OQBYlnVd1WcR4bXTUeOw1uSjfHynZdZIj0JXFNE2IkGSnJhiDI5nZLkE9v8AnooqfBvu8VWF4jRSHlG4lbqB2v8A89V1xlzy8nreH0UtJUaYm+oZKLtZ7FaZFl5PhfF+KTPw6YtlkZhkgZJJl2ivaxXp7cYw2q/EZVMZ7ONl1hVhO3ZK1wHtILTsQbpwboDA7Iw1C1HmKSHQkokztlmGkEjQ4WIWZNTOEhI2Wu8KB7W9WroyyjS5/UNUUcPLbky6K46IDSyEtC1aK3KYQW5BYqtDE2CRze+q0S3qqs7MpLxuukSzMJoQScys87Jpdc/PidRG4sYAPkqor6syE8zdXglup5zXelDKLtWCMUmib6roosSllBzOsr4UnNanZdygPkTNnLvNe6Lyu9SVRZENc3RRiK3m7KRrcgypvNdES5nEJibJhoEDzZZiFsD8rlVkYdVO59kBdmBzLrDEqpjAPlKcZx8k/ZGNQAtyyKHztv1CtMJaFVhuHkdFZa7RYmHWEtzZMJHBMZBlUbni/qXOlTRvcet0ea3qVaF7eblU8pu26lAJnB403Q32co84J1SzAvDWGy1QsNdoFI3ZRgZQlntusTCwkDUsvzVZ1S5hy5lJFOHEjN1U8OWkztUBClGo1TZSs0IsoSyhShqcNCyK5b0aonNN1cdHdRuZZZFUsKHKVZc24yqLluRpEkpS1Bk1sshnboW7KQjoUOUqKWUpZSmRoGbsnSSQIHqE+YpkkBZtU7d0waibsgdMRdOkgbKE2Uo8pTEdCgjyhOiyhLKFAGUIwLJ0TdlQsoQkaqTKEnboI8nsmyhSJIIspSylHlKWUoI8nshIF1Pl0UbmoASSI6FJAkSFO3dFnzOjQJIgweoRN2QDcom7ovuEHJ27JkSIdu6duyFE3ZAYFkbEAFkbN0EsYRpo1KGoEGkqVkZ08qULQVaa0dlATWonbIen0RO3WQBN0BcndsgdsgZJJJAQckmbulmCAsxTJJLYFO7dMksBIETtkDtkDE3TE9SkT1KEuQEXIc2qYmybMVsEmzBCkgQPUIm7IUQJWAWYJ8/umzFBmKCQlNmKC5TLUA8xQPKYlyYm6sAcwT50KSB8wTE9SnzFCTZA6SBJSw7t0yEm6ZUJJNmCWYKWHJsmzFJ2yZVgkCIuQoEkR0KJ2yZ26KZNlCdAhMjSQI0QkkkkCSSTg2QEHIgboMwSzBAYPUJJsxSzFAJchSJ6lDmKBO3TFyRchQJJJCXLYTt0yd26jQGmzBFmKWYqBkkCJuyodJNmCWYIBSSSQGknzFDlCMHSTA3Tq22SLQhCkoCzBOmyhOrTJJWKPIllsogMru6fKU6SNmylLKU6S0E1vuiLfdOkN1mmbNlCf6pJ8vutLZZSllKJJFXsxTg3TZSmQEdEzUOZIG6JJ0OUp8wTZiiUCQBws9twnvplaNExbcp2suhSvJRU8z872XJRAUGEsdVyERtA1JKinxihoqjw9Q/K5c9xDiNM+cVFM8TNI5dgf8OqytNCp+0bAaZ+Vzpcv5g26oVX2k4JVztNBBJUOjtZuXf/ACXGT4nNQyy0GIUAInF2kdlizufhGfkkC5uLG6Tkrrcc+0GnxN7OVQmlfA7OHFwuqOP8U/zjoaceCOenN8wIXN8PyQ1GJk1bGuYBn1GxR43i8ENY6PDoAxvxC1hdZ5jteC8TocSzU7MNaKyIEh7iSXKHGq/BMXn8NiNRUQPidpaPRpXKYLjGN4NUivbGGseCDcbhauLYhhGIvdOyGQOkALpbaAqXyWXZUnD0UVKJayvjxGGL9k0tsG/osHiRr6OqbimFP/ALPxW5TZtv71DhnFkEMFNDS0hlqA7lSA7exXV8S8T0uGYE2lxBkAqagWytH4YUkiXleI8U1FdW56TLCCLHKNCe6rQ08MD5JK7Qy6t+S6gswjD8ShqK2Skkoqhu9tnlZsuIYXT4o8YfRNqmuNwD0VRl/cTpoWThz8jxcmHUALblwHA5stMxz4ZRHfO06OKgkxjD4pH1AvSVFrBgGixKzEIKyKz53NczYM2Ky1CHEDbyGkLXRmxkt6lRzBwGbopRPNIwQyzEs91HJEI3gCQOBUVI0xucx87yGBwvY7jsuxx3BuE8Nbh1dhOImr5ljNA4ki3uuNBZla1sQFjfTqtSapwbwrf6IeaG2uNEFzEcSwuadrJGcto9IYbABas+JQYzR0mHYtXRNp47RxOYfPf3WAyg8PgrMRkjs/mXA6EKtSRQTScyok5TDuGhEldx3B6jhqvp6iGTmxPN2uJ1v2/io6fG4Iqw11VS+Z4s7L1Sc2Eh/Pnnmp42kxi/pKritpZ8NFI+EXHpkG4RWrQVdJLVCfBsNkkeATJG8aH3WrgEMbZBjuGPy1sUp5tMdQ5nyUXBeNsjiOESMAljbnhPce6p8OYoyDEK3Qtqpy8NY7RhK0KuMRwVWKVM9NVBzZLykEWLT2WXK2zrLYrosEgEsGKQyCvY43yHQg7LCzErINsT5qhsIYS072FzZXaqKDI2noWukmBsNNPqp+G8XiwbE4a2aMyB55dm7i/VbNXhOE4pxOaduIeDZU+dzmG4Bsg5ybDJ6J4ZXvMDiLsA1uFBFVui0LA75qxxLSfd2Ly0bcSNc2PQTXuLLMa66DqKriSDEY2Unh20lxaSUdQqcb/ux8lJh1VG9k4uXZbkLIY7PIGN37LawjEIMMlMwp4qnmtLCC/0lbGNK97XbxG/WLYqIG5RzZDK+RmmdxOXshZma67VgFawuVcw6EVVVEBq3XN8lCHREfiX+hsr1FLSxTSMpMxYW7uNygknL5K80lEL2U+MR3dEy3mY3VR8N4E+doxXFJzHRBxYch1079kqeNgfUyPbLNCJPU0XBHdSRlua+Jwe5qvUUrJRzJat0OTcNGgU1fHT8ps9JSl9NILhzgqdI8RslGX9orQvuilopw+Z4kBBs4HRwV/Da19PPT4pT0uRsYIny9QsWKokfTRwSPuIb2+qvYZNI3D5+WLtjcTe+6xMDUlxeg5z4cShvG9xfDUNG5KqcOYh4TEZZ/gkuzMdgk4Mm5EFKRkeSHAjZWKHD6vDGTRwx82N5Bs06Ea7rI2sMq6p1Q6agm5zAWGSIDWQBbXEePTU1a6to489NicIOW2rT1v9VzWGl9HiQxXB3tzQaupybXHUL03BOH8IxWijraiijzEk2O4O/wDiuo5zgfwrKjFjIC7DpY4xKwDQE91v1fDtG3C4IaGMztidzIbnXIT6PkrGF4U/CsVkqqT9lKDHIzq6236f4rWFNyJXRQs/AJuP3fZbiGJhQoIPBfh0oLYZneeEn9kVeyhFy3hxsbNO47p8oW0owNlIDdRa3RZQijzXTl1io7gp3C/mVgE511G8XRNNhlSIBUEBGijI6KcjVCRdClYhAQpnhCRddIGJW0oDi7llZ74wxudm66WWNrm2Komkj6tXbXk5ZQwDKHepytUTmuzM67hW58PhILmgD5LPLOTJksbrvGTCene5suRyvt01WXHvnV6KRxaLrMwsSs5wUWUKqJW3y5lKwg/Jc6WxPNgoHOUkxdl0KgDtdVIhTP1YS3dQOcdlYOh9lE+G4OXRdoZlAJLKVuuqDwcxbnDwQlCXZshBuESkjGnMcoVgNOVKJnk1TlwvlWJUD3aKBrznylTSFVpHhj1YJWWtu8PCun9kfkqVPKMozKy93kusz6rDLqJnQyWOitxxyXEhYQCEDckkreY0E36rVytLQ3oNklPerB1/UopZR6Qiqs0TDkVTmXF0iG7Qyu/E9SUcha71KGaQ5t1E2Uh267cXO27BO1zdSrAkYRdY9PIXX/eKuRyuaVwy1ukZLrXZnZVOI8ypMmZmBV6EZmB/QrjlDcSNkYulJT3F2qZrbFSW0XOVZMkRDtkBjWlKy+llXdHqoKvL9kBh02V1kVz6VKaYFBklh6hBkWlNS5BcBVnxFRpWyJi1TGM7oTGpIjtrdOjylDlCimST5QllCBkYQ5QnSAaSa+l06LBJ+v1TI0JD1+qZJJEJGgRZggLMU52TN3ToBSSS6/VAzdkYahA6BO3ZYDkaKNw0Urt0DxotiuRZJEWoUWrJJJJEO3dOmbunQE3ZOhbuiQgTdkTd0Dd07dkUadu6FuyJu6IkbspI91G3ZTMQSs6KZnRRNCsNbogOG4Vtvpaq8TbKxmKAS5M7dMmdssBnoE7t0yAEkaaw7IHSSSQJJJJAkkkkAu9KB26N26A7oGLkBOic7oS5bAlyFO7dMi3RJ8xTJIWIOR5gogeoTg2RDlwSzBCSlf2KwHsUsxQ5gmuVsHmKAnqUkkCSTZglmClh0CSSoSF26JC7dAN9bJEp0IborLBJswRoVGyQI0OUoCSQI0DZghSSRmjO2SdsnTO2RZCknduk7dGSzFLMUySBIweoQt3RIEkkkD1CLR8xTJJtAEX0MXJk5cgWoZJJJM7ZULKEwanBuk7ZZDO3TI0CsAEkklQWYJZgllCFAkkkkCT6kpZSnyhAsoRZSnyhPY9kCST5SllKMBLU4anyhOrTZJAdEbgEnAKpRCyZ2yQPRImyJYUkkkaJJPqSiQNb3KXVOkjPqdyJC5EhkSSazkYFkX0XS1w6pEFWC1CWqQqqWptPZWHR6bKItt0S2SQ5SmOibMFLQyJhyuBTZgmRXK8Y0T3SzVWS7ctwvOH4dVSVrGPoZMhcLlj7hey4rAyrpTG9ZxwKKHlVVKSHMt5TqP0Rp5xiuEmorYaeZlUYiAGXOwQcUcPw8OiERVRmZLfyu3G3+a7zirDBiEFNBHOYJg7O0gXXN1HA9a6jfPX18kpYLhc5gcZBAwuFm+Z+w7qN0EZkc2QSRPZ16Lr24ngeF0QjeBK9nQt1P1WdBjuETxTUTMOkndUdDusUOfnxCrmHhp5+YBtolE+o5PJa/wAnUKSSjNLU2qonROOwPZQvqvDzEclzx7bLcNy0cLxSTBpnSMhjkZILSB2/tZRYtWMxGmzzzukt6QegVGV93XQct7xpsqKYYMwcbmx6lWMshY6dl7MCBzWtNlGZZGmwdosC3BE2eEyT6s99woZxDGBy5AW+wUbpH5dDZX2U0LcLE0zbyP1VpIZ2W4v0QEsBHsrwgp5MPM3PyyxnRlt0EFXS5AKmibKPfS6ipYooM3jaOnnmjH7TTZPHh1VWNE8L42g+900OL1FG2WHDjyYZd2b6fNNheIRUTnMroBPE69xdBZxapxJk8TZHcxsbdXN1CoCdrJC8sB+YXS4jxdh02FDC6XDSxh3duVzMFM6ZkknLkLIzq63RBbpayljeHPNmPBD1EaWnnZLPSSBrItgeqLDThAle6uJDcpyG3VHRVs9A+Q0QZLHJu1zbghBrUGFPxeqp5OFYZedTgPnD9A2/VbOPYBS4FPR109RHJXvdd8LAbOZouZwDH8Q4dxps8DHDmyBksZFswPT+K7D7V8Iqp6mlxWhY50T4QSANR80YcTj1dDiOIOqoY3MuACD7LOBtdE+Pl6SepMxskh/BjzW31tYI1Dfg4ZmFJR4rO4SUspBcR8AXRw/Z1jeIUszuH5sOmo2OzsYZrySaXsCsfAa+lpKUUOITHSYSRNOscoJ1BP6L0h+P8K8EY7S1baUl88Xmgp9R87Iy8h4gjoi5op6WSnlhuyeN+4It/wAVmkNa0NAXofGGG0PFdfWcR4Ti9PC2UAmnkjyyEgf3lecFznDVGsZaOCy4bT1/MxWDm0xFnaXIUFZLS+NmOGscyF5uLqAbFO3RbUIY47bp8pJs3dWI5KR0JifEea7Z6J12luZo07LAqF+U2IVzDuXHTzzdQbWVWf8AElzxsI01BVmjbI4GFjfL1ug1I2xfd00Lnvjc/wA+Rp0Ko0k0lL56WdwDxYtdqLKYQTunaS+MveQxl3aXPdde/wCy/wANRST12JCKW/8AVuzRgnv2+a58xx9RVc/lwRstkFgAdL9UUYpYMNkMzPxy7a9ixG6g5LXyMkfMA45ixvpCgfTvqYBUMhLmAkEjcfNdIyCZBzYjOZA3LqSe3Ra+HRxYhQ0lOyp/GkPLLdrnoqzQyWKnnLBliAj5IFiWdVBIJ8LrKaopJBIOaJ4mDQgjoVjIdPDhMeG1zqSrpXR1MdvONf4KTB6Tk4kyknq7NqZAI5G7A+6DEuJZsRldjEdJZ0kQY6Nx2I+iiw+Wapo4p6SQC8oEwOhjIO6yHq8Nr4cSrzHSlrqdt5AwXDve66jg+qxnB8EGKTPc+nrXFn4xN4SNRf2Kn4VpJ8fxutj8WY2Tw5J2k3LwdyF3X3bFiuGy4TSVcMbDHy7OOoIsqJYYfFUNFXQMy528z6lW3A5fMrdDAxuE0bGD9nCGH5hRzQ6LvYo5QhyKflp8jlthXyOTeZWMjU3LHdClZ2yR9KORtigIQxDeySEusUHOZfLmWgTvUk3ZCXjugzjoURI4aKFw6KW9wgeNFlUV1E9gupHbITqtxIj5LHizkD6SFovkBI7qeM2Nk0o0XXGWHNVkEkMpcPSpaeqaY8hU2LMsCs2I2C9EfmcUz5fxc11ZhlNr5wqDjmKG7gVeBbYJJj0UQF1TiqnsYA9xKmZVi4ddZnFbWMhKEtcjimjcL3UzS1yNK0YeHAO2VrlQb21Tlo+EIZWXZuQQsqc5QFXcBm0UkUckgyuKB0b4nnPsi0CSN7m+VVpIzbVq0onNc3yqtWROarEszCrGHgWCv3/BChiic5misthfkACkrCjK0k5hutCkkvEGvd5huo/DHNYhTeGc0XapMtCmjbMwi9lluhMT7PC1A+w1VKqLXG4VhMmdUxg+cN1UUbDm8wVmc5W3UUXnvZuy6xLmmp7td7Kd7nB6CEtaQ091Yl5eQuWJIDGWNOaQ2C2KSRpiFtlgscZGlamH+RuXMSCVjPHydIlptUoOirh11M1+i8sw6QLKCFDIzzaKyHaIXM10WaEYY1EgLtUg7VGhyRtcxUpIdVbLkLg0hBnSR66KJw6K7JHa6qvYboIMoTosoSyhQ9ApJZPZPlKyqOzk+Uo8pSylALdk6LK3umylAySSSA0ztkg1E7dAyBJGi+h27pkkkQkSFEhBJ8xTJ8xQMhLtESAuQRndAjdugRYJJJO3dCJMiTN2ToSdu6JM3ZOiQSkb6VGjHVFk7fUFK3dQjdSsUhEjd1IxRt3Usaomj9QVluyhjbYBTRg3QTM3CkJshA0Qk3QPmCEuTZihJssBE2SbsgLkbdkDp3bpkkCTu3TJi5bDO3SzFBmKTd1gSE2QpJ3boBLlGd0SB2yBO2QoyNUxGi2IzumSSRZCkkks+qCSQpLRZ8wTk2QpnbIETZLMEzt0yB8xT5ghSQL6JvN2TpKUGzBOgRZQr6BspS6fVEmyhAKSItTO3QMkknylSqA5QllKLKUspVAJJyNUspQRpI0kZiaAmyhOkgZ2ydJM7ZEA7dMndumRqBJJJ8pRkSSSSBnbKNEhRfUkkkltDO2SbsnTZQshE2S0ITptAFQsoSyhOkgjLUKkdumVDZU6SfKUgMnylEB0CSz6MkkBdPlCdaQkk9illK02ZPlKZPmKlBkk+YpZiqGT5imSQJJCiRmYozdkTd0waiAQEGogOycMRhqIANT5W91IIyn5bkaRZPZFlCkDHIuWe6IjEYRZFNyk+QdkRcylDlCWYpeXssNgfsoH7qd3pUTtSqID2QqR7VG8WWmZCXdkOYJ0BPUpCQTzdCkT1KHMUbVKylE+IUz3+lh1KyuL553NMFM8hwaAbHcLezNvcqrV0NNM51RNl9ySsjz+bA8KpYqd05jD5DciY6hURxBgeESyCgpQXEW5jRcLSxnB5+IsTFJh0mamiF3SO2C5/EcGw2nnmZSzyF8Yv/muIyp6yoxeuEtQ+1zYney23wsoaJtJI1ssTzbmEWKrx4bVYLSsxWaPll5ssusxeqxB4jkyhnRoC2tgxAU8dUYqV5LRuSlTkZ2szgNtqqkmkjfknBjLDeO7mncKNJJHR2czcXuCqhNkZjeGCToVCSSVkSOLC2zLqQzVEkXKMhLRtdQtHZSh9jlC1AAkuFilHDM70MuETJYIpBzoy5vWynfIzNenach2BWRC6lkjFzb6I4Yo2ubNNZzDoR2WjTwwuhzl7hpq0qtljY92VnVBXrBCyTLA8Edh0RQYlPT08tIzKWS7gjYq5PKH0/JZStLujwNVBTeFjgkhqqLM950f1agoN3WmypiioYSyoizMdnc0aGwVN7BJpAzXsrFNhE1TC98kHoNrX1/RBu8WY5w9j2H0mIUDXxV8JDZQW2uOh39l2cfHOF1HAN62pimruUad0APnadtV5dJS0MTmE3IJsRdWa/An4c2Orzg0tQ28bh0PZEmGZPJzZiVrcK4S/F66Wn8WaaMtIM2UOAPS4WRKLHMFrcLNbUVFRT/eMtHI9t2lmx+aK6ulFRw3LJg2IxRVEMerZpY9Ceyg4gw+XCmU+JSPb4Uedobu0nYAqjhGJz1ta7hzHJefTPu1rnDzA+xVGve+hfLQ1FUamKnksIHG4cOl1WFCTFhV54auXmMebtd1B+aqyiIfs2/P3U1bNh1QweGojA4db3uo/GHlcrlxlo20UXEgG5fdAWXaehG4QvLicovc7AIbhzgbEO2I6rbSRvqVprs7MrWkOGocVTz+a1iPmrVJHz5hG8kC10Ac2SWQGR1yBZGJZGX5brX3Txxxu5scwO9gQVNBRTvpnPgbfl7jusDc4D4TZxZU10U9VNAKeLOHs2udgV0nAr6iKpxnh2VklU8MeyK5uHZOirfZHR4tLNiddSMjfTsjEdVEdzodv4qQ0kuGV9NiODRcuspDeXMdJQnFhLVUGK8H4FzDDA52IzWizN88ZPQ/4Kg/hSlpaurwuWWcTy04qIhn6/GP1Xe1EUPG3hq3EoPCR4ceY9l/UTre6854zrMUrMZkxUnlxfs4XN6sCsY21Eho8AopGwQuxgZpR6bWLT230UMuBUdW4sjqjDMQeUS64uOh7L1PhPh/B6uiwHHXUIa+V13RkeUm1rrfZwLg0NXiNLLTCWGcc+BpABjPUArM61eMRcP11Fhba57mVFM8XzsOx6ghKjo3YdLSh0fOgfapc5h1DPkvTeGcMwio4edhFQbw1c/KkI/qpA/QE9Liy2jwNhDJ301PAWtjisxw3aPYqcByv2d8P1LWT8TUVNJV2kLIix1rM7OCyq3i/inhnGJI48PkhyzGUxvbmNj09l2MHD1Vwd4HGMCdKWSycushe67TfY+2q3qrDDidU8zwASz5HvcNOWB7rVMH4axlmLinqYGGNlXERI07xyDcfxW3NBrayrYLgUGF1tUYz5JSJQy2jX2sT9VqSgFxK02yJYrHZR5FoTMabqoWEOVsRGL2UJY5Xmx3CEwttq1IRmy6BVXPstWWnYT6VWkw9kgI1HyW4ZZzpWuFlXlAzXarTsKdC4vY9x9io5ocjA49V0hIUZKgRm10zKgu9LlXqh+IR1UccjmrrTm2IpCRlc5SFZUczwQbq1HUXNnLOWLcSlc3VA5tkb3AhRuOixRZAXKmlb+HcbtVZj7PF1azXYqTDn8V55eMouqktGWZcjLXFz81vTwteSVn1EcnRq7683OYY+zrorBHJE/ObtsgNx5XNsvREw58QvB7ILHspmx5zbqi8JMXCzNynKDiBoew3V2llN7lM6mLG+dqZgyhc5lqIX4zexUuh9SpRzAWzbK6CDFzAsS1Bmjz6bIpGZgq7ah2ayvNIc0WWaVUEYadGogzmaPajLXA5cqJjbFZtoLYA3Zqlax1tVMG2CYG6xOS0jDNfMiLLjL0Rp2rPJaZs8MgaQ1Z7oajzEvFuy6CSNrhlKqmlbm1C645MZQ5/K+R+QalXG0b4IvTq/UrUFHE12cMARSxh0TtNguniMUyOS9+zUXLdazkcctnltlIGtLfLutWUhDQ1vl6KejmF0HJcQjgpzHsFM/RcWs0aKVhsCoKcnKL7qdrl5ZdYGCiLlGCkTdYlYCSSUwckd03X6KNCzBMXIEW4RkDioHi6slqEsF0aUyxDlCsmLXZDy/ZQQZQnR5bJ8pRUYFkraI8iLLosiJNlClylPkHZBERdICylyDskY/ZBFbRJS5QhyaII8pTIiOhQFqB0kCNAk7dkyduyB0k2YpZigZRuNypEDmAhYAISLKXJ7JnbLYjSR6kpO3QMEikEikBI02UJwOgQEGp07d04FkCUjBZC3ZE0WCCRimjFyo2tsp4m2sgma3QKeNnRRM2ViPYrAfKEDt1Io5DqgYuQE9SnduhdsthsxRMQt3ThywJCepQ5ihLk2YoHLkzt0yRNkCGySSSCRu6Tt0yS2BduhylGRdNlKAQNEyLp9E2UoqEtQ5QpSLJkRHlCZSEXQlqAUkSSCJJGkgBJGks0I8oRdfqkWpO3VAO3TJJKhI0CJuyAspRJJIBylC7ZHmCFGDZQnT5io0bGhJumSQJCkXJi5Ai5CT1KRPUoC5GRZgnQJ8xRBIEabKEApspR5QmylF9ADdG3dC3ZGNyr6eSDbsmIsiTO2UEfX6pi1E7ZCtQBS16pJagpQQNikTcp290nd1TkZJJAs+gNJJJaDO2SAsnSQJM7ZOkgW6duya3uiadbKzDIgLJwOgTgXRBqiGyhMWo8pTI2idumUzmhRli0BST5SllKAEWUJ7FPZxRLMAjDEg1SBl0ZAGlSBhRhhspGR9kEYa5TRx6KRkfZHHHqrAERfuo+VfopxH7KZlM0i5QUOSn5dj6Vo8lgGgUUkTdfKo0p+Xul5e6m5SLl+yFI8oQO2UibKFiJVGTdDmClLGpsre6rCF2yida+qsOjuEBZ7KxLam7dAQ5WjF7IHR+yqIExCkczX0oCLIQgMUbt7/qoaymhfQzsANzH3VktQorO4cp4aag/DbbPe65DA8I5+N4gyrZYxOtZd1HGymDms0aTe3ZRshY6odUBgBeNbdViYHI8S0rayOkwc6ea9lhVX2fYnS1uelpi+Jm5J2C9Eq6COSojqY47Sxm+ZahL5TnaN1mh41VYHHUzufJCfI3QMGunsqVXhrKXCXVLITmzZAOo+a9ax+LB6ON1W+eCGpYCb39Z00XnU1ZVcRTtoaGhGVl3uLOoRtx0jtLKJdjDDwzA575oTLNGbANvdYuKQyYzXzVdBTPDOubQ3ClDNiiMno6Kd0EMGXO9ri8XBHRDh0VRPM6Cmj5ji09bLYwbCG0cVRUYpHG5oFsl7lUYzw0FWqmjMNKyqgqY5Li5ZsQeyrVEkcspkg0YdggzEIDdPIbDmG3UJCX4VCQ4DMnB+JBebLPE38N9h2RGoEkZDzqVWilJ0f2UT9DZBKwlpvGbO6FTCWd88TBVvLybGwtZVWyWK0+F6uio8cjqa9+WEggm3XoginpIWynxTxMe4T19Xz6VtFC+RrQb8t2oXQ4jgUEVRPiXJjkoqgktkvrGfZcjO4c8mM3b0KATG7KGla/DVXRYdNJLOwiUt8rxsFlZ7hDzD2QTwzvlxB9XXSeYuJva+qsVcuG1H9KDHGVhGcHaQdlQGvqTFoKgs4qaB0rJKDRrxdzPylPFXUEMLozQhznixcqMjbOsnjGiCWjl8PUMntfIb2Q1U3PqpJrWzm9uymgo5pYH1UYzMjOp9lXl5YddnVA7S4qWmkcydrxveyhjNlJCCX5ujCqLAcDLbN5nuuR2XQcK0M1dW1NLBWxwyhudrXD9pbWwXNF1pHkDW9wV22H0eG1fBsGKQQZcXpKryyg2zsvqD/BYHUcBnF8C4gNRNSyeFrW8qq6AHo/+/VdfLwl98YhHzGRMgp3mxBuSL3S4XqX4hiMUc9CaOUQfisJBBFtCF18MJh+FWBmt4apTU1Uc9jS1LAwtHsF5JjfDUMjcWnpZpG09A8RxA6gm+y9mxV1a+jMdE/K5+hd1AVKkwGgiwiXB5I7tl/aE7krriKHBRfNwxR0VRGGzYW4ReXbrYj6Lpq2tZR0c2IvYH8oXt1XP8J089HDX0tQ8kx1BYL9gtmqp21kbWEXaDcg7FSB5Hj2AYnhWEQcR0T52w1EpfUNabBjw/yG3W69opXsqaKkrGameCN5PuQCsniDDPF8OS4ZG/KzR97X2N1rUL/6BT/+Ez+5ZykSuax8boZmB7HixaeqUTGwxNhZfKwWbfeybN3Rt1WEmRN0N0L5TeyIaBA/UoqR2oULm5kYa4hPk0utiNrbBM5oypHRIG4UFWUHVDFmB1U8jAVHkcrbB52MkZss2qoHSu3tl9lqhpKYsTk05apwGrlkMjHtI+ap1WE1UDQ6112Do3BCGMe0h4Xfmy5eHDajlAujVeoY+nFyLLqpYWxDTZY9XAyoe5hWuds0y4qxxNiVZY+5sqstE6OTRqk5nLtfdaRZfE0C4SadLIopWPaEQYPhWZWUbtUBYOysZQllFrWWbKVJKanlFiwfNVzh0AuzISOi0LNal5VvnJTIkoGROLWCx7qFzpmyct5uLdVtSMa8KrPTsIGi6c5Y4qwifMy790D4Wxix0VptwLWITOAcLEJzKZ7Ig42up2gsaY3bKYRgHQp5Ggm9k5pSs2NxdfKrULi0hqFrdbWUoYVOSpyQ5qIRtDr5d1XL7DRWIpLjzLEtwNCju0pliWgogLIUSyEhcy5RJIAyhGGttZNl90Td05SKVVSsEmZjLX91GyINO60JG5gg5MfZdfEYpSexwcLhTxtaQFNy2dQiDANknLktHY2zRlRpmp1zlvE17JyepTEpnbrCYiQdfokndujZi5NmCZNmCILMEQahzFExAsifKLWsnSRLV5IdToouWO6viNxQmHuFmVVWwu7IuX7K7HDpsnfTO3soqlkPZLIeytGA9k4i/dQUyxyWT2Vt8fsg5fsgrFuqBwVoscgMXsgqlijI6FXHRNUTm6oICLpsoUxj09KHIgjbumdoitZC9A2YIUkQai+h0SbKE6IA7KN26lfsoyLoBT9PqnyhLKEDZdE+UJ0+UoGTgXRZEQagENRgXThilEfsgFrNVKxumyJkWuynZF7II2R67KyxlgnZFb4UYagTQpWbKMDZSN3QJ26F26dA7ZBEkmdsmWBKmJsmzFJ262BzBLMEOYJZgsAswTJswSzBBIGowNEykbutgCLJAXTu2TZiimQI0Lt0lDJJ3bpsuqAU+UospT5QgDKUsjUeUJZQggLLdE2UKxlBSMaCvlCbK7upuWmyKegiylMjQ5dUiQBFkkSEiyoHKUOUKXKU2T2UEbtkm7KURpGP2VDJJvN3SbsgFJGkgBC7dHlKEtQBmCEuRkdChLUZJAT1KkylRkdCjQE2UI8oSyhW2EaJPYplQaPJpZC0aKQDRBGRZMjLUJFllQdfqiTEXSbstIPKEun0S6fROsgTsEySS1IjyhMiQoEknylLKVQySSSBJJWPZJAun1T5SnDdU+UoByhOiyORhvsgjAUganEeu6kDVYA5RsnDCpMjlI1nlSIEGRyfLlUxFkLm3VEJFkBapHbISNUQOT2TZCpQOqcRklGUYYUTWd1KIiUQiKCIRqRkWuykEXspGxrSgEWuaykEaPJ7KSOO6yiMNPZSxRvJ2spWtAUgCAhlARZh2QoSEBZ0JN0ycHVGrNk9k+VikyOSLCES1BJBHIwty5wSNwjXOQ+UpZSiYwn0tujyEbkD6qtIspRCJSCPr0RBqCpJAd1CYStQx3ChkhsljNfFqoHR6rQe0XQOgB6KxPEZxagc2zSewV91OD8KA07ADcXFtlbHL19TT0lM6uxKYtYNmt3cVmYRxnUYnUGGmwu1Kw2MkhtYd0uL8OrcUjihmmbG2SX8NjN7d1j8UQM4dwmDBaT9rKLynqbqSNbEPtMwGiqnUopJp8m7mHS6yIcW4oxqqZNh+JOpaeoNmi40HXRUpeCII+H6etrGyCacgeUX0PVdPQ4V93V2GwSN/DY0ebqpzHP4rhmDYfWvhxIz1dQ9u73XBPy6Lc+zXAoIIJsXcLunvG32GqhxjDG1+O1NWyLN4cC7O4WrguLYdhcElPSuL4f2u1hET0WbHL4lhdFwrxZ4qqkk8LP+JEbXsT0/iucxCFk2LVlbSz8qluSHXWlxVjUvFOJsbOzwsUTfw+YNCR7rj3zSNcQJDboQjaSjlko6t88L7aWBRVb6uofz5q0yCTsVDSw1dW8shZntug5M+dzAzUboC5bwwuNgAo/N0TXINn/D0RgiyAGglI6GykidleHKORoDi5rbBZUvN2U8LTK05tSFE0AmxNvdGLsfoRqrSFKzJ5gpITBGY6mZglDDcxnqmzMB8+3VRPblNm+np8lRv4lxTJilG2hFI2CJlrZTqAsblse7KX291C0vJ5EcZdI/YBaVdg0mHUsNRLO3NL/Vk+cIM4hwcR0TZgrdFSPq3ubs0C5Kelw7xVX4cG2hN0FenMGSTnNdm/qyFOYmcpsjHaP6dlHPEIZDG3Zuija6zlgTy4aTEaoPvb4baqOKna6n5+w6q3T4gW2tlNujhogxJ9VC9lLJHHHE8XDWi1loL7wf93+CjeQ7qSNwqBFlZqRTMeORPzA5va1lDbORZgBsgGx+FTQG92qPVvlsQVOwGI5u4VBEaD52XqP2b0/LNZwtiDYJ6aeHxEDXbk21sPoF5hCx8+YC1mC5BNlcwzGcUw3EqPFYJ5XPojcZd8nUIPWOCOKqqmx6iwnGAWcuR8HOfs0G9gV61KMjiF4PhlbQ8Y8dtmZSTGirTkJcLESW3/UL2LAMJxnDacw4ji0lQNQGvFzboboLztrKI6KZwsU2X5oypFjGSOnYMrn+q3VSsmICeWK4VJ75Izly31stRKL/ADGvDoydHixU8JyRtjGzRYKjBFY5w6491cjvsUyi2olZj2KkBcFGw2CmGUrHopZgh+P6o7WQndZEjdknbIA5OXaIIpRqhDU8huD3QsabaoDA6BJJPlKALlM4lS2CjeFSgHKVA/yODgpi4NdZA7I4FaEM4zRlZEw5UputVztLLGxeGaTLJG+1jcrriwiqBnbnj1PZZ0zvNroVfjY8M3VStjDPxCF1iWZKnOzsyuh7T6SsiOdmfK1ytRTtvutzDNtAC+qWUKJkrcuZEJP3lxpqyIsmIsE5N0zjoqocxUb902cg3Uee79XLUQkyl5QcPMo3Q2KsRi7Qnc2+6kyQqGEk+yYwm2quZQmyghTkKAaWnVEHqy9g6oH07bXC6QKw9fmUzDohF3fhvZqNihyOjeddFGE7JbOyuUoddV3ZAM3xJRyEOyuWabiVkCyQCYOTrMw1yJM3ZOkshswRZigSWVtIDdLL3QA2R5lpJgGVOkmbspZxSJIEagSVrpJx1RZLTullKJv+CZ2yL7kJGUocjron7p1pk2X3RN3Qk2TB7b7JQnGyezrpmHRENysgmhShl0mt0UkY1WZaFFF5hop+X7Iom3Nlbjp7tHVZVQkprsOiquZkOVdCYA2M3b0WVNB5jogpFgso3DVWnR5bhRcolyor5HJsnsrRi9kBiKCq9tlCWXKvGK41UMkNigqOFlGrEkZsVCWIInqMi6mLUPLPdBDlCMCyMRo8nugiAullKlyu7oCwoIyNUsoR5Qll0QR5E4aiAujDEEeT3T5GqbKUg110EeREGqQRohH+6gjDXKxDFmITRs12VuGOxCB2Q2OymYwBP5e6JAJaEBFlKhdusAALp8wTJICUbt1IgdsFtZRJJ8pT5QiI0+YoiAhdusBO3Qk2TkWTO2WwswRN3QZSjaLlBK3dG7dCxJ2yAC5CnduhdspdA8wSJshRAOKoDKEQa5SNjbZFkcgDKU+UI8ru6XLcghyhLKFImyhAg3VGhA1UmXRBGRooiLKV/VAgjyDsmLVOBZA5qCAjoUgOgUhamDEDZEuWe6mEaMReygr5EixWOX+6mMfsqyqmNBkdfZXOWmMY3RpXEf7qRj/dVjIhLUEHLQloHVWCLoCGoIMuYICwBTkWUZHQoISLoC1S5QmIstMIspT5QjyeyWT2QRkWSylSZPZLJ7IHaNEeUpkWUKWALUJF1MR0KEtVFfKUspUhHQpspVEaK3uUWUJ0EZFkJF1IWoCOhUCTZQiy6p8hVOQMoTFqPKUJF0AO3TI8pThgugFPlKla0Igy60IgwqQMJUwiARCNBCI0QY5S5PdOBdShGGImstspGtRtaSrECPKEsoUnLKfloiIjRBlCmcyyEjoURFk9kuWpMoT2QAGIw1EGpw1aQ4aiDcxTAWRsb1UoE2L2UgYAmzpw8oCyABOLdUOYJ0EjcvVFcfCorlIXvoCqqxHHdScj94KGMvBCsC5WRG6OyDQHVWxG077LOrKpkUmSOPMOpOillrrSCN00zcjM6yZuJMMw+D8Zksr/AMrQuZxTjmvqJWspacwRi/kJ1TniAgxunjs2nnp5amV23NFx8l1NPDVcoGRi+djPfzCKSE9CDqF0nDXFvGNPMKfDa41Mf5Kh91lp6jXV2MVtYaHCqiOJo3FrEFS0/CHn5mI4pUTS9bS2XGR8b4XidR4fHKWahr2kgz017H5rYw/i+lpKgUFbPVSX/Zvc7WyDpn0OJYawvwucTs6wzHX6FT4ZjNFijzTZjBVMNnQSixB+fVVKXHHzE+CjNRbo3dBicFJiwZJPQVVNURm7Z4xrf3WR0BDWetIMzBcfxBxLjOD4aGVeFeMY8257RsurwOspsWwqmraSQSZ2+axvY9kEVRAQcwCqg2W6+BkjcpCzqqhLTeNqCkXsCBxBBAUxpJOyhNPID5gtChFQ0kT+dyQX9yudxHA/vOsfVTaXNgLXsF1jmkKuYlRj4xQ//CGQxDSIg/ohlgfXRQSRjQAEFadVC+WnkjA3FkETHwUzA5ti0WUmByuL1jcNw2sqnuy535LrkarFcPmljpKWpnDC3zcodfdd3jGBfedP4V3pc65WFUUNBgNcWQU/4pi8rGDUlShg8axw1UOH+EaDcEaC19lydXRci7ZniNw6HqtitxierrHPxgmDkOtGALC6ycarfvCpdMzY7KNqtG+oDntpZMpI190F3jUuId3urrmUb6eN1KTHUN9V+qqTRSQvySNsVgBqSnbsnI09KexWyzgWIPZPL5ySNionboxtbogB26cAqdtLJnAdGQ090Zbd3LY8BoO290EMuSzbb7EJEOabHtdXaNuGxVIkxGEyR2tYdFDXeEfVO8CXcroHIHwiSigxOKornSBkZuC3oVocS4qzGK1skZjcyP0ua2xPzWPyJzqGXCQDm7oNGmlfDSSlvUWQ4dO+kroJO5sfkpaSJsuAVFXnu6J+Q6bqs5vPpoJA8Mfvr1ssCzPFCKx75WZmXuQsupMLqhxgBDOgI1V8zCopjBGb1Z2b1PyVDf1+vqDugCS4iK38b+45eHsNnoqrNWkBkrdy3uFiFlwih/CIfkB+YWwEbdiVZgP48dx5c2qaR8Bd5WkadlZw1jpXc8MPKbu46WQDVxPM1wy46lQ5StX8CWqDY5myNuA4Dt3U1VBhtO2qY+niIJBicBcgrAr0dDQS0D5n8yOVltBoLlPLTYlgLrTx8t1RFeN24KucPNrJqiEYdHz3vOSRhGjh2WhxW37zqBAxkkT6OMCSN41aVYkdn9lvB1ThfEFBi9XPHJz4Xl7GjSN52Nl7I/dv9lcLwHR1uFGnoHnnsNJHNC941aOrF2TaoSgHbRaDuCjUpHQoS1BA4Cyikha5TO0UetlcUtTEnKcdbBTR1Wz82iaaFpBcoP2fr2WmWtFKHjM1TCTKsaF74/Ow+pWGVEhOVzliYabAkaUzzdU45dFYD7opkx3CbMUwcsA0aiJuizrYNJRZ7FO6YRtuT11WBImI0SuA3O52idjmuHlddBXmisLtVd2ZXpGkhVXMOcrViAszBU52Mewtc3qtPlu/KoKimPLOi3EpMMWcZG5h0WfPOxzSyZq2H0+YZcqrzYVzW5ctvdbxyYmHNyx5HZ2HQpopJc4007q+/CqpzwJINB7qyKdrR54gCvVzhypVkqCIx5kMVV5tSrdbh7mw3jaCL9CoJMO8gew2WbhU7JWnqpNSqXJeyHOXWQRVDmixcpxW1x5+FVZb5w4GymaWyendRSt69Vqi12E5WBS5lShebjVWg/VccoVJoQllCKMAoy1qjanMSBcoGThzQFPO3yFZUwe0ksXbBmV2Rpb58wv7IJCDGXN3bqqBrHDQo4ap1+7barXBnkUFUJXEHoVcy3NxssySF0UomjcCwi+iuU0x5YD04Ja2wiyIO1UIf2SzarlMNYrGZIi+oUQc5G13ULnOLUSLKUySfKFmmzJ27pZQllCjJ0kkkbJJJJA7d04N0m7JwNEYE3ZOkkgid6kzTZG5vZROfkOq1EB5EDGku2R+V1lZhj00WxCHW0spmC6kFPc5lMyAADRYkJsb7bKRrdVNkBYnEbbrnLaeEC4V+C2UKhGrsDtBqoJZR5FUlhu3NZXC4FqAj8ywM6Slub2URgy9FqEMVeUsb5cq2KZi02UZit0Vpu6jkGiCi9uUqN40ViQ6qtIdUETowq8keqslyAjoUFMxlDyyrJahtrdBCWFLK7upsoTFqCKxSLUdikW6IIspTKQtKYMKAQ0Iwz8qMRohGUAZSnDHKUMJRBiCNrSjEaka26LJYoHZFpspQ3KhaC1GpYbMEQcmQg2WQZN0xNkibIC5bBO3TByEm6Td0B7lORdIDoE+oKCPKEwNkRcgI1QMTZA7ZOmyhAyE7ogLpOagjb6kfxJ0kRMNik7dRo0UkCNAgZuylaLBAN1K38ykCVrdFK2K4QRuClDtgFQ/JUbo9Vc8uVQv9SCsY/ZCY1MT1KEuQR5QE6SSCJw1Q5SpcpSyIIxsnAujyu7qSOK6CAxk9EwY1XXQaXsh5XsVBHHHrsjLVNFHrqEbo9NkFQsQuaFYMaRiVZVsoTZFa5fsh5XsEaVixyFzSrRi9kBjG6CoWOQkXVh4soXNKCBR5Spi1CR0KMoSLpspR5SllK0gcoSyhFlCWUKAMpT5QiylLKFoMB0CfKEsoTlqAUk7tkykiMtSDVIkqI8oRZSiT5SgAtbZRkXUpF0ORAIanyhEGOsiazutCMjRDkU+UJuX7LIr8v2RCNT5D2TiP2WmUQY5SMjNlO2PTZOI0aAG6IxGiDXIw1GQcr2CExa7KxlKWUoIBFqpeXboiylOTZBG7dCAVKwBxUnLYOiCq5pQ5SrTwL7KPIPyoIcoSyhTFjQoyLLSFkCawBTGRQvccy3EJae4uizt/wCSq3M0TZyrTPJbzoswVZshaE5kUo5JxIjEmuqqcxo2cia8vKUcmjCYS4XKuCOE9QPmsYyOjF7JpsTeyNxOgAWJajJrmeiZJk5t3DcAXsrcUcUg8hXnL+LHwyZYoSQOhC0qXieqm8oeY/mFx5N27l9KGRmVzwAN7lcPxZj1JQyGGF2fNuduypYnxPV1LnUsdYZdrhgXPV7ad45k4zS9lyyyLRT45PO8vD1EcQpd5hd/XVZT53vkNwAOgARNYyTUjVcVYOEQ/fNQaWqkOQC7bdCiwjDmOxWSmdPI2KLZ7DYrUxnhv7hlpn4VNzJnjzM3JK2aDhuM4YyZ9Jlqn6yuc4gfNelti8M4LDieLzyVZMsEY9Ttwe6ysWo8RrsblpKF89ZyzaPKdgu6ZhTsLiNJhU8cjqsXIcditjhPhen4cppa6us2pIuSNVYlycnhs3EHBjYauuZIIpSAddRr2C72s+0GipMCkxKhPOlYw2Y8WsuEqOI8Nrsbmq8blc2mYckRI0PtZDUcRcM1j/DUNKJD01y3Wmiwr7VKuHxAxyldVmQ+VrbWaO1lXj41dSTSV+B4pNhs0huYGx+Ryz+Ja7B4jE+gLWzxjUAbFZFJiVJLWtquIKWSelG5asj2rhPijjDFMNjrn4jhdSx/+ss0/Vd3DOyWOPx0IhfILjKfJfsvnCtm4ZqnA8OsrYOsrG3IIXXcA/aFhTIjglZVlsrNInVkhyke172Qexz0sETeZMA1vuVky4ngB2qs39kXXH4zxLBRMFRiUz46aT/Uy82KQfTZa+CfaFwOWQwvmo2iX4ty0nuEGnIaScXpXl7e9iFVlh81l1klHFXQMloquINeLtfGQQQuZxen4joqWapkrMPbFELmRzT/AHLQ5PiKompazk4NXf0o6yRnVttFnO46gw6dtPxFTtaX/wBfAQRf3XL8QcXYrUiWmnFKHA25kGxXNV2FYsII8RnZemf8d9kHr0nFvCRaHjE2XO1mElclj3FFLh1aaoCOsfK3ylp2/guNfVPp6ZlOGx6i+Zu6oE5pGsG7zYLIbFsT+9a19a+HlOk3aBpdU4o8zw0rYkg8A0w19KyR0vna8G4sqfhmRSNyPvf3SJbR8phaS3dhQVUr6kN8g0691YJcIjG57RbVVgZOXvpdYDsaMpHVM0s1D0Tcg1LH6blTyNhla0QM2HfdbFTlPcfI26UsE0ADpBa+y0sHw59XIZZLhkd7gHdDWYaXyMqI5/wZXBgzfAgCl+7paTmVlRKJGmwaDuqknLa68ZNjsVaxfDDhMjWOnEmcXuBZUQHSN8x0QS1BB5RdpdqhIs5HO68TWfkUIJWBNvpnI+SZ5cb31KcNJ1G6Tm5DmeLLViaiqauOhqMOYxphk85vrsreEQ1WMPgoYHNjyEm+XQD3VrBcFqsQfFHS1TY/EtNw5mn6r0Wk+5MJwuOkonxS4hTXZoyxv7pLChwnBhXD0VXU4pT089XTuzxPjjvIQfZcNxgaCbEvvKhZNGKu8kkcgtkN+i9X+4sOwzBH12I0LauWS73G1iL9AuExTgfiPFaB+LPmhdHTtflgBuSwdlGolzmHQw08rX4zSyMpp2fhP7lUCWB5DHZmg6G247rRxfHPvLDqKidBy3U259wLKDDjTx1tNNVC8QdrqtKs4HG+rqHxGMuYG9W6D6q0ylxLFqPwVDaZ0TtY4xqB3K2abFaGifU18EIqHE2ffyiNntooKPFPurF/vSlJFHX6G4sAP+SgqYRg09JhjMbkijkiklNMW/1kb0dThs/JlEjAZpPPHY6ELpMNnppMClhs0tGIiYXF7DVRYzR4ZDjJrsOhmq2ykFsEMZMbid7FB1fDn2f0uDcGu4ggnMtfVwB8Y6XJ0A10WVhBqBjOIV+P0kYZUM8HOy1y19t9Fp8I4nX41gpwCelnkw2mnZnlGhgZuWH5L0PCODMEwycvoZ5pmSi5D7EXQWcMwqCklhr4/wDsjIhYabDUKnidOKStjqGO/CndsNA09v4rP4c4njmxOtwSUcvkTPEGlrgbhbeJ0/3lQhkT7ASB4NuoQWw8SsbIzZ4uonm6eLWFvTRCRZYAOKHMid6kF9bLUJJz0VaWEPFs1lNmKd2ysSqJgbFHld0VeOfNUWGymq2PfTvYz4gsrD4KmnuyV2u63DDbbNlcGlysNk0WQ+R58t7HurlNI7li7rlZmGl4SIgeoVcOUgd+ZZVLmKZAizBZEU0l25B6jssQY1PDVxYY+MS1UjM+U6LcdFzLOabEbOG4XF4zw7Qtqn4rildKJYYi+KTPY6agg/VB0mNY3QYcxj60uO1gBc2KkocXhxTCZquhMkbLENzssV5u3jCjxLkSYw+oMsTcjJ4huzpfush2MVUtaI46o+STlslEhAczpmCnJ04vbKOupZcNZXPqmiF4uCdLq0Irta/uLrz+DiGHEMBdg1JhpHh4wObbyEdwutwnHvFQUwkgkjvGNXbE9bf5aI5tlsTLbJPhaRayeCZk7S5l7BEwiQZmEEKikaLXQJhReyvlqTWqjHmotwsyqorXbZdFO3UuWfOy7iusDBqebDDkjF/ZQ07nSR679lqzwNeC6yzGMc2ZwIW4lzpFVwPdEbNuVnSwva0my3HtzNVWRgJLcq645UzMMiKocDlKtsAkGZV8SpJoW+IjZdo3UFNW+UBdfVz9GixuUnSysM1VSKe4F3XCsxuyi4OizMNRKyx7WjVIzdiqrpfdQy1DWjRyxxbtclkYWHVZ8yhNYVHNUue1dMMaZnJFUCOxdfVQRSPjJyHcWSc4v3QZQuzmsRTPy5L6K5TlmQh30Wc11lchqWM8tgT7rMwRK7E9rvIEnOs4qqx2V5edrqYyZ7rnMNxKdpzWspA5UGve19s2iuROztzLE4tRI8ylbsoXA9FIx2ixMLA0kh1T5VzbxPlCdAkT3Upok7d0wT9VWBI0zdkj0+alNRFHQncJDcpyLqshcSopIjL6d1MW6JR5brYjjheHDM1XoW2UQF1IJAwhqItsbcqURiyqRzEHa6txyhwvsuUqO1gkhL0Q2uo2NpsFPFI0AKqpGmyxMC6HgpF7bKoHlOXaLIKSZvpuoZJM5QP3UWchBabsmOUhQ8z3SD9dlsQTR66KvJH7K87KVXkasCvyz3Q5W91O5tlGQtiqW6psvspiLIC1YDBuuqYsBR5QnWxAWpsru6sZW902UoK+UJ8nsp+WllKCINUgHQJAdAjDUA5QllKkQIWICyJu6FuydATdkWYKNMTZBLmKHMEAJTk9Sgcm6ZCXJZigci6kYLAIGgkhTAXQPlCFSZdEDt0Au3QG/VFlunyhBDlKZS5QgLVpApspRZSmWVNlCYDVEkgLME6azikBZAsoTogwJy1BE3dSA3Ubt0Ybogka4og9yiDUYGqCdj3uO6nEdwghido5Tg2QRmHsoyxWXHyqPKUFYixQu3U8ob8KhyhAOV3dOGqQCySAQ1WadouEDWGymiFlgWhELelQSQ+YlWY3aapSuaB5QgrMj9kXKuk0+ZWo2XstCryT3T8j3V7knshMPsqKBi9kJiVx0Th0Vdw1UFdzbKGQFWnDoq8gVFYi6jf1UzhYoHjRVhWcOijIspnAoS3RUQkXQqVwKFSgwFk4HQJwLqQCyCEtTgdApuW1CWrQjQu3RkaocoQCWoVJlCEtQCkkkgSJCiQNlCcNThqIC6B7N/KkGNRJAXQCI0YiUjGI+WVRAW/lTtiJKnEN1YjptAkyKmSyIRK8KX91P4a3wqWKXLcnEfsrRgN/Sly/ZLFcMCMReylETr7IxF7JaUg5PZqidG6/matBsbQEMkV+isSqiMrETXZzZuqlNM93RSxUnL6K2K5i02QllvUrMhbGq7pAfMrDMopBYKrIbFWpJGkZWjVRCIHUjVaRXBJCbl3Vnls7JiGjRoC3DEwqmKybQKSRQn8y0yPO3sgLkJKAnXUrVFpMwU8dS1oDcyp3BCjeCRlzH6JRbaZLA/dKughNI7yrBbLVNPkJKzMbrcZMTmPzOj6ABefNqGbURvNYXMfG/wBlYkq6qKM5pIvoFgtYGvzyF1/mnfLBM7kmUtB3JK8ueTtEFU1U/Mc5kxHyVR9dJez36qxJStEghgeZHHa6uQ4NAKd89dT3ewXtdcZOKtgmGsxWoc2eTKAL/Na2I4PQ0UGenDg/oS+6yaeSMyh0I5dtsuiixDEJG/hmR6rTJiElQ8GlqpWz/FNKdAE2Etq63Go6SaqrKylF+by3EXWTiWO1WL17bCKniAtcbrosN4rh4fw00kAimk1tKPf+9ehl3ph4U4cpY8RqKU09r2BJJC47iP7WIqsOpcNomGM7Odrf9FxOL8SYrjL71s+b6KlHy95I85Uht0R44xWrgdQNoaNkUgsbs1ValoIauUR8yOOY9XmwWJI9o8wahmhqxEJvDvydDZUdBjmFOwZjHzTQztf+XcLPw17SS+OYhnVjtiFjnnO9ZlHsSVdpJ+Sws5Yd80YbcmJ4bh9MDhckniybFlrgLCBfiNQM5jjJG9kzg+aTIwXcdgp6rB5aSmZVTSFrnm2TqEFzDpaXDKwsxmCeem0LWtkIAPyuifWcPzYlzJGSw0zyBdouQsQEEW85/irGH07KutipnzCK5uHO2QehUnEBwDDxXcFcS1VO1nqgndcO+hUf8++JeNHVFPX1tZK3lkllKy9/0WLV0s2JvdRvkjfNG20bmN0Pst/7OcH4lwSWTFKSWBro3GKrophaSQdwhycVibamkeYJaaSJvw5xquqoMPrcZ4YY778pIIg3VpGo1+at8fwji2q52DwzU74BeZs7Mt36aDVcth4NTiFBhOJUwpryhkridHDsqlqYpLteyGAyysv5m7FQw0tRKznBnoIK9Q4pi4ZwKiloMAfA+V8ecsDrn5rzmlhrq2B9KatkDWWu1xsVJhQ43iVLXtgjjp5BLGP0VLklkXOnBDeuqvR0FJQS8yepIlHpeDcK0yCGtw+qq6p51Hly6AjuisyeWhdEI4h5+6pPY4/ErsWHmR0bY3ttIL3OihfDPFXilZpI0gA9Ao0aOWRsXJKUAzOZ9StDF8Hmw4RPMjZHSbtbuFmOdJTybGN7O4VEtRLI2fJBK5rXdknSz5hDJKJLbEKJ0hl879Xd1JgdG2uxF9O572OIuLIBq4iYOe6QuI6FH92TDDmYiyQSMIu9o+H6qWBnNcWVDJORch2VRvpHisNDhU8jm/CNroE6KkfCx9KSS/oTdUnDK5arsMNNSOnrY54HsJF3HdZ2S71JGo3C2w0EFcZLicgEW9KOelga/JT1UcvY7XUuGUbKyUQVMkkMUbSRnOl1XjNEMRnbXWaxjsgsNz0WRWnqq2ikdAyd0VvhadFr4VxAGQRQzUTHM5jDPJn87hdZNYaUuMsQEg6XCnwCbDJKpzMR5cbDGbHYXW7SYe34zWQO4cdUcwM/BD477k7hefT8RVeLGWipGDDoni5eCb3+Su8O4nHScNSzBkVZNQSnltk1AjXX4PinDGP4JPiww6JromkSx8vUHttqpKvBKlkDiTFmzXIeSb3PdCXWiDCtGtwmtFS+rdSmKmnlL2lx1AKKdsksAp6d0ZZHYuyaEFUQNrnxUrYGTWYdxbopn8+PCY5GV14s2kThqFnGJwHm6JR5C8NebN7oNGR0lVyuXzo9LEk6fquwouL/ALuwuN2HzgSxHlOpiLhwXJ4bXz4ax8MLObBKbOY47E9Qp8FignqKmCajkdT2zu/1jff3WB6fw5xbSxYlR1VLOKaarFqiJw/CmHuej/8ANeqYNi2G18Us9FUwjkNPMhBsYj8l84YlS0uE4nTUNcx09HJEJGuI1aO66DhShw+ooqrEqWScPzGItbMfxQNhqrEjvMKwV4Y/FZ4Px55xUNff9+/+a7OPFMPjxQ4a8kvlbzWADb2XmWGY5I3hmSonrKmVkd2upmEZ4R/itnhx+HwYrTR09TJWzSwc+lLZRe3UFaHdzSRxtLnO2VeOdkt7Ln5cRxvGJzHh2HCFrHftTNv9FvnQAn1nV3zQMgRoXboI0kkkBqGYeXVTNczNZ7rI3RgsPlugoP5Yj+aGDI3YpS0sz3esgjayZtPIywO/dbYXWSNd6UcmrPTc9PmghiEYSqHx5THJch4tYblYybBLVmEPbJNYsF7NKeKfM0PBkGfbN1XB8RY/NS1BbNlno4nZJWB1iT016qjHxlijqKCWllET5nGJ0cg/CA7rnI9Mrq5uH0L6ouZeK12k2vdYmN1+EY/wrUT4lTmOHPygSdWkm1wuUOO0kdFh1P4gVE0AeJAy5FwffVR1OPR1s7cDlZyaKotOGkFov+iox8LNLQVknJqZaujj/D5ZjzCx6jXVBHh+CVmKz0HOmjeQPD6Zbv7HsixfDX4fKypwySWI62dGLgmyycHmrXVEmIywc+YAx/iDLY9dFxluJdTT1kGGwVXj63kTPh5Zj5gJv00UmG4dhs3haqTGJIc5YY2GY+aQb3XJ0WC1WL1UtQx0bWUkZkfKX3Dj0t+i62mweCTBX4zisk7pGODDHFFffqQNQpEpk2afiejw6oh5nELadsshLY7cwX9yF22B8VYZi04oYKmOWZwLxyBcEd/ZeLVkmEVmHT1EMfJnpMhhINwT7dl6Z9nVW2swiLF5oYoqwtLJAxtjIwdSF1jJKduTqmTNkEjQ/uEN8xWkDKLgrMrCyLV7rdvcrWkZosLFKXxkHLeXRvfcsI6EFbiVpDNPHEw8x1rqtI0CxzA/Jc/i/EMmDV9PhVXG2RrxzC4kZwB2VuCrZBUkvjkFJL54i0XFvfst2lNC1lHIGAXvqpQRMHGC0gDrEgqpUl4NrELcMytRMbLFkc27SqFXw22YF9IQw9Qeqv0D7MaCtG2i1yRyX3JitO8tztLBtqo2yyROLHts4Lp5nZt1j4hRsla58bPP0XWM+TjMKfPz+UpnsuwnMqzZWsOSRhBUodc5WbLpTMq8mhyoFadT8z4gFF4WQdlpJV8iblm11Lqx1iiu0rTKtYjdOHOabtTvBumIIVEwlLzq5SRTvzKqRZO02V4qvnXZSxTcsWVeAhwAcVM6M7rlOLcSsiVh6qVrrqlFo4K9Gzy3XLKG4SA62RN3QAaqQ62K406RISbKN8oulMbMNlWcXNGYrUQza01zbbJ3StHVVY5DbdM90h22WuKc1sT6XT89p+JZ4bLqQdkTXvB1KeGcl8S3+JPzvmqfig3ykpOqQW+Up4ZyXc4Kdhss5tXIDo5TxVTjuU8M5LokTcxt/MoJJ2ZPIdVFzipxOTTjfoFYjdosuKbYXVyGTyrnOLUSvsPcXR57DyqvHLcKTMCsOg+ZqEZdpuoG7o8yxIluUxkUeYoS5BI5zSoybIMxTIHzBOD1CjzFLMVgS5ihdumDkibqqBw6KPKFYd6VGR0KiIi1A5pUrh0QFq2AyhLzd06SLRJwLpkTtkPQ2UoXbJ0kRHlCNuydJA2YJZgmt7hMgSNAkgSbME6bKUDhyRcmylLKUDIwNUCmiHVAcYGnlUlh2QsFkSAnbIMoUiSCLKUspUmRLItiHKExHQqVCRqifRFlKHKFIksKHKUspRJw1AIFkQDeifInylEswan5Y7o2t1UwjCKqGP2ThhCtcoeyjLBeyCNrGlSsiTZLbKxC02QWIIQ5m4CGVljlUUkjm3sg57ygPMUidNFGJLlO51kDnZR+XuiSAcgdrdPSiy/lRhuidoUCY02UrW2UgYLJOZ2WQLSpCLjVAwaqUjRBFG3zK7FHsoGR3IV+NlgEBBosnLWW2ToHHKFsVZmKpIzVWJprEqo+YEpxDPaoZGqRz7qJzsytMK5YT0KF0dgrFtFE/ZUVi1ARdSO3CBUBkQZPZSO3TIARBqdO3dQMhduiQKgHoUZ3UbtkCzBCiIuNUwY0bCyAXbJlIRdMWoGRpAdAiDUYMBdEwWSDVJG1GyDNNU+VvdSZQiEaAFLGLpxDcKRsVuiAmNHZWYhZQgWViPZBMxvxJPGiQOiHMVgBkuUYp83xWTHRJzitBPiEYyhBlCIlxToBy2RgXTJIJLMaNlDLJbZHqVBL6vqqKs7iSVVlzlW5W3UD4iQt4+rnKoCQ5TMc86BpSbEA7XdWGvyjYLcqrPfY2KAkEeZFKbvueqAggKwygmPRQqaTfzI44cw9K3HkwrH2QmN/Vq04qAFwuFYNEwD0KcoOLEDLI+WewWlJSMALiqlS+GCJ0jyGhupJU5QvBAGRg6myyMVrWSxmOmdcd+6zMR4nY6V0VPGZWn4nCyyKvFKgxHK0N+S47M3WEj6S7s0jgPqqc76GJ+WOFsrvdUDNNN+0eXj3VqmZDHcvGq8uU03iOOOUyc98pYegCVTUVElw6ZxHYlNPPHcFryomjmjRcr5NBppeXJ5QnnbLV1LYmRAd3dPqnjyRvu5uZTeOibtotRA4GojyyvHJdHrfIdwkYnkWjpZGjrmXo/2p4TBh8tLWwsa19rGw3suehGMcUU7xHymCnaH3JtdehzcnLE+P1tso2iRzxHGLk7BdThzafEquPC8XpTLY5GiMa37q1xLgx4WqIpKabNCdo3tF27I1EuVxPDp6CNnMIJkFwBuFu0GHYI3Bw/EcQnbI8AgA6KiznY/iTS6Fu3mt2WvLw87GpRQ8OGQvjbeRsh3+SjTn452SRGFmYn95R07Hzz8hg83RPLFVYNiBpKqHK9j8kuZuwWtieAsFK3EcLeXtIuSHaWVGFUQVOHyh7hZ3RdBScPY1xHw3Ljja6PJTu/ZuZ0trr/wWE7DcSmdmFPNJ9L2Xe8D4zhUOCzcI42DA+pvaV3odf+7ZGHm9NVvoaqGqsWmM3seoWzVwUrp24jC/M14va1rL0Oj4don0c8da2Gqp4hYExi9vmuCrMRwxkslJT0MnhWPOU31H1QDhcuL0srq7CXmCxs5l9StjDuLqusxx8zJjYi7mg9QO65cVFTBVCSnmljaPgcnqKh80/iAOW/u0oO9w53E3HdVNh1LX09PLAD6x57dv4IZMBohM/C+I6UwV9ONJeZYO9wVzOCcSMwankY6kEsz/AOvDsr/qsmtxCuxSpM9dUyTFujcxvYLRxTOc/DsTlyOZIYnFmYagqrV1LpKp9QNHvtqo3zZCQxRFznm7lybHUVU9SQZ3A2GwFgn5ruSIw8jXYdEDQy+UoZBlkyNQa1ZWf0GCkkZmezVsoKyJ5C+TmOkObunI0UdiTYNuVsSU887ayGTnklhuOYdAtLE2vqneNMkOoAysN7fVY/JkcDJEL26K9NVxyNZy6XkuAs5o2QDAx89TDRD+vkEZ+S7XHMBbwditDVUEJn57CHggDey46gD5a2HkMJmBzx27hdpS0HFXEZZUV02ZtJKQGuaB9AfotDk6+CqoKvxEwy+JJkDeg9v4qsZZDUeIjn5PdwOoWnjVHiNXW1DyRJyHcu2xH0WHPFLC7lzNseyyOjrK2pkhjGLR+NjIuyQ6ELDI7KeV9fJQQOkH9GZs47lNExmZr5BdjDdwG9kFmTF6uaJsXOZDEwWIA1KrxU4ndlYWAe5W9jTW11JHW0mHEMyesb/VYTGj0jdZmBXmaIpTG3oujweobSYcI6+hjkaw54swtYE7qTh2OetYKSLCnVbmyXcTvZNxTiIGJ+EMEgbA0NI7qwOuwfB8OpeKJ6GMBtLX0mjSdiVX4Ig8DxBi2BTSDKWEWOmYgqpw82sp2sxjFKSSelg89O7N+I0A7ALS8XT/AHnUcUyYNHNSzyAxyTuyuB9gqJ8T4amOG1lPVw8uCMcwS7knoF5VTSPFQxkEZOf1W10Xd/aLxi2oaMKoKgiExjmsaLXPULFdhEGFxRYhQ14HMh1BFwb7oM2LCZ8Rrfu6lcM5FwXaBVKjDjQMME7/AOlRvMcjLbe63OH6ilixCfEK6cNfHGTHcblY9XiEFZPLPJH+K9+rroIZC8BuQ2sQVqYS2lNYyfFK6WlZMLtlYba+6hqsNqqLD46iuj5QnF4tb5ghlo4ajh1ldzm54ndfmg3uI56/E6ikfVRgTsaYxbqOhWPFU/duKRQ1Mjo4mSAyhpIBCu4fSYhiGF0mIx1rWCN2R3M3aVYbTMqr4XXUZM7pdKxguL9EFzH3YdTY9JxBwnO+WGPI+YHUPuNvdaOB4jDgvFuF8WNF6Im0rGH9nnFjooZ2UmLcMswvDIYqSugPLlgYbiQi3p+a5+sw2qwnC6aoqHva2R2RzG6aey5j6LxTD2TRHFMNaNAJI5IjoT1Vxx8rT3F15f8AZJxXQ0tFPh1ZJUARu8r3OJHLPcdLL1F2VzGyRvbIx3pc3YhaiRE7dBmKkcLIVoJQyyOjF/0UoFknNa4WeFLFOES1bgw5hlO9lqgZWqGLI14Vq/kKohdk3coTlJuFMR3UJDQFoE3ZIsBNwy7rWCbMU4dYghZHjeIUdLNNVUjs/wCHO4SEG1iD291Vw+qe2N8eI0ni6UNPKLQAb+67finDMGjnqKukZy5gc7n7i1/OP8VxPFOEycNVUM0GKfeENTFcFu0TzsD89f0WMml2jpJ8Jq458Okjp5iw3MpuLWWZiePY3jtO6Wt8JFStvq2wB26WWdh1RHWSOqsbjfWPYw+QeWS/+PRVsQomMozJAJYmj0wk/wDNlmIYlp0XFlZTYMaWKYOkY5nhyR6R1/wVnD8TxKWWowiso+ZLUWzEnRptueyn4KbS0+NYacZwrlQSQviifI3Qk7EFdC7A6DCeJudhtWC2rhe+UPkFrdACE4s8nJyRT0lfS0GH1eUSG8lP6mtIPXuF3Zbi2E1UFXJiNFSROjD62hDTqL2JH0XJUUkNJWUWG8YTWo6h7zFLTn8QPvsSOmqsVlPzsckfi9a6agYWQwOedRBtcHrYqw3Crhs+DVJx2khroxSkcynaRYuGfQj5Cy9O4UwesghwuSqp42mCIMc0HUC3rK8prIsPp69mDU5kfURymFszAOWYzra69C4CfTYHiLQat76bESWQOLtARfT63Sh6K4siYXvNmjdYsnEtFS4t91zfhB4/De87nstd8jC9zG6kbhcP9ocNRVwRTYbh3NfG7PHM11soYLm56qwsOnZjlK+pqKemqY5zFYyG17EkC11SxqUU9bBUuka17LxHUXsdvmvKaXjfFTyI8lPknkDHmMeeQXvZaXEuIz45ikFfHn5JjADXCziQb2t9QFiMmmT9pGKMrp8MYGumqqRpFU4DcX7qhHX4jDSUsba6UsLw/wAM46/L5arpW0r6vhXEq6qp4IKOOJ/ILTZ5kHf6/wBy5sRUVUymzzBk7AwmF+hlYOx+qsyTD0jBcUZiEoqmBsXLHLmZa1zYahXXHxLy1/0WJw5hfMEtTQ4pH4l72CRj5Mway+m2+i3jS1bag2geLOtfoR3XXDNiYFDE+IizVoNN2BJkTdGuGqfl+63dpSpUi11kQzl7jn0I6LYrIHywuYBqVmto307XyTN2F11wYlm4hTZnuljCqRxTR2kdoppq0u2dor0MTJaIPcLld8ZcphQMhTtzFPI0sJJboE3OzbWC3DIZMhPm3TBgItlQ8txd/ipGNyfGSqisYHglxOiTwB0Vl2oVd7HLUSnFCWIQLqYsf20QOCtlCZmursUhyi7/ANVTYbFSg51jJVu13Zx3VxhGUKnAWki6uM5ewXDN1hP0TO/wTgaKRsWYZlzdFV4cVWn7LVELPisqNXDGxxcx4N1uHJRDrHKpmG4UfLub5VLHE8aO2XWfRJTNY3LayTY9dWqVjADbdTlnlC522oT0rJRfMQVW8PJ8JWk6N7tgm8O/4QtcmeKpFTuvmcEborDaytsieFMadrh7rPNpmtbYZUYjJIsNO6sPpSDspxADEGFOYrNheHC1iPZXoYnlqjjgEegV6nZ5PSuWUtQr5XsKmjD1YfDZpUceUaOXNtLG199kT2uAUkZACUuossqqlyAnqVI4KF3ZQLMUOYIUkDZgmzoElmFgfMHZOJf3lAXJByCzzj3S5gsq+YplEWC8EpEXUScHVbCdumRdPogs5YBN3SdumSQJFlCQajAstraLIlld3UwabJ8oTFFe3uUiFIRZNlC0wiSIupC1ARZAySLKEeUKU15osnsnDVMGp+We60IBHqpg3REGNRhqgYDoETdk+ROBqgcNv0UnL1RMFlK5nZRVYsQ5PZTNa4pi2y0iuWoCLqV26Gw7IyDKEGUKRJGrtHlKNoSyhEBqoyWUp7BOkpYdo1UwFlE05SpmuCrQbFA4gbqY2sq0oJKBhKCrEU7QO6okEFTRh1ggme/OPVcoHM7JwLqZkemyykoGtKPlkqbI1Z+OY5S8P07JJ2c2WX9nE06kdz2RpcyORBjgudj4xqpWCRmDtDTsTMFBP9o1Jh7Y2VWDSSPeL/hzN0+aDq25lIGOXO4Rx/g+J1UVI+inpXymwz6i667wr/yII8oTojFJ+QpuTJ/qigYNRt3TNikHwO/REGPB9B/RBLFop+YAPUoQCOhTlryPK1AfiY/zKKWcOGULPlD4jq0j5qNtV7rTCWY3JVaR2RSOnZluVDJLGUaISE9UOdQuuR5HgJhcep91WVgSIJDdBmd2QEuQC86oETgUKBnbJN2T2uhylAnboXbI3bKN26BE3TJJIGduo1IRqllKAMpSylEkgbKEsikylLKUABmqMNCMMRBqBg1SNZpq1IM1UzW2QCI/ZSNj19KmbGpA0dlAAj9kQj9kensk3ZAwi/dRAORXCYEdEEoamI11S5jbepC6RndICdumuEHMJTgApQfQogLpWshMgvurQIiyHOAoZJHkFrSqUsk0Z1KtMtF0wA9ShdK0qi2oc7YqZj7C5K3S0kfmPpSEZ+JROrWx3s26YVLpBmUZTiNlvdQvjd8LVNCCdXqwGx2zIM0NcD6EnNc74FfeWAGwVdz231WhCyka8Xc1EYOX0smdVsYbZkPimv8AiRlZZuETyAFWBeeqO5+Jyy0GRucELHx2ClbRnmnmb6dFslwtuFj4oxrmv5jQW9rrGWQ84cyT4NFpw1uDwQiCQc2XqMqWIwlvnbCW37qgIc6807HTEE9PSzTGSMcsHYbpvCADy3UstBUQx52R++6rCprW7kLPJpTnhdG8tabpROAFr6o5DJK/V2p3Tx02YXZr7lc7Ebowbm6iEbXOzE6KaQNZ6j+iiLw0ZRstxK+rqvtNo5sRr4KSP0Df20VvB+F4cEweupAcz54g9rrroazDY66tc+brstB9FFJCGW2Fl6nFxn2fcJUVHDJis55tSSRv6Vj8T4e/F55317JcoNoXMFx7arr8Ip/uGWeOeoBhN3k20C5nE/tChpax1Jhro2wxkjO/UfRRYa/DPB+H4NRGqjF5Z4rODmA2KscB4WzD46upOr5JCAbLj6HjmPFaqVuIYlJFFFu30h30XQ8J8Y4bUCpZECKWO7xOToVtVL7W8AhrsPGLQxATxEAkDUjTdZ3DOG4UeFo5qpgcyP8AaEm2pXNcR8aTV2MSQ1FXLPQtJAA06rArcTD2Gnop5RCfhOyDewDG4MOx+qjpLGlJJaC6396g4okwqZjGUNEYZxIC67r37LmI/KcystluFgdDVcY4iaFtDQw+DZa0hBuXfrsufjramIFrMhub6i6cuuEMceeTICLnYE7rItTVzcRDW1UkcT4xZrbbqqWvFzl8vQoJ4ntcQ8WITium5QgswgbXCsSBc/TMELZNLo5Ia57i/wAMdeoVfLJG7LJG4H5KtpZZQT8k9nhgkdFJkPxW0K1eFMEfieLxB8jWX2a/Zx7L0DF/s/xUUbTzoWxucLsIsB9LIPKG3lkaxkchJNgLLfqOFcRo4mTyUFRci+Ut6L2DCfs2wSWOKKryjl2PLbYAHug+0OqwbDcFdh1FVHxkbbNYHXLRZB4RKxrnksZYdr3UcUr6WcSR7jZdXg2D4jxjS1UFLyIn0jS+5Fi76/RVsHjreFq2HFq2hc5l7FrtiEGK3PV1YYyMXnIAY3QErpKPgkmqkgmZPG9zdPJcfqpa3DX8WOlxjB4DA+I3yiwsFv4TPOzDGT01bLNWBuQm+rT7hBg4VR4zhNPK6TkB1O/ywOj1IPUG66LDJqekwytxIVMclXL/AFDTpHf2WZh7+J3SST4py52Fpu19mlcxPBW19dJ4J8jWMPmc020WbFbE6nEKfE6p/PmD3uvc6LNcZpSZJ3lzvdalU+fnMfPUeI5YsHdSFQm/FlLwwi/QrQMTzmmFI70KeDliSPn+gvAf8rqDkystzBZTxUnjmmk5gjcQSCfZB03EEMXD+fCqF73sq4xJHfQxg9lz+HRZqxjJmPv2aLlalfjT6zCqWhxEOjxCiNmyNjvzGdFewbDHuoW8RwV/LqoiQ5r230/VYAYJiM+GYhUxskkpmvFxmj3C2cOoo8flEFLTNdMQZzJLGTb2WXj0j8UZBUMjMT2eS7tr9l6hw3RMwnBqWkcLPEYJ7m+qrDkqqpdGZ6SoopmzCMxjKLtHurMlNFVfZ/zqgW8PGRAdyZBtouoipI6aeoeGBxnO5GwKmw6jpMPoTSmMGAG9j0Wmol8/1wfVavjn8RvOXDqnp6SCkqoXiTnNO7QdV6hjPC9PLg+KV9aL1EhvFldoGdrLmZKGlrqBtbSUTGtp2BhLtNUVz8/i6Z/OgZlH5XN1VemgoatzIH0snNvd2R2pHyWhJNSNqm00k7azObOcDcWR11JBFODS0kmT93UrA3W1HDmP0n3diNLNF93xZKc3sXdLBDxDwXh9NSYLSTMEFdPKwSsYdSw9/dY2E1UjMUgqGPAq6ZwkiLzo63wFeg8PS0vEmKu4jxandnE/KDL3EQtoD/z0WuTDzjHafEuF6mowiKYijkGcNPYrZ4fwp2IYQ7DJp5KOsfaWPmaB412XT8cYbR1dfTUDYo3h0rH+JB1aL2yP7hN9rVFSYfNh0kcTDkoeUAezLAJXJqZcqIK2KVhpGNp6qkvlfA6z/wBVoV/CWLVuHYbAZJH19ZPdrC64DDrd/b/it/7N5IJOVK9kT31kN7GPzNLOg/VdFxBglRXYGcZp5pIaqglM8OU6CPS4TgxTj5uGJ+BGtrqmqbUOrAaeenZa47ELqMcjqME4UpKuJ9pWZMrhoRdctxbi0FTDhlHAXPl0fI5xuSSRut+qwetoauroyw1FLV0fNp+YfS+wuwfJYtuHd4fXQ1NBRvnmAmlgY8g9TbVWPquEwd+WhwGSePmRsjfE7TY7LtYDIzNHN8Bs09SFq1WBskRoha5GtFoi1yCR9RH5mG4VhM5mcWaLoKwrbSZJdG9SrM1mbmyyKzEKSjlMczbu+axPvLFTkiwnEGNiF789o0Htfog6oygtcWHY2SbN5HO3cBcDus3DpmTMEjxUS5/XMRlYT7BWOZr5UHO47xEKwVWB108dCJIxdjYjIdehsFxOJt8BhFc+WPm04kY+MXtd4vbQ77rs8YwXEqvEH4hTVQ5wgIawbuNtFzcmF4xiWL0uEcRctstRBKIHtHlabDW290y81hxLIcWxT/QYHTPiAkNugVbxuMPlOI1s8kjb5G3GgI6FdeeH6jhRpbX0M4Y/yOroJLxuHuOioMw2j8DVUVPikUnMlfIGM3IXOY4kq82P10FJS4TV12Xw55kYkbpH7DsEGEYpSw1NLJVyxSwvzwOayQ6e9lXxCqw3K0NkD3BgY7OPO149/qoeFDSP4hoJ61n9FEvmLW+yvJKdlQ4BMOKMNa/NU09dC+SIlmrdD5LH5L0SbhmKvwKCnxSiAqaanLIyD+zsN/e6y8FjfiPFFVXuq7HD4BTtLtQS/qFT42GLYA6i+6sREz6xz43Nf0sB/DVLXGF6iw7BOH6eObEGQBksZEbIY8z5Adz3vtsudjxjC6HDm0NLV0kzYpXy+dr45IwTpYnchPg3FUApZsRxd04rYw9ga08yOMA7N7X0WdHR0vG7KmbCpBC9lUyR1PJD+LY9Q++3tZMl4uypuMMam5NBHSMm/C5ktUB+JyNd/dcNiWPY0cOZFHWT1GD082QskOUu7gkdFsfduKcG4ry3yCTDqyIx5SNSLagLC4sxCAQUtIKX7vp52+bLJmaBfqB1WYlm6Uavwcr2VdBQijexwIYx36hHNiGJVFU6aKGS0v4bcwJynsLdVBGKpspnp66mm5H7ONuzr6WPyUlHDikGLiqxGqgZFm5haybQHuBZSYWJa+KYjVUWCUmBYpSScl8pklDma8we/wBViTSR8kzy08ghicOQTuGe61sTqaXE8PqZ3TyVQicJDnuCdbWusqkyU2Kh0jOdSkgmIEyZCRpp1ReTr8HxqLhttLUUVLHPDK4mbltuWsPUHp8l0sPFkGLl5oa6rpHxXDRNGGiU9hdeQVMRpMSfV4fJLBSySZAwnUdwpqerrZKmmqqh3MpqKfPGD6gdNFMZpJevYbjdfLEPvWRrBLkkp58vrFtQfqulES80wzHqV1VSV9XOBDE7lcstuAD1Xo+GYjT4nStqqY6P911jJBuiNlWmg8psN1oZbpnxXWkpw9Tw9Vhz3U7Ltve3ZWsNpK+GC09PIGgkXsukeMj8qkb6V3jYxwc/PhjHRO8pusWoo5IBnDF2b2Zjlas+swiaqvl9LtwtY7WeLlRIR6k7JrrUqMBbFE4tcc1r3KxBdnlO69TiuRyNvmU5lYRZwBWe2R1rXR3ezdapVj8P8qjkp7nMxJhzKyw5bFZ5CjJEYz5tkIe5pWrJCyWK4WdJEeZaysFJIXZ3DVbFPBG4DO4qhT0QymQvAt0Uwq2xuDN7dQuWa4tVkcTW6NRtIZdZja1ziA12hVllPVTghjwAe64TDoUsjSC3Ms2WIFxIWozCKjMXSPDwfpZWYsGadyFefEZEDTk0COzhuCF0MeDwAghmqm+62KeJC05uMWcNFZa5h+Jar8Ka46tsmGC0++U5u6nOFpQaGuThndTSUnKNgEAjspag5SJkZRBqlaPzK2I+QeyXJd0CtgMsiAH5ViylTlHsp4srRlcpjGCNlXeNfKl8lhYLrtsqLpss2QqcF43Ub2Nc+5UVPGdFPy3ObmUDNBupmy2blWZVE+J35VC9llc9RSliFrqDMc0oMjlce1qjLVgVS1CRZWSxRll1VV0kb48pQWsVEJN5u6WYJZihR1K2PMMyhbsp43WCAsiEjoUeYoXboIsoRhhKLKVNFHcII2t1Uwj9lLyG/CE/LPdbEZagI6FTFqAtVEJHQoS1SFqEjoVpESbKEZakG3U4shyeyViVYZFdO6KypiijGqkyJMb1UlnIAyNSDVLy/ZOIv3UCy6IWs1U/K0RNi62UaRsF1OwAhLI1N5hsoBLQCo5MqJ5JUL/UrDKOQqHMFM8WULtlsNmKMC6rj1KVr7ItjARNHRCHtRB3ZSkEmdumBsk43CnFsJN0+dyiJ6pA6q0ifmHumMqG+iEi6lJY7XN1JGBbdAwaI81jZKTknaz4lO0WChjk0U2Yo0MNJXj/ABViU9XjtXVVMMkbWO5bcwsAwbL15slk0vLlGaSCJ5923UHhXPEx0u4+wurtBhGI4lLyqXDqiV/tGQP1XscVJQteJGYfTtd3EQuocRrJYGlsVm/JOBycngf2dzgc3GKo05ZrHHBJcsPclbE+AysdYcT4l/6if7zqDu8oTPJL8ZW+DnZR4NWbM4rxIfUFTswHE/6vjCtHzATUwkMgyi60I5H31ZZYnFqJVRw7jXw8YVV/cBP/ADe4jG3Fs31jW5ALtBVgbLLbnjgfFP8A/GDh841G/DOLm/s+LY/rEF0b1SmNig5+opOMo/8A9poXf+SFVcOMW/8A1akd84l0EvnUPIJ3Gi2wxOdxhsKqhd82J+Zxn/8A28/RaszRFsqk9Q8Rmzl1FA1HGbfXT0Dv9pN948YW1wmjd8irjJ3uGpuivnWZhLUBinFY9WBQf7MlkX3vxT//AA63/wBRXo8oO6ssdcJSsiTG+IgPNwwPpISoP5w4+D/+qsn8VvZinMrxs6yzQwP50YwPXwzKP1QHiusHr4bnH1K3jPL0ch8VL3Shg/zteNZMFq2t6lSw8Y4LI4Mmjnhd++3RbLZ3u3APzCGWjoatlqqkikzd2hShFT4phVYLU9axzj0U2TW6yazhDDZ/9ELqd57ahZzzxLw5IM58VT/8/wCaDp8iWRUsPxqjxFnkfy3/AJXK+AUA8s90gzXVGnbutMmDEWVvdK9k4c0ndKaIiyHmNHVO86IY8r3LVJaWPM4jTRWWs7pRgWTueGi5KzMFpAbJy9UzWxMOpTira7qrxVbzApZ7Km+qy7FJlRn3TglrBmI2Tia/qKqPmAaSqr57+p5CcGbajpgBoVEZXWve6z28w7OJUctW9htmW/DOTTZUG+pTvxGGIbn6LE8TKT6yizPf8S1xObWdioLbBhTw1XNBOo+azG7q3ANAFOCWtyVkcIzav+SqSV8cgs6JSSMjDSXhUyGg7LMYljNSSLMaFJFBLKLv0CjBY05gEElXMRZp0WqLXW0sXbVGI2t9KzI6qdrr51bjqbi5cpxLTmVwQmocNnKJ8rLbqAkkpRad1Ye6gdKZOuiEhg3QmRg2Wiy5dt0bSwFRGYnfZFE7OcrQFac1xsruilLrhVBI1m7kXiMwtmsszDpChXTvDnBpWXXVswvDHBI8fmK0sRjZm/BYXufuQFg47M+GAsDXBw3BC8ebopVsrpgS54+qzC7KtChpudFz5x5iqeIEmUjIBbsF5ZdIRz11W9gYx/lA2VMvePK8aqYaeXW/ZKJlMZT4twA7Fc7JlWtY3OyCSTmN/B27q9VDCsw5L7nte6jPI5ZytsiKLKWR13ySNAVeQMjeWh4I7q3NltYIAW6MyA+5XSCHsjAL5zsFk1mNSVH9EwmMzTH4hoArtTeaIw3tdPRwR0cIjjave4uYxCOfC8Gr/vJ558rbBpN9VzmB8A58Fmxivhe55aXxsLrW31K6XjDC56uanqmO3cGZei6Rs3g8FayAWcYrNHuUbfNNWHtqpQ8Wdm2XouD8Jvw3hKXEn1sjZqlmkewBv7qlW/Z7jcuPxumjjLal2e+fcL0DiZjhhFJhzMskoDBYHeywPFaXhuurcY+6o8ud5JJGuUKWi4ZkdjxwSvltYEh7Nr2Xf1UTOHcWp66Si5j7WIi3XI4/jBZjBxClp5KeXpnGoWsYbYuMYRDhcxiZVc0jf2Wfms3Kp8Qq6isqXVFRJme/cqrfrdSYDtcWrRwqGlmY58k+V8V7aXQx01ViFK0+H5TB8Q3WhguCVVbM6OgJa9guXOGn6rmwz5pHSuMjze/VKOsdTtsyGP52WjivDtXhRZ4qpjeXm2VguQs+sw+ekjaZ2Zc4u33CAoqueVwuVtSPw5zc4NnDa6xIYZGRtdZO2Rhk5ciy2vzVs8FVAXsAgDvM9ulx7L0qjxWOjc2lOKxtw+pjvG+pkzWPYLzGOWKSidQgXubt9lO6unFG2hkylo7jULdj2vCYDgVBJWvrRM14JzHqPmvN4MKHFNXJO0FjpZCQ6+jfn3WM3F8VqoG0j6uaSNlxlHZW8LgnMrsvPjawXJaljqK52EcD4dUxsnYaqrhsY2jX5/xXIHHqyeiigFIJIYzfNINVBidfVY3WBtfJJI6MWaX7gI8DoaOrrTQzyPjuPLbqeych1PBvEGF0VK+CoitUyuzhrBoQqbMTxTCq+qgpKGJrKhxPNYy72j2CwnTVOB4sQySz4TY6bhdBHx5T0dTHXTYJOXxiwewiyWKtXjOHU7smGyTyzPBDufoSU9FgNfNy6iGppYxUC0kdOfOB3VDEcfosbxqTGosLdEHtAe0i4+a3J6n7owuOuooY5Zfhfa1inEZOEYRhdNxXJg+M3lhHkjcdPPbTRSfaBw5DhLoa+GqjMIPL5NvM0W/iqlbNzo31872Ty1FnvPUfLsrGAVMc9ZHX47iAqhlyQhxLsh+Sljl6qrgqXNMJ1AtayaCohjlZzxdjjY2WlxXDh3ixPhzGR3vmY0WDNrf4o+DKrDIq2qgxblCGWAjNILgFWJBSikMk9c2uiPKb5eoI913OE4xhWI4A6unhhgihADmhwuTbdeTwMdBV+Jg05chIvsVv4fycWxJjX4dJLzb5oInWEh7lQdPQ8S0OM2q2U4a/D5w/lD4o9rr0mR4q4I6+PZ4uB815TiLcGwrFMOZh0LqM8wwVEMu4BtuF6rTwzQ0raSBlooxaM+y2kwirsUw7DZI46uUt5gvfLcD5qaT8ZhZB5s+xOgUFVgcFXPHVVPndGLWOxV2HyxtafV1CK5X7QJ58KwelggqiwzyGKW35CF5/XzTx4VUYZS15loo5b+Rv7T5let4zgdHjJdJXwCQMjIiJ3BXP8M8HyU9C2nq2WYJjIHEAkjoFB5VUyU8bIX0kJidEQCXbn5rTqq98cjHUk2bML5LaLV+0PhurZxFUT0kEZjeAcrOizRg2JYTNTfeFC+DNtc7hc5gZRM00/jco/CeNhay9NNNVUVPh+JYTQyT02IRCSemjOtwNwuDjw59Xi/3NTTRxCd27jovYOEKCuiZAyuZIIooBBDm69CfqpQ4bDeI5cEifIKE1bTIXxy5gHN7scDup6bBa/wC1LG/HPZJSYWAL3N/YgWWHVUeK1eK1k8dIJPM8vaTo0XXrWFQfclHTtoqaQtqIWERsGnMI/hddIlIcZhlPR8Hccugoo7UFO3lua43ykjUr0qrkhrcHqaWhfzObHkbpbdec4/wtinDcdfxPiWIcySpOeOnjBcQSdif8VW4cHFFFR0+K1mImJtXKGUkEjbvkBNr+3+KWtND7TsK8HJgdXS05Lw5kLra3Isu04jon1UdG+N4a+A3serCNVexPBvvCaikn9FMRKRa/4iuywCWeOTJmyDb5/wDsoOTpcNfFRYRyvM3xNnN6XuV1UrfxCqNFBy4RBIw5RK97bi1iCr5F1qBEibsmIsUJc0OsXarInAumqIp5ojHADd51INrBRSSOFsgJ11t2U8DhWQNqoHNIB77LY5p2JUT8ddhsfLNRBfmRyN1Paylxurw4UBgqBlqXAhrGt0F91yfEVScN4wmxuRkcNgyQCUkCQWtZc3XcawVdRLWPgdzpLct7N4wBa2u6wO/bifEmC01O+CghrKEjI1z5cpb2utJpr6jDpK7wkUNaRflsdcLgKDiSlpf6PxFiFbKywMccYzRuBFxe2y1+HuKq2q4h8dUx1EGDPj5cD+SeWD7uAWbFF3EVXHX1YnxWNlUbRhoboCDt/Bc9jVRVS18GI0WOVjpZHcuXmANMB7LR4sjxDh6uEdVDHVtlElRS1cYuXXN9QsuSinFFTTwxvmjqz4hoaLlrxuHLcTSwpV+NYpNzoaqoq5qYWjqAXWDrLM+96d84fG98ALcglB1b9FDimKxTtlySGGWoeBNG1lhYdVTdTzVcT62CkZHBELOc3qB1WpjkiUVz3sfTvtM6c3JI1JUNBWyQVP4ejrEFh3U0FO/EqqLwJMbiLAhXqcXwuqfVUQ8RTyi0rhq7XquUzQ2+G8cxehH3lhVXHJUwG76d7dJB7haOJcayYvRzSfdzoZahxuc2YsBPwrk5sGqsIxQsfM6rzxiR0kDPIM4vqVo0MsMVRC2SQRsZ3KzGS4IcTq31E9LTxSRQiezHZNLH99dTDxTBgLIsLfQUmHh0XLkqINSSb6g/NYeKVuEOcPD0Q8TEbxvYNHH3WJXx4jFUeJq44C2Y2IYbgWScrVuSYjiNJirhNiprYI43xtmmdcNz6gjsf81lmSSrmgpcUncWmJ8ERa3Um/VKmo6Gd0wmqMjI4tIBoQQjppoHYnTR4U0BjIybzjRpsVGWzjposKwnCLxxF7IeWQwWzHS5JVTGJsLOH0k8NLeEkidzdHMeLbq8yWlq8N52IyCNtASMxsY5D8lyXiMPfLJHksyQ3Dr3BPRaaa1RLhtPhzoaKaphlZIJImSG7JhdHgVZHRNfVSVfJrGO0L9QR8lnNLYGx01XStyi5brcjsQo6unZzIw/XmC6DpZ2SYlGamtr4HAuMgYwWAJtf+5ZcuHwRRRYiyWRzJATYmxD7qjJDPBTyihfaIjOGdCtrCI456KGkkjnmfTHmSsJsJBe9lzlDCrf5gx8botX2d10C9U4Hl+6+HPxRcCUctw25bwCP7yvPMUxPCsTxJjqfBORFERka0AG/Y207LsuCKmrramOiqoDFDUZ5Gxn4bf8hbwkd+JB1RucLLPqmSUzzlebKaKVr2LtSAlLjJfIbKzELi5QFyISBgVBZAH5mqN5Aci5rSoX6uzLWKTDH4hqDGG5NARquWm1kL+66nHKV9RTvazcNXMPppWH0r36fRwyQte5rrhSvldJ6lGA7sUV8p1BXWXJYhN3K4xt2hZ8TrWKtRTZrBzrLEw0ssOiimiJOdgUsMsY0JTuhu7OzZYaVXzyAFnQiyCLUqWWKx1amaBfytRlNTl7JA9q3aSVtwViQtOcLThdYLlk3g2wQWiyljb7KhTS7XKvtmZlXnl1WWDRH5eyqc7s5OJXd1im1ktBNkJaN0zZLo8zUoQvhY8elVZYQPKArhIsoJDcrVpSsIG22QOisdFZJI0UL3Ks5Aa2xCm2F1G1wujz6ZUoLmdENruzJk2YI0IscUBFipOZYKEnMUBZtE4N1EpG7rNkTaYGwQF5vlTZjlUbvUqCcULrJG/RMWrFKA6p7NKZExURuj02UXKVh2yTRqsCq6L2UTmlX3RtIUJi12QVg111M0GyMRfuowwBVUaSkylNk9lECN1NGUAjU8cdkFlh0QlqkZH5UEg0WxEojupHbKN26CN3qSyhGRdLJ7LSBDAUYh7NRRtubK3HG34lRHFEB8Kn5LCPMEWRoSs3sonFXMLBo1qdkTPUp+Ww6pCNoWRHygfhRCH2UoHQI27o0gLSEm7FTvYMuyi8oKWGQEXUhaoHXvZq0iN/pUMhCsFl/MVFJEbqsoTqFC4fCpnMIUdjutQIi2wuhc8fmUj/AEKjI6zvmtRFgnVBB8qmbUNso2xi10xaGrVMcllst1IZbqiZrdUDqm3xJ4ZyW5ZbC6gNSB6nKpLVuPxKs6d56rca0nJu09UyQANupHSBozWWBFVviOhVoVhlZunA5tEVeqczlzvKqUT2huqtUZZIS4LEw1a5A4kaqyCoQA21+qnita7lzlvkMNejDT1QmRg0zIDNb4ljzRZACqVlNHKC3LqpBPf4kxkHxG6eYxpcPeTly6KWGjY212FaWe/whFE1vVq1aUho6Y8y2Sy0TTNaL5UULGsAc1SlYu24QtB+iInqUkllTk3UUsPNCkRAqjOfRyewVapL42EXWu5uYZVQqKBrC4mUlbhhgT1EmfVygdM548y05sPaTmKqzYf5fIQF6MZclQS5dUvFdknQPabKExPvotMpM73v9StU8rmOytUMULrXyqxDGM2qzLcLYNxdIjoUk7t1zl0Blb3TcpifUFLUFZQ2QjZOiTG3VFDnN0YcHjI9oIPdB1+iTdkGJi/C3MvVYS/kzb5b6EqPBuIJIpfurFWFj2bErouZZZeN4TFiUBlDA2aPztICMtVzdbjUJtgudwDHJGOOF4h+0YbAk6rde+620N8jO6TBnPlVN/50zJ3t2K1TlaasdI0BrdFXjmfnGYpnPe45nuuoXPyPWqS3QwzR8sXKiqqmHKW57LG8U/oVC+QyOzPN1fDW000ueUkHRTRiQMFlXhYx7hoteno+Y2zjZSahIVAX7OUwdYIqmB8ILllzTPzbpFSi5NVs1DXKlJO/Ne6hLwmc5dIxS1hlZMBYFMXOd6lBfKn5t1S0oOUq3TljyGKhmLkTHHopK21TJFD5AGo4JPNmedFVp6J7vO8EgK/HSlzQ1oIv1XGWgzTxEWuqT5M2y02YIx37SZx+qL+b0F/2jv1WeQxi9xRN1WhV4dBTNuy5KouyALfIRvNgo2NnnfZkgA91YY2F4zT6M6rCxLEjSVJZSejvdZ5Doo4gzR5Vjk/h5xsuU+/pdBJWR37LVgxeqkpmx0TWyHrdc/Eb4p6yaOBuZ5sqjK6lf/WgfNVOIqqWKh5lVFJHINtFjUD3Tx890eVp6uCz4hxdMKmneNJQVBFiHNqeVH6fzLEfPzc0MbDI3qAtrDKKOOJs0jxFbcFbx2Wk4tiOIkBzgoKiqpac+d1/ZVJsfgprgPEtvyrlpq2rqJTM829rrGe6Iap1NZxFBRsD46TM49b7LCq8Ypqoky0sl3dVkS4pIbx3b8yVUdXPOjXg/JeXPdzXi6OCJkjM7BZYWJ1ckNQWZC75BPBide1nLEmRrdlQmmnfVGaZ1+y5z+ZuJNnqJXZmU7/0SdhmIzHmeElDO5C0YeIqulZkhhhYP7KiqOJ8SnaY3uYG9gFfDW1PkeHF5GOa4fmUbpsxQvmfPq86qK4HlKtFprtPqNk1mHrdQHINTe6QltsrRb2QtRN3SdG4p2w6L2uJENkFntBHYoXxMe0MOwUhjI6J2seT6SpK2ijpIA4PlcAB+YrBnrsPnxGaeBk9XyxfMwXDSp+JI6yrrqTDs7mQv3sgq6nAcDonwQV9PSPtYErNK8wqMVE2MzyT4k8ZHvAB7XVKSTh2omdJiVdLnPQkmyq1MtPU47JAytjfEZH+cbHVFjOD0MU4FA90bmixtoCsy1CkKKgmxdscPMdRn82iKLBY5sSdBC+0I2DjuoXmriGSR6s0X9Ea6qklLnvFhfosK25cWwfD2NpapksnsxS/zsa4NpsHohSR/mIuuTllM8hkkOqsUkrHAtZ6gLgLFI1IW+KrTUTzmXKSDfos/GsQfXVJje67GdEWBl+eTmC2cE2WXUtmiqXvkjLL90xgaPiIzTsaTqCqc0jhMZGC4KhhkDmmN/r+EopZZY2tzjTutUpjVcuXPmspW1NdbxYg5kR62VmalZSOiqxEJY36OB00V2rxSAU8dDg7LtIuQ4bKgIIaullZzWcvOLj3CtDFa2lcWwSWvvomreK2VdOykrsKjhfFbzxjUqlDWQVpdHT00jja4KwOy8Jh89HS1tW/lPqB8I3PZc/jFMcHxWKSN5yscHi/RT0WJE4RDDO78SCXy/IIOKP6XXR+b1RgrIt8TU8dfURYqRzIpY9eiwTVOy+EphmZ0Y43Wjhk7+Q7DavVgtkPYLMrcKko61ow7zB4uOi1YsNpquKkfyIWRZ2kEjsnwLEn1s8eG4piLYaQ7l40Q4/ic0VK2kjjyS5fMLLMogzPHLLGHMjeHuDhoQFuB0dHhtLPjEmGiqM8Ed+XIDvZTyU/DssMkeJ10dNLASLRA3d+iRqcHxCoczhx8dGyOPmOdKMpv2BW9gvBOFYxw01+JY7SUby593tsSfmrMDz+ofzaFz4IbQg2Dibn6rOY6ymq2PwrEKqhocRE8TJCBI34h39lGxumqlC3TtjlYT2F1p8GTFuOxT7viBLG9CeyzYWthpXSKaCGl+7pK2GXlV8BvlB3+Sg0cdosVxDG5H1VHeWXUFmxC7fhzjmc8PjC6mrbT11NdnNk0BAOn8Fz8vDnEEOHwcTQV0eJuewSSRA3Nj03WZwjNHXVOIYdUsBEnnjzbAhUencLYnjONQ1Bqa6IOp5OXrF6vfddBlIdquX4YhwIU8vIqJIXnyyMZtfuugiL6aoNPA+KsZYEtb+0aFqxayhE0qJtTDLIYWtMTwPRJoVNkcN1RGIaWB5qvDCSV27iLkrExvhmDFcIqK/FjKZommSOzrWK6BFbmQvhPpcLFZmB5fwzh0OLYXNS1UAZiNJIyphedCWXXrlM4PpIJPzsBXJ8P0NqWoJ8k1PJy43DdoXQYZJN4fkSixiNlaGTR8OPhnxOaGG/jnBg6AC5J1XWMiZEI2x7MaAP0VZmmqsNk2SiEz4YpwOcwSAfC4XB+iy8TwdmJVEJlpQWw6xgaBpWox11JHZSlQRQcgPbDmc55vYlUpCJaV8NPHzH35ZLpbFveylxLEJqeVsMFLHJY53O2Nvmud4qxHC8KfRV9JHrzrT5ZbEAjc3SkXqGoq8F5cdXVjEYJH5GkuN4z37HdauG1v3iJ6htuSJHBsoFhZcphE+FYxhsWGPxCCjqHuMlPlnbI762O67DB434ThtFh1mWJIlcRoCkCOqEcM4fJJkvoARcEqiKmlkzyeNjeGGzsvQrZxfCWV8Hh31LBn6NOp+S5maOgwrEMN4cpj5pHEyE/LqoLslZNFA5k9LM2F4I58bgbA7FY1RU4th/DlTNJJHQUcDg8nMeZIPZZWO41VwVUFM+ohpH0mIcrk30DAL7dQQVW484jpMbw/wNDWEPLgGs5ZDZHnTc7f8ABBQpaykxGnrMZxHFIsQeYnxxCUXkaO1lzHD9Xw/TYjFPjMLpKUghzXN0VWWurcAqmwQOhnmYLuuNGfIqDE5aIy09TQ0khiIJc3Xf5rGQ7PGIuHMOlgnw+hlElSwvgn8QHA9dQemqoVPHePxwRMhHhZGN5ZdHKOW5nYx7LJxnD6qjw3Dp66ePLLETC3sNNLrOmhqKfKKmF4bILgHqFyHpUuIGlw9kscOHuaYRK2Z5JLJDuANgud4gx/7yw+iq4JzBPYsn5ZsWv7/Wy5oVFK1rWOhk5bDc37q5VYTA6CKekrvwz6WuG11eTTDxekqyI5Kl+ci9u5Giy/ESQA8p5GcZCOhC6uGknxHC3QSTnPGfMLXfGO4WTU8Pvw/lYgyqE0BvZ2XKQ8Dst47CYFS4RXwRxT0VWI8tn80bALopsNmqmx1kOKxtZUGzmbB3f6rmqfE6qoo30ueOPI+99rrVq3U1K+igw/FPFUzyC6B4sY5D9NQkxbLRjxSopcOxOhwiSJjCAZi4XuPYrnnTRVckeZl3MN7jayiex5rJhIXtaSRcHqr9BXQx4JU4PNCxtryRPG9uylNYjqWPbQS1MAB5ZsQR0URxCllw2KnfAZZdeYRsB/iq0bXyVBJkcyLl5Mo2d3ujimMFI6kqYyIXm5tqsNSjjnfBmyekiwCa2aKKtMcjWD9m69rlSYi3D6dj4WySRTMIIY4XYR7FQU08EkU7HVQhANwwjQ/LstsrlaPFU7AY5M+kkkDD5B7qjUBkLmyQstTPs9vcHsVajrJjTifxZZJl5YsNh/ismRk2UsErnMPdbGo+r8a8T1dOGxsGTQ/op6Wlp3YdUYiWVBNMWAMLtCHnf22WS1+awkbmbYCy1hLFJgFVDBi3KkNs1K7YsBuACkiOnqoDKI5nyEn0u3yHuukw+ngdLVRSSZZZIg+LILAhYVJQ0WKxSMM76flhgL8twO6esENJUMhpK6WbltH4h0H09lxmBuUWDVM2IMZREzwwBk9VJsGi/f3XtlI/hvDvCMZDN4kRmSOPlEyXfqSemvzXj8ONZeZNSVLS6WkEVXTlt8wA6Hv1WnwPjGPwcR01FDWyiKphJyuNw3313stY/kR6xVRMmJv16dQq7afl+UI8Iqm4rSHEHDLKZHxytvpdhtcfNXjCvRGdoomN6jkL2g2F7K85mqiexW0ZYqnje4PyRipep5adjhayrCAscRqt2oJJ3SAsc23sqE9ILF+VaDoLedHHlDvMB9VvDPixPm5ZzmNdmc1GIxWNLIIXPcOy6Oswehr9ZIrO7hFT4ZS0jA2Npv3XbxnGnHPY+NxY8WLeibNfZdBxFh/Mca2P1H1e5XOjQ2XbHLkxksNkd1cr1LVtB5b/AItiszW104PZbmCJadVE8PvYlpCh5dilT1tmiOR1tLXU9wVyU0brODVc5irtawG6OVuYLGTUSuQ1AaMt1YjqrHVyy2xvAzKVjJy4AAlc5huJbLJ22Dro2ThUg0sZqiZIR6lypuJaDKgA3aURqr9VSDg9ugUTpMjsqUttPxFx6kIcCqjJCVKw3UpUr+qhkGildsgJAHmQRaAJ82VIviSuw6ZlpkPMb3QulbfdRlgCrm7XZUptbMo7pCQKtmP5lI3KbXVmGE4OqlBuoibIQ8BYaWbCyidoUPMcnF3HzKAkztkQF0eRtllVcjVMDZFINUKAkmjVCpW5barAEm6fJfZAT5lK30oGDbDZC+O3wqVpBSOUhVURGiFo1UrvShbuoiUNbdSMY26Bh1UocgsADKo5Q0NRByjletio7cKNSHdRoCDdVJk9lGGqQbrSDjjcNVM029SjBNkTdkEjpgG7Jmm4zJIm7IJWiwTpA3SWVJO3dEmDkBX0sopGEm6kScPKgi3FkIide7mowLFSbhBCYv3UuUFMhdugqPgH5VUqIngelaZCo1Zc06LriwzZGyEWKpVDST5Velk0NlUkBvddcZYyQ82aNuUOUZnc71FSO2VeRov5V2ZmTulceqAlyEnVIG6tMWRd2QZSjckALLQhtqrETgLNURGqYGxUlLX2DmW84CuUgja7IJB76rGc64TNflCzOLUS6kyM6PBCjlquW02WNFWG2W+ikNSSN1jivJa+8XuOWxUrHuks9xKoQed61GtGQZVmYbgcLnh/qKkN3kZnEWPRVC4tOhUkL3E3K50trrHtuGhWms0VFh1zK1HIViW8Vxp+FSKnzdfUj5ju65tJiWpi5oCrGUuPqTZiUoTOlskyW5UBv0Ub3OadFaRedLpo5VZptS5yiM7u6hfI4pQZ5uo5G3GqJzlHLMxoOZy6RDnKvJCx4yqHw0bdRe6d87b5U2e66wxMGdohabFO510OxVFpp0CJqhY82UoPULlMNwROUps7knepM/ZWmjc1yXNKiJ1SzpUs2kzBNfqhD22T7KUTKTNcJDcqPP7oZJbK0jI4jwgF4xKlb52G5C0MNrGVtFHI0eYCxRyvEsL43fELLGwGcUlZNS3swmwC3xGzN+W6iMb3egZlcLGE6qaKFhVYmGY6nmG4UMkb+oW4YmW9Kry0wJ9Kcl4ssRlCWO+Fq0DS66NTima0ajVdOZxPQQAkHTTutqPltYLLGhJiOa6mOIa5G6e655Rax+VZxIh8Js7Vc3IbvcFpvnc59nPugcyN1rsF0x/KzPmzdeyDM9bfgYsoysS8DCHXLALLfOGeLJENQ7VsZIUkWGV8ozmLlt9ytyFrWu8isshfI5Znb8GvDc2MKxCxf4Y5Qd7qaloakyhphJPVdSxjstkXh29ljxJa4KNNC+IZXtsVI+SynfG0Dyqu7ZcZlqhNmfdW2+YLN5mqnime5tsykrEDqjEGEyLArMQoackyaAdVexKppc5hMcl+rly/EFJScl1VA57z1ba9lnkvFDi1bPiDcmHsLWdysCeScO5FVOPcBWaGWpkYGQvsTt2WfiFGw1ZjFRI55+i5cjitmPDooRCJgXHstnAsQqaLSih5ut8ztLLm48NmppfxI7/VTw0eKSyBkLjGw7WUjJt6DiVdBPhsme0rwLhoF7Lzo4lVStcwus4GxC6KWDFcNw2zHXz7kjVc/AKWMOM7fM83utWL+A4lHSSmLmWfIb2V3HZqp0JdG8a7gG6559VRU0geHrSpcWwxzMxf9EsUIpJw3K+4+aOsilpoC90gJPZWZqvDqp2eOQAe6y66t5ufKwhvQnqucxaMue+bU3TxtYD5dT2UbjnN1Zw/kGTlztP0WFTBs3L5gba3VQyTPcfVdaVUaSnYQGk+11lSTszEiMBCgvebEZSFCzPfpb5p56l77safKq+ctOi2LjZCPK0XKB4eNXCyriSS485CkLJZSAx5kPVagNzETTf5IHxOjOV+6dp+FQep1mLcUiImjw2lH717qOjxfi311eGxTX2Y3T+N0n41j8MTjVYGA3/+nkzD+5ZMP2hQmokgqObBHF6szV72GzUYjxpPfwmBwQds0l7LDxqo48oKUV1XXRwNvkLWNBAKxuJftaqXtdQ8PxPjH+vkGv6WXDy4zjta4PrsSqJ7G9i7RYmWXodZNx7NhZM1bQ2d8bgQ9vyK80lrMcxCpdQyMNbKDbN+dbE/FGL1tK6ifiHJjDdR3ssDDsVrMLlM1FNlcRlNxcFYtpewPCZDXuNXBl5F9L6haE1dSSVDw0Xt1WdBjVbK+XPq6bctGyoSSviksxcp824acjoKl5e8ggHRU62QWEYJLUjOxpaSADbUdFC4cyXmNN2dQVhrJG8HKhidLBJnU7QKqqjpIW3zmxV3GsPpsNgjig0ed10hmmWZiXZoSc3cC6nqHGVoEjvP1W62CKnwiN8LGh1r66qhPQM5TaiRpsfjYbgrMFMaRj4jmGituifV0TsrdYyuw4dhoZaKqDhHeP0ueOhWNBDTvilZFlzPf5i03BVEEL467CSJJ2RuiIBBVc0tBFSRTx4iTPmAcAOilw7EaijbJRSQQmnLvM5zdlFIyNuJRsmhfHFmGh7IOtx/hrAqyjp5uHpxUVtrloJOcdlz1RijBLBA7DTA+I2cALaraNVVYJUHFMNMdO2nb9Hrl6vFazEq52IVDxnfJnNhZamBNiMj46osjdYWvZaUsorIqWZvqYMhWFIZquvjZ65JCGAdyuoxPAuJcLwp0j8KbaP42SA2+YXKcBAY+dYRxhs3WyZslRRS3rblo/Z2GxUBxCakpYpZIY3uk0Lmu1B/xULa2pq3tZAwOIN7EKUCr6WpdQT1czCH3uCQo4ua+hZzmx5rfD1CuYtxFUP5DMkehs5tlZraKkw2duKwx3ilHovYXShlvpMDkrYGS1LoIpf2haL2PddLPhXBuGUX+i89n5+ablchLEyWV0jS6POb2adAr+KAR4dDGxko1td43WokQUOBjF6monoY/D0TDfO/YKjUMZTTugZMH5LahauGR45hVGK+kma6mqSDLF1OvVLiSs+8nslZQRwOjGuU+pWxWq6fJBT/AIoIIsQFGyhkztdyxI3MLta7UhTxwunoYbdyStKSiZVYZDURSFssRIsOqzYlyYrgcjvu2CsNGYwZMsnpv1BCpQV9ZS1wrMEpzHnbkLjFcP8Ac3W3wbjclM2fDq9sTYHHPzJ3WYBrcHTqqePRYTC7PhOLkRXt4cdPl7LpYpw4/wARYLWSVvMEXif2lmix+S7/AIErMNbAcSwp/iK6X9q2pmHMsvJ6x5Mh/HfKzpnNyo6Wsp2vbYVEMrdpGPLVB75NxRhuJ1TsNjZKJad9pjyrhp7AhXpXT0zXuwuOWoA3Bs4ELzX7Ksew7CMUrnYhjEMTKhrPLNqZDc31XqckeHYhAa7h7EgWsF3ch1wB/h1WxWbjLHP5dTQT0jztztAfqrYll/1f8VSosfwbFo/u+fEIJns8jg7Q3+StYnPS4JTisqJ2x0zDZwcLk/JAceePmEQeaU3cb7lRTDGOUfu50ML+ufXMFkQ/aDw/UPy00GIzN/NHSlw/vWpT8UYHK8M51Qxx2D6chBFR8RVUN/vSOHlXyeIgddjSOjuy024tnp/vGMwy0bG3c5pubfJZ8lFBR46ySGMeFxNpEkRFxnHX+KHAKRmFYxX4U0uNNI0SxA9BqgsTcWMpWxVYojVUEmjqmm15R/eHRdFFLHPE2aB+ZkgBae4XHjDIcAx6F9E8toMVkdFJD0jl6Ee267CITsbklDRbssSssrGcHxeopJ5MKjp5ah9mNbJLlAHUryriajnwmtFPxHBAyV4P7Co53NHb2Xr2J46/DI70T6d0rdxMbMA9yvLOLOIOHMUrZn+Fglrn2/pLX5omkdr6rKOOpPHitlmw3DOTHMAMjjlLfl7r0zB/tIwrDGwU8MpkhfHaaCckmGQdjud1522qlgdVNqsRjzmz2kahw9itN9PHxC7wcccAdEAXTQwk69iQpEjs8R+0GKrmpakPpqpkBzkQOMckY7WUuHY9hGO44eI54XQtgN+ZMQADay42jxGkw6qq6eTB4qt0rQxri2/L03/4qtxZQ4dhsVPLBBlfUi5bfRw+S1ZZuKuIqDFMffiGKcmseJMjfBnKx7GWsSVk4pjjMQcORQxwU7B+DCXXDffbdQwwYRI3PDAYvLkN9ACs2WjronRcyknjZK4CN5jIEg7i+6zEho8klVnqpHf2rrQw/Gp6Nsj/AFhvki1togx7DaSjq44IZOWHgXLje5VZ1PTw1ponzSNDdi1t1JFjEqiuxCippqg3jjfYM6jVGat9fTOmayYcu3KF76KrSVjI3zMfP+CBbUaFJ7TGR921dmHcM2usTAFlTMHDQO1tqrEs8ETmw6ll72CqRse2XV+Vh9RSll5kojgcXuB8rgOiUtNOKrfTODKKeSN8hAOU20Rnwxl5dXJK/wDKR3VKeeCsbG/O0SgWksLC6KOGN1PIJHgy+tpvpb37KRFKjnhpMPbJHVU5lll2sQNFWgEk781/LF5wFJN4LwrpjBPIbZW62ynuqsLjkLxoOi6wLU8Vc95eY7veSQL6kfJKSjr4qdlXJG3wzxfOH7KamqWUVIK8NjdUsl/DfuWjt8lTNXOaOeB8DSyql5tw4jVRkcU8gvHf9n5/cqWWshqKRzb2e74UFA6rpsSp53vbmfZhDTclh0stSphgrqmoY+PysALMptfupMNcmNXOlqpYnHU8sAlUHMeHBzh5SdCterfhkFS5zaESsZ6QTumiwmlxjK+kkFLf4bFzf1W+QqTC0YLTuQAFFIzluMbj5x0V+t4dxWjtLbxEAFuZHrb5joqEMLJibycp3wZwQCey3dixTT3LKX0OebBxGhKu09PaOalkYC/ex6HqFWq4fBuhop2edlpSQdR7I/FTVda6cuA5nkyhtgPf5rEi5RQMkeS++RnrANlenY90LY3wRxwsvynkXsPdZwDo3OfEy+tjrbVTTT1cbIw9hAmF2nofkuPIbdBQQupIo/CmSWfax3XQ8I08ja90sMXiJcLb+FlksQTuT3+S5PDMTDY4mSVRpbm0bzHmt3Nlq0+B4lRVTY4MSjkirDdtRA4jOPl0WLHocH2kUWGz0tC3ChCyonDauV51Dz1AXSY7j0eGFo5zYufbktcQC4dT7BeO45iMFfUSwzUsmYRgNf1cQqdVX1uKPikxKQzQMi5UDibGMDsu0Zsvb6LE81E+qq8rcjS6wfckfJWKKpZWsc4CxYbOF9juvG6CqNFBGI6uatfNEeTE7ymK/r6/NdpwHiP3g2oqKR0r3AMjmiIuXdiD9N10jJadoWg9FFJEM91atcagg9QeijczVdGaVnxgj0qq6Ih5stENBQPjuVqJVnFl/iISkmcxoClnppPUwqhNzWOs69u63EsSsgCogySC4WBiOHsjfniZ11W5E91vZVZ4nyyluW7eq64yyznU0b4DZmvdUm0c0j8gGpW9JTtFg0adU0MQE2Vkey34jPFlfdUgGsrSewUPJqY3lrOi6VtG2aM2G6kgwxrG2eFOZwYlOyeRjTlLwToQtqKggks57b2VhlNE0ZWjQdE4AY642XOc7biDGhhkf1A7KWKghadLoeaQiZWMJsCudt0KWBgFsqzah7GXZfVaGdz3eYrOr6Fz5AYHEEnVXFiVZtTJEcuZRT1kufNmUs0Ip5BzGqvVwsmZen1PZejyQ8GIkO1crzMSj/OudddryDuEuZZa4Qxzl1ceIskOXUI5JBI3RcxFUzNFrq9BiL2kBxXOdTUZLkjfMXI43iMepMCJW6boBHJE65Giw2nfJIW52C6qifmPIIIQ+KJ8ibI31K0lileARqibUCwa4qu9wLSoxJ7LVM21GzXaCkJCFUY92TLoiY57T5liYW10FqlBCqRStf8AEpg9c5h0SmTzqUF5VfrdW4g0tC5qgkCjVmRmqgc119kAqSNAGuRjQIAeNUg8joifrogLVKBscFIDdQBtuiNtipQkdsgDdUYN0Td1AmmxRhyDKU2oQT5go3OumJ6lA7ZbDHdMnAunyhAKNiYNRAWWxK3ZEmBujAUA50uYOyZzQAojJYqoscz95PzfcKpzXImvc5KFkOuUdyoYtCLqyQMuZqinbJYjVHnBCpFzg7VOxzyfKVKRbBHRHmCrglFdwShK7dMozJ7oc7Soo3arOrTqVezBV6mJjvhW4RiPbcpWuFf8IwbtUckTBcWXWJc5hkzDUqu5tzutJ0QzeYaKJ0MdzpqusZOcwo8nS+YfJRuFjlU8zbHyqAi66MmzFOHtuoybpA6rQmcWnZQlpBUsMoaTmF0pXhxGllKESSd26HMFULzdlLHJpuoswSzBBbhkIdo5X2VjQ0XcsYFw1CkEju6xMLEtbntcfK5WYC0jMFixSOa5XqapcXBi5zDUS1mF11IJLCyrskBGhSzkarjlDra211yjDlSjncD5lJzw3qs0trV2p2lqpeLYOqd05e2zClHJYmqIYx5nKv4kSek6KuKYF+d9yVMyG4uBZVDOJumLtFOIbDUKGSJxPZFVZ3PLbM3VR8dVlzEC3zWgaYuPrsiFOwDKSStsMYOkc6wuVcpqR8gzSvt+7ZaBgjA8sYCqzzcq7GDXutRNgZYMo8gVfKSdAUbaqQ7p3yvVZRgEbhHp3KiNyms87JS2m5jG7lRvqmfCVEYT1Ub2NC1xLTc9zkJlHVQlzuqmggfI4OskwzaRlzs0qUtfbUK7DS5GeVqd9I5wtmAXKZbZ5uFC7MVoOoJLFzXgqqIzey3EpKNrT8Swq1j6PFYpmizc2q6QQ6rM4npn+EjnZ6mOW+SNNr2kAjqrMLlXwWJlVh0UoIc62q0m07WDZcZyaRF2iidurL2NAVd2yNnGW1mqCW4Kk2CCTzIwqyyj0NUQ3VgQMz3fqgljAPlaukMnZE291NE271XipJpj5XkN7q9TURg9LyfmsymKYNAaoy1znZWturTYb7qeOENsVymXSleClcPOWG6us0CWZoabqB0rfhWbaTc3sU/OPdVuZqkZdFmxLJLc+yryEXzIc990twqInZUxmETS4mwUoiunlp4ch8QSG+yyOar8Vw8Sul5/0smPFWH1NK+jpMOfM94t6fIqOOjA43GCkY57z17LlpZpoAaeGQtaOy5zLaxFiElJVPhPLgaxxuqVTVsNUZ2Scw/mtZUKh5L3dfc7qEOytWZRuUWLkVDefB4kDZh2utZ/HL2yiNuGxj3yqHDKWlw3DxWzsAk631WHWVlJK91RBYSHsFYV6dhtTNOxgqI2kO6ELK4ywvDXQmeNgieDclo0U/C9XQMom1NXiALz0JH+ajx/EKfFWmmp9QffdaHnEMD66rEFOS5v5iVsDA5qR2SRpk9wFsx4bNRUpnFLFFbqSsaPiKujqHESCRvUELjmgJy+J1mUr9Oqz6uaZ8dpGEW2BC06nHq2d4fyI2gfVBJNHXgSVU0bbbC1lmJHOkvYeymp3SF1wrE+GxmQ5KoOA6gKIRcg5GyZvey1cKaaaTZxUZfnCOdkhFyoLgbpEgiwZRqN0N2MGjQUBLCfSh9WysQJOSZfMCmzGLyNeR8k1n9yE/XzKgc5PqSJ7bpX/RNnZe91oT13GPFEjHMmn8LG/owrEosTrqWqz0T8z5dPPqL91nslfK8MqZSWdSVDNM6J58O/Qekr0cmGriMeKRVDpZ4I/PuQLAfRV6LGmUXMMtNzXyCwPZTubVy0LXzSkk91izQ5XkFyzyEs1SJiS1trnZRZhZR3AOVEQ4NBUmGhwyGN+cX09kQkzSXykuQxVT4muYD5SpsME8lVeENuOhRZRTTFztiHBNTyva2Rt9XDRKr55qZHThodf4VGNFKIlpcPxiSodM4eZmyhxCtlrax4kdow2TYbUmjl5jW3BFrJqrkukfPG22c3spStSR3iII+W64j3Csfekc8vhHQ5aY/lCx6SZ0VntdupY2CSQ2O6UUXPhjMlOx5kjvYG9kEJeWtgjAH4mhHQqtUsfDNZ5B9wrMEUkD45JHBo312WaRfrKiow7Dn0HIbJzDfODcKTH6qCqwzDpIJhIWRAG3QrNr66eqqHPke0t6FqFlFVGmdVCO8Y99SqNzEcdw1/CtNhwdzKgD8Q9QLrnYpG8tqgeQfkkDayti1O+aB4mZ5XsN2nsV7bw1PieL8NR1D8WgncQRK4x3Mft72Xhs8z5RmPawWnTzVGHwc3DccdA2dv4jGusCkixxR4KkqvBUNV4pgcTzhoDr2UNDUMpaYTsk8562We1tPmvO53u7OjlEBBipHl5HdKBZZKzE2R78yUaLreKBBSw0NEx92AWv1Oy53DKZwr6N8jnMvKLkOWhxLGz70EbJJZNnkv2v7KTApyx8txA6KKWqqnxh8xdK0dCVNC5klbGZjaN5sVDidL931ZiaZCwgFpzLNCeixOWa1K67YRswO1v81oVEL6ylbyWZpRcXO5HZUMHwqOtD52z8p0ZA+aYO5/MjadGKDUoaGio6eL76mkiufQw6ge6nxdtNQzRVFLLelnivH57/8AO6zqHD543tqaeldOegdqFPW0njacSimEEjv6vay5iGnxClhEsE9LzW1DclzsFpVfDDORDnxiCFkoADeSQb/NYc4nw98Lqml8tyQ12zlo1NBX8SUE2NQYhTtZSP0pXGz2/JdIgYbsOfDUOjqT5GEgvabhUWeaUwxNc4+wWy6mnoaim8XMC2cXJCah8PC5woJAKmWSwc7QELpAo0DYJalsFRox+5G4XZ8PZcLixGuweomifRAP1dcSg9HtXPRcP1slfAS6NjHuyOewXDb9Suiw2bA+FuJa3DsRrebDPCBHUO1ANj/mtCSs414exiohNbgYjmkIZKWGxJ7ghanCtVhMFbPhWM4xU0Zik/o7KrzREd7HqvO6OhZWYw6FjOc0Sm3S4utjiOvpMQyQPoyJ4m5DINyOinMdZjPEOP8ADuJSuoq5sTBbk8po5c7O/uvQsK4kwvGMNZ42WOnfI20jXaa9/deSYjgsEXC9DxNSumGQsgmpX6gHuFucHV7MSpjhWLxRCQO5tOXH1AnUEjrZZ5DrMRkxXDqWGbD5I5W084uXDMADsR/BT4NxRQVmKeDxKDwWIxNLHNcTY/5fVcnWYRiEfEdRBw1M6FjGCXllxMcltci1qPizAsQx2kkqoAX1kfhqxro7hsgNxr9VuPzDupKRkuVs0YcGPDh8+6stDuqaCn8KwQ8zOBsfZWGtB9SwOU4xoKtlBUz02NS0wnGTlRxss5+wuTrbZee0fDdNwYykxHH5oXCouyVj478vtlXtlTR09TERNDG/KDkzNvY915zxdwhT4RSR4xOHzsiEhlD5MzXG2gsdlKIl5lWRw4u6vrMOoY4qSN3lzE3I+uyucM8QSYNgNRS0FSYXvlvI9rMz7dr9FXFXS/dT3xxBsVXI8OadRH8k3C+LUODmTxtJPUvqCDGIWgk2+aUOrwh01PFRVVVBTc7EoXsa2W4EtjsbbLmuLaKqwvE3xYjIWcrK/wAO51zY7201W9xZxphVbQB8mB1UNZCLUj5HABp7+/RO4cSfaxgWHyOp6Roo5SyWfRtz1WUx9WBSU0NK1vEdbSQupg6/hXSFoksdCtPHOJ2cTZKN+FeHe/JyA+e7GD92w0VnF6fEahxpsGgFZRQRhjGxs5gMg6WO2t1yFG2TBcdpK7HIJaflOOaN0Zu36dEpRVsniIuRU0NnSOyNncSRoeimoXfdUpxClqI5+RHYte3MFbe3DcaZV19BRSl9NeVpqXaOHsOgWJNXyTskYAGmQWs0aKiTEKl9fWSVr6eGJ0triL0aDoooJWNc9knQXFlHFDMzC3VGUHlG1j1WjwhRjiSvNLOC1lJEZrsZckjYEdVmhmCCSe7xJmbJ6WtcASfdRxCejmM0Y/FgJXrk32cYJG+DHeI5aSGWtb5qdhLRf909F5hWwx4ZiVS+CFwgjlfGM0hJPulFgq56OolZXRgxPkNpWHYFAyspWxPa9zYHEWElrgfNRSGhmlZHAHsElrXGxVatoqmhl8NPI0ydh0W6Fumnhpq0wVE/MpXizja11PVvopYoI6Kh5kUT9XuPU/3qjSwYRV4iymrZ5IYiLBzTs/3KgePD1/Lgn5jIprg3uHDRXiL9ZR1ENEK59C6OmncWREdSFC7l1Ihkj58rACDGzoVtHi2qknpJJzF4rD6jmRQZQY5Qd7/QfxVvGeJcArKCeupKUUOMvqmSmFrS0W+Q0SIGDTyMw/EWVrInODG6tf7p53cxzywnJL5wCb29kMlTV19KJaqXM/pcKpWBsUoYzRpFwFJhYXajwuQQUkIDwLZnakp5carZY5HsMUPLsDG0W0HVHR1MEmGzyTvHNgeACW/BbdQGCSWnFU6NuU6lw7d1mkaOEVcta6R1I5zZGAaONhKOyrY8yohlYytqJHCaMSBj2gGM7dFHh0U8Vc3kyCN8fnaTsR81ao62gm4kw+q4mmLsOEv47Te+S3T+C1AHDuHq3GcKNdSszmKcw5nu0AAvb9Af0QQwUxaHxuA+ZXacTxUXBkYwvhiu5+F47TeKBlP7PpcH3B2XCCNjKlrKWSSeK9muaNST2UlE1VOWMyNy69jdatU1lXw9QwQzM5kBMg817g7qjihZK5lM2iljkp7slzAEk6LRhjw6j4OdUNnikrKioYHBjrmNl/jHTb+KzODUKbTirGBjA54G1wDZTR4jPHcimkYLWlc7QtCotrX4O6Ooopo5Xyj8Rp1EZPUdlPUVsFdTGie7NPJs/YF/dTirSw7FIaSnqafL4thLPUdQeo91TOJsYZIXQPjgqJebBf4Bpof1QS4ZNFKzk1eR8cIe60dzfsnw/Bp8YoJRQzwPlidkFK6QNkd10upwHR4VJh1RE3nVLp+RVAN5Z/EazfRdnguJPmxnxGHxR8uik82QWM0b9PR7f4LjuBMMpcMjjxPEZTS1FNVCLI1l3RvPdnX2tuvWaPAuZibcRgkDabwoZma0AueX31HyWoxTm0qCR8jHOfcG97HopsmqlbAyLNk+JxeUsvuusJEma0FA6K2qOxCZ5JGq0Icg6qtV0oeLbA9lcdmTZM+6tjFML4mkWJURZP8A1bCt3wh7JnwW6LfJiYc1LJK05ZGnforkEZy6t1V6RvmshDAE5JQ4Q2JmnVFnunEF1KyAAZVLVUyPc+4OidzHhuZXRCwDZRuiuCOicmmLVVE7HBkbL3Ovsnpo5HPDpNFNUQgSg7WVWatEMmRpt7rqjRJACka9r1iuxJ5dlYblS09eXHU6qcZZtYxSlfJE57Nws5kEkELmP2Itdapqc3lukY2SDzC61EpLlWQTGXI4391eZBQvaGPP4o3C2RRQA3DEwoqcOJLb3W+bPFmGkY70BO3B3u1Erwt+lpIBGPKrUMMI+BZ8RrizcOw8wi7rn5q8YoyC1zAQrZaBsFWlkDLrlybpgVlBypfw22Ci5jI2hubzK7V1bHT2LlTkja5111xYlVLjms42CFsuV+UhTSQOtdV3R2doulwxK0yUNIzFTB8bgVSjbezHKVjHA5C0qTCxKxTNY3Rh6q41ypQMe3zWVpsb3PuuUw6RKVh1VmJ9jZVg1wUsY1DlzltdaAW6oXxtvsmDrBRmbzWzarFIJzGpsjUPMuj8uVKVEQ1AR0KGWUsfYJxJcJSHRN2QHbMozIrSrINk4N1WbJbqpRJ7rMwLDACdkTwLKFsmiIyXClBiQg+iRPdDnVoE0aqTKFE0i6maQbJQcMRBuqSWZqqWJrRdKQ5E4eChk11WiUTpDdDoQhkuHI2jTVaZIWvZSsZ7ad1HlF1YBAZ6lJC5ZClabM1VczusQSgZLfqoClcMyKOSwQZWuNypI4xdGkbqiS+UOIyqRtQba7ojCy3pQ8kBA4dmKdMGgIwW2QRXeCkDdJzxdN5eyMk9tx5VA6N3VqnvrdOW5grEjOlFgqUgGYhaz4HPOjVXkpPMdFvCWaY8ovdVpBZbE9FZpcsmZvmXoxyc5hC4ZgolPy3nYXUbm2Oy6Mnbuk7ZMkjBIEaFyLJkaQGqRDroENETdUmRueQArIphH6jdS1pGwGyNryw5mpaAKWJrHtuQszKwsUVS9zwS0ge6svnBKoSuLB+HoFNTPY4guXOYtuJWS5x1A0TSBwbmKkM0Y0Tcxrlni0q53gqaGU2AUghhJJypCNjT5UmBbidcBTNKrQkDRWYgXO9K5S2kAuFFK3UqxazdVG8XKzCSozyui9Kg8Q8qStVAvcAu8M5Lb6vIN1We7nOuVED+cp2ysaqymbGxSckOChjfnKsMZosy0KGihIu9xP1RSU0MY8jbBG1zR1TPIeoM2R2pChc1xOi0jTsJvZLkNv5WrfJlRhpnOeA0LXgo3wsBcFPRRMhaHhvmCnmkMm6xM21SAENQSuL9AifoFGHAOta6w2Jl7WcozSefPf6KwAOiIALVir4dqgxPD/F0EjPyC608re6fl5mOjds8WKzyHPcCn+hzUp9UZuf4reduuTwF78Ox+eke61zZdgY1m7FeVrnN0UQgNtd1dyt7oX2AWrSYUzTE+pqhla2O9lbc4/Cqrw9zzm2ViUpWiZNLMNbNVx1IH6FuiUUZa4WbZWjstTJSINa1uRrbAKSKO5CE+pGx1tlzmWuKyYSBe6je+zcqLn+W1lC4klZsM6U/EozInkHluVWc83WRLmBQufZQZ3Ji9yom5tijbJfqqoN0bM91Rca5U8UMnJfUCT9mL5baFSh9lRr8dw3DxaufYf3lZVxlZxZi2sZpIYh3Md1TqZAKZshpWyl+9hqtPH8Wwuugz01JcDreyr4XWTRxh76SMMGx7rCOZmlEr+T4XkhQxsgkeGCf6kaLexJ9ZVVTpRHEAdgQqMtFUVQymltZZmGlcup5PwH1Ukre3RQvigBtGAGoS8Nl5eQNHdQyPfmytd5VaVMyw1jvm6WKmgr8TgkBhmkaR7XsqoEjWZ2tIt1IRxT+f8Rxv7LnKw1nV9fMzmVs/NNtLlZzLAucNyUU0kbyDDHYfNA0XcW5h9VhAvNzq5AWZjmzgfNNMcrtx9EOeytLSU3c3IxwCAWYS2+Y+yjD7pF77eVXjCEXOkNjsoXsAKLJcF+Y/RKONl7yZiO11rjAjbEZDpZKRvLOVSmNl/JcD5pcpnxXVEFynPp+iORrGDyk/JREkhUgzt0JNk6F3mCok4e4RxnEHVMUVBEWXyGSU2sddlexP7L8RoaB0tIWvczcZrmy6bEqz+bnETpK3meDBJjc3SxXK4x9oVd4+oOGsk5Uv0A/gujCLhXEMPraSXBMUZFG61o3HTX/AAXK4tQfd9UYjPzXdfZVpi90r5CzLnN7dkGUoE0M66JrZpGsucpNlLSMjNWxs5AZ1urQghdigFw2Fjt+6Q0GtwhlFy7zF3M9rWUMlMYGl8dS0H5rVxIsxd8dFQxuleOwWe+GSgdJT1VIbkW1Gy2ygENVNEajlmRo3IUlJRRysc+tjlb+WwWvRsgdgkjKRjnveLOaBstbhnhOvkYKmrr5IIbXGxzK8U5OLeWRvLGG7QpCyLw5m5xDhs0dVocQvpp6p1NQiN/Kd6oW2DljyNdyyDoRuFJxajJqSfdzaKn8LMTKfUCb2Kq1JlY8MuRcb2SJbFG38HLnFw7ooHzyS6SPuFiWlh1pfOd0c9U6WzHi1hbRQRvs7Idkbg3uslI3RjTlk/VTi3hhGZHDuL6IQLJzE8g8yGRpHQhaKVSA0ZENrKUjS6hRYLP5rqRrGO1I1UKsDKxgtusIN5dta56BbDOZJS5JI42yZegss1rIWtE0jvMzWyXiZq2U33OwVkaeGTvp209Re/Ld16Lo8Zjgr8FFfAwZ43DNYdCuNL5o7Rxi9tT7LeZHJPhbaqnqg6mOhaRY3XOhNgMdBWwksg/pMYu6Nw0JTYlTVeItLPDksHpLdwVlQYw+hr4pmsDWsNnW6gq/xTW1dJLGaGYx09S3O1w+IqDKB8I5zTrJ3V7CBTPfM6pksGm9rauUGI4dPG+AOkbnfHeS3dQta+PMWC5Z1VHU4XjYqaoYbQkMjAJOca/RTVFRasNJK+7x0XK88wg1cDjHM3qCrVdVGlqoahsvNf63HNuucwNnE6l8ZgdKwPjYb2PQrmYK18UtXDPG4tqxYW6Fa/FdZzH0GSS0EsfMIt10WVFUwlr2uhL3ONw4btW8ZFmkqMLmpar7wY6Sd7SwZhe1lTpYn8gA6XICQgje50MbfxCLtUjJJwwMhMbHMPmD979l0iQM1ficZdRRyTxkGzmtFswUEEVOaiMYlPLEy/mJ1LR3UhxDxNWJ5WWfaxLRuoaiZ9RWOyeZ3QK8h0NZh1bwvFHjeFVdJUUdRZjTa5A76rNxKjrKPlzVTHhtSOZHm7KI41VVmGjBqmiieWOzscd2lW63F3VeG0mHVrSXUlw125ss2OtgxjDqvhSTADUR5325FzsfcqnVYdJR/d8k0HJfFHzHBx/aEf8Aus6epoquXD56/C54aWC0U7mbk9F0UuLYX4ilwp9THURQSAioedovyH/NUemcM01M6Hx7Ysj5peaBfUAjRctiuH02HcQ1eGUkYJrZopY2uGjTe5N1sTVeMYbSUowqSCJhcQ2GYXDhuAHdFdwPFKPHJn1U9GKTFYCI5ad+4PstRI6ORwuGtOwsije66q8w5vPujjkbqXGwAuSqLoc7ZZvE1BQYphMlLiOIeEhPqd3HyVX+ctC2ojYybnwyOEfMj1sexHRaktHSvcaqaAPewXBKwPDuLKHC2VkUGA4xHXU1SXiWCOIRhthuAO/+C4dlBHE/NTzyxhmrSHdVv4kx7MdrKrG5J2vlleI3MAAAv2CxauojiqZYWUsckcgs0uvoEVe4jr8ProaKmocQfVPiBMwfoCSj4c4rxzhRtSzCJGRMq7Z8wzC40Fuyjwx9PNTx081FDLUgWhym2Ye6p1Edbh0j6OtgDoS7Pdut/keilI1KCSvxKnkkjZXQ1kWeTmQkgTXNzoP+dVlcR1k1RWseTWZGRiN3ijd9x7rf4LxR4xeOlfLUw0z2PigDjYNJG9+qbi6Cljxkz1sgqJdGF8Zuyb302cFplztNX86J8cn4doiwFpIuVYoX10dEH08DJHj1uduAjouG58VoMSxTDo5Y4cObncx437gFU8BnoKitFLiTKiTmWZEIN7qTDUS7LAqfCql7cNxvEi6nnpDM2Unlcl/sf4eyt4RBg/CtTScRYbgVU6hqYH01WyZxcQzT8QO06rI4kxqHDa2lNI2Mvw9vL5MzbiQfJKm4krMfigpMfqHw0T75Yadobc9ASN/0VpadDxvRYbhrqWtgnq63Dq0B8dOZeYYhbV7T0suSq+I6SaimwqXDjU4eR5ZyA2UHuSqFXjZwerkpcDxKdzNA0k2FuxCzBW1Tw+CXLkncXkAblOIatEMNQxnPE8Mbtxobdiow+R9QeWH5Trd3VinNW/MIDyxbrluVUbIyJ7zN5rx5B0se6WWGRjOYQx+Yd7bqelonyVEEbT+3OUtY7z2VmrweKni5kOIRVFhdwY6xAVbx8raMUjBox+eOS/mBWrQ1SyGGtfSwxmIMNsp3CHl55cm7up6lHIySekdi76ky1Bl5bgQO1wULnUwjgmZLL4jN+Kdhb2QacVDiU1AySKntCxvrzA3UQw2uxiCWrgDS+maObGTZ/wBAoJHCSAvzXda5WngVTDhlZzA+UmWIiUjoFzkZEFO94DHuy32J2upm0dbCeSZLuuA3IdCULoppY5THHmYw3J7BaMMj63hl82cRvo5QxpPUIFh9RUUGOQUuIkTRPPKkYRpYrY4pxbDqqWKhpYGN8FJyg8jUhZJnhb4SetcCyssHOtrGRpcFNSNEte2OWZkbXzCKZ7he375VE+PYo/Fq1s4aWxxxiOKMiwaB2H8VR5rmMay2UjfTcqXF4Xw1ErWEOyEx6fRR14gkooqunm/E9HJcfO33WZRVgmqYpXTNqJLvNybqyyR8zjnHLY/1ZTobbXCCkNPLBLTmojimJu3ObBBKRFeOF8c/KIJd0VptOadtTK2DDhzXH4dkzgyilifnvzGZ2nZWuHHTVGJuMHh4puU/KJTlBPse6hxNnLoqWDKTKHm1h7qTAsU+L/d7jNSwSCV4tna7/AoYcLM8H3jIbyzy2JB1v3KUWF89scbZo3PtnDSdCRuFdw+aYSluHQxBg3aW3LSoOr4Q4krYKV8eJYXR1VHTSRw1DxHepPSN/uvT+G5a+KSbDqufK4DmwOsSJY/cHYjquN4OwCnqcErsVqxIfGlkboWi2Uxnf9V6iKUyzNq2NvaIMHyOpWoSYSTRvjiMgbcjohdIxzGuYdxdTSVLIYZJJPTG0vPyC5qLF6h1bBHT0UkkMzTKG3AP8eyqRDcdsmO6y6jGZBiFLh8UJaagG7idNNwO61GtcVVDYo2NuU4jRAWKiJA0NCCVtmokMpJbZBnztsSVE0XOytSsBHmUXLynRq3ElHDnAeVO2c3sUgx5b6UuS8uFmILDdRcITG97tEcURYLbpyJGHyhBWdhbZzeRqr1HD9B6nxkn3etNomckYJT6tVLKc3UYEC0il0PdZ5w2rgmEeS46m67eOk/c/iifRNPwLrzZ4udosOaTnkJOXYK6+INOjCVfFOxp0Cins1hyqciuLPezKqD6k5iB0KvzFxBWRUAxEk7LcQzK0yrLRqVYirmNs4vssYvYDfOTpdPS1TXEsdrqt+HBbpoalkuztFFV7EqrSShtsytyWlbYFcVc++GWrr3gsy5LIzBNE/IbGy1yzL5gsuqlkEpuF0wliYStbn6IuRT/ABxC6ggdNn/d6q4xmcZnLVrQBS01g0NU0dPHa2QFOIuwRsBCzbfExpo7XyBJsY7IyTZMChxCQD0TgXTIm+pZaCS7ooJmybsVvl3RNh03WUZ0b5g/K9yn5+UWKmfCz8qhfTZ2m61bIHujkOZr1JFDdvqVKaF7YyQNVVjrKmMhl1uIS2xI2zd1TJ1KOOpc5guVVke+51WYxLWGPzGykGZZ7Hm6vxy5wLrU4ESO5CEvsVM2MOAzJzRsebhY4raHns/MiErD1VOqoqiKUmP0n+CQbIG66nurxS1xz29Emyt/MqD5XNOoQCZ56LfFObWFR0zJPl6LPge/N5ip55chDbE37LNLCzHI6+qlMgtus0Tvz5LFWGuKzxatZzN7JBygu8Dog8TY2KUi6hMh2VYVN+qka4Sekq0E55unjkGbUp+USh5Tr6qCwJAOqkY8HZyq+HuVM2OzQEEvMf2/gizEqIZuqZ5s3dZaGZWhDzlVMlyna5GeSwXXSbsom3O6lBaEErW6Iw1Rc63pSEh7pTQjuo5DlKT3myrSSG61CchStDm2WVV0nmztGhV/mKKTzCy6Yy55QzBEWjLlVeZuq0XNcCqU485XbHJzmFbIgN76AlWA4gZcqsxTxgWyBamUpQdBNGGueywKZjSTYBa7XwzMyP2Vd0TWutlsE5rSs2mfYHQpPgeBctVyJt3BXuRGW3cFnkUz6URxjM9l0LpGl5voFZdAR0UMmHktL7m5WedrSrI5rtkcLnZU/gnj4rrQpaeOOOzj/BamWaUxHNK3OyMkJpDUwC/h5PnZa7ZQBlUnibMLL6OWObpTBFXLmsVoZC2ESJjSUr5eYXlvsE1bLkhDIyteqEyXmHQp3SPByqjDMYzdWRNc3cnEXqbmPeMwWxTi1mrGpqthA0sr8dVls7MuEukLr2XUUkWQZkDay6mZOwhc1Z9XTl4JyrKkgeHWsumcARqqM1OwuzZVrGWZhiPgfa1kcNIb3IWg+D9xKKIjot8vglIW0ljfKkWdir2RAYmHos8mqUSSCjjNzZWHU4PpalFTEauarbNCZFdSthAGydjTtZWWM02WZltC1rgE+UqzkHZNkasirJFcKPlZSr2RqjfG0pYrZvZO27kTISDqjbFYpYdrVKCGoNAExN0HH8Qxvw/iNlaxtmPIK7BjuYxr/wAwuuc46i/o0NS3dh/yWrgtQyowyOQdBZRpeJuoZDqjchLQVbZQkH02RsiZu5FlKYmy1YZxYzZQyS6p5HBQO1SYZoRlupYjcKq3RWIniwWJaT5gnDkwcEznABZAytJCrFhupnS6KIvARbRlhumLdE5lF7IczeyIKNoUwyNaq4cAnEuq0GlJPlUFdgVFX0t5obv73VjM0lSh922QeXYvSMpppIKN8pydAtXAcMdJSGepr7uYL8u+gXaQ4bh0b+YacOPcqeSnpZIjGyEMB7BYmB5dWV7jWOPJc/J5AGhVJsWq4bXBbfa4svSxDheExSOZSsdlFzmNyV5tjFPVV1W/EJLRNk9LSeixMNsqV8cjzI/d3RM4wloyEgonQ5TsCoyzVUStfNJZj5DIB3TOiyPzEKNrnNTl97XK5zAlNSR5WZQEMU1pQ+9z3KAtYRmUkYjAsQlAZ7OvI93m6KC+YKaQDNo26jeWNPYqAQR0R2Ltk7qhpZkaB+iUTgxwzOIVDtY4bgoZoJmtzvbYKxmLpGx05dI53RW5KfE4IM8+HvLf7CwMUOcETZS7aMFTTThxyeFy+5UZl0s0D9F0gROJudENvyp5LkXTxNHqK1QjLXJBhsrEUbJJcrJNR/BTyRtIszUqDaxrFMfqKBmJ8TcMOla82zlxAH+z+i5vGMdwaXBjS4bRGKY+ohtrH/FdPxfxPRY1A3PxDS+GbvGzd3zXm1WYPMKecytv6iLLqwtYQ7B2ZpcRyc420dsVn4j4eWZ0lKzKzoL3QBl0xje8ZYxc9AgguBum5otlBsrkOD4vL52UhsNfN1R1UzCQ2eOMuAsbN2Rt0/DUVdhOEGvpqahnc831ms9ZEvEsk1VJI6lja+UW5d9AsHPFG8WLwwfCDortbV0Na1nLpBG5gsCCjDToOJKvBnvZJh8Do3uz2vayt4nxDjddSCQScmjPwtXI53OBaXk/NXm4jVOoRQuj5jB7XsF05DpeC6vB8M8R98VUTXz2y5hpbVZOPzYOx9RDSPE/MfeORuwHZZNP4Wao5FZNyWd3BSVsOFRjLQTmQDckLXJpTfLJKwMebhosFG3zOspGNDjYuspaWAGTzWI7LiqIG2pRx1T2O0APzCeUN5hAboCo3AXzNakkytsl5cZkvZT0c09bmZM8uJGl1RlkvDltsngmMVw2+o7oFUMMUjo7g27KAmyJ26j1JQLOcyIE2TN3SzBBNNNzGj2FlNgzGvrDM/0sFyqmmynhqRBEWRtsTuVmhaklPOkftnULah8NxTzkAm+Xoq3Me5xL3JN/EdaEFx62VoaDqhlRHZ7QHdwtnB2SYjyaGqfEYqdugOtwsSPD53S8uMZ/cKaGc0MgY9ly3peyyNPimrbHWRU9KCzIzIARuFTPMgaGSCzuoR12LUeIyQzzyBro/g3KeUxz2qN29FJFWpZk8wddrwCmw2k+8hLCJhEWbX6q5WYe8sbPTi7HC9ugSoaDmvErGOIZ6o2usVIkVJpZnDl1U4k5XkaR2QUkr2zDI63udlb8Pg/4rWMmAk2BOrVDT00NNeR8xktsCrYsT4dPG0ve5p1OjTcoJ6OBsrufNLzjqLdlr4NDRytir6ioAdr+D2V6rp4ayU11FM3PB5Mp6/Vc+Q5CGRlPMcyv4WY5a8upeUx7bH8QXFlsYhw7LFFH46njL5hePKfxLDosHkxUdW9kLpIn5TfNqtRlbTawqbBm1sp8IPFZvXIPI0fJXK7H8Lmp3UlPSRDPs8jUn2XJwOqQ7SznHckK3QySS1bRHTeJmHw7BWmJb8dQPu2fDKuOS87uYDbSyqYPSsdM2RtOJ2U7vxIju4K1XZ5Io4DTAzMs8MYf8UGHmHBpPvGtjkjaNo3G5J9iufNXa0NdT4m92HOxSanYZPwIZWbEdB2Vni1lJJTUmJMrWRz0czIp5GOyuIPXVYVLWt4jfTxRV0bJwAYQ8akfPRX3YRjgxYSVUdHUQ/HlFiV0xzHZ4TWSxZR444jRTAcqo3kBA1B/gtVpY42L3ZeoBsuV4Mo58Miq6OcCJj5jLCxx1Hcf3LomuXUcnxThuIRtM1JixoKWOO0MbBcya9bddVbwWrxLCKegxCqfVukr/wAJ0dRNmLb9j79lvSRQPex04jPLNwHbXWXi0E2K4ryBLJHTUkYkbkOt+ig537YYKItpa5gLqj9nDGxtrk7k97LyebxDZXMqo5G5DbO+MtH8V6XVQ0/ifFzVddFWQm8bZG9bfx2V7iBtbxFw3R4rTV0GWwE1NNECyQ37rJbguFpsLjnrG11TTNLGgQicaE69V0Hha2SKriMEUEVWGRtaJBITruB9SuVxPDG4pjLqHD6GlAgAEslK3yNHcrc4flxngxj6V+FCoaZRKKln4gLBuAUZacHD9Q/E6bgfxFOIPEc2N7wfMfyX+D5rXrOC8IwjiDnYW2Gjq4gAaV558JB630I+YWfU8ZRVdKMYwGhloq+mkJa5wDhIx+hBuutq8PeaSLGeJ8RlmqZ4uUI4QYn3IvkBHqCN8HD8ecYx0dFW8MwiF8tSGB0sMlspvf6qLh2PBeIKdklVhtPR4pQGN8JBy80gaH5bLYxDDattKz7nwakig181XIJJOm5PRcjVV0FJy62V8ctfHLYOjuIXM6sHshxpVxqWtw7F6mSvEEj6h5e6BsgcY9uyxaur5wY9jHRvjPlsdAO1lexWehdSls1Jaulm5hmZ1HZU5oJIcsYp3xlhu0u6joiNOGDB8UmZV1clPTDIRIx77PJ72WBWZoqwwNaLQP8AJYWuFtfeFVPHVZ6SE8xzALi8jCDuo+JcepccxmJ3J5dHE0RXayxB6/xViV9WUZI87ZnsL3HZvZBynTShhIaCfU42ARsgkqZzBSRmUg2aGM85CjlimbCJNcpNtehWYIFWUJppHfiQuaCLFjt1Xc5rbWKNzuYWhzbPt5TawKARSSXdoADrc7LXEpJzXzXe5w0GwGiIyGSJl7F0ejTbYJ80FO5/LJmb8N08TmySsYQGczr2ShZoY4audsdVPyWZMm+5Wtw7QPrZKpzJLOoor3LQWOWbh8bJKw03gfEPkaWMFwLFXZ8TFLTGmwmDwj8ojnkboZD1BH/O6Siq9s7GOn8OQ2RxIB6jqF0WBM4Wq8IxDC5mVMdRXycylYY7iB7Bvn21Oi5xmJ4rHT+Fkq5JIbECN2oF+y1cBn+9KOowR8ETnti/ori0Eh972v8ARZgZeLRYhhMrKTEqGLTzxlrg4EexCkxLFKeuqm07yIGgWLwdE76qnkqhT1rCxp/ClY27n/T9FpUmF4RBz4sNw+sq3EAGCohynJfUhUZM2JRySMpYLkRtDA4i2YBSwT03g6ullY0TTlkkc7hfJk3B+a2Y+GoQ0Yjjj4aGFkWSnjkP40p6X/56qjDg8EuGSzzOlNSX5I2j+r9j8/8ABYGbTRCpZJJI2mLnm4uL2PRPyzndG+PI+QekCwWjBR1b2y0bY3xfhExkjW/ss6lxOeCo58oEhLOXIw7K2LDKd+Yx2vb1N62VeQ+RrZJAMgsL9kfja1rHCBjXh+lzuB2VgS0sTxmpY5oS3zNcNj7FVo0E0NDPSGellZymgG43B6rafDBHHKMMn5pqJNHMv5ewddUMYPPfG8VEklNEBEBI2zx8z1RQzyiFsfiBFc3uBYaLnI3sE4xxHhuhNFzBIy9y1riDf2XZcL8RVPEGKvGJ1vg2MjGXNP1OxF15zNUQzUrqeqgijJfna9jbk+xSFY05Jp6SUm1hb0O7Kcx7jxbxLPg2B1HhY+bUPiyNk0yWvvZcZivElZhODUWH0jrVHLYyWRzdQw9h9FxEOL082LU8lKxzgG8qOCS9myWsD/FdXw7h2N4m+sqKulElPHemmDn3laeoA+i6choYJNPW8U0lPitQ6OCDmGFz3amTTT29vkvTWytG68f4FazFKBz6iP8AFpqtjua8kXBvdehYfPNGwGeoMna63EMukErUVyVmwziW9jqCtCJ3lCtA0zi2ydzhZQl6jRANKikbY3CO9tUzjdWEkmDyp+aR5coTg2TKoQmfGb6FWYpmPGZ2QFVHoEGoCPZFdqzYpTG5Webf4lgWQ5oTPeLKvzP3lFJM4eW6CYvBVKpezUXR83Kb3VCqnYJHeZdcYSQSyR2y3VSaOCSM819lWnrI45TcXUfjoJvw72XaMXNajpaaQeWIFM7Dmtu+KNoKaGpZHbKVaFQ+SMljQQD1UtpQfR1EJ5nMJFth0UlNWVQlEZYdetlZhrHlxD4raqy/O/Vlge4UmQzpW2ynQqMwh/msqdVHUROfOHlwvdT0dY10Od+nzSIEopQdlYipgBYrLkxUseWsP1UoxuP89lrjI1ssYaqU0ha42Vb7z8t86E1jJ78y1iLaKRjIk8Vfqn53Zyq/0ewDToEnSxNGRh1W+CclttQL2KmZOxZkUrLZnu17JGqb8CeGcms2eH8yLnR9HLI8W0bAlP4oO+JZ4nJqmZliLqPMs/xLhsUXinW3TivJceM4sqlRTRO+FN4p/wCZCZ83xJEM2i5WTYlCWEnVSuduVEZH/C5bhmTluUpw4hKziM1kwab36IqdlW8elWIq14OayoZQ11rgo8wHRSjk0TViRuVyrytOuRVC9w6qQVDtnHRKWzSRcwebdQsaYXHO1WOY0+lELP3WoRBzNdFoMaCACs17MknlT897GgByzMENN7IScxTBsY9KyJJ5y7NnJSFZKN3pxW2zdoQPZHILOAWb493dTNrL/ErxLTci2yNkZYbqNtT+8n57fzKSWuNkd1CO/sqXiz3UjKhh6rFLa20tR5gqniAOqY1JCnGVSySOUecn1Iecx3pQuktqlSC5ebzJWsULZvdM+ZvRWkSc0BPzWn1OVF8zwVE6RxK1GKW0fEsHpN0vF2WTzZA691J4gncrXBLXzWX6JhKx24WeZuxTie3VPDLW3O1Qh+vmCiEwPqSLs2qlFppbFqoyxlxvZWc4OjkZDbAALWLLLLCDlcmDWDbdWZ43etqjZHdy6XDNBjkDXBrirDJGE5XAEe6CWnYBtZNySG3BujS4zlu7BTxuA6rIjmLHauV2KpBF1ynGVteeGkXaonjym26dsqTnA7rMNSpOleHWyqRjr2QzPYHeVqHmhaYXGRF3qTuaAFHFVMy+Z6CWoaRlunGVtXqZLO8pUJe6QWcmm1cU0Y8q6Q5naxoOpVpsHMYAzRVTuFepyOUMqzKwdtHNAM8hFvZJsjhoFNHG/JZ5JCF8DdS1Yl0SQya7q1HJqs7MWOAVlriAFzmCJaLJu6d7muGUKnHNc2JUjZBdSYbtIYuqblgJxLm8qclZgiAZSiyNISRZgrYDluzbKZsWiZrlJzAFAhGB0RgABAJQTuiBBUBZ2j4kswUMl+ijD3A7oLJIQu3UYdqnzFA+UITonzFA7VQMTomBsnyJnbrQocRU7KrC5WHoLhZnBVQ+SjfTu3YbrdqYufTyQ33aVyHDU5ocYkpb2DiQoOzSTX1KdUMdlE86qQnRRSP0QRkdCo5B5UZeLoHOzBaZVybpZ3JO7prDqpMNJmTEfEpTICN1TBy9UjJpspMJaeQjuq75LHdAXlRF6RCeqQv/AHkPMsfUoi9NmCtNJzLf4kg/L1UTd0QasMpRKpY5tbKsBZSR7qtLTSSVBWYxhuGf6dU8r6XU7CsbibAGYpAJWOIey+o+i5yrmMd4kp62r5dDNZndY81K+Z2cVE0zyugoOBGVUZke6TM0ErWwvhNkBs9kjD3zXUacfHw7XzROmLCxrBc5ysYuyvI7L1g8PV1V+Cavlwndtv8AisHirg+mpoh4Se8jzbzCywjgTnllGQXujkpnRNu9dJQ8IvY0Pmkv/ZVHE+HsSp3mQ3dD0utqwibKSJgcLvkLPkrE1DNE28kYHzQ09FUVL8kAFxrqVLD0zWNdne24CPIZJedDRNI99FKJJwRSP5YB6t3UzpYaQCKZmdva9rq0Kc1NNKwzWib7AqainpIIrS2Lj0IXXUOKYR4IR0IhZf4XaG6ypeFKiafxFRJDEPYAqcRixSRNqRUR6EG/suhpeLXBogMYkPckLIxGihoiWfhyWWS6QE6AD5KTA2eITU4lIKoQAACwssXIY9wi58n5yfmgdK87rNCGbOfl2ULRc+lSvLlG19lsWqXJnDHmy2YKGiZlmmqJcnUALngdVaBqnC3iyG9iVByMdZRSPAfSxwju1KeekDLMde/VQQ0Bqr5Xhtu6vUmHinIvaUjoRoVplDhlPPiLnwwSNyRi5BU+SGgeBmtKzYraFbUR05bFg/Lb8TmBXMBxjhmL8PFMOdJMTcl0VyrEozm4hRYjTtgnr2Qy/mJ0CzcU4cqcPidWVFdBIw6tLDcEL0LDm8BYhmFXRHNmNozBy7KjxJxPwtHhk2B4PgbcrxbM6M2Cttw80hFKZAagkN7o3ci9oZMw+SvRywUmXNC13sVXrKpkzs8dOyIj8qoikhgbCXiXzdGndHhkRfUASTmFnV43C3eEHYaKeqnrsrnjoQsGbyuc7u8okrOL4U+lqIpH1vOp5N3W1VM0znH+jjyKQF0xzyOuSQFYqaKtw7l53gc3a2qlqzpqd8DwH9U+XIFP+JNVBj43S/ut3K2anDKinqqNkOG/iSN/ZAb/ACVHOauKItyhbvEuHvop2GahlpfgOdlgVhya7IItMt3Xv/BACUXntlQ5Qgc6pgLKzTwMkDi+ZrSOhSY1t75QbFBA6N7bXbZMATsFaJEsgv0Uch83l2RUGS6lFM8tzuIARRFjb52qxBEZYTZgNtm7LIgpGWlbIAHZDexFwVqz12zqTDYoDsSNbhVH8mlhcNpOgVZ1TMeqiLsM0zxKL2cBcFPh5ZWUk0c7v6Te7bjUqj4h9vKd00Mj2PztKASzLIWHdu60GCd9JHDTjM4G5HWyqObJKbsF7rewWR1LTuYReYtNlJFUYjV4ZXNfSvyseA9zbaX+SuVc1DibmT0l4pDYStaLAjup4phiGDS+KjDpI3Zzfos2ammFVEygY481udrQsDV4ibhdHyGUNBHE+WIPke07/RYJhMkfPjb5B1urtbUvdAPHMd4in8vKO9vZVXGQRB7XujYXZ8gOvyWxJC+85gc3yPade6r4ZUz0+INhZIS18nLdfqP+SopqmQSiQGxGyq3zyuPc3U4j0fHcUp5cPMHMtU0NnxP6hcrjNZS4nHFXQNe2pBAkG1ysbY7lWKaYOkFPMzMw7jZOInbNHJFnYwiQfC4aFalRBP4elxTCpPDskbkmLB6T2WJPTciV0wle6PpYrd4bkZJhlfSTm8D487W9ilCpQVFNSzTFk8kx3EjjqStfxlLidBy6ilmmkPpfewA9iuchpnsa0vbudPkteCdsNFVU+RxEoAbbpv8A5rEwBw8MlnL6cua+AZ42k2zW6XXXYdxl4yelZDSmmo6uTI4vsTG/uCsOkwWbFMEwulg5cUzzKQ61y4KKSgqouEnFkRf4acvkI2AunoPYMOrqTFo/CTPZzoNDrrfoQmxfGzg7eXT4XV19T0bC3T6kryAzsnw+nrajmRw8zll9yCP0XT4VxHitLiNJRSYvJVUdUMjXvN8t+l+q7YSL+JVnGGN5aPEqGOOnDs/IiILwfc9FzlVQPpBLO/FZYKp8mSCnjqiX37Hsu8xLD8QwaAVWHzudK9wDjIMwuTa9l5zxTT4nimN1jIqOCbwQEj5aKGxv0BP0VsmFao4rxXD5ZKHEquGvdBYAOGYx+wd0QVePY5LgcOHR0UUGGl3XUk9ysWXDagTN8VBLE+cXbmYbuPy6LVrsLxWnY2fEnn8SO0TRoPkqDwbEsOgxmOepmhpoYm2yRkkTHsSurhghxuugopKeKHDaSI1MzKcnKX3ta/Vczw5TYRJSnDK+hBc8mQvdpYKaj4nrsMjloaSCOSIExtLtQWLlOQ7KoqOEzc1DhM6T8MZAQIx0sOip4xxTimEYjSVtVJJWOw7RsGzHB4Iz39x/csKKshxOCXxWGwROjaMj4XG99bKOofVV9VRTV1HzW0wAcGn9oB7LPOlmTcTcbYjxK4Uj6BuHxSm+g1OnUrCmljj5MUzOfR0/Y219u61OKazCMXr4o8KpJaB3xmV1gfosWh+6Yi9tdRT1VVmPlvaNbibLR1GKMmrXV1JCIb+kDYJxXmrpW09Xq4G4f1VypxG8j3sw+OBkhvkA0WbNKyScvZEGgduqo0iaXEs/iubFkb5chvc+/usmfBq2meAxnNuL+XorUWJVbLRxiMebOTlWga+jq5QKqMUVZG38KRmsbvms2iHAfuqhxujqqWapc1l46okekkbqXGKLAKKmmkOOPrawyeaJot8yFjipxATcyF/KcST5VXH4UshqhmdJrf3K3isNjF4Y6TDaSngZGGG0ozG77+yyqumqYWRPnpDBnboDuQlNUSSxRQvkJZALRg9FHLU1VSQKiofKBtmN7LSoctwpY35Sx4Hl0s7oVrcLcPz49iTYmQySRU8jDUtbocl+/uu54y4Ew2lwekxXCcPm5xnEE0AaTzGdyOjtEHCcPSPGLPkHLMscRfGH6A99eiKbEIK+skqK2GNtr+VgsHWRVbY8KE9RDQyxeLGSLmC5t8Y/uWuzh+hqoqd7yMPiMYAdOCZJTubDdZlJQ1GHYfRYdh9YQWS1HnLS38Mi190uHK3DqXEo4aWBpllz5ntG2mlva6zcXqMK50WHUchHhgGc25Ice1jspsEqzR1JvViF0rSDK1ua/ss0hmxPwKubis8DJCyR5iDiRlffS/6o/vnGcbnFVVTSOdGSA+MZeWPeyv4/Rvmw+mpIxEyMXf4iQ+dx0WRSNqcPd4fOMr/VbqqJJ21dbLzpq6bNECQZzc/IKfDJsS8RJUxP57GtvKXutf3KkldhkDHiOeWJ09mZnG4Cgw+ugw/nsOaRkseQWG5XGZGthDH1FVTx+OjaeZcmU2A+RXO45RVOH49WUs8kchMnMuw3Fjr/AIqSTJPBZ4dTTM/1ZsD81G2mYBlfMXnq52pKQCH4QY1ha0g2cHHW5WnWsZRVDKQD1/6U4bOJ7drKgYow6OQSwRODha585WrPUjEquOhogJqmpcGFxFhnVU1dPJRSiMsZNLEPVsLHaypzVXPhazwkdx/BT4hDA2lZVw1QLonGKanJu7OOyoQTkzc5gLBYjKdiipDIyJ5kdJZz9W5H6NHZbNY/K2ijz8vyg5RsD3WU2SPLknB2IItY69QtivhrZsLixDxjZYoovw2utnsszAv4ZVYvU41R4dQwYfIyaUEuZCLix1K3OIeIcS4Q4jmp8MZy/ESc+TTNG+N4FjY7HQrnKOeQSYbRz18dM7l8zxDRrGff2V7il1RXTumrquimln9L6fUWAtp2VjIXqfiFlPB4V9HFTskbkEzARGe3RdVR1dRTR0tLUzRuc8bsNxbuuLmqaniCjosDkibTvYQHVd7MIA72FlpU+I1mF4o4tqqcuo7Ql3K5jHC3x2XfHNJh6dT5YzmAstGKXyrJoZq99MH17KYCwe18F7EH5qxFUMc8MYVpGgXXOqBKJpe24TmMjzLKhcdUKJ4N0P1WrKPmCLmBVyboM6CwXtTB9+qiJskCQERP5eyLmAdFW5hamM7CcvVKFoTgnLdM57bKoZWtO6IVDHaEpxkKV5c0rGrPEZ3PZESR3Wu5zPzIHGMje664flYlylZHiR/GkiIZeyGnjeXAltyegXVObE4WIBHYoY4aaG2SBvlXWM2aZVNh0xtIdPZaDInxMykKZ82X0gBVqp1RMy0EgYsz5tC5kOfIXeZTRzMta6idTRxsZkGoGpUTvINFKFmdvOiLAd1izRVULuWNbdQFaiq3iqbHI/yHdXJuW4eQqwOfla6Nub4lVLXHVXq1j7uAboFUAk7EfML0YuUp4p7Ns4ojO/oq4gkPRIXC0wN8sl/WU2d/c3QE6+ZK7beVVLlIyplB1cpmzXKqgi6ISAFTiW0I8jhlJtdO5jWnR11UbOLAWCnjmDtysU1CQ3aLp2zNIAy2KPy2QANGrgjSUObl9Sax7Ibs7fxThwHRYoSx7a7pyGDoEAff0py428zlKB+XsmuO6HOL2cnbsihyt6JvN3RZ2hM6Vt90QBamylEHNeEVggYZgEYeQlmb1S6aIorh/leNFG+AjVmye5RsaeqCuGG9nNTOp228rVayhM4K8ime+JzUPmarzhooDG2+y3aIuYR1Rib31TGPRQSDIVWVgzu/MibO4fEql3Wum83RKatpeKd+dLxXdyocx3dLmO7pUJbQFSBsnM+YKkxziiOYLPCFtZE3uhMxDr2UIJukSSUqEtYMrH9EDhZQ5yCpWzAts5WlM8C2YfooibKV+92qF91WDZ0QN0DY3v8AS26kZGUU4cbqRjjayDkv/KjaLHVFxFYnzBTRkka7qHK4bIo3uDlzpUxbogLGNN7aqfykKOQaLLSpO8+lMCRHZqkexpOqbKbrUSyovB9VtE7C4FXDDpoo5IuWL2XS2UtM8sHmJKnle0t0KpszKRZmG4kD5NCqpvdWHsdfQKJ7XNViGJCJTpZFzHkalBlsiWkFdpKlMBIGQ2UMYsQ5WWSDMLrNgZIDGMxR083Ldup6hofHo8XWe+Mt+MFUabKthNlJzWHqsuA2f6f1VkvsLrnMNxK4cnqsmz/lVGOV999FZDhZTiJQ5TRyXKqtJJ3UyzMNWsiRqfm2Va7rbpi4rHFq1rnDunEw7qlnIT81OJa5zbfEnEo/Mq4cSNk6cS1nme6NkpGip5iFIJipSWtGa6HN7KDO5OHOKnFpOHt/MnD2qAHuiBUoTZ2ps6jzBLMEoS5ggcdUmuuk7dKEYdZy4nEWPw7iQSA3zm+3ddrbzhcrxvFypIath817D22QdSHB7GyDYhLmfvLNweq8RhsZvsFZzrTI3TC6AvuED90g4E6LVKdRmS7rXUhNmabqq82KerJ37oVI3dM7RZVG7ZCS5E7ZMRZCUJLkJFlPl7p+X2Uo9VbI5IMcrXLaE3Lb2UtpC0IxsiLUKgNMw6IUk9RMHJ81xlLbjso0QHQLInjeGizAAPZGJCoAEYKvENUz1bBnpxc9VgYhj8LLQzSNlmOxGw+S1MTpqqrj5dLa53ubLn58BnilE1RHTxN7ly5jKrqnFGuc+CsDG72CzosQrql4ZVTmQe6lx+rpo5+RSvz29RGyxRPP8CLGTWrg57chETR3BVOGpgoyMuvdUTJLu/fqgcMxuVFtrwVkEsoysjb9FXxQc6dgh1vuqdMbShb33JI6Pmx10Hyuqq7gkj6SK5pKRtvj6rusMdh4p/FZRI8C+q8rnwyUxOkkqwMnY7qxhuLQYRF5ZZpfduyCHjKWarxqZ0LC1ptqeqx20s4bstGurH4lOZmtOY7BT0cnKYWSwgu7HosDFbf4kS0aswOveNrXdmqGjw2trnOFPAS0dTogoO3Ubi3ordbh1VRTcucAG19CqxZbdAwKlZM53lkOg6JMDCcpNkbYxdaWXPtpWDWM2d0Kvwx8psXN0Lza6ynzhrC5k1pB0sr3i+dTRzGMhwFteq5o2JMUqsPs+B7SzoCN1gzYxVurC4xwtDzfys1Qz1jpmZHFVSAHhyQTDdquIjURZXM5c1gOY11v4WWJU4jG2IwuaS47vB3TyZWszqq9rX6rUSEKp7ohG6L/AGjulC2MuzT6NUGZrSppTlaLLY1cNEDIqnk9RcrMmkzSlBT1b6eUTN1t06FNX13ipTJ4eNvta62JHeWLONwQQtbGaiCroaSsE8fN6sB0UMmGvr5KTDo6Qw1LviJ0XVYhgvD9HhDKGro81Sxt2losb/NQcvwm3EZMehnw9sMksZByPNrjYr0fjqA0z6KvibavzDkt6XXC4fh9FTsEohnbUDZ4OoW5hk5ra5tRjc1RNBTgGIk7EbLHIWOMHeMw2lp8VkkFYCD4e1iV53iVNVU0+RsGQE2Fzdd9x0+CrfS4kyPnQ2Me9ugXAymP/WE/NaiQWJ4TW4ZEyWcscH9hsq0cMr28wQyZO9l1FNQ0eJ8HyV1RNKZmXIJcTqCsKjraykpDDEGkE316K2KWRzRmcrWHxeIkANsvVVc073H8F1+wF1ajqntpTGwAO7hZCq208ch5DfmRsoMuf0tWpQ09M+ia2SO7j1ujmwp8LM8cd0Vjuj8tgNVPHMynhGXSXugfmLs3VTtp/vGMlhaHR6ZRuVRnvkfIbvNynHoze6JtLVOLgyE+U9TZHDSvEbnyva3Xa+6CAalSx5y4Nbt3VloibDIySMAjUFV44ZJTlZp7oL0XMiLYiw69QLhWiKiKdtbGy7W+oX1srWGVHPpXUsgu6EXGu60Y6Wlrw2mZIA+T9o06aey58imdBI19U7wscnh6iOxJbse6nwimhhkdNik0kXIvynMNjZSOhkoGmk5hDW7ELnMUZUirOR8kgIve9lmynatxahrctPT4c2WE7zVDdVyuIuf4xzHMja3pkNwrlLib6bBWUk8wN73AVWLlztMjgbX0RFHlZ3jQG2uqhkY81JLYwB/BaM0EJp/EUtw52hB9lTpRI4ebqNV0hoLmGMkTMIvt7ooXv5jBD676DutCBt489S8FovbMLlV6KdjMQ8W5o8nwgWBVF+sZRzsfBIwRPy52kfxQYXVswylfBNAXGQWDgdlUnq31EzpDpc3t2WhRMhfTCqqBzHMd5S3qVGQzOgpIPBzxB1UXaOIIyKeogqosMe8ESticzmPYLgXV2mnZPjX3pV0UehDJGvOYNHdamAYnhuF47UwQVUbaJ+8Tm3Dj7BZxF7hd1Pw5wn97VT5ZH1LyyJv+rudx2VvBKuCXA8fwqBgnvEJI82zr7/osaXFqnF8MrjVVzOTFNnjicALC+litDgLi7CcPom4TiTI4rPJbORdmvQn/ABWqYNxThcGHfZ7SQMhyuJYTrsdyuYwamqqnDIYRLHDllOUuNiwrquOMUocYlhjocQpp4YL/AIfM0v8APquZltSTsoqtjCyAczyDqszk3Dsa7jjE8SiiwOSGOB1xHPK2W5lA7dkfBVNh9VXVpo3kNjLM5dJ+0sevsV562iq6x800EcnKjJOXZ5+qmbhOK4PV0VRWF1PDO4eeOTW30WYyHovF5hpq2or66jliheWRtc06kAWv8lxmPcQ4VWyw5Jahro9L2uLdwlxnWP8AFx07J55GEW/Fddiwqidj6RtNINjcEbhWRqUNS/DZZIJCJ4ni/PJ1bdQT0OI1dG+uw7CJWUcJsahwsHLHdJU1E0MLDa5Edztr3XXRMxRx+4MYxuWWljazkxt9BKcRh08EkTM0+kb/AFDup/G1MpZUR1BHIdYADW3YnqhGG19ViFTBHNHMKQHM+9owFlyT1tnRxTMMZNgAsSOkoKjAfCYhW47AZJpZPwiBeyw24eWStqzPEA+z23O57Ko11SZDSTzcvOQJIybH2V3BJRhlRyqqMy04cbuOuVWyFSsfVGpPPhDe7QoJmsEwczRkguAtlhoG4xz5AZIZSea2TQtHcKhVQweLLI5RK1rjZw3WoyU0MMYo/EPGzrEqCpMPLBsJHM3uNFp09GyKFz6upjbH8TNyVSqaSTM+fNG6KQWj19Q6JKQjpeWWtkerAdDI11L4X8S9wOXe6jkpHwCmjDPxC3Vl1bpaxlHIXvyTNfGRltrf2QZ9VSUx87G2B3aNrqCOmiif+P03arpgmqGBgYYnP2z6XCzHNJAv6uq6RKun4Xx3E8IqZn4bVxU8MhBm5guCB/H9FtcR8W1b56WbxVTJY52vaS2Fw6iy5vB8L++Iq1r3uhfT0fOibEBeUjofZVZJIPANjq3uD4BaMg21Kti3jFdRYm+hj8WIm8x75HPN+WCRuumpsFGGtpsYdNJXVj6geEqY5S6NwO12npuuLwnEqXC65tbVU/iGiMs5ff8Aguji45gZiFEGUkLcPoJuZkhcQZL36exSxkcXNqaTF6rDqyCISMkM3Ma22YHXRZpLfCQzj1XNlp8S1eFYvNDPS1Essz5jzTJCYzZ5v/BZM7aXxnhqI3a8gN+aiUds75qhr6ieRxvfzEla00Ana2ancM8hsCSsmqo5qOVvOsHjaxunnqHVMMEb2gcjYt0Ky1xak1LB4MPk5ZeCA4F1iCpBQ0sGGOrqp7pHiTlmNm11hyzPkY9r3F7nkG/XRbstbh+KQNpoOfDVVNpJQ4/hh4C5zilJp6ySqpWzU9LBkADC4t/EFuhKVXXQYv8AiVUAhAAALRYtPe/ZZE1PJFeSN5GtrX0Ksw1bABnafICbAXB+alFLLqOko2n72w41bZzeB8chBAHsPmh4fnfhGL01ZBJblOEjg8ZgR2spcErJjXtmdWxeSJ8ULqjRguPkrNPwtPV09f4aoglkppBaexyX3Iv0VxVn1mI+KxR9RJHDJA9zyA1uUC/YDoqN3tlddm+oI2IW3/NjEsKi+9a7wsbIreSKUOLmG1zYfNUpqdk1WYIyYdRlE5y6FdBHSuJr2FhvcAAvGgTPdM1zoQ9xaxx0abfwUUolge+ke4gRusQNrqSMvimayA7OuH+6zMC1R5mwc60habsBIO/ULf4bfSYliUWHSU9bLIQXxx0tuYZO4utThuqjxLC6vBMUwqKuiDn1jWueYxDJax8wXqPBPC+EUlLQ18+GQtxGmhDDIARJYg7D6rFDzbFGcXYdNSYJHhRmbW0kkcUFwS7W9z+8L7Ln6WvfgNa3nsIqIwWTQVERsQehBX0hUUdNUPifNHmMEgkjLt2ELk+MeDqLEsewfiB0IPIlZBUNDPUwmwPvZb40MXhKrhqJY8LjxCpaJwT4QxEZB+6XjZdlDwzDAzKHyk/m0urVDRmmZAKmUyyUl2Ne6xJZ0uVclmN11xmaqUpBFEImZASfcoiLp2uuiAugrSR2CgI6FaEkYLVTla0FBDIFGGqwY7qC7c1lpD5QmI6FPmCROYoqM7KpPmb5gruUKN4ANyt4+TMMpstY8mxJAUgq7Ascwh3W60XsjiYSwWXP11Ry6mQXsu2H52Vl1bm8jrtPuqkkkg9L3qs8vaOcyTN3CB073tuu3CHPk0aSrkDw17yR7q4alx1bqFgGqLm2tb3Tw1k0J0N29inhQzz+LbmmEkVw657KEOaxwe55FuioCsJ20R1DzNG0s0sNVng1yhedXOe0sbpm6qoax2azzossVEg+JLnP7rfCGfEhrt5UpzOUrQWWzPusiGqLT5irbajMN08MjO1xxuL9UzHwNHniZ81A2fSzlMzK4EdCs0qKrkY4FsJt8lnE5XZVpGmv6FE7Dn5s4K1EsTCi83KbKVNNSSR65b/JRtGuq2I3bIVYMY6KIxndEkGYhGyYg6oS3RDb2VpnkuiseeqIVn5ys8GyNrvZOK8mj4i4uwp+c62W6oBzg7yuRiX4XKcWrW2TO7qVsl23vsqWdhFroszhss0trgka8jVPz2tdkuqWd42KHOb3Si2g57SgzhygjlLzlc6ysQ00kjgGnQ9VmWrDcjZGHoHNyvLL3ymyG5VZS5ndHJxO6Pt9UAkCNrGyI3YhXEm3LCJtUS62UKMwa+VSRwgG5XJEokUzWNkCHlMsCpIg1iw3aB7LHZR5BmV9zIz6kmxxE7K2Uo8hqbwgO7VoiGM9EfKaBoE8QplOo/LaypyQSRuy2W4+I30CjfCx3qC3GaTDDyP7ImseTay1fCR9kTaSMHMG6pzc6UKeB5lyqw+nJeAGX1U8cXLkLvdXYspGayniN0xJ2iKQx9lDmvst6emhqB5marPkw0RmzSdVYysyhnu2QXIKsSRlpylRZLrpbI2TNItdEC0+pVzduo3RMe741aTyWDnA8h0QXKISstYG6cObZRBtkJUoabXUBHZSxzACz1mW0uh6oCxylZLAVM3lO91zaQxuIPmUjmZgnkh/KELQ8KKgdFI29wg0C0Mofu1C6mYdgnIpRzBJwEgyqeWlIGYNQBhVtmkHLylPp6VK5pUdiHbLVs0duvlQOZqW5VZYCQny6pzapnPitplQFtt1oywtI8oVd0YA9K1yhmYQDN2TFzgVJIX2Dbeyrm9/MtMphIXDKUQcB5nAH5qAHqEfMPZKCLrelPzXEa2QE3Qu2ShK19ip2yqnmCISAFJgtcE2vqViOe+izxJoibNYqU1bUEmiSpxzqRkvuuU4U1acg9kBv2SEgsm5nYK0tpmP0R52qqHgHVJ0wOynEWszfzJFwCqh7kYeSNXLPET52fmRcxo9JVNzrFSMNwgs81v5k+f2VYmycHqFmhYEqLmXUBf2Sa/VWmloPHdFcEKu1SNKzRZ3bLG4rhfNhLsovYgrZdqFFWwCeiljf1apSsDhKfNTOpnDzMGq23Cy5Phmp8PiLoybZzZdfIFsVXnVRgkFSPy3UXx/VVhODmChkbY+VSN3RkAhQRByjcdVI5pvso5NPUtUTKN0llBzzfzJpTqVETqt44xTFytscDqpmHMqjTdWIdws5Q1jKUjoUOUojso8wXJ0MR0QfVGR0Kax+FZoCnbuns5E1rlKCzFOD1CbKU4apTNiD0RehDEWRqjQQ7Vc5xfh9TiTWspxcg3/ALl0uRqHl63spMDzlvA/EEnmbSjL3L1l4hh1ThtQaacWcvWniot+CLrBrOF2YlV+Lr2G3sVKMXngp5XakgDuSoXMsdF3mKcF4eyMzQSbbjMuKrI4aSZ0G1uiU2rPjey3mH0U9FzObZh17d1WMgcfKia5w9DiD7LQ7irNE2iZDOwNe9osAih4XgqqUF7LLnqbCKoxCqhqxI4dHbKc4/XGHlGYtcOxQaEnDTI9YI9e6FuGUEbTz5gx/VYEmPYkx5DJf71SrcXxCdpM0wIPQMWJhhrMwm2INPMbK15sLLrzQ0VNT8ynmyuAv5TdcPgQmri8SSOsLaDTuuppMNY6LzTv0VpqJclUNqp6+V85LyTuVVrI2MfZj8y3cTpao5neHkAG7raBYrqR8x8rhf3UmFVRlGo3RCWQ7D9AlLSz0/mmZYdCog8DZynEcq9xJujbPIxnL1yqFEHWPpBUFpojdHzG30UcnsEz5i2Ehit4bEytp5GObaVmykCg578uQbKs97mDyq5KOVI6N+jgVRfuVqBII+ZHnuAlzC5uTsmkMOjYy6/W6Zuy6CWHI1+aaMvb+UKSLwEtc38EiM/CTaxVV73A5U2a6DQmq5oqhgjmcHMNo3X1AWpBSYjO4VFbXT373XNBz8zSxuYg3tfddZV1L46KJzDlztXLJpdoI5xUNfPIDD76JY1RROzS8yYAkCzXWH6LFosQqhO1/M+nVdOzC8YrYvxaJxYep0K43xSYcnjOHT4a+JviZXRygnU/891klkhdmay7O67StwOOoe2Pxsr3xiwa/wA1guanw/EWuLPDTho2HKXbHYlI6CSp8PUUsdVymHzkd1FHA+RwY03J2SPh8oY5rs43JCuYJSz1mItgpna9jst2IGc+ll5F8ufRw7oKg0rHhlPrbdbYpH4NUzPrmMkY8WsTcKrWVlIAZMOgbFboFBSlv4Zj4meZhvuulw+tp6+ZkLzcPi102KwoWT1VG+quLsvcWUmCRSVEvPp5gJYPPlPVBDUU/hq+RjWWaHaBTOgjEgnieQ4dkdWIK6sex83Lt1uoaV8kFU+COfnQj3SJVTra+apl5FN+E/qG9VHC+SCOSnnhEmc3BPRSVMLIa500P0KhkLnPuqq4GtliyHqoQeSeSW6KWJ+TLp6d1PiUTHytkp2XbIL3RE3DzqV01W+pcGBjfKCVbwOonq2y1mcB0DyGADouepYnuqRGf6wEW7lWYK6qwEVFIYP2v5uimWI6aWvhqKZ072Bj4zZ4Gy5iTFKlri3yn3IUtRXTUZbymBzJ26hw0JVGJmaMZt1nHAGDzpRLNVRWHwtFv4LWw71PLXMyDQXICo01OyWleTPG0g2Gt1DVRSRRfiNsCbBWYSmzTQsNUIZ3xhtiR591DiGG3z5GvjV2twOiwnB6HEYMRj+8H2vTjUEHZQuL+e+SSQuL7XuueUCrQU8eflznyjcKLEmUME5bRDyLSp5IJZDGenVZlVGyfPfcKxJRRUrjypLWEguFrUsoo4+Tkc4vd5Wja5WVJJLJDFHzS1rBYtCnie980UjzflEH9Fu2nRVmD/dkHKxHmxVEkYfr8QVvgPh2tix+mxmuo4vBSB4a95v5ztZaWK1NLxzQVVXTxup4MMizxvdoXSW2PtouelhxGhwmhr88kcFeQHR5tLrFMtjHuG8Owg1VfiTwx9RP/R2NuQ2P5Bc3DjGHUWF1+HCDPUT6C7S247r0bhmM0mE1lVV4d4ivEwBZlzPcLaDVcRjOF10+IV2L4xSNpHcvPEy+g7Lckww8PxGbB2OjfRwTiezyD0WxhuLwfeUtfPTTcl7fI9ouLrPwmGhq6iqmqqMSs5Nmgmxv7Fd5wnFWYzLNFhrqeSCmjDHQTmw16XCzGLnMsbDa2HFWSS1NVDD8DYnaEqGCHDMSMmHV1VkmhjJgne/KAfks/iXA6/hPFnyVMYY2SQmKFzg8NYfcfNa1a6ip5aYx4BDVmVolu02FliXRzddO+jgjoH4pBWNO5ac3LN+6ypZJA/K7otTHJ48YkfVxULaPlEAwhZscb6x75JJMjIxdzrXstwJISx0bSw3eTb2WphMb5sQ5Na25jaXt9iLFYsFVAyWFk4/CDs9+9lrmviqcQvTnlguyAZdx1WRNUVeLYngvhW1RyRvtymgA6dL9Vm01DV080QMd4h5yO3stGsquXxDyI3Mh8oufzHus6rrZI6h7GSX8xUgX8UkhDWVE8AfyzdpDdCOx7KjUTRucH0pu14uip8ZqKNxnieHEDztcNCOynnNJiHNxDC5oWzSCz6U6Pz9wlDJzyyzlskpIcdSOikcJ43smp7B0Z0ICGKCerqWihgdMbAyAfCT3Sm/Bs2V+Yg2tt/Faoa1TPDMwP5NrAjfuqNGzDYo89cKmXIbZI3WsFVMs8rrR6MjIJUxmMn4bgwDoAFBZke+efn4WyZxj6OdnMYTYaCYpK58n40El8jxYOWbBWyUVQ98c5ia8WJC1oKmke7I4Zw/9oNsx7/NWh0FTHBxBHRVU0nIpZA+ONlrZZbW+qwMSwCekqn0NfDHT1XLMkDzqXhBSunw3FaZhfLJEycSCGTY+11o8WzT4xUR4rJM4+u8ZtaJl9ACuuKsvh8RUNb/8RraikEsdopI9cpO9/bZSY/hhY2jjpXxztmcQ2Rh0kIKzJBYxyMJkZe8jSdbKepr6aWqidBBJ4OCTO2nfJ/iEnNCioxG1z6qFzXuu2O+g03VCQct5Y7cbrsWcWYThuF1DMHw2ohraj43Wcz6X6Llp5J8SqhNWTxROfZguAAApa2CGqn8SJ3vvZuQaLSpKSlipM8zBzZCX5jryx3soMSw+theJzAx0FrcyE3jU2MZoJ6fwk5MJhD2tNvLdBUqpmVR5hfd3Q91UdopXO1LdM3sob3dlRRNIN7gHTqrUBMTuY2+Yi1wNlXygOI7FXqSB8zcrNDyzoUmGVpzvE07pKiGSZgjsCw2se57o6KkmqMLNqKSd0klg5u9/f2VKE1+SSGiDpS9mfI0a5FHFWPiDWMmkhcDcOabC47rnRbSnw3GKF4oJqAxveckQtmDj2BW9g1ZgbcLo4Kt8kc9MXirphnDJxf4zex2XOz1ddH4XEoK+R1ZFfmOkdezwbaLveDuIMDlpZKTEcKhhqK082UPj5jZel476i/ZbocxxZieEYo6khwvDYaOWAFkpht5huNB/epMMiwbHMNrcRxiSTx1JTvLHMiuNPQXKzxVFiMVVV0WEcPikio3eIbKW2eYiLddxe59lxf4zD5JZY2vbZzRcB3zQWYudV2bYulfoABuVsY9hlLT4XQyMZy6mOBkVRTjR/NDzcvHyssyjrn0jHhrWFryw5uoLDcLrsdbBR1NXiWIMs7EaFk8fcPkGo/ipLSTCsGnxnCMT4gosQdDR0Rj58HLIeQWbhdBgXEPFzq37ng5kOLStIZU1sxu6MagAbXtqsLhHH/C4biOBvfajxelJc9+r4ZWDyD5G1l09fQMl4VwXiLEp5ZZZqxh50biDFTlh7bZCFjEdTh/F+I0OKUuAcTYjFLPVuEbZ46Qt85+C/wCbULtZI2SMySC4uD9QbheFV9VjQZRYx90SVUOGVTJRVhxcXkPuC9g2uu/wrj6jq6M4pPS8iaQZzATmEh7t/Iuo7KTc/NQyZli4dxlS1uJMwqvoZ8PmqG8ylMurJx7Hv7LbdZSADHm6nbI2yitrdNqCtImdKAPM5U5n5ibbJ3uddMN0DMPwlVJIZ+eXZbK+1tzoiEdzqrBLLJkjOUo2vvoVcqoLgnIs2WCaNpe3YLbKyDdO6NrhayqU01iM6vNcHAWVkQSRfhOC5XEaVzJHPa8O+S7A6rma+JsXNc0WGc3XXT/E55MppNkUMjWAteAfmo7uBTFq9bgN743E5AAnbHGSOZUBvsNyoBuUnbrSYpXRZT6wR3CtGWN0IjB81lnhxHREHu6JSWaRha8h3RRk+wRvuTmcgQENUTS78yHMEswVpmFltS8dVM2p/M5Uc5B8wRB9+ixxb5NWOr97KQVR/OVkCRw9KmjlceqlN21mzxu0I3Qy0cU2t7HuFQY93dTNqi0ZSVks0tK+P3UUkJtcBX45hINSndEw+gghLKY591G8WWnPSGxIaqboiB5lqJYmFUORJ3MQ5QtoIHVJxuUySAgHXUgcog+/RE1yHJLnTl4A8yBW8MpIa2qbBMbCxvos5NQKnwnFZ4RUw034b9Q6/RWqXn0gDKhhDguopKOOOmjja3RjbBVcQwmSX8SNmy8mW13jFzcrXueXtG6jvrlctpsQp22qYS0dyqk9HEXF8bbXWsdtpOCoGRnYp8j/AIUwjdFIY3vB13CvziMQtMYANui6zmxSszOpC8jYJ4xVhoe2le5vySklucz4XR/7FguVt8RskuNiPmpWm52SZRTiIzPZZtrhRskba9iFmxbYy6kEYCrB/sQrpo6oNB5ZsRuo2ANRBzRuo5YJ4QC43BKhdI4LIvWituoJGdgoOc/8ykjqR1WhAb39KNr22yndTPMDvQ7zdkzYmk3VSkJfYom1UY9SaWnflNgqpiePUFbS18VLPzJF0b1UZrpZSAXUU0kEcg1VKWnew5cq0w1C5sTvXqtRKUxTE4m1lG+J46LblZBl8jBdVSxp9RXWJYmGY0G+VW+S9oAc9p+SkNO3cBIRtb0VZohE4pjCSVK0oyR1WJlpAIiN1Zg8hQjVSsCxm0uxWcPMj5LCqrCQpWzPCw2lELQkBZPHK0nK5T8phXMVzDcIG0zD01V0MAGyJWxQ8GR0UElLZxsFrWugdGx3wpYyREi5BPRXhTsupmwsHwpYzDTm2yrupDmvdbhbH+VROiZ+VXklMWWj0vlWbM3LIQullibbLZZ01Gwk5mrrjmxOLIaHE6JOarMsHKNlA4WXWJc5gKciyEmyLUlaAu3TIiHJvN3UDAuRg2RCDS5cAjEUfR91Qo73CnbmCha5jD5VIZQ5ZlRF5KNrtFBzGd04lb0cFlEpKa6DmX6qQDypMqIbZkiShB1sjDLhQgIueilsQFGdDoiEjrKNC19lIy5QhtxdTU5AOVZDZXdkJuCr/ksq8jGE5lm2kDX66qdkgsoXssMwQCSx1Si16OQKZpaTbus4Sq3DMNCVmltwdfG/DceaWNs3Ney7Zv40LJraPF1zHHTGtq4KtgtdbOA1ni8IZrqwWWxJKN1CTZTSm5UTgtQzIY5NdVaY4EKsWI4nWSmeSd2ygmZe6kc8BCTdqNqE4IBVY3V+VoKqyMANg1bxlykEZylW43XtYqs1utip4wwelJhYTgkjVPoFG6djQonVGdc/DbtYzN7p80bNbhVA4d0g4HcrXhpyXBKzupGBpVAEDZEKh7FJwTnK8WN7ofIDq4BVDWPKDM+UqeGvKV0ywj4wmEsfdVRC6+ysRUrrXNwp4cHmsRuRgeyCKN17OVtsIsFjKG4QBjjs1J9PdvmatGOmBGjVYFGOouuUy04rE8EpKjM99dMx5+AbBVI+FcHgYKitkjk936ldjimEUsrC+MWeNlgtpfDS2xCVoYO6ljg8dosAk5k2HOLchsWgaBYNHSvqHnIy9t12XEND97VTqTBIGtaNXO6XPf8ARcxLguI4XPkmfED/AGt0bdBUGgZhJZC9sLgNW31K492dz3PB0JVupFVkHOF2+yrh2lrKMBAF8xWjDQxysHLnizHoVnWzvDO606HA8z7+OA+Sgt0f/wALa48xr3n4Qp4OJ2Ut2yMFz0vt/BZtVHHR1Ok/NsNdVdq8SwaopTlw6Nsh2y62WW0OIcSPr2GEPDWHoAqEM9FE4SPjzPHW6ytb7qzTunY7NHTGY9gFbFmtrGVzckcZDuxWZLTyw25jLX2V+KOsqKpv9F5J632WzDh1JUOEdVURNf0G6o8saWlt0OYFCyRnLvcIDOSMrWgJCwnOrLJUtRLS1TZoz6OndRNe4t1KY91EaOLiCbLWwD1DzfNZROYohLI1pYDoeiZIWC8rhdD0SA1ROjfa4F10iT1Ru2UrKeQx8wEZfmq5zdWOHzCNkj7ZbmyIuYZhc+Jl7KadjZGbNdu75K5iFVUh0NDUQuifBob9VQwypNHiENXe3KN1oYniBxzFWzjLG0WspS22MDkgwhjsRndHK4izW9E384sbqKrmOqD7NboAEP3pSxP5EeHQG3XZSy4nhtKzmU7D4n2Gy88wosOxCqbOfxiTzOvRWavGsfoahz45I5IxuHtuAsujfy6Y1Ujg17zcXKHE6t81CXtkUpFWU4fPUOnrqmQuJuWsboFPh+JU9NUCDDqCQsf+0cwfiPCzRSVfK572XZ3upqeZraduSM+Jb6Xt3C64o6+iwrDMYrudiUbqWGJmomdlusPGo8LixUspJ2zUnxCMX0VR0uIzxOZUUU5J+N7itHA6QUNPJiNRStkcwgRNJ0utCaXEMFprx0eGzxOkAIdyiAVBhjuRO+eXlNMm5B0A+StY/iuKPjp56qWF234LW7LIgpo6sl72XLtgk5CtitE2GqNXBIJIj1CpMc9spe3S66dtHFJQy0QbYRgELmH80zCMauUsscz5A7JJt0UlHGAXVB9MYuVHUuFRIHTaOYLWC06Knj+56gOG91BmuJD3OYbNJuAtOjkhdhwgzWd0NtlUooecDHI21he/dPEySlqXZIHvZpdt1mwnQTUczJ3MuY3ZwehVSrbVYnUB8jWlz/hA1K1JpppYnCOAuaNS0bgKlDBVGRlXRSWI9JedR9FrkK00kzP6PVOcDHs0jZBG4AZTGXNPQLap8JkxKuLsYklMr4+YwNbYH6rFzZRLGAQ1hsA4a/VdcZVOJo4YxlAL+x6KOolfVgR2sB0uoOYHlHC4h1wVJISPidEWZwS62lyr9PM7KS/sqZNyC7Ww0UZMl/O429lnKEX3NmhqGiHUvF1JFTwVDZTncJo+g6qOnPMDX/kUmVsbuZ8XdcqWx4E1k2KiKQXbkKie50dVNGw6MNrrQwqCB9VeCo5cgF/Tos2by1UuXz55LNPQlaiVbP3pSnBhSCmDZac3Eodqb9CAjxHGHiajpcblIo44RJDHGbi/+CzocGZ4pkNTVNjfLa4v+z+a2McwfwzGCrninZTxWziOxA/VVlY4fvNxJBPJVV0FLK4RySNkIkAOgB9l1X2k8OYVR4W2CkjlmrZPxObNIXPDB/h7LA4b1wl1RVuL2y+cuJ3sgxbi/HMVovuwvEJkGQOGsjo+xKk+Q5tsNFNSGGCflufqdbXKDhhk7cTdDHVVUDxcskik1BH/ALqq9klNK2/wen2UuF1E0U088b8tnamy5jsKWrxaqmfJjFQ+r05YE/mBHt2XP4nUyF5/pkofHIYo2DQNZfYK5hFWXyOrb8x3NGp2I+S6Wkwbhysr/u6fF6OSaoZz25yCGv7BX1aeZz1D4myU7Ro43JO6gpmZniMveGvIYbHcLtj9njKmqrKOKtjmljPlkiP4bR2WVw/wvjtXVyPw6iu6juJDPoL9vmtxFEwx6mnfSymFw9CeSWSWMTPk5VhZpb17qziEpnlNNVUnh6yMlk+u6zJwY3ZOiSlLuHSwGVsuKSPkaNtLro5cLwrFKKKooRaWmIMo3kI+S5QEBlgtilZK6mdWQT8khgyu6k9lIQHFMMEGJRS08cYgqIwcrT19x0WXV0NTQPiqp6GRsTnfhudoJPku/wCA8NoMQxKixusrqUSwOfHNTz6F2lgR3W19pGA11ZX0FfIYnUdOwgtMZLGje57rtwHlNDWT09TLJTQOHPFpADezOpUVa6mNU4iQujvcG3XsrdfHJHiLnUFLdokJGQfhyD37f8FTrDJNKaoU/KZIbhua/wDFSRYbWeeSPw2UyCxc7qpG4biDpG8uklL9XssNCBuq8E745hWCFsgi3B2suiNM/FYcN+7XGllefDucCXMjJ2v81kcvPV814McEceliNyT3UlO57AZ2QvOTc22XTY5gFVhzIMOxHDoo62K74ZILZJYxuTqs12SonDaepMcD4yXc4ZQD2Z7KqGcVtbRR11v2El2m+rSPZaeLy0Uvg5I3iSOVvMlYD6ZAs6jdC/CqiMSefPdumtu6s0XgamjZDPQ3bGeWDfLdZRXlw0vbUVUcYjbys4sNwqkELKS5qoxNTyeQlrrEfJdLjMcdJEYKSPyPiyXJsGg9yufM4o4YqQ0NLM1nnL73J+qBq2hlw7OIJOZCHCxIuAT0UElPPEGuq47F4uPcK9W4rNidNBQmQRmPoG+o91CMYf4MUklPmlYSM+9h8lQNDUGkpaqlkgkdFOL+XYPUMRqsQr2RmaRjQy5LTY2CaebMzJG9w+SejlfEWV20BvTyO/KT1VVRlljdUO5dxr13+qHaz+t7qziVU+rqxflZYBymlkYZmA6lVxZzi15toSqHzal3dXIJjkzwzGOVmrSVng3UjQ4nKtC9T11RTvkMD8vNbkJG4+R6LSwCsZhk88kzIiyeLI4vjzFvW4Hf2WQ9jGNYx1w7exCl5ri3zrEwUkdCytra6eGrc+IOMkbmiwc8+3RaVLjOIU76Cqwqlj8Zh8l8uUuN+lx76qvR0r4aeSR7HxsfbUHca6KKOVkVWKunkL5ebazm2FuxQdViHH3FePRSYaIaOoL4Cx3h4S0gdRqTsubxAwnD4aTxEUs0fnNj6Qfg+d1nk1AqTyGSGXMTZpWhiWFVrHwGrraQvkLI2sbcEA9dQL+5UVYjpKqClcyiooJXsBiqxIRZsj9gCfqnrK2onwyGSve+oPLZBn25cbNmaKkKiqweqq8PyRyh9o52X1IHQf5rawDDoMRpKpkEcNVFFtG9xbKOpyd/kpLCtFCz7vZM0ta8xvZ5DoWC2/dd7RY7NhmDUuCUcUsD62JjC18YcGkkAkE7BcNieHz0GICCgZaIycpvNs0uNrn6e6ipcSno6yGtdiJPIeDrrp+T5LF01b1GOtxI41HwxhuI8yWoB58j4vxGxsILQQPrqrsHDkv8/YqaOQNpg0VMkNtnkagfpdYGBYzS0HHQxV88rGT8tkchN4hG+w89th2+i7yimFVxpidQz/q0DIye1/8A2Vwllp8Q4BT8RUXhXzGCpiPNpaho1ikGx+SrcP4vU18EtJizRDilF+HUR7ZgNpB7FavM1WJxJQVUssGP4Oy+JUAtyhtVRdYz39l0aiW+3ZN1+qp4ZidLjFBFiNE+7JLgtO8bxuw+4U7TqqFKLKISWKnfqLqu9mvlULTtkHdTRvB6rOGZpUsUjgtUWvkB4UEzG5Dog8Rb4lHJOHNypZTMqBlkNtk8FTYgXT1HrQCL4guiNAG6xK6aGGpex7c/cBaYly+pUqynhmeZreY7rWr+JjJz1W0Z+YzY9FE30q1KwjUtVV7Xh2UtXuxebMDhYplJcW1S5eui0yjyeyYNLSpxGTbsmewtSwDWE7hRFjmGxVlu6MMYXHMNCpZxUVYpqV0zruaQ219t04gh9L3XHyWtFiEbI2sG1rWt0TKVxhFDHCy0bowfmEddhQc1s9JDlv6xdTEUrpRMwZPayvsqoGQZLOK5ulOf8I0MLHu852Cq2IFl0csUE0XOayzgsurpd3xMViUpnhxupBIepV1lBG4BwcfdG/BxK4llTl7NypyhmpUmy/lKsR1T4yqJbNC97HMILDqja4u3WlaYqGvGrkiyGQKgD+VyljlLdypQKahadY2kqk+MtdYjVaLam2gckWRy+ctuUgpmZfdNlPdWXQ22QOjsNlu2OKBGnylFld3VkMrmFiaOtZPHa8Z1BO4VVosrlIeU4SAX9lzy9G4d1SOc6MOforTicuVYtBUNLW3cteKS6+flD1QqV1IKqIxvFwqkWG0zXh8kYe4CwJWyQLKPKxS1c1U4VGyR8ojI1uhoqWapmZCxl2g3K6KWNh9TVFQ0zKORxjc4gm4B6LXiC0ISP2jBf5JGCNwsWD9EeclO5wsuVytE2CK1i0H6KrPhEEz2vsfIb27qzG8XUvNanKSlZ9DTltvDt/RR8stJaTp0Vx0rTsoTYpclK0tLzmln8Vkz4RiLZDkylnQgrfaOyWUFajOmacm9lRC4tkjIUPMmccrGPJ7Bq6+aigqGGN7bgpqfDKSlbljistxstODm+RIwZi+7u1lYpKOqnv5gy3ddH4WG2rAU4ggj8zGAFJ2LwZpw18n7F2X+0q9VQTwszvZmb1LRstzT2SyN7qeIcHMMgfK6zIzf3CsNwuZ1+YS36LafYHQBNcK+IcWHPhz42XbdZ78zDlsutDInbi65nE4JYqt4EZLSbgDotY5FKpfdRnYIpAR5iLKNznWXeHGR50zt1DzdcykbIwnzKoZ4ktcDRM15+JTskYdOiMRRnZJ8hE2QhTRPJOqXI7BEyEh2ZYmbbWwW5NVE5wU7IwW2IRCJgGyxKoYy6+VW2yOta6qTAh/kCYPf3XNV8OR+UqlHMdiVKJHdHILGUoS7ogDndU+a5UaFmCkYC72UYy3UrHBZU/L+aExqy1hcPKUXJd2UsZ8kD7ZrJNo2uFzdaQhPxNS5YHVLGBiGETWL42X9lmNwfEJCbQW/tGy7IMzIXRiy6YbcnPi48cPV8jwLRt98yaqwSaj3lbJ8gus5dz5VA+kznzsBWvHyZ4OScWfENlHmB9K2ZsEqgXvLoyN7BUDhdSNWEOXeNkMzhKo5x2SbHN+QrZgw+NgBkF3KU0kd8zGp4sQcJYLoZpHhgjkudtF0tDw5QNjD6qWSR/UXshpoLzDP0Wo0ZDZcct0N8JNHhuEQgtZRx699ULsFwmVtvCtb7gqQxF5u0onXYAGrHOWmZLwtSD9jPKPrdVJsFqoGXYGyfIrdEtypWtjeNU5yOS8BW5rup7D5qaOnnaNWfxXRTCEFU5cnwLXiyzTIkpJic1lE6CVg8wWjM+11TdIXDUrcfmRVJeDuUYe4dSo5SMyAHqF0pi1sTv0bnUnNv8Sp3PdOHt6qUWtZ3EKB79UYlZb1IOS+Z3k2UpTCRSMkIUkWFvOr5AFKcNttNf6pcLTJ4kp/FYW99rmMgqhwjVWY+n7XXQVtE+WilhHxC/6LjsCn8NiOS9tbLErDq5rhyBz9BZWKltzdqqv01W4SQF5uozMRs5Jx1ULtV1iHNZZNm6oxIeipBxBRiYqcS1pzrqByAyu/Mh5qtFnchzO7JzIhzBUIkndMlmbdASStRBYnOKNoJUd9bqeJzBZKBhjj6QnbA862VmJzLZlOHNXKZFdlG0i5U0ULI/SFJmKZYtviLIy+bLqjvog83ZOsNJG7qVpuFA3ZTs9KxKysU8rw8K/HUA7uWY0tan5juy5zCw03yQu9ax8TwPDcQlbLPVSggWs0qXmuTZmncKU053F5JoG+Cw7Ds7Bu+2qgxTh/D4sME1dHepIuT2XWMDCdlKaOmq2cudlwjTxStqMNidkjbLb31WTGGSVALm3Y82svWOMOEMDnoi+GNwqmG4sd15rWYRDhpHiJpA7sg1YMLwHmtNPSGR/QA6qXGY3UlGXw4dNE/o5yo4ZxJS4W7meEL/mVUxziWbGCGxukjY03y3WRlGqljBblFzqbi6j58h6/wUzaOtqWmSKAyNA3UZglyW5DgRvpdLEJIXa8G0dJDB4ifFI43kWI2v8AxXDta4oouSyQA6INniiHLXOfDMZW9xIFDhE8kTwAwH3IuUTKvC2sDTG/5hTUjXV04jw2En56KDyou1RDVNlKQNl0WEry6MXchMpQyTF0YY7ogv7LMweqzkErRy2+bqnZC9vrbZBA9sYN1bHmYLpxKBFC0jyt1TSiSLzPjOXupoYHTfhskynurrY446YQTzNkt2UstmPq5pA0RizWC11Xc0uN8o+i1DNTj8JlK0juUcMEU0gjyAE9kstiqxTtMZEgGo2WjHSMgrBBUwNN9iGoKynfSTct7LX1HuEssEMs7qoSPpec3srM8nPfn8LyfZV4ap8Dw5mvsruV845ogMd+hWJVUllDByj1Uckb4Yg6dhaH+kpqkXeLuAA3WnXTufh0bbNyM291IihmQsm1zvJj6AlTxMkDg6DcbKsJHEK9h0vLk9AJOxPRaGtj+IyjDYmCTzyAZ9OqyDi1Z4YUL6oiJ24HVbIrcKkpxHiFO5x6jZUqd+DUlVzqeijkDvRm6LLLOkEz3F+cu+ZVzCHviqHGbTy6LSlx+OJ0goqCON0gsS7XXuo6jwD6Hnyf6Yfg6LMquYXNymSPIjkMl7+yv4tBg1PTRxVFFGOYCWOay1j8wuaiqnxR5WLRo8efLH4WusWDZxF9FlGE6ibHIRHqr03/AMqMY9RIYPmtCWehDuTZkcZ2eBrdRy0VLS0pdV1ILGuz6D+C1EqqQ4FikcTGMjZC5hvmc/daUVPyXZ5JgZeoY7RUD4WrPLgke0d3GyuWoYnCGnZtu6+qxnIGripoWNkbHq91nG+4KmODUVdFHJhcnJltbLfQqnVxSTWyEkDoTsqs0+K0kQFK+zb2ss8ktqR41W4TUieqj8S2NvLc21rH5rmqhv3pWyyj8IyecNctSlxCpmhfS4hO6UvN7uAKakglqJ3PhpI3UguyU7EL065Vzk8M1PJklZY9PdAxzg4LdrW0UtQ6hpWmWC/lcfhf81jujdE8skYQWm2q2Jy05SWakC6ctYKdkzXkuO47KWjgFROwPHl6p3SuDpYg0BjTsFFFSTMhhkc/4xYLbZhr4qOHEaiGR1G/42GxXOH0E/D0W7PC+LAWxxVEvms8MLtFiYZWmYcxrH1FK3nMkFgdif8ANUTRMfWU7GPEYZ5rE/wTUuLSxYWGvd5o5fKeq6SHBqXEYmYi+ExkND22de4WGkHCFVAeLJ8PxOnjkbWR8sZtcp30Vupw2OakqKGsxEU1GyqJ1dnJA21/zXMSgQ1grrvE0brgjsqdcyaoc6rfqHnPJ/musZpMLbsWkDjheE1sk0XOs1zugv2UuJ1chrW5nRiVlmF8OhcCsSji9VUHgDpc2urhgE9UKhkwc+WQEgdFUpoVmEPNfHBTSZnvaH2cd7qWgwCWWvfhhDqUSRZyRYgHuVLHP4iatr2G01GLRd22U+BYvPhFXO+oj5sNZEWFw3jXOYA+CbFgk8dJBGJaaQc1zDq73CzJcCngpXVtZL4aV7vwoza5uujruJMO+6Jqaho5CGAMBfubFcpXYlVYofHVBBlBvG1osG27KUmLrKTiGg4HwN+DeFqXYjOHvJB8hJ6g/wCCw5+O+J6qnZTQVApYsoZII26ye9ysetxGtr2+IrpnyTkWJPYbKGnqeTJFNa/LN7HYrti1AZ2TMfzJJw6+7s9ymji5sMj3v1jICv4tLzKxlXJSRxcxzH5GOuLIcUqsJnqJH4XTyBk4BcBs0hWYVmuJyabrSlxHnYdTUULCGRi779Ss15Z6m6oi6Rps02ba9lmYSk9LOymqmzuAdkN7FdhScXYtBQPxKpqDUQuNvDOl8jQf491wwYXutcD5q/T1RihbBPBmi15oZvIFqJYl1mJ4rT1VVDSz4jDQs5BIfB+z1HVZpwrhmiw+poX4zDV1jxeKcX5cXzKzMLwyHEIqumpKKUuI5kD72DWDe6gxakmw0x05jaGTtz6a3+vVIUDqmCKEx0oBMgLJHHUEey6jAa3EpMGnrsLoAG0Udp6oaXOnTrZcW7QE/l6L1jCsRopOEq/h+iws0YgojOby5rl+vbqpMqq8U1cB4Pw+vxKgixCumjMYnnJu25uSAF5943MxjJImuyCwutXEsZgjpqekuJ72leHjRmmw7b/wWLWNPic7Y8sbwCNVLW1uLFKilOSnEbAdxl3CuUE8ctTGckkLGaaHQHusefSpHYNC16ONlPTiqGbJUaZj8JHUKZi1iOJSSwSU5farp3guD9Bb/FY1RXSVDmcxzS/W9m2VhzWVFc8Vb+a7o9up/VNHhkhr4ooQL3vqd1iEVHujY2+0g6p5p2OiZblyGQXc7c3UtTh8hrHs5JaLZ7nYjuFRkLA4tYcze9luATRkY55cCBqVEJpnxGkzfhGTmFvupORAWPfmkab6Brb2HcqItnhuxzXNuL2I3WlI6JtCFqUvD9bX0dJV0ThN4lxYWjQxn3VSuoKzDq6XDKunc2aBxZIG6gH5pYrDcqSJzM7c78oJsT2W1wdg1Lj2Ntw2rzBskT8pGn4nQX6KOvwh/DWNzYbiEfngtmOtjcX0WhYkw5lHhsWMt/plNUB8fnjILX3sN1nMZK6PxGT8KN1i6/VdJgVQ/GWM4Vqq2npcIZKJqqZ7rG3YX6kpsbwKiwoc/Da+OWlqKh7IY76gabu2J1XM5sTkQNppx4iaKYuGVoOjgb3uqkk14gxWcQjq45zGRr0sQQVWnMcp/Dh5QsARmvqqNGjZV0NNFV0rHTVJBPLMe5HX3W5T4jjNXhcL8ZwuOOnZIKinq3jlkEanTrsszDqmCeOWoxL8JtPFHDmZudd1LxDjUuLUlJhdBX8yjp2ABhFi436/JER4riGHVlT42le6eqln5jncvKGs7fKyry1lTFeelfyJhICJI9A0jqCsyleyeURl5DztbqVfZNalNKWXaTcm/VBt8jEMdwqqx+rxdslRRxiVrA0EuF7akWt/wWaYqV1FDJVzZXTxkt0vHclW8H5UMEuHVekOKxWD+rCw3Hz6KSm4exGup5aGKSI8h2dpDtL+/YLnMkK2FOmjqRRzz8mMkZtdWi+9uq9j+z98stVjMlROyaU8tjnsNwVY4UwDCZYn4y+hhMtSyPmRujB5MrBZ9vqn4NgjpsZ4hZBGGM8Ro1uw1K1jgOpRA5DooeYlmC0MWrD+GMUkxumj/wDhte8Cua0fsZDoJR8+vyC6KPVucag7EbEKu7lyxPpp4WSxSCzmO2I/5K5mj4lw7g+vfwvjlcWUzLPop3i5Eb9mPPsf4IOudskWqOKeGeNs9PPFNFJq18brgj5o+Y1AEjLNVV0ljurjiC2ypTxhdID87TdA6S5vmURDwctkjmC1SGkOYI6fMXZHbIW6qWIZTdWVDVwvDc7Bp1VJ01tHrZc5kkSxq4Na9zgNFdc+bGSs7k5io5KaKUXGiEucnbLk0K9cONITQtG91T9Li0A2HdbIfC9tt1nzsIkSGZhG23xKRrGOIagyflTcwxP12WwzoLOOVE1nlUgkBGZMY7lYFR7SDlCaN7g7dTlpucyEQguvZbRY5j8lmbpRVhGkhuE0QaHlp7IDFkcdFhWjBWsIydEMlREJSBtbdZeaRp0cnu/KSd0oto00v4+W9wdVcYbPb81hxTEOurRqnt9LzZZnFqMm86Jk8T4X+l4suVq4H0kphduFswVU77aqtWUs1Y4hsPnaQQbqRLUsq/siEp7Ky3Caou/ELWN73ujOCV1g+NokB7LrcOFSgbIpmuUb8PxCIXkpnNHuhtJH62kfNVtaAuhMd9EDJHbKwySMnzaLArGF3ZCWPHRaQYw+lyjfSn4Wq8ilSMRusx7fqpWsdE7u3oUvDkPD3DS+q0JaVgaBCQQdQpMrEJKKosfK5bdBUeUXOq52CKSN+y16UPDNV5c/V2xbLZrjUqVuoWM2d/M5Y3V2Gea2Vw07rjLosSx3N1GG2KeSfSyjEqwLF7KNz3EqMSO+I6ImlriipWlFmKBu6K47qKcm6JRlwCQkaFQTt0gbIb5/MhUplOHWTiS5UOYoeZZVU5lcChdKBsojJcIDJZFSc6yfnqo9xGqiM9uqqLrplEZ1X5wPxKQBjhdaFiKW5Cme1jrEgHTqqbAGlSGXL1WQ0tLE7+rb+izqrD6cyFzmm57aK3JUOPVBnB3K3yZpknBS512SFo+SA4PLGfLdw7rYBUjFvxJZ8OHNSwTROLAzZSYdSyTT/jEtYN7hbclLC9xkI8yGzWela8WU8ODS0oyfht1Wc8zRSBkgLCtZsvdBNGJgAsRlLXGFRsxsrUTmOGYoBStZpZHygwZmhWZSISBsRUctNHbMxuqG4BUzSHBYtqlQxEKWNunmVgxj8qEgfC1LKAWp27Jre6YvsVOSj6/RO02KivrdEHKDQhmsBZSNqAX+YrOEhGyfOT6lRqS1DMnlOqq+J91VzBLMEF1lRdO+oYBdyok2T3uEExqWXUE1Td3lKB26iIugIzzfnKFsh1a7qmyIcoTkJGtaVII2FRsNlPnaXeUJYOOK2qkFx0UYefiUgksgV320Tszu3QiRt05kdsNFgFywPSEi57B5UF3jqo5pSG+YrbCKSRoKic+5QukuU8eXMo2oYg8tus58xC6ovjfFyXxtPzTxUtDGeYyljDh1su2vbTnMOQ5xPwn9EbZF0WJwU1RGSYwHDqAubbGb2HdemM4lzmKTE3S6fVGyiJGbOVHLDNEcvRRlNSyMEgDgCPdazTAPSwBYcPrCvtkeOq55OsLhl83l0Cbnd3KsZdN1GZXd1xW2jG9j8ze4svOsUg8BjjmbfiArtWTvb6VyXFjbVgqer26pEp6OuH4sDJO4Vd7FHw9N4jCYznuWaKzKNV0iUmFKRjtVGGfuq3bMfM1GGMHRdeTEwomD5oDGr7wy1rKMQsst8kpQf5Qo7FW5ac2Krua4GxVjJiYNmCWYISO6cbrZQcwTp8pSylELMUgdUspSylPMWGylqnjqe7VSHvstXDcNdV2MjfwupXPKG4k0dQwmykMjT6VdqcKoomZYWWI6rNEbo3WJ2XL1dIiljNdO1Cw6IgTdYlYSx+YqUMN7p4oypRGsS1QMjkQYVIGndFlK5lAy6Jw13ZGGogxGjtYLKVgsVG02UgfopIGdhLDyWAu7leb8X8O1VXMAYeWL3c+916WHpT5JKd7HsYbi2oUR4fW4PgVDSkvqpJJ/mudyhemVfCnDvjZH184ve5a02C47iKmwqkqxDhL3uZrcn6KNrPCbpaup8A6ZrIiNb7rdxnD6TD4PFMLnSvNgHG4K4ejrH0U3Ojke0+y3KziSmq6RjXyyOlYb+ZBRkpecx0xZb2y2VIeCjFqmmDndyVLPj9dUNyZg0ezVnTSvlfnKCZ1KJXEwizDsFucJxT0GIsndo1iKioqOambJTxSydnW0UkdW6iJjezmW97IPGLH4WkpswU0wZGSxrr+6GQwFgbHq/qttATBuqcNRKsk05SrTXODRncFV19kre6T5i5HMQbtKmZIJAQLfRVGkMjyAI4HFjXFvVYmFSSOa2+T1Kaikkjcype/y31FtlVBJKUXNs5jgS0lZpG9i9UyCITU7QXPtv0WfNjMVZCxlUR4iPZwFtFTq8QqjD4TmeS4NlTBuf+CtC1BKyOUyvcrxr3VNmvkdlZsAs6NuYbEfMK7FEWQZ3NWZgGKZte1/L1LBeyZ77YWwX62V/AontppZHtsDdZcjHyE0w9IKNDpmskIjjYZHey1zSTUNHzX0UcTx6zulh4paOjc2haDK/wBTnDUlSiB8mES5CXPlN9SqMuohf4d8hkDr9uiDC6uOFpjkh5l9ja9lM2jqxBkkPlO4UuGwshnH9HklHUNUBYpBFSPa3xUbs4v2VKSq5zh5r2FlJT4Q3F8XdRwRugZfzOcP2Y7lBiVFFhuIeDpJJKpg+PLZOIkomy1jnRQi7kq+I0T8j+wKrDxVHUi3MheVYxOaSsp2R1TBzmG+cafwWJDXfU0QezUXWq+vM2CxNqZBzB0J1Ky6GSNlOyOTVoN7IsQljmcRAyzOgUhCjcah34BEbPdSmWSBuZrrrJM8tO004YY/mFawioBnMFVJpKLDTqnBGlS4w+ikE0kfMZ8Qt0VuSroJ2l9LLHMH7tcLGMf4rFr28uOWL8oQYO5pfkP5SscFpbrI2CLn008YyG977qeGt5NG+kh5ZD/Pdu4+aox0dUxjo6h8bYyb73smpIjTeJuNBsu2KGb5opYW+o6qIObW0pjmkAq72BdsVdpKNjqY1Tpw10gIa0jcrPkjgzljHx+X8oW4lSp5nYdI6OZ2We1imcHvqyxo3F1fqsKnfTQ4z4UvsPxLakjuqjCWnxpY5rDoDawv7IIZI5Q4sf6Rsuha2llwl/3jJEJWRfgtDrm6yaWOTE2FkNg69tStfh2ggbj0M9c2B9LSC02thc/+ykwjOYzPC1nZgWhhU8cLnGZhIYL3Cv8AEX83IJ+dhckkhld5mgGwPsq9HDhuSZmY898dwCVyyQXiqeoe0zsEUEhsB1VTFsMqIKpsrGDwbjYa3A+arVlUzO2nZ6YjdWaWqfllZBV8yjki2k1t8lIahj4nCKeYxtkZJ7t2VjCskNTCHus2Rh83UFVZpGGA0bMjyJL5rf3KanhkzMZNPG1oNm3PVbmUW5I6mgnmfDNHIyduSTKdgjwqYS1YhqHXaQSFstw2COglmxGOJ7Y4fw8juq5nmPmjYLsby78sgag/NZaaDqd4oaqqZFam5mQOJ0B7LOopBy2sFiYze/QrUwbEvuec0tVIZmVB/EY/YlT4jw4ymnlxGkmEdNOLxMaM1z2HZahlgYhWPrKhwe0Bw2t1VRubMWHdm4XURYRNRYTE6OlMlXOHggRkvI9wsuSnqsHy/eNDJDNUR3cDuQumIhZG+eOjje6w5nLzHZo91erM7ZeRRMBym/NboFQkqIpG2ggMTexN9VZpJaU0ro6iYxPiaLEKS0rSxTSVLnvZ5rXKgllM8jA63Zagmoa+odHHHLEHx2ytNzp7rNq4ZKJrJ81nknTsqJZ6aRk4pmSMkdYG40AHupI6SSjkZ4tgLJG52WO4Wd4qrkDyx+sgs49wrYqX1dJRQ5wDBG9lzp1VhmYW6LHqnAKl33QYiypF3tfHdsZ/X3Wk/HG4lJT0VXSx+FGsjWeY/O9tAudqGMbK5jH52gXBta6lZKyVrp2TmO7cmXqtQlL2LYHHJTzY3h00Ap+eIQzmE231udFc4fxrGaSV08wgfS1A5FRHYXMYCpVOFTRYFHiNDVT1Eea8sPwNPcrTxbgtrMJp8co63my1TQ9tI03fqL73UmFZOIU8kj5/ARCSijdo87tCzRLKIuXnLmX6q1TyxS05pzJlJJFzsFXY1hiIMjwRsB1Sg8eeeTK1qtNlkkibSPnzMGwadbKWjfC6JsUgjgIvZx6/NRywedj6WSEFgtcdVnKAME9XRukjpTlMhvm6hWS6orYIXvkM1QzQkM/EBVZkZdIGPqY856BWo62ekcadj3lrANW7/VYsW45RIGRySPFhYFuhVCroZ6M3ijc5n9YXMvqrlHJSPaZKuI5HiwcdLKaorBDTxiiqHPaw6ucPUOyxbbHlvNTl0cjBYXI6lPNI+poTW1rzK9mjXNOkZ7OCuYgWVdZTuoqWzpDctZtdaAwOqfRVNDDAat1ROJJuTqIR2K3jkxLR+zJ+NYdjbZ2YVUSYcG/jER35d/j9/ku2HCvD2EY43EsWx5tLGZZJYqdm7s9tzrotrguXA4A3C6WuIrYI+W+KYhsjgBpfvouUxvEXYVxlWVeIVsRw81TGSxuiEh9Gmh2C60tg/mrTRYpiuHYBaehndHJBU84OkbINSQeoudfmqPGuGVXEvElNh0c8Ek9BQsjxGoDMsbSCd9e39y53ifHRjbuRDdkMU75ISPKQw7C30/itTgrHJsNpavBqWqgpqytnZLFPOLttsQSpaO04d4N4ShoZfBUsOIvZaTPIc0lrbjt/wT8TnBp8INBW00k0UoDzNHYiHUansdVzmKycScD1tTXSVdPWOxGCSninYQAALG4A7X/isRuKsq+H6bh/BqasmllJ8Q9rS4kEbH9NFcBjY7h8+A40aczxSyMtK1wdpIDrssqOUOlPM8rXkk22C2cbq5qyup8RxLLlDeTy7WkAYOoWZB4E+JfUZm/hHktB3f01TKCCaYQx0efoSutxfgaGTBqPi3BKo1uHyNz1ZjADoXjf+K46me3M4EXztLF0/CNXieHvhwikr5KSjxE8moLow4PvfZvvsPmsUSy6907cQp5G4YY+VTghoFjJGdpLe6hpXUsVJSSOIe6R0j3NvqLHQFdxxrR1te6Piahw+emoYmxUdO6aEtzMFxzBf4dh3Wdh4JwvE4K+igqPBwh8YdHaxN/70GbhuC1eOxOqqOgFVMzM/lwSG7bdx0B/wVjC6t2HYg5kj7xVLnsmhdoYzs+4/VLgjFq3hvEo62nDhFMRHNGf6wa2/Rd5Nw1hXF1OcRbRT0tZVnn8zfKdARv7LEQJvs6xgNdLAaiaZ4PLiY/aSO+hv3C6PhcWxbHn96m39653h7AP5u1VU6OPLeNjGOcbkG2pHbor/DtZ4Cid1dObl3UkErvhrmrHWudZO14KyaTFGVMpiuc1rhTuqgx1s1lqcKZiWkx115/9onDuIOxKLHcPo5KpksIgmaBcxEbadtT+i7WOe/pU8cizS24L7NqvEIMQfhfhqhtNKC8MI8jCL6+y9DJsoYYoKdpjp4I4mk3OUWuVNmupShLio3G5TkIbKgFG/VSEdCo8wByu3W8UNexVhliAVULwETZy0aILZ2VWXU+bdWA4EKvUWabpHqyzamPIS6yqcxt9RmWq9rJRlVKehy+dgXqiXOYVMzg7TZO52Ya7pntdHuFC+V19NAukOcpHEj0oHy8weduqHm/mT/huXRhHcjZyNs5TFvuhIWVtMJGOF72KWaxUGoRhrxHncNEVabKPqjAZJ9VRu5SNkIKKmlga1hLGE5VC6MtYHu2Vlpd1kHyVh1DWeHE01M8NJ0B3WLWmRaymguTayOWm0zM2UlM+OHWRmYjoiLNPz2uB5W3UqaaeoF3BgKouqqqV/kYAEMvigbOcs0tp4ap7pRzLBblHOxkQcVz9JQVlW8h1mjvdacOGRwWvJISPdYyaxPiuKwxSNERvcXKxqyuZUABx2W9kpfjjDvmpmQ0lrimj/RMcuKTDlA9482UgI/ERjcrp3UUEzwZo2kA6Dopo8OoSSXU8dj7LfiwvFyzKljdnrQo466qtyYCWHqdF0TMNooXh0VPGD8lYc/L6bD6LjlstunPy0NTC0PqIsoOl7pmRuAs3Za1a1ssVlmguY4ghSJsmE1OGlwuFdjYoKfl5gr8YZlC5Z+reKEMGa4GqmaQAmIvsmcCAubZpHglMDZDlCcDoEBZ3ApB5CYtcmRU4lchzklRFybmAKIsPla0KIyOKjMlymDwgnbNYepSMlufUqRk1SbLrugvF4CidJpmVZ1RY7qOSfyHVKFrnN/Mk6W43WZziD6kQqvdEXJJDlUJOqgNVpqUua1AZlAOhRx1RGgKrOIJ0TNOUratMTPKcyE+pVYZQdCpi5p2QInqUGayLX2TPGiyGz6qSOTVVzuE4ugtl5LVBIbu8qa7rILG6B8xupG7KJGxaEzTp6k7pGWtdQlyhc+zlLSU0hAN1LC4EAqo91wiilyG3RUXtSUDyBugEqCaQEHK5SxI57Dso93KsJsrrKaOQucFQWUpw1SAaJEWWAFiiSvZNmCBk2YIsoTZEBJ8xQJLa0Z7tFFnd3RHZVpHarCJ+Z+8mLlDnamMvutiQS2KnhfcqlnaVYgfYhQaDrWBQJB92+pC3dSxM2MHXskSAUxflaqz5blQWC9qrTSZkubYKPNdAGUJwbFHZqYtWg4cT0UnOe1uUKAGyd8miBOa6UEFZ0lI8POQK6Hu+FMTZdYyc6Vog5mh3Uha52+qks3smVtKQ2ydAo3G+qldqgeLJMiEuJ6oAdVJkTFiy0YG53WPxZT3oGSN3Y9bLBqosXpufh7/JeyDL4NnJZLATc20C6N7VxHDFRycRazPbOV27irMsoHNsbpg9qle26jEQC1EkwTy0tQN2UjmtQkELfNJgBOijliY4bIzso8zgStQxMK/hn30YkKOXeyttddSh3VWckpnupZmjMWKHlkbha+cndQvjjf8ACrzODOI6JKeSMD3UfL09K6WzRoYpJZAxouCurpQKamY3LZ3ULEwqdkEgDgtV0/NddcdjcLgAl0d8So1mHGJxka8H2VqnksRmQ1s5I8ui8/J0ZeR3ZTwxC93JmtJU8cb+y3ZS3ENAET25QgjNmpOkdsuOToICyJuVQhzkYN1hUiReEmi5ujMV0EJJRsOiWRGyMIEDZOWMlZkk2duiDGqUMFkHHY/9nNFiLZJ6Stkheemq82quFccwyf8AGjtF+e17r3ywWdi9CzEKY0r3Wafa6DxA4IZad1RnkkcN1DR8M11dC+qbaNkd9wdV6zScHUMTMk5kkb87K27hug8LJSMMjWSC26DwSSN8by0oSJD5WC57L0LHuD8IwIeLEzpXdA43C5g0VPO8mOcsPYAoOj4WfW0mFNE8YjFr3G/6LFxLFsDkneHsMrz20WpR4nJRwiGdpmb7CyzMZo6SrtNS0scGt3DNe5QeRubbV/6qWfwxjDoYDGdkdVAx0hewdVJPVvlYI3MbfpYIqiGpKUxlu4sVLR+EEp8aLst/FbiRVTjUp5+XzjyfR0V6hp8KmjLqmeds35ct0RSdokHvYNFJOIGZszHADutmHDxi2HCop2RwRwt9zdBjQ/tA52y0sHpaXEZZIH4l4Yj9mPzFZ3KfK5lNH6nmy9Jk4LwPhelosYfI7m5B6tcxPsg4OswLGBMWR4dPI0fGIzqt7h91BNhNVmw7kvjb62jVxW7Dx3RGkqKW8stRISG3bo0ey50UVfT4VPPh8MxYHfiNj2BUkc+6aqq5GsmGi3KfmYlQ/wBHpieQCH2bbVUMLwPFcZaZqeH8IGxedBf2VmKhrop6inwnnARW5tzv3WZhZRwTywFwY6zFWyTCXnMZdp2VzD2UriYJ45ZZT6Wt1uVfZBxFRztoBhU8bCbZXx6hc6Rj0Qq2TCmgiEks7vK09SrwdUwS8iZk0Mpd5mt2bddFj/B0+DCjkqpJHeIAJmjblEd16TX8J0WOcPUVLSwN5r47CotbKOpJWmnjeOOp6cCKCd3MDRoFbwTD6mFhMdXHHW1DfKwi5AW7j3AVbwhE6cz0uIRSAAlwu8fIdVp8BcO0uHQzY7XMEtQ9p5MAHnb20V4jleE3YRQ45M7H3OZE+4z9L+4S4hnlxmU0mFUkUNDTnmwkDIZPr3Qzyk4xLI+FrRzc5jdqQrGO8Q0M0sTp8NbE1vkzRuF1LZYmKcyOF03Ku4kAZjeyq4ThlRikhkmnjihZ6i7/AJ9ldxiNjaISRyl7ZBcXWRRzvpXEs6iyzl5rMtytw7A6enaykqHOqSbAgrMnpKqncRJGgDpZA2rhYbRm+YBaEVa+qoJOeQMgu2wtqs0eTJHLkP4nRRUDX1GKR5G+VhvdWaqOnyc6AWz7j3V/hXBmYlHUzR1EnOjZoz/iusKpYtNDU1IhjZZ9rE33QUtAIHZp3hp7EqeuhEUbGPhi5j7+dmhOqgiovK6SvZM2EbEv9RVmBaqXxyNYadlt73ddRRyc9k0IHmy790EwojTsNIJI8gs7W11JSUU9JSNqp2ZWSmzT0spTKtT19QaeKBx8kTr2UctCyGfnsd+189gNArTIGeIbJa0J9TugVlop5YjGTd49J6WWOa02qfHSzlYZTQWZHFoJtCVmY1VTYjh0NK1kcMEbs7xfdCYal72YpWzxhvobruFozUE1G6OeF8FR3Zfusqy8Ph5mRmE0ssjI/wBtLbYe60aoQ0FFEc3NkqpLAOGpHdW5uJBhHD+K0M0MMNbXy6Ma3SxCk4G4Ur60R45VCE0+W0ZJ84PcLrEMsmSOGngL5Kfzs2DtCFUwmRkmMM5htnjNvmppcOfVYvXHxcvkkyNEu7x7rMrovBVzoRI1xj2e3/BTiHqWfiFzvXmyuQQzMia6kqwcl7tc3UhdBh+BB1E+St1EmpuNb9ysmsoZaJofI6JwP5dVmBnRxw+I1BDPzKV7pDFy2aDuj8BW5GVLg0skF9DsiAeyF+Rl7ixPYrci62XDqkWqZNGN2tuqYjjkZJIwZWMNmqEttA1nVxsVYe6BjTTQOu0eyxQeWrErQHMZmtYk6qzBiNdDE2CCY5I/OAdx7BUoaSeqqBBTRhzzsC6ysy0r6WdsdSzyssXBKG9V4xU0FQJKGtq6R9TFmcZLSZj7Lna+fEa6V8+Ic19VJ33/AEV+un8VjVO9kF4g0Zdf1V7EIKbGC+qZUyUb4zYvuMgKRJ4aKv4Gf/NyHGMGxGSqaWg1FPIAJGv62XOzPhjLo4RI4uaGHrrfVdhh2DyCJufGxPz9YbG5eOqyXYZA2ITQzSZy7JlB1Kc2ohz0sU9I5khu1zhcELSwOZ8zamOdrZrlhDZxpupp6kUtVFJLT82OIWynV91LicEWJufX0s73SZdYmtNzbv2VjNeKricUFHUDl0hEcw0ja69nnqFmTN5Ya+WEExm+Rymw+F81RkYRDIBcAm6mq4Z2xy1NQGkMNsjhe/zW4klXlxFk4lAoI4TJbl33aPYrVwrB6yrbFW4VSgSQDVzz5JD+RY1XNDVRiSOQ5h/VONy1HhtViMRbS0lVJFGJBK430bbW63DDqcWlxannkw2aSja+WG7oojo0Hp81Uo5cDxGidQR0D6OoijvzxKDIX9gVWx2rwpovRYj4ipkdebTQ+4PRZUkTDaQPvzBcjst8mOIzg9VFq8QBvW0tz/cijwyrLwwPiaOxdqfl3UDYpnRl0bLtZueyu4rP4mWnmhY4mNliR0XFtFXsgY9rIwIyPUCdynpr6U8QaCNBc2zeypk55C7qd07JXx1IkIzADT5qzC+q8JmUVeyQM/EBsRfZBJI2krm8917m8n12VWNvNeZDudytDw1HUAMkIzvIBe46lYmEXponTOyTR5GP/ZWOllWefDUbqF7bSh2do7hSkP5Tp89+UAxpt26/xVKKlnc6zIyZH7EixIWZh0tpTOwvwTK2n5vN15rWnRh7hLA5sdo6p9fS1UlKyS0jg/UOj16dbf4qCKlkY19JNZr+rSUzYKiFzqiAySRZeWWzDUd9EhnJ1PBWOMpqfEYMYma7DZbvbVBlniUnS3z/AMF2+I8BS4/gdFRYhV01PWtcZ31RbeSRljYn6FeLCXLSyUkcjy2RwflOwsuqpPtM4gj4cPDHIEzjEYueSS/IdxZdeSUqcL8KVvEmKz0/iuVSUzsj6gR3Dhrt87Lva3BeBqLDjg0rKWGaVgymoJMmnUE31+S86w/GKykoJMIp3yx8+QGzTaRp6g+y1RQP4oqHTyVU0VZTQ3dLUH8Mgbe+xWbSlWtwXFZ60UsJfPRMcWCY+VguPddNScJz8O01NjIxes/ChNRJkgDhG8d+4WdVz4k6nbg+I1ZlFNapgcQDFIALjU7rSwTjccR4ZiGB4lNJSuq4eXC5+jM5vnYH9OlgtYyU4jiCpjrZ5a6qZzZq6XmxTEWuAALAdFkl/JijeQCx5tcFdLxrhuK0tTRUuJPcWU9OBG9sDxHn7A9fcrGpyxsDfIx7ul05ELeDRYJWUslNiD4w+C/IedLjXU++gUWHVQwfFIpMSp6gRZSQ1pFxfYhS0ldBl5dVRxEgACQN1+o2K16CmbVluLv5DqeNvKLXamw9v8FjkPSeB56Xjfh7+YePSmV1HH4jKSWnJf8ADGm9rg/QLzXEMBxmHF8ZwqgbJJTUYfJzJWFodFH2JHutng3iT+aXGT8Ux5sjmV8BpoalguI9rHJ12svZKPJxbw3y8ZgMcxdJTT5PLmsbG3zFv1TkPnWlqpjTPbppb1ixPy9l7vwbhzJeHsOxGCl5HiKVmeK256n67q7i/B+D4hS0mFUuGxRMbVRyF4GrWDcX99F0UNLT0dOympY+XFELNbfYJElMHEMElq2ula2zg0k+64nCA6WhYwMJczew21Xq41zD9w/3Lzzgyhmq6CeqhmDHRzFmo3BXp1bGcsVamqGU9Y0u6BWKqZkkhlY7QrcqsFZUtyvY0OHpcBsVzVZS1FFLyqhmU9PcLvjNuX8K3BibWNyEHRWYcU5jw0LCzeyljla1dJ1Qxyl1sNTHJaxVgabLmqGra1+YvW9BVxvAu5eTZqmJdcclgx3CAttuFIyWI7OTusdlyp1QkdCqVeXMiBaNb7q+7ZVquPnRFi1CMpk2c+pWYpABZyryUhEzZNRkFjpv7pSFrIjI12y71yYloicDZUaupdmJJVTxpaM2ZQPndLfMt4avizOSyKvrdSsrQTlWWbpMkI9IXXjDHNqyxNl2VCopHtN2hHFXWGVzrqwypbIL6H5ozbJcCx1nIFpzQQyjNk17qo+mIOi3EsoGufspDslELSDPoEcrTJL5LWsr6ANCtCiY6RojYbFOKGjGz3k+6mp6eSGTMx1gNguects6sgEEpDNB2UUfMkdlYLkrXq6AVTmvD8jhvpe6s4NhETajPMM/ZZ5xVyvAOBYIA5tXURkv6tJuF1Rp2VMVn7lSMghAvZEyw29K8mW2Zm5d6c3i3DdVDGZ6WQO1AyAdz3WZVYNiNHKWTQggfE03C764LMqQax3lIurjupni4GFsMT/xFYPhXaqxxPgtTST+Op4fwJLZspuWn3WB4iQepy74zzcpxppvYbZqecMPYi6F9fWRMGZgI7rM8TLfR5Cc1Ezhlc8kLpRzXhVPlfa4F1cpJJi7l2vbqsUOc05lt4LUtfPleCPdc8oXCbXwHgahDzgw+d4HzK0uUx/RUazCKSeYyFmvXsVxl2xJlfTsGXnAn2TurondVXOB0kbDJDzInfunRVIaCtNSIXi7Opus2rWa5s2gUNTQnLnYb9xZXKSk5V3Abq4yja8edunzWORLnY3EOGVXopHFlsy1HYXS+rkgFV6jDahln00eZva6s5Wpqd9mqR2oVSISxnK8ZSpM5tqsKkDAUnNA2RMFlK2Lu1BHG6N7Mh9SZ9I6xKN0QY7OEAqHZyHHRBSeCCQgdutaOmglOZ+6z61sbXERoK5kCDmIHbocoQIy6pjKo5M+ZLzW9KiDdMe6jfKUBv1QHNdbBZrprOKYGykzgLACxCLMUxfdCgkz33UjEELW31V2OFmXNZbFdpylWYzdqjMWV2ykYBZYEqcjMEwNlILdEVFyfb+KcR26KVSxsZfzBWxVIPZArcsYvoozF+6oK+UoblpU5aQo3j4lVIm6hmBU0bbi/RNLHZmZRlTLiEua5NKW3UWZvZbFnn26pjO0g6quTZRk2KCcOu5WoHWVBhupI5bFBpc6ybnX+JUzLp6kwlWBd5jnBIOsNVVE2iYz9yguCX95PzVQNQEQmQXCbplC2YelxRiS6BydFBJqVM46KB4PqWxGIkzoiDoEQzgqVr/zLAgAPRTNNinc0boFoTxyuuLnRTiVpHqVHNYqQPWRYll0sq5k90D3FR5ggmzqWPZVm7KeN1kEhGijJ6lGXaKJ26BidUJOqTzZATdbDtdrZSZ9FAN1I06IH1KF2hUjXKN48yRKUiOyFxKcplbZN5lJyCR5tEmhuYFS5mq2AjhAN1bbAx8ToyPUCq9+ykje+9syDzeSJ1Fi5sLBktgF3sf4kLJPzi64zimF9Ji5J66hdTgVS6qw6N7ugstSzKykS3snI1Q5SstESOqa7UybKrEgJBuqsjrFWnbfRQuDXG+Vd9bnJMepA5VwQDlKka89irMInzIHAnZPGHPU7I7dNVz9GlN8ZHwlRE65W7reggzWa/Yq0zDaX8ieKzxY2H0LpDzHtWh4fIrMkbIdGCwUepKxOc5N0ONrcqKSkdM0tCdgsArMTiAuVt0ox0Zi6KQNAVxzbm6B0Y6JZxVyLBRuDlZdHoq8gI0RpDzLHzI2TtuoJWGxcoBI4O3VYa7JAphI0rKbU6eqyPxJOzkGqQxCRdU46o/EVN4j3UbTebspA8qsJf3kYk91GE1yhIuUJk90swQC+wCrTOc5pa1Ty6gqubg+VSR5vxfWzGeSk8ETf4jquSpcTmw55EbgHjuLr2bEcEw/E2FtVGb/AJgbELyvijBKakxAw0Umg3zI2xa3FMQqnh8lQ3y7WjVSSaplfnfJmceqnbOylcWOYD8woppY5X57gfIIOPblO+ytVToI42yRR5haxVNnpulLIbZUaQyG7iVE9Txxtk1feyJ0GZpyMsO6uKe41Jhs9Y0mBt8m6v4MGU0r5Jza3kAtus5sssTRBHKY+Y6xI7LusTpKLD+HoGwemRurju4rQ52so6WWriEszXNn+HqPmuniw77qgiwoxgU1QbkncLioopnSeOAMjIyCbDounr+MMPxKniZBDLHUx+kuGgKIt0nB9DScXUMc9U3wZdnHM0AI91t/afiUFaKfCqCphkdHq+zwcosuFxLFq3FsgxLEh5bWa0aBZ08FM6UuhmkLe90F7A8JklxNrQ8SMj9RHW67rAcapcNbiOBxskqKird+FG0aO+a88hlko2uNJPJG54yE3volQ19fhtR4ukqi2axGY6rCutqMB4g4WpDO+Vpo6gkyMiNjGT0CiZhOJ4FhM+JT03LZOLEPlu6x+izpOKMVxaBlJiNQZ5OYCHk2/hsuv44fMcLpIy/yvi1H6Ii19jeB4dU08mN1cDZXxyWjcenuu7xqiosYrG8x7m8oWjezQm+9/wBFz32YSwUvC/Lmp3MzyEtIGhHddPzaE65iPmFZgiVqegpavDjQTmKZhABLt1BQc7DaZtGGBzI9GWOwVSarpImmR0uYD8utlEzHaUf6w/7ClJaaqhZiGKsqKhr3Mh1bYXF02JYRBiT+bRQmCo6PAsPqpaOrjla6cTxljze99lzvGfFtTS0TosPaY2nQyfG4e3ZdViXE8Y1VM/EoqGLDXMq2XEsjTo4jsuGxSOoo5+XUwuY8nY9u66jDa/xVRLV4rVxwPi1aX6k+6xsRnZilWap0wYY3aE9VmcWGVHNJK4MkcS09Lqd0bGtzMV6loqvGqhzoWDlg2zgf4KpJFNBUvpZoy0sNtVznFsIDmRn8f17tvoUckzOT+GfXpl6phTvfKGjLbrdDygag5HeVh/VYoDSS8iqike27GOu8X6LscDmnlx1sdBHGG1AsBewkC44wS1FU2niF3PNl2eA8ilx/CSJo8sclib6bLrCp8d4KkqcW5WE02aWCz53A6Ansue4hgq6TFfu6oIETLPfZm69AoYfvjGMbfQ1ckM9JJnjIdYE67rkeIgcTLampMsVe88vIBo4d/YLCObkp2VMop6d4DT8RK3sU5vgYcNj1YyME/RZmPYE/h6CASYnzZpL5o2bN/wAVBg1by31TKiZxaIxlv9brMov0kbKuKLCYWERPcA8nqFSrqQUFZNSxvc4XuLrpeF6aOuwusnikETqK7/Ns75rna+oZLVyT8lschcbgbBYlqJaWI0V+HKWCOrjL9Lx9XFTYK3DqqgIxCE+IphZoO6g4bw+rxSuzwRx5Yr5pnnRqGogrKeaatcQTzcgIO6gzcYdNj2JllJTvDw30u3XS8MYxxPhnDU3hMDmmhF/xbgBo+XVGxsFDhQqYKWLxkhIzltzr7rWq8Bra3h2HB6HFYIWRtMhs7zuJ3C645pLhqKmkrJYKeWrjPi5eY5nUd1Pi+BcnEp8Q8PI6hjNszDpdQ+IoqKKN8IcKyI+aW1rG+wUro+IcUZSwwEyxVBIiYTqDot2M6oxeprZsj3PEI6A7o4nVVZPIwNklIZoN7BR1GC11FiH3dMGiW4AsbgkrawjD6qixAcxwjcQbk/EOy5y0z4aynOHzMzZXwaNba+igNRysNEDxrPLzD8uisYzSR0lTHVQD8GoJePmsuaplqqiMTHyZtUhlNHE9zxzPK3urmHwQtxV4kjErI2l9iq9VM2GlhY/cyG3yTw1T2zyTs2dEQoLNJNHLI98cBhBN+ZfUfJHVsdMCWEuc/ud1kRVjyOWzS606Q1BlifHG4sgN5XNFwEFvB4mSzVj3jSCKzR7qaCKjqqSsgh25ed/9sKCnnZGyofTEWDy91t1nQYjICZ4eZEXkHN8lODXJcmaaSIQwOyyRNuOtrqvhtS8h8Uhvu8H3RGopnSS1cx87xb5KV1Jy2NnjmMhcLuv0CUqFscZaX4pK2SMtN7dD0VGnrZ6Rr2UknJbJa4Op0VqE+LimZJqNmn3VN2HVGY3blsRaW2h9lYEMjnwETM9QVnEKKbMyXu0H9VLOyGpcCTyX282bYnureIUPIwRlbHXvkfHpa9wR2t0Wolhhuo5mNMmXRT0VD4vnBlXynRtOYW3UtRBWT0ra2WeONtrNhB1t3ShY92HTMZl0duTY/qukSKpY7l5XuDjEOgtojjYxzWQiTlvPxOOh9lGXCIlhFyE8ZY92eYXCWNM4XJSvjikqI5M7Q+URuvyx7q1NgdaeRBSMkimOjz/VyC291Tp44HuyUlKC4i5ANrjsiGMVdso5mQelnNNmqCkKOqzuYY7OZJyyPdWnYNXUuaSSIfh9L+r5KDxFWMnhInjlm50vorzsTqMSqC6ac5TrlGgBUAQU95HwZI4omWsO91ajhAbyBHFGY3XFx6h3WdUPnqPQS573ZNTutvDcBk8YcOq/U3IS8m4aDtdQBU1lmTQxwwx3NyT1B6KGnxHw/OEjWyNNg2UahoCucYYk+Oqn4digEdNTAMjLBrfutGXB6DEODXYg6kMM0ETHwuB1cPznv8t04lM2LDqIVoqpJ2efzkvNiAdrK7WYLJVUk1fDVROZA3QN1LrLDfLHWMk5UcsnIYA1x7Kejw+eOLlvq5ojKDI0NPkcFzaY7Y3RvzSWaexKkilgibUTPikLyzJDI11hH7nupcRh8bXU8GHs5sr2+fXqppsBxCnppJ5DCWRC7iyS4AXWRWYKyCVtRIJGPAvHKNST7q3VY/NUU8VHPOQwOzktGskl72JCzmxg0r6tj3OEbg1wO+q0azBp8Ojpqirj/Cq2iSM7EfMKjSqcQhxV5+++ZRmNgjZGw6WCyqeos9kFdPUfdfM1MFtxsdd9z+qmjhrcZpK3F2fiNo2B85G4ubBWHYdPQcK0VfVOinZWl/JhvYxC/r99v4LLLadxjh1RgdVgbHVYppRyQyQmTyf6wEnQ+yg4N4PpeJMHxSqmrZIpsPbngEepNgTqPoudwODx9czDY6cyzVNo4iL+U9TYb/8ABehzfZ/j/BzRiOAVslUJR4arY0XPKfoTb6rQ5LhvBafE8ArOJ698r6WglEboIxZ7gQNflqoMNHJ5tVTlwY+XlxNkOkgXsdFwVh2GYw92F0zGYfidHGJWZbAPitY27v6rFqOFcGw3CCzELSRwVc9Q2nfIGuIJtkFtT8liYPRyVXgOKmloKsS088FWTI08wDknbW5XacBfaHHh7qrDsfrInu534dS3PZ2m1gDc/osPjiijqoqaDDsLo6TwzWTytp2fisB01C5ekmlwxzTTVAqoxJzHAsAy22WGnuUn2kYKcUhwigMdbNUkCIxTNyDvzCfR/FdPHVc1mfmRu/sOuP1Xz9TVEHGeJ/dtdys5iMkNVT04ieDucw7bL1bhXEGwUEOD8m7KSPltnhF4nW/uK3iy6zxTGse8nZpXI/Z5JbCKwf8Af/4LYqpb0lU6/wDUSf8A7hWDwG/Lg0zvzyA/wXWJpZ83WktsuZ4rhZaCdmlvJbut5r8yapgZNA6N4BB6EK4bKc8sXAN2ukd10JwelbGYWMsBsseuo30ko1u2QXBsvoa9nJ58sQROtsVKyd8RGVyqBzm7BPzHFdOLPJtUFfmlHMK246qEj1rjo5XsNwVap6x4ddz9l58tTthm6V87QM10Ila5qyYq8yODHagqWOqs8sceq4zhMOnK16S1rrElPJhniuSH669FqyF74vIdSsavLorskOrxot6/VM/RTMuiEThuhctrhzhyPHYZJ6qeSJjHZAGbn6rqaDhPAqL/AKpzid3TOzFdst2OE05xi4G99LIQPZdTxBwi+F/isJgc6M3zRgCzLBc009lrDZEufCUWYKQOU5pHyta+OIm42OllWcHRvLHbhaKW2SsLA0FIyKox9lHhlU6spXzSNsY5DH87KFLT4gR5d1GAWu1RktAznZut0TXMkZl5jdr7qpS2Z4ZYGMewB7NnjdJ8swdnL7t+So5gpWSlzCx50WaW2lCWzRmRknm2sEzaqso7eXMB1KzGzGN1wpTVySNyhp/VY4Nc3aUmJwVMIkYfYjsVO2TOuLoMRlozrqF01HVxzwiRj915tuDvhsaIkcTq5TMeB8SotfqpmzWC89N20XFs0ZY/Yri+IOHOVLC3DKdzi8kOA+lv8V1UdRomc4ON1rXsnCTLF5rNFLTyugniLXs3BQhwvsvQK7CqGunFVVU4leBbXayrHBsOhB5VFGPkF6vGcfCcjTROfKNF1uFUNKKZt2aqKjw1rZxnC1mwsjFmCy5ZbbXXroTI4m6AJ8gffKEwaQdkmrhbtSCWFzvhUkFJGPPk1UrrJB42QoQZZStUAkUgkQTaEISLJBwsnJBQQSxRu9QVeSFgOimlJzaqpLIQ5BKLDZSMkFlSzuTc491LRbdICfM5RSOjtoNe6rulceqHO49UtUvPkGxVWUl5u5TISxiqqxaVGdCrT2j0qq9tjcq8kNZpN0DjomdIAMqidIohP3QlqbO1LMUDEWSSJuhLtEAXsUTDqgSa4ILcTgLKw2oyi11niWxRCZBdfPfqkyax3VPm36ohJ+8ir7Zrj1KdknZZzZfdWGyCwQXGkKUOVNsoClZJ8WZBabullKhE+icS3KiFJH7KvJE+/pVlzrp0tbVgxzQpw1skdnJnDMEJlMYytWuRbOrIcsnlCqO0V+oeSS4qjK65WkBnQlwtfMoy5B5uyglzBLOO6hJchzFBZ5txa6cSqrd35kVyEFkz6ITL7qvdydrzdUT53FEJHBQByIPcgtMlU0Uuu6o5iibKQoNDPdIyfCqrZrjdGH+6yLGYJnOso847oJJRZbBGoA3chFRfqqbpLkoebb0lBeM3VE2dp9SzOa7unD391BqF4KBzgqPPf+ZP4h3ZKGhn1UjJVm8890bZ79VRo81Rvc4hVxLrujEmqgcucmTkhLMUBJA2ST5SqogblOW3BUZFkYLuqIhexDlspXBNy7lGDAGyLX3RsaB8KTgLeVG0YfZPnvshO6YbqsOa43ge58NQ30sGQ/wU/BVRngdTzP8ANa/96ucUw8/Cn5RcsIK53hWr5Nc1l7cwELcNOyebIFK9tjYoA1ZZCnylSJEaIIXMNlEWZgruS6YwgreOTMwqRUoc7UKbw8Y+FSCMN2T5f3lbKAI+yka1CjYLFSylyIgAFTiYd1R5g7KQPWGhzSFxQguCfT3TEtWQQkdeynic4KFu6MPQXA/RIuuqnNsd0QmHVRtZAv6kMzY2s1QCdo+JQ1FQCwtugpzysFwHKk4kqWQOTMiL1WENyiuRsrYpfzIvCR22VsVhO/8AMpBM8dU5pAEJjcFBZjmJ6qZslxuqbBZTs9Sli5GOqNRw7fRSt3WV9Au2UJBvsrBamRGbi0WKupT92cvP1zLyfiGGojF66rzTdQ3ovXsR5xpyyMkX7FcZj+C4XDgz6uubebu92qNvLJm8x+YKQUNhepORvdbHDlB4yV8sdIx3L3UtQ1lXXNp6qBoiBu+3ZB5nHHqGIZ4msdbOD8iu6r8CgxrC6TwM8DHxtXL1dBTYXzKatAdOPS5p0Pdb4sMUSBvRGJQRsrEVPzKYyOw6WVvV7VpU9FFW0XJwihMrhu9w2+acW2JDS1FXM18EJcyMh7ndAF1PF2LwmnpcLhNomR51q4Lhr6Thypw5mV85bewK4Gsp6plQ4SRyaeycWE3jZ44TTxzuEbvhCgaS1RtH5lIjdnsHEKchkWjnWT4bSvqq+GBhFy7qtPGMHxB2NeAFKWi9gQ1GbZjRM6VvIhdLfYALWkwKsFB40RODusbtCF2p4OpaDCoJaCaRrmAPcSBdx+a0HcKYXPQirndVyyyAFxz3B+icTxHnGG4TJLAMQdPG10TtYzuu44nrafEsMo2UzHSWjBc4bN+q06fg/CqOlrSyibIJIyQS6+qv8L4Tg1Xg4ZUUTJHXs4WU4M3bc4bkglwOkZTlpAZsDe3/ADZaPIk/Kufjwjh6iqeSyodC47MdNlU2KR4Vg1FLiFbj9ZTxRi5/EuT8tEluGzy9dh9RdVauESHl8sH5Cy4PhbjHH+I8blw+ixcspY9nyxAm3S+nzXctbxFFdjqXD6j97Plv9FDKFet4doKyMQ2fE8/1jXWsvM8co8YxvHXYTBVk01P5I5MvT3XpNZiuOUTXSVfDsBhjF5HRz6gLzlnEmJf0uTAMCn5MrzaU+bln3stRLEMqloYqTFPut5FXWMJEj5z5GjuFS4xw+lpp2+GqY5HyepjdmLEqYz4x75pZC/Mcwv1WlhseCMkc+rZKRl0IdstTKngx+rw+hZSUkbYns/rRuqjKmWaR0s55r3m5c/Up8SbFFOWRqpC+0gCxLUQuOfJnHL2Tuky9bEhHHGHMe8vyBiqxNM8zr622+S4K6DhukpHu8ZKZOYHZA3NoVb4hooaOrfJT09rgPcGjcrHAZTxMzgxg31K6GHGp4aVtLUMaW2td2pcFG137N6sU2Lyx1DCIa9uRpdoCR0V/7Q+IqSnqGUuG0RnrKcesaiIaae6zMMpgTHDBVmngkkvHITq3vZbWN8KQR4bVx0E5c6SEyTTPN+YRrotRLnMPJal8lZI6eWRz3vNySr+B4LPW0NXVwyhhp+h6q1gfC763Dq2qrebG6DyxAaXK3uD8MnosKr6TFY2wsfe7y7UaWXY5Ivs7x6gw2lr6nFXx5ZCPLu4n5LLx7EXY7icmIsw/lMeAI8o3A791miohZUvpcgbTA2JG7h810EtXBIyloYKiOmiFvPl1H1XLIiGNEKmFjmslnhz75Xlt1Lh0LBVxQv5sjTfYrpKum8RJHDVVsclMwWD2n9oq3DGGc/ihlPI8xMDXlrx8OmhusraItr6zFYcHjnEfM/ZZtre67XhmnlkqKrA8XZG2WOM2Me/zBXH4RQPp+MKltU+SQUDXvLyL2IGhK2onxjD4sbfUOZUz1nKleBuCtxgM/E+GKKvxCrnqpOXQ0cfLiezQTSKbg/EKOGhhpxS+IqKaU2/Ey8sHr7re4soIZsTwjh2kjzU4PNc2+p+awMS4WOG4lJQiYwRczPSyBuhHYnoV1ocpi0VZ9/172MLrSvks0aALVocWjpgw1EAkY8XD7ahHxnE3CsamZQnlMqIGGTrc9VnV2MU1bSU1JHAYoqcWuT6iuWWKxK7j9HZslVE8SUpbzDGW6NPcLkYDzKhl/S03K1WYzVGCooXy7aRnpYf+6zOTI1wAb6lmETYoHzTMe14LNmgdEdQMuVt7aWUMbeVIBLoo2hrmOdI85xv1WwALIpLNN1ZkxCR8TqePysfa7W6XTs8HRULK+Ou5lXewZbRvzCqEvMhmebkkm6UtHmGhkYS3vYrXpCw8Lu57v6+7fldZJlvGYSNDueqs857aBtOdWaPslFGfkEWQPzHqVrQlgwWOCef8aoccotrb5rGiF8nubI/MJY5M/wCyN2jssI62hZg+LcuB8ccFSwctwabZrLOmrDEIsOmfZgOYk736BZV5pKkSQ6EuvfshrnZqrmbddys8GoloVgpTG8ywmRu5IGyxeWXWa2+S990pZH3eIy6z90EbmBuV179NVvDEmU3LqHeYwmUDp0ASe7TyRi3ZyjdUzRRPpmPtzNyOyAzPdbProukQhOsXXk39imle0EBm1kUTmsu9rBmvdJ+V1yR9FG6W4JJ4aU18Bty3ZHexU1M2MU8tRO5kTLi1zqSeipSycqldSxylwn87r91NOLRMYTckXJRKTzc5jmiCR7WPabWVKjjmbLzcl2Dqr1FPDG0Pu4uALBfUAJqiwiMgIPyFllgeDPfT4hDOIeaxji8stckLt+Ha/AzSYjSY7iUmG1FXIwx1DmaujA2+QK5+o4afgGCU2P11fIJXyD8CO1wCL/w6rm5MSqnPMkbyWnq85lpXQYxRyCn5VBURVEMBMhqAbGQlUosUxaLBjh0lQ7w+W7WHYfJQ1EzKtjW80QgNGcjTMVE2KtbqyGeSIEEvy3buiN3hmkjfiopPFxubJA8lrQTf2+aJ1RS0c8r6ihqJKNnkhyTWMZ99F2OFYlitFDTV0OByzwywmQgQxgxm35ug0Xl2J18lfX1NfM7IaiV78o2CzQNtVE6rqKtk8sMg/Eic7U391LLK1tK7PPm5o82hFzdUaeSNrzNJm01DgL2K6SLijB6vDoqbFMxeGgS82G7CQd9LKrbBM8baRk1Owg65r+ly3+IK/iGjwahwTHTFO2VrKmN7gOY0dGX/AOd1lYRDDiT56R87KRlQHyRNA0+X1VaZlbWNimkMkjY28vO51wAOg9gojuvs5raCowvFOEmNjhrcZpXsNS5xsSAcgy29z1WNxFhmMYe7CuG8QqIpIIorRyRm4Ivc6aEWuL3VHCKpuHZjyY5HEeUkatPcLaLn4pU4fjFdiogmEkkmaQAtL9Ba3QaBB0P2WcM4dOIcfMkIraeqLYQJA0ujAINxe5NyvUXkj3XlXCVfhHDFXV11fSwfelTL/QnNdmY1hOpt0XrFBOcTw+nrpKYwSSi7mEWsV1FWapliIhZJFHLKCIw8+o/LquH4q59JikD5nS4jj1S21I1sfLji/fB/iu2lwdruKvvWpGZkVCI4b6hkl9bfqqfE9BDFhL8cmky1WFxvkgkJ+N9hqOqDzYcNvqKKoNdjBh4gDyZmzO8hZ0HvfSx91zE+G0viYaeOeeM8gGT8Mm0ovdbPEOJHinF21FJJHSvDY6SWOR2p1tcHtqrlfSY5w3TMwutNDHMxpkimEoleQToB2OnZc8kyZbG4TycOc6arZXGO7nsaSCO3/sun4O4zxnB8XNHjEJdhtXOLObFlDZH6M6aDRcniOIVxhpocVqnwl7C+E2aS0HfXc/qrGEVdDXzuwerxKTkzuiMc73anlnb+K5ky7/FeNvu+aswepp5xWPLzGZDlhDCNgeth/etngx3KwNzf3h/+4Fx/ELYcbfTjEqWWN2FNLHx31cw2tID12H6rsOEIY4cBAhZlbm2uu2Ktts5urLZQRqVQa5isCF7hcbK0AmIubKpNE2VuR40VySF4F1C9lvhK6YTxZnG3OVlNyZSc2h2UELObK2Eep5yhdW/Coa2P0691JQcP0lLZ+Ql/clemN7jOtzLsMq+byoYnSO7tGimdw/i0cQm8OD+6H6/ou5hpo9NFY8LGR5QseMvhPO2wVVK4PqIHRj3Rmqie7M0rrsWwV9azJzGx+51XN4jw3PRRcyB/O7gbrWO3HJZxpYw+cSjJfZXpMHoqtzJalmfl3sDtr3XP0HiKRzaqpp3ti72XUNqmFjXDqLrlsbxW8MbFRMyBrRc3NmgXP/NlpxVLZHZQuf5uuZX6Oa1nrh6t03GSADK8+VczxDhNEyWmlomRw3JDg1m91seJY47qliOSWMXF7G4WsZKZ0NHGDeyn8HTkhxgBPeydru6Lmd3K85KZuNNjo44fC0gLpHeZ1tGsGpXkM+M1ZZVTwThrH1D4wI9BYndes1XFeH0+JDDH/iki+guF5DNyMVgmpaWljhfHJJURhmhNn+g/IHT5J4slOsxHE/B4K4Ofep5I9gCRpquJhxXlNzvlmMsf5DpZa1VTYjU0pknPnkbaRvszYW7rDqJqeml5dOwR2ADrdSuviy4+G7Wm4jw00seV0hsLHS2q1aKdlZTtng1a8XXnE9NUylkdK65MAkDBu4k2XQ8LYpNgzJsNrYOU7KXlxfoF1w22xODq3se0Xsha8gqvR4mytsGaC1yCrOUhdoZk4N1doq51LI0F3lvqqLD5kbH5Hg5fSVMsISJdbTVzJWiQ6eyuRyh4uFysWJRskDI4PLfcnVbUUoaLly8WzVMS74ZtYFSDMqEVbDezX3Ktxy5m3XGYdvVYbqmcyyBslik6S4UEZfZ6lErSqkhOa6TJPdFXMwTZgFBzTb1KCWqLSlC26ZqiMzbrMmxBzXalD95AesOA7rfGWeTWEzUTZx3Wa2pa8BwKMSFqxONLbUEw7o2zM7rKFSO6NlSO6lK0pHXCz6h4a4o/E36qtUOzXUDGVMTdQZiia+y0ynfoFDntsk6TMoJC6+6gs81DzVVLyUJk/eVVZdPqoJJrqF8nuoZHIHke691EZfdA5xUZeiJc+qfOO6r53Ig9BNmKdRg2Rt2QOdkBcpH2UTt0DZ/dISKNzrp0EnMRCRQhydu6CyyT3U8cqpsUjS5BfEvurMTwQswPKnilII1VVe2NyiDhdVxUX3Th+Y6KC0Ho2uCiYCRojyG2qiJxYtUU7AW72SaSFDPLvqqqjVaBUJNz81bnfmKrEXKIHIlym3ujTdfotgeWAEJa2+gUpGiDT2QNlKAsUoc0KOtkZDQzT5w1zGnKT36IADLpiLLkoMF4qqrTzTmHmb3IBt00XWxRcmFsecuyC13blAOYIwbqPr9UQOigNPmKizBOZGt8yCUPITia3qKrmf8AdUTpbuKotOqvyuUbpyeqql7kxcSgm5nunzjsq5cmzuVhhZ5gTcxQZ/dLP7qibOfzJs/uos/uln91RMJHKRkmvqVbMUQcs0LrZNd1PG4j1OVCN+u6sxzWso2tlxKLMVG03CJrgFBK3dTMjKrslF1YbK2yBOiTCNwKkLxdR52oDylLT3UJkUZl13WRZ65UiGqFknupgdEAub5tEGX5KZ2WyiJK1CWjqoWVFHNC7Ului86w93hsSjP+rksvSQ7cdxZeeYvB4fE5hltrcLWMjv8ANzY2y3vnF0CrYXPzsOiN/QLKznSWUrdk7cqjDndkTTYqA8oTIgeoSsOysMyFC7dO4FAQd10BZSnvYIHSOCAyOWuLNpte6bMVHzD2SzjsrwktMJndSi547qvzWdShMjVngvJdZJmKsN8yz4nOFlchdeyxOLUSmLHIALKcFuVRuDSsOiIvCrvfrqrD4xfZRmIIwguEbZGhIwNvsiFKXbIJBKE/Num8NkHncB/aNk5axgJdKwAbnMFmw+a6EjoVRdxFw9HLyH4tEHnor8D2VbeZTSNkZ0IKWAylHGCoZq2hpX5KqrhiPZzrFAcewKIefFKYf+YFbF9hsp2uWC3jHhoy8n7yYPcjRa1PV0lYzmUtVFI32KirGdqReAoC7KdULph3UE7iOqpV+G0OIxcirgzs7XspebdLmI0o0nDmFUTSyipcrTuM65mHBGSY+f6AGs+K4Xac2yjlqJKeJ80Dc72C4F90HzjhLaen/FZHLPpawkIsurw/hHBsSpY6qenJ5hsWuNz9ViYfwfjsTnSR52WF2lovm/iu04HwzEmuNXXyZ2DyZbLvDlC1hnBVJA7kVLHNpB/VXuF0UGF4ZHTmio6GKGNwt+G0XCtAc5wZ8PVKSrggtDRwGeTsDZOSxLyeSln4R4kNJWkmKcmxOxHssv7RXUlRVU0FA0Pk2swbnp/iuo+2DDauSCLFr3cAQGDZvVeaV+JQ1UUU8bCJotyei2JK3hXEcKo21mI5Ys97M3N/+Sst2i6mqrH49gTKvEayOIwtOSJuwPcoeD+Gn4659XA8SvpiCWP0uNVjitosIwCqFLS8R0jmzNidndGNCLaL0Kjw2DH6qDGIGgsi9RJtqqVFQ/dNTU0UkEkMVTo1rgLX9v1Wtw5BUUM7qTmB0Ehu23RduLnybsgAIyttYbKKY2AV/wABLIegQOwOWRwc55sOl1ioVWifmDoSfK9liuGxav4gwfEXYFw0M/N84dluW/8AOq9GdhwYOg+Sqw4XHBK6eFoa95uTdSobcxhHCFJh1sb4hrZK2rf57PvZpVXGeDMQ4orHVtdXcqmjb+E1ttFpcT8WYPhDmQyvM8zHehm31UcXFE+MYW+Whp4qSHKfxJHaD+CzwZhj/ZJhM9HiuKPfqzKYg7u666uu4ur6TmYXQYdI+aK4MshyRtHcrzfBOOMZwKSWnp+TLE+cvkke25d8lc484mxHFIaSmpyImVLPMxgtcaaE9VJa5KdZxDjM+HVkdRXmRpcdGk2F+y6rA8aquHeDYMOo8KlbLOSTM6xjAPUry0Usx8onjzj4L6rUPFHEIj5EkgcAzIzW1gsW0jxSgooWTzwYu2qnkfmILbXKxqaqngmEkb7OGysyB5hzyNs926pWs7Ms20lllfM/mSO8xUcIcaoW1CAuztVijaXMdIy2dm11EKaU7NKejqHxSvIYHG2xVdw1JzDUq5RUs0rQYv2shyBvUqTAlE0lXWMlkNmgWyjZX5qx9Y4Esc3ILaqrLQyYc5zKqzHs3F1ZoiHwmXKQ14sLhYmGokMddyHtZFJpI4M+S7PDcaZhMZwfG45XU0j7c6M6n2Xm0MvhpZPIHWuzVTx1lTUTRU5keQ+QANJuAtxCS+gpOHcGbhkcUYtD68x1JC8g4vqa7EOIn0FFO+eGA3jazYD37r0zHHVo4aOH0IzTMhYzTe3cLh8SoYuGsMdVUk8TnVEZBc8XkL/8F0pypV4QEDq6qnxSnpTSyfhSh+haQuz4iwTADg/hqfD44eZZ8c7Ix5R815hS0jvCc6XRzyT8wvS+DKvEqajbV12ImsoJG/shqYjrqsU6Ob4oosVosJpGmlj8BGAPEMduei0+EQ+bA6yqq2BrqkCOK2h0GnyVzjKmrOKmOjp3cjDcPbzTMBo9/wAly2AYRHHJhk5rpBNJPYNcdLdFYwHW8J4dFFhuIwVb2x1BlyVLzvkC4qKKsxXHhSQvJpmOfJC0aAAey9Nwx7J2VdBXUjGvmJjE7Bv81VpMDZFxW10LLxUlFZx9yP8AgrQs8L4MyWmGLYnJI+vfoXXsQBewV/GqJlbRucw35HnIO1lZoxI1jedG2MgWLW7LJx2Oqw2orcdYObSyNAka11n372WoHHfaVgtVLHTV1NRXigjAGXfW1797WXN0mA1dTgEjKmIRA3kiNs7yemnRdvz6Gr8J99U9dTsqSWQfjGx20UfEtPWz4RXysqDS01JHy4W31cBuSt8bYeeRcO11a0+AjD2AAF7n2JPVZwlqYZTBM6/IJFuxWlRYxLh2GCmjc+R8oJJBty/ksWOS2fmPuSSSTuSuc48WolZfMZ43Z2R3GxHRM+OeFzWFnrCrMkMd7DdX5KtlWRJNGAbWsCucqozRcp5zsAKUcgz5rXb2U9U1xDSDawsqdrOy3SxPlsMyUUrQ60jQ8WtYp48jgR2aVER5GkbqqlpxlEh65tEZJVjCsJxTGc7qCleY4zZ0h2utGs4XxLDo/FSuhmhHqfEbZT7grMlM9lU+BoAbuL/VVpq1lRHGeTleBZzgbh6nw+KSqlbHuOh7qlV0s1LM6CdtnsNiEpDGXdjLkkWUZD2NyvcL+yViEbWAjMVppHdzgHfRP5xupaWmmqqltJTszSSHyi9v4p6umkpXugnZlkZuL3sqgbZbuO5GgQmR5PnNz1QZ3kgk6hI3Jvf6oqeGN878rTYAXJPRWJWwtpnvY27gbZu6rU85hz93Cyc3MQynRQSUssLYsrpfPe9rLRppKWVjYpoQCG5MwOrj7rEAyOWlhLYJalrKuq5ERuS/tZJgdjFX0GP4QzB6qK82Hi7X5vwwP9Yb2Ob5rn8UpMENQDhta+ohBtLMY8oD1rU/DeG12C1Nf+PBCGl8bQLPlPV+vraewUGEY5BRwtoocGgmpuWDMZBYukG4uUGAMOkqXufSlhY0X9Y0C28C4hko5mSVEckrIpAJYmAFjmWsdO6sYtS4PS8IUmN00MBqq6se8uYbmOP/AFZH0UAwd9U6XFZy6DO0SCKPQ27rIbGOKavF5XUFE3wOGskJEDbxl409f+S1uDsHwDFqCrp5omQ4jRuL2mY5rsOwt1/4rAxHCqOGSp5FZnDCC0tFt1FwziT8GxcYlHURiSBpIY82E3st4DpqvgasivPiHh6SknJdLyQLNI62WBhGGcPzY3Lgck8soqG2ppicoMu7AQNr7XUGL8TYvjOJVGIzc6n57eW2OM6NjWcab8COqfYsnvlIPYqzLL2HFvswxLEGVOJTT0ENTHTlkNPTRXDrdz/iF5I+mr8PpIhVUksMMj9JyLsdY62XffZlxozBY6ukxmqqjQmzIX5S4Mfr79V0mMU2E4j9n0+H0wM8kDHmlbOMpMmfcC6Dyx8XhquJl75wT9LIcIayqq+QyOSXmHl2YLlbOD8O+Mop6KvlFPiUZY6MkZpAy2gt1+i1/s14DranEpqitqpIKUF7LMu2SUjseixxR2f2fYfg2MYHX0mJ4fS+KpiKZzzDlIFtCPddu6SGiw+WQVEDhTMeGt5osSBf/BeYcKSV9BUYtRYXW033Wyps4VJDy4j1kHdZlTVsw+sr6HB8RrKqKNoFOyTeMPvnFvloiw9Dw7iGixp1PVfe1LDkawyQh1rPPRc39rGIyYrHFg+H10Io6eQTVD2EyOL/AGt2+fZcJiYnwuSbCaSdlUTGySWQxhpaOyCfFMfhp6SCF/MOr4gLOLhfW4WLaZ2FUL8SqzTwGzQHukeRrGB1KKrquTUxx0NO2ofRSPzyObnD+gN+tloUeJ076iWqrZ2llfSvp6x8cPL5B2CgwOvoKJ4oJRJmrIyBI42bk11A+it2zTKqsRrahrH1t5OU0MbmGlvYqxh2HGrxBzZIZI2wwve42vlNtFHNR08ENVSzVZ58JvFc6OB1VpslVC9tVLLG17Gs0JuJARsQskvYaPhzD5eAnVFdyqqeGj58co/aRkAHJfsNNFrcEQsfwtE9+rjK/wDvXkPDXElfh33mJWS+HrYDFK1jfw7nYgfIWXqnCWN4TTYNS0UleyOSWV/KEgMfMPYXW8JG0aUCTM64V1gs1RyOF7I4ZWOAZm1XWyIERdSwwQubZ7N+qhI811I5zgMwQmDiBlK7R17qQVDPzKAyZmqq6TK7Ks2lNWKUK7C+7Vhw1BzC7lpQT/DdIVdcwSBZ9ZRzljuRIWOOgI6K14oKaOVsjQVEpytZR4jCzPO8zHqbWVbO4brs3tieNRdc/jVEyEtkhIN73B0W+fIiKVWG9lPHI5mzlUjzC2ZSsc7MpKr0c0g2KGaV7nXe4koGnqicLjMooAeyKQWjdJcWYLlBfKdVNJSMq6Ysc+7Hixt2TkPPcdraSkqIZ4Y4p3MnMps70jb9VyuKY3AMdbUw4dZ8nne3ra2l/f8AyXqk3AvDszJWDDhmkGl3aA915rxlgclHURVNJSRRctuScwkEONxYj6FFY2I434oxMhdcHmEnrra9x/gsyt5ZljdG4Fz262K6ScYOaIzeFp6dzGvBY0edwfsR10tsuRpI/wCkRQO0N7H2WbZbmBYbU1FHX1kM8jKyniDw0a3j1vb3WxNR/d7YRiVLNz52CUxP0sOmpT8OYlheEXjqjI2WRxEuVubKAdAfqvTON8Op34DHxBMwSGFjHi41yPtddtWyLqWMoeeU1d91ukiEHMlnlFgfLlB00W6Od4+QjzRNtYjYH/FcRWmWvr5qTC6R16cEyEOBHT9F2GGZMPpYcOfIZJQLuA1Nzuvbjm884roaLovi3UkFLV1ErYYKWUvfsMq34eDHyi89XLE7ty7/AOK1zTi51p82inZWVEbQGPU2KYPW4RMeez8AkBsvRypjK5Z/iWPJpUFQ2SYGc691stxGNj+X07rmaeOqmeG0lNNL1u1txZT+Ia3yyOs5c5wh0wydVFVxvNr6qbmgrkqeqk5wIetyKZ+XdefPVTrGS8RdRua4DZFG65UuZtrLlTavmAOqoV8nwsOqu1AdmFlTki5hOZbhGLPJOJAHtcQewT8qF7CZGyXOxJ2WlKeWMlrk91A9gfGXOGq9GMuVK1JU1EJyPeHDp7LUjnEgu4rAkkIf5bhWaesLPW7ZM9UT5pjl8W2SCm5jgqsddDJbzKbM07LzTi7RKTnO6lMZfdRHNdJYVM3VM82KjDwEjJfqgLNlUUku+qjMnuonOVDvk90JkQEhMToohZ/dA52qLKVEYyTogRN0Ngj5ZHwpBjkDZQmyW2U3LJCXKPUIIswTg9QpDH7JctAOdATZS8r2SMSCvlCkZGVIItVM2MAIK4YSnEJurQiARhjUFURH8qkEdlPkP5UBFigjAspG7oEgXIJxsp6YBzhdQNOllNTua05iVVa0bII2BQyzMBtdQOmDhZqjtdQPLU7taqsshO6scoWuVDLGgqSHVQk9SrEjXXUeRqIjSUlmoDugTzZQ50b9lFLJHBG6aV4axm5WwUs0cETppn5QFTiikr5W1dcyzBrHBfQe5QMikrZxV1bLMZrCw/3lWzJZAbnXdmQufoonPvshzusgRksU/MQO3QEWQGZHAIDKShLtPMgJCCTmfvJi5qizBLMEBl4TF35VHnQ5tVpEmYJ8/uoUs/ujKbP7pZ/dQ5/dJUWMzeyWdQhycOUEocnDlGDdODZUTA63UjSVG3dStCzInbI8DLdGJHHcqJIGyirTX2PqUzZrKmHKQPUaWTPom591Wc/8qG5uqwtGQFDn1UTCVLHE6TZFSteQpWONkIp39VL4eQD0otGzGyEnREI3fEkW6IyhLiuT4ritUsmGlxZdc5liuf4sgvSidrtWEBWAfCUxko5IXm+Q3WwuX4UqWxVjo37SLq35AfMVoLUIgeqi5gPpKAyHZa4MWs5kYc3oqoc78yXOyfEr4ZcLYI6py0EKkKjpdEagkbpwo5QGV2QnVQmVPIXEqIt7LtjHk5TIzNpug5rvzKPI9TRRs6q0IzOfzJ2PeXeZysmKHL5Qq4jJfZjVPIXYj5R5ipObkF7qu2KcbBRmKdx8zVmYaiV0Vp/OiFb+ZUhA/wDKnFPN+Vc+MLyle8Y3uqlZxJg9AP6XWxj+yc39ypYq6akoJJrP06ALyyqrZJ6iSR4817LM4tRL0mt+0TCKX/Rad9T9LLDrftNxSXSlpooB2Iv/AIriXVUrzuo3Skepcpapp13EOMV0vMq8Slk9gSP8VWGJ1t8orpwO3MKqDM7a31NkJbY7j6Fc6bWCYf66U3U8OMVNGzJSV88bT0Dlmm53ckGZuoClLS3NVvqZDJNNJI49XOukHD8o+qpvHL0uD8kPMLfSUoXSA7Zqkhr6qkf+DUTR/wBk2WeypINzsrIhrKpnMgp3yDuAsjuMK+0aaCJtPWwc0/nLtVqVXHd6bmUIYH9iAvLopXh5Y71DdWRK+2hVHXyfaFjJaQGMa7oQNv4Ko77QOJh/1uP/ANMLmJXyMYXlyt0dPz4A8i6g9B4a42OItbDiTmiU/EP8l1RI+N4HzXjMckdBLzvCOJZtYp8T4gqsRhEIfJExvS63Eo9NNL0YwD5JpZoaGlku8Ne0X7rkMP45rsXw+erpMgMTTcFuxVf7L8UkxTF6yOul5s7yTd2tl6+Tm3eHcZq+IzUUrXRwtiNnHqV1FNSxUcJMbfMPi6lcTiMcvAmOz4r4WSbD5wT5N7n9VawjiSTFv/idbiEcdNe0cXMAFlhYc19olHxjV0r58UxOjjoYznbFA7U+5v1XDUWB8qgNXibjDEf2ZGt1vcVYvPLjz6KerEtG4gZxsliuLUmJ1NBhdK8uhiABLhZjv+bIrnqLBp8Tfy6GeM3Ng0mxXonAP2b8QYZiAxOuqo6eIWBjbdxk30KrcC4ZNhnGDRVU7RBPfKTqDovXi5Bz+I4FS1lc3mN/Zm40VmLDoYH52M2VoC9U56sWBCAIXa+ZW2uaQqh38qy8Z4ywnAS2J4fVTuF+XH0+ZSJZpuSQse059G91zVbDilTUOp6IZ4/iPQj5qzg2OVvFMYqGYcaWlvaxdfMtptM6IW5hY32W4mmnjHFmC0s/EEVEz8JthzjvcpcQ09RM+j4eomiOGQgWAuu9wrh3nYzXYtWwMk1tCT7LQjoaJlca+ePM/ot+Iw8ox7g77ogZDSycyV5sW6anv7KevpYWYTh+LVDA2SnIjcANT7r0WshirK5sgZy2NFruUOJcL0D8Lljoqb8QC4c43t9Fmx4viNZRVGKS1MZDA61gGbqvCx81ZlZrcL0er4MwTEOH6mqFMYJotTIN3Fc1PhddPh1PTUVEadkX7Srdo2/zXmyh0iXO1gMFT4V40KzJw+N5YVexAZ2iQ1QmkaLF7XXCo8uSXzE3so2ja60bmd+qKne+OQAOs3qmyflTxxvLgwaXKIVTEGyEsaV13DWAsxmoJnnEDqaESRmM63WRDhsdXVRwCqjtILn2XWUOASYbgNXUTm808nh4XWsCD1+SsqwpcTBnnjIjqpi7lZ3N0+YVnEa2GYUmGGEwujbq0m91h4lhs2D1AppXh2cXDmjQqQGStn8TnAIADCVimVGWNwlfp8RVnCKf/wCJ0dROLU3OGdy6HBsGjxOWTD6p8kZa3O5zDoQtvhPhukoKupZOeaz4WuB0IXo14E5O8Ls0sdWP2RjFj3CwsVw2nrcZbVPpY3U0Y0BGl+60og+KMNY8n5lDUmeSB7IyzORYEjZenwnm5uA40hbJi7oI5Gcw8tgjYO66/wC4vubDm/dZeXtgtKx2zh3VaPBMNpK+nqap3NqJCC5zjbVdJJLSue0F1yTYBPCxObhsX49GIwQYOcOloqB7gJZ3i1wOlrKgMXwvDOKmvjoJa2mjZaNkTbmM9wPotTjkvruJcLwZlNmhzCRzW9bnVW6qahw7j6gcwBgkiERLRZgOtlnLGnTGbdFT1r5ZeeyGSJkoBs8WI+ieWq5OLRzxs0LLOdfqtGqp6YzHOch/gsGuxCEztbTctlGw/iVchsD7N7q0y3hXg/GiNYwDR6zMKxXAcYlfBh1a2SSPdpFiR3CnnbkdYK8MTkrcRU0GK0bBe0sEokjNv1C4Pi/FOJsSxSPCaSPw9NO27QQNQNyV3FQx8uW3wG4+a57FqHEcQxEPZ+G6KIhj7AnXdb4QzyeXTR1NDPLA+TNa7MwGh+SqDewXVcSU1Y2m8HGy7IH3ledyVz0WHTlomLRlOy8OTvigBsMp3VmjMZfaZhLXduiiq4w2chjLN6IqOF7522eGNGriey5zDpKxXw8vzxNOXsTeypXJOXLqugnmpagsyxgNaLbWus+sgjjaZoxYrnaKTmvjOR4IJCZz7M+QR1Dcr2i5Je26ryu8uQ7ONluFh6Jgs1Th3DlBHTzlrXgz+XfVbdXhldUUXIxLDfF01TEHZ4TZ4Hb6KnXYdNRUVLFyJBDHAwZ8tx/zquoosbw7EqRrIp2xPpGi+c2uOwXCf4n1MNeGWuHm+IYA/DKZ2K4HVyTx0/7SBwAMQ7rmMQravEZBPVSFxAsNNgu5xGVkGDYpOdHGIsZ8yuEjymAN6rrjLyb9ca5qFbMB0191KT2AQSsa0jKmuQC236hbedp4DNVOxSGClfBG7XzyjRQcQSB+JPyRx+TyF7CSJD3Wc8FT2vA0IE2nlLM7GXUYLeu6OKN00rYy/IHHUk6KWnonzVBjuwtGt77oK/X6ro+EsIwnF6vwmK1XK5lhE3NYyHtdVqSiYwSV4YLROz6my7/hnDqyHDw+toqQwmUVDXcsExa6/og8+4qwRnDnEFVhcRkMEZvC55uSw/8AJTUFNBFEHV+HOk5pD4HH+s9vZX/tJlgl4tqvC1QmYzIAWuzAaXt/FNX11dg82H0tdh0cvhgyeMPdY6i6Mg4gxfHJvD4JitTG+LDz+Cxo/Zgjb9FknKWCMHyD4RsrWL17MUq5a6SljZNKb3b/AM6qi14ygIsSMxxht2BakXFGJwsjpY3R8i3LdnbmIHVZkIzB3sL2G6iGnmSYV0eKV0FSZJKQtyPBjLQyxJ0tosFzLvcOxspAbxZkItv3UjyDzSTTNBmdmDBkt7dl2WKU/DLeGfvWniMsZljMVPJe0b9M49/muP5VwtNuJzPwaLBnscWRTiSNw2Z7FVJhp4Q2iqcPqMDpo56N9bURyQulF2xxje59v4qXiTCMSwdzuH5JJq+mpn+KppjcXjeDfRV63HJ8bmq3zwkGjbeAU41aOt/bZd8/jbgSswegh4sxSOtrWRMu6ip3kgCxs8G316LcS5uawPAK40dfXxvfJVYU2KWlD22kBvd7LH4F6rV1E9FQE0LI2Pk1Lsu1xqdFRpsa4RxzGHYpgWN0zjLAKaqpJbtkkZrkNjvbXQJOx2qr62WgwrAZpWROyvqKg8qP6XuUhcXjVXPBhmKTciKoa0TCWFziQeXvf3utisr8Px/E3sra2Sgc+TPFV+oR6bO20916LxBwscYgrcsMUj5ImRQNIAA11PsfdeZ8S4PieFl1cWQ0kUGSExxS8wOJv16pODSrX4BiWHxVNYyHxFLGLeLZ6JWXsDdC/HqONlLDhNLI97B5pZGhsjR2uN1DhnFmP4dTCip5Ym01iHtdCHB1/nsqtNRvxGSq5DQQxpluDl+i4TisSuS47SxMbSSQxm7Xslc1mtj3V6fGKV+HxRQcPwCmiayJtVODnOlrlYEFLJWMlMcAme9ut9z/AJpQVMkN6OaM5OkTz6fl81LasNVCwuMJcJS+2V7To4lW5W1UFO59RCQ4BgaQQfZKSOklZmjpeS+Ih57Edx3UkL5qjDmUz6fNncXtk3trrojM+bfwN2KVWD12JuEM0FGWMOcG5vYWFvmui4PpqXijBKilrppHOintUsabGnPwSj+4/JcvQtxDCaV9F42A0dQWPIa++vdem8F4HSYXGcVgk/EqYgyQNNgSDutRgjYw44pFT+ExZ7JpofTUMHlmj6H2PdWw5wdeyEzMGlwq8tUGg2XbHC0X45yT5irLX3CwGYkwShhd9VqtkYSG5twtcOKckspObybKJ3dTaFRTEdFloGYjZTMqXgW9lXvcqRougmhEz/2e61onZRZZbJhFbJcFWY6olu6wDqMRnilLY4iAzcuWNX1klQ4h7r9loT+ZhHdU308YFyFYFJji4eZTxlA5ttkzXWK0LzXXRczRZeIYtT4dC18h8zzZrdrn59FzWI8W4q+andhscccOYiZz3Aj5e3VSYG1xPicNOI6KqqH0Yn0E1rsPsVl1+NcR+MIwTmRsjijYS5oLQLblZ+OYhRY+KVj6KtEhkAMjgcjRtcDZaTK3CcLjOGyYrKIngAiQGQvB0vf9FkdgRJWULzVzyROjbZ2Tc9z7rg+L8Yw11HSfc76eaV7vEHOLAiPQXHc7LsaRseEc7wkxl8S5giMjrEAfAL79V5K7haaafEsNqq2Ns1LLna8i7bP13voVAGHcQUVR42SrwrxHiSC3I0ExP+Z1/wDZc5LDJNiksIZlcXaNvsq1RAaR/LOpN7J6YvmqQ904jefidskkrVNG+WeR3KfIyL1ZTZenVf2h4VivDTaCtjkiaYsgY0XEYYwjv1uvMpvFzPLI2W5pvJy+p91vYTh9EIMJlnjFWySpBljA9TAdQpBKfg7D8SdUOp6UytdWu5jXEXJA6r1vlYbwnBSQyYeJZZBrIG3eT1JKzMOxHgbFMWo63CLw8hxpyAC2Pa+36qDiLi2glxgT4dNDWMI8HCA6wa873J/RdObHF6LTCCSJk0erZBdpta4VsEW2WLg+IUVZByKN120gZEfY22WmxwWeUulHrKOnrYXQVEYkY8EEFYNJwXhVM1zKhjp7m4uSLDtuukuLKJ8lit+IUekp6ejgZS0kYiiZs0KCqw3D52PjfTsLZBY2AurEUgepNFjxCnIM4O8PKH0tVmt8Lh/jdNNG+kfZ7DbuAutEbScyjlo4ZWlkguCteIU5qKS4upM1ypcSwh9DSST0Ty8MFyzrb2WJDXzO+CT9FpltOGdqhEeqaGoMgFlKGu7KKjfCxzctgs6qpZ4zdjLtWvESTZyeQWWoyHPzszQuGQZgNFlujOfl5SHdituspRnL4yQfZZ7aPO8vfckHqvTjl5OGUIHQPhs+QBwVyCrjygbeyCVhka4Fh0VOxjSazhL4tpkrXDRyRKpQuIcLK0wOI1Xmyxp1xyJ7wEHMPZSGEnomFI6+6502gebILlWTT6+pD4X3UFbzd01ndlfFLqiMDfyoM4ByMNVo04v6UnRMHRBGIr+pPyXflRAjopY3N+JBAISPhRsi6WVgFpSNh1VaRclvxIDCPhUj5NUHN1UZPyT3QOZZT52qGpjkkZeN4HzWxGMt0+ZvZROgmAuyZp7pgHD1OurSWl5zRuE/imfCVCQT0QhoHwrfGGblYbOxO6aJx3VUlAX2CxMNQuuljy+pQvmbsCqhe5AXuuoWviQqRk2UrOZM8BLnottdtT7qYVLO6w+f7JCsA3QbXiWdwlz2HqsbxQ7pvGt/Mg05ZIwMwKrumA9RWfJVuPVDzy7dShaM5ug5yrufdMc9rqiyZQVlXdidVzHj+h0zsjWnUOPdTzzthp5JMzQ4MNrlUsNrsPpKCFktWA4i5AF9UGqZCmz32Wc/G6Abc1/ybZQuxxh/ZUjyO7jZBrFyQcsV+OVF/JTMA7E3UDsYxE+mSNh7tjQb5cgLljUOKVc0/JqZuYDsSNQtIyhAnvddR5/dC6TVAtMJczeySDMEswU8mvIWYpkOYp8wT1ZOkns2+ybr9VoM3ZOiA6BJA4RAeyEC6MNUsO3ZSBqAHVGHKCVgsjAuha74VIHBQGkmBupGsv8ACigUsbbhSNaB0UrcqCIQkpcrX0q7FGCE74ulkFNsXsr1PFYA5UcNONH5FYDAOiBDKNkZkZl9WqjLlE52VEE7zFM0aqLnI2OB+JAb4gR5lh4/RzTUMnLZewut7Oo3wsqI3wv2e03VgeaUMz6eqY9umtl2skM2Rj7esXXE1sfIr5mN2bJcBeh4fLDWYXFN+6ujLNEhb9UbZbop6dzXEjZA2Nd4mKcpHmugduiaNVMyFm7gpcEK1ndkYzDdWsrG9Ahcxh9IUs4yrn1Ji5O5huhIsVuJ8kFuqGL4xDg0RkmtmHQhXXOLdl57xRJNPisnMkDrbW6LK4tocf2GtI0/X/goh9oteHWjpYGt7WuuTFM+U5WuA+aP7tNr+KjXnnPJ14umr+Oq2rAyGOH+yFlv4kxGTfFaj5NKxHRHX8Rp+qgMci588l4t2TiLFXtLTiEnzQw47iMLs/3i4rCJIKAuypzyV1j+NsZdA6ndVNkaRa7marnHmTO459zfZVhI8nyqRkj5HiNjHucewus3k0Mabq1h8OFzykYhOWgbADdT0mEGolbzIn6exW1LhlDSUxfyXtez926KpRxcMRvDmwyut+6VLNVU896SgwmENOznDVUGVb5DqI/90K5R0sNRNnmrWwNHUndBRq+H5qRuaaVuY/CBcqlJhGINs8UcxYeojK7zDsJwN1UKqrxiGct2BlstSo4w4SpJTDNiDC4dWtJCDyuShmj8ohlP/llE/B8QbFzpqd7W+69Jl4l4TrH3gxKMu7csrJx2t+8AI6GkbK3q4C1kHAikqZT5IJCPkugoMchwmm8PNGT7A7LTpcNcaTPJM6Jw+FuyxarAcRxCqMNFHnv1ugsYVLw9ilaaevhDeYb5grHFHCtPhFOK2hml6+Un5K9wrwHU4dXtq8ZhBA6A6BdVxFV4OKF1HJaUHYB1kmB5LRy1NRMIY4xIfddTT4Xijo2sZR6u/LsqeGU9HhVSah0PNZe93OXbQcXcOGIyGSOF35bpMDkhwfijJBJPRVIaOuZFBw/Q1dRyOW+/U8xa2I/aTRxOMFJQmVp+Iv8A+C56krq+ulfV0TJY7m+nRKHm0E0wbmhmkbffKbXWlhVbiOC1DMUoX5Zh1VFzZM5GQC3ZBLLLo3N6VVdHjvHvEmP0/hauq/CtYtsNVX4UmggkmjqoOYyTc223WW6R4pc76chv57aKKGR+S7Hlt+yML2MMp21bjTtkDezuitsoavFaOKqpIRaD1OHTuqtAJC+KtqoJ6inJsLDQn5roqgvpp4J34dW0lMSDJHl8hHTRdIlHp1Bhkdbg2FTSPcHwxMII6roo35wuYw3jXhaemjhZXthDABaQEFaMmetaWR1v9HP+pdqf8lbDnG6enllp4YZKuYO1bELgfVUqvjDCqKqbSYowUsrxfK6YE2+ibGKiHhbh2pqqKMmZ4yMI3Lyd15ni/BNazhp3E+JPklrKl2ewfcgGywPRMS+0jhHD6Sd7K/mTBvlY0XJK4zBuLsSxyhrpvDwOdHe92+dcXhvCnEGLSthpMJmGf4naAfMrqaDGqThOajw2ehMM0Z5c4LfgK1A7T7KcXfLgtRGOWZRNY5jv1XS1VVik73RcyIM7BYNLhElDi7MQwhwZTVjRzGjZdEyJjQAGpIKmZPyw2SS9u6d8VjbS3sEYdYJF6CuIYWellz7p6qdlPHkAJfKLNSJ1TT5ZoeQWXz6b7IK2HYYTCKCsZ5QS8juuH+2TG4KShi4Yw0RgvAMrGnYX0/xWlx5xJX8PU48JNllDbFxFyQvH6usqcSldW10zpZX7krORiqwtOZkD9A7dT1VIaOYMY/M2QaFQPjbfr+qIzvMjc7yeX6fZZpuZRmCSM5Qw2UlNGXODiCApJq8vaWZdT1UlPNmj/EF9dlCJS1X9HZDPB67gBvQr13EZ5uJMOwvBpCYMUEYla0CzNup6LyqgmjGNUVRMLxRSMOTvqu4xviymwP7QIMSqoJWwci+QjU6aWVaUftIwubCcEoKKtMMlSJrl8UlxttbouJpJ5Isn9FYchvboUeIYgcXxKoqpeYTLKZGFx2BXQcF4OzE8RfA6rED6dvNALb5vZRmW4IcVwhtNi9fhUMTKtoZ5TZ9iu1kwDEZaaHFoBHJE+L81rLi+KsdxDHXwUtJSuFFSFjJHAdey72q41wvhGOlwCGmEjRGwm7rZQf712w28HKceSvHR1ojBkpSG9wbrnMTxgNqzBTOuGbn3XXQfafgs/OoX0kjbN9bdQuEo8MZjOLVLWVIbSsJJcTYH6rp+IlPBVat9RVyxzy/1Zu0/3rSpa2omiZOGHIOp7rUoaDA6ZwZV0zs40aW6gBPiVPhdZh89RS8xvhPO47aLHPI8Nx+LY6+KeoxTEpiKjlmGnDRqB3A6BVaCifVcImulaJKlspkilB13WPjlTUY3XtLI5NDy4mW1JXY0MTMF4YjopMsNU+IkxE6kreMpOFNeLFH1WEQsqr88RgOA1XI8aV73spIJHubTgWsDpcWXQYHiFDWUb5IoHxzQNHNznUHXdclU08OJOqK/E8x8v4UbTYtUy2cUxdRw3j3C2EshFfyvEkXbVsjNre/uu0c+GeJs8MokY/ZzdivGo5G0ETGMe7JIbEHXRd3w3R09H/SsFqjNC8Wlpr3EZ7hXHbaTDopHRsaXveA0bkrE4q4iouHaWKol5kpl9LIjqR3WtXUHi2NZEwZ2m4zOIC8147wvGZZTiuMOhyRizWxXIaF2nOmccWLiOOz4pK55jEUZdnDb6rNqa6csEEbrNGyUUTwBIRYKGosZncttm9F4snsgjIxxa1rnPPXREbOPLJsD7qFvlKJjXFwAFz2WVXmzAtOT4NAnrQZaAzAWKrWLfR1RTzDwkVMx1yCXu+azQVT+3hPTlpQwx1VdTUztpJQCoOY55Be7Za3CcTKriWiik9IJf9QFVevRYjTyOEArbzZsvLftZS1eCYFi8742cqOrvYvYTcEey42f8Wokl7uKlo8RrMNnbUUrwHs2zC4XknKPe+xj02XCKlW48w/7mwN9L4qOfmzs1aCD9V52M18q9A+0fGPvLBcNEkQEz5SZHDqBsVwMbfNmXfVVPn9T/EUmhARZwTZyYh5OqOGYUj3F8GcvbYa7Lo8yWlooZ2ufNMI265L9VJUQ07cvhY3CM+lztyq7ZnhogeAYurbKxVV76gxfhhrIm5WsGwRlDG2EB2dwzu9JvspssUXkbOHPG5A0CqOLnOv0UtM19TL4d8lmuaSDtayLbbpc8+Hsw3JaaokZG0g6SAlen41UQYPwzUVsEnLlFOIg3qH2tsfqvKcBhrq7EaOngke4UzuebaFtt1PxHUCvxauq6qtkkc+XJG7o63RBl4PhE+MYtRYbSt/FqZQwfqvTON+CeHKZtTjuK4pLNNHS2jY0gRueNAy2/wDFcLwjibMD4gpcWkhEsdNd5aTYnTp7rfxzEJPtNxylpsOpaijw+mhfqWXF9zcj36IjiafDairiNVDrdzWBv+KCqpKqiqDSTx2mBALb91r4rg2K8O0tLP4jK6sD3sgbpIIxs8rEnFQck9QyX8Sz2uePUPbuhC7SV5wzFKSr8KWchwvlGpHVWuK8cw/HK5j8OoBTsjBu7KAZCe9lmyVDKhhE3qA8p91CyPO62ZVpJEHuby2hLzBWcPo5Kqo5EcnL8peS4dAvQMG+zBlRh+H4zUVR5VRSmSWMxEi5Ht7KDgmTufAxjtmXt9VNDUUzAJJJsjvy2uPqrnE5wqpFDNhkbo5uUWTtY02Lx/7Lns+uY7IjpYqrE38P1NHg0lo2O5k7mC0hYTt/eon8XYtUYNDhfPjJi/AeDECXjpc/oqvD1XLS4izJWmnjO7rXF+gIXe4Xi/DNZwZXz10GGfeNPNa8cYjkOvkkZ772WsYSYcjHgGN0DabFqChngrIzpHyyZL/nt7rq8Pp+Mf5tHjTCcaxBz4JSZ6aZwyOjA1IHWxuF2OC8YcJcVT0GfGDFiVJIRG2S7eaANL9DvtfdbYlwSlpHso4Y6htGH5nF9oWm9z191Ep5E/jLiymxep+66+rliqBeKGdplIBGpFxpZR4hWYpJFDJXU2aKKQRCNx1kJGthtbVeoYPgstbE/i7GXxRVdRCGRNbHy2QxA6D67/VcZj9DRDGWMrsTnMsjJJsjIw1sEY+W662y5ypiwgc2KOighmMWfLJcAew7rGpnT5pIYY+Y2W3MZewIHdW6/EoKiJsEEjXZ2i+l/wBCs1t3S5GPLXdwvLk3C7EaVte4Nhkipnuz2BtY+yixDkz/AI8jyHs0BB6XSpoaurbIIDzHs3DjqfkgqxHFMKOdliRmuDcWWWj09dWwsZHRT3zm1nC60cOp8dbFJXQwckUcoDnSiwu82At1CwzHS5Pw3lzyD5dgD0K9NwfGfvjA6qeoMUUj6fw7WPk3kjG4106LeLKvivCnIlGIuj5UFXHAJYBr57nOQf4rtMHjkpcCpo4i9zY87B1sAbBcTiXGEdfhzKCmgka1nLeHuNsr2Czxb5/oul4P4pp5aaDDam5mkqJIwQRYC1xdd9c44ZMNXxM1/UnNS+QWcVy1VxWaCkAqad4czPHK8G5jOwuLdVVZxLXU7Gwh8M5YcznAWJZ10Xq8bByrJ1xF3LYoZmyx6O1Gi5zBsSZjlTJHSQuyRND3OOm66CGhfCbsCZbMczGMmk15I1ScVUp5J8zmSMt2VloJNyvJMOsSdEx2Up+WO6flo2VweqdsluqhIcCm1CwLbZXEapiA71KuH2Uscl0Bmna8WaqE0ZikyrUh1Veuh5js7dwrYqjlEWc1rvmLrnpeJKWeafB6vBeZEfIWtjufmqGI13EfOqqQ0rgTIHxOaLWtrcFWjW4S/Cpq6qqvD+Qxk/EH9gEsQVmEYrLhopIGSxwn9oZB+I1g2NlnT0VLNRxvjmcXUw5ZaOvY9yj4S4mqqwsw7FMR5cEl44nEAPJ7EroXU+AUczmTSMcdPONz9VmRzfEuM4tiVFS0ktqNtNOx7vMQQLWFiuPxriWatq6qeleYxUOHMuNyAvROKqPDix1VicjX01THy+Za5A915VXYfHSvc+OfNGSQ073slLSpTziSrYa6S7bkk27oqVx5bW215hP6KIx+dnZzgtLDY2PmpWkXz1GT6Eoi/wALf0yskwd0cmWqLSZGmwjAGpPYWuutrxSYbQT1WDVENRFTlkQt/VMyEn5nQ6rDxPE8EwOJ0nDlCIJpD4eSU3sAd90pMXw8cN1EEdO1zczBI0u9RItf23/ghLDoK6aNtRSxVElNFIOYXNNtv/ddBg/D7KekpajGKommlJlEMMliT3J/RcgJJsghiYC6byG/ZdHh+HQTxMqqiOUvgljvGHEh4JN9Pp07oPSuFeLnxYo7BoMNjp4Hu/Da2KxtbcnqvQG1rD1XGUn3ZRUcVVLJEZXtOV1/S87gDp0U1Ji7y+OGd4ZJKM7GW1t3Qdj40AbqrJV5nXDlk+LuPUmZVWfug2mVDrjVWI6h175lkibQEOViOo8o1UmGmzHKwjNdEHtusls7h1U0c7i4aoLr4+awsI0IsVz02DxtqM+uhW8+oyNuFRkeXXLlqMhCyBjWBrWhEIbqPmEJGa2xUtlLJDZtwsuokqYpf2JI9lotmL+qlyh48y3Eq5qWsjmdkjddzk2h0+IaEKXE8OgY/OwWzf3qjFFKJWut5D1XfH0YyTkNAKKKn5gu1ot7qKV+R4HRy1WxBrG27KIqCmdfUKVkWlrKUApwLrlMtRCPki10DmkK2BoontYs20qE2TtkZ1UhjaSmNO0WI6oCB6hEC1JrBlSDMzsgOqiIJX+byqKQ3bruppWua8hzVE/8RoFtQgrkkJuYjIFlEW6oJRUaJGoPdVjmugJKCwZnX3TCZVXOKQce62LnPsExqPdVWk/mRAEoJvEaalNzv+bKIMN0+UpyEhkahzqN+m6i5hBvdbthYJ0UT0JnUfNJKjQnbogLhREkpB2UIykI0USstZmjujFDI6Iv7LLaiS5RnZSyNLDqgy3CCMvchz+6NzUORyAc3zTsfqE/Je45lE7Q2KMM+vxGrM74Y5jGGG3lVN0kz2/jTSO+blJXN/pclhvqor3Cw2V28vysSDbgOsm0DkrkAkK8loT8gamaRZINlcM/LNu9kJNkko7t0PT6Juv1RKWWejPLqWPzWF1sO3WGC4SjL3W1muMy1CBLgmzFMktsHzuRZtECSokzBEDqhAsjDVkEBokBqia0uUrYlQwbonyeylDHJBjlBER0KbKFOYk3K19KCJO3dTinJHpT+GKQIweoUrGPPRSMpfZTsha21wrwEMUTswvsrYYbI2OiHl0UoMZG6s4ylwCOJ7jlyqwykv1sjhDQL3Rk2PqWFCyEM+K6mY0EqHMU4e4LDbSLmZbBQOOirmcgbqI1Did1tLSySWuq5JeUZ1CYZWogSzRMA4G9kZI6pN2RDmS26YThvxIXDRV35QVuIS3GcSQcnE3Oa7SQAre4RrSaB1O/XL1+Sy+LoHiWOoG1rFLhea1S6P2JXWkdcJI3+pBIxgGYNVYytZ8SqTTVBNxUAN7KsLd9dkZnt6isjmScwF8xt1RSknzhxcPYrTDQNU0+l6dtQVnRF23VW42PA2QTPqmxML5D5Qsas4ywKmuwySSPHwtam4obV/djnUpAtvc2XnkHJEg5u6zk3i7WfjOiqKV3Ijkif0zLk5qhtXUl4fe/8FUqZm8wtjblb0Ct4LO2CXzUEUtu65c2+I5aKYW0OvZWIeG62rZnbXcsdnLUlx+OFn9HoQH9WrMxDiuqkaYIKd0d9yAqqlXYS+glEb52S36tKmhpHPjzsgLvkqZmqZPNJ5nHqd0TZ6qP0afIrllJEiFLQuJFUy4vq1dBw7g/DlUwvdh40NvMbrmXvf6pBlSFbLDfkv33WeaurfJws2tME+HRxsj3NjqtiDH+DMPiEVPDGz3bGvN5K2WT9obqDni+yzzW3rLMewKrcG01TEH9LsCCuxrD5Kd1FMyJziLNIIXl7ahqTp4j1TmW08Qw6aWoLxJFG0bBqqyPETOS87Kkay20ihdVBx1cSVORbUo6eCohcXuiGvXdZ9TBTNeWRzsPyF1XMz/zJCZ/Vy1bSSnjMMoe2axWkzEq2lPMiq5b+yxXSOJzFTCVzo8ocg6Oj4p8XLyK6tMbO50Ckr5MNa0SYXjDhKOuZcxFh9TO3mRgZT3KikHhJbTaOCDpKPi3HKYFk2ICWM9SLrMxirnr6vmsnuOmUqGnlZK27LFRSyGOe+Qn5IOmpKUOo2xz3zH62UFZhkMcBfnH10VNuO1QaBlj/RXXY1SSQct8WZ/e26DNpYqQyi6nfiFXQy2o3mJo6EKuZs0jX5ALG+i6KSqoJ6LJJZ3a42WYlHmVBTz1mJR0QmLXF2QkdFu4hwdiUWKtocOf4jOL53+Vc3I6ShqXuhmOYm9ypY8fxtjw9te/MNlpXay8IcSU7GQyGkmiJy5GHX+5YvG2CQ4VUxx0kcUbXgXs7r3UUHHfE4gNO+uGUC+bKLn6rn67EK7E6gz1sjnH3N7Kq9l+znC55MAlwPEYwJYnc+N7XBxudRqF6BRTB0DY6mRrJY/IQTZfN/DuN4jgVQ2ekqnNZ1bdbruMpOa6rqpDKT0LipLD2Wt4c4fqv9Iw+icT1ba65Wq4JoY7vwPiOai/cjmBBXlGI8T4rVTPfRYjPBC/4GSGy3+C+JJ8CycyiZUMlNiSdR9VqJRK3inEaCqOB4wJaynppblzRZ5t1v13XpVLxVwnxFStofGGnde4jmYQVl45wwzHDTY1hTxDVRecaC0nsbqU0eHYrEKHH8DjhqXC3OY3T+Coi4o43OBuZBg2Fmtyt/0gE2+i8kxmpnxytNfHBPZ5AeSLhpXpM/DmM4FnnwmqiqoI2n8KQage11R4RnwWqoKzA8RIgqamQk3Ftfmg63hGTEMNwmkw/GcokMeeMg3Fl0fTRU8Lo/BUEdFOY5hE20bra2VsPAQLX3SzFC9wshzO+ErVBwLqRsrIjmedLKuZJi/KLW7pOhgiA8RMLvNhfqsq8i4xhr8VxmWGCO8VRMGZrrGxrhuTAMTjw6acyRyAHNlsAvbKynpg1skMUZYw55DbYLguLJcP4qxqiwehrRmiD7vaMwuenREcfR8PR4tXnD6EuzgZrk6LAqKSoo6uSkqGZXxOLCL9l6nw9PBwvBX4ZW0whqYGF7Khw0k9l5fV1UlbVz1c5u98hJKjZoeQ12eYXPRaVLRvmibUcsFoNywmxVSjwuprrllo2D43bK/SxTUUBZIcwHUDRYpIbVVS1fEFFzcMwtlLDh7Oa6TMATboFo/aC6lxnhTCsZ5wZMYxFkO5t1XacAcKYc/h51XM6V3j2kOaX6NC4us4Yop8WqsLq4G0/hhnjAcfxFVt50y7Vehq6qlMdXSTmGcGwkGtvopKLAMQrsZOExMyOuTmI0aB3UFSx1O99E/1Rm10HpHElDPhfA1A2lquY6tkEk7xqXF+un6IOHuEa6jf/OriqUx0bG5I/FavPa6Hgut+9G4SzEpopYaacR+HcTntrr7LpeL8RouJOKKbAAJJKOn1fy29ex/TdXJhwfDmGux3i84PRT8ujqJTJI3YPjGpsvYHCjoKp+G4dgcDmUcdw0GxP8N1wldiOC8JcfUUkEXKZyTG4NHU7f4rWxbE58MeavDaiQTVO+dwc/Is2nJtY7WSYhRUGIYbTlvMcY5Y7as9iucoah8dfV4bJBJlk+HuVV4L4hroceNJLPI+nlL5HNAuM/ddJg1dXVM9eaaGAOjkvd4uVbVzGG8KyMx+XEqqNkDKV4ywuOuvW65x9R94cSVuLYg8mGgHkDdOnRXMT4nnOJVj3Ou+Q2uNAB8lhYnVzS0rxDaJktmFoG4WbANxKqp5nVTJbQYhrI3oCjk8zbZljz1E4bFSSMDeRtZTxTmoLQHaHdc+RxWMQiy0T39tlewKQ4dVkRzSNgliuddyqb5iW8iVmdqd0xiAMdmHuSryXi6atx6bD8PMtPK/nVElruOgWHLUMqoiaueSVgILwTcn5KjLUGU56h97bAbBV4p2TX5d8vQrp4uScQ1U0ckrmRss3ss+WB9yWWPsFdq7umDYmG9raKORppWl5sZCNr7JdulUzo9FLHJyjnZ6rWuoLFu+3VSMd5QqoXStGnxdUzrFocENrzKyaWMxNImaMwvYoKhJJsN10/2fUb5MSrat4/YU5IPYrng4ROD2M8w2PRdlwRePBa2rcP8ASJgB9FnP+F01eeyIa3L82lrITGVLcJibW/eNl873v0uLn+MIZKvEaHD6dl3mnsxvc3UGP8M/dEFLNA6SQygibMfSV0vEmC1OG45R427I6GBrATbVp6FUMXkxDGJ20EAvJPZ7RsAP8F9PTr/K/OdXsjLbMw5GWJxsIWeZotbutjEuFHRUFHU4bG6R7IebPIToT0/xUYwqpgxJ2GyjLMDYH3XSy4u2kwo4fPIA5pByg3P6rc/leWJefukBY179H3II6hWJICKLnfGfSO6vY/Rxh4qp5h4mS2djdrqKWSA08Rf/AFYC5NM8NsNVYouRFLzpz6AS0dSVYjw987QSwt8+x0K67gXhuirJKn74pObTl1h2jO1yfqqS4vCq6qo5JjRBpnqwYx1IB7BBPLLO6Onmf5YLeQ7grbq6ulwnimukwuia2nps8cYGov3v1WJG2Wpq3TE53ykvcfdUiGvw9w7Pjb5ZTKaalp/2s1hv7LvqbB8UwLhigwzBMRinqKuqfmqYR6AddO64vD6h8OEOw6sqvwqiflU7WyZeW+/rd3C9Y+zzDYMN4cjoDWU9bLFM98j4jdgJ6Aq4pk5DHuBJG4th7KvGZ66WtgkfPz38sXFrd7Nudl0GMcI0eJYhScRChBgw6jNK6ke2/L5bCQQeu4UOOcNYtivEMVW+v/BrHGF0Y0y043t22C6Khrhw9h0eE1876qeEPjhjAvJKy+hP+a1MGOTyzDPs8L8Jfi2Py+FikgfUwMjZaSQja5v2We3giufG6afDZIT4F9ZE9h0lAtc/xXt+F4cx48di0LAWNIjhveOBnZT5MJrIJJqURG8L4I5GbAHt/wA6qU1bwbgrCvvPFKMV9O2SgMlpnneO+1xvYnQr3CGCXDaBuHU7GBsReIy0WAFjbT6rn8enwTg+mw6rge0+EeKWQRxjmFj9yQN9lXg+1vhypzMZh1WJSbQNda0vzPwfVZRaw3AYMIw7CZ46AuyNn57XtFxJIRqf0XknFnC7+GsTmoHYiJnTgTxDlnYn+C9Oj+0OqgxN9FjwpoqdzDJCaeO4Jt6M/VZkGL4a/EHcc8QTxCLEYJKaCAMvlFtvf5+60PLJIH0lYaUSRTPb/qnXG3dSgVVGHzH8MSeRw62TVQwqGYHCoZmNyjMX6EP1vb2QOm5vke8uHUFSBr8OeAMs0D6Z88tQ0RwtAuS++3t816LhWHY3lg+8cDo4Kai1ginqWBt+pda5f00uvP8AB5qSeikpZsOfmpwWNqGNtqdgvV6XFqHBcIon43w74asqCyIsgbmEhtuOx1V4jC+0rEMdfRUcJxOd1PUh7BHTRBsZeNvci1/0XFTmdn9OpIzS08jRTzPHxnYs+q7zF6yOrxumxHD8KqG09NG9j3VAsQTsQPZcRBPVTObSvjjmpY6iSV2YWDie6xOXFJhzkLI8z3B3J1s3MrkLKWnDudJHPLIPLlPp91oUVRSx5pqlkZZHdjaf1AHuNFpYTjVBgMVTVzYSKplfEY4PKPwydNb7b/wXOZtYZGFuZ4kEzthcASwuGhPZLHI5460TRxA5CCwiO4I9wrNZwnxFQ1uH4fPSGHxsoZBUO1Du9/kug4g4cwKkmz4lizhUMgPOhgjIEj7fh29rrQ4qKnEsv48BD3kCwBaFp0s/gaeVkZLBTysLHgXIJ0P9y6KLgLH8M4eZjuIzR897SG08gNxf0AHbOVTxOGOiw+WlrqR+HufVl7mvNy57IwCGfX+9Uc9PA9tRyzPzfLmD79fktWKaaCqgkgrsvLAfZuwk6lAKRjoIoK48vmATQk6loJtYqCKq8JVG8cUjIWefL1HsuTUwvPpcSxAVjIZn1stRKx8gYwl9mfL/AII3MqXymCuglDKedjTI5thGHjQH9F3/ANmvDtFh8EmN4fWyzCsjAma4BpikvsbLfxbAMNxennhqoBec5y8bh9rArpxYmHI8EzUdPjjY4qp8TjGY6iJ4sDqLEHtdehskB9S52h4Zjw/EjU045kUkUbJSRYkhlt/oCt3KR8JXWFjyWyyI7WPySDbIaSMSEB7g33Ktcl8TywsB991LFfKVI0aeZSmH2TBlkVBJB8QVd4dctWkAL+YKCeNubRqCmBcomggogLKdgZZRDwusmqvNE/3Cdo81lkYbiFVJjFVQ13kZHIeXm0uAgrYmH09HPXTzRwPAIaHnQkd15DiNbVYs8smd5ObnDBsCvQONsffh1Y6lbikYcHZizRwDDsLEey8+qaibEq1tbBHlBdaR+2Ynb+4oIoax9NilNHSvtkcyQnoD1XW0nEsIiljZhzZH88ySXOYfpZcdSU1M5jj4hsdULjI4a3HRXaWknFMyqojy31DD4rqAL7IOto8cjxOeqwzF5I46Zg5kB20XHcTmk8cTRcgt0IdB6HM7rocJwunxBx5lWyHw8BYS0aW6XXIGhdU189PCzywOeB7C6qq9bD5444Bq8XAWtwoaSOaF84Icxz5HEC50FrlY1XIIJ2wB1+VcHuL6I6SZlNKzxTy1hafSdTfsoNXG3wVGJcjDZjJSySA5nbmTqs+T/R2scPc+3RdDhGHR1uA0xnqDTy00kshc6K+hva5usR0FfXPjpaeMOdUOIjaNL26/LRBTjPnBjdsujwF88lVTUM0/IZcSSPe0E6HQe6lq+HaTBa1koqmSWsBG0G4fob69FdFX97Q1sjIQ19PkdI7YPAP8FEdDiVJ4Gio7Me6KKoNjzLEgjsr+GCR2KeKl0l5drdAOyxcLpqrGaWjxRjyWRSnNE6S4AGwA7raosKxkV/ixUf0Z785a8au/yQb2dwSEjgo6mqp6Z0bJ5hGZTZgPUp87UEwmk/Mpo5n3DrqmJWp+cOiDXZV36q1DUeUarDZM6/lVmKV/dGm4JyeqLUlUIp+hVtszXBIBcth0VeaB4JOhU5eB1QySAjyoKzHBnqUkcwJy3VWU+cpNZmF7rUC2/I6+bULLqoJM5MZuOyskyt3cnY0yHVW0plM807bsIstHdSvhcPkq7n5Xq8rSITEXSEZKeJzTurTA0tWbWEIa4IZoAWZ2O17K2YhZUpSbluYqKq37pZ9vZC+901tbIg+aAEwl810OUJFmUaKIle4OOYqGWRjQcvVROkcFBM5xVUz3tuhzNUMhfqoXl5uiJnSsBUTpAfSojnKA36oJS5MHapmkXRsYbgrYJrS46KeKF1rpMbYqdrrIGELz0QSxkGy0YXQ5dUb4YXa6FBhSDVRFl+i13UsRkytbun+6on7PKtjDLCmDCt44O0+h1yq8+EVMOtrg9ipYzWtspmQNeMyPwlQN4nqxDC5g1UsoEEbWepWg5trJ4oGPte6uNpI7aBSxkVFJE5peNwqD4gPSF1Iw9kgy5d1QxDDvDnzNCWMMMJ+FLlEfCrZhtshdG8DZUVHhwChkFyrDmv7KFzXKjCxciOs16tCo838pWhjrWiaJ9tS0grKksBosqlMjer0HOcwOJ7KMbpSBxaVFT8P00uJSzslmkLA1zg7MdPpdBSyPdEM6vcIeSpnZ3b/mqtsrns7OKkhsrro+l0rtASVAFtnZlpwm8LT3CznnK2618OpnVFKx7BoFYlJRhEG6q8MOff0qQYc/8i3DnMKAYiylaDcOf+RSx4d3YrZizGxv7KZsEh9LVssw6Jo85CnZS0rXBucXUsZEdLLb0KQUsw6LeFLGPRY/JSGnyi5aFFYgopiNkwo5BuFvNpwReyIU8f5UGF4Un4VYZSWHp1Wy2jYdmJGlt8CloyvC/uoWU+tshWtyh2Quj10arEjMfTlouAoHhy3Yo2vNiNFVqMHlzZ2Pbkd3XXHJnLFkuzBOHXK0XYTERlbNZ3ZMMBqQc+cZe668ocuEgh/ZqTX2RinfELbohHZcMnc0YHxJPIvo66cMRZGrCoHbIG6OU5Zc6KFzXZkQ8wmDbhqCMTyGwYtKmcHtDZE0jREcrLW9lbFJ0b2tzOUbZR6StCS0jCMqotoxzTnLrfNaDGSyrlkjnLRFLB8JJQmFjTZWMmXP8R0jpMMe8DZcxhUzo6+LJ1NivRaqniqaCeF3xt3Xl0D3RVGVhs5j9CrzLdzJQTudnkYbIamsgo4Ax9ID7gKeCetq6VrmSAX7ouUSwiqaxzuqcimFLiEM8L+TBI1/RNRVM7Y/xhp3WtUUlPynNYyxWfTm7HxzMEZZfZZ5FAGM0lJJ+Lm09lSxHiKrqmkQTsgiPxW1XK4jiRqaiRjHWsbLMkNnG5WeeRTpqqUxjnT1nPv7rAk5ZLiyN2p3Sjk5bNHiyFjzK/IxtysTsydKFC4RvzmHmexWpT1EsbMzKEarGklljcReyNuMYjC0Rx1Ry9iFGqdFRRYbI909bUCB3RhKpYlBAai9DO2ZhF9OizqbEZHSCSoyuI6kLqMNpuH8bhcyetfHU3tkabBOTNOVklERLD6gips8jcxat2v4TgoKsQTGRwebNeTYLQk4Mro4nSNqon5emWyerPFx+IzhzGsY0AjqFmmV42fqtGqbTue7PI4EG1gFRkbGBojYBzXeopi535gEsxWhSRy5Dy8OdMCNbjVBnB+tsw+iF2douVoRYVPOXzQU/KtrlOio1EhvyzuEEOdx32ThwGwKJkBc0vb06K9hooGSWrSWdgrxEYp3TQ5Yo5ru76qmYp2utp+q3qiR7I3SUMpEQGgJ1WBJI55zHdKE5p35QeW4/JRFhad8oUYmmB0mcB2BRNZnID3WHdaE1PUyQPzslcfquihq8NrKcMfBHI/u5q58U8fwSBSMJiFmGy52Np2GxRMLoGU/0KwoqySnqHPfEx3a6aR9Rf8AbfwQudr5t0sDV1bqiYyCFsd+jU0VW+I5hce9k5yP9LUILCbO2WbGlLisc0Pmjbm7hU2TB5tzlGQ14ysblRNpJMue7f1TEcwZWSsEjz5kDXa5lC05gr4jj8IT1C2Kr5HZxl6oyTHdjmjVR5HFucDRTRxGVgysFgjQWF9g1qU9w0e6lZBJ0BKKpgL7AMIt3UZDTUxMBZI35LRw6tMEQpyxjwza/RQRSvyBj+iFrAycuHXdB3FJxhiNHSskbijXcvTkAW09l6DgGMw47h4qovVexHULxijqWU7w+SAStbuFs4VxPVYVO+fDQIhIbyMOoK1Ekw9bIGbI7bqFm4vw1hGMxWdAIJR6ZWbgrK4f4vfi7hBXQMil/O06FdIZMvpXVxctHi+OcGTx0mMReKw4mzZWDUD/AJK6ulxClxCBlTSSiRjxe4UVQYayB1LVwiVj9wVxmI01ZwdVNrsNnlfRTnVjhcDbRB3md3ZLOqGHYlBi9IKuik0O4O4KmcZGetaVdErI2ukebBgus+hjfiFUcXqmOsw2iYRYAfJG2rph5Zn3H5RuUTKuSWMx0sJItbM7QLnMDmeK6nEo8EkdHISK2fljs0ey4nh3DajDuL6WlqG2LH6HuNF6RxBQEYZSjNfl1Od3sFDW0UEWIVGPGMPdFB5SehCU0437VsS5tVHhnho/IeZzOu2y8/Is3TqtDHMSq8XxOWtq35nk2v7BQQPjhPOkgEtvhOy5tp4KjFJaVlKJjkBs1g910UuHYlhop4MUkilZUt0YNS35rNNbSzNhmYxsbmG5aBsuhxHFKbHKmly0cr4qSIZiNQT9EsdFg/HcfDuDNwaqpJaupAIp2Qa6dibaLLxuuraqOkqJ44qPFJb5WjcsPdcxUYnU4PiMeLUVO0tsQRIFDR4xiGMcS0NdiBuJJeW22wv0QdlwjQSCabHZjZ0Y5PLiFy73uuNxmjMWP1fPhdH+ITZ24B2XbT4pBwriVdDI1+VwMsLRfU/8lcPX1NbXmpxSSCQ812Zxtt7KSIYah8dQJGC1l2nDPE+FYA51U+nlc948xYL2K4ukpua3NnP1VqklewZ6eUsfcjRc+TS1xTFFVmXiOd8sM1ZUXhY7bInrK+pMkMjpA95jA0O6lfgWLcSOhzVEPIi9TnHUD5fRXThmG4PxBRfddO+eONuR19QT3UiWZh1/2ZRUsmET1NRGDNziCTuNFYZWT0MWNVdOQH2uLi4urFBg9bTtzUsfkm8wHRvyWjQ4RBM+eOScysDrOa4aErtEJTxGSgxeaRxfSSN5pvzHCwKevqqd7Im048sW4916Xx1G2amNbHIAKQ5RGfWRZecV+HwtoYqinYRzBcgm9lzyahgGZznukccxt1QRVD2kMjeRfspqqmlYx3Lgy5RdxJ0Wa06KI22y8xuYvAFrXWbPEyOocXue5gNrk6oIza+mn8ED8xf5lqBfhMEj2QQm5ebXvspwOVI6nyAGPt1WXdkbhZ1ik+eaKUSskNz3SYPRoulc1xcwkH2VeQSVBLXzOBed1UfVyySZ8++4Rtqhez5LN+S1AtYnHh0DWNpPxHBpEjjoXHTVUJB5Yx2apZIy5gkIs1/p9woyLqtCOQyAtb5bJ3wsltnfky7Jqc2lDt7C6eSGedxmkfq7oiBky3t0XeYHAKThnDoGamoHN/VcOIA2NrpHjXsu9wurpMRwei5VZHTywR8u0gsLjsue3+F6umyjDK5TOZJBbnxyR32zNIv+qkgb4iVkLNyQteDG8fjZy6ukgxGm2Jc3Pp2uFVxbEaWokmxGgw7wbKenIItYF/debXh+Z9T8TFXC/HH9/wCJTYdi0ccbWNZbIbCUI6vD4afiKmZRQiM09EcpHTXRcJFi9UKmnxmGQMli3JNswTVnGuI1GISYi+UQSmIQxZdQ0L6+O7Gqfn9mP5pKsxllPIa2SPnVTJH3NtXPWZRVc9RLJNVRxl8u5sqT5rHLbc5yTuSkypkjlFl5sslhqTyU73l/KjkcW5DmF1kVAeZHuyHkxu8xA0HzVmnqpHVTZA0XMnLNxe5KlxCOfC66swp3LlbJbmEbey52BFfUWzMks8+26txVOKVEU8MFVO1j2/itbJYO+azHte5rXxjy3uFPPVsgihjh0mb55D/cstQeKmvSgMm5RjJBaTY7pQ07mUclQYgWRGziUD5KWZrn2tLId/ZK8z4nwDmFklswb1S27VZcj7Z2Zm31F9wu/wCHOIcSwbAYWcOVrHB8hvRuiMhLz77hcbTUzjIynbCRm2BXf/ZtiOGQRVmHcwQPjkzkPNrnXVdcZcsvzOiwziXiPG7wyYGcHyHJUTv1LR2YO62KZmH4K3mRmeaZ+7/VLIViScVUMlU+rgrQ6hiiPNPKJF+/yU1Ji01dSvkw7iGnlBNw1kN3NHbddqZiHKcZ47xDj2KuwikpKzB4mQPlqWTyZZJmDXUDS3+axcOwnEKSkpMUmxWWCu1qIoXym4jGx9l3tRXcMMfFLLV2rqMGNz6p3nt1F9iFyPFmLYZHVyVmG1cNXLO0B7x5g0WAAA/VccmoTVPHEmJz0ba3DYTMwAtfHGCZZdh/enx3CcSwmixbEqvD6AOLQZZxCRmJIFm3Ou65zh6hlx+rgbTzilko2l5meNL7i3/PRLiWrxGd7qKs4hkr4dzldcA9FnirPoMdfHPRMPPdFSSEjNrkBFirL30uI1gjgM0NH53tklbq3/JYpIh1h27rp+AMOwriDGjg+L808+M8pwksGkb/AD6LeKOWqi01MvJfmiv+GTuQjljeykgJhjbnD/M03LjputjGcJxnA5YafFMHn5URfFHI6PfVRYY+ngqBVl8LhHne2GSLMLfLooO1oMWpqevo8WdhVXVUT6GJ7m5CGExgC/uAVQ404/ruInU9J4F1JFB+JlcLFz+h+QU/DX2kcnB5MAqsBhnYI5GAtn5Zs/oBZS41htLxHgeFYs2aOGrqmTmQSDTJHp6hta38UHH/AHrVes11WJRs/nHT6IqSeoLRVCodlErGOL3XBJ62VZkLI3yZyyRmTRw2VnDqMz0YZTxtzvcbPGhWc4FviSSKmr4qSlyWYGF2SOwBPT3V7jTg1uB49Jh+G4jLMw0rKmXPp5zfQfosGrpqmWaQVXO8WTbKRcm3Za1VSYhidQyoxHFSa8xhhiqGeodr7bWWKHTYHxvQYngFDgeOU8jJcOlilp6iElxlIJvp9bLrPs4wKnZV4xVSzSVcI5dNC2cXBj9XX5ri+CMNfhvEME+IkUjYAQZmeZhJtYWt/wA2XrrZreaGRpb3AXSBaNOwUvg3tDoW6BjhoBfQLyjjngPiPGMYOI00LjTz1B5UQvI+MPtd5toBfVeoc+T8yeOR7jqqPL8b4XrcJqcLrsVdG4mMwsjadAWi9/47LG4q4RrcOwyGvjmLpaybl8uFujgRe4I+a7z7RHXfhrPyPkP8AulpoInYfQmSNpMUTCwncHINlKGZwbhDMN4Yw5z6U01TNTjnt6uPd3crcDASn3CTdlRdpMkYAV2LD4ZnZj/cstszARqr1JWgGxcjS8MLpfyJ3UjOgRirFvUhNTHbLmUZQyRM1VOU2crcsumioynzIGJ7oX66JiVFnOZIDOiN0JBarAIePdA8fCrCwjz9yudqo3zYg2OtfFyKW7iXP1IO3+K2a6SOGmc98mU7N06ri+KaPEpYI5mwyZrh5fbVvv8AJEczj+F4VHWMxOirpamKTO+dua7gd9lXxPEIa2GGKOZsfKjBJdubLpcSwaGmocNxWegnzCPJVeU2cwj/AJP1XMYxRUssQNDJGx0oLI8o1aNNCFgUcRZGyKjnEcjJmNeyUuN9tlDFIZImwioMcRdkmDe3SwUkFFixidzwRzByW5iSy/ueioSUjqaOnZUfhudISXb6dFuJHT1dBUQwVM+FSGNsAZJk3fKSdj2XP4ZU1cOK+OngJERebHQX3W3w/VS4bhtbNyzDDcDxDtSRba3Xf+K5+lqHmGcsk8r9bW2JKrStVvzfjzxgPlc9xcNiCdFcy8jDmtfEC+XzRl24WZPFP5SdWl/LDuhK0qiin+8RSxte6a9m3NxZZZakOJ1UuGfdsUIiYz9tdt3uvqDZZmHYlHBUeIfUSR8qIsjLRexK2nUdZXUvhHwcmqeY4AWnS7zbdXuJGx4o77gwRkUdNFGCCyKxc+PR47m5VE8GLsq8INXLGXMjj/GkLcplI2A79U0XEuH1OHeAhoDGxljO539YwX0t87LGno8Vwqqw+qxKCSKjqXCMQyHQ22BH1utfGsGpmSV9RT1DZnztYywHpOlz8t/kisaXGX1gEdPG+kbmtaOU2J7r0PgnEJJKeSicfwqYnmSOkvcnovOoKSKCgZVQEyvppwZGhps4v2/uK1cFrJBg1bDGZGOrZtMhF2/82RHQxO+88YkqK7zUtBrFn0BJ0vZdFBXU9Y4tppM1tzY2UOA8GOpcFbJznOqqmMEukOgPRbVHgVLSRcuPQ3JPzQUxGTsjELz0V8UYbsibCQiqjYHqVjZG/CrrW38pUgg0URAwEbqw0kbFM6F4QuEg9KrQnyvvuibM0N1VbM/qmJf2QKSQZr3UkctlWdmRg3VRYkkBTNmsVD0+qH/ZUFl0xeLXVeTfMnsUzg0jVEKMuFjdWI5iOqp6gp+cGizkVpCe40co5TcKpHUNsnNS38yKBwG6EDVJ0g/MmDh3UZSgWSykoA9SNk1QB4V5+FL7vJ6K21/dWmOjMeVBly4fBlADbqs7DWE+SQAdlveGjOyjko2AXaq0512Gm/UqaLBWObme9abo7H0qxDEzLqERktwJg9LkJwsN6LdAjsmJiKiME0DgfSU4oStwNjcU/IjPRUZIow3q5GIbbX+quyU7m9FGIyeilitynWvZOHuZupntcFG5oVEkMvm1Vy4c3RZmaxU0E5HlcUFieBr4jlCz3QBvRbDTmYqk0TLltlGlFjLKZkpao3HIcqjLiSrKLrKoBu+qz8QmfUSW1OXsiBKJrA0oij4adxsGEfNI0s43YtEmyYvugy30ryLOFlE2gMnULSkaXHyoBGWnVqDk+K6J9LFTzEjVxZb/AJ+S50tu1dzxhAyTAzO71RSAj+5cS30I0jfo3bUIGa+bdSvAuogMjygscPSvjxnlg+V7TdSVjOXWzs/7wn9VSw48vGoW/nNlqYwzLikvuAUFJ3rtZETrZM5t7P8AiCTjtp1UgyNJ6XLsOEIBV4Q/S/LlsuQIztyrtfs7/EoK6H8kgP8ABWEbcVBAwehTCjiPqYmmbUxOzM1ambVv6uWrZof3fB+T+CPwUBZk5VvkovGHujjqSfiRQ/dsY2af1QHDYS67oQT7q5zQfiS5wCqWFjMthkUocAdQg5rUOcOOZqwqzzW2yDZMxjS7NlTRtG5Kk8rBdA5IazQKF5c4ehM6dn5lGagd0EcgLXZkxkuET5mO9SA5SdFqBLEbPCuvdG+HJZZ4NlO2RpZZzlAIhDXqwJH5cl+ihaLO3upgNEmSgGNn5VA+IE3yq2LX1RFsJ9RVsmGU5tjZOCVpSxwWsAqb6Q6litpSIgO9KjMeqltk9ShdJ5tFhRBpHdGGOveydkjVKHgq8g8TYgPOmnEX9W7dERdDkvsU5CBkDA7PmKUrWHdPIXtNkOYHdOQqVDqppdHTOszuvNMXgmo8RnikFnZtl6mHwu05i4DjikbT4mySM6SCy1Es8XS8N1LKnCmPZ/V7rSm5csfk3aub4DqXuhnpCfMDe36rpHMaHOaxhBHUhRpl1o8llmRxZn5C9oB3BNirGMeKqmSwMAjeB5SAuYwmvlkkc6Z5c69rlBVxbharlxdz8HpeZG/Xe1ynqeHMUDAyqw8B37jr2XYUbXRRtmqH2anrMUwencJ5KkhvsqtOEq31+HUpZJTQGKxGrNVgMeWr0jiCkwfGMKNVSvzhhudOi8/bTQyScuCRg7ArnSUeAXeBJD9XjRaxwd9VDaCWjzHs7VZlZUTyXiqHXCjzyU0AdTMLe91riqvNHPRzujlFnAoPFyxPEkb8rh1AUL5Jp3l8kxJ9yo3m3lCxMJK+/GMQmIz4jMfqpnY1jIg5P3jNk7XWbEGdVo0MlK4lkzrKwKWdyE6q7PSsklPI1HdBNhdVDEZyDlG91RQDp4ncxjdRsunwLjV9C0x4jRZ26ZHM3G656PIR5yrcU8MTMvJY75oOmq8Xo8djfTUMjYHP3z6G65Ktwx+HvAfNzL7G6jldGHl/L3UUsr5n3JQaVBG98d42ZtVXrIHxynOMp6hNSTyQG7JLeyGeqkknN3aIInTnZriB8lEW5v6wK3FLTwsJ55B6jIoOdAXZ3NB+iAHQtY0PzhJsv74Cjkla4+XZR5rKxIlcX/6w/Qp+doBcn5qLmNG6nMHkzhZoNHUEnK5EZWX1sFCG6pOA6tShbjazpIEJiDnZrqqHWRc54ShM/wAuyYZ7aq5g9IzEHlkht7q1XUMdJmjD4zbrdShwkcMoePIjqZLXjutJkHh6XxNUeXKdm9Vnztie3mAG/UrQGB3kMPdW4YKqCzOQ4g9Qq0NM6SMyM6LqGxc+ljfF5nNFrIMfmcr4rJGpz9brdioqKNgfiUcUd+2qPGoWRUUfhaKKOG4Ae8an5LI585suezQonlt7qV/LDrFwPyQmFjjcK0FGRbNdTskzelwt3CiFK9o6fqrcFLZtyQFzlYB+O39jJIPqvTODMWqa/D+XXPGeO9rC2i8/jEAItr8lpYZUPbMIIZnR9gOquOTnMPUXD4kL4IKiEwVMLJGP6Fc3nq8OtNOJLdnFdLBVQ+FE+hZ3IXrjJlxeL4dX8FVrMQwol9FJbOwG4PzXXYRNS45Sipjq5nj42NNgCqlRjbKiU4dRUIrXSeQ5tGAriKvCsW4ar2eOMsFLUH8Q08mluy6D0OerwfDyedVRB35Qcz/0UI4kw4W8LBV1De7ITYKfDqPBhSsqcOga4SC+d2pV2N1vhb+gWBShx3A8VvQvkeHP3jkaWkrhftJ4hkgmdw7QG0QsXP6kaaLo+O8WwahouTimGzyveLwzxAANfp1Xj8pNTMZnvJJ6uNysy6ALLC56p3RWizlSOYXtY1nQ3VptHJXV8FBF/WO1XKZbbf2Y4DDj2NzQVUV6ZkXnPUX2XYYhwni/BwlOG5cRopXcwxv0LfYKHA2YJwlxXHDBLHAx9KOaS7S/v+q67iySoxGhfSUjw1hjMjpewGuiSxLxjHp5sUrJZJA2BnoEYPZWcNo5IsLEzB54Jg9pU2GUlPNgVdXSfiS0ZJt0Klgr4WNp614vTSHlytH+C5w27Z8uHYjVwSMfC+pfAOWSRdp7riMVElHW1NAasT3dncWtAAPZPxmZaLEYK2kmMUpaA3LvbpqtGfhGeTDopzWkzVdjoLlx+a058XGTzBrzHGCGjsVLQSecZh5R1XQ4zg9FReCwXOObK78Vw1JWXieHxYTWy0MJc5jNy7crEw3DRoqpjsRbhzXiPmt8pvrf/FddUYP+DBDS5g9nkcAdwsN9F43hqKqw+ijieN3v9drrco+JcPhwqJk9VkrY25CbXWsWZW58UrsDEDWZncrdriSbDv8AqnHFtFHS1TI5jJPUuLw1gvy/YrlxxBiM1HJMYS9z7sMxGgVN1ZNhGECrZyzNK7y9dFqVTY9jUtTWA5/M2MDIf8Vi1VZPJG2F4axgOgAVWerdM7m57udugkqs7Wi2y45fmaiR1r4ZaY08hs4rAlpeS3ltbt1VyslvKfNsqwmY+/OzHtlW8TJEZOWwWBtdOZrxZAdDugc5mbI70qIva1thuuiHcblJrnZw86lB/aUgy5S3uoBJ3sNSjjDmObIz1DXVDYZtEjLY26KiXJM5xneQe9k4sTl6J4WmSJ2vpUYsG3JQXoI3Uk8czHG1rOsOiCVzDO524KeGalfCS55a4Czeyr57Ny7nuguwUD6qaKCNmeRjs4Z0K7bAMPjnaOfAwZSQWgaAri8MqYm1BZMZGZxYFm916Jw+wMgj94y8/NKRhcY4g/CK+OLDah9O4x53ZCRfXqsWo4qxauo3U1TVj5iMXcg4xmNVxFUi1sgDd+wWJFma9ZprxFoSSxwRsk6G4UbZY84dM247J521F88rJA0jyl3VHSUFVVStZDDme4Eht97LNMzPI8L4ZZix8fMJ9OtrFWmRwZ3g6WYVZwqkoKKlrK/EYOZy3ZBGRc6qkx0ZZJPz49QTlJ1C2QRihDXMjd6zdC5rIfM+QF53PVO2WhmgmJY5kzz+ED2HVRVEEkZbzGWvt7hSg738yPlsO3p9lHUfgG77FxHQ7pi+Nh/ZuzdHAaI6ksfEIc4e4u3HRJhpaqY2UVPCxrAZpRfXopJZ4X0tK9gtOTZzOotsqtW3mSg5icgtqbo6OeMNMT448wF2i24WJgWpI31cgfDdtQz9mL6XNuqqx0WIc2aPwv40ZJvm0JW81sda2KOm5dPG8Wk7gq3JRZaiofJSCFnJDMpP7UjUmysJDPnrK2LDThlbVRl0hDLt1fGzqw/P/BSyY3jOGN8KysjisAWBkIDyD7/RMammgwkVUtPFUyyTmPI42LR9Vk1VS+sr4Z2RSNZHuHOBt3W7bp1eFUtHikBxGsj5jIxzJGzHQPB/iuc4lo8IdjL6qgqI4IZACGM2z9QB0Vuhpq2tkbSsjliiqCSJCLBwGvXdZNdLTnwcIkzOiDy59tSb7JDlSTxFU2idSwTl0DBeRsfQdyrmDUmFYlRVbJIZG1LIbRgOIBkJsCtDhCgwfFRWwVGIRQc10fMBdYCK932PusjiGlw3Ca/l4TiMNVHmeS6BxsBfQFYVjSST0s76eeMZ4zZwPQrpODKPCq/Eia7Fzhk8BvTzxt1vrfqsKKhNbBXYjNMc0DmaW1df3XpX2O4JCx1ViNXJRytqYeXFGXAyWv59D9FuFhVxTDsZ4rwqtxOu4nmlbg7gAxzbgxnd+nXTX5LgRFZ5D45cv9WQLA/5L3TCMAiwzh3HaSujyRVEshezry7GwBHcLy2bh3HX0WJzMa59Dhdo5C/TKLAgfMXCIzZOGcfbgP8AOYYd/QASzntI0INjp2v1XRcCcZSUjocCxDw7qeUPhY+/nBff+F163w5SU2MfZ/DhOL0jYGVdOYpWAWPa/wA14DxRw7R8OY/X4VhtQZ4KK2Z7twfdB0GKYfS0nENNw34UZI6ci4uOY9+tzf5LJqsDqMPxDw1JBLI1kYtKDdhvuqTsfr6ySJ9bUyFzI+VzgPxCzsSrE1dVMw2JslceZCQyPIbadljIX8MY2PEYa+omzDVjXXvZ5FgsqQ14rXUtWBJaUgykaX7qegmYJ44KbU1Lo2HNqASdwujq8OosHfXvqaSczUcFjzn6Olf8Y/56qUMmnZNTQubHisr2yuD5GA6XBuF69gnGGDYuI6USZa+wEseXUleQNl5+GSCCPNKZzLZrbHlsA1P6rQ4FZjbuJaSOlpJHEu8QedoOXsX/AMVYHtLZA8aX+RGoUgd3CJ2QjM1qjuLrsOP+0FwNdhrO0Jf/ABXX0zgKOD/wY/7guJ47dmxuhZ2pz/eV2ULv6NAO0TB/BWi1nPYKN04CC5IQGO6yJBOCVKyot1sqJa4dUg54+JUttw1t2jVTeIDzq5YbZCzZSsqnfmWJhbbfOFtXKtJICdHKp4ltt0PNupSrQdmKPJdVo3aqwJdFQxDmuQueboyc4UMnkaXnYC5QO5gdusqtp694nrZyBynZ4rP0I7FR0vFGFTueBVizbWJ0uq+M4pSYrDJgTZ3g1Ed454teW8ahGWfiGJ43WV9PTRxNEDHBkjGyZg49/pZT41w7gcszpZqLLVMJH4AykjqT3XJcEYpPglRmxGoLg+pjhyucLxkE5/7xqvTp6OlrHcy+a/xsKg4jhihw6WkxLC63EROKkWa0gggi9j/bC4/EaTEGxyQSQ8vEINJI3m4ItYFi9hjwuhpp+THRRmNw8zXC5ce91wP2mU33JiGC19BNI58biMj9gwWdk976pA5Woq+VgNPQsdI7xBHNG5aGaFTYjw3Ph1KytLMsUpYRCx1w5mhP8FZqOIMGxPFy+PmMhlaSc7bWPb+G6mr8eqKylpDWvjibFA8DI3z9r2uqrkcRqWeAhw9m1NOXtd1N9l1dPgn3ZwjW8RVVaJZ5msja06EWeNvdc/4eGbGsKp3t80kkYkPcX0H6LoMeqH4bOKSSokEVO7JmLLxtDz1+l0Rq8L4FPVVreJMRmjip3QisEJ1yjUMBPyBPtcKMup8Sx+Wr4brTHTXMldVPb5IgddCd76qlT4NiWIUsbMKknYypuJWk3Ecd7B/1/wAFVw/DX4bV1tI6fnUcUIlnlabA9tOp9lBsTSff7zTYFGZuXGXz1M8tzYfH7Kh4l9HOGCOpzMlOaR7LAn2XQ8G49hzcEr8OpYTzBCOZPO0NFzpe4WxheGcOSwnFPvSKvqaa93A6X+RVaYcFfSYXRSsOHuF3HUm75CuZr2TurT42SGlfOc8bI2k3B2f7r0mqo58QqqGmhjjZVROfJK8bRC3X9f4KSn4Hwjx8eJyVFXPVQON3TO0dpbboto5N2GcWsyvZxDOaN7mBzgC0uFh0/hqvTaebPSwl3qLdSqlZiuH4cTRcxkThvmGg+ZXIit4mmxOprsOLqyjcABNL5YwL9AP+dlhXdMddSBlis+LEqA1RoY6uOWZguWNNyjpKiq/E8SLeby/JbRfAsVZY24VATuLt1YZK4DRYVaygqGUC9ik2TXVO8tJzZlBVe0BLQ+lyORjiqzgWKoUgFyojbqiLrjVRO9SIlaVagsXbKipWTctBdkbe7VWdC+1rapNqgTqiNQ3ugoyRSAkOaoXBw3VuadhKrSSXVVDIHHrZMC9vxuKcnqUOYqIMSu6lSRyjoVXLwgzC/lKDRzXGiUcrQfMVSZLY7lHzLndFaDZmEbqaOfVZ8d9Fah3UpGjFOpXz6KmC0DdPzG/mRo0jgTmzJGpY0bqvJKy1syrvNwqi394AIDW6+pUTIxMPN5kF5tYQb3U8db5t1nI27ojW5+ZvmchM0Y6qi2SS2hQSCZwKQsJqmqY5vlchimbI3yql4eY7qeJrohYqiVwsUm3BUZkulzD2URbjncAhlmcTe6q82yYzXQHIeqjDkJeCh1KCw1zLoiQbkKmXFvlKKKaxQO5zwUPMcpn2eMoUEsZaMygIT2R85jhqqLi71IOY/t/BAuJcsnD1W1vwAH+K4EbLuMScZcJrIT1iK4dvmYHKhpHMjGeTQIGlko5jNul1OQ12jhoo7BgIYAPkoKrHcvEKab/vAFucRC2JRP7x2/isKp/CDZst+W4PstvG5WTQ0dW3XMCL99kaVtCPohu3ZBzT8Si5hLzqq2mJytXYfZVL/TsQpSPVHnuuKlfZq6P7OKzwvEpjcfLPEQVGXpksLSCCNFXDYIvK2ED3U0tQC46qCR7XDRWGKZ2MQSROE9PGXA726KjFWPHlebFdDDUZW2kKyMUw2+aqpWeXt1XbFzyQGsd0epqeZ8u5UVHHBVQctwvKtCGhETLNWsmQofOPSVOadwTtjPxLjbsCJ0xNgpZROGenRGGgHQKQtLm2asjKMr75VJTRSTutqPdGKQsnBePKtWMMyjRBX+6HkB3iR9ULqB0QOYg/JWi57TZJ8vlIsgyZpTG7KnhqmndyjrQb6KiZHNK0NlkzDup2yA7LHZUNGxU7apzVJgX5H6KAPsdTZV3VZ7qB9TmJUGmTYX5w/VQms5fxrPdOT8SjeXuWxbnqhICWOVcSk9VWe8jZDzXj4kF5kru6sxSPKyecImmRxsBuuerftKjpJjDhtEJsmhe8kC/ssDvhnUkZNvMQfkvHcW+0DiTEm8uOeKmYekOh/Vc9HPXxOD2YjMHD/vSg9/qBnbZUnnJoV5Rh32h8T4ZKDUVT6uIfBId/qvQuGuJ6DiZjuXC+OZguWk3/AOdkFzlgTZxdYPHNIZaWKsZvGQF0ro7FZPEsL6jCZgPgF1YHOcCVkdLjbROfLILL0iYQfEF4xh1V4eeCcmwY/VesQ1UdZTxzxyNLXt6FdMhRxRjGQvnjYGlnYLyeasz8QB8el5dhsvXsSic6jlt6bWJ7Ly92G0+HYk2orXtkhkduWrA3MZ4mxKjoRkoIo9bZgSuPFdNWTfjOu49Vc4pxCKWVsdBVcymIB7rOwkMNYySRxDR2F1kt1VTikFFgrqamZd2UjK4LipZ5JHl4jDb9l1NVUYWXZ55jl9tFQqJsEiY6Sne8uPtcJI52V8w85BPyQOqprWzlbFZRNij57pY3MPRix5o4y7NGXAHoVLEPMJ0TbHUInC2+yB3JtZhdfrdUTcy3pUoxGcRiIBoA621VLPrZEglFRM12dkjg49ij8RVSCz5nEdiVWBN0Qf3cipC+w9WqkbKLA7d1A6RgZ5dSgbJpqokrRew9QVGS4H1j6qHmtRseJDY6qA2y3OXMEUht73ULoWjUEocxHUqwQktmKGSIsGY2QZ3DZG0STbW+pVEYBOymFPMW3sjZTct7eYRYnotZhgbGGteb+wWJniMeAOEgdGAXDutOWSWohyGKNvuApHPaNmNH0Sa67bFc/EGW2B+axsEE8D2G9s3yWtO6ExZY2DMqjncmIzWDgteIM7UbtP6J83sf91Wo8Xe7SNjPqFXkcC4vO5K1E8g0U80JvG8tKeSokdfM+5KiIcfS24RMikP9WVsSR0wxbDJMSqzIXgXY1mzVXwugrZ6eSePl5WfmG69JixPAp8OkFJAC3L6BFb+K46mpKunwyqmY3K03s3qFviObje+7hpofh2Th0jtDNIB2Bsp6ajkfGZOW+zj0CruZlkcMjgfcLA0sEBildUPeH8sXDZDe5XQyY8yo/BfRMjDurxouPzPDQ0GyuRztmH49ZJdBbrsVkE7stPTub0IG6qU0JlF1abPHA3I2z/mFGJw06NWMj3jJaz8N4H0TksNrG2ii5jXFNJIGC5XHiJmkAq3hs7KaujqJPSy91kPqwPKGuLjtYIedVSNyOZlCnEd7iHENPKz8ar5zn+mNjFo8GYNXYowVeJYoYKe9m0jTqfn/AAXn+FVMeHVbajl5n93G67fBiyOqE+dw5jtCDZdMcmHpcNDSU8eSlp42N+SpYlgNFjFI+irY9H7OG4KkifXAtjLwlX0dfVxch1VLC07mM2P6r069iTDz2jxocBYjUYHj00jaa2eE2vp8vdV8S+1qhhlDMPwuSpZ+aQ5b/RbHFn2dUb8NfiNHJVS1kXne6SXMXD5LlsTrsAxLhiQ1PLjxGLyCPY3uuvK2KUeM+Pf5zUcNDT0skEQ88gc6+umn8FyofYKJwDTa6V7dVznJ2iFlszg0WNiTZdzhWBfcNO7HKqo5svLvcbf86rgIWvncI42Fx7AK+cTxR8DaGqq3yUoOsd7XHa6xHmsrtdJW+DmxatgkDquTR1tAey7rDOKWT8AyzSP5D4wYHG5dcFcFxLxCzE4IKGlpTBTU7QwAuvdbs2AYtScEZJZoRHI5jjGBZ5+atObk4Jq58c+G0NVK+KYlhANg4e6s4b4iqmosEkIijfNrYahdrwhw79xiSongjlzx6uabkXG1v+dlcw7AIcQx9kdPTshEwvI8x3kaEmGuRcd4bSxYAycReeGRjGuvqArmGYnMcGpuTTCZ1PD6idAVtYzwpBiGBS4RTTud4dxlD5jc3Hc/VZfDWG0zeGXQYdrP+zLet79FmYSIcp92T1L5MbrwJZ/WADYXSw/C2xSnE8TqGgk35drldqzgiqOGPmxCvMQYL5Wi2q4urwurMUj3yRugYbZr6uUmXSFJ2J1RBZz5o6e5PLOypS1Ekkboifw37uG6uTR0kUfhjF5x7rPqmtha5sY/DFrH36rMSkw1MHraYQzYW9rnh4JLnH/BZNNWPFWx72yPiiuwt3Cgp5JWyPmhkyuDd1pQcNVbaN1dWz8iI3cR1+q3M2tKEr4jI58LS2MnQdlDI5oBy7gKVtVA2NuWINAGp3uq0ssbi541DjouVLEsxxNydbne6C5VpsTXO6XJ0uidScpzg8fJdIRRcHkDTUmyjuWus5uoW3SQwRMc4jz2uD2WfPAzIJtLnsFRCI7xc47XQg6oANbJ22DvMLopHdC4FGbE+XZNp7qokjlfHoDodwmdqbo4mtkIYUpm5Gj52RTQnyuHsnaLR+Y+bsnZFJk8sb/mQp4YbRukmka22wJ3RGvDNDPUxwRG7eX6V32EM/Cjd2Fl55gVLM6tjnazybXXoVuThs37sT/7lEeYYpO+fF6mqf8A1kpse4UVO29Qx59LDchKoMxZBzJMzA02bba5UsPKjw90mWznyb90Fo13icNloKjXlvMjHX6/JQRzSNaJc5AjbY5dys90zwTZ260aeV1JGWysB8THYE7j5LMwKcskcluXcW631KenZJz2Z4RKAb5Ds72So6KatqvCwOAO93GwAW5hWC0pwqqxirrY4zTufExjhYAjaxVghVxLEBW1IdyOS1gsG9VGTAJI+e/KxhuFRmJkooZsrxLISS87EDsjmimjjhmqTZsrPL303VF2UwMja+GAwlxIyDQSDuoBMHSEsykkWHltYpSMnkYWs5riwXzO6KowSNDi02+Ski8PwKeUVBYXyG4sfSoGsqmuFQ9g8+xGl0bAyqgFK4fjbMf1JViZr5HR0r6bVgEZLUkhs4FBAcOqK+qhE/JIJYw7n3ueiz6ytrcPxQ1dRTt5vQONwWKT7tiiaadtVJn/AC9FehD/AAVLPJYSc4wF5IzknbRcyGJW1NPLK6oYMmcAke6h8RHDJG/9ppeRu2i6qjwnDqzCcYNRThopM7zP1B6D69lxkEfPqI6fNbmOyX7LUS1yercD1lbi3DFTh5YXPppDFC6fS7CLjXovMK+nfR1c1JJ+2gkLH9rrdwbFMUl4mho4MWiaTMBEXOywuIGgKu4zwXxI6vrsUxGha4mQyvEMgyOZbWx+nZdZYlyVNPNDd18p6H3W/BxCyKAsbSQP58PImzNuZPf2U+PcDYphGGvxHlCSmZEycvvrHn+A9/msmmyOlbDNmpmvyZdNQb9ViSHaYDwL4zCMTNbLaScs5TYzq0gA2Pe912WFcF8O0TYJ2UpjqIvPnjkLSH9e6v0cb3xCNoJMbWMLyPVpurFVVUWGUUtfiNQ2GCAXc4qss/ivwtXRSOfX8pgBMUBfYTSA7+++yq1GFMpsJxHAsQcGU+MNErZmaPjIYC+/ysNV53xjxTT8QYsJMDbKIWQmCMyCxvuSAu24+nxTGeES/A54hQQQMlqpw6/MAFuWPrqVW4lX4M+0KDD+GatmO1xkmpHSRUrhHcSADQX7ryiaSqh58dRfPUOzyFxuXBaIhxGLhx1DW4aY6Z9QyVtQN2vIH94WxwBgFLxLjkc8skDfCSESNkksZn2uLIri81rM9ltcOPghlmmq4IpocoDg+Qtt7jv1/Vdjx59n8GGYbWY5zZDUGXPI5htDCDYBgHW+nyXD4Jg82PVtLhEMoidVScsPIuAfkivUuEuDKCarp+IBPT1MEf4kTINuZ006WXQ8V4bS4phzpq2MHwzHvaLftHmwF0fC0NRFgVMyuphFWRjkTENA5mTQHT2WsGh1s+tiCiOa+z/gPDocRxCoxCjaDBTw0pi3Ej3sDnvPvfTRdpHgGG0tV4ykpmwvZEyCPJ8MY6BPSO5Ls7NC/wBfurvO6rMwjNqGEE3UGUk2yrSkMEw87tVE9kDR5VsefcdxcriSjjve1GD/ABeu0ijtDH/4Y/uXGccuz8U0/tRsH8XLuSQ1kY7Rj+5URCN4GyFwLR5mqdsjeqjrabxcYaw2cDcapDCuXoQfdVnGoiNpo3MHc7KRkxcA0OXXizyTkaJtAU9yAoy65XHKHRLmd2Rtc4lV8xQmR7T5SVlWiw2ViLVctjXF+HcP8oVzpJHz+lkYube642t+1XGXSVTKWnhjpnjLC1wIfGO9xujT02s4l4cw6XkVeM08Ut7FpcNEp8ZwecupYMVpZHvFgBINfkvnyepFTK6eY3e83ce5UJmiWeQ9RxaiwPBpP6Q3K+S72Z9Yx0F+y5yt4mlp6oRirMoicHh8Z0cVykmITSWE7jI33N1G93ON9k5C/V4p4yZ0szhd7i82HcrRwfjLGcJlibS4lOIIz6PVouYmD7gsamZI8ODXi11gepUvF9XxNibqFs0scRGeIg2cCN7qpj9U6mlpZKipkqpqY2ImHlkJ7e64WlkmEjTDKYnsN2uG4K7OLimlw2ghgr6GKsrSM8srtQB0KRKMeZzMPxGSpmgbzqg547N0if2sd0GL4icSHPnomslbYF0Y/UK1jHEUGMte00kUbjHbM29wRa1v0UeFS0uI0X3dVNcZogS2TNa5N9CraMareaWpge193AcwHsVPiOJT1lBHA83Y9z38w7yP7lVDSySYjHDNMcoIZmOtrIA6asqmhvpEmQAbAey3Y7zBqzDxg9TTyTyTS6SOJJBIDLD6b6LEkqsRpcNrYaelOSpmEWcjXIOisjC62lrYIYIuW2rDI3MvfMf8FY4so5sEiEL59TDz263tra38UHOiskpYDT0jTkls6UEWuRey7bAMVpvuicPZHTsA0sLOledbN+S4bD4X4lWxRvcMoOxO57Lp+D8Cqqx0OJVFPHFTU8147nQkbn+8Kq7jhts+HwMxvEDO6vqXDNFta17D+KN/EYho3wwsl8VUSl5eTcNF/wDkKGtxzC6t7R4wwSkWicdD9PmsepopmPZPTGJ7n35jBqQFpGuyuocUrp555mg+ezTs02t/gU0VLin3caahrKQUzHEuL4iTZUKuixoxD7uoonsLQTe2hPRb+DQzUlIIKh5Lh+Y3KlFqVHgUFNiXiqSZ0UsZBlN/2p6gDoP810ma6gAbe6MSfCtKlDxupmzEdVTLrFLOT1WaLaUdQ3a6I1Gm6yeY7ujD9Bqs0i+aj3UT6iL4n6qEy3+JRSiN7STv81ugRqG5rIucz8ypEAfEh5n7ytJa/wA5mXdBzAVUD3KSM3Cwq1msU7neU5VBm/eKWvfRFRyyOuojIVJKxxVdwddapCMvugMtviUTs1/Koy9wQT83X1J+ZYbqsDmddE7O0+yosiUHqibNl3cqQmIRNmJCDUiqGWvnU8NbGTka9Yge62iQe9huHaqDpPFNtuq89V5TkOqyxVvtYlC6d3dKFoVFRqXHQKR1Q4wnIdeyzjOcpF1CKkj0lKGpA50jSXepTZ2xixWG6plHpehNVKd3lKS2vNWgEBhUkdabBpKwzOb+pG2f95KXk6SKrYRe6PxIXPx1gAtdTsqr7lSliW1z2FA591nCpbfdGaxg9T0pFp5sonPI2cqktey1mlV/G9ylC++dw3KHxX7yz5Ku4JaVFz/3loawqNd1IJ/3ljip91Iyo90Gpzs2lgoX3DtFAyW6lDwRss0LMMxB8xVjMHjVZ4dqphKoJHxMPmUDoXP2ajzn8ycSOCCM0Mkscjds8bx/Beet9Nv+d16dSyuL8t15nM+GOpmhL/MyQgj6osH6fRCR0KN2VC5zO6iqFc20LlsYj+LhFFIOgAWVWlkkRAcrVI6olwG884kbG6zQOgVEeQ/FZCGa3QCdnxuUhkaR5IZSO4aUB2HZWeGaiSn4mo3s+N2X9VULKt7csdHO4HqIyreC4Ji8uJwz+ElgbE4PL3tsAFqmXqbg4OdfuoybIHVRc7NdMZW21UoG0a2VqFotZUopm5x5lZbMOhVoTmGBhzRxgE7lNso2zsv5ipmvYQshAZ0Yp7boCQ0XCKOqZ8awD5Q7KURWCHnxludpTCqYtBy1nxBACbeQpSzMLfKqclRlOjrKxItPcWi8mygkqGAZgqsleS0hxVV1Tf0lBLO/mArNnzC91bEub1KOobmBLVYFETWPmcpRUuHpcq7oy4p53UtDAaisqGxNA+LqtMrfOe7dO3VefYpx/iJldHRQwxM7jdNh/wBoWLR6VEcMoG122J+qw09HjpnyWs1WBRSW1avPIftYxGAFk+CQSnoWuITO+2TFY3i+BQCPtc3/AFQegvpGHooJ6aCBvOqJI4mDq51llw8c0OJ4PLW0J5MzBcsedQf0XleM41iuMyGavrJHgeluwaEHXcUcYQTsdhODkSczR0x2HsuSfhGKs1kDWt7g3WXFIY/QtaHiDFS8McGyX/MxSxen4OxKGjFaMQhc21yADosd5qKd+R5WzUYhikDR+PdhF+WBYKOtrMKr2RRyTZHs2DxYfqqKYnxGEZzG4MPxBqt8KYqcKx6GUTctkpyvDha4XQNwalrsBgkuI3s2c7a6yeKmVcOFwQ1dPSu2/EjKRI9UpK+hxRrn0VVHKRuGm9k1XTvfTSxvZdpbqF4phGI1OEyeLw6sML/YXRyY/j01SZ34xVXd+WQgKwLs8b4qqaBzbBjtF6VgETa3DWTskbG+1ibWuvMovM0SHVx3PUrquF8Wqo4nULH/AL7V1yHW3khY6GQ+X8zXLyjizxFJijqeaQyM6A/RdnVVVVz8/PeHs2ssLiidmNUzaeaERzRbPdp+q5jn8PimFMZnUocACbErSg4wpYqfz0cbnj4eXZUoaPGcPw+V8jGy0wb8LtVzgLiS73usi/iOJfeNU6oyBt/hbsFWzVPwROy97Kuc7TcEKwcQquXkzoBfNI02Oh90TZ7syFoVZ8pcfM+5RMifJ6NUBeSQG5A+ahLCDoiLS05Xt1am190DxsgOs179LISzX2UjW5jlQSt5RtnKBX+FMVPyKflh/ivMRfL2VdzdCUWCNiLoHbpF5HyRsZdoN91ckRXsdUTSb+VPK1jDvr2ITZDa7VBIHm6LmtHwXQNY0jzH9EvQdAT8grQMlhP7MLUZhULqds7ZCb9ALWWW2Ook2hefotXD8XqJYvCCJhtsoJPBUWXO5rrt/eUJhLb8oOejqebH5pLAnoFUdNKBnY5eeYE73SlhzxuaW91CaghuT0peMqntIljz9rKB0ZkF1Yx+IuU8TJiA83DlqeDpmgDl6+6xaJ74pQ0nRbt3OGYrpxgReBom+mmj/RLkQD0wMH0UiSseQFrYwMohj/RNlZ+Rn6IxG4+YBM5habEKjP8A53ziPl0lDAx3dUqvFMSrGOElg1+4boFjMk5brozVylpaHboNzDqYuos4qjCL2ADb3Khmwqoa7mVXMBdqMw3C3uEpZ6nDSHRsbyjkuOqjx18kLcsliXmzSDsqMOnhkkuylp+c4bhX6U4fG1sGJ0LGyj3QYfXMw0CR5s0791arMQweoYZmxkuPxoJKyHDjSGWPlue3Zrd1z9VyhJlkcQfZXeRC6oBpZOY49L7K23C67d0MQ+brqSywntjjZ+FZrndzuUmUjyzmTTXPZTz/ANILzI0Z2Gwso2yOa0MPRc5alEGG/b3SaJGn1cwo5JdMrR+qhdceYuP0WBMBIT6Fv0WPVMbGwzRloHxDVYEEmvVWeeWbKwO94Y4ip2YkyB1c5/NIF5TYNsvVpIXy6xi7e6+fMHEeI1bKSWEBp6t3XcRVOJ4UGnAcbkm7xSnMCF0wlaejCkl/rBZnVfPPF1LTw8T18NKRyw7ygH2XfYz9pr34bLR4pQz0dZI3IGMByEd7/VeVRF7pnSPfmcTe53XaMkjBD4R0tXfLooq4QxHl5NVfmblYXqHwUopm4xLC0wxu+LUFZhulubAa+ipYa6heJHSeto3AWpwnhVJI2ogxilzza+RwsR8im4axilnr+ViLxG0n8G5sGnsvUWUb307Gsmp2yvFwRGNQrDMw8ug4YdUYvH4aklMUbg+QHWwC7TiWBlXQBrJiGwuDMo2+qipHx4HLM+eqzOJIkjabi/cKhT19PWSinYZyc2YtaLquUw6LhhsEFH/R7mXMGEk7iyv0tVDFWVNRUTxNfK3I0t0N1lUoqZpRBRvZEDswNuVZr8PbQ08dRFDza+V9ow46X7o3bVqp6fCuG6xlRUR5i09bdVh8AYhDFg9dLUSgBgu0uOru64PimOvir5IKzFW1MxN3RM2b81miSWWLlRSTlw6A6KTJEvUabiusxnDH0kuDzeGkuwyRm9wua4gxeow0iipYjC17c5a46gqXhzFKp+A+BjyxOgeMzmjUgn+Ci+0SihpKuknjkH4kQBa52o2XKUtyUtVlBfM/zK9GYKrh8ANsXkD5LKrYGSHmFXMPjqqdjOZGHxk2DHbJENWnmpqeKkia54GTqdyoa6uDYgDUyuB6FykxHPXVrKelZflx3I7d1m4mHSOp6Vnp0ZftdaW0Mhc5nlT5oiwDl2sLXutHFsMOD1dNDUSNMcjRsNlBXsoW0RlgkJe8gAHcKTCs8v5cvcdLjdSuffzHqoTK8hoJGVnQhSRuhc279+isCcSxmncSAwg6NB1KpzNlqS2Rz22tYNHROJAWZy4XUZlsoIZIHMGYhViLlWp5y6LQ9VVLloLYJuv1ThjnCyRGQ6lAccmV19kTpHEp6Snmr6qOkgbd8hsFujgnFB6q+kH1KmezHXF5TRTCZLJ30Rukc5hC3jwHXjzPxalYO5BRN4ErPhxemc0/EGmy4fi9P8zXh5LPCznvcxm7WbfULpeJKjwfDdXI02cWhg+pWZgODtwYZHVcdQ4G5yi1lexuD74oRQ8wQtvcuOt1Pxmj+ZfCyecyXdE2Yem1gk+oYaZsDNmFdOzg7l+nFWj5RBA/glkhzHFR9GAJ+M0/zNeBk5N2q14sU59EKKSGNrYxZhA1uqFVTeHqpqVpz8t+S9rXVjB3cisAkpmPEjSAXD0nuF6XBtcK8OVeNPkmoqgxTQyMjaQ3MPPfUj6LS4swCv4Xkp6OvfR1jHl74zF5bv09QOy6f7PuKOH+H+GawRwZsUjkuYg38Sck2Zb2v06Ll8bwbG4H1tdjlOJKmojNTzuZfKL7D9UFHCMRw8eHgxWFkDKR2duWLMHE9+yWI8iu4frMTDI5akYjbN1bEL5AB01QmspYaIUlThzZJZ2h4f8AmHuEPgsKiwcQ8/kV9RU2IOwi7lGUE8U5pI5pqeqjmrW5InkWjcewWfAwxyvopBaRlwR7rqsUdjk2CUNAI6XEIqM+IimpTcxj47rl5LTVb60mxmN/qjTSwGMUmKSOdC5z8v4dtrpqaorxXSckxte8vBc7srfCWEz4tWSQMxKKDoQ4fiSDsNVpcR4XRUVTh9LC8PgpxaR2Wxv+q5yOUiaJq0PqhPLkd/UaE/JdlXYRg1JHSYoyFzZpW8xrXOzAnsf81Rwikhgr5KljCPDmzWuFiB1XV4tTDFsNpaympbeGqshD9QAVKXBysA4YgiqJ+IKKaQ81kZp45CGk63Nv07qtHwxRV/CmI8RUk/KhwyYxRuEd5J7uFr36AEL0zDOCsPFfRvxChZM2iiewvMYEchPo062WkzgrBKXA8TwSlEnKxDPJI1xuA/pZa1kvIuGeEJsYwR2PRy5ZqeujjiidHpISRtr7/wAF0fF3GlfQYnVYdh1PlpoJAyqErdJT1HyXXfZKY48GqsKngu+iqr2cPTcaW/Rb+J8NYTiVLXUs1O1vjyDK8DUkG4W0Z2M0I4i4fpaSMtEFYYnzDpyuo+WixazgHC8Qx6evqo+ZBIwPjDHWAIFgF18WHMo4WU9LpFG0Ma3oPkpIoTrn0WxDmYxojYLNAAA+ioYpQU9fBkmp45nM1ja8XAPey1paf4gqxaQdkYeO4nhNLg3EjauvtHTGOWWLMdJSNBp7kkrW+zeKpxng3GsHYLMnqrl/5QRr/cvQsRwbDcRZFPXU7ZRTNfYO1AFuoXH/AGW0VRFSYzhMjJKKWrDJYS7Q5CCLhbHNcV46+DC8M4SwekHIlDJJoZNTIc/k1991LNwDxFg1LSz0sT3V8VQZ3xxNBEJGoN+uyj4sdBh3HdPTso+Z4aSkYSW6uDNy0e917PVxfiB/5xdQc7UmbjThKhjxKEQsqXxyVrSbAsYbv2ta+T+K4PhJmAUvEcFRRMbS0ceIyGKeZ1g5gZtcr0zF6CoqMFq6HD4mOlmiMcbC7KDfcX6LyPF6Ss4YrmSySUnNpJ+YaQHmRxE+/VZmG7e19MzdWu2PcJm7LG4V4nZxThLKx8YiqY/JOwdD3+q1uabqUytsJAWiyxYPks2F4LVOarJHlapMNRJStjBLg83VcuJsh54N7pw9h+JIHDcVfi8Wn9yCMfxXaOl2+Q/uC4rGZObxhVt/1bY2/wBy7F/VaEjZfdWY57BZmYgqRs1kGrlZKyz2Ag9CqVbSatkgaBbTLskyqs2105qC/wCJaiUk5a0sVd8b/hClz2CyONuJf5r8PPrY7Gac8qHuCeqSzyUMS40wnDZa2nz82WkaCGgj8Q9QPkudpvtNfV1GWthdSwndzfMV554p5GgsDv7lA6d7T5VwbavEeItxiuNXBG+9zdznXJB2/QLHfLVN0yE/IIziDwNSmFa++psFlYlHBFJK/VtwQrIpGD0ybG26gmrLt0NvkEDC8x6+notUqwWRAgvINugKkZNE5oZsbbqk5+b0E/QI6fNGXufbziw0UoWtiGEE6boJG5oyzW49KLnx/E5I1LRobAHt1WAAFTG7bSyLmFou+2vSyJs8bvT/AMUbZoz/AGu6YiFs+trEIr3ddrypS6HN0ue6r1QfGGyw2BvqLqiw0vve5utzBqekqHN5sjLxefzaBx7Fc4KmTTMrtBVVLJOVBMIud5CTtZbiR3jKtlNSHiF7c0NK58cIaba/uBcxX+M4orGSvfZ0rX8tvWNg/wA9V3NDgj4sPijqKqKYmPRwN2ZNdB/mqhbg2BxA1D4BzyDmdGAY44zYrUMuOZh74KiKlZ+BWN1ynUt9/oF0MWLUVFRzQSYjG59LCBGTLrIO1rLBx6vfBX/e1Fh3KZWNeyDMLAMFgD8+/wA1lxx4aaMvljnE35mjr3VlYlq4XicFVxBBNW1UtPTvdo+5c9rP00uvVqX7orqaqpMKfBIyNo5oi6E7X9145wtQ02I1E8D2TyTcv8GOL1yG+oC6nBcY/mGcVo+IA6DE+QzwjAMwIsSNfmf71cEd7AZo2tjfHlDAABe6mLnLA4JxfEuJqapxvEqdsUZIjgAG4G5/VdHyxZdmEYkN0Ymt8SCSF97sCiyS/lXOTksGa/VITtHVVjzBuoecTK6G/mAvbqotr0lQ3TzbmyDxDe5WRjEuWgdMeblY5lyPcqSlqxV04qYx5Hkhp+SFr/Pd0Ukc1/iVAON+v6KxFmstIlfujZTzOOo07oNzqrcVUxrMltFu/Kma87QyQvi3t9EzJHD0rQpnwF2d+tgr8mHUBjE8cLSXi5KxLcQw+f8AvIzNHl1cjr6SGFwdGOiz3i65y2ldPYlt0BeD1CrvDw9MHKwiV2uqhcLp8wUkcYKCMRpyx9rK62n0zNQmF3VUZ5hKQY9vRWxDYouVbogqxNfl2Klhi54OVpB6DqVo0RbE7N0WizIRmDAsDANDU9IiqsgLXFh3C6iQ6W0VKemjcCWxszW3V5Dn3OsoXOV2ppzH8KpyM1W0RZndkxkKTvSoyepVZHzCh5pUaWgCok5x/MnE7h6XKHKUm7oLfin903iJPzKDN7JZglNLBmNt0jMSq4d7Im7pQl5jiU4zOQN3UjFGTgPupY2vKTBqpmhATAQp2l1vKkxudTwwsJCzMtohzlK0T9Wq8KaPorDaQBly4LAz2Nf1RZSr3IYOiB9GHG4QQUxtKFk4lwRg1fWvrWVMtK6TV7YwLE99dltupS0+VPynIMODgXBo7tdVVU1+77WVeThXBouJqag5MrqaeAvLHPv5+i6Eu5ZWPilQ+PijCng2zxEK0rQbwlwwNPueA/O5/wAVw+FhlDPPSGMSx09XYNdtZegtqA11yVwExyYziY7zF4XSnN3Qmw+K/Lw6laP/AAh/khfXW/YxRxjs0WVJjnPhZI7q0JLrGEOc5ysCtlz3JuOyF05JzB5/VQpre63whz8SU8c5BUj57qqCjB1WZ1w14ixBznv8jC+29loNheW5lSpR5c17fVTFz2/EuWUcXXGU97IxM0dVnPqHXtmQioy/EscZbtpvqOyiMpKpsqmjRxU0c8JObMs8S1qF8jhYX+qlc1wGqg8bGOqF1cw/GlLYpJXNKgklc46oX1UZOYuUL6qNpu0q0CcCdlE+4Qmqjt61BLUh/pelCYTEFEZhI2wcqPMufUsPiPip/DzmwR0vNfILtN9v4e6UOirJoKKlkq6iTKyMXJXkWJ1kuI1ks8k75BmIaHOuAFNiXE+LYyCyuqiWH4BoAqcMTpAWxgFZEjcOh8NzvFgSgekqHlyDbJ+qeTmRuLJRYjdR50BSxVLRnzt+ihNRMzc3+akmkMTC7U+xUdJDWYk8soqUyuG4HRYiQD6iY+3yWxhDOHJqcnEc5n/I7qsqswzGMPaH1dI+Np2Kri51Vsd9h2BYHi7L09DJTvBsLiwUc2HMo6o0jeVHINi/QLlKHiPGMM0pKvyflcLhac3GcuJiIVeGwOljBAff/BIB43BNhlVH4qSN8UgvdhuiOE4FikPi31zaIDcHVYtbLJUS82TQdkvC1MtskeZnzVEs0k2Fv8PQ4iJouhabhA/GK90DqR3LljPwuCFrPDScmSCNgO3W6CphZlc9hIv2KQIfDTcrmW0SgkyDVW6LEX08fLe0OHUFS1Hg6sCaGARyHe2y1AtUcmaEK7BWTUU0dRCdQbLMoifQrc7HckrV2Oxkhq5SJXx+R4vdYeM8LVuJykwVOW2+d9gtDhTHnYjCMOnjLXwCwv1C36t8RhJMYa7oQg5jCMLquTJhVXVxOu34TcFc/jvCk1DI9lLGCGfC03K3nVToKznj1hVOJcWdMYapkMsb72zA6LUwMCm4XxKSEF9JIT3sofuCcPcJw6No6gL1DD8ep4cBbWvPMy/DsuExzjMYlIRS0gib2JusTA5ero2wyDlyZ2d0VK99NJnZYnsVNNPziS9oAPZU3ubfRShLM2SR+d0RaT3T8qQjRgt/aUf3jVZeXzjl7J4a2FjMj6USG++aytLZjzIT6bKJ8nMPm1V+PJiLvD0tKeYdgXbqrW4fV4e8Mq4C0na2qygOYwNs1ov3SD7i11A7U+VMAVsSObc3QCRwI7JwehKY2Dtlga0WJM8KY5KeKTSwJGoWZZ9vKR8k7MoRssTbqtjU4eZDPMIJ4I3NPsumOF0NA9sjKSMOJsuawISRVgdk8vfsugmmfPUgPmzWNxotDom/dRHJloeWT+7dchjuDMw3EWTU8bjHNsAF27HQVVC2MvAdawuuOqaauirGsq5xKzONb7LMwwx6h01w8wkfNQPq4y3JJGB8l22O0UEmEvlbE242IXFQ1/LjyGmjd/Bccmok8clGG+cSXUZndmPJjNul1BJM2R+rAPkgzG+hssqmaZBKCF0FOWmBtlzbX+cZySOy36M/0cBaiBKQQo5NQjdsgd6StUM2Yv5pVumecouoZhc5kVMdAtQsOTDmHc2RAtd5WK7S4Q+bD3Vxd6PhtqVWjpKuNviRCQ0brSOq4XhfR0/OmmlcL3bC3crZrqKOVgxKSEsYdmu/xXK4dj4p2sfVMuY72awLdnrMZxyFszIY6SmAsMx9SwM6r+73sc+eC3YNGgWW1kU2m7eoVyqM2d0FTVRlg3KvUboKNhBj5nva5QLDYqqJokpKSOJo3fLqFTrKirkrS77wBt+TQfRXxjckRLZKS0J6XsVQeKWeTmQx2HzWJA17M7myRxtAtbTqooYJKlmeNlwFLUODYiC5KKvnpKTl07GkHuucqzJjy3lj4zcKN8rD8H8VPUl8nmfuUhhkxHMLwAoiCO4dduyO7ybqN7JGaKRs7Y7MkFiqNGibmdkBcCeoOq7/AIQhkqogw00cDYzYuvqV55R1cUUgffZdVDxrDQYW57PDiotla0aLcDI4/r5qvGzhz5GOipfTl3v7rn4Y7vQxmStqH1T/AFyPcXKXPyT5WF7uwW7axS1bc0JTx1fjKKnwTJZhd5nXvf6KWMw5GtrnSNYfU1vVV8SiwiECTDaqQu/KeisSKNdQihndC4XI1B7hdVFxxU1GCx0lU3K6NojBbe9lyMjnyHM95PzR5ssIHdZtlblqszyQ+VvzeVoYVxDU4QXeFLJA8ah7b691z2ZykjcR0J+QWLG7NxFiclWa4VHLcNmsFgFtw8WVNYy9TjGIEflaLBcMZ9bFXm17YqJzGG2cWJG4Sxqx4zBWsqYYKRoYBYykeYqPBcRZhkE0jLOqHmwuNlk0tQcPpJA7+t1HyTRzstmPVaiSm7Hjs0LJOVSxgvN3SdQVkSS1dXVeJnmdKPzEpmyOzgPFm91K2SnvkJ3WVpFJzMmd7vKpPGmVrctwQoXOMzXR5gAzZQwuc5hcG7FVG5T1zqankkp2RsmeLXssHl1FW9rOcGOYbm/dIz1DHZMyIXHnO53Kg3IqmoLfCz5DLKMnPfuB2WLWQmnndA6QSZPiGxTz1Tpg3OfSLKvnaVu2klPSmQZy5rR2ch5TBqRohLkhICc2ZUKZ8EUZ3J6D3VKT0td1IV5kDZnEF4ZpfVRRwZ3hzmgtG2qClkJ+SawAWtU0hnabMAH5QqD6WaMXMLgFQdJSzVk0dPA275DYBXBQ1NC+sZUUrTyG6ufGbfqmwiWakrIKqCl574iSGHYre4l4ikxPD/BzUJpKkm8hJFnM6hGWbwnT0v3tSVYkDjlL5QNmhWK9/jBV1/LMjHTBkUhdq0KfgijDDVF0DQ1wuHFupGqvfdXC/N5hmjH7pkNl87qM4jLzerT6IpI48QxGCgq5JOTHSh2W+5HVVKPEzhcNQyN78j5SyLS9gtarpeGq14fUVURIFhaS2isxT4PFB4WCopeXYjLdeTxMaqpenjDN4Te2anqZs5c8y+YkqB3FdeHODMDL2g2BObX+C3MOgpaWmZBSRhsY7Dc9yUU9fSUcphqKuOF43DzYrlOePiTeFrcOWxPivE30b4ThDqTP/Waiy1+GsWrK+lkkrKcs5f8AWWsJFbnxHAqlojqq+nlDTcBztLqKtxnCW0U7mYjCfIQxoOpNl2njljxxwZm/W3I4vS8mrfLzGu5zi+w6KKat5radsceR0Hqda2YqmHPOpddSwtD3ZOrl9zD+Gnz8nsfB+F4PjHDo4wrqZhqGF7GtaLMhDNNAOvuuF4k4lk4qNJHG8xup2mNznuDbknf+ClwrjQYLw47ABSuE1pA1zb2JeVe4O4CoMYwWpxjFJnvfYsigY7KA8X1Ovy0UHVcPUNHW8NYdJSUVNJWWFPLI2MOc0DqTfb/NebYzF4niTEJ8n4bJiB7LewXDa3hCdlbDVU7a0DJJS8y4kB6lc++tfWuxCeSMC87yS3a5OgRmJbvBNfh2G4jXVGKTiKjdSvZb81+gC5F+r3xBhaGOOS46X0/grBmYYDHH6rcwO7KSeaOtpGiqZkkAtnabD6rA6DgvDJoWDHZ45Y5GPPg3OFoZu/m9l2H8zo8YrXVTJYnRxedsA1Oe1zcfNc5wr9pWKYLg8XCtRh1JVYfrynSgnKN9Le61aEYtQ4DXcWUniGvfEXxuBEkYG1jbW6CzHwtVwxTVNRh00bwCZHZ7j2Fh0XTYLBQ1eGNjbG6zwAXNdYaey5/A/tBiOFx4xiuHyFtPanqpo39zYEttqN13GG0mEOwFtRh1T4ykl88UrNTdbWCi8smRWDC9xuAs3EMUjwiMO+6cWq//AOWiLl57L9okGMcc4aKWKdtCwmLlzXje2TuR1tbZamKJX+NeIcR4K4wNXhdPGYaynjMjJBdkhHbsui/n7hzqKlqJmiCaoj5pp3Gz2hYP2oYU/wC6qjiCtiknmljip6MM1EbBq9+n0XC0lLiHEFMKitjyy09RBCbtsSHjqsJT3OkrY6uFs0LzqLkEWIVhjrnVV+VDSgRx7NAF+9hZEyVt91uGFrOoyxucuITCVvdF5XBGwvaxwIOx0IQeFphJHMI2cyJhZG4aEDspCwhAXAbqjiftE4OfjsgxamquRNRU92m5F7ahdhh0FYMIw5tXrMKSPmHreylMbJo7PFwU7p5/hddYCMT29F5D9oGDsqJ8TxKCnkdMMQ5ZynQMDB0XsQrH8rlyMH6qkaOhlLzJSROMhuSW31QcJ9n3DmIYTCMSqDLAayMnk20Iv5L6/P8AVdo4nqtOZjJgCdwqcsOui0ImTFo3Tmpv1T+Ckf6SFFJSyMNja6tBzOSdEmyuzDVVDHODZEwSBwuCdUphylY7mcYYk/8A70f4Ltpneay4QEycU4l//MW/iu1mf+KU4thzHunbsozI26Jh13RhMTZE15aqGJ4lBhNE+uqhIY4/UWi9vmuLp/tFq3YkZGwxuo5DpHYXjHa6lj0aN2chq83+1DiHCsZiosOw+RsxpJHyOlB0bsLK9hfH+HBkjMVY9r6kyPGukYOw+S83rZomGUQizbmw7C+ytis5wfJyWG10DrQyXMwcfnsqxJF3BwCQZNK20bL291inSl1sjXC55Z+aqTyR5rDc/oiAMYLHtNxoQEo5Gx7gEe6zEIjAeR5UhfoVZ5zJIHHl2ybPtseiOeSCUQz5JC6OLIc3Wy00ihm5ZuNeiJ8ziLi/0CrSOLnXumDnA2zAD32Sha5rQ4XN/mjjfHHcjLcncKuwxNLm6H36ICxodcKTAtStIdzWnRRskaQM2h6hCJczCx7kJZb4tEoWmTxteHlocR3VoSRyjIQNeyy75SrDJAWgNvf2UmBb5MN23bexSvFE8hwJB1Fyq7eZvnBHUXUt2yNF3WIOylDqqPiSkfgLMLllnilpoyY38zQlVMUx2pqfCTzwNyUukBeL5z3Pe2mi5/OwbLQpcaxnDWR4rQ8gtp3AEviDm3PSxWoSYPi+Oz4zM2SaoMrI7mMHoTb/ACVCkhnc9zBKX3u/VHXYlJitU6tnhgie/cRx5f4KvzXtF2OsVUaWBVww7EIcV5ebkG4Ga2vTULQ4kx2fjDiJlcaCQZI2RcuJ2YhgPf6rCb63xjVl/I7vZaWFSVFAxuKQTNNQXPp4qdjvO4nuP00Vsl7Lw7FHFw1SMoaWeGAMs1k4tINequtDgdUFHgE/DXD3javEvXGyWVlRN+zNrkC/93srkWF1tRSsq4XxSskALbO6Fd3HIzCMoUgyqSDAcRkcOa1kbfZ1yrRwWSEeQuf81iWsWRUwvmgkYxuuUkW3uuUpMXgkxWjnqIzFI9sjDf4hbQruTBI15YIZM3QWXkeIY1HFUwTUsH40M0gMMu7TnuAVluYbXG1ZVNq4MKpwY4eUamZzh091d4GrYq/AmQs0fTuLJB7nW/11XP8A2iYlUtxSJrGtD5KERytOobnOtls/Z5JHPh9VDT0vJjicxgvuTa5uiU6YR9kiLKbwxYLl/wCijkife+XRVUReAhzuTmM9kKMnzuVuLFKmKA08dspN7ncKmpG5EgSzVUjmkv3VV8zwFYLGOFkYow8KTDUKPNLt0JcdldFFHm87D+qJ9EwML2AadFLKZ2Z3ZSRzWKTomE7a9lHyDrqArYvx1oAy3R+Jjd1WSQ5nVOJiOqK0g4XViLI+wcsts3urMU2U3usC/kDTlCmbOGtAus98/YoDUEdVaGgZxfdIzM7rM8Sb73T8/wB0oWp445VSloLtPmCIVXujE9/iQUTQPJIyqtPSvh9YsFsXF7J544qmlmhmbdpif9NFbHOuZbdDl7rR4Vroca4fbPOGvkicYnWFtQf8rK2+hpH/ANWrEpMMS2myVltfdUD9iWfJJ3DjSwvgq7u7Ob/xWrZ4sVPlKuz4TWweZ0eYfu6qt4Wo/wBRJ+ittI8pT5QjEMzfVC79E/Lf/q3fooya1lK3RBkkv+yd+ikZBVym0dLK8+wWVGx4HqRhyFlHVH1st7FH4SUa2WkTRyaq1FIOpVRtNPf0KZsE43Z/FZGlFPp61bimaRZzrrJiin3LbfVWYJmscA469VG2pl6pFzQq/jIw3LcIDVM6KcZOWKy+VllCZxdVXVV9kPNzbrXGWecJJW53XXPcRxPhxLCJn6DmWJW+JY97rC4ud5KCbpHOLrpjDMtmSN+fIFyGNRmDiSoY9tubEH/wXXyTWlJC5biw/wDxmkm6yREH9V2c27QB0mHwSDblhGQq2DvJwxjC70HIrNwrDMllCbL7pa+6G8nRi1bnR2j3RZgq8pxKNwMUcOS9iHKdgqXmzo2D6qS2swlxb6kZLA31aqKJj2jUhFkPxLhP5pbj0Qv3TKd0QITx00eXOblauKTzQZIn7i5Q8lgN9VZMWUaAD6oeQ8jMNVzluJQOJHVRGV/5lZmgkDdQqj4yo2B0mvlKbO5DkkvsnLSBqpQHO5LMEJchc4LTKRrrHdcbxfEyux+CnLo3tIsG5rEf82XWZl5XjxrabiCon8QM7HWaQL2CFt/HeF6TCqI1kE8jyCBZwXNNnm5jeWCO91pQcXVjW5a9kdQ4elxFiAiqeI4a+ndDNRMDj8bdwFiYdYlQqZ2ysPMkOb3VSOeaGTOw7KPyg6yE/NNJIwNWZIWHVpf5pLX9gtvDuKKGnhNOyhc3OLFzTrfuuV5rUmyNvolK6DGMdmrstF4qY07Lau3KzJWhrM0cgP0VIyFx1cijvI8Nc8Nb1JKkwhF35lLDJy3By1n8OUsVP4qSqEze4kss2CikqahkdJHfKb2urQsPlzC+Uj5qzHNagFo35hs62iHEI6iJpZUU/Jd2Sw3EZoKSekyAh4uCei4irZrmh8k2dyYh/wAEhCpme7yRpdTMkJcLFBMKmmEeR7Iw4dbKuKjK67L6DuoZG5Hm+908Zj/rBcLcQNvDZWPa2bqVqNPMjcOllz8FTBI1sbCYyDtfdbtIb2DtiFuBmPmmoqvnwTOieNiFu0XGtW8cmtEU9/iLbH9VE6gpav8ADkY4E9dlhVkEFJUmFkl7dbLQ6OOrhrJi97MgOwvdB4dtXBPRe+dqgw2nnlpTJS/iW9QG6OkdUeID4YzYaOJ0sljW4dwiOooXwVM0mZgIcy+gWLVcAVstQ59DUxiPoHFdbw+GRSVN3ftBdbtLS07aa5+iMvMP5j4lFKIah8JB7OWHiOEVdHUugbSyWbtovX4qDxcp5kTh2usmfhplRUcvEZJImn9nlPqRYl5TPRVVM3PLFa6rjN8W69K4lwWtoGfh4Tmpx/WM1C4SdkJlOhj+YRbR0lQ+mlErN2rSreJZ62m8LUCM33Nt1Xjwaqmg58Fnt9lnZWHdqkgSLIhFIWZwLhGYwQrdEyhkp3Mk5rJB2Koz3ggXIISzFTcrN65DbsQgfG1h8qBoiLOud9k4Fza9j3QlpAzJ4o8z/Pt80HQYM6Bz+TLIWE7ELZ+7WPna9mJRj2cuboqB9VKzJVRxX/N1WnV4dJRsa/mMlubeU7KxLDcqC+nDRHVNdft0WDilRNHMDzASe3RTNp5tunXVWcSwFjaB1XG9zyzoQtAa3GmS4TyIZLO+Id1zAYrLZIzBZzvN2TwRU8ptPce4K5ZRyXFSLWCQh97fJWxS4Vy+Y+aRrlDWxQRSfgSFw91WdqsNNGlosNqmksqiLdCVpUwjFoITchc9RioNUwQMzHsupoKaZr2meMRu7BaAvieDlIQuikPRa/3eHecPKLwdlBz00Lw06KowmOTVdFUUlydFj18GQZmjZagZv37JDpDSRRs6AbD6qvh2H4zxFLNBhskDQw3LA6wUEeJSeH8E2kaS/wCIuWxwvDUcNcRU8leY2MnjJzF2ll0GPXYZNRusynl50DrSHcX9l1cjMUxbCKedzMjWDM5rT0WlitRhEmJTVEOIU5ZINWt2J7qlJiNKyldDFWx5W+6zODlyczWmndI2Njrdyuswqvo6WiZAY2yW6u3K4qrmgfMeSdlJSYlPSEloD791z5OqzjDnmsdyzZvZQ03MAu/ZCajxMpmmOUfrZXfHUbqXw8cLnfvLMzYgklYQ3yh5v1QzFxOYsEfsoXvbccsWsUM87YyJJHn2WGinkAbq5TQTCSMtL1kV1UXOZluGg6Jc8OOdnTp0Sk8miQGuNzfsUzmx2zOARuyAAixaRpqhJjAOdlwsiAwvkeOXax6kqMiTNY2PySdKHHIz0dAjpSHShjvLfqukIswRZWKdkUELTUv1cP4oqmGRtMRG4ByzX5oW2eSVWk9XiAmH7HLbss578zy9qU0t2ktQeiEdyrAkaDbzbKaXlPdHy3+Vg1FlAxznRge6lbDMOZnjIFuqpMBec8zrDydCpqGcRVPLky5Hix0QwRv2kaQO5CmdHTMfm391iZS1CqZG17zHJqDoCmg5jWiQA3apqtsDyHQ6gbqs5xBs02CqJHVR1JYHOPxFC173AjKSD1CQi8uZ7gEQDA7IzdaaXZckmW0mw2KYNYfU/wDRBHbMA9pulN5HkWWaZWIhGXWzkDuFK9sFE4ZGg279Vm5njYpy/MAH3JShafEJHF7rRk9UBa9kXnOqkkJMQ0BHuozLcefUJTpCF2Xom2vsmdLdxygBRlxcS223Vapkm3O2pUrYjfZHFFZoky7qYZLdA73QQgMfJ+Je1rbJZywWCnbFGWFzzY2Nteqqlj3W2BAVE8c8zRmvr0UdRO8SuyO/DNkEQqJC4PJAZ3TSB+S/uoAEr2uzseWnpY2Ukrp6t5fNI6RxFtVE0KzzRcGOPKALb3QbFDxHVUFCKTwDHlkfLD89rD5KhPWPrKVlKYImiM3Dg3X9VFzNLPPzChsTss+QrugZFUskkGZgcLhdDOMPqIYpKLWV5tlGgssaN3MfkK1KIc2ri120WqXk7rB4I/CMZk1C4XFzQV+L4g6rlexzCS0jqRsu9o5fDUks+X9nGTdeeYjVwVbGVOnMN8xDbX1SoY5ZKQpI5oGzMZbW2UHQKCUeHY5wpn5XksBcFPC+Plhofr29lLjeKR1j44KLMKeJttRueqnBq5ZLXBWIInzyiGP1HZQQwTVMwhhFyVv00tJhkQpIOWZf6yd7blp9luY4ss2TD5IXkT2z9bG6UMk7crnV0rGg3MLXWKkdUGaYPdu42JTGmfFUipb5mrmsSKSoFXWGQ53u6lzySp8ObQOc90tSYnxyXDOjllQSkOleOrlNTueySR8cYe94sL9FqJRLVVU8lVLNI5tiLeVugHspaOZkjg+o/ZkEKGRl6WOF/qBuVpYdDA9sFFJFe7S86qSNnGS2Dh7CRDSSxtomDmSuFg4vvoF3H2U0T2U9bQ107pHSiOcUoIMZj/MNNPfus+bhSg4o4cosUZVz0cTLc8uOYZGb2GlztZc5xVxnHFiNF/M6Oswz7vp/Dc6QWdKz5dtFMIHqPF2FMxfBzhvh2UrJryukbGARy9bC3dcUKTjGP7OW4Vg1DK6OWq58c1LMCRHqSw9WaqxgvHlRiEUeHUFRS1L3nIDiEuWQvOp2XWYB984fjM1PX09LHRVo5kLIn8yMPA1YDbc/4LpxWMnkWA8WcY4JT1vD9JJVCsrJr5ZNSCRra+x2Wxw/wfhWLcHYhib4qinxjD5X5nynLY7jT5f3rpftC4Xoqakq8dMks+KVlXGykc0ZTGSdALb7ey6Xhitw3iLh5z62lEVTJ+DiENrHmjQ3UmC+TSwid+O8IUcFW0h09OwShw1uLIRwjRGStmeAfG1cdVIBteMWACuUjaWjgZTwF2SNoY3MbmwVxtRG4epSkVpaKGU+a4PzQDDohtIQrtwdksreqtikcOl/qpASpGYTiFrtdEPmVabKGqTxjhs5WxRloMSjbd0Ac3uHKsYJ3nzxkFapr7syl2qqSVFz6lLFbwkgGk1vmo3QVTNQWuHsVK+cqM1Lxs5bEVpuqAmQHUKYSgnVEW5xmDkEccjycxVkMY5ucKuLBRmSzvVZOI0mu5bRZMIYZT5gqAqXn404qZG7FBcdRQZtNQl93sLm5LDVVG1cwO6tU1beZoe5B5pTtz8U4l//ADI/vXf+GpZJy2TRcBhcrZsdrXt61n/+67maS0yC6yjp4neSMH5qYUNE8XfTNJVOOpfcElTirJCgCu4YoMWpXUokfC64IOfS+u68bx6CowDDZsKqsKfDUSTkmocMocAdCw/Je1w1Lw8OXm32yxc2poZsuXPDYO6Eg7fxKyw4IfdMMln1c0uQX1OhKypqibEKl0kdNHE0/C3YLSxekocPjjignM9RNEyQuItyifgWVC/kiyzEtjdDThmSRgzdUEL8vkFwPZSSS81tr3ICqCbLaPILBBefFBN5h6rbhRmjjI8ziT3VfxGQXapS54GrtwgvUrDLFLSRj9rawuAwEXuSstr5A50cws+M2cOxVqKQCSOQn9m4PSrahkstQWAZJwL2FtVRQ5hafKna6z87xfVC4OvomzBaUWgcT0KROiEAuIvsk71IFe9vmpCXlgHuha2yt0xjLmgszOPRRVfI61ykwuGzrK1LEwNc9jDlv+iq9UsWYpdLOKkkOQteNiFXjifIPIRdWXRTsjLJB8rFQFDllPmWphFTPSc57KF1VTCzJ2jprusKnkfG/wAwd9VrU+JVUDHsjfZsos4dCEgXcZm4RlwihfhcMkGIvcXy2cXHbrdYAkufxnLRggopBLNI4NlB/CueioVdLJT1QjmYW8xge24sch2VZSwt5kgbAC51rWAWhLSVeC1lOaqFgmYGTNAdcD5qbB8ap8LopMPNFHMZ3F5kedQdLfpZRYtUT4jUOxGZ7SCAM2xPyCK1+IOKKjG6ekimJdDTx+aM7OfrqvZvsrlYeFKWrpmZGyXBY5hsCNNPnoV4zwYzhtorZ8adBJKyEmnhm0BeAToep02XbUX2l1o4Uo8H4awd9VXCi87qdlzSm/XTU6KpT16V5vnGijfUcpuYmy874a4k4q4xNHWSh1BhtEDHOSfxKuUAaHawGn6+y7AE/nP6rNkQuVlbHFSyTTGzWC5I3svJpsFoaziJ+KUM4GeUlokOjiBv/BelVT4zSz8/0Mie8/ILyilxWinpq2swjmTvjnlZSAE3IyX0HstRI5biTEcUr8RFJV1HP8IeXzA2zCA/e67L7N4Y6SvxFxkJdJCyQF22pOv16Lj8TL24LSU0juXOWmd3W57fxWtw/jwinp6GrnipYKZwklcB55bDQEqWPUjn+NPp7KpS1j62hiqHRmMyguAI6dP4IvN2W4EhIuo+UCbpebsiBsgjlpT6orW7FQyNfDbPbVWg56jqwZowG7jZaYRiWyssqSG5WqkYpAU7C9oWWoXjK5xVqFpy+ZZTZZGu8pViOrkadSsNLE2GQSxOey7ZLX06rLfTzDTKt2CqiyBxKExxSycy90Jc3NT1X5T9FWfHUM8xjd+i6iYxA6C3yUJe3oraOc5s7N4ZP0ReKkO7X/ottsgY7MmkMROZzB+itjHZVk90/iX9ir72RHTICh5FOR5mBUUue4ohI8jqrHh4wdAnELB0QQBxRiQhXYqHnG0YuVZZgj39Wj56LAymyuUvjGRU87n9IXn+CvnAD8Ug+iirOHQzDK2qzm0dPKSN/gKDgfsvxRposTpydBMJR8jp/guwFbGDuuF+xaOiq8arsJq816mAPjINh5Dr/eF7TTYBhVOzJ4USe7zcqQObgrYnW1V6ORzhotaXAsJLs4pbewOijdQwReWNlgtxkKTGyE3vb5Kw4tdYvAUwp2ppWsY3JbXupyClgbG0FsbSCOoUAbFIchAYewCkbK8+Q3+iglOV9/iUsWKelhEoJYD81pMawaFjR8lheNyH1qeLFB6S5BdqMJhqZDIJLE9LKlU4O6nbzHM8vdXG1jC0HOjbUMd5Xvu3sljDcWH0oVpYhTx1VhHaMt1u0LJfeJ5Yd11jzc5/KsxvDbXRTRwVAzACM9wqbXPecouSrdOIiz8SQhzSpMURKBtHPmPLbnAF9Cog7stQUTjFzmTD5KWCQskBEcYPUgbrcZ0k4scH6fNPm9wuinpqKog5ksQz9DZUTh9KP6ofor4qeGyr6+oLJ4rGbCQ/8kzCur+76b8h/wB5ZXF+HwR8NVcjAczCwi/zTxMTgjbYwwv7xgrB4wYI34fVO9LHFh+tl1mFtpZ8Jo5+SZM8Q1HyXP8A2gPg+5YGCSPPHMDlvcldPExOA8BLnU8zOzloFjABnkY3+06y5duNVWEm9LBHIJ4wTmO2gWXLUz4tW8/Emtk7NboAFyndji34M5z5OynxTCqJwZV18UZPY3/uVSbi7DYSXU7JJwPitlH6rkRQsje9zGi7zexF0Qm5ZyyRuZ/Z2XLPq/g9GvpPi0a3iPFq6YPppOQwG4DevzViLijGYWAVFNDUf7OVZbJ6XaOS5HQixUjZy70vXn/E5u34XW3ouNKUODKvC6qPuWkOC0KbibAapt2Vhi/8ZhaP1XKZ5h1SsZP2j0/EZH4bX8HYnHcFt5cShfb8jrrExTFaeRr5qDGJs/wxAWCxHUUEh00+SA0EAP7ST6OW/Hj3sfhcV/DqODFY5TjONyQy3GQOPTusuWSvwXECyixF8zIyC1zXXB+imkgfKwxve14+C7dR9VVbRzwnyQEj2N1PGxPwzdqeNcYjZHM6mpC0i5beymwHjD74xAUNXSCF0n7MtP6/4LCLqY6VVE1jjs4RlKKGmMrTS1bI5BsWnUK+Kx+HehyQ8pxaoZKZ8voC5eHi7G6J+Seupasf94zX9VqUn2gDIfF4fCbfFTu1/Qrr4tuWWult2H1Z/q1H91V/+q/ijj464fleGvkmhd+/Hp+q04McwiqsKfEqdxPTmC6c2eLJ+68S1cyKxAve68dxJ1TJiVU+oZleZNRdfQpvfQLzXjX7PjTtqcewuZ77nPLBk2+RV5Mzi87MT3bNSb5VPTxTyWdM/kRv9LnDcKevwnw4ZJDVxzNdbW1lm24Z8m2hF1JFJAGfiM5juyiAA9e6fOxuw1UWD54Q/P4cfVQOAFz7ozKT6lE9wVVI0NezSwITZb6GyhcXDytKYF9rEpKLDGvkHLzE+xKlDqvDqhhaTG7caqs2S2x1UscdbXvAZG5xj69lpWtiVdLWBks3wCwVKKeMA67iyaomY1vIfJdw9lBG0FedEBDg/ZW4yMrTayeWGMtEjWS36/hlItaGNsT+irQpmxyESGZgB3CryNjBtA/MFMYJ5G2hBKjEEkbtRZw31XTWyKOnfYPe1dHhzvwGrClMzQByXj3totrDoZ44AZhbstjVppJhKBEA5x2BQTYHVS1JGK4ZmZ0LTayKldy6iJ/7y6eQZhcb9Ct0wzaLC6XDc76UOaHixaTdSGCejYahjGOin8hBCkqHPZCTK2w6notnDYRWUIZHqQpTVuNjxV9PiBhccoOy7HCmSy04fJe3RYHFmDww0YrbRxyxEAi1rhdHw3VeJwKKa91lleYLFVMVIZC2oy3czZXNQVRr5PEubRRi5cUXE8Er8Sia6q1Z1Z0KGs4ewOshMcmHRexG4WjSQQwU7WW83VKTKjTlMUwJuH0jWUgEkWt25dR8iuJraLBCxz3wTUtQ7YuGi9e0vmVTEcIwrFWFtZRRuv12Kw528UdRT5DIySORg3N7KCB745crPU/Yd11fE2BQ4NPyxMOVJawA9K5psbIajPG/NkOjrWW3RZhwqqq3B0l4r9S3RPPhnhCWmdknyVrEsY8RTNbFLZ9rFtlkuqJrboBlpXxi4GnZQgWKTqiXugbJfXcrAvwvL2hit0sTBUsbUyyRxHc7rPpqljJWtet+kqMLlnYyqYMnW+q1EpMNZrsGiaGRSyyO6LpqSjMtByXsu14ssE09G+xw6ljkPQ2XT4e93h2sfG5pb3V5M8XnsuAVTsQdTc5sTelxdRDApIp3QS1DGuHULp8dZ4fEmyf86qBsd8Yjf+cXWoGRBgtIJWtqqyzugcFRxnBBSNE8MzZWv7Cy9BxLDsNnp/6UOV++wary7GC6OpdBT1E08Q76rPFtNhE78OmdK+gEwIt5hsr1Tj73VTTkEduyzqPEXiAMkk1HSyqySCSQvWJHotKM1Mx/cXRlqpcP1IqcPblkzWWhy1BUnb5SAFhYrF5HXXUGEn0i6wsfp5o4nSZN0geexP5VRGCwuDDcgDdbNbjNLi1fTRy0OWNlmXtr+iyxJS1Mkk0maG5uA06LNdPlmL43uBG1l0HZzcScP4U7w9JgudvX/krCxfEqLEncykohTu1vbqseSofKcz9UcTyhxamDMge0sqWZnHqpq6imGeSCnkyDbRZUU8sDwYyilxGumdmkmIb2CwE2WwzXGqsx1TIxZ77FVo5WC7+WDZV+bJKXF4AaOgUoXX1IPoKqy55HXe8Htboog/oApY4ZJDoNE4tJAfEesAZNAo5DHD6GEv6p5QWjlt0PVREcohx6qiyx75Mrn20PRaE8plgzCMjS2iz6eLO65ukKyZsjoOYQLqUHjBGhBCsQw5pW5iQAVC3MXi5JV+BjTqoNIEPZlWVjL2AhjG2tutJvkic93pYLlYj2mrzyZrarOIqhrnuDRuSgn9YZ0ZorEMb45c8moYL6Kq7zOJ7ldGVmhuZyOwzLTjn/AKKXmMnXoFm0ruS9z2i7iLBPVYjPTyGOE5e6TC2suxenf5J43A+zE/iMP/1hVBzo5YjPu/qVHHIwOD3LnSLZZROkGsjWA3uOqrS5HSl0d8t9M26PmGXSwQOGQ2WwZdzPV0VqihpDeWZ5DiNAOirRzRtaRlu8jqrVK1giBtqshGnzTh3Oyg9Smlby35CQ/wBwgMhvYkadkzjGxoeCSfdGjuDQhmto2PXTVDLO8gbBDG8vcrEJCxRu0LHuNradUExyjIms+PzC490AIkdqBfrZWgG4QZiHEBG4kb/RMxgL7lVRCeSOPI2ycTPmIe/cJOiDzlabJCinvmjdp8lkWiDI0e6jlZleO4CmLaiOEAMJPcttZJ8UZGpOZ26ohMryQLi3umLmXy5kDRCZMrhKSD20UvKaJvxG5WWuPdAhARHnLVe8FGaYcnzO7KnWSPZBExjt3K1M+BkERebkO1CChzbOcB6tikXOYzTdHLFIX52xCzzpZJwsSyZha4dFKAwi4Lhutrh+J0tXZ5vk12WPB6Suh4WiznndTclGXVVMkMWCVDppOWwxkF1r20Xl9M9kTzDVQyZyb39l6HxHLyMDyW/aSsH0XE47NBJXRPgd6IwCgpVkEkThJDGeU/8AZkdQqJc65uuifiVO7DDStLWm1g0DULn3i7dfV1W4lYlZw2eSCe8bQSR1WtTujZVRvq5HtNtc2tliNvHYtdurI0F7klSZSV+nipXVrRe0bybHslVsDHzGlm5rIxfa2vZUuc2PdXXRPp2WmBDJxplF7nssyQzZHPmLWPZHGBrYFKmDxJduljuoMzQ7L0VunLBqdQ4bKwQlmeyWQ5Fu8NQwy1JfWE8qCG5I07rBhiuVqwfhwOY70vFig9B4d4k8VRNoaKj/AKPRkiKBxsHyXvncbegdl5zjOI1uIYhiTaqYTVNRNYinaDHYdB2C06PERRUsoaJRJI3IMu4PzVvhuioqGsEdRPHnliN3BnT3KsZDluGaXDavH6Kmxd8jKOZ/Lkcw2IJ21+a9mosMdw/SGl8VJPDST+JZJI65A7LyqWipaCup6ylgnloG1UcjpXixcb6he3T0sILXW9YuuseZKnNxNR1bIicKqKs8wPhAj2I2f7K9DV0pkeKaERvJvMI26CTrc9SuYxLBZ6GLE6vCJC+aogIjiPw/I3XMUfF3EHB1FFg0+F5Hx3JEouD7q0w9U58jfiUGL47DgWFPxSqk0ZoxvVx7LG4cxvFcSgeMboY6SoZbl2GUS37Kh9pWL0MXDz8IbODWPkZZg3aARf8AgsS1DquHuLafHqBtVTD8QftIr3IWnJiwI0jcPmvDMExN/DmOUlaTzAxo5rWb5CvXvvjDpMOdi7J2yUbIzI5w7LNqunGoI5o4J6qKOWUXawu1IR/eUZ3eT9F4Xj2JyYxxJNitK+Tlcy8L37gfJehcBYxJj2HVjcWbHJLSvYwOA0sRt/BaHZtr4ztKEnVkNt1m+HovhaR8imNPTdAT8ytUwsPqY/zhQPqtfWFVfTsHpef1QGDtM5aoXBV/vBTMrLNtmWXyX9Jf1RCGbrKClDQNVr6kxmDlk1EktM5r5H8uFty51r/op6SrZVQieMSBpNhmFrpQtVFX4aIycmSW3RguU2HVtXPEHVdIIiSdQ6+nZVa8zOp3GB5D2C4sd1So8Xw+Kc0HPvUkAvFt39lymB0mYJ4jZ4PZcFBiFVHVeLw8vEscr43smkuBrtZdNHi5p6d8ldIzmMaeYWDRG3J8N61sz+9Z/jddvM8vfdcHw3NlgfWWvaYmy7N84exr8trja4K1DCYTPHVEKtzTmJVEz+6HnBaG23EICNXrlePMXNFRw1xoY6uFv4ZDxcNJKuCsgNV4K/4obzCPZZuOU1FjjZMLmruTyPM5txkkH+Nv8U4jyrF6jxldNiXLETJTfK3YfJZ8khXQzw0lVS8uNjImgkFxPQHRc5LoVxdCbKQcyEm8ud/1AQgPcfINt0eQhVVq9K62fMAEX4FiGvcb91UJN0eQ5cwUmESczK3L81E577nMdD7JElpBv1TOc1xytborQRF0DtE+Wy1OHaHDsQrnQV0Jle2J74WBxbmI3Fx7IMy/5U31XXQYXhFU0vg4VnkaDYls8h1QScM4fu/A8Zb/AGZNP7k5Ongy5Xzd0s72+h1j3XTHAMGHrgxeH5gH/BM3hrBp5WQxYliQlkNmB8ItdZ5HhZOfbIXMs9xTiK5vnCVTAaSqmpTI2XlSGPM3Y2KawsHdwkOcpISYnGztwrMdTc2LtFTJsExe8ehWYJhoXZJmZfzDWyJ2gsq8b+W4vYBrqQpTICA7uFhRNc2/naCOxSqBUS5Kua7myHI2Rz7kgdPkhYC52UKUUs742kW5fwi+y0Co6jwlUyq5YkyX8rtjpZNFHVVsrYKaGSaV5s1jG3U9FSXlDJ2E2fkIAvYd1dwatreE+IBUMdmY43zHUOH+CWJOGOEa/iOrfSSTijLWF55kRuALdNO66jAPs4x/DcaLJK8igM7I5jC4xmVm+oI26fqjH2jMq5jVcmlNSw8unhykEX0uStSl48xalpah+MYS2WWmi5rXwHJH8iFOTL0E1DdrNYBs0CwASbKx2l15dw/xvxljUrqoUsE1NGcklOyO17i+hO3zXokFSHwR1EzPDZxcskcLhULiHF/uXCJsRjgbPyhcxk7jqvKeGazD8NrazFWNkbLLLKY6drc0bWWOn8f4LsePscwcYKYXVczJZWHl8uxDh7+2i4iV0GGYZFVVNTzxUD+p2jsP/ZIDso2YiKiF4a46STPL8uUkiw12sAo+HaGStqKvC6SjieIpDLNUNcDmHRg+qw6KWqxGsZh7J8kMhMsjb6Gy6Wk4nGFMZFHhuRzZMhdTtDRKdbAjfcqjusLxammZ4GSpzVMAAkAGg06LRble27XXXG4Hg1bw5gVfjOJQ5qqpaJB+JqGHqe263eE5p63CfvKdgDqiR5FtrDT/AAKkSNXqj5eiF5suX4440xPhLwDKLC45nVBeHOkBIuLaC3XVaiR0hMzOiNozNBWNQ8Z4HU4RFimIyspJpRd1MNXA/KyoycfUNVKKfB6LmPPWc2FvYdVodVksNlE2kkd0XLM4xxUS3fBTvj/JYj+N1ZZxtVW/+WxD/wAwokOhbSPjeC5S8tp3auRk4wxmSVro44I2DdhZe/1Tu+0RuH1DI8Vwe7JBcPpzqPoVmYW3TSAxut0Vavr24bSSV0r7RRWzHsCbf4hPhvFnB2LkMZiTIpT/AFc/4Z/iuA4949LnYpwzQ4XGyEO5QqecSXAG5IG3RSlp6IKjmMa9r8weAQRsQUxlWPgVS+XAcNkf6vCxj+Cu8661TCV0qjdKfzKSONjiLmw7qdlLBIA7LcEXBWWlPmO7pCRy0RhtP+UhE3DoA7urYyTUvFaaK3mbEJCbqZrnlWJ24VS8RUkEj2mqq4CyJt9bDX/ArQfTwH1MSxSilkjAcDYqZlVKXXL7lWxSU4H7P+KLwlMf6v8Ais2Im1rybEqSrrIIMIxGeZ9mMpJCT/sFSx0WHj1xvaO7SouJ4sPo+EMYkLvKKOS5+mn8bKwPHvsalfHx9Rhn9ZDKw/K1/wDBfQBcwHzFfN/2ZVUlLx3gckbrcyUxO9wWFfQ1ZIWPJBUlpYmkhY31qpK8OBKrurCBc6lVn1ckhve3srEJMtCmN5m5zpdWKyOG/kasuCflODr3sVZM/OWaQoizOL6BT1VFBNAOTbmHW5VQC6uc5hhA0BA6KjFqcPngfabLbuDdVzTP+BXq6N7nB8bye91XjkfG4F3QqwBho65x8jLD942VgU2KxszPhs0dbhTurs7M2yKHEpMnLsCOqCrDPndZ79FcFHRSjO9mpWMJQ6ocWAMF9grrJH5L3WpgWY4qejlz07LHusypbUU5fMWWY9xN1oxPzm71NV0fiKcZFi2OLFjrJvzK1T1MhlFyiGFv6ZR81HXx/c+H1GIzGN3LZ5R7rXJcWBxHxvi9NiBocHmjY2IDmNygknruqUXF3ExaJBVwSg/9y1cRU1BrK11U8lr3SEgtNiFZkxOfKWtyx33t1K5ZPTjDsjxxxJHu+k/2ogquKcXY5iNBLRVElPklFjkjsbLjDieU6yg/I3R/fr3OZG7OWE2tlCy1WPwdJDimOTUDKKPFHMiYLAN0IHzWbW0LYad0j/M/82a5VDEcQfRubGy9ni+hsqTcWmkfy3Elp3ubpeS1i6bPzKSme7/VAKJssMErTI+19lFLN/8AAxL/AKsWXOR4jVyNDuRGPqUzZ15O3DGOGdtrHbUKMkNFw1p+oXHHFKwacuAD5IPvOsO3IP8A5a58XbxHXkh/qDFBJA51uRU8sD4S7T9Fzf3viPan/wDTTHFcScfVB9W3TiniOl51dFvLC7+zJr/cibWMkdklnA+ZXNCuxMj1U/8A6X/FBJilVC0l8MDz/ZsnBfEdfngt+1B+RSAZbyXXI0+LVk+Yx0kAyb+YhXoMQxCRt42Q/wCzKVOCeI6INf8ABG8/RL+k/CLLmvv/ABGJ5jfBZw/fKL+cdWPWyUfKS6zxXm3+ZPJ5N0vAPk1exc9/Op/5pP8Ae/4Jfzrk/NJ/v/8ABb4J4jeOFxjoW/IpeBcz9nUG/ci9lgfzrk/O/wD9ROOKZz6M5/8AM/4Jxk8RsT0E0jTmiDj3GireGMH7SnkCqfzkqxvDIij4jqJHhlphf3Tkw1KepMQcaWrnp3d+YRdXJcWx1tA9kOMvnY8Wc14z3CzXSzztyvN790o4ZmD0X+Sc2vy/BlYDMwiWknhbLY3AKKuwyuqoeTGMjb3Gl1eo8FZFUuq2Mc3Pfy2sAtEROB1LR8yuninhYuJmwTEQ6zcjvrb+9VZcLxOEjn0UoaeoGb+5ehAAD+qPzKYxRn4Yh8nkJ4rPhYuAkw2ZsBqWOu0bi1iFQdlXpBoYTfNN5TuM+h/gs+q4YwyovZ8DP4LUbLc5wcJmb0T3BNl1ruD6HeSYho6skB/vU+FcJ0DKp9RDWGUsb6HiwK1y5MTi48DIu04bdRGivSxubI+3MB69lj4nHAMRLo8NjZyiQb/EnpKiSnrGzQ5Yx1YXaW6rcSU6CppqSMGdtDAHD9xFh9VHI0s8LC23UBG2aGtgcYnh4tfQ3WfhzyyrMf5gsTDNNWrq209M57mR2+S4usfnkdN3W9xJWMgiFMR6he65mSZj25Wrmo2u0W3wjhbMZqnsmmAybNI3WLQSxx1kDp23YHahe6UVNSS0ENRSUkTM7RqG2K7a2WHBgEcLQxsDXfNVuJMFnhw/nsiDQzey6nkSflUNbh7qihmYW9LrbNPNWyask7G69Fw+npZ6WOa17tC80YPO5nY2Xf8ACZmnwsdchstTKNqBtPEf2Tbe4QOgbTymqwpoheTd7fhcnyvHRM55Cyjm+O4mYnhfPkbyqqn9TW7OHv8AopOAJ4ZcGNKX2cwkrUxTDYcWoZIHvdHJbRzd1x/Bc5wquqKKp0Pf9UV1OJ4maJxDkWHNZMG1pGrwbFchi8k+JY66NxvEwghb1L95QObEz0dkG+X6oC9vdPyJ8t3sIQGF9/SgWdqcSAqMxSDonbG8bhXyZ82HxZw1BjVI6WO7ZY/OCDuvLTBPSvLJIjcG1gvcAHelcFxbRjC6t9RK3ySusD3UlqJck+qjkYWPjA+SkOGwT0ZnErhboFsw4dglTRGtYQXAXcDpr2WFynyNMcZeG9r7I2zMt3ZWq/LhNRHT+KHLcwfl3UkOGTtfnaLrVBlfSugyNj9wg52mlihfmMYk+avU9VC6dj/CxjIbrLmj5MxjvfKrFFJDz2se+11B6DhfFODx5KV9KYndxqunhqIKhvMp352rgqL7upJWzSMaR1JXeYfVUVXTB9JyhbcNKow+LxymxTfJUnG88E7diLLe4iovG4c9gbdw2XA0OMTGqjpJztJYrcSPT4ooZqT8eMPb1BXG43VYDBzIaKlMUhFjYLpp8TpaKjBlkBaegK87qqaeprZH08jRHIdLrA5+UASucwqNheD5loYphlXQ5XFgeHmxIOyrinJOug7rMwLFFiNVQnPS7q0eKsULSwtIv1BVBpgj/rAfkpIX0odm6rlmLMmLYzuytkCjOI4jUNyVVfIR2R3pzu+wQzRU+S8UgeRvZZ5jl9YWg52kvGw6Ks4a3RZiiHmBauzSNtviUkckbRa6iIDHalR5rlBczx9HIJJfKfKT8lEL3CnY0aOO6BmsJGqGzutgEUryzYG3dRXzHUqoljYHOyNN/daEJZFlzAgdws8kQs0bqp6ds0kX7QhvVRR1EnOm8g8o691Vmdy9mknopbhriwdFOSx0IYGgIDoyxkIkm0PZU4hBLM94bdwO6llZJMGQB9heyUdK2lPLduEE8TMzwtKnjFrBqpQAC1lqwxMDQO6zkIay3hixxIBWO0CDMINRsRdatcQZAwHQDZZ5sfIpAhEj4WDykknZBOY5MoLcumZTzRNs081oAtYFOWsa3meSRyWK9K6M1DIWNQ1tJI6qdsB7q3TQiSra8sAt2U+Iwu5uZwy/2lqxRpsHje3PJVsHsFNVUsdHEYuSD7lB/wBVLgJbdC3YqtHHI5ufxUvyJuoDFOcge1411son5gbFI1Y2kfcKQQQPIfcj2Vlk9LE6Rz3v2A0Ttlt6SpgWtBydlXDDmyhPJaK9rpzJduREQxzdvMmhppZSQGXCzaDyUssf4ZIkGp91HG7Lp1RZOUbsfY+xUZOpd1WxdkLDTWbIy46KCMMZfz3uoLuKkp5RC15eMxeeqgj2fchFuU5kzEaBShrHvbbQHQpbUnY27Dbfp8lM6WdlKJ4ZCHA2+iTomMtkJAJ0Cn5cbKTlm1ydLlBVhqX1AJnfKQNW5k4AeQ7fRPJTyu2cA3q5WIZGQMcCQ+4sARZSQQlfLEYeWfIPUAqsga6Mh/RI1NRG0jUNI2CTHjl3KkCB4ExaGmwYLao2tjddj3vJBFySos45gLmE6qQyiUObkGpQWZ5bOYxhFrbDooJ5ubJmfqbWuo7WN1GxuaVrHPyhx3tstA9N11/DMPKpm99ly0tLy8vnvc203Xa4LBlhjCJKnx9K9tHQ0zD+0kL7d7BcfFDHKDnkDZ/crd+0CpD8Vgp89+RDe3a65eNx5geoi1TRwO8szC50Zu4joE8jYLPOexGu2iRJY0kPLSbA26qOMMHkdsqBjhkmdaMXI1ViON0lg3e9rIYZGRS52GxsQgivzC7O4OtoQbWKC7NQyGKLP6wLO0UtdWweCZTsfewuFLR1lVlinYbuykOAGpWLV8sFnL9/70EA85PmVpoytbbqFV5bw4ty6gXKtRm7W5emhWlaFKM1lZllbmaI5A7ILEdkFK0ZM3tdQSj8dz4X3Mlzey4q1ImZoRJs5DheI5TVx1EAmbmAc4npfZUnVc8dO0PA16lHhU1O18UM0Qy5gZXu2v0WoJW8dra2vbz4aWSKljFo8uwPclet8Hvr8V4eo6qqltKYwJGPOvsvKDiceD1gwnFqKSaOCUTBrJLcwbg9rL1ej4uwjEoo6rBpKN0kgvJSvm5Ut+wv9V15ueS3VwzQvjczqbH5KnPwzhWLzCSvg5k7GGON9/SD7LRrZxI+iY+F1O6eQi0h1vb2TOqY6GqnM18scfMsBcn2W4Rn1tczCKOaTGKWPJRtAjIPqPSy80LKfiSuqcWxmv5DmNzgWuCFFxRBjmHYiylx7miaraakB0pJaCT0+ixZpWZTG53lK5zK4pZ3iqmfNHLy22v5hupJMaxSTChg0dU51Pmz8saAn3WVdmexarEb2W8izbS7UsmpYqYCZr2m1gOi7XAcSwvBauomNaAyWNnMbGbgnXouDjmkjj5EhYGEAgEXSimY2YBurXbjuVqJHtNPXUVW9kdPVxPfI0vDWuuSFNqvHaSvOH4jT1cZsWSA/S69A4240gpKKno+HjFI6tbndLr+GzTY9DutW5S6LX3S191X4DmlxfhilnqmeaMvjz31dY7lbjsPiJ2W7aZQbIT5XJnR1TBe2i1hhMtrxuCRw6pH5T9Vm0pg1jXS0skc4uCDoq9JVQUtAckjpeWCTfv2XSDDKrrTsPzKya7C6mg8RVPgYyIi7QCLE9ilnFVc+TEqCRwElPnb5XAgm3dcjTVrBK2qL4TLGbB3Xcf5Lr/u6krKB1bFVyCVjM92u0YbXt8lw1RhvhzBPHUEw3AM5GjiOoC47WsWjTYhRxYwK6q0iL+Y5jBcX+S0MTxDDayM1X3s6QsBexjYSCDbqucndz8VPOla2N4Fnu2sp+c+Bsoo5opRkILjsRY6Ly47MmlnAc7MBlLTa7iruH1ENDC2p50z+bNy2vk2t8lRoPwuF5j+YkfxU9HXU7DTz1UHPMbrtYX2Hufpou2WfAdZWQ0VHFA/mFvNJkkcd/kFPTNoZLTMZPE4dJR/guSxrGH4zXUrIGcuGJ+oc6xsugosUZiT5cseRlOAM17hy7atsTPkkw5qZmK4HXGaopRO8yEyStN2OifsPbWy43FsUqoZZ6F9LHA8EscWdj7rvsW4n5VNHUYWyKoaHfjseDcD2XBYxNS4jiElVBTujgkdnyuOq1lmnFjND6hrr3ys3VeU6Fw2cpJpXQvLGHRVXnOPe6xDQonWablMZNcxCizBOXLoqw0tuC5Sl1h5NlVDhZPmPdZmD1FJKx+p/RA11vSgeb6pWCUJC82R01TNSVENXAbSQOztKhDUeV1r2Qel8M4lDBiTXMf/AEPEm82NvUSdb/VauGRyYhKcZr5JC97pI44gfwxHqNvcLz3h6d8tFPhfMtLEfE05tqCNwF3OEyYti8DMSw3EIbSsySU07fKx40OvRcYh2mbi0+HYjHgeIYhhs00vg4I2VMXMJJjvoWD+C5fHMampIpMZk0rcQvHTxnQxx9X/AOCu4lUc6rfDXVfOhpjzaycNte39WB0C4jFa+bE619bK3LfSNgOjWDYBWmpynDHy96oCbd/mpi8WFuyg6/VPfTL1XV5zuc4o45ZALMch9TUcUZA0KgsxA5BfdGX2UjYjoQQUYijcDmb5h0WFJlQCclg1wFge4T82RjgQ+6aQMcLPZq32shjGawVkakVTmafJ+iesk8TTy0szWSNIvFK/dvyVMh7GgN+I5SmqJHsDIy+4BuCFynFltR02FTU4qYozCWN5QDXaXGx+azHVL2sFNLJ5ySJBbtsge9kFPlLiPkVWYWPaC113KY4jUw7EG4Q4vgqJ2ucLOEb8oPZJvFGOCDwjK6QxXJs43NzvqVkvk1UWYrrjA6msrWVuCNbEJCA4mQOPxgFV8SxGljoaJ9DHGWXPMY8XIt1WKyp5UDmte4XNyAdCVFJIxzsjGW767lagXG1fJqjVUbuW69x1spKesxGqn1qRn5glDzoQRss8PAUsEn4rfndUdRWcX4ziEJw6uxFrjMQyQubbKwLusIx/A+H8GhwqlfPXmC9nZQ0G+u68qEIcXVDgC7rqrv8AOTGL/wDyqD/eKDu8R47xyZrmYbhdJBfZ73ZiP8P4LD+8sYqPEVeOVQm5bs8d9mgdh06LA/nJipH/AMpgP+0UE+O4rPTzQHB4Y2vjILhLsO6DqWSQyMbJy4yHi4NuiEvhbpyGfouch4hxWKBkbcMimDGhgcZCNAoP504lzuSMEje/8rJcxWpkdJVYhT0UDqmpEojZuQ26pjjHhu37SQH/AMIhVqqeuqqN0deKWhhk0J1c8+1lSYzDKNz3UlAKud/9ZPoL/LosWtNj+eHDxPkqZD/5ayYXMDjDSzOqHSG7Wh+aw/wQtlw8y86Skkoqk7yQtzR2/sqwYcQOtMYath+KC1/0WuRRpqNsTM+K1FHSMbuCRJKfosfFKvC5KcMoIZCQf273akdrK2I6eUluWPMNw61wqGLwshiYQGjObaBQbeE8TsoIYoo6moo3Wtdp5kchHUjp/wAV2FPxjKYnSOw6Kut8VHMLn5sOy88wiKOajkY/4Hdlep4Y6Z3Np/w392iy1EpMOxH2kYX5gMKqw4MOdriAQV2+GDmUFFUt9MlPG8fovGpQZ3F8r7uPWwCt0WM41hrRDQ4xURxs2jLrt/RVKe1sseqPl6XGh7ry+k+0jGIGnxtBTzP/ADNu3+CfC/tHxWGtqJKhkczZ/wBnG42EduxXLJT8d1NVh32gYNK2qjbJFEyzns0bd5GvtZd2cQGH8pmL1VGHT/sjHE67vpqvHuJsUxHHsUNVXGM1EbeW0M2A7fxXb49xe+d+HfcPKLDYF0jQ7K8W0Pb5qcx6IY/wiVETkZdeVScbY3g1dz43Qy88AuaJC5g+XZd1g/EEHEWGCvgjMTibSNcbm6YzY1ue1ZXG+IspeC8TkyCUSRctzXbEE2UzpiFzn2g1YHBmIMJ9fLA+edbHlvB9bHhnFGD1k+kMFWwuPYbL6GrKiYuztdodl8x/l/tBfQoqWGnhvJf8KP8A/cCSLJqCNymbO0/Eqb5GOHqUecjZy0NQT/mKt08zXNtfVYmeVWKaV7UGxm826PLp+0CzPEP/AOSl4j8yzQ0DvZVaiO72npfVMyZ17J5ZRa5urEB6hsccQdG++qej1ksg5L5WaIoqSRv4glGnRUVa6X+nPAaBbsFJSyl4ylW5IqCKkkmmpeZMTcBYWI4h90Yf4t4Ie/yRtOlyksLOJcU4NgtQKerfM51rnlx3y/NNF9pPDjYwMla7/wAj/ivMKvF3trXyMLXscbkOF8x91JS4lhdVZ0cjaeYbwuNgR2C5ZPRhg9BxL7TIMhGFYW+R4+KXQX+QXIYzj/EGNRPZVzu5W4ja2wBWeMdoS9sbaWS7zYecIZsbpY3OZJRua4GxBkC583Xjgynx1DCdLKrIbyWkBHY5loz4pSygnk6exCzedHLKGh2pPVahZ/KrZrxOktbIR0VxzHCpiygEGxugytEU8Lt1fiAMUZ/dT0ZseKwsmZFK7XK1beE0dBT4Y2c0sWYAkuLbrKkbmpvN0W1hw52Fvj7tP9xW3NGzFqds4gmZHG1/7NwFw5V8cpKcNZUMha197EhZGEvnoxTVmJFklP5ooyf6o9LroMYZmw5zuxBWjBzRoiJC8klp1t2U4oYi25jd+inOrFcibeKPzX0XCG9jna2lEbWctvUgqmI8sjXOvv3W5iUQyEB4OR17rKlaLBzXA2K6sNaOMXDnNAWbi8QZVANPrF7LWh1ja49Qq+KtaTE87D9VwifzOzNwx8EMsrZ5uXnbppdalDXYdA4unrWO7ZW2VYUg/KFLFSsvsFvmzwHV8morDNS6xPA1SNLG9uUhWA1oTk2WLaZhwcyDQEKGXA5GbPA+a6ala1zbo5oGO9bbL0OLijRTNPnyqajpbTZXu/RaUsIbKWnWxUAAjmDmhDHLzSz0zxHeEkuUsMVog17dVZDRZEV5spdled9bDFzYJyADY3F7KBuIYq//AK7/APitMwNmo5B2KpQ0rIgRmvmUtAiTGXi4qhb3ITZMYP8A1ht7dwiniZcHKlh7GCrB6EK8WLR5cY/17f8AeCaR+KstmqR9CFsGljcf2YVSsgDTfIB8lujHLzZhnxH/ALUh8VXjapVsxscC0tUZgjYzyhZhuYb+DYKK6GKeohljm1zGR1mlU6zE8ShqHUOB0Qcb2BHmv8tF6PwyyrxLAKY8uMMYzJmcVp0+DsY8PZHFm7gBdMYcpl5JNw9xviDB4vDLe1spW5D9lVLPGx02MywvI87HRi7T2Xpc+GyzRF2djLd1jyl8b+WZ2XHTMF3hxlylfg9Lw/Tujkw+zfhljGhPuuXD8uI52+l+y9UDJC0smjEjT0OxXk3GFHPheKz0zPKxwBaPZTinJkcS1XicQLWPuyMWCymOc31OTvLnPuU7YmGcNOyzOLUZWMkW0XrH2UY62pw1+GVMmsRvH7D/AJC83kpaVkIlYQfkVe4Gx37ix1j3t/Cnsx2u2v8AxWMfyt8Xt75YwdFJE4OBYDvoq72sdZzCC0i4KOHIDqujDy3HYH0mPVEb+puui4DnDaipgefXt/FUePaVkOKsrRvKACoOEqnlY3Ttz2EhsukMPQnsN8uVQStcDqFtSYWTd4naQqslJHH+0IPyWYyTiz4fVsuC47ozh2NR1EPlbUak++i9IFMzN+G1cb9qOHzCipq87Mdkt/z8lbZpVdRMgfS1AN3Sau0XTtAErVjQUrqugoZB0AC3fDSFwdlS2pWzVRluW6cMBGYLPkYynkElTLHE3942QO4t4ZoQWT4xAHDpco01CyyhkyhY832icI5S5ldJJbo2IlQN+0LhWX8N0kzb/E6MgI02swzLmePqDxuEGQEB8ZuLnfZacXEnDs7gyHF4ST3uFX4ip4sSwx7IKiCS234iObyyin5TCHtGU3BVR80jZXFryCCpZ/wnGIaZDZUZHPznzfqjpDSZjNZHHbmlQvxmrOxy/JVYI3zvya/QrSjwDmbPZ9ZQsNMqV4e+7kwYL3C2JuGZWMzsmjFvzOWbJA+nfke+N/8AZN0DBl/jk/VbuBYzLg4FjzB0DjssNr7IjLdB69w/jlLi8AygtkG4K4HiSgZR447l2DHnMLG6wG1VSPRNI3+ybKanknqXtM88kju7nXSJSYbE4kEYcx5c3qCVUNTSN/Z1Rjl/KSmljq4WF8dVGW9llTC813NaM3YJaUs1GJzStyPlLvmqjquT03WgMKjlh5nNDfmsaU2kISyk+djvM/dEJ2MOZqrjU2LTZSeHZ0bZShZNSZG2a36pRTmL0i6rAFnRLMeqxUDHANsxTtJROLPSLoQL+ldFOQA7zWunGp8rQo8t5MrtlKModYIGkaS4EdFNKbRZ0DtCmmcHNDBsgaCYuGSRxId0I3SkY2InYdgFDFlLtOiaVxeblFTwnM8B7QQpnNkA02VWO41CnZUSMtnfeyAouY5pzREKWKPmNcM4bbXVOyshN2Fh1FlJHSiYjLIG3UkAyPJ57gkdkTRndm7o3ZIWvjyF3uE8DSImvyE3PRBPRtIlGi0mvyjO7YKGCBjWB9tVPIeXTOOUFx6FYmRkTSZ5XyMvZx6qNpjLHPL9R0UQq3vc4ZACwqMDLmPcqwE5rHyZ3gH5q5EIeWHPVG9xexTHmStLA0lo6INKnZIKhkzGaZxlKs19QysquS92cBNRT0vLbDDBIHM+N50WfROghxD+ln8F7vMsKtQ1LqJ4v5ISbZBsFNiOFwQs59I8PiIuLJ8boaaCdppJ+Yx4uBfZZ1pAfIbe3RBA6ny+blAJh5FNNLIWhj7fRDFC+YPcLeTut2iIzWGWyOF8YsTugaxjnkSX9rKWKOGNxc4Ek7LUyAOUuzhpA90bJHhrrPI06Ipo/I3I3UnVRxRMkmEcnp6rIAw3AdC8k9ipIaWefNkjPk0N1YLRHcRxm3QqUNlgJGlyLm6Cp931J6NHzKaajlhALnMeDvlN7K/HMBEZKq1lXM/LkLI3nL0VgUA0gqZmcewV3msczKRcKCSB8YFruDglhx5XxzM8xHRTyv8AEgh4Fj0CpxztjGR7TfspYZpCc7dG+6xYLNI5ltQ3sijbYjzHXume9w2v5+6bmPaQQB9UZTNvHewBuOqqH3ta/RTOeX+ZRDKXZLtB91qBBJlPoafqjifIyE2s0vda5Cklu1oy2OXsoy7YOVaO0RSFzX3sB0RObEbPj0J3ugvb4QhG6C3TXfVRR/mcF3mEw2yLiMKhfNXx26LvsNFmNf2F1pl5/wAUufWcR1U2QvaLRi3QAKrTxw07S0scf7SOaXn4hUTZbcyUmyrzS62G6NAdcucS643ATZ82vVROMjiXJRXJQTMy5hf09VOWsI5kfluLBVjpb3U8Epe0tsSGa6DoiLrSYI2PYbW6IZXiuka1rA0MFgAFmiV5nzE6K1GZP6mbXtZZRVkEoDpDq+QkaKSljLbX3V+Bhhzh742SEbHqoIG+YqxK2utOWldJ2CgiqZJw2C8bdD5uqPEs8GGHIfW4BUaQ2LZCuatKYMdRBjCC6MWFxur+A4PBiVVRYdL5TPUM5jrXIA1QURpJ4hkbd8m+YWGnZbOFxU9I9uJeN5c1O/yxsdZx76qwzLV4kFXR4/V4i6jo4IYIWQU75xmJA2AHS68+q5qtpdDUM5bxKXEZQCH/ADXt5wJmJ4dTCql5z2TMqS/cEDof1XCfaXw5KKs4lQQf0eniDJOpL7kn6arrxYRs4lrcU4RfXV+IMfiNFVQCib8cYGl/e4XVcMcc0OM4i6TEoxBWR0+sINw4M1vt7Lx1kk9NVMgfmheRcgt1t0UzKt9HJPNT1DmvlbynOG9lZ8kjFc4j4hr+KMcqMYqnh7AckWUWDWDZUaZuaS/LEluhVekLIyI3mzDuUb5TE4iJ1j3WJboecRGZ8bADJofZDfzZ2Ake6jjNwrTHMbB59zoqq5FBLWUrpjSyBojIJ6hUGHI7MpTXTsi5Mc7g0726qFrgEgXC5hYGncjT3QslkZ543EdCCq8crw8ebVSGMi73PvcnbosjfwPiHFaGnfh8GI8mlnIzNds3Xf8AivWP5wYXgGEUYr650rjF5XAXzD5rxKCR0IZIw2LDdbMFFX19IK+GAeFk2vJ/d7LTDr8QHEldLV4rh2JVkbHuD4WtNgQQux4RlqpcBp24lN/TQSCHn8Rw7kLzCHF8X+7jQ+KmluLNIGgHzXW8P0jOHcO+/wDFamRs8jQC6f1m/RFdOz75gibC8w5onF5lMpvYm+y4rGMTwuV8VJJjktXDGSZWWJzv02/iro48pK+mxClq3tDD5IZA3LcG+4XAuFPT/g08kkrh8Q2XLLY26d2PGjoJqHA2NjMm7nNs+ys4NgUXEmDy0kdc5tXFkJkeC5gPYBcXDNBHVCqqWGUgEWL1vw8U0kNLF93U89LKBYtbKMp73C480pnYpQz0mIyUNXKz+h3ZIWv0d2UtZ4RlO+OjjkaRHcucb3uoTieHh8pZRTCWc3cMwddBPi9KyJ9JS4dK18zXMM0x0A9gprx/M6tnC4PF4CKfmNjzAnM7YWVLD5IHcyOoY/exLNQ23ZW6NkIwKJs/oe4MP1Kerwevw6WU4a+LkviDC54AuPZdNzmg8bSwVBqHvnmI0N9LhScSYxHFw1JDSx8u87BC1ps8ste57qOjoWudK7FIeTnLAH5Tb/nZHjGCQ4hVxywM50OWzZWPsLLGueKS4p8c7XuImLb+o5rao3SskFrgaIKySlbJJFGXmNhIBJ3somtnaA7kFgcLjMLXC36iKqhgIMjn6/JZ7wAVpTm0BYWC7tyst51XXEC8De6FO7dDfWy6w1KRu6ZM3ZG7ZVlGiTgXKRagePKjzZfSo2dkbcp3RpYo6iajqYauI2fC/OP8l2UGJ1uHPmZhuseKND4SNLPN72PtquIyrewfEJ6fh2umZYvoz+A5wuWl+/8AesZw1hnwQ8R1zGBuBUr88UDs07+kkv8AwWMf8E3+V/ql0SlnLmQ2TphslmC25wdSwus6+W6hbspomNLT5rLBC5HKQB7Kds7HC7QAb791QEnlud0VOXvcddAFmYVbku8a3+YQwuMflfv79EL5TGwl4J9goY3AtMrWhpJ2vsg0433dbN0/ikbPzcy+Zm2m6oOqMkvMephPHM06mxHTdSmUdZO9+UOJLB1sqglc03G6GaZ8tyb5ehsjbFCBo9xP7y3EB2TWkDiBa26Nz/Lf3UT2tZbMQ7XoUT3scMrBZAzpDa19Eg7zKJ2bojGjbrawnZIevRSxnzZ27hRjKWh3XqmzFuyzRS1z5O6MTSnqjw+l8QRotZuFwgLmrMa+rj0ZUNDe3L/4ppparlPzVAylpBAba4Wt92x91WxGhiiop5B6hGSFq1pSopTJyue18kNjeNul/qtSnkqJMzKOmipI29GN1PzKjwCJkuHh5F/MR/ctWhhYzm2bbzf4BYtuma7Dpy/Pe57vF0Ya+ncY3zNuNw2K61Jvwo3TdGC5QxSYbCGxsrohnGck7kqWkyzMhqXhjJ4wT0fFYqB+GTRVV4nCN3dui1qt+Gz3jnr4HZBdrr2IKGJ3NEMrt3t1VZtlVNZUZg3FKSOsA2fbLIB8wsfEZDIGtu7JzPK1+7R2XWVNKyUjM1YvEVCyGkY9zbXctRLTPoJ3xROySBl3Hdt7qz4yo/7Sz/0yo8EbDJK6OT8unzWx93MOzFbSmWcRqGn/AEiH/wBIphWVLvO2WIj/AMMrY+7GAbBMMLYd2KWvFlGorC0jnQ/7pQCaXv8AotZ+GRW2APuVlVUJpnkFzS3uCsTNszBGax7rdweBlRTPfz+SyciIXOxJ1JXMsk8wKtR1bxG2F+rB0XOGHY4hgXhcFkhZRGM0zi+Op5osQe66jgOlwOfB2w09U+GqYfxuY6xcT1suFw3FKSLCZKGEyiUyjQi99brSosar8WqRR4jPEymgJkdLBGAQRsbjddLoemvwaMC/iZCuU+06klp+DXmmu4Goj5lzsLn/ABstTgaqnrYqwMfUzUrZPwZ59PoovtZ8bFwd4SihEnj52U779tT/AILtEjwyZ+SBx6r6LwXh2OvwbDq6TEAOfSRP0H7g9185lvMcyO9r2YSdrr2r7Oselj4ZosHfWCqlZWSQtsfTELH9NVJ8x2UfCtCBmNdM76BQVGDUsAux7jboVoDEGRxFmeypSV8b7gvuoKghjHmsgdYO8ikMkb/JntfqopILC7Jsw9itwwTucOiJpneL8slC2ORxysJJVnlzsaLxll+6oASZd1IKxoAa5oNjdVpIpnG4sPqkxsjfWGFFWhiAadUbK5h6qlI1knQD5JhTuB8j7ojSNSAsrHMNouIzEzEameJsW3JNif8AmykdSVrWl97+yoxYnSTZufNHCWG1nOvdBzPFPA2E4fh4q6Kvq3+bI5r3aWP/ALLEpODMLqOFKrH3zTeJjkIAzaWvay7jG6/D6vC5KSCTmvO2mi5fB5nDh/HMOe0FjGiSxGxv/wAEdrYs9BHJS09R3aGZhuqb8IgdXOp3vkNrWcSt+NrHYDG8tvkcLfqsuSW+Ic+3QC3Qrg6RLJrMPjp5THvbqq1Oy9ayMd7rWxLz1Esg/trMaeRUsqAwP1sQeyy01IcpFYx9vOLgnopKezImgG4HVBTiN00heQG5NCSpIiMmpB+SUi085aQu9wFp4BLzKZ8fY2WYXMNE/O8Mt3VzhqeAyzMZIHfHp9V0YRsxATNqKGmwN80DJLSDmCxK0MRaHUMkTBYZQQFi3o6PHavxWIVEDS/mRtGgd3/wW5zIaikdNA/Ox7TY91pmPViNF2K1AZOW3ltB06hVWHyJ4MVpaMmObmA3/KuEOuSaWGYvvy4/lbRQS078jnihgNhf0o347hz3Zs8g/wBlA7GMOe1zOdKLi2jVphHTvzMBygHsEdTyzEedCZB0UNO+GzmxyGQdCQp5shgLS8tDuoXn/wCp2RM1jCOJpfII2bnZA0sDA1jw8d0UTstTE7N8S2LYpXj1q0IaBrA2aCQu6lrk8xyNJ/KLrmKvF6mWRz2Zo2H4Oy05c2pS48KPOw0hc3MbOKgq8cEtVni/Zdlgmrmc6znmxRiTIF0jJyt1tHBhGKNM3hpw/qOcBdNiFNg2GxtkmpJnXNgBMFzFBi0tJKHtsbdCEVVi9RPMZCGkHpuraOkhkwKZt2U0/wAubqrRp8KDA/kVWvTmhcmyqfdsjHj5AWWtR4vJUBlO+Muts4LlMcnbHJrMZhmQs8JPr15qjdTYdHYup6qx2POGqimc+Jt2K1JLmoWSZAdhqFODfNA6LCHbwVX/AKwRw0eCstN4WoPzlRT08DYM4ZYlNB56Z7PkulOMr5bhrrPbHPYjo9BUQYU+mc2SOcgD2SisYmtb0FkUjAYXXNtFaRmNo8Jccv8AS/8AeCGWiwlrrf0v9QjpTeot3SqxaUrNN8l3CsZqsHbyMOxGtDCbiM2LLq9NxvjDXZJKqoaf3YmXWPTEWF+6nqc+bOzS63DnKefE58SbmqMVxc36B1goI8Lwr1tqq1j+/Msmu805LnXQQ3LQbrpEouAyQi0OO4i1o6c0LNxmR2I8llbiM07Y/IM2+v8A7I5/WVRqAbF3ZW0opeFuGW+bn1wv2sVn49g+F0mGsq6FsjnRu9TxY2W3Ui9PFN+ZirYjC6fBZ2Mbd3LNgpkQ5ynEE8d45ru/LbdUYzGytYZ/Qx4uE2CycusZGeoIT1DGGplzj4llvk99weSOow2B9ILsyiwWjBBK+QDluH0Xh2H41iVNTAU+NTwj8odorbOL+I4/K3HahOTM4vQftGw3LhsdduYjb5bLiMNq/D1VPVF1gx+qrT8QY5XjkYhis88Z3Y91wnpgHx5FqMknye14fjGG4hSNfT1kTnWtYFNI27t14hT4rX4LUGSgn5Tu9rrSg+1DiummDpKiGob+WRmiyW9eiDIzmkcGtG5Oy5n7QsTwGt4eqKQYpB4iPzxC97my88x7jrFcfc1j4RTttYiN5sVhCPDz65/xeocsjqcG47OG4VHS/djpXxelzdiq9X9o/E8/lgbFSt9mXP6rkNTNylrw4aA3SUm/ddOTXBUqcQxCrJ8VXySZtwZTZVssA9Tmj6q/UUzadud9KyRvUgbKKOSif6DGPY6FOTpwQsliafXdTsqYj6D/AARkxxjMGscq752uPoDB2Ck5CwJ475nxh590JrJh+xyNHYFVXytsbKMP7rlzlniN+Ykk6klQyNiLvOy/1Wnh8EEzsnqU78MbnIiBHsQniStMyjNLG9ofSvl+RWgWYafM2iMbu5fdP4VlP+2jAB7aIiyiOxIPzWOUtUDwkk+0MZb35pR/cuHAfjyGM9gibJHGLMv+qTpY/U+QD5lXlJSpJhlJILUr5L9yq78Nnj3cFqiNhF2yt+hUEhscpJKc0mD0NDRH9ux5d7FWKuKkEYfDGQR1UMVSY9WfUKU1ub4QE5S1GMe9FDIZG6giyGXD6aYXjY4P9knS5T5bBO2fL1V5M0hiwydot+I757KCfDanUhrbfNXjWSXu2Ypc8vHqUspmxsijAZNo5SvoqmUZ4Ibt+anbTc6QeS91u0TWQRhjdFqkpyEsT4zkcLO7KNkT3OAAWvXYVW+K55IfH3CdsLGjzRgLTm5NuR17Ak+yYBw0IIKmhmFKS2O23ZMX5zd41W1DHF5s7x5UMuUP8uylL4gy2TXvdV5N/K5ApC0jQ6oWMeT0SY18hs0XRtblcWm90VNFRse12ttO6qH8J5hbqB1VqOpMEgI1RNiFS/mCMNJ3sgrRR1EzwyOB7/kikhkiNpBY9le88VO/JMWgdB1VDmGQi6gOIGJ7ZCARdaURpJb855DgLi3RVZ2CONuXci6aLTzdSgN4ziyu0mcMDQqceUlXqdzG2u8D5rIvwusMp7p6lzpRdmzd0TIs5GTYqrVTMiJbusSKcNFEXSEWAdqqbmZTl1KmmkcBdm/RHTMmlhdJpoOqsA8OhMs7adhjzyG1r7LToeHxaaOWqeya5IjaLD+9YVFJURYnDON2O0W3iuMzw1DJoGAPG5Ot7LnlIpyvqKF5p54QYx3VGcxvcXsbYFdBVGlxih8Wyzaj4gSqdDQTwysqhG14j9QO1lYlWdFKwts5yYyAK5j5b4iN8EbAwt0sFlGRdUG9wOqOCVrXaG191WkeSLIWO1yq0yuSsjDz5wfkhY10jwB3QdN07TZSmltsbJJMl9+qiMDROchug5x6I2Sgb3us0LLZuUMt0pH85+fc2sqzpWvFkbHkDRQV6mRwsx7bBxQiwLXjXW5HsrwjzMc/sq3OjsNCHg6u6LcATNNrkisD7KzAJJI8r3i3ZKGWGKAGaMveNnjZW4KeOSn5gfYk32ugpSsZGPxGqnNMxrDy3LQnjZM0sc+zugtusualfG4sd6VBotgrpoWzvjyttpmUD5LHI92qfxBEbQ2R/k7lP4uGV+d9jJ8lIZNlkyuLWad1Fm1B7BSPkuDY2Ve/croJDJfyjdCGPffKEwc1vqUsMU0xOXLa19TZKKRjN1Rte9jtALe6J9rWynN19kmhjmu89iOio2eHQHTAs1tuuwdOaLDZaof1cZN1y3C8BjjOcbuNvktviidlNw4+P4pyGD5dVoefxBwc14d1ujz3z/h5nHY9lO90biAyPKLIIgHODwfLdYlYOKCIxZ+cdRfZZ8bOW4szA2NrhbohjDHNbMy5632WNyXxzvD9ADuqqTLZoZ3Ripkij5EQAt2RlrWOEMjyxpF7lQxsYCfPn+qjVInm7ySAD7KSkmEVRG+QmzDfRTRxRl+bclE6iIcx7bHODoDsqytVrqaaEVUcZBJ6qvTDqlMx8VM1mY6uVmkY27QG7rDVGxcPdSsgYLv3ssmmkEc1nus0G11p4tLy5nMtc5bA9lRiw2eeN0jMmgvYndVGpSPfKzPStLmdCAtyioCaeKrfIyVpvcjSwWHhMNRTUTLVAbn3sdwttmIMkyR5Z8gYWOs7T9FiWZhuYiavDsMaYcbkcyUehpub9As+r4rxWopW0lOwRRMiMcgk83MHZZFRNNyHRxW8js4eT51WpZpJX8uQ3BU8TJji3ONKyOb7pxCCCFj5ImEuMZEhI6H27LBxh1BU0nIoYI3SZs7pzpI49fkFFjuM1+KObLXTA8h+RrGiwaFThnstxlyWFQhwuAga52bKRur0mSWN8bBrbP8AVVuVm9GjltRxMBd8lMMjnGHy2Pc/3JU8Lxd1tQN1FrJAZiWgg7bKDUr8SNdTwU08cDGU4s0Ri1z3KzRHI70BVmSW+IK5FOAwDKkAHjkhud1y462RxuAaL3+iDzSyAOFr9VJGHMOe4yjqUIhYzWiFlv8ADmL5pKTDYWEtPks534fU6iyrYNFRGkE1RRc2ac3s4aAf4K66ldM5jY4Iogw6ZDr8ljk7Y6zU9VaaTDjFIGiQgmEaD6o6+fEayRkdXWzVUURvGJToETIhAwMaxoI7KGQPDTd3nsSJNgFxyza8E08DWxNLBZz+pN1H52EsfHdv96qNrvEvDA8k9LlHV1vJBDiG26LkcRSCme0vgiMYYNibmyibM2O+XQHZRRyzSASxDQi9+4QxETSNY94FzqT/ABSmEsr2RyRvYJA/l3zZkcYzQeL9yMyOsgc6rY6nb+Cw8u99kchkjDoc7SO7Qt4fxNtytY/+a8AjF3GRht9UNJj7mUDaXEoTUOgkAjLSAWjpZT1b2R8Ow53bZLfOyyaZ+Qtc5ge525K7ZzTEQ6PCcV4ahzmsrKyozm/h3xiyVFh1LW4uK+N9qaP1QCIgEnsPYLHgMLKsTyAud2GqjxTiSTBa+nbSc0lg5krB5Qb7A/xXPGbZzxpW444ewigraQ4RPJHT1GcSR/6s/wDJWBiOImeWNr25WxRiNovfQBDXYma6odUSNyyPffe4CiBjcLv37rrEMAkmjILDHmuNDeyoTgse7MNArkpY4sfCwhtr6qvPqSXfFqtYiuBnPqsny2SG6Q1K6w0Tt0g1ItQt2VZSJibJwbXQu3RQ5inDk2UpkVKD1C2oHcrhCskt+1qmM+gIWH0+q3Xjl8Fw/wDe1Z/xW2WLm0smNw1PlsEJNlkLME4HQIEYHQLLR8xRxPcB5uiAC6dxsAAkAjlfIWs9N1cpiyOF0jzpsFUpRn/VTucx8dnEkg3GqCy2Vs1ZTwMJyyHln6lQSU8McsjJJw0McQD8lFCbTwucbWlZf9U9ZpXVDDuJTcfVYplG8m+pvrv3Txuf6WfVRvHW4+QRtLI/WAbhKEkm3Le5tx2TRRMlkEbZg0nXVREfiljdBmT2cw2DxcdVRLLGY5CyxLejrbpmk9FIXl0Qa2Uu11Cj5bnNdmcAb2HutgRsjERygt6oIw22psevyVuR9OyB5aHHILN6osIc9mtZ1A1TsPmUSmiaeUZLEgOsVJGxhlZT0sZ5z4xf8y6JmWRjXsdcEXuuawSCOqqoxMy7b3supe5kdrRn6LztQAM1VDiAcvDHyDp5SPYghaQdf+rk/RVcaax+CVmdh8secXHVIltU4UyHDXsduyU/3BatPpJP/aH9yxODjelqh1uD/gtyl0ln/tf4LKjnH9Fn/so5YIbTeSD090nDmMczuoTQx2vJMWDuYxYLUMyJkELWUz8kGre6gp2Z44f9v+8qlRuhqKcPjeTYkGzb9SmnZDLlZNXFkIFuXGbHf+C2lNGqqaCmcBPWsb3G5XOcS4lTVcMFLTvzWlDy7ZXx4WE/0SBuvxuF3/qsTiCVxnj5wzPGodoLDsqhuHZmRYpzJDZpaQuvEdxnZ6e64fCwHVWZ5tYXHzXQ00lXE1phl0b8J2Rps8u2qVrAu0BAvqqseIljR4mnIFvU03urEdbRTC2cHMLWOhWaWZc9WVVVPVc4SAWbnsw6KjNHPKBliI737rQxGkgw3zsnDmyAgNA9IVFhY9mcFJhifNBlyLSwGJ9XiUNCxrD4g8o3F7ArJkkAeWt1VihMz6yBsHre4MYegNwszDNOkxLhp+DVAw+pr4DNISY27XGlkGC4jJgj5o3UgndJIGGztWj9FBDiPLxiar4ijkqpIwQHZtAeirnFmVFRK+KkjjZK4EC9w0Dooj3HDI2VWG09VTVAhE8ecsa4EArmPtExagbwzUYa3GIpKxkkcsbGuzG7L7HpoVwFGMSnY4x1QbFe2Vs3LCU2Fztppsoi/ZHaUFd8ZWnN1VSyoqvEDTUG3vay6/gPiWiwOGqdV4fLO+R9mvjOoHULg2usV0vD8FRJRl8cdwXHW4WrR3//AEi4bfy4XW/VwWlQ8Y8N12RktVJSSP8AhnbYD6hcG2jqz/U//kEjh1Q7eEfWQLMrT1VlHRTQeKir88Z+JrrhOcLZNSOMVXy3H0Sg6ArycUddT3dDeNp3LJR/mrmGcQYthBcIKsPY83LJ7OCzGSU9Aw41eFeJZXVPigBzI3X1Pf8AwV9uNyTYeySKijdzBcAy62620XldTxBNWVgqg6SnmcbF8ch2WozE8TpqB0xxSYtN2G4zWKnO0p6DBUNniZM34hcjq09ipWSc4Hl65DY/NcbwzPSy07qSLiOQTSHOWZBe/tcqQ4di0zZXjGZufa8ZDrMJ91uJap13J188b/0UNTBdlo5Hxu7hcrbjWngZy8XmOurRJew7pVJ4vjqmUk2IeSQXFQBcA2vZDw3Q8mVmHywT1ks126kmy4sQtPqVnEqnjbDKV1XUYxGWMdkOUAn+5UaXGuJpqxlNzKeoc8XDJY26hS3TgtUzA2YR9CuZxiMtqz/H5LqI4MVfVMNXStgmJuA03Bt2XPcRMkgqM5gc75LMZJONNPDDzuHXNt6DZYZBDiFp8N19O/DauCWRsNpLZXmx2WRUV5gkLGQtkHRw1ujUFJ6HA9RZUD5Stagw/GMdgkqsPoOYyLfUNJPYX3WTM4sId+9kt1BWZhbWxTRvDXPbrZWI42xCzNAs04nLDaMQxm3W6vNnZI0P2usN2vt1pJR3YVNw3GxlcQG2ztKq0juZFIzuwqrw1XTvxWON79Duuzk6Ktrq1uKfd0LqOLNHzA6cb+yu07ZjCPE8vOd8noWbjtC+XE6es+7zWwsjs9gsCD0WhQVT54xfD5aXlgDLJbVaYZtVg1Vh1MKmR8T2PdpZ2v6LPOW/mY0/MLoeKJ534bC18EeVkt830XHuxGBlSOa/ydV584d8ZtdNRRRutJJEw+4UU74XgSU8sZIPRYVTURvqXOD9HnMpopqRzBC6sETj1tey51Kc2iKm35R8lK2te4WuFjVFoi0wVfiAfyt2Sh8SHBzY3/7pWeMnJriupWm08haf7N1boI2YlUspqGrHNJuBJoD9Vzs7uZJmcwj5hXMJnfBWRzs3YtVJybOOYhiWGVDqGthY15ZnaRs8Lm5akyuzujD2ndp2K9K4go6XHsEndVwxl7IubDMw6tNrry7Cad9XiVNhtVO6EF3meOi6sCnlY92dkbYz2aos3ddBxRwlTYByqukxkVrJ3FmUjVvv7rEbRVVR+zgkP0WoYmANEfV1kbXs6Oupm4RWfFFKz5KOpw6qgjMnKld8whxPmcRYNBHurtHXeDiDI4SXdwVlNFUPVTTD5xlTRGV2hjkH+wVheLrKTE6WWl/pRLZD3CQxJzI+Syoa5nYwlY9BC4eSWJw7XFloNYBstNRK6MaZkySPzD2iKOnxCmkfyR+GHblzbKk1rgjLQ8ZSAVladAx8DI8ni4D/ALSdtRB/2iL/AHlzbqOL4Rb5LoKTD6OfCgXQgOO5Cl5FKppDzeZDNTj/AGk7qSpebnwp+UipHCdSBKBl7hAMOu7IXj9Vi8mmlHRzt9cbPbLJdUMQr6ynmFOaFkjB72QS4YWtL2S7dLqGOmfPC54dsbaq+JmnFchrJnvDBEBf3VpsVT/qmf8Aqf8ABYRppacEvqQ72AU0NKJm5xUOHstc8jg1jBUO9UTT/wCZ/wAFFNSylhb4X/8AJZskIpyM07zfsVZiw2SrjzwVF+7eZr+ic8jgkgdVPh8KKdkrGdDJlUgp6p0bo/DQxAgj9sscse2bkvHmC224XT1eGPBuJHtORwOoKvPJz4uCmoajCcU5dRbMSSC03FkWJHl1Af8AC8XJVJzZYagifnF8ZsbjdaeKPY/D4JmebQXXWZsmDUhYWDK6N38VI+VkT/PT6Dq1ZtC78YeST9FuRChcwGaflO7OaVyWgNqIJHBscnm7OFlqURbnGcqj4TDZD/pEd1bgAYWjsu+LEoMYblcSOyoU9DLMR5d1u1GQvzvF00UsBcGZAPomRihp8HmLbNEY98wVeq4NnmmM4qmxg72WwIYSbuapc74vIx/k7KOjDh4TELg/xt7e61I6IRsDXOurZYkGfvx/qgpSspIf9Jfy2991Unh4fl1kljdb6K1izcsN7tPyKwJXqraxVnAGwk0zXFw6grDlnZm30U1S67Ss8g33RzSmo102Tmo8mxKrlo3RxGR1mRuAd0upxGtg8lLHKH+JkzdWrbM7nf6PMA/3WFRYXiEczZpILNZvY3W8JofV4eNcdjsrTtqnOtVPBHcKAUkN/wDSv4K1NNDM7zNAHYKJzqWLTJdc4ZkXKgtYEOQ+BgkbzBO0O/KVXksbvDbWUL5QNF0hhoSSxQtysjDSOoVKSbma3VSWdxFuYbdlCKiVp8jtPkrwRpw1PK9cIkHuo6qoEpvDGWjtdRc5roxpqoDI5h3W6Wzl8rdnuKTfGE5yPL3QcwpNqfySH6JS2tGVg/aC47KRtQxwzAEfNUi/MVM1wDfPsoWvNxGZotnafoiGIVMn9ZdZEkn4mZh0U0U3m3RLakb5vjebfNPJLSvZlkYdPeyp+Lt3+ijdOHHNqFLRzZNjrv3RkSNbnaNEc9IIs0heBc3AQSSGOmcGutcWXWEgVw9tyoXkByaOdjYbOvmUfrWhJnIF2bomTPvd5JNuqdrWZS0qM6OyoDuC7Uad1Ya+RsR5WgIshkpw2iL/AIuigFVIWiM+kIqxGX5MpOh3Ub2tY6zehRxyABPNHzXAg6dfdSYE0bTM5r3attZG6mY1+cbAaoBJyogNfopMzn3cTuLLAgpC81QaNQtU0nNbnYNQs6ijdnPsVtQvsQ3KT8kBRPfR0RfMfMzZYklU+V5eStHHZHtp2sY7WQ2CxLkbIJnPc5WmVDMNcGaOjkGtys8uKlY5k0Jhe0l41bZWhfZHkn50beoKgxSYmp82lwChbLPBH5j6e6qVdQ+Z4eegssUL8Na2PLy4wAB+qt/eX9Hdd4a4nZYAmk7p+a78y1xF2pq+a0R9BsVUJuh5jj6kgR0W6Q5t1S6pgTdLMELSZndks/umyhNYoiVstkfMDiqpdqkXJSrGawujjqXA7BVQ6/qNkQIvuoi+yUnXZKpEM1LkkNjfcKs1+tkbTfdZaX6aqiFEYQy3KHluLqemq44oGxSP1OqoRuABA6hE3KXBo3CzYOV45rrKnWy2cIcvmIursovbyhQmGeZ2XkNNtndUsUo5RELSMztUxdRStBjZyyE08GXRz9ewQwQgkhz2j5laAyE2zNcPqqxddyOoD45jGfolA1hks/sumLUHbEZDl6dSpQOToLn6piMvpU8IbYl7b3UmUpJh4ziSQkAMCFz4OWTFoWGyEkQAmNlj8ymfGHhh6l2yiZOuwFhETNE/HMuWkoaW2sjiR/BXMJiyNY0Ln+PJ3vxemjabmCK5Ha5W4Zc9VT5Zvw2gADZFDWCKHI6MG56dlSeHOeVLHTzSAjluszRSYa5LJmhLXMa8kdLiyAv5g87zfuom01UJG5oiG33RPcxzyGbX0WaWz1R8Q8OkN8jQAjgEAZyTcX37IIqaplkEeSzj3U89O+Joikfd7HWsBslFjbDypmtY8OZ0sp4ZGB7c9wLncbqBoLJmrTmqmVcbWVL72HlJGgKyijWOZJURuYCNdlew+O72rPeAaoWIIAABC16BtjfsCpIwcRle6vnu64DrBTUhnMY5byHE2A7hZs7i6peR1eT/ABRxukY0SMOrCtsumhY0xsa+PVw1dfZU35453Fz3AdAq8U5lhMpaA8bNKsw1fMbaqhzWsAQVxnyBGaltpGQ4bEuuijpXywOqI2kgdL6FWycAsGsojK8m175QB3QSvomvyQMka0dL3UHP1TnvdJJICHE6hFCwlzPLpfX5K7iMNP63vIz2yhVHzZfKzfuu0NBnkMNQcrvKhNQG6s3QTkSBzzuq2pK2i9JVEtby8zSBqfdQtD3gsY0l3YIWtLjZu6sRZIjf4lKpEIjI9Q1Ti7bW6I5X3eSUwl7IpAvdJne65K0I2MkYIJCRci9lRbq6/RakdYyGLLHBHzSLl5GwWMka/D8rHtkfM8uzkBgJ6ArVbLZ7mXALN7rlzXNa4GPTvYWuVA3EHxTGZ5uTuuGTtr2uoqpRFB4jIZB0Ddys2tqY66JkHMkjyXzBr7Xus4Ym+VjwDoTdDFMxxuXarFNZbUT6qOmlfymBpHlJ9lC6odN5HAvzjY6XUklPG5xMgEnYjS/zUp89rNFwLBdLc4zNE57rMc/JkFiL7BTOZFy88PnF7bKqH+fLmaPfujkqAGWz2DB0U4pfJbpIHUz2zzizv6v5KZ5ke8uky5T0aqsdQyeJmd58hvsp21HOlcy2wvdMP4mnS4p/8gpW95Gf/ulZgms4MEeW/W+iv4/KymwSjzn+sZ/+6uddi8Et4mMdp1K3sXBsMqmQn8djmk7G2igxespXQ+IizGojFg0DRwVGTFZALBwDexF1WknEsckzPVGLfr/7LlH5WMptTfI9/lfHGPm236KMxnoiLMpF9QdUVy0kDQrtDCO8jgY8rNBfy7lVXFzm3cwj2KtCZhLcjBoLZuqicc5y9TrdbxFQu9gE0Zu/KrDoYQDm9XRVmAtkXZTyDVJu6IAOdqiLWNvlOxRfUBNkGYKR2v8Amgt5g1EO3dEmAs4/up0KAtrEH5eGMKpsvrkfJ+l/81iuC3eIDkoMFhO7ICT/AAVmVY5LrWKjO6ladbIJBlcoiNuykQJ82uVFSjy+ZN6nE90un0Sb6kQVyzQNuT+icEuDYw0Xv+qEnMcp76Jg8xuuxxv3WRcjjaC18zdA4X/VRYkbYrWf+KUpah3LI9wUWNf/ADWb3DD/APgEFUGx3tdOWPI+SFFd1jmudFpEsUrdBvY3seqUoJeXvbYFQC9kYOwQSxvtoNkTjYjMBfqombp5MzTkJuR1WAbi3dHEGvDr/wCreoEdyG+rpZbVNLA2NkWQbxhxRQOtfsdwmbMSAHm9hYIqRt3ttbe+qwrVw+CtNpqQeboumpZZHgtkjLXDcFVsO5hkjc+mgdZjyLR2sdFc8XN/2Kk/iFxVMdll8S1DIMCmLhcz2iA+fVaRrD/2Cm/9QrD4tqedhQj8LHF+Kx92neykNqvBV/6d/ZZ/eVsOxChonzmonYCX+kG5XNYFIyOCTnVskDJHatZvJ2V+Gocx+bDaSNl9p3+Z5WkaLsRxCeLnUlF4SP8A1tSbfwWRW4jhWZz6uoqcTed2tOWP9LqKrjpXTlmJ4leVnSTogbFgbP8Ar0H6rQalr8KMUfMw+emey/4sEhvv2urcbZS0MoKyGvtuPTJ+h0Vfl4J/26n/AN5RmnwY6x10Qd0IdYhUpcFUxjjDUsMD2EEiQWG/dZuMvfK6Nrz5SDp7grRNRX0zDHM/xUQ+GbVZWIOonRxvpaUwOaTzGZrj6dkDYc28jW/vkrbBe1tmd7/3LEoM7I+YxmYtW1hbarFnVEcFNZ8DQWhzh+Ib7IJC9+cec2ABTFrX+Yi7r3v1Upw7Fmn/AEKPa37Yf5Jfd+MFrsmG5v7MoVthXllMTHGR4cLWaHC9lhP+a2n4Ni0rS04bMf8AzAqsnDmMjbDyPnIFJFGOSAWjfBf3zKSMiF4fC21tkddg+I4bAyqrIgxj3ZNHXN1S5uXZyxMDVfNBNRyNLLSvtZ1/dDT0jJaYPedbkFUoZLuGqvRyvAbHGbN3+qg2cDw2klpcQE9s3KGUE2vumw3DqZmCVklQY81ja0guRZZpnnAyxsDndrKtPNiYje5+GyAZTc8olajJtjLewxsc2EuYfVHJcFc+H3C1sLkbHAbtvc3Gq1aU66HDKUvpHvZ5NA7XoVRhwCrxTiR1JQU5kbHUDmOGzWX1VEGZzQ9sTSHdzqur8W/hnhdsdI0uraxoc4jT/nQrhs2cY8nTGLS1PAuBQ17vGcVRQcw35cUWYj6k/wCCpVOCfZ9BKYX8ZVeYf/0gK5gy1VW7PW2D/cqtN6ixc9fiVcl4+52FNw3wrXRiSi4uqNdhJSAfrqhqcDrsPni5jxVUcj/2sDrjTv23XGNvE4SRPLXDqFu4HjdWJWUEtTI4SfET/es5xtwi8VwnGZqXZYPhVC+qkr+URLAbCx01HZawMjHXjWfgjiG1ns5n9y0oTqvd01+HHL1TIQfUE5TslnqmCwt9QpA4bJnOv0K7sAlb4iJ0VUyOVr9w5uhVHEsMhqoOTThtO5p8j2t1aP1V9C7dG3F4xR1GEuj8RiMkznglhBOn6qjj+GNc+KuOJzHngbi+vfdbXGUtNFLTc8E3abWF+yir5aH7qp557OYy1iQvm7/LJv8AjxlyUuGNy5vFh39oW/xQQUVZC4yx0wcy1szCtzxmGEaUv/4qSkq6b8QRx5bt/LZdHJDws7GfETsoMRdRNL/MC0EE691SlwaqZUSiWW55hJLepWvgdQyWtaxgI+YUGMVNRBiVRGx9mh21lqJYU2YWxu1HJIf3tVZgpp2CwogB7hV3YlUBnlyj/ZUU2KVYgztkuDtopLUS0KWNsU7mfFaxCz6c4fRY1SvhOZxkt8kuHKjmGo5zcz8ws7tusqSJ0GI8/JbJMDf6hdXJ3XEtRUUTqV8NRLDG+UskdHqbIMErpJ6x0D62aqpgLh8jLFpCvYxRSYlBCISzNHIJdeoV0+ZaWHPYuxgr5M8j/NbS+ighpmXuYWfOyt4/DS+LbJMzzPbvdZBfODljqnNb2uuObrC1U0kjpPTH/tBV/uYS+Z7IP0KkMVWBf7xv7XUf4/WeT9VgHBQTxaQuga0dA1Suo6uQ38S0Ko7yi5mdbuCpTBBlBNY4newUQZop/inicPdt1AYeW7Zv0Fk0zBZpie8jY3cgaMh9RRVqKJgAcOYPk4oZ8PpZyHl7hrsUHMIHkKjMxy5JH3Wi2jLw/Ty0DZ4LtLDc31UDKZ40E+U/NPT10sMT4nPux4tZVpZ6fYtQte5M4AHiCoGtlppXhsrnCQXuVVhe2JwcyQht9W9CrdRVQTZXsjDSBsCpZaQTv6uS5r/hKz3VrGHW4HdWQXKtLHOefUUOdwKhDiEJmY17QXgG6NUnNRlHmSbM4oMalZaKZjGhxABss3xUg+NHNrc553KCWeqa38E2PzKyZauzvi+iXiyWZ3OzntdEtfFRinxuv7khWmTTfG43WC/Enn0CzvdOzEKh3qIupRbckdJJ8RUcFRJy3Mcdis5tbIRkLk7Z8u+ypaxK572nKmiFWyMPD2/JR3a43bIbKTmsaLB5+qlLEnixLnD0jQ6jdWW4gYAJo2EkKk+lEMfPa4Fr9dEzZY5PI5KW1rxXi5/En1HdWYppAb8xw+RVJkHhQ3K/M0nRTNfZVtYJadbMJ72T5ha2VtvkoRInDr+VGUucD4I/91M+W/leGn5hRk2PmTEgnRGqGHM/1bf0UTvWC1qZj9bI+UZDotYT5sbI8ksti0PUJ0BcFehoH1DeXzA33KhqKN1G4sdIH+4W8nPBAKl7Wi5sFBVVOYXjfftYqviEkmURsYTfsqlO2Zkg3t2K4W7rYxSQWZMwg/NTCoje3S2qpz08kj2v2t/FTwwBoBJv7LPNk8rnFqrugc4ZrLQEbJG2aEHKMBDlrxmWXLT5QbtWZNa/pXWNdSS25sdyjdhuESgXpwPkn4hHEOyrQpMJgqohIa2Nl+l1vVfDuG1DbR3Z8lWi4e5do+Y0+614y44ggjmgZyxPzGj3uhkbMSXCGS3eyux0baOzASQNrlRSy1Ebw+HZu6zybpmc5/76ESTyGwJHzC0n4jBPZr4Gtf8AmQyOgktZzbjoFpylRLZ4m6SD6KGR7yNd+q0JaR5BLHD9VnSPDXWdut0hrNPxj6qUU5acz26KEyD4VI2pldGY7+VaQ8rcumige6zVI8yFQlr72yrK0awePdIRvzbaJ2erzhTWHwqlK5icT5ipAwgWv+qN1rJi6/q3CloHzW8tk7cwN0HMPwpZr7oJeae6XMF7PcoiGOGpS5YvodVqhiuc95zPJJ90jI8tyX0TkZjZEKV5ALrBbEAB3UzdlaiogxmpCFlG9zjewb3ClpYCwgixBREFozFWGQsY/uPdQTm78nRUDmMnk6lRSxOicG9UTmHdpsjLwWgHogeIuIAO6m2Cip3Z5MwYdFYmyO2UUzQ5416I2MeXXCdk0baflujObujgdGXWFz81KE9OWt1k0HdWDU1To7UsRa0f1p/yTjw0UXMn+gVKsrJ6p2shaxuzBslNq5hfJMXvmc9wF7noq4F3H2Kttfy4X+bUqiHEOAaL3VYSCJziLbdT2VyCDJGZoYHSBptmVaPU2uB81uYawvw2YNy6E9VBjPm5pdpYnoqs7LalTEGNznlo0KGoaH0/Oy6jZBW0te6QIQlqZui0gybpiUw3THohAwW2TZggbsk7ZWyx5glnH5lHmKTd0pUoIun8vZRpIDuCE40KBu6IGyUkpgbG6fme6hzFOHfupxFsS33U9M9jnbhU2O1R5mjquc4qvvkaBfMnZMwxEX8xWc55eLNciY1g82azu6ULcNHHzCX5yT7o/Dx81zCGsyjyH3UbJbjMyQEoJZXg52uusitVMnZL+MBZ3oIUHNew2AHzsrNW4lzXE+oKu6F9g62p1C6QDgkDnWd2RtkDHAqsx+VylAu3OjK1K+5upMOPOrIo76XuqYcXblanD0EclfcjYFaJd1hMV3MB7LheKarxePVcjXXaH5B8gvQcPAjjdMdo4yV5RU1DJamaZmz5CQqmJMY4vsAMx7laVXJeMCNxJOjiFiOlc/dXoZi4hh3so2eoqJIY+Wx45ux9goYWsDCH9U88jJpA2MXd1PdG2lkLmmSRgA6dSoNKGntT84u8wbe6qlzS8vurfPzR8hpszss+WNkMpNyAeu6xLK+2SN8erAHf3oZc5IazYKCOQZGNcTcnorADnMcbm1tCq0jYOZULWik5FHJLtYFZ1M28gJ+qt18vIw2Q6avA1WZHOwNvMc3cqR7Q1pZ73WtPS0EEDPFU5D5Rna4dlQMcHwAlaiQELmsHmIB6XUzZJGuLiDltvZPGIWexO9xdStySGzOo/gsTPJKSxyU7RldUS5ehDU8EsbqhwznKBvZBymgZGbBEWRQU5y+p+t1mkPVQsrJAyScMaD5brK5TyTYL1nBMJwWqw6mZVYdDLK1vnc4arhOIGwQYxUQ0sEcTBazWbLtiQwRC+RpAb80JgLY8+a56hWrxtBAeGE7piw8kOb1NlbDTUT6WCnqHPBbUXtbpZV5TY+y2OJWOihw+Af1cZJHZZEkb3wsYwXc82S1si3M0OQ5Lbr1eDD6GrxCOPkRCmw+LzMEYsZCN7+y82xKoZU4pVTGMR3dYBo0sEVVivZ2u6twmEkh4BBHVUwCBodSdkQLm7qSyt2b1UzcJrquldXQ0shpmG2cDS6oueeU5dLPLLM/BcGhlvSwNie5oOhJOpK5ZQsMaOinpWNE0Ul3i4OXcIBLCPjXokmLUWFS1A5jXvf8As2AXAJXDPw+OWV8kjhnkeXmwta6iz5KrZGH47obVJecrgQRcEq43Co8oawgfmvu5DJhsrmgc7KBoSClsqMRe8EyjMNxZT5A6zeWCSbC6eodDCRT0r8wYLAAJqWR7J43WLix1xba601EpocNxVj878LqRH2yqTD3vllN4Syxtr1XQVPE1XLC+FgDM4tcBZVM28oUpm2nxox78HoGskDXCUGx6+RcjTQ4lMXcigkqgw2dyoyQD9F1/GUj46KiA9/7lNi+MVWCUOG4Hg9RJSsipmTTuYbGWR4+Pupsyp0hxc0jXyCTlAW0sNDf3Vilp3yUD65s17OyTMtr81pY7GBikmbqxjz8yNVnxOdFzWtd5ZW5SEZSxYFjFXDFNSUEksRbo7a6z5s8c7o3ts4bhdPS8WHDqGDD46cFkDcpud/4Lmql8c87pXO8z910xZQZrFA5wIspJGC4LXjXTVJlO1+YvvcaDstqrZgmdup5KaRozZC8XsMuqjc18YvJG4D3C2FT0NVW5/DMB5Yu65tYJpqeamjDZLa7EG4K2uFYWTOq8/pLWAj9VR4gZ4evZTg3aGAj6oqg3f0pPYWuBYEwkyna6RkuBbREHf9U6hBOYXUpb+VCgEXW3xW209DD/AKumCxA1zp44fzyALouK2MOJuYW3yUrAP1KkkOdvrdO4366qIGylYx8tgyNxN7bKgEHxqY01VGM8lLKAOtkBY613NIHuEtRqanp3VDnAOAyC6huO6u4UPPKf3QgqkZDb+Kdu6uT0kkz80cJOlz2QCgrRb+jON9rLKKU2bIVbxjXEC/o+KMj9FDLFO71xuHTYqbE2ESQE9aWP+5BUB1upHGNzP3h/FR/u9UgySxfyzYbrQvYfhT8QY97JMoYbbXVxvD8zL/0mM2JGy0OEoebhsrz1lsr4bcO/tuWMhiwYFPUSiCOOSSQ7Bjb/AK6qTF+D5sFpzV4jilJH2jBu9x9h1XQYbW12FVPPpJPI8ZJYztIPn0sq/E1LQcQ2qvCSU1Wyw5hfmBHYhZiVcxSYLiOIxvqcNopqiKMXzNbYEfXf5KGnpZ62dtLA2MSncPkDbH6rq8L4jxbhqg+7msgqo4h+Cdi3591y80wxHF3YljrHVHNJMrYhyye1rLcScVnFeHcZwaNk+I0gjicbZ2uDgPnbZRYfHzZmMGlzZdZiPGkk+HxUGG0PhWyRaGWzrMtYWCy8Dw3+kCfJoOi55I6CgpH0sQLqh8pykDNbS6IYdJUuAZWyQf2QDdWmR2hj+iOHym68/J1xVBglZnyDHZBoTsFFX8Kz1sHh6nHJJG3BH4YNitVh/FH9ko3S6pyac7/0f07mCOXGMw7cpXYOHIMOiipIZ+aLPINvktQSeYFKQ3qovk//AAWuYxK3gKhxCrfWSYjNE59tGxi2gsoh9nWHj/6zUf8ApNXUuNmEqPPd7R3CchzEn2eYa3fE6h3+yE7fs6wh1/8A4jV6C/7MLfxGrFHAZAdeiyJsXlmv/Sst+jDos+JlAt/zaovjll+hsoP5g8MOF3eLLv8AxigpMTNNzfxy4AAjOfIrEXFELb54o5PkCFnxMhB/NfB8OrqXDoGSmKvBEmeS5Fuys/c2D4BVxsw2WbNLIGPEjr2CCoxeCtqqGvgsBRklzSbE36BUMQrjiVcapzMri0MIB00WueQ6KXDqV7iZq7lH8tkwo6Rvn+8hoLWAtdc5Flp7l7JznFi7mXA+QKqupMPzEAzSP7ul3/RdORxdOMKOfm+IJv0IUsGHPjJLJ7X/AHVUHEUb2CPwxjezqX6D+ClpMYzv/GbGG92vuseIzxV8Tw2CufNS1rOayKLmjob20VLC+DsCxDBqWeopnGWVudz2yEFFiONHxMktO3SWHler+KGh4ifRU1PQx0oLYm5M2fT+5PEOKQ8BYPDHmg55yAk5n3usyXD8NcyWSaHyR7R5tbhXavF6qeoFXHVmFsTf2fQpT4s6ohc+aojF/hA3WPEOKlh2eGendRU7zLPuwG9vquvMALSx8ssjZG6h0htYrlYsYlMXLppIo3sABOXX6I2Y7XNa6Oed0gP5Rr9E5I1xwtw1fXBqb9CilwThWnuz7rgbb4bFZLMdnFM6JkhGfcvb57KsZ4ZbscdTuepTm2hqRTR1EkcAs291uYxXQYcMNqp6YzxPifGA02N1zk01IA6NjJMx6uK3MYdRVeC0LHm5Y7XTbRXZl6LqUKjiKKoovCyYPEQHZ2vG41uFl4xjL8WjZHJFEx7DcPY2xPzV6GSmjzRxxABhtve6pYtQxzuGIUcYabfisb19wrhsZnFlAvtZ26Ome+GqZM8XN7KEzRh2dpPyKeM81wcHjdehzqb8ne4XjjMFlkbUxySxzuZmdf06LraWuoK2IT0E4lj623B7Fc9w/DS1T62mq2RuZNFGbOPt0WrhGD0WDtkjpJjIJDexcDb/AJuumE8XfOKagliADi+10RczLpKB80mRk9P8U5jLPhFu1l25OQRy+srb/NQ184paOWqjjE74hdrAd1K5sJ3jb+iZjWtOZjAPkotOAxKvxTEKhjsSpYoHAWja5pBsocOqGTxupHy2jYbbXV/7QT/8apf/AAVz8Plhe5vqzmwXj24/mdtXnbcqocLysa+RznC/mawa/NVzS0NPIJBVsy9eiz4/FNbZ4/immiEjMrwuXFxpYw+ugoattQXh2ToDunxKtgxLEZKiO7WyW0PyWeKGe+keYe2qEh0ZaO62wslrMgY4690eWiZG0Fjnm1zroq4pKl+jpGua8aEFNLFWs2gegKKWKkky0sWUPOtkctSx0ZLWg23uqdqqYhkcfnv3UbWVXOdHI0jTVbSnptJLzaGlf3iUVNHOwu50gdmNxYbKlhpfPw35nvzMhIZbQ3HurGEy87DaeQvzEt1PuuozOLY3WpZB3IWA2N/V36FdJxPGTQNkAvy5AVyjp3slEb4yAe655w3EroZyx5n3HzTSTgbOWfLMyJ4YWStJGgOyFz5Ro95y9QFmmV5tW343NsglrWDWw+TVmk2cc4fbpdAY5pHjkwuI7hXijTFVmbcXHsVC6qmacztu6p53seWPJIRyPfl307pxOK3HXuvlLh9U0tbDI6z5CCOyxpLwyiS9zfW3VXIyKs3jAbbQ2KcRa58sVjnJHS6bxYLru3UUXLzGOSQWBscyeWKnhcLWIPUFKVP42w9k8c5l1jePqoPwI28zO0j8p3SZVYcRmzEfIpQ0C2OVmVynhlbkDA8G3ZZUddHEToH37rSoGVNe9rKOkDnv+EED+9SYbhPI/IwvWNUSmSXO4rfq+HOJXsyMwiY37EFZcnDfE0XqwOqd8gFaaylSlqJpLNe69kmuJVkYFxE//wCiTg/vWCZnDvEh3wmQf7aUzSsY5Rq94PyQeVzshdZXv5rcTf8AYv8A8wjHCHEr96WIf7aUcZZwa0KQODeiu/zO4i7w/wC8f8kTODMfO74m/MlKOEs10VTI+8YUgZVMHnAWxDwjjrG5RLF9LlTjhDFJGFkzg/5ApZ4WTDbLfqmM1nE836WW/FwZiDXAtFx2ylNLwDNI/O900d+g0C58zwslaRwdhg+QWVE9zpW5V1cPCGIwQGna5z2kWF+ijHAOLjzRmEf2n2KzzdOEs4l/hWgstkKFtVENCVsycI442ldGZIDtrzFnUfC9XiLHFldStLDZzXO1BViUmEPPjtfmt/VHFPH3Un/R/Vg3OLUTfm5W4eDKlrAz76o9FbZUZZo/zKB9Sw+XMtCo4Zmp3ZH4rTu/si/+KgHD0R1kr5M3drdEoUBLIHh2ZWoauT8ykOB29GJuPziCtS01PFT+SPzj4rrpjDmlw7EDE6zxdFXVsdQTH4Utt1JWXHmv5VPG/IQSuksY/lRuaSdFI2iD2czQO91ZkqKXJ/ogDu4WTVYo5t2xssOxK8WV35PRayZKYm0gvbZV5SwO8mgWJNiDuYS16Dx73+qVSNUs21+afz/ollLxmc8rOiq76F4HzSlrXNbkDlZ1zDC8XPYbgq1BUFwAzj6rn5a97gbPR0tY4uDCVjw5WHRVM8oZla0/RDDMXWuD9VDC6Zwy89tuhO6N3iAMj3AjoQrEOl0VcJ+XzI7W91l+LedM5R1j3EkF58qoE21K7Y4szKWY5tUIvbVNFIQ8aAj3ViSBjxdjAHHqu1JStI6QXIdp7Kq8m+qkL5GkiT9FG/UKsCZmcPI4X7FFHMIyWO69VDZwdtogc7OVoxacM+YgEXapiGvGUjTosqKWRg3B+itMqPLd7yuc4qndGwO8oQOsNkLpwXbmyXNZ3KzjErKF0hH1QZz+ZSyEHVQkjstsnuU+cnRDp7JsoWwiT8KJrtbZAULd1I1uma4UGY0WeFPEQ59nIA2+26mjAjs8x+b3WkTZHkXynL3Q5JpHBkZs13VIATkPkkNh0CuRVEFOwhjQSehWUDU4dHHTGSOp16hyzDGtbDqsz1/IeGFp7q5i9IwQZ2Qjy9tFmJPRzguBqgIzOyqwGZzYtIUMuSN9guipIw6K+trqWKMyn27qoMxdmLtFeprviLY3dN1FDK1rH5QSUopHxSXsoS2YnM9qeme503LOqDSzRVEAfJoPyoHUk0jea8BvsSjgi5RzSMuB0UddU+U5z8giWjnLI2hjAwnr1VERiNxf+YqSAGeUMYrRo2EWPq6IqmyX8QXWrw3UvdUTwPPkedugWRPC+J9i1WKKZlEzmM0cVBJX2ZUOB9JOq1cZipBhkUsEbQT2WNVO515O6uyS87BofL6FgY80YGoUXXVaeIsh8NFPYgkXIassai66QiQZBdyA5ScyNuT4nJjbNmCoEaFC83Unrdqd0pY2s2eD8lREB3SzFMm83dRR5gkXIQeoRf7KAm7osjg26eIsuWuAUudw2W0VkYPUJnDzJN2WSEma6duyUbGluu6lZASLh4UD+UtsCnBsonMc11sh+adxBCTAma+3YKQBpF2OuFWYr0MWgFwucqrVEM1jJbRGZAWNjkAuB2VuZmcjsOigmg5v4jyAVYklWmhY4h7GeVDZoFtgpBAbXjuSegKeGJzx52G61bKD4iFv8MMvLI73t/eswUuvmaui4cpmRwkjcvKsSS2sRqPCYFVyZrOMdh9V5e2GRzNB1XpXEzcuDCADWdwYFxfgw30BTmQyeUR6gk9rwtjwvlJLNR0Vc0rypyaVIgchIGoUtJKWF0eX16k9VOKZ8TrH9FLCxjZhIWhamSYVjK7mk2O6d84Ny8eX8oUzqQOc54fb2so3UUh0kYQeyzcAIZvEP87wMmiuOL4wYM4IRQUUMUTWvbqDqU0ts2inKBNSNupa6aGKKOOYXbIbJqfyhBisHNjpn5HOyHopIznVcs/7R5dba52CZr1o0lGxrMz27qxDQ0r8149jZTkrDEheVdgb+FzNQ8dCFbqcPg5ZMDfMjlhicW6EeUbFOQgBvZj3aHsopXRhouBuPfRafhYvCkBl3O63WVPRytzF3fZTHKJTi6+m47wqlDniCckR2bpbVcjW4iKyrkqyzJzDfLe9lU8NMdgo8tjlcukSJRODsxp+YVimdzJGQZAA+QdFWAa0XsreGWkxGnb0uT+gWkpPxTLmrWN/JGAslspbkeN4zcfNauMxxy4rK2TYBg/gqnJw+NxZywT3IuU5Dt8M4iw3D8EfDPURuq5QZJDmtmeRb/BcG9xBzNU8kDspfBA+49kzY2Hyvb5uoTkKxmk7gfQJCU33B+isy4dPIbxAFpTtw/w7CauwcFLFqmpoZWgvYL/JdHgnDtRjFKX0lRGHxmzmvFgD8/8Aguaw6SaSsjYz9je5K1gHjaaRv9l1lyk5q+IxSUBc/wARHKbll2XsCFXinbYPedVJi2flwUoZoGGU67/82VERPJJOgUpInku+JlJIY/K23XqoZ6lzGHzg3VF8l3ljD89CmpoA53OqnkMvYNG5WuK0CJskso5YJV6GGeEm7wB2tsiY5kdmQQuA9hslHOZT5IXfNx0KWqzzWeku16KzR+d6zZJZr2EYNhc2Ct4eMSc8GiZTmxFzKUj+Jlq8amwoGfM/3LNx6qZLibiPgjjj/grPELsUfXYbBinhdbWEXu8DVZ+KtzcRVFL8HNYz+5Y2OkJMandUYpNI1lrZGfoAqJBPUD5pY/UPhxytjj0aJf8AAKh44385XRlZeXAbNKhzNO6KOfOPMwN+Shmc1jt91uGSeboo6hrT5mg6KLMChIscqqteklhNBKeZ54zzMttxqtXieiYcFj837KGORum1zYrlmSva4sYbZxlPyXTT43DPC2mnpxIzksgcC4bhS2hcNQsjw0zRlpbJaxBudO6wOJ3Xxlw7RMH8Fq4VU0eFU74Iy/K831OygmjoKuqfPWkOfMbjKdALaKc04S512yQFlvCkwWLV7I3XNvM+9vkEjQ4LL5QGRuPpcHFOcNUxAzzNN+qsyZcxcOpV4UuGQMa6RkrnX6O0KkNPhclnN5rb9Cd05wUzKWPmV0Df+9Z/etbiCpZJi9U1mtgIvlZTUNBQSVVNJDH+KJWaf8/JFiEWHTYhVve0h5mfexTnBTHpKCxEj/oFeALfKx2VWY6GbmDkwSvjtYFrCQtHBOHa3GqnwFPRTPna0uOa8YsPcrnluxiLylx2btWnGc9mUREetyoRVIEZY+xI3JRCohtlkYSL31GgXTxfZPxW+8rqOBh1FuaP70oPst4tfmY6npWOYLhvPGy4R1mifTOPu8Ud57dP/r4/eHIT4dhU935JGvPUHT9FJSYdDDm8PLe41zFdb/0VcUStD5PDx/KYaoT9lPFmQsayN3uJwU/F6f54+6/5x27/AHsfvDnGU9jrOLjcAp5BJEzMLEDddPTfZVxnH5JMNjIsbkyG5Rj7M+O2zAfduaEbNY46qfjen/nj7sT3ntv+/j/3Q48VAJ9SkrKejn5HiI812m2u2q7E/ZFxu4ufTYIc7zcZrABKf7I/tC5EbYcFcXGMi4ubalPxvT/zx90/znt3+/j/AN0OL8HTYewciPl9tUPiZfY/MLs4/se+0eSECTAJfbdPH9jP2qRuvHgbhF2c4AlPxvT/AM8fc/znt3+/j/3Q42HEZaVhhZljYTcgDcqaKukdrYWXUVP2KfaPIM8fDshtv5wpqf7GftJLMruHHg9uYNU/GdP/ADx9z/Oe3f7+P/dDApZs7Pkgq5nstl0HUrqqf7IftFju0YI0X7zBTzfY39okkJb9104d0Bq2BI6vR/PDX+c9v/3sfvDzqQyG+bVVXUj5X5wF6HH9in2mZwZsOoXN7eNYFHXfYx9oFDSPrZMMgc1lgWQ1bXPP0C1j1enKajKHTDu/QbMoxx3YzM/OHJiASRw+XURABbODkZAzqw2K1qL7MeJJreKf4d8QAMLIXSkfwC1I/s7rIW5ZJsR/2aYt/wAFuc4y97388PjDLf6Whu9mWH6pQ/s2u+f962GcBNGrziTz3yOH+Css4MiYwM8LiBt+67/JYmY+Jzw+MOffo5j+90V87QVtS8HymN3h6OrDwPLnDrErn3YTxxTuMbaCllDTYnMLFSPP0dsOOXpKcGyQfnqYi70i9yqToONW/tMDhPyQ/wD6Wx+vh+/+wf8ANWnXw8fi0pJt2LA4nxr7tpmxQyZZpNj2V7NxXf8A/VgH6FQ1MWNTOz1nBvNPfKSlnh4+5yc2KudRHnzyyuf1MgFlSjrXva2RmmY3vfddfJTzlpceBxp/3R/yTx4dXnLMzgB9vhcIHaf/AIrVpOuI9ZczFiM4cWOeC0iwA6IaevlqahsDmNDn7uG66wYZXv8A/wBgZP8A0Xf/APKJuBVs72xt+z+a7jYWgcpxhJjCPOZctDVSMqOW94kPSxUpq2RyykSWLmEA+67ofZ9xA30cAyD6FN/0ecSk/wD6jH6grXk8X4zo/wDex+8OAjxEm/rdbrbQKOKp5k9gd16Q37MuMz+z4Aj+t1J/0X8bnU/Z7FfuAU/KfjOj/wB7H7w83mqQWtbIfQLD5KuKu0Zjjkk1N75ivTm/ZPxuf/2Dg/3z/mpf+i3j74OBqQfRn+afkPxnR/72P3h5hFVSBmS9/c7oJ62RzeRnIt1BNyvUP+iz7QgdeB6T/dZ/gUR+yj7Qspkbwdh4DRc5iA7+9OOJ+L6Sf/Vx+8PLoq4+QOmPkFvMbqTnReq4+gK7iPhHjV7iP5uUzfZzWBa2F/Z79pGIudFR4Xh8eQXIeQ3+5KejPw9ePLPOIj6vNoXNe/nMMhd2ypTRVszs8ccrOxMRXtmHfZHx4WiXERQRua70MlbYj9Fvs+zjHidKWgj+Tgo+T1HfOi0ZccZnL6PneOkxiTWSmmce4aVYhw3F5DduF1ZI2DRuve8Q+yLievawU+JU1Jl35Z3WPX/Ynx5FG11DxJFK8nUPJAA/RYb0d67fuqJz4zPx/v6PHzw9j00ZlGFzMf0zWA/vWl9119bQMoqewfCzzlx0vay6er4G4wo3yR1mPU7TF6xrt+iz8O4MxHEIOfHjjI9bEWJVp93XjriLifJz8XA/EJu41MEZ+d1oU3CmIxMyVOI0wd1IabldCPs5nezPJxFUA/uRE/4qNnAOH/1vEdVIe/LTgz+Rlt4SoGuElTiMeb2Yyx/VWnYDgDKd9oqVzww+azArzeAMA/rcVqXjtoFMzgDhQ/8AW6gfOUf5J5t8tbBo8PpbcyvcAzLYXPXutCiocCqJmwRPY5zjbQuVnBcNw/EpJqfEWcxkWoHvey7/AIJ4RwCGofXSYaOUWloLrkkrGObzdx6zX0fT5bffHp85chJwnDE1zm1bgGi/keb/AN6wZ8SMTnU9NW1zZgbAFxH96+gm4Tw9GfwMNZf3jJXN4vwdiNVik1VRYZTZHm7XENC7RL8z2v2gjfsyw6qYiK8p9GPg/wBm3EL4KatqeKTzD+IYzE1zSOi33cG4y6J0YxeiGYWv4QXH8F1PDuF4ozDmQYg4CZpPm3AHRbLMFqHf9YZ9ASniZvzPW9x6jZvmZ2XU+VT5fo8L4g+wnFcZPif5ywPqGaND4Cxob9F5+/7PMdpsefwrLW0bakOyiS5aw/VfWc2DMZFI51blIaSMrPZfPmMCdv2gubWSGZz3guNtTopnlb9N7P8AX9V1fONmdxHx9bn/AIYg+w7jYm1JiuE1BO0bKoZnfwVOr+yX7QcP8tXg74/fM0j9V7BS4fR4VW01ZLNA1rXB+YS3Nl0lRx7Exr46eLxTCLBrxorbydx73v6XOMdOzHO/l6fu8CwT7OOJvFtjximkjpnA5jC7zey6B32XYS6/M+8fqGrrzLLK98gbkD3E5QdAna+QepxXLPVOc3ymPo+Lv7x12/LlGycflHlH/LyCp+y7i8zyCGGMxZjkLpW3t0Usf2YcaOi5Zhp/rM0L1x0zvzFE2V1tHFdfN7I9outiKuPs8g/6LuL2nSGkH/nhSj7MeMXaztoie4lAXrlyfUUnfVb82f8AxJ1vy+zzuj4SxvBsGfHXNjdkJaTE7MDuuco48WwuLwcNfh7owSbO3XtLz/8ADKsO6xvK8LpqSI4jV00+FmotOfxAfSPkumM2/Q9m6/b3DXlO2IuJ9zYxmHx2GzQB9nPtYqjhn2Z4zilGypp6+AtuWtLyQdFuCjlnpZqhmXJF6rusVt8CzzvmnpXSuMTGBzWdASVMno7n1Ozp+ny26Ji8au3OUX2Q4jLMRiuKxMjtpkaSbrVZ9j9EyPlvxud3Yhg0/Vd+2JEY9N1jzfjdne+u2ZcudfR50/7HcLe0sfi1S9p6ctquYb9luFYeyoZ46eXxEfLvI1vk9x7ruA32QlqkxyipefPunWbMeOWyaefj7HsD/rq+pJ7+UJnfY7gDj5q6st2DgF6DkKWQrdy6/wCb9d/uy89P2LcMu1fU1p/2wuuouFeE6OBsUfDkDrNDcz3XJt1WqGIhHdc9mvHb5ZPJ1HWdR1URG3OZr51/Ri1XCfC08rZBw9SMsLWAJBUTeEuHWm8eD07D7N0W+YwSmbCCpjjGEcYYx3bcYqMp+8sOPhPAI3CQYRR5hrcxqdvDvDReXz8O0D3d+XZa3IalyR+ULVtfiN388/eWYeH+EiMv826dv9hrUdLgHDNLUMnhpmxZPgMIsVo8sflS5Z7KTFxUsz1O2Yqcp+8gq8OwqqYwQ1VJTFpuHGJwBR0OEVNK509LXYVV6WALiAUhHbomyLEa8Yx4R6Mxu2Rjx5TX1lYml4p5DgzAcPfcEXYAQFi0tDjtLTtbNhEUlySZH5brUjdNC7PDIWu7hXYcZxKN4dJUc1vVr2ghMNeOv+FrDqNuvGcccpqfnLBnwyrrGgSUUEJ6kOtdZWJ8FT1d5o8S5DmMNg19m3916BQ0WC4/icdGI5IKmodlbmOVt/7S8/4zpuJKLFanD6aaIspnOheI5WeZajZjz8O/Orr5Pq9qw63qtt6c5ivWZlxVVT4vSzPppMViuw2JEgIKh5VW5tjjhB7ZrhItLSQ8WI0OiVh3XWn9AxyyiIiZ8zxUr4x5sYdm7gmyNrHg+fGJXDsAobhSiCZ4GSKQn2ai8pSc90PmixCoH0VKqrK6e4krpnA9MysT0Nc6IllJL/uLl6h9fhs8jaqPI15uDfquOyHo1y1ebiDGhjal2UdL7IJKicNJ58zi3ccwhZLcbmBsGhS1U9TUxianjsfy3/xXCm7btBLRVEIfIypzX1PMKjpIqVtZM2RkmXMbByyMCra9l+dCRrYX0V6SflzmTKvRDllFteU4XC3P4Qu+Tyqc9dQsu9kQjb+U6pqSr57smdjD0DuqtPpYpPLU8lw7Ly7Nk3UOnBVpMSpKh2Tlxn6K4RHJswAeyyKnDRBPehGVh+q0aarfCyz6Tm/Vb15uc60op47bIZIIzE/TojlrahxAgo4m276phXYiLMMdMPfLZeiNkOXh5MVtmaHcFSO1bdqinjeJS3KbnWwUkUcxZ6V3jZEwxOKeXlspc7zYLAqKV9WS+AXLui2zFMRYxl3s3VS0+HYnUycuio5vkW2XKWohw9RQ10chz0sig5E4P7J9/kvTm8IcQT2ZNDC1p+JxvZW4OA6i5E1ZCPcMVjJqnlggqh64JB9EMxLNHNI+YXsUXA9JtNXTPHUABW4+C8AYQZKd81ujyCniFPCXv7KWmkYHXNx8wvdm8I8Nj/6ZEjbwzw0w64TTC3cJyKeLRSwO1c65UvNqhpC6QN7csle2NouHIdYaGjjcOojCPmYQw6Mox9AsyPCpGVD/AC8uUuO14yEzMMxKX9nSSFe24jU8PzU5jrZaa3SwF1yHjMBwhz5JOZOwm4deyRklONi4X4gkaHNw2Vl+rtAtCLgPiWVgcxkIv0MllvRfaHw61/h46KdoHU2sqdd9oDrGKhpYYx0cHarXKhzNZw3jNDLyaukynuHXBVf7te0/0mXkjumxLiLF8TmvUVTso6AqkWuPre4/MrqL89FhscZezFMxHRZzxGD+GSR3Klja38ouk8AiyjKsJCHWNlYErbWCgMLw6+4Rlv5UmLBF4CQeLerRMI3OCcRj0qUHztPVObIC1rTq4oc7R6SSlILrunQF9/hSzm17D5q0JMxSDvdQ3KVylCHMWnROXvI1cpHQxuObUITExpy7qhxJljTtdfzP0Q3j9J37Ii0W82yjJRSGKoE8Z1C3qfGaWsifS1XkF7X3uufJgHoTxtYHZiLrNCzUPhjDmQAEdCqYez42ElWH5Sbt0ChkH5FoDvorUEvLgEEbCT3VZtPPK7QH6KxFHK25Y06I0O/4ZPVPhcH9K50jLqBwtJnK0sIZ4gyZJg3li5v1QWXnny+mw6hYFdMZqp7Wt8oNh7rop5GUsMkzt2N2XMkcyQ32JJUxZho0EJpo/EO9R2SqXWbfqmjmeXAv/YsFrITUskeXZLt6JTQ8U0ghmtqRYrObnllZDG25ebWWhWSsqKFsYPnYb2UWECGnmfV1Js6IXb7qg8VhbRsjpg78U7t7KxA7w2HiCbS6zM01fiJqLaHYdgrtWyaWQxt1A2CgnkojUwtaZwIR2KaKohphyeVHy/caq7JRz0lKIzGTcXuFiYiHMsO65Aq99LL54GWWcjcS9qisQvRiHLkKIxmxPZCtJdDa7VIlpCAalGIn2zWQBoAnzFO6M/CpqekmqCQzooqDr9UWZ4VrwsbYs4NyFFkulgQeoRtLb6NQ5eyTWvNrMJB620WbS0ociyy5c4jdbv0TZNLqdpklLIzI7LsiK4ZIfxHbKRsbTupZY+W7IXaNGnuga6yltJYWsb63IpJLOsDooHuubdimL9VmRdbLf1O1TSSF5sq4dYe6ONwzeZZiBK1z2utYhWA2wDlA2Rx3R59FlUjpBoF0ODsIgZ5d1yziTLEB+ZdrhkIEbW9l0wZlT4uku6mpzsBcrnjI1gWnxdWNkxYxs2jiYLLnnzkFMiFrneW2clRGQDfuqhqXfE5MKi+5WaLWZJWE3uhE2Xoqhm3CHmnurSL4qCNe3RSiYE53O+SzGy6hSiQXUpbaAeSd7oXl3NFmoInX83VG0Oc/Pm0SmlyNtzYKzLK8zcvygMtuq0J84VmaeBnr/wDdZyAnli5dugNYG+TPcfJRSzw5S7P8gs19T5zZcqGr4rzb+WyaKcOPnt7WWaKhh9eo7JxUW6pxkbHibjyNsBpZMSx7vPsVntkLhmJNlaimAALQD80qmll0EBFo4XE+yoyYbVE3joKlx7sC16OofHaWB+Q9wFpsrq94szE3RDsQt4yU42TDK/4KGc/+WVPhuHYvFXwTfdc5aDY6d11LjVO9eLE/JqjdEXftMXqB/YFl3tKc7iGGY9LVyzfdczmvcbW10U9NQ44xloMBee5dHqtzwlMB/plaf9pE2GC2Tn1f+8f81jJriyBFxHlLJMKm19k33Vi84GfDZbjYnSy2mnD4m2FRiH0kCrVM9EG3jlxCQ9iSFhqlA4VxBDaOOiLfcSgFGcAxadjTLGM3aSYaKcVkbDZjawDsFciET2B75Jxf8xW2eLHGD4lhxE1RJBy72sx9yE7i5XnQQ1L3EYlDAAbZZTcphhTz5vH0jm9+ZukuGS63gXiTGWU+LULaWKF0QEbpZrEgdbWU0f2W8TuZzpMQofkCT/goon1LYmQR1U+SMWaBLYAKpWT44yZngWzy2F5PxToCo3gxw6ucCIMPkkIJF2RXChjosWazIzB6o/Ntl08Ms/Ka17+W7qL2Q1b6gwEMeSdzdZVz4oMedthczfmFFLhWOFtnYbMfkFvQvqQzPM8HtZWIah4u54JA7BG6YdDhGLVUuSoj8Izq+bZaVBhhwuUwOrY6nOb3YCAFbfWUp9eGxn/ZP+aGCehdUh7I+S3s2NbiWJharMNgxHEqSrnqnQup7ZWgXDgDfU9FBV4BhhrnYiyrmc4yCTlOAsrE+IYcS3k1MLtLHOw6FReIpD/1mi/3SplhbcK03B2B4pNJX1GJTxOlcXlrALAqq7g7heFxBxPEHW7NC6CgwqqxK/gpMOkyi9hLrb5WQ1nC/FTjy4MNpBbc81qszxamWA7hvhRvpmxJ31YFGOH+F7/s8Rd83hdNDw7xJC0CTC6Rx/dcCrEPD/Fk8ogpcFZJIdmtYCVObOWUYxc+jl28P8KAeanqwP3pP+KmGAcJn9nQuf8AOZdT/NDj/wD/AIYm/wDQas+swzjWjqfBO4ZndUZcxY2nB07qc3GOr0T6Zx94ZIwXABtg7T/5oUrcPwNvl+6IQe11lV9ZitFJyK6CphkHqa+Eiyzzib43PeyZzXvNyS0hY55PTGUTFw6plNw8HiN+Gxa7gOITSu4eE/IocJidb1OfqL+y4t2KAuLZ5hIe4VqLEKV1Nka92X8o0WfzuluopazA5onSQ4XTuLDbKGK03FcDjGafhegt3IuuSp5yHZIH8s9PPZXHYm6B/Lke2fsCNT9Vx45wW6WHihlO8Oo8GoBF0BjvZT41X4RxE+lFTh7IGwFwu2PobdrdlzvioKiAhmdr8ucENvYqZlewxNsxw+eiuvL8zjnNu3wPhPguQGWCmiqZWStbndduVar+D+CXvLv5twSOO5de5K83ZVyslpxG+UZpMxyabBTU2P1FLVRywVL+aDduZxI/Req35vqu1dXszyzw6ifp/wDr+z2HCW0GBUoosMwWjihDi6xjB1KvNx2RpuMOoi783IaF5THxxj0l8k8RtofIpI+NscZI0ySRloIzAM3C5z02nKbyxi/o+Ds9meq2zOefGZn5+v7PWf50YjlLG09MGnTSIKia5xcXmnguf3AufpOKcKqYY5H1ccT3j9m46g9lrNcHAEbEXTDTr1/wYxD4mfSZdLNZYcf0WPHzDaKMHsGBO3F62NwcwRtINx5FXSyey3xiWKhr/wA8+If+0s/9NqH+eHEfStAHbI1ZOUIspXH8Lo/kj7Q5fh9X8sfZqfzv4j/+5v8Ao1qX87eInb4nIsvKUspT8Lo/kj7H4fV/LH2an86Mf/8AuEqY8T484/6fJ/vFZ17ap8wV/CaP5I+x+H1fyx9mh/OTG+tfJt3KY8R404Fhr5tRbcqhf2Sv7J+E0/yR9oTwNX8sGdJM4kyTSG/d7kiZT6pHEe5T/QpfQrrUO1QG8vwuP6p6abEKKpjraKqMNREbteNU+b2P6J2/VZnCJipgmInyl1eB/adxDhRea+jpcQB9JdEGlq6GP7aqf/rXCQt3DgV5smyheXZ27p85ucXmy6bVlNzD1IfbJgErLzcMTNPYRtKZv2t8HvP42B1EfzhC8uy+6drfdT/K+m/l/eU/B6fh+8vVB9q/A7h/8rm+sQWA/F/sfq6oy1GAzgyvzPdl77lcZywUuU1ax7fp13wuP1l6dOM9Nc6cpxv4TMf8u4rar7EaeEvpsHnmff0tYVRGO/ZNH6eFK3/dP+a5XlO+AJ+Ueq1h0eOMVOWU/rLtj1HUYxU7s5//ACn+7qxjv2Un/wDZes+rT/mqdXiv2fSyB1JgdXC23pyk6/qsLlNQ5GhdcNGOE3Ez95l1w6zqMJuNuX/dP92p948KA/8AyWqt/ZC2KfjrB6akjoYsJqRDDfI2zdFytgla66zjE+pt6vduiI2ZzNfGZdX/AD5wb4sKqR/sNS/nphO/hawf7DVymRvdLI3ulOPiS6k8dUfSmrz+iidx4y/4eFV8n/mWXNXbGn58P5kodH/P4/8A2Wq+sgT/AM/T/wDZp/8AfH+S5l00TviXS4NwNDj0rKXDsfjkqXgu5TIyDce91zzyw1YznnNRHrM+hETM1ATxvP8ABhDvrKo3ca4gfRhEJ/tTf8FuYl9kGO4NRvqqueQmP1MjjDrLmX4PBGcs1dPG7s6BctO7X1GPLVNx8mMZjOZr3eSyeNsSH/0em/8AW/4IXccYr8GGwD/aKhGDUJHnxSU/KJEMHw1v/X5z/wCUF2t18PIA4sxTmmbwNHqLWF1M3jTGW+mipx8guIx7EH0OKzUtPiUgjYdLwNKpNx2pI/0+Y/KBqt4vvavZrrt2uNmMxUxfrP8AZ6H/ADzxw/8AVqcj3al/O3Fyc3hYB8gvO3Y9Pe33jKD/AOC1L76q+mLyf+iEvFv/AMK9d8cfv/7PRf52Y18TIR7ZUw4rxn/Uw/ovO/vyuPqxi/8A5DUvvur/APvD/wD0mpeJ/wCFeu+OP3/9i4truJ6aaaUPi5FSfMQza/TdYuFz4wyVmHYfBG6/pDmglScQVLpGw1k2JvqHl2QRluUfouy4Q4Qw59PDjdTiswmmZnjYIr5B76rlyp+m2bt3bOgj8RMTn6RXx9y1Q09XBTMjqYaaV+5dkAVjlhvmZh9OHd8gWx9z0I2xN4/8tL7ppP8A7q//ANILp4mL8Nl1vW55TlOyfuxjFm3w2j/9IKKtgqZaV7aaGnhktcP5YNlvfdNJ/wDdX/8ApBJ2EUjgWnFQARbWI/5p4mJh1vW45Rl4k/eXknD+HV9ZVVAonhri9zZn5LgDNv8AxXplJXYvSwRwtrB5GhuYRDVUMLwnBcHq56eKo5UlTK5gc43zHfRbjcIv/wDUox/5RXm1bIm3v7512zr8sOMVhHp5fdH9+44Bl8c76NAUb8Wxp/rxKf8A3la+5h/91hHzjIV3DuCcexjOMGdDWGP1BjrWXXb1OnRhz25REfGZqHwsdWeU1jFsY4njDdsVqh8n2QuxHF5R+JitWf8AzCujqfst+0amgfVzcNyGCMZnyNkaRZc9LQ4hTNzVVHPFbfNGbLOnqtPUxOWnOMoj4TE/0XPVlr/iilZ3Pebvqqh593lcTi1ZS0PFDZm0hfJEbvcXXzrtueGg+cCwuV5tj9TBUcQyywyB7HAWIWt/8Fvv+y+mN3VZRlHlxn93okEUEsLJWRC5aDftcKTlNHRYfBNZE/DjTPnzSxyPJaTcgdF0PQrpjPlD4vW9PPS9Rlqn3Sj5Y/Kly/zBGi0JW3mRGMEWulylKWpZfmoyiEaIxtH+SPI5Kx7qhQxMlD4pBdr22I9l41xRSYfh3EFZHBjUlFmcC6O2xXtNN5Xlx7LyH7RMMrqHjds1NHTVQroMwjlbpZdMX6f2Z2xjvy1zPrHlH/z5IaSXmUoDaw1N95LbrpuAxatqf/DH9652IZIgDDHE4i7ms2BXTcBt/ptTf/Vj+9XL0fou8/8A9HZ9P+YdnlKWUqTImyhcn84BlCe2iLKEg0XSiw5W902UI7aIUZNlCXTRFlKZQJIOASSyuVsNnsUr9tk+T2SsPSoGLtUs5T2HZIgBatqjF1kswTkJooJqmZlPALvkOiyyF8gCYTsPpYXfIL2Phv8Ak+PxTDabFp8ZiJnZnDHNJstes+yqHhx0JqJKSWN51EcZBXwsfabtWe6emx2xOcTMVU+seselOu/Rs6fVO/ZH5YeU8NYDBiVXE/nVcMg1/Y5g0d91wnE+HVEWOVsMENLPkmcM0jct/wCK+lZsRw3DCxpYxjdG5gNQvPPtK+y2oxGmfjPCJE7qv1Rt1cXH4h7LXS91x3bpjPyifR39l+96dfWZYb54xlFR8L+cvAKjCKCkzPxTEo47m5YzYKpJifC9LaOCkfVuHUAn+KgxzhTijh6uFJxPh81FJICWGcZQ4LMdTiM7sPyK+3jsjKLjzh/UNeeG3GM9cxMT7482s7il0Y/omERsHQn/ANlVl4hxiri8lVDDf8rdVnT03iG5OYWD2TR4dPGC2iqsvs4Xus826RVlbisEZmOKylo3ylZ/g/vSETmquX/nOq0jzy50dcITbcJo4sNeSynleHDdgOynJWQeHaiMF4qYz7LSojHDCIJDZ46WVkMbbKw3VCrfBG/WF+cfGNQs8k5Lz8rbeU69SoanKYXZnAaXTwujmiaY5gTbYlBmo5iY52NJb2XLkclKiqYeV+NOGO7FNVTNcTNHijRboSp6yjwx8RyOH1VRtNRzQ8mOnJI7my03yamF4k+eHM6S9jqrpxWCPy57Lm6GirqXPyx5Xm4F9ldDA8XmitIPdc5xbjZTRmxRgYXQygEd1ixYi+WrE07ruVtnJlcI26k9FlzUM8E5LCExheToZsQpYKcVHKzu+aVFjlJXROy0rgR2WDLR10sOfOBboShojU0rTklLXd2ldKcubo8PrHurWxuexj2G9j1XYy8Q01LSh80t3D4dl5bHLLzudPJm7lX58U57MjyHHoXDVOSO9w/H2V03Lb5R3ut6NlwvHKTGaugqxyrEdiF3eG8YQmnElZSysv1j2XSMkdBV4lSUUZkkft0WeOM8GzBj2SC/VcfjuIS1cjpoZLArnZKmuD+VnYb9Qlj2inqqWrZzKV1wmmp2v9YJC5Tg2vfDGyCduj/ivsuzvfSyWrKnwiGR+djbLk+LcFxYxB8FKHR63dH6+i9AIb0SWsR4fUxOowbMnY8fmaVRqKqokbkfIdV7vWfdgp3PrYYiel4wbrzPE6vCHVsmWCnkiOwDVuxxYp4ycxCPkRAbLrcL4dwTHKoMaZIGP+NhuQtef7In+qhxpzv3Zov8brpY82kiaNQE1v3gfku8d9meOUzsr+RN/Zcp6Th6sw1pNTgDZW9yLhOaU4Kmpqqcl0VLLI0dQFa+7ppB56Z7T7rsnGSlcfCYe6Bh1yN2uqdVPPVvBkjtZcvEKc4zCrD8S91DNQAP8oXSkMy3eNFSdA0m9lnxjiwTTHo1RmnPVq3nUn7iDwp/1S340JTC8MSdkXgC70C5W4aWKRv7MAoPAOHms63cbq+NBTG+75B6mFRS0zozbISuhjhDBYjP80LqWHpGVnxoKYEdPmF89vZEab8y1jQMvmyOHui8EwfmWvFxKc+2GYszsZcJ44o73nlNx0CbNJ0cU+Q7uXRxC+KG5ey+g3Krcx53KtPDg1VstigkYYQcuRxPsprx5Lag9Ao2PeBawsjGoUsRFzydFMyJzQDIQL9AkxjAb2Qyb5lRbgmMMjQDZqtOqI5CXNezMVjCR7nBSF0YeJJCRYW0ShPM0Em6jgMtPK18IN7os5naZI9gpoJWRygzahZa4rWKPL6dod6yLuWPEzM3MVq1NQyYFjGNt0IVPKBo1IOKJ0kmXls0b1Fk1iGqR7bG3VMWE9VtqkTmPLcx6ILeUqV0bx0Q8t+UtIIuo5lRuMbxIxbVFUwtc+SZoIP8FlxR5B6dFIC65yrMuro6LF2ZuSw2jcocZ4epcRHOpJeS/t0WAHPHxKya2sI/0qT9VzYZj6eWmeYZmEOYbfNNy2lTSOL3Xe8k9yhydl0jIQPbc5MqjMTQbWVljbfNFyS83tqt2KrInh3pV2MAsAKcQEAHLupBTHQ317JYiNJmBtsVPRMYJDGLjTe6lYLHICLu6IwA1+gA+S5zkyh8OyOYjVzD3RPiiEZs1rbbXCseU7oXQMcbrl4kik2jNg6xGU6+6OSMG5tZo6dlpRwmS0MbbgqF8DWOLLi491vxJa4s97yABkAAQtlkac7Ga9CpzBKXnKy4RysPLyAgW6LXIUyXyHKXXcdh1KB7Zo3ESRub81O2uqKNp8JEGyk3Mh1KCKWtrZwKh+dx6uctQIRdpuiJumc3RzfiZv2UPNaPiWklYDrFO19iDuoA4EJw53ZQW+YTtojBuq7XuPlUoB9RJUpV3Dmcyvh9jddzhsXlC47BIs9YHnou4pckcEtQT6IyQFtHnuNVDp8Wq5M1xzLD6LOc6yIySTPfLkPncSoX3soqJ7rhCx+upRZM3xWSbTySWEQDyejSqkwFzkBN1ZbheIuDi6kcAB1TjDqkta6Nma+hF9kEDRc2VmNltDuCnfQTQvtuR0Vk5HubaAsIbqLdVKaKMPtsrMR8qqNkqXODBTZRfUuOyuNp5HHLHO2/cBZoTQZjK0dygnkjE7g92oTNo6+GTnNqGuMeoaeqdtHJnMlVmaJBqWq03SjNLdttfmFVllA0C2hR0MTLESSfNyt07qSGkyPpGn5aKUU5fm3KnYXOsulhqcKYwZcLAP7x/wCCsjFKWMfg4WL97f8ABWxz8TJmMtkJ9wtCkpJJY82Qj5qzLKysaCHiNztwToFcpn0sUeSqrxKR0Y29lyzgRUEUscnLkZa3XoVfdHH4kAMuegVc1tHFpGHub+dxtb6I462gkdmE4e7sFmI4kJTEzMeaw+2qTXRS35LMlutkMtfQgjOJNfqndNBy+Yx5jsb91poLZCIycvl21CdrZS0yCQnINGEaJzVUQd5XOdI8aXGgTGupWNs2qs5g1a0FA7GzSxl7o2XPwtCPlljfM6yjbIx0bXZwXEX8riip3Ujbw8wvc83ObZAJa+MAvnuCbeVP93Qy5n1b3NaB03up/wCjczlxzDTYITKw/tJ2OGbU9EEL6ane/JkPkHsnbQ4a6NwyGKQC+YjdPJyQXP8AK3P2Oh+ilhgxGpYHw0D6hgFmuZE9wRmcox85RU8b2y3Y1rmkEC6m5eX+qiHy1VmDhzi6dueDA6kt7lpCsQcFcWSSl89A+G+5LiP4ZVLZ8TD4wziG9oz/ALJCZ5YNGjUrrcO+zDHsSDyMQhZk3L2uaVLX/ZjFgobUYrVyzGTyDK+wv+iW5R1nTzs8GM45fBxWYD12+o0UczqcjIyY3PYrrxw3w+31UUr/AJuVmHDcDhaBHhMAI+J4BKPS4GYtiyCivK4vGYFpOi2oMDxzFWtdhmA1MsI0L2R2Ad9V2bCIhanZExvZoAWxg/EuIYQHND2yMI0Y52g91XDqMt2OF6IicvhLmKX7FuKqmBs5fRwZhfLJLZwXSYX9huGNjidjGKyF9vxGQDRx/tLTPH1c7/q8H+8U38/sQG0EH+8VLl+d3a+97vK4j6TEL9B9mvCHD1NIaGEwNNubM593O+v1XNVbsOjrw2n5vhWmzrnU97KfF+KK3F4hDIWRMG4Y71fNYhaT/Wv/AN1H0u29J1WvGcur2TMz7r9P/ddxGXDjK37vEoZbXmHW6hjlkidzIZGscNjmsVVfEf8AXn9FE5knSYD6I+rhhGOPCfP6uno+MMZoadsEckTwOr3XK6XDOMaGsl/EbFDKGDM95Av7LzE4dVPOYTvB7ObZCMLxF7sgnjHvslPldV2TpepiajjPxh6tWV2BzxSGUUUl2m4Bbc6e68uxCSjZUPNFhg5V/LmDL/3KB/D2JF4eHiZw+HmABDLh+OUkb5GYYwuYNBnzAntuucS69v7ZHQXWc5X8VWow/D6+/icEpHX3LoxdctxHw3wm6Jn3fzIKi9yIzdoAtuunkxbiTkZMQ4QppGDqP+ComvoT5qjgqna7ucw/wVjN9WMXFVHDYfk8DVvY69yJATc/4KLwVRR1f9Me0th85ym4K6fEg2tqObT0ooWWtkZewVE4XDMJBV1z5OYbks8pCTlyZyhnSSZweRKfxCGC2lwtGY0sFHnqpo2v/dCqVDY6Wn8Fh9K55/MTc/qggbPiFPyMQj5NnXY62lwucw5zC1QzQy3EMhcXnJYqOalfG50kYdEwEMBLb2PVO0Mwy8cDo5QfiRxU2M1rH8yCohppHh7ZXbNCJS285cNE8cd+WC8jNv3OiDBql5rGSMgZKHl5s8XB07IqLDq/DXtp3vp5c77AsF3OB7r0PCP5PXHGLmLEsKwtkLXBr2ufM2Ozfdrlz2btemp2ZRDzdV1vTdFET1GcY36XLmqSXC8UrG0NTgj4JnHR9G6xHuQvecG+xjGqXDI+TijJTIwOjEsh297BaH2f/YRg3B9VT47j+NOqcTiaBLA3Ly7na5+Jel1WM01Kx0jJwGMaPKDr+i+T1Xd88tkYdL5/o/nXtF7SfjNkdP0VTjHvrzu/d/f328SxjgviLBJhHU0RfGP6yN12rJyRwEirZVxu6BrQP712tVi+N1uP1TJ65jKWMl8cI3N+v8FNnfJ5ZrSN7EaL7WjPZ4ceL/E8GqM+EeJ6uF5lCNn1h+rERfSutyY5r9S8j/BdbUcLUNdBLPDRuhc31SxjQf2h1XHsaWk/Mhdsconyh0yxEkwMzfiNfb93dGmGy3dsDtR/6mf/AHwmtR9YJz/5g/yTJJaGyflFk1ijylOGqNjjNJa09JPKe7ZbBLNhv/22f/8AyEOUpZSjIubhw9OGyg9zObKOQ5nkhmVvQXvZOkoGt3RxckOvOx7m9mmxUaduyrSxnw3/ALDU/wDrpZsM/wCw1H/rf8FFlCdAZdQWtHSTsd3MtwoUWUJ0ukqwKWF8LJAZ4jK0fCH5b/VNlKWUqK6fCMQ+zhtE0YzwtVOqcxuYqpwbborNRiH2VugfyOGcREuU5L1jrX6Ljkl83PtsZ7J2eLnHndRlNfb4O0bqiuMfZOarC/hwFwP/APNOVc7l1rDt2ReXsmX0nE2UKRobBZ9RRCZrxoC4j+5MGp8v7yyOh4YxvgKjldJxNwlLPlcDGaeTp7hy+guDuJPsa4gpYjhdFhuH1co5bYKiBolIC+XcjU9OX0szKmlkdHKw3a9psQV+Y777N495jlG7LHKPTzmo8vh9re3pusnp/LjEw9x+0D7DMQioZcY4Nw/DsQkLnSFg8pI7AXXic8/EvC8jDxDw5V4VM4eSTK5oK0YeMOMKR16LiXEIe2WU2/TZekYD9s+HVeHsg49wWnxSeABjHviJL2+5Xj6fHvXZOn49TH4nG/d/FH38ph0znp+pzvD8k/s8ri4zrJPxYseqGF24MxIP+8tuj4wxIuYaqKlqYb+Zzm+Yhe1PwH7CeO+H4qufDMJwttU3M2RknKlZ9AAuB4z+xf7LsNwirxfhLj8Q1cEeaKB9QHsce2pU6P2v6Ldl+H2a89eczxrjP09Y8oNnbs8YnLGYmPW7cNxhxlhAmZU03DrqeBjQxxYzNr38qwfvGrqohVUGIQiM7B8d7qvUfzhpmPYa6ir2PFvI6xCGJ1a7DvCGhhY49S7/AIL9brwjXjGMe506XuWrp9Ua8tOOVe+fVlYhgdVX1j6qethL5N7RusoDwo/pWRD/AGV0bsSxVz2sNFC0WtcSX/wVwYo+FgEjYjfsF0fXw9rt2vGMcdcREfOXEy8P08D8lRisUbiL2cy3+KyIKWjqcVmpI8TiDIABmI0J6rv5MNwriCqbJXF1NUegEEFlv0VSX7F34SyrqaDEPFVE7g9rDIA1vdy8m3rdWnLhsuP08n0tXth0Ucceomccp+U193O/cuGM/wDrFL/stQOwjDX+jG6b6tv/AIqrxHTVnDFUaSujbI9oveN1wfkeqy3YvSGKJ+Qlzxdzcw8p7L1YzGccsZuH6LV1eO7CNmubifSYaVZwpQ1eX/8ASaliyX0bGbLqcPxabD6CGgjx2gcyBmQEsIK4SPGKS2bkAfORHHjFHI2/hx9JFqmd+GHU4xjuxiY+b0Kk4ka2oYazGqTk38wbGb2XVtn4Ukja9/FtEMwBtlNwvC5sbpOe1jIjl+MZgk7G8O/7NN9ApxfI6rsujqJidf5Po915nCvwcX0R/wBgrLquIuGaOZ8DcbjkLdA5sZsV479+UPwwzD5gf5pfftJ/q5f0H+azxcdPs/pwyvPKco+zoWYpO/i17psQY8Rvzxva3yD/AGbr0/C67BMQp2PPElLFKTlLXtIJPsvEKarjnq5apna9kbeIaJjw9j5A5puDl2K568amYfQ7h2zV1mvHCPyzj6V/R9AjDKD+s4ipf90q1SRw0WbwfFjYc2+Rrhf9F4jT/ak6GFkTqfmlgtmde5U4+1gj/qzf90reWqM445RcPyc9j7hjP5Yj7vbXzVMrHMfxmS1wsQ5zgCpqGobTSRtm4oY+AOGdtr6dbLxA/a2y3+gt/QqvJ9rAlY6NtCBmFrgm4WcdGGGM44RV/DyY/wAk7jlMcoj7vsHGONfsZwngwzVseB4hMITE2FsQZUAuBAJt2XxtxfTcPU1dT1uBuY1tQ5wlYxxIBJCw6zHn1UjZYCQ8uzSGQ6OHZBPiMVbJEI4DHkdc63uvj9s7Jh2jDbOOzLOc5uZyn9v3/V+07f0uPSxUevlb1r7NH/Z9gfEdFiWP04rInvZywyYgB5PqkHVq+kKmP7IOPYzRU9BHCxrs7n4dKC4W9gviCauYLwMY/TqRZafCfGeK8N4iyTDqh0HMOR2V1hY9V4+8ezm3umzHq+n6jLXsxjy+FvL3Loo3ROcRc/B9o1P8m/g3GcMkdwlj9SyruMpmeHBv+yvMuMfsT424Lcw1VK6ujkJAlpWFwaPdc5hH2pYjhNdHXMx0zGMEZHzEAr1DC/5XGE4JQxniSd9W95IMUTc1v9pfFyj2q7JMVMdTh7/fP0+NQ/Ia+n6frsuGGM45PFXNcx7o3tLXNNiD0Kde9cL8e/ybPtYnqsOho24NWvjL3T1DG0+p9RDr5brmeJfsCxDDYZ8XwjibC6nDW3dDLNVsaXN6eba6+90ftT0+zZOjrsJ0ZxXln5XfweXqu09R08+lvKzfoms4bI5In07uXM0gna5uCkG9n/wX6uJfMmK8gA+ZcX9plLBEaXH6oTHw7TEDG25AJ6ruA34syr4lBFV4dUU80Ye0xPNj3A0WsZezt/Uz0fUY7Y/+RLx+ixSkxVrjRmQ5RfzNI0XofBuFCkpDWyMeyaXykO7dFzf2YYRTV1EZKgZhSyOaG20PmXpDmBgDQLAaALWUv0PtD3K4no9f6/2MklY9k/nvquT8kHKErDsnuduiVz2+qpRrO/KllIPpRe9rpwT2sEhlFZ3ZPb2CPObjy6JZh8TDb2UaRm1w3YpW/eKk5sf5T+iQAd6W2QRZXD4imy3P7T+CmsPiSyhUQ2KVipbAn169ksov5QgisdkzZZ6eaOqpn5ZYjcKbJ7JixW0p9E/Zx9sHC9bRUHD1ZnoquKLI58pAjvfv9V1XEXDuJ18kuIU0rZoAAWNa65+i+TGh0bw9psQbg9lsU/GnF9EQ+k4jrorbASL+f9V7GZaut/G9tzjGcr5Rlc3Mzc18Le3dvw6zp/w/UxNR6V5PVuKOC8aqaYyx0crQw5jcbLjaPEcZwKUNZUTUrx8BNwul4b/lC4swGl4lpopqdkYaCGFz3EdytKv+2D7KsaiFNieCVMRvcvihBsvnx/mujb4XU9LlPzw/NFPkbuxapx5dPtj6ZeUsCbialxSNsWP4HS17W6NzMbouTxn7L/sj4nrXVz4azCZZAA5kT7MH0XrOEcDfZlxVTRVXD3FE1NJMwuZC+QB31B1WO/7FuOIpJDHPQVEV/K4S2cfmrq79o1ZZYzs8PLHymMvyz+kT6mjou8dBHidNM/8A4zbwmb+TPi9TNI7AeKsPlp8xyNlJa63S64TFvso+0fC6uambw5PVNhdlEsADmO9wvoyt4W4lw972VWC1Qym2djC5qqU3EOJUcnJgxiojezdhkvb6L9Do71ty8/LJ9PR7Zdy6eZ8fCMo+cU+VMWwXEaCrdT4zQz0lQDZzH+UqlT0dPRPMkDLEixX1ZidScSfI/E6WirHTesyQ3LiuGf8AZhwnPMXVGHsPMcSQwhhX09XdtOUf6kTD9D0ntv0W3H/6jGcJ+/19HhMss8LjPA85h0cbhZrMQxGVzhUsdYi1sq9yxb7KcCZVZcMwuZlLlF3uqPiUUP2O4PUOtzmR3/NMQvZj1fTZRfOH2dftH2vZhGfjYxfxmp+zwmKHEYn52RCQe6sRF8s3noTGT2XpuPfZ3T4RWSwR4m5nK3cGBzP1vZZX8za8sL6fE4pu3ktf9F6MeOccsfOH19W3XvwjPXMTE/Bxc1JRQ3qpnk2+A7KeiqMNqZOWykMLz8YNgt6TgvFwfxaSOQdQJP8Agq7sFr6XQYRLbtbMtcFpnT1LKdpgfJld8lQbNUF3kkBF+ykxLBqueoMlQyaAns02KGlwumhs018mYdCLJxWmoYcsDZ4WM54IILVBOx8rxVF+V9rOZbRWIZnAC50VDGGwBnihLZ4/KVz4FI5DPUPdGw2HdVmUVc6/KbnVfxc/Ryu0tXUU7Oe+x9gV0plmNknY/LJob7KWera2I3ViZ9FiUrTU0skX77Dqq/gmUri0vdOzpZa4jYw/G6KqgEFVTse7o8M1BQVGMS05MMM+RvyWNFVQ003lgkH0UtTy6sF7IxmO5C14aLBxJlUclRObn4gmcaCEibx0pI2s1ZvhZGvD7bK9BTQzMvIE4K1qPE4TGORMXN7rTh4iromhjMQnaB0uuZaKZnljqiPmFNE5rv6wO+Ss624l1jeNK+PeqY7+1GpovtBmEgM1PE9vVuoXMMlgLCx8pbfsoJY4tHQkuHyU4tOtxni+DEqJ0DMPyOP711wNW5xnL1oZnkeRpPyUTqd8h1p5P0WuLNosLxHEKGpE1LJa3Rel4VxZV1kLbxR53LzptARq2Cb6BXaaCYsDLSNt9FMoW3ozsSxJw81Ez9VQrMTrmx2kbkv73XKRtxSHWOvmHtdStqsXJySVRc3sV5spVquxGpDb3B9iFWmr3SDzwx/oqcs9XGy4a0/NUn1tS/8AaAD5LjMiarcwtu0AaqrzLHZRTVXlOt1UdVPO2iUNLmXbexQBxJVKOpkuGkq1FUUz3ZeaA7sVmpFghhHosm5bSLBycOY4aa/VLRxuN1LkQPZlKRygXcik0vdROewg32WoCIJ1tdROcfhU0Msb9C4D5oJY2suRb6K+Y5psEgHmankFjboibIXMz9EJkuvoOCMiyHlh3RO52qAyAekqwhnRlp0SYXndPmJ3Tc5g8mU3WmhecJnNvukJbC3XuhLykQGYLOB6hTUUDZ6vK70MFyogNVbkc+Ci/D0e87KswpSyHxDmRsAj6BSlgP7N1x3TT0slNFG+S+Z4vqoWPe1ot1WWpW+Y2wDNLaJzNDC0l+rz26KpmsC4JQt8RLZ8mUDX5qsrhjeIxUyEa62CdhzG1gE73NMjY23I2ACeSBsbS+3m7dVlrkE5YjnIv7IC/MS4uAQHNYk7BVzI1wsFaFkSA7EWRhwt5VRBu4NVtjSGZsqlJZ3H8qbmaWQGaNu7kUDRUvLIwSbdFKaopKZ4yyGwahzM7op4J435HteW/K6BzeW25aQPdKSjjKTmGyMPAUcJzK0IWjQjzJZxM12UhzXE+x6KyxkW5ab+xUIg5Zvcn5hSBsjm5g+30UlqgvblJdonY9VpXlhylyh50l9HLPEpf5ubQNuVJCHud+ILN6lRUJiL3PkdYMF7oRVySzZbgNOwAU4lLzTYOIeQ0bWTavb8+vVPqIzZ+W/smiDg3vfus0UdsDQ3zOURpmZieZb5qZzJ82cTxho3uNlCTTf1teC7qSFYhVaWIukyZ227qvLC9pysOvsrJbRN87K0E9wkyMSm7TnHddcZpKZw8TC/Oxl/Yqfw4qvOyIh/5Vbko+ZtHdNJB4TK2NuVxFyt2UzzA9j8milFNGzSR2Y92rRc6lbG1s8fMfe2YA6KN4jY8MjlysI3tcqWzaF5dNb8MCw6BRsblc43JudlZOrSyOVtup7p4KOdrjK9mj+qNQ18Biu0yDYvNl0OKSPpeH6mRmhLcgPzWVgkWWNrFq4+b4IKXpI4XW2Hn9Pn5IANtSn8NG/1lbYwSkDi4Voy9j0Tuw2hb/10E+zVzhuGL4TTSMn5IWxvjeHxx2I2K3fA0wP7OQ/NTw4dRyuIfG8AC5Icq0xIpqvNnmfoEBY9x8np7roW4bQnR3NF9geoRjCYBoI9PmpyGC2OZnUO9+qTY5pX2tdbzsMYx2WNgSOFw6cxgv8AxWuaMkMma3lx02Yj4jqAly6rR5hcT+4FqikyA/h5gNgpYIpJjkeTk6N2UsYQbUPeMjHfVWW01bJGBJGcrdg0rXdE17QzLlsb2BRGnjLbMkDD1JVtbYoikjOXwpcCbG6Kqjex7Y20sgFr91r8uGEhgnDr7gKMNopXue8uA2OdRGMKeoebCnkHzCM0New5n2c0dAtpppS50cclswsS7ZQsbG2/Miu4+nId0tWWcLrZWiRlMcnclSMopIzZtgSr0rKt1hSMmb+7lutCGHE52iBmD1Ob2iIWJ/MM3w0kRMdwHDf2UYpprk3Dz3HRdEOE8flP+h07faSWxVqD7Pqt4L6vE2wPPwweYD6qo5Q0z2tJmkJ9g1R82nbcSQySA62Pdd9Q8C4PSXdPVVU7iLXL7K3/ADM4YOs1AZnd3uuUHm7J42vDnUouOjjdWuTNiDXPho3OkOzoonFej02D4JRW8LhdPHl2tGtmjxqtw6Iw0BZG0m/oRz2TnGN64iZ+fl/d5nScNYrWMa0cPTtd1eRYLosD+yTiPEzFPUPp6OB51c92ZzfoF6JTcZmOBjZ6ASyNFnPJGpUh44uLNwwgezljnL4/UdT3PKJx1aan43E/2c4/7DKeTSbicub1ywEH+9XR9j3DtFC55ldM1oJPMuQtb+fDf+wH/eU9JxbSVr3U9ZDyInNILib/AEUt8zKe94Ryyuo//wAs7A+AuGgY6+GkpZ2EXymMa/NdfSUkdJCKajiEUbdmNAssmixDhrCmujpKkMzalupVj+c+Djerb+ixOT5vXR13VZzcZzj7rj/iPJfmim5b3MDpHgXa0utcrm8R4jxXCpWxVWFsa5wuLPJ0WnLxZhMcLnxVAfIBcN7lcbj+P1GMVEc7KZ0eRuW173WYers/bM9uyuo1fl+M3E/1hfreNK6eB0UNKYHnZ7SbhYldiOJV7WsrJZpQw3AcNlW8ROWkuBuOiQnncP2J/Vah+u6foOm6X/ysIj+v3Bl+f6JyIz63o3PLRd4NlA/EaVsbpJqqONjN7kLVvYLLTf69o+ZTthgd/Wx/76oy8S4NE0vBfMR05SpScY0srSylwTmHq4mwarY3/CyEHw45lt7G6OPD66RmbIR81w9RV1VdPdkwgZ1ZAbBNzMQi/wBFxOeI98xKzY7XwM4Pnk/RTCgkcPXK9cJzOIHNMn3rLJbrzLKzS4jxdTQZqetkez943Sx1/IfDvCT8wn8RINGwj/dXLM4r4ugdm/AkPZ4RQcccTTPySYXTub7ghLHUGrkPreojOPzhYR4tq2NbPVYTSujvZwZ6wr0PEeFVTrfdtU352ViRd8U2/msfmpBWR2tyz9AqcmLYOzV8VRGO+UFR/fmAjyurZ2nsYStDRZVU49DnszfNGZ2H+tKoDF8EvY4tEb9LFSitw1wuzEqe3TzIJ+ZI7+sik/deFFyubvQU8h+gRMfRPF21dOflKFKIXPNmNz/2QSiXEKvIijPnwen+jQglw3CZHc1+C0rnnfyrqcL4G4nxqkFZh+FTSQklocRl1C1cN+zHFJKkwY1L93sDSczhmN+y8u3qdOu+WUeTw7+69D00ZTt24xXrFxf29Xn78OwOZjqc4LCA/f3/AILosP8AsdqcUw2GuhwsiO1mZ6gsOX5L13CPsvwN1PDE3CRNLEAXVE4LGucDvZd1T8P0ERb4kl7miwYzysb/AJr5PUd4j00R934zuXttjERj0GNzfnOUeX6ebhOAfsa4OwCOmr6vB4qqtgPNjlnOZzT8vZeg4lLUxQZafRzyAXDdo7hTPlggYXSVDG26ZtSubxLjEwVRjggE7R6nXtYr48Yb+uz8rmX4Tdt6rue2c9szll/T6fBYpqOOKJ9TOXNda7nyu3XG4picdZWPmhtnOnl9Oi0cVxw4vTOgfTvOb4S/Rc/UYhhlA3l1MzGPHwN1K+72vocunvPZFS9/Q9JlpvLOPNJQ00Mcn4sj7yuvI8nzEq9jVRg1HyXRVLY42ssSRZzj8lyM/FNXKLUVO2AdHOF3rIkbJM/mTPMju7jcr6s4XlGV+nufQnG8oyv0dHV8ZVxw+ownC5ZoaWpsZr284G1uy587hIB9sqQDraq44Y43OMVfr8zKZn1ONrBKx7J27p10tk2UpZQnSVCTt3TJ9PdAGvuiGhDUJBvskAb7I0JM3ZOdkKMi0ATZiEyfKVWhN3T27oQbaXST1SRpJJKKSNAkgSSScWv5kDjqhRDcoUSBdfolm1QpIo07d1GjQHp7p73Qa3TrKWbKwnYfonyMHwBMCL5QlfWylKcAD4UJDCdGolG8G4aHgZzbVUJ7Be/RRlrVcmwauhidJJNCcgvbmi9ls8NfZvxjxfh/3ngGFGrpiS3MH2II7rzdR1OnpMee/OMY+MzUfu3hqyzmsYuXKldZwnPNPSyslkLmxuDWg9BZbX/QH9ph/wDo8YP/AIg/yUNd9nn2jcA0L8ZxTBOZQh4bKIpMxBPtZfJ6zuHQ9Zr8HTuwnKaqOUX+nm59b0W7ZpmIxn7N6HCeB8XpIKbing6jxFsOvNcHZvqL6rJ4n+xPgXF5mVXBuD0DHus11Ifww33u4qvhHF1NKI/ERFjXbEdF0lLWU9bEKikmDm9HDcL5Onqt3SZfln090vkdB3jre17Iy1ZTUf8ATN19njfFX2UYpw1UiGXhOgcXNzNa3Mbj5hcTOKGjfy6zhmngf2IK+sIeI8SipzS1DWVcTmlpEoGax/eWLX8FcI8TcmGjfFhTwCHx1Dc4cfnZfX6bvOGXluip+L992r250baw67HjPxjzj7esfu+aObhbv/oFEf8AZUeSgPo4cpD8tF7txB/Jna2lnr8Ixh3iY23jgYyzJD2XlvEX2fca8IQiqxzBpWwE5RNH5g4r36us0b/4Mn7Doe9dt6/y0bYmfSvSftPm5kNpG+b+bdJ9ZCnD6cat4covl1Tvlzus1wL+xNkxts71dV6eT7Xg4qrs8deK2nw2OAWsYgND3VttTTH9pw7R/O2iTo3DcImR6eoLj6N8MUonpW2ezAKG3ci6TqtnwYFhw+TUAb3Fx2vqh5ZOzh9V05Sx4eKQ1bPV9x4eD3y3SNRE8a4Hh7vflhCYtdXOHsRcfqmMelrfUapyk8LFI2WlA0wbDz847KniMba1rG09FT0j2G94xYlWAMpT39k9YqTwsTRVMAgZDW4VSyOYLF1jqpOdhH/2SAn9QgysPlzC/ulkb2WPP0hfCx+CVrsMFn/cdJ/EKYVGHD/6DSH5hVAwBOCBu266c5TwcPgndLhTjrgNKPkSFu1fHmJ1+DRcP1rHTYdDlyU7pTlbl2/Rc0cpN8pCV29yuWzXhtnHLZjEzjNxcek/GPgxOjXPri7bhLHeE6iuhw3iQ1eGYeQQ6eIiQs+TV7FwR9l32TcZz1MfDH2sV7pIWBzo6yJrGtF+gO6+Z7go4qiSEl0Er2E75XEXXg7h0nVdVEz0+/LXPyjGY+sxMX+753Udl6XqImJxjz+Uf/t9MYl/Jz+0enle7C6SmraW5yStqGguHQ2K8p4hlfwtiU+DcRUdVSzREskD4nZT3s7qFHhn29/a1g0MVNhnE74o4WBjGuaHAAKX7QPtlxP7QaSj++8OhdUUtG+CeVtgZ3E3zLh2vZ3vX1EYdfGGWub88biYr0mYn4+nk/PR7J4Y5fmma+U/+zmuB24dT1mIx4fI3kPfnjFrWv7LrZCHFeL8NVjKbGqV1RI9kLn5XOYbEL2GOXDJKFkNFVl1Wx5LnvfmaR0Frr9JsymaeDu3Yeoy2eLo/NdR8/KKv9k3XdMRrddyz7D/ALRaigpsRw7DPGxVETZRy3tBF+llzuO8H8S8MAHiDCZqMnbmiy+fp7r0O/Pwte3GcvhcX9n5bPRs1xeeMxH0Y7u6a57pmyxONgU+YdCvoW50QJT5ja10nbpkQ+YJ8xsEKSNDz65c2ibN80KWnujIkswTZgmG6NC0SQp8yvG2RDKhd6kwNkRcoFp21Su38iby9k922Ro5DLat0QmOF39WER9Ka9gkCWhqJMMqm1mHuMMzdA9u4W237QONYjeHiStaR2eFzp9k4PdeXf0HS9Vlz36scp+MxE/1hvHbswisZmHo+BfbjxThNJ4etAr5MxPNlcSbdtFzWK4xh+NPrMdqWNjxCaa7YWC3kO5uud8vZLy9l4+n7H0PR7ct3TYcMsvWvSYu6qfKI+kQ3s357seGybj5rkc1MzOXS1ZLtnF97KjWPxKedgikhcy/mPpsEd/YpXC9Wfb+m2TMzi+dn0WjO5nFQlnxOnfaM1LT+ZjgQrdPi2Pcgvcx8rGmxL23KkuO6cPc30vI+RXkz7NpmPy5TEvJl2zXMflmbUqzHIcSgkw7FMNpaiGTRzc1rqpFgXAg1+5Hxf8AhFwC03U9G+QTvpojI3Z2QIBQ0XNY57ZQwG5aw7rzz2vqNUVpz8vsmOnrOmjj0+2Yj5TMf0HT0fC7HAwQRZujZozYqpNwxLU1M0sGJQRxSOu2JkYAYOwV51DhMj80c1TEOxaFA/Cw3SPEYy7oHtI/xXLXp7lrm8b+8S69P1/eemynLXsy8/jN/wBbDHwbRE/jz1QHdgYQpWfZhwLiUnLr66pGfq6AAfwV7DMJxGLJOK+ieHeqJstj/cugdhWJwhrp8PlaCLnKMwXPZ1XXaprOZj9Ibz9pu9a5rLbMfpH9mVB/Jc4LrohU0eJzPZ0MbgvPeI/5NvEOG4hOzDLvpGk5JJyBmHezV7BQVVbhkvMoHTQd2lhsu6wnjHDcSApqsxRTO05T/iK5R3TqcJuZv6u/Te2HdNGXLPKMvlMf2p8Wz/ZbWRzEVNRGyxsSGuAQv+zCOQBsuL2H/hL6y4n4S4clrpMQkwGeSSXR4YQWFcbH9muGV7XzVVdLSBziQHwuswdl9bT3fRnH5/KX7To/bTt2/H/WmcJr3xcX8qeC0/2eU8LeWcde5nVvKU7fs+wT+srZ3fKwXsOL/ZJidPO1mAyNxGMtu57fJlPbzLlMQwGtwqqkosQpnRzRmzmnoV79XUat/nryt+h6TuXSdfET0+yMvfXv+3q46L7P+FI3cySGaV370ikPCHCjT5MO/wDyK6AxQMOUhDkgv5Wr0PcyBw3gGwwmP9VFNwjhMjs8FI2HvYXut3ygelNz5BsULc67hCBvohB/2QgHDEEfroo/0XS8+SyEF7vWtWOXkwOK/wCxCF+Ax9J42/RdQYIz8KfkRD4Etact90iM/t2n5NCNuEufv5vouotAd44v0RAUjfyA+ycinMfdAA1jd/soDh0HVs2nsuqM1PGPNOwD5qtNX4a3eeMn5pZTl3QUg9ccyikZRX8oePmujlqqJ7S1scZv7LKrqWJ5zxQAHuFxzwsYVYYmRkMIIWJVyhw8q2qylc++RhBXNVjp6aUsmYW9j0K5eFLNoZn6ZmvUImfYlgv7KnNI/P5LDvZV/EyteLOsvRGqV5NeOthJs9haVPzYsucWcFjCpefWVJHO4G+YJ4UjajlYReN2qnZUFu5WXSysluDoWi9wps379j2K5TqltdkqAfUVVdOA82cohHITrKEE8RY3PnCx4UudrTXMe0PYRdAZZAbSEWVGJzy4ZT/FHJI4jK9wK34craCNhLRm2UUscZNmEj6q/lp2M884AUefCi3KBIT3XpYUJRDGPNIAUwgppGh/MJPsrvOw4HyYeJD+ZxRtqILfhwRMHyWobUM0Ef7RrsvsjlmpJYuXTUri7uRqpZJHzXjy5W97KNh5JtGbnutMqwoq2U+SmkViDBcUlcGsja3+0VM2onb/AFrv1SNTK4avRaT0+ATtlBqauBo+asPw9kMrZm1cL2i97hZQk6mPM7vdRTTvk0f6eyyys1VLQTSmSfFhr8LdghjnwGjdYOfUEdwsyYML7sFiVIKGURc+ePlt+atC5W19PVx5IYBEPYWuqsLeW8PdsoN/K1SxDmEQd0ZaNO1jbTh+2wKY5PEte/OXP3uE7Ypo5BCy1gO6mhhkrKtjGNAybrEtYrbMKp56tscjiGHoFWqsPoIJSxlA6Vv52u0WhV00wvMy1xtYqtHLJcCSbKfksulIfA4flD4oJGOP7l0bKSJrtbu+YstJ8z2Rl73scG7qu0sqGZxHY9St2xMK7oQPRDHfvZRg8sF7PK8bkBaFMAH2dSTy23AFrJShhktHSyNH7yzbdIqepky5y7zHdTB7Kohs8cZA6EaFSsomW/Ea76KKSKCJ+VkU36Jya4q8wEzxD4OGGx3ARNiaPICC5u9ldhF2WfGbdLpy6OHyeFhB9isWlKsFFDLIWP8AK3qVFV0LY2/0aojydFadl8SL+g+pTiKgJ8s7gOwjv/itWkwyGYMydwElOZC7d17JHBqIOLGRyAD8q1zNhzfIyd5HZzbJx4bdrnD5Ja0xRhQjY9jc8TXixc8aBWIcDZCzmeIjkI36E/RaDgHEM3b1T8uKJpdcD2vqlnFTkpKRlMyaqp5yb6ZBuoOfC+XO2mmaOjS1bUQZlzyfRMXsefIYx/AqFMzJUy6soZ8vfKihpqd7byQB1+61nDoTMfkVAaKQvvDM1g/K7dClJuF0sbw7kh37h2KtGOPph0LR7Kfw097vMR+Sk8KWxCTmNOboFqZJhVZS0krgw0YLj0anlwahALn0tj26qw1k8LxIxxaR1BSneJZBJm827r9SsWlKRoqGJoeylMjx+YlLwVJN5xSNzdGlXJIiXB5LQ0Dok+lEsTX84guOtluykHhjcsNFEB3ACflQRsDYqVt2neyvPhY1rWwx2dsS6S5PzULnvizZD01tqo0GhbeTN3Kv4oI5Io4C4XHQbqvhw8zQgxSUiqfkB0sLhdJcI/iQMpKUOHMg+V91HNT4eCeUJC4bhDNM8ANexxL9uqnYabLk9UtrnRcnYJfdoYyNo01NtU8tHTkF4nuW7N2ScYIpOVIS5rwCHt0smHhTIAeby3nzEDWynNU5jY19PMJCW8vVo0uim87szSWjoApJqaSpfGzCKSqlaBY3j2Kss4Vx10jc8PLb1L3hatJhmNa++qke1oHkGq2YuEKvMXT4tDB3Aj5isxcJ4dmzz4tNKf3IgApaU5jM9nspY5WA5nS39huuybg2Bt2w4O/taqeOmpIWZIKKnjA6ZArbThuTNNI0MpZyXmwHLKvRYRi00hjgoCW/mfouy5kn5o/4JZyRq8fqpY5WLhHEJpGmqkp6do6g5iVbHBGGZw6SvdYdGN3/AIreBs7NcfqkXAncfqtWMocH4E34Zn/NytQYFgVPa2FQPt1eLlXOY3/VlEJWjeMn9FA8LoIQGQRwxAbZW2U3Pk/1pUJqKUj0EIC+lcMvn/RQTauN3yk/VPl/7z9Sq+WhHW3zTWov9flHW3VBZzBC55UWXDm/s6o39yj5tKP6+P8AVA5k/MhEzBuUxniudGkd8wUfPYT+zJ+WqCyKuI+XNqiE7O1/qqUszGNvyC4DoGqpJiNILCdkkN9sw3Qawff0puY7o26yTiuE2/08fJupVeTiWigIEUc0pPRwsg2vEvB8u6IVFa/Z65ufiqrELnw4XEzs4yAn9LLM/nHjNZHeWQ0jR1BACyO1lmqom556mONv75CryY1Qxft6+GX2jFyuJkEc7eZznykfEZCbqSWaomiELnw3HtssDo6riqlhu6koppbfm8oKpHijEaixYKWMHaLqs+MXiYDo4CxLdLpmsZyiwgZr3a4nzgrUA6irqpswqqqQB+mVrtPqqvLYP2zCQe5sVM9rHWYQRc2uQnin5bXPmAlHpB7hURuMZAec3k18wSDyYvNGLDr1ROfmAGU2LtBuEzXQMjlEj8zmi2g0CstHEz7WY0ABOJ3gF77P0sB0CdkTDE4vJB7BNGIG3zvt9NFniFFUvi0fG/Xr0Uxra6DyiPkMPtuFM2fDSHRPjZI7S1ypKWWectjkoo3RNN81wB9UopSM/wDrC4E7EiynZNVPiBfQumZYmNze47q9WVc8EuXEeUInnytacxsoG414SJpo4SGx+nMba9yFUtWlf5s8mGSC/wCe4t+iMOw13qbPGOhYDr/BXqbivEKqZsMGHxTzONmBjsziV23DXDn2j8VSvpqfhYUjQ3NzaphYw+2y57NuGqLzmnm6jrtHSYzlvzjGI+LzSTkskIbO4j3KiknhZ/WN+pX0fgf2ByTQPl40kw/n5vw/DMzWHuvQcK4E4IwXD4aFnD1BNyRZss1O17z8yvnbu8aNc1h+Z+W6z226Dpp46YnOfl5R93yNgfC2N8TPkjwPAauvfELv5bdLLteHf5PfFGOVUkeN0/3Q0NzNdJZ9/Y2X0xDFh9AXDC6OCmDhb8GMMBSMtU45c5y/NfP2972ZXGvGnwOr9uuozuOm1xjHumfOY/4/ZwHD/wDJ/wDs3wKmppqummq62CxfIZnZHuB/L2XesiwpgDabCIA21gRA0f4KQU+cee/6qUcinaGOeR7DUlfK2dRt3TeeUy/I9V3Lq+sy5b9kz+vpfwQxNdEOXFEyJv5WtACLlNZIKt9NFK9nUhZ1fxHS0zQIW5i82DnaEfRc/i9bHiD/AMOSex1NpC3X5L0aeg3dRMeVR8WdXTbd83Xl8XTYtxXSUMLpJKd7nfkG5WG/jd8pHLpJom9ja6xJo2xAS1czY427OlfYLErOK8OpnBlBD4t+vmH7MfXqvs6e0acca2ecvp6+26scaz85b2JVsmI1hqYInRZmgEl1ysavx6hw55jklM84+Bg/x6Ll67FcWxKYOqpgIhtCwZWAKERRj4V9LXrx1YxjjHlD2YY468YxxaFZxDiVcwxgsp2HpGNf1VAM+vud04awdEbW30uu7pZsruzkWUo+n0SyhER2cPUlp7o3boButB7a7pWPdO426JON+iJZJXzJAByRAKNUYJ9CEnJKSwSbMUr+xSylAyXRLp9U/wDcq0Td0xPmISTbG6AmWJT5rILDrf2siBFup+aoLMUSjyhE3dQEkmzBLMFaLOlb3SSUCSSSRks2UJX6pW72KSNEiB0TWF9krD4UD3PulmCQACWnsgFOSETBS8smSd7X9Ghu/wBUAzEZigK7ghdc6FgP0RpKT5CEx22j/guk4f4+4r4cohSYLVRwRxOLg4N1ud1gDZMQ2923Xk6ro+n67DwupwjPH1qYuLbw2Z6p5YTUvojgX+UXh9bysM4pjMD2x3krnua3M73AXsNLV4VxDhjaqkqIayjqG6PY7MHDuF8KOYtvhzjXiPharjq8KxKZhiaWtjLyWW+S/nvd/wDD3Vsmd3bMuGXrxn0/SfWPP9IfX6bu2Ufl3RcPpHir7DeDOIKWSaj51LiTcxgnEhLWu/eb8S8MxvhHjjgWUPlw6oZTOc5gqAAGOLRu4dF6pwN/KFw3F3U2HcTwNpq2Q2NUxtogff8AKvWZX0WIwRuqGxVER8zHFoe35r83Hd+6dg3eB3jCc4n0mZ8/h+WffH1/5Xqe1dF3THnr8pj4f8vlLD+M4pmgVcTgXC4c3quhglp6yPmM1C67jv7BqTHcYjxjAeTG7NmkiL8jXm+wHZea8YYFxBwVK8u4arWRRHK6qp/NET7BfqOl6zpO4Y4z0uyMspi+PpMPx/V9j6rpPOrj5Orw/G8Twl+WmqC6L8knmH8V1VHxdgmJjk4lStikG4e0Pj/ivHsN43jnjDquIOaR+0Z/kujpq+iqgOTUsN9tV3qcZfJjKcJ+ba4h+wjgDiCSeuio3w1M2ZwlgkszMRpp2XiXFn2D8ZcNlkuGsGMQuuS+m3b8wV7JRYtX4ZIHUlQ4N6sOoXV4ZxfQ1QbHiLDBKfiA8pXs0dx6jR77j5v0/bfa3ufQT/Hzx+GXn9vfH6PjGtpKrC6g0WKU8lNUNAJjl8rgCgHmFw4/Qr7B4x+zbhP7Qoo5cSia5wN2zwnK92mxK8Xxz+TdxThzJ6jBa+CqijzObCTaTL0HzX2em7np24/nnjL+j9o9sui6zCI6jLhn74n0v5S8oY0lpc17T7dUNj6Q85uzlYr6HEMMlZBjOGS0ctvLmZldZQXzNz5+aL213X04mJi4fr9e3HZHLGbgiZma7H9QmEjz5tD7hOHhrhy3lruztkjI4HLo0/u7KNnY4Pu1paT7ixQ2ePWWN7X1Ca4kb+IWyA7dwjYbG3ODb/m2VkNlcfNlBA6tNwkHFuzQfkVK6HS+37zNlG5smXIHsk+e4UCDtdWD5O0RF9nD8P8AxUYLx5cwYf3homOcHqD3bstCUtvezL/IoBqCGmMH942KY5pADcOPvoU48vmkcB/aFwUZBqOv0spLPcAWM/QqMnXTT+zsnGZ5zfh37HQhBJ5yMwI/XVEMtjmG4tYhQ3eHec2b8rqRpsczDcdmnRMRylOctQOwl/xXWyMseYw2cfiabLlpmuirJWno6/8AFdMyS0bNvSF2yebDCLmJeo4J/KX+1rAKGnwygxam5FKwRxMlpw4WG2t1t4h/KcxXinhapwDjrCY6qaWoikiqKaNsYZG31N11uV4nfVP01Xw59n+2eJG3HTjGUTdxFTcTfrDE9Fpn/ph9VcJw/wAl3iTAaWesxN+CVbxrBNVkPafc2VHjT7GKeopIse+yrEpMfw6V/J5bPxHhw3Jd2uvmO1x5Vt8M8a8UcH1lPV4HjNRA2nk5ggZKeW4+4Xz/APKOu6TbPUdJ1WWU+f5M5vGvh8fpNvj7/Z3pd0zlMefr/wDJjzerz/ZT9pVLEyWXg+uOYXIZlcR9L3XKTOdSVs+H1jHwVNM7JLC8Wew9iukP8rn7T300lLO2ge2RhYTy8rxcWuHLz88bRYvVF2LQyulqHjmykh4JPUuX1O17+67JynuOGGMR6cZmZ/d8WfZKcomYyqfd729uluV7PwH9kf2Q41hFHQ1PGEDsXnN+XTVQ66gBvdcsPsqq+IcTxCLgc1E1Lh4c1z6llg97TYtaddVx6f2n6Dftz1xMxGHrOUVHrUec/GfR8HqezdTouYi4j/5bgspSy+63avgHjqg89XwtXNZ+YRgj+CxKiOelldBUwujkZo5rhYhfb19Vp3TWvOJn5TEvkzryx9YAkmBulYFd0Pe9h1STXG6FGSRoEkBF2uybbqiQu3Vtqj5gkDdCkN1AVwnUZal5vZGR3HdPp3UWvulcoJ9PZM5Q5ndkZce10IFc90gTfdBmO3VE2RGhOGt8qZPzdMp2TXG/RGTO2StdP5bJs6IWR3yPsrDMQxKMZYsQqmDsJCq+YJZgkxE+qz5+q6Maxdos6tfIP3rFWaTiaspieZRQTBxu/M1ZN3dv4p9Pdc89WGyOOUXDOWGOcVlDuoftKpqemdFT4ZVQPtoeaXNv+imoPtOhEh8cS9ltAWWN1wHl7pZQF5Y7b00Yzjx9XmjotEYzHF6d/PLhuq8/hoS7sJA0qDFOHOCeK2Q1dZK+GRgNmxz2cPnYarzPJH+QIuWAczCWn2K5f5Vqxm9UzjPxiV19NHT5xs6fKcMo98T5u+i+zTgV7xDFXTvc42AMp1/grP8A0R8NdIBJ85SvPYquvgcHxYhOxw2Ictii454moI+UyqE1ze8ozFct3Q9VHnq2zP1l6NvU9zj/AMvqMp+sy6o/ZPwy39pg0p/sVhKrYp9lHDUlDJHhtHU01SfQ+R5e0fMLNi+03iAeSakp5Gnpstem+02mETfGUs7JOuTZePPR3HVWXKZ/W3n/AB/etMxnG7Ka+d/s5Z/2N1Z3x6lb84nrDxj7N8cw2qbBRNbiDS3NzIdh7ar0uf7RcOmYGw1ssX9uO6yMQ4kkxDWGuonk9cpYV1w6juWM3lhf6Pp6PajvWvPlsxjKPhUR/R58eAeLhvgk/wDD/Na1F9l1bU0jaiuxamoZH3vDK05m/ouhp5MbqphFE9rmEdJ2gFFz56ebk1Eb3O6W1U39y6zDynDjP0XqvbHuc48ccIwn41f9bcjL9kbLn/8ASSL6ROQj7F6edoP88Kdt+hieuzFdCD5w5nzCsRT08w/aNP1Xj/zbqvj+0Pn/APjDu388f9sf2eZ439i1fhr4mUdczEhICSYLtDfndZMn2V8Twu8nDNXO3oWuaT+i9l5UTttfknaZo/2cszfkV3x71uiKyiJe7V7c9dhhGOeGOU/Hzj+nk8GxXh3H8CY2Wv4VxCnjJyhzm2BPZYE2OvgeWPwmqHuQvpmaeqmYI55zM0G4EgzAFQvcHxPidR0bmvaW3NOCRcdF2198ivz4efyfT0e3uPH/AF9Pn8p8v3fLtTxjyHW8AT9FhYvxFFiLS1+HiP3K+mn/AGfcLzMcavDY53WNtLarjZ/sgoqgvbLgNNucpZL06L36+59Ns/6q+r7/AE/tX2vqL/1ONfzRT5uleHuJaouRJfNl0X0P/wBAWAVRdneaR39olUca/k+4dQYeauHHHyWcG5GXLteq9ePX9PlMRGcPVr792zblGGG7G5eDiInYKxFSv6tXpH/Q1iLj/Qa58h7OjAVKu+zTibDZHRvMD3AXLbgGy9OGzHLyiX0cN+rZNYZRM/KYcbHSSRvD8vlG6l6eVbMvC2ORPyPw51u4N1A/DJKWQtqqWSMd7JL0M3mSAqCokMnrKv1RpWRHltOb3WXPJGXaSBSIZCyYRm5Fx1TyVcThYQut3uoi2/xD9Uwib1eAtVArtcC4ZSpw17vSFbdhQhkINRnkHYKu5k8UuR2icoEWSbNY6Ig50ZHkujla+MjLmPe4Vz/R4RJyiSR2UnKBTzTveLwkD30TySPz5uX+ik8RNqX0xA72UTKkvdYxu+jSsxLQc8zj+zsgLNczleibIQXGnccw6hQ+EqXn9hIPmFq2EEZ18iJ1HLK0NBDL91oU+HSRRF8zN0fLPwRud8mlTkM2LDYmOzuk5hHUaLS8NJUUTmRUvOtvp6QpBFIIHllFMXPFrBpVvCJK+mpOVJSNzEk+lZt0c9S0bpZQ2OF5+eikZhdRFWjM3ddFDh9QZ872ujzdlcGHQZg+Vpc4dSVmxzVXh1VTETzMs1DS00xqGNt6zZdDVUVLKCJHyuB6BRUeHNjA5YPlWrSjyUVU0mOHmSHtuFn4rAaekMzmZZAQCtUOrJH5GVYj+YTTYe6pblrfxvclZt0pXwuOGpwiN00Rde97KCpqxS/gQxFrR3Vs0kbfI2sfB+5GVBLhkM4tnnf7k3KSzMJqWukqjy2TtAtrdWGyx2yPmDrdlU+56SJmZnNzdibIoaMwZZpI7NPvdIUjVsjeGRh2YndwTEyOuX1AcSeivysY5jc0MNiNC8o4hCyERs5Ad1y9VRQjjYTaQ5m9kXIp3zkRQH6arVbhFE6+eq8wbnIZusyOOqLQIY5Mv5nHRBYjYYmASQ3HWzblSB8LfNHDlPu2yqNNdI8MZVW730AQRV/KqRDJM17hu4tVEhbGA4Oj67kKMhnZXKrkzROZnFztZVI4uUBnuWjqeqgsNlglJ5MPLIG5UMkTZAXysBPcaFTuoy1pmfGSCNMpSJZCBmY+3dBBFB5bZBE5h69VbIBLCRENeicRYUXXragt0uGgdUIie88yP0IJ6eqdBM4uja5lrZioJDhz3kkyX7N2QCeMOIcfK3dQ8yOSX8MWzdUEpkjADY4ZNDu4pMlfI7lsHKH5iUm+F5dnzyEqu+ojiItmk+SKvuotMxrmi/dRihY5pE1QD2LNUqceKF2UFQXdhGSrwwbGKoMbR0Tob+oyt5YH1RFRsNPA20lVUZOtgk80j9GRymPpbRbFLwVWn8bEMTiFujSTZXY+GsIDQyesllPXzWug5KSJglyMMjY/zFyU8VNC0FlaZHONrBdwMIwT4aGB39oXJWbxp4aKGipYIY4xmzvDW27LbDHoBZ4AbqdAO5VyThXH+c6TJEM5vYu1Ciw6HnVULG6XkC7N1PMf+txrWbnHq5T+aVWP9Kr44z2bqpqXhWgju+armkf0cwAWXQGlmP8AXRu+qcU0jfNYLLoz6TCMKotYKXmOHWbVXL29EELPlGpuRIfhS8NJ+RYVFzJCLOfYdhogy9y8qQwyW8wQ5JrFoY76IoeXrqz9UbWjshz5T52uv8kJk12cPoiJjta6QDFW8QwHa5+SXiAT1A7BBaFj8SceQ+pUm1Nj5hr2SMxPma0j2KCwXoc9lT5z+yjfVlps4FGmjzvn9UxleR5ln+KmJtkP0SArZXWYw29yjK4ZnD1CP9UDqixt5folFhWIu89xbrdCaShpjzMSr4mX6ZgiBdNI/wBOqAF7v6on5IX8RcM0kobRk1EvRrW2P8VFVcX1sTQynoGU19jLqja9HhlXK4DkMF+pcpnYLBF/pmIRxDqL2XLz4xxdXSGGkr4I77iIAFo7qjV1FTJVeFq30kjmD8VznZuYUHWOxbhGhIhM/Pf2jJddKTi+hjdy6HCzboH+U/3LnGiCO80eHtLma+QW07oGzyTTNe54Eg3sb2Qa1ZxHjtYCI42tZ+VossmRlVWzf0977DVt9bFSlr4pWyskOa9rblBI6ojnMj5HwW2c+Kwughgow2Uv58rR0aNP/dTmQvaxjxG6QWZ6d0zalkjw5snMtuLdUqqSfM9jOUbi1wNQstCmiijhMnZSclk8QLW5yRclVyWx1WZ4MkMYuAepKkiqaolzIYyxsgsT0sgAQchpfktkJGXqfohFQzPZtK8X7NVh0PVzwHdSVfgpKiYNqS9rWMFs5OoQmGU2X8Qv5cmgvayOSrh/NOrLad82ebkOkdewHR/zVh0ZicyQwtgLRYGQEgolM+GGesqIoKISScy3mdoArNRh0MVW6hjq55ZmC72iIAab6rRggraiXOyugzACQCPc9lDLgdU5xkvLHKHZwTu1EZtVTzU03h46GaSXJn/aAG3ySjw3FZGh5w6Vo7OIH+Kusgc175KmQZ9W5jJq0eymw4Bz2xjNNfUykkC31Rpjkvj9bCOmqlw7EcOdNNS1LJDKNo3Nve3v0XQ12I4XC8mSmNQYthp6uu6s8OtxfiGsbQ8M8EzGpcCeY5haAB+8mWUYxeUsbNuGnGc9kxER8VXA6WnfdxwuOJwFw5wzX+q0KylgDHQcgOc7RuVttF6dg/2D8UV9AyXHeIjhEmueGBge4fJwXV8IfYfwhwxXjEayomxWZtjG6qv5XdwNl8rb3Xp8ImpuYfm+q9ru2aMcuGfLKPdET5z9fR4DgnAHFXFVc2gwjB5J5GsJ5tQ4xsBG13EfNesYF/JnbPTwVXFmPSNrABzYKZoLf7JcvaX4jTxuMcJYQNsjVX8XVSHTRfJ39337f4Pyw/D9x9s+v6ua0f6cfLzn7z/Zl4LwBwJww6GfDeGqOGpgtlndHd4PfVbE2MeYsj/Et2UVSX5Q+R5IvY3UTo4ImGaRwawbucbAL5mWeWc3lNvyu3ds3zy25TM/ObM+aqmcXWYL9kxhkefxFUn4iweimZAc04c3NmhGYBVRxlBOCKWncwj0ulbr+i3r07Nv8GMyYas9n8MW2RCyNud5AHcmyoYhxHheGEBxfK4mzRG2/wDFclW4rilbUmaSpkLRoGF3lCgnNXUuzzakC1yvraOzZzMTtny+Xq+jq7ZlMxOyfJu1nFks1OeRI5hJ0aG20/tLFq8Zqae9VE6Ykm2Vrr2usqtxrC8OAa+fmv8AyRi5XO4jj1ZiTSyDmUjO7TZ6+t0/QaNH8MX9fN9DV0mrT6R93QV/FeC0uJtoJKxrpCMz3D4XdlSreMHhxZhtK7TaV+n8Fy8dJDHd7I/OdS8m5J91KM6971iqJayvfzq6qkmde9jt+iKNjAMtgAEzd0YapEIe7U1gVJltoWp7C3mVpLAMqk6jKLBIAdEtPZEDp7JIUloO7ZOkkgSSSXX6IGbsn6JJKhrHulbTdOkpZYUSFOBdAySSSBJJJiLoBLwPUiBuoyNUbSAe6okDvzJddNkOl/ZODZQOnBH1Q5ilmKAzoPMn+SG+uXonJsgdJDd1051KBzskzomB6JX1sgK2tkySWx1QLY3SG6a+p+ae47oH3umPZPca5UJNkSDpJJIpzfqn6WG6YbFO3ZQLU3CDlm6N290Tt0Ec1PyC38RrswzeU7LtuA/tVx/hLEGOmrJKqiIDJIZjma1g6hq4kjoU2ULxdd2/pu46p09ThGUT94+k+501bc9OXLCafXnBX2lcMcbx3wuc01S12V1PUHIb/u+y6+qoaWugdSV0DJo3etjhoV8P4biVdg9R4zDqh8Eti3M3ey9a4H/lDYjhzabDOKIBWUrW2lneSZL+y/kfe/YLq+h2Tv7ZPLGPOIv80f3+Vefyfe6XumvZHDd5T+zuePf5P/D3EcdTXYFI/Dax7fw4oWtEWf5LxLG+DuMfs6ro4scwuSoYwEmejYZIrdydm7r6p4X4vwHjPDW4lw9X85pveKTyvb9FqzQwVUMlLVQNmilGV7HN8pC+R0ftT1vQT+G67HnGPl5+WUe71/4n7sdb2TpOux5Y+U/GHyJhvGNPP5KgEvb8TdV0dNW0tXAHwzNeD2Xo3Fv8nrhfGpDU8OgYRLlLjHDYNll6Xv6ey8Z4k4Y4s+zkwvxqgLXTmzZIDzGkd7r9j0fdOk7lUdNneU/9M+WX29/6W/Gdf2Pquhm65Y/GHWwYxW4K3xFNVObGN2nULqsF42wyvY3xrOS8/H0K8ohx5s1E+LEoeU5+Uh52HzWpg1bRzULMtQHPBI23Xryx90vj3T0Li3gzhf7QcNFNXwwvLSMtQxl5GDsCvIeLf5NFTCWT8G13OuCZGVLg23yK7GlrKqklEtFOY3duh+a2qT7QqmlfycVw0SM6yR6W/VerR1u/p6jCfL4Pt9v9o+v7ZEY6M/yx7p84fKVbgeP4LJJFimETxtYSHOfG6wt72Wc18T3Zg9o+a+3n1WAcS4ZyZeRNT1TXNdFM3UheS8Qfya8MqjV1uBYy+lebuhpwwGMdhcr7Wju2vZ5bY4/u/ova/bzpepvHrY8Ofd6zE/t5PAAyntctAf3CAtIPlyuXX439kf2gcP0jqvE8JMsTSAXRPz7+wXK1FHPRWFTS1NM87ZmEL6eOeGyLwm37bput6fq8eWnOMo+U2ivy/Kyblnt0TluY55Iw795pQF0m72hze/VMHR5SWVHLN/SSuj0cksbpgM0czbfkeUQDvVI58fYt2UAkp/6+MvHcFCZmFtqWraQzdrioq5mk2AZMO4OqiMkwJyfh+z9bKLxdCwHnz8t3cOUUuLUuzaoTj5aqiy0bvkZ83MTaP0awSnsVUGOUA0/Fi/tMul980d/9Z/ZFkF6xZpZ0bexOiYQud5gyN47hUvvyhGnJnH0unGJwn00M7vkEGfiTf6e5g7D9Vsxk8qM8si7BoNVhYgaqorDJDA5thqD1W1QStnpw98clO4eQtvoT3WnHH+KUlnuBFmn5HVOGNbuXfVP+G/ctcR7WKWW3la9wH7yOiRu3kB+YSzE+ox299Cowxh1zC/dp1S8p9c5P9sLLUwPy/E9wHUDUJrM6MDr+9ihyjcNNj1aUrOufxB8iLFaZW6OtrsKq4q3DpX0tRCc0crTYtPcOXb/Z99tvG/2dzzvw6tfWQVBc51PNIS3OTcu06rgQCG7uA9jcJgRuLEe68fVdD03Xa519RhGUT63/APLc89OOcVlD7AwD+WJwlHh9PJxBBWNq8g5wjpSWtd2B6hd5g2N/Yv8AafSxYwaHDXy17S4eJaBOfmL3uvgQuf6XXVvBsWxHAq+PE8LreRUxXySNdqLr8Z1XsH02OM59t25a8/r5fTyqav6vndR2jRsio/d9q1v8lzg6tJmoMfr6UPJOXyuaFwfFv8m/irBS+owCoixKkjjzOc94ZJfsAvG8F+3T7R8KxOnrpeJa2pp4n5n075LtkHYr0ym/ln8TU9my8LUlSB+aZ11546b2w7fuxnDZG7GvS4r9+M2+Bu9mscorHH7S4qpwHHqQllXgWIw+76Z4H9yzn/hvMbwWuG7XAghfS3Dv8qn7OsWpKSHHmSUlXU5WvhbEZGMeTa110XEH2dfZRx2yeVkVHBW1ABNRG/lvH+yvVj7c7+k3xq7p0uWvGff5z7/pFx9LfB6rsOeiPLLz+fk+R7XCZo6L6BqP5KsAF8M4wc5n5JIx/fdea8X/AGP8dcHxyVlbhDpqNsmRssDuYXdjYL9J0PtT2ruOzwdG2OXwm4v6XVvk7Oh36seWWPk4hGndDOzMZYJWFu+dpBCA5V+gjKMvSXnLMUySSqH190yfMUyBJJnbJhugTt0ja/lTtTq0wBJFmTZfdKDX1snuQMqWUplFFm18ybMUrdSmQFmCWYIMwSzBEHmCfP7qPME6CXOlnb2QX8qWwQodwkCgGycCypY02qC7kWYqLZZk53Caw7plYgiT2B8pFwhEUf5AnzXRXb9VAwGX0vcPkVNBiGI0sgfTVcjSNupCgSSYifKU9fJoycQ4y8Wkmhk93RBQMxSYPDpaeF5Bv5dLqs42Q5VynTq/lj7J4ev+WG1LxG0+jDeSP3XkpocfYZmiSpMcd/MS0k2WInyleee29NMVwcZ6HRMfwuoOJUx/Y18Tv4JfeEugZDzcxsMtlyuRv5QllHw3H1Xn/wAn6f5/dx/yzR83ZvdURX5kDtN7EKNlfBcNMmUnpouSL6j/ALZKB81LT11XTvbJHPmc03Bc29lwnskV5Z/s809qivLJ1hnppNzr7hMOS7YNWA/Hal+s0DHHuNFA7Eczw8snZY3s1+683+S7v5o/dw/yvb8YdI+GA+i4+qy6zhfB6+oNXU0kc8xABe83Ngg++qZw0ZKz2L7oGYzIZGtHLyE6nawXHHt3V6/zYxX6sYdL1eieWu4n5T/YbOGcOZ6KGL9EbcAw5nrw2ld841Z+8YmbSZ/kQoJschZ64Zvoy65cOs+GX7unjdw/mz+8o/5tcHyf6VwXh03e0dlhYx9lf2Z4zXGrlwSegYWhuSnAy6dV0sOIU08QmYXta7bM2ylZUUMps6dpPzXPDqeo05eWUxKae6dw6XPnr25RPp63/VxR/k//AGT1INsQxGC/7qo4p/Jl4GfRPdgvFNS+qFsrZyGNP1XprI6cf+6PJTfl/iumPc+qib5vbh7Ud1wyjKd0zXumIr+j5MhpcOgZZksV/wAx3Rsip81xyyO9lmzOjFxywEoWczZxA9l+vp/b6atqT4+X+icCkvcZT9FnyRyxNz8skDorNJBJIwvlAjHbZRpZeaJ/mMMZPyQSTUsQuYI7ezVXLKVl/E1R/shQc2OoBEYeGg9QirYr6IjWJv0amNbR9IL/AEVMwEANYw3907G8s/iRj9Fq0W3YlRs9cCc45CdGQWUbBS5Q98LQOqdsVA2XmRwF/wA1kmBNxnN6ICfkEf3nM/0Up+oTOxGR9mQtDW9g1MaOSdpnDx8kUfiK5/8A1Rw+iaWeaGwns0kaDdCyCcjz1El+11Wmhm5vo5jhtcoJxWSHyhjD72ULn1D75GuBPZTiJ/LbYDMegTRxSR3M8JB90RA6B9rvms/5oKeJ87+XPVSBqvMpxVAnmRx/2ioTTubKY4auP3RYyLwNK0XhqHF3uE8OHVcj8/3i6IDpfdCIZA8PZPJf5omw1r9Hykt7AoT+ZLK0B2SeYOHsoqeJjXZ3vOXudUvD8r9pERfrdXpcPfFQCTmFg6AKwiqYmftOZmb8tkF42vysb+gUVqprQyRhcCbbKYOqIXg2Fh0KqUvUxIppHseGvfpf2UA0bkELs3uU0s14c5LQ6+oCGOujA8pMRIt5jdZbAIpIxnkjF+yOKeQt8kEYI7tuhs+V5bC8yuP5Qp2YbjMxyU9HIb9SLKwzMIvxXfjSDRHG17/O9l4h1V+l4V4gltHUcqKAfmdqPotGLhCkj0qsXcf3WgAJaU50uZIPNVNGXsVCJmNP4jw75G67KPBOHKdoHh3Se7nFTRR4NT+alw6MO7nVVXDyNqK2Rvh6WaRo6iMrYg4VxWZmeOlmj/tGwXTDEKi2WNkcbezQmMtTL653fqgwY+DKp3+lz00I6kG5VyLg7AWZfE4jJKGfC2wBWllJ9SblR9WIAbh3DsJywYQ1zm9SrEc8EQywUEMXyjCHlt+GO/1RC8Yy5GAoyc1tafjt8tExmqD65pP1Rcxtg1OMiCER3F3En5lLKLbBS3NykX2CgjDSDouZ4rqH1WKwwOFmwMt+uq6iP8R4bdcdikvisbme03GbQreBLRwRgFZC648tzqukfKXHNmWDg0TPEPnftG3y/MrXMmiufqxCYSEblMXOOucqAzOHQIPEfDlA+Sw2s8149DyEhUT/AOtcFTM5+FyjNQ/uiNHxUx/rP1TfeE7RpPHbsQs5j3SGzTr7pnv6ZwSe2qDS+85dxyyl97yndsSzOVUu/Zxl3ySFHVO9bLf3o00TioG8LT8kxxWnZvEw/JVWYM0nNI+Qf2pFBUTYPQeWSbmOHRlnFEaAxOmk3w9x+idjoKh2UULmLCbxS+JzhTYW6RvwlzrW+iCbinG5YfJyqcdg25/VGnQS4W6X0MLPmVnVYwfDmh1RijHSHdrTmIXN1NXiWIxmN1RO49Xc02AUNFQ1EFQ7kU0MrCLF8vRBuycV4ZTtyU1HJO8bEbIRj2LS2ka2OnYRfQXI+qCNlJSwcpmHwvkuSX9B8vZUhBKHl/ipGOOwB0CxyTikqq/EHy/jYjUFvZrrBZ76Jhk5zJn5ugd5grkhsWvMhlB+M9Vbgmljjkp4aWFzZxZxLdfonNtkeHfHKGQSZnkXLrKSVwlew1MzpDGNAehU8cWWXPH5W3AspDI0EhjQcvcKxKUr86CVlpHiN7/VYbhTSQYcAMkHOcfhyapRSuY6zHZXJzJkl5wBLhtlFyryUHMDnefQPGgGllIBTwtbCGeZhubKM1N6rPVQTxtG73N2Vrl0YaX8mZtxcOcdSokoHTyCUTMNgNipZq99Uy0j8zu/uhe+E0roy0F3V1tQVA2SmbvTNdfunJE4pBmdJlaGgauDt08MEOXMxpsTYucd1TLIoySyFzARpY3sp485jEbpiCdswsnJbHd0ejC35EXR1EhDIm8+Vz2t81ugUJieJW2/EPUN1KOKJjA9noLzch+hCIJ7yyJoiBlcdS7cgJMZnOXMb/vOsAULpo43lkEQaL2Lgd1MylkmguwXbe9y9aW1uGXl3ia9spZd4tpqrML8Rq4Wnlsk5rPMOg+SgoaNr6NwqmOuw2Dr2JW1gVJWVr5YcDw+fEJIA0mOAZnWWJyjGLmUz2Ya8eec1Hxlmww10d6elpRHfVz2ncqxSR1sLyZ3mTSwu7Zd7Q/ybeN8Ua2tqMVpaDnAPMcoPMaOxC9FwP8Ak98E0FNAcUfVVtayzpZBOWxucD+Xsvn7e6dNrjym/o/NdV7Wdr6aPLPlPwiP70+faOjxrGcUjw3BcONRUS6NETA669J4e+wLi7G6Jk/EuIjDZA4gQNAJB7+U6r32DDsJw9rfCUNNBk2McQbb62SdiH+rdqvk7+87c/LVHH935Dr/AG36rfUdJjwj4+s/2/ZwXCf2EcIYBEX4xAMUqxKJGzTXsLdMq9AM1BRuyU9PTsDf9UwBU5H1ExOeYi/ZM2G3q1Xy92/Zvnlsm35XrO4dT1+fidRnOU//AD0j3LMlfLKcsceh7qB0UkwHMJKnihu4WCrV2MYTh04paura2Yi+QC9vmueOOWc1jFvHGM5TUQkZABYZVYLWQjPK8Mb3JWBV8WU7IHmjAvazTe7j7rmKnGK+tYW1s3NJ+LYr3dP23d1HnVR83q6fodu7z9I+bsMRx/DsrqWGVxk6OIsGrlcZxOvxRvJlxN0Ia70xtaWkLPcGP8xGY9ybqKaaClbzKqaOJvQuNl9nR2jTqnll5z+z6unt2rX55ecrLpX5WxifLYW8jQNEDmsa0vlqZw0bkuACwKnjCnZ5KCDmf947QLDrK2rxGTmVc5f+6NGfovqa8IwjjjFQ92OGOMVjDo63ijDKVxZRB1U8deYci56rxjE8QOWSo5bPyxkgKBjGj4U9m/lC6U0ANZHbRFmb+YfqiylM2IXzOaEpgwa7ujDHKQDoE4dYaboEGEIg1MjS0nyCBc5kSH91PfWyEmdumRoW7qxJYLaXT/7XVMkqokkkKBJJ79ksxQNbS6du6ZJA4OuqXluMqEmyV9Lqh0k3VLKFGTpJfNJGjeW/unGqDrmSzWQP5vhch810Td0jv5UD3JtmT9PqhAsb57+yIh4JGwQSoc1gkNHeaxsmsSL2WTEiQDuN0/X6prN+K31SJ20utB/3uqWYpi7ylNmCArbHulmKZPlKByL6JuuVMmyhA+hCfzWTJ/8AaWvRPUhmG6cXATakpllST3KZJAWp2T3090OYp8wQPYWJSI10ddMNQUtggXT6IuvlQ/7SR30QFpZAQL7fqi9O4TZgbj2UoaOCcUY9w1I+XBcQkpnvFrtOoXt/Bn8o+mncyg4sgcxjIxeqFg8v9+6+fMqbJrqvzvePZjt/eYmd+FZ/zR5T/wC/629nTdZt6f8Ahny+D7owXHMLx2jixHCK2Kqp5PM1zDrb3CPGMIwviCjfh2L0jZqeT1A7/qvlH7MMaxqHEG4bTcUUuGMa9rs84Ljr8I7XsvoqHj7BMOk8DivEtDUTxGzmscGuv7r+Ld39n+s7H1sa9EzlPrjOMTcRHvmo8v0l+j0dXr6rXeXl8bef8c/ycvGR1NVwXingyGXZRyBzw89dV4ziWH8UcDYoabH8MqMPLWBwfIM7Cvs+kxClrYm1FJJmjcNxuCquJ0+C4rTy0ldTQziRpY7MwEgexXs7d7YbtMRq67HxMfj6ZR+vv/Xz+b5fXez/AE3VReEcZ+T5Mwvixs0bZqidjmkXD2i110EVdDiNOHwvzArt+Kf5PXDElBLVcIPqKPEmi8eeUyRuHUZO68ZquFvtI4dM0uK4LVQUrNp2Ma1jfmv2vRdf0fc8Ms+l2ek1WVRl+kX5/o/I9b2Lqej86uPk6uPPSS/gSPjvqcpsCVchxKqicHx1UvyLzZcKOOZKf8PEstxu4ssVsUnEeG1AaWPcA/UEjde2dWeH8UU+LOGWPnlFOrPH7aRvKxahc4fC6N5LSEFazgbjmKKKvc1jm3MYfdpH6Ln5ZqaoIY97dfdVqV2G1RMd7OBsNNUwyy1zyxmpdNPU7unyjPVlOMx74lTqP5M+FYhM+qpuMIoS5xLWtBsBfQKpVfyVazw8jqLi2nmkDSWtMILnHsujhZiNE3nYVipGf1xO8wcrEdbxFK98k2IiF9rgxEgH6XXsjuXVR/1ftD9Hj7Zd4xr/AFP2j+zyGf7APtKgbdtAX+wACqy/Yf8AaBEzmT8KSu7uY8L3CDjDiaJoZVVQlYOpaLopeO8WbGX0tVkeNgRcFdv843fyx+763/8AIXcP9vH9/wC75txThLFMBqxT1uDPjlDQ4sk00VAxwB4ZUUAhf7ar6bi+0LEo3ZsRwuhrmEWPNiF/4BatLxzwVV/6ZwlSxDv4dh/wXbHvXl+bDz+r6Ov/ABFiMI8XTc++p8v6S+UmxnXlwNlYd8zLoHHDmHK6lML+4bovsRtd9l9ZFaTCKRoeCP8ARm/5LnX/AGWfYdUudI8vaXOLj+O5o1XXX3rVl/HjMfu+j0/+IXRZ34+ucfhXn/Z8zNqagjMGRzt+ajdNA5+aR7oHdgdF9CY19iX2VzUoHDuNeEnLrl0s7ngj5LmpvsY4fheWP4woC3rmYSvVh3Pps4ucq+r62j2z7Ruw5Ts4/KYm/wBreRhx/rntmjQmelb5YYzGOxXp9b9iGFtpZK7DOK6ebLqIYyQT8sygw/7BOKMSrIqcvnjjkdlMsjAQ33K9OPVaM8eUZxT6Onv/AG3fhOzDdjUfOv2nzectlkkvnhje0bOabFGx3RlS6Mflc/Res43/ACY+KMIpBU0WMR1zi4N5cMRzD30WB/0FcdfDQOd84imHU6dkcscod9Hfe3dRhz17sa+tf1pw34IN5Ig4dHM3Ttc0+g2/8RdufsO49j/Z4fIPk0oHfYz9oTfVgznfMFb8bV/NH3df816L/dx/7o/u48CQeZ7GD3bsk92fd9/mV1zfsb+0GO5GETtPdpSZ9kf2ij1YE5/zTxtX80fc/wA16L/dx/7o/u48AD0tP0Rh7/SQ3/aXX/8ARF9oZH4eBSMPcG6AfYz9pBP/AMqf/urHjav5o+5/mfRf7uP/AHR/dyNnDzHZL8MnUB3zXYO+xfj5jHSTYfLG1gLibHKB+iKh+yDi6ugZUQUrqiF+oLYi4Fb8XXV8o+5/mPRzHLxMa+sf3cdmAHkuPmbpczXzOB+Qsu8/6EeLBqKKsH+wVM37GOIh+1wyukPfJZXxtX80feGP816L/dx/7o/u4OmqWU9TFUFzzypGvtfsbr2rh/7YsExFro8RcaEsAawuebvXPUf2P4vHLGw8P1Ru4AueLgX6r0DB/sCihroZsVlpJKYG72NBa4/VfN7hHR7sY8bLzj0p+X9ouo7J1WuPxe3ziJrjNz7vh5fd6Twbxnj9M2FrKh8tGPPlePUD7r1nDccosWDYo5GPkIzFhF7LzKGHDqSFkLZGNbE0MFndALLTwHiDBcGrvEVFQ3KW5Sc3p91/NO89l09Try3aca2RcxXv+r+ddr7zt6fdGnPK9czXn7o+TseJ+BeGOMsOkoMYw+KNz/TNE0RvB6L5j+077G8Y+z+QVLKrx9E8F3Na2xjHUOX0TU/aZwmGlkGNUjye7kFDUcOcbxy0L6yGrY0XIabhp97r4fZu8939np58MvCu8omJr7zHlL9p1HTdP1flccvc+NA24uCk4Huvovif+TJR1EfO4VxlsEz5C+QVJLrg9AOi8q4n+x3j7hiojikwiWtY5uYy0zczflov65272t7V3KIjDbGOU+7Lyny/b93wdvb9+n1i4+TiUk9THNR1D6WshdDLGbPjeLEFCHsI3X6XDKM4jLGbiXimK8pOWpy5Pcd0GbzZgq0FON07h5k9m2VAJJEflSS2T31uk7dMklNGuEYJQO2TgWVtkkkiLpKBJnbJrlPf2KBuiSe2iTt0QsxSuOyZJA+YJ73QpvM1FGCeqQOuiYXQuLtwhKTMUsxUeYJZ9bIiXMEgbIL+ZPm1QoWYFLMFHcjZPmKCRPpZRtcEgQijSQ50s7UZoi1DlCO4Tae6KbNdJK35U2UIp0k9tU17m2VE5G/VS8+a1uYbKNODoixKTxMzWlmfyn4XapmTRtkbJ4SFz2m46KLKEsoUnGJ8phZxifVoHFCd6Zzf7BTfeIO1fVQke11RSXm/CaP5I+zl+H1fyx9nzuGsL7uZcdldhpXkXjgZEO5OqihkjkaAL37WRmGSxJzD2JsvU/spTU8+761gb2ChzUvpnM0juw0RNFVG4COOK3vqrRqn5CDy79lkVWwQ2zW/VFy2RjMnyukGjwD2SFK87kn5IEY31DLMY8e9kzKJlO273uLvdSQwzxnzzuDewVWV7GycvPIde6Byad5tPqzqjllaLcj0dNFYghjhfncwH5oqmlcXcyP0nZBRbYuu4G6vROaGBU3GqD7tkBHZWWteW3zgfRAbj5S5qqse2SXMG5irEXNDsskgc0nsk6eJr8lw0d7IgOVUSSDIwi2yklhqLBknTqmL3u80clx7JGoyX5z7gdkDSU0LbERXPU3SbCzMH5Uo3uqXZIaedzvaNXmYDjtTD+BQyMH5njKEVDFTvldYRgDo4uQGF4BZNUtb8itCl4HxcvbPV4k2IDpe60/5n0Dx/TcUMg/KywCDmOWzmsa6cSNBuQXKxWSmve3wkgGQWyM1XSQ4Jw3SPzx0jnOH5iStKKWiaMsFNHG32CMcnEDBcYcPLTTOHQu0U8fCGOTgSPMUTf3nahdp4of+4TGoD/l7LRbnW8IUrW/03E2l/wC42/8Air1LgvDtPb+iiZ3UvV9zad3raSm5NJ/qb/NG7IVVLAMlLQQxj2aAhNdOeoCn8NTO+D+KfwlL+Q/qiKLi997yHX3UfLZ+W60X0lKfiePkoxRwOF2Oksss2psaAfMjaxt7q34SP3+qbwrOhQtDlb3TjJ8Kl8Gek8f1KXhH/BJGT7FUR2b+dBzGZsmcX7J3YfNmzGMk+xTOpqkC4pde/VReSUgDrZI6lVjFVk6xvP0QvE7fKYyPoqidLP7qo50g62+aA1OT1vCou5/zOCF0gta6oGoPtZMJ3W6I00I3DI5/YErjKYcyZ03dxW9W1j4aKd46sLP1WJho8oXXFyyb2HuEcT7/AB6Kd0zR1VOnjncLQix7qwMOrZHXcHSdw0Lnn6tY+gnVTB6X3Ub57uLsyldS09MzPVhsP9tyifXYDFE6SapZMB8Mf+awtBE/7ykjikndlDSfkLqg7izDYLfduBGSXvJsFnVHE2PVFxDOYR+WJtl0R1BoY4xepfE0fvusqkmMYDRZg6pBcP8AVtv/ABXOVFMcji+aVz/zO0QmBkkQhfzCOzVhW2/iyMShlFTf7cxsEq/EeIoHtGcRgi5cwWWaXzZ2F1PGMgspqXEX8zkv5krY/g6KNBdJSTOE+JV1RK7q0dPqlSV0giApHtZIPytu9QutMXiR+5umpInxnnUvq1YbrDQZ3TTyB8j+XID5gOqIsc24YwZR0UhiljLnywtLn7m6lh5TYJKmocewA3KsLRQ4g6nty6WIEb2G6jgdztbct5JOZ2gSGSRokyW7lSCfINI2n6KFGdBUyPb5SGE3Eh2KCYC7mMdd1rZhsrVPJnjylpJB0BUfi8Oik89iNiAFlaNUwsa0ML43uP5TewQU7HwPjqLdbNKjlqfK2HlSZjch2W4spKKR5jAq6Solt6bRGyrJ4pXuqHyPsTqUfNEoDswjefitohrBJbywR0t9Mg3t7pqnkRMYGi56nutUohBG1nMMjH6gEt3F0hJHDJ+G6TT8yCAiO/LqBCOxBIKUrTI0zCVkjRo5x01ULSHmSi+cvHulG+eN2cAFo3uEqekqKocuAhoPxE6K2zAa6PzMr4Wu7lENSUZrmlnOjzHpfUBQSYdBTy2fUXcOmX/ip6jCK6zc9fG4k2u0ooOH8RglaTiMIa82PnubfJIlZhm4jSSVsrWUzpHFhzNc0dO6g8NXxPHNnMg6grp3YPkc2mdWzS9GtiaACuji+xPinEIY63CqPmc5oc1s03T3HRYz3a9PnsyiHl6nrem6KInqM4xv0uacFEyo8rIGA5QSSBZTw07aqqy1ElzJa4adrBercKfye+OZq+P+cLoMPocxc+Rkge/9F7BR/ZDwLQtYJcJiqbAAF2gK8O/u2jTMRj+b6PzvcfbDoOiyjHX/AKl/y1Ufq+a+G+A5+Kal0GEYcXgC7ZHOc1rva67ei/k98Z1csXObSUUAIDwXuLrX1X0HheG4PgFL4PB8Pgo4Ll2SJoaLqeSvjHx6r5O3vW7LKfD8ofkur9t+t2bZnpojHD3RMXLi+GvsS4OwBrhVRPxPOACKmxA+QXW4bhOA8PtMWE4XTUTSLO5LMtwkaurkPkICHlzSavIJXzdm7Ztm88pl+U6jrup6rKct2czM+vmsS4j5g1jCbqtJPVSSWBs22mikEbQibHmcABqubyqrYpJHZXkomw2NgnmxDDKJhdVYhCxw+EOBP6LkcQ+0F/OdFhFBmjbpzZRY3+S9GjpdvUTWEO2rp9m6awh2MphpG82rmZEzbM4rPl4mwqz2UM7Kl7QTo6zFweKYrWYzIJK0vIPwepvzUQD3egG/uF9bT2W8YnZlUvpa+13jeyfNrycWY9UvJmEMcROsbBos6rnNZP4h7Gh1g3RR+FrT8F/qqlbidJhvknma6X8jDmK+xq6XTpnlhjUvpa+n1ap5YRUrhkmtrZQz1lLRjPW1Ucft1/RYFZxJXT/6JC2BvRzm3KxzCHPMkl3OPVxuV3dm5W8VvkaY8OjyA/1jt/0WM+SaqfzKuZ0r+5KYM/K1G1t+i2wEMZ2TgBSW1skGIAAukBdSZRlSjIeCRfTuECy2KW4REWQBzyT5PKgMHROmIJ9Gh7lP5upuVYkCSdkkklQ2YIm7pk4Nh6boDdslfWybsmdceYbqASOhS2KVyd0lQ5OiZJJAkkk19NUCuEswTZilmKMnzBOkkq0BFc2zJ0G92pdp6DTaAJ0lFMDdJ2yaxS7ZlQmk21Q3t6jZIlua2idISTjXzNCcO7oczb5TdOwEktLktRk6lLMkNymJuokCI/eTN2T9PokD8PRFPmvpZMP3kI3RIGbuETfUUJuHD5ov/wB5AgLEApagpa/mukHXdZAtb67JfvJ97ocuqBw5Ly9Uyct0QLT3TJJdfogbME6WWyfUlASa+tk17gfJMgI2ttum190/X5JaFAhmzFC4ao/KCmzIkAs4BCcxBynVGTdx0QgOsULRmDMQbuBabgt6FDPBJNKamWrmMjtS5xuSVPY9k5F1z4xM3Xmcp9HS4F9pfFXD1Iyloa4vLCbOfq5djwt9vHEU2LR03E9S00knlDomZXh3u7svJ7apOjuvz/WeynaetjPlpiMsvfHrfxe/X12/XVZeUPrnCeIMAxt5pKDihss7xdkb3gOv+qfEeHuNHU7iKqKupj/VkXzf7K+SqCurMJrGVtDM6KVh3abXHUfVfVP2Z8XU2J8NUeI4i3EmzytuRGzmMI/Rfyb2n9lM/ZuMN+rPlhM1fpMT6vudF10dXeOUVLi8c4A4cxuudJxHh81G8tDSyNhDRb2XLY39kNPS1TqPg2saYGQGYGc31Hwgr6WNbgeMAUUckvMds2op7X/yWezgplFN4rDqcF/5HG4Xg6H2r7h0tY5588Yioxy84j5/onU9s6bqorPGHx1iuBcc4XC6onwciBuhkYDlVLD8T5/LjmkMU7tCR6QvtfEarDH0xw7iDhdnKcQTk8zSVxvEf2R/Zlxm6J1M2OglBJPKjyvPzuv1/R+23buojh1mmdc/HHziP+Xw+p9mtc4/6X9nzdDj89C+zpY3W+qv0/FznkOs11tLBq77Gf5L+PsriOG8ap5aItBaaggOv10VCL+TLxnFUZKmupDDe/4LyCV93HuHZ9uqNmvqsYmfdM1949YfGz9m9+NxjE39HKfzkpXSOD4bX/KoZ8Wg3jbIQe7bLseIfscqsAY1roJBNKCWOZ+I0EfmsNF5tiUGM4PUNjqIqdt72OYkH/m69XTdNn1muNvTzGWM++JiXz8+09TrmssaltQ1kUlO4/F0FlNSmGW+npXJT4xi+UsZDTn5XCPDsbqGD+mRTMeXANMYzBdNnbOpwjlOP2cdvb9+EXT0ShpoTGPIrRpYQNlxjMZqmmzK+Eno0uAKOPiWqjdme+M/+YF4vDy+Dy8Z+DrxTx7ZEFRQt5R/BafmuVPGL5DaKRjZOxcFIeK6yP1uYfmQrOGXwScZ+DSjwWGWoBfHYHcA6Law6SpoW+Hp66oa3o0PNguPPGbmnURg/RWaXjOAOzygSH/xAFrhl8Eqfg7hmLYww6YnUf75VlmN440aV8j/AJrijx3h4P7Fv1kCMcfYaP6kf+s1Y8PL4HGfg7P7+x//ALU7+P8AmnHEGODetcPouQHHmE9j9HAqeLjTBpjbmOv8rpwmPWDjMOqHEmNjTxx/RP8Aznx7/wC4u/3VyUnEeHOdnE+Vve11agxjD5Gh8WJQG/R4sVmkdGOJscJ81eT9FPHxNjIH+l3+bVz/AN5QW1npXX7OsgGMYc02krIAewkBWowmfSFiJdG/iHGZGljqppa4WI5e4UcGNYpSxNgp6kRxt2a1lgFjNxjCQNa+H6OCMYlhxGYVkVj1CcZiPMvKIr3Ng49jJ/8AqEn0FlH974qf+vyLHdjuBsOV+JQtPYuAQHiPAWH/AOZw/Qgpwn4JxlsnEMXO9bMW/OydstRM3MamW/8AbWFNxfgkbbQVPiX2uI47Zly9f9qlMRJT09FLEDoHnT+K76Oi29TlWEO+nptm+axh02J8QVVLWugge2RgA131WZWYvWVj8z5nMFrWa6wXJwcQPaefLC6YONwG7o5MbrKpp8BQO/tOK/T6Oj06IisYuPe/Qaul1aoio8/i32ubf3U9FxVW8P1sVTg+Lsp6tjswYXeV3zC4StrcYkcDUxPi7WGihia11ycubu5dt2rX1GudW2LxnymJerHKcJ5Y+r6N4O/lHVc746Li2gj8S6UsMsdmNA6XXrOF/aRwXiDxHTcR0rJXGwZJIA4lfDrZns8t0ccuWRsoJa9pu0joV+F7h/h30HVbJ2dPnOu/dHnH/tD6eru23CKzi33XivAnCOPOklxLh+jllkveblgPP+0vIuK/5LdHJll4QxWSCRziZGVRzNA9l4fFxxxnDdsXE1dG3sJiug4b+2jj/hx8pbjE9c2QABtTKS1vyuvB0nsx7Q9midnRdRjlP8vnU/fy8nXZ1vSdR+XZhXzbNZ/Jv+0Slu9jaSZg+Jj7rmMG+y/i7GsdfgFNQuinZfzztLGm3Yrr2fym+P2nzUNA75tW1Xfykp34TFLHh1LUYiLZmvgLWtPWxXt/He1/T/l2accpy8omK/LPxmp9Pr5Ofh9Bn5xlMV+7Gd/Jg+1Iemmo3ewnF1wHFnB3EHBFecMx6hfDKLWdls0+wcu+d/KZ4+k0Yynib2Fyuf43+2Hinj3DocMxoxOhheXiwJ8xFri6+n2jb7TfisY7jhh4U+tTFx8/KfNx3x0XCfBmbcLcJwboHnL5rX9gh8VAB68vs4L9k+amdumTNcx4D2m7T1CYl4OUHTsUaElfRJJEsw1GqWxSuPdOMvwrSmzFI26pZSmQK57p8xSylNbVA99U1z3S65U2UKs2dJJJQNmCXT6IU9iiFlKe+lk6DNZEg5clqSlm0SzFFD5morlK9/Uh0IQHfS6bMUun1TIUfMU+YJjl6JkKHc90rlNmCWYX3VZPc9CkXIQRfy7JE3dpso0IHqE+YoM2qd2yAwWpHJa/VBmCTtkDjdOdkOYJAm90D6e6Vz3SvokDpZB4IxsdO/PNFm7C9lNPXGrlbkpYxlN1XEDuuymjjY1wLFyf2FebUQOeGCkgbfq7ZSXoh/pE0AZ/3Y1WXOWE5CLoYnBrQy9mjZaiReqqqkc21JDIP3nKBuf1B5HyVaWoAdkYHm3YIoqXFp/2FLUFvcNsqLgIA87C5C6ZjdoWj3srNLw1xA9ufkOBO2c7LSh4LqsoNdXxtHUZSisR8kErAGuDXdSSq7oXN9Fbf6r1rhb7Ifs4xPDmYjj32jUtBVOe5pp5muzADY6d1BxJwjwNw7iDKfBMWpcaiczMZog4AHtqvHr6/Tt3T0+N8o/+3KI8vnMV+7EbomeLydua/nY4+zdSVdhosSqXAU+E1Bb3Xa+Jgg0gpIGjpoo5MUq3eVmUfJewtz7OG8bcQ00bGNO5c7ZWW8EQO/03FQP3WNC0TUV0nx2UfKkJu6yLYaPhbhWmkjE09S8ZgHOL9AOp2WzjWGcF4XiDoMAPjqcMaWzOBaSSNRb2WSYpPZFHC/sFicZnOMr8vh7v/kJfmtR4i+EZaambEPZRvxGd5/FeAOyEM7o2NY34AfmFstCZZpBoTbvdJsUj97qz0zBjP0RZvZqLiriC3wpwzWxBU+mxQktB0RDCNnun8nS6a7k+fTKgVgd0fkA7qLNqlzYxu/VESlzT0KWZ3S6DngGwYfqo3Tvv5SqqfmPA7ITNL/rDZQtfnka17g25AJPRazsKwm3/AOsEH+6VHDb1GGmuV+fwiZ/pDNM7m/GkKl/xEIKuOCCZzIKhs7Bs9o0KgMjQjrjlGcRlHvWTWTfAyO3eyUdXIXX5TC7vlVTmj2RU9c+kmE8JGYbXF0MriJr1Wn1pOjmj6FL7yawWdCT9VRdI15Ly9tyblQGd59LUVqjE2/6spHEaY+okLJHMkOUKzFQzy7hVV77zorftD+iE1lC/eNsn+yohhcFOx0lRJFGAL3JWRFxNhbLjw5kN7ANVGtKKU7Usf6II6Fk18lCfoFlVPEtcTysKoIwf37FZVbjnEReeZMIndWtAsstNbiCinosPeZBG0PIAbmuVm0MeVgNlnuFVPI2eqn5jj77LVjbakef3F2wcMk83FXgoW+EwhwsLZn63/gpYcbq62gdUVBdETrGwWDLrFfDNNEI5JTJGDe5V90LOQLukcGbNOq45+rvj6KMcl3iXFA2dw+PNcfRNOcPllLRTuB6ADQq9BSVFPUMmfQgsZf8ABdufordRXZajxUOGCKU/622n0WGmYfwja1xbZG+SV4ayGzL62ATC2V0kxdI55vrolGXC7nssR73W0oDTc/jTR5e90dQ2KIHw8b5SNiLJhlEDpGxx5Du62qjmLYg2zwy5tqsWtCaS5gdIy0h6XUtOWszuYz1ixUlEGxVTOdNEWdboJOTSyvtVxFjzcWK1ErQqp0MpbNyRGQLCyqOm84ZGfMeikmhZOwGOqDnMNw0iwKC2SMMl5ccn5lSkwmpZIpZahmV0PkbkOpPdJsMcMTppJbl5ucxUbi+XLmnEhYLC3QInPkNmt/Ef26lBYnFMS1lLo0i7mne6gkgqGSjyWATxeIl/D8KA87PJ1HyV2jwnE6izzIOX1e91gFhmVZ80cMGdwPN7AKJkpcc0d7dyLIcYdFhz2xiuhnebaRagfVBDVB9o4mcwd72UpLWzNXMH9FfID+6pGw4kYGz1FbISTYNzEkIYJ2U4MlXh88tvSGmy1IZZ628bMIFNbrI4ZyocmbLRTvgMkkl3na7SSfqpWYDXStaTGNB1dZa9NKQ0wOeQY97e6Fxc2YPdWyOBNsvZbg5KBweeK0s1LAWj1AHUoHGaV7Y4adoda1rbrY8exsvJc1VMK4O4qxyolrsNwqqqmySWhMDbtLVzz2Y64vOac93UaunjluyjGPnNf1KCB1FE7mTxyPcbnoo3QVFbiEbaOldIZXBjiOrivVuFPsH4gxeClq8fjZQNZKTLBObPc0L0zDvsV4AoJI6g4c+WaJwc0vkNgR7L5+7uujVPGJufk/N9d7X9v6TKdeEzlPyi4v8Ab9ngbPs84u57Y8O4flllYb+V4tfsu94U+w7GsQFLXcRiGhEbi59KfNf6r3dvKgZkhhjjaPysAULquNvxL4u3uvUbMeMeX0fh+q9su5dTr8OJjH5xHn+8y5PD/sj4IoKuOt8AHzQuzMcNLFdezw9IC2n/AA2npdVZqh722Zoqx5j93XXgy2Z7JvObfmt3VbupmJ3ZzlMfGZlcnxB8Vn8wEbWVeorZ5PQ0AEqKWnvTudl2VkwsDs1ul1zcUHKkebyElFFC0HZTNie42YFTxXFKDBKQVlVIXEuADGG7tVrDDLZlGOPrLWOM5zGOPrLQ5YHpTktjZmmkZG3u91lyh49qZmltBhXKb/rZjqudqfE4hUmpxCtknf8A2rNsvp6O07tkz4n5Ye/V27bnP5/J2mMcW0WFPENHF42c62Bsy39pYdbxZi2IQSQZG0rX2/ZnoskMFgBf6lGGL7WjtmjVEcouY976GroNWuI5RcqrKCkY8SiNzn/mc65UvLjv6QpJXU1O3nVU8cTBuXFYWI8ZUjQ6nwmHmv6Su0C+i9raL8m7QANz0Cyq3imhpXmOnYalzOxsy65qqxHEsQ0q6kvb0aNB+irhhB6IL9Zj+N1/kfUiKLqxml/qs7lMb6UeRycMdfotoSQDiUVt/IAnaXK0GGcDK1uqJt/iRAdAlruoHI0zJk99NkOYIJL3O2ybQhLp9Er5fKqBSSTt3UDJ8xQaX83ZPr7LQQPmTpuv0ToF89k1xlunSQPcdk2cJaJaItkXaJr2SvpdOiSZ+wToLIswVkLME6BPlKgWpTJ+n1TLV0lWScb3smSA1SSBaAXTZrhL+11TKKSIbIUQ36KykF5iUB3+aLp9UJ9By79FFI6dEgbpZQbOO9ktrKAuuUpBpL/JJZoTZrpBzW/VAaROpSvpZMdkSC136J019PKnRSTi1/Mm1afL2T7hA/X6Jdfoh090kBtzdUyFF+6gY7aJ821902UJjuqCSN73chSuVA+YJ/hGyQyk7pO0+JA2UJW1sjDmNY5pjzl4sHdkGYoHAskTqPdM31J7/m1QOc1khtfohLr6J26ICAuHO6ITZOHalCdwiQdL95PfUps2iKbMU+6CxBv0RA3QK2tk/lAsh6ovLqsgW52PY8W0N7EXBXuv2Zfbhg+B8NR4Fj1LJA6lFmSxDSQLwwOSJ1Xx+9dj6bv3TfhupuruJj1iXfp+oz6bPng+guKv5QWCspIjgFPJVTl5uZSGmP3CDg/+UziFTiDaLiHDYfDv0a8O8xd0F18/W13SBINwSCOoX5/X/h/2jX086eMzl5/mmfP+3l9Hs/zXqOfK/L4PuOj424WxeLJJWRU83+rmeP4Hqpqrh/DMRu9jWX6PZb/BfDjaqqD2yCplu0hw853Xf8KfbVxLw/I91VUyztsBGxpADfovxvcf8Oes6bDn0mcbJ98ek/p5zH7w9+nu2vOa2RT6X+48Ww/XD6rmtGzHHX9UcfEc8GWLEaSWG/Vzbhef8I/yg8JxhkbMYpmw1UryxsbHjMe2i9MpeIeH8VZy/FxH92TQr8L1nb+p7fs8LqcJxy+f/Hx/R9PXuw248sJuFeVuE4/AaepAkaRbyPsWn6LzDjH+TlQYrGyXAK8wyMcXObJu6/uvWKjhWgmfzIL0zxs5m6ovoeI8MeXwSCtZ1DvWt9B3Tq+2Z+J0mycZ+Xp+sekpt1a90Vsi3yXj32Scd8P8+evwWRlNFciW9wWjrdcm1kkekkbmk66iy+4G8Tx5hTYtQOgzbtczM39Vh4t9mv2YcfTNqqjDQJoQ5hdTnl2P0X9E7Z/iTnE8e4a7j44+v2mfP7w+Tv7RExeqfu+OXkOPnbf5qKSlpJT56Zh+q+j+J/5MNPMwVHCuLNYGA3jm1zHovIcS+yf7Q8LkcKvheqbECQJQAWn3uv3XbfavtXcsOeG2MZ98ZVE/v5faZfM3dDv0zU439HHNw6jje2aOBgew7k3Vkvc79o24/dGqUkcsEr4JmFr2EtcD0KZ2y/QRlGUXHo8lUN8FKRdleQf+8isohD+Vgt10CIi6cEjZWxBJRU8p80TR8ghOF0Thbki/fqp3IOY4JxDeGBZks3LbYhHRt8I0tjY1rrWzAJcx5Sa4k6pljjlFZR5JNZRUjdUVH5yPkkKifluYJ3C/VDpfzIg249K5fhtP8kfaHPwtX8sfZXma6paxlTLJIGagk2uoThsBN7v/AFV/I1ARZdccMcIrGKhvHGMYqIVhTtZq1zvqtXDp53NED8SbA1vpDuqpJiLqZ68NkVlFmWGOcVlFpMQooxVCSSoiqBJcGRjruHzUMdJTOzcyQR2FxcaIgLKRWMYxioWIiIqENK59NUx1VHIyOWI3aXC61jjuIzNLJ6Khnvv+EFlvbD1ahjnax12PsApTondLmlL+Sxlz6RsEmmZpcY5Hxl4scptdRvrX8/wwDXdRpqVZykbixV4wcgGarkZyZZ5Ht7EqJ0WYg2vZWLD4mpiANkqEQlotoCnDfzKS/wCZMMvxLTIwQlzG3y3Q5GA6I8rT8wge7R1uhBsbt2SMZ30TDOPkgK7fqnvbYke6C+qcObqqBkD3OF5Tp2ROs8ZZAHD3CFpv01RaHdUJo5bbMbYIs+YebVKzreyAgaE307ICzdxZO7KgJFtHJgbbnRBMkmzBIOHxLNJQL5TZO3VNcHTLr3SvrdaU5NimvrdPYFLLb1IFe51TpNahIfn3RnISSbzApFxaMzkDoTv5UWlkCJEHdum8xCNJEsPT6oXbJ0zsqKWYJ02UJ8t0UxNk6HKUSBJ27oW7ItyqwZNfWydJRqDaAJZghTjYrcMk7dIE7JN3SPdu6w0LNZDmKWpCZAYeE+dqiyhOiJLtSBCi1G6cai6K8WFJXzfsMPnkvt5LK3BwxjU7M/hxD7POq6p2L2/Zwx/ooJcSqpNiG/II/r7Ej4Fr5m3rMSbE3qGXWpR8JcP0Pmmlknd++7T9EjPORrM4/VCXPf6zdZW2iBgdKfwaSG/s1G7F2N0gjaPoswNRBqMrcmI1LzfPb5Ks+WWQ5jK4pWJ6Jw0g3Kqow3XZWKmomqsnMawctoaMrbae6HYogOgUYnGJmMp9YQcslOI9MylIJ9GiYD4UaINRJJIpJrn4U6Vgd0D3tvonuCLWumdktohJsiDGZosNk9xlUOfRLM611VGT7pZm9lA51ktRuglMtxZNmd2UBc66cvNvUozKUm6G9j5lHmc3qFCZLv8APsqyuNkj7qN8kYdZrlTdURsB82yp+Oax3nmb+qrpDWfKEBkafiCzm4gJbhrnAfmy6IamqbTRtfJnJebAELKNHM4DU3UElWIzZxVqChvT8+tqhB7A3Wb4/h2rqjR080tRJ7aAoqXxDSLsBPyCVMZJz+Jkbfo46qo/HX0cr6WgoIwY/ifYlZNZiOIeaZ55RP5eqwjpJGQRyWfNGPqrpxDA6GENeTI/8oGpXO4hUvgo6Z7GZhJHcaXVSOGonvNNPa+zdis8xs1PEj5MrqHCxAGG93OuT9LKrWcQ4pVCzJeQz8sYtf5qr4bpzpD9ULxBG3kMZK49yF1jJtX8RVtqGz+Kkdbo7UKcRU9RVidmUO/K3ZMKd50a3RFDRuikc6Cmu/q611OQlkax7szH6+xRwDK4iS0ucW1Gyjhiq3vvFVwZh8OYXUkMz3uPjpM2Q2y2slls/wAjqwtZ6VrZHNpCD1Flms5cta98bMrS7yNHQLTlhmmYxnOELR6nOOgXf/pcMv4lVgc2MNy+lTRyyMOSRpAO1woqjEsNp2eG8dz5P+7jv/FSVXEddizIaZ9KIoIhYG1ifmvNl6vTj6BcamomdI9+YlHURGEuguS8dT0QwPgYRUvcTk+EHUqSrq4XNE7XeZ5sGndMWpVZGz5rHL9SkfvGKldMYxC3NkNtbqaKsp2uz1TLjvdJ7aZxfI8XbplHRGKVKCnrYX+Igq8sZBB5zt/onsJZXc+dkrxuW7KYOczYi3snawE3Nh8girVNHgUoDZ6q0x26LFqyyGuYBGKgC9wFoTT+iPIBkN7jqozWNgGrCb9RugqusZ2TSUMzQz4D1V2qiYSJH0rpGv2uFOfE2ZDO4OcdQewQT86R7nySR5QbC5sPottKUT4GzsDCNeytB0bX3JUNKYaoSPZSimMe73i10dOaaV1uWXHq4bKZMrDX0TnFnMkLumRv+KmhwuasidnJi0uM8uiqzcgP/Ablaf1Cj5UB9cl/qsihPQUsj2CRjXZjbQ7K1Fg0dFKyTnuZpcBXKSmgmElPSwmWaRtmsAuSrlDw9j+MVkNGzBa0ve4Na50RAaPdMs8cP4ppx27tWnz2ZRH1lFFiADQyoGdo2F7KR2LyStP9GLnSjK0M2/Vex8K/ydcSqaVw4lDKZ2by5XW097LvOF/sN4Q4Y5jZ3zYjzCCBUAWbbsAvmbu7dPruMfOX5fq/bDt3T8o1zOcx8PSf1fNzcA4yqmuFBwzWSyODTeOP1D3XqXCH8n2uxPDo6riZrqSWXzENkLcgPQt7r3yngocPiENDTxwstazBZOaog+tfI39237seOP5fo/G9w9seu6zCMNX+n88btwHDX8n/AIJ4cr2Ym41NbMxpbkmk8lj+6V6DR0WHYXEIMPooaWIfDEzLb6KJ1a/4TdRPqJXfEvBs3bN03sm35vqes6jrMuXUZzlPzlfknYGnO9VZarS8dyFCI7tDtb+6NrVzeYDnySM1JQmK7R5dlOG2FlI2E2usCKOK7bpxBrpuqWK8R4LgjWCsrWB5+Abrm6rjjCa/xIigqGylpZA/LcW/MF6dfSbtsRljjNO+HTbdkXjj5OwqwIaSWWd2SNo8x7Ln28W0Je+RjuYz0Ndex0XnVNQ8SS1LpcTxeZ8Upu5pluf0VpuD0UExlbn/AHW9B8l9fR2bHGZndN/TyfU1dsxxn/Um2txDxDjuKTmCgqTDRMNwQ4NJKzoDPE3I9rpD7vLkwZLfQCysQseXebT5L7Gnp9enGMcI8ofQ168NWMY4wfNWn+qCQhry4DIBf2VetxugoD+LUZnfliOYrFrOK8RqGFlDGYI39XEF6703ydBWVMeGRcyurY4/YC5XP1PF9S7TDYrt/O8f4LCkBmeZKgmSQ7uduiaxtvKF0aFM+qrJDNW1DpndLnQJANbsi3TEXSIYNtsE/wA0WS6fIgbzdE4BsiyhPp+cA+6ojaHNHlTg22RXcU2XRUFqGpJE3KZ2yyHTiwOaybNYJHU7KgMxJ8qItdb1XCW22ya7dlQSFuoTJ27oGRWuEsgJToEmGyV9bJ0DuTC3xJtCN7JifiRKEhO4RJIpIXJxuUJ11ciQSSf91Minbul1+qXX6pO3VQySSSikibsgyhOteqeh3HWyTd0Nwk3ZQOmbsi6fVI+2ivqBO3m3TpJJRYcpuH3t7JyO6G1iR0unbuook7d0ANynB+HsgM7bJ0gdU2fKCOnqUCA0sUgABoNAma66Q0brqgLqSmvrZOnsPiWQyaxHVO7MQLDqns4gGy0lhBaHEEFOmcbuF06KWvulpZP0ASIF1QyQF0ibAmyWbQaqJYvL0QpDN8KXXVFPfQJdSmT30sgbL/knvYpkkDuuTdMN07d0w3QONymJCcblMBrZEgV27Jj2CQy3N0jqMtkU2t7OPyTtNin3IPUJiLICDt0Lrg3tof4JNF/L1TF3dEgibp2pz2cU+gHlWVP8N2tTWuEzSU+b6q0EWpZSjs0pWFr2UEcZexwfG8tcDcEGxCtR4xiUMrJWV9QXRODgDK46jvqq5aDugygnKueenXt8s8Yn6w1GU4+kvWOFPt+4gwgSHFnyT3IytY0ZcvuvSOFf5RfDOLySxYrh78NbGARIXAtd8gV8w5PcpuWCLar8h1vsJ2jq+WWGM4ZT/LPlH0j0e/X3PfrqJm4+b7ypMUwPHoA+lmp6uIgOaHbkfJZ1VwjQSzGpo3yUkvQtOg+i+MaLifiDDamCpp8VqrQEFjRIQCB0XrHCv8pHFKCAs4iaassd5Gtb0+YX897p7Adx6KOfT1tj5ev2fV09007PLPye1yU/FuENLoJhWRN+E2ujpuL6TNycVpJYT1Dm3H6rJ4Y+2LhTiOnhl5ppZJRdkUjg5xXY/wDw/GKW/wCBOx/xNN1+K36dvTZzr3YzjlHul9HHLHOLxm4clj/2ffZzxzCRXUMTXF2cyUwETyfctXnfFX8mTDW0BfwpiEjqjOAPEvLhbrqvXKjg2nzczDqp1LKdsuypTQ8W4UWuewVsQ3IGq+l0XtB3Lt0Rj0+7KMYm6vy+zns6XTt88sYt8rY/9lfHPD1ZJTVGAVU0TNqiJl2O+S5mrpKmgnNLWQPhlABLHixsV9s0nGtG6QU1fC+Ano8XH6qnjXAv2e8ZxTeKoaXnTsyCoiNpAPYr9z0H+JO2Mscet1ROPvnH1v416fo+Zt7PjUzry+74qeo76r6RxX+SzhTIZH4Lj08jy05Gy7X6XK8h4p+yHjnhKeOOtwiWpY5pcZKVplaPmv3fbPaztfdM/D07Ky+GXlf0fM29Dv0ReUeXycj5bZU4DMp7qSpoa6jLRWUssF9s7SLoCPLZmmm6/RYZ47I5YTcfJ45iY8pMG62cEtjYFOOc3R7gUgzUlaQun1RBt9Sh6/VPmuEBEbpgzskcxCQ2GUoByOJThmqIPHZPf2ULQSB7XegkJFzZRlFv0VkEX84zDsgIYToLKtRLPlpntfzxrboN1Yhr5pv2kT2kdT1VktBSbTxEbICAdIBIHXBTKCSCaJ+eF9yNQCpnMmfC2cty5t2f5IyK19LJPiAKANNv2zmjtZE0263QCG5Ntk4zeoJw9g6aoTlccwuPZA/m7Ig4/EmSA6BUDldm840unkaL+VOQ4BBnIKWG2HsiFzsk7TfqkHtGxQLmPB0P0RCQuBvuhuwjy7pibHRAacbbKLMVJzDbTVAW4SLA/wCLZMHsOzbJakoFlKbYJwbJlQmnqnzFMD1CcG58qAwSBdqTfSm2KbPbZQOkDvmCEm6QB6Kh8rR8Sfy90CMFpFwVZZoPVPZ1r2Szapcxw2UaJwb0KYC6TvUU+X5oybNrdMS4DQJI0aAkn/2UyBJZrJrBL+0jJZghRpsoQMb9UxzXGVPt5r6J/wCyiBSSSt1sgfMUvL2TG56pWHZAkkrnukhMkkkkg88DGl2psiIAOVlyjMIabp26HyhR/XEeWwThl0dnEpw1w1upxboWXTZDZw9QR38qElpRCBBCYut0uhFr+ZEXooQTfZEXP/LZR8yx9WqYyuuiJ8wCEytUZdcIC9BNzmBMZWlQ8wfEUjLAB6z+iqpOfqugwLAIsTjM1VViJrtGBrgXE/JcnLiNJGfiPyCGHH/CTNnpuYx7Ddp7KPN1Wvdt1Tjoy45fGreuR4Dg7Y2sNBE8tAGYjUp34Hgn/YIP0Xl8n2iYyDpVkexVZ/2gcRPPkrMo9lafmJ7J3Gf/AFv3l03F2F0uGVTJKZ2Vs9zkAsG/Jc9zSDZZldxPieJFpraoyZNr9FS+8Kpx/CzX9gq/TdFq26dGOG6byj3t8uulzLBY0UmLTHyscfotOnwvEHASTwyu+WiPUT6jsoHSVRPlatNkVBSNz1HKFt7m9k8vEHD8LL+Izns0XWSVBlNXyH0q9FgVbKwOJ39lWHHcA/8Al2G5ndHF1v71SxDibGagWp547/1mQek/NEaw4aiij5+Iviaet3Ib8NwtDAyOZ3QNBuuaz4rO3PVvknv0edFNDX4jSWzZYox2C0rSxnEp2wtZQwcgdbhc/K+rrgGTm57q/VVEFZHn5xkd2KrSujpWXdcO7LlxaQjCaJnlq6uV7j8LOqjlwXDTbwrzBbrfVWIquNzbvGqimlY0jXdOIlw/C/DOL4akSO6F50CtVuDV9fFkMlKD3Dlnc2Mj1uHyViKvZSwX5MzzsDus8WEPNxWOIUkhD2MPLGmllYgpNmTVEUQHxEqON00gzvba+tk/NLfStxi2mqIIWgBmImUnoNlJh8tNT5ubO7zW31WbNV1UTsvJzf2QrNJNWSNzsjbH7ObqtcRMJTq5gIbmNiRumo6/7ue93K5zZNw4qvWYliBBhaYz/wCWVO2NtThXNpr+JPpuVJg5ClquVq3CYoCfSWx63UbaGvqIX100M0TGC5BsAjw3HqqnibSVWBTz1A9btLW+aOpreJcQifA48ilfbyvtdWi7VMKilfM1+TdXMRFpcnsL/NZFFV8nmxSV450b8jbG2iDD8dg+93sqHSTtjFwOl1vknBqQw8plpmjuMg3CAizr+bL0BK0GY5hVYxzmwua47AC5VKatp4zzow11vgcP8FhURmhgPMMAkt0KumcSQ3pcOZTyd5Rc/oq7eIK4ycjwsMLHdBFa6mjr42vvLA4+wReSJklZK28jqX6Nsnioop7+KrWRt9irVbJS0szP6NuL+VUZ31rWFkdOM7yS256IWGJsEJ5MfpCmtGf2Zuhjwp8cfOkdK95AJDfQFIIQLNjmAf8AlKkwRKpKTYPeWFp3HZSU1SfOKV0We17vbeykhqoudlniBf18hTVJmfK4U9NKxrtLNZopaTlSCmrqvnA1sYGnqd5VpUlThwu2qki517gnoOyp09FidVVxU0VE+aWRwaxoYXG5Xd4H/J/+0rGK5ja7CY8OpiDeeRwBaf7K57eo1aYvPKIeTqu5dL0WPLfsjH3+vn+ke9yFTVOrIXxl+ZpOlhYAdlQhpKyVzoqSCSaQGzmMaSf4L3mm/krTuafvHjuUdo44F6pwx9mfA/CEV8Owlj6l0YY+aU53Osvm7+7aMMf9P80vzXW+2nQaML6e88vh6R+t/wDFvlXhngLiPimr8HQ4NVPcWl13NLGWH7xC9a4d/kw4bJSwVXEWI1ME2hfBG5rgD2JXuvMp4j5GtY3sAgkq4+6+bv7tu2/+X+WH5Pr/AGy67qvLR/px8vOfvP8AZzGFfZR9nGCTQ1VNw9AaiG2WRxcbHuusEsUVxAyJjezYwqMlUHbKPPJJuV83LZln55Tb8tt6jbvnltynKfnNr0lTy/U/+CrOxG5GRhII3UYiFspBKJjGgWXPk4gLp5Ha7J2xOvZ11I1n4gPRSWsSgh5YCkjbY3smmmhpY+bUuDW3ss2fjDhmku11eZJPyMbcrrho27IvDGZhrDVszi8YmYbLGl2wUFbXUmFs51bKGMvbe5XAzcY8RyOcMPPKZc5XFt9FRqTV4hL4nFJpamW1i57tF9bV2bOconZPl8n09fa8uUc58naS8fYKxpFEyapktoRF5QuSxHHeK8VmdN4mWkicdI4jYfVRM/D0ZG1o9gi5juwX1NHbtGiZmIv6+b6GrotWmbiL+qoaGSSXmzDmybZnC7lZFOWN1ACJvMcgqJ4KRvMqqqKFp/OV7YiMYqHqiIiKghECRe31ROijaLvma1vzWBUcXQi7KKmdL+88WH6LEq6usxSXmVdQ97ejNgAqrpq/irC6NxjpGOnmZvbQD6rnavHMWxJoEkwhjO7IxYKryGDZoCNrC1aYBHGxpzMGp69VIGndLLYp8xVAZHndwRAWRW07IgPLeyCK2t0YF93WTE3SD7+YLQduyTdk2vumQFZt9bhOgRWsLoCDvzJ/9pMHJlkGkkknmlyEixsU42Ta+6VytKR3Sy65kyLpugWt/ZMd0ycWuXIHvp5UvMUtAEswQCn6/VJ26XT6oH6/RM7dD1+iI7oH0ATZil0+qWpKBk2YJa+yFAWbVJ3qQp8xQEnzFMkqHbumP7qHW6Tt0BJJv3kXqaraUAANGiJJCfIdwopyf3Uuv0SBtdOga7bpmSMJ8rrpG1/KmPdAra7pJA6p27qBbFE31oI42ts0HROD58uw7oCkDyLB+RHE4x2LbX90Lt0m7qhkaj/tIsxUBHT0pNy2THZIbW6oSLMUyYWunPZZCSSSWg/+STt0yQ/j3QJJM7KXFzevRPlyOuTsrRZ3bpr6WTHZONrKBJJJIDG6ZDqEhsbqQEl1+qScdiqGuDqE+YpvknLfhO6MmGt/ZJObWSGuiNECdQU4HwoU+YoHQ2ukl1+qAQbaZUR19KYtvul2QOD1CfY7oDY6W+qe2mf96yAg7XVGdr3+ig6ZkYd2KCbI4jQ2TNaUGYg9SnL3XKgfW6Vtd02Z35UX96iBPmu1Dl/RSC3VL+5CD0VVU0NQ2popnQys9L27hdBhP2h8XYRXRVkWNVbhG7MYzJZr/YrnG6Oa4ojZxvYD5LydT0HS9Zf4jXjlcV5xEzX19XXDbnr/AIZmHvvDX8pQx0duIGN5zXWDI4ydP7S9P4Q+17gzjFkvg60080bspinGVzv7K+McjVbwzEq7B6oVmG1DoZQC0OHYr8L3P/DzoN+GeXRzOGc+lzeP09Lr9ZfT09224zEbPOP3fc1Zh2EYtDeenika/ZzViT8EwRXmwmeWB/QZl8uYD9rXGuC1sc7sYqJYWnzQ5iGu+i9c4b/lF0ElAyXH2Rtma4hzWNIIHQ3C/A9y9ie69uiMow8SPjjc/t6/tT6unuOjd5XX1egCfivCXBnLbUD+Ku0nG9OZuTiNJJS+7tlX4X+0zhDi6m8RhuIxNs7Lkn8jr+y6GbDsOxGIOMLHDo5hBC/MbdOzp8517cZxyj3T5S9uOUZxeM3DBxzg/gLjVzZMUw+mncL5XtGUheYYp/JboBJJPhXEE743OLmxuYzy36XXqU3BDGuMmG1UkR6EHZVnTcVYNrL/AEhg+LNqvpdB3zuPbMZx6XbOMT7vWPtN1+jlt6XTum88bfL3GP2T8YcIOdUVWGSSUfMyMkj85cOhIC5R9JUxtzyUkzGjcujIC+1afjSlNqfE6flP7PjQYpw1wJxjRugrKGKRr9S+MFhX7noP8SN+vHHDrNUZfGYmvL6fH9YfL29nwmZnXlT4pEQclyrL6Jx7+THRVla+q4f4hFPC61oHx3aP9oLzji/7F+NeGHPliw2Sto4mZ31DNh30C/cdD7X9p6/LHDDbWU+6fL9L9L/V8zb0G/VEzOPk88ydU3L13UvLnIyOhkB6jKUL2ZCWk6noQv0sZRPpLx0EMslsU9zZIai60h0zdkpDZXJcLy0/iqWsgqGD1ZXWLforMrCmT8SWb9E3mymyQt7hRBNkubKnPPPQVDpHnm0xNwPyq3csF2NH1QvcyVhZNACCgDxtFMzmMke0dnhGx8crfw3glVxG+AZI2iSInRjtwpoxC0Zo2Zb76IJS1w7JJun0SsbIBS9XqPRLQphlHxFAWW3xFMiuLaOBQZgrAkY8MvmYHdro5ZKKVgd4UxSjS7ToVGBZPywfUEEdgDmaE7nAhO6zPKk3KgDUlO3ZLMUbTlGWwN0AWF/ZLNbZIuAJBjN+4KG9j5lRIHfmSPdv6pNyH1JHNe2bRQPrl8qbP7FONAnBsoI46mNxyFrw7sQiOpREsJ/Zi6Hc30VDhovqkT5tLpf7SQsTqqHSI6FFlCjkkdEQCwm4vcIDbuk3dAKmA7vMZ92I2lkh8kjSPmshktR8kjcHzCyWoWg9ymSuOxTFwG6B0vkkCAfNsidbpsgY3+JCDdE5Iht7oxJycxQoidfSUDtkZCjTDZIAfEjZZQmbuicNPIQUPXVAyfKUtCU/9lA6HKU/9lP82hVgIB6JkaHr9U9Rwpp6n8qbk1Tf6sKyIpjtWfwSdHVi3Jqh73Uf2JX5dV/qf4IC2tH9WVc52KRgAMiekKmv+OljP1RGflnHrYQkXGNty0lXS+qvd9J/FDzrftaU2+SjEKPPB23S5jyrXPpD6oHfQJXoT/1eRRtTu0eYvCESMJ3VoxYd8Ucn6ITTYaG5rO+SopvlANsyB0v5XK54PD3axQge7kElHhpI51UQfZVVF8hUTru3K1Rh+FHarc89gUQwmlP7OUtH7xslObEfT5z6wPmnbhsjvhW66mwqntzqqEO7ZlQ/nNglPJoHOA7BappXZw0+Xeoj19lcpuCY2tsZ3H/n5qnPx6xtvuygu7u4XVCq4pxmriMklWW3+FgsApMr6Oqg4bwajtznjMOhcgqsT4bwt13SZiPhbqVw0M01a455ZZD1OZIRh0vLEN/crHIdZPx7Sax4bhpc7o5zbLAxLjLF6pvJfUGF7/hYLBDG40Mp5FIJGe8dkEscE7874Wt+SchnysrpogXzGR8hturNPg2JR+SSRob1N1Y0iYBl0CeI1MsRngJjaPiIV5Ct4NkMZhuSDv7qSmp3hl6VvkOyGUTyyZpap7r79FeooZpAIoZA0Dqd1OQpGKocRn5oseqmkga+O5mmc49L6I53tGZj6hxt1JUINQ1zQyCSVp+IBWmgCkrIWtDCDmPUrQ+7a2SnLpYHOJ6qSSCSOPmTROYB1OyhixKOWI0rHygv6tNypMCszAqk/wBT/wDknfg88LTLPAWtHUlBNJI1hZFLUB52JN0qTmStLKp87o2C5F7LNAL0QH7QX+akpGQTzNjDLyE2DjoFOJuGAcszJc3YklWRiOBhv4E0sfuyLX9VoWaujipHCOV7ZHdbHZNHhtDO0SGEAHYl6i5Lp2OqoKZ73d6g2J+irwUFdWZ2V1VGxkYuImDS6DZiFPQs5jJGZRvfVZwxGnrKx8kZ8oF7qrTx0BY4T0jjkNhfYpOpKCrfdjuREwguAO47IqxU10LLMz80n4GDVQtdHOwPySQN+eqtUGOUOHjwseGRxfld6iSrNVXQztyyMjJ+SMzLBnZVPe5japzm6Ws/ZV5JMQY5rKOdsuS+bO+41/5K2IKaetY54kijsbWIslRMno667YI5HBpuNkS2KMDZWMdNiFZFA0bhkev6qeHh3D4/JQvPN6PIuV0c+BTzOfI+1OH6nW4uhpKKOjIeMRgvsX+3yRpjNoZuby/GS/MxgK1TmKnq2sdSOc8m2e23yXTTT4RKG8yqa7IbiwQczChLyXBoJ0aHGxJTkKEtb4y1DSUbs/53agKCTCa42yzQ/TRXKzEWUUpipaSEuG1nhZtNiOJSsMUU0F5nDyAeYlOSeizNh1JMGlz3PewWz30us2eCqlqMgniD9mXd0/Rej4P9jHHOO4bDUNjhp2zeZzZGlpHzXY8OfycWxvlPE+NaOaOW2lOVwPuV4dncem1xN5ej4XUe03a+njK9sTMe6POXhrJfBnlVWKas9R6D6rsOFOEeI+KayJtLw48U8zC9lfIy8buy+kKHgHg3D4IohglJUuiaGh80YcTbut6BsNPG2GngihiboGRtsAvl7u9zONasan4y/J9b7dZ54Tj0mvjPxmb/AG/vbwyl+wPiqd5bUVmG04/MWOv9F3nBn2M8PcMROlxSU4jVSEh73aMLegsV3JeSqstexrizNsvm7u4dRvx4Z5eT811ntH3HrtU6d2z8s/CIj+iSDDsBoLeCwmjhcNnNhbmH1Vk1OfU3J7rIfWyOvZn6IGzzTDzPNl5Zm3wpynL1loT1QYSXP/RVn1Zd6b/MqqWuvuSXKXl+VunVRlIZHkWQiPzXupDFqiEajYQxpIsEYaeiNsf5dT7KOerpaV2Ssq4oDa/mcrjjOU1jFrETl5QlcNLoSQBc7BZGJ8VYXTUck9DUtqpmHK2JpsCVyE+PcXYrA+KSpbRRv25LbO/3l6+n7du6jzqo+b1aei27vOIqPm7p/EOA0jrVeKwMOwF91z+NcdBta2PAQ2eLKLyEX1XIs4eYJBLNUyTOH5labhkbei+50/atOnLlPn9X1dfb9WvK58/qtV2J4ljEjJK2rGUCwA8oCgjNLSmzJIr97XRfd0HuiZQxsNmsH1X0McMcI44xUPdjjGEVjFQT8RjH9eP0QGvEhsyS/wAgFJN4CiaJKqeCNvuses4vw6IcvDqU1BPx+kD/ADWmmu173+oH9FTrMbwui0mqC5/RrBcrl6vFMSrfJPUkMO7GaBU+WwHRgW2WxX8U4pUnLh7BSx++r1kvY+ZxmqJTI87klSN0Gyb/AGVukMBbonylOkgDM6yIXI8yZNv5VoFmCfW4cCR8k1tbIsttCshanqkBrdH1+qbLZBG135mFFmKJoYTle+3ZFEYmXMgJPQLQWUJ8umifc5su56JbmygZK90x0JQh7iPQR81A2UIgLfEmRaEIGDUVtbBM3ZEMx2QMmtpZLr9E2YrQY5tkk39pOgJ2oKQFkjsnRcg7lLygIdAEtwiF1ypaeybX3SHqAvugJOd/KmsL7pswQE70odAE6SAb63TJyNUiQAgJAn7abpkEmYoOn1SzFEqB6pZilmKSjJ82mVLp9E2YplWhZtU1tLpbnRM47fNAkkkTRn0UDdT5km7pW0ul0+qBmhoJ136JeYbtTE+YWF7ovKSMpVDk2TAXSeHgHLo7umQOC23XRIOD/MGkfNM0bua69xsiabAt7qBa+yYb2Tu2TgX9kD5tE4GYoUVtLoEW28oCFOXWOVO3ZAPT3RDofZMWphtfTRAaSQcCLt2TgXBtugRd+ZvRMmBNrF103mujIiepTR93JDsk3ZVo6QumdsnQPl66WTb6BMTqnHtugQ0SOqXWySBJJswToF0+qWp0KLp9E2o1WQxPUpDskkqF0+qVyknbugcG48yFPmvmQk2RII7hOcwNjskbWCSKK1zbohDdcpTkkHME+Zn1UA6W90+YJWub9Er2KBy5IPN01yUgBezlBJe/XVK4BtfVR/h5iL7JjogMnTUJs50HRADdE3L8SA2Xc62gHdCc47JibebKbdbJnWf5RoVQQLrapFrPUhL7tHeyYF1ttVEtYpK+eilZLTzOYWODgA4gXHsuxoftm4/w2Zr6PGS1oIuwsBBC4ezfiAScR0AXg6vtnRddMZdTqxymPjES6YdRs1fwTMPqDhr+UZgE+HRux8xU8zh5gH2IXpPDHGvDvGNJ4vAcQiq4yS0t+K43Xwt11AWhh2OVmETxTUFRKx0Ts4aHHKT7hfge4f4b9Nnjll0eyccruInzj6fH9bfW1d3ziYjZFw+56rC6Kdtn07QfksCr4KjjvNh03JeNgDovnzhb+ULxVhFdfGDHV0mU/hcsb9F69gH28cIYjRRVGJTMoHvbdzXOuG+x0X8+7l7L907XMeLruJ9+PnH9/wBn19PXaN/8M/duQu4qwt+R7G1Te19StGl4xhBMOKU8kV9wWnRaGDY/g3EVIK/Bq+Ktp3fEw3ujrMJpK24mjuD0K/P5Rlhlxyiph6ImJi4UZsG4Ox1wz4RRyjo9sbAf4LguNP5OeBY6Jqvh+fwlZKQbzOcWj9F1lbwW+BwnweomgePhadFDHiHFeENDZgJ2s631Xs6PunV9v2xu6fZOM/X+vxhz2ade3HjnFw8Kxz+TtxhgdFLVl7aoxtu1kDC7OewXls0MlNK+nnjdHIwlrmOGxC+2aXj+gcWtq4ZYXHo9HWYJwRjY582F4dISb5+SwOC/d9q/xF6vp4yjr8fE+FVjMfs+Zu7ThnXhTX7viHLmCFsJG6+puPf5POG8QU7anh6tZR1sflYx18hbfW4HVeX138nTj6hBkfNSSsA9UVyF+37f7b9p6vTGzds8PL34z/eIfO29t368qxi4+Lyvlv6IQ15F3NVyvoa/Cq19BiNM+CeI6te3Lcd1ENdl+twzx2YxnhNxPpLwTExNShdHp5jsdEmA+rMpXtJ0KDJY5W7LaE5ul0spREOsTbROT5RlVEIGtkjn2B0RAd2omjS6gDLZA31KU6E3T5GEXIRlF/YACYXvlcpuX+UISwg5lWgZ8pyndPzB2TvY4nMhcwhA+4u5IaadENiEsxQHoU3X6pmkfmTja7UAddURFxtdIXtlzaJN0HmVDhv5kxFz5boj2TagqCMkgaomm4zNcizZfUAhzs7KgSSCnzO7J7s+JDoUBAE+fRPfpfVLK3umyu6IDY42SLimDXgelJ7cw9VkBcwv9TR+ihdT0pN3wi/caIxnbvsnzFBH4SMfs6mZh9ypbn4kxehzD4kEuYpkLnEDN/ek2S3qQEmtrdMZWdtU2e+6A76Jg64TByIfJAg7VNmizWLrHsnO90OnVo+aM0ezT6Ug259SctZk0NkhohROcPSmyhP0uiy6XBRKRWsnzFIgokUJv1TIzlvZyHy90Q5IuidlQ5QmtcqsvL+TO34j+qVp2/1zgjMjj8Sj5kncKP6+TZ6thuyaQ/VMa/ExtMUg54+SmjpamXaA2PVC0P3jin+uKX3hiXxvJWhDg8588hAb1VxmGUg6vk+iIxhidf8Akap4qzEnbRhbbMNawfhUt1HMaOkbnq6uGG24J1QU2x17/W+EfRWRSOcL5bqpVcT4NSf6FTyVT/fZZtXxVjMzg6kZBSj8tgbo03nUMjGZ5i2NvuVnVdfg9E7NNURucPhtcrjsQrcbq5ediFdLkHRugQ0RgdLepYTH3PVGm/NxjA1/9CpYRb4nG/8ABZVbiGJYw4vfVCNv5W6BWHmIyhlBh0R/fDf8FMaWceetMUd+x1/RZVmUmFsMwfWz54PzXur1XhdC6G9K7K3u7qirp43ZWUsjYQwW23VWKnOXnzVPOPbols0q0sk1GT/RebH2SFXHLUAy0jvdg0AVxtWQMo0CilqIydYy75K23S9FVUUjeWWCO43AWZUVklOS+AxaHQk3urBZCfhH6pMpqUSB3KaXdCeizLMwaPE6qti/GvfbawQyCoDiI6aRw72U4Zy9gPqLppsRqwOWx4SClIfeBP8Ao5HzUzHz6Mk0PZTyVjDTmH4zuVawag+8g9rp+VCDZxG5CrpTMm5znkwN8vRXaeJrqa7yQ4b2VrFp8HwzJHRQBz2m7iTp9FlVGJ1WKtFNHGyGIG/lG/zRiYSGnha10jPOR1drZTUstexrTn/AHRM+dxj5Aa0M7AJhM+1mMDj7onFM6tkmsyeB7ovyufuoOW+neaqKkMbejiUcLZ3EyTyNAO19LKxPh1VVRskkq/wWCwF0aVmYhM94fkaLdkfiHyOL3vsD7JRwUsQyPqAfdqKaEwZJYZBZlstkBy1sAsWRMFhbVqDkmo/Gbyx/BBWTV2IxfiM5gZ0ay26CLDZqzKKh8kTANG23QFIwynKZ3fqhgLMOidkfIc5uequSYfhsMdxUSRvHvdS0FRhxoTSmlDnEWL37k91hFBs3PbnY2Y99bK9Q0VLVMMIw6Rrhs69wrMeHTiAMhLWg91egNdS04hcA1vV4FgtQMk8OEPL6slg9uiuRYDhWmWaX6lPPWc2U0kNWC/pfY/JTMdy6cF7CXjewVSVuHDWRgCnju1WJMDw2Q56mTLL+7JayxBWPr6uOiZPK3MQA2MOLiuk/6FvtCxapbU0FG6SEi7XOeBdefZv16f8AzMoh5eo67pejmI6jZGN+lzTncW8XBmjhfLO38ztVSbXQRUnLfTxiYdV7NwB9hfEFJXiTjB0baWKzmwsIcHHsT2Xqh+zH7PXav4ToT/sn/NeLd3fRpy4x+b6Pz/cPbPoOj2eHhez541T5X4c4e4k4jngpqLCw6SYEsmynJZd7QfyduOKzEoXY4aKOlLvxXNku9o9mr6JocPoMLpo6LDaOGmp4hlZHG3QBTulZH63L5m7vO7OZ8OKh+S6z2363dlMdPEYY/efrfxefcP8A2I8L4LTinq6NtecxdnlsXfJb9NwbwdhcgfBwjTxuYbiRkd3ArbfXRt+JRHEM37MEfNfMy37c5ucpfltvceq3ZTnnsymZ9fOU/jGSaMzD2dugdJGPW9ZWLVEwr4eU+3MbqnAkd633XJ5Vl1dC3QaqrJWSOeA24unEHsjELeyMmAe4auKblAk5gVYaANU5NxYKiAwXbdqKGNoFkfntayaBoa0mWVjbnd7gpETPo1VkYmnZShhyjRUsSx3C8KpHyumjnlAu2NjtSuck+0LFpW5KLC4omd3vK9Wnot/UY8sI8nbV0m3dF4Q7OQxQNvU1EcXs82WfW8SYRh8LpRUidw2ZHqV5/WPdic3icTrZJn9G5vKB2Txy0VM3JDCAvr6+y41E7MvrD6WHbIqJzya03GnFNXMDSU8NFD0JF3rImo5aufxWI10tVJ+/0R+LBGjUJlk+Fl19XV02rRN68afR16Neqbwik8UNPD6IhdSGS5zKvGJ3Dayiqq2io7+Lro2EdAbn9F2p1tbznMjAJ6gfNc1V8VsPkoYL/vv/AMliVNdXVj71NVIW/lBsFR11bj+GUXl55lf+VrbrHruKcRq2FlDH4dh6n1LEDWt2aizO7LfFTSGSY5qmZ8p65k4yAeVgTgdUtApTIQ6/wnTqitrdOZLG7QboWn8/6gKhJXcW5nABDmKdtnC/RaBjXUJvN3T5ddFIM1rZVAIb+ZI5bo81ihFg3a6LiJuyFApG7XSJQJNkV7izuqG1/UkSLqBN9Q2SOyQP7qbMbLQcSGxy6aWSzBCnAN0BJWuewSy3dqQns1ZA+XojtY6tTAdk2fWy0H1BT2cB5moT0vtZIZicoQPuUySbX2QFqSnGZrkh2TdfqgE/mS6/RJ2/slmCBFyFEPfRK36IEcxSbsnSQNlCXX6JZdL3+idA7t0yE7okDOTAEpONynA0+SMkHmwaToEtPZN1+qWUo0TvUUktfSkRZGT69wmvrdLKUtPdGiLNNUj6fUmSOg1VC33TCxJt8JsnStZATn5je2qYOPxJgTf2Sub7ICH5UmAuGZva6a7Rq7ZK+vlQMlfVNrdPsCgVySlayQva5SLmt80j2hvS6lhmm2iLMUILS0vDb6aW6por5TeN8fUX6oLDfM3XT5prDKO6G2YXLkviKgcHMEs7bhlxcbJslgD0KctaTmI1VAu9SJuZLKUP1sgPQhN1+qRzXTICtpZIX3SzBLOfogVvcpy3y36Jv3unZNe4BDbFA5AaRYg6JA3TWzHMnzXF0DZtcqduyLOGxuY5gN+ttQg6Kh72OgToEaTBE2br9Euv0S03SbsoG1CVxa3uncXEoAMxQG3ZOmGyQ1CB0kJ3CJAkkx2SGU/NA4sRfKnFuqYaCyV7lAjrdJJI9kDk9EwtfVOQbprdLarIc9fmmTEJ0ZNsEBLvqpGGzgTrZCcue4Wmgkjd43HROcp2TPN7JwNEC3YLIm/v/qhP7qXmtqglact9d1CTyyDe1zZGb9EBddti1QCRY7o2yuZcZtCor6+lFy2luZUHcnzdEINk5BBy9EwGbRQIm7bh2qYOFkTYr9bJZLHZQMDc7oi0dShtbdqe77HNqEGxgHFeO8NVMNRheITRCF2ZsYd5T8wvUOGv5THE1NV24ijjnpg3TKwlxPzXi5cSl+q+D3L2b7b3S56jVHKffHlL06er3aP4MvJ9bcMfb7w5j0Uk08LqZrHZRzHgE/qF6NQVuE43RsrqGeOphl1D2OuCvgFdhwd9p3EnB0ZhoJHys0DGmUtDR2Fl+A7v/hzOGM7O253N/wAM1Hl9f70+po7vc1uj9X2NW8OYPWtcyRlidjbULArOAwX58Oq3QjpZ115Twv8Aymq55ezinCoBExoLXwNJe4+69gwb7Q+E8Zp4pI8WhgklAcI5nBpF/dfhO49k6/tWXHqtcx8/WJ+kx5Pqaep1b4vCWS3EePOHrxmnFZENnht/8FZo/tJkZaPEsKDX9wbLr7sB1eD7tNws2t4dwavc6SaPzHqviu6gaXgPjXPLimCUUlS4NB54Ad9Ddcxxf9gfDeM4ZJBgbhQzjzMPlIJ6A+y1q37Pqd15KSqkjf0DtVFHVcacNNzyuFbTDvqR8tF9DpO6dZ0meOerZMcfTzmvs556deyJjKPV4TU/yd/tJorl9PTTsHxRSXC4vF+H8V4fqZabEKOS8XqeGEs/VfWkX2mRj/SsNla/2KsVlPw7x5QGlr3wPjnFnRPFnn/my/b9F/iN1+vbE9Vhjlj8oqXzdnaNU4/kmYl8XeJj/Miu0r6vxX+T7wNW0UlNh9I2ime2wky57e41XkXE/wDJx4zwipa3BcuJU5bdz8wa4HtlX7jtvt12vuGU4bJnXP8A93p9/R87d2zfqi48/o8uDwG5XIC4X0XQYr9n/E+D0clViNC1jInDOGuzH9AueaxtrA7br9T0vW9N12HPps4zj4xNvDnrz1TWcUksH26JEZNDb6KJ0TfzfqlyWh2i9bCW7CDbohQZbHMh/BJ87iCoCLkmSNB1TAR9Xg3TFg+FUM4xuu21kNwNCkAwmziU7hY+yiWF2VgzJh5m5gbJ3ZCLEXSa1pGjhfsikHOFxnBR3uPUgIN0BB/OQqJm6FET0coRcj1X+af8f4XD6oH8wNibhC+I3zNfp2Th0nxMB+SXNba2QhUM3N1QnU2vqpWujJs42Pui5UZFyNUEQcb26qTPYDN1SdEXDQoOQ+181/ZBMPmmO4TM7JHNdAQv1bcJbHZCXm/shzuQSEC6EtF03muibsgEj8uycgZQ2yY5h8SfMUA8sEpjGPdSD81kt/NZAAvdFcj4k7mg7JrfmQMXkbtSH8E+RP6bjugG3YlEM35kDdkfT6IH16pX7Ibnqbpea6Ar90hrsm1v7Ja9kCs46pbBOh190ZPc2SuE+qSJTztuGT/GLKZuG/uKtJxdR5yylpZKjsQNCq8/E2NytyQUIhb7o/rjZZh749WiMDu5PNPRUjc9XXxD2bqVxWIYjis3lq8UdGD8DXaKsJGH1zXUHT1nEgf5MIp8zhvI/QBUTiWOOH42Pxsd+Vg0CymeEO7HOPsncXx+aGAR+5CrosVIxzECQOKJAB+YWVX+bVK93NrsfMkvWxSnpHSxB3Mdc76qk3LDOGOWUpqyQw8rl8wO7EIaeLJb2VGXmZstLCXv6WVpsMwhz1BLHflBTktJ3maf9jCLM1JcqzgJrcxgFuoTNqn25MFK7KPiJTtm5r+WyM3UtV6iZNEHOpqjkNO56qGeK0pJe5xPUpo5aqH4Bb3UUss0r75VUsIMcPmcxp+YUwr3lmRjBb2CqywmUgkOAb1VuKOAxCOSoEYBvZo1RQR0dVPHzBCMt9r2TmGPLJ5Gtyb2O6qzRTEjl1BYb2AvpZWZIZgBGXXNrkjVYaV6eljERnmjeCPicdkpZWZclLIS87KQOaWinkjL2neyZ/Jp5R4eneSOqAWwTxwl9U2cnvfdSQRQTwiR8Mo9iVJHUyPqAX5j8grNXFRE8mMy83qCbLbKlFV0sZ5clEHAbEIaqV5HOgg5cXyV2Kkiy2LfMEhYS8oNBPuEaiWFUVNVWXpaWPLGdzbdWcNpa2ljkM8flDNHdFp1dNPIx0MNNkd+du4Qg1cUXInlce4KCnDMwAh7ydUnyPMvKjY/MevZXo6WNzc4nY3vcbIo6R0jnciUOI3LtAVhLBQ0bHyfiT2P7ysZshMbTombgeIVmr5GtH5QFai4bxEH8BsIHdxuStQjNfR3aX5nj+yFEIqmF4gp4/PuLla2I4RjEEQFRUNyv+GMalUIqV9IOYXPzjv0VU3heIAL/hRN6uuNE1VA+BofLiRkeRc2GyGskfXtbHW1pyA3sButaDgvH6rDmVOGYLPUQyC4eGh11nLPHCLymnLbv1aI5bcoxj5zEf1QUXD81SwVU8jOR1cDqfkppW0NFlNBQMmbGfM551J7LYg+z/7UamBlFh3Cte2BgsS9paP716Lwl/Jmc1zK3irGGljowfDwktcx3uV5t3XaNOPLLK/p5vl9b3/t/Q6+eeyJ+UTcz9v+Xj02O1FWGc6DlCM3u13/AAV3D+HuJ+M2SQ4HSVVaId2RtuvpCj+wn7OKMAigqJ3j4pJiV2mDYNhvD1Eyhwmjip4GCwDGtF/meq+fu71hGP8ApR5/N+Y63250Rq/+jwmc/wD7o8v2l8u0P2H/AGkuiyP4fFID8b3AkfRemcA/YjSRYXJHxfE6WbmGzQ612r2N0+X1OAUMlQxvxhfO3d137seHp9Lfmuu9ruv63T4Xlj88bif6sHBvs74M4eqG1eE4FBBM0EZ75ibrpOY63RUJcRZH1VWWve70PI+S+fnnlsm85uX5zd1G7qcuW7Kcp+c2031DR1Cp1GLRQvyXBd2CokyOHqUXIzSAv3WHBediL5BZjSFUJne67nkqwYxG0kbgKGFwIzHcqKkLTyyVNFH5Qha1xUwFkFSujkGJUr8lwIyFaiaLZlBVyXrYXF3QiysMjkI9OUe6NH0CG9zooqzE8Jw6Evqqtpe1t3MbqSuY/wCkUSvLcNwmVrSNJpHDL+i76el29REzri6dtXT7N3nhFutcxxNi5rfm5UsQxrB8KgfPU10f4Qu5rTmK8/r6nFMTqHVFdWSvDtwHZQghpxGHBkTiHdXOLivrY9kny5Z/s+hj2uZq8nR1f2kGYmnwTCnsfYHmyiwC56rkxKvqpKisrnP5jr5WGwCk5b75coCkZEO4X19HR6enm9ceb6OrptembwhVFKGgAMAPe6NtM8dlbbEHfG39UTmxQN5lRM2Ng3cSvS7qfhySjFGHalU6jirA6U2hkNU7szb9VhV3FOJVbDHTjw7T2sT+q206uTwVIzPUTMjb3cVl1HE+GwuyUjH1B/d0C5Ul03mne57u5JKdjWN2ag0azH8Wq2ujYTDGdwzf9VnCL8xcfdxuVKzVJ2iMA8lkVxbMgNrpFw0yuVoSeXLdBcJg02v3T6Ab6qrZXASuw6FxJ7BMw382VPlYHF6Ie35U7A4jL0TyDyHInEt2i7iDbqshjGbG3/sijaWjzPL/AJpAu2RdPqqGSRZQhVsJJP0+qHQhSAuv0RZ2ocoSyhQFdpS8gtdPpshBa86Kofp9UP8AZ3R6Wyt3QjQajVRSTnynMhz2t5eqfNdaCSQk5f1RDXZAidfKhLdfon07apBuqByb+kWCZgygkOKXX6ItSUDJruCe1/LmslbKPdA/9pIuTJ29UDIba2RITt5XWQF6SWpJruSy6oHRPHUbFDsAU1/Yqh81tmptCErdwnVoCd0ydu6QF1kOc1rn5Jv9pK5GiQF1QySR3HskoHdunb6kKfpvsqHO3lQp27p27IehruPb2QkZtxdP8QT7lAtd0yc5QDZqYa2UBXFtkzd0raXSBsgZLfqkR0Ka191Qx1uxpF00RIZ+I2xBIT5YyScmt7XTHQkPtZA5JBJBFuyT44ZGgyAG+gQsjY0kBu5TGEZgQTob2vookJW5Q0MY2wHROX3c0ZTr17JhayWXLs4n5opOkZGA15snzW+HdIC6cmxQETY2SB1QF3mzn6og5paDfQolH83RCfypAj4kxFnX6Ipy5LpmS8vRF0+qBst0sjiN04dYE9Uszu6JZhJrkykkoyBYaag7oE+b2Qoi7Qeyfqk7ZLp9EQ4kDQWuaDcdUx5tgWMvc2CY6/CEQdbbfushnB8Zs9tnDQhLMEnB7vOdbnUpzlHoFgtNEhzFMi0uP3lfRLsBNzmRN3T9ChAvsoqRu6RHl02TJKEhJdfM5LMU+vskRdaDdPqkNyn82X2ambuoC6g9EkI/KnzBA4OoT31OiHMEswQInVOle4smbsgRNkv3roU4Hms7a10Dk2QjdF0+iZpczzIHJBPqv7pgWnRMfZCQS7y6WRISWazqldnRuqQcDsms4FFFpY9wlYEA9+iZmvl6otgge2mXLomGW5YLp+n0SUAdNUrAbJ0rGyoZ2Zo0T8xwtcJ7XFsyYtzBQIm5shEdv6y47FC5tlOGfh97oGAvppbuEuTqcqTGAeXYIr66KCMtAS85GgUjxmN0wdYWvqqIxzALO1UsFVU088c8crw+MhzTfYhR6lORc2WcteOcccouFiZjzh6Nwt9t/E+Axv8AGTzVri4Foc/ygdrL13g/7fMDx809Ni8Taasn9QiGjf8AZXy41tkQzMcHxvcxw2LTYhfjO6ew3a+victWPh5z78fT0+HpX0p79Pct+qombj5vu2jxfC8SjD6DEaecn4WPF1daxrvLILt6hfB+H4rimGVsVdRV0zJonZmnmHf3C9Y4O/lD49hkbKHiKD7wbJIBncQ0Mb7lfg+5/wCHvX9Hhz6XKNse+PSftfn931NPddWyazji+hazhbCcQc581MMx7Cy5nEeCn0zy6ikk02HZWOGvtH4T4jdmoK/IYzZ1j5GH5rrYZWVDc8E8cze7TdfhM9WenKcNmMxMe6Yqfs+nGUZRcS4OmxXifACGHmyQM+GTzBbdBx9SP8ldSyROG5AuujFJGRq1ZuIcJ4bWed0PKf8AmauarUkfD/EFKW1EVNUseLHOwXXB1/8AJ84DqHGegoRA+98tyRdX6zgvEYHOnw2o5uTZuax/vTUnFOPYG4RV1LzWjfODf+9erpuu6ro7/D7Msb9amY/oxnrw2fxxbxvi7+TxxPhUT6jBWnEQ6Q/gsFixq8+xbg3iLAad1Ri2HS07GENJcNblfYuG8c4LXvyTukpnfv2t+q1K7C8C4kpnxV1JS1kLwWkkZjr7r9t2/wDxD7j08Y4dREZxE+c+k1+nl+z523tWnO5w8nwZdqZoYPU1fVGIfyaOBZpubh7qqnLvhdIXBeUfaL9hfFfDtQKrAsKmrqN3lLYLPLLDckr972/257V3DfGiJnCZ9+VRH3t8vb23fqx5ev0eXugg5XMgmF+rXHVQq7inD+NYMwS4vhk9K1xygygNBKz8t1+t079XUY89OUZR8Ym4/Z4cscsJrKKGCb3AJ+QTucw+lrgRvcKMVNRRy86mkyO+W6kmr56sNdOGlw6garqyRkAGpATAtsoZZGW84JCUcjLaHRUTE9SgJ1Q8+B3lvr19kQ2BbsgQcE9wExuAmLyUoSi/bRLMFEDa9iiabneyAwMx90fMIGVxUTzkcOqclpOigkBukRIfS5VTC97rsnLD80rVcf8AXgrQs3d+RDmChzVZbfK2/wA1LHnPlcQfcIDJ/RLy20bZBd2xbZEx2W+l0BEADNcJg5M7VLp9EBb7FPYEZSVF1+qPf0oHsWaOToHEpAlAXm66IP8AaRF5PqQdfqgkBF7Jjm6tQt9SdAXwpm5kvT8kg5ASbQBNv6UyBJ27pNtfK7ZI5QdECO6ZK5PRDqSjIwbJr576WTAW0ARf/vIPDT4ykaTBVDN3AVuhpMYr2B78Xjjv0soWMe5wYG3up5KeeJmdrbNCy/rzUiwulgYec2KaT8xNkTRg8YEdRHGbfC0XKxWSPk0Ezh8lHPUvp35HTyH6o1Tblr6Wld/RcPaOznBZtS4Tv5jx5iq75CRpI53zUJmeN3kIlNBsV4CXSMt0b1VQGhMnLmtn91AKyXPkjGvchTSspsrCWEyO1NkVfpaqKkjeBDmc8WAsoBTTzSaNtdRtIDgSr1Zi0la0U8MYgZaxtv8AqgAYFitszZoWN7ko4KKCikElXi1Iy24us6ojp72qJpQzr5imw7D+HayrbFJNNY9TcBYYatZ4LEnCSkqw4Rixy9UNHTupHCoqOW1nUuV7+bOG0zCcOnJeexWSKOOB7hPPzH9ddFtQ1VSySpyM1jKYxwBzTBAXP6WRw4MxwE76iS/8FYgo6s5vD0z5COo6I0g5E4HnZC09t0rubHy2mxKKVtTTPyVNFNf2FwpqOnp8QlEb4ZmW1J6AIAbhXLi51TWWvs1rdSiDWTNAZAbN6nqtsYbQse1hMhadyRsmnw3B82Tnu/VFty88zxdkH4br2zbqtNRSlnOmfPIR8V9l18OG4U1+QXdfuVfiwPcCEZD8N9ERw+HYaypfriEg7Nab3V04ViLCfDszN+EnddjHgEDNTDy/7IUoojGQ5jQLIy5KKl4g5fLEO/W6IYDisrQ+eohD+oK6t0b3nVgTQ4OampAie97zsxmpT09UmYjzlzf3FlYDVPYT2ab2WrDy2saKanjBAtey2arhzEzCcmHVzj0Ahd/koMF4N4qrZoqNlFLDzXZQZoyLfMrE7MIi5lxy6nTjE5TnFR81WEEOBIsnZVS1U3hcNpJaqfoyIXv9V6/T/YHg1TTRfeeJ1QmLRzGx2IB9l2/DfAfDfC1NTx4fh8Tp4G5RUvbeQ/Mr5W7vGnCP9Pzl+T63216HThP4eJzy+lR9b+D5rhwP7QquURjh2oHuY1v4F9hnGvE0jK7FnMpKZ0mV7ZBZ4aOy+mHSFo3Qul03Xg2d63Z41jEQ/PdT7cdbu1zhqwjGfj6/1eY4Z/Jz4CoJ46mZ9RM+NwcfPYOseq9HpMKwbD2tZRYdAwMFmjLshlq2R9VEK8O9K+Zt6jburxMpl+U6vuHVddX4nZOVely0c5d6So3SgHzLL8dO6UsvYKGR8mc+crk8VtSSshj8rTc9lWkrpSLNZYKlELuu5TZbqKGSaaS93lO2M5LEkpOFiiafKO6ojlYALuKAC+wNvdSz5RGS8aBDE+Mxk5tgo0UbDZDldzWnoEhOwJOlc2zrAX7rDCaUtyHL2UEIaSGFUKzibB6V2SWqDn2uAwXB+qxqriJ9bOBRSOhaDo7Jqu2vp9u7zwxmXow07Nv8EW7QNLd3AD3Kgqq+npWF75WOA1s03Xm1TS11TWiorcWqZQ3QMa/K2yliY9jhlkeAdDqvrYdl2zETOUQ92PbNkxEzLWxLitlbUNZRUso5TrcxjrByycS+98SI5mMTRRt1yh1ipw3RLL819XR27RomMoi5+b6GrotWqeUR5s2PCp2E/wDxIuB3IBJ/VTRYby25PFTH5kK8xuqPKF7oxjH0h6oiI9FL7rzf9bn/AFTjCYA4OfNO63/eJVePYVh4tNVCR/RrNSsSr41mlaPu6hcO7nkKq6NtHTN7/wC1IqFZjOBUTHcycyPHwMuSVyNXiOKYjLzK2V+Xo1ugChETPyfqto16/iMVbQKehEbe7ib/AKLHljNU3JPJI9vYu0UmUD1FGNEEDI2MaGMFg1SAuBAIP0CJ9gLkaJhI7oD9VpLPd1v2ZB7lM8AEZdTZOM5sla+uXUIWY+26dznBuyFrXuJz3HZGG9SbooPpr0ROB0tv27o7DLZPYXzZVBE2wdbr2ROYCLnL7AI8pSvcKrZANARJspIzIg1ZRGHDW+6YgHVIAfl6p1oFr7IszbIkPX6qJdnteyRNtErn6JdPooIwb6NSykHKU2226IuzbLSmO9kyW5ukgNINFi8NANtk1v1SGp12QO9O03an03QnZFxFoUtL5gbpbEpWHZRA2dbQC103pcncdNEw2VCzBEL3v0SbqEyAiNLp0OZvpTi2l0DFyXT6pidUkDdPonQ5ilrmt1QMnvpZK2tk9tPZAx1CXX6pDYpkBWv1St7oS7VIOF0SiSPsnO6a2iIROtk4NkSA6Ky0e+mXoiQkEbpN3UCPZIbpk43QM4C1yNE7d0r2CYHXZGSBcToNk0j3NAsy9yjOYfNCXHMGZSSeqrRJmi1/ndFqQmA1QLUG/ulfdMTqP4p0BZnEBnQbKJ8Yl8jwbXRE5SPnZGRY23UAD0ebdP1+iQtpl6JHRWSCJ0UUWZznl7r66eykuCo4xZ7/ADXJN0Bk2BuLEGyRcG3O4CYSMcD+6UTjHls7qECbI17rB4JUigaI2+fIAe6TpOSwvjfcddeigmzBDcndAyVstsh0IRNie2W/iC5v5eiAx+9si/sqK8mYgtGUbFOsh9PdLcpFwBt7XSBstA3aBDr7JHX4tLojcfNANvcpm+oJE3S6oDJN90sxS3KH1ORkW241Ttfa+l0idLflQBrlkONfiTEaNIdr1CVgiv8ADlWmi1sMycG4TX0smQF0Dkw3TBut0r/CssnLkgbIXepJ3qW2hagoc1yl0PdNFIx188TmOHdBJn29k13EpeSyWXVECC4E+VOf3U7t0iPLcKKZJuqSSBJswS6/REDZAsxTg6JvLf1IXepARclr7pkwcgLMUi9oFybIT1KfyuGRzbgoGz2TCSwsAiNg7RqSBhbU21KOzrZkLdUWXQoHByn0py5DfMLIgQWjKoHBHp6p+gb1QWN8yWvugbXX2T9LpddUvlqFQxtbyuThzRYO6pe52TkAnMBogI32ah848rSn19kuv0UCHmPmTndA5jj7JxmsATeyA2kjoEI8x8zbFIFqWbVAxvdO0pXCdxDBe2iANeifM7qrEEVLUXDKuNrgdignpXwSiMyNeCNHNOigAGyV8xSDbWJTi3RUT01fVUgPhKiSEONyGvIv+i3eGvtG4s4VqHVWFYg8yOtrIS8D5XXN5eyZhuvH1PQdL1mE4b9cZRPrcN4bc9c3jNPo7h3+U5h1RLDRY7hkx5uVr52ANDD1PyXqeDcXcOcQR8zB8YinN7ZScpv9V8OOboVZoMWxXD2GKgr5oGONy1jrAlfge6f4c9Jui+gznCfhPnH931dPds8fLbFvvEtt+0aT8kM0EEzOVkBB3BXyhwB9tHFfC1QafFMRlrKOTV/Oc55b8l7Nwr9u/CXEcslNVMdQviAJkfdrD8rr+f8Ad/ZLuPac55YTnhH/AFRHk+po67Tvjymp+Dq63grDJ7mNj4Ceo2WI/h3HcKl52F1T3gf6l1iu4pqllXEJ6GqjnhOocx2a6N4hOs0eV3UtK/N+j2OPpOO8coCIcYpHSW+JwLSuhoONMExB4Z4jkvPR40/VWpsOoqxnLnibOw9Dof1XPV/ANDkzUE5p3/lft+qg2sZwGgxqlcKjB8Prrm7eYwXv815VXfybcGxKtmqo3yUT5XZjGZLsHyK6Sai4p4df+DLK5nR8fmCt4f8AaFiMLWx4lBHVD8zQGn/ivo9H3PrO3zM9LsnC/hLGzRr3fxxEvBeNPsF4mwGWoq6SmfUUTNWZGue5y4TFOCuI8JgFXPhk3hi3OZSzKAF9uUPFeB4lpHVBjxuyTRSYrgGC43SPosRw5ksTxZwaLAj5r9l0H+IfX9NhGvqMI2V7/Sa/5n5vnbe06s5vCafA5Zolkj/KvpzFv5LvD1VUSVOG41U07XuLuS5tw32BXmn2gfYdxDwlE2toaKeqpmC0rmHOb30sAv6B0Ptp2nr9uOjDOYyy+MVF/C3ydvbt+rGcpjyh5eIowb5AldrBly2srxwzFCLfdlQD25TlVMOpa46jQgr9Rhtw2fwTE/SXinGY9UXMafK5LT2RCNiXLb0W2AtDR00CLS+jUWXKUiG9EAu3SBCI2shyNKKV7HN0TXB2Ra5chNwo+XlOZFSBxBT5hfTRAn//AHlfQGSCmJb0QA65UmvN1RINbpXuLaXQ3CQN0DpA2THTzFODbzB5QPmHv9UtDskXX902u6BWKfM70p7t6pXCBxukXJgXBONUDaeyFE7Kh6fVAkRAt60PX6pEAjKUBJxugEQF8jyD2tol+INDZAWeyQLSNN0snxX+iWXVGSL3dgk0myRIv6dU4KDx3mZGF8NPJmCgFVXTTZJY5Mp6WRkYoXmSNn8UVVidU2n5U3r66rL+vSPwfN2kEfzVZuH08spkqXh2Q2ACyZakl3nmI+SuQ11DFGG82ywW1GxQBukdmj3VfLM7zQxhw91z9TLiL6syUQe9nbdb2DzY1LEGnD3tA3cY9EaXZqOGnpG/hl8zxfTos0CSIfiRnN2C6ClwqerJmqpgzuQLn+9BWCloHhlPUmSX3ViRiGKryiR8PKafzHVRWIeCXXXcx0VJLCwzw5n2udVSxPDZ5gIqGCONg633WhzTJpYz5KXmLTp8PrKyITuw+zO+VaWEeFwZjhiM9IXvNxmKszcUUJf4anAcT/q9lhm1P7urpoS3kuYD3UkGCU0cYMgvJ1V2OukyiSSCQN+SGplr6qIx0dFKGn4yNULVm4ZWyOELJ4YYfyAXKuRYdPE3LSyadTbdR0sT8MiE1VzC7rfVNV8RQU7Q6PmD2yrbSwyOqvaWTVC6j5c+fmyH/JRRYlU1bRJTRtcXeyUlLjdTflxlrztcGwRJmI9Ur5I3u8KAS89AFYDMPihySQNPueiqRYFjLJWzT1Ye0fC1pF103DHBGIcVYg2jjjcyNpBlkI9DT1WM9mOvGcspqIefd1Grp9c7duURjHrLk3eFc/m0g5TgfUVbp62tDgyOWWQ72YLleus+wOGMaYu3/wBNdpw59nvD/DohqIaOOSrjYWOlI9V9zZfN2936fDG8JuX5jrPbPt2jXy0zOeXwiJj72+fs2NzxZ46erv0/CK6/gj7Osb4op31OKTS0dLa0bsnme4HUEFe6cuEbxR/7oUjXRsFmlrR2Asvn7u9bM8a141Pxfmur9uuo36pw6fDhl8bv/h5gPsMw8Nt99z/+mF0XCH2b4NwnK+qaTVVJPkle0AsFtQLLqn1ULDbOqk2ItBswELxbOv6jbjOGeXk+B1Pf+5dXqnTu2zOM+vpH9IaPpFwhD2HchZP3hLJfLdB4uYnVx/ReKnxaa8lTBGLl4UBrmdNVnWBNyi5Vh5VBNJWzvv0UTZZHeom6EtckxuiLBpA4hBH5NbXROBv5UbdAgTAM5OXdMfV9U19S5CXtB85AVKGDZEHKrHOx0mQX+aCoqoYmOe+YNIUFxzweqKOYDYXXLv4vwYSGGGodI8GxsgqMcq+RmpMrQ7QgnUe69GHSb85iIxnz+Tvj023KYiMZdHi9R/8AD5mC2Y9CbLkZuJo6GYU88jxm6MbmBVUVFY6USVVXJJ3HdE58ZOdkTc/U2X1NPZsuX+rPl8n0tPbcuX+pPl8l5uM1FaMkZdTs/O4a/os2rZLLVF02L1LmjQMa6wPujL5HnVPyr6r6GjtejRlyiL+r16+h06p5Vf1A58DsmSJpLNAS3UJxnJ2UoZ7IsoHrNh3JX0MdeOMVjFPXFYxUI8pRjMBbKqj8aweF5ZJiMYI6LKrOM6YeSgpzMfzHRdKW3RWO1lWnxGipAfE1MbXDouNqsX4gr9JK50cf5YxYKo2nB1ke9z++eykwjo6vjGEEw0Mb5D8rf4LDrMTxXEdKqocG/lboFGI7elPkcrQjjibEQQ4D3O6ONrBsExapGCwWg3kAOm6bdP5eqRIGlkA2J6Jau6jREfzJ7tAsUDPASDRfNdIgW3TkW8lvqi5HA0SOycA9Ew1UQF9SOyfbzOciQ3+E6qgifypHN8Iumtb5J7aIFc7EJwf1Tjc+6XX6KArFBfW6K7gQns0lBGiu26Q1ciBDXFrWjfqqHJsle/wpaXTea6iE7QJgdB5LeybzXyoxmtq/Miok1he6PKEKoQHQJ9d07mWTOdYalAeg1O6Vzrp7lDbzapxugTTe7hqkfV6Ug6xskXaokH1vobJ27e6bcodj5UUbm2SGyYk5bF18qQ9IyouJk+ezb2ueyHykdSU2xRCBOmYJ7OO6LO4HyhNmde7giWYagOS810gddkzjZFLromTt3SG6B/U5OmdsmGY7IEN0yfulpb36oGO49kkkztkDnM8gN0skkkgSR2skbDVLXqgQHQJJJxugThYpZj3TJIyYl/5v4IgbFN1+iViN0BZv1TalpZ36pkreyrRms5Yy5ifcp0k2axUCOpzFOB0CScmwsqGI6FIbIbk2u69tk4OuuyBz7JE3GVM03bm2TWLzfOQFAQFkLoopMwN9BfRGcvwm6i5d5C65CgTYo4r2BJI11SDWyRmOxHROB5DqdD1SYdFQ4uyIse4FvUkKAT07iYywKa7ZWkFp36qPw1OSHOjOnS6okPLhaAG2GykJ2seiAtu3LsmbHl2vZQGCbWNvokTpZIdUx3CAxlLUhoQUjt5eyBjgTY79kgFcA5XI2uuLIX7ZQ0fNM3dGTuTuNuiRCZ2vS6LJx3zapm7J02bVWkECL+ZPcZbhRi19U+wUaE12U5k5IeWuay3dCAHepyRB2BQMS8Pt8N0WxUbTNtJbTsnzFAQe9l2N2O6cOA8xUdntffPdpGxGyIPs4O7ICGvTqhLkrm4snQD1+qcvcfUluSmB1VCAuj1BQ5gl5uiBbepqTtkswTqBJtAEjskNQgZ26XT6pkfluS3r0QMRcJdMpQp/i8yAk1tbpXsM3VK7bID8gboUFvNsmNrWKduyBhlGwsEepKjtYFFrl0QIuT5rpsuiTdkDdfqj6/VCWpWu3KUEjTbUdU9ydFGh1uFkSgAN9V0zSQ+zNz0QA312R7EOWg4zEnOLIr+XKExN7uKQ2uoGvpZJpHZC5MzoqkJvK4oBuQRr01S0sns0a21RQ7G6kBBJcmGZvq1CEE32UBFuqQ/KnBufN1Qi43H6oInQgnOzyu7hJj54x52835KUmyYbqhNlEozi/wAiE4OlkjbXKkCDoVkFd1zqguQQi+Ipi0ZbudZAgSb3T9PolY5bhMA8HV2ioPW9yhdHGT7pFyTd7oN2l4840w9rWUHEVZThjQ1uRw2C9k4M/lHRNwttLxkwzSx2aZmAl7x3K+fg4IXL893T2W7Z3TV4ezXGPnd4xET/AE/q9WnrN2ibib+r7Z4b424T4spIa3CMVa0Si7Yp3Bj/ANCugJm/clYvhPDMTrcLqGVdFUujkj9JvoPouswH7aOPMCxCKb74lnp2OvJA8Czh2X856/8Aw46rDOcuj2Rlj7onyn6fD9fJ9PV3fCYrZFS+ww6n9mlUazA6CsaOfQQSdntNivMuD/5QvDPEQZBxFA3Dat8mQBurXdl6vSTUmIRCowqsimj7sK/C9b27qu27PC6rCcZ+fv8ApPpP6Psad2G6OWE25Ou4CD3mTDqoWP8AVv0t9brNfVcW8O2h/H5TNsxLgV6G4OYfx49O7Qhdy3WYJQR+STZeFtylB9ocT2htfSlp7tN/8F0tFjtBiVOX0FbHJfdp/wAQszEeGMIrnGR+Hcl5+KDZc5VcC4lTv5+HTCdv5eZleFYlXWvwWixVxZiOF0pY744xYj9F5RxZ/Jnw3E691fgGKyQNk3psgFndTmPddFR8RY1gcwjmjky/lkj3+q6qi4+wifSqLoHdnEWX0O3916vtWydnR5zjM+U//qfJy26de+OOyLfLf2g/ZHxBwO2OpFJU1NI+wMrWXs7tYLh2QVO5pZ//AEyvvunqaTEIc8L454zu2+Zv1WbifBvDWLxujnwuFhcCC6JjQV+56D/EfqdGmNfVaueUf9V1f6U+Xt7PhnleGVR8HwmZGbApN1X0NxH/ACWHVNbLX8PY3HFTvN2wSAXB6rybjn7N8f4EqWx4hSyvpn2AqAy7c3a4X9C7b7Vdr7pnjq07Pz5R6Tcfp8L/AFfK3dDv0ROWUeTkXhtstkOWxRljj6GvJ9wobyZ9LH5r9DExPo8Q3Xtom1IuiGbUOSAWmgHbM1OD7XSylBl81w4hA+YIi4gA5SfkhzBIOI8ubRBIhz23Qu2SOyEQJup8yWxQ5gi6/VAr2GZydj+yYXbqLEdkI39KokLtUN/yom2v59knBt/JsssmzfNOC5DlKa7ujloSE/mTjL0UYzH1IgbGyNC0Gjki2/sO6Y6jNmCYudbKEDlpYdDdPnFvO7VBc2909uvVAjunaUyFqJaQuSQ2PZJzdEV4691Y5+emqo2sKp1FLVukOcc0nctW3h2HYaYQKiST+y0K/S1dJRZo6ShJI3cVl/XHOQcNyV0QZFhbhfd79lJJwzDh7/DvAc8dtitjEeIKh4EdOeWBvlTvjnrKSPmMkDn7myzSoYKhlKwMdLTwn91uqmnrue0MpaoHN1Ol1BNgcMLCI/NM/cuGymo8IjZD+M+QSd7rXFR1FNXvgP48Q+SCmw2rbEHZYj8yiOHsjPk5r/kVcpaiGOMQyxyZh7KUIXYmyhjtK3mP/K0rHrsUNTKXvqJIh0a3ot77qp6upM00d29l0/C/2bT8UVbvu+mjjY0EmV7btv2usZ5Y68eWc1Dzb+o1dLrnbuy44x75ebUmHUlY81DTVTuG97hb2G4fHRtEzKVsVvSDqV6q77C+IH+T7wpWt7NBCuYP9h9bBWwsxKrifRtd5wxxzW9l5Z7h00RfOHyM/aXtWGM5eNE18HljY5nvMhJ9h0XecO/ZjxPxHhpqXVDKMPP4ZlYSSO4svaIOGMBgiZEzC6YtY0NuYwSbBaUMMULBFEwNa0WDQLABfM396yyxrTFT934/rvbnZs18ejw4zfrNT+zw1v8AJ6x7m8yfiSld7cpytN/k8uPnqMTp5nncujcvaXG26hfWQM9T15P826r+b9ofJ/8AGHdv9yP+2P7OI4Z+x/AcIZHPXx8+rikztewkMt0BBXbHDaIb0FN/6Tf8lBJi0IPlzKN+KTON2Lx7eq278uWc2+H1nc+r6/Z4m/OZn+n0j3LXgKAf9Qpv/Sb/AJImMo6Vx5MUMROhLGAX/RZj6uok3UZzn1uP6rjOUz6y807M5iplrurYejwVC7ENSA5UALkAKYRC93OJKywKSrlLt1E+SR5vnKRj817pwOyEGEZBukW65sqktfTNZPltpdFREOtbZC0SA5cot3UztrAhMHNIOuyARG++uynOUbqrz2f6wKN1TNI7lwRmQ9bdFbFovYgMsY+JVX09cxhmlLY29czhoqBkpS+0+NUbXHZvM1Vxwyy/hi1jDLL0hqy1MY2KrS1R+ArIxDEBRROko8s8mzbOubrHirsfqPPWTQRZvhaP8V7dHbd++OWMfd6dXRbd0XEfd0slc6Jhe6UAWue6zZ+IqfNy4GNll6Ziqj88jGiZ+YjXMNL/ADQBoj9K+jq7NeP+rNT8nu1drvH/AFJ8/khrq/iesqRyHw0kQ0Jvcu9wndDVzRubVVkk2be5sFPmd2Szr6eroNGqIiMY8ve9+HS6dcREY+ioyhji0jAZ7gKwIh8RJUobm+FVqvEcOomE1NVG09r6r1PQk5bU/KaPhXPz8a0IdlpKR8x6E6LMquJ8eqWuZG8Qtf2AuFR2ckkELM8sscbR1c6yyazivC6fSmzVTv8AuxouSbHNNrUVT5Xd3OCkbC1mzh+q0y1ZuLq+RmSlpBF8xdZc9Zi9c/PV1TyOwFgntbqP1SzBaaRCljHRG1rQkQSd0svu5QGCOyYkAlPbRLLfQoAzflT38vqF0+xy6JAX9SrJv3mpA2HmTkOJ7BD8VundGjNdcXbG/wCuid1+qWdwIBBIPZEQ2+iJBru+JO8+X0bdUnBodlYf1Tm98ma6EhBb+VJORZPcIoc1hqkmt5rIspQN5g7XZFbXMhLdE42t0QP7JZfa6azr6iyK52QOIySbDYXKBxsddu6ISWBdmvqla4sdRugEOBFxqnBsDZNYfC2wTgAaIDzX+GyEg9Eh7JG5OqLRW9rJ7aapX1TuPsjNm09kfp6boB5RmSuS4F2yKkvYJ81xb6obG+yYluygRkedE9gdxdCnaPiSQybzdN0+zr7pEg+kWKobf1b9U3xD5p/S5LL2RII+olOkkikDr9Et/MkcwCbrlQIHKUruJzJWbZLYIFbXdNc90+bW1k5GiJYdynIFkIJSQokra7pybpWQsIP6pDTT+9IjYpr2bdFEn6JhqLpyRc/LZEkxbrula7k2e/SyQJDgUU7r3skkTv8ANO3e52VCOo72Qku9STiBsnUZMDvcIhqmS0AQJJI9ShablA/xJ2jzHTdCPUW3905Nh7oHTXdfLbpdPc20TXAF7FVo57jZCT5m7JyTe3RNZ99BfX6oHvrYdE13fkSscpy72TNBAJzboCAJ+ScmyFmcNs74UiRsb6dUDNDywZ3ea5umiLshvpcm6kBumcdLqBgABcAWPZCBJzA5+jR/FEzZRT3fblyZSEEsjb+S5CrCWVspYxgLXjY9FZBzAO6qvLFJzee3cCyCZhNszhY9k0TZeYXveLdAEopeY1peDcjqFJoFQ5OqTQG3y31PVLyWLn7AKGSaZrhGGgXOmiCVzCTbOL90smYefXMkLkeq2ifcb2UCjGS4J+SRAOvXa6SWZjSQ8j6oHtonzOtdo+SY/T6Jjoge+lkIADr5rJxskWgtQHvqonuBuxzrIw6xQuDHk2buiUAPy+XdSg5rA7X3QZBf0pwDfTZCkhMJP4N7A21TZghGVp9PW6e+t0U+YIQCPUjuweu9r9ENtbIAa2QPc5z7tOw7KUBuXzO+iVwPK1Cd1QSY7JtQUsxQPbYpaeyYk6AJO3UCdupG+gH6IXHQC+iTEAgXNktfdMdU4N2/TVAibpAkahLp9U/T6IEGpOypaEJWPxIGJ0unIOh7oU5GzroGsSQE5HZK2tknboEdW2KQ7pk+wzFAuuqWUpZilcXQGSS/K/ZKzU0mQuzMbom6fRA2Yp9fmmFhu1IgOPlbb2CAnDZJrTluAmO9kjfoiwWfRK+ltU2UogdEQzdSj0Cj1unzBAV9bJg7VOMuU90yJB9QSHJ7hCTqnAuC7sinB8xRWBHk36oWiP49PdIuAOh1WWTkFrk5cSBdNmzFI7o0RN0hbLrunB0207pkWDu3Sbuk0XNk2WxsqhX1uiBuBmS8o3QjN8KAmjzad0s2qe8g0LemhQm+56oE7dNdxOUNuU3T6JEaZtlA+vVtj2SJIRGxGbr3Q/NAwJA3skd7p0kZJrrODg4gjUEHULfwLjzinhpsjcIxJ7Oa4OJd5jf6rARaLzdT0mjrMPD6jCMsfhMW1hsy1zeE1L2ngH+UJxBFWGk4rqYqiGU2ZI5gaGfMr13BftX4Bx2eGjGO0zKyY5RGb6n5r45cbpoZZIXtlhe5j2m7XNNiCvxndP8AD/t/XbZ26ZnXNemNcb+Nf1p9PT3Tdqx45ef19X301lQ0ZoJuY3uDcIM7ATz4CH/mabL5GwT7cOO+HoY6fDaqIRxtDS18ea/zXr/A/wDKAwrGqFv854oaeoZZssuYMBPcN7L+b9z9je6dr1+NnjGWN1+Xzn60+rp7hp3zxian5vV5KeOpaWSwQ1MR6O3WBiXBOF1j89K+SkkPQ6haWEcRcM8Rgv4exqCpePVkdq1aJFS02kYJB7r8xnrz1Zcc4qfm98ZRlFw89m4c4qwB/i8OeTbd0Dhf9FJS/aJjlM4x1sDZnDfM0teu9Y+A+h8kLx7KtV4Vh+InNiOGxyj/AFgFiVyotXwnjXDMSAY+9NIekrhb9VrzR01bGYquljmYNS17A5p/VcbWcBwPucLrstto5gT/ABWcW8Y8PxiEc1sfSw5gXTHKcZuGXUYr9n3B2LQSQHBoKd0jS1zo2AGx9rLxPiL+S1i7a6WXhvHaZ1K43ZFIDnA9yV6rhPH0gyw4rBm7ytIH8F09NjmD1jc9LXwuP5XHKV9jtntB1/aMssul2VfrfnH2lw3dLq6iIjOPR8c8d/Z1jPALad+MSNIqHlgs0gCwv1XJCRrtngr76raLDsTjbDiNBT1TWm7RLGH2XL4r9kvAuLwyx/dIpXStLS+GzbX7DKv3fbf8SJ1accOt1znn78oqPf8ACvdD5m7tF5TOuaj4Pi/K4m4SIf7L2XH/AOTDxbR1M0mBYjT1NIHfhtdmz29159xT9nvEvBrGS4tSyGN28jWWa35r990ntJ2rrs8dejdE5Zeke/6fV83Z0e/XE5ZY+UOZLLISDbZSMLHC4fdNex8jrFfceZF08qZ26lIuLndRkFVgOvsln91IGjqmLAjYRe6a9k+UpiOiBNlf3BCLNf0qOxGye9jojCcSPCQOurQoy67kTcqjYs2vkGnZMclhkvfqo25gd0/96jmK37yZNubtSzOutOhOJRNJKHMEswQSZX9rhNsdUGd/5k+/qQHexvbRC51zukHEIc9na/ojNPLaHA8Ym/HqK8xQDcgWKlxKlp6GIBlTJLn/ADHZdtgfBPEHE7pG4fCXiMXcScoXYcL/AGH1U2Itk4lp8sMdnesOz+xXk3dXp0RPPKLj3e9/S+s7x0XQxl42yInH3X5/Z4phMdPEWyTTCR43AXo3DnCeOcUQyS4XTlrI7eaUZQfl3Xt0f2YcBxejhukH+yVv0OHUWFUsdFh0DYYYxZjG7AL5W7vUTjWnHz+b8X13t1jOqujwmMvjlVftLyDhD7IcTp8VbXcUMhdHDZzI2uDg89Q72XozeDOF374BSf8ApreI6FRl7W7r5W/rd2/Llll9vJ+R67vnXdw2eLszmPlFxDJ/mXwp1wGkH+wnHBPCW7sBpD82LTdVxtF7qM4hH8JXHxtn80/d4p6/qv8Acy+8/wB1D+Z3Cw/+h0v+4tHD8Ow/C4jBh9LHBGTctYLC6gOIB2wQOq5CMoFljLbnlFZTMuezq9+3HjszmY+czLTdKxvqKifVwtOpWaXOduUJ0C528687EmD0AlV5a6aQWZoocrinay3RW0gD5ZSLveVFluc5691aewW2QMaL2AUVGIrlTMisLWRMGU5moeac+W6rSQRaoT6rZLpc26Eu+LMgkj03ai16KESsyEmQKu6uAJYw69kFx0ecIA0Nss99XUE5gbKrJWgnz1mvZtlIiZ9FiPg23PaNyqk9cA7IzVZlTWeDi5z3HcDK54uVyeJ8b4u2WWlpMHaGtdZsufUrvo6bb1Ezjrh11aNm6awh3Ek88jSGFVZal8By1FQxp7FwC4qDiLiOVnh5aVkbT1Ljc/VVK3DZsVqzPVlzRlAytf2Xv09o3Z5Vt8o+72au27Msq2eUO0nxeihgkk8RDNI1pLIw8auXNQcecbyuEcVBHG391oTUWFUVGLx0oDh1JutCLTYAfJfT0dq06onn+Z79PbtWuJ5+apV02JY3HfFaqYF3qa126KkwHCqS3JpTmHxOdcq7c+yJoLl9DXqw048dcVD2a9eOuOOEVBuW1vobZDmde2VScs97fNZ9dj+D4bYSVImkPws1K7RLa8xxdupGxvO65ao41lcy1DRBl/iesaoxPFa0nxVY7KejdApLLs6vHMLovLNVDN+VupWTVcZQeaKgp5ZT0c4WXNCGPspIm5dQtwsSsVGMY5WXbNMWtPRosFUNMJLGTmF3clWQ72Ql5CqWYRxxvaQ3QBP5fhagdnvdltd7or2HpupQRy/Cnfq0C102pRZfzKhgHgC6bW/mT3aDojRoKQNhomdskBogMG6QOpQjPbyNuUzX6ebfqgc6pfiX6WQk301+acbBvRBJc90tQUyAvcbjKfmss2JuqdCGotAFpoKK1iha5ttdCnOo3QOdR7pBtwkxthbdEW2CJIdgl5na9Uhl6oXHtsijO3RAhJJGlrp2Zi0OLbXQEbWtm1TN/tJ8pToEjZZxtcC290LhbS2qXT5oWbpdNdxS/uT3v0QM1uUIr2aW21KVySG9EvKdUCs7qn6IvKTmt9EPU2QNpYg9Uh6UiNilfSyB/MLE3shTn3cSmQFl90iLhM5yTRYeVEstfdIa2SOew7HokN0UfW7m3TG40d11TDNrdNoDfNsiQfW/kdYosxTfEUxy9kUvM4Jun1RWb+dAgcm6cZbJrCxzD5JgLbIEQfqkMwFi69uqQuUkD5imSTi19UDJ9ymGb4kkCSB11HzCSYAX90BOPsB7BMhdqntcKlloTqE6Z2yfQhAibJMdrtdNuP7kibHMEDnUEJnbJC/+STrHdRknbpxsL6pgxuYuc467BNn00Fz2VCBvcJm+pLU6bFPbKcrkaP0JamTggAtTRRuyuNxugLT2TE5Ql6moS1rgA9twNkCOpB7JvxC7yD6p2kgnNa3RPchA7/TYO1KR122Q5tUbG9UCb5UztrpX1tl0KYA3Jve6BwOnsmLdMuY7IXZi4exSAsT5jqgQLLkZifkENVlbFmBDTfcomXA8xue6dwa9urQfmoE0Pjjax5BIGpCRzHZLRgJPQJRyxytzRuuLqgtwCd05F1XfVsEvJyOv36KVok9iECaQBk1PcEKUctxY6Rl2g62Qj3KZ7vK7JY2USTO0Tt2UcEskgLpGj2Uvw5kUlHJDHKbOvc9bortLsoSEeV18xQJreWAy5NkiWg9USATxyFzYyCWHVAYHQJKKQvuHvj8ncFSNF2j3QSwtgc/8Z5YLbhQOHmuEeRoSAFjmQMM3xKU5Mg8nmHxd0IBGpPyTE6IHLH5M+XTumGYbKOQGVvKEjgNyFINB/ZQH5rKJkjzIWOZp0KLMU5eAQGoE3ZMd1YpZKQZ/FQudf02NrKB4sgR2CWwSd6kT8lgBqgjT+Xr9EW9moEBW0vZNrsn+FMO6oWxsmSdmDr90lAkWbRM0Zzonbl6oGB/RJxuU4GV56JyGoAS32KSSB8xT+W46oNb2T3APugd3qPZI6hIatzFLrogZGQLZmoEJkY31PDCdgUEmnT6pxsgv5bh/0S1VBpsrt0wKWc7KQyR9STd0spTbdEEpGqG+U26Ji5JrbqNGbny3ISPtukHPZo15t1BSLMzT7Kh2n4U23lumubnMEs2cX2PZAZva6Zw+JCL9dkiezkBDMSicGXsUEbg6/cIiGH1Eg90BAgNslodUD819EQyi13elQPYICDezSPmnc7XMw2SvYZi3dVIk7AD59QmIOozXT73TncLJBEXsnA6JC3VIus9uUaIWWvpRggtQONn+yfL8WbdVSIsdU5Iym7kxv30TeYWIaoGbIA4h4dYjQ9E+5SDj2smLkBFj7ZkwOqAvN8xKMai4VDtsXBt0je3lSIH1TWPZQOD3Tj8wQpA2NrIGNwnDmO0cNUif1QgeZEpr8O8R4nwxiDK/DJnMLTdzL2D/AJr0rBf5SnGVFPDDiFNTzUod58wJeB7FeQJa+6+R3D2f7b3PPxOq1RllVX73p1dVu0xWGVQ+w+Evtg4I4thdLJiDaGZjspbUODc31K7eJnPibPSSslifq17HZmlfAbm32K7vgX7YOJuBqTwNLL4mnbYRRSu8rR2F1/Ou8f4c568Z2dtz5Tf8M1Hl9ff+z6nT92iZrdFfN9hnN6p4QT3GiTXRM0hnPsx2y8V4L/lJxY5XDDuJcMZTcywjkh2J9wdl61h+NcN4yA7DcTppSdmB4zfov593DtHWdr3eD1WExl6/Hy+seT6urqNe/HlhPkOuwDB8QGeuw4Nf+aM5SuZxLgKdrTNhdW14/I/yn+9dg5kjdpCfmmuwjMWuD/zAr5zpyefsxfizhZ4hqIZTEz4ZwXM/VbWGfaJRTsDK2GSB/cPzMP8AkuqdG+RhZIyKojdu12t1h4hwdw/WZvw5KJ5/INAorcosUpqyITUlXFKOuV2qWIU1Fi9K6ixSjhqqeT1xyt0K89reBMfw3+kUMrZmjZ0Zsf70FNxjxBhNoMQhdN/44sf1VxznCYyxmphqYifKWtiP2K/Z/XO5lNgsVI4/kbcLx7jT+THxVT1zqrhaWnrKZ3mIklDC09gOy9qw37QMNq3hldG6jPfNmYukhqZKiET0FXHMx2xadF9/tftL3Lte2dmrZM3FVlcx/V493R6d8VlH2fEPEXBPE/C8scGLYfIHSNLhkaXAAd7bLCDxaxFl97TudVRvhxfC454Xgh2ZgOi4DEfsJ+zfEHPkosOjo5X6nMRZfue3f4k4xr49fqmcvjj7/wBJny/d8zd2ebvVl5fN8jOLCdHJ8pXqnGH8nLjnB6+R+B0jK+jd5mvZIGuF+mX2XnmOcP4xwxO2lxekMMjhmFxfT5r+gdB3zt/ceMdPtiZyi6vz+3q+Vs6bbpvnj6M++l1HbWyl5jCNd0I1OVfWcQ8vMLsTWOoA+SMEx6dExNytIiubhhbb3RI0OS5WVMXE/EkboxGEJ091pogbFK+t7JaHZMcwFkEwcCOyjeHg+YaIQXIszgQgb+9Nr1RnXXqgQE1MDb3Tgtt5k3lt7ozL3vC8NwzAqRtFh8TYYm7Dr+qsOrYWf1gWBNK8nK95shbcm2Ylfgcspym59X5HZsz25TnnNzPvbTsUhYbb/JRnFT0Ys8x2sjDVzZSSYhUEk3sqj6qofJl5hsjmNjlUTBqCtCw0E9SpANLJQjS5UhsOqjQcumyfKEWoKV2keUKBk4AumvZOHMBvmugIBIixSdURxi91UkrrnQ3RmFjP8KWZo3KzWyVErjZpHzUclNPmvLNZvzVbaD6uJhtnAVY10IcSXLFxjF6DBoGySZpcxtYFZB44onRllNhMzn20LrHVerV0e/djy143Dvr6bbtjlhFw6yStdNpG4WUU87o4HTVFU2KJouSei89fXYtVyvqKmbw7XC5yktUMUhcXGaKpqgdjmcG3X0seyT5cs/2e7HtM+V5Ori4g8TM5tHRSTRg2zg6KKo4xqaKUxjCbu/MXrMoq3GJdBSNiB9Iy2ACuOoqis/Er525+zWr6Eds6b+X95e2Og0fBHUcbYnNC5joWQ32dvZc3LjeJsmLaNoLnG4dkuSugZw5hcbw9z53W6GTRXYIaKnZy4II2j5L06em1dPExrj1d9WjXpisIcoKTibEnh8jXR33c86BX6Lhyop5DJU1hqJHdOy3y/MPKhyvOy78Ih2pV8JL7KWGB7PUnmmfDa0D5HHZrRuqeI4pjmH0xq2YFK2MfHJoFeI0jEopK3DqNmerroI/9u64WsxzGcVeTNVujafhZoFUbRhpD5Hl3zV8Nl2M3GGFscWQCWYt7Nssuq4vxeZ16SCKJnyusnLY3CNvlCvFIkpq3GKsnxFfJl/KComwBvT6qQi6QDvhWlg4jaG3sk7dGA70pi1ROJaOGUbpml17FhCcN0Dwnud1ViCbukC7MTZK9zbKU7W6o0WRycRgi+ZNzAHWL/onNt9wjJn6+VoCLMUBIv5dfZPm1Ro6YPTE3ThiBPNyNUxOUJ2gflSuEA2JIdnIt0CPcpG1tEzdkDnKDYJrfE5J2qVrAnUoF5uiJu6ItOTPcAXTIAGfN5ntsia0yE21ASIFkrC2jjcnZAso6pOOU7InZgcrQmRLEC7cIXOcRnbclMXOtluLdULCfT/BCia8n2T+U3voE/mBy2RW0RqwWA0AST5QllCIYm4sESbKE/wDegVnE5uiMDXMhbJlv5boQTbayAnDLvsna1p6oTzHG77fROTkGrf0QORr6kKcAggFts2qRtdA+UJdfomym9uqWYoGT9PqhcbBODpdA99gkARe6cjYgXskD5c2uvQoGBDk9vypmhuth7p76FA5J07ocpTkm+hSI13ugWYIb62ReXNZNdt7dUCaLDr9Uh6QlbS5KZA/XRFv6UOl7dExBH1QEQS2++tkx8n0SDyRYi1imQPd1vLumF76pE6+6dGQnMPSnFw3XdK42O6bMEaOO6Y6lPr7JvKN0DpswTB5NtEkBIfNq1JO3ZWkuyuLHMkNTlCYgEi4uEjlvoiiTO2TAW8qJD1MevyUccVvW+5UtvynVBfKM2pQFpmF9LpmHX/BNv2v7p72Nx10CBybuuEj6i86XTBhABI0RIBOoI6JABlmNbokNTp9U97aZUDZLHNmKTtLe+ycnRBIBoQbE6aoHAs26TcwGu6VvLl+qQ1tc3BQIhpsmysvnJsU9iHeycEONnICFr2Iuh+Ita7ZOLNdmc6wS1Du3ugcaXzeZA8eXQkXSzEOF9dUHKOYljib7IJAbWa7foe6E2BzZkor5de6eR7GNJdoOqBgOYw36pNaynY95Ol7omOu0PY4Frx0SOvayCIywyloZMLv6IwDENCgbFHG6zGNZdHG7mSvhykNZoboHMnMzNZrbdFf5XUbaWGOUTMvcHXXdGW+YKJJ2DS3VPmuLApmpPY3KQw2PQoSFzTfMDZSakpgXMe19xcb+6lLxISXAA+yyqK9vKg5TA90jWDM5GNymLsq0BfI1rTm7bd08RvGDtdItvqisNTpb2QEDdMdT7JN3SO6yHJ82+iY5T6UI/eHRLdtsq0CAYDn+JI7piTkvbVLXogdrrOBte3RKRzLukkbbrp0QuLhbypZbjUXQO0tIzAghFGfNsPqooo2QjK1lrqWLLfXRA8mVpy6363QWy6dTqiEjXAjsULs5c1zRfJv8kDnMR59k7S2190xOhd03+iF5s3Mxl9FQROluiX9yASZhdODogI/w6Ib62TnZNa90Dl3mDtkidUxtsU3QWQF0HcpgTc7oAHNJ85IT3ta6gmT9fqowNb3Tu8pHW/ZSAyWhPmTE2F0zPM0Pb1VBH906JXaN0r5joLdEjqLKhgeo2chdG2S2dt7IvYJje3lUCAy6N0RJgdNkr66qhwdPdOTsgDtTrokTceVRlLoWi24THYh3VMN07nWAdq4oFYWyphfonJv5kIzAosCOW99SOqYAOcDG42smdY3F9UgDZEDqbhJjSGljjfXQow23mcmc4A3tof4I0TPTldqbWQ2aH6b9lITGLEvsQdNEx/OgW3TVImwzOTg3SGptmb9Sga7SjdHGW3JP0TP0ORzdQmPRAxGmWwKNjX20IHzQua4bpZu6MmJIfkcPqiaHE+pNzPNlP6pdb3Ro4OqkJtsFHdt8p0Tn1KAxdwPslZuXfVDfW4cnD8m4vdA7t0wNzYbpOLb3DbBMCPUECu65zNTpIM1kT1PYfEnGmxsiDhezwl5b6FFMDZM8vy+RI6DQhMx3dAILntvmukJH/Fqj09ki0DVl/qoyfzvAdYZTsU5a0ajRMBkFm6Am9kBPUos5CBunsTsohuiVSxtGYpFtjqg8xGjkWY5fM5QJjshDo3ajqCrNNiFdRVMVdR1k0U8JDmua4jVVruabtSBusZ68dkVlFkTMej2fA/5TGP4ZRQ0tbReNLW2eZDq497r1bgz7deCuLxBTVFQMNr5hfw8rg6x/tL5DygIWOljkD4nuY4bFpIK/F9w9gu1dVhPgROvL1uJmf2n3fSn0NXc92E/mm4ffQbT1DRJSVDZW9436J7v2fYjsQving3j3iHhDEhVUmK1RgksJo85ddvtfYr2TBf5UNI6aKjxvBJ3RPcAajMwZPcr+edz9hu59Ds46MfFxq7jy/aZ/u+vp7np2ReU8Ze4fhR6wTyxP7EJ6iKGuhMdfRR1DDucio4NxHwzxFTsqcMrWO5gzMD3BtwtJ8ckY/ANl+MyxnGZxyiph9CJvzhzFZ9n+E1LnOwuqdSuPwkZh/FYVRw1xRw/d9GyeRo3fC7T9AvQBcG88Z/tAIgZBpBO4jo12iQrhsM4/xKlAir4GysH+/wD3LoKLirh/EnnNI6CT8sgsP1V6swvB65rm4hhbRId3AarnMQ+zvnefB66N37klgUHaRVdtngt6OBBBWVxFwtw3xfTmlx3DYaqMEOJ2dce64J7eKeGJhndUw27DMxa+G/aA6J/9Mw8v/fhsCumjbs0Zxs1TMZR6THrDOWMZRUx5MjFf5N/AczJH4RA6ke4GzXPcQCvD+JPsF+0bA62VtFg8uIUrTds0LLtPsvrLDsfw3FWh9PWx5vyPOR4+i02z8v47L9P2z2y7p23Ocpz8SJ92czP287ebd2/RuiIqvo+A8Vw7EMFqfCYrRvgmsHFkg1sqrTG8ZsoX3FxZ9n/B3HUAh4gwmOZwdmEsfkf+q89xL+TDwkYn/c1TPG4tIY2eZ1gbaWK/fdD/AIi9Fnpx/GYZRn76i4/rb5G3tGyMp8OYp8u6tKIONl2PEn2P8f8AD2IS0jsEqKqJhsJ4W3Y75Lma/BsXwdjJMSoJadshs0vFrlfttHcuj6rj4O3GeXpETF/b1fNy07MP4sZVvMUFnKTMwjypiHsGoXuczae6BzTfTbdEHOvmTFzrrTQBoLpFxPmTudbVPkIaH9HIlAubpXPdJIWRkj5xonA0te6IPOxYLICxzXXYdEV7HIxgeXkK1GGus4W2QFrCbFTx2DR5gAv59b8eAizsqZ5yGyYzwuebOvZRyuzOu1LbA5lypCNQewUPNcDlfoVFK6SR9gUsXTUMY3NfZPHVNcMxWc1kuudSx2abvksERfNY0IDWlwystf5KnJV0jb7n6rPr+JKLDY+Y+IlpNrDUrWGGWeXHGLlrHGc5rGPNr5ql3W6ZsbneaSa3zK5scUzVJLIKaZptoeXZZMtLxFXSunqK10DXbNY7QL36O2b90zExx+r16ug27J84r6u7M9CNH1Tf1WRi/FGFYXUNp4g+d7hcZOi56mw4xP5k9VNM7952ismGAyCR7GnLpqvbp7NWV7ZuPk9ertlZXsm4XP52VFQclNTyxdjZZdVJjWINHiqvllp0Eel1bMsfwhQPxGhZ+0njDvyg3K+jp7fo0Tyxjz+b3aui06puIQx0A8rah75A3o43VkRQM0DAnbLHO0PjvY7XScY2ep8f6r2RMYxUQ9cYxHlBzTxnXIFKCQPg+qzKhuOVd24cBk7jdZs9FiVKOfU5jKzubqc26dNd7h5ACq8szo/XJDH83LDgpuIK+zY4ZrnaxsFrRcK1sEDqid8c0n7ztlnkcFqimpCw8+qjfL0DDoinxLDqTNzo48w6AXWRJhGM1UwFKyGmb/H9VboeFp5ZwK6rlmv1afJ9VWOJv50yHSiojIOgEa0KfEcXns6roYqSEAmRzxYgJpI8NweoGeZjWDdznkfosTG+N2vnccDHM8uQyOFrH5LsjR5uJY/PmpahmGULCAKiXyvdbso+IuKcHwvBpMCoKuXFKqTQyO1A91wlfNiuMPZ964jLKxhuGDQJ4YIYGZIGZW9royKBuRhzjzFGLEatTagotX2y6BaA7DKllv8AEU7WOF+Zb2RCwNkaMAMqdugunGY/RIbIGs73SylOZLiyEu07oCcdrBOSBv2QiMu3KFsTCSXOKCQzeXIGAIbuG6G2uie2u2qMnDnE7JdfMU40CYkfE5Gi8vZEywBOW90BsNXbJHU7/RATn3SzWQ2B3RbhA+v0QluvqTl5v5WXTC7hmcgduyRciylCQD6kD5bC9k5HwpXcRZx+SbNrl6oELi+icEAdym+EpgQRcFA4ObTKmI1zFM3ltByAnuhdK9zjG3QIJjIA0/mTG+9teyTdN7IiL+ZQA1r7Xtujy2I9ki55N/ZIOUSzbhJLNlQh19RoVpRu3TWuUwc5zjcWREtAugYljWDSxSBdYe6bQgHQg7J7EboBDz0ZdHe6YiwF+qYblAQFu6fMnjeYyXDqLIESh5XHzPO6YssUrWAN/omJvshR7e6WXshRh2TXLfRCgMe0nunLtcttE75GWGcW7aIg1pCNWjRHZOTnN3FAiCbtbqmdulqCludECu38yRGxQsBtYoumbogZPfXMmS16IC8pPUJraXukNb3/AFSt1QMMwOtkkw9WVOgSXzT+U/RNubKhsovfqk3ZOSAbJEXQIn2TXtb3SJ0SFiEDpnelMPURskdLjdQN5dXZuiVjokQ8GwcLIm7oFpdIkN8xTXa35pzr9FQ0btMw/RI6lP5iTdIWt5dkDtPVLbYJiLJhpdyAjsgLjpbRM8vLdP1TjK4adFGTEEnyhFo68fW3RNtr2TuGt0DWta5JKcEJAGw18xSLTfS31VpqzNI8xale4ul1v3SN7GwueyBgLpDS4Fid7JM3udClcZs+twrAZw8ujAkx4uGZT3SBeT/aKIXvlKgf4vUkA7XW+qQO7XC2qVst0AvsNbG3snMuwyHU2unOg1uEBzX8uiB3t1BSc0vaWBE6xtlFtAoJZ44jlzm/YIJY4xGCG3PzREBzTdDC4Swl3uhBnEnlsWdQUA86CkiAe+zR7KZrmOYHg6EXCBkcckYYWhw90ZF2ho2HRAtPdC51pMvRGGkkN/Mo2mOW7hc2PZBId/YJDcnfsgmibOwN5jm69EoIuS3JnLtdygkREaXy6IXEi1hclOSbWWbCylO42NvZCx9iCdgicAdRsgYb3StchJuh9kxOllVgnF0T7ZfZN1B6JwO7Ubn5x/BEAXNcARb6JhY3bmsel+qTWsAtl+SYjztf2QHlsEJJAva6e90ZY4AX7XQB0+ibnRHYj9U5zfCgMMb35izXuglNiARsmbumBDW+wSzAoHbunOwQPLxbIL666orfqgYadBqiZI6M3CHqf4JIHkdmLemiEOGbI13REN0LwR5gLqh7BWatlGIYn0hNyPOD0KrJm7IHBAIuU50Ka5HS46pXBFwgRF9Ug7cJgTZI3tpuOqgYgH1NTXI6XuisSA7NfRDr7KhjcH27oh5fMmGa+VORZATSzN5dUP0sEgxoNxuUjcOtY2UBAWQ9fqnvY90jr6RZUOkhaCfKUjcHK3dA7d0Mhd6Wbo9gmL7nzWSygeZos5tikPLZ3RGQLJgACW9OijJOcwjRxRAZhvZDyyWl4FwOqTbh2qAk4y9UgQdEyBO0ck7NYWTEuLtUQOU3O3VVo4JLbtGiG19Cj0aC1hNkCga+7CE7Bu6/0T5L+cdEzo7gO7FA4v2SOR3q/RJpvuNwhLeiB7ba3T2vuhGlk97oDOqE2Ojgmacxswgnsns74tCgbJ7J/wDZSuLIfN2QFLYZWnqE/NzADLt1Q77pwAgIujLg5jbHqhJJTZRfMn8vdBK1pLS4kadEg2Nurnn5IbFOBc6usoCzRnQXuheAL2O6ctaLHdMXD4UDkuPrQ5QDdOSUswQCTYpxqLhIkAeZJu6IN1nDexaox+8ncRYdD3CAFxURIShtonYGpOuEZCBYa7Jy0nVrkOZ9rHVEwmxRo2YgWTg5t0JNykXuHqKtNJNAEDcwN0Uby70aHpdETJctNkDPkBQ5rtA7JZRfVqWQt83RRknaIyP0Qk6ofxhv6eiULzcaxeCFscGJVDBGLMAeQAF6BwP9vHFnDHhaCrrH1FBF6w8ZnH6rzG7iiIsvmdd2Xoe4651dRqiYn5VN/G/i7auo26Z5YS+r+E/5RHCHEE8lJicMmGGNodzZHNyH5L0ymlpK+mZWYfUsnhkFw5jtCF8B2PZdVwv9o/FPCc/Po8RnlY1mRkMkvkA+S/nvd/8ADnGp2dsz/wDxy/4n+76/T93n03R+sPtBzahnlI0Ub6SB1pGfgy/mavFeGv5TWHiKBnE0Er3vAEghiJs4nv2XsWDcT8N8Rx5sLxCB77AmMyAOHzuv5t13Z+t7bNdVrnH185jymvhL7Grfr3fwTa0Zq9nkmjbVRDqN1m1XDnDGLHmvp3QSHq27Strl9AopmsBHPp7j8zdwvnw6uFxb7PcUpfxqCeOpj+uf9VQpsb4iwKUMM88bhuyUaL0djsjhyJz7NcUVQ2GtiMGI0scrD3F1mxy9D9oNA9whraSSlf8Anabt/uXUUlZDVxCekqBUMPVnRc1iH2eYbVuJwurNK4/C7Uf3rnKrhzifh2YyRB2Q7yQO0/51W4ll6kJm2s8EA7tOxXP8UcBcI8YUEtBi2FQ/iAgSRiz2+4XMYdx7iVK+2IQx1bO7jlI/zXUUnFWC1bW8vEmxvPwSHULWrdnozjZrmso84mGcsYyip9Hj+O/yXaKmZJJw9WyzgtPLEj7WPS+q8XxvgXjTAauSjrMCrWmJ2XmCIlh+Tl9uR1Dx5v2jPzN1CtiWKqYY5mCVh3a7Zfs+1e3nc+gnLx58WJ/mn0/V4t3bNO2uP5fo+AKuixGgy+PpZYM3pzttdQjMQvtDjL7IOEeM6LwstKaOQCzJotSz5BeT45/JbrcOoZpcFxaWslDfw2Ftrn3K/ddr/wAQu39Trj8Z/p53VVMx9b/u+Zu7Vtwn/T84eByM0ukwkDfQ7hX6/AOIcLqpaKvwesjkjcWk8l1j8jZVHxSQAMqaeSInUB7C2/6r93hv1banDKJv5vmzjMesAs63l6pumUhEGx7MefkUix3RdWQPDyPKmDi3Q7KQfvIH90HsnMUUk8h0zWSSX89fjTwUFTM3mRjTulPA+lZnqKsRt9kkkaFR1eBNivJVGQ9yFz9Xxnw3BUvbTSkOabEpJL6Pb+lw6qZjP3Pb0fT4dRMxl7lX+e9BUkx0cjpHgaAN3VGSXiOskMslS6midswOSSX3un7do1TM1f1831tPRaddzV/Vo4bhOJ10h5FZM8xi58ysYhQGkqR94MzSW0zHokks4Zxh1ng44xEV8PNMcox6nw8cYiK+CvzvyAAeyYyPPxJJL6D2mdmKHKSkkinbFnaQXiMdXHYLHmxzhzDKpzIWBzx6pLXuUkkXFK/jHh2X11Dh/skIG8T8KXu98jv9kpJLk9C+zjfhSlZkhmnt2bGVBLx5w6P2dNVSn2bZJJdXNXf9oWFj9hhVTIe2ayAfaO0izcDlHzkSSWqS0LPtDqeYHnBoyO1yoMR4yxjEmcmBvg4juGXvb5pJJTNMTkgm7y957vcSpNmpJLbmHzH4SkBJYXZZJJEFbQpwSGpJIFcEpHe6SSKTJAL6Ap7knoG90kkCBFkiAPMkkjRXdbKDolfukkgfzZTlCZt3Nv0SSQMbhP0u46pJIFf3Sbo4u3SSQFluiylJJAOpI7IspSSQIN/eT6eySSBiOqawI6pJIH00a76oHepJJEg2UlpDU7RYJJIqSw6i6K9gR16FJJQMS7Kha03N3JJKMjayw3/VIZbXSSVagxGYIQ158vRJJUF5QAG9EgdSkkgVrm5RWuT2SSQD5QmSSQJP5QkkgcmybT3SSQKw0d+VLr9UkkDFosXBxStdoPdJJAWwTaEEJJIGAsN0miwyt2SSQFdhtfRAHF4zBqSSBA6AjZyQI2SSQPe17dUkkkDXACfcJJKhtgAlfS6SSBst0V9bJJJIbYpr6X90kkT3haSbl2yJJJRQ+QkX9Q2TtLdUkkDh1yEXVJJGTOFwhI0yW3SSRo59kGW3m690kkIPe42TscbHN0SSQO4XF+t01ma5ndEkkZJuVPn/ACpJKtAzuDhbohjfNK8kx2G490kkE4sAb2QZjfN0SSUCsT+t0n6m3tukkqHeH9fokM2UZiEkkDO2UL6Zjnl7idUkkn1BgcpjTkP0R9fokkgjgljdGeQ4lrDZGZPKbNv/AHpJIK5nke4R2Ou6njDmsALdkklAWUIZJoYtJH2J290klZhIESAQ4NKV3FJJRT2PwtQsgDRnkec3ZJJBINki0scM3zSSQFufZCdEkkD6kpH6pJIkI2ts/O3UqQEMy3u699CkkhJ3Oz65bICbC97FJJFMGeWxcSE/pakkgQ51hntbon1tukkgdu6WV5Jt0FykkgTmPaRnBFx1CfPYJJII7kvtlSFxmz99EkkD5gTa+qWYXt1SSQP1+iZub4nXSSVDa6Zdihu+5FvMEkkkOXae6bmXsHC/+CSSBA+YNz9LAlPfbN+qSSAjoUr7kJJKAhmyklBfX3SSVkPc7NT2u3zNsUklAibITmH1SSVDG4PX5pX01dqkkgch1szdkzTrlc6ySSgczNaMuS5SEjXgEbJJIE7N0R6lJJABBPlBIPsiBddJJULXonGoJSSUAkPJ017JrPb6wQeySSB8o3Gh9kbiCBnHm790kkCLRfM06IdL26pJIE54v5tE+gF0kkD7hNsR5tEkkDvNvQ66Rc8fCkkpAcFxKdu6SSJJ9whSSVU3LY4WebFMMjNL3SSQSZ4eqAuGazEklAWV1g7MhczOb5iCkkoCAuLdeqa2UpJKwH6py24sUklAFrGzUYzfEkkrKT6ETZAc1kkkWBNyu9bbIi4DTNokkgFsgB7ocwSSUZEHIwD0SSVaMdrK5hOOYxgb3zYRXSUz36l7NEklx3adfUYTr24xljPumLhMcpwm8ZqXrP2d/wAoTHMLgdQcVV7qtrLuZNKLvd7XXuuBfaNgGNQRzyxPgEzQ5rnEWISSX8c9vOx9H23LHqemx4znPnHu9PdHufou19Ts3XhnN06Hw9NUN5tM+N47tddCYZ4/h0SSX84l9cBI67o2zzDQNjkb2SSUFGuwPBcUZaroY43fmabELl8Q+z2ra0zYbWtnb+UtAP8Aekkgx4qribhqUgmqhaNw4fhldBRfaDDJpX0AafzxnX+5JJWB0uHY7huItHhMQzvP9W7daQqLaPY9nzSSVA1MFDWC1TBBNfcltyvMvtF+wXBOOqmnq6XE34XNAwtAjAcDc32SSXv6Hr+o7dujqOmy45x7/r5e9x26sN2PDOLh5hxH/Jl4hwiglqsMxB2JTNbdkbGgFx+a8YqYqqhnfS1tPJDJG4tcHNtqNCkkv697Dd963u8bsOsy5caqaiJ87+H0fn+59Nr6fjOuKsHNjPxapCx6pJL+hvlv/9k=" } ] } diff --git a/samples/js/node/notebooks/hello-world.nnb b/samples/js/node/notebooks/hello-world.nnb index dbc46d196345ab..83d4ca8bec29f5 100644 --- a/samples/js/node/notebooks/hello-world.nnb +++ b/samples/js/node/notebooks/hello-world.nnb @@ -17,62 +17,10 @@ { "language": "javascript", "source": [ - "const { cv } = require('opencv-wasm');\nconst { display } = require('node-kernel');\nconst { getImageData, displayImage, downloadFile } = require('../helpers.js');\n\nconst { addon: ov } = require('openvino-node');\n" + "const { display } = require('node-kernel');\nconst { addon: ov } = require('openvino-node');\n\nconst Image = require('../image.js');\nconst imagenetClassesMap = require('../../assets/datasets/imagenet_class_index.json');\n" ], "outputs": [] }, - { - "language": "markdown", - "source": [ - "## Download the Model" - ], - "outputs": [] - }, - { - "language": "javascript", - "source": [ - "const baseArtifactsDir = '../../assets/models';\n\nconst modelName = 'v3-small_224_1.0_float';\nconst modelXMLName = `${modelName}.xml`;\nconst modelBINName = `${modelName}.bin`;\n\nconst modelXMLPath = baseArtifactsDir + '/' + modelXMLName;\n\nconst baseURL = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/mobelinet-v3-tf/FP32/';\n\nawait downloadFile(baseURL + modelXMLName, modelXMLName, baseArtifactsDir);\nawait downloadFile(baseURL + modelBINName, modelBINName, baseArtifactsDir);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/models/v3-small_224_1.0_float.bin'", - "" - ] - } - ] - } - ] - }, - { - "language": "markdown", - "source": [ - "## Download an Image and Imagenet Classes" - ], - "outputs": [] - }, - { - "language": "javascript", - "source": [ - "const imgUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg';\nconst classesUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_class_index.json';\n\nawait downloadFile(imgUrl, 'coco.jpg', '../../assets/images');\nawait downloadFile(classesUrl, 'imagenet_class_index.json', '../../assets/datasets');\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/datasets/imagenet_class_index.json'", - "" - ] - } - ] - } - ] - }, { "language": "markdown", "source": [ @@ -83,7 +31,7 @@ { "language": "javascript", "source": [ - "const core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'CPU');\n\nconst outputLayer = compiledModel.outputs[0];\n" + "const modelXMLPath = '../../assets/models/v3-small_224_1.0_float.xml';\n\nconst core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'CPU');\n\nconst inputLayer = compiledModel.inputs[0];\nconst outputLayer = compiledModel.outputs[0];\n" ], "outputs": [] }, @@ -97,14 +45,14 @@ { "language": "javascript", "source": [ - "const imgData = await getImageData('../../assets/images/coco.jpg');\n\n// Use opencv-wasm to preprocess image.\nconst originalImage = cv.matFromImageData(imgData);\nconst image = new cv.Mat();\n// The MobileNet model expects images in RGB format.\ncv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB);\n// Resize to MobileNet image shape.\ncv.resize(image, image, new cv.Size(224, 224));\n\ndisplayImage(imgData, display);\n" + "const img = await Image.load('../../assets/images/coco.jpg');\n\nimg.display(display);\n" ], "outputs": [ { "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAJBAyADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDL3/ZJ2aB3LSOSsZTamCBjIzx3/SrCXdyB5LusO3lQHPB/Hn2qGW5C3UMU2AuwJtMeGjI5UKc/MD68VQumNzDvkYxtu24BDjP4dKzqzkpNIiK6mhDPBcqJJk3S7mXc52gdOSf61B5Tz/OIcSP9zewB68jrz681mwXG2Qx7My44YdR9K1InE21wrSK2QE2H5T1waxZoijcXUX2d7OTdvUnPHIOcj9f8KrNGlzbSKJy/lsN42MSv49P8mn3st5Des+cRyLvLbV2+jcfj+tVxuFx5vnB45OVeMFTu6EZ/xFCQEcIg3BjKyjb1ByCffnp71pwWd5E2/wCXKncIyR0zyffg/wCFUm2+SWknjZgxcAjDA9/zx0qa0uywZju+VcrgbtvsR6U3fdCLcV3mCa2mj+ff5iGfY4+U52844/2vSq7mKW3dpgGiYhMIuzDbuQuDg5Aqyb+OIsk8ayx4ZkB6qSP5H0qvbvHbJNGu4lOQ6qNqn0we+DRd2sNIgEYdJkSB3iVOS58x1X/OK0tKtLaO3llbcpwVeNwGwpx8278+3BqnFHcpJuttr/dcnPAzzwBj6H0q2jOlw3n24eZFd3wuwgHodw7dOOazkhlS8w0gIaORg5XbJkbx14/Lr2pzXsS2scc0b2zOo2BjkMoz6/h61XvUvBNCvklzKqvjBbb2xuI5/pjHaoIdTlkt5Z5EZndtxkV/kHzHI+jA4B68e9Uo6CvYks4Le40uH55NqfKIy5BXPVlPcZxx/hSJcXizpD5axRrz93G4/wB73NVtHR1uhcRsHigVsuSRu+9tyAfrVq/fzpNhkYrGWVSmThuuMDjNVy6gSLceW8zPIhcY2Bl+8fTI46U+0meGWbzAAqgNsDZG0jP8qijf7PYlJUaMHdvZlAYj5fmXsfvCkiktvOWNkkuYWRVXYpD7cE7t3Hf8BzTsFhVkNuoaNBK4yvDDHfaW46DufftU94rRuHCSwkhA4JDHkZIA7Dnv1qF5ZFLxs8s2Hwwkf5s9/mXGTj+LH5VMsJnhWaZ32sxjVzgu+OdvtnGOTx70lvoIjS7RJi00SMY3JXflS27+93OPfpVrRbq4V7jLQRJKcDEeeM/MOnTFZjwRC+WUvJgv/qnAc56KknIINX9FRoftKywyMqqGMMb7Sufry3XHr610Ln3T1M/dtqafnSuptPPDIiGKMkkcNjo3rgYrQjddVM0f2hlSNMCdJSMZ+8WX5eeNuTkjJ9awIyRYRRKvyj7/AO93ewyD0+vStK2unmtka3kJRXAEO1QrsCDu+bjPb6+1deFqa2k7mNaLtohZ7E211smtHnhkPBADDce5Py8dc/xDA61GNsFlLDdQmeNlKttPmFTgAMScHa3PKnjNX7q+luIpik9p50bsYopGdGYKMfOFJHMnTHGBms6ICS4k1GeGOe0dmZwy8Mh52xkc56g9VOR0rWpTjJ3WiFCbS1M6SD/S7+zmLtCieZGzdAGAC/NwflORyPWtDQorF7MabqEkcMTHfDdBfnjkHT5sEj/DNVZQ1xbR+WI/tM0QSZ7lcK+PlIVs/KTgEn17c1Ss55RaywylhPBLuTA+ZD0IP9329c151SLhJ2OqnK6Orle1tYmuZH8zaxDKko4Ydwc/LnGfxrjdTWNtRvri5EfzSMWG7eWU8qy568YHH1rZ0wub4N9riK3A3O2fmR+NoZccg9Pf8axfEE8Y1q6W4VZJSImAClcN5a84I4yK66tV1aKl2MYQUajSIZLKJdOkEErbkXzHMithvx9R+uc1s+Bo8xXc20lkaOHk9QwZgB6fNjn1NZkFxIdLltw4McTjyyE52k8r79QK7nQdKj0nSkg2qZZP3k7A7g7n+gHFaZbTbk5dicTJKFu5laneQ3ISKZykJXMZY+XPDN/tL2I7MOCGI5rQskv5NSmmkXyYwC0sZbIMpGchc4xg7i3qTxxVe4sL+G8ElkkUU7SM8cjHzUI5+Vgy4U7TkE9cFc5wavzyywyzyvFELK3+R0ZeZlwGLKDx8vOB/ENw9K7IQm5OVRHO2lFKJRkt43117efMsclrsRDJ9/BCsD6Z3YGP7rc1WWS0gu7m7ljacwoF8qNQ/lg5+bnhjjHXsQed1M1q1nMyOnlvDGJYvMVN3VR8u/oc5+XpypqPToZrmIC3i8gC42AW6hHtkVMHzM9dx28Z5+97VlKSnNxSTZpa0U2y+0Uc8AeDP7ybz2xFhNzKFIXp25IGPu5+tWNoJZjD9lhHlvtDsvBKlvut1XCsx2jr71pLYRwxW1mztvMTrK6SH5QducFueW9eT36VX1mCaayO/wAt54JI2Ucx+cozu2/XJ6HqKK+GbjzPddAhW1siBLN0ktJoZpLqwxIfLVl3FNpG0MePlPbjp9a0LS5tDfC3meWRQVMUfKjj+DDDDNzyAx+7/DVTRp7eK3t1gn8uRZty7MgFcfMGU459c4POeTWkbYus7o+6RlUeXIS0asAOQDyuevHPzVLwvPFKGnUPa2vzFeKARSSi5gSEl9zSiHaq9CCHz69c8Fs881XvtLjdLh496+bLEwRG2JG+QNwKjPPue/fir8sc7NA8aCUg7xBPMEIB+8pwMOFGOv61Oz7FTz2jiYqJEIwy4zgMrf8A6vSuhUVKLpy/4czlUs+ZFGz0/wAm4WXzpBdNFtc/eUnPLcjHHbn61j3rzRXCRzKJYstI4RSEXAKk/NjBXJORx82eRxXSBUlUuixzgqybwwIyccfLxz6daxdTsobi4jgmLwrcZj3yNkiQ4Cnb2HBrCth4QXNFao0pzk3qZHnypBLJDakyRvAkrbF2rwPmO77pbA7fdJNXba4ZLdLK9t1KwsI7cqkjCUgsQ0X3WyDjHfCgjpS2TW0dubxiR5k+4m3Q5Mco2MoJ++PLIbHB3IcfeqRb63a3EkjRRSRBfNUKcGTDLsUY5zsLA479snPJOlzQUoysb81pamMzqjvFuZzBI2wtty2N7Asw+UsT16ZFRvcNcWUapNIkytymGwMsF+Rs4+vb5vwqzfiGO/1CANGASGEoH/LMpyx/u7W7dsdqSApmS6KJP5fyOqqzCRSMA5+6TnkE8cduK53SblqXfS460Z72QIEjeQRCCJBAcoyg+WD2LdVB5wCpNSXdhLcX4htoHuJC6xlFl3MGxnb2CH9Oo9KtWcxtbBY7ZhH+6WVwmCVOwRltpIzllI9t1TWLw3ltOuGlWZW/1vICbvlDycZYEk4z17Y5pSjCUuXYLtK5RtpI7e8Q+ad53Qyyb2RW4wuMDtgY+XGaR5DDqbje8jvKoiJwB2Kx9OMALxnpuzioZ7WNLGJ2k83ZNJGYj8zvjLeauMdscAk5DGi3PKW6M0jSvzII+dxY7MYGemCRz3zxXOpOJdjotOvgtxJaxw+ZcGEFPMjKgyBWVtuPuntjGORjOKfZPda1bTpHMiZj5kMTIythi4bZyAv+z83TpVe0VWv2j2wFdgit5C/zMw25VeSMrt3e3PPWpraSaG6jltYmuZCrRPeGVVMK9cORwwz1HJxXRShNx5ua1/xMp2btYq/2gbXyN6O0Z+WdWViCnADr/eXGct7jI5pzyLq1m5s7RfNI8t5ZI9kcZX+HJGWPPAx8vfb0q8LeGZJbWe6lllZmdoBKyxBW5GE3fNg/389RwKsw28dtBHDEuI412qDycfXue5NezRoTkuWo7x8zjnUitY7lS005IEjaVmmmXnLnKRseuxei/Xr71P8AZ4vMkcxqWkRUbIyCFzgY/E1P6epOB70FSOtdsIxiuVdDByk9WMoxTsUmKsgbijFPx7UmKAG4pMVJikxQA3FJin4oxQMZijFPxRj2oAZijFOxRigBm2lxTsUYoAZRin4oxQAzFG2n45oxQA0ClxTsUuKAGY4pQMDPenbNzdM4qPzEYlVzJj7wQbv/AK360AOUZ5P4U8AnoM03963GFQep+Y/kOP1pDArD96Wl9nPH/fI4oATzo9xVW3t/cjG4/p0/GgmY/dRI/wDfO8/kOP1qfAChQMKOgHA/KgL+dAEH2dWGJXeb2c/L/wB8jAqYKAMAAD0FOwT0pcUrhqNxS4/KnBckDH3vyp0myXCRg56M2OFyR+fSsKtdQ0NoUnLVkZzx8uGIBx6UjbI0Z3ZUQAszucBQOSSewFSBMcAZPoBWcu3Wp8J+802GTAwflu5VPQesaMPozD0XnVSsieW7v0HW0bXsqahcKY4kBNpDIMFFI5mcf3mHQfwr7k05pN8YmKMySnZFCOGm7hR6L/Ex/oOX3Mi3G/zPns0OJQF3G4kJ+VB6jPX+8cDoDU8MT+Y1xcYM7jbgHIjX+4p7+7dz7AVSJYQQeXueRlkuJMeZIBgcdFX0Vew+pPJp7DKsP7zhf5VJimrz5fuS38/8adhbjiM5rlbK2e41HVE3qqf2jcHAzuHzY4/OusI61xsMkkGqarMscpU3t0N2eA2TtXHv15715+Y/wl6nXgviZ2fh8oZ3mblHvZpDhcYAlIUY+irSqMjJ6nmk0dBBIIsjakrqCfy/wqUCssr2l8i8d9kbg1Der+5RGco7sBGMcsfTkEfieKsY4rO1SBzavhgkbMqjy2BJP3sMDxgkY7/hXRmFV06EmjDCw5qiONvSplilcxywMeDGw2/d6qOo6Z+oNXIxa3ocxCNNysuXPJYcjgDjP+c1zyumyTcpDbtwIbIUZ5yPf9K2rHTmey8/ezkNteOKVcjHT6+ufyr59nooqwW6O8TusglDdSPkb071euWh2rbi32MvzLiJfmPcHHU1nXEZlcqoGUAVyj5Rj13D0+hqS0S7trxHkSUMuDlmIIHqKx66lE8I86Vg6QM3/PQjO4f7S98cVRkEyQeW8QQbsFXUhW+hzwenStaCaNzJC0zyIrsBnpSTKyIzpbiKaRlPnRyYTJH44yoI+tHNZg0YTooaO5Ulhuw4HzA9MfpWxZWlrPK/2a5Mlwo+SKEDavcHPTpn/wCvVGe0meWZbqASsuQLiH5kmz0+ZcAcHrtotru907YhhDKWVuVwz8DAzj+nrTk9NAL94IxcQukZDqDgKOVyOCPXkH9agvpgfMa43xW4DM/lJhd2xVXJ7dRg9+abdTSSN9nTISQAkvljhicK3TbyOnsPXFTNZTz6dGHYXQEnzxScbvTLewIAPakvMCppcn70XEatJ5YwREcMR357flV69Mz30F3YagBbqAr/AGhwpXuRjoevvjj2qFNNtbCZo8FlmIaLgllA69Tx+J9OaqJJ9kmJt3uI5c9JHViCO+OlDSeqAfb3kP2HyRKytnEYLfdG0g5P8+BnFZjyrPcrE8iok6Ln5AoI9fqdv51aSGO8tZUkeMSRRsUG/JDkHafbPpUJgMtnHc2AeMI6sVLAiQFRtOMZzu4x/KqjZBcmf7N/bE8puDBCcbXgGVVcDgLye+PpVqGW3iu2ZSBHA4CMjEHPJy3Zj/D07/SsswXRtIJkSVNpKRvjaN2c4z+PWtKNb7zhBcNGyNtjnDAyfUkfwnnk4IoGLerGsrhDAwTbJ5rYxl+mxRwTt5weMevSqt3EbOeKRUkiSRNyZPOM5Yq2cnknnrSQZubOU+cx+bbIHj3Nnsc/1XpgZqe1uVAgQyymEbgyZ3fJ3A7jHpVCZLbFHnafY0kJH3sZyx65zwPxrcMEMAWJrdJopArq6N86r6EdyDWBa3Qt7bEKDbJ2k7c9V/T/AOvU9zdzH9zMU3ldwkRAS+7rz15pxmo7ENXH3CSyTm5ls1DbhHG6pnj/AHux+hqO0vs3Uv2mWQfIMhfvsB0A4zn3qtFcsFaDCMkkZi2lN3fI/iGPr2re0qOWK2mv0kEzpjzWKHgd1B6A/wC0uewrsoU1Ule5lNuKK0VvNcFIYola5RmwCp2spBbZ7nGMfSpTZMLSaCWKRHdgYfKkO1nJ5iK4+U9+f9rrVmS1t4PNvYotzWzgi3mZmZl3lVKMpGS28ptxn5hxkcxyzXM9xOJWMFpKcSRkDdGy5Cq6jIDcDuefyrtVGNON7XMedyZnQTRxQTQXBIY4IZOeGHLcZ9MY65xSy3Vy7IwXFtId/lh2ZcnocY5b3qKO2iMqyQvMTJIXAKbdq/dYc8MwY89BjpWpdWyxfapf3c7xqHS5jZVU/KMY+bI+8R/wGuCUZJNPRG91uZNvbNHN5s0pNrG6hwsg/eMfugcZyc/yqbUYYbe4ju0PmxZ8iaOVmLI+CArKQD6c+1Vb3VI7mWe5KjDqPlPAI6Z4PDcfdx34qy8k2pWUctw2ZN0YM853fKoOAAPXPzZ+me1EZQ9k4vcPe5kylKHunNuyhWB3CVcqQcccj2yKoz291d36/bcvNKuGuHGWYL0J9+n6V0JWVZDEXZlK7SI85B45BHUdvxrPutmJFCuszN+93NuKnvx+VcsZtaGpJo+mnUdYghd/9GVDM5jbBVFUBFz7tt/WvQAm1QMk9sk5rhPC96tleTrPPFGHC7mJ2k4zwpIx6Z+grftprw6mLkTmaylOUliZTHIpwVBH3lIzx09693AzjGkrI4cQpSkXbjVre3uo4CkjMW2ykdIlI4YnoefT1/Csi9YahdRXj3bNpattVJU8pFkG4MrfxZ+VeW/vY4BFO1CCXWkQx2AZ0l4xcFI5odpKsu5eCCx4IAzk88VcnsblIZFknsrazMGJQhdRGBzwuDkbjnqCOnpTqVJ1L7cvcmMYwa7mLZu4t/tMkjm0jKo6RxlljPzAZCncB0Ib39RiuksHhurKGXKNLsAZ4xg5x14H8XB+tYerCCbSbrUWumu7SBkRCqne0bEYTPHK5AyeODnkVqaNbJZYSIsUZdsuZNx3hiN20fdByv5n0rLBwlCo7qxdZqUNGI7N/aZtnBmtZV3ADaJIyy8oVOC2R0GPT0qF76NDJJE7yWNuI/PxCzjDL94EnII/iHUccVsTWdq12l3NDGZ4htV5D93nPfgH3rmLy7hstTvvs98plHyyRou5ZOfuSLkYIzgOvI961r+5rJ9SKb5tEbkNhaLdrMkMc80sfMindujwM7ecFenA6ZBHU0huTZyGOVJJCQcMpG47cLgg455X61UtIpNSOZ3/ANHkj8xIoIwI13ZCt2YdSO/oasK5s9QgWaS5xNcNDAs6iUlRtG4OBlQVyBnnua1hUbgmlYmUVzau5otGpO1gG2nIz600xssgkQ4cDGOxGc49ue9R2M1xKsy3ds8DxvtXcwcOuOqsOo/CnR3trLfPZpMHuYwWaNeSoBGc/n+dbOatd6GSi72RXuZoIpEnHmb1yN4VhGgwSWYD72Qu0de/TrWPqs1tdwRzIkMkYKJm4jLn5mAzjO44UMcgdG+lartDcRm1sbaOVzcbSZImEaM2QZGG3DfNjJGT82eKxNW32kFq0yxSzmSXzwqFgqZ29FGdqsWx0z79vHxFS0m77nfShoilFdXNzILUxtHNauGkddzlQrHcCRnC8YHX9am0u6bStUeSMukY3hUAwxRlwnphST/IVpLLDaSzq8SvEgA2l8PIS7Zf0x83fuOgqrcNBqMpmuHaO2VGddrq7Z2jKemTtPTp05rks7JXuzbuZ64VrYI4t3W3xMI3yRzgqvcHaT17cVWVkMaTI8KSODAYU4DZHXaecYZT83ODgdBWpcyxw6hIiIAIHcqQApjV8PtAGRn5h2xgHFULyOXc800rTec27LH7m4gZYtjhie3T5T7VaqOEbMLXdyVIJpdOne1knF15rBURB+8BIBO0fdIOTuGeR64qe6lv7aC1YTrHB5jOCgWIpcLHtbIVeWHTce+e4NZUaMrSwyPJLDGpjVzk/Nkcr7Hbn16dasy3NtBbF1jjklilZoWjURo6ZIy2085O7+mRWVSUZdBpFq4ZrPbbPdH7HKUuUuABhTkoDt6E5XqPl9O5qGBo472x8m2SMxlf3e0ohBxiTjoQc5x2A+lQRxPe3Cwz3kSP8se+TLDhcjp/D7jjvVqSPFzaJdeetts8tXjYI27HyPs5+VeAPUL/AHq5ZS0syhZdQdNPmsPMzbFNiPl1DJubDbQck+i8ZzzW3ZWL29lBBDFCfLk3KWkK/wB3Em4A4zx8owevNU4dCjnRbaEXAmZUiuIpflDMwJCqp+b5scDAIO3k5qa4uJ7OWERX0bFjH5NuwbHzYAA3ehzkZx0zWlDEKF5WuKcObqbSyQtCri5DDbnB+8Dnkbev+fapAFXa7sNhTfxzx24rOuIpo7iGVESKJWXda+SS5JJVj8o+UDr+PPar2nN9oVmgUW9rjaWL7Q2MKRjv2wCBjFenQzKT92bOWeFW6JY1UooDO0m4KodAwYjPJ284Hc8frTGG5RII2VQFHPzFc8KDj8KatvItuzhfMlcNsjVdwPU7eCOxzn1p2QAnzEBgAvz9eP6810Uqi9s3F30M5x9zVDcUYp3WjFeqmcYzFGKdijHtQA3FJT8UYoAZijFP20YoGMx70YFOxRjmgQ3FGKdigCgYzFGKfigilcBmKMc044VcthV9ScUzeG+4rv7gYH5mmAuPWlxx7etNKyt1ZEH+yNx/M/4U2S3RoZNwLnY3Lnd/Ce3T9KluyuNK7FEiMTszIQcERjdj8en60HzCONif+PH+g/nS2m82y+aTvzyWYcnA/wAKl28bu1ZUqyqRTRdSm4NoiMSYw26T/fPH5dKkAwoXt+lKF5z0pWHy5I4wc9xWjkSkJijFEYb592CQ+APbGf609lIQsq7yoLbR/FURq3LdOw1RknjkHGKXADKM5YICSPf/APVSxj5flO7jOf72abGX+bru3sBv9uAB/ntUSnqilHQkGd7DbwP50jLxuHUdqApjjWI9QRvfGc5HB/HB/wA4p0rp5YGSD1HGD/kVzVMT7NeZtGipMhuZDHF8m5gflyi4/WnxuY2IXlmcMxzj34Hpj/OadcYaN3+6o79jt5/HpWaZ5nRLLTQTqEkYYzzD5YV6bj6tnIVf4mB7KayTXxTNHdaRJLtpNQuZrC2aSKNT/pVwB8y8Z8lSCPmIPzH+EH+8wpZ5ooLdYoykVpGmzcowuxeMD0Qeo6/dHWo44IYLdLcFY7KFXEkjycSBfmdmY4+XJyzfxHPNS2tub6RL6dD5AKvbRFdu7H3ZWX9VXtwf7uNotylYmSSRJa27EpNKmwKP3MJGDGCOWb/bbv8A3R8o71cp23mlArviklZHFK97saeFNIq/OP8AZT+f/wCqnP8AcPvxSqPnc+4H6VRIuMc1x1qZDrOoQuwCtqU7A/3gjOSD/wCO12Z6d64rTgT4g10TEBF1CXy2b+AlznntnjmvPzH+GvU7MF8TOw0pm+0mFlUr9okZm78EYX8f6VOKq6Y27WJoUQkLO53ejEqdo9eGBq5wuNxwCcZrDLGkptmmNTbikNKlhgEqfUVm31ilzI7tG0bcp5iszNgfx7e/JUeo7VdkvIVlIxuWPJf5NxHo2M8AH+LnrWTNJcNJuLzvHnc/mYXylYfcUDO7qGx19wBUY/FUp0+WOosNSnGd2ctNpiDf5DyTMpOQVALL0x0xnOOlVdPtEm1KPZHKSjKVKDO0D1BzkVWe6umXyQ7Bl3IzbskHOSfx/rSwXLwsrs5jLZy445z7dPwryrO1jrNn7fCl+9nNDGgAJ8zZw2Tzk56jpn9O9WbeGAKfnWQn5cvjkAYUH3xxWLbgXkZiZ083cxDE43exJrQVZNnkyKQ6BvKK7cluv155/wAKyaAvwqsTuDbrGrcruXaV9mOSGHHB7ZqNRDLPLLCgU8iRpOMY4wQM5H51GuqWyRqEiIDhciRsAY69Pfjr/Wq/n3F9O0SiGLCtsXaMR8DkZ9aVikTXO+OFJYXaK2KBnRORwfT+71OKx5Hlmlhtwx3x4VSdw3DPB/GtK0vonhBmh88KMlurIB3H58HvinSi2uZ0f57YIWVdh4LD+H27fTg007CJIbabe9q9qjSQKDuQBgucgjP1Bx349qZH5kEnks7uyPyD867cgdPTOPpzU8kt/BKsds0LK0WU25yyl1GBx69j781FPKWmDzzbJFLbwjhue/Pt3FTuOxFO8n2mRJAAHQ7gV3fMB02k8emR6VlX/wBnFw00OxIZVCtGM/Myn05+vpxXSl0liE3yohjDMdxYgjjpj/ODWZqiRTacZIbTfIzZ3NHkA4+Yj/a9M007OwmU7eezjt1mM8f21P8AV8HcuD8p54H19O1NQLaKsyW0iKeU8xRjccn+vtWYcoo+T5WG7Ljj8+9XjC09nLiHzIUTYzRs+VDHG73UZX3FaWAvJdQ6ja+VE3mEICBvxn1GPrxVma5/dFbuZ444YfkUkYYtjC4IOPT0rEjnayu1EDv5zcLJvyrL22/3hkEfhWja3O6JYpiJflzuBDtwuB16jJH5VEo2C5nCSd5Jg7IsLc5Khc4PHTvVaFpYJF8t2SU8j/JrUUz28jobmRY1XCny8snfAByOMnge9PsbYXczxSuWkZtyxSK2O5ySuSM9OMjn8K1gnJ2Qm+pnQBJnVQ6BmIVlf5STk9x0HephK8F2xk2jbwEYbh/9erWqIICIdJO+HazMdp+UZ4CsMc+3b8QKpWbqbdkmQxkAgAr8p9Dn9KqUeV2Yk7q5NbN5V3G6hTn+EDg54xz9a0rIXUGoxExie3QOcIW2oDwxHv8ALjp2rGKiNkJYrlcqX+uCf8a0LbUVumhh0rTXkusKvm7SCWbKk7jyu4nHbp0rfDNRnczqXcTVkubHVb+CX7SI/syNIzFUZ2lb5ApDAKuBuI68kHjrWvNa6YybbyRLsxlcrEu5o/m2qWKfw59R9c1zdrpt5ZatbNdRyi5aTdm3jjuZAc/fZdxGAcD5sZxjFPuJ7W/Hli8uZ0b5fPLOiqfTZFH7DGT7GvTVblXvo5pUrvRmldWyWM8sYQNEVWKUc7MA5+9zgnj5aoSXKJJcB1hKhVeHlVRQVwFUZyT8vbnOayTKbQPZpKPMGRKY43Y+uGbnHcEe1Xo44rmwAaQkYBZSSoJ655HOfcehrgryUpbWN4Kyte5Wl8ue0QzxvhsZUICQvPzY6cc81LaypHc27wySRxpIoeEpu577ckbs4zg/Sqpnw8ka2qCVV3LtQKWI9uh9eB06dafIpmSVDP5oT5sR5Kt3C468dPwFYP3Xoy9y/cS7GdoIk81+TIcIuGyMH+736Y6VgTRSx3W5AhOSu8DAz3H1q+48qYpHmEKeARs684x7UC6gS38tGV3GAVbtWUr8xS2K8Nw9spujD5qL8jFt3y4wT0/z0rtrCXTpFeSxtryKcx5YlCnpkbThePoB+dYmlTLaafMI0DhpFDbiSVyNvTPIxxV+OBYbNvOmd/MIgVAGXaETgE/xdOO2D9K9DD1nTj7nzOerDm3NuOdn1MxIwhiSIM6BBhyT68dMe/U1Nd2bXY2m4kjQdFj4zyp+b16MMdPm9RVKxSZjGEuSvmJvZhAG3EYAUsTnPf0wSOtbGDivXopThr1OKd4y0OVa2udC1CeRTM1nNuVWihV+GyxDE4EeCW+bgcdGqtcaUtrHZyaci3VsyhBJbtwTtP8ACp54BPXHtmuovNOF7cWsj3NzElu5cxwybBJ0+9+X6motXvYobeVJZHikMTSxz7cqki8p3zu3DhQOxrCphYOLvp+hrGs7qxzeqteRt5LXMjN5gllt5ZBHIUZjglcsG2nuM/e6cVrRXGleJLPyZreEuvz+Q5wA2Nu5XXt2zwemRUN5qcd8Xt7iM2dnOjIZVnUKJMKY5AR82MgjPboe+NPTdMkspppJgk9xIuZbhI1T5iSWGB/CT831Y9qywylJ76bO5VW0V5kOkmGTS7dLOOWFrVfLe2mb95Ge6v7n7wPQ5z9JY7901J7ORU8yTDwJ5mwumBlTxjcMNznt0xV8QxiUyiNBKw2s4UbiPQnvTsD07YrucLqydjn5le5Wt547qAyR5GGZHRuqMOqmqUVnqEN3dXEN3CHnuBKsZTKlVGFDdOcHB+la59T196bG22SOXauxWzyASvGMHsayqwU1yy1sXBuLuupzdjYzW1tZ3P2+d4plDG2UD53IO7LegAbPHBHHWq13cQyCK3t2nnnuJGRxA5R3TqrBCOOm7PfaewrpRLFGjFbaNrlZGacRZbDNjdjqF5xleM5NYV55F1fWdtZmWRoWlj8uJvLbyCEbbhh6koM8c1x1cPBxbireR0wqSvZlfUZmjgktk2uqyhopDF+8bc20Alfv/KBn5e2cd6rrteyhtLRy9wqqqmNeTzn5cnBXGOQcdc1oXcepQaWXimZbtIWnAjbLsG4LL91Y8jKhRnJOB0NULyCFSkazxNd4CSBQ0gd/kXHX5dvzjcemK5J0EleJsp9GS2if2ZK1veBvME0U5ZGOxvlAYdNxCgrhgO+cd6yL2QzSJaCaGFQzOzCJ2JLMMBlPfnr9M1vpDc29l57SWrbpAXKT7mQj/bfd2J5XHU/NVPbHDBGY7ycI+6Uq6fvAm7KkZbJDAFcLn7rVyVrJKN9jSC6mB9glEUr/AL0LtUSpGhCnOWC9MirdpBcajp9xJI5ZYy4iJBHmEJu2sV46/llvUCrF1dbROLaULbecAskSZR92GxuJySf4vp24q4lxcw6dOjwho5m4DIyeavBLYGAc7ef1znNcybsaWIo7SyvLJGtoW82NldfKYZbd8pzu4Gwg57d/UU+3NrPa/ZEMMKTfKT5i+XuGcDI/iJHOB+GTSQyG1mc4k814RE3LMZecI6nbtb0z6frPNGdT07MsqvcbgxjiDK7OwO8bRgKTjv79c1bacRW1B5V1TEtzerqEtx+7WK7+WR2QgcbSflyvQgE1bjv4YLyLc4RLUlzKqiRirDau/LYK+nfmssadcXVq11Jc+RHMhGZ0Zd33dpKr1+6G68da0ILS7D2zySRiOERLJHa/vC7M2AjdBzzk9u/WspRSYzSXzZoTBEnlOEbcqOSyKkhG75e23k5Geamhf999mWPbOVDRjg7d2QZOvTnB9/asqW83fu7wPZ2yr53/AE0deNq72A+bA9c/U1M89zNdEQvuDKBMkjiN1xkqAxX3/Q9+KTqPmUkrDtZGnO1ssBknZTFbyDeJMKsZ7nJ6DntxzipGl8yGV/OcYXcp27jlTliFHOSvNVWnlguIpIdw3/L5KqCMMvyruOcqPUH07062uPMjnzsYojOe/mDk4I7dOnP481rTrTjpYmUE1cuyW8gJIjkwcYO316e9Q7MHDcHqO9RRkpPLHKsiCTEuS7Om4EgjPX39Bn1qYLGhlVt8Pl4xxtUfLx+eenuK92hjpOK9p1OCeHu3yjStGKldCp56+/FMxXpxlzK5yNWdhhFGKfijHpTuIZiin4o25NAXGYoxSb0zgHcfRRupfnPRAv8AvnJ/If40AGKaxVCNzBSex6/l1pTHn7zsfYfKP0/xpyoqDCKq/QYpgR5Y/dQ/Vvl/+vRtc9XwPRBj9TUuKMUrgRCNFOQoz/ePJ/OnGnYoxQAgFI25Q5DFdqN8oXnd25+opWjdthUMQrAkKeSPb1NLIjhQVXlTkjnjA4+mGxXLUqK7TN4R0TRXtgy2sbOSrKN2X53E455+tTL93Cr8g67s5+v1qGxKC0QEnIO9uOoC/pmp2ieGHykfhUCoTjgY/p09a4qNS0UdU43buOKnGSOT/KmY2li5GzB+uPT8elPGP4EPOFLHvx6/5/WkcyZk+YfI2Ww2duORx68nn6Vv9Y0StqZ+y1uRx7khQv8ANKwZuOQOcce3AH4VMinGQzH/AMdVvSkijC/wqrAlc564qSPILnnHUZHt39qekYXuLeVgjkTZuQYGc5xz0pNyABt/3chVKnHf2/2j/nNQyNvjlO4ltjDA6DcB/IGmecNpjVZyz7d4VM7QBjPH0/GuBybna51JJK9iwQss7NKqeZndvPzfmR/nijYhlR0Q7FORx0yP8BTdwTAZME88MW3cdvf3rO1m5js9PYb/AN9MwhSCN+SxzkZ6Zx+XXsa2q35bvYiFrhdSDyo7FEinnkBRY/MwJHDHcTjpGvBZv+Ajk1NaWcggYRCWRZGDzyu4Rrh+mcDov8Ix0A2rxkmPS4VWN7iS5aV5FCSGLbt2LwsSY/5ZjnnPznJxzT7qX+0Haxt3eO1UeVdPFhC3byU9B2YjoPlHOcYttxLsrkaxjUJ3mldJLAPtBBLJcsrdAD/yyRh/wJl/urzqDBDeYC75O9ie/wBM1FEim3aIR/ZFGI0SPaflUfKFx+OBxTn2orc5RABgAnfk+n+RnNdMZRtoZNO5KrMwVQ25/frnHX6VIvIJ698imyDB8kbGb+Mbd2e4JJ6/4/SpVUhADj6jvXRSvfQxqcvUYeqj1aljGVz6sT+tKeGHoATTkUiNc9doruRyMQj5T9K4uHyZdU123eUASX1wNmwtzlmGfThTXbEZU1xUEnleIdYi/dq0uouy4+/95Rn6df5V5+Y/w16nZgvjZ0uiOWmS4CbjMTOT0zuKt/6Dtq/cQmVWRdq5PLHORz1GKpaY226Tgf6oOfb5RWow5I7c8Vy4CCnGcZeRrjJuLjJHO6xAbebEDRQkoUjcqCdzEc5P8Wf4eeD71ijeY0shKZCsTtKxiP7xY12+uFwCoy3Jx9K6HWYmjhOyPZBJhZnjj3bVHdh3wM4Fc1NZ6n9uRYftEtvEI5Qypt2n5vupngjP4dfSuDGUHRk10NqNTnVyhJdWqQ7TaxpdKpJIHyyL128/dYfwnPseDVRrje7u6YbaHWSJQpHHYfkSO3Na7QQpLFMbaMjaoHG713Ar+NZmpWyxT+VZqQ0L7hHncVcd1JOQcZBHIrFz5maWSIIpJUEUqyAT5xwMA/pUuoLMtvmFQ0ZO8M5G7cB03Z96qLI73BWONFDJuRQDjPT8Me9aEMcxtJkcYkjU742xhu/5VMhWKunmZZEe5jZ4gjZQxMdx/u5HT/69SYkW4hNmXRnd1ITghGHf1P8AhU1pJDLcRea5j5yFXKE8dsden/66S7uFuLxTujVCvlpIo2cd85zlvei+oya2v4LRmjjhk3gAYAxkgcf7opJZyb5VkjKQIPNjO0naT/e7nnGfT0qC1ntNP1JG3uwUMFk8vdtI/n7j3p+panYz7fKti0z58w524Y+2Dz9DS6gXLabzZrplYANFGNrE5Mu5myB1687s9frUtxP9gtIfkaVnY73U8HCjqce/y/jXNht7gqpKs/VJGymDxjuCOatzahNPZymWSOJxIIwplGHVj1G0/K2QOoI47GnyDJrGWKOKZnXzSc4U9unbu3IINatuPs9ncpM8TNt3t5krYDYxyB2x1IrnkuJEON+IixyUOD/3yQDn3q7Fb28aK/2uL/V7vKcEBs5ypz26fQ0pIRn6lbpBeB7Rv9HYDARWUK23Dbd38OeK1rOSODTzNhmcWzYyxGI84Xkd2JbkY4FVo1W4ukmRLqa4MbMIiAqsBkthepx/dVs8ZrTtp2kt4pPtP2ZmXBSNFy3+yVboCuPl4pt9xmUsKzH91HFB5SjaPugAZxjH8u1X5YjEPNVvMuNiuYmCkHOeMd1/zjmq9xcr56M7tK6jY3y8IO+Md/8ACqa33nwkyHy2hIZRjAPPB6+lLcktSXcdzKEW3kk+T7qFy64xkLzjj+QpLON4Qiboy20oUfIUdcjtyec96ht9ViOY/uRyyBg9vGMv7Nu/kOai89WmeWUybtrF/wCEY7O68AkZ61ouwWLD6hLFbfY3MbWxIcQwcLHz09U47Lx0zUV7EHjkeKDDp8rochgp/iK9+fT1polgidPtILJsz5jx7TjaQeg3fxHjJHT2qk00tzMbgJ5UfMac4JBz1HfgfhVNCLX2dvsYTIv5I8rGjuNkH+03I5PYdTU0Om3Oz7HKq+csoZWUnc24ZLf7XUVY0p7CaRPPDxW4ywESLjOOWOfwqbUZha3EGJTJbCIPtcqxU+oK8kc9Tz157V0Rh+75rmUpe9YI729sreOOHbBbxdBFwS3oD0J9/X3NWJ7n7Wn2p0t45bhWeSMK2JJMlTuyfvHA9BnOc1nRSJdiVpLYy7cMkRkx1ZeefvfTPQ+tbKQR2xgL27SW02ZWzMyNMrDBBVuhUgde46nPG8NY6vQzlo9CGYXls+HaR2RsxFX3jbgEjA6jj0xTb21NxHE9n5qxXDhbhpD8qN03MOoXseh71Yn8QJbs8cNrGvl4SAqqsAv94n+92x7Vfv7i4uPDsV5bzATMrR/vUEfmAsCAoGRuG3jvg9++vs6ck7PYhTkt0c/KjW7mwuLjfOV8vzB/Ceqj69qy55ltpS6GRHkUMBuz82cMf602dTFeMksJLKdx5yGHsehqykUM8ImVHJViQjA7yDx8oHJPB5x2ri5JSex0J2Ii5mj/ANJ2+dHIWdkGDk4IGDzjqfxqKe3jjMcjSOwkO3aWHpU9/C8EMJLoxEajfg/Mp+ZDz6A7fQ4qrbNuljDgBC3zZbGfxrOas9S1qjVtYXFsjdVYjb8oxu/E9OnHfBro45Ev7EvbNCsavlQFztdvuqFz8oPp27Vl6eSd8EPnG2CgM7sAqgE8kd1BPPcZz05pdVuI9TvJGhtrw3McW1OAZJFyM4K5yR6HI6GuuhH3L/gYTetjq9LSxDZt0jabZta4X/lpzyAT1xxxjjIrSxXnWn388bxzxTyM1m0hkjlUv5ajjI5w3HbI+6QK7zTrtr21WYx7Rgc54c45IHYeleth6ilG1rHHVg09yzimPDHIpV40dSckMuecYzUuKMVu1fRmSujhPEmkwWF5C1rZXDtcxyfP5jOFlz1+uOcdx61s+HYpbuyt7i7hkhmtW2RBoiu5NvHXqvP5j8K6Ejp14ORg/wCfWs/ULiSzeIRrKTdyiFCmzEchHDHcRwcc+/ua5Y4aMJua+42dVyjyl4IT0Un6UYweaoahpb3zOQYU8yJoZCryqSv3l5U9m79cH8Kn0y0ms7GK1nlScwrtWQbgXX/aznB+nFa+0admtO5nyrvqTs0gjcRbA+DtYrnB/wAimtAtwpSUcSfeHT9etTKgWMlpY3WNtrSIwI+v6j86gS9DThLGL7ZKGK/K6rGvUZLt8vB7Lu5GK55Yig7u+ptGnUWliXYwiCsC77yoCIcLzxjsOPxOOKxNVkjiv0uIT5l1bHFxABuBjI5LnOFx/tEcVo/v7y3j+237zsc/uLRmWJmXht56+vynA746Ukfkw2EdtJJGFYMiiNQqEtnCjHy5+YD3rCNZzp2NOS0rmXc6fPfN+9uLaGNVwZopAyCMcsSwI3KM8nhQcctVKDR4/O86Helsdv2eQfIWyexU5PAyGAGMirosxHp7Ws0x+zCWRt0CrsK/MS2ApB6njGBs6VS0a3soCkkTeQZYgBdTy7Sw3fIzAcBTwAuB7149TErqrnaoW2K95ZC0kvUtAwRYSzzq7J5jZwV34PAA/vA+vXNVFuWijhgkjikdXY5fDBFbJ2lWHUk9B2zzxW7cXUEluyzl7SNldQVj3Aqq8lcDO05AXI7r61y728l3Ha+UJXimHlIGBGHA+ZefbacgY/GsXPmlfYpKxYRE8p03eWsobbDJMOGCjcduOAefU8Z9KpW8aPcQo5uPNZ1WIOzLnKYGSTlcEL2PAx9bskjrcRRRthYwvlEqP324lduT+R2njcfSi0dre7cXgRp8K8TvCrYIVhjqfm+foP4V9eKL2KE0hpnjSCe2VVgkT5HYqrId2B0OVDdP+BVbtbC2R2GpxvBCs0kS5+ZAF+bbucfMWHTv/ScXhuLNJ2ieRFR4niLkP5uNyqAFG35s85zlueKa16kkYEGWbzEliDptO4HYRlVwCAcHHUkHvyN3VthbF97aSS8Fu9tIjSplYnkDwLGp28My8MTt4B68Y4pu827xO8rQFiW8lUOchsYz36ZDYHb1qnahWtYmnxLl2uGidymCA3J285HIOMDk81DPDp8lnmBJozKseyaN3EeDkAIpJxyTj39aUYxUbAacDC+jeWNZIxGzfvSoKzKMklRnoOm4f3eDUgSYjyJljjWFv3ckqg7h97IHJ7446nHamxhYPKjt44lXeXj8wgYXZtw3GeCNvy+nrxVezu7XyZ2uJoWEb7xI0+NzKcN+JbOCBk464BqHG3QpM0ysQuAYp/mVtrGNVJjP+1zjdx9Tk+hqoWhkkXCM8jkeaqn7w7ICR0x3+uOtJItvFL5uyVCpQxRlpIVxjAYYGOmff068JvaOaVHFskZj3HDEb14G3dz8wbBHs3vRTjdtMGXbbckc5klNw7BeGddgXPOODkc8ZPXHvU0sTQWu3I3cRlVUKM7ht+nI544/KoIpEWNfMi8qeRw3mDo43fLynZffP3RUxDCUl1VSjs0TFQHVOmS3r9456DnrXXSqRpayTZlODlsTIGIGFIXPUnqOfbP0z61Js+TdlSDkcH0ODSpK0jhrba33sHjaD/e/Q9/yqKKJ1YlpN+5eXXgn0weu3rxXuUKzkkqadjzqkLN8245sIMsQv1OKbuz91Gb3xgfrUixqvKqAaXHNd5zkW1z3Vfpz/n8qDEp+8N3+8c//AFqlxSYpANxx7elJinkUmKYDcUYp+KMUgGYpdpxTguTTFO71AKg+h+lS5O+haSsLtNJjAP8AKlA3uDwdvqOlMldFUJhsMvADct/n/Cs5TkkUoq4OPnUYLIVzuxlfYfzqKZ3Vgql1OD13Y6Hv65xz7GoXnmRpFDgnbnqMj8+3XnrUd1K+YwYY91wuw7ztR2OOh3c8DOPb8/MrV24u+5206VmrbEscrweYsjYZCMeb83zbee/Pyhfzq5LF5gZ8rgqP3Yj6ntnHU44rKslkkCwwujqxLMfvMYxgKCcYG7v/ALorQ23EUaoqws2RgnJf35P9axoqcoXS2NJuKdmxwdIWjRnUb/lUM3PqQaSUeUDIucRjcqqcHsOPxNEKuypePLISAyEbsjGeWJ6k8D24pJWyg8tlYeZ8rb+JH5+X2APH1GK6FUtDXdmbheWhaiXy1VpWG50LYDZOO+M9qrGRrooxdxGC2XbAAbHY+39fxqLyHmwMKULFDImMnj5sbj93PHQ9M4pka24Im8tGMg2YRR6Hhckk9Dkk4HNc0m1qbJJkKs5ligXaPNQyvt+6ijj5m75bHU8gVdiPlgx+RkbSpzhcZ6988DjNRRq6zuHELFmzsjkwqHHyr0wO5wQxNXZCkcTSsVSHlnJIAA643fnVQjd6oUn0Qya4t4raW5uP3cMZO4v8oHOD0znnAAHc1i29u9xqy3F+PLhjs98cDuPLiWV8fN74Vtzd+APlHMjSCSOLVdShaCwSQyW1uykljyTNIvXgfcT8erVBHaagDLc3LwLeXCb3Ty1IghBOOBxlicY6Hp61rol7upFn10NGW+Zma2tnSABQJZY3VTCvZR2DYxgfwDnk7RSWc9tBbJDYRCZF+T9wpbb2wW+7z6fTNU4tPkdkVraZYU3bgzKnm5O75m52kjBJAJb7vFbNvByr7lZk/dtHHGF2r/dxtzj3/KlGMp3lIbajaKGRxyySK8jLIUwCsaELzz03fNnNSRJMuxg8ZGcY8kqM9+/Gf8KmCHyUZx2CsPMKkYHQ9umPSn/u3YrjlSo4PTv1H4V0wp05Ja6mE5zi9tBiAjk4C7ySu7Oc9Py4HNTKQ3de/Rs0Iozv2Krc4wOcU4qDz39RXTTpuK0ZhUmpbjHU/N9Av5mpcc0zHzdSfmHU56CpMcV0JvqYsAuSB68VwkGJ9W1C6ABdb2T5sc482PAz26V38K5mjHq6/wA6840UzSJdMMNHM7sxzyCZFZfz+navPzF/u0jswS95nV2sQMqjA3SRKjYHJ+Xj+v61uEfMe3NY1hk6lbLgYMSk/gprcI+duO9ZZW/iNMcvhMnVLcfYLppb6S3RoyvmhdzR5649tvGB7muHubTUGWC5S5e3jhLSxbSVfzFBDLtY5zt253eo4NehalDcT2LxWdylrcOQsczx+YEb3XvXB+Kob6C8lW9uWuowq+Z5MXl4ADEs7Jzv6HpgZB71eY0lOPN1RlhJ2dig2ofabAMsa7o3+cfl0qvd3MLwFmaMuGDB0GGyO/Xjp6VUt5yjBHHmKDglT0/+tTWk/fIqxuT0x1+b3rw1E7is8/lz+fazfNyG3Y5Deo/Oti1vHSIoZ5HlZApk3heMcBgQenrWfcq77WZiX6gMeWHPNNsh58s8Uu5P4cK4U7v4T+W6m1cDSCSG/VLgSOhwU8kgDt26fy61dma1ZJmZFZpZm2xtGPl7E7wfbv3zUcV5FZ2Daff27TRr/pEhVCpU44zweO2eMd/WsvPl7keSURzxZjLfxc5G5hnp3rK12MuIUtpS8eHLIyruw6jPoOo6dfwpUgiurT7RDAsLygskcchYsy9R/sk56fTFZt9aOi741UK+5hG/HQcrzxkjBo05Ll9ORmtXaJ13rKW+8ucde38/Sr5QNeO1SGCL7TBMkMeT5m8KF9d4yG4OPxrN1iOJVW1N158sMiu8rnBY7QOAfbuff1q9a3djdrFbXEzw3KPuDPGjhm64Vu+cdDS6jEt3EbqSXM0L7Nj/ADpIrc/gQe565qVLXUBtxBb3drE9rMZJ402zBgVDHr35yBxnpWfBl415Kn+Ebufy9KtnymgOBl2A8w7gPk7YH94evfv2q9pklusF0IZIZEbMP77aGb0Py9RnuKb0Ay2vHto40lQlof777RndjA9PX60pvmnxc3E0caxlgzMnzbf4Pm/ibrkflmpL7ZsDT/K+wkBFDBscYORz1P5VSm8u3ZMyPIIlIWJskKp53euT6/hxTSuIdLcW8lxGsM8irIdqq4+bb65zkHI600K15MVMZTc3y8bRj6HHvUVyrLFFcLcxAkb3KfNg+g9eP/r003MYd/nd0QfunCj5j/telVYC7DHFFqG0bYod/wA7r83IHP8An2p6Ssk0nlmP99FgfLkMc8/L37k1kxvC9wzS3KoBnhg3zD2Hf3FWZZpE06OKJ4vMYZBIxzuILd+xAzVJAMV5JLfy8ZhCj7PG7fIpB+8x6tx90fzrSW8RLNVtpt5ZWEqlT8rAZOwjGe3vxWdbr+7igR/LVvvZbA3dOfaprLzEdpAhkGQ5Zn2OexwTk/j9K0jYTFgvZI5S3LM/zEq2D+Har9oLma5Ecbi4VcrtGVdcnLFgeM/NjOcenAph1O7jnkkmNu6sVjl82281kHTbv4zwM8dfTNLb3t+t3JbxzykRq6OogUsIyxz8p5PDdDmtoRitzN3N6ysZTbCS2v51VEbYt1Eh+XbnYG6/nwfaqii9ubbzRtgbqnkw7QSD827k5/DJHerNrr7jTXmngs7rDYkRCyTeX2YFgVZenVs/hVFNeeS6uLGzW4MM0nnSReUHORnonOfl79flBz3rqn7OyMVz3ZCbm1u9lvKII5pGzHciPaFyeckH9SDirAtmuLF0sXluXVlcQSMq8f3jg468dj3B7HJ1Sxtob5X/ALUEjk42z2ssb8DhTlfmOMfmDUuiagLO5PlXLmKVdyPDEDkjo21h0HIOPWsorllyyKltdG7Jp94uli4+xQywr5u2NrRneJNu0qzBhyvXOD6io2Fu11ZedbFWaJIdjEREliSSZGPzKWC8jByTyOavazqtvBcI+nzvZxfLvdfm81Qw+V4+OR+fUZrOtdQshc3gawjlZkWMW0c27c2Cu5cEg79+epbIIrsTh0Zl73Y5/UrjzBFC8Tr5SeWiyEscZPvj7xPTt+dVyNwVlXbuO0DtkdB9eakvY8vLslV44z8pUHp079+Oev1qvYxySzonyuGYfeOK8uScpXOuOiNmznVJwoRIwCys4c7/ADDjaDwe4xx2PNaWjvqgmleyQhYi07oG3E/wsrnIYZUnAPdfoa1fCFhAsayXM0f2ti6ogbDGNdp2nnJ+b5uRnjg4rol0OxieV4IFjaYky453knLEn72W9c+lenQw2ik2clSturHJ6VpSWOsafcJJdJHcucM7KPn2gjj5gy9R2+91yCK7GG3gtJG8mGONZm6Iu0bvT+dVbLRhaalPcGVpRIfNVnC53nhs8YyRg7gBnvWoUzt5PBzXbTgoKxzTnzMZjNGKfj1NLt7Ac1pcgjxSEA4yoODkZGcH1pst1BE5jZw0v/PKMb5P++V5/PFR4vJugitU/wBr97L+Q+Qfm1FwsSTSR28XnTSJFEP45G2iqxuJrmPbZQsN6nE10hSMDuQh+d/phR71YhtIYJjMAz3B6zStuk/A/wAI9lwKG89tQ3OoEACkMHJc4xx04GSeM81zYmUlFWRrSSbMeeGK8sp5ZHvrq3CFImfKRcFdjrGpweuAWyflzyKtXF5aRXRtgjhUO9HlAXdhv9WrdwOOR/eq2ZyGcMGPGwIZMfKMNn/Z9fesx7uOO+ia42NAFlEjOn3MsnzBeSSy++fXk8+O8PFOzZ6Cm2tiZHt2W4a3uFCNFvZ94LoAp+Zj1Gc9OygdadLeG40toFjYNIgy7KIlViuRhW6gHbVPcGvQ0cKuly2XEe1nLg856YzkfLx8wq/PfQ3AiYWkbu8ixtLPCWVgELHP+0MHPp71hObguWP4bFqKdmzPkl8mzmV2gQeVulM+7aWDZYNj+HqO31qvaWkPlGWKON7iVC8kgQAqMgLtA/gwFC9qkv4me3FncCZQrCX5NqtjBYOWPBXnOOp24PpVa305bSK7Ivw/2gBLQqobb1K52nnn3GAK5oXW+5oyO/gs576T7S+Io/lJxkImfuhh3J/LgE9qxYrlLcTFswzMy7tibn2qfkVCW69jj+Va+n6RZG3Zrm5kG4yY+YRB8HO52J+8DyeQAPXiqS2iGwhus+dIu5zHG7bGy21VjXGWVcDH48YqXFrcCS/unhKXrXcKSzR4LNsEcbcFkDEfeOcj0I68VS1UBngtxCrPt2piMfvxklSR2PoPYZ4qJo7va15PBtRl2tibeFYNsBXP3Rn5frxT7m4tkX99CXlbDEtKrk7XYAMQO+D+lNQ6i5i1axXkirMXS4+1EuoTZliFB5467uMH8xVu8s7iMGO3eWIFBbKC7FT3ZmbHbgZ65APvVGC8uXjVba3jdYmbyUC/NgDAO0dR94e4C9aqtNc3ckk0btJNDjKfJwwUgEKOG7cdeamz6juXLSOIwpMNiySHZ5kSsSg4GxevLEMTnvn1qe7tL+O2tEgsis8dw86SSRLEjHZgOQDgqABzxkkDBzUGmX8DQSCW4SOWPd5JlRn+Y/KCV6cbm+6Dj0qxeyS+RJcxTGa3IK/vJA2U2nnbj5fz4yDwDRGTTHYp6ZbS391BdTSq1o0ipNIkpHnybX+UhuWXr0wq7sD+Krkd+sQZPNMAESJI1oA/ykEbFTdwAwxx1xU+n2RW32STRyQyCNUlA2yKuNpXrxzjg9jnpWh9mS0ihMBeK9K7ZJFff5ZyMblVQcnDAYzyAacqiuFjPS1ka6lls0aIeVtje8RwWZudzDG0Njj2xxjOauHTriZY2guUjG4GJynyh8nCBhxnPoO3OarmWRbpTC8bRlt27aByBkrno3XOMfxZzWnay+asjSlXWNWVuG+Veu7afvNkAduBzjmrTV/dAjs5Pt5VJkKkuE3nDbZMc8HjHJHT/GpILeZAzJC/kLyqsibVXj5c92GPwqveR/MWt76MA5kVA4WQ84255P3s5b8Ktwi5VTMZX2ytsXcFbcMZU/U4b29OK1jGM73Jk2tiyfLEbIzPu3bljkYMB9Mctwc4PSrTqdzcHAP1xVZGGpWjSQfaQ+3zIpZN0Z6+jdO/BHSlVLya5kuJG227HcqqMsoYgbN393aOo77sV6GDnKm+Va3OPERTV3oS4oxT8UAV7RwDMUmBT8c0uKLgR49qTFSYox7UXAjxRgetPxTSATz+fpUzkl1Liridz2qJ1fLFMHjkE8Hv2pWXZMrGVsEE4bacqMHsPr+tR/al3KxuEjBXciPGVY8bu5PGO4rnnWSTNlTegrO3m7EH3slyxPygYwKf5QdlVt2d24k5Ht/nFQsyiTzkTbGi8sSRkYLEHjkcjk/0NH2hSoZGJz2bgE9h7fjxXG6rtZnSqfUhIEkytjew7bcn5f4eegP9Kp3ZeRjIqSEoBKZY5cYGPm3e+O3vV15Ea3kkkQlH+6rcAc9M896qtCkkTWwSNQyuiRLJ5iMdrHdxjnjiuaTUluapWZNpBxbLlwWPMjAZ2n+Ec4BXAGTjOalS4a6dkQTl3QrlfmGB/Fj8ht65YVRtZY4rN1lmeURnezSYyXKhhtHXqSOav28xjborznjlhtjVfXn5QPyPanRqNLlFUgr3HtH5qku4EMTFVVmznacA5z6/yqtcl3eO9iuY41gkUsCQoWPBVnPPADbfXOPerMX7v5A2HC71lZVwuTln684yOfoBVOKWISzCaN0jQqqfOA+5T0K9iMZ71VSpdWQoR1uPEZSwZGkYW4Q/NbqWbkn5dzDqx7ADrjoppj2sccgOyOQEkKBukLYC5X2469qdDcM/m3N1HIHVN0UkqbEXd8oIJJG4g9fTge74L2NI0lSNt852SXAgl8shTwEAXcw98Dd1qKajKS5mXJuK0HyXKLt8xYxJt3Kn2diFUclicEbRn+dVY0gvmXUbmWR7bftgV5GQTMCDu8skYjUDhPvMTubAxUcs8WqXEqxSNd75PLSMx/LNIo+5tXlYl6tn73TPU1ZlE1pbma4mlLAFUQMYpJF+8QrgfIn97bxjHNdE3ryIyitOZmZr13ajULItHHIxke4zJLtU7V+63975iMnsF2qMkmrNhdPcGNnjSGRf3u2ctgMT94nHzNzheyjgck1Wi0qJ7mOAWsKkss2otJ8mxRlo7dRyQvRmXjP8R+atuSaNGMj3c4Jcl0LCV2bGMqMlACB0HHentG6YdbFb7SibYJII3jb5WHlOw5YNk57fX2xnFTyXjTyLbkxgE53xS7iq/wAI698fdH1rM+1OZpN+9lXAxNIe/cDcSMeg79KLR0ilVE+eTf8ANuTcSpP90nk8HnpXnVqznonodVOmlqacMtzJCzoZpck5Xbgn175+ner0TuSE2OTnbuOG2nqR04NZ6vHHO1ynyuzMWALEgZyp4Pr3oU3AuoWkjEh3Pu3HcVAbjCnk5GfrzWVKsqcr9Spw5kbCy4B8xQgZsKMHd196kUNgb/vdyepp5h8vlU2I3IHrmlKMoyRivoqCXxXPIqt7WIlGWH4n9akwKbGD+SgH/P41IBXUYDouJoz/ALQryfwhJutEyM79xBzyDvTivVXby43c9FRm/JSa8n8Hn/iVQHC7S4yQfmDboj+RH8q83MX7qO7BLVneWDtdamnluRHAiklU5dipJXnoOB0roWHzt9a57R1a3uFdxtklYv8AKef4tuD+FdHuSRi6HIYnHvz2rHLZJOReNTsivcfaDERa+WZcgYc4GO/Pb8j7CuJ15beyvo4ov3CxtIkyxLtViyNn5XzuAVccdNzHnt2d9PDZxie5nMMIO0kx71z78dTjAJ4Ga8p1i78q8kFvceerdJ96yI2eXVSTwm44A/EGunHVLQt3OfDRvK5hwSFJg8oaIZwHI4qxJFPLebXTYxI59Pf6HrRbB7WUu4B4IyT1B77amkvfMYsFXLMdqD7q+2O1eIegT2duhmuIblhsKNvK7Wbj2b3/ABpmtQ+VcWaWcLuqwlEkeLZ5vcDd0I5PoQc0r2juEnt0kuCW+eFEJbb6+h/+tUmk2FneXbrP56QCLcJdwRY8/wB5T+dR15hlhLsQ2CxizDbW+eK5A2SfUA565+UnrmptOt4EWUb44lyJFST5ti87h6Hr/Kqt/MWuojK0UzPg+akXl+YB8oyp6fdqUXEF2EaaIbEc71H8XPOcVL2EOtbveHfzHby2McYK5Cr1Az3A/wDrUXMr3reS9qgON3mIpwV6FvwPbrV37BH5YuUWBYc/Ptdd/X72P4jj2xUUziS58xXCwLyoJXDNnsO9RuMy2Nwgnt44k2QcsdgAyByfY/jVW1unWSWNFWRJFHOc9+SO3t+Na26Ge/cTM0cMikh0Gxm45B9e/Pb0rMiZUuHeNg0Z+dZVyQfqP51YiMqbJoN8eE2thSOWwec/yoikiG3LhFyRhCfl7jaOnTt3xTNQvZZZo4pRGwj2tGn90e/Pv1qMPvuYyYsqzYOHxke3FXZ2A07uecxsQccfKX+Yn/dPbNVYreGR5Uup5FjWMSeajq6Ju75B6HnK/wD66sm4EVv5tzGwDYeQRH5hnj06dBio7zW55vMtolQRH+PZjJA4Jxzx6/nSSfQZrJZ6etuTAls85fKcYRmGMn/Z/wA5rm/PVLly6YDltwBx37elLaX7LGYMqAGycjcM0+WdLmNiimMq/UjAz14/nTSsIryxRrGjsAF3Z8wYxs/x9/51DaGW5cOWaPaeFAB4JHy5Pfird3bzSJbQNNCEJffNuGz2zjnPtj0qivUxuTvx0I5/KtEBakQJboTjzCT8w65Hr7f/AF60NOYI8LvJHvZyU5BXGOVI7N94Y77qy5FZog6qzHIVVC5z61O9obVg8xTDKCyDJxnp9aadhM6ELpV7Ni8inEa7TkEq/wAo5Dc+vfH9czXEElvJNbWhVoeUiErr5jFlBKlRuUkEErt59xms2APc/uoYkllkbanz4+b2NWrRJbaFHdZILeVV+eN8bz1B3jJX5SMcY5rqp1e6MZRI5dMa0EcrOZZ5W2bNgZW+QMrbs4bPIKtg81VghzIrRIYJQyiJULxsxPQq3/fPX+tbUiaLdacytdF71N8CXFx99gTlWb5vm43cDHI4rNgbF5iSPKxhlWXy8iX5Rj5W6tg5xnscEVUlHm90Sbtqbemyz6jYz2WrTsq265WPCsxUfK3yv2IJxz2+lNlEGn3155Mn2/7sMj+UGDLtBWToMyYXDY4YDPXINOS9k0+VSYbZmUkNgkpeQ/xMFbDDt64qN9Str+7eay/0ZEj/AHcchbCsCNpLc9iQe3yitXUsrPchR7CNaKsgkVI9r7pM+azMvUqQw6Kex9hnFR37fb1ykLokIWIBlBW4JC8Kw74688+1XLXVrqy02SezltWdmEc8DLu+9znHZd3RvUc9a7LwnYeFpvDsV5q2n3AEszxEvMzKGHLgBex3e9S5KMC4xcpHmFnouqarqL2OnWT3s6nDrbDKoc4+Y/dX8TXWR/DTxn4evba8EEU+IncGwk3tFJjAXDbTnk8rnvx6+/aZDYw6fCunQwxWhG6NYUCqQe+BXN3XibyNTNrJcW+IZSCGGC2PT0wfzxXLGXK7nQ4pqxyFhpsVoIBcAfbVUt+8QLIMjuOvC8ewrQxWrFq9lqG23vLPfcx8xm4t2yrMcbgwGACO9Q6zZJaJcT6cZJRGfmhYq21QcFhuK9/Vvwr1KOMhL3WrHBUw0lqtSgBknHJHJxVa8uBbwSETW0Uqr8rXEgVAf9rufoOay7Wyu728S9GprJZyxgiP5ZGwctwdoUY3HHytjOM1fi0PTIpjOLKNpyMGZyWkx6Buw46LgV2XbWhzNJbmfpF7q11I7vaK9vsCxnz12btxJJYjc3BA4WtM2skxJurhmQ/8sYcxp+J+834kD2q0ibVA6nGM4xx2p2OaIppag5XehDDBDbxGKCGOGP8AuxoFB/LrT6fioykm7iUqgwVVRg5/2m9D6DFEm1sriSu9RcUSfaUty8Jzt3MqNt2SNt4U9zjG4Ac1Cs0qSNHLEzrGVQPvDO3y53nngHnrk/L9KDLm4t5LeRioJnJJwrLgjcrH0wfQc+9cOIq+0g4pa9v8jop03GV+hjvqAWRX8qGMsqvslnHzEY4C5yfmHC49uhpsN9Lb26SOu2381/MeU+Y0jDG1iCQc85zxjaOKtMsYf7GIIUlm+eK7bc2ZD82VbqGAG5QDj5eOnNLV4DBqdhLJcTmLckqbU3lIwwGW5G7DYO7r8zbvu15s8JLlUm9TrjVTdkiexMFpeMJ3RpXYebcwKMMxLNt355GGGDx07ml8xLW9W7gmhkkRpFgnkLZwy/MzKoHzHyyo46dM5zVeKWaSM2EthKvyOAkb43bgzLESOACQT3OP4RVa81KO2vgL13dJLMF3EoVWdVLMrNjoy8BcDG33FZuHLOzd13LvdFswyTPZzISSY2McH+sDx/Mcg/wgE/XnBzwajghEGpiO6Eks+9gEiRSGIHyttzhVYHpyfvZzVWB7iJoNSnjaQRqk90wVkViqNuQdRtDHOAOTgmrl1eyQXipcPEzXGT5RVd0a4DeWdo47nnnrgUrKacYopXjuNZk1B/LMa+SrBfvKzq38SqM53dePRvasrUb0QTTpceeHRkJlVijSKV3Bg5wP7owOpGc81u3MheK7jtoZGlj3gDDKpkwAx8s4PUkc+tc/qkUL6czqrIV2OsbsQd3yrgE8cBm5GFBzwKiFPkdrDvczH1Fvsd7CbeKORpTBvjfdt+bAXbyuDtJ7dMj1rSlsp7y0tI/Kt41YN5jTSLGI1Y/IFXavzFueN2fvE0zUNFudNWZJ5bb7Et1CsiBtyxSbtrYX++F+8x6q2MZrXkuIoYEhVEW3a43xofmdstsBLEnYWYAdfunPFaxjZO5LW1jmbDUCLPy1RpVt2O0QybZPM7bePmH3/wAOPSn3ssMt2rxBBMFDEREIGXOd25T8p5IOeT2NWrfTLPWIZigjiuYpdhh3fPGd2SWwB8xxsVVHGRnvTtR027tAUhVVe5X5JIJVwERgzdAN0jA4CElcK3Paue13Y0szEt3a0uxG6Rjb1RsfMAezL7jANdPYzNLpsBlhEiTNhPNl2M24kfiPlHy479xXNzabNBZy3Kq9wqtuSOAghVOSBtP3RwOma2rbQNkdvNNHIZCFLSD5Sh/hOOitnHJ3ZyKlxtqJPoaNlctcSn7aGt5Q2DIMLkZ+6ckbWyGOOmCKvfbVVJ4GimQxsFEaJyvGcgAZB/U55HOazb6C5S8kDSwPcSokci542nd97OcgYIxjuT0qvE1uLeSIWyxbj8yktkN/C2QThsKAD3Nc8o3dzQ1JA73UTpJGp3BXVz5O9SN25vQqQD2HPvTftKTIJfKkj2sIItnyHbgDawxz97p+vql02xpJF3B2RA/lxyENGV5KIOvOTknBH4Glha4uXgL+YQ3LJcJgY6AnJzyD26VcItiJkMBcW0iyxXUYKiRm+8cZI4PXnkCrUErzHy4rYxW5jVSX+UBupCsOvX8j+NYe8SSZDI29VVm3OxJBG4cnn+H2z65pJp41nT7JOROcRtslRcuWypB6Aruw2PbvVS546ktrY6C3uHRJTkSSqTkryueduG9PlH4H14qx5k0mx5ZvmxhhG5aNuOOoHTOePbtVW0YtLA8y/v4/vDAfcR1AYD9ffNLa+Y217zy1GDwB06Z+bv1HT09K7aCatUvsZTtL3SyXDOqKec4PvxmpAOKiX76uDhMk4OTyRjP8/wA6sY717WHq+0TZwV6fLZDMUYp1FdRgNxRinYpOO5pXCwwsq/ebB7Ad6rvkZAY7ixCs/wAxXjIwPp61JJcLh1jPzKdrF/lUfj+fFQMvlkP5ZOSruGGSxPAH06ce1efiZJvRnZQi0tRVZ7jhoiSyjd83y/iQPbpSEyJJtD4DKx5+9nd6dxj6dqliUhwyOXYuU2lSc4HsPrz+FVH8ySMSMoRpG4HLF92fu59Rx0P5Vyqq2rHRyWJTGHd/njY53FpBkHHzc+2QP6YpLySSK44VQ6sXBXOA3qcjsf1qSG5ysifLE68NENu7B/Q9Ov8AKqEkkzytLFiLJXAdM7v9knouRnnv61NS7jZFR0epXuFeOF3KoTwzb8FR2y390kc5IxzTALmX9959p9lGFzG+4NgdgPXgde3StOAQs26NhkqCEOflTt34Xt+PWkgsjMzfZk2xbw85wuIgp5dlJAVRuyfXHHNYOK5dDRSd9TPS0u9QvINPsLd5JSUErMNojRTky84HI43A8H14Fddpvhmy1JfLtLiMXYQyEyRb4ZBnGV6N+JyTzWZYXsOmzxizvCswdnd5wzGRlV497AY2jByF/nXT/D02ccd1axFnuYlBMrrtBViSQq9hu59yaUfdKeph6l4T163kANr9pUKq+dbFXIYdG2sR059f5Vzc2lvaTGK9heE4KpHMjIz7m3OSOrA+gznHPFew+IZ9Ut7NP7MtjcSyPtbjOwY64yO9cP4j03Uk0mw1PUYF+3RzvCzYDMY2+ZR8vA5HahiRzPkv5UMMNv5DybZmjjUM0Cf7p6t1wOmSOMCltYGu5ktYdzISFnkSTYf+uYb+LOfmYdPujnJEikNdTRx7sMRFLdJxhscRI3btlu316Oe8gSNbW2S5mQBYBBb/AC7v+ma+gzn5vZjg9aunOOjW5Ek2SzPbW1rNfXCDEqpDbRMflIXOxUB4GTz7DGaoPbGVYpo1KTSMcCPn92MHueFX72M/MeT2AvafGLq9/tC6QTTxOyQNtUwxDAyY8846gv1YjjAqK5uDKbmZNrrIPKtSWZDJxkrt64PX8vXNVWq+55/1qKnH3io9lZzTme8L3chw7SSqvXkljhepP8R596LwSTzswiVbcw+WjfoVIPU49vypk6hyJN8DS4DB3Ztir9044YL09Bn1qvJE9oizTwTsIyJGnaXyYWHcfLuPpkH9K4eeWzZ02Q6W2ghtwuwmB/kBVG2jnjd/EMbu/WnsqNdRu0SuUVm8xSAD+GeDxnr60WXyXDSmdkIZQGR1kxgZwWznHXqenSrkCKbYOj8MpQMqYB45JVePT8+lRcY0r/piqku87QEyxIbpkrzx9atllKxzPHu+Thg3Cj6nv3weuD60nnrL+5R/mHSQNtBwCcZ/GoWuVjggkRhudsRHdwWwWUHjJ+7/ACqJjNCzvWR5YrdGMAPlwEuMEDqw98HgcYIq3HdhreeWYsgyD+8mEkgXOzc3oN2fWsG2nhu7NNpMltOM+Yz8qnO0fN3yOB0Az3xUVxcTiO6tCsat52ZDtC529MHsM4Ppz+V0K7oyUt7GNWMZxcTbt9Rik1NLZ5VkdUmOIj95g4jXjoTgE/nitFZ08945EMeGVUYsMPnp9OeK5LSLlLjWb24niaKV4ljeIDhmJ3ZGPckj0LZ9qt6lq8c1x5NskkGongvMfuxhlz2+QseN3UYOK76eYz97mlr0OZ4eGjsbuozJFpl8yupKW84GD1YRM2Py5ryfwsWFlwcKzLt4yMjZmu2url08M6rh086KzuFLMp+X7yytzyW4VfxrjfDAdrBVRyhDvvGOMFVOD+tb4iu60E3oGGhyt2O/0wPFfQq53A9GPXhm4+gGKs/bhb3dyji4n2yfPAHBCu7EjcxA78bT2wRuxUOlr5mpQgseWA+nzYp1zr1rLLcbba3luEmkiAL7+MlM/L1JB6dRnHbnmoKPNduzRtiL2sle5ka5rkcN6EuVMsyLJF5NrdcLI+35A/BzgDcxGPmK47VxWpwQWdzIE8ue0Fs0zCHLbd23dnkElTycZUZIHSrOpadAHvJlRUW2R4yvmkETchY2/iyv3sDp0rnDHJeo0mxIImZY41ij+VucMv4cHnv9a3rVJSTUjGEEtjotQtreIumyONHG1Sh3Ae688/8A16wnIkuCsJVWJyBK4bd69qtS3MrW32fduJA2KYgMf/XqlGiKX84MHXofQ1xRNjTs7qWOQo6fL6cgfStJZ4YVMqNFlcgCRS42Y5ByeevQVz8biFkYDduGa0I4/tF1sBReOFk6Z9M/0pSAqXExvGaWO3VNx25Tgf8AAh/DVy1vw9slq1s8jMMbw23jGOfWnSSCG3l3+TIBhSCzBlycDp0PcZ44qjYyG21H7Nd4AlO0GVeC38H/AI9jkUboLm8bq6s4Aomkkto8Hc21jgjdwcZH0zRfO00cU0ioF+YiRedy44J+nP4Vjyaw01whdERe8SphfQ5Ht6/jVie7ilhQG4C26KqIYvmAA44+vvU8oEcSwzXV0kFwsKSKV3TRbmUjkY64Pv8ASqMouNOMtv5rSxHCyMm4KRt3ZHbipnAtbx1xISDzscKD3Vj/ALOD2xVgQG7siy3/AO7CAFZSW3KvIHH+fu1ewXZjSIyuWCJ5bRruIxlTn29cVfVnNowSDbJINozwEx1PXPPH5ZpyRxQGP/j0WTG/eufm79Mc/lUpETIDcy7F2nccHJGPXp+FO9wGWnmvMqsTs2tkSj5cf1HpS6xJbRhrSFLWSP73mKPmxjIP1+nTpzTj5IjiIIQZzweH6fNnHfjrVDUAjv1ULHlWVDkHv+FJbgU0VYZRlwF46oc/kKmmlfDMEwjHaFB6Y7ev51Pdm0aKKTYxmCqGRUK0+eW4mssJIJMqoCkYKqMk5/Tk9qoRVjgVW8x5BEjcbyvU/X0q5C8ryRwqA0Y3F3YBUCjnG7+n+1iorqNLOL7J5LCZX3ecy9V2j5STyAPbrU1hcwLdJ5xVYY9uI59xDNkcDAPvweMZqlYNS2bDUIYoZRYhbeYMYHOAFbvnuv0bHbNMkmuElQS+UXC71VVyR398ipdTnV78CKzSKL/lqFQFWHRX3Hr2H/AcdqljvLW8nCzQwrLtCLJ0G7n8uv6Vfut6E69TW0eWzhsmmmhubhYii3JijDOsZJb7235WXnp2BB9RU1RoJ5UuIbny4mBkQeU0YZPVOB0bPr1JFQRNf6TdO6yeS2GAkibOccNgHr/9evX7fw54DbTrRZJHVkiiYbZ5PlIAPGPXv61rOvTUOWbsOFCpUleCuePQ3ryweXJJtQY3bPmyPU57/wA6sT6dNpktvLenAZ+csHRkwCR3wSrdGHGRzXql14P+HVzKZGe5Rixb91cTLyep4qZPDngFbOW0ae8likADeZPMzcHK4brwazjiKC3kavB1+kH9x5xqCQXFm8ojgMPm+aIgDgfKAoGeRnGd3XnHNYmlpPJfjY8rsoOzam7dxyM9zj+XNewf8I14BG4Rz3se7jCTzdMdPpWjY/Dnwrcxi7shqCLIGUst1Im8H72R6Gj21Oq9JfcTLDVKS96LS8zzPw/bQXVvefLdfY4vnZYk2+ZnHy9Pu7fm28fXpXrfhbQbG48HWcF9aW1wpaSQboxjDOcEd/u7ff1q3ZeC9I04OIFuGVgAUknZ1IGcLz25IroI0WONURQiqMBQOBWtSpFxUYmUINSbYkMMdvCkMKKkcahUVRgADtXil/P9p1qZyDlmycj+8xb/ANmr1fWNdttN328rHzniLIF656CvHUAaVpccsc9TWSVzQ3ku5bbTbiaKVlIWR054zuCr+tUrWQ38z29yiTR3IO/JYduvXivQdI0TTm0G0WexgmMsKtIZYwxYn5ufxq3HoWkQuHi0uzjcdGSIA1pTajK5E1dWPHm028sb0NY6ddSGFgVnEibgC3KOvO5crnsTnI9+vhdpoI5WTYXUNtznGfwFdn/Y2lmbzv7OtvN6b9nP51J/Zlif+XSH/vmu+OMiuhySwzfU4vbQFrtf7KsP+fOH/vmj+y9P/wCfOH/vmq+ux7C+qy7nFYpP4lX+JjgDFduNL0//AJ84f++aadI00uH+wW5Zeh2ZIqXjV0QLCvqzhlt1e4kkRHD4GJE+9uA7AjsOvtjtWdLexJLFKj20oUBB8ijargk4XqOMZOOhHrXZand+F9OuFhvEsEmbnDAEjjGT+HFcnr3inw3AwTTNG067lYZeVoPkX9OTXJUlVqqysjqhQ5dWU5rqG9t/L8qdpJGV5WgwzHa2VX5umOOe23Pas1kBluIGR5cW7Q5JY/aFYuWCofvHlsY4+XsSK29H8YaPI3lalptnArHdujt8qTjHQe3Faus6/wCEorCW/sotPutRjUrbq0R3AtgHqOB649KzlCTkueXpY1jQlZtLQ5KEK+YknaZ3B2zklXkcbNsmGxwCO5P6Umq2aXkFywmzIZCSd4XZJnarMCDnA6nqc1reB7uPVtbMF9a21zJHDuilMChlxgcY9sD8K9Dm0jRlRprnT7IBBuZ5IwNo6kkn86ThKOidyWtUzw9bz7ffbFWQiRTJLAy7AWVflVmJGVJI5x6+laGqyx2ptfJkR1jt5POQPsBIaLBHH8IGMdcFfWtvXvEvhyG4kXR9E06eZgqtdyQZBC9MDqcds1zCarcT6jaSyhGCT/u4xCoVdxCsFUDAz09aI4dpa9Snqy7dXbs1qHZCmBkDrE3cLjHzbe/qDjrQ8ksllcC2xc3QRWLplQAWzuywGW4K++AOa9Kv9M8LaVZSahf6dYwxrIJGk8rlpOgwB1b6Vx6+J/Bsc8axeFlSBeBJ5agr/wABDewrOOHk2O5zLTm0soFgYPa2kqqlt5R3M7Ow3FsDlipO4/Mc++Kzr6JxPDaW7m9hfdOoaLbtCNt+XaQw3SF/lHAwuADXtNn4d8L3wTUrXSrJjKuRKikbh74NEPg3w3bS+ZDotrG4YMGXcCCCSO/qSabgyeY8wtUaxhzNHiOSDzTInfpENox1VQoAyck5/ipsM9vLGlzFD9mkkfzZonysfyEgfeGT8q42jjtxXrEfhjQo12JpcAHGBluMcjHzcdaqzaN4VtYDdTWllDCCFMsjFVySMDk9c44rP2bKckzy1oWs4BM1tcQytGHLkH5mbc20dx1x+Jz2qO4a2/s+aVCEmMX2fZB8uWZQuz/a+8cD27V2p1D4bPKsf9iROUfj9yCAQfr6iie/+HdjMksnh5FmQh0UQqWB7HG7j8aiM4xaTeps8PUcea2hxNy80c1st+LgBZAq7ZuIW2jjc3LLtzlvXPXNR6aylgs7sJc7PKEW7cW3ZVkIOei/KP0610v/AAkXgdbl3TwbDhySzsE3E+uPX8a6PQ4PBPiG5drLwxDvGPNeSJAVz0zzk5xiidFJepCi3scVZ3KGNU8iVYoTvMMxzwCWK8c9uvI/q3fP5Z2O0q7QznO35sk8HucDHt3z0rvbifwnpF6EXw7biaNvM3xIh2sRgjOeD2IqGTV/Csp3N4bRmzncY0yT7nNc0atGm2nJHVHA4ia5owdmcdDFtuJHQNHGkofdsLHjn68g+p4HQYqG7t7aZlmngjM+VRXeQquPl2sFHUjHqM4wegr1jSNK0PVbY3Y8PW8CSHK+bEuX9/p/OtL/AIRzRtu0aZagegiFdkOVxXY45xcW4vc8rt2eWNQhkUY6DMYQdTnbkjntnv26VKIraPcVh3z8lQX5YdcZY/KOma9PHh7RVdcaZaBkOVxGMinr4f0hc40y0ySWP7kck9TWsWoqxDijyy12vEkqyICzBdmQShHq2eeKvbsnHH/fYNejpoelRoUTTbRVPUCJRmlGi6YoAGn2oAGBiJa3o140+hjUpOfU83xn0FHH94fnXpf9kad/z4W3/fpaX+y9P/58bb/v0tdP19djH6q+55kxVRksAKikVpFO1wg7jdgnHqecV6NqI0fS7Vrq6tbZUXgARLuY+g965Y+MLADA0GDH+8n+Fc1fNqUPdnodeGyqvW96mrnPbkjQKzIB0BVuSM549DzilxDJGm9wdn04UHrg+tb/APwmVlgj+w4MHr8y8/8AjtA8Y2IJI0O2BPXlf/ia5f7Xw/RnZ/YuL6x/I5V5YpLpmfCzIp3AlQMHkfN2H41DJLFBkeftk68S7dx78Z3fd9Pxrsj4wsi27+xbbceMlh/8TS/8JnbH/mE2v/fY/wDia53mGHf2jVZVi/5PyOLubqN4Uj87AXOwF1bb7jJ6e9RxzKLJhFJG2W3AF9jHg/p0/HFdsfGcIAX+x7TA6fP0/wDHaUeNYSSf7Jtc/wC//wDY1msdRX2i/wCysV/J+KOHMtqFQI0cdxgjdBtD4HBG7/647mozPGEe1NzGgZRuUszBl67cHrgk/wCzzXdnxnFjI0u0+m7/AOxrotDvrjVo/tMtjBDbFcRsOS/04+7WtLE06j5Yu79DGtgq1GPNNWR4uL8TTr+/jcbvmCtn/wDX2rq/BOofZ/EVmwPyTq1u+P0/VV/Ouk8eWCCys7iMQpGkhQqEGSWHB/DBrh42a2lWWHCujB1IHRh0rptpc5D2/GayvEekHXNCuNPWYwvJgpIOqsrBgf0pnhzWjremm4ZAkiuUdQeBWzSEeOeKNJXwittb27+fBc/IEL8q27pz0HzZ3Zzx71QjtZTC8qIkDJlSZIztiBHzBFzuZmx97rz0wK9wVFUfKAPpS4oVk7g9dDwnUrkWejizsrW+hLfu42a1MeR/sZONxycbuetQvZRJAkQjcuq/Njdg9PvMc5I7+pHtXvZFRzSx28LSyuqIoyzMcAClWftNWFNKOiPBJZ3vFVbdJP3K+WBhsvggY3behHGcetJJaXlzFEsoYoqL8ghciNgeql0zxxg5r0a+8dyi6ZbGGI244Vpc7m98A8Cqw8eaj/zxtfyb/GvMeJw6duZ6HqxyzFSV+X8ThbTRrgJ5f2SeAruXc0TZxjGQQMD2PP4VLFa3karavaXaFTlf3En3SuM8DBz357V23/Ceaj/zxtf++W/xpD481H/nja/98t/jR9cod39xX9l4v+X8TjYLSdoXC2d48cSnnyJAX9MfL7d+aivbXUJ7SPbpWoygus6yR20odSq7lONvr2HNdsfHmp9orX/vhv8A4qj/AITvU/8Anna/98N/8VSeLwz6v7g/srFv7P4nnJ0/VRM+dG1ss7BRGtjIqfMfvAhewCgip9Qh1WK58ye1uYD520vPBIiy9+Cy4Pfoc8eld9/wnWqEcJa/9+2/+KrmvGXimbUrWxt78hY/PMgEClSSq9eSem78an2tCbtFu/oY1ctxFKDnJaLzOe+1lLRZBkPv8zc5LY27iGJOOpxnt7VU0typJjMZO0LLK7MjyyPnHQYZUXgKePnzTILuO4hR3ZplklW3+zIesjMRk5+mOlSJbGV1tvkk8kbZ3EYIVzuy3memflLfd4HNXBNbnmsf4huGuvD+rXD7T+6L5R9qM2NuQoODgVl+FSqWc4dcfvm5I4/1a1a1qeceG9Th2NEhhihZCxRiTIFZgOflYjIHfvUXh5fLguY8Ruq3LfXjYOPau2nJuLuVBHbadME1KFejyjei/wDAic/Sub1vT5rpGVrf7DAt/vM0q5zG8jBWDfeI+bcQR2XHUVsaCzHVJZXYsrAKv0VyvH1OTXNL4rvdLv8AUbZjHKIbuXyIPN/dszSkkHZyxHB+ZvlIHpit8O4KT5hV02lYyry+1K6t7bzrdFubWJhJO5y0xLHaox0OV+p28474015POxdgDMo+aTkklj7nv6Ctm8hvWRNQW/t5DeTSJaCMMskqb2VptnVByRzknc3rms+HULeyihhlkzGGVpYD9+PhuQcEFfmDbeDxjHeqqxu9DOLLdzFbuiNG5SQD51bJ/L1rHNukLD/SA4YnO3oB659+a3mKQM8U5YThQW8wAAH8eT+lU7iO3Vlmd/MLMzkOpTGfauVM0KcbBm8vYeORjp+FW5LoKkDxPtZRtPOw5yf05rPlWKRlWGSRu/Izg9sYp8ySRqrvw7FlbPyk45OR78VW4Fqe9nvWRjuwiOuzoSoUkqT/ABA4qvDZQ30Kr9pzcNJtijdiPl9zjHtn6VKIN0KzbPmwduDwc8Zz9CarAlZkcRk7WztUdfw/L9aEAsW6C1d0eaO/EqA5YBQnJP8AwLOB6fWn2yxqiB4ZJok+8FYL6H0rod9g9osKRvv3b5iDl1YcqvJBxnvzjvVINBdOnmRFgoKsm7Bx35HBx/WlzdwJ0EENrCk6DLZbepP3OhUnoAOD+LVWe3jumlitt6bCySMrZAHZvow69etV7lhb3JlhvGBUMojLAbfbA6DH1q3Fd6fawiXcZJpsbECcKOAc4ODtx3AP51IFKGUWuDDz842kjJUj+76dxVG8lLq8SRkgHPA61sXipcbblJnZ0X5gFzu/H261jtBcSsBBH5rkjAzgY+vaqQFKJ5IpEj2lQp4A64Na+nm32SxTmP5hgFxjBJ+UH1H/ANeq8d1fWMrwuUQqDGUAV+f7v0oN5BFZTmZT9rLDygh+TgnLFv0xV7gakUcF3Gf9Gnd85cMMY/hGOCf84plyWiO6K28qGM7MGJmZD/sk4/ziodAe71a+ci9EWB8ueMf7I9+n1rR1ETRZS6drqX7kUjlkVV4/hI5z3wcg1D0YGXcPPcbXut7hQPLEo/hPYHrj61bj8Qyx2cVsqxJCN0W0J91T68/N361jXUsyZCtGQRu/d8jPII9agR41fPQY53djVoRqyedOnl+dGxZ+meWUDjrnb/u08WgwA7gFhyCd2PTPGPXiqVtOkzDY+0ju/IAokuHmmwAoz949icdTQBrRxo7i3jiWZ5HGzLAk8crjvnjvn0r0W0Yixtxu6RKD+AxXl9msTBZZnZTvC4CkD6k9ua9Rtp5LuztriV0kd4hlkHB2/L+fHNedmaTpxfme/kL/AHkl5E24n+I0u4460m3itbQdCl1i824K26H95J/Qe9ePTpyqSUYo+irVYUYOc9kWvDPh+TVrjz58izjPP+2fQf1r0kBIYsDCoo+gApttbxWlvHBCgSNBhQOwrP1a6cKbNAyPKvyybgB15H9Pxr6TDYeNCFlv1Pi8bjJYqpzPbojTjmilXdHIrD2NPrz+GxvHvpRbpKH34wMrj611EEE+mWE1zeXckpjjZyuflXAz9TXQjkaR594v1DzdW1OdTxFiBPw+U/qWrO0fSp9XuzbWzRqwQuWkJxgY9PrVySygudFu7m7i8x/vDLsuXJ9vdq2PAFvtuL2XrsiSME+5/wDsa0WgmdlZ2/2Sxt7bzC/lRqm8/wAWB1qU9KdQOtBJCsgLlRnj2qrqWr2Wj2bXV/cLDCvGTyWPoAOSfpSafp1zZ3d7LNcpJDIw8iKOPYEUd29W9/avKvicl+PFEU0yTGw8hY7eTb8iv8xdc9mPH1ArSMbhY9Y0jV7XWrBL2zk3wszKMjBBU4II7VkeNfEUnh/SEktjH9qnkCRBz07lsdwP61wGheMf7C8LJZWUQa8eWR3lk+5GCeMD+I4A9vrXNXd9c6revc3lxJNKf43bJ+n09hU88VLQ7I4Oo0n32X6nX23xK1ZbRozDbvPn/XMDj/vmsXUfEmr6mW+1ajO6H+BW2L+S4FYMJxJg1M5wpNc9aUpTPby6nRjhnUcfeVyF3yxbpk05F3fM3I7VATkmrKcKv0q6j5Y6HDltGNeu5T2WpHIvlsCvANEspZFXOe5p0o/d1DGNz+w5og1KN2PFQnRryoUtpE1vIVk3KxVlGQwOMVcvNc1O/tBZT391Lag5ETyEg/XP/wCqs5xtkYDvUqrtGO/ehz5ffQYahOv/ALO1ZRerNbw9baA95/xP5bkJ/Bt/1X/A9vzf0r0+w8KeHUuLfVNPs4AUB8uSBsxvnuQOCw7HqMmvGycDPpV/R9c1DRrnzrC5MRzlk6o/+8vQ/wA/eqjiakleWxpi8voqap05WlbY1/H+tPqfiE6fAWe309vJSNP+Wk5wGP6hB/wL1qzefDm9i0T7VBcma/RN72uwbX7lUbru9M8HpxmsrwVBHfeNLaS6kTIaScB2x5kpJIA9Tlice1e1hcDI4xXVJxSVjxJJxPL/AIYa75d0+lPJmCdfNt89m7gfUc16NqV2LK381mCjOOQT2yegPQAn8K8f1+3fwv49kktV2Isq3sCj5QVfJZPpuDD6VW8ReKNT8SRm2v5QlsWDJb2oKbWHQhuWLf49KqVNTaJaO21/4lWmmxm301UvL/HzHP7mI47n+I89B+debXOo6p4i1iBrqea9vZJFEaAE7fmH3VHCitvRPh7qWqyedcxnT7Q8gH/WH6Kfu/jXpNjo2leFdMmngtgkcEZllccu4UZOT3q5RhTjbqUrLc8MVtrH5mDKx/MMavWWmzajulMhSMnmRuSx9v8AGnQRvqN8DKADKxd8dh1xXTxxLFGFQYUcADtXz+Pxbo2jH4j28swCrtzn8Ke3cyf+EdtTHtE04fs5I/lisiynez1BQshDB/KbacBhuwfwrptQu/sUGV5nb/Vr/wCzfQVg6fafaLxF25Cncx/z61nhalR0pzqvQ3x1KjGvTp0F719TpQFHTvzx3rqvCvhc6my316hFmDlEP/LY/wDxP8/p1Z4X8MHVpFu7tSLFTwOhmP8A8T7/AIU/UfinbWuo/ZdNsfNs4n2NcM20Njg7FHUcdTj2rPL8vdV88lp0NM0zPlToUXr1f6I9JUBQABgU7iqGkarb6xYpdWzAo4zxV+vYcXF2Z8wRtFGzh2RSy9GxyKkpaKQDeKXilzRQA3gVS1LU7XSrJ7q6fbGvAA5LHsAO5NLqWpW+lWb3N1JtQcAAZZm7ADua8r1nWbnW73z5zsRciKEHIQf1Pqa48VilRVluejl+XzxU7vSK3f6IZrWtXOtXnnz5VF4iiByIx/Unuf6VnZFOIPtRivn5zc3zPc+0pUoUoKEFZIaTSZFP2mjbUmugzPtS59qdtNBFAXGZz2o4Pan7TjPaup8L+FDqTJe3yYs/vJGes3uf9n+f0660aUqsuWJzYnFU8PTc5sj8L+F21Rlvb1CLLqid5v8A7H+f8/R1VUUKqYUDAAHapUQIoVQABxgUtfQUMPGjG0T4nGYypiqnPPbouxxHj+bEOnwYIDO8p/AAf+zVwz/dr0Lx7aGTT7W7UZ8mXY3+63/1wPzrz9hwa61scZ0ngLU1tdXlsXbC3C7l/wB5a7G/8UaXYBg0/mSL1SIbiPr6V5zpUzQJK8IAniZLlSOp29V/X9a9C1PQrHxDaR3SHyppEDJOg5I6gMP4hUlGZZeMpdS1H7PDaGNP77fNtXnLHkcDj8665NwjG8gtjkjua8zs9Jv9L1Q3Aa3jmt3ZU8zdhuo3YHOP5/rXZaXqDw6RJeandREqfndOEXgZCj0z070m0tx2vojWubmG0gknnkWOKMZZmPAFeZeIvEkutymKLdHZKflTu/u39BR4g16fWp9v+rtEOY4s8n/ab39u1YpXHevDxmO9p7kNvzPqMsyxUrVavxdPL/gjPwpfwp2PekrzT3ROfSlGfSlzSg+9IQ3n0pefSl/EUuR6igLjcH0rD8RyNCkE6s6yRK+3GAqlyqqWJ9x07112kaRc6zd+TAMIvMkpHyoP6n2rG+JaW+iatFaW0MMiW1gkjGbnJaR8kn1O0e3Fd+EoTf7zojws3xkFTdBPV/gc5p7Pa6ktvLHMzKqy+Y2BJj+LnH97I445qqIJmiR7ONFlukW3gWQNIXY4ChsJ146jjOakkmtkup8K20BI2bzEYSZ4XnPXDY5+varUDMktvbOrM+ZLZHEpEYlK/wAQOC2Bjnkc884r0E5JnypX8WJDFpM9tFcBxHcRA4c87mxu2kA8sG/CoNDVIo7poTiIyMQM8g4RiBWdfReR4cd/KG6e4t1/14kCIgZgBx39ck8e9WfDErbryPbwsikHGcjaoP8AKuqGkS4He6PGv26JDsEewoMdFG5Tg/n1rzzxBZsJZrwvE0c91chZVlViHV2JXaOf8V5HNegaeNt2sfXKFSfXGBXA+L9TE0rw4jWXT7meD7RsVHaPgKN/fnf9AAO9TrdWLnsYZ1S6igS2Rxt80zqRENyuy7cq2N3TsOKtBIdSWWS/jeMWVquyNPl3RoQqqF+9nByepPWsgXDuEkyHfdlCSRuORyT3+nvWrY2cWoyCK6a6F6ZN775DEmVOCmGyc9OcYGcYrWUm1a5ikassoXbNLtYTHZG6MuMjGfXaevHFZF2FaYv+7OchgjHr7gmp/wC0Liyu2GyEqV+ZGhU7h36flVC6mjuQQF4bJ2Mwypz0B7ioSKYyLEfz8qd33eorRS3hvJWHm7DtB6ZXJ47dvpWErMPlycdMGrcc7hSeVXaFwpPzc02hG1eQvbL9lDqFxs6dG7Y9qhtkhS1YzJvkmuY1VSpwQoYsp92yAOlIbxisccyxy7GYBv8Aexz9RzTrO9ns7zzYCknUNkFs/QDrS1QD/s7wrFCjtC53hJJCOeeBz1HPH41VeO4jluBK6ER/IzCUInA6kd+nYE1pX2ozXbI7KrIj7QOn4+3071j6jcO6SyomCyc7VwEOcZz3pLXcCGa4LxhvJMqqfmKru/M445pbSVo5WURJj5pQkq5AXG1vy/Os4ptl8kuW4AkKev074q5amRQiSuCJFbEhGQ3sD6nhee/0zWlgJQzmchPNCqdsSI2cKT049OuO+KSOZtN1ELNtaQHIDrkfU4qu0s0cjK8YLqxVFR8jr0B9vzq1qJQeW6gspiUMwbA9sce/8qYFG5heWTdawyF5WJOTnnq38xVl4JPsUEPlkTFDFsYgY+Ykc/j1/wAKhhnneEJHPJGdwAwxAx1AH41a1eOe1uBBM7eZGkaFi4P8IYZx069O31oQDLeJtLliSFo5ZpfuzIwKEf3V79e568Y45OtcajFqke3OwkMGV+g5H5HPb6VysMuyfesjRynlXXjnHGfatO8eW+iRzBIFSMKW+8rEe3bjP5VLQgnhi3OZZjHMp27Sm0fSsqWfadgI28jPrV8Wy/Z4pIbiNnwd+18HHpVaOPZPvfawzwpXpVARqdhym5V6cjNXUnYjDDLFssR/OpJ5PKtQ0SwmKVuQnVOPustQW8imMqFxls5HGB6U1qIvxTMVMZYlXYHbnvXpXh+XzNCtyh4VnXoQPvds15XExZ9u7B7N0r1LwFDcappcVpEN85uHHtyAST7VwZhBypq3c9nJakYVm5PSx0ukaXc6vepbw8DrI5HCD1r1WwsYNOtI7a3XbGg/E+596g0jSoNIskt4hknl3PVj61oitMHhVRjd7sxzHHvEzsvhW3+YtYviYrFo0lxx5kJDJ7knGPxzW1Va8sre/h8m6iWWPIOxumRXajzjjtJ8WeU6wzI7qTjCfMR9B1rU8V6gv/CMvs3j7SyxDcpU4PJ4PsDW/b2lvapst4Y4l9EULXHeNJvtGq2NkDwimRvqxwP0Bp6X0GYF/wDutJtrfvITI34dP1P6V0PgePbplzL/AH59v5KP/iq5rVHEmoNGDxEip+PU/wA67LwnF5fh2Bv+eju//j2P6UxM2zSim0v86aJCRgsZZiFAGSSeBXj/AI68Ux62r2FiAbJHVzKRzKyngj0Udu5+lanj3xX9pkk0axk/0dDtuZFP+sb+4P8AZHf1NcAwLZ75rGVazsj38Fld6Tq1Fq1p/mQBv9HwPUinRLtSq6kjCE96te9Kr7qsup0ZYvay539lWIJPkmyOnWpZW+Qf7VMnXKBvQ1Ez7okHcVcVzpM5sRN4eVWj/Nqhp4xVlJF8sZOCKilXCIfQYpqxFgCGGKqXLJanPhnXw1VqnG7aHPIZX2gcCliOGIqQJ5aEYOcd6hTmSkmnFpG0qVSlXhUqv3mxZD+8apPNXbkn8KSRNw9xUSIXOM4NSuWUdeg26+FxEo01fm2HFzIwAH0FPjbhvpSqoXgDFMf5W3DvQpRk+VBWw1agvrM3d9fmOi4r0nwj47Znj07WZsk4WK7c/ksh/k35+teax96Uvgcc0nzKfumlOOHngkqzs9bdz0D4pWRFxpmoBTj95aPx6/Ov8mFc/wCFNcg0XUFM2nwz+fKifaC214wSBxx0/KsO6vb2+8r7TczzrAmyJXckRqOyj/Jqt5wxlXIORg+h7V3wu7NfM8JxafkfRiAbcisbxckjeD9ZWJtr/ZHIIOOnJ/QGszwX4ui1y0W1uWVNQiX51z/rB/eH+Haul1C3W8065tm5WaJkI+q1EouMtTPqfPdncT2c/mxoG6jDDtmtCTXb5lxFBHEf72Cx/XisWK6Pkx5ODtqxBeQqw86MuPZiK4q2HpTlzyjdnfh8VXgvZxlZMdHFdXc5+/JIx+ZmOfzr0jwV4PN4iy3GRZq2XboZ29B7ep/AVzOja5oC3cSaiXtrYcssaZL+2R0Hqa9a0fxp4c1GVbLS7pZHjjyIY0I2qOwH9BWaw9WsrzhaK2Xc6KlWGGj+6lzTe77ehtX0RXSbiK3UIRA6oF4A+U4xXzBArrZwnBGIl/8AQRXvHxN1p9K8FzCFyk99ItojDgqG5c/98Bq4zwN4Hk1GHTdevJk+xsTKlmY87lGQmTnp3xj0rvpxtG/Q8rcofDfxDNputLp8zkW1wflz/C2Ole6K2UDe1fNfiCyl8KeLrmziYhbeZZbcnnMbfMn1x93/AIDV3xF8QNX8RqlnG721lwggiJ3TH/aI68/wjj6121cN7VpxZVrnsWp+P/D2lztBJfLLMvVYRvx9SOKwG+LmlMFMdrdAMAcOoyPrj0rg9L+HXiPUYVkkjis4zyPPb5v++RXP61ZPomrz6bJcpNLBgO0akDJGcc1UcLQXW9gsj2ix+JukXlykJd4mboZFwM+ma6a51u2stPe9ncCJeBjkuewHqTXzKs+7Iya6vT9XuL23hinmLiGJUjXoFAG3p6/LyeteXmFSFGk5QWp14HCrE1VBuyOh1nWLvWr03Fx8qLkRRA8Rr/U+prOw2aTzKA5r5Kc5TfNJ6n3FKjGlBQgrJC4NJhvWjfgd6XzB6VBrqJhscGja1O81doGOc0nmCjUFcTa1IVbFSCZVHK8V1vhfw0L3Zf6hCRa/eihYf6z/AGiP7voO/wBK1o0ZVZcsUcuKxcMNDnn/AMOReFfCT6gUv9RT/RBzFC3/AC19z/s+3f6dfRRGQABgAdqBIqjAU4HtR5w/umvoqFCNGNonxOLxdTFVOefyXYXy2o8tqPN9jSeb7GtjlMrxHEk+jT2TOBNcqUgU/wAUiguAP++f0ryvOQGHQ817Bf3CwWNxcFf9TE75PbCmvHl4RQTzgVURMlspxb3kUjfcV9r/AO6eDXoHh+a8k0GewtHiF1aSGJTLnGw8qeP88V5u5+bHrXY+Er8rqlrIzcXURt5CeP3icj8xmk9GUtiGXQteutRe3eM8MN0wbEfPOcnk1b8X6Qllo9j5MxSC3ynllSQ7n+I+h68mu8rM1+w/tHRLq2Ay5Tcn+8OR+orLEx9pScTowdX2VeM/M8jKnNJsNOWUYHHWl8z2r5U+9TYzYaNppxf2pu40D1F2ml2GkzS7jQLUNnrWronh+51q42oTHbqf3k2OnsPVv5VL4f8AD0+tzB33R2SnDy929l/x7V6ba2kNlbpBbxLHEgwFUV6ODwTqe/Pb8zxMyzT2P7qk/e/Ii0/T7fTbRLa2jCRr6dz6k9zXhPxUuLmTx5f/AGa6khaC2tIsJ6OWyWHcc9PevoHGBXzt8SL2f/hMtamtERGSWOJpZF2+aqomY93H94cfr0r2anuxsj5WTcnd7mBcFp5TIZGktm2shCFgzKcOFz3+X04J702/XyJmiLzQxeWzBhh3k3sM4IK7MMVG0c06GBpb6W7kSf8AdEl7cL5UiyY+d/m64GCdmeueaWGe31SaP5LeOFVYm6l6h/lynOeTn8q5rtMm5F4gs4tO0tUead52vFQb8EOqo25sj0ZsfjjtRoTeVc3iPt8p2jUv6HIAxUHiGKM2mnoTDLOJTGWi+URpgnbj3JDVP4bBle8G772FGepYbGrop607mkdzvNPJF+mRyEcn8xXn2q3cmj+NdTlntFVUuZGj324bzsHK/wCsB+Vj1K49q7/TZc3cuEBEabmOf73QD8B+teceJhHH4w1b7TP5sou5N3mFioXdlV4PPXBHbtSRUyqPKuElut2mKfNXMckm3c33t21eQBznGB03c1UsriaScRRwksWYyFjznHUsf94/nxUTyW0ckaRgqXVgAu75mzwMdieOP9rmoLtXitd8EjnAAkdQcDcOAfTJBHP92mldGR0l1eSXivewymKVNzSjzNxOe/3fQnPNY0NjJcxkxw3DlcDKRZ5P3fwrpYLG4sRcRX9vFe6UZhA11Cx8otwQpPVSc8bu/Ga6+bw7bN4dub6K/wBt6mZpIzkrNGi8ZU9Dk5yMdD9azdRQNVByZ4/LEFmZA54zzgjB+lKk3lxqd5y5xxz+fpV2fTpUvJkiURqrZCnP3T0p+ppujjLIv2kqIvNyRuX3HRueK2uZEK3Jik3o4IxwBg1NFfBJ0l24O7dgr1HtWfHaTC68g4jmzhkkGOnarkOnsbx7eSaCNfKaSLe/yN+PahoC7PdhpDNb3jF5VYtF5eCqn+Hd3qHfdRoUMXm2nl7smPzNrHoOnB/wqK2i8pv9IjkjOzq3OQe47Hsanh3hJI4rmdbvkbt3yADncQOc/T1qVoBnTWNxaL5zRFYpAVG8YP8Ate/1+tW9N0iW5eKQzPa2TSL591KuUUZxnGeSPz9KUXBmhD3Mgf5NuCvOMYqNpVhJkifgKPlBOOmfxp3Arm1ex1icOFlhjc9AVWT06889fWp5kQqkULSOhXam7qPpV7WkUaxPJE7iOZVcEtuyCPX+76VXNsrZRZS21M7k759e4qrgRww+d5ccXzyKSzHpuP8A+rinXVpMzlpCqp3PfjrxT9MPl3SyxxtNGrEhgPukdzjpVrWLqZICs1vOob5t2Tgg5wBkA461PURyl1IokDRIoGOwq/aalF9leC7X5GztkXO7/dPPSklS1dkEabgVVclMlfpzzVIiQFXhj49ccVdhF6HTZbhwIUY7sFP92knszCzKjmTbxgA4YevNLaT3Vq/7re9xOCjAAk7fTHv/ACpryTJdKZYXU8YEgJBPv60rATRtutnmk2GTPy5HzA/4VDtjWQDBU5xye5p8ty86jIiRYxyEXbn8PWpoIyFLJ8wPDkclfwzzTsA6ON2YfdbHHHFe9/A2OFdAvxt/frcAlj97aVGB+hrxeCykt7oSLPE6ryoZT93HB/PFewfBVWS41cYjUOsTFY+mRuFVYpNrY9gopMCjAoAWiiigBDXm9zOl/wCKb26PEcb7Ac9k4z/Ou91S7Fhpd1dkgCGJn/HHH615ajG20S4dv9ZIPLz6s3X+tNDiVRKZ5GmPJkYt+dekaGgi0KwRenkK358/1rzO3Ydew5r0XwvKZ/CejytwXs4z+lANaXNiuW8ceIjomk+RbPtvrsMsRHWNf4n/AKD3PtXTMyopd2Coo3MxPAA6mvB/EWtPrutXF8xPlv8AJCp/hiH3R+PX8airPliejlWE+sVry2juZytxXd+AtG0rU9N1CbUbC3uXhmXaZU3FV2ZOPyrgFOK9K+GB36dq8f8Atp+qMKxofEfQ5peOGbTtt+Z51JHaSeIzHak/YJrwCIqCpETsMYz0IB/Sr+vaNPoGrS2M7BwB5kUoGBJGc4OOx4wR6isMsyLE6nDKikH3A4r0r4iql94f0TVQAGlyh/3ZIw38wa6atpwTtqeRgaroYl0ujv8A8AzpfCVnZ+ALjVNQlKX9xCHtsscRseY1Cj7zEDnPr7ZrgQTuX3IFeqfFBjDaaNarxGGkbHuqIo/RjXlU+VVyBkr8wFa0ZK/s7aHHiuacY4ibvdsvOvDIwwwyCD2IqGJsZU9BzWrroj/t++aE5ikkEyfR0V//AGasiUYbjjNc8dZODPTxl6Xs8TA9M8a+DrCx0RtR0m1WA2p3TohJDxn+LknlTg/TNcHottFda5aQToHieTDIc8j8K2rrx7cXnhK40yRM3EsZiMmegyvP4jdWT4UHm+I7M44DMfyFehDD8kjxpV6k0lJ3tewurWottevrO1iO2O6eKGMHJxnCjn64qC4tX0TxO9lfwRzLazKJUIJV1IB49eDXS20CzfFkxOMj+1Wcj/dG4fyqp8Rdp8YTEADFtAGx3O0n+v6VyUHGDk2j0cwqSappdFc1Ne8MaTPoja34bkzDGpeWAOWUqPvFd3zKy919M9K5VIbGTQ9ReWdY9QjaJrRSx/eLk71wODxjrXSfDe/B1WfSpmBiuk3YPcj5W/8AHSPyrjwhjXyifu5Q/hx/SomlTnGaW5rhZyxWHnSm27bPy7CWVuLu8SB7uK1jYEtNL91frWhqWnWdh9nW21aDUXdWMrQDCxkEbR/P8qyUO2QEVZzn3q8RWtBRijLK8NTqtynrbp0EqTSBGmsQxTIJLaf93Kh6Mp9ajpgcpKjr95G3D60YCs4TaexvnFOMYxaNPXdDvvC19FcwzSG1Zs2t4vVT/cb/AGvfow/KvQfBnjiLWRHZXzJHfrxjoJP9of4Vf0V7TxD4aWG4RJ4JI/LkRxkH2ry7xL4ZufC2sQ+W8jWckqm2nJ5HzD5GP97+Y/GvSfJNOMt0fPxim7MyFEVvcJE+PkbaePfFdVB4StbvTLzVZbxdN0+1jLSXTrvTcOwXufp3IHWuc1yFV1O4/drnzGxj61e1jXLrV/DGmaUwS3tLYNJIsTH5mXhMj2+ZsepBrzKWGqVOWs3ZLddzunOFJTpJXd9+xh22LmdYoUlmnkJ2RpHudv8AgK55/lXX+EfDurzeJrOVrK5s0t2LtJNHt7fdAPrXZfD/AMHt4ZsPPmiiku7tQ7yryVUjhAfQfqa7uOEO24Jhu9dlSdpX3OF6M81+Mc3l2uhxu3ymSd8f7QQAf+hGu78Lxw23g/ScOFjSxiO5jgY2Ak/zrjfjHo8l3oWn3QOEtbo+Y2M4DrtH4bgK4OXxLqz+Do/DImSO1RTG0yk+ZJF2jPoPXHUcVMYc8Er2EN8b+KLPxL4mkvLUKLSCP7PC+OZVBJ3n2JPHtz3rqvhNp+nXcl1evEj3Ub7Ududi4/h9Pc1xln4Puz4T1LX5LZPs0SL5BZsFvnG5h7VtfDOSez1026FQkg+cemK73SiqclB9LFWPeFSNVxkYryr4h+Bo2uJ9dtLuKJpCPNil6SN0G3/a9v5V6LqeoWei6VJf38xjhjA6DLMT0VR3JryTU/EjeI9RDyiWNFyIYcgiMf1Y9z/QV4VXE1MPFzgrnTgsPGvVUJuy/rY5ePR5QPm2j8a17K1S2AXIJUZz65JNW/s4Pc0fZxnO414FfG1K6tNn12Hy6hQfNFajw6jvR5i+tN8hfU0v2df7xrj0PQF3r6ijevrSeQvrQYB60tAF3pjik3rg5Io8lcc8V2PhTwatyU1HUUPkjDQwOPv+jMPT0H51tRoyqy5YnLisXTw1Pnn/AMOJ4W8Lrd+XqOpJi3+9DCw/1nozD+76DvXoPmx4x6Un2WPOSTz70fZY/f8AOvoaFCNGPLE+IxeKqYmpzz+XkL5yUnnpR9kj/wBqj7JH71scweegoNwlL9kj9/zpPskXv+dFgM7XpFk8O6kqnBNtJ/6DXlLKc5r17ULBJ9NuoUzukhdBn1INeMXmoRWVos82fmGFQdXb0H+PamnZNsai5OyFupordEeZ1QM4QZ/iJ4wPzrQ0u4eMypGSJY2W4i/3l6/p/KvPL67nv5Xllf5yMKF6IPQV11hqA/0S/HO4K5A75+8P51lGpzt2OipQ9mlc90s7hLyzhuYvuSoHX8RU/UGua8G3QawnsScm1k+T/rm3K/1rpRWpztWZ5F4ishp2v3UGMIzeag/2W5/nkVl7hXd+P9ODx2uoKvKHynPseR+v864by1r5nGU/Z1mj7jLa6rYaLe60fyG7hS7xR5YpTGtcx3XQm8Z9xXQ+G/DcmsOLm4zHYg9ehl9h7e/5VZ8NeEhflL2/Qi16pGeDL7n/AGf5/Tr6GkaxoFVQqqMAAYAFephMDze/U27Hz+ZZqo3pUHr1f+Q2CKG2hWKFFREGFVRgAVJuHrS4HpRgele0lbY+YvfcCeK+bfEd7Z3Hi3Xri6Z9vn3KNsQurIrbQfROV2k4Oa+kjXyPqTeXHfeZAIZJpZJHlkVnCMZGK7sdj+OOKzqq6SAV7uzj1hL2C5fEbFIoypDqewLNuHBzjHsDWzcLFeTW8tqotfJmAlDjJTdk49FJPr2xXG3Np/pD28NyJnZVcHGwZ27myTgYXn61ZsJo5fNjs3KsELGXO5mAAP3FGGAPOfSsZU72aZKLeuDHkSJ55t2djE0yIpJ2rn5U4A7/AONaPhaYpFc/ufM8q4TA3bS4ZV3gn/gKkfSuZnmhezgWGTf5ZdpZBzmQ7c4/2eenua1fDjut1OnVGVc+m4MuD/3zkVulaFjSO56LpAkUtJIDvdmdlzkLnGB+WK828UC3Hi/WhMrFmvJWDxy56/dz9DnI68V6VYsY2+Z2+6d5/vf735VxGp2GnXHjHXri4njN5Dds8enSIGFyMDIHI57/AJ4yazT5dWXNXRkHVLGPw61iNMDsWG+5zucMy/MTuGBkhQMY+7VXT7WzmhgiQhJ3JBkbczSN0Cqo/DB29zWnqTptEdr4eFkszlYEO5Zih7yyZ4Hbb0rmh/oyTRKu3nEoZMNkHIwP4cf1OacY6GTNeW4mt8wRyyi1uCsrxhyA7AEAkeoya6bRNd8y1NndBiNnykM3PY/oa5ry4ZLYk78RDeOei96fYPKz+TbmQSKWwFfb5keMsnv0z705wTKuzpr+wtdQjklgnSKUcoZDy+0gE/KOR6ema5aKGF3aO7jYq3ytlmUL+f8A9anxyS2txHdw7CsvzFDnBxwQffj9aY181wUXOIVDBU2jPtuP8Rxxn2pRi0I0bzw5bqrJbXk4n2boPNYYLryEJ6ncMbT2IIrKto1eCTYAbm3k82RTw0inhlHbjA46g11VlIt9pP75c3FqoEh/vRcYb6jP6VS1O1ZBNd2yAMp/eKP4gB8/49GHqN1O76jsZV1HPOw/essmw5AXh2weeO7DIwfSkt50kT/Uxr8uUDHATPX8KqyXG+NCfqrDqKsQWbBEu3YGJVZ2L5bO3GRgd8sB+NPoSU7gTNqJgWMeXdtuVGwoJDdQegqGWGZS8Ls0bI3IHQ5/pjpWjf228Qwm6US2r/u50Viowc/L7fdb8e1Ovbgap9juwrb449lxPE+/5gwAbHHqOvXIwR2Y7EbSxXUNsy+bJcMkaMAuAeMce/A/nVUzpHL8wIOSuParF5bPZzxtLIJY5wZY5omLJKucE4PKn2PSobcpLcvG2398pUl13FTnO4e/H607CLNubWGYz2Mknmom7dkgr8vr9e1Ymp3U804laZmZ87l7A+3t/Ktu005v7O1O6SXdDbxpiQcCQM+3gd6zms7W4kdRKQ4XOGbk89VH9KEBnx3bpOkq7cKQdjZxx+vUdqtSOsV/JIw+bdkAnO1v64pX03ayrFHJKxQkgr029Tx+FMiiDHJ5IGCgHP1qybGppl3PFqUF9Gd8ybni3sflwDnn1wDU95dRnZvicArgbgT9Kzora5EkMg85SMug429e/pV8R77d7ljJv8w/JJn5V3cnnOevT+dKwzHljKAkN8vqFPNWbU7lLJkNuGDWvJNNd6WISkX38koCGcL0UnOP8+1V7FLhS8tiYULFciWPI2+x6+tFgN3TNEvbqGFEuIE8o7sSkgIemdwHTBr0j4OyOniDVbV1I2wD5h0JV8HB79a8cuB9njaGc5WRt/yZ4PbHPQf1rvfAuv3OgyLqFpardLJE0bxqeRz32/d5HfrVXstRo+iqK89tfiS9zGd+m+TIP+WbSjLcZ4NPT4kq5wLDPIGfNUAHrjOeOPXFZe1j3KUGzv6Wsbw/rJ1y0lufKEaJJ5a4OckDn+dbJq07knL+N7ry9GjtlPzXEqrj/ZX5j/IVwesTNDb2lshX5t0jBlDD0HX8a6fxhcefr1tag5WCLcR6Mx/wUfnXGaoXm1aUjlIwsSj6Dn9SabLjoiCVzFpt5JgAiF8BRt528V6j4aXy/DWmx/8APO3WP/vnivMZwkdrtuUzG+MpuI3DcPTnHbNemeGbmK88PWs8MXlRsXATeXxhyCMnk1HLLnv0saOcPYuPW/4GV8Q9V/s/wxJbxtia+b7OOf4ern8uPxrxomu2+J2oG48RQWYOUtIASMfxvyf0C1xFYVZXkfV5RQ9nhk+r1FFek/Co5XVl94f/AGcV5qK6Twp4pPhlrtvsYuRcKgI8zZt2k+x9aVKSjK7NcxpSq4dwhvp+Zy0vy7EOQQNuCMGuu1zVlvfBHhmy3fvDE0zr/dABjX+v5VjeKdV/4SHV/wC0TAYSY1j2GTfwPfFUEUhVHoMV21pQjStF3ueNg6EqmMcqitY9J8cK2s+DNJ1qBS6xYklwM7FdArE/R1GfSvLJWUs2CCD3BzXY+GvGV54fha1aL7TZsxYRl9rIT12n39DxXO63JDfazc3ttC8MU7bvLfbwfw4pUXCTcr6mWMw9anT9k4+6m2n6kfmGQI7dfLRf++VC/wBKY4DClUYApGXcjL6giuaU7VebzPchRVXDKEuqKaK291Ixg9DXWeA7Bn1sT84jB6e46Vn6tdWN9JFNa2UlvLtAlLOpV/lA4x05FW/DOuLoN+8ssDzW8ibXRCAwPZhn8fzr1PrdOM21sz53+zayTstTSuLpdP8AinLcucJHqa7ieykKp/Rql+J1i9v4hjuSMJcW6qrdi8ZYMPrtKmud1vUI9V1y+v44nSO5k3hHwSPlA5x9K6mx8c2tzpA03xBZvdKqgCZUWTdjoWVv4h6ivPhOLbTdj0sVhanJCcY392zRy3hK9+weLdPuCQERm35/u7ST/KqIYv8AOerZY/ic/wBaWeK0jvZXsfOEDZCCUAMFPbgmkrXGTh7sYO9h5XhpU4y51qysc+dt6c1aU8CnTJatDA6eaLlFKyZUbW+Y4wc+ntUY6YpYpwcY8rFl9CdGtK+w8moHJ3GpSaaRk9KxoOz1NM3pSqQjy9GehfDS8Rhd2fSRcSj/AGlPB/Jv/Qq7vUdNttUtWtbuJZInK5B7EHII/KvHPCuojS/EtjOx2xNJ5MvP8D/KfyO0/hXt+DnDcMOtdiqNu58/iqTpSi+6PnzxAr/2/e7flXzmCqewycVnbJTG6BxllIrtNVt9B8S6pJeWN9LbTTfvGhdVIzjPoO3asmbRLWGSeH+0yJoVDfPavsA/3h/SuurXg0uV6GLfM7nuXhXUrfWPDtjdRBcGFVZf7rKMMv4GtwIB0FeH+GtQv/COoFPtkMtrKBJJbpFMWORkMPlxnH513A+KGimREU+YXJAaKQOuQu4gsOmB61w1HG+jJsdhf2NvqFjNaXUKTQTKUeNxkMPSvMo/hLAmsK76hcSaeG3C3dRu/wB0t3H4ZroF+JujtH5gimK88gZ/Hp0pW+JOkqSGhnUjqGUjH144op1+TZhqTePY4bL4barFFGiRi3EaIowASwAAryTwLdpaeIxcTtiKNWZj6fh3J6AV1njrxhZeI9Jj0q0uUtt0yySmVHfcq/MANgPfaa5nR7XRNPkcahqdx5yjLLFDs5ztwPMHJzxiuiGJVPDySd5O5rRUHJe02NvxRqd34mWTe3lRgYt4s8JjoT6k9zXHadBdLqCB1ZWRvmB7V2rS+G1cqZtVzz0RG3fTCnNNWTw06eas+qbM7d21Ov8A3zXgRWJUZKVnfzPanWwDnCUbrl8ikA2OtL83rV0XPhvAzNqvTdyqDj/vmpAdAY4B1Qn/AIAP/Za4vqNXy+89P+2sN5/cZuG9aMH1FaYGhf8AUT/F0/8Aiaap0BmwDqX1LoAf/HKX1Gr5B/bWG8/uM3n1pCSM5YCtCSfw/GSCuqMR2V4yT/47S2ms+GrG7S5NnqNyIznbKVKA9cnAHSqWBqN62JlneHSbV/uOk8KeEmnMeo6mmIvvQwMPvejMPT0FegAEVwH/AAtTTvlH2OYEgHB7Z9aa/wAV9OV9hs5twx3Hr/nivYo0qdGPLE+YxWJqYmpzz/4Y9CwfWk2t/erz6P4q2ckqINNulDnCu2Av55qvJ8YNPjjaT+zbxgDhsL0Odv8AOt7nNY9J2t/epNp/vVxDfEe1Fh9q8u33Yz9m+0jzwMZ3FPT8agt/iZDci522Lg2673RmyxUHBZQM7sd8c0XEkd9tb+9QVb+9XDJ8Q0l024vo7WMpAcPH5h8wdMfL7545qGL4lLPIyR2Odp+Ztxwo7k+1LmQ7M6vW9bttDs/OnfdI3EcKnDOfT6epr5x1Saa51K4eZgWWRkUL91V3HAUeldnfeO9C1bULi4vNMvmkTKkm82rgHGEXPArn57vw1qV9JOLPU4DMQ23zwyjjGc59q5Zxqyk72SPRoVcNSprfm66fkYQroNCl36a8PeGVl4/ut8w/mauv4f0yKzjuWtr9VflQ04z0zzUcFnDbxu9grx+aBkTuZM4J6dMHrV0qbiyMRXhUjZHc+DtQMOo2Ts3yzBrST/eHKn+X516b2rw/RvMi+0wq5MrYnRj13r/9bH5V7Pp92l9p9vdp92aMP9M10I4pEOsWA1LSbm0PBkQhT6HsfzxXj5BBIYYYHBHoe9e3GvIvG+u6J4X8RyWl5oFxO86C5SWK5Kh9xOeM9mB/OuDG4SVezjuj1cszCOF5oz2fYzeAM5rsfDPhIz7L/U0Ii4aO3Yct7t7e351wdt8RvDkEwni8N3LPF8677rcPrgnHFbkfxyt5WVI9EuGduAolTI9zzxWWGy/llzVNTfHZy6sOSjp3Z6+AAMDilrzD/hbaggHSpM4ycOpx+tRn4wIGI/smTjnIkX/GvY9lO2x8/wA6PU6K8lm+NUcEzRtpEuVzz5q8/rTLX40vetKLbw/I/lKHctdRRgKf4vmPTjk9qThJdBp3PV7mXyLWWXA+RC3PsM18mwwxJHvvEgms7lAHlLNGzPtBA5IGFPYc13WrfHq5uNMnistCCGaFlWa4uB8pZeu1RzjPTIry1/FGoC2SGOxskjVFTKwdcDBPUjJ+nYYxzUSp824M37iQzwRzQvbz3EZCGJQWUr2aPcccEdOQT9Ky9asX1MJaxXDSyq29EWM7ULH5zK/AzxxtBzxVSHxRJHG6f2bb4dcMBbx4YZzggAccn8zUP/CTpF8selRQgAjNuzxE5/vDcwP9O1SqPLqmIk1Kw/st4oPIeI+SrB3k3efzguF/hGeAPatTRo3g1hG+Yx7tjqOjfLyOffFY134hsdRKveWt+XSPy42W8DY5zyGU5+mRXReH7iK6ha7Dg7mY7AV+UhehPX0/yabuo6lx3O6sAWzuYFisgbt/nrXJas9ivj+/KwT/AGlGy0yoX/ebM4VO/UD5uOp9q6aK6CsRwHUFlUEA4xjp9cc1wPiu8hh8W6n+5YTLe7y2T8w2r8uPTgHPWsUrmk9hl/qUv9kvMjFYZS0QdduGDdV6ZzwRn25rmY3Qv5ZkdW3ZVjyAuO/rV7ULo3CozfIATtTzNwjUknCj8f51nxxqZC+Fc+v9a0hGyMWbAmmhcMsrjB7MRn2/EVMWa3vPtERITfujOedpH/16r7g6bSOQevtTo8kFC3APr0qxmk0Xm2ySrhkkdun8Ljr+YOfx9qzzCpPmKNr9G9/erumzOJGsyCUkdSBjnePu4+vI/GomKpcMGB8rcQcdcdakZq6XfoI4Zhnz41MMsbf8tIz0Pvjp+VXNQJtbhgrkwuI9x9v4H/mp/CuatAvnrHMpZd2PkbaT9D+tbFzDqbWpXyor2OODdE6HZI0ZPBIPDYPXFS0NFOaCO3kkSVB8n763wOPvfMv+6R+RHvTbB47kzWG11jYl4A3VXHRcfTj6CpP9fGIX+eWF+FPB+625cdRkEfiKpR3xgdUmJdB9yQnkN1BP5CmGxYkZoZmt5UdEZ8kSDhWB27h6AZINRW1rHDdT5yE2HeoPyr04b1AOQfwI6VcnVLhEd5oy7RvKFJYBdzZwP6/jxUsuy3totQjcmZ1JVY3Vkfbndu/u5UfdPWp5irdSK/uLdrXS4FiXda7lkJ6sW9fyFZXlSNFdKseZFIlaQ9oxzwP5095Wu7lZN7N5h3Hf3J5/+tU1vc+TdHzlbKnh+hCH1B6jH9arYm12aV1DNo2iavpM6ESSTwNnPAHDY9jx+lcwHFzchbYSIFzgsQWHHzE+tbupWzXV5MkMkjxRR+bLLI+7A2jr+HQdea52wtvOjmdtwhT5nbGASc7RnsSatEmtaNOLhpYyjL/qzvXO5cfN+dV9MVPPdsZITIJ6/wCeatwEwWqK/JU4bHf/AOv/AIVV05PK1ie2dgQFdVYdDzkH/PrTF1LwgE8aDcy44OD94elTzrts3XPAQjrUds2Y09eRU8nzRN6kf0oBlOwlItVY9UmP5cGtUIEdlAA+n6ViWBzBMuejr1/3a2kbLKTQD2INQjR7KZtgLCFmU+lXPC3iSy0XTZQ6Sfad5dgOVlGPlXrx+XvUFym63dOxicfpXLK2YFIznFEoqSsxHS3niG+vX3F1jReVSMdMdDk8/wD6qUa1ePEN0zcDLknqAc/0FYSNnp3Fbnh/TW1jV9KsAM/abhImx/dLcn8BupxhHaxSbPprwZay2vhHTRcBfPlhE0u0Y+Z/mPH410FMRQiKqjCgYA9qo63e/wBn6JeXQOGSM7P948D9SKXoBwE1yLvWL+/JyhlbaevyrwP0WsJrq1tjFJd3EcT3EmyIOGbzJDzjC849T7+9XnH2bRtmeZMJ/j+gNcZr87p4gsx5txEtvbAAxRq4+dt0gYE8ggIM9vwpSbSujToal7DcQIZLm6iunlZMyQg4AHONv8I9BXoPw+uEbwdaIzqHM0q7Secls9Pxry3w2YpdQllhRfLZJFH7zcVAIwuf4sdPzr13wdj+zHGBkXJ5/BaUJXWpDPJvFVybzxXqs+cg3LIPovyj/wBBrI/n9Knun867uJT/ABzSP+bsa9L+FcIXStSn7vcqmfYJ/wDXrmjHnlY+0rYj6phVO17WR5jHFvPy9foa0INMZsbsjPpXuWq6xY6Fpz32oSbYV4AA3M7dlUdzXlev+P8AVtXZ47GX+y7I94z+9I937f8AAfzNX7DXRnkvPZP7H4mXNoEgh3EBTjI8xgufzq9B4cd4Qwi38f8ALNgxH5Vwl3qOkW0heUPcztyzM27J9STVdPF8cMube2hiHtnP51Toyatc5/7ValzKJ6P/AMIxIX2+Vg+lVH8Nv5uGGB1xu/rXPWXxO1G2QKl5cKo6DzMj8myK6PTvi5OpAuIIbkE/8tIV/muP5VCo1I7DebOWjQsXhiZvl2g47lgKV/C9wNwWHOOflkWur0/4mabezRI+lLsf77RSqzJ7lCASPpmu7g+wXMKSwiCSJ13KyqMEGsJ80Xqi45tNKyR4dJ4euFH+qbvkcVlT2UkLYbr6Gvon7LaN/wAu8J/4AP8ACopNJ0ubmXT7Rz/tQqf6URnc0jnFt4nzkRRivoZvDmhMcNo2nn62yf4VWl8K6CGx/YWmkf8AXqn+Fbwp8+zNf7bgt4P7zwIg+lLtb+6fyr3M+FPD3fQdM/8AAZaYfCHhw/8AMB078IBWn1d9xf25R/lZ4cUb+6fyo2t/dP5V7afBnho/8wKw/CM/400+CvDR/wCYJZj6Bh/7NR9Xl3D+26G/K/wPFNp9D+VIeK9qPgbwyf8AmDW34M4/9mrK8Q+CdDg8P6jPZackN1DbvLG6SPkMoz0LY6A0KjJalf2vQqtQcXqeU43BgOCRjNe+aNe/2jo9hfk8zwJIx98c/qDXgeecjp2r2H4ez+f4Ot4+pglmi/DcWH/odOjLVozziilRjJdH+Zy/hu0YWUDXVtMHkQRiRYHXhh2bGMHuP/r1VWLy9eWVflhmTlSqupYDywAB0AVUz77hXWaR4ks5INNge+aOSGyjt1AViWYp80i44+XG0Z77q5mbMDWsU0kqkxhXjTYF2DsW/wB4L8vf5snmrqSvGx8/Fe9c527gu3mnvHiiWJD5e4rtVcHACj0z29Sayr+1kkheEJvVY2EUSYCID9989+dvXPGBW/OzS3Wm2d5Ky2sSNdTYBxJ2AC9zyeP9qoo3Oo2t/q8KeXaXjApGD91AuGP+zubnsuT3qJJRSZSbbYs0A+zkMcxYXzCG4wAeeMY5wMelF6Ra3BhaGRcdnHMa+v459+tTGaSVrM+ZJJ5iKnJGCwOTuHbr+hp9yii8a6ubpWWS4kMUjRlvm4A+XjnPAz7Vhy3YFFLaOSTy3jIYfe53BV4IPIwM46/403U4Vs9LuisksjRbHMSuQGjHzM2M87cD/Iq/9m/tG7ayjEdxMyrFbj75kf5dygk4+XnOV456Vm6kEi0uW6ht91tbNt3wTcpG3EoZTwytnp1XaDVpWdhoq2kk1zcyzCJIZfmRFEh+/wDed2x8vyr6Z6eprS06cQ28mclTtYiTII47n8c8dcmsvSiLKz0vznkd502sx5GSvAyDhRwgwevPpWpEHRm2qFjgc7ijcA4PB54POPxpySYD/tX74J5UJYDPyoMoO2SBxkep5q3bblyQAwboc8YyfzrNH7u1RkiuH3H7zKM5H97nlun5irMU3lqu21uSo4XCDp68n61m0M0D8ikNjIOfu55qCYZiCgKXJyFkGR9fpTJJZDG3+izg45yF5x7bqrTXbxxGXypQp4yWQZ/8epWuNsfMTcXAUL82cjjBA9h/n0qBtu9kiOWUbjxvB9z0/WoY7p3iGE45GfNVQPTPWpY7OVwjmOLK7SrmX05Gfl55NJK25IyORvPaP5GOze5fAC845U+p7H0qO+WZNPkgjuAlxswoEYHPTgdsj69+RVuNJRFIJ3it/K2sZSC3TttGRjv+uOKjmnvEtXMbxnIZdhtDv28HGAcBSD1xxWgyvbWlmq2TxCISSfJ8hbcCFO7cw4BJxx1HtikuTskaKSbzbUzoJpN7KIhwBuGeevA9etE0lzA8I+zRusysJRcEHzio3bydnQepYntSadA2v6dqJtxDbu06qsbMwQKm5iFOCdvynHQ5bmqbS1bEadj5CapHbKC8kieT5sqqfl2kh9vrwM59etZuqxXdjHHrciy2szv+7kJ2sx3c8Dsffb1rttJttJWyM6RRyXMnlidnCicPg7UAzx/FgDrzXCeJbKAXVwX1C4lZ5iwjlXCElmLISODtHGBzWdGv7ZyVmraFyhypM6PV/K0vSzcWUaS2NxJbuke7727IyzdcjkDn8K5q61VbWdYrd/JuFYPJlh91hyGPH8qt3MM0HhPTzMwlRroPGrBTtG1iM5+99TW3BeaRPbWyPZ2kl0IzHGBbqdkjcFun3ucDnofy0VlLUVrnC/6NDY3l/Km5ZG3xLvVQo8wDjcCMYPp06VLZSrPbrIsaGWRi0Zz/AKtQQOMDA/CtXQ5TZzWsaJAwLGFRcJuXkFSCOvrVe0sRDPcvFvUx5Y/Pu/jHKj0J6CqlJbE8vU7a83x6bbWxijWNnYIFXBG0HqfzrJjSdUG+Am3KKqsOSzHORtp/ia+fdF5G5mtpg7AN+JU+nFa1i3+jbF6I7Lx9eP0pw7EyIo7eXTVtrqaZZ3Rh5hQfw+/4HFei+DbofZrnT2bJt5N0f+43P881xDRebE8R6ONtaHhbUTbajYzSHaJM2c2ex/h/Xb+tWG6PUK8u+NPh59T0TT9St4meezuPKbauSY5cL+jBDXqNZuvaTDrug32lTnEd3C0W7+6SOG/A4P4UyT5RsbWBdauLe8ucLF5kOMffI/Hj6d63LHTLfSopLh0ZpDyqyYLKOy/X+VZ9taXGm3RL2u+8jcqIiuAkuShJ9htLce1a8pEqRtcTxxhecKvLH1xnj6c1tC0NepnO8tCB32IzStlz8znPf/PFVxJyq/xyN+v/AOqnPNZNg7ZJV67ieP6VCbiGclI7c8jGFIy3/juacpuS1YRjrZIyNRuA93K6tld7bT2xntT7PVL+xt7q3tbmWCG7Ty7hYyAJV9Cce5/OmyW808wc2TohOAwk7np1FTw6dOq7mtZEHcGWNv6iolWj3OyGBry+FECBowrxuyMOhXjFRSxKylmTLY61qsunwRuLm6ZZdmVRLfcucnILbs9AMEL168c1ny3FsufLk5HIyppxakrowq05Upcs9GRvpRiMKCLe7KcLGCSKik09VXLxsh98r/OtJdEa+0hNSbVdPiw7AwSXTRTtjgtyu0DP+1Wd5fk/KmpXUZHG37Sf5UGZXWzgC7tmcdc96v6RerbXeM4ikR1OB0bHDfU7dv5VFCpmO1LjzzjONqlgPXIxTGtnjmV44zuBBDbuhB+tRKzLipbpHu3gLwfpGu+F7PVtQiuJbmcNuIuHVQN33QqkcfKOtee/GHSdO0jxpDHaW0Ucd1YxzSHc3MnmMpPXqQFH4dK9P+GvkXHgmyAZC8LyqUST5oyXJwwHTI/SvNvjgixeMtOwHOzS0wPT969Z6dENp9TzDyS8mxOcevQUkEZR9jD8jTVl81mKHZ6cfzqwudoLHOKEQW/U0p+9nHGKv6nbeXdSSoMKx+Yf3W/+vVDGM/SmUKshV8gkEelaLkXtq918vnLPtlA44bGG/wC+sj/gVZf8WfaprSZYbrMgzE67JMf3T1/xpMBxj+V0+7JG+eO4rp9Mm8+1t23rlvMTnosh6qf9luv4n0rCltnGpTW0jDzVcLu/vE8q30YYNP0uZYNRe0vMxxTOYZT3jfkK/wDwFv0zUvVDRo+IYY2miu2iD7o1Dk/eAB45/HFYjRCO8xDh1YEBJORtK4IP+fStzVDJNpEsM/FzbblfHJZVPzfqP1rnkuEYfIVcK4bYw68//W/lUwLaJrdrmKzNvGn7uLDMQdzNnByR1x9PSpVuFnSaJpUDyIU3XDEKRlWXOP4vlHaq0xZ7h5opUwcqEQYK4b5cH0we/piqt4bseU8QMiMAoGMlAOcfTk1ViTXsTFeSw28ttuz5uZoj97aCSR74rPuAHv8A7KySNlmjDqowzY+U/wBan0y6ubC4ieWNFW1uQJdmN7LIpBPXBGO/Wq+qtLa6qj20jJIAuxkPBwT835EfhTsIvS7xZzQxspu5E2yomd2SCFXP+etVtWgj02yGjxSMzwbZbvDY8yY4+XH+wOPzq3pcN5ea80FkEN00iCIy/Ku5UJ3Z/Cs4Wk0SXtpeJIt4Duw55DjuT3/+uPSqQFgJ5lvO6NkIokUnjch7/h3/ABqvZhn1VWHUB1IHpim6fcGIRyY3IoHyH68r/MVNpyeX4stII8skjrsBP3lZen17fhTJ6lyH19XzUzcrTZ4Psdy0Qbcvysp9jTl7d6aB7mTYEZuR0Pyn+lbUTAhc9eCKwbc+XfXUfP3f/Zq1I7qNNu9tvHNHUOhoH5lHHqtcjGP9FVemOOK6Q3qRXDQuxWRZNu0qcjGOtc7tK+YvTDsPyNMRNF2PtXqvwV0r7Z4lS+ZXMdlBKdxTAEjNtUA9/lZvpXlUXQV9EfA3TmtfB9zfOCDeXbFMj+FQF/mGo2RSPUB0rk/HNwRYWtoOs824/wC6vP8AMrXWV5/4nm+1eKFiGdttEq/8Cb5j/wCy1I1uYOojdNbWvQAbm9u38ga4W+vvtGo3N7DzG0ZZ1K7vlJ9MEcACuq1e5Yf2hcofmRPLj/3jhR/7Ma4L95byXLuVW3uVK5D7doznaMdBng+2aiWppc0vCJlk1y8Zpw8JhZkQOPly43YX+FQwIHbHSvZPBZzZXS56XCn81/8ArV5H4WdDqV7shjii8pWiWEtsROBt5A5zznv39a9X8FyqsN9uOFV42JPbhv8ACqjsRc8cZfmbcwXDNktx/Ea1bHx9N4X0iW004QM7ymZpJBnnaBjHpxXPeI9Qt47q4igHmRLLIY37N85w1cbdXZIaRu3rU06fLqeljswddezj8Jua18QdV1W8F1qTxXTqMIHTAQeigcCsS88ZX88YSBY7UYwxjHJ/GuekkMjliajHWtDyydp5ZWJd2Yn1NSBSRS2sHmsMg46mr5t8L0OKpCKXzD+L3xUqXUkQ++asG2H3ulRS242k9KYi1a6wwYI7MP7rKcFTXpfgr4l32g3aQ3MhntJW+YM3H19m9/zrxthhuM1o2F1ykUpGG+6fSpZVz7Is/EEWqaZBfWJVo3l2MOrr/wAB9enHp0zU7arM8EjRWwDBHYb27rjgjHX5untXz14H8cT+FNWCXDs0BwkiZ+8v+TkH+hr6E0nxNo+sael9a38JiYchnClD6MOxqbIYrasUkkBiJCEKvOMn5s5+u3ge49aWLUGubkW8sHyM2Acg/wB7H/oB/Srg1KwPS9tj/wBtV/xqVbiAj5ZoyPZxQIpSffNR4PrT35dvrTa0uQGKMc0UUCDFQXUIntJ4SOHidD+Kkf1qemnhWPYA/wAqHsXD4kfOMIJhi/3F/kK9W+FzZ0S8jP8ADe5+mY1/wrymBiIIuT9xf5V6p8LCTpd+SePtqD/yEK5KPxH1mbf7r80cdZyu+nQ3XkwR74FjYyEOqt8xIyc+uff2xS3lzYSeGgqy2F3cLN5/lQzoD8vA+VevA71xcUQTyUexieLyv9e58uQ+oHPB/nVq9lnliWKxsobSFYlVi9vGXkYj5iW2naOvQj8TV+yuz5rnsrWL1rr9rFp2oLLbyDUrgv8AZy8A2DC/MfYenrmm6G0dx4JmEjW1m28wwiWYJllIPzL1IG4jPTms+7a8kZvLe4kRAPLVZsKMAZIXqM+9N07UdShjdbcXG93VgqlBuIPOWbouMdat07x0M1PlZpaKIE160Imid3CxkW5BYfdOc/dIBzg89D6VsyanfaPrWi20nljznjuldGz+7aXC4ZuRnIyOvP0NZptb+2uo7y6Z3vIZmCOky7o/kYAncf4SRjHX05qlPcWttc3LqGa5Db1cKCcjGGJ4APFRGnrZjnNPVHV6pqU1pafabe3EF7qCuSh3eXD0XBBOTyG/P2rI1yYwaLqiGzklQu8O6Y4TlEDMvoQuOB13flyVxe6jqGz7TNv8kFl3sr7AOS3c9eSf1rU825WzfzLW5f7VbsjCNj95sfvNv+6gAOP4cVoqTi7k86Y+ymM1u9tJHuX5XiK8rnIwG9vTHI7ZrR0whpbczbAftAtyd4YMT3yDzjJB44C/WqWjJdWFo8qadcTkSfNGX2yMSpAkX5f4eeDjt1q6nnWcq27+JWtHOGSK10beY88jDMyjdzyfU9aTh0Y+bsTRkXMN08cMh27jGdjATbDzt47cfgRxTUvUVhPujK4yMv8AfPHA/P8AlW1ql3Zrostss8l9FbJiR3aSJ1klTbuYMWB+ZU4Q4VSwPWuSttQuNYR4fsoAkjj3ObgjaoxtKhe/y+4PNZumHMbSzu8Z2vhAvTb09KikPmRGOQky4ztTnHpUtpYp/ZwhBVgHL+az7trAnbjoMdufertrb2qRSMhMhVd482fGcLyAob5snJwe2O+a55OMNblLUyQsluuGjde/K4BxUyXMqsA75jYnDZ+8fpUt3aLKImMbABBvMe+TcOP4N23n/wDV60fbYbFknkuoFg3KJXuItrD5wepwqjsOM5FaRSl1JuWFWVVlVoZ1ZVw5ki2gR42spyO2QfUcVPZQ3FzfzXX2eeZiTHcTSurSGPCbOTjO07gcDkEU7XPENjqGjSfYbtHSPA3wXKsY88fN1Ck9ckYOK84ltZdS1aS4mLybxkTzIqiQn5QoPOBkdugFbKlrYFNdTqLy9C3aQTGTT3tfPSE4EjCMN98L0AJ3AFvT2q14eebToIf7US6hgQxuYI9PmnZtyk7ty/Kw27eec7q5K1i097qCKR3/AHzhceVj5E4YMM8Zw31r3Sy1CCw8HS3Q1Swt9Q8gSPNHJ5xLDoDGrLvbbtUAd60atsJs5RNW06203/RNStZLiSQlDJbywtGScgOWjYe3OAMmsnVNWvtSsfs099DPwAyC1jGzPUK3LdMcjrWDqN3c65qx1W6mV7cblSLzGbyz/cVjncP65+taWneFBqhmn1LT9SgYRCVJLW8i8sR7Th/3iYI47NzShRhG/nqV7WRc1WN59C0qztLiESqxZmEgYRhUYZOAe59K5uPYl1b3S6xa7YsuokimDFsEA/cxjnNbGn6ftzBpWnXeoGbjbvXOwrjMjquBnn2HQE1e1Hw7FYWdhNeWU+n3bgCG2a9E6+Uo5LKV+TlgMZzn8aFCC9Qam1z9DDNm4tIYTEl1Pc3JC70J3c+mVGSSPTHNWriy1XQJLeK4021h85v3RWNWDFTnghzjBIPNdD4ZSylvWGsafeT2UnyLnT3MAbs3mhuvqT612g8PeFm/1WlTYP3vKnkj2r/39FaRtbYipCSa13PMUbWtakmaPSFkdSvmmHERyRwD8+D3PQ1t6DdNcW5LqUdlyyk5KsrFGB/ECu4bQNAknld49RtN3G4XTqhIA6Ydh3FcxqenadoesW0OnXrTpIjtKkkis6Mx4JwoOG65I5IPNS0t0iVcsDpUAXbdzwA4M6echHZ16/40LNTLmXYsVyv3rdw/1U8NSktC4vU9Z0m9Go6Ta3feWMFh6N3H55q6elcn4KuxtvNPzxG/nRf7jf8A1x+tdb3qQa1PA/ijZvp/ixlCqLS6T7UM9Cx+Vx6D5sH/AIFXFW/+kPK7PGIovlIQnOf7vA6/SvZvjToh1DwlFqcY/e6bMJGI/wCeT/K/5Ha3/Aa8Ku7oWloihAruS5QdBnn/AD9avcRZvriOKPy4Ygig8ACqI1G5tovOs5JEulcFZEHK+tRxXa3VmzTYR93v64qCaaGPcsfz7eMjpWt1bUjW5NbeItTWdJjcxySqwdS0CZBH4fSrt74n1G7V9/2eGaQHzLi3gCSsD23ZOPwwax4WjlzuiwRzuJHHP6CtYaE05gEN3pJeZSdh1ONNnJxu3YxnHHXtnFZ8qLU5LZmK4mvJPnlZv9uRyxx9TWjoOhR6/rsGlJOtu0u8mZssqhVz0HWqtxBPp8oimSNWYbgFlSQEZx95GI7VNp2r3Gl30d5ZM9tcxghZYJSrAHr601oS7t3Z6LY/C3WLKeVNN1+BXRdrgSkBs8Z2tGR/9cfSvMPEnh++8N6zLpV5JbSzxxpI7QyFlO4bhyfb+ddfb/E7xPGOdau3PX96sL8/8Cjrm9XvJdd1OfUb6aaa7nK+ZJ8g+6oUABVAAAFJsDHtAY5kcfeHzYzwfbPbNdXYr4dv7SN9WljtbgSFJfOyTs7FSo+b05IxXMyKls+7zlGRwHPNOhlLBY8o7Nxuc4X8azlBS3N6WIqU1aL0PbPhRZpaT6hqenia4tbgLbIIEGwsjFuSWGMKQP8AgVc78a2Y+MNOeSB42/s/5lZgwx5rbehPqa9B+FMMcXw9sk4DefM0o9HL8j+VcD8b+PFenHOR/ZuPp+9any2jYidRzfNLc8t2NmTDdunrToxuXDdPpUMSgSEhgSvGQOoq1FgYz068UIg6e5kS7d7lQcsu2ZB/6FWNJC0Oxs5BHX1reuFEGpNNbDIZS2wj+EjlaoSKqbEbmGTPln09vwqEUZbqfkb1FN7++KtBR5hgm+XkqG9D2qo2Y2w2QwbaR6VRJdMn2mzEm4/abZVwc/ejz/7KSPwJ9KszIdSs5rlRi48xWOP4mx/WsqOQwOsgGR0I9qtRStAv7o4SRQVb3HQ/zFJoaL9xfvc6Ul4gH2iNGSQ9d2B3+qn9KoXUcL4nt0CxNzgH7tEspb7Vs+WO4Ull/uvg7v8AH8arRuy2hdTlCAG/KpS1LvoVIWeN4jhSG86M+/Izn86sx3Lic70Xy1wfLJ3Ajn168YrOjZl1FH3H5ZMk1oZgluJp/tCq8n3QWHQDuPWrJCaSJIDFAWCsd2GYk/T8+9Xxbw6laxzRMsdxaAHDn5WTdu+b/d5/A1n3lr5U4P3cfKeOoP8AhVmwinuZJpvN5WLaSVC7x93HAxnbTWwrnXeF08vWpXkCo0NrNtJG4Kx+XOfxPNc5rkVxLeS3c22N5H8p9ucblUKOffFdL4Umto57w3MqK7QJHCGbG87xxzx6VX/s2G8jl0lDhftLRkAdMScGobtK5W6RyqN59vFOU8tzGqONuNzc4Yf7w/lUT3BttY0+8UZa3kiZh64bmq9kGieaFydyuO+fpT7kx/aoWGNsyDv9x1f/AD+daLYh7m/qjIdSfyw+10DAnp9R9aiQsULDOVBb8KqGZmmk3vkKSq+wp8V0scgJwRnBB7g0o6KwPczpCBrNwy/ddWx/OoJJn3KVcjgcikmkUakxDfIcgH6riq9wHjjRDjf6A5piLUG5d7GQgHk47VM7mTrj8KzI7hv4vyzWgjl1ztJ4OaYiSL+HAJYdh3r6/wDCmkjQ/Cml6bjDQW6K/wDv4y3/AI8TXhPhP4UeJn1rTLnUtOit9PWeOaVmukZigIb7q5619HDpRJl9AJwMnpXln2oXF1f6mTwzvKM+n8P6AV3viW8+w+H7yZT85jKJz/E3A/nXm1z/AKPpCRDjzCF/Dqf5VJcEYOpu0drZqX2K8rTyt3CqNo4/3mP5VzRjknR7NpQUl3beOM7Wx/nNdPqdnOb/AD59q0Qt0j2b/mX+JgwIAySexPasUaVfDV47kSW/2dfmZWuF3Aj+7gVHW5TTtsV/CSCPVdTYbVPlRh4xnIbPzN+JFeiaPqMdjoniGaU/KtquB6kllA/NhWToXw/1WCe6u2uNLjS6RCqtdEkYJJzhcc5FVPE1lc6NIthPPbuZlEr/AGeQsu0HgNwOc81aaexm4tbnm2rByV8xgzbsEiuY1KTDeSO3Wux1EKgYnkIC+a42a0nlhkvn+WMsQD6n0qiWZzAikp6xswzjipIoGklCKOf5UtQOh0Kye4G8bVVRkkgnH4Ctu40gShmWMjavA7msy11M2lqlvEwVUXqO/wBasJqcskIVEOVGFI4A/CmFyvNpckKqZB8p+YEVWksjKrNxsXpWsbmaxjE1zKPKY48vqWHtWNdap5spZIiID/Bu2/XkUAVG0xQhlmlWJScKvVm+gqSDTklsop4t+CxDLU1rPZ3G8OoZ8/uhK5G1u2cA7gK3rWCzsdKSP7T5j5Z5AI+DnqM5z17/AIUh2KeqWDzafBfxkeYiLHKAf4f4W/p+Vdt8ENTng8V3GnyRTTQT253IqhlRgRh29u3HqK4wanPpjeQckM2Bu/u91Na2kfZDdtJZO8bspRk+6QD2BHUZH1oZdNJySbsfTD2VswwbCDHTBt1/wrOl8L6VKMHTyme8Zdf614X9puVyPPnBH/TVh/Wnre3a/du7kfSd/wDGsfbpdD2lkUpK6qL7j6FWMqiqsbBVAUDaegp2xv7rf98189rqeoA8aheD6XMn/wAVUg1nVV6apfj/ALe5P/iqaxC7B/YE/wCdfce/EH0NFeBjXtYHTV9R/wDAuT/4qnjxHrg6azqI/wC3p/8AGn9Yj2J/sGr/ADL8T3g1R1m4Fpomo3JbHlWsr9f9g/1xXiv/AAk2vL/zG9R/8CGqO58QazeW8lvc6tezQyDa8by5Vh6EUvbrsOGRVVJNyRmohVEX+6oH6V638MYfL8OSORxLfOR9FVV/oa8lB5yenevb/BlqbHwjpSOMM0Xnt7F2Lf1FRQV5XO3O5cuHUe7PAdGs77WbBo7edCsG+SQErHt2fd68sc4/KqVibSZJ7i6W+uHZd2XgBYSdevQD6HNegr8BtYaaR5tVtmjYEJEs8ihSe/3Ox5xUZ/Z7vQgJvrWd+flkmZR+kea6Lny92cLBqatbqglL3bT7sDAPl7hkegG0kZqLSIn+1qsE0cW5ZCkEduxJVEZgznoASMYz7kCvRl+BWprKrvPZN+7EZ/02Rcjpj5YhUkPwZFtJG41LTreePcuUuZMgMCCCT14OPzpcyiFnI47U/FIubxpo7ORYbK3UyOZZHd0K7FLNJ94ZIIx8o7Vm67cXH9nQKuxiJAj/AGZw7yfL/EBz1X+VewW3w2sYzEZ73TpDHGsKs8jSkIOijcentnFaZ8KRouyLULFR0J+0FePbbjH51DlG+5XJLseMaXHrzae8VuVt/tePtDwvIWkjP8BRVKqOeaveHotb0u6uZf7BkmDowzPK0YdugYAMCD3wenXNert4VU/8xLT3P+3cO38zR/wjDZydU0xQOm2QjFV7RdxeyfY4mebxFrl+95f6JYW05Xb57MoDDPRlGcnuGzms+TwTeXjv9o1qSO3YIDb2iNtk2k7dxLep65r0yLQFiHOr6eTjBbzaefD8Ryf7Zstx7+b/APXpc67h7OfY88g8BwpCyS3uoSIY1QISkSqF6epPJ6962o9FsYWDLDCp6Yji/h9OwxXUDQFUcaxp+fd//r1Nb+FprhWaHUbOUA4JTLAfkaOZPYHGS3Ry32XB+TzAOmd3IFPEfkLnzrn6liv5fLXYf8IndlQDc2+B0IVv8adH4WvYTuivbdD6iDJ/Onyom5ySxzmFjJJfkEZXbLISPrxVSSKX5vP+0SIe0p6+mdy84rvH8MXkjbpNQV2xj5kZh+RahfC96h+TVCo9FRsfluxQkhnmOpaBpusqvnC7UqCv7omHOfcDa3/Aqwp/Aej21vLDbT6ra3DAZ81A4Y/w7hsHH0NezyeEJ5T8+oRE/wDXqP8AGoz4JkP/ADEU/wC/B/8Aiqa0EfP0vhDW7eUSR3VtKYwRG250Y+pyw4OCe5p623iKyNtPcXF1qEC/NLbI28huwXruA454r6EPg/MagXojkXqUiO0/gW6/SoLjwwtrC81zq0ccaj5neLGPx3UOVhpX2PmtdWjtdOiglguBcRvysi7eo7Z9D7VpW3irVILJraSJ4bGQ4kzdNhR3O0MePwr3I2WjgYbxJbn/AHkB/rVebRPDM6lZdXsHB67rVGqPbQ7o1+r1X9h/czzrw944sNBvrqZNRSF5GbnqGXORg9PwpNf8UxeJria7juQyJBHEH27MDeWfH4nk13X/AAiXgxCXj1HTI5T1dLGLJ/SuevPhp4Vu757lfFsNtuTYIraxREUYxwPU+vej29P+ZB9Wq/yv7iHQPHtpoNsbR9SMYyTsEXmKf1roLT4jaSkoZJ7RSAQxFrIhz6fLmsf/AIVf4H+zRxnxIyzocm4ij2tJ7MMkfkBVf/hVXg8Rv/xWFyZCRtbyVwPqMc9qf1in/Mg+q1f5X9zOrTxzpN3GPPuNPc7mYBpZUxk+49AKzde1TTb+2g+xR6ck0bhy9sQzsADwzdSOa4bWPB2g2t2trbeIZUEUahpF3P5rEZ3YI+X6DinaRpemaXcmX/hIHuRsZRHLGyqCeN3C849Kv2kWrpol0ai05WdWsnpUhl+Q55XowPp3qGytxdW6yW06TR9A4Vlzj6102ieFodWjmV7u4heMDIVUYEH0yKttWMbNMh8K3zWep2Tu33XNnL9D939dtep15Tq2mDSfENxZRblimhWWA56ED+hU16Po19/aWk210cb3TDgdmHB/WsUay7k2pWMOp6ZdWFyu6C5iaGQeqsMH+dfKOo6W/wDwkQ06ZyrWxdJ3dSfLKHaxIHX7uffIr64rgda+GFhq/iW61tdQubaa52eZHHHGVyuATyM8hRmtIszPnifTxC8sLykSQzur4QnK8YIFUJLd5ZhHCsjE/wCztFe/3PwTsLm7muW17UQ0rFiFihxz/wABot/gjpsDZbW9QkXn5XSMA8Y7KKd0Fjwm1s4oruNUnLzbtgxaiVM8gjBYbh+HarwsbeOR1dLK5TaVBe3ljBOMbv3cgPB/A17VN8IfD2nW0lxNq1/CEU/vMp8g9ht/CslPAvgVwFbWNW3kj5z8uP8Axys5VacdJSSN1QnO7pxbR5CdIQfdSyUHn5JbiP8Amp/nVb+z/nlAAIXH3LxPl/76UZrqPGtto/hXxLPo8L3t6IlR2lW5EZUsN20gqwOBjkY61zCajpuxix1BGY54eNgB+Qq7mDVgGmsAPkuP+AyQv/IimPpUxb5Gu4yP7sAP8mqYXukv/wAvl2vp5lqjfyekWWwOSmpIM/37V1/lmgQipfQApHdPGpOWDWW7J/HNQ3SsYQHktzMpLR+VZLCzN025GOP61a3QlcJqNq3sfMX+a05PtUtpts74oyMwKx3GAc4IOM80tiox5nY9T+Edw9j4LkeZ3YXOpSBAecbURf6VgfGof8VHpROM/YXyf+2tdb4HbZ4Zitvs+7EzswEyoUfIJx14B9PeuS+MZkl1PRjJaxxym1lbckpb93vGFIKgcHJyPWn0I6nnCByy7iW9zSnAcgDHPrTV3BV4dfYikw5JcghOvA5qRnT3N2IJI5PMBaN9rMvIx/e+nNRyorwuquqRs+9ec7fp6jrWQZ1dWEkTYb7rlf5+v4U23uLnT4HhuIoriKQYC7+3Udwf8isrS6FMde3ClU3MBKp+YDoR1zTH1G3urLlsTw/xYx5if4j+VTPNbz3P+j2ttFAE5VwN/wBPmPP1FVBDaJM42bTyf3bkf4iqT7iuVYtQj8sB+OTU0epRorQt8yEHkdvSq0sMCndGoI6Zaj7MWPy+X+HBp3C5ZttRIuk6MGZeD0JqNbp7KaSJdhBUowzkY/8ArVE9i6FvukdRj0qFlXO3YTt/2qLi5mQ3Erecrjafbt+NaumtD894QkUnmgRsBvCtjoV9DWb5cbAlwRn0PWnxxwLjeJAO2BmmO51GoR28Vo1p9p3TGcOmEYYXbyuT17flTNHuJESW34ZI5STx/eA/wrn4AUkUgsqq3QZANbmlkJJdknAZkwe3f/GnFWQN3N+M6bEziRIZDjOJFLBiGyG4IxxwaI5Nu+XTlM12riV1a5zz6AY+bn05+Wub1JcuMbQ+wlfzqratcQTLJ+5dV/gkZiprOcG3dMpTsWNX0m+gna5e2NujP98N8jHOcDms66gnJUy4weRtbP6/0rTLmR1Mk0GVPy4XAX1oxayApO65YcywRYZD+YB+mKIqSWpLMtpWZHTDIHfOEIwB6d6ckcbRr/rN2efmp5t0LM3nEj1wOfypDDCNuyUtkchuKsRGIom+4rAY+YM3U96QW8JDbhtx/dqwIowo28/U1YEkCIFFpyP4mk3E/kBTEUTp7IwdiV2nHzLz7GrpnSQZSKNTtwRjINWFuYdoCWahvXzjj8sVIk7MSEt41c8bsu3/ALNimho+s/D9yk2jacqnL/Y4XbHYFRj+RrWqjpNkunaTZ2aZ2wQJEP8AgKgVepFs4/x1cbksbBT/AKyQysPZeB+p/SuSv4/Nvra3BG1R8w9M/wD1hW5rcv2/xfKoOVt1WIfX7x/Vq5+SYSXV5dZ4XIU/oP0BrOtPkg5PodGGp+0qRiurOB1CYXGpXk2M+ZPI3T/aNVgA427Rzx0pN+QGP8XNT2a+be28Q6tKo/WvHcna7Po4xXNZI7TaFwvACjBPpiuP1DUDdXMkoA8tiNgH90dK1fE+oNa2gtowd9wDlgei56fjXFuXOQGOK6MsotJ1JdTgzrEqUlRj03IfEM/lWAOR+89fSuPmvZZbeKAn93GuAPxzmui8TNtsYd3zFnKg/TvXLRRSTOEjUsx4wK9U8InEuFAVeMc5q1bgyDEKnsWYdfpWmnh6GN/37sQAOFBYscfpzWxBawrbqEiWM+jIWbPccd6Yjn41ePAKd+/UVpJG02AFII52hMVe+x2tsqvcnfcEfJERz9TzUJ1GOXURCdu2N+vXnjpQBV1uInyCOVVAvQ8VkQOA5R2ARu/pXTX0T3J5QDavPYVRh0QSsH3AHPb5sigCnb6XHJOjRy4BOCSOnvVwW8qRqjjHpWt/Y8L7SpZZeh28k/n0qW4s2CRjaWKfIXLdKBnMXcbS+WijHlnpmnbp7blkKns1bEumSJ8xZeffNTJdJ9nMM0YIVeARmgCnY+IboNtu2+0L0+fhh/wKu18P2uj6+/lDXo7C47R3NsSp/wCBhwPzxXGSWtqSWDqjDs3Q1lLc+VdcOY8HqhxUunF6tHXRx2IorlhKy+89DutPa11qbTDKrtHc/ZvN24BO4Lux+NdmfhVqWSF1W0ODj/j2k/xrzTR9VAuIftjYRZEZZVXoAwPIFem+N5bvxBdadqHh77ZfWZhkVpLEuQrbwcMF6NjsaxVJJNtHsLMp15QjTnytrW66mJrvgfV9BtHvJDDc2sf+skg3Axj1KnnHuM1zOa9X8Bw6rBo+opr6XEdj/wAs0vidwG1vMPzchcY6+9eSR/6pMdNoxn07VlUikk0ehgcROpKdOo03Hquo/NHWkFFZo9EsWNlJqWoW1hED5lzKsK47bjgn8Bk/hX0GqJEgjjAEaAKgHYAYH6V5b8M9JNzq8+qOv7uzTy4z6yuP6Ln/AL6Fep120I2jc+Tzuvz1lTX2fzNYdBRRwK5TxT4mNgpsrE5u2HzuOREP/ivb8frFWrGlFykebQoTrzUILUZ4q8UfYVawsXzdsMPIP+WQ/wDiv5V54VBJZgCxOSTyTTyjkkncSTkknJJ9aTy29K+dxGJlWld/cfaYLB08LT5Vv1fcbsX+6v5Unlr/AHV/KnbH9KNjelYX8ztshvlr/cX8qNi/3Vp2x/Sk2P6UXHZCCNT/AAr+VJsXoAv5U7Y/XFa2heHrnXLrAzFbIcSzf0X/AGv5VcISqS5Y6sxrVqdGDnN2SGaF4fn1y72Rjy7aM/vZsdPYerfyr1WwsLfTbNLW1jWOJBwB/M+p96LGxg060jtraMRxIMBR/nk+9Wq+hwuGjRj5nxmPx88VPtFbIWiikrqOAWiikoASgcVXsrn7ZarP5M0O4keXMu1xgkcj8Kj1DULbTbR7m5kCRr09WPYD1NKUlFXY4xcnZbjr++t9OtHurmQRxIOSe/sPU+1eV69r9zrl0GfMdqhzFDnp/tN6n+VJresXeu3fmzfJCh/dQg8J7n1b3/Ksvy2rwcXjXVfLD4fzPrssyuNBe0q/F+X/AAROaOfWl8tqPLeuA9vQT8aM+5/Ol2N6UeW1AXQc+poyfU/nS7Go2MBzQF0c7rwZb6J8HDxf+gk/41Y0XRXvdtzdoy2vVEPBm/wX3710tz4Wl+zWepajCRAXZUhbq2RkF/b5en5+lWd1e/gsO3BSmj5DM8bH2so0n8yaHChUVQqKMKAMAD2rqPCj7NSkQ9HiP5g1y8b5rf8ADs6pqtvnjJK/mK9Gx4dyz4+sz9jtNUjX57SUBuP4W/8Ar4/Ol8GXapNeWGflbFxF9Dwf6fnXTanZJqOm3Nm/3ZoymfQ9j+dea6FePYXlnNL8rW0xt5weyk4/T+lQaLWNj1WmO4jUs3QU8U0qGBBGQfWmQL3qrfX1vp1o9xcyhI179yfQepqHUNUtdLtHnnk4U4Cr1J7KB615pq+qXes3fnTsFRf9XED8qD+p9648Vi40FbqehgMBPFSu9I9yTWtbn1u6zI3lWyEmOInge5x1apdI0J7y7jy0UkXBOJCAx/u5x1wM9KxvKNWLK9vdPlMlnNskA6YDdvSvDhVUqnPV1Pp6lB06PJQdjy34rxQRfEnVootylPJVw3Td5S5x3xz3964qS1mRtpR/xBFdf41g1C6+IOr3OqW966fa2RpYbUncqgKu0cDoB3qniwDc6rJbkdReWMifqm7+VfURtZWPial+Z3OaMMcYzM5J/urx+ZNSRTWqf62J1Tlf3ZztrqRpjTJui13w/cFjn5tR8sn2xKq1heItJvdPnhmuo4hFcL+7aG4jlVivB5RiPSmTYzmbzJyIJHKD7u/jFa2noJmETw+b8rH5gCAACaxYgFXkY+td54Kk0S0jea+innvZGOzy50AWPb/dPJJOc/hSk7IEemfC0258MJLI8aIsrqvzADseKrfEW0tNZ1jT5kugyQ27RERSrzlwcEVzjarp0CfPdTJHkJFF9iVAi9/uvzSN4g09oo1hu7VMKAztA0TMff5T+FctSrUtZIqNGpNXihq+G9JAGI7iXPXfdtgfgF+lSReHdDkKv/Z0mO+66kVf509NYhc/ub22POVUSxgj27fyq2l47MRBksB99VynfjI68VzOpV7h7Cqt4s82j0XVZ8yRwXM/ukW4fmDRJbXqL5d5HKAOi7NoH5ChLhixV3ZF6nCBuO/y1fhksoUjEzStuXvHtP5f/Xruc7EXRUVvLVR9lhHH/PLBP48/pRvhZGRvsz5OfmVmI+hXGK1GvLCQuYRcqduAqRhR7tz/ACpRDZrII0CySYB+Xgg/72fwqfaLsPmM9Vt1wy6fksMFjI4HvxTIbC1Mx3zQQKegifzMfmQa0L6GAwyK9t5Rwu0rOrfjnq3+c1Wj0m2aGOR9VgLAElfmOfTt9Bjn60Kog5ieax0ZYiBq8IkB6m1lXd9eWFVJoLFYyIntyx5UAtn9QK1PsllFvQ3doBwSWRm2tjOB7VnalNFbMbeF7Z1xziEgL9M0RqXdkgcjNVoUPzqM+m4CppDCoV0jUdCQdrZ/EAUyK5mjx5cwVT/dUdKvvZR3Z3WxJbOcu+wfrVOTRNzNkkXbkIi47LTIpLjDfIURjjOwkH0Ga1JLAWiF36cfxKR+Yq5HFqF3bb7S5ghRR82bvb9Omf5U+aXRDMKeznuoY2ynmR8k+aPmHbA60gtnZgptpGLHt2rZih1ae7eJtSs3l27j+8Zyw/2Ttwfwp8V0txIbe6uJE+sQA/PdQ3PsGpkDT5CoU2o3KSxy/wAzA9O/QUrxQBR50EsZA4MWB+h61avLW2hk3JJM69AWAXn8P8ari6CpGRahlzgtyCf8cGnyyFqJDCZWItklLj5lxjkCmmFQx8+TyyPvFxgg/TFSy6pdOwYmMv8A39oY/jnNSPdSybZFmj8xuGj3rnI9qfK+oakK2kJH/HzHnP3Nr/z24pxghjAG5W9dpB571NBbXk48mKKZ0bOBhsD/AAqRfDesyn5LCXb2ZhtU/iafKu4+VkLWu1VbMewjILSLz61aijdLmJrNsSmRRGG+Yhyw29uecc1bt/CuorgXkEvlHny4riJDn/ec4X8jXYeCPB0kvjbTJJBZx29u/nmKOdpWOwZBJP8AtbaqPLfcFF3Pf4FkSGNZWDyBQGYDGT3NOkdY42kY4VRkn2p4rF8V3RtPDt2VOHkXyV+rHH8s0ijhrW4Zob7U3+8++X8WPH8xWBqUn2Xw9Pg/MyNg/wDjo/ma3p0EOkxQjrK449hz/hWHrtm93Yi2icKzLHjKk5A5PT3rkxilKCjHq0enlrhGrzSeyOF+XpWnoSKdWjYjhEdv0x/WpV8H+IZDmHSppl/vJhR/49trqfhx4Ve/1C7uL+F4oLfETRSKVZnydykdgMYPrXE6E5rlS3PUWJpU3zyeiMvxPpEw0my1WRQIHkNvtJ5dSNwcD0BGM/7VcVLp9z9qXagVOme1emfGi51TTbS0MN6p0+Z9v2X7OARtG7O7uvHT3FeC6j4j1C5UosrLF6Bj09K9OhS9lTUF0Pn8TXdeq6jVri+Kr2Ge+it7dt0Vum3Pq1VdJ3Qq1wARhsBvSso1Yhu3jQRnlM56c1qYHUWuo/vMBmyTyT836VtRTjZ5zKYwq7mct8uexx+Yrjbe9hiff53ze6dKs3XiCSZPLVyE4+YfL+lVcRroDcXjzh/mY8IV3fnUN5p8VlbyOjiSXO7eFx36AVlJqtyiq8a+Yp+8vX8zV+yv7lb/AH3G0+Yvy45CDHpQBBpmp3MTOkm+XnlXbGPwrfh1QEmSSaK2XoeUGPpn+Vcxe3Aa7eRd8u5uB13VJbazd2rARWWGU9PLHb8KAOoa5eeNUtJAzdXIyN2enNVEsYJ5cfvRMOryNuxVSDUNVuJC98/2OIkbQ8R+b2Gela8SOkyu0gcHo2Cp+lAx9rJsHk/eTrkA9KWSSzLfvYivHXtTVuEYvLhMdCrjGP8APrU9wLaWHMke9FUOZNp2/hjqaQEF5psM0O+N+GHU8fhXNSW7Q3WxIonOME7ev4nvXVyXEX2hLdApQLnl+V9OKo6rborLMn4nOfyFFwOh8KQaKNOuEvrpEvNoWO32tlSx4ZsjBGAcYPWrU1zN4E1ZJtK1ONnkVWeMSLKCv92QDg/zHtXEzTMkASKQpOmSpJ4bp8p9OmR6H61RtYb7UtQjaRioVsMFOfxz0pt2RUYOUlFanvvirU7jXvhva6vZu8MDuPtkCnhlyUIz12h8H3B5rzE/rXbeDvEGlWug6joWtT+VZTKfLJUtww2uvAPs31JriiNuV3iTaSu8dGx3/HrXJVlze8fXZXSnQUqM1s/vuJT445JpY4oozJLIwSONerMTgD8TTRXo3w48NEsuv3afLgizU9+xk/mF/E9xUU48zsdeMxMcNSc5b9PU7Pw9oyaDolvYKVaRAWmcD78jcsf6D2ArTPSne1A6iu/ZHws5OpNyluzH8UeKV04NZWTBrwj5nHIiH+PtXn5mYksxLMTkk8kmmtIXdmJLMzEkk5JOetJkYr5XE4iVad3sfb4HBQw9Oy1b3Y7zDSeYaTP0pcr7VzndZC76C5pMj2pMj2pBYN5xzR5hFG4egrY8P6BNrlwTylmhxLKOpP8AdX39+1aU6UqkuWK1Mq1anRg5z0SDw/oVxrlz1aOzjOJJR/6Cvv8Ayr0+0tYbK2S3t4wkSDCqBS2lrBZW0dvbxrHEgwqqOBU9fRYbCxoR8z4zHY6eKnd6JbIWiiiuo4QooooASg+tFZ2r6vbaNZm4uX68Ig+87egqZSUVd7FQhKclGK1Yuqana6RZNc3T7UHCqPvMewA7mvLtY1u51q7864GxF4jiByEH9T6mm6rq9xq92bi6YccRxj7sa+g/qe9UNw9q8HF4x1nyx+E+uy3LFh1z1Pj/ACF8z2o3k0m4e1G4etcFj2LAXNJval3Um4etAw3GjeaNw9RQCCQM8k4Ax1+nrTt0B2WrAyYGTwB1rufC3hRyUv8AVI9v8UVuw6ejN/h+dS+FfCX2YpqGpoDP96KA8iL3Pq38vrXaCvYweBStUqfcfL5pmvNelQ26v/IxPFkCy+HZyRzGyyA+mGFeebQK9V1KD7Tpl1DjO+Jl/SvKVGVB9q9iJ86yVMDFaFlJ5U6OOqsGrMHBq1BJg4zVCR6mCGXI715tr9ktp4quon/1F9H5igHGW9vxzXb2+p2sOn2bXE6xmSMbQT97HWub8aS2tzb2d1DKrSwy4GOu0jP8wKyZpE6XQb7+0NFtp8nft2Pk5O5eDn8q065DwfOUluLf/ljMPPjweA3Rh/6Ca6/tQhM8/wDHEEkWo205LGKSMhRnhWHX8wR+VcrvHSvS/F1l9s0CZlGZIP3y4Hp1/TNeZZUDk/TGMV89mNNxrN9z6zKKqqYfl6rQesu1xlQQD0I6/WtXT/Ed3pwKQwW5yoVQycj2z1P41nxXFuE3vCrsR8uDhOO55yffpWppWrO13FFb2emiRctkRqrfmTkn6c1jQupqzszfFWlB3hdebsTzPJeTPO1+gkdiWSUvGF/2QSCMD61DJYXQj8xrZ3j6+ZH+8X81JqQWRuZnk+3QCV2LOAzBtx65Ugd6Fsbm2lDR3USOTwTviY/RttfSrY+Olu7GXJZ2twcSRxSH0eJH/wDQgapS+GdEmYl9I0py3UmzVT+a11jPqM3E9pBfqB/EiSnHruXDVAy2BOya0urWTH8DEj/vmT/Gnck49vA/hx5Vm/sO1WRCGUwyumCOnGcV0Fwsd3Dsu7SOaMdpYVcfpmrxsY3x9mvoH/2Zw0Lfrlf1pkmn6hEN7WsjJ/fjHmL+a5o3A5yfwt4XuAVbTYomP/PCeSL9M1myfDjR5STDfajDk5GSkox6fdFdW1xhirkZzjDdfyNJmE/8sVB9Vyv8qXKi1OUdjiJvhkwjdbXXY5HJyouLUrx9VY/yrOb4aa5bNutrnTJXwf8AVTtEw/76WvSgqN8yySr/AMC3D9adtcfcmB9AyUuU0WImjwS80fVbJBJc2kkCnoZGTn8mNVRHJn5pAB7tmuqS11C/0vfb21uyMv8Az1Qcfi1c5Np2oxuyvbuu3qQAw/QmrUUcrjYYsI2fO5PpgU+GKxDqLiS62A/8sdgP/j2a0LHRGvIFaa7a2bPO9BjHrncKlHhWEqf+J1atj0Rj/LNFoDUSvLZ6M0Qe2mvXkxjEsqY/8dQU23isGiYfZC0o/iaaT+QYVpWnheJDu+0zyn0itWI/XFaAs47XDxabvkHR5/LjH5eZ/OjToOxgwf2fP8kdjGk6nkFncH8CxqKS33OyR24jI42pF0roVvgZg9xZ27leB5arGB77lq2NT2n5JbaA9eZnkb9VNGvRBZHKHSNRmPyQMr4+62Ez7jNMHhnXpAW/s2bHqXT/AOKro5b2Rx/x+SSZ5+WJV/UgGmrqFxHgoik+pG4/+hcU1zBaJzsPh3VA+028cJP3vNlRP61tWWntp8R+2IMdP3VzGB+tXYtRviT92MdysWD/ACqU6jLuy5kP0gj/AK0Wkw0RkjTbL7cZra8jVAdwVnLYPsVUfzq7NZ6fcv5jr8wOT9nRm3H8WGKLq5jZS80Ux95LhFH/AHyKiguLeT5IobdmPOMtMfwXBo5WFxXbTIoxB9jvHPTYZFTp9Nxq7DY2bKjJoMkrY27pFxkenUcUkOm6rN/x72F35bdQln5YP/jorWt/D2qlBu0qNfebaD/WjlQXMvy3U7ZND0mLb0MtqrEf99E1btJ7qPK2v2ML/dggjz+G1K0G8K6i/wDywsV9d9xIB+S7akj8Hyov7yeBWPOILZ5f5tS0C7KcmqalGpDW+oyv/eclVH6f0qqJb+5wGs4woxx5E0p/Lcq1cn8GXEr5875P9uLZ+lL/AMILNgY1BCP7gtyR/wCh1VohdlV/Ntj+7MVux+8zpHCR+G4tXovwrtJpH1HUZrjzgNsCESMyj+JurH/ZrhU8H3YYAXJgT+8iqM/ka9j8FaKuh+GLa2EjyM5aZ3k6sWOf5Ypu1tAV+p0dcZ43uDJcafYKerNMw+nyr/M12ZrzvUZxqHi27k6xwYgU/wC71/8AHialDRlaxLicxjpDDj8Tz/hVSyvLpdkSXMyID0VyOO9V7mSW9a5dcq0z5G4fw7v8BU9lp18x+WBm4IyFPXFTYq9h6yvKS7uzHk5Zia7G21RNI0LVtXlUM7XGFUnG99qqBn3auZGl3VnJAtygVZWCqwbI69DT/Gt5JYfDuxuYVjcy6kCVkXcpDM45/SmlYls8y13xDeeKbiV725e4RNxic/KhDHkKvZQAPc15vqdoLUoY3BV2b5Owx/St+5kubWSJoGEjxIBsOPnUcFaxvEcflaoYOAY0UlR/CWG7H60yTGbG48Y9hSbeM1KsqhQkkYZfUcEfQ1NDELiNlUElc7VXk/j/APWpjKYpxXikZdrYIpe1AE9pdvatxyp7VqZOotGd4WRwWPbIH+TWHVgXLxSAxttwu0cZ7f8A66ANMGS3jVQ2wvKBv28gYq3d6FqCuXeYGf76RlwokTuVYnGeelZkGpbg0c/3W6kGui0nVJrZEXyUu7MSK8cbk4DdipPRv89KARBp9zFIkSCYxr0aULmRP94d+fUGte5vobXUlttokjlQGN1w2845yPWrd5o1n4hjiuNIupJJ0T/UtFGk0W0H/d8wYz0w3sapLpfiTSrq522kU0tsm6R45EEjDPTafmLc5xjNI05bkP263yyb0dOAARtdf931HtQ0zxY8oeaCN2YJOT/wH1qjdReT9p+1afeI8nzKUTKozDd1Xg5B9e9Q6etjPdpbyXJsWLD/AEht3lqf9rjIx600Lk1sX5Lxd53oueoEkRVh9TTJLrzPl24yccZPHY4PT6V1l78L9djs0ubbVINRBG7yo3KMR6gtw3HvXM29hNa6n5d9EzFTtZJkww+o7Ggv2Ek1zaGWVubllAiYZG6upsYGt7VFfBkxzgYC+wqdbeOAlUjjTnPyrjNO71yVKt9EfT5flcKD9pJ3fQBx3pe9HQE5x712PhLwJca0UvdRSS30w8qvKvcD27qvv1Pb1rOMXJ2R6WIxFOhDnm7f10IPBnhF/EN0Lm6Vl0uJsOc4M7D+BT6f3j+HXp7GqKiBEVVVRtVVGAB2AHpRDbxW0EcEEaRQxqFREGAqjsBUmK7qcFBHxeNxk8VU5nstkMoHUU4ikqnsci3PIsAZ+p/nRgUnb8T/ADpcV8g9z9Fh8KDFGPalAFGKkoTAowMdKMccVueHfDcutzebLujskOGcdXP91f8AGtKdKVSSjFamNevChBzm7JDfDvhuXXJ97bo7KNvnkHVj/dX/AB7fWvTrW1hsraO3t41jhjGFVRwBT7e3itbdIII1SJBtVVGABUtfQ4bDRoRstWfGY7HTxU7vRLZC0UUV1HCFFFFACGiisbXdft9EtgXBkuH/ANVCDy3ufQe9TOcYR5pPQunTlUkowV2yTWtbtdFs/OnO524jiU/M59vb1NeW6lqFzqt213dvuY5CqPuoPQf55pt9e3Oo3b3V3JvlbjPZR6AdhVUgkHHHpXz+Kxcqzstj7DLstjhlzy1l/WwpA9KTaPalYAuxVcKTwCc4pMY7VxnrINopcDvSUnPcUALtHqKNoHpSY9qfDBLcTJDDG0krnCovVj/nvTSbdkKUlFXb0ESJpJFjiUu7ttVVGSx9AK9G8MeEk00Je3yq96R8q9Vh+nqff8vex4a8LQ6PGLifbJfMMF+0Y9F/x710le1g8Eoe/U3/ACPk8zzV1m6VJ+737/8AADFLRRXpnhiEA15Lcwvb3U0H/PORk/IkV63XmviCIQa9eLjhn3/99AGqjuJ7GQc0Bip704kZqNqoRLq+p/bNKsbVkw1qX+bP3g3+FYIkdo3QuxCkMAWOP8/40ajqlvDeRWYO+RnCyEHiPPr79OKYsE5JkSMshBU4/wA+orK8XexryyVm1udP4d1BbaSyvGJ/0d/LfH90/Lz7V6uDxXhulM6TPBMDtn+XHpkf/WFeueG783+h28jkebGPKk/3l4/lg/jQhSNRlDoVYAgjBBryS6tv7I1qSExpJ9nl+VZBlSOq5H0Ir12uC8c2IivYL5V4mXy3PuOR+mfyrz8xp3p866HqZRWUazpvaSOcuL+4uZkkmkV2XOwFQQuTngVs6VqOrTSj7NaJNtYfMsKqI/UnA6kZ71gLK0akKuF6scnn0z9K7Pww9wthdma1W381dsLrFsaQ7Sep5P1PHvXm4NSqVfiZ6+YONKjflT7HPl43GX5zzQIowPkcqPQHFNGQAM8gYpDDhg3zH09BX0R8m3qWVmvByl5ccekpP9aWW7u7iPZPcyyLnO13JFVycDOB9cc04TvkjqPzoEJkryNwp6SvG2Y5Np9UJU0qurckYHtS4RjjIz7igCx/ad2V2PO7r6SfOD/31mmC6U5D2dlJ/vRbT+a4qDyFH3eM9xQY2Hf86AJR9lZjmxZM/wDPG4I/9CDVKtrZSAlLueFvSeHcP++lP9KpFHzg5GO4OKcDIh7j60WA8kuJbJGJF07kewP6VQknSRQpacxjoCAB+VadtpkUkwUWd/L6iF84/wCBBcVvRaN5sf8Ao+hTFyMb7mZmH6sPyq9EI4tJ0tyGiZ0cdzIBmtGDV2eMLJqV856ERuWxXVW/hm93gvbW0Q+8V3qg/wDHAzVuR+HyItrrZqPT7KZP/Q2/pRdAebSZuN37y546tPKFH86jjtrYn5p7eZ/7izsSf/HRXqUXh6zj58uB/raxKPyC1ajs2iA8u68lP7sUSp/Kp50Fjy9dKuZfuaUiL2eSRv8AA1ox+EdWdci1sh64d8/X5tor0JbQM2+S8v3HoZ2RT+AxmrUCNJJ5dvE8rDsiFjS5+w7HncfgLVZEJe5tIT6b2b/0EVatfhxt5ub/AC3pEjAfma9Jj0fU5D/x67F9XdV/rTzo5h+a51Kzg9Rv3H8uKXtGHKeeR/D2zDAvMzqOqnJrRj8E6Ai82Qkx1JdsfpXWOmiRA+bqFzc47RRBR+tNGoaUmRbaQZefvTSFs/gOKOZjsYEPhvw/bcR6Pp6H/bhVj+ua17exbhbKxwB0ENvtH5gVabXLpP8Aj2tbW2H+xEM/maqS6nqU4YzXc4U9t/H6YpXkwsXv7Hv8ZlRIVP8Az2lVcUhs7OAnz9VhHqsMbOayN6s3zSlj9anjs7mf/V208o9o2alYC4zaQi/LJeSv7bU/oapSTwNgRWsmf9uZm/ltFTNpt5Hw1uYc8gSFV/maF0fUbhl8u2lI/vDCj8zT0Aq/P1wo+rUm1y23I+gGa2I/DGp7f+WAHfzJckfktRvo32Zj9u1OzhA7JIWP5YougsZ8Nu9xNHCGfdIwTggdT9K9NRFjjVF4VQAK5PQrHTW1FZLdrmcwguJWOIwenTAyea6/tVIRXvrlLKxuLp8bYY2c/gK8shd4tOmmc/vZs5P+03X+ZrtvGt15eirbAjdcyqmP9kfMf5D864i8+WOGAdWOcfpTQ0W9JAt4t56tWul7yOCex5rGlLw2MjoCNijkfw54zWTa6lDpn2me5l2QiPLEnqc8Y96TVyrnSeItTjt7O3YuABKWz6YU/wBTXlPivxc+pSWWlJKVtrLG4jPzNj0xxjms7xN4yfVZSImLIpIji7D6isRprcQkXdyszYHyRfex6McdaaViLkd/bPLdZhYEyYwR1A61zF6XW4L7nbdzuY5J/Gu78Kz6fc+IreG8kW3s5GG89lwpCZ9Bk81n6p4bEf7iZlEkEvl5DZDjtz+FWot7AlfY44FZTgjDU5rfj5DyOxq7qOkPYzqIZFmRk8xWQ54zg0ulrDczlLhtgx97+H8aLW3F6GcsbyOEWNi57AVGeDiumg060nt1edWBfcysDyFzgcd6pT6dFFD5aATSZyJFYrtHurCpAxutPSMuaVo9vUOD6Ypysy9BgehpAMaPA4qezvpLOXcoDL3U9DUyWkl0C0WEXcFCtzz9a6X4f+FNO8XeMRot7dSQwmKRlkgA3MyrnAzx6n8KY0ijaap9oO9Qqt/EAcMfxP3vxrdbxz4hsrZoEvbiWLbt2SNk7ffr+HPavR5PgTZ2eH0y+SZwOFvozz/wJOP/AB2su6+FuuQQ+XHpkbqo4a0uEb9Gxn8qhyt0Oulh1ON1UV+2xx9lf6lfgxzy2kbOMqpl27s9mYDA/HinHS5obiKW9tI2uFcEus6MVbtyD7deRWnceBPE6EhdEnfIxgxN/wCy5/SsC90HxNpzlbnw3qMQJwuyCRkP0O001rsTJyivePUvBXiUXc93oV00IuLb54GhkDpLCe6kd17jt+FaXiPRLfWLFkl2pcxjNvPjJQjsfVT3FeXaL4J8aXuo219p2kX1tJCQ0dxcgQAY6ffOSOvGOc4NezG11CGztl1WGGK7kX5vJffGXxyFP64/nSfuu6OqlVVSHLLc8su7Wa2cxTJiSPAIHOV7EHvU2k6LqOuTeXpto8+DhpOkafVzx/WvTbDSNPv7yGO+s4blA/3ZV3DOP/rV20UMcMSxRRrHGowqIAFH0ArP2SlK52/2pPDUlSSu+7OK8OfDqy0x0utTZL67XlU2/uYz7A/ePufwArtuaXFGK2ilHRHiV8RUry5qjuxMUU7FGKZiRkUmORT8UYpPYaep5Fs4P1P86aRx0NdJI2jB2H9jQ43MOsn50D+yWQONChIPfzJBXz/1FuVlNH1azmEUrwkc3igDPaujI0kED+xIBn1nkp7R6SMg6HBn3ncVf9mVO6F/btH+V/gVvDnhuTWZRPNuSxU8sODIfQe3qf8AI9Mhhjt4VhhQJGgwqqMACuVh8TyRRLHFYQoiKAqKx4Hp04py+LZyM/Y4x7GQj+lephsPGhGy36ng4zGVMVPmlolsjrqK5P8A4Syfg/ZIiD1YSn5f0pf+EquMZazjX0BlPr9K6TjOrork/wDhK5sn/RosD0lP59Kj/wCEvn3EGzjXHdpWGf8Ax2gDsKK5EeLZ26WcX/f0/wCFA8WzcD7LFnrjzGH9KANHX/EUOi2+3iS7cfu4s/qfQV5jdXNxe3MlxcytJK/3mP8AIeg9q6O6m028upLifTIZJpDl2N1JnP8AhURj0kHH9jQAgdruSvNxOGr15bqx7WAx2Fwsb8rcnu9PwObwfejB9a6Py9JzgaNDjHX7XJSbNJyP+JLEPrcy1yf2ZV7o9H+3aH8r/A5zB9/yowa6UxaXsUf2PCQPuj7XJ+NNMelBSf7GhxjP/H3JR/ZlXug/t6h/KznNp9/yp0sMkMzwyDDodrDII/McV0UY0yGRJF0aENGQysLiQ4IOQev0oKaY0hJ0SEkksT9ok5Pen/ZlTuhf27Sv8Lt/XmYFtZ3F9dJb2sTPK54UdvcnsPevT/D3hy30S33HEt24xJNj9B6Cud07VYtKVxZaTbxFsFiJWJPpyc1e/wCEvu8kfYoRgd5D/hXbhMFGj70tWeVmGZzxPuR0j+Z2VFcaPF92efscOPXzGo/4S67JA+yQjP8AttXeeUdlSZrjj4vuv+fOH672pR4suv8An2gP/A2oA7GuD8ZxCPV4ZeB50P5lT/8AXFWx4tuScG2gHvvYisvWb641gweYkcBi3YxubdnH+FNPUTRhkZYeua5/Wdd8kva2Tfvc4eYdEPcL6t79vrXSSafJIjRvN8rDDfKw4/A5rMl8NafFBI4tYSEXdgFx+H3qiq5NWgbUHTjK81c4R1JUgcH1967bT7gzWyzp/ENx/Hn/ABrl9Z1LStInETaKZcorhluSv3gT059KueEfEdpqlzLZwWUtoFhMiK8gcMA3OO/8VY06UoJ3Z0V8RCpblWp108Zktcj7y/Op9DXVeC9QAv5bYn5LqMTx/wC8OGH5EflWdLYM+lyXqfdiKBgP7rd/w4/OszTbptPvElXO60mEn1Q9R+W6tkcz1PXMc1j+JbD7foVxGq7pEHmJ/vDn/wCtWsjq6q6kFWGQR3pTyKU4qcXF9RU5unNTW6PJLMoQPMuBbxjDho4g0hI6YJ/xxXW+HLiJdLulmu5ZEfcVFzgEqB8x6k4/QVV1GHRLDUJbZ9FDFeciYgMDz0oGrWEVlcW9tpvkGdWVm83PX9a4MLg50Z3bR6mMx1OvBqKd36Fi40C1lUPbyPBu5A++h+n/AOusyfRL6IkrGJl9Yzk/98nmoILma05t5ig9Fbj8QeK1LbxBIPluIFYf34jtI+oNd+qPK0MMDkrggj/Z6fnRsBGW611hudN1JAJTFIR2l+Vh/n2NVJvDtvIS9vKydtr/ADj8+tHMKxz5YluC31oDEd/pir1zpF3bZZodyf3ovmH+P6VRGC2Qw46+tVcQ9S3Vfr1pwkyfm2gfrUYBHIwQacCw7YH0pgPynHJH1p209juqEH5vfvTlXbyoXOe3BNIB24AYy2OwJoEiAfMp9+c5qqzOx+8R9eMU4byMEr9etQUTtMoyvJFN3YPA/MVGQxI+c/hxSGDcDkfTLGmIlJP99R680zegPf8A4CM0RQ+ZJ5UaBpD/AAoMk/hWjHoOoSsFNu6qe7jgfhnNIDPLA/wSf8C4pN0pGzOF7hWJrd/4RyOMKbq8aP1Cqq5/76b+lXY9N0pMYt726I6YBI/8dwKLhY5MjdwXyT+FSQ2ck4/cwvM3+whb+VdU8F6mfsGh2cIPAeZl3/lVWay8TSD99dpGvoswQfoKLjsZn9iXkce+SJIF/vzSKg/U0i2ljET52rxHHaFTL+oGKnk0yxgJN3qDzyd1hKlj+J3VJAuiPxFpl5dyg9Gct/I0AVDPokRA/wBPlYdMssYP4dauW7GUZsvDgc/wvPlx/wCPACtG3t9TQf6Hptjp69iwBcflmrJ0vUJ8G51iYjusCBB+dK4FG3tdfjXdLd2djH3HloMfkP60CW1RwbvX7m8k7RQcA/gBVoWGi20o80rNPnpLIZXP/AR/hSvPqQZk03S4YYxwJJFCZ/Dii4EX2+QIfsGlXje7Ls/pzUTPq7Ms1xdpZJ6EZP5Ef1qSTTdeuzmfUY4x/di3AfoBVWTQLSIlrzVwGA5+7kfmTRoAjyaW6g3uq3t9gfcXcq/kP8aibV9OtjtstJiU/wB+UAn64/8Ar0pj8OWpGXuLlvXccH+VW7a8AX/iWaCwHaR1wPz/APr00M1NBurm+tXuLgjBbaiquAAOpFbFQWvmm2jM4AlKjeB0B7ipj0rQhnCeL7kXGvwWwOVt4txH+03/ANYD865wnzdUI6iMY/Krdxdfa9Tv788q8rFc/wB1eB+gqnp4YrJM3VjTKRYvp2jsJIgcLKVD++DkV5f4+kl+zQ/ORF5m0IP4j616fcRJcQ7GJHOQR2rz/wAbpFJe21pHtIghZyN3IZuhI9T70k0JnmjSFAQB0qo8km7vVqWNjLjsOtOWLdEGVTtPTiqJGWkzpIDnFdlFqe7RZIr7dIFCGO5BBKhW+649ACfmrjVjO/kfSrcNy8alCgYYxTU2ti4TcdjXbTrcy4CKUlB2OGI8tyc8EdmH61ot4djmiubmzhCIECSqqna4Pr798iuZsLq6sJNkUoaJjuEMg4x7HtXqvhm6srrSNsEmLp2zLGW55/hxkA+1XOakjO2rPLpr2S122t5GwZfkUr0OPaozNDI/lwJJPM/GEBOfoK7jxLZxRXOoJcWv7uLLGJhk4Cgr0/zzXJaYNU0x7c2d0bSLUdjKqIHyeex52j696iwGRNp13HJ++s54T/01jZP5077FKrKWVdprstQTXbrS3tLjUrIwylWzJCVLbTkf5xXI3Gm6ikfn3KCFZ3ZV2AH5lwCuAfl+8p56huKQ7FK41GZGaKFvLUMTlfy/lXa/BaJR8R7G6kl8qG3SV5WPQ/IQB+JPT2rEm8IfZJBFLdtLMwG2OOLkH/aJNLYa03h8Wpsztb7cJWx8paNPl+bHqS35UrWQ0z6qn12JSRaxPJj+OTgfl1quPEE27DQ2+B/tNzWCJopNkkJLI6hlI/iBAPrUn2gIAQy7VGM7s/n/ACrO7Gbf/CRkruW3hfnHEhH6daRPEkpYbbQAc7suykenasUyoWxsw3T5/wClMBYk7kwDgYA4HHShMLHQp4hVz/x6kk995/ntqO61m1u7eSC5sBLC/G1pPvdwRgZH16isUSM5PBZdvLY3D6Z/zipIrS4n/wBWuF27dx4VT7fjipnUUFeTsUotvQuadd6Vp7M1vaytPt4eWcuT+LdK1F1+KTiK0mkYdkKnH49BWbDp37qP7URLLgbhGSqZ74B5x9akkuIbRNiJkKM7Ih3/AJVxSx93y0Vdm/sW/eqM2F1JGUFoZEPoSD/Kmy6xbQj5w+7rtXBOPzrn2nu7qQxwbFQZDbBxj6nFTQ6ckeGkYu3oowB+PU1LxU6etWS9EP2UZfCjWj1y2kYBIblv9pUBH55qx/aMPpJ/3zWRLPHbrtxyBwqDH/6qzLi6uZsqF2L0KqefzqoYjEVneKsvMUqdKG7uzo5dbsImKtIxYdVRdxFRnxHp0fLNMMc48o1ye3DcKdvuMcf57U1lkB4OxupJ6ge1d8W0tTmdri7l3M+9gf8AfP8Aj9O1KWPyq7yHn+N923/PNNSCT5lO3B53DqaVoXWMd1z83BJ/zxUqnBO6X4FOcno2A+Ykc4+uePanLs3N+8AJ5wwyKaI4wAyo+1Tnlcg8U7DEkL03cEgjriqJAiH+8xy3HB/zijKGXIZfl6oPr/8AWpdkjLvXAAwfbryOtNQOsaq43dBlYwufwzxQApK/P8uf+Ajv7UjN8+3qMcYBbkdqczZLITjk8lOv4dc/hTNzKo/eAfUc0wGOVGeTknuMDPbtmmYQf6xFy3GR3qVgVI/iQ/eL88UxIm3D5B83XngfWgQ4mEDd8n91Tj+tSAZ6DJPTPr7fhSBiAQqEE9aejM3yfMF/2tvSgBq7RgHLYHI6/n/Ok24ZtwA7Yx/nil8twFAKgenJx7imFMDC4OcElcdPXmmAMMdVPpnbimgsAADz7k04Jxu5Yk9AOeev0+tIuOAvyn1agBu5Gz8ysMdeopwPHynBz3FN2kfM21vf+lO289cNjbjdwf8APFIBCCGJLFD0wDTs8/fBB9TTFXYNxZQnp6CkIfJI2+4zQMl+7wfl9QO9IZFVl+bBPQGmbc9WUCmkoE6OSeFwpNAiXdtH3sZ4Xjmmn5ht7deXqJQQcR+WHJ+YleT+NPK5PG0ZOeTn8KYDkY4GY/mAx1yPzprAHqz4AzkcYpDBG38IGe+eT+VIVfoY84weuTQBNuZW9QO+RxUm4gZLJj0qBGVct5bZXvnIP0pnAO9EQEDB65J/nQFi4CjgEMUxgHIzkf41BdwJdWVzb74z5iMgz69j+eKRN25SHdT6f1qU3O3Y3lttJ5YLnafegDyDxZH57283A3wL+G1m/wDi/wBKzvCsq2HiK2myR1Vh/sscH9P5V1Pi+wUrdInIjeQoR/dYbl/pXDaGTNfXMy/MqxqNxPQt/wDqqugup9SeGY47/wAM3NqxBYs8L+v3eP0rhWOy5hkYY8weVIPf/wDWP1qTwn4mfTpEuShkiuIF8xA2PmHcfjmo52F3HLjhmJcY7HOag0R6R4TvTdaJHGxzJbEwtn2+7/47ityuA8Gajs1Py2OEvIun/TRf/rbvyFd/1qiWcr4us1Y295tY4PlPt6+q/rn865rZu7V6Hqlr9s06eADLFSV/3hyK89Vgw+6Bn8cUCI23Aj7v4kU8Nx8yj8DinMAByRx7UgVTlSPfigBCQ6YwPpVi1u57ZgUdgPTORVXyV6jKr7GnINo4yPeiwHRWmuIci6BXHR0XI/HHNaDQ2epRbysVwv8AfHVfxHIrkMlRkc+9KkjxsJEZo2HRkbaanlHc27nw4rDNpcFD/clG4fn1rKuNLvbVWaW2baP4kO8fpV2LXruE4bEwH/PRRn8xWnDr9oxAlLwPjOcbh+Y5/Slqh6HLDZ97dke3NIMMcDPp0rsZdL0+/HneVG+7/lrE2M/iKz5vDK5zBckEdpF/qP8ACnzIVjlml5+WHB9TzTt0mP7ox3FBi3KR5nPfgU7yW5z85647UgEEjMMF/wDvkVLA8ayI8kBmUHLI7Fc/jTRFtyPLUA84Jo6k/uwwHfdxSA37fX41i2LH9kQfwW8Sn9WP9KnXW7GXPmJqEo64dgo/JSBXNNIgbJ5x1wOtPErdMEZ9qOULnSjX7C3YeTpzD1OFBqGXxXcMSIoIkGON5LH+lc4ZXDHgfXmlCyFd25cH86LILmlLrOoTA+ZeuAR0iAX+VZ7nccyMzkd5CWNAQZGXGO4xzQYkAHzbu+cU9ALmnPpkcha+jlb+6qcr+Pets+JLWJPLtLV2QdBgIo/AVy6gdEduOw6VJlAuMZGc0rBc3R4hllJ3s8IHaKJf/QmJ/lUMmr2bAeZbTXLHoLi4PP8AwEcfpWMHZ+MN65Pejyn2bvlC47HmhRC5srr13CmILezt4+wRKgm1vUJM/wClcD/nnhazfLJHz5z+OPamhMknKg+uMU7ICw91LKN8k8jntmRj/Op4W0tQvnLdyMvJVWVVHtxVZI933gAegXPH6Uux4k7ADjp1FNIDbg1jTIBuh08of7w2k/nVyz8QQ3t5FbpbS7pGxksOK5VEBCl2Cuf4B97n6VveFoC93JKVAES4H+8aOVAdaOlZuvXhsdDvLhThxGVQ/wC0eB+prTrkfHFz+4srFTzNLvYf7K//AFyKoRx0ipBpwicblYbSD3pjajbW8QV3Icc7QpzzUOqT7Z4oR/Cu4/0rndTFzPfzqnyoDsU+wAH+NSUaN/4kMVuXt7fdIxCRIx5dz0H+e1cP4ivDpmpRQTSedI9uZrhweZJHP8gAMe1WJr37DH/a1xtdE3RWaHgyH+Jh/L8PeuG1DUJtSv5LmZy0jnr6DsPoKpKxLZoyRCS1EmAY5fkzjjnnira2sbWgjUqsu7HsBj9O1T6GBc6YITy6n5Rjt60rWEkZby2+Rwd2f1qhGfJY4cbTkYXJ/PP8qRdOcqZQAFwDk9K17O3edcbSAW5I7VpT2W+N0jJRlGAv3cr/ACFIDm1tIZGKEYQfMY3O1h64NXhAsP7kfNLgMJA2CV9COh7f/WqYR4lw0mHAyrkZDfXNWwo2OzpujxnaOx9aBmXc6jq1sDLC3mOuB5cq7ldR6HOQcds81S07xFo63CyXGnC0uAT+9QFwvrx1X8K17l3GHXDJIM4jbqv97Pc+386wzZWFzO0l0j7W43qcMh/2vWnzMVjpW1jQ0YzfbrY5HUEs38s1lz65YzXomSbbb7SsvUg4+6WGOucdPSsjUbNNNMdrLbxz2khLRysu10b+6XHXt1rHvZ5GAiWHyIVOFjGcZ+p602youx00+s6agmnt7w+c4Ztm1iruR6EcHPcVyRmaWZNxzgBRVc09Adwx61LbCTvqe+fDXVjqnhgW8rb57FhCfm+Yr1Q8+2R+FdaI0DYLz+g3SBv515D8I7iQ+KG04SBUvYHXB6Fl+YfpmveY9GgDB5maZ/fgD8ua8zFypQlecmvRnVRlO2iXzRk+QM7LcTSTEAMZCMfkB3q9Fpc0kY8zZC2eWxnI/wB3sfrWi9xa2vyeYiMf4EGTn8KqnUWnkKW6twOdq72z9OgrnjWrSVqasu7KlGF7y38iwlrbW8Zdl3Duz8/hj+lNlv4g21csQOp+VahWyubmTfcHy17ZOWH4dBVuO0trePdsB29Xk5x/QVhN0071Jcz8tik5fZVkUy13dqojPyNkMfuhffP8WP1qeOwSMjLvnOSEOBn1/wDrD1pLjVLaFRtYO3Zc7Rx7n+lVri7mkLDzAiHgBBz+vP5VrCFep8C5ETKUI/Fqy7JPb2ybFwSn/LKPr/n61n3F9cSM6xlYU2/KSNxPuCPaqxdN2wcr0xwv/wCuo06gbC7D0AxnI/nXTHL4Jau77mf1l38hkjksxO9iCRkRPn39OaRtv2dXRi7f3PLZV3E89fw71IY0V2AIwR1f+ZphDxtuwxGMbtuPxrenh3B35mxTrKatypCg8YIw3DbSdx9/8ig7VCjO1eNwPT2/CjerkIu87c/Lng49KF8rH9w4zhyBnpx/M10mAw5XJy7BM5BAYL+YpyqPlIc/Q/p+dDbW2YOSD95Owx370HDklsbmwOflPTr9ffpQApJMnAYKpGRtByPQAjg//qpYyrxruy2AflTIzz+tKFWRgm0KNvY7uMc4NKBmQcqGYY3fdH05/wDrUARM2MspAP8AdB//AFUNsLEtGOgUep6ev+elP2nfuVlKt6j8yDTNwJ3Y+X+Hdx9Bt7D396ABuQVAkUkHIJOV+uOKaBtJJPHPzY/z7Ck2xkt8rJghhhtw3Dp+I9aPM2B9xJP3iBwfTrk5/wD1UxDDncxIKjG3qe/T/OakDKuPMRmXuFx83P1oQJvRwSq8lRu/p3oV2RQG++TuZGIztx/+ugBV3tgK65yQo29P8KFBLnc4O72wPqP8KRiVRt7n5sqc5P59hj6UuPkC4BXqoOfy/lQA3COhw+4g+ucN74o3ZDblcZHQ4xinkZHy7i3TOM496E3MxUdDn6e/+e9AEe/jcFBwO3fv+dOfIVtuMNzmlZgFHDLjPG3aB6U0ruUE7g2SWBIx7UAHBYeZGCvYEZx3o6qAvAYEqAPvU44UfKMq3K7X6j8aaWXHKcnoGH4/l9KAG7s9QopfkzjAL9ee4pMIPuFU2gDHenYd8fMQT2OP60DGk5bgfiaXahGR25waYA3JO1mx1Ukj86dyE+VfmJ6evvQIaclRg4PoVpTnjO0EcHFOwuGyilW4yT1/SjaGOG3nPHPHNMBpPGRyPZsfpThtBj2hjjPQUvlMFwsZJ6AHrj0oxJtO1lG31HFIB25ePl5J7mkAVuxXHoMFaRo8hsEDnGMdPwoBKN/rMDPGWwKAELHgsqfgO3enEhcHIDA9mwaaW3AshYjnJHOaPnbBBXtnf9KYHnvxOvXtFjt7ZXeS8g/euqk7EViM/Vuntj6VynhOwkuNL1e4jeMCDy2ZT1IVWbAP516v4mtvtGmo27b5UikHGMBuD/SvPkuZbRrgWF0tvNJhZ41278jpnPQ4PX0NUthMba69Pp0ZMTxmL75SRcj3x3FdzFfwo43ZVTghuox2rzqK2tE2xoQD93B4/DPSumheX7FChQ70QIc+3A/TFEojUjsLK9SOcT2jhzBL5i7ehKnkf59a9et5kuII5ojmORQ6n1BHFeA6FM8M8iycKXVh+Pyn/wBlr2Dwdd+dpBtifntnKY/2Tyv+fapRTOjrgNasxZ6pMozsc+YuemG5/nmu/rnfFNp5kENyo+ZW8tj7N0/X+dMk5IlSdvOPY9aP4QOg9DTnGMgjtwR1pm3b/Ceec9KAHqWUnqMUgRSA2RnOQcUm7phSPf1pc5BIZh6igBzIxywRQD6VBsV1+ZAR6b8VOF2nIPysKaEU59OxPagBEUJ/yzYAf7WadGdrbw3I5GOv0oG5B/s92pS438u5HYHnFAD4LiaKUNCXjPUujlT/APX/ABratfEMkXy3SLMP+ekfyt+XQ/pXPhiCMjbu4GAcH8aUkeVv4U4wyNSsh3Ghi6jAxj2xxR8pXqCO5psY+bbnLenelaNh8zKMf3uvNSA7IOSP4aRU7iPnsaVCD8obO3tjFDqd7Z/A9vyoEHlsBk4ANPKAAk4+nSmhR93bkH06Uirtk4GVPqDQA4KM/fGT2IxSK2SRt6dae4UlecHb3/pTeMj5WII5ycYpBYbucfKPLHoQTmpQvTIHHWm7P3jZAGPbOPwpoViWL8nsdvSmAjLG7KTJKOfu5wD+FG9sD5XHoeCT+FOXcuTksvYkbSR9KEiWQ4ZcAk56j+lAweUOpZ+MdMqKAXLL3GOMH/OfrUwRTtG3D9On6UzZJ5h8xISAOoLBv14FMBCqqBuJLei0pQHCnDOBlSopzAbjwGJJ6jrSbySy7c9uCaNQDauBkcn+EN+tKdxbYFU56YbdigO/zZCLt43H5sHpwKbvGDgZ2jr0FACgY/iD/wB7bzj2H0rsPDcATTPNHWVs59QOK40pv6ZII+UDDfyr0S0h8i1ii6lFCn600BMWGSK878Q3P2zxRPg5S1RYhj+91P6t+legTypbwSTyHCRqWY+gHNeURSu8E97L/rJnaQ9+pz/WmBQa2S/1CQ/NnO3IPYVT19La0i+yNKU85DJczg48qAffb/eb7q+5rXtWg0/Tbi5nVTGF+dm5+Udse5rzLx3f3MZ+zzr5VxdYuLlB/wAs158qL6KMn6mjcDl/EGtNq18ZtiwW0a+Xb269Iox0X/GsGJvmOe9PYmaXBPy+1O2hCaog6Tw1fCC4KcneOx6CuoMcV1HtiIw/TeOMeledW8hUgr96uz0hpJoQ8jYCjg/0oGaLQwW8nylXO0K2AMfn61YTDLczeWG2t5RBPJHsfp0qmHjMe+RgfVgflGf/AK1OtGd5sEnZ0I7KT0/lQBK0NqJmUKsiuTjPDepwfWs7VHlkglihDHGHVUbkr6/WrhiaWYwtyME5/wBrsRXLalceQH+cebgo4A4J9x6HNICxHeIkOd8TM+ThuBIMdf8AZbqDWbJqyOrwxOWhbDKkgz16gHtWNc3TOAAxz/F/te/1osrG6vSwgiL45pgad1e/aLNbTzTjZ8uR1IPH+H5U6SQ3Hh5kcBJLaXzGQ8FmOB0+mau2Xh5xcILpWMccXn/d4PPI/Q1naxH9hvp4lIaCcE8NnLfWgDcl0uxEMD/Zy6bV2k9ieQM9/p2rO/sr7QJWjCqjOwTaMnjvmtfT4ZZNNtRIRsjiD4+6S+3A59uOaiSV7a7KbifKBADd+ef8+9IZU8K3U/hrxrp8kgAe3uV3A9GU8flg19JNHqF43+s2w4xtZdueev8AkV8+axGl/DFcRP5M8PzRnPBX0z9ea958O+J4dZ8MWWqSMPPli/fIv/PRflb6cjP415+OcotThG7OihZpps0oNNjj5kLOewB2j/GrJnt7ZcHZCp7fd3fT1rMl1gCPKeXEvGMncev6flVYz+a4bep34XBbDDk8n/PvzXJHC1qzvWkaOpGPwo0J9WUEpFF84zxICPocDtWXPO12d0m/j58HIT2A5wPekYK4bqQ3QcnjP+OaRQEfewy+3kgg4/Ou6lhqdL4UYSqyluRmZYwf9FnLIM7Qnv2y2SetWTIiyBREVCYxJwvApg2JtfLA8c5DGm7ncurxszbcBsdCeuTXQkZiiT5zj5SOvQ8H69KjV3lLNh48YypPB9QRn0/n3qVtgdSjY39Pkz2//XUbJkbFTOfRAD6454//AF0wBrjIXZgDjA6cUqMrL8gHIx6/r3x/hRLteIsrPtGDtAx+HTkdO9BX5d0jOB2JYbW79eo6UAxm9cDjdHvDYz8vpn3pQ2AyI3K46qCPrxTiuGHKqD/CEHAz7dKaAS5CsJSF5HCgc+vTsaGFxA5+XKfM3Jz6etJ5yqoXBC4zlf4fwpdmVOSuGYHBHX/64pQGjfaXyBnAb5sD0/8ArUAKsqGVuNwOF+UYI/8A1/40NL+7xNGSNw+cnYG+n8qUBJZMqGXd33bQB+Ht3prIh+VeUXCn0Zv6duv5UAHmbQj785OA/OMfT9PWmq2AvzMNoPzbTjPsP89c0KoJ3KWJA53KOnTr9aJA6AEOjFThjxjnPfI/yKYCcsygCNxuyysSu3296RJd2MspGOc5/EU4h1jG4lAox2+br71CLqJlbLj5SVOOffHHf60mA5FR9pDxY5GCvT/PSpGZXyN6YUfdHp/+v9aYxyQy/MSaVZEH3d7AYK85NMBAwGOcqcMfLxwO+M96l+YY3vvDcZQdvf2pkkcYYK5B2/ONv38d6EiIyS+T6/jx+NACvvXy8pvIb94M4xx659e3PWk3NJHlk3r8zEjIx7daiEShWKAHp9Pz/GlZHbGWyoIBYHafz9KBEjCQOTuxu/u8kcdR6inkNv2Y5ychc8fjUKp82GfO3tnj/P8AjRtJDH5cY7Lwp/3h+NADxGxkyyE5IX5X3YHfI7UoT5shmJ28naT26UxQcHaipGnGX6dP6/SlWJmYIhZ2UhiSoyB2/wA96AAQu6jyyB6blz/WnlNiBmOSMfOTx+dR+XJGPmbGOCV6j8qGXaAuSevTGPcUACud2Ock5AP09KkIZWK7VAz2phLdScDHAbJ/EccUjD5cNvVmGVOeo9fegCXfuLoWAwduR6evtRFEsahRMwTbj7xyPxOc9aZGjHcrouzPUruA4/WgZRtzcE4wRnP5elADgpz8vzdic8j/AApCW+ZPKGOOT37UjKxmO9/MOc+wH86kDbydzLnPzcfe70AMIwq7Vj656DPufajysryML6Y/nQWIAO8KV592ah33SKih8nlgQOfbrQA9VzyACQOqjoaa6k4cZJA6Fuvv6VF5gdCo+U9PnQHa3YYqRsFgTFjB+4Dzx70ARXdv9psbmJioMkbAc98cH35xXJDTdN1W0hlvNPtp22bd0kYLj23df1rsMyxsjmUAZzkx4Hr1rm4FENzd246JKWX/AHW5H86qIGO/gfQJZA62s0JzyIbhlDexzn9K65NEsDbJHFEUCoApLsWx/vEk/nVEn5W+hrqtQtPsl2ImOB5UZVdvT5RnH4g1UhI5K50aS3V5I38yLBBYDlR711ngzUfK1SEFvkuk8pv94cqf/Qh+NVsBlzty3QoCM/zwaybXfZ3UkcLYZGEsJHYjkVCLPZh0qrf2i3tjPbN0kQrn09P1pbC7S+sILqP7sqBwPTParJ6UyTzMKEc+ZnfyMDqO1Ip7DeGHc84rX1y0+zapMwC+XN+9A9+/6/zrH2oMMDyPToDQArYZWBJ9sUgDKONpP+9mk+bcc7OTg8ZxTxLtb5cjjjsaAASFV5IAPXHU0B9oPy/L9O1AZSST8pHqMU4AEE/Nt6cd6AGk5KujbWz/AJ60jKoJLA54PAxTgquRnAHpnH5UPEpJZcZC9BzzQA0bAo2cOODhcEj+VDuRxwRnGWUdKcYlCruxuwCQTxn+frRKhjwyMpj3bct1+tAFTzNx28qT2K9aegYsTlWkP3s9vb9akC7eVVduM4PQ0KjoB5R6cbVbBI/rWYxCkynczhFUjocj/wCtTlKc8PJ33cgUzdk7nB5Uk5y1LtMqHLEg/wALLj8R3pgLIzkfICD9Ad30psLOrYwF45+Xk59qj8kJtLF2Ofl4/Xp1q0ijZgEluwL8/nSARiMrjc3f7tAaSTcDtVe4x+uaGCLwiZkHYNwPU1KQwUKVjzjqTnBoAjjgDIHSLdtHOG25p81uwdSQnryecUsrOH+Zsr0PycHj9PrSSCMISwI3dSOf0pgLt28+ZEGOcAnBI9elNDyO4wod9v8Animl8qFUO6DgAHrTvLckERMM5IBPIH+NIAeWV2+ZlGByOufoO1RyKm3cnBx0HPNSspzkKTtxncu7kf8A16SOJQpLFiCM5A6n6VQAVcqRsIyM7iuCaZIXiUMAY1/n7VKUTcwwjkKSWKdvypomJbCnJPykgH/OKAIzG+SzphWGRnr+VTCF9uNu0f7u3n/Ippn2nYUYMfunuDTcgsDlg3fJ5YZoA0NGtln1O3QYZVbzG3YzgDP867uuX8Kw75bm62IOiBk7/wCeK6imhHMeNb17bRjAgH+ksI855x1OPy/WuJvv3VnDEByxHH6103jWTz9W0607Rq0rficD+RriNaeTUdZh0e3cqWQNcyr1iiPXH+03QfiaYyW2UanL5+FOnWJyjHpNKo+Zz/sJ29Sc9q8V8U6r/aus3l4M7ZZcRk/3Bwv6V6h8Qtbi0bw/DotiFikuARtQY8uLpj8en51487Ak5AJHc8049yWQogWMfL83UnPWmHrUnmZ44PaopDyRimIYkpil3dcVsRawxGAATisTqwPapIvvegoA63T9Re4nVH2nuAemfpXWbgLVS5Bkds+mBj26nmuB0WXF/CGfaCcHnGfxrpdc1N7a4WGHgJ29OOx/qaBo0wySmWRvkUAYPTH+TXKahpk9/M7Aj3Y9z1qey1WTbgt+7T5jx972x6VQ1PWZf9UkmAuTkd+KAIJvDsmmxLeXiK0bMUWM/wAWOufTrjPqK0LTXltHi8iGJY1jEfKgnAJxzTtKT+0dOaG6cgOdsWST8x/l9aw7uzl0+7eF0I9qQz0mxntLyxSdkwGyG2jpuHP4/wCFc5qxOn+IktpFjkhmjXG5QRtbP/16NJupl8PXnlHBUq3XP3eenuM03XXW5020mB5jYMHI6hu3r1HShAT6+5swwSQbVKLx12/wjNc4b3zTv5Jb1ra1SJ723iuFddrR7wBj72enHQc1z1xY3FvvLIwxyARjNAEy377BEXAGe1et/DCVLjwrKjSSLtvHHy8/eCnsa8O35PHQ17T8KkdfD1ydpIa5BBx6LWOIoxqw5ZGtGq6b5kd6RGRtM1wgz0T5WP0w1E4iaMx/ZEjf5TmRlJ6f7P49zTUVThTDJ2KhUJ5/n/8AqpfLdMokJAYDttrlp4KnTldX+80nipzVmQR28kQUvI/zcYI+T8OST+lTJu2jJG8HIQclfbOadskU48ncG7IrAY+v580m1pIwGR92cYMX+Pvj9a7TnHLFumVh8+PmGBgH/H9KQyKoyz4ycnJ/SmsplcALIS3QFTznsOBntTFQiQDy2IxgYkY5XryP8mkA5WYs5hXAPzZbg9t2M9cU5ZCHWIjg8YfBH1OPTih42G7Cgc9SnA9SfepNsqSMZGbknhIuR6fl9KAIS67yzgMwX5lfk+vX/PvSiKJWkOFGTngcAe/9KCzspwWU8D73AH94nH171HgkfI7nJIZSnGfXkZ70ALLtDMQS7FT0PLH09gf0pE3kBN6lsdQ549hn270/bJkMByPmwq/Mo9c0iB5VXYSVb5B8m7Prxxx9aQDldY3bzDn+pz2qFMBSu/5d4Z84GfYdv8KmfO0/K5CjKgchf/1dPxqGVZtjOjyRHb8uFEg6/Q59PSmAxmVw/wDqyT91GJ56YHoe1SFM5VEDsMKcNjGe/wCPSh5LohNkUKbEByyyN6e4wc9qSJp0lxKgDLyTBnaOevzZPP40rjtYQF5lnEJbMTfMUGCrHOAflA6Y5HXHfmniSQKxZWZiD8jADP5dvSm3Fv8AamTz7Ys6nKF1LeWfrj9fagqxUNsbHKgbTgfh+FGomRO7MsZLxr0+6T8o/EdaaCed0mNi4BJzkD3pxSZ5U3eYE6MUU5z6VIN8YVzFKdvCrszj8cc/59KoQ1fLRw/mPxwMHlce5/GneYyxKgOF4+XjseN36fSo/IZSGWJ41X5Qq/KQO2PX+dPW2cTPiGQsGBYHLZ68cj/9VADtsQ2qWGP4XbjLelNy6Rk4yTnlzz04Ge3bijyZC5KQyF8ZEjLtHHSlMcuH+Ri0Z5B+ncf/AKjRYBN0rIpYM5UKxxjuf05o/wCWxDHLY7D/ACKGSZFRG3qzZYKAw3Y6/wCf8alZWKglCFDdCn3gffp7/pQBES6um3cQT94Z/AZ9aaFQYCkEjj5n6/1FL5Eio7CGTzn+9+6z+v51R1g31jpk15ptrHM6fvGRlJDDuRyMd6LBc0AA0rZ2yMW4KqATz707J3Z2SP8AOo+fnj69a4lfiPYbS39m3zNwRl4uP1+vFOT4hac6SrNDqEe4YGIYZSeMcN/DT5JCujtBkDBOeDg84A/H05psjCPJV+uOGfGB71xkPxG00yqkmmXMFvnHmLIrEL7r279M11Fnq+n3hR7PUbeQd1jkTI/M5z/KizW47ouffGN7IvcgZP4dP8miZo4XKqcuedpPP0GB06UvlSKrACQLt9OlAicM+5CQ2chhgD3wMdqQDTzKzCZlLHkZO0eh9qGbdjcVbPIbHNAx8wcHIJYfLy35+nrUflyvgQ3IVs84iEqt/vA9PwNDAcOE5JZTwBng80Y2gbiw+Yc9QB6DP86ncAMcDDbf3gHp+dNEG1VALsw5PGPxHXP+fSgCMqVyMyAjnhz1/wAKaifKyyspQMVwjFtvqD/nvUqrIGIC47beB+dNJZZNgLruxwTuwOnYcf8A16AAhWI5ZzxgMecD8aaYkbaApKH7vPT604QZmKllBBByw3fn3pwVgu5D5gyeMHt1xQFw2qItxRgoOBlju6dMDv8AhWBqyiDXYZO1xb4P+8px/IrW/wAhTuG35OC2cY9vxrH8QrixtrkIUNvcDjb0VvlOfyWqQDtOgN1qVpb/APPSZFP0zz+ldv4qiJmtblZPLBDRk5xjByPb1rmvCMHneIoW7Qo8v6YH/oVdprqbtGkfAJiZX5PH90/zpzYkcYUZZBtWOVG6Bjgk9/8APTpVC/HkS29zlvlOxvYdq1AjpIFJWRmLA5JG0/Q1DdQNdW8sOQ25flwNu0jpxUjOt8G3e+znsm6wPvQf7Dc/z3V1FeZeENQMOpWkjcCXNvJ9T93/AMeH616bTGzC8TW/mWCzj/lk43cZyp68d+cVyYjLsFYqpA+Y8c89a9EuIUuLeSF/uyKVPHrXn5g8kyRSbmZSUb5uAV+nf9aBFSTzdiqkLSO3QbtvH1NQm7ulY7rHaV9ZgBn8Aa0FibB2lj3KAAlfc4NRqjoMDK7u6ru3DHWkMjR32ZwA4+UqCG/WnnK8N6/3ev5GkePP+sRCQNvCjK9/505CxYKCv0Ug4OOP/r0xDQ5VOSvJxtJ5p5kCOjPk578gHjpSSR7eCuf7gfq3+fal4AXfkluMMc4Pf9e1ACb+M7t27+FuTTVkKBW2j5sgkAZqZ4HEmzZt2jpjlhj0/wAajjjDZ4PzDPAwD+HfigCsjOfkdlZiedjZX/HNThTn7hLEfLx/jShACzEorjBOBkkezCpAVLGT92/JBJXcfpWYxvlgKfM+XC7vvdMfWlVI1JdfvD1OcCo1Ku/l/dzn77H8qn3BAWjDKm7r6juf/wBVMCBfnRlDMEYkFVTjg+9KqlFLM7quwqCnGP8APvUxLsf4cr0AzmkV2jkxuVD93OOKQWGNvIyBtY4OSozj/Io3bmCDJ57rtNSE+azMHY7W+UEHI4H+Bo2bhl3w3YnqfbOeKAG4TczgM46EZ/Pmk2Zk2LCFXd6j06UuwtgkF/4c/LjPr0pQkcgQBVYnn2Hb8+aAGqE+VsBOcYcgk+/tUgI8l8Fzu+6VU8j0A/rTWEaKY33rzwCu0GoLgorIA27GD984z6Gge5OxXcSATjjBOf6cUPIxdhLGVYjoj5zx344/+tUAuCColaIAfKT8w49PcYqXeI4y8jgJjn5uPrTERMCMbi/AIbPU8dz6UrFwCzAqM/MTn2pxu1VTtALMBgY7fj1zSxnBDnbhVJ5yce35UXYxAQnHGR8zk9vTmo5LiN8HyB0JAZcc5qYnZuRWdth4GMkE45/lT408+cRoWLSMEztGT2/lzTEdf4fg8jSISV2tJmQj0z0/TFax6UyONYo1RRhVAAFVdT1CHS9OnvJjhI1JA7s3ZQO5JqhbnnniDVB/b97OiedMXFraxZxvYDn8B8xJ9KytCgWyNxeXE3mSAtNPOfl3t1J9gAOB2AqSztZYrSfUrrDX0yMFGciJSfuj3J5Y9/wrM8Xzx6X8ObuQsVnvJBbQDPUfxH/vkH86iU1EpRbPJvEetnXdcutQffskc+WD2jH3f8+9YDMdvHT0FOuWO3FMVTt5rRGY2NGeTjgDrRMeelOAZDnoKbIOAadwI06/4VPEpLDioBjn0q5aSFGztRv97kUAathFIs6GFY3f0LYz/hU2t3HlPtkZGnxlgDkIPrU9mvmMrcA9TxgVka3C3n5jf5F53FcHPvQUVxcIFLFz74zVAyiWbrjPc1WfOcE/rTakR3ekypLZLCmeG3DZyRU96kV7YFJmErQfKkyJnaPRsdq4KG4mhcNHIysOhU4IrqLWa8uDFeRXK+cMIxCEFv8AeIHP1NMZLo0pt9UjtpZCttcfLJgK29fYH9DU2vXFtBpsUNjO8ltNIfL3xlWQKeU/DNbCWZCxPKLaN1LO0czgbiOTww68dq4LUdWmvo4bcki3t8+Uh6/UnuaANmDxO1nZeSI1LKflLchfoKzJPEF09z5xbexOTv5De2KxiaVRzSA3Y7q2uFYfZYIuP4U7/nxW5DdXNlYW8MFzcQg5ZhFIy5z06EVz+jIr3KqzYyOAfWti4cSXBKLgKNi8dhVwWpMmWW1S+k+/qF64znBupOP/AB6nrq2pKONRvwCd2PtcnP8A49VAA7uelKePatLEF7+2NRG7OpXx3cN/pcvzD/vqkbVdQcYbUb1l64a6kx/6FVLdn19eaXnPT8aLCuy6NV1AA41G8ycc/apO3T+KlGrakVwNTvwP+vqT/wCKrPLDJwRkHkCn575/SnYNS5/aeojlr+844y1zJkf+PUDVtSGMalfcdP8ASpMD/wAeqlnqT1PU0vXIxgUWHc0V13WFGP7X1FV9BeSf/FUx9a1RlwdS1Agf9Pj/APxVUeg9qDyAMdfSlYV2WU1C8zxeXQJ9J3/xqf8AtjV1P/IS1I5GOLqTp6ferP6qfr0PTFdwuo/D0xxeZoo34G4LayYz35380padClr1OSfUb9/ne/vTt6M1xJkfrSpqeorjbqN8D0DLdSAj2HzV1qXnw7KHdpkobpj7PIM89ciTH/66UXnw6XH/ABKpCp/heCVm/H95/n8Knm8h28zkG1C/fO+8vG9jO5B/8eqNb28jUIlzdKo4wJnH9eK6z7V4AJDjSbtj0KGJ8frJj/ClS8+H7MN+jyKSf+eLn/2enfyC3mcodQvwv/H7elB/08SdP++qP7R1Jl/4/wC9IPrcSf8AxVda174AjVgmkmRsdRay4z7/ALwc0z+0/ATyYOhFEH8QtmJb8PM9e1Lm8gt5nKNe353Br2+IbOQbiTnPX+KmLc3aoVFzdDIwcXEnT/vqutF38P8Ak/2Qw+lo/wCGP3lPN/4AUIg0cno277I2M/8AfeafN5BbzOQW+vjx9tu/p50mP/QqU3V1kg3V1t/67yf/ABVdedW8BZ40ckEfx2R+XtwN3OakOreADHk6Mu4k/wDLiwOPwai/kFjjVvbzkC5uwrdf9IkwR/31TVurthjzrrPXHnvg+/Wuv/tDwB9oXOkOwJGD9kcBfdhvGfwqSK7+He0M2nYk5z5lrIR+HzUubyC3mcat5ej5VursDrxPJ/8AFe1Buro8G5uSMYx58n/xVdil78PMHOmMfb7FIo6dc7qT7V8O8APppDeotJef/Hqd/ILeZxxu7vH/AB83ePQXEg/9mpJLm8kXYby6weSPtEhH5bq7B7n4eIrbdOkYcYAs5QT7g7/50v2v4eMyqdNlwerC3lUD/wAeo5vILHDfMAOCxpw3HsT+Fdx5nw7Jx9nOwdzFc579s0qS/DrbhrQ8+sNxn+f0o5vIXKcKSe4xmmsd24zxxuAAFwDlAPoDmuu1GHwPNCfsd1fWcpPWC1kdfxVz/IiuXuIreCYpb3f2mPs/kPEfxVun5mqTuK1hsV26DbDcTLt/55yyKB+GasDUr8McX95z6XUn/wAVVTnNT2ckEN5HJc26XECsPMiPR17inZATf2jqWdv26/wO32mT/wCKpv8AaV+P+Yheg9f+PqT/AOKrsfP+HTMrfY0w3VTbTA/5/wAOKpXtj4DuonNnqlxYTc7cwzOhb3VlP5gio5l2Ksc8ms6oiAJqmoKF4GLqQY/8eqRdd1VcY1jUsnnm6k/+KrM5BpDuPT5T+dXoTdmtD4g1uLBGs3/GODcMy/kxNdho3xC5WLWI2yelzAP/AEJf6r+VedI5B5ap4bmKJsyWVtP/AL/mKfzR1qXFMEz3CC4tb6P7bbzRyxug2yIysuf9ruG/X2qeSeONQwc5ZVYHceR05wOleW6D4p0zRphImhTxlh+9MOou4f8A4DIMdff8a6iD4h6I8iGQ30fX/WQbgPfcrHjisnBo0TOs2gjCYkLcL82cr7H19qg1e0a60q6hUHc8b4yMYZeR+oFU4PEOi3j4t9as1d+3mKpP1VgM1pR48z91iXack/e3AdOR2NTqhk3w6AuYLi+UZBijjB+uSf5Cu4mgNzazQMOJY2Tr6iuN+HBW3s9Y07YUks79gVPZGUMn6V2wOCPahgcE0lsyr5vlgbcfvHXO4fqaQKUZioZiFyuQeKl1K3jttYvFCjIdiu4buGG78OvH0qkkriQuC2RySUwP93IP8/woYFCNDb6jcW6HHmfvYiOOf/116zpt6uoabb3a/wDLVAxHoe4/PNeValF5Jgu1QqqPtPzFgQf9o123g27zDcWRP3G86Pn+Fuv/AI9n86OozqT0rjfElu0GpeYCqRSrv3N03dD/AErs6xvEdsJdN84AF4DvBJxgdDTEcZACw6lNr9Y/u/n/AEqfO1sEyktkqY1PBxgnGf0P9KhZ9zZeUMxQ7CrYyv8AgMUh27doORndkEfMB6HvjrSGTKAInC7QM/Nnq3fFRysEkIIf0+7xjrTY5ot20yeW/wB0YJA46cjr/k1IruWUBABuzu7tj3PSgCJhGpZflPc56CnbWjkB2xBW/wBrO7uR6dqmaN1yz4Ub9p4X0yfy98/rUSSSZVFByeQCuCeeh469OtDAeJMn5I9m3oE/T9KR9xfJO33A+96UkoBLBnVmzwM9/Ue/+FM4Byyh13ZYgjA9OO1AhyxITwB82Nu4nGfagt8yKixovf5cH35xj8RTljDMu4tjopznbTYnKnluuGQGP73P5moYwV5ml3QxsuBwufXFIzsSpdty9Blf8+9CRdEXJTdsA/iz/OpFQRsdgX5uchgOv0oAijEgUKqgL/E3zcn9KnLPs+aQJljksvQfj601nMiv0+XjrjH58jPFNMscm0OnC9GNMBw2NuzhwegxuGO9NLjb8gzjnCrjFP8APBbAT5echR+Xao5J3xtYkAqMDnp2/D2pNpbgk2NKZXmIucZ+8pwMdxQMjbGi7F25OOn6mlKqwYkBR6kj7x6cZ/lzQCDGTmQ7vlYYAGe4xyf1oUojsxWkVQVJCluSA+fzqJmdtuDlV3FSrA4OR26+nNTBMAZVx83R+MexH9aD87E+Zwx9P5GhgkKQXYOjSEtznbgj0xTIgVH3kYg4O8Y7dQexqVAXiGGeP5suwwpP4GnynZHtLLub+ELuz6ZApXS3BIrGI7j5jfN0Yb+uD0JqRkAJyG3cAKHzx71bg024dUdlWNu4ccj+v4Vai0qNX3ySO7+q/KP8a5qmMpQ3ZrGjJmMTI+3LFE9F5+n6+9bXh3T5vtyTSiUqvI8zHPofXqauwwxxYEUYX6dauwXMNpDJPK3Vgigclm/ugdz7VlQxjrVVCC0KnSUI3ZpyzJDEXc4A/X0rlvFYLaan2j/XTyqscfXy1HJ/4EcDJ98V0MEUspE90u1/4Is5Ef8Ai3v+Vcz4ll+0a7BbjlbeLc31b/6wFemznRz2onZFBAnfn8q87+LGpgyaTpEa4jtYWkYhgdzMcZ/IH869H2fa9dji/hQhf6n+deP/ABU2xeOb+FGJSJIkVf7vybsZ/wCBVhzRdRRe+5bvyto4KQ5lwOgpw/KhE5LE0HA781uYiyFSqnncetRzHgUYzn1okG5A34UAQkZI9KtW77SM8fhmq/Pap0AIznn0pgdPo8qPKANu773zdPwA6msvxLdHzTD5mcMTjHSrelZEZkfnauQBkZH5VzF9MZruV88MxIoKKxNJ1oopAOHWuo0GTzIHheNGVuu7P+T+NctVu2maFgQcbefvYoA6nUrqaa9htGLeTH84KqTtAHVc5/nXGOcsT0yeldJK4msr+8lYpmFIUIQNubPru46dRXNHrQAlOWm1JEu5setAG7oKEPJNk4Ra0GB4ZsDd0rt/h14R06/0GW61G385ZpdigyMmMd/lIrrn8A+GmCldJXYO8c8q9OP7/wCveqjNIlxPGwPfNLs55HWvXf8AhAfDsmR9jmiYYxtupc/q1B+HeiRDcFv1YnaCLyTcuf596r2kRcrPIguCSCCB1NLyBj/9detL8ONAztiju26YDXxUKP8AP50w/D3w8z+XEl625tq/6Y344p+0QuVnk7Dcv0o2kjHavVl+HOiFYihu3BGdxvMBvTov9aefh1oDmN1F9gEBj9sbj3+7zR7RByM8n4DY4yOmKTkE5/xr1Zfh3oTfKPtobkZN4Sc+4204/DzQUVOL9lLDk3LL+P3aXtIj5GeUew5/HpShNx/z1r1h/h54eBysN8U/hL3rAk/gPShvAOgbBIILvHQlb2QAN7ZH+eaFUQuRnkpHzc8evGaNyxg72RSfVsV63/wgHhzyyPs86uvd7ibKn8xnr0q1p2gaVpEMotQHDsGYSozjjI4Zw2O36Ue0QKDPGDJCeDNH+LrS+ZHu/wBdEGx13g/TvXvwsLOMc21ofmVtpt4x9O3B9qjWyjVyPsVuBjABgjH4kYyQeaXtSuQ8CM8Wf9fH9N604SRnBEqZP+2P8a96ayQNuS0gcbB87WyPnrzjb9aha1sIlYyWdtx8zOYVPHvx+HTpR7TyDlPChJH3ljP/AAIUheMY/eR/i617mLDTlTZJYWEg2YDeQg4xx/Xp6UwWtq+Fj06wRGO1c2yg59cEDI96PaBynhvmQg8TRc9tw/xpwkiOV86P8HFe5/YbAxSyNbwncRvzCuP/AEH096ctlYeYu6zhljGF2GGMdR8uMrn04NHtF2DlPCjJEuQZovT74x/Ok8yEDPmx4Hq4r3UabbQFSbaED7uPs0a/U9PU/r7U5bSHzFIs7ZXbJ+SGPJPqO/5+9HtQ5WeEiaP/AJ7Rn/totI00WQDLGP8AtoP8a95FtaNbgNZ2+cjaohj+XJ6fd6c037BZgOTaW8bg43CJTu5wR93pR7QXIeEebGf+W0eP98UBkJ/1qfg45/Wvc202yZR/oVmSB/HbxkfT7vJ4oGk6Ydx/s2zY9Tst4sflj/Gj2gch4coQ8h1OO+4cUgdPvCROvUMK9ufRdKV1J07Tw+G2iS1Tk+vT+XFE2mWzBJTpGnXD7grgoq4Ue+0cgYo9qh8h4juXqGQ9uCDS8npg7evNe3NoekOu/wDsrTg6KT/qEJ49flweD+dRf8I/oTIg/sawKDOwm3TC+u35fX60e1DlPFcjJ7nHTFBZTj5hn617Wvh7SWXaNI0wDAJAtFJ2+vTjmmDRdGzgaVYhWPG6zTJ/r+FHtRch4vj36elGRn7yn8a9pGg6K7/PpOnONpIzZxqfbjbQ+h6OjAHStLCEY+e0Uk/8Cx1o9qHIeLYweSBz0pT97JNe0P4e0cRFf7C04HpgW0ePpjFZmo+BNFvMCC3awmIG02vyj/gSt8v8vrTVVByHlGe/QdqUcDHBFdXqHw/1e33Gykhv0X+4fKf/AL5Y4P4GuXmt5Le4NvcRSQT/APPKZCjfkf51aaexLTQzHejv6cdcUbTk88e9IAewx2piAHjHX60p49fpSg4AJwPpU9pLBDdxS3NnHdwqfngkkZA4/wB5eQfegEVmyykEEqeNrU62+0JOEs0mErcKtuWDN7AL1r0rTtC8HavYteWdkzKnyyIZ5VaNj0BBf369OOKvDwN4aZmb7FJsUZ2x3UwO7PXqf6Vm5ouxS+EXiKaPxfd2Gp3EwmvLYjN0SrF4iCoO7nIXcOa9T1bx14X0MlL/AFyySVf+WMcnmyf98pk1zgZNgiCJMiDcfPUyueO7Nksc1JGY1GEjiVR1MUQVD0+YY68Vk3dlosz+ItN8TWdvf2MbpFJuRZZR5bNg9Cp7fr+dMRHbBLFWyR8zjp349ailJnj+cRzYHG4ZBUemf881FutIlQyYgz8qqGyD0OR2OOlAx81utxZzA7t5X3O70+nP1607wtfm3vbSVjgBvIl57Nx/PBqGNyIwYrh3XecB+TtPse3+NU4UEGqT224BZRuUg8A96APYBTZEWSNkb7rAg/Sqmk3n2/S7e5P32T5x6MOD+oq9TEeb3cDQTSxzGQtExVQigEdjtPXnr154qusM0aKQPMXGAWO4r+v6V0PiSD7PqiXKkr5yEZ7ZHB/TFYJCEHLqE3bQuTn+mPpSGJF+7TzCykgbTtx3459vrQrtGyhkkQ4ysbdOOPp7cUkYPnABowCn8YxnqOOMfnT4n8rIRGaNtpY8Y3dMnBy3btQBYDKHd3fy/lwFB646/KaWJnl8tGyCT8uSGU+5OOAfSq0iqsjSxDaCQo+bavqc/wB4Uq3jQx7pSg3cghMDn3HXmhgTeRmMhhjB2omzIUY6cdD/AI0gIVf3hbPzcdu3Ude2KaHmuFXPz52yKXJwf9nr+nFDyQkfuy7k7l2lg2w85B7ntj19qBGNL4n0KBc/2xZnPy7EYycfUD9azrjx3olu5ktTNdyLyhSAp07bmP6YryvJHAzTjzzWipojnZ3M/wASLhnYW+mRoGPBknZyP5fWqFx4/wBcmyEe2hQdBHAOPxOa5Xk9OnuafyRnHOarkihczNxvGGvseb9+uRlVz+eKltfGGtq4Q3iuueVeNce56VzwC4DdCeuTWnpeh32rtm1izGOHkP3RQ0khps1p9dv747p7+VIz/wAsom2Rgf7o/rXXeG7u9v7dkTUb35OuxlYflg4rItfh7HFtN3qbITwVii4z6ZNdXpOk2un25itUknKrtLA5JP5CvMxOGU1pNnfQxHI9Y3NBrHUmH73UvJgYdGhQH+XA9qrqtvbBbi4u2adFb5YS2wrzjIzj39jWlDpLypiWNY0xjJ5J/D/GrkWjxIiq8sjBc4C/KK85+yp6SqX9DV1JS2jYwLTz5pI5oPNx95fLBJYHpj2x3OK1Y9MldmLiND2L/MfyBrVhtIbcHyowmevzEn+dDyLAcyOEB+7upzzCcvdpr/MhUluyqmmwDmQO59zt59RirUcaRLtjjVf90YqrNqaBA0CeZkjlvlGM/nVJ727nfajMGz0hB6fjUewxNbWb08x88I7GrLPDAP3siKfTPJ/CqU+qxqpWJSW9T0H5ZzTIdOnKhpZAoUbiZME/X2/Gs0XB1B3t/D+yfB2y6jMubeL/AHF/5aN9OPerjQw8Pid7fcLmnLbQZqWu3Mcy2sMUtxezbhHZQ8FvQn+6o7s3FdX4c0SWwtIZ9RmM99s9cpADyUT29W6tVHQ9BtdKykG6S4mbM9zN80sp9z6ew4FdaBgAV6GDnGd+RWSOerdWTYtcAZRdahf3zH5XkO0n+6OP5Cuy1a4NrpVzMv3ljIX6ngfqa4eWMW2jiM8lht+uf/rV2SM4jPD4R7me7lZRxxuOPmY14R8QrqK+8barcwSeZC05Cv8A3toCk/mK9i1m8TRPD9xqRd1eGFjGmcIztgL+OcfrXz/OWkAaQ72zkk9z61lGl+9dS/Sw5T93lKpBHtxULnHXmpJDj5qgZu2eCK3MhSRx70Bxkx8YPSovxo++49c0ASKnOM1fgjWGPc33j0HrVEtuwfTg1oWpeVCi/wB3rjpTAuRXG3TLjy+G28ncciuWbqa6pUJ09lnuG8schA3yr/8AWrFW0tpbtoRKepAOOM0DM2lxU1zAbeUoTkdjjrUPWkMmQKqhmPXpToyZ7hE6AkDC1EqlmxXaeHvBksqnUb8+TbQASkk4yOtAir4jtU03S7SCHy5VmJkaQKvYYVf7wYD1454rkyOa2Nf1d9W1BpTt2R/u4yq4JUE4z61jk80DENTW52yA9cGosZ71Ztogz4PfimkB9JeDovsnhHTEJCyJbo53tgfOd2cfjW40SMzFg7IRx8uDjt749hXPaJc399p1vbW5s2hgVLbc3YgYyeeeO/HXpXTKLJYTvuGF2OsAYujN0OGzyBg+lTOLi9RRknsQFUhjYNGFiA+VdueM/wAPHHrRIzjy28uQ53AbGHXjG725+tPYKhd9rHy8Zxlg39ee1MmjikCyYVmACsG3bvr6n/PWpKE+Rs7TGXwSBGef/rj0IzTUDk5k2RqTzhSvb3709nPlooJTcQVdYtw+6f6ZqcNu8s+QU+nt6UwKiRiN4UBUhhg/N83epm89I2CiNsLu2ueCMHkgH/HmgpKLjzndSH2/u0jyOmM5/p7U2SYRMy7X3sAxAwoUD26kGgBFEiBZCI2DJlgcpwOnTP5U0klNufmbnarYzz/9elUeZ95huHG0vjvyCelOUbVJ8pj0OHIcdeNo/wAKAI9p3Nxc+SFKsh9eucZHb1p0cygFUUJ5nzKin+LuMDPbPIx6UMAI5AEk3r91hGTs68H8/X8ajVtyOxmikCL8zBTkH1/kKAHjyUIVhGXw23nbgdyFPTpSqC0wY8u/zIhB+UZyDx39aezw+blVk+QBTtyXPzZ+X8/wqPZ5kUgdQhkO8jaR68n1wM/0ouA0yJIwkJeQBwdobaN3pj+tNCyTMZpljWReFjIJ3Z57fj/nmpHAkj2BlY7Mr0+Ucjbz6DP40bYpE2zbUGA/zjLEdMqo5HXuKAGoo2EyPCcJkPjqPpwMYOf60zywXaOBSwy27jGcD14J/wA9aelunmAl5C75wFGBtxkDp796jUsibm+RnXChvmI+bufb2oAYqOQrbWPO4Bxk8H6j3o+aWTZ/qwWxuzzj1479P5UkgAm/eZLsxV1Re3qB+H401SjRcO7oV/1m4OMfX86GIcjFJEJC91G5l698dhx6D1qRH2Mrqd5f5S3mbgR6+3B7+lQ7P3xd0YcbQTJnd2+UdCB/+qpowAcPvxu4jBCkDH+7x26+nakBXJcl9wUBVHyu+3Iz1HH65/OpomVoj8iYLlSCegOOfp/h0oCy7C0O1PlHzbcZJ46evvj0oEeXKLHJI4zxgfK3faM9O9ADWjaYESYwfvnlcDjo2fqKcqMTjhgCPlxuH0PP0/OpNw3fNvVTwqdz7DB596ryy5JMZLKAQhlZuSD9M5/TrTAsGNZPuCLdtJ+U8D8eevApJWiGcOBIr55YEds8+uc/pVeF7i4dUwuW7N3PY+w5P4/pYaZEKrEHQt94AHP+c0rDI0lwjxuFHZdsmdy+pBHH4fXNDt+84EhY5yAxAx6dMdj1p7M7Ssv8LHcFIKle3Tt1pkETnYuUO0fKHBHPXB+tMQ4SsrZiZCuOd/p6e1LtdmMMfl7mbaA2V28cDcevYZpRHt27yGPzbV3dTnkfrTQvlK5GSBJ5eEfOR3zu98UARBB9nEk22TBLAHIXPPQdfzxzjrSguIwkqspX76A8g9fz71KyAKCiAEDd8w79uKYQrHghcjaXPBH496AHEiP+FmRMl9nTrx2yMDiiOPbI8JBDKq/LkgjPc8frUeI/Me52BggZtqsMEe5PUD8Kas00QLlVEakchju+p/8A1UATpGvmN5hChfl65yD79qaqoYwPJVgT8jOoAb0yOc/hSRvGy7VTynZSikR/MR36njr0phAxyqojYx5mV4H4UmMlZCXCfO5C7imB+OR26VWu7C2v7NLfUbdLiPBO2cCQH/d7g847VYAky0ezBKb9oIU/UHjAxzTgvyhPu8gEscbPrk+/UU0I4bUfh5YykSaVeS20j8+TOnmw5OcLkHcvTHO6uO1Tw7qujkve2jpCDgTp88f5jp/wLFeyvK0ikQsGIOFiLBMrjtxk/wD16nhieLzdse948FkWT759yeNo9PrVqo0LlPAGHyhlIx2PY0BDjnr9K9Y1Pwdo1+zv9k+zXZyBJZyBc98soBU+nI9a5TUvh/qlp89jJBqEJ+5tHlSfgh4P51oppkOLRgaZqV9pN0LmyunilA5xjay/3WHcV6roWuWetWy3ESR/aoU/fwuzAxjt9V/2vz5ryGe3mgmMNzFJBOOsMqFW/wC+TU1re3Njcx3VrI0E8f3ZAM49RzwR6iiUbgnY9sZQmJJEDg4LEEZB5/rR9oWNgnzlOpfrtOOmM965DSPH9tMAmrxm3n/57wKxjP1UZZfr8wrqLS7sr5jdWt1HcIy/MY5Q+3jj3HpyBisuVrc0uSxuk582J22sRtAbIJHpkdutSuJIn++GCrxwQPyoKMbhMoyj5SWb5xt7n1zwMH60BHhclm4PJzk8jkgevX8KQDZJSfvbVwxxl8rn+h96pXsckK290zR/I/RARtUn3/lVolZkIKFk9fMPy/ljn9KdLYmVJYUcOu0jYB0wO/of8M0Adf4RugRcWpPXE6fQ8H9R+tdPXmfhe/MEtpMx/wBW/lS/7rcfz2n8K9MoQ2Y/iWAzaNLIqlmg/eDB9Ov6ZriZAXieOWYndhI/Nxn268fjXprqrqysMgjBFebXVo8Es9vcxlY45SAsuHjkxjHPXp0HagRmNDqVpIwN5AY92FLR8HjH3ccH/Gr7xyPCqzOiu3zLtbA/D059aPLkMJ23Pl7m/eg7BnnGPyx8wqKItDMx83zCuOFbCr6Ac8DH1o0AmnEzBVgjeQYywjcIQPbsfzpAn7wbgUIONjHgdjn0P9elR+S8sjSrlweAd/GD7f1qXMCRK4TKsMkqwA/Dn29e/SgBZIfmz5SkZz87fN19alGwYhkjkeZztyiso284AX0qLG9mVm2OCMF+cccHr0+opBPMUZ2nbLFfvDhueo/rSA8IXH0pTyxxmphYXYb5rWcf9szU9vpWo3UqQ2+nXcsj/cjSIszfhXSZWI7G2+23cNsH2b2C7tvT/Gu/tPh5pyoGubm9nPXCssQH4cnH+FaHgv4ca5bTteajZx2r7MIsk4Lj8F3AV38fhYhmM7o+7qqDaDXPWrcmxrCF9zjLXQtC03Z5OnQGQn/WTJvce2Tmt2C3kvJFEbFFClXXYNnvzjrXUR6UYP8AVRRrnrt6/nT2tZVUlgcD1NeZVxVbaMGdEacOrMa30iNG3SOxHZYxsH1J61oLGkahY41QDoAuMU25N4iYt7KSRjnGeAPeqn2XWLkNvikReygqo9veuT2GJru83oa88IbFpriKNmV5olZRuZS4yo9cdarz6pBEPkPmt/s9Pzqunhy9ZndokWVuGfcoLfUjmpk8K3Lt++uY9pUhioJP9K3hlsFrJ3Idd9ClJql2xVcJEOh8s559M/8A6qFs7uebzRtHy7cyIfmHcZrdGkWlggkeWKPAx5kmF/8ArVkzeIdKE5t7Kd9Tux/ywsY/NP4t91fxNW41IaUafzEnF/EyaHTYo0w+XA7dv/r1WvdXtNOuRaQRyXN+V4srVcufTd2T6tVldM13UlzcOumW5/5YWsm6dvZpMYX/AICPxpf7PfRYDBp2lYjJy3lgN5h7knO4n3NckqWJm7TTfl0NVKC2M3+xrzWMSa9KogPK6basfK/7aP1kPtwta7y21jEsR2RIi4WNVxtHoFFZ8x1SQMs0EiEHIWOFtrfXvVZIbvcrPbTRtyeI/mz2GOa1hl9Sf8R2XZEuslsdHpNzDd3sqwsjrCPmZX3c/wBO9bvasnw/biGwLbWDSOSdw5OOK1q9WjRjRjyxOacnJ3Zg+JpC0NtaKeZZMn6D/wCua5vVwWlt7dM924/If1rbv2+1eIG/552yBc9gep/mK5HXNXht7XVdWimRo7S2YwyZwC23amM/7bDFW9WGyOS+LWoxRadDpccqeaZx5kakEhVBxn05IryFzxWtqySJBbl1YCQF1c5/eAcbs/xc96xXJDAdau1jNkMg5qFl3H3xVh8EVFQIrspUdaW2+abHPQ0r9BnvRb/LcJ78UAWWtPLw4PB6irunReYxjU4LY5q3Z28d6rIWYED8P5VJawJZDzGSRnzgEjhfpQUVNRXZH9nHIB3O+04b0GKyHdYWZYt6HPIPUVrateo9o0Q2KwKrx1wP61z0j7mPOaYrBI5dskk/WkFNpwpDL2lWzXmpQQgE7nGceldz4z8Rxw2CaJZMC2F88jovoPr3rjbK7XS7d5UJN5KuF/6Zqf6n+VaPhDw9L4s8SxWjsRDzNdSd1jHX8T0H1oEOvfCNza+A7PxIxcie4KMmOEjx8jfiQf0rle9fS/jOxhu/AWqWVrEvyWuYoEGCqx4YbR6fKa+asc9KSGFXLRGLg/lVaNN7gdq1okG1QOxqthH0z4Rm0628K6Ui2cj+bbLKXeTlmbljnA7/AKYqzLEftUpfyYEfGY7dcZAHH17+9Yng9g3gzRWkALG1CjIOeM9x7VuPbgn975m3je474HykdcAe1RKTk7tjSS2GrsYAGNt/3fnYcjjkAf1xSICEOZ2GeCm3LD1JPpTkgaGBA8sjK2DhuQAfTjvg0mFkAmwMKeGZSR+A6A0h2IWSY4+SKU4JTLbQo9j3PPXHrSBhuk3plnUn5Bv/AC46+3ep2ciEgqybizZZQGP4fWmk7k+VQoHdBy3dh1xz09txpgRmRInRR+7LIBkn3zt/lwPWpSnnTINm1jk9/lGP1P8Ak0kZWKARxQYQcMEXhfTbn+fvUiLM8Jh3xiNRw0SFefXHp/8AXoAYVVVDMzspAyoUHy/Qbcd8dqGcRojeYyu/KAoefx7f596hmSGAGV/LV2OGcLjPBzz3/CnK4XzHlkMQkT7x43N2HJ54Ao0Acy7kA2DIb5jgfMT047/jQyFYx8saJkHCy4BH+7t6Dn8+tFmvO9DghOp4XjHrweuMU9LdkO+MAydcleAf936H/wCtQBVkjmYkRsEBGAkSFgT3y3T196ljmEQ3rLkjgKXJyegJHegK8bA5QqAoLkgFuOw/+vUpJx5mxUAO7IYHDH0P0zQOxWj89nIWNpWjONj4H1J7gYAxwakYRwsXh2KWP3t2wgk49Cce9RxRshRfL8uPglZAPu5wMgc9R+Jo2ywwF44zsB2h923Pp7jv/wDXoEOKuS+91Xa/zhFGOcfMSfp19xVU7Lht5TdGu4D5B+Jb0x7VdKylFWONlX0PHbPT8/8APFQiNGREjI3Km1QqdeeV6H/IoAroYMlY0DkBeA24MD03E/Qdc0zYT9+ZHkP3gi7Scnv+OPzqzEnl4dsBsgqQd3T6euD6enamlXTedrMi4+YruyCff249qBDtwA7xnP8AvHPp2wOO1MhknBITbwP4gI+cdqWNXkZXdLdUReJDyxb/AGv8+lPeGMj55I2CKNwfk8d+mMfr60DGNmeQp5BL4wcE8HGSe3QetRqGVSQgWQH7sbHC/X/9fep5YYCyyq5G2Papwemc/dHv69RTI/NKmNJHwW6FVG0d1oAYPLK8qsRdgd/PzDptz3zz/wDXpzRB2XZ5qqODIvylc9F/L6cmpGXIX99lW+b5hwOMj+dV5YQ8QkkVGGzzQmOF/L/CgCRF8tQjM0iLliS3BPX+X06ULMVt1wquW+UAEEtyO2fVuD6U7ds2OkMoTOcCL+LHt/XrSwuIgPMV03ZYBo9pOPY9gfp/KgQCOIOofAUDa2Oh9s/n+fvUBdSjbZB5gAY70DHb/ex9OP8A9dTrEh8xHPyv87JG7Zz1yO1RMBIoXzGcIP4DjIxxnHf6UANyzQAwNjGCIwNpHTaQPeplmcqsfmR4xhcFj8p7Ar179KYY0eKKN1B2tgHc2V9x/jU0ihZDk73X5iMZLc9P5c96AGt5m4lEyz8YVvujqMe3+e1RhUdNqTYVkGSBznP6elKbdRjPlj3yT/Loeo/KnqqRzRSNJuHV0OBt6/07e9A7FclvMUJ5koOd0j7QeB3/AMKcSoU/u3IKn74K9O/4/Snptkct3Rc4zu55P3eKQyEY8tYmPA2uxU+44znp9MUAHmfuwkYODhVUYALHtnHoP8KRid/nI251GPN5UntjNWEP2dcq21lXBBbG3J6qMDPpxVQzhkwr7tse1SX79/09PSiwD23jbEgt8KSxwdx3fpz7e9I6IJGlRG42tkt0PTucDvxz+lOlmd1JCYEnyYV9ydstj8uf5UjK6tsEqhgQMKqnBP1FADXmii2hYw2c9CpGRyApP5/SpQzsm9ZmAUH7oBZunHPGO31phaQod4EkTHbyOTjvj/EU7Akd3uCzM3Vlbbnj7vHOc9Pp0oAGLNxkO24s7uM5+tVpHWGVWkkiAAbyyWC7x3+Ujjr29am2nO4Db0ITcAB9ev58D2qdQwJjYo4z8saDIHPXqQeuKBGffWttqcIguIo75NzZtZkUqvurEZH/AAE1y2p/D60kcnTbiS0fn93ITNHj/wBCX82rsiiC4IL7LhRs7AgdQp446fjiphGCGGyONQNwH3lYfjj8/amm0Frnj2o+FtY0xPMlsXnh258+1zKg+uBuH4ism3unhlE9vM8Tj5fNifaw9tyn9K90Ijgk822CptyQA2wL7nGO/oevrWRqXh/R9cbzLqwzOcDzbc7JTn1YDB9OQfwrRT7kuJxFj451q3cCd4rxOjNKNsnth1+ndTmuq0/xto998ty72ErNuKXAzHnPaQdB9cdq57UPh9eRLJNp1zHc26qTi4/dsT2VSOGP1C1y95ZXWnSIl7aXFq7cr5qbQ30bofzp2jIWqPaZZnmt1uIEFzGflWVGG3HTryPX86coKQlfKkbdtQofn2+jEjgfqfpXidneXOnzebZzzW7/APTF9oP1HQ/iK67TviJdxhU1O2julHPnw7Y5B6fL91v/AB2pcH0GpHZ2mIdUlgf/AFdwm5ef8/5Feo6RdG80yCVj+8xtfP8AeHBrxY+JNN1KaCezmPnw4aWKSMxvjucHgjnGQa9P8K3gaSaDJxKomT/0Fv6VGzL3R1Fcd4otCl/HcYLJImCue4PP6EV2NY/iO2e40eUx48yLEg+XPA6/pmmI4XyfmPyK0YXOQ/Q59OgFCTNIoVTMGTH3Y25bPAPBH49RUoJZkTZ+6RjJ5mMhx39/bpUEjIsrgKV+bOYwHz3I4+lJgTskciBXL5PygjjB9+BTkjfzt4LsCNv3uPTjOfTj6VF50hRFVnB3HarOMZ9DilhWNCF2GMbSFSQ7gqkZ5/2sE/SgBGBRGWXzNu7C7hgM2OME/Xp9aSCVHaPy3L8HYMFSQQMn/wCt2xxSEuW3o5KFTkEbnOByBjgDNTHM0bBYiT93kYz1529xjP50AcRpfh291u6Frpz3CNjLOJPkjHq24GuqHw58Y6KTPoPiu3klK4Zbi28o/Tcu79VrvfDPh9PD2mi38wSzO2+ZwuAT6D2FbLMFGTWkp9hJWPEb7U/i/pOftFpcTp/z0s4orgfp83/jtY7fEzxZaybL28mtn7i4tBEf/HlFfQwy3bbSNGroVcblbqG5BqLp7oq54LD8SfEEuMawG5/gSM/0q2vjzxE3TUZSPQxR/wDxNeuXHhjQbos0+iadIzdWa1Qk/jiqh8CeFj/zALAf7sQFF12Eeajx74hRQW1A+5MSf4VVn+J+tL8q6kmfaNM/livVU8D+GUII0OxJH96IN/OtO00nTrEg2lha25x1ihVf5CndAeNQa78Qtax9ih1VweAywiJP++mC/pWza+EfiDqij+0dcFjHnlTcvK/5Lhf1r1nFFHN2A4Gw+FWkxsJNXvb3VpAckTSbI/8Avlf6mu1stPs9OtxBZW0VvEP4IkCj9Ks4pAT3FK7e4C4oxR1paVgExRS0GmBn3epiymWOS0vHQjPmxQ71H128/pT4tUspkLR3CEjkqThh+B5q5ikKKxBZQcdMigDzyx0HU9XvpLm4aW2jmdpizDBOew/CuosvC2k2cQj+yRzHIJaZQ3I6cHitymgYzyTk55pWHc+dfjhcpN4vtoVj2G2sUUjA43Mx7fhXk+7LFvwr0P4u3i3Pj7ViDkRGOEenyov9WNedL8qc0yXuMbim9h609huOasJYzSR+Yqkx+tAFNosge4/Kkji2zp9a3YdGne4ELqEZgdu498cVGNLli3mWMqy880AJpcpW8TOCgOPYVo6i6rHKyszIV3LuOQvuPX/69Y9qds2cLkc89Kn1B/3YC568knrTAxLxvMl8w9cY+lUe9XrlR8p65NUu9IAxS/dpQMDJpOSaBjslm5P516d8M/HegeFIZLe/0KSeWY/vLqK4+Yr6bDgfrXl5PGKVHKNkUBsfaGhXfhbxZp7z6Y0c8ZGyRCzK8eR0ZSeP5V8ha/pbaN4h1HTGBzaXMkPPXCsQP0xW34L8WXfhvXLfULWQ+ZGcPH2lT+JG+v6HBqx8UpbK98d3OqabIslnqUUV3HjqCyAMD6EMrZFA2cpBGAue9aUK521RiyQvrWpAAAh96BLc+ovh/pVlP8OtEJi3eZZxltzHBPPXn1ro10HTldn+zDexyx3Hr+dZPw3R4vhzoKPkN9jQ8+/SuppDe5lrodoGGFYIO2453euaeNHtQSArkE7mzIeTWlSEgDJosIyzoOnZz5HPb5iTSvoGnyJtkhLgf3mzWiOW3flSgqTxzQBlR+HtOWJUeEvtJIJOMflStoOnKijy32JyF8w4Fa1McbhtGOvNMDOXRbVHQqJCoHIMh+tIdDs2PKNgjBBc1p7hkDuadSsBlrotooA/eDHON/8An1pRo9kAVKswPGC+a06i8tVYsirlmyxx1oAzV0LT5QreVIACSBvYc+uKG8Pae23MJO37oJzt57Z6Vqjpzj8KU0AYjaBprPK7QyIc8vvKjGO3tipG0GyYbh5hJXH3+K1guM+tIBsXnn3oAxW8NacVG7zyF5x5h9fSoW8L2rAGG5uUUg/u22kfyz+tdF1HFIPrmkBz7+FrWW38tpplyOQCMdvb2pq+FIQpxcvuJzkRrXR4pPx4osBzp8KQFiftMpO3au5F+X8Rg1E/hBHkLC6Cnr8sXv8AXiulOd4xjA60pGRwce9IZzEXhbyyCL4eYu0lfLyFI6d/anjwrETxcqTnJ3Q549OtdJsXJJUZIwTikwp6HnpTA5w+FTsdTeqwY5IMH/2XFQN4OUDKTxM4zjfFwD+prp2RZhtlRSvIw3enkME+QBj6MaAOUXwtducvcQruPzrGDt9iPeg+FbuNh5N1EO/I2jOPQCuoi8wgmUYJ7A5Ap7AsuASue/Q0WA5NPC95GuPNhILBiCzNz+I/pSjw3eghklt9vHy7m5Hpuxn8vQV1pFB4HpQBx7+FL0wNGk8Y3KNwSVl3N37HHbmmR+FtQjhUNPbM57ktke3A5x68GuxAI6nJpsjbSv8AdJ5OelCA4+PwpeKvmBbXzfu8SMMr0wTigeFL3zwxeyAwVCqCCv0ODXYo6Om5XVh6g07YpwSMn1oC5xU3hTVHjwlxZAsNpDbsEY9hz+I9ajk8MaikqtHbWcg+UEiTB4+q/mK7CG1t7I4ggjiRidxUY5NWsUWYHCXPh6/bl7beC3JVlJU9m7D8eKP+Ec1JY4gsKM2fnd5s7s45/wA/hXdFVIxigqDQFzg28PanJPskt42jYYbM2C3PXOc+n5dKUeGr9FG20GduM+apIP8AnFd3tAIpcUAcWPDupXEDBtkUhHzEnBP028H8aik8L6mSdqQAbuAkpGBXcBeoowRgdfekBxP9gaqyMEjSN1wMtMu1l/2cDI/GqbeGdWBOLVXPOG85PmB654HtXoR7cVDLIykJGu6Ru2cAD1NMDgV8P6mNqfZpYfVIyhwOo/vZGf7351ZXw9qjxN/oqIWY7izLuxjqOwrtki2ZLHLNyxx/nipFVQSwHJ6mlcDg5dIvICDLbIqY6CdQAR65PpUL2Fy7RpDZjPO4QMvzgc9m6/5FehBACcd6cQCOeaLsDzlNL1RZ1k+wzYYbdwGHUfy/EU19HuJYp4G0+f7OzYeOS33Kx+hHOeK9H2gDFQRAyzys+CEbanH+yM/rTTDQ8V1b4d2ksRe00+806Q5yYYmaL/v2Rx/wEgVy2pfDzxNp4EqabcXsAGd9rC27Huh+YfhmvprbnnmmhApyB170/aSQrI+TUS80fVI/OsrqKZMq0MkDI5BGDwQK9J8JeMEtjYNcxyJsmC5K7dkbfKd2frn/AIDXsl3Y2moQGC8toriFuscqB1/I1xmp/C/Spy0mlyy6dIeqD97Ef+Ascj/gJFNyvuCVjvRTWwVINZtnDqlvpUFu72r3EcSoZSWIYgYzjr+tV7fTNYF+Li91tZoe9rFaLHGR9SWbPvmkBzb6fMl3LAodnTKKREcNz67fTHHSqtvDnc6QzqVO4Hy26dQM9T9P516SKWgDy6ZUkEkT3BLAB1wjZZs+33f/AK1JFHFEZi0pZXOTs/ebf/HhtOeccj3r0qRHD+bF1H3k/vf/AF6hksLG9TMtrFIGOTuTn8aBnna79oP2lY35OZF646Yx0HapI4m3OxuguXKJmQncfcH0Pc16HHYW0PEcKIMYGFHAp4tYkJfYDIerkDd+dIRPUD2qvcpOXfcmdq5+X8qzF8RWqLunEik9FVd3t1HvxTB4otFVnmikRBj5uD+H1p6DsbhB7Gg5ArFh8TWkqsSkq8nbgbsinv4l05FBLyZx93Yc0BY2Acil4rD/AOEjtWlIEUpRf49vf86WXxLaRxuwinYr2C/1ouI2jTFkRyQrAkcEA9KyU8R2ZX5t27G75cY/DOM0jeJdNUqzGUbjt3GM0DsbVIqhVwP1Oazotd0+aPzElfb6mNv8KmGqWhcIJhuPbFAF2iqxvIFQs0sYx1G8cU2LUIJSgWRSXGVAOc0XEWGQE7u4HBpkbrvKFh5g6jPP1qTdn7uDVO6ljtZklwu9/k2heW/HsB78UDLtLxVe3nklQtJEsYzxh92aBN5km2M5UfebGR9KYixRTQwPQikLdgQTjpQA+o5pUhheVzhEUsx9AOtNklZGRRGWLdfmA21n6lO7adfq8kIj+zydGyy/Kc5oCx8j67qEmr6xc3chzJczNMxx3ZiazXjGat2cDzlGVWYlQB+VbS6IgZWlO3nOSfb+VUIxLPTvtA5ZlA64Xr7UyfW3sJXt7VUWNWPbdmrOt6qlqotbNAvybSe9csQzHJGTSAvyazcSSB97Ajpz0qe11e7lmjiMzEMdmGx0P1rKWMsTwR+FWLOLbdQluBvHbNIDfltvJm27WXvg1XvUIbLEHbwcVs6mJIjE7L5i49cZrMmPm7cADI6HnFAzDuxt2g5z1qgCM8itLVPlmK9BtHSsygBxbNJSUooATFKKKKYEiMVYEVYmkaYRFudnyj6VUBrb0TQb7WrPVbi0jBi020N3MzdlDAYHvyT/AMBNAivChDj3rSiQ7F5/i5qhDgxqPStCJ8RYz8p5pDR9V/DW4+0/DjQZSf8Al0VP++cr/SurDAjI6V5b8K/E2nWnw8toLu6Akt5poxGFJO3eSOB/vCtm58fmRc2Fl8ufvzn/ANlH+NCQ2dzmkJGOa80l8b6tuU+fCgJ4Cw5z7d6h/wCEy1m6VN0qouQSnlbCRnv3FPlYj1DBcDqKcBiqmn3a3losq288AP8ABMm01bpABpabt9zSAkPtzmgBSuWDdxTqKQnjjrQA1kDkE54OetOFNBbHYn2prS7Ccq5+i5oAeRznvSg8U1WLDO0r7GkO1n/2gKQDxnHNBoHSopHlVlCRhhgksWwBTAcFK5x3PGaSKTzVJCOuDj51xmlL4KhuCfyz6UpUc44PqKQDulFReYVzvA2/3qeNp5GD70ACnIp3SkzzSAnOMUDHHFMk+7x97sfSkPmAnGCvp3qIOpuFUyru25EZXB+tAE4+YAkfnSk4FAFRSN8wVWG/rgnGaAGx3UbsoAky/I3IQPp7VK5IUsBkgUwmPeiuU3H7oP8ASnMyoMcZ7D1pAJHJvhRmRk3DO1uopxZQm4nA96ATu5xtpSQD1GTQAobIzUXLyDpsX9TUo5FRsEVSzNtHUndigB+1cfdHNKD2xxUEiOwysjDBBGGx/Sj5oSW3yyBnA2lc7c/TtTAkkBePbgc9c+lPbPbFQ78zANGwIHDZGD+tS5bHQD8aAHdaaTjHWk2tnlvoAKNuB0+vNIBJV3oVBK5H3h2poR9+WYFRjGOD75oi85k/fKitnojEjH5CpePWgBGbaMkH8qjhuIpw3lSK+07Tg9DU1RSjCMy4344NMBzyLGu5jgVHEjF2mcYZuAP7q+lOUpKEbO7byMipFZWHFAhaaw44pkjAKCTxuAqQe9IBpYj7w/EUCQH1H1px4HSjFFh3DqKgO+EDClkJx7r7+4qwKOtFhDVIKgqcinGoVt0jZ2jGwucsV7n1x0pS+DtkHB43DpTAlooAooAKKT1OOaNx9DTAUEHpS1G0qJjcwXJwM8U8HJ9qQAQSOKieLLb1+Vx39frU1JTAhSQMTHIu18dOx+hqUAAcdKa4UKSRkDnFR4mLoVKhMfMrDJz9aWoHnZnNsku9bp5W3B9qL3Pc54/CnRwvJKpQIoXkhX5i79uvH88UhlJ8pJEO1XwkoHGB7ex7mp3uDHuAlljDcIJCGKf3TkDnDbj0qUiyKTcQ8KFZXVsYX5QARj884596lEhkUyvIUZD0RgVHbg4yabEbgZaeZpp2P718jIOOuFxgj0xSzxhmKNjaifIw/jbjjjPPT+tAitM8scwSJJAkZOJSpJOe2cZxViSQSxRB3V3YcbuSuD+WPp6VFKhE4CYVl++5iLHHfvgfp1qQmNpGe3lT7OMxkNIGPPUY6f196LgDRLKpklEckZ5ELdWHQkj2x3pXaPBFvZhVxkEjke2MmkjJQb0QyyDo5bAVfXb2q1tkEYRwG3AgpG4x9eOfzoDURPMWBGSKQO33vnCqfU9c+lIPmZfkbkEtkdB6+4qHIRhEf3ip/dTaF56Ak8/WrQkV3y+WjP8AyzYn9cUhj1McaqsaLHIq5JRNqN9aRLpGSTbIMHDNs6Y9BioZ1jXAby2fPyyKpzn2OP0prMwWTafLz1288D8aBFn7dPE7CJ3SM8MQx6545zUj3V0/ztO5dAFDqSfw46k1lyyGBsu2TuK98Dj2/n71O2ws7Y2qAVKDnH5cE59aAHb55JNrCViw3EuMgVILq4k3BCxC/eZGwOnpUWUfzBmZAx4yCACB3poDCIhIwGX5owG3b/U47fl9aYakgkkj8sRuUyNzMRknu3ANJ58wl2RykSsOpbaMnsaZJcZtxKnmIqYf7/yZI2+nemyugxMxSPLY2ou4ZxwxP50ATPLuwhmkzgNkt0z0BPr1rF8Y3Mtp4P1nbtR1tXTC98/L3+tbMgCs4hG/5tzSKACenU/0rI8S2s+qeF9R062jdpJICoQryxX5jg/8BoQHiNm4t2QIVG3ozHrXQgxzWWV/ec/Mc/pXGmYmTfjG7nBrY0+9mit1bzBlgQq5xgVqSUL3w/dhzKYG6nPtzVQaJd43GB15xyvU10p1e7YiI3L47BUXAq4mqeTAC4aQ7vnyTzSA5P8A4Ry/wcwv8vUVoQ+FLhEErMCcjowrfk1h1ZF/gYDk8E/596yLvWJHYhWRU3Z5AJ9etFhEOuKyhwm0NGdr7D0rDjkxw3Bq/ey+fJI7gs0mGzgelZ1spErqyAnoQe1IZR1kk3nbhQODWb2rS1kKLpdowNvas0DJoASloxzSUAKKWkFWILeS5lWONdzn9KYhkMLzSLHGhd2OAo6mvSPD+qX2haE+k2k6RRXDF7kCJT5xxjDMeqgdFrC02xhsYmON0zcFz+oFW1kIfqApGMHnP4U0guc5GhgkkiI+4xT8qtI2Qq9Kj1YGPUmYjHmKG6Y56HipLYb5Ae4A4FTYZ0Gg6hcw2UsMM8yIsm4qjY5x1P5VqHV78yIz3cjMmSpc5HT0x/Oua0391NMm773zcjPf/wCvWpwOVZix9ewq0Jmuut6kilPtch2LjO1S348Ura3qAyDcyD6YX+nFZBPC72JbtluB7f59acJOQysT3xupiOy0v4ieItJKp9sE0KceRcRhgPx4P613vh/x9feIYbg+VBbSRc4T5hj8efSvECzbs7cKPm+Y8Vo6Zrd5pFw8tsyKGG1lcbg3/wBepa7Due8/2zfMGRpED/whOpX+97UDX7tQrO8QUnbwmTznHH4V5xpvjuwnxHf2ksD7dzyxsZMt9OCOcetdBp2safqEaw2uqQ7sbggfDdzgZA55/Ws2pIu6OnfxDd4O7YiDCllXOCRwev6CmxeIrt2jC4fcBy64DcdP1H5VlPEiKrscFhlUduB67hjIHHT86kaLZGJke3+TGeGX1HH0pAao8RXOWbYhHBVQv3hnqDxgfWlfXL4jcv2cL3bkjHqB/nFYhMcPlrNP5PmHjzAWz6jdwAaHCxnCybiYiDhjjPbGM4z+PFAWNVPE1zKszCIxKhwPNgIZj14H+eop0Ou3R2kzI69TkDjnnn6c/wCNY88jKyZiPzfOO/bkY7d/50l2iTSCWFNzn7rRqUO3svtj+nvQBsHxPdGQJFHGwP8AEw4/PvTP+Eovdjs8MKbFLcZO78O1YskcrGTyY1lIYbyhyOM5JYdhxx/OnKXknxK37xv3uxCcAAnHPfPp3ouI2P8AhJLlCCtvDukx86j73PQ89aUeINSEqo6Ip3bSojPX0POfyrBRm3sFTZGCuyHbuKsT93PqAc49qmhzCoMmSQdpJ+bK/wBPz+tAG0PFFw8uHt41i4IKybWbt/Fx1x61YHiWQMoe1AJ/h3cjrwPXpXMXDSR8QrJK/OSQMqvbp1p0NwzKTKXfcQFRsDB5HO0cg+nFMDoG8WMsiK1ntBOPncjP5D+dT/8ACT5mQLAuxxnl+Rzj/CuZZ5kiKdA+Ds3feU9Pof8AH2xUTIsQMit1OcFOAQQTk5x+VAzqh4nO9AbYZbjG8g5zjGCOtJ/wlFsZUL27lgPkZWyOfc4xXPW0h8xT5zTHdwSpzj2/w9MVEsuyPCMVC4jwuQXPoF7Yz7d8UrsDrJvEgjLBLWQkLncWGM+nHP40z/hKBuJNqQgH3vMHP/6q5SeJfsymMbFVdrIMctnn3A7dPehCkmXjd51UYwow2Mf59aNQsdKfEsCt54t0V2OxiX5wOg9f8jrQ3ieKSMM1sxG4FYw/zev0rnh5rJGZojtweCwODk9sccjjv6VCqyrvHmNEoz8hfBPtnn88/rQhHUN4sUzxrHaNgnbJufBXjjHGDVh/EdrJbnYsqy7eEkXYQc46/wCFciyMIldSRJJtkVfKG4enP4mmksVR4IyF5VipJ3E/eYEDt7dM4pjOti163cGS6nkSJlGFERG0+5HrViXV9KEaCaVmVDjDqfTqc9a5FSYWK5YMQWJYBR15+b0H1/LmiaRRF5jykyMu4rGOQB9M56ikFjsW1+yxiJnlbj5VXn9alfW9Oi5ku41/2SDn8q4WWTzLh0D4XdzsYAFQv0BJ5PNO3SSsnyfK2NoYlufQcjGAO1FxHaLremyOp+0DIPRgRj3pya1p8yhluguDxnjNcI5EDsVSNJOocfdOB3Hrx2pzK8LAybldifmCjqT8pA/HuPrTuOyO9i1azkYgXUB2rlsSdKlOo2XDC8twP+ui8/rXnLCUyRI0xXZl2Sbn5u3b681ZlkRDI8WxPKTcyjGHb2PbHpQI7iPWLCWXykuUY/pU1zewW4+d1JPbcB/M15+VXMUquzONoO5Fz0PXnnPoPX61HKIXGJFQxAt821sjnjoTz9M/nSuB6JHdW0j7FuYmfj5VkBx6VOCDg59uteafO0TvGiTZcKrCINux79eOOo/Om3M0qoshDFXQFsDD7s84UHHU9PrjpTA9O6jj8KhiljMroGPmD7wYn+RrzqC+uIBICj/MeQSOp9cH0/h+lSbpHk8xA7eWSCAhbI7+/wBf6UAehyPFu2O6gnnBIqCe+hszCksgLzPtiUdW/wA+tcJFJDBNmEGRQeGb5Djpz7j8veoGc3MjrBJLMkWd0bsWzz2xnB/w7UMD0kTHKgxONx9M4+tSbjj7tecxXd8I9v2maEqx+Vtw3Z9centjFRR3V2ZlmjuXk3fcLbyR0+6e/agLHoxnIfHkybcZ3nAA/M08u2wHy2z/AHeMivP5b6a4jCTTugcbR+8Zlz7gn+meaT+1XtZW8q5nVvu7fNJyPXnP6UXCx6EWIIwpOeuKaJlLbGBVvQiuHfVru9Epja6HzYRC5UgDr+I/rSPrmoAZaZ0VWAX5ie3GT3/z1phY7dn+YohAcjIOe9ShgR1rhjrGoI6n7RJIGJBChlz+ff8Awpq6nqEcwK3cqowJkHBAPIzkk+nOKAsdxDN50QkVXXP8LrgipCfauEOr3yMdl3JKwPzFATtA9vw9KfNrV7cs8MlwyBV6LHs8w455z93vigDt8qwzwRUUMol35jkjKtjDjGfce1cQ+p3a2628c0u4IXVYxs467iV6D2/xpV1bUmRGNxckY2pjbhj7HnP4/lQB3TbsjaeO49adk46159c6xc3djbeZNuMY3sOrs3PTA/8A1VZttYvorWQR3Ej8cbhnnIJxnpgMKQHbbgX27hkDp3pqsrOwH3xgH0riV1rU1mJeYiRFIJMakf7p4/8Ar0Lqt6Zkl+2zFojudSQox2yO45zTCxmrLshYMoZ3IKknoBkcMD+B9amElw8R3L5ZYhNiMWBx/eOev0Feb6N4nv8Aw7cGy1VJxGvy7JfvR+4PQj/Gu+hnttSshNaTQtC6+g6cDjP+RSasPcuGaZXHls23kRnG4SNkY9McfypzRXExCzM4RsjC/Ienp0z7+/WmkqPv+ZsIx979D68inStH5ol4UqNzAnOW74P5VIDQ8ETtb+SrAYHBOOnIPHpxSGDlSoV1VivIxtweO3v39Kky5VkMxY8ts3LsUHuP6U2aYP8APL1kOFUH7p/3e/SgBFeeV2haQghuiAHJ69f84qaOHKlzsRl4+9z19fxqvE2cqVO9l+UAgdufxpPMkkHRNhfcFJ5x2Jz0/GmBNcSC3faUD7hkH+9no2c80KY5SQ3mqsf+szhR9PqagadlUHchXkKV57fxdv8AD3qNkJjxKiyBQcLnIH0IpAWWlWNGCgvJkFHUfzJpwnJVW84/N8rjYSB61FI7gBBuZ256DK0gjkgDwyuxZvm4k3dPp/jQBPN8zShsKPlbeVx+vamO+fktuYzkhV6fp3+ppJnnlCSrGzBlKkvx+HrSAKSp2MVXgHaOP7ufTmgBsTRxtkRlnclc7uQ2TwO2PapUmhZkcocKWV+CwP8A9ao43LK7KQqhtqsD0PoP896kZkaBmHkh1xhtmCf9mgbGGQqN6xNtClfnU/NT0liE7Oy5kwrNvII57n29qZ9oj2/PH5mDlAD94+h9v1pyzBkVw+95FYhlVWKgYzwM/wCfpQIfdSBsAbNnI4J+fjOSeuafJsgjQJyJBkZO7aM8jmmAxSbZHTEbptLBjn6c+9JJJBEwG3zFcFcDp9O44ouB8661b/YdfvrTaoEV1IqgdAu47fwxiq0kxhkwG+7xkd67b4l6I9p4ih1DZiG8jUsQvCyqMEHtyFB/OuDmRmYn3rREFmHUPLIfj29am+3mQ7m59s1jMSDTWkwBTA3Jr0SLvLZf+76Vn/aWkc/l1qiZiPX86BJkf4Urhc1bmQrKu4c+Wuee+KbaeW10CVOPrjiqQk3xRc/dXbVyxwJkL8gHp60DKGvxmO9CntkdMVVtbfdC7/5FanimIjVlzx5g3Y9M+3ao5Y/s9isUONzdeKAMRgAx5zTcVZaDyhlwN3oDURHPApiFij8yRUHJJrqLG2S2iwqgY6kHlveuWSZ4n3I2D61cXWbxUC7kIHqgpgdQHYN8hbnnJFC7wd2D6nPSuY/tu+wR5o59FAqMaldeZ5hlYt7nNO4GzrSBoo5AMENj6A1PZ27NbNc4+QtsX3wBmskX0l3lGYkZBwfWt2W+8pYIUVUSKFEGP73JY/mTUsCO1Ypd5DFdyleDir/DLyQR/n8ayUmH2iNxyNw4rUIcuAFOQcdaqImTCQK2MdD0PQ8dPr/jT0ZgxfcVIbALeuPTvWjoOhXOs3BihG22jYGWTOFHsPU/416E/gzQLmLclltkKFtsMrDaP9o9PQf40OVgtc8sXeSowVPrmmAYbgg54bnvXpEvw/04swjkvCFON4mHHXtgnt09CKbJ8PNOaNfJv75jIfkUMrf8C5Xkcc896XOh2PPSxGOQx9B6VPb2t7fOwtLeWVl+YmMfd/HoPzrpJ/h3qCSsIL+xeMfKC+4OPwUY/Wu50bS7Dw3pjR2yTPJnfPLJnII6beg7nA/+vQ5roCRm+GdM1i2ti2qXLtE6qBAxEgGDn5nx69uea2ZoVubhpo3kTbH5XK5ySefbnrU0U7NIqJDukOSMLyCR0GPxp8Eu6Qhi8kbYd1crkKG/lx29PSs73LHpHGyYkl3boy+6VQc8ct+Q+nNQxTJIytIzzIrbtsan2J3H1x6H+dTCLNt5TzlwzkykS5O3dnd7fTtio45UUiFf3ke4hHJ3DvzuX+gpAPJbdJCylJVQLujOAoI4b3bnGPTNQB8zu4kWV96Kxh2LjjqfT7v9KmBcXiBGAITaqk4AXP3fTGKpzKYpIky8w3sdkVuECkkbdxI984x6UATvEkR6p+8+Zv3QXA28ZB4z/tc1Wyx8x45MEpgRqA2Onyk59h+lTPsguWWUyPtyqt5eS5/3c9PX1qGOOBZREIy8gXAMgA+Ufy570MRJFLJKoiZQZFcgvyPMPORn15/GmSbEjlFy6RqH3Rq0nTGMd+3P5YqbYYZt4Q+WFO4blI3Z6YHv2FQ/Z45ZYjmNlAO5NqqCfbHbH49uxNAyWCSUTosKOpV96OMc54AK9CucYyw5xQAGlVfvjqpYNjOP4QCDyAO+KeuYmLGTCY3ZD56cc8f7o79qdvja1eScybdgzuYKDjcOpPv0PpQITyfIaSHG1FbPmBOCvYgY4OR/L8Yz5kciHahYL/rMfMx9O2Tk9s4+lSRuiyErOp39NgHJxtH3s5/+v60x5T9nRcZH+zHgEjr+HYkf0oQEUULx7hvdlH3pHOAp6EHsORjP19qSJSsZRdzK4wQ6D5OMgkgjj/69PjnVy8hT96uADH8o5/vY+oHv1NNCwsCrLA/XrITuzzkDjPfqO/0oGPji82FvOjbcD8gwAGLdeR07f1qOCCWeQMsw4GTGBjbjtnPOMD+VPVWeFTbAvsG35WAIyOF249xyPekCkOszSTPGgYEFQyqeP8596YiR41lgkjlcCMYCkrtLYPB+U598Zxz2pjOtmpl8wh/vl4045P3gR0/x9aZJJHFvUiJSyeWWkPBJXJHXk5JH5VJv8pGffIJmXAKqQhbpyv3fujp160gHyQLKx3kkNuZkMm1t3Htz2/zmo1xLbjcXZ0IKys4DYAwox6np9OetReaVt0zhVKk4UYVTnr9f8BTljW3uI2YCQYzjdkqucqfT8+4oAdJGAsh8oljyiKclTj06/wD1qVV3o/lRMCG/1bYOCAAWA9fw6GmOR5KQgLH91BvG/PU7uBxnj9aihu0Yx75AZNuUfygCrY4zkd84oHYlMkcsjEIsyt8qv5ZO4Dvkc49v8KJJUd0/f4ZjlBtOB7jJ5OBmniTyhtaRo8NuU4xuYdMenUfnT/MVUk2SSAScMT05PIDAgHp+dAhhhMlwG8kurfKh3hNx/X29KW4T7ZIflVm2qqfN86/U49Sev51Wg8q4xL5q/cUYJUpJjPU/5PTilk2OZCfMijdVJKkKG9WH0J6j9aAuSk7YtyhBuAf5crtz0PsevJP1oVUQqXIRlO07WyF+XgMfXH4UKAFDQrIyx5Q9JEyQMnOM5x1qEsshiuX3TSAnqWGzHQ8YHr04xmnoBI6GJQyIVb5juKZPTgkHIBPP4UjzpbxPK8ToijyySVKMW6Yx90ZH88jtSzskk7srROWIyN+HDY4zgHrg/nnNM85DMUWWMfOBtLAcDg9fccD60ANhYSyLEF/elcMmBISO3bFWiESKNZWcwPh0CvtLLnAAx26/h6UxElMbOithFJMKc7T03AYPQYA7+tPW48mMOythcliI2x79Ocjnr70AU5re3FwzKnnXJOQ/l7iWI6k9+g59+vFPe3PmJcI5hK5wzrkOBjI5Od3/AHzUhmQW8ZTDlh2HzNyTkFRgYHY9DzTY5oPkhiWcKyhtgAJK44wuevrjPFFwLHkTQ7iIlIHBVmJH3uhP4Hqf8aie2xukfzC8Zw8iHZnJwOmfptol/wBWHzGEkPpzjd83AGTk859qrtdQtcYK+WqblYsCqE45JbaM9RnPrigCWZTFFiQSPLu2onQ/TGaYxRw6MJY5pCp3726HdxnO7mpIcsIgpRhnBYTDnrj68Z98dKUyx3LpGeQ6EBJ3YgH+LJJOe/p+FAEbnBAEqxxlNu0Ab+cZz/dyB1/Op38tkDyqh2KFXnKeu3gDGenX1NKyCGFXzOrbM/PGvHygY9ucjLdR61VhQIZpY3cQomJNpVWC/dIPTHPbnPShjJJNoTeGdvukSO+3jjI9zzg/T3qONpywV/siorAFNjASf8B3c1ZVRLCP325W+4SRmTn7ysORn3xgcUu5Ed5WkdY5G2PG53EL26+mP5HvQhMlVRGzed8oMgcsELcNkrxwPp6VAE/eFFSMozBYxHjL/Tn26808BNhDpIPMfdhHIG/A6/j/AC4xSNGZZNsKeZyq7U+ZCO4yfx7e9MCOdEclkeJw/Ow9DnnHXqOlPM0hZIjzJtGxUYK349e3X6VLI370yfKkTZ8vaDnAHPz+ox+vaqhkhuFEKrIHAON4OCd38OQD/n3oAc5gKEqoDLJu+QtmNfTJPuf8inSRmRY1GZJDhQ6ruPfPPbtwenrSoqMqRllCNv8AnQMGI6fe5wPfpQ8iDBD7CCyk28eFQZ9/vCgBibJJDIsbR7wU8pEYPu49+Rx/OkaeaNHh8xDGCcOkeT153Me/GMeoxTysJUfuZJGVfmikBbLBT/jnt2yKfEHDxBZFQ7QEkCqm0HBJLemR7daAI1uXQsXieRNisWOZNvPy8jpn9M0sZElutybiIxMRuLEqqhs8YI75P40qfeYrhJcllk4Rsjn6fNz+XNMR3aGUbBuX51kYH1zgjAHTjpwKLAiLUNJttQtVjuYJLtDiNCCN+exQ4PAzWPHoWr+GLqR9Ldbu33gyWU2dzH1G3qwHTIH0rpkeOeNlwy73H+yDx39RSvF5EjLsxlSAxlPU8nG3r+P41KbHYjtruHVLcTWrbUX/AFgkjwyv/EMfw4p7s64LEAD5hhh16DgVzPisadYxtePfz2d50je1OWPoGXow5/8Ar1gWnj3WHjZJEtrj5s+ZKCjE+vy9f0p2vqgPQhE77DNEVY5zKytg47A+uO1W4I5nDr9n2ovL/NtRUxzk9O3XrXml14v1KbkQ2iKefkDEH65OD+VM/wCEt1l8YltcA5A8nijlYXPRQ8ciO0DlJCSe+W9vpihYzIp6GNlyRvAH09q8+/4SbWmmL+aCG5KtFuA99xGfxzR/wkesmTKTABgV6dPUg0crA9AtpoXJJUEdGC5UjtuGc1I0kQyTk5+5x156D8K85bWtW37YdQKKevyDk0jeINdlO2XUYe5LgLGfXsPrRygeilFW4M5RjJHli7Ybc3+1zjvTMskYMPTbsymCvX1/CvPJtUvyrI97JKjDkbhj9KiTVLm3Vc3VwU7bXJH1xRyhc9KRJeWcOo6t8vVTxwe1Qvclm5ZiFyqqFYDGf4uPb9K87bxBcRyMIbmcMx+YFmxTP+EkuPmWK4uU+bjbMzfpmnyiPTIgkOOhk6dTgemOnWlZniZcz/7IXdkA/wC76/Q1543iXUIowsl1cEL1DScfzqOXxxfLMrfaZQV4B3LkevUd6OULnpc4dbtopHLSFlYgAcN1w2M8DoOabLN5cUodiACyl4x8pAPOT2Fec/8ACa3rxskc207vuHGP0FVW8RalIAhvC6sRwg5+p2ilysLo9KFxazMWmgWKBPmUvwMjoRntWPfeOtD0xsPdQloz8sdvmR8j6cD654ryrWp57lQ7MXK/eIO4fn0rn2ikYjg4PNNQC52viPx9Jr1v9hhs4obbByX+d25yDnt+FcXIwweg5/CmzKbRNufmbrj0qk8ucAmqsTckkKn0qDGTnBxTi4Tntjoai+1qMf0pCH+Xz2x6U4wuF3bCF6dKtW1ldSwmd7ScL2Cx8/Xnt+FRXupTofs6xFJF6u7bnJ/p9KdhkUf3RzV+wYrcIw/hIODVRbaWMBHBEn8YPUGtDS7dnuOV+6M80hkHiKUNqFtPuyBxke3/AOuqs9+gUYJqzqqq9gc5yp3rXO0wLJuSTwF+pFMd2YVEOtTpEZSAORTEVzRV9bL6042XIyOT270coXM6lAJ6Vt23hzUL4p9ksLyUN3SBiM/XGP1rqdN+EeuXrYu3ttPIwWFxJ8+D6KOv50rWGcLa3LW4kVUVt4AyRyCDnIrXtQLj5mySe3euvvPg/dW2nXN3baj9taJN6RxWzL5nf7xOOgbHriuOtrfyQgBODyDmjfYC9c2sUar5Z3DHOa7vwLpWh61ZzTX/AJkuoQSY8hnJVlONrbRy3fIzjj3rj44Qy4JyR+lTWGoXOh6vBf2qktGfubiA6nqp9jS6Ae7wiFBIIoWW0RWKbWCbuflyvYN/nFSGUAqsKtGudqo2VjYjqAv0xwTVLSbuz1HT4byHe8M6K5ZuoxyVJHQgj8MVa8oRhI0yAwP3lO9m6jnNQUSi4LSLMo8tNjAncGAGQTuP8P4/0pGmYMjrzIuCshC9Og9Qc47jtQY5Y/JEjmBCSpyCQT13cNn1qWJXwkaQ+WwOBOwbsp56+uaAKgvDa7W2sRGqiSXcB/EO2P16jNPkuISCxMXJwgLEhlP3gcdf5fjU0Soqqx2bU4TD7dy4JZs8cYxxj5s02F4xYkP5Um5RtJHI47k9j/7KaQEclrJItskKMmFbywi5QN/F7kLUIiB2zSTrlh8pccf7x57fX8etWWkVHkUbSnGPMk+TcF4+729+pqG3aZVaWVYQARwnPmHHGDjpjHXgZ6U0AIk08TfMQdq45PI7An9akMpiVp2+Vlc/IH4HOSMZ6cg9MnFMBVZMfarnLRAFimAT659PU4pot1eTyDHlvM5O7OMemTwc8/hmmBYgUSRRfMEh+aUEKF38EEjrnkf/AKqimLxrG6yyRAbtrRPy3oTz9BzUEjTRSbFuHlBGdzbWJfnKggctjpSMcNujLksm3yY2Dg5HUj2yOntikIstAiYkUqSFA3cDjI47YH5d8U2IeRcRiK4Zi/G0yd8dgPbiqpud06GQ7ZFkbBPZ1PPTqfr+XNWldxGuUi+Z/M37Dn2GOenPTuaHsMbPCsz4M2G252Bdu0+m08+vJNOlfEflXG52Rcx87ZCQu7p688//AFqgEblwUk3Qs+5ecYGe57dvwoARkyglmmDFeV4HGCMn65ODTAfbz5s0Nzai3lAbKhlkA5GG3Ln5c5546dqdFNGybVG1sE/vFOenp/EOn5VLEzmNIjtWN+cM4O/0Pf8AAHmkkkKuZQPLySdsacMBx65Oce+PSkIaGIVNy7wgWJX3/IW5zz1xhscHHtUcRd/mYs3/ADyCfMc7hjK9h36/40950MCzC2MaSkcqvp14/wAaWZNxihUxRnblwTnt0+Uc59/SgdyvbyuZCJY5FjQ4WR5crIG56nlSQBxQrW85Z/tDF14cjDMpzjHHTj39KmkglikaNypwvG18EDHoTxyRz708F4VVIUjaX5CqoDwducbR95vcmmAj+VHlkjVCx+ctjDZwST34x/OoZIy8oMiS7VY5Z23bV7EEdOmB+FQrKRsEkexzy52qSM/xY64FOjErrDP5yswO8FUO0YHDc8c4/WkBNI7gN8uXh2g/KeAW7eoz29cmmBlZACgSSMbxs6txjPzcHnp/k0k8CeYm5gE3ZOEwF6Yxx/tLTmmaN/uH5Ys+ZxnPb5SfT68ULYRC8g+1bUEcqZw0JZk+72JHp/8Aqq3FvgkVmYMTl8SQlQu71H8PPAbPao5AXdpHVkdgGVM4ycZH3j1/HtwOarvdIObhvnLf6wfNtI454/Q0PYAZRGsu9vKIjbZh2A3DnHqR/wDqqdxNH5SyXPzNlt5DYYtzkA8tjPOKIjvKTQopG7928jbXHzHBUbuc7gaSVZGmZk8yXYWJHyBm46d1IPegCoY44rnzVUxzfx5Q+3PoPUf41M2JS4SOQLgAuDgEMDlR6t2wfT3pJIxKm6Zy5V9x3OygA53EDp1xnPTAqQqjiMkNlidznGGPr37cHPrQgFVVKk2tuSCpfCqGAXGPm454BPb0poP+j4RwCDuc5y+MYBGeON36+wqQtGJczvMqYK7gxZs7RnJ6ewB60LP54Uuq7JG/eM+flbAUYI9hTAaX2OuxCWjbfLI8m49vvdOnQ0yWF54BaecZEc/LnLFD/eCqe3bkd8Uk8MsMjMWhKlur4XAPXjqTwBnGOc1NEXMaN5oAztyqnnH8IYYJ+U4z6+woARYiggSYRo6xEvgEh+2WBAbPDcnpxSoZB5kyxRMIf78vRM5wcgcknA7GkCuuw4UQoCJG3ZB5znv+GevNI7NAiXAXzZjGeWj+ZV3fKevU5J9ffkUAOjEslx5kkmZckI6YADf8BPpkY6/rTJPM83cCiM2MNuwMADlQPw+nFLC8vJ2qVWQr+8wV+bgKR0LE55pohBiZbfcZFXbK7o3BzgrjuB7YP4YoESRSSgQs8hTYCwPmktz6j/61IrOLORnySzBn3AHaSMZz75Pt16VTgmAbFxH5ZdMNg8DHPAPJ/PvVtTumY7/m37l8tlBUjPP1wMfhQMSBZUjLRLvAIZIwgwSB2BOWPbjrUha4gVkWTzNjZVT8wHf+Zxx2qOeIeYizW8cQDDa7RngevXnGcfjSqHR08rLbcN5e8KMH0z756gcdKAESK4czZm5ALNtzzkdgPXjr0qYB1Zd8e5ZkbYQoX5iORjP6GqkB86GQyjyt4+WOONlbqPlbP50pvTFK8O1fKkH73yyPLb/YHXn5eTz060wHFjuaURO5kU/Lhwxx8uAuTn09qkjnE5aVRIX2+WQ6bcD7oUqQPU/lTZLl51ZQ+1UU7neXCs2eMr+f+TUsMiRO7GAGNwVzG3Unrgc5+h4pARyvGiwKwYCPJ2KmXPHBJ59uPxp7ZMrxyOrAnLZQJu46gAZz2x7dKiuFuHRpYBDGo2K6mXnI43LznBx/gKdaTeVM+IpTMFHl+WSQD83G7jnn2/SmAiIrKU8oqQu7cOVxjrjPTipEMR+WVUVcLtZ8MSO4I6fgOnFR+SQ53pHECzNuPIUdRwCCFIxx9etNlSQKVeE7j/rAnKt/F8vTjlfyoAmi8sFEV/JYsVIDc4xnqOAeg29aQMJJZIzIDjngEgHGMFfXjr/OmnY8sPmLGpKF0EjBQ+Bztzg7ulNJndmE3mOsSbUdPvNnIXBx9ePcmkMnEUaySoqMG8sEYcjJ64/UAZ9KSXAVJCrn5/llPQ44xkDJ+vGKhhiS1gOyaSTYu7BAYrj3644+71HWpAzRuUWZ1cL8rHgqNvIJDH3yOP8AAQhzI7XhDNC3mEGIF8xucdAenp+eKga4X5WOQGZdyIxKoeuRgcnrj6VDGylXdg2QcPIMdD91T2P3fTpSwxyAu2QpVSuxpOSOSO/t0+gxTAnCLLIdjFocFXEuSeOB6bjz0HvUqeakwuPtDxhWXPlIoDHGM4688jNNiJlVUeR1YgKFI2ll9iB14PIx+OaZ5yrL5ixyS4yC4VcBv7oOe46/WgC2RNGkkaiORurcbdv1P+TzVeZ5raF/szMCdxRWwSx/hxnt2qaSHzUc+fIzM247CsbY6d+h5/KmSbpSzfuwVAY7jnb789eahFHjHiM3T6vNJe3Nu0oJHlxN8kf+yB/D9KzI7jDjEhAxxjmveDCtwqpNbLcxbjtjkGQ2RztGPp09BWDqPgnRLu4KTWItZWc7mi+Rj+A+UVSmTY8yhvdzbUyW7lxVtZgDv3r+HOK6w/Dy2QMbe5O3k7ZX3Y9u26s//hCNViKeQ9m5z6FcfgDVcyCzKMcwmhHmSk4PGT0oa8hVeZGDfT+lWpvDPiBThLWF1HOd+OnXiq0mka1CIxFp07tjP7plVT707oRGZHDfuSXY9Cikce/YUsgcrnO1s/eX+dD2OrfcbTrgNjlR1HvxUMVhfKoVdKuAScKI7VgWP1P4UXAk3FIwBOzSZwDI2Dj65zTVfzF3AOcDqcbf5UTWd+o3Ppt0u3lgqHIx1zioZpI/LWS8WREX/lmFbHNAFe5vIQdu8OF6ABjk+4qkJJJ22wDao6sVyDWpBd2G1Vt58lsjYoZz7jCr/Wo3mtvnCmRiOg8h1H8qdwKYiAkYF5HKf3ZPu/lnFPJygVIYyx+XMjE/jmp90gx5EE7Iy5UiAhefTIxU0WiapfSBIbC6K+6bAx92c4/Ki4ama1nIhLSSxxxjnO4KAPb1q7ap5iq8e9o+gGcFvYDt9a2YPAt9HLGz2cQZuQZ5lYk/Rc4FbMPgq+uGdLjUIrdfSK3wV9RuY/e9OKXMkByssZwqTnzW/giH8P4f16VFLFBa7ZZhBFNIoVIyw9TwOpP1rvT4CsZIEiln1GaPg7Hm2K/r8qrk49SfWud8Q22i6TeTWWm2EUT28SvJMXd3J25ABP8ADjn8cGlzIVjh7/TLu4nkuXVVjJ/H8B2qF9Ato7cTvq1sqEfMXByP8fSuiTUbN4lS6wrysVXKblK4xnNZN/bW1pIxHkG5B7JnH07UwsZUlvo8Qzi8vCOMquxPr3/nTItWsLJD9lswkn95vmb8z0qY32oKHjF5KIyDwpxisuTT0Y5WQr/vc0gLFx4kvJWLRnyzjAJbcw/Gsy3kP22KRssfMBOT15qX+z5T0ZD7Zpn2aRDksoIPHzZoGdXHAZ7jeYgxmbdhj61qpKtu8SomAx7L1H1rO0nEqgPjJ5OeorYnhMrwujbSPmGTxQByniMgSSpHFtjGMFRx9a5xULniu28U5MUcbEl9vOTnqa5mKBAVyMN6+tNK4m7FeG1LHLY+ldj4d8DaxrKRzQwrDaMpbz5jgEf7I6t/nmrHgjwp/wAJBrDy3AZbC1G+d/U/wp+P8q9ymlSG1gis4vLhVVVY0h6ADGWHf+QFKcraIErnB6Z8MNMtlWXUb77UzAhQJPKiVvQ7SWP0z3rr9N8P6Jpsf+h6fZ21wFzJJHFvl2nHRsE569OtX9kXlorJE7jGPmwN2O/b3/rUtxI8ql3K435ARcfXBA4HHFRzNlpFSKOK7AMVvmFmCHLlnJ55VfoO/Gat70GMbUzuyFwRH6jI59uecZ60puRuUwDKcMeMA+nPXgdqrpmPyvmQs2fv5yDnPp796QWEaRDeqVm3SKMMjNle2R714DqUdu19em1tpordJ2ZIZxh1XdwrA9PXHoa+gWaNvLV4wrqPvgjcRk5b8u/tXM+MPDH9t2hvrbcdXtVZoZFG0zIP4HP57T2P1qouwM85+0R6lbxzQwrG3zcL149vTNWV0lGWHzmURsm5sN8wOPQ1gWmqPa3LTbhJG6hNh4285J+v1rQui+oxG4h3iLqwHJ3VRJq6bfz+E7sgX1wsLnc8Xkbo5ffGfz713Ol/ETTb65hWQi3KjodwQt6jGec+teRXml3Udi1y4/eJyys+GUdtw7Zq1o8VtJp3mSD98WzknBVaLIZ7vHLFNDlLmO4EqfJPjcOB1Udc54+tL+9hbCZZ0bcGPWZj3yewxXmel65c6NqCGGaSeGTkwGMsGOOR0xubsfWvVbbY8UN3b2o2TBGV3J+6RkHg+nY+nrUNDKzwzSGTJ2tG5kxJyNoHPzdlORz3pyznyygWWRpGHzb8beSBkEY606UzsiuhEY+8Xz1PcYPf7oOM9KXywkcjzM6eV+8PHAz/AA5GCR/Ft9sUAQW5dYporgRlAC538oDjGSf8OmKndp52Rdv7mP5o/LK7yMjn05ODz25NCXCM32hWk3ZwJMbdhx3GOfXsefanuYZt8hjxuAIby8ZAOGxleT9emKAKamdfs80kSQ4LFY/m2hcgNnvjPrxSeS9xL5nlKIpPnlDHa3Yfr27DmrTXEjzSRhgAXy4ADMw44Jx7DgfnS+dDIV+WJAUZmWQYwc5HJ+Zu3A9qAsVtlowLSMFUjGI8fKufvMCPX09sntRbeSXaFgF8x8/XnjOD/nJPSlAa5VBOwUYw4DFQwHIHzenHTvx70nl26ho5IkZhtPKEPjGSq55HemAye5mmRVcqg4jMWNu4D0Iz8vBHqc07ZvM0pdvmX5UjkbhSDkY9OOppsqbE+UbOCBs645ONwz+OPanvEBII2k+0TZw4KYGevTgdu/pSCwwIs7qXj2KwLKFB+UZ9BgcsfXoDxTp1CyZbCAo2Q7FQMd29Og4+n0qVZERIy21jjEZRQy7eRwDyDz2H8qiby45mRmGLeMLwmeFzuY/hjnHSmIZNdoq7UaRYgAx+UoSp56EevSmqWltQ0iMkqr5pRyW7jknjA+meOabZo8E5l2u3mYaJmbd0yc8dAf15qVbgwTriGRJHzxnqAeOWPYf55pAQFRMzbrhSxARecBTwOhz6ewq00CiOJ/LkKRIHVwx4/i7YOcc8f0qETyeegVWzKqZMq7gBtPzAccYPU+pqTDxLIUlU5Vf3gORn+LHHAzjj86YyuombLRRzJEAFwxyGx1+Yjv8A/qpzSSiVonmjbd8scWWGOeD/ALI/SrEaQ8TSmM5xlmY468deTzwM9cVTQzJtka2Mec+VLtDBsDnpyT7+1IBSohVm8rdIjqigIBtOACPy749amhZ0QExP9ze2flHC9m6dB/8ArqJYJPsEiruWIdSo9s/KeOvpk+9QeXcpGWh2FN+0uG3nkcKPX/IphYvRkvDtmZy7MFQSg4BIB7+o7fxdKhnuFVsXSrGGYiNPLwVOemevIPb156VBCZFm2y+WUQk5+UEckED3PAA7VP5rxw+a5RGj+Qo+1vlbsPc85P8AUUCGIspWELFKo3H93sKYXv2+bPY9c0SrHJFK7xySBCEb93sOCBlc+u3r3qdLrerGRAHU4Zn5BwRxt/qMZ9KUuRbiRke3jjTcjRAKBkgA8DA/HrSGV7e4BtwEZAEGESQZx1Bx/Wm+dFLEBtVgqN1ViCPw+p49cU5yWmWbzXYLgl+p29Mc9j78n2pBMyRqr/u/KTccp82BwBnuenX6UhlqVVij8mYbC2QzueSDg/d9frUbSo8DKGAZQWOCdin7pPPHbHJoaWBmiWQPGS/+smG4gjkjnsxI7A4NVoFzlJHljkAztD4yOcgLnrTQie2eYXLQlZPKPIzgKRzgc8DkdueKZPJFcqpZcqqlhNtLfLt5JU/45pqt5sQaIrHlscYPBGDx3/un+lOkWF5V3RbIVRn3ShSD1XAwP1JJGOlMQ4R4u4vMTzkXbtcjGNw5AXHzf/WNSyRxRxl/lQth2YK3IPTGT/EM/Me/amGWOOLBklQGPCMSW2Y4z/u9ew+tNE8JeGRpVMgT5SWZcMDwd3b/AHenNADfNjSIyx/KWYKhCkDByM+o54HFSsPJYKJJIXVfu9M465x1/rQQRh5R0ZiQ42s3qAcfgfp61GyXRmdJbcGBU+UoQPmI4OcdgeCR0pAUG1ERs0LrPw2d5TagHYrg/geK0f8ARmbCRW8XnMGYjLEMckg5PJJ7UjSBZvOR1R0+aN/u+3p396jldCywsgJWNmj+TqAenpkYJ9ePwpgWLiZhCi3SAqjM+87SpPcZx04XkjtSMYUgSWBmijO0kjco3Z5+o9fr0qOzuFKgtJtjkb5hEVK4z/CMcfiTTjJMX2wzM5ZWddpB+U+/04x65oAfI4MZ2SDDkg/KAN3YZyMAe/c+9K19HEqssKCRirLuiUDC9yNvzYP8VEoVxkzl/J4A2KCMYPQcnjqPaorgCW5We4f975heUMWUkHpnr279BkUXAbHKjH92sKxsQrPHjA6Fj6A9KfIjLK6vFK7IpXbgr5Z478cc8DNJcpCqxM7Axx/LHAFY7eGyBjsP59elJ8sTx7YQqsoO5lBEhOeoycj2B4NFwCJGiubZXDqQW2PxxkjB9/f8qk3IJ90W5BvGPLJ3DBzn6Z5//VSrviuU/eLvlHzISVyvqxHTnPB5NQhDB/pEc8yHOGxu4JX5SFPT9MUwFYziUeTavL+6G7cFzzzkbucHHr6dKbAkRBPlybsksCSrNnuNvp2+lWGnSS4b5/NyrYB4VcckbcdP0zUf7pJ44tsksKoAXdfvDAGOxC+nP8qAsMCzLI8cTyLLnbtlIjb5R8ucD/a6DvQ8QkVRsCr91ihyzFe57r2xxU1ohIcrF5cpwgV/42BJ6nt0POOlPa4nUhIpRcbl2ySKS4wpxnHGBSAgieMS+XslMcgBed0xkYJGMZ/InqBUe67jmkRZJEb7jZyDnr835/56UjLF05WQnJzMflGcjrlenp6808xhWmh8zzA4O7O7arZ6YB454+p60ATgw+So81tzbdpypwp6buBj69TzURSFnVGdwvlblOCRjPAb9enpSw7AJEtwytjcFfGDgAfMDxzx6YIq3Du2IBDCJDHuyGGXQkjk54YZP1oBEECSBo0lSWRgccocSAHkcjGewOPSmo2yLaj/ALphtdQdpOe2WyP+BAdj601N9mwdt37luNsux39MHoOowfz71DM29onWeRJ1Zl3vuZSvpt7NxyN3emBavHRgNvniJtwOR7A7ffCqOn5npSXCl5JJoJcMhULIrFQxIx6Zzjdntn0ojiE8yoFVY2OWdF3NwpAx29PYZqBvtCTghXa3VSyoY2yu7Bz6egPOOetAy0TKmxYgqBurFF6888jOafFEhSMnbHH0Z3QNuJ9Bxk0rSyyRLCjKEJ+Td/Dn730z/Sh2czLu42/d7biBjoOh561BQjNHuaHbsxySDk7efXGPSoII4o5CIEPmspRsDnB6jrj1pxCeSjjIVi3O/cnHPI6+lJGqlXXdlcDkKfck/wD1qBClWiL5KiVergN8v+TSKUilmkCCBJMYIxgqemfT1qYSv+8kZFXhY9rEDGSOnrn9OtRZ8qRZAH2dVLL/AOPHPHfigCMXCbo0aUKTnADZB9BkVOzb2eVEkBbkgqp38Yyef8801RKiK7zOu75Rlfv+pDY/SnZhhZUkBlRWPJH6H0+tADIlldiQxaHlfkG0n+71z1p21GbD7/KCbmIGT7++OaIbhX+SGbzSr5I3YZR1Vs9/yFNjjmnwwhwgVyMnaWOff734UCBY3lVmSNuxVUj2Ee5Jp6QpuyYoZT1OfnDH+QqMiSbfvVgcZ2BOPcgf5NKvlJIY2Yb9rYJ/hXrknHy0AL5cZPywqcc424U/gO1ItnEAAkZOPvEALnPPOOo4qQNJE2UUPIygYK4GMY4x0H86a8rwrGjCaOLG/Iyi4xxyc/57UDQotY7i5P7oF87l8wfKv9afbl24TjHzEb+PbPP86rRSySlpNw8lRnL7lzjp8p6/SrSmL7JlFj++C2Uwceg/HmgLDkhLwGT/AFZQFiPLyDj6UsEnlqsgtScNtL8/Ug/lxSwxrIjPNtVfvCQrgdeBhT046UfZBGELAQpvb5kZgGz/AFoAhcOjFVWQqwwEbk9cnFcX4u0W+lvzf6dam7zEIrqKLAf5c7WC98AkHHtxXbESH5UeRyDuXK8EZ/hB5/LvSPHE6hWhlZwSrKw3L+P+fWhOwWPBbqIRXTLPC2xiD5ZQq8fuFNXGaBrcGeAPA3CzquQfx7Y9DXtdza2ur7YL20hlt1/hlVSoBxwo/wBr25rBvPAXh6VjNbQz6cc4c2Evlhh2+U5B/EdarmJsePajZLBMxO1U+9WWYhI/fFen6p8MriaETwa4zgllVbq2AHA4G5WH/oNcXqvgzxLpMTXEmntNbrj99auJF/L7w/KquhGHPYFo/kbLe1RQwpErJMdjAZBPet3TIIrqGKTzCJmJAR+FODyM+tSarot7qUpuILXJXI8lBzwuT+gzQAaFAJLpBnd1GPUVth86kNnAjUBcrj/Pes7QInjZDGuf3TEFgOcjFdHbRxSQw3hdsoAMnsR/WgDkdbt5Ly8uHiUkRqznJxwKw7aCSe6ihgXfJIwSNf7zE8Cuu1xFtZDLuJinVkbBAwDxS/D7RLnUfE0NxC6JDY4nnkK7vl6bQPVv5ZNUnZCerPVvDPh1fD2gQae7xxybd8sqxnEjE/M/PboBz0HatcC4DNlkSKPGCygszZxjI55GePSnTRqSqQxzo4UFkKryeu3GeBVe3l8uF7uZwArbkC5fJz7Vj1NB8pd3SKTMakh9gwRsPoOvI9aWCDbGx/fJEzLtyR8gJ+9/hRGryTC5RlbO3cHXgMevvjjp1+lPRXeHEbyFS23727aR9TSAaY0imjjkDuq5ZoivGTzx69qeI5AYo03lSv3Cnc4JXrU7W9t5bo7AH7olRWJ55wcdOn596rzp50Y8xlfP3lBy+f73p60JgR7Ny4GEfqxWPG7HAGT9BUIhuIZfmUyBZCArlsN9MHr19anFqEcKzqse0qAD5hbnjOcev4mltYmeEvCzzKBhvMXbuCkkA56dOue1MRy+r+BdG1ud3n32Vx/FcWkWctnqw+6w5Pv+dcjqXw+1zQsvptwdRtiM7YFMU+3ufL5U8ejfhXrjLbo8kzs4faWVmG4Lz27D8B6U2DyoJ0MlxcOemzyuvfr9W+9TTFY8WsVubdGtodJ1J5DuyHtJM7iMDOVqzNpF9K8Ns/hm/tZIQC6BCilufmLMdq554zivV0tQFRkDmRMb90hYY9eOPwNTfYpllG+3aAmFmGCQcZ6j5h8xXP0qrgeZ2HhHxL9oN3DqNnZyOqq8L3Dkuv8ADuKAqAPx/Cu+0m3u7XSbaLUps3ijd5sWGRdzHHzY+Y8dSKtww+WrlWaOLChkkxuRODsJ9e+PpT9sMaunyFZCF/ebs/d+6e/RiRSuCITM4SMfbcvhipPyqynvxSwtJsb5l+VflJ5zxyFycZqeNlMEb+RHIuPld2xuHQY98dvaqp2RXvCBnB+YyqyjPXleh9ce9Idy8jXCYWLEUqjuF3btvPGeuCP15pp3yNCh3MqZR9qAr07Dp7561HHNus3mH7tgFDIQMEZztPrimOJSltcgI0WdxQJny2I2nPtxn0NAE0m4WqoqgRlSPMxkhc9/1471XEcogkZJQfMxkgBdp5xzjpkHn2qm7rLOvnxO0QyMEYyVPGAPr0HFTNJ8sby46ZLZOeeOOw5yenWgLluzDtYib/UmO58tHWT5JBkhWyeVbpwcZPSo5UuZonZlV9ilpHnfbIuWxn5u/GB+lUpYLS4j2PBFMGx5iOhUMM4xz16dRzzwRU0KeR5SNLNP5EaoplYsDx8o/rnq3UnOKi00XeILbOkihEnfJAYYIAz/ALxzx6nH4U4yTFUVIN20hg0UYXGf7xP3vug9PWo90cbxPJvhOcKk6ck+2fXPVv5VIUCWreZlhhmDluCBj+IegqyBtvBtvU83G3cu6UMDszzub6/1qypUPlncqW6Pjn6EcgAgcVGbgy5dLzdGWCg+Vu2rn7oGODxxz9KhN3FFIkMZymRkuwGwnnGPTJBP1z2oAlkH7lJjFG6vJ8pOCTgYLY6+3NQ+fBLKINmZWJz8nK8/L0bnuenpT5iGYxtEzKqlfLIweAAfXnJ9eKjTzBBFGzofMTI3KFG0++717/yoAnnfzJnKBl/d8RlBGAQf7ucEY6jpnPFBAjuUeU/fO8uM8c8rnP1JP61RZT54nWVZHUdeSc8YU++M/pVi4BeJZnuJmRUw4UgqFxzjpj72OffrTAkmUGLf5hWYso8tnzzlvm/DpzyOKQyFygbCF22qVUDd07/T09aGV5LWScZMjsVOxcErt6qSMdf06804+b5zsTCUUKChLcqQMDn5j+PPTFIBlwqojIrRtvfezSOF3c9D/L8KZ5kkrZmJjRCI9yqMnrknpj8PzpQqeb5sOG8v587D97oFAHfjue2famtIi3G1CGZCFCliPl98cgf45piGDYxiW3Vo0VguSxDDOMk9eTjr2/WnLa/OpQkSDvnaWP8AeH4DPvmnIB5KPFvRUUcB+d2TwfbnHt+NOuGhe4EKSqybN3O1sPnH/fP496LgBBLKZIzGvIUbwNuFwQOcfXp/giKEmRWEuwIShXCk8/8Aj2716cVHsnx9wL0COG+4WPXPAGccDsPzqy0KIPKR0i3nayOzOflHCqQxyO54HvxQgKy26tdECOZTuG5D/Dn3PBboetTxW0iQODDIOVDc/dx2I+nuc81GjLABGQGZjtUsQRj37dO+PlzSW8oUBmtsKqluEOSg64/DufyoAZJDLGzfeLEYOBywBxz9SelQiCGR2LvE+QTll9eoyeQfXHp3qdopmhRGjtwjbuNpB+91+XP+J/Sm+VDbDzAjgNw4dmxKPQ5HHsMA0IAIKIiEqNo6puPy57Hqfvc8d6e0Uj27SLG5DqWcr028dMHI7HDY7VII23bB5bDLKAhxuXtjILDBHXrQWeWKRtjwq3yFWJ27sdGDE54XHNAESrIJVto5BGdwPzZUqRx1bp1PegtKH+aRGYbvlGGztJxjPTocHHOaa3ClcNlj84Iyx/4DwC3T61EpDSMkkU0gkycSYAbndj68HpQBPEYY83BVmJB/eByd3JOVx9Bwc459KeskEojk2wPHuXI5X1IG7Gdv+HPrVIqsO2RnSQ5+XZCY1HU9OpPsPzqwGEcxwcQ/L8hYbXHofU89MdqAHKyK43BpEk6smRz7hhjnHSlVoxDEnlr5gLMEYgcjk/QbcHqe/FRhGeUDylVUibBaTbz65/p3xTVAZli/dzREg/NEMde456ZOOfwpgSShjbBTxlvlR+QzYPXHX8Oh/CnRyRxSIhd1Xld+3aGZvVuPlxkdu3NQxEuzQKuNr/JhmYE9cqexBHUf3fWn71EPneWisFbd5e3I5GcL6cnp17e6AlYukmOTt3fODwMY7jrxxyf6UxpljHlsofnDKcY9gSBz6gZ9KhKrOyxsoZAxZQj4O7Py5GP7v/66UQ/O67tx24DYyM+gHTnNAFpzcNbxvMqMHUAyFuN2SQ24E89fTvQshYRIESVyAhZpMDHQ84x1PYj9ahUN5n7mAS/MFUs4DemAv149MrinyLMIVyp5OFj3DIy3ZcZbPsfyosA0yLCSqqCm8jZsOQue/rx/ICpFZWchVaQNll2LnG0ZBx7j696bIkhugI1aOXfxku0cjYOflx+vHvRHbRXWYRHHIrNwEVVbpwd3Ttnjn1ouBJ9oEjIkgk37lK/NkLzxlsZ9ev0qORAsrNPGGG77oQtkD7uD3B9PSmI5hUvH99cs6ZPPHOc89OPw4q1NI80sWSHud5U/NuHOPuknj26UAROkgWI4kVlyuyNjlBu4xyCAc9+c+1LGj/apUHn/ALyELKxTeAc/MG9Bx15pEZXZcFTISoCqPmZu/Xodw+91pjM6RpthjkVF2RibaVVieTjoBn+Lrx9aAHvJ5kriWRRFOvQ7sPxjlQvXOG/HmmuLgbJkhUrJgqCPlznBXIzg9euOgqKRIzG7xsT0XBXaMf7x6DJHYH8KnaTzQZsR4A5AkwVPTDA8++enPWmBJD5xmJuUudrDpu3CTH+3+AOe1VnZ0lZxn5yNgT5gAfl2r19sD2qXI8qKOLyh+63IEG4yN14I9vlPPSmmGdZjtCKhGflVTnjkqOnHp2zSAci+WIpoI4N0f7p2Taqn5cMv0/22746Y5j8xjEFado0UlWi25ztboD0JPf8AClZkRsXB3btpd+ASuPugZ/n06inAB1G2DD52vI0jEd8c465BHqaYEDhWdwkis7DB3JsDDHzZ54HHX2HFTqWTy4xAuwOV3LHtGN27G3sSOeev0qEMzeSzRSrNIiq0gIYEL1wDx/d604QbZEvPKt1UrmPLL8xyVAK5IPH0x+lICdMbin70Fl+VXwSfmGOOPSrcsmzgmCMJ/ACSdv8AtLjG78u1V44vMVRJbSOX3KoWRkOf9rj+RFChEiX/AEPzWRtzyIpLA84x68ZyD6VJTJsPJA0mD5fQEY+9/vDuDwQRwKbAzgLHOjBOuUHByvqO3TFQbR5Yea3dlZQwAfkAHtz3GOvPFPVBFMEMcpTd8nlDJyBwMg8dvSgRIwjmVmhiZ/JTezFuWyfc9cn9c1BA7BlDjYZGUkHAA68lumSaWV08sx7ZopDwDs24X3Y+uW9+lVVWby97PwrlQI/m2nrniiwy5JLHMcyTSY/hRPuqxzndj9cVOg89WE0b+ZwGkTkM3X04H58ZyKo2+0ybQhKb/mR35J/L8e1WHhV32Oxw33dxGNy+p/x7UCFA2Cd4RsiO7dhlYkE9jx1GOnal+Vg2+YBNvyhyTyO3sORULxSyAN5zh9y7k8sAAY4PHP4Yp0sscCyhpknmcHopO3gKAOMZ4zTAkjw3PkYCKBKyOe/qw96RhgRyRh0LgLvZMDjr9evU9KYkeUWRhiSQ5IGF3nqSfQdOOhpVkScbQpkdEVVGNwVeh7AY/wDrUhkhYW7uySgiMBUDZX/vnrj605pyRDK6IUALYyeCOcjsf/11GVmb5JUDbVBXfglD2JB/Ol8gr8tw5dwGAVcHnOcDv3+lAD5kmDHYJ1lwTsUIxyec4+uaSWdUwUY7FXYy7gFZvx6n1+lI1qr+VnywXXPljkg5wflxxiq16VlUK1usoTCBCuGHXnHHNFgL4RUQNErxO2M/P3OfXtioiHYgM24x4LfOWwO4z+XTOagQSOXILWir8oRgSM45yR/KpIY2WMReQR5m0sI8LnnjJznnnmgCG7t7S4wAk2cZO8lS6kdDz0z9KmtkhjtfLhUeVhUCSHcV9AOeBz71JdN524OV3INkikghccYPY/8A6qiWRHl/erHHsx8kQBDAe/Ge3agRbISOdEZl2biFkH0/hxwPxqJzcGRXwmwN1ckkjPUjjA/OopZE+VxGHVlLYZtpTsOPTPFRPNMkXzwKrcMxKbmK5+XGDx19aQEq+SJWMREisdrBMnj/AIEOSKmjl/eHAh2eZ90fwD6nP51E84WNg7EtJklNrALtzgZGePpUsjGeHe4RmCALh+Fz93JboO1MDndb8C6XqAFykosb1pDL9ojj8xZGwc+ZGMBs9zx35rjdb0XWtMaMyrpUMUoaHzLaSTe25clgrD5RtyMZ716ik6rAiJahdvzbkyfmH3jk47Z4z6V5t8RNRVbgI0kSLb2gVAvRnkOW/ELj/JqkwZz0mr2MVnJ5OB5aeXGo46d6p6V4pa0HNtvBJypYdK55BJdWjzICIweWxVKxR5bsh3O0Hk1ZB1Y8QSXpltLiNBDLG0Xlrzsz059jivR/hdEIfC0zMsaT3F22Qww7BVXGd2OmTjHrXklrB9g10Q3EKzx7Qw9GDDg17r4XuP7Q8I2bowfy/wBw29d3zIxHynqDtGamWxUTbZkkuGLieJ2wqvt3nOM8r7e3SomCRkgNEI+hcg/IPXt/k09EjNurPNhccMF5zz/9bg+tPkiKSSSLLiLIKiRO3br27+uKgoY8b+Ud+YF5fezbdoz+px/nmi0tIYZJ4ZZmLEBpCwJG3HbPc7hTSLp5TNJIq7lOMIGOO3H+NPtb1o38kot2Xw3KlTx23D3xTEWJRDFdyBZLjGCFIjwFYcfMB1Pb/DFNaBJMRW6RxyIAD07H0zgmmh83MSJjcWLbwpUHJO4EA/hnj6VDIHIdIX3/AD8CFPl/3if0HNICx5Gy1CzSIpVhuRYvnIPHPPJ//XTlQSyyPIFcfe2A7d4Xvjr6nFVyLiNndJdoixtR13An+LJB7ev8qmiuEgmjZIvLZApZIlIEkg6Ajjof50AImFk3QvcMdjbmVVGMnAHb6dKeqTeRC0hCsoUiOIg7V3Z7HgdifamzoFulR2jYqBuKocu33jkegz361UUMP3gkeJwmW3dcdWb269aYFk5CAIVl/ibaOC2epPUDr+VQXUjzygL9njdRjeSx35xyMnFWDCkwMkJOWHCbSCx254I+9zjrxVedbqSMoJhHLJ8rKMFlA7bug6dKBFsPNHbSF13SRtuV1IIC8Hn1yTxn071Hd3KOwtkjaV4iW3Fgdqd92evfn370y2kRbSGFpo1xuH7nOZDz97H16nimW90siuse4MwB8x1ysJxnOVzk9vyoQCkbhut3ISXG4+WRtYLjhSc9CvsKh8hlhBLAB32bMF9q4+Y+x9APQ1IYEhfdtCzMo+Yx7y2Mjbgflnr0qKGaeSRn80I8XzuhyrEg9cZ6Ac80wRI1k1zZ/aHdopVbEZ87BZcDoe/JyfTHvSyJLKw/eT+UylnkXcC+OeWPXg+3pzTn2sBLPvkjToxU5k53HP8A30Pzpwa1VU82MoSioFdtxbp90evTLY6UhkHkL5TLHMUZYzh2XOV42g46Ed+f4hTYyiJ5ZZppS2GYf8syO4GMg8H24qd08meMTeaFbAb5QgD7efrjgfhiqdu6R3UvlzNLvRgiKu0tz0wBzmmBZFmQk84STzEzIzliPlJGMdAPXd9Ka6vAqjIKqpd3jfcMYGCeOBk9/altrj7YZm86Hyo0Xchyw2kHqf7wxzmnbpVjVIbrEPIEkTli2fYcYPTBNAED8TK17zJJncJEbLDtz3A//V0p9ygjjMgwZFPGU+6mRj5emeR1+lPt4mMEQg8zfu+d1BC9ei/3sfh1pkFnIplEs32gMrNkNkocZPXufb+YpAQPiJdpkZHfO4J0LZ5GOM8Ee44qe4dI4URLGZ9xfH7xQiDvtJ+bsMnk8daaIY7p496F0wpkd246Mv1/hx/+qpp5ow3kyKoJARvPJVwuVOCeijAXH40wGQojkeQPLmIDBuuc5J9+OM5PPFQPJcNfmx+zuLYx71k4O5uBymOCR75q4k7+YUM0kcYQqr5ZRk87sDoAP5dahj8+TUFNrJH5Lbfuq2Q+MbSGb6+2M0gJJkVUKyIVXfuC7yUGc5weAGxk/lUDt5SKqSJErH91vG3KjcQ3p0GOmBU73MaXO0jzZt6nazZBPUL34B44x0qNEi3SuqA53HY7HO373X17Y60AQtc7gu2TI++VVvnbHUrj/PfnNK4l+0K6r+9YKcvFkqTyCM5xwP1HSrDCaYBoZIltlGcwtwgXuemCTjmniQSKzo5h8zkNsZQynoo7Ecf/AF6EBWiiSJXk8krGjLsLSndnvkK3v+tOZApWUykCTO0tKcx7QeASASQcD9OSKfDbQG1aECQjLMuVbBK4JwxPI46e+e1KstksAkTEkPmBd8pAHmZOd5z0xnHHX3oArzziOSLfMFjYnGSfmHTg4PPX35+lPA2MJR+6WRFHmFNuSM87Seeu3P0pWYmIiCEusiAMM52jPQEe3c88UoETwlpcKplUSon3thB2qvYjDdc9u9MQyAbJPNnCqCcb3+ZWPuy+56j9easyAW8hX7+wngDGTsXJVuf4f7x420QyF5EkabeuQ7r/ABZx8o+XqeRwPfBqFpFZkEILqVZTktu55Xr8u3IPuT0oGMaGUncctj7rBFzGOjMRnqPlH4ZpJZhIXa4zIobaqQn/AFi449yuePbJxUss/lt5rxK0UOFBdCRyenPTbj+L1NV59Rme2SBHkkjZlLKTu+Yt2OOOfugUgHNJE1yUG5QjYbMQOw4wQMcDngnPH8pYHQqZolYn5t68OEX+8cn72P5GnPJLagtsiG4sW3AbUGMbc44I6c8d+oqKQzPIXRYHZmbzJM7z0wVDA+30PGTQAzE8k7JIqzHO5lIwVXbg5Zew5596njt/tcIit02HbtV/vKASeRzgjAx0JAGe9R2kpfzCUXy1XbucYxyT2OelPhRJWR8F/M3rKxXJLDGPlz93+XPXimIrRxH7PluELdAGw49cH37j1p5tY42kI5jMZDSEY+bBAOBk9h7/AIVJEiiLfH5cPJyysdzE9A3sR6D0p6KVL2xLttG4/wB5lVR8xxnp0H970NICo4+1Blg2l41CoURWbP3Qo7/T6mq6WTvHvTzWZYt3KHHfrj9fTGavxECPdtUu0bFVLY5bp09i2AAKbE8kVs/3ELbmGX5EmMZOOny5FO4Fdf3cRG8EvkkH5SuOmfUf1NNKOShYx7XbJL/IvI5z+vrng46VaVFmn8hIWiCxASuASFIb5v4t35dM+lS7cxDJBb5mdxH3GBn6YyP1oAoTSMA7OfOkU7lGOwGeBjCqM8//AKqW2eSZ0KwTo6vtAhI4zjgYHbnrz2q6scKLNseaVWG5cZLMP7xHyrxuIHB6VDLCIYxc20W7KfK4mALc8e3X2z0ouA6UzGRY0dvKyB+9O4sOq9Rx3OBjrg9Ko3TTC4hjSNncqrYVlcEAEHAHGM9snHrVq2Fw1ixYRRjcEjc9VxkfLnt835+9WZkIhjkPlr+9BUDDP0+bIHbrxRcCnZTx3LzfZl8vcQEV0wcgc9h0z0/I0ATySOskSFzgK+w7cccdz6Y55/WrKWsHybPLEi7YWkODgAZOR0PHfhugppkZU8uLeGRt7RRnPGf4WOMDnPrzQAyaL91MDLIzKuI9w56cZx/DUTyx+ZmGRFDKGViMg/Lgn155+tEitdxQhvKURqUYFBsUfw8/99H88mppokltso2ZwN7OicFMfwttIwvGc9zQBXmu1t5NotjIqg71WLcgTqM7gcdRn6DrQjosbNl9hXPzABhj5e/PbpjFWAjK/mDy51bOHKKQzFcjLKBxnH64PWo1hQMgMimN0KshfJXuSMcdf/1eoAguHVAjpNHHIuwo2UPrgkHPPB96kMaPDMZUTEh4AKs5x1GMcY4684FDQfLJLGzRGTMR3JtG3I/i75Ptxgc9qSKNkiRSGLyKHHznCqMBiwxhl6YGaBkN2o81cwkqzFT5SjDHvjJ6g+tJLZpFFbyIRKkqdZhyzHsucFu3rzV9YZ1i3S7njkZdzKFXaAvb0HH1qnPexrDvFwUUK37lyW8tSTjnHp655z0oENtreO11BZEuU2PIqsOqHqNxQ46Ho2f4TVgvIIxskjUqp2MF+br03DouM/wkUzzLS+PyE2+5TGqxr8y98hQNvJ6kfSpI0hlIjmEYuHdo5Ez93jA+X+EZY/U59KYEHmMT8ryeYzdWDD227s9+eeOoqQvvjfafLjkfau58gEdA3bofvfnTQkSW+/KmXcqIAy7OMbsr256H68Utqsk7KiJtO0nKpvXpnk4+90HPHSkBXhMjRsm0+WSyRMOnmKT7fUA+xqdVd4pGQJOCjR7og2Oeeg/HHbPPSpms/LmRHTaq/LsLjjBwRt5yMkDnnIpj27OcIjtIIwSHIBbP8QAHzfxfTA5FMC26SzsHuGGwkr8sW04/vL+XfORVNnLXZ3mMSqA2c7ApHQDJJJP4flUo8ySOUh42VV3DkNhTxx9SentUrMkoa4luVkkYENGw+Zs8blxz68H0qLlECSgypkK4AZMKoI9SMd+/SpFVnk3s2UaXc8me5wAcfp1qa2MMXyO3yKvEe4pvweCCcgYHc+9VDLbm83LIrQscA7gSwJOOT/ng0gLWM2iKibvm3LJt+8T/AAn6Y4607DGaQRywQOrEjzMAqeG29xz/AEqu82GIR1lVV3btuQre3P6+9TgwjEhXchxuB/iY9xz1474piIpBKpljeLhS6K5jJAJ5wNp7+vSlIWSRi7tt39kBZ8qflBHb/PNMImVSREp2hV3puGBnG0D/ABqR1GWaOEgyDlY+in0+bnn/ABpgNtLeWZZAXhkO7hScEE9+nNOlhk3EMI1MmBHHEynaR246D1zz+VLJNKbVQNo+cAkc5x0xz1qB1O8SBd7yN80kxBwcYwM9D780rjsOVnVzbzQxpFjDCDCuCPc9R7Y70/euI0Fusn+1uzlc5GRjjgdvakttx+RcqPl2jZnJ6dO/U8e9PWK4KJHGofBKoABhsnnIJ46UXEQLKZpt6XDANkFJkzH9fp6d81IJN0pf7RGdrkKd/wApIHP3h/OmgwSS/M0jllwd52c/Qf5xVZLKW4cxzBorWMFvKjQqG68lupB7UAXEDM/z/MC25kI3bfQ9R/hTlwPM2N5uejhcbvRuR0p4jhkj+0PiTlUjC4Gx8cbs8kY/ug1BKsixHY4jkQE7PmZlPTI7DFACLAZNkbsVQnJAzz75HP4mpMmO2WPzGfkllyyj2OOtC+fa24ZrNgAT+9lY88c8A8/Q8VTa9vJJpI7eBlctw5jwfwP09OKALU8048xnwzMdufL3ZPscEf8A6qUQJuYTfK+M4I+8TyR/s5xTl8oQEGRi5O5jH1GOfp3p6MbmR0Zz8zBsyMPmxnOdvFAEYto5AS/7rd0CDnHb6j3J7U6dnEPlKvkIi/KflVsHsvrmp3bJ8xDHHJsCoqdG75I7flUG7Ym1/mZTuWQMCQOR19DQBIqG2mKW9znK/L5YbdvGfxPp2HtUSou7ZJIXRE3KAuTnHIP51Xa4tYcRJs8x/k24bG7OATn8+tW7eSOVTGDIkuwhgh3MzD1B/lQBJbefbDyS25du5Y84yR3K14t45tft91NcyqfOa+kt1hBwS3J/BQMfnXr6iIGXzYPKQrl2YYZvxPNZt14X0vVr+S/uYneaRfKWUOSi/LjIU8hjwu73NUhHic9jKZprSFMRxfPtToOBmq+j6LcOJ5XQxp5oQ7+McZBNen3ngDU1uDCk9nv27DIA5BVvXHHt9arv4V1e9nu0MsCfa/JtvNZ1G0LjdMo/4D04PNVcVji7GIjT7hwm6+mYQW2CMu2VAC/m3SvZ/D2mS6JodrpakyNEm2RztUK5+Yjr6nqfxqpp3hez0WNGsDDmQ7PMkbezYJGdxGMHHRa1xHJ5BWJBCy/KzuWxyfTrj/EVL1GiztyhRwqL5gLOTjAxj6HnvTAVuIz5O42y7QUIbO/OduevbqMUn2mSS1/eGF5Q2N5bjaOnrk+1PTyYFUzOxLHKZiLP09Aee9IYKwUsGi2NyRjKgjv7kDinyTRlUmaQyIilGByN3Q4+o9sUgurpmj3l4wq7gjEcDtuA46Hp9KYqmZJcPv8ALbJDxcrnvnuODjFACvPD5Bd5HjcxgKEK/wB3hcjp/OppC8jFVcN5saruJCrnAP3cDt3qpaadbs9uPLQXLjcm4/JIM9OeR7+uO1TJbfOzNGXd5PLTY/Cn69+h47UALI8bKYWQeU3zRRcfIp9vc022kuB1SZI2wEjG4Bzxge3HPXiqEs86gLFBIxj43lgvfjPXvmrBZz5gYTsQvyjH06jp68ZoEWYYt3ly7mUPlmOB0zxgHpkD6nFSqojEojePCqXYF2cLGPfkkn0qsIlYF5o42VVAlEnU+nRvu+9StC7XKCNJPMiba8S5znBIxzyOBgUxlaVoWmh+1NtWTG2I5Zz68L2qVJBiTzsxJn5dnv8AX+fXHvUck0ySlxFG8rBVeSRuc/MTgdsH04/WrERKxsl5FtlY/KgO7ecY57/xCkA1ZAslz5X2coIl5Lbt567frgc1BIlukTnzAp+Vn/iAA6fQZz09RUoePZM48vcY9xwvzSH09uQOcDjNSQt9oYEPEjNkvIX2g++R0xzn60wKrxebG8wa4SU8fLIGJU8sWA5yem31qaRo2idG2TAy73yRllUHkdzzipQsYZE3w792UjiHGSpyRnk9hycfjUcCmaVTNGYSP3mIA3zgrjaOcfQjryPegQsysbNzM0ojZvlZuWVsc4HuD+oxVdLaJZWbyg+xVVhK+0oepJPBJzwAOlTS3CTzNvErhXLbGbyz83fGPlPHT6VHLPNsD5cncQXBUDce7Hqx4xSGN8xZE8uKPyodqlmKkKFx1A4z/wDr9qsoyRNLDMjLE+MN5ZXYwHGwZ7889uKUXkUsz/ZmtS4wEnb5zHx18vpnOcEk1UithH5OwvcNjeshlA2nnPQDk9MfjSTux2LCrGY5fLRi6bWkcHBj9c+vBH86hu7lUUmG3uS0khRX2DDZOFy/rjv+HFD+ejK8pjLy5XYfmPJ5LE57fypwWNoha4QzKuFCuThs44HTdxyAOMDvTF1FNxJHfuYlXzlT5CsW4gr6c8gZPtTfs3kXTK7oRtARAuxt3QAjB474HvSzp5dilzBPEzK5RgyndjJHG77gHXaPU0QwPGFaS5MkY3SbY2wMZ2llH8R9jzQAkEUxVdkskvBLngbQQxI3dNx/kfwpHkiiheWMKJEdnzIpA5UBt2M9+4x0OalSCVZ0zbbCqkDcqt8wI9Txnpj1pljNIs7qw8rlQ6R7W4zx8v6FeenSgLEMUGX85SMIGfzD91vTv16Y7fyqR7cXNxLtMbuzbS+CVIADY46+/A+tNg85cGZDGV3AOpGegzwcheuCPfFWF8tpB5vmTFgU5UDPHyjp7Z4/woApxN9lbdvbzWI5HyblPr6fT1qVoYV3PJL5WHUq4JYBgeWzjB3fNwPpUb2dreupeD98oypCtGMcZIZjnAyTnnmrX2aFYZLeVMBCfJCOx8sfeyc+mOfrnvQBWlV2mkbY7C43MSVwrce5IAXjP+TRNIY5xK/7tkYCQY2gHGdxH8PHPI70/a6wpNGqpK3l7FA+XPvznj145xUskT/am3SnfBFnzE4b5uOeexDDnJ4oAWYqVZmdiysyl4nxuwOcc9Se/wCFVF+z27PILc5G5m+Qt8vTpgcEnr1NL5nmbA4kYiUht21tucAe/wDe+vNTR3MZfyEZzbh/LLo/b+ELnOB/QkfQAayjy5ftO4QrhxFFhf8Ax3+7936kc8iniQxSEqT8hzlnGc55K7cj+nTvzSXMxRwzRxgy8Elwvy/QAnPGccUwTIzeRb7Csm7Krh2ZsbvY4P8AnpQBO0IgcRfaAu4hZHT5uo6HPH3eM+5qFZIkbYyTAbWJjfKgvtzuMhPzDd6j+VQSia5iRrdDa4Xd5hl2lhjaOfcg9PSpJklEsYjSLzF4lMpwzMzd8+2OaAD5ojIxkKSL5bwz4wyNnAIBzxjjA601rWDyOHZzyBGy7twbb8xweOc9etXI4Jov3lx8kjbW3Bclc/c+UDK5OSPccVWmO0TNO7M8jrHGgAwu3nb/AI8GmBDc72X/AEe3EkjIGeQ8ALuOFI4/Klii3MyBTJIrIzxu33DglsnA/iOepqRbS4t5XG1xIh+XGUI/2lJPHpxz2p9xJ9qmV5JAYQdxEmAmPl6jg4ODyfegCLyWLolrIEuWOEAjUY5LZOc89xz3po8s2/zOfNjQl/7qnOORnt/OnnISNdrQthkZj1znqeOOpxx60PGluxWRfNT7gfLBmJ7Adu3t+BoAlkif7UHDPKNnltG4PljqDk4HB9/Q1CkMVoD8/kJkjYm3J5wcDcQOvXH0pIo0Rd3nG4VyfmHIP+zkk47k/XtxUEjF7lnhRQkeArojYxnIPTOScYzQIkaQQuX+TymyJGztbbkjJPr1H94Z96bAsTtHDGcFsxuE9j9efTHTipy8sdzFFHvEhzgKmAWOGIIbj/8AVUQl/ceUrNgnflWLMox0Htjg+nrRcGPRZFRQs0RUqW2FtoZhkY5BXOMrxzyOtDGV5yu4rCflx2PPykk9eg5OO/Slt5JlmYow8tMo5SInG4dScY/D60kvlRgSPN5hWQCNzuycgncB0xx6dfSgCwJYsRICjLH8xjMh28c/dI4/ofWoZ/LlhZoy20oA0kWBhuoGe/PY0sbstojQtOIpSN6snG7G0/8AAlznGOn0zVKS6tlvIUKzB3KvE8cnlpgdd+PUc59u/NAFsxIsjJIZZfMClVldf1+XPHX/APXUlxDJFqEcc3kNH93ZuHG4EY4/E0FEeM7WaSFYyIjtVhHxxgZ4A9Qeo/CpGXybSNZnhJX7oAJ2ng9cZz1bLdRxnNAEYk+UytFGQfleM/MZBkfMx6YGOmfxomEU0JRJFRvNHyMSFxz+Xr3/AK0kpeJJZHkLIw5EbEEZBIyvpjBx7jr0pINRtxDIfOia2ymGjiKKEK7QRx33dOOfrRqA5IoV2Otv50YRgzxLtZmYdcH72PlwOn40SrcN5Xl7MR+ZtVdwTjG8gfwjp1z1IqdGjE+xE42YACkoecYYr/Cx6c4FV4iUimiSU57RuocY4yQecMMY/HrQFhywPHK/nO3mW7YXOF2FueQRxjdnPTp1pFSLdsJZv74XOOeccAgcDjPP6U2MMh8ppWUNnCIQPl68fNnA469+lSnzIvMtjOD5fQxvkqepLevTI4H/AAGgBolmjmGySdXfKoo444GMjJ9PWoF8tbQLLLIgETER+YHQMHJzt+vPvnoKUyHywIw0kM0eVE3Oc9Spxxk+vJ5phdZUhaFlRlJCg7S+wHg5/j68k+4oAoxtfwhCUM6ModQUIHc9FxwMZz6VqQQvFeRSCGSMD5mO9mGD1O75cjrxznNV2tgMueAG2u4ypOf5cfnzUkczBreMNGqr8ys0e7aDyfoBz+R9qAG/Zktps20H3WDxTh125x35wD3wOnHpT2Hn28Ue6ZIo5MrlW2/3ux25G45yO4OKeYHJCxKrNMrZLAfIvGRjnI47jPvUq3EkUBZzLvb5WfaSH24w2CDk/XtigCOaAvOSpeTaxyuMKH6j5T156Hb1471Z+3XP9nRWaX0qRpIYwI8LuXaMDcuCc5/TmqlxvkdpVVWDneQM5RcnByRwBwfxFQrG1xJHFb2wdlXIUN94g53A9NwA7ge3WgZYEmQ1y584ooVgoUNnbgluCD1/z1pqsjboPOmO3aoLKS4JHXJ79Rg96Rorg/M8agHMiJnBHRmx3xg/l9KjW6adViLuQqny/McKFH3mz3PHQE/zpgywBH5DFonD7QVfpkH7px9TSsott5laTzlH7squ5VT0wT36VGHuEDxNMkaMVbc6lcEDrt6/TtyahSJd6QtNCIWbGxCccdB8w9SR1xUAGwHEcc2FRc/MDh/9nj+lTRp5NqQiR5X5WKKPlwSVAJ5HGTinIsETHdbbMqNsX3Qo5+vGc8jmnRiNYt/ByflGQwHy8nP97p19KBjyEjjSR1m2OfvMquARzj03Yz/9eo8yeYfJjkjjP/LZ/lLEA5xjt/hTYojKjTBFyNzMq/KBnHOTx+frTmiNyreSibh8uEJfd/vbe/XkntigRE0tuFRHT94EAPmBhhe304NC3QuJHZCGfaSWRMZxgYGOD9acbaIWwnJL4wrB0YHjkgcY7jrSK4YxmOOFGjOMyqgRfT8RjrjOTTASFXLebIECpkAJGuR6k8dvz+lSTrC05eIvORuSEMmwnuSPbrjNBaW4kkcFnMi7mdMDcB64FS26CaGNYvmTYTIHHEYH6dD9eTSuPUlhaGV0jllePCKMnGwf1b6cdOtMj82G3yj/AGpfvNyGHHAGByB7frUv2omGJFkeNBJtKvEu0cY5H3vT2qAMY7Xa88jIAxB4OB/un1z26UAVVnkaaMrG0aYziSPacA9sc9fWpg4Z2wTGVT7pUYPGNpOemPalfASWNh5KquHD7Q2RkfL36fWmxwMqyRRq5G/lR1Y4yMHnP0pgJDLOdzwypGp+UbH8sKcc4+g7il8wkjYjZI8tNj55Hv8ArirCmOOIR4cOmdsbYcMv90gd844FVrfzpQ0kh8uMZ+RVHmHjPtjP5UhErXEjtCEjnecqd0jncNvp1z29KsBIYCzSWxUswyY2Aw23knr7cdqhhmUrGkhVlzu2nLc/3voKa0EVwV3ys6Fd3zMFVf8A6/pSSsU3cib7VLIA8MccDIZEUEnbzwTgVPBEVVXYxtMo58tiSeevP9aktY1nkWOF8hg2QxVV3dvc5HfFRwwBplWMyttPdflI7jd/kUxCxjCnMgBkO3ueveoZZzcS58yNn/hXZxnjqOafcKpRY2lji2qEzy54J6A/0qJrQFWI2Mow6usZDtzznmgQkgcM8jTREIwRg/BB9Mc981PbhpGfLxoi/wARO38e7etCxwx7JJYElUS/J5hyQvGR14pLg74i63DwRD7kbSL0P0A4xigLCCRLd0by/MBwFMbnPHVR7VLbkRjDLH5rHDGLB2Z5GT0J46cVPbvsVSreYzIV+YZ3fgelRM2YlWVGaPdwjDcM9l9ifWi4AgKSl1SEM6lchiBxyAccY/EninQurQpEVjQkksrouMHuW9v61DPEcsk8exG+ZEV87d3eoopZV3wbkVg207Nqn2P144oQFxmSS1Ito1PzIgQ9T83POOe3HpUBaEQOJ5GlVnZy7FCfTHpjocD86iUpbrIqIUDKcRfKTkjqzN0PHaksVi8rdMh2Fd37t2L/AIHGB70wLKSi/QW3yG3VSxDAoMD0Hb2FVoYINplhcKyDaS+7eT6Ke469qvHb9oEZlMgQ+YskkfU7e3fGT1HpU4VJpfntJZ3SNhuY5K8cfT6Z5oEQosccPnJEGh8tSA7Lye/1P09+lRM0kcxFs07rJhvNb5u3Y578imzbiPNaEOqgcOoVs8d/XNQXEEIu3ZJJGMmFWRAVC92we/PegZJKGllnjS2hMQU+ZvcOcfn1GD0qaVG81fOYbpFzIiSbWVsY2kcdvSq05t5Y4EO553baCDjdgfKcdP8AgWck1OJY0dfOfK4wXwGcdeSD356+1AFho7OIRySbIo2mEgj5ZgO27noRj8qplssJGZnkXMjAbW+Xr1HA47U5JY5V2MjBUVnyuRkdyx7UsRK27tHcnchPkSNxnPbHUccY7+wpiJGiTdPDJgSSIxjBK4VjtLEg+3fNJFIw8z97M6Jg7QyrnsfXgevvUkTg3RwIQ8kRSND0Ruv8Q5HvnvTSxihUTeUZVG5wVwqe3GfT9aQxJsO6QCcLcOoblQqj0z68Z6YplqJBbIrI5jf96xkI+c8EfUZHI9/rUSoTHL++2ZQKWU4JPHKjbnk09W3xtbQuSW4IK8ejbiTgjAz14oQC26kSqyW29uDJGu5iqnp8pHqQfwpWVp4vOTesY/gRfukng+nLY9frVOaUx7PtOC8hLH5zI7fXHHb9a0IG+1WsRa4jZFXmF18w/N9Dx09OPen0BEkccFqwhF/83Ctvi+/zzt9PvYwcVXkDxpNBsjjC+jOV2rwWAbp9fbFPjSMXDT/ZpJtzsRG8YYEnpj0Dc9OmKiRisAgTbLKSpLIVby067TnnGTn8KAsKY/MwrQiNt5jy784yD1zwBuXA/pTDHMkZVYiJC5OxgCvYfNxjPA9amYBYizLGpRpA6ZLMuMZLcc9cnGOAo7VWEr2zxTeUnmI+SSpG5SBt2nqcEN9ODQFivb3rTRyxTWgUl2ZpAnJPQhif89auy3MoZfLKPeEAx/LjaNvBI75/PNVHgiMUBilnZpOVK8BWH8TD6d/p71JKn+hDzXPzS7QJMYIHPI65Pr7UkBDD5k1sGDYC8Oobo5/Dp16d+tSb1lJZ4wsSjqg3s3I5bnjJx836Uzzbp5XOxBErAKCu4Z5wFI6Dn9adIwmuIA/75g+ZWST73PJ2jG7k+vfHejqBNcS3kcr27K8KBgjyjlT09s+nNRwuEkkhllkTy428xol5JHb5enYZ6mkeKd9zJIm1RwjHaTjJ+79BnmpIJntg0tt8jOMnb94ZzhvYdR3NACNdJLcO022Dcp2xKu0p32ZyQSSOf901XSYt5lyZ/MCqrmI/LCpxxzj2HSrQe4igWPy5UkBVQhIJ8sliQu7kbm6moViEOJXlkJjU5B4xzz6kkf40agQQXbMI/Mi3KyFiCMmXvgHOfbP4d6nt0Z5wvm/vE3OokY7mJxtAPbr09jUk0cy7iSQYztIAU7vlVQxHqQRjB/Cny+Tat5fns7LMVYnqp/oOP07dwBqEXELp50vls7bMk8YwFP8Au4b/AOtTysha4eV/MaU7AfNGCSRwMZwTz14pmZLeESSMvnu+IwsZc4GduePr75p8IlZOI5I2ViVyvzIWJ/LrzxnkdqAGJEsOxZVcPcJ1HGFI4zz/ALPp3qusd5FFG0Ilf5/uBvkV8YG1fbn8/apisCyNMoXcrlhjnbyccdznvnpxTbAzx3TmZWypUSrLFtXdnlWJ9Bk5FMBz2xLFJIUA5VXIHGCOM846/nxUi2cNuzebI+4gqqh1PybtoJIP17dxUcwju2hgt1M0DrsRAQSFGM+/PHP+BqO00+3F1kQNEnKujjJX5mHOOenGMfSlqBaEczTKIlTc5YGRYtxHZs4684JxjOc+tQy28jyB1mUbP9ZK2wEKOM+2T+ODS38LxJL5EkaksrMpJ+YHOCcscYxTUvA9ilnPPCg2AMTuO4n1HA69fpRsA5mbYgEqyOCQ2RtbggY/T8hUMkryz3IV4mnMolDEZZDgnb+RzgccZ4qZFeVkSYSOXyZM/Nlgw3bicnP3uOQOtPYwpHIrvht7B9vIGF6bs/MxBwPb3phZEUMjjyy0TDcxRBsJ/h3bvTp03Z6+9NilKpcERJGwLI4XcRjno3UD5gMeo61AZWVIrXcqlW3lg/XLZxnnt/Fxx2p4M0TTOCnzTbVh3A5OcDC8ZHIOTjGCaBEoW2ulWW2cTyY+YPGSY+DkLk/7uDnP0qCPmzLqRHKp8vzGIZ8sODk9O4z/AIVLI8fBcW6yRnaQspLL6ruPDc8ZOetO8vzFkISSR0iyuMAAjPTPH3c44/CgZHHDDEu1J3Llt0hlwxVR0+U89RwOfx6UCMoY4+fubpEDgjAA5HOQO/HPPbpTmkazVHmVovMK7W3bnkUfMyjjjnPJ7n3oWVZ2/dqT5wOPkMZBzzkdB+ff60BYigDGxZ0KRvFk71b5gME8YA3Z5GeuOPekgRR+/EbtKx+VNzNjIJ45/L6VK0ANjF5ltGqhceWVxuxj39R04P4VAjolpIkTkRMx2BEy2OvXv1x9KALIwUkfz3ilZA6md3KKOeBxk9+PfFKZDKjMZgqShQzmPIJ5baRxg89/enrIrQRMscUMccW5o3U5ZgOSec4PykHGOvtR5Ssdu9UdZD8pZWLKy/LuJOWOc9M/qaQhzXUMcaOquTGCvmq+BJjGN34cnp6dKpMBPOAJ4TDJGH2E7Bt68H/ZbPH881PczGO3Rp7pAEOxUJwjDb90Z65wTn1z61Hbsrxi4+zwN5UjFxHCCucH+DsvAzt4z2pgSIWnj2lojJkln3fICq4bJz2wPw/GmyeZdokflqJFYjMRIZuOT8owc/L8v0qbZtkJulO1idodRiQkfN8w7rn6USyxLbJ/rI5A4VpCdpJGPmHPVcdOKAK7RMzTYufMf5uZIx5bBemcNgt1wOOv40XMjbpHEY89SF2tlJNo24YdjwMY9MZ9aVriO5DwpcrIsB3SouHcjIDHOecfQ9abIFnnfc0kgwpZkj7cbep4b72cn19KLAWoLh2hWWBZIxI3lRDaqksRkqMdOAvy+9NdsQO0DQecsbLlGDbQG4xg8Hn9R6GlSJ/Jfy9picM0SM3mbznapzjGfoecetRzfYwIlMdwyxjfLIWwCDy2VH6UAJNLc2yEQhDIdqqGmUbRuy2QuPlO7qSOtEiszNJGHf5yx/hZx7DjJwx4x71Iq+VeZDpHvUlXiJ2/iMZz3yfSoTDLIi/feVVO/btyueeuOpwGwDzjNACWuTGEz+9h+aPGGznndkYzxt4B+gqwyllZQzBkVU8pYx8vQ5XH41QudOmuXEiea+1A3759se05K7SM5Pze3THYVZhQ3VtbvDFtYt/qm+VkzyzY67Vwc5PGaAGhPtG+V0kmjkzyG2t3+UbevQdQMAYoWOMs0XmTyInAUSEKV/vAYz90A9O+RU9mgjPzsYfJYAs6llUHPy8EemevTioZ7jcZmz5amMHBfGU288/ljvQA9o38sPGFaIq0h8pSwjI+98v5c5+veoftAj3MVQLwRG4DA4Gdw6Y+ox04qcyvOUAThXJP7pmxzjlc4A+96cH3pHtyNqPOGaN9hkRtvr/3z931HpQAlvN++TmTzPMVXfrhePvAHPqT/jUVt5xgbygWmiVyy4VTgsAWBP8AwEYwc5qUpADGzvIN0TBjswVbOApOPmU9D60ohlZ1CmVcfP5eOzcjHf054HvQFyB3PnwKkLTM4XysyLGEKr/EGzgHjB74/CmG1aOBI5ZXLA7nYhtpIJ+bH8XPGP8AaGak3oxkklhtl85yu2VyRuPpk8bQfbmgROVyq7VjO5HST7xDDGO2O+Tz+dAD45HnuJ32jzYR+9Me35N2Mk47ewppAdnLxRqudiKzMyAfKck5HIz3HbiqqySyxu7xyA+YW8ohkcZ/IE5x3AOcCpxCHjCmBlAjVS21sfN9/dnqcflj3oA2X/5D0n+7H/6LWjUP+QVaf7sn9KKKkZUtv+PG1/3j/IVnab/yHrb/AK+l/pRRQMvL/qpv+uv+NZ2s/wDINj/65f0aiigRuxf8hG3/AOuU/wDKs6T/AI8bv/c/pRRQgRWvv+POy/3E/wDQRW5H/wAgzVP+uqf+hUUUhlB/+PSf/cP9Ks6j/B/1wh/9CFFFNDK03+tvf+uh/mKbb/6yP/fk/kaKKYFlP9SP+A/0qdf+PaD/AK6S/wDoIoopIRSi/wBVP9B/SrWof8elv/13P/oFFFIQtv0uf+veT+QqKH/kCj/rqn/oNFFMaLEn+uufq38jVfTun4r/ACoopiGr0b/Pc03U/wDkNP8A7yf+ixRRSQGjc/8AHqP+uH9BTPEH/HlY/VP/AEBqKKAMWy/5BTfT+gqW2/1o/wB9v/RdFFNCGW33pv8AdH/oNaUP/H1bfSP+T0UUMfQj1P8A5DQ/3/8A2oaml/489S/7Z/8AoQoopCKJ/wCQxD/12H/oZq/e/wCrf8P/AEEUUUwKtn/x72v+4v8A7NRb/wDIVH+7/Q0UUiixb/8AIw2//Xc1k3vRf+uh/wDQaKKoksJ/yGp/r/7LVq//AOPqf6yf+gGiikA646XX+8n/ALNVCP8A1b/7i/8AoVFFMCTVP+Q2v+4v/oNWJf8Al1/69x/6DRRQMsQ/8eL/AO//AEasjSv+Pu8/3P8ACiikBsW3/IIX6f8Asi1Tuv8Aj9P4/wDoVFFAFmb/AI/j9JP/AEJazL3/AJZ/7z/+hLRRQDL4/wCQCn/Xf/CoNM/1Uf8AvH/0ZRRQwF1f70f/AF9P/Okv/wDXR/8AXmf5miihAWJP+PO5+i/yqiv34P8Arov/AKMoopgjo9L/ANQn/XdP/QWqg/8AyOI/3YP/AEEUUUgI4Puy/wDX6n/oVJF/qG+tFFJjW4lr/wAfVp/wL/0E1W1X/VD6Rf8AoMtFFNCFtPv3n/XvH/7LVTV/+QJq/wD1wX+ZoopsBw/487P/AK5f+zmkk/5CN5/17j+UdFFICi//AB+w/Vf/AEbUup/el+k38zRRTA1JP9Ro3+9J/wChiqOrf625/wByT+bUUUCI7b/j+m/3JP8A0IVcH/IyP/vf+0hRRTBFa+/5FW2/661Lqv8Ay3/3k/8ARVFFICO1/wCQTN/12P8A6CtW9a+/p3/XOD/2SiihAMT/AFh/6+E/9DWq8H/HyfxoooYGpd/8i5a/73+Fc1e/8eGo/wC7/WiimA+7/wCPCf8A3V/9ArWH/HoP+u9x/JaKKQGddf8AIZtv916ki/5DR/33/rRRTQM1j9x/+vr/ANmNUh/qIP8Ar5T/ANDeiikAzUfvaj/1+/8AszVdh6yf9fK0UUAQ6V/yHV/64yf+ixWhL/x+6V/12uf/AEKiigCtqf8AyA4P+Bf+gmoL7/j9n/66j/0BaKKAZNZf8jTd/wDXZf5rTT961/66L/6LFFFICC1/1Mv+7J/7NVfxD97/ALdbX+VFFMC2f+P1/wDr2krAT/j/ALr/AK6D/wBAoooA25P+PKH/AK5H/wBCWrkX/H4P+2//AKBHRRSQH//Z" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAJBAyADASIAAhEBAxEB/8QAHgAAAAcBAQEBAAAAAAAAAAAAAgMEBQYHCAEACQr/xABkEAABAwIFAgQEAwQEBRAGARUBAgMEBREABhITIQcxFCJBUQgjMmEVcYEzQpGhFiRSsQlDYoLBNDU2U3JzdHWDkrKztNHh8BclN2OTovEYJqPCxNNUdtInREVltcPiRoWUlfL/xAAcAQACAwEBAQEAAAAAAAAAAAABAgADBAUGBwj/xABEEQABAwIEAwUFBgQEBQQDAAABAAIDBBEFEiExBkFREyIyYXEHFIGRoUJSscHR8BUjkuEzYnLxFhc0U9IkQ4KissLi/9oADAMBAAIRAxEAPwCEmUnL9RckUWVIW7NkLWxEXFLUXQtCdGpvUNAPnuQTbym/GHONmKvoQKVMlNU4skrZQmSsltz3BXdYHGng+vPbDfPrKGazT6bVdCWjGRGLK4m29EWgamm2VaiHUKHGo6LWsRiM195dbg+KnvLiPF7aKUqbkNhwX4Gg3Qbdz25N+2MeI1cscroo9AeY5rJBE0gPO6lVNqlIrbaZ1WjpeneIdjh59zaS3cp87jgNrDsFm/HoABZt8BKqx8YikhEqXxF8Q8lLhsrzNnzDWSfMAo3PvyBiHUqr7MpUFMLcmaLtuo4U2QOSkXsbg8ix+2JrTpKKiG5iI7s1p4rbbYLCgGljzbaiOeDexvcj04xyZNOXxW5l9lGavXqeKXJyrPLpkMLWVnbGtKyouI54NgsmwPIBKOeMNMiJHrlJlx26y5IMN1vxKDFdUpji9yr6Da5Fgbjz8emFOaZ+aqZmFyTrKIkxoSXHQy0GObJd8oHNyoH/ADj98NSA+3VTU1VRuRDnjcafiIcYd3wNtTZUTzYm/mR7W7YLYyeajrDQJHT00fcS6uovtI2gAttzUlTgIsFWUPlkc6u49sTOk5dzRBcMwbGtpYdRDW4i5QHPMu9yFgpWLi97C6CCeY26WTBW/PrUR15t5yW2lxvS8ly43L8clWkAoJAvybHuvy/mFbqHXnA6NporZDaQ8WuOUqTceW1uf9GLJDJ4mJQn6n5gK6dUKDVIP9ZEozIy6kY0loBpwKDRLhSdHPDosdBN/YtUpymzqU+/VENvQ3loiBMdnwxbdDtnEtFCghZU2jgkWuQbC/LyvNsGnrciVqA1UIZS67GQ4SXGHHG7AC5uG1C3lNxa1u2GujSoNGj1GGxvrVEAUl9lpG0w4RYILbg/aaFAk24A4BvcDtHhmS6LWNvm5psTDblMT4sajypUGPGJWuSozJLUcEWPAJsCE29RbgYluQcu0CHSp1QdLrTgS4zJivoQ+EtLDY3t24AuAoEaAULAF+cMUGHX4ssSMvbEokMSnFa/KkLBUClKNJsOUrsLJvY9wcPkd6VFqrorFCRJqMZiTKk6WvDKShdtKw+gCzdy3dJ1kjjknGKZmlrqwJhzIQ/KQtL0Oa8iQtjama0+IRcqAbuAeQknWeU9ubC6h3M1NYosSFU4EiiuSWUCKl5ZcQ6wjUAQXLEA+W5sskWte2GjM8fNSJ0BkUxyUqoNNSikJcfDAICQkuuJAWCCASD5dOjum+G2nZ5qcymTqzPhvvSJTweVLakjwjZLzgU2ASPI624G0K4WCkeiubooczQd0C/IlWXaXRKvkynq8TLLUT5SYa5LiXGNw2W8wr99vWUEJPIBJ5KcAiVjNLNRj0zwTEGLHIcI2g2HXLcv3PDjnYX7+nvhp6cRpLFZRW4DzciHSGHS4+takB3l3aDiUKBNxqFibWuCOb4es2yvxGWYy5z7jcJTsdpyMFuBuQfMUhKAUbhNiQT+Z4JxcYssliUHd5KWquqG/Pekz4y30aBFbdauHXARdsKbug+T1JB4NjfCnLlQmU6ZUUzkIbajpbe8M28XGww4kKsL2JG3fub3J54wjhyjR8uOQ6jGfhpWXTIdeaS26tsBoF5rkocHzW+RcC5uBwSXT5mX/HNQZESXW4EiMwwx4ZlaJRYDalF3dITY7hA7lCQV8WuS+QkIZCutS3KO2mRBhonyGwtoBt1stAebZW8C2fI2eVuEi5ULBAw5ZkYkRH0y0Q59MWtuMiUhbjbzt1thSkNptdCBqBGrlQNwb3wglTp7KpEF6ZPqRRJCHkypA3gu/ms61pC1BFgHSgXIuAkd17NNdq0FmqVSZJLLzrkNqQ5tqkyC2Cos3uNsK0lN3F2SRYBY5wrQM4LFW7mksbMESPUFyKtTYzyoUlxTXiQppx0um1nb3W4EAWAV+zv3sAcPvTCu1tl+rKW9SoLFQWEoKIgUSgK+a3wkgoLYsQbcHkHi0LkUunt5iZqC5ctKHJNzCkoblL120tRJfzErQsjgkAkCxAPGJR0xiyKZ+Mx6nSZj7LTTby6fDk7LrQX3NlkLdI1BBtZYAusAG+O5Gau/axOs4fRYj2LmWeFMzUKjKaVllVaQ/Gix106Gpbi0AIfLZO2921lLe2b9hew4xJ4cmPn52oQE1t5liBFS2mpRqgpOkOG7rjrF2fMNOyHFBS0lTgBuq4q+GtxvLUOmMR7NIJMr+v74J4S2XErvtnkeYEIufW2JbRq5JqVIYeok1xUdmShtNN2GksyXkLQ4JAL5LYWbaBe13COSjt6XhysBkLZnZr73J+P6rkYrC7IHQtAsjqplZ6iVkRKrlmRVafNcGhaENuoD6xbW4olm7Y85cNt1OhsjV3KRGzScvzKXmalOVSLIZW07suGU4yvShtp1xSyle07ZQDjTgKQogmxOJNXc1VKsQ57kOr5dTPgSn1QYUt2TGefQ0nbtIbZU4i6pdtspOgNtlZ9DiJQAmZVJefaxS4NUoMl112Ql1obTsRz5haiLbJc3AdaVnztOFSAdsgg7q2ihmkEkV2tB5kEW6kBU0tW+GI59SonLpYFczTleqqkP06JFE6I89YobQ+2lLN3iG13aXqQCpF+V3uecS3pRAyfKoKMgZ3nwqdBdWZdMraGv6zEmIIKLOhKloNiSeD5AoEWGGOaiRWaREMJqGavVKeiNUpNYZKWJBbu0pLTwcO044W23FuEklfOkhROI9lyq1BFFm0mpOut1SkVAvxdDZ3orgJSptdyC0QD5CLhzUbni58TXwPpJnGM6XuCF6ihqWyxa9Nbq7p7+W6DDezDOlKl7TriXWY09BCHW73W2sq+VqCddxcDXYn1xnzPLMJ7NeZa3mBEMh+a+48ku+LW+04dxp5orF1jbLafKOQdYsAL2JkZyS7mEPnMsFTNZQHn3Q4N6LLOktNusafmIUQEki2ogEcqOK36v1WEM/wBbYrrLEuctFPeSlLSmNuQILN16Vp+WHEXBtbtY9gceixHEZMWwpkx0ykDT01J9fouNRUrKStc1mtwT8L7JFMyxTWcrSk0eov7kVnx0lcpp3Q+DbgKPZbfIAIs4FBYsQbz74WIl4NeqmytTkaRCpZ1LFnG3UvPNoSe7ZDukBX9twXuL2iNKrE9WTJlCbmNuRKfKQYS0RzubC3LqY5JK/qSkX4IFr9saS6UZBg9P8lxqPsMLmzf65VnUubyZEtYseTwUJRZsC1hY+98buBKN76h0wOjfzVXElRHFS9lzf+yoVnjMdLrYYptXlOxqeuOHYi3lmJVqbVADw60BdtaRbQ6m6FIeUk6wDplWWY2cpmbqjUprJpsRptb0+IqQVtOVBxGrU20FbegtrDq3bi61LBSSnDXVsp50plfTOyjGp9Pqb8x+TCmPKM6O4gagWnkOtFDCtlwuIWoHVpWzq17a8SSqzqjTptTqcqm09OXKNphyIzrRDtSZCUuuSGW1+QiOSrQkD5qN5AHCL+qpqerkqHz17bWOnSwB352XAkfFFCyOm1ve/X4KPSaPDkdRpFCrSlz4s2hGLFYXLP8AWg2pLLzaj/iyreCUBABGw75hYnDLHl5ZpNZq+Z6lAdqiqVGbYEGG0iQIiHNy7pKxodWG9u5UeEKbXdQeNiepeX60uexMi+EkU6GioQBLajl2wcabO0ZIG24XA4A0PJ8xlwnkm6TJtNqlbhoaoVMFJbarAjtIpLDceTRY0eJtueM3ASovuFhWkLu4SXSSAG8cyWdtbVOp42Nc4EkHfcarf2TooGySuIFrfVSl6DBqtNRIoxV/X6iKs8UwQmIXX2UMKQ0LoAGgBa0t6QAxuXFxrZYkikTqgqkqy5TU+DlbLb7rFm3HGnHQSw8Bra0suvOFpPCwB9XIxK2spw6bDo+VH5TyZC4EpifJizFAR0LDOoocdBWdTpABcutYN3DZAs19SKVWallxZlqiSqnSJkF1lFlxDUmEBwO7Nv7W4onbWPmNkci1xjHD75IfeHWLmDRvI2GxS0mLAPMTCbHmm+JluSxKoNVpVVlV3LATNUYrTzReVE2lJLLbzlkEMuAgtnbJCQQTZYMry9W8srzEih1WVPmtIcZcgwwVsJ+X/wDo2mQ3odkHUS4226u5aBBaJxH+m9WocCmUlmjVkQZTNSLrXhg4htyPtkOoeaXpu4DcLLgQslWsa14l66MqUipSo81Lsx5mO2Ictxx2C06htshaW1+doKF1XSdd3ybgmwrfw8aynjiphlGjrg2AJ31GunIao/xVsRd2+vLXn5prg0tunyZzeYKPHpjq5KnnZrdNDLDJulTbiZIUSLrADm5dC3Qs6rquWnNORIL7FWlQfEMfiNQpshDDDojRoUncbbEhtTSQ4Sq5J1Lv8w8LOkiUVGLW33abLp0NE5Ta/FN0ypVNuK622sEPtKLaS3JQ0gtgBQIHJus8lxflGO0wa5Ig055xlE2KtKkPsBsrKG3mniAD6g3DZBui3qezHhTJonUNSSb7E7uuLE32+iwT1xZIKiL+yjmW8oJpdTbqCavLbrj9PLUlRG+w4sOWU9ZadBLZ+gaxcElYNr4gWZpVWg1NiDV2G6hB3HZslMZlbcZgIQppTh8QU6FtBxTgW2QizoWdaAEYtppiDUW1TYceDVkrZkRDIQ62porcKRtna8hDnIKSddwPtivs75WpVZqsWj1VcmmM1kLhiTKdDim57mlLCtrshFkrI5tcG4NrHkYrgtLRsM0DAHR3ANzz3uNrkc+a34dWTTy2kNweSghq1Ti02bPpOXFLlwZFIizXDGa2mjtJO85uj5SnwhNxoBLS3F2F8SGh1mRHpcfKGbqG0tmlupg0ZxmNOdTPcQt1SXoHLUgOJWU7Z4WUMJWknSLDyy/l6FS15skuKR4yqqkKXS4zmtcKoIMd5pLiuZLZiKQ9tnbcD8VWkXdBKljNlEfpTc+c/T4MqnNteOYS04UuTS260IzTem7m4YrjqHNP73dIUsL81UYaKmmZPBLltyvb1XbNS1k5Egv9VAHpbEZ+RTRIdlLo8t8RlPFkOvhsyHUqeeQQ048pZIWLIKkcAjuUUmsPVfL0VqLVZ0aoMSAVxg26Wmw462z/AFd3UW+DYrJIQN4jn6MPGa00uHmbNdHQ9DSha0OpnIbv/UnI13HVc3a2nbXSOElIBKLG/KUuGVSsyORY9UMMGNJZZZedalx3EFttW5ctOOByykKV8u7Y4R5TjiSYfK+YtlNzfQ6n4raHjsw8bW2RuXnpmZpKIyI0ORKbgIo8GOmlK3I77CVGElw3CFui6mG3Lr0IcacWCDbCrMWUJ9XzMmk5fo8irynJLUJUdmfvvNvhO4Wjy2iM4L2sSUDzoJJ0APeW6k7QctN0+gPohFMBifJRH0OKacERuIp7acW3rDjrDjfB+WXSQAAML8qSqVmWk1OOGn6gzVWHQPGguNtxg8S0mVMskLeStSlBOu5WCQgoJWRPBTTzCAENNtze1xdB0j2R5rXUZos6JSK+w6aksPrDtMnShJkRWHgEltgNhtu129KNs7OguC4tzfkmaunZwkJTLkTZMqoR24K1aUIAulxmCbpJbKUNsWTrI0F0L27Ww31WgQo2XYUl+eJwiVGbBVAWN2RI2yp4z2NGkEBGklLbi1laXVgWtblIcIVGokKQ9McqMoBcxEQhzdcdc8NthtJWbo0KWm6rgqDgKDzwmzyU7iSTp+PULSWZt1bWTM0tM1KXluFTDNq6qYhyKJURbLTk1tp5p0MbZOysgBOkICAFJ0lWk3PyxJzH1OpNSjQatHjByINctUF+M806G3VyW3RG86UNC3LNneUkhAIGGzLjEd7MzsAs0pxvwrdPo0pyUS886gNFxhoBSkBxgs7wFyUgrGo2cOHCiTKpTq1FqOW6a9W5amX6bJzAuoNMuU1i+rbkKbGh1sOcOJs44UEEGwIHaw6lrJIRP2uUO03PevpqB0voVzawxyOy5bkJjVm1yhCmmbDkPRVDw9UadaecSqMAlCZbIIJdaDYcLjxJN1IC2/NgUmXH6gUF9eU8tMmcpswpM2ZEEeFFU1cbJW42VyHLK+WgN3aChqLX0GTilUmpsz8t1nMtQnTn3X5btNROdap6GpB3ElEYOkvFtfrJKwNSLJRxh0ptKp9EpsSkwGC3FhMiOylR1K0Dkkq/fWSSpajySST3x9UwzB6ydhp612aC1rO3uNL9PO/VeQrq6mj70As/ryTHl/J8SlsRXqk87UqgwNYW+4VR4jqx8zwrH7NoemqxcsD5gDYOZpFNVJlTFwWHXZ0diNI3EakuNtFRbQWzcGxcVzb29hhy0A6Am2pxWlI/tH0A+57Ae+OLZUhWlVwfY8f+eCD+uPWU1PT0zBTxDRq4ks9RMe0kO6TkX8yr3Pc45ovg7buMcKbY2WWS6J0Y8UqF7d8HaT/Zx4pviFQIgIUfqTjhb5wo0nHNH2wqKJ2//IxzR6YOKCMdDfHOImsk+2nHgj3weUY7t/5OIpdJtFvpuceCMH6MdCCfXEQa1Jg0oH7Y6W8H6TjhQr+1xiJkRY49pJ9MHhN8eKAPzxFERpOPbVjg7RzjoTziKItKP7Pf3wII7YMCSPpwPTyMRAokpTpJPFvXHkNhCdZ4V3Pt/wDQMHCPvOAaCoNm/Hv7nCYy4rzio8crllsAuojILp57AkeQfe6x6e+Ih4fCjGm1KO6tP+9gi1h/44PShbhshBUfYC+C/wD1i6ShKGIqf7Th33L/AGQghH8VnAVUll9ITUXX54/sPqs0D9mkWb/iD+eImQfxOCHVMMvGU8O7ERBeUPsdHCD/ALsjAVuVd1QDMSNCBHeQrxDp++22QgW+7h/LDnpQhpLKEBDSLBDaAEoH5JHGPNsq1X03Ur29PtieaibhRo7qdFSkyKj7pkr+Tf8A3lsIb/K4OHBDSUIDTaEIQOAlIAAH2A4wdtqXfQBcHHQgX73wuZEiRF6L4MDSSNRTdNiePsL/AOjBzUcuLQjbILt7EiyexPf72sPubYPliJNSiNAacK/2bzuiyGgtaQQAf2hJTYX5F/S/PGxLGI6QBjdSV1qHDH1N5JNgkK9xOi7O2+ttCig/4sqAJv8Ale2Ar8LFYclS5DMZhttbz8iQ4EoabQCpTjij9CUgEk+gGFQiaVaENlS/7KUXJ/h3xE20sdTaglqLeXlGlzA2gpUCzmGpNOfSg9nIMd1PP7j8hFuW2SVb46gRxgE96yoNMZJC8+Afv5odEiP5kmMZ4rbLkSDHbccy/TpTZQqMwpshypSWz2fdbvpSeWY5t+0cWoGuTvFxk1V2G+9HqS/CwaekhL1U4KkMJJ/Zs2Bdfc7WHfbb+Yor8xireIVUT4mgRHEpnJS0X3axNWsbUNtN7uNhYuof41zSi+2lwl0psKYZLterZbXVJKC1obUFtQo9wfDNKHCrkAuOi24sC1kNtgXR2Lbg3VMp72umX7K9SKUqHuyp8hqZVZhQZctDZSghFwhlkH9nHaBIbT3uVKVdThwc6ncbdb0335CGre4ukH+QOFoQSQq+CGhrMMXIK3Fvq572CrfzcGLg2ypc7tO8jinXr8wHc3xSmV6LLrGas6RfGNMxznOuL0N691AMgNnSbWFwog88cWxeCk/XxawvigKVNnUrOOd6mzAnqZXmfMaA8HBtJkbjgaa0g9nLhV1D6wbHkjHiuORagb/q/Jer4O/6h/p+a0R0ecjOVGTVXrrjS80VSYopaKQltFQUllISTyA200Bb2vbnAGQVDW79avOSfUnk/wB+Dem8ZFLlN07WgNRJ0plCl+wISDb7eU/ngxLVtQ7W9PbHN9nurZz/AKfwK1cb5mmHL/mQNPB084QZiavCZivSFxZMt9tERsIGp9f9ga21ItYk61WQPe9sOgQjQAU3+1u+IrnylyjQ5Gh5uLDefaYbEV1tx1xZu7pdQ5ZAQ4tsNgecgdtJPPa42xB2HYRLI0A3sNfP81xOGqX3vEGhxtbXRZ8zQtlydBqMxcSdTZDlkriOtlgXauVtJI1tklJcB4BW2sWuDh/ht5czQ2+umtRYu6060FyVEqU6gFSTtoTdvVxaxJHP1WxVDEiMWJQeYWl7eLrbiHipLCCobmpFuyrDkEaLdj3xYuVcmyJGX/xnxLspaHS1Iiwp0cqTo5bAF/ODwoKB9SEc4+LSEOOy+itb1TLSqRFkyIMmWzLTObePmUgGM8OySDqBF+Rb3Hf0xJa25TA03RUUQxXWB4hkIgtHdXcBxtRRy4sCxHrwO2IlWIbtRkLZZQ3riNpZkuMSSuO84bq3U83bBBA21AEC17nnCjLkfM9ErzEioRJ7bzIbcCnXVNqbQDwtJ9bW9Li1r8HHMc27rvV2bLonWnI/Eprwlw6U+9bmYpvUHW+3zWSRr02Sbj7jEXlt1SLTfAzKaiKneKFtSWVpZe54KVBVm3AdAIT9h64ntLqUCSZdLkVSTLjR5DqWy4LtG9zewNyOeCeSLg9hguox3o8dyZDoaKfPmvNLFQizAmMFrbIuCQoo1NIU3Y3sscc4HbNY7RAtuqyfjMJXEzAwtbqN0NyUps8hwXTp9bE6L+trfliwcs0DLtVmPmg19cyqx0gxoFPbQGWgLuJXuCyASjULAk9j5+QI3VMvVaRMnsZkorc51grQ3VaeC9HqWu5QN2OG0JshwWVtA8AHkchouYM3ZO8PGXSW32nHmHTraCXpQ20lKC5pJ7X7oPO4CODh5nlzLMNii1tlKMyIgJqlPlxYC0yWUL0IZQC4wFtkpcbBFl3W2ojnvuEDnDZmiotrEp6upkU+koTIekGDGKWN8RmmWgpy/kN1J0KP7Q6gARgvMFTmTHTQ4qnG489KHHHHyp5wNuuKKWXeE+HIW2DpBNwlNyNRRha7lisVbKkVEt5uutoljxMKUNO+CTthx24JKUKSltZuEd/fFbe7q8qO0KZMhzB41FbgsOzBAQUOIguFLriALOHV3RcAmwRc8D74kuZlVSVmSm5myXnhtFJjtoakfislLDjQIKlNhs3Q5bUeQFFJ0CxIRhuj5Ky3lOc5B0OPs1VSJFPIQtTzCEE6hZxVkECxIUsXIR5gDfDHGmKy9OUuhyaxCnayduVJadWlxBBKym2i/APHta5sDgviZIc7SpmSikZlpH9HBS0VB5l0OBuI2p0HYbEdSXCpwmxNiSsBKNQSLm4uIdInM1art02VOYixqxFa1XjNtJU2bgOG1wFqDVxf9+x7dniLTYWZaPOiz5cNEunwnVxUmQVOIkrbcLTg7aCo/ukm4JBBvhC5Sl1CgQ69kxqTDbivsOuNOPNuNy0LaSWnCkp3CoOkt6eQLggosb3QtawlDOnCSaCM+1KorrblKgLLYZfpqNTDUfS2S2lk7i7AqCbEcN3NrWu802fRKfXHHmFoTFpMpKIzsd1SFa7qUVukAIeWDdojRYhQ4NkYhaqVmRdEpdThxKhGDC3IsKTtllIkawoJDhFrnXfVc2Hv2xLobOcTObo9ckQnY0gsQqq262qZe9txxxJA2VgKBW4EKRY345tDa4ddNmRmaGYLE2S1DdpbyIgjzDPd2y2XJBIT4dpADa17Q1BLg0BAvZRIRhmzDAcy3UIc5iHNgxpsbfjlbhDhbKip1TTpVrcGtxRCjZZBHcWOC6WHK3Qpjn4q+580tS0Pwg8+XARpUVggciwDjX0hLYWBwcOVBrjKEU2Gqoz3KegPtyGAovWjWJU22La0FJA8oIvyb2xpBFt0Hus1LqG5FlVJ6sGG/NpziDZ4J1FyQu+rXrG2g2/tWAIHqRey3KTSKUlunP0SLUYU9DElqRHfAksMA20KRbzuJcsQSeSRwASMVXQa+3R6Tt0qGkMzgBtzOyQF2K2rE25CTyeOwC+DhyrWYaqpRpVWXG8StovNymIqFLkbtyq6r6wFWJPcnkWHfFkFVHABlGv4emm6yvY5+hSirx6jMqK6/UcrMJdU83BhSWo4VdFyOHedCyRbyrFrEegwly7mtK6xOGYKnMTeKhKm2bCW822QUtt3SF6yR+0sSbWva5DTT63ISy9RtuO6xPiLppZVGDtwHA42R81sIINyFEHTYG49LMyFCqcCkVHOUSeipSYwQJzrkVZLaCbrZS4QEIcAIAda3ONCOAQMenwijjxGXPmPmuVWyvpY9r9E0waRVKwY1MptMaeq8R94IStlZZeYcbU74Ww4cXoCdBPILZANibLV5ZeTRKhRqjTJcaTKfQqmCBLcDLslbgCqetrSC05yVDULcO/WOcOsuhUWmqn5qptML71BlIdbpFRfeffeYVJU0yuK6w4kqceMhUYtBG5Z1Fk7iSFp5tTr9UqdUaqT6qTQqgpSJcVTbYfhvMakNMymkFTaHjttgWcUAs2JudGPVNwunw+DOG5ibjSy5TquaokaAbN/MKJUupQ6fTalRa2txLjm24h1gBRDb7fndG3qFgW9JQSFhejjAqhXa9LeYeaZLdHmL8X4VuS86yHFiyVBOka3bXOq3buTbCOHRKYqa1OpUyouKmzVyUpXF2izHuWn2/ONt15t1XzLhLei+g3IxNK9Q49PFbqJ8JVpEBpuVGrMN6O0wo7KdBb+aFouXXGxa/LQFhex8bNDM1ro5DlZ0BuT+a7XaREh41KgtHoj0ScajVqi45Q4clpuUhqam8t9Zuy22NOvW5rJPFx5T7EL8402lUiqRczQ1+PhJcNJqUOc88p+LL0OJbZeacbbWBykahcWTe57hlzRnun1ubUswux0aJbLYDK/lpcQAGwsbajoeAbN2iiwDl0Edi8yZtWzrl+JUK5ICpe9BQ5U6mreJYaQ4G0tpQQAVFV3S5Y9kBQBCMGKel/hz6aUd/cHmkcJu3bI06KOz0yq++qhSI6GnUrL6JzWplTbgTdvzIB7o1p5P9+I1VqNmTMOZGv6W7kifUmgl2qyElTzrbA8q1G99dikWPcFHpe1rrj1JmSqnrmOusuNbK0xArdSslI1oWgAuIAukjv5h3xFK+qLplMoYkM1F94mduvbq2V283l9CfL2txxjzkFVIwGM7dF1T94pV06yWvOGfKbS5cpRpDMVdUkKhultxmM00luO0FkGxU9tAHubLPHBxqdDBZZQ3rcUANAccWVE29ye+My9CcyxstVyqMVqsU+ImS20XXluFp1wI1EttKcSUAW0lfAJ0psSRbFoUWp5pXm1FfRWV1PLlSWFR5sFxl2FMZcCVstuI5daWgOWQQEAgHWQBYfYeEqmCloGhjbkk32XiuIIZqmpNyA0DRSKsdQKLRqzDoq4st55x8NT1pACae0tB0urVfQ4Cuwsk8BRJsbIMDzK8M4VeDmiZmV13JDD+00xPiiDGamoLyHmXbfPCwWmgVui93dsBtCkEn5ypdQ6lxozkHJSXpEScNvRV3I8OpUstLUy+1vsgtqQ46slKkJb1hS7q8pD/VMsV5iBLZnVjLNEy85SSiemO5JYaiNtgqISwUqC291QcuVIWkktgkFBL1lZU17ZL5TBfR17f72OhCy00ENE9h1EltlXuWpEpNM/Hp86SuhQlsRJUaLEceZiOfOSjcbaUH0IBCXG37A+bglTejFtZWlUuuUGBU0rjuzjHQiRJht6XNwpF3LoSCN0BDhBHKyTa/OK36gIotRyRW88vZjfzBQqO7EiRXGmVJlPQn1NlMYL8l3GCttGpwFFkuagFp4mnTehxMrERKYp9bD7RZn65odUZbb7jZfLKL7DbgcZIFyPM4eNPODhimloa1xkjDLjre58lsxiRtVSDI4mx+S5JefGb1ZfmIXUqLUmi6ltBjtzYTjrJC4zjLmlcgONghtIQSSUC4LYu3v5phRDKn02XKmZYorcP8VCaY/KbLbrRBeSpaitC2xfxCRZbYDZ2+L4nlRy7lt2uxszVelwl1Omt7TEiWsDYBc1AgOHQhdzYL7i9h3xTmYcwUrLWbsyqomcGVzkfJmRWGA8zNs5zEmMamw2pAcKESWjuJH9q9h0cYHuX82dxIzEjy56n1HPRY6B7as5IxqArHpuVMstVpurR6RDqk+pQrKlMuF7ehlDerZ82hccHb+W3bSFIcQPMrADWV5bfVAqcObOUpCyh1lbZeX4fbZKVtrKbuEKZN7+e97G5syUKDPzuddal2pU+EJ0aFTITbcJgOhTbLw4S62SFOJv8wEANrta2HZqSvLGZqYzU51bCKpWXqXS2akymcpTDewnebktpLjDami42gOHWB51nvjoUlbLLTNeyMx9b20+HMG6zz0zDMRK7NYfv5KVvRGlL2X223Q05qRqF7LHAIPofy+45wSuI6zKTUIjpQ+hCmwjkJcQV6i2bGyAV2OsC4NvQkFLlio12oNVBnMuXpFJkwpW00XX25DUhopBDjTrdtab37pBAsLXBwdCzJlydmKRlOLVESaxDbW6/EYBUphCFpDhcNtItuAEC51kC1uR1H1kbYxLKMpOgvvf0/Jc1tPKHmJmrR8k11ipUemyWKw0JYfYK2zJQy+3DjoKVKcdebRbdKkNbSOFkArvpuF4gOfqhl3MFNi1aHGp0yKlUaIV1aIuSsF91tvWUhe65paQ65uJRbQ6LkWSDOHnaZWoq8t5Py/EqElVY2FqlQXW4MV50qbcmvJ2tDo3gnWpJUsbus7Y5Nc9QxLy/TqK9U49PqFScm1EVVDLC3UsxA5tAFLKde0y846Gz5C4CB5yQUfMccrzDK9wdq4a6fCw5eq91hdLZjXW0G35/voo1T69Xq3LTltyC7Dn5flIfmPs7shbSGHlb6FOIKttoBsJbACgbEXIUCHLItedyBnGRPhLkRYiBJbZjoQG33Ij7RbjAnylDSlqNwOCA2i4BOJe3UKZl6ZVGZNOalQojaE7LkopkzFuSXdyWSfJoJetZywDibFKL3wyVd6jZznKqlckvw6PHiuymg1Kjy3wsMtlUUWsgOOBlz6eE3CPMBc+ZLHuY1jX5nDlyXYAGZ2mh5qKMbTDtIbiS0Ud+PRtNTRDlBSm7q23GGibrbUWluElRtoIbA5vhpZfjLjMVaLMp0aXKbcpCqfGBbS8HG76y0saw3tutOfOJWW3AhPCUkTStT4FNzPLjQ4yG00iTJW0tKW2HIbMstytpKUFxvWA6kC6NBbSoo54xFsyRagXpFWqdRdqYqb28HHnARFLriWwt1ToTdDrjnOk3bu0scWRjQysfRwmOUWP4XUMQkfnb/ul8elVOflSpSstzao3WxPdQzHYjtgTULW22tey2fkqbWFObrZcGtsXAXou51+bnWiU2ivtVpiHSzMfkpVHbap7seuMQQ07uNtMjW8kHSHVE+cL7LbXaCxI77D82lz5kudT4bS4TMlQWs7+4kBxoHgNq2twXssnRfVYgPNQreXqXSFSWIMKXOp86Q9TXYrSIkSRGDimw48WlDXqWHbAWsblGpskjnV08NULBtiOfn1VkTC25cbp8rDz+XC1l+bmRf8AR+oLiV2PVkobCGVhTkVtWyAULXrYPmTdg8lAuVrDZSnYMXMOWVUqgxoioTjH9TDS40VxDgTtzRoJDa23CsrDdgUNoAAIKC2w4ErNFTZpdYzVT48hO1CMiZrdSAhkqQQWzwyL23EgINysc3w+TImir0GLmQ1VijmMYTL8N5MR4uhFokkRrKG00NLbdiNxDNyd2+PP1FQcgbJrodeavaM2yHUc4SY2VahklM8uUhyN4aK+HJTLTsQuuhMgMNqLi1kAltry6tQ3AQeLHy1labR8u0uj0um0xwQppfaW7NcY4AZ25u62lRbCgUnZToWSVgq5IMepvSiBVGG6BSmqwme6xGgVeFNBaQ866lxSY7TThD5DumyEhtC0rDQLiiq5cqzWavlqbTmabnGE8p5UI0ykOtyAgbpSlKG92/KXNWtOoI4bCwSRjbg+NspGumy5wBYa2sTt66qqrpTPYZrXVgNSqS7T2pqK8h9vZLhQq+82rXZTez9ZN7dhc2NxdJwpbajMhqVKeT4dxgSgUEKJbNim6QbgEe9uDcX5xF6zBq8Kp0+oxY0aBDjvx92ifhzi5Dji1qadV8hN46Ek7lz/AGjuA+Q4k2TXRWW3X6PHRR6KlsNOPLk7SHyjS0pBbFiuwCShDjaSnbI5tj3+D8eSyEQVLxfQG4sdTqbjTT01XmK3hlnjiCVwWI5jstNPyXpW8hllEiK28l9xAcJWrZIWW2wQVuHQL2A+oXTyGlPNInJgustIS03ZQLrrOs7bSXNF7EgJFrfnawsWzRprNLeltsmZMltyBFiMtb7TnKlBk7bjdhpXqCr8r5NxwDStKBG/rC0JkNttxwZJJcugG1zblR1jgC4A55GO1htbH/Enuhlz2budhc8zf4ajdZKylJpQ17La8kmKVXtwMBKOcKDZXmTpIV2I7YCUn0GPozX3be9145wa26T6PvjgTg/RjhTz9OJdRFaPvgNlW9sHace0XwVEQE8/bHdFj9XGD9ke2OFvjlOIjqidCf7WOaEj3wcG1D3GPBCicRBE6Me0/ng0oucdSgf2cRMEQUXxwo7YUqSb9scUCO5wudFJwj3x4N3VbBq9DTZdfUhpv+2tYSP4nBAkod/1IxIkjjzIb0t/85ywP6XwyAQwg2OrHdJsVAeUd1eg/M9hgtTFSdIC5EeKj2ZQX3P/AIjgAH/wz+eCplGiPQJaXkOSXFRZAC5Ky+QS0oAhJ8gt34QPzxS9/ZtL+gKuYwPdluhiVGcUpMUrlqSrSpMVG7pPsVXCB3HdY4OOKTUFJ8iY8cq7A3fc/lpR/NeDqEiU5SGPxJSy/uWWp51N1LDaRwOLW09gLDsOMLNi6A9p8p4ufT7Y52G4rHiELZG7n8ei1V2HPo5XMPJN7lOilGiRvzD2AkLum/2bADY9+2FaEbbSGVcJTyBayR6k29MGJZUVa1XSL2/LBjiCWiXGvIULDgICk2sRz9rY3vma3Y6rPFC5zsuVFaUptc31KIx0It6W49TgUNt1JkpeSha0yC2hINxo0BQJPYfXc29SMHPx3UMLkMMmQphC3Q0m9nSBcDjn3/P9MY4cREoJAW6TDzEbFAab3FrGghSFben37c/z/lgYZQlxtsO7jiGEKcUnsCs3tx/uDgUFCtmzDgdu2HNfbdK+b39CTx7jBUVUv5xIWHTJfSgSSO6CEobSRcH1P5JPGMtRU5XtudP12WiKnDmEhqWI1pfdSWbIQALkWuv2/Mf6Rgp2OvTvNhOps/STYH/z/owNppcOKzTXAdaXECU/p3A4FoO2snuNRSq/qPXugk6fKjCKlpTi0rI1tjRpUbcE2F7BJtxe/IxwKzHv4fGSQS6/+2q7EGFNq3gXsEgrUpcKGUxFOvpWQwVsNBNrg8hR4Btfk9yOBfB0WSuE6tuPdxx2S28+4FlsDgKAbSeNGjgHt73WcH1gNvxJMn9i0gn5ndKwwdzkEgrBKQD27m3IGIg5VatJYjZSyClas1TIbbzlSqCdTFNjhKkb6r3DjuvUhlns66lVyW2lEc+N0RtUVZ0IuB8dF0JQ+Npig0SjMT07N9Xn5Ky8/NgRmlWr1WbT85kbYV+GMKQpJEhxCrvOA2Ybct+1dTYdVqVNplKagQXI9PoEOKY5fabCWBGaATtt25bjNgAFSeXCNpu+okpYdJpdLpkaiIW1Dy5S2ZLc2W/Mu3LQyN6Q888rRdouKLjzwuXVlQ1WBOFlCpTuapMbOFXjuJpjamJVEhOMlgv7dtme+zxbkhTLRA0+Vwi4aCOlTySTzOYDvoSeiy1MUcMQJ5cvNKaJSXnXI1XqEZUVLDZFMp6m9KoTbibLddH/AOFOi4XbhpB2kf4wl+Cb+nGDtlRPKT+Z9cCS2B9Khx7Y9nTtjjjDIzoF42qL3PzvG6Tr8ralJ9Af7jgLDVpCE34aj9reqyB/9rg+QlIZUlX79kX/ADNv9ODGW078hzTxrQ2L/ZIv/M4vuszTou6EpGo/nig8uuzXM+ZppUt9CWZGdqu62ogXdbjOyFONq7XF9mxPrbGgVDyK1arW/hjO+SkOOdTepjdVW2mMxnGomG64RaMtctwuXUfo1AJ81+1seK44/wCib/q/Jev4O1qHN8vzWg8gvvCrKpT7LTjf4tPddeJJWChxsJaA7WUb3v2Df3NliRyR/f3w15Fe38+z6VFhrUlmqy1h4n6XXHGlBpsXus7brbhPAAIFyeMPNkNKSp5aUJWsIKiCADf19hjk8BSsiiqXvNgCD9Ct/GbXSPp2tHX8kU40t1BQ0440o/vIAJ/S4Nvztx6YieaMrQq3JkSnoDsN5QXF8Y0+86+Et8+J2edYC1NNg2C03JRckAyCXmOjsTVICC+1A3HJF4266mwGl4J1AobSs2Ltli6gLCxxCKjMrbssPqk1SRD3N6QJQQwYLDqCREYS2FF79o26EcOEgXcbQmxzcZ8RUFbQmCG0h/C3MKrhzC6qlqu1k0VJVDI8VBk/g0mZU3I6160qZS247Htp0kBIRuBRSSE2sB3wx5Py7GqObopjU+ctcV9pxpyM3qDCEWN3G1hRcb5twb2uRzxhpkV7MjzZpKJTyXWg7Gfe3gtSHNepxRtcHVfue+o47Sa1Lpr7Ux+U5DVIKwp9sFA16zyQ3bQL35SOLX49fm7mPa3s3br1hLfsqwVZtpkXMsnK9VpkWKlLa1mX4YlL5WrzBxYVfW2fKFEW9Nu4C8PNGpdGDSkqktS1rAa1vlu6m0J0tJcsACsNnT+XB57V3R0N5kjKpzsmP44vOuNvLcKQ/wBrpUtYN+xN+5HubHEpZYmmP+FT2Vpkxm3fAuNBoKU6RqAB4WdQChcG/YaTcnGCSOxsESbqU01iPBkPtOUFmI06A4yXWthxnnlDzgUpDqLpIQu/lKibkEnCRlqlT6nMqFKjIYWA43NelWbKQ2dJS4lBUFouD6K45474SM57oEaO0iNTnEoltsFxEp0NpRo+q4Rcm67pJCx6XB8wDZ+J1zNFSdprIp0HbakeGZ2GymEShPzGw4TYqtckdwe1hilzDmVrE4VvxMSnx6lTZb0GjKjIfkxow1IuhYsQmws0LqVpHAJv2xX02VUqnMp1CaecEmGUMMuK3U7resltyxPJVyASbXJFxiYZfzVTpNPQup0g1ZDCAtx0eZ6K2gABbZPa+qyFG4UEkelwOoNZerVRYmXlUVEZx1lnw6gG3H0EEtEG5QCNI4+khC7EE3sY4s5JT3tktptEqxkSMtycuRn5dHjtrD0ZtDrTBWHEqb1X/ttqKLjWQki908pYSZ9Kl/hT0uRKcjSDuNuEyGQ1rSkEt+qCvbBPZN1n0OHOXPznSZrMLLz1NdakU/WwWgvceacktJ0N3SbEr7pURY6iFdxhFVai47UG5NZqZjTGnJAkojSUOnWT5jrAHKb2cSCAQOBcjFAc55N7I5Ujqsmf+LSoc5DaUSYyw8hTW/Z1CTdsNOKsgW8pU2f3bkG/MHzd+CN1R6rUwx49PqDSGHYbeqzz7SxyU3Va4IVYkIGnj2xcLsqLPgt1PU1FjLiIeeUHVuqQtADdg2Gzwe4sTYJVfjEMz1GptRyoufSMsCVKddLm89D1IbcCSHVJNhd0gjQVAge98PG8scGJXpho9TypDprNXcrEI5jicxCG1F1rbXdlR1kNo9QXL30WskkYIjJj5dZZqsXL8uOysa4/i2myndcKnOAOw+Z7I4sOMQ9WuO0nVGBZfb3dyQ38s82I1dl+hPtc4kaqY/VaBNDdLMuBFj+GddiPSQpht1zb3Ta4W0kqaIJ87ZuOxON/Zhvi5obqSRq7Sc6UU02nPCYpuK2ttvxJSXDa6kbdrA67ptySL8G+HupVwmEtnM9UkQ4lJpoMVpxTZQ84/tltkIcSootbSbeT1t2xW0Oqv5ZrbKKNLk+PeG2zKEkOMPsk3bLQHDqCtKkm9zdJF7YldArZdhNU6ruInWaCg8lbch4hDO2kWcIC0BbiOCedIHuMZJoBHsoH5VFRNrEuTPRJkR2abIKXAtTSWA4G3CUX0AAr5INuefbDNTn6jS5LXgpj8WcvzIJB7jsfP2IBB9j64mrCq1RpcmGuvzGYjLW20rwmt6LYhWhKVhSPKFKulJIF1AW9FWVqG1mCdJp1Slrelvu77UKWw6EAAqcDi1tBS0bg8pCQtu6hybBGOjTROncI27lI9+UF5URpTcOqPtx0S47brrjbL7Mn5KlOFSiCFIBCEcFXvxwOww4NT5lIrbrs8MtmP5ER3W91JBte39vj1BNrnnth6z5GRSlopXTJZlU7akSHVFlVmmy58tDT6AkFYHGkXCb9xqSgRzLsphylvQ6tGciLbQ42lDjRLLhuClwuHtwdN+eQL8jGiaAQuMTtwkY/OwPThRZCYVciSWEMKKyQGkpBbc1gp0ELPYhQBuRwe+JhlpvMdJzXBccgt1akxkyXNEdx7ZjNr8rymybEr+SE8oIGmxtwcV2ppmI9GcddcYC2C4yt+3IC9Klc8EAiyx29++JZRc4sV96nUjptkGTNre3Ha8bsLbU467uNKcD6yFx91agkAhH0glPIOOtgTmwVGc7+X5rDiLXyw6Kbza3k/P2ZKZUk5hTCNBiyJjzy2Yz0h6oOgxksqS62lpopTuqQfmWWptV0/XidVKhdPHo5j5qnRsxqhraCmYDRcfifN2WlPrjEDaCwQdxA7efXYHFSUDJGa8tZ3pD+YoM5NYdll0KpUSFWpqFlZ/rLzIeU2UJcKW7uhGoJKNskkg+sVXLmbE+CazRXKsxIG0amp+TGZYWQBp8NT4gJAKEhAcXYCyVEAY+gsxU08d6pmp2H70XnJ8N7V7eyfopbWqHEyrUZlPbhofhqaaptQSdQj7aF6gS+NRQ4ry2aF+De1r2jU2uRo8qqNymKYppDTEmmEKZZjNIcY20tNJKtxbgLIF0nWHAsm18QZc9WX0ycrRqi2ZaAtuoriRJT67klWh2QdWgm6krbN7lJufeSQ4dNreWUoenOKRoQt1pxxxlDiyNwuHWnzlVyDqRa+lfFiD4zF5o6iXMGZL7rtUMJgjyF2ayZ6h4KrUSM7WIMjTIKNbTcVK1Jas4Q6GwNFk3UNw+nfsCFtAnxIdWpMylzpkKJGltJlU5Ubd85vq2tak7+oJ1aFW48lyLYZHKqUPzIMfLUdM5lrfZ2oqGFOrQQAAgANrBHmOlAJQLo/aXJ8xt2pMTIbtY8emIA6ExS4tl8ghSWik+chJ8vPIKU+hOOQ4mB4LHLZla/dSisTjHdkSKLTo4myQXFy1huOwW3SpISoc7RNlWCQggpueCL1bU4FSiVjfhtR3Fa1tGQlGlIcAOpse6rDg9jbEpfaTAnORoG5TkNKJQlxsR7BdlaNsGw0kC3p973xxqvUaNSjBivMSn0aG1tOi4bA9jbgDkWvcXGOfNnEpPVaWNuyyaqfWJNBZVmRykmfHaIjPreLp2tGlSgQggci1yeAdB9MaNynOyHMbfqGTsv5lp9Ucha3VLjuRiTdIU3tOFLBLZ54ShB5IvqxXnT+qMZcyvPagwkyG5EuOl4vLcWpkOJLRIRq+YgoITbuDYknjEniUuNTKA8mqVWRKE9SKQ0w2h1rabjRrpQ4si7pOnyc6NCr+qce0wXE5KCL/0ova2a4tb0K4GJUoqHXkNuisOLVX5OcHKdGfbp0GLBbekMIioLclxxzglzygFGggcr+pfHY4XZgy49XkCO7XJcOMi2hmKNOo7jSiXSSdwWbdTt20WdubrSDiP5XiVd9cVuFX3GfGRfFvvJpDbwdW3tpSyp1ag4Vixct9AQ4tHCxcz9LZKRe1/XH0zDGtq6S8tiHakcvT0uvG1hkp6jubjmqQdomYOlOZqlUWHak9l+ql9hl2FTI8g6HQp1SX1LLaIZbW46Q7dCCEgaHTwGqsZBZoEbL83Icdiu0h1lMZMykvANqXsuEDaZUNd0trVfWEGxBSF2BubMuTkZmqdDnSMw1ynsUSQ5JXEp0wx0TSdFg8RzYaPT0Uscd8IuouZ6dTqbOh1GoSKfLXBdqMKp7QUzFmMEORrWVqLpdbuhpKTcNrvYXxyK/h2lkhe15ygG4N/D1t0HkV06bFXtcwsFydD5qos/O5rhvmlPV+U84JjdQnUmfMbhTlRn3VlKltBbqHQ0sgamyo2dAKbNgYnNPrPTfrTQhSatRKc5Ijnxaaa+4EIQ7pLW6xJaFw2T5dY0uABBW2LAYQZlz1BzUp+iVyCvLuX6vFfjqnM1WOllucUtOQ5yVNgPhsrbcSHBYpHkXyFhEwyZkeflmoVCdVkRarVZjSVTqvHiNRyHVrW4+1ttgENOOAP3JPneUBZIAHPwFs1S65eDH4XBwB1F7HUnfQq/EgyBt7EP3FvPU7aaJB0+cpMzJ1Ji5WhTqa7l5rwcmj1B4CdDWblxiRfspwguoXYNrCgoAAnQth5ukxc3ycrTmIpkzC3LpMbxfhVyIhbTuMuXSUbrZbfJVuG+kWToxJ0U6nomOVFECMmY+gNOyUspDziBayVuWusCwsD2txg0tJN/IPo08i50+3/0Y9dJTOdH2TXZeWn5dPIrgipbnz2vz+P6JnpNTgV2nqnQQ43tuuxpLDv7WLIbNnGXLcXBtyDYggg2OGGnZczxTa3Wq3TMz0xt+r1luexDcjamHGGm9tlt4WTdZQrSsj0bHJuTideQAvOqKiBxqF7AD3uAPT1GCYbxZlRKmYzRjx3Qs620rU0Sgp0qHKFm5A4vyb82xzsRp2VcXu813FguDqNduX2unJaaOR1O/tY7DPy30/RVFlXLFRotJy9mH+m9VkQqoyh5yjNNpJlS1tuF8OOgkbbbaXg58u6VpOg3VbDVmCtUee3BotCeqlWqVZlyIspNMkriyZUUXcYdbirSA3bQXgu4DhZUeUJIxb6J1MiR3FsUGI7WI82Q/VEQdx8Iff0l0Nk6kNDWEFbJCCsqWb+hrPMIo9ezFl/LuU3KhLXS36lDMWA6Ibwo60x3y0UupIADinIyA4NBCiO5tjy9fg1NLTufE0tI0ync6i9r338l6SlxGZsvZyEEHmPp8AmvOVRkQ6dKoETYksx5zbsGUuABMf8AEO7TaXFNC8omO2nWdomySspNgsNTBYk5fp2WMsy1yqtHjx2mlRWiXXDqKiWdbgQtooLdlpWEEBwrAxKq/Dz9SsmqmU2qvR67Gp0iqtpiuhyS6h0htTzVtliEVIDjCGW9wqccKEEFCjiL5jpVIZVGgs1inu15SW4stDLb0xMiSPCNbYJWC0Wh4pG659Jb9CRbzVVhLGNLoRpvqQPS2u/ULrx1ezJD8kty7HOR5ztDzY0+JiKnTqsp5h5YjPfIS26g2b33G2kKZ0PpSRZWspAGvECzPMcqUljLTVWplOZQ7IlOutwJLrilvupCW32lmxXdYuqwuQkrJNyLViU/MFHy8aw9PoD2/NQuQtiql1+KtBAF5UkOk2bU4Atnb/aKs6bkGOhmBTabEVBzRVERpe/PUzJjDxjcYuhTS0hboW428hKmCloOEhp08kG/nMVc2KNkOexbr11PpyXQpYmkmQc1Vv8ARKoNxJklXj0t7MdM6NEjKbZUV6nW2idJWgAjg8d7m3Iw+ZdpFbznlarT58xbzcBySiCpSVtmWtEXdDLrjQDZO5wLi6Sp0XAU2jDvX6+lpFUay/UW2KOKm22zLhRdyLILoS+UF9xRcW4o3LoJvdNyGwUASGLV8wUzKtUiy6Sh6JVXflpeiuxvHxzpU48UoKUOBzausgEWI3NwKK8cFkjyxbGsF7lIIeXcoZky9Hey/SnfGw3mJTJguthx4yCGnA4XQUIMdbags30EecdlIwfRlZaqtHOWYy6dTI1UAaW4ZrPgi+2FBtJcbIs6pxPnLbY4JOkOKIIaZNdoM+SsomeMlU5EB75jrrk4lwJiymlbW06f8Xq7FsWBsPM6VOCvPGVAupVJqTVd1t1cOEh1qQ7JkJV4tIaRtoZUopHLhuPPysqJxrkcyWK1v2OSTJZ9wuSp0fPeipV3NrWcJtbIhswK/dqdIkRFJbG2WlqHhytqwSpCHFEC1iRd8hZrpNKr0FT8lEaNl9Tkpc1phua84063ssGUXHdDjItZBsF3VYkkcQoZLrlforuYp+YDS4lUjuNldTivM7pBaDK3GY4IWAGkujzkt2DlrkHEspWXsyofo0qbPhoi0lFOYmxaIDNdkOvulKYzoJbRZQDhWom6TyseY25csUcL8x09CLJ2ku5KYsCo1OCqi02L4GQiK6JDUaStT0diPNcbDxDR5b2gFLDiNZLlvuV9NkaZ4y+zB2qm4yiRESdtQYDupLk4nWRoJVoWTYhYIN0Yg87MYdPgs1tyMuUaO1+KWPMyVHASGmvEupbO9oSOSoLte4Wrgr5FUzFUa0tqlyd9EhptFTjy5KIktkN6lMoS+4yASNQIIJF21nlYKMVPrXulbNG3KRsrgzIyxUzqjlAbpq51aeYchUOY2JCJZQ0zDWQQpRU5whFl8FI0AOBFhwQrfnqlwZskVOS2W2S6yosl13WwrU6tLSAF63GvMCSeALEG9mVyq1GlVODOpG+ky7MimtMoWgofaAaaDqwouNJvfU2skeSw13ODqLV/FxqkFGK+uJGdlKAAUJiLuK21JsS2BpBKLLBNgAolZx0qLFqincIi3nfz/uqKijilZm5p+n0Sa2pbjcCWULLZSraNvmctg/v3IItxzx68YbzGDSih4lCyNSALKB/UcYSwlmNUpsGpsTIzc7RUNbkp+RH3ULUlxIcuVm1g4LeROqwGs4XtxoLCpzEhMineDS2EfLDTCQGhptawuorNk8X1JJ57/YsI4vlfAz30DvGwte4A6jkvD1uAtkkcYeSSqaIPmTbHAi6beuHCXFWwopX9QNiFeU359PXt6YS6bm3bHv4JmzxiQc15OaF0T8hSZSTby++PFNvq74U6Ej0vjhTc+UYvzKmyK0jHjb2wbo9xj21qJ0puB3+2AmzojTjxT/aGOGTE1lpDinlp7pZQXSP4cD9SMeHjF2LcVtj7yHNSv0S3/wDl4moQQttN7euC3nY8YpTIebaUvslZ85/zR5z+gx1UMuWS/LfWLcpQdlJ/RHJ/VZwNmKzGSURY7TIPfQgJv+vc/rg3QRG68ojZhuH/ACnrND+Buv8AljgalK/azAhP9lhvTb/PXc/wAwrKMe0HEzKJI3DitLDqI6NwdnVkuOf888/3YOVc8nk+5PODAnHttX5YF7psyC0yVc2uL2ue1/T+J4HueMceTIaD60SFtbTDo20tDUXe6CFdwQtIAHbm+BriSniwqOh9xMd5DjjbKyHFNg3OkDla+LgC17ccjBs2LMaQh1lq62Vbi03WNJQ2Sm5IugJd2ybjsDf2x52srG5pY5Da1iOXX92XbpKbMxj2D1TVRkSGaPDfkLcYcabS8XJJCg6tenzELA5uvk3ve9+9sODTetCUsMWjIFnC6FFRAIGrkjz+oAsCTze2EGVlRk0OO04twuIc8S7duxUhtoEE9tGomwHqLk34w5vQZdOgmmx5ADbEZDUZS9J0oLZAvze6bBNvrJv3BvjyeFVgjgjy8wdOV77r01bB2sr+02uF1xh3TuOpstZIHH7l+B74J0KZU85JUgsbawRfzaCLBFvUqN0i3r+eFKA19MaM4S4UNLdUOVgItcrPYX7d/wCKr4DJVOCpakvNgxXQt0Jd1FoN2UiySLbgKlHWeL6eDjrnGw6NsYb3r6+i5/8ADCyQvJ0SSIl+NT4y5nzZryHXbpJcQ2CvSQkiwCAG0J/NJw4RmXCgLbkOr5uAAWmXbcova3a5AufU8cHAYURti4Uy0062tbRVquF6L83sOAPTte/rhXDS6hch1SVhuxU2FJI4CLHVc/szcnsO1/bFt2QU3aF1+dkCHSTZAEKHMiBgPREaU6wrWEELHlBI5He9+ee/HOCkuQ0JTI8STs7iWW1tL06gVWBOk2JLqr8Afrrw3y3hIiTVpecU8qM62EN222y+lNiSL3KUKuPfgGxwT+KNba4LDNVW9L2PEIZY1lptCCkLsggk2QLAcKIva17+NfM6Wp7PtNDc6r0jWMZFmybJzW2zOqLr1QYjmWVB4SF3e5sR8xxAJtxxcd0kAc46Y0YzGJUSIrYjr1IIb+krSBxwDyhN7ntx6nBW63HKUPRS2pYLhCHVOl+zYHywD9fbzAAXPFhiJdSq3Cy5ld5Bkn8QqjqKZFpcWSdTkhwKDjYURoLm2ByAAkgL5CVX6WIdo2AOdfLposlIWOlsG2Ouq9XpiRCiZPhxqfVapPQ5EZhiWENy5Lbqg644WyCiE0QlyQ7wT+xbsty+F+Xstz001xqnJqE1mdIbk1Sc/ITGeq8kfLLgS3cttEgMoDZIS2gNNeQKWpLkWmMMxZFdnV92fImMtxpi4AZ2DGaAS1AilFiYSSXPMV3lOFSwmyjhTXZxzhIeyhRJMmHRWmzArsmFojOPWskUuLbltP7r7ieG0XaQdZWUct8r3wkE6k21GwHTotwY3NYbJGzDRnGoyapUZceXldqQGW1ocU7GrTzDxIbbSux/Do7zY4AIfdZHJaZsuapDbiHVT23JMkrWZDq1kHc9SU6iAeCe5sO/OEEGMy7S3KaIJy+22G4MWPFDKhsNIG02yWyQACFaGzt2A49Tg6VsxmXlFYXGitob0pSpzxJcWBwjtcXuOSjWVk8Xx3op4ezGQ+g8+ZXMmZJnsRolzbkh9DbLTpekEkDWLqK9ABWLW8hueLDvY4UtIKkKd1BQ+sqQbj2PPrz/AH4LnIUhX4UhMZ53gSkbIeKgDqS4pSz5x6Djld/RIwrabUhlCFpQCOdSbgqN+L+gFrcDg47eGmQvs0a/RcevEIb3ykywoqZQQfO6B+Xc/wCjHIadTZdPdx1xX6aiB/IYNcGh1ClcJbbccP6DBsZlaYzIWfNtIJ/Mi/8Apx69p6rybt+6glPy3B28pxnmk/hk7OPU2iTKi2hMzNVZb2PCqfu4FOvp1AW0AIYcIJ72IHONGKb1Nqsm/Bxnejy00/qjn6nnwbT1QzlLeZKRaSSHmmw5aw8g83N/XR6m/iuOf+hb/q/Jeu4O/wCqd6fmri6XyVvz2K6iMXXKo4uruKAKdwyFMOj7gloNd72tbi2JLVacqa09EYLTJU4St1Rc1NkLBC0hBAuLXNwewtYi+I/kZ9TNYjnQiyoDclYuBpO0n+8gnnnEwfR8xxJTdN1gpIvxzcWx57gyljroamnl2OX811OLap9JJBPHuLqqepFMVSJwRR3afTlrYXEiSVMtqUX33GwXNxdrPBwAFolY0KJsQo2rtJlORo+Umqi5LUxBkvz3V09QMxiFHDVySoIZKW3Gkhx261aQCQAnFq9SIL8OA54OEYtMnJbaqL8WHvlhlAA1OtkgL22wrQ2Lk2JJte1P1LLXUFOZIzFKNZqFJpyYc9DrMUMFldnQAxGKjoW2Vgk3Om24T+zGPGcUYQ/CZ3REjIdidLfL6rr4TXiuiEnPmorMrmXI8Ax15biRq1HacW4tCCGZrV9W0Q4SWnm+7LgXa521jQrhieqypD78mVEKHNluW1KhNIaUm6SCUt/clKlpP0DWDcjE/fpVKjToVVXQYi0llhtBDe6QRq3kOM2FwQvgjtb8sQvO1GjwqmaZlZhbbtMkh9uLuF9TMls/tGXFqC0KKNaFpIW2RYHnk8t9X27gOY0W/Ixje6m2BLqMUQqkzMQmphWg7aClK+3BATYXFxxwDb0wtzezVGKYFUtlDsVxfi0vSHGy+H20myd3VYC6uO1jYduzE1MmS6oqPCgRmUOxi9GZbbc2td9NgQDthBIB1CwsASLg4ldMg1VdDqMWS2lEuEyvxUR0pKHiCFH1toI9iDe1sVSuDTdJlumPJyqoxKjycwwnpEFuK6XI64Dzm64CTs6kfQeeDcglXNiDhWEz2qrTnMruyY70qTLjuJjAtrbiOtkWcJuVrue1gQEjnuMOFAl0yfVIQqcxyIVKC22mQuO64Akm7ZRcLN08X5/3XbAcw1hmsV5l0PRGIzjQhxZbSDHJbv5tRXqK3SDcqPJJueDbCZwXbI6hLqNm2i5fdcgQaRKD6EoQUpQUhxaE+Qgk2abseAAbc82OOTaqtzMbLM6C5HpkZv8AEIig0tW0ty9w6L612WUhZt5b302PLZQarljJ+bI8jxMl1Edt1tmZ4QO7TiLXIANnLAkLTcWCu5wfnTO+T6uWvw3Lzjs+SXDMXuFoofX3O3tq85B5KV9gBYEXwpH8zQKBSOh1BM+dW347yEtvQYaQ06pYUuoF110qSmwWCV/M3SsWXz2UcLKzVVZRolNvEdnvSXXBJfaUAlwIaSLKWE3IG4Q1x6KtbjFPpdMl9tTLC1tSJAOuNMe3Y+hwbYSD8xDiTqBJJuB72u/VHONXq1CmrqU+HT325aILbS57emTHdUbKTsuHadKm2wNTa0eUk6VDBbTFxuFYl+VahTokOoPyWTPU4VhtpR4bI0nlJuFukqSpCu1ja3ribUdtVHoFXh1aVAfeLQlvCXOdKUv6Cn5jaACEFsHcUk83vYixFVMVmbGIbMzbguOr3FRl7boNwQQ0tCVhYBICrdiQMSCn0ehQ47UlWZ4AIiF0wH21JQ6HA5rac128gBSDbhKxfsbYWePW11Xbqotnajw6XXW5mWHiKS+22UJjMvtNMPlrS+GQ6BdkrujkHgnvfE8y5LhUvK6qqlp12Q3RZBRuOrSEwy5ttEuIuCt5xx4aklF22+SLg4Z4TLNXrLFThxq/UaquE683CWhLDLzbYU47ttXDjhSLDaadCyE6+exmVDqsibS4U5OYVUV51oocYixmg68RYFlTDp8jamgkFqyQO9ycPJICAHG9k4FlBmqbHqZV+GwoFLFOab2UcMIbQgqKS3o5PNzpH03Nu2JPUIDsBIqUd4y6qIzUlUJ5tpaHAvUA3tgedoXNiLAH93zctVZrbP4jHflyX6hIabEZ8FoBEVAIDm2EHly/JHAOn0xHms0iqwFLmqMN2mLbdYbLehtR1kIWfMbnTY8HkD1sMT/F1VZT5NzDBrcxMNmhzJo8KQWo6pLktjRpKkMnUGwEgAAWF0Jsb3wHLkWXTUxoan4a3Q0uK4xJDiWW/wBoVN2ukharqC7+cXI++G+jZ+pywuAlQjRajNQ83JpURJckm9wh0PEg3JI0pAWBb8ii/FmHJ0mpVByXuhl1cq/ymyixCZMlo6W1uJKgArg24JI4xrbdvd2AQy806yc31KDSRlaW5EdpC1IlJp9NOhiJdfCACdcU6LXS0NBskrBB4Q5opzciNLkU6j7b8S7MqMsrQ8hhw3DqmgSF2USTpt9XIwBuoUWnvsf0gQt+OIu4Zb8TZUG9lxKuW0h8gF9d061IJCQb2RaNv1Ko1qcqutxvAxUhcSOAotuqbWFfUmw1nQm97eU2F+DjQ9g0cDdKSnw0d8UJEZLiM2S4ZcZiR5MlAi0oXADzoCkgLUbaEElaj/DC6nZJzCGP6J1KOyJ7U9t1l1pai+8XU6lSLi5dJ1p59BYW4Fnbp/KyZUZUf8ZbkwKUjceSmFHaLWvTdTzhXxyQnkj8yBhfnGppoNVpyUVNyXRkQEywxIcYedYXYEuJcaAW4k7nClkr+oahYIHaipwaM1AdqFzJ57TCG2hXIWZs35ZpUSFStil0mm/s0whoU4/ceVLhuhaxawUCbLtwVqw7VWuJzCwcxSodHhTK01IkzIrbUgJmTw4ppwulxVt1QbQSAEoCwvXrvxEqfMi5hTMem5fXPLBbkRYS5hb+t9rzgOEB0k2BbKxdCgb6xfFgxqVBoiqauZRH5tGqZcqLoVU3oz1SYdQG1IU06SULaWlFws3K2z5nNR0dmmIfD/NfZpte/wC+a507uzfdg1SCpJzVRJBakyJkp6I6XILjUgSUhrQlSmyhFgtACRY7ejk2NwMFZnoLlZiwZOVfxBqFWZCGau9KWSxHfNkiQ63YLQzc6VjyLvdYAuRhzqfV6HRnZEKk5ciM+DCI1JU0y06lDXFnnHLkh4ABsoBFykG/cmU5tq9dq/SuFmyiVRCKg8w9DtNitwvGIW62422022VNl1JaARezm2ojz2OvoGhoapjuzeCWjb+5WRlVURWztsCVVc+O9SHlZJrdcEqpLaMLxTZ4aWPM0kEk3WCdB9LWtxxiE1SpMUWYuZEVMjSJrLbzaQ6FAyNWl1w3Nzc3XzwQPe4wVVY64Ffdh1GluOOMubrhDmtp5sjslwEoX3IHPJ7A3w9RqfSKrARV2Y0hxxp1xaI7zazJcQslsFpKLrUu6HLL0AEpNgSbY8maWapfo1d1sjYxclIDJXUY168lgT4ctbsl6O2W3NbgbUlspWde2PMu3I8wsbWwhqtIhQ1xJzs2Q8mY5tbTjrdxZsHgAC/PP24v9nTNlOl0qBAcXMjPKbiMN7+hR347gLsVd1gW0tqLRI8ii3xfk4ZKG+p6bEaltIbjOOgOhbob1juDqPpfvbmxOMVUx0MpD+S2ROD2DKpxQaZKRSGHlXU3IUja+Q2Gw/ft51W0WCDpJGspV3xbsSZGzblxUrLr1OaiR5IcaQlrc2ZbpIaabZCrtIcJI2zwkkaDc4hOT3HVpfotMVUV0hLLbbsmS62lhlttawVuJIs4y24oldgFo3NZGgheBZ9rcHPVelvUnLuZVViHTw1FulCpk2PuN69LjGorcSQBtq1IAKXL2IGPSYTABSmUC4P2eq41bJeYRONj1V35Ej5QQ8XaJEhPT/DBl2rMg2meclbaVLsV6CU3QEWRrQLnEw0m3a+Ml5SzZWIkmJWqbWJjz2VXZi5kWeyqSIbCLtlxsBQbdGi4Le4g/KcQjm19OZNzC9mmit1VyDspKWwHAsaZC9PzHEpBOhF7aASSQQfXn6RgtayogDQ3LbT5Lx+K0r4ZSXOungt3wRJptPmNLYmQo8hpxe4pDrQUCsJKQrkdwCQD6Xwv204CW7398deRrZAWPFwVy2l0ZzNWZetfT+jZTr9Pfy5lCryna7DmESPGPyENVDUSXObnWG/mFNwFoHZdiMWB0cp9RzBl+k13NFJl0yoZddMWC27BWwXomyA2TuElxqyiQRYhxsjgDRi2VIuUq1LG2sOI0rIsbEeh54Ue9x9r4iecatOy4/CTBbnqVmSc3TY7jBjluLPcSdDyxIUkBtenzEG5X2BWoX4EGBU9HVuq2HQ/ZtYDb4rsPxKSophTAXd1UlbiOOWDLTi78AJFyT6dscLZSSlzgjuCORiM5wyJKzQuQ4hymxVTqe7S5ampU9hTjBIdaAU04D5Xh9RAXoVbtdsuGRaBVcu5ah5crFRj1RVKaEdmYkupcktAk3eDhVoWO10nRYAgC1h0ffHtkLZIzkAvmvp6WWD3dlgGuGc8k7OPTm4j7dO8O3JLa9l5xrVpWbEHuLcp9ORfg4KcpjNZSuLUGwETCN5sHbFrg21Dz+gP6d/TC9uChmOt2RVIchuG8WJEqM82tq97hZsbAWU2DY3JULA3w2xMytPVJEbJsAZlmtvrZGzJjsQ2fqSHHJMg7RKHByloOkLSUWJuMcKbGsIfnkbIC4i9r2ueS6sNBXR2aYyAD5JcqO81DQw805IkCStptMeOvbaGslIQSdCPln1OtzQQgE8YrnPkuFBzLFrVMdE2t0JRTV6ahAdS5AW2dSpSgoIYDduS8435PawOJaDWczUuGnN+dZFWec3CKZQnXWKe/IaGl3xLhs4QLOEtOBtsX1lBBRcEL8KpmWYlBnz4TbDyJEZlMVltqO44+XA3HbKAGi581LYsLuH0F8cqDEn1lE5l7Aa6a2ttY+vP4LomjEFSHjW+nT1uoXW8nVjNToNQrdEp0RhgJcqMCW27GTCb8z6nHW3Gw7HSFkrUNMdLhR5nrEiOUrptBTP/ABSlCTFo7haNIlpBjqf1rIAQtpQW4QhIUh9tCCncSSb9pEMuIh5Xdy3Vau6aO3NnvbtLZaMdbQLynHdtDSkOEBxQDe2EJEUnTzbEe6aUfKNKUxPp0j8JXUYCEJrlSnBpT7e8fCSHW0WbQy5ZtttrQkGx13Itj5bW4+2wbK3OQbjlfnt0XsWUeQ/yzZNeZcqoy7LzFFyy06mNHpjjsqqNSno4mOhwpcZ8SUuEobbTxeQhdwdXCteGBiuPQItOos+BT5khiQ+5qk7b7ceO7qVtOMupJK3HFglCTfbKyFDbFrMq9fo02kus1p2Rl6LIZkstqah7zTkdholbjQQkL2VbiG2i4i3maFvMCaVk0mfmCNRPw1uoSodUT+HRUOtuJ25KEkvNfMJJs2WHC42go78ayBjluqhUT9oBlvrb1WljBEAGp3jRY3g5MTxAiM1Bt/ap0uptkpfbaSHVlkt3Q2o7gBBW5dsLPZF47RokWVVKfElrrHjXZMdmA2+860HCuKUp1OOK1tFK0si2hQ0JCCALa5BOmTGKpCpsB5KG4DbXgFuNJvUS+4praKnLEEXLayyqyA64TYpFu5dlSKNXJLWa0R3qlpalQJL9MaeCHER3khsEOOHdAkW0JH7JoE2WAjAbI1l2uVviQenTlTkRI9HrNBaaZo8uMTGfecZYdjOB8JSbpVuMpeuG7A2O6QTcDD5Q8p5ejyXW+ocCTSqdHqM2ntazuxkNsXe2Q/JSS8463bQSC4QBxySh0RmVdZoEasP02VMjR40mnyoTklxEn8QDYdYZSlDSTHG6FHXq3C48CsBIIwU7miNMiJRRS468ZcaoQG5DBaV4htfh3G9xpotocS2spWE3CnFNr4CjrL3ieLsr5Rulc0MKkr9DnTK8miy6DNiu1CMVR4Uia3IpLMOOoM/Leda8jyl7R0ocBKzo0+UnBZkrpEiFJl1F2lKdccdFPairLpcbeKS3q4K+UlxDu22Bdu5s4RhhoIZeo8KRWA3OK5T9ZfgSJSo5bcbS8C4rYIcLjd1tr0htuyljcNzhuqtOyRMoG5RYlThqqDUIR6jFlSkwwhwqS23GYWpzbAWpRQDyV2NlWOEhghbD2drlHNlNyplSnm81Q36lAjzIaYb748a4ylxmpR0BxSnGk6v2aSSnebBPyrocsScK0RamtH4NVWYUJmkukwpk1pCg8g/N3G27uLJG4UeUfMc0j6CbEwkM0kQ4VEgwGGTJXKiGYtKdLAjFjbdOkr8jiSz8kHlIIJWSjDZlvMOW/AVKRXKvTHW4cnxTct6rBsvvtKCXSbAgrceLhbUlvWoN21BCVjGSSF0JJLVax4IUycZpzdVQqnVqzkd0tPORGmVORFnkh+7gbD3ywTyFual2ACVjDA69TJkppKIT8mVLcR49plYIdbt5YzalpJKNA4cAJF1BBJUbcms0KnzTUvBzoymnIi4MNTsunMlBSW0upDaSj9mHBydzkFBAcJQAyn4dQnQ5jdEjxXIm84EvrQZDQLaQ0X7qAdbdCXEd7Nvc/UbGghMkjmylF/kpHREyYsWpuTqiusSXm2rIdksmMiPuXUE2S4Fo8xLepZBWEWsColfPp8ik0Uxy42Hvlw3GmWkMNhwvJ2BcCzZKk3c8nlPbkJw2wZkdiO0mdTfAVOfJbkeKSohEhG6C0CqMR8tkEkatdyygXAFy4LbkInOOzGWGFxX3X4LqmUpltRiA3qcdJICyd1RcF0N3XyokDHp8NrqfCgHTtc7kdfqFzaulkqP8I2SyOh1xCEtx1ttpc+pTli435iLAp12BPkCrEBR9uVPhSI4eDjSkrK0eVfIKFlKgR3BBHY9wQcHMTXpkhMigBp0/N0OAp2W1jkO8C9gUqH1+hPkFjhBChSmnVOyqh4gPNAOPtANrWbDbCXACvaBCjoJ7ng2Jx9dwfEnTxRtoWOLOd7X8r+XReAxGjbE9zqggP8tl10tR0a33ENA+Ybiwm/8AHBYc122Iz7o/tadtP8V2/kDhW1EYYOtllKFd7gXJ/Mnn+eBaSVeYY9jdedSItS1d3Gmh/kDcP8TYf/JjioLK7byC/bturLgH6fQP0GFxRfAQjnthULIkJskJ7JHZI4A/TAdu/ZPGFCm/tgJRz7Ya6KJKLADVj2i545woLV8dS2m3fC3COXvJOG/TgYF4dZTf0J44wchnUpKdXfBLTinwPKtCFtIcJuUuckjR9vv/AC9cZZZnh9mrZDCxzO8vBhY+lKj9wMcCLJUVC+kX0+p9gMGNoMp5LhS24WibFSRZJPsLC1haw9uDgifIjR0JiKQ6EutEtttu/NfsbkAHvbgknj6b8YyVFXNHGS0arVHTRucOi7IaG+0gNOOx1tBwu7ZUyeSUoPqP3zcjjt3wjqEuQy8lmOuQy4G1i690tAFtViV2Nl69JDgPGlYPthvk1aqR3ZjKJTallouE607qfY3csA2DqOoXWSPQcYTV6bI1xGnKXDL1ZZEZe+vZiSnVhsWSrdJXZCNwpsAA2Ra9wv59ieLukgeHi7jzAXsaLDeykYWEZUshVGVShKYmyS27GUgtmeQ/d3aBUSColY2Us3PpqB4BF36oQFTEOydbQQ4y3eKmECVLNtBc0XK3NA08cehBFziFZYYny0N0qlSI0pp1Tj0hwAOvOQ2w2llLitJQjdPCwQDZpsEXFsSoN1qBGaisMU55zcbCFqK1yLkeazixYcC11XPIFz3xzcLjqZqYPa0kNG/6LdVvhikyvcAT6/ojRKh012LDfltJ8SCy0l535twApxCvUHsALflwBguchUFKp7GsIhtl5llpzSq922xpJPcrULc8ckg4FTm5T7UfNcmpTlLSl2M42XtxJbCyFOuLsHFuAtpHbQQngAg3Knva46PAvNPJ8WAy8ZQDUuT5rsgkXQhLhKeQbrbKLCwx2Y6wspbyDvOOnJYH0vaT9w3A3T5BaVCaZkVKQ2XZcZx8Npf1qCCfmFvWQQ325NhwOBe+Gdcx6uqjvLkyG4ja3S5JeDbaUyAnjbcFxZsHnnsri4GsoTSpNS0IKGVRnHnIzktgNhxy6QHdIdVbaDgKRdDhOkrKeCCniNUJC0VMwozrkxsRgmO02SSEOEIa1qUty5bUFqcWEIsogAi44Ez3QntCbnpyXWY1smlk3tPylTIdFYLSRUIy6jKLVwzHYb8t3XgTubjukAOL8yEmwPpJYDpiJVB/B9xOythQWEMlIXwoW1Bw6UeXcsBc27chDEalNVGSmUinPrkOhwx4s0IZjrKAGmb6Q22DZxRS426tyxNjwRI5q4USG7UpC2YsAFx2S4txCGm0cqKQ6Tc2AXcjsB2AF8X0sAkcTI02PMfgq5pbdxhHxSSpVmh06kzMwV0KhU+EpZdcfs0lA3NLgsgK13XoShtI5ccAAxXdIpEyr51aredkGFToeXDLhU2TKbEOC1UJWizpPJcKGnd139+7baPlJupW7KTMjQ+pefqS/SssRJjk2i0l5hSnHljUpyqTGrhZKU3MaMbWuFkbjtw3Q8vZ2Q5MruYZNKazFWowmSo4iMrTSqY245oISi7et1bmnbN0OH5d+HCel3ImjsQXEG5/ueXwWVsby49oQ2+g/t1UqnZrkOOu5fy/Mj0pDbTaZ06JKaadpzRA22WwLoQ8UaShPBjN3cOpZaQfZZqmXaVSY9MyVTm6mw0BEIpbTj4a4CdDjxAaGoG2i5JuguG9zhgp+UZ8h2OzIoNSagRg6HW3XWo/jitZeG++dQaWpBbU4pLa3HSdoFFgDP6PS1BbcwvtOuRv6i9EiwkM7DFyQ1pLQWUcjzHjvotzhIopq0PqZTa3K23mmlkipC2Bmt0TEi1GXKakznmZa42htbUSKpDQ12ULp3rvFQVa5559BxhXT49UZDDzcyIpAWWygU1xpJXzuEgLG3q5F+L2SCMLURXFQWH5jSrFtDTyfFrYUkoRbQsfszZASL+U/ng8iG+6pkNAqacabJSsXbIAVytBsLeXg979sdyjoqGeNpD/AOYR15+n4rj1VbV07yCzuX6IqM242nU7ttsiStxxoSCrUV/SOxNkgJRZV7AcemFjS0P9lMjlY8joVyCfa9u1+ecejMp1iSqG0y9dYSUtgKCL/wAQPXn3wcptpZ3L3Uqw1JPPt3x3qCikp29x3wXFrquKo8YSaUyfmp0mxbQ0kH1K12wsU2nUUpVexwSWyXLla1fPQPO4Vfs0el/ucKUpJBN+2O2x7yO+uLI1rXWavIaDikN/2ylP8TbGaKUW6pnXNWZEobcks5lnkPFHm2xUohSgKBIQBtk88+vGNRU1oLnw06b6pDQt7/MGMkdLlVeXFrbzSUOw6nKkuvKLl1NuLmtPNEc8lQJFtHZNieRfxfHEn/omM8/yXrOD25p3u8grzoEFpc5prbb3JtPZjPFLfmUNkhIPrYeax+67YsR5tRfc/d8yuP1xA8p7rmbaMyptISuntOK4sBoZcBA+3PbFhvNq8Q5pTxrV/fjmez1+s49PzXQ43b3YT6/koLnmktpy3W3qlnCXR4zsRxrx6WA69D1kBZbIFwjaBb0pAtdSySSTjN1Zy/nl9um5hh5gk0eJS1yKhA2FKYleMjpcS80WnVhZWWtrXui9lNjSsXvqvOtMrtVy5Kp+Vswx6DVpSkMxKjIgia1HdJ4KmSCHOxtcWBscZj69U3N9Lr06Nmyuv1+K2w14v8OgGHtoQ26tTsl2MAvxIslZAbDbYUhdzqvjVxxh0dXAJrd5trLn8LVRid2bjob+v79VGJGcFVvLTb7EJrehSB4lNyARZPDY9ODYkd7Gx9mrMFcpkimOPSHoan23kSESWEbbwcRY6ydQ0EaQb6OSObkghio9VdjPIhzUCY0hwtrUy5fbBufQXKL839OcFvyiZ0ZlmBJWu2kIJCvm97KFr2sDY+thj5I2HvZl7W5TNLqwh1H8Yy3UwXSVtul7SdSHbH5iTxz5r9+4+2J/QsyTIsJcVdYlyZsiK2yuUJSGLthIDaHkrSrhu481zwLgYilZYmStl6RIWt8jcSl5fzXmxq+YDYDggAg8i4wDLKfxWZUqdUS7FI+UUNSksqD4F2VG4Nhtl4DiwPrix7A4XKgUwRGmrzI3GrbU2RGcCFRvw9aENXBSR5TZBAHBHl4Ve9wcSCoPZcdYqLz8Nh16oVN0x4r0RHyTbSpfim1C5s2bagAV6/XulhZjp2WstuZHznQ36lDZBrMtbMdTDrTm3ZJVZKhoNynUNGkEBQHCxCtQh7sObNnIiVeAXITjtiXTqLjYdeRqAKRbWDwb3ueDjmmLtX3JsAnzKQR1xqHNXMhBEhx+K600HtEhhsud9LdtaCCgfMJ5uUG4NsDjUynV2imt0yjs02TUEuOxYcOWt5x91g/MbII+Q45rHyyeCUlGIhmnL8yO2ZMJlhDMoyHW4sg6CS2i7jRKwEa3GwlQN78X9Dj2S42YJOVWH5GW5D8GUyqWzOU7YPsFYbsFA+Q39DZZFyi+NXYhveBUzdFPIlBh02mQlV+jVGPAgBbgleIbZS0SSVCSnUl3yrLfHPn9SDiI9SIdObaZy2rMRqk6ly2Jcqa+stuPLLLbYLaVkkANkXWq/Zd7lWJHQswZOzC3BoFbqcim1mLKL6HZESNKQ66AVFtp8gFwK0jyqB4Nu9jgecoDGYoa8yT6hqqNMkmNsPgyI0th0FRv221pWCdxXCgqxBuSKopSJO+o5yJrNKoeYKPCk5bqq5dShxQxU0vIW006sXV2cusrbR5CoDRYAm1uYrS1OSYzPzHGlk6WkF0lwC3cIsRoBsD73vY4fnPw52nOJQ2HJL6GxMc3m0ERgBtltJBs6kXGq5CgQF3snEkyPNorVOriKTNpkyPILlMAqGyl969ttYDX1o1D9o2RwTYdsO+8Y6pfRQp3McqiRYkOpQ3FPUrvvyCwkObunbbFvILguCwuFk88A4MOan6rprtcq0OI3CcdQ+67GJeLRI8N8+xL7ttWtsi4AGjUTbCnNIihht+tXZkCO4tDbDSHkOlA07bhcTZdwpZ5N/KDxwMR2pCDR3GC5PkTEU+OttqC6VrSwws7gdP75W4TfUO3KBpsb2Rsa/VC9kZPrFEl1OKzSKxOZbmKLTTMhsB3aHIcKtWtC9aLbl+CR37EtLT2Y56mVwHIviHrs3b2khv7JWUggHUbi3bj2wjrTDrMKDW4+YYKHFt+JkKjHeKF35Sgj6xoN/0IWBa+Cna3BD76lSZEmPGbHgZDbSLOrJuN0HlAAPpz3/W1rLDuoKQ0uHTYGZ9lGxBp5kASJDR3QFoQQ5wQAbkCx9dJsb4VR578adLTAVDP4pBKUAta2n1ly67tEfMBAWpYAuD2HriBQpVKkVN56o5hajJb3LJeS6S6gg/s2xcruSNaSRcc344eJ9SnRcqxKbT5VPEt9vU2442UkrEhxKnuyrWbcSkKtwbn14uaxzghdFNS5sylCBoKqc20gUeLIdvHaWhfLrqgAt2zZu03cgE3sq3EvZzJDj0JpigVQSVOsuonNONOAsPto1KMVxspKwSUEEjXdJ72OIjSGf6pBo8SYYjLpIeK3tCd3lN3CAfKO9gCQALc4ccspnxX3ZyIq5qSpElTzskx5KhbSotrcKlmx51AAHykkXtjoQlnhIVcoJ2QqTmeZBmreSlx92SQ+4tp7Q6bG5KSDouR6kWPrwTiT5fRX6lVUQIEpusMsBxnZQVtSWgtwOOqeSu6CsB0pDhc0X+klCcJ3M85lhVGVOqjlGkNSFtQp/j6J456O2DpLIkgthz5adXlI1AkFJWL4FR8z5zYrkqhwKxOWmG1LjSWW6THU+3CceUXPlLAW4Ah25SrcI4APFx1KSGCM965HSyxy9s7UABWflnK9QcpCJ9AzrVmGY0d/wAM1W4EZRLGzqMVLwAcBsQAHLoUBYaTYhkjpzZW6Qai0WqW7fVHNPpmw244hd3g8NTmu4IFk61o/eFrEutD6uSEZTfqtao+Wq+W3VIlsR1PRqp4AGzbyVuoVHkR/pJCnQscg+S4xHIvVmXMrNWyblePWXKdVphqcuCYDchSVoLhIbigOBdmTfUAXAWkL1EgLHfqRQhrcpOulh1XGZ71ndmtpzTc7W8t1/YolSRS4lQmPbkKtIh7SWg4slWpxtQ4uLhSm16bgkHnDu1RXaxlx+Jk6TPrj8d9iUily3o7F0WN3nAhWg2WCgi6FmwW2q4KFwPPuVaDTMxNTD1HbmyHFFvaqdBqEOWdtIKGnA4yA64ElHY3JcQs8G4WdMc3Jy7VnBS8wylwqgwH4sinwG16nEE7bpYkJPy03U2sJ7FQ5FsYKeLsJuxmFr8+YWmdziztI9bclZ83J+a2cmpry8pUydTmBUA1Eey+9Ik0+MGtpxl95t5PmauHArbUSCFoIPZFJRRX6zlsVKgOMPOwYlM8M8tMBbjj63FLUqW8v5sdTrbRDiShzWpYCkAKGJL1Kz7RqXVI8nJNZlZahfKMqSzd41Bht5shmTEukFxsA8nz8OI1W4ERoWbspN1jMCHslQ57kiKxDbosOoh4OPltbO+yUKUhYk+JDly44+HG3EEWAJ9Ox1MARG69rA3XKPbOFywi/wA1Vuc6wmYiFS5dNktCnRzDjMzHHHVhAU4SCSrQbOqcACQAG7D0Cy0rQXUNSGWQ0X1KYbSB8ouIAKEA99fm9e/bCvM0S8iamNUGJUSE58lTKHBdBukXDhJDnluvlY72UQBhsyrDn1Cpx4nyZDb77fDzmnubEfqL84+fTNdPOXHe69VBaOIBWDlqqR2KklluHGhpbcdaefbkOGQZrgSGkOeVRtrSEjbAOhwaweMS/pxK6ionTpeUIziWaat2sSozb4dUs22nmZKgpL6AppSghKhytkWAOheJt8OuU6O1Eam5gq0Q12Q5LjxoyHQh5cNotK8O4NYWuzvzhqb1gtktqLfe2WelmTYD86XSKO1EcqilKnWuvxSluannFOE7pcduQVFw86Da4x77CcBPZsne7VeUr8XbmdGGfFUXkHIUPKmfMq1uNUK9Fh1+Q4EPPvsoHi9ptSSGwHUPMkFbZB02L19zWlbeNA02jUfLsp5NKpUSEzVHgSmOyGkh+xISQOLHzEccG47EWY8sdMm8u5uqVdXUX56Jriag08+21uiWu7bxcskIK3E6XC62hsqIOu9gcTYx0qU0pSlgtr3LA2BVYjn3HJ4/LHq6KlbSsytC87XVTqh4zFEEX5SnHiB7YUbareYnHgxchCG1FRNgALn7cY35ljuEnUnnBa20r062W1ltW4nWgK0rHZQv2I9D3GCp9co9OkKhPTA9OTx4GGgy5hPtsNXWPzVpHuR3wmKc2VIjaagZfjEcl4CoT/0SgiM2r81vW9jbCmQc9UWscfJKanLgUaCapWJ8aBCRf+sSnQ03f2BPc/YXPsDhr/GKpXIezlGkOpEthzbqNbiuRIbbYIDjiYzlpMmwIARoaQb33ABcuVMy9TKTNXV0Ifk1RzldSmvF+Z2tZDh/ZIt+40G0fbA3TV3czCRKjtopraGnEPIkLVIcDem7fCRobC1LOnWSog8WNxwscqJYoW5W3BNjYX3/AHuurhMcL5SCbac1AKpTKbmbL1TqVQk5pr9JaiuRIDsjXHgAoUyY0pmI0rbXyoJQp7U4NrXdxABw+VjMeVqfWlZdRElIbiOCVFkTm22d7bd/1Cw6ANbafl+ZsAjduRbs+LqjjTslt5t9Z0eGbjrllJ2kFLoWBezQB+Z2BULAAgXxDZWYIMLMkJ+t+GfpjbNQRMdkxwDH1uxvnJZAUtbj7V7AqLg41gLUNXy6XBII35ZHg72O9wTYX6fBe6jrJHsJLfVOEaRQ3WqtIolbaTHfgGS7JMhDkiKhEdQ3X3CAtsuBVim9m2kouVm5J0/Maqzk12isQX23JkZAckOtIgtMSHGi43oadIK223C0SbAE3I4AxH0vpfr6H4NKYksV50LktxQ0/Ickocu4FAlOjcKkEtXQN1JFwL4lNWzRSqwiE+nLUSRIly2oLs6p0xT7DraIzjy9znh1vQrXe+nkDWSL8mrq30cZp6cXO1m+HTe/otsVK2ZzZHDTzUSmTjTqFPjSXqXHT4Av1BdTEjZW+h0uPtulFwWgdSbkIHJGrkjDXl7L1HRCcqVOgQpNVqEZyTLltxUpU0gKS21spQADF0Nspav5BYEDucK82092VTE5XrjVRZTHeRPPhgy09oKHH25TjrlkONXcKtsXcUWttdraMM1Gya1QYNdW1nNEk1ptEbLzjTTb20QFKa1eHV57rJAs4gJQ3wCb44NHnbdzgC/W3Nb5LGxakub6TlWq5kmJzDL0Q4PylqLYWiLG3Lhpt5sWK3Fg8g3bslC1EAIxXECtRKQmoOvblMnvvRy/4ePvSCw0sGMzGUt0DUbhLhbANr8FIOJ7lDpzlBVLdfr2YZbanXJ4SA83BakFtzVvSXluCzrayFODWltKBYhZKMR9rLkVzLUDMe7+Iy2t6SqJEkOiM7reLTUeEyUbjsdkNp0C1zdfl0nnM+B7XZnkG/Mfv6INsb2CU5tr8mnORs3PZnpkedU4mhx50Rkw4bwDan4yHlpA3XNWpHFm3EkhVk3xHeoCG3pFMoSKWy8+GizFDcJB/FkbjimXFIJshQJshIIuW0ld0AYRPQ8zhl7NVaopaYdZEd0JqXiGmnUP+FStncILTYUdkcka/IL2thRW6zQYrf8A61pipE6RofWp6c1IU4GJTqW23XEI4Kily4FrXb4NyS7KTNaQlKZtCnqgws0Tmm6muZFrAzG4uU02wYwU842ylw7nkuVh0lJS4OOPMk9n3MuWa1ESqBRJM6AlyM3QmULkuuMqJ+Y8867pBIbO2kODz7iW3LWGvEapeYa/KitsZeokOSzT3nTTWEskP7aEaUObCDZbYO82R3UhtojcPGGR6p5gzFLlVaDLfm1GlBsKjnwp0voacS2pLKAUSBwglsDX8zkEk4p7J7T3grM+neCkOXYlPcp7FWR4ZmXPc8IqZCjvLXFbu234VoWcGt9Tbziy5cFe5zZw3dMw5dztCpFBjUXKS2alDq0uqxZUuntQY7znhilMpSUKLbjSUISNRCQpakoCVlROGrIubKK7TpaKlXY8KbDLxprk5h2TeQ4S0lbjRIbBTuvE7SFhIBOmw4eMzzakabKzBT6qqpUtxDjBEqWl0ORC0q69kJs0bjizl2w4ldkIULCGZ8L7BWNY1zblR/JFDqWa6zTcx1SoMO0KRLai1GXHnOJNUmBiRZpTb4Dkhoea4b0NNB3bbsA6cP8AEzczCS5FNSXS0Ip0aNNeoSESAGHEOJMZqKXfIhLqNsaRdWm5ABIDlk/LDjVLEWfVIcyBNTAjxpqUlia0wE7LjIssFFllICVA2QorvoOJV+CQ8vQ6e7RlyKfmJyPtTZbUjxBhuFxOgvMtNJWXFaX0NlO4daULvbgWVFbE6UC/9kMjsqisagz3qxMqeVIjsBIp6Y8N/MEWSl1112yi883thhDu2dvgjSEHSGwrWZE5kyuVFqI/Ra9HiI3kKpz64tmUydxwtxm3UEIKwsH6WxbSdZWbjDOuoT2KwyukyojsRbxeLoaSjzoQFuNbg+W6bqCinbsd0r1AcYmlBnGoMy5FScYkNQ2nWpFkugNMXCt8NLB3ntbaUg+TyN2XoOs40sezOOxFyN/RK3Nl1SbL005sW1GqcVxhbkpuJvuFLxamFJCvluDQWxqU2PISDcW5KyppdHqkZDsiNSpH4ZHIW007FjbDTHlIaKjbcdbDdwDbTY2tbDXmOGA4t6iZwhtoWVzmo6JCWpivmaQ0VgOLFnirW72HKB3JD5TU19lpVUcqMot1B4xmg82y6HkBsuMKuTwtwJduDdu9tFkEW6cEEFcHl/MaDa391TUOezvMTq54BMV6I+7ILod3WIcx1L7bZJJBSUDW6ChzUW3OE8gkngPMuOsuOFLSwhtZHJ1aR2sD7CwAtx/HDVGfRnWhuz6Ma0iSWRNgz5Rdiq0lR4CXR5ADqGlSLBBvyMCQxmqoVeXW5zwapbqi8yyykKeYbdU2nwm7/tQabsHGzYL3dFgSD7fheploJBBG3OHeYBAPI9fJeRx6Bk7TI7u/BKihPtjgRe/GFBT625x4C482Pq5drqV4ayTFu3fHCyD9Iwo0Eqx3QR9ODmQSUtEemObKrXuMKQ3fHCi37uF7RRJ9r9cd20C+lQJTa9j2/P8AiMHbZvq08+wwAtIdVZXYA3UDbSLe4/uxmqahsQvmstdND2xtZFWVqUm5T2PHci/b9e2EkhiSHHVwlIWCg7iHFWSrurgoFwe/uD6j1wa/GMac1IdqLulaFqKHgysqaRpcuNCb82VYnkec34GEqa6yHmnna3Ghoca3Y7EiGpl5QLe7ca1KGgN3upPJ4AvxfiVWLRxtfff9812IsOkGTouuvvGYIkZoEvhbkhbqlfJbQEhLdwLAm4NvQE8gWOFQgIkPNx5CndzeDq1LC0k86ebcgdrBNj3sD3wgfdYTKNUhxtmHEaGt5xxxO42UuOuNqO2Q42CpslSr9xySlWOCsMqZTIivOKCyLoeOhC1kWbQTc6LgGxUA2LHkWx5mTEXZCxxOvXTT/dejjw9tw4Jvcbalz2ni0mQ62T8sMlxzU0D8ohZ8iFGxJuT8sEXJxHswrly3nKgzFnKciNoqKpsOoBvbbLZ3Q7cWCyjnSQT8y9wbDEkkzYjtMlTp0ZxbEonaaeIS0myz5dzzclwk3PoB2AJwzO02HOhO0BEaGwh1iXGiwGZgmR3XC06oPDQEjWA2Ag8jvzcC3BmeypiPe0/NdWOJzHg21S/p2sCkMpXMbU64NyW8hJWWnDfYbssJQtkISkLUEFZWTx7rI1XerjzsOG1VXH5MdbQcZs82Et3u8U2FgQW29oAua3W7iwviO0KfBgUGQzUqpInNwlCS69L291UlyOh1O02BrJKnHEgKJN+fW5klHqLkJwKLTMmqOXSdTzexEYaJJLllANNtgm4ACFdkW9LMMrZI2CC+mvoPOyrxCjY5/a2uUqeiCeytyVJbbp9PfcZZZcf1FzYWEpcKyo3sscA8HTc3NgGauKlyH4eb6fXocNqkS2lvoWttlpqGUONOylEKJbbbkbPAC9RTYEhQw8wEpiEQ0SAl9tkSWZrzLIbZC3Nx2TYK+YE7iRqNhyhCCbE4j0GdTUzJ7dXgSYsSK40zGvISmRvsLuUONCxQ4kJChcrIvf2Bur60yRCJp1PP02+iSlpyHmR2w2/ulDcR2Nll2M9PdTS24rhLtKaW6+dalDw5efSLrfcJs22hAGrQCENKOCJVCgwpSHPBxJaFqcbYQndmOvqQhoKaB4CBptrN9AFxxgynVl+UJuYMyQJyH48UuwJU6P4eMyXfkpcS4tSmy8pCrhdraLob4J1qKXmaFGiR6hGhOmRWHPCy6sik1AQ3ENL8rcZLbO+63cnzaG90krvYC2WhZFUzt7Z1/wC3l5rXUPkp4j2YSmdW47RZTPZhplFneYjijPKbaaQQpTriilTYab1A3sCCVEXIGGWFEouanWc+V6pTJFIEkNUpl+a7GaqLqFpcEgxHFJDcJpDY24wG++tW88UthsFHPq1Oz3U5rNNnO5iEmb4NiG5EOzU5rTfEXaZAXHpzAJce1H5xJRuEFxwvE9NWy7SXKrXanPccQ2tiLHQ65BmS2LB1xtmU2m8WLyN3ZARoCbuEkX7lXITKaRmwFzbYev5rlQRNDO3fudrqGdWswZcbzRlt16FDnPOS5NaK5U8sMubDJIadJF3fmuN7igSG22S00nWpay75Rr0ysLiSJFPj02XHPj9mpLkaUyHHBeQtwJO66QoIa5KGkeRv5ilWaYOQafKq8SjJy3TWFuvM1TOT0s+H8PHTqdh0Vhs6nG2h5H32hpChYuuXdtix5lSgx3HJ0rMlUQpyStcqMt5E+W+6EhOtlAUqMhCm0X0tgt2Ov1IA0jiD2uAGl9bfX9U12l4ZYk8uf+yZjW4sYM0eoUWFKivktPN+BlPJIcdS6XFhYttji5UObo0hekgO0vMr1Ukt0RbkNKFuat+FO3nWmO7SAdy41aQNpPtr73GIYa9LXUJXjFSHWmNtstz5arFK+NxKd5xbZbJtpSSdwXbBAGPZelRqfNaiRVeKlCT80vRw86ppa+AWXFAOL8qwHB8sWPe3HiMUxSSqHZsccg2PM/qvR0VBHCe0I1PVTKnTq9LgvSYqqlPDi1lbIaKVOWNlAnUFkm10D9oP1xJYMmUpSIgiSXFKcLAeWUOllwDU42fKNtdyO/HHp3xFGZEKJUXK9E+RJkOuuPNpcfUptGvcaUQhXYLB81zyD6Wx1ldabrFPfnwUTV78ne3V7ymENugpLbS7LcKm9VvRVl34sTzaDFY6CcPcCXA76DTn5j4LVWUZqYiNFPmJxbSpU9lEZt10tsthCt62uwJ1gG54IAAHrz3J6A6pKEyUkuJA1LURqWfW4A4t2/TCp6miH8yPGMeO8S4lIJsvXzz6W5/mfTHVxn2UbrjRQPS4t6X7fl+mPuuEMjytmEl77a3v576r5Pibnhxhybb6JC0nU4hSu6dxf8Vgf6MKwlNsEw21C+lPKGm0r/hf/wC2wrSnHoG33XD8TkbTxonxV+zyD/A3/wC/GGvhwmqeobGtBWZJdcbXr8yV+LigIF+3p/Li2NvPvJiRpExZATHiyHiT/kMrV/oxhv4b3B/QqmqDbO05JbCnEufObf34KgCPRCkk8n1SceD46daKNvqvb8Hx2e8+i0/lOU/X83xjAmLRFpEaOpxTUf5sh9cdxxbJKwdDY0JBCRcn1AOLSfSkSHSFXu4TxirOm7D1IqjciS0WZVQkOSjsrOoAh0NFCjaxsm32I+2LYccjTHFyozpcQ844UE/vgLtdJ9R9/wDvxzOA5mRvmDja9lp4zY90cTmja6aKwa4YakZbERyaVoSUPr0JKCbKuoG6ABc3CVk2slNzcZ06sNUTK+YotOphNJZhuzItRZgshph9ciK+XbtSQrdQlpkJITfTuurAUR8vQuaKrS8txkVqvVldNp6ViO44qGZDAWTc7gCTZSgktpU4Q2kq5te+MP8AUjMBgV+W3RK2mqMupu3UvFNS47255pDTLjijtxg8rShJsLjWlR4OO3xfWCGkMdtXaeg6rh8NQOfNnvoFWtJmOxZ7cupIegIKwht9SAWiTcDvxcntf174dJsCsT6/sS4wivOOIsomwSCLarjkIVYKNuQTgVETMoExUqShtYLa29S3ANTa7gL2jfgjgi/ryMLp2ZUzHXHkMNBTryyywkAMNCwG2E/uDsb/AHx8lJ6L3icsu0aG5PqlLzA+gR3I7/inGQy895DwQ26LcLAJ519/TCbqZTkwapl6HlekSpTTFNXGjy5MERjUBYOJSHxZtxuylEcJWleq4ItgcjLsqSmPWaHFmVhS3R4qnR461PbVrlwXGhwWHcG902I5GFfT3KWVcx1t9ms/isemIg+Ibn77cdmFuf7cy4O9vMCO1iLG4JzEASdsToOSa6do2YUU3LLcJrK6H9qRaVBrCE+GmqBBGtLay4POFXacWQFhZsQSC4ZMo9HjNzWvGQ4DJUiczGlAu+Fauout+iFgardgR5e44wyZsqK36zCVUHoVRdlFChNjwTCEttADTZWys2b/AGVjwLm3AwtRV6PmBMd6r01sMRJCzKZQP2wK7qCtABJuff2PFsUSju5m7FKjqBmFMhD8lU+Q8ITy4MJDjJW2w0PmJQFAWWhsk2IueNBv3wGtVCTmZ4UyTlyMhwo8QJUZlQSpgkpVINiANKyCU31i5PrfEjOUoRiIr0RmksU4KBklqQ0JN9Ys+W7jdcKbC5RosO4tfCSpSW5dX8exMQzTWvMylS2tL7oXyW2+y73A4P2xnBDjoU7QoU+5W2E1KiQKbF8NRyVvLEZCGy4hACnAedCyBySv7G4AwxUKvS2ZM6FFZYlx5jSCHNzWCSu6nE/uHsUAm/CvvxOy9TKrmWS1VH3YcCaw443IjNiM86A2SptziyyQVAKABSQPKQDiGU9xiNVH5UB9D8RwCW1NZClJcNzfW3wOL+ewHN7ffVvuEpSYsuZZdpfjIO2wWHihpaRqeLbnn1A8EgeX2II4NufU+XTkbSnZLcZoOLSUxluDZP1N7TZugAIvdPGrST7YT5vzLUZ1QiU2pNQ3UQg09EYIIDLZFgXCFeuonV9wLYRtyDIq8VbtN1NPvhtzRJDZU2D+75eBYc3uLm/F8asji0XQU0r9TrSojzjbhTZFmlyRuurAtw0o/s9QF782Btf1wyU+j0uW/OjZkrcxqLHiNzjNjSWpMSOX+ytxCrhCvOFtW5IvYGzgeVVlFPpQqOYID6EyCiTMTBc+cnX5SAdJsjhtJTYEcEX7lFmTqhWKiJWXqczGRCc4MjwwSVuIbOlxRQAu7dz5jc99dxbCMY7ZqdTyNlzJDVLU5R4tDkVJyTuRiEFuK9IbKS4q/wDij29bWtrHFxT/AOKR4tXkuzIxQ3KXIS+lLgQPr5DfB0C4Pe/IPfAsv5tfYjKoylMtpQ9uOFSA+2V8/bsP48jCqfVodciuuxmFwlsSgApTZS3rB1DTybHkrHA/PjDRxGMnVJubpqnQIDMSPKfQ2loO6zMSEBJjC4ANhYqBI81h7Eq4s3ZfVUq5IRLW+7E2FfLZQ22shC3EnZCl8a/L3tyfb1fcwUepzY9Hor1UpzbC3JIkVDfbMQm/kDhQC4HO90FHco78kRlopC3IUxxwv6AC24gBfvYp9wOf9ONkeXLqonubFajUyMp3bMtxxwbqQNQWgc6ze5R7G1jdYHIOJXk11EaRT5kqdDMh+Q45H86VNaAka2lpuA26LOp0cag6CORiEzGX3oSJLEd91QcQyy021rCiPq4HJ4sePU29b4dJOXnKC4mXVHIwQ80248wgqVt6wNIBJ8/ccfoeeMXRO7I3Vb9W2KtZtnpzmeft5pptUREj+HXrbU41LIabIcbdGr/bCbKKL2sTyF7jhV6ZOpMmoUDLLrD9OJXEgtzZTRnOrfjtqVHcYQH2lrS4lS2i0dd7DUnUBiH0tuXWx+H0qmx586e8Y8X+taQXSQLpcPJ/Ii9j6G2HzL0SpUSBHkzI8yl0qoNNAyYsnQJDhuptQko1LaIacToAQWyFA8Ec+iosRLgWyN+I0XKnpnNIyuv5HVIp+RX8voiVJ6SudVKg/wCGEcRUvMPKMVLrLu+VlDoUdxtbLoSsagQeDZnpdNC5bb1NhuUmal1hNPbjOSYbzrjlwlxh8ji5DV9Q7kcjWCbCmxuktfyq4y9mNyTmOKqXSY1XqqiJDra1hTTr43bPEIL50pKbuIJTe9jEqW6pNdKJ8DU3AbkMMzhEDjc+zLek7Ltwt7Q4FFvWAA24W1ItfFs0cJlAg2/NUxSyFh7RWJkubW855cqeUOptYeZaojOtiGG2nn1x0HadszJBJbUhStA1CxbFuCjAKgKRlDMNfNJmHNoBj0yW/wCBbeS8xspUzMtoRuTtDRbd2wW30Nldy5qQpgmZmm5QmtuLpVDfcjqcbdKVqXHzHTADuutsu6HUX+WbXc0kW49UsjO1AzXXH6tlL/1IxFhXiRJTjxaYkIcSGluO2c4La1pcsAj5TdrEDHQlrzDFkeQHDnzWWOmubsvYot3L8diSiexEhlmUHZuoTnn3mSNSmVtutgBtpdjtqN76WwsoB5R5rdVm5orjUqRHjUxDUBtLrSVMVhxaWrIadQSA5otuXWS4e+g3OJFQeoeY8s5TlVfK1SoL8l11uHV6Y80HiRIAUVlsgBDRkAFDvIDiRuDzXOhfh7yl8N9T6VQc0dTMkVlCKhUpcBxyTUZD7KZDZ1SW0paIO2ougH6wQLcEHGeSWKClLjoStUEElRUBo1AWL8t9L+o2f80v5OyHlKTmeqMr25TFHQHGYqwrQQ+5cNR7KufMscG4uMXnE+Bz4s+jmYKRmtqkU+qaIMmUheVpgkPwZgbKWmCl8sLKyFKO4zrHCwUkfX9Ucj03JtMyvAYyHSaZT6E62mREapsdLMdSV86glAHJ+4vfvziocwdcxS83Ky7NrdICaZUHEuNvJKVSNFx8s8aClR9bhZQRxfHnoqj3d4eBey7r6dsrDGs85TyTT8uimorbbf8ASNhpbw8ZHQxOSHG+Qps+e6WvJxwlAt6XMr06jqNh9sT6D1DyjnLZoma8piTV4ACojlXo7utl11zTutvoTobbU2bagRe4ue+GzqblaHl1iqVfIaptQEFf9Yp7q2XQww2oocdQXXGQkbhHLjpFhcJOPomFcU085EMjcpPyXiMR4dnhvJGcwUUShThUltJWpI1LCRcge5A/LDNmSsNUemy1tVehwJzTSi07V5jbMRtzuN3nWodrpSCux/LEMoGWMz5mr7GbWuobUugVKI24iHdma8G1lboDbmylpGgur0Dad0hwt6uAcSaB0t6eU6aurNZShu1JxG2uoSFOPy9NwdKHibtp8oIS1toHoBcg+nErpWnIvOvjZE7+aVFenmaOp9ekvSpeWWpFJEZDENYqzIjbpdU444p9aTIkWbU22NtoiwHINyZm5QahUlqVmOvPPsKFvw6m7kKL/wAooHfkfkpxLf8A7s4e2IxYaSjUFKCAnXpCSUC+kWAA4BtgwI5xIInMbZ5uUks+d/cFk302lUqiwlU6iUuHTohNyxDjJYbUfchAGr8zc4UlJv7nCgIsLnCYx5pfKkVJxphG2400ygJUVjkh103JbUf3EhBA4viSyPjAMbcx+Q+qVrQ938w2XinULdjgcoV6PSVSaW4VFjeeajuloR5bu0dtpwEFa9spLqEpGskkcA4QNVCoMSXoVTpzshmApiI3I8Sh2Q+CwHDLUAr5baiXAQrWsbJNrFBPjMP4pSp1Emvrbbccq61qc0MPNbbjfiGnlggbYSoWBQgBQ9FEnyWNYiK+kfTxtvIPsm4+LSN7b3Xfw6gdTVAmce4Rv5eYUAkZxQxKbmfhtOhqkNNSdidVEfOUgp8jbBc3HLupOhgNkknRYpVYE03NNSo1NjzpTSmKUqdLM2RPcE12XIbDZZdcStTa9d1BRVdBTtN+Ui1np5EJt8ZXao9OizamrxMGvuh10OTFnf3Gn7biXkttl5ltKwglohs2TZUe6h0tVMzdlepzq3VVwQ7EnxgzHEhcWEh9tsuP+ZsyA3IKVbt9YDrpdJDVx4Oq4bn7Jsz5e+SLb3AvqfUfFergxOOVxjYzzTnlV2j5ezA4iryI706U82Z9YpbSAl19xbrpaMoKO4jbdToVZA8psVrJuIS4lAzC1mej1WnTJcR2ezS6nKckFRbkMgOuutNNps6rwi2kHbuWzdvXqKw10+fVpcVeS6lkic0RGktpjw5RSXfEIeeagOON+RDbjiVOXAUsN/4pB5DRmPOsGjZiS3m2VIkRpuW0LlSEz0sMPy47Sn3WXXSgEIdaAShnbQWy0ASQ4nGOSmdTVQic7Mzm4i2v3R5HmCtIf20dxoeQT2unTai/QKnEcWpS4jq4lLv41uTAVvqDiVkksNtrUCR9fmKFFwlKwmpdORSc3og5iRMqFT8U+03HgMNKaeWhADDoZKi2yy624QEjW4bOlZc5IY6XIrdPepmf6zBemohtRKxXnW2pEdh5xqK7vRUjzILLbrgUG0oOpzQtwkm2JFX8zT6TmBmLW5cF9yt63BCWy1vw4+ht4QlbSQW7AqV5iXASotpuLGkNjrWSQwNttsbEC1vqeivbngsXlEPuw83v+CchNGnMSENG7rD8tp0EB5ppvUVh2+sbZNgh0WJDeIRnTMrdLqFTjVtNUS/GdjOLntPLjPy47kfdbkJlObbY7tNhLZGpaSsuEqOLNrs12TDrkGg0ya7NgGS22kokMsKm7baXVCI4EuXClOJu5YAuA2sLCrM+wKTKyo7Jjsux1RzGktQ5LrjbpkfJY20uOXQNtDrvzEhDSV6vInm1FNQ+5ydmIwAOupPp+quMgkF7qFSs5yE5fzFSl0SDElvz3KUJMWUHywfEaW2dklxgIVsKVcabFsLQbELxMJ+V61mOh0CCKdRojMhEgzX6lLZhphNOrIitsM7LRS+495wUbocF3nFAEYIzh0xr+SG6jGrFQogy3Hr1MjzY7bpfYgTy8GHQ2yLWlNs8POuEFbT+gJDgIM+m1im0+mx6SzEisUt2sGTEjKIekPhx4x0uKecW4Iy3XUtti67bSgv5Z4x0IYeyZIH220HPrf0VL25nNLSqZyhnFxNA8EzDentUN1eyinyy1M8Yf2YaOk7rZHiR3A0EIvfQcG5nnUyfW2ZVNRHbnoZQ+pMFbcdt1grKt3dYVdpYLim16jrVY6FG98P9GyPlfqTAqKojUGDWafUBGXTt3+swnA8FKce0JSN1W2YzLLSCU7jZWfrOB5yyRmfLyFRqUyyzIrrVosul1BrS3GjutuvHyJTuy3W3NtERTi2tDTxDhNkDhlge4Nvff5rZkc0d7ZVvRpUjL1bRBlRoiTHJLjDxbIdbbcBAS80SOHElKFAm1hY4ufK1TfqGU6Y7UaW3Mj1R3RH8dUBGfdLrim+AR52wWkjaCASFc60kkVBUsj1al0KZmGOzIq7LUgOxYlNcS40xHWXFpTsrHyUfLSSGwscggXBAsSh9IfCxqXVKvCmuS3EtOOykksrirJBacDZGhl0r2wFqDoUFJsQe2eWDsx2hCVrjfIpfleuP1ma5/S1DtGmNvBtUpG2zqRruGVBxSdp0uNuuFJsgNqSeSbYkgzSwxHqVFeptRjKhuoYbix4oDjIKSoKQhCSttR7iwDitwhabOaxEs00qvx67KQ/UaVKqtQjRIMxoOEI2Fh42eDmrWhOhxOnR2cUvhBIDTAdoiKZLp7VBYg77id1pantSH+S06VJUoIdDbSW0KI86zf1GOLUQds4SNOnktubS3NTOWiTJrUKVDmwmlB5DMlmSv8N8Uw4jdDrtgC2thxKFEjQjzAEgqIADXI1RjJqP4fMiFl1ujwdgiO5sFttPh3U7YDgJdFm+5t9XYrKrriYzsqaxvtyZEWMmSIcSW40/DW0NS4sZH1ncLiy44rQpFgAfIvBtNfrlckU1yYmWtL9lvsVaNpbLdglpbm4ouXU25+6Dtk3Ha400sMjhmB15JS7kUvjLo6n05fnx59PrUNtyOiY6+Lur2wpTZCFWDhKrrSk2Nxb1w8UufKqahAp+XnKfSnITTLi5JDTSX7anEtPIN3OFAHkkoUQBbzmuC+3NlbjUiM94tphp14uyXVKcQpvdQC4oBwWLVrEoK+2sKtgmp1SC1UmBlisLbqS9EF0sTozJVJcd3GXG1HyNra39D22CbBNjrBGNE4qaW0gJv8wqJXMIyEK2KPVpMWNNdK0TZjKl7imQVMlZ3A0G3b3LZDTZtfhChwV2QHMyqrMEaVU6vd4J0vNxJKn4ToDYCQA4lJGnWFXSReyByjDNlx95+ZTJNVZ/9ZQLF5sNokbriCQ4226hIJFhbUefma7mwODaAZ76WZmaTDYSEuHbQ2SUkFvcAd/xgu42CEix09igc+twhjosleXE5dr7fBcyqayoBgtYEfNOSnm332obLg17hQoXsVEIDlvysbm3cWAthQkWA5wU1w8zJbcKWCpxZQoLUStaQkLBvY/vccDzXtxhboJVq55HbkC35Y+r4JiH8QjdLfX5LwuNUPuhbG1JtGOlOD8dIvxjvLg5Um03OO6b+XTg4oJ5IwEbYJStQFrd/ubDC58veKZsbneFEF+OwRvPFC73QlN/NYgWv979vX05w3S9aS4lDzgdU84206+A8pmyNxOhFheyLEarcdyScKJNWaCH2ILo3Wlhp1T/AMplvi5BcPBNgryg34N7YbpDXg1JkiAtwrcakyUuo3HZC1jSlsD0QPKQnt5bX9ceKx2dkjiWOuV7LBoHRx/zBZDZek1gaX6ctan2UF0l35JAsfMpCe5KSNIHe3cC+OOOT40ostzChDzLqjqI3i5vDgtkkrb0EkklA4RwfVbBZWh9D0OSZDzklcQMraUoKCEA3shIINirzcgXCORhhlCbNiJnPsIjOzHhtp8zqpBdKv2G4Bcqb8vCFW4PKOcedbXvlaGXuV3TSBh7qWqiNSpEi8mI6rcD7jspsrS5tkOkquACjWlFz6kWRoGOZjmT4NTBbZYbfafXJbcZDmlEi1tThcSLhLhJtzdZBIFicKqZW7ty4aixBktHS9CSWi9oWQe1ghdgm2vuOeEesXlzKtJmuVKnJRADjjQbTIjBwOiwAacc5ba1N6rOEEK5uo3sKKwSOh7NhsN7fv8ANXU7Qx137pqqzMyJAkTFsx1LBQ88JJbUw2LlJcdIBDS1IuoLUgNjURc84ToRXp6fxP8AGsu/grehkuRJReQ+UJtdKWxYarpQSV38v0m98TKlN0x14PwZCCtxptbbCis7UbjbsQ5Ztq50iwsCrhRwGkZXXUnXVUCMpiCqQ3LqqttoNwG2lkKlOtLU2hplIe1ucfM2wWxr78d8DOxGTnyXRjmcX2conGy9mfN9dpeSMk0SVNnOKjJnOutlluFEaVqcn3cKUAqQS3uoV8tYN9RCEYv3JHQzKGdWzCyxW4aa83GcnKXLp/iadMbLm2HGbFDosbguLKlqOomxuBCspZmpOSqjFaytmhbNQS+7Lkv1NLr7suQxHkRPFutoCQy2EOFxDBsO5JXfm6fg5VlaHGrOWqYt6RWKa02pc2QyGkuMvOOKUlloEhDe7dR9VLcufQYWnaYdLqyVwk71lWudfh8600aSlLmWhW2Ux2Wkz6OpqSpLyLhLoYfU2RpBVYEOAE+4ScU9UsiTMvVBdPzdSZdNcUlbUeHUYz0Z6UJEguylqbAC3m3PVLZVqLZC7t8D6CdY6t1Eo9Ajq6eZfcrE+ZI2HBoCxGb0E7mjUm91AC5NhfkYzT1pyRn6LkjK3ULPdHZGZoNXlUx54pS++qC/85hu0chDYDjZFkEWuDySQQ8WFhyUYLgKmlUyT4SBSKZRPwqRPDFTdixGEPPUmMLA/JWbOSAQvbT+z1uJ8pQ3fHaDSX8wTmMuUxLr8ZxaGarMjzTHcJIP9Tbft80qCiHXW7FIuy2QvcWhYw4h2sT4MFT4S+4iBUK3HJSUP6bJgRnjYoJGkOP2AbBIB3DZs2RmWjxYrWXMvxq5UY7aWKU3TKQQ1vkAgQmSQA2grCgXr3AbdXpXysbaKqiaGvYLu1+qxTMe8m+yW1OVQaHR5+cK1Fb0VBqNS6LBecBYU20FeEjxm1kNthxd3CBYNthJWABbEYkURyc1Cq8KOuNPnvLKUwwXLwEbagLrUNtlg/NLe5d1ZK3L+RCZHlCI1XswjO+Y4yKjU6bIdi0t4NNOUyntltJU5DLhCw2BuNuST53VpOgIQAAjrlXXUF1irQ9iSzNbMCgOKddjqmAIClNFoAuFtRsokAAeQc6teLMWxAGlFtH6gDr1cfU7DoqsPp2iY5tW/h5fqmKRlfKtRqSqzmlyTmGUstynZU9prhYKnFPKDbRBW4sAbywXCBwq459mVE+rVF15FNaapLlM8FHdJBIJulTSkuW1r0diEA25uk2wnqjLUpaJyZdKenBDbyX5DrwjNNW2lFsFDqGj5bmyEarWKgL2aZtPmZeYbqlYo9UdRDW3OfqTs802mPoNtTYLW6sjhOtKiCQOyfXyRq5Q0sld009F6Ls4vE0I+dRKLTKU2x4RxdMkkxErajPBlJCzp3RYuoCS7f5hOoE2IHIOdZjO1iJMep7UlUZp14zGXEttOWtc7e4A255Qq5WO7gFiefZYKYtTdqC6y7GUh5ptt1iU1LDehG4UqeLhXotuABxdii+36jEgpUdldIRJjSQEusuRW3Wo2lLg03W4ppkBA/xYsCASq4SDjJ2gGwVu+6LcavXmmY1R8SosoRGCnlrbeHlK1tALs3yLBVu9yObjD644w4zFq0yFvkR7tvtveRlFuCVOXO4blYSoebSoEXVYgNVZqA/CokwF1sHRKQ6GEKDaXFFAcHBICh9yQTa4OG9+tsw6bTZ0V9velu6YLm8Ntx0pW602q6Stdw0ebm/ywO98ZKs7Ei6bw6qVZazM9HemU+gw3VU1LngqUpcpASpDdtbzZNhrKFAobsgJW2Rz6PsfMKHqZUajUlvx29xtwGZUkTJqWi4Ypee/sN7oVa5WAPa1sVfQ6pS8w0KMlha5tHq7ZcMp2QSpmISoNJIdAIcK2/InhCUFw2CygFDWKvWW41aywpmK07+Jlcte0ljVtCyS25YlKNYQ4Tcos4eL/RpwfGpcHqI6jxZbgAnQX3sPP6LkYlBBWQOgta9rnnorFo2caZMzfGoEqoNTn2o1UcUmC4PnuoltxGiE30LXttuKPIFirRcDEuYqEUVGRT6hGchBt9pqM8t1splbltNhwWzrOmxvzb3BNFdOq7Cq/UHMVarFPfgTZcBqHJhJQA08+tQeK2w2SRdxSnE8kpW7rB7oL9njqLDqdUNJy7GmUvNyhpVIqLn7CA280VkgJAjKdcO2XQCtOlwIPJx7Sh44qSJ2zS2cDdoAzeo+IsPVeelwKla1j8lxqCrIzjPjU/KOZJDMllxcakVZtASv63W6e86UAjuQjzcemMM9BVvJy8ChwttOvR9olGpsLb8OV373uD+oBxo6uVqZF6Q54S3JjqqFPy1W2lvPNL+QTusz3bLGtbp22WgCSAFgmxuMZ/6FIlO5aajxpTkdbcmSJLegFstrYYUG1exI1A8e1rEXx1sZxZ2MU0cjxlNtQm4eo20jpAw6XH4LU+RkSYOYqezLWp1Cwdt5fKhoedJbPH0pRoA59TwLg4dF5qbpVcrMKW1WasGpf9YpqJKFoakyXXHGw86tKbEuAp2lH6NC0bobwiyI2ZmbKe0t5YLjqE3NyEEuhu4Ha/J7d7W9sCrnVnLtRnVYx6DRp1Vi1GfAbQqSJFkbjkfWA0DuOOIUAGwQtO4UH6Tr4eEMgMxlkkyObYj59OfourjvadmGRszB2hUB6qdU4VLzAiHX2HKhPiNTKeKfQ68dpmZJ2QIqZQ0rCy22jddcbKCH1MluwKDnXPNLo+XKvMRFMSq0FFFfqj6IBcfLO6Gt0qupK1raX53AkraSVLSmwTh3zvk2iofr9WZhtRWaBFkwXGTUFocbqg1NswnR+1K2gC6UJB2yQgkkHFRLhzszR3qgIcalwX3o8KExAiEsPWVpfYBuCCkaVncuNZIv5icdXFKyaoY6OYje/n8PI9FyaKlbA4GPaytnOFEoUBUiKYsSJHlN7LS47hdaSSAdxolR13B4BJtqv25xW0hTUypuM0l1plxbhWhudIS6Hf7XOkAC/Nvc2vbDxPrNRfo4oSXy6txtAisuQG0FNwR3BJK+ffjEehR4bKnzVUPtyI99DgPDa7G5tcEn88eUiBAsV2Cpjluu1CFJVDlxhthVtIKkpuDYo7gfp34xMGarSqW0upR3oG4wFttpltOSWvDBHzEqK1fMJKuEp5+/bFVRJKKe4w+0gu7qA5c2tz3NrkfpiUxIhrFZEZC4zRU38tqVYNldrlOsXNybWQRbg3wkrdblBMtZqbmZHXahCojEYuuBoLj3bb59HkEEscAm1zybXNhh/oObUSKQxlx6gyJjj6S34hMgNEI06QFEkBwC/Prb3vgyXMRTKXOMkUyY23oZcQp11DzOtZbbN0AhCwbrQVAoukWub4jeVJTlDzV/R/MwbQmoLLKFTmQUKfIIjEm3HzSk6k9rk3HOGBEjLAKZlZqq7mPLlPQyipzZtHh7ay48Wn3ShxO7ZtW3rR2+gL4474FmqTIqMSFVZzEVLR3VJlNHUXo4RdLiiSR5SFA3t5Li17HFfzeo0io1SO5JjRo7NiHILUUpYBPlc3Eg28tjdXfgrNucOlTzFT58Fhp2uttUmKwxEjOQPmtNttkN2bt3Kie6vXufTFHu+10Sk9PbpNRrFbh0WttUyNLYcZL1Rgl99laLOILYAUW1m9gq3qm+IzORW8mLnUQ1B2dCcCGpbsYOoZcRtbutNyEHSCb34555Fg5SW0UKuyI+1NWttfzCxKbZaWeFNOuEiwaLahfSUEknkdsOTdJdzDl9b7OdrRERUJU1PW47vsNEuJTdHAF7Dng/KBuRc6tIwhmcq9lx3mpC5KY0YxZEJreWgoCmXC5YCyOSFFAB474lDLsl2hvIjUcMypiAwkODSmKUXKlghQWSq7YN7fSF83IwdGiQKSuGpKsvNSg2JJkMhYD4uHBdspu5Ym5OgW5HI5wuWinux0rr9REdsMr3lBKypSAjizgOi4BuEm4I49cMZc4tZRJqAalInsx5DixGS06FImoJY2wQSOOXEEi6B3B7G4GDOpEygxEuZapcShTIp+aZbKDvaCnU2o3P1giw0k6SCg67mxrppKY0FxpxuO3qK/luDTIJ02eCwgWKvKCFDji1sRfOKYch+xW02zC1tOtRnNaVXOocAANkdiCT374DPEoo9Haj02alLkxLbICba4y9RsLAaUXueLG5F/fDjU6hLCXJDcPbjOq2W2kuA7YQBZs/vg2APmH5E4cq8rLDsKHO8K+5UG2WkPMMx3GuwF+5va5BB59bH1wqqs6u1GgbUae3M1tMNoaUkJcaYRuOKLlxfvt/MVc6OOORi495yUJihUyO0742ZORBjukp31NXClkeq/7HI554viQ0yRUJEqJSGUNvREeIckPvJQzHbYQC5oD1hzwSUi5JdCADfCKuxImW4Zyz+FPIntSBI/EHWQNxgsp+QpTg3EISbEBuwXe5uLHDhlKt0VitR01VbDVPhlgNw6mHXG3ndaSUp0IcFyAq6VEI0aySL3F8Qj0JNwp3k9qylnqnQqdUG8mNs0qpNurpb6y2hLLosXNwg7jJP9l3TclsLtcXIm1Gtx5kZFQMBT6GRKaaaaC1t8BRHOoLQLjuOAOw9V2eKpHkZmQ1TcqR4EJNzPSzGQ4w+2CUtSd5d9wEaUkG3DWixKb4Ww8xZbzLUkR6vS6c1MLKIzUwgpTu2UPfyCzgA9tsdwSDrywvkHZaeqzu7Ro7wup303n5UpmXXKnVKTXKu1TVxmq0uDCQ/IahOLU7w6GLNOs3cJ2wflpUhVgA4mP58eotVmsV2l14Q4Lrbk2K2YLsJp+Jc2VGGlIOl3WCAXLFS1oIAsW2C/nTp5Wn5UeemmOBt1tEqA8F6tFku6EufWO3BAPmB4BIxv8Ao3Rj4K3sqUGPNmyWHo0GnPNlmrTLtOIaQsFOk2GogagBZXYj0xvqcWoYaQRVbgw7X/dkaXBq2vmMlJGXgb2BNvldfPum5mkzqcYE6cGY7YRu7ADocRcguKKzwsAnnnUSe3fDtV8mVTIsyk1LNzm2mRJAcDjzciM9G0pcUm3mCHFMvg6XW7pKkWUe43BXvhy+AytzVT351aiOrdckEQazUWAHF21KAT2vbt25JtycL4/Rj4I2aBOyxIreYp0Ke2226ZdWqbz1m3NxrQ8fmDSq5ABsLnGCDHMGiuXTi/LUfqulLwrjLgMtNJ/Q79FkPN0Sj1ihSKgiDS108TvHtwW21hpA8OltlCdyziNWkKDoIX59s6xzitshRqxKzMlcKXPkOsIcLBZj72+du62yocLWGwTYnktkKNxj6AK6LfBClTiIFczLE3gW9EasVEANlIG3Y3BSLXAN7Em1r4mGVPgt+GqtxUZmykjN8ZmYh5lx5mvzowlNuXDoUgqF0ruQbAX/AJ4Y4rRYo8mOUE/5bH81nl4fxDCmfz4XRt5FwI1+IWMukVHpFfpeYEJj18Zfp58VIahRSyZespu1fSRsloF0sXTcC+qxRbe3QTpPlCrdBKBR845aotZbdenTUFyGkoKHZLm04nUAsEshoXPnNvMSbnD1ln4Yel2TUyGqNHrL7MhLSFxplWekNONthYQ0Q4TZvStadPbSbYteFFjQorUWJHbjstICENNpAS2kDgADgAY6FdWQyQRwQ3NtyQubR0skEz5HnfZBptMgUanxqVS4jUWHDaSxHYaTpS22kWCQPYDHzizZVTWeoE+S42vU+6XF6kWBL7zj3F/Szo5Htjc/Ujqtl7JO/Q6hIWKjJp7j0ZDfKis3Sjj0FwTf0Ax8+4yWnpr1Q2zreUXBdSjYE3Fhew9DYW5xz2RiQ2K6RJCtCNmKpUXKlWqlNqLzKkMTJMca/KFl1thmwJsPOScRqhync3z5VDzBDjVGJXkLEkOOOp5CT5xZV0Cw5ANj7X5xq3pz0vyG/wBNaEzWMm0uorqFMYkS1T4iXlurcO9ZRI7BZuB6WGH2J0o6YU59MundOstxH0AhDrFPbbWAeDYjGyicymqBJbZY6pnbRFnVfPl3JGasrZgQ/k/IVelqpjzamqombGLyG3HfmRpLXzA60FtaibNrUFa08j5l90yQ9VKfFqT8XwqpbKHtkrLmnXz3KUn+IBxo49NOnCqh+LHIlE8dYJ8T4UB2wBAGofYkfrhWMiZLX3ytTj/yX/jj2MHE1Ow6MPzC8rLw7LIB3h9VnAtAf/RjiWh/5TjSf9AMknvlSnH/AJL/AMceOQcjEf7E6cLf+6/8caP+LIf+2fmFT/wvL/3B8is2bdhpIwCx3Wo6b7jy9LadB5NieT2HA9e/Yc40ynIWRh//AEtTL/7wMEnp1kFT7cpOSqQ48xq21mMFKRfg2PpfFMnFbbAxxm/mU8PDDw7+ZJ8t1mSPRmpNTlTosSQmQUo0yo4s6XUNkDSlxNrNtk7hPdG3c6BiKTcy06PLg1CNJok5tlDcZBEZpJZYkNqUotskhxF29OtegAIU36OAHQ2eMwfDpk2qt0rNMXKUec780tuJStaRo29agCSPJ5efTjFGdWevHQCkuojdPulOTMwzHklyROepaRGZJ7ADSC4q4BNrAcc48xXT4hibAxuVlySTcj09PzXpqXBBTO7R4JA01GnxUbqlepGaKSIKKfVXZc92PKnOUzS884Y7u40yS+Lo02SQ4TdsNFYJ0i8RfjoXMq1HfjSJ4RR3aYVKVIcNYYkOSHHW2Yqz81fzHtBSS2CyLBC3G7WT02+I/pTMcFN6gZBy3SmnXN4vw6NqaW5t7d9CBcHR5RcEAEgYm3Unq/8ADFBy1NzrlGn5NzBm2CwtikNPU9wPpW+UtqA3EhSG7AFYSRcN4xz08z52mplFwLNLelrG+q6UGCztidLDES3mQCQOm2yoenNsSiqnw6w7UX5TayzUlOOMSpclBj7U4JdLfkStsEbi12JsTZIABn7LkTMlOrMhFV1SnJq1uLEptgRpmossuutuNqCwlButwEOKLgsLCxn3wr5hgdQeojlHzflyi1uVBppkQZq6Uy261tlKAE7YH7ulNh6JH541hUunfSlhh6qV7IuWUJioLz8mZDbTtoB3FOKUrtz5rk98JLT1FMDEx4eDtofn6ql8QLmSHQr5qtZjObsyCIwzNcRMZcnVCmusmO0qQw1dlp95aklbSluJsrRyCo8bZOJX1AnQaGuhilzY0lmFRp34lGRI8Mhbjb0EtOpOk2LSW9JSAVlC2SSQs2snqz1u6C0+qS2ulvSHJtVqDqGmXa9MpSXELQ0QWg0k+dwIsNBVYDsBbvS8XP8AXarmyh1CpNRn0xKpeHERTo7bDJkLbadbYaQgIQXBZJI85sOeMSDBZImlriAXfE6/grZQJJAeSkVfzDJfeoiJT0dccNtlSUE7sB0gbjbG3pu6Gr3ctfWlWjlzHpEuoTMv1ZFA263WkR2HXH45cZSlDj+5vFx1tO48ClTJJsFaUNncNyNjZtyN8OPT/L8vPOdMhZVp0SPNbnOyjBO47N5S2W0oN3HjcgBPJuRwL4z+z1y+E6JUYrNP+HBqNTY5KESjDZSpoEi5DKHSbHQk257DjGOLBppHAE3A+F9ep/JO1w1sFTMirLy9l6mx6O83IomW5zTMajeBc3X5bst1sPuPBKbLdcacVuqO65qNiA5t4iWaafMTUaflmiSlZmp0ku1Zpt2DsBpuNIDIDIjqS6jemKkkMJshJS1tobXcn6MZd6MfDpmlDHUDLvTbLLqqgyHG50ZlbRdbItZWlY7DixHBGPU34a+gFFmioUjpHQIklDqHUutb6VpUhalJIO5xZbi1W7XJPfDSUbwSDvf6dLqsTNa5YroMZ/KsEqq0DRDnUn8SXLj87hJbgoDKdNgtppLTbbZWsuLUV/40glU2qUWfFjZgp9KFElzJRqVShSApqGDEW42kDfSVrAaa29hsBFyUAt9zumH0M6Nw2RGi9OaW22AgIQHHyEhBKkhN3fJYqJ4+3sMMtT6afDbQKcvMdVy1lemU9Cm46p0x9xlrWtxJQ2C4uxUXNPlAvcDGQUMoJcDrb5Kx8zZLNbdYmepkjLtObqr2X6xTpr8RuUuS4lZLr7+66GU2JW3+00AAAAqcC+dAwiq7uXf6LVCfDW3GqLlP/BAxS7tFch9lLQjcEB+++5oTcA7dxoJ50g7nH4AH5bUAdIqfJXEk/LvTUqQlxCxbncsbLbH2BHvfHKrm34EsrTo9Qn9C47NQiLTJjNt0xpx9KrWSoI3fIADwVWA4tjDT1kVPI2F8oLjyI3+i60mA1z4TUOiIaNzpp5b/AN1nCtSqvCn0hjObdXSmPLbaZLNTsmmu7KTtl5/zutFoLC3bG69QAUFDCTJLjKnUNViU8mYFmMISIJdL6390OsvRVtuBdy20Q02AbWILZuvFxnrJ8HbVXflRvhRp2iUta3X3Uxy+tRFi4U3I1keuu/3xbvSyk/CF1jq8mRlP4daaZLYR+ISJUCMhTJcBKS4NwqUFFvTcA3JAPfD1WGMY0i+XNta5sT+CzRxPlBLRewvy5eqzllytxXIrMUUeexCpihJXTqi4XBobWp1ccFsBy4CT8wakDgXPBWQJFZEVfhpT85rZQ6+4HAwBIDileVQuXFhtITxwkg7mq4RjUNdrnwwdMswpiNdDKO3UIjvjA/Bjxlll9aC2tJUFXSqw0qSftcYa5nVP4Zag4l9/oLEeeUvWXFwoupS7g3UrVqJ4HJx52LFcJw+R8ctQ2+3Pf5L0MPB+O1UbZoaVxa7Y6W/FUHTaeWanKlw234kONPRLLoiuOrAbu4SQDrGptRN9S/I2fK3pAw35hotAqLrNUrNFiO1PU1EjyH5bjLOgbRZdQyg2WtJT33GwrSULBKUX3V04yF0d6g0dWaE9DqRSmZiy40ZsFkqlBVyXbC/lJJsT9XJHFiZkei3SfaLKentADZKSUpgoAuCSOB7En+OPWU3ZSxMLTdp+o9CvL1MLoJXQzCzhodtPlosO0hyTNiNNRFzmUlJJQ2HITUVF9Thc2ipaLrAOnc51chIOgL0waDCLy2aV4ip2WtlK5XzXkE6igKeV8pBskrJAuOe4GNpJ6OdJWnmS30+y6h2Mrca0wmwpsq4JHHF+33we10f6XMatvp7l+63FuqP4c3da121KPHJNgOfQAdhjoU8rYGZLXG1uVljfCxxWI6EGZEKNUY8+Ohch5EfY1tqcirbNrKe1HWNHoRcAAW4sJApwrWEBLY1XJAfQri5597XuBxf7cY2HH6WdNYjCo0TIGX2Gl31Ibp7SErv3uAOf1wNPTLp40hDbeRqClLY0pCYDQAHt27Y7GF4vDhosGG/kfyXLr8NdW/b+ixzpSrlSkJPqDgJDXbeRf/dDGzR05yCDxkuh/wD+va/7sGDIOSBwnJ1Et/xe1/3Y7v8AxjENoj8wuKeF3n7Y+RWLHVsso1uvNtpva5P9wHJ/IYRyY78xha2ZbcdIICmy8G1OaDfzKGrQCB2AufUjtjX2dUdKsh0R7MuZMuUNiNHGhAFPaLjjh7NoFuVH2/MmwBOKJc+J/ILSNkdGaZpHYbsfSefs3bHnsY9peF0DhDWHITy/2FwvT4B7M8ZxcGfD2doBpfQC/wASAbc7Xsqv34kNhDLz0Ztu4Q2pp0BS0agq6PRC7q0kAc2472wIN0qdGjKkym1+GsBcoBTHbctuBtZIBUACTbubCwHNmH4qsjgKQekFKCVAagXWvNbt/iseR8U2RUqUtrpDRkqcN1HW1cn3Nmvtjzh9qOA3JZILeh/ResHsj4nIBdAf6mf+SpmVPp8ysvPTNDFRiMrMhKnGUtFtY3GwXQToQACbBZP5XAw3zZtPpIW0aztSgC5ZucWN9YsVDTrDpIaufLYH982AGL6X8UWSFPF//wBE1D3VApK1ON3INri+19h/DBg+KzLCrkdM6GCRY3kJ5/8AsWOM/jvA5HX7bXXkf0XTZ7OOJGtsaY/Nv6rP1arsOTBYgiqbbcfWI6XJMd7ZJ/xjetR+XwAV83JIubjCWFU2W8vPIp0+I/uPbqEqkiM84dtXe1zoADZ55KwmxsbY0K58VlBQlDJ6V0DbTygCSCE+vbZ4wJHxY0JalOHpjQwoDgmQL/8AVYwN4zwZrjap+h/RaD7OuIXDWmP9Tf1WdFzsvNtx24siFDquhaN2mbKZBbRdtTZf9b27FaAbuL9sJHKrCbYkZadr0SOl5lAdZcedcaeaACiyW1nzltxShYfKGq9saUV8VNG06munmXze907nv3/xfri4+lGaq71FhjMVTyfSqZSFtbcN1JK3JAv3SCkWaHv6ntxzjp4bxBQYi/soJMzv9JXMxXhLFMFg94rI8jb8yNT0Avr+i+dqc2N1CpN/+uoclO6Q820+HADxY9yV2JSb9r39rYvH4Xs4Ck9VcvyEOgR6wy7RpOmxFyLtf/ZGmu3+2HFw/FrlKIigUCuwGqbFiRZrkRxlEVIWtx5PlUCBayQ2u4PvjNMJ56iTGalS9DMiM+3KZUluwS62QUkgWvyBjv5AW5wvMZhew2X0oCUqAJAP54g3Wnpyrqr04q+SGaoumSZoacjTEAlTD7TqXUKFue6LG3NibYT9Fepq+qWUVV5+KiNLYlOR5DKF3QhXcWPtYjvziwyARY4QG+qXbZfPfrz08j/Dq3R6JRJaavTK6DFTHXIAUy7u3LRCyShsh0uF7WHLpI51kiMxKFUnIMmpRYcekuRCtlxcuE5sU9txA3kRmAouuuvbdg9y4SoXToSBj6UNx47KdLLSWx7IGkfywMpHfBiDIpO1A1+iLu+3Idl8xM61tOW8hIyvlHLuaacp/wDqUN56gKh7iCSLRS4ooDytSigugLJ1G3phBLyxT41OjU1ECQqRHa+aECRoWfln5rywoOOJNtw9nHEjtptj6krSLWFz9r4R1GowKNBfqdRltRYsdBdeddXpQ2kckknsMJih9/s95tb6D9FKJjacdkwEk/X99AvlvPq0rMbTbNFjywKWyYLaAh0uyi2tKdAeLQG2pBCQ4UXN1EgEXPJuXM01qJBZqLb7jEeOxeO3TZKm4Tra7BbTkmOXAWwEbbm4Oe4NrnV2cPjBnMVx6PkujwH6Y35G3pm5uun1XpSoaUn0B5tybXsGf/6snO6V2XQ6Hb323f8A75j5zNxJw/TvMbp3EsO+Un6819Ip/Z9xHNE2UU4AOti8A/Ecj5LOOXemdaTHEJOWKrSnGPENbrsB8q2ykJDiXG0BCL3JbUA5aw/Znusg0DNMRpnLsnLOYY6o7hUyTSpYBYW1tlY0NhCyo2KyF/4vg9jjQQ+MnOpP+s1Dv/vDv/3zHF/GVncA2o1FH/IO/wD3zC/8XYE7eZ39BV49nXEl/wDAb/WFQtKy/WHYD6Gcp5jkRKewslf4XLSqQLnSGxtAkXT2USskkk2OEWZ6DnqqUSJsdM851BtclqsR5UOiT0yWFsNbrTmksD9+40p85Bt9saEV8ZGfj9FJoY/OM6f/AOLgs/GRn8//AKroYH/BXf8A77gO4s4efvK7+lR3s54lf/7Lf6wsqO5M6kpnP7nSjqg49JebabitZVnMxzuq4kNrQyQA2hLSFtkgWAvcgkOWc6X1Jp1VE+sZardJc/EQyuRUqTLjtTuNRCHHmAhw2CgSleshN7aBfGmz8YXUBTWpun0S/t4Rz/77ioviP+IGqZ+omW6HnXRGiJq7sxtFMZU2pS2Y/wBZUpSvp3rgAea5GKm4vgdfJ2VI9zpDt3dPiuRiXAON4VSvq6ljQxu/eudwNAqoTmJcWiMzW9bcnxQml19xToSGC6pDzji9IstwoC7gItbygAYY8hyXGVrXCchrUlltmoTpDz0eTOmSw5osUJ0Osx2gG0NKsi8oud+yamZggVmFGmS5D1RamT49E/B2F8uTH3nEhaivgm6QkgJAPI9SQti0NyfIay+THl/hiS1VJCIiVIYlr3i494u/De5Zpx0fKJSj5lxjZStdE0l+gC8HK77KVdYqzIrvS/PVbmbCyYCpIcjSNmO7IKNouJabcCHClAJJNzyOL2GIV0BWzHoNSbkslv8A9ZSAFqRZJtCZsL+vA7D7YeupdWrCOk2cqUqK/T4y6ZT6W6wt1yO84tcxtp51Ld1AMurTqbSbahyvgA4SdHWDCp1XgBEOS21W3+CbuXb8O2QnuNBBsT7gY9ZQTPfTvdIbm6vpImtd3Vo/JlSajZtp0dR0yagjxMVr1sHnFbn2bFgL9r2HfFPdUMo1euNOR5FEGVaXHzcZKqjNZCyqHLnPNtOpdA3Vo+bvLbUkkBtrTbUm1hdJnXlZxmVCW8t9l1ttpixPDceWpkhN/RTm4q3b9TimWfiBzbkLM+bqC+7EqCKXmCoCl0wz7w3nXp6lKSoRRuSFJASsB10bS0p7bZQe1gklMyZwqNiEmNMkexhh3BUHzLmvP1fpdINXokZms5cgyEy6m+sLfqbjj7my03otoc3GiCTdxwtWcKbkLr6p5jrNVcXLfbbVUWW7vTBuKcUt1w3uVqvZRJGlI4sb+pxYWZqZm52NHz0xnajy1ZpqcyHl5uGh5qbOjeJeafqgik64qLqWnzFa3C9IJ4VrEVpmcKLliFT6ZUp4cioeZdnU1xI8RCu2+C4lRbcQtq76Xtoltd29spv58X4hBndmaDbfXosNO8NbYp9rdPoUiPHfhy1xZTaP6yy6HHCAfVIsCvtfg+tu4xAFUiNTXUqVW0SUuuOFe0CUpb4svUfVXmNvS3fFoPKjUp1+m1hb7dUbZbceEptKUJcJBBAWNazcEggIBFu57R6sw6Ew41VJkzxa5DzslxD7TkcoK+R8sWABJuALDgDkXx51j+RW6yj8J1t1zwJiLJbJcb0EhJAP7pA54vcDvh9l15LDFNl06SWXmWy0slzw7m5rV3FiCiyuQTc2PI4Ai1Qaps11lmkz5bx+uym9QS5c6SkI/Q8C/GFNRjz4bTMyTdEh9TrL2s7SlFA1KKkE8BQKQDwDz64sID0E8VTMtazM5HkOqdKI0WSzscJdcaRHUpTSlixdbVt2t6XuLGxw2U7K9JzTAaYTXtyqyJZagRJLqk/1ewN3HNJR6lJUbWIQbG/C1ukh2A1U/DWc23NnS58tQcBSFlR5HlUqxHrbj1wzocdZnR5LUJxXh3te002bOHi3yxe/ISSPWyu3fDsA2CFkKAXqXRpMqJKqMPNCJ8RtwqdSlpEQhSlG4BJd1aWyCdA5tqJsFFEYgNMRkSaVLqUKJfdbafSwDcpUD9JuLE8H39O+LVMjJMmiN0qLAkGRu+Jqa0KDkph9B3GWbuKSsI3Dyoa9vsoHEeQ7SK+/G8fT1vpjtrZeY3tLmixDh1IG24E8Wv8A2uBccVma4IcEU5RW6LTKNTo1ZjIK5Gt/xLK1XMSxSphSrltDaTpVcH954HvYM0uj0+vuTadl5MiOIy3Ykt5p0OJbbJ8jpIPKHUAa+VHzfkcNVbdbpFXVUqTml5Co7brLcRTyU7QJJ2ylFihBR2B1k3JJPbD9T6/kigwUVIPLmz6rtiPGbi/LabG2lzWUKDaw0U3upCVg24OoWpINrhRR6l1BFBKXKRdf9YQWlKQFqZcbuAWr8oH1JuD7jntiNZlqDj7UimxIbikoUFjSi4Ubeg9OOfyH2xYGY2odXDNeiVWS/JisndShoKD9hcEKH9g+fj8+OcV+9S63UHkt0eCJz61IDbe5tt6D3Oo8It35PocXs2BKG6jcGVOgyY8DZW020saG0CyghZvce3e9ziwMnrogjzqdWXYhD6C22uQkN7bi1ktNqJHnbsOb3HKhxwcNMSvZwypNkUqWuPHW025CcYShqT8y5G1yD5TyCPY249AuZjo8GgVJyqMOGuLeb8A3HUTGOhStx5x03+6dPY/mMXlrniyKm1PiUjMEReqgVSRIDhclNut6dAttJCbJUu1wR7AgINgQcJqyt6Asv06giBT4bgjbaoLr70Vd7AtOLCQTcEkjtdN79yh6RSMzdQMxyHWs2twChBU0FHSGyAQGmwbDXym1z5hc3FsSvOKKvT9yNmOY/XpygI1PlPuPxWmI4KbXacQAsuXIWErK0rHsbnO8dm611FDKvJrFY2ZmYvESENJbERM1APylk2QlXCy3wbFXufYYfInWSpwaDCy+yxAjQG/EU8Mpig7LDh7OWVd211EFVyeBexOK9r06rxitLMiGttxIkf1UhaQ5dSVJN/OCLG47YaosmAxIDgJSgIBXumwSsC5AsPccY1NblCRzlOZv4rVmBTzVIjzj8gkIKjrdaQgBsnWVBocghoC4Jvzc4Ut5fRpbaly0IXIR8xCnA9oF/l6gUhA7Kum5JB9O4jdDq0WqPJEKSGVoJuqSCpKEHk2tc/qO57YFKq0yo1DbbQ0guEB9dgEqXoAClm1iSPte/wB8NrmUa5TqHDhyZKaJApzFSkzpLfhdbqFKUSk7jWkkBYUQnssL48lz21HTHHmaJTWy+UlFPjtqGq4uhGg8/p64yBlximvpaqVVlPsL8UhgoQ0pLYBHClufuHWPT3+2NhRaxOzFSaRWqjJjTJEqntBx+MmyHFNXaBtYXVZsaj6m59Tf577SGRuoIXu3D/oQV9o9jchFdURHYtB+RCPLzzhuJCrfnju+skJS8TgtLNwQRpt9sWX0W6OT+o1cShxtxilxlgy5Vvp9m0+6z/Ic4+R4dQzYlUMp6dtyXL7nieI0mEUrquqdZjU+fD90VqHUCqJr1cS63QILgvcafFLHO2k+39o/p3xtNtuLTYgQkIZjx2/bSlCQO/2AAwnodGpmXaXFo9JitxokRsNtNo7JSP8Az39cRjqDX5LbLmV4zTsZ+pMjZmF1CW7ahuI5PBt5f88Y/Q/D+AQcPUfYx6uPiPU/oF+UeK+KKjiuv7Z+kY8Leg6nzO5UwiVKnz2w9BmMvIPq2sHCm6e4xlal5WzZJzFMbosaoJk+I0FCCpot24uo8AD7nF1UymVfI+WZ+Yc15nl1BUCI7KUxuWZaCElVgfqUeO59+2PQMu4a6LzT42t5rKXxHZxVPzvnKsR3Lop+ijxj3AKLMq/+yKe/hiI9Nun9a6iVxeXqC9DZdbiuS1uy1KDQbRp/sAm51C3H8MSCZlii1zp/XMwZnp/jZJIeb1SHWS5JWvuS2QSdbpOJ/wDCHRgxVMxVDSVCNBjQ0qUbnzrvyfU2aHPrjVH/ACwVU/urQeXKSrL2XKVl8zFyjTobUQvqHLpQgDV/59Le2F6wSkqwYU2GojA0hOoEi9iDb3+2Hb3lRdNrMxDj7kdGsFHuggH3sexwz536iZR6b0B7Mmc62xTIDRCAtd1KdcPZtttF1uLPskE+vbBWUcmZgy5XMxVCq1+LLp895sUqDFieHTGZSCSp7my3iTYqHcJueTxh746Y+dW+scKqVOHUV5Y/C48Ojyto+GZlkuqktahwh1YCDzYqQmwJscaoYg8uPID5pmtuVunp31Ey51MyzHzblaYZMGQ66ynWkoWlxpZQpKkHlBBH8CD64gnxO9ZZvR/IzE6gqimt1eWiJARIVcIFip17b7rSkDn0usC+Mq9LPiYPSXo5EydlGmJfzDNnz5cidKF40JCnfLpR/jXNCU97JHrqI04p3MWbK/nuvyMwZrrc2pzHAUmTJd1rI9E37BF+dCAAOLDjFRqY4piW6gfL+69TDwlXSMiktYy+BvM23d5D11PJX3Sfjg6mQ6A9EcpNHlVLXzPeSvSlP2YBAJvbkqI+2Kvzh136o5/dcazDnyrPsuW/qzL3hmAn222tKf4g4riMbrcYKrlSVA/mMDZ0pKnCnkJ068crEJZJqjU7220HyX2b2f4fhMHD8mJSwAzR5rm2umvP8kKRJCnlOgBOpV7jgn/vx4rSG990Fd+EAq7/AJ/lhEpepS1KA+on3wZI1qLaAbJShKv+dzjRVPMUYa06rxvs6weDiPHZZ60XY3vWO1ydNPJGvFLRQ/GulK+QDzgyXLUphtpDurWrWu38hghKlKjvJAOltYAJ/LBcVAce1H6UeY4MLmyxCR+4TcT0VVguNzYJhukdRl06X1/fkl1MmPNSC7DfcZeaQVIdQopKPyIIIxIsxdVOoma6C3lKt51rlQorSg4mHJlKcQpQ7KXruVW9ASUDggXxEZAU3KcDY/aHgD745IcLX9VSDqI+Yof9HAdU9gfeGHU7eSHDWDVnEJ/4eLA1kLiZH2F99APW1/xVjdGqZ0Om5hU31qqdaQyCDH2Qfw/7+I2vn/qLot398bWyl8P/AEJjVSk9RMi5VpKFxW3DEmUt/VDkawAHFISS246kA6F8KTqV74+cTKll8lKPlNDld/3v7OJr056o566YVgVfI+YHYF1hciJbXFlJHo8yfKrt38qx6LGLYsZq5Rnm8PUb/wB10uKuAsJhq2UGH1GSocy4YdQbdSNWk8rqw/i96nzM9dVF5Ioq3ZFKyW9+FRYrBv4usOaUvqt6kFSY6Pazv9o4ecx/Bbm6n9OjmSj5hcqeaYscy5VE2EBiVYanGY7o84dAvo1XQ4QUeTUDiEfDBSYWaviBpE3Mc6MVpdnVhtD7oT4yoLWpSUpv9StTrjgTe5025tj6OIYKEBTYKSkgg+oPcfzx3ppImMaGEH8h08r7lfGZ4nU+juaxV8CfVdMWsSemsucFU2rNCoUcK40u2u42kempFl29wca4ztmFvLFLFQeebaSpZTdxLigAElSjZtKjZLaXFGw4CSfTGAurtGmdB/iYmVDLbPh4zE9jNdKabO0hUeSVKdjdjZsPNvt9uEEYZus3XnqD1siroOdak1Fozj7b0Wk0VKoxaeRfQ428AqQp0AkXvYhRGmx4vlo21j2uB1tr1NtvmN/RZ3xl2o3WkOrfxt5XyVEXQ8gsRsy5oCAHl7hNMp6ykH5jg5dXZQIbb491C+Me1rOnUXrNnymycw1mpZmzFMmx2obAbUvYCnm+GGEeRlsepHp3JxZHS/4Oc+5/lfimYYLmTqC55koULzXE37JaXfaB93OT3scbDyr026adAcoVGs0agojRaTEcqM59PzJUhDCS4rU4eVk27dr29saZoqWii7Np7x5DU/E7AeQV0bWMd318zGnzHcUQ88l1h9xNwedaHVf92H/L+UqhmlLlTkTVxYi3FBcp1JccfXfnQD3I/tE2/PCmnQ5OccwpfqaEtmovuSpRQL7bZJUUi3tcDFoNssx4qWIrADLHkSgdkJHpj4lxtxS/A2NhpwBM4b2vYL7B7PuCocdfJV1hJpmusG7Zj1J+Xzsoiel2WFsCO3VKul5VkokqcQbFXF9Gm36ffEPy1VpOXM0NoYnOJeRIMB8tLKUuth3bUODcouL2+w9sWjmKrf0cpqX2k65zyf6kyDzz/j1eyEd/uQB74gOVKH+JVpptbJU2yrfeWfsfX7rOMHDVfiEuGVNZiryY7aX+pvzGoC63F2G4VDjVFRYLEBPmGa33biwPnoSfqrIc2Gi4sEKK1qUrSB5iT3+5OL++HH4enM6qYzvnCMtNAQvXEjOC34gQe592Lj/lO30X1J/h9+H5fUCW1nDNkZaMusua22iNKqi4D2Hs0DwVevKR6nDvnP4+cvUHNRy5kHJwqFAp0kxH6u68G23Q2dKjGZTytsabBaim/dIIsTg4G4GkxSRtfVMs37DSd/M/kNz6Ld7RPaB7qHYNhLu/s93QbZW+fU8tt9tgMttMIS22gJSkAAAWAHoMHEp9sRjp51AofUbLcbMlCfS4xIbDl0m4/Q+ovfEp5PIx9XfC6FxjeLEcui/PBvfUpI9BgvyG5b0Rpx5o3bcKBqR+R74U3Hrf9cGY9hLXQsi7pPvjvktjur7Y7c+2IBZGyJugeht2xHc856y70/y9KzLmSZ4eHHslKQNS3nD9LaE/vKV6D9TYAnAs853oHT+gScw5km7EZmyW0pGp150/S22n95Z9B+psATjBPVbqrmTqzmM1WprMaJFK0QaehV24zZ7/AO6Wf3levYWAx5TibiaLAYg1usrth5dT0tyHNe54J4JqOKqnO/uwN8Tv/wBW+fU7BEdWurlf6r141aqamIrBUiDASrU3EbPf/dLVxqVbnsLJGIMpwBIS4kk+/tg47oJCEp57qvgIQEjzHV74+C1ldLXSmolN3Hc9V+scOoKXC6ZtJSMDGN2A/f46lEKWgXCknAd5sdknCoML+q3HvqwHwxV/Z/52Mov0W8PbzSfdIN9B82BbqQPoODxHXbgY8poj6rD9MU5ipnaUmSQ6i5RcewwJJDh0JbP2wpMdQ0uJBAX2NuFH1xoD4efhuezg7HznniNpoFw7FhquFVC3ZSvZj7fv/wC5+vt4ThdRjE7aenFyPk0dSuJj3EVDw7RurKx1gNhzcegHM/spL8PXw8vZ4WxnLOkNacvDzRYpSQqom/c+zH/T9OLlWzGGGI7KWGou00hIQhKUgAJAsAAOwAwrjx247SGGW0IQgBISgWAH2Hpg02te4BGPvmCYHT4HTCGDU8z1/fRfkfijimt4prPeak2aPC3k0fr1PP0Wbfi8qRTByrR0oWlDsmXPWB7ttpbH/WnGZ5Gkt97DgY1f8XGXlzMsUXM7KSr8LnGK+AL2akCwP/xENj/OxlV5slCk3ubY9NG0dmvLl3eVvfCRnlmhZ6nZPmPaWq4xutAn/Htfb7gnGhs2ddunOUkutv1rxsprhUaEgurCvZVuEn88ZByDU36WxNlUpCEVKnPRq8wpPCnAwbONEjkiyu3+Ucayzx0pyb1iosXMkZw0+oTIrb8WpsIBUtsjUhDyOzqLHseR6EYpGpVuUCxIUNyx8SlQzrmr8DpGWFwoxIPiHQHyy1ZWuQ6kLTZCSEAgX+u5tbF7xTJEZAlOJcd0DWpIAClethjG2XOn2dMiZxVXGpFGiT6NJkMRxM3Sl82Und0ossNm/FyCsX9PPjQuRc4SadkWXmnqHmWCosOXkvsANx2iltOpppIudIWSEi5UePU2xU57Yrl50Hy/2VhZ2xDYhc9Bz8lO61W6VlymyaxWag1DhQ0F1151VkoT/wCeAO5PAxiLrh15qXVScqi0nfh5cZWC2weFySOzrov+qUdh3Nz251t6y1nqhPMEEw8vx3CqJECvMsjs676FXsnskcC5ucVU7HSgfUAfe+Pi3F/Grq4mjoT/ACubvvfDkOn1X6I9n/s/jwhrcRxMXn+y3cM8/N34ckRpsfpOOhP+Tf8APBxSEjlxOOApsTqTj55ZgX17NdEgLvwjBiW3P7GO7gPcjBqFix84xnDtUC4ooJcv9OB6V/7Vgdkj/GoGB6mwLh5ODn6pC4otDVlHWgXxBOrUxymU+m1iK9JZlwWpZZ0htLDTkhTTTTjzizwNbfCRyrnkC9706Y9LK/1RrSabSUlqLHIVNmuIu1HQe3+6WfRHr3NhziH/ABwxKB0rzrAyvl2kU6VGoWUI0x9VRAUVOPzpJU4tdxdxWykhJGjy24AAx7rhLA61x/iWW0befN1+npzK+Re0/imkhoH4Ox15X2v0aAQdfM8gNVUGUHpdAzY1RajT6i8600xPMp0pTNDZBDpKwkXG9uNgp8h1XAHfDKilVR6HGk5Xhxmp2Y4zVDpDMpt+auS84W0tNu7ccWcO2SVJsjcLlyDyFU6oUGPWKntMvlptMaE86JcZ5MwLBS0dwKJC9DxRZRFr6+AkHD3SX34s2k0CVHefka59DjSEVBTcJNRcZP7Zte2t4pbCBuHU2NV1gKKSfoMMk7X3cOl1+cHAON01/EFFpUHJNUy9Tq2iS3CrNPQ4W5Lllbr5RvbC0trAU82+QLEhFr9zhu6VMRoEWtPUx3RBclvqS2V3cbWExX1NpNu3a54J9++IjmmnqpfSh+WqmoD1XrNDYURVUTG4saI266lKSEixUbgq1qX8u1yFcPfQuoPlzMMFLKSmPLYcS4W9RU2WGUqAvwfp5B59cekpm5ackdVqo2rUPTaGwMyQ4rojoi+FcioKQA2w2JDLgQq3a2u+o9xb0xk3rFl11E2oZoXKp78SsV7MCGJrU9l5aZMeW+p1ksoJWLcW4AcautF1g31VlBGxW2YKfPuRnGXFf7ZoLab9ubg9/a+Ms/EdnhuozZNLSiE1NyXXKvSvxXwrUeU/CJbSwkSRy4dYk9jZtCUJNyq4oGcSNy/FbKsjIqyVn3McCnR8vw5qCz+ILrDS24De+1LdY2S408W924QOEp8gPIueMPqItMzs1Nn51gSoicp0KOmPDj3YDsOM42y0wloAOpc0KKnCSpxy5c4AxAW6tNlIjTStuRI3dcdalLQXXCtNnFOH9oBYAIFvq9b4nOU8uU/OkpFNzHIryMyLmGXK35qoMYONObbkTS8HFhYBbGsoDaSoN6TcHHTmnldHkc4geRXIiYL7Kc1CehotVOpbT6KooRokhh9ooLiAkKF/MGnAQo6LJ4Av6YgNfbYeqCpeqIoOa0uoYeUPmEkgKStRIAFhx+VhbDj/AExreWa2614OnLacYG6w9TmFbzYAKvoAN7nQTe4IN72xGK/U4NaS403HOh0LWGHXUFxlZWTpS4ANaOex7+/OMTIyCrnpNB0Q7y1BbCi6PlEhTfBvyP5/bEvYo9KzHNdaTUTHXtIcsUa2txfkBJQOEX5Om5tfFYsvvoJZC16ACgpV2GH6JU5SWVrUpxprabaKWnFjd8wPNr37e1uMXPYbaJVYmZKZKobIy61LaS2psxiNHCHb/LDZ48nfji974QUONSI1GcXVoolSqnWoTLLTjThaWiOl9TzSiOQ46XG22wCi/NyeRgpeZHXGYsOqMRagYjrrbbvawdLYCxxYrTZQvcC3J7YPy7metZarwqdFXHlg623Qptb4WSCCUpRwuxBI/l3vikZmBRKBSJVOZg0yJKfp0lZkojzJS27ruobbfn4cQQ55Da4soHnjDK/BrkOZVG6nJjqbhXjSHkzm4scaE2ClJBBcBCSSEoUvm9sS3NWc6nmB1iW+ww8xElFptsp2wCRwq9roPc6RYL49RiAZxq8mTHm1CNG0uPRwFBprSmO4VhsL3LjWT6GwsRzfEjJee8okFSqyn4iHvwtc5ppYMhTTW9ySRdTgT5LrNh+XvfBmXZz0Oa4winRygB2oNsTmtTbbWjadte19IJHPn4HY2xEXYymJppi5jjp20Ny1R+wXwSNJtrCSe3b1B9cP9AXOZQxEqMlKkzmXS3MUkuNPGwBbbc5O4o6GiFWIWQTbTrxrbG0aIXS5D8xypLRF8chplwx4LDDxUEsOKPksjsG/qDYuHNPJtckMSoyMk5qTHqaWHpSXA4lt9kqRYX+YrQeCDYADseeww0vVCqQ5TzEmAlT8d5xqO1GkamiQqxbSo3JKTY/2+PQ4e86ORk+DlsIW+yuAwh55DulN7XSUjTex1G57/Txhy0bKFRms02XPlF7LdKmrkVB1bjhWvVdywU8Ab9xrTxe9jfnDxJpU85fplLENbc9cZyniO842kJO6pxJKjwBdwkuE27jgJvhuplWrMinoiwa1Lhr3UIRtvqSnR9SEpIHF3OQe/bjD91EhVig1Vui1aS74uBFhRXH1yULNgyl5JUGyQixcNkAnSRydd8O3RRuySUiC9kSbCiUp6HPqNRIDNQjPJcjuIBF2mSBrJCwQVKsHPLpGi61zysZzp+fYgjqWY7jiHW3mZBO02Sps8XI0LKwbpHBAQfTFG06oCLUjJZnPQ5q/mNPsDTdzSdIVyPJcAHvxewxM8yyKjmuFGmO0WWhuLCbjreILrLy0EG4bNtv5es2F/oH6USszOBShy7VqbTdx9dRqy4lRjuBrZXHLSSLkBsk35vxfsL4g82qllZhoUgsgLQHLcOC97m3F+cSpuismmQ5tMrcJ6Sltwyg1I0ulAJ8hve5sR7dh374Y4cTwtSEyWlp9KVjS0toHSR627G3t64uBslckrK/Cr1wy60zfSNxAX+Q4AH6fl98SWLVXlJLchsreU7qecSOObWc/X2HbCqqzEwaKh6ms0xyDUHgFojk7sbyctOskW55uQeww1UiYy5EWw3H0bjuvWny7aLHy29PSxwzCHoeFSun1R9Ta4K3nFtSXUOFsOGxWO3Ha/JxrPplPMrp3Slw1kobcksABtaUgh1RISFkngn8j6cYxjAedckBgvhKgPI6fLc24JI/vxtH4UaDW+o2UaflqnI8RUVVeSkqA8oStDbilq44SNRuceH49pJKqgYyJt3FwHzX1b2TV0VFikrpnWbkJN9gAQblWx0u6c5g6mZjjUimpU0yPPMklN0x2/wC0fc+gHqftfG98o5Vo+S6HFy/RI2zFiosL8qWfVSj6qJ5Jw19OOn1G6c5cj0Glt3WfPJkKHnfc9VH/AED0H64lybEDSeMb+E+GI8Bp87/8V2/kOgXK464yk4nquzh0gZfKOv8AmP5DkPMoflxXPXNyNAyDLrpKfF0xxt2KD/jHFLDe3/nBf8r+mLGFvTDPmTK9CzbTzSsxU5qdDK0OFhwnSVJNwTb2x7Btr67Lwnos+9PPiDFPkNUmqxpElpatOhgby0n3SkXX+lsTb4gs4x//AEPv+D8Q2cwPs05sOsrZWErOt26FgEfLQsfri06Rl+i0BgRqJSYcBqwGiMwloce+kc4z38TtRNXzplnKDSriGw5OeA/turDbd/8ANQ5/HFndc7uhNfMquzcfw/JNHonlC5ji5joHsgHTf/PcH/Nxa3wrQ9rKFZqFv9VVQNXt6Nso/wBLpxTufpSJeaXqe0ryU6O1EHsF21K/m5/LGhfh7geC6U0x61vGSZcn8wXdI/k3gtN7oStVjL7YGjSBzguxPA7Y6DeyQm6jwBi1ndVBH3UOa621GW886hpttJUpSlAJSALkknsAOST2x8+fir+IWl9U1ycg5NDbmXoT7Uxycpv5lQfZXdtbd/2bKTco/fWeeEWCph8YfxGirSZXSPJk8fhkZWxXZbSuJbw7xUkf4pJ+s/vKFuwVfJrJEp9SyhStYUm6U6r4wzYg5rwyM6c1914M9mgmwqXFcSYcz2Hsx0uNHEdeg5DVdQr+oWQbWWtv/T/pwS2pLJbjg3I86sEtLWkCM6qxS53+/bAX3dT7sgXCkeW6sV1QEbcjT4tfgu77NoW4zUGrlGtOwRDyKNeUWJSXkCyeFi38/wCf9+D3LtNKGnyvL1D/ACU4RqSpcFKlFOtldv8ANP8A44E++XYbCCfMkEcfni+JgqGsk6LzuO1TuF6rE8IG01nN+J1t9USoqRp1XAIv+Ywr+U80hxbwQUJSlQP1HT7YLqDY2GFJ4206DglMUqQ274hoJV79x+YxbKYpo8zzYLz/AA5LjfC+KOiw6HO9zRp5Gxv8Ect0SHBFjNWbb5A/04GwFgLQlSkk4E2TGIZQy4nUhWpS06ScFNfMeXb+yrCNeySN7WbBdmpw3EcHxujxDFJLzSuueYGtrfJdkL/rLhT/AJJT/ox14MvJLjjukOd0hPOAqbS4C0eFpCtH3/yTgplkvagtZQQL2HNxitjYp4hn2anldjnB2PTQYc0ONSbtvz1J+Y2KMbWlZTGjNaUJPkQOfNhTFXdDoSbkJVYe/fCRLyUuLaaRthPcn6tWBF9CimUyq4Vqv5f3vUYjJYp3dgwWHJLi/D2M4C0cR17+1lJPaeQcCBr5X1RilONRllh3bWnSpBtchSeQfz4xsH4Z/i7emPRMg9Xashb6ghmFXpCwCVHhDUtR9+Ah/wDIOeijjwhS47iWm0qJ4spVsAYjNRSpxKitS+CL8fr/AGk/b1wr3StqD2QXTw+nwCr4PbFi7wx93Fh0Lwb8udvI6Fa2+PbLDqKnkzPSWVBF5mXZXl5BWkSWL/q2+n9f41Z8PnVSj9M8ztqqeSKdVTWZ0aJ+Kqd2pUJtaktnSSkgt3INgUG98VnXsy5rzUICcwV+rVZukRvDU9mTIU4mGygfS0g8Djgn67AC9hhoFRatrZkrbWFoKVjgpXe6SD73A/XHraYyPLJIgSLWd6fkviMlO+F1wLs5Ha/mvrmwhotBaeL84rz4ios6R0H6gs018tSf6PS1pWlWkgIAU5z6XbS4P1xD/hk+IqndVKI3l2vyGIua6ayBKZKrCWgWHiG/sSRdP7pv6WxcWb6MzmPKtZy8+EqaqdPkRHE+4W0RjLNFJTTgO01HxHX0WDK5rrOXyey7V6zl2oKqMGKh0KS42UOoNtsquBwbji3a+JE91IzSW1IplGhwFqF93Qp5afy1+QfwOIDBr6zAiqdXpUWgSP8Auw70rMdMZcT+JwnJTY4sl9Td/wCHfHksTwPDq2b3ueDtJB+/IfkvaYHxHi9HGcPpqnso3HX9b2JHw1RkWDmOvVBSv6zLkvKBdddUVE/dRP8Ad+gxrz4YPhrezS01Vq9rTlxl0rkLvpXVHwbFtJHZsdlKHuUp5uRTHTTqb0VTmCGxn9+VRqIhWpyPFjFapB7BBUjlKTfzK72FhYm43t02+JvoLnKW3lHp1mOPMkw4iVop8SKpBaaQLBKEEC9uBpTe1x74xNwXEcbaJa2mMdO3Vrd81uZt9kdOa71fiVNwxE5uGTdtVSjvy8mA8m31zHry9drGzTBWxkmq06iMpjKRSpDEVDIDaUEMKCAkDgAcWt2x8V6SxLZoFPcLS0BNPj3J4sQ0m9yPy74+oPxzdTpOQfh9qDVJlORalm6XHy7GeQdK2kSLqkKHqCIzT35EjGd/hV+FibnODk/rRmurRf6PyFOT4uXVQtW+ykqRGLrhVYpNistaOfl82vj2NDD2cIkOjb/gOS+a3LtT+/NRn4J+sdWyT1AZyTVJTiaPW1kNBV7NP6T5R6AH29/zx9MmntbKXjwCL3x8dusGWah8PvXKr5Xp7ziWKJUWqhR1rGrXBdO9H7/WEC7R+7WJJ1m+MDqf1nbYyrDkyKHltQbit0yA4sv1Jzi5fWixcursyjyAEA6jzj1OIYJ/FXRSQuGo1PO3I26lXOizmy+ged/i66GZEqLtFmZyaqE9n9oxT21SA2fZS0+QH7XJxVz3+EU6bPpZchZbrrSHUpXpfZSFp1DkK0kgFPrYn+eMw5E+DHrznKCzPmw4OW4jgCmxUniX7Hm5Zbvb14JGKo6oZYldLc8VPIM+vxqlNpJbblOxGlIaClo1aBq5JAIv+f54vhwDCIiWl+ctFz3vyCfsWNGq+imVvjk6YZjqsalKlSoDkjSG1S2ShBWTbTq7A/ni4q51Ry/lbKsnN1clpbgMnS2W1Ba5Dh+ltsA+ZSvT+JsOcfGhqrJdCm9azxzcdsXRl/qLXs00inUyq1dyUilQI8eGzYNpZaQ0GidIH1naBUo+c3t2GPnvG9fSYHhz6mjZZ97Ab/H+y9Hwfw5HxNibKSR2Ru59Og9evJWV1W6q5o6q5iVWqvqZhMam4EFDhLUVo/8AScNhqX69hYADEGWiUHQo3OrvgsVHQQgAlI4tjyZridSSpRSrucflmrqp66Uz1D7vPiH7/YX6+w/DYcKpmUlIwNY0WA/fPmepXS0+k8H+OA6HwfKrjHDO0jscCFQZtyhX8Mc+zvsLfaQclwtvlOkLVb21YCWJB7r4/PCv8Ui7CUFolQX7c2wSJzRPCT39sEtf1StdJ91c8LJ06UXJ979sAUxLCdIuoA3Nz2GF6KlHaNlsnSoElRTwAO5vjRnw+dBEZg2M8Z5pBFIVZ6nwHm+ZnqHnEnsz/ZSeV9z5bX7GEYVUY1UCmp2afadyA6nzXnuIeJqbhmkNXW7fZHNx6D96blNvw6fDVKzYqNnrqFFP4Kmy6fTnAR423ZbgPZj2T+/3Pl+rYyILraUoQEoSngAdsebmtNNhLTKgE8ABNuMd/EUjsyu/5Y/QGC4LTYJTdjA3U7nmT+9hyX5K4o4nreKqw1VWbAeFo2aOg8+p3KMER7+1/PHPCP8Av/PHPH/+6X/DHPHqsPlLx1e6vOKD9aoESq9P6nlN+WhE+usuxqW0q935jTSpKUJt62YUebDy4w6VbiUvITZDg3B+R5tj6B5srDNLy7VK44yL0yBJlBah9OhpR/8ADHz6aBbZZQ4sFQbSDb3tjRCdCkel2WKsijV6HOeUfDtP7UkA92HBpV/I/wAsas6PVPNM3prVMlZalQEVvLcxyAw5NKw2Iy/Oy55AT2UQP9xjHsgpDhSrgODnnGg/h5zctnOVFmSHrt5hp6qLLUry/wBdi+Zo/mpGsfrit/cermasSCo9KetNfzVJocmAqzbqC7UEPKTBGsatwLWAtwj1ABN+Dh3+Jvp3Gy109y4qkVZyNTaGFRRDLS1plSl8h5RTwlw2cJWq/JsBc41IAn2FsQ7q9lEZ16d1vL6GwuQ7HLsX3D7fnbt7eZIH645mP038Rw6WnaNSNPUahdvhbEP4TjFPVuOgcL+h0P0Pw5L5zrZfUrgXtgBjOEeYXwqTOS2o3ZVzfg+nuP0wIzx32jj8uC7Ceq/Zoe/kEk8Ksf8A/WOhhdj2wYqYpf0t/wAsF76zaw/lg9oQnu87oQjr7EpGBpirPqMA3FKN9N8CDqk6RY/7nTiSXG6U5kZ4NKrhwi/viw+kfRGvdUKnpbKolIjuf1qfpvoH9hu/CnD/AASOT6AuPRTofV+qdQTOml+HluMvS/L7KkEd22fc+6+yfucbcoGX6TlilR6LQ6ezDhRUbbbLSbAD/ST6k8k8nH0Xg/g1+JkV1eLR8v8AN/bz5r5Lx57QxgoOH4c68/M8mf8A9eXLmkeTcn0PJFCj5fy9DTHiRhwE8lavVald1KPqT3x8w/j9q+YJfxLZp/AsxS6a7SKJlqnlMdQBKJDj5Up1NruNjcAKRYguAgjnH1Z0BtPHAGPkj8bWaKwrr71EqeWYcaK9FnQ6e/PlNbQnNNRooXBDwKb8uoISSASSC4DoA+v1gFPTiOIDoByX5wmlkqHmWU5nHU+qqusOPVWaqc5Pfl0eQWH460x1vJekMKKZbbOu1nAWCLhFkrcPKgOS82tClVB2mLmVGmQfCOvtvNlEmTM8S+ncLam1MmMEyFNJDCbuEjng2BlOpcifmKdmWdHqv/q9bi5VJQ1+HzWZgbHipfzr6glBQpwR9fLhWQq5OB02qULPU+HeHR4lOjsyHFVudw6mT8oKjXXqAWrUO4ICBwSCAOA2R8UoaNgNfJZ86Q9XMt07JuTmo0uqVSTUncytRmzJ23ESo7ER4PvBTZPCZDwTbkguaDcpJwb0pd8DVq/DmFkQZL8Blx8E3bXrbS3p/M2ubcD1w09YKfBVQ8qw3XKdOqSag5DU9CJZbiRA2pzYKLAAqW4l0GxsABfDn0TQ7PkZgQl4EyA20nULqU6gxneD6Cw/Ln3x26Ih9EHE9VvpvEFqHJ7i0ZljlbXmbjyXFAEcEONi18ZRz/mCb04+ITOdSq+WWG24lcmOwxIpDbxqOhwqYJMtDmll9Y87jBRwToIvbGqMk1BKq5MU3FbcRDY3XXC4QQXzdKEAf5DZJuf3hbGQ+uTcKJ14z0nMFYE6cjMUwumWt1bLbG6VMMgIUNYs4ELRYaRco7klWWFwRdaazwhMaDTKuxOzFv5HYX4+OFxJcvYLzoG8H9iONbbbZ1BenbbILRd84Aww5Wq9VmVBumQaS4p5Tr7stTq7qKy2fmKdWe/zXL2NvmApuQDhFIlUGJIiRoCVsuSmXUJS0HSHXQvyNlP7ilANjSDcB0FZPo2ZhYlwaN4ykTZDgQhtMyS02vaQHW7JQogktlTiXEgK4O1e9zYWsjzMLVzL5bFXDX8yT8yNv5tpFSXAmxt92oIMsuqc13u4LND9xSgu6+xxXlMynPrcZa4FKrMpTRQnXHp5V53CQ0OSPKSOSB6Ed7YuSl5UreVUVSm50oVPzNkd2pIo71dpzqzTlO+VTbSlAbkdaiqyN0Dzgthz0xftS6M5ef6VVnOFPzrtZkjByqS4aipTVRhRmSGwplY8jgWoq3E6bhKr9gvGCSsZSWvzXRjpHTvIdyXz/qFPQzUHYbclzyawVlKklKx6FJufcevpfA41TMSI0vxSyqUrQNA1Wt3DluUEexGJHVsm1KLX6jEprDcNph4uNtOBdw0s3SARcGw4Fz6YNzxGS/FhuSIbP4u403AM8KcRus24DiQSh06jpvxYd7nHVDg4dVzikLFaXBlGTGmIWkp8iUaFW9yCQbYcYOakRqlGqQZDakuh0NraNlIvyEkWHp6c4iULLtXbrRoywiFPDhbejSkaSSjkoHHcAE/e3BOJFTcnPrrz9Dn1elxGjT5E2CZEoGO/YgWLgB2yTfvbm1+DfAewHVRSCq19D8tdUoeanVSag0+t6EIZbUww4bFovkAOA3NgRcm5vyMIfEZihMqiu08z6EIQeLi4QmBp9ywS2LNnQ5cHggkaee18IqJB8A5euwpcRxUbhx4hQU24AQpPdtwi6VC9wPbnDjTPGJjzIFOzBVGq78xBeLwMZLbZ1B5SW7r3LjuntuD2xUwZNlFEqjlSu5dZ/FXqatiFMQtpBkN6VccOixssW418d1exw/ZJ6c1KtPw566pJoGXHZjX4tXJrZVEab3NO4Gwobi0kDg2c58gItgaKuupwEya9PblARQ0EuNXcCNBSbGxubcEHvcH0wjfqDNNW5Op8ohCGm7NJW4UkhAUTxwsc2v3uDbjFhkchZNaqFMypnyptSUMVCBCkuctocaZm25b4cAcAcHmNwF2JtbDlU4sZTMan0p6dJjuNbMYugh1sAD6CAD2Hr2HBva+JP1NjR057qcymyZKYtTaaltqcf3txC08/MtYtEC6Li9u/IwzqojMgKjM1Fx8tRQsOxgDrKzayh9aBwRY29DexxZnFlCktNp34kIkKmjxMtpxbrqr6d1XJuPcaLI/T0wZXsvVSRIW/OcaajE+dV7qAQPOCnuLe3scKciLTErTNTgwH6jEYdcUl1KbhhxHZTgQQWwRzcEXHa9rYe+o9cqsemqaqtDqjCZBLu8Vq0qQvVpQkOJS5otq4PBHvxip184ska3RUVXpcdEtD1OiMISEi4bR27c2xKMuZ1ppo79FzPHuw6FlmU1r3jcfslnUBo9RwbHt3NwVBjLchyO1Bhh1LjLTRUqPqUyQCbNnUSvmwHrbEfcRPbU1LpELy+ii2NN/SwPGNIYCl2UjpuSahV30NUqK6oSChca44DRBJuO4AAIuRxY45VMuKprzrMaWuWWhpDbbawl1H9sFduCOeOfX8h5dquY6E+BTRKk1WsJXGeShta3A0LkpKRyCq4Pb6L4IkzKpGrLK6lSpLTnkLaJbanEuLAudRPC73vewuLYGVSycYcgvUd+rTvCrlKcAZ3GyXkOAHuLABBHN7/mOcN2xBakpbCHGVFYSA4q9lrNwB/owonVqVVmUbjcCOzDSAtMZkM7nIFykca784cKRDcDS5MO7qFWRJcSQ46yL82bKvmC32NuCO2CGqbI+HDlOvpCQy9oBT5bJIHpyeLi//AH9sfUr/AAVcKkM9Ms0oTGtU2Ku0px1zl3ZcYQUJ+wJSs8d/0x86KTlmdRqwmaxWIEluONbSHWV28OUHSvuQTr0AXP3OPoD/AIMFp6NU8/NFqGymWxTn1NReE6kF1N7HkcH+N8XZb7jZXxPexpaw2vut/wCO4Bto/wDJx0ISO2CghY9j2PYiiAvtjIFaq0bNvWXMWY1gIiw31REK1EjajDRq5HF7KNvvjUWfMxNZRydWsyuKSkU2C9JF/VYQdIH3KrD9cYljvu0Tp1VZb5vLmNiHqJ5cddNlEe/dw/phm6AlWwt1zJjbnO1WW9VHVFTkx1yQb+6ySB+gsP0xsHpZFRB6b5XjM6Qn8KYd47Xcu4f5qxjWkPI5cVwlsavyAF/7hjXHQioO1TojkCpPpKXJGW4DpB7+Zv8A7rYLdNFHxuLDJyU+7i+KN+LLrgOkuQvwqiTdrMuY0Ox4KkH5kRkCz0nT7i4S3/lrB/dVi633mYzS5Et9LLDKFOuurNkobQLqUT7AAn9MfKHrx1Xl9YuodZzY8tSIT6tikNL/AMXAbuGU/msqLh+7h/ycVVc3ZR2G5X0D2X8JjiXFw+pH8mGxf5k7D8z5KIMS1F15SklSAlA7c/8A82NY/Bv0y6XZ9yXmqs53yNRq4/TKkyllydFDq2mfC7iggmyrEpJt98Y4jSlRnSFJ1BQSDjcX+D+WJOTM/wADSABMiEA+yojqf9GMdBYTar9A+1FstNw690Tiw3j1abaZgCsryouWZ3VgwcurX/Req5kQ3T1sJWw4mnSH06NAcBU2pCHCACD9I73wv6ydMK30d6gy8mVl9MtCkCZBnIRoTMhqKwh3SPoUFJLbifRST6KGIU249Dh06RHdUl6PFYW2sdwttA0n+IGNffG7HjZt6XdM+pLTSEPT1uQ1kejMyEl9QP2DjR/icdau7OrgbJlAcD9CvkXBOLS8OcSPwoG8UjnN13uAS03+Buoo78OOUsqfC9Vuo2eZ6ouaa3TESqKVOqCIbrhC4cdDSf2rziUjWVXACuAEpKsZiaccDrSRYhxxCQByLk2tf8z3xtn4+JTlLoPTrLjJ0xhImvBCeBqZjMMo/gl1f8cYhquthEhbSNSmQXm0j1I5GNeGzxiQUbmjKCLnnqvJcSGprYIeIKt+YzPeB0AB7oHloU+PJKJD8F9KkuJUttSVJ5QtCiCD+RCsJ4yUuIVHdHlbOsm/+UL/AOnEm6xKgNdW8yO0deqFUJyKpGVe425cViT5f1dOIpKA1BxtRRucAj2OOVCz+a6lcdifxX0fi7tcJbQ8SUY+yAfQjT9Fsb4oPhkyLlPp4vqB0ty0zSXcuq3qtHjuOLTKguJALulalfMZXoXxa7Zc72GMzdMaJTK/1IodFrUVEiDLl7chhRUA4ix4uggjt3BBxY1f+LWuZl6H1bp7Pia6rUoa6c5LC/2SNTXmF+4U3ui3obehOIL8PyfxDqtl93QSlt91w/YIbP8A349dTYN7rMC4ggnX0tofivkk2NVtayNk8ucxlxab62Ou++nJIeodBjUPqrmfJ2WactLcDMEin02E2srWUlxKWWgVkk/WEC5P1YBV6DK6Y9YJGUM7UeFVGMv1JpqoxloUpiVHWhty6U3BWChw25HIH6WpQ6NFqfx4qp01IKUZ+kSlJPYhpgvp/m2D/m4Q/Gfsr681BbbaEFuiUlDpA/aLLLiyf4KA/JIxwsKkhpHyuewG5sL7Dr9F9A49r55I6GOK+ZsLZS7nrYaH6lSvrF8P/S+qdO3+s/w+yNcOnMLkzqW1IcfZXHQfnKYDvzWXmb3U0SAUhVgkgXoqgQcizOmucZM6rRoGb4cimyKBGU85/XGdSxKZDafIu6Cg3V2sFA8nF4fBPm5tzOdU6bVV5LkLMMTeCHCbLWkbDv6lpaP+aMZrhxnKdJFMJsmPuRF2VwpTai2VfxRjNUtbQVEVU1os7l8bLpcNVs3FmBVuE1crnZAHMdfUt1IaTzsRbr5pxyxSG8xV5mkS8zQKBFdQtTtQn32GgLW1EA9yQMSHP2VMq5N/CY2XOplJzrImR5Ds96mAJZhrQpsNN2Oo3IUs+b+xiGxlluQFo4J4vexxxSkKccQp9a1NHTpUrti/F8QMdO2GJgF9zbXfquV7NOHKDGHvnqjmMR8Frt12P70Q2ZcgzQ3dIb4T9OJB06bgRc+QKdVoiJdIrBMGdHXyl1pfBCv7wQQQeQb4i7SpG8FrZ0pvwdXfDm3JcizI0lmxdjOiQ2TxZYNxh+FsTdR1TmvJym113va3h0FNDTSRta21xp6eSmfVjpVnDoHmSDXaTVZjlFfdDlCzA1wtlwi4jPnkB0dgo+R5H3ujGrvho+Ken9SxFypnF+PDzOyQnSPKicLcut39eeU349LjEn6aScsdYukTVKrcOPVqZNieDmx5CNSXPQoI7gggc8EEAjm2MT9b+h1e6CZ8pyoMuW/QJtQjuUWprX8xsh5JMV5Qt80c2P76PchePaP93rY5KebRzbkfjcdPML4NDC2WVsT97j8VAW002jVRinSS2TFeVHcJSCDZRQTi8KN8PeXcyZQzD1LqWamck5Vy9Ecfm1uQz4qPuosA201cFxVyBZJvrUlABWQE1F1TpzLObqqr8PYK/GyVJLYuLhwgke3OJN1J6q5i6i9Hcl9N5CItIodAS/OmswXVfOfj+SOXEnjyjde0c/MUhRJIGPn1BgVbiXYYq+QxxsALm/ev5cxdewrKymweKqw1sQc/No8/ZA3t59FWlEU3WqgzTqVDn1KpTFrMWIxELsp0C/8AiWtR127gXCe1z3N+/Dn0a6n1Pq9l+pPZSrmXI9DfXJflVKGWCBosWUpWbnVex9u/Jxoj4QPhwf6GZZNZq1Pp8yvZjaakyZ7JKnGY6kAtxkqP+LQDc2sFLubGwONNw6aiQ54hEbS4bXJ7kY9TWVfYzGQd742ANrWA8l4+TuPzXusdf4SmpCHRum0KU+kNuTKvL0+hdbiNpSQPcB1X8TjTnQeJSaL0HyIUS0NQ42Vqc4XXXNKQ34VtalEnsBdVz6Wxnz/CUdNZuYumuU8xIdKGMvV9Xi3dvUW25TBZSfsndS2D+Yxl+f1v6nSOgsP4eW6tHhUOKyuC/UGVOCbMp9yUw1EmzbdzZZTypACOBe9UVOK2kjia8NN7n0ub/FV5bon4pOvOVut/V+ZmnLzTCaFSYYolMlabO1Bpt1SzKV/kKWo7YPZuyzy5YXl/g9cn5EzDKrebZdPjSK5BkhqLIdsrw7BR/ih+4TzrVa54HbGd8ufDdmVzohm/rVUKBGNIp8eOaUt58pdfHimg6+2AD8sJKhzYnm3BvixPgbm1fLnUd2hxVsIjTkEyEhXDZRxwfa/FrY9bJQRMo5W0kmobl9LakX8wtOTKO6dV9Pmo8NloISpGm3YkcYw58ZPwrU52o1LrPlzM9PpzsxxoVCDOuG5btghOwRc7xAtosQbX8ljjW+es4ZV6Y5LmZ1znU3IlPhISSG063nnFkBtppH7ziiQAP42AJxgbqP1nldbc0JlTxOhRWdaKdTiptTcVFuSSk+Z1X7yvtYWSnn5DiXEFbw9TvqqNmd2xGlgOp9F3eD8Eix7E2UtVLkjO99z5N8/PZUzDyDJQkJkKZT7+e4xL6JT4NFaTGbW2t1pGorSPqC3FL/kbjDgqA24fXn744KWyFXLi7Wtb9b4+E41xpiGPMdHVPbY9BZfqLA+B8DwCXt6VpzeZujEyI6eQoX/PHTLjjsRgH4Yx/tiseNMYt+2Vjx4I+8vX/wArqV4SWD3tjniGD+8B+mPfhjANt048aY2PMHTfC5G/e+qN4+qGXYwb0osSf5YBuI2ypSkkAXve1vucGIp0dSrOEpJFzY24xpn4c/heZqpjZ+6hQ3PAp0vU2lvo4keqXnkn93sUoPfgqFrDHbwjBqjGp2wUzf8AUeQHU/vXkvPcR8S4fwzROq6t2v2W83HoPzOwGpRHw6/D1GrQi5+6iRLU0kPU6mvIsZfql11J/wAX6pSfq7njg668fDSkJFrJ4wH8ChElSlukq5J19zjxoEE8Ert/usff8GwanwSm7CAa8zzJ/fyX5C4l4kreJ6x1XVu/0gbNHQfmeaF+Jwx7DHDVIfv/ADx7+j0EfvO/xx4Zeg+7n6nHWsvPrwqsNPt/HHlVeHzjv9HYH+X/AM7HP6O08eb5n/OwMgUUR6tzWZfSvN8dhwIWqiTLH/kjjDT7CwouKsNRxvrOGUo1VynWqXECg9Op0mM2VcgLW0oDj8zj52ZjzfTcsURus1dS7upCWYyDZ2S7a5bTftb1V2SOTyQDZG9sUbnvOgTxxPnkEUYuTshV2o0+jsRpNUmNR0vym4req93VrISEJA7/AFAk+g74luQ6vMhKmxIS1pnQnWqzAPtIjkaxb7pt/wA04yXmzMVYzdNfn1GSN9SS2wlq4bio9ENj0sbEnuTyftfuUs3o/wDUOdW/N4hMeS4hAJKtwWeSABcm5c4+2OdBXtq3uyDZd+vwY4XFE55uTe/5L6a5arMPMlBgZgp5Bj1GM3Jasb2C0g2/Tt+mHJXmSQe2Kb+GqvpdyzUsmuOal5fmHw1+Lw37uNH8r7g/IDFyJ9sbwbi4Xn3tyuIXzy68ZVbyT1UrdK07cSQ746KkCwLT11WH5K1p/TFeF9sBKyeTxjWnxnZEaqFOo2eWGRuQXDT5Kh/ta/M2T+SwR/n4yk5FZJ7WGPzjxbh/8MxaSEjRxuPQ6/TZfr3gTGWYzgVPO43e0ZXerdPqLH4ohUpo8aucCElrHUwGebJwI09kkeTtySDwB7482CDoAvYF0aCH0a9KD50+lsXV0G6DT+pL7eaMyIch5cbWNCgSlyeR3S2fRHusd+yebkPPQT4Zms1Kj5xzxEcRR02ciw13Sqb7KX6hr7fv/wC577AjQY0RhuPHYbabZSG0IQkJSlIFgAB2AHpj6fwpwS6pIrcQHc+yw8/M+XTqvinH3tIZSB2GYM68mzn/AHeob1PU8vXYmlwKXRYLNMpcZmLGjNhpllpIShCR2AAwtDrY/fGO7aB+6Me22xyEi+PsTWhgs3ZfnwuL3Znm5XlKBAsfUY+O/WrM+VKv1w6oVvMb8kt/itdiO7EdySy/EYe2Eq7hEYBbBacUUrKjzYAXx9iV+n5jHwTzu94SNmfx9GbpkurT50yTOltPSExZC5zqmd0tg+Vwgm5CtJLf6c/EYxNG1p6pHOsF6RmHKkPPkbN9Fr0nbhurjU+GphxMphw/S2487uoASvc0BN7gNoXbtixKs1Tcxz6VUMusN0IUmooRPQ+2HFxw+VOFsWBQ0tTg5Cj9Gix44z3Wsu3qciiUuvN1OS8w1JbXo8O0XPDh5/UpZSjQ0Cu976hYjDxlOpQZ4mQMqy3GHWo6n1TQ4HH3kNoQonwjSQh1LagVFRIs2Te9r45stAHlksbjcC3wVbU+9UwUmmzYYqiqVIkvqgO1BiKwtThaaKjtRgENtm+oX5Purk4l/QWoriwqun8LEs06sRChO8WFSUPstGQ2pwAkfsmlIsOCn74pqqVOlSqBS2aZNMoQ1yX58pJCguYsNBW2QBZqyiQkCw1LsBc4nHRWRJarVSjBRUxIZYKhzp3232NKrdr7WpPvY46rIy2mylbqd3fC1v07TOZW5OnpcEmW9IkvM7gUloL2wlsEWuNAbIHpc4x515boievHUVuqMOuPO5kqL6H4s4KIK7bIcAJACVlRcbsF+UDjm+xsrPuw3NL0x0EMuCUq3LxAFy6PuE39rgewxmzPWUMiVnr11PrlcrMFVfpmYZEmHlKVGbeTXGwlBU2khxJ3LWWBcE3UEa18HE2UQ3c7p6rVWML2C3VQFzPeToXSx7JrfTxuU6p5BkVgqL8pLrrRDqlF5OhAUttlKNOkAMgjnkseUKHlWpwqbToziI1SluLbVMe8Q+9LfAKUtNNNmx7p0OFoga1XtiaZ0kQwy3Ay30Lbyy1U5TjNKjr8QzVHIy7ALnSw55EnhAYtouebW5pxBVRWahTmWQ1de3UEOR9D4cQoKSW03+UWzew9lKC73sLqeEFhI0J13v8A2XMkJuFP51ZqtHK6PCqU9FDrLjFRkREyVpakPtpcS244kGxW3uOAE+/5YuPpf1X8bR3MrZiQ6pJikNOJeduoAaVE25FkrtzxyRwMU/4KlzKStbviAmnNmUglXKWuSr8wAb254v7YUZRk1J980ihLmJlsrkbaWZG0ZkIoLj0Y9tf07lifMTb2GJU0zZhZai57TqVcubMp5czlFlVGj1qPBnN/MjOS1DVIDCkpcUAwm60cHQLeUK4FuRSUCnUuRIdgZnguqakfKd3HnWksi/JAWL37H9ztzfgYURJdSoVTi5npKozjNRO6qOvcCHCg6VIUeLOEJ4NiLKHcDCd7NT1YXHjhemnx0OpajlpAcAJJSXnBy64EAN6j6JFsCGGSIWvdVut4lLMy9F6E0y7EoOaKompiOX6WJrzehUlo6kxFOHzrDrYSWVXslaVoJNrYhFEhsyKbKMNpC6zRpZqUtlR23prDgCX2kH6CG9KTpI1pWbgkGwvDK81nNOSCmqshdWy802mWq9t+nkp23bf206gSf8k+5xHc80B6Mio5noMZDb0dwGYy2Cd1CGz4mwN7OABL6LHzI3hzYWYSP2ci5jbZgoPXolaqjiU/iDrMoRnNxLbHypT5QolfkPC3kbjZSo2OkC4sLF0aqRJccn8LiMkshyKh5QbRGKwAo3NyEEX4vfsL8cskusmVEYU7ckguMut33U24JuP43/Lng4dqTlx5DDGZ5MhtUJpl2TIcfC3SraKQ4jSjkr1utp786vTgm37Kq3TBWW6o7mldHjwEGJmV8vsMPaGEOLbduFoUPKkd+RYG5FjbDdOplVaU/SZbzsNyK9dTbdtK9YBva3KS2botYkEYlmbKKZSKfSXcxtJmZeknwlSYZdUwkoVqIZItZA+S6OL+e3lvg/NFZbz5+AZlbZfD8CEIlZqkKSJB3W3kpS7oGkj9om2o+YqToUm5CLGuRyFJHp1OrtPozzCp82qvxocV9tDIQ0ohvb8vusaUgji/KyTfDK5VocKbd5paHC4trTb9y1jf1/MH1t7YdcyUWVlyoxHqlObqESrJXUIVQgOqdjTmivSpwpcAcZcuLFtVynscIKMqJPq0mE/tf+tGVsrW+1uutL1hwOt8X1jSexBso/lhnNzJU7UheW6ZPXWsnzphmxYxd3tTjamAGSCdRIBOskBNue9zfFcZ6r1YqNTRUJFWedck6t9oGyG197Ni9gggiwBsmxHGLHy7kuQcq5zzHGqQdgUaHGLcxIKWpjb8kskJTzrtYG3cHtb1ib+Wst1iXJjtVFxuQhkLCXnTqcOqxcaT6i3Ojg83PHYsGVAhRKHmKRHqUapRywW2Ftq8M9rKLoINzY6/qTe6TcX4w9y5TFPzNLmPMgPbxWhtThWGHSb3Fv2m3fgnvcHA5GSksuts02DLnvKjuOOJUyLt7VipwBBIAsU3ue5P2wRBp7T69x0Fam0KQthLd3AO5c/QA3xosFVlUzyLmGswM3UzOMFzxE+MHZcDxDyztFttW4NVwSvQlyw55NhzhyzLXIKzH8XTpKEuNaW91Liu9tIPPPaxv2GItT6JXkS4E1r8SaUgLlRm/IGLBwX1EH5dyAeOe3Y84lDcMyKXJrz65viTLXZiWV3ZY3jqcGsq1k6r6SeO91ei5U1lX9RgrjpU409dmwIW20oBy3sf1w6UBanm3H4xcS6HUBCgbAX9yOb37Yn8yp1XMOTW6W7Dp5tJ1qXHbcQ/IQ1cNtOKLmggXJIAF+L20jDXlWNWmlSKhlBymR1uuNakTYZUktdxoUDrFiVDn05vgNab6ouCs3I/S3NtegU6JFrdKjinueIKJrjiW4q7hOvebSfJoUDYg98bA/wbU6ZG6oZ3y7LYUnYpLY3Ug7Ti2pAQoJV2WAV8Eehx8+Kyj8GjO0usOBxue74slgrGlwHyhN1EaE9hcA2VY8jGovhQ6vV7pNKZzxljLbFeZmwHYUmG0oam/OCSraJMc62+NwWUD3vi0yNjZ3zZMzdfXH0xy3r9sZQoPxuSa1GX4nIH4dMQbCI9OQVO+QqGlRIHNgOffB8f42mJC9LeSw4dSGyv8QabbStQ1FJUVeWyObr0g2Nr2xiOIU4Ni5aW0kr72G2q1RcjAj2xXnR/qW71Vok6vinJiR4s0wmilRVuKQkFR/LzAfx7YsNRsCcamuDxcKgiypf4pa6IWQomXml/NrlQaZKBxdpr5q/5pbH64y51IqL1OplDoEZbFnS7OkIdjtupIFm27hYI4JcNxz98XT8SFY/FepdHy62vW1R6fvuAc6XXlk8/fQ0n/n4zzntUypZ1mra80eElqAyn2DaLufxcUr+GI8F1gFrgORhKa6jLXTsoZgqO02gtUuVtpabDQ17RCeBx3IGNr9D2hE6QZQgE38BSGIf/AMK7f+jGL6siNBohZr0TXDlFsrY3Vtl9sPNki6POEEjRrHa5742V0OrdOzF0sodXpFNFPiPKlNpjCQuRtqRJcQpO4sBS+RfkeuM4im98D/shtviTp9FskqqX+FOprHtS8Hyy5SN/VQD41+o39B+ic2jwpO3Pze9+DNEKsoR7a5Kh/wAmko/5THzNkOlWtKtI0LJFvvjU3+EEzuar1XpOTm1lcfLVIDi0W0jxMw7ih/8ADQyn/OOMoabAX9/4Yz1L+0kN+S/VnsgwQYVw5HO4WfN3z8dvp+KUxFBcgKdcANx+uNwf4Oh9t6BnyMjkByl6v1TITjDTaCsgpb7d+cXz8NfxCPdCF5gW7lFNYTXG4oKROMbaLCnCDfSq99ZHp9OEppGRS9o82Fiup7R8LqcXwGWloxmkOSzbgbOB/BVPKHh248NaVtqQktFK0FKx5iOQoA+mLx6ydRmMxfDp0Yyol+01UJ+qyGzyWm22lw2L+1yXCPshR9sQbrr1CT1ozt/T4UddMWuI1D2FTDI8jd7K1FI9/b0xAqhHkrDCQ4pKWW0tjlR49h9uT5fvjv10tNBQtjgdmzW16b3C+NcJYLPiPFr6ivZ2ZjJda99SLW+pW0vixYl9UPh5yH1coLCpTdPDU2oFCNRjRpMVLT7iwOQGpDTYcP7t1E2AvjFM59pS16XW3ELBspKwoG49CMXh0G+KXNHRilOZamU9Vfy+t1TqIqn9h2K4olKy07YpAURyhQKSq54JJNbdTajSM4Z9q2bsv0qRTYVXeD3hXy1dty1jbb8gBt2A9/fCYa6mlc6V0lnADTzC5XF3D+L4VQfwx8N4I5HOZIOYd9kjcWumWoTVzHI859V1IhRWP9ylpltofybGEkOW1MSWnElBUCpBP2x2oJU2UIA4KcN62bMrbB0qU24hKv7OoKTqxyJKjLVum55rr7ZQ4RDi/DcdJUi+aMAfL9UZGafEl9lbSkFtZulXcH1xe/wmZTfk9RE1ghe3AbcA0HglaSNJ/Tn9B74hnULNWTM2SIVVy5lGdRpoaSmoLektLakEMpTdsNi6DqSTc+h98PfRHrHE6QZmkVSqUiRUqVUI5afjR3G23kOA3adbLnHF1gg9wr7DHtjxDSQ1D3RG7HAD0Nl+fm+z7FmtkMUZz62Gmv7ClNZzFEyb8cc2uS1BDMLO7K3XFfSll1pplZ/RL1/0w9fHVlaVR+q8TMLjZTFr1GjsNOW8q5UJx1t1oHtq2lsqt30gm1hxRvVfPtPz/wBT8y52plLfjQ69NTJbYkFCnEAsNtkKKFWPmSTx7pxfOTPi+yxXsj/+j/rplOVXG4zSGk1FqPHl76UCyFvsOkfNSON1s3PlJCSSD5OCeN8jmPOUE3v8V9E4m4YxE0lLV08BfaHspGX71iBqPMO5KoPh5zN/RPrhlStuOpSxHckGTq7bAYUpV/bhP8hiCiW5ImofNgqQpyS4NP0l1Zc0/wDz4cqxHytErs+Zk1VSRTXy43GTOSlt5tpfdFkKVYWJA57YZnXNmQhZF9wpH5Y6HEM9M4ww0zw7IDc7A3N/orPZjw9UYZDUCrZZzxa3pf8AUoa1K8bsEaSF3H3GD5KNte8geZYscOcoZXlUukyISpyKxFYW1N1tILDvzlFJQvVe4bIHKMNM9YDatPFsUY4+nfFF7s69wL+R5hU8A4NV4JjVQ2QEMdoL8xfRF6yG2GvXUL8/fCqQ44HVFKrG+G/VfQtaOE82w6J0vpCym9++MGGvDH2dzXR9rOFzYjRwGHZriT8VrT4IMyxX269lRVkS46kVJs3tux3CEOC3u27b9HR7Y07nHJeXs+0V3LeZ6a1MgyXGipCx9K0OBSVIPcEFIIPpbHz06BZ1byF1fyzWH3dqFIlilzvNYeEl2aUT/uVlpz/k8fSkIUFhp4aXEkhX2IvcfyOO/FWOlOa+o0XwLiTDJMKmhmaLZ2g/EaH6r5TdYWZZ6m5iDKdloVKQlppR/Zt7itIt/f8Ae+IkItSVFfiokpCn2ltgk9iQcaKz/Sei/W/OUzNOT841Ciz6okzH6e+y0pOsp3P7KSCUclN72PGIHUul2WqbJqlJT1EcRPprCJB8TQZXhEtk2BLzZNxwfpBIsfTnHocQxWkljaYZLNsNDpqOS5csxnkL76ndfTDoHnWidSOleW8yU1LJQ5TmWHmwQSxIaQEOtH7pWCPytiy247aPoQB+WPmr0QzlnL4dM0ORDmqmVCi1FDc6ZSWIFSU6rWkrS62S1oCykG4/eA9xjSqPjx6SqkxobKzLclLW0l2BLbksBxDO8tKnUCyClvkhViOxF+MeTrpacPJjkBB5dPJUOYScyv7NmV6JnDLtRy1mKlx6jTKmwuNKiSEakPNqHKSP+72HtjGcT/B50SPntuXJzxWZuVW3t5FKktN7xAPDLj/dxv0PlCyOCTi1Wvjm6VuxhNbp9TcZso60NlQ0i11/T9PI57cjA3vja6YsLUh+lVVpST50uMKQUdz5yU2SOCSSQBY4lHjBog4RuHxt9FG528k4fFvCpmV/hLzxTKfBjxordGbhRWG0BKEKW8220lI+xIsPtjB3wpZii5c6rIrlbkFqDCYdefcsToFrgBI+tSj5Uo9SR7Ei9Piu+JHKHWnJEXprlrMMeiiRUY8yoKnRpUjfYaBdabbEZCibubSr+tgBcm2Ka6dUHo9lGTIZz11BrH4g0jU6xBppijWV7QQkzG7rcKjpCAASSRbgnHZpsdZQYNM2JwdM/MQCeoA1PQLqYU2lfUNFeSIydSN7eSnHXTqTmPrqqYJEk0+ChGijwg5dEVQVdCl24UtVgFK7AGw4veicpU7MEfMbAltPMuRHNTqFC2jg3Fz6e3vjRD3/ANTwxILC6hn/AFpC+G40R0Om5Fm9tpWskggAXJtgthfw9So/jm6xn4xdzYKyxF8rlgSD8m47j+PuDj4pTR8QwwTRTuY7teZcND1225W5L61V4twXLU0s9OXxiDQAM3ANwL33vrfnc3VfoakaiQ4LEn1wJYkJ5K04nzcz4dykFU/P3mbLtlMxkHRyOLs9/wCOFiYfQh5ZbbRnxxVrm/h0j9DsWP8AHHijwPix1zM/qXu/+bHD99S/+j+6rMIetfd/njhaX/to/jiz0UvocbnaztZJsdT8QW/P5HGCmofQl5zbaTnMjkBapMZKVEGxsfD27i/5YDuBcUHNn9X9k3/NfAB9/wDo/uq0G4kBW4ATjllJfKFSUk2ub8BOLDlv9BYSlpVDzy6prkpZkQ1KV+QLQJ/h7n0wblrqH8O+UK6xmM5RzhWPAr1hqYWHIyHQNQcWEoSDpse5tf3sMaIuB8QkkAkc1rfJ11TP7XMBijc5geXWNhltc8hcnRW38OXw1vzlRs/dRo39WBDtOpbzf7U90vvJP7vqls9+CfQDWyW1oACVJAHpjK6fj8yGNpH9FqmlTiEOaVcFAWbDV6Dng8nnBMj/AAgmRGJBiuZUqQdQUXG4j1UBb08wuCU9wObWBx9awfDKHBacU9OdeZ5k9T+i/O/EmP1/E1Yayt+AGzR0H5nmtY7bn+2/ywDad/27GUYX+EAytNmR4reQK60iUspakPFtDB5A/aau/I4tx64apn+EhyLEiuTT0/zG620oodDbV1IVu7VrEj9+3I4sRzjs9o3quAWlbDDT/q/gO07/ALdjNbnxqZcRlj+kXgKQXtAcNGNabFVS3t6y6qMezYHclfphuo/xx0utJrBj5MkocojSpUiO47qeVHQvS46y23q3Qg31hPnHPFhg5w3cqNaSNFqQsvX8z+OKYduFb5OM0RfjKiz8p1TOELLsJcekLKJUTxavGJVdIbG0BY7gWCjz8j7i2G+B8b7NUlOxYWTdQjqO86XVBDLaQCpxR9EAc3+2K+3jdpdN2Thqdld/Vbqpl3pPQDWK5LL0p0lEKA2oJdluDnSL9k/2lHgD3NgfkJnmq1iu5tq0urvNqcZmyIzLbVwywyH1lDbST2QOD7k8n7aJzd8VnRTqPmarV3NvTvNMiXFKmVLXmQtMlLaigpjtBQ0puCbcXvc8kYq2p5g+HvOmYpNWZypnilGpuNuhr8VQ602dsN6g4VEkkti45Nz2x56rgxOpqHCUtbEBoL636nT6cl73B8R4fwuia7vOqD4iW2AH3W6333JAJVWo1AeuLb6TT/FZRfpilELpc95jy3BDToDyRf8A5RwfpiQyujvTyDQoeYHqBm5lmWAtlt6qJCx5Ndlcm3FjYgH1wlpOW6VSIj8rJEeRDNQSjUmpyVSgoIWoApA0hCxdQAJsb824xow6hkp5b3Fis2OY3R4jT9mwG4N9lpn4bc3rpuastyn3rNVVt3Lss+hdR5mFHjuSEj/lMbNBum/vj5p9NDOp4q9KZluKmvbdXiursFGXHI9uB5NsWHADY++PonlDMMTNeVqVmWGRtVOI3KA/slYuU/mDcfpjvMGXReOm71nJu6kZSRnfJFYywpQSufFW20o/uu90K/RQTj5xyWVsynGpgLbjalJcRb6VA2UP0Nx+mPqGq3PH88YD+Kfqj0b6FdV5WWc2dDqtVX6xGRXI0+HWSy3IS8twO/L3BYpdbUCLAeYH1x4fjDhWbHzFLTWztuDfoduXJfRvZ7xvBws2anrQTG6xGWxIOx3I0It8lWhbSLqQo2t6nGlegPw2LnJjZ36iRiIpKXoVJdRYu+oceB7J9Qg9+6uOMZnoPxn9AqRUE1mmdAK06/TrSGQ9XA+2T3C9CllJ027kWBt7Ys+F/hVaFNcYjQej9YdfeJQllNRilSbclSjqskD3OObw5wEKKb3jEe+4bAbX8/yHzXZ4u9qZxSm9zwgFjT4nHe3QW28zv0W/G0IbQEIGkD0wK/6Yxaf8IlGQ4hpfTaXrKNxYRKaVtg+pOoDnm3vY2vhMv/CQxkvLbPTCaQkaipMtnyg9r+bv/wB2Pq4w+pDQch+i+JGqiPNbcx7GEan/AIT+BSp7sCR0tnFbJWNQnMWWR6fVxhNQv8J5MzQ7Mby90QmShT2W5Mlb1fgRENsquN0l1Ys2CghazYI4J4Iuj6WVhsWqxj2y+Erc9cnGl0adUkpSTEjOvgK7eRBV/ox8IqbTadGimZmqJSajQK7HbblTlOuwnn5ZabcbbGtSUaWlkkJT5yDc2PfT3UP/AAtteq+UKnTMp9G245qdMejtVCr1du7CnWrbpYaSQsJ1E6dab2F7YxHK68Z3RSGKVAybliNFjxmIwWzSrlQbQUuOG7ikanAReybDSjbCbqvmlohUWa42+B/JK89ArRq8xyqU6LVqRMo1UqsJTcVcBlDjzKmbgNOwg8oNjbWm5bupDi+Tw2RiFdTMrSc7txsuU2tuzprT5lx4rMJe1GW6u0p2oyiG2wsbYDYYQvUNA5JF2Sn9eZsOM/GOQaQG5LSW30t0aJtvt6irSpKG27oupRIN/qUCbHDcOusSBaPC6b0+mttpcbSqkPyqepxLnfdTuutuWPIuPIQNNrYzMwz3Y5mvuRslbe+oSzOmU/6CvwqMKPKgKVTWnUyZUvdNUO5pVKbaBIZSVAoQ3cEhHIBBxNOmcKTSs9R5BU+qIJHhH2UgaXvlXUnz/wCWEnjn0xXuYesWTs5rYk5oy7mxUmJCESG41mNLyW7Obl3EyGHCsEk3QFpAvcW7Ytzo9V6fmCA9mVMltRkPPOeHSpolhxLQBSpRAXb6SLWHe/1Yd4kbT2kNzZbICM4C0zlJtbwWZEhCnHGJiHRcJ4NyRz35VwOT9uMUR1Dk5NY+J7M7rFHqgrMZ/W9UWoypAM3wgVpaijhweZKLu+QDW5zwjFzQK6hhxTXy0SWG3HWmkOISooKSkkJPIsvT5u3IGMsfEDmOk07rfnNKqW6moMZm8U4+HF2ebDDQDJSeyLpSvUDrubWAscctkRkuAeS3VfdZdJ8252qQyO/VYby2oFRU7AbfZLWl5t3lxqxSVlwlKklQPGnz3POKZhyIqpPgnZkhh1T2tp1fmbS0GzbUCQVEm1ucSXONddq7bDz/APVUtuLLUfxe6mI044pW20n2GruQCSFnuTiKwYjC5K5RQxJcNxrsbC/71uxPr+uN9LAImWAsuS85irCbqdUp0hMhmoyUFCwQUSFtlQ9UXHNlIuD+eHBTr1Irv43TVuNxvEl6EoK8+0tHY27EBRH6Yag61Jj7K2rqbXfV2Og+n/n74NiFSkqjLeJQhy4JXwkm38uBjQ5t1Zmd9pTB+CmoUiNUGAh2PNkPj5ZPyJKLFQ+2pCgoD2ULfTiKOU5hSxPZa2JN9t48kKt2UfsRx+mJFkmpSUyHsqutOLYmvtONoDZKvFoNmim39oFbdv8AKHthC+qPGqbjUhpww99bawm2rbJ1C1+9r/6PTFBblcnPhU4yJm2MiLTqo0pYqcBlymVCK6DeXCXy2sei9uxTf22+Lg4kGb1uUKqONsyVrp8tuAXlW52yCYkn9LqZWfYpB74p3LyY/wCJNQKoy48zvlPyHS04on/a1DsTwofewxYFap3UJ6juR1U+DmaJDpJkQJEZwRprsBa7NuKbWC28Ur4WGyCOQOwxU+KxzBMw5u6mCp0uFSJcqHUoaCIxNTo+2kbVhIBdYJ5O04iwBF9taexCjgjKUqDW11HJamJLMR5xcqlJdHmZkouW2gkGxsghP3Qm5tbCu34pERS5hMmbSpV0MrGl0Xjuh1kpNnEFSVIIBH1pHriPQs1LpL7UaqrclRmyTHlqXdSX7hSVK9bnSnkdz3ww1aQUxswp1mOyKdOdoVRjSY8aRICnEzE+VmQhwtB1N+UIb1KSu3cd72GENFoECnVmppWXEx/DL8Q02sbLJBTdDo/fbS5rbWBcnyrQfLiQ1diNWY8aZMqsRb8iHJnoaUuQlpoOulWlPJuLghZJuCFHSQcLZ4h0ikQ88Qpji6hLZWtlqJKYejSDHDm/vgmzRUy3YNOD5nNu4tR7wB3VaWZbPSDN1XobtGyXRo9NY3cvGQ1LWoEqeW6OdVxYi6UG9gb34BxCBTpr0KtR48Hclx3G57spVvlw0HUQlNuBcgrsCbAexwfJnyMw1ZmcJjr/AI9zdWX7AuOOeYW97E6ftb1w40etinVlX4oy6FMKAS+DpUmIs8hSViziNJIA543P00i7W3VDWCR5Kl9dplV6a9PM/dMqxFWmXNqtHd3NYKEIOl4IA50LOn3sQ2T3timG5Ca1VksZfRMjJY17bji0qfSdHzVLIACxYGwPYWBvzizc60ORXq/UI1JnzJUGnwxUKhOmSi9pQGU2Kjb0QLITbXdy33FS5SoYqUafMeU6iBEG7IeLZbSpawoNNlY4QtS/c+/tjQzwlyqc2ynNAerTVTcqMJcd1q/gVh9nUH2tuzvAII1E2Nvce2GrI7UQ1KS9oDi24xW24v6hyRfjjkKN/wAhh9pTi6VRYzMqy1x1ht0tjlfNybf27Ei/+ThmyVGEDPlSoMl5tbaGpbLTqOUqssKbct7Eev8AlfbFoHdzKv7SkrdJbqsZhBeeZ27ocLa7F1vi6Se/Pe47Ww5VdkMUGS0FEpRGWi5X7Djn9O+EtDf3Isbi6iXEE27cXw4zhvQ3gnlSmyAPuUm2IBog5MGUagtFFZeXbVGqK/TgoJSo/n3OJw3HRHkuMNNISEi10gA27o/kcVtlFQXTagzrvtymjZXNrtA/6MWNGeU48ypZ+q4t+n/hiN7zk0vgCbc4wo0jLtRfVGQp1umvutLIHkIF7j+GH/oH1syl0xynNblQ5orHinJTyEfMZqCC38lo3VZAANj5D6r5OGytRy/SpEZNwhynyW+PcN2A/kcUmw8ldNZWjUVbY5t/34M0EdQ3s5BcKtpI1CuHM3WLOOZny87LaiMMkOMsRUmzZQbpWFrJWSOOf8kccYMb6n5qkwU79VdIbQFSVqWPOhDgc545B0p/K1h6jFZRXipQSi93GyL+2LK6PZIe6kZ6yRkptrWK9WI1PfCBa8dbwLiz/uGw7f8ALBio4AMgYLeiujke0nKTroV9nfhmoNRoXQ3JzdbSymqVCmt1Oftt6AZEkbyvL6WCgLfbFpHtgmMy2ww2yw2ENNpCEJHYJHAH8MRrqlmc5P6eZgzChRS9EhOCOR/ty/I1/wDOpOFygGzdkTusr1OtpzFnvNOdHHdcdc50tKJ1fJY+Wn9NLQxWb1ey5RTDm5lrkOBIrc0xICJLbrvi5i/MEFLQK9sEgLUP7QA5ViSykCi5C8JrIXMLcUX7kE+Y/c6Er/jjO3V6qSY/U7LzZqFZgt0WiobbXAhtSEFUl7emJebcUNaFNpithQI0nnunFU73RMLmbrcW5WWU2zNTa5SY6p9fzFT68/UX4xclU5Cg222glWktG5ZbAACEGwtx376t+D2rw3uglBjPTGm5LlRqDW0pVnQtcgqA09wfN64xH0Scp8/NM2pUuMwYr8eY03abvuMNoW3paLlhulI+WSQbHVz2xvn4bEoGT5KQhAUmtruoAA8tsnApJnTNu8WKyOavnh8S9cfzR12z1WrbjRrT8VspVf5ccJYT+lmjiuNpyW5dpCrptq0tlX/RT/0sP+apyZ+Yq1Uy0FKlVObJUSeFBUt5X+nG2v8AB2U5LfT7NtbCAlcyuR42oC10NxAbf85asc+KM1Eha5fs3FuIP+B+GIqtsWcxtYwC9t7DzWG6dl1ya4AyyoLHqW3Lf9HE7ovTZ53bEkrQVcDSDcn+yBbzY+oOfuo+TOlOVJGcM7VDZp7JDTbbaA6/KdV9LTKb/MWf5C5NhyMO9Xfi+6m9RnZMHJtSTkPLqhbVEVaoKR3O5IsdBPs1YD+2vBfh73Psxy+SVHtvqagaUtv/AJ3/ACUIn9HZ7NO3ltIYVtlxAlOoZ3Lc2AWQb4ktO6IPTILb4pniSUDUIrqX1JuOCQkmwxmXMGc+mFFkuS6i3JrlSkEqddee3ypdr63HFkk+v35w1R/iPg0yYF0PL1NgtpNxo3A7e3fUCLX+2Hkw2aSMRl684Pac+KoM7IbOPmtb/wDoAkqf2RSghRB8pNiPXnDM/wBDpYmhp9CW0fWUh21weAAr05/884qjK/x058obCWImZ6w02LFtBmlxH6IdDiCO/GLdyd/hFKvHWlut0im1pDqwD4qmtAnt/jGdPt6oxkbhtbTm8ZB+P6q2T2oS1TSyaK481yL0AqMn5LjCF7Y5Wp1CQfuOe2Ayvh8rI3W2KRrKBqu3LaPH6nn9MX5lH45MhZoqEGJK6bs+GlEiQ9AnMPuxxf8AaKjOIQ4tHrdvWfYHGm6V/QyuQGKlSmqVMhS2i6y6002W3G1d7cdr9/v35xyKp9TTv/mtWuD2o1kTcjI9AvmXN6I1xhu6Ke/ca9SboJFvtfFfVzKNWpbpQs+YK5So2Kfa4x9fhQ8tOA2olPX6XMVs8f8AN7YRS+nvTqpm9RyVl6So8EvU1hZ/mnEp6xznWcujT+1t0Ls0tPm+I/RfGhbC0X1JKSnngHHghepR0rP5IPOPsG90U6MvKKX+lGT1kjuqhxTf/wCTDPUfh96KodKB0ZyQpKvegRwP5Jx3aaL3s2afouufbXStF30jv6x+i+S2852LL5/5M4NLa16bx5Kvb5K8fUtz4dugar6+h+RL+34EwP8ARhO58Nvw+uCx6IZI/wA2jtj+4jG52FOI3H1VbfbVhPi92kHxavl8G5SCdMZ/T/vSsAdL7oH9XfCk/wDulY+navhj+HlRseimTwf8mAR/9vhOr4Wfh2cNj0ZywB9mXk/3O4AwqS1rj6oN9s2C5u093kv/APH9V8yUoddOksOjT7tqwojPuoBDragkK4NsfSh34TPhxWLno/QR/uFyU/3PYgvWn4U+itO6UZwqmT+nsSl1mm0SZUIL8eXK1peZb3B5VOlBBShQsQe+AMNlju4EaequPtYwPF3MpHQSNDiBqBpfnusKOLU+hSWlFKlpKEr/ALJtwf0x9VumWaU53yHlXOxWSqs0mJNcUDzuqaAc5/3aV4+TqXm1KCEHyrWS3+n04+ivwXVgVj4faPDN1qpM+qU0EHsgPKdbH/NfTbFmHyd9zTzF1z/a1hLYcJgqGDwSW+Dv7hU30Ty+8nL1MdzHQammROjIgtzGaVJZIQ62T5XQgI0OG5WkXBFzz5sMjNP8F1KZqcdOxT6rGsppxpqQyqQ2gRG20No5Q2201GC72+Zuo44xevTvrZlWbTsn0eVm92JLpeWIFGYbDLy1vSHIo35rQRdBDQQWUFVvOXSeLE0zUQulOUWnVWfUGlqhoalQ2PDNtGMgglCnrkWLrbA2rjcs6VKAUTiytmEkOQ8l8Hp25Zy9VNmKk5lkzqlmqZT6exCiOeCDrjGywxtr20NstkkaA4SQ2L2W4q1+bQbNlBnzYD9JRFTKZZhuogQo4QiPFbcB8VL3LAOWc2jdwrOgIbHBINpVV96oVjJ2Vs11F1qi0+NIzFUQhCtuXYaW0NtEAuOHU4AnuN4+xOEcKSrOdHzR1PpMZMOg5oebcjQ0OWLURDRS64eTtB54BQACWg4o21C5xjmYynY1wO6tie+V7hbRCqVJR+GqQ8sKhpQx4tSHvLtttuWcs3p0ArKEaefISTbAczraoVUVSJFJlsFHGiQ3Z2I1wQu173c1XFisgOc9rFxXUp853LzqZ8uaJkePGAcU3pcfQoqcDrdzt3DgAuAPKseoworcaO3XnMyZhzI061OrExyBMdhOOnds2lsbF2/mFZDaAo8ANmxAF+U6HO/fRO7ugqMx6HBmyvAzIK0PJIDvzN1LLQCVNqIWjQjUE31enIsdXBeeaZHy3k2tOsT50tynGNJXCZlKSl6E38597bKruFopbAHFwCB9IvKfwJOcq49lGA3ErM91qPT6Q2bSXZcn5RfaSpbhQNoBwL1NeU67bY5xD87phwMnTcyUmibtIoTxa36bUSHY8N35dRbdacsiQy7qBCR8xrZbWABcY0MYWPyk3RbfLcpky9Lq9bq8yptU2NTpl340Vlua5fxdy7JlOhFmjtNG1k67FvglarmX5LqyKdS5QXrUysR3Vpl6kqbBTay3LX7q1+UEr1LuDcAQrIC0ZZoWS1VeTLkSKxGDTryyFNa1tHbQFoVoaRdMVOlVi4AvglJvNqemZFde2WUNQ6PJXurjujabWW1XbcJVdCxuBsEceYW7YtliY+4KZveSk1689ENVNpi3kN7nyY6A5HbIskKU2kFvU3fhS7qABseTh9opeYKnEIbdS6DpUHDo0alW4/fsT6geoviGo/qVHjPQ6dV5G+4SHnWmy4VoIB3iHLuOnyd7XCkgcC+Hmn1NUNloR8u1xTKPKyURkEls9lELcBBJ1d/Sx9cYnxBrbBONCpWo+GbWh8NhSHA5y0FWWCDYj78d8NlRSpcRLKENLkOOFSGpTetI5/aAEEBHHF/5YIlzprkV6+W6qlQTdZcQwQoo5+ne47/ft+mGWo5gkxISqkmmz0NLGgKW7FRq47ftrm/tx64p7MyEZSrHvy63SmouO1mqNx0R7u7hU3dvSpKB30N+nY39APJ2HLY7siQ7Dprup1hsOrAQJCHDY3ccA0882AUbk9r9i3Q67LkQk7cUhJC0qcM5hpKfVOoAqJAPpb29r4cIeW6lIEaWuBTw5HLCmn1Trg7fmbKwWbrJUvk+wAHPcNjLDZ5WbNmSWHLe/EnYJ8O8sRjLfXJCUpZAc0+ZpdxdxdgEuXB2yBb0TZqaqkbLEqjQK23Hq3hghpCYaWxufsztptZGpHYHWCdVlIJGHyHFqSIUpFal0+kCnlp5U1xDj9igkhIaQFI0XJUQQTYbgT5bYTVGq5qYo0lcGVEUFtus+Hcy84ZZjkpc0FKFBttlSHB5tACLk25Bxta3UWT2HNNFFy7lRprLsuntU9uXOHhgIzjpfStDLhf3XkeRDillvyjzpNuUFJOAVxQiynKbMqonUVdUit1OWZDzTUBA222w82FfMIKjtti51ga7jkdqM3MNKfpzRoMOS3VWnUTU1VxCjUnGkl0yXFCMAENgkhTjri7HQbA3wDJ1Lf6vZVzc5QmaZSJEirMMtRHXnUxUNRd19bbKtta9qzSym4Sord83JsLZHNjPaOdp8/wSWaNVM8qGjxs4RKAyhyTKnR/wsT5zTSj4ctKcbklokArs2kLCiQNX1XGIfn6BmfKsWJ1dns1CgVCVKtElKUGXnnC6Aq6UXIQ4CeVbQOq4B7nRnT2i9NmaGazCp0ObV5phJqz0lDKas3JCFBmM22VXbABdCEt8L+YSSbgZn645XpCK1VVSc71eoOS6it9uHOY0RHHHH3XHoqnGyUObSPKG2xr45tYEYMHxsY2+ZjYnNERy3Ol/T99FvnovdmNkJvforc6i/huQ8nqruUIEedlmszaNLjRN0Dd3S42VuvDzlxuy0oAXYf7WbXNO17qBHoNRZptElmmVSO6iVM1OoN2H0jUh5zykcdvICOebm2H2uUyq0rollVdVfbnRpFfRKiMuoaUGWy0+42FFYO6QAPMo3OLHpOZel1VpFHiS8q5fl1tMJcKIhNHjqEea+Clx76QS6NW2j5hAQoXAIOjewxtlAcVTkbKNTZZk/wDrepmXcw5zqMPdZmOmVAaEhplLDZnJSAndSpso0K5BRbRwj3wuyvUI9UpjU5mBGXMmOuPwllYAhMIcS2SnQkIRe3BSPS55vec9LKgvLc6iQI0WlvoW65TGm6tFLzJ3EuNKbWkXXf6gCDcEXGGnL2VW6ZU6vKp/iWlQCp5wCUXwCZaSFtJ4GlS76EEgEelsaJ543Hszus/YloEltL2Wj8yCTCyjRqAunQ2Yj8h1EVLTOl1IaQ4fMogX4Krk9yRziDQo1aaYR4yjOKpTkZppl5B3FvOr3NxIaAuBYCxPqPTCjrlmmYHYIoyn3XKDUm5TyUSBYg3ccacAvoGgC9za4HNhic5XeH4SIsdR0xH3WQU9iNV0m/r5CMNSOzXYOSpm3ukkOkVLJLNIzHVKszVH4jqBMUw2QSweLquTc6FFBJ5PBIBvjW/w019IpFYyM89qVQ5pfic94sglQt9gvc/iMZsdgpnw36a5YJltlnk2AvwP52xK+gmc3KHmrLFWmrDYmbmWamFWGldxsk/koNfoVY1m7TdQWfHZbUsCMYo/wnXR2Xnnp5lXP1Dp7siqZXq4p7+y0VLVBqBS0e3fTITFVzwOffG1x2xEOrXT6mdV+mmZem9YUERcyUyRTi7a5ZW4ght0fdC9Kx90jFgWcGy+F+VqFRmOoVVomaa9ttU0TaZpLZBkLQCCAQo7dubJBJUSBbnFk5XyRQ8hQ5Valxn3Zbh1sszNK3mGx+zaIAALnbj9y/vc4ilFy9W8kVhbkvLficww5C2UQVs6URagFqjuLcI5LbZaceOnknb7d8T+ctufHiPVyrxIaWRuFDTRLj7norbKjo5v5fPz646lGGUhMu7uXQeaw1eecCMaDn1PkmqTJMVhx6oyNUhwl6UvVe7h9BfuB9I+ww1tzCpTLCbl+Y7c3F/Of7gEcfaxODpVTyg9pWlidUGySpLqnLJPvaxSLfphv/GKZVFrhwqI4CtGgIacbCnb8EXDWvtbi5++LKiplqG/zJAFKena1+SNt7qA5yrKJFbmyo7wUyZLpaWFXSUaiLpPse4/PCjLefM7ZWpdboWW8wVCl0/M0UQqu1FWhKZ0exG24rSSBZShwR9RHY4Lm0qq1OemWvKEqPHccS2h1Mwg61kBPC0g8kjm1uQexw4U3KVVYa8TIy/LiNpHnSuoQ3bc/dTaj64yS4nTg5S/ZerpeDcYnANPETfpZN0dL0NLcyBJejuoIKVs+Ut2+4wjnU+O8049IjBxwouVWuTx74mqm8iU6FJTX8yPMztjWxHj0hTrJcC1lTbjoe1i6Eo0LQ0RuGyvINeIrOrFBa1GDNUVIOtsqZWDfuL9x3GLYpGTNzM1C4uI0FThU5pqpuVw5f7JLJyCuAuBGFNTJkPsr0NRUuKUmwuQQOexvx98IpWT47TOuVBdiq7/ADA40f8A5wMTBnpe9mnI7GfXupOUYBbkvtqpkyvO0+qu6CG3HgVs7SEBZAF3eb+l7YiPhRTgGonUCvRFNEp2vxpVgQbEaSben/04mg3CxNd1Ka2ctUVDPiFRdzb5VqNwr/z/AN2JP07zQzRK4WlOhEKZFkx1BtIITILV23bAfWoNbP6t+owipjD1SWWIlb/FFlBcKAyyt5KAeXNSAk8cXvcc/rgh2iTIc5qXAhKLyHEONuboAQ4hYINgq9wR6/z9ctRkcbXWmCOdo7RrDbrY2+a+nXwkfDh0t6sdGsv9Ts80usT6xV25AecTWJLDCG9+2yhppaQEfJaN1Ekm/POMn/4SPp5kLp58QMCn5ZoECJEzDlWHUpii+7dUwS32VOWKidxaGmk3vayT5T3xtT4IRRax8PGXWkSI6pFLkVBlyNGmfNhLckqUEOoQbIKkEkAjlBBHGMdf4VGMxB6+ZSS03IWI2RIxQi1gn/1jLtb+BP6YwXZsxqtkbLb+Z8lis012TKEONZRb585ulItzc+p7fxx6kwnWHxGeSABwdKrj8gcEtz1VB5xcVZjlQujykEjuCr7X9cObJcDKHXnNZbAKQBYC2GZcaLKpCTcKdsQRxbse9u2OuC7hUlNkFA4GJVnqiCHWJVQitFLUhZ3kejTtxe3sFcG354iyW1JC799P92I12YXVx0NkNqW60/uoWtKkeqTYgjkEH0IPbEtlON5nosnMitoVGPVQ1PSgBJLcgN7b35bxUk+xdHoeIV/jQvgHTzxhwy/UmKbWCuegqhyWTGl6e5YX9Q/ThQ+4GA4aXURjkRQbfjcMy4ckOApNipsn09bg2P6/bF05FqSapRaW/wCJaC3zMi3VbSzNWSXI7guPkvj5g9AXFWvpOKzqFDkozbPoE55sTGpCI+92Dy3AFNPf7h1BSr+I9MHZCqLNKzVJyzmnchQapIXTKgv96FJBUlqSPu06bn0KCv0OKXjOwpmd0qXdZKZAdqEHMz1ORJEiHHTKWsEPJQhYCPPwQQVlJPqO/YYrh6nohV4opW3JZkIW22xMs40GltFKkquQTYE25vfQRyLYsvPip1SyNOpVYGmsUIusytB1KejtOWetbvZaR+jh98VPHrMZ5F4y2pCI8pt3YdTcuWXzYeoskA2Pqkj0xmpiRfyW17BlBKX0d7MMChKoUGJeHTih51SFB590LCXNTiSQstkAAlB/dHAOFjNYZqzFQpr1RipkTIrkUPVd1aWVNlxp1orDdjugtN2ukiwvwbHDTUlvP1N+rU+oxilZcaRHjNhCmtt07QSoEXSW1ADUOdOgm4F2XMqszI8DJp6HJseQltltAQHHY7aCVhAIBO2Cpwj2BsTbGhsQduFnLj4Sp3lZdNzHMp1EqOX98uCoByoQV8vbTanHHE+hXo5BFri3viL1hDcjM39G5ESW6HHXYaX2mk7br5QC053uBcBRB4ABsCezhkiv5hynVYMqpwYrDeXq2hNQ8MEmQ+xMZU244QFbbidsXCvrubduA05+eqNBzrHlZemuxpaG2jHejOWbWEOLAdIJFvIpP2KDY83vY1nJK45XaKTTzLboNRpMN5hyuzY4anR45c3itxtSWGQrsTwLAWvqHe+GnqFSYOScvt9LadOfdfo4an5iKHdImVRwN/KCPaM2QgX7ErPHOHvIdMzXmTqY5RsoNRlVt+XGRAVNJaYLseMpwu6iOLFJ5t7C+IojL9TgR8x5XzVCmsZhbUXSiQsFaZaCLrcX++CCQSDYhxJ/dxa1uXu3SgZk5pjibTKlJivbiIjDc5pSyEl2Ms2C+/dN7rH2UMNWWkPSs6NPIHmQmTHWlIsdvbPP5ev8ME5Qq6oKIk/QXmGUoIjOWII1nU0R7EakkHsThyyXFMTrdQ6PB1vR5kpoRkqVYvNOxzZBP9v9zn1Ti21wqftJ/p5SCXdIG5JC+O3IPbC93zN6b3JA7C+C6vSU5aq71MQ/vM/1d1py3OhYB59yCTg1gpJSr6xfn74dhDm3CD/EoNlJxoLrDdtK7x3D6Ad0nFiQXULQ0lfK7oWj9QR/pxV9FcELMdcghSzZo9z3s9e/8Diawq/AjlnxTwa+WNRJ/wDN+18Laz1HasUrcG6ynSi/7Ro/kQRf+f8APFCwkkUVplVkFIDZKQb/AHJxb6szxYNUepcl9TUtmWWi0ppepso0khQ9O4H54qUNLaMxnSUluS+iwtwEOEX/AIYtIVYPMJxgXIbdPojt+mNxf4MLIAzL1djZykR5DsXKVKqLu6tjS0ia68GmUpVfznYdfJuBpNhz3xhynklKO9lcXP8Adj61f4KvJb9C6D1nN8pLgVmfMMhyOFpt/V46EsAj83EO/wAMQHK0lXR6ahbVSLJAxRfxVVZbeWqHlho2XVqkHXB7tRxq/wCsU1i9BwOcZX66VM13rG1Tmiot0GA0yRq43XSXVfkbFofpikK6Jt3Krc5oS/UKNl3VpbSguunvpB8t/wBEJcOMyZpzV+NZqq+bqSrciyIa5EppTO98lxYFtJSpBCUNtm4IsLngYvHqJW32hmuvQ3buRo3gYR/96spYT+nLqv0xl0GfR5dXkyVstUmvMuNFxEkNbSC4FbSdB8iNwBCx/YKvY3yT991gtpe0AlS74dnKlL6j5hferDcmAumOvRY6JCSGCuUndCWRy00l1LiWwRoKPoPBx9DPhkWV5frTOq+3V2V/lqjj/wDJxgroLJjKzbmIQ6RDgwzBjuwGqep3w0WN8tJaO42m7hc8xWCdVjquRrO5vhjnsMwMyqkuBDTEqFIcUo2CEhDtyfsAnFsOrdljc5oN184V0t+TPlplvNsNIfkJUt4pSB89fvizMofFnU/h/wAgy8p5ARS3n35yqi7LmI1gq2ktlsNkgFNkjn3OKp60ZvocOs1enURAmQmqjMXEfsNDwVJUUugHm1jx+WM913ManEPTZAICQbBXb8sJS0vZ/wA3mV9H434+m4gjGHw2EDbfEjRWX1P+MPqTn6vozD1AmQa7IYTtxUPx9LUZu/KWW0EIbv6kcn1JxW+ZfiWztVoqYtGjwqEko23VRG7uKFu2o9u5xU82W7MfW84u5P8AD/wwlT3xrvbZfMrJxeq1SqDqlS5jzynDclar3wsQy4pIvf7A/wCjAqBSfHvIK0qKfqV68X7jEpcpCm2/oWUcCxHH5XxfEbC5VblHCX27aXr282knjCqLXqhBSf60vm3F+/2w7LoyBZ5SdA9B3J+18JJ9Ib21LCQkex98PZpS6hPVB6jyEuphy3nU2ILLzS9LjSxbkEc42R8MPxxZw6TVtil5gmKq1BqLoDqHHbJPFtV+dt3j9p2N/OD3HzzeToc8hWD6fbEuynX1FbFNqTqNDpG04f3T74zuAd4lcHL9C+XOsFMz3k+mZzyc407FkzvCPtk65DI9g1xdw+Q6TY6CSnVxdyfz9VJNOlvU+gIbeRGlOtiQ8eFNbY0qSEEhV3TdB/smxI5x8nfhX+KesfD/AJ3bjV6W+7S1luNMjhz9sxY2Fu3GoKbc9LW+hRGPq/0+65dLOpOV42cMt51pjkGQ2FKS9JQ07HV6tupJ8jg9R/eLHFRY1qe6E/1CcjSpaF01a0RnEsNWVoKnAXwoLJ4BVsjQn1Kkc+bgcHN8itVcUOoUYCM+6WkKLiVc2eKSQORzHcv2t5Pc2kKM7ZMXbbzdRVX7WqDR/wDtsLWqzRlpBYqsNafTRIQR/fiAAahAqNTv26/QgkHCYBV/qwplaVSHVXBuokc34vhOArG9p7oWJ268G1H74CW1BXbArKx7BQXikji2G2v0xFUotSpjqAUy4MqMq4487C0//bYclDAHPK26SfKltwn8ggnCud3StNK5zZ2HoR+IXxnpzK1QYitPKo7VlBKu+2jG/PgAfS50zr0FZISzmdKgnT9G5BZV2/zL4wFTpLjVOh6XV3TGZ51d/KMb2/we7rruQsyvPPKUj+k0ZOoqKv8A9XtqP8lY4VFczW8iv1V7V25uFy533mfiqEy3OmSsqwMxJpVLhCXSmoTzkpxuQw06S6443rWHLXDgcvwVEc6QnBuZqzkqZ0jSxHqOU8wVZipGqiBTqpGQu7A0pGw1YuHQnkOCxBJNzbGdKfT24xp8SZk6nyIQgc1OQoQ5rhFypts6vIu9h7Kt3w95on1qfDZp2T8o0zL9PagsNOuP0eIuVMfWmzqnHw04WkG6gNtaDYA+dRIBdh4kkuSvzh732cXZtj57qS0Dq5lun5VzVHqNDmozhW1SfwZb9JR4RO2z81Z5AbQRcIsLOFRsBwcJ+lb8Gr/DxUUTpFGy28JS6bTG59SRHDjrSm3LutE61tpDqkhRBR5j3AxFsyP5onPOiDLrEuNGbQYbTNSKGGyhCQ4tDQOtAUSSNwD1A9cEZLznn6lxZEehorBkSZLTrbTTkVAecQohzceePka29s3URYk2B7jZJQdtAchGiwtq/d5bOGimPTBNFi9S6EtqrwZEiSlqEtukONreTy0ouBY+UttKwooXZZIbXexSBiwpeeM49OOoPTvL88xECqyIeYGpDDwWTCdnlLIS6+CtsqDiS4kjX8wg3IQsRFVBztRK1EzTmR+RJr9LqTrcWRGqLRfiERnW0rUXXACGluNlAb+o8aQFE4jVVrGXKHV6zMYbfeq7bxlsyUNNqUVoLZbdUoFLaFkpJ5AJJI9MYoaPK/s3m55J6upZM7tGCwV557zvVMu0UZholDbpeZM6MyVKYVveDpxBS0G1JWoLcs4l61ySNy9iEgYgfVWouUrp/nWGvKs2oMOSZNNDs9zTGu5GjpfeaAALbiWi35E8r3eDwSmhqxmbPecBGTmCpiSaY2t5gPvR5Hh20EqceFg4sALJUtZt91Ymon15qgPpm5brco5gozsR5MV5wEvu6bTtog3JaioShzQB8ooBvjazDnQP7Qm6ze8sk0KVZZqDlSpUnL82BvNK2pMFbJC2S4Fp0tvjuEcXb0nW2fp1gkGW5GcbdmUl2pqjpWKwihrUZKH23VrBIWHEKusNhxSVnQdKGeTwu0c6aR8y5Socmow8i1iqrbmEuxXJAanOuOMqS3NY+RYlk6iUuBBFkcuekji/iuW5jFCk/EE9l59wNvRoND6a+IVD3LqSEvPvMI3fOS4sXGtR83NsK+lsS1+ifthbuJdFcRWoNblwaRNWWPEORHPDPJTUvCOEubF02skaSbclCknTYk4BHzNDYcbrG9DUyUBbe5IP9Ycun5aRex+rknn6R6jFi57zBlZjp9Oy8zWZma4dAj7cuQ+9NgSGZs+KWTIebeU624N5iOA3GWENNKdbXcquaHomcK31HYk0kZcQ23PiQy8+qsKSWGEFJZWyhoHzktd/OhXnPFhbC+gBBN9ETMArEj1WZJiqSzLCYzbQJb2BdAv5bk8kEdj+XOEcpZmQlQJrq3Jm3ubTF3Cm/CfS1v8AuOFWXcqxP6KIpSHGHUtyFyzOdk7u1IbUoNaR8tBQAdJ3ATyoi5taR0Gj5cjQpj8NxyYtpoSUGbVA2FhtoBSG2m3QHS44FuaVXsjQOVlZPFmlhogXBw09VdGO2IAUGQzPpDRakQpDV7uHW1pS4G7Xtf8AO/B5AF+2HCPW6kw6luTJCorq17boUAXVki/lHFwSObe36ra/l5moJhvrgvtobio8SuGJMsvIGk38KHS0Co3B9bcIH748MzUrKzrFan5ipLNMLzTdQfqsANPpvJbcsHHNLTDZACUeQuBbYJJFrboI46ghxduq3PGpCd47VQaZnR3qTVGHY7RbkuSqeWmkwtAadaUHEAeXWlV+XE/LItYHDllel1utZln5k/AqpUnFqXBq9QnSGnZjkItxRFspZbLgZXvNuFKCVIUk8kXwZ1W6y5OzfkGWnJ2ZosqPBKE+JpdbYeVDS5dADpJcQypVtQUpsoUGzYEkWyJPoVRztneVXKquTL8U2FpqVQjtNJlrX8ptptwattsOJ4CR5UJIt79VuGlrsoOiEVSw6vGyurMuaGma3Ho9UdmZPkZfNYjUxwttzX0w0Pf6rQybNobUveShTvbbuL7ZOHro3JquTKdA/wDSRGr1MpsZcOSqmxMoVSqPPl9lTgfL8ezTqA1tfMG4FF4emKHocHJEms02mzZUm1UkttBBg7d4kYFL7bzeoaNRbdAtcq47G9/ptljOFFyn0Em5kb6i5TpGavwtE6VUIcz8TWp1Bu0hyHHfZ8S7s7TKEpIOsjvbGqSMMOZm6R8ubQqjInUPIVDyoTlnqBQJdWnTXFR3JVGn016EtxW4luS47DdbBFtBDmhtOtfbk4gufuoecM7ZdGXqznKmVYFtCXYzdBhJEYrNlNsu3U6LoCRuJA1AkDjFX5yr9f6q52X1JzJVWpNJbLseLCEx58wlgk+GaecLhdQLmx4IWVkg21mYZN+H1Ge11Cs9QMjZ1pT6ICKhGlUTMcAQm4AaUW5RE2KUOIGkj5bt1EHgc2Slwqlp2uLdMxzG3M+qt/iM9sulk/dQIcqrdNsjZWyxXaciYw84+86mW28mIhqM6nUoIS4T5nBxoFz+uKeiGLHrFKzGx1WoAbpxXJZRMgVRt5bxbUltZtHKAgagq1/cc4sHJ2UNnXRumuQ8xZxXUxoDPimg4Ii2SkuS5TLIQ3qO4AbBCbhCFrIuZNnHo1TcpUHK9UzZk+q5NrspLaabRns0pqrIpzTYLi3WVsgxiFutt6S4Va7nmysGOkpmcru1TPiqXxe9AAN23F/gL3+irVeW5aKHT6SabHrtUr9aW3HD8ZThfus2sjcaQXHFuNkklsJBWeLGzzVstdTekMuk0+udPqBTfxN68FTUKO80+5HWFWS43IVoKVrSuy7X9L+lt9DYuUZ2YHG+qWR8yVTLk8eEYC8nSV0hLoN0SBUW3ra7A61LA+q3FicaIa6OfDg6B+HdNKiUqF3jAqsyFssi1zfx6PWwFv4Y2xGNzfBt1CyVtNPC9t5PEL6HT/dYvjP9XOp0uoPU/pe1NkR1sePXTwiC7uLbJbQv+sBDhIK1AFtRF/viyekteerNLW7JjrjyHWg4+ytZUpp9p1Ud9tVwOQtCb3Fx2ONKO9Ieic6pzZcmBnTL+6Q3upr8pqMpbbaCQnRIeR2Uk9hze9zimM9ZMyH0qz3R6TkTNjtWjz40l2dHlTWXZMV11y6VuBDTawl22oLUglS0q8xxW+OInM1oB8lRG2XMSdQnZvlNlYbEMqarlUo6F7aqxFFTiqTxoltH5lvv+9jrFSv3uBhNW56orMOvs/tqJKRJta+thfldH5WI/niuaI5Vop3jNlPNbv6eZnRnPJNFzOkWXUIbbrqf7DoFnE/osKH6YkKhdJGKL+GLMDQbzBkjcBRAkipwbnkxpHcD7BxJP/KDF7etsUt1Cj25XEL5bfHplqZk7rc7GSwy3QsxxjmJsHhC3VnZloUSQhNnQ24TfnfHfnGcKME1h+bLelREQqcNpxthxzVuXA2gUJHn9tJtwcfRD/CedLl5v6HwuoUFs+NyHU25b60i5/DpFmZHFudKiw79g0cfMnMNfRl6hxmW4aGX5K1yVx0322yu6uR7nvb/ACvQYvDi+wVZFtk75prUKBFEGmU9uKyhelttLYA/h/34jCc5V6hQxV8rTZcWtMSUKjyo7d1sjnUeQR245BHPOE8DMLNdoDkirBEeQHbknUQQHAk2uSfUE9+CThqqNTpcTdZh6pG15CpPCSb2IBPe3v646Yc0x2cdVms4OTlRus3UNmoxqovMMOZOjvIktLepMYKbWggjkJFhcJuBYcemJDmbrxnzMTcjxJo1MnzELEyrUmlpjz321m5b3SpW2CQLlsIWfU2xAaa9An6kvU0JUghYcUpNk2WLkn9xFjyfXtidI6UyKmaW3Ssz9PlSKo0twR1Z4gxzFAWoIDxkBsNlzT5ACvgpK9NxjAYGEXK2sqZ2G7XEKuZaarmWWFS6k+8QAC/KkLedCAOLqWSbew+/GJZ0l6TwurvUil9N4lYapDlQElxyoOhx1lpthkqN0IBKyTbkD19MMtXpVZyhORT6vGhNOPI3UpanxZqVoCym4djOrQeUEckHjtbDhkzqNW8h5iiZryg/JodZgJcbYn02c4y+hDgAcAJChyO4tbjFrS2MaLNKXyuzPWuMqfAV1VyvUZ0PIHWultSYjW1JSmoLSh7WNJc2HoTiBzb1uFpHNwkjFfW7o9nHoln6d01zTOos+pwokSZIep0tx9hXiG91vzLAJJSbkkfvd+cX/Rvjp+ImGk6urWYpK7hY8cxTZYC78EF2KTxxbn34xT/UTMlR6r5wqWfM51ao1Ku1ZTRly7RmgQ00lpttKGmktpSlCQAAB2wj3AqcrWVe0BhcaexJSlJcQdzaKzoXbkt6hykKHFx2v64vHKcboRmuhQ5XUqqxcv1VmWqPO8eVKIjCxQ5HU0geINvJZxSSm3c+tPTWIdFkF/8AFGEBaPlofUA4T2BFsG0yep1DMJTkeS46SkuSFaGAByNRsb8D27kfnjDNTMnN3HVdrD8er8LiMNO+zCbkGxF+tiCL+a+j3+D2yzFy5Uc1dQMjIqVZotcbZocdFMjpMVT0d9Tw1uOPJ0lDK20mwPLx54IxU3+FAedX14yjJnUiRCdOTgJDbjzbiS2J721bbUq3KnAQTjVv+D8p8Cn/AAs5dikIS8qq1R+c2BbRJXJupv3FgWxb8sZa/wAKbZHWvKThcKkDJRQCCAEkVJ4W/wC7DiIRQhgKxVla+ulM8xu4+VvoNFiJUaQTJCJA8yBZJ5LgHcfa3r6dsKYad1oIeN0J9NHphrgsIRIK0PIWpoKb1pbNlI9bXtb/AE4foBQgo3L6LaiEi+A3ZZWq6K1Mi5hkSK8whYU+yWanHSex4IeA/QXPuDivJtNepwjPKd3ErRwu4Ic7euLPq7KKTmxyq0FrU2+046YykWuwtBLjVvcG5xGZjDMYxociyqdPDhiOEfsxc/LP3T6fY4yRGyuKhEllZDEgA2Wgj9cFixUO4Vp7W7/+OHltloS1UWqksXUtpt4C4Su/kP5Xtf7E4Y3w5EeDboWl1DpacSTctkd8aBqqnZlI1zTWqAmfvL/GaEzHCVbn7eEFm/3JacUkgf7WpY/dw71KM5negT6+y0U1Qy2nlBHd18IAPHqVCx9rgYg8SYulyGZrSdaRdtxI/sEcj9Rxh6gz5FJaKqesJYmtNqadAvZaP2ayPyKkke98I9rjsnaeqlNYzbJreS42aYiEfi8CM/EmLICg8UNkAq9yplQB450j2xF69CpUkorFEjIZhOkLLba77NiOAPS3A/h745OnOvCteDSGolZYW46yB+wl7at0Afn5h9lEYaYcp5ihqkx1hyOttCHhfsSkc29AbA398UMjs9ajLdmVMNOflxJEF0tMuIdFUhLNh8wakhYV6mwcFr3seRa2HiJWpaak4qTFY8K0UKEVbm60pBKrjz8rGjT973tiJRH3ms1R5SXl3allSlAiw9Tx2HHF+2JVro06p1GsJrjDMmZYMIW8gEtoQLaki1l8gEetja+NTQs9yVypzKbFpyqbRXXUNvr39t51S1Ajktm/qFkkLuSQfSwxKE0il52o8OqU19iHVstNoWUSFnZeibwdO6RyNo6+bHyKFxxiK5koBg1Bt2+0WwGXBoJ1NuHuLd9JHPtcYeMp0+tVuVUar+JAragBlSlNIaEhsDZLYDaQgrDRFyeSOSTfFjfClL7q++hEbwXUCdJmtsRnKXQKmGlKbLrTTrlmgsqFuDuLsq4vf0xUXVSn1ydXpmZ6slqJImyvw6VslWkOtR22kkK9Q4GyfzPfFxfD7UsvwqjmBVeqUZmTIpUaDTW3XQnxDnikgtkL8lgNJIPcCw74a/6E0jMsWd0xirU2ymtSIKm0JILZRNOlYJ54BI7+4xme8Rylx6K8NL42hqo6M7+J0qDWlwzDkLiMRJSS1t7zoKi0+n33WwD+aScIX605RM+5VzYw0FO0aZTnXUWIDmh2znPse1/vhpy027T3qhS5jiy7HkoAsrUCLXSR7cgn7E4U1xUH8Xp77ZbDVUjt3AXzHltSbEfbmxH2c+2NbcpZoszrtebq1M+SIys3SRAakBmTFbebUrhJAPCk/ZQsfzvhBFW8qOt5AVrjoW6QDzoHc4YXKo87PlCTKLiY61stXA8rd7gX9gSf54UwK8zClIdXtqTqKFJWRZxCwQR+oOJCMkYZ0SSHM/MolNWlOfqq8yr5Ull8puLWuA5b9DcfoMNcupyd1lyPJW2ClF1oJHvaxx6py47ebXXUPDw6ytKVjmwW1p5t+Qw0VdEyFEYiuJQHxwUocDgta4tb7Ww25S+JPFJTIaL8hyepCF+ZwpPKRxe1/v8Ar3w5yZDszzLSm55JSLEjj+WITFq7wUQ8oElQ8mri+JdFlLktbqmVq+WvcAHPbv8ApbFoaSq2pZT7nb0IW44m9koFyo34A/Ptj79/D90+b6WdFMlZADZS7R6NGZk3N7ySjW+r9XVLP64+Y3w9f4Pv4hpHULJtfz7kODSMqsVWHU57rtdiuvORG1peADLRcJKrAWJFr4+vaRxhZnCwAWwaNXFqCUlSyAB3J7YxGK8msVnNPUFxZKH5MuejVzZsX2h/zENjGoutuYzlbpfmGpsqs+uGuJH81iXXrNot97rv+mMeVo/g2RmKa0Sgz3G2T92x8xX8dIH64oIIGnNaqVn2iqvzzKeh0egR3ZIjtyZ79Vnvd1IYjt7TZ0DkgvPOXPppxTjkOZVI7+VXqihUeoB0sWSAncDTxbFjc2BveyxckXPFsXRnnLVaVmYOis0J2C3R40MRvE2eYPLr6XkuJSjWtxz91xYACbi+K7RkHOCM9Ra83OpBpLVnnmnqzHL7biBYlrQkXBuQefuO+Mg1kzX0WiSN+SzW6nyKaPh3jtQs6Z0eaLTDngITcmInXqS7uDddNyQdS03vYH7WxrfptnODlTp31YqdTWS0zQWC2gG2pxxTzKUj81uoH6nEE6T/AAfdSaTUazmiRXMhQo2YIsVbLLtdUpxBSpSllQbZKAFhaTwT639MMHXPK9e6aSGMl1ms0iSuqMNz3/wiat9nabcIbS7qbbIUFgqA5HGNkcsbu60i/wC+iwyU8sY7zfobfVY+6hImKW2ZzyHXi7tqUn19h+QxSud5hbeFMQkEI/aevONC5yRHioddX5kRW1ySskWJ5tYfrjPVTy9V58CZnCZZmK48tDbh7qWP3QPYC3PvixZ3nMVEXm1oI1C1wD3wAWuPL+mD2Ykh5IWEHSOxOFUGlSJkxERlN1Hkkdkj1J/IYTvI5sqtnpPliVV0iSjaZZYbClOONrUE8k20o5N7enbjFkVnpy3OQ48xBcTsNAoSL6lDuD7AkHgc98Qug54Vl6ixqJT5CGmYzXCmwQHODfVbk3+3oAMO0fPNRl09DMaIsrjpDbTiTpSkn1Kb27AC3GHS501VHIk6mMtuzmjtOEuoUk3AHHf/AM/34aJeWVTm3HyQGGgdJFgAB/cfzxOXK3VsqRUVbMNQSIUhwt+EI1qdRa4DYJFhfknsecV7Xc+GfMXIi01xNNWSPDbpa7/USpHc+v54N0Uwu5HZTHVUqtUmIDK1bbLNip58/wDu0+v59u4wrpeS4s7L8KsU3xJSt5xt9oj1HYgdxx97YcKBVMq1kvtS47b8jcvBTPlONhh390qLaVbrafYgHnvi0KFS8qZUyXHgJzAmW+lx6VLS3CG0pKramwor1ghQNl9iDoINr4S5JVjWCyYM+5Sk1PK1LznBcQZUSK1BnIbVYloEhl2178X2z/mH740d/gsc8Vel9aqtkebTqlUqbWaM4XYzLCH2YzrbiS3JfB5DYHy7pBN1I9MZ5bzxWMiu/hDusokPbbYeB/ZH62li3II4sfX8wcTrp4jLTtaem5QlSoL7zTkV2OFFpxtDliENrQQHG9aR2OsWHB5xH2A7wutNFDHNUsilfkDjYm17DrZfZSTlnL7w2l5Lpe2UlOlVIaIse4+j1xFJ3QjppOQUqyO7G13BVDclMGxFrcKIA+1sfK6RmDONNWtLlfrCNC7XTOfSAfpKVefy2/s4E3nbO4aLsXN9bZHfyVaSk/8AWYxivbHoGn5r7PH7DqmoY2WGvY5rueU2+FivsIxCdjsNRmIT6W2G0NNpLKuEIAAHb2GB+Hkesd79WiP9GPkEx1H6hpUnb6h5kF/7NamC3/NdwrR1Z6qMp8nVHNYA9Px6b/8AfcO3FW7ZSld7CK77NYz+ly+uCmli6VMrH6YAEpH/ANGPk2jrX1jbF2+q2bbp/wD27N/++4Mb6+9ckW2+rubkj7VmUf8ApKwwxNjuR+io/wCRGK/ZqY//ALfovq+rQSLmxxGOptbay307zbmFcgNfhtBqEu5PomMu3/zFI/M4+Yx+ILr608UN9Ys52H/7VeIt/nE4TZg679YcyUSbQMwdTczVGDOQpiTFkVHW062fqQpJTyOMB2INsbNP0V1L7DcWiqI5H1EZaCCbXva46hQ5mI4iMxHJRrbbDZur2QE//a4+hXwC0owOjr0h1KgioZplrBPqllphn+9Kxj50o4dbXIUdsnUs3/dv5sfVP4X6A5lPoPkSJKa233qf+LSAU8pXKdW+Un/4iR+mKKCO8pcvYe2uZtLgUVKHeOQfS6+WfTPLecOpWWnoVErMZTNIEmdNStyPEDQj32gQsFchZWU2JsCU2F+bR3KrmWKjHqdczIzmurvyGS+XJFJSp5M0DUQFElttA4B0r1kEgjjGrGP8Ep1TeqEuXVupdEeivNuIjwGatNaaZWu9nNQj3ISslYbtb0JthI5/gdM2JjpdXnGhVOQAr5cupusNG/F7twisEXv3tcWtbnG5zr6L8zdo/mVmSmZ6ZfpjUVuoqk15+qJeCUlLajDD6Q4ggkoQktKUkKPNrE884Q9OadJNdaj0irQ4BfZmOx6bEoz61qjxor7yZElRu2hDi2wgt6yQCFqS2Bzrtr/BQdQm5rcmVWMsSLQ24bh/pNMZ1tpGnbIap6bi3qTfCumf4M5FDlRZTefsmUeqQQ+wVxq5N3UtuoW26hS1AFd21lJBA4Kh64HasgbYkAKBj6h1w3Nb98ln3PHXpFarz1WhZUmMU/KtGaVNkGdMkSJUZTQiMrddl2LqStbamwkhpsElJPcw7qxVq6MrUxhlUZ9aJqI0k0aS3JkzQWCCX20EuH5jR7iwOnnkY+gNE+CLJ8JUJdYzbk2UqBCapjTkma9OUiI39DCPEKvtj0bKigXPGJm58P8ADitiNTc8ZUaT9KnPxdxg6PRKAyElA7caz/IYyvqIu0Ds2o8wrvdJnNtkPyK+deQoXWp3LEin0RxmjHMwQaxJp0qWt6bBcPEZcVhlUdpACiFjg8cWF7ybo5A6wZErVZqH/oUl1JEmK6gqqk52EiU6bJbeSlDyXELNioNuGySQvUCBjdTvQBhfCs/ZPkKtqJk1iU7Y/bWo8YD/AOguQV6l9RcishAIRszHE6f4WH8b4t9+ab3ePmEjcPlH/tn5H9Fm6rVLrz1VzPJzVnbpFlKh1JxhTJqbrsdKHkFX0OtNhRWscKQ6Va/y4tFpfwu5ozG/ITXercuFS30RkLpVBjPFqWI61FndU47cnUonVrJ5+2NlQOkDEFJ3Op2UFr06VOGcSb+5+/3uMHudHqaoKWnqrlgPLPK/HH/8rCGriO7x8wj7lUjaM/I/osmUv4S6TFguxKlm7OE1hcRiK2wtyLBZabauW/RTjhC1X1E+a3N8WPD6ZZNpryXmaRTGFghGmJBFy1e5buClAR2tYHF0p6QR2U3R1UygVHuXJJN//mw40joLV6u07IpWe8t1BKF6FLjhx1KTbsSlXfkG2CJWS6NIPxB/NVup5oheVpA9PoqTFC0KKo3jEN/swoOkqSj7AC3pa1iMHoifhbYWKnXAb3KlPLa4HYIIaBH/AI8Y0APh7zOptLTmYaPpQLJWliQSP/mtg6F0EzdTFh6m5so0Zwc60UnUr/nG5/nhnQsdyVYcW7KhmIlZVBedmz82KS4gLa2ahOUpNuPOA2QL/f8AhhjmQakN4Vg1iXHcBBROcIKhxoKt1o7mn0BFvTGoJHQvNE1zen55akOFOk70d91I/JK3SB/DAWOg+bIxIjdRVMJ7hLUZ8J/5u7o/liNja3kjcu5rGGd+kOQOpjbP4u1mBhTTa2rQHHKaXCsgm7iG9h3sLbtj7WxWVV+EvpXQ6ZOpOX6xn6gVZ9tBWZ8ZMpt9wXLJdR4ZPywe+2oX98fRKZ8OVanKvLzzBcUe5NARc/qFA/zwmV8L09Q/2dxx3HFKc+k9xbexa05dtEOWUr5TT/hz6wUiWmfBzHRaguAlSYbu7KjOrBuVOFb6Rtr0qURdTlyeeLYUMUTrxlc0es1uu17ONLYIkTqMw94tSXxfbbaILgdQk6SXBpIAIA4x9YFfDeVxW2ms2IiSmeFORqe4GXE/dtbxsr7pIH2w2VnoUxQKfIqmYOp8SJDYRd2RJp5QEDgC6i969h+fAwHzZQS46DmUzI87g1gv6br45sdQINBypBo9QpFWbqkCVdbMposAqcQRZvc5ulZsfKB27m2JlRevnUSl0B2gzoEmm5anuBE0rrrxbaRxrUWUvK8pAAJ0gW4se2PpecudKEJLT3X6juD2dioXb7C6+2GqpdMfh4qzK2Kl1TyjIQsWWHqDEdBF783HPPvjL/FqVo/xW/MfqugcDxB+9M/+h36LJHRz4qcl9Jsx1uqxs9R6fJnvPjcN1tPRy4HE6VEaCCCCEi1rehJGAdXOvVO651Wo5ng19t+PGpUCnIkhrw22gSlPytIsLkqVdagR278c6ZHw+fCVFcVMhZ9yPCmuE7kmNlSnBSgb3H0Hvc3974qbMnwQfDTmHMUivs/E9TaLvxvCIhUfK8SPFaRt6SUtgkalC5Kv3iScD+L0POZvzCb+AYm7X3aT+h36JB0j+LfLHSWkuZYldQFwkFxbhjpgeLZUD7gKFx6dv4jFrZf+NPpnHmJeh1rLzC0JW26tFBlx1a7jyEs6uwHa/wDDFfD4Efg5/CIkJ34gXGanFc1rqsOKGn5af7LySpTZH3bS2fvhsHwBfCqmNIH/ANVVWlSlrQWnRTY4S2Be4WkIuu908kg8d+cOMdodjM35j9URw3idgBTSf0O/RXlG+KnplmGKn8XruT5Dm666ht6fPjaNauB50gX0JT3OId1az70/zdSKacoQsmRZ8GUiSuRRnG3ZT7aEKGl12+txI1X5vYgYzR1I+GzopQa6zlugddJ0VNNgx0Pymi/IE91aSov7bqdLJV222yUjB/TrIXTnIdXVUh1wlVpJjOx24s2I6y0244NAdu21dekf4vsT3xpFfTPbnY9vz/uqXYRiEZLDC4fAj8leLM7m6CoA8i/qMK11BOwsO+ZogtupV2LZ4c/kSf0w3ZYoqK7Smp9BrEeowwS2iQll1oOFHBtrAuB2uLi/5HFzdL+glM6hxaixLzLWaZJhoRqS0zGebcQ4CLpK034t643Pews3vdcjJJE+zhskPQLNL2Xc45ckynyNiS5lqoG/dtwgMk/54axtwcgHGGeoORE9PeqdVyjTg61BqtOj1GluFRu2ptIFgbk3SppXr7Y1700zWM7ZHpGZFKRvyY6UykpPCZCPK6P+cD/LHLZobLo1DcwEnVL87ZVpGesoVvJVfZD1Mr9PkUyYj+0y82W1/wAlHHwxzlkSUOqych1aWthygKkxKpIfZW4Ii4zhaeUpKPr/AGWof2txHvj70kXxlvqd8COSuonVyt9XWM8Vyh1CvCL4yLEiRHGCtoNpUsFxsrG4lpvUL9wT641RODTqsZvbRfJaq5Obp0idS5VRcTKpdUkx5ARGUvcaG3tqQn0Jub3PtiLy6NMqE5un0xmW+4sk8shpIH3NzYf+RfH1Urf+C6yZXK5UK9I60ZybeqT7j7iG4FO0hSzc2BZPsP4Y5SP8Fr0/pT27I6u5xnNALuy/FgpbUSkpB+W0k3F7ix72/LFglZzSZDm7y+YtBy1T6fW4jEOsLlVAPGMnTQEToxcIUlaNtx9IdRa97oI8t/QHEkRlWiQpcliVFyzW46WlsJXJo0+Glxwo0h3VDlpc8q+bE6FEcixtj6N1T/BzdC8mUmXXap1NzbTm4zCx4wuRf6ug2B20Fki5A0jgnmwxAY3ws/BfJCIz/VbP5kqWlIkLJaCSeObRwLdsYZ8To6UhlTM1pPK67MWEVNdmfh8L3sHMAn8FgdfTmI3cx42WGkOHURGn1iGCbeocacA9b3UfzwzjJ6S/NQ02hxLAQfkZjjANX/39lJXf24ti6/ifofSnoF1dqfS2kTM0ZnbprEWS7PYrjcRbKn0l0MLQth1DhSjSSpOn9oOLg4pePnTp/wCHeceXnGO684VXRJhPpSj25Q2Sf4X+2NjXtcA5puFyXNLNHbryMjyEJQUw6yexJamUySLf5jiThPIyDU1uf1Z7MMRbZBu1S21cffQ/Yj88OSM0dMpR0pzVmBi9wjxVBjO24/8AdyP9GAN1DJbhWuJn2O2FnkP0GU1/Et6h+eF8SpQGo2cqS2qNBzJIhsrcCnkPZYDoUvjmzm4QLAAgG325OG2vNPOQEtzZ9IcqDS1vRBCywzTJDrpASGitAbu33IFjZdjh7L1KWzpi57oDxI4SozGLj7bjIwbHGYp1E2cqZvcjOxX3kLZiVgIS5rCVNK2ysBXJIJsTgGzBdaYIXTyBgIF+uy29/g7KvKyt8P0qVVpMh1FdztObjoUSopDEWM0QLnvdBsB39++Kr/wnyT/6Vsjur0BX9FpIUq/b+vm38z/PF8/Cu94fpDDy9+CF7RU5L0hCKkzGVHllTbjoRYKOhKyQCkE219r4ob/CULnTc39PFz8uRIU5VDqDwdjzlvEQhJQEMrStptAKXNSgpPfcsQLYbeO6yWs8tWQoyJhcaTIUt0g8rcIv+uBrLQeUhtBSdZ/e/lgpjfSy1ZqQ1zyhbfe/rf29e+CtEtSnJLiFoYBUoBDZKhYWxSNFZmyq7K5mBqlSos/xranIEgNSHWgVJDd7B0D1QQrnCSoMsSqfJZYktRorsnxLPzA4WuQSUkW1o+rk+lsQJVVYlNvNzac/pfF2JCmjYXHBUCLuAHkaf9GE9HrFfyhTpNKrVNgVeHMbShLQlXsgXUmx1JcFiQbgixCPbHOLJRq1XPd0SjM1XjKbjh99CZsdR3UpN0qbsFBwHtYj+YwmlZzoley9qceCalTOS6UFPi4xIPJt+0buTzyUH7YXyanRKpVyaJlujU+mIijU0+hvxBPqk7qzrJ9VJ9v1wxN0/LMafJQmGGlDWoGLJWkE25I5UgenB9/XFzJDbvjVJmTNAzhAMVCJKgkFxYJPaxJthfCzxBjsvUp/57DjawFp5LZ7pIPvcfwvhmqFLo0de9DjpcRwgqdtwTz2sBxjhobry7R/BkgA2SSlQv39PTF2fqldKWp7oec3EVmPbQ6l11oFKr6VLvY3/wDNrYRM16XlmoSqcyI60ONLjOt7mpOjmxBHBKfQ4b5GVZcdbiSlpxu2tGj1R7/z9e2ED7DIWY/hlqLVxy6L39vv/DAz5tkvbutZIqvUHxPbkpLCuASi90mx7KH3xOckP0u0nNS0x4Ew1BCITqUGQll0p/ZrZBvpV24ve5xC0xYLyS5JS4nUedK+FfnhXDh0ZkoMpqYlFvIG2yod+b4ZxuNFY2VXXm+JQqfRHcsKr5dnrqjcqLpjSGwiPtfMZ3Fizn7vYg/L9DcYT9NqzOjx5lEs29HhT3HF3bsQXW0259f2f2ta2KppQWxKbW2480y079KAtKV34PFrdjziy8iKTFlV1bitKJDsYIWbaeAqx/XWR+mHgYWNtdB785urShudPqc6+ibGpkxZb17ctlx1p9aJG4275FJ0WQSld7+hsbY7CmhoyKlkWOuo15qQipSWX61qGsm+2lASPEAqIJ0jc+USRYkCoM7sFT6QhTSZAirU0Dc/vi5sfS3H6/bDFQnq3Sp7U3VTpTbXHh5brqmFA9+AQQfyOMlVSSOeXsdvyV0VS5gyp16idPs40qpO16Tl5dIjSJHlktu3jurKi4EpJUeb3IAtYH0xEq/TKwotOVDQErs42WHQsj8lDsQRyk9vytiZrfcmPsuTqpSg5HWSyENaWmSfqAAJBFrd+eD3vjxay3KQ5ErUtkKeTdc2lwNt6Or7WWlDg45SUAEHEhZMxoD1TLdxUIdqL7zEmIlt2OmTI3NLC0hKEHujkKIPAN7++DosOE7Gat4sOhZ1/NJBtzcX/n7YUmjxnXnHhVHFotfUUJBUPc6OL9vf88AXS6W3tKiVBbwcQS4l0abH7W7406qqznJIiBTZBKobL7aQ2C6h54eZdzqsR6YCmj0paHi8lTG2QSWj2FrH+GHRFPp6GkGOkqA4IUu3N/TjgYdkTaLHYSy3lhIWgn5r00OqV628iEj/AM84bVLlc3moyvKEiK4mW864yGnAizrRCrWBSo2FuxHOJMatEmDXGp0RhW1oWkICkqNrX+1/Xn1w6MVylbCWoeVGEvWADgqThSbeunTz2HBNuMKolWkvKU3EokJiQuyQ6VSXTc8Cw3Qgd+1je4xZG43AVseXMF93+kFbiVPIGU2WlhT5y1TJboTyEJcYTpBP30q/h+WJ2O2Iz08yzHybkmgZUjatqj0uLT2wbg6Wmko9fW4OJN2GEO59VokIubLP3xW1kusZZyYw4CZktyoSEA86GhpR+mty/wDyeKJzdD8dmOjUJC0bcdsKdST9JWQT/wDIn+eLI6pTjmzrrNaQpKmqG1HpzftrA3XL/wCc7b9MVTUasmTVswZl1AoaU4hlR7AX20n/AOG2o/kcYcTqm0dI+d32QT8eS9Bw9QnEKyGmbu9wHw3Kypm+rIrWbswVcthXjatNeSSjmxdXb+WGZSUSGVsJaTdwhtNk9yeB/eMFtygpCJCtQ3ruEd+VG+HGgIE6vUuAlNy/OYSR/k6gVfyBx8nlqZMpc46i/wCGq/QMEERqQ1jdARb5q/QlzeaYbWG0NjQo8aUJQOT+gGKZzfm9yvVaZUWkN+EdUgRUpNztIFmxx6kc2++JT1kzU7QKCmjwEOeJriVlbqFW22AqxT/nH+Q/PFBSlzF60IeXYi9uwvawPfv6Y7vs6wh8cL8SnP8AiHT06/E/QLwPtX4gjnmbg8O0ervU7D4BN/WOq/h+WUrStv8ArhNyo8lA54t3xn+o5nqVQpkOjOOARILRbSgc3JUVlZPuSf5Ytrri/s5bgB4bq3ZDjCFdgAgXKre5vxikoMCbUpCY0JhbzqyEhKfc8DH00r4zZOaJ+22ltiPZIRZRUfXD1R21zU7VMYUSQhTzqeFH/Jt7d7++JnH6OUqG+PxiW8ttttuyWW1uqfWUC4FuEAL4ucWDSqBSGKUy3EpzURwkgIdirde128yAEG+u9u/H6YKrcqoiMyYmhp2IL6r+Y+ZI/T9O+JlFhSKiUITHcQtshZaRHKePW/v/AOe2JIMuZbobLcmvL8RVltkx4C0EucEDWo6rC17i+G9WdIE/NaKUtTW1DlWLlysBdkg6fXuPy4PpgqZftJk6o091X4atHmZjxUNAltYKR3tzcepub39++IFSZTSJC4sl5DcZ0WC+RpBNyfbFz5pgya4oBcRCdhklQJKU2PqQDcn/AM9sRmndLkVB1uSp9CVblyW/mhSLXuSTxzcceo7WwEyj1IyJAmVGO/AqOlLjmhalp/Zk9le9h627XviQt0ipRYzcaYgoBF0jtce+J1/6N6RILK2XXWJhGlzZOpahbsNfKDa/bucLaxlx5tiKhMdb6owEZb7joAbue5N+bn7YiYKmMwQ5M8xY0dGgwlkBsLsCDxa2DA7WqGNb0ZxhRsUOgEG9rgg+va+LBnZHnRwH1vMEL5BC9QNuCP0w4Rq9FNLXSqnBQpLDJCEuI1Dn3PsLm354inPMo3lnq9mB1Qi5rcRWo5si0nyvo+wd9eORe+NAdI8i9I+r0s01rrbGyjUlHyQavRFuNLKgbASkyEJvx2UED7qxQ83L+W1qW83KZiuot5XRZKjYdvfv/fiDtVv8OrJDUxyJtuW1sKKbkdhcH88VupoXnM4L1mDcb49gMXY0FQ5rOhs4D0BvZaVr+Q38t9Tp/TH8Uakuw66KH+IbJS24svIaD4buSANaTpufp7nGlXP8HNnZWppjqfQVhCilV6JN5UFXsbOnGRenWd2FVenO5sdUhqLNiyGKgxHBU2hp5DmlbaO6eO/oVe2NjfFvOzn1gqmT87fD8rMeaaC5T57UiVllyUtDT3ikqCHksEFt3ST5FjVa+MIpGNDnPbe21l9cHtFreIpqKlw2rED3Nd2rpGiweACDrYa67fJVT1i+D/qj0foUjNrz1NrlFggGXJpgebchIFvmOMO3WG7nlxClBN7nSnUcUM7IW8nb06dPdOPob8IFM6qUvp7m6N14i1eJl0gGHGzMtRdQ1sveNVZ4lSWCgo/acEhwj1v87GNLe2ltRW2ptO3qNyUEeS/+Va2MtQxrAHtBB6FfQeBOIK3E6irw/EJGTPpyMsjdngi/zC74hOnTzcevrjpc1u9hc+uEw+o9/wDTgY3L6j/HCtFl9JMYUjyNlSZnzOtEyNAStUquzmaa2U/uh1QQtf8AuUNalH7IOPr+zGiU+O3DgNJbixm0MR0JH0tNpCUD/mgYwl8AHTRVbzvVep85kmLlmMYUNRTwahJQRcH3bj6j9t9ON5Hgcfyx2MPjyxl/Vfkz21442vxlmHRO7sA1/wBR1PyFlO0kFKbDi38MB9ObA/3Y6ClAue4Avih/iF69LyVHcynk90O119Nn5CSCmAgjv7F0+iT2HmPoFc/E8Tp8Jp3VVUbNH16fFfNMHwiqxyrZR0jbuPyHmfIfvVJfiI+INvJjL2SsmyQ7X3k6ZMlNiICSPT0LxvcD93ufQHHL6i84uTIVrecWVrUslSlEm5JJ5JJuST3OFEhqe44VrDrrjilLW44oqUpSjcqJPJJJJJPJPOCEwX9d9tRHrj8/Y9xHVY5U9q/wjZnIfqeq/WPCnC1DwtR9hDYvOrnc3H8h0CI0kDgJsPyxzaV/ZRb8hgxUSZf9kr9Tjhiyh/ij/HHAzk6l5+q9YMnkiyylP7iP4DHikg3BQP1H/djqoUw92se8FLT/AIv+GJ2h8/qmAZ1C4hgvHzaLDva3/djyktqGllYAHe9sC8FLSjcSi1u6cWN0c6F17qvWN3Q5AokZYTNnWBI9S21fguEfom9z6A76GlqMRqRT07S5x5a//byXPxPFKLB6Z1XVvDWN/HoBzJ5Ino30XzF1XroajqVDosJweOqBSCEeu22CLKcI/RI5PoDvXKWU6Fkmgxsu5dgNw4URNkNoF7q9VKPdSieSo8k4FlTKtFyZQ4uXsvU9EODEQEttIufzJJ5UonkqPJJJOHsAfnfH3vhvhyDAafTWQ+I/kPIfVflDjPjOp4sqrnuwt8Lfzd1P0HJdtfHhwMe5+2OWv7Y9OvGIWOEDHuT7YCdV++IgTZF3II9NWOJ8lx69yRhryzWxmSjM1lVJqVM31OJ8JUmQ1IbKHFIOpIJHOm45NwQfXCXOWcaBkehSMwZhnojRY/bm6nVn6UIH7yj6D+4c4qmlZAwvkNmjUlPBBJPKIom3cToALn0Hqjs3ZsoOSaHKzLmKoNw4UROpxxXJPslIHKlE8BI5JxgnrP1szF1ZrQUsuQqHGWVQYGv6fZ123CnCP0SDYepJXWDqnmjq7WTUKiDEpcRavw+nIXdDAPGtRH7R0jgq7AGyePMqvzClKRpXYfe2PiHFfGLsWc6lpHWhb83evl5fE9F+nvZ97PYMAjFfiIBqDy3DPTq7qeWw5klXWf31H/OOPXUPqWr/AJxwYIMzjygY74Gd/ZH/ADseEuOhX1nMzqEVf/3yx/nY9rP+3OX/AN3gzwUwi+hI/XHhAmAfSnFPe6/VTMzqEHU5/ty/449uPAftV/8AOwYIUke2BJgySbG3a5ue2CMxNtUC9g5hVz1J3265Bl6VKRKpwGoH95lxY/uWMSLpf0wmZoLWYczxX2KH9cdhSil2o/fjlDPpr7r5A4ucW3N+Hups0LL3UDPtMUmnrkPx41NdFi+VoDiHJA7hs7Rs33PGuw8hkIf5uoWv/wCf/Ix9+4PwKV1HFNWt15D9V+ZvaDxfE2vlpsMdfXVw5eQ6689k4UzQ2hmMyyllhpIbabQkJS2gCwCQOAAOLDF1fD/K8Nm6VGXwmXBUAPdaFg/3XxSkSSlVh6YtTo1VGGM60tLnlK1uM/nrbIH87Y96W5W5bL48XOdq43Kd/i6y0v8AAKD1EgM3k5bqCUP2T3jvkA3PtuBA/wA84M+GXMLEafX8lhy7LxRW4HqC24Al0A/mGz/nHFx54yxGznlGr5VmWDdTiOR9R/dUR5Vfoqx/TGPOk+Y5OUa/QatU/lPUCpuUOqJUbFthxZbN/wDcnn/k8ZD3TdbY/wCbCWLcWE8qQ1EZU+9whNrkDt98Hp7YLcaQ6hSHEBSVCxCuxGLFkXbAquEjj1vhkzZmyhZLokiu5iqCIsSOm5J5UpR7JSBypR9AOThBnLP2XMh0GTWK1UBojq20NMgFxx210soT6qI/gOTYc4xH1Q6j5s6pV01OqPhiJHURCgIVdqOn/wC2WfVZ79hYcY8jxPxVT8Px5d5XeEcvU+X4r23BvBdRxRN2j+5AN3dT0HU9TsE4dVerdY6s1y8yT+HUSMtaocNS7pRYcKc0g6nDbj0F7D1JcemHRyRmOvRXnJUGZCASpWmYptDi7m7QWEFRWEp12CeRxcHFVJprt1BSbn88SDJma87ZNmmVlGolmSkEkaEuiwSRykg+hP8A34+K0WJR1mICrxZplu6/mcvLL5dF+hq3CHUWFGiwWRsVm2F9vUne/U7rG3+EFp9Gp/xbZ6plOL7JiCltSEOklG/+HsFe2T5tHmFtVyTrNyLYznNoVWjvbK4ckexWhSQT62Fu3OL++KCk56r/AMUWfK/1HoOZ5LCsxPRn5sCgOL3o8dtDLG0khLZGhtHOvtzzfEfLWTG3ypfUibR1NkFacw5WnR7H3Ko29b7+TH6Zpwzs2losLbL8j1uczuzm5vuqccpkGGkrqkpxayAUtNDSO/ZSljj9OcK6fU8uxj/6yp8lmOoLaPhV6yySOCCbX/8ApxdicjP1FjxFO6ydH6wp9Wv5mcxDWoE30lNQYat+tsVn1n6d5uyfUafVcyQIKIVZYUIj9Oq8OoMPLa8qrORXXEXsUnmxIPGH0usxYoe694yoLTR50lcZKjsh/wApTfuLAn+/E9yfGbqbzdNmUkVD5TqyHkIUlKENqUTe1+wPrxbFcwWkMt3cRYdyVmwtjUPwwSukWXokip5xp1Vq2Yp76/C+DqkZLTELasAWFgrcWpZcKwSARoAHc4Sd5jYSBcpom3K2T8BK6IejsepT5sONGanyWmfmtto1ENueX07A9vbDP8aOX8r9Ss+ZUqsTMjbrFLo7sFxECeydRclJVoUkgm4sBcWtcd8VA7n3IlLY1SsyVKPFS4iLAg/0YajNRWiDqJLEghf52HAA97lvdX8iPQ4selZpoUYoYQl196lv0959dyfmWaUDx9Bv2Fje18ecr8Srwzs44iPO60QYTX1jDJAy49USz0V6aNpRt06sztQJWX8wu7STewGltoA9ke49OfRZA6O9HZqm5K8hyylRssO12ayzc24ACxyD+Qta2FEfqNSpCwKVnGhuXUlbTKajDbUnk3SSSk2BJtZAI9b98P0fMUx1xbVEUXXENgiSyzuRhyoFGpu5XZB5+38ccB+IYk03LykODYpDq6F3yv8AgsfxOmPUmqhU2BRa5Vjc3cYg7qLe5UhRHv7W98em0XN0dsQs0wKgltBBQyYu0lIFxa6Em/r3JwCLWHluKZlSX47IBW4W47b40WOr5R4J7HviU06blGmMQ0VV6oPmQ13VC2FAnm23e1gCOdfqOx4x7J9X2Tb2us3bMTGy+mG220cu05BSiwIghClcixCiFWIHF02uTzjok0t2O5EkCiSCtwqIejvvOp440qaLYR3secTN/M2S5i5K6WitsKSyEoZYiNtNgXBU/ZZN7i/yyeSeCLYEinZWZlNw4iGJcrbQsBkht1tZAIAfDnbkovYC4N/Q4zOr2kZiwotnsouyzRGQmQzkcKU6gNqeXNkoQfVVkk/3k2GCKdlHLjk5zxlTpdKacJ224UkzdI72G4ptZ/j6YlGaqfRnIEtmTQfAKKGA0pmqNOgEgjcCr63iABxcjk67cYZonT3L78CLNmdSKUp5CHHFsAPKKz3RyE8d0NlsajzcKwzK6PLqLI9u3xWTvU8q9JmYSkNdUaY3LbXYqXQqgxuj3UdTrZ9rjv78Yj9TpWTmYqmqfLpDjrgKmUNuOl37khbaf/N++Jn/AEeynAMiM7mbLzaRocccejSHS0+UhRbTY2KADa59z2xEc61Kn0VxdCpcuhyWS2Cst0xbbbXf9nufexwkNaJH5GNQlqb+EKItPUuKr+uMtlQNykPJbv8AlcG354Xy3KY0lqTDhNNgBCnG1hl0KvyPOhCT+d8JqfWqrCKRCqbbLayCdppJuji/A5PH3xKZeWIeYVpeoLrjjmrcKpEkR2yDx/jD2v7E41PlczVVdoojMlshkuNxY7IbB+W1xf17Xwlgzq4UPARnIkeQoNlZjKUlRI8rZUB3OJtLymjLrCpMo/LIRcl1pTR9T5kE+mJBCp2d8w0rxeWa/SaZHaSS6VZi2SUj6RZBURYW7Jv+mGFRN9hqZuZ3hVaVXLtXr0CI/uxjMgWWpRnIu63+4EpJ1kgew9cAbosp91Md3L0x9bq7ANm2k9zcf6b8YsGBTeptUrcinv5+y5KnBoPLHjXZC3WwL3aUGNDh4Fwk34vg+n1xiryVULMldmRrXFnICEpJve2rd4B9DaxwXPqiRZoRvIoGjKM9xlMdWXEBxla3XAuVZ55tdgnjV9CbE8WPJN+cdk0+ittpFWo0+IpDd0Lhbaf10rB1jn1N/vh8zLQaBTZPiIk2pSmjdCXHm22fmDm3kBB+9l9xhsTXENR4rreXG32y4ULeJcSpzmwIIIC9Kx7fr6YZtPM7VVua5EU2nuT3Ft5eiz1voBdaCQkFxtHcgHgn0tfvgKqayl1xVanGEpBIeVIbKVpX6Ap08k+gHYfpdZOz5mOU6h5xcVT4IG+GkuOkdvMXCrm3qAD98K5FdnzAzOZqcQy3xtuxfFNboW366Qb2t62t3GLRTyfaIRyuckDOX6WtAIzBBDgWAWNqUeCO+5taDzfgHsMGuUumQ0JbU808CAFFlxCrrH1nkCwP+nC+lUPNdUBpFPp1TlMOlZQ2W3S0km9ze1keuFjHRDqvOUPB5JqAbsCl55sMsqv7KXa4J4v9x6Ys7KMbuTdg93JNztB2G2pG7D2HGytKnpjI1AEBy1iTcE8ggEffD5ToUuPV4L2VpARMXNjtw23SXlplrdbDIBCfP8wpsqwv64fqP0Bz0wUIzTR55hL+YIcKs0+K5uGwuXpKihoEA3slRJxf/wALPw2zp3xDZNqE9vLcWk0OUKsqFDqjtQfWIySttbji7gnf2TcG3BsBiyIQBwObZMyncH3svqnS2prECKzUZCZEtDKEvvJTpS44EjUoD0ubm2DZcliFFdmyFhDTLanXFH0SBcn+AwoAsMVz8QFeXl/pTXXGXNEieyKYwQbHVIUGzb8klR/TCbm60DUrNNBrLz0HMvUKV+2lJl1AX77jqiW/4FSR+mKm6hTfwDpXUA2uz8mM9oUee9mEf/MpZxa1XYbp2SYVKRwqoSEeUd9tvzf36cVf1dyrLzRl9FBp0pDDrrMMpK2nHNxtKipYsgXN1DuPUY8vxYyapo208Avmc2/oNfyX0PgB1JBiJnqXWyNcR6nRZh+QAlKUpAR2A7acS7pkw09nGNIU2D4OLJfH2OnbB/8AsmHZj4aOustW5SunFTqbSuQ7GUhpJ/8A8ja/hzi7fgp+HyRm/NNezBnalSoNOohRTnYMtlTTz0pLii6ypB5SlBRpV/a7Di+PHy4LW1bTTxttmH7K+kN4gwzDy6rmkDms1039PiVCOtXTerM5By11HnMJbpsua5RthayHJDLiS6iSlNvobcQU6uP2o9O+d6hlCuqrTaWYzbMa5TqvdIAP9/rjZX+E3rnUfJFDoaqVm5k5VqcpTAoX4M2hbfh2w6pXiACVtDSPLxypPe2Pl7nLrTnetNLhx6m+xCPASh1RAR/Z59Pz5x9IwfDRhVDHRsNw0L4Vj2MPx6vkrpG5cx2/fPr1Quv+ZqVVsxwqJQ3y7Do0bZKwRZx0kkng+1hhl6el6mtu1tppaSl0IQ8D9Fhc2HqcQRRJNzzfDvTswSoTCYSyFRwvcsEjUD9jjo81xyrpoWciJe0iQ6FuKG4pRLxva/Kb/bFjwKsjY/E3mFxEx2t1+S48QyFk+Vei1+PMnvbjjGeKNmelQJAkiqEOKPJVGPlHsP8Avw8V3q/NqTIhR5i0sWQC6izVgBbhI9rD/RizMhZT6KhdZrsisImEuuuAoYUyXQTcAauOBwO/2FrYQZjyhT8s0qXJjSW5s4uB3xKWgkg67httIHNzz3FgLYg8XP8AXorTUuEyZrbn7ZskqNgeNShz3tz3++JRlfN2YWczeJruwoz2DtaRuNxW9BJG2e9+/Pr+eIl08KbMi56r8F1+NMTKn3WC40+7p0mxBsm359zbFqU7PaFLVOnVaBRWjZLnzIzYbBH7uu9+w8v/ANOKVzPV2365JmsiRO3X7ISTq3b8DgAXvb2tz2wsovUzM1BcSmn5SCXWVEaRETcae/JQe3ucRNZXU9XJNUiNRssTkOuAa5DiQtvdDg8tlCw55Pf2PFuWKLlWi1SYUf8ArBE9F9T8t0ulJvyBwLc8m3fuMMVKzf1IrMlUvOEkZcgrUjaS/AcG7ckaElf0fmf0GJ7AjTItQakyJzchDhul3Qpl0342yAQLfcEcHtiJkfQJaYqDSRZ5gkK1pSsjQO9+LcXHJPfA5svKynAajT1tDb4cPCQb8E/w74JYrMZ5b9Q0Ryn6FNPo29I4tcd7jnzdzhzrLeX6hAC5sEyI7DTcpcstOFm17EJ0EFa+3BsDf1wpKibMy5IplRg+MgzPI+gkOOWSOB9B/n+ZI5xTsujPU6sGHEp1Pkr0BtSy0LOWPfUv98/bF5zaxTzVGKJDQw5Gba3CHJVlNA/s7J9eDzfsRz64jPUCjxGnGqnDJIuQte4FD7BKPQc9/XED1FbXw/0npEnKtWiZyzJFi5g2W2YdKDLpUwt1XlfdK0htxBbS4EBK/rsDbi7zUa3U/hOztGqvTbqHDekT2WnpMNuY1OaWwSBsy20Hbc5JIsdaQLgp74zhUqm9GpyItNmri1OIFuNOOKOl0HSdpRHKACgOI76Vg/2jiMUKn5yzrmiM/UJDjDUd4JdSysr+5WXB5DYcj2Bw5kDG3WinpX1UzYYhmc42A6lfUb4js8Zh6t/CDSup2T5sil02XJbOZac2oBLzG6qK62V21lpuUEKtfzIV8y9rYwY4hxDoCFlTh5J+/rjV/wAMPXHphSummb+i3V+tpp2Xaq0sQlLbcdBbkNKaktAISpSTcIeHH1qVyCMZVlRURnHo4mtTEsOqaRKaBSiQEqIDgB5AUEhYB7A449XKJCJL6FfrL2W4fV8PxVWCVjMrmODg7Lo4P/zc8p09EjKlKJKz39bYd6VAnVmVDolLgrmT5r7caJFZsVPvOLCW0A+6lkD9dX04a2xqUhJSVW4tjbvwL9BHGi11xzPD0oKXG8tNKT31AodnEf2bakM/5Jdc7KSRVDGZngBev4z4kp+FcLdWzeIeAdXch+vRaU6LdLoPR7prRshRnGnZENtT9RkoTxJnO+Z9y/qNQCU/5KE4mirBJV6YPIFtKeMeSLLSb3sexx6A2jZpsF+GKqplxGrfU1Bu+Q3J6klV18QHxCRshsP5Uym8l/MLqPmvJAUmCkjub8FwjkJ9ByfQHH7lYmPKcmS3FvPOqK1uLOpSlE3JJPJJPN8BqU1cipPyHXFuuOvOLW44oqUtRUbqJPJJ98JFLSG1ISBY4/MvEmPVGOVbnv8AC3Ych+p81+vOEuFKPhuiEUQzPdYucdyenkByCM/EnCb2v+mAie5fsOcFpcRp+lFvvgYdY/soGPM5XH7S9bkaPsoRnrtpsk44qe7/AJOOKcZSDYIwFTrfqlHOIWH7ygY3ohmarTqdGtWOeNKQHCoBv74CFNaS6Ep/LFo9EuilU6sVPfeSuLl+K4EzJaRYqUn/ABTXuu3BV2Te/ewxvoMMnxSpbT00feO4/X+65+K4nR4LSurKw5WN/dgOvRB6J9Ha71ZrO+ouwcuwnNMuaByo/wC0te6z6nsgfcgY3Ll6gUnLFIj0OiQW4sKIgNstITwkf6Se5J5JJJx3L+X6RlikxaFQqe1DgxG9tplpNkpH+kk8k9ySScOgBta5x+guHeG6fAKfK3vSHd21/IdAvynxdxfU8U1WZwyxN8LenmerjzPLYIQx3Hsex6XfdeQXscOPXGPYKiD6cYCoG+oix9DfAre2If1G6kZd6Y0BdezFLAuSiNGSQXZLtuEIH957AcnFM9RHSxmWU2aNz0V1NTTVkzYKdpc9xsANzdG5+z5lrpvl1/MOZJmxGbG200ixdfcPKW20/vKNjx2AuTYAnGEeqHV6v9WK6qfWEGLEYuiFCQ5dEZs/9JZ41L9ewsLYI6kdUaz1OryqxmOSkJbJTEiNk7URon6U37k8alHlRHoAAIcp5KlBN0gp7W9cfDOKuLZsZk91pjaAc+Z8z+Q/Nfp3gL2fRcORisrW5qkj4MHRvU9T8BpuIzz2CTxxzgBlrV6WxxLjRVdSk4FvoHZYP6Y8Jk7T/EK+oZQNmoCpbhGki33wDxL3ucDLwUfMofwwBT7WrhxI/TEyeacNHRdTJdTdStRH59scMxy9uMeVIYA4WhSvywaw2H1oQlWpSlhCUhJJWSbAJA5USTaw5w4jLj2YKDsrBmcFxMt0nsAP3j7Y1d8PXw5SW1MZ36kQAhwEO0+lup5QR9LrwPr6hs9u6ueEuHw7/DYMueGzz1DiIXVuHYFNWApME+ji/RT3t6I+6uRpJCQCL24479sfX+EeCmwltfXDvbhh5eZ8+g+a/PXtC9o/vBdhWDHubOeOfUNPTkTz5aamt/iFpbM7pXU1qRzAdjzEKHdOh1Nz/An+OMmbKEkp1euNx51paa3lCt0kpCvF0+Q0B9y2bfzxhtpBW0haj3QDb9MfWoDplC+GPtulkctpKcSzK8swalGmIV52XUOjntYg4hjfyz3w90uYEKCCsi4tcel/XFrtkjFt5K0uthaDwoXBxjzq5lePl3rRXKbJBFOzdC8cyhLmgqfN7hJsQDuBRv8AfGkqNnrLtOytQJNcrDENyfDTtJcVy6WwAsj+X8cU98Tk/LtbpmXsxUuoMOzaZPLadChqLTjer0PYKbQeffHOlOVbae+ZXF0kzUM39P6RVytZkJZ8NKDigpYfaO2vURxc6b/riZW4vig/hwqrsWZVKIFXgVVAq8MoUNLb30PNW9CRtOAeouffF+jtiMdmF1XI3KbBZK+Lmk1CnZko9ZLjrlPmRnGm0Fd0MvIIK7D3WlSST3O39sZ+EttZDRBKvQ42/wDEhlMZo6WVJcdnck0kipMBKLk6L6wPzQVjGHtcVCFIUolwjyKTp0/cm/29vtj4H7QsOfT4w6a+kjQR+BDfp81+l/ZjiLK/Amwkd6IlunTcH5H6I+LVFsSkEsNqQ2q5SpHC+b2VYgkfqDixMlddc1ZMSqDSaBSXEuNIYZDkXzNi/CdYIWsE+iibk98RKn1qgxoiZs2jNSXlpGwlCgmOQk8qUNRUu5BChdPcWtie9N+o86XXYlOoeT8mNSY5U+FiIyw7wLcOKUVKXcggJBUfyBxzMEbJDVtEU5je7kG3P6BdniHs56VxmpBIwb5nho0531PknupyJuY6hJrD+dIyJUl1SnmJzkqIlkkn5SVOJU3oSbgeYfphul5TzImIZ71BkyooGsyodpjH5lyOpQH64Vt5XVW58icM5UlubIdcfkJS+626XVElYU0tKTfUT2BvjrGVcw0Wa3JhZigRpK1fLUsSqe+4f8l3ZAJ/I4/RMd8gubr8rVJaZXljbC5/FQiZlvLdYWGpsKBLWAbokQYsm36OtqOI7UOhvSGpvLdldMOn8hTvKluZbYaUT/umgMXw69nupXbrOWaXm1oI53o0aesI/tB1opfB+5w1us5LUoRqnlev5fllN/6s8pafvZmWL/oHL84s7RoVLm5ln974WOgkiazVx0doLUqI4l1pynT5McJWkgpO3uBBII7EEe4xa1XagZgg+DzLliJUoqLfLnUxqQkW7EFAURb7EYka8pwJISrL+caTI1dmKk27Tnifa6wpon76hgmZk/PNPbEl/Ls16OBcSIiBNZ/5zJVbCus/Ryjbt2VQVToL8OdZStmRkCBBdc5vS6pLgG/uGw4Bf7WxEZfwV9Kp61OUzOOdKaXHNxOtcWe2Ef2eWknk2N73xeb9YUl1TMtaCvXoLbw8w/zV2I/UYKKqSqxVSmEqVwVN6mT+hRbC9izktcVZUQDKw6LNtT+BiQ3GfYy71kiS31ubjSarQFMjQB2KmnlG/bnRbEQd+B3rBRXQ/l+vZGnSS2skwavIp7yQeCBusDv73t98bHSzGeG9HnVBkkckPB1JI7fWD/owYWpiRqjVNty30JdY7D15BxX2J3Gq2x45VQ7H9/BfLbMfTXqXllkTMwZYmUppfKVypEYavewS6om35cYZEQp4Ud6c2lFiPM6Vfnbv64vKNQM85tyb4uh5epDsZ1q9jOjIJR68LdHPB/h9xioanknPkKQ6zMokhoMXCloQh1IFr8FClDj88bGQxuXmnwFmtkmYprexplzHHB+6UIJsD6c/rhVToGTW5DSa3Ormwlw38AmI2s88XLwVyPcDEqyr0tezHTG5FTzM7RXNdlh+KjRo9FhReTwfywuHQGlLbV/+duhPlAFwiK64R9vllXr6/cd8QtpmmxTxwEi6aqhlvpQ/DTJy9VszSpW3thufOihHvxtR0m32vgijwMmPRHUHLCn5rYsHXqlLAI/3KHUj29PbEyy90Gp8ZW9+PVWcsGwRCoLrjV+ebr2/Qe4xKU5cgUIomU3IQkykcNv1LwkNJPe5SZZJI4Hm459CL4Q9nezRdXNia7dVhShkiqXiQsmwo9Sjq87bjsmSlwc38q3Vfw7jCObR0vPvRIFCbhrRdBaYgkBu3NwDcj0/TFsMZpQ5PRKrmV6PJXHslvwbMeI0i3Ze81cn24/jh8RnhLSrRqjQ6WskOEOVOVMeH/xGnLDt2729MQ9ruxiPZR8yqLV04z1U1DwlFfYkFsEMuhMbcB/fQFkYIT0L60TELe/oDUS2LgKVJjC/fsC7yOO/bFvzszzZSTbNcuaVnUdqnsNA3PbUtLaz27n/AL8EtZxrcPSuNDaWoC+txIedHFu27ZA9eBi1nb9AkcyHkqkpnRnqK3JLK6HCpinOXRNnxY449/OTf9MWNljJ72UISv6WRmwknQTCrURLRFje+sXv9u2JFT855xcWoHYiJTwpbVPDbgt7+Xn3wtczlUdwLkmcsf8Au6REPPPq4L4JbM8W/BFpjYoKnJWUP6RKqlAzZEZYQ4HW2npSn9KxyQlxppsA355WTiQ1HLWRq2/42UwC6hzWsUiLIfLyyBwSt5Ogfa57k3wKu1uC+2qZVKfUXCP3pdZitI+3ykAWt+eEFJrFClkRadSaO88ohZQVP1J24/stBLnF+wFucD3d5FnFQyt5BDlO9O6fETRxlXMclRs3sOS2Y/0HgkI3lnnnviR07KuVHmWHofRebUHg2Gg9Ka0hSP7Bspu6ACRwOfX3IadkjqRU/wDWPJeYRFdPmSxlswkufxZbP88TqjdG+pS2Eb3TaG1xbVUNpCz9iLKP8cDsmDmm7RxUM8HLaUGJvRzp5BDXCXJ1CYeUgW55ecV7AXtzh6y7V8yQgtjLf9G0NjnapdIglQ4sdIZjcW7fliWv9AM9yAFJo+VGje6xJrE1Kf8A4cfaH8b4UwvhtqEZq8+sUlp1Y1Fum0OVOsffzvj0PbnAyxc1M77phmZ9z9DZW0/Q85zpCrEuSHFtMpNuCDoJ/KyB64Yk1HO1aLaH8qwm2kBB0/hdTnuge4TustD7cYkVU+Gat1B8Oiqgx03AL8AxCLH0Tc/zP6nHR8KFTLadOeYy0Ei0ZNGcUkW+/iQP1IxYGQJTJIeaZJX4lRVf1FyBR3l/tXZMaJTlN8H93dU7a57XH64118AeXqrLfzbnqrVw1JKfD0iKpuY68yk/tnrBbrgvyzyDjMkX4b8ztuJaRX10qOO7rDEdsufkG1Ej9cfQf4YOmUfpV0co+XW5kqY9KceqkmTKILrrj69QJ/JGgfkMWP7Jre6owv3erdHbGePilrC5tUypklhZIcddqj6R/kDaaP8AFbh/TGh1GwxkvOlURm/rlXJqVaotI26U0ruAGr7n/wBlU5/DFLeadm6g3UadoqK4DX0UqmaSPZxzz3/gW/4YY8s5izEwI1Ph5iqUeMhy221KUgBF7qAtzY+2Gmtyqlmd2sSWErZdqcrcRvNnlrd4Ht9CQMOeWsl5wdUSxR33TtrQHENLsFlBAPIAIF8UOizarTmc0d1KWp0qeVy5Uh59Z1qC3XVOEGx55PfGg6JnuJ056b576n1BhDsl6r6GWVuafESQ0wy2kq9Ap0kk+gJOKYbyFmPLcqlsV6G20zUH0MtOodCkk6hdCrdjY359MKPihzJMyl8KuWcwUyNElOVDO7brjUtrdYW3IdlpAUO3Ypt7Gx9MPE3IFTI4uGqxh1X6x5q6+VSdKzbmGRV48QOuU+SuzUdTb6/OlhofQylCU2Butd+TjIOeMvNUFcd2FKbU3KfdAjEjS2EEW4POg3NifbFpVyZmGgy4cijvIlv0+MhvYUEASGkeVTVxe/AHPuBivOtUI0/OTlICUtqhxWHXGE3+Ut1sOlHPtrANhi0HoqLKvHtBdUQ2G7k+VPYfbnAS0Q2F2tftfCxmey22mLOgNvte6flupH+Sq394OHGnU9NYjOMxkLUpoKU002Qtdh6rHBIse6QfyGDZMmBHF7HBimiUhWoH8u2OOtFlwoWgi3vjoPlPm4PcYgCidMv5ilUJ07aQtpY5Te38/wBMTULdzm5DWqUhqVJbckLBIQXEI4AJFgAbK4+wtis7H6vXDumuTIMpLsF4tbTIYQCgKAsm1wCODyo37gnDc7KKZtrnUeK0wh4RnJM5CPE7N1Jb0kE3vb3POH3MPSjPDLy5kurtrqSrSY8RclLLUyILBTjT61BBWNQJb7gXJ7AGGUvO5eQ5CrVw3I/aLSsgD9LHt7ffFt9PM+1ehsR2TS4+YMvty2pUSHJWopTIBuhbDix8t0eg9b2IKMAlBnmm7J9dp02PBit1RURkEtvTkta5kXns6kGyxqPdSFd8TutZopdCzYzl9LCZsWoxUORJLWh0SXCgbmpux81+LDuLexw/Zk6a5V6xx4Ve6XZllzKjFi3FPep8SPVKeGEquLgNeObKNVgkpdBH0rviOsZC6/ZArFZMfLECoTKBHD0uRDlxkzHWw5YILKjuuO+YqLYTrAuebYQldFtP2zbhN4zVRCt2MmVFkMEobQhbe1IYsOA1cWcb4HlNiL2vawxx+qTKeEKpqBOQ4jd3KZMspQNu7R4CuO5ve3piN1+B+GfjP9IskZkjSJqd5hTEcOMRn3Uh4ELauhQUhQNgrgqPa1g25Qj5Oqtbj0GfmFeVn3HWz+LPF0RGVkXJeGkrb0k/UARzzYc4Zgzaqn3V2cMB3Upl5lYLyhIjMFf1JRLp62XmxYX1Ocg3uPY2498J5ddVNBYDIRuKCQUBxQDfGle2u5RcH6R2+/Nr0zR8CHWaFQmK/l7qNSc5JWgPGFDfXGeWggEKbU98t66CCDrHcWvfFN0fKFVy/nAQc5U115bThafjVCOUPNn2cSeUOC4IPFxyLi+Je9yFs/g87JWNm7odz5DzUIcYzBWXWwmmut6kboFrmxJHbub2P8MWtS236TRo9MdKFyi2A8UICUtarHaQE8D/ACuO+FsuCihvqgRY0OMnWpd47Ib1BXZWrvY9++EzRcbcKgCL+trnHJqqoyDKBZfpr2f+y+mwCRuJ1Uglf9jkB56/iuoJatqcUEgkKCVY42jW7dKPW/f+AwMNsoYUVu6XAL/bT/lH0xpv4bPg5rnVFyLnPqPFlUjJy0hxhghbMusIBuNA4WywfV0gLWn6AAdwZ4mOmcGgL6RxBxHQcNUbqytdlG1ubj/lHMny29E0fCt8MkvrHXBmnNcV9jJFPd0P2UUKqrye8RlXfbH0urHYamxdRUUfSFpiNGYbiRmGmGWEBppplsIQhAFkpSkcBIAAA9ABgNMpFMoNMiUSiU+NAp8FlEeLFjNhtphpHAQlCewwrCE28vbHep4m07bc1+MuM+MavjCv95mNo26MZ0H6nmk+OgXUL+/+jBq0JA+nALHVY+/+jFkpGR1l5KNvfb6hfO6YhKX3wR/jF9/92cJ0pQEDjB0r9q7wb61f9M4T6SUDjH5Pl7knc/zL92w+AIYaSRcjHChF7kAfnjqQPbHFJue3GMsj9Nk913bacG8sA3748G2VpOpICRwB/ax5CPKURwSTi2uh3QWodUqimrVkOxMuxnNL7w4VJWL3aaP8ivsOw57dHDcMnxWZlNSsu47/AO65mLYxSYHSurK1+VjfmegA6nkiOhfQeqdVamZs/dh5ahu2kSUiypCh3aa+/ur93083bcdAoVJyxSItCodPahQITYbjsNJshCR7f3knkkknB1Go1My/TI1Ho0JmJDiNhpllpOlKED0Awv7jj1x9/wCHeHKfh+n7Nneedz18vID6r8pcX8X1nFdVnk7sTT3W8h5nq48zy2CFjuPY9j0i8ivY9j2PYiiLVe9wBj17/e+PG5sfbFedW+sOX+lVGD828uqygRBgJVZTxHdSjzoQL8q/QXPGM9VVQ0UJnndZo3K00NDUYjUNpqVhc92wH7+JOw5pV1S6qZa6WZfNZrTpdkO3TChNrAdlLt2T7JFxqUeB9yQDg/qDnrMXUmtuZlzJKDjjmpEdpCrNRmtVw22PQdrk8qKbnnBGc845iz3X5GZc0zVSZb10pAFkMoB4bbTfypHtzc8kkknEc2n3ULdbFuCBx2OPhHFPFc2NTGKLSEbD73mfPy5L9S8DcB0/DEQqZ7OqTu7k2/2W/mea6ttBHlSOfXAUstEj6cGykMuSnVxo5aYWq6EKXqKB7X9fXBJTovZvHkC+3IL6M03Gi8WW/wB1WOlpk21FOC7H91N8cOseVacV6BPY9Ud4dsDSVpse2OeGabN1qRc9sFhClpKlJ4HbC+iZfqmYqjHpFLgPTZ0pzaYjNJup1X29AB3KjYAcnFkUUs7msYFVLKynYZZX2aNyioVJfqE9iBSIzkqTLdDLDLSNS3lnslKfU/8An0xtLoB8N0Hp+0xm3N8ZmXmZxN22wQtqnAjkIP77noV+nITxcqdug/w+UfpbCTW6olmZmWQ3pdkAXRFSe7bN+33V3V+VgLnBAJJ/jj7Twlwa3DwKyuF5OQ+6PPz/AA9V+bPaB7SJcXLsNwtxEGxdzf5f6fx9NF1KU2+nHcdx7H0VfIEFSUqBSoAg8HGDK5TZNHrVRo6hzCmPxuPZDikj+QGN6HtjHPV+nt0vqZmJnTZLspMoffdbSon+N8XQeJVv8Kgii563GOJeW0q91cc4MccaC9N+bXt9sJ3Sk383AHP2xp9FWEr6i54/pJkzLOXH4e27l9cn5+q4eQ7a3Hpp/M3xWTc2U/FfiOynVNsOIdQ2p1RSQRY8H2IH/OOOZ0z7QqZXoOVUr8TMflNszFJUNMIL4AWfVy5R5R2He3bCdmk1pxxc+NAcejKQ4y5t8k/p34Wkfwxyy6KYuDTey6fYTwBrntIDtldHRnN7FCk5czRIWu1DkqhySk/4hz5R1f5IBv8AnY43OlQKQoKBHuPXHzRyA/JjVCTRam2oM1cFlSSCNJWg+4FuUp/U43l0TzYvNvTqlzJKkmbCSafMAFrOs+Un9U6Vf52JELaBVzN0ups8w3KjuR3mkrQ6kpUlQuCDwR/DHz8zLRf/AEYdTplNcgxZgo81SmWZTRW0439TOtIIuNKkG1+4x9CTa3PGMpfF3lFuFWqZnRiOCKg0YMhVuA63dTZPuSgrH+YMeF9oOHunw4VsXiiOb4f72X0T2XYo2nxV+Hy+CdpFvMaj6XCput5zzFX58ep1SWzJVH3DGStlCkMhatRCUEWAvawPa2LL6c556o1ichFByjGqAjvp/rDNPYabik8qUopSDqUkKF9Q5/hin49TkwmSzGjhDSjuPrCljctyjUb8BNyRa3Pe+NMdBJNebytXnKtllqimoNBimSGoQjPTXUsuK+pR1ucAELV5eTZRx8+4QjqMRxIF0zhfUka6jzOg+Pw6r6fxs+mwrCi/3eN1tGgm255AWJPOwPmeYVXqlQJSdcsBzWNZJAN7n2seeb/rgSIEBCB4WW4wn+ylZSAfsBxfBCAtCEp12U2kJNrCxtyLf34Aqm6HkvkursboBN0pPuOMffQLBfmdziXap6aqebEgLi5sq9m+BpnOKFv+dgVQzBmerxvC1jMNQmtBYWGn5K1JBHY2J78nDSpe2gLUhF/7WgBQ+/GDkVSUFKQbLT6XsvEy9ELuQNTjRC0KfbBHpc/3cfywfFnyYTpdgz9pRP1R3Fsq/lY4GzKZfstbYbSoX8tx/IgYHtxX1aCtsq9lI5wSgnU54zOtnwsmsSJDJ4KZdpKVD/lQrBCK6y5qRMytliYT6uwAyr/nMFu+Gz8KZbuqPZOvkqbNscchPpJTuH7ax/34ACN3JwT/AEceWoryc/GC/Wn1dxIHvYOpdH88LGaDlKWlbkbM9UpjoJs3UqcHUf8AxY6j/HTiOKiygsNrS4gIJOptzSP1APODG1zI6tSS4kAW85BH5374mS3hUJusHVmoZSjPKWjMUmS4g2Fm0uWB9kgW+3BxFZlUhy2ksOvVVcVu4QhaW20n76bffE1oeRabLnJZayvm2cSDrbp0nWU8HnebYCPTuDzfjnFoQOmaZ0UfgnRmpuSVpDYk1qpvOp4A5st5PH+Tf3J7402ZG5VuzOas5RqpDo60vU2RJivo41rltpCu9uLcYmFK6iyJERLM/P2aZSgNCkRJK39J9BwRbj0vi76R0NzeH0uycvUSCk/NW14qPHSD62EVLrp9uSMWTC6QOphCPJZy4y2Rwg0JyYPyvJese/8AYGIZIuaUNd4Vj2bqrBd0zq18s2U9VKglhPPqQVE/wGEsKiUBS9L1ao9Skg/sGaq8p1XpaxaT9rcn1xtqD0cypC+YYFKkkcgLoMBpof8AJtsj+Zw+RMtSKelIg5kFNYHAahU9qP8AoC3a/wCZBxUamJP2bmrE7XT/ADBUCBF6bxozRtofly5Aufz0q9vbEvh/Dt1Mkta28uZYTfhWmVK3B/l3eLKAffi2NXs5fbeeMmZmrNkpIHDblVdjsq55+W2ElY/Mn3GHylR3pknwVDp0mc8jkoYjuPKH3JN/4nFbqn7qLYiskxPhH6kTY6nJNfy/TVE30iU86P12kkH+J798PdC+ClLJDmYM6bjwBIbgxXUtk2t+0c59fQY2HF6c9QZqyDlwxWr21yZTLJP6aif5YPX03dp3zsw5+y5SuwcSZRfWAPZI0i+K/fX7XCYQhZOh/BxlVLyFzKu/IZbN3Gl6lfy9P4/piXQ/hd6IxmxuZSRNKBZxTkp0JJ+4RbF6yI/SCnpV+JZ3rVa0G4TAp6Gk/oXB/O+E6M4dN45U3l/pe5OAXcPVKW49q+5bbuj+eB7xJyT9m1VfTeifQ6ifKgdK8nx1dvn01p502/3zUb4ntHyo+AhjKWT9sI+lFNowaR+WtCAB/HDy71TzHG4oOW6FRE9h4anthz7nUon+7DJOzxn6qpdVVM0VZLTh/ZiSQkfojSLYXNK9DIApH/6OM6lGuoQ4tMaUL3qFQaY0/oCT/LAF5cytS1q/GOpNMSU8KZp0N2UoG3uLC98QQSI77h8RUXHVXBI3P9HJw5RMuZgqg/qWXqtOTybtwnnbi1+SQBhSw8ym0UhkP9LYzIUzMzLPftcBAZii/wB7pURiNTKpRXtLdNy3MCgbgP1N5+3PNwjaRfDi7krNcM7cihrp24NaEy1NMX+3nVcnHmOnGequ42INBnqQQPnJLbTXtbU4bfwxMrBzU1+ymQmUSXNplsDkhTtgOfW17AX++ObEpbmzrRc86UoKieb35xPofQrqFslKTSm0KF1eLqBWpIPe+2yR/PCaR0zTQ3lqzj1By1Tm0D6WJi3nRfj6SkW/gcNnYNkuU7qKU6kSazUIlJbekl6c6iKAlaGwNagOLJPv642bFisw4zMVgaW2G0toHskCwxRXSnKvT97NLM2iP1urOU1K5KZrqtENDn0gBIQnWrzcd8X56fpi+PwpXd1NOa65FyxluqZimFIapkR2Wu/roSTb9bWxiSmyZdPypUKrMdtNqYWXHPUuunzn87rUcaO+KCvGF0+ay80tO9Xp7UYp92mzuuf9BI/zsZuzGQzGp9GbI1ur1lJ9beUfzJw7fJMxP/T5tqjQxKWm7josDbkDE+j5oupKg2pQSdKiVdsV3UHJNNy5LlQ0OJMRlA3EjhrWQkLJ9OTb9RiB5fzrSsiir1jMFSEWnJiFx1S1crcChpCfdZucVPa55V+YN0VwdZ88wKRQKU+uU22lueuQHCQAnbZVa/sCtQH54wz8QHxEyc7ysu9NYlRcao2VNsOuJLlnpARwCktgo0kqNzfki2In1x+JiRn6atqnvrejMOLREg3+W3yL7ib8kj3P6WxWz9RojcBbeZa+1UnihADEI/NDdiNDrmkAL7AkXJt34GLmMybrI593JJmyiy51aC6Y8hSpgQUKRYKS2AVG4HBv7/Y+uKYzQqU1U1zPESHg9ZW88srU5x3Kv0HGNO9Aavkat9VKVSM0TWKNl6Y+2JKrXaaLbSkx9wn6Gw4q67ehP6RbPvRJEO9Gqr7TUqjzxEK0O6kSGyCE/M7A+Um/I9u+NLYHPF2p2xOk8Aus9oVHnq0LQEPEWHoMGvUYFJ8M7daOSlWJDnPp1LytU226ZOYqMZ6N41h2M4HDthZSrtwSCMGZEZpdcqColbkeFSEpAeJIaKrgWUfQn09L/ngGPJpIFSD90qItQ5kx9MNmE64+s8ISnk/phMsqQdCu/Y3xcdLybleq0pqXWGX23JJdfYebWdxLAUUpu3cax2JtYg27i4MequTKbT6eYUVtuoy1LK0Sm31M7TfqHGnUi578gnFJCZrlXhGoDkfocKY0NchdgRbuST2x16GGhdbchKj2SW+/64NYdeavoQUpI+lY4P64XVBEPxdtF09/XDnlvNM/LUwPsth5q91ML4Qri1+O35jC+Nl2dXUqk07bjsl5DLbLp1HWbE+a3IAN7+uLl+ED4f8AInxFdeUdI83ZkmU6AuBNeamU1KQ+/IZa1BtvcBQBw4vtyG7cXwdb6JmtJ3UZy7nz8WcEuOyy08LqebQsIecN+AVrJDpueCq1v44s574rOumWKS7RoubKzUIRa2THlPFaiwALaiQoW4GjzkjSDwRjXc7/AATuV8thErpznWPUJLSToazRBUSokc3djK0W9rtKtxiC1v4C+tdIgfh8Pp5BmssIIQ9l+sx3TzYn5UgtlQJHYpv6Ypklc06tK9VhuBx1kPaRVsTX/cccp+ot9VQ2V835+zYhUCsVLL8RyS3qZZXUC0HivkNuvoSW27Gx+YNHuRg1eQ6tTapDqWbctQ36szKbWt9mqRXlMPd2yVNuEWsgkK86CPcWxMqx8JvxFRluIZ6Q1aRuJ0BK4D9hwByWty3a90n8hiq80dJviHyW+pnMHQHOlPStYRHMelS346gRxpcDTl+eQDY9x+UjAee6s9Q6opGhtSAfMEH8CVtz4YutqMxVGudGcxvUxqr0C8ukvQJqJMafS18hTSkEjU1fzIv5Qf8AJOJl1o6X0TqVlp2NUA1FrEJtSqTVNvUuO4i5DarcuNK7Lb5te4sRziTpn8LvxcZozVSM45F6YZpocylOIfiVesNtUptsovtqHiVBa08qGkoOoK0r7nH0SNCzvTKBRmOpNKpkCuTWAHTTpXiIK5IQS4hpagLEgatoi9uxXpJwr2mB2Zq9HhmIRYjS+7TkB42B5hYPz5lerUlRi1OKW59P0IWlB1hbKvoUhafrGq4vgHTPo71J6tzl0/p5leTU20L25E4kNwY4/wDeSFfLH5JKl+wON55W6b5GzjXIFPznlSmV2OmVwzOY3UBYSSbj1F0AkG44FwbY0bBp0KmQmadT4TESJHRtsx47SW2mx7JQkAAfkMZxSCZ5cTp5L23/ADRrOFMLjwmGLtJANHv2A5XA1JHyWZ+hXwPZK6dOR8xdQpMfN2YGSl1phbJFNhO2Fi22rl9Q/dW5wLXShB5xpsAk83Ku5ueTgzbHfV/PHQlI/dxujayIZWiy+M41j2IcQVPvWIy53/QegGg/d0UE8++O2+2DCkfwx7SMOuQk60G+ABHIKvf/AEYUaeftgJRqIAUkc8E9sK7RpBTsdlc1y+d8mMUPOkg3Utf7v+UcJCgaCFtr/hi/5i+k6X3kq6T0yweeQLqnA2BN1kAcAjn8+PTAEtdMHo6ZLfRilLbXwFmZLSCq9rcjve38fzx8IHBM8s5EdXGT92/5BfpRvtSpIYwZKWblrlaB/wDkqDDafXX/AM3HEoSofs1n8k4v1TPS9K0tK6P0lKldtVUlgn8h7W574Ocp3TFG4k9G6VqHquqymwRY+/8A5740O9mleT/jN+v6Ko+2HCf+zL8m/wDko10K6ETupk1uvVdDsPLcdd1uJJSuaoHlDZ9E+il/mBzcp23TaZT6PAYpdLiNxYsZsNMstJCUISOAAB2GKNp3XSfT4TMGn5Kp0aNGZQlmMw+uzaQLBFgiyLC2Dm/iIrCk3/orDb+ryuTXEkW97t/bH0vh3AKbAKfsotXnxHr+g6L43xXxTWcU1Zmm7sbfC297fqT1+WivoY7iiP8A6oOtXQtOWYK0LNnHEz1WaJ7AjTc39Lccd+cDHX6uaCt/KkRofupM9ZJGu3PksOATycehBBXlLWV6Y9iiv/qgasCoHL1PIQeSmoOEWHdV9Pb/ALxhKPiNrG4tDmVYbQSUjU7OeTqvbsNr7/xw1wor/wAcBuOMUK38Q1Ze/Z5Wg+vBqDgvbv8Au48n4hqtZKDluBrIC9HjHUkAi97aLkenbAuopT1k61UXpXSShW3MrstCvBQddrjtuOEfS2D69z2HqRhrM2ZK/musSq/mGovTJks3cdULceiUjslI9EjgX97nF6V+o9Pcz1mVXq306p0yoznNch5yvTgorFwOOPIABawt5rAd8I10/pilQQrpJSUqQkX05hmEaeebjv684+fcScP4xj01myNbENhr8zpv+C+s8G8XcOcKwZjDI6d3ifZtvRve0H1O55BUAEOEfUo/e2OjdSNOpX8MX4Kf0wKg2jpLTCkoJCzmGYASB2HN/wBcAMPpgVJT/wCiGAm47rrU4el8eSHs2xNm0zfr+i91/wA4sG/7Mnyb/wCSoMtOAAWXzc30H+XvgBQ7x5FfqnGglU3psWGkK6T01aE32UmvzbWJuq1z78nBTlN6ZJbW7/6JaXpCQo3r80fr3xHezXEn/wDuR/M/ooPbJg43hk+Tf/JUO1EecUShpaVqtpun37YPqdHqFHqD9JqbATJiOFp1KXEuJBHeykEpPcdj/A4vWEx07psuPPjdI6ah2E42+y8mszVhCkqCkquT6EJ/L7jHHYfTqTLW850gpq1rUt1a/wAWm+dZJKiOeeTyfvi//lvWZdZG5vjb8FX/AM5MM7X/AAZMlujb35W721vqfLWlct5Rrmca3Ho2WoL0ibLXZDKRYJHqpSj9KAO6j/fYY3V0W6I0LpNSAs6Z9dlNhM2oFFuO+22D9LYPp3J5P2qfJmfaZ0/bkpyj0yosFUhSVvLRPfW4v+yNSgo2+17fbElHxGZnC1o/ojT07adR1TXRx7/Rj2PC/CEOCDt6g55eXQenn1K+dcb+0Ko4mHulKDHB0Nru9bch0v5ny0KAALY7jPafiNzK4NQytTtJH1CY6f5Wxz/6ovMqlpR/RqnJK+eZT3b34GPbXXzVaE/XHApJ+lWM9q+I3MgI/wDrWpwBAOoynrc/p/LA0/ELmQ9qDSlH28U8L/fnsMS6i0ETxjL3xN09qFnmDUQEpTU6ba5/eWwsg/wDiMPzfxD19att2gUtIJHIlPqSQeO9sQvqZmmt9SlU0TY8KlLp2+G9Bdd3QsJ4NyABdGLInhrkr23CrVTZcdSlIUVBfAHrf0++Kp6m9WEUsv5cyi9eeFFuVUUfTFWOFNsn990dirsnkC6+1wTMoTpUd2DJqoLTqS26A06k6DwRdDgX+oIxDah0PyRAp0qYjLtMcTFjl0tpVJSLAX03LxA4/wBGM2IPmkj7OmNr7krq4LJQ0s3a1rS63IWt9Vl+Q26ptSWlKQs3IUSb673B/O9jjSeUKu5UKQzWIxIDzYfWLdg4NVv0JUP0xSfUvO/TfpxUE0+R0kVP3I0eSh1qtOM8OocUBpIV/tZHfEh+HbrXljPlWmZUo2Uahl5DVMXMjMyZjcht1tt0agkgBYsHr8+mOTh+Gz0bXdo7Q/iu7jOP0mIFohYQ4dbbfBX3Vobs2ja0Ku9HtJZUO6Vjm36gfyxd/wAMecGk5lnZfWsBjMMRFWic8B9sBLqR9yhST/yZxE5+U3ZOTZWb4oJZpq4zchKRwWXRbX/mkouPZRPpiF5Lr7+T68xU2tYdy3UkTQP7cRy+4n/mF4fqMdSN1l5yVweCt6FIKgT6Yr/rnlH+mPTOr01hjdlxm/GxQPq3WvMAPzAUn9cTyPIalNNyWHEraeQFtrT2Uki4P8MDcTqFvf8AngVdO2rgfBJs4EfNJQ1j8Pqo6qLxMII+Bv8AJfP3KbkBSCmp1lFEitFMpD0WCl2W44i2kJUrkXve2oJ4PGNCdEa3TGsl1pmqZonTWJQkONIrKUIUtltBDy7hSlFv3udKeALXwzZzpXR7KeZp1Bm9ImnltWUFoqCkpdbcGoEJvx6i32+2PJ6hZJgZfq1CoOQBSzVo7rLjwqAXbcFibkFXc3t2vz3x4fhrhSswSozyuaWi9rXv8eVui+i8VcYUGPUjo4GPD3WIuGWFvS7vjfknar9I8tzm0y6JPk0sOjW2j/VMVX+5BNwPyUfyxDqp0uzhAUtbMFFSaAJ1w16lWH/ul2X/AABw2UutVPL/AJ6HU1xk9ylp75f+c2vyfyxM6H1hnIIZrlHZfRYHfhKDSk/dba7g/oRj3GWRnmvm12HxKtUIGtTO0tKkdwWSCg/kvkfrbAvDocSFvpGscnm3HvfF6KreQM7x0oqC6fMcRwG5p2X2/wAiSCP81dsMVT6M0KYpcmiVJ+NqujakjxLV/YKBCx/PEbN95KY/uqqVurU8Ah10ni6iQQQBjjbzqDpLnIBKQk+nvY4lFb6dZlogW87SS/GAFnoJL6R+YA1j9UYjKNC3dbUhvycL4IUP09P1xe1wcEmXLuj2lvJAWzY913K/9F7fpg1uXqWC+GkpPuCFH7YSpQsALSpC0q9Dzg5LjyRpKNKST3buf0PphkNOSPLkYaCpbjdzwVCxP64OLLp8zTofB7WHGECFXcJSRqtZXe364NZZ2TrjttFWu5KfKpz7Ej+WFJURin0pQGtbukW0pUuw/THkzIyU2eZXxwrz6ir7/wDhhledlPLsHnE89lWTpH5dxg1AkqTpdcaBB4XcufxFhzjGRfdX2Tm9UY6AqOlLikfccD8rYKDuhQ0NAfZSeT/5OEa0PLUlKZbnPI2wEX/vxxVLLyXEuNXIN0a3VWBHpx+uGa26rSxxxy/mmNIsPMA4LH7d8JjJjJUUp1k37NIKr+32wOBTkS5P4dBjIflrtdmOkuOq/JIBJxLIfSXPE51DKqFKYZVxrkNjSgfdOrX+gGAco8SaxUVW80olIizB2BLpCbnBW/UlJMYO6GybLQ064oc82PP/AJ++LQHRWFCS27mPNLsO3Km2mWWir3ILzv8A9piQwsk9N41iiiZnrym7aNKVON/l8opR/E4UyNHhQyOVFqSp4bTknUpZ7WDdzbvze+FdOy5PqgIplJk1F09/DRluj722xbF4SKVm2KSMl9HMuU1CxpRIqDzJkfYlKb2P2JOGKpZZ+IaakmqZoiwmQPpZqKYyQL27NpHAHPfEEnTRNlUO/wDRdmqDF8VNpselM8fPqMtiMn/5lX/lgLOXsnwVq/FOqFPUEgnbpzDk8g+wUhIR3v6nDlLyNk+lrW7mjPMirS+7jNOVHU8r/OcLpv8AqMKqS10ikfLp/T7MuYJqHLht6Sp0k+9krtb9MNmzeJG3RMblV6QQSlBGbJzqCdIW6xDQvtfyjzgfn7nEgpDq6gjXlLoKiUoA7UipbkhN/e7qUpP8cSykUjqFHSDlXp/lbKDBF0uPpSqQkH30ajf9MPK8hZ3q1l17qnU1oA87VNjIjJ+/mv8AzthC8A/3/RSyjFIoHW6G0JFQzLlzK0X99JhxUhI7iwQm3H3VjyJ2XYslK8z9bq1mOVclECm2S2v/AJNtJv8AqbYe05S6R0WYgVJ1mqVFKvpmzHJ8hRJv+yTe55/s4MkVPPyHnYmQOnVOpkRJ20ypbaY+4n0IbOkj8iDiF99tvl+qKQDN05thZyX00zK+RxuPM+GF/fhN1/xwjfldUnnGqpXMzx8rsG6g2tBWu1r8pWmxt7FfH2wqm5K60ZhVqrOfIkNsW+TB3kp73N9tKb+nrhlm9I8sQVKezX1QSl1Kef2QUn8g4tw/yucABv71UQZMvpxJZSvNnUjM2adLdvDNB1pg88nSgD/p4b3+o2Q6GrZyl0zp7K72EmclC19uFaeST9isYNVE6CUJSNcus1x0Agq3nAlXHrYNj+Aw+0XMraWtPTzou8hPZEx9gJSCPUqNj/8AP64doF9lFNOklfr+aaLJrlcWgNuPbMVptnbbSlA8yk+4JPr/AGcWAOBzhuoZqS6PEcrKW0zlspMlLY8qXCLqSOTwDx+mF6uRyLY2jZUON3WWYfiMrSav1OpeX0O6mqHTy+4n+y6+b/8AQbT/AM7FSKcNQzmtP1IhJDY9roFz/Mn+GH2rVwZjzdmjOrt1sypzqmSrn+rteVP/AMjYxHsnIeU3KqkgfMkOev35P8zhxZXNsNE75sq0iLlqXT23SlmouMIfAsNQbVuJ5tfuL8YxP8XMypCkU+0lbcPxYabYSbbrlidX6f6cbTrMCNWaeYsh1bZCtba02ug/r9uMZW+KSPTZeYKNliBsLRSKY/LcTv8AzG3XTZtak2N1qsQNVgADYdjgRvb8UkrljV+WuMlTSEcJ7ntz+eGJ+bO1myV88f8Ak4e50J5yYW7nSjlQHYc2ODmICXoaHmY7hZcuUEot/EemL3FZ7Zkmy9U5MeUhSVlBHPf1vjREHPSXun0qmZwU9MS03EciVpDiXFtNtO8NSUdyhKFKAdHI4BBAuM8MxVeI87ZAJISQOQf/ACMPlLrsyC25EcjJeQUFsk88f+Tgx1DoTdq20dXJSm7Sp7IyZRHJxQmGyuNUkLMV9LykGHLWoKBbU2foeQCBcEa/SxGJfI6MQqnBq9eytSG4sdEdMSoNMtOFqQhYsAsG4Cr3XuJN+Oe+KZynXcx5TkiLTqkh+C+rdRTpibpKAQbtLHCb29B6ci4xuLobXspV/Iwj0eft1uW+HJ0Vx3zELFtrb1pQ4Ba6CQfW45xsqatk7AQFz3Md2rnDYrFFSzNPoCW8uZpgutux/wCrMuMgaHNAsDp4sbG1/Xi44wldqNKmyPBUeHMq0+UNIQw24oqJtcNpFyfyAA4ONKdbst0yn1fNkSvZdBh0/cdVBeTqcKG2UqaJCPU3vx21W9L4ofI7fUTI79JcyvmReXYWeTGdaajR25AcVZX+LcustNg+qhfWLXtjI5hFkGlQKo5NzPCk2quU6tTnOSfHQnY4Hvy4AP4HBpyxUGX2VSGWi0vg25/8CMaFzfF6y13Jr+WK5n/K64FQcYdKpVNcZU6GnAtINiQQSASAg3/LFCVnJGeo0Q1rMEVunNVeRIjteGQhQ3o5SlxopQq7J+a0oBQGpDoKSb4rcDyT5VHaxnOpxnXqZSnhEabeWorauCTbQLX+myRbj3PONG/4MSCyj4sMsZjn1I0+n0aLUZc55wENqBiqbabv6lS3Pp7kJPfFb1H4cRl2Uin1PMr86oyEo2ocOANSFm191TigAB7C5Nu2DMp9Tn+jzdFcyu7sO/0pTPdIIYW7DiEtAPFI7KccdPrYJwpYWNv1VzHAFfcWq9VqYypxGXKfImaLFUmX5GiPWyR5z6dyO+Gpvq/Uw4UP0ujhLfJO87ZVxcW54P2xV6KlT5YjT6Utx+PJZRIZcTYbqFtpULEKsSQbetj2wq/Fm46UqbeaDUdvQVboV6Wuokcdynvzf7YwZ3py3qFY6es61NF9iiU6SAvT8uapJJAuRp5XccXAHBNvfAY/W2pLcQmPlhLaQXA+FynmXU/2LDSQb978C2K7XUIrjwQIoS79HzzYk88pHBtye3e3PoMJ0uPurWp+KG0rKEBCUEJSdBuiwNwewv6c9hbBa8jcqZbbK2I/WNiUf9jrjil383il27HkqLRsLi3vhJX+peXcxUqXRq9ktNQp8sKQGHZgKniEhSVJCUa0kHs4DrSRcWJGK4alvSVKVocfaLQKnQ3upIvwkKIAFiebHyEW7nCuFl+t1UHwDOhssloPL8jDS7dm+QTZenkAkjg84oqa+Kjj7Sd4aPNXRQvlf/K3UhyZmPprlB51+hZbnu1Is3RImVZyQ4oHggLdFkdrEgXI9TfmaMdXqdMu3T8s1OU4nuiM404EG17FX0I/U/bEPp2UQYkX+kTjc6cEJ3kQ1ONRNwgBehKjuFu4uAomwJ74VS6rTKCwYcaMlaGW9QjwU2Go8+2gepvyT7cY8jNxmJpPd8JiMjviB9NfwC7Iwlzz21e+3x1Vgs52iONpW9SpUVSiflrcbUf/AJDbBc7qRQKcn+uNyC4RqDLW2pzR7kauB9ziqHapmavSVw6P4ZphBW26WGyU6e3Li9JHre3Iva2F9PyfEhlL851ctz1QwnQ0knm5VwtZ7C5I7dubYodxJWYYc+KTtB/7bWgn4nl8Uww2Co0pmH1JsP1U9idVcvTXEtxqTXHQbjcajNraFuD8wLt34v62OHX+mdINk7cu59A1f/Tiv59ShUhkMqSQW27tsMICQAOw44QPT/RiG1euV+oBcdphUZofLWy2vzGxBPnsCTbsOB+eLqPHMcxl4dTRCKPq4Xv6dfgknoMOom2lcXO6BW/O6pZKp7imX5763kGy2WGt1xJ9iAeO36etsJHOtGQ4hDkh+pp0HUUmnrv+Q578f34osMKQ8ChlwNEdnEFJKSTYi3JPP0Ajt6YKfYmtqIQsxXeVKWokuJRbnSLEkEA2783vj2kL5GsDZDc9bW/VcF+Quu0WCHvxy85MEx1KlEAAynBe5vcHXySCmw0d+bWx1Trl22ZUmUvzCwflF0NJvybX57qFxz7YJj0mYd1laWChy7m63cqUb+4sOeLW+/pgx2nSWYqUpGtoqBdulxSgTa17dzdI47g+/GKmUdMx/aMYAeuVt1e+rnlbkfISPUrifnKW0pTgbUL3CtV0emnmwvYgkj0wcwYwcdV41tLjgC9LrZU3Y2HBNgfa/fjtgpuFDSlL7MOSG2HNZ1s6kuHQDxYkDvbm3IP54ODchwqbYsGy6QhbiXGwQvSR3uLk8W+3fkYua3N3is6CtNMt5Xn1Bx0hBCV3PIHBANkevFyD39wIrjOTQtqQ0doErYTfi6ub37GyQeT257E4GI011vxLO2lLZQoWJ2wNd1I5Vcg2/wC71wTGbksRm48psvk6GwpmI2yFeh8pUQ33JBNx6WBAxGqLri2AX1JjBfNzdpB4XcXKb9uQQeTbi3bHHnkmQWTdbekFspQp07iBYJA7Dixvf+Ywa8+XFvRlrCDqcBWqMRqF7k6eF6yRa+g39eOcJy88y2lXjUIJsLKbOojsCRckWItwbdzwDi5RJpLjCA4nWvUtVrLb0pCwfLa6ddr89zwfSxwnDcRFlTYzRU+dAWgn5nFhfg3tf74XutraWngvsLuHXHwFAI73Jtx6D3APfCeNBeDqAIyPm3KwVgJTyP2hJv3IIt7n3tiJSjVrpjaA+VRwU/JacCRxxYnUe/b0+/tbCpKdZOhrUpZ8hcBIKwP3T/uOLki9sBbfcbSpLMRaFODzi3qLkEm3N725/I2wdGdedJhq30tkEje2eWzcXPJA57XueMRBFMBhvQlzcfCE/MSbK5HNlEG/bzcD9ccDWh50voQjsgJ2/Q9wOLhF+f77YEYkttLKEusJbHAbIWotkd1pJPPPJ4P5i1sJ1xihG0zoUXAha1M6CSi9tY1n/wCn07YZRdeb0kbsddzZGos6RbuCb3sLc39/TBSXHkJSlCwV9/mKVci/BJsbc+mBojXQXkhbqlrJ0Ibs4Cv6jcnye2rnn37Y4yEApQyrZV31Oi1j35Iub9+bYBURJehu6iJDTqNBAUDrRx39P0t+uDUrISCwsJUVgnU3Ygev8/Xt/HBYadbs89tOk28w4AJF9N+Pf2Ha/F8GFlWoqU4EPaNopD1kquTyRb/cn+POAjdBUhbbzjrryoygdBQlw3BNrg3v27jn7c4NK06hqloWlRuQpVh2ub3/AEv7A4IaZ8OkPLkNIYF/IbjbQL2/hzz64LW3K1KdQGgf30Fwg97An/6PfETJb+wshYLBIBWhNwFHg2uOTz298ccmssONaXttSyNLahwAOeLjt3/nhKWNRKnZDDaLWJHYX+4N+57cd8FrXGbYGpqQpTg0NBtpblr8C97kDtyfyxEpSzfS0jSl7SF+VqyLKPuCe1rdgPc3wBZ3kBlRsgjcIXIsRz6AAWt25vhC0laVlEAQ233HLuuONHUoCwuFWPPb7HChTQWu6FNJK16yVL3OOAUCwvew45wyCNjvrKEa4PzEJKQdzU2fY7gsDx7YKdShViuRICEJ3CpshOnnvfvb8sBXS4L50mOhOvkLC7OOW5A8g/lgtxiZYtOwQ4EaFAhwqUQDYdgbnkEfb8sRTdOG+8w4LeZKBYuamyEj7i1z73vhYXlpTurfjhFwQixta3Nj6d/S+GyO8yyVvCC+VtH6tQUlR/yLcm3r+XGE5DTavExosZC22y25w5uKX9ref1v+lsRTKn9Co0hKXG5C44QEJVqbKwpuxuAbg67evP5Ya8xUqNXcv1iiJmRHBPiuxk7guQ5a7aux7OBJHrf2xyMXw8ytqTIaXbhsDi/oskep9revvhautJY8M8Ibu2tfzHktBQaXyTuG/F/yPpziKLAfxCw01WRSavpQnxNKYvfjQWn3bg/kJRuP8nEQ6AT2cp9VKNVVLcRYLZeTcG7Dqw2qxv20m9vTTi7viNymw4zW4cRQWiBJmKjLbJIMeQ3vtWPcgeUc+oxmfpWtyp5iq9UZTvMx4jKQ8pXKVvkAWB9Plkfx9Di8d6OyTL3wvtr0MhwM2dH6vlt5aFvOOyaVKF/MPlAIJ/zCkj/wxmZ0qj1enTpCNPjUmBLTbgOEcE/8om3+d98Kvh765Ssmyo9fcjGbArlLj+MjJd0kyG7jUk+hC9Q59DhLV5CMwQ5yWiEPOuLlt6eyXCrc4/Im2MYdlIC2tHNa9+HzMyq/07hwJDhXKoTiqU6VHkhuxaJ/NpTf6g4s098ZW+GTOIjZvEF5e2xmmCFBB9JjAuB9rtl0f8mnGqR5h2ti8bLO8WKo/wCIvLbDppOa9h1RQr8PkFokKsq62jwQbBQUP8/FNGNu+YIBNuewH641rnugf0lyrUqQ2jU+6ypTH++p8yP/AJgB+ROMnNPNuoCksoSV+hOrT9v07YKF0ldD6FpSEtWPFlLQLi3oSb4PS8oJIkMot2GlYT/549MGvpaQk7i2wEcDyYChllYWy60TeywUm1hiIICnESY+2pDdzeySRz9sO1AzBWKK6lcaS623zZJcCknn0I7/APnjDIaaySXG9xlu5uUKJIv7c/3WwdHSGAFIS4n0CiRb29P9OAWh26itrLvVOIoLRmRDjJR9MiO0HEgd/MEHWP0BxK36blPO0PxTrNPrDSRYSGzdxr/ORZxH5HFAla2ka0J3Lj6iRYffjvgcaXJiPJnRZDsN5AJQ8w8WlX9+CPT0OKHQ63borRJ95WRW+i0d5Jcy1W3Iqv8A8Hmt7zR9/mCyx+oOIRV8h5voLTkipUF0soPL8dXiGgPclu5H6gYkNP6tZnpygiQU1JCQn/VbSA6f+UbIuPub4mtN6uZXeKE1FyTSn9OvUUlxr2PmQNYH5p/XAvLHvqmtGVSDZjECR4grQkdkDUCD98ASEPrLaCsAEtm6CAcaFnZCyRm5v8WNOhyN+x8fAd29z81NmxP5gnETqnQ1gq10evuIUjnblsXv9txsj+af1w3bNcO9oldEfsqkXZ11DZpZS531K8wH/hgzenlB8waTpFitAI/PvwP1x1UDfaU2meQskldm0Ekfe/pg4U54BRdUJKyQvQQNIBHpb8/tityVFomvuoCFyyOLENNi5P8A5++F9LkwWJLEufRV1NlCtTsZ95bIcHaxULm3b09LeuCkQAyVNJhNISslZQ4si9uw5PH545cKWv8AqKH0tkeYOjSfy9yPvhVFadH6uwWYhjRoBy+wk28NSafHWCBz+0eULn/k8OLXVDJs/WmbGzjPSPOUyH22W/y0tLSj9LYp52XFQ5rX5yjlWhvhRv8Alf1/iMHoqDxOgtrQF37IH6E35vbEdCCEc6uJPV7JVGeBpGRH0kDzuFDLagfz5J/jhBO+ICtvrWmm0aDGTp8pkKW+q/vYFIxULk6SlxSUoRa9tQ12v7dgL/ywNDU5xvxG80UrPNgdV+OL8/xwvZN6IZ1Mp/UvO1TSoTs3Sm23EGzcFCGLf83n+eIlJc31bs95+S4gX3JTi3lcHg3JJ9bcY6iOgqTuTEBP76Qk6h9yfbHF0+MhKTvl0Aa9ei4A7e/37jFoDGt2UzFSDJcjp3CkrezlCnO8AsNRhqYI7+YAhZv9jb3xZS+tmXKfGELK+W5TzDYslGluM0j7aU3JxSjDaCC3Gku3QDdKOE+vNj6/ywrCoyGttKApOrcJJt+nFvthSwFEPKsxHWKpT1KEqRKpyUGwRAgMknj/AG15xVva+3hBM6j5XeQkz8vVOtuqAKEVarqIXcX/AGSBt/8AyYrtuQ9KGjQ6APOFKFgo372PIwLwMosmQktNthFiUqusj7cX7f8AnnAbCBvojnKsJrqzmimsFuj0PLlIik2S3GjkkD2IuLn8sNlR6oZ4m6j/AEkAQgdoYQyfy4B5xDjCWpH9ZDhWO5svaFz5QAbXPv8A+RgtuOFqcWFsIUTYLKNNza36c3H3w3ZtUzJ1k1+ozkiTPrUuUsG6QuW8uw7EgL9eTwMOdNe6cspZNWjZglutedxpl1lhlBH7tkC9uO97n1wzxoRdGl5tCFAaEt7g2iQO40AenofXHTHlQWBfQhKPJYJ+pBIJFv8ARxh2tbySqyqV1J6e0lHiKXkZUYhXDqUsqX7X1EE4kGXOsNLzNX4NDiUGoFyY7oC3HkkNgAkki17C2KNjR0LQ25LeQy+u39WRy8dd+Bo4445xaHQOlKk1yXUnWEpTTmNscEkOucEEkX4CT/HB7FpOqYE2V9I7XxD+rWZF5U6c1+tMu7chuGtmOr2fc+W2R/nKB/TExHbFDfFNW70zL2TmXPPVJxlvJHfaYHF/+UWj+GL1WNVnyaxDpOVUU2U3usvoDCkKJ8wPf19vvhM/nPL9GhJYlyXEPo85aSysqIXyDe1v54QZ9q4bqUKmIIO00X1j8zZP8gf44qTPLdfquZakzDs1HbWIzSgeShDaUk/bkKxS5rnO02V2YKWZt61uU+mKk0SiF2XIcRGp7Dzl1SJK/pQAOLDuTf6AcZr6z5kXkfN0Oj1WeajKk0Zyp1mShfmlzZLhsQPRCW20hsXsEAD1w61PM39Fov8A6TK6WpMaIHYGW4znlVLc7OvpA9P3b+yb/vYzRnHONXztmWZmGrzFvzJKwCscaUdktj2QkcAemLomZFne9S2ZT0TaMifoSqLUAIxVtgJJX5iEn3uPTD6xQID1DRBYW01O3ggDslKNBtcD6LWTwfe/rhz6VJarmUG6Y6kLksLIZb02sgXusW59QP7sCfylOiKcVCkXYkpXu67AWAGq/wBuQLnFqraopNytofTsr1oCGtxQ9CQrVcD2Kefb1wFjJcxTa6g02hLJQhwqWLNgen6XsPyP64nmW6PJqrQRsuJS46C44m12ySb3Bt3AN8TGq5Z8TGkRoLior0dsNoaADOtoiw4+hFzYA3tyPfACZU81l+lTHlRnWwiOizzkSSssvN/29tdib3/hax7YkyaXHph/CkWfnJbQ63KS8UuuNdi2pu224Laee3a+m4JcEwimbtvz9MhtBcakqSHG3rHsoL54N7kc+h98P6GEeHfemRN6LthwtJvZtfYrt3BBNwQf07gQmya6g9bzl1MoraqlTJBlvxwhAiTmg8zIaQCAW3NWttQT+6HCFeliCDHMm9ZelTNUbm1zIScv1VDi7T4yVSGmCb6rNk62hyeEA9zxifVuTKbCJbKW32J7e4G4bxu40bkuhQvrWCBdJsfWyhfFbLyvkuuVJ6ZmKNK2XjoEhlQS9HcHJDoHCx25A5B9LWw7ZnjRIWNKuN/qR0cjuKqxzjRV6mwApK1uvkWHAGnWP5YhNU6p5MqOYEVSLVtmkllbE8ArWhejlpx1Om5VrKQNPACBf0xAs5Zbj5IVDy7UaDCqlDnuOPwpjzWxLjP9iwqUgXcHCCAq4seADfFe5nq095CKczShSqc0tSWoreop135JWskqPAub+g4GC+Ryvp5RByufNXPVepeQI7c+sULNaxPkIfkeGDL62pElaDYlDidLaio3K02v98UQ5UX585kvq1JbbQy2CO9v9JJJP3Jw0qB9b/kcHRm3C6gpBtq79u33xW6RzrX5JppRM7NYAr6l/BD1Bcz50cRQqjI8TUsouopbgD1nlsE7kVZ18EBGpPt8vnsBi+UQoyHS0udVDY6EF2Uh2/vwuxuD6WvwcYF/wddYnr6yPZGTPbZYzVS5LCkuEhCnWAXmyQAbnQHBb17Y+n8TpxR0uJlVN5+pyQbjVdCUEc3GnzXPqSeeBxa2PnvFE2G0dQXVc0jXEbNcR9NgvRYTUVXZhsUbSOrmgqECkt6/C0VuozqgtLaXXJam9PBuSW20n6vzHax9cSKFkepSIiTPESnO6/O5thwrRz2a/dWAe6iOebW4xLpVXy7Qrw1TmI7znCY7DZUorvbnT35PqfQ/fDKvN66lJVEosZ0ltJCy0z4h5KxzYpHkSft3IN78Y4lPiuL1MPZ4ezs2fflN/qfyVs8FKX56g3d0aLfKyc41CoVIiKkOMb6bDW7JJUDbkJDfbj+zYm/PrgmbmqnNulhvW8ttKTqWoMtWPaxPJ54sBhG3lqvViQZVbdERq4KNThU+gc8FAOhPJPc+vrh6iUGg0iMHkRkuBkeaTMsrQOPU2SnsP4DHHqnUEbw7EZjUS9G+H5/otEb57Wp2BjepTEpzMteZbTAUTHd1odWPkNNcW1hZuXSk8WAOojuBhwhZXiwym8mSVBwuKQw4W0KcNvNYkkcjhKSANRsOccq+eaDT2U7UpMl6xLbYVso8nfzrsLD2Tc8dsM9Wr9Vkl5oz24sdwlttEZsh0iwJvrus8EfTa/bjvjpUlLjGJC1HEKaI/A/Pf8FTNUUkH+Mc7k/y6pR6OyY7BbUuPwIUWxWCe1xfj81WOIzVs0VuU5JYgKZprBjjaW42XVrJN9Ta0cfSCLED0PfDO5Lh7wjtjU3YNlFm2TYkcm4859OLEg2FzbCSOPOhtcVcp1N7BDbekLCk2N+91Dgc34t647sHBFJHGe1cXSH7X6fqbrF/HpWvDmgZRy5W80mlynXXHXXBIfWlxSSpEKSFkn6yLlPn4vf0F+MBe2vwtqXHeXJeFj4fwjrDJdWvzEFwi3Oi3nINjcWwtchxWpDiGltlLjdgqTcm9wdah6WFge/ew98J3ETIjm+UPuJSkpDxaKRpJuFgXIPci1uw7469BgbqKQP95e63IkW/BV1eLsq2FvYNbfmAf1Q0KISW3UaHPJI2VObquxDnPoOe6SOOOfUKgyyhlCnS00NBdSoApFuGze30XII9fS3e5niGZWmMyJDmwXCWtY21FAvdsnuB6Hm9+eMFsinBAClCOvQFFt9aWw5Ypsn1JAOpR5twB3Ax6EC64zkQ4Fshxe7JdTGK9xLjaHksEm/76SQefyt+uDGGUANuIkrIN7JUBa/+LI4sSrngkcjtjzpjvGMlCypTa7l1gH5aNF/mX84JNwbA3sDYY84ESC46/t7juhslQ2lkFNtViAdfI8wOgAXNrjAzFBGKcdVLBQ2+22wpG4jZSrUgEXQltaQELPHNzceQA3GBQ1syIjPiFKeCELs1H3G9RKyTfmyzYjm/bi3bA0NMS3ExFMoZRtE2S4HbIKfNpUebnjki1gL83wJLYXKbSlbSHpDYb3Qdpo3tZFnLEAEemm9ue18RrsqiRPv6AqQyttKze7KVE9zfnlJuPTgH1OPPeGccW4/AQRpQ03cWdcvp4Osm47kgC1ygHthQWXA+X2XmHGXTyXG+PTUtKrD1Jub8XA9cJi80pYe2iWxcM7o0m1xpQWu6EEdlW41c3wzW6ZkLrr51IWyhEtpa0r1oUtepkH/bAgFH2vew9uMFJTtLW66sBN1gvbY457gEdvpTcjk/xwAsw1LcSGXYwbW262EvF1sOtnykAgWWknhXqee+OCWmMmQX3XFOAB5aGwUuCxLYAVqUFkk82ItdJPYnDXSolYcDrji2lsp2yzcKWSSsgp44APFvr7jthWl1hrSZsR19smyktFsF3z34ursRc+gH6YDFRGL7ExDi2WxrUwkug3NrEhI+sfcEi57+3mJUiO0lqQm0lxYdfYdcTqDQb44AtzZXII4PIODdHdDYMp8JQzJY3ApaWkhgHSbAEXN9B5BH5gj1x1pLi5ClPTELDp5GgtpvyA4kX97XSbEfcYC6p1lhzxcly7pcZc1BTht7rIGhGnk2KD27kXx3SrYQzoQtm5U0lwLNyBfbA9rhI4NuL+gwUEWERJEde1MDqkLtYq1aXQLHUUDgixBB9xc+oCXtaXd9mQgrRYJc0FIR3uLG3v6+/HIwpWjcSUs76nU+Xc06g2AbFVxbi5A5vz72vjsUPOurZRYocC7kC6eOFAAi9ybHtdX374iiRmR5A8iOhwoSD5LgK43ABc31+wJueecHSNaG3EsaNLvmLhN/Q2Pbv6ckfrgbz7SGkpUh1oI1kJ2S02kWGk3PF/ztbnvY4KW1vtNrXuodK1qdStSQk99IJIN+bG3obc4ii5dtbiUz4KFtC5bQpGoN8auOTc9+4PfHb6m0tMpCESG1raSlu26AAb/exI57cW+2DXC2ym8dBW3I8zRakcqbuQLBwc83Jta38MFOPMlBDsYBSyAlDrYFv3rWBsUcH6f09cRRE7ylWLjTCeAATYkG1+ObkHn0H9+BAxgvb2kKkg7hCwAVN9rgcfw5NyfTsEJjJuqGtiOGkNthJI1gm4Ha/seL+uDQiZIDel5aFLIshWgngjiy+38h3574iYIpxWp3ShBAsLrVa9yOxA9fb+7Ai1GWkqQB5PMEOCx49jzzz9r/AGwnQ28QtxaWnXNI87K1KTyL8qte329OeMGgLTH1R4w3XFWCLWCuB57jn1Pf0xEqKXrU0lKF6XDcFLjRuLe359vtgbmsBOstNqQShegG9x6YOLbIQ4HYrSmnzo1rVbVY9raTySeee5OOlhp1YTIEhwLukavKA4Pz4JP8+9xggqJOpwFIW0NaQTcJdKeO30/w49x64PQGUqi7LT6yjXYpbvY24v6c+n3wPwLzbIbYhOKVYNIQ4QFaBaybk82+3rbAQifsqDLzSS1z5m7p45ueRcfr3wFEeX2RpBZJWtZF1uHgcX4NvX1wWlDD99KFtaASdDZSpm3JJPYdh/o5OOOxA4h4tLQgatJSUEFIIv8ASb3P8vvjiFrjvHVPCUaiUhx8pSR3BNuPb+HB9MRRcW65dK3mY5FySUoPCP3r+wPsPXBq1NNaVqcbbcbWCCl4pVfuB7XtY8YIW9voU9EcdWgBZUpICtVxfgkXPa1+O/OBDxLwS42toFRQVB8cgaebXuCftg3UWTfjqzRJy81FoVAZkSZeaaWPHyGWHFiPGYeUkquAbKdJDY58oSTxdOKM+HrKM2r5Oz9XYMuIlukmG680sHU4hph94oSrsLAK79+2N09cqKms5TjPqeLQp8xlba9ASUodG2rm/Y3bNha/fi2Mox63UsuvVVvJeY2KPPmFtqqxGdgSNxAOnUHAShWlRAWP3FdzcY0sc0ssq37oqg9XK1kyKpdOkw3IV/Erjymg4mxF1FJBC0XHsbcXtjTNPzdS40hsyFKaZUUKS79SC2RdJPqAQRjIsCi5ajFqBFcbbUBtBtzyi440BVygm3bkH7YuamyqmMv0+G7FWJMSK3Gc1XBBb8qTz7oCcJPCHahWRTG60FljNEaLUUVnLEpuSukTxMYLJJStxpQK27+oINv87G+aRUYdXpsWq09wORZrDcllX9ptaQUn+Bx8sOk9SmU6oy2ZyghpchiQ3c8AOAsufz2Tj6A/DZmJNSyKvL7ixv5fkrihPr4dfnZP8CR/mHFLO73Vc/UZlbZF8ZY6n5bby1nKox2isRpLnjGAoEJCXbqIB9gvc/ljVGKj695e8XTqfX2W/mx3vBOqAPDbp8pJ9gsD/nYsVKodSmVLDOlRQbX21myvv+uBAJLaUKSEpItpUb2/K3v98GSEJRqSto8N3QpJuq/oSDbi/F8Ji0poHSwolyy9VikC9+f4DEUSphbzSllKVoCPfuf0wFEaOpLbxW2VBwuIWUEDv35PH5frgoOgBGhlab/vC5Cr9jf/ALsDDhcQt1p59FwAtFu//hxiKI12K8oKeRGaSlYPCAeR3JI7DDcYrMlm8iI2tBNi2JRSR79gb/w/XDi2wppWtDhLTqOR2PbvzccYKTFZWVauEqN23HAfKbAWFvQHnEUQIzKY1tUB9CW7gEPbg/LjgYNiKS054lp4FTfmRpF1Dj6e392Op3o6dRJ2QSVui/P6W9v/AKcGKkIL/nkyHEC+2hdnNPsPSw+9yfscRRH0us1eBMQ9SnZUNX1OSI0lbKj/AC+Zz3CgPzxY1C6xToNmMyRmqmkabyooDD1j6ls/LWR7jRiqEvrQpO62Gi+dtOhtzSq5t9Z7i/tfBjikmEZI22lbYbeYcBBJvySDwR9sIY2O3Th5GyKQ67JZTob0BHclvSSj1++PEsqZAStCkmxcWDcnnvfi59cExEpLoZLoU9zZJPmv2uB/pwN+G8n5zzCNIBIdI1fMJ/8AAj9O+MzhlcojtxCgpaOS0ASDyT6ce2AMRuNaIRKgToUT2BP3scDjLacBZbkFza5KAnSTf349PbHpLSy+9ruSP2aiTptYfu2Fjb15/XBalKD4KQ2gLcDaELJAt6n2wpcjpSlS1lF7CybabcfzOCm2kABlLJWlZ7IsU/Y89v54C0xtSSlpvU05cWUhYtbjgG4Avf8AhgucgjW2mtQUZaCtZPkU2UEewIItb8uMAad1LcbLJ8hss24BIuDx3/L2woktx1Lb1LKFhoE7hNwPXTxyBgoBAWgpZdWlaSFhay2E8cEX9z7DnAuplRO9MSDHQmEgEgpUlTmrj1HAHv3H54WJasEpWhAKBdw2JA9QQB2H8cF+GIkubjbaS2b/AEhwJ9CQi/f8ucEoYeUt1cxQWoi7awzYti/t2I5sL4Jbm2RCA6zAkONrXNqDYC7hrcKW1EG/Ddx37848ZTxQjSzJbtcIV5Fqc/JJIA/X/wAcHM+IZK17q32SflrUgNKcR2J0g8fx5xyLAjy1aHo4Slbi9QstAAI47pIIv6YZMhSJyJLS3pIKCgeUKZRwfXte17d8eQ5KU63qSXEFHl0LJABPawsFk2FlDC9EVlRbQGSl8nSDpIIP9gkeo9hgjws8y1qmxqY4hDYGptx5L1r2H1jQjt3At273xMrQoi3GWWkpD7qnHu+hkkWHsbk888/rgxcVpzSytKHZLaNbTjKQT3Fu/Y3Frehwc8lBdWraQ+4tayS4gHUL97elwO17fxwUZC3Fus7IWfpJQtR4IFyeLWA7e3IOG7yi6WGChOtF1LIG0l3i45KyPc+/cEemBrLy3vCNMsua+UaHQ6EgC5N7825B9eMcblSfmB1qK0Y90h1VntKz5fKn37AW9fT1wX4hASpLTZcLCLFQG2mxPe3IvYWtf3wVF1tvbuUvJkhJs6GgFab/ALqTbnSO/bi3e2NAdE6W1FygKigAqqLpc1+qkp8gufXscZ6cjKkk6CtaVNgtJQEu3uQLDRfv6H7c41tl+m/hNHhU0nUqMw20o37qA5P8b4Zo5qbNS9TqdSkHuBfGSusdcGZestSDawtjLsVqno0+jttxf/zOgf5uNVVafEpFOl1ierTHhMOPvK/stoSVE/wGMMQKhKk06qZtqVxLqsqROcJ8xBcUVAfexUB+mHQCiz1DjZtzTKd1OlWoNBaHLeRvjEf6vR8v5fgjLL1RXGNUjuTq3UkOBBg0dv8A1S6OCQ66SGGgRcrcJ5tif0J6j5OylVcw1plpcNto+JddGolpHdASObqWbC3ckDvjGXxY5uzBEUKJWI/gatmEN1quRkG4iNDUIFOuL+RlGpw88uOXN7YNsxsFCcqpPrD1Qf6g5lXVfBs0qkQmBDpFKaADdPhNizbIAHfgFZ9T+WKxp7tnFqXe7huAR2GD3nF1CbtqWQ2Dzp9fywZsojLUrTyAP/oxee7os2bMVcXQ/NbVJqi4ytxRkoNy2rhtsck/nYcAepxdaoVPrsQs051solgaDIQQkoIHlsOQfS1v1GMj0aW6ypC45s5e1/8ATjQ3Tt6fUoKZc98IbYSdtQsQBb6L8d7Xtz74RM1S96mUekSvkLakLDSI7xS2gpuCQPN3C78cXuL4dI+h9mr1dUBt/adMBaFr8ziCeyVWvYItoNiOfsMR9uVBXEMqa+2q5AU6hZDDYWDYA24Gi5tYHv74Py8/LlTtC1uCMTtuNjhLDi7lP5fSB+QOImSx6mZcbnusoZaltSVL06/K8CRqUEqv9Y7ix98RHPkmoTKbNptKQ+4EBEppqO75nGiSCu/bWDxbjuDxyMSBcB+oTl0yRZTe2tZUBYB0DyrbPN+QLffj1xSed6x+FokJEtHjktrjSEpbIQ44O5Um/wBDgUCQOxF/fA3Rd3U6Q8yQ4tPC/EwHnZIcXoeGlMtGkkLA42nR5krtbkX7XxD5vUOJKZk0mnyXH4DpbdaYlIDhAX+0bSo3KObEXJHcccYryuV16UlLaX1lwlW6b8O/5Z/yu4vjuWMq5kzQt1FHpzkot+YgWvc8jvzzbvgpbOUzr+aPxigt5Y/EnCnw12txFg4ttRLZB7AkAt8dyE4NmTV1jpa5EmJTHlUCf456Mr5a3X1lLZBSe9mwok+lwLG+JHljo5JRVI7eY2XlQ4cBNV/ZEpcBUC4kn0AKVX72/I4iXUmGnK2YqnTmFtu02spU4ND25qfBsfP34J/I4m6jcysmfkLJyafTJf4GqRG2mi04oAFLjg1JQFDlYtY6Tcp559on/QBNZRMkQEMssSJTqY4ZQHFWQD5gv+drcg9+MT7J9MqUzKVFTNW34aFAbk6AA0pyWWi2glVjbb8vzPXkc+qCPOk0SuLjJeWs09txtCHTwsBQCibjn1sfQqBthLJymLoHX6v0Q+ITKs2c2hEijVyOHkqHkeZcO2eT+4W3Dz98fYN2HnbMTt1TwxT1J0llxrZCxrFlg8nsOxb54N8fKXqPDi5sgw65TpIp1Tpl3Yay4AlTVwdG53uHBqB7Am3bH1A6N9cKX1L6P5b6jT5CDUqlTx+JR2rEic0S0+LmwR50lXP9rHiOMH1FK+Oqo4A+Q6XIuR6LvYPkmY5kjyANbbKX0vJ0CH82a47KUCQlCXC00D29LLJ+5I/LDuupUajMaHTFpzS1W022g6om3lFruG/54h8/qI0iKXInhIDZ0FKlLDqiSv3FgiwFySk/w5w0uVbx76ZCZTC/E7bRSt0JeTZSrqUPyNgL+y/PwMebp+HcUxp3aYrMWjpfX5bD6rfLXU9KP5DLnqpPVM/sMqVEptNBkICxomoUgDjyK0pudJJ9SPyxDapVH8wKL8/xBKB4lKVFaY4uSEoTdWhAtfWQD3H5ALrbMtLqQVqQ6QG0aVrJbDnAIJt9eoiw9+TgDKWor5lyEpU+WgFqQtKtBN7XK7i4vwCR2P549jhvD1BhesEevU6n5/ouNUYjLU6OOiSuVFmIhenLdVcdit6tlEYAAFRsW9x3Wtf19rAAc2JGHhcyGxJSymmuMojaAmWQ21dDZAFyLWsOT3tcgemErYiRgzJS48lZKAVlaXVAn1AuO4uOD6DnBZdlyFyGZkB91zaCG3SggtrXbUHFdgOLG1xzz3GO21pHNYXO6Ibc0l9aWrNKR9X0q8i+QAV/QOARxc9+MImpMyct58IkRC2Ea2nFAtqJ5WhSNXogAnkfUDdYuAvd8MmQ0uJI2/Em6AY+4SC3yTcAAk6rC3Nib2GEbsZSkiNHjFe4O6IzaVEW1FAKzosSDzf94WvcXsc5palXX6upxLPhNCUAo0I+khABHB7EWPc2Hbvgcd5l5nVGbbJcQUi43AO97KHC9P8APyX9SfTjHkQ3H2HpAabCFBptAbtxYpBKbuIHluNYA49xjjjFmd+e9JQjkJcU8gsukDUBqJK0AabC4A7Hm+Ebui9FCWyUISpovRRIQ6EBYLFxdIUQLBZ4A4I9hzgSX9tDkSI8NTQQBrZbWkA/4whA4v7W+x9cDWzocSrcaZbWSS0mO2NLevi+39B5I5FifU2OCkIdXIW2xIRUFIaAW38tpDY12B1AFAPlUTzcXufbDuH3VMyCmSs7IXEs66C4rcsQUA8q9/YD/wCnADUWWWkx9pxLZb1bjIuWQeSNNgAeTxzbk9+wzH1Mq1utFDryFBCmxdwg/qCtIsBz7YEht6JILK5mpCCsNodBdKUf2QL25HOgnjnthWtzIITFQjLmuakbqFluOC02W1Jub2FhcBQ4Nzb6+5uMefnARSipwVqbLzd31L8Ol49vKeOBykm/oDgbbcWdKK2Wn2g+T5y9spSi45sg8eQ/Ve978c3BbsaMqzLKdxiPoaV30OyOwvf6DbSfNfgcJ9cDL95RBM1LKY8wyS5uOBtuQde2Eeo03IsT5Txr+/bBbLyUpaO86gx0OAO7Ki2F88NpuL3v25PmKyMeYaQpe8y4+tbbZKy602Bo5T9RvYhZ44PoRjstMyKhDrcyM8phe28o6dslZV/jNSQbc34v5bW5xaFEWNx1xptDUSS3uhTrby1M7J5O2AANYsbG55sOcAj1DdCNbzS0BFl6yu4AuFJsAft9z3J5wcpMpqMnfWuKiOjSLFv5pIUCblR4HccWANuOMNzdeprzLilykEMOLaUU+YgE6ijyd3COBqP8MI/xaIWR0VmNK2lokwSklaNK2QQkdr3Hqbafbk9sKnXmZIca8VHDbDZIZSb/AC73NhcWuv1J4WfXnBL60qU3IjneU4ojjte9wAeDb37/AKmwwJiXDRYRxIfbQUOMkLLirG4N797n3IvccC+LUUFL6EhJK9xteh5ZihshKDYKCdfAWPyAub+pwtAfbKfFSRIQ+NGuMgElAPBULfRfvf7W9cJpcSEl1DElSHNq0psNAmRtgjUb/wAeL3PPHHHI0BbetbsvUpQtrFyQQo6QAABrF+Lg8c98C6iHKExjwpXD8Spt0eLQXA2WwEGxCtXYLAGkBX1Hj1xzekTI269DEhsB11a0BbYSAfpI1E2sbep4FuOMIUQGG23Vw0NqICCCSSm/PdQ5HKrepub2uQMDejyXSnceKmUOIbcebWWVAi4J1G/kvftfkYKRKnkzEPrXv7ZdJ5ZspxshJstIAIWgcXuew4vxg9SZBkGHoGvWsOJa1Ep5HOok29ODY9ucIWoqg8UPSdRaBBQXAE9h78k3A4Hq4ePXHCw4pDzulopDdgUs2S0uxud1B4A8w5tex/UXRCUIhPOSSuREWouKQ0C1J39KDfUVJNggXtxz35PHI0RlB3cS8+taWgHFllagfKLIIsAe/B+wt6WIaSsNqSzFajxYwLZXJ5aN03NyBcau/KLC/f1wJmC884iHGcdkPMKQ64pxlvcbbIsnjggegNhqAsCcS6hXk06TIbSIC0JJsUBxrXfvz9QJ455PFub9ieuOI0dDz7ocWgNnxLiwUgEG93COAebe4B/PCXwkyGn+sPFOghta2idxJNj2Re33A/lfHnWtlCGd1atOsEJKCkXHzEWsb354+17+5QXWpKi6Gk7gUtRWlCjYW08+UGxJPr6Wv2wrcQ8064zstNpKyQU82+/sBbkW/wBOEy3HidxxehOkFKHdxdrX86QEkI7G3t2NsAfQA1pe8Q066guMqC+VosLLBt57cX5B+4vzFEuMlTqn4rshDehzaK2yRdABJVwfljve47g/fHINPYiNtst1V5tjZ0XLzhUkEkkFSyrWCVE8c9ubADCeHFfWHGZkNgxyoWUpndSk6DzzyuxuTbsLXHGPIC4roeeSULXthDjevdBvySn1QObfa1+MRRHNsLKhskPG4S4sLspN7XNj9BHFwbdsBUt47sM05AT8uy1G+4TdJ4IAAuQe5vc3GC3mX1TlCTL8YvcLgBJDbaLgAc+ckX5HNr+w4VJfTKWveeY1hwh0hsgO2GoEW4I+/oe9sRREOJS220llmGLKLgAbbKrW8ywRco/Id79vTHjBK2rOoDbZFynQLc3+oegPHGOOPOISh0S0sKYIcA7qdd5FgeAe/oTjsmQXZTUaOiSVrGp5tSGyHO10AhRseTyQCPbEUR7TKVgrbabWptPC2k20rvbv6jv9uB74LktOqKJKNalobNkrd/aAnlVvoJB9LH34whMxEphbKEhhX0/1iMhW07fypKSDe44F7XJB9MKnShx1t12mlIQsAR0rsq6LHhQN/wCFhYn3xFNUizFSDWst1emvutIVNiPttkOXAXt3bXYHzkOBP29sUOjJPT/qBQ4FSzVkei1Z5UYMh2ZESqQkf2N0Wc4uLDXYemNA6qlDejyVVBCUlesKciBLXbUPMCPUevA/PFPUthFMq1coiPoiT3HWhcn5Tp3E2vzayhi6HVyDtlAJfwr9Ep8lMmPl2pUxWoKcRT6y+0h4C/kWFlVgf8kg/fF9x+l2Sl0mNAptOcjIjsNtsrVIeW8WwBxuuKUskD+0TwMRlSjsu2vfbWLD3sbYvDOGXP6O1tFNecCUGnwlNNbJum7CAopIHPnQ4T7cYtqNAkivfRURWumc+jMyJsCWJcLbWhx5tNnWkAg+YenNiL+1+cXp8M+cxAzjAbcf+RmCMae9zx4hu7jK/wBfmp/zhhmCW3mdYj6nrFK2EuIDgBva4KtC7i3BJP39MQbL4k5crMyBSXtDsSQ3UKctJF0rQdSbfkR2/PGVq1b6FfQ9P0jDLmzL7OaMt1KgSOET4y2Qod0m3lV+YVY/pgWUcwQ81ZZpmY4J+TUorcpKb306xcp/Q3H6YeFcg4sVKxolhEaQrx4WZF1tltFwpJ+m5PIHN+O/HpgLS7nS14hDiLgKUCrSL+3Ydrk/c8Yn/VXLpomcp7rSWTDqlqg22RYFZ4cJNwANQuT/AJYxXwaithLyFnUi5u3cJbX+fHpe3viKIT2h5lxpS1W40lAABN/W/pYemONokMJGgMKX6gO6iOex9h25/ngBD26rWI51q0OWbLhSeTb7Gxta3OFDc/ZeBZ1tkIO3cBKrdz2FjY+/BF8C6NkFMxbDd3VIShZ+YEmylC/A9h39DfjHUStpKvkEtH0DZ+j+f9+OIdZWpS1pLS0C/wAxIQSD6gj078nBzaEOIcX80Ni6RoJJUBYk29RyMS6CLcc1qblw39p4KFrcgj8lixHFre5wW6yw2tanmllRCFEpRt34uPTn/wAMHIYZkKSFlCEWvpKikH7psCb8/wDfxjkiAw4pTzKUFxDRBSkFdlm4seBwePfj78Yl1EWgRkNIMU6H2/lq0taVOIJuTYeQfkAD63wGRIWglCktuICtvcdZQbtk3A7AgW9+cHLp7KGm/EFAd20KUlxYLYXcAkdlkDz2t7Y9OjrhFL8Z5pyHvBkqeJ1EnjWLA8cnv+WIEeSYRMD6gypS2VrAOlTJIVY3ueLH7Wt2/XCqOl5Ti3dxp2WsAPFXJTa9kAEnjzE247k4UtsKYAWyyyWdAc0OH5Sjf1Fjzybcj1x5iLLjIT+HOcNjSGmnghSke1jysdgPa2MaszIDkeptLMh+SiO00tAulwKSe/PH0Di5JsbWwewuJdz5ciaLa94620geh5HN/YkWse+EweSpRfmJXdbS3FbgU7za3I7elr/ywINKnMLLkhxSV8Bp5nbItyFpBGsn7dvtgluVKjZb0lxIMRtbawLEBDag6B6JB4P9/GCqc7MYcKNttq6BqAaspwr5vpAuByT9v1wjFORHDS5Dkh1eqzQCSb+y+E/X2F7ADth7jstFgoQtxx4i7aXJJCifYKsSeO9u388DdHwoDymyppKEuv8AOqwaI4HIF+47n15tjzb0+WXW17TTIvrbCALG31FXcADm2OOojMkojRNyU3YhKXSUtgi6lAnsR/3++FykSG2ksLZhlwIHzFKKg25zcXta/bEQSWJSUPMNy4dOLu0k7hbeDQUe9yLk2t64PqVIeRIbW43HF7L86wXNHB4vcjk/l6+uDKg7MbfvIe1NG6VAxiEq8nYWN0cnlZ+3vguWiClhS3mlpL4GtSSFXv3IRYe3Yn0GGzFNlQywWCHPHU9Dy9ZShxWlxSL8KACSR29OLnBaJE+U+lSI6JD4auSb2Isf3eCefte+CVytbaWY7UmSwizaEpWbq54FyRzyOAeDxgzwklxaHUU59AWFrS2pYLiUDggj0X39+LXOF7zkhQpM+oyHNL0hpsNtkqSCVlwkWISm/kvbm5wjltRA0ZEYFtW2AEIBUNfHI73HI/uwtfYXcuIjrUI4b1h1oukLQOOOSQV/wuO9sBiQWUMqdkOPLS4ncK22z5nLXF2zyOeDa3GLWuzbori2pim1NGM42XEhwuqa0rVbk2HbsL9rCwwnmKk09lL6GnIjZtcAkhXrovYkEkD7H8zhcqLDDrrQajSFNtLWt5UcEBvuCAE34PF7A++CU1Na3i2w7qW5Zpa0trvcE2sCeEAX72/TDqJM5ElFan5MQoafQXEBQ8wvyCEDkE+n6YcE02SWi3s7SByLNbADhuODx209xwLm4GCVVQsrER2M+h5Ys0u/mSu/pY37W7c3wUFtKeSvW6l7kuFbh1PI1Am3JI4vY/Yg4BUUp6aUVmq5wpUVtLbrLL/i3d0J3NKEahwOPrA59iOcaeHbFKdAacmTMrOYlRYyP2cRLrAIDh5J4sOw0C5F/wAsXVewGLGaBKVTHxP5mmUXIDlGjISBX30QSvVZYb+pwJHrwm3t58ZvzUDAoVPpqEErdWgFI9beY/zti5fifm/imeMo5ZHKIbD9Rdt/lqDab/8Aw1Yzf1Okzc65/p/SuiTHGFvxkvVqc0bKp9PcuVBJHaQ6Btt+oBWv0GCNdE2XupZRGEZ6mCsaGV5Tye4XIzrhIaqVRab+dKUTwY0bkI9FOK19k3x84+vnUD/0gZ/r+bEayzUJykRFLFiYzYDbXHp5Bf1798bW+MfqjTumvS6ndJcnoagy6y242WmE6RDp4ASU973UfKD7BePn5KfQ4VlbaFKQRZazqAHqAPf1v9sPC293lVSHkm+LGQxFRpZIc+tSir6h6XwQogq81vMbnnCvxqVEoshXAQDaw/PCGY4kLUgItx+d8OqURHnLgzBIV5tHa3JOJ9T+pElaChLaFLCCLXIAH39P9GK4trcSvUCkYWQeHNXCUX7YiivbJ+cpdXqTMaWWnD9baVglIWBYEpsRwAObc4vUOtIozK5jqFSpLwVydGlsIt+5ytd1eovz3OMt9MZ+jM0BD0nabccDbgDgSXBftqPYfbFx9Vc8SqHVWaVSvImN+5a235OyVAk/5yueTwMRWMUyQ/DnrnTn/wCqtoQ2G1cp2wLXNvQhwAk+/wCeKMzfkWsZtnyHmnUXBIU6oWLiz5ieBa97/nfDnlnP87Z2lvWixgXVktgh30CAnjyX4Bvb1OIvnjqZUuKbFmaUtFa9bfBUdFv7yee9gMRN6ptqPRmbkqEzmvNUNp6G+65FZhqNy+UAbuq3LYGopC+bLRe57YlOXerUfLr8H8GpMBiLHiNwvO0lxW2hStJKvXgi59ftbB+QYys6ZVepeZJikIlrLUDcUtwb67XuO6D2IVexN/vitcxZcqOTq2/SpkNxIAvoIFwD6iwF7/YWv+WEKZbCypVstZky6xWHohQh0uNvFlNy2H0AKAFvrPNrDgpPAviouoa3cndVo2XpkaHMgVOFH2y6whSSw7q7kjkglQJHPAPqMe6fV2psdLMw/hqylyOuO9cL1WLVlGyb2OpGq44vYWPGCOq0pmtZRoNVQu6oDyHW5C0AFxD4Nkg/XYLbACTxYYLVE5dXpCsuIeRDnIDLDkZqzdtzaF9hBUO3uQPtio3MzeOX4oFalOgklZvYEA+nPe/54sXPkGXmilQ67HksbL8TxKEI2z8/XbQQjhCLK7e5xVFXyrW6QJK5EZ1sNguIQ4gpKhYG1u49e/e1sG5UTgzm6UqOmnOSUISXLjQfYEW/hxjeHwKzI9b6LTYz1QlsGPmWUmzF1A7rbSgLIWLWI5uOb/fHzT8SpSwlPKV/5P8AD+OPoz/g/wCNJa6WVl0sLcRIrTbiFhBIuhntcck88duCeeOeTjeFw4vTe71F7XB0NtvRdHCcSlwyV00Vr+YB/FagdZjLQI7tarDCNaQUsAtPOCwNk6Hr82t3tb3wKqNU12IuCcsx4chSml65j7TjoOj0DBJ5AULlauxv7guO0wsIZXTJtwW1tIZjqcBcNza9ws3PYEfunn0x0wZUYLhw6YttD7bY5SWh7iwtfgAcXtxjztBwdQYdMJmF5I6uJHyXRrOIquuj7OW3yA/BN0Ojz6ehlcyfIIfBQG1IAj8+ifMpaxYk/uckegOF8cvFlIddb8QhZcbYbGp1rm2gOaiL9z3PNxawGDPDz2FltNKL6XzcpYZeSkov21exJV5j2PF+BgJYemxUtvQ5BdLmgoVBPIAJuQvudenkAgkKtbm/rbBefKNYgB2cy83aQUAuoCUBtCvQXN7rtf2T2I4AwFc1lpJW9KCNa9bgW4eRzZItaw7EWHc9+9iXmHJ76GmmZi1O3LaVMqIc19kpOlIctZPFwAbdwMEsRnEy0tiC6tOnQnTLdVuND5l1J5BJPJJOviwFjhHDMijGnnnHZC6YyEoUA7reJSoghO6Uhz69NwO/Fhfm4wczNW3Iap7iAEuWbCJO241Yi+tZRz5fKLcC44uOMekRH2y6EMoQS5cKXHJSkXO445wPOCR7nke2FexUo8lxc952zi1jRHgnUni7dgRwUgG90EWHFgMJYqJvVIjh5b81CHXG2vmtSTrc7hQ845B7cX5A8/pjqafTWnZTmhhAcWVjQ35W0Wvdw3PAJOgEkcgW746t+VIaUltb7LhKEEB0lKUHu64rSLXGoAa+9yb8YSlDriT4aTJcK1LQ605GGkLuQFgrSVkWVxf39L3w9hlTOQ5wjNuOOoWuQ4tpZ8qwVPLPZsm10IUR3I8vsQMFxzKcQmMZbSnAmxcRIUAm97ISVgXsgj5nFyLgc4UlmbqQ8hHmbG7oaas6w3b6io9gbC1+PXgm+Ax0y5zTPg1rUy6RGT/Vg7ruSFApumyLEAar8X45wtz4UMqNYkx4j7vjnCvUObkXUsLIJSTY8WHIvcke9sN8VKEtKYEyzQkIkSNRbTqF7FKeNHoLEkab3uLjC6RultaSxJUlpsuNJSSptgkjjkXATwm9u6uOBy31Biqlh2XDlzIC9q7QbabmI+v2KVblwQmx8gsAQCTieHko1tzZEuPMSUSeYbinAQyw6tR1EaQlBP0LI8oBBtYe9rq1RtW4zEholPI0MrKX9st6zysgC91WCSb82HpxgMiZmRxEdMOnU6N4aMhRceYnOkgaTcWUnQsrt5bWseBa5wCC9WI8wIqUVtLse61LpZcDLfnuHBulTg1XtYahe1rAcwPzDZO5gZzugoXKqbdTRSnXwqnPWdWw2G1surLhShRLSUEBGkBxJIVp7rsohSiXOQ24t5h555bayGHW0JLhve40EeQcFBt3BNrAYJrFGVXXY6axQHH5LDhcjLkR3HhDct3KtItcC5UAAdIHI4x5xl5TSHlRHtIC2ko2nNtIIFvlkc209wBa/wDGNLnbpXhoOiQypTz7MVS5cRoeQ2ZW5ZlBIvbWkgL4N7+o7XNsFJUshSXpwQYjQQ2tThUVNtkggqBH29ByD3uLGri1WTNYU941LAOh1TDKi7uC4CRye5JJ7mxwqT4yEhuS5TZ6gyClpoRioJHoSrTZwXF7kjj7pJNtwq9UU0IUZ9MxM2R8sbaC2sh1nQLm6nBYE+bm9/W98GiXIahMxkLKWyGwGhoJ8ihpLt/X6Tckaebn1wm/CXmVoeZpsiEy0C0hlmzSktkgt6Qbhy5sSPr9fzUNUSSmc+lqmS3Hm3W1OocC3S4QVAtkLSbAAkkA2SQbAHnBQRuxTkbLLjyAg3DL7x2w47e+3z3HuL83A9cFByXFiKWlsOKcLh1SXCXCNuyU6uSgCyQGwAB25x40yeuQpcalTFv7e4mU80Wk3QBpHIHe359sdch1AIkKVEfU7CWAtLlgTdFhqTyTyT3KF37d8HKpZcLtSeYZW8h2QppDTqg2EADWuwJ7BFlmxNubXv7+AvPWh5wKdCOChs3B7XuboItcce1+LY4/FqcduPEeTJadfDjrbKG5De/oFnDYgkWFu1+bC/mNlrzUhbSXXIi0NNvXKVxgC8hYJN1GyAedQHJBBQfQ4G6myRLXJafjiMp9bbjl91O4QAAdKA52CuPUDufcYKbYjNhKGXUKW2dF3pFys8E+utHcAg+wt3vgZpcxhh+QilTBUJYIe/qJURY8EuAHiwUfuBYHEZ6kuZyytlCfmnp9luJU34gEx6M6y44h9v8Axi2zrb0HhR4J9eDYYIbm0CObK1SxCEuzXNzZlvLdBbUyyhtxXnsPrJ5I9+RfBhU7uF0xpcoGU02A+S58u/8AbBC+4INj6nsMZva+NXJZaU9/QHNLrxLa0lUinjbI5uCV9uVWbI4uOcGxPjGyJIjzWKnS84Qi8gIb0Uylz3FWTpJQ6T8rgk/mL3sbYZ1NLbZViVl9StFpLqEbbjpV5XC24CsNNoHsVi/l81re9ueMFy3moZWpiWCFlvyuyNIS2Rf5l7i/rawN/a3Odqb8anT9c1mJO6d1ulUtLmgS2ZjL7qGCBy4yLaALqPyys+nNhi7Mt9SMkZkXGk5Uz1RZidPmaiS4xUk3sD8xW5rI4sRdN79sDs5GDvBMHsdsVILeISUKkvxWjYOKSjcUSe4T9Iv7ci+rHKk/Apz62WXSuQ4SotLUdVgbFtsIbvoB2+/I+4IwLwE1lt1CETEtBomxQSUgcXHHoSLEccHjnHkQJiXHy9DW4l0L1odbLbSSAPPpQEjlHbki5OBZFEqAVNeeRVXWlvqAWgOLDCQLBtRBA278enfggjHX3t4pVIW0+FjWh0N2c55vYX9LC9xe2ONhqzqJiHNTbi3W/lHcdBNiQViw03HmN783sLYS+EqEgpapNeQw8FkL0U9ucw6SBYOtuEaBfmyVjsefZX91FveKPQdLB1rcdbWdtDZWdpVle/NiB7c4Dp2Eo3nH0jeQS4AVJbRblKddrDt5uT/HDpJQ0lxSUNaHtoGW22CQUEkX4UbcgkA3HFrnk4IRSdptpCFyH3WyXFnQUg8k7jf1awLA2+9x9NsHdQJKppbIWgrmJUjz/LkOX1+twf3OxAv7H3GC48ZRacZqDzS46HVtBLDy3i0SfMhzixPPob+YdrYWMszm3FoajlIBDe2ChNgeSVccknuSSL9zgtS3mJQhoXIZLug6FKLxbQPKLhDdm7c+vOr2GDugvOIjvKQNb8pQCNtt1fzQ2i59FAEgW5tzYWOCHKfDdDSEMuKYXYNDVct2AI1Hm38vW+D00tLk9TC3mm1tuIcKnUB0m5Jso312F+44/PnBqGJDbW/EcEtJWu6Ntw/RbUG/QDn9SO/riKZlwssoh7y4rqGkOaGwp5ze5T9BSgfXcXJ0Aj8sVb1CYRS+pFPnC2it0nbcAJtux3Cngkm/kUyb+uLTJW2yvfaDI8P5Fu7hSEAWskXsLrv2PFyTxfFfdZGVN5co9fRGcjro1WbO2Wh5WJALLgUb9rpZNrWueL9zbH3So7UI7JdKVXs3UOiJTcT6nGZXx+4XAVf/ACA40n19gKXPomYGJ5iIcQ9CW4HNOkoWFt2udA4KuTc8cDFOfDrSvxHqpT5BTdFLjSp9z6EJ22z/ABdxovqxGD+QZUnQ2pVOfZlgOKISBfaUSRzwHL29bYepd3gEse11ndyO+1KTsMQ50d0DbbeWUqWs/Vc3NzzawBR9HOItm1JpcylZhS678pQiv3FtKDwkEWFrG/bE1RGlxpSWXXWpjjq3W3A4txstOWv9CwT2t3IAJAAOG+v0mRXqXNpaltuh9m7WhIaLLiACm6fU8dwfQn3xSCmCvn4acw+IoFTyk8bKpMrxMYE//o0i6wB+TgdH5EYuo9vfGMvhzzeunZuoE2QrQioBdFmX/tr/AGV/vupA/wCUxs0G4Bw6Z26rPrlRjMyyzWUEXp0hG8CnUFMLUAoFNjrsrbNuOx5xRCYbsh1Ed5TTTjaLPKOklQ1my+bA8W/hjWtYpkas0uVSZQuzMZWy4NIPlUCOx/PGVlUn8NXKp0/dfdYU5GfBds2lbB7XRYa7kcElZv64BSpimGpCO0xDpT82S+LNoLoYsi3HmcNjwDcH3Hawu2qzFmRp5SZGTiypq9tVTbbaDhA7FCVDuRxcW498S1mA8ULUyt9wEFxUdtCXHWgLWWotq7A9zb1/KyJqPJjo2mtxoSADraaL4db0C6hwbdwDc2PFu2AE6TRpEvY1pbQ3JQdpbLakvkE+mogC5Hbmw578YUOF1m6ZCh9YBGzYqA7m6FWJuO1sBlQ0qAVOhx3FtpLQKWkB1kG6h6XF1GxFx2PsLDirfW62yhbAsQShpaHCleklPBJF72K+59Cb4ZKEW3KW0wCt1gBxYb2lrOoixNyAOwJ9MHqltx3470tTjgcIus6m0qsm2gACwANzx24vgM2JsENLZCwSDHbkmy3gTe9z35uPLzyLE4EAhCG/Ehbi3yU6XVBelzso24sNfGnk9rk8YmiCB4iyQ4mQXd3gsveZwC5tz6WPbAGpjkdDTyWE/P1tuKbQjUPcG/HI9bfzw4SqTMRJEYQ9rabACdADj6Ai/wBNwAe/1ehPPGEcSGh0OWaXd1sOeRspbVxfhIBC/IL8WI+4GBoomiK9LXaNLkNPPOLJX4Z3ca73AJJ16/QmwFweOOXNDDpWP6mtbziTtWSb3vbsvj17g378Y6mO2lTrzq47L6A244Ut61KQBfyutkC1j2I+wucK0KjqcVNSIkm7i21KcZL6rk2KBwCDxb0tYd+cYbp0T4RDbLnjzshDQfB3h8sINjcLsLAnubfzwNqPBZU5JYSNxs3utYUUt3sfva472P54TNLYkP8AgFAsBZcA33lm5FjtjuCeD39z3w5bqIqFyIDbrDAdtq7BxFrqWABxxwNPPHb1xY3dRNrF5Ed5hp51LD6nG1tMxbtkIUbjz8nsP0wJhhyMyp56TIZZEdxpC2AElsXA4N+b3t5uPzvhe4qS+sq1NamrhtCQoL7gDkenPr74C1IehyQ0l5iK4fkhwIBSACLEHi1uO/f1HphXORyJO+Ji0lbSCw8sIUFKYbCw2R3PfghJI5sf0tjwdL7iYyCtzzWOtraVckEEACwuR3He3OFTjgqDjr7Ut1ew6S0lxtQWm6EjgEHg6VHvzxa2BCPvoLkuTpdI+WpROpw2+gq1DRcAi/J+2IgidMYuOSkIekouUrQHLjudVlHki9hzbvgBj65Jhs0xDDReB/aJuAUfQCbXsb8e59L4GI63wl1xpyRf5IVdotFYAusgJ49bG5v397dTHgy0sNtstOqWNd+dtJB02IF7rsoe179vXETZkWy3GG1IU0iONegNyHEKUo24XYWCBbjubG/c8YVNrbECTockr3wNpxlpw7jY50JbABB7nUe33GCXkwIzaoEvxDHms2lTO005b29bf+QDfhvq64zTrDbcgOhAQonxTmkLufKoA3sbXJHAFxwTgEhu6Ibn8KcXy0HlOpaWrbARpW4VEm3e+nyXJPHP29belS3lPvNVCA404tsXbjyAoOAJtdQKBoJ9PumwJw2t1h1DjSKm9T0Nt2YUqzqLoFiUC5BWjQf4AcgYWmU1DjKlz5LaY+3Zz5x0kAk6+Rc9xax9fthmu6IObZIXm3GyneckAoS4h0qSdSrpuCpQPKLWv37+mBOqmISuQ+hxpAWN1xRWQLaRwBa5BsOfS2DV5gZZaUGUNqddbbKE7ZHkPNgVm69QFu45Pra2BxFba0zFhgtx2XHBr1L27EkpNxb6O3N+wwe1ciW3QULajnQnQHGzuyFrsCkC+k6hb19DY88Xwml1mDJKHFUdI8q1tpdaKTrKrC1jftY88f6VzixG3YbL0l4xlDbQEBSkrWEnUQLdwUkAcDkYVw434vUxAhuvKdnPoihRaRqUCdNr2/sHUO97c2w4fdBrbq/Oj9JVS8iwFux9h2cVzFt25Tr+kG3roCb/AHxOVXtxhPCiMwYrMRhIS3HbS02kCwAAsMMuec4UvImValmqrO6WILKnEoAut523kabT+8tarJAHcnGhVnvFZN6wZ7QOpmY63Ei/iU9UpvLtCghWnxUhtFlC/wC62k7rjjnYNgn2xBelNJZyyqrZqrdVEuWhTtTqlUc+UZD99SnATwhCUN2bTxpQkYV5boFRgUOpdQMwhD2ZarHdbab3NTdPYcXfw7Z/tqV8x1QHmNh2SMQ34jKrByF8KNenrkrZqeaZqKFTEBzlbZI31AWubNIWO4Hm9cZaiqjgtfmQPiVe2F0m3IXWEetXVFzqt1FreeJJkGPPkr8GlZ5bho4ZFuw45PHdRxVjr61MlSdRTfhKfW/2worTyw3tBRFzawOE7LS9vUs9hxbG4d0WWFzsyTRIz8mSdtJQlu2u6vbsPzwKprOoBKLW5N8HIQ9HUF8oQQbWPf3OCpaPKldweLkfbBzJUljKsoptwochI9cOVPZWt5ISgAdybcnDU3oAVpJCR66rYkOXpi47mosx3eLfNGpP8O2DomDbqc5Tgz2qjGdpLMOTJJ4bcdCQq/uR9H+nC/qjWfASAxNfjPVLb1vJbcC246D2Go83t78j+eHLLjXjHGX0ltKhdxfkCWreo45P5cYgfVCmyPxIuwZV2GDqLy2dLusgX1eg9gOSAB74Ct8PhTS3V4zbSnlSV2tdYb1A/lc+npc974jC56J84jdDYWfqVzb9cNEguBWhSyf864OCBf3xXzQy3Wnunk+NOy+3SIZUS28X0bADim+LXI7/AJ24w55njU7M2WlRam+ioPUc7MWosRy5st+qHdFyEHsQQQODcEc5cptYqlOfbegTn2HWzdK23ClSfyPpi66BU80VlUPNNOzC0Kg3ojPrTHcbde991TabLtcedVzyMHdMlnTWe5Rs5xcv1CepmkVobMwoSy8JDAuQUpc4PIshXryL3w49W6vQKVlKFTMm1mTNo9UmOCIZERbL0VDC7qjG4Fykq4PBIt3tiwY2W1oZhS6g1Q4b7S35bsWoyG0h9xA3HCG3Wx8zygjTYk3HJHOWs5dQqrmmLT6GpSkUmjFwQI6vqFz+0cV3Wsi1z9sTZDNmVh0rro9lrL4pTUFhT0dYLKnfmIYt2CUE9xe/Nx9sQ2X1gzJKqxqrkkyXlOKcWXwFIdve6S32IN+3/fivFKUSb8XwJpN1Aab3I4wNyirNh12g1lpxAy3SoACFAbUWxC+LEEqujtyfX2GLLp1fr2WMtUmkUbMNZpyXAt91EGa8wXCsgJJDakgiwPNjxxiqumsWPJqrTL0jQVIUG0rvYr7WFgeefy73OLBq8lEypKdjM6G2kiM0Qgj5aOARf8ica6ZuaS6zzuyiyd389ZxlnVKzvmeQnUHNK67MOm3Yj5v6fxwoZ6h5/ZTpaz5mxtC3C6U/0gnWUT3Nt7EXbbd3PmKVp7WBx1RUg3BKPXvfGvKOizZnfZKlA6kZ7SHS7n7NKvEeR7/64J4DqPQGz3PvY4Ld6gZ5kpDUjPuaHmuVBDtenFI/Ib2I6XS55bOEfX5v77W4x4hesaWbDsVW4/ng5B0VeeTqpI31BzulC1Iz5mULcCLqFdmXIR9P+N9MDR1Ez8tsoR1EzclFrn/1/Nt7cDdtb0xFVOt63EtvIKm12cSkgEH+632weFnlaV9/TSLYbKOiW8nVSE57z4k63s7Zlu2NGpytTCpII7WLvHp29hjqeoefk6FI6gZqsi2kfj8wpRxYWBd444xGtwnUtdytZspRvc/cnAwSsKQlGlI8oA9f1xMreidsjuql7XVnqo0nbHVLOjTfAKE5kli4Ha1ne3pxb0wlk9TuorrYQ5n/ADg4lFxxmOUb3ABP7YWPA5HpiMkFCDpHlPoRz/446pRUlCFN3C++nt+mFyDol7R/3k8x84ZqC/JmivoWsEHTVZQNieeS4TyALj7euHEdSuqTS7Kz9ndWtGk7dfmW0WtpsHb2tc2+3OIsFJU0ocX1g7ah5Sge5Hoe3+nGlWs5fAy5Fhib0lSZIbQXkM0CWEFfG55jIOs35BA5t+mK5g1lu7dXRlz/ALaoiTnTO8gCTJznmdezcNuu1eWVpuT2JXx37D35wONnrPrISWM+ZqQsANtvNV2clSQP3UkO9u3A+2L3i5l+BJbK97p5UEu3KA2aNOSVAr/aFxuWUA/kP3h3ODG8yfAczov01mKZWCdmRS57r3Y/WTLt3sDzcXFuE4oMo/7Z+QVgid99UA/nLOcoLMnNWY3wo8pcqspxKubXILvuBxax/TCVnNOaoTKY0av11ltHkDbdSlJAHrayxo/Ti2L1Fe+CJS25SOmWYXlgBLkdUGTtelzZcwtk2uRx5cGRcx/A+662JnSqawtbnI/DZSiOLC9pFh6EW5/jhxJp4PoEMjvvKjHM451S2QnNuZlMIF+azNtoHqAHTYc9scGc8/PNlP8ATPNC0OG5CqzOIJ+93b9rd+/bti+Hsz/BBCbdRG6YuTHijhaKFPKSs3BKwZjZCxweAQbfxTjPXwUSJYaX0ZXFjN8F1NDdcU8Ld9AmAi6+dJ5/hgdqfufRPl/zKi3s053c3EPZuzS4h/XuJcrMw7muxVf5vrb1wSxXMzMx1MtV+vpC2w25oq8sDQDex+bwL8+3fF8IzB8D3mWelzrYJJITl2Vfg2TpvMIF/W59+9sHOZt+CBpMeM10tWuwQ6XBl18IDh5sSZOsgW7EEEH72xO2P/bUyf5lQjWac5OfLTm/MFjYlIqU3T7C43ebgcHAlV7MpWpC8xV0tn0FWl8H7fN/j+Q9sX6vqH8FIUdvpStaXEDh/K6yGCBpslO9dYUCTa9727c4VL6g/BC5FDq+kzBcW4u//wBazqFaLi19DthxwLHi17XvgulP3Si1n+ZZ6azPmwhbaMwZiQ27Yq/9cyylSAe5s72vf+/BTVdzM+gtKq9eKgdRR+KStKuAQojc7/c88Yv3+mPwSfiral9LpD6VrQEr/o7JShiw+t1PiE6wfZu5tzcXthXT8wfAhtJkPZDCJQ3A4ZVAmKTbnlIDx/QE/n7YQy8yw/IIGPN9tZ6bzLm1sGOzmLMCUfWAmryyPzI3efpH8McOYMyKAS5Xa4tCkBG3+KS7d+37W9ja/wBvTGhI+Z/gTKXC709dUSLBIyzLaSPJ9Wre9zbgXt3v3wWa/wDAiUJbl5BcQ4k3K28vVABwe37ck8W5vbnDtk6MS5NfEs/KzBmUpCf6Q5gCbcJTVpqQPyAd+2ATa1muayYrmaK+UOXWtJq8tSTc83Tu2/li/wCRXPgTjtPbOQ5b6QEbaU5bnocVf99KjIsPvqtx2wP+kPwKOuMsu5AqG24RuPIo1QZS2PQEb3J5A4v7k4PbW+x9E7or81mf+sISlKm1vKB5Pa597+//AHjBiS+QFFtaj/uPy/78aX8X8CTqy0KE4Y6P31U+t6jfULbZXa44uQQSORzYY6xP+A3aCH8suDWbguUysl2wvwTruTwkDvYfyPvH+Uql1P5rMq1rBOtGkrJIJ9e/I98FOrS6XnKzCiyW0toQ1pbWFR0IFgQEJVrPcm4JN+4ti+s5U74OalT1JyvmLNWXJy3BdymUCdIYPBtuMyVcm5AulaCQO4vilKzAodLnuRKHmcVyJe6ZP4VJgK++tp6+gj0stV+cWsd2iURuZzRVPzHMjJLNIrlTa2je0WfLZbQTwSE6hbgAEW7WB4w6ozvnMOqUnOuZBuXBKK7LF+3Bs9z29cMWolV1dk+9z/HDnlubR6fXoc6vUWPWKbHkIM2A5fTJYv5m79xxf1HNucOWttsjm804/wBMs+lQZVnHNhQi3l/Gpthb2Ad4H5/fBSs8Z1av/wDXvmhCknVc12Xe54J/a37EjGgvxX4D3nm3hlWOEOlRUy5RaklYuRYcAgHmxsTym6DyTiOZoyt8F1dhPu5T6j1nKVRsva106oyoynew3GHWlGxN7qbcbPPAxlbK2/hVjmEbFVRH6ndRI7CERuo+cmG2rNthuvTU6e44Ad4HOFLXVfqUzpUjqnnUKX57qr8uwI4HG7z/AOGIaN1tZFr+WxPNifW1+bfnzgKy+qymjtL9iNY/h6Y1Wb0VHayKdU7rB1dp5Qprqvmy6C2dCqw+6yCDe+26pQIPtyOcX/0z+Ma62qb1WgP6lny1imp/hvsHk2H7zR/NOMkRZS0q0uPWtftwOfsDxhyptap0F8OTso0OrACxEkzWF/o5GkNG9/Ug4qfAyTcJmyuve6+ldKrGW80xDm6iVWJPiSYqNmYw608wFmxIe/fQ9a4IJ1jjy84c5lWgQ2kyESVlbzTT6Fbzh1NkaSXNCSQg97AEixHA5xiTpL196d9NagifE6N1SGt9AE5VOznKkIkkW5LExOg+cEg67j0Ue2LrpXxldIJEqMue7muFYrFplLDyGyQBqL0d1V0EJBtb9OMc2Wmew6BbGSh3NXsWUOICIikTVSPK0C6Vhxrvdtzjz2uNIva59uWzqNl2RX8l1ukRm1hyTDklO42WymQ1ZxsC5NwHG2x39ftiP0rrJ0kzHIDdE6t5bafkn6fFtMuqJ5upp1tsuEWHYE8evbEwhaBKCqcEVDw7gWpRJdLzaOUkqbsAhRuebgXxT327hW3CcvgwbRWqbVc4MoK0uQIUNtXJ5WVOKH52Sj+ONJ1OkrrVHqNFfR5KhDfikarHztkD9b2xnr4KlsUegZ/yGmMuLMyxm51tbKhYpjSI7bsUcE2ARcDv+eNJNuKQpJ9UEEfpguOYotbZqy07NoL7TIqSYSUqZDZEuS1r3W/Xka3PYWHew9ccQyuO66tlp91aGStorSv5d7Ed+QO/A7C/qcLs70eDRc+ZhjoYQFIlPOsbze8NDre8OT9F9QCCORp4xG406SmWqS0t4rbAcWtUYJSQSLNBSHLn0vqHceQ+wcgoxDYdo+a6tQ4y9AmWqEFxI02c7gj2+YL/AKY3ZkjMzGcMp0nMzAAFQjJdWn+w52Wn9FhQ/TGHc7wfw1dMzOxGcZaiSg04Q+t9Km1gdnVkk8/lYn3ONIfDVmEOQatlJx0HwzwqUQar/Jf/AGgH2DoWf+UGD9pNuruVyMZ5610h6l5s8el1qLBqDPii68CWg6LIdB9BwEHi5ue2NEYrvrTRW52UzVktIU/R3PFJWpZRpbI0ucj0sbkdjbnDJVnelpcfRq3Vx9iUBriG7HNwbqBJtexKR6n2w6BW06EOuT3Fv7imFQ2XLpXt6VuFvVwABcIVa/NwdIw3OyN1wrl1FDri47gjqZd07jRFwPS6EhJv3455HYCxHDYZSvUkLDuptaPmtovbSoG6ym+q9rD73widODTbaYUlDKWkJDnztdtx2xCggWuTa9/tf0vhNOfRHlqbW1IIJ0GzI29H1EC4NjfggHgWtzgiFU6bvBlU8RJBIZQEuLQ2AjhPmRYOEkkkG4J89+DhWzIkqcaaRDbbSHg4Xh+1eKOLhSxdHoeQeLXPOIg1qRuogsrcZUphVyFrC+EpueB3A9jc9uB64M2X4UpK1R6elt83IDoc3SfM4kj6LnSR7+/IvhydiSWNcmUpthvxBZWdDVjdAUq/JtptYlzULgcWULIY0ydrZix2l61/MQhbW2tXzLFCgEiy7hJGog25GI5SyPRMClDwcLwxaN20MG9ifoJJIH0H07i3fHJAfce3VubRSRZxLZBdBvpt7A2PHtf1HIJ6EOKdaektOuhR0N6wDrAIKkjuVkHggm2n2vhOdCVBb0dEpreC3VtuNlDduGxpsNHqbdzYkE9jEqMZgxlq0pabAkaCyHXFhJc72T3sb3t+XoMeceu9HjxGYUdkW3CWghw82curb0c2vqT9/W+D2ojTzrKpC3Qi5bZXuFW0OLggAgWNxcX5PNsJ4Mp1lel14fMKHYza4hIeG52vfWs2P5exGMjlblQmZVVfmB6kQH2dtslLRXyA4U8gG3e49LEdhfBb8l5amnJT5fZIKEBbRIPoQBzaxCrkckj749Hp48kZhS1MF3wyEC+/r5PPJcJIPcX78W9VTMdEJ1YjNsAvguBxDrabBYtf5Z5HB/ne18FKkUJMxLaWI7KENg3dc+bZSxY2uQkHm3a/f7YdVuSxH+dNbihbqytTjQGhAvbhw2sqwFzbBD0lctt9NkHYAQTr07dyPRfnRq8pNvyHe2AOVGDN20S4wKWjdLzlgBccccX7c37XFsEkOTNQ0+Ee3EupbkIWAUNlJdRot5rWINxb15Jvxa2CVSEFoqjNlwoO4ENslsNg3HA7nseb8H374UCqIceLTcW7NllxtlIAHHlt5fQH+FyT2wml1WUEbDzriEOMoLbYDl9H7t7C2gckJHocK57GWzm3roo1j5PCEQuMlbQUunOSF6NwgyGlFKAjkOJH8ffjHWy6kNwYjAjt7QU5psE9zzZaiTb+d+PsYphl5DinENsoHBWpxBIdWboISXABawsEHX9wcBQpDkZbmuasvkNPt6G0Nlw/tEaRqWQbD98ke5vhmzxE2CZ0Tm94ob0xhlLkdbrbSnQVONokbgAJsdQtcX+/AwidflPFlKFlbLO6uOtp5CtDgcTYhI84vZI1G4Nu3Fi4NxtIb1sSE3dJKHyUbY7hKk2vcC51WwJRMh5a1TgEPrJJ0DkWPDauwFwOT3BJ9jhXKMacuiE4hchxMmG/LWp0FWstBKk8eXTe1/QWP64TQGltJ1b8d5SVlKw+nSQQggOJcFyhYJJ9Qe2F0ZLsqGlKH5ES7pXIebKGVKF7EBKwfYci1wcH1Bao0Xw6nmt5/nZQzvFyw8oUlHHce3cnnnCGRrO85NHG5xTQqnubi/GvK3v2bqBJPzAhf0KUCPUj0ubi5wrdjJStRcQ+XgENIZTJKiWwRfUCOeL8enrfD1S8lVx9mPKejsw3TcqTISApAPtwVjj93+eHqDkOAw+Jk2fJkSAb6mQGkg8ci4Kr8Dm4x5+u4qwyg0e+56DUrdFhk8utvyVfqXNk7RW85FYAIDbR1ADkt344OsE218WtbtixujWUamcxsVSppnOMsAuJMoN+cgWSsca/qUSPYE+2H6n0+FCATTILbJACRtjzfbnv/H74klGrNLy5TpNYqcixU8iK022Nx190i6WW0Dlbh/sjn9Mc7BuK3Y3iDKaljszW5J5AK+pw9tJAXvOqms+oxabDXLlu6W02HHKlE8JAA7kngAd8Ul8QDbj+UmDWxaoVee1HhxfqENhB3FkHsXVaUhSh2CtI4BKrXpNPqVQcbrWYmQ1IHMWHq1JiA+5HC3SO6hwOyfUmnOt9Q/GOpVLoSDqZokEyHR6Bx03/AI6W0/xx9BecoXEjCqfOSvCw6bR4wF1nXb7IAA/mR/DGSf8ACEZ5Q5LyN0whNFuLl2myJz60OpUHnn1bYUQBxZLa7Dn6ie5xrwxTmHqPGpouWoqm2j9gBuL/AJqt+mPn/wDH4piD8RuaaXGecXHpsenx2WiL7IMYO7er1sXTyf4Y5Lpqd9bHTO8YBf8AktbmyNp3St229Vl6atK5gQlV0I9O2DEECwVdNhcWHrgMaMoqU8tdwe/HOPOFtP7/AJieBjsXXKshS1MrbZWNe4u+u9u3thLUV8ITcH1uMcKSvXZVlff/AM+mOy07sdD2qwA27+lxgJk3qQVOJt9PuPXD9RpCWFp3RpN+AE6v4YZ0lZVpQoAi3PbDpHbQpBXrJUP3bWP53wwSlXT03qEaVMbbQWg7cO3eHlB/yUt3K1ngdh7ffEK63190zF0nx2oJeU5t7f035tx+frc/fD30/K0RVTpidew0XEoTuJ1NgXvfTYgc2HqftimM01NVSrU2YF3Q88tSAPQX4/lbBJV26aFqvf05wH6scPJx4G2ERRrYGoJ4+98XV0lmCXTn6VKgR3mX7FW5uWAtbsDZfpwrvikxwQb3Pth9otSkU55taHdBaG6PmlF/sB74ICBKu7OtdqtRzFAyxIW8IEG8lCmmVLLKEJuXGteojuBcLtyQbYzvIXqdWsAJ1KJ0gWA59sW/OlN1HL2Z811B5Ubcp0amxlIjNvB53cFwV73kJCL7iQfuBinF3KjiHRQCyDg5ongJ7nBOFUBpL7oa7FRABwFDsrN6SxViRKqetzREZOokEi54sPTEqdbX5XZBS3ugFN+eP/P9+NJ/Bj8OmQ82dNJ2Y8+UL8TZqc8RWW3JjscN7ZAK7tKSe5PP2xfcn4Rfh+eS04x0zYEdu41RarPYuUeW1xItewJLl/MRY9ji+Gqjj0VL4S5fPRCFEatwqHrYW/K3vgYjJ1kLasF2HfkAnv8Afvewxvc/CJ0GlhTSsrVGnvNlBTs1+fuW9rLePPA54HPrbHlfBj0fp4EhEbNjDy1eHbU3mOWH2NZIBBsdfAVzYc3I4Ixb77Eqvdn/AHlgdLSkqUtohaEWC1W7XNgLYGd1KNCU27mx4UQT62vjeDXwV9FCosU2FmN0EtpbS9mpbKGEWsbkAEn1sOFDjixwQv4Ouhr0jwNNj5ndD7u00TmN32OvTxyfbiwseeDd/fokPdndVhN1CnW7f2OBYcfoMe2VKQEaVBJCgeeePtjczPwVdHlMwVw3MxSG1oLgdczJpaeubIA22QSb3B84vYcHCpfwW9EZK4smOzmrbbWhDyjmN0hFu6rFgldiAQeBxbn0PvsaHuj1hEJQhzaTo1I+gJJ7D8/44B50LVr97EhWqx/hjdLPwZdGXgY6P6UJd86NxeZFqVr72UkMAD7259sDV8G3RaMyxqbze80482ApdcdZ4NrOAFoXB557jA9/iCb3N6wuefIi6yfUq+m/v9uMDTGLpPFrGxuedfscbslfBp0LbVrZpOa3I9jtLkZndQ64sXJA0I7BFr8XPobjAXvhE6IiMieijZgLYO2pTeaJyUJfIuA3uNnjkEXsbBVz2xG1sZSmleN1hBaE7hS4pTZSbL8gUBf8iMcL0eGlSZj0ZlSxcFbobNhcnuRx3P6Y3mPhA6DJiraNEqjT7SSSqRVqlqYcNrAWcbCh5iCkA9rkji75kzpD016dQJjeXEJkokvIfeRNjOykgN7jY235LboQPp4HJGnsTfANdG3YFRtG526+dTkuluEtrqcQi3AVKaNh6kc9vy98GGXA3AlVVgJcUgEqMltQPqkDzexH5Xx9UBlDKUNADlAy6oB1p0tLo0RJPPlN9u6FkdknjuB9kzGV4DT62v6JUZKdvQ2lVKhJsB+8pGkrWhQKrXuL3FzY4rdiDeiubSZea+WC6rTdZUKxCIF/L4prnjnm+DUy4CtC2qnHC3OQDIQRf89WPqQ9leIl4vQ8sUuSkxkfPeosWSFDzAL0hrtbUAeDc88i4b36DkyC04ublahgo/rDz7lNjqG2RYlQLZNxym5QDoHre+J79f7P1U93818xkS4NtTtShq5sQHm+/secFqlQk6U+Oh27EqkNXPB5tf7d8fTRvKWRGY5hzcmZTmI8MW23RSozdmwkFvgg2A8xGknlJB9MJU0HLkjRHg5DyjHjvqLLRcorKVayCdzbWhOtBFiHLjuORfE99A5I9h5r5niZSkqCkVaBZY4SJDfYe3m7YPTNp6tTKarCJHNkyW7/AN/2x9MhlTJi4c2a9QqYoPKR4oOU1gNi4F7gsmx0W41m9+Owwe1lfJXiWt/KtNqERBQ14ddNgpFltktBIW0F3B0gJUOLgWJN8Q17fuqe7nqvmI5KprJWhdTg2J038U3pJPa3m/jgAm0tKdz8RhBKLX1SW+Lj8/scfTpvJVApS2Vqy9Tm27lrSaLEaJsfmLBCCQQtQBtx5iLEpNzmcu0kym3EZWobT8gOKJj0yJqUsg2WkEbhABIJUL2K79rA/wARb0Q93cvmGmqU/wCpVWhKPawltXtcW4v3wB2o0xS0ocqENBPIvLbFzY+mr/xx9RG6Lll2mJQ9lajhQcRtNJpkSzRWokN8tC6Dq73va5wT/Q/KzaH3VZXpER9tZb3UwY6g7ZwpUi+0DoFgb3IBv3ucT3+3JI6kK+YAnQVEhNVihNv/AMKb4/S+BIkR3Fj/ANZx7HnyvpOu/qRqx9M38k5SfbR/9aGWnFtpI/rNHiONgWNkW2rrXdPBPAIJB5IxxHT3p4rddPT7LT6yAtwsUanlu/ANkhsG3Nr3UT98D39vT6qCl8180WkRlXcRJZWUXIVuos329ycARKiG7yJ8YhKrakvIsD683tj6TSemXTRl9pbuRMoJfS2/sIlUKNdS+wcsEXNuOE+Tvx3t6o5Gy68lipL6X5Mq8gOttSEKjMtaWECxsoMpstKNIsRcWHNrXP8AEG9EzqbNzXzaVIZAK0Px1gHSdDiV2NuO3b2vgz5jhIbS2otWCyV9r9geeMfSV7pV0vkNiSOmmS0PxWVuAfhUVajoI5cu0ULGlwEkc6zb2wjPR3oy8xGQnpTlJTDYX4ZxyjRShrm6g0QzYnc4JOvk27YAxAdEXU+m6+cF0lS9J1LKfp0jt+WBKdaUEpLrYWByQsXPB/hxj6OtdHemLzIYa6XZHSnShxSG8vR1ObBPLhG2Sjzi4FibW73vghPS/pRqLSOmmVEtvuAID2WYpU52sbAbl/dI4vfgAct7+OiVtMW7FfOgN2BUHAQg2JSq/wCvGPBbRXw80uw4IUP/AKT64+jaOkvSaVIIk9M8mSkbK3E68twWlXBASQna7g2tfvc+gwGT0r6UxXUNu9NMiIjuIDf9Yy7HUtQuL/N2yNZH29DYgEAB2IAclHUpcvnMptIUUrdQg6uEjurn/Rga7bm647fVymwAtfv/ACA4x9F5PR3pWmG4wejeS0rICA2miRCnsSElO3ySPQ2tcc3xDc5fCb0jzKUoo1CdylPcQ2Gl0Q7TQBty6w78gdzc2T2NlHjBZiDCdQgaU+JYVCxytRKU3sk9h+ox1IW2nQChSLd+5P64vbOXwe9T6MXncpTKZmyM1bmMoQJR5PAZkKCHD2B23O/pik6lSJtHqi6HXKdMpNTTyYVQjLjvj7htyxI/yhcffGpkrH+ErO+F7N0kCbHX79reuOnTrP7t0fVpuT9vywIMO61p1q0+ursPywWlCydLaCkfQbH073xaql4OXQG0/M0m91Jvb3war5YHLiu1kkXt+o7Y6lYS2lbm2nmygnm2HTLs+j02twqlmDKsLMNPYc/rVMmS3Y7cpsixAdYIW24O4X2vwbgnEvYJo+85Mz2t1tSFtKU0sFBadBIv+R74OoX45FqLcbK0SoomunQ01SHHUPuk9koSwQVkn0secbHyZ0p+FLqJltzNeVcovutRiI81hdUqDD8OQsXS242uQQOFk6hdB03QfTElb+FT4fnnXn/6JzQww3rDMOu1JCt3XwokuK9QLG7YHYEk8YnVbG6ELZ2R3BUb/wAHN1nqsPrpX8ldQq3Um5+Z6K4jVXFqafckwFJU02sv2XqS0Xk2UewHtjb/AFD+K34c+lS1Rs7dYssxZrPenxJgnTfXjYjbix+oGKhS7DEdFNajR6kxGQHVfibSp8hwBBALjr+pby9fFyeARyebq4aoDCAiHAgMNt8KVAgNtRlEabup0Abnk9Ce3HbHPe/O+61MbbRPFU6y5B65UGk50ybCkx4cwOxmp85Ihvv7ayAlTTl7JHcE+exFuFYSx48x/Stx5xh5S3EfNktgBBA1XRb6+QfzPJOEVQWuqxR4lEKohtICd5JU0ppABGkL44NrE9iocnjCIvZagtRnJxRS9y8dppL4U24LJUFJ50LKSdNrHkC45wE6U1OjM1mg1FC1Oh9xokk7jhdJN0m1vJ578XXbUO+DegublUfMFBqb6y2lt40if5gLtu2SCfycDav0wghyVpiJXTq7IktCQ4EIfJW7tL7gJcBsgG3fmyrci2I/ToyaXnKqZeLyEtVFG80ptYKUuH6rEccGx4xEWr6AJ1euCJsZmbFehvglqQ2tpyxt5SCD/fhk6eZjObMmUivO8PyI4TIH9l5HldH/AD0nEjsMOq1kHMFLk0ufOhVN2Wp6nOrZaTGabS42fpVtKsFjV+0JCrqukemGlqm1WIwyttsTGinbbU85vKY4twSoXA76CbWsfti1+tlKFGzjHzBHU40alFWncv5NxICVAi3coKee/GKvUmMtKguWwmOHS021uKCjcepGnRYDlI5PAPBF0O6ZBg2iRxOW+0txDe04WSkWCzp+ZwbII76u9+L441JkRHGWno0yKsICmYbwGk6BpseSggfTdJHpccjHISFie22iREQhyMAPEI07hupN0gJ0AgcEqItcEdzhRAkqghTUaI67FdDTryrt7W+QWwtzQoOOgeUAaObXJtgBS6dEPspkSZUuYYQ2tDbTbh8wQPNdpy5JNuL8cm3YYFBfmTvCxpG4la3PlBS23WFEf4xxRT5EKB5bJtzYC4vhplsMsynqjTU7CVrbabBeLLAsNSg4LHfR6kX4J72PI2MyPUyKHqg7GSXQHG1IjlLRK7m2pF9ZKyTzYA3HIwXIpwFJSuMtp5Gjbc8PGjmMXEsthFtA0WCFgccgW1AdyTgpDjTLRE5b4cAdsixLdzpsHGyQsWDYTyLE4KbkVWrtM7qRKK9qY05JcVocJJG0BqIJAsdN0Dg9iOOyZdMcSVQ1yZLig+yWXHkP+FcBVqbUSNaxfToBNlG3KLYiX/Mq8qHXXozS2gtXVPLSysbOxGdVLsg+gcQn8zqv2Pf0xEqz8WXR+jyFz8tuVLMEtqy4zkaluRwCgWCd99YsP8kJ7gG+MPhek6W1LAT7cfl274GpRXZSQb9v1/0Y3NoIxzWR1W9aaqnxs1t595FD6dwoqXV/LXMqrslxNze9wGwOfMO9r+thiL1n4v8ArFUSpqJJodMYQfKmJSWyUfmtwqJJIuT6mx4xRqdSgEtpOlXbU5YcfbB51qTrLd1Ffa1uw9PTFzaaJnJU9vI47qzn/iR63PLIXnaQQHC4nU00XBcdgrTe353P3wsoPxIdYGn24q80sSGQvztSYTGkEg6l/TwT7/YDFTNtMBKZCrJUsgKKlgAfxxNMh9K849RHQrLdN1RUEIfluEhpPIv2BJ7+mBLHE1tyLK6OV+ZTurdV865qX4ir51nx4ix5YUGQI8NtFrcNIFiSO5VfF79F8x5yzZS3YkTO+aFGLpC/DPNPNjtYaVNq0D7C2IPQfg5gwdleZeoj0dxRCVtQoJDYc7lsuODvYji1+1vXF49Pen2W8oUxVPy3Gl1RTTRZU8lzcWtw3BIGlJ72uQOCbD0x874g4eFfGS2rc0+YFvyXtcIxwUbrOpw4epUnXlnqE8kmp58NNpryBdt6nRW3DwL2JSA2m1iU2JJ9sNrDNEoiGq3XszOu1KIy6CzT3JBjrY8wQFo1FB4AVewKV3xL6bkCTNjhmpQ2ojCkJTuOEOLV6myLAgduFfe2H6D08psaOzGk1KU+iPq0Ns2Yb5FiD3JH648O5mHYb3KmtMhHJth+H6rpPxCac3jhDQeqrHL34vUZcSqUhU4JPzWTES4pbza76NP+RoJupzRcD0xNoeSqlJcdcmIhxlpPy3H/AJzgHe4S2uwPpz6YmlOy/SaU2pNOhJjhYsoh1alH15JUT3GPSJTFLUFTZSI7aydsuqte3cD8u+Hq+N6yf+Th0dhsL6u+Swsw+IkySnX5BM8fJ1HRZUxEmUoEEBxZaAWL+ZOg3Hc+vbDtFjRITYZhQ2WBax2kBN/zI5P64Z5+dIzcdL1JjeM1qSNbpLKQnVYmxGs9j2GI/JzHmWpv7MV15twLBLdPSoeQkC3n+1ze57YytwfiDGP5lXJlb/mNh8grPeaam0jFz5KaTqlTKSkJnzo7DhsdBXZavaye/P8AD74YKpnyAy2pqnx3HHSP2jlttJtcCyNRXftYW74T07J9WW0mRUZiGUMp3XFTAlxQsOVXvZIsTfUQOL4iCK2vN0iTQ+iPh6oG17M7N9QaC6NAVfkR2hbxz49knbHcrP041QYPgdFrUymQt3t4R6nb4Xv5IGepn7rBl8yk2dOq9ehTWcu0mnz6xmKqB1EPLdOAS46AAEuKJPyGU87jrpCBxY3BGL06L9LajlGhU+sZ6qiqtmcxjeyyuNS0ueZceIP7N+FOm7jpFybWSIv0r6UZdyJri0gPzKpVnQ5VazUCHZ9Rcvcl1z0SPRpPlSLd++L8SgJSlIHAGPdcLVdNWCR1HEGxNsAeZO59BtZcPEw+JrWSOuTuhkWGMrrqCK9mjM+cHlksyZig0pR7MN+Uf/I2P440N1DrDtByVWaox+2aiLSzzb5q/Kj/AOZQxmqfEboeREwlWUqQgRyLfVuGx/8Alvj1EzrWCwQN5pJ0gbjSqrU8zVOQy2FIO2X3Am7rq78epsPa+PmF8Y9fp2aviHztXqNOE2A/VVNtSLftg02lpSvsAtsgDvYD3x9COpWZofS7phVc/rkyGH6VTn3IjAcCY78mRpSyCO5cCw2RbsNXGPlNV1vTEJkTXTId1lxxxXJcWeSo/mSTjn0+Gj+IOrnO1LcoHQf3VtRVEwCADQG6ZHELRdKfIdPp3/MYb5Cwkea6j9/TCuaooO8e/pbucNbrwHkKvKtH64665yEpaLpCeNZvyPTAkSU61QztlDhJQSexwgJSSNSzb8+xOPC8l9IuNWsWBHcYiiVtxtS9Cli5NsSqjwY1Ljbzxu44CG0g31C3Jt+XriMqeLpSvi7Z0uD2++JZQFyZ8dUZji7Rs5otpHt35uAR/wB+GASp/gVjayfVfAEIdDVnFl5wqT6cA3Fz+X8MUk8VFarknn1xeTMdassOR6xW3zEbs4mOl75LRPrYfuC9ze1/S+K4Yy/QahXHKY1UHBda0oc27IK78D3wSFa3uhRCxx3Tx64c63SlUiauItWpKfpVpsFYb/qtYc4RG6XxW2WG0yHnbblwkAXt9/b9MGRFLqlUjRlJ0NuOISUtC1hfki+ETDK33AhN7nj88aK6OfDPUKgyrPec3RTqPR0N1FxbjgSHGwdXf8xgg2Q3TJ1poEbJWTKDRqX4KczVXFy3ZbbTXBQnS01yfENuttk3C/IdwFJPfFFLbUFci2LB6vdRJXULNL9QdUwY8P8AqcRbTIQpyO2pQbKyPq4JtfkAgXsBivlLSVfb29sQm6IQFDm2HKjLDUptYurQq5HbjCLb1HVqvh2okFt2QEOchwaLHjvxh2tuVCbL7DfDdT/6OdDMmxXVoalRqPGkuB94pTeQrdLm2L3ACu9+/ob8WY/T4brjzj7ch1haPISyEqCCbJF76ygE30pFrm59jUfS6t5zzXlSk5foa8tv06ktRqFvO2IQ4hATrcu4C4AjjUSnlQ8pHOLpYbyixAV4yuut15vVqpiH3JEd50kJVpfCiFpTpUf3TwOLi+KKmB8DrSix6cx6quGoZOcrE2rYi0yK8iRAQ1CQg7DRa3LIKiTtAJJRcjc5ueT3AxyU/KaMV9MGWsubqWyw8gALATpLoBJCDuH012vwLkYPeRHirkS0sOr8EGy4UFbqXfyFty6uNABJHH5YIqMSny0tzlIaecQ2hl5twPB8A3JWSAVuWJ44HY312BNLcpWjwoNorutTK4jkkpW42mK5ZRI4FiTZaOCUKSFc34wRHTLWvVN8NEZWqyylhxrgt8gBZPn7cG4t3ucKH5C/DR2UOORt1aFsSG6eXUX2lAm5Fh5Ar2JtxbDoh1Lxiupo645sQCi44bBILYBtc2ue578DDWQKYY8RESRT4zbrC0yG9CzvXfAJVbgWBFx2Av3724cXjWIsRxEdEJ7ba3Q0+uyXEFCgFLS2ojngfveexPY2CqLUW6oapJktKTJDQEOPE1NgBsJCw4eb8CzfoU3vcEkuXUWoLjjIjyBIkJQ6pCNDTTLaBwCn61oV7C4AvyO2Dm1yooDSZ0ZDM9bcJ9LsfceSvXHAbRymxb1Ek8WSbji/c4KUpao5jlxQckWWGWnSkruvuTcAX13v373sTYdYb8WDvPI3kXSWlSQgiyvO2pVijg97XNrHi2DmG1NNrX+GuuG6FhMhTchJGvy7KfQkd9JBI7nscRRJiy6XHbN1wQG2nGXo7gPLhsoLKCpJIKLElRHfn2wZEqMdKFsxo6I3jwXmY7Tl7PkDUgJRq/c1eZIQOyPufPJbTGloRGlmRHI2XkwlOCOLqOlYIJAAVexcJBH1emEjDqX478hdWgS24zI3XkNLDiVkAFwjm3AS2SSebAetiAolKPwuOtEd5ENb5S6GTubJSgWKnG2nBdsgJIBuPQ2AtgTLbjtQbeWkuPyRvRY7jayWEbgcS58si6zwFnk3INycKXpVIE3cYYm3jNojuFnW5Jc+bqJa4AJIVY+ib2NwcJBGTLhympbLcZcxwSlI2FJA5UCtwm5cLaC4bjmxGi5GJmQCKcmxpjyJzq5MtKJCHC028GkF0EfLCSQSLkDUbji/HNyksTqm65VKmxEYlMXSzEcS4vfC/MLFs9uFG5tcHgfv4VSG2psXwyH2n1CNuM8t2YQApO157nythw3HdZt2Ax7ap0yP4arbEVJbbk/PSHHlN8JK2mkHWg/MBJcRYAX4B4WyKKjMIDC1zpNMcCIxU3IKPqQCDYpGlGjQrVcDvbzckYT+EaVIcgUdlx5Ot0vWRp3ChIuQ4SlbgNr88C/7x7qI1IhiW2tyXLU/JLhQhlIbaLWjUlIJTex1H6gLgkWtyUjCno0cPvAx3pbWhlLoDq2wHQAFKuAQkXN0ggEnvcWllEmajyXENPKYfWCouttvoDjhLbntqSCOFXNhwRwSMeIenSxDH9UQt7Tu7hDmi5JcugW18pPN7myOOTgM1KE1D+uqcckOvLYksx2h9AP1tt2uLls9yAo2wWwuK9CKUTJElhbR/rZdbkNbZ5B3Abd9QJNjcWIuThnJEOK65HksOLaaIs4ykOutEaxcuaQLtoJQeQlFgNeFkeT4d1qYysSVS/kuPGXupU333AAfJ5FC2oA+XuMN3hf685Mkw3U2bLDalyyrdudI2k3CFobt3tfnyAd8OUJLTa9EzxBRugIituJZU2gIN+S0S3c6QdR/dt5BzhFE0qckqMhT6GkJaZQA0/JLOpG4RrSC2eQALKCx7+cjC6C/HfhrtFjFLkpcdxDih5W16SFEixCL+t7jSbJsCR1tiobCpFMDEe7LYDxZKQ44vy3KRyXLAgqKDe6CeCbdbhFb7kViFLmyUBw6Q2i7D/O5sp1EBAN1XsQe9x2MFzooiXoj1TQtqelGlwDxLhDjWlBCSQl0KJ76k3J47g83wexGkOKCCW3koUi7JQXW7HnSo6gB/iyb8nVfuMKQ+nd+cqSy0shLLAFnF8AFLZQqy7g3Wb27cdhhqnz1KUpcFbjrTbbiGFznnvMtDnb6Q4HOAOTtg6+L2wXDMiHdU6Lhx5pJiogbm0tYLTnlbFwQdQ1EaiEpvyPyuTjk52nNlZRLQ3LYk7l1PJUkkBJcOvsVlZV29NJthppsqu1l9qMlDBW9Y6Xr2cXyEqAJG2i7iu4PnsT2JS6v1OHHU2zTWpMdT9t5KG1F0+hIsCQNdzwLmx4wrY0b31SaPOSmO/AlIYbBshjZl6y6wRytSVo8hFxcJB76wocX5JetJG01MceWVhxLby0pKCb6T5Q3YaVDzcGxv6YPfekuzXmVKG06rdbaW2ppxq3lvpJsggqAAJsbG9yMJ6XT5LhjMJcjKLSbtIfS4kblgrQ4CLgq73sQbC1he72QujkT5DTwdpr0dxotjc37k7YP7MgHyXPNyfYegGBBmY86qlQUwy869stpd1tBkFFkpDq+F3G2kKuCbAAcDA24W1sqluIdWA7ss7v1LDnzEk3Hq5a5sbe9wSS2z4FuSprWpKJYhFtiVq3GzcKKi6BcBYSbC9r8HtYc1LpG3HQKYmdVSxNKFuOttr1oY3CHAC2k3WLC31aDr0DzgHAkLktxkRqky6wpo3lR23BrSs2cNyDcLIAUeObG/vha/HQltLkSMhKm0B07qbHWRdsFsi9zyeQSb/lchaWHlgNONs62w0p9w6FNgAXsr98kWsALC3b1wVAjXFIiXKY77seMXHJIYJDQsuyTwjWgpRZPYjtb7eiQgzJfpbiHEPMMtHZLi0qbDgJClXTcAkfVwfQAc2SlEHxL+YfBofTES69ssvI0qRe1lOL+tCR6Ao9BybDBDNSqsBC5S2WkRI7jZ1pdcL3YWWoevNiCG+PzOIgnSNDZMlzxq22ksEM2LmoOIXwLOE2Rx3A9reuCWWIzkVCFUxp1C3P6u7IZSlp3gbepu7hX7gJ9QDbkg8hyYL7ZZjRjT5LrLkZhSIXzlIsCoDWoFA81tPJ9QOMJVNt6AHGWI0eRt6fGBTRKEEA28trC1xa3J9b2COTBL3mHHH0xAmTKWhkvKYCEEEAkKLib3bB0kEm3PAHbDNmLKVBzZQI9Ez5Qo1YhhtxzaqiW5aVg3ILRI3G1+Yp7oI4Nx6OqUzwXYHg9K3IxkhptSWnQO+tKhpCEaPMDe/N7Hvg1pq7aYxOz50NuOOuBIjixBLgcVyBqvqSeRzYDDMzNKDi3ZyzTnX4NMn1BSZvTbNFQocuWdz8Oqkbx9O1rCi20HELMiP8ASU2UXRew59c9576NdS+m61TM25alRqchzbRU458TCNja5cbF2wfQPBs8/bn6GyJz01txulPNvKQ5pagKeRFCmgi42xpC3AQLmwt8wGxOHOmQZkDxgjwRIfh7a3YzUu4krtyHHFjQWkm3lIJ+vsbHGtlZJHvqqnQtK+Vb7R20vMuoKDyhfdKhf0Ix5uMsJO4q6vbR2AH3P8sbtzz8NnSjNrj8tOWk0WuubiEzMvTUslRHmK3mW21MOcXT5kDgLJN7YorO3wf9Rsu2lZOn0vOMBwjYDKTT5t+/ljOEtuW+zgvxYdxjbFWRv30KyPp3s8KqvI2d85dPq0ivZQzHIp8xDdnAnSWX2vVp1siziD37cHkWPfcnSjqrlTqfSWa3T4UMVulxf/WtPfedQqGgkBu4Bs40SBZ0cG1l2XxjAdWo9VpU9VJr1NmUupoHzIE2Oth4EHn5TgBP6Xw5UDM+YMqVeLmPLs52l1KDcsSkpCtN+FpIWNC0EcLbIII/Q4M8AmF2lCKXsnd5fSJ1lEYomz4yJCXAhbq0rSVNr8wAF7AErAubAW9++OGsswXkxtUlccguOSBdwMrCOGy3qsSoD29D39KA6cfF9QakhEPqhAco9SIAFUprLrkJVr2C2m9UiObE3UN1Hc8Xti7cu5iylmtxeYsuZjhViO60A6uJPTIDVkktiwO43xZF1IQUkX5tfHMdC9niC3Ne13eCWwpUOqLFSpsx0tPqbLTaHSpLi0HkN60EeXlXBHb174WyEzYD+lUxt1tpkhILa20gXB5SDYE3BuO5Hc2x5yM8uqsBcR1pv5TjjroMhAaAAUon6yslKdCjzbVx2B4mPLp0hbkh66FgKXrKl/MRdSm0jusjUObeW4tfsFT3RcyoLcB3ywyUPL2wqQXGA5YC9iSULJP1E2/PgYjmZoc2ms0vMTz0P+qSPpjtuJ2WlrIIIc7e4T6cmwvh7cWzUWFIXGcejXBCzNcAaBvzZBSQsXN73R2JBsMGT8qLqEeZS40xuQyWlpLDaDdBQggbgt5FG/Fxc6Ss2FhieSl+9dX98PFfbUiq5cU7cOaKrGHuhzyO/wAFpB/5QYunn1xjPoPmxdJmUOqSHLeAk/h84k8lh2zZJ+wXtK/zcbLBuMBmmiLt1AOt9JXUcgTp0dpTr1IPj0BKiCAgELPBHZBUbfbGb5iVyIT8Oo1Zat8ojQhOKSrsS3w55CR2ChYngG9gMbLkMMyGXGH2wttxJQtJ7FJFiP4Yx9X8vyaTOqVEzBCU1EgTltpZmhEmFMDZTtkqPnBLdihNvLa9792KVQt+nZ+y/LcQ7mulORQ9tsuPQjtKITptsbZLbgJsbm11dvTEplRJsqntsVeTGakPndaLLuloDkHSP8Xddz5vTt6DHhCnLgr2cwGIXXSJ6HDGQHCV6Si5NxdGkF1JvcenNkUFb1OqDy01EzFMBBCGntthkC2lCSFHbb0Aiw1Dji/IM7qicaq3VX0tM0eDImoLYU6mLKRHdbQL8p5KHL8cFd+PzwU3FtLb3kOR1IUUGM65dKLAJcKh/i1g354OvlHGEhpkmdLeqMdLkltZCG3DJs1trJAukkAknnUR3B7E4XFdHjQ2paIpWzIRuOONPIQm5JFk/MINtPI1n6jdIBsAjdGTKZZ3dTTmlt69wmS8N8gOAWLgN+e9u5tf3wsT4ZJTSJkCXJqMlwNa2GnWmywSrbQlqwJQDYD1uOO98IgnxTzsd6QI8ltxspVIGoJJQdKvqPkFiLqQRa3e4wFurVRcd2S9WH9UhTVw63Zp0FQ8yTYfYLJsLi1+OFU37q+YTRQo8q0G3ci/P+jA1+dxSUayfccXwubyjmdDumRluqItcWMJzv7du+HOj5Az5XZsek0TIeYZ8yWQI0WNAcdeePfhNr9uSTYAdyMd1czJ3kkypQk5mrkCgCV4Yy3UMqdLV9NzYkC41kX7X5xqjLnwaZDYjoer+Ycz1RVgsoadYgJSfskanCg373BOm/bEr+GH4KusNEqTmbM+5VhUF9UctxWplUbckJBPcpa3UI45sbkW9DjU8ToM4l1xdYkRZBfHzGo6QyhY5vySTbn2xwsVxQUWjLn0Fz+i6dJSNk8dh6rPNC6UdGskhgUjIdKVMWojxlQY8TISb8oLjhUL/kL84s2k0mdmSS01CeXFZbaWxKZMVBjW/wAYQvTwsmwsLcAEE2OLoh5AdpQ1U6nQ2Cs2UWwNZ/NR5P8AHCh/L1SZaU6+hSkIBJKlC1vU8nHgMR4kxV12QUrr9SL/AEXbho6Vti6QfBV5SunsGI4H5kt1xKbFLURHhklVjdSlAlZPPuP5nEpZjRYrTceDDZjttj5baGgkJ4txbBdbGaYzGmh5SmzH1BWkqGlCfZV7839r/phj/o91TrKHvF02XHbN7MNqaYR38o76yB6kn9MeWOC8QcQOz1j8rPPT5AfW63mppaQdwfJOz1XgRnno0uqwGXo7YddZVJQHGm7Gyii+oA27kc4balnijQWyYivxB09gxwgfcqt/IA/mMN0fovmt1yTKepsRmbIG27IL7SXXrE21OJBWRzxf09Bhwj9AMwSHQKrmCIG3WlNuqZS4tXPcj6efvcdz3x26P2f0cNn1Ly7yFgFllxZztGABRmXnnMzzjbAZjQmyAlZirCyVm5CQo3P8kn7Y6zl3M9TnCothhsbQaK5kZV3UdlJCiSQByRYf9+LMHTvLWUYyJ0qpU+Ftp0eLmaGvTk3JCAT6kC59ziC1Lq/01TUnKFlGtys915N7U3LEQznCR/beSQw0PutYxrlp66gBZg9Fb/Np/upFJDNrPLfyThT8m0+KwG5RMlKbhKLkIsTfn1Ub+uGXNHUbLeTasjK9KgzK1md1nUjLtFZDktSbDSp8/RGT/lvHtewOHlnInWTOrIdrstnItLcFzTKJND9WdH9l2cUhtj7hhJPP7TAhkuV0ypy6PkTpqUxlLDrxiNpd8WsnzLcXq3XHL91ukk+l8eYmw/HquXLVtc8n7Ozfif0W+OopIh3CB+Kh46b5p6j6JvWipMimKIcayZRX1ini3I8bIFnJqh6pGhoeiVDE9kzqHleEzAUqLAjxGg2xDZaCQy2OAlDSBwOwAA5xGKkeokxLseqUaXHUhWtLUSmvbTwBuAo2KyextwL4aItMzMpxp+ZQalCdAWsaYRL2vslATZQA5vckcX9cdGm4IrKy3v7w1g2a3l6cr9d7qiXFY2aRi581bfTqr0uvV+axSno8lumJO68zIDoC1cAWH0nhVwfbFoDgd8QPpBRkUzLLkgx3W3ZkhalF5JDigjygm/5G2J5wBzj6ThWFU+D04pqYafVefq6h1VJncqu64TVuwKRlplQ11GXuOAf2G/8A+ZQ/hioOoqFvTKVQo4USdbvl5Po2nj9VHFj5ve/Hepzun/UtCioYUsmyErPzFknsPqQP0xQvVPqNSqTRs79TKZVozsPLVEkLp0vcCW1SA1tRSkrt3kvpKOxNgRjS675LBOBkYFQ/+ENzlTYGVKf06gVGMZy6o2ZsVpxCloYjtqKdwA+QlxSTY2J7m4AxgOS4QmxJAHIxOuoUadGptJcksvpbnpckx5LgWfGNos2Xg4f2oKwfNzex5xXMpbqHktfUDybDGzIGCy5znElIJjZKtI5uPb1w3ux1Oq7XUpPa/wDLDtJ21oKgOQOQMIlcXsR2PbEJCVNDrK2E3Uv09MDoqd+cEKUQQ2rm3b2wKUQEJ1gnXxjtHuxVGCONR0kD1GAmT0/l1MTTMbWSlfCkW74keTYPi3HILKy249oGoAmw9eAOeOMPWW6RAzMy7DdkuoU2k9x5B7dkkk/wwsoVLh5XQZ70SY7I1lttxTZCGjax03sCbHm/a4tzg25qxuyYc5tKjRRQWvM225uypAaXpeJFm0hPoAeD3ueTiCSpTNOcdZp6ZMder5iVkBSeO3b8+xxOuoWZo0mhuU1CYzTiFNNDbJCihHqTe5Xfv/dip5chTrqlFZV9z3P54N0uRelynJTxddcUs+6jfAEDg4KwYjkYVNZSTIFEfzFm2mUltpS/ESEhYR30epH5Dn9MaZ+JjrRCpmWo/R/KT7bj221+LOI/ZtAC7bfsVn6j7Xtye2e8sV+PkWlP1GG6pdfqLBaYI7Q2VHk3/wBsXb9Efc8S34cejdR+IPq5Cy1KfcRTjuVOuzOSpqE2RuWPopRIbQT2Kr+mAeqAbmcj8zfDpX6F8NNA6/PKkLRWKw5FdjBACI0IptHeJ93HEufkC374o8fV+uPsl8TOVqVmH4Z875Qy3TmP6pQddPpcdvSppiGUvNhluwJQAwo2sT639MfHIo830kA8i/tgNN0x0XgTewOH7L8V5chDtwLcovzY9wSPa9sNEKOl+QhCj5Sbm3tidU+O3sspQBZtywFrYsaciWwJX2X+HWoZDovRfJEdjKsyQKjRWKgqRIm3U8/IBcecKglP+MJ7dkBA74eahBcFYmrl/htKYk7Zci0pgI1ISnykg2DnGoC4Llzz3sK4+G51t74f+nL09DbjrlBbYb1tqCiUarWcRew2+bX44NuebJk0htS71JMvaG2JD6ADq0N3aUkgKKG0gixSSDfnnGeeeSokL5HXJ3J5lOyJkWkYsgM+DdS2hyC+ZAGzd91JKm7pIcbS2Cb8W8xRyCLd8FRkLbjubtYfSF2Q4xslTyTc6lKUL2QAOACOTwOMHRqVIplOjolVKY6y7odSl4FTSULHG35bgq0qIJN72sMc248xKartI0R1ApddZUpN+QAlJBQhfHHtxY3FsUh1k+RNz8eqLCVJiU+cvStyLqd2UsoAAulVzrX5hdRQARqHB7gS61uSxMja3ZTC1DYb8SOewbJSPPY8IBIVwBYC2HN+UtuAptbLsXfW+7rdaSl9wX5skC19fPcgeg4wS4tTrHyGUMJbIAWwghbthqdQSVaPMBpv3SHVkEHtdmulKSrlw6e9HYReG47GQ3rU5c316g1Y97eXypAI1YWKjpqM6Ogxtp5wuK5CwGmwkd7cLWbe97A68ciKZg05MGm0cpjNgoeRGZ+WwTwkNFZt2NgoAXKiDbCqKxVJMFdIMqGmKw2Nt2FGUyCs2JcCeSUAdyfUrBBsCSiiHGGWWkSH3pDrTjaNxlDKFeEFjpQGtIPmCSBt3J73JGPOyUQo7DxnusPyzuRkORlHcIPN1H6CQL29QO19Vm+qRaVSkLqMpUNh99wNvSUNFJc8rmo7n74NuAkX9eLWBzMhDQlSajOcp6J8YEPLBbD744S2StR1jQlPBHbn1wO6VPChusqejpbEZBcQ6S65tps6ty+n5Y+sC5vqNzewJIx12O4xFb/q8KMxrQ4EszwlLjYB7tbNghs7hsQTZX1DnHsuNXX4mGspWiOLuKulklBSLXXdDh8+kJBPY/c4UR6RIjqMmGhCpdwsLUx5UrN7fJ5A8ih63AAOm4xAomSXEqjy1NQHkRUrbKUsQYzjyXHO6tx42QRbVb9/1NwQSviVFEFIks1MqU2QhtlclxQcWRtpU42AS5YcHnmw474421JiOtqUuOplCGG1vqUlKnvLbyoAPNgLkL5PrY2C5xStHjzEaipbWHytt5K9t1dyNtRtcFOoAEWJJI5F8BzgE+UJnhisPyFoZgP1ByEsIEZ/bbuTypaiCFoRobRoGhy1we+FL6INMeck0zwzS3XOHt0RnG1rWEmwCVLKLWusnnta1xhLT4T0dbDPgTDiEIW4zLS2QGi4G29xtA13K0nnspYBtYgY6WajTqauTBhOGMhewmQHdouX5TxYOIsdQNgeAOFm2DukRrjMlapAly2mtiUTKSw02U+cN/OcU4DcWbA1WPKkEWvbDGsQ6y8JLsbdhtB1CSIyRe4s4t08BASL3KTYn7kgSJbVRUw2zAgusM8XQ5ZBsEarhHBvwq1jckccnRhAmFEfYjRoC07zUcMNNsxyQoBZ1NWKVEngEW9EgkdgCommOqjhS2YERuQtCGjttul1L6F8pLqnBwfKk2UF2IHFrXJEdbgKplUjSpbhIeQw0GHXCtVwXBY2GvTcAm2q45w7wIyYmmS9oS8FIcaW2ou8IPBGjmyglQNgjsEWBTwS4xLj+JdUy66xHLYLymt0OIcURzrv2SAkHunjgWOAlKO3m0IKlJXEUXBcftlFwfud0htA0i4TwbA9+yemzawlS24waIQ3YF1tuICsItdJI7WJJvfgC974HCakzHWpMuJSGY0ZohMpZCn1Oj/bhftz7EcJPNjhRIpkFSB4mdDeREaQHUyfMvyC5cHl0bYseCdZ4C+/EdsmSZ/XVJS4f4MtUgI21htxXy3NIcUokaTdKLElQt7HgXSsIeaaW61FbYltuAhmK+opYuQbrA4ItzYrP1XJw5TqdRlutVBmStO1E2mVlKhdBXq5abJABXa4cuVIIIBvwRCFSU25Ahz5BQt0XQpiO3tIt5mri4tYG55sBzzxgqJOgwnGbuR2KeuW6hXiLKs8jhIZDlgXNQ1HvwRbzd8GuwUSnG/DKnstoOhyW18h1orNktEX5sgXsSjzqv63Cx1hSks3q5Uy6S+N1F0pJbDieAeCQrgD1vxxfDTPpyJUNEyazGfQY5ntx9A0tG/azYsTftdBBt6c3G6iVxWPBtJjPvOzI7AW64446NKl/V7EmzYIFwn6eT6nzNSUxS20tstSFSDstpQ42468dSTfSVWHndIQoc6B244HvqjeHmRKVPQwFbmlNPAIdKb8BBBAHHKjZV+/pgymykQEtpnx5MXdC3W0vRCw6otgc6V8aEqItwkAHg30WiXbZeRCp6H2mpIQhlCdp7SAUqBN9vVe5/eHc8q72VhtVIZUw54ea34pttDzniYzbrgaHJdLfAPkBR3NiAASFYcWqfFcMqJLdG1LIlPMRJMjVrIKipJBKADySTYcX9RhE823LbRHTOfkpiNkEML0hSA3ZvcKLDWLi+3e5N+LjBUsiQ48/Tm3KNIDejQ4mIhBaUkXTtLbSbWKgAB/5OHFmpTFNtwDPhlBQW2glTrhDS+QlLkcHXeyrhPB5POE6okWTChQpMdCtp3bbXuva2bWGpIFgbC1lA2tcc2OHCYyhiUsrWJD7RLykhvU4/5/oIHFh5fNcBQBFjbgKWRLpnl5bkWMVOyyWyhl03aQBqb0ggeTj0F78X8pGEiWYcpjZi1MttyIqC4tKfMV67cf7WDYJJ9b2PAwe7R2EbYcMNJNiHNxxViSLX2/ocJCki4JN0m/IsYy1Dh1CFOkT91sArkR1BtBYIKgASQTcoFtJAB1EAnvieJPlTU4qQZbLUYTKghZcL0x/abdIbb7OD2BP08ethfuepTKWVkwJCkLZWQJKVsAhB+q9h+0uLkIsQRY98HxzHmyFvc64zW4UFwPXWdSrbQ024NrH3JBHbBa5q2ygQo9PdcOhssyHnGXCeQ4kbYc1kBJHcoCOwJ4xLoIRmqMZMOA04EL0NMsp0toU653Rq2zfyJvfnm2g8EEDrizI/FYb2/IabLYmjW064D5dGockCx4HPcC1hh0jufg7YWzJLTjDO24lx3b2StRAW0nSkuEgaQU2BB7XvhjVVUOx9DMnf2Imy0tyUD8y5CgQb2ui1insU+vIwuT7SiUPGS2GqZEapBQ04t1ZSovK3yDweUkOAchsgiyrWwGVFjJkvVCNDfJQGng4t4fLXbTYFattF/NdI1C3FxpODKhUpkhlTiIZbRP/qpQ1J3YxJCdbxbPYn5YCjxzcBGAvsSWHPDIqbSHkLQkpZZZWUrWeDZaeO5J72FrixAw9iULImTUadBLSGYSHS4HCNKmlp1oAcSllxdySACrsToN7YXJdkusCWzVXW22kuX2m0KeeB0kIGu6NsmyebHXew5wQt+e4w54pLc2E+54cBSQFOFHJcKQBc2AICkdhyewwMpblyH5NbW8667bceZd2CoFN9ohBK9esXQSeSkgpI4EUshOqee8gWiS6HXHZEh9sKChY2ChwOwte1h35wzzZDNMmtPTZ8BKENuiGtx5LBlIvdwFlaLIF1WIT/a5vfC4tOlYfQgNG6HERg62lpNiQdVgoEkD6vKjv5cObLbzalwXnY0pJcGzFjthxtsFXCyApxCx5ig25t6WvYIWUWzXQMvZ4gJo9cp8LNUcPO7lFqEdlbLV7eZp5betANgQWljsOe2KPzz8HWWJr6l5ArsvL0kBf9VlrXUoIQBcWuRIZAva+t0W54tzodUeG3VHGlyjHqrSPDC4S2pLd9xLSgEDQbJuTcFWnnjjC9uGhaXkKiRIbaEiQ22RusvINrWCykWsLBRvyk29sOyR7NilMTX7r5+5y6C9U8jMGbUsmyapTktFw1OhlU+MkD/bNCQ632J8zdgLG/N8QOjV6VTZqazRKtJp8hu7XjoEksvJB7p3WlXv/kk+nIx9OHEwKVJ/EaA01F8OVqShL3h0NA/4xRQWwAVkC6XPrvwvkYgWdej3Szqm54/MWSdypObbfj6Qoxqg5uHkuPtp212I0+ZDnY/TYnGyOrvpIFQ6n+6s15V+Kzq7RpDaKvMg5jjElDrk9vYmEiwTplNWAICR9TStV7G/fF4ZQ+KPpXmohnMEqRlKbIdDqo1WQXIQc1XsiY2ToQLXG4GzwkHFTZv+DzNlPbl1XIdeiVulMNLc01UiE+pwWKWmnG7tursQLqQ0O/Pcik8x5YzJkuUxEzflir0KRIGtkzoxaS8L28rvLbnII4Ub24vi7JBPskvJHuvo3Pqcqo0tmt0aK3XIrhDTM+M8jYKCdIBV5mz2VYA381zbBrKVx4C2fw+Y9ulqMuMomQWSblLqnGzoRybA/MWeLlBFsfN/LmZa9k+callOsVOjP83NPkbKHARyVN8truLX1IIPr2xfWTPjLzPCQzD6h5ei19tuyzUqcGoU5s3JT8q2w76WILRtc3F8USUrgO5qrWzA7rRWXNFNzlNo0kgRKzGDzQCzbkdwTY39f83G1enleVmTKFOqMhZMoN7ErULHfb8jh/IkXH2Ix851dbMgZ3qNMrOVaq5+J0wIenwZURyJIDZI1LDa7oWgFWkqbWe4JA4xtToFmNp2TPowWst1FlFVjAiwuLNPW/XbP6nGI9x/eWjxtV14z315y+5FzLErpbW7HnRg043qFtTawFnngXbUm/FrA97DGgx2xAOtFDlVnIU1cDR4unlExu7IcJSg/MABPq3r/li1ALMf4YC6tJjNPRW2i4XEyjdtwOdtu5Q2iw+xItwL3Ho9RkS2Ux2V1Ft6JtkBmI9dToUS22qzamySDYKI1ouO+FyFrddjRlRh4KM+uYZQSFJkNgEKAIJXc20gBFwO5w2zHobEyQhDK2bvBRXFQiSFADUtBKPcJF7cm3fgjCPU2Ti7FhTY6WJbsgrcBaSpPlLa7cFXkb97kWJJubdzhRHiSjOMxp2Q+haCyBvAJBB03SFlQB+WNBAudN/YYQ/icxbEeOw9KbWHXCyy7IbDYXf6VBANr24txwRe4FxU5iDFWhlMZcROy421HlLLrTDDjeoFXFw6UKUbWBTc8YAU3QH0ORY7keoeN2VO6Wg83pS8/pBSW1LI4uoXRexuv7Y5SahGkOxVQZjkg6V+FQEKZdcQ4hJUoHm/a+kW0hIKObnAFrmLd8TGkrWw4ysLSpO7JVob86UbY0IQFkki9+AbcE4cFFdSjOts01a3ANohxISVXCiFBoD5iAgq5v8A4wDtxgoLNeQejGbuqNZTl3IkusRntGt2QiZeNDbJ/aOl1KgB7Dkn0B5xeSfgt+KvpktVX6L/ABKUeZOca23WavQ/AKIH7gfZLoIJ/tNdwD+WoOhnR2L0dyimiGeifUZTviqhJQ1oQt21ghANzoSOBc39eL2xYrzzbKdS1WF7D7n/AE42TVGY93ZJEwsC+bWac8/4Urp4VCt5XrNWjfT4rL1PgVdAHbUA2N/0uQWr/livn/jl+JqgyjFzbmqpUN9J0qRV8uogOD7f1hhP54+siQt0ElG1Y29CSP8ARjjsOPJYXHlNB5t0WWh0a0qH3BuMZyY5PGwK7ORsV8uad8bfXGeEFPVRLwKhYsRoKri/2bOHxn4s+uz/ANOfag4k/urgxDc+37K33xvesdCeiteW49WukWTJrj37Rx+gxVLV+atF/wCeGBXwn/Dert0Uym2PZqnoQP5WxC+O9g2yW6x0j4tuucdtK5GenLD6lLp8Ye3f5f6frhmqnx2dXGfkMZ+ihfP0RIpcJT6aQkk+l7DG4o/wsfDxFWl1vo5lVSkm4LsBLv8AJV8TPL/TzIeVFpcyxkmhUhQSBqgU1lg8fdCQcN2jG8rpbFfPGldV/jn6n6TlKl5+ktuHSh5qmIp8bn1LzqWhbj90ng4sKg/Dr8cee20/066xpytEK/M2qtyqhIsbgnbZKGu3b5nrjdpSPXn88dAHbA7a3hACYFZZyl8APTOI83P6o5vzN1BlpVrUmoSzGiE+nyWjc/5zhxo3LGUMrZOpiKLlLL8CjwUdo8GOlhH5kJAufuecPWnjg4ClahfUgge+Ky+R2jjcILunSMcKR6jHQQoXBBGBXGBkFrKbbLmgff8AjjxFu2OnuMcULg274awUUVr+eUZZntwZ2WsxSo60BZnwacZTCOeyg0S4D/mYUwc95TqcdT0KuxVFAK1tOL23kgcm7S7LH8MSHT98FORo7ykreYbcLfKSpIJH5YmyOiyflfpJ1B6h5ik1+uPTqLEqcp+qLfdbKHFBw2DabezZsAfY4urK/QXpllyCiEcrw6mQpDinagyl67iTdKtCgUgg8p449LYsojBLbe2HPOtRUoqGo3t9h7D7YXJ0VhlJFl8j/wDCpVuJUeutJpceAYqqDlaMwtBbSC3vPPuAXSbdko49MYTD2t5TunUB5b98a1/wjGY2a58TWe3W3A4inKhUtA1XSCxEaJH57jquP44yQ0A0yCvj1JHpiwBwCzyC70jfGm6rAm3fBISNKQb6r2PHbCpxCnXLk2/IWth2i5WqcyN49mOpyKTysc2wEALpgfp+6lGoWC0cn+zgqJB8PUowP7q+eO3tizqb01rEmqJpExlEd51Lm1urAAWE3SCfQH+IvhMjIdQgGQuowXGHWAVWULffv6kD2J4ODZEhF5CnusVyNrCHI7aiggkhLZPrxzf8sS/OUqOzFnPMyHXY7jW+yXXCpLVjYLSB9djx73UMV7QTsTwsIaC0XWNweUff/wAcOmb5H9VSiPuAlfnWtQ8w9LDuLc98G+iI7oVbZleM2Z4xfKtsNi5+m3bEcPJxJK0ygBtfCtauSDfi2I4R5yB2wqgXtP54Hfatbgn19sDSjbbC3O3p+eCxqUrUrETI0OLdXdxZJPJKsbS+Bn4suiPw/wAGXRM59G5tUm1RYE2uQawd9xgG4b8K5pbIB9nBe3a+MUuODQEJTY+p98CjSVx3Q4g8j09MTdQd03C/RX0ozB8N/wAQmV5Fa6dyIlViLSY02Mp15qTE3E2KHWVKui4JF7aT5rE84+BHV3Icjpr1SzZ08eQ4F5ark6lfMIKihl5SUm/rdISb/fFj/DH8QmaOifUek53y3OUJUBwIfik/LnxCfmxXfcKHY/uLCFjth1+PeflHM/xJ1jqPkGc1Ly/nuDT8yQym24hbsZKH0OD9xxLzTupB5BOIRYpnAEXCoumQkIaCyLKJt3/liZUxnXtexNu9uMRqnhaktJKQVD0+2JrSUIQmOtRsdz8xbESsHeX2x+EHp/lKp/Cp02U7TQ+ZmW4S3y684UrcAUTqAULjUTx+Xti3GukmQmn3ZQy+2ZEhzcdc3nSoqIsQCVEhP+T24HtiCfBPFk0/4UOl0aYlaXRlyKuzncBVyn+RGLvHPNv44XK1yd5s4qFNdK8sNvIDTLqYyBfb3nCou3HzNRPsALW9B7YPT02y0hSkJZkKbccDroVLcupY7evtiXgWwFa0oSVrIAAuScTKEt1CV9JMiA6hRiHLEJJfUpQHsCbkD047jvfA5PSPJEyOI8+mLlJQLDeeKrD2t2t9rYlqBuOb5TbiyR6298GJUytepKgo9uDiWQUGidHcisQ2okqlrleHWtxtalqRpJvwA3YWANgLfzwJ3pLkRlhtrwEgRox1ob8W5pRwQD3ubXNr3xPMJpCd5vZaKSSoavWwvc4NkbqIsdMsux5MdbDc1TSEELSua4RfVqBse5uTftcHm+OL6V5UdWdcR8pW2pC0KlOEm5Jve59/0xMy6gLS2r6li4GB2SDewvgZQgoW10wyuyhLQ8YjQUqCRIIA03A9O3mPfvgaOm2UW0raU0+6hXyylyQVDmx7ena/88TLj6bYRGG006t6NGZDj7oceVpsVWFr3Hc2AGAQOiihzXSjJFQS0+afLQ224pxCPEOtgq5Gop49z+d74E90dyM9tBymOLDFtlK3NwMnUTdsLBCO5HFrDgWxOG76fPpvf93AlA2Om32xModqjdVu90j6fPSJ0l+lzIqwuzkkyFtIKNIFkHsE6eOB9r4WO9Jcpvo3mjNUpTQbBMolOn7C1h3vYC1/TE5S1pKlEkqPc/8AdgKUiO3dwlZHdVufzOFsEFXT/Q/Iim0b/wCKOJjnWECauwOq99AFib/b0sMNz/QfLjyUrpderUVpSVnwroaU1c+pugL7/wCXi2hpWjUi1jyCPXHEhQJuvUCb9rWwLZUbqqpHQTLs+l+CkVSpshaAFoQ42UXunt5b28vAv64IY+H+kttr0Zgkh1ate4mExwQCBYEG3dV7d7nFvFN/TASOQoLOntYDAyKXVSL+H+jLecd/pDOcUGdpjdjsks834WgJURa4sT684RSPhzjSJSnk5kQypVlENQLAq1Ekj5nkJBI47A8euLjUHC8nSlGhIurjk+1sDWgqTpaWUf5QsT/PAa1FUtA6BiGpK284tiUwGXHGRCCkNLQQUkALBtdAPNzcX+2D0dAacpZDeYGFq3AtZcp2vyWsGxddrc3tzzz3N8XB4dgqUtbLZUtIQtWkXI9j9ucB0sLJ0OeYXTcHke+CVFULnQFwsvsOZtYdQ85uKC6UD6HsN0BHc9rDnDc/8NrLaC7ErEF+S2FbZkQbpSvgi3KlgX7gH7i3bFzPxmakjw9RitOMqKk6XDcKsoW49b2vhQ4h5tg+GQl1y5IDrlgf1scGx5IqjGOgmZZC9cyuU5rfc/rLcVKwyAD5VJ4BLg4sbAD783E50BzNEdQaTmKnpH1kKbLTYWEAAhttNrcDv7fkBdcEzVJUqoNFDij+zS4FoSPsbAke98HOpU82W0KW0VGxUPKoAe2AWZkFRcboNmqG0EJqdNWlbqXnEreedBXYXJ1JsewsAkWAA54wJPRLN6VpkQ6nSNvyEMF96ymwAAnd0FYuO5SQTpR+WL4W2LAXJ9D98ecCUJvq0hPN8HZRZ+k/D7m1dNfgRq1CSHmm95DE59nedBuo3KF7Z4QAoC4F7WPOCIXQXPMOA23IrFFfkLIOtbjwUgEAFolCLOaRyHLJWST6AY0G2lxIJWoqUSOOwH5DBUx0sLZPG0tSkrUVW08Ej0+1vtfBaoqAifD/AJoZbM5LNC8aUlkBMt1JW0fKUrc2yTwLg+hJ4x5v4f8ANYqKZDsnLKE7amm2mEONqaHfyOFKjxc24HAF++NAx5USTHEhiYy8i9txKwRf88G+GZVZakhahyFEX598JlJ8KOZZyqfw/dRZUYpiVvLCVuoLLiXS9ocbKSLEJbAX6fUk91evOE03oZnmLNZehZdy1LSktJU43MCVeQEXGtkWB7rRzfjn1xf1NoNEywvRR6LEgx5Di1PLYbCLLUb3/Ikn8j+eHvQkcq/kOMTI9NmWZKz0dzm+dybl7xSXHiVqZeZW4ysm6HgCUoJB/eGi1r8EXwA9FM/NRoaGaVGdcLh8TIkVMOF3c0gqHNyLEkjvx5b406phlaCjQLd+Ba2PLYQ4BdIJHYkA2OC1papmWXnejvUObUTEn0OE7EkJU28V1IJW75r7gXq182QOBYaR5RzgxHRDO0ZpGzllIUGdAUKgypbaxfkEkDmyTbsLkW9cabLSUqTcn2H3/PBoSLWuPzGCM2yRZzR0Zz9V6c8iSI0CY4gB5TighSj3BSWroXb/ACha4B+2EU3oR1DcKizGpSE790oYqDiQhFxc8+vBP5k+wxpRLVgpIVcexGPBLiQlNis83UOLYRpcis3npF1LdjupiwI0N9oobCnqk1tOtEEEM7be4m3Js4bcgi3IDA90M6mJUvRlxmSoawh78SiWfbX9QWAhPskcegGNYrCrpO3weDzhunynmFpiQWN+U8NSUlQShCb/AFKPe35XPtguc4I3zLLTXR7qEnajKy/PpwJ+ZHiORlBKPqTyS6HEFQIIdHrwoW5dmujfUeTCeH9HIsYvur3XHnWQ8U6LaklJ0p57AggXPbGkotPMfW46rcef8zq9N7n2Hskeg+/vhUwywhxbyEDUv6lW74XO7ZSyzBUOnOaaUpDtSoEdiMUgFLdUYbSlxAt8wrVe2gGwAPpfnnDdIylXpL0WHTMqJ1p3C83S3WrSG0ea9kOm6ySQSLcEaLcg6xTHbbUrQLBw3NvfBikIcGlQCh98HM5DurIkfIfUhmpNTf6GVIpebLKnG2y3JZRyLcgovaw1psee3c4JkdN63OhVKjP5Gqxpb7pblRZdH3WHV8dkrSdYVZNze3HfGwA0lKNPoMNcFK51RnPSQ2tuI/tRbJtYbSdfPr5yofa2C1xv5od1fOXqF8GeWZ0NczLWR8y5NlLK9xdPp7r9PBB4JhLbJR6i7LiEW5te16Qzr8GvxC5OQmpRun1WzPTEJKg/RKc6Xw3fjciuAOoPF7N7l798fZwshR13IP2NsFCK0ysqbSAV2BVbnj74u98ljSmKMr4SRYmaOnGcYqqtlCvU+oxC407TplKkRZDjbjelwBtxCTyD3AI4742J8PHxHw6KrK7uYIMyKItSbZLi2SyY8N0Bl3fC7HjUV8Afsr83Bx9CsxZVy1nCmro2aqBT6xAe+qLPjNyGjx30LBHrjO+evgP6a1VxyodOKnOybMcCtUdI8dT137gsuq1tj7NOIH27Yd8wmHeGqjGZDotQpvbnvgp8IU2pCwCCOQfUYh2W6d1Fo+SqXQpkqhSqtAgsRHJylPqaecbSEl0t2CubXtr7+uGujZG6qpzKiuZt6vtVKngkLocLLrESEpBHqpxx1/UP7W5+mEujZU/IydVI1dm0ZlEh+RGLkZlTcFRaes4SAHA0RYoDfyydFzb7YZaPTdYemRqRVWFMK3m1iG7YNghSUBQGtZBP0AX4IJUCQNgtiwCfbBmANVCsT1JqHMEumzK64p1DaJTOmM6HX3ddyCECzRtzcWACbEYBBh02CuouyKg461LXuLMc+LDXYXtutlpZXZQbspsc+YdsbFmRZjb4qFPBK0A70fsH+1jf+0AOD+hw3TMp5NzOwXKll6nzEuqCl70YayR6K9bi/Y+vOBYI3WS2BJDSF/0hahSbuKKpTJBcKAQ2UFs2QgcpNr3I73JAWQ4MgOvyHcxIY1ylxo4VLcXvLsQdSVkcJWQSpw2sRcXxrGHlLL9Os3BpMWOjQG0hDKbISOwAtx2B/MYUN0Cmx3FSRGQ5KXcKkqQkPG/fzADAseiVOY5JAuPvhskUFmTV49ZdlyS5FCg0zrBaFxYnTbv98Q1rrNluO0H6wmawt22hhtjesb6SNTZIPn8t+Bf+OEyeu+WWGnplTp02LGQUpDx0qFyAdBAN9ViDYA3H5YmZjgnDHDUKzFIct5FgG/qL464VJTcAE4rimdc8rz23VuQ5zVlL2ghsO7iRzfj6SfY+4wfI63ZFjtpcclSwvSkhoxV6uf77X5tfEb5KZCrBQrUPMmx9sd8uK2PWrLr01bTdNqC4zJ/1TsiwVe1rFQIv6E/qBgU7rfliJFffRT6o6pg2KUsgDvwSq9gDY8/Y/a5zJcpVjKIt5vfBDU2NIW42w8hxbZ0rSlXKT98QKL1qyq4yVPbxdDYdIaKSkgi406ykqNrenfjvgLvW/ILCmnnnKggPuBoOKhqPfseObflgbohpVjXUftgDDKGG9tF7cnlRUf4nESgdWMj1GL46LVHyyCrzGE8OxsR9HvhxRnrK7khEVFVbLyyBtlKgRfkE3HAwdG7qZSpDa+PWGGleZaKww48/U4SdoedIkJJT9uMEwc3UaoOR240xpa5adxpKF6tQ/wBHHODnaNEMpTq9GStYeHDiEkJV7YTxJUcPOQ1PoElHKmy4Cop9FW72OFQd3BdkoVY2Nj2wwV6fBoU+PUtDRkSrRS0lgF19PJHzONKE8klXlAv62xNtQiNdFI/Lx98e8uGqjVSZPjrfnU9qIgKsgokh0KHv2FsCbqQmygzBWVstE7roRqQSP3Ab9+R74a4QsU6XGO4JbebXfbWk82JBvzgDjvdDS0qcCb6Cbfx9sS4QyuSgC2EtSnRqbAkVKY4G48Rpb7qyPpQgEqP8AcEy5z8dyO01BW8XyQoh1KQ0Ldzcgn9ATiK52qcl3KmaGpk2mNxBRptgh4qeaIYXq1+lhY84l02U7r4J9V84z+omfavmea5uSq/UpFUfUEfvvuqc/QAED8gMRGRDQF6VXFuORh8y5SpdUXGeZjOuqcaQhs83NkDvixmel8NDjUioObNlhzU44OLoBIPH0D+d/ti9UuzF11XGW8mCsout51pDd9ZQ0CVe6Bf1+54B9MJat1SlZRmP0HLjMZmIy84CAkO6r+pJHPoR+uHrqh1Ah0FpOXspxkNXihlxy91C/c3/AFA/8cUi4l55ZW4gqWTck35OKzuipTM6lV2XKRMVMeStFwkBZGkd7D2H+nDjQeouZ58+JTXaq+4h1YjBt0oPkXwbFfA798QZqI68VDaWn8kHDtlqnqarUByQnSgSG+CnVfkemBdTmrUqFEFMnlksvs384bVa9u3N+Pzw1ZmjLS5redQstAtq0g2Ha5ue/f8Aliws9Nz4C4MmQx4tooJIDmkOW7A29Be2IXUl/iBaCG0N602CVHVp+35D3wE+6rXMDamtpC9WsAqI+1sRhC0BV1JviXZ9uzPUymyU7TYsn19yf4Yh2IojHHS4fYe3tgF1H1xzAkmw4FzggXUXNP3wJNr2x69zjx4v/fhgFEojvFhxLiCTbDvUZz1RTCdeVqEYbSPsjk2wwoWQcWR0u6SZy6m0HO9byzBQ5DyJl1zMtTddJCUx23UJKUkAguEKWsA24Zc9sF1rJbFNdNjrTIRc3CwACewxMYEZYYaVrt80BXBt7EfniK0zSuI23blBtcffn+GJXT5SkQi0ldm1kKt+RxWnbuvuT8EVZ/G/hO6X1F1dyMvMRib+rRU0f+hi8kOocQFoN0q7H3xib4A+uWQ8vfCzRqPmbMLbUuh1GqQkQ22lLc2vFLcbshA7EOptf3xYNc+L5ctoLyTlGzJUoCTUnD2Btfab7ci3Kx/3xjDyTSWutMlQGC3FoCbq5He1uf4YxtUPil6mB1tX4zTIqVLIQhqnBZXwTptdRPAJ49sIf/ql+q1dZjqk1JiKyVpW5GEDw7riAsGzlyVt3AtYe498Wdi8JFtTQqQ2lStxsGxsDY/r/wB2D0pCRYCw9hhgydmBnMdEbqTFEqlLSv8AxFQjFl0fcA9x7H1w/wBu2k2GK1F5Rt6Y7YYLLXe61/nfBaVrS+GNwq4ve3bEUQltanUvAeZAI7dwfTB+OWGAqUopO3Yq9L4iiJfitSFtqc1nbVqACyBf7gd/1wck90g8j0wWhb2gFQbUonnSeLYIeniOV6o8lQHYoaKwfytgbKJQtI1agohQ44P9+BpVYC5uft64JYeU8krVHW1z9K7X/Pi+AubDr4HG8hFwSOwP3wLqJSgqKfN3+2OK/LvjqeEi/GEE2RUGnm2osJDqFJUtbq3tKU27DsTz7+mCVEehpxoOaFDzKunUSbXwVT5qag0t1EaSztuKatIZLZVY9wD6exwNUnSW0PgtqcAsRynX/Zv6/wDhg1TSDuFF0LIsVp74VRGDSj8jj11D7nCIzFsa/FISGgQA6OB+t/vhSgsqs6jQq44UD3GIovMKK0lSu97Ee1sGfT6YAXAFaSeSbDjHEKWSUKRaxsLm9/viJkNZQbak3J9METCdo6BZwg6VgX0GxscAWJrbi1NlLjdrBJHnB49b8jvhG3KZcqbUZ2os7xaLiYq2glz6raxc347fwxFE4tjdSla0C4sRq9DbA3FaUE3At9r8Y6lJuTcn2+2EUt47rbLLqd8nUG1LKQtINjfg4HhCnNEQ67ClOMttNzQqSCpBcjOJSO/lJIsk8djzhdKWtDK3mm9akoNgB3+2E7jkIPR2Zi44ecuWQrubcHSD9iP44OdeZjo27p3CCUN3AKvsMIdAiAgQ5niYDEl6K7HLrYcLTo8yD3IP3wc4+yhjxDqtKBySoEYChToe+Zp27etrk++BuOIQoJC0BSwbXPJ/IYPqghtu60bluD2wiIdlS0i6PDMXJP8AbcvYAfYC9/ufthaka0XNxfkgjn8sJXm4bTSnn3tpAOpaw7tjj1J4wbKI/YY0lJZRZfcEd/T+7A0qsdtLdkgCx9Pyw2zIst1BUxNeSEONuJ0OhNx6g2SeLfxwEh+mKU94moS0yJCEhpTQWGdZtxpAIbF+Sb2AwyiWTW1yYxjpaTdwgKCuRp9b/px+uFDu6bbakg35uL8YQ+JKp6WnoDza20EodK0aF3NiB5r34B5GFgW+Ry2lJvblV+MLlUsjCAoEE98AU4EaUkKOo2va9vzwUGnwslT1wSSlKU2FrcA48GilJBQCFElRKz3/AFwHDoogT2TJjuMIdcaLif2qO6T6H7/lgtuPL8QXZL6FNI0qb0XSsm1laubEE8jtbHKf+LPR71WPFYc1GyIzynE6PS5KU89/S2FoCAbJWPy74ljuogvPBlsuLSuw9Qi/8hzhNTKvT6ql406a1I2FlpzbXfQsdwR3B/PC61wfbCKoN6WHXo4bEjSdC7cg27/ew9MMd7qI+TLZiMl55WhA9f5C3uT6DCSnxXlvu1WYjbdeAQ22Ty00Dwn8ybk/mB6YMaVDqCGHgveDRC0FSSPNY82PrYnCpp5l1HyjcD7EDA0KFkYe+n3wW6g6ToVb9MJpjyG20OrV5N1DZBNrkqA4/X+OFSQbWUq5557YDgDopchEqdW2Dutki31JOBNy21DkKQbXsoc4NVdKSdJNh27k44Eck3IHtYcYXI4I5gUIeZIKTe/N8NThmU1KQ0wp1hxzQQBdTF/3j/bTfv683w7JNxexHpzjxCVevfD5ULothaFtIWysKRbgg3/ngagSfKbYQNUaJEdkPwEGK5LXuOqaPCnPVZSfLc+ptc4MVJLawxNSAlyyQ6n6SfY+xwbZgpdLBfspOPY4lKQAEqPAtxjvII83P3wGtQXrc6cdNxcntgBuCpaUAqt798c3VWF21C/tzi1TdGIUlQ8vbAsJXZ0SNpS++21rUEJ3DpuT6DByXNSrJsU+4OEvZReWlak2bUAfc84RyIAU8Jkc7UlAtrHZY9Aoeo/mPTC+9xcY4NRHm4w11E2xpqH1qp81nZklFygG6XB2ulXr/eML20IQnSj6fzvgqSllDSnHG9xKBr02vyPYe+EhRV1vMOMrZbY2yXmXklbgWbWsoEWtzfv6WwO8oslLqiqHGmCWxX5E50upklmOyB8xfO4oKAb9jpBN79sGRKZKlzGnYqIzKGgVKQ1IsuDwVD6Adfk/6RQQO+AqnuuGDGnxnA21J22JyU3SG0W7NgHlK7EuKJB9b+jnIrLsPcaRUJ8NMgFEZEpxD7kc2syrWhJCyl7dctoI9LkA4ytb3cy1FyQzA+pL9MjLYnyI8gN6GVFppCFoKSBybLDgbs4exVz6YXNzFS2VVOTPXHdjOWCWHkuMpAs2QlwJC3DYk8WP27YIgLrbZW9WKq7Uqk+r+vSNaC4lekjWW2gnbWkWBTosPU+uDarEbedVFe2w3FjgR3Ug2kOjSQgaNVnLFN7HkHz3xAkLtU0VN6oQ56YlNjy22IallM9xhS3HAsWCSrRrKO3BsOeSeMOcuUifChImSmJD7qbpDllOsaFkEWtthBJ40n92xA5winMLRUkNwwhp5gf1iQqC4+stgeb9/Q3yeLhFioXNsK3HoLsl2ZQ6hG/C0FcJaXZbbqvOCHElsWRxyOCVg/vc8NnU1XXaexUGlTqimJLiLJWiA9bU6ixSpakix8ukHzcciw9cdkuwShSaJlRDLe2FIccb8zdhYpCdxRA7cn724scAgrVGSZMOEuoSkDySFSCltpjkbhavZvvwD6emHsMz0RURZbaHfEIcQtiLJQGiQQdR2zr9SLKsR7G+A7uqd5Fx/GsU5h+NTpaH3iC9/WkssqPdSwdQWRbTwRe9xawvjifnPNJ8G+CttanSpu4SgjlQN/mI4v2t/PDcXERnm6etQmNxgRZqKGGmjrNkJU4rz+vmHqffD03LYkPlclLj8RZForzi7Hg2CtHcDufS47HCqxKmXIENppmDGYhSo7O4pTEbaju3BN1cEkkextyO2Axa7GeZlJjzWwlYQ674YAJ0dtCdHBHPcEc37js31VqC0EofTDffC7sy2mXC4F2sNtRSSBY/Tfgj9cEOvvNtSksr8EFgbm0SqzaOTxq4J5N+4BJxEt08/wBKaxAfeapsqTGiODQ6pDrguvUQk7gURe3Ngb+lvXCp+v5ikWlPViQuRFShlL7KlLOoi2myBda1jtf+GIZNlLpj27Mkbiw84z2c2kkpJBu2D3HG4f7Q++HF7wa3n39BabQlbSoyQF6SQLC7YKHF6yD5rm/HpxFNW6o0SazMlCO83PdXITurXJb1JbI7XF7k257DtfCxFerUveajPPqS1+2ejultIISCbpsATa3A54OEJchyhLb1VOMh1Y0bja22krQm41m17W7k+w4GCUodbhrRFgIQ5HIeiIbd3hINhqcKbgoAIHGg2t5wcPok7yVomT4giNwZLkUKSHXXVt6lKHKnQEtqsbnnjt68YL/Fqu3NMSFUnG5rqLFa3QwnWs3CFAEci32ABHtfBMyshdMbqUVMyKzG0SbiV/Vw4tO0bApBOoXAAvaxwTOkxQE1d9ceGVvaA3HZ3UFZQQl1xXNuCq5FvsMC/RMnGRO3QiOuryyoNodC1PElOvhtKlAEawdV+fv6XxXHxJ1qpZe6DdQ9ktRZDFBkxAlkABRcBaBIWeRZR7AEG3POLDlpaZekIpTQlDe3pEpltDalgaRdThFwDxds3+2ID1woVYz50bzdkOgQpL0qdSnI6WFsXddcYG84W13B7s2Nxfi9ucFt8wSm+VfNvLklqjPMIiuNJ2AdDrqiNRAAFvv/AOOLXSqFUcvl1m8u6yHXC4CL8jQO5N7jnjGel1NapIl6NO6ArSoWHI9BiwcoZmqsGmNSPHNhTqFtss69ISgE8mwv+d+TjeqM2ZRbNHR/M6X1VF2jvi616zfhICja/wDD74Ym+l+ZdJeco8hoBe3dxo+Zdxx/P8sXE51EzM8pFNdr0gIJulDUdrS2D3IHJAH3N7Yf4+e/w6mpXLQ/NVvESNalkOC3AJJ/Pgf/AECyKosdF85lLmulSAWuVo49r+n2xKqb8Ptcjx0VB55ClKW2QEOpta47H1vzzbFqS+pEllxhlVgw62i7iiEuqHv2t+ZVxxxiAZi6jz5Tq2o8iO1G3i4AptKne5VYKAF+efQXPa1sLkSJv6qMvspkojJaS7DcLMkMOXCQTYgi/rYXHvfFbQ5mmzT/AJFjvzbj2FsSbM041SVMkyWluuz9Egq0J5JQCCR79sRKiMOJmvsvRm1OE6VpWf2fvb+fOBysnUa6mOLXXRfR5WkNgpVe4A74h9rovxxiY9TEMorTIZaDadkAhJuOD/LEPQ2pw6U4CiBjoIx0psdKuLYCRbEUQ06Sf+/AiL/vYAgX9L4dqTRp1amNwIEbdfcIsL20j3J9Bh9SlKTU6mzKjLZgwYzkh99YbbbbF1LJ9BjX/SDPmculPTeT0zyxWosCDWnVyK2lEFpZqThb2w2864LraS3cIava5JHJxWWSMqUzK0F1wNF6oyAG3JLhtYD60N27A8XvycPzUtaH77rbba0hG255tR5udNue/wBsWtiFtUuZVDDjqpcqVTXEXEZ5cbkd9BtftxwBh8ivBxDbKk6eD/8AThL1BQqHm16QtvSZrLb4AQG0hQ8qvKOAOAf1OFNFR4iUh1SSFtobJSjjjuOf5/nfFBblJCdrla3SbN+Yadl6bS6RWKnFYZmeIcaYeLYCyiwUrkceUD8wOMTVXUbOrkqPIk5lmOvRta2nH1akglFr6dNiSOLqBtyRbFPZKV4CoT4pe/bAOgKbC7kK9L9vq7++Jp8pPnZedU6u/CiLJRxxx3I5v6Y1RDuqt51U9Y6oZ9jMuRv6TzVGM0UbhZZU+ef3vJf2H6+3IE/1RzwkLQvMMtIJIG2G2ie/f5d0H7HtxiCKUnS0JLy1vAEIC3SUoAIsi459+Qeyjfm2DES7qQ+zIWv98o3SOe1yPX2xakWhchfGX126fLZjJzWipwIwDf4ZVoiXUJHfhYCXAebX1kfbGouj3xeZy6xQaq4KdS6LKpwDmhgh5oo7D9oQs3JSLgce3BOPmst99ThdTHLbaSXRvOWSbd73sAASbYmGR+qWa+nVUfqOX5EZpEhOy8y+2HUu3HBsedd+b+n5YzyRAjubqwSr6if+kzODzb0WRNjIkHTtIj2K1NE/teQSj1HIN+LDm2Ao6vZlYS1Ikyqe20pwNHTH1LJXq2ypAN+Sgg2sBbm3Nsh5K+LHJNWKIWdssT6VILQdkT4chcsLfAHOklLiAF6SDdYB5seRi08m9ScjZviNUrLfUWnF4Nl5LDcotvXAU4EJK20nWdRFhxdR9bAc97JmbrS17HK6pXWPM+le8Y0ZhoNtOOsshzStafKs+Y9yD5UgkfocFwesuZn3IjcctyA8hshUhgNodJRym45vdaTe1rJJ5GILIp8aM23MeWptbzYWxHffJbTwSoOtlJWhHlFmxY3trPGFb0BMWMirRJNIAjbYWA26yL+ZJ27jgJFgADybHtxhc1xumPopy31mr4W8/wCFjqQAhTLKGCd5GojUlRKdKOLkr5tYgWN8dk9Vc5KTvMijtt+Yrd86klu1w4lsgGxBBNyNI73JGK4cXApnhGKpWhTRMWNsSm3HQ4Oymy75UIXcXIuDa1iBxjklDMJWhmYHVrguNuFt5wJKwQUhO3qKC4T6agUWPBPC5nIZFOo3XKvTm57yKc5AaiqUhvxtLcS8+u2oFDdwbAAHkc6084NpnVjMKtpa6rHkM/W4FtoGmzhDg19jZHmv6EEefgYr+qzJLTscLpywXbyUc6hwmykFPZFvmcWHPn7C4LzDHiVGU3UqZFL8hw3adiNOR1FgABLNgbI2yQefRNyDqvg5lLKwV9c8wqltxqdEhPoWeHXkHQQQbWUCNRuO3F7/AHwnPXjNwYkyJNLpscRWVvAp1r3gPdPBb/O57WtcYrmbEqD6pf4VBanKQ8gSVMOam7IKtxbjrYICE2b+WLDk/WQMGMuTJdRCKlIPi3f/AFiY8dxwpQhtxRbsrkuFwEWTxqvY8DBzOQ7qsIdba9FWlbVFpwemFs+JaRcO3UQEKuoEKtfsCPW/cYEjrDn5uY1ElQ4zCt8srbREWfOCPIs6isEg8lIIBHfvirorrxkPIjxDFhtraMenFndU0+tYu0FXuXEocKgkkDykcHDhTQ7TG0rn7iltuBpanBvao4NiCTfbHIIAWB21gkcrmKlwrHR12rsqaWpVChsQgUKbcbmbT7tzosA6NBs4UXsVEA+/Z1b63zUutol5cQ2pwqGyHwXEEBZ0JvYLNkEnkWvY2xS1YenQ7opMeZPknc1rU23qZYAGknRcrJF+eO/Avzg6m1h91pa6kuTIEhxCWozxbSErupJCtpI1oVb6boI4vc84tagDZWq/8QslqXGZfysGUrc0XflKQF8HzApSbX7ALA9Teww5HroHJ0ZuPRWvDyk6wXJXnSQrSTYC1j5bXI5PNrg4px1+pRoTkYnQiWELLAfHzmlnykEXLax3FwD8zgjSUFG/EYgpVOjvkFThc0qjWQlxtxKnNxQcDZ79k8EWJtcHEzBNoruT11UHmGncvpKnbN7YkKQ4XC4E6AlbYOocm3tY354AevGXjNYclUKQpxCD4Z1l0KSdfBGtYSEci35i2Kookpwyml/i79RWHgUuKaWXSi9/KeRwO6PVGm/vhFHmpjxduJIcZSyUQQlsuIVKctYNttD6A2FX5Kf39HcDCZnKFoIV61LrY1EU63Gy5LWptkL3FPNlBcvYtgIJX341EAcE9hhJ/wCnlouKWvLa0x0J1bhlt2Ub82vb6fX3vxijapT2E0ht2E2Y7bTW06w2UAqfKruWHC0NgeTlBPGsnm4FGMaYVSYMmRVmo6NspZbDbpQUG4HBNib83UDYm/JwC56JaFcS+ttJZcNZaoUVmQ+4IzrjkjzaE3LabWCiSLngW5TbXwcBd6602XFS/Iy+8pJeQ41ETJAfGmygSeEG/sDcWIN+2KoZ/EHWIi6pTVhnQsBtbzbhbWFuAjTt+QBxshFyFkAlB9kLbNQY8QgT34DbZWPDKkFLqrj6NQ1c3t5iu4HqFm4ZpSuarpd+IRhyow48LLD5bW4WZgekaXWiUXQUgJKFgk2ve3F+3OHeR1py7LpjiokeoNTlNeSNLZ8Optwq08rJsRf1TcH0OKDdjvohMyo7riZc8NTGGhAQHk8WSNzi4JU5x78dxcBUt5bMeTR4TiGgFx5C2nFKDriyQ88lSEnhIsCWzZOoI5uRguRIV6U7qxRJSFTsx1mXHgvtICGUQVpLS7EEqcbJJ1EGwsLW7nDtP6idNxFjt1WpPOtRHAkpfYctcJvrcCxZYHf15F+44oRlxynPKZLr7bi0OPKW822y3cKuqzpIGhu97BZIsOE+YY7UpkdqF42XU3HJkhoOuMxUEOJQ2Lm+2VazZSSePQ8DvhMzuSmUc1oN/q3lEjRTJEme6dBDLLNlEHkka7dgCbYWSuqGRIPnm5lhsEgfJWle7zex02v6H+HfGZJ8pU2pyYyJOhoPAuCM62lpbSGuCbpS4twBTg1AAc835ODg7OnOx9MazL+jabeU46AsgkpSNSSgpQkDy+3sAQ3afeQcFotnqlkGY80tNbbCkLsEPtqbKePrFxyLEevYjB0fqbkmpspfZzI2ztuHTrBSV2F/Y8Ec+9sZglKRTJDqmYkSPKvqbfSAGVBCSTuNn98BJsEgG59Tzg91iZTXkrmqfYfdcXd1DLZAWtV2nG0kD0cBGpANx5+bEFr7KZWrUUDqFlSa6ptGZKSrZaDrxRLvt/xA4se5wsXnPKRDbyM10dKD6mY15he3Hm9xjIrqaiqXCjPVVxoRNyS9GnguDdAukgBINwColfJFiDckYeZ0yDFXMk04x4xp8bdfZbDZblPkAjS5e7ZSbkt3F7G/e5NwhZq0rD6j5KnTRT4eYIrriuL6iElXawJHJ47DthwrWZaRSEgS5LS1OXAQHm0kD1J1qAAHvjKzjTJVCqDEl1+QgR0qD0ZrWLoUbuHVZersUpA+om4svCSamlykBFQZjuQUOOne2nQ4k6xpttqUAs2JBTrNr/v3wmfMplC1nErlBmP+GYr0B98BBDLctCikEeXgG/P87Yc0ONKspLoAF0kBQsD7fmMY41Sn4b8yBFj1HXJbZYebp7boe2+AAqwcG2Qm+pAPcErAOCa1Uqi1GbnutvrZkxW3Hg2gNy90qs4G2kKDYGpweW5JGvQPKTh23U0W0DpcTqbV3+k9x+eG6DOgqmSIiZC/FN2Lzbi1i1zwQlR7H7cYyPSc01ukJmNLhybvruttTiRZawLFRQoEDQR8ogA+Qk3FsLd2fKl+PiNSHvAOLS4lMVbu42LBVrAr7G67k3uD5LYPezKZQtZTZNNLghypTCVL82hbiQfzsTft6jDZVs1UvLrlOiVGalT9VkFiC0jlT5tfgnjgclRIH64zFBl0yl1DVSmnJrSFXS68DHdCCNIKhYALSSDxdHFtXNw1uyl1uU+zR59QqManFZehyXlvarqAJSGwoNrvcngghuwCe+C5TKFsNFQd1NIXTpSd5VgdIUEcXusg8dv5jCsOuBBu0QR6Dm+MjwcxZvai7Jr1Spy2nXAWXjIbLxWOyig3On/3ZTpvcg2Iw3Qq7mZU9mrwa9Ilh3mMp4SnHE/TcNKJOsgFFiOCLmxwGucoGBa7cqrjb+2KTNLWgqMhQbQ2k+x1qB/lbClch4MIdEB4rNrtAp1Jv9724+xxlOoZpq1ZioiVWsyYqJKAw2DLefa3DySpK1E+gsAjX5uQOBgH/pAmUGa6abmGqMPGzBbM9bgWg/4wFwqF+DbSB6g8g4OZHItYOPONqSlDLiys3UU2Nh/HBTdRYcd8I82409a4S42QD+R7H+OM1SOoOZczInOQH66kF4Jix1yHGVpbbA3CeQdaQQTzbzXvyLAk9U89JbCnqtIjNMuoSyS+pViU3SVqAOsEC/JPH9u9gdUMi0o7ISXXIsRaUvuNlaFFVxrHFv8AvwubdSpF9d7cE/f1xmZzqTniPJbV+NzJyH1OIUhlLrIUbXAGscL9rEC6SOecEsZ5zvCqCXGMy1BmPIS4uWklC0NuDUnWHFqVwNI1hINifXjAzHdTIFpWm1IVOGmczGkMhdxsyWi24kg2sQe2FTjhAJCex9Tbj1xmNfUbOMZ5Xhc0Tag8hag85HbcUGkIubaDfkaeSUWtf3JCmo9Ts3VtyTSp9cfipjsA6GIgj+LXosohZUDtHlQSQCbgffBvohYLSRUy8gLCkrSexBuP0wip09FQD+uBLhrZeLZTIQEFVuy0EEgpI9Qf54zXIz1mhmls0OFVqhuojOSWm4iPDHRYLDzimgNDYBN2+DwB+9cmMdQ8/ux47y69W1I0qZjlsNFLqxxdLgKtwjkXUASQRpvzgqObZabd8RdJZWLC+pJ/e9ufTBhUrTfVz2sMZSrXUnMGYMuUbxtVLq4SDIeQbqkOyE6hZGhIue1ibaSDcHDvROpGcINGlphV6XIAbO2XW9dlhSVOBJc5TpQ6gdiDe4PFiuWymULSJfSuQWQ+3qSm5R2Vz2P8jgph5l195Ldt9sISu1ykHkgX9cZwa6ndRGZynZlWcTMjNLQ4pURpSQQAS0ohvmxJ7WXewt3wFjqBm1yoR6j/AEuqK3Ke4XpTCnEtI2zYpDjZFlpIc1cH2HFsG6ORRJqeY0FxqQwh+RJcQtlxTnKUICk/LdSsEA321m1lAXtxhe3MrsmGveY8Gp9TcQR4zqnm1aL3DzhVYLtyA2gC/rzjH3TPrtnPozVV5S6lw6qmJHG1sTQQ/DFvrbWQUONi97XIOrvY2Op6bVsv52y8ir5aqtOdgSme2lPDZ0pugLIuTYj/ACe9gMVSRmPzCsDg9PyqnVWn0JgSHy3ZxuIoJ3W5ju4kpHOnQCi/v9I4Pqa9BrdRWhmqPSG2HdaAlkmOr6AOUg6Nz/3hBJCvq9iVKZSQmV40RnG9Fw8bC/JQoE2X50jtfgc3vwdUHYImN1IJQwppvddQtwqKn72cKVcj0SSCRweDyL1olFNyaLBkOUQ0xp9LehJLa3Ck/LOpCvLa+jy/xHJN8FuUtWtpbLTEplp9yONaNIaCF3SLaASRqJBUONI7Wwt1TFtOxnao4+olx8Rw+0IrCFgAKSBb0vot24ufTBNSqLUgGZU1ArmrLbTKXLhhfoS1fz30gA9ue9+MRBcak1ioPuUp+cttbbtwmO2lwqXfVcuAH0J450+2HCHTFONLkuCNGdYBbBLtlAa+AXARYea/a59ALYaYL4Vqjux3DIeaJaS24lsiyCVi3AC7jkHnvY3wWqdOmJCdqOYy5IdS0twbhQOEuK3CQi3bzG/Pp3wXHMjpbvJxrExujPll2KiRvJK0KANnSsDS7qKgVk8c9iceZVCnqU2/+IMtQ7GWV6GmuLHQAD+0UeLggEDvhskVWQ2ylanoymU60sraBUbbZJDpsUAi9rdkgnlRwkeirVELVSjNTER23Ahkr1pb7EBLiOCbd7cW55NhgKd3Mnd6ezCjutMoXIllxCo0hpPYH0U4s8Cx9Aeb2sMHN1ZxTLbyamsF4BmSkMOKSm5sq5IsQQe3Y+h74STZEtDaYzW69JfAULNoKmT2BBPe1xye1ybC18Ftw5lLQ/TKlJdW86PEHRLL/KBa52wRzzchfp2wNkN05VM7r01DyW2Uq2njIW0UW9RdzkIuBax5Pb74IlSkuWjUGy4jgW4hpo/KsO9tFhuAEDzLtYjj3BUZFYnJjVBiE66l9pbS3JN27AcFsggruLgA+gPpxgDaWXFsu+FfU00C2lW0g6QQNoLAI0ALubn3HGCjdEw34MRwOtQFvSZKlxw4HhqS6Fq8jYF0bdrHTaxN+cLI1RpL7rEt2K5paU6xJGhTrbnfgWtZHfseecJ4sh15mRIjrbZaQ6plp1Dlg24b+VJNgQee4t8wc4VPPxnac5IbTTm32NstveH21OAWAZsB35JseL3PN+YmckypTjSfEx6Y8WkNONAPtOWdufsBaxINybAemD486npqTsx9kuSwlp50SFpU0dZF1KsOUd7JFjza+CBV4GyfGQvGaHA5GQ24CH1gABCibWQBc2A1+hHaxrVRadYbkok+JfntPuJeZZaecZQgpKgW0bh54FwRwbnlJxL+SrCVV+c2/oabVH2EhxA0OOEyPJqLilcLDg5IF+LWt64OmeEpcVhqKAtExJcbC3N0tIC7OIO53FwOR6XHN8J0OU+aW5kyNpiS44aW63IUVAkklFlkgjWOALXH3uACZLo0B5CEsCW1LbWyEpuEkcDbuQpACfYk3J47AYGZNoF8kOp9I/or1NzNlgstITTq9MZZSkApS0JCtr1PkKCk9zhml1NymydCJFtj5epAsVW9RjSHxwdLpOXuq0DPCYu3T80wo6n1tskIZqDSC2pKuAi7jbSFXFhcKB5BxmCoxnnnFr+myza57jG5hzN0WR3dTxTs3mItElQbB7pufN9/y7nC8ZuXMUX3vMBbyFwgDv2/S1sV88paFC3Nxb74KdmlIQnVcDtY4a6l1ZVRzM1LbEt2QVPm1mbmyQBzbnt/5viKmtPS31HsD5T8ztfEXcqLgtwq/qCr7duMdEy6NKiQb86fQYBehnU4rctbM1pLyLKMSOXAFgnXtjn29sF5eMF2sNrcYcKTwRqCQUepJ9P05OI43LEiHCssANMhoAciwJ4GJBlYNtz2HZNlpQoEoIJ1cj2wE26i3VyI5EzAmO6LFoKbuUFJsLWuPe1uPvhkoFIU/T5EzsbWBH7uJr18gON53ZDgCfGth8JBA0Fdr+UfR6G3tbCadC/CMuN02lhsvPmzg2ybpPcm3Ppg2Uuq2dQhLqkles++Cggk2Bw8P0kQEFcttBc/sJcBt9jbscIlpssaEfYdzbD2Qa66FBh+MktxGwVKcWBx3xdmVaJGocIoZZQgoAKnErup3v5z2se3H3xSMeoyqe+Hob224OQoAYf2OpeammER/ERlpQALrjIJP58c4NwCpZXWmU625aM4+dfnKlNjueADzz6nmwwJozErD6UL5OpwKJDdrcEji454GKVHVDOJStKqimzl7hLKE8H04Hb7YSN52zJ4sTV1N1bgIsXFlY9gBc8D8vbFna3SuaSrE6mxUPRIk5trQtt7b45CEOC9rkc2sD+uHLLVFecpLtfS0RGcf8Kzc3Ki2lOo+9ucQRGa52YAqJIkuOJKkL0LII1C/Y2v9ub4s2oZrNPbptJistR49PpkaK3p7l3zOOuH7lxxX5ADFL7O2Ra1JaE6qNXAtDzjRkMuNEtuFJsRyLj3AtiU3Q83ZbiFtg3IF7W7DsQvt6YgUapINSiSWgFI3UHQTbi9jz6YnDjclchCGo6yptZbHnJt9zb0HHNrY0Q+GyR7e8nBuWhpwNlsHQq+lRG0o6L6L3+uxHuRq7XwdGekJeXM31sqS7pS48QLuae4R+/9+LfriX9JelGYeptVVTqQgM0iG625PmFWllsC1m0XtuOc2sORquSAQcavkfDJ0TrcMvxMpbEtcdbxap9QfSGmwL/NcB0X+lJ7Ad7qwslQ2NwCgiLxdYha8SpTTSm1tKHZwKNjza9rXP6c4IQ2UueRaF67peJWSNY9geLEDt3xsGd8H+RFuuogzcxrS0stmSipt2SSVXG2W1LBuk+Xk6FI5BvYmb8GuRH4jSaZnbNbypjh8M0h1h0did6y2fOgBPnJXzq+3M96i6o9mVlBb60FKi426u/KU9gi3BI7+9sOFGoObs1PvNZZoU+e40A8tyK3cNC4H7QkNt888rHOLhqnwZZ5jTXUUfO2VJUVBLTbj4kJkJtzfS0koIBPAKxccgWvjS/TLIOSeimUHoOXodRkS9ZlVWfMCyptaLaNoApRxqWEN8ng8k67LLUtDe7qmZG4lRLodkfqrRKS4/1FzDJehy2mm26W6puWlAQ5uHdklNyQsCzYKxrHtwbDqlNYrVTcq9PlTYpZhCAStoKLi1uHUAbaLqIDgA78mwFsLoFVfdltxotK3pThWpsts+ZLi27baQggEW1A8A83vzhRSp6XZKkPuyZcN0NyZTMhTIcS0h0c2AsUDQQS2CQE8jRjCXh5uVp7yVx4cJ+MG5tSD/iIbksPz2EKCrN2ceuOeUN2ABCAXDe9ycNsKoxprrT0+RIqMdh0O7URpZJtpU5vquSHNB4Da+wsbBRs4/h+5SDTplaXKTIlOOTlJna3SwHdwungaABY6RwkNkWHbCOFPiMlulMkTYgeW3HfUsOt2BUCovM97Wv8tFrkX5JOKi7kiEepT25MpTzK4sxiKhndiOFCWkLRdt2/Ot4hRSU2sUBwC4vhAzIDlQkyW57E5/xERp1ynCM0GzpvrcvygANWsAL3LZuTYLG1SkV2O1GfS2puOY7TSlFLTbG5favco0FBPIJPvY8YYqk0qDIhxlLkVFAkPOGNCpDcZDLji0hoPqWm9rK1bYRYgII7kCBQpfKgQ6eskLjXmguvgQEsBLZa8pU24SgLHPzfPwDciwAZ9bznipMKeErXF0txWkNu6SdIEdTmoEkhKb8XF0W4JGHKQIlLq7zNRXLkFgrZadMTcVIWAASWtVggEHXwdVhxzbCGLDorUxNNRBckykMbba5jbbZ2kWsTwdvzgkKuCbm5PFneUuVKoFQm1BtFPfZbclx5C21yBrSJqwVBSSo8ldlGwsNQFuDhNM8NEjTUV+THiNokb0Rl6WBpDYSUmxVyU+cW45TovfjDkWFUyeZaIrgiIZcLre8y40HQ5YoKUG9g4LaUgjg37Xw2mkwZ82CsOQ3m20uB9gstNIcXYCySgE6Sg89lk+Sx0rXhcubUJkto82pJqUaPSo0hlxqT4uPJbCRuBYCUpU0Loca3Cko1OoJXo55OPJbQ7NaYsJCOHGlvNu6degkbSW1NrIUlKbWXtkAk8EDChsOQXlPrn7ccILoU3KDgsizZ3Bp5uS0kjz3IbtyTY4SIL9Gkzaw7LDfhUBe86llLhRutglS1G58xOlV7aQTfizeaTyRf4cKW5KpZRsRmHdzxaIvy1Nd0qSjQChe4kcW4ukXsAVJHBOiS2HFMR1vNsXEoN/PdXx5QCU6163ALpK9FrklBwsgy4rMtZZrDTglBJR4dCbuOBvaSDulWsjg899RAOsnCeXOc/C47CkBxIItswtKXFoNlEk2BR3bWpIFzYj6ReNKNspSOn06TDDjXjJDzbZs/LfWEtNOctqSoi6EHWnRqNrDUePIcAgMOMxHIzIdebko23G5EZsiLdGpLilIUkhBBJBHchQPJuVEOqx5S5E9cYmaxoQlcUbLYDhJO7t9yS4hNv3gAtYsQcAQzS3kKjvM0mRcrNlzVrLoc8wcbT5SsA7l9SCbqN7XQSqiOiU7x8B1VUguh5CwI6AhCUvOO2DhKkfQANsDkHsFgg8IqVSqjVJaXmKogBCQ4uGlvSWQi40lWuzmkJbsQfUtiwGFLTL0iC27QEOSPCI2jtPNtqb3GyEtFrSbABSTqTYftPUC4EsuokM1d+bUZMSM2+lxK2UOsNOAJAABFwb2Gomw1XsAcWNddRxSp+IxPgTINQktpho0IZUpnYW8UKOlwFpe4ORq0hYQNRA0XOCH5EfLjK6h49aZJvJckRIpLQ3Fm7qVN2CCRYEcgLJuFdyRLmQoKn47iKewt2P4NTspXylrcaDjiQdVnF61OJ54A27WvfCsyhAjvSUyZaahIZLba2mltx3HR5bONEFoWaQRt/Xysjg8Kd0Bsj5lJZnOqTMW4tDofdfYVN2ni6A2OLJAcIskkflwBqujY251LSl52S8/EcQ41OdlNpe20I0tN6Tca1XCTbsgaydd7ojUHWKZHSoNstLZcUEMpKWWFhYBUSOQuxsCSQdKSAbYOahMUiqRJD7bc1O3rKd3W6y0XAppwkeQ8XN1AArbHB4wfJBDlwkIalLFOcU8sFUWO04VKaWEE2DfLnfgA3ugE3JPBjUfxEeT+G019CkPH+quhDm2tDaUuOttgAhy9+yOW3DwbYKlOtiAxSUIahizUZsPt+I1Ealbp22wEBR08AgC6jYcgoabmKE+uGmZObcmbQXFf/D0tusyAmzYUFt3JVqKbni3Nze2FacqayWOTYE+S6tENipNP/KakiE44XkI4Dm4gBYQLGyTYdgD5celz40mRGSayUOPLDkZBYcLaTYEKSXFXWoIQFcWte9gLYOTO8AksPT3Ym2+XWVaC2HnW7hITzZAupIt6aiLX4KjxrLLMoRJ8xLc4lL61XCSXFEuIbdQpKFglPckDWRcdhg+JL9pJ3KY5MqaHhSHJLLoLUdfiURg84DYnncI/dIBDZJHA4wbWI5zHLWrYadcDLEeKS/8A1hkXsNxzTYEqU4LKAF7ea9gGakim1jTUvxBoWjsN6VLZcjyy2FWDirm9ri5PnJ02SAbgc0Q5S5a1qm0+LLaaW4plaGm3RYanW+xIS4oHUk8X5C72wGtRzJY4tLEMvMtR0CQ2iWSzuMhjWCUuCxOhYsoanFgjkruDyJhqNGU27JW3FeacLLhZe1tNWa8rbrgAJVoJ5voPccDAWktIZbkUtmY+1C3Iq7FuXHLi20hxW5pCyso5WbWAN+fRA4/HlqhZgmKfqMxtSyNxTqTFDdttwFGlsXIcIKRoKCqwv3s7qCVSo64DaHocNbLl5Cw6qKVrsWxpWpCypCFq8wuf3L2B7YBIqsajQn6hLp8iNGjtiG4pa2lR3lumyQ2WyQ03rTwOCbOa0gkoBlWdjTKjJkMSIEhUhbYcT4jTKbdKLpDm2lXKglQN7EatZUb8p/xKGueqGzUIaT4ltJaU82nyIISq+sEfWk6EkjgLJ74FlEVTnWp0tmntMATVslDsYNtzHVIFg3aydHPPfgkcmwvh5W3EiwojNTkSF0uSESo6GpG0460HNttDYQLhsnUCR2RYAoJvhPGjVFcVyXEZfKYzS3F0+MNwsrvpDzaSlVilGlLdhrAN1kWOFDFZ/DIiJT0d8tsa3HXG4jwaJBAc+gFetPm+o2uVggAcNsoo/UaRQkVR55mH+I1lxwONyTDLrin1oB3HFAgL+lIDhvfcsVXScKJFHd8WxXI0pdOUwHNLr7QcTKbQUlxN3FFZdJNjy1buSQcK3KlFRS4q4yW5Lj6BYIbu8+StThU240koRpRYFKjdKyF89wVDqNIJYpFPj1RLLzCHRGS2hSnGtI07bQVfctcrKQvyEE2tiZk32U8Gl1enB1aKcy4lAKFMuvOKbA3R5HFX55SqwUvjse5WG6TRQ2XZstMxx+GsokzI6hGCtawlsAt6uQTpLQNrkE3GB1AgREzNyI2xMWQSGxrLYds6C2hGtepXzNwg8t3BNxZoer1MeqhQtkwm4wdZdccbW1GcXt3UtbxaSHDZxOvUSbKKLAWJVuyBS6psuwIe3ObmSJodDMdgHbc4AJQElXNxcngWPABsDhK6qHIQ/EkNVCHUZimlh8yXidtYdAbCioOgK7EcgAE9rkK6cVvJgtx1xn0lwocdTU02VbUW+xIcIRr7+cIPksLYEudBrT7EB0lTcmM4hEapSXVJSsAB0OKWpQcAAUAAU82Nk3OGQSWQ4lK0IbqDESIuMGtpKUGQQsJ3Aog2aKkJPzCSbAa+cOUgQ346JdSYjrERpDLIKyqJ/aDQ20pCAoDSQVgEBawRfkx6OimwWpKnKow4I258+I0SkFptsIsCQjz6m9ToGpF7a/VlpcVqK5UalAkyUwIsfTL2XGmHkxwC04lR8u3dZB02WVHyA35xHJw66WTNlLHiWpEl79ktEqRJ2vJ5Q42BcFa7q0rIIFkgfvHCKG/WVOIZmDL7TDDyG1xzGdQmXc3Fmt67hB7ckcci3GHhtlE6EB+KB5mQbRVLW2HZY1cutOoBW2Vc/UEBKAUAeuBh2JHfkVJ6fJYizHjElQ5Ci6pLQPlJCxcBJbAPBtZC+6uY1K5LWmkQ3XTU/lJcmtynHUR1PkIfDimfLdKAbk6BclJJv2OG1uKoS1xmY0JbD7rbEJuKG9yUSRbSdQuLptqGsD73NlDbcbw625UOa2Z8kPBLEpTafE6W7hQ7kaySbkE6SGygjkt+G7PkiPSohmjWwyGY93Y6m+NTYWvkkErH0dyVi5wyCTVaPEkqcfjS4EpEsBZYc4aUXPMEjzcLSRpJBPYWFrgnuVOc46xTXAFywy2IrLDqWHuwACiSojyAhYtc6RzcgYXy3rzVzdTEeE+XDELKVhwNtou4BKPIWnTybW81/Ib4j6plMrDaaSzHmpkoS5pEhtwJcWHbHaK0pWPUgWAPe5CgcRM1t0fJXR1R1uMsIS61L8QAw49uQmCbhGtavQqULC/YHjSMHS4TktmIyjXLlL0NJkNtF1QN1BQLlgUG5SdKhds281iLiitRnWY8F19puPIMsiQw26h5xsEJA3rKCG+5DgGgAAcc37Imw29C0TDFcQ5IYcVSoQQzHQHAb+f9q2bm5sLgXsQAcRKksYxpklU9mA7D8ShcYwo0Z1MgPgpAJAVdxHlta1rau9zgLtVqcNiTSlTYy4iFubcmPEDijZZLgddWR5zpCdIPK29HFhhQtqlKZT/6plzHGmiHoUpLjocdbacsT78KCgTpFtAWk8nCiCmYiRDaZnsRVhpCYsptlqMWkL0uKWt7nyhxJJPkPmFuSbRRJGq1KirdXKpsmbHEZqQp1wrmloBZ2QXEE6Aq9r9khQB7jA4TjUylt15dcgKhvuN7rjrimmWm39Q29taSAFFThA7BfYWOBRyoPOOsqRFmlxx1qXdEV8uIGoXI8nzTquCTfSCscgEmNJmvQZzaYiA6wDJamOoc4O4FFCm9KWyAm6Rds6UC1gDiZUWJDnDp5l/OFGahZgo8vMMdZRCjqQ4gyA5a6VxlaVEISXBzxfgEG9xX0PpP1R6EVqXL6cyWMxUkyG1y8t1EObzzhtdbZaIDjraCdG4hAJFik3GLoiSoFViOsJadZ8TJQDyWW13RbzWPnQeLcXNz7Y7Mgfhcp5jwYQVtLSl1U5VitfnUWw0bL4two3B5XxYYoZK8KzIN0kouYKRnumN1PLb21GYuJbcuGUvMy7kOtlvuwE8WBA79gBhRJemM6VvuIQhF3kFt1BIWLJbJSi4B+4N7+mKa+IJORMrRnc2TM7VTLeYTZMKTRFlT7luW0vs3230EKJJJB7HVirMv/Fx1UkxXI02NQ6z80ueMnNuR31L485LQOv0tfTY83OH7LOM7UpcButYNwZUvw66pTXGXFleqc807tuFHJbbcJICgi40get+4w/UqHV5KZDH4HsR49lyjuliO1E2yHC44DoBBSCF8rPNrXN8b174js/1GziKRl6M2shREdt1TavYOBxRQ4PtoF/XCcfEP1YkBBRUKFpbcLiUCnAJv6m9/5eluLYBgfbTdHOOa1siRAlsSH6NMciylrWs8OB14kHyAngNhAt6Eg89hjrUVcxtRs2uHIY1LQJSEpIsfLcHyECx4vfv64ymeuPVl+cqSak2tD5DimXYJdabtwVh5xO5cnnWVki9uccPWjquqUVxqu22h9txq2gAIBHmWld/W5B+xN8TsXjcqZgVquiVKlyVLcWy24i2h1DRU0ps8pDrZXqPbuOL9zhS9MpqdTi9ai5YRbtgayHBZCQQAbIBJ5ubC33yO91N6m+J2aVndyK2u5WPDtkKXwQbG97W9729Tgt3q/wBZpygzUs904i61uSEIZiOnkqPCE2sLqta3BAOB7ueqi1uqKyzVHKwuM8uXCC31yXNLpddPHzhqLdzr49R9zggrkRoyHaVYILPhNTBSpi+u5JcPBJLYAA9ByLnjJdRz7nVxl2JLzfOnR3UkOJ3Wwnknj5ZufU8/bnCONnzMNIbZK8yVhTF7Nhp9TjQF/r03tbsOPc++D2B5lDMtjx41TBU/MbkMJBDjp2SdxpYCRpcB8l+eee/oRfDbIrTjrh3X31IYDjDDKWnW06CsX3hpAJGkWJJPlHtzkl3rDXIclxFLr9UQ4+5d5KnXdrt7AgWt6A+vbBH/AKa66d6PT65Wo53bILVSfdF72Hl1WAA9Af7sP7uUMy2XTkxKdpOlDkwEN2DiyhNx5S3fSbqFwbj09yDgyQ9MgPNblXP+1JZ3tTaHOwBaABKyRzpXxcdsZLf64Z5gRkMTcxVhxDRO4h6XdrkcnlXe5JuAbnnjnCSd8VOcGp7bxzDPQ4x8tDgdaCk3A1G62yfNx7+uC6E/ZQDhzWyKqmUxXHqbOkuOynHmnnEIbR5ZBAUUu7YcshNwlHnHIF73wTOqfgoc1qS84hLbkiO4/FbGyptC7rCnCLIbHPrwD25vjIo+J/N8mK7DhVcMObp+QvbCCSbXIQ3ftyAeBzbvhkf6zZ9loTDXmpyU264gFEdADgB/fVtNgWvfm4798I2nfzR7Rq2ImsZbqLy3qpR2oFNiWdaXJASncQPI42XDyjk2uLjvfnEBzT8VvR3IzhblZjpzjsBd2IlJ1y5AcQASCUWbQTYWVuWQcYc6m1StVptMl99chbQO64hwvtg376iSgHixAPHY4ql6BPfUnShwpWNYsPbjn2xY2mA3KQy2WjetHxdT+rdK/odTMrQKZRg2suKknxMp0hYUlZcPDdj6NjknknGdpbrYQskoSS53J8o55/M4IqTKsvMBjdu66AVaewRcixv9xiPSKgpQShaza5Nif54uyBuypc+6VTHWTYakEC17HDWG9aivQsI7mw9MGLktxfmKUCkoNkrPH8PXCP8AH2EBGknjnyDv9gMLul2R4hgrCtKdJ/dBwoXTZSG97wq0tklAuk8nD3Q8sZinwVVmXlqqIaFy2hqJ5iBzqJcP0AX5CLYQZnzvWIyxRGaauLLaI3H5Du/LUsj0PZv/AHIHthsqOVI4gUWkqKzcE3/O+JNlJ5bNVYfQQS04hQSrkGxvY4ZGqJUoaG4ktDiZQ/1QlZupLh5I/PEqyJSJEmp2WyflIKgFdr24uBe/OFKtTX1mnNv5npFYEgOIQC2XE+pQQf8A7bDLVs2Q0tI2nXO3AB7jDxn5lmVllaXFK1MOeJYJA7cg3/O/8hipSSe5vgg2FkCLp4XW3FLJQho35KlJucJpMp51JUpVgTc29ThE33w4xYTk5SW206kgXJJtiwd5LlaE2qNzjmJQ1lpJsbKA9L+pwYrLSQpKVoOpYPlHCuO9vfE7EodoFE7YGhC1HSnFkUXornbNa4ycs5KzHUEv8BUalPrSVd+FlIbtb1KgB64u3JP+Dq6xZmcLWZZlEygpvbW63VpgMjQu37NlsELIv21jsfQYRzQzcpwbrMtBrj9ITKjtRmXRLQhAUtI1NrQsLCkn0PH6jjE8oSEVe7z24tS+Qg/V7n+7F/Zl/wAHBmOiZUrGZcv56OaHadG8VFhwqK814uwKiA8pRQLttulv1WU9gCMZ7olI/DEsIbcUW3DrSrcNgCBa4PIGBo/VhRNxupPWqDTobLaoDpdSW7uFZHBPoCPscad+FHp/0c6m0Ko1TOni5+bKPLDf4a9JcUw7HXpDUgMIst833AtJWEDSP7XOf4VOadZKHF7ikdiQSUgXvb7Eg4X5SzjX+lWeaZnTLjK1uQFElgurbalR18OMqIsQhwEg25BscA3LdEBuvp9TkUmOmY1TqW61Q4jTrkUNOpjF3zAtbjPOhDtrAAcmx8p7KjUGm1tM0qO/EaCy0zHdC2Ybq0cqS20QLkIKSG1LuB3BNgI70+zBlbOWVoGaKSJMmnViO1LceduXUhAClx1OIuUONqTY2Hl273sbl88CiGiNAibiUOtrI3WXBJfdI1Ju4FWsbXuTcDj3xi727lfy7qWJq6npLNVZSIbHhnW1KDrb6Ut6kqUXXAAWu/Gq5vwANNyB+ovIejzI6Q5KaLbjMtSWvouUpuLqQvUEggKR3SceXEqEQQG50xdKjrccaWVtuONKc4UXbodDnYq5JAIsOORhbT2ZKUR4cOmGG4hwNoqbyHiLoaUC4RrJAKwq9rWAv6XMLr7opiRmN2ghl5LLriYbTTcueHUJtZ5sk6Sm1+9lC60hQ7AYPmVilOIU+4uCdbgbjocdWpt9pw3dbVosF3Pt5OT++Dhxp7MZhluQrwoaiXRFCZRaDrQbUp50uDSCgN6Tp0XdCjxwMFU6RBTlxbctVPmCQwjZWps7qTpFwpa+dCgOCLW2lm1rnAR8STS6DPmt0eLSIz0fbakCGmMwVR23TbesT51oaHa5AFwSbnDf+HpcDVXm1poqkNksqfTZo8cvKGonyj01juBqsF4eFzWY0mYyhLTkazYbEqWTGEhto6bBi90WuQoWWogC3u30h6qMsu1Kpx6YlCHGwExgXDNWE3bCVaSAgI0WDl0JKrae+HYhzXo8erVWG8nfWhYaY03W4NxBPlQ4oEEAWLnYix9R3WqnrpzbtYfJYdYkLIjtyrtt3VuKRthQIQSpKj5Nai3Y3tYI0rjsydBzHXCqRAQ2465GKW3F8kqKiCC2BYLVoIFyBfi5SaMzJlGjrgBx4y/MsOle3oFyUa1HbcK7OXFgdsLt3w2VSyd6U03MhQ/npjwPnz0qbZSx4q6VJU4kHUVkrTcHt3sE3OEFRXMhtxJbNTmQEth8MvQpHndBN0rcOq47oSNRv39+WuY/VafLMSPW5E9DjZWXHiy8pUm6tTTakJs46UWCAbjgG45AC+6Wnd6AuQ4t+OWhT4ryJDagtF9TiT30lSb6fUJKCbYDm95LmCepFJjRiieytla0MJSHvlpGgqSNsAaShIJIuQgElei/FioLf4TU4jVNrb765J29pUu5Dm2bhLaASbIski5J44HOGRdaU9UIy5zoaltS3ilawCEy2lHcIDfK12AN1Hi5sk6gcPbUmS3EZC4dPIkSjOL/AIZQdFyNKdNlcJuq2m11q7jmwd4UwRNVpzNSe0OVPS9tbhjttbRZX2De0s6yR5vMpYF+fW2BzpKW4wptdS9JdjMlyJdQZmKcbaDo8twNz5l1kA89uUkYbxCmOPocjTy9TpEjdZJWEhDYWAdSh9BBCSALXQRYdiONohvMhcVqfUaih5bILjPym7p0qb1OEXA1a1ltZIBIAHBwzdlAlFGqwcoEZdey2ijzm23Q4yh2PNabBUktu70fcO0VlXmAQbp5CSLg2DU4D7AYjo2XC24sCUy4VDyEgFsW3UH5fNwRpF/W62nvTHIjFNWlpmJJsvbdlIX4g/uuXIUbf2EqAWQRe1iACZLcafXUkNCGXHFuFmLFO26hB02HmK17hRbnXpIJKQCMLmzKtFpdUlqOl1kSEREMU1mQJAMdTpKg55h8wthDoRZtwIBIsm3ISQHJkr577j73H9SRGAeUXN1OgLa7IRwVi7hta57FZPkVSKqms1ZFBchxqgpFlMtAXDdtXltcWBNtRtY37c4HUYweXCpLLsCE4GS5JSpQVY7d9IDTfnCiRYuW+kX72JaO93k+ZNdHqExctSKlBmMRYjgbZlPzgtiY2+CoeZyzjK3G22zp5BNhe1r+YeolWW7LVW3XJLQKJSm9t95lYc0hHyz5PIL21k302tc4cJdLqMGW9BlqacDbVk7UopU2gIHOlxQDfnWk6geC4SL2AwqbXJprTUOlRYjs4CMppphCzpWGg5oLKCQ86COXFOEkAi3ezudZElFyBTISXH40BqOuQsiSt7SG3y4UqcW4RdYCSgEkHglV+LXbZcJcqYhydEnlph1YU8+7vBhrnSpKkfR9GhB5AJb9zhAzPcbEZE6D4eU5819W00pSQskl7TcLKE27ggG9rHBsRuoSmqdWjVWHXUOCUlTTCyw2W03S6NZCPMEm4sQC563virveJROEqRLQ298gKkUosNrsysFCFu9m/wC2grABT31hayRYDCdL0d+OhC4aYsqGjxKTHsHX7p0hZDw0OWcA0AD39LrIKtSo3iI+9Ibbjl1K3NMXS20DtlJSNNv8c1e9gPY8HBj1TfiSLmMv5UEOeLOkO6+yQWlquDt8X+YQgggE2szPClckEqY0KwI8VEOcwlwtu05br0cENX+WpSL/AE8iwIP9g2vh+gmZS5bT0h5t5bmuSG5VNcYaaLt7BSRbaIX5UOhYA0i1wQcJJaXZL7s2Yw7HkSG0OtMaykqXthSSPEKJKyQeSvskhCQVWDRJr8Nu7tbePiHHb+KSN3aW3ZN1eUggg8pVwTY2xHeFKhOttQW5viX/AALjcR3w2iS623vt+YIN7OLSCTa5BJugA2uXKSmqxfAsz69d54uO760uhp1boDgUlK/O6UhXnLfqQBfvj1OWmSpiq0uK0psugxJEt0syE/OUEraSHjr1B1tV7mxH54DPYnu1B56MJk/wjjrikBMZL7pKSNsgamnG1Eec9iACLYXkm1TGYcCBVxUI7LkSoDiSFRnAT9PzLkaEEgBSOL3BIsvC97RPckIjQZiGUobQ4+hQS2pt1CgppsfvuggpLagbBJuDq5BNiN1BgPVOWuSpiSX1bsl5pKW1lzeUlu+i4WU6yrlOlAPI4UuMxpKYi1oe1yHFl+SvQEvuAkhd7qF9B0rKjxqFuDfDMUzI1pplTKl5boa1pcZcl6WWUPJS0EFJLt0kLJbbcVayLnyG17BO2oGlBEOShtxtQelK3CqQW9AbbW2FnRdO7YAC41Em4SnC1bsBuYF1iTUWGAhxndQ6t54L2k6i44TtggHShKr672F7E4CzVlVZLLsxhrYmunxr0jWdl3bbYbKXEXAIQnsAeQO9icWJUWuV4WQ2mLGWp6A6ZM6U/NL6rgpB3ydJAbuEr78eguRhPPpkmqU1GWfxRc1iWslgqC3VxVkcPJaZWLhN/ICtI5UUXtfAKrTKlTpLshb9Ocbce5ckaWQ224Lrsm5W44QlKQooCAHCvm+F9PckriR3jUm20FezrbZUQ4UE3aS83pWuzSgnUCDrte6E8QDRRBYp646abGqqIUWQxT1uSSlLimpRttlT6XEpf1+V0bihZJKDYcnBkdU1szKqzTae6ilcgSZ4ATG1hQS4HEpBcUtRSgjyKItYWFy0sy2iwtTbTcCMhbcx3eLjSjq1FdiVXIH0bnCzqHrjkh16lR2K22yKhPXEcJcdh/OaaLt2lW1EhaipTluVi58xCk4iiNhpqUuq+PmziuYVrbjyWAhCEvm1iS0sAAthxO3fXYEcXWSmmGd4zeacixXXyjbeDwS1obSnzspQDYWKRc30gti45uKnSakNbuy0tpiWtomZpWzZ/wCWlpTZuhbqllwlwcD1A7kpFOQuE6xQlOqlsMiPOfkxnflubgbcjlIJK20k3unQuwNxo0nAS3SuDNqKEU96TNXFERtx5tfjlKesuxs4m/cEAkhAuSObC+AMOyUUKW/M3FuSH23pW8lKtlxaCnXrIvdQUoA8osF/SQLx+l1FCHdFcgGIqXH0vBLhKUhB3AG212cWeBYBYuVfaxf2FF6e8tMm7okB1kxXWEraWjUSsEgDcDbYRwAPlk97YKZF0pmoxoy36cyJKG1oejRUxkBDjjaP3UrVrfXYFN03Ll+QATZUt6t0pl2GxOMzwjoU0yobrTdhuA8AkkLcCSEgeQ8kkjCarU9syo8ep0SHAbQ6jafdhKO2jmyz8wlZTqCSACBqJBJGBsolRZDCqcHHSwEO+FMlDLYQ4P3dZsQVhQOpCboJ0G9sS91EXHp9blLqG7ViVoQt90M6wHAtvjbSgEEOEJuXLabWse+HBtuS060qZBS61U47ojLSyhmzq0DUgJKiSebaVHkHsQQcMNLX+JwJa6i14ASW7sxIsN1p/wCpNmnQs2sAdQHrb15JErMztPmSaTstCFPSBO8KpsQ3hzaMm+oly7XnUNY8ourkkHkma6yOW+7vO1JFPkSHJrKwWimSl5Qbs0UJZClBd/pPFk2Itc8LIVUbqinaky1LU/tCEpMiMGttsWabacaWlIBJUqwPHl9CcEy65JqjTrLcnZbiNLL78idpYdd3BthTIBN/2gBJ5v3AVcLKbLhwX333aOhyLLQ40XIrxspawdwJbBUVkAjyquixABHFwlSWdIgx26ZHeQ6lMDWsR2oxXLUA3ZtalAOXvZsabWuAvsScKX91c2RAnTGHW1r3HtcVuOXzt33EtoSFhwg7ZbFySkgpuMIqyxXZLDtSordOhNtmI1KZcnAO7iLpDrQKgvQoJPABJJOhNzwZl+ppp85/RTZ66ihpHghFW4pLa/mjSXbt3cJVb9zm/wDZ5NlEGLHZdbXD/DnGFttboebJUyUBFtZbCj8sFJPra44AvhXGcp6wGKkxGZbUhotPP7bylIF9xKk30XAvZKbaRoJuThIKatL6zJjRIKFvPu7yiVttN21JJbQtLjbSkaRtmx5URrHGCahFnJacZlUpe44f62iNdbLpBLoDQskFvzNGwJNkggkEgHbVROMEwEKjxWJJprq3VsuJS6QoNlIcNnEEIQseVAatrBPJPGONvNzJsqCuc25o+YAltam0rCdOhTJsSsaACq3FjysWATueGkTKcJzERpbkVcmKiW8llqQUJIcDQWUr3QSnsLXHHNsFqXWJDziKn4uQzTo+1HfjcPOlYUlotq025JUCm5AKlrJPBCO7yflqnNunwmZMyMzDeQ74RtbeiUpsOLuHAi1uT8xKUBQtdu/IPIJ+hLMeetmSseJAZnOX2nNvy6NxCQtzsCVXQWxf0N8N9NgRqDTVJi1aZMMZkOlKm23lMFBuRuX17YKf2R87Y85sBhU269CfVEZqklh9DR2nlkJU02GvM2tSHVXBGoLSNFhyT30FqB3RrsWU7XXEPPUx7xq23IKVSS5BkObYuhty4buLo7mw1aDaww2vVhgbLziXEIkOtB1iM6tTEVZGoLToSS44buBBIAITySQMN0R1hTcmXJQ+VNq0yZidAJQs/KacFg2sjaty2DoJJucGUyJOQqRIK0MKaaWwWHpoClNkqU2RZy5tt8JvYgIRpuL4ZKnNuMxUJKkxpC3qeW1syETdbiiG/KmwundcAUQUpAIAV+QWRxUos5NcNbkxENOtBQgx2m0vubZSHAkDWdRKkFRAJubg+hcFbs9puJMmyGHHG0NJZWnZU+0RztqQm5XdKjqb0XFh5yrCX8SYZmePZgTahtlxC5LbUfbS7xZpKtVzqb4WLXIVwBwcC6ikK01SJHlQGERJrxu49ZIZDRtYFSwQAACbc67OAcG9mmpyarRIT5oLzyVLL7kVp4IUp9wj5WnWSCg8JJJNxe18OMunJqEaQtVYnPvPvB1wxlNRHtH03IXbQsFRAJI8nHphLOL09Tsn+qJU02h1ReWFbFyBrGsXcAX6W7/cXxkatJXzr60qzJJzzPn5tr9HemocW2IkB/8Aq0Qg8stpAAat/ZHIvz3xDIdXLb6S3OKGwmyC3ZQPb2vf9MfUFdNj1hpqJVqC1WoJeXtQ5TZWl4rQNwNJCbgm6fp9UoHa96vzj8LvSHMVTVDqeT26FMdlL3noA8K8okEGyWyGkG47kEi3ryMaW1OXQhZnRZnLGNOzPvulmHuKeNi45Ib4HFz72Ath9ZqSELEkSWuf9ruot/Y8cdsXqv4NqDHS87Qq84WQFqLU+TvBs9w2CdsuixHHfvyMRX/6lnqVAXH/AAWZluQsqHISprbPr5EKJP6m9+Di0Ts6odm9RqHUW6lT21TqgtYbcBbLiiQkcn1/uHPHbHH8x0hpkpdnvodHBu3a/t5ewvz7X+2HupdDOt7StqHl6mSG27r1mTpsEGznl5I9OTb19jhpm9OOrVNTERTciVSQ6Ww4fAPNNMqPbcsDcjm3v3PYYfOw7FCzkjXLlocCqStyS6v6FsMrSAgejhJCEevqeB64MmNyXG9YWGXCv9syCQBflXFgbXPPv2OAysq9TuYr2Q6ylzbBcZbF1JJ7LAQTxyefXi3Jw30/KOcmGUsx+mlZStbhbaREoLyVOr7AFxYuBfSOAL9/TBzA7qapWXXY8VLaKy6/L3A22qU7pXo+ytWsfcXsMFsyTMZEhpEhQQjlatGyLdj9PAuTa5xyo5czo0nfmZBr7G0Nx1LUVYUnQbK3NF7WIIuRfDdUZsLwzM/NbMuNGaI/qaGnw3ZfvexI4A5/PjBsEE1VvMdJSrw6pSZCWCdttDbqi457qTwP4m3HtiOplz6q4I9Ha2WUglx1bOtKr8ewt298TSl5gyYWmmaFWCpUgLQY7KZEhRJ5UgIjtXFxfgrP3wmkVOgnfQw7MdWj6GxSpLLQIHcAp5A55NgLYObVRR9FPQmS4hUqbJXGF7tTAAx97I1bZPoD/LChSw5HRHiUuGt5yzRVMeU5a3+M1XI+1iefYYcy9OTpFHo1UdjvshxpaaU4ljzi90haQ3c8W735PphfA6W9Rc1yURqRkuuqaAJAXFEdDqx21PyVBAvyRtjv+uJmHNS7lEHstzo61SZ1ShQoqDrKt1DDaUf5ItZf5AH298SOhRfGstyYSpDsSwSlAVpU8b8JbTxoAufMbnjgWtiwqT8KGcoU2JIlZWgIdeG4hdTqLD7qljkeVrVtt8k3JF7G18WFTvhgzjWXX4tbzvT6Q1e4bgUgtraI+pAedcsHebIsgG9ufZDMxu5RykqjahFUUtQ6u5490ECLCbIJaJFySi/bsCogIve3bCObBpFBLNRqjdKgVGe0hqNEU6jvrVZLY8y3OSLqAAI4F7XxqRXwk5OmU6PT6jWM6VOGChZjSKl4dmRz5yGWmtbmkfvKX/bIxUfWSh9JenlfqGUsg5MgQH6NBjy5lRckSZMtTha3G0JU5YhoIJV2BOooWeLYAmY7RqTIVmvNmRs0VipS8wSmWmIi3OD3VYcAJSLlHY2BuT64QSOkNAh0xNZmdTaG1GWgl1b6FhTZ5FrA3XyCngd/44tqLnPKkmEzGzIpDUioOuNNa4+8ypooCdRUDxe4Fva2IJm6h0DL0t5xCqW5WEOW8kXcDYPPlAugE39fv2wxKXIoPMo/SynpK9GZcyqTdG4wwYsfj9/gKJH+cDx2wng9Q8mZYYcOXMqJjSr3D7tnX+39pd9HJP3w4LzXnhhL8BrNdQTFcQsaGnNIbBNzb2v9vfEKmZQivK3WZq2dRt80ahe3v+d8ImTnV+tea57yn4LhhrLe2lwul51HINwtQ+3/AJ4xDqRLc/pBCmPlTqxLbdWVrJKrLBNz635/jhWnKNRXfbfiuabkpDlj/MYJFDmxl7qnmELbVdJDurkc+l/X+44iIKvOHSl1aqmY7TkPrqbxdKXVWA1kk3t+vbE4i1CPR34ceLE0NvrIIQyCHEDj6gO1uO+In0+2qg023JS3qcIcVqvqTcXIBHAxYFWprs9+ny4r2ytsbyApY0mxt3/gD+WIiqM60qS3JnRYNM2IyCjQptshJF+VX7c35GKgYjOSFEI9Bck40h18UtUKLAkOrck7PzCpWr61X4H54pqBSo7amkup0uC91eiueB9sWsiL9VW9+XRNNOoLjqgt1SbeqTe/8saE6M/Cn1T6lsRKrSaSzTKDIaXIFSqKtLamwSCWmwCt0XHNrD/Kw6/Cx8Pw6wZ9kVGtoeaytl5sSqq+RcOOHhmKOCbuG9yOQgGxBtj6ZVKbGptFpdOytT/B0+OwwyzEYp3KG20hsLdTay+STawQ2gDucVVM3YnIwapoo3S6lZdyL8CfTuhstVDPmcPx9x1C22UtyzAgMujnQraUp1wkCwSVgea5JAti/skdHukOSIp/otkfLlDqrbW5LlRKf4iobDgTyl0pU4Vm6zdNg4CBfnEn8LTvCR478OBJkICCg7pSkOlINlXuggW1G3e/1WN8LqxMk1BpcqYtrb8SVITHY0kEjzFK227pb+WNF73txYEYyGZ79CVpaxvRMNPhU/MCErp9CCqe663Gc1SlvSVOAKuthgXAslJuVXQFkdrXD6JMNGjQpqOVl0LSztuJh8/MQFN3WQbhN1HWUFZsoC+BrrLW62ujoDkcaHXBtlKVm9myVW3PK3wWwTzzze2GmMXYRh6noy3nS4fn6wpK9wKtyg2I1chzixAJA4NSJFlx6bDczAytmppfmMtht6O86FMgXTuJte7lz9uALi2PlfnaFRHsy5jXlvL1Rg0uJVHXYtPqiS3JYYDt0supWAWyR5gk8hDgHNr4+rDz8NxMRmTAQxJYRfxKHGw8psOOanRe1iEH6ibeX1vxTXxI9C09UKKrOFA3VZ+y4y+9TJTKNpVSjIAJiSHDYCwDhaUPocJAuFY007ww680sovZZDNXgZ3pkWq0qlNQnAX/ltftBoABG2Bbb1nj73Bw8M9PYz7dP/F5DTcR2MXntLtnm3NB423ObXtwOT/DFW5dz7MoNXdqm8ibEksIjeHcO2GBqClKsLHc4F9XN++JVXlTM5RF1uleJTBPmebSC4oPggE8d72B9gT740bHRU7qc5KzbWfh6rakNZzrDFOkrLsmB+F78GcbABRbKgOwAWQQsj1GNM5C+Mzp/mqr09ma4ijuR0W0qLyI7j57KbLYUQvWbkuAni17DnA2ZMh5jhZbezDMa/rcQanWXpWl9hBI0h1J5QVdwD6c8XGHvpvAoMvKhnT2z+IuPhzUpwocaav2H3PJB+xxHMY7VM1xavp7CqFNqkDXDr8SrpqMW7FTCN1oaEX1tJI16wu6bAGyyeCRx0GpUxzajFx9+M6XUvKJLtSdWSQsOOWIQ3tkEnt2AucY1yJ1Ur3TPNDDlIqs2qwJhLiqYqIt5Dqwn5iSNJQHngDocJBLgtc3xuSiGHKh07M1Ey22I1TTEfakyVuAFlxIUhyzaiRdBtpcAJ08gLxkezKrQ7Mmh+m1aYuXrcDLsOU5O0ygHG9hCRuEuiwQ0rW3ZVyFc2tYXMaqy0xFxG48+W9Mdbs74oN7IDikp3G1p0ABfI4sD2uTbBs5ysOx25kV1uGnh5UkK+pfZxAbcP1k7SV6Qu2ngcYH4JuPGlSqo9JjCnnxjh0AISFjhoOIKVuIHDoZFraSgEjAd3VLJvo6pbUKo02uIhqYQhyWrfIXHQ4EBOtSiLEkjgpJKS2TYgWw5SHaxVHI7Jjn8PhkuxDEcZMtTe4k7nogal7bnzDwiy1jjnketx33BXGZEsvbmluUWwx4VegjzNlJLlhdR+ld1DuU4Pku0qoiRPdhBHiEtuNveCCQpDay26Ua2iVrtc3UbNlsWHbAuhZR9lVZZFKqs6nRqYUKfcZhgulltrcbS6VDlwo1G11DbJuLC2C/w6XV5hnKprSYMw+JnJecLTw5S2CQLC7ljoAAQmygSOAXx+tTZM6VBTIQ2hcjcktobS8+62QkbbiggXuEJOlJB4HmBwYqpUuapopj0+MlUV195mW3tltZWXEgqXd10myflt2vdNzxfDl2imVMgi5aeQ49PeQ0ypvb0RAgbLQUTvOpWixBXwNIty3rUR5Meov4YqQ7SXkIaM2QVEG5KiF+XVoUABcdr2Gpa7FAFzW0v1xqM1V30MIDeiShLzjCHUNkKbbJdPIT5QQm9lgI5J14AIVDYS9Am02M+43tOWcirRI0adSmmiuy2yPN63ufbjBB+8hlCIqlZq9RjtMzHGIqflw1wg2Gi6hAF9txGq7Nm1JNjrVqPoBgwxlyF1CpLmOkOsANR4s10bbS0qKkhsjhBCfqUSSO3pYqoRxGYPh2/CjbW23sE7gR5nNsOoKgSTcLCQOdA5I4UyIDKJaID078ZqJWG5CVxQlsOElwEpulFxp7KvcJN73Bwt1MqTJis1aQy5MgeHafQ460202obKNy30o0oGp1XJCyQ2lwFPubVGEMyg7I247a4z6nEyHVMpb0fvO3B0WCUnTwLaOB9GFrEyJFjxVvJYdVt7cRyOy28xtDU3dKVkLQs7lrpbNx/uTZC6YUKoPRHnm9FEhIYumLueRrUHnnCQf3Nuyij6AbgkDDG6gak1RzFGYa8PGemNQW0odcsw5HU4wvzGyVp4OsHbNh634IsBhb1QoyH50R2NNaZ8euPIW49++mzjjhKShu3YJCrIJX2JsVlpiVS6iqohqS940tvQHXng7YIKnNZCLlttVgBc+bzcCwIXNVh2k1BtLVMmR5ckOHSFghSELO35nVWOlFuRc3sAbKsRbKplsmpbCKi+8l6utKcW23GZIc0pZXZKTdK9VwNPYlKCb24Ng9vUmOiPCkqhy3I9OjIktSEvOHSbF0CyCleoI8xDfYnm2k3Qoqs81NhDMd/XUGYwWqa0XkpRtLIebbBTdAQ5w44STuLJI74VJbmQUSnYdRacK2WgZSHCtsOWO6Gzo8iNZSC2Dze6yQAcP8AZTck0x01R4OSKbBqMaGltDJbdc1JdLfLg3XECwcBBIuLWAQQcGuS6kmY9TZVXiOh0hmJADjyNuzlkrtwGkGwNySi1jzexdY0alXRVqguGvVtlx551zaBCiGwNd1uHWClGoAL0/cWYY7lUi7U96guQi4F+AnhlL7TxQ3dRs2Nbi7c6gbeUm1u1e6GwXlsoprLkhNP3pcSU1EabRHbSGnClKVJsbkApAIUUXF19uMONOfmRmG3V02RxGMp7UNpJKGhfS7fQfIkWHqLX3L4QtUmaMsy2Wd9mEixU403xYJLh2FWSQVcnTrXckldh3bTDr0eM49SvDrjmV4dUlLviVALbGllPHzDYWNhbsjjm1l8ymUKUQnHJMAM1WRIcffdQzFTNbXtIWtCXCRrtYqTYBIuHRdHHF22rVlhhwt5kYahpfdW3EjGJtraWHPoK+VjUhRuEgABVl/TbDXTFzmJ4YqCoi2Iy3FldmkqSCtTakJPIC3DpShI4SRxa4OHPx8mHBNQlLjR3Yd4qo8jadIYd7pT3Otyyitwk9rcLTgZQluiY7VQU1Aaj02eyjecvDEVyOEMC+oEBJ3Qok6FDzlfAve2AT2oEqHNmS6fMmJiKRGdJhCMoocQkOM6hxr2gC4Pr7j1OHZivpksvLnw2232HNt16RdxLgQtslva4ANzcKb0arEhI746p91ulpnPRpFIiwmA7HdhNpZS2VrSlKvlthCLWvdV9wkgkHvUU4dlTXRqw2ulIREfjNoiIKI0eU2VlskuJUGwL8kCy+BewFuCQUKnTqhDQ0GGX0NRnyCpp5xpSDzcaB2upVmweVhHIGDJTrz9QbqxqUl5LGhS5F9Tu0CW9BDlyEKuAArzqABGj0KbqkiLGbZl2hCnx91WuON8IbAbS3qFi4sWSbqFgPJcYGU8k/dT9NbZgRRS6o0Iqnw426/IcOpxtwtqI2SP2gtYbl7diCcIXahGk012OiQ224w25IWEqcEdldtpxy6wUC4SU+ZYuCL24wJ+bR3nYbU9uTCW5JAEuot76m1o8ykncHDbq3EkHQ2stqI5HOGWltFRXDmyZ8KWhIc2m5BRrQAorQlgqsF2AFje/wDDFkeZVuTrRJNVbq7tLdjzPBOEuJCw2llxBCi22Au6GwXEkgpJXZIIvYYTVWXTq2204+yFNNMrfbqIZcdOwGrKWppzg3NzcOBd73uL2LYfVOhIfp7jEQuPaAGwlVkOJLazpAs4SflLv2uCAj0HMZpkqa34iniLT2I78kOz22lNODzNFCQhs29DqUta29NykkjDNckRyYmiuQVTIpqUZraDMhTZTtl1F1NpaCSXQAQLgcFKwCbDC2bCpsOMuYkNRlPlEp11th0FSF8JLZWq13Wwr5qjwv8AdBNsEuVCBChlDs6fFbciFMV5bjjpils6Sq5JO1bcBBbQTwdQF8EpqtMVIp86RUWFykRgWlKedZDchCxtubpF2+By0Ro8w7X5ZREidAjQ3KhAAYXIfbZiqS0tLQQvUnXe5cQCsaUXQQCSbgWwvebTTnG46JkynSY7QuyCElQRfXqCD8w3uLkgqPHpgDja0hMyptn5brq1ofQGHnQSQpCHCnkXASs2I+WP3+CS/FzIufIjVKiIcpjUU7LjDiEkSFou2vc27gJQ4QhSkW0EggEcVlEKKO5yahvO0uYzVPlvBZfcihmMlu5La2ShQBsQErBRe1jcXJxLSKA+4EQ6fSKf+KOtvurbC33G3V6nFNqLizrWpYHlHa/a3cL05DNQ/E4spqO/Gs9EkW2bWu2TbTcBXJuoEkg83JwlqEiMt5ulSIbalx4Uh2IPDW1NtuAaTwG9aQlTguddk8D9zBBugRlTnWam8iDGj5khtuNRnXZQfVtKZUu5KmwopuGwUtDU4i50m/e2AuO0iNTY1SorztPiuGO44tJkMtl8qG4Cb/MQCBrsACXL6RYAJctVhlbKFvTg1EnujfTCW0pnQHDbab2xoJ5/aLWQbDm4AOcmVZcgM0mrOyVOtOy2iy4ldo6xxdwgXATZst3JK9YBuDhs2VM1KJspCoikxp7ZbkuOIcOwltvd4KUbhUkIQ2LHzH61X7KwN/NcGCy28zTIyZT62nWQ9AYS1ttAXccbDQLulzndBPccWwGc2zIQXF1hcn8MAbQ2Y0dKklvSo3Sg63PJwtNuCmxBsBhHWG26hVmq1W5NpwmLkzkuuOsuKQsEthRsonyCwULoRuIvyDY5kqKhz4z6z4JinMRXXER3n4obKUDyuPrNwEIX9IBFgL2tck4PlxnmpkmPMptQlOxGltFktqZMNZ02OqyRoAVZCdwXuLg3AIa1FpjLUN6S+25DhWZhUxtl1wMEIdCkN7f7iSSbG/nN1304LIZp8iKmPTEMMvsIWHnWm1Jmrc1X1JLiitFwBtpWCldz3uMTMohQYr0Gq0hiUiQwpDjojSDoASVqSErI4CwDyu4HYouLYUl+I3Ud+mpfioMhsNiK44HmwhRUFC17Nhy6rAmx45CeRMmTAq0YqntCRUGzux3HFMBTB4KnVIACBr1DSrzqsR63w3txXaWfxuBWajFVrDbxSXvlrWyAyptlwgNi9rGydN+LDBcoBmRj66wmYhVKy3InjwLYeLqGSshYCi4kukrKFFu585tZJGi5wRSY1OWla1QJu8FrU8hS3GnnS4DZaS0bWb7oubHTYkAjDu9VY0yqOKVJE/W06W0KISw1oBUpsNFJ+Xxc3AQFggkA3wkIp0eoxacWZc+BHjoSqRIZN3UaUtlsjyuNtXA0XcJ7G3l5KORENtVdqS/ApsmWzOKtoNTnG4b/AMpHytwNt34LpJSn9/sTcEelQUTGWkCMhlm5YeXHc1vvONWu4u51sd0hADdiL8i/K/LsZxSH1s04wprhbiIZkE/1p9DilXDi7EouEqBcCOU2HAsVL1ZrLK0xKfUm6wZDIZlzGVrkp22HC3uFICdtsAkC9gQL9xhUE2QJUBuYIKos9cSa225KqciNoDiAlxxvSUarkg2KVLJ1pRe1hdKH8zQqhLiMz5sZ65jOhzWl3cF1AOjsQNVj7DvcAIIHWKcflguMzFuanNdTc+UgL1JN3Cpi5QeNJJGrzg2woVDQ07UaSmb41ElCw6F7uyy6FDyaUKBR57JF7ArVcq5F4onJDlIMBpCqi6XnfDhpzcZWUtLsUh27aQjkXLgAWqy7g2thEqLS35DcSRMkJZEEvNL21LSWyoaW3rW55VcpuLpJHe2O0zwyUS4dCQ+06Uh1tl8JKVFCUps6lwhF1EpsTo0rbJPBvh/pwfLEdtqmU5EtyGZBWh1IckxFqUklxQcJbdSVOE2N1H8jgIsTXSYs9LkOJUYc+Y62rQNcVwNTG0L+Yga2ygucBCFFBJsi17YKjvKjQwxGmDwT6dmQ0lYaUoOXs2VPhSDcH9qlBA0rBI1GxUXxWW3kyZCn70x0adqf4aVKNwGwlQJQ2bOJ0KBsbALPKiG+oveJegymK1Nj1Jh59oPyQ88ypoAkNloi6Hhp87YdJJVzwbl0Wp6zJKhuoQI/4qiE/vtuBSAf8WlQZseV6Wmkkhsg2sApZujHKy0uRJlVaj1AtuxnGkNTGnXGWnlrb0/2dYWUbusfQFngo4xyFARVZ7cNDLDUWQrcdkR2A8+dDKkpLdyUWsU2J8iSoXPF8Njv43GqSVojyX6U0y460wuI6FMB3SrWeC2QCEpcsvbJcNlc4Fk6flrqEcR49NbajpduHHVxmh8wBQKvOkuBwDkc9iLjjB8GDDcjw1ubMSGSUuyZDCXQ6tz1bSQkrWL8Ak8i4tgbs2oyobdLjOtNsOOERy6T8gLJDt7HyBVgSP8AJ79wOSXpSqgyl5RQGrhocpDzjadskpRfQs7gBXe3A5xjzX3VqA+/BLjtKEcxij5i3EuFx0sXURcLKS2b2buALe9wLt1KhU2FLUijQ3BOdaciulKSXChYspu5Vo5JUL97YNUiKIEeWlS0tPrdBWZJcj/LJV5k2DlzdNibi3IPfBUNplbT7PiNbOhvzJZXcGylKcF72FgLoJ7C49RiIF1katl+CqQHFMpmsfVJbS78kAjg35uF35va3BtfBbKo0CbPmojN0mPP0ba0aAlbSx5S4Afli4KiSL8Cw7EuCJ8sCXPehtNDbYhbTy0oDZWtJu3blwqsSbjy313vhCVGny2ZzSJXhjdbS3mDbgGzrgc8hI1XQDzxcc4iGZJk1eLuQ4z1SQ044VlKG3QppwWslGpA97m5v+WHF50vuyajGjS0LfJcW2tppzxHyykOOEqAN+bnvws+2C2E1GKw3Ml1SSzukNNhTNjIuRqcQ9p4INgU3Nzbvzg0O0unOsw57bk6Mw64SpSLXHNkqA4bBuLuHm1iAO2HU0SeC3U5LilofcegDcavHb2lKAALRJXqHmNr9wLAffBhajPOFEwyvAtxg88tLYccAIuriwWEXUL88EXF7jHqbWGZQMOl1Uz3GpIU4gO6XmkfUy7uA+f7kISAO9hguFDq9ULbzdLKIyGpK29bgaW84Fg8lyxdt66bCxAFzc4iRcaiypzTj0OC/wDuLaaYiGO6kkX3FKWL35t2FhYC2D49Oh724un06crlxwrIktPLAPccIRxewuOR7jCRbc2pB/xMd9C9GsMJijRe4Dim2x6ADn98WNzccGMGnRpaoUh5sydp0IUsEbTQGorW4UfK4AFhe/FzY3xFP9SEIUJawWaY05ou4W9nbZUSSfpbB8gHJJ9uR2wFnLlNQltuJAWsII3VtoSyVhdlXVoN1osnt2+2FiHpsB0uRmUSJT7TaClxoNoDZQUnTtk6Gzfta6rHnscFSJsuntxYbyKnChafFFaQ5HYCNNk+ZYUCeD2F+CNI4uCHItPRCRQYVXq7gNPbcklZda8UghloHuAACvkjtYGxvwMH0hyS8SiGQgIO64nxIDRIHlCrqsQCONRsb4Z6fMnzlOzUyGzAaSF63y81qDZOkhpfLhvby3sSB6C2Hxo00UPVEahi8pDjpcihLgbNgUJ4sBrAWBbkHvzgo5SjY1OVIpypxKYaoyFvOJMQrS5oHayL/qf9BvgdLliG0zOby6te28Glv2cNwTqcbVyCbaRot2sTfAqbDZmx3ZlWU000QXUSlslKSQu7bZS04SEeU2TwT39LY8aA3FRHW+hunR/EO/OjOupQ+VgHki5Dg4uOw7Ei+Edopz7yQSGpcdxxlhmWtp9GhDDx1OjzlStsgD34ABsQQO+M5/EX0yzjOzMvOuQ8uHMG5Bbp9dhQg2mV8rUWnkscFeltTiXAnnhB03JGNILROcIYiy5UlaHC80VNWbUgqF9ptZ12INrpB84HFr4DKhU2U22y9Sp78htxbTzLyS6wQL23CLX7AkEW+vDsfk1ChZ91fLiu09ECtPR61THSxIW2oRXIy2JMQkAhSWl2I4BuOxHfk4kDrtFfpSF1mjtyqa/8tqpNMlSVECxBcAugpHBSqxB98fR2tUDLfUXZoubssUyfSo9gGZrDSmUoc0koaSCbbpA+nz+oI7YrDMvwkdC5ziqnl6kVXJjhWG5K8rT/AATTyP8AFgsOFSF9iCXE3C+Ob8XNmbzVORfP3OmWWKVUHXVbTLBG9wRc3+w+1sQhcBuXIGoOFN+9/TG08+fAvW6nT01qkdYXpCVuSI7LddoqUJGhsFKN9l1IJJIFtoXtf1sM65/+Gf4gunkJ2uT8iu1OktBJNRoj6ZrNvVRQLPIHry329cXZ2HYqotKrOq5Tcdjf1N7U5a9k/nzyfthFTabDgMvRqq6Yrradbbix9XPt784s3I1Kp1fp8GcmYtM+Q4tDcZ8BtlwoWApAcvbXyODbuPcYWdQOmWb87TVVuj5dK1xytsU+O35rNtFxRHPohsqPvb1wVLLvSektTKzHaUvdKgtASLDUgC5PPpwDxix25W5mxCYg20Q2kNs7jRSAfzsSeNRv98RLpDAlRHWHYDIc/qLrqXHEo+ZuI0i1wbX4N/S2LdosSmzafTs1LkvlcZCEbjgHlWgkAWH754P+jviIqhOqNGnZlrtWk02O4tMNl2W4VK0nbbAvYG/34xWlDpU+q1mHTKPHMiVPfbixGRa7rrigG0fa5I/IXxffVSOxQJSqiZDioNXZdjOlKkp0tufLPY8gE3t62wL4Oul+Yc6dXqdXaVJjxqflDbq1VlqaDtmrlPh0pPBcd5FiRZGtfpjRG8MYSqntzvC3N0O6Mx+jnTKmZIlTIcSUGPFVCa1DUW5jrjhL0k67EpF0pR5rbbY+i4xPgittuupW/Gjw4JRpW60hT7r+vSWwpALhKmyqyR+4BfkXwbUYkdxTUalQapHfbabU/GcZZOpz6toJDlkIJB4AvwPtdspU/wAHTpOZqvJQhth0ux0NbkjU4XL8hv0AAuL9xbm5xynOzOLit7W6I+cqbKfj0+eHIbSlol+HbDakeGcAILaT5xqbH732AGPUmkliM46FVGPDfdYLQWtFo6HHAC6PUmwGgm1iLcdsChsTJc5uvRnmntZaLyZDQCUvuHzWH7QNkpA08ruB9F8HxmpMmAUQJMtxlbu0bvF/aWi5Js4okgX9TYA35xWRZMXZUUuDEgT4kGeiRKZaC33oKmiUbizqISSPmA2bJIFuRb3J7cKY2uHBjJkKaW0LMKik2cXpU4yLqsQDzc8mx79g6O0igeDkxpTzba7bTc5hl5xwlZKg2oNkBs/L4uba/wB482a6pFTU4zfj3mJIcSA6025qkawAS7b9mDyo3JAHItxchrwTYIFul0i8MHWihARHkklx9xqHoDu35UtlSyCTdLYuD6kDsMIG6bW6bMAeZXMSxMWhtmSt0Je9PLoUbOAlR5KyBzfDmmhIjPtsvSWGIYaW02ltfi1vWc8oVr0juom/AUsGxscCoEGRKp7kmmPSKi2hAQ8ZbRZ3UNOKcShWvlsHQSVBfISByOzbBV7Kleo3wo9KOqFRkSq0mTliqkEO1egQdwKd1AlbqeWJCBuKJP7S/r5rYoLO3we9Y+lJXMyDXVZyoziS5tUtpyDVS1yFr8IdTSzoBNm3b9vLcHG9nmaFGkyqnLfkJk7TjrDryS+lgBZADYuG0AkG5SjvpsLi5KpaabS6lGcnVuryVEBvw5gX1cBRsocA63T83sO4OLo5CEjmL5z5VazDRo71BpXTLOsqYsv60ycuzdYkOJKW9QWz6cGxt274d6n07zhOfgZek/D3m6gy6UlDkmMmKuOwqR5jvKdeVtNbh1XbC9FwSADfG6ItCQ21HfiIkuSomjxRdmOPthFvrOiyASSCUqueRb1OHA5YqjM1vxVEfpS10x15strWhYRrN3EgupvIU1quf3b8AEWxb2nQJbLGuUfh0+IH8UXmalZ8y3lqZLaaYkU+RV5K1SGudrdVGQphtCbcXKx2toNsao6eUnM1ByNRqfn6qhVfaSXRPg6XozQddcLZ3i2N5wBIu4pAII7Wth8plOENt9xh52JBKGkPx5Qb34sSyVeGccJJ1kHUUgmwLZuMKTHpcRh+MTGUzMcbaIll0q4aA2XABr4bdW4ggWHPthHOJCZqblVKY3HiN/0t1PlDrjS1EtMvsOAWcug3BsTwbEAWFxbA4D08MPWeaIjtANOLBVuWbG420VqKC4OSO1jbuSAHOJIacp0OWKNEltlADL8h8pL7fCW9snjWEWITY3DZ45AwyOKh0/MA0xUPyUKu6ue0+wkOGyrraF0L7FYb4sHACAeMK45k2ZSSK7XI+hmm6KfOjt3AU20Hw7tDUC2VfWEOJsfcKJUOcEueMlu0+I7uusxNcWQGoyXGSdNrtpJ2xa4UVCywTcntgiFUg/QpFVaHg3W0tIdYU2kpcbCioNOdyspIBN7+178YTyWqi4xSMwttR3oIXvOR0xioRJDiC0rcAI+X8sKJ5QrgD0OFsjdLpm+3RW4zDLaIrjK2xLCA4pDAVwFXAvcahpPLnJ5HIam4NQRTpT0OpNrM0thxSG22i0vzabOFJOgLQoFQv+zIAuL4jsqQxUKm0K1TZD0FvcRoUjSVONOWbDbaDx9XKUnRYn2ALm7NBahzKglslKQpbwcWVkOeWyTyhBLhUojQLrt6cFmtcUzjzT3lpEx3LiaskGnKg1wQ4slmaDFmt61IakBSxraf+kbbgRqXwm9wcJJ0ev1GHIkPMMyBGYW/LkVOUWZzIW7pLg3QSHPKUoJJsDZAPAMfnUrLNXjGLJo8CpJd2/GMPx3GUuo1lvbBcHntpsXE2WColCkei+mxxShBhvVCo1Y0iHHiNOTnlutqAbOy3bkcC6iokuOnzrVrCLZslS06EWv57LQXwEXtr+fVBYokmNKbTGh1SUHHENvI0LQlvc7C7rgcJSf31aSeTZJNic5Lqy2WGIdJLoaWh5DsGE21thYAG8pZO6LtJUAEWJ1n2wkD0CJIhzJ6ZFMVqLbMapR/mLWRa7ZWSQVFz6nRcW7BHdaqKiPRXRUEuPoLb7zclx7yLQ2E2IdRyQlFiL2tyTxYY0c+6sh8SJpFJDVfYVUdBa3Wi9NS62rwpXdwuvD0LhHIseVdjfh3ZU03IK35khxpx82RI2yXO5AS4i7iG0rbSQk3HAv3AwmcrLtQ1zIubA9EW+hltz8P3w01ruWkJKbNqO2Cga+LeTvfDcrMVMp8uPTICy4wVIKlSHW0CO45dRbDZJu3rUlS7jkKK/3cFzXOUBulswKECPVV02JJZkyyWHF7anXNDelTunlZA4buo8m/e9sN6qrRZ8wUfw2qc8pZUPDeZoBwFm4Q6AvkuKuUHko5NrA+orQ64uC/TXX2mmltCKpASsBDaUqBB3Dua1knz+UWNgOySOZzdOhwJEmO54yOXG95pLKQwvi4cLpA85B1+hvyi5GC0W8SmaxTnVpHjag+5DadaJiANw1Rm4iW3EOWHyCrQtGi5WggI16/Lga0Ih1aPLqSyfFrEpUhsOAp+YS40VBVibbilqAsLE6rgAxZ1hYqSKvHqDUyQw3cqAW4rWQnS056lwoCj7kaB2Iu9VhDsiE3U5NdqL8dqKW5LbKm3GEtBs6ikHSUD5ukar8hQ8/Fi5EpRUI6HIZmGctioLfaR4N2QVkuBx0l0XBJKeUgKJcTZJPrgK5i5K47TykxXJTvh2lNMoSHeE3Jc5PKO+kA+axNrAedalS6PKrCA45KkurZWI7OlxxoNABxpS2wi4WQDYXCOVkrtgxYqQnvyFrpjkdhLTbjDjjxCmFpTpbusF1wEi43POLJKLXACOdmRRFXZZjMOxo70J/xcgSXXpcptkPALvoUbk9wUk8fs+LWFyPFzqg7rqa3IbEZxEIuMstlarBWpSr6dBsL3Sb8k6rm+OoYimYahTC2+IQ8WFmMonxF9LbTbbZsVjSBZS7/ACws3towU9NiM1UR4y0OvRHG2m21PKbIauLBRRdaGyCALeigs9uXDhySeFJWxFdVDj0Rl2GxHdQ0FLdcQ8krKStxw+Y7ig2Lq5CQOByVk5mgpMhtURaky2wSFa9px1dh81Jub+ROqwACi4BYXuDoqECAxJpypMVuMygbaJN1b4cUC24TYlF1hNySU3ANtXB1YdpUiqIpUOpNPMCNugObT+3LC9s2JPDQAHJWeVE2uRZ8yVcWl1brLs+C5EauttlBkoSGdDWlxCbq0E3I1gabdub3QCOwiPOYYeRUDGRGWqOpnbYU4Q5Y2AuXd0DlwkIJb5vcHCVUeshI0xENABtMeQl0kR3HV8Oa7pQjUUkISLlKLi3Osu79NjRUCnw5UeniWvaejyHnZLl2kEttMqbdVrbBupY0I7nWNAuI1HMRomhijMu1hbaIVSaWHW92MsWLOs2tqWdC3baXAQsi5FrWNnOn0SfHp0ht2lzW/O026QoWZ0XIQpIsLlsA23HNVlWwmiPR6UlEFaG3nXnC00484lTWi4ICrHRcIBOoIJZKiOxtgFHqCGUIdeoBbZjsreumMvdcjNklWkEgABAvuK4sL6QTYq511LW3SabTKhCee/aqdW3oUUoG68hCtJ1WNyFLc4b4sBcgkjDcik0mW+65MlQJGttbm48yDfXcuN6l+dCzbzlJIGnuu2HN+DVHqfHhvwaQiPIEg6NlaVC7oJcBZ1WIvccharEgc6ME+BplFT45ESS2iQdEpt910pqCAbFKi4k7dyQUJ20rF7jkYLEF5aVxo7ERx1hvab+qNuuDY18htRIcWBu+caLgq9zwpdp86RTHpzMCQpuSyt2Spo/K2Lt/ToWVti+lWl0IP0H1vhUmG8HhFQITySt+O2mOop3WCbJDZcSp1sJW2PMfOb2IBIOOuPSZ8KU8mNIprL9oy2XVr2Q+Ufs3W3VKCxoaKfMSSb8C4BNgokLLU9ExmgwZqIay62v5pcYcaWi6SCp8AoBClE+e5At62xxx6oIfs9OjvONh8hlG28FBhStsNlzlsAJc0KKCXAqx4tYt67bbkdLb4cdcJkIWgOOuAeuzdKFu3CRYk6gLWPN0TDiH5LsSfTqjLROC3AmVpQh0BzdDY5tueVXCSbnse+IonKC5S4mutux3XVLS5aUmUtQdAccUFtBAtwEpulWvT5jYhN8KGZlFnIiz1M0qTF3mNxHzGbkanGm97TrDRNuQLnbssA+cx1TLNNDc6RJjzFahs+HpqoTKQCpRISAVuOX50pNjzdVhh2bdah1BYbdCYFmv6s46ksyEAgFKjYFbl1kFJQLaSDe18RRGtOxmX0h9p+axMN3HmCtsBZIJCm3Wwglek2TyORe4scCadgogwYaoLHi23H3kR3XEIstBClAWIKGw0G1X1q51eXtdIiO9JmIQqnNNMxoLpQt6XsHcNgVBweg5AbsCrSRzfBLSG3nWaaoxKjCccbcO7BbLRAXe6knUAE6laBrv6abC2C1uUIO7yVzm3lUlDK7oDjwDMd8FSXn9Dh8wb5cAt+6SULve50jB0KbAp8tiM7JkMs3ca3yzsIdddtbU6Q3ZrQVNkeQ3KPN3Ab4C1ynHaOywUCPIPhgl515Kli6g404RZtxLibFSRazR4CzbCgSWUU/8VMKOw821I3fC7QUkBxIcCWSPoupyxT9XBRax1BFLnVS48soTuK2g/aS2oaElvb43EAlwFslI1L9SQB5CE71RYiI8BIjNyLL0PMuBG2ACAlKnG0ncIPmQnXbhF+b3b1tM1ZxmA8yh6O26t5pEeQUOGQHBtFSdBAO0L2FwQPqIBx1NOSHn44e33AxttulsKaLlwShtJIQCoKPb3AAOIonuQutv0yLJqjMd1EthDa5Tjp2y+XFOJdLqFK+ZcrJ+ix1gjsTxmW46mHDREjT5LjaIy3XZulO2DpUNRRtgBbg4S4gEEix1Xw3tNvmWfwqjtz7PojsrdkpS+O6Q2lokAErskGwQXGggcYUzGaqmAzuMrstwttQ95BU2HHQBpa29bu4QDdK72A4RaxmVRFuTWaYtUeOw2uMZC2/D+GUFNtBwk6r8rsi55JICW0cHCpp1p6QpDLL81Mgrea8M0VFO02XEqDZJJ1IABuFEgKuPYqXFnKrTaIMd+JNEklJU5Jdhy3QhWoFopIB5BLh0XsQsi+PQqLTq+VUxuDEmNOvWbQw1HYfJ0EJVvXDYuElXlOu9gvtcTMolIq7UtyNGntyzILzSmiXg420dw7Zcd06za7gIVb0RfscJpkZDUt6RWYDbyA6PktxnHS4huwa21CxWhVzZNraLXNxymiynacyuTCHz44W9IYC3CHCG7LCguyx5AEk3v5SEEg2L3UpkmpTISXVolVjxS46/ml5vzhJJaUty6Bc+ThuxtzbEUSORGmttw3dqYy4yVtFiI64VR0F0bYSSpLiG1FQILg16zf6Bx2JHkmtTIbaaoTNpqGpzrkcyEpWF2ebeJACG/IfMSoAgnvbAY78eU60ptbC5a3Gm22WWwXnZHIVYrB0L3Ukl0grNyeRbCZ96XHisKj0iJNaiNGJDRUQwplqQtQCnNJJQhGsEB6xWNsn1UMQqJTJmmbNkt1KcyiFV2uW3C7tSvJpGtlDVi5rDbvfgOArt3JUlutoEerxqWwtmYUONJcQNkrLgbU1qQFFCx8y5UUkhKfXuilxoDkWTLgvOOXLbO2pktNFHA/ar+hJWpPJQlfH9i4w4vzE1BKqvtRNCG/mIRMKHGV329Lra7rPq4FE7fmA1C1sEKJbTPxVVRUuvQ62WnUDyF/eamluwI8UDcAaUq1E+UX4NsM8t+XGmuSkKcAkqQWExxvpbbcJa2WgNVjYJCE/5JHe5KwONGFBgU1UFP9QL0ZuM2XlS3z5gUqbAAIbs0sBf0Ag+owUunVpicvw7cZmO4gODZYaWHPlnU4ykgo8pt5SQW9wknjAy5nJsyNjMqiJh1WiwKXuwh4CQ8x4dllR2tL7QIA8hJsZLo+vQCU7fmSCa85CSy9WXYTDCnGHYOyXAosPfs21HyLWoEBdgSbo5FhYTr0OM7t1tZdDpYVJkANpUqPt38OlOsjuCLOXKTdaQTbBqEtyW0bNH0ySrZkynZjykk2Vt2UEga9aVpNzrct2vYlkqbZLcd+RIbjT2H5LqNtwvRfDIebKRu7gKwEIAbtqJAJbRdPcYcmVvRvCQW6MyYyJK2A61ELSC2XS7oDIIKFuIIX5gQr1sgYQIffeNPkPU+ezUZsVhl+Yhxt1Km2h5i2lZCCB8k3VYE9xg1FJUxJj5o/DqQ02trVC1vNXfcDjjSUKY1Kbc+XewIQEgA3NtAXZROsfQXFxtU9K5DJ2mnw2t1Q3WykFICQD8v7i/Yd7PVQlqjEJUulwkRifkIWtSy1wCXWg2EB0m3YoPLYsbXw2RYAlsMom0CZKcl77LDbUt6O7uWBAeunixBFkrbP5WvgLTcOPCbAyqJz0R0vyJkdpxx5LnmKdJFwsaNQWlQIskEehxmdur3bpwKZM2mvTihfhSC22pvbILtrWLqBcLbXdK0qQAlBPbBdJfmNpZgViI62wfOFx0Aoc1tWFlI7tmzYQLjta1iThtLLQjJk1SiSH2n2W30NtyrLShCxcpuoklSCkWV5/KSbdsHNRUU+ciGuDPVHD5EfwTZU4HEN3S2XG1Wb4LdzdBI7HjgpUrfRAqLLz9Jp78n8NjGS88t4broW4SBZbn1haueSBqK+BbDdSZUht5pqSjwrkx5hS0OBKEpPms4p0+QrUsC3JHB9sdnSowiKgKZqMCUsBttfh9oJYJAsp1YuQoKdBBuu2n8sMrTNVEXxT8vytSlsobiAO7C+VBwBs35I7i3Bt6YGVGwUgl1CDUV7s2rzQkAlmOwQGWnV6g4HdscAXuvT3N+CDw6R0CqsupqcCUJd20uymAVNOv3KrnyjQggix8x0BetNwBiMUgMqlllqKtyN4m7zD8gh1S7EkghJta5VclBFrdjh1kU5mQ+Isx5ZTII2N5TYTutAC6lHi5PPmAujk3OCghhKYyKnJpaPCwnA8Hdt5p5Sm1uEEJUNIupGnlIFkWseCMDsw6hxUqqNoYU0kNNyFOOArbIIQQPoR5k3/MnsLBvfp9RmoRIVVZKX0utbscw0ttttlI0qBbBXexAKSgE3/XB9QnQaU3Mbeqcaqz5LawS2wtWwAhLSG0jSEa7JLlweR398TzQ8kqhlt7zmkFtEVlCJ7saQocLANnHWyONYNiDex+98FvoU2iHPp7UiO5JSiOH3oxbaAQfNYG+4LqALih5b3IwRFhlxlufIRplzVBS0p0tGQsEqcU56tosUgJACFC/J9DGJ0Oqo2ER1y34kdphpJQXkstDyqAGltAQeO1zwj7nEVluqUuSGqQ/IfjVJtxMFDbUZLqXGCATyWratAvfzE3IIJBvwc5VlrRAqMqHHcjtoXICA44C2tACy4i9kLAJ4IFgQv74SuR6m7aNUIzbwjsoU0JAStUdwAbbikrJIAJ1WPAJ/I46aS4yTHrktyU+huQhtloNrBWVaihsjzi+u/oi/HtdL62KOX7qU1KPVGnFGMiqMzlJcWI7SYz7utz5m4EgngLDl/Uc37YDUKtHj7bkZ1QjMMiK+0X20suukXvdditZuQsXH08C2C5FBZkGDr8G2qWyVGGghS23A5pUS0EgthI5sTyRx64aMzuM1BtDMiiMTkxdEZthbOh5uxUArb8tlk3N7H74bKlccqlKIrMWOh6mx5UB93QVEyhyte5awcP0aDwPS1xbnCB5qW+pLb0gOqh6FPDxDjpbbHDiSoWN76bhIVqFr9jZsjJmyVSFtrdy+zHAaRGdbUtIXossOOIBAI58h7nv64V06JIbjIpyaO4gTS0p5EXba3AXLI1KKgsBQKrKtYDk98FHMHBN+YKRlishKERqkF7Ycc33HGHZTC2wdKgFfSFAgE6CbC/fDhQmKVEogg0xlsQg2xFTHlrLqmgTdLaQV+RAKrk2V7YVV1/8T3kSlNByIgxZbSnELaZ0At6FAfLXYW4v30ck90LE2JJmf8ArJmHEMQoAYhNoWl1CBYXUAkLuNHIbHsffC2S3yp9UmNDqLEN95gxi8tLUpFuQGzfZ0EoRYkX1C/f2thFJcrq5bUktxxGbetqkLWpSmwsHWpJCQhFjf8Af9exthHOlwzsyUQkSWn2XHgh13aVHsQlN2x/i9V0gX7A3txdFKqdWiwv63R2WXjodfccjh15TQc+SEFCgUA7ljdYuO9sAaIJYyKUma4unOomtuqLLyY4WoaOD3dQS4tsX7n1HYDDhEnESlqQ3TAwJZIaQDeK2RYEOLKiCP7RB7+gJwikVRDEZ5Ex9xbswLcWwWnUpZLQUEt7iAo7dh2SCSDyTzZZLfXVIAkzGo7rjcVDbOmUQlkLHyi447+zQPoBse3fEUVTdUPhQ6c5xSnMESpN5VzG9MXPNVhxBNZluhCtYlwkBLcjUCdbl0kXV5j6Z66o9MurmR3oi6lH6f06DUEyKWJlFmzPEOh2OXHJCWnk3YR4cKSU6yRqI79tsMVVlinRocbLaGixZ7ejhagX0Czqy4vSCC3ufLC7W0k82xjz4y85MsVRuJInQI7dEy6hiOhkgpekzHNx2wv9aY4bufufVWNMTi42Sva1VNM6jZPp9BlGlJQnwccQYjKQUjyADcIt2IubD3GI/kDr3Iy6khdAMhK1L1MrdbN0d7XNgCfz9cVOwmdX6JIqsNpbcNtRCnSkgC3YX9e+I9laPJqFaW3LkrLTSgFKIJt+g/jxjVbRZcyvNHV+dmhU3LNchRkQKjCkU/wrRCvD6xdshXoUr0G4xrz4DKe1TejFReejwo1UrmYJBcQ8kIkvIjsMlrUHdJOgqVo03vqNr3Ixgyg0n+iPUhFIrdJYq0TabdTc3Q8262SlQII7C5t7gj0x9OehFXTnHoXl6XGebk+CtSnS+yHrvxHXG7NOW3G1BlIcFzyDb2vRM7Kyy0QNzHMVZDr8aXVHlyk1SDId0NMPlkSFFwICvO0CLaRcXSCU2seTfCJ5uLEUptMinoiE6FSFIcvHQbWWbhII4tYH97gm2FEaLAXTGnpNULbYb8jrbJDgWdRHJIFyNNkqA+ojiwODpkByPJlTmakG4RcbWymVFFy3xpFlkgo5KrA69Fye2MatJuk8iHK8EtUkuUtsBckyHXQwWkBywFxbccKDe54sPdRx7L2XqVTZNTpdSqj63VoQ9LW624pOxp52ysWK1brYAJ5JAHYYAtrMUmYurz57DAkMrLZQw26rbuQiyQTcEXNlW45FrG6mg5kdhSPwhcNjMCpe2/5mlsOnRb5ZeQbW3Cm1rnt3J4sshmTpPbpUCtSkNT6zp23EtLRD222nUENnebRcuOfud/XkICSSU9SY00Jp1EjRIkuIhDblggkaFG4CdRQ45bsfuRyMEJkqNXgw4gQHFurdEhEdxlLmtxQeQpKFXFidvXZBsD5eblvmpmLTIh0iYJBEohtFPjfJI4BeUq5J48qLrBHtzhTtZBOwpYi0ZDFUnR2VR3Ww7HZp5EhxCwUgqOoa1jkm55I1jsADmo7U+ZKmTWmJSRd0MIcLXim2gQV6Qddj8xWnjsRYjgtTrVbhuyJMOpBpNO0BpiQzutOOEWd1ONqFi2beY3PI+gAEr6fWYdKnxH4tO8E7GQwp+NBZU23NmIB0oUkFJs2sk3J5KiTfsAfNTRFxy21LD1KlVd1fhnd55lhpITrXtpb7pNiPKTovc2t3IUtRKsKbT3pziGXo6I6m4kJSVFloOhRBKFXbbAAS4o3N0mwJtgmqR22aw1GmSIT6o6UbymozgVKdILrgU2SPIkuWAUbKIF7c3Y2UupvPTNkU+QiNuPl0efbIKnXrEjRYK/aE9xwMM2yJTw5upYQ3GcYnWvIdLKAW1OlwALUuwW23YqIvc+X0uMNldlyqrMbaY/B4T7De2JDjjrhkBYSQpvWrRawAva45Iv3w6OU2HU0rnUp1ZU+38uNtOJU6vaChpU2DunWU/Udsckn0w11RrMkuIuKiqtwpsyzTrLZbceYbQDdsugltF9IunkgW9sWqpPiZFUhUmWuWyX5UB4usvsuIU22wNKhuW4cCluDQHBfy/vWvhLmKtxZDqMvw4L89+nrceDrjyVbEYE7m8F/tLAqu5wQXCLqIsE1EmQ2KJApEiqxGC2H0n8OKyqY5dVi7tgkX1/U4CiwP2OCaRX401uSzBD6XXW21mVIZ1M05wNlzWHGirWskhsg+mi3OAxRdLW8N+hyXEsVENl5zwjidh1tnSdtlagu5QpocHQnvcG9kJpD7MBLi3m0IkyPDFjbckllop+a4SLbbl76G0i5Lar2AvhSqlw6bIMhTCGqi6yizqogkuPaNxIZCWxcG5KS4AXDdu9gMIqXVK1MkuzPxBEWTTiJMiOsLYeUtCwSop1cNpQdR1Gx7DucFzUzXBu6VvZYerlA/G5Mt2nzGHQ3EV+IlC345Sk2SqxDllqCli5KQni2o4MmR6hUHUqM+qeBeZW7JmMh1LskoIUdTrgGsltQBA2+wQAo9jZJjupRUKz4mZDin5bq2nC7LAcLyi5yAP2yTxyS5cA2JBjb2XGG46alAVGUuPHiIZkOl1TxJSLMpJN1k6Qt0o+gkAixATN3kyblUpjwTzMCqriusQ3Nt95oq1NeUNNqKLlC0gDWAsg7rZsMERlxIscQlyHanOW9oddQbeEcQQdaU6Attw7a7clBCeQCeXaTF/DqjFaqZnJaf20O2abjJbk7IKrC516RpT3tZvRfucR2lSokSsTfA1R+eZMZ1MaO0yWlP+YHbKW03cKiOfS1zccYZu6ATwnLbiGKlWExpfi4hXOdkOOrbG0443thIIShBIJcLtwbhINxe5EpiXS2mUFaHGWmly5D8OQHmtAbbCHHPLdCNxwAhRvYt2AtfA6JWP6SLqLwqtNEKBGaDsde46naWlwAqcItupDZKw5Yj1wcH6izEajUnMYRAJWhuVCkOPqeC7cbaAEaFDy7al+1wCBh1MuXdN0s7c9iRm6y5c3WXUS47pU62LhN1jlxtN78AXJAR9OFFajNw4zk1oNrlR1eUqi22ogU2EnasUBdnEmznpZBBIwfSafIcpsFqjGZ4kPWkyGkrSxYucIZuSXSkEEDygFVuThNSMtzWlzkVCq/jKZDTrxW2/qVHWEFSrFwkBbgAFm7i5IuFpxTzR5Juk7cBox3JrseTJDgeSxcNrdCyFoCfl6wUONjjzpsjm3BcaxIiwoEaJEyfU5IfVJ2x41huNHbJ8xaU580WCUlbh1r8ouqxFi0UyBmB+L4uKuSwEtOTZD71mgNLzRuLFwgFoJFuDckgab4X1epQEuGlzmGkqcbRGfNUccakJaDjSg244bIYQEJZCASASVk8ADDh2VQIimRoshSBRWhEqK0Nupe5WHA4XFOdwV3SCnWFLAcs36kHDbIl1t7Mq8nKochFGXCMpqXdKt6R5U2cjaQULcbJIBc1m5N7C+H6JVZYkKirq8yJFTHWw0+VPNJK3BuB4pQAW0JQbE3JukXVY8oIRrU3M7S8vT4YgOmPYMsuh1MkI29pxMh0m3KuTZGgKI74hGZK3u6pZU4rTcZcefFcaaL5dbZElxcdIXq1aHAEtoe29aiCTe6DzxhqlPpgsNsxZseC0+u0EyGwxqjoLqm3rWCCChJSBoKE3572w6yK3BjVYsOoE+o+JYWGnntaVOABxLJAKvlpX8vy6TdJ72AwljxKaXZspmG2or33DGkurKixy6DuW4csCjSfPzcemFc1Nsm5dcU6lrZnbiFAyVstOkyHtFtxxrbAsDzex5I1+fVwKU3UfxRqSyyRNkJYXqfg63WHFjU2psOaiizbfFxa6knyXGHR1FTqaUvUydAao7DZVuU526YqGuQtwWSW3FLKfmA3IJtY2upEtqa09Jiy3KaJ4Km3Aw8yl9hZJbZSblDjYLfJNuALqvgsU5pnhwI0FqROTS1tRYj7AjLdnuB/cBuoqSw8e2sk37lwEXtwc9GQ2tqou1JaETAsMqdnLLkTYQohKVLSlxbiVhKbg3JIQNa0ix9OotHXSHKUhEtbZW+8wFMvBKlsaXFFt1awHEApI0+zmv8AdsRsT8pM01E2IpubT/FoYL81xtLXjCtevxLgcN2w3uFshFy4RYheA095RNVUqiIkuF4mqNsQ3XFlvWpZLqCdJDatKjrILhIJ12UD20YPbQYrjdQbvAamx46RKVHDOpbYVZRYWs67hW1qIsbI7gWwN2QtcJbdHpTkpmfFQh5AcKy02FiyEuN8glAuFKJWQkEkm9jEIpsmCqRUwhplydHTOjMX3xEWhRYjs38jiNt0kq12um4Cyb4szW0S2y6pLS2lRZQqVYbYYStzb8RIO6w64ebKdaPNlqA1JFhcEBfnOHiYlFIlLYVeT4RSxtpQUhxzwzGtbTp3DfaJHzV+UNEkeg9TZjsmTGmvVfxTOtEmU13fKygbTfyLFayVJ8qQSPMEK4N0L8xh56O3S0qkNONOsq1uP7/zPMyCVjYEfW24Lkbil3KLjC5sqI7yIep1QcWHlpcf0CzLyYzWqI3bS66tvWbuJOy2LixCdZ4tguoVJqap96tlc1tt0tNRqe5xNj6LN/5a2gsBNxfSFLKDe111QqiYbgqUynNOwqVoabckxVKb87gJb8/CNotgjd4+Yr8y11bOlUkUiPR4cuXMiSHmFusqWXSXVuggpUE/L+ZYNJSbDtckghUEY7OprtWXDb3WExHQ27rp6FiOvbDakp0HQgFZDa1FwBJJuL8IW0uVFdbVVKcy6pyz/iGhplojtAD5ytbhu7oBHBPCXLkXFzZEqpUJCnhGp6C65IcdDqUbMVGgJLOoJGhbYOkFwhBvrBK08IJbtSky1S4selS3XXZBlytwyFElJbU0262sAfszcg6FEoDirEgxuZMk5RWZdQdiz2GqmoOGQ8y42W3Wmg0UqKnWbWQ3ZRChb6iL3Bu5Q6McwwU0+iRTGUWA0xJB3mUtrWrzNgOBC2y2nSfIpaUN6xyoYSZfqC5AlLditGJHZDW6+2UhJLinLgIVrJCASQbDm3NhdVTIkSoLjS0tLlGf4lqoOraLjin29JSAyFD5Vjz6oAWLrBTd/CFX4kywoKzTC69ZMdx0WSht0JlI5AVtuXA85BDiRwXDf2woXQoEJyU6iyoq4i0PS3Gy2S+EKSlwNoKnBy2kgEFfa5CCRhXBjMJh+JheCpxCnAp1p5RfecXfSh0XJDakE8pbsSEnixwpjMLYVJoC1yHjHQXlXID78dppJ3nCgquEkhKCb7traVk3EUTDJb/HUOR6QlpT8NlDUdyNFZdeLgJaS0kABdzyUXBtqc5vyGqNliVJiGZEFQedYp4e+ZGVt/vDzFBP+eQLpKSuxAIEngOITED6mGlSH4j7jDS3tuy3SQk2QSCShTuhtKEEkkAG2AQJM2BSH9Rjxlvl11G5K8zczTpDjgRbQNoON3vccg+oL5lE0tXhQloElClydxxSVkNKaKAdOoC4Wggi1hYLVz2sCnIslxUZchcItS3dxxx8iMwA4klwq4IFruGx1lXlWGwNOHtiNHqNRFFiUh2ChinttT5CUuKS0tDo3bkuh0G5F9u5b1f2Dhdta4SAtxC3QZDsl9MQga0BKSvvco0biQSLgALANgSHZkzVFanMfSiQ9LcFTlx3C8y2WyAG20FRLbZSW2mUlw6x6n0PkwKhyJ9SkRls0WqR5LEkx2005abp1lJ0J227kpAUbuee/kN7m0jag0mMzPEaTUZzMhBdZDZU4+6iwAecTZpgFveUlHkWLtjsbnDfUKY1TIia9l+m7u5G+S+3U20rd+YS2fVAJWLcoKwA2eCcHMo5GzlVRcpmBGmOiGVoT/X1l5x5BAcaBK0gIB86i2kJBKtC+UgiN152rN1SBAiQHZMhbTT4Q081IacQhCkqLaWwEBBWCA2VrKADdVxfD1RW669lx5x9qBDTuojQpCrF1jbCmztBfGgF0Em/1i9ivDxU460wIk9fg2LTm3GkJDb8g3bO9qS2LhsWV5bgci4HbEzJVH8s1aFWn56qDHMQyFNpitPxylQWhNlcBKR8vUfl3FjfQrjHUprEuVJZnU+Mt9ehDUkRlbIQdI0mwUsXsnQdYCri5514do9Bo58OqN4NMqOGqY/Lc21FKG0FStSbBtY0EnUQl24SgdjgDk15pjwFN8Q27EdEt+DEVuWQV2AZdWUhDZKgruVkq4BtcRp6qJLUYI8FPbdqM111hrbibzZ3SCmzYWEEgNWBNgew5PYYRSZ8LxIXSJ8ZlD7Tchp1bYU058rbUogkL8xDh5FlHsR3x6axIzBCp7chUBlMBpyM62uKgxmkCwa+YbG/Dqu5NtZWoG+HCpU+NOpO7EeLlSQkynX2IwDaowQLlp0tKQENeUOblxrUO/OJsFE01HMLNIlbLVBcmtx21+IaZgl2MmJ9SQ4XUq0Eam9fvtpF1ADHY8qMxFcka5JjONblnm0NupKLtEAuDXYhs+UoCLixsRfDo3FkNPia0mHVmnQ4G5CozS0PSFs6myp1lCbN69JPJBOsoUfNZOxTYzb0dt2cy5Fkx1x3WFStbrIACnHGw2SjhfYgggHhNwCuXBGiiLRWZTUdEWTGqcOLNZ8K4w9qjqNzqDalocKzqBSo2NljtYkDCtyHGkU+euow44TNX8tKFsvSVlseZAb0/LCSEjzefbSbcXOPP0tOxMqUF52AuYFwF7sbZa2taQAXQPOFLBA8h06UXcBGjAIMN+PBix3UOrkTGkykESVhDUdsJS64+gpLbrR8obTrtzYXBIxW3xJ+Sb8xMo8Y1u0pbjch1xpfgWkBLq7gOFO4o+dtY43OQABawBwXPy3Ggw6VMjLROYqMcALqLdnHXV90tBzS46BdsEnX57XuCCJQ1TqwxCD1RU9KizHY4edabaaLDbbQNkeiEAN+2v25AwwVXM8Bqn+JarbkVlDTv/q99bjvgmnHFFu7hTf9mSPMVjXqF0A4dDL3UVRaRAoOaGp8OvR/DyZbTLySdURZspvxCoyw2BpXcIdC/wDFKJIuQXZcmeiMjw0+I0uO0sRXUMgvgBwEJ3WwAhoNhVrtKRa4J4BCUTcsZrVpiLXSDIZcgssxWrvtH9oHEsITtHUv61N8HhFsK4UWmVBSYNUbht1SXKfhzY+4CWLt7bZDVgGUBx1wjm6l7nPlALbpU3CY8tZLMyX4t162t9t1IuCEhrdCzbVZz5lk21J4NhdU5I8RGf8ADumHEnyvDsh6QVJS42RtofP0W0uEbpHqSsm9sEtx6bGpQkqWwucX2I0dDb7Ij2QGw6FNXOiy7bblyCCshJsMGUJidVHWosaLsrDC3CtmOZDJ8m4ApzSTukFKbKsg+U3tyAominOTnorkTYcEVa34sF5Nw34xpxXNtIHNnEoVzfSogECwdGWJUqFLfhtRqohcZ2GHIKXikbh1W0o9blRbudGsFYsm4wvfy14OdHiSYZYZjkMbCpKTpLbgS42WvNuIC1NpOq6y4m17WuRJo70pZREhyHpaITalpkuJbW8XB/qhKUJBd7Om3dBSg6kgWxDsgU/SYlSqbqJVdkI8MpS2iWoOwrQAfmsgk8nSCSrXqRYE9iWB+U4/W3EyVw0TmkNvhQV4RDLiB8ttOtSluLUSDYaLDuAgXwtBnzI01aJEN5lhoPN2W2+EsLGm7ZAuNS1ctkA3TcAAYWPuxJyXa5UsxMS5TyXG3YbqLvP67JLrQbIWRcODSo2+WQOQBjNmWhwv4U1xag05NY1oakobQ/FCGmm3Gxa6nEFP+MI+YbJPck/msjx3pMnxkh4OR35xelSgsCziwlKVbZ4t2TyrnngDjDjQ1UuCPCTJIMdpm6Ie65H8QW3CW1tuL3G2ylFiVKIv5gcMSqhRFV4vR57DtPeUUpUHkqcdQ4tQbJcXcD0JAFjpXYXwl9bItbdPIbK6GxGiRN4B0usygzffW5yWXASPpKRoJCxe3qcG7by58tEKo0ulPtPuON+MKUuMueV3aIs4gly9+eBt29DhrlVFKX1NRpTE5lpkuh3aLjbT/YhNlcAGwCuLBQPPfDihdLSG5jrG8w5t7zazcOvr/fQSo+caRcqKRYgcmxwwSbOSSa3UmvHQZdNulpyTGZkuRHFIbcXZwoTtKA8177nKODxbkdWlmXKcdkynS0JNyURkKelBbShspUgGyOASfW9zrItghxuqssLWinsuCO2w0ZLBdSW2wvSGkpBNr3NtRt6jnCqU0glx6BSloXPQApqKSG2VkX0kPALGq4IBJBsr2vhygEVl2j1KotykOS6dLcDoLbK3AlaXF3Oq2m7lgRcA8cd7jBtQp1QLqm3kRGlzA23CiQn2XA0tAHyyGyAhu4ssq+ZcAA+Ugjm1KoOUZtCEsJPiG0OLTdW5o+kpJUbrubcC3N78izbIYcMhE9qMJMme8Q7KqLjatC9stlCddghYsbKuu9vywuZPkQ2X5cd9VCqtKhxoYb23m6aUMSUuI5B3HCStA/s6CPMQRhV4tkJiw2qE1LFgd0SArcjlwqbDiAkhvyJIGkWADd+LXBRUvEmIwlbSLtFpsxgrUu+keW4K/qVZItyr1woZg11bDECCy3JKHHGoyEpSUPFagVakrUCgEJJHa9jye2JmQzEJtZnrqlQMmJXHW0ulxLkaoRiqGQOznpdAN9AFlhYv6kYVIll6YuV+NxFlqStDSjKAaWtCPODupIPcEBVjyecFtuUaZMKnnpslyQyptzfX4caxYcJbJAA7AWtoHHOGWPlio1d9dPq6H4NFhocdEKLGcYad+qy3HiQtxChwjgEjm9+MGw5qXPiUgjpfffCpKt1tx7ddYcQHgyDwlweZN7WBAuEG3B5vg1otJ8Z4R4zi4SW30MbYdtyl0hxNyiw4HoRxe18Km4VLmxRXJam5hLjUWIhkttiPJ0kNh8rBWtIQOA0lXNgbcHDXUGZrEJfg5LcKXES44I93n32nL6dxJHkQUg9z3viXuoe7ugNUtcvw0GY8tiMtZcW2kKAcJ516m/mH7qV9/thZrdhUlqCZ7sj5i3HWdTzTRHG2rTcrPt9vXi+OMpq9ApiX38qOoDalkTp7zllEJsoaEK89geW1EIHHriPvZnzVMnyoVEo77EhbxDb6omh0XvylRPAKOSE3QQL+2IiWp6qlSrCfFyJKkOvSHPDlYiF8qXxy25oUggAWB4F088C+DE0uLuPNVUbMhTe4UKSRvrX5lI7ANFQT3tcE9sGM/hzdPcacmurkuLDrrkU+ZsI8xIuNsklQFu4vwe4wpiurrUmRGekrHiHkPhcp9sF8oDmrVtEIBsfXngW74ilkkbokKahSphFP3eG0x0nVotdIJJBWiwN1KXbyj74Nqz0punmmsMJpTEZoFlfyWnihZsEtC5Lmrv7+vNuXSW/uqFQhOQ4koR0MRWWB8p8cqDjiTbbI47IJ4vfDWXhGY2JRDrrLgdZloeQpaWyFJsFG/kUeQe4sDbEy6pM1tUraju0WeuHRMwhwraBZMNLpfElAUbWIK3L8pJshBP7vHKFmKzvCNOnrksRIoeaQlorWFlsa0KsOACok2BA5BPPDU9WMt0wop0Twvi5P9W2dDpaDuopSpRcHsNQ8/c+nAw/UiZBntrgIXMjTPDONvJjOB5959B7KSvk9yNsEE2F+2IillE/GaIk0xbwdZDW6zFKy3qWg2KlNG/pz2FwCL84+cnxVUIZsrVQzDUmVCfIzXPorVObUUrVIBccIte7bTbYbItcnUAScfQCOmnNuTTUaKKewtrckuvIKXnbEW8y/md73IJIIvY4h9e6D9OuoWZpmdK/TZMqozGjAZmpkqciM3aCdxDTl3ESHBttbvfzL4B7WROsbpHNuvm9VMq1BydUctUiMEQ6deSWWBdpv5beo8cW4HPr3w09NumVckoqVSlQ3IUbxyIyxIG2WwWi42pV7WBuBfnvjaeZfhD6hsVRdKjVrLe+GREXMbRJUhxh3gBzQCgkfSSDYLFuL4aJPQDqfmapV2M7UqVHOZRTKCZrshpIabaLZeqbDZPIs0LtkIWSq9uTjU14Kpy5VnXKsFacsVaYiMX8yVN1FKooQ43qlSNTKUttAm5sFOjyjgnnjH0S6O5HqXS7p1Q+nDK1zHqfGLU19ZaaaakrJdcbPmAuFqIClC5AGs3AsyZM6D5V6YxI8jJi6duTHBFMuW+ZL7+2tTYcLy06NCikkpZAAJ9e+J4iHPNMWxTYrdNdZAZdkPuOhvzuAkFu5WUDgH1GpJtjPKc+gVrE7lrVHXGmNNRmfGIU7IWoo0oKCkn0QsFYHm7AC9j3wS2pqrxnPwsuuUhoMIcYU27q8SVhWzuEhZBLYutvSCCQMBTWp8yjBM9dMlTEPFvxLjo2vDoA0gXKtbl7WTc2t73wqjqplKbaXVpT6lOq1xyuAt6QRoNyEoUNZvqNj6i5IHekBOSi2XkMKeRIpojuELcbCNbTSkEgOAnla0NkJ9bXte9jdRMqMBbUaqvTnJkaI0uM82sKb3/pUU3PJWk2+nSeSQQBfHEV/Mb7sUyXZEJEdreRGdcbO2i4DZebQSgApUPl3N7puMJ2WV1KPOKZhlGG6FqRIgHca3OAoqBGtACVBGkdyL34GCRlUXZVWpP4cuTMnSIUlyIhthEdbQI+SNMfWj9mALXtdw83IvhylrlTXFMR5LbwqMSPHLq1tsMhwtpUBs6U2O3cBzkEkW7HDFl7JlDekUlHgIzdYlp3YxeWDHlt6uEALAWgW5WSCFFJ+nDhHogEh16RBckPypfgYxYlANNLJIB1XAWLJUdJICbd+BiWUujJsmC62qlyIjYhOnfgQfIfDsLsbaRcXUvgnvwABfjCehy64knXGqMaI+G24sRAebTIX8sJSObtgoAUbrBSAAeeBGJ9UrTCUM06kTXXIV2xJcfSzc7l067BVyHCoE3JNhY8Yc3HpjnikPIqjykNEst7ZBuQkEqRcIAsFWSF8cX7kh8t0uaye6bA3zFqAkPNJkhyQ8sIbIDYc8ulCyAgKQkgd1qKTYeuFrTLcJM5FPmQtEdpcl1tUp6S2xAb7EueZbjjl7hN+LgWthmRAYeSuVVYMR9mOy23PTLsVOcHSbNun5RNgXL8kC5scLX6XLeq7DcGJKEunvlqTAZ1BZWUKU3pBcutA0thCR34JNhg5WtTJpnuUl6dAGYpAYZnFAZhK1OyVgkhRLbNjouCLXFvW4GFkeW1pmGrlyDGCxteHBsdY5BC7AXAuXD59Ha67AJplTq0ecqW3TYsibIQwzIlTH7OlZLrittsW2wlwWs2NBIABPnJdIC1txHouaabsTH1HaYS4HxIcDe2dy5C7gupI/IC55wjmqbIlqW2zKrAp34QphuC0ApToeEpdyosgWtrCEgr5PqLcm7bLjUKLDfcTPQyoiO9I/wAcltCOUnnltvXqBDYPKkDjvhYiTAEadMbVDDzkRUhwoaO9LcNhtk9kedtJKghACArvfCqmPJrLqVtyqfFefLi5MxUkMocuB5ypsEoKQVa72tq9TbFjfCimWVTxPjSamh6sx5qwEksy23XHGljU646lu69xRASGhYaybnggOEt6A9CkxpHhaklycJcjU42FOsNJUAtscOLAXtk88WPc9liGISXWIaZlMMjdLkWJARZrcW0rU4nWQtyw20kqXoJNx5iSEtJYcqM1pdVgOU1xBE0ppqXbym3Gg3sNnUEE2BKFJuVXcQCOV4IVSNqbT66FIXVnqg3EfeAYddsp5p0NgubaRY3UhVgTbhSQg9sNEai01qW8+KaiUYjTTDqJr5adjrsFKcUu6XFrKztoS2SW/b1w4z6vEqtQeMpqfKS1Kce8O8+YigXeNQTpIac8oskGx8gBNsJajVKuqOiZuSVqL621vocjtpDrnZx1QG46s6Qggkgg37kWrt3tVYk/i2prAg02AYEANNKdecZcbYba03LjafKV3NyCSASHDyAmzzHejQXJtJqsR1iDKDYS94JTAjPttgpMdsKuSoFXnI8o0Ei98DRmWm1GoP8A9H3qEp9sIbi1J4iU7ECWwSoQiQgLDgUELcWq1gbHsWOBRG4JgJjLk1h3SZTUtdQSgtOEq3CC22m61C6S2QeLrFiQBUx5dIRZMWAC906sMQVxpqoEN9T8Tw70uQlZSqISLuBVuHCUOJFj7BZ4OG/MFZZisldJoVaU9OlrisyBGQG39awlkqkWtqCATqPe2gBPFvSjWI62pdSXCVJqAcZEdwB10hxwBa3XHNVhoNhc9mz2BGDUMwX4SMuFuOqoMtKbaQzJW5ofDgSAhIJb3To+YlKDpCUD6ySLB3Ql+0hLrU+FmV9ymssePajf1dTUEvOJcaHBbIVZaElThsDouSQD6FChmk1pxmZJjrb2kNxY6WjGfLtwltCmylRCONwoSTxqBvjtWi+Ey3Gr1HrUB96PKXFeQ605vFBcUn5e6B4ZDY8wab7BS7nm2B0ylSoYaenV1cyIguzdqK6Et6NwMqeYSOHXD20qOskCw9MTM0oWciqVTqsttkRqhLqF21uSHAEJLSFtvOKQHuEB1VxawJ0OG3IKMBkyqdAgv1KGhpMqLIdk6pTTiEWW0lLodCCqx1gnWnTctrCzbjC6NSqixU4wdoIiqaacQneZadvIQtvkhxRDeoWRpIACzwSRYk5WqkxioPtSG/AWU0iTHihp8lBUdu7He55StklQIbA0g8iXzIZbJsgUvcf/ABOOpsNxG35RlKPyHxa6SRquFglOgizYHJJ+jCuTR261U5mw7ElSH3Sy5I21KYcbQlL2m6Lax3KyEIuTbVYgEulKqjAQuqQ3Ia2i+22+ytAdALbe4QlwKQzYuFLjZ5s4Edxh1ZMJ+U2moGZUVvpXGstptO5ZNmk20gG2nUAmwJ5vbQA2ZqOXqo/AcVQnS8JLpmOuN/MTaOXWFgkagSQg8303uFgE8Dla9TKXHLsmfUDT9ElpbUhtxb7bb6F/Mf1adCy6A6NKTwboJsL4SyMu5bzLIadk0cePab3GVpaeiNFALepxt51QWEJ1KUXLqBcAPY3w8iiUtmDModSi7aYy3DTEx5LyzCRy6Vublh8sJOu9+HCv97hRmtoimie1Meny31RJDqa1vuqUprSy6QkEDzqU2hDQCdZHPHYGyx2pzVxKg3U5doTsZ1tuW2Wy0ltwpKt1xNyGAEWUQpHAcF7HknlmY1TotUgstRp0gQjFaS3ZkuHuVjVr+WCBuHSSspwtl06Wau4ZFScEmjwS4JUYFt8F+7ZCiVWulaHU3VuL8tgLciN7pUQ6gplxh96RKdU6067HckQZJTvBts6w2AqwWpZA1ADtoFrAYYmfwSjuyJ4oTmpsSHnSIqnrtW2wQ3oTZtS1AFwjWq9wAb3MXNM0x0TGpr6m560SC6GndrWG0p9Ss8F0ggeY7liCBdfFrcBT4o0SRKXSW5HgnH2JNrNgjabaK9wIRY3sQSUOLRcEWSXfeU3RLrKFRZpr6nUU6OUS24UIIYuDwLNAg+H5Z/3xbdlnWkXUJmOQpSnI61kRHCrW7JbDgcDllLa2tTYIuB6osUC2sa8BrlVXGkJeehQ211CyXFrlIZPhxcX20NqXrOkKKQU8gG/ulTUobrxo9EMdbcwulxpotyX33S2XexKVlCrm1+QBYA6TYeLwoOF05O01ulSEU/8AHUMmQtDUuTHBeI1p5bUXDtgloFIc7gOLAJB4b2ZdOjveDfiVFKS1IW5EkhbKVSw1qDypbirOt7osQpFhYAfTYts5uq1qHGeokZyg7bIdMpVQ2lOt6dpsBy9vmONqvpt9IAIubqqjFqImRGoMOAJTHkqC5qi2+6+68LFRc9NGkbl7EEAGwN2a0uHeUFuSF8+nqlvKmrjymjClUyp7eh6O/uBKVIbWFDRpJb0puVEgE97lPUKjinakTHZK/OhuK6zvB1t0NWdUEKAbIWHB5gQo8gW5xIYlKqlPT46tpMeVILTodbaC1NFYHhgWUJ3GipepSALedslBNhdmqqyyioSKxMdfkTpLUSJHSlspY2rOBi4vY3trOhX27kCN+6im2tiW+0PwShtzZT8VD0qYs6Wm2i6ottKT5QCSLBIBNu9u+DIEFT7z0NplyXKjvRnZESS9cx1lClOlbgQkWDqtRutYIHe2FbWXq9Rpr7exJblRnLRykqjqSexdaUtY0WJKCE+ckbdiBworMwV2c3JnT0KgIVvuJlBKYpR8qwUkaVltwtqJcUL3Cri17uEC3VIPwx5x+NGy3Pbi1mQoNxkIhMtlIC1Olxwr1AOG5U2As/UOfTBaDCXTPnSVibDjrVJIsGGnNQbJU3rN9IF7WsHCPW+FCi6iPFZ2Haa8lDsV55Q+brCwQ4ohJDYupW3dvi7gvbt6REjUZ5TM1nxzFxFbkBbqX3VrFwhLZJLfdJHBQAe+hROIUS4BLpkGYawiW29JntiP4J6LIQ4YSeVNqClaUjQokW1AX0qHphujU2nZdQtJkilxy4tIYjlouqGrbVpSXlNoA1EhQQSARowGBEixmzJVVHKy3LUsB1B3EuAXs1uLWotiwUpd7kBwm6PIcNc112TV3ZNKjMIjwghtp+NHdDQRrCkrPlKw4pwJCAoAmx45OIl8W6VPzWqa+qUVRzBf1ty3g4WX9oLUnWpVz8zhTYAG62VAg2VcApTVNlPRaVBWUrdC4MpEYm10LvcnVZZIGnTco+WBc3uXFUmoxKtDpsBMhuW4XAlDLBQlTqwl1SFJdsgj1II7JIANjhEidammnsPOhDjniipl1bzzKAkgttg3O2EAJWLHR2Cj3wcyDkoitTWGGWo9Up7ja2lveHW6GEvvoKk7Z3EqaKwjU0dvzjUg+a3HHXKjKqS2UyFs09whrbJIaUQ5dlTilg7hBbbGpzQbazZIuCOkS6ozUHFxn0GJGLkWStiAtZSX0ghalBJb9L7ZueHBwDwCofh0JKZ8upiYpiWhuHJcD2pwrQpzdS2BoCBp4ARfc9U25iCd0zqaUQ4jSo7zUMh1UVc1ZYIbJcI2VpsgEEC5+lf9u4Ab6p4CfAeehuuhlcZtt+ZC206XydxLe5zueexKVCxA9DgUOU+xRGH6W7VkwagpBksux/KXQjZWATcF1rcCwnQBoNuCncxG5teoLNegQ3WamiRLcjyafJizRDjhCL7niS2LnU2NRUAQNIuXAVEp3syif1QIrUl2NPXUJ3j22FMtTpMfsCTZQ2iu7dtQ47ellXKur06dBzPDhVX8Kfi22gwXkWTuoUkoO2Qb33HLEC3r3x5cWNKjKMeQ/NgMQlogubMd9MQBPlKUlVkNpsQHEucrbuASdGFTzQplDis1aZTlqj8MoS2pRZcISojcKAsLuFO6neFIsgKKsOokaJZ2l1F+mxFoWPDyIqyXVS29TdnXVCyAhOkgJKwbm2qwJHammm1KAuJEntRXEzmyYzri0shBKrDgWQD+0PC/b1K8AnLkwI82fKnuPxpCCXExXnEuJDjbimy4xYENlGlW3bspu+v6MBpedKIinynfxWnvUfVG0Ow4K47LcZbIZS42NJIDgdBKboIWRzdWB3lEdFgUtkR5bNEFShojOtvvwmi088+62bOBtZG8G/laEg7fuCu5AagxXHhB8CY+iEJhaaaLrcezYT4lxLZJ2kfs+FbgupaAQADh1juQUVLwkaGAgRttCUNOKjqAUW9t5xq5DTyxZHnCE347Ya6eoxodQpsapOBRB24klpMhBQQ2HFpcJUEOtlISeAAFX1EmwXMplQmaTMiTXzU5jpl0Z0NslZQ14Vbt1HU2tu6NJdKioEIN27BY7cajU3eENbrr54EhDYWUjXdRSA2hSGxoT5A4As+pFkYKiJejKNNeqT7SH9zbjMLbSdgjUdv5usISAkjUbldwjtwrc8dBMugrrLaxCFm3IkgqdacILilvEC6z5dSLIRaxALRF8Ed1RBbm1WFOSIs6qMSJetqKyg7ZCCUthJW2FOCx27g6hxxexJbWjT2aI2xPqM2MhMCQ4mIJaJMZD6JSlhezwR5iVi4srXcpQCSPLmOGKlqAh+ZT6pCLjSagdRcC7hTjTgSNsuLsCFErUNVr3vghUhmoMQHqY8xHdjqW20leyqT4ZDgDay4bCTcqBcUqwBC0AkA4GUuTZlGYTuc6YhhbsZdVjyGW5LTa46m27eZQBbaCfIkJDgWLeQgk8Ym1Lpkun12FPapcuG22PEPLMl15sNrI3Fl27IcbsFeQawoKAIPfDQ/REJKpi1BKEP8Ah5D7YcaU5rJJ9Btgo7EglRC+3cq4lRebcpcFD8JlmOd5p52IXQwhyylA2N0IbBUTcWslRP7mGS/aRYocShzg7l6jm0d1EqDU0yo+yF6OyhqKEOA3VpTwk7fokgqHUpq1LgwkvVGNChyy40FtOlkC4eudDm0Vth5WsKQQQ4hYTckE9VKlLWhmmMMPPVNl8qU62m8do6QpvbGouIsk2Km9ZsCHBc4WMVmdApy35i6hvunw78kNOKTKLG3tu6VpVrXck2UbFBSO4wRZRJanS1yak45Hdkyw085uM6dLCZNypI2FjzgrICFFoXXZAB1cvRzTXhlaFlWLnGoRokSauChMUtsF5oNJLad9nStWoqIFhe6bLBuSGGrpmTJDlRjssPokqEpSUBzcjNBStK9TiPIhPlUD2G437E4bmYkisy4tNodBRIdjtBaGm3gA+tCtwvJcFmw8lCQPMhBF/Ibq4RwvoFbzTqmYVJcr0x01JUZltl9LKGUvBYZ21OPApU2s3Ve9ieQL/vgtl+G8XKOmq1Nza2Gm1OsrVLbcW2AXApw3C7lxIbUCAsd7WwB2DW13emQWkocK5sWOFhKm/pfdKbEuaC24LEgkoHPLeErFefqjTNPXLkLTHacEQypKGktNgl90KvdaxoPy0qWb8WsVXFmyV2ydEohCnOLep0ht8tIcZfB06kLuGlFJA4K1E3JBPpa18GPtNUQPrqEiaKjHReEtpovMsxgTZO24oAaiCkWNgb2v3wkD9bipkU16px4cd9bDu7JZca23EIHzCyAVgW4bF9BClHuRhujU+OX49MeqdORAkO6SxHcVpIQAUpIdR2K1KTwsoF+/AJyW1Tea74ZDmmBAqulmM1uHdQvbkEkXZ+WDzxe6T6Dt6uMJg0yiLREjQ9Ue7Ty2GU/KCFKcZQ244NbZCCpeknkD1INzWGaPBcVvUBMXcabMeAQWmmGxqFrAq8msK8yfmG4JPrgcRuE1DMwpQouKAYbDiHkJ+UQ4oqB/agbZ83YJNxzfB3T7JQpMSJEjTJLNT8NLXYOvNMyEtuIsoIubIDugOG5v6GysJiqaJTiqRAmworhv+ISPkKdcQhzVthu/kuebEghINwexECA7UGXKmiM0Vtl2Q80yQ0lsrLZLhcWSgmxFiq3CrdzcHuwF1xt78Khxg4gbITHWuSXRYEB4tA3cF3CVOOW8ugDuMFIkb1QojTUeJLjDxTcZtKjLbeTpasNNrC6AEK45sT9r4CxX2qvLkPQnW3ZAacWp1iMElzRpb20hBCFngXUBc+pNrYOcodOFJTWHHFySkoZeS/GeSoaAVONp8oQDdTYAUew5IJwFuU0+uGunwKdFcgL0hU1mMmMzYXSQCCC4kJJCg2F61EdjbB7ttFAi6c1KU7+IzW4yWY2ttDceI0FpuLqW4QkglNvssEADRxhXVGqW9UFyaY5Jqy2w7FpiHGDHccIAcU42ASNBAVoCiDYkHkY4p6pVmXKmNKfkuTGt559jQ2HkIJI1BCeLi/A4tbtzhfR46KnAiRqcN2N4VbkxElB0xEIJIJuCgWQogH6/MsC174XOrO8llNfpM6RHgVKoyYZRFYSVOFIjtgAXF/rd720jQfL9XbBEIVKnUwLiyxXWj814lSHk2R5UoCUWWhABvp9T+9xhemuuLgwozM2VEYbmbS2pEBjYbAbCSHEC7p5CbX8lgeTcjDc26uJRQxLrMx2O23IW24Q2opbAPdlztqKuAn6fUXIGJlQDeqYW6pPenxHGYDsSNt7hTLhBlRbQrunQQ5YL/tAXuCB64cG5DTsh7acXDcjxj8pxlBS4SgpLTiio+QoPPkJItxyDg2UUIjzYDyDTm2GgiSmSWW3ytAUkFoEFy233uVG4PINhhPFpTzLUqmwGZC0+Ks402CXXXNsrSUuXUHCCSSnji5NgMM4WQLVyn1CsKDkqk1CPEZWCygx5RiNsr2zubdgBdKCQVJHHIuOb98atxbaokR0LW34KMY0kqO42BwXLX7jVpNjY374dm3IUKG3B0SW34wWGYj2iQ0+0STtOIQLhZWG7Npve/Jwy0X8WnJenz1mFERuf1ZplPjnLthYuAU6Aq172KAAe9jhLoE5jZL36xNkvU5EaDVJFUcYWHZUlzdTtXJCB5tYN03I0dzze2HVESlUtT0idQFsKdebLjkN1CQ26GhqWskKvfynT3SSQMN9MqbS2ocOe4061r3dlRU8C4QPm+gCE8WuBc++CXaXTautkzKk/IYUyXruutsstAkg3AuQs28nBIJFgcKxgZpe6ulkz8kjfVmOdJS1JpUSJTZEZcyK0hxxzaAVZK3AhJJAFyeB2Pe5w5UqAtltqY85CdnsJ8/hXXFKc8/7QhYNyO9legPAthVQojNUktwaRLDjbqZG4lxxplku3sj0K1hSDbVo49r4T06kNvTWmoS6g6GF8BTJDC2xcOt7xsSCD6WQQBwbjFir3XoaEobWp2a2hU1zZFitVivkOdgCOCOPXi3rhBOqy6vMK0zoTsjgss+Fskr8tw43ZVzcnggiwt7WUVliOqO1BeqUKAY7SIgUdUl2za1fSlZBuQP8AFkA8XsMInsuoU08434Z5pvbksyWoTiJL11EK1EqNrDtwL/awxEiBNTJS4/OeqdPcbiPIiutyQUKSs8hIT5r+fct6i1z7YcqM1ImPP7kmJFjsAEvLc2ubCzgACneRqPc9vQ844zEpEPw0+pUWLUGm54EXxagtxLV0hTZGq6O47CwI7gA45WVGTDXJjVuTS4LYBixXpbRu2skmwQlPk0aDzxxbm+IHX1S5UBEyHR32JBhCWlZQhlyK+sKIRyppNxwg3PcX5+2FlIdbhpCJDEQTHXCl1cIIX4UuAqSXHAChxdk2LfywOLE3w50iQqKyytl4zHX4q2QHEhRf4IuG1ghA57E9/QeqN54rhNs1KG+7D3TpYdSHmtdiW2vQIcVa+rk2wMyOjVyOhyPMVKZi0xLsphxkuIdWlohBKm0qKLoCCb8a1LukX7XwdTZLL8KPT1x4cdS1uOOsyIzJb21jlbjwuCU8gEkEBw2OG+rU9ZU5FrMARozt3Y0diUFBjdIuogAEmxIIPPA78YQ06dUGC/RRIjNOtvBpXh9ply1/KvkgFdkkIPNr2IuL4jUSpE87FmUdbVBgNKJdiRG4zhstXzTquoIsvkp8pP7OwA5Jw2qdpSKa+3WZ7s9t+U7JU+8uMpZIGktkA6Nu5SopTcXNtV7AIWXItHamMRoa46XWXA3BHh1KK1oPzHXnb6HCUgjSADyBa5JBlZmmeCL9UhrEdbQeIhynXJJ5H0qKChAJIK+CCBz24sccvdQTtHnIzawigkRlUphlx1SXkLitlCBb5aRYoNgQhPPqbdhhoptLo4aVUqVMQ07ERtLVIDpkqWRYIZcsQtuwUSQji3qTbEkWWBU0Ql1ByWmI4JrMyZENlLDVrNkkL2ypX1JAJCQbWw6NsxqlM0ycsVCqSI0R1suPOFSmrIu2LX8gHFk67K5vbtiafaVabo7MKJA/Fo1OQ/A8Iw4luQ80Apd7OXIB1rFgPl2PCydAtdG+7Oh1BacvyKpIbmFDvj3kh3gIPKVajbUSpPIBsAOPQqo+IKDUnqYiS1HbbsmQyhh4uAptdQ7uayAATc89rCzVWKVSU1yQ9DnTX1TQ2yzLjIcYbaIJU6Uq4LhC7jVxcnjtcq2xTudlS2oNvVCXU4MPL1MXCbaWZpfktyFFA4P7xIcToURpta/a1hhxnx3vFtJqryQ7Na3JjEebsvtO6QktOJukAbYA8oFwRyecM9WXRJ0OlxFqfk1KS8WUKbcKS9tpO0opJCATcAulZWpwkW4uHJuoQY7zJq8oON7YbXICUvyEi7hK3G128/mICuR8vubgYLXDMp9lObsLK8FEWbPManxZFRRObh2ceeS2LlougKI0qRpI450k3HfDC4/uOonPPuyJjQXOdS3tPjaI1AhSPI2QgE7dr2PN+TgcadAqDJhvQ30txmHZIW0HEbjdjrU6o30cm1wDcDtzwbAcdYpUh+BXll2MtZpcp26C5rNyjQBrbGjyhPJUTyUoxZdIlb1Ohh6pUqelDcqZFdXEbcca2mn3NpTylJcNhZBADhXf2tguDNebVK1VGoyI0YIc2m3WWQrjS5YgKuhIAAc5uVck3vhXAkoXWVaG6al+bAcixIyiQmNIICuzqLrRYGytfdVgTYYKcfXAp7SKoYCprTZekNuNFDMU2PlJQVHgIBJ9NR4JBxW52ZP4UGeESH49HRWEM1SW028AtpLDDYIISVCwLgCA59JQO5NuxT0JE5NIjx340lcWWPHvLlOJAkLJS40TcDWnWmy0iw8wsfrOEEeOtUaao1QRguMhlx1lzQpazpJW02Wi4dS7W5HN7kAmyhD6pUR2gUqYtSnQULSpr5QuNt0uqWooWjbbDlysBJIPe9ixHmhUZp1MxmRGoIlPHbXLitB19xlpZBbJYW2B9biVWvayRfgYMeYeqsJVVhpkNQ21E+GYZtsrW5ZLnbRdT2kkEKvcHVYcR6p1BcLYVmFSFSJji3XB4pUuQ8ORZRRZsDy3uDYhwewxK6U6mvUWE49W4b8dholdPkMGW5d2xsNDgCLFNwSiyLfvX4c+FBuyWRIdIoDiaW1nIl1QbZf8TBH9aAWNQavwj9qW9tWgm1yTYYaZbcuExPo/hYcJLQuNt6SpgR2jpcdbbkHyc91XIO3otzyqiMU9NUerP4FMqW9KdcRFfhJdStxdwkotYNodJUSUgFvTcC+G+O8tmmoosZTE+ctxhxT0Zxp8xIgsoR1BwFwIC1FwixJ03BwGlANRi4XiwmO9Sm4bpkLhhUiTZzQFJcuHNR0Np3WQhJsOe5QTZMuFVI8RTLFOUiU5JcX4d5LambgtpO6NsIDgKW7crA44Atde6lDMJUl6PEaXGfmIkxw44680EBO4t4hs7gJcCllspuhtpAtpIDSmoyKI/Dqf4fG8XElFxa1tLbDzDiG9gMq+tRbW26Te2k7az6DEcVC1NdGzO/UYs2nVXLDbLjkl196YmKApawdKkuuL7G4vYAHlXpa0hqFaqSHGvAOx5FfW2hyJ/VwnYbDVkrcbtZeokkE3XrsBYG+GGRSqeYVNXTahVHnZg1NKaJS2y6gj5zqbXFkA3WQLgpPovCqfHBy+g1KYsF+oFpKJhbKXG2xqIUkXc1uAAlRI5SeCOcK0Zt0XHLskNMM2p0lMhD+2lkaJLSHh5JK73vdN9vhR8ptr+s4ViUzPUt+VAQ1CYQRqjp8S8750+d4Fd0BxZTd0drcJPOEpnZjkzH1JjRkQ47yG2m1sh1rWCoNttOIFkIGvi5tZQJuSMHy3m6nU6c3IvUXG5Wqc8xMPzfOS4osICQ/ZagCdZPmDYvqBEd4lM2YZkurFQzZDmyaJIYk05ht9EaRNbBUwogJsSCkuEEhsblwQSOebYT02SiLLlUmo1GXHECHIE12CyS44tvugbJsg8Np3L61E9rjBUmn1iUHnoc+OWY6SEMPOFpxRQHFE7VuwbSFEqN7+oBGFlJqMuhtuVHL+mK9JRrWGbbydzVpdvxtoPmbtda+55vfCNNtEw7yKersOdU33qrtUsvsuBmCw0GlRQBqEUuFSkLcU4kBZsSAysG174a41RceMqvO1gy0tMtSlwF3YpzKymyTq08/S3bSOATye4e2365T6YzC8HOjykuR2ExlLQp3wbjjq1Ns7oK0F10DcWOBYD3JQMU5FNKalLqMtS4DThW2q7YT8wBQJupbi08gEXvZwWte87ylwm2k5gkPIieOpofaejLeW2pvUqfcatttwqCyARpLnI7oB83LnSIr0mooZ/ErS4xkSmm5brgedWvbDSW1jhBOogItY7bnNhbCypxKs0HlrW4lyE5srbShpYeJaaaS8pAsNbiFNhBbcN+TpGDZ5pVCeMIVh2Q81U3GnnFEFbC734JHkbO2LWAJLZN0gXUdku6BGU1WIMiMqp1DwciU+IwUtfyy3pSysggXaCHrgA8gEBJ4GDlx5y3arLqMnxbtSWIyVmc2WnHFuJuhIb1FC1HcuVEo54BIFyN2fRoCJ095oVKXJ24bbURUhQbRq2dRKbWI3BYjXrAINucG0xupPMfLgS4jrTq3Gipqz0dx1xRJsQQ2PMdZCCuykWsjBzZUQio9PYpiWI9QZkpfrkWwUkhJbaWkbe4Cq9yGuSUC2o8DvhoZhZqgRIj1IRUJJ8SLMNujwjUvRtpLDRvcpJcB9AXLcFJIcnWaKxJdqrLbO6xIceQUHUGbLVpukcOLKrgqK+EXRY2sE+UV1mHWX11aM6VNONNz2J1P2mg6FeZh9xZBOlAcUFJuexNzfDt2UCPlUdbjq4c6kx20krZZfWlB0hDibtlwatBOrtcDXZHHJwpay3SqM48moT5JecbcaZaTIaUBGL2024pxtQsPq7oAIUnkgWwmqbUHMT0Ci0JhdRpslkRIsZtSXFpjtlsuWNwsBQKbqsARYC5bUcJMu5NoSK1rbozsCPZxmSxJRrW0S86klwIO4fIAkp0WPBQb96u8pontqJU3Z7KaazHLktbrTkpmAXXWwRpdCtAssbmlSwkJ1FRX2CsN1Ro02TLTLZqrDZjEeMnPGMlSWEWbCyLnbLiwCLEL0KFuL4NzbTZMCPOTRZsJlxx2PIdZU44C82vcDanNx1W3p27Cw45BPAwUxmZEjLsbK1XrNOjJTFbQ6pzdUHVuEkam7JQbLJ1k3N0mxPq7f5aiE86/sR226mxNktuLbeC2w0+dC0pCLi/HlBPqUJF7k4QS5sudUayiPLgP1RdQbntvLQFPR1hClFq5FydCg4EpJQdJX5LHDhGak1ByPGqrcyQqWVrmlVnQ4628kPF5Swpe4bO2SNaGz8y17YOdXSosaVHlyS28qS+3J2jqSkIatoL5UN91SHAlu5sEXBsvjDd5DKEkpk6Sgw3ZFNfSHnlRY6fDuKJAZ3Q8BfRYoACN0qHmsPqIBMKoOMs1RbVPjRHEOOxpSGUvKTtgqsUOiy22xutpCLfW2LqueWxU59iPCy6HmmlsPGSp1MgEq1vBRSVDVztkXdukaALp4AwpbVVKe9PltLj3kVIsNU4OocDjgWUpCWvKFoAcbUFKKdIbUs3HODlCF8qWoZoNdabqVBlt1aWEAuokxHFLhnbUVNshxQABIaKHCvXcC2gDDXCOqguSY7jcKbHV4ITHnG35BW62ClepYIQRZxsuc+gAGnldNkw7BcxukNSoawy4hqctx9og2U0HnDtu3WdsqcKuFXBsLkzwgmNynUQ5kx+NT9bW2EoabcRqBCSslAs1q0XRxYDTcC7J0mh06k09oR4lYkOSFul6WucW3nGGEX2yWXBrtqbuhvzc3I130Y4iE5GVEg/MuIodlxkSkOJ0NoSS42ArcQjnUAnzjcN9AugHOy5GW2Y0mqx34JmLjhp4vB2VLjoJddZQSk6LOFfzHP8Yo8jULcanM1ZwmAytZqbawgiOqI625ru5qQLtouCb2WblQ9Cs4CXKEkpTbzmW3pkZceJIpoWoyGnxvJQW1OWToSndKrON6j5wghAvwsgpUaM3etJhSHp0hV2mA88+GyttTgCSVgk9yg2udJPJuQtepTK8tw/HZeiMNIY0CI4zth0tlvknUCCFoN0gpWSAblAILZGkRI9Elw6bKcbhOuuCMmNG1vFu5USXCPOAVbfBHkvbgG41cpoU8JDS48qYaxIgzJEZuWy5U5MpUdlvzApSdG45ezhKbAWVo9Lnq5jk6O68uqpaYqKWEPvrhBxLiwVPBpxJCShw6gPNyAVEn0KpqWy7T4b7EKn02HDg7rsR9pzW+6hI3FK8+soUC0pCtBRfVYiyTgJgR31llUliPJjy13aU+0647HdZBa3XFq1vrKwqxbK7Akm+peFb3u85ByPdr1MhxGJLDMlS4SFtCa1J0Jl6C3pD4JJAKbKXcIsTtm6ATiOPoRVqkhpFYpy4E2I3LDC1mM1tGygUuXsNh0KJTe/Atq1Yc61UnIdMjvVjMUZKYqxFYjKcLcZ1BaJDTZWQFlRQ4oK4GsqB+rlHRn2ZURutmiUt8U6Y65JTFpiXGAvQrjwptoZ8re5tDQVi5TwDixRqWRVP1aIY7j0BUoqccfkh4CI2tprS7uKLljp0pF+ToJsOV4ImGdmGOxA8EwiWw+4jXCccbfeJbupXykaFhyzI2rc2RfjjDkI5ZluLzIw4WZDjhaQ+yjbluLRZ0h9sghxoKNiBoseQTY49UKhTmqRH1JmQpbclDDsxbhacU4gJ+am7nLjRSSGwU2Fr3IF4p9pNb0J996fozAJkgh26pcNvwb7cflIUG3dtx03c0NkIAKu/deOVyXILkqUiAg1OOptrZeC48wNNlrS8kkFtwBCQ3p9EBsKF/mHr9Xg1xMmkxswtTWqQ4JE6MzpkylNlxKXVlQV59NwD5FEFR9RwXLTHqtRkqkSJstIbaW89GiWs35Q0SFueSRfdC9SyBZZN9N8TKo5PdKq8p6C3UqKzNhJnvGBAQGY7DinVt6nI7e2LN+RLRLXF9zgnnBUl7bpkp6jvUsVBiFIj7jDrbwabQ9dsoKFEocBUeeCQpBH7NWOxYMsQH/AbSoMlt1+Cw86JZlObgaacCiko1gEkltfm2xa6wcJ6n/RZCIjK4NZdahtiTPlre221Nr+Y8XGWzc2N9FyLAXAvzhXeFRqKqU/MNFYUimNx1yl7LDSHqk0nabU8XHQ4hjTdpRdB1OOIIKu9rgdmNPOuPT4CJEgJkuOruQ09KR6htJ29a9Drg0hHBAWLgglYzH8DXwtuZGhGTHW4w/CcWWCACAVJCS4HLEqCnOPKSR2s2OUyoS2GVJEiTMaaWJGyGS60FkODzhJ86tLbulKxqCdY4HAy7FRyDQtSoyIiVnxtLG9D0lt4uBwBRe1N6dYDZaGlKzcm6E35w7PNLfadZQ+6h6Iy1GEFmIklm5bUHGi2bkklQuCSebk4itbyZVK1ITPjePk7EVDpFQkhiEI7mpTJZU2FFayXQLAIvt6OyUHD1TI7lfpFLmUynhl5x2xgOgMvxSs6nn9N9wNNaVBwuLG3qA+5YpUUmP+L79SkxpdSizQ4ApLxaeNyr5SQ0DuWCUkbiEFKE6DfuOMxIC3HKb4+qy2Iw222W5im2FMAk7yGwkrADTbaj5DwoLRhzy5FahLCpjy6aKW6hK3ZDLjrLSHC5ZoFtSewSFfWBoOi4sQW+qVkuqqL2sw21xEL0rkhvVGDQKiXL3NrJ0AguEWtxbBUSh6JM8ImTBaZfhradmuCE0463DWgDeJaF7/4uyisi99YBCzhuNYRDLrzrMdtsaHG4shtLyXC2kK3U2CQgkH6k6D5bI5vhyXOl1VUdtEazbUlbirQZDukBzbutkqCEIA3TYaTocFhdXIZFHW2WYkmtIedhyRGVKYe2TcFQte3yrbQIGtsi4Qb2OIog0aoqE+MlTkvxRltMyXx5glq7YIeS2rWOApSzbjv9d8I6J+LLpr4prTj0+nMyXXWQ200vQ46lKnkrWLpAIaSGw2or1AmwBwrXFo6VxHpkmWgyIDrbqxFCVNPhwpS0pwN2daUfKu9i4b3JsMGIplQdfaQy7PZ0AyhFDZJLbpCkhsgbgIsklRKEA/vDgYl7I502yZCxUqaxDpD1RclpjiCVTWojcdbDQI3W39RQ2q6dDgN1bZBvbRhK5QHodOYhVGoSVOtqD0l1TbwZWtClXkBu4LpKxpKTeweSV8XOFYkRXVSp9SpdDYFTkrZ258pakl9wnlOtQ0FpCrcFBKz2tbHW4ExxoOR2dlqA4X48liWDurQ8kJKQSUaLgq1KuvgHkKIEd1QR8OZMqlTqUxTKDMpjZNQXF2f6qH9IWtwIJAQALBKQSLC1xcgtxtEhySqVTYbDO54SO06889HSi7SitxwqbJWkqFtSDwnyWBuWRqZUZ8R+VMgTUq8WuR4Jxt2NLb1kgkElKFuBem9lpQoq0I7jDkmmtyoiWF0V1pKIjDLjwZkbVnT/AFkvBYOtYRwRb5ZT6hQOITmUVhyP/aZK/wB4h/8AYmsdzj/sKoX+9TP7049j2M6tTFQv9jlE/wB9X/1beIlkr/2lUf8A4/j/AN6cex7ChONlJGv9R1D/AIf/AKFYiHUv/YjD/wCL0/8AQex7HsTkl+yrOgf7LKX/AMArH/VjETmf7HK7/wAHP/VnHsewWbqN3TTmn/WLLf8AwWJ/1KcWTD/2IZ1/4fF/7Rj2PYU7KwqMyP8AWWpf8HX/AHpw8Zy7Rv8Aimmf9a3j2PYdm6iZal/qzMX/AAxf/WJwXSP9Vw/+EzP+g5j2PYd2yieGP9QJ/Nr+9OHFn/Wimf8ADKj/ANUnHsexU1Dmo7B/1JU/9y3/AHN4eM3/AOstJ/41X/2XHsewBuk5odH+mr/8Tzf+rThJT/8A2fo/4bF/6nHsewSi3mnaZ/q6r/7p7/oLwz5O7H/dMf8AV49j2Hd4UHckU19Dv5j/AKSsFZ6/9oEn/fY3/Y28ex7CNSqXVv8A1mR/xSP+rbwn6v8A+x7Lf++Rv+zPY9j2GUPiVc5Z/wBhb/8AvY/6DeF1E/1aj/hT3/YTj2PYZnhSu3Sei/tah/vLf/VYmFM/14pP+9wP+rkY9j2Gd4k32Ulzx/7QG/8AhX/3crC6d/rDnH84f/XN49j2KjuUG7KOK/2eU/8A4xb/AO0qxJszf6mk/k3/ANSnHseww8KDky5Z/wBbKJ/wdr+5/HaP/s0b/wB5T/0V49j2AN1aNk6Uf/2pUn/jVf8AoxBczfS3/wALV/1Qx7HsXHZVNTrF/wDaDUf98H/UjD1mz/Xipf7qZ/2ZWPY9itu6jkfWPorf++xv/wCLiLRP9TSP+Ctf9aMex7FjdkClWff/AGiM/wDBWf8Aqhh2ndqL/wATN/8AUDHsewysCdaZ/sdk/wDCD/1buIHkH/XzMH/BVf3px7HsVO3S81YFE/2Ct/7j/wC5m8Ryu/7IVfk5/wBdj2PYQot3TzUf9kS/97mf9a1iHZm7Rf8AfpP/AFrOPY9gjdR+ylCf/Zkx/wAb/wChGGvI/wDqKJ/vrn/bRj2PYZ+5QdshdRv2sX/j6T/0sBzZ/q2N/wDi4v8A6a8ex7Cs3R+ynWb/AKwVn/cx/wDonEZZ/wBUUz/hcf8A7Ycex7FpQZsreyJ/rdG/42jf9U9iMTP/AG9J/wB6pH/UN49j2KgiN0lpf7Gd/wDjNF/7RgEH/W93/fP+/HsewrvCmb40XQv9eaF+Tv8A1TmGjqD/AKjR/vdO/wCzzsex7FkaBXcv/t8wf8Tw/wC9jDJ1G/8AZ7n/AP4pj/8ATcx7HsF+yI3R7f8ArBQP+Lv/ALqcwXO/2VV//iZH/Vw8ex7CnZIFGJH+yCB/u2P/AN4DC7PH7Wd/uKr/ANYvHsexY7xBFTaZ/rb08/32b/2pvEY6g/6srH/B53/Sdx7HsQ+IKtJqJ/sjqH/B5/8A1ycSBP8A7W5H+/8A/wBwJx7HsOEW7pmzT/7F6P8A8Pwuz/3qn++xP/3dj2PYCZqSUH/YTUf+Ml/9U1h86nf6oyn/AMCpP98fHsewsexQciI3+ql/8cRv+1tYbaX/AK6L/wByv+849j2C/ko1THMX/soov++/6U4p7M3+xjNv+8H/AKYx7HsOErvEjswf7Gqn/vLX/ZTieI/1kR/xtWf+ixj2PYQKKJV3/Z9SP94k/wBxwqgf+0Jz/hEn/wC3x7HsWN5ovU7X+xkf8f8A/wB0LxG0f63Uz/jqN/2mXj2PYQ7IIjOn7TNv/wCM3/3Q/iR0765f/HTGPY9gclEgyD/7SWv+Lpv/AGJvEqnf7Isj/wDGVf8A+0DHsexComnPP/s5pv5O/wDZ1Yac1f6/1P8A4c3/ANmYx7HsK1Fycss/+2evf8Ys/wDWtYKP7ai/8LY/7E3j2PYHJKN010P/AFBN/wB4m/3vYaesn7b/APsWX/8AojHseww2RKfXP9kEj/iOZ/ecVbF/2T1z/hiP+yY9j2CVFZUr/Y7B/wCAK/65nD/A/wBfUfnV/wDs0PHsewjETuF//9k=" } ] } @@ -120,7 +68,7 @@ { "language": "javascript", "source": [ - "const tensorData = new Float32Array(image.data);\nconst tensor = new ov.Tensor(ov.element.f32, Int32Array.from([1, 224, 224, 3]), tensorData);\n" + "const [N, inputHeight, inputWidth] = inputLayer.shape;\nconst resizedImg = img.resize(inputWidth, inputHeight);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, new Float32Array(resizedImg.rgb));\n" ], "outputs": [] }, @@ -134,14 +82,21 @@ { "language": "javascript", "source": [ - "const inferRequest = compiledModel.createInferRequest();\ninferRequest.setInputTensor(tensor);\ninferRequest.infer();\n\nconst resultInfer = inferRequest.getTensor(outputLayer);\nconst resultIndex = resultInfer.data.indexOf(Math.max(...resultInfer.data));\n" + "const inferRequest = compiledModel.createInferRequest();\nconst result = await inferRequest.inferAsync([tensor]);\n\nconst outputData = result[outputLayer].data;\nconst resultIndex = outputData.indexOf(Math.max(...outputData));\nconst prediction = outputData[resultIndex];\n" + ], + "outputs": [] + }, + { + "language": "markdown", + "source": [ + "## Display result based on maximum probability" ], "outputs": [] }, { "language": "javascript", "source": [ - "const imagenetClassesMap = require('../../assets/datasets/imagenet_class_index.json');\nconst imagenetClasses = ['background', ...Object.values(imagenetClassesMap)];\n\nconsole.log(`Result: ${imagenetClasses[resultIndex][1]}`);\n" + "const imagenetClassesMap = require('../../assets/datasets/imagenet_class_index.json');\nconst imagenetClasses = ['background', ...Object.values(imagenetClassesMap)];\n\nconsole.log('Result:\\n\\nid\\tprobability\\tlabel');\nconsole.log('---------------------------------');\nconsole.log(`${resultIndex}\\t${prediction.toFixed(7)}\\t${imagenetClasses[resultIndex][1]}`);\n" ], "outputs": [ { @@ -149,7 +104,11 @@ { "mime": "application/vnd.code.notebook.stdout", "value": [ - "Result: flat-coated_retriever", + "Result:", + "", + "id\tprobability\tlabel", + "---------------------------------", + "206\t0.7470666\tflat-coated_retriever", "" ] } diff --git a/samples/js/node/notebooks/optical-character-recognition.nnb b/samples/js/node/notebooks/optical-character-recognition.nnb index b7e8e109ff857f..ce450dbce0a0d7 100644 --- a/samples/js/node/notebooks/optical-character-recognition.nnb +++ b/samples/js/node/notebooks/optical-character-recognition.nnb @@ -24,60 +24,42 @@ { "language": "typescript", "source": [ - "const fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst { createCanvas, Image, ImageData } = require(\"canvas\");\nconst { addon: ov } = require(\"openvino-node\");\nconst { display } = require(\"node-kernel\");\nconst { cv } = require(\"opencv-wasm\");\nconst {\n transform,\n getImageData,\n displayArrayAsImage,\n downloadFile,\n arrayToImageData,\n getImageBuffer,\n argMax,\n setShape,\n} = require(\"../helpers.js\");\n" + "const { addon: ov } = require('openvino-node');\nconst { display } = require('node-kernel');\n\nconst Image = require('../image');\nconst { transform, argMax, setShape } = require('../helpers.js');\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "# Download Models" + "# Load a Detection Model" ], "outputs": [] }, { "language": "typescript", "source": [ - "// Intializing Images, Models\nconst baseArtifactsDir = '../../assets/models';\nconst detBaseURL = 'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/';\nconst recBaseURL = 'https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/text-recognition-resnet-fc/';\nconst detectionModelName = 'horizontal-text-detection-0001';\nconst textRecModelName = 'text-recognition-resnet-fc';\n\nconst detModelXMLName = `${detectionModelName}.xml`;\nconst detModelBINName = `${detectionModelName}.bin`;\n\nconst detModelXMLPath = `${baseArtifactsDir}/${detModelXMLName}`;\nconst detModelBINPath = `${baseArtifactsDir}/${detModelBINName}`;\n\nconst recModelXMLName = `${textRecModelName}.xml`;\nconst recModelBINName = `${textRecModelName}.bin`;\n\nconst recModelXMLPath = `${baseArtifactsDir}/${textRecModelName}.xml`;\nconst recModelBINPath = `${baseArtifactsDir}/${textRecModelName}.bin`;\n\nawait downloadFile(\n detBaseURL + detModelXMLName,\n detModelXMLName,\n baseArtifactsDir\n);\n\nawait downloadFile(\n detBaseURL + detModelBINName,\n detModelBINName,\n baseArtifactsDir\n);\n\nawait downloadFile(\n recBaseURL + recModelXMLName,\n recModelXMLName,\n baseArtifactsDir\n);\n\nawait downloadFile(\n recBaseURL + recModelBINName,\n recModelBINName,\n baseArtifactsDir\n);\n" + "const detModelXMLPath = '../../assets/models/horizontal-text-detection-0001.xml';\n\n// Initialize OpenVINO core and load the detection model\nconst core = new ov.Core();\nconst detModel = await core.readModel(detModelXMLPath);\nconst detCompiledModel = await core.compileModel(detModel, 'AUTO');\nconst detInputLayer = detCompiledModel.input(0);\nconst detOutputLayer = detCompiledModel.output('boxes');\n" ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample/assets/models/horizontal-text-detection-0001.xml'", - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample/assets/models/horizontal-text-detection-0001.bin'", - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample/assets/models/text-recognition-resnet-fc.xml'", - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample/assets/models/text-recognition-resnet-fc.bin'", - "" - ] - } - ] - } - ] + "outputs": [] }, { "language": "markdown", "source": [ - "# Dowload Image" + "# Prepare Image for Inference" ], "outputs": [] }, { "language": "typescript", "source": [ - "const baseImagesDir = '../../assets/images';\nconst imgUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/intel_rnb.jpg';\nconst imgName = 'intel_rnb.jpg';\nawait downloadFile(imgUrl, imgName, baseImagesDir);\n" + "const imagePath = '../../assets/images/intel_rnb.jpg';\nconst img = await Image.load(imagePath);\nimg.display(display);\n\n// Resize the image to meet network input size\nconst [detInputHeight, detInputWidth] = detInputLayer.shape.slice(2);\nconst resizedImg = img.resize(detInputWidth, detInputHeight);\n\n// Prepare input tensor\nconst inputImageTransformedData = transform(\n resizedImg.rgb,\n { width: detInputWidth, height: detInputHeight },\n [0, 1, 2],\n);\nconst tensorData = new Float32Array(inputImageTransformedData);\nconst tensor = new ov.Tensor(ov.element.f32, detInputLayer.shape, tensorData);\n" ], "outputs": [ { "items": [ { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample/assets/images/intel_rnb.jpg'", - "" - ] + "mime": "image/jpeg", + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAIFArIDASIAAhEBAxEB/8QAHgAAAAYDAQEAAAAAAAAAAAAAAwQFBgcIAAECCQr/xABgEAABAwEGAwQHBQUEBwMHBg8BAgMRBAAFBhIhMQdBURMiYXEIFDKBkaHwI7HB0eEJFUJS8RYzYnIXJEOCkqLCJVOyGDRUY5PD0iZEc4OEo7PT1OIZdIWVJzY3V5S08v/EABwBAAEFAQEBAAAAAAAAAAAAAAMAAQIEBQYHCP/EAD4RAAEDAgQDBQcDAwMEAgMAAAEAAgMEEQUSITETQVEGFCJhkRUyQlJxgaEjsfAzwdE0U+EWJENicvGSouL/2gAMAwEAAhEDEQA/ALv1/D+mrU9hULp3EdolwdxxB7qgobOdQOVq9Yn/AGbfA7E9TW1jqr/pXq91b7qqW98vfUvOSA605Go22tbZ8fa/71u0J0gbRatBg9DSOJp4g2/QWR3YjUztAleTbqqJXn+yo4ePsli6cd4tpARlBdepaka8o7NuzTvr9lJei6ZdNc3GCqCDEesXAgnQ9U1U/LmbejyQRHxt2ArQCTY5pY0MTOXlje37Lvi/TBBu3HmHqkI5VFNWs5tD/I0sdBvtZr1/7PH0hrvcccYbw9X5hH+r3mWjpz/1hDfKLevIHMmeeot2LDNG081LjnmF4w1HoU+kvSNvLcwEl5ZJhLF9Xc6c07kJqCdieXOyRW+jP6Ql2LbNTwjxgtKVStbF0vPp2M95oLHIaz8be2a6dh0/asNrHigGxddyXO4cy7spSevZAG0O522KlxwvDauwBxAuB5X77wriKhGgJrbreZynX+dI8Nv0smJNQjMhyo74ChOpMg7wfjrb3dTcV2IENU6mo/7txSPuIsRvLA+Gb3SUXndrdWgjVNQO1HwXItE0juqQnC8PB61CApRTzJjWYnYeVjSVVIcbUgqUG0zEcx8iBPj8rex15ejpwbvYZavh1hpfPMq5KNR+JamzarfQ44D1qys4CudJUCO4w40Pgy4gWAaN6Mydo3XjxQvOhKmlPqH2rqTmBIKsx8OvP4dbKdO9UKZpEthUBgd6J3Agx0j7xb1Ir/2fnAV9KvUcKIpVKWVZmLzrkak6+264B8LNmu/ZwcL3UoRR11/UobASgNXw0pIgR/HSE7f4rAjpJI3XIVh9RG9tgV5udqsOPKIWpIIQQB7vdv8AdYBx57KEuKzSDlVlIMbaj4cuYt6BXn+zQusqUq68fXyyCdEuUlPUwPMONTy5WaN6/s0sZNoSi6cfMPZABNVdKmwQDMHs3XPHkbH4bhyVbMCqYKWorQS4ggGCDodjrHn15j3W6Qp1SlpDyoEDmQRIJnny+/xtaW8f2d/GmkWVU984ZqEjXKF1zZ6mJpoHxNmxePoMekJdqHPVcM3RWZ5ILN80zafCA8pB5DcWYsd0TgjdQGaioTT/AN6oEa97XKTy135fW4yHKtDqU9qojX2hsI/W0oXn6KvH+7cod4V3s+E/xUZaqxoJ2YUvn4WbtZwd4uXQpa704Y4wpUITGd25KpCTvuSn65WgWlTDrpqIWt9SgQmASCcp26+Gg+Xw7DrgCASMrgkBJkjToDvIsM/dd6UKEt3jR1FK4pRSUvMls94xpI+o8LcoacU4nOAvLuDrroOn1p5WGQUQboVLjhJSAqBqoAAxHkd/CzUvWqqWcX0QQSlRQkE7QIVoT7hpvoN7OtCVlRcSgGO6CI1Hx8febM/EBLd+UbqUhQUJCgrY5zEfAbdfG1eUXCLGdU76GtU7TD7QLhwpTyOsdP0sIagpAVmKhM6HfQxv7+nzsVutRZoxUgEqzwnUCNI+P11sKtZSlDAQo9p0AJ8dvH52hCLNsiyHXVbVUj2QpIA1OUkHnEe7x5Wxx1aU5t5IGYAHXff3ePnbSVIUtXdByawQY168v0Ng1KZLZW6gHPsCI1nQa+GvWxiNEJCLcfLhUVnQDcATvt8frmH2rmaXHezMlUeB6Dy1+HhbboST2crlROYgkDmdJ32J/rboM5kkmco7oGbTLO53Ez91hndSC4Ln93nUpCiQUjmPEbmJJ+duczql5lgpgbKIgDpHXzsL2S8wSCpStyRtt+Y0Fu22HUqJSQRlAiABtM6npPhaBcNlJYhS0biY1OxA5HXccrBPqU9TrSgkSkwCoxrET7wDYyEOBScyUgqMZYVqCdRPlpvbtxhwtrDgKwpBVAJ9xEjXn87BcbKYFwmRhmpeYXWoOYlRbKgBEjvEggeeh56eVnj6rTOhL4ZCAqVkEbp5nx0J+PnZn3W6tqvrmEtnrAXEEHrM7E+Pv0L+u3Mq7mDkOVxAmQdT16fXvsB3hddFaLtsk792MuZAplQJ5SdTpymenwtyu629crzomE6KEmfPnP1FnClpKEhRCEyARrAAjp7j9+lulst5R2bWYkTMkCfGPhZuImyJtm5/9iioUdBmBG35/jrYA3E4SSstqBHdzp021j3b+XhZ1GmaUSkgjpG5HXTy+t7Y3Slaxl1MTM7b6jp+ZNomXKpCO6a6MMKeAWpinWoQDqUyOp002+jbHsNLywijBhRMJX4bn4geVnuiiAjKkaSOkHmNbcLYKgpMAAxPOImfn+FhGZEEQUduXBUHRFI6r/ECJUJGkn+n4FH8PPpUFON1TQCtcyCIMQBr9RNpNXTlIJUnIUGRmA0P56H5+FgX6JzsFOBCjCTA10+6efztESlIQgqJUB5p6AM0EpIgkaGZjzO3gOeyi1QF9lClhtKuSp31289j8bavRnsalcpJBWYIPMkGfPT5ednphNmnqrkbS6w2qFrbUCnMd5A+cCzufsUzW8kx6rDLbbC3qm76VwJlUJbSSY8D7/iNptHWMbiLVKqqprqSyEqEFpvLpr0EQZHj7rT7f9LRFCqRphISDKykECZ9nT3/AI+EcYrogLnqUtp7qFhRO/ekfn5+VtKhlc1wN1TqWCxsoQLTzZlKFiOgItsVdYiQH3RJzEZjv1+6zkcpyT7a/tCdVDnJIOn1sdrFfVgtBQO9oSJTIgRJ193wt0YlB3WQWW5pFNbWaKLyjlgCdYtoVbs94A89RZTVSIyBwoSZB0CRvrp+nlbj1RKkSpjKNydhvtH1r7rSD2qNj1SaalZGoEnnbr1o5MhEgCBr9ePxsZVSN5ZUlSYE6a6bfX0LAPU6ENZkZt9ZtIFpTeJY447CXF5u8IBJ5W49ad73e0VuOth3vtKSmCie6lSB5hRP/ULFQyogmRoJ52cWKRuuxUlJzBInnbv11ZEZQJnYxYAtrHLxtrKomAk2ewTZijLVS2lQK0d2RpvYyq8myns20qy7ZiYMT4WTi24BqhW3S3MHpZsoKWYo76y2pARzHUaTJ3FtB5OoJEyNZixPWLZryFllCV0NUHNCpGvSwUzFtWy0homJQtNUO01S1UtLKVtLStJG4IMg2+kvhvik4u4d4XxU2sKTfFzUVfI59qwlf42+a8biLe+foIYmViz0ReGt5Lc7RynupV3rPT1Z5xgD/hbFuZ7Sx3jjk6Ej1/8ApaOHWLiCp+VUuRvYI1Du4NuTvbStbcmCtgNb0Wdu4T7Vs7dzUz8LcxztqCbGCnlC6YfcS+4gq0ISr7x+FjHbOz7RsTSSmoQdsyFD4EWHMzboaM5ogq8oAKG7Z06lRtoPu/zW4mRFsMnna0AhErrt3Brmtnbu8lbWD52zbnaQF026F9YdOmY2T7xecD9K8VnRZQdeRFjnw91iV6pWaUrbIlCkrnyNk4aJ26Fddu9Ht25NQ8P44i2gZTIG9uFD32YjRPsuxUukRPnYdNS6UTPIcrFI0MjWwrAlAHutXc2yI3VDGpeiCr52QcV3hetHh+9Lwbvc0RpaV2oStFPnKAhJVqDObbkBOtld5fZpkIKlKMJSOZ/AflZBxLhlvEVyV913tVPuIrmVtAIOVDBI0KUiMxBAPenUctrBcwyeEKZIaLqvtb6TdwIum777vC5Hq6rrgy+lhbDCV0ockF3OHVBJA1LaghY2UQdLN3iZifBF4JrUVGI78Xfq3E0jtGkLcNGcx7nYJhttSvaKHVoUEmTJ3auIeH1z4lwxfuE2r0dp8QYKfco/WnezUpTKUgrkhQBCyc0nUEZddU2rWzipVLSUGHnr6YdvO6K3KyENoU0wlDS1ZFFaSFpSvUEqKQqImYtnVmHiWBzSbaH1We6tkY4Ai4KngYnwGkBLlPifOBCovCmQJ5wlNIQPIEgdTvbLNxn0g6ltlDdVSXut5KQHFJSkgqjUgpWARPQAeA2tlvODhtbf3f8A9v8AhaPFh6r00fH2unU27bAUItw+T20RPW3aDpv/AFt9IO3XIsFghkjrHvt3AAtwklO9u0kWHZSXYEH9bbAjbblbkEKI193S3YPKySWwBO3vt0NtjbkRztsGySXUbWwTG58ranwi29Js1k4XQJne2wSeY00twPf1t2Dv94tEqS3MA6W34C3MmSAf6WyYmySXQI2Nt8teduZERMe+2TO+1kkujAOmltnvD5W4Jts+NmskuF01K5o5TtL/AMyQbBG67s3/AHfTiBulsA/EWMSAOnKbZmAHusrBPcom7ct2vI7J2nJR/KVqjfpNm9ePCXhve6iu88FXJVlW6qi72XifetBs7c2kZjbJjnItEsb0ThxHNRbeXoxcEbykO8NsNImT9nc1K375Q2k8zz52Zt9egj6N99uoqKrh7RpcbgJWzVVjMdNEPgTqeVrBzr52wq/mG1hGCM7hSEzxsVVyo/Z58Du+LuavigB1Cae815QY3+1S59RZtXr+zdwS8srujGl/0hCSEh19l8A6awGm522kWuQCImbZPLoLR7tH0UuO/mVQyv8A2a14pacRRcTXVqXr3rmQE+UipnkOVm1eX7OXiYktmgxdcjyWdU+sMVLc6RPcQ5yn5W9GQZETbJBM/jaBo4ypCpeF5fXr6BXHSgQ+umbw9XrI0DFc42VdD9s22PmLNS8PQ/473UhPaYBYdZbJUssXpRO6Cd0h2ddeXK3rbMG3C0NrTDiUqG3eFhOoWnmiNq3cwvHeo4BcaKdpVUvhPil1vdJprsee0Ame4kiCbIN4cPMa3PkbvjCV/UJBlRqbufZHiAVpHhrb2ZZu27lMJSugpiAIjsk/lbf7luoiRRpSTuUKUj7iLAOGki4KK2tA3C8T1MBLzheXBb2BV4anXXythpnCySh0qcXIToNR8NOWtvaerwtcVcCippVug6Qt1agfcSbNy8OCXC+9ZN4YMuV8k7vXVSOke9bRPzsB2GSciisrWdF4gFhdPfTzI1UtCspJ3Mjl9fKz/wAPhblz0yVKISjMQdUlRnbeNvhb1QvP0O/R5vZwvVfDO4e0IIzt0QYMHfVkoj3WRX/Qf4Glo093YeRRNme6xVVYVr0Kn1D5WBJhsxGiKysi5rziSD2WctaAADuwCeRjz+UeFhEUPahLJKte8VHyEyNxuOdr+13oD8MnVZ6C879Yyg5Ui8GsvwVTqPzshu+gPd1Mpxd3Yxr0FQyp7Snaeypjb2m7U34fUDYKwyrg6qlibto1OEqQgAAgefMkWGZu1lAAUkdo4ZlKpPu6jz62tzXegrfzYQKDGTbwSc3293hsTB17jy+fhys3rx9DTiRSl1ynrrlfWYCAE1Qnz+wI687U5KOpbqWlWWTwHZyrQaNtx09nmSGkgFSj7W5kGOXOwaqF0iO1IUuSMyY8tJHytPFb6J3F2myhvD1HUJTqpabwYQSfJakn5ToLIFVwB4qUJccdwBey1AQgUyA+B1nsyoHrvy91q5gnbu0+iIJIzsQohcu5RWpKMhABUpWUgDwHWRJ8osXdpXjTEEN96RouCdxG3OD8bSZX8LcdUK0t1WCsR0qUpCnFO3c+kdYkpj4aWalfc9TQKAqmalkpEJQtGVRPM6jx3iwjnG4RBl5KCsU0zjL6kqQjNmClFKpgRqT0P6WXMG3itm5nmwStaXjl0gQQNfHbeweM6dxpNSktDtnRmUQQTAI06RH32T8GEvUNQ2ZjMkgEzGhnfxA3tdY0uaqrjlfolioUVOFZEqV/MfGd/KRp1izQxRTtuUNal0SEtzmJ9525ab9Dpzh4LQsoKpGYmZjeOnWLNy/mu0aqUdmcq2VAa5pGXz1/XcWu0/hIsq0wuFEjzZglCSspBPtZp13j4+MneycQpQLWQ5VCSfEffEHz+FlRSsoSlJcBI5QoGefhtBkdegsmqKHESAdIkHmBz+ult9iyHIJ3u5ieQGpn5e4fpYN1KRlKwgKGh72/1pHlYQrTKlNrBV7KtfaHMjw0+VgFKXKc6CEq2GhMSeXLf52KAolaXkICey0He8hyHx+uhaqALTgTEggkk8/A/XOwy8wSCEqO8iZ25zYJzvIcCsw011nX6ixQo25INKpups5NGqhXe/zJGn/LbuG15UEBWYHUHWeW/kPnbinSV3bVoynuLbXoNvaH4i3KMywJB3g66H38rSKbktltmII3MEnQzt8jbMqUEJVEjoYiLYVkmVCJ8YERE/dbkgqOgGo73Ub/AJWSS2UBU8xpGuvPfrbSmkpJTJmYBI++3RK8xQpskHc89LczEFWhO08/ysklzlAJ0BIEeztblTWsKA12k2EzbnLmmCAVaQPvtpUT3ANehj6FnumsgFoSkab+dg/Owy092dbBAHUeFpAqBC0N7ezH7JXFKr79F6puNx3MvD2I6ylSn+VtxDTw/wCZxdvGcbi3p1+xnxOSxxNwa47oFXdebSJ/+mbWY/8AZ/K2PjrM9GT0IP8Ab+6uUDrTW6r0vVI3NucwG1tr2sGfPa3DhdC0XCE32tzEWxB2EW7AB2scC6bZBqGVbSjAhUfERYY6WAqP7on+UhQ9xsY20623KA/p2VeYc1mlt7crYJ520QCdvCbXwECywxFtDU23HIi2jPM2e1kitknb77BVCA4y4iRCkkRYUSdANbaUJGnjaYbcJr2SfTnMygkz3bbVJMxYKldQvtmEKSVMuqQoT7PMA+4/dYYyNOtoqV7Lka6WEY/iB1OhsGNz0t0FFJOQCSDEmAT0+hYMlgLlSjJvZDpbW4sISmVHQWLvVV1IafQ/XtJ7MFR74JSoDXQaz4WVKUXWy16zeVTTgKSSO1cASB5ExaLMR8TbgRjd3AhvKop3HmBVIbYIRnbGiw4QFOAAHNkSmf5ykA2zpcQFI3OSAeV/79FIjiuyhVX9Jhy8rt4vf21wJX0lO09dvqt4O1TDlI2t0lTZaW64lKZKM0EHMIIjQC1bsPcI8l5tY/xxjS6MOXLeLy00LLjwb/eBKiVFIAKyiDBUhJTmj2gDF4+L/EK5au42rmvPD9XeVI+UP01AxdqqenpRPcWXXWuxUtAhUZwiQUybUjxDwexbjYGvxNiynpO3fW8mvvJwUgLCRAAClAOEJ0OQEaEawTbDqMUdM0tdIGA7m2v23Qn07WPzBubyVgaK7vRhp6NhhuopXUNtJQlxdM6VLAAAJIqkiT/lHkNrZaGqTCuGaWlZpX8YXG44y2lta+wq+8oCCf7sfdbLckaB1/8AUv8A59la4v8A6BesLwPaanmbdpBjnpbToh3mJnlbtAAPzt9DO3XJN2WFRG9kyqqC2Xni66lLQJISqNAmdLKS5Inpysi1yS41VNE5QtKkExtKY/GxIxcoUhsod4b+mXwX4nX7Q4auDEN60V53ioN0tPeV3qaDqyJCM4zICjsASNdN9LTqxeFROV4gnlpaA8L+i/hC471uy+lOsVDt3PtVTbqWsqittQUk7nmkc7PX/SE3XY+VhGhpK5H7upi9VvVFK4whS1qGRLecDOISs5kynaCYMFyBw1CgHkHRSo29mAUIE2AfvLsl5Gmgop9okwBYGncIZnnvaOeOdJWXtwnxDdFDUOMu3my1RFbaoV2bzyEOAEaiUKUPI2GIxdTMhUgUWJ6SvQXaN6lqmkqyqXTPpcCVdCUzr4TZUbrG3Eg6687VI9HDgtWcKMcJvG66t1q77zpXaerpQtRaUQkrQqD/ABBSd/8AERzNrOitS1W+qkjVAWPfI/CzuiHJJshS84+00guLMJTvpNghedISCXonqg/lYlVuk0wE7qSPmLVy9KjGHGrCN4YUc4SX4xRodarnLwpnqFp9NUUFjsgStJKQJcHdInNrtYYiuph5vorQt1LDolp5CttjtYTMI018rMLCF+V1dhS48QX2wilra+hpn6poApDbriE5kgHUd5URZ2MP5kkg6EWgYraJxLdKMwRI+NsCjOhNmBjjE72FMI3/AItprqdvN66aKprk0TTnZrqS0lSg2lUGCcsTB32tBuG/TKevpxtlzg1jNhxUJKadYeKSTH8qbLhW3Kdry7YK2RV/NbM0gybJtHXLqWkO95IKQSFb2J1V6PNrqXe3LbVPMwmdAkKJ2nnZhGTsnMgG6Xs3Tbfe2FWu8C0K4B9Kbg5xLvZi48FcRqO8LwqZLNK7RP0y3YGYhPaoQCYBOUGYB0tLVNeSnQC4iDzA2sxjISEgKP5gedsJjnZEvfEdJdTjDTym0mozZO0dCAYAkCdzrYNGJ2nAFJplKB5pcCh7utmylSzBL+bqedsmNd5slUl+MVb3YJYdSYJlWWPvsoocSsSk2jZSQgV1tk8h77czHKwL9XT0ye0qX22kA6rcWEiTyk2iQkjClSf1tmfxsC3UMvJztuIWnqhQIPwt3mkSNpsyS6zdbbCup+NuJm2oJ5+dknWNHukTzI+dhM/ObF2v40nXvq++wgII3+NmCRQoV1A1tokg6TbknTytiTqQI31s6YLqdiZ+Ftheg59LBn5zbcgbCzFTBuhQrnGlszGelgkkbm25J062FayVkIFDcae+2ZzOlg+ZtgPjZJITNHP320tDbgyuISoHcETYMHnboK162YgIjSgTdt2LIUbvpyRseyEj5W5dui7Xk5F05KTyC1AfAGxgbaC2wR77QyN6I2Z3VNq9OFPDe+ypV8YLuWtKxCjVUDL+YdD2iTNmnU+ir6PtXnP+ivDjJXGb1e7WWCQNtWkpOk6WlIdbbBg2gYIzyT8R45qB709Bv0dryzKTgv1VZEZqevrEn4B6PlZl33+zb4HXqVOU15YkoSrkxeKIG+3aMuHmedrXZjpJkC2wrTQ+c2h3WLkFLjPtYlUDvj9kXw9fzC5OJeJqQFMJ9YFNVR5gNtE/HnZgX3+x5v1KZuHjN2ipJAqbiCEjXTVFUo6T/LyFvT1KhvFu0rGotPhW2KhfyXkBe/7I3j7TFa7rxvhCrSmcgWuvaWR7qZYB9/vswb7/AGY/pX3WhaafCtyXlGYJVT32w0I/+0Fsj4W9vgsH8rdJUN+VlZw5peE8l4AXz6DPpV3OV+t8F75qCJA9QfpqyYG8MOLmzQvf0afSFuQKVe/AzH1OlOpWvDdYUAdcwbKY9/nb6LVtsuj7VtCv8wBsXF13UTP7upgY5MpFkHvG6bhtK+Z57DGJLnTWMXzcN5XeoNHu1VI40ZChocwEc7I4SoNzty1t9Or1xXVUJKHqcqSd0B1YSfcDFmve/BLhNfxUb64fYfr83teuXWxUT/7RCrPxnDkkYgea+bMocEZtNNdCNOU20pRM9yATtNvoWvj0KvRdvxJFbwTwinNrNPc9PTn4tISfnZi3v+zK9EO9ypX+i+npFKMzTXjXNQfIP5flZd5tu1R4PmvCSCD3UxG2ttlLihCUmJ0jW3tTe/7JH0Y64f8AZzWI7ujb1a9zt/8AXNu2Y1+fscOGTqnF3BxDxTSAju+sLpaqD4jsmpG/OzGraNwUuCSvI9CDlzDQzrIjT6/C2dkiJKhGmo13FvTS+P2Nl8IBVcnF9xwnlVXA2B4SUVZJ/wCGzDvz9kVx2oCs3TjHCNcn+FLyK5pR/wCBhxM++yFXGU/AeqCqSj+ImT42BKSDFrhX/wDsvvSwuwRRYcuC8ttaa+m2gT/9o7I/LnaPr59Aj0uLiUF1fBi8qhJUB/qNbSVh+DLqiPeLFbPGfiCiYX9FXvUWu/8AsjcRqur0k7zuJTuVu+8M1TQST7S2nWXR8krtXC+/Rg9I3D6lG9eBOPWG0/7T+z9UtEf50oKfnaT/AEDW8TcOfS84fVN/3Fed1oq6167HPW6RxmfWGHGkg5gP4lJsGvyy0rwDyT07XMmabc17kKMggW4522VamLaHjbz4DVdKFnusIgzIFuBr4W2BrY7Eiu3EZ21IP8QIttpWdtCjzSLYk5hB3txTK+yUjTuLUny1kD4EW16A2JCrS6BCj3W3FtCDvpbYmNLaoCr3Wc5m2hlgcrbJ03tke+LOEyyCBIM25URtbek/ibYZ2FihRJ1SahIRW1CEpAzwvQb8rdq31GptzVAiubWD7aCkj527VKhMzaFksy5G4IG3Kx+6ruFW+X1tpyta7bq5CxFsdo+3TI1ccMJTv7/IWdvYeo0JbpU5lJRKdPaV1PvtXka06nkmfIWaDcpt3xQ3Ve7qziR5hm52VJbDbjvZpfeUYhZkSASEhM6qJkGBaMuLXD3hxdlwV9fT05w2mphp28aeudoHHgogqSMiVuPKMJ7mQ5tdbTS8GKTItbWcNkFAy5lZoiR46n4myPifCVFjK7Td+KqRD9A6tLhogshKinUBwpIzCYOX2dNZti1VFHIx0b2hznc7bfflbknZKWuBB0VSmlPXtThq7r0x5f13UjfaJXWMuPtKUlKyQy0ptxSFQ2YByLy90e1Brzih3CWIcbU9wVeNsVu1F/VyqZ2gqaPs1sNkpzuZE5glAyz3koACZCXIMXz4m33U1l10l1YBoFIYoX6dKKimYCmlpCi2tlkJICYSCCpRSjZIVMlNQ+N+M2g1fdVgm+Lpuy+H7qDFe3dRbdqWVtocYCHKhBkAdqhTeTKFDtNJFuIqaLu02Vri4deQPodlrts+LMdPJMuvYwNS11RTU/ol8SFNNOrQgqdr21FIJAlCachJjkCQNgTbLRX+4OMqu9dtVfIpDrTh15IWG/4cwKhBiJ0FssT2fBzkP/5uWfxz8v4C9jnxD0a26A0j522+gdqOptiRpJ3t7s46rmW7LgxyMWbGKgpdwXy2DGekqBv/AOrNnUpEg6TOwtCXEk8a2MUVFDhqruN3Dt4ZW+zqGFB9lCkhLneG5nMR0kTtYjHNb7xshPY5xsAq+cD+BGL+GuMbgv65MX3kKBuoZbq6NVQS08wSErSobHunSZiBGokW6vZtg3lQurbBXlcSkxqB3dPKxdjDNXR3C45QpbVXNMqUwlWiS4BKQT0kCzDwXjTifizEjd1Yu4d010U9Iha/XGq0LBWIGUIJkg7zOmWxjIzS7kMRv6KaaUxTb7J52b+M2/WMLwnvAu0yjPTtE62ctIwpLGRQ5Rp5WirEfFO6+Hle9hziFdte1dzhUaS82KdTzCmiZyOBPeSpO2gMgA6WUbgTome0gJ23Mwhp2mWlIgSNtu4q3dVUZ8TlCYhFK3OvPMo/daO3fSS4HXVS9uzi5yscAhqnp6J8uL/wjMgCfMjzs4OGtdeuLkvYtr7vdoU3m6XWaZ722mAAlsKn+LKkE+JNnkeL6pRxuAUiPkilbJH+0Ry/xC0I+kT6QaeCF44cp3eHL2Jqa+G6px19moCFUYZU1HdLas2btDzEZec6TNiC8KK5LndvS9KxqlpKYoU686oIQ2nMO8onQDW0Z4pc4WcRaqkqazG9wPCjbcbaSmtYcBzkT/Fp7ItAG+ykR1SrTvUHFDD+G+INzXveTFGtlN409GVBDTwUg6PJAkqTJjvQCNuYf11uq7FIVMxr1tGT+NcDYNuq6sB4cvOlrat9SKOkpqV9DikIJlbiwk9xITmOu5gDwki6AtVOhShEj32Y7papGxVV3Fd+FL7vDFLymbnpaSqevB0pUrLTJSpTioSCowgEwATpoLRThfiz6MFU8y3hvHFD2zykoaQ4y+gqUSAAM7Y1JI0m0t4hu5u/MN3xcamw569TVNNkVEKzhSYM6c+doywl6PuE7ir2rxew5QpeaUlSVdkhRBBBBkeNhSa7i6LFpzspqoT9mAgDLG4snOjMa9tWynCCfNtIsp0iUNtBAVoBFk1SAXbwTPtKPL/1abTZtZQfuoMwn6MWGsLYoubFDQp2qu6ahFShxlBCllMmNRsZg+FpaubGFLe991l3ULNUhN3rDLqqimWxmcgE5UrAJSAU96IM6EjW1UeF3B/HvDbGlwX1dGN73dudNY23VUD1UtbTjKyEqSRMHQzrMeYBtbspaNawsphZSoBUcpGnuk2ThlCduuyReLfCy4OKNysUN/UYfVSKU5TrzlJbUoQSIjcCLIfCzh/SYGwBW4QpFLS01UVgSSokgODNuTM942lhgZmQCSdLIaUpS/eLYG7xPxbT+VmGoNynvYgWULej5wovPAN6uXnUYmvq8EuU3YIarKtTjbYKgSQk8+6NfE2shTkBEEAWbGGqdtLDZSkeyPOzoQiBpp0Nq1iNCbqxodghiqzH4qKUq4Utg93t0KIGs2e5O8aWZnEtAXcZJ1hxJtNouDdRcU0sC4j/AHe47drildk/30CfZXGvxH3WTcMX1UDGia1T7qkLqnioZjBnMNttzaLqzGFdc3GsYMqZRS12H2Lyu1w/+kNPvJqEeeQsq9yrPHBztQqsadqW1NuevP6Hp2ysp96YjwsPZgBUhq64ViaR8OpmbGQZ0H62SLpUpTQk8hZVBgWThYpLlr2nJ5L5+QsLMaxYFsgLcGs5gY9wt2TPKbMBdJNvFGMjhypYYaoRUrdSVLlzJlHLkd9bD0+LqN65Dfbram0NtqU6gKzFJTumY1/UWaXEgH96056tHkep/OzIrMTMKwviJuiqgsUzL7TyQofZvNo1BHIxB8QRYLn8OpEbvdIVlsXEp87dwVM2HcYUWIu0SxSvsqaSlRz5dZ8jZaU8IBSPKbRZgt9VHVBLayQ418dRZ04wvx66cNVVayftQjKggxClGAfnNnpnGVlzuo1EbYn2CP1mOML0FSqkqr4aDqDlWlCVOZT0JSCAfDeynd18XZerPb3bXM1LYgEtqnKfEcvI2qDxIuviBfd2sNYAv83XUtqW86sNBRd07qJMwNyY128QXDwIxbiZVPc1fiIKbvJThorwSABmIWUEkDT+VWkCRpYTXvc3iHa9kQxNByDdWnW6lKcyiAneegttDza0haCCCORs0cd3dU4iwZel0UTpbeqadSWyFFPeGoBjkSAD1BItDvCfGLl3Vie3loOj1SrbUfZcSdCRyIPyUbWnxZWB/K9j5Ks05nFvNWQDiSqBbqQBJMWgyqxbed94pQunvWrYo+1S0hNO8pGZsHU6Eb6md9RaYUV6V06n8xCInvdLV43CVxDeSNJGYwCeaUA6Npt2FAiRHhaBeIPGupuC76zELL9Yi66SAhFFTh59+SACB4zO4AAk7TZV4QcaWsd0FBeLNWqpu+8wQy462EPNOAlOVYTpuCOfIgmwjKL2tpe10UQuAvfXeymYK26WwLAgmJsj35frNzXPVXq+kqRStlzKCAVRy9+gsyl8YKKmDZvCkYpw4cqM9aE5zEwJSJ0tOVzYQC/mmia6a4aNlJwUPL3W2FTZsYWxtQYnU8zSsqQplAWSXEqBBMbizjSqedmY8PF2qTmFhsUMF6aRbebxsDm00Pvsi1OMLhoqpyiqr4pWnmzC0rXGUxt0tImwuVEAk2CcIX4267TlrZJu2/bqvRRbobxpahYEkNPJUQPEAzY+XAEk9LR0cLhIgjdGQsEaG3Qc87FG3Q4mYOmltioSpWSdRaNk4dbdGguNfjbfaCJBsB2gSnMT520lwKEpMi0HAqV77IwHDoOVug6ORsWKwNSdLbCtgPvtGymLI12pnU6W32s7WLZo0m2ZoG5i0UrAoyVk24JB3sBnJnWTbO0nS0SErWQqjPT4WBcS05IU2lU9QDbCsnYC3Kj4izFo5qQNkWcu+hUcyqNgnr2YsA5dNCuCWimNgHFCPcDY6penP324Kj1sMxNI2RWvI5og5dFNlIbdfR4h0n75sCbqXMt3jUf7wQR/4bKSvPa3A9oePUWAaOF3wowleOaTV0Vcn+7rWj0zMT9yhbnsrzQdfVV+9SPwNlJY03iwWnvtDuER5KXHciYVXo1NG2SP5XvzAsQumpvdx2rVUXe00pTuYN9tJAiNwCD7PzstGedirY7OsWJjOJjyj9bJtI2IgtT8YuFiF0l2qjvUh8crgP5W2agpmaV8R0SD9xsPsJ5G28xHQWthh6oOZFjWsDRQdT5tK/K3IvGhKoVVIBA2On32NnXU21AIgj42bK4c09wghU07glD7avJYNuiZ2I+NuHKWncELZbV4lANgF3XdylKPqbQKtyEx91peJRJARe93006WH1BUJeSNBsCYtwKk1Ci3TQcuil7pT+ZsSv8Aw829ddSihedYdyKKFlxa0pMb5CqDz0sfo0JRTNJbEJyiABA26WXiLiEjlAuE4rkugUKDVOJJed9oq9qPwstKcSlkvKCiEAkhKSokeAGpPhZGpr9ZRTIRUZlOAQTHTY2Zt78U8WYXDlTeHDh28KMvFDT913gh4pRBILqHEtqSdDojPrAEkgWo1dXFRsMkxIA52JQhDJO6zRqn8+yoH1+oKm+5AQVd1tPjynqfdytEfFbGOJr/AHEYLwNSMlKSl2vqKxakN1TWaFU7aEguLSZBWpMDKCnMCrRuY746X1fFUjCzLFNcbrikuFovpXVvNgahKTATBIBJSqI2HJpXxj2uuK4n0U123I1U3k7kqF1iy6p4oSIU44rMpzQgDNISAQJ2tw2JY/NUPbHQg8M2u62p672stinwtzGZp7X6XQOJGL1RcdXduLcCVfaPBZq7wonGbxbJgAKNPmQy3DaUpSghRSAhIB7pNauINw8J8Mm7bwwtW3qugvFxLLpW0W3UvaOJcebXlSlBKoSlKEiFE6zaY73x/iRFEf3HeWHlVT6FNoRTXc0ptDkEpUVhsJyJMGSoa6QSYMNX9iKiqr27bGr9Vie+EEIStkqpbsIlJAKEJ7RWaNYDbhKT3VFOmTJO6STQG3RWZImxstcEqJ6ziHxOoKt+hpMNYmqGKdxTTTq+yWpxCTAUVZBJIEzAm2WnRdXg5S1KYwuA2SSjs7szoy8sqlZlKHQlRJ5k72y0Pa0X+x+/+VT4D+q9HK1ID5002AFuECdbD14+28zysEkaAje3v+65FYEnmLAP0TbypUmTY0lMxIt3Gm1mukiaaVKUZQnfSxZFzU7b5fQ0ArmQNbKuWPdbeWNxZiE40QCGsqYsmXzh6ivhos1bCHEq0IUkEGy0RbYRPKzGx0KVyNlGzPCTCtPV+ss3PSoWFTmSykH4gWeN3XU1RNpbbSEpSNNLK5QneLZk8LQbG1puAiF7njVJd93LRX5dNRdN40zVRTVKOzcacSFJUnoQbQzeXoxcPKl0rGH6VJJJ7qMv3WnuNY2i2lNpVrFk5ufmQosdl5KHsD8DsI4LrfXrpudhp+IDmXMuJ/mOvO0pU9Lkby5YseDCQZCBbYbA0jeyY3JzTvOc3UC8SeFPEC9cTVl9YW4o3/c9PVZVCkZdJabUEgHKkECDE+ZNmuzw89ICjVDXGa8lJ2lyhQ5/4jr8rWfXTocPeTYH1Fon2bJ3EJ0d+yk0sA1aku4qapRQMN1z6nn0NpStwpAK1AQVQOptHHEpji9SYlQvAlfc6bqqEIS81WUylLS5JClBSdxGXTqPG0wIYCNrcO0jb0ZgNPCxRI5vuoRY07hR5WYWq0YYqP3e22quaYK6cKkJLidUgnlqBZtcO8R8TMR3w5SYzwpd91sUqCpLzFSVKWskDKEEmExOpPIeNpoFIkJKQNCLF27saadLiUJBO8c7J0rzZIMYNVqnZKWwmZ05WijHPFCiwNit6467DWIKxValt1p+hou0aMjLlKiod7T5i0xJbCRBFk+8bjpq91LrzQUpBBEizGQtGgT5Gk6pMw2y6mmaUtBHdG/lZxIiwNPSpa7g0FjQSRzm0C6+6f6LShzizU4hKabuJx2ocQ022tJUtaglKdeZNnYYsh4uwzdmLLhqrhvilRUUlWkJcbWNDBBHzANptcBuolt9lWbi5hb+0NPRYtw0609emGyirplMrCs6Uk52iRyUhR06xZwYUvBurTQXi1HZPlpaOcTH17rSZgvhXhzBF3O3Xc13IZZdcU4oanMowJJOp0AHus0mvR2w5RYzTimmerkZaj1lNOH1BoLmQMo5Axp4RZnyMcwA30U2scHXBCk5m86e7KL1qqUoIGVOgkkkgAD42WKG8qeta7ZkrAnUKGvWzVxlharv7C1RdV2Xi9d1UooU1UsgFbakqB0kEco99g+GtwX/AHBcvqOI7+qb2qi4VqffQlBgwAkBIiBHnqbQc5ugCQaSLp7oIDqyD0sIYiB52DRHan/KPxsJrGny1sgbJlHvEVsGtoTqJSsH5G1abvYewpxvxVcN6OKduPiA0iupQZCUVbTCGX2v99AQeWxtYrjeMVU1zUt4YRuFF71bbxQthTgQA2RqZO50A352j29eHWIMb4GpLxvG7W7vxBQuJvCmbQ4Fll5JJCMw01T3Ty15xaMkQkmu46EW8wUeKUxxWA1BunnhOWXaRBWV5GuzzEbxGtl3iInPg2tVPsBCjH+dJtGnC/FGJ7/xCbvvDAd5XS3ToUXqipWAnMDAATAJJmegFpixDc7t8YavC7ElIXUU60IJ2C47vzi0aBhibleeaeueJHBzeijLBFO09UPIcAPdGh982bTVM3d2Jq9lgZQ3eJWlIIG+VRPz+fwRcP8AEy7bivF27b+fTdF70Us1FPW9xKj1B25Tv5SLK1y3xdGKMTFdBeVLUv1b4fcbpnA5lACQToTGiRv1tGcmGEwuGt1KJvElEoOllYW7yXmQlQ9oSbQZxCwyvC2PEV9Ex/qF/hanUpGiKhIkn/eE+ZJtOt0sqSykqSQSOlmbxkp2UYfp612Einq0qUTskEEHXkLXYiHtLDsQqjyWuDhyKaWA7sFXWpeCISyAkCI1/paVL3aW3h+tSgwoUzkeeU2YnCd5itpqlTS0ryPDVJB3SPyNpNqmQ9RONEGHEFJ99qFEwM1Kt1j87rKBLnuakv6nprtq2kONPISopIme7NiuA+Bt54Txg0u7sVVjNwU9Waxq7W2UBMk5spWQTGbpGgjxsewheLVI+2HVAu3e+umfSN0KBKdRuOtlurxXiF6/akXbWFijZDaWShCVdpIOcmZnUx7rQe0QGQOOt9kZhdNky7WT/wAdtF7Bd7J3PqjioPgmfwtA2LMCYf4i3PRUOI6RL7NO4mpRqQUqyFMgggjRRtYS+0+v3DXMAhXa0riSOspNq+X1hZeOcIUlzovK8LtWvsHA/RP9k6CncBQ5HW0q8foMcDazv3CjQn9V7SL6KRuBeB8H4Boai78M0iWn31BypXlVncAJCZUSdBJ08T1tMCFAjS0O8DOHDuBaasdqsQXxe79UUp7S8axT3ZoHJA2Enc76C0wtwIEWFFfLqbqUpu7QWXSzCSqBpaueL2K5yoxExQVPY1ZeqksO5QoocIORUHTQwddNLWLcEII8LQPithTOLb6Z2BdSsH/MgGxpm3pJD0sfyoU7rVTPuPwom9HzF/EWsYvJGPHQu+LjvINtVTdOGe0QUgiQmE/orW1wq6rd/ddS7TLyudgtSD0VlMWp1wu4mYsv7Gl64Oxfh2koK26EJq2HqZa+zeRngaL6778iItbS5a395XMzVEAdu0CoDkrmPKZtVpvDOQ4WvY25KxU6wgg3tcXUY8KL6ql3rRVT9W8760y424VrUqVQTJnnmTZOv3H2JWOIl7tJvaraZu6raShlt0oQWsoMFMwZk6n8NC/D4qo65uiUqDRXk9TGdP4yPuNknHbAo+Kd8NgkGtpaapGnROT/AKbWclu9M6EkeoKCTmdA7qNfQqx98Xv6hctbeaEhz1enceCSfaKUkx8rMThdjy/MQVqWr2qmltP0xeQlLSUwvunQjlBO/wCFnJQqF7YRZKiT63QpB/3m9fvtEXB+rLdddCFqMhC6dXmAoR8hYUv9Bsg+YflThH6pYeh/ClniTjheDropaxqkFS9VVKaZsKVlSkkE5jGpGnzFj2BsVuYquU3g/SIp3W3lMqQhZUNAkgz/AL1mNx+aWvBbFakEmjvFh0lPQkp+9Qsa4NvqFFeFMDol5Dm38wj/AKbKpbkbG4c7g/ZRhGYvB5WUp5tJJgWaD/EG6mrxqaBAqnV0zhbWUNjLmAkgSRMbWdWYlo+VoHraZpWI8QXW6ohDlQ8hcEpIDgmQeWhsKoJjpXys3bb97KcQDpmxu2P+FLl14zuutqE0peW24vRIdbKJPSdvnZwFYgkjYWptga5MUcOuKNXhC8MSXneV1V1CirovXX1OlohRBAKjpsqYiRl6Ta113166+5G3s5Lq2ilRj+ICCfiLAhdJxBFLbUAg+RRZGsDC5nI2QtBim4rxvB26qW+7verWZz0zdShTqY3lAMj4WUKioSykLV15m1P+ENcbs4q3U84YUt9ynVP+NC0ffFpW9J+sqk4GpW6d1xLTtXkdCVQFdxRTPUSLaXdCap1MDsqgmHAE3VTQKlCmi6NR52xp1LqSocjztSHBHF298J4GxHhlu8HGDUsioonkKOZp0KSFhJPs5kbbQU6b2kL0TsXXtX35f103tfFdWBVMh5hNRVLdSgIcIOXMSBIWnbpau6LLEJfOyKJLyZLcrqzi3ENpzKMCwYfaV3QseVos488W3uGuH2lXZSt1F5VoV2Ad/u2wClJWoSCYKhA0nrpatN1+krxfob7pUXlfgfRWoDjDdRdzaWXo3CSlIJGm4V4WC4ZCARujtOa5HJXqJ3iwcmY1+Nmxw1x3ScQsI0mJKdsNOOZmqhnNPZPJ9pM8xsR4KFnOZJ0982YG4upWI0K0TpYBaQKhtXXSw5sDUEwhZ/hVPu+hZiEhoULyg7W2OpFsgeVtDQWkm3Wz+FtGPxtsxbRjysklzy099sAG8aWw+0BbfLeLOAoE3QT6M6FIOx08bJd2LUujQD7TZLavAgxZYKQQfvsjXelbVZXU5QUgPB1JPRQn8LORqCkDcI3zsKxTJrWKilISSUZ0Trqk6WDI2jnbpisp6BZq6peRlCFZ1QTpHIDU68hqdhNq9ZE2SFzX7WSY4tddqhniRhbD143k7dV9XVR16ajI96u8hKikAZS7CtgkzqNdY52jOvwLetY0wrCGOr0u5EuEu1bKa9socVmJbS6QpCU6BOVaZAG8ZrTPWXRWXled84irxUUZqXQxT077Ce0ZaQhJQTvBJVmyzAJJUCRAZ2Kqegw/hWqqL5YcNG42pIVUBZ9ZUUmEFQEkqOm0axoLcuGxTh0j2hreS0XPdlGtyojvPC4xxfT12VN90JumgQH2001OA7VMKAGd5/MokLV2gGUoTAWQkQCqrvEa5r6fxRe9z4rqTddxpqVUFJV1N6rWt2hSv7HMlJU6lgZEqg5UGCRBM2t7dFNdd0XCqrvGqup29rzfFdXBx1CWqZRAgBH+AAJSk8xPIm1M/S4wPi7EGPX2MMvu3hddV6rXNGnqszBdUgNrWsiQpcok7kZwAABFhU9JTMcA0ht9z0VeWRwZnOqkWixXh2moqemp+M16sNNNIQhpm63ltoSAAEpUTKkgaA8xbLVcpMAY6apWWximlZyNpT2YVIRA9nQxptpbLVjglJf/AFP7f4Qu/SfKvoEvBMPTE62AQkAzEWjJXpNcMa5wFKr2ZkxDlIlUHxyLNjjHH7hY4QF37UNHnnu6o0+CDb03vtOfjCyjSTjXKVIoEamTboJ5RttZmMcZeGFSQG8YUqSqQA4063/4kiyizxI4ev8AsY2uQHoqtbSfmbSFVC7Z49VAwSjdp9E4x1tmnI2TKbFmE6whNJie6HyZIDda0o/JVlFqppXxLNS05PNCwfuNiCVjtio8N43C7CRz2tuBprbYBPl4W3GmvOz3B2UcpXIE6xbUbQPK3cC2FMiPDeyulYriATG1thM7i3Uaz1thERAsk65jprp8bZljXrboCOdsg8gPGySXOU7wbaIFhBPO2ojU2SS4CegtoDwsJGvnbUDysklxEn5b21APK3ZTrpbRGnKySXOWbajmBpbsiNbaAOmlkdklxHO248LbAJNsIFo3SWoi2iARFtkAWwxOhs4KSDLSCdtRbSmUkju2FjlbCOUWYpIEspIjLpbQp0o1Fh9+Vtc7MkDZAAQ9BH8H42F/G2iPtkx/Kfwt0Brz1sk5QD1Ml9JDmo8rF03e22kpSjQ6WUIEW1AsrJrpLRc9Oy8XktpCj0FjYYATE2NRO3lbRAA2swFtlIuJ3TJxXw0wrishd93FR1ixsp5lKiPIkfU2Awvwxw3hVR/c1z0tGknUMtJRPwGtn5CToQTbMoBmPjaDow7VFZI5osi7LJQkAJ2sk4uwtdeL7iqLivmlD1LUxnTJGxkajxFl4pk6aWzLB8D4WkPCLKGl7lMvAnD+5MCXebruKhRTsFwuqAklSjzJOpMAD3CztLYiCPfYfKBy3thSOQ3tBtwk52bkoP4j8EbwvG/nsW4HxG9cl5VA/wBZR2QdYfPUoMa9T74kkkrgnh5julvIO4sxGzWttnut09IGQT1UZJPkIFp4LSTopMjxtx6q0NQkWhM0zm7iiwzCEWCTW6E+qKZP8Scutq13hcXHG4apygu+5sP1dEyvKypanc5bB0JMgT8tedrUdkBpFi713svd5SATztKQufHw9LeaeKRrHZlBPCwcTqjFTCsS4fu+76BlKluONVClLWYISkJ23M67AdbT62DA3NizV2MsrzIQPdY4hJToLAiZwxZFkfxDdbIlO/haAOMd9U2EcWLrKu7rxdRXMIUFUtKp0ApEEqI0Gw8bWAKZG1k687npLzATUMpXG0ixi48NzLXuLIbG2kD77Kq1xYhwhe2I0Vt23fXqvh1j1QRQvJPZ580KMZRrzVEdetlcM0S6O56elM/Zog+Z1PzNuqfCN3MO9o3SoB8BZdYpEtJygaARanDE5kmd3TrdXJpmvZlCrJiK/KfAfEm+Lovl31RqufF40LzhhtzOdRPLvSP90+EgYgrWMQ4rZxGipQmKFNGptKgvOrOVAgj3iPH4z9jbhxhnHFMinxBdNPWdlPZqWnvInfKrcfGzQw5wAwPhi8mryu25kIfa9hS1rXk8RmJg2PVylxc6O4LhY7WKHTNaAA+xynTqnhhKlfpcM0FLUAhxthIUFbp8D5bWhfBxF14qdpFLCfVL8eaAHJPbEfibWJYpQ0yG4G0Wjp7gxci8cOYvcerlFx71j1YVKksdrM5igb6666WiXAU3Cd5fhJo/X4g8/wAo1xjpPXeHF8ICdWmkvjwyLSr8LNThhf1Ldb7q6qpbbbqWUKCnCEpkHqT4m0r3/cjV93FW3O7KU1lOthRGpGZJE2gB7gpxMoV9lQcQAWkJhKHLuQrbaDM/0s1TK3gsF9QSefP6JU8ZMjjyIVh7svmgvRtSaSupnlJElLbqVEDqQDaHcVA0nEq8Wtk1TTL4HIwkJJ+Is4OEeC8T4bdqa7FGIE3g+62GW0N0qWUITIJUYJJJgdAIOmuiRxiw1jI4go8R4Ru6krf9V9XeQ+8pEQokEQDO/wArMyVslNI19hcH/jdIxFszC3kU3cfVDC8f4IrGWzndbrGXCOY7JJGvSc1piwmvNc6G9SEkgSPGbQncrHE6tvBinvXB9EyAYNQasKDaecAJJMwNLTzcF3+pULbIT7KdfE8zbPjnfPUMcRYNbbe/O6tuiEUThe9zdVKeAw1xXVJyJor/AFK6AJD2Ye6Dad+P93/vDhpWPBoLVRvMVAA6Zwk/JRtFPF/B9/ucTqtu57irqpV4OtvsraYJaBKACSvYQpOsnTe0746uypvXh1fNGtuX3bucISnX7QIkAddRbojOzv7ZQd8t/QXWS2NwpTGR1Xn3ftYbuvWgoH0j1a8TUUi9IhyMyNtpCVj3i0w+i/WLu7iNR0jjgJqaR+mJ2JypChPn2YPvs06Lh5WcSlXpS3RSrefoGFVlK8EnK3UoUCElWwKgFp95PKx3g5UVl3cSsOOMsrU6qtbbcZSk50BRKVynlAKiegtWqWBocGciiwuNwXcwpY9LiiccauioB7qqd9M/ylK21fdaIb1fwtfPDvCF2VV41VLfNKmpcDrCEHIgPKRBKtJICYAk6eU2K9Jy53KzA9NeyEHLQVMOq/lbcTln/iy/G1QLzuK4b/w4u471oSa2kqUPUbwBKS2VpLjZ5xAJ6mTtZ5WtLYpfIi/Q/wAKlGSDIzzB+quT6NNRhZjCNTc2HReRVTPJeq3K0pzuuOJgKGQwBCIgAbe+0xe61a/RfvamVf183fTkBLtKhYAEAdm4Ux7s5FrJjQcrZcTrt1V6UAHRYSOf32AqhmZUNOXwsMTrbhYlBSIkgxYiDm1WNrK2kK6pBsII8D42BpyFMAARqR89LCwroRZwE91s21Bjb326A0g20Tl2tKyiSuYBVMW3sNLb3E21ykDezgXTFcxqYFkp0djfAg919nXzSfyNlcmdhZKvRLiKqifQUhIeKHJmcqknb3gWk4aKLSbowRAg2T2Ev1N8+tLATSUKkhkA+29rnUfBIISPHP0BsokHlbjKRyG8/Ow5Yw8C+ydr7LvGdEhTYVplLfTSR9C0J4wu2mW5QU79Q3UJ7cut0TErqKp5CcyG0iQIEZlE9wAakCbSvjS9MVLpWmLlwgK5KmQfWX69DDYVqCnQKV/CDtztFmOafElx3JeN6Ul3q9ZNGhVT2KF1JLpABbQoABDQWUnUAGFLOaABxWNYkyJggidlcfLofNaFMwubZ2qiS/KGpvrGBoMc381d131LZ9ToWVkBLTaAVl6pAISDmzFKMkCMyyBZIviguvEpv/1LBN03w4puoU9elJkcTd4SUpbCUalS1DM4ZDclQSP5rOPGF13jeDbNNft5LoLu7MdpSMinFStRITlSjOpEbqzJUSmQIJEGHMYUzhvN5q7OJCaO4qVCHEUdNUFpa3FJKjCSMzhSkI1JWSqAkTojh6yaorMzJTa3Mixv9v8A6V4MjA8IToprs4bMUzTD90XitxtCULUq87vSVKAgkjszHlJ8zbLNym9H+jrKZqrreJ1bTVD6EuOslxxRbWRKkyNDBJE+Fstl54v98+hUO7u+T8owmsfZcICjERqrbX7j+FlmivntIC3O8AJJ+ug+tbN2pCu1zoVKo1PLTwsNTKSAEkqSQDsTI57+/wCFvYHAOUmnVPRitQ6MyVGJ1P19bWOtvQkGTJgmPrzs0GKp1pRQZgSnQa6T+dlenr3EaFRKSPamD8vo2rubbZGCWezpVqIVTNEHmUja3aAw2oFDLaVcoSBr9fdYo1VJJAIzDaNtrDpey+zz3ObUifnYWoU7ApTp71raYZqevqmRMy08pPwIP1NlBjGmLKYhLGKb4T4CudEdf4rIGaUgKBMb89fzsIFKju5Y9qSdrSEjxsVHhsO4Top+I+Pqf2MXXvOvt1a1j5k2PU/GDiSyMreK6k9M7TS+n8yTNmSFEg6nfeZ/p+tuiSQCFGDMg8yenW0xUzN2cfVRMER3aPRSIzxz4jte1fLLw09uja/6Ug2UGfSEx63o43dLpEHv0yh80rFouJVIg65hEb23mUCM3MQCN4ixBXVLdnlCNHA74QphpvSMxUEkVFx3S5z7gcQfms2U2PSKvAEJqcJUyidJRXKTr0jIbQe0qFZjoJ5jb6+uVj4JTAKdY11+tbOcVq2bPSGG0zvhU3s+kRSE/wCsYTeRoD3K0K+9A+hY416QmG5+1uC9kx/KWlf9YtA6c2TMI0gAwNfgbdQRHIRr4fXSzjG6tu7vwonCKY8lYFnj3gtyO0ob4bkfxMNn7nDY6zxs4fOiV3jVs/56Nw/+EG1cBJiDOoI0ts94QR4b72KMeqh0QzgtPyurMt8W+HjsD+0aEE8nKd5B+aLHWuImBX9G8V3dP+J4I/8AFFqsBR1BXA315nmLYFZQf8R57WK3tDPzaEM4HEdnFWzaxdhN7RjE90r/AMta2fxsbavK7KkA095Uzs80PJI+RtUMx3cyhodlDy+Vs7MAQAhR2ggRYg7RP+JigcCbyerigpUJBB56W3lPIfG1O0uraMoWtvrCog2Ms3zfFPBp73rG/wCXK8tMHyBsQdohzZ+UP2EeT/wrdFJki2ssbDQ2qk3jHFzRQGsTXwIjatcjb/N9eVj7XEnHTMZMVV2n86grT/eB1sQdoYubCoHA5OTgrP5TMHW2ik7jytWxri9xBZ0OIe0H+KmZJ/8ABY61xux417VRQPf/AElKNf8AhIixW4/THcFCOCVA2IVh8p6WwDTa0Cs8fMYJGZ277ncA/wDUup/95Y4x6Qd8pI7fDNEsEEAoqFIn4hVjNxqkPP8ACGcIqhyU0uJPbNmZkED5W7KVbHpaF1ekM+p1n/5HNqgkri8iI0j/ALrrZRZ9IO6yQKvDFWgk69lUoWB8QmxBi9GfjQ3YZVN3apXiNbaA1tGzHH3CTpAdui+W5P8A3bKh0/7yx9njfgJ4AuVFcwSf46RR/wDDNitxKlds8IZoKkfAU+oi2Qf6WaDXGDh04Qn+0BQf8dG+n55LHGOJWAqgAIxVQCdB2iij/wAQFiCsp3bPHqhmlnG7D6JxgTuLYNwPwsks4ywe+qGcV3QszsK5v/4rHWr4uiogsXvROf5KhB+42IJozs4JcJ7dwUZG2otgTvbG1trEtuJUOqVA27CSBt8rIuB2KgWlcbxOtsIn87dZfD5W3AJ20s11GxQcTbevW3URrbeXpZJWXBAO41trKI1sIQR/W2o1328bLVTa0LkjlbI6i3RTbcGLRRRouI8LbCSNNyLdAco87YBzsk60ExuOW1ugBGmlsANuh1skwN1wUz5m2ZU9Ld76RbNedklsuABO9uQgTO9hIjS2tzZrJ1yUAjTnYJdM2r2kjXwsYgRobaMTaJATglAIpkI2AFtPU6HRC0CPHWxkjwtnKw8o2RLlJwuxgKzBsAjnFjDbSUaAaWHO1syjff8AG0crW7J85O6I1F3sPvJWpsFSdASNrCGnT2ZbIBB0jlYwBOgGk20oa6WiGAG6mHl26Rqa4KOjUv1ZhCAskmBGtiNLg+56C83Lzortp2ah0krWhsBSid5Is5YHObai0DE3opiRyTbxuqlvOgeu+tpkP076FNutuJCkqSRqCDuLQ7W+jNgYVxfpKWoaZJJ7BNQvswT0EyI5a6WnIwExAsGUCdQLJ7Mzct0mOyG9lG/DHg/hbhoHBcN3lC1jJ2jjilqCJkgEnQE6mNzvaQ9hFhCgDby2topgaHSzMjyCyd7y/dcGYtuARoLbgbC2xIG+tiAIaLUWhdb/AJXDHlt94NjPu+VizJy1rySPaAUPL+pNjOvOyGyc7rUSN9bZOu9tnTW2jyjz2s6ZYZ5W142yYNsnraTQouWaEWIXskqpFrTuhQcHuM2PconnYB9Hasrb2zpKfiLEy3CgTlIQRhUFPMSLckachNuKRZXSNFUzlAPmLdkEmAJ8rROyfmjlb2rmE6j1YgPtZktEicqtwdxtPUWiDGtNVVl206KW8lO0SaxmrdbZ7M+uBvcBS1ACeomTAkWlW7EOVtFUi8EAQA81TTIQobFRjVW2mw8YksDGLLLVU4cpzNkq0EyI+W/ytyNfhTquodODYGw213utCmmbG0sOvNRVju6E4iv25PW7iZ9RKFO1hL6G3oSUltpboClIaJCtG5Ku6CUpJUWTxFevNi6K697n4YYPRW3fSuKo6x55Lwpg02rs1hosCYCQQiYmJjUhzXzidiVUtwUK7zq0lSnmqNSUttLMqV2jqiEJ10iSrX2d7IqaavfadqcSvFKCoBNFQjMFZoHZqKhLkwRIyCCZGmYCdgVPw+NOcz7eVgkKt2bIwWCqSjj5d7aEt1fFHE5fSAHSze7oQVjfLodJ21Olss8qnhE05UurquO2D6J5S1FymRR3clLCp1QAgZAEnSE93TTS2W5rg4d/D/8Ayj3l6/z1T1eYUHFlTQKQqdQZ1t2w0AlJicumgH0bHq5tindAfdS0snKAo5ZPQdTbmmCFpBDyPNKwR7vr426dsheLhaXDDTqtttZRGUq015TznlH6+NhkII1CZI5kdBtHy+tRUNACEKhQgDKI5bfG3SWURKCBz5QAOXyn32fMeafRdsOrZhCgNwe7Ikz4an63scp34y50E6bxr9b2KJCSQG1BR035ba8uVttZoUEkpn2goROvl9fG0NSkldqqMyTOUaAySN+XusOHErSQVKGUCTvr9ac7IyXVg5pAB1E6jx2saS64mJJMwZj5f152ikErar7ySDpmnUx+e/L+uABI7yyRzER5WJofaJhJ19oGdrGEOEApHI+IPy91olSFkKNUnTNrI393lbsJGbvEnXQE9frnbhKkFWRSpJOutuoA2KSN9Oenj5fKzXS3XTcEhKVHTl9fX3WUmhrMnXcT9fQslpISuQIOhk9LKbIUUAAEc42/rYL3IzAhQOoAkb7gR02+vkISmII02idd+mxtwAAkETqRoPv+uttaD2UgEgASJ90WDdFXahGgIAAiR5W4zDUZRJB0Enl9fO2ylSBBOp11525WmJjKQkHTcz+VkHJrBYDmJAOg5qGk+62JSQhSAkE+HMG2kJJASVQDvpvbsZMw0zTuBaWZINCxKlRpA7pIO/1+ttgaKATMQBB05/X9bbjLJUdY0OWOmnytzMgkCN/hZsxSsF1KJzK5aGfmR9b25I/mA1122t0QZAlRJ130H1FtEmAZJBMz9ctrK6VliU6aiYMEA8o/rbYBUNADpzHL6i29IgQDy8vr7ra0UkhEQI25fX10srpZQszCQQZ06WxWaRm1zawZtmWMpKtQZJtikiTCfa05QLK6kGrFFM9IOk24KzBPdGmwkC3ZUCsAkQBsNR/T63sErVYTOm6tBr9aWldRLViElR0SASdQTbshMQSdNBPPS2IkLBBG0CTrbQSQMo8xH3zZXUcq6BSqZTuAIOh62wqCSTuROv3fn8bB5SATmk767eHjbM0kHLJ3GtlcqWVCBYJg6lPe+fP6/TEqnULO0z9fW9uQRIO8aax9dfoW5zEJ0ImAd+XSzJWCFzARlnXTnv8AU22EoUkhQBM7lPI2CzJTosHUaEk7aT9edsUojed40/P6/N7lItBQg7JKcyEAEHSBHXWxpm87wYEsXjUsj/1by0/cd7EEHXukzNu4TlKTpG3P6/WziRzdio8Np3CV28WYnp1DscS3sgdPX3o/8VjzPETHNPOTFd4nnCnQsj/imzbGpzATvtvr/S21AJGUp8ztaYqpW7OPqoOponaloTxZ4t8QWpBxGpR6Lp2DH/J+NjzXGnHjUZq6je1g9pSJE/8ADHjZhBZ1GoI3gW1miSspPU+E2IK+pGzz6oZoqc7sHopKa4642RAcobncJ5erOA//AITwPKxxnj7fyT9vcd3L/wAinEfeTaKUqjcmddYH1426zqQNTpzmJ/rYgxOrbs8qJw6mPwKYUekC+NHsKtkTEpriP/dmx6n4+3WrR/DtUjWPs30r+8C0I51d2ZHIjeT0torOsK0iTp93xtMYxVj4vwoeyqV3wqfWuOuFF6O3bezXU9m2Y/57HmeM+B3P7yorGZ/npyf/AAk2rulzU6AjlpGvS3ZdSpBHOSJ6+FiDHKpu9j9lE4NTHqrJM8WsAPaC/cp2IVTPDX/hix5riDgl7RGJaIHfvrKP/EBar3aHMCCDOxjf7rCIdIGYaggxztMdoKgbtCH7Cg5OKtM3jDCTh7mKbqM7f643+djbN83NUa097Ubv+R9B/G1UA8UjMeQ38Ldl4HURAnf8eXWxG9opObB6qBwFnJ5VtkKbWJQtKh/hM23lIExapCH0NglshMExACfr9bGmr7vOmEMXpVtRqMj6xHwNijtGOcf5UDgJ5P8AwrXR0FtAR0tWCmxtihkkJxHeXd1M1Kz+NlFniRjJtP2eIakEie+Qr/xA2mO0UR3YUM4FKNnBWOgzbIJ1mbV+p+LGNkGF34XP8zDX35bHW+MuLmwM7lG4T/M0PjpGlpjH6Y7gqJwaoGxCnQjS2iNBFoXa424hTo7Q3ev/AHFj7lfU2OtccK0D7e42FRuUOFI+c2K3HKR3NDOE1I5KWcvUW0RPMzytGLPG5lyO0w/5xUz/ANFjbXGi5lf3t0VYMx3FJV98WkMXoz8f7oZwyqb8KkAjTS2iJMmzHTxiworRxmvRPVpJ/wCqxpHFXBbioNe63/nYV+E2MMQpXbPCh3OoZuwp2Eac4to/UWbieI2C3YKL7QOWrLgj/lsOjHWEHTKcQ0Y6Z15fvi0hVwHZ49U3d5Ru0+iWSP6W0Rynaya3i3C1QYZxHdiyeSaxske6bG27yu5/RivpXOmV5J+42IJY3bEKBjeNwhjAi2trYClWqSFeRm2bH7jYgIPNRsRuizgy1rawfbSUx8TYyI5WKVi+zXTrzR9oEz9/3WNTNnGqYmyyR10trz52w2yZ52mBZQJutTrNs2E2yTrblci0gNVG9ltREwNR5W4JGwBtsmRpztogDcx52Khk3RCmkds0f4HVD3HX8bCmBztwRkrXdNFpSofdYRPZpUVrbKpjYkbG1eZz2x5ohmPS9lMAX1QlGspfE7KkHlvaMsV1uH8U1T9Ca9qoDCRT113heVaVAyc5BCikiIiARvIOkjBlpb6qjOpDqiCDnJJ126QLQnxi9G+7uJd5pxPhrFVRcN9pUFmraSpbShBSMpQpJQqYJjNMDQb283xTtTWUVSGVlOWMO/O/Qgj8rQZC0tzMNykl+gum7gKO7aJuhokLKGmGx3GUcthsAdTHxtEF+4ixtdV5uX41w0U9eDLgoEqTe4qqdpAJU6rsktoUnlmc7ydU94JylU23fw2xPgO54vjEVbjV5tta0gppqNU5MoRncKirvEwYHtTrlSmyFT3PfF9X+HXcMLud2jQ2F0nZ0ricsAlTjqUaoEQMqk90pISCqLZtT2tbi8nAo3ZWMF9Tv9lYipQwZ37lV3q8OYK9ae9evhwVPaK7bMaGc896YqDznnbLT07gThE46td5YdxG7VqUS+5+5anvuT3joobmeQ8hbLcycQk+f8q3wWdFBH7SN91PCzC6W82ZV+hRCZB0pXQNj1Vt42p9wmerH3qY+sPFJeQrL2qj/tIMz59LW3/aUOBOA8I00qld7Oq0J2SwP/i+dqqcHWypqgfcKSXX946rBgdfZ1529c7INDsNjzdT+6r9o3EVr7dB+ynBg1aSnsqmoSsHQJdIkTqLGU1F7wkKvOrmeTy4A6an8bD0rUkhPeBkgAagdT01H1yUUspCMypkDOQdQI5z9bW7gwxn4QuWbM/kUVRfGJGtr8rjJghSyoRr8o+8WHRiLFaQQL3dzDTvoSdtuVh/VW0d0IG+mms+X199hAwEk6SToBMAkeJ+tB7hmlhPwD0UxUzN2cVprFWKVGDXIWoEe20mT8o5WNN40xQ0hUrpVmQYLZ0PlPXz+63Ip4ICkknXUq1P0frnbXqiFRAjkIAMD+htHuFM7dgS79UDZxRxviJf7aIXS0SiNBAUP+rpOsdLG2eJt7ApR+6WN5hLhEn4GyP6ohw5VpB2khMdN45W36kjQGNDsdvl7vjZjhVI7dgUhiVSNnJyI4p1MlK7jBlO4fiOp1TYdPFunQftLkqJTz7RJgz7vocrNY0pSk6k6+Ex9bTYF6hbAkDluFbRH6WF7Eo3fCpDF6ofElq8/SMwlctYilvS5b6+0QHQppppxIExBlYM/XPR30PGLC9VStP+r3kgOoCk52U7ETOivK1UOLCRT4hpwkZZpUlJA55lDnqOlpXw/SB26KLMASqmbJnSZSOlq3/T1G9x0PqrIxyqY0bKZE8W8JjQOVgJM606uv4mLGWeKODXJi8XUkclUzgPxy+fw6Wif92JUAlCCDMzGp167WEF2AiASdYT3Znf9LRPZijOxPqnHaGp5gKXEcQ8HPwEX8wgnUhSSnUyOYsYRjfCjxTkxFQ6TMvJAHxOp3+jaGTdiUHIlESNp5bW4/dzcglIAUIOn42Cey0HJ5RG9o5ubQpyZxJh9xOZu+6JUDRQqEH7vxsbReF3KJyV7BJI2dST4c/D5WgIXQlZClogA8+Rg87JOIbjbN2jMhKe8IkjTQ76eVgP7LMtdsn4VhnaJ5OrPyrNJcbcTAWlQB3GutukwSkAwU6mOnl7rUTNzOoZxC0y842S52iileUwUDXQyPMW4oziK5GbnqF4ivZoq7R10Jr3QJUM2ozbDkDtFqR7OO5P/CsjHm82flXw7oKVE5Y5D7/G2ZSVnQCNByg/DwtQ+6cRcSW27xWrHGJEKfp/WWki9nzkBUqAJXA0jbw6WNU2OeLlHhWorE8Rb+VVracfC3atThQYVCQFzoJGnnaH/Tkx2cFMY9FzaVeiDIVl0gbGdPxtrJJGbXzHyNvPZv0kuN93qpEDiHUu/ZCUuUtOrOqSCTLcjXoeVnfeXpLcX2K1qhuzEKFpQ60h5bt3sqzZtx3UDl+O9gu7P1IOhCMMcp7agq7ZICYUox8PKLZlG6c0zp+Fqtscf+KTKJeXdLmm7lEtPjqErHSyW16VnFf94rof7PYaezVCkolh9spaBMrVDx5+7Q2G/AatvT1RI8bpT1Vt/AaxAIJIPX8rBN5ZnUk6zHW1c6v0mcXtU3af2XutYQMy4fcTmEbc4nTXW3OHfSxvW9aJVVWcPGachzIUN3sVcgebQ8R7rQOC1bfh/KmMXpHa5vwrH5SU7wIg68uluAFE66j2o52gCg9Lahr7zcov9H9elticzzde2sSOQlKZ+Njd1elphCurXrvqMI4hZcpwVKUkMLQqI2PaAn4WGcLqx8Cn7SpfnU7KkQE6dJGptoyQAJTsY5D32iVr0mMCLJbduq/mjMwaVpQ+Tp2/Gwa/Sp4QMPFirvC82SlWUlV2uKAOuncmfjaD6CpZuwqYrqZ2zwpdMqJSCRzkdYtpS1yZ0E6D7/rxtFdB6T/BO8nwyxix4Oq1SF3XVjlrqW4sof8AlBcGkDI9xCu5pQMw8lxv3d5I6iwzSVAFyw+iIKqA/GPVSHDgUQkanSPrfzt0ToCUkwdI62ZtLxk4UVlMKtniJh4spJClqr0JSI3kqgfXusoUfETh9XJD1Fjq4Hkq0Cm7yZXPl3rDMMo3afREEsR2cE4QtciVAd4zrEfWmvhbtBgyrQRvGvw87JTeJMOVaiilv67XiDJSirbUR8D9a2PNVDDolp9Czr7KgZ1/pYZjf0Uw5pO6MIUJjkByj5/E/O2AkaDSJ5zHOwYWFHMoqgHkeWn18LbzqSlRHvnX3wdrRylEuF2VBSgRruRH10toqC+6IgxB2A/KwfaFQEjz0PhOn1vbAtJMKEx3QFcvD5xZWUbC6FCiU59QowRPKLdQCSEg/pv9edggrMnvE67SLaUsAZSozEAcjvaJKmuwYICY1OoHwttJEzqIgSRM2DU5IOswdTER7vM2wuc8uhEifrxs6VkMleVZIUQRsR99ulOKyxlkRAnXnzjyFixWCISFaxuRB6W0FAR3Vd4bbkT9CyToyFKkwYMx1k6c+dug6QmZGus7/W1ipdSUwhwCOXIfUW2pxRJJ0O4gaxy++zWCSNKcIUcsmddtBytovpKjBjoQTOuv4WJl5RkgzsRJ6iwangYO2mYg/n7trSDVEmyP9uToTqfZ12130sEqpHsyRrIJHlNiaqlJ0JBCvjFgXXEpEkiZJiY08PfZw3VRLilNqpBVMgbETNjKKrUCeUab9bICKgSJJ3MTsff8LG0VEphIJJ1B8Pj7rMWJw5KwqjKjm55o5/hbaakwVZ9dBqoiPGyUupKUyVagxG2vT8LcqqFgSBqNJmPo/XiW4afNZLPrJjRZnWTGtsTVd3KIEDUn+vnZFFWUJ3X+m5+vDlbBWLygKJEHy1suGlnS6mqBmDHgJ+H11sIKyEgKOaTAmZ/TnZvpvA6lWyTuDpYRNcEggEkHQTp9fpaPCTh10uOVatSTqd4Jg2Lmr725lMbaT9RZLXWAolZ1Gm02LqrkhRVMfAg6aizhlki5ORutGxWMp8dvz/pZTo6sFtcFJnfys0KatzDTNodROtlGgqVnOkq0KfcRaBbZLdLpqydDG068tpsEp5gnVttUn+UHz+6yaag6EqV4gjQ2CXXJQM4OaRKoO4+hZ81gllBSsmoabOdpCUET7Jgz7udtP33VtKDSbyvELUe41TVTqVK2/lULI1JU1d6KPqaQloEzUESmP8P8x+X3Wcd03PTU+wzuq0W6rVa4Okn8LMJJHGzSmLY2i7gnDgdnEVbfTNfeV/3mplkLy0i61x1pRUIClhRMkSYjTztNaVZkg9QCLRth1k07YUBEQR10tIVAsqpW9ZKQU/AxPyt22FROjhAcbkrk8ReJJbhGDrbRVp525nTe2iQdd7a4aLLOJXRX15W0VTrtbmfG2ieVnAsok3XW1smIOmh2tzOsb+VsJ5mzlocLFMmRV3RjK5L2L1zXh69d7qlKbpnSntGZM5Ez7SZmNRAPQAErXY9vy43xTX1hvZIKnUdqgBJMaDIoEyU/xRvrMAviukobcSfYWknlodDbeYwRy3g7W4ebsxiFJK6TC6xzAfhd4mj6X2V1tVG4ASsv57KOF8fMI0rymaihqUAJIU4mHFZp0AToIy7nltrvZZf4/wDCu80UbbN8qYeeb7qC0SJy5ssolJVoRCSTNnU7hnDl407tRfVyUFQh320rpkFThG0kjxPxNilbwywFW3EXWMMUzKHKz1tbbRyDtu+kKJTEwHFGOsdLYsrMcjjk45ZIG3Bdb8bIzXQZ25QQmO5xF4fioXSoxdS17j7SnBSsKCh2YgKK1L7oyg7CI3M72MUmELqxAh69KW6WmrwdUGVoDhabyIMJBMFKgmVQcp1UYiZshYj4HcN27s7Wsqa+7RSNoWqqbqsqWw3uvvEhOkydI5QNLMa+cLYyuXDVy1XC2/7/AE0ja6pp1uoS409eXbrCWsichyjKlULzIIkuSmc1vKMQpe8zNe1uRx0NtB+FsZ3NYU4rxuTCtDeFVRHihTUBp3ltGkRWUiE0+VRHZhJZJSExEE6RbLEWeFHEBTKFJuTBNOCkEMvUNM843p7KnOz76hsVczrbLZ/cJvmP4S4jlVP9pNUpbuHA7BOUqrq1QBjk0z+dq28I6eWrt7MSDVg6bATBB9yp8vdawH7S99xDGAWCrRTt5LO2kJpgNDrz5HwtBPB1A7G68xzpU6lUE+z3x9Tb6R7JtLcNi+/7rP7Q61sn0H7KeKenSIUsSSo6DkdOtjrbYJC9DmICoET0tlM2pwDMgEkHNInT85mxxLYKc5B08OfXT+nhbuFygRfIrMogaROkR9fl8BEtNgxlJA32P1z+Fhy1kMqGmggE727CTIzJlKo5x0G5sk65CJMGTCgY3M68vra3RbkBAUZIzHymJ+vnbaUFaQsDQp1HP3WFSMqJ3BkwTt5zZDRIhFyjvQgRBInqT08bYWwO7Mge4T+n52M5MyjOm5nr9aWEbQQsyCDMGRyHM2kHFRyhFcpSc0kkecxufrzsAtrKMxIRzKuWn32PKbCiEqSCmZ328PDnYFxAyBKhA9k676nw8voaECE4KC+MrahiOiISdaQZknYnOrY7c/6c5cw22n9y0BKSqaVklQH+FM/1tFfGhJRiChJghVJ/ESNlqJJjfl8JtLeFU57luxYUrL6ozrm1IyD42gz3ipP9wJZbaISUlsTvH3/WlhlNKIyJza/ETH142xtGcAHQ/wAR2n4+7ysaaQYOg8OY+trFQwEWLQ7xJMnWIGn4HT77dFgwEwkbR4fp+djfZj2tI156T93LrztvsUrHsEDw+ulo3U7IoWUpJ7gzAyZEST9fdZMxAwTdxICNFpmY67n4ffZdDZXCkchtoeX52T8QJX+7nNUg5kED3jW0XHRTbuoZviA/eFIyAFVLzWYdEpRJPjsB7+Vhb2ok1d03YyhP2joQhJUdgpOp+HzNlKqpEO3teb24ap8gB5lSdfkPn4WTW+1rKq7lQezplMsmP5imTGmp0A+NqGbkrFlzeLTF33g72iE5fUuzSOW8DTz/AB5WLUNMheGkpyaBtQIPvH5Gyte9I1WX4xTqjKhBcWAYzZRIHjqQSPKyMmqcpbhbpaZMvvFSUgEEAZjJ6xFptfYprKBb3pHmK5Tysqe3ddcKdw0M+VKd5OieelpRormYo7rulpuCpTiHXFlMFaiJ931Fo6xE4WqhtDolP2kAkDXOSmeca9eZ87Sqwkqu26Vbn7E9f4Z002P4fCBsnA0TuqA1R0q6l0oyNpiTzPT5WRsN3emorK2ucT31wkEkbSZge762so1ua97yRdjTn2KCHH1Ae0BGnXpbq4WQ1X3gltEDtIUEnQROnhaRdchK1gha272QytISkqAMHmRGv3fGzWomE09xrZbQO1edUG4TrsNdeY39/us/H2dCHEgI1EnrtZGue7kVdT61ADFNDbKDBM/zH6+60HHVSaNLopdmHG7vo0oDaO1UQtxQT/F+m30DZKu66kJxLVKDY7wV3ADA202s+XWgDnI9rXaIjn91kFhnJiCoU6YzJUdTpGUbfP6FmcQLJgCUUvWjFJTKAZCVnRIVy8fAAeFmfUXE072knvkZySYzHX8/nZ53i4qodW842CkHKmfd18vqLFmbue9Vqatxak5Gzl19o/kPxsF7s5RWtyjVJFwYWpmKUVicgdfkkjUAdBOu/WzB4k0IbecEGcqenvEDXl89xaZLsSk3WxBJ0UN4jvfrtaMOKzYS444dEhgEmRroQJ58jItZAGSyCb3RTBN3ovLBbqlnMkLdSYSCSNT+nu52cd03MmiuVplLIzrnKnTNr0/LnvYjwkdQrDdWHmwsIqlSkEHTKkhPnr8xZ+XFTprX1VruVQbgITyST7unjy8bDIBUwSBdIl34KapmnHXG0l92C4qNhyA+P1AsYOHuzWClJSdwdR3hOvh8vyea2kkynNlnQxEdD+FuCyAdUcpiDKfeLTDG9FDO7qo9xFdl4Ju9Cma+rbhWYFD6kyNddCAN7GrmcxUzd1N2OJb3QpDeXMK51K9CRMhXn9buW/KUOXevKjREKk/l749/nYG6adJu5KSTCSrKOe8zPx+tLC4UZdqFMSvtoSiTN/cQWyezxtf4O4AvB7QjpKo6fKxhOPeKlMO7ja9lqTIBcqCsnror+ulj6aNKBJQoZRzgz0H3WTbwCQU09OkqcWSICeXTyGnwtF9PCfhHopiomGzj6oGt40ca2F+rXdjmpLuaO/TML0I3lTZ0++OQsoYp4u+krg/hg/xFXfrT1C1WNUYfqbqZCHXFSSlMIGgA1JO+mpmF7h1g7C1Vei3MXX01Q0zKO1VmHffXE5EkAgDqTy+QvpT4oocTcNqy4KKrpX6dllK2GaRRDFOlsQlOXROaNNtLUZKWHihjYx56K0yqmDC4vPqouub00+Nj1DSVFQbgdLxhal3eofxRPdcH3cjpaa7m4/cQ6ulpqiqoLmUt5tLpCWXEjUeKzHS1G7lX2V00iyUhbLyiNJKe94ef3WtzhRpp65aFUJI7IAGOmnP6/EU9BT5tGBWoK6e2ripCTx3xZlh24LscgScq3ESOe5Nkq/PSkvy4KhltfD9ipQ6gqzpvFTeUjSILZ/PXaxJujbKMyZlQjMNT8fePl7mvjS52nfVnSgfxJChGpOU/He1Y4bTk+6rPf57XDk6GPTGXH+s8M6hJ3IbvZKtCd9WR9T0saR6ZGHf/AJ7gS/mlRPdeZWNh1I18LRA7c7JQSGQMoBjw2Aj3jpvYBVxMKSW0tp7o1JOoPL36j4D3L2VTnl+VH2nUDmpuR6YeAlJHbYaxM1lmfsGD16PWNtel5woWtCapF9UxIMldATH/AAqMjpavb2HaYBLgZAVmgDLAEdAOcWKvYcplgpI12MEg+f3mzeyYPNP7UnHRWcR6VXBgpleJ6tnQBIcuupHjuEH4Hpbs+lBwOdjtOINKwSY+2pn2yNBvmQLVQqML07g6zCenWJJnx+jZo42wkGsNV1a0oyhtLglKYhKhMabxvEa++0m4LATuUxxidvIK9NL6QnBeoyqa4mXCkL0GasSjXc+15iy7S8YeGFWlKafiFhxcjSLzYk/80g/lbyjCQbt9sgtPkAQJOYfH+HytJdzYeYqqBiqzD7RlKylQkAESZ16CedmmwOJguHFPHjUh3aF6UMYzwvWpilxJdT0iU5K1pU9I71jib2onATT1jbo1EoWDO/IG3m8jB7Trax2bcKVuoazHL5WLVGEG0oIZSEagHQjWfDw/rztV9jt+b8I/tk/KvS1FQpau6k/zbG3K6gpCcxUDyMxH19b28z3LrvmnQUUF41dO4ASns6laSDsBIIj+lsums4puVCGbpxriRnZIKLzqEADzzac/CzDBbj3/AMKQxgbZPyvS4ViidVTqBBM/1sIKs5j3zoNzbznqsY8WsPIqUq4o4peebpi73r3dUEHKCAApWuubWOVvQv8AZ3XhT8WeCV4V3EBIvy97txA/Rqq6wZnuzNOw4hJOhgKWuPfajX0JoYeNe4urVPiLZn5Mtij5qEwUp0OsgdOmti5qiFHKdxsTv5bWss5wqwC6sFWHGgRp3XXBpHgqxKp4LcPHUn/sVxGsyiqdH/VbD76w7hXuICq/U1WlR9rSPM/W1lGmvBKVxmmQR3Ty52l1XAjA+cqbXejRIiE1UwPDMk2S724DXOKCoVdOIr4YqA2sslamVoC4MSOzBI20BGnjrYT62MorHC6jV683C6mnZK3HVaIQg5iT4R/TX4H6G4nqpSKm9FglGqaZJ+zH+b+Y+ER52Gum62LraCAkrd0Lji9VrPOSPHlZXZlXskkA9d7FAL9kYkN3RylaSAlISBAGg5eHh+ll27GSVhMjTWI2sl0jXf118J+tP0s4rsaOYCPr8OVtehpLuuVm1U9gnJdyQ20kZrPK63SqkE8o+YBsz6YykQIB3s5bidzNKTGwAHun8xbsKZuUWXLzuuSUqGB8banlOm9sVrvbU8totcVRbm2TG/W3JM79LamCRFkkuiRFtkwIm3OYDf3W0SOR2sRouoOKDqgVU6wD/DP425bV2gSQfaiwiwFAid50sVo1zTo1hSZBM7EGzSNuNEmmyUq+oKECjbVlSkArG0+FlC4FdvdlVTkAlBzDnuP0sgLJJJJ1O5ssYUdiqeZ3K2iY6kEfgbZbaFlPSmLfmfPqiGQl11H/ABGuqoqVJqQhD9Oy1nFKoEIUvcqXyUAIhJEAyTJCSmMLzvXE17KqLwF6OpZpHwXVKJyoQQo5UnRIVmLZyiSRrGk2mrHN2P19E7TuVK6OnAUlRa0dOh0kapBMajXe0d3Fd923vclbhp1TdGhNZTpT2RgqK1wkRtEiNAJ6zJt5RiBMMksnC8LXN3Gtr62C6mN+aJrk3W71xU42lxqVoUkKSrt3u8Dsf7s/fbLOV65eKSnVqoqinYpyollputShLaJ7qQk6pAECDtbLQ7/S/K3+fZQzu6/lUG/aVvKVeWBGEgZkNXkSOYzGmA+vC0QcIkZKe5pEAuiIGhAWB9eZtJH7Ri8k3jfOAXAsEO3bVVGUnRWdTUjwMDf87R/wUSU0tymA4hIlBI0iQdB7z7yT429I7Li2HQ38/wByqOPm9ZL/ADkp2aASNikpAO+qdY+jZQQlBg77FSYnTnYGmaTGpEATO342NMtpSoOJiRqe7rH1Nu0K5Zi67M5YkmAYB2nztgbymQM3Lp5/hYSFAaKEbQR7/fbbYUUTOidyTr4/cfrdlPZBoQgEbmTHiNNP6WES0kiQiSrQfXw/QW6y94AnbUbaH6Bt0EEJIBA0gac+k/W1m5p7LhSE54ClGFSRzjrtboJU4EqKozAiQNhpr8bCBoklKN9Y6+/4W6S0QrMCRGsxtvPO0m7qB8kGpvPBUIM6dB5xBPLf52LuAgArPdXvpPWx0tiCDlOgOmgGvv8ACxd9sEEpOkjnqfHxP1pYgKE7RQZxqQE3/QqOX/zQxzg51a+7r4Wl7CaALguskpBNIyCUjXRCefx+t4e45PKYxBQKUTrSGNZjvqncdPqdbTNg1Pa4ZupW00DJI13yDT7t9bQabuKd3uBLSQZKROWZAj4a/X42MBJI3WDI5azvrP1zsEhpSITm217p5yPzsZQjuyRmKRoDpE8vjH5WndIN6LBmTmkjWNfztsypMkH4cuk8v6+YFyiFKBSrcpgfKOv1pbZaK0yjRJEz9dPqLQzKeRcFtt2cqBr743EWTL/Qk3O4Z1BTBPPUH8N7KxCj9mEazESfLymCbJ+Ims91FRQUwtChBAHtb/daDz4bqQteyi90ttVV7uOQlKEImTzKNuvhYqpluguq7HFFKQt8PLOpjuk8t401t1eqKh6tr6OTlWQ+4QmYQhEhPxiw95ssv0V00IVlU/2YIgk5con4gx77Z4N1bLbJKo0OPXkqrqMx9aZcdyie6g+yPKBYvddChFG7eJbkraWhnT2UDUnwPP8ALmtXm63R3q4pxSAlqiCQBuCTAjnrG1ilKhCMLqTkDf2CiSeW/X6199pNOqidlX/GjRpn6YQFFRWVACSQSCPHcWk27Uo/svdlQ4AUtNNEZjrITt933+UZY9palm9Gal4q7J5aQwkz/dhIOY6fxEmz6w2upvDD7JGZNPQ07bZgxLmg5dPw8bO86aKLVI+H6FVPSl99RNRU/aKJBA6xB56j5W1c6CL5vBuSZykz0/rZQuxSnKKnehXebSYI8Jj77J5qEXbfNY45KR2IVrpJ7unhaDn2AUgCSu72ddqH27sphC1n7VSU6BM8/G3GG0LS1UtGe67IEb6Gxy46NSkLvCpb+2qSdDyG+lgsPqIfrG0ggBYJnzMHX6087QzG4KJawsj5aK+6ogAgaQTGo+o/OzdvbsKeqdQ2SpxyM5iMqQBp8RPw5WcNfVikpy4QS4rQCdvrrZqOrK3Vd2XHPdJ6WhLJyTxs5oJqkdrKhLSUwZIJCdBG+mll6spG6W7FtgiEtkeY+P177D3TdvqlPnWgdusBR5EeH3W6vQoNE8sEphByab9NPrn7nYLC5TO1KR7obP7oQlaZIKok+PT4j32jjiqwFtKKk589OSJ5mTy56H52kq6gP3bK1kgFYM777fO0XcWq0tVtBRM95VRmSokR3BqZ8ADr9G1kP8KCW3KT+EKXqluupW5Qy26HVkARqiABvqcvwFpQwgo9k/3CVSkGTBMg7cuQ26jbW0bcEXgp++KYO6lDK99J74jbTUm0l4UIVVVCTmAgEEkDYxH3fG0Bo4JyLhONLZCipSSeY1326/nbakhw6DWADzsOlGkAgAwIgQr3e/5W0sFMqCASZGu4672IXWQ7JLvBkqpnwRMpzCI0ixK4GvsXACkd8qM6GCBH3fA2W6hsKYWI1UCkTrB26a7WS6JtV2UxU6lQccOYJHtJ6DTn9eAA99ijMbosq3Q0kpaBUpZhISZPhGus6+VuWKBDRDrywt5emugG8gfP42NUtEsE1dSk9sU6g6ZQY+vo2HWhKjmKQogzod9R+vnr75NN9SmcCNAiLzZ3TGkakAacvo2bmM6P1nDN604JhdK4BG3sGB152dikQ2nMZCdo0PXp52SL3Z9YoqhkNkpW2pJ25jUx7+tjB10NVRuBIXcyykat1RV7MwCgfltztbjhs8iqwrdz8JKS0CAozlJE/cd9/K1RbkzBm8KZ0GW3EmOcDMAZPP3aT7rWo4K1SarBVGAUnswEiCemgM9NeVqE+hV+nN1IiEgJAUAQQSTOpEneOW39bI2LqcGibWmCpL0qy7TB2+htZaSTpBMTATMc+envslYoYcXdayNezWlUGZJ1n8dPO1cHW6snayZi2gV51AFIgrIIPL5crBqaSVShISrLMTPTY9NbGiVQViNeih8Pr8bAkKIEqBUvUEyDp4jz+fvtO6FlKKvNlXdciBAJ2B3nfwn6mxdynASI72ZRMxzjWden3WOrHIQpKRIMz8jz0+psC4DEEAzqSZUfv6zr158rSDkxBCSnWoKgITExlEyfDf7vush4hoVVWH70pkmC9RvBIAgHuqgT5x9HV0Og6JKSJEGBoT4AddLEFNNrXkUkZXDliNwRB/H84sVj0J4uquIAVd9W2D7K23B0AkiZ/wB4b/jaXsBn1jDdA8pBSpLYbHkFZZHwn32iembLSLwpCAn7I7nmlQPTwNpO4XuqXcLbaiAWX3G8pXBjQxHLVXzsSY+FDjGqejdMFNwptMxBAJkDbltPv0FsqaVJHdgGQoZun5ef42PUTC6hQQhuSNxMjnpZy0FwBoBdWiVghRBGsxvp5WomSytNjzJrXfhior3gqpQUN6QojU6A6/1jfay8i6mKJnsmGkJ3IjnJ308ffvZwBlKUZUaRHs7j6j3WK1DBSCoAFMSVAamPMeXwtHPcooZl2UW44owmrJKSQ/TwT0MFPlOgtbv9ktfpcuLiJhtxzv09Vdt4oSVa99DzazHL+7b+ItVbHNPlbpnUp0Wl1AMTB7p085+tbTV+yuvlV38cMV4acXlbvO4XyhEmFLYqWiOgJyrXrvali8fFw+QfdEpnZKhq9SlKzBKxztysbmNvlblBOTIRJSbdnYE9Lec5rhdBayLySY8LcOd5JB59bYruqPibcrJ6687VXuRwogxPdqqS+ahoAgFXaJI5pVr8NxYmw0SrmCBtFnrji7ypdNXBP8zSvvT+NmuhgJUCD7RCtetulwz9WJpSnfZHKNGylAizgu1CUictkiiaBIlMjTUdOdl6jQEpA20Bi3WUkVlg1Ut0rMCNJ8tbLlyOFLhRAgg/E6/gbIbO4kTtZTu1woqkkiNR8zH3G2ywWWU83Fk4s3L5xbQOk7W523tkzyG/S1iyAuzANtE6c7clXKPfbWbSQR02tMC6i46LeYz0i250gA624J10+RtudvqLEAtshreaNedijJyrebVoQskacjrY2ZA+VipBTVL6KQFaWdzdFEHVbVI11ix+4ngzezCp9olHxEWIK2gHytth0svNu6koWFAdYM2DK27SpDeyUsWs9omqYBGYgKA8x+dq135dj1ViE063VMtzmcWnVSUDfKB7SuSRzMWtDilphoLvR1xLbXq5zqJgBI1k/G1c78pqyrvh6vraL1elzzTU7iJW8mBC3kqEATJDfP8AiiChXA4g8xVRZzcNP2XTULg+nt0QPaYjGlFganXT/wCxUuhU6pSP4SVz3zEd7nvbLAOofcdW4uuqFKUokkrJJNstR9nu6D1KNlcqD/tFgaTGmEaBpay0xdT/AHQnUA1CgnUchlHysyuF1+MXTd1yPLoHnQzTNlWWO8ZPMxvHys6v2j7yFcVsOsqmUXGDIMb1Tw3jwP1oWRgVtCLgu7MpK1+rwVAbwpQ0A8vD3W3sHlMGFwuG9v7oGKxiXEJQVNdPxNuwIzLuq8UR3vYQfOO9ZQb4lXCtKZo7wEawGQdd/wCbxtGjJCtU6a5QdPrp8LH29dycpSFEnXz92/uNtH2vUeSoDDYTsFJKeIuG1EZk1if4ZNOdtddPrawox7hnNnFY+haeZZXEzt4/raNVEpkZuUkbe4/XW3TTn8JXqTMRI/TSyGMT9AkcNiHVSYMcYXWkAXmQo7Zm3E6c+Xn1sO3jDDIhRvZCTMGUKTp7x5fnaMkEBGZAhQ21Ik/ntpbBlKjISSNdYgjp7xtaYxiboE3s2MDcqV04pw2vX990yIOmZUfH65zYdWJMPLISL8oDP8PbDX3ToNrRGMqRGSQARp05aDe3C0oKspQEzoNDprO48/lYgxqQbtCGcNZyKmBV93MQUi9qEmJCvWExoeUmwLl6XcpJQmvpVyP++STHM/faHVBKgVkp03ISPGTp4jrZOqG0IGbKmBpqkSBsPf8ApY7cZdzahPwxp+JEvSEvClZxJdrjdQFAUiwSjvJJznQxv+k8rTrgOoZVhO53UPNyqgp9AQdChO9qecTKhulvWlhOUlgmAIJ1M/Ln/W0o4JQ1+4LtUlrL/qrZPjmSNz1384sV+KGJoky7oLaASeAHZWYbamCkkjeM31P62MpaWRGYmIJ3133tBbSyJKHFpIJEhagN9R9a2OsuViFQK6rBA0yvLEbfgRYftxp3Z+UZuFH5lNqUrACgjUGSSd9bcKQFhKUCCVd46yPH8LQ81WXoFAi96+I0ioWIO/M+djLd832lRIviviT7T50MzpPOyGNM5tKRwx4+JS6G3CAFEnmfHY2IYhZIu1yRGVSDB0nW0dtYhv8A3F9VszrLkgfEfXxt07ft9PMFt+931pJAhYSIM7xG82TsZiItlKQwuS97hE32A5UX5ULTK0NpQCBqAETHjr93xTqZtdVeFBXLJDbbiWGkjolBzGOWv11UC5VqU8fXCF1Byud0d6NNfjbhHrKAwhupBTTd5AKEwPqdz8LVe/tKOaN3VFr4o01l8upPeRTsdq4J10BgQOROvusRvBwUWEgy7MvU6kpOYaq1P6+6ymk1HbPVPaoLtQkocJQIUnb3crJN6Ui1sIYWpPZtJLaO54Dx6c7TbXNuhmjcBZQ7xUZ7J66lJJByiFCTpkG3U6c4G3lZ8YTpWlYNYDaEguskqjfNJnz208rR/wAYKqpbvOkaecZ7JDSShITBBGk76iJ+Nnnw9rKp7DVIhCmw0UrQAUmR3j+mnjaw+qaGhyC2nObKpDw04hV0UjhgwkT3toJ1+XzsDWpRVXy1Qg5QofaKJjNl5H652Bul6qoKRNIylooaGxmYOw315/EWEpaNVPVtVqVBS0KKzmJAVJ5fOwXVjDZGbSuGqcJROVKR3tk8tIgCOnvsjXQtDV716VqgDveIEz79Dv8AfY2bwqITmpW+8BJC4/CxBbalOVD7bZSuoA2cnTy8TFovrGHZJtM7mid61yqlztcwDY0SNNxGo+uXhYK7Uti8qclMyCSOR9qPmLDLux1xwLUgZZ7wB1jw6GxhxsCuZq2WMqGk5coAJPvJsFtQ1zrlT4JAsEuISMwgFI72ggz+ViN5qc9SqAncIUSE68j+thHL0bylAZeMa6QZ8NPrX4l3q5laFtFh0ZioSE/PQ9LWzUsI3QOA66Q2K4UlzmQEuFZSgH+Lx6xIsyMaXS4xd5r6pJdfcQ5EicgIGg+vus+W26cVIqKptamW0yygJkCOZ+uYs1uJV9XexdiVOdoCHMmrSiCMp0+AP67WnBOwmxKhJC7kEyOBr039eTedULpkkjlIWPd/EOfIWlXC5AveqYSpUAGQobd4T56RaFODla1R44qEuGErYdRojUnMlXlyO3M6bWmi5nWmL3cqDPZrCyFAEASZGmp5b+FiPka0i6i2IkJ6AKTA3MyUawR/S3OYgBOYka6kxA931HSwCK+kVC0uaSQRkIjwPxH0bcmto0JEOglIjQHX6+uVkZ29VEQO6IV5wNBRLkEnKJ1InfSyPdrvrN4rU+SVISSBG2oBP11saaNPVPqfqqlKUtGEJGhmN94PKxK7ghm+M7iClolSc61aGNdLAMoJRhGQFIGFsHVV/IXWVDTjVC2YK0IJ7VendSQfif0s9m8GUraRU1NMM5gtsKplAFPXYwPrlobuevw6zRU1QusoA22kIYYC2+8rbvQsGPPebNfGePGKtLt3XJUoGfR6oByk9W24J7um/uEWpcaSaTKDYKwI2sbcpv4weuamqF3fdTVO6AYdqENwkGdkaTp1+iyKtoOtKacSoiNiY57+X15KroQ6jKciSDoREDr9eNiL7SYIXCzqAAY+vO2vEWsFrrPeCTeyqOKcUWIL+oVpMocUlJCfZhyNdOh/K1h+AdSp3C6k9pmLTxgTMyVaef6+doHxcw1Q8R79SogBa3VeJkJVtvyPnAtL/o5VYcoK6mCtW3NRMwAE7aaaqI91gT66hWafQ2U5D2+6ZI7x05EffFid+JWq6X4Ue6kKBnQAEH362HaUEbKEAQOQI5gDblbi8kLVdz7eTMOzUkBPPQ6j77Vb9VbITEhQbUSCCkb69I5+/TpFtKme9OVEgac9dfo9LYMygM57vTbXrqZ5fK2nFQAoAZTIJjYzB9/62WYqGqDcSogbxJ0nlpt7jy8bF3CDIShSkxp4zr7/AOlhnDIClRMZRIJEDX387BrAGYrVPj7IJ16+X1ye9kxF0TdSCjNmMk/xaxA/IfebE1kpJVm0kyEmI8vl030sfWghBRlM6Du6H3aa+Fk5yCoqbJQREa6yNxy+Hl0ixGOUCxV3vOhFLi+9qKAn7eobTmJ5lUHbyP6WenBqgeq6SupUKUOzdSsmTpmTEf8AKLEca4ddYxtedcqEtpcbqEJIjtApCSr7zPzsucEHQxe16URUkKcaQ4AddEkif+bz2sSaUFhsoRM8YBUyXVd7NIj7BAznUqGm2u1lNORCIUQBvmgGfw6WLMBCTomQNBM+8DX6+VjTeZKgT7QEkAwAOURtr+NssvJWk0Zdl0AcoBSJBABA12/SxWraQpBJWJMhWpy7H8Z5/jY9nbyltXsjYlW+3vsA+vPLeaYEacvrTztNjkzhcKPcdtH1BLqMwyu97wKkq1G33WX/AEEL3GGvTAw4gryt3l6zRr5SHaRzLPh2iUgDqNhZPxe12l0PpTIIAUBmgykgTp4HkDZpcHr5GGPSM4dX2p1LSW7+uxTi9oR6ylKx55VQYsSZvFppGdQVXHhkafNe4TgCHjoIULdQcsg2yrEBKk/w6W0FZkZh1t5aTYkLpBq0FFnCM2k/G3Czp4m3bx72YHysC4tKUZlKypAkk8rVHHWysNSbf9Ga266hCRKmwHE+adfumzHbbBMQI3iNPdZ7XBii4cS1l4UNz1yav1HIHlISS33s2iVbK9kzEizYqqI0VY7Rx/dqKRP8o2+VuowAktLCg1ZLRYrVIyAEpywPvssUoAjY/hYlTtkiefjI8bH6cQZP6/X5W7inaucqDqj7UaAHaxymVkdBHIEj4WItgnU6x00mxxpQDiB0InS2gAVSKc8yJtgOu/jYGnWTTtkmTlAPiRp+Fu82k2sNF1XJXebbS3M+XwtqdJNskSdTFiAWUVu250gW0DztknXfSxGhQcV1uQZOti9QYfZPJWZJ+Gn42HzSIsWrzlbS5rCHEkkeOn42d2yiN12uY1ANuFE8rdr2iLBKj9LCIuFO9tU73w3WXPQ1C0BRQkakTBAyk/faNMQXWw9fKEO0rD6nVJQgPOFIBJ3ge1y+7ykW5HQ9cBaKSeycKYB6kH8bMfHtMUStsHTQkC3DdooA5wsbHa638IdmuwrhWL8IUyjTJuWgcS0cgWEQFAaT7FstH6KetCQBSlQjQyrX52y3M+xm/wC8fwr/AHZnX915kftFqlL3GW62CdG7gZASdv8AzioM/Dy3s1cFXrdVPcN3sKr6VCk0yM4U8kETrJE6fW+tnB+0Bq546tN51IyXDToVlOuVTjxI+B1tXJ1a26t4pUClCUyrUmAnkfceUdbdnhkAlw2Fu3hCq4i/JXSu81ZmnvK6VhJXelICdY9YTry/Hlzso095XcoS3XsKTtAcSY+B0P5m1VkrKSG8w6FJPOZPOBt91hA+4qftCkp2TG48+vLToLGND5quKu3JWtS6wCAFIlQiMwP9B+Ws2HadbKoBTpBOuny89otU3198O5mXVKyaAJJA322E2GF6VvtIraprN/K8oCYj4k7fpaPcD1TmrvrZWyQNsyMwG06/ERHW3RPfSSNCd55yBPv026/GqP7+vplGRF61k7oKX1yDrpM/P6I6MUX822rs74vAJjNHrKwJB6A9fx87IUThzS72DyVqM6T3lD3jofD32CXqBsVLmYHPQ7D3fpar6cZ4padStGILyTAlIFWoyfj5+8RGuopx7ixtMJxHeLgGhJfXI03k8pAP0ZXc39U3em9FZJxzQ6aCT3dJ2+WliNUoqbKW0knYJjbXl0HjaAP7fYxC0n+0NYRCRose6Ad9RHvNsc4i4vQslV/VCiNJUlJM890zGlptpXBRNQ0pV4tKP72ozMqDJ7qpIkL5+Pn+dpbwEQrDl2GFKmlakHQg5ROkfXlauV939XX4tp29Krt6hsZCopAAEzEAR46/dZy3XxKxPdFAxS0NSjsWWwltKmkqyoA6kbDTqeVizwOdG1oQ4pAxxKspTlJJBkEbkax+Gwjl7rH2YAKMvdOqRmmdPHztW9jjTjRvMf3hSLypEE0yRqRodNxzsKjjtjZmXlqoCEyO+wY8van4dbVO6SK02pYFZCAU9nlk6yYB2+/66WFUIJUmDodFJjLGsz00PxtXRHHvGTYSHmLqMjUqYWNOYHf1HUe7wsMj0hsWNjK7d90rIAEZXNRMT7fj9/jEe6SqXeWFWGQoIUkJSSTyKQJHTw+trDFwABMZgfCPfavzfpE4jcAcNz3Yon2gAsSf+Lp9dR//ACiL2ySrDtIoxMIeXrv4Gef0bD7tLfZOKlnVTwtaUAIKjvlgj6PU25LgUhJJVI7xM6fX6++DT6Rtdp2mGKdQkQU1SvgJR+PvsKn0iCSpbmGGxlISIqzqd49nw++bS7tKOSTp4zzU1F0qAIBUB3tOn46mxCqWSCEpmTEknnvvtytErfpF0+QH+y6wZ/8AS5IOw/gncx8bdOcfLpfht/D1WnOJBDiVQQR4Df3WkIJRyUOMzqkTjU+pm8KR5YzAtgmDrIKtvH56bWcnCt/PhOiJzjVYnb+NXnaM+ImN6PFq2amjo3EFpIQouK3Gs7acvv8AGy1gLiDd9yXQihrW3lLbUv2EAiM07kz03takY4wgc0Bj28UlTtROpQgtgTBmTv5aWPBQ9uD5QAOcjz8uvjaMabizhtJyPmqBO/2cx8DzIjrr5WUmOLeGFjv1bySZlKmTI09/3/pSLHdFazN6p/FacxCEKEkg+J6/L522H5Ep0iRI016WYSOLGEnUpzV7iQqRJaVpyMaaWEPFPCbkFN5Ebe02rpodvgfK0cjuibO3qnwp5OmZSSkaEE7W47RMEpIyzJiCT08LMxvifhRRE3qnfUqbWAPCfj99hDxFwrCc96sogTJKgTr8+vvssruie7eqd4cQSc2VI6zIUZ/rbha5OUyTsYO31+fLSzU/0hYVUlSf3ywMp1BBTMfXL9baXj/Cao/7bphEfxHmOfnHhys4DkszU4ahYKJkgREDfeeVmJxHR21wLhE5HUK05jXT4kedld3G+F1lShfVGTOxdA100jlpv/WGxjLEN01dz1TTF407q0EHKlxMjWNp87FizByE+xCjnho+lnHjICHAHUutAjYHISRt4WsPQuNyhJ7vOVbG1Z8D1JYx5QvO1CQ2XVQpaoE5CPhvHj77WJu+taUQA43ooQAsER4662tVRIcLdEGntlTnQ5pnWlWkEAnU6dfKdfEW6UmQc/dJiCInf+vysTafBR3VJWFbkqzGJ5df1sIV5+4UjMZISSZBOn5fCwMxR7BDZgDAkynT8o+vGwZCEg9xKsuswCNt/C3AczxIkAgAdNiR4crcrKoHdkxJEzI8Pz+hDOQU2UEoZSmZns0idJiBp9f1sCtDajqjUTpz32+o+duA8kyqQka6a7f0n4W5S4BJIEjqPqN/rS0muI1CcgELhxumBV9mdpVz005GxN9phMjMJkp7itD5/n42NuqXOdRmI7pJjQ/pPv52IVbpyaKHe89dNo+vdpYrZHIRYFAvE+mpqHGdcptpSHFobX3SRqpAGo+O362WOCdbUZLybYefaKVtkKbcKT3grQ/8E+4WKcYk9niFhax3XaZuROspWsae4ixDgpU9nfd5UqlFGdhCyADulUGf+I2vbw3VVotIp6o6+8sySL0q8hTmOZxXhr7/ABsfTeF4uoLZvN1SVe1KpnbSLItIoQCTBSnlMA7n4afDnNlFCiuCFpz5tTB+P3DyFqRerll03TpSezLiipJgz/KfuGse+3PqKW5Qe6YiZ6aDyibCtAwSQF5J1Pj1NuswBCUjPrzO+34j5WbiJZEVVQqcTCXzvr4Dw0+osEqiW0dHVBI1GbWPd462UFFQCAoBQG53I8dLcrAzEEwNpjNrHT42QkT5Aks0CiAAqFaScs/p1+hYSiuhIXKpUQSYB/A+Nji1JTKVAyQc2pAPv9/3W7pipOfKoZo06aTO3O0XSkJ2x3UUcWrvLF8B9KYL9CCCOUFSIHwHL9Gpwoq+yxmQCQmqYeSfEApVIkbaRp0jS0i8XGULN11KkZkkOskjYgKSf+oxaJ8CVDlNjG7H1riV5B3twoEdOZM8+vjY8RLoygvblkCsU25lgpUSAdkgwBrGsfUGx1oyBCDA2mfqbJTDxy6CEn4zHQDXysptuAqIWEjMZ0Op08/qLU72VyyHSsIkpEzqoETuOvvNizroCfZ2gGFcvKPD6i3bjyCsHRJ29qY/WxV5R7M5iND3jO/kQNOVlnsoloSHiFfb0NWyQe/TrMRJ2MfP66Qnfd4uUd5XPeVPUErpnCtKpGhBSUx4fmLTrXMocPYrEpV3DG4B029/1raAMUsqaoqdb+YervBKiZiYIJJnUfiogWuwOz+EqrM3Lqvfa6bzYvy4aG96ZQLV4UrVU2oHQpcQFA/A2Fp1ygojY2h3gLxRwvSei7w/xviu/qS7qJGHaKmefqHQAXWWw0tI5qVmbVoJOlo0xn6aGE6x9+7sK3i/RUmoNUunV2rgmJTyQN+qv8pt5lJSzOmc1jb2JC6OnAkaATb6qwuMcfXFhVBRUOmqrYJTSMEFw+KuSR4n3TaDca8TL0xJmarqkNUavYomSez8M5/2nvhO3d52hiv44YXqFKDV5KlxRUouBRUtU6lROpJgySZsg1fFi4qkdp+8RBIzTOg8J5/rYseGy3uWrZiNLAPeBKs56OmIlVeNrzutTuZNTd/bQf5m3EjT3LNpdxXR9neDdSgAJfRrA/iTpPwi1SvRp4i3NU8ZLku+nrkqcvFFRSxrJ+xUoD4pHytdHE1H6xdpdCZUwoLnwOhts4cw08wDgsnEXtkcS03TYp0QBEa766WNITA2JjeOtgGUmNSJiT52NIBMb6c58bdrBouXm1KGaPeGg+FjKCIjWbFQggwRIImw6IjqALXwqbjZOChcLjGm4Vrp11/GxgEga72IXWsFlSd/oj7osZqKlikZXUVTyGWkAqWtZypSOck6Wss91Vzuh5B1P3WyZMfGLErvvi6r0Cv3deDFTl3DTgJA8rCvV1JTKSioq2WlL9kLcCSfKbSBvqmNxojExtbYURubcTIzA6WxDrbkltSVwdcqpixAoG5QgIkfLzsBeCM1E9lE90ke7WwwMH8BbagFIKCB3tNbORfRMEXaUlxhC07KSDYNRBGwmwdAT6mlB1UglJ8wbdq02Ox1sMpwbpzYNd7RNZTKOikpUPmD+Fid90jK2jUVaQpFOMykKHtnkPrlNucIvdje4bUdHm1J167/AIWO3+3Vllxlqkbc1UVAu5Y6cukW5PtM0CAuIP2/4Wjh7iJALqO3cQ1/aryVlalOYwEvhKQJ5DLoPC2WFVQXlmMPJQJ9kPmE+A7tst5pxaXq78rq+DB0Xjt6d9QH/SBqEK2Zuuja1PIpKoj/AHrV+dRlrXg5kSZE6c51PUnfr5WnX04HQv0hL4gpJbo6FEEbf6sknbz++0Hpl111KgVZjJCANSCem34Tb1PCfDQQ/wDxCwcTN6yT6lY2tSVGRqARoRvyMcvHr87YoDdSlyRMcvH8fnYXskqUgnUgyNRrHIc+h8rbKBKUKRJUQcytCvkDa6SFVsuA6sGMqCcpMEkRJ+vEn3W5UVAlQIAXMwZBiD4jSwrjQQe1KgnNtJAn6gW6yntcx9pRIPlGs/D4RZgQmIRZK1JAJCgRJIUcsn3am2FAIhKVAHQEzJkcjtPOxwUxVBaSNJ0KYHXrrp9x6W0lghZMZyYEHkZ0jofxBsswTWRXMSU5Mqc2wI1PMj4R8Bbckp7NOYgA6H4R9c43saaZWoKzLKkKUE5MsyYO/Xx+hblDE5QpKpIAUCDKdNIB8h46WWYJWRRTnfUUglc666kzofv38ettFwk50JS2F65Rpm941A0ixkMFAyuwdu6pMwYgac4/XW3Cmm0ELDiRlB7u5MbR8+gtIOGyiWlFHu73ipPdUUyFeyRsB1Gm/vscbV9i2DrkTlPe08ZjfSefPxiwlz3BeeJ74u/D1wUIqryvCpTSUtOlQGd5ZASmSQBMxJIGnhaZ2vQz9IjKnPw+Da0gZpvOk1MAf97E/lYc9RDCAJHAfVEhgkk1YCVCZdKEw2DJE5ojXxP5xtYJeZa5LZMch8/Ly8bTw16FfpFPNKcVgVjKTAUL1o9CeUB3Tn9C2K9CL0j31AHAzKUmCVqvajEjkf7zpytXGIUvzj1R+51HyH0UFZ0hQWcqVJ001BPjHhpG3lrPJUOQzJUCU6mTrsfqNPC0/U/oH+ktUoUn+xdGlcSAb0pBqDpsvb8OlhFegj6S60JT/YmnGg0/elMZMc/tPDbUfOz+0aQacQeqXc6n5D6KAmaw0y1KU224E6QtJEgxt8v6WDcqFOKLuiQBO2kc48Plv1tOFV6FfpFUL6mXcEdotRnKiqaUEncR3o35gnWwafQ09It5SGG8BOdoo+0urZCBvBJKoH15WcVtKdQ8eqj3WcHVpUKh0lRCsoBGpSZ+Hl566+NtKqEg/wCGYWRroSOuh28vO01VXodekQy6pl7AL5KVAkoqmFpmBsQvpE6mxdfoh+kM0Y/0d1kbaPsxsdiF+PLws4q6b5x6pGnmHwlQ2y6ACjNJ1lXnMCOWg38B4WxFSkKSsoJCgQsq00k9fDTTx8LS4PRH9INtGdXDWrkzmBfYA9raAqwb/om8f0hShw3rRmyxDjPTX+PX+mtpiqpyffHqocCb5SogdcWlAEmJJgCMw6zy016fiPROuBpS0LJSdiDBkGYPjqLSo76JvpBPR2PDSvOg2daGwP8Ai+tfcNR+iP6RmY9vw0vGEnTK80ND1Gc9BaRq6fL749Uwpps3un0UXCoCRnCjEZVp0J0HhqNjH522mrWlsgFxOXuQFEx5dPaHnpaW/wDyTfSBUkp/0Y3jomP71oGQIjRX4fqGr0S+PpKh/o1vIlSZSSpoAe4K6Tt1sAVdOd3j1RO7zfKVE4qFElTqwCSMpKTsPCdLdIrFtNnvEKKdRO45e+NvM+FpSd9E3j+tAC+G145yNs7Yj/m5e/Qne2j6J/pC8uGV4qndRW0PCNFSZEbfPW0u80x+MeqjwJvlPoosNeIRKVBQTlCdZj3/AFM+Fuk16m2woAJUTqdZ023kdP62lRHol+kOMyTw4vDUnTtWRAMae34fRttHoj+kIUmeHFaNIhTrCuZnXPvy1/Oy7xTfOPVPwJj8JUTGqcKiVASdfHXnr110tyqsXlBUUnN3lZdSfr3be60wUvogekVVKLTfDerQcsqDlVTtgbjYrg6RYm96JvH8OhocOLyMzIDrcbbHvRrPXwsu9Uw0Lx6p+7T75T6KKfXnHVZjJ7+k90kdfDfn4205XJgJzglGgI0Khz193ytKi/RN9IVPcPDa8VFCoCwtBmdCR3o5DX3+YZ9E30hFZQeGl5gL9mVogR5K25bjraQq6T5x6hR7tP8AIfRQ4zULS+k5u9m2B56Tv7rCuVa/ZClCRqdI3IPly8rSqPRK9IcjKOGl4pymZzNiJJ/xfGwg9EX0hgkn/RtWmei0TtzGaTt99imtpL/1G+oUBTVA2YfRRSm8atB7RNStBBjRZSANfHwt3+/b3ZWEC8aoZdSA6RqBvvvtp4Wk1fokekIBH+jWvSMsHvI1A0/m3841t0fRJ9IVauzHDi8MwJUR2jYnx9rQ+z87OKuk+dvqEjT1Hyn0UcNYwxElv7K/7yGbUgVKwAdgfa6fR3sMnHWLUggYmvXKnQJFY5I16yR7uptITvolekOCVL4b14OnsutydDyzaa+Hwtx/5IvpEZj/APw2rIgwS41+KvGyFTRn42+oTd3qflPomEjH2L2niTiW98+bVKqxZkcxv52Ms8ScYoBR/aW8CdMoNQodQdeY5e/wmzzHokekQVZDw4rgNVBXaNxA2nveduv/ACRPSFSSP9H1WQZEesNCSdt1/Plz5WRqKP52+oTiCp+UpoDihjYDL/aWs7xMZoObXy89vH3iK4oY0nK7iKqVlGgUEq320KY6z1s8mPRA9I50Ky8OagQMxipagzznPrrv00sI/wCh36RlK6KdfDupJSAoFupZUCd9FBZE78/wtDj0fzt9Qn4FT8pUV37im/b7Uly+bwU+42ChtRbSCU9O6BOvUT8LB4exHeuGrxXX3a8EPOJUlX2YWMogzB05TItKb3oe+kS240g8PKllLyghKlVDYTqNicx+XjY8z6EnpFrcCP7FJlYE/wCvMpHQDVUWJ3ykAsXj1CYUlUTfIfRM+k40YtpUABVA6EjvBTG/iYVoNrH0ceMUJASaK7FEGP7lwaTy73h9a2XXvQr9I9Mf/IJE8z+8Kcgjr7evX6NiS/Q/9IlogKwCQnQAiupxv17/AI2HxqJ3xt9VPg1YPuH0QSOPt/JEOXNdpIJGUlyZjU+1Y4xx+rwuVYdpSEzJD6tDJ3H4+Gx2toeh76RmbshgY5hyTeFNA0HMOdB93U2T8XejJxxwPh+sxfibBC6O6buQlVQ/67TuhCSpKQYQszqRyMTaOajeQ1rhf6qWSpaLuafRLjHH15JLT1wU6lA65amB5+yfryEjDjrTrRK8NHltVDUdfZ+f32gpDqScoUZCdJJk6/1sMisWCA0U5iIJHj79dvusQ0jELvDlOjPHK7FhAcuWoTmTrlfCo59OQ/ppZSuzi5cr1WW03XVoJBKSFJ18teX1pE1+RUulXf1EglJBPj4eP4WV7jrlivbICiVEtgde78/zJsCWlAaSEWOc31Ur4/xjd9+0NMzTU1ShxpalhKgADKRpIO8gfQ0iukrBSYhYq+yJSxWdovL3tA5Ow8Of3aWXax5LjYhJWU+zmIBEadekWZ9epxFdUKIgFSSAkyIgRty0+Vo0guCE8ztQVYCnxtc0aLeSdDBR8Bryj4ab2UqfHdwZUpD7vd6smY2AiNbQ9TPj1ftE5hm/igxG8dI/DnBix/tlBKFgpIA0gkpBJ+ufI62pvaQUZshKls41uDIhSqogZjqELmdNzFuFYtuIqCPXBKgIJQrmPL9NLROmpKWgSVJMAeKvx5+Fki9cXXfdMtBYdeEdxBTod9Ty3PWyjifKbNCd8wbq5THV4iuQs9qLyQhIGq4IHI7kRv8Al1tA+PMTXS9X1t33YoPoeqSS4kygJzT3dBOs8ufibN2+cVXnfK1B15LTSlZuzbACfCepFkJwkmSZ1mbbNLRcLV+6zqiqzCzVKmAr341cQGKfB2GsSvP0eH6ZRpKOodSWqdtTpWoNpUCBK1knTnvtZ4VHDb0j2UFa3KF0azFOwZ08W+kfK0HXNiG+MM3i5W3FeD1G8oFGdpUEpkGPEaDeztp+PHFSmbDIxdUOJ59q22v/AMSfK0Kimnz3pwy3mNUemqKcMtOXX8inJX3Fx/pCfWKJokDVSael0Hnl0skOPcYKYAVFA2SZGrLIM6jlEGNNNbJr/GviHUp/1i/AuTJ/1dvU/DysQe4nYrqEdm7XpI6hpI6fDblppZmQVHxsYmfNBfwOcpd9HfG3ELD/AB84eXne1MGqRGJLvaqF5QPsnH0tq2P8qjb3KdYTUMuMLHdcSUH32+di7uI+IaO8qO8PW1BdLUIqElOhBSoEa+75+VvoiuO9Ke/blu++6Q5mbwpWqto8ihxAUD8CLY+LQujex5aB9FcoZQ4OaCT9UzUMqbWppYhTZKTpzBiw7YHjoI99j1/U3YXipYHddAXMc9j8wbEgNhyGnutqUzszQ5VphZ1l2lMkdPusKkaagadDYNPtDTT8bCjUTpBtoBU3C4SldLgJKBznX4R+NkXibct8X9hh2guJCXKntW1ltS8gWkKBIn3A+6ynd6gH0jxB+8fjY7etYLuu2prigrFOytzKNzlEx8rHZq2yrk2ddRTd11qwfii7qekrHl1D6GV1dOp3tQytRhQSdwkj+HbSQBNlHEmGb7vS+r2qat6poKdDanmK5JbWjRIypyqkx10G2h52TOGlddt5V1RinEd5sisdcK0pWSYJG/kBoB0FnLxFvdbyKO6KJ0BqqAdcczQnL/DJ6c/daLcoYTyRHE5gk2lvq8nsBVQLq0inqGm8xOobUpMp6xr8LJlzvV+Hb4u+9hea3ae8hPZFKQG8qsq0SB3gZB11BnXYB4C6rsRgqtua7K2nqX/V1uKLa0qUpwCZIHjHysxql1NXh673WlhS2Ktad9Q24jMD5EpsW1gChHc2Srik35e2ILypKGuS1UU4KqZK0lSIS2F5YBETr3ht42e3D2+n73uQesvKdcYUE5lGVKQQCCTz52adfUBnGN3Vyu4iupmHFA6DWUq+Ay2P8MnF0lfeF1qOjcoTP/q1lA+UWINDdQOye7cIdqGxpC8w94ti5UdNracBTeCyVaONgx5G21DXSyKYI1dLwprzpn9gl1M+RMGz4vCmCnV6e0i0diAd4jUG0lKWKmmYqQNHGwr4ibZWKQiSFwPRGhcWu0TGdoQHVjXRR5+Nss5F0iSonJubZbzI056Lb72V4K+mj3/SIxElBVPZUKYH/wCqtdbQ8UqcqHgW1KHakZ52n3dTG2utpu9L6hqLz9ILEdXdzJqqZfqWR5glaTlpmgdU+ItHQTdVI5UpqaRaX+1IjMBy13Ply1t3VC7JRxNt8I/YIdb4qqQ+ZTfNOvIVdmXIMEARoRoD4mCN+Vum0KUMqgYmSkIgnu6Hw3s42b+uWmSWkUhb7Q6gqSABy5zpztpNfhDR00jp00MmUmdNSoSefS1i7uir3HVN5NOtQSCFDIJJjbTczz11526TTKSQrs1qOgEE7xp5f1svJvDCi/sadp3vryJGZQiPMzz84B6ahorMNLUnKFoSvu7geMb7fITZru6Jhbe6RhTK7RBZUQANYTOu8nx25fp2lsOBbYBQghSVExB1PMzO55zrvZ3XFdFFftO7U0YORtZQVqWB3okiOehn8rKQwa0lKgnKIEAFcmQJjlGmthGUg2RRGXahR+pgEkZDm9jQDfw6a+7QG3JowBlghSN4BHI9OUH7+VpEZwSwk5XklKcxSR2qZBBI3jflz2EeBhGBLrUQlZcAk5T2qNTrtO58+gtHj2UxASoxUylWiGxKUEFMTJH6fHnIsG9TOskZ21QRA1ISTtM/dytK5wTcwSsOVDwC+jiMpJEztzn5ixd7h/cDo1vB5IMk6tkgbRtqNPhNk2qF05pnEaJF9Hp3Jx74al0pAViiggbjvPo01/W3uUmloHgla6RgnL7RbTP3W8VOGVzU9w+kPw2pGHnHUDEt2rK1lJOY1CBuPK3tMmqCW0x/CItQxSPjuY63JXMPcY2uaeqEN33aglQoKYf/AFQ/K3aqK7VaepsE7atpsA7ejKmxThiFzq5OkeVg1VC4y5pGkmd7ZxogFeFTfdDGiu4HSiY8fsxbZobuUO9Rsmf/AFY+elihvFFKStxCnJ0ATv524RXlYzGAN4mRZdyFr2T95J0uu13bQF7tU0rIUTybEnzNulUdINSw2I/wjewZqlHvAfG2i/Jjl1tNlKBpZMZ/NGW6Ojyx6ugQP5RvtYZNFRAEdg1GsDKLE26pI3V7tjYUVOmpGg2iyNN5JhPfmhzR0mqfV24Oh7oNgxd9Eoz2KPegfjbkVOYwUjrbn1sTEj4z8LIU9uSRm80KigokqJ7BsGI9gdbCt0tIP9mkTzyixUVAnp5CbdesmRoI5gmzmm8kuN5o0mnpEpILKRp/LblynpZksok7EpFi/rYgiQD+NuVVRmUmTt52j3XySE3mhPVmEq1QjQxqgeFuls0uiuxQSR/L87F1VSo9tXh4W4NSdif8Vpik8k3H80OaSjgD1VuR/hEWDVS0YBHq6IGh7osCKobAHa3HrRGvLfe0hS+SXeD1QppqQDL6syY/wWLVdNS9lpTteyf4BrbFVBgzJG2n19aWL1dQgNgmUwCJ8bQlo/Dspsqdd0LTtUxcWoMtzl0gC3KmKUttJLLRk7FINidNUIzokiVIn3zbhNUcyNZCSNZ6k/laiaMjkrPeB1Rw0lGWCo07R3klI5Hy1sXNBSBgqFM3Ko2SNPlYJVcAxCtyo87dPVI9VmRMA6ajey7nfkl3i3NGU0VFH/mrIEfyixdykpVLX/q7crIGqR8bYa0BvMCCI5mwCKxLj8rJGUEjl9fpYvc9ALKPeB1Q1VSUbaW/sG+kBA1t2qgoVJj1VnQDdsWLVtUFJSSNlHnrNuzWJUEqAExOlpso7HZQdU+aKu0NEapI9VbgJiEpAsVqqCkfUGkU7Mjc5IPKTYeqqwHisBMJTPgd9LF2XyUhawoKWrrMaHT5TaQpNU3eQEUqbsoWlJSmnYCYBICQDv5R9TzsiY3xHhPBlzLr75yUyHCllpTdOFLJUYlIjkDvECLK1+1rV3Ublc8VFDTS3lhOphIJ08Yn5+NoQ4ecV3Mb4pp75vAyUsetBouy2wlZAbaGsGJk76gnSNLMVAXnyUHVYbZPpGI7wxS8tGCOGN64gdY0RV1zq6eka/wJ7RRSo6knK2Umedg8S1XGbDF3euO8G2qtE5nDSVDTwbSQMwCMoV58trWCw1elB6m0xSMNiESCjQKOu/Unfzsevq8l07IU6ltKlwMugPv1m1luHQBuZyn3mRzg0BQNhDGOHcbUqwzdrdLXspHrNI8hCXEGNSUnvRPgR42PVV0UTy8qaJkRqqGx3dfdzstYmuS4W3nsUMXOzS1obzKrWBCnAnXKobEECPf8C13w9Tt1KSIdAXIiACJ395+hanHTtc4taE9RI6IAlEGLtpUVHZqp2vaBAygwLNT0grgoL34K43u8UjJAw9XLSnsk6KSwpaY6EFIPmB0s9KmvS24qmTJgZdAkDNmCs8xmmAU7xB2nWxHGVMi9sKXxdgSVeu0FSwZ6LaUn8TazFSBjwQFTkqszCF4h5UZoygAmDHx+vK3aCkyFLylJCZ2/pbpdMtsqVJkHv5tANSBPvHhbvsXHXClCBCk7JTqdfrbqbdXcLm7ELaELKu6tJCRvBhX1B+Njl1lTVYyvXuuJGZI0Eke75TNgGGlzmLYJBE5hOkgayfr7jbbaG4dQhKoVmkxAjYa76RMfKwJDcEIjBrdOqoUkJCFAZld4BSOh6jWdBGvPxs2r6UoV6Va+wI7s5tT79tyfnZ1VKE9iEtmUq0UCmZI0HPw38OtmriYgJaWlMjKqToZ1EHfTUqtSpCC+yszaNunDdLinKJhxMx2aO8lUkRp74GnusHeV60tCz6xVuZDHcj2lJ5QOc/gOmjTRiZdDQM0lOwntG5+0WJiSSBHPc7/CyLUvu1DxdfcLq1ays62tMosziXbKu6osLDdLd8YxrK9JZoppmtZP8Z8yPKzdKpOqjPjbR00Bi2jMa9NBa+yNsYs0Ko97n6uWxIgfdbSgZ1523Cuu56227BAM76xFiKFlysagpESBbREDS3RBCUK1gyAffbakZR9aWSVkGCelsOltwd7ZB2skloEzMW+gL0Q8TDGPoxcM7+zlS1YcpKRxUnVynR2C/wDmaNvn9O1vaj9lpiY396Jd13ctQKsPXzeN276wVioH/wDs/K2PjTM0LXdCtDDXWkLfJWgxLThdO3UBMFCsp8jZux7vfZ6V1P6zRusACVI08+VmbBB1EG1fDn3ZlVuqbZ110AZkEeXIGwiVEka6+f1424GU9QDbsCQRlPiTbXas8lDMOdm6FE6CSfdr+Flh9hqqYcp30BbTqShaeoIg2RRBOVZ7p0PlsbK9EtTlGwtftKaRm841+djRlVnjmmgxwgwXSyKRiuYBnRuvfA+SrKd74Dui+GqVpb1Wx6m32LamX1JOTSAeu1nGCYInztuYMDexg1vRQLnHmm5h3A1Bh2rcraetrH1LQW8r7gUACZ6eXwslvcKbr9eNTQ3pV01MpZcNIkIU2kkycpUkqSPAGBsIFnwd+VtiNgdtpNpBjbWso5nXum5iXA9DiGipWEVTtFU0IinfagkJIAKVA6KSYEjwBEHW3eEsHqw4XXaiu9bfcGUrDeQAeAk/fZwiAJmBbtKjvpaQaN02Y7IpWJy1NO71KkE+Yn8LdLSDuAD0i27wgNIcP8DiVfOD+NtrEAfGzEalK6BIg66Wf2H3RU3DTmZLctn3EgfKLMJXx/GzwwW6V3fUsGfs3Mw94/S1WqbmYQptKUSjU6Wyw5RJJtluU7i1Wsy+aKnqV0TLbTbriTJBKVQo6kgT79rYupf7ZSHUOJWhSgoFKe71GniTp5c7ErrRUPM06e0lSjHdIEHYDz8PlZw3dwt4kX7eTrd10HaPIQp05qpsbECSSdTJHnNulbAXOsAhGXKLkpKRXLJyw6cw7sjUHp9DWNfAVq8AiPsXE94SFIBJOkeOhP1vZyDgFxjQO9cyEiZM1rO//F4bmxpjgDxgUpSTddOTlJE1jJ13B38D9RYvc3n4UPvLPmTTTeBUlJLJjUElrUnQgGPKI8Oe1uvXErhCKdefY/ZSYIjpB+Pws5TwF4xpQS3crJCz/BVMTIn/ABCdLcI4KcY2kR+5BABTpUMczP8AN9TaJo5OicVLOqRmbzq20GlpnapCDKy220QFH+JRKfHLyHnYRF+X0FZU1l46gpGVTkDXbnHLygC0l4G4H41qKGpcxBhvtKhD47OKhsSMp7oKFa6xqfGzne4G30hSVUmFVNgCClNQCSZTrqvTTWdxpas+FzXWyq0x4cLhyg9q/b+efSUVd4EqTlBGeQR+oHT4myg1V4zWgGnTeeoBlLSwVancAHx8J1i0v0/BLETTqVKw3UNAKGoqEk6kyPa8APjY+nh3i6iWtNLh10BJlMZVctt/r52rSCQe7H+FYiyH3nqFm2+JTqClqhvtSBBMsLUZjQ6J93Tffe3NQviTQNhdTdd9ozGVLVSLlX/LpqPhNp2RceNqVK0OYXW5BmSyAdSdsvMQPjYk7duNy6Gqm461IcGfSnUlIEDaDpyA8rAa6e+seit8OntpIbqPeB9ZflRxzwCm8EVSHEYju1MvtFsgesNnmBvJ+t/Z9tw5QZiYOlvIzCN3V9Dx64fP3hQu0q1Ygu4gKbKCQKtI5nb9PG3rg2QUJGWFRO9lUsz5TayhTuyZgDddKUCRJJIHvm2jWtoEKGw+vvsE4HMhKUmBz6e/4WJL7RRJSox1ja0WQjZNJNbZGi4y4vOSNIk/XnbsOM5co166/dZPQh51RShJUqJIAtyXSFBK5B5z52L3cHRCFQQlULY3UOe9tlxoSQSn652S0vAjQg/hbC8JIzeR/GyFMEjVEpUFSxvqPCNhbBV0/JXh42SC6IJB5RpbkvnNGaROhn8LTFK0qPeiEtqqmSYMkjTe2GpZkEEn+tkUPySAr/NNsD+XvEyN43+jZd1AS70Ut+tMzEE7RbYqmTM6TvrZE7eUwTPPlpp1t0KgACSQPPlZxTCyXenJX9Zp5g8vvtpVVTzqPjZIL8GMxHnbXrIUrUieXl9Cy7q1LvTkrmqYzSNJtz6wxzSoRpZK7bTQK0H420qo2AmDG1n7sEu9OSsaljcoG3X87cmop5jKdZ5zZJU/zUrfXfS2lPnWDM89vCzimCgak8kql6mH8A8JNi1dUU5bIybggQbEvWFaAq0+Q+tLF66oCkJSN/Pwsn04spNqTdGG6uk7Rr7OPso+FuG6qm7ErCIlwDfTSyR61kQy6NB2SjbTbw9QJB2XJ1Gus2rmlaiiqclM1jJEZdZJ1OlhV1dMaUSn+GRrHQbWRS9nDywQYlI/H7rb7UmlCBBCkxE77WburVLvTilV+8KRLIT2YTmHUzpbVLW0xdCuzVKknUzIshPvqUqAe6BAVI0+NhkuparEQdkZddtv0FnbTC6Y1Tglm8axgNCG9AQLCJq2FITDWkdddtrIdc7LG+oMkGd55fXjYQ1IQ2FKVASnlB1+vv8AGxO7tCH3lyNVVZTrUlkszm8TPl8/qbDpepk5R2QypIhP4x9bWbztSMyVpJAKp9qSQDrz8fD32URUAnTWDBEnXr+Bs4p2pjUush72cpahCW1sIhYLahJ2Pj7vvtT2huS+Lr4o3VgG6rlvK7WzXKU89UUpYUulQohK050gqCjICkyIOh1M20q35Q2pKEnKdEqEz4b6frZCvbC9Bd9WxjCiTSsXVdjaW007Lcu1DrypfccUTOZICVCAPZJObQgUwMTSQFeobTu8RTpvaivdFzihuZNTkd7jy2qs0y8kAR22vZDQkkSoxAHSHeGuDb5XxCqnDTXZR05ePrDlHfFRebi2ZIAU8/MKjWABExrEmaqrE1Jh+6n61x1LyKZrN2TRAL6tMonb3k87RvVcT3aamqKujxzhK770Lkpu11lLzCWx/CEoU2oqMR2hJEzAgRbOa9rhYro44XXzNSDju7cV0HEKsu0O3k83TKytqpL1LJSwvvJdUye46M3dggkQdRNpWo1Jprup6NISSwy23sCAQmNuXW0TMcSr0xPeP72xHRXA4+hSWQihV2mQQIUFEmCJnyI90stMLcp01B9rKmRsdt4O36/CWG5JJH25Klj7XQsjzbn/AISReCyp9WQhPUAnWf6WFgrSG3CQFd0687Gl3VUVIeqaVJU2ynv94AzBMAc9EkwOlgygBgJOsJBEiNB+v1tbS4Wq59slwvNm8sCcHLvvu8LuqnrhFRTVbtO62qrSlSVIWpJBHaCCIiPC3bWC+CRICTcS0nvDLeBAPwc00+trNLjhwLvV7jDjN+kqHEt1F91j7Tfq61aOOFYggbDONf62YrnAnEAWA1UAhR0+yXMe4dCLVXQQk6zkFXmzSgf0AVOKeG3BVxtB7O6w257ATeSiFDYH2/EWOU/CfgzUFSUpoxlBKibzc/Az0087V/HAvEaRPraZ2MMOEBU7SUiPx194DvBbFDUjtmySDA7NzUjlMb/XOzGljdtUH+fdEFU9uppwrOK4c8ICnskNNqXlCQG7wcUmYEjzAFqc3iLxrb2qaAqW8tpbjaEN6gZZmAPBO/vsv1PCLFTCe0DjWTkTnEnp7NmZX0dVdN4u3fVAh6ncKFiTuD8bXsNpGQF2WTOT+Fn4jUumDbxZP7p9XDhe4a7ClYbxuS90X96wU072YCnDRA9pJ70iF+FuVYBu+FIKqxKgdsyT06J6fh0s38J4VxBiv1v9zPpHqSUrc7R0p0VIEfCysOGWMgExWsAiYAfVpzI2tZkeGOIMllTaC9oIZdSdgXhJwbxRW0lxPO4pTXvMkrcK2UMlaEyvLCVabxNpHc9D/hS072buIL7ZOmgrGCU+ctfWlq50XD3iOw+HrvvMtOgZQtutUg+QOnX52U2cJ8Z2wkM4orW8qcoAvZxOUCDG+m3ytmzRyudeOqsFq080TW2kprlT+PQ14WOphGKr/SIkS9TDX/2c/jpZgcePRkwhwy4fuYrw3iO8q2pYqmWXGKhbSkhC5E9xIMg5fjysyxh3j8ggJxjevcOYRfbndPX2tLB3lhHjtfV2que9MUV1fd7qg4unfvhTjRXMyUqVBPPa0YW1EUjXPqgWg6jTVFnkppYnMZSkOI0KiTIC0DmEhUWuxhD0DcD4rwtc2J0cTrxT+9rvp67sk0TXd7RtKymc+sTFqzI4GcRilSfUGACrUGpRqRtz8fnZ9XXdPpQXNd1Pdd1YwvVijpEBqnp270ORCEiAlImAkACBt87HxOZ07AKSoawjfbVVMMiZA5xq4C8HZTLWfs7bkRpR8RbyWraP3ahUq5ahyyY7+zrrQFOIx7VoQJGZdyk69P7wC0c1N+el/hygqr2/ttfzbFNTlx51F4gqS0kEqVvOgmSNbM930rPSHfc7V/ipfLqoABcUhcR0lOm3Le2dDT4zKLxVTXD6f8LQmqMIjNpaZzf59VNDn7PG/hAZx4XAoEibnWNBv/tOXO16/wBndwlvvgjgfFeBb2vYV7b96tXqwoUy2cpcaDaxCiZ/uU7W88sE8YPTexrdn77wfiW9K+iDymu2TRUhSFpAJT3m9BBGm2vna5/7PnGHpPXlxFv+6ePDla/dT1zFygcfp6ZsJqUPokS0kKJKVq3kd21eU4iHcOpqGOA3A979lMCgczPTQPaep2/dX0Eyd5s0L0p/V651uNCrMNeRs8Dpz+FkLEVPq1UAanuq/C1+hdlfbqqFS3M26RQkK84iI5WETA184jS2RsInn4fWttyDzIjebbjVllbAIOpI5WVaNwqZjmFKj3mR8jZK1EwD0Nj9CsdkpAPMHT4fhY8e6rP2R0RuAdrbBIEGfjbgGBoSIttKidBz6WsoS7EDczbAflbkdZ8NLbmdY162ko2QgO8iBbtJ1iRYESnQi3YOgPwtNMua5BcpXgPayGNOY1FuGFh6nadJMqSDYdRERBI6WKUcJpuyM/ZqUjfofytB26fkuliBFnFgd/JWVFPycbzecH9TZvOa9d+dlPCjpavlkkwFSk+MiwpRdqdqeZQqT52yxssgmbZbG7mUXiBfMtcjA7aibK0/3qClMcyofD63tZ3hc36xf1YhCkIHqyl94EQntE89eg+XlaugumruW/W7uvOkeo66iqEMvsutdm42sLAIUCMyTrz6aWsrwoYR+8a59TqJSwlGTKRoVSeWu3n+G3R6vVaq0jKkZdMppAl5CkKITlSqR4ae+PrUJIKHQsEpBVm2nQHr77HcjRkKfSJEknpPM9BbYZbGZfboBGkZuu8/XxtsgmyyQUSWlWYo7Mnp3oEfX0RbR70KKRBIkg6n3fXLxsdfYYOV3t0HQagmVEe7cQDrYFbSfZDiQIHsmAfL57/rZkjqlbDRSadyASUOpMxABj4fHxstKUlStOZlI57DSLJeHWULYeQFBQDiBGsDfpZXXTLSUkJOmo5SNJ8BztTl95aEHuIE6HOAZBJH/N0sGoLQoOIKCokknfpv99jCkZm0tBG0xGmup28/usGsJSoZkERqI122/wDDHxsFGGqKqzlPaTOxPPSDpHvB+teHRnhACQlMDKPMc/d91jAZUpvvpXqoJlU7aTYvV1LdHTO1VUshptBWoxpAE/jt99mNgNVMAnRQ/jNeT0hOE7cJzJvqjUop3A9aa+Wvvjwt6ctLSpIn3z99vLS9Gaq/fSD4bLW6Kd6uvyhT2rmoZ/1lsIEc430G8nc29L2KO+0o717g5tQex/W2FWuzPuFp0oLGkFONVe+ulVQJy9i5qrTWdOfwsWU1CVJMGBpZKFDfK+7+945/3c6WGFBewSqL0UDEQECwmEnkneAOaNMVzt1ZnWMvaKGUZkzAsjl9alKWtUqUdfPf8LFqu7b47WXL3cyFRgBAsSeu+8CIN8PgjnA87W27WsqjjqlX1kb5zvEDTlbPWcycyVE6TG9oM46Y9xDw0cuGoo8Q1KGq9x9t1HZoOfKEkHVM/wAR2NowX6SWLgsZb2UtMgqJCfdACeevuFkXBptZSa0kXureqqhmJzGAfo20qoE6Ekcp3+dqjtekdiEkhV8PAjT2Uyd9+7p52EHpE4i0y31UwqIhKJ+Q0mzZ/JPa+xVtPWhB36RI0+Fs9aSRJWZiNDGlqmH0hMQd0G/a47pgBMzoZOnnbn/T5f5Ol+VyRMESkH7tbNxQOSYMPVW2FWEyc2069NLb9bkwCmRpEWqV/p7xCdVX1XA8wVjTly8j8LCU/G3EdQSVX9XQTJUXfZ+G9mNQ0clIROPNWx9ZHXXTUW5VVECSRHhoZtU2q46XyVkN35eMCRIfIMjn4f1sGjjXfjhzfvu8SAQO8+rXXz+tLN3hvRLhHqrb+tDLpJBmYtwqqMiAdOfutWq6uKtdVFJevW8jO/8ArTiQJGmk2cCMcuVCErTe9fBEE+trnx5/XvtXfXtbyRmUbnbFTv6yVaJnoLaVUkCASOW0i0L1eKKlqlacN61ozpgkVi5knzsVRiJ51p1z98Xr3RoPXV9dt7C9qMteyMMNeeam81JzBOXUE6g2BrXz2QUOZ1kfXhaCXb8rKkBDWJLwaVkIbKqt1SSepAUNJj5CwN137eBeyvX9XVCk6k+sOhOgOuUqJGvj91huxmP5URuFP+ZTCuqUthoQZlSTO0TzsIalQpHgCqA5oY+utonZv+rKCo17pzLSgFVUvTnP6+Fi19YpfpXVoReD8KUSMtQuCB0B/HwsAYzETbKjHCpAL3UwNVLqg6QFEFRiIkfXW22X3vVUgNLmDMDSfD5Wgr+2zyVgmtq1CCTNQojXbn5/CxhnHDq25Qt5Qj2u1VqfKbSOLxj4VEYa/qpmaW84+ErScqQCox8vhNhHalxNa2pKFFJESR4nnaB7yxw42goXUOqJ3KXj+BiNvhZBXjZ4KPaVTpB1Ci+slPXn9aWePFmO+FDfQObzVl66rdLEgKI0Pd1+vr3l3K1x0NUqEmcokiQR8uUm1bnMc1IkGodAgCDUbTz38/DXzsUVjp4pKy+pUaH7YzHjrpzHw6WsiuaeSGaM9VZyrqlN9inUEGIAGpnffz+djhqnNglUyf4ZJPKBGvPlap7mOHgtPioEgLk85ETry+/yKHH4OZJU0O9qC7KcwjQ5fMaeHS021YOwUDSG2pVsbxqldmlHZqJBJyxOn19TaMPSJxTjnCnCisxZgJNcm8qNTLTqW1fYKpXe64X2j3FoTIVKhoY5G0LnHynm0eqqpShLkKcCgrNtpvE7QCJJOk2ScF8QF3txiw3ct+UZXdNRUVNNLi1BLlSqle7NfLKUODunNIORW4GW9Rs71K1hGhKE+8Au11lIXDT0m7qpqumwJxDvLtkhxNHdt91tKmkReCkJHdfaStaWnpAiFqSsQZSoFFrC0103FV3c/eVxpuVmsqCpbtSwsUzq1QdStEFS45yeWsW8++M/CKsww1+/ENVd/wCFryf1p6koTVXZUFagtKqpQygGdFOA5jEqBIKmRUcU77unA1ZePD7HS8PpuqsFEblvO+k1NWpvROdpl1lSijaQl6E5TCY1tOrwaKbNNSOA6tO/mtCmx2SjLYatpPQhXK4yYwwVwJuavx/WUl2VuIq1Xq9Gl51T9RVviCltS1KKuzBBJAMQdNYtYrBF93hiXBNz37fN0JuevvS72KqpoFOZjSuLQFFsqOuk8xO0i3kRgjF1XefECi4gcW37yxxdeG6+ncqEKr1BC2i5CcgdTAbDmUqTlSFARIkm3pNgb0sfR+xYlqkfvV3DTroPZi+qZLDKjvo8gqaSnoVqTNqFPQdzbrqTuVHEcUOIuBAs0bKV62rqKcpaoah5AWkoX2f8YJ1mNPr32KB1wU4TEkAjfb61+XSbGHsPUV50qX7rZpqhh5vtkLbKS2pEe0FTlKYjWY1FkdrD9K2pxtVGhKkKKFJUgAhWs7+/4GxXAhUWmyrnxXQzR8Rb57NKFhxTa0EkZUKU2gkx5n60s2A26j7ZbCcqlykFyCRzI0mPHaNLSHxpuVFJjlKadDbKXrvac/hSBlzJ5+CR8epsyGcqJyMl0+ylS/ZSNTt5C3F1pLJ3DzXdULeJTsd5LhgMoYzZVJcKYy66HqTv10+6LH2mXngKiqaUltBCUKU53SY/hAEmZ5bEWDZoXHSArNCtBmPdHn8fnZUTdL7pSy2pdQ5tDae6B4E6nWOUb2znSW5rTZH5ImKqgQpt31N1SGgdVuAIB0hWUCOnhFqN+kXTFvi/fj4IWKksv5+SiplEkeGabeiF13DX3dFYinpW3G5WHKhtDoSRyAUCkk77T5Wpb6bVHUjinQXvUKzLr7paK3A2lAUtDriSYAAGgTbb7N1F6wsvuFido6Y9zD+hC59C6n/e2OcSYVDvZt3lcrskqEDK4gAmd4Cztr94tUOHVUzeVHedNXUyl07yXyhLQyZwyhtSRprIC9uZGuhtUT0QHew9IC7LvIKhedNVsiDz7Eug7TugfI29J7rwq5UOJHYFSU+0IgRoTJ2+Nqnauc01YDfdoP8AZT7N0sdVR+MbEqPMJ4QbovWHb3u2jNZVViqjtGmwvs0FCU5RmAgyJmOehGlnpdeAsPVdO56xdVOiUkLqHmEFI1kBMe0dzz6xvL2p8POOSw02hLSo7jXsqM8zur39DtZdpsOUdMhJqFKWoaBpG4HjytxMuIk6grq2U8UIDbKPKbhjherMIuelcQFQ48ujbAGkTJM/j4WUqbhfhCmeCV4fuqoWUHKF3Y2RmJmAI1Gp5c9rSQzdzi8qWW1JCR3SqCoeXIfCyxQ4ddbkgZVnfWVK8z+dqJrppNiVCSaCIXcAo7oOFeCSpT14YQuxxajISqiaBPQkAaWXLv4V4LcgNYNuQKX7S13YxI8u7oPnaUrlwJWVagpTEN75z3RY9el64AwIAL3vBuqrNk0rHeUVchE2uwYdW1DeLK7IzqTb05n7LBqcZhzcOFuZ3QBRbV+jvhHFNxXncacHXW4LwpHaYlNE2kDOhQnNGhE7i3hzxG4dXxw2xpeeEb/aQ1V3dVO0TyErCgl1CilQmI3B+FvoO/fHE/HFP2OFbmbw3diyUioqgUuFP8yUDX/w+dvC70trkvXCPpA4uw7ftb65W3ffNQhx9ZBLoLkpcMndSSFak7629D7JRGlc6OMuc0jciwNul9Vy+LSvqG5pi3MOQNyPryV2P2ceMMVcTOCN68JcNcMrgcbuS80et3sWUoUtD4zhTqjmKnAUKAMeyEiNJN2uG/o7v8P6lzFl8YkeqrxCFgUzCA1SthW4Cd1eZPLa1Gf2K+NW7vvfijgV1SZfYoL1Z8m1utL/APwjdvTm8757VtTM6KFliUNFS1r5JSS7Sw5BCgr6p0Igi0bz80kDbaDYnerBfoXAAJRCx7rHJ1525MKGU7EG1+E5SCjPFwQmeOsgz42wTMjQGwlS0WnnGlE91ZE++wc5uZ/AW6JhuLrHcLGy6TqOZPjv9a2OUCu8U9Qfl/8A9WKSeW/LnYakUA8kAjptGkH9LGYbFVnahKUmYjY22FQJiPEWDzE6je28wBiLWggEXQmaZTvytuTGhmbBBRmY526Ss9ZtJOhAZ8bdpUOWgsCCTBB1Put2lR+OlphQQoXsZINiVIspq6xhX8LiVjyUkfkbG83LSxMyi95HsvMdOaVf/nWi8bFONkO7oJPxFh7pdLVc06P4FhW9gHNBofhbu7z/AKwD1NmcLhIbqVBqJtlk+nvBHq7WYiciZ+FstTUrFfNrWV1bXJRet51rtTVKQl156ocK1qypAEqJJMJGk9By0tYngjUprKSovBsaPU7CoIgiQTEe+LV2qGgLlSpBRm9WJUCYkZee/M+Um00ejVWqqcIOu5iXG1JpieYIJI6clC1ujFpECqN2KbC+5AheUwJmdD0+utuVPgAwkQFTM5RP19b2Khas2VCu8dBrB1+vvtwpRUCrkkhOXTveevWOlthZeVKDT2dpSQJKVBQGXSI18d48rAhwySnQq0EeHMfpYohamnFdwEK7p1Oug+YHw0tijlcJJUrNqeYIImNfIfnaOymnThSsy1D1KUAlwBaZPdGU6/DMT7rONbyCQgiVITAVO8FJ1+MWjm7KxNBWsPAEhtUkZdxz05GNvGz8SISEoXmSQIV/MO7B+61OYWKu05u2yMMvISsApygQRIB5R+I362AcLagcqSd1eB0PTTmbdylTaSBtuNoJIGg22tpRJUZUqFnb4Db37bWDcKyAuUaryqOhJJ18T87I2Lgs3E4gIGjyFwQNO9p8I+7nZYUnLlJXKTBGg20/GfzslYlyKud7MYSSg9Ce8PlqbQkPgKLH7wUMKqOx4/8AC6pOzV/UMgE/+lta+PP6iPUlpsxMEwCZHW3lZeYnjhw1hZEX5Qmcsf8Azps/n8fHT1bZQopAJFsaUXsrrTqUWUS0A2imzLJnPPs7frboBWVRB1Ow8rGVNDmN/C2nE9xUdOfSzxhQkKJ0ztE27F6MKdaA0SkwZshrQnOpSUQCo5QRqB4+NlRxBzKmd+n142LKbjQER4WtgWVRzrlVf9M656u9qHCTNO6tuKypzZACSOzSNbVsRgG+VFIcrKzSNCEkaeY6/WptfLiPhFjFt7YeoqmQGXKhxJH+VOnvANuKDgbd7jyHiwSknNISAfAQBZxTmQ3Tcfhiyo23w/vNcj1qtBOvdKNP+WfH32PIwJXJSCl+tOoCZCZ93d+tLXsXwHuju5GVSmAJTP8ATexlrgVdqDPqvxH4Wn3MqPegqIJwHeEBWesIBPtZdd9u6PxG9hhgO8gJCKzQiCVDUdfZ00+6192uB9zpVnTTQeo0sMOCVyDUUZgHTS0DQuUxVjmqG0GAK591IcTXgaD24jbbTw+tLLF58Na273TTUVNWrbCBqp32tj003td5rgvcqSFeqGec62Wn+EFyuHtnKRJlA3+dq76B+ZWI6tll51u4FvZHdXTVZnQ/aRI3108rHKHh7XLeSk0tVvAhwxH1+Fr5VPCO5ipR9QB3GwtjPCK6GzIowPCNjafs1xG6H31oOyq1hLg9QVYS7WM3hHMetKSZ6bfPe0iUvBPCqyls3ZW6gZj+8nzO8fxxawdz8OqKmPcpwnTTay1/YulzFXYjWIITFsqfCZS7RyvR4lGOSr89wiwfTtMIF2POyCoFV41BI8j2mnWwdTwswozROrauasStEJCfX6jIZ1JIK9d/qLTveuCahwJDLJjKU5kAfH6628z8cYZ/aNqxbfbWG2sYrulNc8KVSDShJRmUE5QUhRERB3587Uzg8p0LrfVWhiUe4F1ZxXDK6xUKdapX2kJbjWteMGN9VGyUOHFAp1akNqKgDqmqdkADX+L7vO1Y2cAftBK51v8AeNJj5wsHtRD9MgDSNI2JlQ05ac7KdRgn0z3r8uupYuDH10MNPI9Y9YvoOodGcSVZj7O8gRvYLsFfb+oFaZirR8CsJT4Fu9jLLJlpfdT27vd8fasUxJgpht7KGCtJzEkvLk6jUiYn65Wl+mulS6VXb06J9kefhYLENztIC1LYQgI0BI121PytmjDZA+4Kv99YWbKBmsFpddCVUIWCDI7VZg+czZbpuHt1qYBN1hWWIJWvlp18LSUzcbZeUUJQVpakwNvjZbp7nb9VQUtCAAkmOfjaE9DKdiox1cfRV8vrANMg5WrvSlQ1lWvyP32a39imC6lpd3MnOqAC0DAM/wBfoWshfVyoVVju6BJkH5mzZaw8isvgLQM6Ge9qoHcHl9a2NTUEttSgyVMbnbKMk8MbqLKC9dFKpSid2hr05bRbP9F91ggG6qYKTHeDYkyfHn9SbTe1cgdrkJUgJSlJy6aTyI87N3irXnA+B77xGyUdtRUi+zOSD2yhkbPuUR8PO15lJITa6Z08YbchV6xJWcN8LVC6KoqbtNY2ohbDbaFKCphXe29qAdZ5RaO784jYYbzO3ZdFIcpCSpNKlZSMxmAfE6SOXIG0H4mxA8isdqHHnFKSSnvKJKgrnr4qmPDaDYOnqRW0pccKpA0bIKshH8swTtMctDzgdLT4VGwAuJJXPT4i95s0WT3vjiPeb1S0G10rjTaloW2uUKAncKghIGndMb++xa4MT1lNja4L6CkMro71pS2oHulKXRmBnWSFDX/EnlNmiw/V0jpDThAI0UZJBkAgSNyY2nkecEdtpVMhT2dSHCgrStIyKROqCCBMSEEHQg8tBGxTMbBI1zRss6SV0gsSvQXGV4YGwhfd4UeKcU3BS0t5MN1FXd1dVM9sTBAWlgErIUIBASQYEHS1cr89HrhXiJmuxzhelCrsCnOzbu+uz0bikGFZA2SQSdShKkgA6AaS9MK8McO35h6nqWKSodVWzUVL1QhtLri1AZlOEqKnDMklYk69CLC02AcRYAVUN4Dfao1X19nWNICGGe6f78NQWe0RoElKEiFwSqBbpMS7LT1UxrovCHWJAR6XE2CIQSjNba6hG4LiwtgehvnDeJcO3y7WYkpy09R3fRJcRQtLOVDinCrVSRC8iUkqJEETJhVyuvW6w/dbNaXQw640VFpSFwkwVDMkLEmTCo6RuLXbu3hu5dzjiryzrUFFTpceU4tSlbrUtRJJOWdSdtY1mpPGTDLmGMf3zSIS6Gapw1rXbd4kLGYyfNIPKfG1TFcKdTUzZCNAbIImDnWCVeF/pBcUOE1b22DMT1NHTklL1F/eUTxO4cp3B2apEwSnN4ybXO4E+nhhziJiSgwnj+7kXdfl8VQYp6ykcz0j76zoFhfeaKlaAyqSRsLeatSvIwiFSRBUDtyEA78o91nFwh7ao4r4PaTmWVX7QAJG5PrCBvbl5IwRcozHkFeqnHG6qV7EN1V9StMKo3Gy3qCtSVmBMae2qzAYuXOlvtXWWGPZTlkz7iJVEa9J91pm4yXS9UouqrbpwsIddbzLXCAk5TmPX2fDW0dUNzrdWG0I7VU6JRJA10HMW82xl4jqnBem4I3iUbD/ADdFKG52QAmhbSt7LmlaSpXIa6QN+c+dly77odox6s4GHFE5g4pailCQNyI3205dOqvQ3HUttQ5UlKj3C21OWOijME/GzmufDi6hQWtpa0aEhLkfPbf62tzU9UG810LIRa6bzOHWnVhKn0vKMI7V3MAAdZSkCE/MxtaoX7RLB5uqowNiBqVtVbNdRlzIBJbU0rQHl3yfPpb0EpMOF9SEppwgJ2yDl4nfkLVj/aWYNSrhJhzEKlrcXd9+dgpaRo2l1hZiY6tJ26c7W+zeIEYpGL6G4/Cyu0LGPoHs57qkPox1f7u9JDh/UOr7tVejDEzH96C0Nv8ANb2WoriSWkNtAhAOYlSjAJ3EbTbxT4cOruXilgvEJSoC7r3oqlRAMFLT6Vmemk299Lrwo7VFMoXl5SJ0tr9uYJKmoh4QuSCPz/yuZwCvFFTva421v+E0KW4EKgoWrJoO5oTPLUaD32WrvwsXIbbbSE8zMk2kCnwlS0jIfvB5LLaBqVKAgW0nEFAyr1XCt1OXk+NO1AhtJ/zH8Lc9B2dcyzqx2QdN3H6Aaok+PSS3EAv58h90lXZgNcdpUIS02N1rMe+Bbt2+sK3C56ldFK5fdeNmqdOYA+JGgsqqwrfmISHMUXqtDMz6pTHIg/5jubL13XRc9xMdhd9G20kcwNTbp6PAxEA6JgjHzP1d9m7D7rEnxDP/AFnl56DQfc7n7JlvXBxDxn3b2vRFwXar/wCb0ozPKT4q2Hzsr4a4bYMwco1NBdqHKxft1T57R1XXvHbyECy9U3k2nTOJ8LI1Ze5JMKseWWgoTn/qSdXa+g2CCJamobw2+FvQaevM/dLNReLbSSlOg5AW8I/2m9ym5/S5xZUpSQ3eTdFeCARvnpmwT/xJV8Le1VZeiiCSvS3kn+1suZTfGTDWJUIHZ3jh9DKlcytl92f+VaLTwTGnVmJiNx3BspS0Jhp3PTX/AGW+Lk4a9Kr92lwJRiG4a6iAmASkIqBHX+5NvXqrvYTova3hB6IWJ3MJ+k1w4vhCgkOXyxQrI/kqJYJ+DlvbWprZTJWetsjt259PWxvbs5v7Fa3Z6AVMRvuCpDYcDrLbwiFpChbsn3WTsPVAqbmpXSZOTLPkY/CygRPS3RUcnFgY/qAgzNyPLTySDflNlqQ9ycHPwsnbQmbOC+WO1owoAyhU6dLIBkHTT3W6KlddiyahtnrBpqEgT1sIyrIsKUNiPv1+63Bk6A7cptitQcupIIHvFrgVN6UgsEQBbYPLadCbAoWlaEuCNRmAt2DPTzm1oFVl2DJ/W3QJjbQ2DzQJJ+dsMbREfK0wSUkNm1mBJ0tsKFgcwkazHhYRK4OvLnraQ3UShQfmOtita4GqmjcUDBcLStP5knf3gWHzToCd7Er5JTQKfjVhaHh/uqBPymzu2SG6PLAGsW6oD9tGYb/KwayFAKSeVuqEy/EjU9bPa6ZOHtgnTMNNLZbjtY0z/Xwtlo5U914CjDF51TzeGKVDS611RpQkudwrHdOp3E2lLgZgHEmBbtvS7sQNMtpfqUuMBt4OSQkydNv4dN7NCgvRq78XU9/PtvFCK1VQ4EEFWqiYGw/DW0nJ4tXD2edVBXBJ1MhBAGvRX1tZ4A1pzFBmzOGUBO/snDyVCvPrO0W0hAzQsaSJ2BNmsnilcqT9jQ16hoTojQ+Pe8LCOcVbneWEro64qjeGzz12V4/0td4zeqqmF/ROMIQE6iBB0A8NPLa2EAqKSJkgghQ1Ph+fhZqjiZcSgJu6vnLzCJI5D2vz2nlbj/Sdc0hSaCs7xgwhHPkdfL61tHjNPNNwXdE7MkEpAkgaCJ5bfXWz3uO8F1d1MBTiitr7NUSNdPwj52hb/SfcsEpu6uGwGUJKZ6b2WLl4zXNd6HBUXZeC0rWFApCNwddyLCme0jRWYGFh1UyNr7LveGmnlr9dLBqKe66VLOXnA8/wGnjaNFcecMoUpSLnvUhMn2UaaHX2uv10Bc4+YcUSpVyXnodT9mJG++bxtUuVcu1SgVlYKu6MiYSIAGk6eW3wslYsUHLldAaUkKLaRCZjvjU+8g+7pZhf6esOp7v7kvFUrAA7kHb/ABaR8fvsh4g9IXDNTQrpE3DeoUohRV9mUgAjcZtNh9A2hJctKnG5oIuka/n0N8aeHC0AJSq/aIZVT3pqmjqI93xPjb1gYTz8Zt450WPbvxxxmwC5QUdXTt0t/wBDKX8uZU1KehOkJj429jWMuUqtnSggC6sxkOJIXZA8bBuCUkT4jlFhZzDU8uti61GNRoREE2ePdQek9xJKjGvj+FgHEgjbflYw7lmBpvpG1gHCNSdevS1oKod1GnFvifcHCJVx4mxGwHqR6peowQ6hspUpoqBlZA2R5xNm9d/p4cHKJIDwS5IEA3ixz13BM9bS1eXD/BfEPs7sxvhW6r+oqUmpap7ypEVDSHQICwlYICsqlCY2JsJSejXwFa77fBnBDcq0/wCwabf3otaiY8i7TZAe9l7EKKnf2hHBptJKEUupgZr0QNJ30QbBp/aLcHogJu/XSDeit9dNGPC05U/o/cEWPY4QYKTGki4aX/4NbKlPwW4PsgdjwqwinyuKlH/RYhEt9XfhQvH0VdHP2j/CBoED90CP571dHkdKY2B//SQ8K1hRaRcRSBJJvZ+I6/8AmvutaFrhZwyb/uuHWGE9SLopxP8AyWOtcP8AAbOreCrhTH8t2sD/AKbMRKfi/CcGMclVBX7RnhslIXkw+kHc/vWo8Y19VjlZx8P/AE12eLl+jCPDzCt13xeop11RYavF5IDSN1Fa2EpGp2mTayzOD8JtK+zwxdKdI7tE0PuFkXEeEMOUuJMOX22qkudVK5VNKfZbaazBbJGUlSYiQNOoFo8OXcuUhKwaBqYycccd6lvtafgzd6klZSCb6b5Eg7gcxbpvGHH6deEN1thO83u2f+sTaQ7rvDBeH7vYuX+2NI8GQopdrL0bW8tSlFRKlFQkyr7rcv4vwBTuFD+MLjTG+e82Rr/x2nY295Qv/wCqZNLi7jyCFL4Z3WhM8qtC9f8A2wsrJxVxrfAIwPdSOpKkKAHT/wA4squ4/wCF7cJc4hYZQNfavdiYHjn8R8bFHuKHBptaVK4m4QTBMhV8Uxn/AJ/CwXRB27z+ERshHwoL+0fGwoQBhO6Ukci2g8v/ANatir341LaKv3DcbawJGdsR8qk2Bd4y8EEgpc4p4MSRv/2tTkj/AJvOxNfHLgCwpOfivg9J03vRgg/81h93b859Qp8U/KEkIxfxgqcT3lhdVHh9ipu6npqtwuAJQ42+XQnIe3JMFlc6CNPcBV0PF2tWS/V4XKCIID8Ry6mLN9rjv6PlBxlvm/HuI+GvU67Dd3MesNPBxtTjVTVnICkEZgHZj9LOR70p/RoZlw8T7k5SEtOqPyR9TYJpWOHiefVHFQ9uzB6JNGF+JACQqvwwCFSAKwiAP902AvLBXEqvIKL2ww0Jkk1qlSTz/uvL42UXfS79Ghru/wCkuhmD7FHUE6eTdi6vTF9GlOUJ4hNuFX/d3dVEdf8Au7B9mU97lx9UX2hPawaPRIqOH3FJDhJv7DQVGXuvk6dP7n8LKFFw84oBsoOJLhaB1MOFQA6asW096Z/o4NwP7Z1C9/YuqpP/AEWJu+mx6OiFjs8SXm4BAhN1Pa+RI8LI4bSu3J9UhX1LdgPRavXhpxCTUN0q8d3Il6pQtbaEoClLyiVR9lrAPum1frrvX0o3kmsu/gdip1twdol5D93ZXkkSFJzjY6EaDe063h6a3o3VgT67UXnWBsktpcucrCVbGMx00tN+EKmkvPCl13tQAopaykZqGUqTkKW1oCkSBtoQI5WYYZBfwE+qXf5j7w/CpWh/0vVLC08E8VMKCplVdc45eI5bWjT0hr69ISgwCpjibgW87humvq0NIfq667nQt5ILiW4pxnH92TOg7oHO3pe42lWuZSo130FqQftPb1FLhPBOH2n1j1mqraxeXcdmhtAO/wD61VpsoY43Ai6d1bI9pBXmpfleaipfo9VJBykEEco0iQdvOeY3OsPVqFt9lUIQS1o53u8Y7uvSZHWNTFi150jz7/qxfl9CU5QQSV6+0k7GdIjUiPKyVdC3Wb5WioQtCghasq1FMq3106Hn8RbSGipHVPNAh5KwpQJVBSQO0KVc/foROghR3GhxwsOMJNMlJSheckJBJVAjLGkbkxGknUmyQ68yKY5ohTeVKUxmiQZBJ5jLygag84VKdCHaft22kkvZVgxCpVqTqBMd1MAnaLEB1QyFdHgTiFN58Lborbrap6q8KakQy6pwCUqQMpkJPtd0HfprAssVd4XpUX3RLvOnYC3g8zlQgBJGRRA0HRJjrA15Wjf0XKxdZgN1ph5Lz1FeD9P2mYg5VK7RAkRMhwGfPraWb/rGbsoGUXheDC6x19DFKvLnzLPdEjQ5UiTPMa29ao5jNRRvJ5BUMoa8oq4xTUzLeSVvvqLhStSkjLmkb8tU8to6gCsnpY3O36/dWJkNKyPB6lXmlIJIC0ySY0g+Gm2lrTUIrXaVs1iaUuinCXXmimFLCf8AKmBuRoNOdoU9KO6BX4BcrUTnu91urQtAJIAVCh19lZ6T100p4vTiegkHlf01RGutILKlt6UjrKAtQCm80CE90adR7vL42mb0HMDPY39JPCVMFJDV1PKvh8qVEtsJKoAG8rKNLRXedCqppKgtKW6UDNoNIA0M89Ofh01tLvoD37T3H6UGFRVultNciqo0cgVuMLge8gfK3kktw0rVi94XXqxxKuxuvuimzoSQ3VJV3gVASkjbnrGm3zs1rnwglynQtCg3BBJVoOegSNTsLLPGPiLc/Dfh3e+PL9u+trbvuZLT77VIhJeKVOpQCkKUBoVgmSNAedq5UP7SXgLTJSo4Oxok5o7tJTHl/wDTjW3mfaHC6+qqeJSsJBC9EwTFKWlpOHM4B11ZGkwlffrlMzTqpRd0y6VpVnnllSNDz18rPShuFCkpabBCQZUY0nymLVPb/ah8BmVBDuEMbDkQKSmk+f2/SylS/tU/RrZCUqwjjxShqR6lSwP/AL+3MO7NYxKfFEbK3P2ggI8LwrfUVwNlSQlRjSTl3tDfp38Oq3EvozX8q76QvPXZUUlekIElKUOhK1dAAhaiT0BmwPDX9pF6K+KWKqorqnEFx+rIzobvG70y+RulPYrXB/zZQZ3tHPHz9qBwsvrAmI8D4Nw/frKr1u6oom6ldKypJzpKTmlzuggkTBInrbWwrs9NSyNlkvnB0AFz/gLAqcRkqTl+HmSV504YF13Nf91VF+NOLpaesCqhpqQSkqCjlUecAxuRmt9AuEMXf2uw3dN9YOut0UN50TFWzVVQy/ZuICkmJ1MEeFvnIvbFlPeLv+rsraV2gXJAiNY0HmfibeqPBz9rN6PODOEGDMJYnuTGH74uS4qK7a40l3MrZU8wyltakKLwJSSmRIB12t31TRyyObISR1sLn7dFgPkYW5Wi/wB/5dX/AP7Kt1boqL+rXK5Q1CFaNg+CRpZVQKGgaDdMyhtI0ASItRB79sX6MhScmH8en/8AZtP/APlFkiq/bDejg7OXDOPv/wB30v8A+UWpOZJS37pTku6kXPqULK6SwkeLdL6K/dVe6EJMKsjVd9Eg97a1DH/2u3o7uJ7uGMee+gpf/wAosnP/ALWn0fXdE4Yxz/8A4NL/APj7c/Ww41UknhOWhTxUrPeeFeuqveQTnsk1V7knRR91qNu/tXeASz//ACvjffnR03/4+xZz9qlwEUSRhbG2o50lN/8Aj7c/NgWNy/8Ahd+FsQzUDN5ArtVNepYMkxzt59ftWbjN4YfwPiBtAJpna+kUoH+YMrSP+RZjz6Wcq/2pvAY7YXxqf/sdNp/9/aEPSt9Nbg7x34fUuGrgw9iZmvpa9NU25WU7CEZOzWhQlLqjMqTy5G17s/geLUWJRVEsJDRufqFOuq6GWkfGyQXsqdYPvtzDmLbhxEyqHbqvGnq0naC26lY+63vO24qqbQtkKUHkhSMvMHa3gAXGoOUrCs8pECI8+u1vQ+6v2leC6PB9y3HWXfidytprspqas9VpmaZgvIaSlfeDxdcTmCjJWienK3U9ssFqMWEJp23Ivf72Wd2cxGKi4jZSBe269LsFFaLnNM8QHGXVJKc3eE96D0OuxsvH52qt6CXpE4W47XfjCmw7RVVG5cz9I+8y/TIa/vw4AUlK1Ff9zqVGflFqCTy57WVBTSUlOyGUWcBZFqpGTSufGbgoN9CXmltH+IEWajoKVqQT7Oh5WdiiYjptZuXs0WaxRSNDChHW27RHUhZlUNMyLFcmT8YtmbY6H3/hYMqG+Xa2T1M21AFmk3R2mUexSAT3ZR7gYsLPONTpFidOsHOkKJAVtzgpB++bGAox7VrLdQq50KFJO2sx0tgUDr99uArlI91thQnY2INEyECpgHXw526QSICTt99gSSYAMaW6SoDY677WkFBDBXXQddrc1CQ/TuMqAIWgp+ItyFDUdRtboq52kkuaR3taNlRkkoE+ca2M0J+3G8DlZPu+Ay5TwPsnVpA8Jn8bH6IgO6ybME3NOEAwNVjT+U2ywE/4494/O2WbVOvBLFj1RdVzVFVSOBL7CUqCikESVBJEDQTJ8JnY7POl4L8R32EKTjO6klSAsRSHTNy9n7vHaTZp4+addw1WNpRJWWyB/vp28fz8bXP4bcLsVcRlmlwpc79c5TMhx1LSUhKExuSSAJ1jXrFngYx9y9AqHvjtkVa0cEeJAUD/AGzurLvpSaAx5dOf52OM8B+IjmZs4yuwqA/9CO/w+ptYiuw7X3Jejl1XrTqZqadwtPIdTlUlQ3kdfMWn7g36OTeP7mXfTleKdpIyJzJzZiRsI5DT8bHfBCwZjsqzaiVxsvPRXAviUpIYRjCgccyapRQKJnTQaz8bBVfBLiRSns6nGFG0txEoCrtIPmJjQxHu+Fz+LnDF/h9fT13qqJLZA7RM7bxpr0+Nogryt1Z7RS3FqTlGYyoDkDrP372dtPG7UJd4kGhUCK4M4/UEhOPqGMuXS7xrvO/ntbocGcdOlLTnEOhSUKETdiSNNf5tt/qRaaydZKQJEaazpy+NgVCEgISZUNRMeOtp91jS7y++qjZPoy8THvs1cTbvPabkXYJI+Omn4eNu2vRj4gKzLd4p0kHci6ZEkbyVdJ1+dp/uCo9dulp7tSVtQ0oHckAR8RZWaaQlQCgCI70E6kae/e1F7A02WrGA5oKrafRZxmpUq4uU4MkkJuYaf8402+hZNvv0VMTUtGqre4rNumUpUn9yhJgq5faTufna0ak5kqDigNpBG8yNpsk4uJNxvvrKRlW1ISQZ+0SDHjpvYLxZpIKO1rSbWVJcH4aq8J8drjuCpvX15V34guuHUNBorCnG3CCkT1A3O3lb21p9WwSCR57728a6tzJ6TNG2lQM4hulRKdI1ZG46EgW9ladOVkJ13/G2fMSbFTjADiAuiBrCjO1i7o38NrGzH8p157WLPEkRPO0Y908mqT3x1PnYosD6HKx51IVqoSZsUcAzGSdTztdaqTt0g4jx+3w6p6e9nrpdrvWnxShDbiUFJKFLzEnl3PnZF/8AKhaCspwZUHKo6evJHOP5LJXHdSRhq7FBWSL0SJ56sPWhcrCFGUgjoTP1z+hbSpxdiz6hxDtFYJPpVZDCcBu6fzXkAZ9zdulelo4lHcwGD4KvOBtyhq1eC6ADCoyiD1n89futvPKiCZGx0kR8I6fCxiwKvxXBWEHpaXidRgGnTO03kr7+ysEr0tr7UTkwPd4CTBJvFZj/AO7tABUZKDBIMAADUe/3eNtFzLmKNdAJkb/X32YRhPxXFT9/5WeJCPssGXbmJ29ccP8A062jPjrxRxDx4uK6uHF43XQ3VRXheIW7UUzq1vJ7NpawAFaaqTB8LM5DqjCgkASIkCAI3tpt4C/bjWFjuVsQQebTg/H5+6zujBFikJHA3UeXn6N9yXS+lly/rwd7T2VBLYA5aSnoBYJv0fsMEBJve80iDoFNa6bexvaasbAB+mKz7aTlPUTIizadS6QlNMop111gTpYLqeIGwCK2olIvdR2jgBg1IPaXhei0lUd59AJJj/Dp/X3n6b0fcAuJUVPXooySo+tAAAeQ2+t7OpXrCNFE9yCYnnH6WU7sbqVqLocJZSTIBJ5dP0snQRgbJ2zyE7pns+j1w+ToBeUE6j1uOcRtrzEWFPo/cOwCVUlYRuQatR5eFn22+JHdAB0nbUTYwHhJSo5tYPXw/pYfCj6Iwlk6qKxwZwJS4zcpPUaxdIbiRU9gurWUl0VK2yrcH2SNJjawGI+F+AqGgNTRXIUlCtvWXTmHT2uWnjzs/wCpj+2NOAlQL2HqpGpOvZ1jB06aOfMWJYgbS/dlSkojKmdfDX3bfdZNiYWnRM6V9xqq+1F2YdocYpoauhSiiNKt3Kp1YCSJ55pnQWR10NNX095111uf6vQZ1JVmOZahEASdEgCyvxBCXLzu+7GVILl41CGSJM9mkyr7wIsnXShLFNjChIc+zUslO0pyHaNfD4+FgcME2Rw4gXuuL8TRMYQu69qenCXVuNhxRJMg7zr0+NlDF120jF001TdlKlBD7bi8p3RHekg7ADX39RZv19QHeEFI5lGdCWiJVlg9oNZ5acvEWGxxfjQwlS0/antH2mllKFDNlSBOnL9dLOWNASDyUvUNDSXpX0l5OUCWqBVShDKQTDqTzKehOvPa0gMekTxsurHtfguk4k3wxdF306WqSmYUhKWUpIQAFBIMACIn+GzPerW2rloVtRp2KgpIGUdwxrz57a/IWI0V309+8YMXFXaJdoH1BsNuCSlSlSTrBGgPu85rl7Y9URoLzYJ8Yy9IHjhQ1tyCk4uYkZZqKrsn0prSMwzJBG08ztraNePHEHFONa6hpMSYnvO+F3e0tTfrtWt/ss6xmCcxIAMD4DrZ/V+DLjvFFNU1dS9FKsKaIcgJjcQNP5flrFoa4x0dLcWJaxlpwllNGzUOBagpSlqJAA1E6GI6Zd7RiqGSPsCpuhfGLuCjW9WbmZYFZeTrocHeATEue1oJ13nXpBnqhXU+aquXUKDjbKGyEJMkJBVpvuAfu99hU3bX30+XnnA0ypQCUZ5AJPw2AJ8culuKBz1e/wBxpa0rIZCTlIOWNNCdPERtp0i1y6CAlStIQlTq3G0ZoIycuZ567mSTqCZ9rRQuFdL6g+yttE0jhCyCnKUqAKRJMGCFCdD4kwLJNQ/nRmU4MpOUzOkAAakwZjYfqQ7nr2qW9w0FJyvJLalZAOzcnTWJGo5awrrrZwUiLqyfow34xTLvi7XnlNKqa1DwUteYZVMp0JJ72bLI13OgAm0xYii8b1ZbpQrLRvNLSEoJBOcRkERAI3HkIM2gX0cK+hpb0v5h2mbfdaYZdZW23KQsqVMa92cqToNhB1Gtg2qijuG5U194IJqKlxBKFwIWVARMyImTOwSelvRsCcZ6JrCdAqsoyOzJTfevlbgRTILbTOeQFQVyUgqnfaN4jMN50j/jXR1tVgesoaynQl2qpXVEqCiJGx25k6CBAjbe0rUbl7UIFJeCkhltxZ7VB9mZEQBI/jnU6kbSRaD+OlVijGSqy67mqV0VPTJUkrRIUBGoEQBJJlWsDpbWrv06Z25JFrIAsXKoFI6s04qFZlFsqCgsghMAaEnlJnnv1Ih8ejXcldXekRgahuZKg+zf9NUJUknvMIV2ijOmyEqnaRaPaEKoXaijWQHWVqQogagpkE6eZ36aeE2ehzUXm16TmCn7vofW+0edZfIJORvsF5l+BSmTr0iZt49OMoK1Y9SAvUXiNw6Z4q4Fvvh69WmiRf8ASGj9ZDQd7ElQIWEEgKggGCeloJw3+yDuh8pdf4qVrrR1KlXO2lIEzElw/KbWww/VmjvelebTT5kOCO3MNEzAzdBrvy3i0tLwtf8AiBXaYsxKlFJ/6Dd0ttqH+JZ7yvkLcfiT6lsrRC426CwH1JO32W1E9jW+Kw8zcn7BULxx+zN4F0uGV4duHHF/Xli3OlLRoqZlxAE94KQfZ0nXtBqduVkPCX7GFm8UCsxbxdrLvS5B9XYuxtbgA2lRXA05Cbell20WHsOUwpboomKdCf5EiT4k87Y9fiQD3xapHX9yJdUTX/8AVu3qdf2UZJDMMsTPud/QaD8qgzv7GvhmyiaTjVilpyAMyqGnUNOUaWbOI/2OWDKS6KyspuOF+u1DFO4tptV1MhK1JSSkGF9dPfb0Nqb8PJXlZHr73C21IUuQQQdbZdX2pyG8Rsiw0U0lg7ZfOJiPD7mH3+xW4Xc5WknLlhQ/rPvsgkk762lHjldT12Y2vu63GC3+7r2raWTpOVwj/pn3j3RcbeiUcpmha87kLLqWCOQtCwEjrrbWpMzZwYXwBjbG63GsGYUva/nWBmeau2jcqXGkyAFLS2CUgkwCRBIMbWc7Po38f6gEs8E8dKCQVKIw/VQANyTk0HjYrp42GznAH6hDbC94uGkhRvHj8rZH1FnyngbxmVd5vUcKMX+pAhPrH7kqezzHQDNkixu7PR44730Cq6eDuMqpI1zN3JUER55LMamFupePUKXdpT8B9Co7n6m2ptKn/kq+kh//AGOxt/8AuV8f9NtK9Fj0jU78EMaj/wDYz/8A8Nm77Tf7jfUJu6VB+A+hUWTbJNpTT6LPpGKH/wDRTGI0nW6Xh96bIONOC/FLh1SUtdjvA173DT1y1t0zldTKaDq0pzFKZ3MWdlXTvdla8E/UJnUs7G5nMIH0KZxKOzSQCVGQokaDyta70W/Qww16QnDeoxxX8Qa66KilvV67V0jNCh1PcaaWFZisHUO7RytU4CTA10tfn9mji1LeHcb4UdcSOwq6O8Gkk6nOhaFn/wC7b+Ns7Hpp6aidLTmzhb91pYFDDU1jYpxcG6tD6Efow3T6N+NcQuXRjisvlrEF3NtOMP0iWkpW05mSoFKjJhaxEc7XEUAB0tAvDe+P/l3d7ZX3XS40T5tq08dQLT0sgm3N4ZVzVkZknN3XXQYlSRUcoZCLNsuTtp8bJd/U6i2h5P8ACcp8rKZE6T8bB1iQ9SLQRrEi21TuyuBWRKMzSE0gQRr8OQt0NQAk67W6W3lXlI3+G+otwQRzJ87bQN1jrthSU1CtNFtg/An/AOKxoEakjfexFJipaXOqgpGviJ/6bGgokyNxroLWI9UF267WsBClRMAxA18rETfjYb7Ruiqz3gkJyd6T4T4fUi3dQ3WLUg0tSlCdMySkEqPmdtJ1twwxegMvXmFgH+FoCR7/AH2sC3NQWv36Nhd1aZ00b8/H4+Nh03oolQRRvrCUhSTEBYMbTz12MWBTS3j2SgbyPakkpUGwQE6QI5nTfxtgpbxJSTeikpgZgECVEAbdOvTXytMZVDUJQpKpVQz2qqdbJkpyrGog7/jYxm55vOyQijvI6qvlRBER2YAmInrM62U0kaAnUDXxNmPknC4YHZ1dSkyO0SlwT7wfuFlCintZI21M2TVKCK9gn/aJWjptB/OynQwHdQNdpsyY7pYJcn+P/itlu8q/5j8D+Vsskrrwix0Ut3O64AmVvN5UgR3ivn035G12+DvGzFfCRblbhlVKU1bKG6hmoaztuAbEwQQRPI8/G1Hcfqi7MpTJNS2kykbZtvLzB5xvBtvgekbrr8u26n31N01ZUM0dQsESEKWATBHjt1Fo09g1xdsq1W7IA7pdODEWNr1xhieuxNfjra66ue7V7KnKgdIA5AD5a2lrhP6Rl84Au9y6aZNPUU7hBh0TCo3GotCOO2XbrxZet0VFUKhd1Vb1AhQBnIy4UjyGnzskCsW0vVR7u+5nra+A2WMdFRa/4gpE4k8R67Hl5PXneVSlx15ROZOgHwtHVSorzKKwrQ5v4swnX46726FQt0FOYBRkmDqBtH3/ADtyumdySUgxA8+WotJoDdAkdURX9mVBa0lJ1JmZ1+fPzsC4gJC+8mSDmnw5mfvsO8ypkKGcApOgkgzGu+362Kr7EgGPaVlgiYAPTfaetpqVku4OrS1eDlE4MyKgFQMmM41HxmPfZ8inKgtbYPdCiIEgaSPPW0UtPLYqEvtuFK0rCx5gjr5fK0rXZUoraFmub9h1tKiPZAJ3Hxtm1rcpzBalC7MC1a7BRILghMQCDHlHwsj4kZUbmqe0AUlKm1GdiM4PPx+FnI6krSY7oTJ1O2ug+unKyLiUpbuKtKkKOVvMAP8AMJ+Gvvtnvf4CtNjRmCp3eXZf+UpRuoSQRiC7EjSRopnaNDrPXTpb2UZ9kSdZt41XshbHpJ0a3m0JWu/LqJykAkHsTOn/ABafdNvZlhsqG8wdPC1N+wKdos931WKA3IA8bF3AJOXTrpY4ptIAJEE9dJsXdEbAd6zxqMlkQfT/ABGxNY1knfbWyg8kEa8uosVdGkHny+vfa4xUnbqLePCJwrRKRui8myD/APVOjl52gtSVESpwiecTawPGulNRhBsSO7XNK1H+FY5+doL9VOQ5IMGNOk/QtpU7hlWfUC7kSKHE+06SQZIHPQ6WyEhJK1bEqBj5WNCiKl5QrlIMbDmQfjp/Wwhu45gsZYVHTQ/Gxy8KuGEpOWnUHMesgb23lURmUo68pkedj3qLijlCisAGdJOnh7xZSGCsUuaowxehCgYUKNzaPLptZuIBumLcoSAAAoEOCR46HUaRHXS28sXpdBKlaXg0NY/ilMfOzhbwXiYmFYfvBASI7zKkjbTePGwNXh2su+puk19C5Tl28KbIF6EgOoCo672eQPa3MRomicx7soOqU8ZsNvOUzmZRAQuIMDkdf6WQH6SiYbJqHy2FzMqglQ5fX52kLHFFSU6KPMNcq9zue7zmdvrWzDqqKjrlBpzVKDMAxrEWBmzOurIZlbZI+e7O1ITUI7pglKzGpPTcfjZZux66lMFtypl9xXdbBIBP1Fk9Vx0CVplWm8lXQWV7pui6FINVCs7ZzIlU9I+vO0pDcJmA3Q/7uaUkuEKB5iSLdpu2kV3SkyQCRmyjx0sYbcaQgAqA0mI57zba3QFE5yEgHQnW1Y5lYBaEitXbTPcRcLUj8hiqprxpV5SQcpcpVEAnnAJtYo+jVw7rad6neqb5hYKFZ3kAHkdkTavS6lDGOcIVadOzq69uVA86dK//AHYteqnKF5VpU7KpC+6DPlP4WdhNyEQNadVX4ehbwGr69ivqrvvSrrKdyWXFVzqQ31PdTvFq6q4G4Iub01bw4Su3atzDF80dNUdh6wvOsKp3CZXIVPaI5Hp1t6FpLKKxFO264CpQMEgc9rU44vu09z/tEcEValhBvO5aJkAfxK7V9ESOcdfw0T9BdStqpFV6Gno5U92i42MDldCjRTLl41K0HWYADg5iy/Reh36NtRTobf4X3W8lXdX27j7q4GwSSuQOqZg6dLSUimSlbgNP2ozGSpBUd+tlKjWhitQ00ltDaVbjKB99nyhTK8oeJ1z3VcWLsSYeudopaui86mjaZjRLbT6kITEyISNz1+LYu6vbuPi/iu8jop+haqCDPeKmWzv/ALwPPU2lD0iOxTx/xndAZCnBej7hWhspzFZC9B5L3/O0NY8qv3VxJdfhpLddc1ACc2i4ZbAB96OfXxtgTvzZmq3C0MIcny/idx9r1dNMUzBUEg+O4O+p+pNq/cfMRF/ESqenq/tlsMioCTp3QrKPCM34crTSw6EpyoUSkkSY358vM8udq/Y7rbiuvEF4VlawuovJ+ocWULSFKPeMKHJIMCNJgDTe0MNitIXKxWyXYAmrQ3UpTX7wvmsU3StpDmVLkK1nQRt9QDNidxEVt6POMjszULCUJ1UQkqGnWdtfzsSve+7yvJXZ1CuzanMGhonpPjt+UWPYWYZX21TVEpYbGVRyk77bAzoDobbw1KyksVyUOiG15iqVKAVJEqVB3BIjXWJnxEAUNMyh1Km1r7SJXASoBO55kcyI59NbdVVQqtdW3TH7NRhKQgqBSNJA3946z5K1z3a52SHnyvtFpzNk5VH+Hr1kE6z5SDaQTXspU9G5+maxnedOtHZmooGnWyXEqSpSXCConY6q0HtR1nSxNUmkvq9WaN9C/VKZKgooBObQTsfPQ6E8zAip3CtxN3Yu7OnXToNTSko7QCCCpBBTH8UAxA6nu2tfgHLWAO1YDdMwTnzQSogGQJOqTJEa7jfe3oPZQ54cnmqdUbJ//vKlvC7qZ6vQhvM22pwkTBiCoHLI5eMAdNI1xBfdK8XqOnpEvOOLWArKYHidNDHUTIsttXbfl7FdTTlTFDXIb7FlDZSppABMgiFEkRoTpJ8ij46qbg4Y4bcvK8B2lW4nsaWnBSVrXlA7oMRoB3jEAa7ablTNI65Is0c1ARty76qkXEe628McQ7zoG3dA8KhfeMIUtIWU+4k/LeLWo/Z3cP8ACuLOId441r/WfX8IUjTt3ttuFDa1v9ohTi41VlSmAj2RnJM2pvi696+98U3hX3opHrD7pK8iRlkcgee0Ttpppa2H7NTGDN3cY7ywvU1CUpvu43ENIOmZ5haXAB45S6fcfK3kdeQ57y3a5WlT6EXXpWtJgoCimRGvI/XXp4Wk67cXisuqjqUOn7ZhDmvikG0aEZACJBkEHx008YsTwzfS0Xf6kt3WmqamnA27qHlpT8ki3l/bR0kMEckZ52XW4HStrHuY7lqpaN+5v9p75sWdvmBAXy62Z7F5LIkqgdbdprHnJKQVC3mMlZKRuumGFsZunE5eqlbqsUerVLzEqs3b+xJcOFKMV+K78orppyCQupeCCqOSU+0o+ABtENf6XOA6y+hhfh9QvX3eTiHC29VTTU2ZCFKjYrJISYBCZsSCkq6zVjTbqmLIotBuvO/0xbo/dHF3G9IJHZ349VJGXbtjnGu+zgtWVY70Hfr1tYr0l8Y3lj7G154xvqjpad+/GmansadGVIypCAIMqBhB5mTm8LV4fTleWidlHc297wMPbRMa/cALgcUt3hxCsl6CnGK9uEfEe+nLsulq8F3zcy6YIWCcqkOoWFRmE6BQ1nfa3qbw8xXjjiVgq578rXbvpGq9txLqC0HFMOtOFCoTAQc2ihKSRmiTpbxm9HV15PFy5KZh7s11Rep80SO80vlz1j329XPR4r7wuS7X7lqL0dqKRCzUNIWkZW1rAC8kCQDkTpMaeJtzHa+kbm47R4tLroOzt5Yw0C9id1NzOELnbfTW3s4/elSmMrlY52gR/lT7KfcLKq6plhIbYQlCQIASIFkdy9JTmK9vlYhUXmn2s5PlztwJc9xXVMpC7V6Vn7wzSUka87Enq/2hm/h6WQai+EJIBJ73dPn9ffYkq+0FwIBVOaDyOtpNieVbZAxqWHq9UnUkjpao37Raj/e3B65rwSlS1XdiGnKo0hDjTyDt4lNrF1N9CNUE+AE/G0Fel4j9/cBcTsgEuUSaeuRB0T2L7az5HLmtt4KHQVsT/MflVMWjbJRSNHReVbYAeSlQnvAGbWa9AG/1XZxbvS6VOEIvS5XUZR/EtDrah8s9q13m32N4VDfLtFERGxMjbTaLSn6Kt9m4+PeG6kKypqHX6YiND2jK0pHxKbep4lHxqORvUFeX4ZIYayN3mvU7DV6+oYruisU5AbrmCoyNElwA7+B/C1tVSBsR77UOcv1xpaVoPfTCkqTvprP0Putei6a5N7XPR17eqKumafTpyUkKHyNuLwhpYHNK7PGCHFrghdTO8W5UNNvObCEd3aOlgtSQdd9bbrVhOTerGexfcbKdAZFi0QSN4iLKl+tlLiHAJBTFkjOIiSZM22YTmYCsiZuVxWLARkUDP2iADPUx9xsaUDvrMWJvLzU7gTuEkiORAkfOw5UFd4bK1Enra0xVXLuNASB8LYJ3g624JCjIMQOm1sJOsWM0qKEBAOs+E23MjYx42DmCbZKgO8SOh6WkN0yHCjPSLdJOkA2AnQk26BA269LTCig6x0t9g+f9m8nNtoDofvstUSSHQqeetkWvldI6ge1lMeBFlW7XA8GnU7LAV5yLNzTFL47KBK1T/mtlg5XyHyNss6ivCPFLK3vUaZMKDl40yQo7jM4np5n7rWkpHWbrvWmeYqFuCkfQ7mCQmQkyCPhasV8oS5e9yNgI+1vqkB5SC6OWx5b9bWQCmO0zdoolMqmJkD7/AK5WalAc1wKDWNzANKdPEi9rtxBj2/7+w+4t2hvCsXVNKKCk98Z1Ag6yFFfwJs28xQjNBCZmD5fnYamepmVDMlBSVDvBOpj6H32AcUA+oMCRIKDoIT019/x+NyMcNoYOSoNjytsE48OUVG/XNCrcSlskZsx0I+/lt+trD8e+GXCfCGBbnvDCV8Nv3lUhJ7FL6Vl1BQSXYHswQB5ExsbVYZrw2orQrKoEd7WRtE8/CxuovesqEIQpwkbAE+/rpP4fFnMLnA32RGkAEEIjWFJWpCI906ifAfU2JLSqZIMHY7DQ/rYy48V5SlKes9Tty+uVgXYUpRgERsRqOUfW/usZMizhXn7qo5TI08fGz+wFePbUDtA6onsHMyEgwQFdNev32YqilQOaITsAr6+jZWwreKKC+W1OrCWXQWXCZ0Ctj7iB8LAqWcRhVmlfkkF1JqkrcGYwAdIB0kfW9kLFb7X7gr1OtlUMrJCUnSNP1s6G2iV95EmdQNgQfu+/TrYK9sM3Zet016qu6aisKWVuOZKtbAUjLqkwRp4jW3PyyBjTddLHCXOFlSPFLbbPpH0IzAkXvdJVuI1YOx+7wNvZil1bBIHnO9vIDiu7d49KKjF03O3dTIvO6AKZLzrwBlrMSpxRUVQPLUC3sDSp+yA89d7AzZmNPkhEFsjh5rtSYOUn5WLPIMd2QT1NjkaRqYsA8kkSYE87Ti3Q5EmPNFIzAeJsSVnBnUjf7rKTqVEEQdPr8rFFt9NLXWbKk7dR5xjH/wAjXiEgqTUNKAJ09qNPjaCVuKAJBBUI5ESRayWPcPV+Jbgdue7snbvLQW86so7qgTry0Fo5a4E44en7S7kpOgBfUYHlHnY8MgAsUJ1PJJq0XUasOvjtIy90DulBn2t7Yc6gFrg5f8IP1+VpSY9HzGc96tuxJI/71Z2/3bGEejpi05gu97tQCRMKcM/8utj8RvVN3Sb5VEzailSVJUBE94JEabfA2sXT0eK7wxQmloKaaJdIX4FIsoIyjvF+SkOZz7EezrZpN+jbf5JLt/0KZ1gNKMfd4WsZdzS6G76KkcKlqapm2yUqIEpSATYT421Dw6+yHJSvDbSCyhG88L40D5aXTVCW5knsZkEnQGCD8edmHxSpKukVh128KJ1hz1/RLqVAkB6n6jbU2ts25n1S0sjrnItG/HjBtDiSiuy8Kt6oaVdSXX2wgiCoKbVCpnmi2xUYiZqbu5b91m0eCiCp4zXX8lCPFNKWW6RYTnjPIgkgkTHytGaqvUpSiNRO43+ulrJYz4fXZfCKcVdTUoCe8cigJ0Om3ibNNHA7CHcdcvipaKpUEu1KUkbiYyzyNs2PIxt5CB9VozRPbqoZcUlSQ4CZGuuomRA+duEVTqUBJMAazGv1r99poqeDWGkLZWzXqcZz/bldQoymI0hO+1jl18FcM3nUep0dK6+s94Ht3ECJ11IAInz1NtBlI6RnEHu9VTEjS7IHC6gw1D/eUVJJnURofDW2hWPgZHFjLrBJiB5/C1jD6NtGqR+6WkAwBmr1H3xNhmvRwu1CMpuakWZ1mtWY133tAQs2LgjcF/VVmq6gi9sLOqUJavladejlDUjUcvZte+gqr0fZS7TM0wBQgpK1K1BSDMR4/K1YOIeCLjw3f+E6amoWmSjE9M2+lKyQZbebO/8Anj32ny7ajiDUJo13bQ3KqgU0gJU++4FhvL7WVOhOg0tQkZlkIBV1sZDBdK9Q9i92qC0+opQDKSS6dJ8B52qH6WNS7cXpl8EL7co0B+sRQUrrgkpyprlZo/49Z8OtrW1h4mIW23SLw602lJWo9g+dJ10I1PhOtof464WTfeLsIYnxvdty195Xa7/2LWMsvNmnUHUKJKFHecp1nnYEjAxhJd6osERc6w1U8VDvZVL7eUQVbEAzz5+djV2sMuTVVORKAuAcolR9ws0X7g4pOFC0YvuRKzq4Dd4USdNRKhpY5S0PEhthVLW44u5aVtlttLF1IC23FbLEOQYO4Omp02tF1Q5p0YT6f5RTCLe8FQf0uaNNL6TGJ6pFC6lD6adztFJAzBVOiQJ5Ei0C8YqepN8XDUUbai0bmZc7yohSHHUTm6kCBH9PQDjbw/YViCorsRUNFet6di0V1y6BtLiwG+6CNdojfpau67ipK647tcfu6kqE9g4wv1hsFZyvOGAVCYlWn6254y3neD1Wm2jORpB3ChO+7wqLqw3WXnRFPbtMFTMapCyAAfGJnTf5Wr7UC/l1r15XvW0pUtwrdccZQtZHMlRg+HI8tNBa7XGPDLTfCu8ks0NPTKFOw8XmmwVNthaFKOgkd2R4a9Dan99XBU1VQxSuU7nq7RzEBv8AvFbgRrp0G2scxOthzW5SRuqVe0scAUxqhVBeFS67TXYopAzFZWQCBuY9x08LH7gaQ5QrQpglSASQATBBiSD4yfjZRvlygu8pYapQl52EIbgIKiecEd3UDUkHWOWiHh6sSHgl2YKlSE6kSQTruToPhz2toWsVn8k46ah9Zd7R4BDaJAR3gNNdQdJ18PjFllYCiVaZQkKG6ZBTAPUgQRoTPhsUxiscK0DtUtkgFYCoVqRBkba7CBMKEnax5ASUoZMpaIGQa5SeWWBvCTtoQANgLEFlE3Tj4bXDdN74urn78qkMMMMNraUl/IrOVqBHMnRMayII91orgxhhGjSKKmPrbBCQvMSG1I1J1JkySN94IE2pLf8AeS7vvOhCbwq6dLzayXKdIcdCTAgBZGhKZ3j2tySbG2sSVq1oScT4gdDeV5X2jTGTvR/ATpr/ADJjXXXTdw3tAcKi4bGXPVDdTCY3cVf268dUCqJ1ugYbW+whxLVOkjtFd4htvr4DfztCmOMJXjeDzmLeKOMriw+04fsmaoOuFtG+QNgCVQrwPnpEc3Vf1PiKixJdt3XTd1319Fh929LrrKWtvNb6qhp1swntap5pRyKXqUJ1GhBAmE7zosUXrXrqb9avCsL4LKH3lFX2hQVIhexkiY5ifO0n9shW3Y+Pbz0Vp+H8FoLTundxMw7w9eZvC9sMX9eVbXUykEKcok01O4galSUFSlxEwSBOmmtkfgXxE/0XcV8MY3KVFu6rzYdfWCqfVpKHwE7HM2tW4nQeMtKvw5iK52FGpYcbQYzpQrMOe5GnXn18bJLSwk6xbBqp21bs4aB9FBoLF79NOs1KE1VK8HG30hxpaVEhaSNCPAg/pZAuy7qoXleqgclO1XKV2ijlQApCHDJOg1WdzaKPQk4kq4k+jth52rqA9XYezXBWKzEmWMvZE+bC2Z6kG0t1uBncS3sqprL6qGqBBT/qzaiO+EiTJ2+HO3n/AGsp2y0dnm1iux7NzGOpJbzCGvDHWEMPhLT9W5Xvo0DNImRMfzK0jxE2j7ilxJ4pHB1ZfeE7qcw9RUICnHw3mcWhSgkd9WoEqB7oB0tMFxYNwnh8A0V1sl0a9o4nOqfM7W3i3933pctXdNdStP01S0W3GVpCkqBG0Hxt5zEKencMjL/Vde4OmNivOq+nr7xa469fN6Vt51rpzFbq1KJ18dfjPjbrDGCr2ui86e+0kNGiUFgCRCVaHnyBtL153BRXPeT1PTUiGktuFIKURsdPh0/SyVXU6hdlU0CEq9XcAEazlJ0/rbtoA3J4RosmRmV2qq56TOC/7H3tdtI26p5NZdhdbKuudyR7ifnatlXlNQsp2nTvZvna5/paU6Lxw5ga/SkrU7SusuKE8uzIBPLVRtTSubKXSIjSD5jT8Lddgz81OLrkMZYGVBsnNwgvL908UsKV+bKG73pUqIEylTgSoR5E29acA1HqSllIIKgM0bbj9fhbxvu2tdoLxpaxpZSumeQ6gjkUqBHzt694Yr23WmH6cqyuN5wJ3BAItm9pY87W+a2Oy0li8dLKWk34exCo0MJiYsBUP1q2PWgw6pkAEqIMfHazOq6q+XrtfcuHsXK5KCtpt0ZkLIkhJE8wPKfhZHvG+8cUF4uKpahqqo/9vV1BZaUhsLSp7sk54WjsfZJH+1BPsxbhe6i9gu54lgnfVXkEyQRIEExuPfYj+9MiwpB1Cs0A8/H9bRD/AGl4h1VO4jtmmiw6puoqHHqYN07YeMqcIkocCMkgiIUo5QQAUh7FPENbfb017XZ2IabU4XKilKQpWUqCCFbDK4BOsLmTGlttJYIJnCma8azK+4BlgOKA8ACev1rZj8UaYX7w+xPci0hXrt0VTKQT/GplUfMj6i2sP3lfVTRLqL3eLy1vqNO8S2S4wQkpV9kcn8REjkATFh36kFDjTpSUqBzK3O3nqLGhZw3gjkUCd/EjLeoXk5e6Yqg5/wB62hcxEmNfPWdbLnDm9E4d4h4Xvxz+7pbypKhU6d1LozfcbEsXXau6b0eu10ELon36VUjctuqFkrO4E07wJGQlIM7EGf8Aqt6hYSR26heUaxzX6FeodTVBCnElSd9YInrqOXONbXi4L3qb34W4arcwWRQJYJmZ7Ils68/Zt543HiD99Yeu2+P4a2iYqTrM50BXLnJ+Pwtd30Ub6/evChmmKyTdt41NL7iQ6P8A8Lbh6QFkxaV29a7iQtcpiW4CdvjYIFIVoPjbpYUFEczYMq33HytsNWM5Fb3bD1MrmUCRFm2oiDuZnQ2dSxmSUnYgzZsPo7J9SI1m2lSvuMqz6pvxLgZTuBE7T7rapyoUzQVqoJynxI0M/C2gSnROkfX4Wxmci0ad1Zj394ffbQYs526GS57/AB52zMVAmdOnO3BBSMusb72TcU3hX3Xhu9rzupppyspKF9+mS6CUKdQhSkpUAQYJA5ix2tubBQJtqlfVXI6HnrbCZ1B28bUmwN6aPE3FeMrhw5XUOG6SmvW8qWjfebo3iptDrqUKUnM7EjNz6a6WDxn6ZnFPDuJ72uW66W43aOlq3WqV6pul9h15lKyErUguSkkDY9DttbSGFz5suipe0IcuZXeQogga/dbvMTqCNrRd6PXE29OK3DWlxXfTdI3eHrVRS1ApkFLYUhfdgEmO4pM6nWfK0mBeuadrVJGGJxY7cKyx4kaHN2KGIBBBOn4WM4bJVTNJIMtEtxG2Ux+FiYUNhPMGbD4c7tXVsSDldDg/3kj8QbD5pynWA3HtK9yv1tlszDnr4xbLOorwsvFXaYiw4JVlcvqkBT4Z59/9Da1eH+GeNMUYavbF9z4eU9c1xNlyvrFvtNtsjLmiVqGdUa5UAq1SI1AtVQFbuMMMJkEfvmnJzbEAq2/S1hRfF4GiN2+vVKaFbgeXTJeIZWsCErKNirlJE+O9npAcpshVRFxdGG6tpKx2uciYGVUEH6ixiorbsLY7FnQiZWSZMb/H8bJBUDrmUYMHkYOlhmUodKc1QEQogSnSIGvQ/pa6QqjTyQodbOZKlDQSlMcz9ffbSVlRMwDEJ6+7x+utjNRQ3Y3SreYvVLjgUkBIGqgQTIPh+J8rECiMoBCu0GwIJ30sg66T25U9qDhfjG98BVXEq7Ll9ZuKgdW1W1DLzalNQBOdrNnSnvA5ogAnYWZjry1AlZlJOxMg8pH1ysZo6pxumVSevPsUz5HbJbXKVBJkZkgwqInXaycpyFrzIBEEnWJ059LM24vdLQrvMZKiSDMmDvHn9b2CQ4CvNlBKZkjcnkfrxti1Zs06lRGXu6H8Im2kZcojvCD9T0tI6pWI1U7YWdF63LS3oO92rSUrlMnMNFfnqbLdddtSm5quW1rIYUCk7ER9eVmTwOvBNQqtuN4pzpAqWcx1UNlgf8p+Np2XcD711vuCQosKUJgBRAPTf9LcPis/dXuYV3+Exd5ia8Lzc4xsPp9J2k9Z7y03hdRKYnL/AHR1+Xu8pt7CUgHZJHKLeS3pH0iaf0r2xCoVW3OpRUQYJDUxA1Ezp4n3+tdKMzYG+nysSF+eCNw6LOqWZKiQeaEUCRziJ05WLupBkEbWNGCIIsXdAPKPfazGqkiIPJ3PP52LLBEEifPSx14abb8osUcQCSP62uA2Cpu3WqFkPXiw2oCFKj5Wcn7qSowgRz3slYdYadvqmS8FLBKoiAZymLPpFIxoMhKyYiLU5qjgvstWitw9U3EXTMCdRrvYYXdklWcCBFnA9dyWnMpSZiSI2tyaFKjog2Ea031VwFh1Cb4u9PenIqeZBsa7JP2aQFdxMaRt7zZV9RVICU2i3HDmK6fGdBddy3BUVdHVOhFRUpLiUsiEzqCEpjU67xG9iQV0mfLG3MT9lQxBrOHmUiIQkKBUpKI11KPwmzc4lJaqLgcCVJWpLLswoGO74WjnE+FMcLSpV0IrHFKgR2io35DMTtY1hK5MS0VNe7GIbqqGW/UwGnXUnvq70jUkE7W6uaiAoDVF4Bt7vNcxRYm+SubTGIgX35Jy32xmpaZZbnOAdvCwD+FL4vegut+6KV5Y7TKtbamk5AVnvqLg7yUxqkanNZdqGE1F20ZWAqUIOomO5ZzYZShOHWApWgcUkHwm2A3JVgRuO2q36uLLHqmHiHhxiFxeeidWoBKgA27oOYOWQfvtrBWGcTXDe5Vel2OIpFMhIqCtJ7+cQDBm0qJS0AVhC1eMH8LcXqibtT2QKSl1tUbfxCd7dGzEnx03dgBlXNtwqJtQJwTdNdLyytQXUBWXUd//APOtwy8DW5jVNhtAkjONPnYk/RVJrHUU93Z0CcpzkSN9Zt0xRrbqCp27UJAjve1+lqBcStUBQJxre7XFLIU2kCkxRdbydBpmqW0b/wD1nztO+GHH03Ddz6GkkCmSglRG4/py6WhDjDTmpxZeVIgKzt1t01MRoIraUz4/xWmvC6S7cdHTrCoZcdSvWIKHFJiPMfK2XT1YlqXRX1H9lp1VPkp2v+iVKt91xoF8tMgHKftNz8LRP6QLbBurDFc2FOdjeRBcSdEkgaT4wfhaXLxS0WW0qfcSdVQk/kLRpx1b7DAFNVFC3BTXmyvMqZEyB7rPi1+6PI5IeG/6hqkCrQczb6l/3qQQYnkN9LdsKQYCnFKVoBCDH3WApq0qoKN9DpPa0zSwnswqJSNjY41WVK4IQqOoSRra1GbsBVaQWcQo64y3RmvVpxPsv0yEqCzOska/PS1asH4fFdd6qAuNU4ZvKuaUsyUpAcJkgRrCveSbXJxvdibzTSvrpVnIjLtJkEHW0A4PwA5TX7iK565ioOS9XahpaUz3FpQobGNYjXnbzzEq4U9fM08l2VDGJaeJM1zCtXelz1VCqkYdaVRuBTTmVKXUZCChQXpqNIPWLecDSnqYrWoO9jT/ANy0lKlCOUQPADx13Jt6/wBVgKnWFNrYqlkpnRAmPEzuJt5U4twjWYZ4g35gdmlqnHqC9ai722A0pTi1B0pQEpA1JkQAJMjfSdns3iUdW57QdRZZ+P0xjDHBQ/iq76h8evu9x5HflAgpGndOs6dfPobN26socCSlI27xBEe8edrFYp9F3j7U3Sa1zgziwobaLsLup1Dkc4bKc51MwBPnFq6M07rFQ7T1QdYdaWULbWjIpKhoQRyMyCI/TqQ9rj4SuaLHN94WT4u9aUNJ7sh0FYSVZpSrQAjcmDoSNvdB1Va1TsIeqVko0Q7vGpKYgQdgDzOh9yJd9W6+wGkLLpXlAQO8YgbRvOnjqNZ1srU+R2mV2ri9ICpSUqjmJIHh4HveBBDtdDG64p7sujE9Yy/U09RVpbbDQyrCCoiSZ6SVSdASZM7gvuk4dYFBaqV3cEIBSvItySUyDrpsdevSRANm1hGlpW1ApaDebMstApgJEHUGe7toeg5mLSM1RKr6dxLVQ4ytSCEpSJlcER3dCJJ1/ITUe7VW42i109KzE2JmmBclTibEVRRJRl9UevqrdpMsD/YrcKCNAfZjwEgFrXlU1tSlVMzTodfXKVlxCSkEADSdxPvAMgawVpVCxUZnF0q3FK75Qaggie9B1mfZBET3T71+5bhYu/8A199hDLmUFAbKiEpTvqdZjXcaEbmZotj8eiuF4DdVD2O8Fk0tPdD9a7U11e+X3VqRo00kZin3lSZB5DTYi0PY7wevDlSHUCW1mCMuUDoRrqCQrXwtYzEGIk/vOqbuWlF43s6kNgAgopmwdSSJynfnpGuoi0eYk4XX256xeN7XoXHX0BbrSUjKnSDB9+mm0baC16N1tFTkZfVS3+zJ4mKuLiVfvDKsf/1XFFB61SIUY/1ymlUD/Myp2T/gTb0raqjTKypJAIzDSBPP8LeJXAPFiuHHHjB2JV1Abbu2/adFQudOwU4G3dehbUu3tXXNPdokMtrXlJCoExEW57tRFnonn6futrs3IG1jQfP9koKvEx3TJJnfn1s3sQ3oQ0pKjoNQZ36/jYRp0rdNNIzA+yDKp8vzs3sZpfoshcQtPaoIlSdd/wBRbzKCIOcAvSiWsGii3FzQF5uupJBc75HIA8/hZoOvMZ3GVuIyuJI1Me77/efGzb9K67r6vjC12uXGxW1LzVaha26VtayUlpUkhPSB4CfG1YF4Oxo+sJXhm+1KCVJCjROkHXxB3/LS3dUFG2SEOc6y5DEa10MxY1l1MfH5lq8eCOHa1PfN3XkunJSSYQQsa66juga8xalt7oW3UqDhJUFKSSdzBi1wqu67wHowXndl4UjtPU3Re2cNuJUlYBcTuk7e2dAOdqj4iaCax4hMDMFeyREjYj3D4m23g/gzMHIrAxe73NkPMBI6SAqSSdot6ncIbyF48PMKV/a5lVFz0ZUSdz2KZ+dvNDCmBMWYzZvFzC9zLrxdbBqKvItCS23B70KIJ2O08rX89GK/EVvBHCri1KUpll1hRV/D2by0/wDhSkRvFmx5odCCORVjs68smcDzCn27a5lBKHKgMBSZQ9knKQDv4QT8rM++8B4Vqyum/eVTUsuOPLU0lKUtJLq21OJAKSSCltLcaAIJAG0HVXgrtFIGXdQ3BB8/nrZNfrXHElOeRMdJ3Ma7D9PC3HiOxuF2xm0subnuOhw8p5TNU+92zaWk9uUkJSHHFxIGveeWdZ0IHIyjHCNzUzjLtNU1iTTsllAzpMBTDLJJBSQZQy3vIke6x9yqQYSVbQM0gn4/CyXV4iuikKzVXhTgMhIWC4BBUYEjoZgeJsUMdyQXStCUqNTV03bS3bTKcW1SMJZQXInIJAkJAGmmwiwb1YtQJCQOsjvEzHWbM7EnEu6bkHq6E1dbUtPFhTbKJIWUBUiY0gk2Srg4iKvi/XLlqLvcpklbgacK8wUps94EEaEDX32kIHWzWQHVLb5bqnPHy60XZxNxMw2Dl/ertQDyyvpDoH/MfjaPQqaMI17jkjwzD/8ANFpw9LC6fU+IVTWhGtfQ0lWVCd0lbROvLuJ+ItBrWYtupGoACjr4x+Nu+on56djvILzyubkqXjzV3uCN7qvLhRhx8rzKbpjTqE/904tsD4JFrw+hXfHbXXiq5Fr1YqaarSJ/nQpJI/8AZpt58+i7V+vcMCwXCDQXi80BMQClC9+Q75tdj0Oa/wBU4h3rdpOlfdK1gciW3UEH3BSrcvM0RVjh5rqYncSiafIK4qyTpHhtzsEogEgHTxFu1HQBR8NrckSJO4EG18bLOcgzptsLIV8NZX88ABQmy6RH8PlpZNvhouMByYKDNrVM7K9Vp25mpECtRtvbQJQ+sH+LKr8PwtsEjWI0+VuXTleSqVd5J1+H621mFZDguys+f3e62nEIdbU0sBSHElKgeYPKwZJMkE/CwiTm0J1262O02N1A6rzUwPdbuFcb41u67aNmoxJcNHWt3K280HIqWKlKXFNoUIU6KdL6k7kESNQmyxflK/x0uG9eIH7pormvqirspdqL4cNPeCFJW6622at5QQtnVYQghORRTlBCQXPxXr8H4L9ILElJia4Kena9YReVHeVKy+qr7Z7s3ZkPJSIKnYKQNUpEzJsTxFxi4ZY07BzFar5vI0zRSy3U0IeDaiolRBW6dFdySEySCSTpbr2vc7LIByC51zGtu0lSv6Bl9B7COJsOqOYUdezVtidMrreX4S1a0yTrBJtTf0T8c3JUcXr2w9h+7qWkuuvuou05TRpYeccbLZIcykgxmcjwtcVKhHdMDlH9bYWJDLUE23WrRG8IA5IYHmBFh7nWpN+Fue69TgjzSqP+qxUHcETpuR0t3TrLd60D2aE9oppXhmSY+aRairaeBWQSA2rToB+dst0Qf5h/w2y0rKC8LaOf7X4WbSqc15tqkmYTlJk++0+BtfZgI72YyVCPO0PcNbmpsR8Y8EXDVkhmrvQ5+zVBIS2smDvy15ee1rGY1wZeGCq5FIuoRV0r5Utl1RyqIEaKE77Dx08rBpaiNruETqVOopZHs4oGgTWKVJM5NJPLUHlI+vnbpCVKJQUkgaSOY15/UW6R2qYJG41/xfOetty9yCSD3Z11Gv4eNtJZpaQu10lY0pptTYCn2wtAS4DAOxMHQ848+tt/u+rU4hkkZlQQUid9uWp187cBb0ZgogDUgcunl+PjbpT78hXaGABqTJj6/K0EuS6coWmnVBSyttGsoSSR+E/lZWoMLU79PU1rd+XZFGwip9WeccbdfSokFDfdIK08xI0IidrJQeeyZS7qDBE6R0+t5t3mcy5+0He00HXz8J++zXKkwDdF/VXVuHK0pMKn2dNbdtXe+tU9ivvK8pPhPn/Wwmd0H+8gDunXYdT8/iLGqdx4K7ripI1jSInXp9G0XPsEVrASnVwvoL6oMY3VeNNd7zqUVCUuhJAzNqGVXxBP32vrS8Onl0hCqcErTAhMR7rU14QXY/VXwy4/fLlOG3EqVlGigCDB/L6HpTSdmqmaUgCCgEfC3I4pQjFanIXWsF0lLVy4bTjh/EV40emnhR7DvpetqRTqYQ47crgVObOe6JI5bK/4bemtOn7MJ6DkOVqM/tK6mjR6UGFWmwA8KC7M8AAk+tOkeegA+HS16qdMNAxvZ4IhBEIfl09Ez5DMeI7c6rsjTQzqOdgHQrmNzYyQAnSxd3RUgz52txoEmyJOpkculiqgORmeVjjwJBg/CxZWnKR0iLWhsqTt0tYLol1F/wBMAkkJzKPlFpPZuSmYcD7aPtB7JKiQPjZgcOnUt3+G51caWn7j+FpSsPu0cri526mZXMGVpSebtKlZlLBJ5xbf7sT/ADfKx+2rIUEPRR4z+qTlXUM2YLIs279o3Ke8EoB0WJnw+hZ7WZ+Mkreq0MIcLZKU99O41sanpY4X5mpnSueLORFDD5khf/MbA3ywwq7FNVL6EBXdClGBJ0H32GZSkaheb/Ny+VkDiI8lrC7z6icrbrStAN8w62JWXdA8A8kSlIErT5o3WUNPR0bNK64gdkhKVGf8Nhbpve6Ke620KveiZCXCSFupGnvsycUYuSjDf7wIkuNsn2huUjb42g/+1rt4VaqTs1JbWVQQqduX3Wy8MjeHZyVcr5gW5Fbqgvu5bzfNPQXtRVTqBnUht1KiBMSYGmpFkS/+JXD+kZqKB/F9zN1dK6hLzKqtCVIUlwBQIMajWRvpas3Cbig7c2Lr2W/RNNs09K4yt3OZUe0bUInwGv4c6t8bcVJr+JuLqZKHMwvm8WlgBJzD1hcRrpztYkxBzah0FtAL3VdlLeISL0lVxV4fpa7Y49w6lDmYoJrW06DnqbJTnHfhC2ShXFLDSldG61C5iZjL5W8h70oO2qCUERIJ0MfDmYmJ/OythdbtE620llspdJylwDTSO6T9a2aTEntFwAnZTNcbFXdvPHN1439IfEgufECLzuP+zrVVTqaI7EOsOsLVGkk906/mLTvcuOsPXbelfhyovkN1jV93gyGyy4oQp9a0gkJyjRQ3NqEcHa5268WXqUENn+z95pMd4KHq6laEHXUD4/G+PDSuFSziq8W1gqexC66DodFsskffbJoXSOry9thcFadU1vdAHa2Wrx48cOqSodbVfF7PFtICBTXHVuSfAhuOdo19IHjRhS9uF1Vdd33fipdY7UUz6FPXHUU7YTnT7S1pCRII3PO0+BbQcDvaLDmqvaB1syvSEu5F48DcTLb7RTrbTb4UqVDMh9CtZMAaeQt0c0b5InNcdLdFjxSCOQOaiOHeIt4MYdutit4XY6VUUtCyy8TdraE50oAJlTg5iy9SY+vSpQlTPC7FEFYB7RVG3kB0ky/pZ0sKdqKSnUFEoUyhXdJ1lIOuhsKGCtktFgqSSJCl7+6LRax4AAck6RpNyE78EIF53A3W3ldDtG84pYVT1Km1rbAMQShSkmQJ0POxS57suR3Gd+/6o0XXAw97MT9mEz4+zYvS38q7WPVpykoCgmR3dI/C0f4axg7RcTK9TrxKKi72gEk6ZkqXr12Atw+L4pTRVrYZGg5TdxIGv8utakop3xPkYbAjT9/7KblXFdJkmjb11Olq1VnAnAtwek7fnFZFxMuXrX09EaVSzKadwtOIdebSRotYbSkkawDEZlTNasdojQjpvaO+KOMW6FNPialby1LaFUxezJypBIUnMD7R0WEgc1e4zjxrDpZWx0zcpPMC1/JHoqGpZJeYXH81RPF9+12IcMX7T4NBqb8oO2p2WWnG0qTUR9mkl0pROqdyE672rB6TfoH3xxa4S1fEWg4WJu/izRULVVUuUlZToF7LRBeQ622tTa3CnNlUIUVBIKiNLWTu2+mMRXawzclTVqr1OGrZKHWwipZXBWwhSicqgqVpkEb6idJZurEdcu5KVF40LgrOySHEOuo1UBqQQTPLlbooXxRXcT/OibEGyEBoHlY/uvnBpVusVZaqEdmtpeVSH0/aJgxGXeQJERzMxNluirHXstE2mFKXBQlsJgkbzy3jXkRpOzn9KG/KW9/SQ4kXrTerIaqcT168tMhSWsweMmFEyoqCieWYqIgaBpYZZcrq1LbKApLac5BB1IGgE9Qk69NdptsB92By57JZ+VSNhu72WWUCqqEBoASlC/4oJUdCRqJ6ezI5w+aaqoqWlXVKUlDQQlkpJgulQBGg31JjSN/ez7mp1LyjtF5MwS6lRSEpRsN+U6g9DoNdT6a5V44gTdbCVrYokdstG/fBImY2Gux5DqZq3VkaJ60RqEIaebIbU0hpOQAJDZA7wmNgSfAkaaRY3iC8W0Xe4q8L2NDRDWurguFBKQO63J5qOmg0g6zIbLTl7NNLF2XJVXgtxa0MttMyAoFW6v4B8EiSQoEgWaF5Grvy/vVb+CMS3q2rPS4du5al01MJ9upcSCDB/gSSesa2ZrFJzwnBQXneeImVuYFo0YYw+2vK9fVa3nqahRkANiJBIPtEaZtxtZJxkL7ujDt40GErqQrtEOF28at1NQ/Ut6lSmiNBok+UGNdbO5N14+o0NM3veV9uZEBKLvuW7mkU7ImQgrdCW4BjmeZI11HdpXFJdvC9MP0tDQKKUupafVVuyBqXEspU2mYiSAJG82TrstZJnjuCqdds72/rGcl3Nnzc80zNvoF9GrE124qwzh7El8sMPJvzDlFeJ7RIWA6tlpaonbVareCuOruo7sxTX0920L1JRF0rp23TJCD0I3EzG/nb2B9FS/3mvRu4d3o24uWrgZYzeCMzceXd+QtVxYnujnsFyBdSoI884iJtc2VxqLDfDe7b1dvunuunTVOkklSipCSd8qCcqfcBaLPShq7lXh+53LuYp0OCrcQpTbYBCS2dJHkLN1eO3yDDx32n6+hZj8TcSO3xczTTjshmpC5JkbEfXu6284irp5rQuY0A9Au3iwcwSCd0hJHUqOLweDtO2tw7oIMiDueVkJ9BSolSArLoPET+Bsoh0u0pQQqW3VxI2mCJ95P1pYg+2VLStWqdoT49PrrbYiuBZHlN9UwMbUSarBPEW6gkSKduuAg6kIzT/wAlqE4jbUlxRJV3kpWJA6662v5i2+7iuWtv27r/AK/1YX7h91llJbWtTq05kFICQdgofRi1Br/RmJABzBGp111Hu6/K3QYOTc3XMY1Y5bKTvRKoLwvviJVYZu+vRRuXnQFGdwdxUOtkpV4QT9+1p79EaocHDy8rkWYVdN+1NLBABywhX3k6edqbYDxcrBOIBfIo/WkFhxhbXaZCQsRIVBggwoHqLWk9DDEH71XjlKW3GkuV7NehrNmy9r2siefsJExrFrmKxXgc76Krg8uWoa36qyt6VxobvfvAI7QMNqcKQTJhOYjnp+u/OKXOJd9OtuuKu5NI8232hBM7OJnfcFtYM7zNpWvSnNTdLjaQIfZUiPAgjTwPnZm4IuHB9+UqU30lvt3rpqiFB7sz2zdOS2TtOrZGXmRzNuQLmxNL3C9l0WJVZpI+JyCiitrcUX1dr1det+VgYYWioyJGQBC3uzUiQP4QJ/rZPvynXSVF6pbecW6aippnVrWZUlCEutz0Pd6fKz4xoxddC9fdz4feD121NO460M8hDiUhaoVzGk/U2Z98OGpvBxQImsdoX0idFdswW1QB8NuVr8JD2gqlHPxWB/VEm8Sl28HL+qAVBupoqpzTUgNlCwI0mT/WzjuasZXjynDTSm1OVq6pHad0hLrCTlMayT9GbNld1N3bc9JebLi1mtpFVOUgHIptbRKfHUKg2cD14C/LyexDc9PVlCU01QkFogqUhZQoaTOhB8uXKzyDTRGYTfVMT0xKMKvK461tvL6xRVVOpQBglpxLg/8AGr42rCx7SkxMoVa3vpZU9PV4auKtCwt1mvWyRm72R1pU6c9UJ18LVEpShNUguju5oPltbpMJdelaFzmMNy1RKsn6IV4lV34nukqJKHaaobRJ/iC0qP8AyptdT0ar2N28ZriClZUVSX6RwEiTmZVlHxSn428/vRUvVdDji8rvERW3avQjdSFoI18s1rfcL8UqpMZYWxEmnfpUNXpS1DjT7fZuNo7ZIUlQkxpM6++LY2KMLKvN9FtYY8Pow36r0tMjTTfabcBY2j3W7WkSRYMyZ10PysduyrOC0TCtJ1PhYCob7VpaOosKfvMGbcq10jcdbGjNihOF01lpyLj4zYs+uFsqJH95G/UH9LKN4tdnUGdcxneIsmV8ppysT3CFbHSDP4W2YzcArHlGUoYKiJM687Zmk/kbB5wdj4bW2FgiQDvqbHCCmhizg5wwxzexvzFuDaG8rwS2lnt3s2YoTsNCBpP1pZPb9Hrgg0YHDO4p379Pm+8mz/Jnmeu3O3WYEyCD0FjCaQCwcbKBjYTchNfDvCvhvhK8G73w1gi57srWkqSh+npkocSkiFAEa6jSzuziQTuTYFMyQDttboK5GY/CycXP1cUgwN2Q2aFQRqTuPrW3T6+xpjUwf9XUh4HwSoE/IGwSTr15AED8rDsNCqbdpiow4hSYHORFlyST2EKAISNddxbLItDfVKaKnLriCvskZiTBmBMi2WV1FeP3o+U5rfSMwIghRSmorHEk6exSPHTx0taf0gm+zVdcqUkqW7qkxO3S1VvR0ve67h9InCV7X0+lmnaF4ZlDUgKpVpCo6SqSbWo9Ie9aSrq7laoFesUzrLzwqmyC1qQIkHQwJ8iPK2Ixru/sNtLLZDmjD3i+t/8ACh9IX7KBBBkQN56/CbYEjMQolREA6fX38xbk+0CNUwTrt+mltLcWlTaEtjKYKl5hpA2321j426XMubcCdl2kFRJJUQCImYEa/f8AW8uzAnDDG/EapepMHXI5Xu0wSp1KXENlCTOpzqE7HbrZqNZipJJM5ZJI5gHTl0+tbTd6NvGC5eFWIKt+/qGofo6+mU248wpOdtQ9nuKgKBk89PGbVauaSOPNELn+XVeXOwXaE38V+jxxUwTci8QYkwuqiu9jV1xb7RjXbKlZOp+4b2jpTSm5JM6wdYMGfr+lrVcffSdwhxGwa5hu4aGuQ8taVhbyk5dCJkAnqfhaqzzrmVAbLamypWckwTtET7+fusGlqHygl4t0UoHOkLtNOS5SFkglSeUEjc/h9eNjdIUthJMAzIEjX8T52IHvJJkkRp9c9PvNt9u6xXU7Xao7BSV9oT7QgCPASJ38LWHG6vRgjdSPh2/F3U2FNqGZSTEbyIA+voej2E8Ts1OGrrqFrkuUbKyZ6oBt5QNYlpkP9mVkIKAoKC9zMbe75i19MKX5e6LhuarReTIu5FzNIcpewlxb2VGVwOZtAAFDLGs725DH5pKLLLEbHVdFh1MMQaYzyVF/2it9euemRdqUOEIapLmRy3DqlDfzOvU+63o9SkltMjcW8o/TfvFd4elqw+pKVDPdCM6lRslBj5/G3q5Sf3SDMyLFpZDNTxyO3IuhSx8GV0XQ2Q2w06WLuA6zNjPKEjaxdyZ9m12NAk2RR3TcTrr42Jr31A8NfxsddAiSqDYksiTvta23ZU3JawY/2GJKI5oCllOvOUkWl31jxtC2HXezvyi1g9smdfr6NpUD6dy58LU6ioMD7DmiRx8QXSqalI0zC2jUp/mFod4nUnGKuvZj/R/f130VDl+17UALnnMpM+H3WZ7GEvSVfWDU8S7uQJByoTBIkaSEdJ+Nsp2MyNcQGo/dNLqyJqQP4vnYnXO08turbbUpSwmSkEx52gqgwDx4NWp68OKbC2IP2Scw321y8rJ94cMOLTRbdreMlQ6jMEdmAsDMrTNoRsT9bgT8Ync33LJ20tzaysGp+kCNGWx/ui1esb3zUVN43pQu1jpaTUOBLRUcoAUY02sewvw8xtc1+sXve3E6trmGgUqpghWVWo17yjvEGQdzpZj40XWVN9XyKOhqHFduuDkVBM7gjlE2nSV76guEumnVWG0xis63NJGM8d3XdmGGbuvEuNBLTQUsxlToNTroNN7RZX3t6k8aynQVBIKgoRCkkcldI8Om9lnHmEl4muenpai6Hq1qqaTTVSFtqIXPdIOm28zZiX/g/GFEf3BdKaBmmQ203ShVSWWqdpDYRkInMDmIAM6xynS1R4jG0lhOqsS4aZrOCRWeIzVff980t1OJhmkfFWAUqOdCM0KAJymNYOugtE/F+voU8XsTUorKf1movSufQyVjP2ZeUVKjoPrY2dTmBcTNC9ayguBhip7Baq9+nvN1IcX2fe0CeaREK8OmpXHfCjGlTebNY5hJNTXN1Sah+obr6p5VQg6qTnU0BCiUkmd0iwH1Eb5y4m5VxtCWRBpNgowva96O7KZuoqKpLLfaNoJGuZSjAAjXf6HJfuPDF8OtuXulS0UqT2pKlFZypBKsqUyTok/E6WdNBwlxzfDjd3XZwifqKhDgdW6ahXsJBJnMkBIBiNOnnaTcK8D7+wjR3fjC8JpHqNDjjt2B5cBOZDYDpMZ9V5oAH5wmnDv049XdFXdTxxt4j3CybnDbD1et2nxGwAq77xuuqLKnEFDvZu0zgBKTqJkCCJtdXgOt1zB941TicwqaqmqTlOvfpGPDwtV66Kq9K69mVKcR6utS0qASJgpMDcmPrrazPoy1J/sS4AqAaK7HU5o1JpUjSP8AL+vQ2EFxnBeLbqpVEOgIBUpd1CR2jZEiRLh69ALN3jBUso4K4yLKkhtm6qh5ztZAyhMkydtAfhZx0bb9W8lIp3d9ZO3ysVxzcYvrhrii7aqnU2mruypSFJElEIMESNwdY2t1jj4T9FjN94XRPDb7l5YWuKspCpSaq66V3OlWhlsHcedlH93VrkypWvUkwfhZD4Xv0qcBYbz3klambqp21lSgCopQEydRExZ4IvO7SNK+nHPV5P3Taq2aMAXcEZ0br6BNLiJet8XPdrCsO3WzeN6mnSG6JyqDAcQlxCVrzkGMqVFW28DnaKa/EqKbF7TgPeQ0UKUDGmkDz1+dpVx6blvBNBfNLU0tY/RF+mbcacS4UBYSVpBB0koTPlaA8ZLCL0F4NABam1d5StQruAQBJjQyfC3lmPRsdizxvf8AwF22G3FCw9P8p3X5xOFz3O9ejjNQ+Gl99NOM69xsPeNN9LMy8uM9fetK/QU2EMRr7oIdFLKAdwcwmCIB948rIVRej5lSX8iErSciUkJEjaRrEjSIn4WI196vrYRDiUpUowhBgTpqdNSQBqZPwtnQ4a9sgkYbWVqWWQ+6bBP/AIVt4Hp8dsv3piJdy1NSs1BQ6RTMFxuFSFPDupWNwkpMjSRNrQU173RfDLFXdN8UlbTLK0h2lcDqCUyCMyZToQQddDahSHM9Q0673ZUMxUdIOwMnvbxETrazvo15XeG1OQSsUlbVNKTJAGYpXtsdFbW7ihmMjTG8LGr25iJb+S8P+MlSLz4vY2vFht4JqcR3m82lxICgFVLh7wGx1E9OtjfD9hosu16mo74SnUzGkQYjkd4k9NbC8W8L3oji7jOiapHVJaxHebUlIBEVbggHbX8bLWCqIXVRIS+UtFtQUCW90iJ170qjp0IjQgdTmGQALmADxCU5nKtu5qA1i1oKhK0hW4TuDrHXz1HhZLuCuF33YuuqWWy/WLcWolBQGxMgaRpAMT1POLI1/Xk9eTzVI04QiW1QYIMqiZ3/AJZnXqOvOIKpmkw+HJcSW2cspaAQTExmT4wec5pOthjdFJsjVxXXiHGFGq9FV1TS0Lsslqgbc7SobKl7qBykScvQDcjUWfOGsM44p2zdFwleGKN9QU52PedcIhOqUEob0P8AEpapPeVBtF3CK7qq91paN7Xsw00FJCKGrU2reYhOsakmNYCjytNN1YNuWqWad++Knt2SA4t++lFYVr1WFEzpGXqOoFgNQb3F067l4dXPSpZrL8qK283kAK7WtqFOpTGoGXNlT10EactBY7WjD1K+t4XK1U1a++z2DailsFOhzT4H/hjlYK7cBNNJC6LF1/IS6hRJavNahJJmBJEQUz4gzsAVZ24q+haQzTX/AFqnXXgS442hSkplXdlKR4GdfCdZUjfDspROs7UqsvpKYcxLT1d04kvRRXRvoXTN6x2apKwIJnUGOnct6N+jJTu0Pop4IQolJFwh+SJgFxbg+R+7laPuCeE8HYprsQ8PMZXK3iG7r/u9Dj6bxlwywsmUKJzNkdoohSSCCmZtZJjD90Ydwi3hfD9Cmhu27qD1KkpkEkNNJbISkE6mBzMzuddbUZf1IDGehVuL9OobIOoUcV5FdVUNa5WVDaqF1biUsvqS28S2pGV1I0WkSSEnZQB6WJ3zVLdplo2SCCNJjXx99gy8pRgDJOwUfl4bn5eFilYttynWrPIKQqCdQN/cfC3BMjs5eiufmam5cN3U92NV1LRh1XrFWurczuqclbk5ozGUiRokQByG9t3rdlFeKGU3gylxLL6KlsKUTldQQUKBG8HXXSw7S/tnQ3ErbSYAjYwJ5zqfrYvWr7QJypSYMjMdI3+vO15tyVTdYCyhL0kqd4XnhapabcUpw1LJCEZiYSDA66n4pFqkYnpFNVFQFtLQ4HlpWlehmOc68t/O3oNWZmMU4XrArKyiscZWSsZYWggEmY3199qZekLdyrp4l4qomwMiL0deBREK7TvSDrp350+Ok22cMltJw1zmLQWbxfNQwQQTr4Wsj6EN4djje/7tj+/utL2WNy26kR5982reqQpQGmpEWmn0RLyNDxmoqQEj94UVTTwEzJydoB/93barm5qZ48ljYe7JVMPmr7gzTJlQUFSI36Tr01tAV53NetNeFdRqq6amVSlLdGpx5I7YKccURG40Vt4Wn1umfUyUIacgHUpQTP1HX8rQ5ijBtU5eOJquku6tXeDDjVTTZW1ysBKHCECIVrmGnXe3HRWJ1XaVbBI0JpNV1x0d7Fb9/qqUIu9a1stNnvqLMKIJ5RqNvvsQbvzDz1Zh5dVdNcs0zCEJfdUlJCUuZUqWmNQDHhrrrbt7B1+rpKWMPXgHHqKkUVCkc0UUuNuJJywNCkkfG3N8XXiG8rquxScNXkzUN0gu97tadSQlaFtrzmYgGFAWttDQswNLdEPR33fd5oFNdd30TNNS3a/UBPZ51NkhaSEzP8Y6bGPCxOuvi8r6eZoWK55lirco1oDaoSnM2oECDp309eZspXVcWIbrxAhtCaVqmdeqKV9x99tKewU6F5gCoanOY+ob13XDfFA8k1C6EhpTCm0qr2IGVwnKZXvClHymzjLdTOYJv8QbscvDCd9XpeTT6bwu9ugrGw4TJBUlKiR1gn5+6ulYkMVzyUH2HVZSPA2tnxRauW9275ra+8qWlbFzvtt/620S4+kKyCAomDIjx6a2qhehBrFL0OcJVIMzoJ+c23cKdeMhYOKttICn7wJrU3fxbuwAqDb6n6cwRspteXz1g+61wUOuN/aBWVxEKkHUKGs7bjTx362ozg6vF044uC880JZrKVxRB5BSZ/G14iF5oiJ9lRI7wnr9fK1LGG2ka7yV/BX3ic3zXqDcF6ovq4LsvlBBRX0bFUkg7hxsK/Gxs7TGvKzD4DXsm+ODWEqsKBLd2opSQZ1ZJZ/93Z9FZid9dLQjN2hPJ7xWirkB+ttEpP8AEPIna2iZ1OnMGLYFCd9POxmoZSXe7RypcQNt/KyHUI7ZlTYGqgRt1s669pCqRUAT1FmusBILZMC2nTO8Nlm1Tdboow52tO2sgkrQJA1A0sMFAHVRsUpcyWygiezcWmOneMfhYcrAM5pOutrrdlSQoMjOTynW3Sl9IB6czYHNpJk/hbclIAkmBB+FnCS7B05TMwPhbpKh1FggqNQQB523PdM6DxM2I1MUYC5ESNNxY5d6z2gOokjlZNCifOx275L0pAI8TytJRRGrumtNU8WsmTtFZdtp0tlnWA0RJCjPOBbLNZJeKXDlSneKVzZs8JYrTqoCT2UT56/P32nK8alxRSlagQ37ObWNNuZ+vC0DcNcx4l3a+3KUilrIJGvsAQefX4fGbKxxa1ZlQM2kCdz49JJNo0oFlCdxCUbocbW/2D6Z7p15HedRvZeRTXWACppaxMQTJH1rZsXOHF1YQ0palAH2Rsn4672cKaWqKc2Rck90DWDp9fCzzuIcpU4BajDbF0iUpZQVQCN9Dv05eFszXXJUmmSBm012259NB8rFvVHSCANANSUyIHS2IoahSlZkEGTrEz89duVg5r7oxaOiMofoE94sCEkyZB26/UWDF5Ui2VBqnbQAZCQkRIjadOY91nPw9qcEXNelS9xBwnVX7d66VaGWKarVTqQ+fZVKSJGhB8510FmTTXah12pqKBisQhyoUpKHnw4ltkgZEDuzI70kmCI0EahbN+oW2+6TmWFwuqq9EtIU4G0IA0OgTpt+P6dYy4p4gdpzS9nWqpkPFTyHmnRmWpP8I0IiCNbSBfNCG7vc7ZKXEqRGVUqkbTHgT87RDjanaq7qVQ366oC7XnTTrGi0oDZKZ3OsHTnHhawZARYKGU2vZIlPi+svFppP+sLbbyOLcRGfss0cup8tjyNvTTB9/qewJdh76YuqmkH2ky0nffWD87eSeDayrXiO76Tt2mmmkn2gopeCQVJmJ0M/Lnz9OcJ1dS7hC7HA9lDlCx2qAmQfsk6AgCNfwty3aOzomA9Sun7NN/Uf9lSr0s60VHpMtOFXdVUXUIIn+FAPloPn5W9faNQDSAYPd59beO/pSOJHpCtuBSVRU3aIIlMZUa+MGLewbDhS0kJj2Qd+djU1m00Y8lUqjeqk+qPKUACSIsVUoKnvflbl2oUUkcuosVU64hJSlX6fUWssdqqj23QrokRPkBYi4VHz3t07WLSIUJI8N7AdoHBIBHWetrrDdUpBYo1dbmS9KRcah5G3mLSGKg8zaNKNeSupiFbPJOnnZUxNibGF2XimluDBjl5sqQFesB9KU5uYiZHvtlYi1zpGhvRaNDlEZJ6p31NR34nlbhl+VgZjaL7yxjxgCwKbhk2QU7qrET8M02BoMWcaXatpL/D1inaPtrcXlyn3qja2MaSYuzC3qtATRhtv7KW1vKSgjMdrJd5N1lRRGoaaUtDLralFMaALHKyRdF642fPY33h2iaaXr27dUJQOgSCcx+Fur8eWLvKEOEfbM6BW/wBom1acljsrlZgYXatR1L/aNgg9fvscuOhadSkqCSV5j46k2ZmIr3vq4LqevS7bpfvt5TjbbdE0Uo7KTClExmO45H3CbR1ScZuOd3VlWbo4G195sFYFPnrm2UpjQgZgCAT1J/C2fVtdkDgRa/UBGlBylrd1M9JhiidoqlKkJOSvqIPT7ZR/G1R6UJxyi+LwvwtJdTeFRQLQ0S2laW1pUhQ0VBnnaSLw4t+kPTUNS9h7hNSvu1L63KhirvFseqOk95sd9OYDQyOtmpwSwTjaiu2+Knifhpq7a9293Kmnp2KhDiFNuJSVElCl/wAQiCfjY+AU0T3yx3HEdbLqDte/4WHiz62FjXsJyjf+ybV716cM3XV0VPhxTlDWp7OoccUteQZcpgpSOXnZXwlg5ji5iW7rxxfdXq13hjsU+rvOtrWy2hRCozkgk8yNYtN9Lhi4b/q6S7r4uplyncWESUKGWTyOX6m0l3LgbC911blZd90MNONMt06kjNqEgK16+2belwQUdHTENjHFIPiXGvfiFVKOJMTGOSjvAvDC7LrpK5nBd1tXeisoEJbfqAtQP2hUoKUVHMYTHvsncVrsu5d5XDTVWHmKn1uqWXA62SFpU2yYGWZjoAdhadaFxhhqqSgobQ1XKQEto0CTlgADzsiXqhjtcN1ZS2VLUCFFrMrZA8+VqmGwtpJONKMzzzVqoa+WPhtcQFRu8KamuvFQu+gZaZY7dCuzSmISoAyOca7+NpF4EMUVVcVMa+9Kllmjw/RukMVq2QopcW1Kwg6gBOlov47XXfFNx4qnbuvtFIyxVsDsywnuNKS2VISqJAJBkeI6WlH0eGkm5qpxx1BULoeZzhOwavB1IHun5m1OZpFTdwtcrepzelsw3NlLdAnCa3pXeFWWzquK6pMjp7VsqsJ4Jvmgfu+gfrnELS6G2lV1VmUSgka9pBGmx0+JsiuM1AZTVrvdwoUY+ybI98AiBZy4R9TecDZvd5a+8hbTiT35SQIO552stc29i5voqnDqr3ymyZ/B7DGHqrBV33kq67uVVPNFFQ4872alFDigAdCDGo8Pufirmu6jpgtu7bndCEglAW2VHrr2ZJ5+O1mR6PwuHA3C6pbrqv1G7rqrK5bz9WvRH26lElR5SvTziyhx04iJuLAVNRYRq01V84yWm7rmNMrMVh2Ap5JHIBQg9VJsMRUkcHFkaCbeV1pRR1NVMI2E2JtfkEqXnc12oubtqO66WjWmpTHZJyyhQVI9lPOOXS0EcSmgy9TOSnIEqT2Z3Ptk68hoOn5Ta3hSswDwiu3Dtbez94Vl3UjIqap90uKcd7QZ0pnZAKiEjkkC0E8TH1OUaXjU5yXwUIygIGm3iZUfjbhu0UAjxKNzW5QWjRdLhjv+zezNms46pq1Abo6ZxReTAAcgrByJ13PT39LM+pxzhtKJReSld/8AkV3oHKQJ2NnQZU1Udq23k7spSJ128REffZvvXbdjIIRdtICSYKW0DWZ1kaj68LEpwAPEpSlxHhKkP0Z8aYGvPiarDeIjdtRQ31d7rSG7yZQWVvIKVoA7TQEgLgbmfK1tb3r8CcOblcdTW3Hhu6kK7VRSqnpGCoiMxUcqZgASSNhrbzsxM3R/uCuUzTtpCWs6YQBoFAx87QHiFls11KU0zchG4CR/EfEa6D4626XD8r4i0D7rn63M2UOJRHiS7RXzxExVe9LUest1d919Sl4rDhdS5ULWFlR3BzAyJ331tH2IA5R1ZpojOgGFwAQrMIJ1/lTHlZ+VFGpL3rDaCVAAajSRHyMRPgJsxMWKDeJFBKsy+waGgOgCBpGpI3119nkINtVrcoWc83KQ3hmrUhJAynRKZSDqCZBiOo6wOYAEmcEsM3Vj7jThHCl73QKyjcql1VbS1DYW06wyw46ttwCNFqSBtOuh1i0bOrX2wSXCpuMqe8SARodz3tQABuJMCZixPoG3Eu9uK1/YpcSSxclxinQQggJeqXkhJKjucjLo1AjYxFpDqhkqNOIfB9PAvi1WYXfp3H7hrgKy6aioTnTUUilSUHkpxCvs1bDuhRhKxMnYduvDdattHqt2LdVlQGQhHd8Ak7QD117u0Ws16RXCFzi5w+Xdt104cxFdLiq65CBlUt2BnpwojZ1ICde7nS2SSE2qbgLEVzVN20iL/KaepQEsIqHWygKAMJzEpEK12keybGifcoZGllI1BcFxUziBTXM0C0kEFSYSJA56k6jU7fCy5UNrQ0W6d0hIRIZUSYUNU6HTdI8N4sWo23Ujtm6gLQBIUkhUDKDA3nn120O9jVTUFhptyoqEBBIPeBOYyIAE9AOfwItYIuLJgbFGuHV/jD/EK5q5Cc3a1CaRSQJlD0NkSOYnN45d4tbtaJQUFO+ijprrHvO3hHzpVdwqKK9LvqlNKaFPVtrU84AhKQlwDSTzhI/C10pSrK4FTm1HMRH1tbNtqQrpOxUHvH1dbrZckpKkEHwO23hP42LuqzIUqICh7WojWx+/Gyi+bxaSAmKpzKZ27x26/lZJzAwcxCBokAkgc9umv4W4tzcryF38bs0Yd5JN7TI4QnMpGQjKdY035eO/3WAcCVNpIIABy5QfP6mwzhJeCciSO8IVqeev18drArCt15RrIA2+o8t7GZsgP3TZxhRIXc7LroMNVjS8wAB9qJ84J+t6n+krdzlHxCvRJVm7SnpnEZtCQltICtNDOVQ5bWuDitvPhyuAcEpbzgHXVJB1+Hjas/pYUA/tPQVoZJ9fuhKSQNMyVLI194+XS16gdaoBWVijc0BKrA8haXFhQykEgjpZ0cKn65jiJh/93VRp6l6uap23c5Rl7Q5NxqBCrNmqSUOEqEBRnT68LHsMXiu58R3Ve6NDR1rD6f8AcWFfhbqZBmjI8lycRyyA+avkcB8Q6snt8UU5CY0NQ4SSY3gfUWZuL8PX9c9Q5R3liZCFt0prEkqUO2E+wk81d0j3/GxNO+XEqaKWgEIUUrTOZwGDKuUg6CI5WinjI1R09XdlWu6KetUAsuFaiAlpMZo1EnvE8+fW3ER3z2XeSxNEeYKJ7nuK8r8vdm6Hr3dZcdbVJgqAytpVMTrmC/dHjYhTXPUP3iu7S7XOPs1RpnsrZKYlYSoKE6d0ctJHSzyp72vpgNXwtFDTtVDz9Aw602C4lSW1JTrvoUiNdbI9cnENQzVXjUX0+Qq7kV6G0EpEFYChAI1GuvOfO15oWcWhJDWFMTVdGPVqGrLy+zhLikpGUpJVuNO8I6ajxs4qXhHeDqUqqahxjvupV9olRDYSOzMZdTPtD87A1dEq4MS0dQ1XVDgbbo3UrWuc7anChcj3p15R7zNYbaW2IJgpmeu33a/GwZXFmyNDC2S91Cj/AAHZvWgTS3teS0rWpZcSyCAQYygTz05zyEaWqdfVO7T1CW3QQpsFozO6FFJ38vlb0cW20qcrpnmSnbnprpr91qFcX7pTc+Ob+ogILV61KgIjuLVnRp5K+uerg0znFzCsnGqdsbGuamcVlAp3UqBKR8CFE/ja91y3gm9LloL2GiKykaqCJ5KQlX4/1tQ/NNMkfyrPPqB+Vrn8IKxV5cNMP1K8qoouwOo/2alNj5JHxsfGGjhtd5oGCu8bmeS9DfRAvb94cIvUSpM3XelTTAA7JVld+9xXwtNizm1J6aC1YvQkvQKu/FdylWjb9LVgbRnStKjr/kTazZOoAn3b2oQG7Ar1QMrytEzpl+Vucxnn4k2w842+RtpUA6g7xa0Aqy2tRUjIdNLNytbLTziQefvNnCCOfLl0skXw2EL7QAz4Wt07rFVagXCbyDkqahE/xBfuIj/pNhe9src62De7l4FRgZ2jpt7J/wDzrCwQTvpOvW2iHLOy3WwVEHlB87YSCdNuttgADMIjz2tzIgGBtsTad1Aiy2CeRjwt1JKpBO+45W51gb9NLbEAkDlpvNpApIT+HnET9fXjY9duj0iRBkEediAgDNqNNTY5dqwXhJ36WnmTWS7mV/Mfj+tst3I5pPwtlnumsF4lcM3GnOI1AEADLSvwQNRKIIPXb63tM1aoq3A0VEjbNy/HS0BYbvNygxbTViS6FBtQBb3BJM7bDTUe60x/vt112nKlJyrTkUlZglXKD8eth0rxsh1DDZOfDlQxT1K3luQ2hKlFJ1JJgQP6bWcn9pLuTKU1QIkQAkjl9R5WZLVSHEKKHZSlI7wMneeX18LJTeMcNVN7G4W79o11qF5Q12kd4bpBmJB5WsSRNebkoUUjoxYKSP7RUQ0TUKhMykJJnpy+pFsOJbuQmEuvTvAT9x56j7rNBSiVZwuQDIG8Drpy1+psVfuulcS46C/nkkZHVAnw3sMwNGimKh19Urf6TLsDjyGk1LnZqUpUDKpIHICZMnN8+thrqxfdbLKnXlLQH5W2Csax3dNfC0SVdO/dle8s9tSvqqELJUSpKUKVBJB3EqEgHn8ClbVBq8Cp99aA3TqWWUA5RrmciNSIEg6e0Olq/CaDqrHFcVI9+cSsP1lSm53rurCqsR2aXM6YCjtPT2YHuHMWhXFN+Xo/iSouxlgU7iGFtLJIlYAGWY9rSfOT42K1d7NPYhReCnUClbzLKe1OZOwAk6pMgR4WPY0Zum/slZdVak1rTRcqsylZy2ACAnrHlyEkRYZYBqFIPJ0KalKw/QVNPedA0X00rAWVIQYCh3TMbDfUyBMyOfpdgqto3ME3MglfaKuumME6BRaSTz21+63nx6P9TUNcW7ho3nHTS1NS5TOtrkocQtKklJTEFJEA78t+d9cHMCmw1dTCNENUjKE/5UoEcvK3KdoJvdjXYdmodHSfZU59KJeXj8l/vKCa27svP/Zo938Q93vI9h6Mj1ZvqEj7rePHpPKqf9OzqUIzJS5dyiJ0ACEanl0+WtvYOhUPVWyI1QmBPha7F/po/osqoFqqT6owqCNI5TYBzQSdLDKJywJIsAvYySDy0sZirP3RV6NdhpYNLiCkSoA+zbp8xuY91iClALmSddLXojZUZBdDVThQyXBopIkR5WkGlrqenp6aodcCkuInmSqEyYtGN6OuGieS1mJKFZQnUzl5Dn5WUrqvCleoqO7b7rn6KspgShtxXZ9qlQIB130V7iLVa7KSLq9h4JBCetXilpK2k0jK051tpKS3lOVcwofCxV/FS+yWqldOcQpBUBCkhQB5/fZLNy3Q8ptZvioKkJaSgodEjs5g6DnOtuFque68110lPX1y3+8GUyoIBMzOkCeptS4cLtbG6vWc3pZLS67tLyqKdtKSkNJW5GgCyTG3MiyXfAJu1eckQttQI1iFgiyw1TU1KghDYT2hK3CTJKj1Nku/XqRu6aoh5uQmYKwCSCNBbnKt5mkuwaBalPZgsUE046WF5idHCNbaoHHHkqbaOy1b+dq6ekTjnGuEsROUVwYnq6KmdpmnOzZUEhMyM207gfU2jfhBxVx5U49aui9Ma3pUtLu+ocUlT6sodbQVBR2nbn5WjNgUtfC14cAE02IRwEtI1VnHa5aai+kZjCa9fLaW0H8bHw6Vsu6eyQR8bVh4c4mxFet63O/XX268X62pRUGqrFJ7Q5SBofbIED4WmzGWP2MAppPWLmrL3F6LUy2i7yFKS4mIB/zToBqSDarhOGvwzHImucDcHXbkoYlUNq8LeWN2t+4Up4VuBy96lhLzeRtSlKLhQdcsEhO31FpPpafsHKxEDVaDMa6oQPwtB/CXi5UXziClwreWCr0uSpZafqmU1jgzLbKADAA01AtJV74urrvvRh6iuVb7DzR7ZfaQltSVN5QdDuCr/hNvR3V0LpMmYE/j1XDilkYzOWlLlI6n1uqoUkh5yoDwCUCco7MHfnvZJdK313Ow2taV9g6EZkwEqCiAfA6WaFXxBrk3tX3bfLN33feqb1Fz0zqqpa2lLfp+3pyVISImEJMDcmDZKxNj/GN38PcI1tNclCbzevZFLUuJCl07QK1pJEkK9tI3smVcLn5A7UJjBIG5rKBOPl01DfFaqfBX2vZ06lkIkAgQSSdNYs4PRtdSfXaRxorKWb5bhaZHdvAqAjn7f9LJHFy9q+9sW3m5ejjIrWH105DTZSlQSs8iTr7/ANUbhhxBc4fCuvVq6m62L2vOlUyp3IPtC05MgHr03NsuWq48pudAVt08PBiFt1Z2vo6ZVO0aamoygCQU06QT/hM2N4ZaYbvNpZYaSudFBpIAPmNffaD1elqmlZWUYApiWXJ71URGXxyjfX4WLNembWF5akYBuxsurCmwqrUVDXfQa/Kyjpmh4kz7eSM6pcWFmXdSNw1obnvvCt/YexBdLV40D173jTPsKSFpWkOoIMcuoO4InQxZL4f8ArpwVxBRiqnvq8Lxuq7qdxq4rorSV/u1bipcKVkmRE5dAe8SSTraO8I8Yb0uKhbeorroS3f2IKhxxDgWexDic3dII00G/nZ0V3HrFN1ur9Ru67dSUgFtShrof4rAOO4bGQyYEub5KzHS4g1juA6zXDVTJj+rTWYPvNtLTKw2xmUW3kqUPtG9RHTXTwtV3G7VOKGDq32yMhUkFOoCfjP1vZ6o40Ymv27Ki7ayhutLVUytl0tMZVpCo2M7iB77R3j+oUigW8SsuEIKSkQE9/Ukxofuje2LjNdDilSyWC+gtqtHDaWWjp3Mk6psNqSSs5gc7KTqTqdDttH5/ErUJEE6CTII57zy3tujcZbqHRmbh5kqSQqTy5H38traeCQe8ZSUn2j4fp7rTjZZFDrjVIl9tKfuWuQoEDsHASE6zB6RHL4jxtAN7sk1lKpSYys65e8SokztqPx08IsVWILjDjOY6oIgqI7xEDz/AFtXu+kLTWMJUVo7iifDU7CR0jWN/cN3DOYWLiVrghFF0oUkNpSFBwkJEAzO2/h8hvrrGGOlrRi11lBStammSsAaEwdgdBvpHOPG0uZUrpgrPokkgJRG52mNdvkZtEeMW3TjKqWQ4AlDSQQDlCUpA5aJ2I35DlvsHTdZCbF6OJbZP2hZ7pAA1JG5AkxzOg8N51uv+z1uYMYAxbfq2Qh68L3p2ElUSW2WVZST/meVptNqZ1rZbU2pohpKt0hJJIG5VlkgQBp7zta/foe0hu3hLd7CgEKqGXH3FzqpZq3N9NwhTY66x0tAu0skWG11YBhYQ82SCiXEmDqNDr4zytVP0m+GrPDnisMT3ZSsvXPj0uVZp1mEtV6dahAIB9sqDyYTut0RATFplvB0LUTMmY3iB7Xuny36WaHpe4VdxNwWcvmmRnfwxVs3qlJQTDAOR/XfRCswjmkWJEOqC7dVZuE07bk3De9RdFSVpK6Z9P2RJg+yTkM/4eR5aS70VN5U7S3qxhLiwgFdTRZS6jaTlV5qO6p00A0sz7rW3e1GyatpLanUZgsIzNqVoIJ5HefETrsFemdvhl9NMilp3A0rKAvPlCZEEb6RBHPWPK6L2TaIKofoK+p9bpnK6pLRCg6XC4G1CAUuNASkTuop5yJMWurhu/rvxLcNFfV2OqVTPsjeApKhoUqHUEHTbmJFqlm82GElLFBSmqpEBZiRl05KOux5eNpn9HS+xel1X62spZeZrEKXThITlC0e2RuMxBH+5ram9mQ36q0w5m/Raxi0lnE94IU4QVrCogfxJBnw36c7IOYI0SsnUCCJ8tfrcdDZ0Y/T2eJVrGYdsy25A5wCPPl4ffZrrhJgKJ2ACTuDH9f6W4upGWdw813lG7NTMPkEnPRnJUnTNOVKtpMeHM/OwtfeNZe9WurvB4O1BSEKWlpLcwMo0SANhyHjYNxSFLKwQZTmnp9R7rBlCgtYUpMpJ1B0mdND+Vostuk7dEr2R211VzOUqK6dY3jXKYgfC1efSipE1eHcF3vlzJVTO05JBB0Q2R8cp67eOtkHEJdZW1MhcpPkRt06f0tAfHen9Y4MXHWnRVJeRZzAHMBDqY98C1qlOWZpVGubmgcFTKqSe0IPSBGs2DGmUjQ2N3o2WqtwHkpWk8pNiidd5gba87dkDcLiSLOXqZgWrRfGEbjvfKkmvu2nqNua2kn7z9a2bPEu72Ha253axoLaNQ4w4CO7C21AyI/w216O16m9uDGEKtRzKboU02sa9ktTfXX2fnZc4jXLXXtcLoutoO1lMtNQwhX8aknUE8pBNuGk/TnI6FehsPFp2kdAoJvairX6G9WqRbBu2mvhDlIECVAuGTqNCDnTGvw3tl1sCsuq6sySBVXVX0Cu9MlJKk6HWeeutnJR4CxtT0i7uacpBT1LbDruZZzJdayxAAOgPjysuU2AHmHmClxrsqavfqEADdl5JGSeRBPwB95zK0c1REDyb2UUXjdrzV23BeCax56aNbyw4rRYQtKy3PQSYHhadqZTb9Oy8mSy4hKwI5ESNNtrNC7+EjlIQivvp99torS22pEJQlaVJUnUnfMD7hZ63ZQKum7Ka7+0W6KZoNJW5BJCREnxgTpYUjw61kaCJzL3QBGubNmOYRJMe/Weg62pv6TV1+pcQ7xcQCG6xmlqxrz7Psjp5tm1zykOKJbGh1EmTzm1WvS4uxTWIrqvHISmsux1kDmC05m19zlr2Evy1FuoVHGGZqa/RVyT/drT4hX4fja23o4VvrXC1hs6+p1j7A00EqC//eH5dLVJb2WkH2k/jP4Wsv6KVcHsNX5dsyaatbfAnX7Rsgx/7O2zizb0xPQhYWDutUgdQVez0NLwNJxEva7VKlFbdSymNAVNuoI89FKNrhkCJBBnUfUWoj6Nt5fuvjVcUqKUVZfpVyrQ5mVgefeCffa9qtR1nmLYtG7MxbNa2z7rkkbTr4W1I/mHnbCevKbaKp1BB8elr4OqolYVayOXU2J3ijOxm0kGZsaOg0V+dg3EhbZTrtGtjxmzkKQZhZM68YRVUbs6Z1tzPVJP3pFhFKMT1nn8rc4hBapu1CR9g82uToAAoT8ptyXCdFKI1tpN1WY4WQwJ1BPWdbcZxOaee2/ysGFzuoaeFsSoxIjQT7rFQzqhCr/FAI+hboKGmo1sAVA6lQgn5W2les8wNT7rSCijCVCQTpr52ULuMvCSNfhFkpKjoYBGkb2PXYoh4lOkmzpJw9465FGedst0kqCQAQABoNfztlldNZeFt2uLcv5pTK4IbKjpmA7x5/Xy0khd4ssNttpWQ9HfVvsDoDytGVHUvM1ofadSCtoIiSTJUoZSI015b9edlOmeNRWINZV5G0rH2Sf4txlHnPS1OGTK+yPK3My6k2jr6JbdZW1dc3SMss9h2qnIT7OpknWPM2gfDt3qrMa0d00zrK4rUI7VAELSkznBAJ1ACtfnZ2OYgvHEeMF3XcDKKmmo6MIcpnmwWgSoZu6d+WsEjXTezfxdTVt3Xi5idvLRVhWCWko7MNKCP4QOQI5TuN7FkqA6QN6ITIMrMysxIQSpzTNuCPzg7ee1kqura3M9QUTjQeKc6Co65dR8otFGCOMVbfj7d24jdS28mSXUJAD7eypT/MBrp/LtZyXziWhorwarrqWqpWzqsZSQtOkjSZ1+467i1wzNIuqYiN9UnY14gLTVfutSG+1bb7+QEnLl5yJBnTTl1myNc+J6Jy7mBVQHIepS4rQhtfsE7SAREnXWzRxFUqvW+qq9A2XjUAx2h76UnyjbbrqLJ2d0NBzsysICSlREKzcwTtMz7o2tTdISbq01uiXroQxUVgNQouq7ZGUwDnSdNZ6aGenTeyxildwUNW0y48ilqEoUlKgmSdu8Fb7TodIJHMCzc7WjprnZrWQj1kLLayFGEEiN9xpr9GyFez66iriofZdVKUpWlRJAzTEDwjaRaObRSy6p8cEK5DPGHCyG4UF3glskCI7x2Hjrpp0629C8PN+rXbSU8AFDSW+6Omn15284eFFS5d/FDDAcKghV6UuZIVIyZ09OQIn3jxt6R3emClClAlKtYG5+63H9ov6jfou47MaxP+qpb6Uqn18a6ktsqWhpy7yswDlBSjWOWsDx8dLev1BPqTJ/9Wn36C3kd6TAW5xiv1ZnI0i7TIOhkIAnp3iPhy1t643WqbvpjG7SP/CLaUJ/7aP6LFqv9VJ9UOqSNyD0+NgFkDfWw6+u3vsA5CTMmJibHYqkm6KPE5SUzZOeJCjGkazPKyg+dCT9aWTniUqOb3C11ipO3QTrhQ2FlQ0Oxs8WWrvvIJRX0zTyBqlLiAoA2YV6LyUDqkaZUEaEb6/Gzeuzite1Cy1TrYpXltISC8oGVmBJIBAE+Atk4pFM97HRcls4U6MMeHqX6m57ru9SF0FC0yXQSrIInpZUuWpaep1oSrvIWUK5QbQ5WcWq2uhJp6VAAgKAOpmeulgUcWL8pyX6ZFKjMgIgNzoCYOusyTapNFPJGBzV8GMNsFNd7UBrLqrKbOUl5hxEjlKSJtVo4Pbu91utVedQsMLaecBEJCkKBI15aa2fo4uYweQWl1NN39IQ0n62szcTXhU1V2V3eUkOU7mZIG/dPTzsfDWvpY3sfzQJWZyHHku+MfCrEHFPFLd/4QraStusUYpHlIcQoB1ClHQ5t+8LRlhDgBi1jHlW3TPH1lmiqaVZW0W2/tG8hKVmUqIzgwPG0o4dxFeN00C7suyscpqUqkNNGE6gAkH3WUrprKt6822zVOlS1GVZzJlPz1tWjNVAzhtItyVl8EM13OCiGj4PYhw9fFLSP3i0XbgvIvVAiQtKwlcJHx+tLSPj55arvuZwCezvajVtzKo29/zshKvK9r6vm9GRTOl9KkpcyZipRSVJ133AHw0ssXtSX7eNzXfdzGH7yW+iop1rcLUJCUlKlGTB2PTpbncbZK6Rk0h1Gmi08P4LGOYxSBSqZp/SDuRmjBLdRddQgyddW0HSd9bSiH01dPWoomchd7GEDmQp1JP/ACWjVWH78XxlwriJikKqGmpnGHl9ohORSmgdZI0EGfMdbPbD96rp3GEKbLiKpx5sOZkgI7N11UHXmFC0KGbJAL9VmV8YfJp0UY49uTE7l7GtorsqKhTeKMN141T/AHaGcjitSPZjz87P3EN1XnW4Oorv9RUmopL6ZqchWB3PWnVTIO2UTYHHF93mwm7627rsadp6h9g1LinsuTIe7yPU2PVWOLpfVWpq6htAWhtKcn2moU500mFTaD8V4TiCfomiw5zg1zRf+BVx4533Q4fxpfDroLiKmudWgtgEkZp357je0S1eJXLpoH0N0CqhFdf6yAV5ezC6NhyTod491nfx8bqryvtxNyrccZbeKiVgCRA3H1vaPL5o36i5+wFOVuN3tQqUBEpzUWU6+bZtvYdM6ogEjhuq1VEIZDGOSb2JMYV1NT1BoWksuFKnEhcKElI2MdR05+GonBzEFXjd+8aS9lIS5QXS7XNOtpCe0eSpMSD/AAwdh4WCvnC9XeDislFVS02EfZvAJnXT2T9352C9HfCl8IxGhbjnqtPX0FQyVDvLKCYIhQiZSRPX4W0q0uZRPew2ICp0wDqprXbEqer6u6kuzBWE6ykzD1i82ah5RUT9opMEidBbeI+6pS0uyQZjYgdfhY7fl3preGN3sqcdSbuelDiCJCmlgDl+H6tzEN21KlLcXfdcsETBWmD8BbzaneZnEvOtyuykZw22aEo3BWOKd7k6J0kTOnz5fP3gY1vJ1y73W3E50ZQQG0gKQoTr1O/yFkXD93L9YQpdVXENnMiX1CDOugjoPrSx7GNSF0aUqUo5G1gBP8QJGubwk26CkiBKz55Dw7JisXvVqvMMGtdKEFSUjMkQlIICDpqJgzpYd696p7MguZTBk+6IB843/OxBwA3yQpaQkIJT706g/P424chuUjdMqlQ0jntt/X39XHC0hYgmdshHLxqYUpVQEwJJSqCTp7+XjytFWNKb1atafZbgLp0wmdSozJ2+fv01tJRgKPfMHRI3k6D6/rEf8QW15aEpJgJUkhM5dFQdSANo6bnxtfpmhrrBVKhxc3VJFMEIpmm21LkAAaxBPOY036xaK8XpbcxZWGVZVJbhSVFMSAd4EwI1E6AgRNpbaC2mmUqHdWghJExsNTtJ1+YtE+MwhGKKhIyolCJWI0ievgTvpqCTBtek2VJu6Qn2XO2Cex1EKgxMhOwlJkRHlJ9qNL9ei204zw+u9haCjJSONgT/ACs0qyI5d5S9DO3utQ9m6a687xZu2ha7SqqXfV2G0albqyUoGpGpKkADf8PSXhthFOEq2tw22nOi7r2q6EGCkqSGqhE/BlPht4Wz5KhrJWxk6lW2RF8bnDknu3lKMxKVSnbnO2u/L8h4vJFFSX7hp26a5rNT19Kad4RuFIjU78z8BrZnobAIyNJI1SZERp/QbeXQvK5HAq7KcK5JKSJ5Ax/UxbUh1WZICF58XfddZgvEd+YOqmlF26K0sFDumZqe4oAaGUxr5+NlleJ7rZayVzTiUBQlcCEyOfIa8vxOsm+lTge7bvxvdWMX2Cilv9tV3VjidCl5JKm1GOpJET+NoXqLrFCtNLfa1qQAEtPgZUkRPeG0iB8NbHFwmFk6ruqKGtYUKJxCg4ggrI74zHfSSdRy8LLvo6X5U3TxONzuvlabxYfoHpOuZCS6hRnUx2agPPxtFfqNTh50Xjd7yXqZRUVBObKRm/VXxEQLPHhverN3cWcP3wiE09bUtZikwJWOz+ROvny516gE2JVmAgXarFcT0lm86N5wlPa06kAdcqv/AM6zKeKs2ZRggRBO0HlI8BrtaQ+KDJW1dlQQDlU6gnXnlP4ffaPygwcq16HloCY+Ox+QtxuI+Gpcu3wo56RqKPaHRJzBWsA68p21On324CJ1UFRyjUkmNLGHUACCqVbkHefjr5/1tgbCSE95JMAg/Dn4H6Nq7XK4Qi6EQUiI1yzzPhHx+fvhvizSJc4JYkpkpK/3fexUkDU/+cJB+Sj9TaaQjvJ7qQSZM8vrX62jfG91GrwTxIuzRSktmqhWsS0F/enz0saJ9pAfNVqhmaMjyKoFiFtSK1xSgqVHMcw28PrwslAHefdZbxI1DyFlSdUaRzg/rZEgpIJ687dvEbsC4CQWcVfn0Nr0/eHBZimMKN3XlU04E/5Xf/ebeVp3q0BTi0lByqhRPnOuvl9C1YPQJvAv4YxXc+4pa6mqYB1HatqT/wC6tad5pQUkgBQKEkwNenwi3GYlaOqePP8Add9hX6lIw+SbrlMhJWooJSOZB1Ov19RYq6CO8Rv7RA2ssPMdmrKpKjBM6AHQn7+nn7yzjKkJyrJGx16dfL8LVA7RXS3qk1TRSSQYiBOUnx/GwLrZKQRqUiFJn3Rt1/Cx5bSoKAICTGUHw/LW3DrZUFDKESQAcsmf6n+vJZlAtSZ6uS4A5Ble400PMfXwtAfpa3IFYcuS8YUr1avcppSkzDzRny/ugPo2sT2aQoK1HMHcHmD4Wi30m7oVePC28qwA9pd9TTVaIO0OBJP/AArVa1QyhtSw+ao18OelePJUKYJQ6I3UCn4iOdp19EqtCcR35dhVHb0Tb4SZhRbXH3OG0G1QDNc6EiAh0keAnS0qejHW+pcWaej/APTaaop+uyc//Rbrq4ZqZ/0XF4e7JVM+qupgG8U3JjvDt7Z4FLedOtROpyh1M+Wk/fb0YXvChzNvNLsnEEFsAKBlPWdNQenj+tvR65q9N63LQXmmCmspWXxB3zpCp+duZoHXuF1GINtYo3JAIBI8baOUGSZg21oNTz5Dba2lanTxnztqBZRWiTEJi2oAgSdd7Yo6ae6bcEgezzHWxGqJSBiSnz0z6EjVbagPONLJLbwdbQ6lI7wCvlNnJejeZrMNdLNC7yPU22809kS2PJJI/C2lAbtWbM2zkcSsRqQR4W2VJTy36CwQWJ31md7aU4RBkE8tbWEA6IUuGdBrPPY24SogjLsI0sGXB/Fy6flbErJOunIeHnaQUCjGYSJ08bKF2Ll0EyD5WSg5qZ8rH7tX9qFE6zBnpFnSTlzeHzNssW7cnULV8Ytln0SXhtToPbdo0ohfZgRP8MqmPl7/ACsl3lWu094KDLigVEhIGggbxJ5DQfGbHlBDVQU5kqcKUicxJ/i0A8Bz+YmyHenapqyW2ySrUInTUbn63nebZLTd6su0YnJwCcbViy8nFKzFdJI7uYzmHx3g+fnZxcfblo7vu677yacf7esqXAtKl5kju6x02Gg+divo7Ubjd/X04R9iKZCSojmpUxrry+6yl6StTFDcCASmXXlKABmQE6jlztQe93tENG3/AAtBjWmhLioSoVLaq2HE9popBUYgDw8f00s9VXkWVpbWVpSQFpAOUK0Gh5kD8POzFpXXHXAylYUCCkyJA5j693msNorKgpyrygnUExJ8Ou/LXX3HZKyLXR+sfL6hUR33BqUkpTI2GvmfrWxVWoBb7yIzJkwBAPL4a/1sI25Rs0q2nHnO2S3lQotiJJ0Opk7CxFSlpUpS5kJCYBmTED89OQHuhdTDbIwe0RdrhS4stqdBykQNtY8do6jSxZtinS0CzVla0lIHcMHUGROxBnwiLH2altFw3i0pJDgdZyJHQEgxZz8EsM3NizEVbTX3SLep26PtUtpcUgZgtI1ymdj1IO9k52VtypRszuDQieCKRf8ApGwwUZVBN6UygpLk/wC0TOu4OpMHfXwt6W0yCH8gP8RBA5T4/X4WqtdfDnCNx1Ld4XRc/ZPMLS62ovLUc41ESo7ETaSKbHOKFVBm9VkKXJUEpGvPlHn525vFqV1c5pYbWXWYNUNoGua8XuoQ9KdxLPF++EFZyuNXaQiIChCQTOnWfdy5+uFzqP7tpTzLKJ5/wi3jF6Qt511dxMTU1tV6w4pdKlRJ1UAgQPPp529mLkcz3XRqECadskDxQLXWxmOBjTyCx5ZBLUSPHMo+rWRpr8rAr7o8Ok2FnSAJ8LAOGRqN9NbFjQZEUqFGCOY0+vjZLqVD8IspVE5SdxEWSqhQzd4kRpubXY9AqTt0n3oo+pPpAPsH7rQ+h0lITPLMNeVpRxHXpoqBS1ICpWlEGBuQLRNKgfsxGhO3142r1R1Cv4fsUdcccQcxMyIkjcHmPrewjLzimgAsnQRB0J5D7/hYFplxSA6dErEAzpp0svO3Cmjuakvc3jd7oqVFHqzTpXUNQN1pIgDfYncWqFwG61W2SahzKiCvNzBzbfptYO9Q47dtZlMDsHDrJjuWPCmQSomdR0Bjp+Nua1DblBVtoVlHYrBPuPT+thlymRoi1yqMLjVICVEk66izkuZIN7U6kdwJUXCCZ0G4n42bFwLbDUZs32bevIyD8LLDLrjJS+0ohSZgzqDtYLnIrBcI6w83QY4vp4QhJbaVPQZ3Nfus6arENHT3ezX07qFMpppWlImDlSPwFoxvVxSr6cedJUpymRmV4gkGfj99lJKj/ZJ4TCvVleMQJi3M4/8A0g4dVdw+IGQgqSq/Hl20artRU0j77hUpEpUEpyuMqSQZ15C0ccPsc4gdua86FVcrs6a9qppogSoJBB3InmfrSwmIFqULodTAipZKj7lD8vhZr8PVZP7RU6yqU3y8QCd5Sg/jbnWyvNM4XV10DGSDRPl2vqak/wCsvrUJmFK9+g5W7p3YYdIQQMyDodjrZNaWVAhASTJGxgHSxqnUBS1Ikx3DBGg70e/e2TLdw1WhDYHRRlxGUkXy6twEQpsoG4Pd+vnZl3kHm1VhfbLZ9aup5zTUSXUSnl0+PlLo4mlRvolIASWk7nQQd/62ZF9EG4Kp6A4ct3LKkqGsVKwN+ma3pmDEChYuPxJpNU8pRqmb+XeVQi5KtppbnZlYcUmDmCwD0Jifo2S+Da6qjxLQU1Y4e0acq2AAQYTnXtHKR87aIq26xaxTqWCykqlGmUKMHpsT+Z5FeF9YXMZMrS1l/wC0H2zrIzaz7vztp1MjX0kjPI/ss6KJzahjz1CnKq+0wFXtR3W6ipBG8aqj7jv0s27/AINMlSUpgpnMTvpsdPLpz87L6HO1wnfzAUD2dY7KSIiUn8x8fKzUvSpSu72FwlwqaQRJ2gDX4fC3m1K20h+q7Oc+FEbrdWiobT2mx25dDYXFwzXUah0pAMJUBoSJ2Hwsm0a0+uSkZioDbYGeYO4so40dyYfnLmUmSMoiBy8utupoG+KxWLMTkJTDcP8A25ToQhQJQ0mI2OQb/rbhxwZkyAqQBMxtrz3ttlxJxBdjgXJeYAhQkEAKTy/yi23EJQokmDPMyd+cW6dgsAsflqgCnXUgFWpJJjfrz3/CzSx6wHbqpQgFSjVlJA0kEKOvTX7x1s7VIVCdhAnVJn68bI2JKJFVRphCcqKyVDf2kxv4kp+FrcRs4IEgBCZ1e83S19LTtrjK2FEb6kdeoET5+M2iDHhdbxG8nLoUoVmgzlEpGgMjY+EdNZka/albmK1IAStLRyyQYAAG3vsweJNODfs/xFhCglOYKhRXrmAjkZAP6XpPdVNu6kf0PLsw1ePpD4OVid1pNBRmqvL/AFhxLaEVFOw48xmnUQ602YmDAE6xa/l2ss/2pvp5t9p5k4gbWlbawUkO5Z1iP/nJET5m3nH6P9MmuxZRvnuOUdHULCVGMwy5BvvPbTIHIWu1gXFfbqvChXlQqnqKKqSUp9kfYI2O393qPo85W0TjVNqw7YWt91r0k7QwxEbqTbwp+wyrbT3wqVFSiRPhOx3+t1jDzyl3cO8cyVLCiQBrM7ddRv42L37SuGhU4gpTACp9/LXx8PlYLDDx7BxsqIUl5Wk9QNduv1pNt6hkzBZlfHwym96QeEjjDhdezLDWaru1k19MOZLfeIka6p5c4i1U7sq3LzuxqqCkuO5EoWh0DvEAjyG3LbytetS21NkKKS2qJS4AQQRJBH1obUkxPhRzh7ja+MLk5EUb3a0bhRJXTOd9oH+Y5YB/xJMaW0gs4bJBXedBTuOU9XdymAFlLiuzKg31GmoH5xZHqaG8KWtpKy45cu9tZqWXkLBLK5BiCZyyJjw2s6y/SV+ZvtGfWhqhsHNJG8E7gz9aWal54aVRPKqWS5ShRAV2JhPgY8RpEWhKzOEWN2Q3Kt7f15t4jwLc1+tKT/rIaeBCtDmQZ+JFmaGFJSUhtWih4ieZ6cvv6W64VXw3fXB5u7lJKn7oqUsuyqIGeQRHgo/WljrrCCdjlgSkjUc/ONNrcLjX6dT9l32A+Ol06pPcazJyJGgjUmJ8umszt+NuAyUnWZMmeUjTfx91lJVIe/CSZOoA85+j08LYmmIJbVCcwJEk7z05fpbObItd0d0nFlQMhsAgbETzP17turQr7s9bvrFVzutApvG5wQAdT3FNny5fpZ+dkTIypzg6nWZ5e/WzbrGjT8QqMkAJrbrdaOb/AALBgctlT+liCSxugyRaLzRxQyWi0SFqWkrQSUwCfqfP42biSM2qrP7ijdq7sv29bvISBTXk+wU8wErUkbT05mzAJ/pbv6V2aIFebVTckpCtb6Ad4BOLMVXRrFTd7FT1gtu5Zj/62121s52wQAZBHKN9flbyg4f8QL44eV9Re1xXneNDVvs9gXaKoDKi2SCUklKtJSk7crPWp9JfiJUQHcT4nUiDAN+upJk/4Ep+VsXEcIlq6gysOi6DC8bio6YRPGoXozU0y85WUwqdQr66/Wtij7PY/arypB3JGUEzprbzWqOOWNKpZcevK83TO7t8Viv/AHosj1vEi+a8RVUtO9rIL7jz0f8AG4q1duAS83Ky/tLDyYvSupvS46VIU/fNDT6QS7VISB8TptZKq8dYBowUVeNbhYiUkLvJlO2n83X66eddXje713GxTUeH2Wb0SuXqpSGFtqTroGy1I5bqOtks40xBMtv07es9ykZT/wBNit7Pnm5Af2kA2YvRB/i3wqZyqd4hXDmA7wFe2qPckzZjcVuKXC2/sAX/AHBQ40oqmqrbsebp2WULWVu5CUJEJ3mN9Nd7UlGMsUBGRF9VCADPcVl193nYJeKcSL1Vf1frp/5wv87WIsDbG8Pzbfzoq0vaF0jCzLui16ILdYsFKhMKhQ12s8ODFci7+LOGanQJcr2mjJ/7zuHb/NZjvPPVDhdfdW4tW6lqJJ95spXFeK7ov26r3QrvUdQy+NIgoXP4W2pWZoizyWDC/LM1/mF6Mu02ZClJOsRJGgPu+t7Xg4LXsm+OFmHajtSotUYpD1lolvX/AIbU3NEh4Z0rCklIIUn+U6g/Py1taT0aaxT/AA4VRL3orwfbAGwCsrn3rVbiMPk/WLV3mJRfohyleQZVOswbaXA5HT7raKwDPQ8utuToOfibbwNlzxCwkc7cz3tVfrbZidJMTYMkphWliNKgUHUp7RoyIB982YiT2NZXU8Zcr+YCYEKAP3zZ/GeXdmzFvlBpcQvI2D7CFjT+UqBPzFrtOdbKpUt0uuSsT5bRbCsjUfxHTf6+jYuVQoiO9PP47212hKj3TrFr40CziTzRgLkxPObbCpkGR52Lhz2iTIB/O2docsCNefIWYJIyl3vROx5iOdjt3KQFhUgzpFkrPBJ02JMn8bHbvcJcmfHW0kk4gWo1Qj/hNssT7Y81GfP9LZZJLxJbShx1JGSEpRIPNZGumnnJ62bV89yoAUtJlSiDlMefnqZjpZ63E80KlxL1zIqiWkBvNKshCNB3dRvGvSzNvgMP3wrs8jas8DtDlAJM8xtrEx91spnvq04eBFKC+7wupLwu+scbK0pzAKgSkzm035n3CxW+sSXxfxR+9LzfqAyT2aHHVKCQd45chtZT/c4p7tdr3XkunIowkBSMoHL5a/1snYeo2K6vU3VoWUpSe6kgT4a6Ry958rWLNJzWQbuAy3QF3FCXD2iQVESkpgR5+74QdrLKVJUITUNpCEqJka+cxrZMokssXv2N4JBaT2mZJOhACiNt+Xy1sUefcIIWo5hy5DroP05b2kRdII6muLhLSFEpTKYGw3126A+dh/3felQhVSxTOLGhzhJJ0HMgfXU2IZEC71OhgjMYTz1nWOQEiB1087LeD6p9mrSz6yOzfaX3PaPdBIMHy+doW6KXJCUV2VIuS9XKqlXTuoDQQtwHWTqOh2nnZ+ejEioVxbui6C0H2b4cFFUtkqnIVicsHQyBy/KyDVsXtXUblFKEKbbShtZhISU9IE/HWBp4yZ6Il1Vdx8fsJ1FYtotpvBlSggkkAKkjY7wPz52pYo8x0cjgbWB1VzDh/wB1HpfUadVde8uBV2Xa+Gaq7rwoXCgKLTxKCUk6HUAxvZEPBmnrsU4cw3czvqb181RpA68VLShWWQTp4G1sMfuUeMMRm87vzqZDKG0lSCNt9PfaJeJiavA1fhTFtIgKdu29kvpaOgchCiUz4gEc97eN0ONVLsVFO2YvZcgG+hXqMppZ8NzGJrJSOmxXnp6aXDi8+FnHWqwdeNUxWv0hoXg4ylRQtLjKVCJ1G5BB3+/1pw+ofuahIE/6s1zn+EW8p/T54hL4m8fa3Fa7tTd6Q1RsBie1VlQ0BJJAAJ18reqWG1RcN3yFf+as79OzG9vXic0TSvMg1zXuDt+aVnVg6D5dLAqI8QbcrcMxynrYFaySRMWkxM8armpV3CkdJ1HSyPUqIVAEFPysovLTBnTUeBsl1RAzEDW12MKi/dIWIKf1qiIBAhxCpUdNCDaL2EoVGcQCN8vlr9fO0jYnrPV6VALaVBbyEAHkCRr9eFo9pUutOB0t5pGh3kf1tUrCGkXWhhoLmmyO05GVLKQQc4GYnXX6FjzFM+KlFKkoK3FJy/agATy6AfL3CxJgrWorICNgByj32N0TgaWQHgYJ0kc/K1FzxbdarWuulO/7jvbDV4uXZeHZOOJShYUy4HEFKkhSSCPP60NkqobfqmXUobVJbUI8YNjoq6hxtTDDKio7QkmPrWwoum9HmVLZpKrMpBBKWVHceAsDjR8yEcMdbVIuHrveQw0lTZbBZbk+Q/X7rOu5MP0d4vPU9ffjF2IbZK0uOtlYcVIhIA2Pn02Nke4MN32/TU7q6avP2CBm9XVqYH62Muqu273HEVF50bK2yUrS9WNoUDMEEKMgzPlYZka7QFTa1wbrokS9FNt3m2M8EsqAI0kBX6iw7UO4aqG1GT2LonmNCPOyRfuILjor5ZqHb+ursGad1LrjdcysI1QQDlUTJ1IToTBgGCbF/wDSRwvaw5WrqOI9yM1YaeU3SKdzLUrXKCAeZjlztj4xC+WAZATr0VuilZHIc7gE4b+dKrsuh4JWPtKUn/iGtm3ggOKvTE1OJBF55xoRllCfft+W9nVfxQrDVx1ly1CaxippqR9DoQUyhQBB1PL8bM6kvG9rtvm+zdxp1OOVKFLKyAIyR+FsKmpDJC5p/mqvzSDOCnmlt5AJyKKIkzuP6zY5RUL1RSVzil5S2ylwJGv+0SP+q0VY5xRxDwi6husfYQH0B1tLSWlHL0IOu0dbN65OOV+XI5XVmI311tK5SqaDLKmEqzykgwFTysX/AKenkjzssVU9rRRPyO0Tox9QuOXxTuLGqm0gSNTr/SzPxV/2XdV55cpKKKncCjpIRVoP/V8DtZKvrj1dN53k3U1WHqzsEo7PIKltCyudN0kRqevL3iO4ho8Y3BU11NRO0jNRd1ewGy6l1WZtxhU5gACNZ26a26mhopKenayQWssipqop5XFhusxTj5Vx3+miSFLF4MJpYBEQuCN+UT9CyTw3PZX88862EuG/DkSnUpSQkqMzp7XwmzjS3dKHGEFvtMzJV9otIM93WSFfdZIwq7SUWIrzcdbQyo16VIWupSmJbQZkogjx2kRrbS4DeE4W5KjxnGQa6XUv09Uhu7MRMOOIQfWUKTmIT7SBtOh/OzXdbcq7oYTTqbUvsEJKc6R3oEjfl+JstVV94GpmLxpqi9roSy+8pxIqLwRJzNhJgjLPPYWR0cScAXXddLRqxLcNMQgIQtNaSox0lcHQTNuJjw+UPJa0rqHVERaLuHqkekoa5FeCtATPtEyd99uUdJs4cS3S7V3ItlS0EKSOoPz23087N+o4rYHdWFnGVM62ZXmStS0wn2tUjoJ05DlYzenGG4Kijauiiv5au0CUJT2KkhR0PIagzGvjbbpaeZrgcpWbNLAGkZgmo7Shq65UELcQkoS4ESUp7RWk7jy8bBO01S8ciKOoXuYQhR5jw/H4WLXxiapu67luNVtUwh4uaNrUJggnQEfzaT4+Nm9XY0rEUxfCbxqkhJKW0Fbi1wNkJzySSdhbfaHbBZBcy2qdf7pvBaNLnqxlAMinVp4gjxn5b6WLVV2VyKZ9x+kXlYyrmBKOU685HMTr42IXthvjLd1DT3m/wYxE5TV1V6oyS/Rgl3IpUKHbkoASg95WUap/mTmJ0VxcXu3eeq+FrdBQMEKq6moxHdmZluJKg0l4rWe8YSkEkiBJ0tZjhmvchV3TRWsCojrHSvE1SkupPeIKdJAzRvv9bbWbeOkF2/G1KdCCKJnN0iVZTBBG86/fFlanqQb4vV4rlOdScs65dTE6gCBPXfbQFJxasm/EuIbIQKdhBOUd2UEjYHSfLzOlrsujFUjN3Jwej9CceN0RqG6cKp6oKU7IiAT3oB1OXpufOLM4arqejxHWsovFkrdoElBSXMqsgdVEkCCCkHWPZ+NXOGVQaXHdO8w+EF1l8pJCRP2e8mRsTtpoDrraacM3ktvH11tOOhYrqd5gAgjvKSoDWeYUI8uU2qGPO25RhLwnWVvb4xmVti6UUqcgbSguEnMRlTrH4xy+AmEapDi6hSQApCkkmIGoifr7osxTVIeTS1AM9pTMrnMBMoTp57n6NnJgysQF1TYRlJSDMHKTP6nbla/BE2P3VSnqHzHxJ9oeMZzIO0gwBvzPT8BavfpS3Kmnrrgxi1lQHELu2pUE8hLjRJHIZnRr/N8Z29ZSJCF6lWsj3/nFo74+3Z++eGd5paRLlEpFagp1goOoieh+uVpAG6qXe12sVawqmcVS1EaKRPfPP7t+eo0sni/sZYZa/wC2WG75u7ZS0GKhtG0xML2894mLOGhbpqgpadbHaglSeRI6ePnvYW87vfVTeriidfSvvJypzECOfMxMafqXHkpJ/cCcS3bUVN4XZRVIXR3xSds0UmSlxCk6EHYkQAD06zaXBdy1DOSRpmlQM7ePmP1tTHB9deuD8WC8qVxTK6KozOsQoJWhR1PwJ1O3vtKt9en7w8w/V1F2PYIxC7VsSHUqXStJzc4JcUfKRNuMx+iqaqdroG30Xa9nq+mpYHMndbVT2LuIGQJggxO493uHy5WaGO8QvYYqaZDFMFl1srClAAGCNPEwLQdV/tJbtaKv3bwukcu3vkJM+SGFzv1sy8W+n1eOIXm3f9H2HZaSpKS7UPumCRpJaTGw5WyYsFxIm5Zp9QtiXHcOAsJPwVNq+KN6dt2bNG0dAQDMDfwgmwNNi69r5xPc9QxR0xqEIfbbLtSmnRK49pxyEjQc+ZjzrbX+mtf9SkAYCwuSkHLnpnF5SZndQ+jZPc9M7GGdK2cF4RTCgdbsVt0/vBa2MFr3aFoH3VF2O0Pzk/ZNHjk4qoxziVwpQhx283nXG0LzpSpa5gHmNdCNNBaKVgpMEROsRZz44x/W49xBWYivWipqd6tKCtmjbDTKcqQnup1iconXcmzYdWVrJTMTp5W7KkidDC1j9wFxNZK2WZz2bXXEkn9LZJtsDwNukMur9htR8hNrVwFVAJ2QfKJtllWmwtiWuKfUsPXlUZtuypXFz8BZbpeDvFiuT2lHw0xQ8nqi6XyPjltEysG5CmIZHbNKZ5tlpFZ9HfjY8ATw3vlmf/SGex/8ZFlam9FTjfUkZsMUNOOtRflA1HuU9PysJ1XTt3ePUIjaKodsw+iiS2WnOl9Dviq8tCau8cKUebm7fbS4/wDZZj+NlFj0NMVqhVTxIwShEEns36x1WngKeD7jYRxKkG8gRhhlW7/xlV7sICezSeijr8LTze3oj3zRNJ/d2Pbnr3ioBSRSVaEBJHtZuzJJ20y9elku+fRxxDT09IKK9E1jwSUPpRRuIDcE5SFKjMDsIEiNQLMMSpXbPSOHVTN2KYLk9MbBVFhu7aGrwzez1UxSMsvLFQwkKcQhIUr251IJEj7rW89Abj1c/F9vGl03fdT1Cq6FUNRDr4cLgdDqSRAER2aRudxbzYZ9GnGjzq0LrqBhAPcLpczKHIwlJA5c+trZ+gDhtXAPiPfVXizFd1/u2/rp9WVkDoKX0OpU3Eo1MFYjxtkmnw+JxfEfEtdtTiEoEco8K9LiBBBMdPK3BEkFU2baOJODHBrfraY3lpwf9NuzxAwhl1vtpMSPZX/8NiAhQIKXzJGWNPG2lHUkiROulm8riHg3VJxBTyN9FA/MW0OIGDlwRf8AS6kcjr8rFaQoWS+ZOhI8bMzGgTT3vdtSRHbIdYn3JVP/ACmx88RcFqOUYho5iIkzt5WaHEDiTgZd30dQ3iu7ZZrW0yXgkAElB1O0E78rHhdZ4QJm3Yhy6mZzdd9ibaU8MsgQAJ56+P42Z6+JeAp7mMbnVzPZ1aF/cdbAL4r4GbGX9+hYEqPY0zzmkxplSZtqBwssstN09i6rKYIk6aibYHV7/GZtH7vGLCKE50G9HQRmHZ3dUQQPNAEe+xN/jXcwSVMYcxJVka5WqJIJBPLMsfdZZ29UshUmIdklWY66QB+djdC6rPvpO4M2h5fGrKErYwRfaUknV9bDWUzz7552CTxxr0OpbYw1StKkgiovZtKiY6JSo/XnZcRvVNlKnr1hX/dzbLQOfSEqp790UKVcx+8FaH/2Vss/EYnyFecWG73prsrnFVlQ23IahM75Up1kcgNbN7Eibscqn30XYhROZye2V3lFR1A6GR8eU2WLpuJi+u2XUuVCXG+zA7MBUS2CZ8dPmd7GXMNUhfX2y1uBKYcJA1010Gn142zmtANwjuJLbJnUrwVhZ1fZqbydqhISnSYOniYPwnyshXRU1LDjjlMklcphQBOn3/XlaWUYXu9V2Jo2WVntVLXlRmTMgCfl7p8LN7DGH6Zq9HkdittxKCCHASJCtToYO/hpaecBR4ZTFaQ/WV+iFF1ZIG4JVOnOehtst1DAW28nIuSlQOneG8+P66WkGluSiOK1JNK2odqVEqOitAoQM3htvvFiFdRUbVa4h6kZLnalJzJBJUegJ8fy52YypxGmW29V9kA6oRlkTrrPv5nfw+JzD5q/3nSpacASlckjURGun4+VnOlttlKUM01PCQCSGkASNYk6/lB62cFz3u9dylKPqq0ue0SClQMKlOngB578tYcbyRBEDuUSYvphwlo0z5LitSlgke6PADkd7SRwVxjdGC+JOG8X3o1XPUdNUmoeSzSlTmUaEAaA7zG/SyVdlfc9Y6C664lxWhBWU5vgZ5bG1g+B/EC+LsxDdVw4forhpS66hAqnLoZcdKSdu0UCrmee4tkYxUPFK9uW4IN9baLYwqlD52kOsbi31XpFwsuW7cb4RuvGFFUrapb1pkVLLbzOVxKFCQFCdD4WH4o8BGOIdzUd2ovlFCKWo9YU4aftMwCCIAzDmRzsNw2xOGblZVUpr62qKYcdRTHIs9QE90e6zyfxfUdiQ1h6vfCkkEJyII03OdQtw3Z+HAhA2SRtpOW5RMQqMQiq3AO2Pl/deGXppXF/ZnizUUyHy+lYp1BxKN9FAbmeU/Dbe3q5ht1IuG7gND6mx/8Ag028qfTwvBD/ABuqAzSGlSGaXKha0lSAlKhymNj7/O3qVh11P7gutYWNaGn08ezTzt3NEXyUUT37kKvW5WVcgG10vF47lQ0PKwC3SRvy26aWA7aJUVkE2CU+lIJBMza7GFSedLrbqlEEkiB87JtUuZB10iQRYR6qOoB+f5+6xB+qSoKhRB56xra9GqLymvjZ1SKBASVZy+0BB/xg/ODaPF8VqClSpn1/DzawezCTQUxUFD/MfM2fmL1F6gRlT7DzaxHgr5crVSqOG2Nqm9qx5i87oZZdqXFN9rWEHIVHTKlJ8PeLUq+kiqbcVXcOqZIL5AprPG1i7kKzYioGAhPtNUlKiI/yoNkCp9J6qF1m+6CqxjXUJLQRUUdxVaULLpR2QSoU6Ukq7VvKATOYRpZiNcF8Q1zeStxPcDJMSQ68RvB/2ImdrOO8eH/Em9sDVeA7x4yXPRUlaw005UXbcznbgoXmzJUXUgEnOSAAJcVAACMtCLDMPB8S0ZMQrLeEIWo9I3HdS4UU/D3iw8YnM5QuU6cpkSC4tGmh6bWTneNnG69FqbujhHfThADYFfiqjp1KUrTVCnySIIOk72I/6Dr/AHFKcqvSYxMhayVqNDc1PTHVOUmUKBEjf42Nsej5h+sR/wBuceOJ1eUjLJvdTaQOaQnNtoDHha2KLDI/hVc1mIO529Es4Yf9IrEWFf7QXXceC6dph5d3KTU4hdrSXm1BJlTDZb1Xoe/IyqBE5AuvLt53tfF4V15VdV2K7yfNU4yw+0tCFqkqCVQoECYkHUczM2sfdvArhlctyi56XHeN3qRTrtQpkXi0ErdchTi1/ZHOpRSkkmSYBNspeBnBK72g21SXupAGVKXbwCRHMJyISP6Wm2Wji/pMKG9lVN/VcoHo3nkUF5tpfcUvNTOZyUGVd8SMqfjp7usfXyy47er9Q4+lt4p7NYLhB5HknQ6+O3SbXd4f4C9HGmxXX0+MrvdaucXch1PaVdW4VPpdEA9monYmOXnZ+P3B6ANM+updwg1WOjvKKmrwcnTotUbco6+NrcTGyNz3A+qozF7HZSL2Rz0VsE4f4pej7cddf97pp74uxs0NQGHy802lMKZ33IbW3JmCQYkEG0k1Ho18PXX1O0+LLypnnoLq2nWyVmImFJIHlZiYT40eiZwsoHbt4e4Mq7ppKlztHWKG7ktocWB7RzOCTrv4mx2s9MLhAynPR4LvZ/SQVtsJnX/MTbn58BvMXRvAB+q0o8TmyAHN+Fxf3oRYIxRXiqvji1iJSEApaQ2ikSUg6xmLZnWTtZEqf2efCzs0pZ4jYodMqBUamjSQDMQBTGYnn0HvMv8ApuYNbVF3cO6lQCZBXWIT9yDZPe9OELBFBgBhvkC7WqUPkkWsMwt7G5eNb6BV3Tve7NlufOyUL4/Z58Kv3c0qhxZfoq0tgqcU/TkFQj+ENA6gkb2QMbei3dWDr+wnw+wPU19VR37QXww7X1ie1RTvrabKc/ZpASIb209k22z6auI6usRSruC5qJtaoLi0vOhOmg9tOvnG9ka8fSsxpfF+Plhu6kO4eD9dTON0xhZFO4lUhSiDovad7F7iGnMZnH7JMllB90LVL+zvxIxTNPI4o3SF5NUroXhlkaj2zrtrZMT6C18XZTXhdruH8N33WvU7zTd5OZp7ZeYh2VpCgUk7D+YiYSkWGvj0weJC8L0153XjGjF4uPllylTdjAI/xQUkxGszGsaGJZ1T6VfHerckY1daSdYboqdvMfcjwI+tbrIQzd5P2CBd7twPyjCPQh4vmmYo0YCwAk05KkOu1LRQoqHflJplHU6iSY1EmbLTHoX8cHDTn93cLaQMoS3lyv8AdSkJiOzZTGoJIECzIqeP/HSs7Qrx1iASIhtwoAPhlA6GyU9xT4xXiSKjGOJXAUzlNe8fCdDtp87HbwhyJQ8khO4CnK5fQ84yUlyLuivxtg9IXVLqlONUdQ4U5lBQSmcsgQB3sxICQfZTAFD6CD13qZqL14pUqDTlJATdsAkebvMeW9q+V+L8bvq/7QxJeCgBr214Lg6c8xjxsRbxrRUz03ziqgayJkJevRoHNGk5lTGg8dbJ72W9z8qTIjzcrcJ9E7A1WpuivLiQFoQFANs09OSAQJ0UVxsOX5W096IvCSmpyKziDWtrElFQxT07LqCdlBUGFDkY5C1dMN4lbTTIvK78V3C626jOgU19Urjzic0CGkuFyJ11T+dlml4juVbZQKhS+zJHNMwddSQCPLfflbn6yjq5ps8L8jei1qd8LWZZDc/ZTnjPg9w6xRdlNc2IuOuO6iloK1q8aZNFX0dO43UoCsqwpqmzpIzEyFTt0Fo+vTgRwHp2H6g4p4j3vUsIW4x69iuqca7USpMp7oV3tSCCDPjZpnHt3sNntjVJcAGvZiNJ9qFGPnYgriLQh3su1aWTplW8WyN+qDGmpHKLPFSVgILpnFT4lK0WDAql0CEsXrXNF9anUuHPpJUqdgBt77JOJqpFTiB2idVDblM2oK3IITOnTc+8bRBsp36WrmxtX05ZkOuqjMqcxMgCPu8OUmS2ceN1z96M1FG4e19RQ+dcoSQpYieum2sb8tOjlbdtlixkBylT0e37voeJFEbyu+mq30M1QaRVMJdQrMytSjkUCDEaeY2MWsXe98pTWUj7F03XTt09ShRRTU7bJJ1BEhO0aeHTSLU24ZYrql4mu2qpUst1zVQGlB/MpuSFAylIkjX5HQTFp/r7/wAT1jYp2a65UkfaZfVHEBX8MFUEjUg7fwaWqCIk3VgygCym648UhN30zDyuzNPTpQDnUUqygDSE9Bz+Vlq68e0NPVlDy3HEpbOYyNDIMTIJO2w+Vq502M8ZUSECruRFWB3CaN5KsoiR7WTSYnbysoXVjqrZce/fFxX22jIChKaBbpJBB2RmhUHrHnaXElZsVFscL9wrIP8AFK4kZkoS8SBKRkIjTp08fumyVf3Eq77wu6tu8Xc86h5pbZ+02RGnLUaRaETjWpeSr1bBOKXs47uWhQiZ0ABWtMeI5aDTkC7i3EeSWOGuJVAOEZnDRpCvH+/5yOWnSy7xKVMU8I2TPrHKuiUUUDSu1Z+ybCU5u0ImAQOcT+HMWHouIVdSU/7vvu7F0r7gKEPAKUiSJBmSY2jXTxto1lW3eKk19zuXa+9Ck0ry0qVB0IBQSD066crKrDRq6IU1Td7eUjKlK/aIOhkeGkxy99tJhLm6Kg8ZXWUbsLqaXE7zlfVmqdqGg4XE6COURuJHmOe4szMacGb9xziFD2GXKFtfZTUduvsUp2gzBnf9BaR8cfuG4nKOloe0L6AXEyvNkQDvqO7OnhoI3gH8DVT4vlhDbjSfWQCQqQkRIymNtz59elGokdELs3V2BjJSGv2UOU3olY+djtL9uBtRMZfWXFEDXUwjQaWU6X0PMSuEJrMW3a2SrKQ00tyI33jUWt61hWqWjtV1rKQoZpCSoRpB5HXTXXnPiZ/stTlcKeUdZlLStOeknr5fxG2OcTqid/wtgYZS72VTqf0OWQlKqvHbpGpPZXcNvCXPqD0soseiFhgNgOYivl1ZJkpYaRtyAk2tMjDTCE51POkhQUQEpkkxA5/QHSLcJuRCAQpDnfkxMFOwA2jX63tA19UfiUvZ9KNmqrTfonYZaWVGsvd5IPsqLYkf7v52UKT0fsNU/ZB3DNO4GNQVuLOYwPaP8Xlt8bWabuWlChNKoKToSXOe/wCP1rY03ctAkqBaHtaEpKpg6+fh+dl3uodu4pxRU7dgFCV24FwywUzgy4E6ADLc9OpQjlJaM/ebPq6KG5acBpDZpRySw0hoDyCEAnb5+cPj9z0rawOzAGWYCYKj+f1rYdm62ATm7MA84AMeX1vFqspkk3JVmJjGbJvsNXIdDWXs5rpFYuNOWg6fcfeaXdOHHgC/SV7p3lSlL000knXpy99loUVMSFtvZeUjlGkdPrW3WRhAKu1Qe9BzGAB4/XO1R0DirrZmgJDTh3CiYb/ckhQkjsEpgaiNQLGG7luZuOxuhttJgE5ANPLz+7lZQ7ahbQSKxlMmVBOUFXnHh0sGuqusArVeCE8zlTqIB/KfjZhSOdqUu8sCLqo6RlIZao0pTqlMNhURp1Ej68LBOUza4LTB/l9kR8fcLGFXrdyYArFLSToAFd0jTSdP6+8l3b9oUd3I8SgmZIBjkZOoOtjNondEJ1a3qiz91B0LUhpRUnYaajruR+FijtwuqKciQDvGYjofhpt1sOrEqKdXaN0rqwoc3SBttsQPMfGwasXvhGVqna1AgBRMGfCJ6WMKF3IIBrWdUUXhN5z2Xm2wZASSrcz+njbTWAXg+ioTWUh7NQISEqMkeM9bCOYsr4zAhsrICYRMyPnufxsFUYlvFxMh9Y1IEo0O3h4/fY7aF26A+sbZPz+2XElCswvO7VJTpoyohI02Mxy38ffbSuI/ElCEhN53eVgzm7AamBpPLytHSr/vInKu8HSknNCVQPl1/CwDl4vKTPrfaSRCQdxvvP18rXmQOG5VIzMOwUkDipxJaUkE3OEEz9oyTpM8+nu3sIrjfi9kBVUxdUHMZCwnnOkp2AP1Fovdq86lAJBInQmdvMzsBrrysTdUUq0akpGwMR0+psdsPVAdKOQUqVnH7EQaCWKOgDhAClSFCdNu7tr42a198Y8V3pdtVRFqk7VxIKc9OlbIWFDQpyDoedmWsvAKQGgkCJ70fHT62sEITmQAFDMNAZG+nysdkQaq75CUZVxEx/kaDDd0UyhGjdAgADfnr0/WxR7H3EhSxlvlhhajrkom4g+YsCoA6KUo5pEZjIJkfhYFRSk6uKWknuxp7v08PjYGgVY3ujBxVjpwfaYsrBmH+ybQmOswNP112sVqLyxHUFPbYsv/ACohWRqrLZUImO75/dbgF0FKQoBMmdNvGfP77chLqHFxmk7FKfa06c5185s4N0yDU7W5i65e18vKByku3g6sHTmCojb7zrbVJQUS3FOuUBVJzSZPIHmeo/pbS1uFEIKswMa7gzBiLd0rr6HA4VqMEbkHNy/Dpz99nUeaV1UtMklPqdNoY9q2WD7cjTt6Yf5iqffbLJSUBYTpKpPrLaLpqHytwFPZtKXICEyrSZEzp8jY1U3NiJ50qZuGvymFaUjpk7b5dedkJ2/b0Cm6egvyoQ0RAp6eoUWwEgbBKvLl46WDapMR3stTbVLetToR3ad5yYBGmWQTsCJ8bUuIAbkhHLdLWKcbVz4mYJdXd1akDSFslsak/wA3ujTbrNkeow/edO6qtp2qRhwuKWQutaSVAmCIJBBj5iJ6KFBws4g3o+kXbw9xJWKCVK+zut5YzR1y7jbSZjWdrOSl9HfjdVDNS8JcUw1lzZ7rcbnyzAT0/O0DUxDd49U4he4eFpTHqrhfdf8AW275oGnyElYVXMkGCCYyqJGw018bdVtzUtc0n1jEN1IegpW4ha1FQ3juIIO206T8ZFovRR9Iy80hFPwiv1CETlLyW2gnXYZ1pMf16Q57k9Cv0laqkFK5w9do0O95zt71pkpXERKQudvHkdp1G6tpm7yD1Ck2mmcbZCoKqcL3aw32T2LKNZjOvs23jIIBGnZwIE6cjHmCbtLctOMovqocy91RbpjuFTOsRz+trOL/AGenpC3mpta13KylOUlNTeZB/wAQSpCFxJJ8tOYss0n7M7ixUECuxphqkbUDIbW8sg+A7ODvvpqBpaHtKiaNZAiCiqTsxVIbqKBltCk11cQkDKCyk89/7wGZidPnZw3NxavfCt60943TVVyamnWFsudi1lCkkwqFZs3vEDpa2NN+zGvpsJVePE670LUZhmjWoZv94iRJPQ2TMRfs6blwxQqr774yKPfhDFPcwCl66JTmeUDtzSRYL8Uw+UZHOuDysjxUVcw5mCx+qhwenhx9pmSzR4wvVthIlXZerIyzAGgpxGp/PpZv4g9Mfjpf6CzeGNb7fQhSsgNeWpO0DsUo8eukbb2ma4PRQ4SXSnNfbF9X+oxnTU1XYtKUZIIS0kHmNCSB42Wbl9GTgvdJXWN4McfQ4uQ3V1LjsRuEdBrB5bedqjThEJvHCPQKz3fEX6ufv5qnbH9suOOM7ow9QUia6/L4rGaKkU7UOEurWopQlS3VGEjMdZ2mOQt7SXJhvFdLc9BQruJ4uU1Kyy4ovNgFSEAGJUOYtVvhlwr4WYQ4gXBjFOEbpuZu6rxYrF1qlrZRTpQoHtCoqCQkAH2hFrbv+klwIYPe4s4ZJBBhu8EOH4JJNtyidBXMuPCAsqsjmpH2dqSgxh3E5VBuohX/ANK3+CrYrCOKlIn1Sn22L+vyBFir/pR8CpAZx9T1OYSn1ajqXfeMjZ6WAX6T3CpwD1SsvurB1HY3DWxy1lTQHS1zgUzd3flUjJO7QNR5WAsVPCJoWxvq+o/9FsHCy/XR9re1I3m6NqXr8R4WSXPSZwiof6phXF7+4GW6w3P/ALRxNgV+k3duUKpuG+LFZpyqd9TQFQfF+0g+lZ8Q9UwhqH/CUp3hwSq7xZDNViZsSQruUZ3Go3c6gG1ZscYGvfBmNrzw/S+sXimmWhaakpS2FlaAsjeABmjmbT0/6S95rSk0XC6sUVSP9Zvhhr/wBdqocbuH2NuK/EO9eJLV3Ul0M3t6u09Qt3gt5SFMtJQlRWEICswQTsIg87VqmWleAA7VWaWGpiNy0p1Iu+/SEj/U2l7BK69lB3B5q8/rY41h29alAXUX7czM6FK7yQqDz9mfoWiZjhBjmmQlKr+eGQZUZaxeg/Lb4fFRa4L4mqaJyrfvlLrlOB2jfrC1fZye/udtARGkg9Tanan+ZXCKgn3VI5wuttwCpxVc6Ecsjrq9IP8AKj8bCfuW4WiXKjiHd7Ujf1d8GB5pHL62tF44NPPAl6tQhURISSofEHx+Fke/eEF7oeU5ddTX1lSEStptZSlwRqUDYK0iOc/GYfT7XUeHPvZTU2xw4aSpD/FBpa1CMrVMCJ8QpwWBdvnhPSJUl/GV5vKQdS1TMEGOUdrM9P6C1Z14dShYBaqMqVBCwXFdomdIIInQmOs6RYw1hdCnVKepMhnvZ1SUj+EHqJ6RGp8bSAi6KNpFLl+474aM1Ieu3EN4IQtvs3C/SElW2VGRtKlTuc22h11Fo9vPixciVrTSUd81SUHKFhumbS4BzAceSYPjrrrEGxWnuO7kKk0KAf4wo5ZPgTM6E6EcuekdruG7lhJpmG8hjMkwVpM8pGh5QdN9RFpBzNgFAxPKTjxKp3iS1hy9FOJBVDtVSIEDxQ6uN+li7vEyscTKML0YiCBUXupOUxpORhX1O1l5i56NLKltsU4UCQlWbKtI0kKH5aanewrVJRNlTdRT0JSD3ihsJIHOQN9ht8LPnaOSXCd1TUVxBxSs5qTDtxIJMgfvCpe93/m6RzB/G3CMe46qSDSsXAjLCu5d9S8oeEKdQeXSz0TQXeQFt9kpCBJOSU+Gk6aA+/3ACqpGgpCXVNO51EZ0yABGsGJn4fCz8RvRLhO6pkvYp4iVSM6bzo2FEQFMXFoUxvKqhfjuOVg6W9OKDJNSxiOrQp1tSF9nQ0SISqApMqbXuCefLY8n0UNtj7GrKm82Ud4kaK3zcvCZ326YtN3kOIDLqEoSJUEqUknfcanSN+k72WcHknEVuaYwRjqpSttWJL33yryro2gCf8SKcR7jYBVyYp0m9b/cBUZUq93U9Ng2pJPhFpCUllaA2h1LkwSYKUg7zIEQPvBsUUulpVQXSwUju65dI1PTmT8bLOeQUTE0c0yH8PXu+Slyvr+zMEB29q5QCifF73e7xsWTw+bMO1VPRvISSYqC4/mMHUdotUc9OelpAVeF3lkOvusLGXRSklJjTYyZkT5/G3CL8uLtBMoTsC4nuxGpJEjl79dtLPndyS4bBzTGRw1p1EH1G5BK9k3dT5iNdNUmDPjpPlYdrBaaAksIp2ERKizRtInXllSIj3naN7PE3vQq7zb5dBUDBToNDrJAkydPZHzkmu+Wm1JUgVaFH7Pvagxz3gQAddN+tnDnkJZWJCbw29TAJVfFfKoltuoPenkIIM6kRH52DXhRJKfWLwvFQX9p9pUq+ZnqOnMbQBZWXftRHbJSHz7MklKzroBMgnbXYR11sD++CSta6aoZW0Ncq8xSRz5wIB5nppzcZioEsCSncF0SnEreS4pYO+dR7x8NYI0Mc9ekW03hWhbC0JpAk94lJSSCkEEnczrzAIGtlJV5vOIWpCG3mykthOvtGQDI56fLwsXVV1jTZbZLK1ZSpKVNnQ6A7nYGOg231tINclmaNkwOJjBpryYrGGkDOhIzCEiBy+MmAdPLZIvqvURdFWhxWRbCmVKnTRYnfWe/8jPi7cYNO3wzSBCe0U2tSHFIQUwcoM7bkGI+in/6Nq+/rvy0VQhlxh9KgCNNRBHloOg02BsYtzNVfPlcmngfs7j4n3RWuUyX6f1hD5S4S2VpTm6+HUcx1tauj4i4RQM9XdTlKVgFJ9XQ4gzOwSoq2B3Gs6QbVWw4067fl3VTS1NVFAp1KylWVUFtXQydQIA5E+QerlVeLjy00ynV5CV5e3JWEiACQRO+kDSdRIJIrmLiI7ZuHsrP3XjnB1WhIZxDQU7RkZXkGmgiJMupHjqNJ18bOWlFDeQDtBX09Qkpkpb7N0AwRMp3G+418TtTF5+pUwXKt1LaHgMgeQEAhOg7pTBAHxB5wMum6ZapyJCnHSrs3GYCgYTqNATHdOnTnpAnUAJuHIza+27VdZN1A5JS2VA5VQlSZ1/wjl5jSdNLdKu9AOZypUkKVJCHACAVcp8lD3adLVAo7w4gtIU5dl9XwOzSM6E132aNDIyFW+3IiAIibLNPxB4m3WpNO9WULiFOe1VNZQrQj/ZpH825Mkk66pFoGiI5ogrmnkpl4jYeYqlU9VSvpFSiYzKSpQJgaeACR+PKzQa9YRlSpalPnuqQ2iQ3G+o03G8czvtZCo+KVbeakM112UruZc9uzXtJKQVCVFrMTHMQZ5cps5GKhupWVs5Ql6BKBqecCNOY/W2jTtyMylZ87g5+ZMa97lqXA5eNW12inHSvMUZlZUmBqRy2033625u6pXQv01UhnMqnUkkkGMydSNtOfWzrqKO8bzZe+0QaMKI7Rbm4EzEjUb2Zj6YfCCS4lKsgJOveJSojoTAn42BIy2hVmN3MKzWGsVXVV3QyFU7ilsogxkGh0CpmdgDz0B3spqxLSnNlpnFAnUKKYHnA00jYfHnCXDm+lCmNCXRkTISknvTPdnfl93xfPrpglTSBzCijuidDJ/p4TvaiaRm6vCsdbROtzFCASn93NgEc3BMAHUmBpJ/LbQsb/qHTnTTsJjXQkySNx4STy5WbqUuqzkJdgHVSRB93XlqND4cukKXklp+QCdMugnkep8Y5edmFKwJd5eUtuX/WlSiWkIIhIOhjXfU7mLF1XxeRVC6ooAJMBrSJ69dB8DZNQt9KZcQ2oKI2Okaajl9c97bUakgpLaTr3VJXJHl0P62mKdvRRNQ5GV3nWLzJU+ohRIgaKHhNgV1tSvKsOVZyynQnT7rBdmsKynMhJH8SRA6n77dlIH8KlJ2griZBmz8FvROJndUF6w+pJU6p9SojvqO/uNgCtbye1kFJAEFU6axzscLLKzk7IjWAUrIA6nU6b25XTEkykzoQT3h5SOf1rZxEByUeKSk5ZUqAqphZ/lVIJidtpP1vbaXHm0jsy4Qf5kyBt5QR5crGlsqSofaEzsVHRXn1+/SwC6dwgKSnTf2hOmm8aj8vO0xGFAyFABbgXmD6xlEzlj39I+X4cds85JCxrqDMeEEcxHzNjKYalYpgVezJ+/Qn657W5cUy6AFNrzSNMoB+Ea6eNlw+ibiFF3HCpJJQc0xOhM9frrblVW7JSpGaAU8oynwG3152NpbpJBzpSVCQI8ySdbcpaYSQpsz3dRl08NuW/n4WcMUeIgBeb6Up7enATvO8x4/X3W5XWBIhQVmJGkj468vlvYwpTYWEHs5BOp6chE+dglFOfIgIKpOZWWQT4wNPdaQYomRFV1UHMArKkDWNYnQx9eU2364gtlS+UaEgEmBrv9c7C5UhQDYQAAYypgADbQ+7624cQoFKkOazrJnpr+n5m0gwBRL7aoNT7LiSnKjTukBM6fU/py5X6stJiIImQ3O/u0+tbcvJcXK0rSUE5gc0Dr1j68rAO0j4UVpKErB0IPMg69Zm08qhxEIUsqCkpIBnSFEGdvyGnhzsEtDLqVBM6+ySomJ8J+XxtnY1bZWtVQkwCQpEj3dOR+fnYMpfy5VOI5zrpz1Gn1NnSJutBppMErSYEaD37b/1+IZQUAwsnKAMp03nTrYYiqOZBQ0FIESdpiOW/KwKwpCftWExMKTlMn468vlZhYJkCtxzMYdCiPEA7RMxvv4WCJyHs8+p0235zM9eU/GLGF1DSCEPUwCRoDsPOAI+EWCS8wrLKXVyiAUpMRHXbpp/Wz5gFEtKLOOEBZIWoZYIDJBPjvG3KwrK+8QUpgakwZ6g6672MpaSoAMB0IgElaD3p5z4WOU91rWnK0jInYBQiCT5T9RaOcAJ+ESi4LahmcYbzHVWnO2WWE3UnKPs6c6b5x+dss3ECfhFemDOCcMU7aW6a4ruZQkylKKRAAPkBY81c9G0kJS02ANgGwBbLZbysDmugzuWlUbZVAJSPC2l3a1spaiCLZbLFawHdFD3DYoBVOw0fYKvM20420kBRRO9stlpZQEZrid0A8hLeoSkgQdRbRcPZ5ghG8bWy2WgNCjt1tdJN8Xg5QUD1chtKi2kkJ2+doGxD65iWt9dvWvcWZKUoQkJSnyHutlssamF7kqyNkjN4epFvoQpxw5yATpznWwbt0UalZ1hZCQAlIV7I+vnNstlroUCuU3PQ1YVTPtl1CiEqDpzhW2457/Kwjty3RQVLtNRXZStBhZaBDKZOWNdt5M2y2WsROIGhQXtB3C6ZaOU9kUtgJBgJ01/rYaoazuoZmOzbSJA3mSfvtlssYOJ5oeUDkglICVBKiVSqN42BV/02FS029QNqKT3XlpTJkjQSfPS2WyxWqFkCmlRCtRokKHdH80fh87HKKjS6p1hSzl7FatBr3UlQ/8ADHvNstln5qKKu3ZTJUVkEgTI66n3cultUdOyxWJep0BK2pOsKCtNiOYiR77ZbLTCbmlG88NUdM+2thWRt9tLyUBPsBU92eYkf15kDhmjfLrSllJVELQkBSTMZgeotlss5UVGfE3BV1G4l42bKkVbDxp6pEAipCHuzCp/hXporXTQ5tCImcbp00yCadJLqdY00k7dNRMjrbLZbSpdWC6zanR+iIu3hTtU7MUeZNQ6lEKUk5SQVBRlOvsj36+FiSMQPs0jaktznqOxlRGYabzGuw0+62Wy10AKmXFGW78qX3mWlNt5ylHfKZ1UAdvqbEay/Lwp+wUFpyvgaJSEkBRPPYxlHK2WyxA0KDnFAu3/AHmahspqD2iVwFlKTptG22vPpYVmtvAVZoqmvdeStsOjXKBJiIHv+XSbZbLEDRZBLjddPdrTutvO1Dj32yGgCYgKSNZGvP5DxkOqU+h5omqdJdkI7x7iYBg9efTe2Wy0g0KNyiqAXqVpxMI7RAMCYB113125/pbpqrWhtmkqJcU64W86CG4iNSANfKbZbLPYJXKMIYaU6WinKSgjM3CfZ0H4W5ao1J0D6ozaAgHYkc9eXW2Wy0VFxKEVdrfbJSh1beUTKAAdCTE+87zvYuFR30rcIUsAZlSRKtRIAEeEWy2Weya5XCXKZDgacpyo1BDeYKAgDroZ38PnYneF4ponlqQ0tbhVBUtwGYT3dI1iOetstlpBMiDd8orVIUKFKFwO8VzG+2gj2jtZzXfdabyaCw4lkAxCWknqJncnTckna2WyzlRRm9sIquui/eD9e3UpXWIQEBgoIKp1nMQdxpHLygRm7BTkoafUC04jKYE7kfh87ZbLEbsou3UOfudqjxIwpDplFUlKyEiVy5BMGRyO4I185sXRcNcLVlR6oll5ChMkqTljLmgBKUn4kxPutlstUYd1ZIFlqqwFhy66pLCGKlRTupNa+gK15pC4P9etu2cLYdcZW45dTToBHdfJemT/AIyefSLZbLEuVEALlFw4SaUhJwtQBSnEolDLYGugJlBnfr12k2VH8PUN00yapimpBmyyEUyU848Rsek2y2WiN1MgLlla3m0qKjKBIkyJOm2x2G/TSLNN9S1XmrKoJ7VQTokaAkEbDkeke62WyxYt0OXqmnjTiJeVx3mMK0FIh1SkOPGoq3VuDRShAQkpj2eR2jpaPl8RXXXqti8rnadcpCUFdO6WkuAAH2VBcbxv162y2WHL7yND7icHDTiWbyv391MXMacBttXaGpzEkzEgIExB8dRqItY0pacAUtKoypV3VQZO/KPlbLZYbt1Jmy6FGlaA+lxaYbC45klAO/kQPdbWVxGqXlSO6CQDz8bZbLMnXSW1HtBn0B7o1EaeBH0Lbapg+o5XXEwspMkKBgnr5Wy2WSa5utuULjOVfrMhPRMHrvNiK3nUuDOsuELCDm1BM7m2Wy0gE91yupcStCFZVaADSBHl1sJSvl1bVOJCnAO+YMEgcotlsskgjd50TtNFQakKOYAAIIiVKHU8k/OyahxaiSFxBSmMoIPyn52y2WdJcuZ1uKSVDuiZj422ptJBJkkqMTEaAfr8bZbLJILHLuUlDjhfBAAhOTSJ23sVLUuKSCkajZO5m2WyzFMgkMdsla82UJ7sRJJBiZNtmmGVBSqAZ0InYn8vnbLZZX0SQakONFQS6QE6R74sNTtuuICy/MxlCkggDeOhtlstFPYI4i7FK/vXkrEbdnHQ8j42E/c1PGVxxZjQ5DEyJ5z0i2WyzOJT2C6YuGjqELWlx9OUjRTmaZ+Fjhw3SUrgQFlWROhjx5yTNstlhucUVrQtJudhTgADQUteSS0DBjewT9MhlGdpKE8kiDoQTrvbLZYYcbqQaEWSiUoeSopzlWmmgG2vXxsNT0frLvZLWkCSkdwHafytlss9ynaAumaCmeLjamkDTMTlmZIH5fCwzV20iSFKSqCf4SAdQDrIPW2WywwSpWF0IugpMxlmTO5O9stlstJPZf/Z" } ] } @@ -86,141 +68,72 @@ { "language": "markdown", "source": [ - "# Load a Detection Model" + "## Run inference on the detection model" ], "outputs": [] }, { "language": "typescript", "source": [ - "// Initialize OpenVINO core and load the detection model\nconst core = new ov.Core();\nconst detModel = await core.readModel(detModelXMLPath);\nconst detCompiledModel = await core.compileModel(detModel, 'AUTO');\nconst detInputLayer = detCompiledModel.input(0);\nconst detOutputLayer = detCompiledModel.output('boxes');\n" + "const detInferRequest = detCompiledModel.createInferRequest();\nconst detResult = await detInferRequest.inferAsync([tensor]);\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "# Prepare Image for Inference" + "# Load Text Recognition Model" ], "outputs": [] }, { "language": "typescript", "source": [ - "const imageData = await getImageData(`${baseImagesDir}/intel_rnb.jpg`);\nconst inputImageMat = cv.matFromImageData(imageData);\nconst displayImageMat = inputImageMat.clone();\n\n// Resize the image to meet network input size\nconst [B, C, H, W] = detInputLayer.shape;\nconst resizedImage = new cv.Mat();\ncv.cvtColor(inputImageMat, inputImageMat, cv.COLOR_RGBA2RGB);\ncv.cvtColor(inputImageMat, inputImageMat, cv.COLOR_BGR2RGB);\ncv.resize(inputImageMat, resizedImage, new cv.Size(W, H));\n\n// Prepare input tensor\nconst inputImage = transform(resizedImage.data,\n { width: W, height: H },\n [0, 1, 2]);\nconst tensorData = new Float32Array(inputImage);\nconst tensor = new ov.Tensor(\n ov.element.f32,\n detInputLayer.shape,\n tensorData\n);\n" + "const recModelXMLPath = '../../assets/models/text-recognition-resnet-fc.xml';\n\n// Load the recognition model and prepare the inference request\nconst recModel = await core.readModel(recModelXMLPath);\nconst recModelCompiled = await core.compileModel(recModel, 'AUTO');\nconst recInferRequest = recModelCompiled.createInferRequest();\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "## Define Post-Processing Functions" + "# Define Post-Processing Functions" ], "outputs": [] }, { "language": "typescript", "source": [ - "// Function to extract bounding boxes from the model output\nfunction extractBoundingBoxes(output) {\n console.log(`Output shape: ${output.getData()}`);\n const { data: boxes } = output;\n const foldingCoefficient = 5;\n const numberOfBoxes = boxes.length / foldingCoefficient;\n\n return setShape(boxes, [numberOfBoxes, foldingCoefficient]);\n}\n" + "async function performTextRecognition(model, inferenceRequest, img) {\n const inputLayerShape = model.input(0).shape;\n const outputLayer = model.output(0);\n\n const [,, inputHeight, inputWidth] = inputLayerShape;\n const resizedImg = img.resize(inputWidth, inputHeight);\n\n // Convert image to grayscale and create tensor\n const tensor = new ov.Tensor(\n ov.element.f32,\n inputLayerShape,\n new Float32Array(resizedImg.grayscale),\n );\n\n const result = await inferenceRequest.inferAsync([tensor]);\n const recognitionResults = extractRecognitionResults(result[outputLayer]);\n const annotation = parseAnnotations(recognitionResults);\n\n return annotation;\n}\n\n// Function to extract bounding boxes from the model output\nfunction extractBoundingBoxes(output) {\n const { data: boxes } = output;\n const foldingCoefficient = 5;\n const numberOfBoxes = boxes.length / foldingCoefficient;\n\n return setShape(boxes, [numberOfBoxes, foldingCoefficient]);\n}\n\n// Function to adjust bounding box coordinates by a given ratio\nfunction multiplyByRatio(ratioX, ratioY, box) {\n const scaleShape = (shape, idx) => {\n const position = idx % 2\n ? Math.max(shape * ratioY, 10)\n : shape * ratioX;\n\n return Math.floor(position);\n }\n\n return box.map(scaleShape);\n}\n\n// Function to extract recognition results from the model output\nfunction extractRecognitionResults(output) {\n const outputData = output.getData();\n const outputShape = output.getShape();\n const [, height, width] = outputShape;\n\n return setShape(outputData, [height, width]);\n}\n\n// Function to parse annotations from the recognition results\nfunction parseAnnotations(recognitionResults) {\n const letters = '~0123456789abcdefghijklmnopqrstuvwxyz';\n const annotation = [];\n\n for (const row of recognitionResults) {\n const letterIndex = argMax(row);\n const parsedLetter = letters[letterIndex];\n\n // Stop if end character is encountered\n if (parsedLetter === letters[0]) break;\n\n annotation.push(parsedLetter);\n }\n\n return annotation.join('');\n}\n\n// Takes original image and bounding boxes with annotations\n// and returns the image with annotations\nasync function putAnnotationsOnImage(img, boxesWithAnnotations, options) {\n const defaultOptions = { threshold: 0.3, confLabels: true };\n const { threshold, confLabels } = Object.assign(defaultOptions, options);\n\n let finalImage = img;\n\n for (const item of boxesWithAnnotations) {\n const { box, annotation } = item;\n const conf = box[box.length - 1];\n\n if (conf < threshold) continue;\n\n const [xMin, yMin, xMax, yMax] = box;\n const yOffset = 10;\n\n finalImage = finalImage.drawRect(\n xMin, yMin,\n xMax - xMin, yMax - yMin,\n { color: 'green', width: 3 },\n );\n finalImage = finalImage.drawText(\n annotation,\n xMin, yMin - yOffset,\n { font: '30px Arial' },\n );\n\n if (!confLabels) continue;\n\n finalImage = finalImage.drawText(\n conf.toFixed(2),\n xMin, yMax + 2 * yOffset,\n { font: '20px Arial' },\n );\n }\n\n return finalImage;\n}\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "# Do Inference" + "### Do Inference and Show Detected Text Boxes and OCR Results for the Image\n" ], "outputs": [] }, { "language": "typescript", "source": [ - "// Create infer request\nconst detInferRequest = detCompiledModel.createInferRequest();\n\nconst detResult = await detInferRequest.inferAsync([tensor]);\nconst boundingBoxesArray = extractBoundingBoxes(detResult[detOutputLayer]);\n\n// Show original image\ndisplayArrayAsImage(\n displayImageMat.data,\n displayImageMat.cols,\n displayImageMat.rows,\n display\n);\n" + "// Calculate ratios\nconst [ratioX, ratioY] = [img.width / detInputWidth, img.height / detInputHeight];\nconst boundingBoxesArray = extractBoundingBoxes(detResult[detOutputLayer]);\n// Resize bounding boxes to the original image size\nconst boundingBoxesOriginalSizeArray = boundingBoxesArray.map(box =>\n [...multiplyByRatio(ratioX, ratioY, box), box[4]]);\n\n// Process each bounding box and run inference on the recognition model\nconst boxesWithAnnotations = [];\nfor (let i = 0; i < boundingBoxesOriginalSizeArray.length; i++) {\n const box = boundingBoxesOriginalSizeArray[i];\n const [xMin, yMin, xMax, yMax] = box;\n const croppedImg = img.crop(xMin, yMin, xMax - xMin, yMax - yMin);\n croppedImg.display(display);\n\n const annotation = await performTextRecognition(recModel, recInferRequest, croppedImg);\n\n boxesWithAnnotations.push({ box, annotation });\n\n console.log(`Box ${i}: [${box.join(',')}], Annotation: '${annotation}'`);\n}\n" ], "outputs": [ { "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAIFArIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1SXSkkG1ihGQehHQ59a429+D3h+9klkY3SNKxZjHcY5Jz/Epr0RvvfjSgVEcPSg/djYp1pyWrPJpvgbpjKVh1G/j/AN5kf+grPufgXMYysGtvj/btR/R69rFLVciFzM8Cn+CetpjytRtHx/fSRc/kpqhL8JfE0RYqLSX/AHJtv/oQFfRtLS9mh8x8yP8ADjxXGrE6cGPolzE38nqtJ4M8TQld+i6gRnkrAzD81zX1IUVuqg/hTDbQHrEn/fNL2Ycx8py6VqVqx+0Wd3F/11gZcfmKgG4ZBbnmvrMWsI6KV/3SRUU2mWlwCJYhIPR/m/nS9mw5j5W+fAzxTwW3AjJAFfS03hHQpx8+l2Z9zbRn/wBlqjJ8PfDshz/Z1uPopX/0Eip9mylJHzZEzYI3H7x6/Wp0ZiseM/d6173L8KfDjA+XZhDnOUmkH82NUZfg/pDABJLpAOBtuAR+sdSqbTLc00eJbjuY8kdOKYWbHPPocV7DN8Gockw6lcL7NGj/ANVrOn+Dd8ABDqStj/npBj+TGqsyLnmJPI5FKCSSNxrvpvhJr0Zys9k49MyD/wBkqhN8MvE0IbZaW8mf7twgH/jxFFmByO9gn3j+PanAuGAyfxrem8DeJIcZ0edsd49sn/oJNUpPD2tW5Jm0nUEAHVrZwP5UrDuZ4JYnp+VLuPHofSnNBNEAJUdGzjDLjrQFJYZ5xUlCgnOOfes+d3XU4scHA/rWiAc5x7Vm3fF3EQM57596mQ0aMUm6PrnnAp2/AznNRwHbFv754pxOAFweaUdimBftkY9qCxAz+tAIJPtTSV25I6+tUSOJbOc03cc8nHelbHTn3o25Ht069qkYmfu5yD2oyScnj60u05xyTTgpB49KVxgCRTWO5Dj09afg5HAye1KVODnnIzUsZl2TsplHPbI/OtLYjYbbgHms2Bis0q4/WteHJgTjgipe41sQ+SpxlTSGAf3m9OtXAoAzwKUqMcLmi4WKX2f+EMffNM+ynPOCO2RWhsB4oCZPFLmHYoCyLAEqhP5UNZnHCd+gNagj9BSFc5qeYfKYxtW7Ix9/Wo2tGByQ6jPcVulMDnjHrTGjOwnB6Ucwcpzo3K/r2qdYtyAnAPrSTrtkPHetSwVXtACoPJB4zQ2CRlPZAIWaJD34ArF1C12xl0hC8/wjFdfdJHgoFGO+KxL6P/RZMdAc/jW1KTTM5o5bawPANHmOP4m/OrxT3PPrUezII6/hXZzHPYq+Y/8AePFHmHvU5jGM4H5Unlgj7uPei6EQbzS7+MU8xjHIIpjIAuRmnoGopJ4JzzTd555609uY09gR+v8A9eo9poAXfil8w+lNwaTB9KYXJFcA8jinmYYwM49ahwfQ0lKwXJd4IxRuFRUUWAc/NNoopiHI5SRXBwQcg19uaPffb9EsLwHInt45fzUGviGvrv4X3pv/AIa6HKTkrAYj/wAAYr/SuLGLRM2o7nXlzSbzSUhrhOiyF3GjcaSkqh2FViHYZ9DT9x9ajHDj3Bp1ddPWJEh24+tG4+tJRViF3H1o3NTaKYh28+tQzMd8bZ74qWopwfLyOxzQxoXc3rSb29aTtSGkAu808OcVHTl6VLQ0O3tVS+lmjsriUT+XsQsCEzjAz+NWGOB0yT0FVLyyF3aTQzOxEileOAv0Hf8AGpavoPY46TxpbC2huJLdpZJNrBSqgpn+LO44+nB9ap6ze6fKJQ11cm7JCNGMny/baOAT6MQcVn3ekwXlhd2YmZLzT2MfmNgkqBz379f0riFvtkUNs06tPBJhcAFVABODkc4PvjNY1KV4tGTqNM6z7bpwHK3ue+JUH6eXRVJfFjBQHS4LAckAdfzorx/Y1OxtzRPdG+9Sjmkb71KK+wZ56HClpBxSikMWlpOtLQAtFFFABS0lLQAUtJS0hhS0lFAC0UlFAC0daSloAaUQ9VU/UU3yIv8Anmn4CpKM0ARNbRMMFePTJqnNoOlXBzLYW0me7xK38xWhmilZDuYM3gvQJs7tKsx9LdB/ICsy5+GHha5YM+mRhh0KvIv8mrsaM1PKg5mcE/wm8P8APlC4i9kmOP8Ax4GqM/we09jmC/uo+ONzK3/sor0uijkQ+ZnkkvwclCsI9WYk+tuP/i6ozfCHVhjy722YL03q4/kDXtNLS9mh87PBp/hb4hiDlBaSn/ZlIz/30BWfL8P/ABFABnTVZRyds8bfpur6JpCARyAfrUukhqoz5sfwrryKXOjXzDtshZv5Cqcuk39vgT2V1F674WX+Yr6aWGIoAY0/75FH2aH+4B9Dip9iUqh8ulcM249PejYdpwck9K+n5LG3l4ZC31YmqUvhvSJ/9bYWzH1aBG/mtQ6LGqiPlfaUu2XuRxWxaZNqgPQZ9s179N8PvDM7bn0m13eqx7f/AEHFVW+Gnh/btithGvoryZ/VjUujIpVInig+7nH6U4RbgF5+tevy/CzSWOY5bpcdB5q4/wDQDVRvhZEhJivpRngZQNj9RWbpTLVSJ5gIULHIGKcsKjr1Poa9Gl+GNyMeXfBsf34sfyY1Sm+HeqpuKyWzHtw//wATWbpz7FqUe5w/lgtxkbR1PemmI468n1FdbJ4E1tMYto3A6kSqP5kVTfwrrEWSdNnJ7BBu/lmo5Zdh3RzhhOSBj3OKYyN5fbn3rcl0TUYiA9heIAOS0LD+lUJbd4j86upHQEYNLUehyd8hVyCBnOTg1a06YrasOpDcUmooVDjHzNyah075oXHuK0SuiHuWXOWz3PrWbeoDDKD0A61pEHGe9Urpcq4xwVPvWkCJHOsPTnHvmoOTxjg1YJwABn+dQHBFdaOdjW4zSMBxnGfrS56kHnofemknjI4NMBDjpj3pj/dbFOOcd6aeQetUIQH/AEccfdfr9R/9al4OB1zSIMwSD0INIMkUwDC0YwcGjNHX+tABjNBUDijnOCOKPrQAmP8AOKQrTs+2fxpD7UCGkYptOI4pKYhK+mfgNffafh89uWybW8kQD0BAb+ZNfM1e7fs73vya7Yk94plH/fQP9K58Ur0zSk/ePcTRmg0leYdaFopBS1QCHgqfenU1/u/TmnV1UtiJC0UUVqSFJS0lAC01xlSPUU6kNOwEKcoPpSmmowO5RjKsQfanUgEpy96bS5xnHWpY0OAJOAOaazwhXDSKMe/Q1Onkqu+V05H8R4rAvNathqzad5roWXeFTjI756n3wB9cVlKryK493Y4DxmZYfE32/TpI0VodkrSKYwTyMFiAOmfeuJtNAxOupahf29naysfKUtjzeecd8e4HX1r1XxBq0ElosEttJNG2GSJISiJ6HLLtyOvXFeWXfh+91L95d3iR7mLCWc+X8o+p549K5Z13JWbsS4JPa52McPhJIkUMjAKACUbJ/wDHxRXMx2Nokao19bFlABO1+f0org9k/wCdl83kfQjfepRQ33qUV9azgQZqB3xubLAD0NTmqsoysi9MgimhM5rR/iJoOtXkVrbXM8c8pwiTxFdx9M8jNdYsrfxVyFl4Ksra4gnJV2iZXDBcHI5Fav8Aawl1k2UaSjyk3SM6FQSTwBnr0PI4qrCub4bIpjTbTgDOOtNQ/JWL4mjefw5eQRsVaZRHkHnDMAf0JpWHc2I71JRlGR1BwSjA4/KpxICK878H+HH0LVhLC7LDMhSSPPB4yD9cj9a7rzNsuz2zQ4gmXCwUZPSm+cn979Kikb93+IrivHGoa9YT6edFuFjDLIZUaJWD4246jjv09aXKO53gdW6MDTs1kafcyS6da3NwoSWSJGdRxgkDI/M1oq2RS5Q5iajNY+p3rWOm3l6kLTNBG8giU4L7QTgH8K5Sz+IbXLBToWoqx4wh3Y/QUcoJnoeaM1BHKXUHkcd6jedgXO7Cr7e2aVh3LeaXNctpXjfQ9ZuUt7DVI5Zn+7G0bIW78bgK6JJtw5FFguTZoqrcXiQFAxA3ZxlsU0XoI+4SPY5osFy5miq8dysjbQrD61MDmkMdmikprSKgy7BR6k4pAPJozTQ6sMggj1Bpc0ALmjNJRQAi9KdmmL3HvTu1AC5opKBQAtGaSigY7NGaaKKkB2aM0lFAC5oIB4IBptLmgaG+TEefLT/vmka3iYYK8fU0+ilZFXZRn0LS7nPn2FtJnr5kStn8xWe/gbw1Jn/iT2a567IVX/0ECt+ilyod2clP8M/DE2SLDYfVJZB/7NWZc/B7w/OSVlvIif7kw/qpr0HNGaXJEOZnkNx8A9MbP2fVr2Pjjfsf+i1j3P7PtwB/o+u5P+3a4H6Oa93BpQafKI+b7j4CeJEyYdQ0+QDpkyqf/QDWRc/BfxlADts7ab0KXKr/AOhYr6ozQDRqGh8gXHwy8Y2+d+g3D/8AXFkk/wDQSazbjwb4mtgTN4e1VAO5s5MfyxX2iQrdQD9aZ5EP/PJP++RRdhZHw01ldW4lWe2miO3pJGV7+9VsHFfdjWsLjDLkem44qjceG9Gus+fplpLnr5kCv/MGjmYcp8RYPegn2r7IuPhx4RuQfM0CwGf7luif+ggVk3HwY8E3GT/ZCRk/3JpV/wDZqOfyFynybRgnpX1BcfAXwlL/AKoXkP8AuXH/AMUGrJuf2etJYk22p30fpvKP/wCyrR7RBynzqBxRtHrXulx+zzOMm31sn/rpaj+kn9KyLn4B+Ios+TfWEo7BhIp/RSKPaIfKzyEgd6bjmvSbr4KeMof9Xa2s3+5chf8A0PFY9x8LfGtqcvoUzjP/ACykjk/9BY01JdxcrONr1T4CXnkeO57fOBcWTrj1IZW/oa4m58FeKLQnzvDuqKB3+yOR+YFb3wuF3pHxL0d7m3mgDyNCfMjK/eUqOvuRU1bODCCakj6rNJRmivKO0KUUlFUgHEZBHrQpyoPtQOaRD8pHoSK6KREh1FFFbkBRRRQAUhpaKoRABiVwB15pTSP/AK5T6jFKeaQXCpYIfMfcQMCohy6oOp6CtHb5UOEGSBx7moaQN2KVxFDcMftTKtspA2lsBmPr/LFYWvaTpcNnNIq/Yw/ytMkrRFs9RwCWPTjHNdO22PBIzjpxk5qte2EeoweVeIGhYg+Vng49cdfp0rmnTTTTV2wUtTztS06Yim1S6hQZBkUspwD91SCQePY4471xt61ld6sls9/fM9zKUaJ48FRxk4GcDj0H0Net61cvJbxw6bGQkbKA6LkEZwVXHt3OB2z6ec+JdRG26ewnghuWg2yrb4Z1IBXBYf7wIxjPNebOnyysnc6FrG5lyrp6Sui+C9YKqxAy0o4+gTiiuf8AsuunmJ7jy/4NzDOO2eaKfso9/wAWY83kfSzfeopWHzUCvqGcaEqhfc2dyPWNv5VoEVy2sf2+uoPHavbNZS4GHU7lBGDz+dNNLclps47w14XvdG1Ozube9m8oOokjL8MvQg/hXos4Xz4iRzg4/SmLZPHZsYwDKFyoPTPasnTtR1a/vhDe6WluiAnzFkzz6Y/z0qrruKzOoT/V/hVLURv0/jn5kP6ir0akJiufvNch0mZrXU4pVhOfLnRCylfQ45BFCYmjRt1CtGf89DSu+dQwOyD+ZrGbxj4fgj3LemRv4USNsn25FXNGlmvw17JE0QmbcqN1Vegz74FDYJM2W/1Y+o/nXLeLvFn/AAjU9kraW16k4csyvgx7dvbBznP6V013LHbWrTTOqRpgszHAArDvjo+ryRtJqFq3lghQJFPX8fakMsI0et2VlqcE0yxFRKkecBsj+IeorXgY7BmsJtS0/Tra3061lSSRiI40jYEgdycdBjNbdvkoCaAK19JbxaddS3jFbZI3aVsE4QZJ6c9K56y17wk7KLXUItzEBQVYZP4iuiu4Rc2NzbkZ8xHTB75yKw7DwpZWsyytaxBgQQdoNSyonURdOOlQNz5wPc/0FTxgBcZqEj5ph6n+lNCZydh4LtbHULa7GxZIHDgqOTiuit9QS4u5Io1cCI7WLoVyfbPbpzXn2ieH9S0fVbO4h1C4a28wB4mclSp4r0bA81DjnHWhgirr2h22t2qR3Me4oSUOcYJqroelJpmjS2SZCq8mOfXn+tdCvK1UAw8w/wBr+goA5jwnoUulXDStd3MoKbQskhIH+cV2ydKoWaAIvHar4FQWPzWVrhJswO24GtSszWRm0/EU0Jmdpl55TNESdrcj2NQWVy39qiQsxBds8/WsGTUJLfxX9gfiOS0WaFv9oMwYfltP51paezGVWcEHzW/9COP0pdB9TtI23Cn1XgOVqehgC9T9aWmjqfrS0gKV7qP2N0UR7yRk84xTkv0a0+0EEAAlhnOMVn6x/wAfCf7tZUl6p0+9Eb5CKwYZ6MBUt2nZ7FqN4XR01pqEd3uCoykAHnFWi3FYGnMY5MA9Vq/qFy0FhJIv3sYH1NODuhTSTJpNTtInKPOu4dQATj8qnhuIp03RSK49j0rzjWINSuYEGm3PkOCWY7c7vQVc8L392UtpLrInJ8uUevOP8DUptq5XKtjvy2BntQGBGR0rO1SF7vS54I2wzoQOcc1zWhagYZRu+XPySA9iP8/rVuNlchb2O2zzS1yj38tzqAKzSLHuAARsZFdIJQULZ496lO70KasTbqXNcjq3iRrWGS5VpBAnQRJuZqseH/Ea6pDDKrl4ZvulhhlPTBxU8w+VnT5ozVa5ult7aSZhkIM49ayz4gjTHmoqZ4GZMZ/SnJqO4RTlsbuaM1RsdSjvSyopBUZ6g1dBoTuNqw7NGabmqz6hbRyGN50Vh1BPSgRczS5qtDdQznEcqOfRWBqbPFG4EmaM1GGyKN4JxSC5JmjNNzgUA5HFJjH5o3UzNGaQyTdS7qjzRmgB+aSm5ozSAcaYQD1AP1ozSGiwxhijPPlr+VNMEZ7Y+hNSE0manlQ0yI2644LD/gVM8g9pX/HH+FTmk71Ps4voVzMgMcg6SL+K/wD16TbKP7h/MVOabS9lEfMyPMg/gH4NUMDzlpC0SqS2cbv8+lWqjHEp96FTS2DmuKGfun5Gjfj+Bvyp9Ga0sTcZ5i/7Q/4CaTzo8/fFSUUrMY3ep6MD+NLSFFPVQfwphgiJPyLz7U9RDLhtgVjnhhSb95wn4nsKiurQNbuI2ZWxwckgfhmpowBGoHTFGtw0Lttb+UN5+8euetWSQFLHPHoM1WS6UIA2S1Zlxrl7Zbml0tpY92FaCUNgf7QIBH4ZrKpUjBXkTyuT0Ndl/wCWjZHHTPSud13ULu6YWGnouBgyvISA690AHJHqR24zzVLVPE89xILNVS2YkHaWzIw9h2/I1nXGqSWtm4SK2V5mw5lO4tgdSTkn8a8ytipSaVPbudEKDSvIbeLMLSSK906TLZ8yaIrMPrsyFHAAA5xwPSuH1a10ay8iWzknMMpCtlcMG6gsDxjnjAHWuluNVuhF/o8toZGBACQqQD2OcYwPrXM3V3G9zuv2e9uRwCuUh/IDJz+B46HFYOTbLcUkc9Jq2rRSNGlpeuiEqrHaSQO+cUV1hksSTts/l7bYcj8Cc5/Oil7eP8pnys9qkHz0gp0v36QV9UcAYpjRhjzUgFLQBEEAGKYLdQ5YDmrGKMUANC4FV7i0juFKuoIPqKtUuKQGGug2aSb1gQHPUKK04YBGoAGBVjFGKSSRV2yC5to7m2eGVFdHGCpGQa5ibwXpjtn7Mg+gxXX0hANDVxJ2Ob0zwzZadL5kMCq/rjJ/Ot9EwMYqXaPSjFCVgepyWsaFqU9/LPZ6vdW6Pg+WrfKDjsKoLpPiSM8a7MR/tRA/zruygPUU3yl9KHfuNW7Fe1RxCokYs4ABbHX3rF1ldbjvwdOktxbuBuWRCSD3P8q6QLikaMN1qrvoTZGNJYuNPfywDKFygPTI6VQ0i81a8uil9ZxQIgyGV8kn0x6V1HljGKYIVVsgChyYWQiLha5/U9bj0zUWt5LS7kMgDK8UeV9MZz1rpAMVDNbJKwZhkii9h2RBZqwjUkdquimogXin4pXADWfqxAsyzEKARkk4ArQqrf2UV/ZyW06B43HINNMVjhtfsftaRXtqytcWmHQqc5x1H4g1csZRIIZV+62CK3NO0O102BoYIgqkkn3NZy+EbWPVReI0ow+8IGO3P0pNqw0nc3VmWGLe+ccCrUUyyLuXP41n6jYvdae8MMrQycFXXqCKbo1rc2tr5d1cvPJnJdgB+HFJtBY1R9406mj734UtAGPq4/exfQ1w0StY+LNQtpiWtNTUSJ6B1UKw/EYrtPEv2xLWOWythcSBsFCccVjT6Rc6lpEcssQivIyJUUHO1h2z9OKHG8tSoytE07D5WjGc4XGTVvVxnS5T6YP61h6Je3d1e+VLp00CoDudzxn6V0t3bm4sJouMshA+tKkrKzHVd3dGFpqKzsD6VRVBDfzKvAE2QPyNVbTWYrWdorlhb3MfyukvANWba4hvb/McqO7tuIQ5x0/wpS92PKwiry5jsYvmUA965XVrI2OsCSNf3NzksB2YV1cCkIM1m+IUUWSSHjbICfatY6qxD0dzO0uHzJQ2OF4rfuFIspcddh/lWToTLJHIVIOG7Vuuu6Jh6jFZU1Yuo7s5G3t0ukSJwCrDpUel+GZrDU1MV5ItmsnmLCqjHrjP1qXT5hG4z96JijD0PSrcl9cteP5T7YlwFwAc+tJrlvcpXlaxr6ou7Srgf9Mya5K/0u21e1ijukDKpDj2OMf1rsbkebZyr13If5Vx1zYnU9MjgEssJO074m2tx70VfhTCl8TRteGNMsdKieK0QK7HLnByfSukBrm/DOjtpiSM9zcXDvgZmkLYHtXSClHYJbinpmuK1BZC96sb7ZNzhWxnB7Gu0PSuSvlK6ldL/tA/mKqS/dsUH76Oe8J6hqkiTjUTm5tpsK4TbkY9v8816TLIfs7lDhtpI/KvNdE1m8utVuLG9tY4pYAHVkJwwz716HbSedaq/wDeXms4aS1Ln8JhaFcubiJ2dm3qQcnPP/6xUN1qt0ut3KiaRVikUBQ2BjHpTNJzHMsZ6xzMn61W1RfL8Q3I/wCeiI/6Y/pV2+NE9Ys7a4uPKtZZQM7ULY9eKydE1S4u5QJnUqybgAoHPFXoj5+mrn+OIfqK5zw/Jia2B9Cp/WlL4Uxx+Kx0OsamdPto3Cb2dwgBOAPepdMvjfWnmMgRg20gHPpWV4qUnSkkH/LOZTx+X9ak8PN+6mT0YH8//wBVE1ZJijrc381mtq0KzvGN7FDg4HGa0M/LXJSIDfXkJPBdge3WpnpByXQcdZJM6KDUYZHCbiGPQMMVdzXmemW13pHiCSxluppreSIPH5rFtv5/jXoMUpltA2fmK4J96mLd7SKaVrodFfW80zQpcRNKvVFcFh+FTO4UZrzfQJfJ8RW7HqWKnPuCK6HxrI40iMKSFaTDYPXg1t7P33Az5vd5jpt4K5oVgwryzTdfnsNIvbQSld67o2B5VsjOPqP5Vs+BL+aW7u4Zp5ZMoGUO5YDB7Z+oqHH3blX1sd2SAOabuB4zWB4o15tGslMSB55M7d3Qdsn864aDxlrcV3GJbjcJBlQ8ICt9MD+tS9ClqesUlUNG1RNW0yO6UbScq65+6w6j/PrV+gYlNI+cU6mv2PoaQDqKKKYBRRRQAlFHeloENYZBFV4STEPUcGrOKrRArLKuMfNkfjTAkpyp5iOnHTIzTaVZFiJdzhQDk1FSKcWmCdmczrFjbTTtDPDHKGw2xgDjtu59KwpdMmkVPsWozwjn5pFEowTnjdyB6YI/rXTyW7zT3N1Jvj3ttRHUZUADH884/Os2+SO006Rp1PlkEZfPzn0z71xWjK7asjVt2Ocmsv7Tu2ie4i+zxDcAifM6nuzZPU59B149eC1e3uG1C4gvH8i1DmOOR5yS0YPy5AywXgH0r0e3SG3sy8rwNcTN5kuWACH6e3QCvM/HumXt3rDraM0sD7JF2PldxGCT6nj9amFOCemhMm7XNmO+tUiRF12dVVQAqwMQB6A9xRXBR6VqKxqPtiLgAbc9Paio+rU/5yfavsfYUo+emAVhHxnpMp6zr9Y8/wAjUq+KtHPW4dfrC/8AhXte0h3MPZy7G1S4rLXxFpD4xfIM/wB5SP5ip11jTG6X9t+MgFHPHuLll2LtFQJf2Un3Lu3b6SKf61Mro33XU/Q1XMhWYuKWil7UBYSiloxQA2lxRS0AJRilooATFFLRQA3FFLRQAlFLiigBMUlLRQAlFLRSASiiimAm0UhUU6ikA3aKTYBT6KAGfx/hTqP4h9KWgBjIGHNMEQAxipqKAK4t1ViwAzUm3ipKTtQO5l32jWd9zcW8ch9WUGm2Wi2tif3ECRj/AGVArWoxSauNNoYq4FV7+xh1Cze3nTdG/UVbxRigRl6XpVvpcBht4wiZyfc1o44p+KMUkDZyuseGpZrxr3T7pradvvjbuVvwqPTdJ1FJ83t0sijoEj211u0dxSbF9KUlzblRlykAi/dlfbFcRLa+ILWRo4oLSSJThSSc4rv9tMaJW6im7tWCLSdzk9D/ALWfUUN1bRRQqCSVckn2xXXCmLCqnIFSgYqYqxTdw7Vx/iG5Sw1Eu8UzCRQcxoW6etdjioJrdJhhlB+tVfRoSWtzgLW7sp74SRRSm5Zdn+qYcZzz2/Ou4sozHaonoKEsIlbIQZq2se0YrOMWndmkpJqxwt3crpeu3ME58tZG82Jj0OaZdyLd6it0GA/deWQDnJzmuw1LR7TU4wtzCkmOhI5H0NZtn4V0+ynWWKAB16EknH51VSV7tdSYJdehpWCMlhEjfeCjIPauY0/9xqLJnGy5Yfhurs1TauKxm8O251Y3xaU5bfs3kLn1xSv7nKwXxXH+IY/M0O5Hoob8iDWfot0kDsXcBXUHJ4FdDdWwubOWA8CRCp/EVx7eG9WiOI9S+UDgGEGiclyoILVnZw3Ec4ISRGI7K2a5q+/d69MvZ1Vv0xVzQNOu7NpJLy5ErsNoCoFAFV/ENnffbY7qyijk+TYwdiO9CknB3DltJWKWqup1nSnUcsJFP5D/AOvXSWBzagelcvbLq0kyLNYxqP7/AJmcfpXW2sXlwqvoKxUnKafZGjjaLPO2/wBD8RnsI7on8N2a63xVF5ugyNjJjZW/XH9a5/xBp9ydfkEFvK5lYMpVeOnr9RXW6nC8+h3MZHzmE8D1xXZzL2ql6GCT5LHjt1J5NxDG33JdyH69R/I10ngqQw65GhP342T9M/0rPj0l9ZNwkKFmjXzEbHAcHpn35FS+HneHXbIqpLGQAqByM8H+tRNb2HF9zofHsZK2zdijD6YINc3O1ncaJpsTyulygc7kA4G4jv8AhXaeNLcyaSkwHEb/ADH0BGP54rzia1t7qxNvNH+9Rw0bdsZGRRJK0WNdUel+DXs1014LXzsowaRpcZYkdePpXS1w/gqdTeXMa9CgOPocf1rt6wi9DWQU1+VNOpD0IqibgDlQfalpqcoKdzQAUdqWjpTEJ3ooooASq7fLcj0Zf5VZqvOCJImGMbsH6GmxIfUKhnut54jjI2+7dz+HT86mpMVMlcaY7UYwVHpiuW1CFC0KswcbtwiTlnYDIA/me1dDqM94Y1W3sfNBX77ShRn07ntWBqaXVtaTzJEd/lgvtBfn0B7LnH6nmvOxNZJcsXZmsFpqc5cxNc6n5eoXKwwuP3cSnooHO5+3XOBj3NV7iKG8N55dhBcEhi08eCIsYAwO5PJ7dce9XdQgllVUuJTFBjmNdu8npgDJHvkHiua1BD9oZYdVEdogBEaPtJJGenU4GPXnp7eZUlOd1I1suhfSHSlRVaCYsAASZohn9KKpJ4USRFeTVpEdgCy5JwfSisLx/mFyPsPEjK3X9asx3Gep5qk+d2R1pyEe4r6B6gjUWQN0NShuKzVcqcfhVlJSO/HrUtFFnCE8ov5Uo2g8KAfpUav+NPDY6fzqRk6TyJysjr/usRUy6jeJ928uB/21b/GqeeKdk9setO7CyL6axqSdL24/GQmpU8QaqowLx/xVT/MVl5pe1Pnl3Fyx7GyvibVF6zq31jX/AAqZfFmojqIG+qH+hrB/xoyaftZ9yfZx7HSJ4vvMfNbwH6ZH9asL4ul/jskP0lI/pXKKec9qm6Ue3qLqHsYPodUvi1P4rJh9JM/0qVfFlr/FbTj6bT/WuRGcZpaf1mp3D6vA7FfFNgesdwPqo/xqRfEmmN1lkX6xn+lcVR1p/Wpk/VoHdDXtMb/l6A+qMP6VIur6c3S8h/FsVwOfejOKpYuXYX1aPc9DW/sm+7dwH6SCpFmif7sqN9GFecemT+dGPoaf1t9hfVV3PSuDS4rzUMVPBI/GpFuJ0+7PIPTDEVX1vyF9V8z0XFGK8+GoXq4xd3H/AH8P+NTLrGor0vJfxOf50/rcewvqz7nd4oxXELr+pr/y85+qL/hUq+JdRXq0TfVP8KpYqBP1aZ2WKK5JfFN8OsVuR/usP61IviycfetIz9HI/wAapYmmL6vM6cj5lpcVzB8WsWX/AEEH1/ff/Y1OviyH+O0kH+64P+FP6xT7kujPsdBRWIviqyY8wXA/BT/Wpl8S6cw5aVfrGf6VSrQ7i9lPsatFZy+INLJ/4+cfWNh/SpF1nTX6XkX4nH86ftIdxckuxdoquuo2LH5by3P/AG1H+NSrcQP92aM/RxT5l3DlY+jFAIPQg/Q0uKLisJRS4ooEJRS0YoASjFLRQNISilxRSKEoxTqSgAxS0UUAGKTFOooAbRilooATFNKA9afRSAYEAoZA3UU+ilYoh8lc5xTwuKdRilZBcieJWYEjkUuwbcVJSGiw7lZLVIydqgZ9KiTT4IpzNHEiu3Ugcmr1JS5UO7IJoEmhaKRQyMMMpGQRXNyeDNP84siOq/3A5xXVdqTFDV1YE7GJovh+z0YH7NHgnjJJJxWzTsUmKSVgbuJRS0VQhkf8Q9DT6YvErD1p9ABRRRQAUlFFNCYtQzjMRI7HNS0xhuUj1FOwhOvSkpIzmNfpS0hkkm46c+z74zt9q5zUkeSBAkpaISK7KuP3mPcmt+EGSJ/NH+0qen196yNQVVkPHI5rgq0HObkawkkrHP6pbi7vLXfbr5WC0mWAbjGATyQOvTr7DmsvV2mW2lng0jTxLEhMcjMG2bQcHbt9ulXri9XlLaMzSDJZYiAFPU5J4H8/aqoSRlZ7psDPEUXOc9jnr+lS8LC3NLVh7R3sjzseKYgAH1e93j7224bGfaitN9ABdi/iLT42JOUEcQCn044/KiuPlo/1/wAMV7xqMvzHjjNKq8D2qWUKjfMQp9+KRMEfeH4Gu1O5tawBcds08D2pwX0PPtQFHai4xVYrwf0qVG6ZFRjHY5oXOD29c0gLKv8ApT8gjqeKqhj1qQMRQBY68j60dOpqMMP608HjFIB3b1pcc00EZxml/KgBR1xmp1quODU65xUtlIdTuKTtSfQVJQppM+1LgikI+nFFwDr/APXoA4IxQB+VLxn1ouFgB/lR68UdOv8AKii4Bx1o+tL+dJQAAUdaWk6jigLC5pD796XHSkI9utAWA4oz9KM80h64/OmFgAzS0DrSY4oFYXg5ozikx15ozQOwuaAfek70Z4pALn0pcAj/AOtTc460E0wsO4AyB+VPWaVfuyuv0Yioh7UvGKLsVkWBf3aH5bqcD/rq3+NTLq+oJ0vJvxbP86o+9B47U+eXcThF9DTXXtTX/l6J+qL/AIVKviPUV6yRt9Yx/SsjNGfXFP2s+4vZw7G4vie/HWO3P/AD/jUi+Krkfet4T9CRXPg0ucVXtqncXsYdjpB4rb+KzH4S/wD1qlTxVCfvWrj6MDXLZPFGfen9Yqdxewg+h1y+J7M9Yp1/Af41MviLTz1aRfqn+FcYDS7gQaf1mYvq8Dt117TW/wCXjH1Rv8KlXVrBul3H+JxXB55pQ1P63PsL6rHud8NQsj0vIP8Av4KkW4gf7s0bfRhXn27HNLuprFvsL6qu56ICD0IP0pcV50GA6cfpUi3MqfdmkX6Mar655C+q+Z6DRXBpqV2vS6m4/wBs1MusXwHF0/480/rcewvqsu521Fccmu3463Gfqq/4VKPEV6OpjP1Wn9agL6vM6uiuYXxJcjrHEfwP+NSr4lk/it1P0OKpYmmL2EzocUVhL4lU9bb8n/8ArVIviOA9YJPwINH1in3J9jPsbFJWWPENmeqyj/gI/wAaeNcsCf8AWMPqpqvawfUXs5roaNJVIavYN0uB/wB8n/CnjU7JulzH+JxT9pHuHJLsWaKgF/Zv926hP0kFSCaJvuyIfowp8yFZjqKOvTmiqEMPEoPqMU+o5DgoffFSUAFJRRTEFFFIaYhTTaWkpiIk/iHoxp1J0lb3ANKMA5IzUybSutRixnD/AFrDvpLa+kePzFfaNssOcEH39q2toLlskMfeuW8Q+D4tZnF3a3j2t0DnzFBKntxgjB/OvHr46pTnapCyNVFWuiu0UMPyRRiKIHCoOiiucurvUILhrgaUWmU+UCLjeqjqTjAI9z06c9M9RFo93pdr+/upNSYAkcJGemMZOc8/z9gKqpbz3N7uNobdowMx4QjHqWA6fQjt61jPHqs+Wm7JFxhbVnFyWdh5jeZOd+Tuz5fXv/HRXXNpeiFiZbW8aQn5j9mfk9+9FcXtX3NOVHJ/GFiPD1hjOTdZwP8AcP8AjXm+gs7MnzNjcDjcfWvRPjGcaPpyc8zk/kv/ANevP/DwysLHHLf1r38vV6KuRjH+8Z1S7xjDuD7NTw8+B+9k/wC+jT0Xr3qYKMZP1r0uVdjhUmMFxdL/AMvEv4nNOF3eD/lu2fcCn7AOMUu3Bpckew+eXcFvrw/8tAT7qKkGo3ag8ofwpNn+c0nlg0vZQ7B7WfclGr3IHKRn8/8AGpF1qbIHkr+Bqt5YPB/lR5Yo9hT7D9tPuXhrj9Db9uzf/Wp415R1t3yPcVn7MCmtEP8AJqfq1PsP6xPuWpvF9lbShJoLnkZyqqR/OtGLxDaPGrbZhkZGVH+Nefa6Nl6mOPkGPzNdBaR7raLPdB/Ko+qU2y/rM0jphr1l6yf98U9dbsT/AMtWHsUP+Fc95IPAFO8n/wCtxR9Sph9bmdENWsW6XCj6jFPGpWbEYuovxYVzPkgcAUnkj86n6jHuNYyXY6tby2I4uIz77xUgliPSRf8AvoVyH2cE5Iqvd2w8jkAc1DwK7lrFvsdyCCOoNKO3tXk/2chbwKxHOTg47Ukf2q2W2Y3U65yWxK3fn1rP6m+5f1pdj1rjjtRjmvJYLvVQsxOoXgLJvUee3HJ96emp61HpzuNUujIVLZaQnHtzS+py7j+tR7Hq9GPWvGx4x8QRGMf2m7fL0KIcn8q0pvGWtrKscVyCAwDFolOfyFS8JMr6zA9S7c0Y+ua4JfFWrqPmMB9zGR/I1XXxzrPnmP7NZtlyB8rDA9T81J4WoNYmB6LTRXFSeM71Uz9kgOOT8xGaLTx3NPEXfTFTnGBPn/2Wl9Wqdh/WKb6na44pOc1yEXj2OW4aP+zZQq9WWUH+gqSDx5ZSytE1ldqy8kjaQf1qfYVOw/bQ7nWGk/SueXxnpxODDdL/AMAU/wDs1IfHGiK215JlwcZMJP8AKk6U10H7WD6nRdTQSeawIvGugTPtW9bcemYH/wAKl/4SvQhw2pwqf9oEfzFL2c+xXPHubXOeKO1ZqeIdGkj3rqdptHUmUAVLHq+myjdHqNqwPcTKf60uWXYfNHuXcn9aUVXF5ayHCXELewkBqVXVujA/Q1NmO6Hg0Umc8nNGcA0WGLnJozmm5zS5Hf8AWgBc8Z70tNzkf40E9qQxaBSE0ZoAcDg0pJxTM/Wkz7GgCTJyaN3FR7uODSk0ASE88Um7mo91NLf407CJt3/1qaXqIuKaxAosFydX5qQPVMPUgfiiwXLO/rQH75/Wq5fApC5pWC5a3+9AfjFVfMwO9HmHHNFguWw9L5nHPNUxL604S4o5R3LRkpnmc+9QGTjmmGUZosFy6JPfip45PlPSs5JM+tTROefTFKwFvzKaWX0B/CoN/uaQygDPX1ouFiwHA5AA+lI1y6naJZsnoqOwJ/I1Wjd5z8gwv989Pw9auwW6J7serHqaLt7CsluXdMW6ku1kluZiq5xGZCynPrmuoByBWHaLsGa2YjmNfbivSoRajqcNZ3kPpM0UldFjEXNGaSigQUtJRRa4GVJb31tcloJfNhYnCNjK+w9RUcuqXFs4S4te3LDI/ofbvWrL0U+hpc15ssFVhJujUa8nqjRTT3RiHxTZIxVo3Axyw5OfpVlvFWjzCILOVZhwNvtnHHetBrK1mRmnt4nDdQUGTUcmi6dJZ7ltEUGTzCF4+bkZ4+prmksSk+azt1KTjdWMo6vpu8oL1JWZSfLQ5475J44p8enw3YaZIVWZjtIB2jA6fXHPbvVS88M6WINzvLDsAJkD4Ax356VlXFjfW1havpFzdCMF1YOCrTbjhccccA85Hr714VWHNJNKx0XaRdmtrOKaSP8AtdItrFfLEiAJg9MbeMUVCuhakVGLfTU4+60SMR7E45PvRWXspdwuzz/4xOBZ6UvTMsn/AKCtcRoCfLBj/np/n+ddh8ZWITSFz/FMf/QK5Pw8Plt88gsD9Oa+wwC/cxMsX/EZ1qIOp65qUDnPr1oQZ6j61IBxmvTOEZg5NOCj0p23HWnY9RwaAEx/OjHGM89aUDIz7U4cCgBmOeKMdqfjJNOA5NFxWI8YphXA9PepSMnpxTSOOfpVCZyniIf6dFx/yz5H4muisx/osPGfkXn8BWB4jGL2L3j7/U10ViM2kByceWvf2pLcHsWQvGMc08qenNIBke/c1IopiGbevrS7e3FSY70bQe1AyPaB25qC7X9wenUVbxnkVDdg+QenUUmNHM3H35kXq7DP4ClnjEltAo6nAGfcVO8Ya5nbsqY/SoBmSSH0QqvHrWVywmVYpmyBjy8Co4kBsAMdjVi4jEl2i9gMn3xVYOUs1RR87ZA/OmmI5G4jZZSxwNzE4/u84A/St+O3WO3t1HUkEnHU1i3Z2uoPTn+fFdAozBbn/d/lSGaL4jjLnGAKq2cQeWWQjk1PJm4nEIPyjlj60tqu2aYAd+cU7gLJEu08DNUI1CWhUD5mY44rXZeOelVbeESPv/gThRSYIjhsxFEBgbjyTjvVeGAC/kOOueK1mXv61UVcXrk9x/SkwIp4/LjPy4PbNZr2obPr1+tacxLsWI46CmLC3lu5JGBx71Ldykita2KLGH43NWRrEWGb6CumhA+zr+P86wddGCT221fQkj02ITaU2eRlu1XYLcR2qrt5PQVFoLA2Mm4Zw54/AVrWqCRzIcEDoPSkMqxaaqKSQN7dTT/smDxxWoVHbOKbt9v/AK1OyFdmPdwyiEFZJBznhiKfbm8WBNt1cAgYz5rA/wA6vXKZgPHTmmwIPIA9M4qeVXHzMiW61IdL+6+nmt/jTxqmsIOL+cn3fP8AOpRGB2PFQy9lUZJ9qHCPYfPLuNk8R6+p2Rag+7PdFP8AMVNfa/4r0/w++qG4VohII9zwLhiew4q3pGn2b3BN7cLEijcc9WPoKd44vY73QpLaN0ZFUFVjPyoB2x0rJwjzWSLU5WvcwLf4ja+0UbN9lbd1JiP9DXUW/irU5I0Z47clgDwpH9a8qtjttozxlW/LmvRbFQ1pEePu1MqUL7Fxqy7myPFF5jm2hP0JFV7nxvcWrqDpquGGciYjH6VEIxjP61Q1G3DbDj1Gaj2MOxftZdy+vxCOPn0lwfacH/2WpB8Q7X/lpp10p9mU1zjW64+70phtVIIAHFHsIB7aZ1I+IOmkfNaXq4/2V/8AiqkXx/oxIDi5T6xf4GuOa0Tg7ec+lRtZoaPYRD28juh450HHN3IvoDA/+FO/4TXw+2M6mi/7yMP5ivPXslP8qztSsNthLID0APQdjTWGiH1iR6unizQXwRq1qM+sgH86tp4g0mQALqdmf+2y/wCNfPmP3HX7rfz/AP1VuW9orwo/qoODSlhorqCxL7HuC6jaSD5LqBvTEin+tSCeM/dcMPUGvExp6sp4HJ6mo308AfKMVH1ddyvrHke4h8nj60hfGM5rw4wToMRyyIe2HIogk1guFhv7xfpMw/rR9W8x/WPI9x8w+tL5nPWvFn1DWrQPnV75mCZ5uGOPzP1r2T4SSrr3hSaTUh9puYrtozJLy2NqkD9TWVWl7OPMXCspO1ibeOgpm/nj9a7g6HprHm1X8GP+NRP4c0xh/wAe5H0dv8a5vaI1ucekgPepklwf8K6M+F9PySDOv0f/AOtUE/heDyX8m6uFfB25KkZ/KpdRFJmG0x3BVyWPQDmporVnIaY9OiDp+PrToIFgXGMt3J6mrK81W5WxIijgYq3CvNV415q7CvIrelT1MZyLsIworTgbMVZydKu2rZUivQgrHDJliikNFaEBRRSUALRRmkqkJiPyh+lIDnHvSnkGmRn5B6ik0CJpXwAgOB3qe1O63kXuOapn9as2LfvGXuV6ViqSjCw76mPq8DOQ+AyKufLPQn1Pr9P/AK2MGae7nLy+cwVG+YnoBzx9c4rqNThaWJlLGNOQSvX/AOtWNaxRXFpLaNiMCRANvfJ4rwqvuuT5dE0dyfuopCe8KgjkEcHc3P6UVea21gsTGyKmflUSAADsMUUvaw7IV2eQ/GRiZ9JUdQsv/slc5oAwlt6bv61t/F2bzrrSDn70Lvj1yVrG8NjEdr3A6V7GC/gxMsV/EkdavHtj9KmAHH6imooxT1GDn+leicKFxxRjHvTufWgZx9KBjQBmnBR6daXHNGOKQxCBnqetLjOD607b2HWgLzTQhCM0w9s9DUuPpTGHpTJZyniMYvIjx/q/6mujsB/ocHT/AFa9PpXNeJm23sJ/6Z/1NdPpw3WFuf8Apkv8qS3B7Foenan4+uaaFI4z+VPA49cUwsHIzR1FOx1PB9KNuRx0pXHYTAbtUN0B9mP4VY9sfhUN2ubcnGOQf1pPYZgthZLkngAD+VRlRFbwE4GW3GlnDNLKnY/MeOwHSnTKrRW8fQtj8sVkaWK8YLTl2z86k49B2pkEQETS46qQvsKtTMI7hiSMLHio0AGnnjHymmhHH6iux075zmt2ED+z4GPQKOv0rC1RHW4V2ztYjaPbHX8a1rMvLZL1CRoB+NDEjatIike5vvvyaLcf6VMKmhJMSNzyBUO8Q3UhPHy5/lSbGLOxdxCnU/eIHakswQsg9GqS2jJBlcfM9JaH55R70rjJdueDVOfakjAck9fYVclk8tCf4jWexyx9TSkxpDVjMjgD+VW5IwkBHoKfBD5aZI+c0s+PKY9OOKEJlW3H+jDI9axdcXKnjOUrbg/1HJ9awdek2ywxryWyD9Ku+hNiHQAziVBwobcfy/8ArVvaeflbjnisTw02XuUz2U/zrcseZH60uoy6BznFKRk04Dj2/nQeOcc1VySvKuY2+maitV+U9OuatOMofU1BGDDHlgcnnHpUNlJBI20YHJPTFIsQU5Y5Y1IkZ5dx82Pyp5AJzjNNAyFhVLUY99hcL2KH+VaJHyjPSq1wu+J1xwQRVXEefWozanHUPnp7V6Lo7B9OhbjG3v2rzm2zsmRuxH9a7/w44fSo+nHFZSNYG0Bxz0qrfp+6BHUNzirQ/wAiq96pNufYg1BZmFecnp3ppUZ44OKk56/1pvOOvJpk2I2GevSmFOPXJqU/mBTT/nvTuBXZevb6VVu4t9lOg/ijbH5Veb6VEVBOMcHiqTJZwQ5gkHuD/n866PS/nsYWxggY/XFc8g2iZOny/wAq3dEbNmAeqsRjNOWwkagTI6Ch09PrzUsalzgDmr0Vrt5cc9ayuWkUIrJpW+YYX1q2IFjXCgCrm0AYH6VG64BPb1ouVaxganHiXp95P/rV6N8B7rNnrVqTyrxSgZ9QwP8AIVwGpphYyB1DD+VdR8D7gxeLdQtScCa1bA9Srr/QmssQr0mODtNHvZOcGg0g6Y9KWvIudYzvSHkGg8GkNQ2Uc3ewmO6ce+R+NRqvNampxZKSf8BNUAuCPfmuyjrFBJkkYq7CMCq0a1bjGBXfTics5Fhat2xw2KqLU8JxIK6EYMu5opKKskWikzRmmJhmlpKKYgzUa8Fh71JUfSQ+4oaEKaltW23Ke/FRGhW2uG9DmpktBk9+uRIvfrXD3MLPe7SSo7kdQPb1Nd7fKqgzEgLs5J9K4q5R5LlpJI9kefkQjlh6sD/L8/Q+XVdp27nZSd4Dc3X/ACz09Cn8JMRYkdue/wBaKYwYsSZHJJ9aKy9kyrM8i+Lv7vVdOjBO1YG4x/tHH8qy9EuVggtWMbMFQZx3rQ+MDA+IrJT2tv8A2dqy9MAFnBnBOzr+Jrqw8uWhFk11erI6hNahxkwzDv0H+NTDWbYgfJL/AN8//XrDXnp9KlH6YzW31iZl7GJuDV7U9RIP+AUv9qWmc72BH+yaxDx3pVPbNH1iQexibn9p2ZH+t59wRTxqFpx++A/AisIdMjrR37U/rEg9ijoBfWp/5eEH1NPN5anj7RF9N1c7wB0ppAzjGKf1l9ifYo6Q3MH/AD2i+u8U0zxEYEiH/gQrmjg88flULgDnj8qpYh9hOiu5F4slRb6Ahs/uzyOe9dZpbqdNtiGGTEv8hXm2suEuY+3y1vaaF+xwHGPkH8qp17K9iVSvodyF9P51IFPrXJqfQkfialUuD/rJPwY0vrK7FKh5nUgHrjn3pCMgY9ea5pZJs/6+X/vs1ILi4B/18v4tR9ZXYPYvudFg96iu1/cH2IrGW7uf+fiT86Gurhkw07EehxQ8RHsHsGRsuXu2I5AA/SoEBeaGQ/dBCqPoKny5LfPy3B4600bxtAbhORxWftUV7NjLiPzLph1Crk1FKfL00Kf4kIHPepxu3M+RuYYJx1qtPGSgU4wowOKaqoXs2c3ri7Xt8enX8K1bBAdLTAHK8/WsfxA7i4jDFdoUYGK09JkdrCMDG3BGMVbmrEqGtjZsyDbRn29abIA90sfQdz602BniiCKFwvrSpGUkWQHJBzz3qXURSgy5joB17VWtyFuZgT71J5rcZQc+9REEs7AYLe9J1ECgyKeUu2c8dqbCB56cf55pxhJOccd6eR+9V1XAUYxUqauPlLYHI7daimJ8p8eh6UpnGMbWpjSqQRtbnPatOdEcrKqy+Xa+jZwPesrUYCsPmONzkH8K1QF8zc4JUD5RiqGs3MSwAnPXHK+1OMkJxZleGW/0ycZPKf1/+vXQWX/HzIoJ/H61y/h6QR6s+ehUjp7g109uwW5LfwnPNNtIFE1OR/hSZ4xmmCVDyD+GKPMQDr09KOZC5WKx2g889KrwtvnO7qBT12u5Z3AC9BUUOFuskYXkZJqeYqxsWOnvdAuwKxDuB94+grVGnIBvZOeylD0p9vLarFGxki2gYVcjn681R1HVFkDRW7jn7zjj8B7VnzOTsXZJFPUGgRzHCEb+84HH4VlSLuUgg1YbDDsKhZeuea3jZGLPO9nl3t3GR0JA46c12XhZy2nnnO1v8a5K/URa7dj1JP8AI10nhCTMMq56H/D/ABqZFwOr71Fcgm2fnoM09TikmBMDjGeCOKg0MnnB9qQ9fYUdevSgn8qLiGkGmHvwSKcf/rUh75P9KAImHH+NRnjnP5VMRxjH5VAeuRxTTFY4yaLZqdxH0+ZwM1qeHYmkjlQZ4YH86i1K0K6rPIeFBDAeuQM1a8NNtubiPIyVB59v/wBdVKWgorU6aCJYx8o59anGAP61GuAakHB96wubLQXtUcigjrz3qXIxg9PrTGOeKaYmY+qL+5BGeG5/KrfwvuPsfxNshnCzb4z/AMCQ/wBQKh1Bc2zjv161n+H7j7F440W4yFAuoST7bwD+lOSvBonZo+qDw31pe1EnYijORXiHZ0Iz1pD0pW600nAznArNlIhuo/Mt3HcciskCtW1vba8lmjgkEnl43EDjnPQ9+lUHj8uVk9DXdhdrE1NBI16cVaSokFSpXpQRyTJl7VKhw1QipVPIrYyL1FNQ/IPpS5qkSLmikoqhC0UUU0Ji1G/31980/NMl4UH0IoYhTTTTj0pppDNFsSWsTEZwP/rVh3cCtdAFFYsQBuOK2rZt1lj+6cVlaomORXmYuN2deHfQQ6hZIdggiIXjOOv6UVjhJMD5M/nRXF9XX8xryI8L+LrhvFEC9harx/wJ6z9NnhSzhUyICEGcsKufFaT/AIq9RkjFqgOPQlq4piRI2DwAOa9GjG9GK8jOs7VZHcpNCQMzR/8AfYqZJoiOJFI+tcADjjP4U7cT3xjtVeyJ9oegBl45H509WGe1ee+a27KknHpTvPk6iR1z6MaXsg9oehD6Zpe4rz77VOowJpPYhjTxe3IU4nlx1++aPZsPaHf5Hemnt6muDGo3isCLmYen7w07+1L0Di6mI/3jR7Nhzo7Yn/IqJzlcAfhXH/2rfZH+lSdu9KdXvQeblz9QP8Kagxc6J9eP+kxeu3ofrXRaXzYwdT8g/lXF3N1JclWmfc44yRV2DWbu3hRI3G1RgAqDgVUotpIUXZncJUy+mOO3NcSviO/Gf3sZ4/uCnDxPqC/MTF+K/wD16z9my1NHa9sU4+o/UdK4seKb4Y3LB+Kn/GnjxbejgxQH2wf8aPZyHzo7IHBHH6U7P41x48XXRwfIhPr1/wAad/wls2ObWMn2Y1PJIOdHWkgDGaM5A6+tcp/wl8ne0Q/Rz/hTh4t6k2g44/1n/wBanySByR0+7I9aic+lc6PFy4/49D/38/8ArUp8Uwtw1tIM+4NPlkLmRV8RsVnjY88Vd0Ns6dF17/zNYer6kl+VaNCMDByas6Xq0VtbCOQMSCeg96tp8pKa5jrY2AGKlz3rCTXrUcNv/Kpl160PV2HsVrOzLujYyM8A0bvSsga7ZMB+8Iz6qaX+3LM/8tfzBpWYXRqlunIxRkdulZY1qzJ/1w/EGnf2vZ8ZmUfnRZhoaWRznA/rTSe1Z/8Aa1mQR568fhQdVszj/SE/OjULouOeKydYG6yPHRgastqVoc/v4/8AvqqGo3cMltIFlRiOwIqo3uSzF0ZtusLweQw/SuziI4HT61w2mPt1iJmYbdx5J9q7OKQHHI6+tXPcmGxfB7nNBHHPFRq3HBBz75pc547+lTcsdn+VJwM8A0mc0h6UrisOJX0FNIB7Um4delIDTTGIQmTx9ajZVHf24NSMT1/SoZG46002TY5HWkSLVJSAQSAePcVZ8NyNicKzLgjkHHr/AIVH4hGL1CehQfzNReG3xdzoTjKg/kf/AK9a/ZIW510csuR++kxjPJqUSysMeaxB681VjP8AKpwc9xnNZ3NACAcZORR5QHHSnL+eKM/jSuFiMxZH3qQxlT94496mOcDPNIev+TRcLFfyuPf6U6O3GeeakJA6/jTkyM880OQJHPa9Ftutw6tF/iKz9Ck26p7OrD+Rra19QTA5GRyv8v8AGue0tymqQMT3x19aqOqJa1OzBx0P5VKv0/Oq6tx7VODzzjmszQeDjp+NMZuOn60pYZHb8ajY8HP40XCxVuzuikX1Q1y1zMY57aVW5Q5B/KuslUE7T0PFcffLtiQtn5Wwf8/561pF30M5I+vYJlubOKdPuyorg+xGach4IrmvC2t2kfw+0fULy5jhiFpGjO7fxKNpHucg1h6j8RbKR3is5Wjj/wCehQ7j9PT+f0rxnCXM0kdkNUdlqGq29iMMd8vaNOv4+lcpqWszXmRI22M9Il6fj6/yrmJfEto5OJTycknOSfeqkmu278+bx3pqjLsdEeSPU7rwjd+Zq08JOQ8W78iP8a6O+jxMrjow/UV534N1eB/FFrEsgJlDJ/46T/SvT72PfBu7qc100Vyy1MKzTehQQdKkApiipBXoxOKQ5etSCo8U4VqZsuRHKfjUlQwH5CKe7rGpZ2CqOSTwBVrYgfR3qKK4hnz5Uiv/ALppzSohAZ1Unpk4pgPozSe/agMD0IP0NMQ6mSjMTfSnUHkY9aYhqnKAjuKaabF/qgO44pxqRl7TmyJE9QDUVzGpXc/IXkg96LBttyAf4gRUt0H2soQH1+bFcONXu3NaT1MVruTccPIBns2P6UU4xS5+8B7Bun6UV43NDzO/lifNnxQfd40cdlgjH9f61xzDErZwK6z4ltnxrc9OI4x/44K5UfMzA859K9uhpSj6HLW/iMASDQfqadtBIpcdBjr3PetTMbuPt0pDnOfWnlcc5xmjHzZ7mkAwHHr+PFGOOAak2Z+7/KgLz60XAjz0xgZ9aXtjnFPVSc85BOMYpAvTIPv7UXAjJ5OOtGecgAZ7etP24GD+RFIVA5yOO1O4rEbcc5HXHXpUoPyj2GOtLb2st7dQ2ttHvnlcIiA9WPQV06/DvxPgZ0zBHX98n/xVKU4x3Y4xb2OW3YHH5005J6V1q/DjxQyknTlx6+fH/wDFUH4a+KWP/IPUD1M8f+NR7WHcv2c+xyeec8AijP4jtXXp8L/Fbgj7BGD/ANd0/wAaU/DDxWQB9gT/AL/p/jR7an3D2c+xyKybCSQD9RTS5Jz0rqn+HHieJyp0/JPYOpx+tIPh34oYhRpp3HuZFx/On7SHcXJLscvu57fhQXH4d66d/h74nViraa3B7Op/rTD4A8TL/wAwyT/vpf8AGj2kO4ckuxzSt2z+NAcZBx165rov+EC8TAZOlSe/zL6/WkbwJ4kGf+JVJzjoV/xp88O4uWXY5tiQPb+dPjY7SQeK328CeJW+7pUv/fS/40+PwF4oyd2lTcejL/jT9pC24ckr7GBvxznjuKBIQO4xx1/z610f/CCeJSD/AMSmbp/eH+NNPgPxJk/8Sqbkccr/AI1HtIdx8kuxz289z9OKUSFV684rfbwJ4kI50qbP1H+NJ/wgnib/AKBMx98j/Gnzw7i5ZdjA83pwc4xilEpC56Gt8eA/E/I/suX/AL6X/GlHgLxNg/8AErk/FlP9aOeHcfLLsc9vOeaaZDjnHPJxXSp8P/E7kgaVIOOcuo/rUTeBPEm7H9lzf99D/Gjnh3Dkl2Of80sfx+lIZRwM8jv610B8CeJhx/ZUxweuR/jSf8IJ4mOP+JVNz05H+NP2lPuLkl2OZVyHHrmnGQ9Mmug/4QPxP0/sqYY9x/jTv+EB8TY/5BUn5j/Gn7Sn3FyT7HPCZxyHI/HFO+1TqQPNfj/ardPgLxN/0CpRx6j/ABo/4QLxMTj+y5c9eo/xp+0p9w5J9jFXULoLxczc+jn/ABpw1O9A/wCPufA7eYa2G8B+J++lS/gw/wAaT/hAfE+f+QVJ+Y/xo56fdByT7GQNVvVb/j7uM56GQ1IusXwGPtcvt85rT/4QLxPnH9ly/XI/xpf+EB8TD/mGSf8Afa/40c9Pug5Z9jN/tvUOn2uT8e9OOt3/AEN05x64P9K0l+H/AIpbONLf/vtf8ac3w+8UI206Y+R6Op/XNLmp90Pln2MC6vri5IM8hZhwCQOn4U20vJrOcyRNhiCDxnit9vh94nBUf2Y6hjgEuMfzqVfht4oLAfYBz/01Uf1p+0p9xezn2M2PxHeoP+WTY65X/wCvUw8UXg48uEn/AHT/AI1bb4ceKR/zDh9fNT/Goj8P/E69dN4/66p/jS5qb6j5anYaPFVwOsEP05/xqRfFUmebVOP9o0f8K+8UZx/Z/PtKn+NQ3/gvxBpllJe3enmO3iALv5iNgZA7Glem+o7TXQtL4qYfK1shPs//ANan/wDCTqRzafk//wBauTDds9qcJD2xmn7NE87OsXxNEcZgcZHZs1NDr8DSY8mQenIrjw5zz+VWbaU+cOvPH6VMoaFKR0Gq6hFdQoqo4YHOD9KwI5PLvUfHCyZOOe9W5GyvTPpms2UkSv8AUdKVMJM7BNSgx1YfhUyapbYA3N/3zXNo3yZGee9TbuAeMVm0UmdF/aVtgHf39DSG/t848zr7GueD4XuD/Oq09/FBxnc3oMUKLew3Kx0sl3b7c+aAPWuS1S9haaWKI7gz/eHTr2qlcX01wTlgqk52jpVU9a6YU7bmM532N/S7jXtVRLG0u2aK1Q+XE7DagLZOAfc1pPo/ilQSTE3/AABf8K5W3u57KdpLeRo2PGVPatFPFGsIu37a5H+0Af6Upwlf3bFQnC3vXLstr4kjPzRr9Qif4VXLa2n3ox+KrUDeJNTcfNcZ/wCACoW1q8cYMg/75FJRn1SByj0bOj8Jalqdp4x0aWZNsYvIlc4/hLAH9DX1Yyh1KnoRivjCHWLmOeOXeco4YY9jX2bbTrdWsNwnKyorr9CM1z14tNOxpSle6MwLglT1FPFS3SbZyex5qKt4O6uRLcUCnCkHWlrUzZPAe1Vtatp7rT2jtwDJuBwTjIzU8R+cVLPJ5MDyYztUnH0q1sR1OehgOn6hCqOxdgpkQtu2k9ce1T3llcT3Vwzs8SAFllGCOnAwar6NLFNK95dSr5hOQDV/V7gsI4I2+V/mLZ4x2pK1inuQJcyto8nzEbXUZ9jjiq9u0lpdQzeaWSX+HA4wcEe9afkRDSpYIZEdthJ2kEk1kuwksoSDkrIR+BGf6VRJYvvtE97OkcgV05QEZHAzitTSblri0+dizKep7is+V9uqQSdBIik/yP8ASptFJjmmhPbgfgcUxGoOGce+aDyaDxMfcUGgCSBtlwjdgwrVlTLH3FYtbhO9Eb1GawrxvFlRdmZTRfMfrRV0xjJ4orxeQ6faHyP8RufG97jP3Yxj/gC1zeCXbgn5utdT4/iabxpevEu9D5eGXkcIvpWLiGMvvQh93rXqUnanFeRNTWbKew4zjPtSgZ4P5Y9qurdQINoTGfcUCWy67G/w/Wq1JKYQnHXigIQQcE1cEtmflUNycDk00SWhIxkA8UaiK2w5G0/pSgZyOg71o2tvHdIzp0BxknvU/wDZygHp+JqeYqxjleTxz0pPL7dxW0umqD8wwM4+8KeNLhPBzjt8wo5h8pglQeg6DpSMhXqD/Sug/s2DByzc+4xTG0q2Yf61h+VCmHIyt4TbHjHQ845vY/8A0IV9WBI2wSi5x1xXy9otutr420NFYsPtkRycdd4r6eD4Ue1ZV1zNM0pOyY7yogf9Wn/fNKY4j/Av5CmtOpG3bz/eppc1j7M15x3lxZ/1a/lS+VEesa/lUfnBOSCfpTRLnmj2Yc4phj37gi5+lKY0/uj8qTfSbqagHMSCOPH3R+VOEcf91fyqIOPWnb6OQOYf5af3R+VN8qM/wj8qN+aTzKOQOYcIowfuj8qcET0H5VHvpd9HIHMPCIB90flQUT+6PypnmUhejkDmHbFB6D8qCqf3R+VMLn1NJvp+zFzD/Lj/ALg/KmlE/uj8qbvpN9HIHOO2J/dX8qjkRNv3V6elBemSOMelKVPQamORU3H5R0pCqbVG1fyqNHGR7ikD8is/Zl85J5abD8q/lTPKTZnYMn2pDL8nPrQz/u/ej2Yc5II48fcX8qYY0JPyjJ9qTzPlzTRJl+e1P2Yc4540AHyj8qUxRkfcX8qZI+QPrS+YCAaapicyNoo/MHyDp2FMeJGOAq/lTnk+bPHApitxk5yTT9mLnI3hjUgBVx9Kq6leWenWpknwgPyqQmT+FWLqQQxGRs4VSxx7Vy2k66dS1BJ5OoXft3cKD0FXGlcTqGsLyW+YjT9Jnu2XpJKxRF9hk8/gMU28fXbKDedDWQdT5bq2B9MZrsbOePylVFHTt3qW5mKJyACe1WqMbD523Y5HT9QtdSjO2IJMo+eNgAR+FTPbxscCNffjpVm9trcM12sCpLjJlTqcdjTIvmQOP4uazUE3Yc20QrCgfBVevpWf4rtY7jwrqsWxf+PSQgbR1Ckj+Vajy4YqPp265zn19qi1FBPp1zFjPmROv5qRVxp2Zm53R8sYGf8AClGO5xjilKEZ/XNLtLNwO3YV3HGABzwRx+tSwfLKp9xyKaqnrj86eAByADzmpY0aDkYwep55FUbknzgfb061oOBtGOh68Vn3vRSPT/P9azp7ly2LsBJiQj0HINNmnSJNznHp6ms8XpihVFUZHc1VdmdtzHcferVO71Icy1cag8o2x/Iv61Sz70UVqklsQ22FIaWlamIQ0lL2BoIxQAlFFFABX2D4Avf7Q8AaFcZyTZohPug2n9Vr4+r6f+CN79q+G8ER62txLD+u/wD9nrnxKvFM1ovU7y8TKBvQ4qlWpKm+Jl9RWZ3qaL0NJrUWlBpKWt0YjlOGzVllDoVYZVhgiq3selWIzmJCepUZqkQzOXw/YJ9xZV/3ZW/xqe40uC4WNS0i+WNoKsRxV2iqshXZStNMjtJDIskjEjGGOagbQofOLxzSIhOfLGCB9MjIrUpadkF2UrzTI7uKNQ7RvH9x19PQ+opbDT/se5mk8xzxnGKuUoNOwrkcgxIh+opSKJfug+hBpTSAbWvaNvs09uKyDWlpzZgdfQ5qJq6GibFFPxRXD7JGlz4cRzGqgMR9DQXbcQwIIJzwOKigDMiDPPtV2HRNUup2EMeWAzy4rsUbsnmsVxKf9rnpTllx/CRz3FXf+EV1wD/UAD/rov8AjUi+FdbJI8lOn/PRar2b7C513M4S5A+X/wAdpfMzwFOf92r3/CLa4Bxbrz6Ov+NIPDeuKP8Aj34/31/xpezYc6KqzOBsUuB1wq9fU8fhThcz5wHm9OCa3NM8NX7wubm1y4bj5x6exq+3hmcEFLMj2D/T3qHFp7Fp36nKrdXLOMPLyMd6mWS+I+UTfgp5rpE8N3SsCbV15/vj/GpRpF7GSEtW46dDUO/RFxt1ZzAGqsOI7kj/AHSf6UjnVYhl4bkZ6kxnn9K6wW1+gINoT/wH/Comh1DdhreQZ5+4QKlOXY0tDuY/hqS4fxdpAlDgi8iHzLj+MV9Ng8V86WEMkXjHR2ljZCbuLqMfxivoodBxzRNXsKDtcUmk8wCmnOOBUZyTSURORJlSc0uVxUADMeBk0bucGq5Bc5PlaXK/SoA1Juo5A5yxvWjzF9arbutG7mnyIXOWi60b1qru/wDr0bsfSjkDnLW9aN61V3cUu+jkDnZZ3rSF1qvu96Tfk0ciDnZZ3rmk3r6GoN31pC9HIHOyxvX0pN61XLe9BanyC5yxuX0qOV129Kj3n1qOV8gUOA1MkEibl4/hpodNucd6r78BW/2TQG/cn61HIh87J/MX0pTInl9O1Vt2dx/Cjd+7x6ijkQ+dlhpUCYxjNIkibuh5FVWYk+3rTgwWUfTFCgHOyzNIu3pThIpA44qpK3yU7fhc+gp8iJ52SPIpIXb1pwZBjjgdqps/II6Z9am30+RBzsfOUcAFRg8GvNoraeDxBb6bDBNCPNJZnTaSgPBGRzmvRJG4BAHHY1Unso4pEvowi28IwEUcuzH5iT6jg/hUy0RrS956l+eOYWojg34bhismw4/3u3865rRtOnOtSHZDGm75zHcPMSvuzd66h71LS3eQsGCLnav8R7ViPrRSN3j1CwinzxAyhlA+gIOfesU0dai9ypqkN5FrckWZmVDgGOfbhTyCV6HniugjISBEGPlUCueXWZr2fzrqO1LghcRHOPfNdCqkoG74FOjZtmeKTikV5Tlzjil6jB78VIYGfc6cqo55/wA+lJj5PwrblORM8Qm0vQ4ruaJ2td6SMrAyAEEEj1pV07QO32UjrxL/APXrP8S+GJm8T6myMQr3MjKNhPU5/rWSfDFznhuv+yahxj/MaqUv5Tqho+glRxBtPTE3X9alTQtCckAR8df3xrj/APhGLofxj/vk/wCFMbw5dr/EPyNLkX84+d/yHdnSNExgAE4xxKTXmk3myXLx5LFSQAPb/wDVVx9AvFGcrj8f8KzJY3gnaJ/vKcGtaNNRvZ3Mq03K3u2NW1sraXTpPNt7gXe/CNn5NvuOvrQdKjwRmQH6j/CqdhY3N95nkMP3YyctirH9i33H7xf++jVt2e5mtVsbumaDod7LHbsb4TMvJyoXIHOODW2fh/oythrm5U/9dF4/8drio9J1RX3RS7W9RIRU62GujG28kHGP9eRisZKTekzeEopawOw/4V5o7Di8uh/wJP8ACsjxR4LsdF0Y3lrdTSOrqpVypGD9B9Ky/sniQdL6fj/p5PH60k1h4iuYDBNdyywsclGuCRn6E0oqaabnoVJwcWlDU5zHy/jXqOn/AAu0++0+2uxq0w8+JZNojXjIBx1rhh4Z1TBHlrjP98VrQQeLreBIYb6dY0G1EE/AHoParrScl7k0jOjFRb543Omk+EluPuapMT/1xB/rUDfCOTkjUZAPU23/ANesR7nxvZwyTfb7oIiZZhN0A61mt458TM2W1i4Y+5B/pWMYYh/DNM1lPDreDR05+Etx/DqOc+tuf8a9Y+Emg3HhrSdQ0+abzVadZlOwrjK4PX/dFeNab4g8f6lb/aLG6nli3FdwjjxkduRXp3wo1DxbNrl5D4iMjW7W+YiyIMOGH90ehP5VEvbXtOaY/wB01eEWj1ys2dNkzDtmtKql2n3W/A1rSdmZTWhVxmnCiiupGItTxnKfiagqWI/KRVIhktFFANWSLRSUUxDqUU3pS0wElGY29cUincit6inmo4+I9voSKTADV3TGxK6+ozVM1PYttul9+KmWw0aeDmipNtFc/sx3Phi2X5ohkfeGB+Nd3og33koGANmefqK4vyHtrxYpUaOWNwrKy4IOe/pXcaEo8+ViRwoGMe9dNPciextFCo+8CDxgGmjhs9Oc1Lgc5YfjRtHJ3D866DAiIOcY/WjrjipWVeDuH+NMKj1H4UAWbPGw+zCrJIJ/lUFooKMM55FWShGOKzluax2Gd8/570hyDkYzTyMgDFNOAen+f8ipKIznGaRueOMDtT9pK8g9cc0yRxHGzuflAyaQzm9ROPGvh4cZ+0xk4/3xXuykEV4HOr3XjTQyTsaS5jG4/wAPzjFe4rHcAf67r/s1zVHdm8NEXTKxjMYxtPWmFeCKr+VOf+W36U/ypsH98fyqUDHrK0GSuNx45FVtxJJJ5NMkhn3czHGfSomik/57tWiM2WN/vRvyODXKeJ9UutGa0ZLlwshYMMDnGPb3rBPjG9z/AK4kd+n+FFxpHo5fk0F686Xxfc952z9B/hS/8Jddf893x9BSuB6Jvo3jHWvO/wDhLLnj/SJfTjFH/CU3P/PxKKOYLHonmYo8z6V53/wlN13uJfzpyeJLp/8Al5l+u7pRzofKz0LfSF689fxPPk4uJv8AvqmjxJcHn7RN+LGlzoOU9F38UheuHg1yR8bppv8Avsirg1MuARPL/wB/DUuqkUqbZ1u/PSkL1zEl6yxqfOk5H/PQ1GLtmVj58/H/AE0NT7dFexZ1W/nGKbI3y5rk2uXfhbqVTjjMjEfzpkF1Luw1zK5H+22PyzSeIXYaoPudKXJRfypd58tuvWueW6fGfMbk4yXNMub5kYgSt17Oaj6xHsV7BnSK5O7r1oVm8scGuT/tJgf9ZIf+BmnrqZK8Fj77jT+sLsL2LOnUsX5HA60rORKCAcVyU2pkDBZj9GqodSbPLt9Sx4oVddhOkzuJZDs7/hTTIW2oB27VxB1N/wC83/fdRnU2xncT/wACq/aoXszupHI2ipd59D+VeenU2yPr2NR/2r1GV6/3u/4U1UF7M9Cmf5QMH6VheLr7ULHw5Je6d5onjKqwU/KUbg7l6EDrzXMf2qWA2FMA8nOc1W07VTP4nsoLiPNuzunzE4LlGwfbB/ofprTXPJIl+71NnRvGkKSpp2py7hkRw3UqCMS4HRlBIVvxIPseK7JILeSB5bcWyyNks6HYxP1HU1474i0B7JftAWS6sJm+4+A8LZOcuePxPX19ct9cuINIll0zUTaCGTy/ss9yHcjpkKVPH0bjHSnUw8ZXlB/I1hinC0Zo9M8Q6hYeGLWbUnSGS9kO2MMxZpG7AknOK7PTbmS80m2uJ4BbyzRK7xE52EjOM1856bfvNrUWpa002pwWkqlwZTgjPGMjpnGRxXt2meO/DV+FRpms2PQXSBVP/AhkY+pFZQpciCtX9q/I6CSRkwsbMMjBx3qPcdmKe1pHNGGhVHRhuBXGCPXPTFVltEBIKAEHBBFUzJHF66Fj1y5wAckEexwKoYI+YqME8ZNbXiO2EerjaAoaJT2HqP6VlLgdF3dgT0FebU0kz06SvBCLtCdCDjpUqqzfM4IA4BJ/lSLEW9efXpU4gZsKCXPsOKybN0iPfGCp2MQvqePrivKfFyY8TXTdd+1s+vyivZoLWSH5wqBhzucBsfnxXmHxJjf/AISGGduTJAuTjGSCR/hXTg5/vLHNjIfu7h8OU8/V76zzgS2x7+hH+Nd//ZDrPHKkiEqwbAXjO0Aj+dec/D9tvjOCPr5yOv8A47u/pXt8FiXYfLkDrWeOly1B4OCnT1Maw08R72nij8x5C25RnAwBjmtODS7aRG3QoOOXZRj8PWtRLQn5VACnsvQ/41bSzRAN2Sf7orzZVjuUIx0MZNFtJDxAhGeWMYqdNEskbBtoHOOMwittYScBQRjpnrVmK0I9j+prL2smJyitzGi0OwyWksoST2Ma1ai0OwPSxtsnqTCv+Fb9tpbyHO3j16VLPPpul/6+UPL2jXk1rGjUkrt2RyzxEb2irswJPCVlfWc9v9hgPmxsnEYHUHvXyrq+kT6Pqs9lcgLJE5jYA5wQcGvsX7Rq2ppts4BZwHjfJwfwH/6q+UfHttNYeM9RtbiTzJYrhgXP8XPB/EV62AjyNpXaOKvJyV5Wueo/CDULzWfCdxotppNqRbzDzLjaASG5+Y85PB/SvUtH8JNpUjXs90zzYPyINqDPt3ryj9nLUhFc69p5I+ZYpl/Asp/mK92muNylaK0acKjb3JjVny8sditUc67om9uakpOtaxKZm0tK67XIPY03rXWjnYo6VLEecVHToz8wqkQyejNJmjNaEi5pabmlBoAWlBptKDTEOzUUZxJKvuD+YqTNR9Ln2Zf5H/69DAc1PgbbMp9DmmGli+/SYHQUVCko2Ln0orMZ8RySySgTSyM8hAZmc5Jx7/Suz8NOJInlH8SKa4x1/wBFGMZ2c/lXT+DZN+mMc8ghP8/nWlPcmex1O4+uKQt7cZ+lR5OcDrSE5GfTjHrXQYWJlbKkehz0pueuO9RglSeOvFB4buaQy/YyYdkx15H4f/rq6WHTuB1rFhk8qVG9D6VrjpgHI9fXpWcjSGw9WAPTAphwc4HvS8FR/Kg9ep5qSxB15qtf5+yEY/iB/WrBGMc8VXvMG1b04/nSexS3OZL7fGegt6XUf/oa174or5/m/wCRt0Pn/l5j7f7Yr6CUHFc8jRDD8uAFyfX0pecGnlaCODQhMiRow375Sy+gqqRycDjPFTkcmmEVoQ2cH8Rbd54tOVWI/ePnH0FcQNKnOMvJ+letaxYLf3NnG3RSxH5Cki8MxFg23j2FHJcXNY8qGlSn+OT8Mf4VKNLkA+9J7f5xXrJ8Lw8YU8e1PXwxEP4Kfsw5zyUaXJ1zJ+OP8Kd/ZcvpJ+f/ANavXV8NQA5CU7/hG7f+5S9kx+0PJYtKkZhnzfzqzNo0kTFY1kK47t1r1NfDkA/gq03h+3PzFB0qXSdylUR4u2mTDqkn51LFpMhYfI/5162+gQZP7uhdAhH8FP2LF7RHAWHh+OQAusv/AH2RW0nhuzOB5Unv++b/ABrsbfSI06LirX9nJn7v6VhKhK+5qqyOPbQLFFQeUzd+Zn/xpr6HZrExEEgI4x5r4/nXWz6axA2r2xkV4bqdl8UTqV0tquoG3ErbCNmMZOMcZrP6vLuX7ZHdHRYd5KoyjH/PRv8AGq/9jxliQDn2dv8AGuFXSviVKy+amqnb833kH+e9TvpvjtruB1ttVt0Vhv33O4Nz3z2qXhn3LVddjsU0yJcfL0PA3Hj9ajvNNUNjbkf7xrpEgJjO5RnpTbu3AzlQAKx9i7mvtFY5JdODNjy8/wDAjVpNJhKD9znHua3FthuOAMhatJbjyxx7UpUpAqiOOudKUcCIA+9Z/wDZq7gDEvJ/u121zbAyfhVFbQSXWRyF561UKUiXNXMIaLDtG6FCT/s0f2JD/wA8UyO+K6pbbdMMjAA4qlrkv9maRdXS43RxnBx/EeB+taKmxOSscbeSaXYuY2eHzAeUABOe/NY1zq9oMmKGPjjIQHFcpe3bCVmLEkccnrn/APXSI/mRknP064+ldsKCW5ySrNmrcaxKzrgoVBIIPB/PtTLW9dNWs58hTHOmD2wG5/n+orNVnjbg8etPC7AWyQ2Mgjgj0/pXRBKLTRk5NnsWoy6fp93Ml5eWqRyqGkhldd31C9efpXFXPhPR7tZdQtEzBk4WGXMZI64x/LIrTsdFtbmyR1R2MnzOzgBiT1zzk/jTk0q60ouNOZYzccSKMKvH8W37uR2wB1712VsDKcvaLS5UKy5eV6nLWtrZ6ZFc2t3a3DSXabWjhjBESngEnPUdcAc1y5lmgDwrJu2sVztIPHfkZ/OvUodHMJJlyTnLbmyST3JP0rzvxDZGy1m5QBtrnzF3c9ef6VnXoOEEyea7LGieK9X0GXdY3bxp0aLrG31Q8fpXpvhj4oWur30VlqUQhu532pLGcozH1zyuT9a8Pc4Qc/Wrugbn8R6aBkk3UXA/3hXE0UmfQHiaBGvbeRyOYyMepB/+vWOttkDJVU6DH+ea6bxDAzi3cLnDMMk8dv8ACsWK3LHAG4+grxsQ7TZ7WGV6aI4rdekYBbHcZNW4rcx/IdpPXJJwBViK2YLy+D02r0/Gr9vZlzkgkexrjlOx1qJSW0DHG4Mem5s/oK84+Len+Q+lXI5V1kjzj0IP9a9ijs9xAC4x6Vwnxk04Hw1ZXOSTFc7SR2DKf8BWmDq/v0YYtJ0mjyzwXJ5PjrR2Y8POq/8AfXy/1r6YjtRtAHTrya+X9HY23iHS7nB/dXEbn6Bga+u4LFnxwcVvmcXKceU4sLV9nB3M5LUHoTj2qzFY5wABitlLBI13SsFUeppBdxqdlnCZn/vdh+NckcI1rUdhyxTfwleHSzjLAKvqac1zZ2reXAhuZv7qDNTmxuLvm7mIX/nmnA/GrcNvBaptiQKK7aeGtrFW83v9xzSq3+J3MxrXU9R4mmFrB/cj5Y/jVmz0ew08744gZT1kflj+NW3mA71VkuPerlKlT13fmTzTkrLRFp5go4r5M+NFt9n+JWouB8sojlH4oP6g19QST+9fO/x5t8eKLG6A+WW0Ck+6s39CKeGxLnWsxypcsGyh8Er8WfxE8rOBdWskf8n/APZa+jZJ+etfJngC9Nh4+0SccZuFjP0b5f619SPJx1rnzRuNRNdUb4SPNE2VO5Vb1GaWoLR99rGfbFTV103eKZMlZ2KlymH3etQ+1XLhd0X0NU67IPQwmtQpynBzSUH2rQzZNmlpgOQDS1ZA6ikzRTAdnmlzTM04GmIdUch2vEffB/KnZqO44hLd1Ib8jQwJTSxfe60080sX36ALm7Heik3e9FFgPkH7FK7i0QKZSdmM8Z6Vv+GdKutMguIrkKAzgrhs9qzopxFqSXDBsCTcQOvWt0a9bbc+XLj8P8aI2WpMrmltPvSAc89KoDW4B92OU/l/jSnXIGPKS5/D/GtOZEcrLuBijvj+Rqh/bNuf+WUvT2/xpv8AbUGeI5OfYUcyDlZo4rVtpTJbpknK8GuX/tqD/nlL+GKs23iKCINuhlIJzxipk0XFWOmB280044OTWGfFFoCcQT4HsP8AGmnxTanrbzfp/jWZpob2c88cDiq9+c2rcEZwOB71kf8ACU2o4+zzHn2/xqrd+LLR4Sgt58nvxj+dJ7DTRVumA8VaIRwPtUfB7/OtfQiivmuPVI9T8U6QY45ECXUfDYyfnH+FfSq4xWUi0LTT0p3WmE0ITISOaaRT2xTDVmZh69rVtoBtbu6XdGztH94DBK57/SqUXxQ0OMfNg/8AbZa6KbSrDVsRahZwXUSfOqTRhwD64PfmnR+DvDi8jQtNHP8Az6p/hWkU+hLaOfb4saEBwE/Gcf4U0fF3Q8dIv+/5/wDia6tPCnh9emiaaP8At1T/AAqwnhzQ1+7o9gP+3VP8KfvdxaHFH4waIv8Az7/jO3/xFN/4XBo5ztFrj/ru3/xFd6uh6SOml2Q/7d0/wqRdK05elhaj6Qr/AIUe93DQ8+Pxe0oDOLQD/ru//wARV3SviQuv3n2LTLOC4uNhfYszD5R3yVAruF0+zU8WkA+ka1WvNPtUv7K4Hl25QuCyhVzlenIpWl3HzLsZI1PxE43LoURGcf8AHyKUah4k/wCgJAMf9PA/xrZglsLSFIPt0bBc4aScFjk55OfekbUNNQkNfWw+sy/40/mL5GWl/wCI+p0mAD/roD/7NVgX2vt/zD4B+IP/ALPVhtV0gcHU7ID3uF/xqNtb0IEE6tp4+twn+NS4+Y0/Ib9s1/A/0KAfgP8A4ug3GvFf+Pa2B9x/9nTW8ReHxkHWNOH/AG3T/Goj4m8NqRnWdPH/AG3X/GlyLuPm8iuNQ1t7+e02WivEiOd3AIbdjHzf7JpkkWtyH5nssf71U18UeGYvFFzcNqln5clnEu9WyCVeTjjv81Xm8ceFF5/te2/BWP8ASp5E92VztdCD7FqvGZLLr/z0/wDrUybTdVlxiayX/toT/wCy1M3j7wov/MVi/CN/8KafiF4UGP8AiZgn2hf/AAqfYw7le1l2Ko0nVw3/AB82eenDf/Y1NHpOr7cfarVfxz/7LSN8RfCw/wCX9z9IH/wqNviR4XB4upj9IGo9jBh7WYT6NqYcIdRtgzgkADJOOv8ADXHQT+L2BeLw/fMp5DBosMPUZrrJfiP4WkA8xppMdAbfOPzrqtPdJtOt5o+I5I1ZQRjAIyKXsY9A9rLqeXhvGxORoF8pz3lt6w/Ftz4mi0Yrq2nTWtvJIFDySwtlhyB8vPb9K9yIB7k15X8ap9mnaXbBj87ySHHsAP8A2Y01SSYOo2jw+5l3yPH1HSktJAVwwGV688+lMmjZn2bvnA49/pVe3LLdEMCDgnBOOa2MzTH3gcnr074/z/WpTtKDaBgHPA71WZl2HPpgAdf89KsIAybgB82D78/5AqhHp3hi787w9bSQqjzJGFYnsRx/SrMksz3cRlVctuXAHsf8KxPBMhk0dgrBmjlZM598j+ddDdSLDCoklUyMwVDjOT0/Svepy5qaZlazGFURRjl2OcEkcZrhfHduPOt7sA4bchzx7iu+i8xo13hN2zDMuOTj6CuX8bW/m6M0g6xMHBH6/oazxEOakxp6nmE8ZUZ6rn0rpvhnpjal4705cjbCxuGyeyjP88VgTRb43xlsc10fwsultviBp+84EgeMfUqa8CWxvHc+gdZhEtsmQOHB557Vn2+ngoCDj6/4Va8Q6vBo+iXGo3EUkkMAVmWMDdgsBxk+9cVF8YvDiAH7DqI57Rp/8VXjYuhVnO8Eevhq8IU7SZ20dhcebGqlPJ/iyDn8BWpFaggAdO9eeD42eHVOGstS/wC/af8AxVTp8cfCqgf6FqhP/XNP/iq4ng8Q94lyxcejPR47UZGD+lc18UNIkvPAV2Yk3NC6SjHYBsE/kTTdG+MPg++SRpGu7bYMgTRD5vYbSf1xWL4p+Nej3Oj3un2NtdKZoWjDlFI5GOea3oYSUGpPc5Z1nLToeLWXk295btcAmNZMuq+mc8H/AD1r7C0+/wDt9jbz2MLeVNGrrJJxwRkV8Vz36zN8qlTnNe/eHvjv4Z07wzplld2+ofabe1jhl8uFSpZVAJB3dOK9WdOTaZytq1j2D7CJG3XMhlPoen5VOPLiXCqAPavJm/aE8J44ttU/78p/8XVZ/wBoLws3/Lpqv/fpP/i6zacPghqTZvdnrz3AA61WkufevJG+Pvhgji01T/v0n/xdQN8ePDTdLTU/+/Sf/FVyVI4mX2WawjBbs9Ze496rPcV5U3x08Nn/AJc9S/79p/8AFUw/HHw4f+XPUv8Av2n/AMVXJLC4l/ZZ0RlSXU9ReUmvHfjnbebZaVcgfcaVCfrtI/kavH43+HP+fTUv+/af/FVy3jr4j6H4o0WO0tra9WZJd4MiKBjBHZj6itcJhq9OtGUo6Dq1KTptJnmun3Js9StLpfvQzK4/Ag19bAlwCuSGGRivkDIxxnOeK9lg+MlhHplrbvHemVIUSTy0VF3BQD/FuIznuK7sxw063LyIywdaMLpnuOm5FqUbqrEYzz61crgPhh4us/E8OpJaxvGbdkZlZAv3s9ME5+73rvqKUHCCjLcc2pSbQjDcpHrWe3BI9K0TVGddspxXTTMZjM0ZpuaWtzEkQ/KPbinVGh6j3p+ferRAtLmm5pc0wFzSim0A0xD80jjchU9xikzS5pgJG26JT7VJF9+oYvulfRiKlj+9SAu9u/5UUz8aKAPka/ZoLV3Q4dQDnHvitNPDmqMgP2+AZGf9X/8AWrP1VS1hKMdcfzFem6Pol5q52WcDSsi5YKBwKIpPcmba2OIHhrVcj/T4Mf8AXOpF8L6mcj7dDn/rnXZy2kltcGGZSrqdrBhgg12Hh7wgNVtTOZNijgZGc1bjFakKcjxo+GNVI2i+iLY6CI0knhvVU4e+jUkcZhr03X9FbSbpot3TuK5uXLHkkkjHPWhQTDnZyR8O6kcY1KLp/wA8qP8AhHdRbCnU4hg/88R/jXUU09OBzT5EHOzEHgzVm4/taLn/AKY0q+C9SOSdYjx/1w/+vXX2r+ZbK2eR8pqyqgHn8ayasbrVHEf8IRfk860n4W//ANeoLnwNdpEXbWA3QEfZsd/rXekZByar3/8Ax6OxxwV6fUVL2KSR5dp9m9h4vtLZ5vNMV3D8wXbnJB6V9Rp92vmiQ48exAH/AJe4Dx/wGvpdBhcVlIa3FpjVJ+FMakgZC1RmpWGajPWtUZsqXmqjSESZoWl3tswpAxwTn9Kq/wDCbDOPsD8H/nqP8Kr+KD/oEBzj9+P/AEFq5jODW0NjKb1OxHjnH/MOb8Zv/rUp8eEDjTvzm/8Asa4zd+lGeaqxHMzsv+E8l/6BqD/tsf8A4mmnx7cHpp8X/f4/4VyGe1JnGcUrBzM7D/hPLr+Gwhz/ANdD/hWF4n1u68UWlvpcsMUEUs2WdGJYYBPf3FZoY9cUgb/TLQ56Sf0NNoLsxpvB9vA4U3MrZ6HA/wAKaPClp/z2mH4r/hXUal99M9xxVFgeNpxUuESlORjDwrYjrLORnuw/wqVPCmmkHLTn1+etA7h17VPCHJzn5RQ4oFJmavhPTR/z2/7+U7/hFNM7pJ/38Nawb2p+78anlRXMzA/4R3Tk1Rk8uQx/ZQ+wyHGd5GaZeaJpsUO+O3xg/wB88/rWu/8AyFE6/NaOPykX/GorsBrdxjoKFFWByZx7w2sWqCN4wIthbBY8frVYxLKk8kJ+SPJBzyT/AIVY1bBnhiUjdK4X8B1qGABU1KPngnj2xU2KuJciNdMhmVcMSMmpr+FFtkaFAPmBOPTvVOV93hmM45AHfHenancj+zY1z8zKDgHnFFkFy3FEk80cpjCwlwFA/iH0rYXxdr8GsS2CarcLbRIBGikAKBx1x/nFZrSBbWIj/ZOR06VFHEt14n1HqGiY42ntk1N0ilqauo+K/EEUtrs1q8VWfDASdeRWH4o1a81KaJLq7muDEpI82QttyecZrYl063mCO7t8hyvPT/PFcz4hjS1v5FU/KI1Y5OSSf8/ypRmmxuLRhzrAqb5S2euB361UgbfKWOQoHA9OacIZLl9zHapPAz/n/OKSI7L1lJB+XHHatCSxJwCSQM+n+f8AP41LalPJdSB8h56dD0/rVd2yMk8dKS3lCXOMjDDBOOh/z/OgDt/BVyqG5iZipeQNknPVf1ziulu/31woTOI2BGB79q5HwfLGlxdqyhmCqykDjOT+XQV2KulraiSUfO5HB9c/5/KvYwr5qaRnLRk7NOWwowq579enP+fWsfxJHI+kyRuoDOhPOa6CMzRfJLjaCfmHb/PNcr4ne71EywwOY0QHkf5/Wt6ukGT1POI2OwNycZzntWr4OtpJfG2lRwA71ukcEd1Byf0BrGi/dM8Z+8pIP4V1Hw9eVfH2ltHH5mWKt7Dacn8BXz8tDdHver6QuuaRdaY0nli5j8vft3beeuO9clZ/ACBsFtYkZfU24A/nXoVpJ5dzGwCZB/j+7+NdEbG5uzm9uwI/+eUPAP1PU159Zz5lys6YtJankep/Bnw8lgbW21C6m1HIC+Uike+R2/Oqth+zqsw33utyRA/wJCCfzzXuEMdrZpsgjVAPQUrXI9azVX2espfJCb5tEjyFv2eNJUfJr18repiQ1RvP2e7GO2ldPEF0zqhKgwLgkDjvXsr3PvVaW4yCCawqY63wlRpye58UXlobR9pO7OR0xzVSt/xNA0OrXUJXHlTyJ+v/ANasCvXpy5opmE1Z2CirllpV/qRYWFnPdMvLLBGXKj1IHSr6+D/Ej/d0DUzjr/oj/wCFU5JbsSi30MSitUeGddMPnf2NqHldN/2Z8fnipIfCXiK5GYdD1Fx6rbP/AIUuePcOSXYxaK6D/hBvFP8A0L+pf+Azf4Uh8D+KB/zL+pf+A7f4Ue0h3Qezn2MCit8eCPFB/wCYBqH/AH4b/Cqmo+HNX0iOOTUdPuLVJCQhlQruIGcChVIN2TBwkldozeNo9e9eg+CPh1aeLNCfUJNSlt3SdoTGsQYcBTnOfevPa9d+DV/ix1WzJHyyRyqPqCD/ACFZYqUo03KG5thYxlUSkd58NvBUPg/VbwwahJcLdQgFXjCgFTkHr7mvSjXI6Pcf8TiEZ4OR+hrrjXHRqSmry3OqtTjCVo7CVXuk+UMO3FWKbIN0RFdEHZmEtUZ1L2pSMHFJXSc4qnDn3H+f51JUI/1in1yP8/lUmatEsUngn+VRfaRtyI5OuMY5pXDkjY4A7jHWkVZs/NLn6LVCE+1f9MpPyp4n64RjgZHvTQku0/vfm7HHajZLx++IHfA609BE0b71ztK9sGn596rCOX/nufyqcUAIvErj1ANTR/eqAnEy+4IqxF96gCzz7/nRS4NFAHyZqeBbMeOWGB75r1Lw94kvNBLSWhTDqA6uuQa8q1U/uOn8YHI969F0yMS3cELMQkjrG5HoTShs7kVHbUuXepTahqEt3cEGWRtzYGBXRaF4vn0qAwoEdD/e9a5XVFMGpXEDPvMMjRA+ynFV/MKn6Vro0ZJ9TZ1jWJNUnaWVgWY9RWM5zk5z6980by3GeaQocU0BEeM5IxTSMZ5HvT2XbnnkVGdv5mmMt6fJtmMZ5DfzrV2ZyR2rn1Yq4YHBBzXQQuJIVkHRgD6VjUXU2pO+gbT36VWvFP2V88gEH9ausMj0AqreYFnLweBn9axb0N0tTzabb/wncRH/AD9w/wA1r6YXpXzPOCvjuIsACbmHp/wH/wDXX0yozWbBbsDTD7VIRTGpoTImFRGpmFRtWiM2YPigf8S6IjtMP/QWrkyD3Ndh4kTfpg9pQf51ymzjitoPQynuRYI6n8KO3J96k8vJxTvJ5zxg1VyLEBHvS4PepfKJOOtTf2beHpaT89/LNFwtYqdxzRj/AEi35P8ArV/Xiro067720oHupFMktHikt/MjKFpUxn/eFDulcUWm7FjUVDMhyeh6VTaONR8zYz7962NTjRBH9D/Ssl40lOD0FTe7LtZFbMW77w/A1ZhaEpgv85PAqE20YI5/WrMFvARv5yORzQwQ/wAlSM80ohQ8f1xTwQBQW5PPFRqXoVVhRtcsEb7jpKhx9UNdp/whumSIys9xzwcsP8K40uF1fTnHaSQZP+4D/SvWEwcEbsnrxTQ7I4//AIVz4clmSR455JVPysZWGPyFcWfDOn2/xVm0VoibG4jRtm85OUPfr1FeyDaJQoY8mvNfEDLb/G3SnJx51tGv1O5hQxmyfh34XSD7Oun5iHVTM5H86uR/D7wq6ANpEDA9d5Zj+HP6VuBACfl3c9xmp4yFlCjAUH2osM+e9at4bXUr22gGFgmeNV9ArED9Kowyi28TahL3aJX+uVH+Nbvi3aPGepwbct57HIGM55/rXNao/ka6zfKBJbx9+vyj/CuWTvdFxVtTVa9LLt2Y9cVx/im73XxVX+Yqu/H44/nXTq2BweK4/VJLeC9meRS87OTgjJ69falRjrcuo9DPigJXzZ5CIwM4BqK1/eXDFeCxwB171FcXMsxwxwvXaOlTWKqdzPwo4PFdRgWZQG6HOevP1pkSKGBBOe+MHildzIxCdD0GM8VYt4TtDNncRkd/SgDoPB7IuqTqRjdErDkEEg//AF67NwlzcKjA+Wg5x3rz3Qz5OpYUoN6cZ+o6e/8AnivQdKxJy/CL1z3/APrV62B1jYzmbHnJLAjSADIBP+PSsO7uUbdGqBmJPOKtLDcT5dcrFIBtUDBUfzqtqb22i2Blk5kI2onGScdv8a6Zyb9BWVjyzWIBZa3PGD/FuPPQnnFegfCTSrO/1ue/k3+dYxholU4BLZBJ9cAdOnNeaX9xJcahNJMRvZucDivQfg3qCw+KJ7RmAFxbEKPVlIP8s14NXVuxtA9xI7ZxW7DqHmW8bg/eUH9Kw+lRWVyRB5ZP3HdPwDED+VeJmLcYpo78NBTbTOi+1Z70xrj3rNWY+tKJGPTmvFdSR2ewSLpnJ71E0mc81Tury2sYvMvLiOBPV2xn6DvXNy+PtOkuhaaZG1zOQcM/yJwCfr29qcadSeyC0UeM/EK3+z+JdVT0uWcDH97n+tcMetdp4y1CXVdWnvp40R7hVfagwOBj+n864xhhyPevqcNdU0meVX+Nnb/DHxDNoGu3RhhWU3FuUwexDA56/WvfNJvtQ1nSra4kMUayghgVyVZTg8dOevTvXzN4RZh4ltVVsF9yZ+qmvoLwlLLbQPA0zPGDuUEcAnrj8hXFmFNX5ludeE1Vjql0+AOJJi07jo0hzj6DoKnLqowoAHtVcz8ZzULzD1ry7tncqd9yw0uaiaXrz2qo9wB+NRG5G4DnrQos0UUiy0tedfF2Pz/DFtKASYrtc49CrD/Cu0e59q5Px+PtXg6/X+KMLIPbawP8s104a8akWZ10nTaPn8ffGfWu6+FV0YfEtxCTxNbMMepBB/xriJhtnce9b/ga5+zeMbF84DMyfmpFe5WXNTaPEovlqJnvtnP5Wo20hPSVc/TNeiGvJTdFSCOo5BFerwSie1jkHR0DD8RmvOw6tdHo4jWzFoNL2ptdKOZlORdrkYplT3QwwPtVfPFdMdjnkrMDxg+4p5/Wo2OUOPSn5z+NWiGLRSdTRVIQtLTaKYD80Cm0UxCSNjY3owzVmMfNVaXmJh3xViE7tpHfmkBb49T+dFJzRTEfJl8pbykHOZkGfxFd9GywXCMrE7GDZxjpXCXAzc2o4+a5T+ddtld3U8c0oapk1Fcv6xPFd6zeXNsS0UshdTjHXk/rmqWcD2pyMintjPXFNJ+c7fwNaLRWMkrIu2ccbTLvIA75rsvFOi6Np+kW0tlOGnfHyhgdwx972rgVlwcg4PrUj3DuACaGrsaIpMZIFREGnls44FNbkmqAYc561r6XNuhaM/wnIH1rJPPXt71YsZhFdKScKflP41M1dFwdmbpBIzVS+YfYpSRn5T0FXwvPSmz2UU9tMXheQhSTiQrkY6VySdkdijdnlt8AvjmLn/l4hz/47X0yn3a+b9daL/hYMfkwCBfOg+QMzf3c8k19IIPlqb3SFs2OIqNhUtMYU4ksgZcc1Ec1Ow4qMitEZsxvEP8AyC245Dqa5Inj3rt9UtJLyzaCLG9iMZOO9Yq+GNQbvCB/vGqiyXBvYw1ZuenHbHvRyeT29q318J33eSEf8CP+FPHhG95zPCPxP+FXdB7OXY50HBHP44rtUjvJdQCRp+6Me7/VnHTru9c9qzh4Oue9zEP+Amu0hUxQxIckhAMg+gqWlJicHbU5aayv9+0o4X/drJ1uN4zZGSNlbzejA/3lr0QHPQH86xPFGnR3kUEjs6mEMwx+B5/Kuidbmhy2MaeG5Z8yZy2uDasZxnrWEZOeBXb6jpMVwE3u4Hsazh4ZsuCZ3XPOGcD+lYqyWprKLOZJyM0gcgYrp38O2gKlZCVz82XPT8qkg8OWkz7ERmPX7xFaqm2r9DO6va5ym9uTxR5jdCeK7T/hDkP/ACxUfWU05fB8QGPIjP8A20NLlXcrlZw0j/6Tp5J6XBH5xv8A4V61E8zKCqpjAxkn0rg9W023s7zTlSNVIvUDAH2Yf1rr4X1NxEYo7YwlRguxzjHoKya1NUtCw7XrSZHlAdvvV5z47c2vxR8K3BQb3ESMe3EnP869Bk/tYEBDaKoGT8rVzfiexFzqWm3eoRW0s8Tf6NKqsNhyD0P4VLVluOMbs6122yOMd6fCqn53wBn061nNa6uSCL22B7/us/1qVItUCFJNQhIIwAsAyCe/Wk5tdCuXzPIvH0YTx7fuI2AYKckdcoK5HxCjfarRowdv2dTyfQsOteweJNKU3ryXUcc9xtGZTEATxxXGG1SW0gLRRuNpU7xz94+v1rk5veZsqeiOXuZWgsZZo8b1XK+ma45/tJlaWeRMk5ZioJ/OvUfENkB4dnCxoh2K25RyBkE15xc2rO6IVOxeenU10UUrGdVWZkuY5XZkhOOuc1NaqDCQV5HaprgxxYVUwzcAdM/4VVtJBu56Z7VqZF1It7ZbhR25qyeT7Yz6VAshyOQPXnmpR0C9F7Dt+H5UxF3R7WG41KVrhwqKoKkNg5ya7211CyjHlp+8XjOehFeW3UxiuIsSyIGB+ZBk4/H6fzqRbyQkf6XdtjDHkLjn2/xFdVHF+xjZIThzbnr8GpxmJhGoLqCFQdTzwK5fU7CWVmvdXvrW0U/dWTJwPTHrzWLBdLdxX0UUMMU0do00EkckxYsrDpudlPBPYVy80d3PMWuBLJu+UOxzzjI5pvMfaaNFulyrQ0dZtNMZZprS5mklQjlowikfTJNVvDGr/wBieI7DUMHbDMrMefudG4+hNZ0tndW6HepA7gHNVlODXLOSm72EtD6+VlcB0bKsMqQeoqpDC/n3HZBLnJ4HIB/rXPfDbWDrHgiyZ23S2ubWQ5/u4x/46VropNMN5cl3ndYRj5FPevJx8E6ep6GDlaeg6XU7K0wGcyuP4Yx/WsfW9Y1f+y5bizhNpFHyWxkkZx1P17V0lrp1laf6uFd394jJpb/yp7WSGRFZHXBUjINeRHki9Eeg7yPF7lri/ZmnmkmlbnLEmiy02a3uEuB8vlnP510k1rHbzsqoFAPYVXlT/R5B0Ow/yr0o2toYNanA+M9O/s+5gQEsHhyM/U1xEn3zjp9c16d48QTWOlXGMlkKk/l/jXmcow1d+Hd4Hn4hWmXvD83keIdPk6YuEB+hODX0RpT+WT79a+aYZDFPHIDgowYfga+jrKUMqMucEZrHGK6R0YF7nQi5+X9Ka7SFN+1ivrWZI87QObbaZQMqG5B9qrzXOoRTsUZXi/ikfapAyC23nkben19q8zkPTuaLzYqLz8HI9c1zn2zU3QjcqlWIdyyYQbup9DjH5niqzX2pkbkmh27QTudMZ4zjn2P51oqZPMdNNJh26YyaytbT7Vot9bkZ8y3dR9SpotJp3iLTtuJY7G45XjH3eO9PZ+CDjB6mqirMmTurHzzcD95n+8Aat6RP9k1uwuD91JkY/QNzUV/CYLhoj1jZkP4Mar5OEb04/wA/nXtbo8LaR7y74LDivVfDk/2jw9YyZz+6C/8AfPH9K8atrv7TZQT9pI1f8xmvUvAtz5/hxUz/AKqZ0/8AZv8A2avNp6SselUd4pnSk03vSnrSZroRzsjuBujPtVI1fPIxVFhtcitoPoYzQn8qRP8AVrnrjFHShehHof8A69aoyY4GjOaTpUF9LJBY3EsIUypEzIG6EgEjNUkIs9aK8t0z4jatfapaWskdnHHNMkbMI2yAzAEj5vek1H4iaxaahcQQpbNEkhCM8DKWXPBIzxW3sJXsZe1ja56mDS5rB8J61Nrugx3k4jE29kcIMDIPH6EVuZqGrOxad1dD+1Ps+UUf3ePyqLNPs+JJF9Gz+YqRmhx6n86KM0UxHyhMc31l1wblOPxr0C00a/vbC4vYLYtbWwzLIWUBeM9zyfYe1ef8tqdgP+nhev412X2iXyvK8xxETuKBvlJ9cetFPYmY8SAHnP4GnvJFj5V/E1Xz7mnKA2Mtjn0rQhC7hzk/QUA5J/SnvFCIyyzAtkcetRY6d80XBqxqRaJfXGjyarFb77SJisjqwJX6rnIHPWsxmJ69PQ1JG5EZTzGVG+8AeDj1HeoCeTxQgFz3poPOaCc59+nFA6etMDrLFvPtY5uuVGeO/erUsLi1k4JO08Vl+GZQ5lt2xn7657+tdYbVmt3PfaTz3rzK8uRtHq0I88UzxLxCrDx/Hv5Ilh49Pu19JR/dFfO/jCMJ8Rl68yQE5/4DX0QnKinF3imYzVpscaYwqSmNVozZCwqM1K1Rkc1oZsIl3ToPervkA9KgtFVrpA2T1/lWqI19OazlPlZvT2KQgp3k45zV1oQrYxSeUCelL2hpoU/KHPQ1Jt6DngVP5R7CsHUzeJqsMMFs8kTth3GQF6fgKcarvZK5lVStc2QBnkgflVLWQHs25BIU9D7Vi3tjqBB8gSEn3P8AjUlhbXccdwtzC6jy/lZh1613Sp/uue5xU6zdVQ5S7crmNDjrTWsZ7iG3aBGPOCQQMc9TnqB6Vbdd8EWeeB1+lXrIAWKfUiuVWnozrqRsjIu9HuicxkkYPRv6UabZXdrckzRERlcbye+a6AAYzgmknH7gY4wwP612Ks1DkONUI8/MZ4Y5OWz+P/16RW/e/fGB71E0b+awWLI7c0qxkPzEB79ayubnI+JG3aivA+S9hYfi4H9a6yyLCzhYKPuAc1y3iBN+pToM5EkD/wDkRP8A69dRZDdaRKc/KWB/AkVhCpebRtOFoJliRiV+bavbrXPeKwv2ewkGTtm6jtXRTAbVG4g9eKw/E426Mj4J2TKcmiv/AA2Kj8aNiQcq2eopVI7kk/SmpJmGJgfvIDjGe1SLIxxwfyrRbEPcxfEVvm5UjoyDOa4fT7TzYTHlU2zSKT2HNemalCJhGxQ8DFchp+lFLy9gkV+J2ZSB2IBryK1XlqyR6FJXhEzDYvPayRlFZTGcq2AGGOhzXialkyTu2r91QM19HvpanIKuePSvn+/sHstau9PVHLRTvEF2kkndgYFdODrKbaMsVC1mc3fRM37w8MOeO1U4MbgMD612d94J8SPbGQ6Ff4C7uYGB/LGa4tUKuyvuVgcEEYIrtumcdmtzViICjjrzjOetSmQIgZjx0b+VVYpCyYB3Zxx1qwmGjOSffjH+f/11RIiQw3sqs6vIANvBxn/Oa1o9I07KsYsDrgntVGwRAeBjOTtGOn+Fbax+ahAYqccAev4Vm2aJGnJe3ap9ne7u3ixjy2uXaP8A75Jx+lUJnkcFFUM54OQKsmJXyShJPOC/4/4flVy2tVi/eMoU44xngCsktTS+hzeqad+7SFpGeWRtzEj7oHOP1Fc1qmnmzfI+6fbH+e9drd3f+kSCBPOuGGPZB/Ssa80S4O+WabLMMsoHArVMzaOi+DGsm11270l2/d3kW+MH/nonP6qW/IV7gr7Dj8a+W/C1/wD2P4w027LYWG6UOf8AZJw36E19QSq24bQTjrXJjY3ps6cG7VETGaqd3P8AKRSq2W2d/Qdap6iGjxkEZHcV4sY3Z7GiMC/X/SGI781nMy5KkjBFUfHUM9xp8Bt1kdhICQgJ4wa4M6ffsebS5JwRnyzXqUqacbtnBWqOMrJHTeKlE3hOykHPlTFePTn/AArzC4BDnPXJFekyQS/8K/nhkRkeCfO1gQeo/wAa85u1xK3Hf0rpw+l0cmI1aZXHWvfdAm87RNPk3ZLW8eT/AMBFeG2Ol3morMbSAy+Su+TBAwPXmvX/AAXciTwnp5JJKqVOe2GI/kKWKV4l4R2kzr4ZVBwW25HDY6Vm3Ol2cmU813UliVAAUbiCR09Bj6VIZTuI471C0hIxnivPselzCW9tHaFirs24BRuxwMk/zY1W+wQIylXkBVdo5HdVX09FFSlx0z+NQSXcEed8qfLjPPrVWZLkieMiCCOJMlUUKM+n4U1pCe1Zt5rMNsNoEkjq20qo74zVe11c3F4YGjKDJ2nOckdafKyXNbHmnimAQ6/fKOnnlvwb5v61j5/dY9D/AJ/lXVeO4PL1p5Mf62JHz9Mr/QVyi52sPxr1KbvBHkVVabPU/DVwZvDlk2ckJtP4Ej+leq/Di43W+oW5PKujj8QR/QV454Jk83w+Vz/q5mX+R/rXqPw9l8vW7iL/AJ6QE/kR/ia4pK1RndF3po9KNNNKfejtWpixtVLhcPn1q3UNwuUz6VcHqRJXRUz0o6OffBoobhh15FdCMGLmkIDKQeQRg0lKOapCPD9MgNjq2qRRIr3ttHILYMufnVwCQD32BiKs3KN4ns7jUvJjt7qOXG57g7JQcswHmMcFeuBxg9OlX9dlsdO8aXqXdsiLvEsc0asZNzYb+8B3PT0FRXfiHSdR2G8NxMUXCh4t2Dnnq3fjtXoJt2ZxtLY6H4XXO7Tb616+XKsgH+8Mf+y13wrzTwJqdu/iW4traJI7eWDKYjCsSMdcfU16UDXLW+M3p/COp9uSLsjsyfyP/wBeo6VDi4hbPGSD+NZGhpZ9jRS/j+lFMR8oR/8AITsAD1mBrr8HbxznvXN6NbpeeKNKt3+68/OD7Gu01LTpdNmCFhJG2SrHg1MJpPlHODa5jPwR2pQCeMUoyP8APWj5vb0rYxsOMbqVBHLDIwaPKcsF7n0pMt1zxSlm4OfzpABiCscnIHoKsxWKskkguIf3ah9jEhmz2HHUVX3NjGaXJxnNA0M2EnoRz6UqxMT909aXLevtUiFs9TSbGkaGiRTxanbyrEzAOA2O4PBr1xNIYxcr1HYV5n4fhZ7pS05TBBOO9e4R4MakelcFel7adrnZCpKlDTqfM3xGsWtPiYuFKgtbnPXPT/P4V7og+XFeUfGR0HxA08D73lQ5/wC+2r1hB8tEY8seXsDfNqxaa1P7UxutaIlkTCmGpGphqzJlnToy94nHTJreW2RTuA+btzWRpDYvMeqmt6lyJu7HzNEPk5Oc0eSKmoo9lEOZkJg5zmqN1GUmA9a1KzdRBaQKDjgciqhBRd0JybIgrev6024VTAQ7AZ4yacoHrn61T1dtunsx6BgePrTqaxZUPiRJJEscSoxHygZ/KnQXEKW4BnjUA92FZd7fgWPm+oXv7Vyv29pZSmCFOe9YUU73NKstLHo0VzBM+2OaN2HJCsDVW61nTY1eNr23EiMAymQAgg81wug62bfUrgtGoVUKls9eQa4LxJfCXX9RTBz9olU9OfmNW6rU3EhQ9257adc00Lu/tG0APTMgFQHxRog4Or2ZPtID/KvnKeLc/FWLItGwUKMN0z/Sh1mgUEepzanDqXja+8i5E1p9kV0K/d3Kyk/yrrLbU7aG4ltWnxILmVcFSerEjtivI/D0pg1K4xx/okw9c/KTXrWjS711CUHlrtm/NVrnpNurdG80vZ6hN4o0uN2BnnbA42Wzn+lYfivxHZz+H5IYor4yl0YFrZ0HUdSRiuvyM5yd31rL8WQibwjfEZLBQ2TzyGBrskm4tM54uzuiK01eVbKBZNI1MukSq37kAZA9zVuPVZnAK6Re4z3MYx/49V9SXjQ9ioPFO25XG3I9zSSfcG0aWmjzrISSwNGxJyjkEj8iRUdvDbtql38i7jtbp7YqNLryU2dOM49KyLPUDHr0xZuHiXj6E15uIrwVRRa23N6dOTi2jqTaw8/IK4eTwxp1r4/u9ZFupuJUj2E9EO0hmA9TgD/9ZrqDqgrG1vUBEEu0GJANm7IwO4z+v501iaLklBWKp0pp+8RahdSXen3a2Pz3Ue5VVSAQ3YfNgentXB+NPhfPr3huTVI9HEXiGOIO5jkUeeR94MASCcZweucc13ENyt3Ai27yGYneuGGHU9VBPQ55FdDBeSG0jEsZ8zaMhmHWuuLitQqp7HxQhKyYYYIOCGHI/CrUchbEYHJPQDFXvG1ylx461uZNgV72U4QEDO7375z+OazrJTLKAoyAM/5/Kui+lzktrY27OJVUb2G30B79/wDPtWqjxpGXOAuAuP71ZtuhOOTjOGBxgCphKZr0QqCUjG4j3qCzUj3AKw4KhRjpipLuYCE+bN5UX/LWXPOB2H41RUzKp8q3eUkkKqr3579v881mzb7m82XOLy4BylnCSUT3cj+QpJDbLkU0t2hOnILKzBw11KMu59v8feq+o/aLexnjsoQcg7ppGDs477fyrREGpRhVmluTgYENrCoVfbLYFPZCQ0k1skUJxuCsZD+IUECh6AtTzXcd+7PzZzmvsHwbexX1hZ3U6qwuLOOXkZ+YqpP8zXyRqcKQ6jMsUbRxbsorelfSHga6ZfAuizAn5bVVz9Mj+lRX/htoKSvLlPSo7PSoblrhYUEh9TkD6DoKwPG0kBsrYxKgPmEEgdsVTOqN/erJ1q8NxaqCfuvmvHjVlL3bI9OOH5XzNmLK25AT6VUYYPIzU27dHjnhjUTDJBPT2roiVIx9Sj36VrUOP4BJ+mf6V5HeAhj15ANevX9zb20t3FcybPtNoyqME7iMjHH1ryK6GfriuvDnFiehu+A4pbnW5LSKQRtNFjJ6HkcGuu8Auf7Entz1gunT+RrzTS786be+fs3jaVK5x196734dXfnnVQAVBlWULnON2f8AAVpXj7rZGHl76R3M8vlQvLjO0E4H0zXPHWZ2ViYhGwGf1H9DXQTpvtmA/iUis3TbWxuYwJwNzQPyGx8wXj+XSvPukrs661TkVznpJLu5gaSa4k2KQ2BxwWwR+FQ3KGN7gBiW3ujEnqAAwrV1FYYmuoLZt0DoWUZ6EDNZtwd8xP8Az0aNh77lwa1jqjNSurkQvN0xuW5w8bn8sGrtvIp1hMAgmQuM8cMo4qiYBDbRzKSfMj349CCv/wBerjS/aZ2uYFkwAjD5epBwabKRk/EGPM9pIBjdG6k/Qg/1NcIvU/SvRvHaLJYWkmcsspXrzhlP+ArzlMCQZ6ZrsoP3EceIXvnb+AJswX8OejIwH5g/yFeoeDZ/J8U2mTgOGQ/ipx/IV4/4GnMWrTxdpIT+YI/+vXo+iXxj1SwugrIFnRirDBA3DINc1dWqXOmi707HuNJmlIptWiGHemONykU6g1SJZnkYNMY8r9ammXDn3qCXhM+nNdCOeQ7NGeabmlzVEmdf+HtJ1O5+0XtjFNMAF3tnOKgHhPw+p/5BNr+KZrYpc1XM+4rIoWmh6VYTLPaafbwyqCA6IAQO9aOaaKM0O7Cw/PNDHam7+6Q35GkFOVd6smeoIoA1Ovaiq0VynlJkjO0ZooEfN3hNPM8caSOcb5CPwRq7/wAVjBg6jJPSuA8I3ENr4306adgqDzcn/gBFd/4tnSSS1EZ3oys3mD7tcyT9qjpuvZM5oZ6ClxzzzSd/agkggAcdzmu25xsUc+uK0NL0XUNXkZLG3MrJywBAx+ZrPXOR9K6nwd4gg0O9ka5jdo5EILKRkHtwetZ1JNK6IldbFO+8Jaxptobm6tDHCv3mLL/LNYpUivQPFXjSx1fS2tbaOUMSDliMVwDMcDGCM81MJt7ji27iDPtUkfGKh6il3FZkXI2kHPrVs0RtWlyYFyDyRXtlheq9hAxPWNT+lfPa3iB8Z4xnOa9csbmcWds4lXyRbgGPbyWwMHOfrxXn4qThaUTsow9orHlHxcufM+KEAB4EduP1Jr2tPuivnz4lTGX4kI3B5gGSfpX0HH90VUHzQTZMlyyaHUw0+mmtESyJqjNSt9aiNaIzZa05tt9F9cfpXR765e0bF3F/vCug3D1rOc+VjSuT76N49a5rWo9cluU/s24ijix827r/ACrNWw8Vsfn1WEewH/1qweId9ivZnb7/AHqKVlyCQCScZxXJxaV4i8wtJrClP7oz/hUMui60u1pNcdhnGBnqal4iVthqB2JZMfdH5Vx2pXDPPcRs7bQ5wueOtS2Wk6hb3iTzatJKi9UwcH8zWVqJd7q68uNydx7GnTquV7lKFitqOqQw2CxS5XCrk9hWBLP5beYozjnPrVnVLA3tqiNA0iuoRwQee1ZN1p99Gfs0PlKgACZfaqADGPXrWlOsti5UebUrLq6y3lykJHyxt5nQ8gZ59K57xBLGPE18nmJ5jTyMFzzjcc1fOmXa/aJI7ZVfaTK6TMMnHPb09aZqmhX73CubIPKHDs6yuxcdxkr34/Koc05XNFStGxgz3CQxhmcKMgfUmrdtZTsrT5IjHPJz09MVoRaDqFwwii0RncHcW3noPqOK3bHw1c2EUN9L+7aMEtBuPTIHzevXNKUr6LchwSV2ylo9pIWS6XmGWB9pIw2GQ9q9Q8LljpkzkZ3uj8e6LXBW7zS3KkkbCTnj2ruvBj/8So+nlwkZ/wBwf4VWHvzakT+E3uABkfrVLxA6jwrqe0jCwMx3elXow0jAbWqPU7b7ToN/E6kB4XGR24rvexzLcjs2M2n2kiZIeBGyD7VN5Mh9aqaIyDRrLMoJWBQcnrgYrSE0X/PRP++hUKS7lNMztXnnt4ENrCs1xsGImfbkZAJz9DmuflvAmpqe4XBNdBqnkSiKdHSR49yBlIOM4z/IVyOonFwJB1weSe/FeHiknXZ6VH+EjRuda+z2zTFXbaeQnJrMm8RSTxvGtleHj72zj86qPO3UNgAjgDiopZ2KDkAE9BWMaLTujSUn0NfQxp6awrTXRtnc7sN8i5HPVux9sV3iXEFwqPDPHIhyN0Z3D8xxXkgOXUtxzzn/ADzXdeDsNoSd9kjqR9cH+tenSldWZzVVfU+V/EL+d4m1SVQ2HvJmAI5++etSaUo2tJjvgU7XrKYeJdTjCMQt5MvT/bNWtNj8iIBsLg56dv15ruvocXUvGQW8JckZ6jPpUFrL5VuZHUbnJJ4xiqt1M0zqinjg0XbrHZZ5GFxwvH5j/PNSUPtYLrUIjMZHSI/LtiBy4ye/T2rWs7LUEBgts2UTHnbyx7dBwPxJPqawNAhe4IHnTqozxFIQf0/z1rqINOt3O1p33L1LXJzn8/6VdiTQttIgQK9w0kzDndK5YD8M4FSyfZkct5CvIeV2A4HH+fypsOlhRmO9ugCD92Yn/ParDWskShUuZCxb7zAEgc8cChrQcXqcN4xs7tJLe6m5jYFB7Hr/AJ+le0+C0MXw60odP9F3fqTWP4bsLG+mvdMvoBdw3MQLCbn7p7HqOp5HpXbraQ2mmC0toxFBFF5caD+EAcVlLWNi46TTMSX97JFIZHBjYkBWIDcEYI7j2qO4cshHam7ifb61HIQUPPbNeWlqeu3oUrWFYVlRNx3SFzlieT169PpRPDHMFEqghWDjJ6EdDT1PzNjuOlMkOQMY/GtUZs5bxijfaLB1BJO9eBmvO72Mq75BDbiCDXsUmV1CwfOFEhU8+orzLxZCYNe1CMdBOW4755/rXRRlrY5K8dLnMV2/w1l26tdxf3oN2Pow/wAa4k9TXT+AZvK8UxJ/z1jdOntn+ldNVXgzmpO00eu/wDnINchNbzJNKhdEKcRlmHzZJP8AWuvCMVwAfwFcze6c5nvnSKQzKQ6YB56Hj9a86J6VRXRnrLbx3OWuS4ERJVR1O3moRc2rS2ReGUlFADsQOM4BIpW0+4Mcf+iy5aNDnyz6EEdPpSXEF1NbwH7JMriPym3IRggg5/nWisYWHx3NxMNsMcaokLNjGSOo4/GopbiW5dY1kZUcoRg9Mg/1FTwWtzBegAIqMzoxdgPlLZz196pw2s8TDcYuCpAMq9j9fc0aD1KerQmXTbqaUMJohHIN31wa4uQbZmA7McV6Hra29wtzJJKiL9nYD94vLDOO9eez/wCtJ9cGuqg9DlrrU1/DEnleJIOuGLL+YNekBiOc4I/nXlOny+Rq1pLnhZEJ/MV6rzn+R9azxC1TNcM/daPeLWcXNnBOOksauPxGalrI8Lz/AGjwvpz56QhP++fl/pWtmkthvcTNHFFGapEle4XgEVVcblI9a0JVBjNUDxxW0HoYzRGp3Ip9RTs1GmQuPQkfrTs1ojMd2zSk03PFHSmAtKDTc0vamgH5qSI81BmpYvvUxEUkEnmNjGMnFFaHHvRSA+XtIJbxDbZzwsnf2rq5nJwCenTNcjo2f7egYcDZJ/KupkJJ570QFJk1uQX2sO1WwkPcE1Rt8mTAJJ9quBHx0PtRJ6jhsPCw9AozS5h6hB1qPyz/AJFAiYk5FTcqw8NGOdvApPOQqcKB7Yq/pL6fb3EjanZPdQlCFVJChDdjxWUkIZnaNZAC5IDNkBew6fWpUtbA0K84UE4A/SsLXLsp5eJCgb5gytySO1bFxFiFtwBGOh5rnNSQSW5juCf3THYe4GOKu4rFVNQkmVR85UYJYddte56fdbtIg6j9wn4cCvnfTpHN9Cm5VVR3zhscivdbCR20yA7sZiXcMewrhxmsUduDWrPL/Hkm/wAeqc8F4f5Cvo+M/KPpXzZ43I/4TUHIPzw/ToK+kVOFGPSqhpBGc/jZKTxTCc0jOcVGWIHBq0zNoc1RHNK0hA5pmc1ojNkkBxcRn/aFbO+sOM4lT/eFT3t7fQzhLaxMykZ37gBmsKybaNqVrGk780itz1rBm1DW8/JpI6d5B/jTYr/XmkUNpiovck4x+tc/s5XNeZWOiLECq8wd4twBIVhnH1qtbz37fLcWsaqf4w/T8O9LcsfJwD/Evf3FRLR2Lir7EobIqW2iDAZxzWXd3E9rbtNFC1yxIAiXjb6n1rGj8ReIYZZPJ8PSzJn5MyhQPzrKona5UtrI6aOyjaJxgcSt/wChGvOkxqYuZbjaGErRELwCAQQe9bcuveJ0idrXRUZnYl1kmH7s9x1Gaz/Dem38cFy+rWiwzNcF0RHBBBAz0J71WFhFuS6vY5q7qRSa2KNxKLK3kjW1JikGHLEnHGOwqzYaeuv38Mt7Dsh27RsZgSoB5611KWVtdSRxTwqUJx0PH6Vu22mWkEjPFAqsqhSOe3P9a9mMacIaL3u55zdWctZaGLpmixQRyrYQrEJIgAz5PfJyc89Kh12GIz2iPbK+9zncOuQtdXEVVZAMALKRgDtxVWcLusXwMk9duT2rOjFQfM9WXNNqyZ5VKiQaj5UaqqbgdoHY1teF1jezj8yZ1WO0jOEkK5wSvOPpWD4ngnTxhIYrgRqsi/LtHAIGQDW94SUfZZCSM/Z2XOPSVhWcl7+p1Q+DQ6OIWRbmWTHf96/+NI9hYXELxRtKQQ2FMr5PH1qsVbaHM5wf7oq7YbGOPPYnkFSOvFWmu6ItPsZ3h+ytn0qGUwwmRlw5ZsE4JrWNvFHHkRW7YHTIz/Ksrwp9m0zw84kfyoYZJCzSHp82ev41P4n1cWujpHZOHur8iK32HOd3Vh+f6ily01G7RtFTnKyJ5reIWu5IUjIcfdGOOfYVyWsrtZDxjBGPzrqRYvpXhqC1kmaWSKNd8jtnLZ5x7c8e1cnrLZiDbs/NwMcf55rzMXG1ZNKx2UX+7avcz3xGjHcPXr0FZr6nagcSk8+h5q/1V8gY44FU2hiXpFH+AFOHmOV+hs+DNR0+bXza3XkvFcRMAsyjaWGCOv416JcS6dpFoxEltZ24OTgrGv58V4xehPscu1QAFyOPeuRu1HnR4QdPau2lZxsclS/Nch1ho7jW9QnRt4e5kcNnO4Fyc5/Gse7zHJs9R398/wCFa7x4bcBzWRfnF+cHJ2j+X+fyrdIxZVbmUY7dhxW54bsodV8VadZzQeZGXLyRuMqyqpYg/UisRid3XI6Dmuz+F1qZ/Ed3eEfJb220cdGduP0VqBGHq3h8eGPEktmylrST95A7jIdCenuR0P0z0NbtpBayEDZCW4G3A4/D/Pau68XaAdf0UxQrm9gPmW3Yk91z/tDj0yB6V55pd3A8EYucI4wodhjPpnjr/hVRYjbitbdCNkC5UdxVpwQuFPGPun17fypkYIG4Nkeo57VI77VBZhj371Yh+kXX2TWraQc5cIR7Nx/n6V6ORxjH1ry+HdHcQvgrtkBLHgDB/wD1V6dwcHPWsTU5VvkLDPTIpjHIJ/WprkYupgOPnOPzqvn8u1ec1Znqp3VyDOG7kY6Uw4Kj06Ypx+9jA79aac98VSIZR1CMG1UnosinNee+MoTHrVwM5yiEZ+g5/Q16RfDNjLz0GcVw3juL/T4pNv8ArLcD8QTWlJ++YV1eJwjAhj2q/obSLrdn5T7HaVVDZxjPH9aouMNzUtlMbe+t5x/yzkVh+BzXc9UcMdGet/2Xqcn3rtMf75rM1C0ubd2jluwCE8wcn5vYe/FdmjZBGF4HBHf61z/iFUSSBzAkh5zk9FHX+deatz1JRVrnPW9rLc3KwNOylgffoAf1zUKW7NOYt0pdX2Ngcd+c/hWmk9woWciJFdmjVlHIwCB/Kq0ouXWSVp25hEoUcd60RjYrLY3ckXyRybjjhiBxjn9aupoErAFmK8sDyDx27fnTZI/st/Gwkc4EbAk9QTg/0rqMAr+FTJ2KjFM5dvC6zwhJpSCSchf0rzy5Qo4DdR8v5cV7WQD0P6V5H4ggFvq93H6TufwPI/nW+Hk3dGGJgkk0ZucbDnp/jXrNtKJ7WGbtJGrfmAa8lz+7Hsa9O8PyGbQbNjg/u9v5Ej+lXiFomRhnq0ey/D+fzfDXl8fuZ3T88N/U11B5rhPhtPmHUIM9GR/zBB/kK7qso7Gs9wpM0UGrIAnIxVKQbWIq7Va4GDmtIMiZSHEjj3zTqRuJvqv8v/106tbmNgo70UUxBS96SjvTAd2qWH71Q1LCfmp3At596KX8KKBHy3oxB1yHHZG/lXTSHP51yNnMYtSRxuzg9K6X7SWZMkYIwQfWlBimi9ZuqSFieAOlXvtkQyA/6VlK+QcHjHWoBqFo9z9nFxGZQcbc9/SrcUyYto2/tcfZj9MUfbIgOC35Vmk85zUbQIQx+bPsxqeVD52WP7Zi3MAHOCSccEU+DUIVUliRu5Az+Fc7IjQzMfmR94PPIAz/APXqKR9sxLMQAhO0fmf8+9TyovmZt3Os20jiBopMyDAbI6/5FcvfXMzXzxKuwhSpz39KjknDXolJHljJxu5H+FTaisN1h4ZB5oXL5Jzj2qbDuZ6K0UiSxjcEXOQO/SvcdNkQ6TbDncYE/PArx3wo7L4ktI2LeW7lGB6EEEYx6V63p67LC3UdFjUD6Yrhxctkehg47s818bHHjINycSRY/If419JR/wCrX6V82+NS3/CXsAMgGI/oK+kYj+7X6CtI/AjCfxseaYaeelMNUiGRtSAjFK1Qk81rEyY5zhc9xWwkqoiMxyCPz4rCnY+UwGc4OMfSp4JUaKKK4kaOROgJxuz/APrqKtjWkakl8AVCKRkgYxjg1G18dpKHnqM9+ag+zQMQfPfIAAw3pSEwQZhRZZS38I5xWVos01RZMuZ3UAY2gn61Bcf6g59Qf1qyqIgOBjPJ+tQXTILaT5h09a5Kj5nobw0GKTsOfWiIlgQPU1xfi7U7+wvmjtruSNCgO1TjFYnh/XNRfWFgmv53UxMSCxxkDOaUsLKpFO4SqqOh3TSkPdDPSU/yFTbsq1cJpF7dT3Fs0lwzbpHDmSQjP+NdRqOqrpQj3QSXHnHaBDyQf/r1FCi6WJimxVpqdB2Rv2NqbiRAwwCSckenpW8ibTKPcfyFcroOvtcXsdnLYT2zqrOolPJGK3Li/kiuEaOAsjL8xzwDkY/r+Veu6sW7XPM5Glexbjb95JH/ABF93A+lVmyxtlBIO1sZHQ5rOk1aQXM0U6xRXAn8hGLkqSybk6D6Cq97qt9FounSJbxee04R2GSq8kfXqKFUi3a4crtc5HxTAw8RSNzuwpPFW/Bzf61CucLOOR6S5/rVfX55J9SnMxXzVYr8oxnBqrourHSRLMIRJ+/mQqWx12msZT5pHTCPLE7uWNSi7Ejx6hB+VS2SqLhTtUH12iuVPjzYpxpqfKe7+n4VGvxEk3kjTYRk8ZkOaFBXvcpzdrWNvRooLnTru2uYVmha4lRkIyCMiq+leFYdN1oXizyzW8SFbW3l58knrg/y+tYth4gmtYg0cMWLm7YkNn5cjPFX5fFN5Ax8uKH05BP9aj61RWkt0WoVUnyvRnTarJ5mmTgBThcnDAnqK4LUlXyvbcMZHHpWoPEd3dQPE8cAV1KttXBGaxtVciEtzu4xjtzXPiKsas04mtGEoQaZQBHPuoqNxxSxlQ7dPmXI5pG/THemkVcq3KlrWUf7J7VyFwv72MkdF7c812cgyjLnt61x1yCJVByODXTROasRlMjAGc9KwdTJGpMowSVXNdHjMec9PQVzmoA/2pIeeAPpwK6DAozkBeu3ivUPhNb7dG1G4K4aW4VRn0VTj9WNeZyDBGPlB7Y/wr174fR+T4bhU8F1LE+p3n+hFK4WOwU4Ze3Irz/xpoy6R4i+1worW2pZfYeiyj7w/HO7p3b2rvS27JrO8f2JvfCrTqMvZyLOAR/D0b9Dn8KqJLOBtdob/Rp3t3zyjjj8un5Voh5UUtIoJxy8WNw/A/jWbARPEu8AEjOcZBNWUadXCBEIU45zjFaCGu0csm9DK+3ndnOPqO31r1Czuo7yziniYlGX8QfQ1535yqPljTzEGfp+NdP4RufPtrsHCssgymMYyOv4/wBKzasaLVBqC7dQmGepz+lU846Gr+qjF+Tz8yg8VQPB/wAK86ekmepTd4Ihbqc9M9AadLNJcSF5W3PjBIUD27U0kE59s0mDk5I4pIGRTjdbyr1JQ/yrjfG0YksdLnxkFGXn6D/Cu2IDKR68VyPihN/ha1k7pNtz3/iFXDSSMqq91nmTj5qb6VJONsjfU1GK9E83qe+aZILjTbSfAzLCr/moqjrMStLbFxld5U56cg/4UeEZ/P8AC2nOeSIgn/fJI/pVrV7aSezbyRulQhlB7kV5j0kestYI5OeORobhUK+QlwDHjrz/APrFEC+Zb2/+3BJEefTmrsel36RmIGPY4VmyeQwxVtNKZWTBXakrOP8AdYdKrmRnys56aFlgs5A7N+7LHJ64IOK61CGRWH3SAazYtBMfEk7MFyACOACCCP1rUhiMFukeS2xcAt7Um7lRi0N/HPNeZ+M4PL1uYjo6o/6bf6V6djJ4/WuB8ewlb63lxxJCy/8AfJz/AFrSg7TMsQvcOKH3SK9F8Hyb/DyD/nnIy/1/rXnQ7j2rufAsu6wu4v7kgbH1H/1q6a69w5cO/fPWPh3L5et3ER6SQHH1DD/E16VXkvg6byPFVpzgPuQ8+qn+uK9ZNc1N6HTUWolFFJmtTIXNRTDKZqSmkZFUtyWZs3EkTe5H6UpNF38secfdYH9aTPrWyMWOpM0maAaoQufelzTM+9ANMRIDUsP36gBqWE/NxQBc59DRSjOKKAPlCEk3ilTzj6962zMqqBn5u5rDjdllDBhyuP1NTo2+Ub3woP3R39qzi7MqS0N2OWMrJI8gjVV27ieOlcjaRGTVYoUKn94BuHcDv/WtE3ct5qZhtlDokeCjD5ffiqd+kkM5uxiOXP3QMbTjtVOd3YlRsrnc9OTVeWSTLRxld2MjPpXP6b4hkuXEV0QGH8QH3h9PWrtxeRxzLJCS5Xrx1FacyM+Uh1LVSJPJIG4DnH0qrb3sZgTf1+ZNx9D0rOu3M91JNjcW/vdQKhy23OM46E9azbNEi3bhXlG47juGPerN8baKRVLBHAwCB+uapbkS2WRcb84PPSqk7F5PmZWPQEGlcLGr4alC+J7ADnMoH617JaDZBGvouOK8U0JzF4gsM5wZ0yM9sivbYh0HvXn4vdHp4L4WeYeNyx8VyYUkAxZ9uBX0fF/ql+lfOvjPJ8T3Z7KIv6f1r6KgP7lP90fyraPwI5p/Gx5php5pp4qkQyJuhqFutTNULcGtUZMaxwua0lWObAkRWHYEZrInOIWI7CqcOuzRKqlUYgD5j3rnrxk2nE6aDVnc6R7eGIgxxqpPXFT2zqyEA8g4Nc1Jr0kv8KD3po124T5kCDjHSolGTRrpY6ieLzLeRM43KR+lcD/Z4iYSGVztIY+mQa1/7fvmBBdOfRRWZeys9vLyRlDkfhVUU4JpkSV9RfEOh3OuagLmxkjkt/L2MQQfmBPv71haf4VvV1iQI37xY3Q5XA5GOD0PWt+0u5YITFE5SPP3V6VNBI7XCje2Seuahc8VZFuMZas5yPw/dWlzGjSrutptz+4ODxW3qrEw2p9J4z+tVDNNc3U67W3ggHGcnHFWZ47ia1hiW2mLh1Jbbxxg1yYlSbUmb0uVJpGwhVPGlqqdGgYfoK3twkSURrjdt4H1I/pWGbS4PinT7pUzEiFWOQMHbWraTlCgIyHLDORxhmP9aVKVomFVXZg6pbXZufMjhdyL20k7dAuCfwrXu4JZNMii8sh0uVfGe29jTdTuZVEMkUSsjMu8lsYx0qV9ThYyh2AyABjnuf8AGk69mEaLdmjivE1zHaarck8h5WIK/WudkvDBC4EZcSXR74xmNTWl4pDzXZFuSVDZ5rGuI2e227csJ4yfbMeP6V1UZOUbkTjyuxSvNQkRH8tQpxkZ57U7w9dvqTzJMQGigMisBjLAj9KbcWTyk4jfIGOG4/lTfCVjOL4EnYkkTLnqcfj9K2qXVNtGcPjSZ11zCkOlac6Z+aZWY57mi84JOaluYhJoEK5YeU3BHqDVK7hckk3Ep/EV40HzPU9FqyJrWQluPSm6lMxhYHkY7Doaq2kJ3jLy4HT5jUuoPmIZzwD07/jXXTiYyehkrcObjb5jYGcc+nantcO2RnBqE/8AH1yRjHFIeOPT1ruUUc3MxxmbnLY+hrn9RTZKrKOqdPetzv1+lY+rA4i+hHHTrWsFZkTd0VkwI1AJrn9QAOpS9cEDkHFdGuVVc9COK57UcDUH6Dgc1qzJFRlO77vP/wBb6f5969e8EKV0WFSMYjI/8dQ/zzXkqwSTTrFGMyO21QO7HgfzFe3aPYCwlltQMiKd4/TIww/9lFZOaUkjRRumzUGMdjxWmI0urBoZFykibW/EVmgegHpWnbHNun0xW0TFnjkUEmnX13YuDugk24buvY/lVk3sKriQEDPX0rd8caZFFq1vfMuI7keVIR2Ycg1zDweUQlwSR0Vug/GrEaELxyIfLIOR1781b8I3Lwa/5DNkSq0TfUfMD+hrn/Ke0bzYmDRnrjOOv/6609HnWHxJZzjhJHXOPfiomXE7PWhtuImPGUx+R/8Ar1mN1z3rZ1tcrA3oSP5Vj444Jrzq2k2enQ1pojb6c5puPXNPYcdeaMY9R9alM0GAdPyzXNa9GD4Tvkxnyp8gf8CH+NdPjkcCsTUoPM0nW4epA38/TP8ASqi9SJrQ8guxiVuvPPNQVavF+YHjpVXpXpR2PKe5678PJ/N8KonXypnX+Tf1rrZBliMcGuD+Fsu7T9Qg/uSo/wCYI/8AZa79lOR34rzq2k2epQ1popFAM8cVG3r+dWWXBxzTCpA5rO5rYgK4/wD1U1hwP1FSle3p2pGGQeMfhRcViDZ83Pr2rkPHltmxtZeTslKZA/vD/wCxrs8DOawfGlv53h6d/wCKJ0cfnj+RNaUpWmjKrG8GeRrw314rrPAcmL67iz96MNj1wf8A69co/wAszY7NXQeC5PL8SIn/AD0Rl/TP9K76usGedSdpo9P0qYW2sWU2fuTKT9MivaTXh20jp17V7Xbyie0hlHSRFb8xmuKkztqolpO9FBrcwEooNJTQineJmNx6g1XDblBx15q7OMrms6L/AFSjP3ePyraOxjJakgNLnFNzSE1ZI7NID6U3PrSg0xD81NCfmFV81LCfmoAvZopm73NFAHymg+bI64/TmoJpCkx2k89KlOFfsTgd/rVWfIk4HXtWC3Lexd8KkHUpz1zH6Z71d8VWyRQQygtvkc5BOR0qPwjGReXR/h2AZ/Gp/GT/ALqzHT5mJ/Ssm37axqkvZXOWiJWRSM9RmtQzbSAcgdR2zWSjFjtByOlWQHfGDgV0GBNI247u57jioz7cjtSgosZBY7gOCRURJBJPpikOw/kQNgnBbpTAqhflfJGO1Sq4FnMD13LgCr/huygv72VLhCyCPcACRzkUN2QJXZHpsZ/tyxxg4nQ5B9xXuKD58e9efwaRZW0gkgg2spDA7iefzrbTU7wv/rjyeuBXHXg6jVjvw81TTTOV8cHb4muRngrFx69K+i7c/uI/90V8yeLJpJdeDyPvJKA+/FfTNsc28R/2B/KtUrRSOeTvNslNNPAp1MNNEsjc1A5qZ+lV3PNaozZDOf3Tj2rmw3AFb95KI4ScZ5Arne/FRM1pEpJHNOViV601VJAPY1ba1EdrHP5sTbzjYrZdfqKzubogBwOufxpJ8tBL/un+VS7Bk0kgBhkA4+U0rjGWx6+nBq7bj/SUI4wc1RtSNvXPAqyrFcMDyO9S2NEisItWum6DCn9TWg92iQrIrAqE5A+lYU5Juix5JQZNTA/8S1vXYa4sV8NzWlHU3JdUijMAdGZskcHHVSKxNJ1O5a1njMhwk7hfXFOuzn7MR/fXP61Q0k4+2qc8XDf0rkUnyM0cUmazSs/32JHuaVG+RuO4qBTkHGKkQ/u3/D+dYSNYmFq5H2pifUY/KsubcDLuGDvhY/8Ajw4q/rWftXHTaKyrn/jzkbrxEcg/7Z/xr2cP/CR59Ze+yd1uDO4t3VScZyR71X8PF47+FHPKl1/DJo+cSk7SRtGeKj0STOqKQMfvWH41tNp02jKMXzpnVvzo8w7B3/rVK6/1YPHTrVsHdpt2uekh4/Cs+d8wIeDlRXjwWp6MtiKBiHXmnX/NvuOPeoIz+945qbUWxZdMkeld1JanNLYyT/x9oAD0A/SkJ5HehSPtsBz95e/4igjBrtRzjcfmaztUXdbR45PmY/nWiR0/wqreRh4hwMCTkfUf/qq47kszZWCTRqD0HSub1TIvmHsDn26Vt3Tk6iRwQvFY+sJ/pnvtBwM55z3rV7GaNv4fQ2k3jbTTdlRDHvl+cgAMillz+KivX4VX+0bpgysv2sEEHj5sf/F14p4UTzdSibo0cbHB78Y/9mr1LTL7cZozgFXjcYHT7o/pXHUpvnU7m9OStym5Km3BA5zzk1atGzAOeQTmmXSHySRgd6bZN8jDuGNdVJ3MaqsU/Flh/aHh+4VVzJCvmp+HP8q8+hkM1ur5BbGCG716wSCOcYPY15be2J0nVbm06CNt0bY6oeR9eP1FbGJUM0aEq8RXnk4ziqzxSpLG9vzCDvVgfun/AArQ3JLkZXzOw61nTWZjYuuUz129KUlcpOx6PdTC80i2uBj58N19RWbtIGMHrS6HcC58MLF1eB9rc+9SMoz7eleZidJnq4XWBCVyMCk24qcx9aAnbpmsUzdoh2nPSs2WHzLrUICOJbf+hFa+36ZqjIuzWovSSArz7H/69VclxPDr1du3qSMjkVRHWtjW4TDeXEfHyTMuPoSKx69WDvE8eatI9B+FcuNS1CD+/Er/AJHH/s1epFcqK+fNK1WfSZnmt5ZopGXbuifacenQ+grTfxlqb9bu9I/6+mH8gK5q2HlOd0ddDExhDlZ7U6HOcc+9Rsu3k4H6V4e/ia/c5aWZj/tXEh/9mqtJrE8v30Rv94s38zULCy7lvGR7HuLz26AFp4k/3nAqvJqemxjD39qvbmZR/WvF5NSiNoiJbKtwD80hCkH8Nv8AWoP7RuezIPpGo/pVLCeZDxnkezNr2jrjOp2ue/70H+VZOu63pF1o15bR38bySQsEVQTlscdq8u/tG7xgTuPocU0310etxL/32auOGSd7kSxbatYjnGJTwfXmtLw7KIvEli/Yyhfz4/rWUzM5yxJJ7k1PazG3u7ecHlHVvyNdEleNjli7STPamTINep+HJxceHrJs5xHs/wC+eP6V5p5YYZzkY6j0rvvBshbQzGf+WcrDH15/qa82k/esepWj7tzoPeg0ZpK6jkCk70tN6c00IRxlayR8ssq9MNn861/0rJuBsvWH95QfyzWsCJoTNGf1pmefejPPStTEfmjNMz1ozxSAkDc1LCRnNV881JEfmpgXePQflRUe73ooA+WgAWHTgD86o3HD8kdT2rUtWHmHdAH+UYzzjis242tdHGAc9+KwW5b2I4rmSAN5bkZAzz6VHc3k90R50rPt6AsTip/s+yBpGYNx25GKhtI1lmIcHAHQVem5OuwyHAPI57YqyDn+IDANQRhVudsn3RnI/Oo2Y9+tAEolzwDwOKf5UzgsqEj1xUOB5JO3r0q1p7ssgXdwynjr0pDFjhf7JcF0KMMYJrX8Fhj4ltoMblnPluvPTPaqkizSxNHwCAAD0xitzwDA9t4y05nK7RKucfWs67tTbNKPxo9Rm8MRQvteKWI4+63HFVf+EdWXULK1gPltcPs3NkgHFehaqY9QvvNiyV2gDIrndZD6ZNp96gy0U4YL68HivnqWJn7fkUro9uXJKj8KUjxv4jaPNofi+SxldZHTy2yo4IKg19EWh/0WL/cX+VfPvxT1Y614ylvDEIhtjXZ944Ar3+zP+hw9fuL/ACr3/so8WzTdywxpp/WkJppNNAxHPFVnPNTsarv3rRGTKl2m+L8QeawVAPX+Vbd7JsjHAOWArGQFTuxms6mhtR2JU6BR69alVG3hOMkjHzVEuSc9KkjO0/erNs3SZPdW01nOYpdpYAHKnI5GagcM6sADnBqTzGKlVU59hTvImZSVR847KajmRVmVrSJgigjHyitC2tI5nZZLhYQFyGYZz7VWtbO4aNCUl+6Odhp7GKJiGljUjghpAD+tK6Y0nYqzkCdef4Tz+NPX5rBwf7pqtdXdvHdKxuINqoQxEqnHT0P6Uz+2NIWxlLapbLJtYiMtkk9q58RFuOhpTkk9WXLps29s3PVP51S03JuL5f8Aptn6cCr91g2Fo8DCRHRGDYxwazY5pobq68rYSXBOfpXLCneLRrJ6mmAw7HFSxxM8cpzjC5x+Irn9TvdTsGAdlG4bgFCmqdt4muLZpXumMkZQrtUqDnj3qvqkmroz9vFOzL+qxE3SE9wKzL79xbz4xxGpz9HH+NQXPimGacM9tJtxjG8A5/Kla7TULN5EjaNWhlXG4NyCprupU3GCTOec4yk7Bfaqba8EfJ81Qn51X0f5bxmIw32ngDsOM/zq6BCCgxnK55I9vrVexZI72ckBT5oIJcDsPatuVcrM+Z3OjRwLe9UkD5wRnjqKoMDJbIFIJ2jjI61ae509EmRprcKzZG+Udxj2qsNY02C3jQ3VqnGARJz/ADrzVSlfRHa5xtuVo4pBNyMepq5eQGS0KkjkVTfXdPY5+3Iy9cgkinz+ILZ4lgjuCc4AG0jNdMIST2MZSjbcz2Tbb9iwGAQOnJpGR24COfoDUdxevDASsjqGz0J/z3qnLqLhN375xjgDJJ+gzXUrnPdGh5EpH+ok4/2DUbwyCNyyHC4P0qKez12GJJW0K7KSPsX5o/vYJ5+bjgdTjt6jMcdrre9mfSBFCv8ArHe8hyo9dobJ69BVqMiXKJzkjZv3GR16fjVHUxuuwScfu1z+tWUf/Sbhs8Zxj2qtfn/SwQONijp04rSWxmty54U41gR7gnyPkt/X8q7qzlWO+lUSrlohjGcHG4/0/SuC0V9msIytjKtj8q6eymI1mAE58xGXn1IP+NRa6K5rM9HuNRyvkhBjAGe/QUtg4JcjqCKyd4YRt6opzn2FXtOkGZBjHH4VrGKWxnKbluawbvXHeN7YJLZ3wwMgwucfiv8ANvzrrN/XB71i+KoftGgz4HMZEgI9qsg88nhWQjaSj+o71D9qvrJf36i4g7kffA/rVyIK+AR83UU6aJjHt8tmB5GBmgZr+GLyJ5Joo2zHPHuXHYjFdH5JIz+PNeZafLNp+oiVCVMb5ZOcEGugufipplpI8TafdtIv3gSijP5mvOxVOc5JxR6WEqwhFqTOu8ntis3VLtrKRAqZ3DOTXKyfGKJc+VpH/fdxj+Smsu/+Kct26n+zLT5QQNzs3/sorCOGrdjoliqPc6k63NuwqLTUv5rjULZljTeAwG5wg59SeK4iX4j3Lgf8S6y46ZQnH61CfiLfZBWwsBz/AM8T/jWn1aq+hk8VS7md4mO/V744AJmYkA5wSa588Ve1PVZNUvZbqaNEaTGVjG1RgY4H4VRY5NehTi4xSZ5tSScm0NopaUKT0BrQgbRVhLG7lx5dtM+f7qE1aTw9rMozHpV6w9RA3+FLmQ+V9jNoraXwlr7f8wq4X/fXb/OrCeBvED9bSJP9+5iX+bVLqQXUapzfQ52iurT4fawxAeWxjz/euVP8s1Ovw7vDy+qaaB/stI3/ALJS9tT7lexqdjjad2H1rrZ/AU8ajytRt5WzyPLcDH1xVe48IXSJH5cwkbGGAjIx6cnrS9tB9Q9jNdDpbb4hWEdjBG9pO0ixqrHeoyQBnvXo3ws8UweIF1OGKFojAY2wzZzncP6CvEF8G37MQZIlHbdnJ/SvQ/hVZnwtrl097eQeRcwbDjdwwYEdvrWHJSTvHc6FOq9JbHuVJ9aojWLA/wDLwB/wE/4Uv9q2OP8Aj4Ufgf8ACmIuUh/SqZ1ax5H2lP1o/tWxP/LylUhFuszUcJcwN/eDL/I1N/a9gePtUeazdV1jTzDEwvIcrIBy2Paqi9SJLQfuoLcVmnWdN7X1ufpIDTDrunjj7Rnv8qMf5Ct7mNjU3cUbjWO3iGyAyPPb6Qt/hUbeJIMEra3knssY/qaLoLG4G75qSJua5s+I8YK6fcgf7ZVcfrTR4mkDALaIp/25wD+gNF0Kx128+lFcl/wlj94Igf8Arqf/AImii6HY8Us7hIZT5jAfd4/CqV4ITI7CIHqfvHk1ZgtVudxcuCMdOe1SGzTedxJwOaySKexmo2dPY4IxuAwKqW7urMUHPrXQiyiNuEVT8xJwMiqdlaItww2kMB3+tO4rGSoaSbodxowy5DDB6Ee9bKW0f9okbAfmzz37+tQyxxrKwZF3bscjvS5h2MsM+3k9qltN/wBojweAavgBQAqJ/wB8irdvcNCT9wg9ex78UuYfKRrcqTja2Sey1t+HNQg07XbK+mErRI+5gqZOKrwywSN8zEE++M12PhrVZ4b23traO1TcwHmG3Utj6nmsMRN8jVjooQvJantuh20WpaZBfRuVjmQOqsuCAfWna34WXVrWOITiLY+4tsznj60/R73baqXEskmOWCcH8uK0m1Btvy2srcdsD+ZrzMJHC8qbWo6s6qqPU+U/iNa/YvEbqG3A7TkD619BWbD7HD/1zX+Qr5/+KEobxY+1Ng2pgEjIxmvfLRh9jgOf+WS/yFenTu6cWyallUZb3e9MLUzd70hYCtEZMGPWoHNOZ6hZ855rVGbKGpNiEYzncv8AOsY67GgK+ZaA9MeUmc/jWtqB3QjA6MD+tefvo9+9zKyy26qzkjdJ2z6AVlVpxluaUZuOx1H/AAkiwg5uolwOqxoP6VTfxq/2f7RG+oSxfLh47VwDnG3B2Ac5GPrWSvhy5lGJLu1X/gTf/E1dm0rVZ9Ik06XXbeOORQpeG3O7g5yDuHv+Z9sZRo0ups6tToOfxfqLthdM11vcxFB+pFQt4k1+ckQaLcnt+9vo0JJ9i1Rf8Izckkv4rvQTyfKt0T27GpF8J20g/wBI8Ra1Ljv9oIH061p7OiuhHtKrLNk3ie7077TDb6ciqxiIe7MnzA46qMdffsfbPGNNNcTSyu+0ytvKqykAnrg8120PhjSba18hNR1JoyxYr5y4JPJJ+XknAoTwzoES4CXBHQBpcflgCmpU18KE1OW7OSjZhDOAxJyhyce/oKx7hS1w7FgGxg5P/wBavU9K0vwsmozLfRstt5II3SOcsD7GtZrX4bI5drJZG6nKyn+ZrSKTVzKV07EvgfTbbW/BdpJczBLmEeW+xtwAHK/oRW2/g7TGclbyZGb7xVhz+YrJsPEfgzQ4Wi0yxkgjc5ZYoQAT68mpZPiDoijKWE7fUKP61ySwvvXTNlWlYS6+Gun3swefWrsgcKAEGPxxVV/hNo+0BdUvW998Y/8AZKe3xJsQf3WluRjvIB/SoW+Jef8AV6ao/wB6TP8ASrVBpW5iHJt3sTXHwm0fyFMd5deYBySy9fptqnqXgiHT7zT9M095XiuYp1aWQbgjFRjOBx0/SlX4j3UkoQ21vGCfvEM2P1FVZvHN/cXbbRAGtd0iEJ1+Ug9T70/ZW+0wUpdgT4SXSxqw1eDOOhibj9agHwxnhjmiNtZ3MrIwE5z949+Rn/8AX7CnXHxA1U6eksN9H5xbaYxCv59KzX8c+IpD/wAf7KP9mNRn9K1UbdSdR4+Gut+WkY03Sht5DM4wc9eNhqyvw68QHZ+60SPaAMYbjH0UVlv4q8QyZzqN3+Bx/Kq7a5rk33r68PHTzWqlyiszq7b4fa5HaGCS/wBPGXLkrG5xk5x2/XPb0FMi+F7RFWm1dBtx0h/+yrjpdQ1Bj+9upT/vSn+tRDUo0b9/eRLgdGnHX8TQ2uwKPmeijwJp8hEcuq5AzwqL/XNI3gHRUQ79TkB7OiIpHvn1ri7O8AjEsV5asGGRsuULEf7oOf0q0msGQY3E4/CuSpTqSleLsjog4pWZ1eo+H9LvbdILrxDqjxxSLKgiljQhxnByEyOvrWNP4X8Oojt9s1i4kUEr5t85Geo44zzWf/akSr828N9P/r1CdXj3Y3KfYtj+lONOp1kx3h2PPIgFuJRuJbPPuar3rh71o2+6UBz+FT3WLfVpl29WPU9aoaoJGuFZD83lBvTHWuuS0OZbnQeE2ii12LzYkkcK+0OoYH5SehrtLi4xLGywwIFccIgX+leZ6LfOb+B02iUPtO/JH5CuwluruRdqyWwPXHlkZ7df89KjlL5jqra+xDGrHG1AOvBx+FWYNUjSQhiSAOa4tNRvowN9uJO37tgcfnipoNTdWbz7e5AxwBEW/lnmi8kJKLO2bW7cZwG9uKr3WsxywSxeUzBlI69q5b+0mYHZYXzZ6YiA/mRTWv7rHy6VeHnqfLGf/HqOeQ+SJmSF4ziMHcvAwM5p8erSRp5VxEUc8BuSKPMcTnzIDCzciNiCf0qdV8yLY8Yx0APWtlqjJ7mIpdNQYyPvZ1zkVmaj4duNTvQ1oYgdvz7ztArb1P7NatGked45HOcD+lS6Y7fakAKjf69PpWc21saRSejOaTwHqTdbi1B9N5P9KsJ8PrsnD3sIOcfKpNejLYuRkyKM88DNSfYVzyx/BTXP7aZ0exgeeJ8PVwC+ot/wGH/69Tr4AtNozdXDH2VRXfCzUDJZuuegpBbAZ4PP6Uvaz7j9lDscCPAlopzvuGHocf0qaPwpaJtzaIdvqTz9fWu6FsmfuHI96kFtGM/L37jNHtJ9w9nA5aHTLRcf6Baj6W6E/wDoNa1vFAgwBs9kUL/IVq/Z0B6dvTrTlgX2rOV2XFJFNVt/785/7aGpDBasPmSVvqSateWmchsUYUdx171DizVSRVFpZjj7P/46BTxbQD7sAA+lTbowPvqPXGOaaXh6+aB9BS9mw50NMaKNqoAOgwM0woDjavt0qQzxD+MkfjxUbXUY4w3FUqbJdRDGg3ZIBzUbWpOMD9aebwIchGP/AAKmnUGxgKv51XsmT7REZsGPRgPahdKbeGDx8Hpg0pv5OvTPTimveSkfeP5VapMh1Ea/9o6qDnzYSB/snig6xqoAxLFn121im6l6GVsexphmYj7+fatFFmfMjb/tzVVI/wCPfH+0tKfEt8uN6wd++P6VgtJkn1qNjg9OlWokuRvyeKrraNqRZ7nr/SqFz4hvJ4JI8R7iOMoCuc+mKzDu5GMfjTemR1GapRIbHnV9Swu0W6H0EQqNtV1UnidVJ9IxTT75596YcDuSO1WQSfbtRPW9k/4CAKjea6fG69usDnCyYz+VJzwO1Jghj1z7DrQA0tJnJmuGPT5pWP8AWiOKMkkx5oJOOM5pUZgc5P8AjQIsFEBx5afnRSbv9pPxzRQM5CwjcbwIXbJ4wpPYc1I9vdMxK28uOvEZ/wAKqNdTZCx3Dhf7iOccfQ01Y7qckBJ3+iMf5VncqxdW3u1OTFIPquP51We0lRjIojU5JwZFGani0PU53HlaZeSHB+7Ax/pV1PCWvv8Ac0W++XGcwEfzpc8e4creyMp7VmfeJ4lfjOZV/oaWS3SVRuuoA3cgk5/IVsx+BfFEwwuiXQA6bgFx+ZFX7b4ceK3j2HTGjB5O6dAD+GaTqQ7jUJdjk3solGGvYz3OA3+FRskCcee57HCe9d0fhN4mmIJNsoGOHm/PoDVmP4M6y+PMv7OMexY/0pe2prqP2c+x52HjVRiSXjp8o/xq5b69PY3CSwvKHU5U7RjPrznNehp8F5xgy6tED/sxk/zqC7+EVvZQmS4109eES35Ptyx/lUuvSejZcadVao5n/haHiRF2pfThO+3YMf8AjlU7v4heIbobZL+5YAnH73b/AOggV09r4F0WAZuFubo9w77VP4KKs23gvQYMuLAsCfuyOW/Ks19XjtEvkrPdnmy/bvE2qW1tGglu55FjjLOfmJ4AJY9Oa+nbazvEtoYzbtuRFUncOoH1rgtF0PR9P1qzvhZQW4hmWQyklQgB69cY+teiN4x8Or11my/4DKD/ACrppuNRdjCopQeon2S7z/qef94f40GwvCPuJ+LVG3jbw9/DqSPnpsjdv5CmHxro5+5Jcyf7trJ/Va05YLqZXk+hKdLvG/55D/gR/wAKX+w7hus0Y/Amqx8Z2RHyWeoN9IMfzIpp8aRYymlX5z0LeWM/+PU7wXUOWb6E8vht5l2vdj14j/8Ar1w2p6ZPp2rT2yb5ghBD4AzkZ/rXXN4ymIHl6RIc/wB+4Vf5Zrz3xLpN/rut3GqiKO3WfarRCUtgqAAc4Gc4qJyg+pcIzXQ0BFccf6tT6GVR/WpFtJnGWuLdfYzA/wAq55fD+oIADctxwMSGpl8OXbxF2nDFeo3k8etZ+53NPf7G39iIPzXluB7Mx/kKd9mtl5bU4l/4A3+FYH/COs33pAD9OarXXh+YMTC8skmOVU4B+nvTvAVpHUBdLUENq6knsqf4mmtcaNGDuvpmI/uov/xVcMbTB6PgHByTkU9bIFiWTHrk9PSn7oanRXWqaUsm6K5lAIwd8fX2wBWPNrtuCQkdw4HGcIM/mwqNLaIH/Vj3zxSm1iONqjHcdxRdC5WQ/wBsq33bWcsP7zxj+TGmNrLkcWkf/A7gjH5KatrboFJCpnsc4I+tOWOMZDLFjvgYp3QcrM86teHlLW1Ht5rt/wCyCmjVNRfGxbUY54idj+rCtQRRdRtIHtTjGMgHa2T1FF0HKzLa+1NxkTRqfVbXt+LmmpPq6/Mt1ICQQcRRjg9RyDWtgAfK+RnHWgiL5htYADrjIouHKZONRcEG6uPQ4Ma/qEpptrz/AJ7XR56m4YfyIrYIUgAMD+gphKIeu3HTtRcXKjKa0nbgyS7f9qeQ8/8AfVRjSR951jYD+/ls/mTWyZY9uWKnjqRimi5t93oP9ocUXYWRlDRlP/LO269oVz/Knrpwi+6EUd9saj+QrS+0RnkNuGfT/P8ASozcAEEeYD05p3YWRUFmyDHny89g/Wmmx6bpJjnnlzVg3Tfexu/Q037R1JV1I9DnFGotCu2nRkgtkn6nrQLGMZATHXjHapzMxBIAZcYxUZkdVwu0nHAIp2YXRkayuydZFA5A5HFVrmU/6O4JwVKk/j/9etHUFNwseBkgkEgY7VD/AGPJdQ4jYKVYGqtoTfUz9MxbeILaQoGTeGw3GQK9Aj1eyHLwlM9PkBH6HNcBZqzXcLglXjJBIOD0NaZeUsdhY45xu5xUctylKx3kGp2MgG25iRfRhs/9CFXk8ubmORHGOi4avMWZimXIAbpuGOntj/P8gIT0GS2cFevb/wCtUul5lKr5HqAg6cDPQ8Y/lSmId3Iyex9//wBdebxy6moJinuOByBLwPwzVpNW1aAhWeIjPWRcZ/Ie/wDnil7MftUdNq9orlHRhvHqQTWau4YGSW6EAdKqx63JMQskSNk/fWVRjnrtzV1XDklcYb0raCsrGUnd3Mq4tnO6V1yS2c4ycCkhcxOjhclSPzrQeOWZG5Hl56k1mMPnA6gHHPvwalotM7qzvoZLZcqSVHt+dTm8TnCk/XFctpFyfL8vPHYd/atXzPUD644rL2aNfaM0Dejp5Q/OmfambkKoqkATnhseopQTjhv0o5EHOy0bqTJ4A96YbiXPL4/4DUALAcgHNKd3TA9iDT5ELnY8zOcgseaaZHODmT0puDnHIHuKXHsSPrRyoOZjd7EHO4n3NNyWGe31qTap4x+RpCnt/WjlDmITk9X59jQCwAxn8RUhUg9fz70woeuP1p2Fcbk5zuP5Um5jnmnj5ednPSkJVuoOfpRYVxhOR05pDIfTPb8KlAT1AzSBVHT09KdguM85gBuXikMmOuc1ISMgcZ9aYcZwMZ7nFFhXGF/rgUeYNuTTsDPGPwFIR0INOwriFlI6D8qQ7CP/AK1IwJ5yMdetMaNs5GAadhXHYU5H9aaQrA4/DJo2uCSWH1FNw2OSKADaB3FJjHfp2p3z8jC5FNPA5Ue4xQAwk565pvTjNPLqOGXikDKccMeOoFFxWIyevU8f3acp56CpAoP3d2PUjrUiQEjAGB70XHyjODyVGe9FWRAMdE/OilcOU9yXTbRFAS3hUDoBGKlW3RRwB+VFFeGdd2BjGfSgwjuTRRVJDuxhRV7ZpCAOcUUU7FJjWAHYUmflzgUUUi0V7iUxQtIADgdK5K78y8l8yaQn0AGAKKKqBZVFohcAk80jW6E5OeOgz0oorQQgt45Mqw3A8HdzmnNbQRSMkcSLtOPujtRRVxZLQqrxxgcelOdcsF9AKKKq5NhpGDjrz/8AX/pTgoaFT6MQKKKaEMCDn6Z6VLHGGLKTxtJ/IZ/pRRTAjaFQc0RoqyhlGCPXnNFFMRNNZxo6leFYBgMdM9qi+xIxYE49wORRRQBha1psP2M34yJFba4/v4bGfY/55rniFCD5R8wooraGxjPciaVVRf3eQzY5PTvnp7VELtljBA6vt560UVoZ3Hi5dmUYGcDnHrUUlzImw5GG9Bj/AD0ooqrCbGNdS7xhvmB6kCnLJL5nlvIzAjd6UUU7E3FbKMGLFvmA/OmuWDr87fN056CiinYQwfNGp6ZFKshAVG5LHGRxRRQA4KC23GDjqOOlCx4/iPWiikJimEbhgke4pme4J69zRRTAQFA20rndxnNRyzeWxIBJz1JoooAhFwJCD5YB9c/59avxQCZc5C/RRRRTEPnsDBF5rSBwZAMbcdfxpVh2cBjwRg0UU0JnNfZxHfIQejgHjrzXaR6NZyPsCsD9RjpnsBRRWaLB9LtYJAoVyR3EjDP4ZpVsbUqSYVb2f5v50UVQCC1slIH2OLJOOFH+FWGtI4Iw6pHz6JiiikMRSWA56VnsSZzjjJx0ooqokyM7UdXltrj7HGgYkFt8jE9z2GPSsc6uWeRZYFYpxlG25/A5ooqZblR2Lmjaz5155KwbOAc78/0rtMAgZz0B4NFFJjQvlgjcCRxn9KMEdGP40UUAKAfm5+lCpuPBI5+tFFAAYiuDu/SoixDcnPOOaKKAELkEA4NORtxVe570UUASTRlPm3559Pr/AIVACSTz6DpRRTADksRmggfrRRQAGEgE7vwxUe35j0/KiikA0LuBOcYpdnA5oooAaQVzg8CnICRnd9MiiigCUQ56sD+FL9nXuT+FFFJjFW1RwSCwx6nNS/Y0Q4znAoopNjSEFupb+HJOM7aayBRkYHpRRSuOwwDgMDjOacke9sEj06UUUAhViVsjA9elOWFByQfwoopDHGJMn5aKKKAP/9k=" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAAoAJ8DASIAAhEBAxEB/8QAHAAAAgIDAQEAAAAAAAAAAAAABwgABgEEBQID/8QAOBAAAQMDAwEFBgQEBwAAAAAAAQIDBAAFEQYSIQcTMUFRYQgUIoGhwRUjcZEyYqKxFhdSksLR8P/EABkBAAMBAQEAAAAAAAAAAAAAAAECAwQFAP/EACsRAAICAQIEBAYDAAAAAAAAAAECAAMRBBITITFBIlFhsQUycZGhwUJSgf/aAAwDAQACEQMRAD8AePU3UiRZr5ItcW3NupjbQtS3CkqURnjHhiuXpTr5pzULilQlxJcdtYbfdgTkSOyUc/xJA47jxnPB4NcTXbYb1ncNyQO0bZcH+3H2oM9OukjPT3WeodR2uQ0m33tKFCMhJKkubyonGMY/fknwrna5jRawDYwAQPOdbSILalJXOSQfSOs3IbdSFtrBSsBSSDwR51W7Z1E09cLv+Cx333HlOqZSvs/yytOcgK+WP/Cvvp9xSLLDZcUFFqO2gkHIOEgUFdOOKg6nmDdgwb44jxyB2mfuatYSNMbh1GPzI1qGu4R9fxDZeNd6bsFxRb7tdmmHVpC9uxSiE+Z2g4+fkfKu+iQhxoLbUFJUNySPEEUtPVJxaepy9yjtk2lpweHKVqT9qOGlLgqXpa2v78qMRtJV6hIB+oqlqBLQg6EA/eSr8VZfuCRO5FvUCVJXDTOjmQk4LQdTvyOSNuc1uOOhtG88+HFLt00mOMakjSH1bnkz5DKyrvKitaTn15oldYbrNt3T25TIEhxhxPZZW2rCgkuJChnv5Bx86lXWXuanuDiVfwVizzEICHO0QF4ODzXsr9f3oS9FrrIeauMN+U86MMut9osqCR8QOM93hRUSrcnNRU747KVntUhtKtpVz5VlLqVnAUDnyoW6g1lem9XzLTEfSwxHDaUp7JJKiU7iSTnzx8qqWkvaLsN+1G9puFdg7cohX28STDXHcwlWFEEjacHyzxzjxqOovXT2GtgeQB5eRlKqmtTeCMdIwBVjjvrG4E99acK4MzYjUtlR7N1AUPMenyoVN9eoX+YiNGIsLqmnJ3uHvhkAYc37M7Np4z/NmrqpdDYvQSZIV9h6wvqUPKvJUTwD61w9Xaqt2krBIv10U4I7GMhtO5SiTgJHqTx3gVVtA9atI69j3Jy3uSoirYz7w8JqUoPZc5WCFEEDHJ8MjzocNsAgdZ7cM4zCET4fvWDz3VRNE9ZNDa5vT+nrDdXZE1htTpQqM4gFCVAFQURjGVDxzzV2ffDTZX347vU0jKVbaesojAjImcfpx6VhRBzwPnVVHU7Q4nm1q1hY/fEqKSx7+0FhQOCnG7Oc8YqztupdSFtqBBAIxQIEfMEXUxnstYtubTh6Ckd3BIWftS/QOn970x1ua1RaZkxVnviZSp0cOLU0l3G4KI5ABJ447wMd9Mv1h0bqjULcSbpK5x4cpjchxT0ftdyDg4AyMeNDK2aG6xoltsTbnZFMk/mOe5rK8eOAFAZ+npWjXWAuGXB8OMGS0S+Da3Y5hk6fvLcsKUrJ/KdWhOfLOfvQvuKU27qFqSJkDdIZlJGPFaAT9aMml7QbbbWYqsqKE4JIxk+JP60Ket2j9YRb9F15oqAJ7gZMadCzguIB+FY88Z8OeBjPIp9Oos0rUMcEjl9RJWts1ItUd/wZyOsLzKtYaanMEYlRJDCjjnCcKAz8zRW6ZSO20dFRuyWi42e7wWSB+xFL5GvOpNSTYMSZoO+okMKUlkvs/lsKVgKO84wMfuPmKZPRNlXZ7CxCWdy8FazjGVHk49PCke823ICMYXEpwRVU3POTmBu2rNq1rdmP4REvi3AP5VKCvvRY6oRPfunl9ZSDkQnHE478oG4H+mhBruWiwdT7rDXDlLduimHo6WYy3Cs9mlJxtBHeDR3uURdx09JiutkGREW2pJ81III+tafk1pcdDgyJydKF7jMF3ROSkTUpKh+fASf1I2/9mjg2copU9D68sunBEXIu0BuVHbUy4w7IS2rjjBJ+VGfRHVa06luzNnjqirckJUpJZmpcPwpJPwgelYiVqudW8zLkF61ZfISv64HunURwHA94iNPJ45JBKftQ+6g6PhWnXGjtdw2ENOSX1QZCkjlZcGU5Pp8f70Q+rifd9X2KZjAkMusnj/SQf+VVvWjUrUttscKC62FW26sTV71dzaN27B55O7FNqGqp1S3WnAKY/UWgWPQUQZIaF3RUpTloLJwA24QPQYB/uTSu6vdVYeq1xf8A4VRLuZAzjIBcS4n6GmX0PvMNaiOFL4PnS9dereqB1JnuoGBKbjyAPM7NhP8AR/aj8HO/QOh/r7GDXjh6pG9feG7ri09N6X3RbWfyQ0+MHwDic/Sk3s1/maYuTzkNYQqS0/DWgq4cQ4ghSf2558QDjjl1JaBqfpW6jkm4WTIx37lM5H1pCdeRZqYaLhbmy67BktTsAA5bAw7nwP5al59K6FQL6RSP4t7iZrCFvPqIUfZlva4vVC1yCVBM5p6M4D35LauD5fE2KPHtMaoulm0CmNapTsdy4OLaccaVhXZhtRIB9Tj5Z86WHpXPZtuu7Bcm8BP4iw4VeaVKAV9CT+uaZv2lbaJugo7xGQxORuP8qkLR9xUKl42rIbvn2lHPDoBXtE+f0zcomlYmvYkiOYSJ5gyWinGzODuKh6Y7wMZpsfZO1nO1BpW5Wl15yVCtT6EwpJyU7Fg5aCu74doOPALx5UsrGvbxpfpvMtsW1uSYDFzXJuAbZQ4pLSmkhON3CU7m1HOM8eHiwHsx64vl1mps1wW41DetvvUeMthtCmTuRgHaO/ao8ZPd6VzCnCRT6nn+pvB3swHkIzC20uDBAIr5CA1vztFSpXRwDOeCR0n3bZSgd30rD0Zp1JQpIIIqVKGB0gyc5mj+BRN+8MJ5Oc4rdbjpbTtSAKlSkFSKcgShsZhzM1JNoiSZKJKmgXEcJVjkVte77UbKlShsCnIhLsRgwf3foxoW5yVy5GmLa4taitSjGQSSe85x31vaN6a6Y0ZJdlWOyRIbr6dq3G2UpUU9+MgZxkDj0FSpWZqlDbhn7ywtYrie+o3Tu1dQLaxDufboVGWVtLZeU0oEjnlJBI4HHoKHcH2eLdAkocavl87MKypsT1gK9D44+dSpS318X5iY9L7eghgslqatsRuK0nCW07QPIUOOr3SGZru8QrnBu4g7Ggw+DH7QqSFFSSMqGMZNSpTKzaevFZx2gIFzeMZl90/p+PZNOxbCwpxbMSOmOkunKikDHNL7pz2eb63rFci/uwjao/aNIQ3uWt9pSSnaoEAJ4Uc8ny9RKlMdRZWgrU8j1gWpHbeRzEqiPZr1lbdTItduVHRb2ZGY05xwlSWd2RlIGSoD5EimV6g6Ud1hom42FtxIkPtBTClcJDqSFJJ8hkc+lSpQq1VjWmw9RDZQipsHQxKpWmtYWCfPs8uxToypzaosmMqOpaHQrxSRlJ9CDx6ZxRH9nSNqVOvG3P8AD0+NEgNvR3VPRy0nYG8A5OOCsAAd5xnGOalSoamwvZg/WX06Ba8/5P/Z" } ] - } - ] - }, - { - "language": "markdown", - "source": [ - "# Load Text Recognition Model" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "// Loading the text recognition model\nconst recModel = await core.readModel(recModelXMLPath);\nconst recModelCompiled = await core.compileModel(recModel, 'AUTO');\nconst recInputLayer = recModelCompiled.input(0);\nconst recOutputLayer = recModelCompiled.output(0);\n" - ], - "outputs": [] - }, - { - "language": "markdown", - "source": [ - "# Define Post-Processing Functions" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "// Function to calculate the ratios for the image\nfunction calculateRatios(originalImage, resizedImage) {\n const realY = originalImage.rows;\n const realX = originalImage.cols;\n const resizedY = resizedImage.rows;\n const resizedX = resizedImage.cols;\n const ratioX = realX / resizedX;\n const ratioY = realY / resizedY;\n\n return { ratioX, ratioY };\n}\n\n// Function to convert the image to grayscale\nfunction convertToGrayscale(originalImage) {\n const grayscaleImage = new cv.Mat();\n cv.cvtColor(originalImage, grayscaleImage, cv.COLOR_BGR2GRAY);\n\n return grayscaleImage;\n}\n\n\n// Function to adjust bounding box coordinates by a given ratio\nfunction multiplyByRatio(ratioX, ratioY, box) {\n const scaleShape = (shape, idx) => idx % 2\n ? Math.max(shape * ratioY, 10)\n : shape * ratioX;\n\n return box.map(scaleShape);\n}\n\n\n// Function to resize and convert a crop to the recognition model input format\nfunction resizeAndConvertCropToModelInput(crop, netShape) {\n const [netWidth, netHeight] = netShape;\n\n // Resize the crop to the network's input shape\n const tempImg = new cv.Mat();\n cv.resize(crop, tempImg, new cv.Size(netWidth, netHeight));\n\n // Create the reshaped buffer\n const reshapedBuffer = new Uint8Array(netHeight * netWidth);\n let index = 0;\n\n for (let i = 0; i < netHeight; i++) {\n for (let j = 0; j < netWidth; j++) {\n reshapedBuffer[index++] = tempImg.ucharPtr(i, j)[0];\n }\n }\n\n // Clean up\n tempImg.delete();\n\n return reshapedBuffer;\n}\n\n// Function to extract recognition results from the model output\nfunction extractRecognitionResults(output) {\n const outputData = output.getData();\n const outputShape = output.getShape();\n const [batchSize, height, width] = outputShape;\n\n return setShape(outputData, [height, width]);\n}\n\n// Function to parse annotations from the recognition results\nfunction parseAnnotations(recognitionResults) {\n const letters = \"~0123456789abcdefghijklmnopqrstuvwxyz\";\n const annotation = [];\n\n for (const row of recognitionResults) {\n const letterIndex = argMax(row);\n const parsedLetter = letters[letterIndex];\n\n // Stop if end character is encountered\n if (parsedLetter === letters[0]) break;\n annotation.push(parsedLetter);\n }\n\n return annotation.join('');\n}\n\n// Function to crop the image based on the bounding box coordinates\nfunction cropImage(originalImage, xMin, yMin, xMax, yMax) {\n xMin = Math.max(0, xMin);\n yMin = Math.max(0, yMin);\n xMax = Math.min(originalImage.cols, xMax);\n yMax = Math.min(originalImage.rows, yMax);\n if (xMin >= xMax || yMin >= yMax) {\n throw new Error('Invalid crop coordinates');\n }\n const roi = originalImage.roi(\n new cv.Rect(xMin, yMin, xMax - xMin, yMax - yMin)\n );\n const cropped = new cv.Mat();\n roi.copyTo(cropped);\n roi.delete();\n\n return cropped;\n}\n\n// Function to log the bounding boxes with annotations\nfunction printSortedAnnotations(boxesWithAnnotations) {\n /* Sort the boxes with annotations based\n on their position in the input image */\n const sortedAnnotations = boxesWithAnnotations\n .sort((a, b) => {\n const [aXMin, aYMin] = a.box;\n const [bXMin, bYMin] = b.box;\n\n return (aYMin - bYMin) || (aXMin - bXMin);\n })\n .map(item => item.annotation);\n\n console.log('Sorted Annotations:', sortedAnnotations);\n}\n\n// Get Text size\nfunction getTextSize(text, fontFace, fontScale) {\n const canvas = createCanvas(200, 200);\n const ctx = canvas.getContext('2d');\n const adjustedFontScale = fontScale * 35;\n ctx.font = `${adjustedFontScale}px ${fontFace}`;\n const metrics = ctx.measureText(text);\n const width = metrics.width;\n const height =\n metrics.actualBoundingBoxAscent +\n metrics.actualBoundingBoxDescent;\n\n return { width, height };\n}\n\n/* The convertResultToImage function visualizes object detection\n results on an image by drawing bounding boxes around detected\n objects and optionally adding labels to them. */\nfunction convertResultToImage(\n bgrImage,\n resizedImage,\n boxesWithAnnotations,\n options,\n) {\n const defaultOptions = { threshold: 0.3, confLabels: true };\n const { threshold, confLabels } = Object.assign(defaultOptions, options);\n\n const colors = {\n red: [255, 0, 0, 255],\n green: [0, 255, 0, 255],\n white: [255, 255, 255, 255]\n };\n const [realY, realX] = [bgrImage.rows, bgrImage.cols];\n const [resizedY, resizedX] = [resizedImage.rows, resizedImage.cols];\n const [ratioX, ratioY] = [realX / resizedX, realY / resizedY];\n\n const rgbImage = new cv.Mat();\n cv.cvtColor(bgrImage, rgbImage, cv.COLOR_BGR2RGB);\n\n boxesWithAnnotations.forEach(({ box, annotation }) => {\n const conf = box[box.length - 1];\n\n if (conf < threshold) return;\n\n const [xMin, yMin, xMax, yMax] = multiplyByRatio(ratioX, ratioY, box);\n\n cv.rectangle(\n rgbImage,\n new cv.Point(xMin, yMin),\n new cv.Point(xMax, yMax),\n colors.green,\n 3\n );\n\n if (!confLabels) return;\n\n const text = `${annotation}`;\n const fontScale = 0.8;\n const thickness = 1;\n const { width: textW, height: textH } = getTextSize(text, 'Arial', fontScale);\n const imageCopy = rgbImage.clone();\n\n cv.rectangle(\n imageCopy,\n new cv.Point(xMin, yMin - textH - 10),\n new cv.Point(xMin + textW, yMin - 10),\n colors.white,\n cv.FILLED\n );\n cv.addWeighted(imageCopy, 0.4, rgbImage, 0.6, 0, rgbImage);\n cv.putText(\n rgbImage,\n text,\n new cv.Point(xMin, yMin - 10),\n cv.FONT_HERSHEY_SIMPLEX,\n fontScale,\n colors.red,\n thickness,\n cv.LINE_AA\n );\n\n imageCopy.delete();\n\n });\n\n return rgbImage;\n}\n" - ], - "outputs": [] - }, - { - "language": "markdown", - "source": [ - "# Async Inference Helper Function" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "async function inferAsyncProcess(\n tensor,\n recModelCompiled,\n recOutputLayer,\n i,\n annotations,\n) {\n // Create infer request\n const inferRequest = recModelCompiled.createInferRequest();\n\n // Define the completion callback function\n function completionCallback(outputTensor, i, annotations) {\n const recognitionResults = extractRecognitionResults(outputTensor);\n const annotation = parseAnnotations(recognitionResults);\n annotations.push(annotation);\n }\n\n // Start inference in asynchronous mode\n try {\n const result = await inferRequest.inferAsync([tensor]);\n completionCallback(result[recOutputLayer], i, annotations);\n }catch (error) {\n console.error('Error during inference:', error);\n }\n}\n" - ], - "outputs": [] - }, - { - "language": "markdown", - "source": [ - "### Do Inference and Show Detected Text Boxes and OCR Results for the Image\n" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "// Process each bounding box and run inference on the recognition model\nconst [batchSize, channels, height, width] = recInputLayer.shape;\n// Calculate ratios\nconst {\n ratioX,\n ratioY,\n} = calculateRatios(inputImageMat, resizedImage);\n\n// Convert image to grayscale\nconst grayscaleImage = convertToGrayscale(inputImageMat);\n\nconst annotations = [];\nconst croppedImages = [];\n\n\nfor (let i = 0; i < boundingBoxesArray.length; i++) {\n const crop = boundingBoxesArray[i];\n const [xMin, yMin, xMax, yMax] = multiplyByRatio(ratioX, ratioY, crop).map(Math.floor);\n const cropRect = new cv.Rect(xMin, yMin, xMax - xMin, yMax - yMin);\n const croppedImage = grayscaleImage.roi(cropRect);\n\n try {\n const preprocessedCrop = resizeAndConvertCropToModelInput(croppedImage, [width, height]);\n const tensorData = new Float32Array(preprocessedCrop);\n const tensor = new ov.Tensor(\n ov.element.f32,\n Int32Array.from(recInputLayer.shape),\n tensorData\n );\n\n await inferAsyncProcess(\n tensor,\n recModelCompiled,\n recOutputLayer,\n i,\n annotations\n );\n\n croppedImages.push(\n cropImage(inputImageMat, xMin, yMin, xMax, yMax)\n );\n } catch (error) {\n console.error('Error during preprocessing:', error);\n }\n\n croppedImage.delete();\n}\n\ngrayscaleImage.delete();\n\nconst boxesWithAnnotations = boundingBoxesArray.map((box, index) => ({\n box,\n annotation: annotations[index]\n}));\n\nconst resultImage = convertResultToImage(\n inputImageMat,\n resizedImage,\n boxesWithAnnotations,\n { threshold: 0.3, confLabels: true }\n);\n\ndisplayArrayAsImage(\n resultImage.data,\n resultImage.cols,\n resultImage.rows,\n display\n);\n\ncroppedImages.forEach((croppedImage) => {\n displayArrayAsImage(\n croppedImage.data,\n croppedImage.cols,\n croppedImage.rows,\n display\n );\n});\n" - ], - "outputs": [ + }, { "items": [ { "mime": "application/vnd.code.notebook.stdout", "value": [ - "Annotation for box 0: building", - "Cropped Image Size: 159 x 40", - "Annotation for box 1: noyce", - "Original Image Size: 690 x 517", - "Cropping Coordinates: (256, 50) to (377, 88)", - "Cropped Image Size: 121 x 38", - "Cropping Coordinates: (604, 205) to (653, 228)", - "Cropped Image Size: 49 x 23", - "Cropped Image Size: 26 x 32", - "Cropped Image Size: 31 x 23", - "Text: noyce, Width: 74.716796875, Height: 21", - "Text: 2200, Width: 62.2890625, Height: 19", - "Text: robert, Width: 73.14453125, Height: 20", + "Box 0: [391,59,550,99,0,0.518794059753418], Annotation: 'building'", "" ] } @@ -230,15 +143,18 @@ "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAIFArIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1SXSkkG1ihGQehHQ59a429+D3h+9klkY3SNKxZjHcY5Jz/Epr0RvvfjSgVEcPSg/djYp1pyWrPKx8C9HkJUahqKLjj50b/wBlFMm+BFkUKR6veY940r1uM7cn0BNYmla9qOu28F3p+nxJaNjfJcSkFjn5ggAOcdMnGT+dEuWLsa06NSpFzWy3bdt9vyPNZPgZtxs1F2x/eOP5IarP8GJ0JImd8/3ZlH81Fev3eqXT3sljpNtHcXEIBneZykcWRkAkAksfQDjvUunapLNdPYX8C29+ieZtVtySJnG5T9eoPI/Wl7t7DdCooc/ztfW3e29v+H2PEI/hZLIZhGLl2jco4W4iO1uv8jQ3wynjI3W2pn1IUH+SmvUtNuNTOqa5FptlBKov2Z5riQqmdiDaAAST39P6bum3yXrzW1zai3vrfHnQk7hg9GVu6nB9/XFTBpmuIw0qeq1Vk91dXS3XzPCm8CxwMfNGoJ/vqB/NajHhOyHBuLnP+8v+Fe2/2hf38jtotlbyWsbFPPuJmQSsDghAATgEYyevbjmpP7RtbjRLy+ltWWSzWTz7eQ5KOgyVz0I9D3BovFkPDVV0+V1dX7roeJf8IpZ8fvrn/vpf8KVfDFgZCq3FwWQDI3Lx+leuWEJ1sW93FollFp0gB3zhfNcEfeVQp49MnJ68VladoUs3i7VIZdP0/wAhPJ37ZHBVSDgqABknv0qG1pbqbRwkkp87ScVfdd0rf11Pn6JmwRuP3j1+tTozFY8Z+71r3uX4U+HGB8uzCHOcpNIP5saoy/B/SGACSXSAcDbcAj9Y6aptM5nNNHiW47mPJHTimFmxzz6HFewzfBqHJMOpXC+zRo/9VrOn+Dd8ABDqStj/AJ6QY/kxqrMi55iTyORSgkkjca76b4Sa9GcrPZOPTMg/9kqhN8MvE0IbZaW8mf7twgH/AI8RRZgcjvYJ94/j2pwLhgMn8a3pvA3iSHGdHnbHePbJ/wCgk1Sk8Pa1bkmbSdQQAdWtnA/lSsO5nglien5Uu48eh9Kc0E0QAlR0bOMMuOtAUlhnnFSUKCc4596z53ddTixwcD+taIBznHtWbd8XcRAznvn3qZDRoxSbo+uecCnb8DOc1HAdsW/vninE4AXB5pR2KYF+2Rj2oLEDP60Agk+1NJXbkjr61RI4ls5zTdxzycd6VsdOfejbke3Tr2qRiZ+7nIPajJJyePrS7TnHJNOCkHj0pXGAJFNY7kOPT1p+DkcDJ7UpU4OecjNSxmXZOymUc9sj860tiNhtuAeazYGKzSrj9a14cmBOOCKl7jWxD5KnGVNIYB/eb061cCgDPApSoxwuaLhYpfZ/4Qx980z7Kc84I7ZFaGwHigJk8UuYdigLIsASqE/lQ1mccJ36A1qCP0FIVzmp5h8pjG1bsjH39aja0YHJDqM9xW6UwOeMetMaM7CcHpRzBynOjcr+vap1i3ICcA+tJOu2Q8d61LBVe0AKg8kHjNDYJGU9kAhZokPfgCsXULXbGXSELz/CMV190keCgUY74rEvo/8ARZMdAc/jW1KTTM5o5bawPANHmOP4m/OrxT3PPrUezII6/hXZzHPYq+Y/948UeYe9TmMYzgflSeWCPu496LoRBvNLv4xTzGMcgimMgC5GaegaikngnPNN3nnnrT25jT2BH6//AF6j2mgBd+KXzD6U3BpMH0phckVwDyOKeZhjAzj1qHB9DSUrBcl3gjFG4VFRRYBz802iimIcjlJFcHBByDX25o999v0SwvAcie3jl/NQa+Ia+u/hfem/+Guhyk5KwGI/8AYr/SuLGLRM2o7nXlzSbzSUhrhOiyF3GjcaSkqh2FViHYZ9DT9x9ajHDj3Bp1ddPWJEh24+tG4+tJRViF3H1o3NTaKYh28+tQzMd8bZ74qWopwfLyOxzQxoXc3rSb29aTtSGkAu808OcVHTl6VLQ0O3tVS+lmjsriUT+XsQsCEzjAz+NWGOB0yT0FVLyyF3aTQzOxEileOAv0Hf8alq+g9jjpPGlsLaG4kt2lkk2sFKqCmf4s7jj6cH1qnrN7p8olDXVybskI0YyfL9to4BPoxBxWfd6TBeWF3ZiZkvNPYx+Y2CSoHPfv1/SuIW+2RQ2zTq08EmFwAVUAE4ORzg++M1jUpXi0ZOo0zrPtunAcre574lQfp5dFUl8WMFAdLgsByQB1/OivH9jU7G3NE90b71KOaRvvUor7Bnnoeoyrj/AGTWb4Qi8nwlpibNhEIypGOcmrstxDbIHnlSJScAuwAz+NRDVNP/AOf61/7/AC/41jKcIyvKSTKeKhCm6Umldp79r/5lCK7h0DVtRGpOILe8mE8Fy/3GOwBkJ7EbeM9QeOhp9tMus+I4L+0+extIJI/tAGBK7EZC+oG3r0z9DVttR01xte9tGHoZVP8AWnLqemqoVb21AHQCVf8AGs+entzqxq8xw9ua65mrXurbW2728zG0rWLLTNW1uDUZRaGS+Z43mBVHGxBwx4J9verumk6nrt1q0Kstk1stvDIRjzvmLFgOuOQAe/OOKttqOmSLte8tGHo0qn+tOGqacOBfWv8A3+X/ABpKdNaOasOpmWGldxaUmkn7ytpbb7u7ORsNL8OabbfYtcX7LewEhjJcSIsq5OHQ7sEEdhyD1ArZFraJ4N1VdPsZre3mglZBKW3y5TG7BJIzgYzz7CtN9R0uTG+8s2xyMyqcfrT/AO1dO/5/7X/v8v8AjSj7FaKS/A1q5zCo1Jzu73d5afJdPxsS6Onl6LYJt27beMbcYx8o4rJS5g0vxfqBvZVhW9ihMDvwrlcqVB/vcjj3rR/tXTv+f+1/7/L/AI0janpjY3XtocHIzKvH61TqU9LSWnmc0MfQUpuUlaXmu6f5ou0tUv7V07/n/tf+/wAv+NL/AGrp3/P/AGv/AH+X/Gr9vS/mX3nP9Yo/zr70XKKp/wBq6d/z/wBr/wB/l/xo/tXTv+f+1/7/AC/40e3pfzL7w+sUf5196LlHWqf9q6d/z/2v/f5f8aP7V07/AJ/7X/v8v+NHtqX8y+8PrFH+dfei0UQ9VU/UU3yIv+eafgKr/wBq6d/z/wBr/wB/l/xo/tXTv+f+1/7/AC/40e2pfzL7w+s0f5196J2tomGCvHpk1Tm0HSrg5lsLaTPd4lb+YqX+1dO/5/7X/v8AL/jR/aunf8/9r/3+X/Gl7al/MvvH9Zpfzr70Zk3gvQJs7tKsx9LdB/ICsy5+GHha5YM+mRhh0KvIv8mrpv7V07/n/tf+/wAv+NWldZEV0YMrAFWU5BFOLpz+Fplwrxn8Ekzg3+E3h/nyhcReyTHH/jwNU5fg5YOSYL+6j7Dcyt/7KK9JrH8VT3Nv4ekks5fKn86FVfsMyKOfUc0SjGKbsdNGMqtSNNPdpfeefyfBeXawj1nJPrb/AP2dULj4Pasm3y761cDpuVx/IGvQtX0ZdL0a61WO+vH1K3jMxuHmb5yOSCgO3aemMcCumD+ZFG+MblBxUqKbs0XVgoRU4Sum2traq34a/wDAPBp/hb4hiDlBaSn/AGZSM/8AfQFZ8vw/8RQAZ01WUcnbPG36bq+iaQgEcgH603SRiqjPmx/CuvIpc6NfMO2yFm/kKpy6Tf2+BPZXUXrvhZf5ivppYYigBjT/AL5FH2aH+4B9Dip9iUqh4UfCtjuOZrjj/aH+FH/CKWW0/vrnJ/2l/wDia9j1LWbGz1H+zvs89xetGskcEQyZASR3OABgkk8Co7J7O/vHsr3Svsl4sfmiOZY33JnGQRnv1qOWN7G3saqhztab/Lvbe3meIf8ACIWC3DfvrnJH95f8K0rfwvZC3RfOuMDP8Q5/SvUtSsdIjuza2ujfb7xRukjgVU8tT03NkAE9h1NJYaPpF+ssIs5LS6i/1ttIzB19CPmIKnsRxQ4JuwezqKHO1p/Wtt7eex5r/wAIzZY/1k/5j/Cj/hFbIgDzbj/vof4V6TY+HdM1Syju7V7hVbIKswBVgSCCNvUEEVTi0fT5NPuNQW9eK0id082RRg7TtJHIyMggetRyIOWd2rap2+fY4b/hFbDPLzf99D/CnL4WsR/y0nyf9of4V1DWFxHtmns7u3sieLiWIAKOxdQxZR6kjjvirGo6PLp0Ild45GkO2KKMMXlb0UY5P9OanlW5o6VSLUbavtr+Rx58MWRPEtwMD+8Of0rK1nSYtP8AJ8mSQmTdnfjtjHTHrXbLZ363MNvd6fJavMSIjI6lXPXGQcA98HrVDxF4e1OQQFLCdyu7/Vjdjp6VEo+7oglGUJJS/wA/xRwRhOSBj3OKYyN5fbn3rcl0TUYiA9heIAOS0LD+lUJbd4j86upHQEYNY6j0OTvkKuQQM5ycGrWnTFbVh1Ibik1FCocY+ZuTUOnfNC49xWiV0Q9yy5y2e59azb1AYZQegHWtIg4z3qldLlXGOCp960gRI54RtI4SNWdycKBySe3FSf2TqR+X+zrvB6/uW/wqxpRxrFgBn/Xx+/8AEK9N4IrrRzs8pOk6lz/xL7v/AL8t/hSNpGpcZ066z/1xb/CvVvoeaTJ79Kqwjyg6RqPT+zbv/vy3+FMfSNS2nGnXf/flv8K9ZNNyGUkHI+tMLaHk40jU/s6/8S674Y/8sG7j6e1L/ZGpHA/s67/78t/hXqy/6th6EGgc0xHlH9j6j/0Drv8A78t/hR/Y+pDrp13/AN+W/wAK9WJGQCRk0tA9jyf+x9SP/MOu/wDvy3+FB0bUhx/Z13/34b/CvWPwooEeTf2Pqf8A0Drv/vw3+FVHiaORo5FKupIKtwQfQ17JXk+sf8hq/wAf8/En/oRoAoEYptOI4pKYhK+mfgNffafh89uWybW8kQD0BAb+ZNfM1e7fs73vya7Yk94plH/fQP8ASufFK9M0pP3j3E0ZoNJXmHWhaKQUtUAh4Kn3p1Nf7v05p1dVLYiQtFFFakhSUtJQAtNcZUj1FOpDTsBCnKD6UppqMDuUYyrEH2p1IBKcvem0ucZx1qWNDgCTgDmms8IVw0ijHv0NTp5KrvldOR/EeKwLzWrYas2nea6Fl3hU4yO+ep98AfXFZSq8iuPd2OA8ZmWHxN9v06SNFaHZK0imME8jBYgDpn3ribTQMTrqWoX9vZ2srHylLY83nnHfHuB19a9V8QatBJaLBLbSTRthkiSEoiehyy7cjr1xXll34fvdS/eXd4ke5iwlnPl/KPqeePSuWddyVm7EuCT2udjHD4SSJFDIwCgAlGyf/HxRXMx2Nokao19bFlABO1+f0org9k/52XzeR9CN96lFDfepRX1rOBHGeL2Y6nChY7RCCBngEscn9B+VSt4SChib7hf+mP8A9lUPi/8A5C0X/XAf+hNXVSjKyL0yCK+ew+CoYrGV/bRvZq2r8+x8tQwNDF47E+3jezVtX59n5Hmej6p4f1q7itLfVZo55ThEmtCuT6Z3EZrqR4Ryeb7H/bL/AOyqKy8FWVtcQTkq7RMrhguDkcitX+1hLrJso0lHlJukZ0KgkngDPXoeRxXof2Lgf5Pxf+Z3/wBhZf8A8+/xl/mUx4NyP+P/AP8AIP8A9lTG8IYOBfZ9f3P/ANlXUofkrF8TRvP4cvII2KtMojyDzhmAP6E0v7FwP8n4v/Mf9hZf/wA+/wAZf5mXH4XjlBMeoo6g4JRA2Pyapx4OB/5iH/kH/wCyrH8H+HH0LVhLC7LDMhSSPPB4yD9cj9a7rzNsuz2zT/sXA/yfi/8AMFkWX/8APv8AGX+Zz58GBRk6hx/1x/8Asqb/AMIhH/0Ev/IH/wBlXTyN+7/EVxXjjUNesJ9POi3CxhlkMqNErB8bcdRx36etL+xcF/J+L/zH/YWX/wDPv8Zf5mgPBit01IH/ALY//ZUv/CFf9RD/AMg//ZVrafcyS6da3NwoSWSJGdRxgkDI/M1oq2RS/sXBfyfi/wDMP7Cy/wD59/jL/M5j/hCv+oh/5B/+yo/4Qn/qIf8AkH/7KtXU71rHTby9SFpmgjeQRKcF9oJwD+FcpZ/ENrlgp0LUVY8YQ7sfoKP7FwX8n4v/ADH/AGFl/wDz7/GX+Zrf8IT/ANRD/wAg/wD2VL/whP8A1EP/ACD/APZV0scpdQeRx3qN52Bc7sKvt7Zpf2Lgv5Pxf+Yf2Fl3/Pv8Zf5nO/8ACE/9RD/yD/8AZUv/AAhP/UQ/8g//AGVSaV430PWblLew1SOWZ/uxtGyFu/G4CuiSbcORR/Y2C/k/F/5h/YWXf8+/xl/mcx/whX/UQ/8AIP8A9lR/whX/AFEP/IP/ANlXR3F4kBQMQN2cZbFNF6CPuEj2OaP7GwX8n4v/ADH/AGDl3/Pv8Zf5nJ6n4X/s7T5bv7Z5nl4+XysZyQOufetjwe7No7hmJCzMFBPQYU8fiTT/ABDcrLoN0oVgfl6/761F4O/5BEv/AF3P/oK1x08NSw+ZRhSVk4+fd9zho4SjhM3jToKycL7vu+/odFmqOs2L6nphtY3VGMsb5YcYV1Y/yq7XO6h4sis72S3itjN5Z2sxfb8w6joa9XE4ijQhes7J6f1Y9+tj6WB5a1SXLrp119DZ1m1bUtGvbGNgjzwtGrN0BIxVlPlhjTP3VAzXJ/8ACa/9Q/8A8jf/AGNH/Ca/9Q//AMjf/Y1x/wBr4K9+f8H/AJHC+IsA4KHtNN9pf5eR1uaM1y0HjOJ50We0MUZOGcSbtvvjFdRXZh8XRxKbpSvb+up1YTHYfFpuhK9t91+Yi9KdmmL3HvTu1dB1mTCB/wAJ5dtgZ/s6IZx/00eluz/xWum/9ec//oUdam1RIZAo8wjaWxyR6Zo2qZBIVBdQQGI5APUZ/AVHJpbzudX1lc3Nb7PL+FjI0iWOy1nVrW6YR3E1ybiIvx5sZVQNp77cYOOn40sUiX3jET2mHitbVoZ5k5UuWBCZ7kYJPpn3q/e2FpqMQivbaK4RTkLIoYA06KKGwtBFbQiOGNSVjiXHvwBU8j26GjxEHeaT5mreW1r/APA6HNapdXGhajf6faAhtYw9ljok7EJJ+QIf860NZ05bHw5Y29tE8lvYzQPJGq7i0aMMnA6n+L8KW1huNV1eLVbuB7e3tkZbSGX75ZusjD+HjgDryelbe4+pqIwvf8DarifZumktVZy83svuX4tkEus6dFZpdG7iaGTAjMbbzIT0CgZLE+grC8S2wfVdIu57i5sbNFkjaaFgnkuwG3ccEAHBXPqR61rQ6PptveteQ2NvHckkmVYwGyevNXHVZY2jkAdGGGVhkEehFVKLkrMwpV6dCop079b3t1VtN9u/Xsc++l2MWoWST63qV7N5qyxW7TB8lTncQBwB6nA/OulkAOAQD9aoWOl2GmlzZWcNuX+8Y0C5q5knqc04xsTiK/tWrNtLvZfghvkxHny0/wC+aRreJhgrx9TT6KqyMLsoz6Fpdznz7C2kz18yJWz+YrPfwN4akz/xJ7Nc9dkKr/6CBW/RS5UO7OSn+GfhibJFhsPqksg/9mrMufg94fnJKy3kRP8AcmH9VNeg5ozS5IhzM8nT4E6ZBeQz2+qXi+W6uA+x+hz2C1Vn8MRW2rahZfbppJ4ZFCxRWrSMylFJbCngDOK9kU8isPQmi/4STxIFK+Z9oiLD+LHlDH4damad1bv+h14Xk5Krmrrl/wDbonn9p4GvNRgaayvbaRVYo6sHR0YdVYbeD7Gq0fgzV51lNvHDMIpGiYrKBhh1HzYr0yMqfHUv2ftYAXWOm7f+7z743fhR4X+VNWU/eGpzkjuMkEfoQaFJ3sKph4KDmuydvXueSnw1q7SzwrYySPCQsixkNtJGR0PoRWdJpV7HpslwljcC3CsfMETbRjg84x2r2fw/cQ3WveInhdZUFzGN6nIyI1BGfYgisq3Rf+FV3p2jPkXPOP8AbeolO6+TOmlQ9nNK7+Knp6q+q8uh5UYpIY2M0bxjb1ZSKpyzAx4G5SSMZGM817nq8cQuvDpuB/onn/Pk/L5nlny8/j+uKj8aWNgmgF5IIQxuIFBKAk5kXIHfpmnOUnF26CwtKnGtSck25P7tbdte78jxV1TzELAbu1DyqpIJ59BXsOs+HNIPiLQ4RYW6xzPMHVYwu7EeRnFVU8FaHP4tvrZrMCOO0hdQrsMFmcHv7Cm5tNpLr+hlGjFwjKo20o3tf+81ZdkeUpIHJAyCOxGDQ8gQ45J9AMmvTbnwHpJ8VW9lGJkiezklO2TnIdQOoPqap6d8PrO7utTgN3NFcW1wVKnaTsIBRsYHBH8jT9o9hPD0lepZ2sna/d23tt8uy8zz5JFkztPI6j0ryrV1H9tX/P8Ay8Sf+hGvopfABl8R3mnw34LwQRyMTEP4i3Bw3HQH8a4zV/gT4hk1C6uLe9sJFklZ1VxIDySeykU41LrUxxFGMJpQ2aT181c8aIHem45r0m6+CnjKH/V2trN/uXIX/wBDxWPcfC3xranL6FM4z/yykjk/9BY1aku5z8rONr1T4CXnkeO57fOBcWTrj1IZW/oa4m58FeKLQnzvDuqKB3+yOR+YFb3wuF3pHxL0d7m3mgDyNCfMjK/eUqOvuRU1bODCCakj6rNJRmivKO0KUUlFUgHEZBHrQpyoPtQOaRD8pHoSK6KREh1FFFbkBRRRQAUhpaKoRABiVwB15pTSP/rlPqMUp5pBcKlgh8x9xAwKiHLqg6noK0dvlQ4QZIHHuahpA3YpXEUNwx+1Mq2ykDaWwGY+v8sVha9pOlw2c0ir9jD/ACtMkrRFs9RwCWPTjHNdO22PBIzjpxk5qte2EeoweVeIGhYg+Vng49cdfp0rmnTTTTV2wUtTztS06Yim1S6hQZBkUspwD91SCQePY4471xt61ld6sls9/fM9zKUaJ48FRxk4GcDj0H0Net61cvJbxw6bGQkbKA6LkEZwVXHt3OB2z6ec+JdRG26ewnghuWg2yrb4Z1IBXBYf7wIxjPNebOnyysnc6FrG5lyrp6Sui+C9YKqxAy0o4+gTiiuf+y66eYnuPL/g3MM47Z5op+yj3/FmPN5H0s33qKVh81Ar6hnGjivF3/IWi/64D/0Jq6W+5s7kesbfyrm/F/8AyFov+uA/9Caresf2+uoPHavbNZS4GHU7lBGDz+deRlzSxmJv3X6ngZam8divWP6nHeGvC97o2p2dzb3s3lB1EkZfhl6EH8K9FnC+fESOcHH6UxbJ47NjGAZQuVB6Z7Vk6dqOrX98Ib3S0t0QE+YsmefTH+ele1ddz3LM6hP9X+FUtRG/T+OfmQ/qKvRqQmK5+81yHSZmtdTilWE58udELKV9DjkEUJiaNG3UK0Z/z0NK751DA7IP5msZvGPh+CPct6ZG/hRI2yfbkVc0aWa/DXskTRCZtyo3VV6DPvgUNgkzZb/Vj6j+dct4u8Wf8I1PZK2ltepOHLMr4Me3b2wc5z+ldNdyx21q00zqkaYLMxwAKw746Pq8kbSahat5YIUCRT1/H2pDLCNHrdlZanBNMsRUSpHnAbI/iHqK14GOwZrCbUtP062t9OtZUkkYiONI2BIHcnHQYzW3b5KAmgCtfSW8WnXUt4xW2SN2lbBOEGSenPSueste8JOyi11CLcxAUFWGT+IroruEXNjc25GfMR0we+cisOw8KWVrMsrWsQYEEHaDUsqJ1EXTjpUDc+cD3P8AQVPGAFxmoSPmmHqf6U0JnJ2Hgu1sdQtrsbFkgcOCo5OK6K31BLi7kijVwIjtYuhXJ9s9unNefaJ4f1LR9Vs7iHULhrbzAHiZyVKnivRsDzUOOcdaGCKuvaHba3apHcx7ihJQ5xgmquh6UmmaNLZJkKryY59ef610K8rVQDDzD/a/oKAOFtNCl0vS7uV7u5mBRVCySEgfOvb8K6rwf/yCZf8Aruf/AEFaj1ZQPD1wR6J/6EKk8Hj/AIlMv/Xc/wDoK14lT/kaR/wfqzxKv/I6h/g/VnRZrzm4UP4iuVYAgzycEZ7mvRa88k58SXP/AF3l/maM1SdSgn/Mv0FnSTq4ZP8AnX5o19PSzDtHLawPu6Fo1ODUNpDbNqQLW8TJvb5Sgx37VkSahJb+K/sD8RyWizQt/tBmDD8tp/OtLT2Yyqzgg+a3/oRx+len7Clb4V9x7P1ejf4F9yIfE8MMOpRLBEkSmEEhFCjOW9K72uD8UHOpQ/8AXAf+hNXd15eXpLF4hLuv1PGyqKjjcUl3j+oL1P1rP16eW20S5lhcpIAoDDqMsAf0NXx1P1rL8S/8i/df8A/9DFehjG1h6jW/K/yPUx8nHCVWnqoy/JnEW9hLcw+YjIBuK4JOeg9vero8O3RtjP51uEAJI3NkY/CpdJ/48D/11b+Qq1JeqdPvRG+QisGGejAV4mCyrDVaUJzT1Xc8PLsgweIwcK00+ZruZ9r4eu7stskgGAD8zH/CrX/CIah/z2tv++m/+JrX05jHJgHqtX9QuWgsJJF+9jA+prrhkmEavZ/edk+G8Ans/vOPk0GSJyj3toGHBAZzj8lqaLwtdzruiurNx7O3H/jtZ+sQalcwINNufIcEsx253egq54Xv7spbSXWROT5co9ecf4GpWS4W17P7xvhrAbWf3lg+ENQAyZrXH+83/wATQPCF+RkT2p/4G3/xNdNqkL3elzwRthnQgc45rmtC1Awyjd8ufkkB7Ef5/WreRYRK9n95C4cwN7Wf3if8IfqB/wCW1r/303/xNL/wh2of89rX/vpv/iauPfy3OoArNIse4ABGxkV0glBQtnj3qVkuEb0T+8b4awK6P7zjv+EP1D/nta/99N/8TS/8IdqH/Pa1/wC+m/8Aias6t4ka1hkuVaQQJ0ESbmarHh/xGuqQwyq5eGb7pYYZT0wcUv7GwnZ/eV/qzgez+8zv+EO1D/nta/8AfTf/ABNH/CHah/z2tf8Avpv/AImuvubpbe2kmYZCDOPWss+II0x5qKmeBmTGf0pyyXCR3T+8UeGsDLZP7zAuPCl9b20s7y2xWNC5AZs4Az6VqeC7iV4LqBnJjjKlFP8ADnOcflV+bUo73Sb9UUgrbueoOflNZfgr/l+/7Z/+zVywwtPDY+lGlezTv9zOOOApYDNaMKN0pKV9fJnXA1mXegadeXL3LxPHcuctPDI0bngDGQemAOPaoPE19PY6SGt32PJIIyw6gEE8eh4rk4ptYnjDx3s5U9CbrH82ruxWPjTq+xVNydr6HrV87nhMT7GhCUpWu+Xt8jv7Cyt9Nt/ItIxGhJZjnJZj1ZieSfc1TvdA0+/unuJVmSSQAS+TM0YkAGPmCkZ44rlIrfxDMcRXM7n0W8B/9mqX+zvFH/PS6/8AAof/ABVY/wBozat7CX3f8AyjxBi4zc1h6ib62f8AkdnZ2VrYeZ9kgjhD7dwQYBwAo49gAKRdPs1059PECi0dWVouxDEk/nk1xo0/xQekl1/4FD/4qj+z/FBOPMus/wDX0P8A4qj+0an/AD4l93/AJ/t/E3u8PU+59NunQ7iWGGe2e2mjSSBl2NGwyCPSsqPwtpKOrvDLOUx5f2iZ5BHgg/LuPHQdPSud/s7xT/z0uv8AwLH/AMVQNO8UnpJdf+BY/wDiqTzCb3oS+7/gFw4hxdNNQw9RX7J/5HbS28M1xBPJGGlgJMTHquRg/pSrBCl3JdqgE8iKjP3KjJA/U1xH9neKf+el1/4Fj/4qj+zvFP8Az0uv/Asf/FUf2jU/58S+4j+3a9rfVqn3P17HcGCE3i3ZjBnVDGH7hSQSPzArI8Q/2MHgm1Rpbd8FY7mIuhA7qXTsfQ1z39neKf8Anpdf+BY/+KoOm+KSOXusf9fQ/wDiqUsxqNW9hL7jSln9eE1J4app2un99mbHhmzgTUr6/sYGisZY444XcHdOQWLSHdyclup64rpSa4L+zvFP/PS6/wDAsf8AxVH9neKf+el1/wCBY/8AiqUcwqRVvYy+4K/EFetU53hanTo3srdjvDTCAeoB+tcN/Z3in/npdf8AgWP/AIqj+zvFP/PS6/8AAsf/ABVP+0an/PiX3GX9t1v+gWf3P/I7UxRnny1/KmmCM9sfQmuL/s7xR/z0uv8AwKH/AMVR/Z3ij/npdf8AgUP/AIql/aE/+fEvuH/blf8A6Ban3P8AyOyNuuOCw/4FTPIPaV/xx/hXIf2d4n/56XX/AIFD/wCKpP7O8T/37r/wKH/xVT9el/z4l9xX9u1/+gWp9z/yOvMcg6SL+K//AF6TbKP7h/MVxF2+uaY8TXNxcIWOUzPuBxjtk+o6121nM1xY287gBpI1cgdMkZrbC4iGIlKDg4tdzsy/N1i6k6Tg4SjbR+Y7Mg/gH4NUMDzlpC0SqS2cbv8APpVqoxxKfeu5U0tj1ea4oZ+6fkaN+P4G/Kn0ZrSxNxnmL/tD/gJpPOjz98VJRSsxjd6nowP40tIUU9VB/CmGCIk/IvPtT1EMuG2BWOeGFJv3nCfiewqK6tA1u4jZlbHBySB+GamjAEagdMUa3DQu21v5Q3n7x6561ZJAUsc8egzVZLpQgDZLVmXGuXtluaXS2lj3YVoJQ2B/tAgEfhmsqlSMFeRPK5PQ12X/AJaNkcdM9K53XdQu7phYaei4GDK8hIDr3QAckepHbjPNUtU8Tz3Egs1VLZiQdpbMjD2Hb8jWdcapJa2bhIrZXmbDmU7i2B1JOSfxrzK2KlJpU9u50QoNK8ht4swtJIr3TpMtnzJoisw+uzIUcAADnHA9K4fVrXRrLyJbOScwykK2VwwbqCwPGOeMAda6W41W6EX+jy2hkYEAJCpAPY5xjA+tczdXcb3O6/Z725HAK5SH8gMnP4HjocVg5NstxSRz0mratFI0aWl66ISqsdpJA75xRXWGSxJO2z+XtthyPwJzn86KXt4/ymfKz2qQfPSCnS/fpBX1RwHE+MP+QtF/1wH/AKE1dm0YY81z3iTRLu/uIrm1USEJ5bR5AI5Jzk/Ws7+zvFH9+6/8Ch/8VXz8atbCYqtL2UpKTVrL+u58xCtXwWMry9jKSm1Zpdr/AOZ2QQAYpgt1DlgOa5D+zfFH9+6/8Ch/8VR/Zvij+/df+BQ/+Kro/tSr/wA+J/d/wDq/tmt/0DT+5/5HahcCq9xaR3ClXUEH1Fcl/Zvij+/df+BQ/wDiqP7N8Uf37r/wKH/xVH9qVf8AnxP7v+AH9s1v+gaf3P8AyNxdBs0k3rAgOeoUVpwwCNQAMCuQ/s3xR/fuv/Aof/FUf2b4o/v3X/gUP/iqSzOov+XE/u/4A/7arP8A5hp/c/8AI7C5to7m2eGVFdHGCpGQa5ibwXpjtn7Mg+gxVb+zfFH9+6/8Ch/8VR/Zvij+/df+BQ/+KoeZ1X/y4n9zBZzWX/MNP7n/AJGnpnhmy06XzIYFV/XGT+db6JgYxXG/2Z4n/v3P/gUP/iqP7N8Uf37r/wACh/8AFULM6q/5cT+5/wCQPOaz/wCYaf3P/ItaxoWpT38s9nq91bo+D5at8oOOwqguk+JIzxrsxH+1ED/Opf7M8Tnq1z/4FD/4qk/svxN/euf/AAJH/wAVQ80rf8+J/d/wAWc1f+gWf3P/ACOttUcQqJGLOAAWx196xdZXW478HTpLcW7gblkQkg9z/Ks3+zfE4/juf/Aof/FUh0zxMerXJ/7eh/8AFU/7Vq/8+J/d/wAAX9sVf+gaf3P/ACN2Sxcae/lgGULlAemR0qhpF5q15dFL6zigRBkMr5JPpj0qj/Znib+9c/8AgUP/AIqkGleJQcg3AP8A18j/AOKoea1v+fEvuYf2xV/6Bp/c/wDI7NFwtc/qetx6ZqLW8lpdyGQBleKPK+mM561m/wBm+J/79z/4FD/4qmtpPiNzl/PYj1uQf/ZqP7Vq/wDPiX3f8AP7Yq/9A0/uf+RrawpHhy4JHZP/AEIU7wd/yCZf+u5/9BWsaTSPEUsZjkE7oeqtcAg/hurpvD+my6ZpvlTEGR3MjKP4cgDGe/SsaE6uIx6rOm4pRtqv67mOGnWxWZRxDpShFRtqrdX/AJmoa89Yqvia4LMFAnlyScD+KvQq831ERx65eC6tzJGZpMxt8uQScH9Qa0zecYSozlspK5eezjTnh6ktEppsl1+x+1pFe2rK1xaYdCpznHUfiDVyxlEghlX7rYIqpp+qWemQNDb6aApOT+97/lVBDZR6kL1baUEPvEfn/Ln8q3ec4Fq3P+D/AMjp/t7L7/xPwl/kanib/kIw/wDXAf8AoTV3EUyyLuXP4155qN6dVu43SAo20RhAdxJyfb3rsdGtbm1tfLurl55M5LsAPw4rlyyrGriK9SGqbVvxObJasK2KxNWm7xbjb8TVH3jWX4l/5F+6/wCAf+hitQfe/CsvxL/yL91/wD/0MV6WN/3ap/hf5Hq5j/udX/DL8mc5pA/4lw/67N/6CtY0StY+LNQtpiWtNTUSJ6B1UKw/EYrUtftieHjLZWwnkFywKE442rU8+kXOpaRHLLEIryMiVFBztYds/TisctjfC079iMmlbL6Xoadh8rRjOcLjJq3q4zpcp9MH9aw9Evbu6vfKl06aBUB3O54z9K6W7tzcWE0XGWQgfWu6krKzPSqu7ujC01FZ2B9Koqghv5lXgCbIH5GqtprMVrO0Vywt7mP5XSXgGrNtcQ3t/mOVHd23EIc46f4Upe7HlYRV5cx2MXzKAe9crq1kbHWBJGv7m5yWA7MK6uBSEGazfEKKLJJDxtkBPtWsdVYh6O5naXD5kobHC8Vv3CkWUuOuw/yrJ0JlkjkKkHDdq3XXdEw9RisqasXUd2cjb26XSJE4BVh0qPS/DM1hqamK8kWzWTzFhVRj1xn61Lp8wjcZ+9ExRh6HpVuS+uWvH8p9sS4C4AOfWk1y3uUrytY19UXdpVwP+mZNclf6XbavaxR3SBlUhx7HGP612NyPNs5V67kP8q465sTqemRwCWWEnad8TbW496KvwphS+Jo0NM0yx0vRL+K0QK7QuXODk/KcU/wV/wAv3/bP/wBmqPTNHbTNG1Fnubi4d4HGZpC2BtPSpPBX/L9/2z/9mrx6/wDyMKPo/wAmeBjf+Rxh/SX5MueMf+QRF/13H/oLVztuJG0thE22TawVsZwexrofGP8AyCIv+u4/9BasHTs/Zcf5700v+FGf+D9UPDv/AIXZf4P1Rn+E9Q1SRJxqJzc202FcJtyMe3+ea9JlkP2dyhw20kflXmuiazeXWq3Fje2scUsADqyE4YZ969DtpPOtVf8AvLzXqQ0lqfQT+EwtCuXNxE7Ozb1IOTnn/wDWKhutVul1u5UTSKsUigKGwMY9KZpOY5ljPWOZk/Wq2qL5fiG5H/PREf8ATH9Ku3xonrFnbXFx5VrLKBnahbHrxWTomqXF3KBM6lWTcAFA54q9EfP01c/xxD9RXOeH5MTWwPoVP60pfCmOPxWOh1jUzp9tG4TezuEAJwB71Lpl8b608xkCMG2kA59KyvFSk6Ukg/5ZzKePy/rUnh5v3UyejA/n/wDqomrJMUdbm/ms1tWhWd4xvYocHA4zWhn5a5KRAb68hJ4LsD261M9IOS6DjrJJnRQajDI4TcQx6Bhirua8z0y2u9I8QSWMt1NNbyRB4/NYtt/P8a9BilMtoGz8xXBPvUxbvaRTStdDor63mmaFLiJpV6orgsPwqZ3CjNeb6BL5PiK3Y9SxU59wRXQ+NZHGkRhSQrSYbB68GtvZ++4GfN7vMdNvBXNCsGFeWabr89hpF7aCUrvXdGwPKtkZx9R/KtnwJfzS3d3DNPLJlAyh3LAYPbP1FQ4+7cq+tjuyQBzTdwPGawPFGvNo1kpiQPPJnbu6Dtk/nXDQeMtbiu4xLcbhIMqHhAVvpgf1qXoUtTrvGn/Ll/20/wDZa3tM/wCQTZ/9cE/9BFcjrWppq2m6bdKNpPmB1/usNuR/n1rrtM/5BNn/ANcE/wDQRXk4f/kYVvRfkj5/B/8AI5xPpH8kWqaR84p1NfsfQ16p9COooopgFFFFACUUd6WgQ1hkEVXhJMQ9Rwas4qtECssq4x82R+NMCSnKnmI6cdMjNNpVkWIl3OFAOTUVIpxaYJ2ZzOsWNtNO0M8McobDbGAOO27n0rCl0yaRU+xajPCOfmkUSjBOeN3IHpgj+tdPJbvNPc3Um+Pe21EdRlQAMfzzj86zb5I7TTpGnU+WQRl8/OfTPvXFaMrtqyNW3Y5yay/tO7aJ7iL7PENwCJ8zqe7Nk9Tn0HXj14LV7e4bULiC8fyLUOY45HnJLRg/LkDLBeAfSvR7dIbezLyvA1xM3mS5YAIfp7dAK8z8e6Ze3esOtozSwPskXY+V3EYJPqeP1qYU4J6aEybtc2Y761SJEXXZ1VVACrAxAHoD3FFcFHpWorGo+2IuABtz09qKj6tT/nJ9q+x9hSj56YBWEfGekynrOv1jz/I1KvirRz1uHX6wv/hXte0h3MPZy7G1S4rLXxFpD4xfIM/3lI/mKnXWNMbpf234yAUc8e4uWXYu0VAl/ZSfcu7dvpIp/rUyujfddT9DVcyFZi4paKXtQFhKKWjFADaXFFLQAlGKWigBMUUtFADcUUtFACUUuKKAExSUtFACUUtFIBKgmsrW5cPPbQysBgM8YY49OanopSipK0lcmUIzVpK5T/srT/8Anxtv+/K/4UHStP8A+fG2/wC/S/4VcorP2NL+VfcZ/V6P8i+5FWPTrOGQSRWkCOOjLGAR+OKn2AU+irjCMVaKsaQhGCtFWGfx/hWX4l/5F+6/4B/6GK1v4h9KZPBFcwPDOgeNxhlPes8RTdWlKmuqa+9GWKpOtQnSW8k196OS8O6rYWWmyQ3c+xzKWA2MeMD0HtWqPEGjgYF1/wCQ3/wqL/hDtO/57XX/AH0v/wATR/wh2n/89rr/AL6X/wCJryKEcyo01TjGNl6/5nhYaGcYelGjCELLvf8AzHjXdGVtwuhn/rk/+FS/8JHpGP8Aj7/8hv8A4VX/AOEO0/8A57XX/fS//E0f8Idp/wDz2uv++l/+JrXnzT+WH4/5m/tc6/kh+P8AmVr678O3/M7xyH1aFj/Sm2V14fsT+4kjjH+zCw/pVv8A4Q7Tv+e11/30v/xNH/CHaf8A89rr/vpf/iaTlmj+zH8f8xqtna+xD8f8ywviPSAP+Pv/AMhv/hVa/wBX0PULR7ee4DI/UeW/+FL/AMIdp/8Az2uv++l/+Jo/4Q7T/wDntdf99L/8TT9pmn8sfx/zF7XOv5Ifj/mV9MvtB0uDybecImcn92/J/KtD/hI9I/5+/wDyG/8AhVf/AIQ7T/8Antdf99L/APE0f8Idp/8Az2uv++l/+Jo9pmn8sfx/zB1c6/kh+P8AmYOrLp1xeNe6dqrW07/fHlMVb8MU3TrhYpt17qiSKOgSBl/Piug/4Q7T/wDntdf99L/8TR/wh2n/APPa6/76X/4mlKeaS3jH8f8AMca+dR2hD8f8x41/SfLK/a+ox/q3/wAK419Q1CBykLWckSnCkh84rrv+EO0//ntdf99L/wDE0f8ACHWH/Pa5/wC+l/8AiabqZo1Zxh+P+YKtnSd1CH4/5nM2urXEgmS5WGJDC4BUsSxKkAdPetvwX/y/f9s//Zqt/wDCHWH/AD2uf++l/wDia1dN0y20uAxW6n5jlnblm+tZ0cPi54qFauklG+3mhUcNmFfHU8TilFKCe3mmvPuZfjH/AJBEX/Xcf+gtXNrcpYWVvI8UzCRScxoW6MetdJ4x/wCQRD/13H/oLVJoFuk/h613KD9//wBDNXD/AJGc1/c/VFUf+R5N/wBz9Ucta3dlPfCSKKU3LLs/1TDjOee3513FlGY7VE9BQlhErZCDNW1j2jFenGLTuz6OUk1Y4W7uV0vXbmCc+WsjebEx6HNMu5Fu9RW6DAfuvLIBzk5zXYalo9pqcYW5hSTHQkcj6Gs2z8K6fZTrLFAA69CSTj86qpK92upMEuvQ0rBGSwiRvvBRkHtXMaf+41FkzjZcsPw3V2aptXFYzeHbc6sb4tKctv2byFz64pX9zlYL4rj/ABDH5mh3I9FDfkQaz9Fukgdi7gK6g5PArobq2FzZywHgSIVP4iuPbw3q0RxHqXygcAwg0TkuVBBas7OG4jnBCSIxHZWzXNX37vXpl7Oqt+mKuaBp13ZtJJeXIldhtAVAoAqv4hs777bHdWUUcnybGDsR3oUk4O4ctpKxS1V1Os6U6jlhIp/If/XrpLA5tQPSuXtl1aSZFmsY1H9/zM4/SuttYvLhVfQVipOU0+yNHG0Wedt/ofiM9hHdE/huzXW+KovN0GRsZMbK364/rXP+INPuTr8ggt5XMrBlKrx09fqK63U4Xn0O5jI+cwngeuK7OZe1UvQwSfJY8dupPJuIY2+5LuQ/XqP5Guk8FSGHXI0J+/Gyfpn+lZ8ekvrJuEhQs0a+YjY4Dg9M+/IqXw87w67ZFVJYyAFQORng/wBaia3sOL7nQ+PYyVtm7FGH0wQa5udrO40TTYnldLlA53IBwNxHf8K7TxpbmTSUmA4jf5j6AjH88V5xNa291Ym3mj/eo4aNu2MjIoklaLGuqOi3Wi6ZbQWvnZRnaRpcZYkLzx9K9D0z/kE2f/XBP/QRXmEDKwbb0HFen6Z/yCbP/rgn/oIrxMN/v9b0X5I8PCf8jnE+kfyRapr8qadSHoRXrHv3AHKg+1LTU5QU7mgAo7UtHSmITvRRRQAlV2+W5Hoy/wAqs1XnBEkTDGN2D9DTYkPqFQz3W88Rxkbfdu5/Dp+dTUmKmSuNMdqMYKj0xXLahChaFWYON24RJyzsBkAfzPauh1Ge8Marb2Pmgr99pQoz6dz2rA1NLq2tJ5kiO/ywX2gvz6A9lzj9TzXnYmskuWLszWC01OcuYmudT8vULlYYXH7uJT0UDnc/brnAx7mq9xFDeG88uwguCQxaePBEWMAYHcnk9uuPeruoQSyqqXEpigxzGu3eT0wBkj3yDxXNagh+0MsOqiO0QAiNH2kkjPTqcDHrz09vMqSnO6ka2XQvpDpSoqtBMWAAJM0Qz+lFUk8KJIivJq0iOwBZck4PpRWF4/zC5H2HiRlbr+tWY7jPU81SfO7I605CPcV9A9QRqLIG6GpQ3FZquVOPwqykpHfj1qWiizhCeUX8qUbQeFAP0qNX/GnhsdP51IydJ5E5WR1/3WIqZdRvE+7eXA/7at/jVPPFOye2PWndhZF9NY1JOl7cfjITUqeINVUYF4/4qp/mKy80vanzy7i5Y9jZXxNqi9Z1b6xr/hUy+LNRHUQN9UP9DWD/AI0ZNP2s+5Ps49jpE8X3mPmt4D9Mj+tWF8XS/wAdkh+kpH9K5RTzntU3Sj29RdQ9jB9Dql8Wp/FZMPpJn+lSr4stf4racfTaf61yIzjNLT+s1O4fV4HYr4psD1juB9VH+NSL4k0xussi/WM/0riqOtP61Mn6tA7oa9pjf8vQH1Rh/SpF1fTm6XkP4tiuBz70ZxVLFy7C+rR7noa39k33buA/SQVIs0T/AHZUb6MK849Mn86MfQ0/rb7C+qruelcGlxXmoYqeCR+NSLcTp92eQemGIqvrfkL6r5nouKMV58NQvVxi7uP+/h/xqZdY1Fel5L+Jz/On9bj2F9Wfc7vFGK4hdf1Nf+XnP1Rf8KlXxLqK9Wib6p/hVLFQJ+rTOyxRXJL4pvh1ityP91h/WpF8WTj71pGfo5H+NUsTTF9XmdOR8y0uK5g+LWLL/oIPr++/+xqdfFkP8dpIP91wf8Kf1in3JdGfY6CisRfFVkx5guB+Cn+tTL4l05hy0q/WM/0qlWh3F7KfY1aKzl8QaWT/AMfOPrGw/pUi6zpr9LyL8Tj+dP2kO4uSXYu0VXXUbFj8t5bn/tqP8alW4gf7s0Z+jinzLuHKx9GKAQehB+hpcUXFYSilxRQISiloxQAlGKWigaQlFLiikUZXiDTZdT0zyoSPMRxIqn+LAIxnt1rmotI8SQRiOLz40HRUuQAPw3V3dJXn4nLaVep7Rtp7aM8nGZPQxVb20pSjK1tHb9GcP/Zvij+/df8AgUP/AIql/s3xR/fuv/Aof/FV3FFY/wBj0/8An5L7/wDgHN/q9R/5+z/8CX+Rw/8AZvij+/df+BQ/+KpP7N8Uf37r/wACh/8AFV3VFH9j0/8An5L7/wDgB/q9R/5+z/8AAl/kcL/Zvij+/df+BQ/+Ko/s3xR/fuv/AAKH/wAVXc0Uf2PT/wCfkvv/AOAH+r1H/n7P/wACX+Rw39m+KP791/4FD/4qkOmeJ+7XP/gUP/iq7qil/Y9P/n5L7/8AgB/q9R/5+z/8CX+Rw39meJx0a5/8Ch/8VSHTPE56tcn/ALeh/wDFV3VFL+x6X88vv/4A/wDV6j/z9n/4Ev8AI4T+yvEvXNx/4Ej/AOKpw03xOOj3P/gUP/iq7ijFH9j0v55ff/wA/wBXqP8Az9n/AOBL/I4U6V4lLZJuCfU3I/8AiqX+zPE2MbrnH/X0P/iq7mkNH9kU/wCeX3/8Af8Aq9R/5+z/APAl/kcIukeIk+6J1z6XA/8AiqYmia9HIZEikVz1ZZ1BP45rvqSl/ZFP+eX3/wDAD/V2j/z9n/4Ev8jhZNJ8RyxsknnujDBVrgEEfTdVP/hF9RUY+wKB/vJ/jXovakxR/ZFP/n5L7/8AgB/q7R/5+z/8CX+R57D4X1FSsaWiRIT13qAPfg13dtD9mtIYN27y41TdjGcDFT4pMV04XA08NJyi22+53YDKqOClKcG25dW7/ohKKWiu09IZH/EPQ0+mLxKw9afQAUUUUAFJRRTQmLUM4zESOxzUtMYblI9RTsITr0pKSM5jX6UtIZJJuOnPs++M7fauc1JHkgQJKWiEiuyrj95j3JrfhBkifzR/tKnp9fesjUFVZDxyOa4KtBzm5GsJJKxz+qW4u7y1326+VgtJlgG4xgE8kDr06+w5rL1dpltpZ4NI08SxITHIzBtm0HB27fbpV64vV5S2jM0gyWWIgBT1OSeB/P2qqEkZWe6bAzxFFznPY56/pUvCwtzS1Ye0d7I87HimIAB9Xvd4+9tuGxn2orTfQAXYv4i0+NiTlBHEAp9OOPyorj5aP9f8MV7xqMvzHjjNKq8D2qWUKjfMQp9+KRMEfeH4Gu1O5tawBcds08D2pwX0PPtQFHai4xVYrwf0qVG6ZFRjHY5oXOD29c0gLKv+lPyCOp4qqGPWpAxFAFjryPrR06moww/rTweMUgHdvWlxzTQRnGaX8qAFHXGanWq44NTrnFS2Uh1O4pO1J9BUlCmkz7UuCKQj6cUXAOv/ANegDgjFAH5UvGfWi4WAH+VHrxR06/yoouAcdaPrS/nSUAAFHWlpOo4oCwuaQ+/elx0pCPbrQFgOKM/SjPNIeuPzphYAM0tA60mOKBWF4OaM4pMdeaM0DsLmgH3pO9GeKQC59KXAI/8ArU3OOtBNMLDuAMgflT1mlX7srr9GIqIe1Lxii7FZFgX92h+W6nA/66t/jUy6vqCdLyb8Wz/OqPvQeO1Pnl3E4RfQ0117U1/5eifqi/4VKviPUV6yRt9Yx/SsjNGfXFP2s+4vZw7G4vie/HWO3P8AwA/41Iviq5H3reE/QkVz4NLnFV7ap3F7GHY6QeK2/isx+Ev/ANapU8VQn71q4+jA1y2TxRn3p/WKncXsIPodcviezPWKdfwH+NTL4i089WkX6p/hXGA0u4EGn9ZmL6vA7dde01v+XjH1Rv8ACpV1awbpdx/icVweeaUNT+tz7C+qx7nfDULI9LyD/v4KkW4gf7s0bfRhXn27HNLuprFvsL6qu56ICD0IP0pcV50GA6cfpUi3MqfdmkX6Mar655C+q+Z6DRXBpqV2vS6m4/2zUy6xfAcXT/jzT+tx7C+qy7nbUVxya7fjrcZ+qr/hUo8RXo6mM/Vaf1qAvq8zq6K5hfElyOscR/A/41KviWT+K3U/Q4qliaYvYTOhxRWEviVT1tvyf/61SL4jgPWCT8CDR9Yp9yfYz7GxSVljxDZnqso/4CP8aeNcsCf9Yw+qmq9rB9Rezmuho0lUhq9g3S4H/fJ/wp41Oybpcx/icU/aR7hyS7FmioBf2b/duoT9JBUgmib7siH6MKfMhWY6ijr05oqhDDxKD6jFPqOQ4KH3xUlABSUUUxBRRSGmIU02lpKYiJP4h6MadSdJW9wDSjAOSM1Mm0rrUYsZw/1rDvpLa+kePzFfaNssOcEH39q2toLlskMfeuW8Q+D4tZnF3a3j2t0DnzFBKntxgjB/OvHr46pTnapCyNVFWuiu0UMPyRRiKIHCoOiiucurvUILhrgaUWmU+UCLjeqjqTjAI9z06c9M9RFo93pdr+/upNSYAkcJGemMZOc8/wA/YCqqW89ze7jaG3aMDMeEIx6lgOn0I7etYzx6rPlpuyRcYW1ZxclnYeY3mTnfk7s+X17/AMdFdc2l6IWJltbxpCfmP2Z+T370Vxe1fc05Ucn8YWI8PWGM5N1nA/3D/jXm+gs7MnzNjcDjcfWvRPjGcaPpyc8zk/kv/wBevP8Aw8MrCxxy39a9/L1eirkYx/vGdUu8Yw7g+zU8PPgfvZP++jT0Xr3qYKMZP1r0uVdjhUmMFxdL/wAvEv4nNOF3eD/lu2fcCn7AOMUu3Bpckew+eXcFvrw/8tAT7qKkGo3ag8ofwpNn+c0nlg0vZQ7B7WfclGr3IHKRn8/8akXWpsgeSv4Gq3lg8H+VHlij2FPsP20+5eGuP0Nv27N/9anjXlHW3fI9xWfswKa0Q/yan6tT7D+sT7lqbxfZW0oSaC55GcqqkfzrRi8Q2jxq22YZGRlR/jXn2ujZepjj5Bj8zXQWke62iz3QfyqPqlNsv6zNI6Ya9Zesn/fFPXW7E/8ALVh7FD/hXPeSDwBTvJ/+txR9Sph9bmdENWsW6XCj6jFPGpWbEYuovxYVzPkgcAUnkj86n6jHuNYyXY6tby2I4uIz77xUgliPSRf++hXIfZwTkiq93bDyOQBzUPAruWsW+x3III6g0o7e1eT/AGchbwKxHOTg47Ukf2q2W2Y3U65yWxK3fn1rP6m+5f1pdj1rjjtRjmvJYLvVQsxOoXgLJvUee3HJ96emp61HpzuNUujIVLZaQnHtzS+py7j+tR7Hq9GPWvGx4x8QRGMf2m7fL0KIcn8q0pvGWtrKscVyCAwDFolOfyFS8JMr6zA9S7c0Y+ua4JfFWrqPmMB9zGR/I1XXxzrPnmP7NZtlyB8rDA9T81J4WoNYmB6LTRXFSeM71Uz9kgOOT8xGaLTx3NPEXfTFTnGBPn/2Wl9Wqdh/WKb6na44pOc1yEXj2OW4aP8As2UKvVllB/oKkg8eWUsrRNZXasvJI2kH9an2FTsP20O51hpP0rnl8Z6cTgw3S/8AAFP/ALNSHxxoitteSZcHGTCT/Kk6U10H7WD6nRdTQSeawIvGugTPtW9bcemYH/wqX/hK9CHDanCp/wBoEfzFL2c+xXPHubXOeKO1ZqeIdGkj3rqdptHUmUAVLHq+myjdHqNqwPcTKf60uWXYfNHuXcn9aUVXF5ayHCXELewkBqVXVujA/Q1NmO6Hg0Umc8nNGcA0WGLnJozmm5zS5Hf9aAFzxnvS03OR/jQT2pDFoFITRmgBwODSknFMz9aTPsaAJMnJo3cVHu44NKTQBITzxSbuaj3U0t/jTsIm3f8A1qaXqIuKaxAosFydX5qQPVMPUgfiiwXLO/rQH75/Wq5fApC5pWC5a3+9AfjFVfMwO9HmHHNFguWw9L5nHPNUxL604S4o5R3LRkpnmc+9QGTjmmGUZosFy6JPfip45PlPSs5JM+tTROefTFKwFvzKaWX0B/CoN/uaQygDPX1ouFiwHA5AA+lI1y6naJZsnoqOwJ/I1Wjd5z8gwv8AfPT8PWrsFuie7Hqx6mi7ewrJbl3TFupLtZJbmYqucRmQspz65rqAcgVh2i7BmtmI5jX24r0qEWo6nDWd5D6TNFJXRYxFzRmkooEFLSUUWuBlSW99bXJaCXzYWJwjYyvsPUVHLqlxbOEuLXtywyP6H271qy9FPoaXNebLBVYSbo1GvJ6o0U090Yh8U2SMVaNwMcsOTn6VZbxVo8wiCzlWYcDb7Zxx3rQaytZkZp7eJw3UFBk1HJounSWe5bRFBk8whePm5GePqa5pLEpPms7dSk43VjKOr6bvKC9SVmUny0OeO+SeOKfHp8N2GmSFVmY7SAdowOn1xz271UvPDOliDc7yw7ACZA+AMd+elZVxY31tYWr6Rc3QjBdWDgq0244XHHHAPOR6+9eFVhzSTSsdF2kXZrazimkj/tdItrFfLEiAJg9MbeMUVCuhakVGLfTU4+60SMR7E45PvRWXspdwuzz/AOMTgWelL0zLJ/6CtcRoCfLBj/np/n+ddh8ZWITSFz/FMf8A0CuT8PD5bfPILA/TmvsMAv3MTLF/xGdaiDqeualA5z69aEGeo+tSAcZr0zhGYOTTgo9Kdtx1p2PUcGgBMfzoxxjPPWlAyM+1OHAoAZjnijHan4yTTgOTRcViPGKYVwPT3qUjJ6cU0jjn6VQmcp4iH+nRcf8ALPkfia6KzH+iw8Z+RefwFYHiMYvYvePv9TXRWIzaQHJx5a9/aktwexZC8YxzTyp6c0gGR79zUiimIZt6+tLt7cVJjvRtB7UDI9oHbmoLtf3B6dRVvGeRUN2D5B6dRSY0czcffmRersM/gKWeMSW0CjqcAZ9xU7xhrmduypj9KgGZJIfRCq8etZXLCZVimbIGPLwKjiQGwAx2NWLiMSXaL2AyffFVg5SzVFHztkD86aYjkbiNllLHA3MTj+7zgD9K347dY7e3UdSQScdTWLdna6g9Of58V0CjMFuf93+VIZoviOMucYAqrZxB5ZZCOTU8mbicQg/KOWPrS2q7ZpgB35xTuAskS7TwM1QjUJaFQPmZjjitdl456VVt4RI+/wDgThRSYIjhsxFEBgbjyTjvVeGAC/kOOueK1mXv61UVcXrk9x/SkwIp4/LjPy4PbNZr2obPr1+tacxLsWI46CmLC3lu5JGBx71Ldykita2KLGH43NWRrEWGb6CumhA+zr+P86wddGCT221fQkj02ITaU2eRlu1XYLcR2qrt5PQVFoLA2Mm4Zw54/AVrWqCRzIcEDoPSkMqxaaqKSQN7dTT/ALJg8cVqFR2zim7fb/61OyFdmPdwyiEFZJBznhiKfbm8WBNt1cAgYz5rA/zq9cpmA8dOabAg8gD0zip5VcfMyJbrUh0v7r6ea3+NPGqawg4v5yfd8/zqURgdjxUMvZVGSfahwj2Hzy7jZPEevqdkWoPuz3RT/MVNfa/4r0/w++qG4VohII9zwLhiew4q3pGn2b3BN7cLEijcc9WPoKd44vY73QpLaN0ZFUFVjPyoB2x0rJwjzWSLU5WvcwLf4ja+0UbN9lbd1JiP9DXUW/irU5I0Z47clgDwpH9a8qtjttozxlW/LmvRbFQ1pEePu1MqUL7Fxqy7myPFF5jm2hP0JFV7nxvcWrqDpquGGciYjH6VEIxjP61Q1G3DbDj1Gaj2MOxftZdy+vxCOPn0lwfacH/2WpB8Q7X/AJaaddKfZlNc41uuPu9KYbVSCABxR7CAe2mdSPiDppHzWl6uP9lf/iqkXx/oxIDi5T6xf4GuOa0Tg7ec+lRtZoaPYRD28juh450HHN3IvoDA/wDhTv8AhNfD7YzqaL/vIw/mK89eyU/yrO1Kw22EsgPQA9B2NNYaIfWJHq6eLNBfBGrWoz6yAfzq2niDSZAAup2Z/wC2y/418+Y/cdfut/P/APVW5b2ivCj+qg4NKWGiuoLEvse4LqNpIPkuoG9MSKf61IJ4z91ww9Qa8TGnqyngcnqajfTwB8oxUfV13K+seR7iHyePrSF8YzmvDjBOgxHLIh7YciiCTWC4WG/vF+kzD+tH1bzH9Y8j3HzD60vmc9a8WfUNatA+dXvmYJnm4Y4/M/WvZPhJKuveFJpNSH2m5iu2jMkvLY2qQP1NZVaXs48xcKyk7WJt46Cmb+eP1ruDoemsebVfwY/41E/hzTGH/HuR9Hb/ABrm9ojW5x6SA96mSXB/wroz4X0/JIM6/R//AK1QT+F4PJfybq4V8HbkqRn8ql1EUmYbTHcFXJY9AOamitWchpj06IOn4+tOggWBcYy3cnqasrzVblbEiKOBircK81XjXmrsK8it6VPUxnIuwjCitOBsxVnJ0q7atlSK9CCscMmWKKQ0VoQFFFJQAtFGaSqQmI/KH6UgOce9KeQaZGfkHqKTQImlfACA4Hep7U7reRe45qmf1qzYt+8Ze5XpWKpKMLDvqY+rwM5D4DIq58s9CfU+v0/+tjBmnu5y8vnMFRvmJ6Ac8fXOK6jU4WliZSxjTkEr1/8ArVjWsUVxaS2jYjAkQDb3yeK8Kr7rk+XRNHcn7qKQnvCoI5BHB3Nz+lFXmttYLExsipn5VEgAA7DFFL2sOyFdnkPxkYmfSVHULL/7JXOaAMJbem7+tbfxdm8660g5+9C749claxvDYxHa9wOlexgv4MTLFfxJHWrx7Y/SpgBx+opqKMU9Rg5/pXonChccUYx707n1oGcfSgY0AZpwUenWlxzRjikMQgZ6nrS4zg+tO29h1oC800IQjNMPbPQ1Lj6Uxh6UyWcp4jGLyI8f6v8Aqa6OwH+hwdP9WvT6VzXiZtt7Cf8Apn/U10+nDdYW5/6ZL/KktwexaHp2p+PrmmhSOM/lTwOPXFMLByM0dRTsdTwfSjbkcdKVx2EwG7VDdAfZj+FWPbH4VDdrm3JxjkH9aT2GYLYWS5J4AA/lUZURW8BOBltxpZwzSyp2PzHjsB0p0yq0VvH0LY/LFZGlivGC05ds/OpOPQdqZBEBE0uOqkL7CrUzCO4YkjCx4qNABp54x8ppoRx+orsdO+c5rdhA/s+Bj0Cjr9KwtUR1uFds7WI2j2x1/GtazLy2S9QkaAfjQxI2rSIpHub778mi3H+lTCpoSTEjc8gVDvEN1ITx8uf5UmxizsXcQp1P3iB2pLMELIPRqktoyQZXHzPSWh+eUe9K4yXbng1Tn2pIwHJPX2FXJZPLQn+I1nscsfU0pMaQ1YzI4A/lVuSMJAR6CnwQ+WmSPnNLPjymPTjihCZVtx/owyPWsXXFyp4zlK24P9RyfWsHXpNssMa8lsg/SrvoTYh0AM4lQcKG3H8v/rVvaeflbjnisTw02XuUz2U/zrcseZH60uoy6BznFKRk04Dj2/nQeOcc1VySvKuY2+maitV+U9OuatOMofU1BGDDHlgcnnHpUNlJBI20YHJPTFIsQU5Y5Y1IkZ5dx82Pyp5AJzjNNAyFhVLUY99hcL2KH+VaJHyjPSq1wu+J1xwQRVXEefWozanHUPnp7V6Lo7B9OhbjG3v2rzm2zsmRuxH9a7/w44fSo+nHFZSNYG0Bxz0qrfp+6BHUNzirQ/yKr3qk259iDUFmYV5yenemlRnjg4qTnr/Wm8468mmTYjYZ69KYU49cmpT+YFNP+e9O4Fdl69vpVW7i32U6D+KNsflV5vpURUE4xweKpMlnBDmCQe4P+fzro9L+exhbGCBj9cVzyDaJk6fL/Kt3RGzZgHqrEYzTlsJGoEyOgodPT681LGpc4A5q9Fa7eXHPWsrlpFCKyaVvmGF9atiBY1woAq5tAGB+lRuuAT29aLlWsYGpx4l6feT/AOtXo3wHus2etWpPKvFKBn1DA/yFcBqaYWMgdQw/lXUfA+4MXi3ULUnAmtWwPUq6/wBCayxCvSY4O00e9k5waDSDpj0pa8i51jO9IeQaDwaQ1DZRzd7CY7px75H41Gq81qanFkpJ/wABNUAuCPfmuyjrFBJkkYq7CMCq0a1bjGBXfTics5Fhat2xw2KqLU8JxIK6EYMu5opKKskWikzRmmJhmlpKKYgzUa8Fh71JUfSQ+4oaEKaltW23Ke/FRGhW2uG9DmpktBk9+uRIvfrXD3MLPe7SSo7kdQPb1Nd7fKqgzEgLs5J9K4q5R5LlpJI9kefkQjlh6sD/AC/P0Pl1Xadu52UneA3N1/yz09Cn8JMRYkdue/1opjBixJkckn1orL2TKszyL4u/u9V06ME7VgbjH+0cfyrL0S5WCC1YxswVBnHetD4wMD4islPa2/8AZ2rL0wAWcGcE7Ov4murDy5aEWTXV6sjqE1qHGTDMO/Qf41MNZtiB8kv/AHz/APXrDXnp9KlH6YzW31iZl7GJuDV7U9RIP+AUv9qWmc72BH+yaxDx3pVPbNH1iQexibn9p2ZH+t59wRTxqFpx++A/AisIdMjrR37U/rEg9ijoBfWp/wCXhB9TTzeWp4+0RfTdXO8AdKaQM4xin9ZfYn2KOkNzB/z2i+u8U0zxEYEiH/gQrmjg88flULgDnj8qpYh9hOiu5F4slRb6Ahs/uzyOe9dZpbqdNtiGGTEv8hXm2suEuY+3y1vaaF+xwHGPkH8qp17K9iVSvodyF9P51IFPrXJqfQkfialUuD/rJPwY0vrK7FKh5nUgHrjn3pCMgY9ea5pZJs/6+X/vs1ILi4B/18v4tR9ZXYPYvudFg96iu1/cH2IrGW7uf+fiT86Gurhkw07EehxQ8RHsHsGRsuXu2I5AA/SoEBeaGQ/dBCqPoKny5LfPy3B4600bxtAbhORxWftUV7NjLiPzLph1Crk1FKfL00Kf4kIHPepxu3M+RuYYJx1qtPGSgU4wowOKaqoXs2c3ri7Xt8enX8K1bBAdLTAHK8/WsfxA7i4jDFdoUYGK09JkdrCMDG3BGMVbmrEqGtjZsyDbRn29abIA90sfQdz602BniiCKFwvrSpGUkWQHJBzz3qXURSgy5joB17VWtyFuZgT71J5rcZQc+9REEs7AYLe9J1ECgyKeUu2c8dqbCB56cf55pxhJOccd6eR+9V1XAUYxUqauPlLYHI7daimJ8p8eh6UpnGMbWpjSqQRtbnPatOdEcrKqy+Xa+jZwPesrUYCsPmONzkH8K1QF8zc4JUD5RiqGs3MSwAnPXHK+1OMkJxZleGW/0ycZPKf1/wDr10Fl/wAfMign8frXL+HpBHqz56FSOnuDXT27Bbkt/Cc8020gUTU5H+FJnjGaYJUPIP4Yo8xAOvT0o5kLlYrHaDzz0qvC2+c7uoFPXa7lncAL0FRQ4W6yRheRkmp5irGxY6e90C7ArEO4H3j6CtUacgG9k57KUPSn28tqsUbGSLaBhVyOfrzVHUdUWQNFbuOfvOOPwHtWfM5OxdkkU9QaBHMcIRv7zgcfhWVIu5SCDVhsMOwqFl655reNkYs872eXe3cZHQkDjpzXZeFnLaeec7W/xrkr9RFrt2PUk/yNdJ4QkzDKueh/w/xqZFwOr71Fcgm2fnoM09TikmBMDjGeCOKg0MnnB9qQ9fYUdevSgn8qLiGkGmHvwSKcf/rUh75P9KAImHH+NRnjnP5VMRxjH5VAeuRxTTFY4yaLZqdxH0+ZwM1qeHYmkjlQZ4YH86i1K0K6rPIeFBDAeuQM1a8NNtubiPIyVB59v/11UpaCitTpoIljHyjn1qcYA/rUa4BqQcH3rC5stBe1RyKCOvPepcjGD0+tMY54ppiZj6ov7kEZ4bn8qt/C+4+x/E2yGcLNvjP/AAJD/UCodQXNs479etZ/h+4+xeONFuMhQLqEk+28A/pTkrwaJ2aPqg8N9aXtRJ2IozkV4h2dCM9aQ9KVutNJwM5wKzZSIbqPzLdx3HIrJArVtb22vJZo4JBJ5eNxA45z0PfpVB4/LlZPQ13YXaxNTQSNenFWkqJBUqV6UEckyZe1SocNUIqVTyK2Mi9SvMbe1uJljaUxoWEadWwDwKYh+QfSp4P4vwqiU0nqZCWOu3Ea3T6x9nncB/sywK0SH+6c/MfQnI/CrVjqz3egnUDbN5qK+6GM5y6EghfqRxUGp3891cvo+lt/pRX/AEi4H3bVD3Pq5/hX8TgddSztIbCzhtLddsMShVHtURWun9M7K0v3adSKu2mkklp5276WvrpfrrkJY67cRrdPq/2eZwH+zLArRJ/snPzH0JyPwqe21xG8OPq1zGY/JRzKinPzISCAe/I4qPU7+a6uX0fS2/0or/pFwPu2qHufVyPur+J46weIrOKx8F3FrbJiC3jT5euUVgT9eAaV+W7j0X4mqiqrhCqknKStZJWj5276Wvrpfrq6K08QXMC3barHbzuA62ywBokz/CT94/UEVPBrsbaA+qXEZjMCuJ41OdroSGA9eQcVrqwZAV6EZFcTdAv4T8QTKD5Jv5JAMdVV13fnhqJXhquzFRUcS7TSXvRWiS0btb/h+xrx2niC5t1u21WO3ncB1tlgDRJ/sk/ePuQRVjT9QOp2Mdw0flygmOWPOdrqSGAPcZFaykMgK9CMiub0L5hqUi8RvfzFPoDg/qDTtyySuZSn7WlKTSVmrWSW99PP59jqGxJaxMRnA/8ArVh3cCtdAFFYsQBuOK2rZt1lj+6cVlaomORXDi43ZOHfQQ6hZIdggiIXjOOv6UVjhJMD5M/nRXF9XX8xryI8L+LrhvFEC9harx/wJ6z9NnhSzhUyICEGcsKufFaT/ir1GSMWqA49CWrimJEjYPAA5r0aMb0YryM6ztVkdyk0JAzNH/32KmSaIjiRSPrXAA44z+FO3E98Y7VXsifaHoAZeOR+dPVhntXnvmtuypJx6U7z5Ookdc+jGl7IPaHoQ+maXuK8++1TqMCaT2IY08XtyFOJ5cdfvmj2bD2h3+R3pp7eprgxqN4rAi5mHp+8NO/tS9A4upiP940ezYc6O2J/yKic5XAH4Vx/9q32R/pUnbvSnV70Hm5c/UD/AApqDFzon14/6TF67eh+tdFpfNjB1PyD+VcXc3UlyVaZ9zjjJFXYNZu7eFEjcbVGACoOBVSi2khRdmdwlTL6Y47c1xK+I78Z/exnj+4KcPE+oL8xMX4r/wDXrP2bLU0dr2xTj6j9R0rix4pvhjcsH4qf8aePFt6ODFAfbB/xo9nIfOjsgcEcfpTs/jXHjxddHB8iE+vX/Gnf8JbNjm1jJ9mNTySDnR1pIAxmjOQOvrXKf8JfJ3tEP0c/4U4eLepNoOOP9Z/9anySByR0+7I9aic+lc6PFy4/49D/AN/P/rUp8Uwtw1tIM+4NPlkLmRV8RsVnjY88Vd0Ns6dF17/zNYer6kl+VaNCMDByas6Xq0VtbCOQMSCeg96tp8pKa5jrY2AGKlz3rCTXrUcNv/Kpl160PV2HsVrOzLujYyM8A0bvSsga7ZMB+8Iz6qaX+3LM/wDLX8waVmF0apbpyMUZHbpWWNasyf8AXD8Qad/a9nxmZR+dFmGhpZHOcD+tNJ7Vn/2tZkEeevH4UHVbM4/0hPzo1C6LjnisnWBusjx0YGrLalaHP7+P/vqqGo3cMltIFlRiOwIqo3uSzF0ZtusLweQw/SuziI4HT61w2mPt1iJmYbdx5J9q7OKQHHI6+tXPcmGxfB7nNBHHPFRq3HBBz75pc547+lTcsdn+VJwM8A0mc0h6UrisOJX0FNIB7Um4delIDTTGIQmTx9ajZVHf24NSMT1/SoZG46002TY5HWkSLVJSAQSAePcVZ8NyNicKzLgjkHHr/hUfiEYvUJ6FB/M1F4bfF3OhOMqD+R/+vWv2SFuddHLLkfvpMYzyalEsrDHmsQevNVYz/Kpwc9xnNZ3NACAcZORR5QHHSnL+eKM/jSuFiMxZH3qQxlT94496mOcDPNIev+TRcLFfyuPf6U6O3GeeakJA6/jTkyM880OQJHPa9Ftutw6tF/iKz9Ck26p7OrD+Rra19QTA5GRyv8v8a57S3KapAxPfHX1qo6olrU7MHHQ/lUq/T86rq3HtU4PPOOazNB4OOn40xm46frSlhkdvxqNjwc/jRcLFW7O6KRfVDXLXMxjntpVblDkH8q6yVQTtPQ8Vx98u2JC2flbB/wA/561pF30M5I+vYJlubOKdPuyorg+xGach4IrmvC2t2kfw+0fULy5jhiFpGjO7fxKNpHucg1h6j8RbKR3is5Wjj/56FDuP09P5/SvGcJczSR2Q1R2Woarb2Iwx3y9o06/j6VympazNeZEjbYz0iXp+Pr/KuYl8S2jk4lPJySc5J96qSa7bvz5vHemqMux0R5I9TuvCN35mrTwk5DxbvyI/xro76PEyuOjD9RXnfg3V4H8UWsSyAmUMn/jpP9K9PvY98G7upzXTRXLLUwrNN6FBB0qQCmKKkFejE4pDl61IKjxThWpmy5Ecp+NJdfaWspo7SZYLh1wkrJuCH1x3/wA9aSA/IRWJq/iCWyvTbW8SEoBvaQE5JGeMGssTiaeHp89V6bHHisdSwUFWqvS/a+voO0/TNc0y0FvbX9gFyWZ2tXZpGPVmJfJJ9a1m/tN9KkhN3Al8wIW4SH5V99pJ5/GuY/4Sq+/55W//AHy3+NH/AAlV9/zyt/8Avlv8a86OcYOOib/E4qnF2EqS5pb3v8K/pmjp+ma5ploLe2v7ALku7NauzSMerMd+ST61twefJYtBqRhuGcFX2IVVge2CTXJ/8JVff88rf/vlv8aP+Eqvv+eVv/3y3+NOOc4KOzYq3FuFqu8nrvdRSf3o2E07V7eEWltrAW0HyozwbpkX0DZwfqRmtKK0tbXSfsEcAa3WMoVc53A9c+pPOfrXLf8ACV33/PK3/wC+W/xp8Piu581RPBCYs/NsBDY9smnHOMFe139zCXFeFqtKUmtf5ba93Y0o7DV4bcWtrrAW0AwjPBvmRf7obODx3IzWjbWsFlaRW1uhWONdoyck9ySe5J5NPi/1QHccU416ygo6o9ypiKlRWl+CSu/O25e05siRPUA1FcxqV3PyF5IPeiwbbcgH+IEVLdB9rKEB9fmxXHjV7txUnqYrXcm44eQDPZsf0opxilz94D2DdP0orxuaHmd/LE+bPig+7xo47LBGP6/1rjmGJWzgV1nxLbPjW56cRxj/AMcFcqPmZgec+le3Q0pR9DlrfxGAJBoP1NO2gkUuOgx17nvWpmN3H26UhznPrTyuOc4zRj5s9zSAYDj1/HijHHANSbM/d/lQF59aLgR56YwM+tL2xzinqpOecgnGMUgXpkH39qLgRk8nHWjPOQAM9vWn7cDB/IikKgc5HHancViNuOcjrjr0qUH5R7DHWlt7WW9uobW2j3zyuERAerHoK6dfh34nwM6Zgjr++T/4qlKcY7scYt7HLbsDj86ack9K61fhx4oZSTpy49fPj/8AiqD8NfFLH/kHqB6meP8AxqPaw7l+zn2OTzzngEUZ/Edq69Phf4rcEfYIwf8Arun+NKfhh4rIA+wJ/wB/0/xo9tT7h7OfY5FZNhJIB+oppck56V1T/DjxPE5U6fknsHU4/WkHw78UMQo007j3Mi4/nT9pDuLkl2OX3c9vwoLj8O9dO/w98TqxVtNbg9nU/wBaYfAHiZf+YZJ/30v+NHtIdw5Jdjmlbtn8aA4yDjr1zXRf8IF4mAydKk9/mX1+tI3gTxIM/wDEqk5x0K/40+eHcXLLsc2xIHt/OnxsdpIPFb7eBPErfd0qX/vpf8afH4C8UZO7SpuPRl/xp+0hbcOSV9jA345zx3FAkIHcY46/59a6P/hBPEpB/wCJTN0/vD/GmnwH4kyf+JVNyOOV/wAaj2kO4+SXY57ee5+nFKJCq9ecVvt4E8SEc6VNn6j/ABpP+EE8Tf8AQJmPvkf40+eHcXLLsYHm9ODnGMUolIXPQ1vjwH4n5H9ly/8AfS/40o8BeJsH/iVyfiyn+tHPDuPll2Oe3nPNNMhxzjnk4rpU+H/idyQNKkHHOXUf1qJvAniTdj+y5v8Avof40c8O4ckuxz/mlj+P0pDKOBnkd/WugPgTxMOP7KmOD1yP8aT/AIQTxMcf8SqbnpyP8aftKfcXJLscyrkOPXNOMh6ZNdB/wgfifp/ZUwx7j/Gnf8ID4mx/yCpPzH+NP2lPuLkn2OeEzjkOR+OKd9qnUgea/H+1W6fAXib/AKBUo49R/jR/wgXiYnH9ly569R/jT9pT7hyT7GKuoXQXi5m59HP+NOGp3oH/AB9z4HbzDWw3gPxP30qX8GH+NJ/wgPifP/IKk/Mf40c9Pug5J9jIGq3qt/x93Gc9DIakXWL4DH2uX2+c1p/8IF4nzj+y5frkf40v/CA+Jh/zDJP++1/xo56fdByz7Gb/AG3qHT7XJ+PenHW7/obpzj1wf6VpL8P/ABS2caW//fa/405vh94oRtp0x8j0dT+uaXNT7ofLPsYF1fXFyQZ5CzDgEgdPwptpeTWc5kibDEEHjPFb7fD7xOCo/sx1DHAJcY/nUq/DbxQWA+wDn/pqo/rT9pT7i9nPsZsfiO9Qf8smx1yv/wBeph4ovBx5cJP+6f8AGrbfDjxSP+YcPr5qf41Efh/4nXrpvH/XVP8AGlzU31Hy1Ow0eKrgdYIfpz/jUi+KpM82qcf7Ro/4V94ozj+z+faVP8ahv/BfiDTLKS9u9PMdvEAXfzEbAyB2NK9N9R2muhaXxUw+VrZCfZ//AK1P/wCEnUjm0/J//rVyYbtntThIe2M0/ZonnZ1i+JojjMDjI7NmpodfgaTHkyD05FceHOefyqzbSnzh154/SplDQpSOg1XUIrqFFVHDA5wfpWBHJ5d6j44WTJxz3q3I2V6Z9M1mykiV/qOlKmEmdgmpQY6sPwqZNUtsAbm/75rm0b5MjPPept3APGKzaKTOi/tK2wDv7+hpDf2+ceZ19jXPB8L3B/nVae/ig4zub0GKFFvYblY6WS7t9ufNAHrXJapewtNLFEdwZ/vDp17VSuL6a4JywVSc7R0qqetdMKdtzGc77G/pdxr2qoljaXbNFaofLidhtQFsnAPua0n0fxSoJJib/gC/4Vytvdz2U7SW8jRseMqe1aKeKNYRdv21yP8AaAP9KU4Sv7tioThb3rl2W18SRn5o1+oRP8KrltbT70Y/FVqBvEmpuPmuM/8AABULa1eOMGQf98ikoz6pA5R6NnR+EtS1O08Y6NLMm2MXkSucfwlgD+hr6sZQ6lT0IxXxhDrFzHPHLvOUcMMexr7Ntp1urWG4TlZUV1+hGa568WmnY0pSvdGYFwSp6inipbpNs5PY81FW8HdXIluKBThSDrS1qZsngPauR1tHk8QyxoMuzIFHTJKiusiPziuW1STyvFBkxnZJG2PoFrys4/hU/wDGv1PnuIf4VL/HH8mXYYDp+oQqjsXYKZELbtpPXHtU95ZXE91cM7PEgBZZRgjpwMGq+jSxTSveXUq+YTkA1f1e4LCOCNvlf5i2eMdq9dWsfQPcgS5lbR5PmI2uoz7HHFV7dpLS6hm80skv8OBxg4I960/IiGlSwQyI7bCTtIJJrJdhJZQkHJWQj8CM/wBKoksX32ie9nSOQK6coCMjgZxWLqs7zzws7FiIgMnuMn/Gt6V9uqQSdBIik/yP9K53UV2XIQ/wjb+RIryM8/3R+qPA4j/3F+qO8HDOPfNB5NB4mPuKDXsHvEkDbLhG7BhWrKmWPuKxa3Cd6I3qM1hXjeLKi7Mymi+Y/WirpjGTxRXi8h0+0Pkf4jc+N73GfuxjH/AFrm8Eu3BPzda6nx/E03jS9eJd6Hy8MvI4RfSsXEMZfehD7vWvUpO1OK8iams2U9hxnGfalAzwfyx7VdW6gQbQmM+4oEtl12N/h+tVqSUwhOOvFAQgg4Jq4JbM/Kobk4HJpoktCRjIB4o1EVthyNp/SlAzkdB3rRtbeO6RnToDjJPep/7OUA9PxNTzFWMcryeOelJ5fbuK2l01QfmGBnH3hTxpcJ4OcdvmFHMPlMEqD0HQdKRkK9Qf6V0H9mwYOWbn3GKY2lWzD/WsPyoUw5GVvCbY8Y6HnHN7H/6EK+rAkbYJRc464r5e0W3W18baGisWH2yI5OOu8V9PB8KPasq65mmaUnZMd5UQP+rT/vmlMcR/gX8hTWnUjbt5/vU0uax9ma847y4s/wCrX8qXyoj1jX8qj84JyQT9KaJc80ezDnFMMe/cEXP0pTGn90flSb6TdTUA5iQRx4+6Pypwjj/ur+VRBx607fRyBzD/AC0/uj8qb5UZ/hH5Ub80nmUcgcw4RRg/dH5U4InoPyqPfS76OQOYeEQD7o/Kgon90flTPMpC9HIHMO2KD0H5UFU/uj8qYXPqaTfT9mLmH+XH/cH5U0on90flTd9Jvo5A5x2xP7q/lUciJt+6vT0oL0yRxj0pSp6DUxyKm4/KOlIVTao2r+VRo4yPcUgfkVn7MvnJPLTYflX8qZ5SbM7Bk+1IZfk59aGf9370ezDnJBHHj7i/lTDGhJ+UZPtSeZ8uaaJMvz2p+zDnHPGgA+UflSmKMj7i/lTJHyB9aXzAQDTVMTmRtFH5g+QdOwpjxIxwFX8qc8nzZ44FMVuMnOSafsxc5G8MakAKuPpVXUryz061Mk+EB+VSEyfwqxdSCGIyNnCqWOPauW0nXTqWoJPJ1C79u7hQegq40ridQ1heS3zEafpM92y9JJWKIvsMnn8Bim3j67ZQbzoayDqfLdWwPpjNdjZzx+Uqoo6du9S3MxROQAT2q1RjYfO27HI6fqFrqUZ2xBJlHzxsACPwqZ7eNjgRr78dKs3ttbhmu1gVJcZMqdTjsaZF8yBx/FzWagm7Dm2iFYUD4Kr19Kz/ABXax3HhXVYti/8AHpIQNo6hSR/KtR5cMVH07dc5z6+1Raign065ixnzInX81Iq407Mzc7o+WMDP+FKMdzjHFKUIz+uaXaWbgduwruOMADngjj9alg+WVT7jkU1VPXH508ADkAHnNSxo0HIxg9TzyKo3JPnA+3p1rQcDaMdD14rPveiken+f61nT3LlsXYCTEhHoOQabNOkSbnOPT1NZ4vTFCqKoyO5qq7M7bmO4+9Wqd3qQ5lq41B5Rtj+Rf1qln3oorVJLYhtsKQ0tK1MQhpKXsDQRigBKKKKACvsHwBe/2h4A0K4zkmzRCfdBtP6rXx9X0/8ABG9+1fDeCI9bW4lh/Xf/AOz1z4lXima0Xqd5eJlA3ocVSrUlTfEy+orM71NF6Gk1qLSg0lLW6MRynDZrn9c0+8/tVrmGF5FfBUxruwQAOR+FdB7HpViM5iQnqVGa58XhI4qnySdrO+h52Y4GONpKnJtWd013ODSw1CP7ltdr9EYVYkXV5ggeG6+QbVxERx+Art6K5P7Lqf8AP+X3/wDBPO/sat/0Ez+9/wCZxMH9s27l4obkEjHMJP8AMVGsGqLnFtcAZJx5J/wruqWj+yqn/P8Al94f2PW/6CZ/e/8AM4mU6zP5fmQXB8sYXEGMD8BUa6bqV5cIr20oY8bnjKqB7nFd1Sg0pZO6itUrSa7NkTyJ1Fy1a85R7NkcgxIh+opSKJfug+hBpTXsn0A2te0bfZp7cVkGtLTmzA6+hzUTV0NE2KKfiiuH2SNLnw4jmNVAYj6Ggu24hgQQTngcVFAGZEGefarsOiapdTsIY8sBnlxXYo3ZPNYriU/7XPSnLLj+EjnuKu/8IrrgH+oAH/XRf8akXwrrZJHkp0/56LVezfYXOu5nCXIHy/8AjtL5meApz/u1e/4RbXAOLdefR1/xpB4b1xR/x78f76/40vZsOdFVZnA2KXA64VevqePwpwuZ84DzenBNbmmeGr94XNza5cNx849PY1fbwzOCClmR7B/p71Di09i079TlVurlnGHl5GO9TLJfEfKJvwU810ieG7pWBNq68/3x/jUo0i9jJCWrcdOhqHfoi426s5gDVWHEdyR/uk/0pHOqxDLw3Iz1JjPP6V1gtr9AQbQn/gP+FRNDqG7DW8gzz9wgVKcuxpaHcx/DUlw/i7SBKHBF5EPmXH8Yr6bB4r50sIZIvGOjtLGyE3cXUY/jFfRQ6DjmiavYUHa4pNJ5gFNOccCozkmkoiciTKk5pcrioAGY8DJo3c4NVyC5yfK0uV+lQBqTdRyBzljetHmL61W3daN3NPkQuctF1o3rVXd/9ejdj6Ucgc5a3rRvWqu7il30cgc7LO9aQutV93vSb8mjkQc7LO9c0m9fQ1Bu+tIXo5A52WN6+lJvWq5b3oLU+QXOWNy+lRyuu3pUe8+tRyvkChwGpkgkTcvH8NNDptzjvVffgK3+yaA37k/Wo5EPnZP5i+lKZE8vp2qtuzuP4Ubv3ePUUciHzssNKgTGMZpEkTd0PIqqzEn29acGCyj6YoUA52WZpF29KcJFIHHFVJW+Snb8Ln0FPkRPOyR5FJC7etODIMccDtVNn5BHTPrU2+nyIOdj5yjgAqMHg15tFbTweILfTYYJoR5pLM6bSUB4IyOc16JI3AIA47Gqk9lHFIl9GEW3hGAijl2Y/MSfUcH8KmWiNaXvPUvzxzC1EcG/DcMVk2HH+92/nXNaNp051qQ7IY03fOY7h5iV92bvXUPepaW7yFgwRc7V/iPasR9aKRu8eoWEU+eIGUMoH0BBz71imjrUXuVNUhvItbkizMyocAxz7cKeQSvQ88V0EZCQIgx8qgVzy6zNez+ddR2pcELiI5x75roVUlA3fAp0bNszxScUivKcuccUvUYPfipDAz7nTlVHPP8An0pMfJ+FbcpyJniE2l6HFdzRO1rvSRlYGQAggketKunaB2+ykdeJf/r1n+JfDEzeJ9TZGIV7mRlGwnqc/wBayT4Yuc8N1/2TUOMf5jVSl/KdUNH0EqOINp6Ym6/rUqaFoTkgCPjr++Ncf/wjF0P4x/3yf8KY3hy7X+IfkaXIv5x87/kO7OkaJjAAJxjiUmvNJvNkuXjyWKkgAe3/AOqrj6BeKM5XH4/4VmSxvBO0T/eU4Na0aajezuZVpuVvdsatrZW0unSebb3Au9+EbPybfcdfWg6VHgjMgP1H+FU7Cxub7zPIYfuxk5bFWP7FvuP3i/8AfRq27PczWq2N3TNB0O9ljt2N8JmXk5ULkDnHBrbPw/0ZWw1zcqf+ui8f+O1xUek6or7opdreokIqdbDXRjbeSDjH+vIxWMlJvSZvCUUtYHYf8K80dhxeXQ/4En+FZHijwXY6LoxvLW6mkdXVSrlSMH6D6Vl/ZPEg6X0/H/TyeP1pJrDxFcwGCa7llhY5KNcEjP0JpRU003PQqTg4tKGpzmPl/GvUdP8Ahdp99p9tdjVph58SybRGvGQDjrXDDwzqmCPLXGf74rWgg8XW8CQw306xoNqIJ+APQe1XWk5L3JpGdGKi3zxudNJ8JLcfc1SYn/riD/WoG+EcnJGoyAeptv8A69Yj3Pjezhkm+33QREyzCboB1rNbxz4mZstrFwx9yD/SsYwxD+GaZrKeHW8Gjpz8Jbj+HUc59bc/416x8JNBuPDWk6hp803mq06zKdhXGVwev+6K8a03xB4/1K3+0WN1PLFuK7hHHjI7civTvhRqHi2bXLyHxEZGt2t8xFkQYcMP7o9CfyqJe2vac0x/umrwi0euVmzpsmYds1pVUu0+634GtaTszKa0KuM04UUV1IxFqeM5T8TUFSxH5SKpEMloooBqyRaKSimIdSim9KWmAkozG3rikU7kVvUU81HHxHt9CRSYAau6Y2JXX1Gapmp7Ftt0vvxUy2GjTwc0VJtorn9mO58MWy/NEMj7wwPxru9EG+8lAwBszz9RXF+Q9teLFKjRyxuFZWXBBz39K7jQlHnysSOFAxj3rpp7kT2NooVH3gQeMA00cNnpzmpcDnLD8aNo5O4fnXQYERBzjH60dccVKyrwdw/xphUeo/CgCzZ42H2YVZJBP8qgtFBRhnPIqyUIxxWctzWOwzvn/PekOQcjGaeRkAYppwD0/wA/5FSURnOM0jc8cYHan7SV5B645pkjiONnc/KBk0hnN6iceNfDw4z9pjJx/vivdlIIrwOdXuvGmhknY0lzGNx/h+cYr3FY7gD/AF3X/ZrmqO7N4aIumVjGYxjaetMK8EVX8qc/8tv0p/lTYP74/lUoGPWVoMlcbjxyKrbiSSTyaZJDPu5mOM+lRNFJ/wA92rRGbLG/3o35HBrlPE+qXWjNaMly4WQsGGBzjHt71gnxje5/1xI79P8ACi40j0cvyaC9edL4vue87Z+g/wAKX/hLrr/nu+PoKVwPRN9G8Y6153/wllzx/pEvpxij/hKbn/n4lFHMFj0TzMUeZ9K87/4Sm673Ev505PEl0/8Ay8y/Xd0o50PlZ6FvpC9eev4nnycXE3/fVNHiS4PP2ib8WNLnQcp6Lv4pC9cPBrkj43TTf99kVcGplwCJ5f8Av4al1UilTbOt356UheuYkvWWNT50nI/56GoxdsysfPn4/wCmhqfbor2LOq384xTZG+XNcm1y78LdSqccZkYj+dMgupd2GuZXI/22x+WaTxC7DVB9zpS5KL+VLvPlt161zy3T4z5jcnGS5plzfMjECVuvZzUfWI9ivYM6RXJ3detCs3ljg1yf9pMD/rJD/wADNPXUyV4LH33Gn9YXYXsWdOpYvyOB1pWciUEA4rkptTIGCzH6NVQ6k2eXb6ljxQq67CdJncSyHZ3/AAppkLbUA7dq4g6m/wDeb/vuozqbYzuJ/wCBVftUL2Z3UjkbRUu8+h/KvPTqbZH17Go/7V6jK9f73f8ACmqgvZnoUz/KBg/SsLxdfahY+HJL3TvNE8ZVWCn5SjcHcvQgdea5j+1SwGwpgHk5zmq2naqZ/E9lBcR5t2d0+YnBco2D7YP9D9Naa55JEv3eps6N40hSVNO1OXcMiOG6lQRiXA6MoJCt+JB9jxXZJBbyQPLbi2WRslnQ7GJ+o6mvHfEWgPZL9oCyXVhM33HwHhbJzlzx+J6+vrlvrlxBpEsumaibQQyeX9lnuQ7kdMhSp4+jcY6U6mHjK8oP5GsMU4WjNHpniHULDwxazak6QyXsh2xhmLNI3YEk5xXZ6bcyXmk21xPALeWaJXeInOwkZxmvnPTb95tai1LWmm1OC0lUuDKcEZ4xkdM4yOK9u0zx34avwqNM1mx6C6QKp/4EMjH1IrKFLkQVq/tX5HQSSMmFjZhkYOO9R7jsxT2tI5ow0Ko6MNwK4wR656YqstogJBQAg4IIqmZI4vXQseuXOADkgj2OBVDBHzFRgnjJra8R2wj1cbQFDRKew9R/SspcDou7sCegrzamkmenSV4IRdoToQcdKlVWb5nBAHAJP8qRYi3rz69KnEDNhQS59hxWTZukR74wVOxiF9Tx9cV5T4uTHia6brv2tn1+UV7NBayQ/OFQMOdzgNj8+K8w+JMb/wDCQwztyZIFycYyQSP8K6cHP95Y5sZD93cPhynn6vfWecCW2Pf0I/xrv/7IdZ45UkQlWDYC8Z2gEfzrzn4ftt8ZwR9fOR1/8d3f0r2+CxLsPlyB1rPHS5ag8HBTp6mNYaeI97TxR+Y8hbcozgYAxzWnBpdtIjboUHHLsox+HrWoloT8qgBT2Xof8atpZogG7JP90V5sqx3KEY6GMmi2kh4gQjPLGMVOmiWSNg20DnHGYRW2sJOAoIx0z1qzFaEex/U1l7WTE5RW5jRaHYZLSWUJJ7GNatRaHYHpY22T1JhX/Ct+20t5Dnbx69Klnn03S/8AXyh5e0a8mtY0aklduyOWeIje0VdmBJ4Ssr6znt/sMB82Nk4jA6g96+VdX0ifR9VnsrkBZInMbAHOCDg19i/aNW1NNtnALOA8b5OD+A//AFV8o+Pbaaw8Z6ja3EnmSxXDAuf4ueD+Ir1sBHkbSu0cVeTkrytc9R+EGoXms+E7jRbTSbUi3mHmXG0AkNz8x5yeD+lepaP4SbSpGvZ7pnmwfkQbUGfbvXlH7OWpCK517TyR8yxTL+BZT/MV7tNcblK0Vo04VG3uTGrPl5Y7Fao513RN7c1JSda1iUzNpaV12uQexpvWutHOxR0qWI84qOnRn5hVIhk9GaTNGa0JFzS03NKDQAtKDTaUGmIdmooziSVfcH8xUmaj6XPsy/yP/wBehgOanwNtmU+hzTDSxffpMDoKKhSUbFz6UVmM+I5JZJQJpZGeQgMzOck49/pXZ+GnEkTyj+JFNcY6/wCijGM7Ofyrp/Bsm/TGOeQQn+fzrSnuTPY6ncfXFIW9uM/So8nOB1pCcjPpxj1roMLEytlSPQ56U3PXHeowSpPHXig8N3NIZfsZMOyY68j8P/11dLDp3A61iwyeVKjeh9K1x0wDkevr0rORpDYerAHpgUw4OcD3peCo/lQevU81JYg681Wv8/ZCMfxA/rVgjGOeKr3mDat6cfzpPYpbnMl9vjPQW9LqP/0Na98UV8/zf8jbofP/AC8x9v8AbFfQSg4rnkaIYflwAuT6+lLzg08rQRwaEJkSNGG/fKWX0FVSOTgcZ4qcjk0witCGzg/iLbvPFpyqxH7x84+griBpU5xl5P0r1rWLBb+5s426KWI/IUkXhmIsG28ewo5Li5rHlQ0qU/xyfhj/AAqUaXIB96T2/wA4r1k+F4eMKePanr4YiH8FP2Yc55KNLk65k/HH+FO/suX0k/P/AOtXrq+GoAchKd/wjdv/AHKXsmP2h5LFpUjMM+b+dWZtGkiYrGshXHdutepr4cgH8FWm8P25+YoOlS6TuUqiPF20yYdUk/OpYtJkLD5H/OvW30CDJ/d0LoEI/gp+xYvaI4Cw8PxyAF1l/wC+yK2k8N2ZwPKk9/3zf412NvpEadFxVr+zkz939KwlQlfc1VZHHtoFiioPKZu/Mz/4019Ds1iYiCQEcY818fzrrZ9NYgbV7YyK8N1Oy+KJ1K6W1XUDbiVthGzGMnGOM1n9Xl3L9sjujosO8lUZRj/no3+NV/7HjLEgHPs7f41wq6V8SpWXzU1U7fm+8g/z3qd9N8dtdwOttqtuisN++53Bue+e1S8M+5arrsdimmRLj5eh4G48frUd5pqhsbcj/eNdIkBMZ3KM9Kbd24GcqABWPsXc19orHJLpwZseXn/gRq0mkwlB+5zj3Nbi2w3HAGQtWktx5Y49qUqUgVRHHXOlKOBEAfes/wDs1dwBiXk/3a7a5tgZPwqitoJLrI5C89aqFKRLmrmENFh2jdChJ/2aP7Eh/wCeKZHfFdUttumGRgAcVS1yX+zNIurpcbo4zg4/iPA/WtFTYnJWONvJNLsXMbPD5gPKAAnPfmsa51e0GTFDHxxkIDiuUvbthKzFiSOOT1z/APrpEfzIyTn6dcfSu2FBLc5JVmzVuNYlZ1wUKgkEHg/n2plreumrWc+QpjnTB7YDc/z/AFFZqs8bcHj1p4XYC2SGxkEcEen9K6IJRaaMnJs9i1GXT9Pu5kvLy1SOVQ0kMrru+oXrz9K4q58J6PdrLqFomYMnCwy5jJHXGP5ZFadjotrc2SOqOxk+Z2cAMSeuecn8acmlXWlFxpzLGbjiRRhV4/i2/dyO2AOveuytgZTl7RaXKhWXLyvU5a1tbPTIrm1u7W4aS7Ta0cMYIiU8Ak56jrgDmuXMs0AeFZN21iudpB478jP516lDo5hJMuSc5bc2SSe5J+led+IbI2Ws3KANtc+Yu7nrz/Ss69BwgmTzXZY0TxXq+gy7rG7eNOjRdY2+qHj9K9N8MfFC11e+istSiEN3O+1JYzlGY+ueVyfrXh7nCDn61d0Dc/iPTQMkm6i4H+8K4mikz6A8TQI17byORzGRj1IP/wBesdbbIGSqp0GP88103iGBnFu4XOGYZJ47f4VixW5Y4A3H0FeNiHabPawyvTRHFbr0jALY7jJq3FbmP5DtJ65JOAKsRWzBeXwem1en41ft7MuckEj2Nccp2OtRKS2gY43Bj03Nn9BXnHxb0/yH0q5HKuskecehB/rXsUdnuIAXGPSuE+MmnA+GrK5ySYrnaSOwZT/gK0wdX9+jDFpOk0eWeC5PJ8daOzHh51X/AL6+X+tfTEdqNoA6deTXy/o7G28Q6Xc4P7q4jc/QMDX13BYs+ODit8zi5TjynFhavs4O5nJag9Cce1WYrHOAAMVspYJGu6Vgqj1NILuNTss4TM/97sPxrkjhGtajsOWKb+Erw6WcZYBV9TTmubO1by4ENzN/dQZqc2Nxd83cxC/8804H41bht4LVNsSBRXbTw1tYq3m9/uOaVW/xO5mNa6nqPE0wtYP7kfLH8as2ej2GnnfHEDKesj8sfxq28wHeqslx71cpUqeu78yeaclZaItPMFHFfJnxotvs/wAStRcD5ZRHKPxQf1Br6gkn96+d/jzb48UWN0B8stoFJ91Zv6EU8NiXOtZjlS5YNlD4JX4s/iJ5WcC6tZI/5P8A+y19GyT89a+TPAF6bDx9ok44zcLGfo3y/wBa+pHk461z5o3GomuqN8JHmibKncqt6jNLUFo++1jPtipq66bvFMmSs7FS5TD7vWofarlwu6L6GqddkHoYTWoU5Tg5pKD7VoZsmzS0wHIBpasgdRSZopgOzzS5pmacDTEOqOQ7XiPvg/lTs1HccQlu6kN+RoYEppYvvdaaeaWL79AFzdjvRSbveiiwHyD9ildxaIFMpOzGeM9K3/DOlXWmQXEVyFAZwVw2e1Z0U4i1JLhg2BJuIHXrW6NettufLlx+H+NEbLUmVzS2n3pAOeelUBrcA+7HKfy/xpTrkDHlJc/h/jWnMiOVl3AxR3x/I1Q/tm3P/LKXp7f403+2oM8Ryc+wo5kHKzRxWrbSmS3TJOV4Ncv/AG1B/wA8pfwxVm28RQRBt0MpBOeMVMmi4qx0wO3mmnHByawz4otATiCfA9h/jTT4ptT1t5v0/wAazNNDeznnjgcVXvzm1bgjOBwPesj/AISm1HH2eY8+3+NVbvxZaPCUFvPk9+MfzpPYaaKt0wHirRCOB9qj4Pf51r6EUV81x6pHqfinSDHHIgS6j4bGT84/wr6VXGKykWhaaelO60wmhCZCRzTSKe2KYaszMPXtattANrd3S7o2do/vAYJXPf6VSi+KGhxj5sH/ALbLXRTaVYatiLULOC6iT51SaMOAfXB7806Pwd4cXkaFpo5/59U/wrSKfQltHPt8WNCA4CfjOP8ACmj4u6HjpF/3/P8A8TXVp4U8Pr00TTR/26p/hVhPDmhr93R7Af8Abqn+FP3u4tDij8YNEX/n3/Gdv/iKb/wuDRznaLXH/Xdv/iK71dD0kdNLsh/27p/hUi6Vpy9LC1H0hX/Cj3u4aHnx+L2lAZxaAf8AXd//AIirulfEhdfvPsWmWcFxcbC+xZmHyjvkqBXcLp9mp4tIB9I1qteafapf2VwPLtyhcFlCrnK9ORStLuPmXYyRqfiJxuXQoiM4/wCPkUo1DxJ/0BIBj/p4H+NbMEthaQpB9ujYLnDSTgscnPJz70jahpqEhr62H1mX/Gn8xfIy0v8AxH1OkwAf9dAf/ZqsC+19v+YfAPxB/wDZ6sNqukDg6nZAe9wv+NRtrehAgnVtPH1uE/xqXHzGn5Dftmv4H+hQD8B/8XQbjXiv/HtbA+4/+zpreIvD4yDrGnD/ALbp/jUR8TeG1IzrOnj/ALbr/jS5F3HzeRXGoa29/PabLRXiRHO7gENuxj5v9k0ySLW5D8z2WP8AeqmvijwzF4oubhtUs/Lks4l3q2QSryccd/mq83jjwovP9r234Kx/pU8ie7K52uhB9i1XjMll1/56f/Wpk2m6rLjE1kv/AG0J/wDZambx94UX/mKxfhG/+FNPxC8KDH/EzBPtC/8AhU+xh3K9rLsVRpOrhv8Aj5s89OG/+xqaPSdX24+1Wq/jn/2Wkb4i+Fh/y/ufpA/+FRt8SPC4PF1MfpA1HsYMPazCfRtTDhDqNsGcEgAZJx1/hrjoJ/F7AvF4fvmU8hg0WGHqM11kvxH8LSAeY00mOgNvnH511Wnuk2nW80fEckasoIxgEZFL2Mege1l1PLw3jYnI0C+U57y29Yfi258TRaMV1bTprW3kkCh5JYWyw5A+Xnt+le5EA9ya8r+NU+zTtLtgx+d5JDj2AH/sxpqkkwdRtHh9zLvkePqOlJaSArhgMr1559KZNGzPs3fOBx7/AEqvblluiGBBwTgnHNbGZpj7wOT16d8f5/rUp2lBtAwDngd6rMy7Dn0wAOv+elWEAZNwA+bB9+f8gVQj07wxd+d4etpIVR5kjCsT2I4/pVmSWZ7uIyquW3LgD2P+FYngmQyaOwVgzRysmc++R/OuhupFhhUSSqZGYKhxnJ6fpXvU5c1NMytZjCqIoxy7HOCSOM1wvju3HnW92AcNuQ549xXfReY0a7wm7ZhmXHJx9BXL+NrfzdGaQdYmDgj9f0NZ4iHNSY09TzCeMqM9Vz6V03wz0xtS8d6cuRthY3DZPZRn+eKwJot8b4y2Oa6P4WXS23xA0/ecCQPGPqVNeBLY3jufQOswiW2TIHDg889qz7fTwUBBx9f8KteIdXg0fRLjUbiKSSGAKzLGBuwWA4yfeuKi+MXhxAD9h1Ec9o0/+KrxsXQqzneCPXw1eEKdpM7aOwuPNjVSnk/xZBz+ArUitQQAOnevPB8bPDqnDWWpf9+0/wDiqnT44+FVA/0LVCf+uaf/ABVcTweIe8S5YuPRno8dqMjB/Sua+KGkSXngK7MSbmhdJRjsA2CfyJpujfGHwffJI0jXdtsGQJoh83sNpP64rF8U/GvR7nR73T7G2ulM0LRhyikcjHPNb0MJKDUnucs6zlp0PFrLybe8t2uATGsmXVfTOeD/AJ619haff/b7G3nsYW8qaNXWSTjgjIr4rnv1mb5VKnOa9+8PfHfwzp3hnTLK7t9Q+029rHDL5cKlSyqASDu6cV6s6cm0zlbVrHsH2ESNuuZDKfQ9PyqceXEuFUAe1eTN+0J4Txxbap/35T/4uqz/ALQXhZv+XTVf+/Sf/F1m04fBDUmze7PXnuAB1qtJc+9eSN8ffDBHFpqn/fpP/i6gb48eGm6Wmp/9+k/+KrkqRxMvss1hGC3Z6y9x71We4rypvjp4bP8Ay56l/wB+0/8AiqYfjj4cP/LnqX/ftP8A4quSWFxL+yzojKkup6i8pNeO/HO282y0q5A+40qE/XaR/I1ePxv8Of8APpqX/ftP/iq5bx18R9D8UaLHaW1terMku8GRFAxgjsx9RWuEw1enWjKUdB1alJ02kzzXT7k2epWl0v3oZlcfgQa+tgS4BXJDDIxXyBkY4znPFeywfGSwj0y1t3jvTKkKJJ5aKi7goB/i3EZz3Fd2Y4adbl5EZYOtGF0z3HTci1KN1ViMZ59auVwHww8XWfieHUktY3jNuyMysgX72emCc/d7131FKDhBRluObUpNoRhuUj1rPbgkelaJqjOu2U4rppmMxmaM03NLW5iSIflHtxTqjQ9R70/PvVogWlzTc0uaYC5pRTaAaYh+aRxuQqe4xSZpc0wEjbdEp9qki+/UMX3SvoxFSx/epAXe3f8AKimfjRQB8jX7NBau6HDqAc498Vpp4c1RkB+3wDIz/q//AK1Z+qqWsJRjrj+Yr03R9EvNXOyzgaVkXLBQOBRFJ7kzbWxxA8Narkf6fBj/AK51IvhfUzkfboc/9c67OW0ktrgwzKVdTtYMMEGuw8PeEBqtqZzJsUcDIzmrcYrUhTkeNHwxqpG0X0RbHQRGkk8N6qnD30akjjMNem6/oraTdNFu6dxXNy5Y8kkkY560KCYc7OSPh3UjjGpRdP8AnlR/wjuothTqcQwf+eI/xrqKaenA5p8iDnZiDwZqzcf2tFz/ANMaVfBepHJOsR4/64f/AF66+1fzLZWzyPlNWVUA8/jWTVjdao4j/hCL8nnWk/C3/wDr1Bc+BrtIi7awG6Aj7Njv9a70jIOTVe//AOPR2OOCvT6ipexSSPLtPs3sPF9pbPN5piu4fmC7c5IPSvqNPu180SHHj2IA/wDL3AeP+A19LoMLispDW4tMapPwpjUkDIWqM1KwzUZ61qjNlS81UaQiTNC0u9tmFIGOCc/pVX/hNhnH2B+D/wA9R/hVfxQf9AgOcfvx/wCgtXMZwa2hsZTep2I8c4/5hzfjN/8AWpT48IHGnfnN/wDY1xm79KM81ViOZnZf8J5L/wBA1B/22P8A8TTT49uD00+L/v8AH/CuQz2pM4zilYOZnYf8J5dfw2EOf+uh/wAKwvE+t3Xii0t9LlhigilmyzoxLDAJ7+4rNDHrikDf6ZaHPST+hptBdmNN4Pt4HCm5lbPQ4H+FNHhS0/57TD8V/wAK6jUvvpnuOKosDxtOKlwiUpyMYeFbEdZZyM92H+FSp4U00g5ac+vz1oHcOvap4Q5Oc/KKHFApMzV8J6aP+e3/AH8p3/CKaZ3ST/v4a1g3tT9341PKiuZmB/wjunJqjJ5chj+yh9hkOM7yM0y80TTYod8dvjB/vnn9a13/AOQonX5rRx+Ui/41FdgNbuMdBQoqwOTOPeG1i1QRvGBFsLYLHj9arGJZUnkhPyR5IOeSf8KsatgzwxKRulcL+A61DAAqalHzwTx7YqbFXEuRGumQzKuGJGTU1/Ci2yNCgHzAnHp3qnK+7wzGccgDvjvTtTuR/Zsa5+ZlBwDziiyC5biiSeaOUxhYS4CgfxD6VsL4u1+DWJbBNVuFtokAjRSAFA464/zis1pAtrER/snI6dKijiW68T6j1DRMcbT2yam6RS1NXUfFfiCKW12a1eKrPhgJOvIrD8UatealNEl1dzXBiUkebIW25POM1sS6dbzBHd2+Q5Xnp/niuZ8Qxpa38iqflEascnJJP+f5UozTY3Fow51gVN8pbPXA79aqQNvlLHIUDgenNOEMly+5jtUngZ/z/nFJEdl6ykg/LjjtWhJYk4BJIGfT/P8An8altSnkupA+Q89Oh6f1qu7ZGSeOlJbyhLnGRhhgnHQ/5/nQB2/gq5VDcxMxUvIGyTnqv65xXS3f764UJnEbAjA9+1cj4PljS4u1ZQzBVZSBxnJ/LoK7FXS1tRJKPncjg+uf8/lXsYV81NIzloydmnLYUYVc9+vTn/PrWP4kjkfSZI3UBnQnnNdBGZovklxtBPzDt/nmuV8Tvd6iZYYHMaIDyP8AP61vV0gyep5xGx2BuTjOc9q1fB1tJL420qOAHet0jgjuoOT+gNY0X7pnjP3lJB/Cuo+Hryr4+0to4/MyxVvYbTk/gK+flobo971fSF1zSLrTGk8sXMfl79u7bz1x3rkrP4AQNgtrEjL6m3AH869CtJPLuY2ATIP8f3fxrojY3N2c3t2BH/zyh4B+p6mvPrOfMuVnTFpLU8j1P4M+HksDa22oXU2o5AXykUj3yO351VsP2dVmG+91uSIH+BIQT+ea9whjtbNNkEaoB6Cla5HrWaq+z1lL5ITfNokeQt+zxpKj5Nevlb1MSGqN5+z3Yx20rp4gumdUJUGBcEgcd69le596rS3GQQTWFTHW+EqNOT3Pii8tDaPtJ3ZyOmOaqVv+JoGh1a6hK48qeRP1/wDrVgV69OXNFMwmrOwUVcstKv8AUiwsLOe6ZeWWCMuVHqQOlX18H+JH+7oGpnHX/RH/AMKpyS3YlFvoYlFao8M66YfO/sbUPK6b/sz4/PFSQ+EvEVyMw6HqLj1W2f8Awpc8e4ckuxi0V0H/AAg3in/oX9S/8Bm/wpD4H8UD/mX9S/8AAdv8KPaQ7oPZz7GBRW+PBHig/wDMA1D/AL8N/hVTUfDmr6RHHJqOn3FqkhIQyoV3EDOBQqkG7Jg4SSu0ZvG0evevQfBHw6tPFmhPqEmpS27pO0JjWIMOApznPvXnteu/Bq/xY6rZkj5ZI5VH1BB/kKyxUpRpuUNzbCxjKolI7z4beCofB+q3hg1CS4W6hAKvGFAKnIPX3NelGuR0e4/4nEIzwcj9DXXGuOjUlNXludVanGErR2EqvdJ8oYduKsU2QboiK6IOzMJaozqXtSkYOKSuk5xVOHPuP8/zqSoR/rFPrkf5/KpM1aJYpPBP8qi+0jbkRydcYxzSuHJGxwB3GOtIqzZ+aXP0WqEJ9q/6ZSflTxP1wjHAyPemhJdp/e/N2OO1GyXj98QO+B1p6CJo33rnaV7YNPz71WEcv/Pc/lU4oAReJXHqAamj+9UBOJl9wRViL71AFnn3/OilwaKAPkzU8C2Y8csMD3zXqXh7xJeaCWktCmHUB1dcg15Vqp/cdP4wOR716LpkYlu4IWYhJHWNyPQmlDZ3IqO2pcu9Sm1DUJbu4IMsjbmwMCui0LxfPpUBhQI6H+961yuqKYNSuIGfeYZGiB9lOKr+YVP0rXRoyT6mzrGsSapO0srAsx6isZznJzn175o3luM80hQ4poCI8ZyRimkYzyPensu3PPIqM7fzNMZb0+TbMYzyG/nWrszkjtXPqxVwwOCDmughcSQrIOjAH0rGouptSd9A2nv0qteKfsr55AIP61dYZHoBVW8wLOXg8DP61i3obpanm023/hO4iP8An7h/mtfTC9K+Z5wV8dxFgATcw9P+A/8A66+mVGazYLdgaYfapCKY1NCZEwqI1MwqNq0RmzB8UD/iXREdph/6C1cmQe5rsPEib9MHtKD/ADrlNnHFbQehlPciwR1P4UduT71J5eTineTznjBqrkWICPelwe9S+UScdam/s28PS0n57+WaLhaxU7jmjH+kW/J/1q/rxV0add97aUD3UimSWjxSW/mRlC0qYz/vCh3SuKLTdixqKhmQ5PQ9KptHGo+ZsZ9+9bGpxogj+h/pWS8aSnB6Cpvdl2sitmLd94fgaswtCUwX+cngVCbaMEc/rVmC3gI385HI5oYIf5KkZ5pRCh4/ringgCgtyeeKjUvQqrCja5YI33HSVDj6oa7T/hDdMkRlZ7jng5Yf4VxpcLq+nOO0kgyf9wH+lesJg4I3ZPXimh2Rx/8Awrnw5LMkjxzySqflYysMfkK4s+GdPt/irNorRE2NxGjbN5ycoe/XqK9kG0ShQx5Nea+IGW3+NulOTjzraNfqdzChjNk/DvwukH2ddPzEOqmZyP51cj+H3hV0AbSIGB67yzH8Of0rcCAE/Lu57jNTxkLKFGAoPtRYZ8961bw2upXttAMLBM8ar6BWIH6VRhlFt4m1CXu0Sv8AXKj/ABrd8W7R4z1ODblvPY5AxnPP9a5rVH8jXWb5QJLePv1+Uf4Vyyd7ouKtqarXpZduzHriuP8AFN3uviqv8xVd+Pxx/OunVsDg8Vx+qSW8F7M8il52cnBGT16+1KjHW5dR6GfFASvmzyERgZwDUVr+8uGK8FjgDr3qK4uZZjhjheu0dKmsVU7mfhRweK6jAsygN0Oc9efrTIkUMCCc98YPFK7mRiE6HoMZ4qxbwnaGbO4jI7+lAHQeD2RdUnUjG6JWHIIJB/8Ar12bhLm4VGB8tBzjvXnuhnydSwpQb04z9R09/wDPFeg6ViTl+EXrnv8A/Wr1sDrGxnM2POSWBGkAGQCf8elYd3co26NUDMSecVaWG4ny65WKQDaoGCo/nVbU3ttFsDLJzIRtROMk47f410zk36CsrHlmsQCy1ueMH+LceehPOK9A+EmlWd/rc9/Jv86xjDRKpwCWyCT64A6dOa80v7iS41CaSYjezc4HFeg/BvUFh8UT2jMALi2IUerKQf5ZrwaurdjaB7iR2zit2HUPMt43B+8oP6Vh9KisrkiDyyfuO6fgGIH8q8TMW4xTR34aCm2mdF9qz3pjXHvWasx9aUSMenNeK6kjs9gkXTOT3qJpM55qndXltYxeZeXEcCertjP0Heubl8fadJdC00yNrmcg4Z/kTgE/Xt7U406k9kFoo8Z+IVv9n8S6qnpcs4GP73P9a4Y9a7TxlqEuq6tPfTxoj3Cq+1BgcDH9P51xjDDke9fU4a6ppM8qv8bO3+GPiGbQNdujDCspuLcpg9iGBz1+te+aTfahrOlW1xIYo1lBDArkqynB46c9enevmbwizDxLaqrYL7kz9VNfQXhKWW2geBpmeMHcoI4BPXH5CuLMKavzLc68JqrHVLp8AcSTFp3HRpDnH0HQVOXVRhQAPaq5n4zmoXmHrXl3bO5U77lhpc1E0vXntVR7gD8aiNyNwHPWhRZoopFlpa86+Lsfn+GLaUAkxXa5x6FWH+Fdo9z7Vyfj8favB1+v8UYWQe21gf5Zrpw141IszrpOm0fP4++M+td18Krow+JbiEnia2YY9SCD/jXETDbO4963/A1z9m8Y2L5wGZk/NSK9ysuam0eJRfLUTPfbOfytRtpCekq5+ma9ENeSm6KkEdRyCK9XglE9rHIOjoGH4jNedh1a6PRxGtmLQaXtTa6UczKci7XIxTKnuhhgfaq+eK6Y7HPJWYHjB9xTz+tRscocelPzn8atEMWik6miqQhaWm0UwH5oFNopiEkbGxvRhmrMY+aq0vMTDvirEJ3bSO/NIC3x6n86KTmimI+TL5S3lIOczIM/iK76NlguEZWJ2MGzjHSuEuBm5tRx81yn867bK7up45pQ1TJqK5f1ieK71m8ubYlopZC6nGOvJ/XNUs4HtTkZFPbGeuKaT852/ga0WisZJWRds442mXeQB3zXZeKdF0bT9ItpbKcNO+PlDA7hj73tXArLg5BwfWpHuHcAE0NXY0RSYyQKiINPLZxwKa3JNUAw5z1rX0ubdC0Z/hOQPrWSeevb3qxYzCK6Uk4U/KfxqZq6Lg7M3SCRmql8w+xSkjPynoKvheelNnsop7aYvC8hCknEhXIx0rkk7I7FG7PLb4BfHMXP/LxDn/x2vplPu183660X/CwY/JgEC+dB8gZm/u55Jr6QQfLU3ukLZscRUbCpaYwpxJZAy45qI5qdhxUZFaIzZjeIf+QW3HIdTXJE8e9dvqlpJeWbQRY3sRjJx3rFXwxqDd4QP941UWS4N7GGrNz047Y96OTye3tW+vhO+7yQj/gR/wAKePCN7zmeEfif8Ku6D2cuxzoOCOfxxXapHeS6gEjT90Y93+rOOnXd657VnDwdc97mIf8AATXaQqYoYkOSQgGQfQVLSkxODtqctNZX+/aUcL/u1k63G8ZsjJGyt5vRgf7y16IDnoD+dYnijTo7yKCR2dTCGYY/A8/lXROtzQ5bGNPDcs+ZM5bXBtWM4z1rCMnPArt9R0mK4Cb3cD2NZw8M2XBM7rnnDOB/SsVZLU1lFnMk5GaQOQMV07+HbQFSshK5+bLnp+VSQeHLSZ9iIzHr94itVTbV+hndXtc5Te3J4o8xuhPFdp/whyH/AJYqPrKacvg+IDHkRn/toaXKu5XKzhpH/wBJ08k9Lgj843/wr1qJ5mUFVTGBjJPpXB6tptvZ3mnKkaqReoGAPsw/rXXwvqbiIxR2xhKjBdjnGPQVk1qapaFh2vWkyPKA7ferznx25tfij4VuCg3uIkY9uJOf516DJ/awICG0VQMn5Wrm/E9iLnUtNu9QitpZ4m/0aVVYbDkHofwqWrLccY3Z1rttkcY70+FVPzvgDPp1rOa11ckEXtsD3/dZ/rUqRaoEKSahCQRgBYBkE9+tJza6FcvmeRePownj2/cRsAwU5I65QVyPiFG+1WjRg7fs6nk+hYda9g8SaUpvXkuo457jaMymIAnjiuMNqktpAWijcbSp3jn7x9frXJze8zZU9EcvcytBYyzR43quV9M1xz/aTK0s8iZJyzFQT+deo+IbIDw7OFjRDsVtyjkDIJrzi5tWd0Qqdi89OproopWM6qszJcxyuzJCcdc5qa1UGEgryO1TXBjiwqphm4A6Z/wqraSDdz0z2rUyLqRb2y3CjtzVk8n2xn0qBZDkcgevPNSjoF6L2Hb8PypiLuj2sNxqUrXDhUVQVIbBzk13trqFlGPLT94vGc9CK8tupjFcRYlkQMD8yDJx+P0/nUi3khI/0u7bGGPIXHPt/iK6qOL9jGyQnDm3PX4NTjMTCNQXUEKg6nngVy+p2EsrNe6vfWtop+6smTgemPXmsWC6W7ivoooYYpo7RpoJI5JixZWHTc7KeCewrl5o7ueYtcCWTd8odjnnGRzTeY+00aLdLlWho6zaaYyzTWlzNJKhHLRhFI+mSareGNX/ALE8R2GoYO2GZWY8/c6Nx9CazpbO6t0O9SB3AOarKcGuWclN3sJaH18rK4Do2VYZUg9RVSGF/PuOyCXOTwOQD/Wue+G2sHWPBFkztultc2shz/dxj/x0rXRSaYby5LvO6wjHyKe9eTj4J09T0MHK09B0up2VpgM5lcfwxj+tY+t6xq/9ly3FnCbSKPktjJIzjqfr2rpLXTrK0/1cK7v7xGTS3/lT2skMiKyOuCpGQa8iPJF6I9B3keL3LXF+zNPNJNK3OWJNFlps1vcJcD5fLOfzrpJrWO3nZVQKAewqvKn+jyDodh/lXpRtbQwa1OB8Z6d/Z9zAgJYPDkZ+priJPvnHT65r07x4gmsdKuMZLIVJ/L/GvM5Rhq78O7wPPxCtMveH5vI8Q6fJ0xcID9CcGvojSn8sn36180wyGKeOQHBRgw/A19HWUoZUZc4IzWOMV0jowL3OhFz8v6U12kKb9rFfWsyR52gc220ygZUNyD7VXmudQinYoyvF/FI+1SBkFtvPI29Pr7V5nIenc0XmxUXn4OR65rnPtmpuhG5VKsQ7lkwg3dT6HGPzPFVmvtTI3JNDt2gnc6YzxnHPsfzrRUyeY6aaTDt0xk1la2n2rRb63Iz5lu6j6lTRaTTvEWnbcSx2NxyvGPu8d6ez8EHGD1NVFWZMndWPnm4H7zP94A1b0if7JrdhcH7qTIx+gbmor+EwXDRHrGzIfwY1XycI3px/n869rdHhbSPeXfBYcV6r4cn+0eHrGTOf3QX/AL54/pXjVtd/abKCftJGr/mM16l4FufP8OKmf9VM6f8As3/s1ebT0lY9Ko7xTOlJpvelPWkzXQjnZHcDdGfaqRq+eRiqLDa5FbQfQxmhP5Uif6tc9cYo6UL0I9D/APXrVGTHA0ZzSdKgvpZILG4lhCmVImZA3QkAkZqkhFnrRXlumfEbVr7VLS1kjs445pkjZhG2QGYAkfN70mo/ETWLTULiCFLZokkIRngZSy54JGeK29hK9jL2sbXPUwaXNYPhPWptd0GO8nEYm3sjhBgZB4/QitzNQ1Z2LTurof2p9nyij+7x+VRZp9nxJIvo2fzFSM0OPU/nRRmimI+UJjm+suuDcpx+NegWmjX97YXF7BbFra2GZZCygLxnueT7D2rz/ltTsB/08L1/Guy+0S+V5XmOIidxQN8pPrj1op7EzHiQA85/A095IsfKv4mq+fc05QGxlsc+laEIXcOcn6CgHJP6U94oRGWWYFsjj1qLHTvmi4NWNSLRL640eTVYrffaRMVkdWBK/Vc5A561mMxPXp6GpI3IjKeYyo33gDwceo71ATyeKEAue9NB5zQTnPv04oHT1pgdZYt59rHN1yozx371alhcWsnBJ2nisvwzKHMtu2M/fXPf1rrDas1u577See9eZXlyNo9WhHnimeJeIVYeP49/JEsPHp92vpKP7or538YRhPiMvXmSAnP/AAGvohOVFOLvFMxmrTY40xhUlMarRmyFhUZqVqjI5rQzYRLunQe9XfIB6VBaKrXSBsnr/KtURr6c1nKfKzensUhBTvJxzmrrQhWxik8oE9KXtDTQp+UOehqTb0HPAqfyj2FYOpm8TVYYYLZ5InbDuMgL0/AU41XeyVzKqla5sgDPJA/KqWsgPZtyCQp6H2rFvbHUCD5AkJPuf8aksLa7jjuFuYXUeX8rMOvWu6VP91z3OKnWbqqHKXblcxocdaa1jPcQ27QIx5wSCBjnqc9QPSrbrvgizzwOv0q9ZACxT6kVyq09GddSNkZF3o90TmMkjB6N/SjTbK7tbkmaIiMrjeT3zXQADGcE0k4/cDHGGB/WuxVmochxqhHn5jPDHJy2fx/+vSK37374wPeomjfzWCxZHbmlWMh+YgPfrWVzc5HxI27UV4HyXsLD8XA/rXWWRYWcLBR9wDmuW8QJv1KdBnIkgf8A8iJ/9euoshutIlOflLA/gSKwhUvNo2nC0EyxIxK/NtXt1rnvFYX7PYSDJ2zdR2ropgNqjcQevFYficbdGR8E7JlOTRX/AIbFR+NGxIOVbPUUqkdySfpTUkzDEwP3kBxjPapFkY44P5VotiHuYviK3zcqR0ZBnNcPp9p5sJjyqbZpFJ7DmvTNShEwjYoeBiuQ0/Sil5ewSK/E7MpA7EA15Fary1ZI9CkrwiZhsXntZIyispjOVbADDHQ5rxNSyZJ3bV+6oGa+j30tTkFXPHpXz/f2D2WtXenqjlop3iC7SSTuwMCunB1lNtGWKhazObvomb94eGHPHaqcGNwGB9a7O+8E+JHtjIdCv8Bd3MDA/ljNcWqFXZX3KwOCCMEV23TOOzW5qxEBRx15xnPWpTIEQMx46N/KqsUhZMA7s4461YTDRnJPvxj/AD/+uqJESGG9lVnV5ABt4OM/5zWtHpGnZVjFgdcE9qo2CIDwMZydox0/wrbWPzUIDFTjgD1/Cs2zRI05L27VPs73d28WMeW1y7R/98k4/SqEzyOCiqGc8HIFWTEr5JQknnBf8f8AD8quW1qsX7xlCnHGM8AVklqaX0Ob1TTv3aQtIzyyNuYkfdA5x+ormtU082b5H3T7Y/z3rtbu7/0iQQJ51wwx7IP6VjXmiXB3yzTZZhllA4FapmbR0XwY1k2uu3eku37u8i3xg/8APROf1Ut+Qr3BX2HH418t+Fr/APsfxhpt2WwsN0oc/wCyThv0Jr6glVtw2gnHWuTGxvTZ04N2qImM1U7uf5SKVWy2zv6DrVPUQ0eMgjI7ivFjG7PY0RgX6/6QxHfms5mXJUkYIqj46hnuNPgNusjsJASEBPGDXBnT79jzaXJOCM+Wa9SlTTjds4K1RxlZI6bxUom8J2Ug58qYrx6c/wCFeYXAIc565Ir0mSCX/hX88MiMjwT52sCD1H+Nec3a4lbjv6V04fS6OTEatMrjrXvugTedomnybslrePJ/4CK8NsdLvNRWY2kBl8ld8mCBgevNev8Agu5EnhPTySSVUqc9sMR/IUsUrxLwjtJnXwyqDgttyOGx0rNudLs5Mp5rupLEqAAo3EEjp6DH0qQyncRx3qFpCRjPFefY9LmEt7aO0LFXZtwCjdjgZJ/mxqt9ggRlKvICq7RyO6qvp6KKlLjpn8agku4I875U+XGefWqsyXJE8ZEEEcSZKooUZ9PwprSE9qzbzWYbYbQJJHVtpVR3xmq9rq5uLwwNGUGTtOc5I60+Vkua2PNPFMAh1++UdPPLfg3zf1rHz+6x6H/P8q6rx3B5etPJj/WxI+fplf6CuUXO1h+NepTd4I8iqrTZ6n4auDN4csmzkhNp/Akf0r1X4cXG631C3J5V0cfiCP6CvHPBMnm+Hyuf9XMy/wAj/WvUfh7L5et3EX/PSAn8iP8AE1xSVqjO6LvTR6UaaaU+9HatTFjaqXC4fPrVuobhcpn0q4PUiSuipnpR0c++DRQ3DDryK6EYMXNIQGUg8gjBpKUc1SEeH6ZAbHVtUiiRXvbaOQWwZc/OrgEgHvsDEVZuUbxPZ3GpeTHb3UcuNz3B2Sg5ZgPMY4K9cDjB6dKv67LY6d40vUu7ZEXeJY5o1Yybmw394DuenoKiu/EOk6jsN4biYouFDxbsHPPVu/HavQTbszjaWx0Pwuud2m31r18uVZAP94Y/9lrvhXmngTU7d/EtxbW0SR28sGUxGFYkY64+pr0oGuWt8ZvT+EdT7ckXZHZk/kf/AK9R0qHFxC2eMkH8ayNDSz7Gil/H9KKYj5Qj/wCQnYAHrMDXX4O3jnPeub0a3S88UaVbv915+cH2NdpqWnS6bMELCSNslWPBqYTSfKOcG1zGfgjtSgE8YpRkf560fN7elbGNhxjdSoI5YZGDR5Tlgvc+lJluueKUs3Bz+dIAMQVjk5A9BVmKxVkkkFxD+7UPsYkM2ew46iq+5sYzS5OM5oGhmwk9COfSlWJifunrS5b19qkQtnqaTY0jQ0SKeLU7eVYmYBwGx3B4NeuJpDGLleo7CvM/D8LPdKWnKYIJx3r3CPBjUj0rgr0vbTtc7IVJUoadT5m+I1i1p8TFwpUFrc5656f5/CvdEHy4ryj4yOg+IGngfe8qHP8A321esIPlojHljy9gb5tWLTWp/amN1rREsiYUw1I1MNWZMs6dGXvE46ZNby2yKdwHzduayNIbF5j1U1vUuRN3Y+Zoh8nJzmjyRU1FHsohzMhMHOc1RuoykwHrWpWbqILSBQccDkVUIKLuhOTZEFb1/Wm3CqYCHYDPGTTlA9c/Wqertt09mPQMDx9adTWLKh8SJJIljiVGI+UDP5U6C4hS3AM8agHuwrLvb8Cx831C9/auV+3tLKUwQpz3rCine5pVlpY9GiuYJn2xzRuw5IVgaq3Ws6bGrxte24kRgGUyAEEHmuF0HWzb6lcFo1CqhUtnryDXBeJL4S6/qKYOftEqnpz8xq3Vam4kKHu3PbTrmmhd39o2gB6ZkAqA+KNEHB1ezJ9pAf5V85Txbn4qxZFo2ChRhumf6UOs0Cgj1ObU4dS8bX3kXImtPsiuhX7u5WUn+VdZbanbQ3Etq0+JBcyrgqT1YkdsV5H4elMGpXGOP9EmHrn5Sa9a0aXeuoSg8tds35qtc9Jt1bo3ml7PUJvFGlxuwM87YHGy2c/0rD8V+I7Ofw/JDFFfGUujAtbOg6jqSMV1+RnOTu+tZfiyETeEb4jJYKGyeeQwNdkk3FpnPF2d0RWmryrZQLJpGpl0iVW/cgDIHuatx6rM4BXSL3Ge5jGP/HqvqS8aHsVB4p23K425HuaST7g2jS00edZCSWBo2JOUcgkfkSKjt4bdtUu/kXcdrdPbFRpdeSmzpxnHpWRZ6gY9emLNw8S8fQmvNxFeCqKLW25vTpycW0dSbWHn5BXDyeGNOtfH93rIt1NxKkewnoh2kMwHqcAf/rNdQdUFY2t6gIgl2gxIBs3ZGB3Gf1/OmsTRckoKxVOlNP3iLULqS70+7Wx+e6j3KqqQCG7D5sD09q4Pxp8L59e8NyapHo4i8QxxB3McijzyPvBgCQTjOD1zjmu4huVu4EW3eQzE71www6nqoJ6HPIroYLyQ2kYljPmbRkMw611xcVqFVPY+KEJWTDDBBwQw5H4VajkLYjA5J6AYq942uUuPHWtzJsCveynCAgZ3e/fOfxzWdZKZZQFGQBn/AD+VdF9LnJbWxt2cSqo3sNvoD37/AOfatVHjSMucBcBcf3qzbdCccnGcMDjAFTCUzXohUEpGNxHvUFmpHuAVhwVCjHTFSXcwEJ82byov+WsuecDsPxqipmVT5Vu8pJIVVXvz37f55rNm33N5sucXlwDlLOEkonu5H8hSSG2XIppbtCdOQWVmDhrqUZdz7f4+9V9R+0W9jPHZQg5B3TSMHZx32/lWiINSjCrNLcnAwIbWFQq+2WwKeyEhpJrZIoTjcFYyH8QoIFD0Banmu4792fmznNfYPg29ivrCzup1VhcWccvIz8xVSf5mvkjU4Uh1GZYo2ji3ZRW9K+kPA10y+BdFmBPy2qrn6ZH9Kiv/AA20FJXlynpUdnpUNy1wsKCQ+pyB9B0FYHjaSA2VsYlQHzCCQO2KpnVG/vVk61eG4tVBP3XzXjxqyl7tkenHD8r5mzFlbcgJ9KqMMHkZqbdujxzwxqJhkgnp7V0RKkY+pR79K1qHH8Ak/TP9K8jvAQx68gGvXr+5t7aW7iuZNn2m0ZVGCdxGRjj615FdDP1xXXhzixPQ3fAcUtzrclpFII2mixk9DyODXXeAXP8AYk9uesF06fyNeaaXfnTb3z9m8bSpXOOvvXe/Dq7886qACoMqyhc5xuz/AICtK8fdbIw8vfSO5nl8qF5cZ2gnA+ma546zOysTEI2Az+o/oa6CdN9swH8SkUvh3Q9O1Sw8263CQoy/KwHIUY7e1cKR11qnIrnFySXdzA0k1xJsUhsDjgtgj8KhuUMb3ADEtvdGJPUABhXeXnhnT1FxDDLP5TqSBuHB6+lU5vDNlJKxMtxl2RvvL3XB7VsotmKqJq5xovN0xuW5w8bn8sGrtvIp1hMAgmQuM8cMo4rf/wCEQsIoVdZrk713feXjBB/u+1W38O2U85uI3uv4W6jscelDgxqaPPfiDHme0kAxujdSfoQf6muEXqfpXuni3wpYXtrbmSW4yjkcMucEfT2FcSngnS/N2me7yOT8y/8AxNdVHSCTOasnKbcSDwBNmC/hz0ZGA/MH+Qr1DwbP5Pim0ycBwyH8VOP5CuV8J+FbCy1CYRzXJ8yPBDMvYj2rt9K0q2t9Rs7pHnUpKr7WIGOeh4rCtB890b0pr2dj0ukzVgwL6mq7DDEds1VrE3uJ3pjjcpFOoNNCZnkYNMY8r9ammXDn3qCXhM+nNdCOeQ7NGeabmlzVEmdf+HtJ1O5+0XtjFNMAF3tnOKgHhPw+p/5BNr+KZrYpc1XM+4rIoWmh6VYTLPaafbwyqCA6IAQO9aOaaKM0O7Cw/PNDHam7+6Q35GkFOVd6smeoIoA1Ovaiq0VynlJkjO0ZooEfN3hNPM8caSOcb5CPwRq7/wAVjBg6jJPSuA8I3ENr4306adgqDzcn/gBFd/4tnSSS1EZ3oys3mD7tcyT9qjpuvZM5oZ6ClxzzzSd/agkggAcdzmu25xsUc+uK0NL0XUNXkZLG3MrJywBAx+ZrPXOR9K6nwd4gg0O9ka5jdo5EILKRkHtwetZ1JNK6IldbFO+8Jaxptobm6tDHCv3mLL/LNYpUivQPFXjSx1fS2tbaOUMSDliMVwDMcDGCM81MJt7ji27iDPtUkfGKh6il3FZkXI2kHPrVs0RtWlyYFyDyRXtlheq9hAxPWNT+lfPa3iB8Z4xnOa9csbmcWds4lXyRbgGPbyWwMHOfrxXn4qThaUTsow9orHlHxcufM+KEAB4EduP1Jr2tPuivnz4lTGX4kI3B5gGSfpX0HH90VUHzQTZMlyyaHUw0+mmtESyJqjNSt9aiNaIzZa05tt9F9cfpXR765e0bF3F/vCug3D1rOc+VjSuT76N49a5rWo9cluU/s24ijix827r/ACrNWw8Vsfn1WEewH/1qweId9ivZnb7/AHqKVlyCQCScZxXJxaV4i8wtJrClP7oz/hUMui60u1pNcdhnGBnqal4iVthqB2JZMfdH5Vx2pXDPPcRs7bQ5wueOtS2Wk6hb3iTzatJKi9UwcH8zWVqJd7q68uNydx7GnTquV7lKFitqOqQw2CxS5XCrk9hWBLP5beYozjnPrVnVLA3tqiNA0iuoRwQee1ZN1p99Gfs0PlKgACZfaqADGPXrWlOsti5UebUrLq6y3lykJHyxt5nQ8gZ59K57xBLGPE18nmJ5jTyMFzzjcc1fOmXa/aJI7ZVfaTK6TMMnHPb09aZqmhX73CubIPKHDs6yuxcdxkr34/Koc05XNFStGxgz3CQxhmcKMgfUmrdtZTsrT5IjHPJz09MVoRaDqFwwii0RncHcW3noPqOK3bHw1c2EUN9L+7aMEtBuPTIHzevXNKUr6LchwSV2ylo9pIWS6XmGWB9pIw2GQ9q9Q8LljpkzkZ3uj8e6LXBW7zS3KkkbCTnj2ruvBj/8So+nlwkZ/wBwf4VWHvzakT+E3uABkfrVLxA6jwrqe0jCwMx3elXow0jAbWqPU7b7ToN/E6kB4XGR24rvexzLcjs2M2n2kiZIeBGyD7VN5Mh9aqaIyDRrLMoJWBQcnrgYrSE0X/PRP++hUKS7lNMztXnnt4ENrCs1xsGImfbkZAJz9DmuflvAmpqe4XBNdBqnkSiKdHSR49yBlIOM4z/IVyOonFwJB1weSe/FeHiknXZ6VH+EjRuda+z2zTFXbaeQnJrMm8RSTxvGtleHj72zj86qPO3UNgAjgDiopZ2KDkAE9BWMaLTujSUn0NfQxp6awrTXRtnc7sN8i5HPVux9sV3iXEFwqPDPHIhyN0Z3D8xxXkgOXUtxzzn/ADzXdeDsNoSd9kjqR9cH+tenSldWZzVVfU+V/EL+d4m1SVQ2HvJmAI5++etSaUo2tJjvgU7XrKYeJdTjCMQt5MvT/bNWtNj8iIBsLg56dv15ruvocXUvGQW8JckZ6jPpUFrL5VuZHUbnJJ4xiqt1M0zqinjg0XbrHZZ5GFxwvH5j/PNSUPtYLrUIjMZHSI/LtiBy4ye/T2rWs7LUEBgts2UTHnbyx7dBwPxJPqawNAhe4IHnTqozxFIQf0/z1rqINOt3O1p33L1LXJzn8/6VdiTQttIgQK9w0kzDndK5YD8M4FSyfZkct5CvIeV2A4HH+fypsOlhRmO9ugCD92Yn/ParDWskShUuZCxb7zAEgc8cChrQcXqcN4xs7tJLe6m5jYFB7Hr/AJ+le0+C0MXw60odP9F3fqTWP4bsLG+mvdMvoBdw3MQLCbn7p7HqOp5HpXbraQ2mmC0toxFBFF5caD+EAcVlLWNi46TTMSX97JFIZHBjYkBWIDcEYI7j2qO4cshHam7ifb61HIQUPPbNeWlqeu3oUrWFYVlRNx3SFzlieT169PpRPDHMFEqghWDjJ6EdDT1PzNjuOlMkOQMY/GtUZs5bxijfaLB1BJO9eBmvO72Mq75BDbiCDXsUmV1CwfOFEhU8+orzLxZCYNe1CMdBOW4755/rXRRlrY5K8dLnMV2/w1l26tdxf3oN2Pow/wAa4k9TXT+AZvK8UxJ/z1jdOntn+ldNVXgzmpO00eu/wDnINVdJaSy064DDLC42RLkfOTjH06/pVoIxXAB/AVWtrUC2vCrBbiK6E8audu4BVJx+tefF9j1VGEpxU9rl0NdwyiR54pxty8SKV4xzgnr+OM+1JdX8cbWqKoErL8rynCqB3NQf2hHIqJFE5lKqCHG0L1ByemPpmmag8Un2e4ZY5PLQxyIcE9jkA1rdW91lQherFV4pb7WV9NNNt+vXuSxajPHJFE8sEyOCFKKAVPPbJ44p095dz3ptICFUqjbz0XOc8d+lQWt1Y/bVEC20agndJLtTAz2zg5pkNzbx6jI7Xdrt2JgmdMZBPHWknfqVUUIybcdVHrbe66Ly6fh3WczYmiuiHlCLKjAY4yAQR+NYZVxeP844x29zW5qeoaa+oPLJf2qRi1ddxnTG7IwOtc1Jq2nC6Lfb7VgyjlZlP9a1jsr9GZc2suRK8obWW9+i6aK5r2Nw0WpxpHJHGzZG+Too/rXRW17Il0lvLPDOZASkkXGCOcEZP51xMeraUl/DK95ZyISoK+arHr6Z5rr7W905rgNbx28KKOZXCoT7Adfx/nU1HLn1ZdKNL6v7sbq2r039d1bt1Xqd5DrGqatqd7p9h5ECW3ls1zIpbAdAwAXjJznuMD1zWtGsqRhZ5FklH3nVdoJ+nasfwgd4v7r5fKmePy3BBDhY1UkHvyCK23PzsevNEb2uznxMo3UIJJJL77K+vr0G5o4oozVo5ivcLwCKquNyketaEqgxmqB44raD0MZojU7kU+op2ajTIXHoSP1p2a0RmO7ZpSabnijpTAWlBpuaXtTQD81JEeagzUsX3qYiKSCTzGxjGTiitDj3opAfL2kEt4hts54WTv7V1czk4BPTpmuR0bP9vQMOBsk/lXUyEk896ICkya3IL7WHarYSHuCao2+TJgEk+1XAj46H2ok9Rw2HhYegUZpcw9Qg61H5Z/yKBExJyKm5Vh4aMc7eBSechU4UD2xV/SX0+3uJG1Oye6hKEKqSFCG7HispIQzO0ayAFyQGbIC9h0+tSpa2BoV5woJwB+lYWuXZTy8SFA3zBlbkkdq2LiLELbgCMdDzXOakgktzHcE/umOw9wMcVdxWKqahJMqj5yowSw67a9z0+63aRB1H7hPw4FfO+nSOb6FNyqqjvnDY5Fe62EjtpkB3YzEu4Y9hXDjNYo7cGtWeX+PJN/j1TngvD/IV9Hxn5R9K+bPG5H/Cag5B+eH6dBX0ipwox6VUNIIzn8bJSeKYTmkZzioyxA4NWmZtDmqI5pWkIHNMzmtEZskgOLiM/wC0K2d9YcZxKn+8Knvb2+hnCW1iZlIzv3ADNYVk20bUrWNJ35pFbnrWDNqGt5+TSR07yD/GmxX+vNIobTFRe5Jxj9a5/Zyua8ysdEWIFV5g7xbgCQrDOPrVa3nv2+W4tY1U/wAYfp+HeluWPk4B/iXv7iolo7FxV9iUNkVLbRBgM45rLu7ie1t2miha5YkARLxt9T61jR+IvEMMsnk+HpZkz8mZQoH51lUTtcqW1kdNHZRtE4wOJW/9CNedJjUxcy3G0MJWiIXgEAgg9625de8TpE7WuiozOxLrJMP3Z7jqM1n+G9Nv44Ll9WtFhma4LoiOCCCBnoT3qsLCLcl1exzV3Uik1sUbiUWVvJGtqTFIMOWJOOMdhVmw09dfv4Zb2HZDt2jYzAlQDz1rqUsra6kjinhUoTjoeP0rdttMtIJGeKBVZVCkc9uf617MY04Q0Xvdzzm6s5ay0MXTNFigjlWwhWISRABnye+Tk556VDrsMRntEe2V97nO4dcha6uIqqyAYAWUjAHbiqs4XdYvgZJ67cntWdGKg+Z6suabVkzyqVEg1Hyo1VU3A7QOxra8LrG9nH5kzqsdpGcJIVzglecfSsHxPBOnjCQxXAjVZF+XaOAQMgGt7wko+yyEkZ+zsucekrCs5L39Tqh8Gh0cQsi3MsmO/wC9f/GkewsLiF4o2lIIbCmV8nj61WKttDmc4P8AdFXbDYxx57E8gqR14q013RFp9jO8P2Vs+lQymGEyMuHLNgnBNaxt4o48iK3bA6ZGf5VleFPs2meHnEj+VDDJIWaQ9Pmz1/Gp/E+ri10dI7Jw91fkRW+w5zu6sPz/AFFLlpqN2jaKnOVkTzW8QtdyQpGQ4+6Mcc+wrktZXayHjGCMfnXUixfSvDUFrJM0skUa75HbOWzzj2549q5PWWzEG3Z+bgY4/wA815mLjasmlY7KL/dtXuZ74jRjuHr16Cs19TtQOJSefQ81f6q+QMccCqbQxL0ij/ACnDzHK/Q2fBmo6fNr5tbryXiuImAWZRtLDBHX8a9EuJdO0i0YiS2s7cHJwVjX8+K8YvQn2OXaoAC5HHvXI3ajzo8IOntXbSs42OSpfmuQ6w0dxreoTo28PcyOGzncC5Oc/jWPd5jk2eo7++f8K13jw24Dmsi/OL84OTtH8v8AP5VukYsqtzKMduw4rc8N2UOq+KtOs5oPMjLl5I3GVZVUsQfqRWIxO7rkdBzXZ/C61M/iO7vCPkt7baOOjO3H6K1AjD1bw+PDHiSWzZS1pJ+8gdxkOhPT3I6H6Z6Gt20gtZCBshLcDbgcfh/ntXdeLtAOv6KYoVzewHzLbsSe65/2hx6ZA9K880u7geCMXOEcYUOwxn0zx1/wqosRtxWtuhGyBcqO4q04IXCnjH3T69v5UyMEDcGyPUc9qkd9qgswx796sQ/SLr7JrVtIOcuEI9m4/wA/SvRyOMY+teXw7o7iF8FdsgJY8AYP/wCqvTuDg561ianKt8hYZ6ZFMY5BP61NcjF1MBx85x+dV8/l2rzmrM9VO6uQZw3cjHSmHBUenTFOP3sYHfrTTnviqRDKOoRg2qk9FkU5rz3xlCY9auBnOUQjP0HP6GvSL4ZsZeegziuG8dxf6fFJt/1luB+IJrSk/fMK6vE4RgQx7Vf0NpF1uz8p9jtKqhs4xnj+tUXGG5qWymNvfW84/wCWcisPwOa7nqjhjoz1v+y9Tk+9dpj/AHzUEvh/VpgTG+9QMg7gNx9OTXTo2QRheBwR3+tP1G7ksNIe5iVWaMLw3TGQD+ma8DF4ipR5VTSbk7amGdY+tg40o4eKcpysr7fhY42PwtrEkgWVSi4wW8xTj9e/9KjXwrq5YboH+9gnzE6c89fpWqvijUgiyPFaCN3ZFIDZyAcd/WmnxTqYiZ/KtOIRKBsb1xj71LmzD+WP4/5nD7XPv+fdP8f8zJPhbWvK4tW3ccGZPTnv605/DF1CVE+Yyd38QPHbp+tbE/im+huFTyrYoQjZ2t0Jwf4q6DVQDaqD/fH8jUfWcVTqwhWirS7E08xzKji6VDGQjao2tL30+b7nAN4XWeEJNKQSTkL+leeXKFHAbqPl/LivayAeh/SvI/EEAt9Xu4/Sdz+B5H869rDybuj6DEwSSaM3ONhz0/xr1m2lE9rDN2kjVvzANeS5/dj2NeneH5DNoNmxwf3e38iR/SrxC0TIwz1aPZfh/P5vhry+P3M7p+eG/qa6g81wnw2nzDqEGejI/wCYIP8AIV3VZR2NZ7hSZooNWQBORiqUg2sRV2q1wMHNaQZEykOJHHvmnUjcTfVf5f8A66dWtzGwUd6KKYgpe9JR3pgO7VLD96oalhPzU7gW8+9FL+FFAj5b0Yg65DjsjfyrppDn865GzmMWpI43ZweldL9pLMmSMEYIPrSgxTRes3VJCxPAHSr32yIZAf8ASspXyDg8Y61ANQtHufs4uIzKDjbnv6VbimTFtG39rj7Mfpij7ZEBwW/Ks0nnOajaBCGPzZ9mNTyofOyx/bMW5gA5wSTjginwahCqksSN3IGfwrnZEaGZj8yPvB55AGf/AK9RSPtmJZiAEJ2j8z/n3qeVF8zNu51m2kcQNFJmQYDZHX/Irl765ma+eJV2EKVOe/pUck4a9EpI8sZON3I/wqbUVhusPDIPNC5fJOce1TYdzPRWikSWMbgi5yB36V7jpsiHSbYc7jAn54FeO+FHZfElpGxby3cowPQggjGPSvW9PXZYW6josagfTFcOLlsj0MHHdnmvjY48ZBuTiSLH5D/GvpKP/Vr9K+bfGpb/AIS9gBkAxH9BX0jEf3a/QVpH4EYT+NjzTDTz0phqkQyNqQEYpWqEnmtYmTHOcLnuK2ElVERmOQR+fFYU7HymAznBxj6VPBKjRRRXEjRyJ0BON2f/ANdRVsa0jUkvgCoRSMkDGMcGo2vjtJQ89RnvzUH2aBiD575AAGG9KQmCDMKLLKW/hHOKytFmmqLJlzO6gDG0E/WoLj/UHPqD+tWVREBwMZ5P1qC6ZBbSfMOnrXJUfM9DeGgxSdhz60REsCB6muL8Xanf2F80dtdyRoUB2qcYrE8P65qL6wsE1/O6mJiQWOMgZzSlhZVIp3CVVR0O6aUh7oZ6Sn+QqbdlWrhNIvbqe4tmkuGbdI4cySEZ/wAa6jUdVXShHugkuPOO0CHkg/8A16ihRdLExTYq01Og7I37G1NxIgYYBJOSPT0reRNplHuP5CuV0HX2uL2OzlsJ7Z1VnUSnkjFblxfyRXCNHAWRl+Y54ByMf1/KvXdWLdrnmcjSvYtxt+8kj/iL7uB9KrNljbKCQdrYyOhzWdJq0guZop1iiuBP5CMXJUlk3J0H0FV73Vb6LRdOkS3i89pwjsMlV5I+vUUKpFu1w5Xa5yPimBh4ikbndhSeKt+Dm/1qFc4Wccj0lz/Wq+vzyT6lOZivmqxX5RjODVXRdWOkiWYQiT9/MhUtjrtNYynzSOmEeWJ3csalF2JHj1CD8qlslUXCnaoPrtFcqfHmxTjTU+U939PwqNfiJJvJGmwjJ4zIc0KCve5Tm7WsbejRQXOnXdtcwrNC1xKjIRkEZFV9K8Kw6brQvFnlmt4kK2tvLz5JPXB/l9axbDxBNaxBo4YsXN2xIbPy5GeKvy+KbyBj5cUPpyCf61H1qitJbotQqpPlejOm1WTzNMnACnC5OGBPUVwWpKvle24YyOPStQeI7u6geJ44ArqVbauCM1jaq5EJbndxjHbmufEVY1ZpxNaMJQg0ygCOfdRUbjiljKh26fMuRzSN+mO9NIq5VuVLWso/2T2rkLhf3sZI6L255rs5BlGXPb1rjrkESqDkcGumic1YjKZGAM56Vg6mSNSZRgkqua6PGY856egrnNQB/tSQ88AfTgV0GBRnIC9dvFeofCa326NqNwVw0twqjPoqnH6sa8zkGCMfKD2x/hXr3w+j8nw3Cp4LqWJ9TvP9CKVwsdgpwy9uRXn/AI00ZdI8Rfa4UVrbUsvsPRZR94fjnd07t7V3pbdk1neP7E3vhVp1GXs5FnAI/h6N+hz+FVElnA2u0N/o072755Rxx+XT8q0Q8qKWkUE45eLG4fgfxrNgIniXeACRnOMgmrKNOrhAiEKcc5xitBDXaOWTehlfbzuznH1Hb616hZ3Ud5ZxTxMSjL+IPoa8785VHyxp5iDP0/Gun8I3Pn212DhWWQZTGMZHX8f6Vm1Y0WqDUF26hMM9Tn9Kp5x0NX9VGL8nn5lB4qgeD/hXnT0kz1KbvBELdTnpnoDTpZpLiQvK258YJCge3amkgnPtmkwcnJHFJAyKcbreVepKH+Vcb42jEljpc+Mgoy8/Qf4V2xAZSPXiuR8UJv8AC1rJ3Sbbnv8AxCrhpJGVVe6zzJx81N9KknG2RvqajFeieb1PfNMkFxptpPgZlhV/zUVZ1NVOkyLIAVIUEH6isjwjP5/hbTnPJEQT/vkkf0ra1C2lu9MeCEoJGClS+ccEHnH0r5rMWoVKbeykebxHUjCWEqTdkppv00ODnjkaG4VCvkJcAx468/8A6xRAvmW9v/twSRHn05raTwzqccbRLPbbGCk8t94fhVlPDUyMn7yLakrOBz91h06V1fX8P/Ojt/tvLv8An8jk5oWWCzkDs37sscnrgg4r0TU8G2XPTeP5GsGHwjMmBJdK6jO0c4AIII/Wt/Uf+Pdf97+hrixGIp1sRS9m72b/AEPHxmNw+KzLCewmpWcr287GP+Oea8z8ZweXrcxHR1R/02/0r07GTx+tcD49hK31vLjiSFl/75Of617VB2mfWYhe4cUPukV6L4Pk3+HkH/PORl/r/WvOh3HtXc+BZd1hdxf3JA2PqP8A61dNde4cuHfvnrHw7l8vW7iI9JIDj6hh/ia9KryXwdN5Hiq05wH3IefVT/XFesmuam9DpqLUSiikzWpkLmophlM1JTSMiqW5LM2biSJvcj9KUmi7+WPOPusD+tJn1rZGLHUmaTNANUIXPvS5pmfegGmIkBqWH79QA1LCfm4oAuc+hopRnFFAHyhCSbxSp5x9e9bZmVVAz83c1hxuyyhgw5XH6mp0bfKN74UH7o7+1ZxdmVJaG7HLGVkkeQRqq7dxPHSuRtIjJqsUKFT+8A3DuB3/AK1om7lvNTMNsodEjwUYfL78VTv0khnN2MRy5+6Bjacdqpzu7EqNlc7npyaryySZaOMruxkZ9K5/TfEMly4iuiAw/iA+8Pp61duLyOOZZISXK9eOorTmRnykOpaqRJ5JA3Ac4+lVbe9jMCb+vzJuPoelZ125nupJsbi397qBUOW25xnHQnrWbZokW7cK8o3Hcdwx71ZvjbRSKpYI4GAQP1zVLciWyyLjfnB56VUnYvJ8zKx6Ag0rhY1fDUoXxPYAc5lA/WvZLQbII19FxxXimhOYvEFhnODOmRntkV7bEOg968/F7o9PBfCzzDxuWPiuTCkgGLPtwK+j4v8AVL9K+dfGeT4nuz2URf0/rX0VAf3Kf7o/lW0fgRzT+NjzTDTzTTxVIhkTdDULdamaoW4NaoyY1jhc1pKsc2BIisOwIzWROcQsR2FU4ddmiVVKoxAHzHvXPXjJtOJ00GrO50j28MRBjjVSeuKntnVkIB5Bwa5qTXpJf4UHvTRrtwnzIEHGOlRKMmjXSx1E8XmW8iZxuUj9K4H+zxEwkMrnaQx9Mg1r/wBv3zAgunPoorMvZWe3l5IyhyPwqqKcE0yJK+oviHQ7nXNQFzYyRyW/l7GIIPzAn396wtP8K3q6xIEb94sbocrgcjHB6HrW/aXcsEJiicpHn7q9KmgkdrhRvbJPXNQueKsi3GMtWc5H4furS5jRpV3W025/cHB4rb1ViYbU+k8Z/WqhmmubqddrbwQDjOTjirM8dxNawxLbTFw6ktt44wa5MSpNqTN6XKk0jYQqnjS1VOjQMP0Fb24SJKI1xu28D6kf0rDNpcHxTp90qZiRCrHIGDtrVtJyhQEZDlhnI4wzH+tKlK0TCqrswdUtrs3PmRwu5F7aSdugXBP4Vr3cEsmmRReWQ6XKvjPbexpup3MqiGSKJWRmXeS2MY6VK+pwsZQ7AZAAxz3P+NJ17MI0W7NHFeJrmO01W5J5DysQV+tc7JeGCFwIy4kuj3xjMamtLxSHmuyLckqGzzWNcRs9tt25YTxk+2Y8f0rqoyco3InHldileahIiP5ahTjIzz2p3h67fUnmSYgNFAZFYDGWBH6U24snlJxG+QMcNx/Km+ErGcXwJOxJImXPU4/H6VtUuqbaM4fGkzrrmFIdK050z80ysxz3NF5wSc1LcxCTQIVyw8puCPUGqV3C5JJuJT+IrxoPmep6LVkTWshLcelN1KZjCwPIx2HQ1VtITvGXlwOnzGpdQfMQzngHp3/GuunExk9DJW4c3G3zGwM459O1Pa4dsjODUJ/4+uSMY4pDxx6etdyijm5mOMzc5bH0Nc/qKbJVZR1Tp71ud+v0rH1YHEX0I46da1grMibuismBGoBNc/qAB1KXrggcg4ro1yqrnoRxXPajgag/QcDmtWZIqMp3fd5/+t9P8+9eveCFK6LCpGMRkf8AjqH+ea8lWCSadYoxmR22qB3Y8D+Yr27R7AWEstqBkRTvH6ZGGH/sorJzSkkaKN02agxjseK0xGl1YNDIuUkTa34is0D0A9K07Y5t0+mK2iYs8cigk06+u7Fwd0Em3Dd17H8qsm9hVcSAgZ6+lbvjjTIotWt75lxHcjypCOzDkGuYeDyiEuCSOit0H41YjQheORD5ZByOvfmrfhG5eDX/ACGbIlVom+o+YH9DXP8AlPaN5sTBoz1xnHX/APXWno86w+JLOccJI65x78VEy4nZ60NtxEx4ymPyP/16zG6571s62uVgb0JH8qx8ccE151bSbPToa00Rt9Oc03HrmnsOOvNGMeo+tSmaDAOn5ZrmtejB8J3yYz5U+QP+BD/GunxyOBWJqUHmaTrcPUgb+fpn+lVF6kTWh5BdjErdeeeagq1eL8wPHSqvSvSjseU9z134eT+b4VROvlTOv8m/rXXs7KSATjrXA/C2Xdp+oQf3JUf8wR/7LXfspyO/FeZiIRlJxkrneqFLEUVGrFSXZpNfiQGeYZ+bj6VGbqcfx/oKey4OOaYVIHNc/wBWofyL7kY/2TgP+fEP/AV/kBup/wDnp/46KglkklwXYsQOlPK9vTtSMMg8Y/CqjRpQd4xSfoaUsvwlGXPSpRi+6ik/wRBs+bn17VyHjy2zY2svJ2SlMgf3h/8AY12eBnNYPjS387w9O/8AFE6OPzx/ImuilK00bVY3gzyNeG+vFdZ4DkxfXcWfvRhseuD/APXrlH+WZsdmroPBcnl+JET/AJ6Iy/pn+ld9XWDPOpO00en6VMLbWLKbP3JlJ+mRXtJrw7aR069q9rt5RPaQyjpIit+YzXFSZ21US0neig1uYCUUGkpoRTvEzG49Qarhtyg4681dnGVzWdF/qlGfu8flW0djGS1JAaXOKbmkJqyR2aQH0pufWlBpiH5qaE/MKr5qWE/NQBezRTN3uaKAPlNB82R1x+nNQTSFJjtJ56VKcK/YnA7/AFqrPkScDr2rBblvYu+FSDqU565j9M96u+KrZIoIZQW3yOcgnI6VH4RjIvLo/wAOwDP41P4yf91Zjp8zE/pWTb9tY1SXsrnLRErIpGeozWoZtpAOQOo7ZrJRix2g5HSrIDvjBwK6DAmkbcd3c9xxUZ9uR2pQUWMgsdwHBIqIkgkn0xSHYfyIGwTgt0pgVQvyvkjHapVcCzmB67lwBV/w3ZQX97KlwhZBHuABI5yKG7IErsj02M/25Y4wcTocg+4r3FB8+PevP4NIsraQSQQbWUhgdxPP51tpqd4X/wBceT1wK468HUasd+HmqaaZyvjg7fE1yM8FYuPXpX0Xbn9xH/uivmTxZNJLrweR95JQH34r6Ztjm3iP+wP5VqlaKRzyd5tkppp4FOphpolkbmoHNTP0qu55rVGbIZz+6ce1c2G4ArfvJRHCTjPIFc734qJmtIlJI5pysSvWmqpIB7GrbWojtY5/NibecbFbLr9RWdzdEAOB1z+NJPloJf8AdP8AKpdgyaSQAwyAcfKaVxjLY9fTg1dtx/pKEcYOao2pG3rngVZViuGB5HepbGiRWEWrXTdBhT+prQe7RIVkVgVCcgfSsKck3RY8koMmpgf+Ja3rsNcWK+G5rSjqbkuqRRmAOjM2SODjqpFYmk6nctazxmQ4SdwvrinXZz9mI/vrn9aoaScfbVOeLhv6VyKT5GaOKTNZpWf77Ej3NKjfI3HcVApyDjFSIf3b/h/OsJGsTC1cj7UxPqMflWXNuBl3DB3wsf8Ax4cVf1rP2rjptFZVz/x5yN14iOQf9s/417OH/hI8+svfZO63BncW7qpOM5I96r+Hi8d/CjnlS6/hk0fOJSdpI2jPFR6JJnVFIGP3rD8a2m06bRlGL50zq350eYdg7/1qldf6sHjp1q2Du027XPSQ8fhWfO+YEPByorx4LU9GWxFAxDrzTr/m33HHvUEZ/e8c1NqLYsumSPSu6ktTmlsZJ/4+0AB6AfpSE8jvQpH22A5+8vf8RQRg12o5xuPzNZ2qLuto8cnzMfzrRI6f4VVvIw8Q4GBJyPqP/wBVXHclmbKwSaNQeg6VzeqZF8w9gc+3Stu6cnUSOCF4rH1hP9M99oOBnPOe9avYzRt/D6G0m8baabsqIY98vzkABkUsufxUV6/Cq/2jdMGVl+1ggg8fNj/4uvFPCiebqUTdGjjY4PfjH/s1epaZfbjNGcAq8bjA6fdH9K46lN86nc3pyVuU3JU24IHOecmrVo2YBzyCc0y6Q+SSMDvTbJvkYdwxrqpO5jVVin4ssP7Q8P3CquZIV81Pw5/lXn0Mhmt1fILYwQ3evWCQRzjB7GvLb2xOk6rc2nQRtujbHVDyPrx+orYxKhmjQlXiK88nGcVWeKVJY3t+YQd6sD90/wCFaG5JcjK+Z2HWs6azMbF1ymeu3pSkrlJ2PR7qYXmkW1wMfPhuvqKzdpAxg9aXQ7gXPhhYurwPtbn3qRlGfb0rzMTpM9XC6wISuRgUm3FTmPrQE7dM1imbtEO056Vmyw+ZdahARxLb/wBCK19v0zVGRdmtRekkBXn2P/16q5LieHXq7dvUkZHIqiOtbGtwmG8uI+PkmZcfQkVj16sHeJ481aR6D8K5calqEH9+JX/I4/8AZq9SK5UV8+aVqs+kzPNbyzRSMu3dE+049Oh9BWm/jLU363d6R/19MP5AVzVsPKc7o66GJjCHKz2p0Oc4596jZdvJwP0rw9/E1+5y0szH/auJD/7NVaTWJ5fvojf7xZv5moWFl3LeMj2PcXnt0ALTxJ/vOBVeTU9NjGHv7Ve3Myj+teLyalEbRES2VbgH5pCFIP4bf61B/aNz2ZB9I1H9KpYTzIeM8j2Zte0dcZ1O1z3/AHoP8qydd1vSLrRry2jv43kkhYIqgnLY47V5d/aN3jAncfQ4ppvro9biX/vs1ccMk73Ili21axHOMSng+vNaXh2UReJLF+xlC/nx/WspmZzliST3Jqe1mNvd284PKOrfka6JK8bHLF2kme1MmQa9T8OTi48PWTZziPZ/3zx/SvNPLDDOcjHUeld94NkLaGYz/wAs5WGPrz/U15tJ+9Y9StH3bnQe9BozSV1HIFJ3pab05poQjjK1kj5ZZV6YbP51r/pWTcDZesP7yg/lmtYETQmaM/rTM8+9GeelamI/NGaZnrRnikBIG5qWEjOar55qSI/NTAu8eg/Kio93vRQB8tAAsOnAH51RuOH5I6ntWpasPMO6AP8AKMZ5xxWbcbWujjAOe/FYLct7EcVzJAG8tyMgZ59Kjubye6I86Vn29AWJxU/2fZA0jMG47cjFQ2kayzEODgDoKvTcnXYZDgHkc9sVZBz/ABAYBqCMKtztk+6M5H51GzHv1oAlEueAeBxT/KmcFlQkeuKhwPJJ29elWtPdlkC7uGU8delIYscL/ZLguhRhjBNa/gsMfEttBjcs58t156Z7VUkWaWJo+AQAAemMVueAYHtvGWnM5XaJVzj61nXdqbZpR+NHqM3hiKF9rxSxHH3W44qr/wAI6suoWVrAfLa4fZubJAOK9C1Ux6hfebFkrtAGRXO6yH0ybT71BlopwwX14PFfPUsTP2/IpXR7cuSVH4UpHjfxG0ebQ/F8ljK6yOnltlRwQVBr6ItD/osX+4v8q+ffinqx1rxlLeGIRDbGuz7xwBXv9mf9Dh6/cX+Ve/8AZR4tmm7lhjTT+tITTSaaBiOeKrOeanY1XfvWiMmVLtN8X4g81gqAev8AKtu9k2RjgHLAVjICp3YzWdTQ2o7EqdAo9etSqjbwnGSRj5qiXJOelSRnafvVm2bpMnuraaznMUu0sADlTkcjNQOGdWABzg1J5jFSqqc+wp3kTMpKo+cdlNRzIqzK1pEwRQRj5RWhbWkczsslwsIC5DMM59qrWtncNGhKS/dHOw09jFExDSxqRwQ0gB/WldMaTsVZyBOvP8J5/Gnr81g4P901Wuru3julY3EG1UIYiVTjp6H9KZ/bGkLYyltUtlk2sRGWySe1c+Ii3HQ0pySerLl02be2bnqn86pabk3F8v8A02z9OBV+6wbC0eBhIjojBsY4NZsc00N1deVsJLgnP0rlhTvFo1k9TTAYdjipY4meOU5xhc4/EVz+p3up2DAOyjcNwChTVO28TXFs0r3TGSMoV2qVBzx71X1STV0Z+3inZl/VYibpCe4FZl9+4t58Y4jU5+jj/GoLnxTDNOGe2k24xjeAc/lStdpqFm8iRtGrQyrjcG5BU13UqbjBJnPOcZSdgvtVNteCPk+aoT86r6P8t4zEYb7TwB2HGf51dAhBQYzlc8ke31qvYskd7OSAp80EEuB2HtW3KuVmfM7nRo4FveqSB84Izx1FUGBktkCkE7RxkdatPc6eiTI01uFZsjfKO4x7VWGsabBbxobq1TjAIk5/nXmqlK+iO1zjbcrRxSCbkY9TVy8gMloVJHIqm+u6exz9uRl65BJFPn8QWzxLBHcE5wANpGa6YQknsYylG25nsm237FgMAgdOTSMjtwEc/QGo7i9eGAlZHUNnoT/nvVOXUXCbv3zjHAGST9BmupXOe6NDyJSP9RJx/sGo3hkEblkOFwfpUU9nrsMSStoV2UkfYvzR/ewTz83HA6nHb1GY47XW97M+kCKFf9Y73kOVHrtDZPXoKtRkS5ROckbN+4yOvT8ao6mN12CTj92uf1qyj/6TcNnjOMe1Vr8/6WCBxsUdOnFaS2M1uXPCnGsCPcE+R8lv6/lXdWcqx30qiVctEMYzg43H+n6VwWivs1hGVsZVsflXT2UxGswAnPmIy8+pB/xqLXRXNZno9xqOV8kIMYAz36ClsHBLkdQRWTvDCNvVFOc+wq9p0gzIMY4/CtYxS2M5TctzWDd647xvbBJbO+GBkGFzj8V/m3511m/rg96xfFUP2jQZ8DmMiQEe1WQeeTwrIRtJR/Ud6h+1X1kv79RcQdyPvgf1q5EFfAI+bqKdNExj2+WzA8jAzQM1/DF5E8k0UbZjnj3LjsRiuj8kkZ/HmvMtPlm0/URKhKmN8snOCDXQXPxU0y0keJtPu2kX7wJRRn8zXnYqnOck4o9LCVYQi1JnXeT2xWbql21lIgVM7hnJrlZPjFEufK0j/vu4x/JTWXf/ABTlu3U/2ZafKCBudm/9lFYRw1bsdEsVR7nUnW5t2FRaal/NcahbMsabwGA3OEHPqTxXES/Ee5cD/iXWXHTKE4/WoT8Rb7IK2FgOf+eJ/wAa0+rVX0MniqXczvEx36vfHABMzEgHOCTXPnir2p6rJql7LdTRojSYysY2qMDHA/CqLHJr0KcXGKTPNqSTk2htFLShSegNaEDaKsJY3cuPLtpnz/dQmrSeHtZlGY9KvWHqIG/wpcyHyvsZtFbS+Etfb/mFXC/767f51YTwN4gfraRJ/v3MS/zapdSC6jVOb6HO0V1afD7WGIDy2Mef71yp/lmp1+Hd4eX1TTQP9lpG/wDZKXtqfcr2NTscbTuw+tdbP4CnjUeVqNvK2eR5bgY+uKr3HhC6RI/LmEjYwwEZGPTk9aXtoPqHsZrodLbfEKwjsYI3tJ2kWNVY71GSAM969G+FnimDxAupwxQtEYDG2GbOc7h/QV4gvg2/ZiDJEo7bs5P6V6H8KrM+FtcunvbyDyLmDYcbuGDAjt9aw5KSd47nQp1XpLY9ypPrVEaxYH/l4A/4Cf8ACl/tWxx/x8KPwP8AhTEXKQ/pVM6tY8j7Sn60f2rYn/l5SqQi3WZqOEuYG/vBl/kam/tewPH2qPNZuq6xp5hiYXkOVkA5bHtVRepEloP3UFuKzTrOm9r63P0kBph13Txx9oz3+VGP8hW9zGxqbuKNxrHbxDZAZHnt9IW/wqNvEkGCVtbyT2WMf1NF0FjcDd81JE3Nc2fEeMFdPuQP9squP1po8TSBgFtEU/7c4B/QGi6FY67efSiuS/4Sx+8EQP8A11P/AMTRRdDseKWdwkMp8xgPu8fhVK8EJkdhED1P3jyaswWq3O4uXBGOnPapDZpvO4k4HNZJFPYzUbOnscEY3AYFVLd3VmKDn1roRZRG3CKp+Yk4GRVOytEW4YbSGA7/AFp3FYyVDSTdDuNGGXIYYPQj3rZS2j/tEjYD82ee/f1qGWONZWDIu7djkd6XMOxlhn28ntUtpv8AtEeDwDV8AKAFRP8AvkVbt7hoSfuEHr2PfilzD5SNblScbWyT2Wtvw5qEGna7ZX0wlaJH3MFTJxVeGWCRvmYgn3xmux8NarPDe29tbR2qbmA8w26lsfU81hiJvkasdFCF5LU9t0O2i1LTIL6NyscyB1VlwQD607W/Cy6taxxCcRbH3Ftmc8fWn6Pe7bVS4lkkxywTg/lxWk2oNt+W1lbjtgfzNeZhI4XlTa1HVnVVR6nyn8RrX7F4jdQ24HacgfWvoKzYfY4f+ua/yFfP/wAUJQ3ix9qbBtTAJGRjNe+WjD7HAc/8sl/kK9Ond04tk1LKoy3u96YWpm73pCwFaIyYMetQOacz1Cz5zzWqM2UNSbEIxnO5f51jHXY0BXzLQHpjykzn8a1tQO6EYHRgf1rz99Hv3uZWWW3VWckbpO2fQCsqtOMtzSjNx2Oo/wCEkWEHN1EuB1WNB/Sqb+NX+z/aI31CWL5cPHauAc424OwDnIx9ayV8OXMoxJd2q/8AAm/+Jq7NpWqz6RJp0uu28ccihS8Nud3BzkHcPf8AM+2Mo0aXU2dWp0HP4v1F2wuma63uYig/UioW8Sa/OSINFuT2/e30aEk+xaov+EZuSSX8V3oJ5PlW6J7djUi+E7aQf6R4i1qXHf7QQPp1rT2dFdCPaVWWbJvE93p32mG305FVjEQ92ZPmBx1UY6+/Y+2eMaaa4mlld9plbeVVlIBPXB5rtofDGk21r5CajqTRlixXzlwSeST8vJOBQnhnQIlwEuCOgDS4/LAFNSpr4UJqct2clGzCGcBiTlDk49/QVj3Clrh2LANjByf/AK1ep6VpfhZNRmW+jZbbyQRukc5YH2NazWvw2Ry7WSyN1OVlP8zWkUmrmUrp2JfA+m22t+C7SS5mCXMI8t9jbgAOV/Qitt/B2mM5K3kyM33irDn8xWTYeI/BmhwtFpljJBG5yyxQgAn15NSyfEHRFGUsJ2+oUf1rklhfeumbKtKwl18NdPvZg8+tXZA4UAIMfjiqr/CbR9oC6pet774x/wCyU9viTYg/utLcjHeQD+lQt8S8/wCr01R/vSZ/pVqg0rcxDk272Jrj4TaP5CmO8uvMA5JZev021T1LwRDp95p+mae8rxXMU6tLINwRioxnA46fpSr8R7qSUIba3jBP3iGbH6iqs3jm/uLttogDWu6RCE6/KQep96fsrfaYKUuwJ8JLpY1YavBnHQxNx+tQD4YzwxzRG2s7mVkYCc5+8e/Iz/8Ar9hTrj4gaqdPSWG+j84ttMYhX8+lZr+OfEUh/wCP9lH+zGoz+laqNupOo8fDXW/LSMabpQ28hmcYOevGw1ZX4deIDs/daJHtAGMNxj6KKy38VeIZM51G7/A4/lVdtc1yb719eHjp5rVS5RWZ1dt8PtcjtDBJf6eMuXJWNzjJzjt+ue3oKZF8L2iKtNq6DbjpD/8AZVx0uoagx/e3Up/3pT/WohqUaN+/vIlwOjTjr+JobXYFHzPRR4E0+QiOXVcgZ4VF/rmkbwDoqId+pyA9nREUj3z61xdneARiWK8tWDDI2XKFiP8AdBz+lWk1gyDG4nH4VyVKdSUrxdkdEHFKzOr1Hw/pd7bpBdeIdUeOKRZUEUsaEOM4OQmR19axp/C/h1EdvtmsXEiglfNvnIz1HHGeaz/7UiVfm3hvp/8AXqE6vHuxuU+xbH9KcadTrJjvDseeRALcSjcS2efc1XvXD3rRt90oDn8KnusW+rTLt6sep61Q1QSNcKyH5vKDemOtdclocy3Og8JtFFrsXmxJI4V9odQwPyk9DXaXFxiWNlhgQK44RAv9K8z0W+c38DptEofad+SPyFdhLdXci7VktgeuPLIz26/56VHKXzHVW19iGNWONqAdeDj8KswapGkhDEkAc1xaajfRgb7cSdv3bA4/PFTQam6s3n29yBjgCIt/LPNF5ISUWds2t24zgN7cVXutZjlgli8pmDKR17Vy39pMwOywvmz0xEB/Mimtf3WPl0q8PPU+WM/+PUc8h8kTMkLxnEYO5eBgZzT49WkjTyriIo54DckUeY4nPmQGFm5EbEE/pU6r5kWx4xjoAetbLVGT3MRS6agxkfezrnIrM1Hw7canehrQxA7fn3naBW3qf2a1aNI87xyOc4H9Kl0x2+1IAVG/16fSs5trY0ik9Gc0ngPUm63FqD6byf6VYT4fXZOHvYQc4+VSa9GWxcjJkUZ54Gak+wrnlj+Cmuf20zo9jA88T4ergF9Rb/gMP/16nXwBabRm6uGPsqiu+FmoGSzdc9BSC2Azwef0pe1n3H7KHY4EeBLRTnfcMPQ4/pU0fhS0Tbm0Q7fUnn6+td0LZM/cOR71ILaMZ+Xv3GaPaT7h7OBy0OmWi4/0C1H0t0J/9BrWt4oEGANnsihf5CtX7OgPTt6dacsC+1Zyuy4pIpqtv/fnP/bQ1IYLVh8ySt9STVry0zkNijCjuOveocWaqSKotLMcfZ//AB0Cni2gH3YAB9Km3RgffUeuMc00vD180D6Cl7NhzoaY0UbVQAdBgZphQHG1fbpUhniH8ZI/Hio2uoxxhuKpU2S6iGNBuyQDmo2tScYH6083gQ5CMf8AgVNOoNjAVfzqvZMn2iIzYMejAe1C6U28MHj4PTBpTfydemenFNe8lI+8fyq1SZDqI1/7R1UHPmwkD/ZPFB1jVQBiWLPrtrFN1L0MrY9jTDMxH38+1aKLM+ZG3/bmqqR/x74/2lpT4lvlxvWDv3x/SsFpMk+tRscHp0q1ElyN+TxVdbRtSLPc9f6VQufEN5PBJHiPcRxlAVzn0xWYd3Ixj8ab0yOozVKJDY86vqWF2i3Q+giFRtquqk8TqpPpGKaffPPvTDgdyR2qyCT7dqJ63sn/AAEAVG810+N17dYHOFkxn8qTngdqTBDHrn2HWgBpaTOTNcMenzSsf60RxRkkmPNBJxxnNKjMDnJ/xoEWCiA48tPzopN3+0n45ooGchYRuN4ELtk8YUnsOake3umYlbeXHXiM/wCFVGupshY7hwv9xHOOPoaasd1OSAk7/RGP8qzuVYurb3anJikH1XH86rPaSoxkURqck4MijNTxaHqc7jytMvJDg/dgY/0q6nhLX3+5ot98uM5gI/nS549w5W9kZT2rM+8TxK/Gcyr/AENLJbpKo3XUAbuQSc/kK2Y/AviiYYXRLoAdNwC4/Mir9t8OPFbx7DpjRg8ndOgB/DNJ1IdxqEuxyb2USjDXsZ7nAb/Co2SBOPPc9jhPeu6Pwm8TTEEm2UDHDzfn0BqzH8GdZfHmX9nGPYsf6UvbU11H7OfY87DxqoxJLx0+Uf41ct9ensbhJYXlDqcqdoxn15zmvQ0+C84wZdWiB/2Yyf51Bd/CK3soTJca6evCJb8n25Y/yqXXpPRsuNOqtUcz/wALQ8SIu1L6cJ327Bj/AMcqnd/ELxDdDbJf3LAE4/e7f/QQK6e18C6LAM3C3N0e4d9qn8FFWbbwXoMGXFgWBP3ZHLflWa+rx2iXyVnuzzZft3ibVLa2jQS3c8ixxlnPzE8AEsenNfTttZ3iW0MZt23IiqTuHUD61wWi6Ho+n61Z3wsoLcQzLIZSSoQA9euMfWvRG8Y+HV66zZf8BlB/lXTTcai7GFRSg9RPsl3n/U8/7w/xoNheEfcT8WqNvG3h7+HUkfPTZG7fyFMPjXRz9yS5k/3bWT+q1pywXUyvJ9CU6XeN/wA8h/wI/wCFL/Ydw3WaMfgTVY+M7Ij5LPUG+kGP5kU0+NIsZTSr856FvLGf/Hqd4LqHLN9CeXw28y7Xux68R/8A164bU9Mn07Vp7ZN8wQgh8AZyM/1rrm8ZTEDy9IkOf79wq/yzXnviXSb/AF3W7jVRFHbrPtVohKWwVAAOcDOcVE5QfUuEZroaAiuOP9Wp9DKo/rUi2kzjLXFuvsZgf5Vzy+H9QQAG5bjgYkNTL4cu3iLtOGK9RvJ49az9zuae/wBjb+xEH5ry3A9mY/yFO+zWy8tqcS/8Ab/CsD/hHWb70gB+nNVrrw/MGJheWSTHKqcA/T3p3gK0jqAulqCG1dST2VP8TTWuNGjB3X0zEf3UX/4quGNpg9HwDg5JyKetkCxLJj1yenpT90NTorrVNKWTdFcygEYO+Pr7YArHm123BISO4cDjOEGfzYVGltED/qx754pTaxHG1RjuO4ouhcrIf7ZVvu2s5Yf3njH8mNMbWXI4tI/+B3BGPyU1bW3QKSFTPY5wR9acscYyGWLHfAxTug5WZ51a8PKWtqPbzXb/ANkFNGqai+Ni2oxzxE7H9WFagii6jaQPanGMZAO1snqKLoOVmW19qbjImjU+q2vb8XNNSfV1+ZbqQEgg4ijHB6jkGtbAA+V8jOOtBEXzDawAHXGRRcOUycai4IN1cehwY1/UJTTbXn/Pa6PPU3DD+RFbBCkABgf0FMJRD1246dqLi5UZTWk7cGSXb/tTyHn/AL6qMaSPvOsbAf38tn8ya2TLHtyxU8dSMU0XNvu9B/tDii7CyMoaMp/5Z23XtCuf5U9dOEX3QijvtjUfyFaX2iM8htwz6f5/pUZuACCPMB6c07sLIqCzZBjz5eewfrTTY9N0kxzzy5qwbpvvY3foab9o6kq6kehzijUWhXbToyQWyT9T1oFjGMgJjrxjtU5mYgkAMuMYqMyOq4XaTjgEU7MLoyNZXZOsigcgcjiq1zKf9HcE4KlSfx/+vWjqCm4WPAyQSCQMdqh/seS6hxGwUqwNVbQm+pn6Zi28QW0hQMm8NhuMgV6BHq9kOXhKZ6fICP0Oa4CzVmu4XBKvGSCQcHoa0y8pY7Cxxzjdzio5blKVjvINTsZANtzEi+jDZ/6EKvJ5c3MciOMdFw1eYszFMuQA3TcMdPbH+f5AQnoMls4K9e3/ANapdLzKVXyPUBB04Geh4x/KlMQ7uRk9j7//AK683jl1NQTFPccDkCXgfhmrSatq0BCs8RGesi4z+Q9/88UvZj9qjptXtFco6MN49SCazV3DAyS3QgDpVWPW5JiFkiRsn76yqMc9duauq4ckrjDelbQVlYyk7u5lXFs53SuuSWznGTgUkLmJ0cLkqR+daDxyzI3I8vPUmsxh84HUA459+DUtFpndWd9DJbLlSSo9vzqc3ic4Un64rltIuT5fl547Dv7Vq+Z6gfXHFZezRr7Rmgb0dPKH50z7UzchVFUgCc8Nj1FKCccN+lHIg52WjdSZPAHvTDcS55fH/AagBYDkA5pTu6YHsQafIhc7HmZzkFjzTTI5wcyelNwc45A9xS49iR9aOVBzMbvYg53E+5puSwz2+tSbVPGPyNIU9v60cocxCcnq/PsaAWAGM/iKkKkHr+femFD1x+tOwrjcnOdx/Kk3Mc808fLzs56UhKt1Bz9KLCuMJyOnNIZD6Z7fhUoCeoGaQKo6enpTsFxnnMANy8Uhkx1zmpCRkDjPrTDjOBjPc4osK4wv9cCjzBtyadgZ4x+ApCOhBp2FcQspHQflSHYR/wDWpGBPORjr1pjRtnIwDTsK47CnI/rTSFYHH4ZNG1wSSw+opuGxyRQAbQO4pMY79O1O+fkYXIpp4HKj3GKAGEnPXNN6cZp5dRwy8UgZTjhjx1AouKxGT16nj+7TlPPQVIFB+7ux6kdakSAkYAwPei4+UZweSoz3oqyIBjon50UrhynuS6baIoCW8KgdAIxUq26KOAPyoorwzruwMYz6UGEdyaKKpId2MKKvbNIQBziiinYpMawA7Ckz8ucCiikWivcSmKFpAAcDpXJXfmXkvmTSE+gAwBRRVQLKotELgEnmka3QnJzx0GelFFaCEFvHJlWG4Hg7uc05raCKRkjiRdpx90dqKKuLJaFVeOMDj0pzrlgvoBRRVXJsNIwcdef/AK/9KcFDQqfRiBRRTQhgQc/TPSpY4wxZSeNpP5DP9KKKYEbQqDmiNFWUMowR685oopiJprONHUrwrAMBjpntUX2JGLAnHuByKKKAMLWtNh+xm/GRIrbXH9/DYz7H/PNc8QoQfKPmFFFbQ2MZ7kTSqqL+7yGbHJ6d89PaohdssYIHV9vPWiitDO48XLsyjAzgc49aikuZE2HIw3oMf56UUVVhNjGupd4w3zA9SBTlkl8zy3kZgRu9KKKdibitlGDFi3zAfnTXLB1+dvm6c9BRRTsIYPmjU9MilWQgKjcljjI4oooAcFBbbjBx1HHShY8fxHrRRSExTCNwwSPcUzPcE9e5oopgICgbaVzu4zmo5ZvLYkAk56k0UUAQi4EhB8sA+uf8+tX4oBMuchfoooopiHz2Bgi81pA4MgGNuOv40qw7OAx4IwaKKaEzmvs4jvkIPRwDx15rtI9Gs5H2BWB+ox0z2AoorNFg+l2sEgUK5I7iRhn8M0q2NqVJMKt7P8386KKoBBa2SkD7HFknHCj/AAqw1pHBGHVI+fRMUUUhiKSwHPSs9iTOccZOOlFFVEmRnajq8ttcfY40DEgtvkYnuewx6VjnVyzyLLArFOMo23P4HNFFTLcqOxc0bWfOvPJWDZwDnfn+ldpgEDOegPBoopMaF8sEbgSOM/pRgjox/GiigBQD83P0oVNx4JHP1oooADEVwd36VEWIbk55xzRRQAhcggHBpyNuKr3PeiigCSaMp82/PPp9f8KgBJJ59B0oopgByWIzQQP1oooADCQCd34YqPb8x6flRRSAaF3AnOMUuzgc0UUANIK5weBTkBIzu+mRRRQBKIc9WB/Cl+zr3J/Ciikxirao4JBYY9TmpfsaIcZzgUUUmxpCC3Ut/DknGdtNZAoyMD0oopXHYYBwGBxnNOSPe2CR6dKKKAQqxK2RgevSnLCg5IP4UUUhjjEmT8tFFFAH/9k=" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAAmAHgDASIAAhEBAxEB/8QAHAAAAgIDAQEAAAAAAAAAAAAAAAcFBgIECAMB/8QANhAAAQIFAwIFAgQEBwAAAAAAAQIDAAQFBhESITEHQRMiUXGBFGEWocHwCCMykSQlQmKy0eH/xAAYAQADAQEAAAAAAAAAAAAAAAACAwQBAP/EACURAAICAQQCAgIDAAAAAAAAAAECABEDBBIhMRNRBUEyYXGhwf/aAAwDAQACEQMRAD8A7evC6a5I1xUrTp4y7TaEpwEIVqJ3z5gfXHxG5VL6elbZTVmC19SShkpUNkuFQSdvnI9xEBe7Z/EbhxspCTz8fpCzp9+0+7rMVVKa55Wqu3JzTQ2LT7E6Gnkn2KT8YPBhDP4tUynqpYuPy6dWHdx/Wjc09WpR1yoeD4iFgJ0J05Tj0ye8TU9UZeQlVzc28lplpJWtR4AEL+0FFmYdaRsMJOAe28ZdU6g+xQGWWlkJfmEJXjuACcfkIPSkvh3HuL1KhMu0dT5M9bKXLzCW1UwpbcVpbL00ltaz6BOCM/bJi3W5edIuZpapFa0OtYLjLgwpOe+2xHsffEcl9Ruikp1TlzUFOutzkhLFuVcSo4bVnV/TnGCcZ9QB6DF76Jz89Iu0MTswVzBlRLTKlZytQRg5PfzJBha+QIMjHgmobKm4ovYE6AuK6KPa8impVqdTLsLcS0klJUVLVwkAbngn2BjKjXTRbgl3JmkTqX0NKCHPIpJScZ4UAfn39IrHUi3TeFmzlKaUQ/p8aXVnh1Jyn4yMfMKHp1ek1Sm3ZqYbd1BCmJtnBCg4jgY7HP8AyMVZMYRBk+r5/wAkyW7FPv6nRP4hoyZ0yDtWlETOQnwVPJCsngYJzk5G0SCnEhPIPaEJZ701M3Q1U30hx5LinlKUNtRznHpudvaGzW62KPQZmquI1eA0XNIONRA2HztEunfz3UfmxeEiTqpxprBddQjPGpWM+2Y9wsKSCk88GOSOrF53tSaI5cdv0qWq1SU4VuomtehDQSSrSlGCTwAMgAZ+WB0M6ki4JOlrLyW0VeVDxlC5q8B7RqIA7DZQ7Z2hS5C3NcXUccIQcHmrj3UoJ5OOYPGGcBUU+/rinKFQDNyK9Ly32mUr0g6QpQBO+3Gf7wobu/iKa6ezclK3PVJtJnkrWytFPDreEEAhRQnY+Yc8wzM3h2ggm/UHCpygmwKnSAVt/wCwRQOkvUZrqRQHa1KurdlkveCy6qVWxrwAVYCwCcE4yNu3IgjFbcLqc67TUir8CUV8OLG3gA8f7iI54takK6e9Ta9brjOmh3s+7V5IKOUInSf8Q3vxqGlXuTiH/wBZun91Xe5ITVqXW5RXJbUHtLIc8UHGM7jjfv3iOuPpK9WrSlpNmd/zimhD0nOvIGQ+gYClAdjuCB6/aKXRGyMWPBFfxE48rJjVVHIMlbUdCpnKVf1IGd/36x6dV0Zt2XfIOG5xvJPbORn84gek1rdR6bNTT9+ViSmiUpQy3Ky+gJxyoqwCc7YGNt4v96Wuq5rTn6Ih7wnZhr+U4f8AQ4N0q9sgRmiUY02MZ2rc5H3gSg2OthUvMskDVnJGPUY/SKtaCQzXmG0jSUTzqchPo6sbekVRy6b9sl9ynXLZNYMw0S2mcpjfitvJHB8vGef0HEXLpemfuKqJqJoFQp7DaisKnmw2tazydOSfU5PrCs7MmMYfR7uMxIGc5vY6j2kwHGSCcjHpCOv61zb99vVGT0ok6zLlxaPR9J3PyDn3zD5kpctNhJ5xCs6/T0tQqdTa3NocDTbq2VFttSj5k7bD2/fEXIQ6MnsST8XVvRnv05poWj61YGVqOCPscRaOobJNlVMNg5SzqGPsQf0iD6PzQqtrS1RZZdS26tami42UFSc7HB3/AHkbYi6XFTTVaFPU1OMzMutodtykgfnEujXxgAx2qcuxMU9sU+WqU0ZaYbS4hTRV5hsRt/3G/wBP+jtk2bcj1epFIlWZ5YVocKtS0BXITk+Udtu23EVCyLwlvHMsXA3UZHMvNyrmEOJIOCce4/SJmhNOv3ImozL5dden/qE6jnSjIITz2CQInetNaH8r/qVKGz0w6qXzqugmy5pwZBZcZc59HEwoblq9mUYSj141KQlEvLU0wqccCAo4BIBPfEOXqE2Zux6xpyCJVSwfuBmFLVLWt+7JeVauSny02y0fFQHmQsJUU4JGQcbRvySg48bHqzO+PYhnH6EZvSa5qBcFHX+GpqSmJWRKWdUrMB1AJGdORsD9vaCPTphI2tb9LRQLcRLSyEFTpYYZ8MEkjKsAAHkbwQGGtnBmZid/Mv6mUKG4Bz6xh9OjAGBg7wQRbIJiJVpBJSkc77RmWkkAYz2ggjBCPM0JykSsycONJPuI+SlIlZQ/yUJHbYQQQt1W7qGjtVXJFLWBkHiNSp06WqDH0002lxBPChkQQQVWtTgaa5hT6exJthlkBKU4AA4jcKAoYPBgggV44E1zZ5iw6gdDbEvWeNWqVLDU8capiXcU04rG3mKCNW22+YLB6T29ZrqlU1Li3HNi4+8t1ZAPGVEkD7DaCCEZSWcBjKMPC8S+VSkMVKkTVMfUoNzTC2lFOxAUkg4/vHPU50GrUnMLVIdUbhbSFEpQqYKk75OCMAbekEEDq8zqgUHiN0uNSxsS99IbCq1r1OZn6veE/WVLb8JpD+AhsZBJx3JwB7QQQQOAkoDAz8ORP//Z" } ] }, { "items": [ { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAoAJ8DASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1PUNYe0umiSIMFxkk4yaqWPi22umyux0BwzRSB9v1qHWlxqEpx1Cn9K5PS9DXStRu7mJgIbgA7AOQc5roprmitDOo+V7nqquGAIOQeQaoxaxayXf2YMxcsVBxxke9PsSRZwqSDtQDP4Vy0RMWqMf+edyR+tSlefKN/DzHVXOp2tpMI55gpIzjBP8AKrYZWUEHIPIritfY/wBsHPR4VP5Eiun06YyadA2efLAz+FSvhuNqzsWUuYnkMfmJvH8IYZqUnAzXF6XIU1OJ2OW81lOfXJFb+uzPFos7xsVbjkHnGRVPSKkJK8uU1QcjNLmuc8OTuTcRs7MPlI3HOOtdCDkZptWJWopcZxmgNk1gahqNxHfvFGwVVwMbRzxmsqw8YRXN61skwaZM7o3jKnjrz0qoxclcmUlF2Z2tJmooZlmiWRTwwzWE3idE1tbH7MxVpPL8zf3zjpj+tS9HYpK6ujojTc1Wvr2Oys3uJSdi9gMk+1Z+leIrLVVn8ovGYV3MJQBx69aLoLM2KSsrTvEGn6jdva28zPKoLYKEcDvn8a1GbC5ppit0DFJVD+2tO87yTfWvmZxs81c5/OrwYMMg5pgYOvLi9U+sY/nXFRaXNZ+J1u4XkNvcB/MQEkA9c13niKwu7uON7OVY3XIJZN2RXOxaXrfmBXkt9vc+Wc/zqaS0s+5VR638jqNFcmxAP8LECsO9Hlazdr0+dXH4iuk0+38i3VD2FYfiWwvFuUv7KPzTt2SR+o7GlJ2nzIcVeDiyt4hZTeWUi/xxsv8AWtzQn3aVGM/dyP1rjVuLu7kijewuQykhd68LnrzXcaTbG3sUjPJ6n60+XlixOV5I5s/uNXmHTZck/gTmuj1dPN0a6Uf88yfy5/pXNaw4ttdnj2OWmKsgVCc8AV1roZbNkYfeQgj8Kn/l3Yvapc57w6+Lsj+9H/hXVr0rz6x1OGykjYzRB1BBUsAa6bS9fivblYFCEsDgrID09qvVxTRm9JNMq6wNuqH/AGkVv6Vi6jp0cGoafqCKFLsY3I756f1rd8QDbe2z/wB5WH5f/rrLv3a6tLaKMjMUyuc+gzn+dKHM4NLuOfLGak+x0WkSE2hX+62K4vVj9m1+R+6T7v1Brr9Hz5LH1Ncv4riMerSsP4wrD8sf0pVtKiYUNaTR0/iAGTQpyOwDfqK80gupLC8cocFg0ZGfvAjkV6dj7ZoGP+ett/Na8m1eNxiWMbmjcSY9u/6ZqJaTfoarWC9Tb8IXRXxFbvziRWQ/kf6iuu8YXk1vopWF2UyEgleuMGuC0WQQaxZyjp5ynPsTzXe+LYfN0cH+7IM/iCKp6Q0Jirz1PMRYyrYR6ijJ5Ql8t1x09816F4E1GS60+eBmLxQuBFJ2wf4c+2P1riItTnstDmgSEvEsxeXaoJAKjHXoMiut8F6ncTT/AGeUkRtDvRCoBXkensa1vdsxtZI74qGpnkrnpRRWRqPCAUNGrDBFFFICv9jj3ZCj8qnWMKMCiiq5mxWSIpLVHkDkfMOhqQJgYooouxWMW58N2E8hdraIknJOwVLpui2umyM8ECIzDBIUA4ooquZ2Ja1H6xpMWq2yxybgVOQVYr/KsGPwikcgInuNueR5p5oopxdhSV9zprS2EEQQDAAxWP4g0OXU54pIphGANr/JnPOfWiij4nqNPlWhrWdqtrZR265KxoFG7riuQtvC9wdVL3Jj8hMqAMkuMYwfTrRRSUU3cfM0rIzj4RvYtQWGLaIVb5JSei/T1rudSszfaXNbgjey/KT/AHhyKKKcoK1hKbvc8umtLy0uJomgkQygo8ZQkNn0rQ8O3X9n37Xl7izs7cFJJrgiJFBwBlmwBliAKKKqKtG4pO8j/9k=" + "mime": "application/vnd.code.notebook.stdout", + "value": [ + "Box 1: [257,50,377,88,0,0.48566171526908875], Annotation: 'noyce'", + "" + ] } ] }, @@ -246,15 +162,18 @@ "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAmAHkDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1bV7ye3kjWF9nGTwOaWHU2OmvK5BkjU5yOvpUGuD95Gfaucj1SOeDVIYziS23RSL77dw/MEVolzQ8xN2n5HV6XqM10zrNt4AIwMVps4C5JwPWub0xsTgr0Zc1parKyaZKVOCRj8zSqJJhDVFafxLBE5CxZQHG9nCg1dsdWgvW2LlJMZ2t3+lcJf6FHrsaxSbsR5IwcYJ71L4fjl04W0Ej7ngk2Fj3AP8AgabSvYSva56BcXMVtbtPM+2NBliaitdRtbwsIJdzKMkYI/nTbqFbyylgY8OpBxXH6fcS2F4VfIeElHHqKzW9i3HS520l5bxSBJJ40YjOGYCpsjFcSkzXepLMwzlgwB7DtXXCXEBduwyauUeVIiL5mTGQL94gfU0oYMMg5rgdZ1C8kgluYI1km48tHJ2j8vapPCuuyXEcEkoETyNsliB4BzjgVXKK53fSjdVS9uGt7GaVfvKpI+tcnf8AiqTSo0kuppNjttBWINg4zzgUormHLQ7jNH51z3hzxANbErRsXjTHzGMrye3PWt7NNqwJ3MnWx/qifeuHltzpviU3yjFvqAEUvoJAMKfyyK7nxDp9xqFmqWtwYJVbO4DOR6VQTQjLo/2W6kMsmMmQjHzdQcfWoTajZFNK92M07CyoBxgYxWpqozpU3sAaw9G07V4b8m+niaJQQoRME/jXTywCa2kibo6lT+NOq+Z3FTXKc7pDKJ3U9SM1RnwupzAf89f6Cqc76jo8xjntJpVXhJoBnI9xU2mSS6jeBza3CDOWeZduTTiry5gk7LlO3g5QfSue1+x8u9ivI+A4KSe/pXRwLtQA9azfEbLFpZmYHEbBjgZNZMuLMvRoN8hc9jiujnTNpKo7oR+lYHhiZbm3kkQNs38EqRmulIymD3rSo7siCschZRiSeNWHBHf6VZt/DmnwasL0Qx+bu3AnsfUCqEVwtrfvbsds0LkbW43D1H4VJMxub15WJw20KpPTFVG8rCl7rZ0+oLu024Hfy2/lXGXUtrDAst5IiRcfNIcDJrtX/e2rj1Qj9K417WK7to4rhFeMhchlyOKKPxMKvwI2PDl9Z3AaGzkhcINx8tw2K6Ld7VgaDBYWAMNsI42c5IVMZ/Sug49acr3JjsP2g0mwUUViaCeWucgUuBiiimBFLAkg+YCo0tI4zkAUUU02JosquBTJY1kjKsMg0UUdQ6EUFukIwgwParGKKKGIxtY0Cx1XDXEQLgYDgkN+Yqjp3hu0sbjzI9zN0Bdi2Pzooq4bClqdEqAJj8K4q88Myi4dotUuo1JyFD8D2oopwSbCUmkWtD0ie21BZptQmnCg4Q9M+tcr/wANA+E/+gfrX/fmL/45RRRU0ZMNUf/Z" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAAYADEDASIAAhEBAxEB/8QAGwAAAwACAwAAAAAAAAAAAAAAAAUGAwQBBwj/xAAxEAACAQIFAgQEBQUAAAAAAAABAgMEBQAGERIhIjEHE0FhFSNSkRQWMlGyU3GBoeH/xAAYAQADAQEAAAAAAAAAAAAAAAABAgMABP/EABoRAAMBAQEBAAAAAAAAAAAAAAABAhESITH/2gAMAwEAAhEDEQA/APUd8z3nSozXdrbl9lZLRy9MrIrmNQCz9QO488KSAeBqO5q1z3O2UTeUijarWUQHUaLuOnVp/b017+2NHxJvVFYYlo7dTQrdr18nzUQBxGP1EnTnvtHufbDPK+VqWnyyaG9UsM61Oks0Uq7l400BB440H+cTlNW0mdNNOU2hTl/P98+LRUF/onWCoCtFMyIA6sSA6lCeNe4YAjUYy5o8Q7ta6+rprVbGnFEetEj3yMoALMAWHAB9OdAdNTxhVEozXmmnht1OkVutxCx7R0rGpB+5IGnsPbGzcoxQ+IhU67KsIxGnHWhjP8Rho1z9FrNKKLPcTZXa+/hl85GEJiB6S5I0IP08640sueJbXO9fBrlQCF2K7XCOg6tQpG7hlJBGoPcYkrKjNli+252Jaj2Te+sTnd/HHNTN5dLZLkoIaHzYCfXcrq6+n7bsU1/dE8O6Nze+DCn43R/1B9sGH7ExkL4i5IzNfcwW2/ZdlpN1LF5bLPKyNqG1BUhW98M7Fb8+slXTZhlpxFLTNHGUqTJ8w9jyi6D74MGIqF1pTt84T9st3iNYFeK22dEVzq2yeNwx079S4b5usGZK2K2ZgoaVZrhSxgVVOjBWbsQyE9JIYHg6AgnnjQmDGicTRqpvDFk/Ltxq3uc94t0tCtfFLGySBQSZDqxAVjieNozTThMt11jqGMFR5sc8abopCVK7w47Ag6kNyOe/cmDB58QN9Kv8m3H6v9/9wYMGKcoGn//Z" } ] }, { "items": [ { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAXADEDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDu7rVNQfUriK1wRB1QEA47nnr9K0RqrHSzOFBkB2+2fWotauY7VQkSKLm4+XcBzjv/AIVYsLFE0/y7hFcP8zKw4q3sJblWz1a4+0rHcxkK2CrEDkHuMU++1eeCeRIYSwj6gDJPqetVlH2/UkWJQsEXAx0AFSzDytcx2fB/MY/pSY0XV1QHTjcbBuB27R61FZ60Z7ryJY9pOMHBHXp16is62BOn3cR6x4b/AL5PP8qV22xWso6ruX8QQR/WkM6jmiq32pPX9KKQ7mPrOmXd1ewXNq0eUXBDtjvn0NT2kWpESJdMm0oQMPu5/IUUVV9CbalKGPU7TKxQqATzhwf51b1G2uZI4LpFDTIv7xQQM+47daKKGxIZpls8xneVNiyKQQcd+vQ1QYTxOLGZBvD7lIIw3GM//rooo6jOc/4Wd4J/6D3/AJKz/wDxFFFFK4H/2Q==" + "mime": "application/vnd.code.notebook.stdout", + "value": [ + "Box 2: [604,205,653,229,0,0.45074450969696045], Annotation: '2200'", + "" + ] } ] }, @@ -262,7 +181,18 @@ "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAgABoDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD03Ur25h1BhHMyqAMLnj8qtXOpsNN8+JgJDgeuDVDV0Jvmx121gWGsJqGg29yn+rn8tgP7pJAI/A8fhS6D6nZaVfyXUDNMwLhscDHGK0s1zujHBce9boPApgjH1FAb/P8AsiuK021OlXk+nFf9Hc+fBxwMn5h+B5rsNe0u9u7uOW0vpLdQNrKoBB561FqWiPcWitbttuo+Y3I/PNKzHdD9II3t+FbobgVzugWWoQeY19KjsSNuxcYFTT+LPDVtcSQT+I9JimiYpJHJexqyMDgggtwQe1N3ErH/2Q==" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAApAKMDASIAAhEBAxEB/8QAHQAAAQQDAQEAAAAAAAAAAAAABwMEBQYAAQgCCf/EADcQAAEDAwMCAwcDAwMFAAAAAAECAwQABREGEiEHMRNBUQgUIjJhcYEzkcEVNEJScrFiobLR8P/EABkBAAMBAQEAAAAAAAAAAAAAAAECAwAEBf/EACARAAICAgMBAAMAAAAAAAAAAAABAhEDIRITMUEEUWH/2gAMAwEAAhEDEQA/APoVNKVuIQR2ST6c8Vzf1C9sux9NepV36fXnpvqSQxanmm/6nAdQtDoWyhwqCFbcY8Qpxv7pNdGzQrxwOflP/NBm86U6Laj1vc0XHV9nVqCRJSiVb1zo5eQ5sSAjwid4JTs4IzyPWvRgtaOFvYW7Df496tEG/Wyap+BcY7UqOtWQVNuJCkE55HBGRU4xKLieTzQo1heLppGPYtL6Z0xMXb3X2Iip6VN+7w2G8HaQFb9ygnaPhCefmzgEh2V9TsZDijuJHrWnBUNGTRJyZymgENgFSueewqlxOs2g52q3dDwdeWCRqBhamnbaiUkvJcT8yMA8qGDlI5GDxVldcIdW5/p/gVx1pf2doKpMPWLYLd1YkpuQkIGHC/v8TcVd1Eq55oRxqvDPJ/TtSPcVOJ+NvCvMU7S+lScj71Xp8wQ/BWDgOPbPvwT/ABUgw/vZUoHORSSxr4UU3Qv/AFdk5Bbd4+g5/wC9Kt3SI6oJ8QpJ4G4YzQd9oO16rvPR+7QNFXi6Wu8OuQlMSrY6tuSgCS0pe1SPiAKAoHH+JNMPZwZ6kW7py7H6qXeZPnxZzyY0yd+quJtQUlajyrB3jJ9O+BQeJG7GkHpLqFDhWT9a8qfaRhK3UJJGQCrk1DWq5NT47UppWUuJCwfUEV5muFUxQUDwhOPyTSdVh7ScDzaz8LiT9lCvee3nXHMz2i+vEPWF5sDfSWx3CHBukqJDeakrbU6y26pKFKJWQFFISTx3zgeVdGdNdSX/AFFpSFc9SWFFluD4Wp2GiQHktfEduFjvkAH6Zx5UrjHxMpckra0XoKGfpWFXpzULPkuGQ0whxSRsWo4OMkFI/mgLrX2yOnHT3qBcunupbfq+PKtbrbT05qEh2Ire2hwKSrxN5ACwD8Gcg8Gj1MXtR0qVZ/Fa3c9zUFbL173HZktSUvsPtpdacH+SFDKT9iCK8an1TA0zYpV+ucpiLFiJCnXn1bW20lQG5R9Bml634Mppk/uHesCuO57VQNPdWNNarceY0zqKy3dyOne8iHKS4ptJPBUlJJGTxUk11AtRn/0pU23e9IcS26yJifEQo44KO4PI4+ooOLXqDyTLbu9M1lNhLTjkH9xWUvEYiJv642n/ABxxXHHUP2aLFr/qVqXVjqnWpc24qWpbalDcUhKBkdjwkcYrpPrBpfXF7jwZeh9Zy7FJilYdSy0lxD4VtxuB9NvH3NZoHR9ygWdoajuC59xXlyTIU2Gy64okqVtHA5PlXbDLCOm9nI8U3tCsNl+Loa2x5zin348OK064vJU4oBAKjnzJGeTVnshSWEqGO2eKDl/Y67wNYOWm1zLA/pp6SnZ7xHX4rccqG5OUjlQGcH7UarNDcZYQlR5xQeWErSdhWKa9RokOCU2k/GFFHfzKRiqTpuJ4VnbaUkhaG9pBHmBjH70t1FumrNGyhqXT1hVfYbiA3OtyHNjuBna62cHkZIKcHII9KHq/ac0PEcWqboPWTErOVMC2IJKvMg+IMn696rHIlG7JSxSk6oLGp5IBt7CVAKVKKvqQlBz/AOQqcg591J9BQh0JqPUvVC/L1LO01JsVqiN+72+NJ/WXuOVurGOCdqBjHGO5zkmZqOW4+0ZOE/vUeyL8ZXrlFKwZ9c9cav6edKJ+rNC2uFcL1EXDQxGmoUplaVvtocyEKSrIQpRGD3A79qq/RrX0r2ien11g9RtJKtcuHK90mxWHnG23gUhSHEYVuA75SVKHHOc4p1eut/Ra7xnNO6g1WI/hOpS+07FfQQtCv9RbxjcKRY609FdGWyQrS14aukqUdyIcRC1PSXcYSn5cJ7YyccZ79qa9bFpv4FjTzJgse6BYUGiUAjzxxT58Zmr+HOW0Hv8AVVQmiF3GVaY8q5NhuS8gOvJHYLUMqA+mTxU0+tKJxSedyE/8mlT/AEan9OeJ/tQdC7Jqu76euWkNRR7hbblIhvvptiC268h1SFrSpLmVJKgSDjJHlR00RqC2apsMK/2JD6IU1oOspfaLbgSe25J5FVBPSW2OXuXeZDEZxUmU7J7En41lWDkd+aIVpisQI6I7aNiUJCQB9Klu9ovrjpmp3FyaAPJZcHP+5NB7VfTrpjq3V10N1v1rVdpTqBIhF1lT7avDSEgo3bgSnaRkdiD6UYpoBuLBGf0nBj8prlLrV7Ntm6g9R79q1p1+PPkrZKnWlEbihhtAyOx4SKvXKiF8bD87dJNjl2nS1tsU0QGWEtiepSCylLaQlLXCiveeDykDAOCewtc+1xNQWZ63XBhuRHko2uNuJ3JUPqDx6VWdN+8J0faUz1qfebhxkPOK5KlpCQVH65zVxtuPCx6jmpS06KJasF+helGnunurblMsluZii4wwFhpsJB2OAjsP+o1U5PQvTNw6xzOoT0HdO9/blhfiKA8RCE7VYzjIKQe3lRuuQSi7MHABUw6PzlJ/imcFtJukk4wfFzn14FLkimkPjltk6wkhlAVgkDkmsp4hKQgZA7VlRtlhR1hLowoZrwiKhHyp8qcDt+K2r5R9hTkxm5b2XVBSkDPrS7TAQCAKXPes/wDVAI1kxG5DZQtOQfI1AStGWx5wuKiIJPnirR5H71pXb80rjGXqGjOUfGRECyR4KQhhoIA7ACpMM4SUilR3/FYO34rJKKpGk72yg3/pbpu8yXJMu0sOrcO5RU0Dk/tUdauj2lbTNEyHZorTqT86GUpV+4FFBz5fxSB7I+1SeGDdjxyySoaQoIYQEYAAqhdX+j9m6mtwHLi9KZet/ieE5HeLZwrbkEDv8o/+zRMT8xpN35arbqkTXts5iT7LrkJWbbrbVMfHYN3Ip/ijJ0x0hJ0fYGrNLu9xuSkuLX486QXXDk9txHYDsKth86cMdj+aSCknbk2Um4yVJJFK6raX1LqG0R0aS1NLsU6O8V+OwkK3pKSChQPlkg/cCkND6TusK0NI1NdXbpcVAl+U42lBcJPBwOBgYA+gq/v/AKP702jdh9qssk7pPRPhH2gKJsvWS3ayFtZvdrVplcrKULh5dTH3Z2AgD4sZG4nvz6CjPb43htAE84x2pB/+4H+6pNnuPt/FLHJKTfIPGMVpA/6sXzUelosK86d0m5f1hxTT0duQlkoQoZ35IOeQBjHnS2gZd2vttbvV1syrVIlErVEW6HC3zgZUAM5AB7eePKrPqD+z/JpOy/2zX+w0j/Il2ddDLElHkTCAdo4FZSqflFZWsU//2Q==" + } + ] + }, + { + "items": [ + { + "mime": "application/vnd.code.notebook.stdout", + "value": [ + "Box 3: [22,33,185,74,0,0.3334950804710388], Annotation: 'robert'", + "" + ] } ] }, @@ -270,7 +200,18 @@ "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAXAB8DASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDg538i7nWG3jVi5VGPOecH+lK0lzuNorrG4kKnYvHIBH8jVnUIo1S+yB5qvuQn3wf6VTlEseqQyy4LM0ZbHTJ4/wAa9WWjOFDLeCSaSFZJ5DviZhz3BxirGgttZ4yxIKK3J78g/wAqZ5P7y2TcyGO4dMg846gU/SEMOpyISTuDKue2D0/WlHcGaWq6fNNfAxbfLcZYnsQCP61ANNvJlzO0YYIFXBz908f1/OiitnFXIuSTaS8wk2y7CZfMVgOnGKjktYNKg+23NyypEdzPjP3gF7c9aKKmaUYuS6Djq0j/2Q==" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAAXACADASIAAhEBAxEB/8QAGgAAAQUBAAAAAAAAAAAAAAAAAAECBQYHA//EACsQAAEDAgUDBAEFAAAAAAAAAAECAwQFEQAGEiFBBzFREyIjYYEUJEJxkf/EABkBAAIDAQAAAAAAAAAAAAAAAAIGAAMEBf/EABoRAAIDAQEAAAAAAAAAAAAAAAECAAMREiL/2gAMAwEAAhEDEQA/AMPz5IZomb6lLiUaKXG0haZKyoq9fQHL2v4KvG+I92oV6lpVSHHYsORJ/SO/tmk6S25qQb2G9iUk/wCYuud6VTl16uNT20fPAQ4ypdhZzS42nTfsd7WxmmYG6sYVHrFQLa3VQkq+JJAKEup03PBspIPHm+Eys+RHq4csY+LS6q7NhR6nXJavVqT8VwpWRpUhNxbewCiPxfEl07Sqm5tcguPulLxltOJWq41tu7EePaobXthMxwkumpspeWyBWIspLjavchLyACQeO9746UOIqldTSt51bja5KmQF/wAVuNBSVgeSQRf65wbnpZWnlxNL6o5XqtVqECRSg171JYlFRtoQFhwKG+/YjnFXjZBzRMYaptXehiJGZkxWCFlStDibgnYdlaT/AEMGDGJLGVcE6VlKliTJCZ07lVGNMZcmIaXLixm1KQndDrJ3V93BHI5/Laf00ENSZkqsyJEhtxl8ukAFSmlKIPPcHSfoYMGIbGgilN3J/9k=" + } + ] + }, + { + "items": [ + { + "mime": "application/vnd.code.notebook.stdout", + "value": [ + "Box 4: [506,408,538,431,0,0.32059410214424133], Annotation: 'center'", + "" + ] } ] }, @@ -278,7 +219,18 @@ "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAqAKMDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD2d+QAfWuQ8R+OYfDesLYS6Zc3CtEsnmwsOMkjGD9PXvXXyZ4rnNUtdEudVP269gju9ijypJVB284ODz61kzVGtpWpw6rpkF/bM/kyruAcYYc4IPuCDWgr571gXrNo2kLDp9o8q/dBQjbHuPLHnOOewP4VrWbl4lJ5NCYNFpn2j1NZk3iHTbbUBYTajax3ZxiFnAbnp+daDHLV51d+EodU1W+upRl5Lh23d+pA5+gFDYJHo6Sk9RzUgbNZ8TNFYQl2JYIgYnueM1ajfcuaaYrDzMASMGlE6H1H1FZOtRzy6DqKW0kkc728gjeM4ZWwcEe+awfA8OtwW93Fq9zPcIGUwPOSW77hk8ntT5mFjtww9aCwHUgfjVWCZZVypBAJGR7U6U/Mv0Jo5hWLG4HoR+dLXnmr+K9dsNfurK30m2uLWNlEb7iGOVBOefUmum8P6leahZmW9sltJN5ARX3Ajjmq1FZG7mjNQSscIoJ5P9K5bXfHOneHdUWwvIb9naMSeZCgZMEkdSw54pcwWOwzxRms+w1CO/sobu3k8yCZA6MRg4NWZrgRQPK2AEUsSenAp3CxPRWLaeIbK9lEVteWs0hGdiSAnHrjNWH1WKKTy3eFZMZ2mQA4+lF0FjRopgfIB45opiIpO2K8/wDE/hSDXPEUtzJkOsaICCQcAZ/rXY65a3dzZAWV29tMrbg6DORg8GqWjWN2kbNfzmedjkuV259KzcWWmh+kWrWPh6G0di/kxsoJ9ATj9K0bTBQY9KwNYOv216Rpz2xtnA+WVCSD36V0FlG6wpuxuwM4o5WPmRPkeYw9hWZbptklDdfMb+ZqXVWu7ZVurSMTOgw8JON6+x7GsFvGdjG3+k6dfxS918oHP45GalxY0zoLt9tkozgkqKs2/KCuYs9WuPEF8hjspbayhywMow0jdOnbAzXVRLtQCqs0TdGfq93cWOh311aoj3EMMjxK4ypYAkA+2a5/wp4jvPEsd5Z6tYLbyogJ8osFdTx65H51oX3iPRopLiwvLvy3BKOpRv54xVeHX/DdgrzQXiMxGMKCWPsOKQzcsbdLJBbxcRxgKo9B2q3J99foazdIuZb23FzJGYzKdwU/wjsPyrRkIDr9DQI5HVfF2g6ZrVxZXlldG4jK7pVhBVsqCMHPoQK3tG1G11S0FzZBxESQA6bTx7VWutDiutQluXVTvI69eAB/StS1gS3QIowBV9BEs33o/Xd/Q1z+r6dpd7qWbu5hSfywBG7LuIycHBOfWugl6x/739DXE+LvC0Gu6us7llkWBUDKcHGWP9ago31/4lVnbWVrbSPEDt8wEbUHXJ5z9OK1UAmgKtghhgg1jaHbvaaDBayMXaFCmT6AnH6Vs25G0VSJZhW3h200/WkvIYVRmDKdq46j/wCtVDW/DNrquvrdzR7mWNF6kZAJ/wAa6y4wHiP+1/Q1Xxm6P0FNPUHsTxhhGo9qKnUfKKKokkK5FNEYHQU8UUhkbRBuopyoBT/SloAjZAwwapS6bDI2SgrQopptCtcpw2ccI+VQKtBcU6g9KGC0Mq+0e3vJC8kasT3IqlH4as45Q6wIGHfaM10I6GimpNC5UyvBAI1CgcCqOu6PFrFksEpYbW3qVbBBxj+ta/ekI4pDOAPglo2/dXt4n+7Liug0DS5dNikSW5uJyzZzM+4j6VtkD0pVptp9BK/cpatbT3OnvHbTvbzZBWReoxVDSLG7iiP265a4lz99lA49K3u1MUVNkVdnL6hDrcGpEWM8ItXI+V487fX610dvHtQZNPkAz0FPSnZCuzP1mae2svPt7c3EiMD5YYLn15qnpFzcaghnntDbNnaEZ9xx61tTgGM5FQW4AAwAKFFWuDlrYsjgUV8y/FjxT4h034mavaWOvapa20fk7IYLyREXMKE4UHA5JP40UBc//9k=" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAAgABoDASIAAhEBAxEB/8QAGAABAQEBAQAAAAAAAAAAAAAABwgEBgX/xAArEAABAgUDAwQBBQAAAAAAAAABAgMABAUGEQcSISIxURNBYXEUMpGhsdH/xAAVAQEBAAAAAAAAAAAAAAAAAAADAv/EABwRAAICAwEBAAAAAAAAAAAAAAABAhESITEDQf/aAAwDAQACEQMRAD8As68bqr1Pud5qRqbzDKEI2thXSfJ2x69bvqYbtUVamPoRNL2J5AV6atwzwe47xy2oDC1XDMKR+r0gRn3OP9/uC+1dR5S8dN6RcUooiWrP4b4Qe7K1rShbZHlKiUn5SYWqm18I24pooawrsn6/Tn3ao82t9t/aClAR0bQRwPkmOvC8gHI5+oJtNnFIMw3gY3A4hMSvKQee3mIjwRh7d8sly5FKUM5bTE4WJQ16f3FVLAeZxS5t1VXpIKcpTuVl1seClzCwB2Big9WNPbvuSuSlUtm852jtobDL7LLSFpX1Hq598ED37RhvjSmZrNEYmKFMpl65T8rlJhxOQCeFBQ9wR/OIXPz2m+hqE6VGrT1aQ+8BwSEnGfuEtL5CQMntBTpHaV8Uj8uYvSqSs084UpZEuzsCUjOc+Scj9vmE4yruThzj6g4Y10SV3w//2Q==" + } + ] + }, + { + "items": [ + { + "mime": "application/vnd.code.notebook.stdout", + "value": [ + "Box 5: [203,46,229,78,0,0.3125338554382324], Annotation: 'n'", + "" + ] } ] } @@ -287,28 +239,25 @@ { "language": "markdown", "source": [ - "### Print Annotations in Plain Text Format" + "## Display the OCR Results on the original image" ], "outputs": [] }, { "language": "typescript", "source": [ - "printSortedAnnotations(boxesWithAnnotations);\n" + "const annotatedImg = await putAnnotationsOnImage(\n img,\n boxesWithAnnotations,\n { threshold: 0.3, confLabels: false },\n);\nannotatedImg.display(display);\n" ], "outputs": [ { "items": [ { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "Sorted Annotations: [ 'robert', 'n', 'noyce', 'building', '2200', 'center' ]", - "" - ] + "mime": "image/jpeg", + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAIFArIDASIAAhEBAxEB/8QAHgAAAAYDAQEAAAAAAAAAAAAAAwQFBgcIAAECCQr/xABvEAABAwEGAgYGBQUJCQgOBgsBAgMRBAAFBhIhMQdBCBMiUWFxFDKBkaHwI7HB0eEJFUJS8RYXGDNicpOy0iRDgpKVoqPC4iUmNFNjZXODJzU2NzhHVFVWV3WFs8MZKERFSFhn09RGZnR2hJS08v/EABwBAAEFAQEBAAAAAAAAAAAAAAQBAgMFBgAHCP/EAEwRAAEDAgQDBAcEBQoFAwUBAAEAAgMEEQUSITETQVEGFGGRIjJCUnGBoRWisdEjksHS8BYzNERTYnKTo7IkQ2OC4QdU8Rclc4PC4v/aAAwDAQACEQMRAD8Au/X8P6atT1FQuncR1iXB2HEHsqChs53gcrV6xP8Ak2+B2J6mtrHVX/SvV7q33VUt75e2peckB1pyNRtta2z4+l/wrdoTpA2iw0GD0NI4mniDb9BZTuxGpnaBK8m3VUMv/wDJZ8Maa7qipY4i4nuqkp2luOv1dTSPNtNgEqUoltsBIEkkmBFo4/8Ao6MJ4noq5GBekzdF5sUCEu1SmqSlqPR0TopxTVYco0OpAnW3p1U0VLX0r1BX0rVTS1LamnmHmwtt1CgQpCkmQUkEgg6ETaqXDDhbhG+ukxxVw7U3HRJwzdf5vcFxs06G6J10tgtlbSQEqSmXCEEZSpQUQSkEDV2ankiZG0HOba6a2Jv8NFqezmF0GLUdbPWSOY6nYJPRAII4jGEWJBzHPpqB1KqXSfk88VYkdeYwFxswHiN2gg1LVO8+tTQOYJzhlDoTMEa8gbNnEHQk4rYYvJV312J8DKrHYSmnOIW6V5R5dipDROkaW9EeLWGcN9HTAmP+K/DS72rnvG9bvo7vRS0jDbVNTvdd1SX22wmEqHXFRGxKBpqZcvAbhRgyh4PXCb1w5d96V2IbsZvK96qup01D1Y/UNhxfWLWCVjtwJ5DzsO0TPm7sQM4GY72tewtz1t8vFGVGD4TDQHGmySGnc8RsFmh5eI2vkLtbAMLgBa+e/K115gK6FvSVpWl1X7hG6mTmQGL6u54KJM5iE1BJBBPLUGyPffCHiahxdyHh/e356uZXXXxT0d2qcXTNKTmQp0NBQSkgoIVMGRb0l6OFa9hji1xK4Fl1VVcOHalFfczb30ho2HSCphJOuQdY3A5Qo8zZDw3gC4cY9NDiZS3/AETFVdN3Xfd1Qq7nEAsPvejUwbLidlJTKzlOmbKd0iwct5xBLGLOzlpF+YDgdelxfZaXDcJjwd2LYdVSZoe6tma7LqQ6SBzHBpOj8ry217Aki5Gq8y2cOYlQpx7823oppJyrWujXCFSdCSI7vnSwSPSUy2uoPWdocySZ7j9tvTrpZ4aw/wAFRg/jXw+udi5L0ob8au+uTd6RTpraVxpxam3AmAdGSmd4X4CAek5e1z1/Gbh3wwvG7b4rrgrGjfd7Ul3Uz1XU1yB1nVtBtGZa0yysqSBsqT6sh81UYOIyUDM0tHgc23w536WVZRdim4uaSooJHcCdszjdoL2cAXkbYGziQW5NW3LgDaxK82i1eCEoLqFtjcqKOe/2WMoFWp1sspWsNp2CefLwI18fhb0O40YR4U4w4d3rdeDOA2MqC/g0HLudp+Hr1MS8kghClpYByKEggmNbD1/Ry4dv9G6u4g3xw2o7jxTT4Uqa6oZVQOUbrNY1TrJV1SVICe0nMApJGo0NmcSR2djbEtbmuL28R8Ug7L0sXdaicvZHLLwnMdl4gJALXN1s5pvroLEEa6FeZ9A1Xkrpsr+dLroWktqOsnw+Onh32Xbpp6hwNN1NO6kM00StJHb00jwE++1/uB/Qu4aY44QYexZflE47el8tKrXX2q6oYPaWoJTCVFBhIG6Dr7IZHSC6IWEuEWG8P31dN4Xo9U3vf9Lcr6na9DiCh1t1RUlAYSUq+jGuYjfs66BTNrBSmoADQWg3BNxey0WA4X2cg7TR4W975nsmcwhzW5HBoeL65r2IG4F7ggC1lSxxNUwtxx5h4IMJ1bieXlz+AsUceeyhLis0g5VZSDG2o93LmLegd6fk07uWVG7eIN9NJJkJdpaepj2hxqfdZoXr+TSxk2hKLpx8w9kAE1V0qbBAMwerdc8eRtfMilAs8a+C8lqn0rpSaW4Z/eIJ8wAPoqYKWorQS4ggGCDodjrHn38x7LdIU6pS0h5UCBzIIkEzz5fX42tLeP5O/jTSLKqe+cM1CRrlC65s95iaaB7zZsXj0GOkJdqHPRcM3RWZ5ILN80zafCA8pB5DcWcWO6IcEbqAzUVCaf8AjVAjXta5SeWu/L53GQ5VodSnrVEa+sNhH42lC8+irx/u3KHeFd7PhP6VGWqsaCdmFL5+Fm7WcHeLl0KWu9OGOMKVCExnduSqQk77kp+eVmFpTw66aiFrfUoEJgEgnKdu/wANB8Pd2HXAEAkZXBICTJGncDvIsM/dd6UKEt3jR1FK4pRSUvMls9oxpI+Y8LcoacU4nOAvLuDrroO7508rRkFSDdCpccJKQFQNVAAGI8jv4Wal61VSzi+iCCUqKEgnaBCtCfYNN9BvZ1oSsqLiUAx2QRGo9/j7TZn4gJbvyjdSkKChIUFbHOYj3Dbv8bDyi4UsZ1Tvoa1TtMPpAuHClPI6x3fhYQ1BSArMVCZ0O+hjf293xsVutRZoxUgEqzwnUCNI9/z32FWspShgIUes7gCfHbx+NmQizbKWQ66raqkeqFJAGpykg84j2ePK2OOrSnNvJAzAA677+zx87aSpClq7IOTWCDGvfy/A2DUpktlbqAc+wIjWdBr4a99piNFEhFuPlwqKzoBuAJ329/zzD61zNLjvVmSqPA9w8tfd4W26Ek9XK5UTmIJA5nSd9if226DOZJJnKOyBm0yzudxM/VaM7pwXBc/i86lIUSCkcx4jcxJPxtzmdUvMsFMDZREAd0d/nYXql5gkFSlbkjbb7xoLdtsOpUSkgjKBEADaZ1PdPhZhcNk5YhS0biY1OxA5HXccrBPqU9TrSgkSkwCoxrET7QDYyEOBScyUgqMZYVqCdRPlpvbtxhwtrDgKwpBVAJ9hEjXn8bQuNk8C4TIwzUvMLrUHMSotlQAiR2iQQPPQ89PKzx9FpnQl8MhAVKyCN08z46E+/wA7M+63VtV9cwls98BcQQe+Z2J8fboX9duZV3MHIcriBMg6nv7vn22gd6LrqVou2yTvzYy5kCmVAnlJ1OnKZ7vdbld1t65XnRMJ0UJM+fOfmLOFLSUJCiEJkAjWABHd7D9elulst5R1bWYkTMkCfGPdZOIkyJtm5/7yioUdBmBG33/brYA3E4SSstqBHZzp021j2b+XhZ1GmaUSkgjujcjv08vne2N0pWsZdTEzO2+o7vvJs0y5U4R3TXRhhTwC1MU61CAdSmR3nTTb5Nsew0vLCKMGFEwlfhufeB5We6KICMqRpI7oPMa24WwVBSYABiecRM/H7LRGZSCIKO3LgqDoikdV/KBEqEjST+z7Cj+Hn0qCnG6poBWuZBEGIA1+Ym0mrpykEqTkKDIzAaH79D8fCwL9E51CnAhRhJga6fVPP42aJSuEIKiVAeaegDNBKSIJGhmY8zt4DnsotUBfZQpYbSrkqd9dvPY++2r0Z6mpXKSQVmCDzJBnz0+HnZ6YTZp6q5G0usNqha21ApzHeQPjAsrn7FI1vJMeqwy22wt6pu+lcCZVCW0kmPA+33jabR1jG4i1Sqqqa6kshKhBaby6a9wiDI8fZafb/paIoVSNMJCQZWUggTPq6e37fCOMV0QFz1KW09lCwonftSPv8/K1lQyua4G6DqWCxsoQLTzZlKFiO4EW2KusRID7ok5iMx37/qs5HKck+uv6QnVQ5ySDp87HaxX0YLQUDtaEiUyIESdfZ7raMSg7qoLLc0imtrNFF5RywBOsW0Kt2e0AeeospqpEZA4UJMg6BI310/Dytx6IlSJUxlG5Ow32j519lnB7U2x6pNNSsjUCTzt16UcmQiQBA1+fH32MqpG8sqSpMCdNdNvn5FgHqdCGsyM2+s2cC0pPSWOOOwlxebtCASeVuPSne12tFbjvsO99JSUwUT2UqQPMKJ/1hYqGVEEyNBPOyixXG67FSUnMEiedu/TVkRlAmdjFgC2scvG2sqiYCTZbBJmKMtVLaVArR2ZGm9jKrybKerbSrLtmJgxPhZOLbgGqFbd1uYPdZMoK7MUd9JbUgI5jvGkydxbQeTqCRMjWYsT1i2a8hbsoXXQ1Qc0Kka91gpmLatlnDRIShaaodpqlqpaWUraWlaSNwQZBt9JfDfFJxdw7wviptYUm+Lmoq+Rz61hK/tt8143EW98+ghiZWLOiLw1vJbnWOU91Ku9Z7vRnnGAP8VsWzPaWO8ccnQkef/wrHDrFxBU/KqXI3sEah3cG3J3tpWtsmCrgNb0Wde4T61s69zUz7rcxztqCbTBPyhdMPuJfcQVaEJV9Y+yxjrnZ9Y2JpJTUIO2ZCh7iLDmZtoaM5ogh5QAUN1zp1KjbQfd/WtxMiLYZPOxQCiJXXXuDXNbOvd5K2sHztm3OzgLpN0L6Q6dMxsn3i84H6V4rOiyg68iLHPd7LEr1Ss0pW2RKFJXPkbc4aJW6Fdde9Hr25NQ8P04i2gZTIG9uFD22QjRLsuxUukRPnYdNS6UTPIcrFI0MjWwrAlAHssO5tlI3VDGpeiCr42QcV3hetHh+9Lwbvc0RpaV2oStFPnKAhJVqDObbkBOtld5fVpkIKlKMJSOZ+wfdZBxLhlvEVyV913tVPuIrmVtAIOVDBI0KUiMxBAPanUctrQuYZPRCeSGi6r7W9Ju4EXTd993hcj1dV1wZfSwthhK6UOSC7nDqgkgaltQQsbKIOlm7xMxPgi8E1qKjEd+Lv1biaR2jSFuGjOY9jqEw22pXrFDq0KCTJk7tXEPD658S4Yv3CbV6O0+IMFPuUfpTvVqUplKQVyQoAhZOaTqCMuuqbVrZxUqlpKDDz19MO3ndFblZCG0KaYShpasiitJC0pXqCVFIVETMWrqzDxLA5pNtD5qvdWyMcARcFTwMT4DSAlynxPnAhUXhTIE84SmkIHkCQO872yzcZ6QdS2yhuqpL3W8lIDikpSQVRqQUrAInuAHgNrZbzg4bW39X73/hWPFh6r00fH0unebdtgKEW4fJ66Invt2g6b/tt9IO3WRYLBaq6lNDRP1q2Xnk07SnS2y2VuLCQTlSkaqUdgBqTas3CvE15XNx94iYvvLhvj9i68XOXc1dry8L1iRKEZFFyUfRpBO50jW1lLwvm6blbQ7e960lChwlKFVL6WwojkCoibE044wV/wCl9yf5Qa/tWrKuETSRuzhpYb/Qjr0JWhwbtBT4PTVdLNFn7wwMJz5bAPa/TQ65mN+VxzuE/ixw9o+KXDu/MBVj4YTetNkaeKcwaeSoLaWRuQlaUkjmARaPeGWOsY8OcB3XgPH3C7Fr984epUXay9c93mupK9ppOVpaHkHKiUBMhzJB38JVGOMFEj/fhcgnl+cGv7VuhjnBX/phcn+UGf7VulijdMJ2Pyutbkbi9/py+KZRdpI4MPOFVTGyQZ+IBmLS19spII5OFg4EG9haxF1FnAvhxifDN9Y34z8Q7uNLf+MHzUJuqnIfcoaNuS2zKJzukZQQJ9RPMkCO8HXjxGp+k1jri5cnC7Fj+F7zoKOnep6y63aGreQllhBcYRUBCXFoW0ZRmBKFEiSADZgY5wTzxhcn+UGv7VtjHWCv/TC5P8oM/wBqwzqCLJGxklgx2blcuN9Sfmb6K6h7fnvFZU1MLHuqIxDa5DGRtLLNaAb6CNoac1xa5udVAPFG6MXdKDEGGMF03D/EOHsGXLeab1vq8b/o/Q11JQkpDLDROdUpW4M0R2p2HadnHfhhiuuxpgzjVw8u5u8r8wa8puousuJaVXUK5C0NqVCQtIU5AOhznmADKf7usE/+mFyD/wB4M/2rb/dzgmf+7C5P8oM/2rKaGCQPMj7ucQb6C2X1bfD587qJnbyamkpm0bGMhgbI0RklwcJQRLnJIJLwbG2WwAy2Iuo+xHxZxtiK4Km5eGnDLF1Liaua9HYqL5uv0Kku1xYgvuuuyhwN+tlb6zMQBsbC8Zai+rk4FXphOro8Q4svy9sP1V0Couu6HKlb9WulUnrHEMIhpKlneAkTFn4Mc4JGpxhcn+UGf7VuhjrBOv8AvwuPzF4Nf2rSPiztcHS3LhblYfLr4oCHtFQ000DoKdrWRvz+tdziLWBedQ0W0aANze5sRG/Rdvyqa4Y4fwJeuEcU3Pedx3WhFUb1uSoo2SrMRlbcdSAs6zA5WA6W+BMT464b3YcI3S9eldh/EFHfXoTBHWvtNJcQtKATqoB2Y3OUxJtJ/wC7nBWv+/G5P8oM/wBq2/3dYJE/78bk/wAoNf2rNNPE+k7o99xa19BtspW9rW0/aD+UFKxrXZzJlLswuSS4X0Njc+IHPmmvhTiBjLGuOG2rvwTedz4PprtWurqr9u1yjq3a4rAQ0yhagciU5ipRQQTABtI3LXnZC/d1gnb92Nyf5RZ/tW1+7rBR3xjccf8AtBr+1YmJzWD033Py+ipK/EKWqkBgY2NoAAANyfFxO5N97DkANEvGAdNLbPaHwsgnHWCf/TC4/wDKDP8Aatv93WCeeMbk/wAoNf2rScWP3gguNH7w80sLpqVzRynaX/OSDYI3Xdm/5vpxA3S2AfeLJn7usER/3YXIOU/nFn+1bP3d4JA/7sbj2/8AODP9q3cSPqF3Hj94eaPu3LdryOqdpyUfqlao37ps3rx4S8N73UV3ngq5Ksq3VUXey8T7VoNlH93WCYj92Vyf5QZ/tWz93OCh/wDvjchH/tBn+1ZpdD1CUVLB7Y80yby6MXBG8pDvDbDSJk/R3NSt+2UNpPM8+dmbfXQR6N99uoqKrh7RpcbgJWzVVjMd2iHwJ1PK00fu6wSD/wB2Fya/84M/2rKVDel3XrTiruyup6xjNlDtO6lxBI3GYEibM4cLzYWKlZVXNmuVZaj8nnwO7Yu5q+KAHUJp7zXlBjf6VLnzFoZ49dCWh4aXArFeC6DEV+UFHT1FReTi78o2hQtNpSQspUwlTsjNojXs+It6EgiJm0ddIszwIx4P+YKz/wCGbDVlO1tO9zPRIBIIty15ghX/AGbkjmxemiqoxJG97Wlri4AhxDd2Oa64vcWcNQL3FwfOqj6LGL8WYLpMXcOqK+cRUlRSIqK9xqkYaZp1lIUtlpS6gLqVI9U5GwAoFPrAgKuAOhjifizg+hxtgfGV0P0SlraLVbTVLLjTqdFIX1aXRI89QUm1/OjWwzTcBMCNMNhCVXJTOEDTtKTmUfMkk+20edCAJbwXjphAhtrHV4pQkGAkdVT7d1q2CN/EhEjriRpJ0GhAB0t8ed1sMWpMMkosTdR04idSTNa1wc8lzHOkaQ8OcRcZQQWhulwb7qpd69ArjpQIfXTN4er1kaBiucbKu4/TNtj4izUvDof8d7qQnrMAsOstkqWWL0ondBO6Q7OuvLlb1tmDbhaG1phxKVDbtC1m6haea86bVu5heO9RwC40U7SqpfCfFLre6TTXY89oBM9hJEE2Qbw4eY1ufI3fGEr+oSDKjU3c+yPEArSPDW3syzdt3KYSldBTEARHVJ+62/zLdREijSkncoUpH1EWgOGki4KlbWgbheJ6mAl5wvLgt7Aq8NTrr5Ww0zhZJQ6VOLkJ0Go92nLW1/8Apl8LcKXbfWC8c4audDWK78xVQ0DtbUPvutupCIbSptS8uUFCJygEgb6276RPRT4b4a4S39j65lXkrE1ztIrlV9VVKdTUkOJ6wLZP0YBBVCUpAGgiNLUssdQx0oDQeHqddxa+mm9uW3ivRaHs5hlVDh8rqxzTWEsaDEDleHBpuRJ6mY+v61teHvbzCLC6e+nmRqpaFZSTuZHL5+Fn/h8LcuemSpRCUZiDqkqM7bxt7rXy4hcJOAt29GtrjbXcKcOrvWrw/Q1jSG6EMJFVVIbSkfQlHZC3ZMcgbHOEXQm4X31wqw3e+IaRTt53xdrN4vupdeaU2X0BxKEhDmQBIUB6kyDNlfBLJIIohclube2h281WRYPHTUUmIYlIY42ymEZW53Oe0XdYFzBlaLXN9yABvajSQeqzlrQAAdmATyMefwjwsIih60JZJVr2io+QmRuNxztZjhn0X8IYp4q8QOE1+XjenpGE3kuUD7FU22HqVw6BwKZXKgFN6ggGTpoLEcLdFmjb434n4P3jf1RQ11HSovK5qj0dD3plEqM260DMnMAYIkpXpAtXlszsjmt0cS3fYi+n0VzJ2Ygp21jJZ7yQRtlaGsuJInhpDwS4EWD2lwykgXOtioDTdtGpwlSEAAEDz5kiwzN2soACkjrHDMpVJ9nePPvtYW7OhzX4mx3iO67hxE1UUuFlsUj9a/RZEu1y2+tW3CHF/wAWhTUmTquI0syMJcA8W8S8YYyua5aihdRg6sF3O5+vS289ncQSMraiRLKvWjcWhkbUAgFu5IGu9t/wXR9nqR7HzcchkUTJJSWC7eIWhjGgPOYuzDcssN7EEKMjRtuOnq8yQ0kAqUfW3Mgxy52DVQukR1pClyRmTHlpI+Fpqruh7xfu+spn7vuGldYQCHwm82WwrTQhK1JO/h3WifFFyVmHsZVuFsSti7XrsbT17C30arUlKgMyVEK0UDoeXstBKZoBeVttbeHmlw7s5DjVRw8NmMjGxmV9mem0NdlLcgcbuJy5QHWIcCSNbJDl3KK1JRkIAKlKykAeA75Enyixd2leNMQQ32pGi4J3Ebc4PvsZvV+6KHqn7mqwVJKetbQ4YWOY7p8tLK9Xdb9IlC326hsZQW0qTBVPMSPHeLNjmMhLRyUWN9m5sGp4Kt2bJLmAzsLHAttcFpJ5EEEEgg8iCFBeKaZxl9SVIRmzBSilUwI1J7j+FlzBt4rZuZ5sErWl45dIEEDXx23sHjOncaTUpLQ650ZlEEEwCNO6I+uyfgwl6hqGzMZkkAmY0M7+IG9rJjS5qyLjlfolioUVOFZEqV+sfGd/KRp3xZoYop23KGtS6JCW5zE+07ctN+46c4eC0LKCqRmJmY3ju74s3L+a6xqpR1ZyrZUBrmkZfPX8dxY2n9EiyGmFwokebMEoSVlIJ9bNOu8e/wAZO9k4hSgWshyqEk+I+uIPn7rKhUEhCR1gkciCDPPw2gyO/uFla97hw+rBzWJ7pF4IK6wUxbqXG1DKAok9lI7ha4fO2EtDr+kbfNJQYPPikU8sBb+hYXuBNiWjcjSxtcX1G+iaTvZzE8gNTPw9g/CwbqUjKVhAUND2t/nSPKwhWmVKbWCr1Va+sOZHhp8LAKUuU50EJVsNCYk8uW/xsYAqcrS8hAT1Wg7XkOQ9/wA9xaqALTgTEggkk8/A/POwy8wSCEqO8iZ25zYJztIcCsw011nX5i0oTbckGlU3U2cmjVQrtfzkjT/Nt3Da8qCArMDqDrPLfyHxsq3xQ4dprraTcN6P1pWyy/WZ2svUu9oKSNBpr4+Zmyte2G8OHBDGLLlN5oW5XeiFqqebWMoSokgoQNdBYd1WxuQkEZjYaW18eYV/F2cqpzUMjewmBmd1nh126XLS27Ta4BF9CbdbNAtsxBG5gk6GdvgbZlSghKokdxiIthWSZUInxgRET9VnTgC4MO4pvZu472F4oqKnOUOU7qEpSlKCe0FIJk5eRtJUTtpojK+9hqbdEFg+FTY3XRYfTECSQhrcxsC4mwF7Hc6DkmqUBU8xpGuvPfvtpTSUkpkzMAkfXY8KemRfJoaxxxujFT1broTmWhsLgqgbkDXxsJiGnuajvh9i4a52roUkdS86kpUsEAncDnOsCYs4SguDLHUX205c+vgoXUL2wPqC5tmuDSMwzXIJuG3uWi2rrWBIG5SXlAJ0BIEertblTWsKA12k2EzbnLmmCAVaQPrtpUT2ANe4x8i0t0DZALQlI0387B+dhlp7M62CAOo8LOBTCFob29mPySuKVX30Xqm43Hcy8PYjrKVKf1W3ENPD/OcXbxnG4t6dfkZ8TkscTcGuO6BV3Xm0if8Apm1mP6P4Wp8dZnoyehB/Z+1GUDrTW6r0vVI3NucwG1tr2sGfPa2HC0LRcITfa3MRbEHYRbsAHa04F0myDUMq2lGBCo94iwx0sBUfxRP6pCh7DYxtp32vKA/o7IeYc1mlt7crYJ520QCdvCbHgKCywxFtDU23HIi2jPM2W1lxWyTt9dgqhAcZcRIhSSIsKJOgGttKEjTxs8NuEl7JPpzmZQSZ7NtqkmYsFSuoX1zCFJKmXVIUJ9XmAfYfqsMZGnfZqdey5GulhGP0gdTobBjc91ugopJyASQYkwCe75FoZLAXKdGTeyHS2txYQlMqOgsXeqrqQ0+h+vaT1YKj2wSlQGug1nwsqUoutlr0m8qmnAUkkda4AkDyJi0WYj4m3AjG7uBDeVRTuPMCqQ2wQjO2NFhwgKcAAObIlM/rlIBtXS4gKRuckA8r/t6JxHFdlCqv0mHLyu3i9+7XAlfSU7T12+i3g7VMOUja3SVNlpbriUpkozQQcwgiNALVuw9wjyXm1j/HGNLow5ct4vLTQsuPBv8AOBKiVFIAKyiDBUhJTmj1gDF4+L/EK5au42rmvPD9XeVI+UP01AxdqqenpRPYWXXWupUtAhUZwiQUybUjxDwexbjYGvxNiynpOvfW8mvvJwUgLCRAAClAOEJ0OQEaEawTajqMUdM0tdIGA7m2vy3UT6drH5g3N4KwNFd3Rhp6NhhuopXUNtJQlxdM6VLAAAJIqkiT/NHkNrZaGqTCuGaWlZpX8YXG44y2lta+oq+0oCCf4sfVbLZI0Dr/ANJf/HyRXF/uBesLwPWanmbdpBjnpbToh3mJnlbtAAPxt9DO3WSbsqs9O6hp70u7h3dlWFFirxD1DuUwci0pSYPLQmypUdCTgKyHnDQX2ENAkxeKtgJ5ixPpvE/9jI//AMTo/wBS1g65JcaqmicoWlSCY2lMfbaigoKasxGpNQwOtktcX9lZXuNNWYlUmojDrZLXF/ZVHOHNx9BnihflFhq4avFdFed4qDdLT3iXWQ4siQjOnMgKOwBI1030tODPQg4DE5XbtvmeUXkr7rGsL9F/CFx3rdl9KdYqHbufaqm3UtZVFbagpJ3PNI52ev74Tddj5WEaGkrkfm6mL1W9UUrjCFLWoZEt5wM4hKzmTKdoJgxZ/YWGn/kN8gjPsXDb6QN8gmYjoLdH9aQfzdfev/OavusXf6EPR/bVkbuq+lkesTeigB8LWCp3CGZ572jnjnSVl7cJ8Q3RQ1DjLt5stURW2qFdW88hDgBGolClDyNoxgWHX/mW+QTzguG/2DfIKN6Loa9HSvSXaOkvKqaSrIpdNfXWBKu45QdfCbKjfQZ6PSwD+br8E/8AOavus0OjhwWrOFGOE3jddW61d950rtPV0oWotKISVoVB/SCk7/yiOZtZ0VqWq30UkaoCx7ZH2WV2A4dyhb5Bc3BcN/sGeQUKudBXo9NILi6C/Akb/wC6Sz9lgh0H+jkYJpL7APfeLn9m091bpNMBO6kj4i1culRjDjVhG8MKOcJL8Yo0OtVzl4Uz1C0+mqKCx1QJWklIEuDskTm12swYFh5/5LfIJ/2Lht9KdnkEsN9BXo7uiW6O+VeV6K0+FhP4B3R9/wDN19n/AN5q+60mYQvyursKXHiC+2EUtbX0NM/VNAFIbdcQnMkA6jtKiLOxh/MkkHQizDgVAP8Akt8guGDYYf6uz9UKB/4B3R+H/wB235/lNX3WwdA3o/E/9rr8/wApq+60lY4xO9hTCN/4tprqdvN66aKprk0TTnVrqS0lSg2lUGCcsTB32tBuG+mU9fTjbLnBrGbDioSU06w8UkmP1U277Cw8bxN8gnDBMNdtTt/VCdP8A3o+je7r8/ymr7rYOgb0fY/7XX3/AJTV91p3o65dS0h3tJBSCQrexOqvR5tdS715bap5mEzoEhRO087IMDoDtC3yCU4JhY3p2eQUJ/wDuj7/AObb7/ymr7rbPQN6Pn/m6+x/7zV91l/APSm4OcS72YuPBXEajvC8KmSzSu0T9Mt2BmIT1qEAmATlBmAdLS1TXkp0AuIg8wNrIcDoR/yW+QXDBcKP/Ib+qFAv8A7o+f8Am6+/8pq+62HoHdH3/wA3X3/lNX3Wm698R0l1OMNPKbSajNk6x0IBgCQJ3Otg0YnacAUmmUoHmlwKHs77J9iUH9i3yTvsPC/7BnkFAWJ+g7wFurDV7XpSUF8h+joah9oqvJRGdDalCRGokCyn0DTHR+pNP/vSt/rC0tYtvxiswhiBhLDqT+aqsyrLH8UrxtEfQOI/g/0g/wCdKz+sLAto4KTEoxAwNux17C3NqBZQ01Fi8QpmBt2PvYWvq1WLCu+zG4wYCxNxKwnVYPuLGVNh+kvKnepLxU7dXpqnmXExlR9K31ZGuuu+1nvMcrVtxz007nwbjq/MCtcNMQXvUXFUdQ+9RrSpKtB2ogkDXnY+vmpoYstU6zXac9fLVaVmOt7Ozx12cNe0+iS0PGYaj0SHA23Fxodd1JGBOG/ELAnDcYEpeJt31NTQU9PS3RXqw9lTSMtwCHGvSD1xUkETnTEzBslcDeCOLODT95073Eqkvq6b3r6i9KmkFw+jOGrdCAVh3r1wkBA7OX22jlPTobWMyeBWNSO8N/7Nt/w5e7gRjaP+i/2bVba3CmuY4PN2Cw9fT8/miJP/AFMM0VTC6dmWoOaT9CwZiNjcR3bbcZSLEk7kk2lzd9thXeffat2CumfdeLsd3HgOq4YYiuWqv2oDDLtcpKEp0PaggEjTlaxsE8/O1vS1kNY0ugdcDTn+1VtFX09e0vpnZgDY7jXfmsaPZInmR8bCZ+c2LtfppOvbV9dhAQRv77EBGFVc6ZWPsHMXvw+w+/iGjReNzYvu+8q+mK4XT0sE9aockwRrZ39JvidgGp6Pt8rYxXQrTiq7Hk3MQv8A4cULRmDffEj32nYnTytiTqQI31tWvoZHGY5x+kFvV20t72unw18lt6btVRU8eGsNM8mjcX/zoGclwfa3COUZgObvRuNzmFdLhue4OPvRDpOHWFr/AKSqvClw5dtG5kXpTXgwy04lpwxp22wk7wCbKvCbjnhTCHDa48I8T36vDOJsOUDV1VV211I6HnlMICEuMZUkPhaUhQKM0k2nU/GbbkDYWVtC+NzZWPGYNynTQgbaX0I+PNNqO1NNWwzUVTA4wPlMzQJAHsc4WcM/DIc1wtpkB0FjveBOjlgvElZj7H3HbE9y1NzjGVUlq6aCrQW6hFC2ey46k6oKwlvsnUZSdiLJvS6ob4wTeGD+kThKiD96YOrk0VazJHpNFUHJkURrGdRQI2L5MGLWFvBVYLuqjdoQasMuejhfql3Kcs+ExaHGLo4v8U3sKXRxNwXSYduy4qxm9L6cReLVQm96pgSyhptskoa67K6Qsz2Epg6mwdRSCKm7qy5cTcG3tXvcnYa6/DRXGEY/JWY79v1RjbDG0RuiLwLwiIx8NrScz7sGS4v6ZDjYah88GcI1uCuH13UF9KC78ry5et9O83K+pUXXj5BSso8Ei0OYQpl9G/jlj2uxXS1bOC+ID6L1or6Qwt2mpaoLcWth8oB6qS8uFK7JCU67xZbmbYD42LkomlsfDNjHsd+VjfrcLOUnaiaOWsNWzPHVi0jQcvth7S02OUtcBa4ItcW6Rjd3GCtx3jy57k4V0rF7YaaQ6/iG/nqZ4U7acv0LNM5KUuOqV60ZkpT47R7iu4q3g50m6jjXWXRW12DsYXULtveppKZVQbsqEhoJccbQCoNkMI7QBjMueU2PB526Cte+zZaJ0zQXu9IODgbaAjS1r7W315nVT0HaWHDZnNpae0L4nRPaXXc9rjmLi/KBnDrFpDAAGtBadbwzibjBdeKE0OH+A9Jd2JcQV9W025WG73HrvuymCwXnqhwZACEyEoC8xURpyMvOXRdzyAhxgkRqAtSQfYDYwNtBbYI9tpY4S0l0hvfwsB8tfnqqyurKaZjIqSLI1tzcuzPcTbdwa0WFhlAaLanUlNq9OFPDe+ypV8YLuWtKxCjVUDL+Ydx6xJmzTqeir0favOf3q8OMlcZvR7tZYJA21aSk6TpaUh322DBs8wRnkq7iPHNQPenQb6O15ZlJwX6KsiM1PX1iT7g9Hwsy77/Jt8Dr1KnKa8sSUJVyYvFEDfbrGXDzPO1rsx0kyBbYVpofObM7rFyCdxn2sSqB3x+SL4ev5hcnEvE1ICmE+kCmqo8wG2iffztDNZ0Ar0b4pDo7XPjBq8V0dGcRqr6yhLSAgykIU226oqAUtI0WD7tfWQKCQVHSNzaptfxFwxhjpw1V/wBfedObmqMIN0VTeLbgXT0mZ1BS46saIRnShBUdAXEzA1tXYoxjBFmNhnH7Vvv/AE+ZO+WvNPHncKaSwte5uy2h0J6Ag3OljsqVcafyf2PuFWCKrEuLqzCtTRofRTU9VczVSxUMrXOVTiVthBTIAMLJ1EA7hoVnQp4zcH7kRivH+GsMV+FnFJUK1l5qoXLoAbgOJS5lOmgTprNvRbpg42uTiPhC6eCvDq9aHEOJsU3rTZKagqEvinYQStTzqkSEJkJ1P6OY7JNnD0vsJV6+i3eFy3Sy5Vm427vcWECVFlhaErXHcEyo9wBPKwNQDkqY4jdoaCOdnWN7fKxWswUtbU4DV4i0MqJJ3MddoaXw5ow0vFhcZi9ocRqBa5DRbyVT0OekzgyvVe99cFKipu4DNUOD0SvabpwQpThDa15YSN40E2auKuCPFhV/O4qwxwaxNXXC6tLtHU0uHKh+gdSlISogobLakhQUDB3Bm3tpjri3gO9eBF5XrcF+Xbe1Xe+HX27voqd9Dj7r66ZXY6sSoZNSuR2AlRVEGyZ0QKnC6ujnhT0qouzrqSlqVVWdbedpJq3oK9ZSIjfwsY2Zrq4WIILDfpu39iy8lBV0vZOV0sbmuZVMDbjUehLe2lwM1vC46heIPEW4X6W5rneXhxF11btA4a5tqg9HyvBTfZWkAZVDtaHUa2IVAI4J0o2P56O/8xdvaTo2OYYruLnG+nqa+kWmpxMg0raavKH0/TzkAV2x77N5eFOEmIembe10X1d2GbwuBzBqHUtVbVPUUyqnr2hMOBSCuCrx1NhI5/8Ah4AeUh5+Llpa2kf9t4qWtIvRN5czFASdOZNyfG68ecL3HdV1YPrsdX7d7dctDopaKldkNqWSO2sDca7eB7wQrcL77ob5xtSmpuSioapCHlsOULXVJV2CFIWicp0MggTIjnp6kcZOj3wT4f8AH3D9Tjjh5cdNwpxPRGjdCKFtigoLyAVClpaSlLeaEHNp6yzPZJszukr0duiFw3we5iLg9hy6F4rS404hy57yfqU0dKtYQt51IfW2hCswbTmTqpwZdiQyrna+CqM5Of0gNdhbSw6Hn1RPZ6lnpsW7Px4XEDTvET3kMBzScQ8QufYnMwgAAkZQG2GuvlnSXgu7cSvuooaOoC6hbRRVMJeQAXNwFCJ038TZf4kIprg4hOKu256HqKYMOJpTTjqFHICQpCY0JJnvt6nVn5NDooXphFnGNLfF4ULqqdL63ae/0dWakt9YWz1qHBMmcsgxaJOG35Pfh70hsDNcTLw4lV1zXtUB8VNO25TONsNsrLaVqbWEqCYAJJXEnlY51TH3uMnYscD46tt+1Y+KKpb2arXAelHVQlpt6t2zk2uOZDbjY2F1SXFz9wXfcmG8SIwtdHptfTKcNKin6qnmElS1JSQVRMAE8zNkvEVDcmIMBsYxo7npLsrGK30SpRRpKWXAUgghHIiU/HU6WuJdHQBPF6+77wHcPFSkpXcBVRumnceom3fzkk5ocATUJIjqphOca7iNWtiDoPYyuG9XOAtBjrCVbelUj8+IrXXnWaVKAotlpZShag7LZMQRqNbAwTRxwxSNcc2cjc+rmIt0tZbDFWzV2KYjQSQtMfdGyWDG34ghicH3AzB2a+x2uNrqr2GLjua68H1uO78oEXgtDwpaKleJ6tTmkqWBuBJ0n9E+EHuHFdcOKsXMUV94WulDoQ64z6PTBLa+wZQ42ZSREkGJBTaeOKPQy428LOH9DhHFt3XUl2uvAGhrWa6KJS5HYceeCOqOqiCsJTA30MR/hTo3cWuHOLE37i24KNqhoKZbzz9De9HeCUJchpBWaV1zqwpSwAVQCdBNpKieN8FU+R9njMG62sLaWHQ/XVR4HSTQ4pgFPQQB1LIInSHI1wc8yESl7iD6TNAAT6IDbDXWDMTXgq8L2dV6DQ0oYUplKKSmQykpCjBISACdd/AWuV+SNxGq6uknedxKdyt33hmqaCSfWW06y6Pgldq6Yl6NPSFu2rfrqrgfjpNK84p1uoRcFU40pCjKVBaUFMEEHe0l9A1vE3DnpecPqm/7ivO60Vda9djnpdI4zPpDDjSQcwH6Sk2uqprHULmN91eNVMksuIvllJJLjqfivchRkEC3HO2yrUxbQ8bYMDVXAWeywiDMgW4GvhbYGtp2Liu3EZ21IP6QIttpWdtCjzSLYk5hB3txTK+iUjTsLUny1kD3EWt6A2JCGl0CFHstuLaEHfS2xMaWtQEPdZzmbaGWBytsnTe2R7YsoSLIIEgzblRG1t6T9pthnYWlCaTqk1CQitqEJSBnheg35W7VvqNTbmqBFc2sH10FJHxt2qVCZmzLLsy5G4IG3Kx+6ruFW+X1tpyta7bq5CxFsdY+3TI1ccMJTv7fIWdvUeg0JbpU5lJRKdPWV3n22Hka06nkkfIWaDcpt3xQ3Ve7qziR5hm52VJbDbjvVpfeUYhZkSASEhM6qJkGBaMuLXD3hxdlwV9fT05w2mphp28aeudoHHgogqSMiVuPKMJ7GQ5tdbTS8GKTItbWcNkFAy5lZoiR46n3myPifCVFjK7Td+KqRD9A6tLhogshKinUBwpIzCYOX1dNZtS1VFHIx0b2hznc7bfPlbklZKWuBB0VSmlPXtThq7r0x5f13UjfWJXWMuPtKUlKyQy0ptxSFQ2YByLy9ketBrzih3CWIcbU9wVeNsVu1F/VyqZ2gqaPq1sNkpzuZE5glAyz2koACZCXIMXz4m33U1l10l1YBoFIYoX6dKKimYCmlpCi2tlkJICYSCCpRSjZIVMlNQ+N+M2g1fdVgm+Lpuy+H7qDFe3dRbdqWVtocYCHKhBkAdahTeTKFDrNJFsRU0XdpsrXFw68gfI7K3bZ8WY6eCZdexgalrqimp+iXxIU006tCCp2vbUUgkCUJpyEmOQJA2BNstFf5g4yq7V21V8ikOtOHXkhYb/RzAqEGInQWy0n2fBzkP67lX8c+79AvY58Q9GtugNI+NtvoHWjvNsSNJO9vdnHVZluyq/03Y/7GX/9To/1bTtioKXcF8tgxnpKgb/8mbQX030/97LnOKEf6lnjxJPGtjFFRQ4aq7jdw7eGVvq6hhQfZQpIS52huZzEd0idrVlA5rcQqsxt6n+1UNIxzsSqgB7n+1V84H8CMX8NcY3Bf1yYvvIUDdQy3V0aqglp5gkJWlQ2PZOkzECNRIt1ezbBvKhdW2CvK4lJjUDs6eVi7GGaujuFxyhS2quaZUphKtElwCUgnukCzDwXjTifizEjd1Yu4d010U9Iha/TGq0LBWIGUIJkg7zOmW16ZGaXcrURv6KaaUxTb7J52b+M2/SMLwntAu0yjPd1idbOWkYUljIoco08rRViPindfDyvew5xCu2vau5wqNJebFOp5hTRM5HAntJUnbQGQAdLdG4E6JHtICdtzMIadplpSIEjbbsKt3VVGfE5QmIRStzrzzKP1Wjt3pJcDrqpevZxc5WOAQ1T09E+XF/yRmQBPmR52cHDWuvXFyXsW193u0KbzdLrNM967TAAS2FT+llSCfEmyyPF9V0cbgFIj5IpWyR/fEcv5QtCPSJ6QaeCF44cp3eHL2Jqa+G6px19moCFUYZU1HZLas2brDzEZec6TNiC8KK5LndvS9KxqlpKYoU686oIQ2nMO0onQDW0Z4pc4WcRaqkqazG9wPCjbcbaSmtYcBzkT+lp6oswG+ycR1SrTvUHFDD+G+INzXveTFGtlN409GVBDTwUg6PJAkqTJjtQCNuYf11uq6lIVMxr32jJ/GuBsG3VdWA8OXnS1tW+pFHSU1K+hxSEEytxYSewkJzHXcwB4SRdAWqnQpQiR7bId12qRsVVdxXfhS+7wxS8pm56WkqnrwdKVKy0yUqU4qEgqMIBMAE6aC0U4X4s9GCqeZbw3jih655SUNIcZfQVKJAAGdsakkaTaW8Q3c3fmG74uNTYc9OpqmmyKiFZwpMGdOfO0ZYS6PuE7ir2rxew5QpeaUlSVdUhRBBBBkeNopNdxdSxac7KaqE/RgIAyxuLJzozGvbVspwgnzbSLKdIlDbQQFaARZNUgF28Ez6yjy/5NNns2smP3UGYT6MWGsLYoubFDQp2qu6ahFShxlBCllMmNRsZg+FpaubGFLe991l3ULNUhN3rDLqqimWxmcgE5UrAJSAU9qIM6EjW1UeF3B/HvDbGlwX1dGN73dudNY23VUD1UtbTjKyEqSRMHQzrMeYBtbspaNawsphZSoBUcpGnsk25wyhK3XZIvFvhZcHFG5WKG/qMPqpFKcp15yktqUIJERuBFkPhZw/pMDYArcIUilpaaqKwJJUSQHBm3Jme0bSwwMzIBJOlkNKUpfvFsDd4n3tp+6yDUG5S3sQLKvnDHhPeeAbixFeVTia+rwS5cdUwhqsq1ONtgtkkhJ59ka+Js5+geQOANKDH/bSs/rC0hXiw2nAt9qSkf9qanlr/ABKrR50EUzwBpD/zpWa/4QtQzAjE4rm/oP8AxaqyexxiG3uP/FisWVWpThmVdJDjKATHpNOSBz7Y0tdUneNLUvwinP0kuM86w8yf88WWv1qKb/Gf9jkmLH/iqT/8n/8ADlOGBcR/m9x27XFK6p/toE+quNfePqsm4YvqoGNE1qn3VIXVPFQzGDOYbbbm0XVmMK65uNYwZUyilrsPsXldrh/8oafeTUI88hZV7FWeODnahVY07Utqbc9Of0Pd1ysp9qYjwtZ7MAKvBq64SPxve67pYcGFSd3f65taoGdB+Nql8YlFXSp4MEmf4z+ubWyBgWqaD+kVP+Mf7Wqkwv8ApVX/AIx/sauWvWcnkvn5CyVjRxbODb9daUULRdlUpKkmCCGlQQbKjZAW4NZzAx7BZHxyf95WIP8A2XV//CVaxk9R3wKt5j+jd8CqE8IuE148ROH9FjC8OKWLaepqqmoZWwxWqyIS2QEmSSSTr7rSlT9EujeuQ327xyxu2httSnUB/MUFO6Zza/iLEOi4D+8rdZ5GurRz5LH32k2sxMwrC+Im6KqCxTMvtPJCh9G82jUEcjEHxBFszhuG0cjITMy+ZoJNzuR8Vn8G7O4fV4RFUvju8gEm7tfqmFh3omUWIusDHG3HrKmkpUQt5Os+S7LZ6ESYkcdsbe13/atJ+C31UdUEtrJDjXv1FnTjC/Hrpw1VVrJ+lCMqCDEKUYB+M2sKbBqGVlyzX4n81YVHZnC4n2EX3nfmq2VnRVwtQVCqSq6R2MUuoOVaULU5lPcSkkA+G9lK7uhndl7M+kXb0hcYVDY0JbqJynxGbTyNkTiRdfEC+7tYawBf5uupbUt51YaCi7p2USZgbkxrt4guHgRi3Eyqe5q/EQU3eSnDRXgkADMQsoJIGn6qtIEjS0TcIpHN4hj0vbd35qQ9l8KByCPW3vO/NCL6ECUJzK48Y1Cd567T+tbEdCBC0haOPONSDzD3+1adsd3dU4iwZel0UTpbeqadSWyFFPaGoBjkSAD3gkWh3hPjFy7qxPXy0HR6JVtqPquJOhI5EH4KNi3YBQtYH5NL2OrtPqhm9ncMc4t4Wv8Aid+aTB0HwTA48Y2/pf8Aat1/AcAEnjxjYf8AW/7VnhVYtvO+8UoXT3rVsUfWpaQmneUjM2DqdCN9TO+otMKK9K6dT+YhET2u6w8eEUEriGs28T+alk7L4bGATFv4u/NVs/gQJ2/f4xv/AEv+1bodBwESOPGNv6X/AGrOriDxrqbgu+sxCy/WIuukgIRRU4effkgAgeMzuAAJO02VeEHGlrHdBQXizVqqbvvMEMuOthDzTgJTlWE6bgjnyIJtH9l0F7cPS9r3P5qUdlMNAvw/G2Z35pgjoOf/AKeMb/0v+1bX8B0c+PONv6b/AGrWNvy/WbmueqvV9JUilbLmUEAqjl7dBZlL4wUVMGzeFIxThw5UZ60JzmJgSkTpZ8uEUEIBezfxd+abF2Yw2UkNh2/vO/NQzf8A0LVXNcN5Xu1x0xqtdDRvVKUKegKKEFQBhXhZ69Bu9LxvXgNS1d6V9TWPm86xJdqHVOLIChAzKJNn9X42oMT4MxMxSsqQpm6KpZJWlQILSxuLRt0DT/8AV/pB/wA6Vn9YWFhp4Kevj7uLAtdfU9W9UIzDKfC8ZiZTty3Y++pOxZbclWMJCklJAUk6EbzZtXJwx4aYaqHavDnD3DN0v1DCqZ12huinYW4yogqbUpCASkkCUnQwO60ZdMTGmKMCcFqnEGDr7qbqvFF40rSamnICwhSjmGo52iagwL0kayipap7pXsUy6qnaqC04hct50BQSTESAbFVNaGT8IRF5aAbi2l/iR0Vs/tBVUVQ+gpI5HXALgwgAg3tcFwvsVbW4sG4Nwu89U4awnc10PVH8c5Q0DVOpz+cUJBV7bLSlJWktrTmSoQQQCCD32qPd3CDpQ3ooou/pXU1QoCSGklRA8QLHTwG6W4ST/Ce0H/IL+6zG1slrMp3W8Mv7yhmxzEJn8SallLupLCfMvVkLuwHgO6VVq7pwVcFEq8UKarDT3ay0alChCkuZUjOCNwZmxe7OGHDG5aSvoLn4c4YoKa9WRT1zNNc9O03VNfqOpSgBxOp0VI1tXdvgP0t3EkjpPHT/AJBf3WwcCelspZQOlBqP+QX91o+8P/8Aau+7+aJ/lTi+oME2tr+m3W21/T5cunJWDunhLwnuC8Wb3uPhhhK7q6mJLNTSXLTMutEgglK0oBGhI0OxNi7HBTgtTuoqabhFgpp5pQWhaLgpApCgZCgQ3III3FoIPAXpcJGY9KHT/oF/dbSeAvS4UJT0oJH/AEC/utGZnbd1P3fzU38sMcJLuHPc6H0xsL6ev4nzKtLeFBd170bl23td9NW0jwh2nqGkutrHcpKpBFkZHDnhw3cj+Gm8A4bTdFUpC37vF1MCmdUkhSStrJlUQQCJGhFq6HgJ0uRr/Cg/0C/utg4B9LoxHSg/0C/usrqqR2rqd33fzQ8PaLEoG5YqaVouDo5o1Gx9fccjurDp4W8LUXIrDA4bYVFzKqvTVXeLmphTGoy5euLWTL1mXs5omNJtl18K+Fdyt1rVy8M8KUCLxp1UdYmmuamaFTTqIzNOBKBnQYEpMgwNLV5/eD6XY0/hQ/6Bf3W3+8H0ugP/AAoTH/QL+6zOO7fux+7+an/lXjJaW8GexNz6bdT1Pp7qwlz8J+FWHLyYvrD3DLCd13hTElirorlpmHmiUlJKVoQFJkEgwdiRYrVcFuDVbVO1tZwlwW/UPuKddecuClUtxZMqUpRbkkmSSdbQJ+8J0uz/APihJ/6hf3Wz94TpdH/8UP8AoF/dZvGda3dj93808drsbD+IIp81rXztvbpfPtqVMfFbEmLLlvy5KRHDR7F+C69mpbv1ukpW6qop3Jb6hQZWodYj+MzAJJ0B3ACoo4lYbPGDCdPwg4U8I7xwlct63hTv31eldcabnYpqZpYWQ20oJW66SlMQmO8wZBL94Tpdf/mg/wBAv7raPAPpcc+lB/oF/dYeYzTFwdE6x/w7dAb7FXOFdvqvCGwyU+GP4sWrXZxlLgSWvczPZzm3GU6D0W3B1vZhi5rrpKZmlZoWQ2w2ltHYEgJAA18hbly6aFcEtFMbAOKEewG1ZzwF6XHPpPH+gX91tHgL0t//AMzv+hX91iDUSWt3d33fzWb+3K4m5o5L/Fn7yso5dFNlIbdfR4h0n65sCbqXMt3jUf4QQR/VtW88BelqP/xObf8AIL+63P7wvSzJj+E0de9hf3WhJzf1Z33fzUox6vH9Tk82fvKx66KuT/F1rR7szE/UoW56q80HX0VftUj7DamvE6l6SfBO88I1WIuO1VfNLft8tUKmWElEJzJKs2Yaggxa6+nts6mjiqXOYYy0ttvbn8CUZhuOPr5JIXxujdHa4db2hcbEomFV6NTRtkj9V77wLELpqb3cdq1VF3tNKU7mDfXSQIjcAg+r8bLRnnYq2OrrFiYziY8o/GxbaRsRBarfjFwsQuku1UdqkPjlcB+62zUFMzSviO5IP1Gw+wnkbbzEdwsWGHqocyLGtYGig6nzaV91uReNCVQqqQCBsdPrsbOuptqARBHvsmVw5pbhBCpp3BKH21eSwbdEzsR77cOUtO4IWy2rxKAbALuu7lKUfQ2gVbkJj6rO9JNJARe93006WH1BUJeSNBsCYtwKk1Ci3TQcuil7pT95sSv/AA829ddSihedYdyKKFlxa0pMb5CqDz0sfo0JRTNJbEJyiABA27rd6RcQuOUC4TiuS6BQoNU4kl531ir1o+yy0pxKWS8oKIQCSEpKiR4Aak+Fkamv1lFMhFRmU4BBMd2xszb34p4swuHKm8OHDt4UZeKGn7rvBDxSiCQXUOJbUk6HRGfWAJJAsDV1cVGwyTEgDnYlRCGSd1mjVP59lQPp9QVN9iAgq7LafHlPefZytEfFbGOJr/cRgvA1IyUpKXa+orFqQ3VNZoVTtoSC4tJkFakwMoKcwKtG5jvjpfV8VSMLMsU1xuuKS4Wi+ldW82BqEpMBMEgElKojYcmlfGPa64rifRTXbcjVTeTuSoXWLLqnihIhTjisynNCAM0hIBAna2GxLH5qh7Y6EHhm13W1PXe1lcU+FuYzNPa/S6BxIxeqLjq7txbgSr6x4LNXeFE4zeLZMABRp8yGW4bSlKUEKKQEJAPZJrVxBuHhPhk3beGFq29V0F4uJZdK2i26l7RxLjza8qUoJVCUpQkQonWbTHe+P8SIoj+Y7yw8qqfQptCKa7mlNocglKisNhORJgyVDXSCTBhq/sRUVVe3XY1fqsT3wghCVslVLdhEpIBQhPWKzRrAbcJSeyop0qZJ3SSaA26ImSJsbLXBKies4h8TqCrfoaTDWJqhincU006vqlqcQkwFFWQSSBMwJtlp0XV4OUtSmMLgNkko6u7M6MvLKpWZSh3EqJPMne2WZ9rRf2H4/mg+A/qvRytSA+dNNgBbhAnWw9ePpvM8rBJGgI3t7/usiqvdOAEfvYyP/wB6Ef6lrNP0TbypUmTaOOO3AW7OO913Rdl54hrrn/NFUqrbdpG0qUpSkxHa2jfS0YfwG/8A9O+Nv6X/AGrURFZTVkssUWdr8vtAbC3NZ4trqSummhhztflscwGwtzVl00qUoyhO+liyLmp23y+hoBXMga2rl/Aa/wD08Y2/pf8Aatn8Bo8+O+Nv6X/atKauv/8AbffaiBXYmP6p/qN/JWcQ1lTFky+cPUV8NFmrYQ4lWhCkgg2rt/AaP/r4xt/S/wC1bY6DM/8Aj4xv/S/7VkNZXnQ0332rhX4mNqT/AFG/kpaZ4SYVp6v0lm56VCwqcyWUg+8Czxu66mqJtLbaQlKRppauZ6DI/wDXxjb+l/2rZ/AZ3njvjf8Apf8AasxtRWtNxTffannEcUeNaX/Ub+Ssffdy0V+XTUXTeNM1UU1Sjq3GnEhSVJ7iDaGby6MXDypdKxh+lSSSeyjL9VmuOg1rH7/GNh/1v+1bX8Bgf+vfG39L/tW51TXv/q5H/eEjMQxNv9U++38lIWB+B2EcF1vp103Ow0/EBzLmXE/rHXnaUqelyN5csWrX/AX7uO2Nv6X/AGrZ/AZP/r3xt/S/7VlZVV7P6v8AfC59fibzfun32/knTxJ4U8QL1xNWX1hbijf9z09VlUKRl0lptQSAcqQQIMT5k2a7PDzpAUaoa4zXkpO0uUKHP6x1+FuT0GATrx2xqf8Arf8Aatz/AAF0zrx0xp/Sf7VkdWYm46Qfeb+Sc2vrwNaL/UarE3FTVKKBhuufU8+htKVuFIBWoCCqB3m0ccSmOL1JiVC8CV9zpuqoQhLzVZTKUtLkkKUFJ3EZdO8eNo/HQZj/AMe2NR5O/wC1bSugwlX/AI9canzc/wBq0oxDEW7U/wB9qiNZiJOtH/qNUt1mFqtGGKj83ttqrmmCunCpCS4nVIJ5agWbXDvEfEzEd8OUmM8KXfdbFKgqS8xUlSlrJAyhBJhMTqTyHjZkfwGOX7+uNY/6T/atwOguEqkccsZCeYc1/rWV2IYk616b74XCrxAf1T/Ub+SszTslLYTM6crRRjnihRYGxW9cddhrEFYqtS260/Q0XWNGRlylRUO1p8RZgjoNxvx2xqP+t/2rAvdBVKyFK434xWRzUsH/AFrcK+vH9W++1cazEibmk/1G/kppvNl5OAr7UtBH+5NVv/0KrRt0D4/eApO/86Vn9YWQP4DDbiFMu8ccZFCwUqSXJBB5EZtbTbwW4S3fwWwO1ge672qLyYaqXakP1CEoUS4QSITppYePvVRWsnmiyNa1w9YHcjp8EyBlbUYgyonhyNa1w9YOuSW9Pgn0oc4tTXBCmW+klxqdqHUNNtuMFS1qypT9INSTa5Zi1CMRYnOAePHFk4j4Y3/iO6sSPIpkikplZewpKwrMUwRIG3MWdiszKeSnlk9UPN/1XJMdlZTy000hs0P1Opt6LuikLi5hb90NPRYtw0609emGyirplMrCs6Uk52iRyUhR074s4MKXg3VpoLxajqny0tHOJj59loywb0g8M4Hu5y7Ll4AYwaZdWpxQ6knMowJJIM6AD2Wa1PxKwtR4wTium4OcQWymo9JTSgqDQXMgZQnaY08LSOx7DnNDc508D+SmHaHDQ64l+jv3VJnGurao+k7wdqns2RsOEwJPrkWtfQ3lT1rXXMlYE6hQ177ULxXi+/ePXGfAVTceCsS4XF3FdKaqppiS2pRKgsHLlERztcbhrcF/3BcvoOI7+qb2qi4VqffQlBgwAkBIiBHnqbD4XPHPLPJGbgu0391qZgcrKqWqniN2l4sdvYb1T3QQHVkHusk45j9xOIAD/wDddWf9EqysiOtP80fbZJxx/wBxWIP/AGXV7f8AQqtbSH0HfBXU38274FVT6LLYPA65lbTeVeD7FJNk272HsKcb8VXDejinbj4gNIrqUGQlFW0whl9r/DQEHlsbC8ARiqn6M91XjhG4UXvVt3zWoWwpwIhskSqTvsB7bSHevDrEGN8DUl43jdrd34goXE3hTNocCyy8kkhGYaap7J5a84tXYVEH0sJcdCwD4FCdnZjHg8AA1sD9E88Jyy7SIKyvI11eYjeI1su8RE58G1qp9QIUY/npNo04X4oxPf8AiE3feGA7yulunQovVFSsBOYGAAmASTM9wFpixDc7t8YavC7ElIXUU60IJ2C47Pxi1jQMMTcrzzVvXPEjg5vRRlginaeqHkOAHsjQ+2bNpqmbu7E1eywMobvErSkEDfKon4/H3IuH+Jl23FeLt238+m6L3opZqKet7CVHvB25Tv5SLK1y3xdGKMTFdBeVLUv1b4fcbpnA5lACQToTGiRv32bOTDCYXDW6dE3iSiUHSysLd5LzISoesJNoM4hYZXhbHiK+iY/uC/wtTqUjRFQkST/hCfMk2nW6WVJZSVJIJHdZm8ZKdlGH6etdhIp6tKlE7JBBB15CxsRD2lh2IQjyWuDhyKaWA7sFXWpeCISyAkCI1/ZaVL3aW3h+tSgwoUzkeeU2YnCd5itpqlTS0ryPDVJB3SPuNpNqmQ9RONEGHEFJ9tgKJgZqUXWPzusoEue5qS/qemu2raQ408hKikiZ7M2K4D4G3nhPGDS7uxVWM3BT1ZrGrtbZQEyTmylZBMZu6NBHjY9hC8WqR9sOqBdu99dM+kboUCU6jcd9lurxXiF6/akXbWFijZDaWShCVdZIOcmZnUx7LMe0QGQOOt9lMwumyZdrJ/47aL2C72TufRHFQfBM/ZaBsWYEw/xFueiocR0iX2adxNSjUgpVkKZBBBGijawl9p9PuGuYBCutpXEkd8pNq+X1hZeOcIUlzovK8LtWvqHA/RP9U6CncBQ5HWzq8foGOBtZ34hNoT+le0i+idmA8D4PwDw9xPd2GaRLT9RdtS5UryqzuANLCQVEnQSdPE99ifQPP/YApB/zpWf1hY1gPhw9gXAeK3qnEF83u/VXZUJ6y8axT3VoDSjCAdBJ3O+gsV6B/wD3gaTf/tpWf1han17/ABXN/Rd+LVSVpBxyGwt+jf8Aixb6eH/g/wBX/wC1KL+sbIDLNc7hVxigqeoqzRZWHcoUUOFkFCoOmhg66aWX+nhp0f6sf86UX9Y2JUbKmWGmdgaGjWD/ADqdBtIW3qKk9GMP3im4e63aKT/Az/8AtMDo+Yv4i1jF5Ix46F3xcd5Btqqbpwz1iCkESEwn8Fa2uFXVbv5rqXaZeVzqFqQe5WUxanXC7iZiy/saXrg7F+HaSgrboQmrYeplr6t5GeBovv335ERa2ly1v5yuZmqIA69oFQHJXMeUzYum9GchwtextyWkqdYQQb2uLqMeFF9VS71oqp+red9KZcbcK1qVKoJkzzzJsnX7j7ErHES92k3tVtM3dVtJQy26UILWUGCmYMydT9mhfh8VUdc3RKVBoryepjOn6ZH1GyTjtgUfFO+GwSDW0tNUjTuTk/1bE5Ld6Z0JI8wVCTmdA7qNfIqx98Xv6BctbeaEhz0enceCSfWKUkx8LMThdjy/MQVqWr2qmltP0xeQlLSUwvsnQjlBO/2WclCoXthFkqJPpdCkH/Cb1+u0RcH6st110IWoyELp1eYChHwFopf5hsg94fVPhH6UsPQ/RSzxJxwvB10UtY1SCpeqqlNM2FKypSSCcxjUjT4ix7A2K3MVXKbwfpEU7rbymVIQsqGgSQZ/wrMbj80teC2K1IJNHeLDpKe4kp+tQsa4NvqFFeFMDol5Dm36wj/Vt1S3I2Nw53B+SbCMxeDyspTzaSTAs0H+IN1NXjU0CBVOrpnC2sobGXMBJAkiY2s6sxLR8rQPW0zSsR4gut1RCHKh5C4JSQHBMg8tDaKoJjpXys3bb8bJ8QDpmxu2P5KXLrxnddbUJpS8ttxeiQ62USe6dvjZwFYgkjYWptga5MUcOuKNXhC8MSXneV1V1CirovTX1OlohRBAKjpsqYiRl7pta6769dfcjb2cl1bRSox+kBBPvFoIXScQRS21AIPgVLI1gYXM5GyFoMU3FeN4O3VS33d71azOembqUKdTG8oBke6yhUVCWUhau/mbU/4Q1xuzirdTzhhS33KdU/y0LR9cWlbpP1lUnA1K3TuuJadq8joSqArsKKZ7xItZd0JqnUwOyEEw4Am6qaBUoU0XRqPO2NOpdSVDkedqQ4I4u3vhPA2I8Mt3g4walkVFE8hRzNOhSQsJJ9XMjbaCnTe0hdE7F17V9+X9dN7XxXVgVTIeYTUVS3UoCHCDlzEgSFp27rDuiyxCXxspRJeTJbldWcW4htOZRgWDD7SuyFjytFnHni29w1w+0q7KVuovKtCuoDv8W2AUpK1CQTBUIGk9+lq03X0leL9DfdKi8r8D6K1AcYbqLubSy9G4SUpBI03CvC0LhkIBG6nac1yOSkrpw/8Aixj/ANKEf6lrOSZjX32pz0lcdUvEPCPCbElM2GnHMUJaqGZnqnk5MyZ5jYjwULXGMk6e2bVtKb1c5H938FQ4eCMWrQf+n/tK0TpYBaQKhtXfpYc2BqCYQs/oqn2fItYELQDQoXlB2tsd5FsgeVtDQWck3Wz9ltGPttsxbRjyty5c8tPbbABvGlsPrAW3y3iygJhN0E+jOhSDsdPGyXdi1Lo0A+s2S2rwIMWWCkEH67I13pW1WV1OUFIDwdST3KE/ZZSNQVwNwjfOwrFMmtYqKUhJJRnROuqTpYMjaOdumKynoFmrql5GUIVnVBOkcgNTryGp2E2HrImyQua/ay5ji112qGeJGFsPXjeTt1X1dVHXpqMj3o7yEqKQBlLsK2CTOo11jnaM6/At61jTCsIY6vS7kS4S7Vspr2yhxWYltLpCkJToE5VpkAbxmtM9ZdFZeV53ziKvFRRmpdDFPTvsJ6xlpCElBO8ElWbLMAklQJEBnYqp6DD+Faqovlhw0bjakhVQFn0lRSYQVASSo6bRrGgtlw2KcOke0NbyVi57so1uVEd54XGOL6euypvuhN00CA+2mmpwHaphQAzvP5lEhausAylCYCyEiAVVd4jXNfT+KL3ufFdSbruNNSqgpKupvVa1u0KV/Q5kpKnUsDIlUHKgwSIJm1vboprrui4VVd41V1O3teb4rq4OOoS1TKIEAI/kABKUnmJ5E2pn0uMD4uxBj19jDL7t4XXVei1zRp6rMwXVIDa1rIkKXKJO5GcAAARaKnpKZjgGkNvueiHlkcGZzqpFosV4dpqKnpqfjNerDTTSEIaZut5baEgABKVEypIGgPMWy1XKTAGOmqVlsYppWcjaU9WFSEQPV0MabaWywxwSkv8A0n8PyUXfpPdX0CXgmHpidbAISAZiLRkrpNcMa5wFKr2ZkxDlIlUHxyLNjjHH7hY4QF37UNHnnu6o09yDb03vtOfbCqjSTjXKVIoEamTboJ5RttZmMcZeGFSQG8YUqSqQA4063/WSLKLPEjh6/wCpja5Ae5Va2k/E2cKqF2zx5phglG7T5JxjvtmnI2TKbFmE6whNJie6HyZIDda0o/BVlFqppXxLNS05PNCwfqNpBKx2xTeG8bhdhI57W3A01tsAny8LbjTXnZbg7JuUrkCdYtqNoHlbuBbCmRHhvbrrrFcQCY2tsJncW6jWe+2EREC3JVzHdrp77ZljXvt0BHO2QeQHjbly5yneDbRAsIJ521Eam3LlwE9wtoDwsJGvnbUDyty5cRJ+G9tQDyt2U66W0Rpyty5c5ZtqOYGluyI1toA6aW47LlxHO248LbAJNsIFm3XLURbRAIi2yALYYnQ2UFcgy0gnbUW0plJI7NhY5WwjlFkK5AllJEZdLaFOlGosPvytrnZFwNkABD0EfofbZJxx/wBxV/8AjddX/wDCVZaI+mTH6p+y3FVSU9dTPUVY0HWKltTTiFbLQoEEHwIJs14zNITZWl7S3qFT7oh8ZuEmC+C1Ph7GeObsuu8E3hVOmmqFEKCFKGU6DnaZ09JDo9oSUp4pXJB5dYr+zY//AAXuj9/6q7k/o1f2ra/gvdH7/wBVdyf0av7VqWmp8TpYWwtMZDQB7XJZ2jpsZoqdlMwxENAHt8klo6RHR4adLyOJ9xBR7lq+6xwdJno/hMfvqXJ/SK/s2M/wXuj+duFVyaf8mr+1bR6L3R/A/wC9Vcn9Gr+1aYDFBtw/vIsuxs78L76ZeKuKfRXxYQu+sb4YrFjZTycxHkSmwGGOJ/Rdwqo/mbHeGqNJMkMgon3J1s+/4L3R+2/equU/9Wr+1bX8F7o//wDqruX2oV/asx0eJuNzw/vKRkmOtFhwvvouz0meADaQBxTuQR/yiv7NkrFvHbo2YwuOouG+OJNxv01TGdHWrGxkagd9l09F3o/k6cK7kH/Vq/tWz+C7wAkf9iu5Y/6NX9qzh9qtFhw/vJv/AN73/RffTMwJxc6M+A7uN13HxFuGnYU4XFAOLJUo8ySJJgAewWdv8JvgARH76dyf0iv7Nh/4L3R/58K7k/o1/wBq2HovdH/lwruT/EV/as0fag/s/vLnOxt39l99QlxExZwUvW/3sWYE463Vct5VI/ulEF1h895QQNfH2xJJIGC+J+BaK8Q/i7jjh+saQZS3TsdSCe9R1J8hAtOf8F3gBz4V3L/Rq/tW1/Bb4AAf9625f8RX9qzJhik/rmP7yfFLjcIs3hffSU30j+AYpVMK4pXL2k5dXFfdau9VxrXdFQ5RXVjLAtbQsrysKVUPZy2DoSdNfZGvO1nD0XeAA/8AFZcv9Gr+1bhfRa4Bq24X3MPJCv7VnvOKPZwzwyP+5dHJjbHZhwr/APeq9XBx+uerTeLGIcS4YoaVV3VaQpmtWtx1xTK0oQExGpVz+u0pdBAH94KkgH/tpW/1hZ5J6LPARKteGFzEfzFf2rP3CGDMMYDudOH8IXLT3Xd6VqdTTsAhAWr1jB5nS0FNR1IqWzz5bAEejfnbr8E+GmxGevZWVpZZrXNAbm5kHn8FCXTwH/1fqsztelGP842TMV31S4Rprjrau7rxdRXXJRKCqWlU6AUspBKiNBsPGyp08R/9X6r/APalF/WNpbw1c9JeeCcPJqGUri66WJH/ACSbSxuPfp2WvdjR9XJKQWx2V19mMP1eq43FiHCF7YjRW3bd9eq+HWPRBFC8k9XnzQoxlGvNUR399lcM0S6O56elM/Rog+Z1PxNuqfCN3MO9Y3SoB8BZdYpEtJygaARYiGJzJM7unW61M0zXsyhVkxFflPgPiTfF0Xy76I1XPi8aF5ww25nOonl2pH+CfCQMQVrGIcVs4jRUoTFCmjU2lQXnVnKgQR7RHj75+xtw4wzjimRT4gumnrOqnq1LT2kTvlVuPfZoYc4AYHwxeTV5XbcyEPteopa1ryeIzEwbT1cpcXOjuC4WO1io6ZrQAH2OU6dU8MJUr9LhmgpagEONsJCgrdPgfLa0L4OIuvFTtIpYT6JfjzQA5J64j7TaxLFKGmQ3A2i0dPcGLkXjhzF7j1couPekejCpUljrZnMUDfXXXSzS4Cm4TvD6Lmj9PxB4/VGuMdJ6bw4vhATq00l8eGRaVfZZqcML+pbrfdXVVLbbdSyhQU4QlMg95PibSvf9yNX3cVbc7spTWU62FEakZkkTaAHuCnEyhX1VBxABaQmEocu5CttoMz+yyVMreCwX1BJ58/gup4yZHHkQrD3ZfNBejak0ldTPKSJKW3UqIHeQDaHcVA0nEq8Wtk1TTL4HIwkJJ94s4OEeC8T4bdqa7FGIE3g+62GW0N0qWUITIJUYJJJgdwEHTXRI4xYaxkcQUeI8I3dSVv8Acvo7yH3lIiFEgiAZ3+FkZK2Smka+wuD/AON1xiLZmFvIpu4+qGF4/wAEVjLZzut1jLhHMdUkjXunNaYsJrzXOhvUhJIEjxm0J3KxxOrbwYp71wfRMgGDUGrCg2nnACSTMDS083Bd/oVC2yE+qnXxPM2r453z1DHEWDW23vzui3RCKJwve5uqlPAYa4rqk5E0V/qV3AJD2YeyDad+P93/AJw4aVjwaC1UbzFQAO7OEn4KNop4v4Pv9zidVt3PcVdVKvB1t9lbTBLQJQASV7CFJ1k6b2nfHV2VN68Or5o1ty+7dzhCU6/SBEgDv1FtEZ2d/bKDvlv5C6qWxuFKYyOq8+79rDd160FA+kejXiaikXpEORmRttISse0WmHov1i7u4jUdI44CamkfpidicqQoT59WD7bNOi4eVnEpV6Ut0Uq3n6BhVZSvBJyt1KFAhJVsCoBafaTysd4OVFZd3ErDjjLK1OqrW23GUpOdAUSlcp5QConuFhqlgaHBnIqWFxuC7mFLHS4onHGroqAeyqnfTP6pSttX1WiG9X8LXzw7whdlVeNVS3zSpqXA6whByIDykQSrSSAmAJOnlNiuk5c7lZgemvZCDloKmHVfqtuJyz/jZffaoF53FcN/4cXcd60JNbSVKHqN4AlJbK0lxs84gE95k7WWVrS2KXwIv0P8FOjJBkZ4g/FOviM9hdnCHDu5sOC8iqlxcy9VuVxTndccSkBQynKBCIgAbe23oB7LebONa6ndrsJsU0BP7oqQwBAGVZEeyYt6SjQcrZuhJNVPf+7+CraIAYvW2/6X+xYSOf12AqhmZUNOXusMTrbhYlBSIkgxa0V1m1WNrK2kK70g2EEeB8bA05CmAAI1I+OlhYV3EWUBLdbNtQY29tugNINtE5drOsmkrmAVTFt7DS29xNtcpA3soF0hXMamBZKdHU3wIPZfZ180n7jZXJnYWSr0S4iqon0FISHihyZnKpJ29oFnOGia0m6MEQINk9hL9TfPpSwE0lCpIZAPrva51HwSCEjxz9wNlEg8rcZSOQ3n42jljDwL7JWvsu8Z0SFNhWmUt92kj5FoTxhdtMtygp36huoT15dbomJXUVTyE5kNpEgQIzKJ7AA1IE2lfGl6YqXStMXLhAVyVMg+kv16GGwrUFOgUr9EHbnaLMc0+JLjuS8b0pLvV6SaNCqnqULqSXSAC2hQACGgspOoAMKWc0ADFY1iTImCCJ2Vx8Oh8VYUzC5tnaqJL8oam+sYGgxzfzV3XfUtn0OhZWQEtNoBWXqkAhIObMUoyQIzLIFki+KC68Sm/wD0LBN03w4puoU9elJkcTd4SUpbCUalS1DM4ZDclQSP1rOPGF13jeDbNNft5LoLu6sdZSMinFStRITlSjOpEbqzJUSmQIJEGHMYUzhvN5q7OJCaO4qVCHEUdNUFpa3FJKjCSMzhSkI1JWSqAkTojD1k1RWZmSm1uZFjf5f/AAjgyMD0QnRTXZw2YpmmH7ovFbjaEoWpV53ekqUBBJHVmPKT5m2WblN0f6Ospmqut4nVtNUPoS46yXHFFtZEqTI0MEkT4Wy1Xni/tz5FM7u73PqjCax9lwgKMRGqttfqP2WWaK+esgLc7QAkn57h862btSFdbnQqVRqeWnhYamUkAJJUkgHYmRz39vut7A4Byc06p6MVqHRmSoxOp+fnax1t6EgyZMEx8+dmgxVOtKKDMCU6DXSfvsr09e4jQqJSR60wfh8mw7m22UwSz1dKtRCqZog8ykbW7QGG1AoZbSrlCQNfn6rFGqpJIBGYbRttYdL2X1ee5zakT8bRahPsClOnvWtphmp6+qZEzLTyk+4g/M2UGMaYspiEsYpvhPgK50R3/pWQM0pAUCY356/fYQKVHZyx60k7WcJHjYpvDYdwnRT8R8fU/qYuvedfXq1rHxJsep+MHElkZW8V1J7s7TS+79ZJmzJCiQdTvvM/s/G3RJIBCjBmQeZPd32eKmZuzj5ppgiO7R5KRGeOfEdr1r5ZeGnr0bX+qkGygz0hMet6ON3S6RB7dMofFKxaLiVSIOuYRG9t5lAjNzEAjeItIK6pbs8qI0cDvZCmGm6RmKgkiouO6XOfYDiD8Vmymx0irwBCanCVMonSUVyk690ZDaD2lQrMdBPMbfPzysfBKYBTrGuvzrZTitWzZ64YbTO9lTez0iKQn+6MJvI0B7FaFfWgfIsca6QmG5+luC9kx+qWlf64tA6c2TMI0gAwNfcbdQRHIRr4fPdZRjdW3d30TThFMeSsCzx7wW5HWUN8NyP0mGz9ThsdZ42cPnRK7xq2f59G4f6oNq4CTEGdQRpbZ7Qgjw33tKMeqh0UZwWn5XVmW+LfDx2B+6NCCeTlO8g/FFjrXETAr+jeK7un+U8Ef1otVgKOoK4G+vM8xbArKD/KPPa0re0M/NoUZwOI7OKtm1i7Cb2jGJ7pX/NrWz9tjbV5XZUgGnvKmdnmh5JHwNqhmOzmUNDsoeXwtnVgCAEKO0ECLSDtE/2mJhwJvJ6uKClQkEHnpbeU8h77U7S6toyha2++FRBsZZvm+KeDT3vWN/q5Xlpg+QNpB2iHNn1Uf2EeT/ordFJki2ssbDQ2qk3jHFzRQGsTXwIjatcjb+d8+Vj7XEnHTMZMVV2n66grT/CB1tIO0MXNhTDgcnJwVn8pmDrbRSdx5WrY1xe4gs6HEPWD+VTMk/1LHWuN2PGvWqKB7/pKUa/4pEWlbj9MdwVEcEqBsQrD5T3WwDTa0Cs8fMYJGZ277ncA/wCRdT/8yxxjpB3ykjr8M0SwQQCioUifeFWmbjVIef0UZwiqHJTS4k9c2ZmQQPhbspVse60Lq6Qz6nWf95zaoJK4vIiNI/4rvsos9IO6yQKvDFWgk69VUoWB7wm0gxejPtqN2GVTd2qV4jW2gNbRsxx9wk6QHbovluT/AMWyod3/ABlj7PG/ATwBcqK5gk/p0ij/AFZtK3EqV2zwozQVI9gp9RFsg/ss0GuMHDpwhP7oCg/y6N9PxyWOMcSsBVAARiqgE6DrFFH9YC0grKd2zx5qM0s43YfJOMCdxbBuB9lklnGWD31Qziu6FmdhXN/2rHWr4uiogsXvROfzKhB+o2kE0Z2cF3Ce3cFGRtqLYE72xtbaxLbiVDvSoG3YSQNvhbi4HYphaVxvE62wifvt1l8PhbcAnbSyXTbFBxNt699uojW28vdbl1lwQDuNbayiNbCEEfttqNd9vG3ap7WhckcrZHeLdFNtwYs1SjRRf0iOEt48aeGz2B7rvWmu192sYqQ/UIUpADZJIhOus2iSk6O/SvoaZmhpOkulpinbS00hLC4ShIgAabAWtWByjztgHOwE+HQVEnFdcOtbRxG3wKqqvBqWsmNQ/MHWAu1zm6DbYjqVVocAulvH/hOf6Bf3W2OAPS3G3Se/0C/utaUA26HfaL7Jg95367vzQ47PUh9qT/Mf+8qsfvA9Lgn/AMJ3/QL+62v3gelt/wDmdH9Av7rWo30i2a87d9kQe879d35rv5PUnvSf5j/3lVgcAultP/hPf6Bf3W1+8D0tidOk6P6Bf3WtPEaW1ubd9kQe879d35rv5PUnvSf5j/3lVn94HpbEf+E6I/6Bf3W4V0felkr1ukykz306/utaqBGhtoxNmnCIOrv13fmlHZ6l9+T/ADH/ALyqsno+9LNPq9JpI8mF/dbS+j70s1iHOkwlQ8adf3WtWR4WzlZn2RT7Xd+s781J/J6l9+T/ADH/ALyqgOjt0rZzfwk0SOfo6/usKngB0tE6J6TQHkwv7rWoO1syjff7bIcJp27F36zvzS/yepTu+T/Mf+8qpO9HjpWOuhbnSTQtY2Uadcj4W7PR/wCllkKP4TCSk6EdQuD8LWoAnQDSbaUNdLNGEwdXfrO/NKOz1Id3yf5j/wB5VNY6N/SjpSoU/SLZbzGTlpliT7rFKbow9JShrV19Hx9o2Kl0krdbo1JWoneSE62t3A5zbUWQ4RB1d+s780/+TtL78n+a/wDeVUa3o7dKa8KN2ir+kY1U0zyS26y7TKUhaSNQQRBFmS50JeMTKsqOLN3ZZmE07gAPgIgWvEYCYgWDKBOoFuOFxWsHO/Wd+aQdnKMG+aT/ADH/ALypJcvQb4jsXjdLt4cSLqfpLrrmqtLHozm6VhSsvcTr7TNrrbCLCFAG3ltbRTA0OlpqWijpM2S9zvck7fFH0OGU+HF7oL3fa5Li4m22pJK4MxbcAjQW3A2FtiQN9bGAI9FqLQut/quGPLb6wbGfZ8LFmTlrXkkesAoeX7SbGdeduGyU7rUSN9bZOu9tnTW2jyjz2sqRYZ5W142yYNsnvs5oTXLNCLEL2SVUi1p3QoOD2GbHuUTzsA+jrWVt7Z0lPvFpMtwmE5SEEYVBTzEi3JGnITbikWV0jRVM5QD5i3ZBJgCfKzTsl5o5W9a5hOo9GID7WZLRInKrcHcbT3i0QY1pqqsu2nRS3kp2iTWM1brbPVn0wN7gKWoAT3iZMCRaVbsQ5W0VSLwQBADzVNMhChsVGNVbabDxiSwMYsstVThynM2SrQTIj4b/AAtka/CnVdQ6cGwNhtrvdWFNM2NpYdeairHd0JxFftyel3Ez6CUKdrCX0NvQkpLbS3QFKQ0SFaNyVdkEpSSosniK9ebF0V173Pwwweitu+lcVR1jzyXhTBptXVrDRYEwEghExMTGpDmvnE7EqpbgoV3nVpKlPNUakpbaWZUrrHVEITrpElWvq72RU01e+07U4leKUFQCaKhGYKzQOrUVCXJgiRkEEyNMwidgVPw+NOcz7eFguFW7NkYLBVJRx8u9tCW6vijicvpADpZvd0IKxvl0Ok7anS2WeVTwiacqXV1XHbB9E8pai5TIo7uSlhU6oAQMgCTpCezpppbLZrg4d/B//wAqe8vX+PNPV5hQcWVNApCp1BnW3bDQCUmJy6aAfJserm2Kd0B91LSycoCjlk9w7zbmmCFpBDyPNKwR7Pn3207ZC8XCsuGGnVbbayiMpVprynnPKPx8bDIQRqEyRzI7htHw+dRUNACEKhQgDKI5be+3SWURKCBz5QAOXwn22XMeaXRdsOrZhCgNwezIkz4an53scp34y50E6bxr872KJCSQG1BR035ba8uVttZoUEkpn1goROvl8++zNSuSu1VGZJnKNAZJG/L2WHDiVpIKlDKBJ31+dOdkZLqwc0gA6idR47WNJdcTEkmYMx8P287NXBK2q+0kg6Zp1Mffvy/bgASO0skcxEeViaH2iYSdfWBnaxhDhAKRyPiD8PZZpThZCjVJ0zayN/Z5W7CRm7RJ10BPf887cJUgqyKVJJ11t1AGxSRvpz08fL4WS67ddNwSEpUdOXz8/VZSaGsyddxPz8iyWkhK5Ag6GT3WU2QooAAI5xt+20L3KZgQoHeAJG+4Ed23z8BCUxBGm0Trv3bG3AACQROpGg+v577a0HqpAJAAkT7ItDdSrtQjQEAARI8rcZhqMokg6CTy+fjbZSpAgnU6687crTExlISDpuZ+63BySwWA5iQDoOahpPstiUkIUgJBPhzBtpCSQElUA76b27GTMNM07gWdmXBoWJUqNIHZJB3+fxtsDRQCZiAIOnP5/bbcZZKjrGhyx3afC3MyCQI391kzFdYLqUTmVy0M/Ej53tyR+sBrrttbogyBKiTrvoPmLaJMAySCZn55bW666yxKdNRMGCAeUfttsAqGgB05jl8xbekQIB5eXz9VtaKSQiIEbcvn57rdddlCzMJBBnTutis0jNrm1gzbMsZSVagyTbFJEmE+tpygW66cGrFFM90HSbcFZgnsjTYSBbsqBWASIA2Go/Z872CVqsJnTdWg1+dLOumlqxCSo6JAJOoJt2QmIJOmgnnpbESFggjaBJ1toJIGUeYj65t103KugUqmU7gCDoe+2FQSSdyJ1+r7/fYPKQCc0nfXbw8bZmkg5ZO41t1ynZUIFgmDqU9r48/n8MSqdQs7TPz8725BEg7xprHz3/ItzmIToRMA78u6yLrBC5gIyzrpz3+ZtsJQpJCgCZ3KeRsFmSnRYOo0JJ20n587YpRG87xp9/z963K4tBQg6pKcyEAEHSBHfrY0zed4MCWLxqWR/wAm8tP1HexBB17JMzbuE5Sk6Rtz+fxsokc3YpvDadwldvFmJ6dQ6nEt7IHd6e9H9ax5niJjmnnJiu8TzhToWR/jTZtjU5gJ3231/ZbagEjKU+Z2s8VUrdnHzTHU0TtS0J4s8W+ILUg4jUo9y6dgx/mfbY81xpx41Gauo3tYPWUiRP8Aix42YQWdRqCN4FtZokrKT3nwm0gr6kbPPmozRU53YPJSU1x1xsiA5Q3O4Ty9GcB/+J4HlY4zx9v5J+nuO7l/zFOI+sm0UpVG5M66wPnxt1nUganTnMT+20gxOrbs8ppw6mPsKYUdIF8aPYVbImJTXEf/ACzY9T8fbrVo/h2qRrH0b6V/WBaEc6uzMjkRvJ7raKzrCtIk6fV77PGMVY9r6Jn2VSu9lT61x1wovR27b2a7z1bZj/PseZ4z4Hc/jKisZn9enJ/qk2rulzU6AjlpGvdbsupUgjnJE9/haQY5VN3sfkmnBqY9VZJni1gB7QX7lOxCqZ4a/wCLFjzXEHBL2iMS0QO/bWUf1gLVe6w5gQQZ2Mb/AFWEQ6QMw1BBjnZ47QVA3aFH9hQcnFWmbxhhJw9jFN1Gdv7sb++xtm+bmqNae9qN3+Y+g/baqAeKRmPIb+Fuy8DqIgTv9vLvtI3tFJzYPNMOAs5PKtshTaxKFpUP5Jm28pAmLVIQ+hsEtkJgmIAT8/jY01fd50whi9KtqNRkfWI9xtKO0Y5x/VMOAnk/6K10dwtoCO61YKbG2KGSQnEd5dnUzUrP22UWeJGMm0/R4hqQSJ7ZCv6wNnjtFEd2FRnApRs4Kx0GbZBOszav1PxYxsgwu/C5/OYa+vLY63xlxc2Bnco3Cf1mh79I0s8Y/THcFNODVA2IU6EaW0RoItC7XG3EKdHaG71/4Cx9SvmbHWuOFaB9PcbCo3KHCkfGbStxykdzUZwmpHJSzl7xbRE8zPK0Ys8bmXI6zD/nFTP+pY21xouZX8bdFWDMdhSVfXFnDF6M+3+KjOGVTfZUgEaaW0RJk2Y6eMWFFaOM16J72kn/AFrGkcVcFuKg17rf89hX2TaYYhSu2eEzudQzdhTsI05xbR+Ys3E8RsFuwUX2gctWXBH+bYdGOsIOmU4hox3Z15frizhVwHZ480nd5Ru0+SWSP2W0Rynaya3i3C1QYZxHdiyeSaxskeybG27yu5/RivpXO7K8k/UbSCWN2xCYY3jcIYwItra2ApVqkhXkZtmx+o2kBB5ptiN0WcGWtbWD66SmPebGRHKxSsX1a6deaPpAmfr+qxqZso1SE2WSO/S2vPnbDbJnnZ4Fkwm61Os2zYTbJOtuVyLOA1Tb2W1ETA1HlbgkbAG2yZGnO2iANzHnaVRk3RCmkdc0f0HVD2HX7bCmBztwRkrXdNFpSofVYRPVpUVrbKpjYkbGw8zntjzRDMel7J4AvqhKNZS+J2VIPLe0ZYrrcP4pqn6E17VQGEinrrvC8q0qBk5yCFFJEREAjeQdJGDLS31VGdSHVEEHOSTrt3QLQnxi6N93cS7zTifDWKqi4b7SoLNW0lS2lCCkZShSShUwTGaYGg3t5vinamsoqkMrKcsYd+d+hBH1VgyFpbmYblJL9BdN3AUd20TdDRIWUNMNjsMo5bDYA6mPfaIL9xFja6rzcvxrhop68GXBQJUm9xVU7SASp1XVJbQpPLM52k6p7QTlKptu/htifAdzxfGIq3Grzba1pBTTUapyZQjO4VFXaJgwPWnXKlNkKnue+L6v8Ou4YXc7tGhsLpOrpXE5YBKnHUo1QIgZVJ7JSQkFUWranta3F5OBRuysYL6nf5IiKlDBnfuVXerw5gr0p706+HBU9YrrsxoZzz2pioPOedstPTuBOETjq13lh3EbtWpRL7n5lqe25PaOihuZ5DyFstmTiEnv/VF8FnRQR+UjfdTwswulvNmVfoUQmQdKV0DY96tvG1PuEz1Y+9TH0h4pLyFZetUf75BmfPutbf8AKUOBOA8I00qld7Oq0J2SwP7XxtVTg62VNUD7hSS6/vHesGB3+rrzt652QaHYbHm6n8UP2jcRWvt0H4KcGDVpKeqqahKwdAl0iROosZTUXvCQq86uZ5PLgDu1P22HpWpJCe0DJAA1A7z3aj55KKWUhGZUyBnIOoEc5+drbgwxn2Qss2Z/Ioqi+MSNbX5XGTBCllQjX4R9YsOjEWK0ggXu7mGnbQk7bcrD+ito7IQN9NNZ8vn67CBgJJ0knQCYBI8T86D2RmlhPsDyTxUzN2cVprFWKVGDXIWoEeu0mT8I5WNN40xQ0hUrpVmQYLZ0PlPf5/VbkU8EBSSTrqVan5Pzztr0RCogRyEAGB+w2b3CmduwLu/VA2cUcb4iX+2iF0tEojQQFD/W7p1jusbZ4m3sClH5pY3mEuESfcbI/oiHDlWkHaSEx3bxytv0JGgMaHY7fD2e+yHCqR27AnDEqkbOTkRxTqZKV3GDKdw/Ed51TYdPFunQfpLkqJTz6xJgz7PkcrNY0pSk6k6+Ex87TYF6hbAkDluFbRH4Wi+xKN3spwxeqHtJavPpGYSuWsRS3pct9fSIDoU0004kCYgysGfnno76HjFheqpWn/R7yQHUBSc7KdiJnRXlaqHFhIp8Q04SMs0qSkgc8yhz1HdaV8P0gduiizAEqpmyZ0mUjusN/J6je46HzRIxyqY0bKZE8W8JjQOVgJM606u/7TFjLPFHBrkxeLqSOSqZwH35fP3d1on/ADYlQCUIIMzManXv2sILsBEAk6wnszO/4WaezFGdifNKO0NTzAUuI4h4OfgIv5hBOpCklOpkcxYwjG+FHinJiKh0mZeSAPedTv8AJtDJuxKDkSiJG08trcfm5uQSkAKEHT7bQnstByeVI3tHNzaFOTOJMPuJzN33RKgaKFQg/V9tjaLwu5ROSvYJJGzqSfDn4fC0BC6ErIUtEAHnyMHnZJxDcbZu0ZkJT2hEkaaHfTytA/ssy12yfREM7RPJ1Z9VZpLjbiYC0qAO411t0mCUgGCnUx3eXstRM3M6hnELTLzjZLnWKKV5TBQNdDI8xbijOIrkZueoXiK9mirrHXQmvdAlQzajNsOQO0WCPZx3J/0RIx5vNn1V8OyClROWOQ+vxtmUlZ0AjQcoPu8LUPunEXEltu8VqxxiRCn6f0lpIvZ85AVKgCVwNI28O6xqmxzxco8K1FYniLfyqtbTj4W7VqcKDCoSAudBI087M/k5MdnBPGPRc2lXogyFZdIGxnT7baySRm18x8Dbz2b6SXG+71UiBxDqXfohKXKWnVnVJBJluRr3HlZ33l0luL7Fa1Q3ZiFC0odaQ8t272VZs247KBy+3e0Luz9SDoQphjlPbUFXbJATClGPd5RbMo3TmmdPstVtjj/xSZRLy7pc03colp8dQlY7rJbXSs4r/nFdD+57DT2aoUlEsPtlLQJlaoePP2aG0b8Bq29PNSR43Snqrb+A1iAQSQe/7rBN5ZnUk6zHfaudX0mcXtU3WfuXutYQMy4fcTmEbc4nTXW3OHeljet60Sqqs4eM05DmQobvYq5A82h4j2WYcFq2+z9U8YvSO1zfRWPykp3gRB15d1uAFE66j1o52gCg6W1DX3m5Rfvf16W2JzPN17axI5CUpn32N3V0tMIV1a9d9RhHELLlOCpSkhhaFRGx6wE+60ZwurHsJ/2lS++p2VIgJ07pGptoyQAJTsY5D22iVrpMYEWS27dV/NGZg0rSh8HTt9tg19KnhAw8WKu8LzZKVZSVXa4oA66diZ99mPoKlm7CniupnbPCl0yolIJHOR3xbSlrkzoJ0H1/PjaK6DpP8E7yfDLGLHg6rVIXddWOWupbiyh/CC4NIGR7iFdzSgZh5Ljfs7SR3i0ZpKgC5YfJSCqgPtjzUhw4FEJGp0j5387dE6AlJMHSO+zNpeMnCisphVs8RMPFlJIUtVehKRG8lUD59llCj4icPq5IeosdXA8lWgU3eTK58u1aMwyjdp8lIJYjs4Jwha5EqA7RnWI+dNfC3aDBlWgjeNfd52Sm8SYcq1FFLf12vEGSlFW2oj3H51seaqGHRLT6FnX1VAzr+y0Zjf0Tw5pO6MIUJjkByj4+8/G2AkaDSJ5zHOwYWFHMoqgHkeWnz7rbzqSlRHtnX2wdrNylSXC7KgpQI13Ij57raKgvsiIMQdgPusH1hUBI89D4Tp872wLSTChMdkBXLw+MW6ybYXQoUSnPqFGCJ5RbqASQkH8N/nzsEFZk9onXaRbSlgDKVGYgDkd7NJT12DBATGp1A91tpImdRECSJmwanJB1mDqYiPZ5m2Fznl0IkT8+NlXWQyV5VkhRBGxH126U4rLGWRECdefOPIWLFYIhIVrG5EHutoKAjsq7Q23In5FuSoyFKkwYMx3ydOfO3QdITMjXWd/naxUupKYQ4BHLkPmLbU4okk6HcQNY5fXZLBcjSnCFHLJnXbQcraL6SowY7iCZ11+yxMvKMkGdiJPeLBqeBg7aZiD9/s2s4NTSbI/15OhOp9XXbXfSwSqkerJGsgkeU2JqqUnQkEK98WBdcSkSSJkmJjTw9tlDdU0uKU2qkFUyBsRM2MoqtQJ5Rpv32QEVAkSTuYnY+33WNoqJTCQSTqD4e/2WQsShyVhVGVHNzzRz+y201Jgqz66DVREeNkpdSUpkq1BiNte77LcqqFgSBqNJmPk/PiU4aXNZLPpJjRZnWTGtsTVdnKIEDUn9vnZFFWUJ3X+G5+fDlbBWLygKJEHy1t3DXZ0upqgZgx4Cfd899hBWQkBRzSYEzP4c7N9N4HUq2SdwdLCJrgkEAkg6CdPn8LN4SUOulxyrVqSdTvBMGxc1fa3MpjbSfmLJa6wFErOo02mxdVckKKpj3EHTUWUMsuLk5G60bFYynx2+/wDZZTo6sFtcFJnfys0KatzDTNodROtlGgqVnOkq0KfYRZhbZdul01ZOhjadeW02CU8wTq22qT+qD5/VZNNQdCVK8QRobBLrkoGcHNIlUHcfIsuawXZQUrJqGmznaQlBE+qYM+znbT991bSg0m8rxC1HsNU1U6lStv1VCyNSVNXeij6GkJaBM1BEpj+T+sfh9VnHdNz01PsM7qtFuq1WuDpJ+yyCSRxs0pC2Nou4Jw4HZxFW30zX3lf95qZZC8tIutcdaUVCApYUTJEmI087TWlWZIPeARaNsOsmnbCgIiCO/S0hUCyqlb1kpBT7jE/C22wqJ0cIDjclZPEXiSW4Rg620VaeduZ03tokHXe1uGiyriV0V9/K2iqddrcz420TysoFk0m662tkxB00O1uZ1jfythPM2UtDhYpEyKu6MZXJexeua8PTrvdUpTdM6U9YzJnImfWTMxqIB7gAStdj2/LjfFNfWG9kgqdR1qAEkxoMigTJT+lG+swC+K6ShtxJ9RaSeWh0Nt5jBHLeDtbDzdmMQpJXSYXWOYD7LvSaPhfZGtqo3ACVl/HZRwvj5hGleUzUUNSgBJCnEw4rNOgCdBGXc8ttd7LL/H/hXeaKNtm+VMPPN9lBaJE5c2WUSkq0IhJJmzqdwzhy8ad2ovq5KCoQ766V0yCpwjaSR4n3mxSt4ZYCrbiLrGGKZlDlZ6Wtto5B13bSFEpiYDijHfHdallZjkccnHLJA24LrfTZTNdBnblBCY7nEXh+KhdKjF1LXuPtKcFKwoKHViAorUvsjKDsIjczvYxSYQurECHr0pbpaavB1QZWgOFpvIgwkEwUqCZVBynVRiJmyFiPgdw3buzraypr7tFI2haqpuqypbDe6+0SE6TJ0jlA0sxr5wtjK5cNXLVcLb/v9NI2uqadbqEuNPXl16wlrInIcoypVC8yCJLkpnNbyjEKXvMzXtbkcdDbQfRXGdzWFOK8bkwrQ3hVUR4oU1Aad5bRpEVlIhNPlUR1YSWSUhMRBOkWyxFnhRxAUyhSbkwTTgpBDL1DTPON6eqpzq+2obFXM62y1f3Cb3j9F3Ecqp/lJqlLdw4HYJylVdWqAMcmmfvtW3hHTy1dvViQasHTYCYIPsVPl7LWA/KXvuIYwCwVaKdvJZ20hNMBodefI+FoJ4OoHU3XmOdKnUqgn1e2PmbfSPZNpbhsXz/FV/aHWtk+A/BTxT06RCliSVHQcjp32OttgkL0OYgKgRPdbKZtTgGZAJIOaROn3zNjiWwU5yDp4c+/T9nhbcLKBF8isyiBpE6RHz93uES02DGUkDfY/PP3WHLWQyoaaCATvbsJMjMmUqjnHcNzbkq5CJMGTCgY3M68vna3RbkBAUZIzHymJ+fjbaUFaQsDQp1HP2WFSMqJ3BkwTt5zbhouIRco7UIEQSJ7ye7xthbA7MyB7BP4ffYzkzKM6bme/wCdLCNoIWZBBmDI5DmbODim5QiuUpOaSSPOY3Pz52AW1lGYkI5lXLT67HlNhRCVJBTM77eHhzsC4gZAlQgeqdd9T4eXyNJAonBQXxlbUMR0RCTrSDMk7E51bHbn+znLmG20/mWgJSVTSskqA/kpn9tor40JKMQUJMEKpP0iRstRJMb8vdNpbwqnPct2LClZfRGdc2pGQe+zGesU5/qBLLbRCSktid4+v50sMppRGRObX3iY+fG2NozgA6H9I7T7/Z5WNNIMHQeHMfO1pVGAixaHaJJk6xA0+w6fXbosGAmEjaPD8Pvsb6setpGvPSfq5d/O2+pSseoQPD57rNun2RQspST2BmBkyIkn5+qyZiBgm7iQEaLTMx37n3fXZdDZXCkchtoeX32T8QJX+bnNUg5kED2jWzXHRPbuoZviA/eFIyAFVLzWYdyUoknx2A9vKwt7USau6bsZQn6R0IQkqOwUnU+74mylVUiHb2vN7cNU+QA8ypOvwHx8LJrfW1lVdyoPV0ymWTH6xTJjTU6Ae+wGbkiLLm8WmLvvB3rEJy+hdWkct4Gnn9vKxahpkLw0lOTQNqBB9o+42Vr3pGqy/GKdUZUILiwDGbKJA8dSCR5WRk1TlLcLdLTJl94qSkAggDMZPfEWe19iksoFvekeYrlPKyp6911wp3DQz5Up3k6J56WlGiuZijuu6Wm4KlOIdcWUwVqIn2fMWjrEThaqG0OiU/SQCQNc5KZ5xr38z52lVhJVdt0q3P0J7/0Z002P2e5hslA0TuqA1R0q6l0oyNpiTzPd8LI2G7vTUVlbXOJ7a4SCSNpMwPZ87WUa3Ne95Iuxpz6FBDj6gPWAjTv7rdXCyGq+8EtogdZCgk6CJ08LOLrkLrWCFrbvZDK0hKSoAweZEa/V77NaiYTT3GtltA6151QbhOuw115jf2+yz8fZ0IcSAjUSe/ayNc93Iq6n0qAGKaG2UGCZ/WPz9VmOOqc0aXRS7MON3fRpQG0daohbign9L8NvkGyVd11ITiWqUGx2grsAGBtptZ8utAHOR62u0RHP6rILDOTEFQp0xmSo6nSMo2+PyLI4gWSAEopetGKSmUAyErOiQrl4+AA8LM+ouJp3rJPbIzkkxmOv3/GzzvFxVQ6t5xsFIOVM+zv8vmLFmbue9Fqatxak5Gzl19Y/cPttC92cqVrco1SRcGFqZilFYnIHX5JI1AHcJ1377MHiTQht5wQZyp7vaIGvL47i0yXYlJutiCToobxHa/Ha0YcVmwlxxw6JDAJMjXQgTz5GRYkAZLKE3uimCbvReWC3VLOZIW6kwkEkan8PZzs47puZNFcrTKWRnXOVOmbXu+7nvYjwkdQrDdWHmwsIqlSkEHTKkhPnr8RZ+XFTprX1VruVQbgITyST7O7x5eNoyAU8EgXSJd+CmqZpx1xtJfdguKjYcgPf8wLGDh7q1gpSUncHUdoTr4fD7nmtpJMpzZZ0MRHcfstwWQDqjlMQZT7RZ4Y3omZ3dVHuIrsvBN3oUzX1bcKzAofUmRrroQBvY1czmKmbupupxLe6FIby5hXOpXoSJkK8/ndy35Shy715UaIhUn7vbHt87A3TTpN3JSSYSVZRz3mZ9/zpaLhRl2oTxK+2hKJM39xBbJ6vG1/g7gC8HtCO6VR3fCxhOPeKlMOzja9lqTIBcqCsnv0V+3Sx9NGlAkoUMo5wZ7h9Vk28AkFNPTpKnFkiAnl3eQ091mvp4T7I8k8VEw2cfNA1vGjjWwv0a7sc1JdzR26ZhehG8qbOn1xyFlDFPF3pK4P4YP8AEVd+tPULVY1Rh+pupkIdcVJKUwgaADUk76amYXuHWDsLVV6LcxdfTVDTMo61WYdt9cTkSQCAO8nl8BelPiihxNw2rLgoqulfp2WUrYZpFEMU6WxCU5dE5o020sDJSw8UMbGPHRFMqpgwuLz5qLrm6afGx6hpKioNwOl4wtS7vUP0onsuD6uR0tNdzcfuIdXS01RVUFzKW82l0hLLiRqPFZjutRu5V9VdNIslIWy8ojSSnteHn9Vrc4UaaeuWhVCSOqABju05/P2xT0FPm0YEVBXT21cVISeO+LMsO3BdjkCTlW4iRz3JslX50pL8uCoZbXw/YqUOoKs6bxU3lI0iC2fv12sSbo2yjMmZUIzDU+/2j4exr40udp30Z0oH6SQoRqTlPv3sMcNpyfVRPf57XDk6GOmMuP7p4Z1CTuQ3eyVaE76sj5nusaR0yMO//bcCX80qJ7LzKxsO8jXwtEDtzslBIZAygGPDYCPaO7ewCriYUktpbT2RqSdQeXt1HuHs77Kpzy+qb9p1A5qbkdMPASkjrsNYmayzP0DB7+56xtrpecKFrQmqRfVMSDJXQEx/iqMjutXt7DtMAlwMgKzQBlgCO4DnFir2HKZYKSNdjBIPn9Zsn2TB4pftScdFZxHSq4MFMrxPVs6AJDl11I8dwg+491uz0oOBzsdZxBpWCTH01M+2RoN8yBaqFRhencHfMJ7u+JJnx+TZo42wkGsNV1a0oyhtLglKYhKhMabxvEa+2zm4LATuUhxidvIK9NL0hOC9RlU1xMuFIXoM1YlGu59bzFl2l4w8MKtKU0/ELDi5GkXmxJ/zpB+63lGEg3b65BafIAgScw9/6PlaS7mw8xVUDFVmH0jKVlKhIAIkzr3CedkmwOJguHFLHjUh3aF6UMYzwvWpilxJdT0iU5K1pU90dqxxN7UTgJp6xt0aiULBnfkDbzeRg9p1tY6tuFK3UNZjl8LFqjCDaUEMpCNQDoRrPh4ft52F+x2+99FP9sn3V6WoqFLV2Un9bY25XUFITmKgeRmI+fne3me5dd806CigvGrp3ACU9XUrSQdgJBEfstl01nFNyoQzdONcSM7JBRedQgAeebTn4WQYLcev9E4YwNsn1XpcKxROqp1Agmf22EFWcx7Z0G5t5z1WMeLWHkVKVcUcUvPN0xd7V7uqCDlBAAUrXXNrHK3oX+TuvCn4s8ErwruICRfl73biB+jVV1gzPdWadhxCSdDAUtce2wNfQmhh417i6Kp8RbM/JlsUfNQmClOh1kDu7tbFzVEKOU7jYnfy2tZZzhVgF1YKsONAjTsuuDSPBViVTwW4eOpP+4riNZlFU6P9a1H31h3CO4gKr9TVaVH1tI8z87WUaa8EpXGaZBHZPLnaXVcCMD5yptd6NEiITVTA8MyTZLvbgNc4oKhV04ivhioDayyVqZWgLgxI6sEjbQEaeOton1sZUrHC6jV683C6mnZK3HVaIQg5iT4R+zX3H6G4nqpSKm9FglGqaZJ+jH879Y+ER52Gum62LraCAkrd0Lji9VrPOSPHlZXZlXqkkA9+9pQC/ZTEhu6OUrSQEpCQIA0HLw8Pwsu3YySsJkaaxG1kuka7euvhPzp+FnFdjRzAR8/Zytb0NJd1yq2qnsE5LuSG2kjNZ5XW6VUgnlHxANmfTGUiBAO9nLcTuZpSY2AA9k/eLbCmblFll53XJKVDA99tTynTe2K13tqeW0WMQi3NsmN++3JM791tTBIi3Ll0SItsmBE25zAb+y2iRyO1pGi6Y4oOqBVTrAP6M/bbltXWBJB9aLCLAUCJ3nSxWjXNOjWFJkEzsQbJI240XNNkpV9QUIFG2rKlIBWNp8LKFwK6+7KqnIBKDmHPcfhZAWSSSTqdzZYwo7FU8zuVtEx3kEfYbVbaFlPSmLfmfHqpDIS66j/iNdVRUqTUhCH6dlrOKVQIQpe5UvkoARCSIBkmSElMYXneuJr2VUXgL0dSzSPguqUTlQghRypOiQrMWzlEkjWNJtNWObsfr6J2ncqV0dOApKi1o6dDpI1SCY1Gu9o7uK77tve5K3DTqm6NCaynSnqjBUVrhIjaJEaAT3zJt5RiBMMksnC9Frm7jW19bBamN+aJrk3W71xU42lxqVoUkKSrr3u0Dsf4s/XbLOV65eKSnVqoqinYpyollputShLaJ7KQk6pAECDtbLM7/S+63+PkmZ3dfqqDflK3lKvLAjCQMyGryJHMZjTAfPhaIOESMlPc0iAXREDQgLA+fM2kj8oxeSbxvnALgWCHbtqqjKTorOpqR4GBv99o/wCCiSmluUwHEJEoJGkSDoPafaSfG3pHZcWw6G/j+JQOPm9ZL/HJTs0AkbFJSAd9U6x8myghKDB32KkxOnOwNM0mNSIAmdvtsaZbSlQcTEjU9nWPmbbQrLMXXVnLEkwDAO0+dsDeUyBm5d3n9lhIUBooRtBHt9ttthRRM6J3JOvj9R+d0T9kGhCARuZMeI00/ZYRLSSJCJKtB8+78BbrL2gCdtRtofkG3QQQkgEDSBpz7p+drJzS2XCkJzwFKMKkjnHftboJU4EqKozAiQNhpr77CBoklKN9Y7/b7rdJaIVmBIjWY23nnZzd0w+CDU3ngqEGdO4ecQTy3+Ni7gIAKz2V76T32OlsQQcp0B00A19vhYu+2CCUnSRz1Pj4n50tIConaKDONSAm/wChUcv/AAQxzg51a+zv8LS9hNAFwXWSUgmkZBKRrohPP3/O8PccnlMYgoFKJ1pDGsx21TuO75nW0zYNT1uGbqVtNAySNd8g0+rfWzGm7ild6gS0kGSkTlmQI92vz9tjASSN1gyOWs76z887BIaUiE5tteyecj77GUI7MkZikaA6RPL3x91n3XBvRYMyc0kaxr99tmVJkg+7l3Ty/b5gXKIUoFKtymB8I7/nS2y0VplGiSJn57vmLMzJ+RcFtt2cqBr7Y3EWTL/Qk3O4Z1BTBPPUH7N7KxCj9GEazESfLymCbJ+Ims91FRQUwtChBAHrb/VZjz6N04WvZRe6W2qq93HISlCETJ5lG3f4WKqZboLquxxRSkLfDyzqY7JPLeNNbdXqioera+jk5VkPuEJmEIRIT74sPebLL9FdNCFZVP8AVgiCTlyifeDHttXg3RZbZJVGhx68lVdRmPpTLjuUT2UH1R5QLF7roUIo3bxLclbS0M6eqgak+B5/dzWrzdbo71cU4pAS1RBIA3BJgRz1jaxSlQhGF1JyBv6BRJPLfv8AnX22c06pp2Vf8aNGmfphAUVFZUAJJBII8dxaTbtSj9y92VDgBS000RmOshO31fX5Rlj2lqWb0ZqXirqnlpDCTP8AFhIOY6fpEmz6w2upvDD7JGZNPQ07bZgxLmg5d32eNledNE1qkfD9CqnpS++omoqfpFEggd8Qeeo+FtXOgi+bwbkmcpM937bKF2KU5RU70K7TaTBHhMfXZPNQi7b5rHHJSOpCtdJPZ08LMc+wCcASV3ezrtQ+3dlMIWs/SqSnQJnn424w2haWqloz2XZAjfQ2OXHRqUhd4VLf01SToeQ30sFh9RD9Y2kEALBM+Zg6/OnnZmY3BUlrCyPlor7KiACBpBMaj5j77N29uop6p1DZKnHIzmIypAGnvE+7lZw19WKSnLhBLitAJ2+e+zUdWVuq7MuOeyT3WZLJySxs5oJqkdrKhLSUwZIJCdBG+mll6spG6W7FtgiEtkeY9/z7bD3TdvolPnWgdesBR5EeH1W6vQoNE8sEphByab92nzz9isFhcpHalI90Nn80IStMkFUSfHu949to44qsBbSipOfPTkieZk8ueh+NpKuoD82ytZIBWDO++3xtF3FqtLVbQUTPaVUZkqJEdgamfAA6/JsSH+ioS25SfwhS9Ut11K3KGW3Q6sgCNUQAN9Tl9wtKGEFHqn+wSqUgyYJkHblyG3eNtbRtwReCn74pg7qUMr30ntiNtNSbSXhQhVVUJOYCAQSQNjEfV77MGjglIuE40tkKKlJJ5jXfbv8AvttSQ4dBrAB52HSjSAQAYEQIV7Pb8LaWCmVBAJMjXcd+9pC6yjsku8GSqmfBEynMIjSLErga+hcAKR2yozoYIEfV7jZbqGwphYjVQKROsHbu12sl0Tarspip1Kg44cwSPWT3DTn8+Age+xUzG6LKt0NJKWgVKWYSEmT4RrrOvlbligQ0Q68sLeXproBvIHx99jVLRLBNXUpPXFOoOmUGPn5Nh1oSo5ikKIM6HfUfj56+1zTfUpHAjQIi82d0xpGpAGnL5Nm5jOj9JwzetOCYXSuARt6hgd/OzsUiG05jITtGh7+7zskXuz6RRVDIbJSttSTtzGpj299pg66jVUbgSF3MspGrdUVerMAoH3bc7W44bPIqsK3c/CSktAgKM5SRP1HffytUW5MwZvCmdBltxJjnAzAGTz9mk+y1qOCtUmqwVRgFJ6sBIgnu0Bnu15WAn0KPpzdSIhICQFAEEEkzqRJ3jlt+2yNi6nBom1pgqS9Ksu0wdvkbWWkk6QTEwEzHPnp7bJWKGHF3WsjXq1pVBmSdZ+3TzsODrdEnayZi2gV51AFIgrIIPL4crBqaSVShISrLMTPdse7WxolUFYjXuUPd8/bYEhRAlQKl6gmQdPEefx9tn3UWUoq82VdlyIEAnYHed/CfmbF3KcBIjtZlEzHONZ17vqsdWOQhSUiQZn4Hnp8zYFwGIIBnUkyo/X3zr38+VnByQghJTrUFQEJiYyiZPhv9X1WQ8Q0KqrD96UyTBeo3gkAQD2VQJ84+Tq6HQdElJEiDA0J8AO/SxBTTa15FJGVw5YjcEQft++LSseoni6q4gBV31bYPqrbcHcBJEz/hDf7bS9gM+kYboHlIKVJbDY8grLI90+20T0zZaReFIQE/RHc80qB7vA2k7he6pdwttqIBZfcbylcGNDEctVfG0kx9FRxjVPRumCm4U2mYggEyBty2n26C2VNKkjswDIUM3d93n9tj1EwuoUEIbkjcTI56WctBcAaAXVolYIUQRrMb6eVgTJZFNjzJrXfhior3gqpQUN6QojU6A6/tjfay8i6mKJnqmGkJ3IjnJ308fbvZwBlKUZUaRHq7j5j2WK1DBSCoAFMSVAamPMeXus3PcqUMy7KLccUYTVklJIfp4J7jBT5ToLW7/JLX6XLi4iYbcc7dPVXbeKElWvbQ82sxy/i2/eLVWxzT5W6Z1KdFpdQDEweydPOfnW01fkrr5Vd/HDFeGnF5W7zuF8oRJhS2KlojuBOVa9d7BYvHxcPkHzUlM7JUNXqUpWYJWOduVjcxt8LcoJyZCJKTbs7Anut5zmuFoLWReSTHhbhztJIPPvtiuyo+Jtysnv152Fe5ThRBie7VUl81DQBAKusSRzSrX3bixNholXMEDaLPXHF3lS6auCf1mlfWn7bNdDASoEH1iFa99tLhn6WJpXTvsjlGjZSgRZwXahKROWyRRNAkSmRpqO7nZeo0BKQNtAYtrKSKyoaqW6VmBGk+Wtly5HClwogQQfedfsNkNncSJ2sp3a4UVSSRGo+Jj6ja5YLKqebiycWbl8YtoHSdrc7b2TsRYjuXCdyVeI8RV7VDdtAjraiocBKW0yBsASdSBA11tOSGgucdFHFE+d7YomlziQAALkk6AAcyeQSoYBtonTnaL0cfLoqbqGI7v4fY6rLiU2XhejVzjqVNf8YG1LDxTGshsiNbPPCGM8NY9uCnxPhG92Lxu6qH0bzYIgjdKknVKhzBAIsyGohmOVjrnf5dR1HiFYV2CYjh0XGqYS1oOUnkHe663qu0Pomx0OiW8xnui250gA62ju8ON2Fm8R3hhLDl039ie+LpOWvprmoS4KVXJLjrhQ0FaHTPOh7jY3hHjDhLFuIqjB3V3lc2I6VrrnLovelNNUlv9dAkocT4oUrv2s5tXAXZA4Xvb59L7X8N1z+z2KxQmofA4NDQ46ahptZxb6wabiziMpuNU+M0a87FGTlW82rQhZI05HW2Xtet23DdlVfN81zNHQ0TSnqh95eVDaEiSpROwgWjD+EJhJFGMUPYcxYxhh9KVIv5y6VJoygmA6RPWhsyO2WwNRrZ888MFhI4A7/Lr8PFD0GD4hibS+khc8AgXA9o7NHVx5NFyeQUoqka6xY/cTwZvZhU+sSj3iLJVJW0l40bFfd9U1U01S2l1l5pQUhxChKVJI0IIMizMvLjLh258VvYUua6r7xNfV2ht6tpLmpQ8aRJMp61xakNpUdwnNmjlZlRJGxl3G19vH4dUyiw2rrpXRU8ZcWi7v7oBtdxNg0XIFzYXIG6kzFrPWJqmARmICgPMffatd+XY9VYhNOt1TLc5nFp1UlA3ygesrkkczFp0ujiZgviLTXjetw1r7D905qW9aCvYVTVdA4mTleaVqnSYIkGDBMG0QX5TVlXfD1fW0Xo9Lnmmp3ESt5MCFvJUIAmSG+f6UQUKwdbMG1JDNc40/BaCOmnpGGmqWFr27gixGl/qNR1GqB6zEY0osDU66f+8qXQqdUpH6JK57ZiO1z3tlgHUPuOrcXXVClKUSSVkkm2WD+z3dB5lNyuVB/yiwNJjTCNA0tZaYup/shOoBqFBOo5DKPhZlcLr8Yum7rkeXQPOhmmbKssdoyeZjePhZ1flH3kK4rYdZVMouMGQY3qnhvHgfnQsjAraEXBd2ZSVr9HgqA3hShoB5eHstfYPKYMLhcN7ftUGKxiXEJQVNdPxNuwIzLuq8UR2vUQfOO1ZQb4lXCtKZo7wEawGQdd/wBbxtGjJCtU6a5QdPnu91j7eu5OUpCiTr5+zf2G1j9r1HggBhsJ2CklPEXDaiMyaxP6MmnO2uunztYUY9wzmzisfQtPMsriZ28fxtGqiUyM3KSNvYfnvt005+iV6kzESPw0twxifoFxw2IdVJgxxhdaQBeZCjtmbcTpz5effYdvGGGRCjeyEmYMoUnT2jy++0ZIICMyBChtqRJ+/bS2DKVGQkka6xBHd7RtZ4xiboEn2bGBuVK6cU4bXr+e6ZEHTMqPf885sOrEmHlkJF+UBn9Hrhr7J0G1ojGVIjJIAI07uWg3twtKCrKUBM6DQ6azuPP4WkGNSDdoUZw1nIqYFX3cxBSL2oSYkK9ITGh5SbAuXpdyklCa+lXI/wCOSTHM/XaHVBKgVkp03ISPGTp4jvsnVDaEDNlTA01SJA2Ht/C07cZdzaon4Y0+0iXSEvClZxJdrjdQFAUiwSjtJJznQxv+E8rTrgOoZVhO53UPNyqgp9AQdChO9qecTKhulvWlhOUlgmAIJ1M/Dn+20o4JQ1+YLtUlrL/crZPjmSNz37+cWlfihiaJMu6hbQCT0AdlZhtqYKSSN4zfM/jYylpZEZiYgnfXfe0FtLIkocWkgkSFqA31HzrY6y5WIVArqsEDTK8sRt9hFo/txp3Z9VM3Cj7ym1KVgBQRqDJJO+tuFICwlKBBKu0dZHj9loearL0CgRe9fEaRULEHfmfOxlu+b7SokXxXxJ9Z86GZ0nnbhjTObSuOGPHtKXQ24QAok8z47GxDELJF2uSIyqQYOk62jtrEN/7i+q2Z1lyQPePn326dv2+nmC2/e760kgQsJEGd4jebc7GYiLZSuGFyXvcIm+wHKi/KhaZWhtKAQNQAiY8dfq96dTNrqrwoK5ZIbbcSw0kdyUHMY5a/PeoFyrUp4+mELqDlc7I7Uaa++3CPSUBhDdSCmm7SAUJgfM7n3WF7+0qc0buqLXxRprL5dSe0inY61wTroDAgcidfZYjeDgosJBl2Zep1JScw1Vqfx9llNJqOueqetQXahJQ4SgQpO3s5WSb0pFrYQwtSeraSW0djwHj3c7PbXNuozRuAsod4qM9U9dSkkg5RChJ0yDbvOnOBt5WfGE6VpWDWA2hILrJKo3zSZ89tPK0f8YKqpbvOkaecZ6pDSShITBBGk76iJ99nnw9rKp7DVIhCmw0UrQAUmR2j+GnjYh9U0NDlC2nObKpDw04hV0UjhgwkT2toJ1+HxsDWpRVXy1Qg5QofSKJjNl5H552Bul6qoKRNIylooaGxmYOw315+8WEpaNVPVtVqVBS0KKzmJAVJ5fG0Lqxhspm0rhqnCUTlSkdrZPLSIAju9tka6FoavevStUAdrxAmfbod/rsbN4VEJzUrfaAkhcfZYgttSnKh9tspXUAbOTp5eJizX1jDsubTO5onetcqpc63MA2NEjTcRqPnl4WCu1LYvKnJTMgkjkfWj4iwy7sdccC1IGWe0AdY8O42MONgVzNWyxlQ0nLlABJ9pNoW1DXOuU/gkCwS4hIzCAUjtaCDP3WI3mpz0KoCdwhRITryP42EcvRvKUBl4xrpBnw0+dfeXermVoW0WHRmKhIT8dD3WLNSwjdQcB10hsVwpLnMgJcKylAP6Xj3xIsyMaXS4xd5r6pJdfcQ5EicgIGg+fqs+W26cVIqKptamW0yygJkCOZ+eYs1uJV9XexdiVOdYCHMmrSiCMp09wP47WfBOwmxKZJC7kEyOBr039eTedULpkkjlIWPZ+kOfIWlXC5AveqYSpUAGQobdoT56RaFODla1R44qEuGErYdRojUnMlXlyO3M6bWmi5nWmL3cqDPVrCyFAEASZGmp5b+FpHyNaRdNbESE9AFJgbmZKNYI/ZbnMQAnMSNdSYgez5jusAivpFQtLmkkEZCI8D7x8m3JraNCRDoJSI0B1+fnlbjO3qmiB3RCvOBoKJcgk5ROpE76WR7td9JvFanySpCSQI21AJ+e+xpo09U+p+qqUpS0YQkaGY33g8rEruCGb4zuIKWiVJzrVoY10tAZQSphGQFIGFsHVV/IXWVDTjVC2YK0IJ61enZSQfefws9m8GUraRU1NMM5gtsKplAFPfsYHzy0N3PX4dZoqaoXWUAbbSEMMBbfaVt2oWDHnvNmvjPHjFWl27rkqUDPo9UA5Se9tuCezpv7BFguNJNJlBsEQI2sbcpv4weuamqF3fdTVO6AYdqENwkGdkaTp3/ACWRVtB1pTTiVERsTHPfy+fJVdCHUZTkSQdCIgd/z42IvtJghcLOoABj587W8RawWuq94JN7Ko4pxRYgv6hWkyhxSUkJ9WHI107j91rD8A6lTuF1J6zMWnjAmZkq08/x87QPi5hqh4j36lRAC1uq8TISrbfkfOBaX+jlVhygrqYK1bc1EzAATtppqoj2Wgn11CJp9DZTkPX7JkjtHTkR9cWJ34larpfhR7KQoGdAAQfbrYdpQRsoQBA5AjmANuVuLyQtV3Pt5Mw6tSQE89DqPrsLfqiyExIUG1EggpG+vdHP26d0W0qZ7U5USBpz11+T3WwZlAZz2e7bXv1M8vhbTioAUAMpkExsZg+38bdmKZqg3EqIG8SdJ5abew8vGxdwgyEoUpMaeM6+39lhnDIClRMZRIJEDX287BrAGYrVPj6oJ17/AC+eS3skIuibqQUZsxkn9LWIH3D6zYmslJKs2kmQkxHl8O7fSx9aCEFGUzoOzofZpr4WTnIKipslBERrrI3HL3eXdFpGOTCxV3vOhFLi+9qKAn6eobTmJ5lUHbyP4WenBqgeq6SupUKUOrdSsmTpmTEf5osRxrh11jG151yoS2lxuoQkiOsCkJKvrM/Gy5wQdDF7XpRFSQpxpDgB10SSJ/zvPa0k0oLDZMiZ6YBUyXVd7NIj6BAznUqGm2u1lNORCIUQBvmgGfs7rFmAhJ0TIGgmfaBr8/CxpvMlQJ9YCSAYAHKI21+21WXkqyaMuy6AOUApEggAga7fhYrVtIUgkrEmQrU5dj9s8/tsezt5S2r1RsSrfb22AfXnlvNMCNOXzp52exyRwuFHuO2j6Al1GYZXe14FSVajb6rL/QQvcYa6YGHEFeVu8vSaNfKQ7SOZZ8OsSkAd42Fk/F7XWXQ+lMggBQGaDKSBOngeQNmlwevkYY6RnDq+1OpaS3f12KcXtCPSUpWPPKqDFpJm8WmkZ1BQ49GRp8V7hOAIeOghQt1ByyDbKsQEqT+jpbQVmRmHfby0mxIWkGrQUWcIzaT77cLOnibdvHtZgfKwLi0pRmUrKkCSTysI462RDUm3/RmtuuoQkSpsBxPmnX6psx22wTECN4jT2We1wYouHEtZeFDc9cmr9ByB5SEkt9rNolWyvVMxIs2KqiNFWO0cfxaikT+qNvhbUYASWlhUNWS0WK1SMgBKcsD67LFKAI2P2WJU7ZInn4yPGx+nEGT+Pz91txTtWcqDqj7UaAHaxymVkdBHIEj3WItgnU6x3aTY40oBxA7iJ0tYAFBFOeZE2RsY4SubHmGq/CWIWnHbuvJCUVCG3ChSkhQVAI1GoFlOnWTTtkmTlAPiRp9luysJSVKIAAkzyFpiwSNLXC4KbDUS0srZoHFr2kEEaEEG4IPUHZcKXSUFJKurp6anbkkwlDaEj3AAfC1fehvd1WLlxriinYVT3BiDElRU3O0UZQpkKIK0jkD2U/8AVnus4b5vCv4/1tVhLDdS9ScP6R0sX1fDKild8LSe3R0qh/epEOOjfVKeZtI9wXlhOjr6jh9h7qqd3DtHSldCywpCKZhwKDIBgJIIbVoCSI1ibBANqalkw0a24H94kcvAAH4/ALYF0uB4LVYa9pdPPw3SC1+FGx1xn6Pc5zRY+oND6T7DvDWDbiwnUXzVXNTKbdv+8XL0rlrXmLj60pSY7kwkQNhr3m0IcVKR3EnSs4Z3fhpE3hcFI/eF7PIH8TRkwlKz/KhaQP8AlR32k/iJxIew1UUmEsI3ci+sZXwkm77uCoQygaKqqlQ/i2Eczuo9lMk6EcKYcw5wbu9y/cW32uvxFim8aZi875dZUpdZWvKDbLSUpB6tpJIShPqpGpO5stTHHPanZYNaQ5x5CxzW+JO/QancXfgdRVYXnxepu+aaN8cTNS5+dhiLyN8jGkhvvPAa0ENcWsXpk19RV4cwfgNp9bdPizElLR1mUwVsJUJT5Z1IPmkWm69rkuyvupWHaikbN3VVK5QrYCOx1SkZckd2WRaEOmRQ1FNhzB+PW2FuU+E8SUtZWZRJQwpQ7X+OlCf8IWm6/L7u27rgcxM/VoF30jPpq3wrs9SElRXPdl52WG3fajidGfq2P0vdNxTN/JnCO6b5572/tM7OntZMludrKA+iRi+8KbgVfdLeiy+vBldXUrZUomWkNpeA11gKUsAcgBZR6GtKt/hPUYurFl688S3zWV9dUK1U6sLya+1JPmo2TuiThGvqeBV91V6NlleM6+uqmwpJENLbDIPfBUlZB5giyh0NqpTHCeowlWpLN54avisu+up1aKaXnz6ju7ShPek2q8PDyaTibZHW+Po2+6tn2zMAix/uds3eoc9ulpc3y4u/K9vBGsVVP7lOmvg2soEJTTY5wuaa8UDQOPsh8JcI2KgGWkg7gZu+0pYguth6+UIdpWH1OqShAecKQCTvA9bl9XlE1dTVGO+mlhVigSXaTAOHHaq8Fo1DbjxeCEE7Anr2VAHUgK7rTNj2mKJW2DpoSBbN4nBnkkym3pkX+Qv9bqpx5zTDhrXfzhpY836z8l//ANeS392y4Vi/CFMo0ybloHEtHIFhEBQGk+pbLR+inrQkAUpUI0Mq1+NstU/Yzf7Y/RZXuzOv4rzI/KLVKXuMt1sE6N3AyAk7f8IqDPu8t7NXBV63VT3Dd7Cq+lQpNMjOFPJBE6yROnzvrZwflAaueOrTedSMlw06FZTrlU48SPcdbVydWtureKVApQlMq1JgJ5H2HlHfbZ4ZAJcNhbt6IQuIvyV0rvFWZp7yulYSV3pSAnWPSE68vt5c7KNPeV3KEt17Ck7QHEmPcdD95tVZKykhvMO4pJ5zJ5wNvqsIH3FT9IUlOyY3Hn38tO4WmND4ocVduStal1gEAKRKhEZgf2D7tZsO062VQCnSCddPh57Rapvp74dzMuqVk0ASSBvtsJsML0rfWRW1TWb9V5QExHvJ2/Cze4HqlNXfWytkgbZkZgNp194iO+3RPbSSNCd55yBPt027/fVH8/X0yjIi9ayd0FL65B10mfj8kdGKL+bbV1d8XgExmj0lYEg9wPf9vnbhROHNd3sHkrUZ0ntKHtHcfD22CXqBsVLmYHPQ7D2fhar6cZ4padStGILyTAlIFWoyff5+0RGuopx7ixtMJxHeLgGhJfXI03k8pAPyZ7ub+qTvTeisk45odNBJ7Ok7fDSxGqUVNlLaSTsExtry7h42gD932MQtJ/dDWEQkaLHsgHfUR7TbHOIuL0LJVf1QojSVJSTPPdMxpZ7aVwTTUNKVeLSj+dqMzKgyeyqSJC+fj5/faW8BEKw5dhhSppWpB0IOUTpHz5Wrlfd/V1+LadvSq6+obGQqKQABMxAEeOv1Wct18SsT3RQMUtDUo6llsJbSppKsqAO8jYad55Wlngc6NrQo4pAxxKspTlJJBkEbkax9mwjl7LH2YAKMvZOqRmmdPHztW9jjTjRvMfzhSLypEE0yRqRodNxzsKjjtjZmXlqoCEyO2wY8vWn3d9hO6SIptSwKyEAp6vLJ1kwDt9fz3WFUIJUmDodFJjLGsz3aH32rojj3jJsJDzF1GRqVMLGnMDt6jvHs8LDI6Q2LGxldu+6VkACMrmomJ9fx+vxhvdJU7vLCrDIUEKSEpJJ5FIEju8PnawxcAATGYHwj22r830icRuAOG57sUT6wAWJP+N3fPeP/AAiL2ySrDtIoxMIeXrv4GefybR92lvslFSzqp4WtKAEFR3ywR8nvNuS4FISSVSO0TOnz+Ptg09I2u06zDFOoSIKapXuEo+322FT0iCSpbmGGxlISIqzqd49Xw+ubO7tKOS508Z5qai6VAEAqA7Wnd9upsQqlkghKZkxJJ5777crRK30i6fID+5dYM/8AlckHYfoTuY99unOPl0vw2/h6rTnEghxKoII8Bv7LOEEo5JnGZ1SJxqfUzeFI8sZgWwTB1kFW3j8dNrOThW/nwnRE5xqsTt+mrztGfETG9Hi1bNTR0biC0kIUXFbjWdtOX1+NlrAXEG77kuhFDWtvKW2pfqIBEZp3Jnu3sVIxxhA5qBj28UlTtROpQgtgTBmTv5aWPBQ9eD5QAOcjz8u/xtGNNxZw2k5HzVAnf6OY9x5kR36+VlJji3hhY7dW8kmZSpkyNPb9f4BFjuiKzN6p/FacxCEKEkg+J7/h8bbD8iU6RIkaa91mEjixhJ1Kc1e4kKkSWlacjGmlhDxTwm5BTeRG3rNq7tDt7j5Wbkd0SZ29U+FPJ0zKSUjQgna3HWJglJGWZMQSe7wszG+J+FFETeqd9SptYA8J9/12EPEXCsJz3qyiBMkqBOvx7/bbsruiW7eqd4cQSc2VI75kKM/ttwtcnKZJ2MHb5+/lpZqfvhYVUlSfzywMp1BBTMfPL8baXj/Cao/3bphEfpHmOfnHhysoDl2ZqcNQsFEyQIiBvvPKzE4jo664FwicjqFacxrp7yPOyu7jfC6ypQvqjJnYuga6aRy03/bDYxliG6au56ppi8ad1aCDlS4mRrG0+dpYswcon2IUc8NH0s48ZAQ4A6l1oEbA5CSNvC1h6FxuUJPZ5yrY2rPgepLGPKF52oSGy6qFLVAnIR7t48fbaxN31rSiAHG9FCAFgiPHXWxVUSHC3RQ09sqc6HNM60q0ggE6nTv8p18RbpSZBz9kmIIid/2/CxNp8FHZUlYVuSrMYnl3/jYQrz9gpGYyQkkyCdPu91oMxU9ghswBgSZTp90fPjYMhCQewlWXWYBG2/hbgOZ4kSAQAO7YkeHK3KyqB2ZMSRMyPD7/AJDM5BSZQShlKZmerSJ0mIGnz+2wK0NqOqNROnPfb5j424DyTKpCRrprt+yfdblLgEkgSO8fMb/OlnNcRqEpAIXDjdMCr6M7SrnppyNib7TCZGYTJT2FaHz+/wAbG3VLnOozEdkkxofwn287EKt05NFDteeum0fPs0tK2RyiLAoF4n01NQ4zrlNtKQ4tDa+ySNVIA1Hv2/GyxwTrajJeTbDz7RStshTbhSe0FaH/ABJ9gsU4xJ6vELC1jsu0zcidZStY09hFiHBSp6u+7ypVKKM7CFkAHdKoM/4xsdvDdCtFpFPVHX3lmSRelXkKcxzOK8Nfb42PpvC8XUFs3m6pKvWlUztpFkWkUIBJgpTymAdz7tPdzmyihRXBC0582pg+/wCoeQsEXoyy6bp0pPVlxRUkwZ/VP1DWPbbn0FLcoPZMRM92g8omwrQMEkBeSdT495t1mAISkZ9eZ32+0fCycRdkRVVCpxMJfO+vgPDT5iwSqJbR0dUEjUZtY9njrZQUVAICgFAbncjx0tysDMQTA2mM2sd3vtwkS5Aks0CiAAqFaScs/h3/ACLCUV0JC5VKiCTAP2HxscWpKZSoGSDm1IB9vt+q3dMVJz5VDNGndpM7c7NdKQlbHdRRxau8sXwH0pgv0III5QVIge4cvwanCir6rGZAJCaph5J8QClUiRtpGndGlpF4uMoWbrqVIzJIdZJGxAUk/wCsYtE+BKhymxjdj61xK8g7W4UCO7mTPPv8bTxEujKhe3LIFYptzLBSokA7JBgDWNY+YNjrRkCEGBtM/M2SmHjl0EJPvmO4DXysptuAqIWEjMZ0Op08/mLB3sjLIdKwiSkTOqgRO47/AGmxZ10BPq7QDCuXlHh8xbtx5BWDok7etMfjYq8o9WcxGh7RnfyIGnK3Z7JpaEh4hX19DVskHt06zESdjHx+e6E77vFyjvK57yp6gldM4VpVI0IKSmPD7xada5lDh6lYlKuwY3AOm3t+dbQBillTVFTrfzD0d4JUTMTBBJM6j7VECxsDs/olCzNy6r32um82L8uGhvemUC1eFK1VNqB0KXEBQPuNhadcoKI2Nod4C8UcL0nRd4f43xXf1Jd1EjDtFTPP1DoALrLYaWkc1KzNq0EnS0aYz6aGE6x9+7sK3i/RUmoNUunV1rgmJTyQN+9X8028ykpZnTOaxt7EhaOnAkaATb4qwuMcfXFhVBRUOmqrYJTSMEFw+KuSR4n2TaDca8TL0xJmarqkNUavUomSer8M5/vnthO3Z52hiv44YXqFKDV5KlxRUouBRUtU6lROpJgySZsg1fFi4qkdZ+cRBIzTOg8J5/jaWPDZb3LVcxGlgHrAlWc6OmIlVeNrzutTuZNTd/XQf1m3EjT2LNpdxXR9XeDdSgAJfRrA/STpPui1SujTxFuap4yXJd9PXJU5eKKiljWT9CpQHvSPha6OJqP0i7S6EyphQXPgdDa5w5hp5gHBVOIvbI4lpumxTogCI1310saQmBsTG8d9gGUmNSJiT52NIBMb6c58bbWDRZebUoZo9oaD3WMoIiNZsVCCDBEgibDoiO8AWPCDcbJwULhcY03CtdO/X7bJWN8JUuO8J3jhCvvS8rvp7zbDD1RdzyWn0ozAlKVKSoAKAKTIMpURY9dawWVJ3+SPqi0C8WukbxAwXxTf4bYL4f0V/ONUDdcVLfUheUplZOwgd9mVVTDTwXn9V2mgJvflpqgZsWGCPZWZiHNcMpAJObcWAB5jolCi6H+FLupWqK7+K3E+lpmU5WmWb9bQhA7gkMwB5WfHC3gtcXCqvve8bsxNiW+qq+kU7dQ9fVaipWlLPWZAlSW0kD6RW88oi0P0PSI6Sl5SLv6P9NUxqQ1VqVA9irCP8f8ApOUykoqOj0y0pfqhdUpJPlKrVUNRhsDhJHE4Ebeg/wCHRWGIf+qeJ4pA+lrJpXMf6wMR1sQ7WzAdwD8Qnnf/AEVcK3/i69saq4i8QKG8b5eU7UGhvdplKUzKWk/QlXVpEBKSTAAtzd/RTwvRXtdt71fEviLef5qrqe8Gqavvlt5hbrLgWjOgs6iUjYg9xFmp+/n0p4zDo4af9Mv77aRx36Ujklvo4pVB1y1Cz9tl4mF5s3Cdff1H7+Sk/wDq1jIiEIqJcoAaP0R0AFgL5L6DRWQvW67sv27am5r4oWayhrWlMv07yQpDqFCCkg7g2jSr6PWFRcJw4rE2LX8NMAdXcLt7FVEEJMpbJy9aWxpCC4U6DS0ejjn0qQdOjd/pl/fbMPdJTi4viVhrh5xB4TUuHf3SPdUhxx5ZcyagqSNjqIsVLiFBO4cVjrnS5Y4bnYm2xPI6Kkw/tzJhdo6WWSMOcPZcBm2DhcWDhewcLEcirFXdS0VFdtNR3dTN09KwyhtllpAShCAAAkAbAARHhZj35wgw5eWI6nGFzXrfOGr5r20tVtVc1UGfTEp9XrW1pW2pQGgVlzRzs9qAn0NKDqpBKT5g27Vpsdjra1khjlADxcD6fDorejxOroZHS08haXCzv7wJvZwOjhexsbi4B3WuC3D7C+A6e+Ka46d5yrvRxNTX19W8p+rrHNRmdcVqYnQaASYGtlu+6RlbRqKtIUinGZSFD1zyHzym3OEXupvcNqOjzak69+/2WO3+3Vllxlqkbc1UVAu5Y7uXdFsr2iijhpiGtsB0H5fFGQ1tRW1RnqXlz3bkm56b/DQdBoo7dxDX9avJWVqU5jAS+EpAnkMug8LZYVVBeWYw8lAn1Q+YT4Ds2y3m3FpervqtFwYOi8dunfUB/pA1CFbM3XRtankUlUR/hWr86jLWvBzIkyJ05zqe8nfv8rTr04HQvpCXxBSS3R0KII2/uZJO3n9doPTLrrqVAqzGSEAakE9232Tb1PCfRoIf8IVDiZvWSfErG1qSoyNQCNCN+Rjl49/xtigN1KXJExy8ft+NheqSpSCdSDI1Gschz7j5W2UCUoUiSog5laFfIGxpIQtlwHVgxlQTlJgkiJPz4k+y3KioEqBAC5mDIMQfEaWFcaCD1pUE5tpIE/MC3WU9bmPrKJB8o1n3e6LICEhCLJWpIBIUCJJCjlk+zU2woBEJSoA6AmZMjkdp52OCmKoLSRpOhTA7+/XT6j3W0lghZMZyYEHkZ0juP2g27MElkVzElOTKnNsCNTzI90e4W3JKerTmIAOh90fPON7GmmVqCsyypClBOTLMmDv3+PyLcoYnKFJVJACgQZTppAPkPHS3ZgusiinO2opBK5111JnQ/Xv499tFwk50JS2F65Rpm9o1A0ixkMFAyuwduypMwYgac4/HW3Cmm0ELDiRlB7O5MbR8e4WcHDZNLSij3Z7RUnsqKZCvVI2A7xpv7bHG1fQtg65E5T2tPGY30nnz8YsJc9wXnie+Lvw9cFCKq8rwqU0lLTpUBneWQEpkkATMSSBp4WmdroZ9IjKnPw+Da0gZpvOk1MAf8bE/daOeohhAEjgPipIYJJNWAlQmXShMNgyROaI18T98bWCXmWuS2THIfHy8vG08NdCvpFPNKcVgVjKTAUL1o9CeUB3Tn8i2K6EXSPfUAcDMpSYJWq9qMSOR/jO7lYcYhS++PNT9zqPcPkoKzpCgs5UqTppqCfGPDSNvLWeSochmSoEp1MnXY/MaeFp+p+gf0lqlCk/uLo0riQDelINQdNl7fZ3WEV0EekutCU/uJpxoNPzpTGTHP6Tw21Hxsv2jSDTiDzXdzqfcPkoCZrDTLUpTbbgTpC0kSDG3w/ZYNyoU4ou6JAE7aRzjw+G/facKroV9IqhfUy7gjrFqM5UVTSgk7iO1G/ME62DT0NOkW8pDDeAnOsUfWXVshA3gklUD58rKK2lOoePNN7rODq0qFQ6SohWUAjUpM+7y89dfG2lVCQf5MwsjXQkd+h28vO01VXQ66RDLqmXsAvkpUCSiqYWmYGxC+6J1Ni6+iH0hmjH73dZG2j7MbHYhfjy8LKKum98ea408w9kqG2XQAUZpOsq85gRy0G/gPC2IqUhSVlBIUCFlWmknv8NNPHwtLg6I/SDbRnVw1q5M5gX2APW2gKsG/wBE3j+kKUOG9aM2WIcZ7tf09f2a2eKqnJ9ceaZwJvdKiB1xaUASYkmAIzDvnlpr3faPROuBpS0LJSdiDBkGYPjqLSo70TekE9HU8NK86DZ1obA/yvnX2DUfRH6RmY9fw0vGEnTK80ND3jOe4Wcauny+uPNIKabN6p8lFwqAkZwoxGVadCdB4ajYx99tpq1pbIBcTl7EBRMeXd6w89LS3/BN6QKklP72N46Jj+NaBkCI0V9n4hq6JfH0lQ/e1vIlSZSSpoAewK7p277QCrpzu8eak7vN7pUTioUSVOrAJIykpOw8J0t0isW02e0Qop1E7jl7Y28z4WlJ3om8f1oAXw2vHORtnbEf53L26E720eif0heXDK8VTuoraHhGipMiNvjrZ3eaY+2PNN4E3unyUWGvEIlKgoJyhOsx7fmZ8LdJr1NthQASonU6zptvI7v22lRHRL6Q4zJPDi8NSdOtZEAxp6/h8m20dEfpCFJnhxWjSIU6wrmZ1z78tfvt3eKb3x5peBMfZKiY1ThUSoCTr4689e/XS3KqxeUFRSc3aVl1J+fZt7LTBS9EDpFVSi03w3q0HLKg5VU7YG42K4OkWJvdE3j+HQ0OHF5GZkB1uNtj2o1nv8Ld3qmGhePNL3affKfJRT6c46rMZPb0nskjv8N+fjbTlcmAnOCUaAjQqHPX2fC0qL6JvSFT2Dw2vFRQqAsLQZnQkdqOQ19vmGeib0hFZQeGl5gL9WVogR5K25bjvs4VdJ748wm92n9w+ShxmoWl9JzdrNsDz0nf2WFcq1+qFKEjU6RuQfLl5WlUdErpDkZRw0vFOUzOZsRJP8r32EHRF6QwST+9tWme5aJ25jNJ2+u0praS/wDON8wmCmqBsw+SilN41aD1ialaCDGiykAa+Phbv8+3uysIF41Qy6kB0jUDfffbTwtJq+iR0hAI/e1r0jLB7SNQNP1t/ONbdHok9IVaurHDi8MwJUR1jYnx9bQ+r8bKKuk99vmFxp6j3T5KOGsYYiS39Ff95DNqQKlYAOwPrd3yd7DJx1i1IIGJr1yp0CRWOSNe+SPZ3m0hO9ErpDglS+G9eDp6rrcnQ8s2mvh7rcfwRekRmP8A2NqyIMEuNfarxtwqaM+23zCTu9T7p8kwkY+xe08ScS3vnzapVWLMjmN/OxlniTjFAKP3S3gTplBqFDvB15jl7fCbPMdEjpEFWQ8OK4DVQV1jcQNp7Xnbr+CJ0hUkj976rIMiPSGhJO26/jy58rcaij99vmEogqfdKaA4oY2Ay/ulrO0TGaDm18vPbx9oiuKGNJyu4iqlZRoFBKt9tCmO+e+zyY6IHSOdCsvDmoEDMYqWoM85z66792lhH+h30jKV0U6+HdSSkBQLdSyoE76KCyJ35/ZZnHo/fb5hLwKn3Sorv3FN+32pLl83gp9xsFDai2kEp7uyBOveJ91g8PYjvXDV4rr7teCHnEqSr6MLGUQZg6cpkWlN7oe9IltxpB4eVLKXlBCVKqGwnUbE5j8PGx5noSdItbgR+4pMrAn+7mUjuA1VFpO+UgFi8eYSCkqib5D5Jn0nGjFtKgAKoHQkdoKY38TCtBtY+jjxihICTRXYogx/EuDSeXa8PnWy690K+kemP94SJ5n84U5BHf6+vf8AJsSX0P8ApEtEBWASE6AEV1ON+/t+No+NRO9tvmn8GrB9Q+SCRx9v5Ihy5rtJBIykuTMan1rHGOP1eFyrDtKQmZIfVoZO4+3w2O1tDoe9IzN1QwMcw5JvCmgaDmHO4fV3myfi7oycccD4frMX4mwQujum7kJVUP8AptO6EJKkpBhCzOpHIxNm5qN5DWuF/inZKlou5p8kuMcfXkktPXBTqUDrlqYHn6p+fISMOOtOtErw0eW1UNR3+r8frtBSHUk5QoyE6STJ1/bYZFYsEBopzEQSPH267fVaQ0jFF3hynRnjldiwgOXLUJzJ1yvhUc+7kP2aWUrs4uXK9VltN11aCQSkhSdfLXl86RNfkVLpV29RIJSQT4+Hj9lle465Yr2yAolRLYHf2fj95NoJaUBpIUsc5vqpXx/jG779oaZmmpqlDjS1LCVAAGUjSQd5A+RpFdJWCkxCxV9USlis6xeXtaBydh4c/q0su1jyXGxCSsp9XMQCI07+6LM+vU4iuqFEQCpJASZEQI25afCzaQXBCWZ2oKsBT42uaNFvJOhgo9w15R7tN7KVPju4MqUh93s97JmNgIjW0PUz49H6xOYZv0oMRvHdH2c4MWP9coJQsFJAGkElIJPzz5HWwb2kFTNkJUtnGtwZEKVVEDMdQhczpuYtwrFtxFQR6YJUBBKFcx5fhpaJ01JS0CSpJgDxV9vPwskXri677ploLDrwjsIKdDvqeW577dHE+U2aEr5g3VymOrxFchZ60XkhCQNVwQOR3Ijf7u+0D48xNdL1fW3fdig+h6pJLiTKAnNPZ0E6zy5+Js3b5xVed8rUHXktNKVm6tsAJ8J7yLIThJMkzrM2uaWi4Wr91XVFVmFmqVMBXvxq4gMU+DsNYlefo8P0yjSUdQ6ktU7anStQbSoECVrJOnPfazwqOG3SPZQVrcoXRrMU7BnTxb7o+FoOubEN8YZvFytuK8HqN5QKM7SoJTIMeI0G9nbT8eOKlM2GRi6ocTz61ttf9ZPlZlRTT5704ZbxGqnpqinDLTl1/ApyV9xcf6Qn0iiaJA1UmnpdB55dLJDj3GCmAFRQNkmRqyyDOo5RBjTTWya/xr4h1Kf7ovwLkyf7nb1Pu8rEHuJ2K6hHVu16SO8NJHd7tuWmlkZBUe2xiR80F/Qc5S70d8bcQsP8fOHl53tTBqkRiS72qheUD6Jx9Latj+qo29ynWE1DLjCx2XElB9tvnYu7iPiGjvKjvD0tQXS1CKhJToQUqBGvs+Plb6IrjvSnv25bvvukOZm8KVqraPIocQFA+4i1Pi0Lo3seWgfBGUMocHNBJ+KZqGVNrU0sQpslJ05gxYdsDx0Ee2x6/qbqLxUsDsugLmOex+INiQGw5DT2WtKZ2ZochphZ1l2lMkd31WFSNNQNO42DT6w00+2wo1E6QbWAQbhcJSulwElA5zr7o+21VeJN0XtfnS+q7uuVtLtUvDrSkoUvIFhISoifIWtHd6gH0jxB+sfbavF6Vgu/psVVcUFYp8Krcyjc5Wpj4WBxLVsI/wCo39qzeOaOpj/1WftT+u661YPxRd1PSVjy6h9DK6unU71oZWowoJO4SR+jtpIAmyjiTDN93pfV7VNW9U0FOhtTzFcktrRokZU5VSY79BtoedkzhpXXbeVdUYpxHebIrHXCtKVkmCRv5AaAdws5eIt7reRR3RROgNVQDrjmaE5f0ZPdz9lrNuUMJ5LQOJzBJtLfV5PYCqgXVpFPUNN5idQ2pSZT3xr7rJlzvV+Hb4u+9hea3ae8hPVFKQG8qsq0SB2gZB11BnXYB4C6rsRgqtua7K2nqX/R1uKLa0qUpwCZIHjHwsxql1NXh673WlhS2Ktad9Q24jMD5EptLawBUR3Nkq4pN+XtiC8qShrktVFOCqmStJUiEtheWARE69obeNofxje9Re/SK4OrqnVOrZW4gLUZUpMyJPPe01V9QGcY3dXK7CK6mYcUDoNZSr3DLaBsSNrp+k3wvpCf+D1TzInuStSR8ALVuMW4Df8AGz/cFQ9oP6I3/HH/ALwrftwh2obGkLzD2i2LlR02tpwFN4LJVo42DHkbbUNdLWxV0Eaul4U150z+wS6mfImDZ8XhTBTq9PWRaOxAO8RqDaSlLFTTMVIGjjYV7xNqrFIRJC4HopoXFrtExnaEB1Y10UefjbLORdIkqJybm2W8yNOeiu+9leCvTR7fSIxElBVPVUKYH/8AKtd9oeKVOVDwLalDrSM87T7O8xtrrabul9Q1F59ILEdXdzJqqZfoWR5glaTlpmgdU+ItHQTdVI5UpqaRaX+tIjMBy13Ply1tuqF2Sjibb2R+AUdb6VVIfEpvmnXkKurLkGCAI0I0B8TBG/K3TaFKGVQMTJSEQT2dD4b2cbN/XLTJLSKQt9YdQVJAA5c50520mvwho6aR06aGTKTOmpUJPPusRd3RD3HVN5NOtQSCFDIJJjbTczz11526TTKSQrq1qOgEE7xp5ftsvJvDCi/oadp3tryJGZQiPMzz84B7tQ0VmGlqTlC0JX2dwPGN9vgJsl3dEgtvdIwpldYgsqIAGsJnXeT47cvw7S2HAtsAoQQpKiYg6nmZnc8513s7riuiiv2ndqaMHI2soK1LA7USRHPQz91lIYNaSlQTlECACuTIExyjTW0RlINlKIy7UKP1MAkjIc3qaAb+Hdr7NAbcmjAGWCFI3gEcj3coP18rSIzglhJyvJKU5ikjrUyCCRvG/LnsI8DCMCXWohKy4BJynrUanXadz59ws3j2TxASoxUylWiGxKUEFMTJH4e/nIsG9TOskZ21QRA1ISTtM/VytK5wTcwSsOVDwC+5xGUkiZ25z8RYu9w/uB0a3g8kGSdWyQNo21Gnum3NqhdKaZxGiRej07k498NS6UgKxRQQNx2n0aa/jb3KTS0DwStdIwTl9Ytpn6reKnDK5qe4ekPw2pGHnHUDEt2rK1lJOY1CBuPK3tMmqCW0x+iIsBikfHcx1uSMw9xja5p6oQ3fdqCVCgph/wBUPut2qiu1WnobBO2rabAO3oypsU4Yhc6uTpHlYNVQuMuaRpJne1caIBHCpvuhjRXcDpRMeP0Yts0N3KHao2TP/Jj46WKG8UUpK3EKcnQBO/nbhFeVjMYA3iZFu7kLXsl7yTpddru2gL3WppWQonk2JPmbdKo6QalhsR/JG9gzVKPaA99tF+THLvs9lKBpZIZ/FGW6Ojyx6OgQP1RvtYZNFRAEdQ1GsDKLE26pI3V7NjYUVOmpGg2i3Gm8EgnvzQ5o6TVPo7cHQ9kGwYu+iUZ6lHtQPttyKnMYKR3259LExI98+63CntyXGbxQqKCiSonqGwYj1B32FbpaQf3tInnlFioqBPd5CbdekmRoI5gmymm8F3G8UaTT0iUkFlI0/Vty5T0syWUSdiUixf0sQRIB+23KqozKTJ287N7r4LhN4oT0ZhKtUI0MaoHhbpbNLorqUEkfq/GxdVUqPXV4eFuDUnYn+VZ4pPBJx/FDmko4A9FbkfyRFg1UtGAR6OiBoeyLAiqGwB2tx6URry33s4Uvgu7weqFNNSAZfRmTH8ixarpqXqtKdr1T+gNbYqoMGZI20+fnSxerqEBsEymARPjZktH6Oyeyp13QtO1TFxagy3OXSALcqYpS20kstGTsUg2J01QjOiSJUifbNuE1RzI1kJI1nvJ+6wJoyOSJ7wOqOGkoywVGnaO8kpHI+Wti5oKQMFQpm5VGyRp8LBKrgGIVuVHnbp6pHosyJgHTUb27ud+S7vFuaMpoqKP+CsgR+qLF3KSlUtf9ztysgapHvthrQG8wIIjmbAIrEuPyskZQSOXz+Fpe56AWTe8Dqhqqko20t/QN90BA1t2qgoVJj0VnQDdsWLVtUFJSSNlHnrNuzWJUEqAExOlnso7HZMdU+KKu0NEapI9FbgJiEpAsVqqCkfUGkU7Mjc5IPKTYeqqwHisBMJTPgd9LF2XyUhawoKWrvmNDp8Js4UmqTvICKVN2ULSkpTTsBMAkBIB38o+Z52RMb4jwngy5l1985KZDhSy0punClkqMSkRyB3iBFla/a1q7qNyueKihppbywnUwkE6eMT8fG0IcPOK7mN8U0983gZKWPSg0XZbYSsgNtDWDEyd9QTpGhMVAXnwTHVYbZPpGI7wxS8tGCOGN64gdY0RV1zq6eka/kJ6xRSo6knK2Umedg8S1XGbDF3emO8G2qtE5nDSVDTwbSQMwCMoV58trWCw1elB6G0xSMNiESCjQKOu/eTv52PX1eS6dkKdS2lS4GXQH26zYluHQBuZyf3mRzg0BQNhDGOHcbUqwzdrdLXspHpNI8hCXEGNSUntRPgR42PVV0UTy8qaJkRqqGx2dfZzstYmuS4W3nsUMXOzS1obzKrWBCnAnXKobEECPb7i13w9Tt1KSIdAXIiACJ39p+RYOOna5xa0JaiR0QBKIMXbSoqOrVTtesCBlBgWanSCuCgvfgrje7xSMkDD1ctKeqTopLClpjuIKQfMDus9KmvS24qmTJgZdAkDNmCs8xmmAU7xB2nWxHGVMi9sKXxdgSVem0FSwZ7ltKT9psTFSBjwQEHJVZmELxDyozRlABMGPf8+Vu0FJkKXlKSEzt+y3S6ZbZUqTIPbzaAakCfaPC3fUuOuFKECFJ2SnU6/O3ebau4WbsQtoQsq7K0kJG8GFfMH32OXWVNVjK9ey4kZkjQSR7PhM2AYaXOYtgkETmE6SBrJ+fqNttobh1CEqhWaTECNhrvpEx8LQSG4IUjBrdOqoUkJCFAZldoBSO4941nQRrz8bNq+lKFelWvqCOzObU+3bcn42dVShPUhLZlKtFApmSNBz8N/Dvs1cTEBLS0pkZVSdDOog76alVgqQgvsiZtG3ThulxTlEw4mY6tHaSqSI09sDT2WDvK9aWhZ9Iq3MhjsR6yk8oHOfsHdo00YmXQ0DNJTsJ6xufpFiYkkgRz3O/usi1L7tQ8XX3C6tWsrOtimUWZxLtkO6osLDdLd8YxrK9JZoppmtZP6Z8yPKzdKpOqjPjbR00Bi2jMa92gseyNsYs0IR73P1ctiRA+q2lAzrztuFd+577bdggGd9Yi0iZZcrGoKREgW0RA0t0QQlCtYMgH222pGUfOluXWQYJ7rYdLbg72yDtbly0CZmLfQF0Q8TDGPRi4Z39nKlqw5SUjipOrlOjqF/5zRt8/p2t7UfktMTG/uiXdd3LUCrD183jdu+sFYqB/8A7PwtT40zNC13Qqww11pC3wVoMS04XTt1ATBQrKfI2bsez22eldT+k0brAAlSNPPlZmwQdRBsPhz7syouqbZ110AZkEeXIGwiVEka6+fz424GU94Bt2BIIynxJtbtVeShmHOrdCidBJPs1+y1ceLNy8Wrg6Rb/EjBPDOoxNRLuVF375Wl5kFKxIMyPttYwQTlWeydD5bGyvRLU5RsLX6ymkZvONfjaKppBWMDMxaQQ4EW3HxVNieHjEI2tzlpa4OBFr3G24IVPWLx44Usik6NNewDybvapSPguyne2NOkZe7VK070dqxj0NrqW1M1ziTk0gHXXb67W2BMET523MGBvaIYZN/7h/k391A/ZlYf65J5M/dVSMP436SmHatdbS9H2tfUtBbh+rUpIBMnTTw91k5d+dIwVqqik4BV9NTqWXPREPhTSSTJCcwKkg9wMDYQNLXJO/K2xGwO20mzvsua1u8v8m/upv2bV3v3x/kz91VJxDjrpKYgpaRlXR3fpHqEQw+w+rMEkAFJBkKSYEgjkCIOtkXCmFeOOKuNuB8V4v4W11z0dy1UPvjVCUGSVHXTX67XREATMC3aVHfSyOwd8tuLO5wBBt6PI3GwUb8GlmytmqXuaCDYhtiQQRs2+4RSsTlqad3vKkE+Yn7LdLSDuAD3Rbd4QGkOH9BxKvjB+221iAPfa4I1KvroEiDrpZ/YfdFTcNOZkty2fYSB8Iswle/7bPDBbpXd9SwZ+jczD2j8LC1TczCE9pSiUanS2WHKJJNstlO4tRWZfNFT1K6Jltpt1xJkglKoUdSQJ9u1sXUv9cpDqHErQpQUClPZ7xp4k6eXOxK60VDzNOnrJUox2SBB2A8/D4WcN3cLeJF+3k63ddB1jyEKdOaqbGxAkknUyR5zbStgLnWAURlyi5KSkVyycsOnMOzI1B7vkaxr4CtXgER9C4ntCQpAJJ0jx0J+d7OQcAuMaB2rmQkTJmtZ3/xvDc2NMcAeMClKSbrpycpImsZOu4O/gfmLS9zefZUfeWe8mmm8CpKSWTGoJLWpOhAMeUR4c9rdemJXCEU68+x+ikwRHdB9/us5TwF4xpQS3crJCz+hVMTIn+UJ0twjgpxjaRH5kEAFOlQxzM/rfM2aaOTolFSzqkZm86ttBpaZ2qQgystttEBR/SUSnxy8h52ERfl9BWVNZeOoKRlU5A125xy8oAtJeBuB+NaihqXMQYb6yoQ+OriobEjKeyChWusanxs53uBt9IUlVJhVTYAgpTUAkmU66r001ncaWGfC5rrZUUx4cLhyg9q/b+efSUVd4EqTlBGeQR+IHd7zZQaq8ZrQDTpvPUAylpYKtTuAD4+E6xaX6fgliJp1KlYbqGgFDUVCSdSZHreAHvsfTw7xdRLWmlw66AkymMquW2/z8bDSCQerH9ERFkPrPULNt8SnUFLVDfakCCZYWozGh0T7O7ffe3NQviTQNhdTdd9ozGVLVSLlX+bpqPdNp2RceNqVK0OYXW5BmSyAdSdsvMQPfYk7duNy6Gqm461IcGfSnUlIEDaDpyA8rQNdPfWPRF8OntpIbqPeB9ZflRxzwCm8EVSHEYju1MvtFsgekNnmBvJ+d/Z9tw5QZiYOlvIzCN3V9Dx64fP3hQu0q1Ygu4gKbKCQKtI5nb8PG3rg2QUJGWFRO9uqWZ8ptZMp3ZMwBuulKBIkkkD2zbRrW0CFDYfP12CcDmQlKTA593t91iS+sUSUqMd8bWayEbJJJrbI0XGXF5yRpEn587dhxnLlGvfr9Vk9CHnVFKElSokgC3JdIUErkHnPnaXu4OiiFQQlULY3UOe9tlxoSQSn552S0vAjQg/ZbC8JIzeR+23CmC41RKVBUsb6jwjYWwVdPyV4eNkguiCQeUaW5L5zRmkToZ+yzxStKb3ohLaqpkmDJI03thqWZBBJ/bZFD8kgK/nTbA/l7RMjeN/k27uoC7vRS36UzMQTtFtiqZMzpO+tkTr5TBM8+WmnfboVAAEkgefKyimFl3enJX9Jp5g8vrtpVVTzqPfZIL8GMxHnbXpIUrUieXl8i3d1au705K5qmM0jSbc+kMc0qEaWSuu00CtB9ttKqNgJgxtZe7Bd3pyVjUsblA27/vtyainmMp1nnNklT/NSt9d9LaU+dYMzz28LKKYJhqTySqXqYfoDwk2LV1RTlsjJuCBBsS9IVoCrT4D50sXrqgKQlI38/C3PpxZObUm6MN1dJ1jX0cfRR7rcN1VN1JWERLgG+mlkj0rIhl0aDqlG2m3h6ASDsuTqNdZsOaVqlFU5KZrGSIy6yTqdLCrq6Y0olP6MjWO4bWRS9nDywQYlI+36rb60mlCBBCkxE77WTurU7vTilV+8KRLIT1YTmHeZ0tqlraYuhXVqlSTqZkWQn31KVAPZAgKkae+wyXUtViIOyMuu234CytphdIapwSzeNYwGhDegIFhE1bCkJhrSO/XbayHXOyxvqDJBneeXz42ENSENhSlQEp5Qdfn6/G0nd2hR95cjVVWU61JZLM5vEz5fH5mw6XqZOUdUMqSIT9sfO1m87UjMlaSQCqfWkkA68/Hw9tlEVAJ01gwRJ17/ALDZRTtSGpdZD3s5S1CEtrYRCwW1CTsfH2fXantDcl8XXxRurAN1XLeV2tmuUp56opSwpdKhRCVpzpBUFGQFJkQdDqZtpVvyhtSUJOU6JUJnw30/GyFe2F6C76tjGFEmlYuq7G0tpp2W5dqHXlS+44omcyQEqEAeqSc2hEUwMTSQEdQ2nd6RTpvaivdFzihuZNTkd7Dy2qs0y8kAR12vVDQkkSoxAHdDvDXBt8r4hVThprso6cvH0hyjviovNxbMkAKefmFRrAAiY1iTNVViakw/dT9a46l5FM1m6pogF9WmUTt7Sedo3quJ7tNTVFXR45wld96FyU3a6yl5hLY/RCUKbUVGI6wkiZgQItXNe1wsVo44XXzNSDju7cV0HEKsu0O3k83TKytqpL1LJSwvtJdUyew6M3ZggkQdRNpWo1Jprup6NISSwy23sCAQmNuXfaJmOJV6YnvH87YjorgcfQpLIRQq6zIIEKCiTBEz5EeyWWmFuU6ag+tlTI2O28Hb8fc7Dckkj7ckFj7XQsjzbn/wki8FlT6shCe8AnWf2WFgrSG3CQFdk687Gl3VUVIeqaVJU2ynt9oAzBMAc9EkwO6wZQAwEnWEgiRGg/H52tZcLVZ9slwvNm8sCcHLvvu8LuqnrhFRTVbtO62qrSlSVIWpJBHWCCIiPC3bWC+CRICTcS0ntDLeBAPuc00+drNLjhwLvV7jDjN+kqHEt1F91j7Tfo61aOOFYggbDONf22YrnAnEAWA1UAhR0+iXMewdxFhXQQk6zkFHNmlA/mAVOKeG3BVxtB6u6w256gTeSiFDYH1/EWOU/CfgzUFSUpoxlBKibzc+wz3aedq/jgXiNIn0tM7GGHCAqdpKRH26+0B3gtihqR1zZJBgdW5qRymN/nnZDSxu2qD/AB81IKp7dTThWcVw54QFPVIabUvKEgN3g4pMwJHmALU5vEXjW3tU0BUt5bS3G0Ib1AyzMAeCd/bZfqeEWKmE9YHGsnInOJPd6tmZX0dVdN4u3fVAh6ncKFiTuD77HYbSMgLssmcn6KvxGpdMG3iyftT6uHC9w12FKw3jcl7ov70gpp3swFOGiB6yT2pEL8LcqwDd8KQVViVA7Zknu7k932d1m/hPCuIMV+l/mZ9I9CSlbnWOlOipAj3WVhwyxkAmK1gETAD6tOZG1iZHhjiDJZBtBe0EMupOwLwk4N4oraS4nncUpr3mSVuFbKGStCZXlhKtN4m0judD/hS071buIL7ZOmgrGCU+ctfOlq50XD3iOw+HrvvMtOgZQtutUg+QOnf8bKbOE+M7YSGcUVreVOUAXs4nKBBjfTb4WrZo5XOvHVWCtaeaJrbSU1yp/HQ14WOphGKr/SIkS9TDX+jn7dLMDjx0ZMIcMuH7mK8N4jvKtqWKpllxioW0pIQuRPYSDIOX38rMsYd4/IICcY3r2DmEX252T3+tpYO8sI8dr6u1Vz3piiur7vdUHF0798KcaK5mSlSoJ57WbC2oika59UC0HUaaqWeSmlicxlKQ4jQqJMgLQOYSFRa7GEOgbgfFeFrmxOjideKfztd9PXdUmia7PWNpWUzn1iYtWZHAziMUqT6AwAVag1KNSNufj8bPq67p6UFzXdT3XdWML1Yo6RAap6du9DkQhIgJSJgJAAgbfG0+JzOnYBSVDWEb7aoTDImQOcauAvB2Uy1n5O25EaUfEW8lq2j82oVKuWocsmO/k660BTiMe1aECRmXcpOvd/GAWjmpvzpf4coKq9v3bX82xTU5cedReIKktJBKlbzoJkjWzPd6VnSHfc61/ipfLqoABcUhcR3SnTblvauhp8ZlF4qprh8P/CsJqjCIzaWmc3+Pipoc/J438IDOPC4FAkTc6xoN/wC+cudr1/k7uEt98EcD4rwLe17CvbfvVq9WFCmWzlLjQbWIUTP8Sna3nlgnjB03sa3Z+e8H4lvSvog8prrk0VIUhaQCU9pvQQRptr52uf8Ak+cYdJ68uIt/3Tx4crX7qeuYuUDj9PTNhNSh9EiWkhRJStW8js2HlOIh3DqahjgNwPW/BPAoHMz00D2nqdvxV9BMnebNC9Kf0eudbjQqzDXkbPA6c/dZCxFT6tVAGp7KvssfQuyvt1QFS3M26RQkK84iI5WETA184jS2RsInn4fOttyDzIjebXjVVlbAIOpI5WVaNwqZjmFKj2mR8DZK1EwD3Gx+hWOqUgHmDp7vstPHuhn7I6I3AO1tgkCDPvtwDA0JEW2lROg591iVEuxA3M2wH4W5HfPhpbczrGvfZybZCA7yIFu0nWJFgRKdCLdg6A+6z0i5rkFyleA9bIY05jUW4YWHqdp0kypINh1EREEjusUo4TTdUZ+jUpG/cfusx26XkuliBFnFgd/JWVFPycbzecH8TZvOa9+/OynhR0tXyySYCpSfGRaKUXalanmUKk+dssbLIJm2Wpu5lS8QL5lrkYHXUTZWn+NQUpjmVD3fO9rO8Lm/SL+rEIUhA9GUvtAiE9Ynnr3D4eVq6C6au5b9bu686R6jrqKoQy+y611bjawsAhQIzJOvPu0tZXhQwj841z6nUSlhKMmUjQqk8tdvP7Luj1ehqrSMqRl0ymkCXkKQohOVKpHhp7Y+dQkgodCwSkFWbadAe/22O5GjIU+kSJJPdPM9wtsMtjMvr0AjSM3fvPz77XIJsqkFElpVmKOrJ7u1Aj5+SLaPahRSIJEkHU+z55eNjr7DByu9eg6DUEyoj2biAdbAraT6ocSBA9UwD5fHf8bIuOqVsNFJp3IBJQ6kzEAGPd7/ABstKUlStOZlI57DSLJeHWULYeQFBQDiBGsDfusrrplpKSEnTUcpGk+A52Dl9ZWEHqIE6HOAZBJH+d3WDUFoUHEFBUSSTv3b/XYwpGZtLQRtMRprqdvP6rBrCUqGZBEaiNdtv6se+0KmGqKqzlPWTOxPPSDpHtB+deHRnhACQlMDKPMc/Z9VjAZUpvtpXqoJlU7aTYvV1LdHTO1VUshptBWoxpAE/bt9dkNgNU8AnRQ/jNeTpCcJ24TmTfVGpRTuB6U18NfbHhb05aWlSRPtn67eWl6M1V+9IPhstbop3q6/KFPWuahn+6WwgRzjfQbydzb0vYo77SjtXuDm1B6n8bUVa7M+4VnSgsaQU41V766VVAnL1LmqtNZ05+6xZTUJUkwYGlkoUN8r7P53jn/FzpYYUF7BKovRQMRAQLRMJPJK8Ac0aYrnbqzOsZesUMozJmBZHL61KUtapUo6+e/2WLVd23x1suXu5kKjACBYk9d94EQb4fBHOB52LbtayEcdUq+kjfOd4gacrZ6TmTmSonSY3tBnHTHuIeGjlw1FHiGpQ1XuPtuo6tBz5Qkg6pn9I7G0YL6SWLgsZb2UtMgqJCfZACeevsFuLg02snNaSL3VvVVQzE5jAPybaVUCdCSOU7/G1R2ukdiEkhV8PAjT1Uyd9+zp52EHSJxFplvqphURCUT8BpNkz+CW19iraelCDv3RI091s9KSRJWZiNDGlqmHpCYg7IN+1x3TACZnQydPO3P7/l/k6X5XJEwRKQfq1snFA5JAw9VbYVYTJzbTr3aW36XJgFMjSItUr9/vEJ1VfVcDzBWNOXLyPusJT8bcR1BJVf1dBMlRd9X3b2Q1DRyThE481bH0kd+umotyqqIEkiPDQzaptVx0vkrIbvy8YEiQ+QZHPw/bYNHGu/HDm/Pd4kAgdp9Wuvn86WTvDei7hHqrb+lDLpJBmYtwqqMiAdOfstWq6uKtdVFJevW8jO/91OJAkaaTZwIxy5UIStN718EQT6WufHn8+2w769reSmZRudsVO/pJVome4W0qpIEAkctpFoXq8UVLVK04b1rRnTBIrFzJPnYqjETzrTrn54vXsjQemr79t7RfajLXsphhrzzU3mpOYJy6gnUGwNa+eqChzOsj58LQS7flZUgIaxJeDSshDZVVuqST3kBQ0mPgLA3Xft4F7K9f1dUKTqT6Q6E6A65Soka+P1WjdjMfuqRuFP8AeUwrqlLYaEGZUkztE87CGpUKR4AqgOaGPnvtE7N/1ZQVGvdOZaUAqql6c5/HwsWvrFL9K6tCLwfhSiRlqFwQO4H7fC0AxmIm2VTHCpAL3UwNVLqg6QFEFRiIkfPfbbL73oqQGlzBmBpPh8LQV+7Z5KwTW1ahBJmoURrtz8/dYwzjh1bcoW8oR63Wq1PlNnHF4x7KaMNf1UzNLecfCVpOVIBUY+HumwjtS4mtbUlCikiJI8TztA95Y4cbQULqHVE7lLx+wxG3usgrxs8FHrKp0g6hRfWSnv5/OlljxZjvZUb6BzeasvXVbpYkBRGh7Ovz8+0u5WuOhqlQkzlEkSCPhyk2rc5jmpEg1DoEAQajaee/n4a+diisdPFJWX1KjQ/TGY8ddOY93dYkVzTyUZoz1VnKuqU31KdQQYgAamd9/P42OGqc2CVTJ/Rkk8oEa8+VqnuY4eC0+KgSAuTzkROvL6/Iocfg5klTQ7WoLspzCNDl8xp4d1ntqwdgmGkNtSrY3jVK6tKOrUSCTlidPn5m0YdInFOOcKcKKzFmAk1ybyo1MtOpbV9Aqld7LhfaPYWhMhUqGhjkbQucfKebR6KqlKEuQpwKCs22m8TtAIkk6TZJwXxAXe3GLDdy35Rld01FRU00uLUEuVKqV7q18spQ4Oyc0g5FbgZTqNnepWsI0JUT7wC7XWUhcNOk3dVNV02BOId5dckOJo7tvutpU0iLwUhI7L7SVrS09IEQtSViDKVAotYWmum4qu7n7yuNNys1lQVLdqWFimdWqDqVogqXHOTy1i3n3xn4RVmGGvz4hqrv/C15P609SUJqrsqCtQWlVUoZQDOinAcxiVAkFTIqOKd93TgasvHh9jpeH03VWCiNy3nfSamrU3onO0y6ypRRtIS9CcphMa2fV4NFNmmpHAdWnfxVhTY7JRlsNW0noQrlcZMYYK4E3NX4/rKS7K3EVar0ejS86p+oq3xBS2palFXVggkgGIOmsWsVgi+7wxLgm579vm6E3PX3pd7FVU0CnMxpXFoCi2VHXSeYnaRbyIwRi6rvPiBRcQOLb95Y4uvDdfTuVCFV6ghbRchOQOpgNhzKVJypCgIkSTb0mwN0sej9ixLVI/eruGnXQerF9UyWGVHfR5BU0lPcVqTNgKeg7m3XUncpuI4ocRcCBZo2Ur1tXUU5S1Q1DyAtJQvq/wBME6zGnz7bFA64KcJiSARvt86/Dumxh7D1FedKl+62aaoYeb65C2yktqRHrBU5SmI1mNRZHaw/StqcbVRoSpCihSVIAIVrO/t9xtK4EIFpsq58V0M0fEW+erShYcU2tBJGVClNoJMeZ+dLNgNuo+mWwnKpcpBcgkcyNJjx2jS0h8ablRSY5SmnQ2yl672nP0UgZcyefgke/vNmQzlRORkun1UqX6qRqdvIWxdaSydw8VuqFvEp2O8FwwGUMZsqkuFMZddD3k79+n1RY+0y88BUVTSktoIShSnOyTH6IAkzPLYiwbNC46QFZoVoMx7I8/f8bKibpfdKWW1LqHNobT2QPAnU6xyje1c6S3NWbI/BExVUCFNu+huqQ0DqtwBAOkKygR3eEWo30i6Yt8X78fBCxUll/PyUVMokjwzTb0Quu4a+7orEU9K243Kw5UNodCSOQCgUknfafK1Lem1R1I4p0F71Csy6+6WitwNpQFLQ64kmAABoE2u+zdResLL7hUnaOmPcw/oQuehdT/nbHOJMKh3q27yuV2SVCBlcQATO8BZ21+sWqHDqqZvKjvOmrqZS6d5L5QloZM4ZQ2pI01kBe3MjXQ2qJ0QHeo6QF2XeQVC86arZEHn1JdB2ndA+Bt6T3XhVyocSOoKkp9YRAjQmTt77Cdq5zTVgN92g/sT+zdLHVUfpjYlR5hPCDdF6Q7e920ZrKqsVUdY02F9WgoSnKMwEGRMxz0I0s9LrwFh6rp3PSLqp0SkhdQ8wgpGsgJj1jueffG8vanw845LDTaEtKjsNeqozzO6vb3Hay7TYco6ZCTUKUtQ0DSNwPHlbEy4iTqCtWynihAbZR5TcMcL1ZhFz0riAqHHl0bYA0iZJn7fCylTcL8IUzwSvD91VCyg5Qu7GyMxMwBGo1PLntaSGbucXlSy2pISOyVQVDy5D3WWKHDrrckDKs76ypXmfvsCa6aTYlMkmgiF3AKO6DhXgkqU9eGELscWoyEqomgT3EgDSy5d/CvBbkBrBtyBS/WWu7GJHl2dB8bSlcuBKyrUFKYhvfOeyLHr0vXAGBABe94N1VZsmlY7SirkImxsGHVtQ3iyuyM6k28uZ+SoanGYc3Dhbmd0AUW1fR3wjim4rzuNODrrcF4UjtMSmibSBnQoTmjQidxbw54jcOr44bY0vPCN/tIaq7uqnaJ5CVhQS6hRSoTEbg+630Hfnjifjin6nCtzN4buxZKRUVQKXCn9ZKBr/AFfO3hd0trkvXCPSBxdh2/a30ytu++ahDj6yCXQXJS4ZO6kkK1J31t6H2SiNK50cZc5pG5FgbdL6rL4tK+obmmLcw5A3I+PJXY/Jx4wxVxM4I3rwlw1wyuBxu5LzR6XexZShS0PjOFOqOYqcBQoAx6oSI0k3a4b9Hd/h/UuYsvjEj1VeIQsCmYQGqVsK3ATurzJ5bWoz+RXxq3d978UcCuqTL7FBerPk2t1pf/xG7enN53z1ramZ0ULdiUNFS1r5JSS7Sw5BRQV9U6EQRaN5+KSBttBsTvVgv0LgAEohY9ljk687cmFDKdiDY+E5SCpni4ITPHfIM+NsEzI0BsJUtFp5xpRPZWRPtsHObmfsFtEw3F1TuFjZdJ1HMnx3+dbHKBXaKe8H4f8A/Viknlvy52GpFAPJAI7to0g/haZhsUM7UJSkzEbG2wqBMR4iweYnUb23mAMRYoKAi6EzTKd+VtyY0MzYIKMzHO3SVnvmzkqEBnxt2lQ5aCwIJMEHU+y3aVH36WeExChexkg2JUiymrrGFfouJWPJSR9xsbzctLEzKL3keq8x3c0q/wBqzXjYpRsh3dBJ94sPdLparmnR+gsK3sA5oND7rd3ef7oB7zZHC4XDdSoNRNssn094I9HazETkTPutlg06xXza1ldW1yUXreda7U1SkJdeeqHCtasqQBKiSTCRpPcOWlrE8EalNZSVF4NjR6nYVBEESCYj2xau1Q0BcqVIKM3oxKgTEjLz35nyk2mjo1VqqnCDruYlxtSaYnmCCSO7koWLoxaRQVRuxTYX3IELymBMzoe7577cqfABhIgKmZyifn53sVC1ZsqFdo6DWDr8/XbhSioFXJJCcuna89e+O61wqvKlBp7O0pIElKgoDLpEa+O8eVgQ4ZJToVaCPDmPwsUQtTTiuwCFdk6nXQfED3aWxRyuEkqVm1PMEETGvkPvs3ZPTpwpWZah6lKAS4AtMnsjKdfdmJ9lnGt5BIQRKkJgKneCk6++LRzdlYmgrWHgCQ2qSMu456cjG3jZ+JEJCULzJIEK/WHZg/VYOYWKNpzdtkYZeQlYBTlAgiQDyj7Rv32AcLagcqSd1eB0PdpzNu5SptJA23G0EkDQbbW0okqMqVCzt7ht7dtrQ3CJAXKNV5VHQkk6+J+NkbFwWbicQEDR5C4IGna090fVzssKTlykrlJgjQbafbP32SsS5FXO9mMJJQe4ntD4amzJD6BUsfrBQwqo6nj/AMLqk7NX9QyAT/5W1r48/mI9SWmzEwTAJkd9vKy8xPHDhrCyIvyhM5Y/+1Nn7/f46erbKFFIBItTSi9ka06lFlEtANopsyyZzz6u3426AVlUQdTsPKxlTQ5jfwtpxPYVHdz7rLGEyQonTO0TbsXowp1oDRKTBmyGtCc6lJRAKjlBGoHj42VHEHMqZ37vnxsWU3GgIjwsWBZCOdcqr/TOuervahwkzTurbisqc2QAkjq0jW1bEYBvlRSHKys0jQhJGnmO/wCdTa+XEfCLGLb2w9RVMgMuVDiSP5qdPaAbcUHA273HkPFglJOaQkA+AgCyinMhuk4/DFlRtvh/ea5HpVaCdeyUaf5s+PtseRgSuSkFL9adQEyEz7Oz86WvYvgPdHZyMqlMASmf2b2MtcCrtQZ9F94+yz+5lN70FRBOA7wgKz1hAJ9bLrvt2R9o3sMMB3kBIRWaEQSoajv9XTT6rX3a4H3OlWdNNB7xpYYcErkGoozAOmlmGhcnirHNUNoMAVz7qQ4mvA0HrxG22nh86WWLz4a1t3ummoqatW2EDVTvrbHu03td5rgvcqSFeiGec62Wn+EFyuHrnKRJlA3+Nh30D8yIjq2WXnW7gW9kdldNVmdD9JEjfXTyscoeHtct5KTS1W8CHDEfP2WvlU8I7mKlH0AHcbC2M8IrobMijA8I2Nn/AGa4jdR99aDsqtYS4PUFWEu1jN4RzHpSkme7b472kSl4J4VWUtm7K3UDMfzk+Z3j9OLWDufh1RUx7FOE6abWWv3F0uYq6kaxBCYtVT4TKXaOR0eJRjkq/PcIsH07TCBdjzsgqBVeNQSPI9Zp32DqeFmFGaJ1bVzViVohIT6fUZDOpJBXrv8AMWne9cE1DgSGWTGUpzIA9/z328z8cYZ/KNqxbfbWG2sYrulNc8KVSDShJRmUE5QUhRERB3587BnB5ToXW+KKGJR7gXVnFcMrrFQp1qlfaQluNa14wY31UbJQ4cUCnVqQ2oqAOqap2QANf0vq87VjZwB+UErnW/zjSY+cLB60Q/TIA0jSNiZUNOWnOynUYJ6Z71+XXUsXBj66GGnkekekX0HUOjOJKsx9XeQI3tC7BX2/nAimYq0ewrCU+BbvYyyyZaX2U9e72fH1rFMSYKYbeyhgrScxJLy5Oo1ImJ+eVpfprpUulV19OifVHn4WCxDc7SAtS2EICNASNdtT8LVow2QPuCj++sLNlAzWC0uuhKqELBBkdaswfOZst03D261MAm6wrLEErXy07/C0lM3G2XlFCUFaWpMDb32W6e52/RUFLQgAJJjn42ZPQynYpsdXH0VfL6wDTIOVq70pUNZVr8D9dmt+4pgupaXdzJzqgAtAwDP7fkWshfVyoVVjs6BJkH4mzZaw8isvgLQM6Ge1qoHcHl862mpqCW2pUMlTG52yjJPDG6iygvXRSqUondoa93LaLZ+9fdYIBuqmCkx2g2JMnx5/Mm03tXIHa5CVICUpScumk8iPOzd4q15wPge+8RslHXUVIvqzkg9coZGz7FEe7zscykkJtdI6eMNuQq9YkrOG+FqhdFUVN2msbUQthttClBUwrtbetAOs8otHd+cRsMN5nbsuikOUhJUmlSspGYzAPidJHLkDaD8TYgeRWO1DjzilJJT2lElQVz18VTHhtBsHT1IraUuOFUgaNkFWQj9WYJ2mOWh5wNLT4VGwAuJJWenxF7zZosnvfHEe83qloNrpXGm1LQttcoUBO4VBCQNOyY39ti1wYnrKbG1wX0FIZXR3rSltQPZKUujMDOskKGv8pPKbNFh+rpHSGnCARooySDIBAkbkxtPI84I7bSqZCns6kOFBWlaRkUidUEECYkIIOhB5aCLimY2CRrmjZV0krpBYleguMrwwNhC+7wo8U4puClpbyYbqKu7q6qZ64mCAtLAJWQoQCAkgwIOlq5X50euFeIma7HOF6UKuwKc6tu767PRuKQYVkDZJBJ1KEqSADoBpL0wrwxw7fmHqepYpKh1VbNRUvVCG0uuLUBmU4SoqcMySViTr3EWFpsA4iwAqobwG+1Rqvr6OsaQEMM9k/wAeGoLPWI0CSlCRC4JVAtpMS7LT1UxrovRDrEgKelxNgiEEozW2uoRuC4sLYHob5w3iXDt8u1mJKctPUd30SXEULSzlQ4pwq1UkQvIlJKiRBEyYVcrr1usP3WzWl0MOuNFRaUhcJMFQzJCxJkwqO6Nxa7d28N3LuccVeWdagoqdLjynFqUrdalqJJJyzqTtrGs1J4yYZcwxj++aRCXQzVOGta67tEhYzGT5pB5T42ExXCnU1M2QjQGyhEwc6wSrwv6QXFDhNW9dgzE9TR05JS9RfxlE8TuHKdwdWqRMEpzeMm1zuBPTww5xExJQYTx/dyLuvy+KoMU9ZSOZ6R99Z0CwvtNFStAZVJI2FvNWpXkYRCpIgqB25CAd+Ueyzi4Q9dUcV8HtJzLKr9oAEjcn0hA3tl5IwRcqZjyCvVTjjdVK9iG6q+pWmFUbjZb1BWpKzAmNPXVZgMXLnS31rrLDHqpyyZ9hEqiNe6fZaZuMl0vVKLqq26cLCHXW8y1wgJOU5j3+r4a2jqhudbqw2hHWqnRKJIGug5i3m2MvEdU4L03BG8SjYf43RShudkAJoW0rey5pWkqVyGukDfnPnZcu+6HaMejOBhxROYOKWopQkDciN9tOXd3q9DcdS21DlSUqPYLbU5Y7lGYJ99nNc+HF1CgtbS1o0JCXI+O2/wA7WzU9UG81oWQi103mcOtOrCVPpeUYR1ruYAA6ylIEJ+Jja1QvyiWDzdVRgbEDUraq2a6jLmQCS2ppWgPLtk+fdb0EpMOF9SEppwgJ2yDl4nfkLVj/AClmDUq4SYcxCpa3F3ffnUKWkaNpdYWYmO9pO3dzsX2bxAjFIxfQ3H0VV2hYx9A9nPdUh6MdX+bukhw/qHV9mqvRhiZj+NBaG3863stRXEktIbaBCAcxKlGATuI2m3inw4dXcvFLBeISlQF3XvRVKiAYKWn0rM92k299Lrwo7VFMoXl5SJ0tb9uYJKmoh4QuSCPr/wCVmcArxRU72uNtb/RNCluBCoKFqyaDsaEzy1Gg9tlq78LFyG220hPMzJNpAp8JUtIyH7weSy2galSgIFtJxBQMq9FwrdTl5PjTrQIbSf5x+y2eg7OuZZ1Y7IOm7j8ANVJPj0ktxAL+PIfNJV2YDXHWVCEtNjdazHtgW7dvrCtwuehXRSuX3XjZqnTmAPiRoLKqsK35iEhzFF6rQzM+iUxyIP8AOO5svXddFz3Ex1F30bbSRzA1NtPR4GIgHRMEY95+rvk3YfNUk+IZ/wCeeXnoNB8zufkmW9cHEPGfZva9EXBdqv8A7PSjM8pPirYfGyvhrhtgzByjU0F2ocrF+vVPnrHVd/aO3kIFl6pvJtOmcT4WRqy9ySYVaeWWgoTn/nJOrtfIbBQiWpqG8Nvot6DTz5n5pZqLxbaSUp0HIC3hH+U3uU3P0ucWVKUkN3k3RXggEb56ZsE/4yVe63tVWXoogkr0t5J/lbLmU3xkw1iVCB1d44fQypXMrZfdn/NWiz8Exp1ZiYjcdwbJ0tCYadz01/yW+Lk4a6VX5tLgSjENw11EBMAlIRUCO/8AiTb16q72E6L2t4QdELE7mE+k1w4vhCgkOXyxQrI/UqJYJ9zlvbWprZTJWe+1R27c+nrY3t2c38Crbs9AKmI33BUhsOB1lt4RC0hQt2T7LJ2HqgVNzUrpMnJlnyMfZZQInutoqOTiwMf1AUMzcjy08kg35TZakPcnBz8LJ20JmzgvljraMKAMoVOndZAMg6aey2ipXXYqmobZ6waahIE99hGVZFhShsR9ev1W4MnQHblNsVqDl1JBA9osYEG9KQWCIAtsHltOhNgULStCXBGozAW7Bnu85sUChl2DJ/G3QJjbQ2DzQJJ+NsMbREfCzwSVyGzazAk6W2FCwOYSNZjwsIlcHXlz1s4bppQoPxHfYrWuBqpo3FAwXC0rT9ZJ39oFh806AnexK+SU0Cn41YWh4f4KgT8Jsrtlw3R5YA1i3VAfpozDf4WDWQoBSTyt1QmX4kanvstrpE4euCdMw00tluOtjTP8+62WblS3XgKMMXnVPN4YpUNLrXVGlCS52Csdk6ncTaUuBmAcSYFu29LuxA0y2l+pS4wG3g5JCTJ02/R03s0KC9GrvxdT38+28UIrVVDgQQVaqJgbD7NbScni1cPV51UFcEnUyEEAa9yvnaywBrTmKhmzOGUBO/qnDyVCvPvnaLaQgZoWNJE7AmzWTxSuVJ+hoa9Q0J0RofHteFhHOKtzvLCV0dcVRvDZ567K8f2WN4zeqFML+icYQgJ1ECDoB4aeW1sIBUUkTJBBChqfD7/CzVHEy4lATd1fOXmESRyHrfftPK3H751zSFJoKztGDCEc+R18vnWzeM080nBd0TsyQSkCSBoInlt899nvcd4Lq7qYCnFFbX0aoka6fZHxtC3759ywSm7q4bAZQkpnu3ssXLxmua70OCouy8FpWsKBSEbg67kWime0jREwMLDqpkbX1Xa8NNPLX57rBqKey6VLOXnA8/sGnjaNFcecMoUpSLnvUhMn1UaaHX1u/wCe4Fzj5hxRKlXJeeh1P0Ykb75vGwlyjLtUoFZWCrsjImEiABpOnlt7rJWLFBy5XQGlJCi2kQmY7Y1PtIPs7rML9/rDqez+ZLxVKwAOxB2/laR7/rsh4g6QuGamhXSJuG9QpRCir6MpABG4zabD5BsyS5aU+NzQRdI1/Pob408OFoASlV+0QyqntTVNHUR7PefG3rAwnn4zbxzose3fjjjNgFygo6unbpb/AKGUv5cypqU9xOkJj329jWMuUqtXSggC6JjIcSQuyB42DcEpInxHKLCzmGp5d9i61GNRoREE2WPdMek9xJKjGvj9lgHEgjbflYw7lmBpvpG1gHCNSde/usUEId1GnFvifcHCJVx4mxGwHqR6peowQ6hspUpoqBlZA2R5xNm9d/Tw4OUSQHglyQIBvFjnruCZ77S1eXD/AAXxD6u7Mb4Vuq/qKlJqWqe8qRFQ0h0CAsJWCArKpQmNibCUnRr4CtdtvgzghuVaf7g02/tRYqJjyLtNlA97L2IUVO/lCODTaSUIpdTAzXogaTvog2DT+UW4PRATd+ukG9Fb66aMeFpyp+j9wRY9ThBgpMaSLhpf7GtlSn4LcH2QOp4VYRT5XFSj/UtIRLfV30TLx9FXRz8o/wAIGgQPzQI/XvV0eR0pjYH/AOkh4VrCi0i4ikCSTez8R3/8F9lrQtcLOGTf8Vw6wwnvIuinE/5ljrXD/AbOreCrhTH6t2sD/VshEp9r6JQYxyVUFflGeGyUheTD6Qdz+dajxjX0WOVnHw/6a7PFy/RhHh5hW674vUU66osNXi8kBpG6itbCUjU7TJtZZnB+E2lfR4YulOkdmiaH1CyLiPCGHKXEmHL7bVSXOqlcqmlPsttNZgtkjKSpMRIGneBZvDl3Lk4SsGgamMnHHHepb62n4M3epJWUgm+m+RIO4HMW6bxhx+nXhDdbYTvN7tn/AFxNpDuu8MF4fu9i5f3Y0jwZCil2svRtby1KUVEqUVCTKvqty/i/AFO4UP4wuNMb57zZGv8Aj2fY29ZMv/dTJpcXceQQpfDO60JnlVoXr/TCysnFXGt8AjA91I7yVIUAO7/hFlV3H/C9uEucQsMoGvrXuxMDxz+I99ij3FDg02tKlcTcIJgmQq+KYz/n+FoXRB27z9FI2Qj2UF+6PjYUIAwndKSORbQeX/8ANWxV78altFX5huNtYEjO2I+FSbAu8ZeCCQUucU8GJI3/AN1qckf53nYmvjlwBYUnPxXwek6b3owQf860fd2++fMJ/FPuhJCMX8YKnE95YXVR4fYqbup6arcLgCUONvl0JyHryTBZXOgjT2AVdDxdrVkv1eFygiCA/Ecu8xZvtcd+j5QcZb5vx7iPhr0Ouw3dzHpDTwcbU41U1ZyApBGYB2Y/Czke6U/RoZlw8T7k5SEtOqPwR8zaE0rHD0nnzU4qHt2YPJJowvxIASFV+GAQqQBWEQB/gmwF5YK4lV5BRe2GGhMkmtUqSef8V5e+yi70u+jQ12f3y6GYPqUdQTp5N2Lq6YvRpTlCeITbhV/xd3VRHf8A8XaH7Mp73Lj5qX7QntYNHkkVHD7ikhwk39hoKjL2XydO7+J+yyhRcPOKAbKDiS4WgdTDhUAO7Vi2numf0cG4H7s6he/qXVUn/UsTd6bHR0QsdXiS83AIEJup7XyJHhbjhtK7cnzXCvqW7AeS1evDTiEmobpV47uRL1Sha20JQFKXlEqj6LWAfZNq/XXevSjeSay7+B2KnW3B1iXkP3dleSRIUnONjoRoN7TreHTW6N1YE+m1F51gbJLaXLnKwlWxjMdNLTfhCppLzwpdd7UAKKWspGahlKk5CltaApEgbaECOVkGGQX9AnzXd/mPrD6KlaH+l6pYWngniphQVMqrrnHLxHLa0adIa+ukJQYBUxxNwLedw3TX1aGkP1dddzoW8kFxLcU4zj+LJnQdkDnb0vcbSrXMpUa76C1IPynt6ilwngnD7T6x6TVVtYvLuOrQ2gHf/lVWeyhjjcCLpXVsj2kFeal+V5qKl+j1UkHKQQRyjSJB2855jc6w9WoW31VQhBLWjna7Rjs690yO+NTFi150jz7/AKMX5fQlOUEElevrJOxnSI1IjyslXQt1m+VoqELQoIWrKtRTKt9dO48/eLWQ0QR1TzQIeSsKUCVQUkDrClXP26EToIUdxoccLDjCTTJSUoXnJCQSVQIyxpG5MRpJ1JskOvMimOaIU3lSlMZokGQSeYy8oGoPOFSnQh2n69tpJL2VYMQqVak6gTHZTAJ2i0gOqjIV0eBOIU3nwtuitutqnqrwpqRDLqnAJSpAymQk+t2Qd+7WBZYq7wvSovuiXedOwFvB5nKhACSMiiBoO5Jjvga8rRv0XKxdZgN1ph5Lz1FeD9P1mYg5VK6xAkRMhwGfPvtLN/1jN2UDKLwvBhdY6+hilXlz5lnsiRocqRJnmNbetUcxmoo3k8ggMoa8oq4xTUzLeSVvvqLhStSkjLmkb8tU8to7wBWTpY3O36fdWJkNKyPB6lXmlIJIC0ySY0g+Gm2lrTUIrXaVs1iaUuinCXXmimFLCf5qYG5Gg052hTpR3QK/ALlaic93ut1aFoBJACoUO/1Vnunv00DxenE9BIPC/lqpGutILKlt6UjrKAtQCm80CE9kad49nl77TN0HMDPY36SeEqYKSGrqeVfD5UqJbYSVQAN5WUaWiu86FVTSVBaUt0oGbQaQBoZ56c/Du1tLvQHv2nuPpQYVFW6W01yKqjRyBW4wuB7SB8LeSS3DSrWL1hderHEq7G6+6KbOhJDdUlXaBUBKSNuesabfGzWufCCXKdC0KDcEElWg56BI1Owss8Y+Itz8N+Hd748v2762tu+5ktPvtUiEl4pU6lAKQpQGhWCZI0B52rlQ/lJeAtMlKjg7GiTmjs0lMeX/AE41t5n2hwuvqqniUrCQQvRMExSlpaThzOAddWRpMJX36ZTM06qUXdMulaVZ55ZUjQ89fKz0obhQpKWmwQkGVGNJ8pi1T2/yofAZlQQ7hDGw5ECkppPn9P3WUqX8qn0a2QlKsI48UoakehUsD/T2zDuzWMSn0ojZFz9oICPReFb6iuBsqSEqMaScu9ob6d/DqtxL0Zr+Vd9IXnrsqKSvSECSlKHQlau4AIWok9wM2B4a/lIuiviliqqK6pxBcfoyM6G7xu9MvkbpT1K1wf52UGd7Rzx8/KgcLL6wJiPA+DcP36yq9buqKJupXSsqSc6Sk5pc7IIJEwSJ77W2FdnpqWRssl84OgAufyCoKnEZKk5fZ5kledOGBddzX/dVRfjTi6WnrAqoaakEpKgo5VHnAMbkZrfQLhDF37rsN3TfWDrrdFDedExVs1VUMv0biApJidTBHhb5yL2xZT3i7/c7K2ldYFyQIjWNB5n3m3qjwc/KzdHnBnCDBmEsT3JjD88XJcVFdtcaS7mVsqeYZS2tSFF4EpJTIkA67W31TRyyObISR1sLn5dFQPkYW5Wi/wA/4ur/AP7lW6t0VF/VrlcoahCtGwfBI0sqoFDQNBumZQ2kaAJEWog9+WL6MhScmH8en/3bT/8A7RZIqvyw3RwdnLhnH3+T6X/9osE5klLfulOS7qRc+ZUWV0lhI8W6X0V+6q90ISYVZGq76JB7W1qGP/lduju4ns4Yx57aCl//AGiyc/8Alaej67onDGOf/wDBpf8A9fbP1sONVJJ4TlYU8VKz1nhXrqr3kE57JNVe5J0UfZajbv5V3gEs/wDcvjffnR03/wCvsWc/KpcBFEkYWxtqOdJTf/r7Z+bAsbl/5LvoriGagZvIFdqpr1LBkmOdvPr8qzcZvDD+B8QNoBNM7X0ilA/rBlaR/mLMefdZyr/Km8BjthfGp/8A7Om0/wBPaEOlb01uDvHfh9S4auDD2Jma+lr01TblZTsIRk6taFCUuqMypPLkbHdn8DxaixKKolhIaNz8Qn11XQy0j42SC9lTrB99uYcxbcOImVQ7dV409Wk7QW3UrH1W9523FVTaFshSg8kKRl5g7W8AC41BylYVnlIgRHn37W9D7q/KV4Lo8H3LcdZd+J3K2muympqz0WmZpmC8hpKV9oPF1xOYKMlaJ7uVtT2ywWoxYQmnbci9/nZV3ZzEYqLiNlIF7br0uwUVouc0zxAcZdUkpzdoT2oPcddjZePxtVboJdInC3Ha78YU2HaKqo3LmfpH3mX6ZDX8eHACkpWor/idSoz8ItQSeXPa3UFNJSU7IZRZwFlLVSMmlc+M3BQb6EvNLaP6QIs1HQUrUgn1dDys7FExHdtZuXs0WaxRSNDChHfa9ojqQqyqGmZFiuTJ98WzNsdD7fssGVDfLtbJ7zNrQBVpN0dplHqUgE9mUewGLCzzjU6RYnTrBzpCiQFbc4KQfrmxgKMetYluoQ50KFJO2sx3WwKB1+u3AVykey2woTsbSDRIhAqYB18OdukEiAk7fXYEkmADGlukqA2Ou+1nBMQwV36Dv2tzUJD9O4yoAhaCn3i3IUNR3ja3RVzs5cuaR3raNlRkkoE+ca2M0J+nG8DlZPu+Ay5TwPonVpA8Jn7bH6IgO6ybIEnNOEAwNVjT9U2ywE/y49o++2WTVKvBLFj1RdVzVFVSOBL7CUqCikESVBJEDQTJ8JnY7POl4L8R32EKTjO6klSAsRSHTNy9X6vHaTZp4+addw1WNpRJWWyB/hp28fv8bXP4bcLsVcRlmlwpc79c5TMhx1LSUhKExuSSAJ1jXviywMY+5eoKh747ZFWtHBHiQFA/uzurLvpSaAx5d3P77HGeA/ERzM2cZXYVAf8AkR393zNrEV2Ha+5L0cuq9adTNTTuFp5DqcqkqG8jv8xafuDfRybx/cy76crxTtJGROZObMSNhHIafbad8ELBmOyGbUSuNl56K4F8SlJDCMYUDjmTVKKBRM6aDWffYKr4JcSKU9XU4wo2luIlAVdpB8xMaGI9nuufxc4Yv8Pr6eu9VRJbIHWJnbeNNe732iCvK3VnrFLcWpOUZjKgOQOs/XvZW08btQu7xINCoEVwZx+oJCcfUMZcul3jXed/Pa3Q4M46dKWnOIdCkoUIm7Ekaa/rbb/Mi01k6yUgSI01nTl77AqEJAQkyoaiY8dbP7rGu7y++qjZPRl4mPfRq4m3ees3IuwSR79NPs8bdtdGPiArMt3inSQdyLpkSRvJV3Tr8bT/AHBUem3S091pK2oaUDuSAI94srNNISoBQBEdqCdSNPbvYF7A02VrGA5oKraeizjNSpVxcpwZJITcw0/zxpt8iybffRUxNS0aqt7is26ZSlSfzKEmCrl9JO5+NrRqTmSoOKA2kEbzI2myTi4k3G++spGVbUhJBn6RIMeOm9oXizSQVO1rSbWVJcH4aq8J8drjuCpvX05V34guuHUNBorCnG3CCkT3gbnbyt7a0+rYJBI8997eNdW5k6TNG2lQM4hulRKdI1ZG47iQLeytOnKyE67/AG2r5iTYp8YAcQF0QNYUZ2sXdG/htY2Y/VOvPaxZ4kiJ52bHulk1Se+O8+diiwPkcrHnUhWqhJmxRwDMZJ1POxrUE7dIOI8ft8OqenvZ66Xa70p8UoQ24lBSShS8xJ5dj42Rf4ULQVlODKg5VHT05I5x+pZK47qSMNXYoKyReiRPPVh60LlYQoykEdxM/PP5FrKnF2KvqHEO0Vgk9KrIYTgN3T9a8gDPsbt0rpaOJR2MBg+CrzgbcoatXgugAwqMog98/fr9Vt55UQTI2OkiPdHd7rTFgQ/FcFYQdLS8TqMA06Z2m8lfX1VgldLa+1E5MD3eAkwSbxWY/wBHaACoyUGCQYAAGo9vs8baLmXMUa6ATI3+frsgjCXiuKn7+FniQj6LBl25idvTHD/q62jPjrxRxDx4uK6uHF43XQ3VRXheIW7UUzq1vJ6tpawAFaaqTB8LM5DqjCgkASIkCAI3tpt4C/bjWFjsVsQQebTg+34+yyujBFiuEjgbqPLz6N9yXS+lly/rwd6z1VBLYA5aSnuAsE30fsMEBJve80iDoFNa6bepvaasbAB+mKz66TlPeJkRZtOpdISmmUU666wJ0tC6niBsApW1EpF7qO0cAMGpB6y8L0WkqjtPoBJMfydP2+0/TdH3ALiVFT16KMkqPpQAAHkNvnezqV6QjRRPYgmJ5x+FlO7G6lai6HCWUkyASeXd+FudBGBslbPITumez0euHydALygnUelxziNteYiwp6P3DsAlVJWEbkGrUeXhZ9tviR2QAdJ21E2MB4SUqObWD3+H7LR8KPophLJ1UVjgzgSlxm5Seg1i6Q3Eip6hdWspLoqVtlW4PqkaTG1gMR8L8BUNAamiuQpKFbekunMO71uWnjzs/wCpj92NOAlQL2HqpGpOvV1jB07tHPiLEsQNpfuypSURlTOvhr7NvqtzYmFp0SOlfcaqvtRdmHaHGKaGroUoojSrdyqdWAkieeaZ0FkddDTV9Pedddbn9z0GdSVZjmWoRAEnRIAsr8QQly87vuxlSC5eNQhkiTPVpMq+sCLJ10oSxTYwoSHPo1LJTtKch2jXw9/haDhgmynDiBe64vxNExhC7r2p6cJdW42HFEkyDvOvd77KGLrtpGLppqm7KVKCH23F5TuiO1JB2AGvt7xZv19QHeEFI5lGdCWiJVlg9YNZ5acvEWGxxfjQwlS0/WnrH2mllKFDNlSBOnL8dLKWNAXB5KXqGhpL0r6S8nKBLVAqpQhlIJh1J5lPcTrz2tIDHSJ42XVj2vwXScSb4Yui76dLVJTMKQlLKUkIACgkGABET+jZnvVrbVy0K2o06lQUkDKOwY1589tfgLEaK76e/eMGLirrEu0D6g2G3BJSpSpJ1gjQH2ecjl7Y9VI0F5sE+MZdIHjhQ1tyCk4uYkZZqKrqn0prSMwzJBG08ztraNePHEHFONa6hpMSYnvO+F3e0tTfptWt/qs6xmCcxIAMD3Dvs/q/Blx3iimqaupeilWFNEOQExuIGn6vw1i0NcY6OluLEtYy04SymjZqHAtQUpS1EgAaidDEd2XezYqhkj7Ap7oXxi7go1vVm5mWBWXk66HB2gExLnraCdd517oM96FdT5qq5dQoONsobIQkyQkFWm+4B+r22FTdtffT5eecDTKlAJRnkAk+7YAnxy6W4oHPR7/caWtKyGQk5SDljTQnTxEbad0WMuoQEqVpCEqdW42jNBGTlzPPXcySdQTPraKFwrpfQH2VtomkcIWQU5SlQBSJJgwQoTofEmBZJqH86MynBlJymZ0gADUmDMbD8SHc9e1S3uGgpOV5JbUrIB1bk6axI1HLWFd+tlBXEXVk+jDfjFMu+LteeU0qprUPBS15hlUynQkntZssjXc6ACbTFiKLxvVlulCstG80tISgkE5xGQREAjceQgzaBejhX0NLel/MO0zb7rTDLrK225SFlSpjXszlSdBsIOo1sG1UUdw3KmvvBBNRUuIJQuBCyoCJmREyZ2CT3W9GwJxnomsJ0CFlGR2ZKb718rcCKZBbaZzyAqCuSkFU77RvEZhvOkf8a6OtqsD1lDWU6Eu1VK6olQURI2O3MnQQIEbb2lajcvahApLwUkMtuLPWoPqzIiAJH6c6nUjaSLQfx0qsUYyVWXXc1Suip6ZKklaJCgI1AiAJJMq1gd1rau/R0ztySLWUAsXKoFI6s04qFZlFsqCgsghMAaEnlJnnv3kQ+OjXcldXdIjA1DcyVB9m/wCmqEqST2mEK6xRnTZCVTtItHtCFULtRRrIDrK1IUQNQUyCdPM792nhNnQ5qLza6TmCn7vofS+sedZfIJORvqF5l+BSmTr3RM28enGUFWsepAXqLxG4dM8VcC33w9erTRIv+kNH6SGg71JKgQsIJAVBAME91oJw3+SDuh8pdf4qVrrR1KlXO2lIEzElw/CbWww/VmjvelebTT5kOCOvMNEzAzdw135bxaWl4Wv/ABArrMWYlSik/wDIbulttQ/lLPaV8BbH4k+pbK0QuNugsB8STt8ldRPY1vpWHibk/IKheOPyZvAulwyvDtw44v68sW50paNFTMuIAntBSD6uk69YNTtysh4S/Iws3igVmLeLtZd6XIPo7F2NrcAG0qK4GnITb0su2iw9hymFLdFExToT+okSfEnnbHr8SAe2LCR1/ciXVE1/7rdvM6/gmySGYZYmfM7+Q0H1VBnfyNfDNlE0nGrFLTkAZlUNOoaco0s2cR/kcsGUl0VlZTccL9dqGKdxbTarqZCVqSklIML79Pbb0Nqb8PJXlZHr73C21IUuQQQdbVdX2pyG8RspYaKaSwdsvnExHh9zD7/UrcLucrSTlywoftn22QSSd9bSjxyup67MbX3dbjBb/N17VtLJ0nK4R/qz7R7IuNvRKOUzQtedyFV1LBHIWhYCR3621qTM2cGF8AY2xutxrBmFL2v51gZnmrto3KlxpMgBS0tglIJMAkQSDG1nOz0b+P8AUAlngnjpQSCpRGH6qABuScmg8bSunjYbOcAfiFG2F7xcNJCjePH4WyPmLPlPA3jMq7zeo4UYv9CBCfSPzJU9XmOgGbJFjd2dHjjvfQKrp4O4yqkjXM3clQRHnkshqYW6l48wnd2lPsHyKjufmbam0qfwVekh/wCo7G3+RXx/q20rosdI1O/BDGo/9zP/ANmyd9pv7RvmEndKg+wfIqLJtkm0pp6LPSMUP+8pjEaTrdLw+tNkHGnBfilw6pKWux3ga97hp65a26ZyuplNB1aU5ilM7mLKyrp3uyteCfiEjqWdjczmED4FM4lHVpIBKjIUSNB5Wtd0W+hhhrpCcN6jHFfxBrroqKW9XrtXSM0KHU9hppYVmKwdQ7tHK1TgJMDXS1+fyaOLUt4dxvhR1xI6iro7waSTqc6FoWf9G377V2PTT01E6WnNnC34qywKGGprGxTi4N1aHoR9GG6ejfjXELl0Y4rL5axBdzbTjD9IlpKVtOZkqBSoyYWsRHO1xFAAd1oF4b3x/v7u9sr7LpcaJ821aeOoFp6WQTbN4ZVzVkZknN3XWgxKkio5QyEWbZcnbT32S7+p1FtDyf0TlPlZTInSffYOsSHqRaCNYkWuqd2VwKqJRmaQmkCCNfdyFuhqAEnXa3S28q8pG/u31FuCCOZPna6BuqddsKSmoVpotsH3E/2rGgRqSN97EUmKlpc6qCka+In/AFbGgokyNxroLER6qF267WsBClRMAxA18rETfjYb6xuiqz2gkJydqT4T4fMi3dQ3WLUg0tSlCdMySkEqPmdtJ1twwxegMvXmFgH9FoCR7fbYgW5pi1+fRsLurTOmjfn4+/xsOm9FEqCKN9YSkKSYgLBjaeeuxiwKaW8eqUDeR60klKg2CAnSBHM6b+NsFLeJKSb0UlMDMAgSogDbu7+7Xys8ZUzUJQpKpVQz1qqdbJkpyrGog7/bYxm55vOyQijvI6qvlRBER1YAmInvmdbKaSNATqBr4myHwShcMDq6upSZHWJS4J9oP1CyhRT1skbambJqlBFewT/fErR3bQfvsp0MB3UDXabIkO6WCXJ/T/xrZbvKv9Y+4/dbLcuuvCLHRS3c7rgCZW83lSBHaK+fdvyNrt8HeNmK+Ei3K3DKqUpq2UN1DNQ1nbcA2JgggieR5+NqO4/VF2ZSmSaltJlI2zbeXmDzjeDbfA9I3XX5dt1PvqbpqyoZo6hYIkIUsAmCPHbvFm09g1xdshqt2QB3S6cGIsbXrjDE9dia/HW111c91r2VOVA7oA5AD4a2lrhP0jL5wBd7l00yaeop3CDDomFRuNRaEcdsu3Xiy9boqKoVC7qq3qBCgDORlwpHkNPjZIFYtpeqj2d9zPfY8BssY6IFr/aCkTiTxHrseXk9ed5VKXHXlE5k6Ae60dVKivMorCtDm/SzCdffrvboVC3QU5gFGSYOoG0fX8bcrpncklIMQPPlqLOaA3QLjqiK/oyoLWkpOpMzOvx5+dgXEBIX2kyQc0+HMz9dh3mVMhQzgFJ0EkGY132/GxVfUkAx6yssETAB7t9p77PTrJdwdWlq8HKJwZkVAKgZMZxqPfMe2z5FOVBa2weyFEQJA0keetopaeWxUJfbcKVpWFjzBHf5fC0rXZUoraFmub9R1tKiPVAJ3HvtW1rcpzBWlC7MC1a6hRILghMQCDHlHusj4kZUbmqesAUlKm1GdiM4PPx91nI6krSY7ITJ1O2ug+e7lZFxKUt3FWlSFHK3mAH84T7tfbave/0CrNjRmCp3eXVfwlKN1CSCMQXYkaSNFM7RodZ79O63soz6ok6zbxqvZC2OknRrebQla78uonKQCQepM6f42n1Tb2ZYbKhvMHTwsG/YFK0We74rFAbkAeNi7gEnLp36WOKbSACRBPfpNi7ojYDtWWNNksiD6f0jYmsayTvtrZQeSCNeXeLFXRpB58vn22MYgnbqLePCJwrRKRui8myD/wBU6OXnaC1JURKnCJ5xNrA8a6U1GEGxI7Nc0rUfyVjn52gv0U5DkgwY07p+Rayp3DKq+oF3IkUOJ9Z0kgyQOeh0tkJCSVq2JUDHwsaFEVLyhXKQY2HMg+/T9thDdxzBYywqO7Q++05eEOGEpOWnUHMe+QN7byqIzKUdeUyPOx70FxRyhRWADOknTw9ospDBWKXNUYYvQhQMKFG5tHl3bWTiAbpC3KEgAAKBDgkeOh1GkR36W3li9LoJUrS8GhrH6Upj42cLeC8TEwrD94ICRHaZUkbabx42Bq8O1l31N0mvoXKcu3hTZAvQkB1AVHfvZZA9rcxGiSJzHuyg6pTxmw285TOZlEBC4gwOR1/ZZAfpKJhsmofLYXMyqCVDl8/faQscUVJToo8w1yr3O57POZ2+dbMOqoqOuUGnNUoMwDGsRaDNmddEhmVtkj57s60hNQjsmCUrMak924+2yzdj11KYLblTL7iuy2CQCfmLJ6rjoErTKtN5Ku4WV7pui6FINVCs7ZzIlU90fPnZ0huEjAbof83NKSXCFA8xJFu03bSK7JSZIBIzZR46WMNuNIQAVAaTEc95ttboCic5CQDoTrYY5kQC0JFau2me4i4WpH5DFVTXjSrykg5S5SqIBPOATaxR6NXDutp3qd6pvmFgoVneQAeR2RNq9LqUMY5whVp06urr25UDzp0r/wDli16qcoXlWlTsqkL7IM+U/ZZWE3IUga06qvw6FvAavr2K+qu+9Kusp3JZcVXOpDfeeyneLV1VwNwRc3TVvDhK7dq3MMXzR01R1HpC86wqncJlchU9Yjke7vt6FpLKKxFO264CpQMEgc9rU44vu09z/lEcEValhBvO5aJkAfpK619ESOcd/wBmnP0F062qkVXQ06OVPdouNjA5XQo0Uy5eNStB1mAA4OYsv0XQ76NtRTobf4X3W8lXZX17j7q4GwSSuQO9MwdO60lIpkpW4DT9aMxkqQVHfvspUa0MVqGmktobSrcZQPrsuUJ5XlDxOue6rixdiTD1ztFLV0XnU0bTMaJbafUhCYmRCRue/wB7Yu6vbuPi/iu8jop+haqCDPaKmWzv/hA89TaUOkR1KeP+M7oDIU4L0fcK0NlOYrIXoPJe/wB9oax5VfmriS6/DSW665qAE5tFwy2AD7Uc+/xtQTvzZmouFoYQ5Pl/E7j7Xo6aYpmCoJB8dwd9T8ybV+4+YiL+IlU9PV/TLYZFQEnTshWUeEZvs5Wmlh0JTlQolJIkxvz5eZ5c7V+x3W3FdeILwrK1hdReT9Q4soWkKUe0YUOSQYEaTAGm9mYbFaQuRFbJdgCatDdSlNfnC+axTdK2kOZUuQrWdBG3zAM2J3ERW3o84yOrNQsJQnVRCSoad87a/fYle993leSurqFdW1OYNDRPdPjt90WPYWYZX11TVEpYbGVRyk77bAzoDobXw1KqksVyUOiG15iqVKAVJEqVB3BIjXWJnxEAUNMyh1Km1r6yJXASoBO55kcyI592tuqqoVWurbpj9GowlIQVApGkgb+0d8+Stc92udUh58r6xaczZOVR/R7++QTrPlINnBJeylTo3P0zWM7zp1o6s1FA062S4lSVKS4QVE7HVWg9aO+dLE1SaS+r1Zo30L9EpkqCigE5tBOx89DoTzMCKncK3E3di7q6ddOg1NKSjrAIIKkEFMfpQDEDvPZta/AOWsAdqwG6ZgnPmglRAMgSdUmSI13G+9vQeyhzw5PFB1Rsn/8AnKlvC7qZ6vQhvM22pwkTBiCoHLI5eMAd2ka4gvuleL1HT0iXnHFrAVlMDxOmhjvEyLLbV235exXU05UxQ1yG+pZQ2UqaQATIIhRJEaE6SfIo+Oqm4OGOG3LyvAdZVuJ6mlpwUla15QOyDEaAdoxAGu2l5UzSOuSLNHNMEbcu+qpFxHutvDHEO86Bt3QPCoX2jCFLSFlPsJPw3i1qPyd3D/CuLOId441r/SfT8IUjTt3ttuFDa1v9YhTi41VlSmAj1RnJM2pvi696+98U3hX3opHpD7pK8iRlkcgee0Ttpppa2H5NTGDN3cY7ywvU1CUpvu43ENIOmZ5haXAB45S6fYfK3kdeQ57y3a5VlT6EXXpWtJgoCimRGvI/Pf3eFpOu3F4rLqo6lDp+mYQ5r4pBtGhGQAiQZBB8dNPGLE8M30tF3+hLd1pqmppwNuyh5aU/BIt5f20dJDBHJGedlrcDpW1j3Mdy1UtG/c3989s2LO3zAgL5d9mexeSyJKoHfbtNY85JSCoW8xkrJSN1phhbGbpxOXqpW6rFHq1S8xKrN2/sSXDhSjFfiu/KK6acgkLqXggqjklPrKPgAbRDX9LnAdZfQwvw+oXr7vJxDhbeqppqbMhClRsVkkJMAhM2kgpKus1Y026pCyKLQbrzv6Yt0fmji7jekEjq78eqkjLt1xzjXfZwWrKsdqDv399rFdJfGN5Y+xteeMb6o6Wnfvxpmp6mnRlSMqQgCDKgYQeZk5vC1eH05XlonZR3Nve8DD20TGv3ACwOKW7w4hWS6CnGK9uEfEe+nLsulq8F3zcy6YIWCcqkOoWFRmE6BQ1nfa3qbw8xXjjiVgq578rXbvpGq9txLqC0HFMOtOFCoTAQc2ihKSRmiTpbxm6OrryeLlyUzD3Vrqi9T5okdppfLnrHtt6udHivvC5LtfuWovR2opELNQ0haRlbWsALyQJAOROkxp4m2Y7X0jc3HaPS0utB2dvLGGgXsTupuZwhc7b6a29nH70qUxlcrHOsCP5qfVT7BZVXVMsJDbCEoSBACRAsjuXpKcxXt8LEKi80+tnJ8udsCXPcVqmUhdq9Kz94ZpKSNediT1f6wzfo91kGovhCSASe12T5/P12JKvtBcCAVTmg8jrZzYnlFsgY1LD1eqTqSR3WqN+UWo/ztweua8EpUtV3YhpyqNIQ408g7eJTaxdTfQjVBPgBPvtBXS8R+fuAuJ2QCXKJNPXIg6J6l9tZ8jlzWu8FDoK2J/iPqhMWjbJRSNHReVbYAeSlQntAGbWa6AN/quzi3el0qcIRelyuoyj9JaHW1D4Z7VrvNvqbwqG+XWKIiNiZG2m0WlPoq32bj494bqQrKmodfpiI0PWMrSke8pt6niUfGo5G9QV5fhkhhrI3eK9TsNXr6Biu6KxTkBuuYKjI0SXADv4H7LW1VIGxHttQ5y/XGlpWg9tMKSpO+ms/I+q16Lprk3tc9HXt6oq6Zp9OnJSQofA2xeENLA5pWzxghxa4IXUzvFuVDTbzmwhHZ2jusFqSDrvra9aqJyb1Yz1L7jZToDIsWiCRvERZUv1spcQ4BIKYskZxESTJm1zCczAVUTNyuKxYCMigZ+kQAZ7zH1GxpQO+sxYm8vNTuBO4SSI5ECR8bDlQV2hsrUSe+xTEK5dxoCQPdbBO8HW3BIUZBiB3bWwk6xaZpTUICAdZ8JtuZGxjxsHME2yVAdokdx7rOG6RDhRnui3STpANgJ0JNugQNu/us8JqDrHS31D5/vbyc22gOh+uy1RJIdCp562Ra+V0jqB62Ux4EWVbtcDwadTssBXnIsnNIUvjqoErVP8AOtlg5XyHwNssqavCPFLK3vQaZMKDl40yQo7jM4nu8z9VrSUjrN13rTPMVC3BSPodzBITISZBHutWK+UJcve5GwEfS31SA8pBdHLY8t++1kApjrM3WKJTKpiZA+v55WSlAc1wKhrG5gGlOniRe124gx7f9/YfcW7Q3hWLqmlFBSe2M6gQdZCiv3E2beYoRmghMzB8vvsNTPUzKhmSgpKh2gnUx8j67AOKAfUGBIkFB0EJ7tfb7/eZGOG0MHJANjytsE48OUVG/XNCrcSlskZsx0I+vlt+NrD8e+GXCfCGBbnvDCV8Nv3lUhJ6lL6Vl1BQSXYHqwQB5ExsbVYZrw2orQrKoEdrWRtE8/CxuovesqEIQpwkbAE+3v0n7PejmFzgb7KRpABBCI1hSVqQiPZOonwHzNiS0qmSDB2Ow0P42MuPFeUpSnvnvO3L55WBdhSlGARGxGo5R87+y0yRFnCvP2VRymRp4+Nn9gK8euoHaB1RPUOZkJBggK7te/67MVRSoHNEJ2AV8/JsrYVvFFBfLanVhLLoLLhM6BWx9hA91oKlnEYUTSvySC6k1SVuDMYAOkA6SPneyFit9r8wV6nWyqGVkhKTpGn42dDbRK+0iTOoGwIP1fXp32CvbDN2XrdNeqrumorCllbjmSrWwFIy6pMEaeI1tn5ZAxputLHCXOFlSPFLbbPSPoRmBIve6SrcRqwdj9XgbezFLq2CQPOd7eQHFd27x0oqMXTc7d1Mi87oApkvOvAGWsxKnFFRVA8tQLewNKn6IDz13tBmzMafBREFsjh4rtSYOUn4WLPIMdmQT3mxyNI1MWAeSSJMCednxbqORJjzRSMwHibElZwZ1I3+qyk6lRBEHT5+6xRbfdpY1myCduo84xj/AHmvEJBUmoaUATp60ae+0ErcUASCCoRyIki1kse4er8S3A7c93ZOveWgt51ZR2VAnXloLRy1wJxw9P0l3JSdAC+owPKPO08MgAsVE6nkk1aLqNWHXx1kZeyB2Sgz629sOdQC1wcv8kH5+60pMdHzGc9qtuxJI/41Z2/wbGEdHTFpzBd73agEiYU4Z/zdbT8RvVJ3Sb3VEzailSVJUBE9oJEabe42sXT0eK7wxQmloKaaJdIX4FIsoIyjtF+SkOZz6kerrZpN9G2/ySXb/oUzrAaUY+rwtYy7ml0N30VI4VLU1TNtkpUQJSkAm0T421Dw6+yjkpXhtpBZQjeeF8aB8tLpqhLcyT1MyCToDBB9/OzD4pUlXSKw67eFE6w56fol1KgSA9T9421NrbNuZ9UtLI785Fo348YNocSUV2XhVvVDSrqS6+2EEQVBTaoVM80WuKjETNTd3Lfmq2jwUQVPGa6/goR4ppSy3SLCc8Z5EEkEiY+FozVV6lKURqJ3G/z3WsljPh9dl8IpxV1NSgJ7RyKAnQ6beJs00cDsIdh1y+KloqlQS7UpSRuJjLPI2rY8jG3kIHxVjNE9uqhlxSVJDgJka66iZED424RVOpQEkwBrMa/Ov12mip4NYaQtlbNepxnP9OV1CjKYjSE77WOXXwVwzedR6HR0rr6z2gevcQInXUgAifPU2sGUjpGcQer1QYkaXZA4XUGGof7SipJM6iND4a20Kx8DI4sZdYJMQPP3WsYejbRqkfmlpAMAZq9R9sTYZro4XahGU3NSLM6zWrMa772YIWbFwU3Bf1VZquoIvbCzqlCWr5WnXucoakajl6tr30FVej7KXaZmmAKEFJWpWoKQZiPH4WrBxDwRceG7/wAJ01NQtMlGJ6Zt9KVkgy282d/58e20+XbUcQahNGu7aG5VUCmkBKn33AsN5fWyp0J0GlgJGZZCAUa2Mhguleoexe7VBafQUoBlJJdOk+A87VD6WNS7cXTL4IX25RoD9YigpXXBJTlTXKzR/j6z4d9rW1h4mIW23SLw602lJWo9Q+dJ10I1PhOtof464WTfeLsIYnxvdty195Xa7/uLWMsvNmnUHUKJKFHecp1nnaCRgYwku81LBEXOsNVPFQ71VS+3lEFWxAM8+fnY1drDLk1VTkSgLgHKJUfYLNF+4OKThQtGL7kSs6uA3eFEnTUSoaWOUtDxIbYVS1uOLuWlbZbbSxdSAttxWyxDkGDuDpqdNrNdUOadGE+X5qUwi3rBUH6XNGml6TGJ6pFC6lD6adzrFJAzBVOiQJ5Ei0C8YqepN8XDUUbai0bmZc7SohSHHUTm7yBAj9noBxt4fsKxBUV2IqGivW9OpaK65dA2lxYDfZBGu0Rv3Wruu4qSuuO7XH7upKhPUOML9IbBWcrzhgFQmJVp+Ns8ZbzvB6qzbRnI0g7hQnfd4VF1YbrLzoinr2mCpmNUhZAAPjEzpv8AC1fagX8utevK962lKluFbrjjKFrI5kqMHw5HlpoLXa4x4Zab4V3klmhp6ZQp2Hi802CptsLQpR0EjsyPDXuNqf31cFTVVDFK5TuejtHMQG/4xW4Ea6dw21jmJtsOa3KSN0FXtLHAFMaoVQXhUuu012KKQMxWVkAgbmPYdPCx+4GkOUK0KYJUgEkAEwQYkg+Mn32Ub5coLvKWGqUJedhCG4CConnBHZ1A1JB1jloh4erEh4JdmCpUhOpEkE67k6D3c9rWFrFV/JOOmofSXeseAQ2iQEdoDTXUHSdfD3xZZWAolWmUJChumQUwD3kCCNCZ8NimMVjhWgdalskArAVCtSIMjbXYQJhQk7WPICSlDJlLRAyDXKTyywN4SdtCABsBaQWTTdOPhtcN03vi6ufvyqQwwww2tpSX8is5WoEcydExrIgj2WiuDGGEaNIoqY+lsEJC8xIbUjUnUmTJI33ggTakt/3ku77zoQm8KunS82slynSHHQkwIAWRoSmd49bckmxtrElataEnE+IHQ3leV9I0xk7UfoE6a/rJjXXXS9w3tAcKi4bGXPVRuphMbuKv7deOqBVE63QMNrfYQ4lqnSR1iu0Q233+A387QpjjCV43g85i3ijjK4sPtOH6JmqDrhbRvkDYAlUK8D56RHN1X9T4iosSXbd103dd9fRYfdvS66ylrbzW+qoadbMJ62qeaUcil6lCdRoQQJhO86LFF6166m/WrwrC+Cyh95RV9IUFSIXsZImOYnzs5/bIVt2Pj28dEU/D+C0Fp3Tu4mYd4evM3he2GL+vKtrqZSCFOUSaancQNSpKCpS4iYJAnTTWyPwL4ifvXcV8MY3KVFu6rzYdfWCqfRpKHwE7HM2tW4nQeMtKvw5iK52FGpYcbQYzpQrMOe5Gnfz7/GyS0sJOsWoaqdtW7OGgfBMaCxe/TTrNShNVSvBxt9IcaWlRIWkjQjwIP4WQLsu6qF5XqoHJTtVyldYo5UAKQhwyToNVnc2ijoScSVcSejth52rqA9XYezXBWKzEmWMvVE+bC2Z7yDaW63AzuJb2VU1l9VDVAgp/uZtRHbCRJk7e7nbz/tZTtlo7PNrFbHs3MY6klvMIa8MdYQw+EtP1ble+jQM0iZEx+srSPETaPuKXEnikcHVl94TupzD1FQgKcfDeZxaFKCR21agSoHsgHS0wXFg3CeHwDRXWyXRr1jic6p8ztbeLfzfely1d011K0/TVLRbcZWkKSoEbQfG3nMQp6dwyMv8AFa9wdMbFedV9PX3i1x16+b0rbzrXTmK3VqUTr46++fG3WGMFXtdF5099pIaNEoLAEiEq0PPkDaXrzuCiue8nqempENJbcKQUojY6e7u/CyVXU6hdlU0CEq9HcAEazlJ0/bbbQBuT0RoqmRmV2qq50mcF/uPva7aRt1Tyay7C62Vd+dyR7CfjatlXlNQsp2nTtZvja5/S0p0XjhzA1+lJWp2ldZcUJ5dWQCeWqjamlc2UukRGkHzGn2W12DPzU4ushjLAyoNk5uEF5fmnilhSvzZQ3e9KlRAmUqcCVCPIm3rTgGo9CUspBBUBmjbcfj7reN921rtBeNLWNLKV0zyHUEcilQI+NvXvDFe260w/TlWVxvOBO4IBFq3tLHna3xVx2WksXjpZS0m/D1IVGhhMTFgKh+tWx6UGHVMgAlRBj37WZ1XVXy9dr7lw9S5XJQVtNujMhZEkJInmB5T7rI9433jigvFxVLUNVVH/AH+rqCy0pDYWlT3VJzwtHU+qSP76CfVi2F7qL2C3PEsE76q8gmSCJAgmNx7bEfzpkWFIOoVmgHn4/jaIf3S8Q6qncR1zTRYdU3UVDj1MG6dsPGVOESUOBGSQREKUcoIAKQ9iniGtvr6a9rs6kNNqcLlRSlIUrKVBBCthlcAnWFzJjQttJYKEzhTNeNZlfcAywHFAeABPf862Y/FGmF+8PsT3ItIV6bdFUykE/pqZVHxI+YtrD95X1U0S6i93i8tb6jTvEtkuMEJKVfRHJ+kRI5AExYd+pBQ406UlKgcytzt56i00LOG8EcioJ38SMt6heTl7piqDn/GtoXMRJjXz1nWy5w5vROHeIeF78c/i6W8qSoVOnZS6M31GxLF12rum9HrtdBC6J9+lVI3LbqhZKzuBNO8CRkJSDOxBn/Wt6hYSR26heUaxzX6FeodTVBCnElSd9YInv1HLnGtrxcF71N78LcNVuYLIoEsEzM9US2defq288bjxB+esPXbfH6NbRMVJ1mc6Arlzk+/3Wu70Ub6/OvChmmKyTdt41NL7CQ6P/i2w9ICyYtK29a7iQtcpiW4CdvfYIFIVoPfbpYUFEczYMq33HwtcNVM5Fb3bD1MrmUCRFm2oiDuZnQ2dSxmSUnYgzZsPo6p9SI1m1lSvuMqr6pvtLgZTuBE7T7LapyoUzQVqoJynxI0M+62gSnROkfP2Wxmci0adlZj29ofXawYq526GS57fHnbMxUCZ07uduCCkZdY33sm4pvCvuvDd7XndTTTlZSUL79Ml0EoU6hClJSoAgwSBzFp2tubBMJtqlfVXI6HnrbCZ1B28bUmwN00eJuK8ZXDhyuocN0lNet5UtG+83RvFTaHXUoUpOZ2JGbn3a6WDxn0zOKeHcT3tct10txu0dLVutUr1TdL7DrzKVkJWpBclJIGx7jttayGFz5suiC+0IcuZXeQogga/VbvMTqCNrRd0euJt6cVuGtLiu+m6Ru8PSqilqBTIKWwpC+zAJMdhSZ1Os+VpMC9c07WEkYYnFjtwiWPEjQ5uxQxAIIJ0+yxnDZKqZpJBloluI2ymPssTChsJ5gzYfDnZq6tiQcrocH+EkfaDaPmlKdYDcesr2K/G2WzMOevjFssqavCy8VdZiLDglWVy+qQFPhnn2/sNrV4f4Z40xRhq9sX3Ph5T1zXE2XK+sW+022yMuaJWoZ1RrlQCrVIjUC1VAVu4wwwmQR+eacnNsQCrb8LWFF8XgaI3b6dUpoVuB5dMl4hlawISso2KuUkT472WkBymyiqiLi6MN1bSVjrc5EwMqoIPzFjFRW3YWx1LOhEyskyY39/22SCoHXMowYPIwdLDMpQ6U5qgIhRAlOkQNe4/hY0hCNPJCh1s5kqUNBKUxzPz9dtJWVEzAMQnv9nj899jNRQ3Y3SreYvVLjgUkBIGqgQTIPh9p8rECiMoBCusGwIJ30twddc9uVPag4X4xvfAVVxKuy5fSbioHVtVtQy82pTUATnazZ0p7QOaIAJ2FmY68tQJWZSTsTIPKR88rGaOqcbplUnpz7FM+R1yW1ylQSZGZIMKiJ12snKcha8yARBJ1idOfdZG3F7rtCu8xkqJIMyYO8efzvYJDgK82UEpmSNyeR+fG2LVmzTqVEZezofsibaRlyiO0IPzPdZx1XWI1U7YWdF63LS3oO11rSUrlMnMNFffqbLdddtSm5quW1rIYUCk7ER8+VmTwOvBNQqtuN4pzpAqWcx1UNlgf5p99p2XcD711vuCQosKUJgBRAPdv+FsPis/dXuYVv8ACYu8xNeF5ucY2H09J2k9J7S03hdRKYnL/FHX4ezym3sJSAdUkcot5LdI+kTT9K9sQqFVtzqUVEGCQ1MQNRM6eJ9vrXSjM2Bvp8LSQvzwRuHRV1SzJUSDxQigSOcROnKxd1IMgjaxowRBFi7oB5R7bExoSREHk7nn8bFlgiCRPnpY68NNt+UWKOIBJH7bGA2CDdutULIevFhtQEKVHws5PzUlRhAjnvZKw6w07fVMl4KWCVREAzlMWfSKRjQZCVkxEWDmqOC+ytaK3D1TcRdMwJ1Gu9hhd2SVZwIEWcD13JacylJmJIja3JoUqOiDaI1pvqjAWHUJvi709qcip5kGxrqk/RpAV2ExpG3tNlX0FUgJTaLccOYrp8Z0F13LcFRV0dU6EVFSkuJSyITOoISmNTrvEb2kgrpM+WNuYn5IDEGs4eZSIhCQoFSkojXUo+ybNziUlqouBwJUlaksuzCgY7PhaOcT4UxwtKlXQiscUqBHWKjfkMxO1jWErkxLRU17sYhuqoZb9DAaddSe2rtSNSQTtbVzUQFAaovANvV5rMUWJvkrm0xiIF9+Sct9sZqWmWW5zgHbwsA/hS+L3oLrfuileWOsyrW2ppOQFZ7ai4O0lMapGpzWXahhNRdtGVgKlCDqJjsWc2GUoTh1gKVoHFJB8JtQNyVYEbjtqr+riyx6ph4h4cYhcXnonVqASoANu6DmDlkH67awVhnE1w3uVXpdjiKRTISKgrSe3nEAwZtKiUtAFYQtXjB+y3F6om7U9UCkpdbVG36Qne2jZiT46buwAyrNtwqJtQJwTdNdLyytQXUBWXUdv/atwy8DW5jVNhtAkjONPjYk/RVJrHUU93Z0CcpzkSN9Zt0xRrbqCp27UJAjtet+FgC4lWoCgTjW91uKWQptIFJii63k6DTNUto3/wCs+Np3ww4+m4bufQ0kgUyUEqI3H7OXdaEOMNOanFl5UiArO3W3TUxGgitpTPj+laa8LpLtx0dOsKhlx1K9YgocUmI8x8LVdPViWpdFfUfsVnVU+Sna/wCCVKt91xoF8tMgHKfpNz7rRP0gW2DdWGK5sKc6m8iC4k6JJA0nxg+60uXilostpU+4k6qhJ+4WjTjq31GAKaqKFuCmvNleZUyJkD2WXFr90eRyUeG/0hqkCrQczb6l/wAakEGJ5DfS3bCkGApxSlaAQgx9VgKatKqCjfQ6T1tM0sJ6sKiUjY2ONVlSuCEKjvCSNbFRm7AUNILOIUdcZbozXq04n1X6ZCVBZnWSNfjpatWD8Piuu9VAXGqcM3lXNKWZKUgOEyQI1hXtJNrk43uxN5ppX10qzkRl2kyCDraAcH4Acpr9xFc9cxUHJertQ0tKZ7C0oUNjGsRrzt55iVcKevmaeS2VDGJaeJM1zCtXelz1VCqkYdaVRuBTTmVKXUZCChQXpqNIPfFvOBpT1MVrUHepp/4lpKVKEcogeAHjruTb1/qsBU6wptbFUslM6IEx4mdxNvKnFuEazDPEG/MDs0tU49QXrUXe2wGlKcWoOlKAlIGpMiABJkb6Tc9m8Sjq3PaDqLKvx+mMYY4KH8VXfUPj093sPI7coEFI07J1nTv8+42bt1ZQ4ElKRt2iCI9o87WKxT0XePtTdJrXODOLChtouwu6nUORzhspznUzAE+cWrozTusVDtPVB1h1pZQttaMikqGhBHIzIIj8NSHtcfRKzRY5vrCyfF3rShpPZkOgrCSrNKVaAEbkwdCRt7IOqrWqdhD1SslGiHd41JTECDsAeZ0PsRLvq3X2A0hZdK8oCB2jEDaN508dRrOtlanyO0yutcXpAVKSlUcxJA8PA9rwIkO11GN1xT3ZdGJ6xl+pp6irS22GhlWEFREkz3SVSdASZM7gvuk4dYFBaqV3cEIBSvItySUyDrpsde/ukQDZtYRpaVtQKWg3mzLLQKYCRB1Bns7aHuHMxaRmqJVfTuJaqHGVqQQlKRMrgiOzoRJOv3CRHu1RcbRa6elZibEzTAuSpxNiKookoy+iPX1Vu0mWB/eVuFBGgPqx4CQC1ryqa2pSqmZp0OvrlKy4hJSCABpO4n2gGQNYK0qhYqMzi6VbildsoNQQRPag6zPqgiJ7J9q/ctwsXf8A3e+whlzKCgNlRCUp31OsxruNCNzMgtj9PRGF4DdVD2O8Fk0tPdD9a7U11e+X3VqRo00kZin2lSZB5DTYi0PY7wevDlSHUCW1mCMuUDuI11BIVr4WsZiDESfznVN3LSi8b2dSGwAQUUzYOpJE5Tvz0jXURaPMScLr7c9IvG9r0Ljr6At1pKRlTpBg+3TTaNtBY6N1tEHIy+qlv8mTxMVcXEq/eGVY/wD3Liig9KpEKMf3ZTSqB/OZU7J/kJt6VtVRplZUkgEZhpAnn9lvErgHixXDjjxg7Eq6gNt3bftOioXOnUKcDbuvcW1Lt7V1zT3WJDLa15SQqBMRFs92oiz0Tz8PxV12bkDaxoPj+CUFXiY7Jkkzvz77N7EN6ENKSo6DUGd+/wC2wjTpW6aaRmB9UGVT5ffZvYzS/RZC4haetQRKk67/AIi3mUEQc4BelEtYNFFuLmgLzddSSC52yOQB5+6zQdeYzuMrcRlcSRqY9n1+0+Nm30rruvq+MLXa5cbFbUvNVqFrbpW1rJSWlSSE90DwE+NqwLwdjR9YSvDN9qUEqSFGidIOviDv92lt1QUbZIQ5zrLIYjWuhmLGsupj4/MtXjwRw7Wp7Zu68l05KSTCCFjXXUdkDXmLUtvdC26lQcJKgpSSTuYMWuFV3XeA6MF53ZeFI7T1N0XtnDbiVJWAXE7pO3rnQDnao+ImgmseITAzBXqkRI2I9g95td4P6GZg5FUGL3e5sh5gJHSQFSSTtFvU7hDeQvHh5hSv63MqouejKiTuepTPxt5oYUwJizGbN4uYXuZdeLrYNRV5FoSW24PahRBOx2nla/nRivxFbwRwq4tSlKZZdYUVfo9W8tP9VKRG8WTHmh0II5FEdnXlkzgeYU+3bXMoJQ5UBgKTKHsk5SAd/CCfhZn33gPCtWV035yqallxx5amkpSlpJdW2pxIBSSQUtpbjQBBIA2g6q8FdYpAy7qG4IPn8dbJr9a44kpzyJjuncxrsPw8LY8R2NwtsZtLLm57jocPKeUzVPvdc2lpPXlJCUhxxcSBr2nlnWdCByMoxwjc1M4y7TVNYk07JZQM6TAUwyySQUkGUMt7yJHssfcqkGElW0DNIJ9/usl1eIropCs1V4U4DISFguAQVGBI7jMDxNpQx3JQulaEpUamrpu2lu2mU4tqkYSyguROQSBISANNNhFg3qxagSEgd8jtEzHfNmdiTiXdNyD0dCautqWniwptlEkLKAqRMaQSbJVwcRFXxfrly1F3uUyStwNOFeYKU2e0CCNCBr7bOEDrZrKB1S2+W6pzx8utF2cTcTMNg5fzq7UA8sr6Q6B/nH32j0KmjCNew5I8Mw/2RacOlhdPofEKprQjWvoaSrKhO6StonXl2E+8Wg1rMW3UjUABR18Y+22+on56djvALzyubkqXjxV3uCN7qvLhRhx8rzKbpjTqE/8AFOLbA9yRa8PQrvjrrrxVci16sVNNVpE/roUkkf0abefPRdq/TuGBYLhBoLxeaAmIBShe/Ids2ux0Oa/0TiHet2k6V90rWByJbdQQfYFKtl5miKscPFamJ3EomnwCuKsk6R4bc7BKIBIB08RbtR0AUfDa3JEiTuBBseNlXOQZ022FkK+Gsr+eAAoTZdIj9Hy0sm3w0XGA5MFBmxVM7K9DTtzNSIFajbe2gSh9YP6WVX2fZbYJGsRp8LcunK8lUq7STr7vxtbMKqHBdlZ8/q9ltOIQ62ppYCkOJKVA8weVgySZIJ91hEnNoTrt32nabG6YdV5qYHut3CuN8a3ddtGzUYkuGjrW7lbeaDkVLFSlLim0KEKdFOl9SdyCJGoTZYvylf46XDevED800VzX1RV2Uu1F8OGnvBCkrddbbNW8oIWzqsIQQnIopyghILn4r1+D8F9ILElJia4Kena9IReVHeVKy+qr657q3ZkPJSIKnYKQNUpEzJsTxFxi4ZY06hzFar5vI0zRSy3U0IeDaiolRBW6dFdiSEySCSTpbXte52WQDkFnXMa27SVK/QMvoPYRxNh1RzCjr2atsTpldby+6WrWmSdYJNqb9E/HNyVHF69sPYfu6lpLrr7qLtOU0aWHnHGy2SHMpIMZnI8LXFSoR2TA5R+21FiQy1BNt1a0RvCAOSGB5gRYe51qTfhbnsvU4I80qj/WsVB3BE6bkd1u6dZbvWgezQnrFNK8MyTHxSLAotPArIJAbVp3AffbLdEH9Yf4tss6yYvC2jn91+Fm0qnNebapJmE5SZPttPgbX1YCO1mMlQjztD3DW5qbEfGPBFw1ZIZq70Ofq1QSEtrJg78teXntaxmNcGXhgquRSLqEVdK+VLZdUcqiBGihO+w8dPK0NLURtdwidSn1FLI9nFA0CaxSpJnJpJ5ag8pHz8bdISpRKCkkDSRzGvP5i3SOtTBI3Gv8r4z323L3IJIPZnXUa/Z42slWlpC7XSVjSmm1NgKfbC0BLgMA7EwdDzjz77b/ADfVqcQySMyoIKRO+3LU6+duAt6MwUQBqQOXd5fb426U+/IV1hgAakyY+fusxdyXTlC006oKWVto1lCSSPsn7rK1Bhanfp6mtbvy7Io2EVPozzjjbr6VEgob7JBWnmJGhETtZKDz2TKXdQYInSO753m3eZzLn6wdrTQd/n4T9dkuU5gG6L+iurcOVpSYVPq6a27au99ap6lfaV5SfCfP9thM7oP8ZAHZOuw7z8feLGqdx4K7LipI1jSInXu+TZrn2ClawEp1cL6C+qDGN1XjTXe86lFQlLoSQMzahlV7wT9dr60vDp5dIQqnBK0wITEey1NeEF2P1V8MuP3y5ThtxKlZRooAgwfu+R6U0nVqpmlIAgoBHutkcUoRitTkLrWC0lLVy4bTjh+0V40dNPCj2Hel62pFOphDjtyuBU5s57Ikjlsr/Ft6a06fownuHIcrUZ/KV1NGjpQYVabADwoLszwACT6U6R56AD3d1r1U6YaBjeywRCCIQ+7p5JHyGY8R251XZGmhnUc7AOhXMbmxkgBOli7uipBnzsXGoJNkSdTI5d1iqgORmeVjjwJBg+6xZWnKR3RFihsgnbpawXRLqL/pgEkhOZR8otJ7NyUzDgfbR9IPVJUSB77MDh06lu/w3OrjS0/UfstKVo+7RyuLnbp5lcwZWlJ5u0qVmUsEnnFt/mxP63wsftq3Cgh6JvGf1Scq6hmzBZFm3ftG5T3glAOixM+HyLPazPxklb1WhhDhbJSntp3GtpqeljhfmakdK54s5EUMPmSF/wCcbA3ywwq7FNVL6EBXZClGBJ0H12GZSkaheb+dy+FkDiI8lrC7z6icrbrStAN8w77SVl3QPAPJSUpAlafFG6yhp6OjZpXXEDqkJSoz/JsLdN73RT3W2hV70TIS4SQt1I09tmTijFyUYb/OBElxtk+sNykbe+0H/utdvCrVSdWpLayqCFTty+q1Xhkbw7OSjK+YFuRW6oL7uW83zT0F7UVU6gZ1IbdSogTEmBpqRZEv/iVw/pGaigfxfczdXSuoS8yqrQlSFJcAUCDGo1kb6WrNwm4oO3Ni69lv0TTbNPSuMrdzmVHrG1CJ8Br9nOrfG3FSa/ibi6mShzML5vFpYAScw9IXEa6c7ESYg5tQ6C2gF7odlLeISL0lVxV4fpa6449w6lDmYoJrW06DnqbJTnHfhC2ShXFLDSldzdahcxMxl8reQ96UHXVBKCIkE6GPdzMTE/fZWwut2idbaSy2Uuk5S4BppHZJ+dbJJiT2i4ASspmuNiru3njm68b9IfEgufECLzuP9zrVVTqaI6kOsOsLVGkk9k6/eLTvcuOsPXbelfhyovkN1jV93gyGyy4oQp9a0gkJyjRQ3NqEcHa5268WXqUENn9z95pMdoKHo6laEHXUD3+++PDSuFSziq8W1gqexC66DodFsskfXapoXSOry9thcFWdU1vdAHa2Wrx48cOqSodbVfF7PFtICBTXHVuSfAhuOdo16QPGjCl7cLqq67vu/FS6x2opn0KeuOop2wnOn1lrSEiQRuedp8C2g4HesWHNVesDrZldIS7kXjwNxMtvrFOttNvhSpUMyH0K1kwBp5C2jmjfJE5rjpboqeKQRyBzURw7xFvBjDt1sVvC7HSqiloWWXibtbQnOlABMqcHMWXqTH16VKEqZ4XYogrAPWKo28gOkmX9LOlhTtRSU6golCmUK7JOspB10NhQwVslosFSSRIUvf2RZrWPAADlzpGk3ITvwQgXncDdbeV0O0bzilhVPUqbWtsAxBKFKSZAnQ87FLnuy5HcZ37/AHI0XXAw96sT9GEz4+rYvS38q7WPRpykoCgmR2dI+y0f4axg7RcTK9TrxKKi72gEk6ZkqXr37AWw+L4pTRVrYZGg5TdxIGv8XVtSUU74nyMNgRp+P7FNyriukyTRt66nS1aqzgTgW4Ok7fnFZFxMuXrX09EaVSzKadwtOIdebSRotYbSkkawDEZlTNasdojQju3tHfFHGLdCmnxNSt5altCqYvZk5UgkKTmB9Y6LCQOavYXx41h0srY6ZuUnmBa/gp6KhqWSXmFx/GqJ4vv2uxDhi/afBoNTflB11Oyy042lSaiPo0kulKJ1TuQnXe1YOk30D744tcJaviLQcLE3fxZoqFqqqXKSsp0C9logvIdbbWptbhTmyqEKKgkFRGlrJ3bfTGIrtYZuSpq1V6nDVslDrYRUsrgrYQpROVQVK0yCN9ROks3ViOuXclKi8aFwVnVJDiHXUaqA1IIJnlytooXxRXcT/HRJiDZCA0DwsfxXzg0q3WKstVCOrW0vKpD6fpEwYjLvIEiI5mYmy3RVjr2WibTClLgoS2EwSN55bxryI0nZz9KG/KW9+khxIvWm9GQ1U4nr15aZCktZg8ZMKJlRUFE8sxURA0DSwyy5XVqW2UBSW05yCDqQNAJ7wk692u02uA+7A5Z7JZ+VSNhu72WWUCqqEBoASlC/0oJUdCRqJ7vVkc4fNNVUVLSrqlKShoISyUkwXSoAjQb6kxpG/tZ9zU6l5R1i8mYJdSopCUo2G/KdQe46DXU+muVeOIE3Wwla2KJHXLRv2wSJmNhrseQ7zIt0SNE9aI1CENPNkNqaQ0nIAEhsgdoTGwJPgSNNIsbxBeLaLvcVeF7GhohrXVwXCglIHZbk81HTQaQdZkNlpy9mmli7LkqrwW4taGW2mZAUCrdX6A9yRJIUCQLNC8jV35f3ot/BGJb1bVnpcO3ctS6amE+vUuJBBg/oJJPfGtkaxOc8JwUF53niJlbmBaNGGMPtryvX1Wt56moUZADYiQSD6xGmbcbWScZC+7ow7eNBhK6kK6xDhdvGrdTUP1LepUpojQaJPlBjXWzuTdePqNDTN73lfbmRASi77lu5pFOyJkIK3QluAY5nmSNdR3aVxSXbwvTD9LQ0CilLqWn1VbsgalxLKVNpmIkgCRvNuddlrLmencFU6653r/SM5LubPm55pmbfQL0asTXbirDOHsSXyww8m/MOUV4nrEhYDq2WlqidtVqt4K46u6juzFNfT3bQvUlEXSunbdMkIPcRuJmN/O3sD0VL/ea6N3Du9G3Fy1cDLGbwRmbjy7PwFhcWJ7o57BcgXTqCPPOIibXNlcaiw3w3u29Xb7p7rp01TpJJUoqQknfKgnKn2AWizpQ1dyrw/c7l3MU6HBVuIUptsAhJbOkjyFm6vHb5Bh477T8/Isx+JuJHb4uZppx2QzUhckyNiPn2d9vOIq6ea0LmNAPQLbxYOYJBO6QkjqVHF4PB2nbW4d0EGRB3PKyE+gpUSpAVl0HiJ+w2UQ6XaUoIVLbq4kbTBE+0n50sQfbKlpWrVO0J8e7577XEVwLKeU31TAxtRJqsE8RbqCRIp264CDqQjNP+ZahOI21JcUSVdpKViQO/XW1/MW33cVy1t+3df9f6ML9w+6yyktrWp1acyCkBIOwUPkxag1/ozEgA5gjU666j2d/wtoMHJubrMY1Y5bKTuiVQXhffESqwzd9eijcvOgKM7g7CodbJSrwgn69rT30RqhwcPLyuRZhV037U0sEAHLCFfWTp52ptgPFysE4gF8ij9KQWHGFtdZkJCxEhUGCDCge8WtJ0MMQfnVeOUpbcaS5Xs16Gs2bL1vWyJ5+okTGsWMxWK8DnfBC4PLlqGt+KsrelcaG737wCOsDDanCkEyYTmI56fjvzilziXfTrbriruTSPNt9YQTOziZ33BbWDO8zaVr0pzU3S42kCH2VIjwII08D52ZWDrowVedAHMRLZaWu6apRcL4ah5tg9WSTHNsjLzIG5Nsg5zYWl7hstLX1D4GgsBJ6AXJ+CiutrcUX1dr1det+VgYYWioyJGQBC3urUiQP0QJ/bZPvynXSVF6pbecW6aippnVrWZUlCEutz3Hs93ws8cXvXEwu+bqw3Ul67Kxla2QrNCXQAqASBIJH1Wa19Ol68HVqIBq3aKoAJgEOsFtf3bcrGwSsmbdpTZqKupWNdVwvjLtszXNv8LgXRJvEpdvBy/qgFQbqaKqc01IDZQsCNJk/ts47mrGV48pw00ptTlauqR1nZIS6wk5TGsk/JmzZN1ooLuYq2FOrdqKbrltlPqOJW0rINO0ZSrUfdZeqr0ava8ncRXa3UIbSKZ4dYjLK0LKFCQY2I+7lZHyRlpLXCw31RP2RiUUrIZIHh7zZoLXAuPRotcnbQJjdMSjCryuOtbby+kUVVTqUAYJacS4P66vfasLHrKTEyhVrg9I67xjnD1x1GHUmvfYrnGlJByKDa2jnJzRoClAnytU+8LivXDl6N0V/3e7SuGDlWPWSTEgiQRvtNtHg9TFJA2MOBd0uL2Wf7QYFidFI6pqKaRkYOUucxwAd7pJFgfDdWE6IV4lV34nukqJKHaaobRJ/SC0qP+am11OjVexu3jNcQUrKiqS/SOAkSczKso96U++1IujZQVGHaapvCtZbQa1YKVNuJcUtotylKshJGuuU6g8rWa4YYqNJjXDV/ttO06aa9qV15txASptPWpzJMEj1TqQSPG1DWVEc1Y/IdjbfwC1EvZqvwjC6eoqI3ZZWZ75XANu5zQ0ki1/Rv8CF6WmRppvtNuAsbR7LMW8OOvCWgqqmmdxgy6KJZaqX6amfqKdhY9YOPtoU0iOeZQizhqsX4ap8LP43VfVO9cTFIuvXW0569vqEJKlLT1clQAB0SCdO+xsc0bgbOGniqSowjEKcMMsD25zZt2uGYnYC41J6BK5MK0nU+FgKhvrWlo7xaPrk6RPBjEd1Xpfl146pTQ3N1Irn6mnfpktl3MG0jrUJKyrq1QEydNrKmAeLvDvim5eLWBMRC9VXX1Yq4pn2Q31mbLBcQnNORXqztruJkhqoXOAa8EnbUa/BOquzuL0kcktRSyMbHYPLmOAaTawcSPRvcWva9xbcIdaci4982LPrhbKiR/GRv3g/hZRvFrq6gzrmM7xFkyvlNOViewQrY6QZ+y1/GbgFZCUZShgqIkzrztmaT9xsHnB2PhtbYWCJAO+ptOFCmhizg5wwxzexvzFuDaG8rwS2lnr3s2YoTsNCBpPzpZPb6PXBBowOGdxTv26fN9ZNn+TPM9+3O3WYEyCD3C0wmkAsHGyYY2E3ITXw7wr4b4SvBu98NYIue7K1pKkofp6ZKHEpIhQBGuo0s7s4kE7k2BTMkA7bW6CuRmPstzi5+riuDA3ZDZoVBGpO4+dbdPr6mmNTB/udSHgfBKgT8AbBJOvfyAIH3WHYaFU27TFRhxCkwOciLdyXJ7CFAEJGuu4tlkWhvqlNFTl1xBX1SMxJgzAmRbLddNXj90fKc1vSMwIghRSmorHEk6epSPHTx0tafpBN9Wq65UpJUt3VJidu61Vujpe913D0icJXtfT6WadoXhmUNSAqlWkKjulUk2tR0h71pKuruVqgV6RTOsvPCqbILWpAiQdDAnyI8rUjGu7+w20srkOaMPeL63/JQ+kL9VAggyIG89/um2BIzEKJURAOnz9fMW5PrAjVME67fhpbS3FpU2hLYymCpeYaQNt9tY99tLmWbcCdl2kFRJJUQCImYEa/X87y7MCcMMb8Rql6kwdcjle7TBKnUpcQ2UJM6nOoTsdu+zUazFSSSZyySRzAOnLu+dbTd0beMFy8KsQVb9/UNQ/R19Mptx5hSc7ah6vYVAUDJ56eM2Fq5pI480Quf4uh5c7BdoTfxX0eOKmCbkXiDEmF1UV3sauuLfaMa7ZUrJ1P1De0dKaU3JJnWDrBgz8/starj70ncIcRsGuYbuGhrkPLWlYW8pOXQiZAJ7z7rVWedcyoDZbU2VKzkmCdoifbz9loaWofKCXi3ROgc6Qu005LlIWSCVJ5QSNz9nz42N0hS2EkwDMgSNftPnYge0kmSRGnzz0+s2317rFdTtdajqFJX1hPrCAI8BInfwsQ43R0YI3Uj4dvxd1NhTahmUkxG8iAPn5Ho9hPE7NThq66ha5LlGysme9ANvKBrEtMh/qyshBQFBQXuZjb2fEWvphS/L3RcNzVaLyZF3IuZpDlL1EuLeyoyuBzNoAAoZY1ne2Qx+aSiyyxGx1Wiw6mGINMZ5Ki/5RW+vTOmRdqUOEIapLmRy3DqlDfzOvefZb0epSS2mRuLeUfTfvFd4dLVh9SUqGe6EZ1KjZKDHx99vVyk/ikGZkWlpZDNTxyO3IuopY+DK6LobIbYad1i7gOszYzyhI2sXcmfVsbGoJNkUd03E66+Nia99QPDX7bHXQIkqg2JLIk77WLbsg3JawY/1GJKI5oCllOvOUkWl30jxtC2HXervyi1g9cmdfn5NpUD6dy57rB1FQYH2HNSRx8QXSqalI0zC2jUp/WFod4nUnGKuvZj97+/rvoqHL9L1oAXPOZSZ8Pqsz2MJdJV9YNTxLu5AkHKhMEiRpIR3T77VTsZka4gNU/dNLqyJqQP0vjYnXO08turbbUpSwmSkEx52gqgwDx4NWp68OKbC2IP0Scw321y8rJ94cMOLTRbdreMlQ6jMEdWAsDMrTNoRsT87iJ+MTub6lkraW5tZWDU/SBGjLY/wRavWN75qKm8b0oXax0tJqHAloqOUAKMabWPYX4eY2ua/WL3vbidW1zDQKVUwQrKrUa9pR3iDIO50sx8aLrKm+r5FHQ1DiuvXByKgmdwRyibPpK99QXCXTTqiG0xis63NJGM8d3XdmGGbuvEuNBLTQUsxlToNTroNN7RZX3t6E8aynQVBIKgoRCkkcld0eHdvZZx5hJeJrnp6Wouh6taqmk01UhbaiFz2SDptvM2Yl/4PxhRH8wXSmgZpkNtN0oVUllqnaQ2EZCJzA5iADOscp0Ko8RjaSwnVES4aZrOCRWeIzVff980t1OJhmkfFWAUqOdCM0KAJymNYOugtE/F+voU8XsTUorKf0movSufQyVjP1ZeUVKjuHzsbOpzAuJmhetZQXAwxU9QtVe/T3m6kOL6vtaBPNIiFeHdqVx3woxpU3mzWOYSTU1zdUmofqG6+qeVUIOqk51NAQolJJndItA+ojfOXE3KMbQlkQaTYKML2vejuymbqKiqSy31jaCRrmUowAI13+RyX7jwxfDrbl7pUtFKk9aSpRWcqQSrKlMk6JPvOlnTQcJcc3w43d12cIn6ioQ4HVumoV6iQSZzJASAYjTu87SbhXgff2EaO78YXhNI9RoccduwPLgJzIbAdJjPqvNAA+9k04d+jj1d0Q7qeONvEe4WTc4bYer1u0+I2AFXfeN11RZU4god6t2mcAJSdRMgQRNrq8B1uuYPvGqcTmFTVU1Scp17dIx4eFqvXRVXpXXsypTiPR1qWlQCRMFJgbkx899rM9GWpP7iXAFQDRXY6nNGpNKkaR/N/HumwguM4LxbdCVRDoCAVKXZQkdY2RIkS4e/uAs3eMFSyjgrjIsqSG2bqqHnOtkDKEyTJ20B91nHRtv1byUind31k7fCxXHNxi+uGuKLtqqdTaau7KlIUkSUQgwRI3B1ja2scfRPwVM31hdE8NvuXlha4qykKlJqrrpXc6VaGWwdx52UfzdWuTKla95Jg+6yHwvfpU4Cw3nvJK1M3VTtrKlAFRSgJk6iJizwRed2kaV9OOeryfqmwrZowBdwUzo3X0CaXES9b4ue7WFYdutm8b1NOkN0TlUGA4hLiErXnIMZUqKtt4HO0U1+JUU2L2nAe0hooUoGNNIHnr8bSrj03LeCaC+aWppax+iL9M2404lwoCwkrSCDpJQmfK0B4yWEXoLwaAC1Nq7SlahXYAgCTGhk+FvLMejY7Fnje/5BbbDbihYen5p3X5xOFz3O9ejjNQ+Gl9tNOM69xsPaNN9LMy8uM9fetK/QU2EMRr7IIdFLKAdwcwmCIB9o8rIVRej5lSX8iErSciUkJEjaRrEjSIn3WI196vrYRDiUpUowhBgTpqdNSQBqZPutXQ4a9sgkYbWRUssh9U2Cf/CtvA9Pjtl+9MRLuWpqVmoKHSKZguNwqQp4dlKxuElJkaSJtaCmve6L4ZYq7pvikraZZWkO0rgdQSmQRmTKdCCDrobUKQ5nqGnXezKhmKjpB2Bk9reIidbWd6NeV3htTkErFJW1TSkyQBmKV7bHRW1txQzGRpjeFTV7cxEt/BeH/GSpF58XsbXiw28E1OI7zebS4kBQCqlw9oDY6ie7vsb4fsNFl2vU1HbCU6mY0iDEcjvEnu1sLxbwveiOLuM6JqkdUlrEd5tSUgERVuCAdtftstYKohdVEhL5S0W1BQJb3SInXtSqO7uIjQganMMgAWYAPEJTmcq27moDWLWgqErSFbhO4Osd/nqPCyXcFcLvuxddUstl+sW4tRKCgNiZA0jSAYnvPOLI1/Xk9eTzVI04QiW1QYIMqiZ3/Vmde8d/OIKpmkw+HJcSW2cspaAQTExmT4wec5pOtoxupSbI1cV14hxhRqvRVdU0tC7LJaoG3OsqGype6gcpEnL3Abkaiz5w1hnHFO2bouErwxRvqCnOp7TrhEJ1SglDeh/SUtUntKg2i7hFd1Ve60tG9r2YaaCkhFDVqbVvMQnWNSTGsBR5Wmm6sG3LVLNO/fFT17JAcW/fSisK171hRM6Rl7x3gEBqhvcXTruXh1c9KlmsvyorbzeQArra2oU6lMagZc2VPfoI05aCx2tGHqV9bwuVqpq19tnqG1FLYKdDmnwP+LHKwV24CaaSF0WLr+Ql1CiS1ea1CSTMCSIgpnxBnYAqztxV9C0hmmv+tU668CXHG0KUlMq7MpSPAzr4TrPSN9HZOidZ2pVZekphzEtPV3TiS9FFdG+hdM3rHVqkrAgmdQY7uxb0b6MlO7Q9FPBCFEpIuEPyRMAuLcHwP1crR9wTwng7FNdiHh5jK5W8Q3df93ocfTeMuGWFkyhROZsjrFEKSQQUzNrJMYfujDuEW8L4foU0N23dQehUlMgkhppLZCUgnUwOZmdzrrYGX9JAYz0KLi/R1DZB1CjivIrqqhrXKyobVQurcSll9SW3iW1IyupGi0iSQk7KAPdYnfNUt2mWjZIII0mNfH22DLylGAMk7BR+Hhufh4WKVi23Kdas8gpCoJ1A39h8LYJkdnL0Vz8zU3Lhu6nuxqupaMOq9Iq11bmd1TkrcnNGYykSNEiAOQ3tu9bsorxQym8GUuJZfRUthSicrqCChQI3g666WHaX9M6G4lbaTAEbGBPOdT87F61fWBOVKTBkZjpG/wA+djm3JQbrAWUJdJKneF54WqWm3FKcNSyQhGYmEgwO/U+9ItUjE9IpqoqAtpaHA8tK0r0MxznXlv529BqzMxinC9YFZWUVjjKyVjLC0EAkzG+vttTLpC3cq6eJeKqJsDIi9HXgURCus7Ug66dudPfpNrnDJbScNZzFoLN4vioYIIJ18LWR6EN4dTje/wC7Y/j7rS9ljctupEefbNq3qkKUBpqRFpp6Il5Gh4zUVICR+cKKpp4CZk5OsA/0drqubmpnjwVNh7slUw+KvuDNMmVBQVIjfunXu1tXm/btvOjvl26XC0kNkt06lLEKBWtQJj1fXAg91rDN0z6mShDTkA6lKCZ+Y7/utDuI8H1ar3xFXUtDXLrKd1FS0EtKJdT1YUQgRBMlR057m2FlYZIy1vh+IuvXOzNXTUOLRT1Lsos8B3uucxzWu/7XEG/K1+SaCKu4qa9Sai/lVSEXepxTLbR7S+q1IJ021G21iy7yum+F3W/6A+h2npY614hJU2FqG3LbQzz77dqwhewRTq/c5eSs9NTKBNGvtdlxtU6QmQRI3Ea7g26rqW/qqnuurOFbzacapjQPtrYUgJUkoObXYaqA8rTTtE0bhGNbb7fLX/4UnZqVnZ3EKWTEZmmISF2VrhIGnLYSEsJAsSNAc3o3ttcRm970vNAp7tp6I01NRPO5EslSmwcwKgVcs42ieVkp+8rwvE091U15Ppp6h6nU2UKIASoOCBtGqR7z3WWLtw/f913t9A7RtJcL9M76Q6hKShbmcxKhGqjB7jtpoh01zXk3eHWvKu5hptTKmkGtYypCFkhJ7W5BUfaTZ0bW8XiXJsDuLdNLWF0yvqJG4a/DssDeJK0gMkLySA4F+d0r2sbrY5rF1xyaSEDiVQt1+EL3q3Os9JutFJVtCdDL2VRIHMCfjaL8frXUcPsK1r7hcfaeqmUuK9bIlZAE+SE+60y43uq56m7r3evi+qKlT+bKhCAKptRU6AooBCSSRJiBz9tocxhXYUrsGXZcVNiyncq7r65+PRX4fKiVBKTkgEkxJgT3WMw27YqcZTdrtfROgykG+nUhWHaOZlVieOvNRGWSwgM/TR2cRLE5ob6epDWuOmo23Iu9ujg+3Rt1iiDkVenVCI/Tbyj4kWne+ah5i7XVoVlc7KSQdQdJI8fiJ8bVW4WYwum5D+Z7yqF0y6iuoa2neCCpJW2tBUhUaickTECTO1rUKpamsaearRlD38WJBgctRuZ117rRYlFI2WZtiC4mx5WyjW/x06oTCa2gLcIxCWRjoqdkYkZmBfcVEhc3J63qkOuQGkaXvor0cKsSYyuDhhhq5bh4AXq/dguqmUh1u+LsQiqDjQUp0pU8D9ISVHMAe1qLZ0acH8Q8D1GMrrxRhRy4cP1l6m8LgpF1tO/1CHFOZ2gGXFhISkM9wmY52I8CMbcWG+EmHqRHDZu/mWKT0WgvGlvmnYbW00pTaA8lwhaCnJlJSFzE+FpAwPcWNsMXLf2I8VVrl/Yivd5y8PzbTVR9Gp8qIao6YukJSNIKzlBUqTprZaePMY3el6I6AAabbC/yUeM15ZBXQNEGWd7bOa9z3PIfmDtZXCPS+YvaDZxbYXNot4N3Bdl59JLi7fFdTpfeuyspBTJX2koW4lyXANs4CCkK3AWoD1jM5XZhDDdzYjvjFV2UCKe8r9SwivcSow6GQoNkp2kBagSNTpOwtCnDC6uLGF+KmNsXXtwjvBqgxpW0S0LF73eo0TbecLW4A8SqA5MJBPZMa2sIFCd9POxGHMHD9JtjmcdQRuTrr4Kt7c1cv2gODOHRuhhZ6EjXizIo8zTkcdpGk2PMXHVJd7tHKlxA238rIdQjrmVNgaqBG3fZ117SFUioAnvFmusBILZMC2jpnejZeZ1Tdboow51tO2sgkrQJA1A0sMFAHVRsUpcyWygiercWmO7tGPssOVgGc0nXWxrdkEhQZGcnlOtulL7oB7uZsDm0kyfstuSkASTAg+6yhcuwdOUzMD3W6Sod4sEFRqCAPO257JnQeJm0jUhRgLkRI03Fjl3rPWA6iSOVk0KJ87HbvkvSkAjxPKzk1Eau6a01TxayZOsVl22nS2WdYDREkKM84Fsslly8UuHKlO8UrmzZ4SxWnVQEnqonz1+PttOV41LiilK1Ahv1c2sabcz8+FoG4a5jxLu19uUpFLWQSNfUAg8+/wB3vmyscWtWZUDNpAnc+PdJJs2lAsmTuISjdDja3+ofTPZOvI7zqN7LyKa6wAVNLWJiCZI+dbNi5w4urCGlLUoA+qNk+/XezhTS1RTmyLknsgawdPn3WWdxDk6nALUYbYukSlLKCqARvod+7l4WzNdclSaZIGbTXbbn3aD4WLeiOkEAaAakpkQO62IoahSlZkEGTrEz8dduVoc191MWjojKH6BPaLAhJMmQdu/5iwYvKkWyoNU7aADISEiJEbTpzHss5+HtTgi5r0qXuIOE6q/bvXSrQyxTVaqdSHz6qpSRI0IPnOugsyaa7UOu1NRQMViEOVClJQ8+HEtskDIgdmZHakkwRGgjWFs36Qtt81zmWFwuqq9EtIU4G0IA0OgTpt9v4d8ZcU8QO05perrVUyHip5DzTozLUn9EaERBGtpAvmhDd3udclLiVIjKqVSNpjwJ+NohxtTtVd1Kob9dUBdrzpp1jRaUBslM7nWDpzjwsQZARYJmU2vZIlPi+svFppP90LbbyOLcRGfqs0cu8+Wx5G3ppg+/1PYEuw9tMXVTSD6yZaTvvrB+NvJPBtZVrxHd9J17TTTST6wUUvBIKkzE6Gfhz5+nOE6updwhdjgeyhyhY61ATIP0SdAQBGv2Wy3aOzomA9StP2ab+kf8lSrpZ1oqOky04VdlVRdQgif0UA+Wg+Plb19o1ANIBg9nn328d+lI4kdIVtwKSqKm7RBEpjKjXxgxb2DYcKWkhMeqDvztNTWbTRjwQlUb1UnxR5SgASRFiqlBU9r7rcu1Cikjl3ixVTriElKVfh8xYljtUI9t0K6JET5AWIuFR897dO1i0iFCSPDewHWBwSAR3z32NYboKQWKNXW5kvSkXGoeRt5i0hioPM2jSjXkrqYhWzyTp52VMTYmxhdl4ppbgwY5ebKkBXpAfSlObmImR7bVWItc6Rob0VjQ5RGSeqd9TUduJ5W4ZflYGY2i+8sY8YAsCm4ZNkFO6qxE+7NNgaDFnGl2raS/w9Yp2j663F5cp9qo2tTGkmLswt5qwE0Ybb9iltbykoIzHayXeTdZUURqGmlLQy62pRTGgCxyskXReuNnz1N94domml69e3VCUDuCQTmPut1fjyxd5Qhwj6ZnQK3+kTYacljsrkTAwu1ajqX+sbBB7/rscuOhadSkqCSV5j46k2ZmIr3vq4LqevS7bpfvt5TjbbdE0Uo6qTClExmO45H2CbR1ScZuOd3VlWbo4G195sFYFPnrm2UpjQgZgCAT3k/Zavq2uyBwItfqAppQcpa3dTPSYYonaKpSpCTkr6iD3fTKP22qPShOOUXxeF+FpLqbwqKBaGiW0rS2tKkKGioM87SReHFvpD01DUvYe4TUr7tS+tyoYq7xbHojpPabHbTmA0Mjvs1OCWCcbUV23xU8T8NNXbXu3u5U09OxUIcQptxKSokoUv8ASEQT77T4BTRPfLHccR1suoO17/RUeLPrYWNewnKN/wBibV716cM3XV0VPhxTlDWp6uoccUteQZcpgpSOXnZXwlg5ji5iW7rxxfdXo13hjqU+jvOtrWy2hRCozkgk8yNYtN9Lhi4b/q6S7r4uplyncWESUKGWTyOX5m0l3LgbC911blZd90MNONMt06kjNqEgK17/AFzb0uCCjo6YhsY4pB9JY178QqpRxJiYxyUd4F4YXZddJXM4Lutq70VlAhLb9QFqB+kKlBSio5jCY9tk7itdl3LvK4aaqw8xU+l1Sy4HWyQtKm2TAyzMdwB2Fp1oXGGGqpKChtDVcpAS2jQJOWAAPOyJeqGOtw3VlLZUtQIUWsytkDz5WEw2FtJJxpRmeeaKqGvlj4bXEBUbvCmprrxULvoGWmWOvQrq0piEqAMjnGu/jaReBDFFVXFTGvvSpZZo8P0bpDFatkKKXFtSsIOoATpaL+O113xTceKp27r7RSMsVbA6ssJ7DSktlSEqiQCQZHiO60o9HhpJuaqccdQVC6Hmc4TsGrwdSB7J+JsHM0ipu4WuVfU5vS2YbmylugThNb0rvCrLZ1XFdUmR3etbKrCeCb5oH7voH65xC0uhtpVdVZlEoJGvWQRpsdPebIrjNQGU1a73cKFGPomyPbAIgWcuEfQ3nA2b3eWvtIW04k9uUkCDuediWubexc3yQnDqr3ymyZ/B7DGHqrBV33kq67uVVPNFFQ4871alFDigAdCDGo8Pqfirmu6jpgtu7bndCEglAW2VHv16sk8/HazI6PwuHA3C6pbrqv0G7rqrK5bz9WvRH06lElR5SvTziyhx04iJuLAVNRYRq01V84yWm7rmNMrMVh2Ap5JHIBQg96k2jEVJHBxZGgm3hdWUUdTVTCNhNibX5BKl53NdqLm66juulo1pqUx1ScsoUFSPVTzjl3WgjiU0GXqZyU5AlSerO59cnXkNB3fdNreFKzAPCK7cO1t7P3hWXdSMipqn3S4px3rBnSmdkAqISOSQLQTxMfU5RpeNTnJfBQjKAgabeJlR99sN2igEeJRua3KC0aLS4Y7/AIN7M2azjqmrUBujpnFF5MAByCsHInXc93t7rM+pxzhtKJReSldv9RXagcpAnY2dBlTVR1rbeTsylInXbxER9dm+9dt2MghF20gJJgpbQNZnWRqPnwtJTgAeknSlxHolSH0Z8aYGvPiarDeIjdtRQ31d7rSG7yZQWVvIKVoA6zQEgLgbmfK1tb3r8CcOblcdTW3Hhu6kK61RSqnpGCoiMxUcqZgASSNhrbzsxM3R/mCuUzTtpCWs6YQBoFAx8bQHiFls11KU0zchG4CR+kfEa6D3620uH5XxFoHzWfrczZQ4lEeJLtFfPETFV70tR6S3V33X1KXisOF1LlQtYWVHcHMDInffW0fYgDlHVmmiM6AYXABCswgnX9VMeVn5UUakvekNoJUABqNJEfAxE+AmzExYoN4kUEqzL6hoaA6AIGkakjfXX1eQg2tWtyhVzzcpDeGatSEkDKdEplIOoJkGI7x3wOYAEmcEsM3Vj7jThHCl73QKyjcql1VbS1DYW06wyw46ttwCNFqSBtOuh1i0bOrX1wSXCpuMqe0SARodz2tQABuJMCZixPQNuJd7cVr+xS4kli5LjFOghBAS9UvJCSVHc5GXRqBGxiLOHVRkqNOIfB9PAvi1WYXfp3H7hrgKy6aioTnTUUilSUHkpxCvo1bDshRhKxMnYduvDdattHot2LdVlQGQhHZ8Ak7QD369naLWa6RXCFzi5w+Xdt104cxFdLiq65CBlUt2BnpwojZ1ICdeznS2SSE2qbgLEVzVN20iL/KaepQEsIqHWygKAMJzEpEK12keqbTRPuVGRpZSNQXBcVM4gU1zNAtJBBUmEiQOepOo1O3usuVDa0NFundISESGVEmFDVOh03SPDeLFqNt1I65uoC0ASFJIVAygwN559+2h3saqagsNNuVFQgIJB7QJzGRAAnuA5+4ixBFxZIDYo1w6v8Yf4hXNXITm62oTSKSBMoehsiRzE5vHLvFrdrRKCgp30UdNdY9p28I+NKruFRRXpd9UppTQp6ttannAEJSEuAaSecJH2WulKVZXAqc2o5iI+drVttSEaTsVB7x9HW62XJKSpBB8Dtt4T9ti7qsyFKiAoetqI1sfvxsovm8WkgJiqcymdu0du/7rJOYGDmIQNEgEkDnt3a/ZbFubleQt/G7NGHeCTesyOEJzKRkIynWNN+Xjv9VgHAlTaSCAAcuUHz+ZsM4SXgnIkjtCFannr8+/awKwrdeUayANvmPLe0zNlA/dNnGFEhdzsuugw1WNLzAAH1onzgn53qf0lbuco+IV6JKs3WU9M4jNoSEtpAVpoZyqHLa1wcVt58OVwDglLecA66pIOvu8bVn6WFAP3T0FaGSfT7oSkkDTMlSyNfaPh3WOoHWqAVVYo3NASqwPIWlxYUMpBII7rOjhU/XMcRMP/m6qNPUvVzVO27nKMvWHJuNQIVZs1SShwlQgKM6fPhY9hi8V3PiO6r3RoaOtYfT/AICwr7LamQZoyPBZOI5ZAfFXyOA+IdWT1+KKchMaGocJJMbwPmLMa/7svRDz9KcWNvKp2i7mbKj1igf4sH9bs/GbWQp3y4lTRS0AhCilaZzOAwZVykHQRHK1eL1dbpHHqgXa3VEVzxcKnFJytJjNEESe1PstgHySMlayO1zffwVxj9ZWUk9NSUAaXS59X5rDKAfZTXoLlr7wrW6avvlykbUhQU6qVJTlSIO/6WY+WXxsTauWpU+GX6irkPlpa4GUo7QStJk6aJ5aZhvFnO29X9Q3X1NNQt0lVUVFGyUJkhYQoJJmf0gI75smVFHe3o9ReCq+E/m5Fe22lsJgFQCk98jXWxIFZ/d+qrS3tH/0P9RIv5jvdyj+iDnWnqxlW8gADKSrcadoR7R42cV18MnL0R1qLwICVuoWUupVlRlHVmAnUz6w++3V4Uirovumb9MeeYyUj3biHELcKFyI5ynbb4mQ8DtNlq8mxKUpqCAB3cvdaJ01RFI1kltb7X5JKWrxWLEoKOvEZbLn1ZmuMrb89FHb/Adm9aBNLe15LStallxLIIBBjKBPPTnPIRpap19U7tPUJbdBCmwWjM7oUUnfy+FvRxbbSpyumeZKduemumv1WoVxfulNz45v6iAgtXrUqAiOwtWdGnkr556HBpnOLmFW2NU7Y2Nc1M4rKBTupUCUj3EKJ+217rlvBN6XLQXsNEVlI1UETyUhKvt/bah+aaZI/VWefeB91rn8IKxV5cNMP1K8qoouoOo/valNj4JHvtPjDRw2u8VBgrvTczwXob0QL2/OHCL0EqTN13pU0wAOyVZXfrcV7rTYs5tSe7QWrF0JL0CrvxXcpVo2/S1YG0Z0rSo6/wAxNrNk6gCfZvYCA3YEdUDK8rRM6ZfhbnMZ5+JNsPONvgbaVAOoO8WKAQy2tRUjIdNLNytbLTziQeftNnCCOfLl3WSL4bCF9YAZ8LF07rFC1AuE3kHJU1CJ/SC/YRH+qbC9rZW51sG92LwKjAztHTb1T/tWFggnfSde+1iHKuy3WwVEHlB87YSCdNu+2wABmER57W5kQDA22Js+6YRZbBPIx4W6klUgnfccrc6wN+7S2xAJA5abzZwK5Cfo84ifn58bHrt0ekSIMgjzsQEAZtRpqbHLtWC8JO/dZ+ZJZLuZX6x9/wCNst3I5pPutlluksF4lcM3GnOI1AEADLSvwQNRKIIPft872matUVbgaKiRtm5fbpaAsN3m5QYtpqxJdCg2oAt7gkmdthpqPZaY/wA9uuu05UpOVacikrMEq5Qff32jpXjZR1DDZOfDlQxT1K3luQ2hKlFJ1JJgQP2bWcn7pLuTKU1QIkQAkjl8x5WZLVSHEKKHZSlI7QMneeXz7rJTeMcNVN7G4W79o11qF5Q11kdobpBmJB5WIkia83JUUUjoxYKSP3RUQ0TUKhMykJJnu5fMi2HEt3ITCXXp3gJ+o89R9VmgpRKs4XIBkDeB36ctfmbFX7rpXEuOgv55JGR1QJ8N7RmBo0TxUOvqlb98y7A48hpNS51alKVAyqSByAmTJzfHvsNdWL7rZZU68paA/K2wVjWOzpr4WiSrp37sr3lnrqV9VQhZKiVJShSoJIO4lQkA8/cUraoNXgVPvrQG6dSyygHKNczkRqRAkHT1h3WH4TQdURxXFSPfnErD9ZUpud67qwqrEdWlzOmAo7T3erA9g5i0K4pvy9H8SVF2MsCncQwtpZJErAAyzHraT5yfGxWrvZp7EKLwU6gUreZZT1pzJ2AEnVJkCPCx7GjN039krLqrUmtaaLlVmUrOWwAQE98eXISRFoywDUJweToU1KVh+gqae86BovppWAsqQgwFDsmY2G+pkCZkc/S7BVbRuYJuZBK+sVddMYJ0Ci0knntr9VvPjo/1NQ1xbuGjecdNLU1LlM62uShxC0qSUlMQUkQDvy3531wcwKbDV1MI0Q1SMoT/ADUoEcvK2U7QTerGth2ah0dJ8lTnpRLy8fkv9pQTW3dl5/3tHs/SHs9pHsPRkejN94SPqt48dJ5VT+/s6lCMyUuXcoidAAhGp5d3w1t7B0Kh6K2RGqEwJ8LGxf0aP4KqqBaqk+KMKgjSOU2Ac0EnSwyicsCSLAL2Mkg8tLTMQz90VejXYaWDS4gpEqAPq26fMbmPZYgpQC5knXSx0RsgZBdDVThQyXBopIkR5WkGlrqenp6aodcCkuInmSqEyYtGN6OuGieS1mJKFZQnUzl5Dn5WUrqvCleoqO7b7rn6KspgShtxXV9alQIB130V7CLC12UkXR2HgkEJ61eKWkraTSMrTnW2kpLeU5VzCh7rFX8VL6paqV05xCkFQEKSFAHn9dks3LdDym1m+KgqQlpKCh0SOrmDoOc624Wq57rzXXSU9fXLf7QZTKggEzM6QJ7zYLhwu1sbo6zm9LJaXXdZeVRTtpSUhpK3I0AWSY25kWS74BN2rzkiFtqBGsQsEWWGqampUEIbCesJW4SZJUe82S79epG7pqiHm5CZgrAJII0Fs5VvM0l2DQK0p7MFigmnHSwvMTo4RrbVA448lTbR2WrfztXTpE45xrhLETlFcGJ6uipnaZpzq2VBITMjNtO4HzNo34QcVceVOPWrovTGt6VLS7vqHFJU+rKHW0FQUdp25+VmzYFLXwteHABJNiEcBLSNVZx2uWmovpGYwmvXy2ltB+2x8OlbLunqkEe+1YeHOJsRXretzv119uvF+tqUVBqqxSesOUgaH1yBA91psxlj9jAKaT0i5qy9xei1Mtou8hSkuJiAf506Aakg2FwnDX4ZjkTXOBuDrtyTMSqG1eFvLG7W/EKU8K3A5e9Swl5vI2pSlFwoOuWCQnb5i0n0tP1DlYiBqtBmNdUIH2Wg/hLxcqL5xBS4VvLBV6XJUstP1TKaxwZltlABgAaagWkq98XV133ow9RXKt9h5o9cvrIS2pKm8oOh3BV/im3o7q6F0mTMCfp5rDilkYzOWlLlI6n0uqoUkh5yoDwCUCco6sHfnvZJdK313Ow2taV9Q6EZkwEqCiAfA6WaFXxBrk3tX3bfLN33feqb1Fz0zqqpa2lLfp+vpyVISImEJMDcmDZKxNj/ABjd/D3CNbTXJQm83r2RS1LiQpdO0CtaSRJCvXSN7cyrhc/IHahIYJA3NZQJx8umob4rVT4K+t6unUshEgECCSTprFnB0bXUn02kcaKylm+W4WmR2bwKgI5+v+yyRxcvavvbFt5uXo4yK1h9dOQ02UpUErPIk6+38UbhhxBc4fCuvVq6m62L2vOlUyp3IPpC05MgHv7tzarlquPKbnQFXdPDwYhbdWdr6OmVTtGmpqMoAkFNOkE/yTNjeGWmG7zaWWGkrnRQaSAD5jX22g9XS1TSsrKMAUxLLk9qqIjL45Rvr7rFmumbWF5akYBuxsurCmwqrUVDXfQa/C3R0zQ8SZ9vBTOqXFhZl3UjcNaG577wrf2HsQXS1eNA9e940z7CkhaVpDqCDHLvB3BE6GLJfD/gFdOCuIKMVU99XheN1XdTuNXFdFaSv82rcVLhSskyInLoD2iSSdbR3hHjDelxULb1FddCW7+xBUOOIcCz1IcTm7JBGmg387Oiu49Yput1foN3XbqSkAtqUNdD+laA47hsZDJgS5vgiY6XEGsdwHWa4aqZMf1aazB95tpaZWG2Myi28lSh9I3qI7tdPC1XcbtU4oYOrfXIyFSQU6gJ98/O9nqjjRia/bsqLtrKG60tVTK2XS0xlWkKjYzuIHttHeP6hSKBbxKy4QgpKRAT29STGh+qN7UuM10OKVLJYL6C2qscNpZaOncyTqmw2pJKzmBzspOpOp0O20ff7ytQkQToJMgjnvPLe26NxluodGZuHmSpJCpPLkfby2tp4JB7RlJSfWPh+Hss+NllKHXGqRL7aU/ctchQIHUOAkJ1mD3RHL3jxtAN7sk1lKpSYys65e0SokztqPt08IsVWILjDjOY6oIgqI7REDz/ABtXu+kLTWMJUVo7CifDU7CR3RrG/sF7hnMKlxK1wQii6UKSG0pCg4SEiAZnbfw+A311jDHS1oxa6ygpWtTTJWANCYOwOg30jnHjaXMqV0wVn0SSQEojc7TGu3wM2iPGLbpxlVLIcAShpIIByhKUgctE7Eb8hy3uDpuqhNi9HEtsn6Qs9kgAakjcgSY5nQeG863X/J63MGMAYtv1bIQ9eF707CSqJLbLKspJ/nPK02m1M61stqbU0Q0lW6QkkkDcqyyQIA09p2tfvoe0hu3hLd7CgEKqGXH3FzqpZq3N9NwhTY79Y7rMLtLLiw2urAMLCHmyQUS4kwdRodfGeVqp9Jvhqzw54rDE92UrL1z49LlWadZhLVenWoQCAfXKg8mE7rdEQExaZbwdC1EzJmN4get7J8t+6zQ6XuFXcTcFnL5pkZ38MVbN6pSUEwwDkf130QrMI5pFpIh1ULt1Vm4TTtuTcN71F0VJWkrpn0/REmD6pOQz/J5HlpLvRU3lTtLerGEuLCAV1NFlLqNpOVXmo7qnTQDSzPutbd7UbJq2ktqdRmCwjM2pWggnkd58ROuwV6Z2+GX00yKWncDSsoC8+UJkQRvpEEc9Y8jReyTRBVD9BX1PpdM5XVJaIUHS4XA2oQClxoCUid1FPORJi11cN39d+Jbhor6ux1SqZ9kbwFJUNClQ7wQdNuYkWqWbzYYSUsUFKaqkQFmJGXTko67Hl42mfo6X2L0uq/W1lLLzNYhS6cJCcoWj1yNxmII/wNbBvZkN+qKYczfgtYxaSzie8EKcIK1hUQP0kgz4b93OyDmCNErJ1AgifLX53HcbOjH6erxKtYzDrmW3IHOAR58vD67NdcJMBROwASdwY/b+y2LqRlncPFbyjdmpmHwCTnozkqTpmnKlW0mPDmfjYWvvGsverXV3g8HagpCFLS0luYGUaJAGw5DxsG4pCllYIMpzT3fMeywZQoLWFKTKSdQdJnTQ/dZrLbrnboleyOuuquZylRXTrG8a5TED3Wrz0oqRNXh3Bd75cyVUztOSQQdENke/Ke/bx1sg4hLrK2pkLlJ8iNu7u/ZaA+O9P6RwYuOtOiqS8izmAOYCHUx7YFiqU5ZmlA1zc0DgqZVST1hB7oEazYMaZSNDY3ejZaq3AeSlaTyk2KJ13mBtrztsgbhYkizl6mYFq0XxhG473ypJr7tp6jbmtpJ+s/OtogYoqd+pU7WNhbP53ep3AodnKttcyPZZ8dHa9Te3BjCFWo5lN0KabWNeqWpvv19X42YlRelFTXXfVApwivF5ek0wUglCikkEEjaQTbzyrc2CrbnNgCUd2grqejxDDamqeGMtLqTYasFvqkC9qKtfob1apFsG7aa+EOUgQJUC4ZOo0IOdMa+7e2XWwKy6rqzJIFVdVfQK7UyUkqTodZ56621SvVtLSOXc1eLHo9Qhhx0qQqQ63BEabad/Ky1TP4fp3mCKxXV01fUVCEhgyWXUkFHgQT9ftKNfTj2x5qQdp8EJ/pcf6wTPvG7XmrtuC8E1jz00a3lhxWiwhaVlue4SYHhaUMBZHW7wXBLaqmYidIPKzOu668OU2VNbiGtqG2ysNNmnhKErSpKk6k6HMD7BZ58NAluhrg0StCXgEqWBJATufGLCTVEc88fDde1/wVaMUocRx2hFHM15HFvlINvQG9k4yNc2bMcwiSY9us9w77U36TV1+hcQ7xcQCG6xmlqxrz6vqjp5tm1zykOKJbGh1EmTzm1WulxdimsRXVeOQlNZdjrIHMFpzNr7HLXeEvy1FuoWtxhmamv0Vck/xa0+IV9n22tt0cK30rhaw2dfQ6x9gaaCVBf/AMw/DutUlvZaQfWT9s/Zay/RSrg9hq/LtmTTVrb4E6/SNkGP6O1zizb0xPQhUWDutUgdQVezoaXgaTiJe12qVKK26llMaAqbdQR56KUbXDIESCDOo+YtRHo23l+a+NVxSopRVl+lXKtDmZWB59oJ9tr2q1HfPMWpaN2Ziua1tn3XJI2nXwtqR+sPO2E9/KbaKp1BB8e6x4OqBKwq1kcu82J3ijOxm0kGZsaOg0V99g3EhbZTrtGtp4zZyikGYWTOvGEVVG7Omdbcz3pJ+tIsIpRie+efwtziEFqm60JH0Dza5OgAChPwm3JcJ0UojW1k3VVjhZDAnUE98624zic089t/hYMLndQ08LYlRiRGgn2WlUZ1QhV/KgEfIt0FDTUa2AKgdSoQT8LbSvWeYGp9lnBNRhKhIJ0187KF3GXhJGvuiyUlR0MAjSN7HrsUQ8SnSTZVycPaOuRRnnbLdJKgkAEAAaDX77ZbrpLLwtu1xbl/NKZXBDZUdMwHaPP5+GkkLvFlhtttKyHo7at9gdAeVoyo6l5mtD7TqQVtBESSZKlDKRGmvLfv52U6Z41FYg1lXkbSsfRJ/S3GUec91g4ZMr7KeVuZl1JtHX0S26ytq65ukZZZ6jrVOQn1dTJOseZtA+HbvVWY1o7ppnWVxWoR1qAIWlJnOCATqAFa/GzscxBeOI8YLuu4GUVNNR0YQ5TPNgtAlQzdk78tYJGum9m/i6mrbuvFzE7eWirCsEtJR1YaUEfogcgRyncb2lkqA6QN6KJkGVmZWYkIJU5pm3BH3wdvPayVXVtbmeoKJxoPFOdBUdcuo+EWijBHGKtvx9u7cRupbeTJLqEgB9vZUp/WA10/V2s5L5xLQ0V4NV11LVUrZ1WMpIWnSRpM6/UddxYwzNIugxEb6pOxrxAWmq/NakN9a2328gJOXLzkSDOmnLvmyNc+J6Jy7mBVQHIepS4rQhtfqE7SAREnXWzRxFUqvW+qq9A2XjUAx1h7aUnyjbbv1Fk7O6Gg51ZWEBJSoiFZuYJ2mZ9kbWDdISboprdEvXQhiorAahRdV1yMpgHOk6az3aGe7u3ssYpXcFDVtMuPIpahKFJSoJknbtBW+06HSCRzAs3Oto6a52a1kI9JCy2shRhBIjfcaa/JshXs+uoq4qH2XVSlKVpUSQM0xA8I2kWbm0TsuqfHBCuQzxhwshuFBd4JbJAiO0dh466ad3fb0Lw836NdtJTwAUNJb7I7tPnzt5w8KKly7+KGGA4VBCr0pcyQqRkzp7uQIn2jxt6R3emClClAlKtYG5+q2P7Rfzjfgtx2Y1if8VS3pSqfXxrqS2ypaGnLvKzAOUFKNY5awPHx0t6/UE+hMn/k0+3QW8jukwFucYr9WZyNIu0yDoZCAJ7u0R7uWtvXG61Td9MY3aR/VFrKE/8ADR/BUtV/SpPih1SRuQe732AWQN9bDr79vbYByEmZMTE2nYhJN0UeJykpmyc8SFGNI1meVlB86En50snPEpUc3sFjWIJ26CdcKGwsqGh2Nniy1d95BKK+maeQNUpcQFAGzCvReSgdUjTKgjQjfX32b12cVr2oWWqdbFK8tpCQXlAyswJJAIAnwFqnFIpnvY6LkrnCnRhjw9S/U3Pdd3qQugoWmS6CVZBE91lS5alp6nWhKu0hZQrlBtDlZxara6EmnpUACAoA6mZ79LAo4sX5Tkv0yKVGZARAbnQEwddZkmwk0U8kYHNHgxhtgprvagNZdVZTZykvMOIkcpSRNqtHB7d3ut1qrzqFhhbTzgIhIUhQJGvLTWz9HFzGDyC0uppu3pCGk/O1mbia8Kmquyu7SkhynczJA37J7vO0+GtfSxvY/moJWZyHHku+MfCrEHFPFLd/4QraStusUYpHlIcQoB1ClHQ5t+0LRlhDgBi1jHlW3TPH0lmiqaVZW0W2/pG8hKVmUqIzgwPG0o4dxFeN00C7suyscpqUqkNNGE6gAkH2WUrprKt6822zVOlS1GVZzJlPx1sNGaqBnDaRbkiXwQzXc4KIaPg9iHD18UtI/eLRduC8i9UCJC0rCVwke/50tI+Pnlqu+5nAJ6u9qNW3Mqjb2/GyEq8r2vq+b0ZFM6X0qSlzJmKlFJUnXfcAe7Syxe1Jft43Nd93MYfvJb6KinWtwtQkJSUqUZMHY93dbO42yV0jJpDqNNFZ4fwWMcxikClUzT9IO5GaMEt1F11CDJ11bQdJ31tKIfTV09aiiZyF3qYQOZCnUk/5lo1Vh+/F8ZcK4iYpCqhpqZxh5fWITkUpoHWSNBBnzHfZ7YfvVdO4whTZcRVOPNhzMkBHVuuqg68woWZQzZIBfqqyvjD5NOijHHtyYncvY1tFdlRUKbxRhuvGqf4tDORxWpHqx5+dn7iG6rzrcHUV3+gqTUUl9M1OQrA7HpTqpkHbKJsDji+7zYTd9bd12NO09Q+walxT2XJkPZ5HvNj1Vji6X1VqauobQFobSnJ9JqFOd2kwqbMfivCcQT8EkWHOcGuaL/wFXHjnfdDh/Gl8OuguIqa51aC2ASRmnfnuN7RLV4lcumgfQ3QKqEV1/rIBXl6sLo2HJOh3j2Wd/HxuqvK+3E3Ktxxlt4qJWAJEDcfO9o8vmjfqLn6gU5W43e1CpQESnNRZTr5tm19h0zqiASOG6GqohDIYxyTexJjCupqeoNC0llwpU4kLhQkpGxjvHdz8NRODmIKvG7940l7KQlygul2uadbSE9Y8lSYkH9GDsPCwV84Xq7wcVkoqqWmwj6N4BM66eqfq++wXR3wpfCMRoW456LT19BUMlQ7SygmCIUImUkT3+61lWlzKJ72GxAQdMA6qa12xKnq+rupLswVhOspMw9IvNmoeUVE/SKTBInQW3iPsqUtLskGY2IHf7rHb8u9Nbwxu9lTjqTdz0ocQRIU0sAcvs/FuYhu2pUpbi77rlgiYK0wfcLebU7zM4l51uVspGcNtmhKNwVjinexOidJEzp8eXx9oGNbydcu91txOdGUEBtICkKE6953+Asi4fu5fpCFLqq4hs5kS+oQZ10Edw+dLHsY1IXRpSpSjkbWAE/pAka5vCTbQUkQJVfPIeHZMVi96tV5hg1rpQgqSkZkiEpBAQdNRMGdLDvXvVPZkFzKYMn2RAPnG/wB9iDgBvkhS0hIQSn2p1B+Pvtw5DcpG6ZVKhpHPbb9vt1ccLSFSCZ2yEcvGphSlVATAklKoJOnt5ePK0VY0pvRq1p9luAunTCZ1KjMnb4+3TW0lGAo9swdEjeToPn9sR/xBbXloSkmAlSSEzl0VB1IA2ju3PjY+maGusEJUOLm6pIpghFM022pcgADWIJ5zGm/fForxeltzFlYZVlUluFJUUxIB3gTAjUToCBE2ltoLaaZSodlaCEkTGw1O0nX4i0T4zCEYoqEjKiUIlYjSJ7/AnfTUEmDY6TZBN3SE+y51wT1OohUGJkJ2EpMiI8pPrRpfrottOM8PrvYWgoyUjjYE/qs0qyI5dpS9DO3stQ9m6a687xZu2ha6yqqXfR2G0albqyUoGpGpKkADf7PSXhthFOEq2tw22nOi7r2q6EGCkqSGqhE+5lPht4Wr5KhrJWxk6lFsiL43OHJPdvKUZiUqlO3Odtd+X3DxeSKKkv3DTt01zWanr6U07wjcKRGp35n3DWzPQ2ARkaSRqkyIjT9g28u4vK5HAq7KcK5JKSJ5Ax+0xa0h1VZICF58XfddZgvEd+YOqmlF26K0sFDumZqewoAaGUxr5+NlleJ7rZayVzTiUBQlcCEyOfIa8vtOsm9KnA923fje6sYvsFFLf7arurHE6FLySVNqMd5JET9toXqLrFCtNLfa1qQAEtPgZUkRPaG0iB7tbTi4SCydV3VFDWsKFE4hQcQQVkdsZjvpJOo5eFl3o6X5U3TxONzuvlabxYfoHpOuZCS6hRnUx1agPPxtFfoNTh50Xjd7yXqZRUVBObKRm/FXvEQLPHhverN3cWcP3wiE09bUtZikwJWOr+BOvny5j1AJsSiYCBdqsVxPSWbzo3nCU9bTqQB35Vf7VmU8VZsyjBAiCdoPKR4DXa0h8UGStq7KggHKp1BOvPKfs+u0flBg5Vr0PLQEx79j8BbG4j6NS5bfCjnpGoo9odEnMFawDrynbU6fXbgInVQVHKNSSY0sYdQAIKpVuQd59+vn+22BsJIT2kkwCD7ufgfk2Ha5GEIuhEFIiNcs8z4R7/j7Yb4s0iXOCWJKZKSv833sVJA1P/CEg/BR+ZtNIR2k9lIJMmeXzr87Rvje6jV4J4kXZopSWzVQrWJaC/rT56WmifaQHxQ1QzNGR4FUCxC2pFa4pQVKjmOYbeHz4WSgDvPsst4kah5CypOqNI5wfxsiQUkE9/O23iN2BYCQWcVfnobXp+cOCzFMYUbuvKppwJ/mu/8AzNvK021l0XWt1ybrpyFwonqUzrJmSJtWroE3gX8MYrufcUtdTVMA6jrW1J/+Va07zSgpJACgUJJga93ui2LxIBlU9p6/it3hsbKmkYXgGw5pqruW7EFZ/NtPlHMsjU6+Hz8LFXLsoAMxoKaD60NDT4WcDzHVqyqSowTOgB0J+vu8/aWcZUhOVZI2Ovd3+X2WDGWyJNHB7g8gkVV1UCSSKKnEafxINufRGWUfQMoRp20oAAOkawO/7LKS2lQUAQEmMoPh92tuHWyoKGUIkgA5ZM/tP7eS+iNkjaaKM3a0A/AJM9HJcAcgyvcaaHmPn3WgPpa3IFYcuS8YUr0avcppSkzDzRny/igPk2sT1aQoK1HMHcHmD4Wi3pN3Qq8eFt5VgB6y76mmq0QdocCSf8VarF0MobUsPihq+HPSvHgqFMEodEbqBT7xHO069EqtCcR35dhVHX0Tb4SZhRbXH1OG0G1QDNc6EiAh0keAnS0qdGOt9C4s09H/AOW01RT9+yc/+pbXVwzUz/gsXh7slUz4q6mAbxTcmO8O3tngUt5061E6nKHUz5aT9dvRhe8KHM280uqcQQWwAoGU986ag93j+NvR65q9N63LQXmmCmspWXxB3zpCp+NszQOvcLUYg21ijckAgEjxto5QZJmDbWg1PPkNtraVqdPGfO1oFVFaJMQmLagCBJ13tijpp7JtwSB6vMd9pGppSBiSnz0z6EjVbagPONLJLbwdbQ6lI7QCvhNnJejeZrMNdLNC7yPQ22809US2PJJI+y1lAbtVbM2zkcSsRqQR4W2VJTy37hYILE76zO9tKcIgyCeWtiFAdEKXDOg1nnsbcJUQRl2EaWDLg/S5d33WxKyTrpyHh52cEwoxmEidPGyhdi5dBMg+VkoOamfKx+7V/ShROswZ7osq5OXN4fE2yxbrydQtXvi2WXRcvDanQeu6xpRC+rAif0ZVMfD2+Vku8q12nvBQZcUCokJA0EDeJPIaD3zY8oIaqCnMlThSkTmJP6WgHgOfxE2Q7061NWS22SVahE6ajc/O87zapabvRLtGJycAnG1YsvJxSsxXSSOzmM5h794Pn52cXH25aO77uu+8mnH+vrKlwLSpeZI7Osd2w0HxsV6O1G43f19OEfQimQkqI5qVMa68vqspdJWpihuBAJTLrylAAzICdRy52Ae932iGjb/wrBjWmhLioSoVLaq2HE9ZopBUYgDw8fw0s9VXkWVpbWVpSQFpAOUK0Gh5kD7POzFpXXHXAylYUCCkyJA5j59nmsNorKgpyrygnUExJ8O/flrr7DclVFro/WPl9QqI7bg1KSUpkbDXzPzrYqrUAt9pEZkyYAgHl7tf22Ebco2aVbTjznXJbyoUWxEk6HUydhYipS0qUpcyEhMAzJiB9+nID2MunhtkYPWIu1wpcWW1Og5SIG2seO0d40sWbYp0tAs1ZWtJSB2DB1BkTsQZ8Iix9mpbRcN4tKSQ4HWciR3AkGLOfglhm5sWYiraa+6Rb1O3R9altLikDMFpGuUzse8g725zsrblOjZncGhE8EUi/wB8bDBRlUE3pTKCkuT/AHxM67g6kwd9fC3pbTIIfyA/pEEDlPj8/Zaq118OcI3HUt3hdFz9U8wtLrai8tRzjURKjsRNpIpsc4oVUGb1WQpclQSka8+UefnbN4tSurnNLDay1mDVDaBrmvF7qEOlO4lni/fCCs5XGrtIREBQhIJnTvn2cufrhc6j+baU8yyief6It4xdIW866u4mJqa2q9IcUulSok6qAQIHn3edvZi5HM910ahAmnbJA8UCxrYzHAxp5BU8sglqJHjmUfVrI01+FgV9keHdNhZ0gCfCwDhkajfTW0sahkRSoUYI5jT599kupUPsiylUTlJ3ERZKqFDN2iRGm5sbHoEE7dJ96KPoT6QD6h+q0PodJSEzyzDXlaUcR16aKgUtSAqVpRBgbkC0TSoH6MRoTt8+Nh6o6hH4fsUdcccQcxMyIkjcHmPnewjLzimgAsnQRB0J5D6/dYFplxSA6dErEAzpp3WXnbhTR3NSXubxu90VKij0Zp0rqGoG60kQBvsTuLCFwG6tW2SahzKiCvNzBzbfhtYO9Q47dtZlMDqHDrJjsWPCmQSomdR3Ax3fbbmtQ25QVbaFZR1KwT7D3fttGXJ5GiLXKowuNUgJUSTrqLOS5kg3tTqR2AlRcIJnQbiffZsXAtsNRmzfRt68jIPussMuuMlL7SiFJmDOoO1oXOUrBcI6w83QY4vp4QhJbaVPcM7mv1WdNViGjp7vZr6d1CmU00rSkTBypH2C0Y3q4pV9OPOkqU5TIzK8QSDPv+uyklR/ck8JhXoyvGIExbM4/wDzQcOqNw+IGQgqSq/Hl20artRU0j77hUpEpUEpyuMqSQZ15C0ccPsc4gdua86FVcrq6a9qppogSoJBB3InmfnSwmIFqULodTAipZKj7FD7vdZr8PVZP3RU6yqU3y8QCd5Sg/bbOtleaZwujXQMZINE+Xa+pqT/AHS+tQmYUr26Dlbundhh0hBAzIOh2Otk1pZUCEBJMkbGAdLGqdQFLUiTHYMEaDtR7d7VMt3DVWENgdFGXEZSRfLq3ARCmygbg9n5+NmXeQebVWF9stn0q6nnNNRJdRKeXd7/ACl0cTSo30SkAJLSdzoIO/7bMi+iDcFU9AcOW7llSVDWKlYG/dmt6ZgxAoWLH4k0mqeUo1TN/LvKoRclW00tzqysOKTBzBYB7iYn5Nkvg2uqo8S0FNWOHrGnKtgAEGE517RykfG2iKtusWsU6lgspKpRplCjB7tifvPIrwvrC5jJlaWsv+6D7Z1kZtZ9n32s6mRr6SRngfwVdFE5tQx56hTlVfSYCr2o7LdRUgjeNVR9R37rNu/4NMlSUpgpnMTvpsdPLu5+dl9DnW4Tv5gKB6usdlJERKT949/lZqXpUpXd7C4S4VNIIk7QBr7vdbzalbaQ/FbOc+iiN1urRUNp6zY7cu42FxcM11GodKQDCVAaEidh7rJtGtPpkpGYqA22BnmDuLKONHcmH5y5lJkjKIgcvLvtqaBvpWKpZichKYbh/wB3KdCEKBKGkxGxyDf8bcOODMmQFSAJmNtee9tsuJOILscC5LzAEKEggBSeX80W24hKFEkwZ5mTvzi2nYLAKn5aoAp11IBVqSSY37+e/wBlmlj1gO3VShAKlGrKSBpIIUde7X6x32dqkKhOwgTqkz8+NkbElEiqo0whOVFZKhv6yY38SU+6xcRs4KCQAhM6vebpa+lp21xlbCiN9SO/vAifPxm0QY8LreI3k5dClCs0GcolI0BkbHwju1mRr9qVuYrUgBK0tHLJBgAAbe2zB4k04N+z+kWEKCU5gqFFeuYCORkA/gdJ6qDbupH6Hl2YavHpD4OVid1pNBRmqvL+6HEtoRUU7DjzGadRDrTZiYMATrFr+Xayz+6m+nm32nmTiBtaVtrBSQ7lnWI/+0kRPmbecfR/pk12LKN89hyjo6hYSoxmGXIN9566ZA5C12sC4r69V4UK8qFU9RRVSSlPqj6BGx2/i9R8nOVtE41TasO2FrfNW9JO0MMRG6k28KfqMq209sKlRUokT4Tsd/ndYw88pd3DtHMlSwokAazO3fqN/Gxe/aVw0KnEFKYAVPt5a+Ph8LBYYePUONlRCkvK0nvA127/AJ0m19QyZgqyvj4ZTe6QeEjjDhdezLDWaru1k19MOZLfaIka6p5c4i1U7sq3LzuxqqCkuO5EoWh0DtEAjyG3LbytetS21NkKKS2qJS4AQQRJBHzobUkxPhRzh7ja+MLk5EUb3W0bhRJXTOdtoH9Y5YB/lJMaWsgq4bJBXedBTuOU9XdymAFlLiurKg33jTUD74sj1NDeFLW0lZccuXe2s1LLyFgllcgxBM5ZEx4bWdZfpK/M31jPpQ1Q2DmkjeCdwZ+dLNS88NKonlVLJcpQogK6kwnwMeI0iLMlZnCljdkNyre39ebeI8C3NfrSk/3SGngQrQ5kGfeRZmhhSUlIbVooeInme7l9fdbrhVfDd9cHm7uUkqfuipSy7KogZ5BEeCj86WOusIJ2OWBKSNRz8402thca/R1PyW+wH06XTqk9xrMnIkaCNSYny7tZnb7bcBkpOsyZM8pGm/j7LKSqQ9uEkydQB5z8nu8LYmmIJbVCcwJEk7z3cvwtXNkVu6O6TiyoGQ2AQNiJ5n59m3e0K+7PS76xVc7rQKbxucEAHU9hTZ8uX4WfnVEyMqc4Op1meXt1s26xo0/EKjJACa263Wjm/kLBgctlT+FpBJY3UMkWi80cUMlotEhalpK0ElMAn5nz99m4kjNqqz+4o3au7L9vW7yEgU15PsFPMBK1JG093M2YBP7Lb+ldmiBXm1U3JKQrW9AO8AnFmKro1ipu9ip74LbuWY/62121s52wQAZBHKN9fhbyg4f8QL44eV9Re1xXneNDVvs9QXaKoDKi2SCUklKtJSk7crPWp6S/ESogO4nxOpEGAb9dSTJ/kJT8LUuI4RLV1BlYdFoMLxuKjphE8ahejNTTLzlZTCp1Cvnv+dbFH2ep+lXlSDuSMoJnTW3mtUccsaVSy49eV5umd3b4rFf/ADRZHreJF814iqpad7WQX3Hno/x3FWHbgEvNyJf2lh5MXpXU3pcdKkKfvmhp9IJdqkJA9502slVeOsA0YKKvGtwsRKSF3kynbT9bv+e7zrq8b3eu42Kajw+yzeiVy9VKQwttSddA2WpHLdR1slnGmIJlt+nb1nsUjKf9W0rez55uUD+0gGzF6IP8W+FTOVTvEK4cwHaAr21R7EmbMbitxS4W39gC/wC4KHGlFU1VbdjzdOyyhayt3IShIhO8xvprvakoxligIyIvqoQAZ7Csuvs87BLxTiReqr+r9dP+EL++xEWBtjeH5tv46IaXtC6RhZl3Ra9EFusWClQmFQoa7WeHBiuRd/FnDNToEuV7TRk/8Z2Dt/Osx3nnqhwuvurcWrdS1Ek+02UrivFd0X7dV7oV2qOoZfGkQULn7LXUrM0RZ4Khhflma/xC9GXabMhSknWIkjQH2fO9rwcFr2TfHCzDtR1pUWqMUh75aJb1/wAW1NzRIeGdKwpJSCFJ/VOoPx8tbWk6NNYp/hwqiXvRXg+2ANgFZXPrWq2Iw+T9MWreYlF+hDlK8gyqdZg20uByOn1W0VgGe48u+3J0HPxNr4GyzxCwkc7cz2tVfjbZidJMTYMkphWlpGlMKDqU9Y0ZEA+2bMRJ6msrqeMuV/MBMCFAH65s/jPLszZi3yg0uIXkbB9hCxp+qVAn4ixtOdbISpbpdclYny2i2FZGo/SOm/z8mxcqhREdqefv3trrCVHsnWLHjQKuJPNGAuTE85tsKmQZHnYuHPWJMgH77Z1hywI158hZAuRlLvaidjzEc7HbuUgLCpBnSLJWeCTpsSZP22O3e4S5M+OtnLk4gWo1Qj/FNssT6481GfP8LZbly8SW0ocdSRkhKUSDzWRrpp5ye+zavnsVAClpMqUQcpjz89TMd1nrcTzQqXEvXMiqJaQG80qyEI0HZ1G8a91mbfAYfvhXV5G1Z4HWHKASZ5jbWJj6rVTPXRTh6CKUF93hdSXhd9Y42VpTmAVAlJnNpvzPsFit9Ykvi/ij86Xm/UBknq0OOqUEg7xy5Dayn+ZxT3a7XuvJdORRhICkZQOXw1/bZOw9RsV1epurQspSk9lJAnw10jl7T5WIs0nNZQ3cBlugLuKEuHrEgqIlJTAjz9nug7WWUqSoQmobSEJUTI185jWyZRJZYvfqbwSC0nrMySdCAFEbb8vhrYo8+4QQtRzDlyHfoPw5b2cRdcEdTXFwlpCiUplMDYb67dwPnYf833pUIVUsUzixoc4SSdBzIHz3mxDIgXep0MEZjCees6xyAkQO/Tzst4Pqn2atLPpI6t9pfY9Y9kEgwfL42ZbonckJRXZUi5L1cqqVdO6gNBC3AdZOo7jtPOz86MSKhXFu6LoLQfZvhwUVS2SqchWJywdDIHL7rINWxe1dRuUUoQpttKG1mEhJT3QJ9+sDTxkzoiXVV3Hx+wnUVi2i2m8GVKCCSQAqSNjvA+/nYLFHmOjkcDawOqMw4f8AFR6X1GnVXXvLgVdl2vhmqu68KFwoCi08SglJOh1AMb2RDwZp67FOHMN3M76G9fNUaQOvFS0oVlkE6eBtbDH7lHjDEZvO786mQyhtJUgjbfT22iXiYmrwNX4UxbSICnbtvZL6WjoHIQolM+IBHPe3jdDjVS7FRTtmL2XIBvoV6jKaWfDcxiayUjpsV56dNLhxefCzjrVYOvGqYrX6Q0LwcZSooWlxlKhE6jcgg7/X604fUPzNQkCf7ma5z+iLeU/T54hL4m8fa3Fa7tTd6Q1RsBietVlQ0BJJAAJ18reqWG1RcN3yFf8ABWd+7qxvb14nNE0rzINc17g7fmlZ1YOg+HdYFRHiDblbhmOU99gVrJJExZzEjxqualXYKR3TqO6yPUqIVAEFPwsovLTBnTUeBsl1RAzEDWxsYQL90hYgp/SqIgECHEKlR00INovYShUZxAI3y+Wvz8bSNies9HpUAtpUFvIQAeQJGvz4Wj2lS604HS3mkaHeR+2wlYQ0i6sMNBc02R2nIypZSCDnAzE66/IseYpnxUopUlBW4pOX6UACeXcB8PYLEmCtaisgI2AHKPbY3ROBpZAeBgnSRz8rAueLbq1a110p3/cd7YavFy7Lw6pxxKULCmXA4gpUkKSQR5/OhslVDb9Uy6lDapLahHjBsdFXUONqYYZUVHaEkx862FF03o8ypbNJVZlIIJSyo7jwFoONHzIU4Y62qRcPXe8hhpKmy2Cy3J8h+P1WddyYfo7xeep6+/GLsQ2yVpcdbKw4qRCQBsfPu2Nke4MN32/TU7q6avP0CBm9HVqYH42Muqu273HEVF50bK2yUrS9WNoUDMEEKMgzPlaMyNdoCnta4N10SJeim27zbGeCWVAEaSAr8RYdqHcNVDajJ6l0TzGhHnZIv3EFx0V8s1Dt/XV1DNO6l1xuuZWEaoIByqJk6kJ0JgwDBNi/75HC9rDlauo4j3IzVhp5TdIp3MtStcoIB5mOXO1PjEL5YBkBOvRF0UrI5DncAnDfzpVdl0PBKx9JSk/4w1s28EBxV6YmpxIIvPONCMsoT7dvu3s6r+KFYauOsuWoTWMVNNSPodCCmUKAIOp5fbZnUl43tdt832buNOpxypQpZWQBGSPstRU1IZIXNP8AGqPmkGcFPNLbyATkUURJncftmxyioXqikrnFLyltlLgSNf74kf61oqxzijiHhF1DdY+wgPoDraWktKOXuIOu0d9m9cnHK/LkcrqzEb662lcpVNBllTCVZ5SQYCp5Wl/k9PJHnZYoT7WiifkdonRj6hccvincWNVNpAkanX9lmfir/cu6rzy5SUUVO4FHSQirQf8AW9x2slX1x6um87ybqarD1Z1CUdXkFS2hZXOm6SI1Pfy9ojuIaPGNwVNdTUTtIzUXdXsBsupdVmbcYVOYAAjWdu7W2poaKSnp2skFrKoqaqKeVxYbrMU4+Vcd/pokhSxeDCaWARELgjflE/Isk8Nz1V/PPOthLhvw5Ep1KUkJKjM6et7ps40t3ShxhBb6zMyVfSLSDPZ1khX1WSMKu0lFiK83HW0MqNelSFrqUpiW0GZKII8dpEa2suA3hOFuSB4zjINdLqX6eqQ3dmImHHEIPpKFJzEJ9ZA2nQ/fZrutuVd0MJp1NqX1CElOdI7UCRvy+02Wqq+8DUzF401Re10JZfeU4kVF4Ik5mwkwRlnnsLI6OJOALruulo1YluGmIQEIWmtJUY7pXB0EzbEx4fKHktaVqHVERaLuHmkekoa5FeCtATPrEyd99uUd02cOJbpdq7kWypaCFJHeD8dt9POzfqOK2B3VhZxlTOtmV5krUtMJ9bVI7hOnIcrGb04w3BUUbV0UV/LV1gShKepUkKOh5DUGY18bXdLTzNcDlKrZpYA0jME1HaUNXXKghbiElCXAiSlPWK0nceXjYJ2mqXjkRR1C9zCEKPMeH2+6xa+MTVN3XctxqtqmEPFzRtahMEE6Aj9bSfHxs3q7GlYimL4TeNUkJJS2grcWuBshOeSSTsLX7Q7YKoLmW1Tr/NN4LRpc9WMoBkU6tPEEeM/DfSxaquyuRTPuP0i8rGVcwJRynXnI5idfGxC9sN8Zbuoae83+DGInKauqvRGSX6MEu5FKhQ68lACUHtKyjVP6ycxOiuLi917z1XwtboKBghVXU1GI7szMtxJUGkvFaz2jCUgkkQJOliY4Zr3IQ7porWBUR1jpXiapSXUntEFOkgZo33+dtrNvHSC7fjalOhBFEzm7olWUwQRvOv1xZWp6kG+L1eK5TnUnLOuXUxOoAgT377aApOLVk34lxDZCBTsIJyjsygkbA6T5eZ0sbLoxCRm7k4Oj9CceN0RqG6cKp6oKU7IiAT2oB1OXu3PnFmcNV1PR4jrWUXiyVu0CSgpLmVWQOqiSBBBSDrHq++rnDKoNLjuneYfCC6y+UkhIn6PeTI2J200B11tNOGbyW3j662nHQsV1O8wAQR2lJUBrPMKEeXKbCGPO25Uwl4TrK3t8YzK2xdKKVOQNpQXCTmIyp1j7Y5e4TCNUhxdQpIAUhSSTEDURPz9UWYpqkPJpagGespmVzmAmUJ089z8mzkwZWIC6psIykpBmDlJn8TtysfBE2P1UFPUPmPpJ9oeMZzIO0gwBvzPd9gtXvpS3Kmnrrgxi1lQHELu2pUE8hLjRJHIZnRr+t7529JSJCF6lWsj2/fFo74+3Z+eeGd5paRLlEpFagp1goOoie4/PIpQDdVLva7WKtYVTOKpaiNFIntnn9W/PUaWTxf2MsMtf7ssN3zd2yloMVDaNpiYXt57xMWcNC3TVBS062OtBKk8iR3ePnvYW87vfVTejiidfSvtJypzECOfMxMafiVHgnJ/cCcS3bUVN4XZRVIXR3xSdc0UmSlxCk6EHYkQAD3d82lwXctQzkkaZpUDO3j5j8bUxwfXXrg/FgvKlcUyuiqMzrEKCVoUdT7idTt7bSrfXT94eYfq6i7HsEYhdq2JDqVLpWk5ucEuKPlIm2Mx+iqaqdroG30W17PV9NSwOZO62qnsXcQMgTBBidx7PYPhys0Md4hewxU0yGKYLLrZWFKAAMEaeJgWg6r/KS3a0Vfm3hdI5dffISZ8kMLnfvsy8W9Pq8cQvNu/vfYdlpKkpLtQ+6YJGklpMbDlapiwXEiblmnxCuJcdw4Cwk+hU2r4o3p13Vs0bR0BAMwN/CCbA02Lr2vnE9z1DFHTGoQh9tsu1KadErj1nHISNBz5mPOttf01r/qUgDAWFyUg5c9M4vKTM7qHybJ7nTOxhnStnBeEUwoHW7Fbd38YLFjBa92haB80C7HaH3yfkmjxycVUY5xK4UoQ47ebzrjaF50pUtcwDzGuhGmgtFKwUmCInWIs58cY/rce4grMRXrRU1O9WlBWzRthplOVIT2U6xOUTruTZsOrK1kpmJ08rbKkidDC1j9wFiayVsszns2uuJJP4WyTbYHgbdIZdX6jaj5CbFXAQoBOyD5RNssq02FsS1xT6Fh68qjNt1VK4ufcLLdLwd4sVyeso+GmKHk96LpfI9+WzTKwbkJ4hkds0pnm2WkVno78bHgCeG98sz/5Qz1P9ciytTdFTjfUkZsMUNOO+ovygaj2Ken4WidV07d3jzCkbRVDtmHyUSWy050vQ74qvLQmrvHClHm5u320uP6LMftsosdDTFaoVU8SMEoRBJ6t+sdVp4Cng+w2iOJUg3kCmGGVbv+WVXuwgJ6tJ7lHX3Wnm9uiPfNE0n83Y9ueveKgFJFJVoQEketm6sknbTL391ku+ejjiGnp6QUV6JrHgkofSijcQG4JykKVGYHYQJEagWQYlSu2euOHVTN2KYLk6Y2CqLDd20NXhm9nqpikZZeWKhhIU4hCQpXrzqQSJH1Wt50BuPVz8X28aXTd91PUKroVQ1EOvhwuB0OpJEARHVpG53FvNhno040edWhddQMIB7BdLmZQ5GEpIHLn32tn0AcNq4B8R76q8WYruv8239dPoysgdBS+h1Km4lGpgrEeNqk0+HxOL4j6St21OISgRyj0V6XECCCY7vK3BEkFU2baOJODHBrfraY3lpwf6tuzxAwhl1vtpMSPVX/ZtICEwgpfMkZY08baUdSSJE66WbyuIeDdUnEFPI30UD8RbQ4gYOXBF/wBLqRyOvwtK0hMsl8ydCR42ZmNAmnve7akiOuQ6xPsSqf8ANNj54i4LUcoxDRzERJnbys0OIHEnAy7vo6hvFd2yzWtpkvBIAJKDqdoJ35WnhdZ4UEzbsQ5dTM5u/fYm2lPDLIEACeevj9tmeviXgKexjG51cz1dWhf1HWwC+K+Bmxl/PoWBKj1NM85pMaZUmbWgcLKrLTdPYuqymCJOmom2B1e/vmbR+7xiwihOdBvR0EZh1d3VEEDzQBHtsTf413MElTGHMSVZGuVqiSCQTyzLH1W7O3quyFSYh2SVZjrpAH32N0Lqs++k7gzaHl8asoStjBF9pSSdX1sNZTPPtnnYJPHGvQ6ltjDVK0qSCKi9m0qJjuSlR+fO3cRvVJlKnr0hX/FzbLQOekJVT27ooUq5j84K0P8ARWyy8RiXIV5xYbvemuyucVWVDbchqEzvlSnWRyA1s3sSJuxyqffRdiFE5nJ65XaUVHUDuMj38pssXTcTF9dcupcqEuN9WB1YColsEz46fE72MuYapC+vrlrcCUw4SBrproNPnxtXNaAbhTuJLbJnUrwVhZ1fVqbydahISnSYOniYPunyshXRU1LDjjlMklcphQBOn1/PlaWUYXu9V2Jo2WVnrVLXlRmTMgCfh7J8LN7DGH6Zq9HkdSttxKCCHASJCtToYO/hpZ+cBN4ZTFaQ/WV+iFF1ZIG4JVOnOe422W6hgLbeTkXJSoHTtDefH8dLSDS3JRHFakmlbUOtKiVHRWgUIGbw233ixCuoqNqtcQ9SMlzrSk5kgkqPcCfH7udkMqURpltvVfVAOqEZZE66z7eZ38Pecw+av850qWnAEpXJI1ERrp9vlZzpbbZSlDNNTwkAkhpAEjWJOv3Qe+zgue93ruUpR9FWlz1iQUqBhUp08APPflqzjeCkEQO5RJi+mHCWjTPkuK1KWCR7I8AOR3tJHBXGN0YL4k4bxfejVc9R01Sah5LNKVOZRoQBoDvMb91kq7K+56x0F11xLitCCspze4zy2NrB8D+IF8XZiG6rhw/RXDSl11CBVOXQy46Uk7dYoFXM89xaoxioeKV7ctwQb620VxhVKHztIdY3FvivSLhZct243wjdeMKKpW1S3rTIqWW3mcriUKEgKE6HwsPxR4CMcQ7mo7tRfKKEUtR6Qpw0/WZgEEQBmHMjnYbhticM3KyqpTX1tUUw46imORZ7wE9keyzyfxfUdSQ1h6vfCkkEJyII03OdQthuz8OBCBskjbSctypMQqMQiq3AO2Ph+1eGXTSuL9zPFmopkPl9KxTqDiUb6KA3M8p9229vVzDbqRcN3AaH0Nj/AOGm3lT08LwQ/wAbqgM0hpUhmlyoWtJUgJSocpjY+3zt6lYddT+YLrWFjWhp9PHq087bmiL5KKJ79yEPW5WVcgG10vF47lQ0PKwC3SRvy27tLAddEqKyCbBKfSkEgmZsbGEE86XW3VKIJJED42TapcyDrpEgiwj1UdQD8fv9liD9UlQVCiDz1jWx0aBeU18bOqRQICSrOX2gIP8ALB+MG0eL4rUFKlTPp+Hm1g9WEmgpioKH84+Zs/MXqL1AjKn1Hm1iPBXw5WqlUcNsbVN7VjzF53Qyy7UuKb62sIOQqOmVKT4e0WCr6SKptxUbh1TJBfIFNZ42sXchWbEVAwEJ9ZqkpURH81BsgVPSeqhdZvugqsY11CS0EVFHcVWlCy6UdUEqFOlJKutbygEzmEaWYjXBfENc3krcT3AyTEkOvEbwf7yJnazjvHh/xJvbA1XgO8eMlz0VJWsNNOVF23M514KF5syVF1IBJzkgACXFQAAjKBFhmHg+krGTEKy3ohC1HSNx3UuFFPw94sPGJzOULlOnKZEguLRpoe7ayc7xs43XotTd0cI76cIAbAr8VUdOpSlaaoU+SRBB0nexH946/wBxSnKrpMYmQtZK1GhuanpjqnKTKFAiRv77G2Oj5h+sR/u5x44nV5SMsm91NpA5pCc22gMeFixRYZH7KHNZiDudvJLOGH+kViLCv7oLruPBdO0w8u7lJqcQu1pLzagkyphst6r0PbkZVAicgXXl2872vi8K68quq6ld5PmqcZYfaWhC1SVBKoUCBMSDqOZmbWPu3gVwyuW5Rc9LjvG71Ip12oUyLxaCVuuQpxa/ojnUopSSTJMAm2UvAzgld7QbapL3UgDKlLt4BIjmE5EJH7LPbLRxfzTCo3sqpv51ygejeeRQXm2l9xS81M5nJQZV2xIyp9+ns74+vllx29X6hx9LbxT1awXCDyPJOh18du6bXd4f4C6ONNiuvp8ZXe61c4u5Dqesq6twqfS6IB6tROxMcvOz8fuDoA0z66l3CDVY6O0oqavBydO5ao25R3+Ni4mNkbnuB8UDMXsdlIvZHOitgnD/ABS6Ptx11/3umnvi7GzQ1AYfLzTaUwpnfchtbcmYJBiQQbSTUdGvh66+p2nxZeVM89BdW062SsxEwpJA8rMTCfGjomcLKB27eHuDKu6aSpc6x1ihu5LaHFgesczgk67+JsdrOmFwgZTno8F3s/pIK22Ezr/OJtn58BvMXRvAB+Kso8TmyAHN9Fxf3QiwRiivFVfHFrESkIBS0htFIkpB1jMWzOsnayJU/k8+FnVpSzxGxQ6ZUCo1NGkgGYgCmMxPPuHtMv8ATcwa2qLu4d1KgEyCusQn6kGye904QsEUGAGG+QLtapQ+CRYhmFvY3LxrfAId073uzZbnxslC+PyefCr83NKocWX6KtLYKnFP05BUI/RDQOoJG9kDG3RburB1/YT4fYHqa+qo79oL4Ydr6xPWop31tNlOfq0gJEN7aeqbbZ6auI6usRSruC5qJtaoLi0vOhOmg9dOvnG9ka8elZjS+L8fLDd1Idw8H66mcbpjCyKdxKpClEHRe072l7iGnMZnH5LmSyg+qFql/J34kYpmnkcUbpC8mqV0LwyyNR65121smJ6C18XZTXhdruH8N33WvU7zTd5OZp65eYh2VpCgUk7D9YiYSkWGvjpg8SF4XprzuvGNGLxcfLLlKm7GAR/KgpJiNZmNY0MSzqnpV8d6tyRjV1pJ1huip28x9iPAj51NZCGbvJ+QUF3u3A+qMI6EPF80zFGjAWAEmnJUh12paKFFQ7cpNMo6nUSTGokzZaY6F/HBw05/N3C2kDKEt5cr/ZSkJiOrZTGoJIECzIqeP/HSs6wrx1iASIhtwoAPhlA7jZKe4p8YrxJFRjHErgKZymvePhOh20+Np28IciVHkkJ3AU5XL0POMlJci7or8bYPSF1S6pTjVHUOFOZQUEpnLIEAdrMSAkH1UwBQ9BB671M1F68UqVBpykgJu2ASPN3mPLe1fK/F+N31f7oYkvBQA1668FwdOeYx42It41oqZ6b5xVQNZEyEvXo0DmjScypjQeOtue9lvU+qcyI83K3CeidgarU3RXlxIC0ICgG2aenJAIE6KK42HL7rae6IvCSmpyKziDWtrElFQxT07LqCdlBUGFDkY5C1dMN4lbTTIvK78V3C626jOgU19Urjzic0CGkuFyJ11T99lml4juVbZQKhS+rJHNMwddSQCPLfflbP1lHVzTZ4X5G9FbU74WsyyG5+SnPGfB7h1ii7Ka5sRcdcd1FLQVrV40yaKvo6dxupQFZVhTVNnSRmJkKnbuFo+vTgRwHp2H6g4p4j3vUsIW4x6diuqca60SpMp7IV2tSCCDPjZpnHt3sNnrjVJcAGvViNJ9aFGPjYgriLQh3qutaWTplW8WyN+9BjTUjlFlipKwEF0zin8SlaLBgVS6BCWL1rmi+tTqXDn0kqVOwA29tknE1UipxA7ROqhtymbUFbkEJnTu3PtG0QbKd+lq5sbV9OWZDrqozKnMTIAj6vDlJktnHjdc/ejNRRuHrfQUPnXKEkKWInv021jflpo5W3bZUsZAcpU6Pb930PEiiN5XfTVb6GaoNIqmEuoVmZWpRyKBBiNPMbGLWLve+UprKR9i6brp26epQoopqdtkk6giQnaNPDu0i1NuGWK6peJrtqqVLLdc1UBpQfzKbkhQMpSJI1+B0Exaf6+/8AE9Y2KdmuuVJH0mX0RxAV+jBVBI1IO36GlhBESbogygCym648UhN30zDyurNPTpQDnUUqygDSE9w5/Cy1dePaGnqyh5bjiUtnMZGhkGJkEnbYfC1c6bGeMqJCBV3IirA7BNG8lWURI9bJpMTt5WULqx1VsuPfni4r7bRkBQlNAt0kgg7IzQqD3x52dxJWbFNbHC/cKyD/ABSuJGZKEvEgSkZCI07u7x+qbJV/cSrvvC7q27xdzzqHmltn6TZEactRpFoRONal5KvRsE4pezjs5aFCJnQAFa0x4jloNOQLuLcR5JY4a4lUA4RmcNGkK8f4/nI5ad1u7xKU8U8I2TPrHKuiUUUDSutZ+ibCU5usImAQOcT9nMWHouIVdSU/5vvu7F0r7gKEPAKUiSJBmSY2jXTxto1lW3eKk19zuXa+9Ck0ry0qVB0IBQSD3d+nKyqw0auiFNU3e3lIypSv1iDoZHhpMcvbayYS5uiAeMrrKN2F1NLid5yvqzVO1DQcLidBHKI3EjzHPcWZmNODN+45xCh7DLlC2vqpqOvX1KU7QZgzv+AtI+OPzDcTlHS0PWF9ALiZXmyIB31HZnTw0EbwD+Bqp8XywhtxpPpIBIVISIkZTG258+/uBqJHRC7N0bAxkpDX7KHKbolY+djrL9uBtRMZfSXFEDXUwjQaWU6XoeYlcITWYtu1slWUhppbkRvvGotb1rCtUtHWrrWUhQzSElQjSDyOumuvOfEz+5anK4U8o6zKWlac9JPf5fpG1OcTqid/orgYZS72VTqfocshKVVeO3SNSequ4beEufMHusosdELDAbAcxFfLqyTJSw0jbkBJtaZGGmEJzqedJCgogJTJJiBz+QO6LcJuRCAQpDnbkxMFOwA2jX53sw19UfaTvs+lGzVVpvonYZaWVGsvd5IPqqLYkf4P32UKTo/Yap+qDuGadwMagrcWcxgesf0vLb32s03ctKFCaVQUnQkuc9/t+dbGm7loElQLQ9bQlJVMHXz8Pvt3e6h27ilFFTt2AUJXbgXDLBTODLgToAMtz06lCOUloz9Zs+rooblpwGkNmlHJLDSGgPIIQCdvj5w+PzPStrA6sAZZgJgqP3/Oth2brYBObqwDzgAx5fO8WFlMkm5KJiYxmyb7DVyHQ1l7Oa6RWLjTloO76j7TS7pw48AX6SvdO8qUpemmkk693L22WhRUxIW29l5SOUaR3fOtusjCAVdag9qDmMADx+edhHQOKNbM0BIacO4UTDf5kkKEkdQlMDURqBYw3ctzNx1N0NtpMAnIBp5ef1crKHXULaCRWMpkyoJygq848O6wa6q6wCtV4ITzOVOogH7p99kFI52pXd5YEXVR0jKQy1RpSnVKYbCojTvEj58LBOUza4LTB/V9UR7/AGCxhV63cmAKxS0k6ABXZI00nT9vtJd2/aFHZyPEoJmSAY5GTqDraZtE7oonVreqLP3UHQtSGlFSdhpqO/cj7LFHbhdUU5EgHeMxHcfdpt32HViVFOrrG6V1YUObpA222IHmPfYNWL3wjK1TtagQAomDPhE91phQu5BQGtZ1RReE3nPVebbBkBJKtzP4eNtNYBeD6KhNZSHq1AhISoyR4z32EcxZXxmBDZWQEwiZkfHc/bYKoxLeLiZD6xqQJRodvDx+u07aF26gfWNsn5+7LiShWYXndqkp00ZUQkabGY5b+PttpXEfiShCQm87vKwZzdQNTA0nl5WjpV/3kTlXeDpSTmhKoHw7/ssA5eLykz6X1kkQkHcb7z8/CxzIHDcoIzMOwUkDipxJaUkE3OEEz9IyTpM8+72b2EVxvxeyAqqYuqDmMhYTznSU7AH5i0Xu1edSgEgkToTO3mZ2A115WJuqKVaNSUjYGI7vmbTth6qB0o5BSpWcfsRBoJYo6AOEAKVIUJ027O2vjZrX3xjxXel21VEWqTrXEgpz06VshYUNCnIO487MtZeAUgNBIET2o9+nztYIQnMgAKGYaAyN9PhadkQah3yEoyriJj/I0GG7oplCNG6BAAG/PXu/GxR7H3EhSxlvlhhajrkom4g+YsCoA6KUo5pEZjIJkfZYFRSk6uKWknsxp7Pw8PeQNAhje6MHFWOnB9JiysGYf3ptCY75gafjrtYrUXliOoKeuxZf+VEKyNVZbKhEx2fP6rcAugpSFAJkzpt4z5/XbkJdQ4uM0nYpT62ndznXzmyg3SINTtbmLrl7Xy8oHKS7eDqwdOYKiNvrOttUlBRLcU65QFUnNJk8geZ7x+y2lrcKIQVZgY13BmDEW7pXX0OBwrUYI3IObl9ndz9tlTeaV1UtMklPodNoY9a2WD68jTr6YfziqfbbLcnKAsJ0lUn0ltF01D5W4Cnq2lLkBCZVpMiZ0+BsaqbmxE86VM3DX5TCtKR0ydt8uvOyE7ft6BTdPQX5UIaIgU9PUKLYCQNglXly8dLBtUmI72WptqlvWp0I7NO85MAjTLIJ2BE+NguIAbkhTlulrFONq58TMEuru6tSBpC2S2NSf1vZGm3fNkeow/edO6qtp2qRhwuKWQutaSVAmCIJBBj4iJ7lCg4WcQb0fSLt4e4krFBKlfR3W8sZo78u420mY1nazkpejvxuqhmpeEuKYay5s91uNz5ZgJ7vvsw1MQ3ePNKIXuHotKY9VcL7r/pbd80DT5CSsKrmSDBBMZVEjYaa+Nuq25qWuaT6RiG6kPQUrcQtaiobx2EEHbadJ98i0XRR6Rl5pCKfhFfqEInKXkttBOuwzrSY/b3Q57k6FfSVqqQUrnD12jQ72nOvvWmSlcREpC528eR2nWN1bTN3kHmE5tNM42yFQVU4Xu1hvqnsWUazGdfVtvGQQCNOrgQJ05GPME3aW5acZRfVQ5l7Ki3THcKmdYjn87WcX+T06Qt5qbWtdyspTlJTU3mQf5QSpCFxJJ8tOYss0n5M7ixUECuxphqkbUDIbW8sg+A6uDvvpqBpZn2lRNGsgUgoqk7MVSG6igZbQpNdXEJAygspPPf+MBmYnT42cNzcWr3wretPeN01Vcmpp1hbLnUtZQpJMKhWbN7RA7rWxpvyY19NhKrx4nXehajMM0a1DN/hESJJ7jZMxF+TpuXDFCqvvvjIo9uEMU9zAKXrolOZ5QO3NJFoX4ph8oyOdcHlZTxUVcw5mCx+KhwdPDj7TMlmjxherbCRKuq9GRlmANBTiNT9/dZv4g6Y/HS/0Fm8Ma32+hClZAa8tSdoHUpR49+kbb2ma4OihwkulOa+2L6v9RjOmpqupaUoyQQlpIPMaEkDxss3L0ZOC90ldY3gxx9Di5DdXUuOxG4R3DWDy287CNOEQm8cI8gie74i/Vz9/FU7Y/dlxxxndGHqCkTXX5fFYzRUinahwl1a1FKEqW6owkZjrO0xyFvaS5MN4rpbnoKFdxPFympWWXFF5sAqQgAxKhzFqt8MuFfCzCHEC4MYpwjdNzN3VeLFYutUtbKKdKFA9YVFQSEgA+sItbd/pJcCGD2uLOGSQQYbvBDh9ySTa8onQVzLj0QFVVkc1I+ztSUGMO4nKoN1EK/6Vv7FWxWEcVKRPolPtsX9fgCLFX+lHwKkBnH1PU5hKfRqOpd9oyNnusAvpPcKnAPRKy+6sHUdTcNbHLWVNAd1jOBTN3d9UEZJ3aBqPKwFip4RNC2N9X1H/Utg4WX66Ppb2pG83c2pevvHhZJc6TOEVD+5MK4vf3Ay3WG5/pHE2BX0m7tyhVNw3xYrNOVTvoaAqD4v2cH0rPaHmkENQ/2SlO8OCVXeLIZqsTNiSFdijO41G7neAbVmxxga98GY2vPD9L6ReKaZaFpqSlLYWVoCyN4AGaOZtPT/AEl7zWlJouF1Yoqkf3TfDDX9QLtVDjdw+xtxX4h3rxJau6kuhm9vR2nqFu8FvKQplpKEqKwhAVmCCdhEHnYaplpXgAO1RNLDUxG5aU6kXffpCR/cbS9gldeyg7g81efzscaw7etSgLqL9uZmdCld5IVB5+rPyLRMxwgxzTISlV/PDIMqMtYvQfdt7veotcF8TVNE5Vv3yl1ynA6xv0havo5Pb3O2gIjSQe82DtT+8jCKgn1VI5wuttwCpxVc6Ecsjrq9IP6qPtsJ+ZbhaJcqOId3tSN/R3wYHmkcvna0Xjg088CXq1CFREhJKh7wfH3WR794QXuh5Tl11NfWVIRK2m1lKXBGpQNgrSI5z73h9PtdN4c+9lNTbHDhpKkP8UGlrUIytUwInxCnBYF2+eE9IlSX8ZXm8pB1LVMwQY5R1sz3fsFqzrw6lCwC1UZUqCFguK6xM6QQROhMd86RYw1hdCnVKepMhntZ1SUj9EHvE90RqfGzgIuibaRS5fuO+GjNSHrtxDeCELb6twv0hJVtlRkbSpU7nNtoddRaPbz4sXIla00lHfNUlByhYbpm0uAcwHHkmD4666xBsVp7ju5CpNCgH9MKOWT4EzOhOhHLnpHa7hu5YSaZhvIYzJMFaTPKRoeUHTfURZwczYBMMTyk48Sqd4ktYcvRTiQVQ7VUiBA8UOrjfusXd4mVjiZRhejEQQKi91JymNJyMK+Z2svMXPRpZUttinCgSEqzZVpGkhQ+7TU72FapKJsqbqKehKQe0UNhJA5yBvsNvdZc7RyXcJ3VNRXEHFKzmpMO3EgkyB+cKl72f8HSOYP224Rj3HVSQaVi4EZYV2LvqXlDwhTqDy7rPRNBd5AW31SkIEk5JT4aTpoD7fYAKqkaCkJdU07nURnTIAEawYmfd7rLxG9F3Cd1TJexTxEqkZ03nRsKIgKYuLQpjeVVC/HccrB0t6cUGSaljEdWhTrakL6uhokQlUBSZU2vcE8+Wx5PoobbH0NWVN5so7RI0Vvm5eEzvt3YtN3kOIDLqEoSJUEqUknfcanSN+6d7dnB5JRFbmmMEY6qUrbViS998q8q6NoAn+UinEew2AVcmKdJvW/3AVGVKvd1PdsG1JJ8ItISksrQG0OpcmCTBSkHeZAiB9YNiil0tKqC6WCkdnXLpGp7uZPvt2c8gmmJo5pkP4evd8lLlfX9WYIDt7VygFE+L3s9njYsnh82YdqqejeQkkxUFx/MYOo6xao56c9LSAq8LvLIdfdYWMuilJKTGmxkzInz99uEX5cXWCZQnYFxPZiNSSJHL267aWXO7ku4bBzTGRw1p1EH0G5BK9k3dT5iNdNUmDPjpPlYdrBaaAksIp2ERKizRtInXllSIj2naN7PE3vQq7Tb5dBUDBToNDrJAkydPVHxkmu+Wm1JUgVaFH6Ptagxz3gQAddN++yhzyF2ViQm8NvUwCVXxXyqJbbqD2p5CCDOpER99g14USSn0i8LxUF/SfSVKviZ7x3cxtAFlZd+1EdclIfPqySUrOugEyCdtdhHfrYH88Ela101Qytoa5V5ikjnzgQDzPdpzUZimEsCSncF0SnEreS4pYO+dR7R8NYI0Mc9e6LabwrQthaE0gSe0SkpJBSCCTuZ15gEDWykq83nELUhDbzZSWwnX1jIBkc9Ph4WLqq6xpstsllaspUlKmzodAdzsDHcNt9bODXLszRsmBxMYNNeTFYw0gZ0JGYQkQOXvkwDp5bJF9V6iLoq0OKyLYUypU6aLE76z2/gZ8XbjBp2+GaQIT1im1qQ4pCCmDlBnbcgxHyU/wDe2r7+u/LRVCGXGH0qAI01EEeWg7hpsDaYtzNQ+fK5NPA/V3HxPuitcpkv0/pCHylwlsrSnN3+HeOY77Wro+IuEUDPV3U5SlYBSfR0OIMzsEqKtgdxrOkG1VsONOu35d1U0tTVRQKdSspVlVBbV3GTqBAHInyD1cqrxceWmmU6vISvL15KwkQASCJ30gaTqJBJA5i4inbNw9lZ+68c4Oq0JDOIaCnaMjK8g00ERJl1I8dRpOvjZy0oobyAdoK+nqElMlLfVugGCJlO433GvidqYvP1KmC5VupbQ8BkDyAgEJ0HZKYIA94POBl03TLVORIU46VdW4zAUDCdRoCY7J07uekROoATcOUza+27VdZN1A5JS2VA5VQlSZ1/kjl5jSdNLdKu9AOZypUkKVJCHACAVcp8lD2ad1qgUd4cQWkKcuy+r4HVpGdCa76NGhkZCrfbkRAERNlmn4g8TbrUmnerKFxCnPWqmsoVoR/e0j9bcmSSddUizDREc1IK5p5KZeI2HmKpVPVUr6RUomMykqUCYGngAkfbys0GvSEZUqWpT57KkNokNxvqNNxvHM77WQqPilW3mpDNddlK7mXPXs17SSkFQlRazExzEGeXKbORiobqVlbOUJegSgannAjTmPxtY07cjMpVfO4OfmTGve5alwOXjVtdYpx0rzFGZWVJgakcttN9++3N3VK6F+mqkM5lU6kkkgxmTqRtpz77Ouoo7xvNl76RBowojrFubgTMSNRvZmPph8IJLiUqyAk69olKiO4mBPvtBIy2hRMbuYVmsNYquqruhkKp3FLZRBjINDoFTM7AHnoDvZTViWlObLTOKBOoUUwPOBppGw9/OEuHN9KFMaEujImQlJPamezO/L6ve+fTTBKmkDmFFHZE6GT+zwnewJpGbo4VjraJ1uYoQCU/m5sAjm4JgA6kwNJP3baFjf8AUOnOmnYTGuhJkkbjwknlys3UpdVnIS7AOqkiD7O/lqND4cukKXklp+QCdMugnke8+McvOyClYF3eXlLbl/1pUolpCCISDoY131O5ixdV8XkVQuqKACTAa0ie/v0HuNk1C30plxDagojY6RpqOXzz3ttRqSCktpOvZUlckeXcfxs8U7eiaahyMrvOsXmSp9RCiRA0UPCbArraleVYcqzllOhOn1WC6tYVlOZCSP0kiB3n67dlIH6KlJ2griZBmy8FvRKJndUF6Q+pJU6p9SojtqO/sNgCtbyetkFJAEFU6axzscLLKzk6ojWAUrIA7zqdN7crpiSZSZ0IJ7Q8pHP51soiA5JvFJScsqVAVUws/qqkExO20n53ttLjzaR1ZcIP6yZA28oI8uVjS2VJUPpCZ2Kjorz7/r0sAuncICkp039YTppvGo+7zs8RhMMhQAW4F5g+sZRM5Y9vdHw+zjrnnJIWNdQZjwgjmI+JsZTDUrFMCr1ZP16E/PPa3LimXQApteaRplAPujXTxt3D6JOIUXccKkklBzTE6Ez3/PfblVW7JSpGaAU8oynwG3z52NpbpJBzpSVCQI8ySdbcpaYSQpsz2dRl08NuW/n4WUMTeIgBeb6Up6+nATvO8x4/P1W5XWBIhQVmJGkj368vhvYwpTYWEHq5BOp7uQifOwSinPkQEFUnMrLIJ8YGnss4MTTIiq6qDmAVlSBrGsToY+fKbb9MQWypfKNCQCTA13+edhcqQoBsIAAMZUwABtofZ87cOIUClSHNZ1kz3a/h95s4MATS+2qDU+y4kpyo07JATOnzP4cuV+jLSYiCJkNzv7NPnW3LyXFytK0lBOYHNA7++PnysA7SPhRWkoSsHQg8yDr3zNn5UziIQpZUFJSQDOkKIM7fcNPDnYJaGXUqCZ19UlRMT4T8PfbOpq2ytaqhJgEhSJHs7uR+PnYMpfy5VOI5zrpz1GnzNlXE3Wg00mCVpMCNB7dt/wBvvDKCgGFk5QBlOm86d9hiKo5kFDQUgRJ2mI5b8rArCkJ+lYTEwpOUyffry+FkFgkQK3HMxh0KI8QDtEzG+/hYInIerz6nTbfnMz38p98WMLqGkEIepgEjQHYecAR7osEl5hWWUurlEApSYiO/bu0/bZcwCaWlFnHCAskLUMsEBkgnx3jblYVlfaIKUwNSYM94Ouu9jKWkqADAdCIBJWg9qec+FjlPda1pytIyJ2AUIgk+U/MWbnACXhEouC2oZnGG8x1VpztllhN1Jyj6OnOm+cffbLJxAl4RXpgzgnDFO2lumuK7mUJMpSikQAD5AWPNXPRtJCUtNgDYBsAWy2W8rA5rQZ3LSqNsqgEpHhbS7ta2UtRBFstlpWsB3Uoe4bFAKp2Gj6hV5m2nG2kgKKJ3tlss7KApmuJ3QDyEt6hKSBB1FtFw9XmCEbxtbLZZg0Knbra6Sb4vBygoHq5DaVFtJITt8bQNiH0zEtb6bete4syUpQhISlPkPZbLZaamF7kokbJGbw9SLfQhTjhzkAnTnOtg3boo1KzrCyEgBKQr1R8/GbZbLGhMK5Tc9DVhVM+2XUKISoOnOFbbjnv8LCO3LdFBUu01FdlK0GFloEMpk5Y123kzbLZYiJxA0Khe0HcLplo5T1RS2AkGAnTX9thqhrO6hmY6ttIkDeZJ+u2Wy0wcTzUeUDkglICVBKiVSqN42BV/q2FS029QNqKT2XlpTJkjQSfPS2Wy0rUyyBTSohWo0SFDsj9aPs+NjlFRpdU6wpZy9StWg17KSof1Y9ptlssvNNRV27KZKisgkCZHfqfZy7rao6dlisS9ToCVtSdYUFabEcxEj22y2WeEnNKN54ao6Z9tbCsjb7aXkoCfUCp7M8xI/bzIHDNG+XWlLKSqIWhICkmYzA94tlsspTVGfE3BV1G4l42bKkVbDxp6pEAipCHurCp/RXporXTQ5tCImcbp00yCadJLqdY00k7d2omR32y2WsqXVguq2p0foiLt4U7VOzFHmTUOpRClJOUkFQUZTr6o9uvhYkjED7NI2pLc56jqZURmGm8xrsNPqtlssaAEGXFGW78qX3mWlNt5ylHbKZ1UAdvmbEay/Lwp+oUFpyvgaJSEkBRPPYxlHK2Wy0gaExzigXb/ALzNQ2U1B6xK4CylJ02jbbXn3WFZrbwFWaKpr3XkrbDo1ygSYiB7fh3TbLZaQNFlCXG66e62ndbedqHHvpkNAExAUkayNefwHjIdUp9DzRNU6S7IR2j2EwDB7+fdvbLZZwaE25RVAL1K04mEdYgGBMA66767c/wt01VrQ2zSVEuKdcLedBDcRGpAGvlNstllsF1yjCGGlOlopykoIzNwn1dB9luWqNSdA+qM2gIB2JHPXl32y2WamuJQirtb65KUOrbyiZQADoSYn2ned7FwqO2lbhClgDMqSJVqJAAjwi2Wyy2SXK4S5TIcDTlOVGoIbzBQEAd+hnfw+Nid4XimieWpDS1uFUFS3AZhPZ0jWI562y2WcEiIN3yitUhQoUoXA7RXMb7aCPWO1nNd91pvJoLDiWQDEJaSe8TO5Om5JO1stllKajN7YRVddF+cH69upSusQgIDBQQVTrOYg7jSOXlAjN2CnJQ0+oFpxGUwJ3I+z42y2Wkbsmu3UOfmdqjxIwpDplFUlKyEiVy5BMGRyO4I185sXRcNcLVlR6Ill5ChMkqTljLmgBKUn3kxPstlssIw7okgWWqrAWHLrqksIYqVFO6k1r6ArXmkLg/t77ds4Ww64ytxy6mnQCOy+S9Mn+WTz7otlstJcpoAXKLhwk0pCThagClOJRKGWwNdATKDO/f37SbKj+HqG6aZNUxTUgzZZCKZKeceI2PdNstlmjdPIC5ZWt5tKioygSJMiTptsdhv3aRZpvqWq81ZVBPWqCdEjQEgjYcj3R7LZbLSxbqOXqmnjTiJeVx3mMK0FIh1SkOPGoq3VuDRShAQkpj1eR2jutHy+Irrr1WxeVztOuUhKCundLSXAAD6qguN437++2Wy0cvrKaH1E4OGnEs3lfv5qYuY04DbausNTmJJmJAQJiD46jURaxpS04ApaVRlSrsqgyd+UfC2Wy0bt05my6FGlaA+lxaYbC45klAO/kQPZbWVxGqXlSOyCQDz8bZbLIlXSW1HrBn0B7I1EaeBHyLbapg+o5XXEwspMkKBgnv8rZbLckubrblC4zlX6TIT3Jg9+82IredS4M6y4QsIObUEzubZbLOAS3XK6lxK0IVlVoANIEeXfYSlfLq2qcSFOAdswYJA5RbLZblwRu86J2mioNSFHMAAEERKlDvPJPxsmocWokhcQUpjKCD8J+NstllXLlzOtxSSodkTMe+21NpIJMklRiYjQD8ffbLZblwWOXcpKHHC+CABCcmkTtvYqWpcUkFI1GydzNstlkKRBIY65K15soT2YiSSDEybbNMMqClUAzoROxP3fG2Wy3X0XINSHGioJdICdI9sWGp23XEBZfmYyhSQQBvHcbZbLNS2COIuxSv415KxG3Vx3HkfGwn5mp4yuOLMaHIYmRPOe6LZbLI4lLYLpi4aOoQtaXH05SNFOZpn3WOHDdJSuBAWVZE6GPHnJM2y2Wjc4qVrQtJudhTgADQUteSS0DBjewT9MhlGdpKE8kiDoQTrvbLZaMON04NCLJRKUPJUU5yrTTQDbXv8bDU9H6S71S1pAkpHYB2n7rZbLLcpWgLpmgpni42ppA0zE5ZmSB93usM1dtIkhSkqgn9EgHUA6yD32y2WjBKdYXQi6CkzGWZM7k72y2Wyzktl/9k=" } ] } ] } ] -} \ No newline at end of file +} diff --git a/samples/js/node/notebooks/pose-estimation.nnb b/samples/js/node/notebooks/pose-estimation.nnb index 6ca358ed2b0cf4..16774b4514ab11 100644 --- a/samples/js/node/notebooks/pose-estimation.nnb +++ b/samples/js/node/notebooks/pose-estimation.nnb @@ -17,32 +17,17 @@ { "language": "javascript", "source": [ - "const path = require('node:path');\nconst { cv } = require('opencv-wasm');\nconst { display } = require('node-kernel');\nconst tf = require('@tensorflow/tfjs-node');\nconst {\n getImageData,\n displayArrayAsImage,\n arrayToImageData,\n getImageBuffer,\n transform,\n downloadFile,\n} = require('../helpers.js');\n\nconst { addon: ov } = require('openvino-node');\n" - ], - "outputs": [] - }, - { - "language": "markdown", - "source": [ - "## Download the Model" - ], - "outputs": [] - }, - { - "language": "javascript", - "source": [ - "const baseArtifactsDir = '../../assets/models';\n\nconst modelName = 'human-pose-estimation-0001';\nconst modelXMLName = `${modelName}.xml`;\nconst modelBINName = `${modelName}.bin`;\n\nconst modelXMLPath = baseArtifactsDir + '/' + modelXMLName;\n\nconst baseURL = `https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/3/${modelName}/FP16-INT8/`;\n\nawait downloadFile(baseURL + modelXMLName, modelXMLName, baseArtifactsDir);\nawait downloadFile(baseURL + modelBINName, modelBINName, baseArtifactsDir);\n\nconst imgUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/intel_rnb.jpg';\nconst imageFilename = path.parse(imgUrl).base;\n\nawait downloadFile(imgUrl, imageFilename, '../../assets/images');\n\nconst imagePath = `../../assets/images/${imageFilename}`\n" + "const path = require('node:path');\nconst { display } = require('node-kernel');\nconst tf = require('@tensorflow/tfjs-node');\nconst { addon: ov } = require('openvino-node');\n\nconst { transform } = require('../helpers.js');\nconst Image = require('../image');\n" ], "outputs": [ { "items": [ { - "mime": "application/vnd.code.notebook.stdout", + "mime": "application/vnd.code.notebook.stderr", "value": [ - "Proxy agent configured using: 'http://proxy-dmz.intel.com:911'", - "Proxy agent configured using: 'http://proxy-dmz.intel.com:911'", - "Proxy agent configured using: 'http://proxy-dmz.intel.com:911'", - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/images/intel_rnb.jpg'", + "2024-11-04 17:09:35.860051: I tensorflow/core/util/util.cc:169] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.", + "2024-11-04 17:09:35.872537: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX_VNNI FMA", + "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.", "" ] } @@ -60,7 +45,7 @@ { "language": "typescript", "source": [ - "const core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'CPU', { PERFORMANCE_HINT: 'LATENCY' });\n\nconst inputLayer = compiledModel.inputs[0];\nconst outputLayers = compiledModel.outputs;\n\nconst [height, width] = inputLayer.shape.slice(2);\n\nconst heatmapsOutputKey = 'Mconv7_stage2_L2';\n\nconst THRESHOLD = 0.3;\nconst COLOR = [0, 255, 0, 255];" + "const modelXMLPath = '../../assets/models/human-pose-estimation-0001.xml';\n\nconst core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'AUTO', { PERFORMANCE_HINT: 'LATENCY' });\n\nconst inputLayer = compiledModel.inputs[0];\nconst outputLayers = compiledModel.outputs;\n\nconst [height, width] = inputLayer.shape.slice(2);\n\nconst heatmapsOutputKey = 'Mconv7_stage2_L2';\n\nconst THRESHOLD = 0.3;\nconst COLOR = 'rgb(0,255,0)';\n" ], "outputs": [] }, @@ -74,9 +59,18 @@ { "language": "typescript", "source": [ - "const imgData = await getImageData(imagePath);\n\nconst originalImage = cv.matFromImageData(imgData);\nconst { cols: originalWidth, rows: originalHeight } = originalImage;\n\nconst image = new cv.Mat();\ncv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB);\ncv.cvtColor(image, image, cv.COLOR_BGR2RGB);\ncv.resize(image, image, new cv.Size(width, height), cv.INTER_AREA);\n\n// NHWC to NCHW\nconst inputImage = transform(image.data, { width, height }, [0, 1, 2]);" + "const imagePath = '../../assets/images/intel_rnb.jpg';\nconst img = await Image.load(imagePath);\nimg.display(display);\n\n// Resize the image to meet network input size\nconst [inputHeight, inputWidth] = inputLayer.shape.slice(2);\nconst resizedImg = img.resize(inputWidth, inputHeight);\n\n// Prepare input tensor\nconst inputImageTransformedData = transform(\n resizedImg.rgb,\n { width: inputWidth, height: inputHeight },\n [0, 1, 2],\n);\nconst tensorData = new Float32Array(inputImageTransformedData);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, tensorData);\n" ], - "outputs": [] + "outputs": [ + { + "items": [ + { + "mime": "image/jpeg", + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAIFArIDASIAAhEBAxEB/8QAHgAAAAYDAQEAAAAAAAAAAAAAAwQFBgcIAAECCQr/xABgEAABAwEGAwQHBQUEBwMHBg8BAgMRBAAFBhIhMQdBURMiYXEIFDKBkaHwI7HB0eEJFUJS8RYzYnIXJEOCkqLCJVOyGDRUY5PD0iZEc4OEo7PT1OIZdIWVJzY3V5S08v/EABwBAAEFAQEBAAAAAAAAAAAAAAMAAQIEBQYHCP/EAD4RAAEDAgQDBQcDAwMEAgMAAAEAAgMEEQUSITETQVEGFCJhkRUyQlJxgaEjsfAzwdE0U+EWJENicvGSouL/2gAMAwEAAhEDEQA/ALv1/D+mrU9hULp3EdolwdxxB7qgobOdQOVq9Yn/AGbfA7E9TW1jqr/pXq91b7qqW98vfUvOSA605Go22tbZ8fa/71u0J0gbRatBg9DSOJp4g2/QWR3YjUztAleTbqqJXn+yo4ePsli6cd4tpARlBdepaka8o7NuzTvr9lJei6ZdNc3GCqCDEesXAgnQ9U1U/LmbejyQRHxt2ArQCTY5pY0MTOXlje37Lvi/TBBu3HmHqkI5VFNWs5tD/I0sdBvtZr1/7PH0hrvcccYbw9X5hH+r3mWjpz/1hDfKLevIHMmeeot2LDNG081LjnmF4w1HoU+kvSNvLcwEl5ZJhLF9Xc6c07kJqCdieXOyRW+jP6Ql2LbNTwjxgtKVStbF0vPp2M95oLHIaz8be2a6dh0/asNrHigGxddyXO4cy7spSevZAG0O522KlxwvDauwBxAuB5X77wriKhGgJrbreZynX+dI8Nv0smJNQjMhyo74ChOpMg7wfjrb3dTcV2IENU6mo/7txSPuIsRvLA+Gb3SUXndrdWgjVNQO1HwXItE0juqQnC8PB61CApRTzJjWYnYeVjSVVIcbUgqUG0zEcx8iBPj8rex15ejpwbvYZavh1hpfPMq5KNR+JamzarfQ44D1qys4CudJUCO4w40Pgy4gWAaN6Mydo3XjxQvOhKmlPqH2rqTmBIKsx8OvP4dbKdO9UKZpEthUBgd6J3Agx0j7xb1Ir/2fnAV9KvUcKIpVKWVZmLzrkak6+264B8LNmu/ZwcL3UoRR11/UobASgNXw0pIgR/HSE7f4rAjpJI3XIVh9RG9tgV5udqsOPKIWpIIQQB7vdv8AdYBx57KEuKzSDlVlIMbaj4cuYt6BXn+zQusqUq68fXyyCdEuUlPUwPMONTy5WaN6/s0sZNoSi6cfMPZABNVdKmwQDMHs3XPHkbH4bhyVbMCqYKWorQS4ggGCDodjrHn15j3W6Qp1SlpDyoEDmQRIJnny+/xtaW8f2d/GmkWVU984ZqEjXKF1zZ6mJpoHxNmxePoMekJdqHPVcM3RWZ5ILN80zafCA8pB5DcWYsd0TgjdQGaioTT/AN6oEa97XKTy135fW4yHKtDqU9qojX2hsI/W0oXn6KvH+7cod4V3s+E/xUZaqxoJ2YUvn4WbtZwd4uXQpa704Y4wpUITGd25KpCTvuSn65WgWlTDrpqIWt9SgQmASCcp26+Gg+Xw7DrgCASMrgkBJkjToDvIsM/dd6UKEt3jR1FK4pRSUvMls94xpI+o8LcoacU4nOAvLuDrroOn1p5WGQUQboVLjhJSAqBqoAAxHkd/CzUvWqqWcX0QQSlRQkE7QIVoT7hpvoN7OtCVlRcSgGO6CI1Hx8febM/EBLd+UbqUhQUJCgrY5zEfAbdfG1eUXCLGdU76GtU7TD7QLhwpTyOsdP0sIagpAVmKhM6HfQxv7+nzsVutRZoxUgEqzwnUCNI+P11sKtZSlDAQo9p0AJ8dvH52hCLNsiyHXVbVUj2QpIA1OUkHnEe7x5Wxx1aU5t5IGYAHXff3ePnbSVIUtXdByawQY168v0Ng1KZLZW6gHPsCI1nQa+GvWxiNEJCLcfLhUVnQDcATvt8frmH2rmaXHezMlUeB6Dy1+HhbboST2crlROYgkDmdJ32J/rboM5kkmco7oGbTLO53Ez91hndSC4Ln93nUpCiQUjmPEbmJJ+duczql5lgpgbKIgDpHXzsL2S8wSCpStyRtt+Y0Fu22HUqJSQRlAiABtM6npPhaBcNlJYhS0biY1OxA5HXccrBPqU9TrSgkSkwCoxrET7wDYyEOBScyUgqMZYVqCdRPlpvbtxhwtrDgKwpBVAJ9xEjXn87BcbKYFwmRhmpeYXWoOYlRbKgBEjvEggeeh56eVnj6rTOhL4ZCAqVkEbp5nx0J+PnZn3W6tqvrmEtnrAXEEHrM7E+Pv0L+u3Mq7mDkOVxAmQdT16fXvsB3hddFaLtsk792MuZAplQJ5SdTpymenwtyu629crzomE6KEmfPnP1FnClpKEhRCEyARrAAjp7j9+lulst5R2bWYkTMkCfGPhZuImyJtm5/9iioUdBmBG35/jrYA3E4SSstqBHdzp021j3b+XhZ1GmaUSkgjpG5HXTy+t7Y3Slaxl1MTM7b6jp+ZNomXKpCO6a6MMKeAWpinWoQDqUyOp002+jbHsNLywijBhRMJX4bn4geVnuiiAjKkaSOkHmNbcLYKgpMAAxPOImfn+FhGZEEQUduXBUHRFI6r/ECJUJGkn+n4FH8PPpUFON1TQCtcyCIMQBr9RNpNXTlIJUnIUGRmA0P56H5+FgX6JzsFOBCjCTA10+6efztESlIQgqJUB5p6AM0EpIgkaGZjzO3gOeyi1QF9lClhtKuSp31289j8bavRnsalcpJBWYIPMkGfPT5ednphNmnqrkbS6w2qFrbUCnMd5A+cCzufsUzW8kx6rDLbbC3qm76VwJlUJbSSY8D7/iNptHWMbiLVKqqprqSyEqEFpvLpr0EQZHj7rT7f9LRFCqRphISDKykECZ9nT3/AI+EcYrogLnqUtp7qFhRO/ekfn5+VtKhlc1wN1TqWCxsoQLTzZlKFiOgItsVdYiQH3RJzEZjv1+6zkcpyT7a/tCdVDnJIOn1sdrFfVgtBQO9oSJTIgRJ193wt0YlB3WQWW5pFNbWaKLyjlgCdYtoVbs94A89RZTVSIyBwoSZB0CRvrp+nlbj1RKkSpjKNydhvtH1r7rSD2qNj1SaalZGoEnnbr1o5MhEgCBr9ePxsZVSN5ZUlSYE6a6bfX0LAPU6ENZkZt9ZtIFpTeJY447CXF5u8IBJ5W49ad73e0VuOth3vtKSmCie6lSB5hRP/ULFQyogmRoJ52cWKRuuxUlJzBInnbv11ZEZQJnYxYAtrHLxtrKomAk2ewTZijLVS2lQK0d2RpvYyq8myns20qy7ZiYMT4WTi24BqhW3S3MHpZsoKWYo76y2pARzHUaTJ3FtB5OoJEyNZixPWLZryFllCV0NUHNCpGvSwUzFtWy0homJQtNUO01S1UtLKVtLStJG4IMg2+kvhvik4u4d4XxU2sKTfFzUVfI59qwlf42+a8biLe+foIYmViz0ReGt5Lc7RynupV3rPT1Z5xgD/hbFuZ7Sx3jjk6Ej1/8ApaOHWLiCp+VUuRvYI1Du4NuTvbStbcmCtgNb0Wdu4T7Vs7dzUz8LcxztqCbGCnlC6YfcS+4gq0ISr7x+FjHbOz7RsTSSmoQdsyFD4EWHMzboaM5ogq8oAKG7Z06lRtoPu/zW4mRFsMnna0AhErrt3Brmtnbu8lbWD52zbnaQF026F9YdOmY2T7xecD9K8VnRZQdeRFjnw91iV6pWaUrbIlCkrnyNk4aJ26Fddu9Ht25NQ8P44i2gZTIG9uFD32YjRPsuxUukRPnYdNS6UTPIcrFI0MjWwrAlAHutXc2yI3VDGpeiCr52QcV3hetHh+9Lwbvc0RpaV2oStFPnKAhJVqDObbkBOtld5fZpkIKlKMJSOZ/AflZBxLhlvEVyV913tVPuIrmVtAIOVDBI0KUiMxBAPenUctrBcwyeEKZIaLqvtb6TdwIum777vC5Hq6rrgy+lhbDCV0ockF3OHVBJA1LaghY2UQdLN3iZifBF4JrUVGI78Xfq3E0jtGkLcNGcx7nYJhttSvaKHVoUEmTJ3auIeH1z4lwxfuE2r0dp8QYKfco/WnezUpTKUgrkhQBCyc0nUEZddU2rWzipVLSUGHnr6YdvO6K3KyENoU0wlDS1ZFFaSFpSvUEqKQqImYtnVmHiWBzSbaH1We6tkY4Ai4KngYnwGkBLlPifOBCovCmQJ5wlNIQPIEgdTvbLNxn0g6ltlDdVSXut5KQHFJSkgqjUgpWARPQAeA2tlvODhtbf3f8A9v8AhaPFh6r00fH2unU27bAUItw+T20RPW3aDpv/AFt9IO3XIsFghkjrHvt3AAtwklO9u0kWHZSXYEH9bbAjbblbkEKI193S3YPKySWwBO3vt0NtjbkRztsGySXUbWwTG58ranwi29Js1k4XQJne2wSeY00twPf1t2Dv94tEqS3MA6W34C3MmSAf6WyYmySXQI2Nt8teduZERMe+2TO+1kkujAOmltnvD5W4Jts+NmskuF01K5o5TtL/AMyQbBG67s3/AHfTiBulsA/EWMSAOnKbZmAHusrBPcom7ct2vI7J2nJR/KVqjfpNm9ePCXhve6iu88FXJVlW6qi72XifetBs7c2kZjbJjnItEsb0ThxHNRbeXoxcEbykO8NsNImT9nc1K375Q2k8zz52Zt9egj6N99uoqKrh7RpcbgJWzVVjMdNEPgTqeVrBzr52wq/mG1hGCM7hSEzxsVVyo/Z58Du+LuavigB1Cae815QY3+1S59RZtXr+zdwS8srujGl/0hCSEh19l8A6awGm522kWuQCImbZPLoLR7tH0UuO/mVQyv8A2a14pacRRcTXVqXr3rmQE+UipnkOVm1eX7OXiYktmgxdcjyWdU+sMVLc6RPcQ5yn5W9GQZETbJBM/jaBo4ypCpeF5fXr6BXHSgQ+umbw9XrI0DFc42VdD9s22PmLNS8PQ/473UhPaYBYdZbJUssXpRO6Cd0h2ddeXK3rbMG3C0NrTDiUqG3eFhOoWnmiNq3cwvHeo4BcaKdpVUvhPil1vdJprsee0Ame4kiCbIN4cPMa3PkbvjCV/UJBlRqbufZHiAVpHhrb2ZZu27lMJSugpiAIjsk/lbf7luoiRRpSTuUKUj7iLAOGki4KK2tA3C8T1MBLzheXBb2BV4anXXythpnCySh0qcXIToNR8NOWtvaerwtcVcCippVug6Qt1agfcSbNy8OCXC+9ZN4YMuV8k7vXVSOke9bRPzsB2GSciisrWdF4gFhdPfTzI1UtCspJ3Mjl9fKz/wAPhblz0yVKISjMQdUlRnbeNvhb1QvP0O/R5vZwvVfDO4e0IIzt0QYMHfVkoj3WRX/Qf4Glo093YeRRNme6xVVYVr0Kn1D5WBJhsxGiKysi5rziSD2WctaAADuwCeRjz+UeFhEUPahLJKte8VHyEyNxuOdr+13oD8MnVZ6C879Yyg5Ui8GsvwVTqPzshu+gPd1Mpxd3Yxr0FQyp7Snaeypjb2m7U34fUDYKwyrg6qlibto1OEqQgAAgefMkWGZu1lAAUkdo4ZlKpPu6jz62tzXegrfzYQKDGTbwSc3293hsTB17jy+fhys3rx9DTiRSl1ynrrlfWYCAE1Qnz+wI687U5KOpbqWlWWTwHZyrQaNtx09nmSGkgFSj7W5kGOXOwaqF0iO1IUuSMyY8tJHytPFb6J3F2myhvD1HUJTqpabwYQSfJakn5ToLIFVwB4qUJccdwBey1AQgUyA+B1nsyoHrvy91q5gnbu0+iIJIzsQohcu5RWpKMhABUpWUgDwHWRJ8osXdpXjTEEN96RouCdxG3OD8bSZX8LcdUK0t1WCsR0qUpCnFO3c+kdYkpj4aWalfc9TQKAqmalkpEJQtGVRPM6jx3iwjnG4RBl5KCsU0zjL6kqQjNmClFKpgRqT0P6WXMG3itm5nmwStaXjl0gQQNfHbeweM6dxpNSktDtnRmUQQTAI06RH32T8GEvUNQ2ZjMkgEzGhnfxA3tdY0uaqrjlfolioUVOFZEqV/MfGd/KRp1izQxRTtuUNal0SEtzmJ9525ab9Dpzh4LQsoKpGYmZjeOnWLNy/mu0aqUdmcq2VAa5pGXz1/XcWu0/hIsq0wuFEjzZglCSspBPtZp13j4+MneycQpQLWQ5VCSfEffEHz+FlRSsoSlJcBI5QoGefhtBkdegsmqKHESAdIkHmBz+ult9iyHIJ3u5ieQGpn5e4fpYN1KRlKwgKGh72/1pHlYQrTKlNrBV7KtfaHMjw0+VgFKXKc6CEq2GhMSeXLf52KAolaXkICey0He8hyHx+uhaqALTgTEggkk8/A/XOwy8wSCEqO8iZ25zYJzvIcCsw011nX6ixQo25INKpups5NGqhXe/zJGn/LbuG15UEBWYHUHWeW/kPnbinSV3bVoynuLbXoNvaH4i3KMywJB3g66H38rSKbktltmII3MEnQzt8jbMqUEJVEjoYiLYVkmVCJ8YERE/dbkgqOgGo73Ub/AJWSS2UBU8xpGuvPfrbSmkpJTJmYBI++3RK8xQpskHc89LczEFWhO08/ysklzlAJ0BIEeztblTWsKA12k2EzbnLmmCAVaQPvtpUT3ANehj6FnumsgFoSkab+dg/Owy092dbBAHUeFpAqBC0N7ezH7JXFKr79F6puNx3MvD2I6ylSn+VtxDTw/wCZxdvGcbi3p1+xnxOSxxNwa47oFXdebSJ/+mbWY/8AZ/K2PjrM9GT0IP8Ab+6uUDrTW6r0vVI3NucwG1tr2sGfPa3DhdC0XCE32tzEWxB2EW7AB2scC6bZBqGVbSjAhUfERYY6WAqP7on+UhQ9xsY20623KA/p2VeYc1mlt7crYJ520QCdvCbXwECywxFtDU23HIi2jPM2e1kitknb77BVCA4y4iRCkkRYUSdANbaUJGnjaYbcJr2SfTnMygkz3bbVJMxYKldQvtmEKSVMuqQoT7PMA+4/dYYyNOtoqV7Lka6WEY/iB1OhsGNz0t0FFJOQCSDEmAT0+hYMlgLlSjJvZDpbW4sISmVHQWLvVV1IafQ/XtJ7MFR74JSoDXQaz4WVKUXWy16zeVTTgKSSO1cASB5ExaLMR8TbgRjd3AhvKop3HmBVIbYIRnbGiw4QFOAAHNkSmf5ykA2zpcQFI3OSAeV/79FIjiuyhVX9Jhy8rt4vf21wJX0lO09dvqt4O1TDlI2t0lTZaW64lKZKM0EHMIIjQC1bsPcI8l5tY/xxjS6MOXLeLy00LLjwb/eBKiVFIAKyiDBUhJTmj2gDF4+L/EK5au42rmvPD9XeVI+UP01AxdqqenpRPcWXXWuxUtAhUZwiQUybUjxDwexbjYGvxNiynpO3fW8mvvJwUgLCRAAClAOEJ0OQEaEawTbDqMUdM0tdIGA7m2v23Qn07WPzBubyVgaK7vRhp6NhhuopXUNtJQlxdM6VLAAAJIqkiT/lHkNrZaGqTCuGaWlZpX8YXG44y2lta+wq+8oCCf7sfdbLckaB1/8AUv8A59la4v8A6BesLwPaanmbdpBjnpbToh3mJnlbtAAPzt9DO3XJN2WFRG9kyqqC2Xni66lLQJISqNAmdLKS5Inpysi1yS41VNE5QtKkExtKY/GxIxcoUhsod4b+mXwX4nX7Q4auDEN60V53ioN0tPeV3qaDqyJCM4zICjsASNdN9LTqxeFROV4gnlpaA8L+i/hC471uy+lOsVDt3PtVTbqWsqittQUk7nmkc7PX/SE3XY+VhGhpK5H7upi9VvVFK4whS1qGRLecDOISs5kynaCYMFyBw1CgHkHRSo29mAUIE2AfvLsl5Gmgop9okwBYGncIZnnvaOeOdJWXtwnxDdFDUOMu3my1RFbaoV2bzyEOAEaiUKUPI2GIxdTMhUgUWJ6SvQXaN6lqmkqyqXTPpcCVdCUzr4TZUbrG3Eg6687VI9HDgtWcKMcJvG66t1q77zpXaerpQtRaUQkrQqD/ABBSd/8AERzNrOitS1W+qkjVAWPfI/CzuiHJJshS84+00guLMJTvpNghedISCXonqg/lYlVuk0wE7qSPmLVy9KjGHGrCN4YUc4SX4xRodarnLwpnqFp9NUUFjsgStJKQJcHdInNrtYYiuph5vorQt1LDolp5CttjtYTMI018rMLCF+V1dhS48QX2wilra+hpn6poApDbriE5kgHUd5URZ2MP5kkg6EWgYraJxLdKMwRI+NsCjOhNmBjjE72FMI3/AItprqdvN66aKprk0TTnZrqS0lSg2lUGCcsTB32tBuG/TKevpxtlzg1jNhxUJKadYeKSTH8qbLhW3Kdry7YK2RV/NbM0gybJtHXLqWkO95IKQSFb2J1V6PNrqXe3LbVPMwmdAkKJ2nnZhGTsnMgG6Xs3Tbfe2FWu8C0K4B9Kbg5xLvZi48FcRqO8LwqZLNK7RP0y3YGYhPaoQCYBOUGYB0tLVNeSnQC4iDzA2sxjISEgKP5gedsJjnZEvfEdJdTjDTym0mozZO0dCAYAkCdzrYNGJ2nAFJplKB5pcCh7utmylSzBL+bqedsmNd5slUl+MVb3YJYdSYJlWWPvsoocSsSk2jZSQgV1tk8h77czHKwL9XT0ye0qX22kA6rcWEiTyk2iQkjClSf1tmfxsC3UMvJztuIWnqhQIPwt3mkSNpsyS6zdbbCup+NuJm2oJ5+dknWNHukTzI+dhM/ObF2v40nXvq++wgII3+NmCRQoV1A1tokg6TbknTytiTqQI31s6YLqdiZ+Ftheg59LBn5zbcgbCzFTBuhQrnGlszGelgkkbm25J062FayVkIFDcae+2ZzOlg+ZtgPjZJITNHP320tDbgyuISoHcETYMHnboK162YgIjSgTdt2LIUbvpyRseyEj5W5dui7Xk5F05KTyC1AfAGxgbaC2wR77QyN6I2Z3VNq9OFPDe+ypV8YLuWtKxCjVUDL+YdD2iTNmnU+ir6PtXnP+ivDjJXGb1e7WWCQNtWkpOk6WlIdbbBg2gYIzyT8R45qB709Bv0dryzKTgv1VZEZqevrEn4B6PlZl33+zb4HXqVOU15YkoSrkxeKIG+3aMuHmedrXZjpJkC2wrTQ+c2h3WLkFLjPtYlUDvj9kXw9fzC5OJeJqQFMJ9YFNVR5gNtE/HnZgX3+x5v1KZuHjN2ipJAqbiCEjXTVFUo6T/LyFvT1KhvFu0rGotPhW2KhfyXkBe/7I3j7TFa7rxvhCrSmcgWuvaWR7qZYB9/vswb7/AGY/pX3WhaafCtyXlGYJVT32w0I/+0Fsj4W9vgsH8rdJUN+VlZw5peE8l4AXz6DPpV3OV+t8F75qCJA9QfpqyYG8MOLmzQvf0afSFuQKVe/AzH1OlOpWvDdYUAdcwbKY9/nb6LVtsuj7VtCv8wBsXF13UTP7upgY5MpFkHvG6bhtK+Z57DGJLnTWMXzcN5XeoNHu1VI40ZChocwEc7I4SoNzty1t9Or1xXVUJKHqcqSd0B1YSfcDFmve/BLhNfxUb64fYfr83teuXWxUT/7RCrPxnDkkYgea+bMocEZtNNdCNOU20pRM9yATtNvoWvj0KvRdvxJFbwTwinNrNPc9PTn4tISfnZi3v+zK9EO9ypX+i+npFKMzTXjXNQfIP5flZd5tu1R4PmvCSCD3UxG2ttlLihCUmJ0jW3tTe/7JH0Y64f8AZzWI7ujb1a9zt/8AXNu2Y1+fscOGTqnF3BxDxTSAju+sLpaqD4jsmpG/OzGraNwUuCSvI9CDlzDQzrIjT6/C2dkiJKhGmo13FvTS+P2Nl8IBVcnF9xwnlVXA2B4SUVZJ/wCGzDvz9kVx2oCs3TjHCNcn+FLyK5pR/wCBhxM++yFXGU/AeqCqSj+ImT42BKSDFrhX/wDsvvSwuwRRYcuC8ttaa+m2gT/9o7I/LnaPr59Aj0uLiUF1fBi8qhJUB/qNbSVh+DLqiPeLFbPGfiCiYX9FXvUWu/8AsjcRqur0k7zuJTuVu+8M1TQST7S2nWXR8krtXC+/Rg9I3D6lG9eBOPWG0/7T+z9UtEf50oKfnaT/AEDW8TcOfS84fVN/3Fed1oq6167HPW6RxmfWGHGkg5gP4lJsGvyy0rwDyT07XMmabc17kKMggW4522VamLaHjbz4DVdKFnusIgzIFuBr4W2BrY7Eiu3EZ21IP8QIttpWdtCjzSLYk5hB3txTK+yUjTuLUny1kD4EW16A2JCrS6BCj3W3FtCDvpbYmNLaoCr3Wc5m2hlgcrbJ03tke+LOEyyCBIM25URtbek/ibYZ2FihRJ1SahIRW1CEpAzwvQb8rdq31GptzVAiubWD7aCkj527VKhMzaFksy5G4IG3Kx+6ruFW+X1tpyta7bq5CxFsdo+3TI1ccMJTv7/IWdvYeo0JbpU5lJRKdPaV1PvtXka06nkmfIWaDcpt3xQ3Ve7qziR5hm52VJbDbjvZpfeUYhZkSASEhM6qJkGBaMuLXD3hxdlwV9fT05w2mphp28aeudoHHgogqSMiVuPKMJ7mQ5tdbTS8GKTItbWcNkFAy5lZoiR46n4myPifCVFjK7Td+KqRD9A6tLhogshKinUBwpIzCYOX2dNZti1VFHIx0b2hznc7bfflbknZKWuBB0VSmlPXtThq7r0x5f13UjfaJXWMuPtKUlKyQy0ptxSFQ2YByLy90e1Brzih3CWIcbU9wVeNsVu1F/VyqZ2gqaPs1sNkpzuZE5glAyz3koACZCXIMXz4m33U1l10l1YBoFIYoX6dKKimYCmlpCi2tlkJICYSCCpRSjZIVMlNQ+N+M2g1fdVgm+Lpuy+H7qDFe3dRbdqWVtocYCHKhBkAdqhTeTKFDtNJFuIqaLu02Vri4deQPodlrts+LMdPJMuvYwNS11RTU/ol8SFNNOrQgqdr21FIJAlCachJjkCQNgTbLRX+4OMqu9dtVfIpDrTh15IWG/4cwKhBiJ0FssT2fBzkP/5uWfxz8v4C9jnxD0a26A0j522+gdqOptiRpJ3t7s46rmW7LgxyMWbGKgpdwXy2DGekqBv/AOrNnUpEg6TOwtCXEk8a2MUVFDhqruN3Dt4ZW+zqGFB9lCkhLneG5nMR0kTtYjHNb7xshPY5xsAq+cD+BGL+GuMbgv65MX3kKBuoZbq6NVQS08wSErSobHunSZiBGokW6vZtg3lQurbBXlcSkxqB3dPKxdjDNXR3C45QpbVXNMqUwlWiS4BKQT0kCzDwXjTifizEjd1Yu4d010U9Iha/XGq0LBWIGUIJkg7zOmWxjIzS7kMRv6KaaUxTb7J52b+M2/WMLwnvAu0yjPTtE62ctIwpLGRQ5Rp5WirEfFO6+Hle9hziFdte1dzhUaS82KdTzCmiZyOBPeSpO2gMgA6WUbgTome0gJ23Mwhp2mWlIgSNtu4q3dVUZ8TlCYhFK3OvPMo/daO3fSS4HXVS9uzi5yscAhqnp6J8uL/wjMgCfMjzs4OGtdeuLkvYtr7vdoU3m6XWaZ722mAAlsKn+LKkE+JNnkeL6pRxuAUiPkilbJH+0Ry/xC0I+kT6QaeCF44cp3eHL2Jqa+G6px19moCFUYZU1HdLas2btDzEZec6TNiC8KK5LndvS9KxqlpKYoU686oIQ2nMO8onQDW0Z4pc4WcRaqkqazG9wPCjbcbaSmtYcBzkT/Fp7ItAG+ykR1SrTvUHFDD+G+INzXveTFGtlN409GVBDTwUg6PJAkqTJjvQCNuYf11uq7FIVMxr1tGT+NcDYNuq6sB4cvOlrat9SKOkpqV9DikIJlbiwk9xITmOu5gDwki6AtVOhShEj32Y7papGxVV3Fd+FL7vDFLymbnpaSqevB0pUrLTJSpTioSCowgEwATpoLRThfiz6MFU8y3hvHFD2zykoaQ4y+gqUSAAM7Y1JI0m0t4hu5u/MN3xcamw569TVNNkVEKzhSYM6c+doywl6PuE7ir2rxew5QpeaUlSVdkhRBBBBkeNhSa7i6LFpzspqoT9mAgDLG4snOjMa9tWynCCfNtIsp0iUNtBAVoBFk1SAXbwTPtKPL/1abTZtZQfuoMwn6MWGsLYoubFDQp2qu6ahFShxlBCllMmNRsZg+FpaubGFLe991l3ULNUhN3rDLqqimWxmcgE5UrAJSAU96IM6EjW1UeF3B/HvDbGlwX1dGN73dudNY23VUD1UtbTjKyEqSRMHQzrMeYBtbspaNawsphZSoBUcpGnuk2ThlCduuyReLfCy4OKNysUN/UYfVSKU5TrzlJbUoQSIjcCLIfCzh/SYGwBW4QpFLS01UVgSSokgODNuTM942lhgZmQCSdLIaUpS/eLYG7xPxbT+VmGoNynvYgWULej5wovPAN6uXnUYmvq8EuU3YIarKtTjbYKgSQk8+6NfE2shTkBEEAWbGGqdtLDZSkeyPOzoQiBpp0Nq1iNCbqxodghiqzH4qKUq4Utg93t0KIGs2e5O8aWZnEtAXcZJ1hxJtNouDdRcU0sC4j/AHe47drildk/30CfZXGvxH3WTcMX1UDGia1T7qkLqnioZjBnMNttzaLqzGFdc3GsYMqZRS12H2Lyu1w/+kNPvJqEeeQsq9yrPHBztQqsadqW1NuevP6Hp2ysp96YjwsPZgBUhq64ViaR8OpmbGQZ0H62SLpUpTQk8hZVBgWThYpLlr2nJ5L5+QsLMaxYFsgLcGs5gY9wt2TPKbMBdJNvFGMjhypYYaoRUrdSVLlzJlHLkd9bD0+LqN65Dfbram0NtqU6gKzFJTumY1/UWaXEgH96056tHkep/OzIrMTMKwviJuiqgsUzL7TyQofZvNo1BHIxB8QRYLn8OpEbvdIVlsXEp87dwVM2HcYUWIu0SxSvsqaSlRz5dZ8jZaU8IBSPKbRZgt9VHVBLayQ418dRZ04wvx66cNVVayftQjKggxClGAfnNnpnGVlzuo1EbYn2CP1mOML0FSqkqr4aDqDlWlCVOZT0JSCAfDeynd18XZerPb3bXM1LYgEtqnKfEcvI2qDxIuviBfd2sNYAv83XUtqW86sNBRd07qJMwNyY128QXDwIxbiZVPc1fiIKbvJThorwSABmIWUEkDT+VWkCRpYTXvc3iHa9kQxNByDdWnW6lKcyiAneegttDza0haCCCORs0cd3dU4iwZel0UTpbeqadSWyFFPeGoBjkSAD1BItDvCfGLl3Vie3loOj1SrbUfZcSdCRyIPyUbWnxZWB/K9j5Ks05nFvNWQDiSqBbqQBJMWgyqxbed94pQunvWrYo+1S0hNO8pGZsHU6Eb6md9RaYUV6V06n8xCInvdLV43CVxDeSNJGYwCeaUA6Npt2FAiRHhaBeIPGupuC76zELL9Yi66SAhFFTh59+SACB4zO4AAk7TZV4QcaWsd0FBeLNWqpu+8wQy462EPNOAlOVYTpuCOfIgmwjKL2tpe10UQuAvfXeymYK26WwLAgmJsj35frNzXPVXq+kqRStlzKCAVRy9+gsyl8YKKmDZvCkYpw4cqM9aE5zEwJSJ0tOVzYQC/mmia6a4aNlJwUPL3W2FTZsYWxtQYnU8zSsqQplAWSXEqBBMbizjSqedmY8PF2qTmFhsUMF6aRbebxsDm00Pvsi1OMLhoqpyiqr4pWnmzC0rXGUxt0tImwuVEAk2CcIX4267TlrZJu2/bqvRRbobxpahYEkNPJUQPEAzY+XAEk9LR0cLhIgjdGQsEaG3Qc87FG3Q4mYOmltioSpWSdRaNk4dbdGguNfjbfaCJBsB2gSnMT520lwKEpMi0HAqV77IwHDoOVug6ORsWKwNSdLbCtgPvtGymLI12pnU6W32s7WLZo0m2ZoG5i0UrAoyVk24JB3sBnJnWTbO0nS0SErWQqjPT4WBcS05IU2lU9QDbCsnYC3Kj4izFo5qQNkWcu+hUcyqNgnr2YsA5dNCuCWimNgHFCPcDY6penP324Kj1sMxNI2RWvI5og5dFNlIbdfR4h0n75sCbqXMt3jUf7wQR/4bKSvPa3A9oePUWAaOF3wowleOaTV0Vcn+7rWj0zMT9yhbnsrzQdfVV+9SPwNlJY03iwWnvtDuER5KXHciYVXo1NG2SP5XvzAsQumpvdx2rVUXe00pTuYN9tJAiNwCD7PzstGedirY7OsWJjOJjyj9bJtI2IgtT8YuFiF0l2qjvUh8crgP5W2agpmaV8R0SD9xsPsJ5G28xHQWthh6oOZFjWsDRQdT5tK/K3IvGhKoVVIBA2On32NnXU21AIgj42bK4c09wghU07glD7avJYNuiZ2I+NuHKWncELZbV4lANgF3XdylKPqbQKtyEx91peJRJARe93006WH1BUJeSNBsCYtwKk1Ci3TQcuil7pT+ZsSv8Aw829ddSihedYdyKKFlxa0pMb5CqDz0sfo0JRTNJbEJyiABA26WXiLiEjlAuE4rkugUKDVOJJed9oq9qPwstKcSlkvKCiEAkhKSokeAGpPhZGpr9ZRTIRUZlOAQTHTY2Zt78U8WYXDlTeHDh28KMvFDT913gh4pRBILqHEtqSdDojPrAEkgWo1dXFRsMkxIA52JQhDJO6zRqn8+yoH1+oKm+5AQVd1tPjynqfdytEfFbGOJr/AHEYLwNSMlKSl2vqKxakN1TWaFU7aEguLSZBWpMDKCnMCrRuY746X1fFUjCzLFNcbrikuFovpXVvNgahKTATBIBJSqI2HJpXxj2uuK4n0U123I1U3k7kqF1iy6p4oSIU44rMpzQgDNISAQJ2tw2JY/NUPbHQg8M2u62p672stinwtzGZp7X6XQOJGL1RcdXduLcCVfaPBZq7wonGbxbJgAKNPmQy3DaUpSghRSAhIB7pNauINw8J8Mm7bwwtW3qugvFxLLpW0W3UvaOJcebXlSlBKoSlKEiFE6zaY73x/iRFEf3HeWHlVT6FNoRTXc0ptDkEpUVhsJyJMGSoa6QSYMNX9iKiqr27bGr9Vie+EEIStkqpbsIlJAKEJ7RWaNYDbhKT3VFOmTJO6STQG3RWZImxstcEqJ6ziHxOoKt+hpMNYmqGKdxTTTq+yWpxCTAUVZBJIEzAm2WnRdXg5S1KYwuA2SSjs7szoy8sqlZlKHQlRJ5k72y0Pa0X+x+/+VT4D+q9HK1ID5002AFuECdbD14+28zysEkaAje3v+65FYEnmLAP0TbypUmTY0lMxIt3Gm1mukiaaVKUZQnfSxZFzU7b5fQ0ArmQNbKuWPdbeWNxZiE40QCGsqYsmXzh6ivhos1bCHEq0IUkEGy0RbYRPKzGx0KVyNlGzPCTCtPV+ss3PSoWFTmSykH4gWeN3XU1RNpbbSEpSNNLK5QneLZk8LQbG1puAiF7njVJd93LRX5dNRdN40zVRTVKOzcacSFJUnoQbQzeXoxcPKl0rGH6VJJJ7qMv3WnuNY2i2lNpVrFk5ufmQosdl5KHsD8DsI4LrfXrpudhp+IDmXMuJ/mOvO0pU9Lkby5YseDCQZCBbYbA0jeyY3JzTvOc3UC8SeFPEC9cTVl9YW4o3/c9PVZVCkZdJabUEgHKkECDE+ZNmuzw89ICjVDXGa8lJ2lyhQ5/4jr8rWfXTocPeTYH1Fon2bJ3EJ0d+yk0sA1aku4qapRQMN1z6nn0NpStwpAK1AQVQOptHHEpji9SYlQvAlfc6bqqEIS81WUylLS5JClBSdxGXTqPG0wIYCNrcO0jb0ZgNPCxRI5vuoRY07hR5WYWq0YYqP3e22quaYK6cKkJLidUgnlqBZtcO8R8TMR3w5SYzwpd91sUqCpLzFSVKWskDKEEmExOpPIeNpoFIkJKQNCLF27saadLiUJBO8c7J0rzZIMYNVqnZKWwmZ05WijHPFCiwNit6467DWIKxValt1p+hou0aMjLlKiod7T5i0xJbCRBFk+8bjpq91LrzQUpBBEizGQtGgT5Gk6pMw2y6mmaUtBHdG/lZxIiwNPSpa7g0FjQSRzm0C6+6f6LShzizU4hKabuJx2ocQ022tJUtaglKdeZNnYYsh4uwzdmLLhqrhvilRUUlWkJcbWNDBBHzANptcBuolt9lWbi5hb+0NPRYtw0609emGyirplMrCs6Uk52iRyUhR06xZwYUvBurTQXi1HZPlpaOcTH17rSZgvhXhzBF3O3Xc13IZZdcU4oanMowJJOp0AHus0mvR2w5RYzTimmerkZaj1lNOH1BoLmQMo5Axp4RZnyMcwA30U2scHXBCk5m86e7KL1qqUoIGVOgkkkgAD42WKG8qeta7ZkrAnUKGvWzVxlharv7C1RdV2Xi9d1UooU1UsgFbakqB0kEco99g+GtwX/AHBcvqOI7+qb2qi4VqffQlBgwAkBIiBHnqbQc5ugCQaSLp7oIDqyD0sIYiB52DRHan/KPxsJrGny1sgbJlHvEVsGtoTqJSsH5G1abvYewpxvxVcN6OKduPiA0iupQZCUVbTCGX2v99AQeWxtYrjeMVU1zUt4YRuFF71bbxQthTgQA2RqZO50A352j29eHWIMb4GpLxvG7W7vxBQuJvCmbQ4Fll5JJCMw01T3Ty15xaMkQkmu46EW8wUeKUxxWA1BunnhOWXaRBWV5GuzzEbxGtl3iInPg2tVPsBCjH+dJtGnC/FGJ7/xCbvvDAd5XS3ToUXqipWAnMDAATAJJmegFpixDc7t8YavC7ElIXUU60IJ2C47vzi0aBhibleeaeueJHBzeijLBFO09UPIcAPdGh982bTVM3d2Jq9lgZQ3eJWlIIG+VRPz+fwRcP8AEy7bivF27b+fTdF70Us1FPW9xKj1B25Tv5SLK1y3xdGKMTFdBeVLUv1b4fcbpnA5lACQToTGiRv1tGcmGEwuGt1KJvElEoOllYW7yXmQlQ9oSbQZxCwyvC2PEV9Ex/qF/hanUpGiKhIkn/eE+ZJtOt0sqSykqSQSOlmbxkp2UYfp612Einq0qUTskEEHXkLXYiHtLDsQqjyWuDhyKaWA7sFXWpeCISyAkCI1/paVL3aW3h+tSgwoUzkeeU2YnCd5itpqlTS0ryPDVJB3SPyNpNqmQ9RONEGHEFJ99qFEwM1Kt1j87rKBLnuakv6nprtq2kONPISopIme7NiuA+Bt54Txg0u7sVVjNwU9Waxq7W2UBMk5spWQTGbpGgjxsewheLVI+2HVAu3e+umfSN0KBKdRuOtlurxXiF6/akXbWFijZDaWShCVdpIOcmZnUx7rQe0QGQOOt9kZhdNky7WT/wAdtF7Bd7J3PqjioPgmfwtA2LMCYf4i3PRUOI6RL7NO4mpRqQUqyFMgggjRRtYS+0+v3DXMAhXa0riSOspNq+X1hZeOcIUlzovK8LtWvsHA/RP9k6CncBQ5HW0q8foMcDazv3CjQn9V7SL6KRuBeB8H4Boai78M0iWn31BypXlVncAJCZUSdBJ08T1tMCFAjS0O8DOHDuBaasdqsQXxe79UUp7S8axT3ZoHJA2Enc76C0wtwIEWFFfLqbqUpu7QWXSzCSqBpaueL2K5yoxExQVPY1ZeqksO5QoocIORUHTQwddNLWLcEII8LQPithTOLb6Z2BdSsH/MgGxpm3pJD0sfyoU7rVTPuPwom9HzF/EWsYvJGPHQu+LjvINtVTdOGe0QUgiQmE/orW1wq6rd/ddS7TLyudgtSD0VlMWp1wu4mYsv7Gl64Oxfh2koK26EJq2HqZa+zeRngaL6778iItbS5a395XMzVEAdu0CoDkrmPKZtVpvDOQ4WvY25KxU6wgg3tcXUY8KL6ql3rRVT9W8760y424VrUqVQTJnnmTZOv3H2JWOIl7tJvaraZu6raShlt0oQWsoMFMwZk6n8NC/D4qo65uiUqDRXk9TGdP4yPuNknHbAo+Kd8NgkGtpaapGnROT/AKbWclu9M6EkeoKCTmdA7qNfQqx98Xv6hctbeaEhz1enceCSfaKUkx8rMThdjy/MQVqWr2qmltP0xeQlLSUwvunQjlBO/wCFnJQqF7YRZKiT63QpB/3m9fvtEXB+rLdddCFqMhC6dXmAoR8hYUv9Bsg+YflThH6pYeh/ClniTjheDropaxqkFS9VVKaZsKVlSkkE5jGpGnzFj2BsVuYquU3g/SIp3W3lMqQhZUNAkgz/AL1mNx+aWvBbFakEmjvFh0lPQkp+9Qsa4NvqFFeFMDol5Dm38wj/AKbKpbkbG4c7g/ZRhGYvB5WUp5tJJgWaD/EG6mrxqaBAqnV0zhbWUNjLmAkgSRMbWdWYlo+VoHraZpWI8QXW6ohDlQ8hcEpIDgmQeWhsKoJjpXys3bb97KcQDpmxu2P+FLl14zuutqE0peW24vRIdbKJPSdvnZwFYgkjYWptga5MUcOuKNXhC8MSXneV1V1CirovXX1OlohRBAKjpsqYiRl6Ta113166+5G3s5Lq2ilRj+ICCfiLAhdJxBFLbUAg+RRZGsDC5nI2QtBim4rxvB26qW+7verWZz0zdShTqY3lAMj4WUKioSykLV15m1P+ENcbs4q3U84YUt9ynVP+NC0ffFpW9J+sqk4GpW6d1xLTtXkdCVQFdxRTPUSLaXdCap1MDsqgmHAE3VTQKlCmi6NR52xp1LqSocjztSHBHF298J4GxHhlu8HGDUsioonkKOZp0KSFhJPs5kbbQU6b2kL0TsXXtX35f103tfFdWBVMh5hNRVLdSgIcIOXMSBIWnbpau6LLEJfOyKJLyZLcrqzi3ENpzKMCwYfaV3QseVos488W3uGuH2lXZSt1F5VoV2Ad/u2wClJWoSCYKhA0nrpatN1+krxfob7pUXlfgfRWoDjDdRdzaWXo3CSlIJGm4V4WC4ZCARujtOa5HJXqJ3iwcmY1+Nmxw1x3ScQsI0mJKdsNOOZmqhnNPZPJ9pM8xsR4KFnOZJ0982YG4upWI0K0TpYBaQKhtXXSw5sDUEwhZ/hVPu+hZiEhoULyg7W2OpFsgeVtDQWkm3Wz+FtGPxtsxbRjysklzy099sAG8aWw+0BbfLeLOAoE3QT6M6FIOx08bJd2LUujQD7TZLavAgxZYKQQfvsjXelbVZXU5QUgPB1JPRQn8LORqCkDcI3zsKxTJrWKilISSUZ0Trqk6WDI2jnbpisp6BZq6peRlCFZ1QTpHIDU68hqdhNq9ZE2SFzX7WSY4tddqhniRhbD143k7dV9XVR16ajI96u8hKikAZS7CtgkzqNdY52jOvwLetY0wrCGOr0u5EuEu1bKa9socVmJbS6QpCU6BOVaZAG8ZrTPWXRWXled84irxUUZqXQxT077Ce0ZaQhJQTvBJVmyzAJJUCRAZ2Kqegw/hWqqL5YcNG42pIVUBZ9ZUUmEFQEkqOm0axoLcuGxTh0j2hreS0XPdlGtyojvPC4xxfT12VN90JumgQH2001OA7VMKAGd5/MokLV2gGUoTAWQkQCqrvEa5r6fxRe9z4rqTddxpqVUFJV1N6rWt2hSv7HMlJU6lgZEqg5UGCRBM2t7dFNdd0XCqrvGqup29rzfFdXBx1CWqZRAgBH+AAJSk8xPIm1M/S4wPi7EGPX2MMvu3hddV6rXNGnqszBdUgNrWsiQpcok7kZwAABFhU9JTMcA0ht9z0VeWRwZnOqkWixXh2moqemp+M16sNNNIQhpm63ltoSAAEpUTKkgaA8xbLVcpMAY6apWWximlZyNpT2YVIRA9nQxptpbLVjglJf/AFP7f4Qu/SfKvoEvBMPTE62AQkAzEWjJXpNcMa5wFKr2ZkxDlIlUHxyLNjjHH7hY4QF37UNHnnu6o0+CDb03vtOfjCyjSTjXKVIoEamTboJ5RttZmMcZeGFSQG8YUqSqQA4063/4kiyizxI4ev8AsY2uQHoqtbSfmbSFVC7Z49VAwSjdp9E4x1tmnI2TKbFmE6whNJie6HyZIDda0o/JVlFqppXxLNS05PNCwfuNiCVjtio8N43C7CRz2tuBprbYBPl4W3GmvOz3B2UcpXIE6xbUbQPK3cC2FMiPDeyulYriATG1thM7i3Uaz1thERAsk65jprp8bZljXrboCOdsg8gPGySXOU7wbaIFhBPO2ojU2SS4CegtoDwsJGvnbUDysklxEn5b21APK3ZTrpbRGnKySXOWbajmBpbsiNbaAOmlkdklxHO248LbAJNsIFo3SWoi2iARFtkAWwxOhs4KSDLSCdtRbSmUkju2FjlbCOUWYpIEspIjLpbQp0o1Fh9+Vtc7MkDZAAQ9BH8H42F/G2iPtkx/Kfwt0Brz1sk5QD1Ml9JDmo8rF03e22kpSjQ6WUIEW1AsrJrpLRc9Oy8XktpCj0FjYYATE2NRO3lbRAA2swFtlIuJ3TJxXw0wrishd93FR1ixsp5lKiPIkfU2Awvwxw3hVR/c1z0tGknUMtJRPwGtn5CToQTbMoBmPjaDow7VFZI5osi7LJQkAJ2sk4uwtdeL7iqLivmlD1LUxnTJGxkajxFl4pk6aWzLB8D4WkPCLKGl7lMvAnD+5MCXebruKhRTsFwuqAklSjzJOpMAD3CztLYiCPfYfKBy3thSOQ3tBtwk52bkoP4j8EbwvG/nsW4HxG9cl5VA/wBZR2QdYfPUoMa9T74kkkrgnh5julvIO4sxGzWttnut09IGQT1UZJPkIFp4LSTopMjxtx6q0NQkWhM0zm7iiwzCEWCTW6E+qKZP8Scutq13hcXHG4apygu+5sP1dEyvKypanc5bB0JMgT8tedrUdkBpFi713svd5SATztKQufHw9LeaeKRrHZlBPCwcTqjFTCsS4fu+76BlKluONVClLWYISkJ23M67AdbT62DA3NizV2MsrzIQPdY4hJToLAiZwxZFkfxDdbIlO/haAOMd9U2EcWLrKu7rxdRXMIUFUtKp0ApEEqI0Gw8bWAKZG1k687npLzATUMpXG0ixi48NzLXuLIbG2kD77Kq1xYhwhe2I0Vt23fXqvh1j1QRQvJPZ580KMZRrzVEdetlcM0S6O56elM/Zog+Z1PzNuqfCN3MO9o3SoB8BZdYpEtJygaARanDE5kmd3TrdXJpmvZlCrJiK/KfAfEm+Lovl31RqufF40LzhhtzOdRPLvSP90+EgYgrWMQ4rZxGipQmKFNGptKgvOrOVAgj3iPH4z9jbhxhnHFMinxBdNPWdlPZqWnvInfKrcfGzQw5wAwPhi8mryu25kIfa9hS1rXk8RmJg2PVylxc6O4LhY7WKHTNaAA+xynTqnhhKlfpcM0FLUAhxthIUFbp8D5bWhfBxF14qdpFLCfVL8eaAHJPbEfibWJYpQ0yG4G0Wjp7gxci8cOYvcerlFx71j1YVKksdrM5igb6666WiXAU3Cd5fhJo/X4g8/wAo1xjpPXeHF8ICdWmkvjwyLSr8LNThhf1Ldb7q6qpbbbqWUKCnCEpkHqT4m0r3/cjV93FW3O7KU1lOthRGpGZJE2gB7gpxMoV9lQcQAWkJhKHLuQrbaDM/0s1TK3gsF9QSefP6JU8ZMjjyIVh7svmgvRtSaSupnlJElLbqVEDqQDaHcVA0nEq8Wtk1TTL4HIwkJJ+Is4OEeC8T4bdqa7FGIE3g+62GW0N0qWUITIJUYJJJgdAIOmuiRxiw1jI4go8R4Ru6krf9V9XeQ+8pEQokEQDO/wArMyVslNI19hcH/jdIxFszC3kU3cfVDC8f4IrGWzndbrGXCOY7JJGvSc1piwmvNc6G9SEkgSPGbQncrHE6tvBinvXB9EyAYNQasKDaecAJJMwNLTzcF3+pULbIT7KdfE8zbPjnfPUMcRYNbbe/O6tuiEUThe9zdVKeAw1xXVJyJor/AFK6AJD2Ye6Dad+P93/vDhpWPBoLVRvMVAA6Zwk/JRtFPF/B9/ucTqtu57irqpV4OtvsraYJaBKACSvYQpOsnTe0746uypvXh1fNGtuX3bucISnX7QIkAddRbojOzv7ZQd8t/QXWS2NwpTGR1Xn3ftYbuvWgoH0j1a8TUUi9IhyMyNtpCVj3i0w+i/WLu7iNR0jjgJqaR+mJ2JypChPn2YPvs06Lh5WcSlXpS3RSrefoGFVlK8EnK3UoUCElWwKgFp95PKx3g5UVl3cSsOOMsrU6qtbbcZSk50BRKVynlAKiegtWqWBocGciiwuNwXcwpY9LiiccauioB7qqd9M/ylK21fdaIb1fwtfPDvCF2VV41VLfNKmpcDrCEHIgPKRBKtJICYAk6eU2K9Jy53KzA9NeyEHLQVMOq/lbcTln/iy/G1QLzuK4b/w4u471oSa2kqUPUbwBKS2VpLjZ5xAJ6mTtZ5WtLYpfIi/Q/wAKlGSDIzzB+quT6NNRhZjCNTc2HReRVTPJeq3K0pzuuOJgKGQwBCIgAbe+0xe61a/RfvamVf183fTkBLtKhYAEAdm4Ux7s5FrJjQcrZcTrt1V6UAHRYSOf32AqhmZUNOXwsMTrbhYlBSIkgxYiDm1WNrK2kK6pBsII8D42BpyFMAARqR89LCwroRZwE91s21Bjb326A0g20Tl2tKyiSuYBVMW3sNLb3E21ykDezgXTFcxqYFkp0djfAg919nXzSfyNlcmdhZKvRLiKqifQUhIeKHJmcqknb3gWk4aKLSbowRAg2T2Ev1N8+tLATSUKkhkA+29rnUfBIISPHP0BsokHlbjKRyG8/Ow5Yw8C+ydr7LvGdEhTYVplLfTSR9C0J4wu2mW5QU79Q3UJ7cut0TErqKp5CcyG0iQIEZlE9wAakCbSvjS9MVLpWmLlwgK5KmQfWX69DDYVqCnQKV/CDtztFmOafElx3JeN6Ul3q9ZNGhVT2KF1JLpABbQoABDQWUnUAGFLOaABxWNYkyJggidlcfLofNaFMwubZ2qiS/KGpvrGBoMc381d131LZ9ToWVkBLTaAVl6pAISDmzFKMkCMyyBZIviguvEpv/1LBN03w4puoU9elJkcTd4SUpbCUalS1DM4ZDclQSP5rOPGF13jeDbNNft5LoLu7MdpSMinFStRITlSjOpEbqzJUSmQIJEGHMYUzhvN5q7OJCaO4qVCHEUdNUFpa3FJKjCSMzhSkI1JWSqAkTojh6yaorMzJTa3Mixv9v8A6V4MjA8IToprs4bMUzTD90XitxtCULUq87vSVKAgkjszHlJ8zbLNym9H+jrKZqrreJ1bTVD6EuOslxxRbWRKkyNDBJE+Fstl54v98+hUO7u+T8owmsfZcICjERqrbX7j+FlmivntIC3O8AJJ+ug+tbN2pCu1zoVKo1PLTwsNTKSAEkqSQDsTI57+/wCFvYHAOUmnVPRitQ6MyVGJ1P19bWOtvQkGTJgmPrzs0GKp1pRQZgSnQa6T+dlenr3EaFRKSPamD8vo2rubbZGCWezpVqIVTNEHmUja3aAw2oFDLaVcoSBr9fdYo1VJJAIzDaNtrDpey+zz3ObUifnYWoU7ApTp71raYZqevqmRMy08pPwIP1NlBjGmLKYhLGKb4T4CudEdf4rIGaUgKBMb89fzsIFKju5Y9qSdrSEjxsVHhsO4Top+I+Pqf2MXXvOvt1a1j5k2PU/GDiSyMreK6k9M7TS+n8yTNmSFEg6nfeZ/p+tuiSQCFGDMg8yenW0xUzN2cfVRMER3aPRSIzxz4jte1fLLw09uja/6Ug2UGfSEx63o43dLpEHv0yh80rFouJVIg65hEb23mUCM3MQCN4ixBXVLdnlCNHA74QphpvSMxUEkVFx3S5z7gcQfms2U2PSKvAEJqcJUyidJRXKTr0jIbQe0qFZjoJ5jb6+uVj4JTAKdY11+tbOcVq2bPSGG0zvhU3s+kRSE/wCsYTeRoD3K0K+9A+hY416QmG5+1uC9kx/KWlf9YtA6c2TMI0gAwNfgbdQRHIRr4fXSzjG6tu7vwonCKY8lYFnj3gtyO0ob4bkfxMNn7nDY6zxs4fOiV3jVs/56Nw/+EG1cBJiDOoI0ts94QR4b72KMeqh0QzgtPyurMt8W+HjsD+0aEE8nKd5B+aLHWuImBX9G8V3dP+J4I/8AFFqsBR1BXA315nmLYFZQf8R57WK3tDPzaEM4HEdnFWzaxdhN7RjE90r/AMta2fxsbavK7KkA095Uzs80PJI+RtUMx3cyhodlDy+Vs7MAQAhR2ggRYg7RP+JigcCbyerigpUJBB56W3lPIfG1O0uraMoWtvrCog2Ms3zfFPBp73rG/wCXK8tMHyBsQdohzZ+UP2EeT/wrdFJki2ssbDQ2qk3jHFzRQGsTXwIjatcjb/N9eVj7XEnHTMZMVV2n86grT/eB1sQdoYubCoHA5OTgrP5TMHW2ik7jytWxri9xBZ0OIe0H+KmZJ/8ABY61xux417VRQPf/AElKNf8AhIixW4/THcFCOCVA2IVh8p6WwDTa0Cs8fMYJGZ277ncA/wDUup/95Y4x6Qd8pI7fDNEsEEAoqFIn4hVjNxqkPP8ACGcIqhyU0uJPbNmZkED5W7KVbHpaF1ekM+p1n/5HNqgkri8iI0j/ALrrZRZ9IO6yQKvDFWgk69lUoWB8QmxBi9GfjQ3YZVN3apXiNbaA1tGzHH3CTpAdui+W5P8A3bKh0/7yx9njfgJ4AuVFcwSf46RR/wDDNitxKlds8IZoKkfAU+oi2Qf6WaDXGDh04Qn+0BQf8dG+n55LHGOJWAqgAIxVQCdB2iij/wAQFiCsp3bPHqhmlnG7D6JxgTuLYNwPwsks4ywe+qGcV3QszsK5v/4rHWr4uiogsXvROf5KhB+42IJozs4JcJ7dwUZG2otgTvbG1trEtuJUOqVA27CSBt8rIuB2KgWlcbxOtsIn87dZfD5W3AJ20s11GxQcTbevW3URrbeXpZJWXBAO41trKI1sIQR/W2o1328bLVTa0LkjlbI6i3RTbcGLRRRouI8LbCSNNyLdAco87YBzsk60ExuOW1ugBGmlsANuh1skwN1wUz5m2ZU9Ld76RbNedklsuABO9uQgTO9hIjS2tzZrJ1yUAjTnYJdM2r2kjXwsYgRobaMTaJATglAIpkI2AFtPU6HRC0CPHWxkjwtnKw8o2RLlJwuxgKzBsAjnFjDbSUaAaWHO1syjff8AG0crW7J85O6I1F3sPvJWpsFSdASNrCGnT2ZbIBB0jlYwBOgGk20oa6WiGAG6mHl26Rqa4KOjUv1ZhCAskmBGtiNLg+56C83Lzortp2ah0krWhsBSid5Is5YHObai0DE3opiRyTbxuqlvOgeu+tpkP076FNutuJCkqSRqCDuLQ7W+jNgYVxfpKWoaZJJ7BNQvswT0EyI5a6WnIwExAsGUCdQLJ7Mzct0mOyG9lG/DHg/hbhoHBcN3lC1jJ2jjilqCJkgEnQE6mNzvaQ9hFhCgDby2topgaHSzMjyCyd7y/dcGYtuARoLbgbC2xIG+tiAIaLUWhdb/AJXDHlt94NjPu+VizJy1rySPaAUPL+pNjOvOyGyc7rUSN9bZOu9tnTW2jyjz2s6ZYZ5W142yYNsnraTQouWaEWIXskqpFrTuhQcHuM2PconnYB9Hasrb2zpKfiLEy3CgTlIQRhUFPMSLckachNuKRZXSNFUzlAPmLdkEmAJ8rROyfmjlb2rmE6j1YgPtZktEicqtwdxtPUWiDGtNVVl206KW8lO0SaxmrdbZ7M+uBvcBS1ACeomTAkWlW7EOVtFUi8EAQA81TTIQobFRjVW2mw8YksDGLLLVU4cpzNkq0EyI+W/ytyNfhTquodODYGw213utCmmbG0sOvNRVju6E4iv25PW7iZ9RKFO1hL6G3oSUltpboClIaJCtG5Ku6CUpJUWTxFevNi6K697n4YYPRW3fSuKo6x55Lwpg02rs1hosCYCQQiYmJjUhzXzidiVUtwUK7zq0lSnmqNSUttLMqV2jqiEJ10iSrX2d7IqaavfadqcSvFKCoBNFQjMFZoHZqKhLkwRIyCCZGmYCdgVPw+NOcz7eVgkKt2bIwWCqSjj5d7aEt1fFHE5fSAHSze7oQVjfLodJ21Olss8qnhE05UurquO2D6J5S1FymRR3clLCp1QAgZAEnSE93TTS2W5rg4d/D/8Ayj3l6/z1T1eYUHFlTQKQqdQZ1t2w0AlJicumgH0bHq5tindAfdS0snKAo5ZPQdTbmmCFpBDyPNKwR7vr426dsheLhaXDDTqtttZRGUq015TznlH6+NhkII1CZI5kdBtHy+tRUNACEKhQgDKI5bfG3SWURKCBz5QAOXyn32fMeafRdsOrZhCgNwe7Ikz4an63scp34y50E6bxr9b2KJCSQG1BR035ba8uVttZoUEkpn2goROvl9fG0NSkldqqMyTOUaAySN+XusOHErSQVKGUCTvr9ac7IyXVg5pAB1E6jx2saS64mJJMwZj5f152ikErar7ySDpmnUx+e/L+uABI7yyRzER5WJofaJhJ19oGdrGEOEApHI+IPy91olSFkKNUnTNrI393lbsJGbvEnXQE9frnbhKkFWRSpJOutuoA2KSN9Oenj5fKzXS3XTcEhKVHTl9fX3WUmhrMnXcT9fQslpISuQIOhk9LKbIUUAAEc42/rYL3IzAhQOoAkb7gR02+vkISmII02idd+mxtwAAkETqRoPv+uttaD2UgEgASJ90WDdFXahGgIAAiR5W4zDUZRJB0Enl9fO2ylSBBOp11525WmJjKQkHTcz+VkHJrBYDmJAOg5qGk+62JSQhSAkE+HMG2kJJASVQDvpvbsZMw0zTuBaWZINCxKlRpA7pIO/1+ttgaKATMQBB05/X9bbjLJUdY0OWOmnytzMgkCN/hZsxSsF1KJzK5aGfmR9b25I/mA1122t0QZAlRJ130H1FtEmAZJBMz9ctrK6VliU6aiYMEA8o/rbYBUNADpzHL6i29IgQDy8vr7ra0UkhEQI25fX10srpZQszCQQZ06WxWaRm1zawZtmWMpKtQZJtikiTCfa05QLK6kGrFFM9IOk24KzBPdGmwkC3ZUCsAkQBsNR/T63sErVYTOm6tBr9aWldRLViElR0SASdQTbshMQSdNBPPS2IkLBBG0CTrbQSQMo8xH3zZXUcq6BSqZTuAIOh62wqCSTuROv3fn8bB5SATmk767eHjbM0kHLJ3GtlcqWVCBYJg6lPe+fP6/TEqnULO0z9fW9uQRIO8aax9dfoW5zEJ0ImAd+XSzJWCFzARlnXTnv8AU22EoUkhQBM7lPI2CzJTosHUaEk7aT9edsUojed40/P6/N7lItBQg7JKcyEAEHSBHXWxpm87wYEsXjUsj/1by0/cd7EEHXukzNu4TlKTpG3P6/WziRzdio8Np3CV28WYnp1DscS3sgdPX3o/8VjzPETHNPOTFd4nnCnQsj/imzbGpzATvtvr/S21AJGUp8ztaYqpW7OPqoOponaloTxZ4t8QWpBxGpR6Lp2DH/J+NjzXGnHjUZq6je1g9pSJE/8ADHjZhBZ1GoI3gW1miSspPU+E2IK+pGzz6oZoqc7sHopKa4642RAcobncJ5erOA//AITwPKxxnj7fyT9vcd3L/wAinEfeTaKUqjcmddYH1426zqQNTpzmJ/rYgxOrbs8qJw6mPwKYUekC+NHsKtkTEpriP/dmx6n4+3WrR/DtUjWPs30r+8C0I51d2ZHIjeT0torOsK0iTp93xtMYxVj4vwoeyqV3wqfWuOuFF6O3bezXU9m2Y/57HmeM+B3P7yorGZ/npyf/AAk2rulzU6AjlpGvS3ZdSpBHOSJ6+FiDHKpu9j9lE4NTHqrJM8WsAPaC/cp2IVTPDX/hix5riDgl7RGJaIHfvrKP/EBar3aHMCCDOxjf7rCIdIGYaggxztMdoKgbtCH7Cg5OKtM3jDCTh7mKbqM7f643+djbN83NUa097Ubv+R9B/G1UA8UjMeQ38Ldl4HURAnf8eXWxG9opObB6qBwFnJ5VtkKbWJQtKh/hM23lIExapCH0NglshMExACfr9bGmr7vOmEMXpVtRqMj6xHwNijtGOcf5UDgJ5P8AwrXR0FtAR0tWCmxtihkkJxHeXd1M1Kz+NlFniRjJtP2eIakEie+Qr/xA2mO0UR3YUM4FKNnBWOgzbIJ1mbV+p+LGNkGF34XP8zDX35bHW+MuLmwM7lG4T/M0PjpGlpjH6Y7gqJwaoGxCnQjS2iNBFoXa424hTo7Q3ev/AHFj7lfU2OtccK0D7e42FRuUOFI+c2K3HKR3NDOE1I5KWcvUW0RPMzytGLPG5lyO0w/5xUz/ANFjbXGi5lf3t0VYMx3FJV98WkMXoz8f7oZwyqb8KkAjTS2iJMmzHTxiworRxmvRPVpJ/wCqxpHFXBbioNe63/nYV+E2MMQpXbPCh3OoZuwp2Eac4to/UWbieI2C3YKL7QOWrLgj/lsOjHWEHTKcQ0Y6Z15fvi0hVwHZ49U3d5Ru0+iWSP6W0Rynaya3i3C1QYZxHdiyeSaxske6bG27yu5/RivpXOmV5J+42IJY3bEKBjeNwhjAi2trYClWqSFeRm2bH7jYgIPNRsRuizgy1rawfbSUx8TYyI5WKVi+zXTrzR9oEz9/3WNTNnGqYmyyR10trz52w2yZ52mBZQJutTrNs2E2yTrblci0gNVG9ltREwNR5W4JGwBtsmRpztogDcx52Khk3RCmkds0f4HVD3HX8bCmBztwRkrXdNFpSofdYRPZpUVrbKpjYkbG1eZz2x5ohmPS9lMAX1QlGspfE7KkHlvaMsV1uH8U1T9Ca9qoDCRT113heVaVAyc5BCikiIiARvIOkjBlpb6qjOpDqiCDnJJ126QLQnxi9G+7uJd5pxPhrFVRcN9pUFmraSpbShBSMpQpJQqYJjNMDQb283xTtTWUVSGVlOWMO/O/Qgj8rQZC0tzMNykl+gum7gKO7aJuhokLKGmGx3GUcthsAdTHxtEF+4ixtdV5uX41w0U9eDLgoEqTe4qqdpAJU6rsktoUnlmc7ydU94JylU23fw2xPgO54vjEVbjV5tta0gppqNU5MoRncKirvEwYHtTrlSmyFT3PfF9X+HXcMLud2jQ2F0nZ0ricsAlTjqUaoEQMqk90pISCqLZtT2tbi8nAo3ZWMF9Tv9lYipQwZ37lV3q8OYK9ae9evhwVPaK7bMaGc896YqDznnbLT07gThE46td5YdxG7VqUS+5+5anvuT3joobmeQ8hbLcycQk+f8q3wWdFBH7SN91PCzC6W82ZV+hRCZB0pXQNj1Vt42p9wmerH3qY+sPFJeQrL2qj/tIMz59LW3/aUOBOA8I00qld7Oq0J2SwP/i+dqqcHWypqgfcKSXX946rBgdfZ1529c7INDsNjzdT+6r9o3EVr7dB+ynBg1aSnsqmoSsHQJdIkTqLGU1F7wkKvOrmeTy4A6an8bD0rUkhPeBkgAagdT01H1yUUspCMypkDOQdQI5z9bW7gwxn4QuWbM/kUVRfGJGtr8rjJghSyoRr8o+8WHRiLFaQQL3dzDTvoSdtuVh/VW0d0IG+mms+X199hAwEk6SToBMAkeJ+tB7hmlhPwD0UxUzN2cVprFWKVGDXIWoEe20mT8o5WNN40xQ0hUrpVmQYLZ0PlPXz+63Ip4ICkknXUq1P0frnbXqiFRAjkIAMD+htHuFM7dgS79UDZxRxviJf7aIXS0SiNBAUP+rpOsdLG2eJt7ApR+6WN5hLhEn4GyP6ohw5VpB2khMdN45W36kjQGNDsdvl7vjZjhVI7dgUhiVSNnJyI4p1MlK7jBlO4fiOp1TYdPFunQftLkqJTz7RJgz7vocrNY0pSk6k6+Ex9bTYF6hbAkDluFbRH6WF7Eo3fCpDF6ofElq8/SMwlctYilvS5b6+0QHQppppxIExBlYM/XPR30PGLC9VStP+r3kgOoCk52U7ETOivK1UOLCRT4hpwkZZpUlJA55lDnqOlpXw/SB26KLMASqmbJnSZSOlq3/T1G9x0PqrIxyqY0bKZE8W8JjQOVgJM606uv4mLGWeKODXJi8XUkclUzgPxy+fw6Wif92JUAlCCDMzGp167WEF2AiASdYT3Znf9LRPZijOxPqnHaGp5gKXEcQ8HPwEX8wgnUhSSnUyOYsYRjfCjxTkxFQ6TMvJAHxOp3+jaGTdiUHIlESNp5bW4/dzcglIAUIOn42Cey0HJ5RG9o5ubQpyZxJh9xOZu+6JUDRQqEH7vxsbReF3KJyV7BJI2dST4c/D5WgIXQlZClogA8+Rg87JOIbjbN2jMhKe8IkjTQ76eVgP7LMtdsn4VhnaJ5OrPyrNJcbcTAWlQB3GutukwSkAwU6mOnl7rUTNzOoZxC0y842S52iileUwUDXQyPMW4oziK5GbnqF4ivZoq7R10Jr3QJUM2ozbDkDtFqR7OO5P/CsjHm82flXw7oKVE5Y5D7/G2ZSVnQCNByg/DwtQ+6cRcSW27xWrHGJEKfp/WWki9nzkBUqAJXA0jbw6WNU2OeLlHhWorE8Rb+VVracfC3atThQYVCQFzoJGnnaH/Tkx2cFMY9FzaVeiDIVl0gbGdPxtrJJGbXzHyNvPZv0kuN93qpEDiHUu/ZCUuUtOrOqSCTLcjXoeVnfeXpLcX2K1qhuzEKFpQ60h5bt3sqzZtx3UDl+O9gu7P1IOhCMMcp7agq7ZICYUox8PKLZlG6c0zp+Fqtscf+KTKJeXdLmm7lEtPjqErHSyW16VnFf94rof7PYaezVCkolh9spaBMrVDx5+7Q2G/AatvT1RI8bpT1Vt/AaxAIJIPX8rBN5ZnUk6zHW1c6v0mcXtU3af2XutYQMy4fcTmEbc4nTXW3OHfSxvW9aJVVWcPGachzIUN3sVcgebQ8R7rQOC1bfh/KmMXpHa5vwrH5SU7wIg68uluAFE66j2o52gCg9Lahr7zcov9H9elticzzde2sSOQlKZ+Njd1elphCurXrvqMI4hZcpwVKUkMLQqI2PaAn4WGcLqx8Cn7SpfnU7KkQE6dJGptoyQAJTsY5D32iVr0mMCLJbduq/mjMwaVpQ+Tp2/Gwa/Sp4QMPFirvC82SlWUlV2uKAOuncmfjaD6CpZuwqYrqZ2zwpdMqJSCRzkdYtpS1yZ0E6D7/rxtFdB6T/BO8nwyxix4Oq1SF3XVjlrqW4sof8AlBcGkDI9xCu5pQMw8lxv3d5I6iwzSVAFyw+iIKqA/GPVSHDgUQkanSPrfzt0ToCUkwdI62ZtLxk4UVlMKtniJh4spJClqr0JSI3kqgfXusoUfETh9XJD1Fjq4Hkq0Cm7yZXPl3rDMMo3afREEsR2cE4QtciVAd4zrEfWmvhbtBgyrQRvGvw87JTeJMOVaiilv67XiDJSirbUR8D9a2PNVDDolp9Czr7KgZ1/pYZjf0Uw5pO6MIUJjkByj5/E/O2AkaDSJ5zHOwYWFHMoqgHkeWn18LbzqSlRHvnX3wdrRylEuF2VBSgRruRH10toqC+6IgxB2A/KwfaFQEjz0PhOn1vbAtJMKEx3QFcvD5xZWUbC6FCiU59QowRPKLdQCSEg/pv9edggrMnvE67SLaUsAZSozEAcjvaJKmuwYICY1OoHwttJEzqIgSRM2DU5IOswdTER7vM2wuc8uhEifrxs6VkMleVZIUQRsR99ulOKyxlkRAnXnzjyFixWCISFaxuRB6W0FAR3Vd4bbkT9CyToyFKkwYMx1k6c+dug6QmZGus7/W1ipdSUwhwCOXIfUW2pxRJJ0O4gaxy++zWCSNKcIUcsmddtBytovpKjBjoQTOuv4WJl5RkgzsRJ6iwangYO2mYg/n7trSDVEmyP9uToTqfZ12130sEqpHsyRrIJHlNiaqlJ0JBCvjFgXXEpEkiZJiY08PfZw3VRLilNqpBVMgbETNjKKrUCeUab9bICKgSJJ3MTsff8LG0VEphIJJ1B8Pj7rMWJw5KwqjKjm55o5/hbaakwVZ9dBqoiPGyUupKUyVagxG2vT8LcqqFgSBqNJmPo/XiW4afNZLPrJjRZnWTGtsTVd3KIEDUn+vnZFFWUJ3X+m5+vDlbBWLygKJEHy1suGlnS6mqBmDHgJ+H11sIKyEgKOaTAmZ/TnZvpvA6lWyTuDpYRNcEggEkHQTp9fpaPCTh10uOVatSTqd4Jg2Lmr725lMbaT9RZLXWAolZ1Gm02LqrkhRVMfAg6aizhlki5ORutGxWMp8dvz/pZTo6sFtcFJnfys0KatzDTNodROtlGgqVnOkq0KfcRaBbZLdLpqydDG068tpsEp5gnVttUn+UHz+6yaag6EqV4gjQ2CXXJQM4OaRKoO4+hZ81gllBSsmoabOdpCUET7Jgz7udtP33VtKDSbyvELUe41TVTqVK2/lULI1JU1d6KPqaQloEzUESmP8P8x+X3Wcd03PTU+wzuq0W6rVa4Okn8LMJJHGzSmLY2i7gnDgdnEVbfTNfeV/3mplkLy0i61x1pRUIClhRMkSYjTztNaVZkg9QCLRth1k07YUBEQR10tIVAsqpW9ZKQU/AxPyt22FROjhAcbkrk8ReJJbhGDrbRVp525nTe2iQdd7a4aLLOJXRX15W0VTrtbmfG2ieVnAsok3XW1smIOmh2tzOsb+VsJ5mzlocLFMmRV3RjK5L2L1zXh69d7qlKbpnSntGZM5Ez7SZmNRAPQAErXY9vy43xTX1hvZIKnUdqgBJMaDIoEyU/xRvrMAviukobcSfYWknlodDbeYwRy3g7W4ebsxiFJK6TC6xzAfhd4mj6X2V1tVG4ASsv57KOF8fMI0rymaihqUAJIU4mHFZp0AToIy7nltrvZZf4/wDCu80UbbN8qYeeb7qC0SJy5ssolJVoRCSTNnU7hnDl407tRfVyUFQh320rpkFThG0kjxPxNilbwywFW3EXWMMUzKHKz1tbbRyDtu+kKJTEwHFGOsdLYsrMcjjk45ZIG3Bdb8bIzXQZ25QQmO5xF4fioXSoxdS17j7SnBSsKCh2YgKK1L7oyg7CI3M72MUmELqxAh69KW6WmrwdUGVoDhabyIMJBMFKgmVQcp1UYiZshYj4HcN27s7Wsqa+7RSNoWqqbqsqWw3uvvEhOkydI5QNLMa+cLYyuXDVy1XC2/7/AE0ja6pp1uoS409eXbrCWsichyjKlULzIIkuSmc1vKMQpe8zNe1uRx0NtB+FsZ3NYU4rxuTCtDeFVRHihTUBp3ltGkRWUiE0+VRHZhJZJSExEE6RbLEWeFHEBTKFJuTBNOCkEMvUNM843p7KnOz76hsVczrbLZ/cJvmP4S4jlVP9pNUpbuHA7BOUqrq1QBjk0z+dq28I6eWrt7MSDVg6bATBB9yp8vdawH7S99xDGAWCrRTt5LO2kJpgNDrz5HwtBPB1A7G68xzpU6lUE+z3x9Tb6R7JtLcNi+/7rP7Q61sn0H7KeKenSIUsSSo6DkdOtjrbYJC9DmICoET0tlM2pwDMgEkHNInT85mxxLYKc5B08OfXT+nhbuFygRfIrMogaROkR9fl8BEtNgxlJA32P1z+Fhy1kMqGmggE727CTIzJlKo5x0G5sk65CJMGTCgY3M68vra3RbkBAUZIzHymJ+vnbaUFaQsDQp1HP3WFSMqJ3BkwTt5zZDRIhFyjvQgRBInqT08bYWwO7Mge4T+n52M5MyjOm5nr9aWEbQQsyCDMGRyHM2kHFRyhFcpSc0kkecxufrzsAtrKMxIRzKuWn32PKbCiEqSCmZ328PDnYFxAyBKhA9k676nw8voaECE4KC+MrahiOiISdaQZknYnOrY7c/6c5cw22n9y0BKSqaVklQH+FM/1tFfGhJRiChJghVJ/ESNlqJJjfl8JtLeFU57luxYUrL6ozrm1IyD42gz3ipP9wJZbaISUlsTvH3/WlhlNKIyJza/ETH142xtGcAHQ/wAR2n4+7ysaaQYOg8OY+trFQwEWLQ7xJMnWIGn4HT77dFgwEwkbR4fp+djfZj2tI156T93LrztvsUrHsEDw+ulo3U7IoWUpJ7gzAyZEST9fdZMxAwTdxICNFpmY67n4ffZdDZXCkchtoeX52T8QJX+7nNUg5kED3jW0XHRTbuoZviA/eFIyAFVLzWYdEpRJPjsB7+Vhb2ok1d03YyhP2joQhJUdgpOp+HzNlKqpEO3teb24ap8gB5lSdfkPn4WTW+1rKq7lQezplMsmP5imTGmp0A+NqGbkrFlzeLTF33g72iE5fUuzSOW8DTz/AB5WLUNMheGkpyaBtQIPvH5Gyte9I1WX4xTqjKhBcWAYzZRIHjqQSPKyMmqcpbhbpaZMvvFSUgEEAZjJ6xFptfYprKBb3pHmK5Tysqe3ddcKdw0M+VKd5OieelpRormYo7rulpuCpTiHXFlMFaiJ931Fo6xE4WqhtDolP2kAkDXOSmeca9eZ87Sqwkqu26Vbn7E9f4Z002P4fCBsnA0TuqA1R0q6l0oyNpiTzPT5WRsN3emorK2ucT31wkEkbSZge762so1ua97yRdjTn2KCHH1Ae0BGnXpbq4WQ1X3gltEDtIUEnQROnhaRdchK1gha272QytISkqAMHmRGv3fGzWomE09xrZbQO1edUG4TrsNdeY39/us/H2dCHEgI1EnrtZGue7kVdT61ADFNDbKDBM/zH6+60HHVSaNLopdmHG7vo0oDaO1UQtxQT/F+m30DZKu66kJxLVKDY7wV3ADA202s+XWgDnI9rXaIjn91kFhnJiCoU6YzJUdTpGUbfP6FmcQLJgCUUvWjFJTKAZCVnRIVy8fAAeFmfUXE072knvkZySYzHX8/nZ53i4qodW842CkHKmfd18vqLFmbue9Vqatxak5Gzl19o/kPxsF7s5RWtyjVJFwYWpmKUVicgdfkkjUAdBOu/WzB4k0IbecEGcqenvEDXl89xaZLsSk3WxBJ0UN4jvfrtaMOKzYS444dEhgEmRroQJ58jItZAGSyCb3RTBN3ovLBbqlnMkLdSYSCSNT+nu52cd03MmiuVplLIzrnKnTNr0/LnvYjwkdQrDdWHmwsIqlSkEHTKkhPnr8xZ+XFTprX1VruVQbgITyST7unjy8bDIBUwSBdIl34KapmnHXG0l92C4qNhyA+P1AsYOHuzWClJSdwdR3hOvh8vyea2kkynNlnQxEdD+FuCyAdUcpiDKfeLTDG9FDO7qo9xFdl4Ju9Cma+rbhWYFD6kyNddCAN7GrmcxUzd1N2OJb3QpDeXMK51K9CRMhXn9buW/KUOXevKjREKk/l749/nYG6adJu5KSTCSrKOe8zPx+tLC4UZdqFMSvtoSiTN/cQWyezxtf4O4AvB7QjpKo6fKxhOPeKlMO7ja9lqTIBcqCsnror+ulj6aNKBJQoZRzgz0H3WTbwCQU09OkqcWSICeXTyGnwtF9PCfhHopiomGzj6oGt40ca2F+rXdjmpLuaO/TML0I3lTZ0++OQsoYp4u+krg/hg/xFXfrT1C1WNUYfqbqZCHXFSSlMIGgA1JO+mpmF7h1g7C1Vei3MXX01Q0zKO1VmHffXE5EkAgDqTy+QvpT4oocTcNqy4KKrpX6dllK2GaRRDFOlsQlOXROaNNtLUZKWHihjYx56K0yqmDC4vPqouub00+Nj1DSVFQbgdLxhal3eofxRPdcH3cjpaa7m4/cQ6ulpqiqoLmUt5tLpCWXEjUeKzHS1G7lX2V00iyUhbLyiNJKe94ef3WtzhRpp65aFUJI7IAGOmnP6/EU9BT5tGBWoK6e2ripCTx3xZlh24LscgScq3ESOe5Nkq/PSkvy4KhltfD9ipQ6gqzpvFTeUjSILZ/PXaxJujbKMyZlQjMNT8fePl7mvjS52nfVnSgfxJChGpOU/He1Y4bTk+6rPf57XDk6GPTGXH+s8M6hJ3IbvZKtCd9WR9T0saR6ZGHf/AJ7gS/mlRPdeZWNh1I18LRA7c7JQSGQMoBjw2Aj3jpvYBVxMKSW0tp7o1JOoPL36j4D3L2VTnl+VH2nUDmpuR6YeAlJHbYaxM1lmfsGD16PWNtel5woWtCapF9UxIMldATH/AAqMjpavb2HaYBLgZAVmgDLAEdAOcWKvYcplgpI12MEg+f3mzeyYPNP7UnHRWcR6VXBgpleJ6tnQBIcuupHjuEH4Hpbs+lBwOdjtOINKwSY+2pn2yNBvmQLVQqML07g6zCenWJJnx+jZo42wkGsNV1a0oyhtLglKYhKhMabxvEa++0m4LATuUxxidvIK9NL6QnBeoyqa4mXCkL0GasSjXc+15iy7S8YeGFWlKafiFhxcjSLzYk/80g/lbyjCQbt9sgtPkAQJOYfH+HytJdzYeYqqBiqzD7RlKylQkAESZ16CedmmwOJguHFPHjUh3aF6UMYzwvWpilxJdT0iU5K1pU9I71jib2onATT1jbo1EoWDO/IG3m8jB7Trax2bcKVuoazHL5WLVGEG0oIZSEagHQjWfDw/rztV9jt+b8I/tk/KvS1FQpau6k/zbG3K6gpCcxUDyMxH19b28z3LrvmnQUUF41dO4ASns6laSDsBIIj+lsums4puVCGbpxriRnZIKLzqEADzzac/CzDBbj3/AMKQxgbZPyvS4ViidVTqBBM/1sIKs5j3zoNzbznqsY8WsPIqUq4o4peebpi73r3dUEHKCAApWuubWOVvQv8AZ3XhT8WeCV4V3EBIvy97txA/Rqq6wZnuzNOw4hJOhgKWuPfajX0JoYeNe4urVPiLZn5Mtij5qEwUp0OsgdOmti5qiFHKdxsTv5bWss5wqwC6sFWHGgRp3XXBpHgqxKp4LcPHUn/sVxGsyiqdH/VbD76w7hXuICq/U1WlR9rSPM/W1lGmvBKVxmmQR3Ty52l1XAjA+cqbXejRIiE1UwPDMk2S724DXOKCoVdOIr4YqA2sslamVoC4MSOzBI20BGnjrYT62MorHC6jV683C6mnZK3HVaIQg5iT4R/TX4H6G4nqpSKm9FglGqaZJ+zH+b+Y+ER52Gum62LraCAkrd0Lji9VrPOSPHlZXZlXskkA9d7FAL9kYkN3RylaSAlISBAGg5eHh+ll27GSVhMjTWI2sl0jXf118J+tP0s4rsaOYCPr8OVtehpLuuVm1U9gnJdyQ20kZrPK63SqkE8o+YBsz6YykQIB3s5bidzNKTGwAHun8xbsKZuUWXLzuuSUqGB8banlOm9sVrvbU8totcVRbm2TG/W3JM79LamCRFkkuiRFtkwIm3OYDf3W0SOR2sRouoOKDqgVU6wD/DP425bV2gSQfaiwiwFAid50sVo1zTo1hSZBM7EGzSNuNEmmyUq+oKECjbVlSkArG0+FlC4FdvdlVTkAlBzDnuP0sgLJJJJ1O5ssYUdiqeZ3K2iY6kEfgbZbaFlPSmLfmfPqiGQl11H/ABGuqoqVJqQhD9Oy1nFKoEIUvcqXyUAIhJEAyTJCSmMLzvXE17KqLwF6OpZpHwXVKJyoQQo5UnRIVmLZyiSRrGk2mrHN2P19E7TuVK6OnAUlRa0dOh0kapBMajXe0d3Fd923vclbhp1TdGhNZTpT2RgqK1wkRtEiNAJ6zJt5RiBMMksnC8LXN3Gtr62C6mN+aJrk3W71xU42lxqVoUkKSrt3u8Dsf7s/fbLOV65eKSnVqoqinYpyollputShLaJ7qQk6pAECDtbLQ7/S/K3+fZQzu6/lUG/aVvKVeWBGEgZkNXkSOYzGmA+vC0QcIkZKe5pEAuiIGhAWB9eZtJH7Ri8k3jfOAXAsEO3bVVGUnRWdTUjwMDf87R/wUSU0tymA4hIlBI0iQdB7z7yT429I7Li2HQ38/wByqOPm9ZL/ADkp2aASNikpAO+qdY+jZQQlBg77FSYnTnYGmaTGpEATO342NMtpSoOJiRqe7rH1Nu0K5Zi67M5YkmAYB2nztgbymQM3Lp5/hYSFAaKEbQR7/fbbYUUTOidyTr4/cfrdlPZBoQgEbmTHiNNP6WES0kiQiSrQfXw/QW6y94AnbUbaH6Bt0EEJIBA0gac+k/W1m5p7LhSE54ClGFSRzjrtboJU4EqKozAiQNhpr8bCBoklKN9Y6+/4W6S0QrMCRGsxtvPO0m7qB8kGpvPBUIM6dB5xBPLf52LuAgArPdXvpPWx0tiCDlOgOmgGvv8ACxd9sEEpOkjnqfHxP1pYgKE7RQZxqQE3/QqOX/zQxzg51a+7r4Wl7CaALguskpBNIyCUjXRCefx+t4e45PKYxBQKUTrSGNZjvqncdPqdbTNg1Pa4ZupW00DJI13yDT7t9bQabuKd3uBLSQZKROWZAj4a/X42MBJI3WDI5azvrP1zsEhpSITm217p5yPzsZQjuyRmKRoDpE8vjH5WndIN6LBmTmkjWNfztsypMkH4cuk8v6+YFyiFKBSrcpgfKOv1pbZaK0yjRJEz9dPqLQzKeRcFtt2cqBr743EWTL/Qk3O4Z1BTBPPUH8N7KxCj9mEazESfLymCbJ+Ims91FRQUwtChBAHtb/daDz4bqQteyi90ttVV7uOQlKEImTzKNuvhYqpluguq7HFFKQt8PLOpjuk8t401t1eqKh6tr6OTlWQ+4QmYQhEhPxiw95ssv0V00IVlU/2YIgk5con4gx77Z4N1bLbJKo0OPXkqrqMx9aZcdyie6g+yPKBYvddChFG7eJbkraWhnT2UDUnwPP8ALmtXm63R3q4pxSAlqiCQBuCTAjnrG1ilKhCMLqTkDf2CiSeW/X6199pNOqidlX/GjRpn6YQFFRWVACSQSCPHcWk27Uo/svdlQ4AUtNNEZjrITt933+UZY9palm9Gal4q7J5aQwkz/dhIOY6fxEmz6w2upvDD7JGZNPQ07bZgxLmg5dPw8bO86aKLVI+H6FVPSl99RNRU/aKJBA6xB56j5W1c6CL5vBuSZykz0/rZQuxSnKKnehXebSYI8Jj77J5qEXbfNY45KR2IVrpJ7unhaDn2AUgCSu72ddqH27sphC1n7VSU6BM8/G3GG0LS1UtGe67IEb6Gxy46NSkLvCpb+2qSdDyG+lgsPqIfrG0ggBYJnzMHX6087QzG4KJawsj5aK+6ogAgaQTGo+o/OzdvbsKeqdQ2SpxyM5iMqQBp8RPw5WcNfVikpy4QS4rQCdvrrZqOrK3Vd2XHPdJ6WhLJyTxs5oJqkdrKhLSUwZIJCdBG+mll6spG6W7FtgiEtkeY+P177D3TdvqlPnWgdusBR5EeH3W6vQoNE8sEphByab9NPrn7nYLC5TO1KR7obP7oQlaZIKok+PT4j32jjiqwFtKKk589OSJ5mTy56H52kq6gP3bK1kgFYM777fO0XcWq0tVtBRM95VRmSokR3BqZ8ADr9G1kP8KCW3KT+EKXqluupW5Qy26HVkARqiABvqcvwFpQwgo9k/3CVSkGTBMg7cuQ26jbW0bcEXgp++KYO6lDK99J74jbTUm0l4UIVVVCTmAgEEkDYxH3fG0Bo4JyLhONLZCipSSeY1326/nbakhw6DWADzsOlGkAgAwIgQr3e/5W0sFMqCASZGu4672IXWQ7JLvBkqpnwRMpzCI0ixK4GvsXACkd8qM6GCBH3fA2W6hsKYWI1UCkTrB26a7WS6JtV2UxU6lQccOYJHtJ6DTn9eAA99ijMbosq3Q0kpaBUpZhISZPhGus6+VuWKBDRDrywt5emugG8gfP42NUtEsE1dSk9sU6g6ZQY+vo2HWhKjmKQogzod9R+vnr75NN9SmcCNAiLzZ3TGkakAacvo2bmM6P1nDN604JhdK4BG3sGB152dikQ2nMZCdo0PXp52SL3Z9YoqhkNkpW2pJ25jUx7+tjB10NVRuBIXcyykat1RV7MwCgfltztbjhs8iqwrdz8JKS0CAozlJE/cd9/K1RbkzBm8KZ0GW3EmOcDMAZPP3aT7rWo4K1SarBVGAUnswEiCemgM9NeVqE+hV+nN1IiEgJAUAQQSTOpEneOW39bI2LqcGibWmCpL0qy7TB2+htZaSTpBMTATMc+envslYoYcXdayNezWlUGZJ1n8dPO1cHW6snayZi2gV51AFIgrIIPL5crBqaSVShISrLMTPTY9NbGiVQViNeih8Pr8bAkKIEqBUvUEyDp4jz+fvtO6FlKKvNlXdciBAJ2B3nfwn6mxdynASI72ZRMxzjWden3WOrHIQpKRIMz8jz0+psC4DEEAzqSZUfv6zr158rSDkxBCSnWoKgITExlEyfDf7vush4hoVVWH70pkmC9RvBIAgHuqgT5x9HV0Og6JKSJEGBoT4AddLEFNNrXkUkZXDliNwRB/H84sVj0J4uquIAVd9W2D7K23B0AkiZ/wB4b/jaXsBn1jDdA8pBSpLYbHkFZZHwn32iembLSLwpCAn7I7nmlQPTwNpO4XuqXcLbaiAWX3G8pXBjQxHLVXzsSY+FDjGqejdMFNwptMxBAJkDbltPv0FsqaVJHdgGQoZun5ef42PUTC6hQQhuSNxMjnpZy0FwBoBdWiVghRBGsxvp5WomSytNjzJrXfhior3gqpQUN6QojU6A6/1jfay8i6mKJnsmGkJ3IjnJ308ffvZwBlKUZUaRHs7j6j3WK1DBSCoAFMSVAamPMeXwtHPcooZl2UW44owmrJKSQ/TwT0MFPlOgtbv9ktfpcuLiJhtxzv09Vdt4oSVa99DzazHL+7b+ItVbHNPlbpnUp0Wl1AMTB7p085+tbTV+yuvlV38cMV4acXlbvO4XyhEmFLYqWiOgJyrXrvali8fFw+QfdEpnZKhq9SlKzBKxztysbmNvlblBOTIRJSbdnYE9Lec5rhdBayLySY8LcOd5JB59bYruqPibcrJ6687VXuRwogxPdqqS+ahoAgFXaJI5pVr8NxYmw0SrmCBtFnrji7ypdNXBP8zSvvT+NmuhgJUCD7RCtetulwz9WJpSnfZHKNGylAizgu1CUictkiiaBIlMjTUdOdl6jQEpA20Bi3WUkVlg1Ut0rMCNJ8tbLlyOFLhRAgg/E6/gbIbO4kTtZTu1woqkkiNR8zH3G2ywWWU83Fk4s3L5xbQOk7W523tkzyG/S1iyAuzANtE6c7clXKPfbWbSQR02tMC6i46LeYz0i250gA624J10+RtudvqLEAtshreaNedijJyrebVoQskacjrY2ZA+VipBTVL6KQFaWdzdFEHVbVI11ix+4ngzezCp9olHxEWIK2gHytth0svNu6koWFAdYM2DK27SpDeyUsWs9omqYBGYgKA8x+dq135dj1ViE063VMtzmcWnVSUDfKB7SuSRzMWtDilphoLvR1xLbXq5zqJgBI1k/G1c78pqyrvh6vraL1elzzTU7iJW8mBC3kqEATJDfP8AiiChXA4g8xVRZzcNP2XTULg+nt0QPaYjGlFganXT/wCxUuhU6pSP4SVz3zEd7nvbLAOofcdW4uuqFKUokkrJJNstR9nu6D1KNlcqD/tFgaTGmEaBpay0xdT/AHQnUA1CgnUchlHysyuF1+MXTd1yPLoHnQzTNlWWO8ZPMxvHys6v2j7yFcVsOsqmUXGDIMb1Tw3jwP1oWRgVtCLgu7MpK1+rwVAbwpQ0A8vD3W3sHlMGFwuG9v7oGKxiXEJQVNdPxNuwIzLuq8UR3vYQfOO9ZQb4lXCtKZo7wEawGQdd/wCbxtGjJCtU6a5QdPrp8LH29dycpSFEnXz92/uNtH2vUeSoDDYTsFJKeIuG1EZk1if4ZNOdtddPrawox7hnNnFY+haeZZXEzt4/raNVEpkZuUkbe4/XW3TTn8JXqTMRI/TSyGMT9AkcNiHVSYMcYXWkAXmQo7Zm3E6c+Xn1sO3jDDIhRvZCTMGUKTp7x5fnaMkEBGZAhQ21Ik/ntpbBlKjISSNdYgjp7xtaYxiboE3s2MDcqV04pw2vX990yIOmZUfH65zYdWJMPLISL8oDP8PbDX3ToNrRGMqRGSQARp05aDe3C0oKspQEzoNDprO48/lYgxqQbtCGcNZyKmBV93MQUi9qEmJCvWExoeUmwLl6XcpJQmvpVyP++STHM/faHVBKgVkp03ISPGTp4jrZOqG0IGbKmBpqkSBsPf8ApY7cZdzahPwxp+JEvSEvClZxJdrjdQFAUiwSjvJJznQxv+k8rTrgOoZVhO53UPNyqgp9AQdChO9qecTKhulvWlhOUlgmAIJ1M/Ln/W0o4JQ1+4LtUlrL/qrZPjmSNz1384sV+KGJoky7oLaASeAHZWYbamCkkjeM31P62MpaWRGYmIJ3133tBbSyJKHFpIJEhagN9R9a2OsuViFQK6rBA0yvLEbfgRYftxp3Z+UZuFH5lNqUrACgjUGSSd9bcKQFhKUCCVd46yPH8LQ81WXoFAi96+I0ioWIO/M+djLd832lRIviviT7T50MzpPOyGNM5tKRwx4+JS6G3CAFEnmfHY2IYhZIu1yRGVSDB0nW0dtYhv8A3F9VszrLkgfEfXxt07ft9PMFt+931pJAhYSIM7xG82TsZiItlKQwuS97hE32A5UX5ULTK0NpQCBqAETHjr93xTqZtdVeFBXLJDbbiWGkjolBzGOWv11UC5VqU8fXCF1Byud0d6NNfjbhHrKAwhupBTTd5AKEwPqdz8LVe/tKOaN3VFr4o01l8upPeRTsdq4J10BgQOROvusRvBwUWEgy7MvU6kpOYaq1P6+6ymk1HbPVPaoLtQkocJQIUnb3crJN6Ui1sIYWpPZtJLaO54Dx6c7TbXNuhmjcBZQ7xUZ7J66lJJByiFCTpkG3U6c4G3lZ8YTpWlYNYDaEguskqjfNJnz208rR/wAYKqpbvOkaecZ7JDSShITBBGk76iJ+Nnnw9rKp7DVIhCmw0UrQAUmR3j+mnjaw+qaGhyC2nObKpDw04hV0UjhgwkT3toJ1+XzsDWpRVXy1Qg5QofaKJjNl5H652Bul6qoKRNIylooaGxmYOw315/EWEpaNVPVtVqVBS0KKzmJAVJ5fOwXVjDZGbSuGqcJROVKR3tk8tIgCOnvsjXQtDV716VqgDveIEz79Dv8AfY2bwqITmpW+8BJC4/CxBbalOVD7bZSuoA2cnTy8TFovrGHZJtM7mid61yqlztcwDY0SNNxGo+uXhYK7Uti8qclMyCSOR9qPmLDLux1xwLUgZZ7wB1jw6GxhxsCuZq2WMqGk5coAJPvJsFtQ1zrlT4JAsEuISMwgFI72ggz+ViN5qc9SqAncIUSE68j+thHL0bylAZeMa6QZ8NPrX4l3q5laFtFh0ZioSE/PQ9LWzUsI3QOA66Q2K4UlzmQEuFZSgH+Lx6xIsyMaXS4xd5r6pJdfcQ5EicgIGg+vus+W26cVIqKptamW0yygJkCOZ+uYs1uJV9XexdiVOdoCHMmrSiCMp0+AP67WnBOwmxKhJC7kEyOBr039eTedULpkkjlIWPd/EOfIWlXC5AveqYSpUAGQobd4T56RaFODla1R44qEuGErYdRojUnMlXlyO3M6bWmi5nWmL3cqDPZrCyFAEASZGmp5b+FiPka0i6i2IkJ6AKTA3MyUawR/S3OYgBOYka6kxA931HSwCK+kVC0uaSQRkIjwPxH0bcmto0JEOglIjQHX6+uVkZ29VEQO6IV5wNBRLkEnKJ1InfSyPdrvrN4rU+SVISSBG2oBP11saaNPVPqfqqlKUtGEJGhmN94PKxK7ghm+M7iClolSc61aGNdLAMoJRhGQFIGFsHVV/IXWVDTjVC2YK0IJ7VendSQfif0s9m8GUraRU1NMM5gtsKplAFPXYwPrlobuevw6zRU1QusoA22kIYYC2+8rbvQsGPPebNfGePGKtLt3XJUoGfR6oByk9W24J7um/uEWpcaSaTKDYKwI2sbcpv4weuamqF3fdTVO6AYdqENwkGdkaTp1+iyKtoOtKacSoiNiY57+X15KroQ6jKciSDoREDr9eNiL7SYIXCzqAAY+vO2vEWsFrrPeCTeyqOKcUWIL+oVpMocUlJCfZhyNdOh/K1h+AdSp3C6k9pmLTxgTMyVaef6+doHxcw1Q8R79SogBa3VeJkJVtvyPnAtL/o5VYcoK6mCtW3NRMwAE7aaaqI91gT66hWafQ2U5D2+6ZI7x05EffFid+JWq6X4Ue6kKBnQAEH362HaUEbKEAQOQI5gDblbi8kLVdz7eTMOzUkBPPQ6j77Vb9VbITEhQbUSCCkb69I5+/TpFtKme9OVEgac9dfo9LYMygM57vTbXrqZ5fK2nFQAoAZTIJjYzB9/62WYqGqDcSogbxJ0nlpt7jy8bF3CDIShSkxp4zr7/AOlhnDIClRMZRIJEDX387BrAGYrVPj7IJ16+X1ye9kxF0TdSCjNmMk/xaxA/IfebE1kpJVm0kyEmI8vl030sfWghBRlM6Du6H3aa+Fk5yCoqbJQREa6yNxy+Hl0ixGOUCxV3vOhFLi+9qKAn7eobTmJ5lUHbyP6WenBqgeq6SupUKUOzdSsmTpmTEf8AKLEca4ddYxtedcqEtpcbqEJIjtApCSr7zPzsucEHQxe16URUkKcaQ4AddEkif+bz2sSaUFhsoRM8YBUyXVd7NIj7BAznUqGm2u1lNORCIUQBvmgGfw6WLMBCTomQNBM+8DX6+VjTeZKgT7QEkAwAOURtr+NssvJWk0Zdl0AcoBSJBABA12/SxWraQpBJWJMhWpy7H8Z5/jY9nbyltXsjYlW+3vsA+vPLeaYEacvrTztNjkzhcKPcdtH1BLqMwyu97wKkq1G33WX/AEEL3GGvTAw4gryt3l6zRr5SHaRzLPh2iUgDqNhZPxe12l0PpTIIAUBmgykgTp4HkDZpcHr5GGPSM4dX2p1LSW7+uxTi9oR6ylKx55VQYsSZvFppGdQVXHhkafNe4TgCHjoIULdQcsg2yrEBKk/w6W0FZkZh1t5aTYkLpBq0FFnCM2k/G3Czp4m3bx72YHysC4tKUZlKypAkk8rVHHWysNSbf9Ga266hCRKmwHE+adfumzHbbBMQI3iNPdZ7XBii4cS1l4UNz1yav1HIHlISS33s2iVbK9kzEizYqqI0VY7Rx/dqKRP8o2+VuowAktLCg1ZLRYrVIyAEpywPvssUoAjY/hYlTtkiefjI8bH6cQZP6/X5W7inaucqDqj7UaAHaxymVkdBHIEj4WItgnU6x00mxxpQDiB0InS2gAVSKc8yJtgOu/jYGnWTTtkmTlAPiRp+Fu82k2sNF1XJXebbS3M+XwtqdJNskSdTFiAWUVu250gW0DztknXfSxGhQcV1uQZOti9QYfZPJWZJ+Gn42HzSIsWrzlbS5rCHEkkeOn42d2yiN12uY1ANuFE8rdr2iLBKj9LCIuFO9tU73w3WXPQ1C0BRQkakTBAyk/faNMQXWw9fKEO0rD6nVJQgPOFIBJ3ge1y+7ykW5HQ9cBaKSeycKYB6kH8bMfHtMUStsHTQkC3DdooA5wsbHa638IdmuwrhWL8IUyjTJuWgcS0cgWEQFAaT7FstH6KetCQBSlQjQyrX52y3M+xm/wC8fwr/AHZnX915kftFqlL3GW62CdG7gZASdv8AzioM/Dy3s1cFXrdVPcN3sKr6VCk0yM4U8kETrJE6fW+tnB+0Bq546tN51IyXDToVlOuVTjxI+B1tXJ1a26t4pUClCUyrUmAnkfceUdbdnhkAlw2Fu3hCq4i/JXSu81ZmnvK6VhJXelICdY9YTry/Hlzso095XcoS3XsKTtAcSY+B0P5m1VkrKSG8w6FJPOZPOBt91hA+4qftCkp2TG48+vLToLGND5quKu3JWtS6wCAFIlQiMwP9B+Ws2HadbKoBTpBOuny89otU3198O5mXVKyaAJJA322E2GF6VvtIraprN/K8oCYj4k7fpaPcD1TmrvrZWyQNsyMwG06/ERHW3RPfSSNCd55yBPv026/GqP7+vplGRF61k7oKX1yDrpM/P6I6MUX822rs74vAJjNHrKwJB6A9fx87IUThzS72DyVqM6T3lD3jofD32CXqBsVLmYHPQ7D3fpar6cZ4padStGILyTAlIFWoyfj5+8RGuopx7ixtMJxHeLgGhJfXI03k8pAP0ZXc39U3em9FZJxzQ6aCT3dJ2+WliNUoqbKW0knYJjbXl0HjaAP7fYxC0n+0NYRCRose6Ad9RHvNsc4i4vQslV/VCiNJUlJM890zGlptpXBRNQ0pV4tKP72ozMqDJ7qpIkL5+Pn+dpbwEQrDl2GFKmlakHQg5ROkfXlauV939XX4tp29Krt6hsZCopAAEzEAR46/dZy3XxKxPdFAxS0NSjsWWwltKmkqyoA6kbDTqeVizwOdG1oQ4pAxxKspTlJJBkEbkax+Gwjl7rH2YAKMvdOqRmmdPHztW9jjTjRvMf3hSLypEE0yRqRodNxzsKjjtjZmXlqoCEyO+wY8van4dbVO6SK02pYFZCAU9nlk6yYB2+/66WFUIJUmDodFJjLGsz00PxtXRHHvGTYSHmLqMjUqYWNOYHf1HUe7wsMj0hsWNjK7d90rIAEZXNRMT7fj9/jEe6SqXeWFWGQoIUkJSSTyKQJHTw+trDFwABMZgfCPfavzfpE4jcAcNz3Yon2gAsSf+Lp9dR//ACiL2ySrDtIoxMIeXrv4Gef0bD7tLfZOKlnVTwtaUAIKjvlgj6PU25LgUhJJVI7xM6fX6++DT6Rtdp2mGKdQkQU1SvgJR+PvsKn0iCSpbmGGxlISIqzqd49nw++bS7tKOSTp4zzU1F0qAIBUB3tOn46mxCqWSCEpmTEknnvvtytErfpF0+QH+y6wZ/8AS5IOw/gncx8bdOcfLpfht/D1WnOJBDiVQQR4Df3WkIJRyUOMzqkTjU+pm8KR5YzAtgmDrIKtvH56bWcnCt/PhOiJzjVYnb+NXnaM+ImN6PFq2amjo3EFpIQouK3Gs7acvv8AGy1gLiDd9yXQihrW3lLbUv2EAiM07kz03takY4wgc0Bj28UlTtROpQgtgTBmTv5aWPBQ9uD5QAOcjz8uvjaMabizhtJyPmqBO/2cx8DzIjrr5WUmOLeGFjv1bySZlKmTI09/3/pSLHdFazN6p/FacxCEKEkg+J6/L522H5Ep0iRI016WYSOLGEnUpzV7iQqRJaVpyMaaWEPFPCbkFN5Ebe02rpodvgfK0cjuibO3qnwp5OmZSSkaEE7W47RMEpIyzJiCT08LMxvifhRRE3qnfUqbWAPCfj99hDxFwrCc96sogTJKgTr8+vvssruie7eqd4cQSc2VI6zIUZ/rbha5OUyTsYO31+fLSzU/0hYVUlSf3ywMp1BBTMfXL9baXj/Cao/7bphEfxHmOfnHhys4DkszU4ahYKJkgREDfeeVmJxHR21wLhE5HUK05jXT4kedld3G+F1lShfVGTOxdA100jlpv/WGxjLEN01dz1TTF407q0EHKlxMjWNp87FizByE+xCjnho+lnHjICHAHUutAjYHISRt4WsPQuNyhJ7vOVbG1Z8D1JYx5QvO1CQ2XVQpaoE5CPhvHj77WJu+taUQA43ooQAsER4662tVRIcLdEGntlTnQ5pnWlWkEAnU6dfKdfEW6UmQc/dJiCInf+vysTafBR3VJWFbkqzGJ5df1sIV5+4UjMZISSZBOn5fCwMxR7BDZgDAkynT8o+vGwZCEg9xKsuswCNt/C3AczxIkAgAdNiR4crcrKoHdkxJEzI8Pz+hDOQU2UEoZSmZns0idJiBp9f1sCtDajqjUTpz32+o+duA8kyqQka6a7f0n4W5S4BJIEjqPqN/rS0muI1CcgELhxumBV9mdpVz005GxN9phMjMJkp7itD5/n42NuqXOdRmI7pJjQ/pPv52IVbpyaKHe89dNo+vdpYrZHIRYFAvE+mpqHGdcptpSHFobX3SRqpAGo+O362WOCdbUZLybYefaKVtkKbcKT3grQ/8E+4WKcYk9niFhax3XaZuROspWsae4ixDgpU9nfd5UqlFGdhCyADulUGf+I2vbw3VVotIp6o6+8sySL0q8hTmOZxXhr7/ABsfTeF4uoLZvN1SVe1KpnbSLItIoQCTBSnlMA7n4afDnNlFCiuCFpz5tTB+P3DyFqRerll03TpSezLiipJgz/KfuGse+3PqKW5Qe6YiZ6aDyibCtAwSQF5J1Pj1NuswBCUjPrzO+34j5WbiJZEVVQqcTCXzvr4Dw0+osEqiW0dHVBI1GbWPd462UFFQCAoBQG53I8dLcrAzEEwNpjNrHT42QkT5Aks0CiAAqFaScs/p1+hYSiuhIXKpUQSYB/A+Nji1JTKVAyQc2pAPv9/3W7pipOfKoZo06aTO3O0XSkJ2x3UUcWrvLF8B9KYL9CCCOUFSIHwHL9Gpwoq+yxmQCQmqYeSfEApVIkbaRp0jS0i8XGULN11KkZkkOskjYgKSf+oxaJ8CVDlNjG7H1riV5B3twoEdOZM8+vjY8RLoygvblkCsU25lgpUSAdkgwBrGsfUGx1oyBCDA2mfqbJTDxy6CEn4zHQDXysptuAqIWEjMZ0Op08/qLU72VyyHSsIkpEzqoETuOvvNizroCfZ2gGFcvKPD6i3bjyCsHRJ29qY/WxV5R7M5iND3jO/kQNOVlnsoloSHiFfb0NWyQe/TrMRJ2MfP66Qnfd4uUd5XPeVPUErpnCtKpGhBSUx4fmLTrXMocPYrEpV3DG4B029/1raAMUsqaoqdb+YervBKiZiYIJJnUfiogWuwOz+EqrM3Lqvfa6bzYvy4aG96ZQLV4UrVU2oHQpcQFA/A2Fp1ygojY2h3gLxRwvSei7w/xviu/qS7qJGHaKmefqHQAXWWw0tI5qVmbVoJOlo0xn6aGE6x9+7sK3i/RUmoNUunV2rgmJTyQN+qv8pt5lJSzOmc1jb2JC6OnAkaATb6qwuMcfXFhVBRUOmqrYJTSMEFw+KuSR4n3TaDca8TL0xJmarqkNUavYomSez8M5/2nvhO3d52hiv44YXqFKDV5KlxRUouBRUtU6lROpJgySZsg1fFi4qkdp+8RBIzTOg8J5/rYseGy3uWrZiNLAPeBKs56OmIlVeNrzutTuZNTd/bQf5m3EjT3LNpdxXR9neDdSgAJfRrA/iTpPwi1SvRp4i3NU8ZLku+nrkqcvFFRSxrJ+xUoD4pHytdHE1H6xdpdCZUwoLnwOhts4cw08wDgsnEXtkcS03TYp0QBEa766WNITA2JjeOtgGUmNSJiT52NIBMb6c58bdrBouXm1KGaPeGg+FjKCIjWbFQggwRIImw6IjqALXwqbjZOChcLjGm4Vrp11/GxgEga72IXWsFlSd/oj7osZqKlikZXUVTyGWkAqWtZypSOck6Wss91Vzuh5B1P3WyZMfGLErvvi6r0Cv3deDFTl3DTgJA8rCvV1JTKSioq2WlL9kLcCSfKbSBvqmNxojExtbYURubcTIzA6WxDrbkltSVwdcqpixAoG5QgIkfLzsBeCM1E9lE90ke7WwwMH8BbagFIKCB3tNbORfRMEXaUlxhC07KSDYNRBGwmwdAT6mlB1UglJ8wbdq02Ox1sMpwbpzYNd7RNZTKOikpUPmD+Fid90jK2jUVaQpFOMykKHtnkPrlNucIvdje4bUdHm1J167/AIWO3+3Vllxlqkbc1UVAu5Y6cukW5PtM0CAuIP2/4Wjh7iJALqO3cQ1/aryVlalOYwEvhKQJ5DLoPC2WFVQXlmMPJQJ9kPmE+A7tst5pxaXq78rq+DB0Xjt6d9QH/SBqEK2Zuuja1PIpKoj/AHrV+dRlrXg5kSZE6c51PUnfr5WnX04HQv0hL4gpJbo6FEEbf6sknbz++0Hpl111KgVZjJCANSCem34Tb1PCfDQQ/wDxCwcTN6yT6lY2tSVGRqARoRvyMcvHr87YoDdSlyRMcvH8fnYXskqUgnUgyNRrHIc+h8rbKBKUKRJUQcytCvkDa6SFVsuA6sGMqCcpMEkRJ+vEn3W5UVAlQIAXMwZBiD4jSwrjQQe1KgnNtJAn6gW6yntcx9pRIPlGs/D4RZgQmIRZK1JAJCgRJIUcsn3am2FAIhKVAHQEzJkcjtPOxwUxVBaSNJ0KYHXrrp9x6W0lghZMZyYEHkZ0jofxBsswTWRXMSU5Mqc2wI1PMj4R8Bbckp7NOYgA6H4R9c43saaZWoKzLKkKUE5MsyYO/Xx+hblDE5QpKpIAUCDKdNIB8h46WWYJWRRTnfUUglc666kzofv38ettFwk50JS2F65Rpm941A0ixkMFAyuwdu6pMwYgac4/XW3Cmm0ELDiRlB7u5MbR8+gtIOGyiWlFHu73ipPdUUyFeyRsB1Gm/vscbV9i2DrkTlPe08ZjfSefPxiwlz3BeeJ74u/D1wUIqryvCpTSUtOlQGd5ZASmSQBMxJIGnhaZ2vQz9IjKnPw+Da0gZpvOk1MAf97E/lYc9RDCAJHAfVEhgkk1YCVCZdKEw2DJE5ojXxP5xtYJeZa5LZMch8/Ly8bTw16FfpFPNKcVgVjKTAUL1o9CeUB3Tn9C2K9CL0j31AHAzKUmCVqvajEjkf7zpytXGIUvzj1R+51HyH0UFZ0hQWcqVJ001BPjHhpG3lrPJUOQzJUCU6mTrsfqNPC0/U/oH+ktUoUn+xdGlcSAb0pBqDpsvb8OlhFegj6S60JT/YmnGg0/elMZMc/tPDbUfOz+0aQacQeqXc6n5D6KAmaw0y1KU224E6QtJEgxt8v6WDcqFOKLuiQBO2kc48Plv1tOFV6FfpFUL6mXcEdotRnKiqaUEncR3o35gnWwafQ09It5SGG8BOdoo+0urZCBvBJKoH15WcVtKdQ8eqj3WcHVpUKh0lRCsoBGpSZ+Hl566+NtKqEg/wCGYWRroSOuh28vO01VXodekQy6pl7AL5KVAkoqmFpmBsQvpE6mxdfoh+kM0Y/0d1kbaPsxsdiF+PLws4q6b5x6pGnmHwlQ2y6ACjNJ1lXnMCOWg38B4WxFSkKSsoJCgQsq00k9fDTTx8LS4PRH9INtGdXDWrkzmBfYA9raAqwb/om8f0hShw3rRmyxDjPTX+PX+mtpiqpyffHqocCb5SogdcWlAEmJJgCMw6zy016fiPROuBpS0LJSdiDBkGYPjqLSo76JvpBPR2PDSvOg2daGwP8Ai+tfcNR+iP6RmY9vw0vGEnTK80ND1Gc9BaRq6fL749Uwpps3un0UXCoCRnCjEZVp0J0HhqNjH522mrWlsgFxOXuQFEx5dPaHnpaW/wDyTfSBUkp/0Y3jomP71oGQIjRX4fqGr0S+PpKh/o1vIlSZSSpoAe4K6Tt1sAVdOd3j1RO7zfKVE4qFElTqwCSMpKTsPCdLdIrFtNnvEKKdRO45e+NvM+FpSd9E3j+tAC+G145yNs7Yj/m5e/Qne2j6J/pC8uGV4qndRW0PCNFSZEbfPW0u80x+MeqjwJvlPoosNeIRKVBQTlCdZj3/AFM+Fuk16m2woAJUTqdZ023kdP62lRHol+kOMyTw4vDUnTtWRAMae34fRttHoj+kIUmeHFaNIhTrCuZnXPvy1/Oy7xTfOPVPwJj8JUTGqcKiVASdfHXnr110tyqsXlBUUnN3lZdSfr3be60wUvogekVVKLTfDerQcsqDlVTtgbjYrg6RYm96JvH8OhocOLyMzIDrcbbHvRrPXwsu9Uw0Lx6p+7T75T6KKfXnHVZjJ7+k90kdfDfn4205XJgJzglGgI0Khz193ytKi/RN9IVPcPDa8VFCoCwtBmdCR3o5DX3+YZ9E30hFZQeGl5gL9mVogR5K25bjraQq6T5x6hR7tP8AIfRQ4zULS+k5u9m2B56Tv7rCuVa/ZClCRqdI3IPly8rSqPRK9IcjKOGl4pymZzNiJJ/xfGwg9EX0hgkn/RtWmei0TtzGaTt99imtpL/1G+oUBTVA2YfRRSm8atB7RNStBBjRZSANfHwt3+/b3ZWEC8aoZdSA6RqBvvvtp4Wk1fokekIBH+jWvSMsHvI1A0/m3841t0fRJ9IVauzHDi8MwJUR2jYnx9rQ+z87OKuk+dvqEjT1Hyn0UcNYwxElv7K/7yGbUgVKwAdgfa6fR3sMnHWLUggYmvXKnQJFY5I16yR7uptITvolekOCVL4b14OnsutydDyzaa+Hwtx/5IvpEZj/APw2rIgwS41+KvGyFTRn42+oTd3qflPomEjH2L2niTiW98+bVKqxZkcxv52Ms8ScYoBR/aW8CdMoNQodQdeY5e/wmzzHokekQVZDw4rgNVBXaNxA2nveduv/ACRPSFSSP9H1WQZEesNCSdt1/Plz5WRqKP52+oTiCp+UpoDihjYDL/aWs7xMZoObXy89vH3iK4oY0nK7iKqVlGgUEq320KY6z1s8mPRA9I50Ky8OagQMxipagzznPrrv00sI/wCh36RlK6KdfDupJSAoFupZUCd9FBZE78/wtDj0fzt9Qn4FT8pUV37im/b7Uly+bwU+42ChtRbSCU9O6BOvUT8LB4exHeuGrxXX3a8EPOJUlX2YWMogzB05TItKb3oe+kS240g8PKllLyghKlVDYTqNicx+XjY8z6EnpFrcCP7FJlYE/wCvMpHQDVUWJ3ykAsXj1CYUlUTfIfRM+k40YtpUABVA6EjvBTG/iYVoNrH0ceMUJASaK7FEGP7lwaTy73h9a2XXvQr9I9Mf/IJE8z+8Kcgjr7evX6NiS/Q/9IlogKwCQnQAiupxv17/AI2HxqJ3xt9VPg1YPuH0QSOPt/JEOXNdpIJGUlyZjU+1Y4xx+rwuVYdpSEzJD6tDJ3H4+Gx2toeh76RmbshgY5hyTeFNA0HMOdB93U2T8XejJxxwPh+sxfibBC6O6buQlVQ/67TuhCSpKQYQszqRyMTaOajeQ1rhf6qWSpaLuafRLjHH15JLT1wU6lA65amB5+yfryEjDjrTrRK8NHltVDUdfZ+f32gpDqScoUZCdJJk6/1sMisWCA0U5iIJHj79dvusQ0jELvDlOjPHK7FhAcuWoTmTrlfCo59OQ/ppZSuzi5cr1WW03XVoJBKSFJ18teX1pE1+RUulXf1EglJBPj4eP4WV7jrlivbICiVEtgde78/zJsCWlAaSEWOc31Ur4/xjd9+0NMzTU1ShxpalhKgADKRpIO8gfQ0iukrBSYhYq+yJSxWdovL3tA5Ow8Of3aWXax5LjYhJWU+zmIBEadekWZ9epxFdUKIgFSSAkyIgRty0+Vo0guCE8ztQVYCnxtc0aLeSdDBR8Bryj4ab2UqfHdwZUpD7vd6smY2AiNbQ9TPj1ftE5hm/igxG8dI/DnBix/tlBKFgpIA0gkpBJ+ufI62pvaQUZshKls41uDIhSqogZjqELmdNzFuFYtuIqCPXBKgIJQrmPL9NLROmpKWgSVJMAeKvx5+Fki9cXXfdMtBYdeEdxBTod9Ty3PWyjifKbNCd8wbq5THV4iuQs9qLyQhIGq4IHI7kRv8Al1tA+PMTXS9X1t33YoPoeqSS4kygJzT3dBOs8ufibN2+cVXnfK1B15LTSlZuzbACfCepFkJwkmSZ1mbbNLRcLV+6zqiqzCzVKmAr341cQGKfB2GsSvP0eH6ZRpKOodSWqdtTpWoNpUCBK1knTnvtZ4VHDb0j2UFa3KF0azFOwZ08W+kfK0HXNiG+MM3i5W3FeD1G8oFGdpUEpkGPEaDeztp+PHFSmbDIxdUOJ59q22v/AMSfK0Kimnz3pwy3mNUemqKcMtOXX8inJX3Fx/pCfWKJokDVSael0Hnl0skOPcYKYAVFA2SZGrLIM6jlEGNNNbJr/GviHUp/1i/AuTJ/1dvU/DysQe4nYrqEdm7XpI6hpI6fDblppZmQVHxsYmfNBfwOcpd9HfG3ELD/AB84eXne1MGqRGJLvaqF5QPsnH0tq2P8qjb3KdYTUMuMLHdcSUH32+di7uI+IaO8qO8PW1BdLUIqElOhBSoEa+75+VvoiuO9Ke/blu++6Q5mbwpWqto8ihxAUD8CLY+LQujex5aB9FcoZQ4OaCT9UzUMqbWppYhTZKTpzBiw7YHjoI99j1/U3YXipYHddAXMc9j8wbEgNhyGnutqUzszQ5VphZ1l2lMkdPusKkaagadDYNPtDTT8bCjUTpBtoBU3C4SldLgJKBznX4R+NkXibct8X9hh2guJCXKntW1ltS8gWkKBIn3A+6ynd6gH0jxB+8fjY7etYLuu2prigrFOytzKNzlEx8rHZq2yrk2ddRTd11qwfii7qekrHl1D6GV1dOp3tQytRhQSdwkj+HbSQBNlHEmGb7vS+r2qat6poKdDanmK5JbWjRIypyqkx10G2h52TOGlddt5V1RinEd5sisdcK0pWSYJG/kBoB0FnLxFvdbyKO6KJ0BqqAdcczQnL/DJ6c/daLcoYTyRHE5gk2lvq8nsBVQLq0inqGm8xOobUpMp6xr8LJlzvV+Hb4u+9hea3ae8hPZFKQG8qsq0SB3gZB11BnXYB4C6rsRgqtua7K2nqX/V1uKLa0qUpwCZIHjHysxql1NXh673WlhS2Ktad9Q24jMD5EpsW1gChHc2Srik35e2ILypKGuS1UU4KqZK0lSIS2F5YBETr3ht42e3D2+n73uQesvKdcYUE5lGVKQQCCTz52adfUBnGN3Vyu4iupmHFA6DWUq+Ay2P8MnF0lfeF1qOjcoTP/q1lA+UWINDdQOye7cIdqGxpC8w94ti5UdNracBTeCyVaONgx5G21DXSyKYI1dLwprzpn9gl1M+RMGz4vCmCnV6e0i0diAd4jUG0lKWKmmYqQNHGwr4ibZWKQiSFwPRGhcWu0TGdoQHVjXRR5+Nss5F0iSonJubZbzI056Lb72V4K+mj3/SIxElBVPZUKYH/wCqtdbQ8UqcqHgW1KHakZ52n3dTG2utpu9L6hqLz9ILEdXdzJqqZfqWR5glaTlpmgdU+ItHQTdVI5UpqaRaX+1IjMBy13Ply1t3VC7JRxNt8I/YIdb4qqQ+ZTfNOvIVdmXIMEARoRoD4mCN+Vum0KUMqgYmSkIgnu6Hw3s42b+uWmSWkUhb7Q6gqSABy5zpztpNfhDR00jp00MmUmdNSoSefS1i7uir3HVN5NOtQSCFDIJJjbTczz11526TTKSQrs1qOgEE7xp5f1svJvDCi/sadp3vryJGZQiPMzz84B6ahorMNLUnKFoSvu7geMb7fITZru6Jhbe6RhTK7RBZUQANYTOu8nx25fp2lsOBbYBQghSVExB1PMzO55zrvZ3XFdFFftO7U0YORtZQVqWB3okiOehn8rKQwa0lKgnKIEAFcmQJjlGmthGUg2RRGXahR+pgEkZDm9jQDfw6a+7QG3JowBlghSN4BHI9OUH7+VpEZwSwk5XklKcxSR2qZBBI3jflz2EeBhGBLrUQlZcAk5T2qNTrtO58+gtHj2UxASoxUylWiGxKUEFMTJH6fHnIsG9TOskZ21QRA1ISTtM/dytK5wTcwSsOVDwC+jiMpJEztzn5ixd7h/cDo1vB5IMk6tkgbRtqNPhNk2qF05pnEaJF9Hp3Jx74al0pAViiggbjvPo01/W3uUmloHgla6RgnL7RbTP3W8VOGVzU9w+kPw2pGHnHUDEt2rK1lJOY1CBuPK3tMmqCW0x/CItQxSPjuY63JXMPcY2uaeqEN33aglQoKYf/AFQ/K3aqK7VaepsE7atpsA7ejKmxThiFzq5OkeVg1VC4y5pGkmd7ZxogFeFTfdDGiu4HSiY8fsxbZobuUO9Rsmf/AFY+elihvFFKStxCnJ0ATv524RXlYzGAN4mRZdyFr2T95J0uu13bQF7tU0rIUTybEnzNulUdINSw2I/wjewZqlHvAfG2i/Jjl1tNlKBpZMZ/NGW6Ojyx6ugQP5RvtYZNFRAEdg1GsDKLE26pI3V7tjYUVOmpGg2iyNN5JhPfmhzR0mqfV24Oh7oNgxd9Eoz2KPegfjbkVOYwUjrbn1sTEj4z8LIU9uSRm80KigokqJ7BsGI9gdbCt0tIP9mkTzyixUVAnp5CbdesmRoI5gmzmm8kuN5o0mnpEpILKRp/LblynpZksok7EpFi/rYgiQD+NuVVRmUmTt52j3XySE3mhPVmEq1QjQxqgeFuls0uiuxQSR/L87F1VSo9tXh4W4NSdif8Vpik8k3H80OaSjgD1VuR/hEWDVS0YBHq6IGh7osCKobAHa3HrRGvLfe0hS+SXeD1QppqQDL6syY/wWLVdNS9lpTteyf4BrbFVBgzJG2n19aWL1dQgNgmUwCJ8bQlo/Dspsqdd0LTtUxcWoMtzl0gC3KmKUttJLLRk7FINidNUIzokiVIn3zbhNUcyNZCSNZ6k/laiaMjkrPeB1Rw0lGWCo07R3klI5Hy1sXNBSBgqFM3Ko2SNPlYJVcAxCtyo87dPVI9VmRMA6ajey7nfkl3i3NGU0VFH/mrIEfyixdykpVLX/q7crIGqR8bYa0BvMCCI5mwCKxLj8rJGUEjl9fpYvc9ALKPeB1Q1VSUbaW/sG+kBA1t2qgoVJj1VnQDdsWLVtUFJSSNlHnrNuzWJUEqAExOlpso7HZQdU+aKu0NEapI9VbgJiEpAsVqqCkfUGkU7Mjc5IPKTYeqqwHisBMJTPgd9LF2XyUhawoKWrrMaHT5TaQpNU3eQEUqbsoWlJSmnYCYBICQDv5R9TzsiY3xHhPBlzLr75yUyHCllpTdOFLJUYlIjkDvECLK1+1rV3Ublc8VFDTS3lhOphIJ08Yn5+NoQ4ecV3Mb4pp75vAyUsetBouy2wlZAbaGsGJk76gnSNLMVAXnyUHVYbZPpGI7wxS8tGCOGN64gdY0RV1zq6eka/wJ7RRSo6knK2Umedg8S1XGbDF3euO8G2qtE5nDSVDTwbSQMwCMoV58trWCw1elB6m0xSMNiESCjQKOu/Unfzsevq8l07IU6ltKlwMugPv1m1luHQBuZyn3mRzg0BQNhDGOHcbUqwzdrdLXspHrNI8hCXEGNSUnvRPgR42PVV0UTy8qaJkRqqGx3dfdzstYmuS4W3nsUMXOzS1obzKrWBCnAnXKobEECPf8C13w9Tt1KSIdAXIiACJ395+hanHTtc4taE9RI6IAlEGLtpUVHZqp2vaBAygwLNT0grgoL34K43u8UjJAw9XLSnsk6KSwpaY6EFIPmB0s9KmvS24qmTJgZdAkDNmCs8xmmAU7xB2nWxHGVMi9sKXxdgSVeu0FSwZ6LaUn8TazFSBjwQFTkqszCF4h5UZoygAmDHx+vK3aCkyFLylJCZ2/pbpdMtsqVJkHv5tANSBPvHhbvsXHXClCBCk7JTqdfrbqbdXcLm7ELaELKu6tJCRvBhX1B+Njl1lTVYyvXuuJGZI0Eke75TNgGGlzmLYJBE5hOkgayfr7jbbaG4dQhKoVmkxAjYa76RMfKwJDcEIjBrdOqoUkJCFAZld4BSOh6jWdBGvPxs2r6UoV6Va+wI7s5tT79tyfnZ1VKE9iEtmUq0UCmZI0HPw38OtmriYgJaWlMjKqToZ1EHfTUqtSpCC+yszaNunDdLinKJhxMx2aO8lUkRp74GnusHeV60tCz6xVuZDHcj2lJ5QOc/gOmjTRiZdDQM0lOwntG5+0WJiSSBHPc7/CyLUvu1DxdfcLq1ays62tMosziXbKu6osLDdLd8YxrK9JZoppmtZP8Z8yPKzdKpOqjPjbR00Bi2jMa9NBa+yNsYs0Ko97n6uWxIgfdbSgZ1523Cuu56227BAM76xFiKFlysagpESBbREDS3RBCUK1gyAffbakZR9aWSVkGCelsOltwd7ZB2skloEzMW+gL0Q8TDGPoxcM7+zlS1YcpKRxUnVynR2C/wDmaNvn9O1vaj9lpiY396Jd13ctQKsPXzeN276wVioH/wDs/K2PjTM0LXdCtDDXWkLfJWgxLThdO3UBMFCsp8jZux7vfZ6V1P6zRusACVI08+VmbBB1EG1fDn3ZlVuqbZ110AZkEeXIGwiVEka6+f1424GU9QDbsCQRlPiTbXas8lDMOdm6FE6CSfdr+Flh9hqqYcp30BbTqShaeoIg2RRBOVZ7p0PlsbK9EtTlGwtftKaRm841+djRlVnjmmgxwgwXSyKRiuYBnRuvfA+SrKd74Dui+GqVpb1Wx6m32LamX1JOTSAeu1nGCYInztuYMDexg1vRQLnHmm5h3A1Bh2rcraetrH1LQW8r7gUACZ6eXwslvcKbr9eNTQ3pV01MpZcNIkIU2kkycpUkqSPAGBsIFnwd+VtiNgdtpNpBjbWso5nXum5iXA9DiGipWEVTtFU0IinfagkJIAKVA6KSYEjwBEHW3eEsHqw4XXaiu9bfcGUrDeQAeAk/fZwiAJmBbtKjvpaQaN02Y7IpWJy1NO71KkE+Yn8LdLSDuAD0i27wgNIcP8DiVfOD+NtrEAfGzEalK6BIg66Wf2H3RU3DTmZLctn3EgfKLMJXx/GzwwW6V3fUsGfs3Mw94/S1WqbmYQptKUSjU6Wyw5RJJtluU7i1Wsy+aKnqV0TLbTbriTJBKVQo6kgT79rYupf7ZSHUOJWhSgoFKe71GniTp5c7ErrRUPM06e0lSjHdIEHYDz8PlZw3dwt4kX7eTrd10HaPIQp05qpsbECSSdTJHnNulbAXOsAhGXKLkpKRXLJyw6cw7sjUHp9DWNfAVq8AiPsXE94SFIBJOkeOhP1vZyDgFxjQO9cyEiZM1rO//F4bmxpjgDxgUpSTddOTlJE1jJ13B38D9RYvc3n4UPvLPmTTTeBUlJLJjUElrUnQgGPKI8Oe1uvXErhCKdefY/ZSYIjpB+Pws5TwF4xpQS3crJCz/BVMTIn/ABCdLcI4KcY2kR+5BABTpUMczP8AN9TaJo5OicVLOqRmbzq20GlpnapCDKy220QFH+JRKfHLyHnYRF+X0FZU1l46gpGVTkDXbnHLygC0l4G4H41qKGpcxBhvtKhD47OKhsSMp7oKFa6xqfGzne4G30hSVUmFVNgCClNQCSZTrqvTTWdxpas+FzXWyq0x4cLhyg9q/b+efSUVd4EqTlBGeQR+oHT4myg1V4zWgGnTeeoBlLSwVancAHx8J1i0v0/BLETTqVKw3UNAKGoqEk6kyPa8APjY+nh3i6iWtNLh10BJlMZVctt/r52rSCQe7H+FYiyH3nqFm2+JTqClqhvtSBBMsLUZjQ6J93Tffe3NQviTQNhdTdd9ozGVLVSLlX/LpqPhNp2RceNqVK0OYXW5BmSyAdSdsvMQPjYk7duNy6Gqm461IcGfSnUlIEDaDpyA8rAa6e+seit8OntpIbqPeB9ZflRxzwCm8EVSHEYju1MvtFsgesNnmBvJ+t/Z9tw5QZiYOlvIzCN3V9Dx64fP3hQu0q1Ygu4gKbKCQKtI5nb9PG3rg2QUJGWFRO9lUsz5TayhTuyZgDddKUCRJJIHvm2jWtoEKGw+vvsE4HMhKUmBz6e/4WJL7RRJSox1ja0WQjZNJNbZGi4y4vOSNIk/XnbsOM5co166/dZPQh51RShJUqJIAtyXSFBK5B5z52L3cHRCFQQlULY3UOe9tlxoSQSn652S0vAjQg/hbC8JIzeR/GyFMEjVEpUFSxvqPCNhbBV0/JXh42SC6IJB5RpbkvnNGaROhn8LTFK0qPeiEtqqmSYMkjTe2GpZkEEn+tkUPySAr/NNsD+XvEyN43+jZd1AS70Ut+tMzEE7RbYqmTM6TvrZE7eUwTPPlpp1t0KgACSQPPlZxTCyXenJX9Zp5g8vvtpVVTzqPjZIL8GMxHnbXrIUrUieXl9Cy7q1LvTkrmqYzSNJtz6wxzSoRpZK7bTQK0H420qo2AmDG1n7sEu9OSsaljcoG3X87cmop5jKdZ5zZJU/zUrfXfS2lPnWDM89vCzimCgak8kql6mH8A8JNi1dUU5bIybggQbEvWFaAq0+Q+tLF66oCkJSN/Pwsn04spNqTdGG6uk7Rr7OPso+FuG6qm7ErCIlwDfTSyR61kQy6NB2SjbTbw9QJB2XJ1Gus2rmlaiiqclM1jJEZdZJ1OlhV1dMaUSn+GRrHQbWRS9nDywQYlI/H7rb7UmlCBBCkxE77WburVLvTilV+8KRLIT2YTmHUzpbVLW0xdCuzVKknUzIshPvqUqAe6BAVI0+NhkuparEQdkZddtv0FnbTC6Y1Tglm8axgNCG9AQLCJq2FITDWkdddtrIdc7LG+oMkGd55fXjYQ1IQ2FKVASnlB1+vv8AGxO7tCH3lyNVVZTrUlkszm8TPl8/qbDpepk5R2QypIhP4x9bWbztSMyVpJAKp9qSQDrz8fD32URUAnTWDBEnXr+Bs4p2pjUush72cpahCW1sIhYLahJ2Pj7vvtT2huS+Lr4o3VgG6rlvK7WzXKU89UUpYUulQohK050gqCjICkyIOh1M20q35Q2pKEnKdEqEz4b6frZCvbC9Bd9WxjCiTSsXVdjaW007Lcu1DrypfccUTOZICVCAPZJObQgUwMTSQFeobTu8RTpvaivdFzihuZNTkd7jy2qs0y8kAR22vZDQkkSoxAHSHeGuDb5XxCqnDTXZR05ePrDlHfFRebi2ZIAU8/MKjWABExrEmaqrE1Jh+6n61x1LyKZrN2TRAL6tMonb3k87RvVcT3aamqKujxzhK770Lkpu11lLzCWx/CEoU2oqMR2hJEzAgRbOa9rhYro44XXzNSDju7cV0HEKsu0O3k83TKytqpL1LJSwvvJdUye46M3dggkQdRNpWo1Jprup6NISSwy23sCAQmNuXW0TMcSr0xPeP72xHRXA4+hSWQihV2mQQIUFEmCJnyI90stMLcp01B9rKmRsdt4O36/CWG5JJH25Klj7XQsjzbn/AISReCyp9WQhPUAnWf6WFgrSG3CQFd0687Gl3VUVIeqaVJU2ynv94AzBMAc9EkwOlgygBgJOsJBEiNB+v1tbS4Wq59slwvNm8sCcHLvvu8LuqnrhFRTVbtO62qrSlSVIWpJBHaCCIiPC3bWC+CRICTcS0nvDLeBAPwc00+trNLjhwLvV7jDjN+kqHEt1F91j7Tfq61aOOFYggbDONf62YrnAnEAWA1UAhR0+yXMe4dCLVXQQk6zkFXmzSgf0AVOKeG3BVxtB7O6w257ATeSiFDYH2/EWOU/CfgzUFSUpoxlBKibzc/Az0087V/HAvEaRPraZ2MMOEBU7SUiPx194DvBbFDUjtmySDA7NzUjlMb/XOzGljdtUH+fdEFU9uppwrOK4c8ICnskNNqXlCQG7wcUmYEjzAFqc3iLxrb2qaAqW8tpbjaEN6gZZmAPBO/vsv1PCLFTCe0DjWTkTnEnp7NmZX0dVdN4u3fVAh6ncKFiTuD8bXsNpGQF2WTOT+Fn4jUumDbxZP7p9XDhe4a7ClYbxuS90X96wU072YCnDRA9pJ70iF+FuVYBu+FIKqxKgdsyT06J6fh0s38J4VxBiv1v9zPpHqSUrc7R0p0VIEfCysOGWMgExWsAiYAfVpzI2tZkeGOIMllTaC9oIZdSdgXhJwbxRW0lxPO4pTXvMkrcK2UMlaEyvLCVabxNpHc9D/hS072buIL7ZOmgrGCU+ctfWlq50XD3iOw+HrvvMtOgZQtutUg+QOnX52U2cJ8Z2wkM4orW8qcoAvZxOUCDG+m3ytmzRyudeOqsFq080TW2kprlT+PQ14WOphGKr/SIkS9TDX/2c/jpZgcePRkwhwy4fuYrw3iO8q2pYqmWXGKhbSkhC5E9xIMg5fjysyxh3j8ggJxjevcOYRfbndPX2tLB3lhHjtfV2que9MUV1fd7qg4unfvhTjRXMyUqVBPPa0YW1EUjXPqgWg6jTVFnkppYnMZSkOI0KiTIC0DmEhUWuxhD0DcD4rwtc2J0cTrxT+9rvp67sk0TXd7RtKymc+sTFqzI4GcRilSfUGACrUGpRqRtz8fnZ9XXdPpQXNd1Pdd1YwvVijpEBqnp270ORCEiAlImAkACBt87HxOZ07AKSoawjfbVVMMiZA5xq4C8HZTLWfs7bkRpR8RbyWraP3ahUq5ahyyY7+zrrQFOIx7VoQJGZdyk69P7wC0c1N+el/hygqr2/ttfzbFNTlx51F4gqS0kEqVvOgmSNbM930rPSHfc7V/ipfLqoABcUhcR0lOm3Le2dDT4zKLxVTXD6f8LQmqMIjNpaZzf59VNDn7PG/hAZx4XAoEibnWNBv/tOXO16/wBndwlvvgjgfFeBb2vYV7b96tXqwoUy2cpcaDaxCiZ/uU7W88sE8YPTexrdn77wfiW9K+iDymu2TRUhSFpAJT3m9BBGm2vna5/7PnGHpPXlxFv+6ePDla/dT1zFygcfp6ZsJqUPokS0kKJKVq3kd21eU4iHcOpqGOA3A979lMCgczPTQPaep2/dX0Eyd5s0L0p/V651uNCrMNeRs8Dpz+FkLEVPq1UAanuq/C1+hdlfbqqFS3M26RQkK84iI5WETA184jS2RsInn4fWttyDzIjebbjVllbAIOpI5WVaNwqZjmFKj3mR8jZK1EwD0Nj9CsdkpAPMHT4fhY8e6rP2R0RuAdrbBIEGfjbgGBoSIttKidBz6WsoS7EDczbAflbkdZ8NLbmdY162ko2QgO8iBbtJ1iRYESnQi3YOgPwtNMua5BcpXgPayGNOY1FuGFh6nadJMqSDYdRERBI6WKUcJpuyM/ZqUjfofytB26fkuliBFnFgd/JWVFPycbzecH9TZvOa9d+dlPCjpavlkkwFSk+MiwpRdqdqeZQqT52yxssgmbZbG7mUXiBfMtcjA7aibK0/3qClMcyofD63tZ3hc36xf1YhCkIHqyl94EQntE89eg+XlaugumruW/W7uvOkeo66iqEMvsutdm42sLAIUCMyTrz6aWsrwoYR+8a59TqJSwlGTKRoVSeWu3n+G3R6vVaq0jKkZdMppAl5CkKITlSqR4ae+PrUJIKHQsEpBVm2nQHr77HcjRkKfSJEknpPM9BbYZbGZfboBGkZuu8/XxtsgmyyQUSWlWYo7Mnp3oEfX0RbR70KKRBIkg6n3fXLxsdfYYOV3t0HQagmVEe7cQDrYFbSfZDiQIHsmAfL57/rZkjqlbDRSadyASUOpMxABj4fHxstKUlStOZlI57DSLJeHWULYeQFBQDiBGsDfpZXXTLSUkJOmo5SNJ8BztTl95aEHuIE6HOAZBJH/N0sGoLQoOIKCokknfpv99jCkZm0tBG0xGmup28/usGsJSoZkERqI122/wDDHxsFGGqKqzlPaTOxPPSDpHvB+teHRnhACQlMDKPMc/d91jAZUpvvpXqoJlU7aTYvV1LdHTO1VUshptBWoxpAE/jt99mNgNVMAnRQ/jNeT0hOE7cJzJvqjUop3A9aa+Wvvjwt6ctLSpIn3z99vLS9Gaq/fSD4bLW6Kd6uvyhT2rmoZ/1lsIEc430G8nc29L2KO+0o717g5tQex/W2FWuzPuFp0oLGkFONVe+ulVQJy9i5qrTWdOfwsWU1CVJMGBpZKFDfK+7+945/3c6WGFBewSqL0UDEQECwmEnkneAOaNMVzt1ZnWMvaKGUZkzAsjl9alKWtUqUdfPf8LFqu7b47WXL3cyFRgBAsSeu+8CIN8PgjnA87W27WsqjjqlX1kb5zvEDTlbPWcycyVE6TG9oM46Y9xDw0cuGoo8Q1KGq9x9t1HZoOfKEkHVM/wAR2NowX6SWLgsZb2UtMgqJCfdACeevuFkXBptZSa0kXureqqhmJzGAfo20qoE6Ekcp3+dqjtekdiEkhV8PAjT2Uyd9+7p52EHpE4i0y31UwqIhKJ+Q0mzZ/JPa+xVtPWhB36RI0+Fs9aSRJWZiNDGlqmH0hMQd0G/a47pgBMzoZOnnbn/T5f5Ol+VyRMESkH7tbNxQOSYMPVW2FWEyc2069NLb9bkwCmRpEWqV/p7xCdVX1XA8wVjTly8j8LCU/G3EdQSVX9XQTJUXfZ+G9mNQ0clIROPNWx9ZHXXTUW5VVECSRHhoZtU2q46XyVkN35eMCRIfIMjn4f1sGjjXfjhzfvu8SAQO8+rXXz+tLN3hvRLhHqrb+tDLpJBmYtwqqMiAdOfutWq6uKtdVFJevW8jO/8ArTiQJGmk2cCMcuVCErTe9fBEE+trnx5/XvtXfXtbyRmUbnbFTv6yVaJnoLaVUkCASOW0i0L1eKKlqlacN61ozpgkVi5knzsVRiJ51p1z98Xr3RoPXV9dt7C9qMteyMMNeeam81JzBOXUE6g2BrXz2QUOZ1kfXhaCXb8rKkBDWJLwaVkIbKqt1SSepAUNJj5CwN137eBeyvX9XVCk6k+sOhOgOuUqJGvj91huxmP5URuFP+ZTCuqUthoQZlSTO0TzsIalQpHgCqA5oY+utonZv+rKCo17pzLSgFVUvTnP6+Fi19YpfpXVoReD8KUSMtQuCB0B/HwsAYzETbKjHCpAL3UwNVLqg6QFEFRiIkfXW22X3vVUgNLmDMDSfD5Wgr+2zyVgmtq1CCTNQojXbn5/CxhnHDq25Qt5Qj2u1VqfKbSOLxj4VEYa/qpmaW84+ErScqQCox8vhNhHalxNa2pKFFJESR4nnaB7yxw42goXUOqJ3KXj+BiNvhZBXjZ4KPaVTpB1Ci+slPXn9aWePFmO+FDfQObzVl66rdLEgKI0Pd1+vr3l3K1x0NUqEmcokiQR8uUm1bnMc1IkGodAgCDUbTz38/DXzsUVjp4pKy+pUaH7YzHjrpzHw6WsiuaeSGaM9VZyrqlN9inUEGIAGpnffz+djhqnNglUyf4ZJPKBGvPlap7mOHgtPioEgLk85ETry+/yKHH4OZJU0O9qC7KcwjQ5fMaeHS021YOwUDSG2pVsbxqldmlHZqJBJyxOn19TaMPSJxTjnCnCisxZgJNcm8qNTLTqW1fYKpXe64X2j3FoTIVKhoY5G0LnHynm0eqqpShLkKcCgrNtpvE7QCJJOk2ScF8QF3txiw3ct+UZXdNRUVNNLi1BLlSqle7NfLKUODunNIORW4GW9Rs71K1hGhKE+8Au11lIXDT0m7qpqumwJxDvLtkhxNHdt91tKmkReCkJHdfaStaWnpAiFqSsQZSoFFrC0103FV3c/eVxpuVmsqCpbtSwsUzq1QdStEFS45yeWsW8++M/CKsww1+/ENVd/wCFryf1p6koTVXZUFagtKqpQygGdFOA5jEqBIKmRUcU77unA1ZePD7HS8PpuqsFEblvO+k1NWpvROdpl1lSijaQl6E5TCY1tOrwaKbNNSOA6tO/mtCmx2SjLYatpPQhXK4yYwwVwJuavx/WUl2VuIq1Xq9Gl51T9RVviCltS1KKuzBBJAMQdNYtYrBF93hiXBNz37fN0JuevvS72KqpoFOZjSuLQFFsqOuk8xO0i3kRgjF1XefECi4gcW37yxxdeG6+ncqEKr1BC2i5CcgdTAbDmUqTlSFARIkm3pNgb0sfR+xYlqkfvV3DTroPZi+qZLDKjvo8gqaSnoVqTNqFPQdzbrqTuVHEcUOIuBAs0bKV62rqKcpaoah5AWkoX2f8YJ1mNPr32KB1wU4TEkAjfb61+XSbGHsPUV50qX7rZpqhh5vtkLbKS2pEe0FTlKYjWY1FkdrD9K2pxtVGhKkKKFJUgAhWs7+/4GxXAhUWmyrnxXQzR8Rb57NKFhxTa0EkZUKU2gkx5n60s2A26j7ZbCcqlykFyCRzI0mPHaNLSHxpuVFJjlKadDbKXrvac/hSBlzJ5+CR8epsyGcqJyMl0+ylS/ZSNTt5C3F1pLJ3DzXdULeJTsd5LhgMoYzZVJcKYy66HqTv10+6LH2mXngKiqaUltBCUKU53SY/hAEmZ5bEWDZoXHSArNCtBmPdHn8fnZUTdL7pSy2pdQ5tDae6B4E6nWOUb2znSW5rTZH5ImKqgQpt31N1SGgdVuAIB0hWUCOnhFqN+kXTFvi/fj4IWKksv5+SiplEkeGabeiF13DX3dFYinpW3G5WHKhtDoSRyAUCkk77T5Wpb6bVHUjinQXvUKzLr7paK3A2lAUtDriSYAAGgTbb7N1F6wsvuFido6Y9zD+hC59C6n/e2OcSYVDvZt3lcrskqEDK4gAmd4Cztr94tUOHVUzeVHedNXUyl07yXyhLQyZwyhtSRprIC9uZGuhtUT0QHew9IC7LvIKhedNVsiDz7Eug7TugfI29J7rwq5UOJHYFSU+0IgRoTJ2+Nqnauc01YDfdoP8AZT7N0sdVR+MbEqPMJ4QbovWHb3u2jNZVViqjtGmwvs0FCU5RmAgyJmOehGlnpdeAsPVdO56xdVOiUkLqHmEFI1kBMe0dzz6xvL2p8POOSw02hLSo7jXsqM8zur39DtZdpsOUdMhJqFKWoaBpG4HjytxMuIk6grq2U8UIDbKPKbhjherMIuelcQFQ48ujbAGkTJM/j4WUqbhfhCmeCV4fuqoWUHKF3Y2RmJmAI1Gp5c9rSQzdzi8qWW1JCR3SqCoeXIfCyxQ4ddbkgZVnfWVK8z+dqJrppNiVCSaCIXcAo7oOFeCSpT14YQuxxajISqiaBPQkAaWXLv4V4LcgNYNuQKX7S13YxI8u7oPnaUrlwJWVagpTEN75z3RY9el64AwIAL3vBuqrNk0rHeUVchE2uwYdW1DeLK7IzqTb05n7LBqcZhzcOFuZ3QBRbV+jvhHFNxXncacHXW4LwpHaYlNE2kDOhQnNGhE7i3hzxG4dXxw2xpeeEb/aQ1V3dVO0TyErCgl1CilQmI3B+FvoO/fHE/HFP2OFbmbw3diyUioqgUuFP8yUDX/w+dvC70trkvXCPpA4uw7ftb65W3ffNQhx9ZBLoLkpcMndSSFak7629D7JRGlc6OMuc0jciwNul9Vy+LSvqG5pi3MOQNyPryV2P2ceMMVcTOCN68JcNcMrgcbuS80et3sWUoUtD4zhTqjmKnAUKAMeyEiNJN2uG/o7v8P6lzFl8YkeqrxCFgUzCA1SthW4Cd1eZPLa1Gf2K+NW7vvfijgV1SZfYoL1Z8m1utL/APwjdvTm8757VtTM6KFliUNFS1r5JSS7Sw5BCgr6p0Igi0bz80kDbaDYnerBfoXAAJRCx7rHJ1525MKGU7EG1+E5SCjPFwQmeOsgz42wTMjQGwlS0WnnGlE91ZE++wc5uZ/AW6JhuLrHcLGy6TqOZPjv9a2OUCu8U9Qfl/8A9WKSeW/LnYakUA8kAjptGkH9LGYbFVnahKUmYjY22FQJiPEWDzE6je28wBiLWggEXQmaZTvytuTGhmbBBRmY526Ss9ZtJOhAZ8bdpUOWgsCCTBB1Put2lR+OlphQQoXsZINiVIspq6xhX8LiVjyUkfkbG83LSxMyi95HsvMdOaVf/nWi8bFONkO7oJPxFh7pdLVc06P4FhW9gHNBofhbu7z/AKwD1NmcLhIbqVBqJtlk+nvBHq7WYiciZ+FstTUrFfNrWV1bXJRet51rtTVKQl156ocK1qypAEqJJMJGk9By0tYngjUprKSovBsaPU7CoIgiQTEe+LV2qGgLlSpBRm9WJUCYkZee/M+Um00ejVWqqcIOu5iXG1JpieYIJI6clC1ujFpECqN2KbC+5AheUwJmdD0+utuVPgAwkQFTM5RP19b2Khas2VCu8dBrB1+vvtwpRUCrkkhOXTveevWOlthZeVKDT2dpSQJKVBQGXSI18d48rAhwySnQq0EeHMfpYohamnFdwEK7p1Oug+YHw0tijlcJJUrNqeYIImNfIfnaOymnThSsy1D1KUAlwBaZPdGU6/DMT7rONbyCQgiVITAVO8FJ1+MWjm7KxNBWsPAEhtUkZdxz05GNvGz8SISEoXmSQIV/MO7B+61OYWKu05u2yMMvISsApygQRIB5R+I362AcLagcqSd1eB0PTTmbdylTaSBtuNoJIGg22tpRJUZUqFnb4Db37bWDcKyAuUaryqOhJJ18T87I2Lgs3E4gIGjyFwQNO9p8I+7nZYUnLlJXKTBGg20/GfzslYlyKud7MYSSg9Ce8PlqbQkPgKLH7wUMKqOx4/8AC6pOzV/UMgE/+lta+PP6iPUlpsxMEwCZHW3lZeYnjhw1hZEX5Qmcsf8Azps/n8fHT1bZQopAJFsaUXsrrTqUWUS0A2imzLJnPPs7frboBWVRB1Ow8rGVNDmN/C2nE9xUdOfSzxhQkKJ0ztE27F6MKdaA0SkwZshrQnOpSUQCo5QRqB4+NlRxBzKmd+n142LKbjQER4WtgWVRzrlVf9M656u9qHCTNO6tuKypzZACSOzSNbVsRgG+VFIcrKzSNCEkaeY6/WptfLiPhFjFt7YeoqmQGXKhxJH+VOnvANuKDgbd7jyHiwSknNISAfAQBZxTmQ3Tcfhiyo23w/vNcj1qtBOvdKNP+WfH32PIwJXJSCl+tOoCZCZ93d+tLXsXwHuju5GVSmAJTP8ATexlrgVdqDPqvxH4Wn3MqPegqIJwHeEBWesIBPtZdd9u6PxG9hhgO8gJCKzQiCVDUdfZ00+6192uB9zpVnTTQeo0sMOCVyDUUZgHTS0DQuUxVjmqG0GAK591IcTXgaD24jbbTw+tLLF58Na273TTUVNWrbCBqp32tj003td5rgvcqSFeqGec62Wn+EFyuHtnKRJlA3+dq76B+ZWI6tll51u4FvZHdXTVZnQ/aRI3108rHKHh7XLeSk0tVvAhwxH1+Fr5VPCO5ipR9QB3GwtjPCK6GzIowPCNjafs1xG6H31oOyq1hLg9QVYS7WM3hHMetKSZ6bfPe0iUvBPCqyls3ZW6gZj+8nzO8fxxawdz8OqKmPcpwnTTay1/YulzFXYjWIITFsqfCZS7RyvR4lGOSr89wiwfTtMIF2POyCoFV41BI8j2mnWwdTwswozROrauasStEJCfX6jIZ1JIK9d/qLTveuCahwJDLJjKU5kAfH6628z8cYZ/aNqxbfbWG2sYrulNc8KVSDShJRmUE5QUhRERB3587Uzg8p0LrfVWhiUe4F1ZxXDK6xUKdapX2kJbjWteMGN9VGyUOHFAp1akNqKgDqmqdkADX+L7vO1Y2cAftBK51v8AeNJj5wsHtRD9MgDSNI2JlQ05ac7KdRgn0z3r8uupYuDH10MNPI9Y9YvoOodGcSVZj7O8gRvYLsFfb+oFaZirR8CsJT4Fu9jLLJlpfdT27vd8fasUxJgpht7KGCtJzEkvLk6jUiYn65Wl+mulS6VXb06J9kefhYLENztIC1LYQgI0BI121PytmjDZA+4Kv99YWbKBmsFpddCVUIWCDI7VZg+czZbpuHt1qYBN1hWWIJWvlp18LSUzcbZeUUJQVpakwNvjZbp7nb9VQUtCAAkmOfjaE9DKdiox1cfRV8vrANMg5WrvSlQ1lWvyP32a39imC6lpd3MnOqAC0DAM/wBfoWshfVyoVVju6BJkH5mzZaw8isvgLQM6Ge9qoHcHl9a2NTUEttSgyVMbnbKMk8MbqLKC9dFKpSid2hr05bRbP9F91ggG6qYKTHeDYkyfHn9SbTe1cgdrkJUgJSlJy6aTyI87N3irXnA+B77xGyUdtRUi+zOSD2yhkbPuUR8PO15lJITa6Z08YbchV6xJWcN8LVC6KoqbtNY2ohbDbaFKCphXe29qAdZ5RaO784jYYbzO3ZdFIcpCSpNKlZSMxmAfE6SOXIG0H4mxA8isdqHHnFKSSnvKJKgrnr4qmPDaDYOnqRW0pccKpA0bIKshH8swTtMctDzgdLT4VGwAuJJXPT4i95s0WT3vjiPeb1S0G10rjTaloW2uUKAncKghIGndMb++xa4MT1lNja4L6CkMro71pS2oHulKXRmBnWSFDX/EnlNmiw/V0jpDThAI0UZJBkAgSNyY2nkecEdtpVMhT2dSHCgrStIyKROqCCBMSEEHQg8tBGxTMbBI1zRss6SV0gsSvQXGV4YGwhfd4UeKcU3BS0t5MN1FXd1dVM9sTBAWlgErIUIBASQYEHS1cr89HrhXiJmuxzhelCrsCnOzbu+uz0bikGFZA2SQSdShKkgA6AaS9MK8McO35h6nqWKSodVWzUVL1QhtLri1AZlOEqKnDMklYk69CLC02AcRYAVUN4Dfao1X19nWNICGGe6f78NQWe0RoElKEiFwSqBbpMS7LT1UxrovCHWJAR6XE2CIQSjNba6hG4LiwtgehvnDeJcO3y7WYkpy09R3fRJcRQtLOVDinCrVSRC8iUkqJEETJhVyuvW6w/dbNaXQw640VFpSFwkwVDMkLEmTCo6RuLXbu3hu5dzjiryzrUFFTpceU4tSlbrUtRJJOWdSdtY1mpPGTDLmGMf3zSIS6Gapw1rXbd4kLGYyfNIPKfG1TFcKdTUzZCNAbIImDnWCVeF/pBcUOE1b22DMT1NHTklL1F/eUTxO4cp3B2apEwSnN4ybXO4E+nhhziJiSgwnj+7kXdfl8VQYp6ykcz0j76zoFhfeaKlaAyqSRsLeatSvIwiFSRBUDtyEA78o91nFwh7ao4r4PaTmWVX7QAJG5PrCBvbl5IwRcozHkFeqnHG6qV7EN1V9StMKo3Gy3qCtSVmBMae2qzAYuXOlvtXWWGPZTlkz7iJVEa9J91pm4yXS9UouqrbpwsIddbzLXCAk5TmPX2fDW0dUNzrdWG0I7VU6JRJA10HMW82xl4jqnBem4I3iUbD/ADdFKG52QAmhbSt7LmlaSpXIa6QN+c+dly77odox6s4GHFE5g4pailCQNyI3205dOqvQ3HUttQ5UlKj3C21OWOijME/GzmufDi6hQWtpa0aEhLkfPbf62tzU9UG810LIRa6bzOHWnVhKn0vKMI7V3MAAdZSkCE/MxtaoX7RLB5uqowNiBqVtVbNdRlzIBJbU0rQHl3yfPpb0EpMOF9SEppwgJ2yDl4nfkLVj/aWYNSrhJhzEKlrcXd9+dgpaRo2l1hZiY6tJ26c7W+zeIEYpGL6G4/Cyu0LGPoHs57qkPox1f7u9JDh/UOr7tVejDEzH96C0Nv8ANb2WoriSWkNtAhAOYlSjAJ3EbTbxT4cOruXilgvEJSoC7r3oqlRAMFLT6Vmemk299Lrwo7VFMoXl5SJ0tr9uYJKmoh4QuSCPz/yuZwCvFFTva421v+E0KW4EKgoWrJoO5oTPLUaD32WrvwsXIbbbSE8zMk2kCnwlS0jIfvB5LLaBqVKAgW0nEFAyr1XCt1OXk+NO1AhtJ/zH8Lc9B2dcyzqx2QdN3H6Aaok+PSS3EAv58h90lXZgNcdpUIS02N1rMe+Bbt2+sK3C56ldFK5fdeNmqdOYA+JGgsqqwrfmISHMUXqtDMz6pTHIg/5jubL13XRc9xMdhd9G20kcwNTbp6PAxEA6JgjHzP1d9m7D7rEnxDP/AFnl56DQfc7n7JlvXBxDxn3b2vRFwXar/wCb0ozPKT4q2Hzsr4a4bYMwco1NBdqHKxft1T57R1XXvHbyECy9U3k2nTOJ8LI1Ze5JMKseWWgoTn/qSdXa+g2CCJamobw2+FvQaevM/dLNReLbSSlOg5AW8I/2m9ym5/S5xZUpSQ3eTdFeCARvnpmwT/xJV8Le1VZeiiCSvS3kn+1suZTfGTDWJUIHZ3jh9DKlcytl92f+VaLTwTGnVmJiNx3BspS0Jhp3PTX/AGW+Lk4a9Kr92lwJRiG4a6iAmASkIqBHX+5NvXqrvYTova3hB6IWJ3MJ+k1w4vhCgkOXyxQrI/kqJYJ+DlvbWprZTJWetsjt259PWxvbs5v7Fa3Z6AVMRvuCpDYcDrLbwiFpChbsn3WTsPVAqbmpXSZOTLPkY/CygRPS3RUcnFgY/qAgzNyPLTySDflNlqQ9ycHPwsnbQmbOC+WO1owoAyhU6dLIBkHTT3W6KlddiyahtnrBpqEgT1sIyrIsKUNiPv1+63Bk6A7cptitQcupIIHvFrgVN6UgsEQBbYPLadCbAoWlaEuCNRmAt2DPTzm1oFVl2DJ/W3QJjbQ2DzQJJ+dsMbREfK0wSUkNm1mBJ0tsKFgcwkazHhYRK4OvLnraQ3UShQfmOtita4GqmjcUDBcLStP5knf3gWHzToCd7Er5JTQKfjVhaHh/uqBPymzu2SG6PLAGsW6oD9tGYb/KwayFAKSeVuqEy/EjU9bPa6ZOHtgnTMNNLZbjtY0z/Xwtlo5U914CjDF51TzeGKVDS611RpQkudwrHdOp3E2lLgZgHEmBbtvS7sQNMtpfqUuMBt4OSQkydNv4dN7NCgvRq78XU9/PtvFCK1VQ4EEFWqiYGw/DW0nJ4tXD2edVBXBJ1MhBAGvRX1tZ4A1pzFBmzOGUBO/snDyVCvPrO0W0hAzQsaSJ2BNmsnilcqT9jQ16hoTojQ+Pe8LCOcVbneWEro64qjeGzz12V4/0td4zeqqmF/ROMIQE6iBB0A8NPLa2EAqKSJkgghQ1Ph+fhZqjiZcSgJu6vnLzCJI5D2vz2nlbj/Sdc0hSaCs7xgwhHPkdfL61tHjNPNNwXdE7MkEpAkgaCJ5bfXWz3uO8F1d1MBTiitr7NUSNdPwj52hb/SfcsEpu6uGwGUJKZ6b2WLl4zXNd6HBUXZeC0rWFApCNwddyLCme0jRWYGFh1UyNr7LveGmnlr9dLBqKe66VLOXnA8/wGnjaNFcecMoUpSLnvUhMn2UaaHX2uv10Bc4+YcUSpVyXnodT9mJG++bxtUuVcu1SgVlYKu6MiYSIAGk6eW3wslYsUHLldAaUkKLaRCZjvjU+8g+7pZhf6esOp7v7kvFUrAA7kHb/ABaR8fvsh4g9IXDNTQrpE3DeoUohRV9mUgAjcZtNh9A2hJctKnG5oIuka/n0N8aeHC0AJSq/aIZVT3pqmjqI93xPjb1gYTz8Zt450WPbvxxxmwC5QUdXTt0t/wBDKX8uZU1KehOkJj429jWMuUqtnSggC6sxkOJIXZA8bBuCUkT4jlFhZzDU8uti61GNRoREE2ePdQek9xJKjGvj+FgHEgjbflYw7lmBpvpG1gHCNSdevS1oKod1GnFvifcHCJVx4mxGwHqR6peowQ6hspUpoqBlZA2R5xNm9d/p4cHKJIDwS5IEA3ixz13BM9bS1eXD/BfEPs7sxvhW6r+oqUmpap7ypEVDSHQICwlYICsqlCY2JsJSejXwFa77fBnBDcq0/wCwabf3otaiY8i7TZAe9l7EKKnf2hHBptJKEUupgZr0QNJ30QbBp/aLcHogJu/XSDeit9dNGPC05U/o/cEWPY4QYKTGki4aX/4NbKlPwW4PsgdjwqwinyuKlH/RYhEt9XfhQvH0VdHP2j/CBoED90CP571dHkdKY2B//SQ8K1hRaRcRSBJJvZ+I6/8AmvutaFrhZwyb/uuHWGE9SLopxP8AyWOtcP8AAbOreCrhTH8t2sD/AKbMRKfi/CcGMclVBX7RnhslIXkw+kHc/vWo8Y19VjlZx8P/AE12eLl+jCPDzCt13xeop11RYavF5IDSN1Fa2EpGp2mTayzOD8JtK+zwxdKdI7tE0PuFkXEeEMOUuJMOX22qkudVK5VNKfZbaazBbJGUlSYiQNOoFo8OXcuUhKwaBqYycccd6lvtafgzd6klZSCb6b5Eg7gcxbpvGHH6deEN1thO83u2f+sTaQ7rvDBeH7vYuX+2NI8GQopdrL0bW8tSlFRKlFQkyr7rcv4vwBTuFD+MLjTG+e82Rr/x2nY295Qv/wCqZNLi7jyCFL4Z3WhM8qtC9f8A2wsrJxVxrfAIwPdSOpKkKAHT/wA4squ4/wCF7cJc4hYZQNfavdiYHjn8R8bFHuKHBptaVK4m4QTBMhV8Uxn/AJ/CwXRB27z+ERshHwoL+0fGwoQBhO6Ukci2g8v/ANatir341LaKv3DcbawJGdsR8qk2Bd4y8EEgpc4p4MSRv/2tTkj/AJvOxNfHLgCwpOfivg9J03vRgg/81h93b859Qp8U/KEkIxfxgqcT3lhdVHh9ipu6npqtwuAJQ42+XQnIe3JMFlc6CNPcBV0PF2tWS/V4XKCIID8Ry6mLN9rjv6PlBxlvm/HuI+GvU67Dd3MesNPBxtTjVTVnICkEZgHZj9LOR70p/RoZlw8T7k5SEtOqPyR9TYJpWOHiefVHFQ9uzB6JNGF+JACQqvwwCFSAKwiAP902AvLBXEqvIKL2ww0Jkk1qlSTz/uvL42UXfS79Ghru/wCkuhmD7FHUE6eTdi6vTF9GlOUJ4hNuFX/d3dVEdf8Au7B9mU97lx9UX2hPawaPRIqOH3FJDhJv7DQVGXuvk6dP7n8LKFFw84oBsoOJLhaB1MOFQA6asW096Z/o4NwP7Z1C9/YuqpP/AEWJu+mx6OiFjs8SXm4BAhN1Pa+RI8LI4bSu3J9UhX1LdgPRavXhpxCTUN0q8d3Il6pQtbaEoClLyiVR9lrAPum1frrvX0o3kmsu/gdip1twdol5D93ZXkkSFJzjY6EaDe063h6a3o3VgT67UXnWBsktpcucrCVbGMx00tN+EKmkvPCl13tQAopaykZqGUqTkKW1oCkSBtoQI5WYYZBfwE+qXf5j7w/CpWh/0vVLC08E8VMKCplVdc45eI5bWjT0hr69ISgwCpjibgW87humvq0NIfq667nQt5ILiW4pxnH92TOg7oHO3pe42lWuZSo130FqQftPb1FLhPBOH2n1j1mqraxeXcdmhtAO/wD61VpsoY43Ai6d1bI9pBXmpfleaipfo9VJBykEEco0iQdvOeY3OsPVqFt9lUIQS1o53u8Y7uvSZHWNTFi150jz7/qxfl9CU5QQSV6+0k7GdIjUiPKyVdC3Wb5WioQtCghasq1FMq3106Hn8RbSGipHVPNAh5KwpQJVBSQO0KVc/foROghR3GhxwsOMJNMlJSheckJBJVAjLGkbkxGknUmyQ68yKY5ohTeVKUxmiQZBJ5jLygag84VKdCHaft22kkvZVgxCpVqTqBMd1MAnaLEB1QyFdHgTiFN58Lborbrap6q8KakQy6pwCUqQMpkJPtd0HfprAssVd4XpUX3RLvOnYC3g8zlQgBJGRRA0HRJjrA15Wjf0XKxdZgN1ph5Lz1FeD9P2mYg5VK7RAkRMhwGfPraWb/rGbsoGUXheDC6x19DFKvLnzLPdEjQ5UiTPMa29ao5jNRRvJ5BUMoa8oq4xTUzLeSVvvqLhStSkjLmkb8tU8to6gCsnpY3O36/dWJkNKyPB6lXmlIJIC0ySY0g+Gm2lrTUIrXaVs1iaUuinCXXmimFLCf8AKmBuRoNOdoU9KO6BX4BcrUTnu91urQtAJIAVCh19lZ6T100p4vTiegkHlf01RGutILKlt6UjrKAtQCm80CE90adR7vL42mb0HMDPY39JPCVMFJDV1PKvh8qVEtsJKoAG8rKNLRXedCqppKgtKW6UDNoNIA0M89Ofh01tLvoD37T3H6UGFRVultNciqo0cgVuMLge8gfK3kktw0rVi94XXqxxKuxuvuimzoSQ3VJV3gVASkjbnrGm3zs1rnwglynQtCg3BBJVoOegSNTsLLPGPiLc/Dfh3e+PL9u+trbvuZLT77VIhJeKVOpQCkKUBoVgmSNAedq5UP7SXgLTJSo4Oxok5o7tJTHl/wDTjW3mfaHC6+qqeJSsJBC9EwTFKWlpOHM4B11ZGkwlffrlMzTqpRd0y6VpVnnllSNDz18rPShuFCkpabBCQZUY0nymLVPb/ah8BmVBDuEMbDkQKSmk+f2/SylS/tU/RrZCUqwjjxShqR6lSwP/AL+3MO7NYxKfFEbK3P2ggI8LwrfUVwNlSQlRjSTl3tDfp38Oq3EvozX8q76QvPXZUUlekIElKUOhK1dAAhaiT0BmwPDX9pF6K+KWKqorqnEFx+rIzobvG70y+RulPYrXB/zZQZ3tHPHz9qBwsvrAmI8D4Nw/frKr1u6oom6ldKypJzpKTmlzuggkTBInrbWwrs9NSyNlkvnB0AFz/gLAqcRkqTl+HmSV504YF13Nf91VF+NOLpaesCqhpqQSkqCjlUecAxuRmt9AuEMXf2uw3dN9YOut0UN50TFWzVVQy/ZuICkmJ1MEeFvnIvbFlPeLv+rsraV2gXJAiNY0HmfibeqPBz9rN6PODOEGDMJYnuTGH74uS4qK7a40l3MrZU8wyltakKLwJSSmRIB12t31TRyyObISR1sLn7dFgPkYW5Wi/wB/5dX/AP7Kt1boqL+rXK5Q1CFaNg+CRpZVQKGgaDdMyhtI0ASItRB79sX6MhScmH8en/8AZtP/APlFkiq/bDejg7OXDOPv/wB30v8A+UWpOZJS37pTku6kXPqULK6SwkeLdL6K/dVe6EJMKsjVd9Eg97a1DH/2u3o7uJ7uGMee+gpf/wAosnP/ALWn0fXdE4Yxz/8A4NL/APj7c/Ww41UknhOWhTxUrPeeFeuqveQTnsk1V7knRR91qNu/tXeASz//ACvjffnR03/4+xZz9qlwEUSRhbG2o50lN/8Aj7c/NgWNy/8Ahd+FsQzUDN5ArtVNepYMkxzt59ftWbjN4YfwPiBtAJpna+kUoH+YMrSP+RZjz6Wcq/2pvAY7YXxqf/sdNp/9/aEPSt9Nbg7x34fUuGrgw9iZmvpa9NU25WU7CEZOzWhQlLqjMqTy5G17s/geLUWJRVEsJDRufqFOuq6GWkfGyQXsqdYPvtzDmLbhxEyqHbqvGnq0naC26lY+63vO24qqbQtkKUHkhSMvMHa3gAXGoOUrCs8pECI8+u1vQ+6v2leC6PB9y3HWXfidytprspqas9VpmaZgvIaSlfeDxdcTmCjJWienK3U9ssFqMWEJp23Ivf72Wd2cxGKi4jZSBe269LsFFaLnNM8QHGXVJKc3eE96D0OuxsvH52qt6CXpE4W47XfjCmw7RVVG5cz9I+8y/TIa/vw4AUlK1Ff9zqVGflFqCTy57WVBTSUlOyGUWcBZFqpGTSufGbgoN9CXmltH+IEWajoKVqQT7Oh5WdiiYjptZuXs0WaxRSNDChHW27RHUhZlUNMyLFcmT8YtmbY6H3/hYMqG+Xa2T1M21AFmk3R2mUexSAT3ZR7gYsLPONTpFidOsHOkKJAVtzgpB++bGAox7VrLdQq50KFJO2sx0tgUDr99uArlI91thQnY2INEyECpgHXw526QSICTt99gSSYAMaW6SoDY677WkFBDBXXQddrc1CQ/TuMqAIWgp+ItyFDUdRtboq52kkuaR3taNlRkkoE+ca2M0J+3G8DlZPu+Ay5TwPsnVpA8Jn8bH6IgO6ybME3NOEAwNVjT+U2ywE/4494/O2WbVOvBLFj1RdVzVFVSOBL7CUqCikESVBJEDQTJ8JnY7POl4L8R32EKTjO6klSAsRSHTNy9n7vHaTZp4+addw1WNpRJWWyB/vp28fz8bXP4bcLsVcRlmlwpc79c5TMhx1LSUhKExuSSAJ1jXrFngYx9y9AqHvjtkVa0cEeJAUD/AGzurLvpSaAx5dOf52OM8B+IjmZs4yuwqA/9CO/w+ptYiuw7X3Jejl1XrTqZqadwtPIdTlUlQ3kdfMWn7g36OTeP7mXfTleKdpIyJzJzZiRsI5DT8bHfBCwZjsqzaiVxsvPRXAviUpIYRjCgccyapRQKJnTQaz8bBVfBLiRSns6nGFG0txEoCrtIPmJjQxHu+Fz+LnDF/h9fT13qqJLZA7RM7bxpr0+Nogryt1Z7RS3FqTlGYyoDkDrP372dtPG7UJd4kGhUCK4M4/UEhOPqGMuXS7xrvO/ntbocGcdOlLTnEOhSUKETdiSNNf5tt/qRaaydZKQJEaazpy+NgVCEgISZUNRMeOtp91jS7y++qjZPoy8THvs1cTbvPabkXYJI+Omn4eNu2vRj4gKzLd4p0kHci6ZEkbyVdJ1+dp/uCo9dulp7tSVtQ0oHckAR8RZWaaQlQCgCI70E6kae/e1F7A02WrGA5oKrafRZxmpUq4uU4MkkJuYaf8402+hZNvv0VMTUtGqre4rNumUpUn9yhJgq5faTufna0ak5kqDigNpBG8yNpsk4uJNxvvrKRlW1ISQZ+0SDHjpvYLxZpIKO1rSbWVJcH4aq8J8drjuCpvX15V34guuHUNBorCnG3CCkT1A3O3lb21p9WwSCR57728a6tzJ6TNG2lQM4hulRKdI1ZG46EgW9ladOVkJ13/G2fMSbFTjADiAuiBrCjO1i7o38NrGzH8p157WLPEkRPO0Y908mqT3x1PnYosD6HKx51IVqoSZsUcAzGSdTztdaqTt0g4jx+3w6p6e9nrpdrvWnxShDbiUFJKFLzEnl3PnZF/8AKhaCspwZUHKo6evJHOP5LJXHdSRhq7FBWSL0SJ56sPWhcrCFGUgjoTP1z+hbSpxdiz6hxDtFYJPpVZDCcBu6fzXkAZ9zdulelo4lHcwGD4KvOBtyhq1eC6ADCoyiD1n89futvPKiCZGx0kR8I6fCxiwKvxXBWEHpaXidRgGnTO03kr7+ysEr0tr7UTkwPd4CTBJvFZj/AO7tABUZKDBIMAADUe/3eNtFzLmKNdAJkb/X32YRhPxXFT9/5WeJCPssGXbmJ29ccP8A062jPjrxRxDx4uK6uHF43XQ3VRXheIW7UUzq1vJ7NpawAFaaqTB8LM5DqjCgkASIkCAI3tpt4C/bjWFjuVsQQebTg/H5+6zujBFikJHA3UeXn6N9yXS+lly/rwd7T2VBLYA5aSnoBYJv0fsMEBJve80iDoFNa6bexvaasbAB+mKz7aTlPUTIizadS6QlNMop111gTpYLqeIGwCK2olIvdR2jgBg1IPaXhei0lUd59AJJj/Dp/X3n6b0fcAuJUVPXooySo+tAAAeQ2+t7OpXrCNFE9yCYnnH6WU7sbqVqLocJZSTIBJ5dP0snQRgbJ2zyE7pns+j1w+ToBeUE6j1uOcRtrzEWFPo/cOwCVUlYRuQatR5eFn22+JHdAB0nbUTYwHhJSo5tYPXw/pYfCj6Iwlk6qKxwZwJS4zcpPUaxdIbiRU9gurWUl0VK2yrcH2SNJjawGI+F+AqGgNTRXIUlCtvWXTmHT2uWnjzs/wCpj+2NOAlQL2HqpGpOvZ1jB06aOfMWJYgbS/dlSkojKmdfDX3bfdZNiYWnRM6V9xqq+1F2YdocYpoauhSiiNKt3Kp1YCSJ55pnQWR10NNX095111uf6vQZ1JVmOZahEASdEgCyvxBCXLzu+7GVILl41CGSJM9mkyr7wIsnXShLFNjChIc+zUslO0pyHaNfD4+FgcME2Rw4gXuuL8TRMYQu69qenCXVuNhxRJMg7zr0+NlDF120jF001TdlKlBD7bi8p3RHekg7ADX39RZv19QHeEFI5lGdCWiJVlg9oNZ5acvEWGxxfjQwlS0/antH2mllKFDNlSBOnL9dLOWNASDyUvUNDSXpX0l5OUCWqBVShDKQTDqTzKehOvPa0gMekTxsurHtfguk4k3wxdF306WqSmYUhKWUpIQAFBIMACIn+GzPerW2rloVtRp2KgpIGUdwxrz57a/IWI0V309+8YMXFXaJdoH1BsNuCSlSlSTrBGgPu85rl7Y9URoLzYJ8Yy9IHjhQ1tyCk4uYkZZqKrsn0prSMwzJBG08ztraNePHEHFONa6hpMSYnvO+F3e0tTfrtWt/ss6xmCcxIAMD4DrZ/V+DLjvFFNU1dS9FKsKaIcgJjcQNP5flrFoa4x0dLcWJaxlpwllNGzUOBagpSlqJAA1E6GI6Zd7RiqGSPsCpuhfGLuCjW9WbmZYFZeTrocHeATEue1oJ13nXpBnqhXU+aquXUKDjbKGyEJMkJBVpvuAfu99hU3bX30+XnnA0ypQCUZ5AJPw2AJ8culuKBz1e/wBxpa0rIZCTlIOWNNCdPERtp0i1y6CAlStIQlTq3G0ZoIycuZ567mSTqCZ9rRQuFdL6g+yttE0jhCyCnKUqAKRJMGCFCdD4kwLJNQ/nRmU4MpOUzOkAAakwZjYfqQ7nr2qW9w0FJyvJLalZAOzcnTWJGo5awrrrZwUiLqyfow34xTLvi7XnlNKqa1DwUteYZVMp0JJ72bLI13OgAm0xYii8b1ZbpQrLRvNLSEoJBOcRkERAI3HkIM2gX0cK+hpb0v5h2mbfdaYZdZW23KQsqVMa92cqToNhB1Gtg2qijuG5U194IJqKlxBKFwIWVARMyImTOwSelvRsCcZ6JrCdAqsoyOzJTfevlbgRTILbTOeQFQVyUgqnfaN4jMN50j/jXR1tVgesoaynQl2qpXVEqCiJGx25k6CBAjbe0rUbl7UIFJeCkhltxZ7VB9mZEQBI/jnU6kbSRaD+OlVijGSqy67mqV0VPTJUkrRIUBGoEQBJJlWsDpbWrv06Z25JFrIAsXKoFI6s04qFZlFsqCgsghMAaEnlJnnv1Ih8ejXcldXekRgahuZKg+zf9NUJUknvMIV2ijOmyEqnaRaPaEKoXaijWQHWVqQogagpkE6eZ36aeE2ehzUXm16TmCn7vofW+0edZfIJORvsF5l+BSmTr0iZt49OMoK1Y9SAvUXiNw6Z4q4Fvvh69WmiRf8ASGj9ZDQd7ElQIWEEgKggGCeloJw3+yDuh8pdf4qVrrR1KlXO2lIEzElw/KbWww/VmjvelebTT5kOCO3MNEzAzdBrvy3i0tLwtf8AiBXaYsxKlFJ/6Dd0ttqH+JZ7yvkLcfiT6lsrRC426CwH1JO32W1E9jW+Kw8zcn7BULxx+zN4F0uGV4duHHF/Xli3OlLRoqZlxAE94KQfZ0nXtBqduVkPCX7GFm8UCsxbxdrLvS5B9XYuxtbgA2lRXA05Cbell20WHsOUwpboomKdCf5EiT4k87Y9fiQD3xapHX9yJdUTX/8AVu3qdf2UZJDMMsTPud/QaD8qgzv7GvhmyiaTjVilpyAMyqGnUNOUaWbOI/2OWDKS6KyspuOF+u1DFO4tptV1MhK1JSSkGF9dPfb0Nqb8PJXlZHr73C21IUuQQQdbZdX2pyG8Rsiw0U0lg7ZfOJiPD7mH3+xW4Xc5WknLlhQ/rPvsgkk762lHjldT12Y2vu63GC3+7r2raWTpOVwj/pn3j3RcbeiUcpmha87kLLqWCOQtCwEjrrbWpMzZwYXwBjbG63GsGYUva/nWBmeau2jcqXGkyAFLS2CUgkwCRBIMbWc7Po38f6gEs8E8dKCQVKIw/VQANyTk0HjYrp42GznAH6hDbC94uGkhRvHj8rZH1FnyngbxmVd5vUcKMX+pAhPrH7kqezzHQDNkixu7PR44730Cq6eDuMqpI1zN3JUER55LMamFupePUKXdpT8B9Co7n6m2ptKn/kq+kh//AGOxt/8AuV8f9NtK9Fj0jU78EMaj/wDYz/8A8Nm77Tf7jfUJu6VB+A+hUWTbJNpTT6LPpGKH/wDRTGI0nW6Xh96bIONOC/FLh1SUtdjvA173DT1y1t0zldTKaDq0pzFKZ3MWdlXTvdla8E/UJnUs7G5nMIH0KZxKOzSQCVGQokaDyta70W/Qww16QnDeoxxX8Qa66KilvV67V0jNCh1PcaaWFZisHUO7RytU4CTA10tfn9mji1LeHcb4UdcSOwq6O8Gkk6nOhaFn/wC7b+Ns7Hpp6aidLTmzhb91pYFDDU1jYpxcG6tD6Efow3T6N+NcQuXRjisvlrEF3NtOMP0iWkpW05mSoFKjJhaxEc7XEUAB0tAvDe+P/l3d7ZX3XS40T5tq08dQLT0sgm3N4ZVzVkZknN3XXQYlSRUcoZCLNsuTtp8bJd/U6i2h5P8ACcp8rKZE6T8bB1iQ9SLQRrEi21TuyuBWRKMzSE0gQRr8OQt0NQAk67W6W3lXlI3+G+otwQRzJ87bQN1jrthSU1CtNFtg/An/AOKxoEakjfexFJipaXOqgpGviJ/6bGgokyNxroLWI9UF267WsBClRMAxA18rETfjYb7Ruiqz3gkJyd6T4T4fUi3dQ3WLUg0tSlCdMySkEqPmdtJ1twwxegMvXmFgH+FoCR7/AH2sC3NQWv36Nhd1aZ00b8/H4+Nh03oolQRRvrCUhSTEBYMbTz12MWBTS3j2SgbyPakkpUGwQE6QI5nTfxtgpbxJSTeikpgZgECVEAbdOvTXytMZVDUJQpKpVQz2qqdbJkpyrGog7/jYxm55vOyQijvI6qvlRBER2YAmInrM62U0kaAnUDXxNmPknC4YHZ1dSkyO0SlwT7wfuFlCintZI21M2TVKCK9gn/aJWjptB/OynQwHdQNdpsyY7pYJcn+P/itlu8q/5j8D+Vsskrrwix0Ut3O64AmVvN5UgR3ivn035G12+DvGzFfCRblbhlVKU1bKG6hmoaztuAbEwQQRPI8/G1Hcfqi7MpTJNS2kykbZtvLzB5xvBtvgekbrr8u26n31N01ZUM0dQsESEKWATBHjt1Fo09g1xdsq1W7IA7pdODEWNr1xhieuxNfjra66ue7V7KnKgdIA5AD5a2lrhP6Rl84Au9y6aZNPUU7hBh0TCo3GotCOO2XbrxZet0VFUKhd1Vb1AhQBnIy4UjyGnzskCsW0vVR7u+5nra+A2WMdFRa/4gpE4k8R67Hl5PXneVSlx15ROZOgHwtHVSorzKKwrQ5v4swnX46726FQt0FOYBRkmDqBtH3/ADtyumdySUgxA8+WotJoDdAkdURX9mVBa0lJ1JmZ1+fPzsC4gJC+8mSDmnw5mfvsO8ypkKGcApOgkgzGu+362Kr7EgGPaVlgiYAPTfaetpqVku4OrS1eDlE4MyKgFQMmM41HxmPfZ8inKgtbYPdCiIEgaSPPW0UtPLYqEvtuFK0rCx5gjr5fK0rXZUoraFmub9h1tKiPZAJ3Hxtm1rcpzBalC7MC1a7BRILghMQCDHlHwsj4kZUbmqe0AUlKm1GdiM4PPx+FnI6krSY7oTJ1O2ug+unKyLiUpbuKtKkKOVvMAP8AMJ+Gvvtnvf4CtNjRmCp3eXZf+UpRuoSQRiC7EjSRopnaNDrPXTpb2UZ9kSdZt41XshbHpJ0a3m0JWu/LqJykAkHsTOn/ABafdNvZlhsqG8wdPC1N+wKdos931WKA3IA8bF3AJOXTrpY4ptIAJEE9dJsXdEbAd6zxqMlkQfT/ABGxNY1knfbWyg8kEa8uosVdGkHny+vfa4xUnbqLePCJwrRKRui8myD/APVOjl52gtSVESpwiecTawPGulNRhBsSO7XNK1H+FY5+doL9VOQ5IMGNOk/QtpU7hlWfUC7kSKHE+06SQZIHPQ6WyEhJK1bEqBj5WNCiKl5QrlIMbDmQfjp/Wwhu45gsZYVHTQ/Gxy8KuGEpOWnUHMesgb23lURmUo68pkedj3qLijlCisAGdJOnh7xZSGCsUuaowxehCgYUKNzaPLptZuIBumLcoSAAAoEOCR46HUaRHXS28sXpdBKlaXg0NY/ilMfOzhbwXiYmFYfvBASI7zKkjbTePGwNXh2su+puk19C5Tl28KbIF6EgOoCo672eQPa3MRomicx7soOqU8ZsNvOUzmZRAQuIMDkdf6WQH6SiYbJqHy2FzMqglQ5fX52kLHFFSU6KPMNcq9zue7zmdvrWzDqqKjrlBpzVKDMAxrEWBmzOurIZlbZI+e7O1ITUI7pglKzGpPTcfjZZux66lMFtypl9xXdbBIBP1Fk9Vx0CVplWm8lXQWV7pui6FINVCs7ZzIlU9I+vO0pDcJmA3Q/7uaUkuEKB5iSLdpu2kV3SkyQCRmyjx0sYbcaQgAqA0mI57zba3QFE5yEgHQnW1Y5lYBaEitXbTPcRcLUj8hiqprxpV5SQcpcpVEAnnAJtYo+jVw7rad6neqb5hYKFZ3kAHkdkTavS6lDGOcIVadOzq69uVA86dK//AHYteqnKF5VpU7KpC+6DPlP4WdhNyEQNadVX4ehbwGr69ivqrvvSrrKdyWXFVzqQ31PdTvFq6q4G4Iub01bw4Su3atzDF80dNUdh6wvOsKp3CZXIVPaI5Hp1t6FpLKKxFO264CpQMEgc9rU44vu09z/tEcEValhBvO5aJkAfxK7V9ESOcdfw0T9BdStqpFV6Gno5U92i42MDldCjRTLl41K0HWYADg5iy/Reh36NtRTobf4X3W8lXdX27j7q4GwSSuQOqZg6dLSUimSlbgNP2ozGSpBUd+tlKjWhitQ00ltDaVbjKB99nyhTK8oeJ1z3VcWLsSYeudopaui86mjaZjRLbT6kITEyISNz1+LYu6vbuPi/iu8jop+haqCDPeKmWzv/ALwPPU2lD0iOxTx/xndAZCnBej7hWhspzFZC9B5L3/O0NY8qv3VxJdfhpLddc1ACc2i4ZbAB96OfXxtgTvzZmq3C0MIcny/idx9r1dNMUzBUEg+O4O+p+pNq/cfMRF/ESqenq/tlsMioCTp3QrKPCM34crTSw6EpyoUSkkSY358vM8udq/Y7rbiuvEF4VlawuovJ+ocWULSFKPeMKHJIMCNJgDTe0MNitIXKxWyXYAmrQ3UpTX7wvmsU3StpDmVLkK1nQRt9QDNidxEVt6POMjszULCUJ1UQkqGnWdtfzsSve+7yvJXZ1CuzanMGhonpPjt+UWPYWYZX21TVEpYbGVRyk77bAzoDobbw1KyksVyUOiG15iqVKAVJEqVB3BIjXWJnxEAUNMyh1Km1r7SJXASoBO55kcyI59NbdVVQqtdW3TH7NRhKQgqBSNJA3946z5K1z3a52SHnyvtFpzNk5VH+Hr1kE6z5SDaQTXspU9G5+maxnedOtHZmooGnWyXEqSpSXCConY6q0HtR1nSxNUmkvq9WaN9C/VKZKgooBObQTsfPQ6E8zAip3CtxN3Yu7OnXToNTSko7QCCCpBBTH8UAxA6nu2tfgHLWAO1YDdMwTnzQSogGQJOqTJEa7jfe3oPZQ54cnmqdUbJ//vKlvC7qZ6vQhvM22pwkTBiCoHLI5eMAdNI1xBfdK8XqOnpEvOOLWArKYHidNDHUTIsttXbfl7FdTTlTFDXIb7FlDZSppABMgiFEkRoTpJ8ij46qbg4Y4bcvK8B2lW4nsaWnBSVrXlA7oMRoB3jEAa7ablTNI65Is0c1ARty76qkXEe628McQ7zoG3dA8KhfeMIUtIWU+4k/LeLWo/Z3cP8ACuLOId441r/WfX8IUjTt3ttuFDa1v9ohTi41VlSmAj2RnJM2pvi696+98U3hX3opHrD7pK8iRlkcgee0Ttpppa2H7NTGDN3cY7ywvU1CUpvu43ENIOmZ5haXAB45S6fcfK3kdeQ57y3a5WlT6EXXpWtJgoCimRGvI/XXp4Wk67cXisuqjqUOn7ZhDmvikG0aEZACJBkEHx008YsTwzfS0Xf6kt3WmqamnA27qHlpT8ki3l/bR0kMEckZ52XW4HStrHuY7lqpaN+5v9p75sWdvmBAXy62Z7F5LIkqgdbdprHnJKQVC3mMlZKRuumGFsZunE5eqlbqsUerVLzEqs3b+xJcOFKMV+K78orppyCQupeCCqOSU+0o+ABtENf6XOA6y+hhfh9QvX3eTiHC29VTTU2ZCFKjYrJISYBCZsSCkq6zVjTbqmLIotBuvO/0xbo/dHF3G9IJHZ349VJGXbtjnGu+zgtWVY70Hfr1tYr0l8Y3lj7G154xvqjpad+/GmansadGVIypCAIMqBhB5mTm8LV4fTleWidlHc297wMPbRMa/cALgcUt3hxCsl6CnGK9uEfEe+nLsulq8F3zcy6YIWCcqkOoWFRmE6BQ1nfa3qbw8xXjjiVgq578rXbvpGq9txLqC0HFMOtOFCoTAQc2ihKSRmiTpbxm9HV15PFy5KZh7s11Rep80SO80vlz1j329XPR4r7wuS7X7lqL0dqKRCzUNIWkZW1rAC8kCQDkTpMaeJtzHa+kbm47R4tLroOzt5Yw0C9id1NzOELnbfTW3s4/elSmMrlY52gR/lT7KfcLKq6plhIbYQlCQIASIFkdy9JTmK9vlYhUXmn2s5PlztwJc9xXVMpC7V6Vn7wzSUka87Enq/2hm/h6WQai+EJIBJ73dPn9ffYkq+0FwIBVOaDyOtpNieVbZAxqWHq9UnUkjpao37Raj/e3B65rwSlS1XdiGnKo0hDjTyDt4lNrF1N9CNUE+AE/G0Fel4j9/cBcTsgEuUSaeuRB0T2L7az5HLmtt4KHQVsT/MflVMWjbJRSNHReVbYAeSlQnvAGbWa9AG/1XZxbvS6VOEIvS5XUZR/EtDrah8s9q13m32N4VDfLtFERGxMjbTaLSn6Kt9m4+PeG6kKypqHX6YiND2jK0pHxKbep4lHxqORvUFeX4ZIYayN3mvU7DV6+oYruisU5AbrmCoyNElwA7+B/C1tVSBsR77UOcv1xpaVoPfTCkqTvprP0Putei6a5N7XPR17eqKumafTpyUkKHyNuLwhpYHNK7PGCHFrghdTO8W5UNNvObCEd3aOlgtSQdd9bbrVhOTerGexfcbKdAZFi0QSN4iLKl+tlLiHAJBTFkjOIiSZM22YTmYCsiZuVxWLARkUDP2iADPUx9xsaUDvrMWJvLzU7gTuEkiORAkfOw5UFd4bK1Enra0xVXLuNASB8LYJ3g624JCjIMQOm1sJOsWM0qKEBAOs+E23MjYx42DmCbZKgO8SOh6WkN0yHCjPSLdJOkA2AnQk26BA269LTCig6x0t9g+f9m8nNtoDofvstUSSHQqeetkWvldI6ge1lMeBFlW7XA8GnU7LAV5yLNzTFL47KBK1T/mtlg5XyHyNss6ivCPFLK3vUaZMKDl40yQo7jM4np5n7rWkpHWbrvWmeYqFuCkfQ7mCQmQkyCPhasV8oS5e9yNgI+1vqkB5SC6OWx5b9bWQCmO0zdoolMqmJkD7/AK5WalAc1wKDWNzANKdPEi9rtxBj2/7+w+4t2hvCsXVNKKCk98Z1Ag6yFFfwJs28xQjNBCZmD5fnYamepmVDMlBSVDvBOpj6H32AcUA+oMCRIKDoIT019/x+NyMcNoYOSoNjytsE48OUVG/XNCrcSlskZsx0I+/lt+trD8e+GXCfCGBbnvDCV8Nv3lUhJ7FL6Vl1BQSXYHswQB5ExsbVYZrw2orQrKoEd7WRtE8/CxuovesqEIQpwkbAE+/rpP4fFnMLnA32RGkAEEIjWFJWpCI906ifAfU2JLSqZIMHY7DQ/rYy48V5SlKes9Tty+uVgXYUpRgERsRqOUfW/usZMizhXn7qo5TI08fGz+wFePbUDtA6onsHMyEgwQFdNev32YqilQOaITsAr6+jZWwreKKC+W1OrCWXQWXCZ0Ctj7iB8LAqWcRhVmlfkkF1JqkrcGYwAdIB0kfW9kLFb7X7gr1OtlUMrJCUnSNP1s6G2iV95EmdQNgQfu+/TrYK9sM3Zet016qu6aisKWVuOZKtbAUjLqkwRp4jW3PyyBjTddLHCXOFlSPFLbbPpH0IzAkXvdJVuI1YOx+7wNvZil1bBIHnO9vIDiu7d49KKjF03O3dTIvO6AKZLzrwBlrMSpxRUVQPLUC3sDSp+yA89d7AzZmNPkhEFsjh5rtSYOUn5WLPIMd2QT1NjkaRqYsA8kkSYE87Ti3Q5EmPNFIzAeJsSVnBnUjf7rKTqVEEQdPr8rFFt9NLXWbKk7dR5xjH/wAjXiEgqTUNKAJ09qNPjaCVuKAJBBUI5ESRayWPcPV+Jbgdue7snbvLQW86so7qgTry0Fo5a4E44en7S7kpOgBfUYHlHnY8MgAsUJ1PJJq0XUasOvjtIy90DulBn2t7Yc6gFrg5f8IP1+VpSY9HzGc96tuxJI/71Z2/3bGEejpi05gu97tQCRMKcM/8utj8RvVN3Sb5VEzailSVJUBE94JEabfA2sXT0eK7wxQmloKaaJdIX4FIsoIyjvF+SkOZz7EezrZpN+jbf5JLt/0KZ1gNKMfd4WsZdzS6G76KkcKlqapm2yUqIEpSATYT421Dw6+yHJSvDbSCyhG88L40D5aXTVCW5knsZkEnQGCD8edmHxSpKukVh128KJ1hz1/RLqVAkB6n6jbU2ts25n1S0sjrnItG/HjBtDiSiuy8Kt6oaVdSXX2wgiCoKbVCpnmi2xUYiZqbu5b91m0eCiCp4zXX8lCPFNKWW6RYTnjPIgkgkTHytGaqvUpSiNRO43+ulrJYz4fXZfCKcVdTUoCe8cigJ0Om3ibNNHA7CHcdcvipaKpUEu1KUkbiYyzyNs2PIxt5CB9VozRPbqoZcUlSQ4CZGuuomRA+duEVTqUBJMAazGv1r99poqeDWGkLZWzXqcZz/bldQoymI0hO+1jl18FcM3nUep0dK6+s94Ht3ECJ11IAInz1NtBlI6RnEHu9VTEjS7IHC6gw1D/eUVJJnURofDW2hWPgZHFjLrBJiB5/C1jD6NtGqR+6WkAwBmr1H3xNhmvRwu1CMpuakWZ1mtWY133tAQs2LgjcF/VVmq6gi9sLOqUJavladejlDUjUcvZte+gqr0fZS7TM0wBQgpK1K1BSDMR4/K1YOIeCLjw3f+E6amoWmSjE9M2+lKyQZbebO/8Anj32ny7ajiDUJo13bQ3KqgU0gJU++4FhvL7WVOhOg0tQkZlkIBV1sZDBdK9Q9i92qC0+opQDKSS6dJ8B52qH6WNS7cXpl8EL7co0B+sRQUrrgkpyprlZo/49Z8OtrW1h4mIW23SLw602lJWo9g+dJ10I1PhOtof464WTfeLsIYnxvdty195Xa7/2LWMsvNmnUHUKJKFHecp1nnYEjAxhJd6osERc6w1U8VDvZVL7eUQVbEAzz5+djV2sMuTVVORKAuAcolR9ws0X7g4pOFC0YvuRKzq4Dd4USdNRKhpY5S0PEhthVLW44u5aVtlttLF1IC23FbLEOQYO4Omp02tF1Q5p0YT6f5RTCLe8FQf0uaNNL6TGJ6pFC6lD6adztFJAzBVOiQJ5Ei0C8YqepN8XDUUbai0bmZc7yohSHHUTm6kCBH9PQDjbw/YViCorsRUNFet6di0V1y6BtLiwG+6CNdojfpau67ipK647tcfu6kqE9g4wv1hsFZyvOGAVCYlWn6254y3neD1Wm2jORpB3ChO+7wqLqw3WXnRFPbtMFTMapCyAAfGJnTf5Wr7UC/l1r15XvW0pUtwrdccZQtZHMlRg+HI8tNBa7XGPDLTfCu8ks0NPTKFOw8XmmwVNthaFKOgkd2R4a9Dan99XBU1VQxSuU7nq7RzEBv8AvFbgRrp0G2scxOthzW5SRuqVe0scAUxqhVBeFS67TXYopAzFZWQCBuY9x08LH7gaQ5QrQpglSASQATBBiSD4yfjZRvlygu8pYapQl52EIbgIKiecEd3UDUkHWOWiHh6sSHgl2YKlSE6kSQTruToPhz2toWsVn8k46ah9Zd7R4BDaJAR3gNNdQdJ18PjFllYCiVaZQkKG6ZBTAPUgQRoTPhsUxiscK0DtUtkgFYCoVqRBkba7CBMKEnax5ASUoZMpaIGQa5SeWWBvCTtoQANgLEFlE3Tj4bXDdN74urn78qkMMMMNraUl/IrOVqBHMnRMayII91orgxhhGjSKKmPrbBCQvMSG1I1J1JkySN94IE2pLf8AeS7vvOhCbwq6dLzayXKdIcdCTAgBZGhKZ3j2tySbG2sSVq1oScT4gdDeV5X2jTGTvR/ATpr/ADJjXXXTdw3tAcKi4bGXPVDdTCY3cVf268dUCqJ1ugYbW+whxLVOkjtFd4htvr4DfztCmOMJXjeDzmLeKOMriw+04fsmaoOuFtG+QNgCVQrwPnpEc3Vf1PiKixJdt3XTd1319Fh929LrrKWtvNb6qhp1swntap5pRyKXqUJ1GhBAmE7zosUXrXrqb9avCsL4LKH3lFX2hQVIhexkiY5ifO0n9shW3Y+Pbz0Vp+H8FoLTundxMw7w9eZvC9sMX9eVbXUykEKcok01O4galSUFSlxEwSBOmmtkfgXxE/0XcV8MY3KVFu6rzYdfWCqfVpKHwE7HM2tW4nQeMtKvw5iK52FGpYcbQYzpQrMOe5GnXn18bJLSwk6xbBqp21bs4aB9FBoLF79NOs1KE1VK8HG30hxpaVEhaSNCPAg/pZAuy7qoXleqgclO1XKV2ijlQApCHDJOg1WdzaKPQk4kq4k+jth52rqA9XYezXBWKzEmWMvZE+bC2Z6kG0t1uBncS3sqprL6qGqBBT/qzaiO+EiTJ2+HO3n/AGsp2y0dnm1iux7NzGOpJbzCGvDHWEMPhLT9W5Xvo0DNImRMfzK0jxE2j7ilxJ4pHB1ZfeE7qcw9RUICnHw3mcWhSgkd9WoEqB7oB0tMFxYNwnh8A0V1sl0a9o4nOqfM7W3i3933pctXdNdStP01S0W3GVpCkqBG0Hxt5zEKencMjL/Vde4OmNivOq+nr7xa469fN6Vt51rpzFbq1KJ18dfjPjbrDGCr2ui86e+0kNGiUFgCRCVaHnyBtL153BRXPeT1PTUiGktuFIKURsdPh0/SyVXU6hdlU0CEq9XcAEazlJ0/rbtoA3J4RosmRmV2qq56TOC/7H3tdtI26p5NZdhdbKuudyR7ifnatlXlNQsp2nTvZvna5/paU6Lxw5ga/SkrU7SusuKE8uzIBPLVRtTSubKXSIjSD5jT8Lddgz81OLrkMZYGVBsnNwgvL908UsKV+bKG73pUqIEylTgSoR5E29acA1HqSllIIKgM0bbj9fhbxvu2tdoLxpaxpZSumeQ6gjkUqBHzt694Yr23WmH6cqyuN5wJ3BAItm9pY87W+a2Oy0li8dLKWk34exCo0MJiYsBUP1q2PWgw6pkAEqIMfHazOq6q+XrtfcuHsXK5KCtpt0ZkLIkhJE8wPKfhZHvG+8cUF4uKpahqqo/9vV1BZaUhsLSp7sk54WjsfZJH+1BPsxbhe6i9gu54lgnfVXkEyQRIEExuPfYj+9MiwpB1Cs0A8/H9bRD/AGl4h1VO4jtmmiw6puoqHHqYN07YeMqcIkocCMkgiIUo5QQAUh7FPENbfb017XZ2IabU4XKilKQpWUqCCFbDK4BOsLmTGlttJYIJnCma8azK+4BlgOKA8ACev1rZj8UaYX7w+xPci0hXrt0VTKQT/GplUfMj6i2sP3lfVTRLqL3eLy1vqNO8S2S4wQkpV9kcn8REjkATFh36kFDjTpSUqBzK3O3nqLGhZw3gjkUCd/EjLeoXk5e6Yqg5/wB62hcxEmNfPWdbLnDm9E4d4h4Xvxz+7pbypKhU6d1LozfcbEsXXau6b0eu10ELon36VUjctuqFkrO4E07wJGQlIM7EGf8Aqt6hYSR26heUaxzX6FeodTVBCnElSd9YInrqOXONbXi4L3qb34W4arcwWRQJYJmZ7Ils68/Zt543HiD99Yeu2+P4a2iYqTrM50BXLnJ+Pwtd30Ub6/evChmmKyTdt41NL7iQ6P8A8Lbh6QFkxaV29a7iQtcpiW4CdvjYIFIVoPjbpYUFEczYMq33HytsNWM5Fb3bD1MrmUCRFm2oiDuZnQ2dSxmSUnYgzZsPo7J9SI1m2lSvuMqz6pvxLgZTuBE7T7rapyoUzQVqoJynxI0M/C2gSnROkfX4Wxmci0ad1Zj394ffbQYs526GS57/AB52zMVAmdOnO3BBSMusb72TcU3hX3Xhu9rzupppyspKF9+mS6CUKdQhSkpUAQYJA5ix2tubBQJtqlfVXI6HnrbCZ1B28bUmwN6aPE3FeMrhw5XUOG6SmvW8qWjfebo3iptDrqUKUnM7EjNz6a6WDxn6ZnFPDuJ72uW66W43aOlq3WqV6pul9h15lKyErUguSkkDY9DttbSGFz5suipe0IcuZXeQogga/dbvMTqCNrRd6PXE29OK3DWlxXfTdI3eHrVRS1ApkFLYUhfdgEmO4pM6nWfK0mBeuadrVJGGJxY7cKyx4kaHN2KGIBBBOn4WM4bJVTNJIMtEtxG2Ux+FiYUNhPMGbD4c7tXVsSDldDg/3kj8QbD5pynWA3HtK9yv1tlszDnr4xbLOorwsvFXaYiw4JVlcvqkBT4Z59/9Da1eH+GeNMUYavbF9z4eU9c1xNlyvrFvtNtsjLmiVqGdUa5UAq1SI1AtVQFbuMMMJkEfvmnJzbEAq2/S1hRfF4GiN2+vVKaFbgeXTJeIZWsCErKNirlJE+O9npAcpshVRFxdGG6tpKx2uciYGVUEH6ixiorbsLY7FnQiZWSZMb/H8bJBUDrmUYMHkYOlhmUodKc1QEQogSnSIGvQ/pa6QqjTyQodbOZKlDQSlMcz9ffbSVlRMwDEJ6+7x+utjNRQ3Y3SreYvVLjgUkBIGqgQTIPh+J8rECiMoBCu0GwIJ30sg66T25U9qDhfjG98BVXEq7Ll9ZuKgdW1W1DLzalNQBOdrNnSnvA5ogAnYWZjry1AlZlJOxMg8pH1ysZo6pxumVSevPsUz5HbJbXKVBJkZkgwqInXaycpyFrzIBEEnWJ059LM24vdLQrvMZKiSDMmDvHn9b2CQ4CvNlBKZkjcnkfrxti1Zs06lRGXu6H8Im2kZcojvCD9T0tI6pWI1U7YWdF63LS3oO92rSUrlMnMNFfnqbLdddtSm5quW1rIYUCk7ER9eVmTwOvBNQqtuN4pzpAqWcx1UNlgf8p+Np2XcD711vuCQosKUJgBRAPTf9LcPis/dXuYV3+Exd5ia8Lzc4xsPp9J2k9Z7y03hdRKYnL/AHR1+Xu8pt7CUgHZJHKLeS3pH0iaf0r2xCoVW3OpRUQYJDUxA1Ezp4n3+tdKMzYG+nysSF+eCNw6LOqWZKiQeaEUCRziJ05WLupBkEbWNGCIIsXdAPKPfazGqkiIPJ3PP52LLBEEifPSx14abb8osUcQCSP62uA2Cpu3WqFkPXiw2oCFKj5Wcn7qSowgRz3slYdYadvqmS8FLBKoiAZymLPpFIxoMhKyYiLU5qjgvstWitw9U3EXTMCdRrvYYXdklWcCBFnA9dyWnMpSZiSI2tyaFKjog2Ea031VwFh1Cb4u9PenIqeZBsa7JP2aQFdxMaRt7zZV9RVICU2i3HDmK6fGdBddy3BUVdHVOhFRUpLiUsiEzqCEpjU67xG9iQV0mfLG3MT9lQxBrOHmUiIQkKBUpKI11KPwmzc4lJaqLgcCVJWpLLswoGO74WjnE+FMcLSpV0IrHFKgR2io35DMTtY1hK5MS0VNe7GIbqqGW/UwGnXUnvq70jUkE7W6uaiAoDVF4Bt7vNcxRYm+SubTGIgX35Jy32xmpaZZbnOAdvCwD+FL4vegut+6KV5Y7TKtbamk5AVnvqLg7yUxqkanNZdqGE1F20ZWAqUIOomO5ZzYZShOHWApWgcUkHwm2A3JVgRuO2q36uLLHqmHiHhxiFxeeidWoBKgA27oOYOWQfvtrBWGcTXDe5Vel2OIpFMhIqCtJ7+cQDBm0qJS0AVhC1eMH8LcXqibtT2QKSl1tUbfxCd7dGzEnx03dgBlXNtwqJtQJwTdNdLyytQXUBWXUd//APOtwy8DW5jVNhtAkjONPnYk/RVJrHUU93Z0CcpzkSN9Zt0xRrbqCp27UJAjve1+lqBcStUBQJxre7XFLIU2kCkxRdbydBpmqW0b/wD1nztO+GHH03Ddz6GkkCmSglRG4/py6WhDjDTmpxZeVIgKzt1t01MRoIraUz4/xWmvC6S7cdHTrCoZcdSvWIKHFJiPMfK2XT1YlqXRX1H9lp1VPkp2v+iVKt91xoF8tMgHKftNz8LRP6QLbBurDFc2FOdjeRBcSdEkgaT4wfhaXLxS0WW0qfcSdVQk/kLRpx1b7DAFNVFC3BTXmyvMqZEyB7rPi1+6PI5IeG/6hqkCrQczb6l/3qQQYnkN9LdsKQYCnFKVoBCDH3WApq0qoKN9DpPa0zSwnswqJSNjY41WVK4IQqOoSRra1GbsBVaQWcQo64y3RmvVpxPsv0yEqCzOska/PS1asH4fFdd6qAuNU4ZvKuaUsyUpAcJkgRrCveSbXJxvdibzTSvrpVnIjLtJkEHW0A4PwA5TX7iK565ioOS9XahpaUz3FpQobGNYjXnbzzEq4U9fM08l2VDGJaeJM1zCtXelz1VCqkYdaVRuBTTmVKXUZCChQXpqNIPWLecDSnqYrWoO9jT/ANy0lKlCOUQPADx13Jt6/wBVgKnWFNrYqlkpnRAmPEzuJt5U4twjWYZ4g35gdmlqnHqC9ai722A0pTi1B0pQEpA1JkQAJMjfSdns3iUdW57QdRZZ+P0xjDHBQ/iq76h8evu9x5HflAgpGndOs6dfPobN26socCSlI27xBEe8edrFYp9F3j7U3Sa1zgziwobaLsLup1Dkc4bKc51MwBPnFq6M07rFQ7T1QdYdaWULbWjIpKhoQRyMyCI/TqQ9rj4SuaLHN94WT4u9aUNJ7sh0FYSVZpSrQAjcmDoSNvdB1Va1TsIeqVko0Q7vGpKYgQdgDzOh9yJd9W6+wGkLLpXlAQO8YgbRvOnjqNZ1srU+R2mV2ri9ICpSUqjmJIHh4HveBBDtdDG64p7sujE9Yy/U09RVpbbDQyrCCoiSZ6SVSdASZM7gvuk4dYFBaqV3cEIBSvItySUyDrpsdevSRANm1hGlpW1ApaDebMstApgJEHUGe7toeg5mLSM1RKr6dxLVQ4ytSCEpSJlcER3dCJJ1/ITUe7VW42i109KzE2JmmBclTibEVRRJRl9UevqrdpMsD/YrcKCNAfZjwEgFrXlU1tSlVMzTodfXKVlxCSkEADSdxPvAMgawVpVCxUZnF0q3FK75Qaggie9B1mfZBET3T71+5bhYu/8A199hDLmUFAbKiEpTvqdZjXcaEbmZotj8eiuF4DdVD2O8Fk0tPdD9a7U11e+X3VqRo00kZin3lSZB5DTYi0PY7wevDlSHUCW1mCMuUDoRrqCQrXwtYzEGIk/vOqbuWlF43s6kNgAgopmwdSSJynfnpGuoi0eYk4XX256xeN7XoXHX0BbrSUjKnSDB9+mm0baC16N1tFTkZfVS3+zJ4mKuLiVfvDKsf/1XFFB61SIUY/1ymlUD/Myp2T/gTb0raqjTKypJAIzDSBPP8LeJXAPFiuHHHjB2JV1Abbu2/adFQudOwU4G3dehbUu3tXXNPdokMtrXlJCoExEW57tRFnonn6futrs3IG1jQfP9koKvEx3TJJnfn1s3sQ3oQ0pKjoNQZ36/jYRp0rdNNIzA+yDKp8vzs3sZpfoshcQtPaoIlSdd/wBRbzKCIOcAvSiWsGii3FzQF5uupJBc75HIA8/hZoOvMZ3GVuIyuJI1Me77/efGzb9K67r6vjC12uXGxW1LzVaha26VtayUlpUkhPSB4CfG1YF4Oxo+sJXhm+1KCVJCjROkHXxB3/LS3dUFG2SEOc6y5DEa10MxY1l1MfH5lq8eCOHa1PfN3XkunJSSYQQsa66juga8xalt7oW3UqDhJUFKSSdzBi1wqu67wHowXndl4UjtPU3Re2cNuJUlYBcTuk7e2dAOdqj4iaCax4hMDMFeyREjYj3D4m23g/gzMHIrAxe73NkPMBI6SAqSSdot6ncIbyF48PMKV/a5lVFz0ZUSdz2KZ+dvNDCmBMWYzZvFzC9zLrxdbBqKvItCS23B70KIJ2O08rX89GK/EVvBHCri1KUpll1hRV/D2by0/wDhSkRvFmx5odCCORVjs68smcDzCn27a5lBKHKgMBSZQ9knKQDv4QT8rM++8B4Vqyum/eVTUsuOPLU0lKUtJLq21OJAKSSCltLcaAIJAG0HVXgrtFIGXdQ3BB8/nrZNfrXHElOeRMdJ3Ma7D9PC3HiOxuF2xm0subnuOhw8p5TNU+92zaWk9uUkJSHHFxIGveeWdZ0IHIyjHCNzUzjLtNU1iTTsllAzpMBTDLJJBSQZQy3vIke6x9yqQYSVbQM0gn4/CyXV4iuikKzVXhTgMhIWC4BBUYEjoZgeJsUMdyQXStCUqNTV03bS3bTKcW1SMJZQXInIJAkJAGmmwiwb1YtQJCQOsjvEzHWbM7EnEu6bkHq6E1dbUtPFhTbKJIWUBUiY0gk2Srg4iKvi/XLlqLvcpklbgacK8wUps94EEaEDX32kIHWzWQHVLb5bqnPHy60XZxNxMw2Dl/ertQDyyvpDoH/MfjaPQqaMI17jkjwzD/8ANFpw9LC6fU+IVTWhGtfQ0lWVCd0lbROvLuJ+ItBrWYtupGoACjr4x+Nu+on56djvILzyubkqXjzV3uCN7qvLhRhx8rzKbpjTqE/904tsD4JFrw+hXfHbXXiq5Fr1YqaarSJ/nQpJI/8AZpt58+i7V+vcMCwXCDQXi80BMQClC9+Q75tdj0Oa/wBU4h3rdpOlfdK1gciW3UEH3BSrcvM0RVjh5rqYncSiafIK4qyTpHhtzsEogEgHTxFu1HQBR8NrckSJO4EG18bLOcgzptsLIV8NZX88ABQmy6RH8PlpZNvhouMByYKDNrVM7K9Vp25mpECtRtvbQJQ+sH+LKr8PwtsEjWI0+VuXTleSqVd5J1+H621mFZDguys+f3e62nEIdbU0sBSHElKgeYPKwZJMkE/CwiTm0J1262O02N1A6rzUwPdbuFcb41u67aNmoxJcNHWt3K280HIqWKlKXFNoUIU6KdL6k7kESNQmyxflK/x0uG9eIH7pormvqirspdqL4cNPeCFJW6622at5QQtnVYQghORRTlBCQXPxXr8H4L9ILElJia4Kena9YReVHeVKy+qr7Z7s3ZkPJSIKnYKQNUpEzJsTxFxi4ZY07BzFar5vI0zRSy3U0IeDaiolRBW6dFdySEySCSTpbr2vc7LIByC51zGtu0lSv6Bl9B7COJsOqOYUdezVtidMrreX4S1a0yTrBJtTf0T8c3JUcXr2w9h+7qWkuuvuou05TRpYeccbLZIcykgxmcjwtcVKhHdMDlH9bYWJDLUE23WrRG8IA5IYHmBFh7nWpN+Fue69TgjzSqP+qxUHcETpuR0t3TrLd60D2aE9oppXhmSY+aRairaeBWQSA2rToB+dst0Qf5h/w2y0rKC8LaOf7X4WbSqc15tqkmYTlJk++0+BtfZgI72YyVCPO0PcNbmpsR8Y8EXDVkhmrvQ5+zVBIS2smDvy15ee1rGY1wZeGCq5FIuoRV0r5Utl1RyqIEaKE77Dx08rBpaiNruETqVOopZHs4oGgTWKVJM5NJPLUHlI+vnbpCVKJQUkgaSOY15/UW6R2qYJG41/xfOetty9yCSD3Z11Gv4eNtJZpaQu10lY0pptTYCn2wtAS4DAOxMHQ848+tt/u+rU4hkkZlQQUid9uWp187cBb0ZgogDUgcunl+PjbpT78hXaGABqTJj6/K0EuS6coWmnVBSyttGsoSSR+E/lZWoMLU79PU1rd+XZFGwip9WeccbdfSokFDfdIK08xI0IidrJQeeyZS7qDBE6R0+t5t3mcy5+0He00HXz8J++zXKkwDdF/VXVuHK0pMKn2dNbdtXe+tU9ivvK8pPhPn/Wwmd0H+8gDunXYdT8/iLGqdx4K7ripI1jSInXp9G0XPsEVrASnVwvoL6oMY3VeNNd7zqUVCUuhJAzNqGVXxBP32vrS8Onl0hCqcErTAhMR7rU14QXY/VXwy4/fLlOG3EqVlGigCDB/L6HpTSdmqmaUgCCgEfC3I4pQjFanIXWsF0lLVy4bTjh/EV40emnhR7DvpetqRTqYQ47crgVObOe6JI5bK/4bemtOn7MJ6DkOVqM/tK6mjR6UGFWmwA8KC7M8AAk+tOkeegA+HS16qdMNAxvZ4IhBEIfl09Ez5DMeI7c6rsjTQzqOdgHQrmNzYyQAnSxd3RUgz52txoEmyJOpkculiqgORmeVjjwJBg/CxZWnKR0iLWhsqTt0tYLol1F/wBMAkkJzKPlFpPZuSmYcD7aPtB7JKiQPjZgcOnUt3+G51caWn7j+FpSsPu0cri526mZXMGVpSebtKlZlLBJ5xbf7sT/ADfKx+2rIUEPRR4z+qTlXUM2YLIs279o3Ke8EoB0WJnw+hZ7WZ+Mkreq0MIcLZKU99O41sanpY4X5mpnSueLORFDD5khf/MbA3ywwq7FNVL6EBXdClGBJ0H32GZSkaheb/Ny+VkDiI8lrC7z6icrbrStAN8w62JWXdA8A8kSlIErT5o3WUNPR0bNK64gdkhKVGf8Nhbpve6Ke620KveiZCXCSFupGnvsycUYuSjDf7wIkuNsn2huUjb42g/+1rt4VaqTs1JbWVQQqduX3Wy8MjeHZyVcr5gW5Fbqgvu5bzfNPQXtRVTqBnUht1KiBMSYGmpFkS/+JXD+kZqKB/F9zN1dK6hLzKqtCVIUlwBQIMajWRvpas3Cbig7c2Lr2W/RNNs09K4yt3OZUe0bUInwGv4c6t8bcVJr+JuLqZKHMwvm8WlgBJzD1hcRrpztYkxBzah0FtAL3VdlLeISL0lVxV4fpa7Y49w6lDmYoJrW06DnqbJTnHfhC2ShXFLDSldG61C5iZjL5W8h70oO2qCUERIJ0MfDmYmJ/OythdbtE620llspdJylwDTSO6T9a2aTEntFwAnZTNcbFXdvPHN1439IfEgufECLzuP+zrVVTqaI7EOsOsLVGkk906/mLTvcuOsPXbelfhyovkN1jV93gyGyy4oQp9a0gkJyjRQ3NqEcHa5268WXqUENn+z95pMd4KHq6laEHXUD4/G+PDSuFSziq8W1gqexC66DodFsskffbJoXSOry9thcFadU1vdAHa2Wrx48cOqSodbVfF7PFtICBTXHVuSfAhuOdo19IHjRhS9uF1Vdd33fipdY7UUz6FPXHUU7YTnT7S1pCRII3PO0+BbQcDvaLDmqvaB1syvSEu5F48DcTLb7RTrbTb4UqVDMh9CtZMAaeQt0c0b5InNcdLdFjxSCOQOaiOHeIt4MYdutit4XY6VUUtCyy8TdraE50oAJlTg5iy9SY+vSpQlTPC7FEFYB7RVG3kB0ky/pZ0sKdqKSnUFEoUyhXdJ1lIOuhsKGCtktFgqSSJCl7+6LRax4AAck6RpNyE78EIF53A3W3ldDtG84pYVT1Km1rbAMQShSkmQJ0POxS57suR3Gd+/6o0XXAw97MT9mEz4+zYvS38q7WPVpykoCgmR3dI/C0f4axg7RcTK9TrxKKi72gEk6ZkqXr12Atw+L4pTRVrYZGg5TdxIGv8utakop3xPkYbAjT9/7KblXFdJkmjb11Olq1VnAnAtwek7fnFZFxMuXrX09EaVSzKadwtOIdebSRotYbSkkawDEZlTNasdojQjpvaO+KOMW6FNPialby1LaFUxezJypBIUnMD7R0WEgc1e4zjxrDpZWx0zcpPMC1/JHoqGpZJeYXH81RPF9+12IcMX7T4NBqb8oO2p2WWnG0qTUR9mkl0pROqdyE672rB6TfoH3xxa4S1fEWg4WJu/izRULVVUuUlZToF7LRBeQ622tTa3CnNlUIUVBIKiNLWTu2+mMRXawzclTVqr1OGrZKHWwipZXBWwhSicqgqVpkEb6idJZurEdcu5KVF40LgrOySHEOuo1UBqQQTPLlbooXxRXcT/OibEGyEBoHlY/uvnBpVusVZaqEdmtpeVSH0/aJgxGXeQJERzMxNluirHXstE2mFKXBQlsJgkbzy3jXkRpOzn9KG/KW9/SQ4kXrTerIaqcT168tMhSWsweMmFEyoqCieWYqIgaBpYZZcrq1LbKApLac5BB1IGgE9Qk69NdptsB92By57JZ+VSNhu72WWUCqqEBoASlC/4oJUdCRqJ6ezI5w+aaqoqWlXVKUlDQQlkpJgulQBGg31JjSN/ez7mp1LyjtF5MwS6lRSEpRsN+U6g9DoNdT6a5V44gTdbCVrYokdstG/fBImY2Gux5DqZq3VkaJ60RqEIaebIbU0hpOQAJDZA7wmNgSfAkaaRY3iC8W0Xe4q8L2NDRDWurguFBKQO63J5qOmg0g6zIbLTl7NNLF2XJVXgtxa0MttMyAoFW6v4B8EiSQoEgWaF5Grvy/vVb+CMS3q2rPS4du5al01MJ9upcSCDB/gSSesa2ZrFJzwnBQXneeImVuYFo0YYw+2vK9fVa3nqahRkANiJBIPtEaZtxtZJxkL7ujDt40GErqQrtEOF28at1NQ/Ut6lSmiNBok+UGNdbO5N14+o0NM3veV9uZEBKLvuW7mkU7ImQgrdCW4BjmeZI11HdpXFJdvC9MP0tDQKKUupafVVuyBqXEspU2mYiSAJG82TrstZJnjuCqdds72/rGcl3Nnzc80zNvoF9GrE124qwzh7El8sMPJvzDlFeJ7RIWA6tlpaonbVareCuOruo7sxTX0920L1JRF0rp23TJCD0I3EzG/nb2B9FS/3mvRu4d3o24uWrgZYzeCMzceXd+QtVxYnujnsFyBdSoI884iJtc2VxqLDfDe7b1dvunuunTVOkklSipCSd8qCcqfcBaLPShq7lXh+53LuYp0OCrcQpTbYBCS2dJHkLN1eO3yDDx32n6+hZj8TcSO3xczTTjshmpC5JkbEfXu6284irp5rQuY0A9Au3iwcwSCd0hJHUqOLweDtO2tw7oIMiDueVkJ9BSolSArLoPET+Bsoh0u0pQQqW3VxI2mCJ95P1pYg+2VLStWqdoT49PrrbYiuBZHlN9UwMbUSarBPEW6gkSKduuAg6kIzT/wAlqE4jbUlxRJV3kpWJA6662v5i2+7iuWtv27r/AK/1YX7h91llJbWtTq05kFICQdgofRi1Br/RmJABzBGp111Hu6/K3QYOTc3XMY1Y5bKTvRKoLwvviJVYZu+vRRuXnQFGdwdxUOtkpV4QT9+1p79EaocHDy8rkWYVdN+1NLBABywhX3k6edqbYDxcrBOIBfIo/WkFhxhbXaZCQsRIVBggwoHqLWk9DDEH71XjlKW3GkuV7NehrNmy9r2siefsJExrFrmKxXgc76Krg8uWoa36qyt6VxobvfvAI7QMNqcKQTJhOYjnp+u/OKXOJd9OtuuKu5NI8232hBM7OJnfcFtYM7zNpWvSnNTdLjaQIfZUiPAgjTwPnZm4IuHB9+UqU30lvt3rpqiFB7sz2zdOS2TtOrZGXmRzNuQLmxNL3C9l0WJVZpI+JyCiitrcUX1dr1det+VgYYWioyJGQBC3uzUiQP4QJ/rZPvynXSVF6pbecW6aippnVrWZUlCEutz0Pd6fKz4xoxddC9fdz4feD121NO460M8hDiUhaoVzGk/U2Z98OGpvBxQImsdoX0idFdswW1QB8NuVr8JD2gqlHPxWB/VEm8Sl28HL+qAVBupoqpzTUgNlCwI0mT/WzjuasZXjynDTSm1OVq6pHad0hLrCTlMayT9GbNld1N3bc9JebLi1mtpFVOUgHIptbRKfHUKg2cD14C/LyexDc9PVlCU01QkFogqUhZQoaTOhB8uXKzyDTRGYTfVMT0xKMKvK461tvL6xRVVOpQBglpxLg/8AGr42rCx7SkxMoVa3vpZU9PV4auKtCwt1mvWyRm72R1pU6c9UJ18LVEpShNUguju5oPltbpMJdelaFzmMNy1RKsn6IV4lV34nukqJKHaaobRJ/iC0qP8AyptdT0ar2N28ZriClZUVSX6RwEiTmZVlHxSn428/vRUvVdDji8rvERW3avQjdSFoI18s1rfcL8UqpMZYWxEmnfpUNXpS1DjT7fZuNo7ZIUlQkxpM6++LY2KMLKvN9FtYY8Pow36r0tMjTTfabcBY2j3W7WkSRYMyZ10PysduyrOC0TCtJ1PhYCob7VpaOosKfvMGbcq10jcdbGjNihOF01lpyLj4zYs+uFsqJH95G/UH9LKN4tdnUGdcxneIsmV8ppysT3CFbHSDP4W2YzcArHlGUoYKiJM687Zmk/kbB5wdj4bW2FgiQDvqbHCCmhizg5wwxzexvzFuDaG8rwS2lnt3s2YoTsNCBpP1pZPb9Hrgg0YHDO4p379Pm+8mz/Jnmeu3O3WYEyCD0FjCaQCwcbKBjYTchNfDvCvhvhK8G73w1gi57srWkqSh+npkocSkiFAEa6jSzuziQTuTYFMyQDttboK5GY/CycXP1cUgwN2Q2aFQRqTuPrW3T6+xpjUwf9XUh4HwSoE/IGwSTr15AED8rDsNCqbdpiow4hSYHORFlyST2EKAISNddxbLItDfVKaKnLriCvskZiTBmBMi2WV1FeP3o+U5rfSMwIghRSmorHEk6exSPHTx0taf0gm+zVdcqUkqW7qkxO3S1VvR0ve67h9InCV7X0+lmnaF4ZlDUgKpVpCo6SqSbWo9Ie9aSrq7laoFesUzrLzwqmyC1qQIkHQwJ8iPK2Ixru/sNtLLZDmjD3i+t/8ACh9IX7KBBBkQN56/CbYEjMQolREA6fX38xbk+0CNUwTrt+mltLcWlTaEtjKYKl5hpA2321j426XMubcCdl2kFRJJUQCImYEa/f8AW8uzAnDDG/EapepMHXI5Xu0wSp1KXENlCTOpzqE7HbrZqNZipJJM5ZJI5gHTl0+tbTd6NvGC5eFWIKt+/qGofo6+mU248wpOdtQ9nuKgKBk89PGbVauaSOPNELn+XVeXOwXaE38V+jxxUwTci8QYkwuqiu9jV1xb7RjXbKlZOp+4b2jpTSm5JM6wdYMGfr+lrVcffSdwhxGwa5hu4aGuQ8taVhbyk5dCJkAnqfhaqzzrmVAbLamypWckwTtET7+fusGlqHygl4t0UoHOkLtNOS5SFkglSeUEjc/h9eNjdIUthJMAzIEjX8T52IHvJJkkRp9c9PvNt9u6xXU7Xao7BSV9oT7QgCPASJ38LWHG6vRgjdSPh2/F3U2FNqGZSTEbyIA+voej2E8Ts1OGrrqFrkuUbKyZ6oBt5QNYlpkP9mVkIKAoKC9zMbe75i19MKX5e6LhuarReTIu5FzNIcpewlxb2VGVwOZtAAFDLGs725DH5pKLLLEbHVdFh1MMQaYzyVF/2it9euemRdqUOEIapLmRy3DqlDfzOvU+63o9SkltMjcW8o/TfvFd4elqw+pKVDPdCM6lRslBj5/G3q5Sf3SDMyLFpZDNTxyO3IuhSx8GV0XQ2Q2w06WLuA6zNjPKEjaxdyZ9m12NAk2RR3TcTrr42Jr31A8NfxsddAiSqDYksiTvta23ZU3JawY/2GJKI5oCllOvOUkWl31jxtC2HXezvyi1g9smdfr6NpUD6dy58LU6ioMD7DmiRx8QXSqalI0zC2jUp/mFod4nUnGKuvZj/R/f130VDl+17UALnnMpM+H3WZ7GEvSVfWDU8S7uQJByoTBIkaSEdJ+Nsp2MyNcQGo/dNLqyJqQP4vnYnXO08turbbUpSwmSkEx52gqgwDx4NWp68OKbC2IP2Scw321y8rJ94cMOLTRbdreMlQ6jMEdmAsDMrTNoRsT9bgT8Ync33LJ20tzaysGp+kCNGWx/ui1esb3zUVN43pQu1jpaTUOBLRUcoAUY02sewvw8xtc1+sXve3E6trmGgUqpghWVWo17yjvEGQdzpZj40XWVN9XyKOhqHFduuDkVBM7gjlE2nSV76guEumnVWG0xis63NJGM8d3XdmGGbuvEuNBLTQUsxlToNTroNN7RZX3t6k8aynQVBIKgoRCkkcldI8Om9lnHmEl4muenpai6Hq1qqaTTVSFtqIXPdIOm28zZiX/g/GFEf3BdKaBmmQ203ShVSWWqdpDYRkInMDmIAM6xynS1R4jG0lhOqsS4aZrOCRWeIzVff980t1OJhmkfFWAUqOdCM0KAJymNYOugtE/F+voU8XsTUorKf1movSufQyVjP2ZeUVKjoPrY2dTmBcTNC9ayguBhip7Baq9+nvN1IcX2fe0CeaREK8OmpXHfCjGlTebNY5hJNTXN1Sah+obr6p5VQg6qTnU0BCiUkmd0iwH1Eb5y4m5VxtCWRBpNgowva96O7KZuoqKpLLfaNoJGuZSjAAjXf6HJfuPDF8OtuXulS0UqT2pKlFZypBKsqUyTok/E6WdNBwlxzfDjd3XZwifqKhDgdW6ahXsJBJnMkBIBiNOnnaTcK8D7+wjR3fjC8JpHqNDjjt2B5cBOZDYDpMZ9V5oAH5wmnDv049XdFXdTxxt4j3CybnDbD1et2nxGwAq77xuuqLKnEFDvZu0zgBKTqJkCCJtdXgOt1zB941TicwqaqmqTlOvfpGPDwtV66Kq9K69mVKcR6utS0qASJgpMDcmPrrazPoy1J/sS4AqAaK7HU5o1JpUjSP8AL+vQ2EFxnBeLbqpVEOgIBUpd1CR2jZEiRLh69ALN3jBUso4K4yLKkhtm6qh5ztZAyhMkydtAfhZx0bb9W8lIp3d9ZO3ysVxzcYvrhrii7aqnU2mruypSFJElEIMESNwdY2t1jj4T9FjN94XRPDb7l5YWuKspCpSaq66V3OlWhlsHcedlH93VrkypWvUkwfhZD4Xv0qcBYbz3klambqp21lSgCopQEydRExZ4IvO7SNK+nHPV5P3Taq2aMAXcEZ0br6BNLiJet8XPdrCsO3WzeN6mnSG6JyqDAcQlxCVrzkGMqVFW28DnaKa/EqKbF7TgPeQ0UKUDGmkDz1+dpVx6blvBNBfNLU0tY/RF+mbcacS4UBYSVpBB0koTPlaA8ZLCL0F4NABam1d5StQruAQBJjQyfC3lmPRsdizxvf8AwF22G3FCw9P8p3X5xOFz3O9ejjNQ+Gl99NOM69xsPeNN9LMy8uM9fetK/QU2EMRr7oIdFLKAdwcwmCIB948rIVRej5lSX8iErSciUkJEjaRrEjSIn4WI196vrYRDiUpUowhBgTpqdNSQBqZPwtnQ4a9sgkYbWVqWWQ+6bBP/AIVt4Hp8dsv3piJdy1NSs1BQ6RTMFxuFSFPDupWNwkpMjSRNrQU173RfDLFXdN8UlbTLK0h2lcDqCUyCMyZToQQddDahSHM9Q0673ZUMxUdIOwMnvbxETrazvo15XeG1OQSsUlbVNKTJAGYpXtsdFbW7ihmMjTG8LGr25iJb+S8P+MlSLz4vY2vFht4JqcR3m82lxICgFVLh7wGx1E9OtjfD9hosu16mo74SnUzGkQYjkd4k9NbC8W8L3oji7jOiapHVJaxHebUlIBEVbggHbX8bLWCqIXVRIS+UtFtQUCW90iJ170qjp0IjQgdTmGQALmADxCU5nKtu5qA1i1oKhK0hW4TuDrHXz1HhZLuCuF33YuuqWWy/WLcWolBQGxMgaRpAMT1POLI1/Xk9eTzVI04QiW1QYIMqiZ3/AJZnXqOvOIKpmkw+HJcSW2cspaAQTExmT4wec5pOthjdFJsjVxXXiHGFGq9FV1TS0Lsslqgbc7SobKl7qBykScvQDcjUWfOGsM44p2zdFwleGKN9QU52PedcIhOqUEob0P8AEpapPeVBtF3CK7qq91paN7Xsw00FJCKGrU2reYhOsakmNYCjytNN1YNuWqWad++Knt2SA4t++lFYVr1WFEzpGXqOoFgNQb3F067l4dXPSpZrL8qK283kAK7WtqFOpTGoGXNlT10EactBY7WjD1K+t4XK1U1a++z2DailsFOhzT4H/hjlYK7cBNNJC6LF1/IS6hRJavNahJJmBJEQUz4gzsAVZ24q+haQzTX/AFqnXXgS442hSkplXdlKR4GdfCdZUjfDspROs7UqsvpKYcxLT1d04kvRRXRvoXTN6x2apKwIJnUGOnct6N+jJTu0Pop4IQolJFwh+SJgFxbg+R+7laPuCeE8HYprsQ8PMZXK3iG7r/u9Dj6bxlwywsmUKJzNkdoohSSCCmZtZJjD90Ydwi3hfD9Cmhu27qD1KkpkEkNNJbISkE6mBzMzuddbUZf1IDGehVuL9OobIOoUcV5FdVUNa5WVDaqF1biUsvqS28S2pGV1I0WkSSEnZQB6WJ3zVLdplo2SCCNJjXx99gy8pRgDJOwUfl4bn5eFilYttynWrPIKQqCdQN/cfC3BMjs5eiufmam5cN3U92NV1LRh1XrFWurczuqclbk5ozGUiRokQByG9t3rdlFeKGU3gylxLL6KlsKUTldQQUKBG8HXXSw7S/tnQ3ErbSYAjYwJ5zqfrYvWr7QJypSYMjMdI3+vO15tyVTdYCyhL0kqd4XnhapabcUpw1LJCEZiYSDA66n4pFqkYnpFNVFQFtLQ4HlpWlehmOc68t/O3oNWZmMU4XrArKyiscZWSsZYWggEmY3199qZekLdyrp4l4qomwMiL0deBREK7TvSDrp350+Ok22cMltJw1zmLQWbxfNQwQQTr4Wsj6EN4djje/7tj+/utL2WNy26kR5982reqQpQGmpEWmn0RLyNDxmoqQEj94UVTTwEzJydoB/93barm5qZ48ljYe7JVMPmr7gzTJlQUFSI36Tr01tAV53NetNeFdRqq6amVSlLdGpx5I7YKccURG40Vt4Wn1umfUyUIacgHUpQTP1HX8rQ5ijBtU5eOJquku6tXeDDjVTTZW1ysBKHCECIVrmGnXe3HRWJ1XaVbBI0JpNV1x0d7Fb9/qqUIu9a1stNnvqLMKIJ5RqNvvsQbvzDz1Zh5dVdNcs0zCEJfdUlJCUuZUqWmNQDHhrrrbt7B1+rpKWMPXgHHqKkUVCkc0UUuNuJJywNCkkfG3N8XXiG8rquxScNXkzUN0gu97tadSQlaFtrzmYgGFAWttDQswNLdEPR33fd5oFNdd30TNNS3a/UBPZ51NkhaSEzP8Y6bGPCxOuvi8r6eZoWK55lirco1oDaoSnM2oECDp309eZspXVcWIbrxAhtCaVqmdeqKV9x99tKewU6F5gCoanOY+ob13XDfFA8k1C6EhpTCm0qr2IGVwnKZXvClHymzjLdTOYJv8QbscvDCd9XpeTT6bwu9ugrGw4TJBUlKiR1gn5+6ulYkMVzyUH2HVZSPA2tnxRauW9275ra+8qWlbFzvtt/620S4+kKyCAomDIjx6a2qhehBrFL0OcJVIMzoJ+c23cKdeMhYOKttICn7wJrU3fxbuwAqDb6n6cwRspteXz1g+61wUOuN/aBWVxEKkHUKGs7bjTx362ozg6vF044uC880JZrKVxRB5BSZ/G14iF5oiJ9lRI7wnr9fK1LGG2ka7yV/BX3ic3zXqDcF6ovq4LsvlBBRX0bFUkg7hxsK/Gxs7TGvKzD4DXsm+ODWEqsKBLd2opSQZ1ZJZ/93Z9FZid9dLQjN2hPJ7xWirkB+ttEpP8AEPIna2iZ1OnMGLYFCd9POxmoZSXe7RypcQNt/KyHUI7ZlTYGqgRt1s669pCqRUAT1FmusBILZMC2nTO8Nlm1Tdboow52tO2sgkrQJA1A0sMFAHVRsUpcyWygiezcWmOneMfhYcrAM5pOutrrdlSQoMjOTynW3Sl9IB6czYHNpJk/hbclIAkmBB+FnCS7B05TMwPhbpKh1FggqNQQB523PdM6DxM2I1MUYC5ESNNxY5d6z2gOokjlZNCifOx275L0pAI8TytJRRGrumtNU8WsmTtFZdtp0tlnWA0RJCjPOBbLNZJeKXDlSneKVzZs8JYrTqoCT2UT56/P32nK8alxRSlagQ37ObWNNuZ+vC0DcNcx4l3a+3KUilrIJGvsAQefX4fGbKxxa1ZlQM2kCdz49JJNo0oFlCdxCUbocbW/2D6Z7p15HedRvZeRTXWACppaxMQTJH1rZsXOHF1YQ0palAH2Rsn4672cKaWqKc2Rck90DWDp9fCzzuIcpU4BajDbF0iUpZQVQCN9Dv05eFszXXJUmmSBm012259NB8rFvVHSCANANSUyIHS2IoahSlZkEGTrEz89duVg5r7oxaOiMofoE94sCEkyZB26/UWDF5Ui2VBqnbQAZCQkRIjadOY91nPw9qcEXNelS9xBwnVX7d66VaGWKarVTqQ+fZVKSJGhB8510FmTTXah12pqKBisQhyoUpKHnw4ltkgZEDuzI70kmCI0EahbN+oW2+6TmWFwuqq9EtIU4G0IA0OgTpt+P6dYy4p4gdpzS9nWqpkPFTyHmnRmWpP8I0IiCNbSBfNCG7vc7ZKXEqRGVUqkbTHgT87RDjanaq7qVQ366oC7XnTTrGi0oDZKZ3OsHTnHhawZARYKGU2vZIlPi+svFppP+sLbbyOLcRGfss0cup8tjyNvTTB9/qewJdh76YuqmkH2ky0nffWD87eSeDayrXiO76Tt2mmmkn2gopeCQVJmJ0M/Lnz9OcJ1dS7hC7HA9lDlCx2qAmQfsk6AgCNfwty3aOzomA9Sun7NN/Uf9lSr0s60VHpMtOFXdVUXUIIn+FAPloPn5W9faNQDSAYPd59beO/pSOJHpCtuBSVRU3aIIlMZUa+MGLewbDhS0kJj2Qd+djU1m00Y8lUqjeqk+qPKUACSIsVUoKnvflbl2oUUkcuosVU64hJSlX6fUWssdqqj23QrokRPkBYi4VHz3t07WLSIUJI8N7AdoHBIBHWetrrDdUpBYo1dbmS9KRcah5G3mLSGKg8zaNKNeSupiFbPJOnnZUxNibGF2XimluDBjl5sqQFesB9KU5uYiZHvtlYi1zpGhvRaNDlEZJ6p31NR34nlbhl+VgZjaL7yxjxgCwKbhk2QU7qrET8M02BoMWcaXatpL/D1inaPtrcXlyn3qja2MaSYuzC3qtATRhtv7KW1vKSgjMdrJd5N1lRRGoaaUtDLralFMaALHKyRdF642fPY33h2iaaXr27dUJQOgSCcx+Fur8eWLvKEOEfbM6BW/wBom1acljsrlZgYXatR1L/aNgg9fvscuOhadSkqCSV5j46k2ZmIr3vq4LqevS7bpfvt5TjbbdE0Uo7KTClExmO45H3CbR1ScZuOd3VlWbo4G195sFYFPnrm2UpjQgZgCAT1J/C2fVtdkDgRa/UBGlBylrd1M9JhiidoqlKkJOSvqIPT7ZR/G1R6UJxyi+LwvwtJdTeFRQLQ0S2laW1pUhQ0VBnnaSLw4t+kPTUNS9h7hNSvu1L63KhirvFseqOk95sd9OYDQyOtmpwSwTjaiu2+Knifhpq7a9293Kmnp2KhDiFNuJSVElCl/wAQiCfjY+AU0T3yx3HEdbLqDte/4WHiz62FjXsJyjf+ybV716cM3XV0VPhxTlDWp7OoccUteQZcpgpSOXnZXwlg5ji5iW7rxxfdXq13hjsU+rvOtrWy2hRCozkgk8yNYtN9Lhi4b/q6S7r4uplyncWESUKGWTyOX6m0l3LgbC911blZd90MNONMt06kjNqEgK16+2belwQUdHTENjHFIPiXGvfiFVKOJMTGOSjvAvDC7LrpK5nBd1tXeisoEJbfqAtQP2hUoKUVHMYTHvsncVrsu5d5XDTVWHmKn1uqWXA62SFpU2yYGWZjoAdhadaFxhhqqSgobQ1XKQEto0CTlgADzsiXqhjtcN1ZS2VLUCFFrMrZA8+VqmGwtpJONKMzzzVqoa+WPhtcQFRu8KamuvFQu+gZaZY7dCuzSmISoAyOca7+NpF4EMUVVcVMa+9Kllmjw/RukMVq2QopcW1Kwg6gBOlov47XXfFNx4qnbuvtFIyxVsDsywnuNKS2VISqJAJBkeI6WlH0eGkm5qpxx1BULoeZzhOwavB1IHun5m1OZpFTdwtcrepzelsw3NlLdAnCa3pXeFWWzquK6pMjp7VsqsJ4Jvmgfu+gfrnELS6G2lV1VmUSgka9pBGmx0+JsiuM1AZTVrvdwoUY+ybI98AiBZy4R9TecDZvd5a+8hbTiT35SQIO552stc29i5voqnDqr3ymyZ/B7DGHqrBV33kq67uVVPNFFQ4872alFDigAdCDGo8Pufirmu6jpgtu7bndCEglAW2VHrr2ZJ5+O1mR6PwuHA3C6pbrqv1G7rqrK5bz9WvRH26lElR5SvTziyhx04iJuLAVNRYRq01V84yWm7rmNMrMVh2Ap5JHIBQg9VJsMRUkcHFkaCbeV1pRR1NVMI2E2JtfkEqXnc12oubtqO66WjWmpTHZJyyhQVI9lPOOXS0EcSmgy9TOSnIEqT2Z3Ptk68hoOn5Ta3hSswDwiu3Dtbez94Vl3UjIqap90uKcd7QZ0pnZAKiEjkkC0E8TH1OUaXjU5yXwUIygIGm3iZUfjbhu0UAjxKNzW5QWjRdLhjv+zezNms46pq1Abo6ZxReTAAcgrByJ13PT39LM+pxzhtKJReSld/8AkV3oHKQJ2NnQZU1Udq23k7spSJ128REffZvvXbdjIIRdtICSYKW0DWZ1kaj68LEpwAPEpSlxHhKkP0Z8aYGvPiarDeIjdtRQ31d7rSG7yZQWVvIKVoA7TQEgLgbmfK1tb3r8CcOblcdTW3Hhu6kK7VRSqnpGCoiMxUcqZgASSNhrbzsxM3R/uCuUzTtpCWs6YQBoFAx87QHiFls11KU0zchG4CR/EfEa6D4626XD8r4i0D7rn63M2UOJRHiS7RXzxExVe9LUest1d919Sl4rDhdS5ULWFlR3BzAyJ331tH2IA5R1ZpojOgGFwAQrMIJ1/lTHlZ+VFGpL3rDaCVAAajSRHyMRPgJsxMWKDeJFBKsy+waGgOgCBpGpI3119nkINtVrcoWc83KQ3hmrUhJAynRKZSDqCZBiOo6wOYAEmcEsM3Vj7jThHCl73QKyjcql1VbS1DYW06wyw46ttwCNFqSBtOuh1i0bOrX2wSXCpuMqe8SARodz3tQABuJMCZixPoG3Eu9uK1/YpcSSxclxinQQggJeqXkhJKjucjLo1AjYxFpDqhkqNOIfB9PAvi1WYXfp3H7hrgKy6aioTnTUUilSUHkpxCvs1bDuhRhKxMnYduvDdattHqt2LdVlQGQhHd8Ak7QD117u0Ws16RXCFzi5w+Xdt104cxFdLiq65CBlUt2BnpwojZ1ICde7nS2SSE2qbgLEVzVN20iL/KaepQEsIqHWygKAMJzEpEK12keybGifcoZGllI1BcFxUziBTXM0C0kEFSYSJA56k6jU7fCy5UNrQ0W6d0hIRIZUSYUNU6HTdI8N4sWo23Ujtm6gLQBIUkhUDKDA3nn120O9jVTUFhptyoqEBBIPeBOYyIAE9AOfwItYIuLJgbFGuHV/jD/EK5q5Cc3a1CaRSQJlD0NkSOYnN45d4tbtaJQUFO+ijprrHvO3hHzpVdwqKK9LvqlNKaFPVtrU84AhKQlwDSTzhI/C10pSrK4FTm1HMRH1tbNtqQrpOxUHvH1dbrZckpKkEHwO23hP42LuqzIUqICh7WojWx+/Gyi+bxaSAmKpzKZ27x26/lZJzAwcxCBokAkgc9umv4W4tzcryF38bs0Yd5JN7TI4QnMpGQjKdY035eO/3WAcCVNpIIABy5QfP6mwzhJeCciSO8IVqeev18drArCt15RrIA2+o8t7GZsgP3TZxhRIXc7LroMNVjS8wAB9qJ84J+t6n+krdzlHxCvRJVm7SnpnEZtCQltICtNDOVQ5bWuDitvPhyuAcEpbzgHXVJB1+Hjas/pYUA/tPQVoZJ9fuhKSQNMyVLI194+XS16gdaoBWVijc0BKrA8haXFhQykEgjpZ0cKn65jiJh/93VRp6l6uap23c5Rl7Q5NxqBCrNmqSUOEqEBRnT68LHsMXiu58R3Ve6NDR1rD6f8AcWFfhbqZBmjI8lycRyyA+avkcB8Q6snt8UU5CY0NQ4SSY3gfUWZuL8PX9c9Q5R3liZCFt0prEkqUO2E+wk81d0j3/GxNO+XEqaKWgEIUUrTOZwGDKuUg6CI5WinjI1R09XdlWu6KetUAsuFaiAlpMZo1EnvE8+fW3ER3z2XeSxNEeYKJ7nuK8r8vdm6Hr3dZcdbVJgqAytpVMTrmC/dHjYhTXPUP3iu7S7XOPs1RpnsrZKYlYSoKE6d0ctJHSzyp72vpgNXwtFDTtVDz9Aw602C4lSW1JTrvoUiNdbI9cnENQzVXjUX0+Qq7kV6G0EpEFYChAI1GuvOfO15oWcWhJDWFMTVdGPVqGrLy+zhLikpGUpJVuNO8I6ajxs4qXhHeDqUqqahxjvupV9olRDYSOzMZdTPtD87A1dEq4MS0dQ1XVDgbbo3UrWuc7anChcj3p15R7zNYbaW2IJgpmeu33a/GwZXFmyNDC2S91Cj/AAHZvWgTS3teS0rWpZcSyCAQYygTz05zyEaWqdfVO7T1CW3QQpsFozO6FFJ38vlb0cW20qcrpnmSnbnprpr91qFcX7pTc+Ob+ogILV61KgIjuLVnRp5K+uerg0znFzCsnGqdsbGuamcVlAp3UqBKR8CFE/ja91y3gm9LloL2GiKykaqCJ5KQlX4/1tQ/NNMkfyrPPqB+Vrn8IKxV5cNMP1K8qoouwOo/2alNj5JHxsfGGjhtd5oGCu8bmeS9DfRAvb94cIvUSpM3XelTTAA7JVld+9xXwtNizm1J6aC1YvQkvQKu/FdylWjb9LVgbRnStKjr/kTazZOoAn3b2oQG7Ar1QMrytEzpl+Vucxnn4k2w842+RtpUA6g7xa0Aqy2tRUjIdNLNytbLTziQefvNnCCOfLl0skXw2EL7QAz4Wt07rFVagXCbyDkqahE/xBfuIj/pNhe9src62De7l4FRgZ2jpt7J/wDzrCwQTvpOvW2iHLOy3WwVEHlB87YSCdNuttgADMIjz2tzIgGBtsTad1Aiy2CeRjwt1JKpBO+45W51gb9NLbEAkDlpvNpApIT+HnET9fXjY9duj0iRBkEediAgDNqNNTY5dqwXhJ36WnmTWS7mV/Mfj+tst3I5pPwtlnumsF4lcM3GnOI1AEADLSvwQNRKIIPXb63tM1aoq3A0VEjbNy/HS0BYbvNygxbTViS6FBtQBb3BJM7bDTUe60x/vt112nKlJyrTkUlZglXKD8eth0rxsh1DDZOfDlQxT1K3luQ2hKlFJ1JJgQP6bWcn9pLuTKU1QIkQAkjl9R5WZLVSHEKKHZSlI7wMneeX18LJTeMcNVN7G4W79o11qF5Q12kd4bpBmJB5WsSRNebkoUUjoxYKSP7RUQ0TUKhMykJJnpy+pFsOJbuQmEuvTvAT9x56j7rNBSiVZwuQDIG8Drpy1+psVfuulcS46C/nkkZHVAnw3sMwNGimKh19Urf6TLsDjyGk1LnZqUpUDKpIHICZMnN8+thrqxfdbLKnXlLQH5W2Csax3dNfC0SVdO/dle8s9tSvqqELJUSpKUKVBJB3EqEgHn8ClbVBq8Cp99aA3TqWWUA5RrmciNSIEg6e0Olq/CaDqrHFcVI9+cSsP1lSm53rurCqsR2aXM6YCjtPT2YHuHMWhXFN+Xo/iSouxlgU7iGFtLJIlYAGWY9rSfOT42K1d7NPYhReCnUClbzLKe1OZOwAk6pMgR4WPY0Zum/slZdVak1rTRcqsylZy2ACAnrHlyEkRYZYBqFIPJ0KalKw/QVNPedA0X00rAWVIQYCh3TMbDfUyBMyOfpdgqto3ME3MglfaKuumME6BRaSTz21+63nx6P9TUNcW7ho3nHTS1NS5TOtrkocQtKklJTEFJEA78t+d9cHMCmw1dTCNENUjKE/5UoEcvK3KdoJvdjXYdmodHSfZU59KJeXj8l/vKCa27svP/Zo938Q93vI9h6Mj1ZvqEj7rePHpPKqf9OzqUIzJS5dyiJ0ACEanl0+WtvYOhUPVWyI1QmBPha7F/po/osqoFqqT6owqCNI5TYBzQSdLDKJywJIsAvYySDy0sZirP3RV6NdhpYNLiCkSoA+zbp8xuY91iClALmSddLXojZUZBdDVThQyXBopIkR5WkGlrqenp6aodcCkuInmSqEyYtGN6OuGieS1mJKFZQnUzl5Dn5WUrqvCleoqO7b7rn6KspgShtxXZ9qlQIB130V7iLVa7KSLq9h4JBCetXilpK2k0jK051tpKS3lOVcwofCxV/FS+yWqldOcQpBUBCkhQB5/fZLNy3Q8ptZvioKkJaSgodEjs5g6DnOtuFque68110lPX1y3+8GUyoIBMzOkCeptS4cLtbG6vWc3pZLS67tLyqKdtKSkNJW5GgCyTG3MiyXfAJu1eckQttQI1iFgiyw1TU1KghDYT2hK3CTJKj1Nku/XqRu6aoh5uQmYKwCSCNBbnKt5mkuwaBalPZgsUE046WF5idHCNbaoHHHkqbaOy1b+dq6ekTjnGuEsROUVwYnq6KmdpmnOzZUEhMyM207gfU2jfhBxVx5U49aui9Ma3pUtLu+ocUlT6sodbQVBR2nbn5WjNgUtfC14cAE02IRwEtI1VnHa5aai+kZjCa9fLaW0H8bHw6Vsu6eyQR8bVh4c4mxFet63O/XX268X62pRUGqrFJ7Q5SBofbIED4WmzGWP2MAppPWLmrL3F6LUy2i7yFKS4mIB/zToBqSDarhOGvwzHImucDcHXbkoYlUNq8LeWN2t+4Up4VuBy96lhLzeRtSlKLhQdcsEhO31FpPpafsHKxEDVaDMa6oQPwtB/CXi5UXziClwreWCr0uSpZafqmU1jgzLbKADAA01AtJV74urrvvRh6iuVb7DzR7ZfaQltSVN5QdDuCr/hNvR3V0LpMmYE/j1XDilkYzOWlLlI6n1uqoUkh5yoDwCUCco7MHfnvZJdK313Ow2taV9g6EZkwEqCiAfA6WaFXxBrk3tX3bfLN33feqb1Fz0zqqpa2lLfp+3pyVISImEJMDcmDZKxNj/GN38PcI1tNclCbzevZFLUuJCl07QK1pJEkK9tI3smVcLn5A7UJjBIG5rKBOPl01DfFaqfBX2vZ06lkIkAgQSSdNYs4PRtdSfXaRxorKWb5bhaZHdvAqAjn7f9LJHFy9q+9sW3m5ejjIrWH105DTZSlQSs8iTr7/ANUbhhxBc4fCuvVq6m62L2vOlUyp3IPtC05MgHr03NsuWq48pudAVt08PBiFt1Z2vo6ZVO0aamoygCQU06QT/hM2N4ZaYbvNpZYaSudFBpIAPmNffaD1elqmlZWUYApiWXJ71URGXxyjfX4WLNembWF5akYBuxsurCmwqrUVDXfQa/Kyjpmh4kz7eSM6pcWFmXdSNw1obnvvCt/YexBdLV40D173jTPsKSFpWkOoIMcuoO4InQxZL4f8ArpwVxBRiqnvq8Lxuq7qdxq4rorSV/u1bipcKVkmRE5dAe8SSTraO8I8Yb0uKhbeorroS3f2IKhxxDgWexDic3dII00G/nZ0V3HrFN1ur9Ru67dSUgFtShrof4rAOO4bGQyYEub5KzHS4g1juA6zXDVTJj+rTWYPvNtLTKw2xmUW3kqUPtG9RHTXTwtV3G7VOKGDq32yMhUkFOoCfjP1vZ6o40Ymv27Ki7ayhutLVUytl0tMZVpCo2M7iB77R3j+oUigW8SsuEIKSkQE9/Ukxofuje2LjNdDilSyWC+gtqtHDaWWjp3Mk6psNqSSs5gc7KTqTqdDttH5/ErUJEE6CTII57zy3tujcZbqHRmbh5kqSQqTy5H38traeCQe8ZSUn2j4fp7rTjZZFDrjVIl9tKfuWuQoEDsHASE6zB6RHL4jxtAN7sk1lKpSYys65e8SokztqPx08IsVWILjDjOY6oIgqI7xEDz/AFtXu+kLTWMJUVo7iifDU7CR0jWN/cN3DOYWLiVrghFF0oUkNpSFBwkJEAzO2/h8hvrrGGOlrRi11lBStammSsAaEwdgdBvpHOPG0uZUrpgrPokkgJRG52mNdvkZtEeMW3TjKqWQ4AlDSQQDlCUpA5aJ2I35DlvsHTdZCbF6OJbZP2hZ7pAA1JG5AkxzOg8N51uv+z1uYMYAxbfq2Qh68L3p2ElUSW2WVZST/meVptNqZ1rZbU2pohpKt0hJJIG5VlkgQBp7zta/foe0hu3hLd7CgEKqGXH3FzqpZq3N9NwhTY66x0tAu0skWG11YBhYQ82SCiXEmDqNDr4zytVP0m+GrPDnisMT3ZSsvXPj0uVZp1mEtV6dahAIB9sqDyYTut0RATFplvB0LUTMmY3iB7Xuny36WaHpe4VdxNwWcvmmRnfwxVs3qlJQTDAOR/XfRCswjmkWJEOqC7dVZuE07bk3De9RdFSVpK6Z9P2RJg+yTkM/4eR5aS70VN5U7S3qxhLiwgFdTRZS6jaTlV5qO6p00A0sz7rW3e1GyatpLanUZgsIzNqVoIJ5HefETrsFemdvhl9NMilp3A0rKAvPlCZEEb6RBHPWPK6L2TaIKofoK+p9bpnK6pLRCg6XC4G1CAUuNASkTuop5yJMWurhu/rvxLcNFfV2OqVTPsjeApKhoUqHUEHTbmJFqlm82GElLFBSmqpEBZiRl05KOux5eNpn9HS+xel1X62spZeZrEKXThITlC0e2RuMxBH+5ram9mQ36q0w5m/Raxi0lnE94IU4QVrCogfxJBnw36c7IOYI0SsnUCCJ8tfrcdDZ0Y/T2eJVrGYdsy25A5wCPPl4ffZrrhJgKJ2ACTuDH9f6W4upGWdw813lG7NTMPkEnPRnJUnTNOVKtpMeHM/OwtfeNZe9WurvB4O1BSEKWlpLcwMo0SANhyHjYNxSFLKwQZTmnp9R7rBlCgtYUpMpJ1B0mdND+Vostuk7dEr2R211VzOUqK6dY3jXKYgfC1efSipE1eHcF3vlzJVTO05JBB0Q2R8cp67eOtkHEJdZW1MhcpPkRt06f0tAfHen9Y4MXHWnRVJeRZzAHMBDqY98C1qlOWZpVGubmgcFTKqSe0IPSBGs2DGmUjQ2N3o2WqtwHkpWk8pNiidd5gba87dkDcLiSLOXqZgWrRfGEbjvfKkmvu2nqNua2kn7z9a2bPEu72Ha253axoLaNQ4w4CO7C21AyI/w216O16m9uDGEKtRzKboU02sa9ktTfXX2fnZc4jXLXXtcLoutoO1lMtNQwhX8aknUE8pBNuGk/TnI6FehsPFp2kdAoJvairX6G9WqRbBu2mvhDlIECVAuGTqNCDnTGvw3tl1sCsuq6sySBVXVX0Cu9MlJKk6HWeeutnJR4CxtT0i7uacpBT1LbDruZZzJdayxAAOgPjysuU2AHmHmClxrsqavfqEADdl5JGSeRBPwB95zK0c1REDyb2UUXjdrzV23BeCax56aNbyw4rRYQtKy3PQSYHhadqZTb9Oy8mSy4hKwI5ESNNtrNC7+EjlIQivvp99torS22pEJQlaVJUnUnfMD7hZ63ZQKum7Ka7+0W6KZoNJW5BJCREnxgTpYUjw61kaCJzL3QBGubNmOYRJMe/Weg62pv6TV1+pcQ7xcQCG6xmlqxrz7Psjp5tm1zykOKJbGh1EmTzm1WvS4uxTWIrqvHISmsux1kDmC05m19zlr2Evy1FuoVHGGZqa/RVyT/drT4hX4fja23o4VvrXC1hs6+p1j7A00EqC//eH5dLVJb2WkH2k/jP4Wsv6KVcHsNX5dsyaatbfAnX7Rsgx/7O2zizb0xPQhYWDutUgdQVez0NLwNJxEva7VKlFbdSymNAVNuoI89FKNrhkCJBBnUfUWoj6Nt5fuvjVcUqKUVZfpVyrQ5mVgefeCffa9qtR1nmLYtG7MxbNa2z7rkkbTr4W1I/mHnbCevKbaKp1BB8elr4OqolYVayOXU2J3ijOxm0kGZsaOg0V+dg3EhbZTrtGtjxmzkKQZhZM68YRVUbs6Z1tzPVJP3pFhFKMT1nn8rc4hBapu1CR9g82uToAAoT8ptyXCdFKI1tpN1WY4WQwJ1BPWdbcZxOaee2/ysGFzuoaeFsSoxIjQT7rFQzqhCr/FAI+hboKGmo1sAVA6lQgn5W2les8wNT7rSCijCVCQTpr52ULuMvCSNfhFkpKjoYBGkb2PXYoh4lOkmzpJw9465FGedst0kqCQAQABoNfztlldNZeFt2uLcv5pTK4IbKjpmA7x5/Xy0khd4ssNttpWQ9HfVvsDoDytGVHUvM1ofadSCtoIiSTJUoZSI015b9edlOmeNRWINZV5G0rH2Sf4txlHnPS1OGTK+yPK3My6k2jr6JbdZW1dc3SMss9h2qnIT7OpknWPM2gfDt3qrMa0d00zrK4rUI7VAELSkznBAJ1ACtfnZ2OYgvHEeMF3XcDKKmmo6MIcpnmwWgSoZu6d+WsEjXTezfxdTVt3Xi5idvLRVhWCWko7MNKCP4QOQI5TuN7FkqA6QN6ITIMrMysxIQSpzTNuCPzg7ee1kqura3M9QUTjQeKc6Co65dR8otFGCOMVbfj7d24jdS28mSXUJAD7eypT/MBrp/LtZyXziWhorwarrqWqpWzqsZSQtOkjSZ1+467i1wzNIuqYiN9UnY14gLTVfutSG+1bb7+QEnLl5yJBnTTl1myNc+J6Jy7mBVQHIepS4rQhtfsE7SAREnXWzRxFUqvW+qq9A2XjUAx2h76UnyjbbrqLJ2d0NBzsysICSlREKzcwTtMz7o2tTdISbq01uiXroQxUVgNQouq7ZGUwDnSdNZ6aGenTeyxildwUNW0y48ilqEoUlKgmSdu8Fb7TodIJHMCzc7WjprnZrWQj1kLLayFGEEiN9xpr9GyFez66iriofZdVKUpWlRJAzTEDwjaRaObRSy6p8cEK5DPGHCyG4UF3glskCI7x2Hjrpp0629C8PN+rXbSU8AFDSW+6Omn15284eFFS5d/FDDAcKghV6UuZIVIyZ09OQIn3jxt6R3emClClAlKtYG5+63H9ov6jfou47MaxP+qpb6Uqn18a6ktsqWhpy7yswDlBSjWOWsDx8dLev1BPqTJ/9Wn36C3kd6TAW5xiv1ZnI0i7TIOhkIAnp3iPhy1t643WqbvpjG7SP/CLaUJ/7aP6LFqv9VJ9UOqSNyD0+NgFkDfWw6+u3vsA5CTMmJibHYqkm6KPE5SUzZOeJCjGkazPKyg+dCT9aWTniUqOb3C11ipO3QTrhQ2FlQ0Oxs8WWrvvIJRX0zTyBqlLiAoA2YV6LyUDqkaZUEaEb6/Gzeuzite1Cy1TrYpXltISC8oGVmBJIBAE+Atk4pFM97HRcls4U6MMeHqX6m57ru9SF0FC0yXQSrIInpZUuWpaep1oSrvIWUK5QbQ5WcWq2uhJp6VAAgKAOpmeulgUcWL8pyX6ZFKjMgIgNzoCYOusyTapNFPJGBzV8GMNsFNd7UBrLqrKbOUl5hxEjlKSJtVo4Pbu91utVedQsMLaecBEJCkKBI15aa2fo4uYweQWl1NN39IQ0n62szcTXhU1V2V3eUkOU7mZIG/dPTzsfDWvpY3sfzQJWZyHHku+MfCrEHFPFLd/4QraStusUYpHlIcQoB1ClHQ5t+8LRlhDgBi1jHlW3TPH1lmiqaVZW0W2/tG8hKVmUqIzgwPG0o4dxFeN00C7suyscpqUqkNNGE6gAkH3WUrprKt6822zVOlS1GVZzJlPz1tWjNVAzhtItyVl8EM13OCiGj4PYhw9fFLSP3i0XbgvIvVAiQtKwlcJHx+tLSPj55arvuZwCezvajVtzKo29/zshKvK9r6vm9GRTOl9KkpcyZipRSVJ133AHw0ssXtSX7eNzXfdzGH7yW+iop1rcLUJCUlKlGTB2PTpbncbZK6Rk0h1Gmi08P4LGOYxSBSqZp/SDuRmjBLdRddQgyddW0HSd9bSiH01dPWoomchd7GEDmQp1JP/ACWjVWH78XxlwriJikKqGmpnGHl9ohORSmgdZI0EGfMdbPbD96rp3GEKbLiKpx5sOZkgI7N11UHXmFC0KGbJAL9VmV8YfJp0UY49uTE7l7GtorsqKhTeKMN141T/AHaGcjitSPZjz87P3EN1XnW4Oorv9RUmopL6ZqchWB3PWnVTIO2UTYHHF93mwm7627rsadp6h9g1LinsuTIe7yPU2PVWOLpfVWpq6htAWhtKcn2moU500mFTaD8V4TiCfomiw5zg1zRf+BVx4533Q4fxpfDroLiKmudWgtgEkZp357je0S1eJXLpoH0N0CqhFdf6yAV5ezC6NhyTod491nfx8bqryvtxNyrccZbeKiVgCRA3H1vaPL5o36i5+wFOVuN3tQqUBEpzUWU6+bZtvYdM6ogEjhuq1VEIZDGOSb2JMYV1NT1BoWksuFKnEhcKElI2MdR05+GonBzEFXjd+8aS9lIS5QXS7XNOtpCe0eSpMSD/AAwdh4WCvnC9XeDislFVS02EfZvAJnXT2T9352C9HfCl8IxGhbjnqtPX0FQyVDvLKCYIhQiZSRPX4W0q0uZRPew2ICp0wDqprXbEqer6u6kuzBWE6ykzD1i82ah5RUT9opMEidBbeI+6pS0uyQZjYgdfhY7fl3preGN3sqcdSbuelDiCJCmlgDl+H6tzEN21KlLcXfdcsETBWmD8BbzaneZnEvOtyuykZw22aEo3BWOKd7k6J0kTOnz5fP3gY1vJ1y73W3E50ZQQG0gKQoTr1O/yFkXD93L9YQpdVXENnMiX1CDOugjoPrSx7GNSF0aUqUo5G1gBP8QJGubwk26CkiBKz55Dw7JisXvVqvMMGtdKEFSUjMkQlIICDpqJgzpYd696p7MguZTBk+6IB843/OxBwA3yQpaQkIJT706g/P424chuUjdMqlQ0jntt/X39XHC0hYgmdshHLxqYUpVQEwJJSqCTp7+XjytFWNKb1atafZbgLp0wmdSozJ2+fv01tJRgKPfMHRI3k6D6/rEf8QW15aEpJgJUkhM5dFQdSANo6bnxtfpmhrrBVKhxc3VJFMEIpmm21LkAAaxBPOY036xaK8XpbcxZWGVZVJbhSVFMSAd4EwI1E6AgRNpbaC2mmUqHdWghJExsNTtJ1+YtE+MwhGKKhIyolCJWI0ievgTvpqCTBtek2VJu6Qn2XO2Cex1EKgxMhOwlJkRHlJ9qNL9ei204zw+u9haCjJSONgT/ACs0qyI5d5S9DO3utQ9m6a687xZu2ha7SqqXfV2G0albqyUoGpGpKkADf8PSXhthFOEq2tw22nOi7r2q6EGCkqSGqhE/BlPht4Wz5KhrJWxk6lW2RF8bnDknu3lKMxKVSnbnO2u/L8h4vJFFSX7hp26a5rNT19Kad4RuFIjU78z8BrZnobAIyNJI1SZERp/QbeXQvK5HAq7KcK5JKSJ5Ax/UxbUh1WZICF58XfddZgvEd+YOqmlF26K0sFDumZqe4oAaGUxr5+NlleJ7rZayVzTiUBQlcCEyOfIa8vxOsm+lTge7bvxvdWMX2Cilv9tV3VjidCl5JKm1GOpJET+NoXqLrFCtNLfa1qQAEtPgZUkRPeG0iB8NbHFwmFk6ruqKGtYUKJxCg4ggrI74zHfSSdRy8LLvo6X5U3TxONzuvlabxYfoHpOuZCS6hRnUx2agPPxtFfqNTh50Xjd7yXqZRUVBObKRm/VXxEQLPHhverN3cWcP3wiE09bUtZikwJWOz+ROvny516gE2JVmAgXarFcT0lm86N5wlPa06kAdcqv/AM6zKeKs2ZRggRBO0HlI8BrtaQ+KDJW1dlQQDlU6gnXnlP4ffaPygwcq16HloCY+Ox+QtxuI+Gpcu3wo56RqKPaHRJzBWsA68p21On324CJ1UFRyjUkmNLGHUACCqVbkHefjr5/1tgbCSE95JMAg/Dn4H6Nq7XK4Qi6EQUiI1yzzPhHx+fvhvizSJc4JYkpkpK/3fexUkDU/+cJB+Sj9TaaQjvJ7qQSZM8vrX62jfG91GrwTxIuzRSktmqhWsS0F/enz0saJ9pAfNVqhmaMjyKoFiFtSK1xSgqVHMcw28PrwslAHefdZbxI1DyFlSdUaRzg/rZEgpIJ687dvEbsC4CQWcVfn0Nr0/eHBZimMKN3XlU04E/5Xf/ebeVp3q0BTi0lByqhRPnOuvl9C1YPQJvAv4YxXc+4pa6mqYB1HatqT/wC6tad5pQUkgBQKEkwNenwi3GYlaOqePP8Add9hX6lIw+SbrlMhJWooJSOZB1Ov19RYq6CO8Rv7RA2ssPMdmrKpKjBM6AHQn7+nn7yzjKkJyrJGx16dfL8LVA7RXS3qk1TRSSQYiBOUnx/GwLrZKQRqUiFJn3Rt1/Cx5bSoKAICTGUHw/LW3DrZUFDKESQAcsmf6n+vJZlAtSZ6uS4A5Ble400PMfXwtAfpa3IFYcuS8YUr1avcppSkzDzRny/ugPo2sT2aQoK1HMHcHmD4Wi30m7oVePC28qwA9pd9TTVaIO0OBJP/AArVa1QyhtSw+ao18OelePJUKYJQ6I3UCn4iOdp19EqtCcR35dhVHb0Tb4SZhRbXH3OG0G1QDNc6EiAh0keAnS0qejHW+pcWaej/APTaaop+uyc//Rbrq4ZqZ/0XF4e7JVM+qupgG8U3JjvDt7Z4FLedOtROpyh1M+Wk/fb0YXvChzNvNLsnEEFsAKBlPWdNQenj+tvR65q9N63LQXmmCmspWXxB3zpCp+duZoHXuF1GINtYo3JAIBI8baOUGSZg21oNTz5Dba2lanTxnztqBZRWiTEJi2oAgSdd7Yo6ae6bcEgezzHWxGqJSBiSnz0z6EjVbagPONLJLbwdbQ6lI7wCvlNnJejeZrMNdLNC7yPU22809kS2PJJI/C2lAbtWbM2zkcSsRqQR4W2VJTy36CwQWJ31md7aU4RBkE8tbWEA6IUuGdBrPPY24SogjLsI0sGXB/Fy6flbErJOunIeHnaQUCjGYSJ08bKF2Ll0EyD5WSg5qZ8rH7tX9qFE6zBnpFnSTlzeHzNssW7cnULV8Ytln0SXhtToPbdo0ohfZgRP8MqmPl7/ACsl3lWu094KDLigVEhIGggbxJ5DQfGbHlBDVQU5kqcKUicxJ/i0A8Bz+YmyHenapqyW2ySrUInTUbn63nebZLTd6su0YnJwCcbViy8nFKzFdJI7uYzmHx3g+fnZxcfblo7vu677yacf7esqXAtKl5kju6x02Gg+divo7Ubjd/X04R9iKZCSojmpUxrry+6yl6StTFDcCASmXXlKABmQE6jlztQe93tENG3/AAtBjWmhLioSoVLaq2HE9popBUYgDw8f00s9VXkWVpbWVpSQFpAOUK0Gh5kD8POzFpXXHXAylYUCCkyJA5j693msNorKgpyrygnUExJ8Ou/LXX3HZKyLXR+sfL6hUR33BqUkpTI2GvmfrWxVWoBb7yIzJkwBAPL4a/1sI25Rs0q2nHnO2S3lQotiJJ0Opk7CxFSlpUpS5kJCYBmTED89OQHuhdTDbIwe0RdrhS4stqdBykQNtY8do6jSxZtinS0CzVla0lIHcMHUGROxBnwiLH2altFw3i0pJDgdZyJHQEgxZz8EsM3NizEVbTX3SLep26PtUtpcUgZgtI1ymdj1IO9k52VtypRszuDQieCKRf8ApGwwUZVBN6UygpLk/wC0TOu4OpMHfXwt6W0yCH8gP8RBA5T4/X4WqtdfDnCNx1Ld4XRc/ZPMLS62ovLUc41ESo7ETaSKbHOKFVBm9VkKXJUEpGvPlHn525vFqV1c5pYbWXWYNUNoGua8XuoQ9KdxLPF++EFZyuNXaQiIChCQTOnWfdy5+uFzqP7tpTzLKJ5/wi3jF6Qt511dxMTU1tV6w4pdKlRJ1UAgQPPp529mLkcz3XRqECadskDxQLXWxmOBjTyCx5ZBLUSPHMo+rWRpr8rAr7o8Ok2FnSAJ8LAOGRqN9NbFjQZEUqFGCOY0+vjZLqVD8IspVE5SdxEWSqhQzd4kRpubXY9AqTt0n3oo+pPpAPsH7rQ+h0lITPLMNeVpRxHXpoqBS1ICpWlEGBuQLRNKgfsxGhO3142r1R1Cv4fsUdcccQcxMyIkjcHmPrewjLzimgAsnQRB0J5D7/hYFplxSA6dErEAzpp0svO3Cmjuakvc3jd7oqVFHqzTpXUNQN1pIgDfYncWqFwG61W2SahzKiCvNzBzbfptYO9Q47dtZlMDsHDrJjuWPCmQSomdR0Bjp+Nua1DblBVtoVlHYrBPuPT+thlymRoi1yqMLjVICVEk66izkuZIN7U6kdwJUXCCZ0G4n42bFwLbDUZs32bevIyD8LLDLrjJS+0ohSZgzqDtYLnIrBcI6w83QY4vp4QhJbaVPQZ3Nfus6arENHT3ezX07qFMpppWlImDlSPwFoxvVxSr6cedJUpymRmV4gkGfj99lJKj/ZJ4TCvVleMQJi3M4/8A0g4dVdw+IGQgqSq/Hl20artRU0j77hUpEpUEpyuMqSQZ15C0ccPsc4gdua86FVcrs6a9qppogSoJBB3InmfrSwmIFqULodTAipZKj7lD8vhZr8PVZP7RU6yqU3y8QCd5Sg/jbnWyvNM4XV10DGSDRPl2vqak/wCsvrUJmFK9+g5W7p3YYdIQQMyDodjrZNaWVAhASTJGxgHSxqnUBS1Ikx3DBGg70e/e2TLdw1WhDYHRRlxGUkXy6twEQpsoG4Pd+vnZl3kHm1VhfbLZ9aup5zTUSXUSnl0+PlLo4mlRvolIASWk7nQQd/62ZF9EG4Kp6A4ct3LKkqGsVKwN+ma3pmDEChYuPxJpNU8pRqmb+XeVQi5KtppbnZlYcUmDmCwD0Jifo2S+Da6qjxLQU1Y4e0acq2AAQYTnXtHKR87aIq26xaxTqWCykqlGmUKMHpsT+Z5FeF9YXMZMrS1l/wC0H2zrIzaz7vztp1MjX0kjPI/ss6KJzahjz1CnKq+0wFXtR3W6ipBG8aqj7jv0s27/AINMlSUpgpnMTvpsdPLpz87L6HO1wnfzAUD2dY7KSIiUn8x8fKzUvSpSu72FwlwqaQRJ2gDX4fC3m1K20h+q7Oc+FEbrdWiobT2mx25dDYXFwzXUah0pAMJUBoSJ2Hwsm0a0+uSkZioDbYGeYO4so40dyYfnLmUmSMoiBy8utupoG+KxWLMTkJTDcP8A25ToQhQJQ0mI2OQb/rbhxwZkyAqQBMxtrz3ttlxJxBdjgXJeYAhQkEAKTy/yi23EJQokmDPMyd+cW6dgsAsflqgCnXUgFWpJJjfrz3/CzSx6wHbqpQgFSjVlJA0kEKOvTX7x1s7VIVCdhAnVJn68bI2JKJFVRphCcqKyVDf2kxv4kp+FrcRs4IEgBCZ1e83S19LTtrjK2FEb6kdeoET5+M2iDHhdbxG8nLoUoVmgzlEpGgMjY+EdNZka/albmK1IAStLRyyQYAAG3vsweJNODfs/xFhCglOYKhRXrmAjkZAP6XpPdVNu6kf0PLsw1ePpD4OVid1pNBRmqvL/AFhxLaEVFOw48xmnUQ602YmDAE6xa/l2ss/2pvp5t9p5k4gbWlbawUkO5Z1iP/nJET5m3nH6P9MmuxZRvnuOUdHULCVGMwy5BvvPbTIHIWu1gXFfbqvChXlQqnqKKqSUp9kfYI2O393qPo85W0TjVNqw7YWt91r0k7QwxEbqTbwp+wyrbT3wqVFSiRPhOx3+t1jDzyl3cO8cyVLCiQBrM7ddRv42L37SuGhU4gpTACp9/LXx8PlYLDDx7BxsqIUl5Wk9QNduv1pNt6hkzBZlfHwym96QeEjjDhdezLDWaru1k19MOZLfeIka6p5c4i1U7sq3LzuxqqCkuO5EoWh0DvEAjyG3LbytetS21NkKKS2qJS4AQQRJBH1obUkxPhRzh7ja+MLk5EUb3a0bhRJXTOd9oH+Y5YB/xJMaW0gs4bJBXedBTuOU9XdymAFlLiuzKg31GmoH5xZHqaG8KWtpKy45cu9tZqWXkLBLK5BiCZyyJjw2s6y/SV+ZvtGfWhqhsHNJG8E7gz9aWal54aVRPKqWS5ShRAV2JhPgY8RpEWhKzOEWN2Q3Kt7f15t4jwLc1+tKT/rIaeBCtDmQZ+JFmaGFJSUhtWih4ieZ6cvv6W64VXw3fXB5u7lJKn7oqUsuyqIGeQRHgo/WljrrCCdjlgSkjUc/ONNrcLjX6dT9l32A+Ol06pPcazJyJGgjUmJ8umszt+NuAyUnWZMmeUjTfx91lJVIe/CSZOoA85+j08LYmmIJbVCcwJEk7z05fpbObItd0d0nFlQMhsAgbETzP17turQr7s9bvrFVzutApvG5wQAdT3FNny5fpZ+dkTIypzg6nWZ5e/WzbrGjT8QqMkAJrbrdaOb/AALBgctlT+liCSxugyRaLzRxQyWi0SFqWkrQSUwCfqfP42biSM2qrP7ijdq7sv29bvISBTXk+wU8wErUkbT05mzAJ/pbv6V2aIFebVTckpCtb6Ad4BOLMVXRrFTd7FT1gtu5Zj/62121s52wQAZBHKN9flbyg4f8QL44eV9Re1xXneNDVvs9gXaKoDKi2SCUklKtJSk7crPWp9JfiJUQHcT4nUiDAN+upJk/4Ep+VsXEcIlq6gysOi6DC8bio6YRPGoXozU0y85WUwqdQr66/Wtij7PY/arypB3JGUEzprbzWqOOWNKpZcevK83TO7t8Viv/AHosj1vEi+a8RVUtO9rIL7jz0f8AG4q1duAS83Ky/tLDyYvSupvS46VIU/fNDT6QS7VISB8TptZKq8dYBowUVeNbhYiUkLvJlO2n83X66eddXje713GxTUeH2Wb0SuXqpSGFtqTroGy1I5bqOtks40xBMtv07es9ykZT/wBNit7Pnm5Af2kA2YvRB/i3wqZyqd4hXDmA7wFe2qPckzZjcVuKXC2/sAX/AHBQ40oqmqrbsebp2WULWVu5CUJEJ3mN9Nd7UlGMsUBGRF9VCADPcVl193nYJeKcSL1Vf1frp/5wv87WIsDbG8Pzbfzoq0vaF0jCzLui16ILdYsFKhMKhQ12s8ODFci7+LOGanQJcr2mjJ/7zuHb/NZjvPPVDhdfdW4tW6lqJJ95spXFeK7ov26r3QrvUdQy+NIgoXP4W2pWZoizyWDC/LM1/mF6Mu02ZClJOsRJGgPu+t7Xg4LXsm+OFmHajtSotUYpD1lolvX/AIbU3NEh4Z0rCklIIUn+U6g/Py1taT0aaxT/AA4VRL3orwfbAGwCsrn3rVbiMPk/WLV3mJRfohyleQZVOswbaXA5HT7raKwDPQ8utuToOfibbwNlzxCwkc7cz3tVfrbZidJMTYMkphWliNKgUHUp7RoyIB982YiT2NZXU8Zcr+YCYEKAP3zZ/GeXdmzFvlBpcQvI2D7CFjT+UqBPzFrtOdbKpUt0uuSsT5bRbCsjUfxHTf6+jYuVQoiO9PP47212hKj3TrFr40CziTzRgLkxPObbCpkGR52Lhz2iTIB/O2docsCNefIWYJIyl3vROx5iOdjt3KQFhUgzpFkrPBJ02JMn8bHbvcJcmfHW0kk4gWo1Qj/hNssT7Y81GfP9LZZJLxJbShx1JGSEpRIPNZGumnnJ62bV89yoAUtJlSiDlMefnqZjpZ63E80KlxL1zIqiWkBvNKshCNB3dRvGvSzNvgMP3wrs8jas8DtDlAJM8xtrEx91spnvq04eBFKC+7wupLwu+scbK0pzAKgSkzm035n3CxW+sSXxfxR+9LzfqAyT2aHHVKCQd45chtZT/c4p7tdr3XkunIowkBSMoHL5a/1snYeo2K6vU3VoWUpSe6kgT4a6Ry958rWLNJzWQbuAy3QF3FCXD2iQVESkpgR5+74QdrLKVJUITUNpCEqJka+cxrZMokssXv2N4JBaT2mZJOhACiNt+Xy1sUefcIIWo5hy5DroP05b2kRdII6muLhLSFEpTKYGw3126A+dh/3felQhVSxTOLGhzhJJ0HMgfXU2IZEC71OhgjMYTz1nWOQEiB1087LeD6p9mrSz6yOzfaX3PaPdBIMHy+doW6KXJCUV2VIuS9XKqlXTuoDQQtwHWTqOh2nnZ+ejEioVxbui6C0H2b4cFFUtkqnIVicsHQyBy/KyDVsXtXUblFKEKbbShtZhISU9IE/HWBp4yZ6Il1Vdx8fsJ1FYtotpvBlSggkkAKkjY7wPz52pYo8x0cjgbWB1VzDh/wB1HpfUadVde8uBV2Xa+Gaq7rwoXCgKLTxKCUk6HUAxvZEPBmnrsU4cw3czvqb181RpA68VLShWWQTp4G1sMfuUeMMRm87vzqZDKG0lSCNt9PfaJeJiavA1fhTFtIgKdu29kvpaOgchCiUz4gEc97eN0ONVLsVFO2YvZcgG+hXqMppZ8NzGJrJSOmxXnp6aXDi8+FnHWqwdeNUxWv0hoXg4ylRQtLjKVCJ1G5BB3+/1pw+ofuahIE/6s1zn+EW8p/T54hL4m8fa3Fa7tTd6Q1RsBie1VlQ0BJJAAJ18reqWG1RcN3yFf+as79OzG9vXic0TSvMg1zXuDt+aVnVg6D5dLAqI8QbcrcMxynrYFaySRMWkxM8armpV3CkdJ1HSyPUqIVAEFPysovLTBnTUeBsl1RAzEDW12MKi/dIWIKf1qiIBAhxCpUdNCDaL2EoVGcQCN8vlr9fO0jYnrPV6VALaVBbyEAHkCRr9eFo9pUutOB0t5pGh3kf1tUrCGkXWhhoLmmyO05GVLKQQc4GYnXX6FjzFM+KlFKkoK3FJy/agATy6AfL3CxJgrWorICNgByj32N0TgaWQHgYJ0kc/K1FzxbdarWuulO/7jvbDV4uXZeHZOOJShYUy4HEFKkhSSCPP60NkqobfqmXUobVJbUI8YNjoq6hxtTDDKio7QkmPrWwoum9HmVLZpKrMpBBKWVHceAsDjR8yEcMdbVIuHrveQw0lTZbBZbk+Q/X7rOu5MP0d4vPU9ffjF2IbZK0uOtlYcVIhIA2Pn02Nke4MN32/TU7q6avP2CBm9XVqYH62Muqu273HEVF50bK2yUrS9WNoUDMEEKMgzPlYZka7QFTa1wbrokS9FNt3m2M8EsqAI0kBX6iw7UO4aqG1GT2LonmNCPOyRfuILjor5ZqHb+ursGad1LrjdcysI1QQDlUTJ1IToTBgGCbF/wDSRwvaw5WrqOI9yM1YaeU3SKdzLUrXKCAeZjlztj4xC+WAZATr0VuilZHIc7gE4b+dKrsuh4JWPtKUn/iGtm3ggOKvTE1OJBF55xoRllCfft+W9nVfxQrDVx1ly1CaxippqR9DoQUyhQBB1PL8bM6kvG9rtvm+zdxp1OOVKFLKyAIyR+FsKmpDJC5p/mqvzSDOCnmlt5AJyKKIkzuP6zY5RUL1RSVzil5S2ylwJGv+0SP+q0VY5xRxDwi6husfYQH0B1tLSWlHL0IOu0dbN65OOV+XI5XVmI311tK5SqaDLKmEqzykgwFTysX/AKenkjzssVU9rRRPyO0Tox9QuOXxTuLGqm0gSNTr/SzPxV/2XdV55cpKKKncCjpIRVoP/V8DtZKvrj1dN53k3U1WHqzsEo7PIKltCyudN0kRqevL3iO4ho8Y3BU11NRO0jNRd1ewGy6l1WZtxhU5gACNZ26a26mhopKenayQWssipqop5XFhusxTj5Vx3+miSFLF4MJpYBEQuCN+UT9CyTw3PZX88862EuG/DkSnUpSQkqMzp7XwmzjS3dKHGEFvtMzJV9otIM93WSFfdZIwq7SUWIrzcdbQyo16VIWupSmJbQZkogjx2kRrbS4DeE4W5KjxnGQa6XUv09Uhu7MRMOOIQfWUKTmIT7SBtOh/OzXdbcq7oYTTqbUvsEJKc6R3oEjfl+JstVV94GpmLxpqi9roSy+8pxIqLwRJzNhJgjLPPYWR0cScAXXddLRqxLcNMQgIQtNaSox0lcHQTNuJjw+UPJa0rqHVERaLuHqkekoa5FeCtATPtEyd99uUdJs4cS3S7V3ItlS0EKSOoPz23087N+o4rYHdWFnGVM62ZXmStS0wn2tUjoJ05DlYzenGG4Kijauiiv5au0CUJT2KkhR0PIagzGvjbbpaeZrgcpWbNLAGkZgmo7Shq65UELcQkoS4ESUp7RWk7jy8bBO01S8ciKOoXuYQhR5jw/H4WLXxiapu67luNVtUwh4uaNrUJggnQEfzaT4+Nm9XY0rEUxfCbxqkhJKW0Fbi1wNkJzySSdhbfaHbBZBcy2qdf7pvBaNLnqxlAMinVp4gjxn5b6WLVV2VyKZ9x+kXlYyrmBKOU685HMTr42IXthvjLd1DT3m/wYxE5TV1V6oyS/Rgl3IpUKHbkoASg95WUap/mTmJ0VxcXu3eeq+FrdBQMEKq6moxHdmZluJKg0l4rWe8YSkEkiBJ0tZjhmvchV3TRWsCojrHSvE1SkupPeIKdJAzRvv9bbWbeOkF2/G1KdCCKJnN0iVZTBBG86/fFlanqQb4vV4rlOdScs65dTE6gCBPXfbQFJxasm/EuIbIQKdhBOUd2UEjYHSfLzOlrsujFUjN3Jwej9CceN0RqG6cKp6oKU7IiAT3oB1OXpufOLM4arqejxHWsovFkrdoElBSXMqsgdVEkCCCkHWPZ+NXOGVQaXHdO8w+EF1l8pJCRP2e8mRsTtpoDrraacM3ktvH11tOOhYrqd5gAgjvKSoDWeYUI8uU2qGPO25RhLwnWVvb4xmVti6UUqcgbSguEnMRlTrH4xy+AmEapDi6hSQApCkkmIGoifr7osxTVIeTS1AM9pTMrnMBMoTp57n6NnJgysQF1TYRlJSDMHKTP6nbla/BE2P3VSnqHzHxJ9oeMZzIO0gwBvzPT8BavfpS3Kmnrrgxi1lQHELu2pUE8hLjRJHIZnRr/N8Z29ZSJCF6lWsj3/nFo74+3Z++eGd5paRLlEpFagp1goOoieh+uVpAG6qXe12sVawqmcVS1EaKRPfPP7t+eo0sni/sZYZa/wC2WG75u7ZS0GKhtG0xML2894mLOGhbpqgpadbHaglSeRI6ePnvYW87vfVTeriidfSvvJypzECOfMxMafqXHkpJ/cCcS3bUVN4XZRVIXR3xSds0UmSlxCk6EHYkQAD06zaXBdy1DOSRpmlQM7ePmP1tTHB9deuD8WC8qVxTK6KozOsQoJWhR1PwJ1O3vtKt9en7w8w/V1F2PYIxC7VsSHUqXStJzc4JcUfKRNuMx+iqaqdroG30Xa9nq+mpYHMndbVT2LuIGQJggxO493uHy5WaGO8QvYYqaZDFMFl1srClAAGCNPEwLQdV/tJbtaKv3bwukcu3vkJM+SGFzv1sy8W+n1eOIXm3f9H2HZaSpKS7UPumCRpJaTGw5WyYsFxIm5Zp9QtiXHcOAsJPwVNq+KN6dt2bNG0dAQDMDfwgmwNNi69r5xPc9QxR0xqEIfbbLtSmnRK49pxyEjQc+ZjzrbX+mtf9SkAYCwuSkHLnpnF5SZndQ+jZPc9M7GGdK2cF4RTCgdbsVt0/vBa2MFr3aFoH3VF2O0Pzk/ZNHjk4qoxziVwpQhx283nXG0LzpSpa5gHmNdCNNBaKVgpMEROsRZz44x/W49xBWYivWipqd6tKCtmjbDTKcqQnup1iconXcmzYdWVrJTMTp5W7KkidDC1j9wFxNZK2WZz2bXXEkn9LZJtsDwNukMur9htR8hNrVwFVAJ2QfKJtllWmwtiWuKfUsPXlUZtuypXFz8BZbpeDvFiuT2lHw0xQ8nqi6XyPjltEysG5CmIZHbNKZ5tlpFZ9HfjY8ATw3vlmf/SGex/8ZFlam9FTjfUkZsMUNOOtRflA1HuU9PysJ1XTt3ePUIjaKodsw+iiS2WnOl9Dviq8tCau8cKUebm7fbS4/wDZZj+NlFj0NMVqhVTxIwShEEns36x1WngKeD7jYRxKkG8gRhhlW7/xlV7sICezSeijr8LTze3oj3zRNJ/d2Pbnr3ioBSRSVaEBJHtZuzJJ20y9elku+fRxxDT09IKK9E1jwSUPpRRuIDcE5SFKjMDsIEiNQLMMSpXbPSOHVTN2KYLk9MbBVFhu7aGrwzez1UxSMsvLFQwkKcQhIUr251IJEj7rW89Abj1c/F9vGl03fdT1Cq6FUNRDr4cLgdDqSRAER2aRudxbzYZ9GnGjzq0LrqBhAPcLpczKHIwlJA5c+trZ+gDhtXAPiPfVXizFd1/u2/rp9WVkDoKX0OpU3Eo1MFYjxtkmnw+JxfEfEtdtTiEoEco8K9LiBBBMdPK3BEkFU2baOJODHBrfraY3lpwf9NuzxAwhl1vtpMSPZX/8NiAhQIKXzJGWNPG2lHUkiROulm8riHg3VJxBTyN9FA/MW0OIGDlwRf8AS6kcjr8rFaQoWS+ZOhI8bMzGgTT3vdtSRHbIdYn3JVP/ACmx88RcFqOUYho5iIkzt5WaHEDiTgZd30dQ3iu7ZZrW0yXgkAElB1O0E78rHhdZ4QJm3Yhy6mZzdd9ibaU8MsgQAJ56+P42Z6+JeAp7mMbnVzPZ1aF/cdbAL4r4GbGX9+hYEqPY0zzmkxplSZtqBwssstN09i6rKYIk6aibYHV7/GZtH7vGLCKE50G9HQRmHZ3dUQQPNAEe+xN/jXcwSVMYcxJVka5WqJIJBPLMsfdZZ29UshUmIdklWY66QB+djdC6rPvpO4M2h5fGrKErYwRfaUknV9bDWUzz7552CTxxr0OpbYw1StKkgiovZtKiY6JSo/XnZcRvVNlKnr1hX/dzbLQOfSEqp790UKVcx+8FaH/2Vss/EYnyFecWG73prsrnFVlQ23IahM75Up1kcgNbN7Eibscqn30XYhROZye2V3lFR1A6GR8eU2WLpuJi+u2XUuVCXG+zA7MBUS2CZ8dPmd7GXMNUhfX2y1uBKYcJA1010Gn142zmtANwjuJLbJnUrwVhZ1fZqbydqhISnSYOniYPwnyshXRU1LDjjlMklcphQBOn3/XlaWUYXu9V2Jo2WVntVLXlRmTMgCfl7p8LN7DGH6Zq9HkdittxKCCHASJCtToYO/hpaecBR4ZTFaQ/WV+iFF1ZIG4JVOnOehtst1DAW28nIuSlQOneG8+P66WkGluSiOK1JNK2odqVEqOitAoQM3htvvFiFdRUbVa4h6kZLnalJzJBJUegJ8fy52YypxGmW29V9kA6oRlkTrrPv5nfw+JzD5q/3nSpacASlckjURGun4+VnOlttlKUM01PCQCSGkASNYk6/lB62cFz3u9dylKPqq0ue0SClQMKlOngB578tYcbyRBEDuUSYvphwlo0z5LitSlgke6PADkd7SRwVxjdGC+JOG8X3o1XPUdNUmoeSzSlTmUaEAaA7zG/SyVdlfc9Y6C664lxWhBWU5vgZ5bG1g+B/EC+LsxDdVw4forhpS66hAqnLoZcdKSdu0UCrmee4tkYxUPFK9uW4IN9baLYwqlD52kOsbi31XpFwsuW7cb4RuvGFFUrapb1pkVLLbzOVxKFCQFCdD4WH4o8BGOIdzUd2ovlFCKWo9YU4aftMwCCIAzDmRzsNw2xOGblZVUpr62qKYcdRTHIs9QE90e6zyfxfUdiQ1h6vfCkkEJyII03OdQtw3Z+HAhA2SRtpOW5RMQqMQiq3AO2Pl/deGXppXF/ZnizUUyHy+lYp1BxKN9FAbmeU/Dbe3q5ht1IuG7gND6mx/8Ag028qfTwvBD/ABuqAzSGlSGaXKha0lSAlKhymNj7/O3qVh11P7gutYWNaGn08ezTzt3NEXyUUT37kKvW5WVcgG10vF47lQ0PKwC3SRvy26aWA7aJUVkE2CU+lIJBMza7GFSedLrbqlEEkiB87JtUuZB10iQRYR6qOoB+f5+6xB+qSoKhRB56xra9GqLymvjZ1SKBASVZy+0BB/xg/ODaPF8VqClSpn1/DzawezCTQUxUFD/MfM2fmL1F6gRlT7DzaxHgr5crVSqOG2Nqm9qx5i87oZZdqXFN9rWEHIVHTKlJ8PeLUq+kiqbcVXcOqZIL5AprPG1i7kKzYioGAhPtNUlKiI/yoNkCp9J6qF1m+6CqxjXUJLQRUUdxVaULLpR2QSoU6Ukq7VvKATOYRpZiNcF8Q1zeStxPcDJMSQ68RvB/2ImdrOO8eH/Em9sDVeA7x4yXPRUlaw005UXbcznbgoXmzJUXUgEnOSAAJcVAACMtCLDMPB8S0ZMQrLeEIWo9I3HdS4UU/D3iw8YnM5QuU6cpkSC4tGmh6bWTneNnG69FqbujhHfThADYFfiqjp1KUrTVCnySIIOk72I/6Dr/AHFKcqvSYxMhayVqNDc1PTHVOUmUKBEjf42Nsej5h+sR/wBuceOJ1eUjLJvdTaQOaQnNtoDHha2KLDI/hVc1mIO529Es4Yf9IrEWFf7QXXceC6dph5d3KTU4hdrSXm1BJlTDZb1Xoe/IyqBE5AuvLt53tfF4V15VdV2K7yfNU4yw+0tCFqkqCVQoECYkHUczM2sfdvArhlctyi56XHeN3qRTrtQpkXi0ErdchTi1/ZHOpRSkkmSYBNspeBnBK72g21SXupAGVKXbwCRHMJyISP6Wm2Wji/pMKG9lVN/VcoHo3nkUF5tpfcUvNTOZyUGVd8SMqfjp7usfXyy47er9Q4+lt4p7NYLhB5HknQ6+O3SbXd4f4C9HGmxXX0+MrvdaucXch1PaVdW4VPpdEA9monYmOXnZ+P3B6ANM+updwg1WOjvKKmrwcnTotUbco6+NrcTGyNz3A+qozF7HZSL2Rz0VsE4f4pej7cddf97pp74uxs0NQGHy802lMKZ33IbW3JmCQYkEG0k1Ho18PXX1O0+LLypnnoLq2nWyVmImFJIHlZiYT40eiZwsoHbt4e4Mq7ppKlztHWKG7ktocWB7RzOCTrv4mx2s9MLhAynPR4LvZ/SQVtsJnX/MTbn58BvMXRvAB+q0o8TmyAHN+Fxf3oRYIxRXiqvji1iJSEApaQ2ikSUg6xmLZnWTtZEqf2efCzs0pZ4jYodMqBUamjSQDMQBTGYnn0HvMv8ApuYNbVF3cO6lQCZBXWIT9yDZPe9OELBFBgBhvkC7WqUPkkWsMwt7G5eNb6BV3Tve7NlufOyUL4/Z58Kv3c0qhxZfoq0tgqcU/TkFQj+ENA6gkb2QMbei3dWDr+wnw+wPU19VR37QXww7X1ie1RTvrabKc/ZpASIb209k22z6auI6usRSruC5qJtaoLi0vOhOmg9tOvnG9ka8fSsxpfF+Plhu6kO4eD9dTON0xhZFO4lUhSiDovad7F7iGnMZnH7JMllB90LVL+zvxIxTNPI4o3SF5NUroXhlkaj2zrtrZMT6C18XZTXhdruH8N33WvU7zTd5OZp7ZeYh2VpCgUk7D+YiYSkWGvj0weJC8L0153XjGjF4uPllylTdjAI/xQUkxGszGsaGJZ1T6VfHerckY1daSdYboqdvMfcjwI+tbrIQzd5P2CBd7twPyjCPQh4vmmYo0YCwAk05KkOu1LRQoqHflJplHU6iSY1EmbLTHoX8cHDTn93cLaQMoS3lyv8AdSkJiOzZTGoJIECzIqeP/HSs7Qrx1iASIhtwoAPhlA6GyU9xT4xXiSKjGOJXAUzlNe8fCdDtp87HbwhyJQ8khO4CnK5fQ84yUlyLuivxtg9IXVLqlONUdQ4U5lBQSmcsgQB3sxICQfZTAFD6CD13qZqL14pUqDTlJATdsAkebvMeW9q+V+L8bvq/7QxJeCgBr214Lg6c8xjxsRbxrRUz03ziqgayJkJevRoHNGk5lTGg8dbJ72W9z8qTIjzcrcJ9E7A1WpuivLiQFoQFANs09OSAQJ0UVxsOX5W096IvCSmpyKziDWtrElFQxT07LqCdlBUGFDkY5C1dMN4lbTTIvK78V3C626jOgU19Urjzic0CGkuFyJ11T+dlml4juVbZQKhS+zJHNMwddSQCPLfflbn6yjq5ps8L8jei1qd8LWZZDc/ZTnjPg9w6xRdlNc2IuOuO6iloK1q8aZNFX0dO43UoCsqwpqmzpIzEyFTt0Fo+vTgRwHp2H6g4p4j3vUsIW4x69iuqca7USpMp7oV3tSCCDPjZpnHt3sNntjVJcAGvZiNJ9qFGPnYgriLQh3su1aWTplW8WyN+qDGmpHKLPFSVgILpnFT4lK0WDAql0CEsXrXNF9anUuHPpJUqdgBt77JOJqpFTiB2idVDblM2oK3IITOnTc+8bRBsp36WrmxtX05ZkOuqjMqcxMgCPu8OUmS2ceN1z96M1FG4e19RQ+dcoSQpYieum2sb8tOjlbdtlixkBylT0e37voeJFEbyu+mq30M1QaRVMJdQrMytSjkUCDEaeY2MWsXe98pTWUj7F03XTt09ShRRTU7bJJ1BEhO0aeHTSLU24ZYrql4mu2qpUst1zVQGlB/MpuSFAylIkjX5HQTFp/r7/wAT1jYp2a65UkfaZfVHEBX8MFUEjUg7fwaWqCIk3VgygCym648UhN30zDyuzNPTpQDnUUqygDSE9Bz+Vlq68e0NPVlDy3HEpbOYyNDIMTIJO2w+Vq502M8ZUSECruRFWB3CaN5KsoiR7WTSYnbysoXVjqrZce/fFxX22jIChKaBbpJBB2RmhUHrHnaXElZsVFscL9wrIP8AFK4kZkoS8SBKRkIjTp08fumyVf3Eq77wu6tu8Xc86h5pbZ+02RGnLUaRaETjWpeSr1bBOKXs47uWhQiZ0ABWtMeI5aDTkC7i3EeSWOGuJVAOEZnDRpCvH+/5yOWnSy7xKVMU8I2TPrHKuiUUUDSu1Z+ybCU5u0ImAQOcT+HMWHouIVdSU/7vvu7F0r7gKEPAKUiSJBmSY2jXTxto1lW3eKk19zuXa+9Ck0ry0qVB0IBQSD066crKrDRq6IU1Td7eUjKlK/aIOhkeGkxy99tJhLm6Kg8ZXWUbsLqaXE7zlfVmqdqGg4XE6COURuJHmOe4szMacGb9xziFD2GXKFtfZTUduvsUp2gzBnf9BaR8cfuG4nKOloe0L6AXEyvNkQDvqO7OnhoI3gH8DVT4vlhDbjSfWQCQqQkRIymNtz59elGokdELs3V2BjJSGv2UOU3olY+djtL9uBtRMZfWXFEDXUwjQaWU6X0PMSuEJrMW3a2SrKQ00tyI33jUWt61hWqWjtV1rKQoZpCSoRpB5HXTXXnPiZ/stTlcKeUdZlLStOeknr5fxG2OcTqid/wtgYZS72VTqf0OWQlKqvHbpGpPZXcNvCXPqD0soseiFhgNgOYivl1ZJkpYaRtyAk2tMjDTCE51POkhQUQEpkkxA5/QHSLcJuRCAQpDnfkxMFOwA2jX63tA19UfiUvZ9KNmqrTfonYZaWVGsvd5IPsqLYkf7v52UKT0fsNU/ZB3DNO4GNQVuLOYwPaP8Xlt8bWabuWlChNKoKToSXOe/wCP1rY03ctAkqBaHtaEpKpg6+fh+dl3uodu4pxRU7dgFCV24FwywUzgy4E6ADLc9OpQjlJaM/ebPq6KG5acBpDZpRySw0hoDyCEAnb5+cPj9z0rawOzAGWYCYKj+f1rYdm62ATm7MA84AMeX1vFqspkk3JVmJjGbJvsNXIdDWXs5rpFYuNOWg6fcfeaXdOHHgC/SV7p3lSlL000knXpy99loUVMSFtvZeUjlGkdPrW3WRhAKu1Qe9BzGAB4/XO1R0DirrZmgJDTh3CiYb/ckhQkjsEpgaiNQLGG7luZuOxuhttJgE5ANPLz+7lZQ7ahbQSKxlMmVBOUFXnHh0sGuqusArVeCE8zlTqIB/KfjZhSOdqUu8sCLqo6RlIZao0pTqlMNhURp1Ej68LBOUza4LTB/l9kR8fcLGFXrdyYArFLSToAFd0jTSdP6+8l3b9oUd3I8SgmZIBjkZOoOtjNondEJ1a3qiz91B0LUhpRUnYaajruR+FijtwuqKciQDvGYjofhpt1sOrEqKdXaN0rqwoc3SBttsQPMfGwasXvhGVqna1AgBRMGfCJ6WMKF3IIBrWdUUXhN5z2Xm2wZASSrcz+njbTWAXg+ioTWUh7NQISEqMkeM9bCOYsr4zAhsrICYRMyPnufxsFUYlvFxMh9Y1IEo0O3h4/fY7aF26A+sbZPz+2XElCswvO7VJTpoyohI02Mxy38ffbSuI/ElCEhN53eVgzm7AamBpPLytHSr/vInKu8HSknNCVQPl1/CwDl4vKTPrfaSRCQdxvvP18rXmQOG5VIzMOwUkDipxJaUkE3OEEz9oyTpM8+nu3sIrjfi9kBVUxdUHMZCwnnOkp2AP1Fovdq86lAJBInQmdvMzsBrrysTdUUq0akpGwMR0+psdsPVAdKOQUqVnH7EQaCWKOgDhAClSFCdNu7tr42a198Y8V3pdtVRFqk7VxIKc9OlbIWFDQpyDoedmWsvAKQGgkCJ70fHT62sEITmQAFDMNAZG+nysdkQaq75CUZVxEx/kaDDd0UyhGjdAgADfnr0/WxR7H3EhSxlvlhhajrkom4g+YsCoA6KUo5pEZjIJkfhYFRSk6uKWknuxp7v08PjYGgVY3ujBxVjpwfaYsrBmH+ybQmOswNP112sVqLyxHUFPbYsv/ACohWRqrLZUImO75/dbgF0FKQoBMmdNvGfP77chLqHFxmk7FKfa06c5185s4N0yDU7W5i65e18vKByku3g6sHTmCojb7zrbVJQUS3FOuUBVJzSZPIHmeo/pbS1uFEIKswMa7gzBiLd0rr6HA4VqMEbkHNy/Dpz99nUeaV1UtMklPqdNoY9q2WD7cjTt6Yf5iqffbLJSUBYTpKpPrLaLpqHytwFPZtKXICEyrSZEzp8jY1U3NiJ50qZuGvymFaUjpk7b5dedkJ2/b0Cm6egvyoQ0RAp6eoUWwEgbBKvLl46WDapMR3stTbVLetToR3ad5yYBGmWQTsCJ8bUuIAbkhHLdLWKcbVz4mYJdXd1akDSFslsak/wA3ujTbrNkeow/edO6qtp2qRhwuKWQutaSVAmCIJBBj5iJ6KFBws4g3o+kXbw9xJWKCVK+zut5YzR1y7jbSZjWdrOSl9HfjdVDNS8JcUw1lzZ7rcbnyzAT0/O0DUxDd49U4he4eFpTHqrhfdf8AW275oGnyElYVXMkGCCYyqJGw018bdVtzUtc0n1jEN1IegpW4ha1FQ3juIIO206T8ZFovRR9Iy80hFPwiv1CETlLyW2gnXYZ1pMf16Q57k9Cv0laqkFK5w9do0O95zt71pkpXERKQudvHkdp1G6tpm7yD1Ck2mmcbZCoKqcL3aw32T2LKNZjOvs23jIIBGnZwIE6cjHmCbtLctOMovqocy91RbpjuFTOsRz+trOL/AGenpC3mpta13KylOUlNTeZB/wAQSpCFxJJ8tOYss0n7M7ixUECuxphqkbUDIbW8sg+A7ODvvpqBpaHtKiaNZAiCiqTsxVIbqKBltCk11cQkDKCyk89/7wGZidPnZw3NxavfCt60943TVVyamnWFsudi1lCkkwqFZs3vEDpa2NN+zGvpsJVePE670LUZhmjWoZv94iRJPQ2TMRfs6blwxQqr774yKPfhDFPcwCl66JTmeUDtzSRYL8Uw+UZHOuDysjxUVcw5mCx+qhwenhx9pmSzR4wvVthIlXZerIyzAGgpxGp/PpZv4g9Mfjpf6CzeGNb7fQhSsgNeWpO0DsUo8eukbb2ma4PRQ4SXSnNfbF9X+oxnTU1XYtKUZIIS0kHmNCSB42Wbl9GTgvdJXWN4McfQ4uQ3V1LjsRuEdBrB5bedqjThEJvHCPQKz3fEX6ufv5qnbH9suOOM7ow9QUia6/L4rGaKkU7UOEurWopQlS3VGEjMdZ2mOQt7SXJhvFdLc9BQruJ4uU1Kyy4ovNgFSEAGJUOYtVvhlwr4WYQ4gXBjFOEbpuZu6rxYrF1qlrZRTpQoHtCoqCQkAH2hFrbv+klwIYPe4s4ZJBBhu8EOH4JJNtyidBXMuPCAsqsjmpH2dqSgxh3E5VBuohX/ANK3+CrYrCOKlIn1Sn22L+vyBFir/pR8CpAZx9T1OYSn1ajqXfeMjZ6WAX6T3CpwD1SsvurB1HY3DWxy1lTQHS1zgUzd3flUjJO7QNR5WAsVPCJoWxvq+o/9FsHCy/XR9re1I3m6NqXr8R4WSXPSZwiof6phXF7+4GW6w3P/ALRxNgV+k3duUKpuG+LFZpyqd9TQFQfF+0g+lZ8Q9UwhqH/CUp3hwSq7xZDNViZsSQruUZ3Go3c6gG1ZscYGvfBmNrzw/S+sXimmWhaakpS2FlaAsjeABmjmbT0/6S95rSk0XC6sUVSP9Zvhhr/wBdqocbuH2NuK/EO9eJLV3Ul0M3t6u09Qt3gt5SFMtJQlRWEICswQTsIg87VqmWleAA7VWaWGpiNy0p1Iu+/SEj/U2l7BK69lB3B5q8/rY41h29alAXUX7czM6FK7yQqDz9mfoWiZjhBjmmQlKr+eGQZUZaxeg/Lb4fFRa4L4mqaJyrfvlLrlOB2jfrC1fZye/udtARGkg9Tanan+ZXCKgn3VI5wuttwCpxVc6Ecsjrq9IP8AKj8bCfuW4WiXKjiHd7Ujf1d8GB5pHL62tF44NPPAl6tQhURISSofEHx+Fke/eEF7oeU5ddTX1lSEStptZSlwRqUDYK0iOc/GYfT7XUeHPvZTU2xw4aSpD/FBpa1CMrVMCJ8QpwWBdvnhPSJUl/GV5vKQdS1TMEGOUdrM9P6C1Z14dShYBaqMqVBCwXFdomdIIInQmOs6RYw1hdCnVKepMhnvZ1SUj+EHqJ6RGp8bSAi6KNpFLl+474aM1Ieu3EN4IQtvs3C/SElW2VGRtKlTuc22h11Fo9vPixciVrTSUd81SUHKFhumbS4BzAceSYPjrrrEGxWnuO7kKk0KAf4wo5ZPgTM6E6EcuekdruG7lhJpmG8hjMkwVpM8pGh5QdN9RFpBzNgFAxPKTjxKp3iS1hy9FOJBVDtVSIEDxQ6uN+li7vEyscTKML0YiCBUXupOUxpORhX1O1l5i56NLKltsU4UCQlWbKtI0kKH5aanewrVJRNlTdRT0JSD3ihsJIHOQN9ht8LPnaOSXCd1TUVxBxSs5qTDtxIJMgfvCpe93/m6RzB/G3CMe46qSDSsXAjLCu5d9S8oeEKdQeXSz0TQXeQFt9kpCBJOSU+Gk6aA+/3ACqpGgpCXVNO51EZ0yABGsGJn4fCz8RvRLhO6pkvYp4iVSM6bzo2FEQFMXFoUxvKqhfjuOVg6W9OKDJNSxiOrQp1tSF9nQ0SISqApMqbXuCefLY8n0UNtj7GrKm82Ud4kaK3zcvCZ326YtN3kOIDLqEoSJUEqUknfcanSN+k72WcHknEVuaYwRjqpSttWJL33yryro2gCf8SKcR7jYBVyYp0m9b/cBUZUq93U9Ng2pJPhFpCUllaA2h1LkwSYKUg7zIEQPvBsUUulpVQXSwUju65dI1PTmT8bLOeQUTE0c0yH8PXu+Slyvr+zMEB29q5QCifF73e7xsWTw+bMO1VPRvISSYqC4/mMHUdotUc9OelpAVeF3lkOvusLGXRSklJjTYyZkT5/G3CL8uLtBMoTsC4nuxGpJEjl79dtLPndyS4bBzTGRw1p1EH1G5BK9k3dT5iNdNUmDPjpPlYdrBaaAksIp2ERKizRtInXllSIj3naN7PE3vQq7zb5dBUDBToNDrJAkydPZHzkmu+Wm1JUgVaFH7Pvagxz3gQAddN+tnDnkJZWJCbw29TAJVfFfKoltuoPenkIIM6kRH52DXhRJKfWLwvFQX9p9pUq+ZnqOnMbQBZWXftRHbJSHz7MklKzroBMgnbXYR11sD++CSta6aoZW0Ncq8xSRz5wIB5nppzcZioEsCSncF0SnEreS4pYO+dR7x8NYI0Mc9ekW03hWhbC0JpAk94lJSSCkEEnczrzAIGtlJV5vOIWpCG3mykthOvtGQDI56fLwsXVV1jTZbZLK1ZSpKVNnQ6A7nYGOg231tINclmaNkwOJjBpryYrGGkDOhIzCEiBy+MmAdPLZIvqvURdFWhxWRbCmVKnTRYnfWe/8jPi7cYNO3wzSBCe0U2tSHFIQUwcoM7bkGI+in/6Nq+/rvy0VQhlxh9KgCNNRBHloOg02BsYtzNVfPlcmngfs7j4n3RWuUyX6f1hD5S4S2VpTm6+HUcx1tauj4i4RQM9XdTlKVgFJ9XQ4gzOwSoq2B3Gs6QbVWw4067fl3VTS1NVFAp1KylWVUFtXQydQIA5E+QerlVeLjy00ynV5CV5e3JWEiACQRO+kDSdRIJIrmLiI7ZuHsrP3XjnB1WhIZxDQU7RkZXkGmgiJMupHjqNJ18bOWlFDeQDtBX09Qkpkpb7N0AwRMp3G+418TtTF5+pUwXKt1LaHgMgeQEAhOg7pTBAHxB5wMum6ZapyJCnHSrs3GYCgYTqNATHdOnTnpAnUAJuHIza+27VdZN1A5JS2VA5VQlSZ1/wjl5jSdNLdKu9AOZypUkKVJCHACAVcp8lD3adLVAo7w4gtIU5dl9XwOzSM6E132aNDIyFW+3IiAIibLNPxB4m3WpNO9WULiFOe1VNZQrQj/ZpH825Mkk66pFoGiI5ogrmnkpl4jYeYqlU9VSvpFSiYzKSpQJgaeACR+PKzQa9YRlSpalPnuqQ2iQ3G+o03G8czvtZCo+KVbeakM112UruZc9uzXtJKQVCVFrMTHMQZ5cps5GKhupWVs5Ql6BKBqecCNOY/W2jTtyMylZ87g5+ZMa97lqXA5eNW12inHSvMUZlZUmBqRy2033625u6pXQv01UhnMqnUkkkGMydSNtOfWzrqKO8bzZe+0QaMKI7Rbm4EzEjUb2Zj6YfCCS4lKsgJOveJSojoTAn42BIy2hVmN3MKzWGsVXVV3QyFU7ilsogxkGh0CpmdgDz0B3spqxLSnNlpnFAnUKKYHnA00jYfHnCXDm+lCmNCXRkTISknvTPdnfl93xfPrpglTSBzCijuidDJ/p4TvaiaRm6vCsdbROtzFCASn93NgEc3BMAHUmBpJ/LbQsb/qHTnTTsJjXQkySNx4STy5WbqUuqzkJdgHVSRB93XlqND4cukKXklp+QCdMugnkep8Y5edmFKwJd5eUtuX/WlSiWkIIhIOhjXfU7mLF1XxeRVC6ooAJMBrSJ69dB8DZNQt9KZcQ2oKI2Okaajl9c97bUakgpLaTr3VJXJHl0P62mKdvRRNQ5GV3nWLzJU+ohRIgaKHhNgV1tSvKsOVZyynQnT7rBdmsKynMhJH8SRA6n77dlIH8KlJ2griZBmz8FvROJndUF6w+pJU6p9SojvqO/uNgCtbye1kFJAEFU6axzscLLKzk7IjWAUrIA6nU6b25XTEkykzoQT3h5SOf1rZxEByUeKSk5ZUqAqphZ/lVIJidtpP1vbaXHm0jsy4Qf5kyBt5QR5crGlsqSofaEzsVHRXn1+/SwC6dwgKSnTf2hOmm8aj8vO0xGFAyFABbgXmD6xlEzlj39I+X4cds85JCxrqDMeEEcxHzNjKYalYpgVezJ+/Qn657W5cUy6AFNrzSNMoB+Ea6eNlw+ibiFF3HCpJJQc0xOhM9frrblVW7JSpGaAU8oynwG3152NpbpJBzpSVCQI8ySdbcpaYSQpsz3dRl08NuW/n4WcMUeIgBeb6Up7enATvO8x4/X3W5XWBIhQVmJGkj468vlvYwpTYWEHs5BOp6chE+dglFOfIgIKpOZWWQT4wNPdaQYomRFV1UHMArKkDWNYnQx9eU2364gtlS+UaEgEmBrv9c7C5UhQDYQAAYypgADbQ+7624cQoFKkOazrJnpr+n5m0gwBRL7aoNT7LiSnKjTukBM6fU/py5X6stJiIImQ3O/u0+tbcvJcXK0rSUE5gc0Dr1j68rAO0j4UVpKErB0IPMg69Zm08qhxEIUsqCkpIBnSFEGdvyGnhzsEtDLqVBM6+ySomJ8J+XxtnY1bZWtVQkwCQpEj3dOR+fnYMpfy5VOI5zrpz1Gn1NnSJutBppMErSYEaD37b/1+IZQUAwsnKAMp03nTrYYiqOZBQ0FIESdpiOW/KwKwpCftWExMKTlMn468vlZhYJkCtxzMYdCiPEA7RMxvv4WCJyHs8+p0235zM9eU/GLGF1DSCEPUwCRoDsPOAI+EWCS8wrLKXVyiAUpMRHXbpp/Wz5gFEtKLOOEBZIWoZYIDJBPjvG3KwrK+8QUpgakwZ6g6672MpaSoAMB0IgElaD3p5z4WOU91rWnK0jInYBQiCT5T9RaOcAJ+ESi4LahmcYbzHVWnO2WWE3UnKPs6c6b5x+dss3ECfhFemDOCcMU7aW6a4ruZQkylKKRAAPkBY81c9G0kJS02ANgGwBbLZbysDmugzuWlUbZVAJSPC2l3a1spaiCLZbLFawHdFD3DYoBVOw0fYKvM20420kBRRO9stlpZQEZrid0A8hLeoSkgQdRbRcPZ5ghG8bWy2WgNCjt1tdJN8Xg5QUD1chtKi2kkJ2+doGxD65iWt9dvWvcWZKUoQkJSnyHutlssamF7kqyNkjN4epFvoQpxw5yATpznWwbt0UalZ1hZCQAlIV7I+vnNstlroUCuU3PQ1YVTPtl1CiEqDpzhW2457/Kwjty3RQVLtNRXZStBhZaBDKZOWNdt5M2y2WsROIGhQXtB3C6ZaOU9kUtgJBgJ01/rYaoazuoZmOzbSJA3mSfvtlssYOJ5oeUDkglICVBKiVSqN42BV/02FS029QNqKT3XlpTJkjQSfPS2WyxWqFkCmlRCtRokKHdH80fh87HKKjS6p1hSzl7FatBr3UlQ/8ADHvNstln5qKKu3ZTJUVkEgTI66n3cultUdOyxWJep0BK2pOsKCtNiOYiR77ZbLTCbmlG88NUdM+2thWRt9tLyUBPsBU92eYkf15kDhmjfLrSllJVELQkBSTMZgeotlss5UVGfE3BV1G4l42bKkVbDxp6pEAipCHuzCp/hXporXTQ5tCImcbp00yCadJLqdY00k7dNRMjrbLZbSpdWC6zanR+iIu3hTtU7MUeZNQ6lEKUk5SQVBRlOvsj36+FiSMQPs0jaktznqOxlRGYabzGuw0+62Wy10AKmXFGW78qX3mWlNt5ylHfKZ1UAdvqbEay/Lwp+wUFpyvgaJSEkBRPPYxlHK2WyxA0KDnFAu3/AHmahspqD2iVwFlKTptG22vPpYVmtvAVZoqmvdeStsOjXKBJiIHv+XSbZbLEDRZBLjddPdrTutvO1Dj32yGgCYgKSNZGvP5DxkOqU+h5omqdJdkI7x7iYBg9efTe2Wy0g0KNyiqAXqVpxMI7RAMCYB113125/pbpqrWhtmkqJcU64W86CG4iNSANfKbZbLPYJXKMIYaU6WinKSgjM3CfZ0H4W5ao1J0D6ozaAgHYkc9eXW2Wy0VFxKEVdrfbJSh1beUTKAAdCTE+87zvYuFR30rcIUsAZlSRKtRIAEeEWy2Weya5XCXKZDgacpyo1BDeYKAgDroZ38PnYneF4ponlqQ0tbhVBUtwGYT3dI1iOetstlpBMiDd8orVIUKFKFwO8VzG+2gj2jtZzXfdabyaCw4lkAxCWknqJncnTckna2WyzlRRm9sIquui/eD9e3UpXWIQEBgoIKp1nMQdxpHLygRm7BTkoafUC04jKYE7kfh87ZbLEbsou3UOfudqjxIwpDplFUlKyEiVy5BMGRyO4I185sXRcNcLVlR6oll5ChMkqTljLmgBKUn4kxPutlstUYd1ZIFlqqwFhy66pLCGKlRTupNa+gK15pC4P9etu2cLYdcZW45dTToBHdfJemT/AIyefSLZbLEuVEALlFw4SaUhJwtQBSnEolDLYGugJlBnfr12k2VH8PUN00yapimpBmyyEUyU848Rsek2y2WiN1MgLlla3m0qKjKBIkyJOm2x2G/TSLNN9S1XmrKoJ7VQTokaAkEbDkeke62WyxYt0OXqmnjTiJeVx3mMK0FIh1SkOPGoq3VuDRShAQkpj2eR2jpaPl8RXXXqti8rnadcpCUFdO6WkuAAH2VBcbxv162y2WHL7yND7icHDTiWbyv391MXMacBttXaGpzEkzEgIExB8dRqItY0pacAUtKoypV3VQZO/KPlbLZYbt1Jmy6FGlaA+lxaYbC45klAO/kQPdbWVxGqXlSO6CQDz8bZbLMnXSW1HtBn0B7o1EaeBH0Lbapg+o5XXEwspMkKBgnr5Wy2WSa5utuULjOVfrMhPRMHrvNiK3nUuDOsuELCDm1BM7m2Wy0gE91yupcStCFZVaADSBHl1sJSvl1bVOJCnAO+YMEgcotlsskgjd50TtNFQakKOYAAIIiVKHU8k/OyahxaiSFxBSmMoIPyn52y2WdJcuZ1uKSVDuiZj422ptJBJkkqMTEaAfr8bZbLJILHLuUlDjhfBAAhOTSJ23sVLUuKSCkajZO5m2WyzFMgkMdsla82UJ7sRJJBiZNtmmGVBSqAZ0InYn8vnbLZZX0SQakONFQS6QE6R74sNTtuuICy/MxlCkggDeOhtlstFPYI4i7FK/vXkrEbdnHQ8j42E/c1PGVxxZjQ5DEyJ5z0i2WyzOJT2C6YuGjqELWlx9OUjRTmaZ+Fjhw3SUrgQFlWROhjx5yTNstlhucUVrQtJudhTgADQUteSS0DBjewT9MhlGdpKE8kiDoQTrvbLZYYcbqQaEWSiUoeSopzlWmmgG2vXxsNT0frLvZLWkCSkdwHafytlss9ynaAumaCmeLjamkDTMTlmZIH5fCwzV20iSFKSqCf4SAdQDrIPW2WywwSpWF0IugpMxlmTO5O9stlstJPZf/Z" + } + ] + } + ] }, { "language": "markdown", @@ -88,7 +82,7 @@ { "language": "typescript", "source": [ - "\n// Utility function to get the coordinates of the maximum value in a tensor\nfunction getCoords(tensor) {\n const { values, indices } = tf.topk(tensor.flatten(), 1);\n const [maxVal] = values.dataSync();\n const [maxIndex] = indices.dataSync();\n const x = maxIndex % tensor.shape[1];\n const y = Math.floor(maxIndex / tensor.shape[1]);\n\n return { x, y, confidence: maxVal };\n}\n\n// Draw keypoints on the input image\nfunction drawKeypoints(image, keypoints) {\n keypoints.forEach(keypoint => {\n if (keypoint.confidence > THRESHOLD) {\n cv.circle(image,\n new cv.Point(keypoint.x, keypoint.y),\n 2,\n COLOR,\n 3,\n );\n }\n });\n}\n\n// Draw skeleton (lines between keypoints) on the input image\nfunction drawSkeleton(image, keypoints) {\n const skeleton = [\n [1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9], [9, 10],\n [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16], [0, 15], [15, 17],\n ];\n\n skeleton.forEach(([start, end]) => {\n const startPoint = keypoints[start];\n const endPoint = keypoints[end];\n\n if (startPoint.confidence > THRESHOLD\n && endPoint.confidence > THRESHOLD) {\n cv.line(image,\n new cv.Point(startPoint.x, startPoint.y),\n new cv.Point(endPoint.x, endPoint.y),\n COLOR,\n 2,\n );\n }\n });\n}\n\nfunction toTFTensor(ovTensor) {\n return tf.tensor(ovTensor.data, ovTensor.getShape());\n}" + "// Utility function to get the coordinates of the maximum value in a tensor\nfunction getCoords(tensor) {\n const { values, indices } = tf.topk(tensor.flatten(), 1);\n const [maxVal] = values.dataSync();\n const [maxIndex] = indices.dataSync();\n const x = maxIndex % tensor.shape[1];\n const y = Math.floor(maxIndex / tensor.shape[1]);\n\n return { x, y, confidence: maxVal };\n}\n\n// Draw keypoints on the input image\nfunction drawKeypoints(image, keypoints) {\n let modifiedImage = image;\n\n keypoints.forEach(keypoint => {\n if (keypoint.confidence < THRESHOLD) return;\n\n modifiedImage = modifiedImage.drawCircle(\n keypoint.x,\n keypoint.y,\n 4,\n { color: COLOR, width: 3 },\n );\n });\n\n return modifiedImage;\n}\n\n// Draw skeleton (lines between keypoints) on the input image\nfunction drawSkeleton(image, keypoints) {\n const skeleton = [\n [1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9], [9, 10],\n [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16], [0, 15], [15, 17],\n ];\n\n let modifiedImage = image;\n skeleton.forEach(([start, end]) => {\n const startPoint = keypoints[start];\n const endPoint = keypoints[end];\n\n if (startPoint.confidence < THRESHOLD\n || endPoint.confidence < THRESHOLD) return;\n\n modifiedImage = modifiedImage.drawLine(\n startPoint.x,\n startPoint.y,\n endPoint.x,\n endPoint.y,\n { color: COLOR, width: 3 },\n );\n });\n\n return modifiedImage;\n}\n\nfunction toTFTensor(ovTensor) {\n return tf.tensor(ovTensor.data, ovTensor.getShape());\n}" ], "outputs": [] }, @@ -102,7 +96,7 @@ { "language": "typescript", "source": [ - "const tensorData = new Float32Array(inputImage);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, tensorData);\n\nconst inferRequest = compiledModel.createInferRequest();\ninferRequest.setInputTensor(tensor);\nconst outputs = inferRequest.infer();\n\nconst heatmaps = outputs[heatmapsOutputKey];" + "const inferRequest = compiledModel.createInferRequest();\nconst outputs = await inferRequest.inferAsync([tensor]);\n\nconst heatmaps = outputs[heatmapsOutputKey];" ], "outputs": [] }, @@ -116,18 +110,30 @@ { "language": "typescript", "source": [ - "try {\n const outputHeatmapShape = heatmaps.getShape();\n\n // Define constants\n const numKeypoints = 18;\n const heatmapWidth = outputHeatmapShape[3];\n const heatmapHeight = outputHeatmapShape[2];\n\n // Extract keypoints from heatmaps\n let keypoints = [];\n const heatmapsTFTensors = toTFTensor(heatmaps);\n const xCoef = originalWidth / heatmapWidth;\n const yCoef = originalHeight / heatmapHeight;\n\n for (let i = 0; i < numKeypoints; i++) {\n const heatmap = heatmapsTFTensors.slice(\n [0, i, 0, 0],\n [1, 1, heatmapHeight, heatmapWidth]).squeeze();\n\n const { x, y, confidence } = getCoords(heatmap);\n\n keypoints.push({\n x: x * xCoef,\n y: y * yCoef,\n confidence,\n });\n }\n\n drawKeypoints(originalImage, keypoints);\n drawSkeleton(originalImage, keypoints);\n\n displayArrayAsImage(originalImage.data, originalWidth, originalHeight, display);\n} catch(e) {\n console.log(e);\n}" + "try {\n const [N, C, heatmapHeight, heatmapWidth] = heatmaps.getShape();\n\n // Define constants\n const numKeypoints = 18;\n\n // Extract keypoints from heatmaps\n let keypoints = [];\n const heatmapsTFTensors = toTFTensor(heatmaps);\n const xCoef = img.width / heatmapWidth;\n const yCoef = img.height / heatmapHeight;\n\n for (let i = 0; i < numKeypoints; i++) {\n const heatmap = heatmapsTFTensors.slice(\n [0, i, 0, 0],\n [1, 1, heatmapHeight, heatmapWidth]).squeeze();\n\n const { x, y, confidence } = getCoords(heatmap);\n\n keypoints.push({\n x: x * xCoef,\n y: y * yCoef,\n confidence,\n });\n }\n\n let imgWithKeypoints = drawKeypoints(img, keypoints);\n imgWithKeypoints = drawSkeleton(imgWithKeypoints, keypoints);\n\n imgWithKeypoints.display(display);\n} catch(e) {\n console.log(e);\n}" ], "outputs": [ + { + "items": [ + { + "mime": "application/vnd.code.notebook.stderr", + "value": [ + "(node:3778163) [DEP0051] DeprecationWarning: The `util.isNullOrUndefined` API is deprecated. Please use `arg === null || arg === undefined` instead.", + "(Use `node --trace-deprecation ...` to show where the warning was created)", + "" + ] + } + ] + }, { "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAIFArIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1SXSkkG1ihGQehHQ59a429+D3h+9klkY3SNKxZjHcY5Jz/Epr0RvvfjSgVEcPSg/djYp1pyWrPJpvgbpjKVh1G/j/AN5kf+grPufgXMYysGtvj/btR/R69rFLVciFzM8Cn+CetpjytRtHx/fSRc/kpqhL8JfE0RYqLSX/AHJtv/oQFfRtLS9mh8x8yP8ADjxXGrE6cGPolzE38nqtJ4M8TQld+i6gRnkrAzD81zX1IUVuqg/hTDbQHrEn/fNL2Ycx8py6VqVqx+0Wd3F/11gZcfmKgG4ZBbnmvrMWsI6KV/3SRUU2mWlwCJYhIPR/m/nS9mw5j5W+fAzxTwW3AjJAFfS03hHQpx8+l2Z9zbRn/wBlqjJ8PfDshz/Z1uPopX/0Eip9mylJHzZEzYI3H7x6/Wp0ZiseM/d6173L8KfDjA+XZhDnOUmkH82NUZfg/pDABJLpAOBtuAR+sdSqbTLc00eJbjuY8kdOKYWbHPPocV7DN8Gockw6lcL7NGj/ANVrOn+Dd8ABDqStj/npBj+TGqsyLnmJPI5FKCSSNxrvpvhJr0Zys9k49MyD/wBkqhN8MvE0IbZaW8mf7twgH/jxFFmByO9gn3j+PanAuGAyfxrem8DeJIcZ0edsd49sn/oJNUpPD2tW5Jm0nUEAHVrZwP5UrDuZ4JYnp+VLuPHofSnNBNEAJUdGzjDLjrQFJYZ5xUlCgnOOfes+d3XU4scHA/rWiAc5x7Vm3fF3EQM57596mQ0aMUm6PrnnAp2/AznNRwHbFv754pxOAFweaUdimBftkY9qCxAz+tAIJPtTSV25I6+tUSOJbOc03cc8nHelbHTn3o25Ht069qkYmfu5yD2oyScnj60u05xyTTgpB49KVxgCRTWO5Dj09afg5HAye1KVODnnIzUsZl2TsplHPbI/OtLYjYbbgHms2Bis0q4/WteHJgTjgipe41sQ+SpxlTSGAf3m9OtXAoAzwKUqMcLmi4WKX2f+EMffNM+ynPOCO2RWhsB4oCZPFLmHYoCyLAEqhP5UNZnHCd+gNagj9BSFc5qeYfKYxtW7Ix9/Wo2tGByQ6jPcVulMDnjHrTGjOwnB6Ucwcpzo3K/r2qdYtyAnAPrSTrtkPHetSwVXtACoPJB4zQ2CRlPZAIWaJD34ArF1C12xl0hC8/wjFdfdJHgoFGO+KxL6P/RZMdAc/jW1KTTM5o5bawPANHmOP4m/OrxT3PPrUezII6/hXZzHPYq+Y/8AePFHmHvU5jGM4H5Unlgj7uPei6EQbzS7+MU8xjHIIpjIAuRmnoGopJ4JzzTd555609uY09gR+v8A9eo9poAXfil8w+lNwaTB9KYXJFcA8jinmYYwM49ahwfQ0lKwXJd4IxRuFRUUWAc/NNoopiHI5SRXBwQcg19uaPffb9EsLwHInt45fzUGviGvrv4X3pv/AIa6HKTkrAYj/wAAYr/SuLGLRM2o7nXlzSbzSUhrhOiyF3GjcaSkqh2FViHYZ9DT9x9ajHDj3Bp1ddPWJEh24+tG4+tJRViF3H1o3NTaKYh28+tQzMd8bZ74qWopwfLyOxzQxoXc3rSb29aTtSGkAu808OcVHTl6VLQ0O3tVS+lmjsriUT+XsQsCEzjAz+NWGOB0yT0FVLyyF3aTQzOxEileOAv0Hf8AGpavoPY46TxpbC2huJLdpZJNrBSqgpn+LO44+nB9ap6ze6fKJQ11cm7JCNGMny/baOAT6MQcVn3ekwXlhd2YmZLzT2MfmNgkqBz379f0riFvtkUNs06tPBJhcAFVABODkc4PvjNY1KV4tGTqNM6z7bpwHK3ue+JUH6eXRVJfFjBQHS4LAckAdfzorx/Y1OxtzRPdG+9Sjmkb71KK+wZ56HClpBxSikMWlpOtLQAtFFFABS0lLQAUtJS0hhS0lFAC0UlFAC0daSloAaUQ9VU/UU3yIv8Anmn4CpKM0ARNbRMMFePTJqnNoOlXBzLYW0me7xK38xWhmilZDuYM3gvQJs7tKsx9LdB/ICsy5+GHha5YM+mRhh0KvIv8mrsaM1PKg5mcE/wm8P8APlC4i9kmOP8Ax4GqM/we09jmC/uo+ONzK3/sor0uijkQ+ZnkkvwclCsI9WYk+tuP/i6ozfCHVhjy722YL03q4/kDXtNLS9mh87PBp/hb4hiDlBaSn/ZlIz/30BWfL8P/ABFABnTVZRyds8bfpur6JpCARyAfrUukhqoz5sfwrryKXOjXzDtshZv5Cqcuk39vgT2V1F674WX+Yr6aWGIoAY0/75FH2aH+4B9Dip9iUqh8ulcM249PejYdpwck9K+n5LG3l4ZC31YmqUvhvSJ/9bYWzH1aBG/mtQ6LGqiPlfaUu2XuRxWxaZNqgPQZ9s179N8PvDM7bn0m13eqx7f/AEHFVW+Gnh/btithGvoryZ/VjUujIpVInig+7nH6U4RbgF5+tevy/CzSWOY5bpcdB5q4/wDQDVRvhZEhJivpRngZQNj9RWbpTLVSJ5gIULHIGKcsKjr1Poa9Gl+GNyMeXfBsf34sfyY1Sm+HeqpuKyWzHtw//wATWbpz7FqUe5w/lgtxkbR1PemmI468n1FdbJ4E1tMYto3A6kSqP5kVTfwrrEWSdNnJ7BBu/lmo5Zdh3RzhhOSBj3OKYyN5fbn3rcl0TUYiA9heIAOS0LD+lUJbd4j86upHQEYNLUehyd8hVyCBnOTg1a06YrasOpDcUmooVDjHzNyah075oXHuK0SuiHuWXOWz3PrWbeoDDKD0A61pEHGe9Urpcq4xwVPvWkCJHOsPTnHvmoOTxjg1YJwABn+dQHBFdaOdjW4zSMBxnGfrS56kHnofemknjI4NMBDjpj3pj/dbFOOcd6aeQetUIQH/AEccfdfr9R/9al4OB1zSIMwSD0INIMkUwDC0YwcGjNHX+tABjNBUDijnOCOKPrQAmP8AOKQrTs+2fxpD7UCGkYptOI4pKYhK+mfgNffafh89uWybW8kQD0BAb+ZNfM1e7fs73vya7Yk94plH/fQP9K58Ur0zSk/ePcTRmg0leYdaFopBS1QCHgqfenU1/u/TmnV1UtiJC0UUVqSFJS0lAC01xlSPUU6kNOwEKcoPpSmmowO5RjKsQfanUgEpy96bS5xnHWpY0OAJOAOaazwhXDSKMe/Q1Onkqu+V05H8R4rAvNathqzad5roWXeFTjI756n3wB9cVlKryK493Y4DxmZYfE32/TpI0VodkrSKYwTyMFiAOmfeuJtNAxOupahf29naysfKUtjzeecd8e4HX1r1XxBq0ElosEttJNG2GSJISiJ6HLLtyOvXFeWXfh+91L95d3iR7mLCWc+X8o+p549K5Z13JWbsS4JPa52McPhJIkUMjAKACUbJ/wDHxRXMx2Nokao19bFlABO1+f0org9k/wCdl83kfQjfepRQ33qUV9azgQZqB3xubLAD0NTmqsoysi9MgimhM5rR/iJoOtXkVrbXM8c8pwiTxFdx9M8jNdYsrfxVyFl4Ksra4gnJV2iZXDBcHI5Fav8Aawl1k2UaSjyk3SM6FQSTwBnr0PI4qrCub4bIpjTbTgDOOtNQ/JWL4mjefw5eQRsVaZRHkHnDMAf0JpWHc2I71JRlGR1BwSjA4/KpxICK878H+HH0LVhLC7LDMhSSPPB4yD9cj9a7rzNsuz2zQ4gmXCwUZPSm+cn979Kikb93+IrivHGoa9YT6edFuFjDLIZUaJWD4246jjv09aXKO53gdW6MDTs1kafcyS6da3NwoSWSJGdRxgkDI/M1oq2RS5Q5iajNY+p3rWOm3l6kLTNBG8giU4L7QTgH8K5Sz+IbXLBToWoqx4wh3Y/QUcoJnoeaM1BHKXUHkcd6jedgXO7Cr7e2aVh3LeaXNctpXjfQ9ZuUt7DVI5Zn+7G0bIW78bgK6JJtw5FFguTZoqrcXiQFAxA3ZxlsU0XoI+4SPY5osFy5miq8dysjbQrD61MDmkMdmikprSKgy7BR6k4pAPJozTQ6sMggj1Bpc0ALmjNJRQAi9KdmmL3HvTu1AC5opKBQAtGaSigY7NGaaKKkB2aM0lFAC5oIB4IBptLmgaG+TEefLT/vmka3iYYK8fU0+ilZFXZRn0LS7nPn2FtJnr5kStn8xWe/gbw1Jn/iT2a567IVX/0ECt+ilyod2clP8M/DE2SLDYfVJZB/7NWZc/B7w/OSVlvIif7kw/qpr0HNGaXJEOZnkNx8A9MbP2fVr2Pjjfsf+i1j3P7PtwB/o+u5P+3a4H6Oa93BpQafKI+b7j4CeJEyYdQ0+QDpkyqf/QDWRc/BfxlADts7ab0KXKr/AOhYr6ozQDRqGh8gXHwy8Y2+d+g3D/8AXFkk/wDQSazbjwb4mtgTN4e1VAO5s5MfyxX2iQrdQD9aZ5EP/PJP++RRdhZHw01ldW4lWe2miO3pJGV7+9VsHFfdjWsLjDLkem44qjceG9Gus+fplpLnr5kCv/MGjmYcp8RYPegn2r7IuPhx4RuQfM0CwGf7luif+ggVk3HwY8E3GT/ZCRk/3JpV/wDZqOfyFynybRgnpX1BcfAXwlL/AKoXkP8AuXH/AMUGrJuf2etJYk22p30fpvKP/wCyrR7RBynzqBxRtHrXulx+zzOMm31sn/rpaj+kn9KyLn4B+Ios+TfWEo7BhIp/RSKPaIfKzyEgd6bjmvSbr4KeMof9Xa2s3+5chf8A0PFY9x8LfGtqcvoUzjP/ACykjk/9BY01JdxcrONr1T4CXnkeO57fOBcWTrj1IZW/oa4m58FeKLQnzvDuqKB3+yOR+YFb3wuF3pHxL0d7m3mgDyNCfMjK/eUqOvuRU1bODCCakj6rNJRmivKO0KUUlFUgHEZBHrQpyoPtQOaRD8pHoSK6KREh1FFFbkBRRRQAUhpaKoRABiVwB15pTSP/AK5T6jFKeaQXCpYIfMfcQMCohy6oOp6CtHb5UOEGSBx7moaQN2KVxFDcMftTKtspA2lsBmPr/LFYWvaTpcNnNIq/Yw/ytMkrRFs9RwCWPTjHNdO22PBIzjpxk5qte2EeoweVeIGhYg+Vng49cdfp0rmnTTTTV2wUtTztS06Yim1S6hQZBkUspwD91SCQePY4471xt61ld6sls9/fM9zKUaJ48FRxk4GcDj0H0Net61cvJbxw6bGQkbKA6LkEZwVXHt3OB2z6ec+JdRG26ewnghuWg2yrb4Z1IBXBYf7wIxjPNebOnyysnc6FrG5lyrp6Sui+C9YKqxAy0o4+gTiiuf8AsuunmJ7jy/4NzDOO2eaKfso9/wAWY83kfSzfeopWHzUCvqGcaEqhfc2dyPWNv5VoEVy2sf2+uoPHavbNZS4GHU7lBGDz+dNNLclps47w14XvdG1Ozube9m8oOokjL8MvQg/hXos4Xz4iRzg4/SmLZPHZsYwDKFyoPTPasnTtR1a/vhDe6WluiAnzFkzz6Y/z0qrruKzOoT/V/hVLURv0/jn5kP6ir0akJiufvNch0mZrXU4pVhOfLnRCylfQ45BFCYmjRt1CtGf89DSu+dQwOyD+ZrGbxj4fgj3LemRv4USNsn25FXNGlmvw17JE0QmbcqN1Vegz74FDYJM2W/1Y+o/nXLeLvFn/AAjU9kraW16k4csyvgx7dvbBznP6V013LHbWrTTOqRpgszHAArDvjo+ryRtJqFq3lghQJFPX8fakMsI0et2VlqcE0yxFRKkecBsj+IeorXgY7BmsJtS0/Tra3061lSSRiI40jYEgdycdBjNbdvkoCaAK19JbxaddS3jFbZI3aVsE4QZJ6c9K56y17wk7KLXUItzEBQVYZP4iuiu4Rc2NzbkZ8xHTB75yKw7DwpZWsyytaxBgQQdoNSyonURdOOlQNz5wPc/0FTxgBcZqEj5ph6n+lNCZydh4LtbHULa7GxZIHDgqOTiuit9QS4u5Io1cCI7WLoVyfbPbpzXn2ieH9S0fVbO4h1C4a28wB4mclSp4r0bA81DjnHWhgirr2h22t2qR3Me4oSUOcYJqroelJpmjS2SZCq8mOfXn+tdCvK1UAw8w/wBr+goA5jwnoUulXDStd3MoKbQskhIH+cV2ydKoWaAIvHar4FQWPzWVrhJswO24GtSszWRm0/EU0Jmdpl55TNESdrcj2NQWVy39qiQsxBds8/WsGTUJLfxX9gfiOS0WaFv9oMwYfltP51paezGVWcEHzW/9COP0pdB9TtI23Cn1XgOVqehgC9T9aWmjqfrS0gKV7qP2N0UR7yRk84xTkv0a0+0EEAAlhnOMVn6x/wAfCf7tZUl6p0+9Eb5CKwYZ6MBUt2nZ7FqN4XR01pqEd3uCoykAHnFWi3FYGnMY5MA9Vq/qFy0FhJIv3sYH1NODuhTSTJpNTtInKPOu4dQATj8qnhuIp03RSK49j0rzjWINSuYEGm3PkOCWY7c7vQVc8L392UtpLrInJ8uUevOP8DUptq5XKtjvy2BntQGBGR0rO1SF7vS54I2wzoQOcc1zWhagYZRu+XPySA9iP8/rVuNlchb2O2zzS1yj38tzqAKzSLHuAARsZFdIJQULZ496lO70KasTbqXNcjq3iRrWGS5VpBAnQRJuZqseH/Ea6pDDKrl4ZvulhhlPTBxU8w+VnT5ozVa5ult7aSZhkIM49ayz4gjTHmoqZ4GZMZ/SnJqO4RTlsbuaM1RsdSjvSyopBUZ6g1dBoTuNqw7NGabmqz6hbRyGN50Vh1BPSgRczS5qtDdQznEcqOfRWBqbPFG4EmaM1GGyKN4JxSC5JmjNNzgUA5HFJjH5o3UzNGaQyTdS7qjzRmgB+aSm5ozSAcaYQD1AP1ozSGiwxhijPPlr+VNMEZ7Y+hNSE0manlQ0yI2644LD/gVM8g9pX/HH+FTmk71Ps4voVzMgMcg6SL+K/wD16TbKP7h/MVOabS9lEfMyPMg/gH4NUMDzlpC0SqS2cbv8+lWqjHEp96FTS2DmuKGfun5Gjfj+Bvyp9Ga0sTcZ5i/7Q/4CaTzo8/fFSUUrMY3ep6MD+NLSFFPVQfwphgiJPyLz7U9RDLhtgVjnhhSb95wn4nsKiurQNbuI2ZWxwckgfhmpowBGoHTFGtw0Lttb+UN5+8euetWSQFLHPHoM1WS6UIA2S1Zlxrl7Zbml0tpY92FaCUNgf7QIBH4ZrKpUjBXkTyuT0Ndl/wCWjZHHTPSud13ULu6YWGnouBgyvISA690AHJHqR24zzVLVPE89xILNVS2YkHaWzIw9h2/I1nXGqSWtm4SK2V5mw5lO4tgdSTkn8a8ytipSaVPbudEKDSvIbeLMLSSK906TLZ8yaIrMPrsyFHAAA5xwPSuH1a10ay8iWzknMMpCtlcMG6gsDxjnjAHWuluNVuhF/o8toZGBACQqQD2OcYwPrXM3V3G9zuv2e9uRwCuUh/IDJz+B46HFYOTbLcUkc9Jq2rRSNGlpeuiEqrHaSQO+cUV1hksSTts/l7bYcj8Cc5/Oil7eP8pnys9qkHz0gp0v36QV9UcAYpjRhjzUgFLQBEEAGKYLdQ5YDmrGKMUANC4FV7i0juFKuoIPqKtUuKQGGug2aSb1gQHPUKK04YBGoAGBVjFGKSSRV2yC5to7m2eGVFdHGCpGQa5ibwXpjtn7Mg+gxXX0hANDVxJ2Ob0zwzZadL5kMCq/rjJ/Ot9EwMYqXaPSjFCVgepyWsaFqU9/LPZ6vdW6Pg+WrfKDjsKoLpPiSM8a7MR/tRA/zruygPUU3yl9KHfuNW7Fe1RxCokYs4ABbHX3rF1ldbjvwdOktxbuBuWRCSD3P8q6QLikaMN1qrvoTZGNJYuNPfywDKFygPTI6VQ0i81a8uil9ZxQIgyGV8kn0x6V1HljGKYIVVsgChyYWQiLha5/U9bj0zUWt5LS7kMgDK8UeV9MZz1rpAMVDNbJKwZhkii9h2RBZqwjUkdquimogXin4pXADWfqxAsyzEKARkk4ArQqrf2UV/ZyW06B43HINNMVjhtfsftaRXtqytcWmHQqc5x1H4g1csZRIIZV+62CK3NO0O102BoYIgqkkn3NZy+EbWPVReI0ow+8IGO3P0pNqw0nc3VmWGLe+ccCrUUyyLuXP41n6jYvdae8MMrQycFXXqCKbo1rc2tr5d1cvPJnJdgB+HFJtBY1R9406mj734UtAGPq4/exfQ1w0StY+LNQtpiWtNTUSJ6B1UKw/EYrtPEv2xLWOWythcSBsFCccVjT6Rc6lpEcssQivIyJUUHO1h2z9OKHG8tSoytE07D5WjGc4XGTVvVxnS5T6YP61h6Je3d1e+VLp00CoDudzxn6V0t3bm4sJouMshA+tKkrKzHVd3dGFpqKzsD6VRVBDfzKvAE2QPyNVbTWYrWdorlhb3MfyukvANWba4hvb/McqO7tuIQ5x0/wpS92PKwiry5jsYvmUA965XVrI2OsCSNf3NzksB2YV1cCkIM1m+IUUWSSHjbICfatY6qxD0dzO0uHzJQ2OF4rfuFIspcddh/lWToTLJHIVIOG7Vuuu6Jh6jFZU1Yuo7s5G3t0ukSJwCrDpUel+GZrDU1MV5ItmsnmLCqjHrjP1qXT5hG4z96JijD0PSrcl9cteP5T7YlwFwAc+tJrlvcpXlaxr6ou7Srgf9Mya5K/0u21e1ijukDKpDj2OMf1rsbkebZyr13If5Vx1zYnU9MjgEssJO074m2tx70VfhTCl8TRteGNMsdKieK0QK7HLnByfSukBrm/DOjtpiSM9zcXDvgZmkLYHtXSClHYJbinpmuK1BZC96sb7ZNzhWxnB7Gu0PSuSvlK6ldL/tA/mKqS/dsUH76Oe8J6hqkiTjUTm5tpsK4TbkY9v8816TLIfs7lDhtpI/KvNdE1m8utVuLG9tY4pYAHVkJwwz716HbSedaq/wDeXms4aS1Ln8JhaFcubiJ2dm3qQcnPP/6xUN1qt0ut3KiaRVikUBQ2BjHpTNJzHMsZ6xzMn61W1RfL8Q3I/wCeiI/6Y/pV2+NE9Ys7a4uPKtZZQM7ULY9eKydE1S4u5QJnUqybgAoHPFXoj5+mrn+OIfqK5zw/Jia2B9Cp/WlL4Uxx+Kx0OsamdPto3Cb2dwgBOAPepdMvjfWnmMgRg20gHPpWV4qUnSkkH/LOZTx+X9ak8PN+6mT0YH8//wBVE1ZJijrc381mtq0KzvGN7FDg4HGa0M/LXJSIDfXkJPBdge3WpnpByXQcdZJM6KDUYZHCbiGPQMMVdzXmemW13pHiCSxluppreSIPH5rFtv5/jXoMUpltA2fmK4J96mLd7SKaVrodFfW80zQpcRNKvVFcFh+FTO4UZrzfQJfJ8RW7HqWKnPuCK6HxrI40iMKSFaTDYPXg1t7P33Az5vd5jpt4K5oVgwryzTdfnsNIvbQSld67o2B5VsjOPqP5Vs+BL+aW7u4Zp5ZMoGUO5YDB7Z+oqHH3blX1sd2SAOabuB4zWB4o15tGslMSB55M7d3Qdsn864aDxlrcV3GJbjcJBlQ8ICt9MD+tS9ClqesUlUNG1RNW0yO6UbScq65+6w6j/PrV+gYlNI+cU6mv2PoaQDqKKKYBRRRQAlFHeloENYZBFV4STEPUcGrOKrRArLKuMfNkfjTAkpyp5iOnHTIzTaVZFiJdzhQDk1FSKcWmCdmczrFjbTTtDPDHKGw2xgDjtu59KwpdMmkVPsWozwjn5pFEowTnjdyB6YI/rXTyW7zT3N1Jvj3ttRHUZUADH884/Os2+SO006Rp1PlkEZfPzn0z71xWjK7asjVt2Ocmsv7Tu2ie4i+zxDcAifM6nuzZPU59B149eC1e3uG1C4gvH8i1DmOOR5yS0YPy5AywXgH0r0e3SG3sy8rwNcTN5kuWACH6e3QCvM/HumXt3rDraM0sD7JF2PldxGCT6nj9amFOCemhMm7XNmO+tUiRF12dVVQAqwMQB6A9xRXBR6VqKxqPtiLgAbc9Paio+rU/5yfavsfYUo+emAVhHxnpMp6zr9Y8/wAjUq+KtHPW4dfrC/8AhXte0h3MPZy7G1S4rLXxFpD4xfIM/wB5SP5ip11jTG6X9t+MgFHPHuLll2LtFQJf2Un3Lu3b6SKf61Mro33XU/Q1XMhWYuKWil7UBYSiloxQA2lxRS0AJRilooATFFLRQA3FFLRQAlFLiigBMUlLRQAlFLRSASiiimAm0UhUU6ikA3aKTYBT6KAGfx/hTqP4h9KWgBjIGHNMEQAxipqKAK4t1ViwAzUm3ipKTtQO5l32jWd9zcW8ch9WUGm2Wi2tif3ECRj/AGVArWoxSauNNoYq4FV7+xh1Cze3nTdG/UVbxRigRl6XpVvpcBht4wiZyfc1o44p+KMUkDZyuseGpZrxr3T7pradvvjbuVvwqPTdJ1FJ83t0sijoEj211u0dxSbF9KUlzblRlykAi/dlfbFcRLa+ILWRo4oLSSJThSSc4rv9tMaJW6im7tWCLSdzk9D/ALWfUUN1bRRQqCSVckn2xXXCmLCqnIFSgYqYqxTdw7Vx/iG5Sw1Eu8UzCRQcxoW6etdjioJrdJhhlB+tVfRoSWtzgLW7sp74SRRSm5Zdn+qYcZzz2/Ou4sozHaonoKEsIlbIQZq2se0YrOMWndmkpJqxwt3crpeu3ME58tZG82Jj0OaZdyLd6it0GA/deWQDnJzmuw1LR7TU4wtzCkmOhI5H0NZtn4V0+ynWWKAB16EknH51VSV7tdSYJdehpWCMlhEjfeCjIPauY0/9xqLJnGy5Yfhurs1TauKxm8O251Y3xaU5bfs3kLn1xSv7nKwXxXH+IY/M0O5Hoob8iDWfot0kDsXcBXUHJ4FdDdWwubOWA8CRCp/EVx7eG9WiOI9S+UDgGEGiclyoILVnZw3Ec4ISRGI7K2a5q+/d69MvZ1Vv0xVzQNOu7NpJLy5ErsNoCoFAFV/ENnffbY7qyijk+TYwdiO9CknB3DltJWKWqup1nSnUcsJFP5D/AOvXSWBzagelcvbLq0kyLNYxqP7/AJmcfpXW2sXlwqvoKxUnKafZGjjaLPO2/wBD8RnsI7on8N2a63xVF5ugyNjJjZW/XH9a5/xBp9ydfkEFvK5lYMpVeOnr9RXW6nC8+h3MZHzmE8D1xXZzL2ql6GCT5LHjt1J5NxDG33JdyH69R/I10ngqQw65GhP342T9M/0rPj0l9ZNwkKFmjXzEbHAcHpn35FS+HneHXbIqpLGQAqByM8H+tRNb2HF9zofHsZK2zdijD6YINc3O1ncaJpsTyulygc7kA4G4jv8AhXaeNLcyaSkwHEb/ADH0BGP54rzia1t7qxNvNH+9Rw0bdsZGRRJK0WNdUel+DXs1014LXzsowaRpcZYkdePpXS1w/gqdTeXMa9CgOPocf1rt6wi9DWQU1+VNOpD0IqibgDlQfalpqcoKdzQAUdqWjpTEJ3ooooASq7fLcj0Zf5VZqvOCJImGMbsH6GmxIfUKhnut54jjI2+7dz+HT86mpMVMlcaY7UYwVHpiuW1CFC0KswcbtwiTlnYDIA/me1dDqM94Y1W3sfNBX77ShRn07ntWBqaXVtaTzJEd/lgvtBfn0B7LnH6nmvOxNZJcsXZmsFpqc5cxNc6n5eoXKwwuP3cSnooHO5+3XOBj3NV7iKG8N55dhBcEhi08eCIsYAwO5PJ7dce9XdQgllVUuJTFBjmNdu8npgDJHvkHiua1BD9oZYdVEdogBEaPtJJGenU4GPXnp7eZUlOd1I1suhfSHSlRVaCYsAASZohn9KKpJ4USRFeTVpEdgCy5JwfSisLx/mFyPsPEjK3X9asx3Gep5qk+d2R1pyEe4r6B6gjUWQN0NShuKzVcqcfhVlJSO/HrUtFFnCE8ov5Uo2g8KAfpUav+NPDY6fzqRk6TyJysjr/usRUy6jeJ928uB/21b/GqeeKdk9setO7CyL6axqSdL24/GQmpU8QaqowLx/xVT/MVl5pe1Pnl3Fyx7GyvibVF6zq31jX/AAqZfFmojqIG+qH+hrB/xoyaftZ9yfZx7HSJ4vvMfNbwH6ZH9asL4ul/jskP0lI/pXKKec9qm6Ue3qLqHsYPodUvi1P4rJh9JM/0qVfFlr/FbTj6bT/WuRGcZpaf1mp3D6vA7FfFNgesdwPqo/xqRfEmmN1lkX6xn+lcVR1p/Wpk/VoHdDXtMb/l6A+qMP6VIur6c3S8h/FsVwOfejOKpYuXYX1aPc9DW/sm+7dwH6SCpFmif7sqN9GFecemT+dGPoaf1t9hfVV3PSuDS4rzUMVPBI/GpFuJ0+7PIPTDEVX1vyF9V8z0XFGK8+GoXq4xd3H/AH8P+NTLrGor0vJfxOf50/rcewvqz7nd4oxXELr+pr/y85+qL/hUq+JdRXq0TfVP8KpYqBP1aZ2WKK5JfFN8OsVuR/usP61IviycfetIz9HI/wAapYmmL6vM6cj5lpcVzB8WsWX/AEEH1/ff/Y1OviyH+O0kH+64P+FP6xT7kujPsdBRWIviqyY8wXA/BT/Wpl8S6cw5aVfrGf6VSrQ7i9lPsatFZy+INLJ/4+cfWNh/SpF1nTX6XkX4nH86ftIdxckuxdoquuo2LH5by3P/AG1H+NSrcQP92aM/RxT5l3DlY+jFAIPQg/Q0uKLisJRS4ooEJRS0YoASjFLRQNISilxRSKEoxTqSgAxS0UUAGKTFOooAbRilooATFNKA9afRSAYEAoZA3UU+ilYoh8lc5xTwuKdRilZBcieJWYEjkUuwbcVJSGiw7lZLVIydqgZ9KiTT4IpzNHEiu3Ugcmr1JS5UO7IJoEmhaKRQyMMMpGQRXNyeDNP84siOq/3A5xXVdqTFDV1YE7GJovh+z0YH7NHgnjJJJxWzTsUmKSVgbuJRS0VQhkf8Q9DT6YvErD1p9ABRRRQAUlFFNCYtQzjMRI7HNS0xhuUj1FOwhOvSkpIzmNfpS0hkkm46c+z74zt9q5zUkeSBAkpaISK7KuP3mPcmt+EGSJ/NH+0qen196yNQVVkPHI5rgq0HObkawkkrHP6pbi7vLXfbr5WC0mWAbjGATyQOvTr7DmsvV2mW2lng0jTxLEhMcjMG2bQcHbt9ulXri9XlLaMzSDJZYiAFPU5J4H8/aqoSRlZ7psDPEUXOc9jnr+lS8LC3NLVh7R3sjzseKYgAH1e93j7224bGfaitN9ABdi/iLT42JOUEcQCn044/KiuPlo/1/wAMV7xqMvzHjjNKq8D2qWUKjfMQp9+KRMEfeH4Gu1O5tawBcds08D2pwX0PPtQFHai4xVYrwf0qVG6ZFRjHY5oXOD29c0gLKv8ApT8gjqeKqhj1qQMRQBY68j60dOpqMMP608HjFIB3b1pcc00EZxml/KgBR1xmp1quODU65xUtlIdTuKTtSfQVJQppM+1LgikI+nFFwDr/APXoA4IxQB+VLxn1ouFgB/lR68UdOv8AKii4Bx1o+tL+dJQAAUdaWk6jigLC5pD796XHSkI9utAWA4oz9KM80h64/OmFgAzS0DrSY4oFYXg5ozikx15ozQOwuaAfek70Z4pALn0pcAj/AOtTc460E0wsO4AyB+VPWaVfuyuv0Yioh7UvGKLsVkWBf3aH5bqcD/rq3+NTLq+oJ0vJvxbP86o+9B47U+eXcThF9DTXXtTX/l6J+qL/AIVKviPUV6yRt9Yx/SsjNGfXFP2s+4vZw7G4vie/HWO3P/AD/jUi+Krkfet4T9CRXPg0ucVXtqncXsYdjpB4rb+KzH4S/wD1qlTxVCfvWrj6MDXLZPFGfen9Yqdxewg+h1y+J7M9Yp1/Af41MviLTz1aRfqn+FcYDS7gQaf1mYvq8Dt117TW/wCXjH1Rv8KlXVrBul3H+JxXB55pQ1P63PsL6rHud8NQsj0vIP8Av4KkW4gf7s0bfRhXn27HNLuprFvsL6qu56ICD0IP0pcV50GA6cfpUi3MqfdmkX6Mar655C+q+Z6DRXBpqV2vS6m4/wBs1MusXwHF0/480/rcewvqsu521Fccmu3463Gfqq/4VKPEV6OpjP1Wn9agL6vM6uiuYXxJcjrHEfwP+NSr4lk/it1P0OKpYmmL2EzocUVhL4lU9bb8n/8ArVIviOA9YJPwINH1in3J9jPsbFJWWPENmeqyj/gI/wAaeNcsCf8AWMPqpqvawfUXs5roaNJVIavYN0uB/wB8n/CnjU7JulzH+JxT9pHuHJLsWaKgF/Zv926hP0kFSCaJvuyIfowp8yFZjqKOvTmiqEMPEoPqMU+o5DgoffFSUAFJRRTEFFFIaYhTTaWkpiIk/iHoxp1J0lb3ANKMA5IzUybSutRixnD/AFrDvpLa+kePzFfaNssOcEH39q2toLlskMfeuW8Q+D4tZnF3a3j2t0DnzFBKntxgjB/OvHr46pTnapCyNVFWuiu0UMPyRRiKIHCoOiiucurvUILhrgaUWmU+UCLjeqjqTjAI9z06c9M9RFo93pdr+/upNSYAkcJGemMZOc8/z9gKqpbz3N7uNobdowMx4QjHqWA6fQjt61jPHqs+Wm7JFxhbVnFyWdh5jeZOd+Tuz5fXv/HRXXNpeiFiZbW8aQn5j9mfk9+9FcXtX3NOVHJ/GFiPD1hjOTdZwP8AcP8AjXm+gs7MnzNjcDjcfWvRPjGcaPpyc8zk/kv/ANevP/DwysLHHLf1r38vV6KuRjH+8Z1S7xjDuD7NTw8+B+9k/wC+jT0Xr3qYKMZP1r0uVdjhUmMFxdL/AMvEv4nNOF3eD/lu2fcCn7AOMUu3Bpckew+eXcFvrw/8tAT7qKkGo3ag8ofwpNn+c0nlg0vZQ7B7WfclGr3IHKRn8/8AGpF1qbIHkr+Bqt5YPB/lR5Yo9hT7D9tPuXhrj9Db9uzf/Wp415R1t3yPcVn7MCmtEP8AJqfq1PsP6xPuWpvF9lbShJoLnkZyqqR/OtGLxDaPGrbZhkZGVH+Nefa6Nl6mOPkGPzNdBaR7raLPdB/Ko+qU2y/rM0jphr1l6yf98U9dbsT/AMtWHsUP+Fc95IPAFO8n/wCtxR9Sph9bmdENWsW6XCj6jFPGpWbEYuovxYVzPkgcAUnkj86n6jHuNYyXY6tby2I4uIz77xUgliPSRf8AvoVyH2cE5Iqvd2w8jkAc1DwK7lrFvsdyCCOoNKO3tXk/2chbwKxHOTg47Ukf2q2W2Y3U65yWxK3fn1rP6m+5f1pdj1rjjtRjmvJYLvVQsxOoXgLJvUee3HJ96emp61HpzuNUujIVLZaQnHtzS+py7j+tR7Hq9GPWvGx4x8QRGMf2m7fL0KIcn8q0pvGWtrKscVyCAwDFolOfyFS8JMr6zA9S7c0Y+ua4JfFWrqPmMB9zGR/I1XXxzrPnmP7NZtlyB8rDA9T81J4WoNYmB6LTRXFSeM71Uz9kgOOT8xGaLTx3NPEXfTFTnGBPn/2Wl9Wqdh/WKb6na44pOc1yEXj2OW4aP+zZQq9WWUH+gqSDx5ZSytE1ldqy8kjaQf1qfYVOw/bQ7nWGk/SueXxnpxODDdL/AMAU/wDs1IfHGiK215JlwcZMJP8AKk6U10H7WD6nRdTQSeawIvGugTPtW9bcemYH/wAKl/4SvQhw2pwqf9oEfzFL2c+xXPHubXOeKO1ZqeIdGkj3rqdptHUmUAVLHq+myjdHqNqwPcTKf60uWXYfNHuXcn9aUVXF5ayHCXELewkBqVXVujA/Q1NmO6Hg0Umc8nNGcA0WGLnJozmm5zS5Hf8AWgBc8Z70tNzkf40E9qQxaBSE0ZoAcDg0pJxTM/Wkz7GgCTJyaN3FR7uODSk0ASE88Um7mo91NLf407CJt3/1qaXqIuKaxAosFydX5qQPVMPUgfiiwXLO/rQH75/Wq5fApC5pWC5a3+9AfjFVfMwO9HmHHNFguWw9L5nHPNUxL604S4o5R3LRkpnmc+9QGTjmmGUZosFy6JPfip45PlPSs5JM+tTROefTFKwFvzKaWX0B/CoN/uaQygDPX1ouFiwHA5AA+lI1y6naJZsnoqOwJ/I1Wjd5z8gwv989Pw9auwW6J7serHqaLt7CsluXdMW6ku1kluZiq5xGZCynPrmuoByBWHaLsGa2YjmNfbivSoRajqcNZ3kPpM0UldFjEXNGaSigQUtJRRa4GVJb31tcloJfNhYnCNjK+w9RUcuqXFs4S4te3LDI/ofbvWrL0U+hpc15ssFVhJujUa8nqjRTT3RiHxTZIxVo3Axyw5OfpVlvFWjzCILOVZhwNvtnHHetBrK1mRmnt4nDdQUGTUcmi6dJZ7ltEUGTzCF4+bkZ4+prmksSk+azt1KTjdWMo6vpu8oL1JWZSfLQ5475J44p8enw3YaZIVWZjtIB2jA6fXHPbvVS88M6WINzvLDsAJkD4Ax356VlXFjfW1havpFzdCMF1YOCrTbjhccccA85Hr714VWHNJNKx0XaRdmtrOKaSP8AtdItrFfLEiAJg9MbeMUVCuhakVGLfTU4+60SMR7E45PvRWXspdwuzz/4xOBZ6UvTMsn/AKCtcRoCfLBj/np/n+ddh8ZWITSFz/FMf/QK5Pw8Plt88gsD9Oa+wwC/cxMsX/EZ1qIOp65qUDnPr1oQZ6j61IBxmvTOEZg5NOCj0p23HWnY9RwaAEx/OjHGM89aUDIz7U4cCgBmOeKMdqfjJNOA5NFxWI8YphXA9PepSMnpxTSOOfpVCZyniIf6dFx/yz5H4muisx/osPGfkXn8BWB4jGL2L3j7/U10ViM2kByceWvf2pLcHsWQvGMc08qenNIBke/c1IopiGbevrS7e3FSY70bQe1AyPaB25qC7X9wenUVbxnkVDdg+QenUUmNHM3H35kXq7DP4ClnjEltAo6nAGfcVO8Ya5nbsqY/SoBmSSH0QqvHrWVywmVYpmyBjy8Co4kBsAMdjVi4jEl2i9gMn3xVYOUs1RR87ZA/OmmI5G4jZZSxwNzE4/u84A/St+O3WO3t1HUkEnHU1i3Z2uoPTn+fFdAozBbn/d/lSGaL4jjLnGAKq2cQeWWQjk1PJm4nEIPyjlj60tqu2aYAd+cU7gLJEu08DNUI1CWhUD5mY44rXZeOelVbeESPv/gThRSYIjhsxFEBgbjyTjvVeGAC/kOOueK1mXv61UVcXrk9x/SkwIp4/LjPy4PbNZr2obPr1+tacxLsWI46CmLC3lu5JGBx71Ldykita2KLGH43NWRrEWGb6CumhA+zr+P86wddGCT221fQkj02ITaU2eRlu1XYLcR2qrt5PQVFoLA2Mm4Zw54/AVrWqCRzIcEDoPSkMqxaaqKSQN7dTT/smDxxWoVHbOKbt9v/AK1OyFdmPdwyiEFZJBznhiKfbm8WBNt1cAgYz5rA/wA6vXKZgPHTmmwIPIA9M4qeVXHzMiW61IdL+6+nmt/jTxqmsIOL+cn3fP8AOpRGB2PFQy9lUZJ9qHCPYfPLuNk8R6+p2Rag+7PdFP8AMVNfa/4r0/w++qG4VohII9zwLhiew4q3pGn2b3BN7cLEijcc9WPoKd44vY73QpLaN0ZFUFVjPyoB2x0rJwjzWSLU5WvcwLf4ja+0UbN9lbd1JiP9DXUW/irU5I0Z47clgDwpH9a8qtjttozxlW/LmvRbFQ1pEePu1MqUL7Fxqy7myPFF5jm2hP0JFV7nxvcWrqDpquGGciYjH6VEIxjP61Q1G3DbDj1Gaj2MOxftZdy+vxCOPn0lwfacH/2WpB8Q7X/lpp10p9mU1zjW64+70phtVIIAHFHsIB7aZ1I+IOmkfNaXq4/2V/8AiqkXx/oxIDi5T6xf4GuOa0Tg7ec+lRtZoaPYRD28juh450HHN3IvoDA/+FO/4TXw+2M6mi/7yMP5ivPXslP8qztSsNthLID0APQdjTWGiH1iR6unizQXwRq1qM+sgH86tp4g0mQALqdmf+2y/wCNfPmP3HX7rfz/AP1VuW9orwo/qoODSlhorqCxL7HuC6jaSD5LqBvTEin+tSCeM/dcMPUGvExp6sp4HJ6mo308AfKMVH1ddyvrHke4h8nj60hfGM5rw4wToMRyyIe2HIogk1guFhv7xfpMw/rR9W8x/WPI9x8w+tL5nPWvFn1DWrQPnV75mCZ5uGOPzP1r2T4SSrr3hSaTUh9puYrtozJLy2NqkD9TWVWl7OPMXCspO1ibeOgpm/nj9a7g6HprHm1X8GP+NRP4c0xh/wAe5H0dv8a5vaI1ucekgPepklwf8K6M+F9PySDOv0f/AOtUE/heDyX8m6uFfB25KkZ/KpdRFJmG0x3BVyWPQDmporVnIaY9OiDp+PrToIFgXGMt3J6mrK81W5WxIijgYq3CvNV415q7CvIrelT1MZyLsIworTgbMVZydKu2rZUivQgrHDJliikNFaEBRRSUALRRmkqkJiPyh+lIDnHvSnkGmRn5B6ik0CJpXwAgOB3qe1O63kXuOapn9as2LfvGXuV6ViqSjCw76mPq8DOQ+AyKufLPQn1Pr9P/AK2MGae7nLy+cwVG+YnoBzx9c4rqNThaWJlLGNOQSvX/AOtWNaxRXFpLaNiMCRANvfJ4rwqvuuT5dE0dyfuopCe8KgjkEcHc3P6UVea21gsTGyKmflUSAADsMUUvaw7IV2eQ/GRiZ9JUdQsv/slc5oAwlt6bv61t/F2bzrrSDn70Lvj1yVrG8NjEdr3A6V7GC/gxMsV/EkdavHtj9KmAHH6imooxT1GDn+leicKFxxRjHvTufWgZx9KBjQBmnBR6daXHNGOKQxCBnqetLjOD607b2HWgLzTQhCM0w9s9DUuPpTGHpTJZyniMYvIjx/q/6mujsB/ocHT/AFa9PpXNeJm23sJ/6Z/1NdPpw3WFuf8Apkv8qS3B7Foenan4+uaaFI4z+VPA49cUwsHIzR1FOx1PB9KNuRx0pXHYTAbtUN0B9mP4VY9sfhUN2ubcnGOQf1pPYZgthZLkngAD+VRlRFbwE4GW3GlnDNLKnY/MeOwHSnTKrRW8fQtj8sVkaWK8YLTl2z86k49B2pkEQETS46qQvsKtTMI7hiSMLHio0AGnnjHymmhHH6iux075zmt2ED+z4GPQKOv0rC1RHW4V2ztYjaPbHX8a1rMvLZL1CRoB+NDEjatIike5vvvyaLcf6VMKmhJMSNzyBUO8Q3UhPHy5/lSbGLOxdxCnU/eIHakswQsg9GqS2jJBlcfM9JaH55R70rjJdueDVOfakjAck9fYVclk8tCf4jWexyx9TSkxpDVjMjgD+VW5IwkBHoKfBD5aZI+c0s+PKY9OOKEJlW3H+jDI9axdcXKnjOUrbg/1HJ9awdek2ywxryWyD9Ku+hNiHQAziVBwobcfy/8ArVvaeflbjnisTw02XuUz2U/zrcseZH60uoy6BznFKRk04Dj2/nQeOcc1VySvKuY2+maitV+U9OuatOMofU1BGDDHlgcnnHpUNlJBI20YHJPTFIsQU5Y5Y1IkZ5dx82Pyp5AJzjNNAyFhVLUY99hcL2KH+VaJHyjPSq1wu+J1xwQRVXEefWozanHUPnp7V6Lo7B9OhbjG3v2rzm2zsmRuxH9a7/w44fSo+nHFZSNYG0Bxz0qrfp+6BHUNzirQ/wAiq96pNufYg1BZmFecnp3ppUZ44OKk56/1pvOOvJpk2I2GevSmFOPXJqU/mBTT/nvTuBXZevb6VVu4t9lOg/ijbH5Veb6VEVBOMcHiqTJZwQ5gkHuD/n866PS/nsYWxggY/XFc8g2iZOny/wAq3dEbNmAeqsRjNOWwkagTI6Ch09PrzUsalzgDmr0Vrt5cc9ayuWkUIrJpW+YYX1q2IFjXCgCrm0AYH6VG64BPb1ouVaxganHiXp95P/rV6N8B7rNnrVqTyrxSgZ9QwP8AIVwGpphYyB1DD+VdR8D7gxeLdQtScCa1bA9Srr/QmssQr0mODtNHvZOcGg0g6Y9KWvIudYzvSHkGg8GkNQ2Uc3ewmO6ce+R+NRqvNampxZKSf8BNUAuCPfmuyjrFBJkkYq7CMCq0a1bjGBXfTics5Fhat2xw2KqLU8JxIK6EYMu5opKKskWikzRmmJhmlpKKYgzUa8Fh71JUfSQ+4oaEKaltW23Ke/FRGhW2uG9DmpktBk9+uRIvfrXD3MLPe7SSo7kdQPb1Nd7fKqgzEgLs5J9K4q5R5LlpJI9kefkQjlh6sD/L8/Q+XVdp27nZSd4Dc3X/ACz09Cn8JMRYkdue/wBaKYwYsSZHJJ9aKy9kyrM8i+Lv7vVdOjBO1YG4x/tHH8qy9EuVggtWMbMFQZx3rQ+MDA+IrJT2tv8A2dqy9MAFnBnBOzr+Jrqw8uWhFk11erI6hNahxkwzDv0H+NTDWbYgfJL/AN8//XrDXnp9KlH6YzW31iZl7GJuDV7U9RIP+AUv9qWmc72BH+yaxDx3pVPbNH1iQexibn9p2ZH+t59wRTxqFpx++A/AisIdMjrR37U/rEg9ijoBfWp/5eEH1NPN5anj7RF9N1c7wB0ppAzjGKf1l9ifYo6Q3MH/AD2i+u8U0zxEYEiH/gQrmjg88flULgDnj8qpYh9hOiu5F4slRb6Ahs/uzyOe9dZpbqdNtiGGTEv8hXm2suEuY+3y1vaaF+xwHGPkH8qp17K9iVSvodyF9P51IFPrXJqfQkfialUuD/rJPwY0vrK7FKh5nUgHrjn3pCMgY9ea5pZJs/6+X/vs1ILi4B/18v4tR9ZXYPYvudFg96iu1/cH2IrGW7uf+fiT86Gurhkw07EehxQ8RHsHsGRsuXu2I5AA/SoEBeaGQ/dBCqPoKny5LfPy3B4600bxtAbhORxWftUV7NjLiPzLph1Crk1FKfL00Kf4kIHPepxu3M+RuYYJx1qtPGSgU4wowOKaqoXs2c3ri7Xt8enX8K1bBAdLTAHK8/WsfxA7i4jDFdoUYGK09JkdrCMDG3BGMVbmrEqGtjZsyDbRn29abIA90sfQdz602BniiCKFwvrSpGUkWQHJBzz3qXURSgy5joB17VWtyFuZgT71J5rcZQc+9REEs7AYLe9J1ECgyKeUu2c8dqbCB56cf55pxhJOccd6eR+9V1XAUYxUqauPlLYHI7daimJ8p8eh6UpnGMbWpjSqQRtbnPatOdEcrKqy+Xa+jZwPesrUYCsPmONzkH8K1QF8zc4JUD5RiqGs3MSwAnPXHK+1OMkJxZleGW/0ycZPKf1/+vXQWX/HzIoJ/H61y/h6QR6s+ehUjp7g109uwW5LfwnPNNtIFE1OR/hSZ4xmmCVDyD+GKPMQDr09KOZC5WKx2g889KrwtvnO7qBT12u5Z3AC9BUUOFuskYXkZJqeYqxsWOnvdAuwKxDuB94+grVGnIBvZOeylD0p9vLarFGxki2gYVcjn681R1HVFkDRW7jn7zjj8B7VnzOTsXZJFPUGgRzHCEb+84HH4VlSLuUgg1YbDDsKhZeuea3jZGLPO9nl3t3GR0JA46c12XhZy2nnnO1v8a5K/URa7dj1JP8AI10nhCTMMq56H/D/ABqZFwOr71Fcgm2fnoM09TikmBMDjGeCOKg0MnnB9qQ9fYUdevSgn8qLiGkGmHvwSKcf/rUh75P9KAImHH+NRnjnP5VMRxjH5VAeuRxTTFY4yaLZqdxH0+ZwM1qeHYmkjlQZ4YH86i1K0K6rPIeFBDAeuQM1a8NNtubiPIyVB59v/wBdVKWgorU6aCJYx8o59anGAP61GuAakHB96wubLQXtUcigjrz3qXIxg9PrTGOeKaYmY+qL+5BGeG5/KrfwvuPsfxNshnCzb4z/AMCQ/wBQKh1Bc2zjv161n+H7j7F440W4yFAuoST7bwD+lOSvBonZo+qDw31pe1EnYijORXiHZ0Iz1pD0pW600nAznArNlIhuo/Mt3HcciskCtW1vba8lmjgkEnl43EDjnPQ9+lUHj8uVk9DXdhdrE1NBI16cVaSokFSpXpQRyTJl7VKhw1QipVPIrYyL1FNQ/IPpS5qkSLmikoqhC0UUU0Ji1G/31980/NMl4UH0IoYhTTTTj0pppDNFsSWsTEZwP/rVh3cCtdAFFYsQBuOK2rZt1lj+6cVlaomORXmYuN2deHfQQ6hZIdggiIXjOOv6UVjhJMD5M/nRXF9XX8xryI8L+LrhvFEC9harx/wJ6z9NnhSzhUyICEGcsKufFaT/AIq9RkjFqgOPQlq4piRI2DwAOa9GjG9GK8jOs7VZHcpNCQMzR/8AfYqZJoiOJFI+tcADjjP4U7cT3xjtVeyJ9oegBl45H509WGe1ee+a27KknHpTvPk6iR1z6MaXsg9oehD6Zpe4rz77VOowJpPYhjTxe3IU4nlx1++aPZsPaHf5Hemnt6muDGo3isCLmYen7w07+1L0Di6mI/3jR7Nhzo7Yn/IqJzlcAfhXH/2rfZH+lSdu9KdXvQeblz9QP8Kagxc6J9eP+kxeu3ofrXRaXzYwdT8g/lXF3N1JclWmfc44yRV2DWbu3hRI3G1RgAqDgVUotpIUXZncJUy+mOO3NcSviO/Gf3sZ4/uCnDxPqC/MTF+K/wD16z9my1NHa9sU4+o/UdK4seKb4Y3LB+Kn/GnjxbejgxQH2wf8aPZyHzo7IHBHH6U7P41x48XXRwfIhPr1/wAad/wls2ObWMn2Y1PJIOdHWkgDGaM5A6+tcp/wl8ne0Q/Rz/hTh4t6k2g44/1n/wBanySByR0+7I9aic+lc6PFy4/49D/38/8ArUp8Uwtw1tIM+4NPlkLmRV8RsVnjY88Vd0Ns6dF17/zNYer6kl+VaNCMDByas6Xq0VtbCOQMSCeg96tp8pKa5jrY2AGKlz3rCTXrUcNv/Kpl160PV2HsVrOzLujYyM8A0bvSsga7ZMB+8Iz6qaX+3LM/8tfzBpWYXRqlunIxRkdulZY1qzJ/1w/EGnf2vZ8ZmUfnRZhoaWRznA/rTSe1Z/8Aa1mQR568fhQdVszj/SE/OjULouOeKydYG6yPHRgastqVoc/v4/8AvqqGo3cMltIFlRiOwIqo3uSzF0ZtusLweQw/SuziI4HT61w2mPt1iJmYbdx5J9q7OKQHHI6+tXPcmGxfB7nNBHHPFRq3HBBz75pc547+lTcsdn+VJwM8A0mc0h6UrisOJX0FNIB7Um4delIDTTGIQmTx9ajZVHf24NSMT1/SoZG46002TY5HWkSLVJSAQSAePcVZ8NyNicKzLgjkHHr/AIVH4hGL1CehQfzNReG3xdzoTjKg/kf/AK9a/ZIW510csuR++kxjPJqUSysMeaxB681VjP8AKpwc9xnNZ3NACAcZORR5QHHSnL+eKM/jSuFiMxZH3qQxlT94496mOcDPNIev+TRcLFfyuPf6U6O3GeeakJA6/jTkyM880OQJHPa9Ftutw6tF/iKz9Ck26p7OrD+Rra19QTA5GRyv8v8AGue0tymqQMT3x19aqOqJa1OzBx0P5VKv0/Oq6tx7VODzzjmszQeDjp+NMZuOn60pYZHb8ajY8HP40XCxVuzuikX1Q1y1zMY57aVW5Q5B/KuslUE7T0PFcffLtiQtn5Wwf8/561pF30M5I+vYJlubOKdPuyorg+xGach4IrmvC2t2kfw+0fULy5jhiFpGjO7fxKNpHucg1h6j8RbKR3is5Wjj/wCehQ7j9PT+f0rxnCXM0kdkNUdlqGq29iMMd8vaNOv4+lcpqWszXmRI22M9Il6fj6/yrmJfEto5OJTycknOSfeqkmu278+bx3pqjLsdEeSPU7rwjd+Zq08JOQ8W78iP8a6O+jxMrjow/UV534N1eB/FFrEsgJlDJ/46T/SvT72PfBu7qc100Vyy1MKzTehQQdKkApiipBXoxOKQ5etSCo8U4VqZsuRHKfjUlQwH5CKe7rGpZ2CqOSTwBVrYgfR3qKK4hnz5Uiv/ALppzSohAZ1Unpk4pgPozSe/agMD0IP0NMQ6mSjMTfSnUHkY9aYhqnKAjuKaabF/qgO44pxqRl7TmyJE9QDUVzGpXc/IXkg96LBttyAf4gRUt0H2soQH1+bFcONXu3NaT1MVruTccPIBns2P6UU4xS5+8B7Bun6UV43NDzO/lifNnxQfd40cdlgjH9f61xzDErZwK6z4ltnxrc9OI4x/44K5UfMzA859K9uhpSj6HLW/iMASDQfqadtBIpcdBjr3PetTMbuPt0pDnOfWnlcc5xmjHzZ7mkAwHHr+PFGOOAak2Z+7/KgLz60XAjz0xgZ9aXtjnFPVSc85BOMYpAvTIPv7UXAjJ5OOtGecgAZ7etP24GD+RFIVA5yOO1O4rEbcc5HXHXpUoPyj2GOtLb2st7dQ2ttHvnlcIiA9WPQV06/DvxPgZ0zBHX98n/xVKU4x3Y4xb2OW3YHH5005J6V1q/DjxQyknTlx6+fH/wDFUH4a+KWP/IPUD1M8f+NR7WHcv2c+xyeec8AijP4jtXXp8L/Fbgj7BGD/ANd0/wAaU/DDxWQB9gT/AL/p/jR7an3D2c+xyKybCSQD9RTS5Jz0rqn+HHieJyp0/JPYOpx+tIPh34oYhRpp3HuZFx/On7SHcXJLscvu57fhQXH4d66d/h74nViraa3B7Op/rTD4A8TL/wAwyT/vpf8AGj2kO4ckuxzSt2z+NAcZBx165rov+EC8TAZOlSe/zL6/WkbwJ4kGf+JVJzjoV/xp88O4uWXY5tiQPb+dPjY7SQeK328CeJW+7pUv/fS/40+PwF4oyd2lTcejL/jT9pC24ckr7GBvxznjuKBIQO4xx1/z610f/CCeJSD/AMSmbp/eH+NNPgPxJk/8Sqbkccr/AI1HtIdx8kuxz289z9OKUSFV684rfbwJ4kI50qbP1H+NJ/wgnib/AKBMx98j/Gnzw7i5ZdjA83pwc4xilEpC56Gt8eA/E/I/suX/AL6X/GlHgLxNg/8AErk/FlP9aOeHcfLLsc9vOeaaZDjnHPJxXSp8P/E7kgaVIOOcuo/rUTeBPEm7H9lzf99D/Gjnh3Dkl2Of80sfx+lIZRwM8jv610B8CeJhx/ZUxweuR/jSf8IJ4mOP+JVNz05H+NP2lPuLkl2OZVyHHrmnGQ9Mmug/4QPxP0/sqYY9x/jTv+EB8TY/5BUn5j/Gn7Sn3FyT7HPCZxyHI/HFO+1TqQPNfj/ardPgLxN/0CpRx6j/ABo/4QLxMTj+y5c9eo/xp+0p9w5J9jFXULoLxczc+jn/ABpw1O9A/wCPufA7eYa2G8B+J++lS/gw/wAaT/hAfE+f+QVJ+Y/xo56fdByT7GQNVvVb/j7uM56GQ1IusXwGPtcvt85rT/4QLxPnH9ly/XI/xpf+EB8TD/mGSf8Afa/40c9Pug5Z9jN/tvUOn2uT8e9OOt3/AEN05x64P9K0l+H/AIpbONLf/vtf8ac3w+8UI206Y+R6Op/XNLmp90Pln2MC6vri5IM8hZhwCQOn4U20vJrOcyRNhiCDxnit9/h/4mQrnTWQMcZLgD+dSp8N/E7EYsAcjr5q/wCNHtae10RZuXL1M2PxHeoP+WTY65X/AOvUw8UXg48uEn/dP+NWW+Hfifdt/s9dw6jzk6fnTD8P/E69dN4/66p/jSU6T2aHFylrHUaPFVwOsEP05/xqRfFUmebVOP8AaNH/AAr7xRnH9n8+0qf41Df+C/EGmWUl7d6eY7eIAu/mI2BkDsaL031LtNdC0viph8rWyE+z/wD1qf8A8JOpHNp+T/8A1q5MN2z2pwkPbGafs0Tzs6xfE0RxmBxkdmzU0OvwNJjyZB6ciuPDnPP5VZtpT5w688fpUyhoUpHQarqEV1Ciqjhgc4P0rAjk8u9R8cLJk4571bkbK9M+mazZSRK/1HSlTCTOwTUoMdWH4VMmqW2ANzf981zaN8mRnnvU27gHjFZtFJnRf2lbYB39/Q0hv7fOPM6+xrng+F7g/wA6rT38UHGdzegxQot7DcrHSyXdvtz5oA9a5LVL2FppYojuDP8AeHTr2qlcX01wTlgqk52jpVU9a6YU7bmM532N/S7jXtVRLG0u2aK1Q+XE7DagLZOAfc1pPo/ilQSTE3/AF/wrlbe7nsp2kt5GjY8ZU9q0U8Uawi7ftrkf7QB/pSnCV/dsVCcLe9cuy2viSM/NGv1CJ/hVctrafejH4qtQN4k1Nx81xn/gAqFtavHGDIP++RSUZ9Ugco9Gzo/CWpanaeMdGlmTbGLyJXOP4SwB/Q19WModSp6EYr4wh1i5jnjl3nKOGGPY19m2063VrDcJysqK6/QjNc9eLTTsaUpXujMC4JU9RTxUt0m2cnseaireDurkS3FApwpB1pa1M2TwHtVbWrae609o7cAybgcE4yM1PEfnFSzyeTA8mM7VJx9KtbEdTnoYDp+oQqjsXYKZELbtpPXHtU95ZXE91cM7PEgBZZRgjpwMGq+jSxTSveXUq+YTkA1f1e4LCOCNvlf5i2eMdqStYp7kCXMraPJ8xG11GfY44qvbtJaXUM3mlkl/hwOMHBHvWn5EQ0qWCGRHbYSdpBJNZLsJLKEg5KyEfgRn+lUSWL77RPezpHIFdOUBGRwM4rU0m5a4tPnYsynqe4rPlfbqkEnQSIpP8j/SptFJjmmhPbgfgcUxGoOGce+aDyaDxMfcUGgCSBtlwjdgwrVlTLH3FYtbhO9Eb1GawrxvFlRdmZTRfMfrRV0xjJ4orxeQ6faHyP8AEbnxve4z92MY/wCALXN4JduCfm611Pj+JpvGl68S70Pl4ZeRwi+lYuIYy+9CH3etepSdqcV5E1NZsp7DjOM+1KBng/lj2q6t1Ag2hMZ9xQJbLrsb/D9arUkphCcdeKAhBBwTVwS2Z+VQ3JwOTTRJaEjGQDxRqIrbDkbT+lKBnI6DvWja28d0jOnQHGSe9T/2coB6fianmKsY5Xk8c9KTy+3cVtLpqg/MMDOPvCnjS4Twc47fMKOYfKYJUHoOg6UjIV6g/wBK6D+zYMHLNz7jFMbSrZh/rWH5UKYcjK3hNseMdDzjm9j/APQhX1YEjbBKLnHXFfL2i262vjbQ0Viw+2RHJx13ivp4PhR7VlXXM0zSk7JjvKiB/wBWn/fNKY4j/Av5CmtOpG3bz/eppc1j7M15x3lxZ/1a/lS+VEesa/lUfnBOSCfpTRLnmj2Yc4phj37gi5+lKY0/uj8qTfSbqagHMSCOPH3R+VOEcf8AdX8qiDj1p2+jkDmH+Wn90flTfKjP8I/Kjfmk8yjkDmHCKMH7o/KnBE9B+VR76XfRyBzDwiAfdH5UFE/uj8qZ5lIXo5A5h2xQeg/Kgqn90flTC59TSb6fsxcw/wAuP+4PyppRP7o/Km76TfRyBzjtif3V/Ko5ETb91enpQXpkjjHpSlT0GpjkVNx+UdKQqm1RtX8qjRxke4pA/IrP2ZfOSeWmw/Kv5Uzyk2Z2DJ9qQy/Jz60M/wC796PZhzkgjjx9xfyphjQk/KMn2pPM+XNNEmX57U/ZhzjnjQAfKPypTFGR9xfypkj5A+tL5gIBpqmJzI2ij8wfIOnYUx4kY4Cr+VOeT5s8cCmK3GTnJNP2YucjeGNSAFXH0qrqV5Z6damSfCA/KpCZP4VYupBDEZGzhVLHHtXL6fq9xqEiXzoQrKSnPC+gHrj+dTUcKUXObsjnxWNpYWk6tZ2S/qy8y9dXUV0VeLzdoyMPnH4A/jSy/wBrWNqrppokt5CCJmdcKDjnGM1pC1SDQLqV23zzBCT/AHRuHH+NNEr6wba23NHbW0ShvdgMZ+vYewJ9q8hc0KznJe/JaLte619Ers+LjWr0swlXlH99Vj7kb7XbSvr0Su/06ZtkgubsqJV3J8zjIJ/KtR7eNjgRr78dKs3drbITdJbpHIF/1iDGQO30pkXzIHH8XNelhMJGhHlTu+p9RleWQyyjyKTberfn6dPzIVhQPgqvX0rP8V2sdx4V1WLYv/HpIQNo6hSR/KtR5cMVH07dc5z6+1Raign065ixnzInX81IrsjTsz0XO6PljAz/AIUox3OMcUpQjP65pdpZuB27Cu44wAOeCOP1qWD5ZVPuORTVU9cfnTwAOQAec1LGjQcjGD1PPIqjck+cD7enWtBwNox0PXis+96KR6f5/rWdPcuWxdgJMSEeg5Bps06RJuc49PU1ni9MUKoqjI7mqrsztuY7j71ap3epDmWrjUHlG2P5F/WqWfeiitUktiG2wpDS0rUxCGkpewNBGKAEooooAK+wfAF7/aHgDQrjOSbNEJ90G0/qtfH1fT/wRvftXw3giPW1uJYf13/+z1z4lXima0Xqd5eJlA3ocVSrUlTfEy+orM71NF6Gk1qLSg0lLW6MRynDZqyyh0KsMqwwRVb2PSrEZzEhPUqM1SIZnL4fsE+4sq/7srf41PcaXBcLGpaRfLG0FWI4q7RVWQrspWmmR2khkWSRiRjDHNQNoUPnF45pEQnPljBA+mRkVqUtOyC7KV5pkd3FGodo3j+46+nofUUthp/2PczSeY54zjFXKUGnYVyOQYkQ/UUpFEv3QfQg0ppANrXtG32ae3FZBrS05swOvoc1E1dDRNiin4orh9kjS58OI5jVQGI+hoLtuIYEEE54HFRQBmRBnn2q7DomqXU7CGPLAZ5cV2KN2TzWK4lP+1z0pyy4/hI57irv/CK64B/qAB/10X/GpF8K62SR5KdP+ei1Xs32FzruZwlyB8v/AI7S+ZngKc/7tXv+EW1wDi3Xn0df8aQeG9cUf8e/H++v+NL2bDnRVWZwNilwOuFXr6nj8KcLmfOA83pwTW5pnhq/eFzc2uXDcfOPT2NX28MzggpZkewf6e9Q4tPYtO/U5Vbq5Zxh5eRjvUyyXxHyib8FPNdInhu6VgTauvP98f41KNIvYyQlq3HToah36IuNurOYA1VhxHckf7pP9KRzqsQy8NyM9SYzz+ldYLa/QEG0J/4D/hUTQ6huw1vIM8/cIFSnLsaWh3Mfw1JcP4u0gShwReRD5lx/GK+mweK+dLCGSLxjo7SxshN3F1GP4xX0UOg45omr2FB2uKTSeYBTTnHAqM5JpKInIkypOaXK4qABmPAyaN3ODVcgucnytLlfpUAak3Ucgc5Y3rR5i+tVt3WjdzT5ELnLRdaN61V3f/Xo3Y+lHIHOWt60b1qru4pd9HIHOyzvWkLrVfd70m/Jo5EHOyzvXNJvX0NQbvrSF6OQOdljevpSb1quW96C1PkFzljcvpUcrrt6VHvPrUcr5AocBqZIJE3Lx/DTQ6bc471X34Ct/smgN+5P1qORD52T+YvpSmRPL6dqrbs7j+FG793j1FHIh87LDSoExjGaRJE3dDyKqsxJ9vWnBgso+mKFAOdlmaRdvSnCRSBxxVSVvkp2/C59BT5ETzskeRSQu3rTgyDHHA7VTZ+QR0z61Nvp8iDnY64Kuu3GMgjiuSEqJeQ2AhlSRnKLG0ZTao43c/w+mOv546mRuAQBx2NVJ7KOKRL6MItvCMBFHLsx+Yk+o4P4Vx4rCU6zi59DkxOW0MfOEqzfuO+j/r/Mfq1s4tUdWYxp94Hp6Z+tZemWk0+shh5YiCgMytliPT0B5rb1GVf7MlQOOi9O/IrKGrm3tsR6jZRzrgCGRMjA46Ag5461yRpQ+uufXl/WxFPB01ncqqWvInu922vyW2xW1SG8i1uSLMzKhwDHPtwp5BK9DzxXQRkJAiDHyqBXPLrM17P511HalwQuIjnHvmuhVSUDd8CvQo2bZ7WKTikV5Tlzjil6jB78VIYGfc6cqo55/wA+lJj5PwrblORM8Qm0vQ4ruaJ2td6SMrAyAEEEj1pV07QO32UjrxL/APXrP8S+GJm8T6myMQr3MjKNhPU5/rWSfDFznhuv+yahxj/MaqUv5Tqho+glRxBtPTE3X9alTQtCckAR8df3xrj/APhGLofxj/vk/wCFMbw5dr/EPyNLkX84+d/yHdnSNExgAE4xxKTXmk3myXLx5LFSQAPb/wDVVx9AvFGcrj8f8KzJY3gnaJ/vKcGtaNNRvZ3Mq03K3u2NW1sraXTpPNt7gXe/CNn5NvuOvrQdKjwRmQH6j/CqdhY3N95nkMP3YyctirH9i33H7xf++jVt2e5mtVsbumaDod7LHbsb4TMvJyoXIHOODW2fh/oythrm5U/9dF4/8drio9J1RX3RS7W9RIRU62GujG28kHGP9eRisZKTekzeEopawOw/4V5o7Di8uh/wJP8ACsjxR4LsdF0Y3lrdTSOrqpVypGD9B9Ky/sniQdL6fj/p5PH60k1h4iuYDBNdyywsclGuCRn6E0oqaabnoVJwcWlDU5zHy/jXqOn/AAu0++0+2uxq0w8+JZNojXjIBx1rhh4Z1TBHlrjP98VrQQeLreBIYb6dY0G1EE/AHoParrScl7k0jOjFRb543Omk+EluPuapMT/1xB/rUDfCOTkjUZAPU23/ANesR7nxvZwyTfb7oIiZZhN0A61mt458TM2W1i4Y+5B/pWMYYh/DNM1lPDreDR05+Etx/DqOc+tuf8a9Y+Emg3HhrSdQ0+abzVadZlOwrjK4PX/dFeNab4g8f6lb/aLG6nli3FdwjjxkduRXp3wo1DxbNrl5D4iMjW7W+YiyIMOGH90ehP5VEvbXtOaY/wB01eEWj1ys2dNkzDtmtKql2n3W/A1rSdmZTWhVxmnCiiupGItTxnKfiagqWI/KRVIhktFFANWSLRSUUxDqUU3pS0wElGY29cUincit6inmo4+I9voSKTADV3TGxK6+ozVM1PYttul9+KmWw0aeDmipNtFc/sx3Phi2X5ohkfeGB+Nd3og33koGANmefqK4vyHtrxYpUaOWNwrKy4IOe/pXcaEo8+ViRwoGMe9dNPciextFCo+8CDxgGmjhs9Oc1Lgc5YfjRtHJ3D866DAiIOcY/WjrjipWVeDuH+NMKj1H4UAWbPGw+zCrJIJ/lUFooKMM55FWShGOKzluax2Gd8/570hyDkYzTyMgDFNOAen+f8ipKIznGaRueOMDtT9pK8g9cc0yRxHGzuflAyaQzm9ROPGvh4cZ+0xk4/3xXuykEV4HOr3XjTQyTsaS5jG4/wAPzjFe4rHcAf67r/s1zVHdm8NEXTKxjMYxtPWmFeCKr+VOf+W36U/ypsH98fyqUDHrK0GSuNx45FVtxJJJ5NMkhn3czHGfSomik/57tWiM2WN/vRvyODXKeJ9UutGa0ZLlwshYMMDnGPb3rBPjG9z/AK4kd+n+FFxpHo5fk0F686Xxfc952z9B/hS/8Jddf893x9BSuB6Jvo3jHWvO/wDhLLnj/SJfTjFH/CU3P/PxKKOYLHonmYo8z6V53/wlN13uJfzpyeJLp/8Al5l+u7pRzofKz0LfSF689fxPPk4uJv8AvqmjxJcHn7RN+LGlzoOU9F38UheuHg1yR8bppv8Avsirg1MuARPL/wB/DUuqkUqbZ1u/PSkL1zEl6yxqfOk5H/PQ1GLtmVj58/H/AE0NT7dFexZ1W/nGKbI3y5rk2uXfhbqVTjjMjEfzpkF1Luw1zK5H+22PyzSeIXYaoPudKXJRfypd58tuvWueW6fGfMbk4yXNMub5kYgSt17Oaj6xHsV7BnSK5O7r1oVm8scGuT/tJgf9ZIf+BmnrqZK8Fj77jT+sLsL2LOnUsX5HA60rORKCAcVyU2pkDBZj9GqodSbPLt9Sx4oVddhOkzuJZDs7/hTTIW2oB27VxB1N/wC83/fdRnU2xncT/wACq/aoXszupHI2ipd59D+VcBLeTRCNnRlWT5lPqKg/tXqMr1/vd/wojVUtURFKSvF3R6FM/wAoGD9KwvF19qFj4ckvdO80TxlVYKflKNwdy9CB15rmP7VLAbCmAeTnOaradqpn8T2UFxHm3Z3T5icFyjYPtg/0P02prnkkD93qbOjeKzcOukXbvLIfkguGj2edtGTuUE7WwD3II9DxXXwW0DWm+AW6TEfM6nYxOO5HU155ZaFNp/iazuo3eaykZyBIuXgfa+QW9D79f583rWt3dvJqklhqYt2gvJI2t5bgMzfMRuVSucewbj0rjlRUsbLk/lX5s4KVd080fP8A8+1/6VI9A8Q6hYeGLWbUnSGS9kO2MMxZpG7AknOK7PTbmS80m2uJ4BbyzRK7xE52EjOM1856bfvNrUWpa002pwWkqlwZTgjPGMjpnGRxXuOj+NfD+rOkEZlgmcErHLD97jPBXIx9cVvGmqSuz06+IU/eeiRuySMmFjZhkYOO9R7jsxUj2aSrmNFZSM8dPrmqq2iAkFACDggiqM09Lo4vXQseuXOADkgj2OBVDBHzFRgnjJra8R2wj1cbQFDRKew9R/SspcDou7sCegrzqmkmenSV4IRdoToQcdKlVWb5nBAHAJP8qRYi3rz69KnEDNhQS59hxWTZukR74wVOxiF9Tx9cV5T4uTHia6brv2tn1+UV7NBayQ/OFQMOdzgNj8+K8w+JMb/8JDDO3JkgXJxjJBI/wrpwc/3ljmxkP3dw+HKefq99Z5wJbY9/Qj/Gu/8A7IdZ45UkQlWDYC8Z2gEfzrzn4ftt8ZwR9fOR1/8AHd39K9vgsS7D5cgdazx0uWoPBwU6epjWGniPe08UfmPIW3KM4GAMc1pwaXbSI26FBxy7KMfh61qJaE/KoAU9l6H/ABq2lmiAbsk/3RXmyrHcoRjoYyaLaSHiBCM8sYxU6aJZI2DbQOccZhFbawk4CgjHTPWrMVoR7H9TWXtZMTlFbmNFodhktJZQknsY1q1FodgeljbZPUmFf8K37bS3kOdvHr0qWefTdL/18oeXtGvJrWNGpJXbsjlniI3tFXZgSeErK+s57f7DAfNjZOIwOoPevlXV9In0fVZ7K5AWSJzGwBzgg4NfYv2jVtTTbZwCzgPG+Tg/gP8A9VfKPj22msPGeo2txJ5ksVwwLn+Lng/iK9bAR5G0rtHFXk5K8rXPUfhBqF5rPhO40W00m1It5h5lxtAJDc/Mecng/pXqWj+Em0qRr2e6Z5sH5EG1Bn2715R+zlqQiude08kfMsUy/gWU/wAxXu01xuUrRWjThUbe5Mas+XljsVqjnXdE3tzUlJ1rWJTM2lpXXa5B7Gm9a60c7FHSpYjzio6dGfmFUiGT0ZpM0ZrQkXNLTc0oNAC0oNNpQaYh2aijOJJV9wfzFSZqPpc+zL/I/wD16GA5qfA22ZT6HNMNLF9+kwOgoqFJRsXPpRWYz4jklklAmlkZ5CAzM5yTj3+ldn4acSRPKP4kU1xjr/ooxjOzn8q6fwbJv0xjnkEJ/n860p7kz2Op3H1xSFvbjP0qPJzgdaQnIz6cY9a6DCxMrZUj0OelNz1x3qMEqTx14oPDdzSGX7GTDsmOvI/D/wDXV0sOncDrWLDJ5UqN6H0rXHTAOR6+vSs5GkNh6sAemBTDg5wPel4Kj+VB69TzUliDrzVa/wA/ZCMfxA/rVgjGOeKr3mDat6cfzpPYpbnMl9vjPQW9LqP/ANDWvfFFfP8AN/yNuh8/8vMfb/bFfQSg4rnkaIYflwAuT6+lLzg08rQRwaEJkSNGG/fKWX0FVSOTgcZ4qcjk0witCGzg/iLbvPFpyqxH7x84+griBpU5xl5P0r1rWLBb+5s426KWI/IUkXhmIsG28ewo5Li5rHlQ0qU/xyfhj/CpRpcgH3pPb/OK9ZPheHjCnj2p6+GIh/BT9mHOeSjS5OuZPxx/hTv7Ll9JPz/+tXrq+GoAchKd/wAI3b/3KXsmP2h5LFpUjMM+b+dWZtGkiYrGshXHdutepr4cgH8FWm8P25+YoOlS6TuUqiPF20yYdUk/OpYtJkLD5H/OvW30CDJ/d0LoEI/gp+xYvaI4Cw8PxyAF1l/77IraTw3ZnA8qT3/fN/jXY2+kRp0XFWv7OTP3f0rCVCV9zVVkce2gWKKg8pm78zP/AI019Ds1iYiCQEcY818fzrrZ9NYgbV7YyK8N1Oy+KJ1K6W1XUDbiVthGzGMnGOM1n9Xl3L9sjujosO8lUZRj/no3+NV/7HjLEgHPs7f41wq6V8SpWXzU1U7fm+8g/wA96nfTfHbXcDrbarborDfvudwbnvntUvDPuWq67HYppkS4+XoeBuPH61HeaaobG3I/3jXSJATGdyjPSm3duBnKgAVj7F3NfaKxyS6cGbHl5/4EatJpMJQfuc49zW4tsNxwBkLVpLceWOPalKlIFURx1zpSjgRAH3rP/s1dwBiXk/3a7a5tgZPwqitoJLrI5C89aqFKRLmrmENFh2jdChJ/2adHo8KSKxt422nOCvWuoW23TDIwAOKbcReUhYYJ+lOpHlg3LZIxxFWnCjOU9knf0sc5rGrafY7IbtY5XI3rG4HTkZ5/Kuel8U6QmSujwPjgkBf/AImuY8Y6l5/iO8YM+2JvJVWPTaMHHtuyf171kI/mRknP064+lZ4LKqPsIud7tX3a39GfL5flOHjhYc6fM1d+9Jb67JpHaS+NbHIA0G2fBx80gGB/3xSweL9P+1wkeHrWN/MAWUMuVOeo+T3/AFrilZ424PHrTwuwFskNjII4I9P6V3RyzDJ7P/wKX+Z1PKsL2f8A4FL/ADPb3MMN41sskPnFfMZA678dMleuOnNY+p22nmWe4n0azuXXG6SSNSzYA4ztJzjH4VB4asoIU+0xK26RCXdwASTg8nOSeOpqbVYr793c2chWSVBDMC2EGMkMAcjIJbn3+lcNXLqSzP2ai1FpLd7tNrW99Unpc8aOFw0cz9nKL5bJfFL4nqtb32VrXMP+2dHs0+zy+GIo5ZceZFDAjKFz/EcDPrjBqPQvGGlPrSWa6LDYNKxhWaEDlsjAI2AgE/rjI6kXYdHMJJlyTnLbmyST3JP0rz3XbaTTNenMTSRtv86Ns8g9c59civSxeR0FSdk1fzl+rPVq5PhakHBJq6/ml+rOu8R+MdS0TxbLHprm3EKqsuDuWclQ3zofl4BwOM+/QDpfDHxQtdXvorLUohDdzvtSWM5RmPrnlcn61wPxFRJ7nTNYhRUhv7UMAQA5IwctjvtZR1PT2Fc1oG5/EemgZJN1FwP94Vz4BJ4WFui/Fb/idOUyX1Oml0VvmtH+J9AeJoEa9t5HI5jIx6kH/wCvWOttkDJVU6DH+ea6bxDAzi3cLnDMMk8dv8KxYrcscAbj6CuPEO02fVYZXpojit16RgFsdxk1bitzH8h2k9cknAFWIrZgvL4PTavT8av29mXOSCR7GuOU7HWolJbQMcbgx6bmz+grzj4t6f5D6VcjlXWSPOPQg/1r2KOz3EALjHpXCfGTTgfDVlc5JMVztJHYMp/wFaYOr+/Rhi0nSaPLPBcnk+OtHZjw86r/AN9fL/WvpiO1G0AdOvJr5f0djbeIdLucH91cRufoGBr67gsWfHBxW+Zxcpx5TiwtX2cHczktQehOParMVjnAAGK2UsEjXdKwVR6mkF3Gp2WcJmf+92H41yRwjWtR2HLFN/CV4dLOMsAq+ppzXNnat5cCG5m/uoM1ObG4u+buYhf+eacD8atw28Fqm2JAortp4a2sVbze/wBxzSq3+J3MxrXU9R4mmFrB/cj5Y/jVmz0ew08744gZT1kflj+NW3mA71VkuPerlKlT13fmTzTkrLRFp5go4r5M+NFt9n+JWouB8sojlH4oP6g19QST+9fO/wAebfHiixugPlltApPurN/QinhsS51rMcqXLBsofBK/Fn8RPKzgXVrJH/J//Za+jZJ+etfJngC9Nh4+0SccZuFjP0b5f619SPJx1rnzRuNRNdUb4SPNE2VO5Vb1GaWoLR99rGfbFTV103eKZMlZ2KlymH3etQ+1XLhd0X0NU67IPQwmtQpynBzSUH2rQzZNmlpgOQDS1ZA6ikzRTAdnmlzTM04GmIdUch2vEffB/KnZqO44hLd1Ib8jQwJTSxfe60080sX36ALm7Heik3e9FFgPkH7FK7i0QKZSdmM8Z6Vv+GdKutMguIrkKAzgrhs9qzopxFqSXDBsCTcQOvWt0a9bbc+XLj8P8aI2WpMrmltPvSAc89KoDW4B92OU/l/jSnXIGPKS5/D/ABrTmRHKy7gYo74/kaof2zbn/llL09v8ab/bUGeI5OfYUcyDlZo4rVtpTJbpknK8GuX/ALag/wCeUv4YqzbeIoIg26GUgnPGKmTRcVY6YHbzTTjg5NYZ8UWgJxBPgew/xpp8U2p62836f41maaG9nPPHA4qvfnNq3BGcDge9ZH/CU2o4+zzHn2/xqrd+LLR4Sgt58nvxj+dJ7DTRVumA8VaIRwPtUfB7/OtfQiivmuPVI9T8U6QY45ECXUfDYyfnH+FfSq4xWUi0LTT0p3WmE0ITISOaaRT2xTDVmZh69rVtoBtbu6XdGztH94DBK57/AEqlF8UNDjHzYP8A22Wuim0qw1bEWoWcF1Enzqk0YcA+uD35p0fg7w4vI0LTRz/z6p/hWkU+hLaOfb4saEBwE/Gcf4U0fF3Q8dIv+/5/+Jrq08KeH16aJpo/7dU/wqwnhzQ1+7o9gP8At1T/AAp+93FocUfjBoi/8+/4zt/8RTf+FwaOc7Ra4/67t/8AEV3q6HpI6aXZD/t3T/CpF0rTl6WFqPpCv+FHvdw0PPj8XtKAzi0A/wCu7/8AxFXdK+JC6/efYtMs4Li42F9izMPlHfJUCu4XT7NTxaQD6RrVa80+1S/srgeXblC4LKFXOV6cilaXcfMuxkjU/ETjcuhREZx/x8ilGoeJP+gJAMf9PA/xrZglsLSFIPt0bBc4aScFjk55OfekbUNNQkNfWw+sy/40/mL5GWl/4j6nSYAP+ugP/s1WBfa+3/MPgH4g/wDs9WG1XSBwdTsgPe4X/Go21vQgQTq2nj63Cf41Lj5jT8hv2zX8D/QoB+A/+LoNxrxX/j2tgfcf/Z01vEXh8ZB1jTh/23T/ABqI+JvDakZ1nTx/23X/ABpci7j5vIrjUNbe/ntNlorxIjndwCG3Yx83+yaZJFrch+Z7LH+9VNfFHhmLxRc3DapZ+XJZxLvVsglXk447/NV5vHHhRef7XtvwVj/Sp5E92VztdCD7FqvGZLLr/wA9P/rUybTdVlxiayX/ALaE/wDstTN4+8KL/wAxWL8I3/wpp+IXhQY/4mYJ9oX/AMKn2MO5XtZdiqNJ1cN/x82eenDf/Y1NHpOr7cfarVfxz/7LSN8RfCw/5f3P0gf/AAqNviR4XB4upj9IGo9jBh7WYT6NqYcIdRtgzgkADJOOv8NcdBP4vYF4vD98ynkMGiww9Rmusl+I/haQDzGmkx0Bt84/Ouq090m063mj4jkjVlBGMAjIpexj0D2sup5eG8bE5GgXynPeW3pWutetYZ7jXLCa2t4o2kUPJEzORycbPb1I6j8PVSAe5NeYfFbUPJ02aBHmUkJCCp4BJ3Hv0KjBrjx1NeyVNbyaX37/AIXPNzWrKeH9j/O1H73r+Fzwy9uHnnl8xmdmYlmYkknufemWkgK4YDK9eefSmTRsz7N3zgce/wBKr25ZbohgQcE4JxzXprQ7bGmPvA5PXp3x/n+tSnaUG0DAOeB3qszLsOfTAA6/56VYQBk3AD5sH35/yBVAepeFLnz/AA3DLHhpFjCk46sB3/StJHkntpPOUb1OQB0x/nNYXglpP+EVwd3+vcRseNwyOQe/fn2Nb6SpC6JPKhaQ7FJP3vb9K5nF1aOIxS+KE01/25GP5q6Pm6sHJV68d4zTX/bqX5q6KZVEUY5djnBJHGa4Xx3bjzre7AOG3Ic8e4r0DY6kq+0lRguABn/CuV8bW/m6M0g6xMHBH6/oa+hq8tWhzR1TV0e9CaklKOzMS/T+0/hdbzcNLplzsd5OTsJwFQ88fPHxx932FUfhnpjal4705cjbCxuGyeyjP88Vp+DU/tLTtc0TO9rm3LwiTmNWAI3fXJQ5A/h9hVD4WXS23xA0/ecCQPGPqVNfK4b3J1aXZ3+Utfzuc2X+5VrUe0rr0lr+dz6B1mES2yZA4cHnntWfb6eCgIOPr/hVrxDq8Gj6JcajcRSSQwBWZYwN2CwHGT71xUXxi8OIAfsOojntGn/xVceLoVZzvBH1OGrwhTtJnbR2Fx5saqU8n+LIOfwFakVqCAB07154PjZ4dU4ay1L/AL9p/wDFVOnxx8KqB/oWqE/9c0/+KrieDxD3iXLFx6M9HjtRkYP6VzXxQ0iS88BXZiTc0LpKMdgGwT+RNN0b4w+D75JGka7ttgyBNEPm9htJ/XFYvin416Pc6Pe6fY210pmhaMOUUjkY55rehhJQak9zlnWctOh4tZeTb3lu1wCY1ky6r6Zzwf8APWvsLT7/AO32NvPYwt5U0ausknHBGRXxXPfrM3yqVOc1794e+O/hnTvDOmWV3b6h9pt7WOGXy4VKllUAkHd04r1Z05NpnK2rWPYPsIkbdcyGU+h6flU48uJcKoA9q8mb9oTwnji21T/vyn/xdVn/AGgvCzf8umq/9+k/+LrNpw+CGpNm92evPcADrVaS5968kb4++GCOLTVP+/Sf/F1A3x48NN0tNT/79J/8VXJUjiZfZZrCMFuz1l7j3qs9xXlTfHTw2f8Alz1L/v2n/wAVTD8cfDh/5c9S/wC/af8AxVcksLiX9lnRGVJdT1F5Sa8d+Odt5tlpVyB9xpUJ+u0j+Rq8fjf4c/59NS/79p/8VXLeOviPofijRY7S2tr1Zkl3gyIoGMEdmPqK1wmGr060ZSjoOrUpOm0mea6fcmz1K0ul+9DMrj8CDX1sCXAK5IYZGK+QMjHGc54r2WD4yWEemWtu8d6ZUhRJPLRUXcFAP8W4jOe4ruzHDTrcvIjLB1owume46bkWpRuqsRjPPrVyuA+GHi6z8Tw6klrG8Zt2RmVkC/ez0wTn7veu+opQcIKMtxzalJtCMNyketZ7cEj0rRNUZ12ynFdNMxmMzRmm5pa3MSRD8o9uKdUaHqPen596tEC0uabmlzTAXNKKbQDTEPzSONyFT3GKTNLmmAkbbolPtUkX36hi+6V9GIqWP71IC727/lRTPxooA+Rr9mgtXdDh1AOce+K008OaoyA/b4BkZ/1f/wBas/VVLWEox1x/MV6bo+iXmrnZZwNKyLlgoHAoik9yZtrY4geGtVyP9Pgx/wBc6kXwvqZyPt0Of+uddnLaSW1wYZlKup2sGGCDXYeHvCA1W1M5k2KOBkZzVuMVqQpyPGj4Y1UjaL6ItjoIjSSeG9VTh76NSRxmGvTdf0VtJumi3dO4rm5cseSSSMc9aFBMOdnJHw7qRxjUoun/ADyo/wCEd1FsKdTiGD/zxH+NdRTT04HNPkQc7MQeDNWbj+1ouf8ApjSr4L1I5J1iPH/XD/69dfav5lsrZ5HymrKqAefxrJqxutUcR/whF+TzrSfhb/8A16gufA12kRdtYDdAR9mx3+td6RkHJqvf/wDHo7HHBXp9RUvYpJHl2n2b2Hi+0tnm80xXcPzBduckHpX1Gn3a+aJDjx7EAf8Al7gPH/Aa+l0GFxWUhrcWmNUn4UxqSBkLVGalYZqM9a1RmypeaqNIRJmhaXe2zCkDHBOf0qr/AMJsM4+wPwf+eo/wqv4oP+gQHOP34/8AQWrmM4NbQ2MpvU7EeOcf8w5vxm/+tSnx4QONO/Ob/wCxrjN36UZ5qrEczOy/4TyX/oGoP+2x/wDiaafHtwemnxf9/j/hXIZ7UmcZxSsHMzsP+E8uv4bCHP8A10P+FYXifW7rxRaW+lywxQRSzZZ0YlhgE9/cVmhj1xSBv9MtDnpJ/Q02guzGm8H28DhTcytnocD/AApo8KWn/PaYfiv+FdRqX30z3HFUWB42nFS4RKU5GMPCtiOss5Ge7D/CpU8KaaQctOfX560DuHXtU8IcnOflFDigUmZq+E9NH/Pb/v5Tv+EU0zukn/fw1rBvan7vxqeVFczMD/hHdOTVGTy5DH9lD7DIcZ3kZpl5ommxQ747fGD/AHzz+ta7/wDIUTr81o4/KRf8aiuwGt3GOgoUVYHJnHvDaxaoI3jAi2FsFjx+tVjEsqTyQn5I8kHPJP8AhVjVsGeGJSN0rhfwHWoYAFTUo+eCePbFTYq4lyI10yGZVwxIyamv4UW2RoUA+YE49O9U5X3eGYzjkAd8d6dqdyP7NjXPzMoOAecUWQXLcUSTzRymMLCXAUD+IfSthfF2vwaxLYJqtwttEgEaKQAoHHXH+cVmtIFtYiP9k5HTpUUcS3XifUeoaJjjae2TU3SKWpq6j4r8QRS2uzWrxVZ8MBJ15FQeP76aQ2NpPNJJOsXmyszEhzwoPPU8N19ferdrpUF3e2u47tkgYAnggckY/Cue8bTbtfuCWYokajDHpjjA9uv51wzkqmLhFfZTf6L9Tza8XPHU4fypy+/3V+py06wKm+Utnrgd+tVIG3yljkKBwPTmnCGS5fcx2qTwM/5/zikiOy9ZSQflxx2r0T0SxJwCSQM+n+f8/jUtqU8l1IHyHnp0PT+tV3bIyTx0pLeUJc4yMMME46H/AD/OgD1nQX8jwnpiSkxvIzsA55ILMc89QQR+YqS6Jkuk8vP7p1YcZ5z2FN3x29hpNvKo+0xWakoV+ZOFHI6jkH/vk+lSK6WtqJJR87kcH1z/AJ/Kt8kip4Rt/alJ/e2eTlqUqDm/tSk/vkzQvvO85GiPyFc+meg/w/Oue8SRyPpMkbqAzoTzmuoUOtovmN8yMQx/Mf1rjfE73eomWGBzGiA8j/P61rlsnHCOjLV024fdt+FgyxtUfZPeDcfu2/Cxyngu9Nn4msZC0u2RzC4Q9dwwAeem4g/h61dstKktfizDZwIiFNQWVFiGFEZO8DtjCnGK5S3d7WY7HKSxtkMpwVI7givVo1aT4keGtYtIFMOoQcjGHGFJLNjjIVlHU/dx6V4lX93jIv8AmTXzWq/C5rL93mEJdJxa+cdV+DZ6Xq+kLrmkXWmNJ5YuY/L37d23nrjvXJWfwAgbBbWJGX1NuAP516FaSeXcxsAmQf4/u/jXRGxubs5vbsCP/nlDwD9T1NTWc+ZcrPbi0lqeR6n8GfDyWBtbbULqbUcgL5SKR75Hb86q2H7OqzDfe63JED/AkIJ/PNe4Qx2tmmyCNUA9BStcj1rNVfZ6yl8kJvm0SPIW/Z40lR8mvXyt6mJDVG8/Z7sY7aV08QXTOqEqDAuCQOO9eyvc+9VpbjIIJrCpjrfCVGnJ7nxReWhtH2k7s5HTHNVK3/E0DQ6tdQlceVPIn6//AFqwK9enLmimYTVnYKKuWWlX+pFhYWc90y8ssEZcqPUgdKvr4P8AEj/d0DUzjr/oj/4VTkluxKLfQxKK1R4Z10w+d/Y2oeV03/Znx+eKkh8JeIrkZh0PUXHqts/+FLnj3Dkl2MWiug/4QbxT/wBC/qX/AIDN/hSHwP4oH/Mv6l/4Dt/hR7SHdB7OfYwKK3x4I8UH/mAah/34b/Cqmo+HNX0iOOTUdPuLVJCQhlQruIGcChVIN2TBwkldozeNo9e9eg+CPh1aeLNCfUJNSlt3SdoTGsQYcBTnOfevPa9d+DV/ix1WzJHyyRyqPqCD/IVlipSjTcobm2FjGVRKR3nw28FQ+D9VvDBqElwt1CAVeMKAVOQevua9KNcjo9x/xOIRng5H6GuuNcdGpKavLc6q1OMJWjsJVe6T5Qw7cVYpsg3REV0QdmYS1RnUvalIwcUldJziqcOfcf5/nUlQj/WKfXI/z+VSZq0SxSeCf5VF9pG3Ijk64xjmlcOSNjgDuMdaRVmz80ufotUIT7V/0yk/KnifrhGOBke9NCS7T+9+bscdqNkvH74gd8DrT0ETRvvXO0r2wafn3qsI5f8AnufyqcUAIvErj1ANTR/eqAnEy+4IqxF96gCzz7/nRS4NFAHyZqeBbMeOWGB75r1Lw94kvNBLSWhTDqA6uuQa8q1U/uOn8YHI969F0yMS3cELMQkjrG5HoTShs7kVHbUuXepTahqEt3cEGWRtzYGBXRaF4vn0qAwoEdD/AHvWuV1RTBqVxAz7zDI0QPspxVfzCp+la6NGSfU2dY1iTVJ2llYFmPUVjOc5Oc+vfNG8txnmkKHFNARHjOSMU0jGeR709l2555FRnb+ZpjLenybZjGeQ3861dmckdq59WKuGBwQc10ELiSFZB0YA+lY1F1NqTvoG09+lVrxT9lfPIBB/WrrDI9AKq3mBZy8HgZ/WsW9DdLU82m2/8J3ER/z9w/zWvphelfM84K+O4iwAJuYen/Af/wBdfTKjNZsFuwNMPtUhFMamhMiYVEamYVG1aIzZg+KB/wAS6IjtMP8A0Fq5Mg9zXYeJE36YPaUH+dcps44raD0Mp7kWCOp/CjtyfepPLycU7yec8YNVcixAR70uD3qXyiTjrU39m3h6Wk/PfyzRcLWKncc0Y/0i35P+tX9eKujTrvvbSge6kUyS0eKS38yMoWlTGf8AeFDulcUWm7FjUVDMhyeh6VTaONR8zYz7962NTjRBH9D/AErJeNJTg9BU3uy7WRWzFu+8PwNWYWhKYL/OTwKhNtGCOf1qzBbwEb+cjkc0MEP8lSM80ohQ8f1xTwQBQW5PPFRqXoVVhRtcsEb7jpKhx9UNdp/whumSIys9xzwcsP8ACuNLhdX05x2kkGT/ALgP9K9YTBwRuyevFNDsjj/+Fc+HJZkkeOeSVT8rGVhj8hXFnwzp9v8AFWbRWiJsbiNG2bzk5Q9+vUV7INolChjya818QMtv8bdKcnHnW0a/U7mFDGbJ+HfhdIPs66fmIdVMzkfzq5H8PvCroA2kQMD13lmP4c/pW4EAJ+Xdz3GanjIWUKMBQfaiwz571q3htdSvbaAYWCZ41X0CsQP0qjDKLbxNqEvdolf65Uf41u+Ldo8Z6nBty3nscgYznn+tc1qj+RrrN8oElvH36/KP8K5ZO90XFW1O08PXdvN5kYQrcAZOe6+3pz/n0858c39vPrUv2NmCsBvJP3jzyPTr/npXb+Fz/wATGQA8eSf5ivNtUkt4L2Z5FLzs5OCMnr19q87B0UsbUnd7L8f+GPGoYeMczr1Lu9o9e9/8tOxnxQEr5s8hEYGcA1Fa/vLhivBY4A696iuLmWY4Y4XrtHSprFVO5n4UcHivaPVLMoDdDnPXn60lvAZJkSJXeVyAFQbic9h79sUjuZGIToegxnitjw1bO+u6adrs32hGAAycBgSfy5P0qKkuSDl2RnVn7OnKfZNnealNEfGzxhwWFjGCAR8pDscH3wwP41O4S5uFRgfLQc471zWqzL/wn18Y3jO5VXOc8hVBH14IrpNKxJy/CL1z3/8ArV3ZFHlwdOPkjhy+HJg6S/ur8Vc3reUTxgNwXQZ+uK568uFLPEI8uSR0PFXrYXP2hJ04ibBCBeg7/mKg1+a10mOW5cZeQ/IgOSSevWnd0swmnoqkb/OOj/Br7jOC9ljZR6TV/nHR/g19x5PqkMdjr8sb7mjDh3CttODyQDg4/I/Q17j4XlsNXs7CXS9hgtwDGH5aHaMYPOcgHHXkHrg5rwG/uJLjUJpJiN7NzgcV6D8G9QWHxRPaMwAuLYhR6spB/lmvIx+DjiJxk21yu+hvisvjip05Sk1yO+n9fie4kds4rdh1DzLeNwfvKD+lYfSorK5Ig8sn7jun4BiB/KuDMW4xTR7+GgptpnRfas96Y1x71mrMfWlEjHpzXiupI7PYJF0zk96iaTOeap3V5bWMXmXlxHAnq7Yz9B3rm5fH2nSXQtNMja5nIOGf5E4BP17e1ONOpPZBaKPGfiFb/Z/Euqp6XLOBj+9z/WuGPWu08ZahLqurT308aI9wqvtQYHAx/T+dcYww5HvX1OGuqaTPKr/Gzt/hj4hm0DXbowwrKbi3KYPYhgc9frXvmk32oazpVtcSGKNZQQwK5KspweOnPXp3r5m8Isw8S2qq2C+5M/VTX0F4SlltoHgaZnjB3KCOAT1x+QrizCmr8y3OvCaqx1S6fAHEkxadx0aQ5x9B0FTl1UYUAD2quZ+M5qF5h615d2zuVO+5YaXNRNL157VUe4A/GojcjcBz1oUWaKKRZaWvOvi7H5/hi2lAJMV2ucehVh/hXaPc+1cn4/H2rwdfr/FGFkHttYH+Wa6cNeNSLM66TptHz+PvjPrXdfCq6MPiW4hJ4mtmGPUgg/41xEw2zuPet/wNc/ZvGNi+cBmZPzUivcrLmptHiUXy1Ez32zn8rUbaQnpKufpmvRDXkpuipBHUcgivV4JRPaxyDo6Bh+IzXnYdWuj0cRrZi0Gl7U2ulHMynIu1yMUyp7oYYH2qvniumOxzyVmB4wfcU8/rUbHKHHpT85/GrRDFopOpoqkIWlptFMB+aBTaKYhJGxsb0YZqzGPmqtLzEw74qxCd20jvzSAt8ep/Oik5opiPky+Ut5SDnMyDP4iu+jZYLhGVidjBs4x0rhLgZubUcfNcp/Ou2yu7qeOaUNUyaiuX9Yniu9ZvLm2JaKWQupxjryf1zVLOB7U5GRT2xnrimk/Odv4GtForGSVkXbOONpl3kAd812XinRdG0/SLaWynDTvj5QwO4Y+97VwKy4OQcH1qR7h3ABNDV2NEUmMkCoiDTy2ccCmtyTVAMOc9a19Lm3QtGf4TkD61knnr296sWMwiulJOFPyn8amaui4OzN0gkZqpfMPsUpIz8p6Cr4XnpTZ7KKe2mLwvIQpJxIVyMdK5JOyOxRuzy2+AXxzFz/y8Q5/8dr6ZT7tfN+utF/wsGPyYBAvnQfIGZv7ueSa+kEHy1N7pC2bHEVGwqWmMKcSWQMuOaiOanYcVGRWiM2Y3iH/kFtxyHU1yRPHvXb6paSXlm0EWN7EYycd6xV8Mag3eED/eNVFkuDexhqzc9OO2Pejk8nt7Vvr4Tvu8kI/4Ef8ACnjwje85nhH4n/Crug9nLsc6Dgjn8cV2qR3kuoBI0/dGPd/qzjp13eue1Zw8HXPe5iH/AAE12kKmKGJDkkIBkH0FS0pMTg7anLTWV/v2lHC/7tZOtxvGbIyRsreb0YH+8teiA56A/nWJ4o06O8igkdnUwhmGPwPP5V0Trc0OWxjTw3LPmTOW1wbVjOM9awjJzwK7fUdJiuAm93A9jWcPDNlwTO655wzgf0rFWS1NZRZzJORmkDkDFdO/h20BUrISufmy56flUkHhy0mfYiMx6/eIrVU21foZ3V7XOU3tyeKPMboTxXaf8Ich/wCWKj6ymnL4PiAx5EZ/7aGlyruVys4aR/8ASdPJPS4I/ON/8K9aieZlBVUxgYyT6Vwerabb2d5pypGqkXqBgD7MP6118L6m4iMUdsYSowXY5xj0FZNamqWhYdr1pMjygO33q858dubX4o+FbgoN7iJGPbiTn+degyf2sCAhtFUDJ+Vq5vxPYi51LTbvUIraWeJv9GlVWGw5B6H8Klqy3HGN2da7bZHGO9PhVT874Az6dazmtdXJBF7bA9/3Wf61KkWqBCkmoQkEYAWAZBPfrSc2uhXL5nkXj6MJ49v3EbAMFOSOuUFcj4hRvtVo0YO37Op5PoWHWvYPEmlKb15LqOOe42jMpiAJ44rjDapLaQFoo3G0qd45+8fX61yc3vM2VPRFfwt/yEHPbyT0+q15c/2kytLPImScsxUE/nXt9hbrC4IjRSY+oH0rxq5tWd0Qqdi89OprLCL/AGmr6R/U8iCtmFf0h/7cZLmOV2ZITjrnNTWqgwkFeR2qa4McWFVMM3AHTP8AhVW0kG7npntXpHcXUi3tluFHbmuo8Gxu/im1ZVLKgdmIHQbCAT7Z4/GuZWQ5HIHrzzXY+AMx6tdTNlLeO1be54RfmXqeg4B/L2rkzCXLhKj8mvv0PPzSbjgqr8mvv0MxYLa88V6nLJKvk+c7I6v1y57jqMfhzXY2uoWUY8tP3i8Zz0Iry26mMVxFiWRAwPzIMnH4/T+dSLeSEj/S7tsYY8hcc+3+Ir1MNifq9NQS2OuFFKKjfY9fg1OMxMI1BdQQqDqeeBWX4htY73SrO+1GeCxdMq/mAsDnoOxzwTj3NcjBdLdxX0UUMMU0do00EkckxYsrDpudlPBPYVY8Pm51bwv4g0e6E0khj+0wEAl3YAHGeQeVTjryfw4cwx3M6da2sJfg9H+d/kcmZUvZKniF9iSv6PR/nf5GBrNppjLNNaXM0kqEctGEUj6ZJqt4Y1f+xPEdhqGDthmVmPP3OjcfQms6WzurdDvUgdwDmqynBreclN3segtD6+VlcB0bKsMqQeoqpDC/n3HZBLnJ4HIB/rXPfDbWDrHgiyZ23S2ubWQ5/u4x/wCOla6KTTDeXJd53WEY+RT3rycfBOnqehg5WnoOl1OytMBnMrj+GMf1rH1vWNX/ALLluLOE2kUfJbGSRnHU/XtXSWunWVp/q4V3f3iMmlv/ACp7WSGRFZHXBUjINeRHki9Eeg7yPF7lri/ZmnmkmlbnLEmiy02a3uEuB8vlnP510k1rHbzsqoFAPYVXlT/R5B0Ow/yr0o2toYNanA+M9O/s+5gQEsHhyM/U1xEn3zjp9c16d48QTWOlXGMlkKk/l/jXmcow1d+Hd4Hn4hWmXvD83keIdPk6YuEB+hODX0RpT+WT79a+aYZDFPHIDgowYfga+jrKUMqMucEZrHGK6R0YF7nQi5+X9Ka7SFN+1ivrWZI87QObbaZQMqG5B9qrzXOoRTsUZXi/ikfapAyC23nkben19q8zkPTuaLzYqLz8HI9c1zn2zU3QjcqlWIdyyYQbup9DjH5niqzX2pkbkmh27QTudMZ4zjn2P51oqZPMdNNJh26YyaytbT7Vot9bkZ8y3dR9SpotJp3iLTtuJY7G45XjH3eO9PZ+CDjB6mqirMmTurHzzcD95n+8Aat6RP8AZNbsLg/dSZGP0Dc1FfwmC4aI9Y2ZD+DGq+ThG9OP8/nXtbo8LaR7y74LDivVfDk/2jw9YyZz+6C/988f0rxq2u/tNlBP2kjV/wAxmvUvAtz5/hxUz/qpnT/2b/2avNp6SselUd4pnSk03vSnrSZroRzsjuBujPtVI1fPIxVFhtcitoPoYzQn8qRP9WueuMUdKF6Eeh/+vWqMmOBozmk6VBfSyQWNxLCFMqRMyBuhIBIzVJCLPWivLdM+I2rX2qWlrJHZxxzTJGzCNsgMwBI+b3pNR+ImsWmoXEEKWzRJIQjPAyllzwSM8Vt7CV7GXtY2uepg0uawfCetTa7oMd5OIxNvZHCDAyDx+hFbmahqzsWndXQ/tT7PlFH93j8qizT7PiSRfRs/mKkZocep/OijNFMR8oTHN9ZdcG5Tj8a9AtNGv72wuL2C2LW1sMyyFlAXjPc8n2HtXn/LanYD/p4Xr+Ndl9ol8ryvMcRE7igb5SfXHrRT2JmPEgB5z+Bp7yRY+VfxNV8+5pygNjLY59K0IQu4c5P0FAOSf0p7xQiMsswLZHHrUWOnfNFwasakWiX1xo8mqxW++0iYrI6sCV+q5yBz1rMZievT0NSRuRGU8xlRvvAHg49R3qAnk8UIBc96aDzmgnOffpxQOnrTA6yxbz7WObrlRnjv3q1LC4tZOCTtPFZfhmUOZbdsZ++ue/rXWG1Zrdz32k8968yvLkbR6tCPPFM8S8Qqw8fx7+SJYePT7tfSUf3RXzv4wjCfEZevMkBOf+A19EJyopxd4pmM1abHGmMKkpjVaM2QsKjNStUZHNaGbCJd06D3q75APSoLRVa6QNk9f5VqiNfTms5T5Wb09ikIKd5OOc1daEK2MUnlAnpS9oaaFPyhz0NSbeg54FT+UewrB1M3iarDDBbPJE7YdxkBen4CnGq72SuZVUrXNkAZ5IH5VS1kB7NuQSFPQ+1Yt7Y6gQfIEhJ9z/jUlhbXccdwtzC6jy/lZh1613Sp/uue5xU6zdVQ5S7crmNDjrTWsZ7iG3aBGPOCQQMc9TnqB6Vbdd8EWeeB1+lXrIAWKfUiuVWnozrqRsjIu9HuicxkkYPRv6UabZXdrckzRERlcbye+a6AAYzgmknH7gY4wwP612Ks1DkONUI8/MZ4Y5OWz+P/ANekVv3v3xge9RNG/msFiyO3NKsZD8xAe/Wsrm5yPiRt2orwPkvYWH4uB/Wussiws4WCj7gHNct4gTfqU6DORJA//kRP/r11FkN1pEpz8pYH8CRWEKl5tG04WgmWJGJX5tq9utc94rC/Z7CQZO2bqO1dFMBtUbiD14rD8TjboyPgnZMpyaK/8Nio/GjYkHKtnqKVSO5JP0pqSZhiYH7yA4xntUiyMccH8q0WxD3MXxFb5uVI6MgzmuH0+082Ex5VNs0ik9hzXpmpQiYRsUPAxXIafpRS8vYJFfidmUgdiAa8itV5askehSV4RM5YXSMfLxjsK8h1iayk1WeXTonjtAfkTHBPcqMfKCeg/l0Hv76WpyCrnj0r5/v7B7LWrvT1Ry0U7xBdpJJ3YGBSwFOHt51VJ3fT+v6R5M8rjQxc8UpNufS+i/rp21Obvomb94eGHPHaqcGNwGB9a7O+8E+JHtjIdCv8Bd3MDA/ljNcWqFXZX3KwOCCMEV690zWzW5qxEBRx15xnPWux8MMIfDPiCec7IWtwm9uFLbXG0H15XjryPWuJikLJgHdnHHWuy0+SMfDTUld8F7tVAJwSf3ZwM9eAT+BrjzDWkod5RX4o8zNFzUYw/mlFf+TI5RIYb2VWdXkAG3g4z/nNa0ekadlWMWB1wT2qjYIgPAxnJ2jHT/CttY/NQgMVOOAPX8K6mz1UjTkvbtU+zvd3bxYx5bXLtH/3yTj9K0vDF5ax3rxzbluZBtRyflPT5fz6Z/TvjmJXyShJPOC/4/4flWjp1ultcxTyAJtZW4ycAHmuLFUFWoyh3XQ58woLEYWdJ31XQzfHFpaT6kIreJkmc5mOMA45yB75GfXr654PVNPNm+R90+2P8969I8UvJb65MI4fMnmRSnoq4xn25B/KuTvNEuDvlmmyzDLKBwKvANfVqduyMcuS+p0mnf3V1v0Oi+DGsm11270l2/d3kW+MH/nonP6qW/IV7gr7Dj8a+W/C1/8A2P4w027LYWG6UOf9knDfoTX1BKrbhtBOOtLGxvTZ6+DdqiJjNVO7n+UilVsts7+g61T1ENHjIIyO4rxYxuz2NEYF+v8ApDEd+azmZclSRgiqPjqGe40+A26yOwkBIQE8YNcGdPv2PNpck4Iz5Zr1KVNON2zgrVHGVkjpvFSibwnZSDnypivHpz/hXmFwCHOeuSK9Jkgl/wCFfzwyIyPBPnawIPUf415zdriVuO/pXTh9Lo5MRq0yuOte+6BN52iafJuyWt48n/gIrw2x0u81FZjaQGXyV3yYIGB6816/4LuRJ4T08kklVKnPbDEfyFLFK8S8I7SZ18Mqg4LbcjhsdKzbnS7OTKea7qSxKgAKNxBI6egx9KkMp3Ecd6haQkYzxXn2PS5hLe2jtCxV2bcAo3Y4GSf5sarfYIEZSryAqu0cjuqr6eiipS46Z/GoJLuCPO+VPlxnn1qrMlyRPGRBBHEmSqKFGfT8Ka0hPas281mG2G0CSR1baVUd8Zqva6ubi8MDRlBk7TnOSOtPlZLmtjzTxTAIdfvlHTzy34N839ax8/useh/z/Kuq8dweXrTyY/1sSPn6ZX+grlFztYfjXqU3eCPIqq02ep+GrgzeHLJs5ITafwJH9K9V+HFxut9QtyeVdHH4gj+grxzwTJ5vh8rn/VzMv8j/AFr1H4ey+XrdxF/z0gJ/Ij/E1xSVqjO6LvTR6UaaaU+9HatTFjaqXC4fPrVuobhcpn0q4PUiSuipnpR0c++DRQ3DDryK6EYMXNIQGUg8gjBpKUc1SEeH6ZAbHVtUiiRXvbaOQWwZc/OrgEgHvsDEVZuUbxPZ3GpeTHb3UcuNz3B2Sg5ZgPMY4K9cDjB6dKv67LY6d40vUu7ZEXeJY5o1Yybmw394DuenoKiu/EOk6jsN4biYouFDxbsHPPVu/HavQTbszjaWx0Pwuud2m31r18uVZAP94Y/9lrvhXmngTU7d/EtxbW0SR28sGUxGFYkY64+pr0oGuWt8ZvT+EdT7ckXZHZk/kf8A69R0qHFxC2eMkH8ayNDSz7Gil/H9KKYj5Qj/AOQnYAHrMDXX4O3jnPeub0a3S88UaVbv915+cH2NdpqWnS6bMELCSNslWPBqYTSfKOcG1zGfgjtSgE8YpRkf560fN7elbGNhxjdSoI5YZGDR5Tlgvc+lJluueKUs3Bz+dIAMQVjk5A9BVmKxVkkkFxD+7UPsYkM2ew46iq+5sYzS5OM5oGhmwk9COfSlWJifunrS5b19qkQtnqaTY0jQ0SKeLU7eVYmYBwGx3B4NeuJpDGLleo7CvM/D8LPdKWnKYIJx3r3CPBjUj0rgr0vbTtc7IVJUoadT5m+I1i1p8TFwpUFrc5656f5/CvdEHy4ryj4yOg+IGngfe8qHP/fbV6wg+WiMeWPL2Bvm1YtNan9qY3WtESyJhTDUjUw1Zkyzp0Ze8Tjpk1vLbIp3AfN25rI0hsXmPVTW9S5E3dj5miHycnOaPJFTUUeyiHMyEwc5zVG6jKTAetalZuogtIFBxwORVQgou6E5NkQVvX9abcKpgIdgM8ZNOUD1z9ap6u23T2Y9AwPH1p1NYsqHxIkkiWOJUYj5QM/lToLiFLcAzxqAe7Csu9vwLHzfUL39q5X7e0spTBCnPesKKd7mlWWlj0aK5gmfbHNG7DkhWBqrdazpsavG17biRGAZTIAQQea4XQdbNvqVwWjUKqFS2evINcF4kvhLr+opg5+0SqenPzGrdVqbiQoe7c9tOuaaF3f2jaAHpmQCoD4o0QcHV7Mn2kB/lXzlPFufirFkWjYKFGG6Z/pQ6zQKCPU5tTh1LxtfeRcia0+yK6Ffu7lZSf5V1ltqdtDcS2rT4kFzKuCpPViR2xXkfh6UwalcY4/0SYeuflJr1rRpd66hKDy12zfmq1z0m3VujeaXs9Qm8UaXG7AzztgcbLZz/SsPxX4js5/D8kMUV8ZS6MC1s6DqOpIxXX5Gc5O761l+LIRN4RviMlgobJ55DA12STcWmc8XZ3RFaavKtlAsmkamXSJVb9yAMge5q3HqszgFdIvcZ7mMY/8AHqvqS8aHsVB4p23K425HuaST7g2jS00edZCSWBo2JOUcgkfkSKjt4bdtUu/kXcdrdPbFRpdeSmzpxnHpWRZ6gY9emLNw8S8fQmvNxFeCqKLW25vTpycW0dSbWHn5BXDyeGNOtfH93rIt1NxKkewnoh2kMwHqcAf/AKzXUHVBWNreoCIJdoMSAbN2Rgdxn9fzprE0XJKCsVTpTT94i1C6ku9Pu1sfnuo9yqqkAhuw+bA9PauD8afC+fXvDcmqR6OIvEMcQdzHIo88j7wYAkE4zg9c45ruIblbuBFt3kMxO9cMMOp6qCehzyK6GC8kNpGJYz5m0ZDMOtdcXFahVT2PiqBlS4UTIxVWG9c7WxnkDrg9exr1fUbvQW8EQutqxsZBstoVXayyc9+dpBDZJznn72cHz7xtcpceOtbmTYFe9lOEBAzu9++c/jmuhlXzPhlo4A6XTHH0MtceY0lUlRbbXvLZ+Tf6HzebYdVKlC7a99LR+Tf36b+pkWcSqo3sNvoD37/59q1UeNIy5wFwFx/erNt0JxycZwwOMAVMJTNeiFQSkY3Ee9eie4ake4BWHBUKMdMVJdzAQnzZvKi/5ay55wOw/GqKmZVPlW7ykkhVVe/Pft/nms2bfc3my5xeXAOUs4SSie7kfyFJIbZ13iQ3F9Dp2oWk0dtDcwBnkZcuowCoH13Htx+NcrqP2i3sZ47KEHIO6aRg7OO+38q63UIrt/DOmSy+cs8RMTRWirtI5xkN6bRznHJ4rGZCQ0k1skUJxuCsZD+IUECuLAe7Q5P5W19zZ5mUaYb2f8rlH7pP9DzXcd+7PzZzmvsHwbexX1hZ3U6qwuLOOXkZ+YqpP8zXyRqcKQ6jMsUbRxbsorelfSHga6ZfAuizAn5bVVz9Mj+ldVf+G2j06SvLlPSo7PSoblrhYUEh9TkD6DoKwPG0kBsrYxKgPmEEgdsVTOqN/erJ1q8NxaqCfuvmvHjVlL3bI9OOH5XzNmLK25AT6VUYYPIzU27dHjnhjUTDJBPT2roiVIx9Sj36VrUOP4BJ+mf6V5HeAhj15ANevX9zb20t3FcybPtNoyqME7iMjHH1ryK6GfriuvDnFiehu+A4pbnW5LSKQRtNFjJ6HkcGuu8Auf7Entz1gunT+RrzTS786be+fs3jaVK5x196734dXfnnVQAVBlWULnON2f8AAVpXj7rZGHl76R3M8vlQvLjO0E4H0zXPHWZ2ViYhGwGf1H9DXQTpvtmA/iUis3TbWxuYwJwNzQPyGx8wXj+XSvPukrs661TkVznpJLu5gaSa4k2KQ2BxwWwR+FQ3KGN7gBiW3ujEnqAAwrV1FYYmuoLZt0DoWUZ6EDNZtwd8xP8Az0aNh77lwa1jqjNSurkQvN0xuW5w8bn8sGrtvIp1hMAgmQuM8cMo4qiYBDbRzKSfMj349CCv/wBerjS/aZ2uYFkwAjD5epBwabKRk/EGPM9pIBjdG6k/Qg/1NcIvU/SvRvHaLJYWkmcsspXrzhlP+ArzlMCQZ6ZrsoP3EceIXvnb+AJswX8OejIwH5g/yFeoeDZ/J8U2mTgOGQ/ipx/IV4/4GnMWrTxdpIT+YI/+vXo+iXxj1SwugrIFnRirDBA3DINc1dWqXOmi707HuNJmlIptWiGHemONykU6g1SJZnkYNMY8r9ammXDn3qCXhM+nNdCOeQ7NGeabmlzVEmdf+HtJ1O5+0XtjFNMAF3tnOKgHhPw+p/5BNr+KZrYpc1XM+4rIoWmh6VYTLPaafbwyqCA6IAQO9aOaaKM0O7Cw/PNDHam7+6Q35GkFOVd6smeoIoA1Ovaiq0VynlJkjO0ZooEfN3hNPM8caSOcb5CPwRq7/wAVjBg6jJPSuA8I3ENr4306adgqDzcn/gBFd/4tnSSS1EZ3oys3mD7tcyT9qjpuvZM5oZ6ClxzzzSd/agkggAcdzmu25xsUc+uK0NL0XUNXkZLG3MrJywBAx+ZrPXOR9K6nwd4gg0O9ka5jdo5EILKRkHtwetZ1JNK6IldbFO+8Jaxptobm6tDHCv3mLL/LNYpUivQPFXjSx1fS2tbaOUMSDliMVwDMcDGCM81MJt7ji27iDPtUkfGKh6il3FZkXI2kHPrVs0RtWlyYFyDyRXtlheq9hAxPWNT+lfPa3iB8Z4xnOa9csbmcWds4lXyRbgGPbyWwMHOfrxXn4qThaUTsow9orHlHxcufM+KEAB4EduP1Jr2tPuivnz4lTGX4kI3B5gGSfpX0HH90VUHzQTZMlyyaHUw0+mmtESyJqjNSt9aiNaIzZa05tt9F9cfpXR765e0bF3F/vCug3D1rOc+VjSuT76N49a5rWo9cluU/s24ijix827r/ACrNWw8Vsfn1WEewH/1qweId9ivZnb7/AHqKVlyCQCScZxXJxaV4i8wtJrClP7oz/hUMui60u1pNcdhnGBnqal4iVthqB2JZMfdH5Vx2pXDPPcRs7bQ5wueOtS2Wk6hb3iTzatJKi9UwcH8zWVqJd7q68uNydx7GnTquV7lKFitqOqQw2CxS5XCrk9hWBLP5beYozjnPrVnVLA3tqiNA0iuoRwQee1ZN1p99Gfs0PlKgACZfaqADGPXrWlOsti5UebUrLq6y3lykJHyxt5nQ8gZ59K57xBLGPE18nmJ5jTyMFzzjcc1fOmXa/aJI7ZVfaTK6TMMnHPb09aZqmhX73CubIPKHDs6yuxcdxkr34/Koc05XNFStGxgz3CQxhmcKMgfUmrdtZTsrT5IjHPJz09MVoRaDqFwwii0RncHcW3noPqOK3bHw1c2EUN9L+7aMEtBuPTIHzevXNKUr6LchwSV2ylo9pIWS6XmGWB9pIw2GQ9q9Q8LljpkzkZ3uj8e6LXBW7zS3KkkbCTnj2ruvBj/8So+nlwkZ/wBwf4VWHvzakT+E3uABkfrVLxA6jwrqe0jCwMx3elXow0jAbWqPU7b7ToN/E6kB4XGR24rvexzLcjs2M2n2kiZIeBGyD7VN5Mh9aqaIyDRrLMoJWBQcnrgYrSE0X/PRP++hUKS7lNMztXnnt4ENrCs1xsGImfbkZAJz9DmuflvAmpqe4XBNdBqnkSiKdHSR49yBlIOM4z/IVyOonFwJB1weSe/FeHiknXZ6VH+EjRuda+z2zTFXbaeQnJrMm8RSTxvGtleHj72zj86qPO3UNgAjgDiopZ2KDkAE9BWMaLTujSUn0NfQxp6awrTXRtnc7sN8i5HPVux9sV3iXEFwqPDPHIhyN0Z3D8xxXkgOXUtxzzn/ADzXdeDsNoSd9kjqR9cH+tenSldWZzVVfU+V/EL+d4m1SVQ2HvJmAI5++etdmhB+GGkHHH2lsf8AfUlclr1lMPEupxhGIW8mXp/tmuzVPJ+G+kqxA23Df+hSUYx60v8AEvyZ4GY/xKH+NflIxDILeEuSM9Rn0qC1l8q3MjqNzkk8YxVW6maZ1RTxwaLt1jss8jC44Xj8x/nmuw9QfawXWoRGYyOkR+XbEDlxk9+ntWtZ2WoIDBbZsomPO3lj26DgfiSfU1gaBC9wQPOnVRniKQg/p/nrXUQadbudrTvuXqWuTnP5/wBKuxJ09rYo3hfybtmnEEu9DIeR0646/eP5/Ss2T7MjlvIV5DyuwHA4/wA/lWnotksen38CXMz+dHlQZSxU/NyvoeR+OKpNayRKFS5kLFvvMASBzxwK4cKuWpWh2lf70medl75K1en2lf8A8CSf53+Zw3jGzu0kt7qbmNgUHsev+fpXtPgtDF8OtKHT/Rd36k1j+G7Cxvpr3TL6AXcNzECwm5+6ex6jqeR6V262kNppgtLaMRQRReXGg/hAHFdMtY2PVjpNMxJf3skUhkcGNiQFYgNwRgjuPao7hyyEdqbuJ9vrUchBQ89s15aWp67ehStYVhWVE3HdIXOWJ5PXr0+lE8McwUSqCFYOMnoR0NPU/M2O46UyQ5Axj8a1RmzlvGKN9osHUEk714Ga87vYyrvkENuIINexSZXULB84USFTz6ivMvFkJg17UIx0E5bjvnn+tdFGWtjkrx0ucxXb/DWXbq13F/eg3Y+jD/GuJPU10/gGbyvFMSf89Y3Tp7Z/pXTVV4M5qTtNHrv8A5yDXITW8yTSoXRCnEZZh82ST/WuvCMVwAfwFcze6c5nvnSKQzKQ6YB56Hj9a86J6VRXRnrLbx3OWuS4ERJVR1O3moRc2rS2ReGUlFADsQOM4BIpW0+4Mcf+iy5aNDnyz6EEdPpSXEF1NbwH7JMriPym3IRggg5/nWisYWHx3NxMNsMcaokLNjGSOo4/GopbiW5dY1kZUcoRg9Mg/wBRU8FrcwXoACKjM6MXYD5S2c9feqcNrPEw3GLgqQDKvY/X3NGg9Snq0Jl026mlDCaIRyDd9cGuLkG2ZgOzHFeh62tvcLcySSoi/Z2A/eLywzjvXns/+tJ9cGuqg9DlrrU1/DEnleJIOuGLL+YNekBiOc4I/nXlOny+Rq1pLnhZEJ/MV6rzn+R9azxC1TNcM/daPeLWcXNnBOOksauPxGalrI8Lz/aPC+nPnpCE/wC+fl/pWtmkthvcTNHFFGapEle4XgEVVcblI9a0JVBjNUDxxW0HoYzRGp3Ip9RTs1GmQuPQkfrTs1ojMd2zSk03PFHSmAtKDTc0vamgH5qSI81BmpYvvUxEUkEnmNjGMnFFaHHvRSA+XtIJbxDbZzwsnf2rq5nJwCenTNcjo2f7egYcDZJ/KupkJJ570QFJk1uQX2sO1WwkPcE1Rt8mTAJJ9quBHx0PtRJ6jhsPCw9AozS5h6hB1qPyz/kUCJiTkVNyrDw0Y528Ck85CpwoHtir+kvp9vcSNqdk91CUIVUkKEN2PFZSQhmdo1kALkgM2QF7Dp9alS1sDQrzhQTgD9Kwtcuynl4kKBvmDK3JI7VsXEWIW3AEY6Hmuc1JBJbmO4J/dMdh7gY4q7isVU1CSZVHzlRglh1217np91u0iDqP3CfhwK+d9Okc30KblVVHfOGxyK91sJHbTIDuxmJdwx7CuHGaxR24Nas8v8eSb/HqnPBeH+Qr6PjPyj6V82eNyP8AhNQcg/PD9Ogr6RU4UY9KqGkEZz+NkpPFMJzSM5xUZYgcGrTM2hzVEc0rSEDmmZzWiM2SQHFxGf8AaFbO+sOM4lT/AHhU97e30M4S2sTMpGd+4AZrCsm2jalaxpO/NIrc9awZtQ1vPyaSOneQf402K/15pFDaYqL3JOMfrXP7OVzXmVjoixAqvMHeLcASFYZx9arW89+3y3FrGqn+MP0/DvS3LHycA/xL39xUS0di4q+xKGyKltogwGcc1l3dxPa27TRQtcsSAIl42+p9axo/EXiGGWTyfD0syZ+TMoUD86yqJ2uVLayOmjso2icYHErf+hGvOkxqYuZbjaGErRELwCAQQe9bcuveJ0idrXRUZnYl1kmH7s9x1Gaz/Dem38cFy+rWiwzNcF0RHBBBAz0J71WFhFuS6vY5q7qRSa2KNxKLK3kjW1JikGHLEnHGOwqzYaeuv38Mt7Dsh27RsZgSoB5611KWVtdSRxTwqUJx0PH6Vu22mWkEjPFAqsqhSOe3P9a9mMacIaL3u55zdWctZaGLpmixQRyrYQrEJIgAz5PfJyc89Kh12GIz2iPbK+9zncOuQtdXEVVZAMALKRgDtxVWcLusXwMk9duT2rOjFQfM9WXNNqyZ5VKiQaj5UaqqbgdoHY1teF1jezj8yZ1WO0jOEkK5wSvOPpWD4ngnTxhIYrgRqsi/LtHAIGQDW94SUfZZCSM/Z2XOPSVhWcl7+p1Q+DQ6OIWRbmWTHf8Aev8A40j2FhcQvFG0pBDYUyvk8fWqxVtocznB/uirthsY489ieQVI68Vaa7oi0+xneH7K2fSoZTDCZGXDlmwTgmtY28UceRFbtgdMjP8AKsrwp9m0zw84kfyoYZJCzSHp82ev41P4n1cWujpHZOHur8iK32HOd3Vh+f6ily01G7RtFTnKyJ5reIWu5IUjIcfdGOOfYVyWsrtZDxjBGPzrqRYvpXhqC1kmaWSKNd8jtnLZ5x7c8e1cnrLZiDbs/NwMcf55rzMXG1ZNKx2UX+7avcz3xGjHcPXr0FZr6nagcSk8+h5q/wBVfIGOOBVNoYl6RR/gBTh5jlfobPgzUdPm182t15LxXETALMo2lhgjr+NeiXEunaRaMRJbWduDk4Kxr+fFeMXoT7HLtUABcjj3rkbtR50eEHT2rtpWcbHJUvzXIdYaO41vUJ0beHuZHDZzuBcnOfxrR1jKfDzTV/6eiMH/ALaVmvHhtwHNa2u8eAdNwf8Al57fSSsMYvepf4l+TPHzL46H+NflI4huZRjt2HFbnhuyh1XxVp1nNB5kZcvJG4yrKqliD9SKxGJ3dcjoOa7P4XWpn8R3d4R8lvbbRx0Z24/RWruPRMPVvD48MeJJbNlLWkn7yB3GQ6E9Pcjofpnoa3bSC1kIGyEtwNuBx+H+e1d14u0A6/opihXN7AfMtuxJ7rn/AGhx6ZA9K880u7geCMXOEcYUOwxn0zx1/wAKqLEdVocNvBekRxhGMZUED6H+lRyo0e5Acbf4Cc4I6fyqTTTsvYmzuXOOOeop19mK6k3sMFs9+/P+Fckfdxsl/NFP7m1+p5lP3Mxmv5op/c2v1QzSLr7JrVtIOcuEI9m4/wA/SvRyOMY+teXw7o7iF8FdsgJY8AYP/wCqvTuDg5610HrnKt8hYZ6ZFMY5BP61NcjF1MBx85x+dV8/l2rzmrM9VO6uQZw3cjHSmHBUenTFOP3sYHfrTTnviqRDKOoRg2qk9FkU5rz3xlCY9auBnOUQjP0HP6GvSL4ZsZeegziuG8dxf6fFJt/1luB+IJrSk/fMK6vE4RgQx7Vf0NpF1uz8p9jtKqhs4xnj+tUXGG5qWymNvfW84/5ZyKw/A5rueqOGOjPW/wCy9Tk+9dpj/fNZmoWlzbu0ct2AQnmDk/N7D34rs0bIIwvA4I7/AFrn/EKokkDmBJDznJ6KOv8AOvNW56koq1znre1lublYGnZSwPv0AP65qFLdmnMW6Uur7GwOO/OfwrTSe4ULOREiuzRqyjkYBA/lVaUXLrJK07cwiUKOO9aIxsVlsbuSL5I5NxxwxA4xz+tXU0CVgCzFeWB5B47dvzpskf2W/jYSOcCNgSeoJwf6V1GAV/Cpk7FRimcu3hdZ4Qk0pBJOQv6V55coUcBuo+X8uK9rIB6H9K8j8QQC31e7j9J3P4HkfzrfDybujDEwSSaM3ONhz0/xr1m2lE9rDN2kjVvzANeS5/dj2NeneH5DNoNmxwf3e38iR/SrxC0TIwz1aPZfh/P5vhry+P3M7p+eG/qa6g81wnw2nzDqEGejI/5gg/yFd1WUdjWe4UmaKDVkATkYqlINrEVdqtcDBzWkGRMpDiRx75p1I3E31X+X/wCunVrcxsFHeiimIKXvSUd6YDu1Sw/eqGpYT81O4FvPvRS/hRQI+W9GIOuQ47I38q6aQ5/OuRs5jFqSON2cHpXS/aSzJkjBGCD60oMU0XrN1SQsTwB0q99siGQH/SspXyDg8Y61ANQtHufs4uIzKDjbnv6VbimTFtG39rj7Mfpij7ZEBwW/Ks0nnOajaBCGPzZ9mNTyofOyx/bMW5gA5wSTjginwahCqksSN3IGfwrnZEaGZj8yPvB55AGf/r1FI+2YlmIAQnaPzP8An3qeVF8zNu51m2kcQNFJmQYDZHX/ACK5e+uZmvniVdhClTnv6VHJOGvRKSPLGTjdyP8ACptRWG6w8Mg80Ll8k5x7VNh3M9FaKRJYxuCLnIHfpXuOmyIdJthzuMCfngV474Udl8SWkbFvLdyjA9CCCMY9K9b09dlhbqOixqB9MVw4uWyPQwcd2ea+NjjxkG5OJIsfkP8AGvpKP/Vr9K+bfGpb/hL2AGQDEf0FfSMR/dr9BWkfgRhP42PNMNPPSmGqRDI2pARilaoSea1iZMc5wue4rYSVURGY5BH58VhTsfKYDOcHGPpU8EqNFFFcSNHInQE43Z//AF1FWxrSNSS+AKhFIyQMYxwaja+O0lDz1Ge/NQfZoGIPnvkAAYb0pCYIMwosspb+Ec4rK0WaaosmXM7qAMbQT9aguP8AUHPqD+tWVREBwMZ5P1qC6ZBbSfMOnrXJUfM9DeGgxSdhz60REsCB6muL8Xanf2F80dtdyRoUB2qcYrE8P65qL6wsE1/O6mJiQWOMgZzSlhZVIp3CVVR0O6aUh7oZ6Sn+QqbdlWrhNIvbqe4tmkuGbdI4cySEZ/xrqNR1VdKEe6CS4847QIeSD/8AXqKFF0sTFNirTU6DsjfsbU3EiBhgEk5I9PSt5E2mUe4/kK5XQdfa4vY7OWwntnVWdRKeSMVuXF/JFcI0cBZGX5jngHIx/X8q9d1Yt2ueZyNK9i3G37ySP+Ivu4H0qs2WNsoJB2tjI6HNZ0mrSC5minWKK4E/kIxclSWTcnQfQVXvdVvotF06RLeLz2nCOwyVXkj69RQqkW7XDldrnI+KYGHiKRud2FJ4q34Ob/WoVzhZxyPSXP8AWq+vzyT6lOZivmqxX5RjODVXRdWOkiWYQiT9/MhUtjrtNYynzSOmEeWJ3csalF2JHj1CD8qlslUXCnaoPrtFcqfHmxTjTU+U939PwqNfiJJvJGmwjJ4zIc0KCve5Tm7WsbejRQXOnXdtcwrNC1xKjIRkEZFV9K8Kw6brQvFnlmt4kK2tvLz5JPXB/l9axbDxBNaxBo4YsXN2xIbPy5GeKvy+KbyBj5cUPpyCf61H1qitJbotQqpPlejOm1WTzNMnACnC5OGBPUVwWpKvle24YyOPStQeI7u6geJ44ArqVbauCM1jaq5EJbndxjHbmufEVY1ZpxNaMJQg0ygCOfdRUbjiljKh26fMuRzSN+mO9NIq5VuVLWso/wBk9q5C4X97GSOi9uea7OQZRlz29a465BEqg5HBrponNWIymRgDOelX/EWR4D04cE/a/T/rpVXGY856egq54gyfAmne116enmVljPipf4l+TPFzH4qH+NflI4KcgL128V6h8Jrfbo2o3BXDS3CqM+iqcfqxrzOQYIx8oPbH+FevfD6PyfDcKngupYn1O8/0IrsuenY7BThl7civP/GmjLpHiL7XCitball9h6LKPvD8c7undvau9Lbsms7x/Ym98KtOoy9nIs4BH8PRv0Ofwqoks4TT3jiuI3hkeIq4ZomOA3Ocen5VuaruhvGcKSGQNuQDcvbv9K5+AieJd4AJGc4yCa3tUMpjsZQqPI8fzk5weh/qa5a3u4ulJdVJfk/0PMxHuY6jJfaUo/k1+TMh2jlk3oZX287s5x9R2+teoWd1HeWcU8TEoy/iD6GvO/OVR8saeYgz9Pxrp/CNz59tdg4VlkGUxjGR1/H+ldTVj1lqg1BduoTDPU5/SqecdDV/VRi/J5+ZQeKoHg/4V509JM9Sm7wRC3U56Z6A06WaS4kLytufGCQoHt2ppIJz7ZpMHJyRxSQMinG63lXqSh/lXG+NoxJY6XPjIKMvP0H+FdsQGUj14rkfFCb/AAtayd0m257/AMQq4aSRlVXus8ycfNTfSpJxtkb6moxXonm9T3zTJBcabaT4GZYVf81FUdZiVpbYuMrvKnPTkH/CjwjP5/hbTnPJEQT/AL5JH9KtavbST2beSN0qEMoPcivMekj1lrBHJzxyNDcKhXyEuAY8def/ANYogXzLe3/24JIjz6c1dj0u/SMxAx7HCs2TyGGKtppTKyYK7UlZx/usOlVzIz5Wc9NCywWcgdm/dljk9cEHFdahDIrD7pANZsWgmPiSdmC5ABHABBBH61qQxGC3SPJbYuAW9qTdyoxaG/jnmvM/GcHl63MR0dUf9Nv9K9Oxk8frXA+PYSt9by44khZf++Tn+taUHaZliF7hxQ+6RXovg+Tf4eQf885GX+v9a86Hce1dz4Fl3WF3F/ckDY+o/wDrV0117hy4d++esfDuXy9buIj0kgOPqGH+Jr0qvJfB03keKrTnAfch59VP9cV6ya5qb0OmotRKKKTNamQuaimGUzUlNIyKpbkszZuJIm9yP0pSaLv5Y84+6wP60mfWtkYsdSZpM0A1Qhc+9LmmZ96AaYiQGpYfv1ADUsJ+bigC5z6GilGcUUAfKEJJvFKnnH171tmZVUDPzdzWHG7LKGDDlcfqanRt8o3vhQfujv7VnF2ZUlobscsZWSR5BGqrt3E8dK5G0iMmqxQoVP7wDcO4Hf8ArWibuW81Mw2yh0SPBRh8vvxVO/SSGc3YxHLn7oGNpx2qnO7sSo2VzuenJqvLJJlo4yu7GRn0rn9N8QyXLiK6IDD+ID7w+nrV24vI45lkhJcr146itOZGfKQ6lqpEnkkDcBzj6VVt72MwJv6/Mm4+h6VnXbme6kmxuLf3uoFQ5bbnGcdCetZtmiRbtwryjcdx3DHvVm+NtFIqlgjgYBA/XNUtyJbLIuN+cHnpVSdi8nzMrHoCDSuFjV8NShfE9gBzmUD9a9ktBsgjX0XHFeKaE5i8QWGc4M6ZGe2RXtsQ6D3rz8Xuj08F8LPMPG5Y+K5MKSAYs+3Ar6Pi/wBUv0r518Z5Pie7PZRF/T+tfRUB/cp/uj+VbR+BHNP42PNMNPNNPFUiGRN0NQt1qZqhbg1qjJjWOFzWkqxzYEiKw7AjNZE5xCxHYVTh12aJVUqjEAfMe9c9eMm04nTQas7nSPbwxEGONVJ64qe2dWQgHkHBrmpNekl/hQe9NGu3CfMgQcY6VEoyaNdLHUTxeZbyJnG5SP0rgf7PETCQyudpDH0yDWv/AG/fMCC6c+iisy9lZ7eXkjKHI/CqopwTTIkr6i+IdDudc1AXNjJHJb+XsYgg/MCff3rC0/wrerrEgRv3ixuhyuByMcHoetb9pdywQmKJykefur0qaCR2uFG9sk9c1C54qyLcYy1Zzkfh+6tLmNGlXdbTbn9wcHitvVWJhtT6Txn9aqGaa5up12tvBAOM5OOKszx3E1rDEttMXDqS23jjBrkxKk2pM3pcqTSNhCqeNLVU6NAw/QVvbhIkojXG7bwPqR/SsM2lwfFOn3SpmJEKscgYO2tW0nKFARkOWGcjjDMf60qUrRMKquzB1S2uzc+ZHC7kXtpJ26BcE/hWvdwSyaZFF5ZDpcq+M9t7Gm6ncyqIZIolZGZd5LYxjpUr6nCxlDsBkADHPc/40nXswjRbs0cV4muY7TVbknkPKxBX61zsl4YIXAjLiS6PfGMxqa0vFIea7ItySobPNY1xGz223blhPGT7Zjx/SuqjJyjciceV2KV5qEiI/lqFOMjPPaneHrt9SeZJiA0UBkVgMZYEfpTbiyeUnEb5Axw3H8qb4SsZxfAk7EkiZc9Tj8fpW1S6ptozh8aTOuuYUh0rTnTPzTKzHPc0XnBJzUtzEJNAhXLDym4I9QapXcLkkm4lP4ivGg+Z6notWRNayEtx6U3UpmMLA8jHYdDVW0hO8ZeXA6fMal1B8xDOeAenf8a66cTGT0MlbhzcbfMbAzjn07U9rh2yM4NQn/j65IxjikPHHp613KKObmY4zNzlsfQ1z+opslVlHVOnvW536/SsfVgcRfQjjp1rWCsyJu6K8QTCKWYLxkgcj3/ya6XWrbRn8JKktwyWq/NBIhyxfnoO5OTkcd+mMjnFyqrnoRxVnXhjwLYdv9KPPp/rK4sxpuUqTUmve6ej/wAjwc2oupUoNSa99LT0bv66W+bOKZTu+7z/APW+n+fevXvBCldFhUjGIyP/AB1D/PNeSrBJNOsUYzI7bVA7seB/MV7do9gLCWW1AyIp3j9MjDD/ANlFdrmlJI91RumzUGMdjxWmI0urBoZFykibW/EVmgegHpWnbHNun0xW0TFnjsEDadqF1Yyg/uJdpyOdvYgd+K6q6+w/2OJZJCYEUbZOpHb+fGP5VF440yKLVre+ZcR3I8qQjsw5BrO0hfnm026O6KdPlOO4HbsDjn8BXn5lSk4xrRk1yO+nbr+B42cUJShHERk17N3aXbr80vluQwvHIh8sg5HXvzVvwjcvBr/kM2RKrRN9R8wP6GsGS3l0+ZivIViGH0P/AOutHR51h8SWc44SR1zj34rvk7pNHtwaa0Oz1obbiJjxlMfkf/r1mN1z3rZ1tcrA3oSP5Vj444Jrz62k2enQ1pojb6c5puPXNPYcdeaMY9R9alM0GAdPyzXNa9GD4Tvkxnyp8gf8CH+NdPjkcCsTUoPM0nW4epA38/TP9KqL1ImtDyC7GJW68881BVq8X5geOlVelelHY8p7nrvw8n83wqidfKmdf5N/WutkGWIxwa4P4Wy7tP1CD+5Kj/mCP/Za79lOR34rzq2k2epQ1popFAM8cVG3r+dWWXBxzTCpA5rO5rYgK4//AFU1hwP1FSle3p2pGGQeMfhRcViDZ83Pr2rkPHltmxtZeTslKZA/vD/7GuzwM5rB8aW/neHp3/iidHH54/kTWlKVpoyqxvBnka8N9eK6zwHJi+u4s/ejDY9cH/69co/yzNjs1dB4Lk8vxIif89EZf0z/AErvq6wZ51J2mj0/SphbaxZTZ+5MpP0yK9pNeHbSOnXtXtdvKJ7SGUdJEVvzGa4qTO2qiWk70UGtzASig0lNCKd4mY3HqDVcNuUHHXmrs4yuazov9Uoz93j8q2jsYyWpIDS5xTc0hNWSOzSA+lNz60oNMQ/NTQn5hVfNSwn5qAL2aKZu9zRQB8poPmyOuP05qCaQpMdpPPSpThX7E4Hf61VnyJOB17Vgty3sXfCpB1Kc9cx+me9XfFVskUEMoLb5HOQTkdKj8IxkXl0f4dgGfxqfxk/7qzHT5mJ/Ssm37axqkvZXOWiJWRSM9RmtQzbSAcgdR2zWSjFjtByOlWQHfGDgV0GBNI247u57jioz7cjtSgosZBY7gOCRURJBJPpikOw/kQNgnBbpTAqhflfJGO1Sq4FnMD13LgCr/huygv72VLhCyCPcACRzkUN2QJXZHpsZ/tyxxg4nQ5B9xXuKD58e9efwaRZW0gkgg2spDA7iefzrbTU7wv8A648nrgVx14Oo1Y78PNU00zlfHB2+JrkZ4KxcevSvou3P7iP/AHRXzJ4smkl14PI+8koD78V9M2xzbxH/AGB/KtUrRSOeTvNslNNPAp1MNNEsjc1A5qZ+lV3PNaozZDOf3Tj2rmw3AFb95KI4ScZ5Arne/FRM1pEpJHNOViV601VJAPY1ba1EdrHP5sTbzjYrZdfqKzubogBwOufxpJ8tBL/un+VS7Bk0kgBhkA4+U0rjGWx6+nBq7bj/AElCOMHNUbUjb1zwKsqxXDA8jvUtjRIrCLVrpugwp/U1oPdokKyKwKhOQPpWFOSboseSUGTUwP8AxLW9dhrixXw3NaUdTcl1SKMwB0ZmyRwcdVIrE0nU7lrWeMyHCTuF9cU67OfsxH99c/rVDSTj7apzxcN/SuRSfIzRxSZrNKz/AH2JHuaVG+RuO4qBTkHGKkQ/u3/D+dYSNYmFq5H2pifUY/KsubcDLuGDvhY/+PDir+tZ+1cdNorKuf8AjzkbrxEcg/7Z/wAa9nD/AMJHn1l77J3W4M7i3dVJxnJHvVfw8Xjv4Uc8qXX8Mmj5xKTtJG0Z4qPRJM6opAx+9YfjW02nTaMoxfOmdW/OjzDsHf8ArVK6/wBWDx061bB3abdrnpIePwrPnfMCHg5UV48FqejLYigYh15p1/zb7jj3qCM/veOam1FsWXTJHpXdSWpzS2Mk/wDH2gAPQD9KQnkd6FI+2wHP3l7/AIigjBrtRzjcfmaztUXdbR45PmY/nWiR0/wqreRh4hwMCTkfUf8A6quO5LM2Vgk0ag9B0qbxDn/hBbD2uz2/66VQunJ1EjgheK0PEi/8ULYDB/4+s8H/AK6Vz43el/iX5M8jMPjof41+Uil8PobSbxtppuyohj3y/OQAGRSy5/FRXr8Kr/aN0wZWX7WCCDx82P8A4uvFPCiebqUTdGjjY4PfjH/s1epaZfbjNGcAq8bjA6fdH9KdSm+dTue1TkrcpuSptwQOc85NWrRswDnkE5pl0h8kkYHem2TfIw7hjXVSdzGqrFPxZYf2h4fuFVcyQr5qfhz/ACrgLSdmSOdcM64OG7kf/qr1YkEc4wexry29sTpOq3Np0Ebbo2x1Q8j68fqK1aTVmYOKkmmTayIYblJRHiKdd2/HGe/HUdj+NYrxSpLG9vzCDvVgfun/AAroI9t9pEiAKZrdtwA9D15P4/kK56azMbF1ymeu3pXHg2/ZulLeDt8uj+6xxZZNxpujJ603y/Lo/mrHo91MLzSLa4GPnw3X1FZu0gYwetLodwLnwwsXV4H2tz71IyjPt6VhidJn0uF1gQlcjApNuKnMfWgJ26ZrFM3aIdpz0rNlh8y61CAjiW3/AKEVr7fpmqMi7Nai9JICvPsf/r1VyXE8OvV27epIyORVEda2NbhMN5cR8fJMy4+hIrHr1YO8Tx5q0j0H4Vy41LUIP78Sv+Rx/wCzV6kVyor580rVZ9Jmea3lmikZdu6J9px6dD6CtN/GWpv1u70j/r6YfyArmrYeU53R10MTGEOVntToc5xz71Gy7eTgfpXh7+Jr9zlpZmP+1cSH/wBmqtJrE8v30Rv94s38zULCy7lvGR7HuLz26AFp4k/3nAqvJqemxjD39qvbmZR/WvF5NSiNoiJbKtwD80hCkH8Nv9ag/tG57Mg+kaj+lUsJ5kPGeR7M2vaOuM6na57/AL0H+VZOu63pF1o15bR38bySQsEVQTlscdq8u/tG7xgTuPocU0310etxL/32auOGSd7kSxbatYjnGJTwfXmtLw7KIvEli/Yyhfz4/rWUzM5yxJJ7k1PazG3u7ecHlHVvyNdEleNjli7STPamTINep+HJxceHrJs5xHs/754/pXmnlhhnORjqPSu+8GyFtDMZ/wCWcrDH15/qa82k/esepWj7tzoPeg0ZpK6jkCk70tN6c00IRxlayR8ssq9MNn861/0rJuBsvWH95QfyzWsCJoTNGf1pmefejPPStTEfmjNMz1ozxSAkDc1LCRnNV881JEfmpgXePQflRUe73ooA+WgAWHTgD86o3HD8kdT2rUtWHmHdAH+UYzzjis242tdHGAc9+KwW5b2I4rmSAN5bkZAzz6VHc3k90R50rPt6AsTip/s+yBpGYNx25GKhtI1lmIcHAHQVem5OuwyHAPI57YqyDn+IDANQRhVudsn3RnI/Oo2Y9+tAEolzwDwOKf5UzgsqEj1xUOB5JO3r0q1p7ssgXdwynjr0pDFjhf7JcF0KMMYJrX8Fhj4ltoMblnPluvPTPaqkizSxNHwCAAD0xitzwDA9t4y05nK7RKucfWs67tTbNKPxo9Rm8MRQvteKWI4+63HFVf8AhHVl1CytYD5bXD7NzZIBxXoWqmPUL7zYsldoAyK53WQ+mTafeoMtFOGC+vB4r56liZ+35FK6PblySo/ClI8b+I2jzaH4vksZXWR08tsqOCCoNfRFof8ARYv9xf5V8+/FPVjrXjKW8MQiG2Ndn3jgCvf7M/6HD1+4v8q9/wCyjxbNN3LDGmn9aQmmk00DEc8VWc81Oxqu/etEZMqXab4vxB5rBUA9f5Vt3smyMcA5YCsZAVO7GazqaG1HYlToFHr1qVUbeE4ySMfNUS5Jz0qSM7T96s2zdJk91bTWc5il2lgAcqcjkZqBwzqwAOcGpPMYqVVTn2FO8iZlJVHzjspqOZFWZWtImCKCMfKK0La0jmdlkuFhAXIZhnPtVa1s7ho0JSX7o52GnsYomIaWNSOCGkAP60rpjSdirOQJ15/hPP409fmsHB/umq11d28d0rG4g2qhDESqcdPQ/pTP7Y0hbGUtqlssm1iIy2ST2rnxEW46GlOST1Zcumzb2zc9U/nVLTcm4vl/6bZ+nAq/dYNhaPAwkR0Rg2McGs2OaaG6uvK2ElwTn6Vywp3i0ayeppgMOxxUscTPHKc4wucfiK5/U73U7BgHZRuG4BQpqnbeJri2aV7pjJGUK7VKg5496r6pJq6M/bxTsy/qsRN0hPcCsy+/cW8+McRqc/Rx/jUFz4phmnDPbSbcYxvAOfypWu01CzeRI2jVoZVxuDcgqa7qVNxgkznnOMpOwX2qm2vBHyfNUJ+dV9H+W8ZiMN9p4A7DjP8AOroEIKDGcrnkj2+tV7FkjvZyQFPmgglwOw9q25Vysz5nc6NHAt71SQPnBGeOoqgwMlsgUgnaOMjrVp7nT0SZGmtwrNkb5R3GPaqw1jTYLeNDdWqcYBEnP8681UpX0R2ucbblaOKQTcjHqauXkBktCpI5FU313T2OftyMvXIJIp8/iC2eJYI7gnOABtIzXTCEk9jGUo23M9k22/YsBgEDpyaRkduAjn6A1HcXrwwErI6hs9Cf896py6i4Td++cY4AySfoM11K5z3RoeRKR/qJOP8AYNRvDII3LIcLg/Sop7PXYYklbQrspI+xfmj+9gnn5uOB1OO3qMxx2ut72Z9IEUK/6x3vIcqPXaGyevQVajIlyic5I2b9xkden41s68N3gXT8nH+kf/F1go/+k3DZ4zjHtW/rh/4obTSBx9o/o9c+N3pf4l+TPIzD46H+NflIw/CnGsCPcE+R8lv6/lXdWcqx30qiVctEMYzg43H+n6VwWivs1hGVsZVsflXT2UxGswAnPmIy8+pB/wAa6bXR6vNZno9xqOV8kIMYAz36ClsHBLkdQRWTvDCNvVFOc+wq9p0gzIMY4/CtYxS2M5TctzWDd647xvbBJbO+GBkGFzj8V/m3511m/rg96xfFUP2jQZ8DmMiQEe1WQcdpLxRaqm5VCzKYmBGQ2f8A64FZ1499pN3NDMv2i2ViASRvC9j6HipYgr4BHzdRWrrUX2m1t7tIyxkXbJtHRh+vqPwFcU/3WKjLpNW+a1X3q559T9zjYz6VFyv1Wq+9XXyRJ4YvInkmijbMc8e5cdiMV0fkkjP4815lp8s2n6iJUJUxvlk5wQa6C5+KmmWkjxNp920i/eBKKM/mazxVOc5JxR9HhKsIRakzrvJ7YrN1S7aykQKmdwzk1ysnxiiXPlaR/wB93GP5Kay7/wCKct26n+zLT5QQNzs3/sorCOGrdjoliqPc6k63NuwqLTUv5rjULZljTeAwG5wg59SeK4iX4j3Lgf8AEusuOmUJx+tQn4i32QVsLAc/88T/AI1p9WqvoZPFUu5neJjv1e+OACZmJAOcEmufPFXtT1WTVL2W6mjRGkxlYxtUYGOB+FUWOTXoU4uMUmebUknJtDaKWlCk9Aa0IG0VYSxu5ceXbTPn+6hNWk8PazKMx6VesPUQN/hS5kPlfYzaK2l8Ja+3/MKuF/312/zqwngbxA/W0iT/AH7mJf5tUupBdRqnN9DnaK6tPh9rDEB5bGPP965U/wAs1Ovw7vDy+qaaB/stI3/slL21PuV7Gp2ONp3YfWutn8BTxqPK1G3lbPI8twMfXFV7jwhdIkflzCRsYYCMjHpyetL20H1D2M10OltviFYR2MEb2k7SLGqsd6jJAGe9ejfCzxTB4gXU4YoWiMBjbDNnOdw/oK8QXwbfsxBkiUdt2cn9K9D+FVmfC2uXT3t5B5FzBsON3DBgR2+tYclJO8dzoU6r0lse5Un1qiNYsD/y8Af8BP8AhS/2rY4/4+FH4H/CmIuUh/SqZ1ax5H2lP1o/tWxP/LylUhFuszUcJcwN/eDL/I1N/a9gePtUeazdV1jTzDEwvIcrIBy2Paqi9SJLQfuoLcVmnWdN7X1ufpIDTDrunjj7Rnv8qMf5Ct7mNjU3cUbjWO3iGyAyPPb6Qt/hUbeJIMEra3knssY/qaLoLG4G75qSJua5s+I8YK6fcgf7ZVcfrTR4mkDALaIp/wBucA/oDRdCsddvPpRXJf8ACWP3giB/66n/AOJoouh2PFLO4SGU+YwH3ePwqleCEyOwiB6n7x5NWYLVbncXLgjHTntUhs03ncScDmskinsZqNnT2OCMbgMCqlu7qzFBz610IsojbhFU/MScDIqnZWiLcMNpDAd/rTuKxkqGkm6HcaMMuQwwehHvWyltH/aJGwH5s89+/rUMscaysGRd27HI70uYdjLDPt5PapbTf9ojweAavgBQAqJ/3yKt29w0JP3CD17HvxS5h8pGtypONrZJ7LW34c1CDTtdsr6YStEj7mCpk4qvDLBI3zMQT74zXY+GtVnhvbe2to7VNzAeYbdS2PqeawxE3yNWOihC8lqe26HbRalpkF9G5WOZA6qy4IB9adrfhZdWtY4hOItj7i2zOePrT9HvdtqpcSySY5YJwfy4rSbUG2/Laytx2wP5mvMwkcLyptajqzqqo9T5T+I1r9i8RuobcDtOQPrX0FZsPscP/XNf5Cvn/wCKEobxY+1Ng2pgEjIxmvfLRh9jgOf+WS/yFenTu6cWyallUZb3e9MLUzd70hYCtEZMGPWoHNOZ6hZ855rVGbKGpNiEYzncv86xjrsaAr5loD0x5SZz+Na2oHdCMDowP615++j373MrLLbqrOSN0nbPoBWVWnGW5pRm47HUf8JIsIObqJcDqsaD+lU38av9n+0RvqEsXy4eO1cA5xtwdgHORj61kr4cuZRiS7tV/wCBN/8AE1dm0rVZ9Ik06XXbeOORQpeG3O7g5yDuHv8AmfbGUaNLqbOrU6Dn8X6i7YXTNdb3MRQfqRULeJNfnJEGi3J7fvb6NCSfYtUX/CM3JJL+K70E8nyrdE9uxqRfCdtIP9I8Ra1Ljv8AaCB9OtaezoroR7SqyzZN4nu9O+0w2+nIqsYiHuzJ8wOOqjHX37H2zxjTTXE0srvtMrbyqspAJ64PNdtD4Y0m2tfITUdSaMsWK+cuCTySfl5JwKE8M6BEuAlwR0AaXH5YApqVNfChNTluzko2YQzgMScocnHv6Cse4UtcOxYBsYOT/wDWr1PStL8LJqMy30bLbeSCN0jnLA+xrWa1+GyOXayWRupysp/ma0ik1cyldOxL4H0221vwXaSXMwS5hHlvsbcAByv6EVtv4O0xnJW8mRm+8VYc/mKybDxH4M0OFotMsZII3OWWKEAE+vJqWT4g6IoylhO31Cj+tcksL710zZVpWEuvhrp97MHn1q7IHCgBBj8cVVf4TaPtAXVL1vffGP8A2Snt8SbEH91pbkY7yAf0qFviXn/V6ao/3pM/0q1QaVuYhybd7E1x8JtH8hTHeXXmAcksvX6bap6l4Ih0+80/TNPeV4rmKdWlkG4IxUYzgcdP0pV+I91JKENtbxgn7xDNj9RVWbxzf3F220QBrXdIhCdflIPU+9P2VvtMFKXYE+El0sasNXgzjoYm4/WoB8MZ4Y5ojbWdzKyMBOc/ePfkZ/8A1+wp1x8QNVOnpLDfR+cW2mMQr+fSs1/HPiKQ/wDH+yj/AGY1Gf0rVRt1J1Hj4a635aRjTdKG3kMzjBz142GrK/DrxAdn7rRI9oAxhuMfRRWW/irxDJnOo3f4HH8qrtrmuTfevrw8dPNaqXKKzOrtvh9rkdoYJL/Txly5KxucZOcdv1z29BTIvhe0RVptXQbcdIf/ALKuOl1DUGP726lP+9Kf61ENSjRv395EuB0acdfxNDa7Ao+Z6KPAmnyERy6rkDPCov8AXNI3gHRUQ79TkB7OiIpHvn1ri7O8AjEsV5asGGRsuULEf7oOf0q0msGQY3E4/CuSpTqSleLsjog4pWZ1eo+H9LvbdILrxDqjxxSLKgiljQhxnByEyOvrWNP4X8Oojt9s1i4kUEr5t85Geo44zzWf/akSr828N9P/AK9QnV492Nyn2LY/pTjTqdZMd4djzyIBbiUbiWzz7mui1ra3gPTwTw1x/wDHK5y6xb6tMu3qx6nrW54g8x/h5prIfmFyT1x/z0qcdvS/xL8meHmHx0P8a/KRU8JtFFrsXmxJI4V9odQwPyk9DXaXFxiWNlhgQK44RAv9K8z0W+c38DptEofad+SPyFdhLdXci7VktgeuPLIz26/56V1cp63MdVbX2IY1Y42oB14OPwqzBqkaSEMSQBzXFpqN9GBvtxJ2/dsDj88VNBqbqzefb3IGOAIi38s80XkhJRZ2za3bjOA3txVe61mOWCWLymYMpHXtXLf2kzA7LC+bPTEQH8yKa1/dY+XSrw89T5Yz/wCPUc8h8kTMkLxnEYO5eBgZzWzot1LfWV3YyxbbhU8yL/aPTr2/h/OsnzHE58yAws3IjYgn9K0NOnMFzDIyKoVsc9cHg/zqMVTlUovl+Jar1Wq/yPOzCjKpQkofEtV6rVffsc2pdNQYyPvZ1zkVmaj4duNTvQ1oYgdvz7ztArp/E0Fvp2oCONCpb94oB4APoOwzkfhUGmO32pACo3+vT6U4VlUpqpDqrnXhasMRTjUjtJJnNJ4D1Jutxag+m8n+lWE+H12Th72EHOPlUmvRlsXIyZFGeeBmpPsK55Y/gprP20zt9jA88T4ergF9Rb/gMP8A9ep18AWm0Zurhj7KorvhZqBks3XPQUgtgM8Hn9KXtZ9x+yh2OBHgS0U533DD0OP6VNH4UtE25tEO31J5+vrXdC2TP3Dke9SC2jGfl79xmj2k+4ezgctDplouP9AtR9LdCf8A0Gta3igQYA2eyKF/kK1fs6A9O3p1pywL7VnK7Likimq2/wDfnP8A20NSGC1YfMkrfUk1a8tM5DYowo7jr3qHFmqkiqLSzHH2f/x0Cni2gH3YAB9Km3RgffUeuMc00vD180D6Cl7NhzoaY0UbVQAdBgZphQHG1fbpUhniH8ZI/Hio2uoxxhuKpU2S6iGNBuyQDmo2tScYH6083gQ5CMf+BU06g2MBV/Oq9kyfaIjNgx6MB7ULpTbwwePg9MGlN/J16Z6cU17yUj7x/KrVJkOojX/tHVQc+bCQP9k8UHWNVAGJYs+u2sU3UvQytj2NMMzEffz7Voosz5kbf9uaqpH/AB74/wBpaU+Jb5cb1g798f0rBaTJPrUbHB6dKtRJcjfk8VXW0bUiz3PX+lULnxDeTwSR4j3EcZQFc59MVmHdyMY/Gm9MjqM1SiQ2POr6lhdot0PoIhUbarqpPE6qT6Rimn3zz70w4Hckdqsgk+3aiet7J/wEAVG810+N17dYHOFkxn8qTngdqTBDHrn2HWgBpaTOTNcMenzSsf60RxRkkmPNBJxxnNKjMDnJ/wAaBFgogOPLT86KTd/tJ+OaKBnIWEbjeBC7ZPGFJ7DmpHt7pmJW3lx14jP+FVGupshY7hwv9xHOOPoaasd1OSAk7/RGP8qzuVYurb3anJikH1XH86rPaSoxkURqck4MijNTxaHqc7jytMvJDg/dgY/0q6nhLX3+5ot98uM5gI/nS549w5W9kZT2rM+8TxK/Gcyr/Q0slukqjddQBu5BJz+QrZj8C+KJhhdEugB03ALj8yKv23w48VvHsOmNGDyd06AH8M0nUh3GoS7HJvZRKMNexnucBv8ACo2SBOPPc9jhPeu6Pwm8TTEEm2UDHDzfn0BqzH8GdZfHmX9nGPYsf6UvbU11H7OfY87DxqoxJLx0+Uf41ct9ensbhJYXlDqcqdoxn15zmvQ0+C84wZdWiB/2Yyf51Bd/CK3soTJca6evCJb8n25Y/wAql16T0bLjTqrVHM/8LQ8SIu1L6cJ327Bj/wAcqnd/ELxDdDbJf3LAE4/e7f8A0ECuntfAuiwDNwtzdHuHfap/BRVm28F6DBlxYFgT92Ry35Vmvq8dol8lZ7s82X7d4m1S2to0Et3PIscZZz8xPABLHpzX07bWd4ltDGbdtyIqk7h1A+tcFouh6Pp+tWd8LKC3EMyyGUkqEAPXrjH1r0RvGPh1eus2X/AZQf5V003GouxhUUoPUT7Jd5/1PP8AvD/Gg2F4R9xPxao28beHv4dSR89Nkbt/IUw+NdHP3JLmT/dtZP6rWnLBdTK8n0JTpd43/PIf8CP+FL/Ydw3WaMfgTVY+M7Ij5LPUG+kGP5kU0+NIsZTSr856FvLGf/Hqd4LqHLN9CeXw28y7Xux68R//AF64bU9Mn07Vp7ZN8wQgh8AZyM/1rrm8ZTEDy9IkOf79wq/yzXnviXSb/XdbuNVEUdus+1WiEpbBUAA5wM5xUTlB9S4RmuhoCK44/wBWp9DKo/rUi2kzjLXFuvsZgf5Vzy+H9QQAG5bjgYkNTL4cu3iLtOGK9RvJ49az9zuae/2Nv7EQfmvLcD2Zj/IU77NbLy2pxL/wBv8ACsD/AIR1m+9IAfpzVa68PzBiYXlkkxyqnAP096d4CtI6gLpaghtXUk9lT/E01rjRowd19MxH91F/+KrhjaYPR8A4OScinrZAsSyY9cnp6U/dDU6K61TSlk3RXMoBGDvj6+2AKx5tdtwSEjuHA4zhBn82FRpbRA/6se+eKU2sRxtUY7juKLoXKyH+2Vb7trOWH954x/JjTG1lyOLSP/gdwRj8lNW1t0CkhUz2OcEfWnLHGMhlix3wMU7oOVmedWvDylraj2812/8AZBTRqmovjYtqMc8ROx/VhWoIouo2kD2pxjGQDtbJ6ii6DlZltfam4yJo1Pqtr2/FzTUn1dfmW6kBIIOIoxweo5BrWwAPlfIzjrQRF8w2sAB1xkUXDlMnGouCDdXHocGNf1CU0215/wA9ro89TcMP5EVsEKQAGB/QUwlEPXbjp2ouLlRlNaTtwZJdv+1PIef++qjGkj7zrGwH9/LZ/Mmtkyx7csVPHUjFNFzb7vQf7Q4ouwsjKGjKf+Wdt17Qrn+VPXThF90Io77Y1H8hWl9ojPIbcM+n+f6VGbgAgjzAenNO7CyKgs2QY8+XnsH6002PTdJMc88uasG6b72N36Gm/aOpKupHoc4o1FoV206MkFsk/U9aBYxjICY68Y7VOZmIJADLjGKjMjquF2k44BFOzC6MrVkVbuJ2yqHAZlAJA74Gee/Ga7jUotE/4Q6MG6IslAa3lU7mL89uMkktkcd+mOON1BTcLHgZIJBIGO1RnSZru2EaSYCPuCk8AkYP8h+QrlxWEeIcGptcrueZjsFLEzpyU3Hld9P63/4Ohm6Zi28QW0hQMm8NhuMgV6BHq9kOXhKZ6fICP0Oa4CzVmu4XBKvGSCQcHoa0y8pY7CxxzjdziunluempWO8g1OxkA23MSL6MNn/oQq8nlzcxyI4x0XDV5izMUy5ADdNwx09sf5/kBCegyWzgr17f/WqXS8ylV8j1AQdOBnoeMfypTEO7kZPY+/8A+uvN45dTUExT3HA5Al4H4Zq0mratAQrPERnrIuM/kPf/ADxS9mP2qOm1e0Vyjow3j1IJrOTIZQzNknDbVzt/pVSPW5JiFkiRsn76yqMc9duauq4ckrjDelbRVlYynqzb1a30240JGnfMcSjyZQNzg+3qTjkfyxkcdC5idHC5KkfnWi6zTxsu7MIJxlvzxWWw+cDqAcc+/BrhwmEeGi4uTd3f+v1OHLMC8FCUedyu769P66/kd1Z30MlsuVJKj2/OpzeJzhSfriuW0i5Pl+XnjsO/tWr5nqB9ccVv7NHre0ZoG9HTyh+dM+1M3IVRVIAnPDY9RSgnHDfpRyIOdlo3UmTwB70w3EueXx/wGoAWA5AOaU7umB7EGnyIXOx5mc5BY800yOcHMnpTcHOOQPcUuPYkfWjlQczG72IOdxPuabksM9vrUm1Txj8jSFPb+tHKHMQnJ6vz7GgFgBjP4ipCpB6/n3phQ9cfrTsK43JzncfypNzHPNPHy87OelISrdQc/SiwrjCcjpzSGQ+me34VKAnqBmkCqOnp6U7BcZ5zADcvFIZMdc5qQkZA4z60w4zgYz3OKLCuML/XAo8wbcmnYGeMfgKQjoQadhXELKR0H5Uh2Ef/AFqRgTzkY69aY0bZyMA07CuOwpyP600hWBx+GTRtcEksPqKbhsckUAG0DuKTGO/TtTvn5GFyKaeByo9xigBhJz1zTenGaeXUcMvFIGU44Y8dQKLisRk9ep4/u05Tz0FSBQfu7sepHWpEgJGAMD3ouPlGcHkqM96KsiAY6J+dFK4cp7kum2iKAlvCoHQCMVKtuijgD8qKK8M67sDGM+lBhHcmiiqSHdjCir2zSEAc4oop2KTGsAOwpM/LnAoopFor3EpihaQAHA6VyV35l5L5k0hPoAMAUUVUCyqLRC4BJ5pGt0Jyc8dBnpRRWghBbxyZVhuB4O7nNOa2gikZI4kXacfdHaiiriyWhVXjjA49Kc65YL6AUUVVybDSMHHXn/6/9KcFDQqfRiBRRTQhgQc/TPSpY4wxZSeNpP5DP9KKKYEbQqDmiNFWUMowR685oopiJprONHUrwrAMBjpntUX2JGLAnHuByKKKAMLWtNh+xm/GRIrbXH9/DYz7H/PNc8QoQfKPmFFFbQ2MZ7kTSqqL+7yGbHJ6d89PaohdssYIHV9vPWiitDO48XLsyjAzgc49aikuZE2HIw3oMf56UUVVhNjGupd4w3zA9SBTlkl8zy3kZgRu9KKKdibitlGDFi3zAfnTXLB1+dvm6c9BRRTsIYPmjU9MilWQgKjcljjI4oooAcFBbbjBx1HHShY8fxHrRRSExTCNwwSPcUzPcE9e5oopgICgbaVzu4zmo5ZvLYkAk56k0UUAQi4EhB8sA+uf8+tX4oBMuchfoooopiHz2Bgi81pA4MgGNuOv40qw7OAx4IwaKKaEzmvs4jvkIPRwDx15rtI9Gs5H2BWB+ox0z2AoorNFg+l2sEgUK5I7iRhn8M0q2NqVJMKt7P8AN/OiiqAQWtkpA+xxZJxwo/wqw1pHBGHVI+fRMUUUhiKSwHPSs9iTOccZOOlFFVEmRnajq8ttcfY40DEgtvkYnuewx6VjnVyzyLLArFOMo23P4HNFFTLcqOxc0bWfOvPJWDZwDnfn+ldpgEDOegPBoopMaF8sEbgSOM/pRgjox/GiigBQD83P0oVNx4JHP1oooADEVwd36VEWIbk55xzRRQAhcggHBpyNuKr3PeiigCSaMp82/PPp9f8ACoASSefQdKKKYAcliM0ED9aKKAAwkAnd+GKj2/Men5UUUgGhdwJzjFLs4HNFFADSCucHgU5ASM7vpkUUUASiHPVgfwpfs69yfwoopMYq2qOCQWGPU5qX7GiHGc4FFFJsaQgt1Lfw5JxnbTWQKMjA9KKKVx2GAcBgcZzTkj3tgkenSiigEKsStkYHr0pywoOSD+FFFIY4xJk/LRRRQB//2Q==" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAIFArIDASIAAhEBAxEB/8QAHgAAAAYDAQEAAAAAAAAAAAAAAwQFBgcIAAECCQr/xABhEAABAwEGAwQHBQUEBwMIAhMBAgMRBAAFBhIhMQdBURMiYXEIFDKBkaHwI7HB0eEJFUJS8RYzYnIXJEOCkqLCJVOyGDRUY5Ojw9ImRHODhLPidIWV09QZJyg2N1eUtPL/xAAcAQABBQEBAQAAAAAAAAAAAAADAAECBAUGBwj/xABIEQABAwIEBAMFBAYIBAYDAAABAAIDBBEFEiExBhNBURRhkSJCcYGhFTJSsQcjM8HD8CQ0U3JzdNHhFiVDojViY4KS8VSD4v/aAAwDAQACEQMRAD8Au/X8P6atT2FQuncR2iXB3HEHuqChs51A5Wr1if8AZt8DsT1NbWOqv+ler3Vvuqpb3y99S85IDrTkajba1tnx9r/vW7QnSBtFq0GD0NI4mniDb9hZHdiNTO0CV5Nu6olef7Kjh4+yWLpx3i2kBGUF16lqRryjs27NO+v2Ul6Lpl01zcYKoIMR6xcCCdD1TVT8uZt6PJBEfG3YCtAJNjmljQxM5eWN7fsu+L9MEG7ceYeqQjlUU1azm0P8jSx0G+1mvX/s8fSGu9xxxhvD1fmEf6veZaOnP/WEN8ot68gcyZ56i3YsM0bT1UueeoXjDUehT6S9I28tzASXlkmEsX1dzpzTuQmoJ2J5c7JFb6M/pCXYts1PCPGC0pVK1sXS8+nYz3mgschrPxt7Zrp2HT9qw2seKAbF13Jc7hzLuylJ69kAbQ8HbYqXPC8Nq7AHEC4HlfvvCuIqEaAmtut5nKdf50jw2/SyYk1CMyHKjvgKE6kyDvB+Otvd1NxXYgQ1Tqaj/u3FI+4ixG8sD4ZvdJRed2t1aCNU1A7UfBci0TSO7pCcLw8HrUIClFPMmNZidh5WNJVUhxtSCpQbTMRzHyIE+Pyt7HXl6OnBu9hlq+HWGl88yrko1H4lqbNqt9DjgPWrKzgK50lQI7jDjQ+DLiBYBo3ozJ2jdePFC86EqaU+ofaupOYEgqzHw68/h1sp071QpmkS2FQGB3oncCDHSPvFvUiv/Z+cBX0q9RwoilUpZVmYvOuRqTr7brgHws2a79nBwvdShFHXX9ShsBKA1fDSkiBH8dITt/isCOkkjdchWH1Eb22BXm52qw48ohakghBAHu92/wB1gHHnsoS4rNIOVWUgxtqPhy5i3oFef7NC6ypSrrx9fLIJ0S5SU9TA8w41PLlZo3r+zSxk2hKLpx8w9kAE1V0qbBAMwezdc8eRsfluHRVswKpgpaitBLiCAYIOh2OsefXmPdbpCnVKWkPKgQOZBEgmefL7/G1pbx/Z38aaRZVT3zhmoSNcoXXNnqYmmgfE2bF4+gx6Ql2oc9VwzdFZnkgs3zTNp8IDykHkNxZix3ZOCN1AZqKhNP8A3qgRr3tcpPLXfl9bjIcq0OpT2qiNfaGwj9bShefoq8f7tyh3hXez4T/FRlqrGgnZhS+fhZu1nB3i5dClrvThjjClQhMZ3bkqkJO+5KfrlaBaVMOumoha31KBCYBIJynbr4aD5fDsOuAIBIyuCQEmSNOgO8iwz913pQoS3eNHUUrilFJS8yWz3jGkj6jwtyhpxTic4C8u4Ouug6fWnlYZBRBuhUuOElICoGqgADEeR38LNS9aqpZxfRBBKVFCQTtAhWhPuGm+g3s60JWVFxKAY7oIjUfHx95sz8QEt35RupSFBQkKCtjnMR8Bt18bV5RcIsZ1Tvoa1TtMPtAuHClPI6x0/SwhqCkBWYqEzod9DG/v6fOxW61FmjFSASrPCdQI0j4/XWwq1lKUMBCj2nQAnx28fnaEIs2yLIddVtVSPZCkgDU5SQecR7vHlbHHVpTm3kgZgAdd9/d4+dtJUhS1d0HJrBBjXry/Q2DUpktlbqAc+wIjWdBr4a9bGI0QkItx8uFRWdANwBO+3x+uYfauZpcd7MyVR4HoPLX4eFtuhJPZyuVE5iCQOZ0nfYn+tugzmSSZyjugZtMs7ncTP3WGd1ILguf3edSkKJBSOY8RuYkn525zOqXmWCmBsoiAOkdfOwvZLzBIKlK3JG235jQW7bYdSolJBGUCIAG0zqek+FoFw2UliFLRuJjU7EDkddxysE+pT1OtKCRKTAKjGsRPvANjIQ4FJzJSCoxlhWoJ1E+Wm9u3GHC2sOArCkFUAn3ESNefzsFxspgXCZGGal5hdag5iVFsqAESO8SCB56Hnp5WePqtM6EvhkICpWQRunmfHQn4+dmfdbq2q+uYS2esBcQQeszsT4+/Qv67cyruYOQ5XECZB1PXp9e+wHey66K0XbZJ37sZcyBTKgTyk6nTlM9Phbld1t65XnRMJ0UJM+fOfqLOFLSUJCiEJkAjWABHT3H79LdLZbyjs2sxImZIE+MfCzcxNkTbNz/7FFQo6DMCNvz/AB1sAbicJJWW1Aju506bax7t/Lws6jTNKJSQR0jcjrp5fW9sbpStYy6mJmdt9R0/Mm0TLlUhHdNdGGFPALUxTrUIB1KZHU6abfRtj2Gl5YRRgwomEr8Nz8QPKz3RRARlSNJHSDzGtuFsFQUmAAYnnETPz/CwjMiCIKO3LgqDoikdV/iBEqEjST/T8Cj+Hn0qCnG6poBWuZBEGIA1+om0mrpykEqTkKDIzAaH89D8/CwL9E52CnAhRhJga6fdPP52iJSkIQVEqA809AGaCUkQSNDMx5nbwHPZRaoC+yhSw2lXJU767eex+NtXoz2NSuUkgrMEHmSDPnp8vOz0wmzT1VyNpdYbVC1tqBTmO8gfOBZ3P2KZreiY9Vhltthb1Td9K4EyqEtpJMeB9/xG02jrGNxFqlVVU11JZCVCC03l016CIMjx91p9v+loihVI0wkJBlZSCBM+zp7/AMfCOMV0QFz1KW091CwonfvSPz8/K2lQyua4G6p1LBY2UIFp5sylCxHQEW2KusRID7ok5iMx36/dZyOU5J9tf2hOqhzkkHT62O1ivqwWgoHe0JEpkQIk6+74W6MSg7rILLdUimtrNFF5RywBOsW0Kt2e8AeeospqpEZA4UJMg6BI310/Tytx6olSJUxlG5Ow32j6191pB7VGx7pNNSsjUCTzt160cmQiQBA1+vH42MqpG8sqSpMCdNdNvr6FgHqdCGsyM2+s2kC0pvaWOOOwlxebvCASeVuPWne93tFbjrYd77SkpgonupUgeYUT/wBQsVDKiCZGgnnZxYpG67FSUnMEiedu/XVkRlAmdjFgC2scvG2sqiYCTZ7BNmKMtVLaVArR3ZGm9jKrybKezbSrLtmJgxPhZOLbgGqFbdLcwelmygpZijvrLakBHMdRpMncW0Hk6gkTI1mLE9YtmvIWWUJXQ1Qc0Kka9LBTMW1bLSGiYlC01Q7TVLVS0spW0tK0kbggyDb6S+G+KTi7h3hfFTawpN8XNRV8jn2rCV/jb5rxuIt75+ghiZWLPRF4a3ktztHKe6lXes9PVnnGAP8AhbFuZ4ljvHHJ2JHr/wDS0cOsXEFT8qpcjewRqHdwbcne2la25MFbAa3ss7dwn2rZ27mpn4W5jnbUE2MFPKF0w+4l9xBVoQlX3j8LGO2dn2jYmklNQg7ZkKHwIsOZm3Q0ZzRBV5QAUN2zp1KjbQfd/mtxMiLYZPO1oBCJXXbuDXNbO3d5K2sHztm3O0gLpt0L6w6dMxsn3i84H6V4rOiyg68iLHPh7rEr1Ss0pW2RKFJXPkbJw0Tt0K67d6Pbtyah4fxxFtAymQN7cKHvsxGifZdipdIifOw6al0omeQ5WKRoZGthWBKAPdau5tkRuqGNS9EFXzsg4rvC9aPD96Xg3e5ojS0rtQlaKfOUBCSrUGc23ICdbK7y+zTIQVKUYSkcz+A/KyDiXDLeIrkr7rvaqfcRXMraAQcqGCRoUpEZiCAe9Oo5bWC5hk9kKZIaLqvtb6TdwIum777vC5Hq6rrgy+lhbDCV0ockF3OHVBJA1LaghY2UQdLN3iZifBF4JrUVGI78Xfq3E0jtGkLcNGcx7nYJhttSvaKHVoUEmTJ3auIeH1z4lwxfuE2r0dp8QYKfco/WnezUpTKUgrkhQBCyc0nUEZddU2rWzipVLSUGHnr6YdvO6K3KyENoU0wlDS1ZFFaSFpSvUEqKQqImYtnVmHiWBzSbaH1We6tkY4Ai4KngYnwGkBLlPifOBCovCmQJ5wlNIQPIEgdTvbLNxn0g6ltlDdVSXut5KQHFJSkgqjUgpWARPQAeA2tlvODhtbf7v/d/stHmw916aPj7XTqbdtgKEW4fJ7aInrbtB03/AK2+kHbrkWCwQyR1j327gAW4SSne3aSLDspLsCD+ttgRttytyCFEa+7pbsHlZJLYAnb326G2xtyI522DZJLqNrYJjc+VtT4Rbek2aycLoEzvbYJPMaaW4Hv627B3+8WiVJbmAdLb8BbmTJAP9LZMTZJLoEbG2+WvO3MiImPfbJnfaySXRgHTS2z3h8rcE22fGzWSXC6alc0cp2l/5kg2CN13Zv8Au+nEDdLYB+IsYkAdOU2zMAPdZWCe5RN25bteR2TtOSj+UrVG/SbN68eEvDe91Fd54KuSrKt1VF3svE+9aDZ25tIzG2THORaJY3snDiOqi28vRi4I3lId4bYaRMn7O5qVv3yhtJ5nnzszb69BH0b77dRUVXD2jS43AStmqrGY6aIfAnU8rWDnXzthV/MNrCMEZ3CkJnjYqrlR+zz4Hd8Xc1fFADqE095rygxv9qlz6izavX9m7gl5ZXdGNL/pCEkJDr7L4B01gNNzttItcgERM2yeXQWj4aPspc9/Uqhlf+zWvFLTiKLia6tS9e9cyAnykVM8hys2ry/Zy8TEls0GLrkeSzqn1hipbnSJ7iHOU/K3oyDIibZIJn8bQNHGVIVLwvL69fQK46UCH10zeHq9ZGgYrnGyroftm2x8xZqXh6H/AB3upCe0wCw6y2SpZYvSid0E7pDs668uVvW2YNuFobWmHEpUNu8LCdQtPVEbVu6heO9RwC40U7SqpfCfFLre6TTXY89oBM9xJEE2Qbw4eY1ufI3fGEr+oSDKjU3c+yPEArSPDW3syzdt3KYSldBTEARHZJ/K2/3LdREijSkncoUpH3EWAcNJFwUVtaBuF4nqYCXnC8uC3sCrw1OuvlbDTOFklDpU4uQnQaj4actbe09Xha4q4FFTSrdB0hbq1A+4k2bl4cEuF96ybwwZcr5J3euqkdI962ifnYDsMk6FFZWs7LxALC6e+nmRqpaFZSTuZHL6+Vn/AIfC3LnpkqUQlGYg6pKjO28bfC3qhefod+jzezheq+Gdw9oQRnbogwYO+rJRHusiv+g/wNLRp7uw8iibM91iqqwrXoVPqHysCTDZiNEVlZF1XnEkHss5a0AAHdgE8jHn8o8LCIoe1CWSVa94qPkJkbjcc7X9rvQH4ZOqz0F536xlBypF4NZfgqnUfnZDd9Ae7qZTi7uxjXoKhlT2lO09lTG3tN2pvw+oGwVhlXB3VLE3bRqcJUhAABA8+ZIsMzdrKAApI7RwzKVSfd1Hn1tbmu9BW/mwgUGMm3gk5vt7vDYmDr3Hl8/DlZvXj6GnEilLrlPXXK+swEAJqhPn9gR152pyUdS3UtKssngOzlWg0bbjp7PMkNJAKlH2tzIMcudg1ULpEdqQpckZkx5aSPlaeK30TuLtNlDeHqOoSnVS03gwgk+S1JPynQWQKrgDxUoS447gC9lqAhApkB8DrPZlQPXfl7rVzBO3dp9EQSRnYhRC5dyitSUZCACpSspAHgOsiT5RYu7SvGmIIb70jRcE7iNucH42kyv4W46oVpbqsFYjpUpSFOKdu59I6xJTHw0s1K+56mgUBVM1LJSIShaMqieZ1HjvFhHONwiDL0UFYppnGX1JUhGbMFKKVTAjUnof0suYNvFbNzPNgla0vHLpAgga+O29g8Z07jSalJaHbOjMoggmARp0iPvsn4MJeoahszGZJAJmNDO/iBva6xpc1VXHK/RLFQoqcKyJUr+Y+M7+UjTrFmhiinbcoa1LokJbnMT7zty036HTnDwWhZQVSMxMzG8dOsWbl/Ndo1Uo7M5VsqA1zSMvnr+u4tdp/ZIsq0wuFEjzZglCSspBPtZp13j4+MneycQpQLWQ5VCSfEffEHz+FlRSsoSlJcBI5QoGefhtBkdegsmqKHESAdIkHmBz+ult9iyHIJ3u5ieQGpn5e4fpYN1KRlKwgKGh72/1pHlYQrTKlNrBV7KtfaHMjw0+VgFKXKc6CEq2GhMSeXLf52KAolaXkICey0He8hyHx+uhaqALTgTEggkk8/A/XOwy8wSCEqO8iZ25zYJzvIcCsw011nX6ixQo26INKpups5NGqhXe/wAyRp/y27hteVBAVmB1B1nlv5D524p0ld21aMp7i216Db2h+ItyjMsCQd4Ouh9/K0im6LZbZiCNzBJ0M7fI2zKlBCVRI6GIi2FZJlQifGBERP3W5IKjoBqO91G/5WSS2UBU8xpGuvPfrbSmkpJTJmYBI++3RK8xQpskHc89LczEFWhO08/ysklzlAJ0BIEeztblTWsKA12k2EzbnLmmCAVaQPvtpUT3ANehj6FnumsgFoSkab+dg/Owy092dbBAHUeFpAqBC0N7ezH7JXFKr79F6puNx3MvD2I6ylSn+VtxDTw/5nF28ZxuLenX7GfE5LHE3BrjugVd15tIn/7M2sx/7P5Wx8dZnoyexB/d+9XKB1prd16Xqkbm3OYDa217WDPntbhwuhaLhCb7W5iLYg7CLdgA7WOBdNsg1DKtpRgQqPiIsMdLAVH90T/KQoe42MbadbblAf1dlXmHVZpbe3K2CedtEAnbwm18BAssMRbQ1NtxyItozzNntZIrZJ2++wVQgOMuIkQpJEWFEnQDW2lCRp42mG3Ca9kn05zMoJM9221STMWCpXUL7ZhCklTLqkKE+zzAPuP3WGMjTraKley5GulhGP4gdTobBjc9LdBRSTkAkgxJgE9PoWDJYC5Uoyb2Q6W1uLCEplR0Fi71VdSGn0P17SezBUe+CUqA10Gs+FlSlF1stes3lU04CkkjtXAEgeRMWizEfE24EY3dwIbyqKdx5gVSG2CEZ2xosOEBTgABzZEpn+cpANs6XEBSNzkgHpf9/ZSI5rsoVV/SYcvK7eL39tcCV9JTtPXb6reDtUw5SNrdJU2WluuJSmSjNBBzCCI0AtW7D3CPJebWP8cY0ujDly3i8tNCy48G/wB4EqJUUgArKIMFSElOaPaAMXj4v8Qrlq7jaua88P1d5Uj5Q/TUDF2qp6elE9xZdda7FS0CFRnCJBTJtSPEPB7FuNga/E2LKek7d9bya+8nBSAsJEAAKUA4QnQ5ARoRrBNsOoxR0zS10gYDuba/LdCfTtY/MG5vJWBoru9GGno2GG6ildQ20lCXF0zpUsAAAkiqSJP+UeQ2tloapMK4ZpaVmlfxhcbjjLaW1r7Cr7ygIJ/ux91styRoHX/rL/5+Stc3/wAgXrC8D2mp5m3aQY56W06Id5iZ5W7QAD87fQzt1yTdlhURvZMqqgtl54uupS0CSEqjQJnSykuSJ6crItckuNVTROULSpBMbSmPxsSMXKFIbKHeG/pl8F+J1+0OGrgxDetFed4qDdLT3ld6mg6siQjOMyAo7AEjXTfS06sXhUTleIJ5aWgPC/ov4QuO9bsvpTrFQ7dz7VU26lrKorbUFJO55pHOz1/0hN12PlYRoaSuR+7qYvVb1RSuMIUtahkS3nAziErOZMp2gmDBcgcNQoB5B0UqNvZgFCBNgH7y7JeRpoKKfaJMAWBp3CGZ572jnjnSVl7cJ8Q3RQ1DjLt5stURW2qFdm88hDgBGolClDyNhiMXUzIVIFFiekr0F2jepappKsql0z6XAlXQlM6+E2VG6xtxIOuvO1SPRw4LVnCjHCbxuurdau+86V2nq6ULUWlEJK0Kg/xBSd/8RHM2s6K1LVb6qSNUBY98j8LO6IdEmyFLzj7TSC4swlO+k2CF50hIJeieqD+ViVW6TTATupI+YtXL0qMYcasI3hhRzhJfjFGh1qucvCmeoWn01RQWOyBK0kpAlwd0ic2u1hiK6mHm+itC3UsOiWnkK22O1hMwjTXyswsIX5XV2FLjxBfbCKWtr6GmfqmgCkNuuITmSAdR3lRFnYw/mSSDoRaBitonEt0ozBEj42wKM6E2YGOMTvYUwjf+Laa6nbzeumiqa5NE052a6ktJUoNpVBgnLEwd9rQbhv0ynr6cbZc4NYzYcVCSmnWHikkx/Kmy5Vtyna8u2CtkVfzWzNIMmybR1y6lpDveSCkEhW9idVejza6l3ty21TzMJnQJCidp52YRk7JzIBul7N0233thVrvAtCuAfSm4OcS72YuPBXEajvC8KmSzSu0T9Mt2BmIT2qEAmATlBmAdLS1TXkp0AuIg8wNrMYyEhICj+YHnbCY52RL3xHSXU4w08ptJqM2TtHQgGAJAnc62DRidpwBSaZSgeaXAoe7rZspUswS/m6nnbJjXebJVJfjFW92CWHUmCZVlj77KKHErEpNo2UkIFdbZPIe+3MxysC/V09MntKl9tpAOq3FhIk8pNokJIwpUn9bZn8bAt1DLyc7biFp6oUCD8Ld5pEjabMkus3W2wrqfjbiZtqCefnZJ1jR7pE8yPnYTPzmxdr+NJ176vvsICCN/jZgkUKFdQNbaJIOk25J08rYk6kCN9bOmC6nYmfhbYXoOfSwZ+c23IGwsxUwboUK5xpbMxnpYJJG5tuSdOthWslZCBQ3GnvtmczpYPmbYD42SSEzRz99tLQ24MriEqB3BE2DB526CtetmICI0oE3bdiyFG76ckbHshI+VuXbou15ORdOSk8gtQHwBsYG2gtsEe+0MjeyNmd3TavThTw3vsqVfGC7lrSsQo1VAy/mHQ9okzZp1Poq+j7V5z/orw4yVxm9Xu1lgkDbVpKTpOlpSHW2wYNoGCM9E/MeOqge9PQb9Ha8syk4L9VWRGanr6xJ+Aej5WZd9/s2+B16lTlNeWJKEq5MXiiBvt2jLh5nna12Y6SZAtsK00PnNoeFi6BS5z7WJVA74/ZF8PX8wuTiXiakBTCfWBTVUeYDbRPx52YF9/seb9Smbh4zdoqSQKm4ghI101RVKOk/y8hb09SobxbtKxqLT5VtioX8l5AXv+yN4+0xWu68b4Qq0pnIFrr2lke6mWAff77MG+/2Y/pX3WhaafCtyXlGYJVT32w0I/wDugtkfC3t8Fg/lbpKhvysrOHVL2T0XgBfPoM+lXc5X63wXvmoIkD1B+mrJgbww4ubNC9/Rp9IW5ApV78DMfU6U6la8N1hQB1zBspj3+dvotW2y6PtW0K/zAGxcXXdRM/u6mBjkykWQe8bpuW0r5nnsMYkudNYxfNw3ld6g0e7VUjjRkKGhzARzsjhKg3O3LW306vXFdVQkoepypJ3QHVhJ9wMWa978EuE1/FRvrh9h+vze165dbFRP/tEKs/OcOiRiB6r5syhwRm0010I05TbSlEz3IBO02+ha+PQq9F2/EkVvBPCKc2s09z09Ofi0hJ+dmLe/7Mr0Q73Klf6L6ekUozNNeNc1B8g/l+Vl4m27VHk+a8JIIPdTEba22UuKEJSYnSNbe1N7/skfRjrh/wBnNYju6NvVr3O3/wBubdsxr8/Y4cMnVOLuDiHimkBHd9YXS1UHxHZNSN+dmNW0bgpckleR6EHLmGhnWRGn1+Fs7JESVCNNRruLeml8fsbL4QCq5OL7jhPKquBsDwkoqyT/AMNmHfn7IrjtQFZunGOEa5P8KXkVzSj/AMDDiZ99kKuMp+Q9UFUlH8RMnxsCUkGLXCv/APZfelhdgiiw5cF5ba019NtAn/7o7I/LnaPr59Aj0uLiUF1fBi8qhJUB/qNbSVh+DLqiPeLFbPGfeCiYX9lXvUWu/wDsjcRqur0k7zuJTuVu+8M1TQST7S2nWXR8krtXC+/Rg9I3D6lG9eBOPWG0/wC0/s/VLRH+dKCn52k/0DW8TcOfS84fVN/3Fed1oq6167HPW6RxmfWGHGkg5gP4lJsGvyy0rwD0T07XMmabdV7kKMggW4522VamLaHjbz4DVdKFnusIgzIFuBr4W2BrY7Eiu3EZ21IP8QIttpWdtCjzSLYk5hB3txTK+yUjTuLUny1kD4EW16A2JCrS6BCj3W3FtCDvpbYmNLaoCr3Wc5m2hlgcrbJ03tke+LOEyyCBIM25URtbek/ibYZ2FihRJ1SahIRW1CEpAzwvQb8rdq31GptzVAiubWD7aCkj527VKhMzaFksy5G4IG3Kx+6ruFW+X1tpyta7bq5CxFsdo+3TI1ccMJTv7/IWdvYeo0JbpU5lJRKdPaV1PvtXka06nomfIWaDcpt3xQ3Ve7qziR5hm52VJbDbjvZpfeUYhZkSASEhM6qJkGBaMuLXD3hxdlwV9fT05w2mphp28aeudoHHgogqSMiVuPKMJ7mQ5tdbTS8GKTItbWcNkFAy5lZoiR46n4myPifCVFjK7Td+KqRD9A6tLhogshKinUBwpIzCYOX2dNZti1VFHIx0b2hzndbbfPpbonZKWuBB0VSmlPXtThq7r0x5f13UjfaJXWMuPtKUlKyQy0ptxSFQ2YByLy90e1Brzih3CWIcbU9wVeNsVu1F/VyqZ2gqaPs1sNkpzuZE5glAyz3koACZCXIMXz4m33U1l10l1YBoFIYoX6dKKimYCmlpCi2tlkJICYSCCpRSjZIVMlNQ+N+M2g1fdVgm+Lpuy+H7qDFe3dRbdqWVtocYCHKhBkAdqhTeTKFDtNJFuIqaLw02Vri4d+gPodlrts+LMdPJMuvYwNS11RTU/ol8SFNNOrQgqdr21FIJAlCachJjkCQNgTbLRX+4OMqu9dtVfIpDrTh15IWG/wCHMCoQYidBbLE+z4Osh/8Am5Z/PP4foF7HPiHo1t0BpHztt9A7UdTbEjSTvb3Zx1XMt2XBjkYs2MVBS7gvlsGM9JUDf/1Zs6lIkHSZ2FoS4knjWxiioocNVdxu4dvDK32dQwoPsoUkJc7w3M5iOkidrEY5rfvGyE9jnGwCr5wP4EYv4a4xuC/rkxfeQoG6hluro1VBLTzBIStKhse6dJmIEaiRbq9m2DeVC6tsFeVxKTGoHd08rF2MM1dHcLjlCltVc0ypTCVaJLgEpBPSQLMPBeNOJ+LMSN3Vi7h3TXRT0iFr9carQsFYgZQgmSDvM6ZbGMjNLuQxG/spppTFNvsnnZv4zb9YwvCe8C7TKM9O0TrZy0jCksZFDlGnlaKsR8U7r4eV72HOIV217V3OFRpLzYp1PMKaJnI4E95Kk7aAyADpZRuBOiZ7SAnbczCGnaZaUiBI227ird1VRnxOUJiEUrc688yj91o7d9JLgddVL27OLnKxwCGqenony4v/AAjMgCfMjzs4OGtdeuLkvYtr7vdoU3m6XWaZ722mAAlsKn+LKkE+JNnkeL6pRxuAUiPkilbJH+0Ry/xC0I+kT6QaeCF44cp3eHL2Jqa+G6px19moCFUYZU1HdLas2btDzEZec6TNiC8KK5LndvS9KxqlpKYoU686oIQ2nMO8onQDW0Z4pc4WcRaqkqazG9wPCjbcbaSmtYcBzkT/ABaeyLQBvspEd0q071BxQw/hviDc173kxRrZTeNPRlQQ08FIOjyQJKkyY70AjbmH9dbquxSFTMa9bRk/jXA2DbqurAeHLzpa2rfUijpKalfQ4pCCZW4sJPcSE5jruYA8JIugLVToUoRI99mO6WqRsVVdxXfhS+7wxS8pm56WkqnrwdKVKy0yUqU4qEgqMIBMAE6aC0U4X4s+jBVPMt4bxxQ9s8pKGkOMvoKlEgADO2NSSNJtLeIbubvzDd8XGpsOevU1TTZFRCs4UmDOnPnaMsJej7hO4q9q8XsOUKXmlJUlXZIUQQQQZHjYUmu4uixadbKaqE/ZgIAyxuLJzozGvbVspwgnzbSLKdIlDbQQFaARZNUgF28Ez7Sjy/8AVptNm1lB+6gzCfoxYawtii5sUNCnaq7pqEVKHGUEKWUyY1GxmD4Wlq5sYUt733WXdQs1SE3esMuqqKZbGZyATlSsAlIBT3ogzoSNbVR4XcH8e8NsaXBfV0Y3vd2501jbdVQPVS1tOMrISpJEwdDOsx5gG1uylo1rCymFlKgFRykae6TZOGUJ267JF4t8LLg4o3KxQ39Rh9VIpTlOvOUltShBIiNwIsh8LOH9JgbAFbhCkUtLTVRWBJKiSA4M25Mz3jaWGBmZAJJ0shpSlL94tgbvE/FtP5WYag3Ke9iBZQt6PnCi88A3q5edRia+rwS5Tdghqsq1ONtgqBJCTz7o18TayFOQEQQBZsYap20sNlKR7I87OhCIGmnQ2rWI0JurGh2CGKrMfiopSrhS2D3e3QogazZ7k7xpZmcS0BdxknWHEm02i4N1FxTSwLiP93uO3a4pXZP99An2Vxr8R91k3DF9VAxomtU+6pC6p4qGYwZzDbbc2i6sxhXXNxrGDKmUUtdh9i8rtcP/AKQ0+8moR55Cyr3Ks8cHO1Cqxp2pbU2568/oenbKyn3piPCw9mAFSGrrhWJpHw6mZsZBnQfrZIulSlNCTyFlUGBZOFikuWvacnkvn5CwsxrFgWyAtwazmBj3C3ZM8pswF0k28UYyOHKlhhqhFSt1JUuXMmUcuR31sPT4uo3rkN9utqbQ22pTqArMUlO6ZjX9RZpcSAf3rTnq0eR6n87MisxMwrC+Im6KqCxTMvtPJCh9m82jUEcjEHxBFgufy6kRu+6QrLYuZT527gqZsO4wosRdolilfZU0lKjny6z5Gy0p4QCkeU2izBb6qOqCW1khxr46izpxhfj104aqq1k/ahGVBBiFKMA/ObPTOMrLndRqI2xPsEfrMcYXoKlVJVXw0HUHKtKEqcynoSkEA+G9lO7r4uy9We3u2uZqWxAJbVOU+I5eRtUHiRdfEC+7tYawBf5uupbUt51YaCi7p3USZgbkxrt4guHgRi3Eyqe5q/EQU3eSnDRXgkADMQsoJIGn8qtIEjSwmve5vMO17IhiaDkG6tOt1KU5lEBO89BbaHm1pC0EEEcjZo47u6pxFgy9LoonS29U06ktkKKe8NQDHIkAHqCRaHeE+MXLurE9vLQdHqlW2o+y4k6EjkQfko2tPiysD+l7HyVZpzOLeqsgHElUC3UgCSYtBlVi2877xShdPetWxR9qlpCad5SMzYOp0I31M76i0wor0rp1P5iERPe6WrxuEriG9EaSMxgE9UoB0bTbsKBEiPC0C8QeNdTcF31mIWX6xF10kBCKKnDz78kAEDxmdwABJ2myrwg40tY7oKC8WatVTd95ghlx1sIeacBKcqwnTcEc+RBNhGUXtbS9roohcBe+u9lMwVt0tgWBBMTZHvy/WbmueqvV9JUilbLmUEAqjl79BZlL4wUVMGzeFIxThw5UZ60JzmJgSkTpacrmwgF/VNE101w0bKTgoeXutsKmzYwtjagxOp5mlZUhTKAskuJUCCY3FnGlU87Mx4eLtUnMLDYoYL00i283jYHNpoffZFqcYXDRVTlFVXxStPNmFpWuMpjbpaRNhcqIBJsE4Qvxt12nLWyTdt+3Veii3Q3jS1CwJIaeSogeIBmx8uAJJ6Wjo4XCRBG6MhYI0Nug552KNuhxMwdNLbFQlSsk6i0bJw626NBca/G2+0ESDYDtAlOYnztpLgUJSZFoOBUr32RgOHQcrdB0cjYsVgak6W2FbAffaNlMWRrtTOp0tvtZ2sWzRpNszQNzFopWBRkrJtwSDvYDOTOsm2dpOlokJWshVGenwsC4lpyQptKp6gG2FZOwFuVHxFmLR1UgbIs5d9Co5lUbBPXsxYBy6aFcEtFMbAOKEe4Gx1S9OfvtwVHrYZiaRsiteR1RBy6KbKQ26+jxDpP3zYE3UuZbvGo/3ggj/wANlJXntbge0PHqLANHC73UYSvHVJq6KuT/AHda0emZifuULc9leaDr6qv3qR+BspLGm8WC099oeAiPRS57kTCq9Gpo2yR/K9+YFiF01N7uO1aqi72mlKdzBvtpIERuAQfZ+dlozzsVbHZ1ixMZxMeUfrZNpGxEFqfnFwsQuku1Ud6kPjlcB/K2zUFMzSviOiQfuNh9hPI23mI6C1sMPdBzIsa1gaKDqfNpX5W5F40JVCqpAIGx0++xs66m2oBEEfGzZXDqnuEEKmncEofbV5LBt0TOxHxtw5S07ghbLavEoBsAu67uUpR9TaBVuQmPutL2lEkBF73fTTpYfUFQl5I0GwJi3AqTUKLdNBy6KXulP5mxK/8ADzb111KKF51h3IooWXFrSkxvkKoPPSx+jQlFM0lsQnKIAEDbpZe0XEJHKBcJxXJdAoUGqcSS877RV7UfhZaU4lLJeUFEIBJCUlRI8ANSfCyNTX6yimQiozKcAgmOmxszb34p4swuHKm8OHDt4UZeKGn7rvBDxSiCQXUOJbUk6HRGfWAJJAtRq6uKjYZJiQB1sShCGSd1mjVP59lQPr9QVN9yAgq7rafHlPU+7laI+K2McTX+4jBeBqRkpSUu19RWLUhuqazQqnbQkFxaTIK1JgZQU5gVaNzHfHS+r4qkYWZYprjdcUlwtF9K6t5sDUJSYCYJAJKVRGw5NK+Me11xXE+imu25Gqm8nclQusWXVPFCRCnHFZlOaEAZpCQCBO1uGxLH5qh7Y6EHlm13W1Pfe1lsU+FuYzNPa/a6BxIxeqLjq7txbgSr7R4LNXeFE4zeLZMABRp8yGW4bSlKUEKKQEJAPdJrVxBuHhPhk3beGFq29V0F4uJZdK2i26l7RxLjza8qUoJVCUpQkQonWbTHe+P8SIoj+47yw8qqfQptCKa7mlNocglKisNhORJgyVDXSCTBhq/sRUVVe3bY1fqsT3wghCVslVLdhEpIBQhPaKzRrAbcJSe6op0yZJ3SSaA27KzJE2NlrglRPWcQ+J1BVv0NJhrE1QxTuKaadX2S1OISYCirIJJAmYE2y06Lq8HKWpTGFwGySUdndmdGXllUrMpQ6EqJPMne2Wh9rRf2H5/6qnyH916OVqQHzppsALcIE62Hrx9t5nlYJI0BG9vf91yKwJPMWAfom3lSpMmxpKZiRbuNNrNdJE00qUoyhO+liyLmp23y+hoBXMga2Vcse628sbizEJxogENZUxZMvnD1FfDRZq2EOJVoQpIINloi2wieVmNjoUrkbKNmeEmFaer9ZZuelQsKnMllIPxAs8buupqibS22kJSkaaWVyhO8WzJ4Wg2NrTcBEL3PGqS77uWivy6ai6bxpmqimqUdm404kKSpPQg2hm8vRi4eVLpWMP0qSST3UZfutPcaxtFtKbSrWLJzc/UhRY7L0UPYH4HYRwXW+vXTc7DT8QHMuZcT/MdedpSp6XI3lyxY8GEgyEC2w2BpG9kxuTqnec5uoF4k8KeIF64mrL6wtxRv+56eqyqFIy6S02oJAOVIIEGJ8ybNdnh56QFGqGuM15KTtLlChz/xHX5Ws+unQ4e8mwPqLRPs2TuYTo78lJpYBq1JdxU1SigYbrn1PPobSlbhSAVqAgqgdTaOOJTHF6kxKheBK+503VUIQl5qsplKWlySFKCk7iMunUeNpgQwEbW4dpG3ozAaeFiiRzfuoRY07hR5WYWq0YYqP3e22quaYK6cKkJLidUgnlqBZtcO8R8TMR3w5SYzwpd91sUqCpLzFSVKWskDKEEmExOpPIeNpoFIkJKQNCLF27saadLiUJBO8c7J0rzZIMYNVqnZKWwmZ05WijHPFCiwNit6467DWIKxValt1p+hou0aMjLlKiod7T5i0xJbCRBFk+8bjpq91LrzQUpBBEizGQtGgT5Gk6pMw2y6mmaUtBHdG/lZxIiwNPSpa7g0FjQSRzm0C6+6f4LShzizU4hKabuJx2ocQ022tJUtaglKdeZNnYYsh4uwzdmLLhqrhvilRUUlWkJcbWNDBBHzANptcBuolt9lWbi5hb+0NPRYtw0609emGyirplMrCs6Uk52iRyUhR06xZwYUvBurTQXi1HZPlpaOcTH17rSZgvhXhzBF3O3Xc13IZZdcU4oanMowJJOp0AHus0mvR2w5RYzTimmerkZaj1lNOH1BoLmQMo5Axp4RZnyMcwA30U2scHXBCk5m86e7KL1qqUoIGVOgkkkgAD42WKG8qeta7ZkrAnUKGvWzVxlharv7C1RdV2Xi9d1UooU1UsgFbakqB0kEco99g+GtwX/cFy+o4jv6pvaqLhWp99CUGDACQEiIEeeptBzm6AJBpIunuggOrIPSwhiIHnYNEdqf8o/GwmsafLWyBsmUe8RWwa2hOolKwfkbVpu9h7CnG/FVw3o4p24+IDSK6lBkJRVtMIZfa/30BB5bG1iuN4xVTXNS3hhG4UXvVtvFC2FOBADZGpk7nQDfnaPb14dYgxvgakvG8btbu/EFC4m8KZtDgWWXkkkIzDTVPdPLXnFoyRCSa7joRbzBR4pTHFYDUG6eeE5ZdpEFZXka7PMRvEa2XeIic+Da1U+wEKMf50m0acL8UYnv/EJu+8MB3ldLdOhReqKlYCcwMABMAkmZ6AWmLENzu3xhq8LsSUhdRTrQgnYLju/OLRoGGJuV56p654kcHN7KMsEU7T1Q8hwA90aH3zZtNUzd3Ymr2WBlDd4laUggb5VE/P5/BFw/xMu24rxdu2/n03Re9FLNRT1vcSo9QduU7+Uiytct8XRijExXQXlS1L9W+H3G6ZwOZQAkE6Exokb9bRnJhhMLhrdSibzJRKDpZWFu8l5kJUPaEm0GcQsMrwtjxFfRMf6hf4Wp1KRoioSJJ/3hPmSbTrdLKkspKkkEjpZm8ZKdlGH6etdhIp6tKlE7JBBB15C12Ih7Sw7EKo8lrg4dCmlgO7BV1qXgiEsgJAiNf6WlS92lt4frUoMKFM5HnlNmJwneYraapU0tK8jw1SQd0j8jaTapkPUTjRBhxBSffahRMDNSrdY/O6ygS57mpL+p6a7atpDjTyEqKSJnuzYrgPgbeeE8YNLu7FVYzcFPVmsau1tlATJObKVkExm6RoI8bHsIXi1SPth1QLt3vrpn0jdCgSnUbjrZbq8V4hev2pF21hYo2Q2lkoQlXaSDnJmZ1Me60HtEBkDjrfZGYXTZMu1k/wDHbRewXeydz6o4qD4Jn8LQNizAmH+Itz0VDiOkS+zTuJqUakFKshTIIII0UbWEvtPr9w1zAIV2tK4kjrKTavl9YWXjnCFJc6LyvC7Vr7BwP0T/AGToKdwFDkdbSrx+oY4G1nfmFGhP617SL6KRuBeB8H4Boai78M0iWn31BypXlVncAJCZUSdBJ08T1tMCFAjS0O8DOHDuBaasdqsQXxe79UUp7S8axT3ZoHJA2Enc76C0wtwIEWFFfLqbqUpu7QWXSzCSqBpaueL2K5yoxExQVPY1ZeqksO5QoocIORUHTQwddNLWLcEII8LQPithTOLb6Z2BdSsH/MgGxpm3pJD2sfqoU7rVTPmPoom9HzF/EWsYvJGPHQu+LjvINtVTdOGe0QUgiQmE/orW1wq6rd/ddS7TLyudgtSD0VlMWp1wu4mYsv7Gl64Oxfh2koK26EJq2HqZa+zeRngaL6778iItbS5a395XMzVEAdu0CoDkrmPKZtVpvZnIcLXsbdFYqdYQQb2uLqMeFF9VS71oqp+red9aZcbcK1qVKoJkzzzJsnX7j7ErHES92k3tVtM3dVtJQy26UILWUGCmYMydT+Ghfh8VUdc3RKVBoryepjOn8ZH3GyTjtgUfFO+GwSDW0tNUjTonJ/02s5LeKZ2JI9QUEnM6B3ca+hVj74vf1C5a280JDnq9O48Ek+0UpJj5WYnC7Hl+YgrUtXtVNLafpi8hKWkphfdOhHKCd/ws5KFQvbCLJUSfW6FIP+83r99oi4P1ZbrroQtRkIXTq8wFCPkLCl/YNkH4h9VOEfrSw9j9FLPEnHC8HXRS1jVIKl6qqU0zYUrKlJIJzGNSNPmLHsDYrcxVcpvB+kRTutvKZUhCyoaBJBn/AHrMbj80teC2K1IJNHeLDpKehJT96hY1wbfUKK8KYHRLyHNv5hH/AE2VS3I2Nw63B+SjCMxeD0spTzaSTAs0H+IN1NXjU0CBVOrpnC2sobGXMBJAkiY2s6sxLR8rQPW0zSsR4gut1RCHKh5C4JSQHBMg8tDYVQTHSvlZu2352U4gHTNjdsf9FLl14zuutqE0peW24vRIdbKJPSdvnZwFYgkjYWptga5MUcOuKNXhC8MSXneV1V1CirovXX1OlohRBAKjpsqYiRl6Ta113166+5G3s5Lq2ilRj+ICCfiLAhdJzBFLbUAg+RRZGsDC5nQ2QtBim4rxvB26qW+7verWZz0zdShTqY3lAMj4WUKioSykLV15m1P+ENcbs4q3U84YUt9ynVP+NC0ffFpW9J+sqk4GpW6d1xLTtXkdCVQFdxRTPUSLaXhCap1MDsqgmHIE3dTQKlCmi6NR52xp1LqSocjztSHBHF298J4GxHhlu8HGDUsioonkKOZp0KSFhJPs5kbbQU6b2kL0TsXXtX35f103tfFdWBVMh5hNRVLdSgIcIOXMSBIWnbpau6LLEJfOyKJLyZLdLqzi3ENpzKMCwYfaV3QseVos488W3uGuH2lXZSt1F5VoV2Ad/u2wClJWoSCYKhA0nrpatN1+krxfob7pUXlfgfRWoDjDdRdzaWXo3CSlIJGm4V4WC4ZCARujtOa5HRXqJ3iwcmY1+Nmxw1x3ScQsI0mJKdsNOOZmqhnNPZPJ9pM8xsR4KFnOZJ0982YG4upWI0K0TpYBaQKhtXXSw5sDUEwhZ/hVPu+hZiEhoULyg7W2OpFsgeVtDQWkm3Wz+FtGPxtsxbRjysklzy099sAG8aWw+0BbfLeLOAoE3QT6M6FIOx08bJd2LUujQD7TZLavAgxZYKQQfvsjXelbVZXU5QUgPB1JPRQn8LORqCkDcI3zsKxTJrWKilISSUZ0Trqk6WDI2jnbpisp6BZq6peRlCFZ1QTpHIDU68hqdhNq9ZE2SFzX7WSY4tddqhniRhbD143k7dV9XVR16ajI96u8hKikAZS7CtgkzqNdY52jOvwLetY0wrCGOr0u5EuEu1bKa9socVmJbS6QpCU6BOVaZAG8ZrTPWXRWXled84irxUUZqXQxT077Ce0ZaQhJQTvBJVmyzAJJUCRAZ2Kqegw/hWqqL5YcNG42pIVUBZ9ZUUmEFQEkqOm0axoLcuGxTh0j2hrei0XPdlGtyojvPC4xxfT12VN90JumgQH2001OA7VMKAGd5/MokLV2gGUoTAWQkQCqrvEa5r6fxRe9z4rqTddxpqVUFJV1N6rWt2hSv7HMlJU6lgZEqg5UGCRBM2t7dFNdd0XCqrvGqup29rzfFdXBx1CWqZRAgBH+AAJSk8xPIm1M/S4wPi7EGPX2MMvu3hddV6rXNGnqszBdUgNrWsiQpcok7kZwAABFhU9JTMcA0ht9z2VeWRwZnOqkWixXh2moqemp+M16sNNNIQhpm63ltoSAAEpUTKkgaA8xbLVcpMAY6apWWximlZyNpT2YVIRA9nQxptpbLVjglJf+s/l/oheOk/CvoEvBMPTE62AQkAzEWjJXpNcMa5wFKr2ZkxDlIlUHxyLNjjHH7hY4QF37UNHnnu6o0+CDb03xtOffCyjSTjXKVIoEamTboJ5RttZmMcZeGFSQG8YUqSqQA4063/4kiyizxI4ev+xja5Aeiq1tJ+ZtIVULtnj1UDBKN2n0TjHW2acjZMpsWYTrCE0mJ7ofJkgN1rSj8lWUWqmlfEs1LTk80LB+42IJWO2Kjy3jcLsJHPa24GmttgE+Xhbcaa87PcHZRylcgTrFtRtA8rdwLYUyI8N7K6ViuIBMbW2EzuLdRrPW2ERECyTrmOmunxtmWNetugI52yDyA8bJJc5TvBtogWEE87aiNTZJLgJ6C2gPCwka+dtQPKySXESflvbUA8rdlOultEacrJJc5ZtqOYGluyI1toA6aWR2SXEc7bjwtsAk2wgWjdJaiLaIBEW2QBbDE6GzgpIMtIJ21FtKZSSO7YWOVsI5RZikgSykiMultCnSjUWH35W1zsyQNkABD0EfwfjYX8baI+2TH8p/C3QGvPWyTlAPUyX0kOajysXTd7baSlKNDpZQgRbUCysmuktFz07LxeS2kKPQWNhgBMTY1E7eVtEADazAW2Ui4ndMnFfDTCuKyF33cVHWLGynmUqI8iR9TYDC/DHDeFVH9zXPS0aSdQy0lE/Aa2fkJOhBNsygGY+NoOjDtUVkjmiyLsslCQAnayTi7C114vuKouK+aUPUtTGdMkbGRqPEWXimTppbMsHwPhaQ9kWUNL3KZeBOH9yYEu83XcVCinYLhdUBJKlHmSdSYAHuFnaWxEEe+w+UDlvbCkchvaDbhJzs3RQfxH4I3heN/PYtwPiN65LyqB/rKOyDrD56lBjXqffEkklcE8PMd0t5B3FmI2a1ts91unpAyCeqjJJ8hAtPBaSdFJkeNuPVWhqEi0Jmmc3cUWGYQiwSa3Qn1RTJ/iTl1tWu8Li443DVOUF33Nh+romV5WVLU7nLYOhJkCflrztajsgNIsXeu9l7vKQCedpSFz4+XpbzTxSNY7MoJ4WDidUYqYViXD933fQMpUtxxqoUpazBCUhO25nXYDrafWwYG5sWauxlleZCB7rHEJKdBYETOWLIsj+YbrZEp38LQBxjvqmwjixdZV3deLqK5hCgqlpVOgFIglRGg2HjawBTI2snXnc9JeYCahlK42kWMXHluZa9xZDY20gffZVWuLEOEL2xGitu2769V8OseqCKF5J7PPmhRjKNeaojr1srhmiXR3PT0pn7NEHzOp+Zt1T4Ru5h3tG6VAPgLLrFIlpOUDQCLU4YnMkzu7d7q5NM17MoVZMRX5T4D4k3xdF8u+qNVz4vGhecMNuZzqJ5d6R/unwkDEFaxiHFbOI0VKExQpo1NpUF51ZyoEEe8R4/GfsbcOMM44pkU+ILpp6zsp7NS095E75Vbj42aGHOAGB8MXk1eV23MhD7XsKWta8niMxMGx6uUuLnR3BcLHaxQ6ZrQAH2OU6d08MJUr9LhmgpagEONsJCgrdPgfLa0L4OIuvFTtIpYT6pfjzQA5J7Yj8TaxLFKGmQ3A2i0dPcGLkXjhzF7j1couPeserCpUljtZnMUDfXXXS0S4Cm5TvL6JNH6/mDz+qNcY6T13hxfCAnVppL48Mi0q/CzU4YX9S3W+6uqqW226llCgpwhKZB6k+JtK9/3I1fdxVtzuylNZTrYURqRmSRNoAe4KcTKFfZUHEAFpCYShy7kK22gzP9LNUyt5LBfUEnr1+CVPGTI49CFYe7L5oL0bUmkrqZ5SRJS26lRA6kA2h3FQNJxKvFrZNU0y+ByMJCSfiLODhHgvE+G3amuxRiBN4PuthltDdKllCEyCVGCSSYHQCDprokcYsNYyOIKPEeEbupK3/VfV3kPvKREKJBEAzv8rMyVslNI19hcH/bdIxFszC3oU3cfVDC8f4IrGWzndbrGXCOY7JJGvSc1piwmvNc6G9SEkgSPGbQncrHE6tvBinvXB9EyAYNQasKDaecAJJMwNLTzcF3+pULbIT7KdfE8zbPjnfPUMcRYNbbe/W6tuiEUThe9zdVKeAw1xXVJyJor/UroAkPZh7oNp34/wB3/vDhpWPBoLVRvMVAA6Zwk/JRtFPF/B9/ucTqtu57irqpV4OtvsraYJaBKACSvYQpOsnTe0746uypvXh1fNGtuX3bucISnX7QIkAddRbojOzx7ZQd8t/QXWS2NwpTGR3Xn3ftYbuvWgoH0j1a8TUUi9IhyMyNtpCVj3i0w+i/WLu7iNR0jjgJqaR+mJ2JypChPn2YPvs06Lh5WcSlXpS3RSrefoGFVlK8EnK3UoUCElWwKgFp95PKx3g5UVl3cSsOOMsrU6qtbbcZSk50BRKVynlAKiegtWqWBocGdCiwuNwXdQpY9LiiccauioB7qqd9M/ylK21fdaIb1fwtfPDvCF2VV41VLfNKmpcDrCEHIgPKRBKtJICYAk6eU2K9Jy53KzA9NeyEHLQVMOq/lbcTln/iy/G1QLzuK4b/AMOLuO9aEmtpKlD1G8ASktlaS42ecQCepk7WeVrS2KXyIv2P8lSjJBkZ5g/FXJ9GmowsxhGpubDovIqpnkvVblaU53XHEwFDIYAhEQANvfaYvdatfov3tTKv6+bvpyAl2lQsACAOzcKY92ci1kxoOVsuJ126q9KADosJHP77AVQzMqGnL4WGJ1twsSgpESQYsRBzarG1lbSFdUg2EEeB8bA05CmAAI1I+elhYV0Is4Ce62bagxt77dAaQbaJy7WlZRJXMAqmLb2Glt7iba5SBvZwLpiuY1MCyU6OxvgQe6+zr5pP5GyuTOwslXolxFVRPoKQkPFDkzOVSTt7wLScNFFpN0YIgQbJ7CX6m+fWlgJpKFSQyAfbe1zqPgkEJHjn6A2USDytxlI5DefnYcsYeBfZO19l3jOiQpsK0ylvppI+haE8YXbTLcoKd+obqE9uXW6JiV1FU8hOZDaRIECMyie4ANSBNpXxpemKl0rTFy4QFclTIPrL9ehhsK1BToFK/hB252izHNPiS47kvG9KS71esmjQqp7FC6kl0gAtoUAAhoLKTqADClnNAA4rGsSZEwQROyuPl2PmtCmYXNs7VRJflDU31jA0GOb+au676ls+p0LKyAlptAKy9UgEJBzZilGSBGZZAskXxQXXiU3/AOpYJum+HFN1Cnr0pMjibvCSlLYSjUqWoZnDIbkqCR/NZx4wuu8bwbZpr9vJdBd3ZjtKRkU4qVqJCcqUZ1IjdWZKiUyBBIgw5jCmcN5vNXZxITR3FSoQ4ijpqgtLW4pJUYSRmcKUhGpKyVQEidEcPWTVFZmZKbW6kWN/l/8ASvBkYHshOimuzhsxTNMP3ReK3G0JQtSrzu9JUoCCSOzMeUnzNss3Kb0f6Ospmqut4nVtNUPoS46yXHFFtZEqTI0MEkT4Wy2Xni/tz6FQ8O78H1RhNY+y4QFGIjVW2v3H8LLNFfPaQFud4AST9dB9a2btSFdrnQqVRqeWnhYamUkAJJUkgHYmRz39/wALewOAcpNOqejFah0ZkqMTqfr62sdbehIMmTBMfXnZoMVTrSigzAlOg10n87K9PXuI0KiUke1MH5fRtXc22yMEs9nSrUQqmaIPMpG1u0BhtQKGW0q5QkDX6+6xRqqSSARmG0bbWHS9l9nnuc2pE/OwtQp2BSnT3rW0wzU9fVMiZlp5SfgQfqbKDGNMWUxCWMU3wnwFc6I6/wAVkDNKQFAmN+ev52EClR3cse1JO1pCR42Kjy2HcJ0U/EfH1P7GLr3nX26tax8ybHqfjBxJZGVvFdSemdppfT+ZJmzJCiQdTvvM/wBP1t0SSAQowZkHmT062mKmZuzj6qJgiO7R6KRGeOfEdr2r5ZeGnt0bX/SkGygz6QmPW9HG7pdIg9+mUPmlYtFxKpEHXMIje28ygRm5iARvEWIK6pbs8oRo4He6FMNN6RmKgkiouO6XOfcDiD81mymx6RV4AhNThKmUTpKK5SdekZDaD2lQrMdBPMbfX1ysfBKYBTrGuv1rZzitWzZ6Qw2md7qm9n0iKQn/AFjCbyNAe5WhX3oH0LHGvSEw3P2twXsmP5S0r/rFoHTmyZhGkAGBr8DbqCI5CNfD66WcY3Vt3d9FE4RTHorAs8e8FuR2lDfDcj+Jhs/c4bHWeNnD50Su8atn/PRuH/wg2rgJMQZ1BGltnvCCPDfexRj1UOyGcFp+l1Zlvi3w8dgf2jQgnk5TvIPzRY61xEwK/o3iu7p/xPBH/ii1WAo6grgb68zzFsCsoP8AiPPaxW8Qz9WhDOBxHZxVs2sXYTe0YxPdK/8ALWtn8bG2ryuypANPeVM7PNDySPkbVDMd3MoaHZQ8vlbOzAEAIUdoIEWIOIn+8xQOBN6PVxQUqEgg89LbynkPjanaXVtGULW31hUQbGWb5ving0971jf8uV5aYPkDYg4iHVn1Q/sI9H/RW6KTJFtZY2GhtVJvGOLmigNYmvgRG1a5G3+b68rH2uJOOmYyYqrtP51BWn+8DrYg4hi6sKgcDk6OCs/lMwdbaKTuPK1bGuL3EFnQ4h7Qf4qZkn/wWOtcbseNe1UUD3/2SlGv/CRFitx+mO4KEcEqBsQrD5T0tgGm1oFZ4+YwSMzt33O4B/6l1P8A8Sxxj0g75SR2+GaJYIIBRUKRPxCrGbjVIev0QzhFUOimlxJ7ZszMggfK3ZSrY9LQur0hn1Os/wD0ObVBJXF5ERpH/ddbKLPpB3WSBV4Yq0EnXsqlCwPiE2IMXoz76G7DKpu7VK8RrbQGto2Y4+4SdIDt0Xy3J/7tlQ6f95Y+zxvwE8AXKiuYJP8AHSKP/hmxW4lSu2eEM0FSPcKfURbIP9LNBrjBw6cIT/aAoP8Ajo30/PJY4xxKwFUABGKqAToO0UUf+ICxBWU7tnj1QzSzjdh9E4wJ3FsG4H4WSWcZYPfVDOK7oWZ2Fc3/APNY61fF0VEFi96Jz/JUIP3GxBNGdnBLlPbuCjI21FsCd7Y2ttYltxKh1SoG3YSQNvlZFwOxUC0rjeJ1thE/nbrL4fK24BO2lmuo2KDibb1626iNbby9LJKy4IB3GttZRGthCCP621Gu+3jZaqbWhckcrZHUW6KbbgxaKKNFxHhbYSRpuRboDlHnbAOdknWgmNxy2t0AI00tgBt0OtkmBuuCmfM2zKnpbvfSLZrzsktlwAJ3tyECZ3sJEaW1ubNZOuSgEac7BLpm1e0ka+FjECNDbRibRICcEoBFMhGwAtp6nQ6IWgR462MkeFs5WHlGyJcpOF2MBWYNgEc4sYbaSjQDSw52tmUb7/jaOVrdk+cndEai72H3krU2CpOgJG1hDTp7MtkAg6RysYAnQDSbaUNdLRDADdTDy7dI1NcFHRqX6swhAWSTAjWxGlwfc9Bebl50V207NQ6SVrQ2ApRO8kWcsDnNtRaBib2UxI5Jt43VS3nQPXfW0yH6d9Cm3W3EhSVJI1BB3FodrfRmwMK4v0lLUNMkk9gmoX2YJ6CZEctdLTkYCYgWDKBOoFk9mZuW6THZDeyjfhjwfwtw0DguG7yhaxk7RxxS1BEyQCToCdTG53tIewiwhQBt5bW0UwNDpZmR5BZO95fuuDMW3AI0FtwNhbYkDfWxAENFqLQut/yuGPLb7wbGfd8rFmTlrXkke0AoeX9SbGdedkNk53Wokb62ydd7bOmttHlHntZ0ywzytrxtkwbZPW0mhRcs0IsQvZJVSLWndCg4PcZse5RPOwD6O1ZW3tnSU/EWJluFAnKQgjCoKeYkW5I05CbcUiyukaKpnKAfMW7IJMAT5Widk/VHK3tXMJ1HqxAfazJaJE5Vbg7jaeotEGNaaqrLtp0Ut5Kdok1jNW62z2Z9cDe4ClqAE9RMmBItKt2IcraKpF4IAgB5qmmQhQ2KjGqttNh4xJYGMWWWqpw5TmbJVoJkR8t/lbka/CnVdQ6cGwNhtrvdaFNM2NpYdeqirHd0JxFftyet3Ez6iUKdrCX0NvQkpLbS3QFKQ0SFaNyVd0EpSSosniK9ebF0V173Pwwweitu+lcVR1jzyXhTBptXZrDRYEwEghExMTGpDmvnE7EqpbgoV3nVpKlPNUakpbaWZUrtHVEITrpElWvs72RU01e+07U4leKUFQCaKhGYKzQOzUVCXJgiRkEEyNMwE7Aqfl86c5n28rBIVbs2RgsFUlHHy720Jbq+KOJy+kAOlm93QgrG+XQ6TtqdLZZ5VPCJpypdXVcdsH0TylqLlMiju5KWFTqgBAyAJOkJ7ummlstzXJw7+T//ACj3l7/z6p6vMKDiypoFIVOoM627YaASkxOXTQD6Nj1c2xTugPupaWTlAUcsnoOptzTBC0gh5HmlYI9318bdO2QvFwtLlhp1W22sojKVaa8p5zyj9fGwyEEahMkcyOg2j5fWoqGgBCFQoQBlEctvjbpLKIlBA58oAHL5T77PmPVPou2HVswhQG4PdkSZ8NT9b2OU78Zc6CdN41+t7FEhJIDago6b8tteXK22s0KCSUz7QUInXy+vjaGpSSu1VGZJnKNAZJG/L3WHDiVpIKlDKBJ31+tOdkZLqwc0gA6idR47WNJdcTEkmYMx8v687RSCVtV95JB0zTqY/Pfl/XAAkd5ZI5iI8rE0PtEwk6+0DO1jCHCAUjkfEH5e60SpCyFGqTpm1kb+7yt2EjN3iTroCev1ztwlSCrIpUknXW3UAbFJG+nPTx8vlZrpbrpuCQlKjpy+vr7rKTQ1mTruJ+voWS0kJXIEHQyellNkKKAACOcbf1sF7kZgQoHUASN9wI6bfXyEJTEEabROu/TY24AASCJ1I0H3/XW2tB7KQCQAJE+6LBuirtQjQEAARI8rcZhqMokg6CTy+vnbZSpAgnU6687crTExlISDpuZ/KyDk1gsBzEgHQc1DSfdbEpIQpASCfDmDbSEkgJKoB303t2MmYaZp3AtLMkGhYlSo0gd0kHf6/W2wNFAJmIAg6c/r+ttxlkqOsaHLHTT5W5mQSBG/ws2YpWC6lE5lctDPzI+t7ckfzAa67bW6IMgSok676D6i2iTAMkgmZ+uW1ldKyxKdNRMGCAeUf1tsAqGgB05jl9RbekQIB5eX191taKSQiIEbcvr66WV0soWZhIIM6dLYrNIza5tYM2zLGUlWoMk2xSRJhPtacoFldSDViimekHSbcFZgnujTYSBbsqBWASIA2Go/p9b2CVqsJnTdWg1+tLSuolqxCSo6JAJOoJt2QmIJOmgnnpbESFggjaBJ1toJIGUeYj75srqOVdApVMp3AEHQ9bYVBJJ3InX7vz+Ng8pAJzSd9dvDxtmaSDlk7jWyuVLKhAsEwdSnvfPn9fpiVTqFnaZ+vre3IIkHeNNY+uv0Lc5iE6ETAO/LpZkrBC5gIyzrpz3+ptsJQpJCgCZ3KeRsFmSnRYOo0JJ20n687YpRG87xp+f1+b3KRaChB2SU5kIAIOkCOutjTN53gwJYvGpZH/q3lp+472IIOvdJmbdwnKUnSNuf1+tnEjm7FR5bTuErt4sxPTqHY4lvZA6evvR/4rHmeImOaecmK7xPOFOhZH/FNm2NTmAnfbfX+ltqASMpT5na0xVSt2cfVQdTRO1LQnizxb4gtSDiNSj0XTsGP+T8bHmuNOPGozV1G9rB7SkSJ/4Y8bMILOo1BG8C2s0SVlJ6nwmxBX1I2efVDNFTndg9FJTXHXGyIDlDc7hPL1ZwH/754HlY4zx9v5J+3uO7l/5FOI+8m0UpVG5M66wPrxt1nUganTnMT/WxBidW3Z5UTh1MfcUwo9IF8aPYVbImJTXEf/DNj1Px9utWj+HapGsfZvpX94FoRzq7syORG8npbRWdYVpEnT7vjaYxirHvfRQ+yqV3uqfWuOuFF6O3bezXU9m2Y/57HmeM+B3P7yorGZ/npyf/AAk2rulzU6AjlpGvS3ZdSpBHOSJ6+FiDHKpu9j8lE4NTHurJM8WsAPaC/cp2IVTPDX/hix5riDgl7RGJaIHfvrKP/EBar3aHMCCDOxjf7rCIdIGYaggxztMcQVA3aEP7Cg6OKtM3jDCTh7mKbqM7f643+djbN83NUa097Ubv+R9B/G1UA8UjMeQ38Ldl4HURAnf8eXWxG8RSdWD1UDgLOjyrbIU2sShaVD/CZtvKQJi1SEPobBLZCYJiAE/X62NNX3edMIYvSrajUZH1iPgbFHEY6x/VQOAno/6K10dBbQEdLVgpsbYoZJCcR3l3dTNSs/jZRZ4kYybT9niGpBInvkK/8QNpjiKI7sKGcClGzgrHQZtkE6zNq/U/FjGyDC78Ln+Zhr78tjrfGXFzYGdyjcJ/maHx0jS0xj9MdwVE4NUDYhToRpbRGgi0LtcbcQp0dobvX/uLH3K+psda44VoH29xsKjcocKR85sVuOUjuqGcJqR0Us5eotoieZnlaMWeNzLkdph/zipn/osba40XMr+9uirBmO4pKvvi0hi9Gff/ADQzhlU33VIBGmltESZNmOnjFhRWjjNeierST/1WNI4q4LcVBr3W/wDOwr8JsYYhSu2eFDwdQzdhTsI05xbR+os3E8RsFuwUX2gctWXBH/LYdGOsIOmU4hox0zry/fFpCrgOzx6pvDyjdp9Eskf0tojlO1k1vFuFqgwziO7Fk8k1jZI902Nt3ldz+jFfSudMryT9xsQSxu2IUDG8bhDGBFtbWwFKtUkK8jNs2P3GxAQeqjYjdFnBlrW1g+2kpj4mxkRysUrF9munXmj7QJn7/usambONUxNlkjrpbXnzthtkzztMCygTdanWbZsJtknW3K5FpAaqN7LaiJgajytwSNgDbZMjTnbRAG5jzsVDJuiFNI7Zo/wOqHuOv42FMDnbgjJWu6aLSlQ+6wiezSorW2VTGxI2Nq8zntjzRDMe17KYAvqhKNZS+J2VIPLe0ZYrrcP4pqn6E17VQGEinrrvC8q0qBk5yCFFJEREAjeQdJGDLS31VGdSHVEEHOSTrt0gWhPjF6N93cS7zTifDWKqi4b7SoLNW0lS2lCCkZShSShUwTGaYGg3t5vinFNZRVIZWU5Yw79b9iCPqtBkLS3Mw3KSX6C6buAo7tom6GiQsoaYbHcZRy2GwB1MfG0QX7iLG11Xm5fjXDRT14MuCgSpN7iqp2kAlTquyS2hSeWZzvJ1T3gnKVTbd/DbE+A7ni+MRVuNXm21rSCmmo1TkyhGdwqKu8TBge1OuVKbIVPc98X1f4ddwwu53aNDYXSdnSuJywCVOOpRqgRAyqT3SkhIKotm1PFrcXk5FG7Kxgvqd/krEVKGDO/cqu9XhzBXrT3r18OCp7RXbZjQznnvTFQec87ZaencCcInHVrvLDuI3atSiX3P3LU99ye8dFDczyHkLZbmTiEn4/qrfJZ2UEftI33U8LMLpbzZlX6FEJkHSldA2PVW3jan3CZ6sfepj6w8Ul5CsvaqP+0gzPn0tbf9pQ4E4DwjTSqV3s6rQnZLA/8Am+dqqcHWypqgfcKSXX946rBgdfZ1529c4QaHYbHm7n81X4jcRWvt2H5KcGDVpKeyqahKwdAl0iROosZTUXvCQq86uZ5PLgDpqfxsPStSSE94GSABqB1PTUfXJRSykIzKmQM5B1AjnP1tbuDDGfdC5Zsz+hRVF8Yka2vyuMmCFLKhGvyj7xYdGIsVpBAvd3MNO+hJ225WH9VbR3Qgb6aaz5fX32EDASTpJOgEwCR4n60HuGaWE+4PRTFTM3ZxWmsVYpUYNchagR7bSZPyjlY03jTFDSFSulWZBgtnQ+U9fP7rcinggKSSddSrU/R+udteqIVECOQgAwP6G0fAUzt2BLx1QNnFHG+Il/tohdLRKI0EBQ/6uk6x0sbZ4m3sClH7pY3mEuESfgbI/qiHDlWkHaSEx03jlbfqSNAY0Ox2+Xu+NmOFUjt2BSGJVI2cnIjinUyUruMGU7h+I6nVNh08W6dB+0uSolPPtEmDPu+hys1jSlKTqTr4TH1tNgXqFsCQOW4VtEfpYX2JRu91SGL1Q95LV5+kZhK5axFLely319ogOhTTTTiQJiDKwZ+uejvoeMWF6qlaf9XvJAdQFJzsp2ImdFeVqocWEinxDThIyzSpKSBzzKHPUdLSvh+kDt0UWYAlVM2TOkykdLVv+HqN7jofVWRjlUxo2UyJ4t4TGgcrASZ1p1dfxMWMs8UcGuTF4upI5KpnAfjl8/h0tE/7sSoBKEEGZmNTr12sILsBEAk6wnuzO/6WieGKM7E+qccQ1PUBS4jiHg5+Ai/mEE6kKSU6mRzFjCMb4UeKcmIqHSZl5IA+J1O/0bQybsSg5EoiRtPLa3H7ubkEpAChB0/GwTwtB0eURvEc3VoU5M4kw+4nM3fdEqBooVCD9342NovC7lE5K9gkkbOpJ8Ofh8rQELoSshS0QAefIwedknENxtm7RmQlPeESRpod9PKwH8LMtdsn0VhnETydWfVWaS424mAtKgDuNdbdJglIBgp1MdPL3WombmdQziFpl5xslztFFK8pgoGuhkeYtxRnEVyM3PULxFezRV2jroTXugSoZtRm2HIHaLUjw47o/wCisjHm9WfVXw7oKVE5Y5D7/G2ZSVnQCNByg/DwtQ+6cRcSW27xWrHGJEKfp/WWki9nzkBUqAJXA0jbw6WNU2OeLlHhWorE8Rb+VVracfC3atThQYVCQFzoJGnnaH/Dkx2cFMY9F1aVeiDIVl0gbGdPxtrJJGbXzHyNvPZv0kuN93qpEDiHUu/ZCUuUtOrOqSCTLcjXoeVnfeXpLcX2K1qhuzEKFpQ60h5bt3sqzZtx3UDl+O9gu4fqQdCEYY5T21BV2yQEwpRj4eUWzKN05pnT8LVbY4/8UmUS8u6XNN3KJafHUJWOlktr0rOK/wC8V0P9nsNPZqhSUSw+2UtAmVqh48/dobDfgNW3t6okeN0p7q2/gNYgEEkHr+Vgm8szqSdZjraudX6TOL2qbtP7L3WsIGZcPuJzCNucTprrbnDvpY3retEqqrOHjNOQ5kKG72KuQPNoeI91oHBatvu/VTGL0jtc30Vj8pKd4EQdeXS3ACiddR7Uc7QBQeltQ195uUX+j+vS2xOZ5uvbWJHISlM/Gxu6vS0whXVr131GEcQsuU4KlKSGFoVEbHtAT8LDOF1Y9xT+0qX8anZUiAnTpI1NtGSABKdjHIe+0StekxgRZLbt1X80ZmDStKHydO342DX6VPCBh4sVd4XmyUqykqu1xQB107kz8bQfQVLN2FTFdTO2eFLplRKQSOcjrFtKWuTOgnQff9eNoroPSf4J3k+GWMWPB1WqQu66sctdS3FlD/yguDSBke4hXc0oGYeS437u8kdRYZpKgC5YfREFVAffHqpDhwKISNTpH1v526J0BKSYOkdbM2l4ycKKymFWzxEw8WUkhS1V6EpEbyVQPr3WUKPiJw+rkh6ix1cDyVaBTd5Mrny71hmGUbtPoiCWI7OCcIWuRKgO8Z1iPrTXwt2gwZVoI3jX4edkpvEmHKtRRS39drxBkpRVtqI+B+tbHmqhh0S0+hZ19lQM6/0sMxv7KYc0ndGEKExyA5R8/ifnbASNBpE85jnYMLCjmUVQDyPLT6+Ft51JSoj3zr74O1o5SiXC7KgpQI13Ij66W0VBfdEQYg7AflYPtCoCR56HwnT63tgWkmFCY7oCuXh84srKNhdChRKc+oUYInlFuoBJCQf03+vOwQVmT3iddpFtKWAMpUZiAOR3tElTXYMEBManUD4W2kiZ1EQJImbBqckHWYOpiI93mbYXOeXQiRP142dKyGSvKskKII2I++3SnFZYyyIgTrz5x5CxYrBEJCtY3Ig9LaCgI7qu8NtyJ+hZJ0ZClSYMGY6ydOfO3QdITMjXWd/raxUupKYQ4BHLkPqLbU4okk6HcQNY5ffZrBJGlOEKOWTOu2g5W0X0lRgx0IJnXX8LEy8oyQZ2Ik9RYNTwMHbTMQfz921pBqiTZH+3J0J1Ps67a76WCVUj2ZI1kEjymxNVSk6EghXxiwLriUiSRMkxMaeHvs4bqolxSm1UgqmQNiJmxlFVqBPKNN+tkBFQJEk7mJ2Pv+FjaKiUwkEk6g+Hx91mLE4clYVRlRzc80c/wttNSYKs+ug1URHjZKXUlKZKtQYjbXp+FuVVCwJA1GkzH0frxLctPmsln1kxoszrJjW2Jqu7lECBqT/XzsiirKE7r/Tc/XhytgrF5QFEiD5a2XLSzpdTVAzBjwE/D662EFZCQFHNJgTM/pzs303gdSrZJ3B0sImuCQQCSDoJ0+v0tHlJw66XHKtWpJ1O8EwbFzV97cymNtJ+oslrrAUSs6jTabF1VyQoqmPgQdNRZwyyRcnI3WjYrGU+O35/0sp0dWC2uCkzv5WaFNW5hpm0OonWyjQVKznSVaFPuItAtslul01ZOhjadeW02CU8wTq22qT/ACg+f3WTTUHQlSvEEaGwS65KBnBzSJVB3H0LPmsEsoKVk1DTZztISgifZMGfdztp++6tpQaTeV4haj3GqaqdSpW38qhZGpKmrvRR9TSEtAmagiUx/h/mPy+6zjum56an2Gd1Wi3VarXB0k/hZhJI42aUxbG0XcE4cDs4irb6Zr7yv+81MsheWkXWuOtKKhAUsKJkiTEaedprSrMkHqARaNsOsmnbCgIiCOulpCoFlVK3rJSCn4GJ+Vu2wqJ0cIDjclcniLxJLcIwdbaKtPO3M6b20SDrvbXDRZZxK6K+vK2iqddrcz420Tys4FlEm662tkxB00O1uZ1jfythPM2ctDhYpkyKu6MZXJexeua8PXrvdUpTdM6U9ozJnImfaTMxqIB6AAla7Ht+XG+Ka+sN7JBU6jtUAJJjQZFAmSn+KN9ZgF8V0lDbiT7C0k8tDobbzGCOW8Ha3DzcMYhSSukwuscwH3Xe00fC+yutqo3ACVl/PZRwvj5hGleUzUUNSgBJCnEw4rNOgCdBGXc8ttd7LL/H/hXeaKNtm+VMPPN91BaJE5c2WUSkq0IhJJmzqdwzhy8ad2ovq5KCoQ77aV0yCpwjaSR4n4mxSt4ZYCrbiLrGGKZlDlZ62tto5B23fSFEpiYDijHWOlsWVmORxyc8skDbgut9NkZroM7coITHc4i8PxULpUYupa9x9pTgpWFBQ7MQFFal90ZQdhEbmd7GKTCF1YgQ9elLdLTV4OqDK0BwtN5EGEgmClQTKoOU6qMRM2QsR8DuG7d2drWVNfdopG0LVVN1WVLYb3X3iQnSZOkcoGlmNfOFsZXLhq5arhbf9/ppG11TTrdQlxp68u3WEtZE5DlGVKoXmQRJclM5reUYhS+Jma9rcjjobaD6LYzuawpxXjcmFaG8KqiPFCmoDTvLaNIispEJp8qiOzCSySkJiIJ0i2WIs8KOICmUKTcmCacFIIZeoaZ5xvT2VOdn31DYq5nW2Wz/AAE34j9EuY5VT/aTVKW7hwOwTlKq6tUAY5NM/natvCOnlq7ezEg1YOmwEwQfcqfL3WsB+0vfcQxgFgq0U7eSztpCaYDQ68+R8LQTwdQOxuvMc6VOpVBPs98fU2+keE2luGxfP81n8Q61snwH5KeKenSIUsSSo6DkdOtjrbYJC9DmICoET0tlM2pwDMgEkHNInT85mxxLYKc5B08OfXT+nhbuFygRfIrMogaROkR9fl8BEtNgxlJA32P1z+Fhy1kMqGmggE727CTIzJlKo5x0G5sk65CJMGTCgY3M68vra3RbkBAUZIzHymJ+vnbaUFaQsDQp1HP3WFSMqJ3BkwTt5zZDRIhFyjvQgRBInqT08bYWwO7Mge4T+n52M5MyjOm5nr9aWEbQQsyCDMGRyHM2kHFRyhFcpSc0kkecxufrzsAtrKMxIRzKuWn32PKbCiEqSCmZ328PDnYFxAyBKhA9k676nw8voaECE4KC+MrahiOiISdaQZknYnOrY7c/6c5cw22n9y0BKSqaVklQH+FM/wBbRXxoSUYgoSYIVSfxEjZaiSY35fCbS3hVOe5bsWFKy+qM65tSMg+NoM+8VJ/3AlltohJSWxO8ff8AWlhlNKIyJza/ETH142xtGcAHQ/xHafj7vKxppBg6Dw5j62sVDARYtDvEkydYgafgdPvt0WDATCRtHh+n52N9mPa0jXnpP3cuvO2+xSsewQPD66WjdTsihZSknuDMDJkRJP191kzEDBN3EgI0WmZjrufh99l0NlcKRyG2h5fnZPxAlf7uc1SDmQQPeNbRcdFNu6hm+ID94UjIAVUvNZh0SlEk+OwHv5WFvaiTV3TdjKE/aOhCElR2Ck6n4fM2UqqkQ7e15vbhqnyAHmVJ1+Q+fhZNb7WsqruVB7OmUyyY/mKZMaanQD42oZuisWXN4tMXfeDvaITl9S7NI5bwNPP8eVi1DTIXhpKcmgbUCD7x+RsrXvSNVl+MU6oyoQXFgGM2USB46kEjysjJqnKW4W6WmTL7xUlIBBAGYyesRabX2KaygW96R5iuU8rKnt3XXCncNDPlSneTonnpaUaK5mKO67pabgqU4h1xZTBWoifd9RaOsROFqobQ6JT9pAJA1zkpnnGvXmfO0qsJKrtulW5+xPX+GdNNj+HwgbJwNE7qgNUdKupdKMjaYk8z0+VkbDd3pqKytrnE99cJBJG0mYHu+trKNbmve8kXY059ighx9QHtARp16W6uFkNV94JbRA7SFBJ0ETp4WkXXIStYIWtu9kMrSEpKgDB5kRr93xs1qJhNPca2W0DtXnVBuE67DXXmN/f7rPx9nQhxICNRJ67WRrnu5FXU+tQAxTQ2ygwTP8x+vutBx1UmjS6KXZhxu76NKA2jtVELcUE/xfpt9A2SruupCcS1Sg2O8FdwAwNtNrPl1oA5yPa12iI5/dZBYZyYgqFOmMyVHU6RlG3z+hZnECyYAlFL1oxSUygGQlZ0SFcvHwAHhZn1FxNO9pJ75GckmMx1/P52ed4uKqHVvONgpBypn3dfL6ixZm7nvVamrcWpORs5dfaP5D8bBe7OUVrco1SRcGFqZilFYnIHX5JI1AHQTrv1sweJNCG3nBBnKnp7xA15fPcWmS7EpN1sQSdFDeI7367WjDis2EuOOHRIYBJka6ECefIyLWQBksgm90UwTd6LywW6pZzJC3UmEgkjU/p7udnHdNzJorlaZSyM65yp0za9Py572I8JHUKw3Vh5sLCKpUpBB0ypIT56/MWflxU6a19Va7lUG4CE8kk+7p48vGwyAVMEgXSJd+CmqZpx1xtJfdguKjYcgPj9QLGDh7s1gpSUncHUd4Tr4fL8nmtpJMpzZZ0MRHQ/hbgsgHVHKYgyn3i0wxvZQzu7qPcRXZeCbvQpmvq24VmBQ+pMjXXQgDexq5nMVM3dTdjiW90KQ3lzCudSvQkTIV5/W7lvylDl3ryo0RCpP5e+Pf52BumnSbuSkkwkqyjnvMz8frSwuVGXahTEr7aEokzf3EFsns8bX+DuALwe0I6SqOnysYTj3ipTDu42vZakyAXKgrJ66K/rpY+mjSgSUKGUc4M9B91k28AkFNPTpKnFkiAnl08hp8LRfTwn3R6KYqJhs4+qBreNHGthfq13Y5qS7mjv0zC9CN5U2dPvjkLKGKeLvpK4P4YP8RV3609QtVjVGH6m6mQh1xUkpTCBoANSTvpqZhe4dYOwtVXotzF19NUNMyjtVZh331xORJAIA6k8vkL6U+KKHE3DasuCiq6V+nZZSthmkUQxTpbEJTl0TmjTbS1GSlh5oY2MeeitMqpgwuLz6qLrm9NPjY9Q0lRUG4HS8YWpd3qH8UT3XB93I6Wmu5uP3EOrpaaoqqC5lLebS6QllxI1Hisx0tRu5V9ldNIslIWy8ojSSnveHn91rc4UaaeuWhVCSOyABjppz+vxFPQU+bRgVqCuntq4qQk8d8WZYduC7HIEnKtxEjnuTZKvz0pL8uCoZbXw/YqUOoKs6bxU3lI0iC2fz12sSbo2yjMmZUIzDU/H3j5e5r40udp31Z0oH8SQoRqTlPx3tWOG05P3VZ8fPa4cnQx6Yy4/1nhnUJO5Dd7JVoTvqyPqeljSPTIw7/8AXcCX80qJ7rzKxsOpGvhaIHbnZKCQyBlAMeGwEe8dN7AKuJhSS2ltPdGpJ1B5e/UfAe5fZVOen1UftOoHVTcj0w8BKSO2w1iZrLM/YMHr0esba9LzhQtaE1SL6piQZK6AmP8AhUZHS1e3sO0wCXAyArNAGWAI6Ac4sVew5TLBSRrsYJB8/vNm+yYPNP8Aak47KziPSq4MFMrxPVs6AJDl11I8dwg/A9Ldn0oOBzsdpxBpWCTH21M+2RoN8yBaqFRhencHWYT06xJM+P0bNHG2Eg1hqurWlGUNpcEpTEJUJjTeN4jX32k3BYCdymOMTt6BXppfSE4L1GVTXEy4UhegzViUa7n2vMWXaXjDwwq0pTT8QsOLkaRebEn/AJpB/K3lGEg3b7ZBafIAgScw+P8AD5Wku5sPMVVAxVZh9oylZSoSACJM69BPOzTYHEwXDinjxqQ7tC9KGMZ4XrUxS4kup6RKcla0qekd6xxN7UTgJp6xt0aiULBnfkDbzeRg9p1tY7NuFK3UNZjl8rFqjCDaUEMpCNQDoRrPh4f152q/Y7fxfRH+2T+FelqKhS1d1J/m2NuV1BSE5ioHkZiPr63t5nuXXfNOgooLxq6dwAlPZ1K0kHYCQRH9LZdNZxTcqEM3TjXEjOyQUXnUIAHnm05+FmGC3H3/AKKQxgbZPqvS4ViidVTqBBM/1sIKs5j3zoNzbznqsY8WsPIqUq4o4peebpi73r3dUEHKCAApWuubWOVvQv8AZ3XhT8WeCV4V3EBIvy97txA/Rqq6wZnuzNOw4hJOhgKWuPfajX0JoYede4urVPiLZn5Mtij5qEwUp0OsgdOmti5qiFHKdxsTv5bWss5wqwC6sFWHGgRp3XXBpHgqxKp4LcPHUn/sVxGsyiqdH/VbD8aw7hXuYCq/U1WlR9rSPM/W1lGmvBKVxmmQR3Ty52l1XAjA+cqbXejRIiE1UwPDMk2S724DXOKCoVdOIr4YqA2sslamVoC4MSOzBI20BGnjrYT62MorHC6jV683C6mnZK3HVaIQg5iT4R/TX4H6G4nqpSKm9FglGqaZJ+zH+b+Y+ER52Gum62LraCAkrd0Lji9VrPOSPHlZXZlXskkA9d7FAL9kYkN3RylaSAlISBAGg5eHh+ll27GSVhMjTWI2sl0jXf118J+tP0s4rsaOYCPr8OVtehpLuuVm1U9gnJdyQ20kZrPK63SqkE8o+YBsz6YykQIB3s5bidzNKTGwAHun8xbsKZuUWXLzuuSUqGB8banlOm9sVrvbU8totcVRbm2TG/W3JM79LamCRFkkuiRFtkwIm3OYDf3W0SOR2sRouoOKDqgVU6wD/DP425bV2gSQfaiwiwFAid50sVo1zTo1hSZBM7EGzSNuNEmmyUq+oKECjbVlSkArG0+FlC4FdvdlVTkAlBzDnuP0sgLJJJJ1O5ssYUdiqeZ3K2iY6kEfgbZbaFlPSmLfqfPuiGQl11H/ABGuqoqVJqQhD9Oy1nFKoEIUvcqXyUAIhJEAyTJCSmMLzvXE17KqLwF6OpZpHwXVKJyoQQo5UnRIVmLZyiSRrGk2mrHN2P19E7TuVK6OnAUlRa0dOh0kapBMajXe0d3Fd923vclbhp1TdGhNZTpT2RgqK1wkRtEiNAJ6zJt5RiBMMksnK9lrm7jW19bBdTG/NE1ybrd64qcbS41K0KSFJV273eB2P92fvtlnK9cvFJTq1UVRTsU5USy03WpQltE91ISdUgCBB2tloePpfwt/n5KGd3f6qg37St5SrywIwkDMhq8iRzGY0wH14WiDhEjJT3NIgF0RA0ICwPrzNpI/aMXkm8b5wC4Fgh27aqoyk6KzqakeBgb/AJ2j/gokppblMBxCRKCRpEg6D3n3knxt6RwuLYdDfz/Mqjj5vWS/z0U7NAJGxSUgHfVOsfRsoISgwd9ipMTpzsDTNJjUiAJnb8bGmW0pUHExI1Pd1j6m3aFcsxddmcsSTAMA7T52wN5TIGbl08/wsJCgNFCNoI9/vttsKKJnRO5J18fuP1uynsg0IQCNzJjxGmn9LCJaSRIRJVoPr4foLdZe8ATtqNtD9A26CCEkAgaQNOfSfrazdU9lwpCc8BSjCpI5x12t0EqcCVFUZgRIGw01+NhA0SSlG+sdff8AC3SWiFZgSI1mNt552k3dQPkg1N54KhBnToPOIJ5b/OxdwEAFZ7q99J62OlsQQcp0B00A19/hYu+2CCUnSRz1Pj4n60sQFCdooM41ICb/AKFRy/8AmhjnBzq193XwtL2E0AXBdZJSCaRkEpGuiE8/j9bw9xyeUxiCgUonWkMazHfVO46fU62mbBqe1wzdStpoGSRrvkGn3b62g03cU7vuBLSQZKROWZAj4a/X42MBJI3WDI5azvrP1zsEhpSITm217p5yPzsZQjuyRmKRoDpE8vjH5WndIN7LBmTmkjWNfztsypMkH4cuk8v6+YFyiFKBSrcpgfKOv1pbZaK0yjRJEz9dPqLQzKeRcFtt2cqBr743EWTL/Qk3O4Z1BTBPPUH8N7KxCj9mEazESfLymCbJ+Ims91FRQUwtChBAHtb/AHWg8+zdSFr2UXultqqvdxyEpQhEyeZRt18LFVMt0F1XY4opSFvh5Z1Md0nlvGmtur1RUPVtfRycqyH3CEzCEIkJ+MWHvNll+iumhCsqn+zBEEnLlE/EGPfbPBurZbZJVGhx68lVdRmPrTLjuUT3UH2R5QLF7roUIo3bxLclbS0M6eygak+B5/lzWrzdbo71cU4pAS1RBIA3BJgRz1jaxSlQhGF1JyBv7BRJPLfr9a++0mnVROyr/jRo0z9MICiorKgBJIJBHjuLSbdqUf2XuyocAKWmmiMx1kJ2+77/ACjLHtLUs3ozUvFXZPLSGEmf7sJBzHT+Ik2fWG11N4YfZIzJp6GnbbMGJc0HLp+HjZ3nTRRapHw/Qqp6UvvqJqKn7RRIIHWIPPUfK2rnQRfN4NyTOUmen9bKF2KU5RU70K7zaTBHhMffZPNQi7b5rHHJSOxCtdJPd08LQc+wCkASV3ezrtQ+3dlMIWs/aqSnQJnn424w2haWqloz3XZAjfQ2OXHRqUhd4VLf21SToeQ30sFh9RD9Y2kEALBM+Zg6/WnnaGY3BRLWFkfLRX3VEAEDSCY1H1H52bt7dhT1TqGyVOORnMRlSANPiJ+HKzhr6sUlOXCCXFaATt9dbNR1ZW6ruy457pPS0JZOieNnVBNUjtZUJaSmDJBIToI300svVlI3S3YtsEQlsjzHx+vfYe6bt9Up860Dt1gKPIjw+63V6FBonlglMIOTTfpp9c/c7BYXKZ2pSPdDZ/dCErTJBVEnx6fEe+0ccVWAtpRUnPnpyRPMyeXPQ/O0lXUB+7ZWskArBnffb52i7i1WlqtoKJnvKqMyVEiO4NTPgAdfo2sh/soJbcpP4QpeqW66lblDLbodWQBGqIAG+py/AWlDCCj2T/cJVKQZMEyDty5DbqNtbRtwReCn74pg7qUMr30nviNtNSbSXhQhVVUJOYCAQSQNjEfd8bQGjgnIuE40tkKKlJJ5jXfbr+dtqSHDoNYAPOw6UaQCADAiBCvd7/lbSwUyoIBJka7jrvYhdZDsku8GSqmfBEynMIjSLErga+xcAKR3yozoYIEfd8DZbqGwphYjVQKROsHbprtZLom1XZTFTqVBxw5gke0noNOf14AD32KMxuiyrdDSSloFSlmEhJk+Ea6zr5W5YoENEOvLC3l6a6AbyB8/jY1S0SwTV1KT2xTqDplBj6+jYdaEqOYpCiDOh31H6+evvk031KZwI0CIvNndMaRqQBpy+jZuYzo/WcM3rTgmF0rgEbewYHXnZ2KRDacxkJ2jQ9ennZIvdn1iiqGQ2SlbaknbmNTHv62MHXQ1VG4EhdzLKRq3VFXszAKB+W3O1uOGzyKrCt3PwkpLQICjOUkT9x338rVFuTMGbwpnQZbcSY5wMwBk8/dpPutajgrVJqsFUYBSezASIJ6aAz015WoT6FX6c3UiISAkBQBBBJM6kSd45bf1sjYupwaJtaYKkvSrLtMHb6G1lpJOkExMBMxz56e+yVihhxd1rI17NaVQZknWfx087VwdbqydrJmLaBXnUAUiCsgg8vlysGppJVKEhKssxM9Nj01saJVBWI16KHw+vxsCQogSoFS9QTIOniPP5++07oWUoq82Vd1yIEAnYHed/CfqbF3KcBIjvZlEzHONZ16fdY6schCkpEgzPyPPT6mwLgMQQDOpJlR+/rOvXnytIOTEEJKdagqAhMTGUTJ8N/u+6yHiGhVVYfvSmSYL1G8EgCAe6qBPnH0dXQ6DokpIkQYGhPgB10sQU02teRSRlcOWI3BEH8fzixWPQni6q4gBV31bYPsrbcHQCSJn/eG/42l7AZ9Yw3QPKQUqS2Gx5BWWR8J99onpmy0i8KQgJ+yO55pUD08DaTuF7ql3C22ogFl9xvKVwY0MRy1V87EmPsocY1T0bpgpuFNpmIIBMgbctp9+gtlTSpI7sAyFDN0/Lz/Gx6iYXUKCENyRuJkc9LOWguANALq0SsEKII1mN9PK1EyWVpseZNa78MVFe8FVKChvSFEanQHX+sb7WXkXUxRM9kw0hO5Ec5O+nj797OAMpSjKjSI9ncfUe6xWoYKQVAApiSoDUx5jy+Fo57lFDMuyi3HFGE1ZJSSH6eCehgp8p0Frd/slr9LlxcRMNuOd+nqrtvFCSrXvoebWY5f3bfxFqrY5p8rdM6lOi0uoBiYPdOnnP1raav2V18qu/jhivDTi8rd53C+UIkwpbFS0R0BOVa9d7UsXj5uHyD5olM7JUNXqUpWYJWOduVjcxt8rcoJyZCJKTbs7AnpbznNcLoLWReSTHhbhzvJIPPrbFd1R8TblZPXXnaq9yOFEGJ7tVSXzUNAEAq7RJHNKtfhuLE2GiVcwQNos9ccXeVLpq4J/maV96fxs10MBKgQfaIVr1t0uGfrYmlKd9kco0bKUCLOC7UJSJy2SKJoEiUyNNR052XqNASkDbQGLdZSRWWDVS3SswI0ny1suXI4UuFECCD8Tr+Bshs7iRO1lO7XCiqSSI1HzMfcbbLBZZTzcWTizcvnFtA6Ttbnbe2TPIb9LWLIC7MA20TpztyVco99tZtJBHTa0wLqLjot5jPSLbnSADrbgnXT5G252+osQC2yGt5o152KMnKt5tWhCyRpyOtjZkD5WKkFNUvopAVpZ3N0UQdVtUjXWLH7ieDN7MKn2iUfERYgraAfK22HSy827qShYUB1gzYMrbtKkN7JSxaz2iapgEZiAoDzH52rXfl2PVWITTrdUy3OZxadVJQN8oHtK5JHMxa0OKWmGgu9HXEtternOomAEjWT8bVzvymrKu+Hq+tovV6XPNNTuIlbyYELeSoQBMkN8/wCKIKFcDiDzFVFnVw0/JdNQuD6e3ZA9piMaUWBqddP/ALFS6FTqlI/hJXPfMR3ue9ssA6h9x1bi66oUpSiSSskk2y1H7Pd2HqUbK5UH/aLA0mNMI0DS1lpi6n+6E6gGoUE6jkMo+VmVwuvxi6buuR5dA86GaZsqyx3jJ5mN4+VnV+0feQrith1lUyi4wZBjeqeG8eB+tCyMCtoRcF3ZlJWv1eCoDeFKGgHl4e629g8pgwuFw3t+9AxWMS4hKCprp+Jt2BGZd1XiiO97CD5x3rKDfEq4VpTNHeAjWAyDrv8AzeNo0ZIVqnTXKDp9dPhY+3ruTlKQok6+fu39xto/a9R5KgMNhOwUkp4i4bURmTWJ/hk052110+trCjHuGc2cVj6Fp5llcTO3j+to1USmRm5SRt7j9dbdNOfwlepMxEj9NLIYxP2CRw2Id1JgxxhdaQBeZCjtmbcTpz5efWw7eMMMiFG9kJMwZQpOnvHl+doyQQEZkCFDbUiT+e2lsGUqMhJI11iCOnvG1pjGJuwTfZsYG5UrpxThtev77pkQdMyo+P1zmw6sSYeWQkX5QGf4e2GvunQbWiMZUiMkgAjTpy0G9uFpQVZSgJnQaHTWdx5/KxBjUg3aEM4azoVMCr7uYgpF7UJMSFesJjQ8pNgXL0u5SShNfSrkf98kmOZ++0OqCVArJTpuQkeMnTxHWydUNoQM2VMDTVIkDYe/9LHbjLurUJ+GNPvIl6Ql4UrOJLtcbqAoCkWCUd5JOc6GN/0nladcB1DKsJ3O6h5uVUFPoCDoUJ3tTziZUN0t60sJyksEwBBOpn5c/wCtpRwShr9wXapLWX/VWyfHMkbnrv5xYr8UMTRJl3QW0Ak9gHZWYbamCkkjeM31P62MpaWRGYmIJ3133tBbSyJKHFpIJEhagN9R9a2OsuViFQK6rBA0yvLEbfgRYf2407s+qM3Cj+JTalKwAoI1BkknfW3CkBYSlAglXeOsjx/C0PNVl6BQIveviNIqFiDvzPnYy3fN9pUSL4r4k+0+dDM6TzshjTOrSkcMePeUuhtwgBRJ5nx2NiGIWSLtckRlUgwdJ1tHbWIb/wBxfVbM6y5IHxH18bdO37fTzBbfvd9aSQIWEiDO8RvNk7GYiLZSkMLkve4RN9gOVF+VC0ytDaUAgagBEx46/d8U6mbXVXhQVyyQ224lhpI6JQcxjlr9dVAuValPH1whdQcrndHejTX424R6ygMIbqQU03eQChMD6nc/C1Xx7Sjmjd3Ra+KNNZfLqT3kU7HauCddAYEDkTr7rEbwcFFhIMuzL1OpKTmGqtT+vusppNR2z1T2qC7UJKHCUCFJ293KyTelItbCGFqT2bSS2jueA8enO021zboZo3AWUO8VGeyeupSSQcohQk6ZBt1OnOBt5WfGE6VpWDWA2hILrJKo3zSZ89tPK0f8YKqpbvOkaecZ7JDSShITBBGk76iJ+Nnnw9rKp7DVIhCmw0UrQAUmR3j+mnjaw+qaGhyC2nObKpDw04hV0UjhgwkT3toJ1+XzsDWpRVXy1Qg5QofaKJjNl5H652Bul6qoKRNIylooaGxmYOw315/EWEpaNVPVtVqVBS0KKzmJAVJ5fOwXVjDZGbSuGqcJROVKR3tk8tIgCOnvsjXQtDV716VqgDveIEz79Dv99jZvCohOalb7wEkLj8LEFtqU5UPttlK6gDZydPLxMWi+sYdkm0zuqJ3rXKqXO1zANjRI03Eaj65eFgrtS2LypyUzIJI5H2o+YsMu7HXHAtSBlnvAHWPDobGHGwK5mrZYyoaTlygAk+8mwW1DXOuVPkkCwS4hIzCAUjvaCDP5WI3mpz1KoCdwhRITryP62EcvRvKUBl4xrpBnw0+tfiXermVoW0WHRmKhIT89D0tbNSwjdA5DrpDYrhSXOZAS4VlKAf4vHrEizIxpdLjF3mvqkl19xDkSJyAgaD6+6z5bbpxUioqm1qZbTLKAmQI5n65izW4lX1d7F2JU52gIcyatKIIynT4A/rtacE7CbEqEkLugTI4GvTf15N51QumSSOUhY938Q58haVcLkC96phKlQAZCht3hPnpFoU4OVrVHjioS4YSth1GiNScyVeXI7czptaaLmdaYvdyoM9msLIUAQBJkaanlv4WI+RrSLqLYiQnoApMDczJRrBH9Lc5iAE5iRrqTED3fUdLAIr6RULS5pJBGQiPA/EfRtya2jQkQ6CUiNAdfr65WRnb3URA7shXnA0FEuQSconUid9LI92u+s3itT5JUhJIEbagE/XWxpo09U+p+qqUpS0YQkaGY33g8rEruCGb4zuIKWiVJzrVoY10sAyglGEZAUgYWwdVX8hdZUNONULZgrQgntV6d1JB+J/Sz2bwZStpFTU0wzmC2wqmUAU9djA+uWhu56/DrNFTVC6ygDbaQhhgLb7ytu9CwY895s18Z48Yq0u3dclSgZ9HqgHKT1bbgnu6b+4RalzpJpMoNgrAjaxtym/jB65qaoXd91NU7oBh2oQ3CQZ2RpOnX6LIq2g60ppxKiI2Jjnv5fXkquhDqMpyJIOhEQOv142IvtJghcLOoABj687a8RawWus94JN7Ko4pxRYgv6hWkyhxSUkJ9mHI106H8rWH4B1KncLqT2mYtPGBMzJVp5/r52gfFzDVDxHv1KiAFrdV4mQlW2/I+cC0v+jlVhygrqYK1bc1EzAATtppqoj3WBPrqFZp9DZTkPb7pkjvHTkR98WJ34larpfhR7qQoGdAAQffrYdpQRsoQBA5AjmANuVuLyQtV3Pt5Mw7NSQE89DqPvtVv3VshMSFBtRIIKRvr0jn79OkW0qZ705USBpz11+j0tgzKAznu9Nteupnl8racVACgBlMgmNjMH3/rZZioaoNxKiBvEnSeWm3uPLxsXcIMhKFKTGnjOvv/AKWGcMgKVExlEgkQNffzsGsAZitU+PsgnXr5fXJ72TEXRN1IKM2YyT/FrED8h95sTWSklWbSTISYjy+XTfSx9aCEFGUzoO7ofdpr4WTnIKipslBERrrI3HL4eXSLEY5QLFXe86EUuL72ooCft6htOYnmVQdvI/pZ6cGqB6rpK6lQpQ7N1KyZOmZMR/yixHGuHXWMbXnXKhLaXG6hCSI7QKQkq+8z87LnBB0MXtelEVJCnGkOAHXRJIn/AJvPaxJpQWGyhEz2wCpkuq72aRH2CBnOpUNNtdrKaciEQogDfNAM/h0sWYCEnRMgaCZ94Gv18rGm8yVAn2gJIBgAcojbX8bZZeStJoy7LoA5QCkSCACBrt+litW0hSCSsSZCtTl2P4zz/Gx7O3lLavZGxKt9vfYB9eeW80wI05fWnnabHJnC4Ue47aPqCXUZhld73gVJVqNvusv+ghe4w16YGHEFeVu8vWaNfKQ7SOZZ8O0SkAdRsLJ+L2u0uh9KZBACgM0GUkCdPA8gbNLg9fIwx6RnDq+1OpaS3f12KcXtCPWUpWPPKqDFiTN5tNIzuCq49mRp817hOAIeOghQt1ByyDbKsQEqT/DpbQVmRmHW3lpNiQukGrQUWcIzaT8bcLOnibdvHvZgfKwLi0pRmUrKkCSTytUcdbKw1Jt/0ZrbrqEJEqbAcT5p1+6bMdtsExAjeI091ntcGKLhxLWXhQ3PXJq/UcgeUhJLfezaJVsr2TMSLNiqojRVjtHH92opE/yjb5W6jACS0sKDVktFitUjIASnLA++yxSgCNj+FiVO2SJ5+MjxsfpxBk/r9flbuKdq5yoOqPtRoAdrHKZWR0EcgSPhYi2CdTrHTSbHGlAOIHQidLaABVIpzzIm2A67+NgadZNO2SZOUA+JGn4W7zaTaw0XVcld5ttLcz5fC2p0k2yRJ1MWIBZRW7bnSBbQPO2Sdd9LEaFBxXW5Bk62L1Bh9k8lZkn4afjYfNIixavOVtLmsIcSSR46fjZ3bKI3Xa5jUA24UTyt2vaIsEqP0sIi4U721TvfDdZc9DULQFFCRqRMEDKT99o0xBdbD18oQ7SsPqdUlCA84UgEneB7XL7vKRbkdD1wFopJ7JwpgHqQfxsx8e0xRK2wdNCQLcNxFAHOFjY7XW/hDs12FcKxfhCmUaZNy0DiWjkCwiAoDSfYtlo/RT1oSAKUqEaGVa/O2W5n7Gb/AGx+iv8Ahmd/zXmR+0WqUvcZbrYJ0buBkBJ2/wDOKgz8PLezVwVet1U9w3ewqvpUKTTIzhTyQROskTp9b62cH7QGrnjq03nUjJcNOhWU65VOPEj4HW1cnVrbq3ilQKUJTKtSYCeR9x5R1t2eGQCXDYW7eyFVxF+Suld5qzNPeV0rCSu9KQE6x6wnXl+PLnZRp7yu5QluvYUnaA4kx8DofzNqrJWUkN5h0KSecyecDb7rCB9xU/aFJTsmNx59eWnQWMaHzVcVduital1gEAKRKhEZgf6D8tZsO062VQCnSCddPl57Rapvr74dzMuqVk0ASSBvtsJsML0rfaRW1TWb+V5QExHxJ2/S0fAHunNXfWytkgbZkZgNp1+IiOtuie+kkaE7zzkCffpt1+NUf39fTKMiL1rJ3QUvrkHXSZ+f0R0Yov5ttXZ3xeATGaPWVgSD0B6/j52QonDql4sHorUZ0nvKHvHQ+HvsEvUDYqXMwOeh2Hu/S1X04zxS06laMQXkmBKQKtRk/Hz94iNdRTj3FjaYTiO8XANCS+uRpvJ5SAfoyvBv7pvFN7KyTjmh00Enu6Tt8tLEapRU2UtpJOwTG2vLoPG0Af2+xiFpP9oawiEjRY90A76iPebY5xFxehZKr+qFEaSpKSZ57pmNLTbSuCiahpSrxaUf3tRmZUGT3VSRIXz8fP8AO0t4CIVhy7DClTStSDoQconSPrytXK+7+rr8W07elV29Q2MhUUgACZiAI8dfus5br4lYnuigYpaGpR2LLYS2lTSVZUAdSNhp1PKxZ4HOja0IcUgY4lWUpykkgyCNyNY/DYRy91j7MAFGXunVIzTOnj52rexxpxo3mP7wpF5UiCaZI1I0Om452FRx2xszLy1UBCZHfYMeXtT8OtqnhJFabUsCshAKezyydZMA7ff9dLCqEEqTB0OikxljWZ6aH42rojj3jJsJDzF1GRqVMLGnMDv6jqPd4WGR6Q2LGxldu+6VkACMrmomJ9vx+/xiPhJVLxLCrDIUEKSEpJJ5FIEjp4fW1hi4AAmMwPhHvtX5v0icRuAOG57sUT7QAWJP/F0+uo//AJRF7ZJVh2kUYmEPL138DPP6Nh+GlvsnFSzup4WtKAEFR3ywR9HqbclwKQkkqkd4mdPr9ffBp9I2u07TDFOoSIKapXwEo/H32FT6RBJUtzDDYykJEVZ1O8ez4ffNpeGlHRJ08Z6qai6VAEAqA72nT8dTYhVLJBCUzJiSTz3325WiVv0i6fID/ZdYM/8ApckHYfwTuY+NunOPl0vw2/h6rTnEghxKoII8Bv7rSEEo6KHOZ3SJxqfUzeFI8sZgWwTB1kFW3j89NrOThW/nwnRE5xqsTt/GrztGfETG9Hi1bNTR0biC0kIUXFbjWdtOX3+NlrAXEG77kuhFDWtvKW2pfsIBEZp3Jnpva1IxxhA6oDHt5pKnaidShBbAmDMnfy0seCh7cHygAc5Hn5dfG0Y03FnDaTkfNUCd/s5j4HmRHXXyspMcW8MLHfq3kkzKVMmRp7/v/SkWO7K1mb3T+K05iEIUJJB8T1+XztsPyJTpEiRpr0swkcWMJOpTmr3EhUiS0rTkY00sIeKeE3IKbyI29ptXTQ7fA+Vo5Hdk2dvdPhTydMyklI0IJ2tx2iYJSRlmTEEnp4WZjfE/CiiJvVO+pU2sAeE/H77CHiLhWE571ZRAmSVAnX59ffZZXdk92907w4gk5sqR1mQoz/W3C1ycpknYwdvr8+Wlmp/pCwqpKk/vlgZTqCCmY+uX620vH+E1R/23TCI/iPMc/OPDlZwHJZmpw1CwUTJAiIG+88rMTiOjtrgXCJyOoVpzGunxI87K7uN8LrKlC+qMmdi6BrppHLTf+sNjGWIbpq7nqmmLxp3VoIOVLiZGsbT52LFmDkJ9iFHPDR9LOPGQEOAOpdaBGwOQkjbwtYehcblCT3ecq2Nqz4HqSxjyhedqEhsuqhS1QJyEfDePH32sTd9a0ogBxvRQgBYIjx11taqiQ4W7INPbKnOhzTOtKtIIBOp06+U6+It0pMg5+6TEERO/9flYm0+CjuqSsK3JVmMTy6/rYQrz9wpGYyQkkyCdPy+FgZij2CGzAGBJlOn5R9eNgyEJB7iVZdZgEbb+FuA5niRIBAA6bEjw5W5WVQO7JiSJmR4fn9CGcgpsoJQylMzPZpE6TEDT6/rYFaG1HVGonTnvt9R87cB5JlUhI1012/pPwtylwCSQJHUfUb/WlpNcRqE5AIXDjdMCr7M7SrnppyNib7TCZGYTJT3FaHz/AD8bG3VLnOozEd0kxof0n387EKt05NFDveeum0fXu0sVsjkIsCgXifTU1DjOuU20pDi0Nr7pI1UgDUfHb9bLHBOtqMl5NsPPtFK2yFNuFJ7wVof+CfcLFOMSezxCwtY7rtM3InWUrWNPcRYhwUqezvu8qVSijOwhZAB3SqDP/EbXt4bqq0WkU9UdfeWZJF6VeQpzHM4rw19/jY+m8LxdQWzebqkq9qVTO2kWRaRQgEmClPKYB3Pw0+HObKKFFcELTnzamD8fuHkLUi9XLLpunSk9mXFFSTBn+U/cNY99ufUUtyg90xEz00HlE2FaBgkgLyTqfHqbdZgCEpGfXmd9vxHys3MSyIqqhU4mEvnfXwHhp9RYJVEto6OqCRqM2se7x1soKKgEBQCgNzuR46W5WBmIJgbTGbWOnxshInyBJZoFEABUK0k5Z/Tr9CwlFdCQuVSogkwD+B8bHFqSmUqBkg5tSAff7/ut3TFSc+VQzRp00mdudoulITtjuoo4tXeWL4D6UwX6EEEcoKkQPgOX6NThRV9ljMgEhNUw8k+IBSqRI20jTpGlpF4uMoWbrqVIzJIdZJGxAUk/9Ri0T4EqHKbGN2PrXEryDvbhQI6cyZ59fGx4iXRlBe3LIFYptzLBSokA7JBgDWNY+oNjrRkCEGBtM/U2SmHjl0EJPxmOgGvlZTbcBUQsJGYzodTp5/UWp3srlkOlYRJSJnVQIncdfebFnXQE+ztAMK5eUeH1Fu3HkFYOiTt7Ux+tiryj2ZzEaHvGd/IgacrLPZRLQkPEK+3oatkg9+nWYiTsY+f10hO+7xco7yue8qeoJXTOFaVSNCCkpjw/MWnWuZQ4exWJSruGNwDpt7/rW0AYpZU1RU638w9XeCVEzEwQSTOo/FRAtdgdn9kqrM3Lqvfa6bzYvy4aG96ZQLV4UrVU2oHQpcQFA/A2Fp1ygojY2h3gLxRwvSei7w/xviu/qS7qJGHaKmefqHQAXWWw0tI5qVmbVoJOlo0xn6aGE6x9+7sK3i/RUmoNUunV2rgmJTyQN+qv8pt5lJSzOmc1jb2JC6OnAkaATb4qwuMcfXFhVBRUOmqrYJTSMEFw+KuSR4n3TaDca8TL0xJmarqkNUavYomSez8M5/2nvhO3d52hiv44YXqFKDV5KlxRUouBRUtU6lROpJgySZsg1fFi4qkdp+8RBIzTOg8J5/rYseGy3uWrZiNLAPvAlWc9HTESqvG153Wp3Mmpu/toP8zbiRp7lm0u4ro+zvBupQAEvo1gfxJ0n4RapXo08RbmqeMlyXfT1yVOXiiopY1k/YqUB8Uj5Wujiaj9Yu0uhMqYUFz4HQ22cOYaeYBwWTiL2yOJabpsU6IAiNd9dLGkJgbExvHWwDKTGpExJ87GkAmN9Oc+Nu1g0XLzalDNHvDQfCxlBERrNioQQYIkETYdER1AFr4VNxsnBQuFxjTcK1066/jYwCQNd7ELrWCypO/0R90WM1FSxSMrqKp5DLSAVLWs5UpHOSdLWWfdVc7oeQdT91smTHxixK774uq9Ar93XgxU5dw04CQPKwr1dSUykoqKtlpS/ZC3Aknym0gb6pjcaIxMbW2FEbm3EyMwOlsQ625JbUlcHXKqYsQKBuUICJHy87AXgjNRPZRPdJHu1sMDB/AW2oBSCggd7TWzkX0TBF2lJcYQtOykg2DUQRsJsHQE+ppQdVIJSfMG3atNjsdbDKcG6c2DXe0TWUyjopKVD5g/hYnfdIyto1FWkKRTjMpCh7Z5D65TbnCL3Y3uG1HR5tSdeu/4WO3+3Vllxlqkbc1UVAu5Y6cukW5PiZoEBcQfl/stHD3ESAXUdu4hr+1XkrK1KcxgJfCUgTyGXQeFssKqgvLMYeSgT7IfMJ8B3bZbzTm0vd31XV8mDsvHb076gP8ApA1CFbM3XRtankUlUR/vWr86jLWvBzIkyJ05zqepO/XytOvpwOhfpCXxBSS3R0KII2/1ZJO3n99oPTLrrqVAqzGSEAakE9Nvwm3qeE+zQQ/3QsHEzesk+JWNrUlRkagEaEb8jHLx6/O2KA3UpckTHLx/H52F7JKlIJ1IMjUaxyHPofK2ygSlCkSVEHMrQr5A2ukhVbLgOrBjKgnKTBJESfrxJ91uVFQJUCAFzMGQYg+I0sK40EHtSoJzbSQJ+oFusp7XMfaUSD5RrPw+EWYEJiEWStSQCQoESSFHLJ92pthQCISlQB0BMyZHI7TzscFMVQWkjSdCmB1666fceltJYIWTGcmBB5GdI6H8QbLME1kVzElOTKnNsCNTzI+EfAW3JKezTmIAOh+EfXON7GmmVqCsyypClBOTLMmDv18foW5QxOUKSqSAFAgynTSAfIeOllmCVkUU531FIJXOuupM6H79/HrbRcJOdCUtheuUaZveNQNIsZDBQMrsHbuqTMGIGnOP11twpptBCw4kZQe7uTG0fPoLSDhsolpRR7u94qT3VFMhXskbAdRpv77HG1fYtg65E5T3tPGY30nnz8YsJc9wXnie+Lvw9cFCKq8rwqU0lLTpUBneWQEpkkATMSSBp4Wmdr0M/SIypz8Pg2tIGabzpNTAH/exP5WHPUQwgCRwHxRIYJJNWAlQmXShMNgyROaI18T+cbWCXmWuS2THIfPy8vG08NehX6RTzSnFYFYykwFC9aPQnlAd05/QtivQi9I99QBwMylJglar2oxI5H+86crVxiFL+MeqP4Oo/AfRQVnSFBZypUnTTUE+MeGkbeWs8lQ5DMlQJTqZOux+o08LT9T+gf6S1ShSf7F0aVxIBvSkGoOmy9vw6WEV6CPpLrQlP9iacaDT96Uxkxz+08NtR87P9o0g05g9UvB1P4D6KAmaw0y1KU224E6QtJEgxt8v6WDcqFOKLuiQBO2kc48Plv1tOFV6FfpFUL6mXcEdotRnKiqaUEncR3o35gnWwafQ09It5SGG8BOdoo+0urZCBvBJKoH15WcVtKdQ8eqj4WcHVpUKh0lRCsoBGpSZ+Hl566+NtKqEg/4ZhZGuhI66Hby87TVVeh16RDLqmXsAvkpUCSiqYWmYGxC+kTqbF1+iH6QzRj/R3WRto+zGx2IX48vCzirpvxj1SNPMPdKhtl0AFGaTrKvOYEctBv4DwtiKlIUlZQSFAhZVppJ6+Gmnj4WlweiP6QbaM6uGtXJnMC+wB7W0BVg3/RN4/pClDhvWjNliHGemv8ev9NbTFVTk/fHqocib8JUQOuLSgCTEkwBGYdZ5aa9PxHonXA0paFkpOxBgyDMHx1FpUd9E30gno7HhpXnQbOtDYH/F9a+4aj9Ef0jMx7fhpeMJOmV5oaHqM56C0jV0+X749Uwpps33T6KLhUBIzhRiMq06E6Dw1Gxj87bTVrS2QC4nL3ICiY8untDz0tLf/km+kCpJT/oxvHRMf3rQMgRGivw/UNXol8fSVD/RreRKkyklTQA9wV0nbrYAq6c7vHqieHm/CVE4qFElTqwCSMpKTsPCdLdIrFtNnvEKKdRO45e+NvM+FpSd9E3j+tAC+G145yNs7Yj/AJuXv0J3to+if6QvLhleKp3UVtDwjRUmRG3z1tLxNMffHqo8ib8J9FFhrxCJSoKCcoTrMe/6mfC3Sa9TbYUAEqJ1Os6bbyOn9bSoj0S/SHGZJ4cXhqTp2rIgGNPb8Po22j0R/SEKTPDitGkQp1hXMzrn35a/nZeIpvxj1T8iY+6VExqnColQEnXx1569ddLcqrF5QVFJzd5WXUn6923utMFL6IHpFVSi03w3q0HLKg5VU7YG42K4OkWJveibx/DoaHDi8jMyA63G2x70az18LLxVMNC8eqfw0++U+iin15x1WYye/pPdJHXw35+NtOVyYCc4JRoCNCoc9fd8rSov0TfSFT3Dw2vFRQqAsLQZnQkd6OQ19/mGfRN9IRWUHhpeYC/ZlaIEeStuW462kKuk/GPUKPhp/wAB9FDjNQtL6Tm72bYHnpO/usK5Vr9kKUJGp0jcg+XLytKo9Er0hyMo4aXinKZnM2Ikn/F8bCD0RfSGCSf9G1aZ6LRO3MZpO332Ka2kv+0b6hQFNUDZh9FFKbxq0HtE1K0EGNFlIA18fC3f79vdlYQLxqhl1IDpGoG+++2nhaTV+iR6QgEf6Na9Iywe8jUDT+bfzjW3R9En0hVq7McOLwzAlRHaNifH2tD7Pzs4q6T8bfUJGnqPwn0UcNYwxElv7K/7yGbUgVKwAdgfa6fR3sMnHWLUggYmvXKnQJFY5I16yR7uptITvolekOCVL4b14OnsutydDyzaa+Hwtx/5IvpEZj/+7asiDBLjX4q8bIVNGffb6hN4ep/CfRMJGPsXtPEnEt7582qVVizI5jfzsZZ4k4xQCj+0t4E6ZQahQ6g68xy9/hNnmPRI9IgqyHhxXAaqCu0biBtPe87df+SJ6QqSR/o+qyDIj1hoSTtuv58ufKyNRR/jb6hOIKn8JTQHFDGwGX+0tZ3iYzQc2vl57ePvEVxQxpOV3EVUrKNAoJVvtoUx1nrZ5MeiB6RzoVl4c1AgZjFS1BnnOfXXfppYR/0O/SMpXRTr4d1JKQFAt1LKgTvooLInfn+Foc+j/G31CfkVP4Sorv3FN+32pLl83gp9xsFDai2kEp6d0Cdeon4WDw9iO9cNXiuvu14IecSpKvswsZRBmDpymRaU3vQ99IltxpB4eVLKXlBCVKqGwnUbE5j8vGx5n0JPSLW4Ef2KTKwJ/wBeZSOgGqosTxlIBYvHqEwpKom+Q+iZ9JxoxbSoACqB0JHeCmN/EwrQbWPo48YoSAk0V2KIMf3Lg0nl3vD61suvehX6R6Y/+gSJ5n94U5BHX29ev0bEl+h/6RLRAVgEhOgBFdTjfr3/ABsPnUTvfb6qfJqwfuH0QSOPt/JEOXNdpIJGUlyZjU+1Y4xx+rwuVYdpSEzJD6tDJ3H4+Gx2toeh76RmbshgY5hyTeFNA0HMOdB93U2T8XejJxxwPh+sxfibBC6O6buQlVQ/67TuhCSpKQYQszqRyMTaOajeQ1rhf4qWSpaLuafRLjHH15JLT1wU6lA65amB5+yfryEjDjrTrRK8NHltVDUdfZ+f32gpDqScoUZCdJJk6/1sMisWCA0U5iIJHj79dvusQ0jELxDlOjPHK7FhAcuWoTmTrlfCo59OQ/ppZSuzi5cr1WW03XVoJBKSFJ18teX1pE1+RUulXf1EglJBPj4eP4WV7jrlivbICiVEtgde78/zJsCWlAaSEWOc31Ur4/xjd9+0NMzTU1ShxpalhKgADKRpIO8gfQ0iukrBSYhYq+yJSxWdovL3tA5Ow8Of3aWXax5LjYhJWU+zmIBEadekWZ9epxFdUKIgFSSAkyIgRty0+Vo0guCE8ztQVYCnxtc0aLeSdDBR8Bryj4ab2UqfHdwZUpD7vd6smY2AiNbQ9TPj1ftE5hm/igxG8dI/DnBix/tlBKFgpIA0gkpBJ+ufI62pvaQUZshKls41uDIhSqogZjqELmdNzFuFYtuIqCPXBKgIJQrmPL9NLROmpKWgSVJMAeKvx5+Fki9cXXfdMtBYdeEdxBTod9Ty3PWyjifKbNCd8wbq5THV4iuQs9qLyQhIGq4IHI7kRv8Al1tA+PMTXS9X1t33YoPoeqSS4kygJzT3dBOs8ufibN2+cVXnfK1B15LTSlZuzbACfCepFkJwkmSZ1mbbNLRcrV+6zqiqzCzVKmAr341cQGKfB2GsSvP0eH6ZRpKOodSWqdtTpWoNpUCBK1knTnvtZ4VHDb0j2UFa3KF0azFOwZ08W+kfK0HXNiG+MM3i5W3FeD1G8oFGdpUEpkGPEaDeztp+PHFSmbDIxdUOJ59q22v/AMSfK0Kimnz3pwy3mNUemqKcMtOXX8inJX3Fx/pCfWKJokDVSael0Hnl0skOPcYKYAVFA2SZGrLIM6jlEGNNNbJr/GviHUp/1i/AuTJ/1dvU/DysQe4nYrqEdm7XpI6hpI6fDblppZmQVHvsYmfNBf2HOUu+jvjbiFh/j5w8vO9qYNUiMSXe1ULygfZOPpbVsf5VG3uU6wmoZcYWO64koPvt87F3cR8Q0d5Ud4etqC6WoRUJKdCClQI193z8rfRFcd6U9+3Ld990hzM3hStVbR5FDiAoH4EWx8WhdG9jy0D4K5Qyhwc0En4pmoZU2tTSxCmyUnTmDFh2wPHQR77Hr+puwvFSwO66AuY57H5g2JAbDkNPdbUpnZmhyrTCzrLtKZI6fdYVI01A06GwafaGmn42FGonSDbQCpuFwlK6XASUDnOvwj8bIvE25b4v7DDtBcSEuVPatrLal5AtIUCRPuB91lO71APpHiD94/Gx29awXddtTXFBWKdlbmUbnKJj5WOzVtlXJs66im7rrVg/FF3U9JWPLqH0Mrq6dTvahlajCgk7hJH8O2kgCbKOJMM33el9XtU1b1TQU6G1PMVyS2tGiRlTlVJjroNtDzsmcNK67byrqjFOI7zZFY64VpSskwSN/IDQDoLOXiLe63kUd0UToDVUA645mhOX+GT05+60W5QwnoiOJzBJtLfV5PYCqgXVpFPUNN5idQ2pSZT1jX4WTLner8O3xd97C81u095CeyKUgN5VZVokDvAyDrqDOuwDwF1XYjBVbc12VtPUv+rrcUW1pUpTgEyQPGPlZjVLqavD13utLClsVa076htxGYHyJTYtrAFCO5slXFJvy9sQXlSUNclqopwVUyVpKkQlsLywCInXvDbxs9uHt9P3vcg9ZeU64woJzKMqUggEEnnzs06+oDOMburldxFdTMOKB0GspV8Blsf4ZOLpK+8LrUdG5Qmf/VrKB8osQaG6gdk924Q7UNjSF5h7xbFyo6bW04Cm8Fkq0cbBjyNtqGulkUwRq6XhTXnTP7BLqZ8iYNnxeFMFOr09pFo7EA7xGoNpKUsVNMxUgaONhXxE2ysUhEkLgeyNC4tdomM7QgOrGuijz8bZZyLpElROTc2y3mRpz2W34srwV9NHv+kRiJKCqeyoUwP/AMVa62h4pU5UPAtqUO1IzztPu6mNtdbTd6X1DUXn6QWI6u7mTVUy/UsjzBK0nLTNA6p8RaOgm6qRypTU0i0v9qRGYDlrufLlrbuqF2Sjibb3R+QQ632qqQ+ZTfNOvIVdmXIMEARoRoD4mCN+Vum0KUMqgYmSkIgnu6Hw3s42b+uWmSWkUhb7Q6gqSABy5zpztpNfhDR00jp00MmUmdNSoSefS1i7uyr3HdN5NOtQSCFDIJJjbTczz11526TTKSQrs1qOgEE7xp5f1svJvDCi/sadp3vryJGZQiPMzz84B6ahorMNLUnKFoSvu7geMb7fITZru7Jhbe6RhTK7RBZUQANYTOu8nx25fp2lsOBbYBQghSVExB1PMzO55zrvZ3XFdFFftO7U0YORtZQVqWB3okiOehn8rKQwa0lKgnKIEAFcmQJjlGmthGUg2RRGXahR+pgEkZDm9jQDfw6a+7QG3JowBlghSN4BHI9OUH7+VpEZwSwk5XklKcxSR2qZBBI3jflz2EeBhGBLrUQlZcAk5T2qNTrtO58+gtHn2UxASoxUylWiGxKUEFMTJH6fHnIsG9TOskZ21QRA1ISTtM/dytK5wTcwSsOVDwC+jiMpJEztzn5ixd7h/cDo1vB5IMk6tkgbRtqNPhNk2qF05pnEaJF9Hp3Jx74al0pAViiggbjvPo01/W3uUmloHgla6RgnL7RbTP3W8VOGVzU9w+kPw2pGHnHUDEt2rK1lJOY1CBuPK3tMmqCW0x/CItQxSPnuY63RXMPcY2uae6EN33aglQoKYf8A2oflbtVFdqtPU2CdtW02AdvRlTYpwxC51cnSPKwaqhcZc0jSTO9s40QCvCpvuhjRXcDpRMeP2Yts0N3KHeo2TP8A6sfPSxQ3iilJW4hTk6AJ387cIrysZjAG8TIsvBC17J/Ek6XXa7toC92qaVkKJ5NiT5m3SqOkGpYbEf4RvYM1Sj3gPjbRfkxy62mylA0smM/mjLdHR5Y9XQIH8o32sMmiogCOwajWBlFibdUkbq92xsKKnTUjQbRZGm8kwnv1Q5o6TVPq7cHQ90GwYu+iUZ7FHvQPxtyKnMYKR1tz62JiR8Z+FkKe3RIzeaFRQUSVE9g2DEewOthW6WkH+zSJ55RYqKgT08hNuvWTI0EcwTZzTeSXO80aTT0iUkFlI0/lty5T0syWUSdiUixf1sQRIB/G3KqozKTJ287R8L5JCbzQnqzCVaoRoY1QPC3S2aXRXYoJI/l+di6qpUe2rw8LcGpOxP8AitMUnkm5/mhzSUcAeqtyP8IiwaqWjAI9XRA0PdFgRVDYA7W49aI15b72kKXyS8Qe6FNNSAZfVmTH+CxarpqXstKdr2T/AADW2KqDBmSNtPr60sXq6hAbBMpgET42hLR+zspsqdd0LTtUxcWoMtzl0gC3KmKUttJLLRk7FINidNUIzokiVIn3zbhNUcyNZCSNZ6k/laiaMjorPiB3Rw0lGWCo07R3klI5Hy1sXNBSBgqFM3Ko2SNPlYJVcAxCtyo87dPVI9VmRMA6ajey8Hfol4i3VGU0VFH/AJqyBH8osXcpKVS1/wCrtysgapHxthrQG8wIIjmbAIrEuPyskZQSOX1+li+D0Aso+IHdDVVJRtpb+wb6QEDW3aqChUmPVWdAN2xYtW1QUlJI2Uees27NYlQSoATE6WmyjsdlB1T5oq7Q0Rqkj1VuAmISkCxWqoKR9QaRTsyNzkg8pNh6qrAeKwEwlM+B30sXZfJSFrCgpausxodPlNpCk1TeJARSpuyhaUlKadgJgEgJAO/lH1POyJjfEeE8GXMuvvnJTIcKWWlN04UslRiUiOQO8QIsr33Vt0FKqrdJIQhThCdzAJ08d/ebRbcGI79rcR01/YmDSVjMoU6XipNPPsN9IGpJ1k69IBPPRUUrYqqVrC7a5tf+e+ywcW4vwfA5oqfEalkb5PuhxAvrb5C+lzYX6pcRiO8MUvLRgjhjeuIHWNEVdc6unpGv8Ce0UUqOpJytlJnnYPEtVxmwxd3rjvBtqrROZw0lQ08G0kDMAjKFefLa1gsNXpQeptMUjDYhEgo0Cjrv1J387Hr6vJdOyFOpbSpcDLoD79Ztstw6ANzOW94mRzg0BQNhDGOHcbUqwzdrdLXspHrNI8hCXEGNSUnvRPgR42PVV0UTy8qaJkRqqGx3dfdzstYmuS4W3nsUMXOzS1obzKrWBCnAnXKobEECPf8AAtd8PU7dSkiHQFyIgAid/efoWpx07XOLWhPUSOiAJRBi7aVFR2aqdr2gQMoMCzU9IK4KC9+CuN7vFIyQMPVy0p7JOiksKWmOhBSD5gdLPSpr0tuKpkyYGXQJAzZgrPMZpgFO8Qdp1sRxlTIvbCl8XYElXrtBUsGei2lJ/E2sxUgY8EBU5KrMwheIeVGaMoAJgx8fryt2gpMhS8pSQmdv6W6XTLbKlSZB7+bQDUgT7x4W77Fx1wpQgQpOyU6nX626m3V3C5uxC2hCyrurSQkbwYV9QfjY5dZU1WMr17riRmSNBJHu+UzYBhpc5i2CQROYTpIGsn6+4222huHUISqFZpMQI2Gu+kTHysCQ3BCIwa3TqqFJCQhQGZXeAUjoeo1nQRrz8bNq+lKFelWvsCO7ObU+/bcn52dVShPYhLZlKtFApmSNBz8N/DrZq4mICWlpTIyqk6GdRB301KrUqQgvsrM2jbpw3S4pyiYcTMdmjvJVJEae+Bp7rB3letLQs+sVbmQx3I9pSeUDnP4Dpo00YmXQ0DNJTsJ7RuftFiYkkgRz3O/wsi1L7tQ8XX3C6tWsrOtrTKLM4l2yruqLCw3S3fGMayvSWaKaZrWT/GfMjys3SqTqoz420dNAYtozGvTQWvsjbGLNCqPe5+rlsSIH3W0oGdedtwrruettuwQDO+sRYihZcrGoKREgW0RA0t0QQlCtYMgH322pGUfWlklZBgnpbDpbcHe2QdrJJaBMzFvoC9EPEwxj6MXDO/s5UtWHKSkcVJ1cp0dgv/maNvn9O1vaj9lpiY396Jd13ctQKsPXzeN276wVioH/APs/K2PjTM0LXditDDXWkLfJWgxLThdO3UBMFCsp8jZux7vfZ6V1P6zRusACVI08+VmbBB1EG1fDn3ZlVuqbZ110AZkEeXIGwiVEka6+f1424GU9QDbsCQRlPiTbXas8lDMOdm6FE6CSfdr+Flh9hqqYcp30BbTqShaeoIg2RRBOVZ7p0PlsbK9EtTlGwtftKaRm841+djRlVnjqmgxwgwXSyKRiuYBnRuvfA+SrKd74Dui+GqVpb1Wx6m32LamX1JOTSAeu1nGCYInztuYMDexg1vZQLnHqm5h3A1Bh2rcraetrH1LQW8r7gUACZ6eXwslvcKbr9eNTQ3pV01MpZcNIkIU2kkycpUkqSPAGBsIFnwd+VtiNgdtpNpBjbWso5nXum5iXA9DiGipWEVTtFU0IinfagkJIAKVA6KSYEjwBEHW3eEsHqw4XXaiu9bfcGUrDeQAeAk/fZwiAJmBbtKjvpaQaN02Y7IpWJy1NO71KkE+Yn8LdLSDuAD0i27wgNIcP8DiVfOD+NtrEAfGzEalK6BIg66Wf2H3RU3DTmZLctn3EgfKLMJXx/GzwwW6V3fUsGfs3Mw94/S1WqbmYQptKUSjU6Wyw5RJJtluU8C1Wsy+aKnqV0TLbTbriTJBKVQo6kgT79rYupf7ZSHUOJWhSgoFKe71GniTp5c7ErrRUPM06e0lSjHdIEHYDz8PlZw3dwt4kX7eTrd10HaPIQp05qpsbECSSdTJHnNulbAXOsAhGXKLkpKRXLJyw6cw7sjUHp9DWNfAVq8AiPsXE94SFIBJOkeOhP1vZyDgFxjQO9cyEiZM1rO//ABeG5saY4A8YFKUk3XTk5SRNYyddwd/A/UWL4N591D8Sz8SaabwKkpJZMagktak6EAx5RHhz2t164lcIRTrz7H7KTBEdIPx+FnKeAvGNKCW7lZIWf4KpiZE/4hOluEcFOMbSI/cggAp0qGOZn+b6m0TRydk4qWd0jM3nVtoNLTO1SEGVlttogKP8SiU+OXkPOwiL8voKyprLx1BSMqnIGu3OOXlAFpLwNwPxrUUNS5iDDfaVCHx2cVDYkZT3QUK11jU+NnO9wNvpCkqpMKqbAEFKagEkynXVemms7jS1Z8LmutlVpjw4XDlB7V+388+koq7wJUnKCM8gj9QOnxNlBqrxmtANOm89QDKWlgq1O4APj4TrFpfp+CWImnUqVhuoaAUNRUJJ1Jke14AfGx9PDvF1EtaaXDroCTKYyq5bb/XztWkEg+7H9FYiyH7z1CzbfEp1BS1Q32pAgmWFqMxodE+7pvvvbmoXxJoGwupuu+0ZjKlqpFyr/l01Hwm07IuPG1KlaHMLrcgzJZAOpO2XmIHxsSdu3G5dDVTcdakODPpTqSkCBtB05AeVgNdPfWPRW+XT20kN1HvA+svyo454BTeCKpDiMR3amX2i2QPWGzzA3k/W/s+24coMxMHS3kZhG7q+h49cPn7woXaVasQXcQFNlBIFWkczt+njb1wbIKEjLConeyqWZ8ptZQp3ZMwBuulKBIkkkD3zbRrW0CFDYfX32CcDmQlKTA59Pf8ACxJfaKJKVGOsbWiyEbJpJrbI0XGXF5yRpEn687dhxnLlGvXX7rJ6EPOqKUJKlRJAFuS6QoJXIPOfOxfDg6IQqCEqhbG6hz3tsuNCSCU/XOyWl4EaEH8LYXhJGbyP42QpgkaolKgqWN9R4RsLYKun5K8PGyQXRBIPKNLcl85ozSJ0M/haYpWlR8UQltVUyTBkkab2w1LMggk/1sih+SQFf5ptgfy94mRvG/0bLwoCXiilv1pmYgnaLbFUyZnSd9bInbymCZ58tNOtuhUAASSB58rOKYWS8U5K/rNPMHl99tKqqedR8bJBfgxmI87a9ZClakTy8voWXhWpeKclc1TGaRpNufWGOaVCNLJXbaaBWg/G2lVGwEwY2s/hgl4pyVjUsblA26/nbk1FPMZTrPObJKn+alb676W0p86wZnnt4WcUwUDUnolUvUw/gHhJsWrqinLZGTcECDYl6wrQFWnyH1pYvXVAUhKRv5+Fk+nFlJtSbow3V0naNfZx9lHwtw3VU3YlYREuAb6aWSPWsiGXRoOyUbabeHqBIOy5Oo11m1c0rUUVTkpmsZIjLrJOp0sKurpjSiU/wyNY6DayKXs4eWCDEpH4/dbfak0oQIIUmInfazeFapeKcUqv3hSJZCezCcw6mdLapa2mLoV2apUk6mZFkJ99SlQD3QICpGnxsMl1LVYiDsjLrtt+gs7aYXTGqcEs3jWMBoQ3oCBYRNWwpCYa0jrrttZDrnZY31BkgzvPL68bCGpCGwpSoCU8oOv19/jYnh2hD8S5Gqqsp1qSyWZzeJny+f1Nh0vUyco7IZUkQn8Y+trN52pGZK0kgFU+1JIB15+Ph77KIqATprBgiTr1/A2cU7UxqXWRbGbP7zuhdPStpS6QISCdYIMa/wCW0KXfhJFBfqFUyltVlU+sPpc9t5SicqTOvdmfCPE2mu8KxtlgVDnZhDUkhfSNt9P1swEv3FTYjVelLnRSMKltrJ/eZoC1E7giVLHUjxt5rxrR07a2nzVHL536t4uNWDMb69Lm3a5B3AXiv6SMOoKzGaEVFd4fxREEozNGaEB7rm+wzHLc+zmc07tCku8qC+Gbjbu66PWSlaQ2+61VGmWUhIB+217IGCSRKtIA6Q/w1wbfK+IVU4aa7KOnLx9Yco74qLzcWzJACnn5hUawAImNYky/XY6urDaE07jpqHMmbsmBo5I0KjsBp567WY1XxHvCioHLybxXhu57wecJZu6ro81OW0gSEhKm1FW0rJME7AaW6SHFKB83gGygyD3b66fv+q95ouIsEkxM4HTVUZqWC3LzAu0Got1IGpG4GpTdx3duK6DiFWXaHbyebplZW1Ul6lkpYX3kuqZPcdGbuwQSIOom0rUak013U9GkJJYZbb2BAITG3LraJmOJV6YnvH97YjorgcfQpLIRQq7TIIEKCiTBEz5Ee6WWmFuU6ag+1lTI2O28Hb9fhpYbkkkfbotXH2uhZHm3P+ySLwWVPqyEJ6gE6z/SwsFaQ24SArunXnY0u6qipD1TSpKm2U9/vAGYJgDnokmB0sGUAMBJ1hIIkRoP1+traXK1XPtkuF5s3lgTg5d993hd1U9cIqKardp3W1VaUqSpC1JII7QQRER4W7awXwSJASbiWk94ZbwIB+Dmmn1tZpccOBd6vcYcZv0lQ4luovusfab9XWrRxwrEEDYZxr/WzFc4E4gCwGqgEKOn2S5j3DoRaq6CEnWcgq82aUD9gCpxTw24KuNoPZ3WG3PYCbyUQobA+34ixyn4T8GagqSlNGMoJUTebn4Gemnnav44F4jSJ9bTOxhhwgKnaSkR+OvvAd4LYoakds2SQYHZuakcpjf652Y0sbtqg/z80QVT26mnCs4rhzwgKeyQ02peUJAbvBxSZgSPMAWpzeIvGtvapoCpby2luNoQ3qBlmYA8E7++y/U8IsVMJ7QONZOROcSens2ZlfR1V03i7d9UCHqdwoWJO4Pxtew2kZAXZZM5P0WfiNS6YNvFk/en1cOF7hrsKVhvG5L3Rf3rBTTvZgKcNED2knvSIX4W5VgG74UgqrEqB2zJPTonp+HSzfwnhXEGK/W/3M+kepJStztHSnRUgR8LKw4ZYyATFawCJgB9WnMja1mR4Y4gyWVNoL2ghl1J2BeEnBvFFbSXE87ilNe8yStwrZQyVoTK8sJVpvE2kdz0P+FLTvZu4gvtk6aCsYJT5y19aWrnRcPeI7D4eu+8y06BlC261SD5A6dfnZTZwnxnbCQziitbypygC9nE5QIMb6bfK2bNHK5146qwWrTzRNbaSmuVP49DXhY6mEYqv9IiRL1MNf8A2c/jpZgcePRkwhwy4fuYrw3iO8q2pYqmWXGKhbSkhC5E9xIMg5fjysyxh3j8ggJxjevcOYRfbndPX2tLB3lhHjtfV2que9MUV1fd7qg4unfvhTjRXMyUqVBPPa0YW1EUjXPqgWg6jTVFnkppYnMZSkOI0KiTIC0DmEhUWuxhD0DcD4rwtc2J0cTrxT+9rvp67sk0TXd7RtKymc+sTFqzI4GcRilSfUGACrUGpRqRtz8fnZ9XXdPpQXNd1Pdd1YwvVijpEBqnp270ORCEiAlImAkACBt87HxOZ07AKSoawjfbVVMMiZA5xq4C8HZTLWfs7bkRpR8RbyWraP3ahUq5ahyyY7+zrrQFOIx7VoQJGZdyk69P7wC0c1N+el/hygqr2/ttfzbFNTlx51F4gqS0kEqVvOgmSNbM930rPSHfc7V/ipfLqoABcUhcR0lOm3Le2dDT4zKLxVTXD4f7LQmqMIjNpaZzf5+Kmhz9njfwgM48LgUCRNzrGg3/ANpy52vX+zu4S33wRwPivAt7XsK9t+9Wr1YUKZbOUuNBtYhRM/3KdreeWCeMHpvY1uz994PxLelfRB5TXbJoqQpC0gEp7zeggjTbXztc/wDZ84w9J68uIt/3Tx4crX7qeuYuUDj9PTNhNSh9EiWkhRJStW8ju2rynEQ7l1NQxwG4H3vyUwKBzM9NA9p7nb81fQTJ3mzQvSn9XrnW40Ksw15GzwOnP4WQsRU+rVQBqe6r8LX6F2V9u6oVLczbpFCQrziIjlYRMDXziNLZGwiefh9a23IPMiN5tuNWWVsAg6kjlZVo3CpmOYUqPeZHyNkrUTAPQ2P0Kx2SkA8wdPh+Fjx7qs/ZHRG4B2tsEgQZ+NuAYGhIi20qJ0HPpayhLsQNzNsB+VuR1nw0tuZ1jXraSjZCA7yIFu0nWJFgRKdCLdg6A/C00y5rkFyleA9rIY05jUW4YWHqdp0kypINh1EREEjpYpRwmm7Iz9mpSN+h/K0Hbp+i6WIEWcWB38lZUU/JxvN5wf1Nm85r1352U8KOlq+WSTAVKT4yLClF2p2p5lCpPnbLGyyCZtlsbwZReYF8y1yMDtqJsrT/AHqClMcyofD63tZ3hc36xf1YhCkIHqyl94EQntE89eg+XlaugumruW/W7uvOkeo66iqEMvsutdm42sLAIUCMyTrz6aWsrwoYR+8a59TqJSwlGTKRoVSeWu3n+G3R6vVaq0jKkZdMppAl5CkKITlSqR4ae+PrUJIKHQsEpBVm2nQHr77HcjRkKfSJEknpPM9BbYZbGZfboBGkZuu8/XxtsgmyyQUSWlWYo7Mnp3oEfX0RbR70KKRBIkg6n3fXLxsdfYYOV3t0HQagmVEe7cQDrYFbSfZDiQIHsmAfL57/AK2ZI6pWw0UmncgElDqTMQAY+Hx8bLSlJUrTmZSOew0iyXh1lC2HkBQUA4gRrA36WV10y0lJCTpqOUjSfAc7U5fvLQg+4gToc4BkEkf83SwagtCg4goKiSSd+m/32MKRmbS0EbTEaa6nbz+6wawlKhmQRGojXbb/AMMfGwUYaoqrOU9pM7E89IOke8H614dGeEAJCUwMo8xz933WMBlSm++leqgmVTtpNi9XUt0dM7VVSyGm0FajGkAT+O332Y2A1UwCdFD+M15PSE4TtwnMm+qNSincD1pr5a++PC3py0tKkiffP328tL0Zqr99IPhstbop3q6/KFPauahn/WWwgRzjfQbydzb0vYo77SjvXuDm1B7H9bYVa7M+4WnSgsaQU41V766VVAnL2LmqtNZ05/CxZTUJUkwYGlkoUN8r7v73jn/dzpYYUF7BKovRQMRAQLCYSeid4A6o0xXO3VmdYy9ooZRmTMCyOX1qUpa1SpR189/wsWq7tvjtZcvdzIVGAECxJ677wIg3w+COcDztbbtayqOOqVfWRvnO8QNOVs9ZzJzJUTpMb2gzjpj3EPDRy4aijxDUoar3H23Udmg58oSQdUz/ABHY2jBfpJYuCxlvZS0yCokJ90AJ56+4WRcGm1lJrSRe6t6qqGYnMYB+jbSqgToSRynf52qO16R2ISSFXw8CNPZTJ337unnYQekTiLTLfVTCoiEon5DSbNn8k9r7FW09aEHfpEjT4Wz1pJElZmI0MaWqYfSExB3Qb9rjumAEzOhk6eduf9Pl/k6X5XJEwRKQfu1s3NA6Jgw91bYVYTJzbTr00tv1uTAKZGkRapX+nvEJ1VfVcDzBWNOXLyPwsJT8bcR1BJVf1dBMlRd9n4b2Y1DR0UhE49VbH1kdddNRblVUQJJEeGhm1TarjpfJWQ3fl4wJEh8gyOfh/WwaONd+OHN++7xIBA7z6tdfP60s3iG9kuUe6tv60MukkGZi3CqoyIB05+61arq4q11UUl69byM7/wCtOJAkaaTZwIxy5UIStN718EQT62ufHn9e+1d9e1vRGZRudsVO/rJVomegtpVSQIBI5bSLQvV4oqWqVpw3rWjOmCRWLmSfOxVGInnWnXP3xevdGg9dX123sL7UZa9kYYa89VN5qTmCcuoJ1BsDWvnsgoczrI+vC0Eu35WVICGsSXg0rIQ2VVbqkk9SAoaTHyFgbrv28C9lev6uqFJ1J9YdCdAdcpUSNfH7rDdjMf4URuFP/EphXVKWw0IMypJnaJ52ENSoUjwBVAc0MfXW0Ts3/VlBUa905lpQCqqXpzn9fCxa+sUv0rq0IvB+FKJGWoXBA6A/j4WAMZiJtlRjhUgF7qYGql1QdICiCoxESPrrbbL73qqQGlzBmBpPh8rQV/bZ5KwTW1ahBJmoURrtz8/hYwzjh1bcoW8oR7Xaq1PlNpHF4x7qiMNf3UzNLecfCVpOVIBUY+XwmwjtS4mtbUlCikiJI8TztA95Y4cbQULqHVE7lLx/AxG3wsgrxs8FHtKp0g6hRfWSnrz+tLPHizHe6hvoHN6qy9dVuliQFEaHu6/X17y7la46GqVCTOUSRII+XKTatzmOakSDUOgQBBqNp57+fhr52BYxhXVryKelLtQ84QhCEOkqVPLfzH9Jsc4gwDMRYIElMImF8jgANSToABuSeysvV1Sm+xTqCDEADUzvv5/Oxw1TmwSqZP8ADJJ5QI158rVyvCmxfQUnr7tBmbR33UtO5loSJnQHXlO/XyaJx+DmSVNDvaguynMI0OXzGnh0sGixqkxBpfSPDwNDYg29Fl4XiGHY5G6XDahkrWmxLHB1j2Nj/wDe6tPiNTtVdyqZtCyuZCddgZ+vIc5syMQXtdd0XTUX1Vlymp6Cn7WrJTKUpSnVWnu0jnaDWMbVN4uM0t2Ip3lrdDYKDnLh07sTBO0Ajc6TZwYfwrj13Gt2XliK76L9x0qKhbzS6sqcW8plSG15UjRSFrzJOaQUBUyEhPGcYYVhVbO2pxKpEQLQCLAucGkn2NQQ7UjqDcAheSfpU4bwGpnbW4tXCElgBYWhz3tYXOHL9oOa72nNvZzTcAhSHhrFFLiehZv6hYrW2FuKQ0K5oIcWltWUHKFKGU5dIJ0s87joHL0XeF5U66amASpLeapLCgo6nIUkEGOn83wqF6Ut0YhvClu2sdoEXpgtARTpo2UBhyjqkrWhTi3nJQmUkBOdOXQCUkgrhio4p33dOBqy8eH2Ol4fTdVYKI3Led9JqatTeic7TLrKlFG0hL0JymExrYuAfo6kDhirpeWMzniA3JZe4ALySSQ0+1oCTuTZQ4B4ClwTEocaxN72MMhmbBbMWOcHNbmkc4uJDXe2LC7gMxOVXK4yYwwVwJuavx/WUl2VuIq1Xq9Gl51T9RVviCltS1KKuzBBJAMQdNYtYrBF93hiXBNz37fN0JuevvS72KqpoFOZjSuLQFFsqOuk8xO0i3kRgjF1XefECi4gcW37yxxdeG6+ncqEKr1BC2i5CcgdTAbDmUqTlSFARIkm3p/wy4+cMOJVyvXhcd03xRN0xyqRWUCUpWoiYQtClIUNQdwYIMW7N0dPgkBnqnhreribDyGv8le38Q8SUrIjV1jxFCzq4gDXzPU9An9W1dRTlLVDUPIC0lC+z/jBOsxp9e+xQOuCnCYkgEb7fWvy6TbinobnxJTLcuVpp0IGZxsoylGmpVOgEDysSaw/StqcbVRoSpCihSVIAIVrO/v+BseGeKrjE0Dg5jtiDcH5hVKGtp6+BtTSSB8btQ5pBB+BCrnxXQzR8Rb57NKFhxTa0EkZUKU2gkx5n60s2A26j7ZbCcqlykFyCRzI0mPHaNLSHxpuVFJjlKadDbKXrvac/hSBlzJ5+CR8epsyGcqJyMl0+ylS/ZSNTt5C3I1pLJ3DzXodC3mU7HeS4YDKGM2VSXCmMuuh6k79dPuix9pl54CoqmlJbQQlClOd0mP4QBJmeWxFg2aFx0gKzQrQZj3R5/H52VE3S+6UstqXUObQ2nugeBOp1jlG9s50luq02R+SJiqoEKbd9TdUhoHVbgCAdIVlAjp4RajfpF0xb4v34+CFipLL+fkoqZRJHhmm3ohddw193RWIp6VtxuVhyobQ6EkcgFApJO+0+VqW+m1R1I4p0F71Csy6+6WitwNpQFLQ64kmAABoE22+G6i9YWX3CxOI6Y+DD+xC59C6n/e2OcSYVDvZt3lcrskqEDK4gAmd4Cztr94tUOHVUzeVHedNXUyl07yXyhLQyZwyhtSRprIC9uZGuhtUT0QHew9IC7LvIKhedNVsiDz7Eug7TugfI29J7rwq5UOJHYFSU+0IgRoTJ2+NqnFc5pqwG+7Qf3KfDdLHVUftjYlR5hPCDdF6w7e920ZrKqsVUdo02F9mgoSnKMwEGRMxz0I0s9LrwFh6rp3PWLqp0SkhdQ8wgpGsgJj2juefWN5e1Ph5xyWGm0JaVHca9lRnmd1e/odrLtNhyjpkJNQpS1DQNI3A8eVuJlxEnUFdWynihAbZR5TcMcL1ZhFz0riAqHHl0bYA0iZJn8fCylTcL8IUzwSvD91VCyg5Qu7GyMxMwBGo1PLntaSGbucXlSy2pISO6VQVDy5D4WWKHDrrckDKs76ypXmfztRNdNJsSoSTQRC7gFHdBwrwSVKevDCF2OLUZCVUTQJ6EgDSy5d/CvBbkBrBtyBS/aWu7GJHl3dB87SlcuBKyrUFKYhvfOe6LHr0vXAGBABe94N1VZsmlY7yirkIm12DDq2obzZXZGdybenU/JYNTjMOblwtzO7AKLav0d8I4puK87jTg663BeFI7TEpom0gZ0KE5o0IncW8OeI3Dq+OG2NLzwjf7SGqu7qp2ieQlYUEuoUUqExG4Pwt9B3744n44p+xwrczeG7sWSkVFUClwp/mSga/+Hzt4Xeltcl64R9IHF2Hb9rfXK2775qEOPrIJdBclLhk7qSQrUnfW3ofCURpXOjjLnNI3IsDbtfVcvi0r6huaYtzDoDcj49Fdj9nHjDFXEzgjevCXDXDK4HG7kvNHrd7FlKFLQ+M4U6o5ipwFCgDHshIjSTdrhv6O7/D+pcxZfGJHqq8QhYFMwgNUrYVuAndXmTy2tRn9ivjVu7734o4FdUmX2KC9WfJtbrS/wD743b05vO+e1bUzOihZYlDRUta+SUku0sOgQoK+qdCIItG9fNJA22g2J3qwX6FwACUQse6xydeduTChlOxBtfhOUgozxcEJnjrIM+NsEzI0BsJUtFp5xpRPdWRPvsHObmfwFuiYbi6x3Cxsuk6jmT47/WtjlArvFPUH5f/APViknlvy52GpFAPJAI6bRpB/SxmGxVZ2oSlJmI2NthUCYjxFg8xOo3tvMAYi1oIBF0JmmU78rbkxoZmwQUZmOdukrPWbSToQGfG3aVDloLAgkwQdT7rdpUfjpaYUEKF7GSDYlSLKausYV/C4lY8lJH5GxvNy0sTMoveR7LzHTmlX/4VovGxTjZDu6CT8RYe6XS1XNOj+BYVvYBzQaH4W7u8/wCsA9TZnC4SG6lQaibZZPp7wR6u1mInImfhbLU1KxXza1ldW1yUXreda7U1SkJdeeqHCtasqQBKiSTCRpPQctLWJ4I1KaykqLwbGj1OwqCIIkExHvi1dqhoC5UqQUZvViVAmJGXnvzPlJtNHo1VqqnCDruYlxtSaYnmCCSOnJQtboxaRAqjdimwvuQIXlMCZnQ9PrrblT4AMJEBUzOUT9fW9ioWrNlQrvHQawdfr77cKUVAq5JITl073nr1jpbYWXlSg09naUkCSlQUBl0iNfHePKwIcMkp0KtBHhzH6WKIWppxXcBCu6dTroPmB8NLYo5XCSVKzanmCCJjXyH52jspp04UrMtQ9SlAJcAWmT3RlOvwzE+6zjW8gkIIlSEwFTvBSdfjFo5uysTQVrDwBIbVJGXcc9ORjbxs/EiEhKF5kkCFfzDuwfutTmFirtObtsjDLyErAKcoEESAeUfiN+tgHC2oHKkndXgdD005m3cpU2kgbbjaCSBoNtraUSVGVKhZ2+A29+21g3CsgLlGq8qjoSSdfE/OyNi4LNxOICBo8hcEDTvafCPu52WFJy5SVykwRoNtPxn87JWJcirnezGEkoPQnvD5am0JD7BRY/vBQwqo7Hj/AMLqk7NX9QyAT/6W1r48/qI9SWmzEwTAJkdbeVl5ieOHDWFkRflCZyx/9abP5/Hx09W2UKKQCRbGlF7K606lFlEtANopsyyZzz7O3626AVlUQdTsPKxlTQ5jfwtpxPcVHTn0s8YUJCidM7RNuxejCnWgNEpMGbIa0JzqUlEAqOUEagePjZUcQcypnfp9eNiym40BEeFrYFlUc65VX/TOuervahwkzTurbisqc2QAkjs0jW1bEYBvlRSHKys0jQhJGnmOv1qbXy4j4RYxbe2HqKpkBlyocSR/lTp7wDbig4G3e48h4sEpJzSEgHwEAWcU5kN03P5YsqNt8P7zXI9arQTr3SjT/lnx99jyMCVyUgpfrTqAmQmfd3frS17F8B7o7uRlUpgCUz/TexlrgVdqDPqvxH4Wn4MqPigqIJwHeEBWesIBPtZdd9u6PxG9hhgO8gJCKzQiCVDUdfZ00+6192uB9zpVnTTQeo0sMOCVyDUUZgHTS0DQuUxVjqqG0GAK591IcTXgaD24jbbTw+tLLF58Na273TTUVNWrbCBqp32tj003td5rgvcqSFeqGec62Wn+EFyuHtnKRJlA3+dq76B+ZWI6tll51u4FvZHdXTVZnQ/aRI3108rHKHh7XLeSk0tVvAhwxH1+Fr5VPCO5ipR9QB3GwtjPCK6GzIowPCNjaf2a4jdD8a0HZVawlweoKsJdrGbwjmPWlJM9NvnvaRKXgnhVZS2bsrdQMx/eT5neP44tYO5+HVFTHuU4TpptZa/sXS5irsRrEEJi2VPhMpdo5Xo8SjHRV+e4RYPp2mEC7HnZBUCq8agkeR7TTrYOp4WYUZonVtXNWJWiEhPr9RkM6kkFeu/1Fp3vXBNQ4EhlkxlKcyAPj9dbeZ+OMM/tG1YtvtrDbWMV3SmueFKpBpQkozKCcoKQoiIg78+dqZweU6F1virQxKPcC6s4rhldYqFOtUr7SEtxrWvGDG+qjZKHDigU6tSG1FQB1TVOyABr/F93nasbOAP2glc63+8aTHzhYPaiH6ZAGkaRsTKhpy052U6jBPpnvX5ddSxcGProYaeR6x6xfQdQ6M4kqzH2d5AjewXYK+37QK0zFWj3FYSnwLd7GWWTLS+6nt3e74+1YpiTBTDb2UMFaTmJJeXJ1GpExP1ytL9NdKl0qu3p0T7I8/CwWIbnaQFqWwhARoCRrtqflbNGGyB9wVf8aws2UDNYLS66EqoQsEGR2qzB85my3TcPbrUwCbrCssQStfLTr4Wkpm42y8ooSgrS1JgbfGy3T3O36qgpaEABJMc/G0J6GU7FRjq4+yr5fWAaZBytXelKhrKtfkfvs1v7FMF1LS7uZOdUAFoGAZ/r9C1kL6uVCqsd3QJMg/M2bLWHkVl8BaBnQz3tVA7g8vrWxqagltqUGSpjc7ZRknhjdRZQXropVKUTu0NenLaLKmHMDXfcV7NXg3dtO240FJS4lsBXeBHu3jTrubS21cgdrkJUgJSlJy6aTyI87FsR0X7sonXm8pWSlrMlEKClkICh5TPu87NiVCJaKZk7iGFrrkdrarC4nZRVmB1cFYS2J0Tw4jcNym5HmB307phV1ThvBFDVX3e9axStuEuP1D7gCnFnZInedgn4Wg2v468J6XM6zwwoKrKcqlIbamJ1/gnrv8pm0R+lNjVddxXvRtDyizdDbd3tpnQwkFfv7Rxem+kxraNKepFbSlxwqkDRsgqyEfyzBO0xy0POBn8J8A00tEyuxF7nSSta6wc5mUEXA9lwvofgNgBrfwXgn9H1NJh7cTxeR8k1Q1jjZ74w0Wu1vsPF7A210FgA0a3tRhv0kuGzuKrup6rAtFcyO2WybyU8kmkBBGbRsQkmAe8ICp11tYAV13UV2P30pbCaXKuqW8yrtErbAkLkDvSkDr0BMTbzXYfq6R0hpwgEaKMkgyAQJG5MbTyPODcnh1iynwpw14Z0t5JSaG/lu3Y+VAhKO1Ky2YjYrCE6xAWbYn6QOC6WiEFTQ5yXHKWlznEhrXPNi4kg2BAGovaw3vyX6UeA6SjFPV4fnJcSxzC9zyQ1r5Dlc8uINmkAai9rAG95NxjffD/CT9TdOLr7w5SCrbTUVVLV1bPbOAiATTyVrmAJCSD42jQYW4HV1yVGJbm4TYWvG7mlOZX6RphbbpSYIQW0kT/h0I+E49wTuCldqbtapXXu0fW89UOhvO7mg5nFFRU6eqlCTrvBFkymwDiLACqhvAb7VGq+vs6xpAQwz3T/AH4ags9ojQJKUJELglUC3szf0YYXilPFieHuk5MjWuH66U3DgCDq+97L1PCuF+Gq+kjmYZXNe0OafETjQi405lkiqx1waww0u4K30f8A1WqvNDfrtFQ3ey80hkrgF1QCZMEKCUpUTIjcGz04T454acTsLV+CsOXHRXL6rVPU9VcweLC0tZjNQ2nIlSpOsFIIPtQYlIu3hu5dzjiryzrUFFTpceU4tSlbrUtRJJOWdSdtY1mqPFa5q3AvE28XrvW8yr1kXnSLc1IUSFyT1zIB5THO2VxT+imidh4dTPfG8EEEySPsRtcOcR8xYjv0Odjf6M8MxSm5NHLJHKCHNcZJHgEbey55HzFiO+4NoOL3HG+/R7obgwjgLGL7t+0tS85Upey1Kf3eouFmnqUuSlfdWgDMCqG50JBL74E+nhhziJiSgwnj+7kXdfl8VQYp6ykcz0j76zoFhfeaKlaAyqSRsLVf9NCmYvWswVxRu5sCkxRcrajEEZ0ZViTvJQ8keSLQvwh7ao4r4PaTmWVX7QAJG5PrCBvbI4Kp2NwKJ9yXOzF1/wAeY5h5WdcLb/RkwQ8OQOBJc8vc+/4y85xpoAHAgBeqnHG6qV7EN1V9StMKo3Gy3qCtSVmBMae2qzAYuXOlvtXWWGPZTlkz7iJVEa9J91pm4yXS9UouqrbpwsIddbzLXCAk5TmPX2fDW0dUNzrdWG0I7VU6JRJA10HMWxMZeI6pwX0RgjeZRsP87opQ3OyAE0LaVvZc0rSVK5DXSBvznzsuXfdDtGPVnAw4onMHFLUUoSBuRG+2nLp1V6G46ltqHKkpUe4W2pyx0UZgn42c1z4cXUKC1tLWjQkJcj57b/W1uanqg3quhZCLXTeZw606sJU+l5RhHau5gADrKUgQn5mNrVC/aJYPN1VGBsQNStqrZrqMuZAJLamlaA8u+T59LeglJhwvqQlNOEBO2QcvE78hasf7SzBqVcJMOYhUtbi7vvzsFLSNG0usLMTHVpO3Tna3w3iBGKRi+huPosriFjH0D2dd1SH0Y6v93ekhw/qHV92qvRhiZj+9BaG3+a3stRXEktIbaBCAcxKlGATuI2m3inw4dXcvFLBeISlQF3XvRVKiAYKWn0rM9NJt76XXhR2qKZQvLykTpbX45gkqaiHlC5II+v8AuuZwCvFFTva421v9E0KW4EKgoWrJoO5oTPLUaD32WrvwsXIbbbSE8zMk2kCnwlS0jIfvB5LLaBqVKAgW0nEFAyr1XCt1OXk+NO1AhtJ/zH8Lc9Bw65lnVjsg7buPwA1RJ8ekluIBfz6D5pKuzAa47SoQlpsbrWY98C3bt9YVuFz1K6KVy+68bNU6cwB8SNBZVVhW/MQkOYovVaGZn1SmORB/zHc2Xruui57iY7C76NtpI5gam3T0eBiIB0TBGPxP1d8m7D5rEnxDP+2eXnsNB8zufkmW9cHEPGfdva9EXBdqv/q9KMzyk+Kth87K+GuG2DMHKNTQXahysX7dU+e0dV17x28hAsvVN5Np0zifCyNWXuSTCrHlloKE5/2knd2voNggiWpqG8tvst7DT16n5pZqLxbaSUp0HIC3hH+03uU3P6XOLKlKSG7yborwQCN89M2Cf+JKvhb2qrL0UQSV6W8k/wBrZcym+MmGsSoQOzvHD6GVK5lbL7s/8q0WngmNOrMTEbjuDZSloTDTuemv+y3xcnDXpVfu0uBKMQ3DXUQEwCUhFQI6/wBybevVXewnRe1vCD0QsTuYT9JrhxfCFBIcvlihWR/JUSwT8HLe2tTWymSs9bZHHbn09bG9uzm/kVrcPQCpiN9wVIbDgdZbeEQtIULdk+6ydh6oFTc1K6TJyZZ8jH4WUCJ6W6Kjk5sDH9wEGZuR5aeiQb8pstSHuTg5+Fk7aEzZwXyx2tGFAGUKnTpZAMg6ae63RUrrsWTUNs9YNNQkCethGVZFhShsR9+v3W4MnQHblNsVqDl1JBA94tcCpvSkFgiALbB5bToTYFC0rQlwRqMwFuwZ6ec2tAqsuwZP626BMbaGweaBJPzthjaIj5WmCSkhs2swJOlthQsDmEjWY8LCJXB15c9bSG6iUKD8x1sVrXA1U0bigYLhaVp/Mk7+8Cw+adATvYlfJKaBT8asLQ8P91QJ+U2d2yQ3R5YA1i3VAftozDf5WDWQoBSTyt1QmX4kanrZ7XTJw9sE6ZhppbLcdrGmf6+FstHKnuvAUYYvOqebwxSoaXWuqNKElzuFY7p1O4m0pcDMA4kwLdt6XdiBpltL9SlxgNvBySEmTpt/DpvZoUF6NXfi6nv59t4oRWqqHAggq1UTA2H4a2k5PFq4ezzqoK4JOpkIIA16K+trPAGtOYoM2ZwygJ39k4eSoV59Z2i2kIGaFjSROwJs1k8UrlSfsaGvUNCdEaHx73hYRzirc7ywldHXFUbw2eeuyvH+lrvOb3VUwv7JxhCAnUQIOgHhp5bWwgFRSRMkEEKGp8Pz8LNUcTLiUBN3V85eYRJHIe1+e08rcf6TrmkKTQVneMGEI58jr5fWto85p6puS7snZkglIEkDQRPLb662e9x3gurupgKcUVtfZqiRrp+EfO0Lf6T7lglN3Vw2AyhJTPTeyxcvGa5rvQ4Ki7LwWlawoFIRuDruRYUz2kaKzAwsOqmRtfZd7w008tfrpYNRT3XSpZy84Hn+A08bRorjzhlClKRc96kJk+yjTQ6+11+ugLnHzDiiVKuS89DqfsxI33zeNqlyrl2qUCsrBV3RkTCRAA0nTy2+FkrFig5croDSkhRbSITMd8an3kH3dLML/T1h1Pd/cl4qlYAHcg7f4tI+P32Q8QekLhmpoV0ibhvUKUQoq+zKQARuM2mw+gbQkuWlTjc0EXSNfz6G+NPDhaAEpVftEMqp701TR1Ee74nxt6wMJ5+M28c6LHt3444zYBcoKOrp26W/6GUv5cypqU9CdITHxt7GsZcpVbOlBAF1ZjIcSQuyB42DcEpInxHKLCzmGp5dbF1qMajQiIJs8e6g9J7iSVGNfH8LAOJBG2/Kxh3LMDTfSNrAOEak69elrQVQ7qNOLfE+4OESrjxNiNgPUj1S9Rgh1DZSpTRUDKyBsjzibN67/Tw4OUSQHglyQIBvFjnruCZ62lq8uH+C+IfZ3ZjfCt1X9RUpNS1T3lSIqGkOgQFhKwQFZVKExsTYSk9GvgK132+DOCG5Vp/2DTb+9FrUTHkXabID3svYhRU7+0I4NNpJQil1MDNeiBpO+iDYNP7Rbg9EBN366Qb0Vvrpox4WnKn9H7gix7HCDBSY0kXDS/8Aya2VKfgtwfZA7HhVhFPlcVKP+ixCJb6u+ihePsq6OftH+EDQIH7oEfz3q6PI6UxsD/8AtIeFawotIuIpAkk3s/Edf/Nfda0LXCzhk3/dcOsMJ6kXRTif+Sx1rh/gNnVvBVwpj+W7WB/02YiU+99E4MY6KqCv2jPDZKQvJh9IO5/etR4xr6rHKzj4f+muzxcv0YR4eYVuu+L1FOuqLDV4vJAaRuorWwlI1O0ybWWZwfhNpX2eGLpTpHdomh9wsi4jwhhylxJhy+21UlzqpXKppT7LbTWYLZIykqTESBp1AtHly7lykJWDQNTGTjjjvUt9rT8GbvUkrKQTfTfIkHcDmLdN4w4/TrwhutsJ3m92z/1ibSHdd4YLw/d7Fy/2xpHgyFFLtZeja3lqUoqJUoqEmVfdbl/F+AKdwofxhcaY3z3myNf+O07G33lC/wD5UyaXF3HkEKXwzutCZ5VaF6/+2FlZOKuNb4BGB7qR1JUhQA6f+cWVXcf8L24S5xCwyga+1e7EwPHP4j42KPcUODTa0qVxNwgmCZCr4pjP/P4WC6IO3efoiNkI91Bf2j42FCAMJ3SkjkW0Hl/+NWxV78altFX7huNtYEjO2I+VSbAu8ZeCCQUucU8GJI3/AO1qckf83nYmvjlwBYUnPxXwek6b3owQf+aw/Dt/GfUKfNP4QkhGL+MFTie8sLqo8PsVN3U9NVuFwBKHG3y6E5D25JgsrnQRp7gKuh4u1qyX6vC5QRBAfiOXUxZvtcd/R8oOMt8349xHw16nXYbu5j1hp4ONqcaqas5AUgjMA7MfpZyPelP6NDMuHifcnKQlp1R+SPqbBNKxw9p59UcVD27MHok0YX4kAJCq/DAIVIArCIA/3TYC8sFcSq8govbDDQmSTWqVJPP+68vjZRd9Lv0aGu7/AKS6GYPsUdQTp5N2Lq9MX0aU5QniE24Vf93d1UR1/wC7sH7Mp73Lj6ov2hPawaPRIqOH3FJDhJv7DQVGXuvk6dP7n8LKFFw84oBsoOJLhaB1MOFQA6asW096Z/o4NwP7Z1C9/YuqpP8A0WJu+mx6OiFjs8SXm4BAhN1Pa+RI8LI4bSu3J9UhX1LdgPRavXhpxCTUN0q8d3Il6pQtbaEoClLyiVR9lrAPum1frrvX0o3kmsu/gdip1twdol5D93ZXkkSFJzjY6EaDe063h6a3o3VgT67UXnWBsktpcucrCVbGMx00tN+EKmkvPCl13tQAopaykZqGUqTkKW1oCkSBtoQI5WYYZBf2CfVLx8x+8PoqVof9L1SwtPBPFTCgqZVXXOOXiOW1hmbx440jrtVxewdeOHrop2Hqltyrrbve7VaAJSBTDMISSSTA0G86XecbSrXMpUa76C1RfTyxem5cEXrQU9QoKFAmhTG4cqVQv/3WvutzXFtK1mGmmiJzzOZEP/e4A/8AbmK4r9IWIzTYG7D2GzqlzIR/+xwDv+zMV5eY3xHUYkxFetbVSpddVOvugz7S1EmIkHWfGRuOZHD1ahbfZVCEEtaOd7vGO7r0mR1jUxYtedI8+/6sX5fQlOUEElevtJOxnSI1IjyslXQt1m+VoqELQoIWrKtRTKt9dOh5/EW7eNghaGMFgNB8F0scTYmCNgsALAeQTzQIeSsKUCVQUkDtClXP36EToIUdxpPnEpQ/8nHh09TkQiqW4goAnMA7BTHjrPSTvavDrzIpjmiFN5UpTGaJBkEnmMvKBqDziwvEIh/0auHTqEZy7UKWOozdodNvKBvtbmuIz/TcN/xv4ci4riwf8wwr/MfwpFYDhrjN7GvDS5cY0aGKu9FUiGatTgHdfQIcJCTp3hmidAoa87F6u8L0qL7ol3nTsBbweZyoQAkjIogaDokx1ga8rMn0Rr39bwFUXYupDqaevqGc4VqCT2iYIAmUrmff5SRiKqaumiaavGuaVWuvoYpVKRmKlnROYaGEjWZ2t6FwDUOwp9Rw1KdIrSw+cMhJsP8ADkzM+Bas/hcfY1dU8PO2jPMi/wAJ5JsP7j7t+GVFHGKamZbySt99RcKVqUkZc0jflqnltHUAVk9LG52/X7qxMhpWR4PUq80pBJAWmSTGkHw020taahFa7StmsTSl0U4S680UwpYT/lTA3I0GnO0KelHdAr8AuVqJz3e63VoWgEkAKhQ6+ys9J66adxi9OJ6CQeV/TVd011pBZR7ienOOfQyu+tIS7V4FvbsXFDU9gpRSE6cgioa/9n4GzV9BzAz2N/STwlTBSQ1dTyr4fKlRLbCSqABvKyjSz29GJpvF2GuI/Cd57tP37dCnqbNshwJU3nBO5lbJ/wBwcrNT0C75aw96UeF2bwUtk1aauhAIiHVsLASR5j4xb55wK9JVV+Hfhkzj+7KM30dmXN8HWosSxLCTsybmN/uzAP0+Dw8L1W4lXY3X3RTZ0JIbqkq7wKgJSRtz1jTb52a1z4QS5ToWhQbggkq0HPQJGp2FlnjHxFufhvw7vfHl+3fW1t33Mlp99qkQkvFKnUoBSFKA0KwTJGgPO1cqH9pLwFpkpUcHY0Sc0d2kpjy/+zjW3PcQ4XX1VTzKVhIIXvuCYpS0tJy5nAOurI0mEr79cpmadVKLumXStKs88sqRoeevlZ6UNwoUlLTYISDKjGk+Uxap7f7UPgMyoIdwhjYciBSU0nz+36WUqX9qn6NbISlWEceKUNSPUqWB/wC/tzDuGsYlPtRGytz8QQEey8K31FcDZUkJUY0k5d7Q36d/DqtxL6M1/Ku+kLz12VFJXpCBJSlDoStXQAIWok9AZsDw1/aReiviliqqK6pxBcfqyM6G7xu9MvkbpT2K1wf82UGd7Rzx8/agcLL6wJiPA+DcP36yq9buqKJupXSsqSc6Sk5pc7oIJEwSJ621sK4empZGyyXzg6AC5/0CwKnEZKk5fd6kledOGBddzX/dVRfjTi6WnrAqoaakEpKgo5VHnAMbkZrfQLhDF39rsN3TfWDrrdFDedExVs1VUMv2biApJidTBHhb5yL2xZT3i7/q7K2ldoFyQIjWNB5n4m3qjwc/azejzgzhBgzCWJ7kxh++LkuKiu2uNJdzK2VPMMpbWpCi8CUkpkSAddrd9U0csjmyEkd7C5+XZYD5GFuVov8AP+bq/wD/AGVbq3RUX9WuVyhqEK0bB8EjSyqgUNA0G6ZlDaRoAkRaiD37Yv0ZCk5MP49P/wCTaf8A/WLJFV+2G9HB2cuGcff/AJvpf/1i1JzJKW/hKcl3ci59ShZXSWEjxbtfRX7qr3QhJhVkarvokHvbWoY/+129HdxPdwxjz30FL/8ArFk5/wDa0+j67onDGOf/APBpf/09ufrYcaqSTynLQp4qVn3nhXrqr3kE57JNVe5J0Ufdajbv7V3gEs//AMr43350dN/+nsWc/apcBFEkYWxtqOdJTf8A6e3PzYFjcv8A0XfRbEM1AzeQK7VTXqWDJMc7efX7Vm4zeGH8D4gbQCaZ2vpFKB/mDK0j/kWY8+lnKv8Aam8BjthfGp/+46bT/wB/aEPSt9Nbg7x34fUuGrgw9iZmvpa9NU25WU7CEZOzWhQlLqjMqTy5G17h/A8WosSiqJYSGjc/EKddV0MtI+NkgvZU6wffbmHMW3DiJlUO3VeNPVpO0Ft1Kx91vedtxVU2hbIUoPJCkZeYO1vAAuNQcpWFZ5SIER59dreh91ftK8F0eD7luOsu/E7lbTXZTU1Z6rTM0zBeQ0lK+8Hi64nMFGStE9OVup4ywWoxYQmnbci9/nZZ3DmIxUXMbKQL23XpdgorRc5pniA4y6pJTm7wnvQeh12Nl4/O1VvQS9InC3Ha78YU2HaKqo3LmfpH3mX6ZDX9+HACkpWor/udSoz8otQSeXPayoKaSkp2QyizgLItVIyaVz4zcFBvoS80to/xAizUdBStSCfZ0PKzsUTEdNrNy9mizWKKRoYUI623aI6kLMqhpmRYrkyfjFszbHQ+/wDCwZUN8u1snqZtqALNJujtMo9ikAnuyj3AxYWecanSLE6dYOdIUSArbnBSD982MBRj2rWW6hVzoUKSdtZjpbAoHX77cBXKR7rbChOxsQaJkIFTAOvhzt0gkQEnb77AkkwAY0t0lQGx132tIKCGCuug67W5qEh+ncZUAQtBT8RbkKGo6ja3RVztJJc0jva0bKjJJQJ841sZoT9uN4HKyfd8BlyngfZOrSB4TP42P0RAd1k2YJuqcIBgarGn8ptlgJ/xx7x+dss2qdeCWLHqi6rmqKqkcCX2EpUFFIIkqCSIGgmT4TOx2edLwX4jvsIUnGd1JKkBYikOmbl7P3eO0mzTx8067hqsbSiSstkD/fTt4/n42ufw24XYq4jLNLhS5365ymZDjqWkpCUJjckkATrGvWLPAxj7l6BUPfHbIq1o4I8SAoH+2d1Zd9KTQGPLpz/OxxngPxEczNnGV2FQH/oR3+H1NrEV2Ha+5L0cuq9adTNTTuFp5DqcqkqG8jr5i0/cG/Rybx/cy76crxTtJGROZObMSNhHIafjY74IWDMdlWbUSuNl56K4F8SlJDCMYUDjmTVKKBRM6aDWfjYKr4JcSKU9nU4wo2luIlAVdpB8xMaGI93wufxc4Yv8Pr6eu9VRJbIHaJnbeNNenxtEFeVurPaKW4tScozGVAcgdZ+/eztp43ahLxEg0KgRXBnH6gkJx9Qxly6XeNd5389rdDgzjp0pac4h0KShQibsSRpr/Ntv9SLTWTrJSBIjTWdOXxsCoQkBCTKhqJjx1tPwsaXiX31UbJ9GXiY99mribd57Tci7BJHx00/Dxt216MfEBWZbvFOkg7kXTIkjeSrpOvztP9wVHrt0tPdqStqGlA7kgCPiLKzTSEqAUARHegnUjT372ovYGmy1YwHNBVbT6LOM1KlXFynBkkhNzDT/AJxpt9CybffoqYmpaNVW9xWbdMpSpP7lCTBVy+0nc/O1o1JzJUHFAbSCN5kbTZJxcSbjffWUjKtqQkgz9okGPHTewXizSQUdrWk2sqS4Pw1V4T47XHcFTevryrvxBdcOoaDRWFONuEFInqBudvK3trT6tgkEjz33t411bmT0maNtKgZxDdKiU6RqyNx0JAt7K06crITrv+Ns+Yk2KnGAHEBdEDWFGdrF3Rv4bWNmP5Trz2sWeJIiedox7p5NUnvjqfOxRYH0OVjzqQrVQkzYo4BmMk6nna61UnbpBxHj9vh1T097PXS7XetPilCG3EoKSUKXmJPLufOyL/5ULQVlODKg5VHT15I5x/JZK47qSMNXYoKyReiRPPVh60LlYQoykEdCZ+uf0LaVOLsWfUOIdorBJ9KrIYTgN3T+a8gDPubt0r0tHEo7mAwfBV5wNuUNWrwXQAYVGUQes/nr91t55UQTI2OkiPhHT4WMWBV+a4Kwg9LS8TqMA06Z2m8lff2VgleltfaicmB7vASYJN4rMf8Au7QAVGSgwSDAAA1Hv93jbRcy5ijXQCZG/wBffZhGE/NcVP3/AJWeJCPssGXbmJ29ccP/AE62jPjrxRxDx4uK6uHF43XQ3VRXheIW7UUzq1vJ7NpawAFaaqTB8LM5DqjCgkASIkCAI3tpt4C/bjWFjuVsQQebTg/H5+6zujBFikJHA3UeXn6N9yXS+lly/rwd7T2VBLYA5aSnoBYJv0fsMEBJve80iDoFNa6bexvaasbAB+mKz7aTlPUTIizadS6QlNMop111gTpYLqeIGwCK2olIvdR2jgBg1IPaXhei0lUd59AJJj/Dp/X3n6b0fcAuJUVPXooySo+tAAAeQ2+t7OpXrCNFE9yCYnnH6WU7sbqVqLocJZSTIBJ5dP0snQRgbJ2zyE7pns+j1w+ToBeUE6j1uOcRtrzEWFPo/cOwCVUlYRuQatR5eFn22+JHdAB0nbUTYwHhJSo5tYPXw/pYfKj7Iwlk7qKxwZwJS4zcpPUaxdIbiRU9gurWUl0VK2yrcH2SNJjawGI+F+AqGgNTRXIUlCtvWXTmHT2uWnjzs/6mP7Y04CVAvYeqkak69nWMHTpo58xYliBtL92VKSiMqZ18Nfdt91k2JhadEzpX3Gqr7UXZh2hximhq6FKKI0q3cqnVgJInnmmdBZHXQ01fT3nXXW5/q9BnUlWY5lqEQBJ0SALK/EEJcvO77sZUguXjUIZIkz2aTKvvAiyddKEsU2MKEhz7NSyU7SnIdo18Pj4WBywTZHDiBe64vxNExhC7r2p6cJdW42HFEkyDvOvT42UMXXbSMXTTVN2UqUEPtuLyndEd6SDsANff1Fm/X1Ad4QUjmUZ0JaIlWWD2g1nlpy8RYbHF+NDCVLT9qe0faaWUoUM2VIE6cv10s5Y0BIPJS9Q0NJelfSXk5QJaoFVKEMpBMOpPMp6E689rSAx6RPGy6se1+C6TiTfDF0XfTpapKZhSEpZSkhAAUEgwAIif4bM96tbauWhW1GnYqCkgZR3DGvPntr8hYjRXfT37xgxcVdol2gfUGw24JKVKVJOsEaA+7zmuXtj1RGgvNgnxjL0geOFDW3IKTi5iRlmoquyfSmtIzDMkEbTzO2tivpYYovZ/DeGMPXxedTX3jUoN5Vz1S6XXVKSkIRmKjJHfcAnbKBZZufhXS4rrburFtVDtNRVLa8/aBKEpBlQ5TsNB4crRf6WlfU/6SagVaVopqS6qfsgsR2gzL9iYkSpYPLbnpbjqutp8V4ipKON4PJD5HC40Ngxo+IzE/VcBitRDifFdDhjJGu8OJJntBBs6wjYCOhGdzrHW2uygW9WbmZYFZeTrocHeATEue1oJ13nXpBnqhXU+aquXUKDjbKGyEJMkJBVpvuAfu99hU3bX30+XnnA0ypQCUZ5AJPw2AJ8culuKBz1e/wBxpa0rIZCTlIOWNNCdPERtp0i3d3XeAJUrSEJU6txtGaCMnLmeeu5kk6gmfa0sTjVDTXov8Ou0CC2h52RKSkf3vM6adZHhrANcKh/OjMpwZScpmdIAA1JgzGw/U2Ex9WN0vo0cL0mAHqp5nORGVf2sHaR3hGgnW3K8RH+m4df+2/hyLiuLB/zDCv8AMfwpEc9F6/2qF+96FdQplx+ubqELUvMAC0nck97Nkka7nQATawOP7uN7Juu/6JRUhDyErCElZCwsQQBPMETvoItWX0cK+hpb0v5h2mbfdaYZdZW23KQsqVMa92cqToNhB1Gtp8HJQu5zdNes9tWS+lDh9lzQkSDM6AxuINuwxzNQUlJxLELmkcRJbcwPIEg88hyyDtlPdVONGOwuSn4iiGtO6z7dYX2D/jl0eO2UpOfevlbgRTILbTOeQFQVyUgqnfaN4jMN50j/AI10dbVYHrKGsp0JdqqV1RKgoiRsduZOggQI23tK1G5e1CBSXgpHYIcWe2Qr2ZkbAafxzqdxtJFoP46VWKMZKrLruapXRU9MlSStEhQEagRAEkmVawOlvVat7TSF7PazDS3W666N7ZLOadDrdQZ6MmJ14d4u4brXXVhqsqF3Y+CZgPJyJBM6DtFIV576nRy4cwFeFyenDduG7oYU061i5m8mcsx6qtQqYB6JbKgfAEWg3D7tddN7JaoMxvCkqU9l2aCpfaoV3SANzJ89NPC/jGCcVvek7w24zXRhyLtqaFFLfgecS0uiUpCwFKQohWYJdCdjBbgxpb5vxaeHCOIop5nBrZ43MNyB7THBzd+4c4Dz0XLV9ZT4JxbTVNRI1jKmJ0ZJIAzRuDmXv3DnNHnYdlaziNw6Z4q4Fvvh69WmiRf9IaP1kNB3sSVAhYQSAqCAYJ6WgnDf7IO6Hyl1/ipWutHUqVc7aUgTMSXD8ptbDD9WaO96V5tNPmQ4I7cw0TMDN0Gu/LeLS0vC1/4gV2mLMSpRSf8AoN3S22of4lnvK+Qs+JPqWytELjbsLAfEk7fJesxPY1vtWHmbk/IKheOP2ZvAulwyvDtw44v68sW50paNFTMuIAnvBSD7Ok69oNTtysh4S/Yws3igVmLeLtZd6XIPq7F2NrcAG0qK4GnITb0su2iw9hymFLdFExToT/IkSfEnnbHr8SAe+LVI6/wRLqia/wD5W7ep1/JRkkMwyxM+Z39BoPqqDO/sa+GbKJpONWKWnIAzKoadQ05RpZs4j/Y5YMpLorKym44X67UMU7i2m1XUyErUlJKQYX1099vQ2pvw8leVkevvcLbUhS5BBB1tl1fFOQ3iNkWGimksHbL5xMR4fcw+/wBitwu5ytJOXLCh/WffZBJJ31tKPHK6nrsxtfd1uMFv93XtW0snScrhH/TPvHui429Eo5TNC153IWXUsEchaFgJHXW2tSZmzgwvgDG2N1uNYMwpe1/OsDM81dtG5UuNJkAKWlsEpBJgEiCQY2s52fRv4/1AJZ4J46UEgqURh+qgAbknJoPGxXTxsNnOAPxCG2F7xcNJCjePH5WyPqLPlPA3jMq7zeo4UYv9SBCfWP3JU9nmOgGbJFjd2ejxx3voFV08HcZVSRrmbuSoIjzyWY1MLdS8eoUvDSn3D6FR3P1NtTaVP/JV9JD/APsdjb/8yvj/AKbaV6LHpGp34IY1H/5Gf/8Als3jab+0b6hN4SoPuH0KiybZJtKafRZ9IxQ//opjEaTrdLw+9NkHGnBfilw6pKWux3ga97hp65a26ZyuplNB1aU5ilM7mLOyrp3uyteCfiEzqWdjczmED4FM4lHZpIBKjIUSNB5Wtd6LfoYYa9IThvUY4r+INddFRS3q9dq6RmhQ6nuNNLCsxWDqHdo5WqcBJga6Wvz+zRxalvDuN8KOuJHYVdHeDSSdTnQtCz/7tv42zsemnpqJ0tObOFvzWlgUMNTWNinFwbq0PoR+jDdPo341xC5dGOKy+WsQXc204w/SJaSlbTmZKgUqMmFrERztcRQAHS0C8N74/wDp3d7ZX3XS40T5tq08dQLT0sgm3N4ZVzVkZknN3XXQYlSRUcoZCLNsuTtp8bJd/U6i2h5P8JynyspkTpPxsHWJD1ItBGsSLbVO7K4FZEozNITSBBGvw5C3Q1ACTrtbpbeVeUjf4b6i3BBHMnzttA3WOu2FJTUK00W2D8Cf/msaBGpI33sRSYqWlzqoKRr4if8ApsaCiTI3GugtYj1QXbrtawEKVEwDEDXysRN+NhvtG6KrPeCQnJ3pPhPh9SLd1DdYtSDS1KUJ0zJKQSo+Z20nW3DDF6Ay9eYWAf4WgJHv99rAt1UFr9+jYXdWmdNG/Px+PjYdN6KJUEUb6wlIUkxAWDG089djFgU0t49koG8j2pJKVBsEBOkCOZ038bYKW8SUk3opKYGYBAlRAG3Tr018rTGVQ1CUKSqVUM9qqnWyZKcqxqIO/wCNjGbnm87JCKO8jqq+VEERHZgCYieszrZTSRoCdQNfE2Y+ScLhgdnV1KTI7RKXBPvB+4WUKKe1kjbUzZNUoIr2Cf8AaJWjptB/OynQwHdQNdpsyY7pYJcn+P8A4rZbvKv+Y/A/lbLJK68IsdFLdzuuAJlbzeVIEd4r59N+Rtdvg7xsxXwkW5W4ZVSlNWyhuoZqGs7bgGxMEEETyPPxtR3H6ouzKUyTUtpMpG2bby8wecbwbb4HpG66/Ltup99TdNWVDNHULBEhClgEwR47dRaNPYNcXbKtVuyAO7XTgxFja9cYYnrsTX462uurnu1eypyoHSAOQA+Wtpa4T+kZfOALvcummTT1FO4QYdEwqNxqLQjjtl268WXrdFRVCoXdVW9QIUAZyMuFI8hp87JArFtL1Ue7vuZ62vgNljHZUWv94KROJPEeux5eT153lUpcdeUTmToB8LR1UqK8yisK0Ob+LMJ1+Ou9uhULdBTmAUZJg6gbR9/ztyumdySUgxA8+WotJoDdAkdURX9mVBa0lJ1JmZ1+fPzsC4gJC+8mSDmnw5mfvsO8ypkKGcApOgkgzGu+362Kr7EgGPaVlgiYAPTfaetpqVku4OrS1eDlE4MyKgFQMmM41HxmPfZ8inKgtbYPdCiIEgaSPPW0UtPLYqEvtuFK0rCx5gjr5fK0rXZUoraFmub9h1tKiPZAJ3Hxtm1rcpzBalC7MC1a7BRILghMQCDHlHwsj4kZUbmqe0AUlKm1GdiM4PPx+FnI6krSY7oTJ1O2ug+unKyLiUpbuKtKkKOVvMAP8wn4a++2e9/sFabGjMFTu8uy/wDKUo3UJIIxBdiRpI0UztGh1nrp0t7KM+yJOs28ar2Qtj0k6NbzaErXfl1E5SASD2JnT/i0+6bezLDZUN5g6eFqb9gU7RZ7visUBuQB42LuAScunXSxxTaQASIJ66TYu6I2A71njUZLIg+n+I2JrGsk77a2UHkgjXl1Firo0g8+X177XGKk7dRbx4ROFaJSN0Xk2Qf/ALU6OXnaC1JURKnCJ5xNrA8a6U1GEGxI7tc0rUf4Vjn52gv1U5DkgwY06T9C2lTuGVZ9QLuRIocT7TpJBkgc9DpbISEkrVsSoGPlY0KIqXlCuUgxsOZB+On9bCG7jmCxlhUdND8bHLwq4YSk5adQcx6yBvbeVRGZSjrymR52PeouKOUKKwAZ0k6eHvFlIYKxS5qjDF6EKBhQo3No8um1m5gG6YtyhIAACgQ4JHjodRpEddLbyxel0EqVpeDQ1j+KUx87OFvBeJiYVh+8EBIjvMqSNtN48bA1eHay76m6TX0LlOXbwpsgXoSA6gKjrvZ5A9rcxGiaJzHuyg6pTxmw285TOZlEBC4gwOR1/pZAfpKJhsmofLYXMyqCVDl9fnaQscUVJToo8w1yr3O57vOZ2+tbMOqoqOuUGnNUoMwDGsRYGbM66shmVtkj57s7UhNQjumCUrMak9Nx+Nlm7HrqUwW3KmX3Fd1sEgE/UWT1XHQJWmVabyVdBZXum6LoUg1UKztnMiVT0j687SkNwmYDdD/u5pSS4QoHmJIt2m7aRXdKTJAJGbKPHSxhtxpCACoDSYjnvNtrdAUTnISAdCdbVjmVgFoSK1dtM9xFwtSPyGKqmvGlXlJBylylUQCecAm1ij6NXDutp3qd6pvmFgoVneQAeR2RNq9LqUMY5whVp07Orr25UDzp0r/+GLXqpyheVaVOyqQvugz5T+FnYTchEDWnVV+HoW8Bq+vYr6q770q6yncllxVc6kN9T3U7xauquBuCLm9NW8OErt2rcwxfNHTVHYesLzrCqdwmVyFT2iOR6dbehaSyisRTtuuAqUDBIHPa1OOL7tPc/wC0RwRVqWEG87lomQB/ErtX0RI5x1/DRP0F1K2qkVXoaejlT3aLjYwOV0KNFMuXjUrQdZgAODmLL9F6Hfo21FOht/hfdbyVd1fbuPurgbBJK5A6pmDp0tJSKZKVuA0/ajMZKkFR362UqNaGK1DTSW0NpVuMoH32fKFMryh4nXPdVxYuxJh652ilq6LzqaNpmNEttPqQhMTIhI3PX4ti7q9u4+L+K7yOin6FqoIM94qZbO/+8Dz1NpQ9IjsU8f8AGd0BkKcF6PuFaGynMVkL0Hkvf87Q1jyq/dXEl1+Gkt11zUAJzaLhlsAH3o59fG2BO/NmarcLQwhytFgC9v33cNzLul1tCKVbiLwZ2UkwqDHQnX3+Bit/p0Yqu1D2HcKUte27fNK28/XFsAKSyvL2SVEbGUqVHLf+IWVrrvSrux5FRd1Y6ytKknOgkTzA09/xsk+lxQ3HVXXhrjKzdgeN6Mi7agpbnK8kKUgEnQHRxMmf7sW8zw7AG4RxRT1cj7skMmXSxDyHGzj1uHOttsAdl4hNwY3hnjelxOSXNFO6bLoARI4Pfle7XMCHvynQnK0G9tax0N1KU1+8L5rFN0raQ5lS5CtZ0EbfUAzYncRFbejzjI7M1CwlCdVEJKhp1nbX87Er3vu8ryV2dQrs2pzBoaJ6T47flFj2FmGV9tU1RKWGxlUcpO+2wM6A6G3s41K9iSxXJQ6IbXmKpUoBUkSpUHcEiNdYmfERP3Ehr/8Ahd4XhJV3alxRICVQIeJ30PTnMxGtq+VVQqtdW3TH7NRhKQgqBSNJA3946z5WRxxTKd9GHholwqDiXlqBMKJJ7Tr5yfvG45biL+u4d/jfw5FxPFn/AIhhP+Y/hSpsejc/TNYzvOnWjszUUDTrZLiVJUpLhBUTsdVaD2o6zpZOlvBK8S0TzWdLV3EnMgHvaQdjtqdDoSd9BFROFbibuxd2dOunQamlJR2gEEFSCCmP4oBiB1Pdta/AOWsAdqwG6ZgnPmglRAMgSdUmSI13G+9vZuHYIq6ifSTtzMdcOB2IIsQfiCukxGKOoidDKLtcCCD1BFiPmFJ16vUNRTMVjpR6tUBDgWU7eMgSDEfAdLRNiC+6V4vUdPSJeccWsBWUwPE6aGOomRZ/3WzU4lwvU0LiFUqHMrtFkBSUtwITyInXx79mHjqpuDhjhty8rwHaVbg7GlpgUla15QO6DEaAd4xAGu2mJwXWVFLSzcP1Zu+hdy7nd0R1hd82ez8Wlcfwa90NNLglS68tK7J8YyLxO+bfZPm0qrWGHcO8OPSbu2oxG+hN2U15JqKhxZ+zYW62ShRn+FDi0q/3Z5Wvfg2nxjeV+3qcRlh272XO0w69d7xUa1t5xavtgDENt5EpATqYVmOtvLjF171974pvCvvRSPWH3SV5EjLI5A89onbTTS1t/wBm3xAqWOLVVgq8r2ecpbxuR5NHTuOqyIfZWhz7NJMAlHaTA/h8hbyzjPhJvEFaapjw32Sw5m5rAk+03UWdqdTcbG1wFHivgEcU1TKnmNachjOZmezTf2me03K8ZiLm42NrtC9HaNl9mgp6aoc+2QyhCyf54En4/daVbtxeKy6qOpQ6ftmEOa+KQbRoRkAIkGQQfHTTxixPDN9LRd/qS3daapqacDbuoeWlPySLYHGDX0lLFyidNPlbr6L27hzD45W+HGoaABfU6aaqWjfub/ae+bFnb5gQF8utmexeSyJKoHW3aax5ySkFQt5pJWSkbrqxhbGbpxOXqpW6rFHq1S8xKrN2/sSXDhSjFfiu/KK6acgkLqXggqjklPtKPgAbRDX+lzgOsvoYX4fUL193k4hwtvVU01NmQhSo2KySEmAQmbEgpKus1Y027piyKLQbrzv9MW6P3RxdxvSCR2d+PVSRl27Y5xrvs4LVlWO9B369bWK9JfGN5Y+xteeMb6o6Wnfvxpmp7GnRlSMqQgCDKgYQeZk5vC1eH05XlonZR3Nve8DD20TGv3AC4HFLeIcQrJegpxivbhHxHvpy7LpavBd83MumCFgnKpDqFhUZhOgUNZ32t6m8PMV444lYKue/K1276RqvbcS6gtBxTDrThQqEwEHNooSkkZok6W8ZvR1deTxcuSmYe7NdUXqfNEjvNL5c9Y99vVz0eK+8Lku1+5ai9HaikQs1DSFpGVtawAvJAkA5E6TGnibcxxfSNzc9o9rS66Dh28sYaBexO6m5nCFztvprb2cfvSpTGVysc7QI/wAqfZT7hZVXVMsJDbCEoSBACRAsjuXpKcxXt8rEKi80+1nJ8uduBLnuK6plIXavSs/eGaSkjXnYk9X+0M38PSyDUXwhJAJPe7p8/r77ElX2guBAKpzQeR1tJsTyrbIGNSw9Xqk6kkdLVG/aLUf724PXNeCUqWq7sQ05VGkIcaeQdvEptYupvoRqgnwAn42gr0vEfv7gLidkAlyiTT1yIOiexfbWfI5c1tvBQ6Ctif5j6qpi0bZKKRo7LyrbADyUqE94AzazXoA3+q7OLd6XSpwhF6XK6jKP4lodbUPlntWu82+xvCob5dooiI2JkbabRaU/RVvs3Hx7w3UhWVNQ6/TERoe0ZWlI+JTb1PEo+dRyN7gry/DJDDWRu816nYavX1DFd0VinIDdcwVGRokuAHfwP4WtqqQNiPfahzl+uNLStB76YUlSd9NZ+h91r0XTXJva56Ovb1RV0zT6dOSkhQ+RtxeENLA5pXZ4wQ4tcELqZ3i3Khpt5zYQju7R0sFqSDrvrbdasJyb1Yz2L7jZToDIsWiCRvERZUv1spcQ4BIKYskZxESTJm2zCczAVkTNyuKxYCMigZ+0QAZ6mPuNjSgd9ZixN5eancCdwkkRyIEj52HKgrvDZWok9bWmKq5dxoCQPhbBO8HW3BIUZBiB02thJ1ixmlRQgIB1nwm25kbGPGwcwTbJUB3iR0PS0humQ4UZ6RbpJ0gGwE6Em3QIG3XpaYUUHWOlvsHz/s3k5ttAdD99lqiSQ6FTz1si18rpHUD2spjwIsq3a4Hg06nZYCvORZuqYpfHZQJWqf8ANbLByvkPkbZZ1FeEeKWVveo0yYUHLxpkhR3GZxPTzP3WtJSOs3XetM8xULcFI+h3MEhMhJkEfC1Yr5Qly97kbAR9rfVIDykF0ctjy362sgFMdpm7RRKZVMTIH3/XKzUoDmuBQaxuYBpTp4kXtduIMe3/AH9h9xbtDeFYuqaUUFJ74zqBB1kKK/gTZt5ihGaCEzMHy/Ow1M9TMqGZKCkqHeCdTH0PvsA4oB9QYEiQUHQQnpr7/j8bkY5bQwdFQbHlbYJx4coqN+uaFW4lLZIzZjoR9/Lb9bWH498MuE+EMC3PeGEr4bfvKpCT2KX0rLqCgkuwPZggDyJjY2qwzXhtRWhWVQI72sjaJ5+FjdRe9ZUIQhThI2AJ9/XSfw+LOYXOBvsiNIAIIRGsKStSER7p1E+A+psSWlUyQYOx2Gh/Wxlx4rylKU9Z6nbl9crAuwpSjAIjYjUco+t/dYyZFnCvP3VRymRp4+Nn9gK8e2oHaB1RPYOZkJBggK6a9fvsxVFKgc0QnYBX19GythW8UUF8tqdWEsugsuEzoFbH3ED4WBUs5jCrNK/JILqTVJW4MxgA6QDpI+t7IWK32v3BXqdbKoZWSEpOkafrZ0NtEr7yJM6gbAg/d9+nWwV7YZuy9bpr1Vd01FYUsrccyVa2ApGXVJgjTxGtuflkDGm66WOEucLKkeKW22fSPoRmBIve6SrcRqwdj93gbezFLq2CQPOd7eQHFd27x6UVGLpudu6mRed0AUyXnXgDLWYlTiioqgeWoFvYGlT9kB5672BmzMafJCILZHDzXakwcpPysWeQY7sgnqbHI0jUxYB5JIkwJ52nFuhyJMeaKRmA8TYkrODOpG/3WUnUqIIg6fX5WKLb6aWus2VJ26jzjGP/AKGvEJBUmoaUATp7UafG0ErcUASCCoRyIki1kse4er8S3A7c93ZO3eWgt51ZR3VAnXloLRy1wJxw9P2l3JSdAC+owPKPOx4ZABYoTqeSTVouo1YdfHaRl7oHdKDPtb2w51ALXBy/4Qfr8rSkx6PmM571bdiSR/3qzt/u2MI9HTFpzBd73agEiYU4Z/5dbH5je6bwk34VEzailSVJUBE94JEabfA2sXT0eK7wxQmloKaaJdIX4FIsoIyjvF+SkOZz7EezrZpN+jbf5JLt/wBCmdYDSjH3eFrGXc0uhu+ipHCpamqZtslKiBKUgE2E+NtQ8OvshyUrw20gsoRvPC+NA+Wl01QluZJ7GZBJ0Bgg/HnZh8UqSrpFYddvCidYc9f0S6lQJAep+o21NrbNuZ9UtLI65yLRvx4wbQ4korsvCreqGlXUl19sIIgqCm1QqZ5otsVGImam8OW/NZtHgogqec11/JQjxTSllukWE54zyIJIJEx8rRmqr1KUojUTuN/rpayWM+H12XwinFXU1KAnvHIoCdDpt4mzTRwOwh3HXL4qWiqVBLtSlJG4mMs8jbNjyMbeQgfFaM0T26qGXFJUkOAmRrrqJkQPnbhFU6lASTAGsxr9a/faaKng1hpC2Vs16nGc/wBuV1CjKYjSE77WOXXwVwzedR6nR0rr6z3ge3cQInXUgAifPU20GUjpGcwfd7qmJGl2QOF1BhqH+8oqSTOojQ+GttCsfAyOLGXWCTEDz+FrGH0baNUj90tIBgDNXqPvibDNejhdqEZTc1IszrNasxrvvaAhZsXBG5L+6rNV1BF7YWdUoS1fK069HKGpGo5eza99BVXo+yl2mZpgChBSVqVqCkGYjx+VqwcQ8EXHhu/8J01NQtMlGJ6Zt9KVkgy282d/88e+0+XbUcQahNGu7aG5VUCmkBKn33AsN5fayp0J0GlqEjMshAKutjIYLpXqHsXu1QWn1FKAZSSXTpPgPO1Q/SxqXbi9MvghfblGgP1iKCldcElOVNcrNH/HrPh1ta2sPExC226ReHWm0pK1HsHzpOuhGp8J1tD/AB1wsm+8XYQxPje7blr7yu13/sWsZZebNOoOoUSUKO85TrPOwJGBjCS71RYIi51hqp4qHeyqX28ogq2IBnnz87GrtYZcmqqciUBcA5RKj7hZov3BxScKFoxfciVnVwG7wok6aiVDSxyloeJDbCqWtxxdy0rbLbaWLqQFtuK2WIcgwdwdNTptaLqhzTown0/1RTCLfeCoP6XNGml9JjE9UihdSh9NO52ikgZgqnRIE8iRaBeMVPUm+LhqKNtRaNzMud5UQpDjqJzdSBAj+noBxt4fsKxBUV2IqGivW9OxaK65dA2lxYDfdBGu0Rv0tXddxUldcd2uP3dSVCewcYX6w2Cs5XnDAKhMSrT9bc8ZbzvB7rTbRnI0g7hQnfd4VF1YbrLzoint2mCpmNUhZAAPjEzpv8rG+DC744o8H8bcLMQVLdRe9Is3vdKlJC1Ek5ikD/7ImCdD9udtLSbxjwy03wrvJLNDT0yhTsPF5psFTbYWhSjoJHdkeGvQ2gPg5fN6YK4uYerWmn00TrpoKpCGz9qh7u6gakJUUqAHNIG5E0sZoHVmEyPp/wBrGRIz+8z2h62I+a4f9ImGzVmEvNP+2htLH/fjOYeoBb8CoCqTQ11U85T3WuEjMVqWQNNzB8jp4WP3A0hyhWhTBKkAkgAmCDEkHxk/G07ekPwSxLhLEt+4muHB9RUYfqHzUIqKUJUhoO99eZKdW0oXpmVAiNelfsPViQ8EuzBUqQnUiSCddydB8Oe1uhwnFKXGKVlXSvDg4A6EGxIvY22I6jdFwXGqPH6JldRPDmuAJsQS0kA5TbYi+oOqcdNQ+su9o8AhtEgI7wGmuoOk6+Hxi1i+JmVXo4cPFSCkPuEa5QO67HOYjTQ6+Vq9MVjhWgdqlskArAVCtSIMjbXYQJhQk7WsDxHS0fRs4ctnKlJqVhI8YdiI8AdtOW1sriP+u4b/AI38ORc9xXriGFf5j+FIov4bXDdN74urn78qkMMMMNraUl/IrOVqBHMnRMayII91orgxhhGjSKKmPrbBCQvMSG1I1J1JkySN94IE2pLf95Lu+86EJvCrp0vNrJcp0hx0JMCAFkaEpnePa3JJsbaxJWrWhJxPiB0N5XlfaNMZO9H8BOmv8yY11109Iw3iA4VFy2Mue67F1MJjdxXoFcOOrqWjsWG0ZmSpCWUQXYzwhA57SBv52i/jhgGj/tJUYqxXju57ouh9KfVRXocWUwBnSlCYCjJJEEGCOcWhm6r+p8RUWJLtu66buu+vosPu3pddZS1t5rfVUNOtmE9rVPNKORS9ShOo0IIEvG78P396Rfo5LuGuoqitxjgyrKKF15xKF1bak9z7VUI1QYOupZTJEzbh+IOK2UeNQY+5lmW5M1zYFrjeN57ZH3BPZxXGcUUreGMTpeIc2WJ36mY9A1xvG830Aa/Qk9HqEOJmHeHrzN4Xthi/ryra6mUghTlEmmp3EDUqSgqUuImCQJ001sj8C+In+i7ivhjG5Sot3VebDr6wVT6tJQ+AnY5m1q3E6DxlFxdw24g8Pnl3fiu4q27HSkKLa9UqT/MFJlChuJBPMdbNZpYSdYtsS1kOIgTw2yu6tNwfgV2lNNFNG2aF4c06ggggjyI0K9+mnWalCaqleDjb6Q40tKiQtJGhHgQf0sgXZd1ULyvVQOSnarlK7RRyoAUhDhknQarO5tFHoScSVcSfR2w87V1AersPZrgrFZiTLGXsifNhbM9SDaW63AzuJb2VU1l9VDVAgp/1ZtRHfCRJk7fDnbg+LKdstHZ5tYrt+G5jHUkt6hDXhjrCGHwlp+rcr30aBmkTImP5laR4ibR9xS4k8Ujg6svvCd1OYeoqEBTj4bzOLQpQSO+rUCVA90A6WmC4sG4Tw+AaK62S6Ne0cTnVPmdrbxb+770uWrumupWn6apaLbjK0hSVAjaD4285iFPTuGRl/iuvcHTGxXnVfT194tcdevm9K286105it1alE6+Ovxnxt1hjBV7XRedPfaSGjRKCwBIhKtDz5A2l687gornvJ6npqRDSW3CkFKI2Onw6fpZKrqdQuyqaBCVeruACNZyk6f1t20Abk9kaLJkZldqquekzgv8Asfe120jbqnk1l2F1sq653JHuJ+dq2VeU1CynadO9m+drn+lpTovHDmBr9KStTtK6y4oTy7MgE8tVG1NK5spdIiNIPmNPwt12DPzU4uuQxlgZUGyc3CC8v3TxSwpX5sobvelSogTKVOBKhHkTb1pwDUepKWUggqAzRtuP1+FvG+7a12gvGlrGllK6Z5DqCORSoEfO3r3hivbdaYfpyrK43nAncEAi2bxLHna3zWxwtJYvHaylpN+HsQqNDCYmLAVD9atj1oMOqZABKiDHx2szquqvl67X3Lh7FyuSgrabdGZCyJISRPMDyn4WR7xvvHFBeLiqWoaqqP8A29XUFlpSGwtKnuyTnhaOx9kkf7UE+zFuF8KL2C7nmWCd9VeQTJBEgQTG499iP70yLCkHUKzQDz8f1tEP9peIdVTuI7ZposOqbqKhx6mDdO2HjKnCJKHAjJIIiFKOUEAFIexTxDW329Ne12diGm1OFyopSkKVlKgghWwyuATrC5kxpbbSWCCZwpmvGsyvuAZYDigPAAnr9a2Y/FGmF+8PsT3ItIV67dFUykE/xqZVHzI+otrD95X1U0S6i93i8tb6jTvEtkuMEJKVfZHJ/ERI5AExYd+pBQ406UlKgcytzt56ixoWct4I6FAnfzIy3uF5OXumKoOf962hcxEmNfPWdbLnDm9E4d4h4Xvxz+7pbypKhU6d1LozfcbEsXXau6b0eu10ELon36VUjctuqFkrO4E07wJGQlIM7EGf+q3qFhJHbuF5RrHNfsV6h1NUEKcSVJ31gieuo5c41teLgvepvfhbhqtzBZFAlgmZnsiWzrz9m3njceIP31h67b4/hraJipOsznQFcucn4/C13fRRvr968KGaYrJN23jU0vuJDo/++24ekBZMWldvWu5kLXKYluAnb42CBSFaD426WFBRHM2DKt9x8rbDVjORW92w9TK5lAkRZtqIg7mZ0NnUsZklJ2IM2bD6OyfUiNZtpUr7jKs+qb7y4GU7gRO0+62qcqFM0FaqCcp8SNDPwtoEp0TpH1+FsZnItGndWY9/eH320GLOduhkue/x52zMVAmdOnO3BBSMusb72TcU3hX3Xhu9rzupppyspKF9+mS6CUKdQhSkpUAQYJA5ix2tubBQJtqlfVXI6HnrbCZ1B28bUmwN6aPE3FeMrhw5XUOG6SmvW8qWjfebo3iptDrqUKUnM7EjNz6a6WDxn6ZnFPDuJ72uW66W43aOlq3WqV6pul9h15lKyErUguSkkDY9DttbSGFz5suipfaEOXMrvIUQQNfut3mJ1BG1ou9HribenFbhrS4rvpukbvD1qopagUyClsKQvuwCTHcUmdTrPlaTAvXNO1qkjDE4sduFZY8SNDm7FDEAggnT8LGcNkqpmkkGWiW4jbKY/CxMKGwnmDNh8Od2rq2JByuhwf7yR+INh9U5TrAbj2le5X62y2Zhz18YtlnUV4WXirtMRYcEqyuX1SAp8M8+/wDobWrw/wAM8aYow1e2L7nw8p65ribLlfWLfabbZGXNErUM6o1yoBVqkRqBaqgK3cYYYTII/fNOTm2IBVt+lrCi+LwNEbt9eqU0K3A8umS8QytYEJWUbFXKSJ8d7PSA5TZCqiLi6MN1bSVjtc5EwMqoIP1FjFRW3YWx2LOhEyskyY3+P42SCoHXMowYPIwdLDMpQ6U5qgIhRAlOkQNeh/S10hVGnohQ62cyVKGglKY5n6++2krKiZgGIT193j9dbGaihuxulW8xeqXHApICQNVAgmQfD8T5WIFEZQCFdoNgQTvpZB10ntyp7UHC/GN74CquJV2XL6zcVA6tqtqGXm1KagCc7WbOlPeBzRABOwszHXlqBKzKSdiZB5SPrlYzR1TjdMqk9efYpnyO2S2uUqCTIzJBhUROu1k5TkLXmQCIJOsTpz6WZtxe6WhXeYyVEkGZMHePP63sEhwFebKCUzJG5PI/XjbFqzZp1KiMvd0P4RNtIy5RHeEH6npaR1SsRqp2ws6L1uWlvQd7tWkpXKZOYaK/PU2W667alNzVctrWQwoFJ2Ij68rMngdeCahVbcbxTnSBUs5jqobLA/5T8bTsu4H3rrfcEhRYUoTACiAem/6W4fFZ/Cvcwrv8Ji8TE14Xm5xjYfT6TtJ6z3lpvC6iUxOX+6Ovy93lNvYSkA7JI5RbyW9I+kTT+le2IVCq251KKiDBIamIGomdPE+/1rpRmbA30+ViQvzwRuHZZ1SzJUSDzQigSOcROnKxd1IMgjaxowRBFi7oB5R77WY1UkRB5O55/OxZYIgkT56WOvDTbflFijiASR/W1wGwVN261Qsh68WG1AQpUfKzk/dSVGECOe9krDrDTt9UyXgpYJVEQDOUxZ9IpGNBkJWTERanNUcl9lq0VuXqm4i6ZgTqNd7DC7skqzgQIs4HruS05lKTMSRG1uTQpUdEGwjWm+quAsOoTfF3p705FTzINjXZJ+zSAruJjSNvebKvqKpASm0W44cxXT4zoLruW4Kiro6p0IqKlJcSlkQmdQQlManXeI3sSCukz5Y25ifkqGINZy8ykRCEhQKlJRGupR+E2bnEpLVRcDgSpK1JZdmFAx3fC0c4nwpjhaVKuhFY4pUCO0VG/IZidrGsJXJiWipr3YxDdVQy36mA066k99Xekakgna3VzUQFAaovANvu9VzFFib5K5tMYiBffonLfbGalplluc4B28LAP4Uvi96C637opXljtMq1tqaTkBWe+ouDvJTGqRqc1l2oYTUXbRlYCpQg6iY7lnNhlKE4dYClaBxSQfCbYDclWBG47arfq4sseqYeIeHGIXF56J1agEqADbug5g5ZB++2sFYZxNcN7lV6XY4ikUyEioK0nv5xAMGbSolLQBWELV4wfwtxeqJu1PZApKXW1Rt/EJ3t0bMSfHTeGAGVc23Com1AnBN010vLK1BdQFZdR3//AMK3DLwNbmNU2G0CSM40+diT9FUmsdRT3dnQJynORI31m3TFGtuoKnbtQkCO97X6WoFxK1QFAnGt7tcUshTaQKTFF1vJ0GmapbRv/wDbPnad8MOPpuG7n0NJIFMlBKiNx/Tl0tCHGGnNTiy8qRAVnbrbpqYjQRW0pnx/itNeF0l246OnWFQy46lesQUOKTEeY+VsunqxLUuivqP3LTqqfJTtf8EqVb7rjQL5aZAOU/abn4Wif0gW2DdWGK5sKc7G8iC4k6JJA0nxg/C0uXilostpU+4k6qhJ/IWjTjq32GAKaqKFuCmvNleZUyJkD3WfFr+EeR0Q8N/rDVIFWg5m31L/AL1IIMTyG+lu2FIMBTilK0AhBj7rAU1aVUFG+h0ntaZpYT2YVEpGxscarKlcEIVHUJI1tajN2AqtILOIUdcZbozXq04n2X6ZCVBZnWSNfnpatWD8Piuu9VAXGqcM3lXNKWZKUgOEyQI1hXvJNrk43uxN5ppX10qzkRl2kyCDraAcH4Acpr9xFc9cxUHJertQ0tKZ7i0oUNjGsRrzt55iVcKevmaei7KhjEtPEma5hWrvS56qhVSMOtKo3AppzKlLqMhBQoL01GkHrFqS+jzW3XdXFS63r+eQxSIDrNIXf7tDi0ENmSIGwSCf4j1NvTqqwFTrCm1sVSyUzogTHiZ3E28qcW4RrMM8Qb8wOzS1Tj1BetRd7bAaUpxag6UoCUgakyIAEmRvpNvDpIsepKmgLi3O0tuNxcEXXO8bYKMTw6Shc4tErXMuNxmFrq07V78SKTFFZevEekui5sE3NdlW3V1Rq0uN3hKkZXFNk9yEpVoQPbUnWRbzdp10i7xeVSM9nTqdUprNMoQVd0FQ8Dr5WspjXgD6T984bSxefDfHtVd9K12zdNU0lSQmB/AysE6ToAmekxasbNO6xUO09UHWHWllC21oyKSoaEEcjMgiP0scKcNDh8yvL2kvDRZgLWgNvrq5xLjmNzew0A0C8x4M4Kk4T50kpbeQNFmNLW2Zm1s5ziXOLiSb2GgAACfF3rShpPdkOgrCSrNKVaAEbkwdCRt7on/iXWN03o0cOn6pXc9ZWhzQwQe1GwM/jpaul2VFRWNopWM1Q48pKENJBWpR0gJA1M6ba6jnra0PEzB+JmvRrwVSVlyVyHrpWXrxb7JXaU7ZC9VjdO4mRAkzG4nxPPFFW4aHuAPO6n/03j8yB8SFDi+ohhxHCWyOAPiOpA/6Ug/MgfEgdVW+nuy6MT1jL9TT1FWltsNDKsIKiJJnpJVJ0BJkzuC+6Th1gUFqpXdwQgFK8i3JJTIOumx169JEA2bWEaWlbUCloN5syy0CmAkQdQZ7u2h6DmYtIzVEqvp3EtVDjK1IISlImVwRHd0IknX8hPSvdqvRY2i109KzE2JmmBclTibEVRRJRl9UevqrdpMsD/YrcKCNAfZjwEgGYuD94uVeDqGludym9bp651N7JX7QBSvKsJ0JE9lGolIPlaDlULFRmcXSrcUrvlBqCCJ70HWZ9kERPdPvd+Bru/s/e9JexSKZfat6NLVl7IKGYEneQJ3GhG5meL4rwI47h7oIzlc32hpcEgEWI+enY2PRcn+kTho8V4E+kY/K5hztuLgkNcLEXG99OzgDrayK+lVdTdfg3DuELzvJFdfZvOoqA9k+0TQguEBW/IsJMnvZFHUg2pvjvB68OVIdQJbWYIy5QOhGuoJCtfC1wOPdW5dHESuauq71V163hSsONgzlYagJKieUlKtdtOcRaCMScLr7c9YvG9r0Ljr6At1pKRlTpBg+/TTaNtBa3wPA2mwSAtdmzjOdLAF/tEAdACbKj+jvD20nDNK9j83MHMOlgDIc5aANAGk2+V+tlLf7MniYq4uJV+8Mqx//AFXFFB61SIUY/wBcppVA/wAzKnZP+BNvStqqNMrKkkAjMNIE8/wt4lcA8WK4cceMHYlXUBtu7b9p0VC507BTgbd16FtS7e1dc092iQy2teUkKgTERaxxRFnonn4fmvVOG5A2saD5/klBV4mO6ZJM78+tm9iG9CGlJUdBqDO/X8bCNOlbpppGYH2QZVPl+dm9jNL9FkLiFp7VBEqTrv8AqLeZQRBzgF6US1g0UW4uaAvN11JILnfI5AHn8LNB15jO4ytxGVxJGpj3ff7z42bfpXXdfV8YWu1y42K2pearULW3StrWSktKkkJ6QPAT42rAvB2NH1hK8M32pQSpIUaJ0g6+IO/5aW7qgo2yQhznWXIYjWuhmLGsupj4/MtXjwRw7Wp75u68l05KSTCCFjXXUd0DXmLUtvdC26lQcJKgpSSTuYMWuFV3XeA9GC87svCkdp6m6L2zhtxKkrALid0nb2zoBztUfETQTWPEJgZgr2SIkbEe4fE228H9jMwdCsDF7vc2Q9QEjpICpJJ2i3qdwhvIXjw8wpX9rmVUXPRlRJ3PYpn5280MKYExZjNm8XML3MuvF1sGoq8i0JLbcHvQognY7Tytfz0Yr8RW8EcKuLUpSmWXWFFX8PZvLT/4UpEbxZseaHQgjoVY4deWTOB6hT7dtcyglDlQGApMoeyTlIB38IJ+VmffeA8K1ZXTfvKpqWXHHlqaSlKWkl1banEgFJJBS2luNAEEgDaDqrwV2ikDLuobgg+fz1smv1rjiSnPImOk7mNdh+nhbjxHY3C7YzaWXNz3HQ4eU8pmqfe7ZtLSe3KSEpDji4kDXvPLOs6EDkZRjhG5qZxl2mqaxJp2SygZ0mAphlkkgpIMoZb3kSPdY+5VIMJKtoGaQT8fhZLq8RXRSFZqrwpwGQkLBcAgqMCR0MwPE2KGO6ILpWhKVGpq6btpbtplOLapGEsoLkTkEgSEgDTTYRYN6sWoEhIHWR3iZjrNmdiTiXdNyD1dCautqWniwptlEkLKAqRMaQSbJVwcRFXxfrly1F3uUyStwNOFeYKU2e8CCNCBr77SEDrZrIDqlt8t1Tnj5daLs4m4mYbBy/vV2oB5ZX0h0D/mPxtHoVNGEa9xyR4Zh/8Agi04elhdPqfEKprQjWvoaSrKhO6StonXl3E/EWg1rMW3UjUABR18Y/G3fUT89Ox3kF55XNyVLx5q73BG91Xlwow4+V5lN0xp1Cf+6cW2B8Ei14fQrvjtrrxVci16sVNNVpE/zoUkkf8As028+fRdq/XuGBYLhBoLxeaAmIBShe/Id82ux6HNf6pxDvW7SdK+6VrA5EtuoIPuClW5eZoirHDzXUxO5lE0+QVxVknSPDbnYJRAJAOniLdqOgCj4bW5IkSdwINr42Wc5BnTbYWQr4ayv54AChNl0iP4fLSybfDRcYDkwUGbWqZ2V6rTtzNSIFajbe2gSh9YP8WVX4fhbYJGsRp8rcunK8lUq7yTr8P1trMKyHBdlZ8/u91tOIQ62ppYCkOJKVA8weVgySZIJ+FhEnNoTrt1sdpsbqB1Xmpge63cK43xrd120bNRiS4aOtbuVt5oORUsVKUuKbQoQp0U6X1J3IIkahNli/KV/jpcN68QP3TRXNfVFXZS7UXw4ae8EKSt11ts1byghbOqwhBCciinKCEgufivX4PwX6QWJKTE1wU9O16wi8qO8qVl9VX2z3ZuzIeSkQVOwUgapSJmTYniLjFwyxp2DmK1XzeRpmillupoQ8G1FRKiCt06K7kkJkkEknS3Xte52WQDoFzrmNbdpKlf0DL6D2EcTYdUcwo69mrbE6ZXW8vwlq1pknWCTam/on45uSo4vXth7D93UtJddfdRdpymjSw8442WyQ5lJBjM5Hha4qVCO6YHKP62wsSGWoJtutWiN4QB0QwPMCLD3OtSb8Lc916nBHmlUf8AVYqDuCJ03I6W7p1lu9aB7NCe0U0rwzJMfNItRVtPArIJAbVp0A/O2W6IP8w/4bZaVlBeFtHP9r8LNpVOa821STMJykyffafA2vswEd7MZKhHnaHuGtzU2I+MeCLhqyQzV3oc/ZqgkJbWTB35a8vPa1jMa4MvDBVcikXUIq6V8qWy6o5VECNFCd9h46eVg0tRG13KJ1KnUUsj2c0DQJrFKkmcmknlqDykfXzt0hKlEoKSQNJHMa8/qLdI7VMEjca/4vnPW25e5BJB7s66jX8PG2ks0tIXa6SsaU02psBT7YWgJcBgHYmDoecefW2/3fVqcQySMyoIKRO+3LU6+duAt6MwUQBqQOXTy/Hxt0p9+QrtDAA1Jkx9flaCXRdOULTTqgpZW2jWUJJI/CfysrUGFqd+nqa1u/LsijYRU+rPOONuvpUSChvukFaeYkaERO1koPPZMpd1BgidI6fW827zOZc/aDvaaDr5+E/fZrlSYBui/qrq3DlaUmFT7Omtu2rvfWqexX3leUnwnz/rYTO6D/eQB3TrsOp+fxFjVO48Fd1xUkaxpETr0+jaLn2CK1gJTq4X0F9UGMbqvGmu951KKhKXQkgZm1DKr4gn77X1peHTy6QhVOCVpgQmI91qa8ILsfqr4ZcfvlynDbiVKyjRQBBg/l9D0ppOzVTNKQBBQCPhbkcUoRitTkLrWC6Slq5cNpxy/eK8aPTTwo9h30vW1Ip1MIcduVwKnNnPdEkctlf8NvTWnT9mE9ByHK1Gf2ldTRo9KDCrTYAeFBdmeAASfWnSPPQAfDpa9VOmGgY3s8EQgiEP4dPRM+QzHmO3Oq7I00M6jnYB0K5jc2MkAJ0sXd0VIM+drcaBJsiTqZHLpYqoDkZnlY48CQYPwsWVpykdIi1obKk7dLWC6JdRf9MAkkJzKPlFpPZuSmYcD7aPtB7JKiQPjZgcOnUt3+G51caWn7j+FpSsPw0cri526mZXMGVpSebtKlZlLBJ5xbf7sT/N8rH7ashQQ9lHnP7pOVdQzZgsizbv2jcp7wSgHRYmfD6FntZn4ySt6rQwhwtkpT307jWxqeljhfmamdK54s5EUMPmSF/8xsDfLDCrsU1UvoQFd0KUYEnQffYZlKRqF5v83L5WQOIjyWsLvPqJytutK0A3zDrYlZd0DwD0RKUgStPmjdZQ09HRs0rriB2SEpUZ/wANhbpve6Ke620KveiZCXCSFupGnvsycUYuSjDf7wIkuNsn2huUjb42g/8Ata7eFWqk7NSW1lUEKnbl91svDI3h2clXK+YFuRW6oL7uW83zT0F7UVU6gZ1IbdSogTEmBpqRZEv/AIlcP6RmooH8X3M3V0rqEvMqq0JUhSXAFAgxqNZG+lqzcJuKDtzYuvZb9E02zT0rjK3c5lR7RtQifAa/hzq3xtxUmv4m4upkoczC+bxaWAEnMPWFxGunO1iTEHNqHQW0AvdV2Ut4hIvSVXFXh+lrtjj3DqUOZigmtbToOepslOcd+ELZKFcUsNKV0brULmJmMvlbyHvSg7aoJQREgnQx8OZiYn87K2F1u0TrbSWWyl0nKXANNI7pP1rZpMSe0XACdlM1xsVd288c3Xjf0h8SC58QIvO4/wCzrVVTqaI7EOsOsLVGkk906/mLTvcuOsPXbelfhyovkN1jV93gyGyy4oQp9a0gkJyjRQ3NqEcHa5268WXqUENn+z95pMd4KHq6laEHXUD4/G+PDSuFSziq8W1gqexC66DodFsskffbJoXSOry9thcFadU1vhAHa2Wrx48cOqSodbVfF7PFtICBTXHVuSfAhuOdo19IHjRhS9uF1Vdd33fipdY7UUz6FPXHUU7YTnT7S1pCRII3PO0+BbQcDvaLDmqvaB1syvSEu5F48DcTLb7RTrbTb4UqVDMh9CtZMAaeQt0c0b5InNcdLdljxSCOQOaiOHeIt4MYdutit4XY6VUUtCyy8TdraE50oAJlTg5iy9SY+vSpQlTPC7FEFYB7RVG3kB0ky/pZ0sKdqKSnUFEoUyhXdJ1lIOuhsKGCtktFgqSSJCl7+6LRax4AAck6RpNyE78EIF53A3W3ldDtG84pYVT1Km1rbAMQShSkmQJ0POxS57suR3Gd+/6o0XXAw97MT9mEz4+zYvS38q7WPVpykoCgmR3dI/C0f4axg7RcTK9TrxKKi72gEk6ZkqXr12Atw+L4pTRVrYZGg5TdxIGv83WtSUU74nyMNgRp+f7lNyriukyTRt66nS1aqzgTgW4PSdvzisi4mXL1r6eiNKpZlNO4WnEOvNpI0WsNpSSNYBiMypmtWO0RoR03tHfFHGLdCmnxNSt5altCqYvZk5UgkKTmB9o6LCQOavcZx41h0srY6ZuUnqBa/kj0VDUskvMLj+dUTxfftdiHDF+0+DQam/KDtqdllpxtKk1EfZpJdKUTqnchOu9qwek36B98cWuEtXxFoOFibv4s0VC1VVLlJWU6Bey0QXkOttrU2twpzZVCFFQSCojS1k7tvpjEV2sM3JU1aq9Thq2Sh1sIqWVwVsIUonKoKlaZBG+onSWbqxHXLuSlReNC4KzskhxDrqNVAakEEzy5W6KF8UV3E/z2TYg2QgNA8rH814F+jRW3LdnG/D5xUGmGm33WkGsCU9nUFtSWZB1CgsgDoog6Wve21iSiZYqcSXrQm7mF167yLqAEKpipZaBJMQE5OQgAgzahHpQ35S3v6SHEi9ab1ZDVTievXlpkKS1mDxkwomVFQUTyzFREDQTBjWsva8/Ro4ahVdUumodX2wW6o9sGw4UpVJkgZRE7AdBbguOuHvtavo52yZRKRGbtvYZXuzDXQ2J+Yaei+c/0mcHOxnGaF4lDec7km7c2WzZHZmm+hsT8w030TCoKK7BXVK2HUt3eXlqZbSv2UyoiY09nyPdJ6w76aqoqWlXVKUlDQQlkpJgulQBGg31JjSN/ez7mp1LyjtF5MwS6lRSEpRsN+U6g9DoNdT6a5V44gTdbCVrYokdstG/fBImY2Gux5DqZ9KaMrQ1e5RsEbQwdE9aI1CENPNkNqaQ0nIAEhsgd4TGwJPgSNNIsbxBeKG7vWq8L2NDRJ71dXBcKSlIHdbk6So6aDSDrMhstOXs00sXZclVeC3FrQy20zICgVbq/gHwSJJCgSBZoXkau/L+9Vv4IxLeras9Lh27lqXTUwn26lxIIMH+BJJ6xrZBnREc4KeuOK7/xCjC+KMMXlRXTdd7XU27UXs61mqAhQzNoQORIdOpGmblaD8ZC+7ow7eNBhK6kK7RDhdvGrdTUP1LepUpojQaJPlBjXW094wocRvcH8H1L4vNFdQqNHUUtzMMwdFAJIdIQlCeySJ220INo5dpXFJdvC9MP0tDQKKUupafVVuyBqXEspU2mYiSAJG825bhAmHC20/WJ8jD/AO17gPpZcP8Ao5GTAxRO3gkli/8AhI4D/tsqdds72/rGcl3Nnzc80zNvoF9GrE124qwzh7El8sMPJvzDlFeJ7RIWA6tlpaonbVareCuOruo7sxTX0920L1JRF0rp23TJCD0I3EzG/nb2B9FS/wB5r0buHd6NuLlq4GWM3gjM3Hl3fkLb+LE+Ec9guQLruqCPPOIibXNlcaiw3w3u29Xb7p7rp01TpJJUoqQknfKgnKn3AWiz0oau5V4fudy7mKdDgq3EKU22AQktnSR5CzdXjt8gw8d9p+voWY/E3Ejt8XM0047IZqQuSZGxH17utvOIq6ea0LmNAPYLt4sHMEgndISR3Kji8Hg7TtrcO6CDIg7nlZCfQUqJUgKy6DxE/gbKIdLtKUEKlt1cSNpgifeT9aWIPtlS0rVqnaE+PT6622IrgWR5TfVMDG1EmqwTxFuoJEinbrgIOpCM0/8AJahOI21JcUSVd5KViQOuutr+Ytvu4rlrb9u6/wCv9WF+4fdZZSW1rU6tOZBSAkHYKH0YtQa/0ZiQAcwRqdddR7uvyt0GDk3N1zGNWOWyk70SqC8L74iVWGbvr0Ubl50BRncHcVDrZKVeEE/ftae/RGqHBw8vK5FmFXTftTSwQAcsIV95Onnam2A8XKwTiAXyKP1pBYcYW12mQkLESFQYIMKB6i1pPQwxB+9V45SltxpLlezXoazZsva9rInn7CRMaxa5isV4HO+Cq4PLlqGt+KsrelcaG737wCO0DDanCkEyYTmI56frvzilziXfTrbriruTSPNt9oQTOziZ33BbWDO8zaVr0pzU3S42kCH2VIjwII08D52ZuCLhwfflKlN9Jb7d66aohQe7M9s3Tktk7Tq2Rl5kczbkC5sTS9wvZdFiVWaSPmdAoora3FF9Xa9XXrflYGGFoqMiRkAQt7s1IkD+ECf62T78p10lReqW3nFumoqaZ1a1mVJQhLrc9D3enys+MaMXXQvX3c+H3g9dtTTuOtDPIQ4lIWqFcxpP1NmffDhqbwcUCJrHaF9InRXbMFtUAfDbla/CQ9oKpRz81gf3RJvEpdvBy/qgFQbqaKqc01IDZQsCNJk/1s47mrGV48pw00ptTlauqR2ndIS6wk5TGsk/RmzZXdTd23PSXmy4tZraRVTlIByKbW0Snx1CoNnA9eAvy8nsQ3PT1ZQlNNUJBaIKlIWUKGkzoQfLlys8g00RmE31TE9MSjCryuOtbby+sUVVTqUAYJacS4P/ABq+Nqwse0pMTKFWt76WVPT1eGrirQsLdZr1skZu9kdaVOnPVCdfC1RKUoTVILo7uaD5bW6TCXXpWhc5jDctUSrJ+iFeJVd+J7pKiSh2mqG0Sf4gtKj/AMqbXU9Gq9jdvGa4gpWVFUl+kcBIk5mVZR8Up+NvP70VL1XQ44vK7xEVt2r0I3UhaCNfLNa33C/FKqTGWFsRJp36VDV6UtQ40+32bjaO2SFJUJMaTOvvi2NijCyrzfBbWGPD6MN+K9LTI0032m3AWNo91u1pEkWDMmddD8rHbsqzgtEwrSdT4WAqG+1aWjqLCn7zBm3KtdI3HWxozYoThdNZaci4+M2LPrhbKiR/eRv1B/SyjeLXZ1BnXMZ3iLJlfKacrE9whWx0gz+FtmM3AKx5RlKGCoiTOvO2ZpP5GwecHY+G1thYIkA76mxwgpoYs4OcMMc3sb8xbg2hvK8EtpZ7d7NmKE7DQgaT9aWT2/R64INGBwzuKd+/T5vvJs/yZ5nrtzt1mBMgg9BYwmkAsHGygY2E3ITXw7wr4b4SvBu98NYIue7K1pKkofp6ZKHEpIhQBGuo0s7s4kE7k2BTMkA7bW6CuRmPwsnFz9XFIMDdkNmhUEak7j61t0+vsaY1MH/V1IeB8EqBPyBsEk69eQBA/Kw7DQqm3aYqMOIUmBzkRZdEk9hCgCEjXXcWyyLQ31Smipy64gr7JGYkwZgTItlldRXj96PlOa30jMCIIUUpqKxxJOnsUjx08dLWn9IJvs1XXKlJKlu6pMTt0tVb0dL3uu4fSJwle19PpZp2heGZQ1ICqVaQqOkqkm1qPSHvWkq6u5WqBXrFM6y88KpsgtakCJB0MCfIjytiMa7x7DbSy2Q5ow94vrf/AEUPpC/ZQIIMiBvPX4TbAkZiFEqIgHT6+/mLcn2gRqmCddv00tpbi0qbQlsZTBUvMNIG2+2sfG3S5lzbgTsu0gqJJKiAREzAjX7/AK3l2YE4YY34jVL1Jg65HK92mCVOpS4hsoSZ1OdQnY7dbNRrMVJJJnLJJHMA6cun1rabvRt4wXLwqxBVv39Q1D9HX0ym3HmFJztqHs9xUBQMnnp4zarVzSRx5ohc/wA3VeXOwXaE38V+jxxUwTci8QYkwuqiu9jV1xb7RjXbKlZOp+4b2jpTSm5JM6wdYMGfr+lrVcffSdwhxGwa5hu4aGuQ8taVhbyk5dCJkAnqfhaqzzrmVAbLamypWckwTtET7+fusGlqHygl4t2UoHOkLtNOi5SFkglSeUEjc/h9eNjdIUthJMAzIEjX8T52IHvJJkkRp9c9PvNt9u6xXU7Xao7BSV9oT7QgCPASJ38LWHG6vRgjdSPh2/F3U2FNqGZSTEbyIA+voej2E8Ts1OGrrqFrkuUbKyZ6oBt5QNYlpkP9mVkIKAoKC9zMbe75i19MKX5e6LhuarReTIu5FzNIcpewlxb2VGVwOZtAAFDLGs725DH5pKLLLEbHVdFh1MMQaYz0VF/2it9euemRdqUOEIapLmRy3DqlDfzOvU+63o9SkltMjcW8o/TfvFd4elqw+pKVDPdCM6lRslBj5/G3q5Sf3SDMyLFpZDNTxyO3IuhSx8mV0XY2Q2w06WLuA6zNjPKEjaxdyZ9m12NAk2RR3TcTrr42Jr31A8NfxsddAiSqDYksiTvta23ZU3JawY/2GJKI5oCllOvOUkWl31jxtC2HXezvyi1g9smdfr6NpUD6dy58LU6ioMD7DqiRx8wXSqalI0zC2jUp/mFod4nUnGKuvZj/AEf39d9FQ5fte1AC55zKTPh91mexhL0lX1g1PEu7kCQcqEwSJGkhHSfjbKdjMjXEBqP4TS6siakD+L52J1ztPLbq221KUsJkpBMedoKoMA8eDVqevDimwtiD9knMN9tcvKyfeHDDi00W3a3jJUOozBHZgLAzK0zaEbE/W4E/GJ3N+5ZO2lubWVg1P0gRoy2P90Wr1je+aipvG9KF2sdLSahwJaKjlACjGm1j2F+HmNrmv1i9724nVtcw0ClVMEKyq1GveUd4gyDudLMfGi6ypvq+RR0NQ4rt1wcioJncEcom06SvfUFwl007qw2mMVnW6pIxnju67swwzd14lxoJaaClmMqdBqddBpvaLK+9vUnjWU6CoJBUFCIUkjkrpHh03ss48wkvE1z09LUXQ9WtVTSaaqQttRC57pB023mbMS/8H4woj+4LpTQM0yG2m6UKqSy1TtIbCMhE5gcxABnWOU6WqPEY2ksJ1ViXDTNZwSKzxGar7/vmlupxMM0j4qwClRzoRmhQBOUxrB10Fon4v19Cni9ialFZT+s1F6Vz6GSsZ+zLyipUdB9bGzqcwLiZoXrWUFwMMVPYLVXv095upDi+z72gTzSIhXh01K474UY0qbzZrHMJJqa5uqTUP1DdfVPKqEHVSc6mgIUSkkzukWA+ojfOXE3KuNoSyINJsFGF7XvR3ZTN1FRVJZb7RtBI1zKUYAEa7/Q5L9x4Yvh1ty90qWilSe1JUorOVIJVlSmSdEn4nSzpoOEuOb4cbu67OET9RUIcDq3TUK9hIJM5kgJAMRp087SbhXgff2EaO78YXhNI9RoccduwPLgJzIbAdJjPqvNAA/OE04d+rj1d2Vd1PHG3mPcLJucNsPV63afEbACrvvG66osqcQUO9m7TOAEpOomQIIm11eA63XMH3jVOJzCpqqapOU69+kY8PC1Xroqr0rr2ZUpxHq61LSoBImCkwNyY+utrM+jLUn+xLgCoBorsdTmjUmlSNI/y/r0NhBcZwXi26qVRDoCAVKXdQkdo2RIkS4evQCzd4wVLKOCuMiypIbZuqoec7WQMoTJMnbQH4WcdG2/VvJSKd3fWTt8rFcc3GL64a4ou2qp1Npq7sqUhSRJRCDBEjcHWNrdY4+yfgsZv3hdE8NvuXlha4qykKlJqrrpXc6VaGWwdx52Uf3dWuTKla9STB+FkPhe/SpwFhvPeSVqZuqnbWVKAKilATJ1ETFngi87tI0r6cc9Xk/dNqrZowBdwRnRuvoE0uIl63xc92sKw7dbN43qadIbonKoMBxCXEJWvOQYypUVbbwOdopr8SopsXtOA95DRQpQMaaQPPX52lXHpuW8E0F80tTS1j9EX6ZtxpxLhQFhJWkEHSShM+VoDxksIvQXg0AFqbV3lK1Cu4BAEmNDJ8LeWY9Gx2LPG9/8AQLtsNuKFh7f6p3X5xOFz3O9ejjNQ+Gl99NOM69xsPeNN9LMy8uM9fetK/QU2EMRr7oIdFLKAdwcwmCIB948rIVRej5lSX8iErSciUkJEjaRrEjSIn4WI196vrYRDiUpUowhBgTpqdNSQBqZPwtnQ4a9sgkYbWVqWWQ/dNgn/AMK28D0+O2X70xEu5ampWagodIpmC43CpCnh3UrG4SUmRpIm1oKa97ovhlirum+KStpllaQ7SuB1BKZBGZMp0IIOuhtQpDmeoadd7sqGYqOkHYGT3t4iJ1tZ30a8rvDanIJWKStqmlJkgDMUr22OitrdxQzGRpjeFjV7cxEt/JeH/GSpF58XsbXiw28E1OI7zebS4kBQCqlw94DY6ienW1ir5LavRh4ZvFEBLrhSJM7u843jrE+FoF4t4XvRHF3GdE1SOqS1iO82pKQCIq3BAO2v42sVfDCbv9G3hu06tLfq7ijqBsC5PI6xrp0PlZcQuHicOA/tv4b15DxYP+Z4Vf8A/I/hSqNXKtu5qA1i1oKhK0hW4TuDrHXz1HhZLuCuF33YuuqWWy/WLcWolBQGxMgaRpAMT1POLI1/Xk9eTzVI04QiW1QYIMqiZ3/lmdeo684gqmaTD4clxJbZyyloBBMTGZPjB5zmk626UbrvCbI1cV14hxhRqvRVdU0tC7LJaoG3O0qGype6gcpEnL0A3I1FnzhrDOOKds3RcJXhijfUFOdj3nXCITqlBKG9D/EpapPeVBtF3CK7qq91paN7Xsw00FJCKGrU2reYhOsakmNYCjytNN1YNuWqWad++Knt2SA4t++lFYVr1WFEzpGXqOoFgNQb3F1NOGsL0i+Dzlz33UvXkLvrRVZ6xztFTKVAncADMdBp5bWZ1aMPUr63hcrVTVr77PYNqKWwU6HNPgf+GOVnpwzwwzT4VxJcdLfl4upvGlK0OmtU4ppcLBU2STl3RtzBmzcduKvoWkM01/1qnXXgS442hSkplXdlKR4GdfCdZ5TAm8mtxGk7Sh/ykY0/mCuK4Qd4XFMVoT0mEg+EsbHfmHfO6rL6SmHMS09XdOJL0UV0b6F0zesdmqSsCCZ1Bjp3LejfoyU7tD6KeCEKJSRcIfkiYBcW4Pkfu5Wj7gnhPB2Ka7EPDzGVyt4hu6/7vQ4+m8ZcMsLJlCiczZHaKIUkggpmbWSYw/dGHcIt4Xw/Qpobtu6g9SpKZBJDTSWyEpBOpgczM7nXW27L+sgMZ7FehRfq6hsg7hRxXkV1VQ1rlZUNqoXVuJSy+pLbxLakZXUjRaRJISdlAHpYnfNUt2mWjZIII0mNfH32DLylGAMk7BR+Xhufl4WKVi23Kdas8gpCoJ1A39x8LcEyOzl6K5+Zqblw3dT3Y1XUtGHVesVa6tzO6pyVuTmjMZSJGiRAHIb23et2UV4oZTeDKXEsvoqWwpROV1BBQoEbwdddLDtL+2dDcSttJgCNjAnnOp+ti9avtAnKlJgyMx0jf687Xm3JVN1gLKEvSSp3heeFqlptxSnDUskIRmJhIMDrqfikWqRiekU1UVAW0tDgeWlaV6GY5zry387eg1ZmYxThesCsrKKxxlZKxlhaCASZjfX32pl6Qt3KuniXiqibAyIvR14FEQrtO9IOunfnT46TbZwyW0nLXOYtBZvN81DBBBOvhayPoQ3h2ON7/u2P7+60vZY3LbqRHn3zat6pClAaakRaafREvI0PGaipASP3hRVNPATMnJ2gH/u7bVc3NTPHksbD3ZKph81fcGaZMqCgqRG/SdemtoCvO5r1prwrqNVXTUyqUpbo1OPJHbBTjiiI3GitvC0+t0z6mShDTkA6lKCZ+o6/laHMUYNqnLxxNV0l3Vq7wYcaqabK2uVgJQ4QgRCtcw067246KxOq7SrYJGhNJquuOjvYrfv9VShF3rWtlps99RZhRBPKNRt99iDd+YeerMPLqrprlmmYQhL7qkpISlzKlS0xqAY8NddbdvYOv1dJSxh68A49RUiioUjmiilxtxJOWBoUkj425vi68Q3ldV2KThq8mahukF3vdrTqSErQtteczEAwoC1toaFmBpboh6O+77vNAprru+iZpqW7X6gJ7POpskLSQmZ/jHTYx4WJ118XlfTzNCxXPMsVblGtAbVCU5m1AgQdO+nrzNlK6rixDdeIENoTStUzr1RSvuPvtpT2CnQvMAVDU5zH1Deu64b4oHkmoXQkNKYU2lVexAyuE5TK94Uo+U2cZbqZzBN/iDdjl4YTvq9LyafTeF3t0FY2HCZIKkpUSOsE/P3V0rEhiueSg+w6rKR4G1s+KLVy3u3fNbX3lS0rYud9tv8A1tolx9IVkEBRMGRHj01tVC9CDWKXoc4SqQZnQT85tu4U68ZCwcVbaQFP3gTWpu/i3dgBUG31P05gjZTa8vnrB91rgodcb+0CsriIVIOoUNZ23Gnjv1tRnB1eLpxxcF55oSzWUriiDyCkz+NrxELzRET7KiR3hPX6+VqWMNtI13kr+CvvE5vmvUG4L1RfVwXZfKCCivo2KpJB3DjYV+NjZ2mNeVmHwGvZN8cGsJVYUCW7tRSkgzqySz/8Oz6KzE766WhGbtCeT7xWirkB+ttEpP8AEPIna2iZ1OnMGLYFCd9POxmoZSXe7RypcQNt/KyHUI7ZlTYGqgRt1s669pCqRUAT1FmusBILZMC2nTO9myzaput0UYc7WnbWQSVoEgagaWGCgDqo2KUuZLZQRPZuLTHTvGPwsOVgGc0nXW11uypIUGRnJ5TrbpS+kA9OZsDm0kyfwtuSkASTAg/CzhJdg6cpmYHwt0lQ6iwQVGoIA87bnumdB4mbEamKMBciJGm4scu9Z7QHUSRysmhRPnY7d8l6UgEeJ5WkoojV3TWmqeLWTJ2isu206WyzrAaIkhRnnAtlmskvFLhypTvFK5s2eEsVp1UBJ7KJ89fn77TleNS4opStQIb9nNrGm3M/XhaBuGuY8S7tfblKRS1kEjX2AIPPr8PjNlY4tasyoGbSBO58ekkm0aUCyhO4hKN0ONrf7B9M9068jvOo3svIprrABU0tYmIJkj61s2LnDi6sIaUtSgD7I2T8dd7OFNLVFObIuSe6BrB0+vhZ53EOUqcAtRhti6RKUsoKoBG+h36cvC2ZrrkqTTJAzaa7bc+mg+Vi3qjpBAGgGpKZEDpbEUNQpSsyCDJ1iZ+eu3Kwc190YtHZGUP0Ce8WBCSZMg7dfqLBi8qRbKg1TtoAMhISIkRtOnMe6zn4e1OCLmvSpe4g4Tqr9u9dKtDLFNVqp1IfPsqlJEjQg+c66CzJprtQ67U1FAxWIQ5UKUlDz4cS2yQMiB3Zkd6STBEaCNQtm/WFtvmk5lhcLqqvRLSFOBtCANDoE6bfj+nWMuKeIHac0vZ1qqZDxU8h5p0ZlqT/AAjQiII1tIF80Ibu9ztkpcSpEZVSqRtMeBPztEONqdqrupVDfrqgLtedNOsaLSgNkpnc6wdOceFrBkBFgoZTa9kiU+L6y8Wmk/6wttvI4txEZ+yzRy6ny2PI29NMH3+p7Al2Hvpi6qaQfaTLSd99YPzt5J4NrKteI7vpO3aaaaSfaCil4JBUmYnQz8ufP05wnV1LuELscD2UOULHaoCZB+yToCAI1/C3LcR2dEwHuV0/DTf1j/kqVelnWio9Jlpwq7qqi6hBE/woB8tB8/K3r7RqAaQDB7vPrbx39KRxI9IVtwKSqKm7RBEpjKjXxgxb2DYcKWkhMeyDvzsams2mjHkqlUb1UnxR5SgASRFiqlBU978rcu1Cikjl1FiqnXEJKUq/T6i1ljtVUe26FdEiJ8gLEXCo+e9unaxaRChJHhvYDtA4JAI6z1tdYbqlILFGrrcyXpSLjUPI28xaQxUHmbRpRryV1MQrZ5J087KmJsTYwuy8U0twYMcvNlSAr1gPpSnNzETI99srEWudI0N7LRocojJPdO+pqO/E8rcMvysDMbRfeWMeMAWBTcMmyCndVYifhmmwNBizjS7VtJf4esU7R9tbi8uU+9UbWxjSTF2YW9VoCaMNt+5S2t5SUEZjtZLvJusqKI1DTSloZdbUopjQBY5WSLovXGz57G+8O0TTS9e3bqhKB0CQTmPwt1fjyxd5Qhwj7ZnQK3+0TatOSx2VyswMLtWo6l/tGwQev32OXHQtOpSVBJK8x8dSbMzEV731cF1PXpdt0v328pxttuiaKUdlJhSiYzHccj7hNo6pOM3HO7qyrN0cDa+82CsCnz1zbKUxoQMwBAJ6k/hbPq2uyBwItfuAjSg5S1u6mekwxRO0VSlSEnJX1EHp9so/jao9KE45RfF4X4WkupvCooFoaJbStLa0qQoaKgzztJF4cW/SHpqGpew9wmpX3al9blQxV3i2PVHSe82O+nMBoZHWzU4JYJxtRXbfFTxPw01dte7e7lTT07FQhxCm3EpKiShS/wCIRBPxsfAKaJ75Y7jmOtl1B2vf6LDxZ9bCxr2E5Rv+5Nq969OGbrq6Knw4pyhrU9nUOOKWvIMuUwUpHLzsr4SwcxxcxLd144vur1a7wx2KfV3nW1rZbQohUZyQSeZGsWm+lwxcN/1dJd18XUy5TuLCJKFDLJ5HL9TaS7lwNhe66tysu+6GGnGmW6dSRm1CQFa9fbNvS4IKOjpiGxjmkH2lxr34hVSjmTExjoo7wLwwuy66SuZwXdbV3orKBCW36gLUD9oVKClFRzGEx77J3Fa7LuXeVw01Vh5ip9bqllwOtkhaVNsmBlmY6AHYWnWhcYYaqkoKG0NVykBLaNAk5YAA87Il6oY7XDdWUtlS1AhRazK2QPPlaphsLaSTnSjM89VaqGvlj5bXEBUbvCmprrxULvoGWmWO3Qrs0piEqAMjnGu/jaReBDFFVXFTGvvSpZZo8P0bpDFatkKKXFtSsIOoATpaL+O113xTceKp27r7RSMsVbA7MsJ7jSktlSEqiQCQZHiOlpR9HhpJuaqccdQVC6Hmc4TsGrwdSB7p+ZtTmaRU3cLXK3qc3pbMNzZS3QJwmt6V3hVls6riuqTI6e1bKrCeCb5oH7voH65xC0uhtpVdVZlEoJGvaQRpsdPibIrjNQGU1a73cKFGPsmyPfAIgWcuEfU3nA2b3eWvvIW04k9+UkCDuedrLXNvYub6Kpy6q98psmfwewxh6qwVd95Kuu7lVTzRRUOPO9mpRQ4oAHQgxqPD7n4q5ruo6YLbu253QhIJQFtlR669mSefjtZkej8LhwNwuqW66r9Ru66qyuW8/Vr0R9upRJUeUr084socdOIibiwFTUWEatNVfOMlpu65jTKzFYdgKeSRyAUIPVSbDEVJHBzZGgm3ldaUUdTVTCNhNibX6BKl53NdqLm7ajuulo1pqUx2ScsoUFSPZTzjl0tBHEpoMvUzkpyBKk9mdz7ZOvIaDp+U2t4UrMA8Irtw7W3s/eFZd1IyKmqfdLinHe0GdKZ2QCohI5JAtBPEx9TlGl41Ocl8FCMoCBpt4mVH424biKAR4lG5rcoLRoulwx39DezNms46pq1Abo6ZxReTAAcgrByJ13PT39LM+pxzhtKJReSld/8AkV3oHKQJ2NnQZU1Udq23k7spSJ128REffZvvXbdjIIRdtICSYKW0DWZ1kaj68LEpwAPaUpS4j2SpD9GfGmBrz4mqw3iI3bUUN9Xe60hu8mUFlbyClaAO00BIC4G5nytbW96/AnDm5XHU1tx4bupCu1UUqp6RgqIjMVHKmYAEkjYa287MTN0f7grlM07aQlrOmEAaBQMfO0B4hZbNdSlNM3IRuAkfxHxGug+Otulw/K+ItA+a5+tzNlDiUR4ku0V88RMVXvS1HrLdXfdfUpeKw4XUuVC1hZUdwcwMid99bOzjCF03o94DaGhNW6ghZgQUu6E6+HwszqijUl71htBKgANRpIj5GInwE2e/G0lHAjAISZIq1jQEz3HNOZj47e8ZmPty1eHj/wBb+G9eb8YG+I4T/mP4Mqri8M1akJIGU6JTKQdQTIMR1HWBzAAkzglhm6sfcacI4Uve6BWUblUuqraWobC2nWGWHHVtuARotSQNp10OsWjZ1a+2CS4VNxlT3iQCNDue9qAANxJgTMWJ9A24l3txWv7FLiSWLkuMU6CEEBL1S8kJJUdzkZdGoEbGIt1Q7rsyVGnEPg+ngXxarMLv07j9w1wFZdNRUJzpqKRSpKDyU4hX2ath3QowlYmTsO3XhutW2j1W7FuqyoDIQju+ASdoB6693aLWa9IrhC5xc4fLu266cOYiulxVdchAyqW7Az04URs6kBOvdzpbJJCbVNwFiK5qm7aRF/lNPUoCWEVDrZQFAGE5iUiFa7SPZNjRPuUMjSynDhZd903Xfq6ahu8M9tSFtRCYBHdMa6n2Trty5WK3hTOUq3aVh0hLWYBlRJhSScuh03SPDeNbD4KLjF/0TyHw42peQEEEQpAAE+ZPXbQ72M4tPqN81XbPoS2Xs8KBMkwQAPKPhyi3OQDw/E08Y2lhY75sc5p+jguHpD4TjKojG00Eb/nG9zD9HNRbh1f4w/xCuauQnN2tQmkUkCZQ9DZEjmJzeOXeLW7WiUFBTvoo6a6x7zt4R86VXcKiivS76pTSmhT1ba1POAISkJcA0k84SPwtdKUqyuBU5tRzER9bW2bakL0QnYqD3j6ut1suSUlSCD4Hbbwn8bF3VZkKVEBQ9rURrY/fjZRfN4tJATFU5lM7d47dfysk5gYOYhA0SASQOe3TX8LcW5uV5C7+N2aMO8km9pkcITmUjIRlOsab8vHf7rAOBKm0kEAA5coPn9TYZwkvBORJHeEK1PPX6+O1gVhW68o1kAbfUeW9jM2QH7ps4wokLudl10GGqxpeYAA+1E+cE/W9T/SVu5yj4hXokqzdpT0ziM2hIS2kBWmhnKoctrXBxW3nw5XAOCUt5wDrqkg6/DxtWf0sKAf2noK0Mk+v3QlJIGmZKlka+8fLpa9QOtUArKxRuaAlVgeQtLiwoZSCQR0s6OFT9cxxEw/+7qo09S9XNU7bucoy9ocm41AhVmzVJKHCVCAozp9eFj2GLxXc+I7qvdGho61h9P8AuLCvwt1MgzRkeS5OI5ZAfNXyOA+IdWT2+KKchMaGocJJMbwPqLM3F+Hr+ueoco7yxMhC26U1iSVKHbCfYSeau6R7/jYmnfLiVNFLQCEKKVpnM4DBlXKQdBEcrRTxkao6eruyrXdFPWqAWXCtRAS0mM0aiT3iefPrbiI757LvJYmiPMFE9z3FeV+XuzdD17usuOtqkwVAZW0qmJ1zBfujxsQprnqH7xXdpdrnH2ao0z2VslMSsJUFCdO6OWkjpZ5U97X0wGr4Wihp2qh5+gYdabBcSpLakp130KRGutkeuTiGoZqrxqL6fIVdyK9DaCUiCsBQgEajXXnPna80LOLQkhrCmJqujHq1DVl5fZwlxSUjKUkq3GneEdNR42cVLwjvB1KVVNQ4x33Uq+0SohsJHZmMupn2h+dgauiVcGJaOoarqhwNt0bqVrXOdtThQuR7068o95msNtLbEEwUzPXb7tfjYMrizZGhhbJe6hR/gOzetAmlva8lpWtSy4lkEAgxlAnnpznkI0tU6+qd2nqEtughTYLRmd0KKTv5fK3o4ttpU5XTPMlO3PTXTX7rUK4v3Sm58c39RAQWr1qVARHcWrOjTyV9c9XBpnOLmFZONU7Y2Nc1M4rKBTupUCUj4EKJ/G17rlvBN6XLQXsNEVlI1UETyUhKvx/rah+aaZI/lWefUD8rXP4QViry4aYfqV5VRRdgdR/s1KbHySPjY+MNHLa7zQMFd7bmeS9DfRAvb94cIvUSpM3XelTTAA7JVld+9xXwtNizm1J6aC1YvQkvQKu/FdylWjb9LVgbRnStKjr/AJE2s2TqAJ929qEBuwK9UDK8rRM6ZflbnMZ5+JNsPONvkbaVAOoO8WtAKstrUVIyHTSzcrWy084kHn7zZwgjny5dLJF8NhC+0AM+FrdO6xVWoFwm8g5KmoRP8QX7iI/6TYXvbK3Otg3u5eBUYGdo6beyf/wrCwQTvpOvW2iHLOy3WwVEHlB87YSCdNuttgADMIjz2tzIgGBtsTad1Aiy2CeRjwt1JKpBO+45W51gb9NLbEAkDlpvNpApIT+HnET9fXjY9duj0iRBkEediAgDNqNNTY5dqwXhJ36WnmTWS7mV/Mfj+tst3I5pPwtlnumsF4lcM3GnOI1AEADLSvwQNRKIIPXb63tM1aoq3A0VEjbNy/HS0BYbvNygxbTViS6FBtQBb3BJM7bDTUe60x/vt112nKlJyrTkUlZglXKD8eth0rxsh1DDZOfDlQxT1K3luQ2hKlFJ1JJgQP6bWcn9pLuTKU1QIkQAkjl9R5WZLVSHEKKHZSlI7wMneeX18LJTeMcNVN7G4W79o11qF5Q12kd4bpBmJB5WsSRNebkoUUjoxYKSP7RUQ0TUKhMykJJnpy+pFsOJbuQmEuvTvAT9x56j7rNBSiVZwuQDIG8Drpy1+psVfuulcS46C/nkkZHVAnw3sMwNGimKh19Urf6TLsDjyGk1LnZqUpUDKpIHICZMnN8+thrqxfdbLKnXlLQH5W2Csax3dNfC0SVdO/dle8s9tSvqqELJUSpKUKVBJB3EqEgHn8ClbVBq8Cp99aA3TqWWUA5RrmciNSIEg6e0Olq/KaDqrHNcVI9+cSsP1lSm53rurCqsR2aXM6YCjtPT2YHuHMWhXFN+Xo/iSouxlgU7iGFtLJIlYAGWY9rSfOT42K1d7NPYhReCnUClbzLKe1OZOwAk6pMgR4WPY0Zum/slZdVak1rTRcqsylZy2ACAnrHlyEkRYZYBqFIPJ0KalKw/QVNPedA0X00rAWVIQYCh3TMbDfUyBMyOfpdgqto3ME3MglfaKuumME6BRaSTz21+63nx6P8AU1DXFu4aN5x00tTUuUzra5KHELSpJSUxBSRAO/LfnfXBzApsNXUwjRDVIyhP+VKBHLytynEE33Y12HDUOjpPkqc+lEvLx+S/3lBNbd2Xn/s0e7+Ie73kew9GR6s31CR91vHj0nlVP+nZ1KEZkpcu5RE6ABCNTy6fLW3sHQqHqrZEaoTAnwtdi/q0fwWVUC1VJ8UYVBGkcpsA5oJOlhlE5YEkWAXsZJB5aWMxVn7oq9Guw0sGlxBSJUAfZt0+Y3Me6xBSgFzJOulr0RsqMguhqpwoZLg0UkSI8rSDS11PT09NUOuBSXETzJVCZMWjG9HXDRPJazElCsoTqZy8hz8rKV1XhSvUVHdt91z9FWUwJQ24rs+1SoEA676K9xFqtdlJF1ew8EghPWrxS0lbSaRlac620lJbynKuYUPhYq/ipfZLVSunOIUgqAhSQoA8/vslm5boeU2s3xUFSEtJQUOiR2cwdBznW3C1XPdea66Snr65b/eDKZUEAmZnSBPU2pcuF2tjdXrOb2slpdd2l5VFO2lJSGkrcjQBZJjbmRZLvgE3avOSIW2oEaxCwRZYapqalQQhsJ7QlbhJklR6myXfr1I3dNUQ83ITMFYBJBGgtzlW8zSXYNAtSnswWKCacdLC8xOjhGttUDjjyVNtHZat/O1dPSJxzjXCWInKK4MT1dFTO0zTnZsqCQmZGbadwPqbRvwg4q48qcetXRemNb0qWl3fUOKSp9WUOtoKgo7Ttz8rRmwKWvha8OACabEI4CWkaqzjtctNRfSMxhNevltLaD+Nj4dK2XdPZII+Nqw8OcTYivW9bnfrr7deL9bUoqDVVik9ocpA0PtkCB8LTZjLH7GAU0nrFzVl7i9FqZbRd5ClJcTEA/5p0A1JBtVwnDX4ZjkTXOBuDrt0UMSqG1eFvLG7W/MKU8K3A5e9Swl5vI2pSlFwoOuWCQnb6i0n0tP2DlYiBqtBmNdUIH4Wg/hLxcqL5xBS4VvLBV6XJUstP1TKaxwZltlABgAaagWkq98XV133ow9RXKt9h5o9svtIS2pKm8oOh3BV/wAJt6O6uhdJkzAn6eq4cUsjGZy0pcpHU+t1VCkkPOVAeASgTlHZg7897JLpW+u52G1rSvsHQjMmAlQUQD4HSzQq+INcm9q+7b5Zu+771TeouemdVVLW0pb9P29OSpCREwhJgbkwbJWJsf4xu/h7hGtprkoTeb17IpalxIUunaBWtJIkhXtpG9kyrhc/IHahMYJA3NZQJx8umob4rVT4K+17OnUshEgECCSTprFnB6NrqT67SONFZSzfLcLTI7t4FQEc/b/pZI4uXtX3ti283L0cZFaw+unIabKUqCVnkSdff+qNww4gucPhXXq1dTdbF7XnSqZU7kH2hacmQD16bm2XLVc+U3OgK26eHkxC26s7X0dMqnaNNTUZQBIKadIJ/wAJmxvDLTDd5tLLDSVzooNJAB8xr77Qer0tU0rKyjAFMSy5PeqiIy+OUb6/CxZr0zawvLUjAN2Nl1YU2FVaioa76DX5WUdM0PEmfbyRnVLiwsy7qRuGtDc994Vv7D2ILpavGgeve8aZ9hSQtK0h1BBjl1B3BE6GLJfD/gFdOCuIKMVU99XheN1XdTuNXFdFaSv92rcVLhSskyInLoD3iSSdbR3hHjDelxULb1FddCW7+xBUOOIcCz2IcTm7pBGmg387Oiu49Yput1fqN3XbqSkAtqUNdD/FYBx3DYyGTAlzfJWY6XEGsdyHWa4aqZMf1aazB95tpaZWG2Myi28lSh9o3qI6a6eFqu43apxQwdW+2RkKkgp1AT8Z+t7PVHGjE1+3ZUXbWUN1paqmVsulpjKtIVGxncQPfaO8f1CkUC3iVlwhBSUiAnv6kmND90b2xcZrocUqWSwX0FtVo4bSy0dO5kndNhtSSVnMDnZSdSdTodto/P4lahIgnQSZBHPeeW9t0bjLdQ6MzcPMlSSFSeXI+/ltbTwSD3jKSk+0fD9PdacbLIodcapEvtpT9y1yFAgdg4CQnWYPSI5fEeNoBvdkmspVKTGVnXL3iVEmdtR+OnhFiqxBcYcZzHVBEFRHeIgef62r3fSFprGEqK0dxRPhqdhI6RrG/uG7hnULFxK1wQii6UKSG0pCg4SEiAZnbfw+Q311cnHouo4GYEbGValV6phIg91zly8PGOVkjIF0wVn0SSQEoiZO0xrt8jNnFx3uy8hwQwXNDVJ9TrFKePYqHZJyrgq/kB0AMjcawbZvEbmtrMPzH/rfw3ry7i97W4jhAJ3qP4Ug/ePVVavRxLbJ+0LPdIAGpI3IEmOZ0HhvOt1/2etzBjAGLb9WyEPXhe9OwkqiS2yyrKSf8zytNptTOtbLam1NENJVukJJJA3KsskCANPedrX79D2kN28JbvYUAhVQy4+4udVLNW5vpuEKbHXWOlunLtLLuiw2urAMLCHmyQUS4kwdRodfGeVqp+k3w1Z4c8Vhie7KVl658elyrNOswlqvTrUIBAPtlQeTCd1uiICYtMt4OhaiZkzG8QPa90+W/SzQ9L3CruJuCzl80yM7+GKtm9UpKCYYByP676IVmEc0ixIh3QXbqr+FHqSkvBiouW9am6nkPtuO0j2jau8FaJJyaj+XkZ00mT+ICnqG/HKpFIstu0yFmop8qnkAGDCVDoCdzyEWim61t3tRsmraS2p1GYLCMzalaCCeR3nxE67CU8csXu9d+H7zoqFuomlCKlSkryI0QQdNt1ROvLnI5/FHtpcfoJybB4ljPzDXgerT8yuExt7KLijDKhxsJGzRE/ENe0erDbzKj6ofoK+p9bpnK6pLRCg6XC4G1CAUuNASkTuop5yJMWurhu/rvxLcNFfV2OqVTPsjeApKhoUqHUEHTbmJFqlm82GElLFBSmqpEBZiRl05KOux5eNpn9HS+xel1X62spZeZrEKXThITlC0e2RuMxBH+5rboXsyG/deiMOZvwWsYtJZxPeCFOEFawqIH8SQZ8N+nOyDmCNErJ1AgifLX63HQ2dGP09niVaxmHbMtuQOcAjz5eH32a64SYCidgAk7gx/X+luLqRlncPNd5RuzUzD5BJz0ZyVJ0zTlSraTHhzPzsLX3jWXvVrq7weDtQUhClpaS3MDKNEgDYch42DcUhSysEGU5p6fUe6wZQoLWFKTKSdQdJnTQ/laLLbpO3RK9kdtdVczlKiunWN41ymIHwtXn0oqRNXh3Bd75cyVUztOSQQdENkfHKeu3jrZBxCXWVtTIXKT5EbdOn9LQHx3p/WODFx1p0VSXkWcwBzAQ6mPfAtapTlmaVRrm5oHBUyqkntCD0gRrNgxplI0Njd6NlqrcB5KVpPKTYonXeYG2vO3ZA3C4kizl6mYFq0XxhG473ypJr7tp6jbmtpJ+8/WtmzxLu9h2tud2saC2jUOMOAjuwttQMiP8NtejtepvbgxhCrUcym6FNNrGvZLU3119n52XOI1y117XC6LraDtZTLTUMIV/GpJ1BPKQTbhpP1c5HYr0Nh5tO0jsFBN7UVa/Q3q1SLYN2018IcpAgSoFwydRoQc6Y1+G9sutgVl1XVmSQKq6q+gV3pkpJUnQ6zz11s5KPAWNqekXdzTlIKepbYddzLOZLrWWIAB0B8eVlymwA8w8wUuNdlTV79QgAbsvJIyTyIJ+APvOZWjqqIgeTeyii8bteau24LwTWPPTRreWHFaLCFpWW56CTA8LTtTKbfp2XkyWXEJWBHIiRpttZoXfwkcpCEV99PvttFaW21IhKErSpKk6k75gfcLPW7KBV03ZTXf2i3RTNBpK3IJISIk+MCdLCkeHWsjQROZe6AI1zZsxzCJJj36z0HW1N/Sauv1LiHeLiAQ3WM0tWNefZ9kdPNs2ueUhxRLY0Ookyec2q16XF2KaxFdV45CU1l2OsgcwWnM2vuctewl+Wot3Co4wzNTX7KuSf7tafEK/D8bW29HCt9a4WsNnX1OsfYGmglQX/8Q/LpapLey0g+0n8Z/C1l/RSrg9hq/LtmTTVrb4E6/aNkGP8A2dtnFm3piexCwsHdapA7gq9noaXgaTiJe12qVKK26llMaAqbdQR56KUbXDIESCDOo+otRH0bby/dfGq4pUUoqy/SrlWhzMrA8+8E++17VajrPMWxaN2Zi2a1tn3XJI2nXwtqR/MPO2E9eU20VTqCD49LXwdVRKwq1kcupsTvFGdjNpIMzY0dBor87BuJC2ynXaNbHjNnIUgzCyZ14wiqo3Z0zrbmeqSfvSLCKUYnrPP5W5xCC1TdqEj7B5tcnQABQn5TbkuE6KURrbSbqsxwshgTqCes624zic089t/lYMLndQ08LYlRiRGgn3WKhnVCFX+KAR9C3QUNNRrYAqB1KhBPyttK9Z5gan3WkFFGEqEgnTXzsoXcZeEka/CLJSVHQwCNI3seuxRDxKdJNnSTh7x1yKM87ZbpJUEgAgADQa/nbLK6ay8LbtcW5fzSmVwQ2VHTMB3jz+vlpJC7xZYbbbSsh6O+rfYHQHlaMqOpeZrQ+06kFbQREkmSpQykRpry3687KdM8aisQayryNpWPsk/xbjKPOelqcMmV9keVuZl1JtHX0S26ytq65ukZZZ7DtVOQn2dTJOseZtA+HbvVWY1o7ppnWVxWoR2qAIWlJnOCATqAFa/OzscxBeOI8YLuu4GUVNNR0YQ5TPNgtAlQzd078tYJGum9m/i6mrbuvFzE7eWirCsEtJR2YaUEfwgcgRyncb2LJUB0gb2QmQZWZlZiQglTmmbcEfnB289rJVdW1uZ6gonGg8U50FR1y6j5RaKMEcYq2/H27txG6lt5MkuoSAH29lSn+YDXT+XazkvnEtDRXg1XXUtVStnVYykhadJGkzr9x13FrhmaRdUxEb6pOxrxAWmq/dakN9q2338gJOXLzkSDOmnLrNka58T0Tl3MCqgOQ9SlxWhDa/YJ2kAiJOutmjiKpVet9VV6BsvGoBjtD30pPlG23XUWTs7oaDnZlYQElKiIVm5gnaZn3Rtam6Qk3VprdEvXQhiorAahRdV2yMpgHOk6az00M9Om9ljFK7goatplx5FLUJQpKVBMk7d4K32nQ6QSOYFm52tHTXOzWshHrIWW1kKMIJEb7jTX6NkK9n11FXFQ+y6qUpStKiSBmmIHhG0i0c2ill1T44IVyGeMOFkNwoLvBLZIER3jsPHXTTp1t6F4eb9Wu2kp4AKGkt90dNPrzt5w8KKly7+KGGA4VBCr0pcyQqRkzp6cgRPvHjb0ju9MFKFKBKVawNz91uP4i/aN+C7jhjWJ/wAVS30pVPr411JbZUtDTl3lZgHKClGsctYHj46W9fqCfUmT/wCrT79BbyO9JgLc4xX6szkaRdpkHQyEAT07xHw5a29cbrVN30xjdpH/AIRbShP9Gj+Cxar+tSfFDqkjcg9PjYBZA31sOvrt77AOQkzJiYmx2KpJuijxOUlM2TniQoxpGszysoPnQk/Wlk54lKjm9wtdYqTt0E64UNhZUNDsbPFlq77yCUV9M08gapS4gKANmFei8lA6pGmVBGhG+vxs3rs4rXtQstU62KV5bSEgvKBlZgSSAQBPgLZOKRTPex0XRbOFOjDHh6l+pue67vUhdBQtMl0EqyCJ6WVLlqWnqdaEq7yFlCuUG0OVnFqtroSaelQAICgDqZnrpYFHFi/Kcl+mRSozICIDc6AmDrrMk2qTRTyRgdVfBjDbBTXe1Aay6qymzlJeYcRI5SkibVaOD27vdbrVXnULDC2nnARCQpCgSNeWmtn6OLmMHkFpdTTd/SENJ+trM3E14VNVdld3lJDlO5mSBv3T087Hw1r6WN7H9UCVmchx6LvjHwqxBxTxS3f+EK2krbrFGKR5SHEKAdQpR0ObfvC0ZYQ4AYtYx5Vt0zx9ZZoqmlWVtFtv7RvISlZlKiM4MDxtKOHcRXjdNAu7LsrHKalKpDTRhOoAJB91lK6ayrevNts1TpUtRlWcyZT89bVozVQM5bSLdFZfBDNdzgoho+D2IcPXxS0j94tF24LyL1QIkLSsJXCR8frS0j4+eWq77mcAns72o1bcyqNvf87ISryva+r5vRkUzpfSpKXMmYqUUlSdd9wB8NLLF7Ul+3jc133cxh+8lvoqKda3C1CQlJSpRkwdj06W53G2SukZNIdRpotPD+SxjmMUgUqmaf0g7kZowS3UXXUIMnXVtB0nfW0oh9NXT1qKJnIXexhA5kKdST/yWjVWH78XxlwriJikKqGmpnGHl9ohORSmgdZI0EGfMdbPbD96rp3GEKbLiKpx5sOZkgI7N11UHXmFC0KGbJAL91mV8YfJp2UY49uTE7l7GtorsqKhTeKMN141T/doZyOK1I9mPPzs/cQ3Vedbg6iu/wBRUmopL6ZqchWB3PWnVTIO2UTYHHF93mwm7627rsadp6h9g1LinsuTIe7yPU2PVWOLpfVWpq6htAWhtKcn2moU500mFTaD8V5TiCfgmiw5zg1zRf8AkKuPHO+6HD+NL4ddBcRU1zq0FsAkjNO/Pcb2iWrxK5dNA+hugVUIrr/WQCvL2YXRsOSdDvHus7+PjdVeV9uJuVbjjLbxUSsASIG4+t7R5fNG/UXP2Apytxu9qFSgIlOaiynXzbNt7DpnVEAkcN1WqohDIYx0TexJjCupqeoNC0llwpU4kLhQkpGxjqOnPw1E4OYgq8bv3jSXspCXKC6Xa5p1tIT2jyVJiQf4YOw8LBXzhervBxWSiqpabCPs3gEzrp7J+787BejvhS+EYjQtxz1Wnr6CoZKh3llBMEQoRMpInr8LaVaXMonvYbEBU6YB1U1rtiVPV9XdSXZgrCdZSZh6xebNQ8oqJ+0UmCROgtvEfdUpaXZIMxsQOvwsdvy701vDG72VOOpN3PShxBEhTSwBy/D9W5iG7alSluLvuuWCJgrTB+At5tTvMziXnW5XZSM5bbNCUbgrHFO9ydE6SJnT58vn7wMa3k65d7rbic6MoIDaQFIUJ16nf5CyLh+7l+sIUuqriGzmRL6hBnXQR0H1pY9jGpC6NKVKUcjawAn+IEjXN4SbdBSRAlZ88h5dkxWL3q1XmGDWulCCpKRmSISkEBB01EwZ0sO9e9U9mQXMpgyfdEA+cb/nYg4Ab5IUtISEEp96dQfn8bcOQ3KRumVSoaRz22/r7+rjhaQsQTO2Qjl41MKUqoCYEkpVBJ09/Lx5WirGlN6tWtPstwF06YTOpUZk7fP36a2kowFHvmDokbydB9f1iP8AiC2vLQlJMBKkkJnLoqDqQBtHTc+Nr9M0NdYKpUOLm6pT4SpuoYxuVq8VDsgs9mHB3S9kOTlvmiIMTFp6qHb5cSy1fdLR/utfrovMuqBQKcZskyAIjLO+kzasTBcZbZ3AUk5SJgGBBO0mfvHhZy8c7/vscEcKg3xVI/eFUtqrX2xl5tIXCVk+0NB7WmgJ2t57x5w8/E8RpJWvA5h5eoJy6PdmbqNd/mGrwD9J/CEmOYzQSskA5xEOoJyWbI/M2xGtidPxNYb6Kt19MUwvmqRdrRVSpfWWM3tFAnINUme7G+0n2o0vj6LbTjPD672FoKMlI42BP8rNKsiOXeUvQzt7rUPZumuvO8WbtoWu0qql31dhtGpW6slKBqRqSpAA3/D0l4bYRThKtrcNtpzou69quhBgpKkhqoRPwZT4beFu+dM2GRkROp/cve4YHck21sBqd0928pRmJSqU7c52135fkPF5IoqS/cNO3TXNZqevpTTvCNwpEanfmfgNbM9DYBGRpJGqTIiNP6Dby6F5XI4FXZThXJJSRPIGP6mLa0OqzpAQqH4CucYQ4j1uCr6TCbqvE06kvaAo/wBkqBoZ7mvn42nxt6uezCpp0NNN1DiHM5BC2YVlUP8Al+dml6VGB7tu7G91YyfZKKW/21XdWOJ0KXkkqaUSOpJET+Nm7gC974drH8EYsvB6spbwpFU9M44NT3DInrlzc+VvM/0jcNz4hH9qRuBETfaBBJAubub0vYkkH8IINwvE/wBLfCM+LQ/bULgWwss5pBJDQSXPZrbMASSDb7rSDcJLU/dVXVViLndaXTrccCHE6nKpRgiJJ2Hy52VfR0vypunicbndfK03iw/QPSdcyEl1CjOpjs1AefjaMrwuS8MGXm+WVhbbLq23UAKCe6uNj5q+IiBZ1cN71Zu7izh++EQmnralrMUmBKx2fyJ18+XP0RuUwRljswsLHuLb/Ne14Y6PwzBG7O0tFnb3FtDfz3ViuJ6SzedG84SntadSAOuVX/4VmU8VZsyjBAiCdoPKR4DXa0h8UGStq7KggHKp1BOvPKfw++0flBg5Vr0PLQEx8dj8hblMR9mpcvQMKOekaij2h0ScwVrAOvKdtTp99uAidVBUco1JJjSxh1AAgqlW5B3n46+f9bYGwkhPeSTAIPw5+B+jau1yuEIuhEFIiNcs8z4R8fn74b4s0iXOCWJKZKSv933sVJA1P/nCQfko/U2mkI7ye6kEmTPL61+to3xvdRq8E8SLs0UpLZqoVrEtBf3p89LGifaQHzVaoZmjI8iqBYhbUitcUoKlRzHMNvD68LJQB3n3WW8SNQ8hZUnVGkc4P62RIKSCevO3bxG7AuAkFnFX59Da9P3hwWYpjCjd15VNOBP+V3/4m3lad6tAU4tJQcqoUT5zrr5fQtWD0CbwL+GMV3PuKWupqmAdR2rak/8AwrWneaUFJIAUChJMDXp8ItxmJWjqnjz/ADXfYV+spGHyTdcpkJK1FBKRzIOp1+vqLFXQR3iN/aIG1lh5js1ZVJUYJnQA6E/f08/eWcZUhOVZI2OvTr5fhaoHaK6W90mqaKSSDEQJyk+P42BdbJSCNSkQpM+6Nuv4WPLaVBQBASYyg+H5a24dbKgoZQiSADlkz/U/15LMoFqTPVyXAHIMr3Gmh5j6+FoD9LW5ArDlyXjClerV7lNKUmYeaM+X90B9G1iezSFBWo5g7g8wfC0W+k3dCrx4W3lWAHtLvqaarRB2hwJJ/wCFarWqGUNqWHzVGvhz0rx5KhTBKHRG6gU/ERztOvolVoTiO/LsKo7eibfCTMKLa4+5w2g2qAZrnQkQEOkjwE6WlT0Y631LizT0f/ptNUU/XZOf/ot11cM1M/4Li8PdkqmfFXUwDeKbkx3h29s8ClvOnWonU5Q6mfLSfvt6ML3hQ5m3ml2TiCC2AFAynrOmoPTx/W3o9c1em9bloLzTBTWUrL4g750hU/O3M0Dr3C6jEG2sUbkgEAkeNtHKDJMwba0Gp58httbStTp4z521AsorRJiExbUAQJOu9sUdNPdNuCQPZ5jrYjVEpAxJT56Z9CRqttQHnGlklt4OtodSkd4BXymzkvRvM1mGulmhd5HqbbeaeyJbHkkkfhbSgN2rNmbZyOJWI1II8LbKkp5b9BYILE76zO9tKcIgyCeWtrCAdEKXDOg1nnsbcJUQRl2EaWDLg/i5dPytiVknXTkPDztIKBRjMJE6eNlC7Fy6CZB8rJQc1M+Vj92r+1CidZgz0izpJy5vD5m2WLduTqFq+MWyz6JLw2p0Htu0aUQvswIn+GVTHy9/lZLvKtdp7wUGXFAqJCQNBA3iTyGg+M2PKCGqgpzJU4UpE5iT/FoB4Dn8xNkO9O1TVkttklWoROmo3P1vO82yWm71ZdoxOTgE42rFl5OKVmK6SR3cxnMPjvB8/Ozi4+3LR3fd133k04/29ZUuBaVLzJHd1jpsNB87FfR2o3G7+vpwj7EUyElRHNSpjXXl91lL0lamKG4EAlMuvKUADMgJ1HLnag97vtENG3+y0GNaaEuKhKhUtqrYcT2mikFRiAPDx/TSz1VeRZWltZWlJAWkA5QrQaHmQPw87MWldcdcDKVhQIKTIkDmPr3eaw2isqCnKvKCdQTEnw678tdfcdkrItdH6x8vqFRHfcGpSSlMjYa+Z+tbFVagFvvIjMmTAEA8vhr/AFsI25Rs0q2nHnO2S3lQotiJJ0Opk7CxFSlpUpS5kJCYBmTED89OQHuhdTDbIwe0RdrhS4stqdBykQNtY8do6jSxZtinS0CzVla0lIHcMHUGROxBnwiLH2altFw3i0pJDgdZyJHQEgxZz8EsM3NizEVbTX3SLep26PtUtpcUgZgtI1ymdj1IO9k52VtypRszuDQieCKRf+kbDBRlUE3pTKCkuT/tEzruDqTB318LeltMgh/ID/EQQOU+P1+FqrXXw5wjcdS3eF0XP2TzC0utqLy1HONREqOxE2kimxzihVQZvVZClyVBKRrz5R5+dubxaldXOaWG1l1mDVDaBrmvF7qEPSncSzxfvhBWcrjV2kIiAoQkEzp1n3cufrhc6j+7aU8yyief8It4xekLeddXcTE1NbVesOKXSpUSdVAIEDz6edvZi5HM910ahAmnbJA8UC11sZjgY09AseWQS1Ejx1KPq1kaa/KwK+6PDpNhZ0gCfCwDhkajfTWxY0GRFKhRgjmNPr42S6lQ/CLKVROUncRFkqoUM3eJEabm12PQKk7dJ96KPqT6QD7B+60PodJSEzyzDXlaUcR16aKgUtSAqVpRBgbkC0TSoH7MRoTt9eNq9UdQr+H7FHXHHEHMTMiJI3B5j63sIy84poALJ0EQdCeQ+/4WBaZcUgOnRKxAM6adLLztwpo7mpL3N43e6KlRR6s06V1DUDdaSIA32J3FqhcButVtkmocyogrzcwc236bWDvUOO3bWZTA7Bw6yY7ljwpkEqJnUdAY6fjbmtQ25QVbaFZR2KwT7j0/rYZcpkaItcqjC41SAlRJOuos5LmSDe1OpHcCVFwgmdBuJ+NmxcC2w1GbN9m3ryMg/Cywy64yUvtKIUmYM6g7WC5yKwXCOsPN0GOL6eEISW2lT0GdzX7rOmqxDR093s19O6hTKaaVpSJg5Uj8BaMb1cUq+nHnSVKcpkZleIJBn4/fZSSo/wBknhMK9WV4xAmLczj/AOyDh3V3D4gZCCpKr8eXbRqu1FTSPvuFSkSlQSnK4ypJBnXkLRxw+xziB25rzoVVyuzpr2qmmiBKgkEHcieZ+tLCYgWpQuh1MCKlkqPuUPy+Fmvw9Vk/tFTrKpTfLxAJ3lKD+NudbK80zhdXXQMZINE+Xa+pqT/rL61CZhSvfoOVu6d2GHSEEDMg6HY62TWllQIQEkyRsYB0sap1AUtSJMdwwRoO9Hv3tky3cNVoQ2B0UZcRlJF8urcBEKbKBuD3fr52Zd5B5tVYX2y2fWrqec01El1Ep5dPj5S6OJpUb6JSAElpO50EHf8ArZkX0QbgqnoDhy3csqSoaxUrA36ZremYMQKFi4/Emk1TylGqZv5d5VCLkq2mludmVhxSYOYLAPQmJ+jZL4NrqqPEtBTVjh7RpyrYABBhOde0cpHztoirbrFrFOpYLKSqUaZQowemxP5nkV4X1hcxkytLWX/tB9s6yM2s+787adTI19JIzyP5LOiic2oY89wpyqvtMBV7Ud1uoqQRvGqo+479LNu/4NMlSUpgpnMTvpsdPLpz87L6HO1wnfzAUD2dY7KSIiUn8x8fKzUvSpSu72FwlwqaQRJ2gDX4fC3m1K20h+K7Oc+yiN1urRUNp7TY7cuhsLi4ZrqNQ6UgGEqA0JE7D4WTaNafXJSMxUBtsDPMHcWUcaO5MPzlzKTJGURA5eXW3U0DfasVizE5CUw3D/25ToQhQJQ0mI2OQb/rbhxwZkyAqQBMxtrz3ttlxJxBdjgXJeYAhQkEAKTy/wAottxCUKJJgzzMnfnFunYLALH6aoAp11IBVqSSY3689/ws0sesB26qUIBUo1ZSQNJBCjr01+8dbO1SFQnYQJ1SZ+vGyNiSiRVUaYQnKislQ39pMb+JKfha3EbOCBIAQmdXvN0tfS07a4ythRG+pHXqBE+fjNjvHkrTwPwSEyQmvcBOUgwA4OWos2b9qVuYrUgBK0tHLJBgAAbe+zn9IBrPwOwV3TJrlkBKoOqXOY/pbF4j/rWH/wCN/DevO+K//EsI/wAx/BlSD6Hl2YavH0h8HKxO60mgozVXl/rDiW0IqKdhx5jNOoh1psxMGAJ1i1/LtZZ/tTfTzb7TzJxA2tK21gpIdyzrEf8A1kiJ8zbzj9H+mTXYso3z3HKOjqFhKjGYZcg33ntpkDkLXawLivt1XhQryoVT1FFVJKU+yPsEbHb+71H0S1tE41TasO2FrfNen0k7QwxEbqTbwp+wyrbT3wqVFSiRPhOx3+t1jDzyl3cO8cyVLCiQBrM7ddRv42L37SuGhU4gpTACp9/LXx8PlYLDDx7BxsqIUl5Wk9QNduv1pNt6hkzBZlfHyym96QeEjjDhdezLDWaru1k19MOZLfeIka6p5c4i1WsO3vVOM0l70zgXUsZCUuQMykz8No08trXkUttTZCiktqiUuAEEESQR9aG1JMT4Uc4e42vjC5ORFG92tG4USV0znfaB/mOWAf8AEkxpa/JG2VhjkF2kWI7grKkiZPG6KQXa4EEdwdCErcTxd9JXUd9rpD6re7QUt7syrIoAA5o27pHvm0Z1NDeFLW0lZccuXe2s1LLyFgllcgxBM5ZEx4bWlu73W8W4GrLuT2a626HO2bSlWfTU6E7g98fC0SXnhpVE8qpZLlKFEBXYmE+BjxGkRbleF3PZRvwuU3fTOMfmW7xn5sI9CuS4CnfS0cmDzm8lI8xa7ln3onfOMtHyKt7f15t4jwLc1+tKT/rIaeBCtDmQZ+JFmaGFJSUhtWih4ieZ6cvv6W64VXw3fXB5u7lJKn7oqUsuyqIGeQRHgo/WljrrCCdjlgSkjUc/ONNrZ+Nfq6n5L3TAfbpdO6T3GsyciRoI1JifLprM7fjbgMlJ1mTJnlI038fdZSVSHvwkmTqAPOfo9PC2JpiCW1QnMCRJO89OX6WzmyLXdHdJxZUDIbAIGxE8z9e7bq0K+7PW76xVc7rQKbxucEAHU9xTZ8uX6WfnZEyMqc4Op1meXv1s26xo0/EKjJACa263Wjm/wLBgctlT+liCSxugyRaLzRxQyWi0SFqWkrQSUwCfqfP42biSM2qrP7ijdq7sv29bvISBTXk+wU8wErUkbT05mzAJ/pbv6V2aIFebVTckpCtb6Ad4BOLMVXRrFTd7FT1gtu5Zj/7ba7a2c7YIAMgjlG+vyt5QcP8AiBfHDyvqL2uK87xoat9nsC7RVAZUWyQSkkpVpKUnblZ61PpL8RKiA7ifE6kQYBv11JMn/AlPyti4jhEtXUGVh0XQYXjcVHTCJ41C9GammXnKymFTqFfXX61sUfZ7H7VeVIO5IygmdNbea1RxyxpVLLj15Xm6Z3dvisV/8UWR63iRfNeIqqWne1kF9x56P+NxVq7cAl6uVl/EsPRi9K6m9LjpUhT980NPpBLtUhIHxOm1kqrx1gGjBRV41uFiJSQu8mU7afzdfrp511eN7vXcbFNR4fZZvRK5eqlIYW2pOugbLUjluo62SzjTEEy2/Tt6z3KRlP8A02K3h89XID+JANmL0Qf4t8KmcqneIVw5gO8BXtqj3JM2Y3Fbilwtv7AF/wBwUONKKpqq27Hm6dllC1lbuQlCRCd5jfTXe1JRjLFARkRfVQgAz3FZdfd52CXinEi9VX9X66f+cL/O1iLA2xvD82389lWl4hdIwsy7oteiC3WLBSoTCoUNdrPDgxXIu/izhmp0CXK9poyf+87h2/zWY7zz1Q4XX3VuLVupaiSfebKVxXiu6L9uq90K71HUMvjSIKFz+FtqVmaIs8lgwvyzNf5hejLtNmQpSTrESRoD7vre14OC17JvjhZh2o7UqLVGKQ9ZaJb1/wCG1NzRIeGdKwpJSCFJ/lOoPz8tbWk9GmsU/wAOFUS96K8H2wBsArK5961W4jD5P1xau8xKL9SHKV5BlU6zBtpcDkdPutorAM9Dy625Og5+JtvA2XPELCRztzPe1V+ttmJ0kxNgySmFaWI0qBQdSntGjIgH3zZiJPY1ldTxlyv5gJgQoA/fNn8Z5d2bMW+UGlxC8jYPsIWNP5SoE/MWu051sqlS3S65KxPltFsKyNR/EdN/r6Ni5VCiI708/jvbXaEqPdOsWvjQLOJPVGAuTE85tsKmQZHnYuHPaJMgH87Z2hywI158hZgkjKXe9E7HmI52O3cpAWFSDOkWSs8EnTYkyfxsdu9wlyZ8dbSSTiBajVCP+E2yxPtjzUZ8/wBLZZJLxJbShx1JGSEpRIPNZGumnnJ62bV89yoAUtJlSiDlMefnqZjpZ63E80KlxL1zIqiWkBvNKshCNB3dRvGvSzNvgMP3wrs8jas8DtDlAJM8xtrEx91spn31acPYRSgvu8LqS8LvrHGytKcwCoEpM5tN+Z9wsVvrEl8X8UfvS836gMk9mhx1SgkHeOXIbWU/3OKe7Xa915LpyKMJAUjKBy+Wv9bJ2HqNiur1N1aFlKUnupIE+GukcvefK1izSc1kG7gMt0BdxQlw9okFREpKYEefu+EHayylSVCE1DaQhKiZGvnMa2TKJLLF79jeCQWk9pmSToQAojbfl8tbFHn3CCFqOYcuQ66D9OW9pEXSCOpri4S0hRKUymBsN9dugPnYf933pUIVUsUzixoc4SSdBzIH11NiGRAu9ToYIzGE89Z1jkBIgddPOy3g+qfZq0s+sjs32l9z2j3QSDB8vnaFuyl0QlFdlSLkvVyqpV07qA0ELcB1k6jodp52fnoxIqFcW7ougtB9m+HBRVLZKpyFYnLB0Mgcvysg1bF7V1G5RShCm20obWYSElPSBPx1gaeMmeiJdVXcfH7CdRWLaLabwZUoIJJACpI2O8D8+dqWKPMdHI4G1gdVcw4f0qPS+o07q695cCrsu18M1V3XhQuFAUWniUEpJ0OoBjeyIeDNPXYpw5hu5nfU3r5qjSB14qWlCssgnTwNrYY/co8YYjN53fnUyGUNpKkEbb6e+0S8TE1eBq/CmLaRAU7dt7JfS0dA5CFEpnxAI5728bocaqXYqKdsxey5AN9CvUZTSz4bmMTWSkdtivPT00uHF58LOOtVg68apitfpDQvBxlKihaXGUqETqNyCDv9/rTh9Q/c1CQJ/wBWa5z/AAi3lP6fPEJfE3j7W4rXdqbvSGqNgMT2qsqGgJJIABOvlb1Sw2qLhu+Qr/zVnfp2Y3t68TmiaV5kGua9wdv1Ss6sHQfLpYFRHiDblbhmOU9bArWSSJi0mJnjVc1Ku4UjpOo6WR6lRCoAgp+VlF5aYM6ajwNkuqIGYga2uxhUX7pCxBT+tURAIEOIVKjpoQbRewlCoziARvl8tfr52kbE9Z6vSoBbSoLeQgA8gSNfrwtHtKl1pwOlvNI0O8j+tqlYQ0i60MNBc02R2nIypZSCDnAzE66/QseYpnxUopUlBW4pOX7UACeXQD5e4WJMFa1FZARsAOUe+xuicDSyA8DBOkjn5WoueLbrVa110p3/AHHe2Grxcuy8OyccSlCwplwOIKVJCkkEef1obJVQ2/VMupQ2qS2oR4wbHRV1DjamGGVFR2hJMfWthRdN6PMqWzSVWZSCCUsqO48BYHOj6kI4Y62qRcPXe8hhpKmy2Cy3J8h+v3WddyYfo7xeep6+/GLsQ2yVpcdbKw4qRCQBsfPpsbI9wYbvt+mp3V01efsEDN6urUwP1sZdVdt3uOIqLzo2VtkpWl6sbQoGYIIUZBmfKwzI12gKm1rg3XRIl6KbbvNsZ4JZUARpICv1Fh2odw1UNqMnsXRPMaEedki/cQXHRXyzUO39dXYM07qXXG65lYRqggHKomTqQnQmDAME2L/6SOF7WHK1dRxHuRmrDTym6RTuZala5QQDzMcudsfGIXywDICdeyt0UrI5DncAnDfzpVdl0PBKx9pSk/8AENbNvBAcVemJqcSCLzzjQjLKE+/b8t7Oq/ihWGrjrLlqE1jFTTUj6HQgplCgCDqeX42Z1JeN7XbfN9m7jTqccqUKWVkARkj8LYVNSGSFzT/OqvzSDOCnmlt5AJyKKIkzuP6zY5RUL1RSVzil5S2ylwJGv+0SP+q0VY5xRxDwi6husfYQH0B1tLSWlHL0IOu0dbN65OOV+XI5XVmI311tK5SqaDLKmEqzykgwFTysX/h6eSPOyxVT7WiifkdonRj6hccvincWNVNpAkanX+lmfir/ALLuq88uUlFFTuBR0kIq0H/q+B2slX1x6um87ybqarD1Z2CUdnkFS2hZXOm6SI1PXl7xHcQ0eMbgqa6monaRmou6vYDZdS6rM24wqcwABGs7dNbdTQ0UlPTtZILWWRU1UU8riw3WYpx8q47/AE0SQpYvBhNLAIiFwRvyifoWSeG57K/nnnWwlw34ciU6lKSElRmdPa+E2caW7pQ4wgt9pmZKvtFpBnu6yQr7rJGFXaSixFebjraGVGvSpC11KUxLaDMlEEeO0iNbaXIbynC3RUec4yDXS6l+nqkN3ZiJhxxCD6yhScxCfaQNp0P52a7rblXdDCadTal9ghJTnSO9Akb8vxNlqqvvA1MxeNNUXtdCWX3lOJFReCJOZsJMEZZ57CyOjiTgC67rpaNWJbhpiEBCFprSVGOkrg6CZtxMeHyh5LWldQ6oiLRdw9Uj0lDXIrwVoCZ9omTvvtyjpNnDiW6Xau5FsqWghSR1B+e2+nnZv1HFbA7qws4ypnWzK8yVqWmE+1qkdBOnIcrGb04w3BUUbV0UV/LV2gShKexUkKOh5DUGY18bbdLTzNcDlKzZpYA0jME1HaUNXXKghbiElCXAiSlPaK0nceXjYJ2mqXjkRR1C9zCEKPMeH4/Cxa+MTVN3XctxqtqmEPFzRtahMEE6Aj+bSfHxs3q7GlYimL4TeNUkJJS2grcWuBshOeSSTsLb7Q7YLILmW1Tr/dN4LRpc9WMoBkU6tPEEeM/LfSxaquyuRTPuP0i8rGVcwJRynXnI5idfGxC9sN8Zbuoae83+DGInKauqvVGSX6MEu5FKhQ7clACUHvKyjVP8ycxOiuLi927z1XwtboKBghVXU1GI7szMtxJUGkvFaz3jCUgkkQJOlrMcM17kKu6aK1gVEdY6V4mqUl1J7xBTpIGaN9/rbaz/AONyFOcDsEZnAiKkkmBHsLjcHnHl42jKnqQb4vV4rlOdScs65dTE6gCBPXfbQGT+My44H4EWlJCDUkEgbS2uDEHnHMedsbiTSpw//G/hvXnfFRviWEf5j+DKo69H6E48bojUN04VT1QUp2REAnvQDqcvTc+cWZw1XU9HiOtZReLJW7QJKCkuZVZA6qJIEEFIOsez8aucMqg0uO6d5h8ILrL5SSEifs95MjYnbTQHXW004ZvJbePrracdCxXU7zABBHeUlQGs8woR5cpttmPO25XfiXlOsre3xjMrbF0opU5A2lBcJOYjKnWPxjl8BMI1SHF1CkgBSFJJMQNRE/X3RZimqQ8mlqAZ7SmZXOYCZQnTz3P0bOTBlYgLqmwjKSkGYOUmf1O3K1+CJsf3VSnqHzH2k+0PGM5kHaQYA35np+AtXv0pblTT11wYxayoDiF3bUqCeQlxokjkMzo1/m+M7espEhC9SrWR7/zi0d8fbs/fPDO80tIlyiUitQU6wUHURPQ/XK0gDdV+4d3lSXRjKnJhpF4INI8nXK4VRBjacyRr0J8rNXF9Ri3h9iC8LuvGmTe9zIeJaUkxUtU6tUbmFwnTroYkiLGbuSy442lScr6FZ0EaKj9N53s++JtGrEOHbrxIxRrfW8gMVKUIkpUAdTGsA5x7xbk6r/lePw1Q0ZUt5bu2dt3MJ8yMzfRcPXf8l4ogrBpHVt5Tu3MZd8ZPmW52+iH4E4lu2oqbwuyiqQujvik7ZopMlLiFJ0IOxIgAHp1m0uC7lqGckjTNKgZ28fMframOD669cH4sF5UrimV0VRmdYhQStCjqfgTqdvfaVb69P3h5h+rqLsewRiF2rYkOpUulaTm5wS4o+UibBx+iqaqdroG30Xt/D1fTUsDmTutqp7F3EDIEwQYnce73D5crNDHeIXsMVNMhimCy62VhSgADBGniYFoOq/2kt2tFX7t4XSOXb3yEmfJDC5362ZeLfT6vHELzbv8Ao+w7LSVJSXah90wSNJLSY2HK2TFguJE3LNPiFsS47hwFhJ9CptXxRvTtuzZo2joCAZgb+EE2BpsXXtfOJ7nqGKOmNQhD7bZdqU06JXHtOOQkaDnzMedba/01r/qUgDAWFyUg5c9M4vKTM7qH0bJ7npnYwzpWzgvCKYUDrditun94LWxgte7QtA+aoux2h/GT8k0eOTiqjHOJXClCHHbzedcbQvOlKlrmAeY10I00FopWCkwRE6xFnPjjH9bj3EFZiK9aKmp3q0oK2aNsNMpypCe6nWJyiddybNh1ZWslMxOnlbsqSJ0MLWP3AXE1krZZnPZtdcSSf0tkm2wPA26Qy6v2G1HyE2tXAVUAnZB8om2WVabC2Ja4p9Sw9eVRm27KlcXPwFlul4O8WK5PaUfDTFDyeqLpfI+OW0TKwbkKYhkds0pnm2WkVn0d+NjwBPDe+WZ/9IZ7H/xkWVqb0VON9SRmwxQ0461F+UDUe5T0/KwnVdO3d49QiNoqh2zD6KJLZac6X0O+Kry0Jq7xwpR5ubt9tLj/ANlmP42UWPQ0xWqFVPEjBKEQSezfrHVaeAp4PuNhHEqQbyBGGGVbv+mVXuwgJ7NJ6KOvwtPN7eiPfNE0n93Y9ueveKgFJFJVoQEke1m7MknbTL16WS759HHENPT0gor0TWPBJQ+lFG4gNwTlIUqMwOwgSI1AswxKlds9I4dVM3YpguT0xsFUWG7toavDN7PVTFIyy8sVDCQpxCEhSvbnUgkSPutbz0BuPVz8X28aXTd91PUKroVQ1EOvhwuB0OpJEARHZpG53FvNhn0acaPOrQuuoGEA9wulzMocjCUkDlz62tn6AOG1cA+I99VeLMV3X+7b+un1ZWQOgpfQ6lTcSjUwViPG2SafD4nF8R9pa7anEJQI5R7K9LiBBBMdPK3BEkFU2baOJODHBrfraY3lpwf9NuzxAwhl1vtpMSPZX/8ALYgIUCCl8yRljTxtpR1JIkTrpZvK4h4N1ScQU8jfRQPzFtDiBg5cEX/S6kcjr8rFaQoWS+ZOhI8bMzGgTT3vdtSRHbIdYn3JVP8Aymx88RcFqOUYho5iIkzt5WaHEDiTgZd30dQ3iu7ZZrW0yXgkAElB1O0E78rHhdZ4QJm3Yhy6mZzdd9ibaU8MsgQAJ56+P42Z6+JeAp7mMbnVzPZ1aF/cdbAL4r4GbGX9+hYEqPY0zzmkxplSZtqBwssstN09i6rKYIk6aibYHV7/ABmbR+7xiwihOdBvR0EZh2d3VEEDzQBHvsTf413MElTGHMSVZGuVqiSCQTyzLH3WWdvdLIVJiHZJVmOukAfnY3Quqz76TuDNoeXxqyhK2MEX2lJJ1fWw1lM8++edgk8ca9DqW2MNUrSpIIqL2bSomOiUqP152XMb3TZSp69YV/3c2y0Dn0hKqe/dFClXMfvBWh/9lbLPzGJ8hXnFhu96a7K5xVZUNtyGoTO+VKdZHIDWzexIm7HKp99F2IUTmcntld5RUdQOhkfHlNli6biYvrtl1LlQlxvswOzAVEtgmfHT5nexlzDVIX19stbgSmHCQNdNdBp9eNs5rQDcI7iS2yZ1K8FYWdX2am8naoSEp0mDp4mD8J8rIV0VNSw445TJJXKYUATp9/15WllGF7vVdiaNllZ7VS15UZkzIAn5e6fCzewxh+mavR5HYrbcSgghwEiQrU6GDv4aWnnAUeWUxWkP1lfohRdWSBuCVTpznobbLdQwFtvJyLkpUDp3hvPj+ulpBpbkojitSTStqHalRKjorQKEDN4bb7xYhXUVG1WuIepGS52pScyQSVHoCfH8udmMqcRpltvVfZAOqEZZE66z7+Z38Picw+av950qWnAEpXJI1ERrp+PlZzpbbZSlDNNTwkAkhpAEjWJOv5QetnBc97vXcpSj6qtLntEgpUDCpTp4Aee/LWHO8kQRA7lEmL6YcJaNM+S4rUpYJHujwA5He0kcFcY3RgviThvF96NVz1HTVJqHks0pU5lGhAGgO8xv0slXZX3PWOguuuJcVoQVlOb4GeWxtYPgfxAvi7MQ3VcOH6K4aUuuoQKpy6GXHSknbtFAq5nnuLZGMVDxSvbluCDfW2i2MKpQ+dpDrG4t8V6RcLLlu3G+EbrxhRVK2qW9aZFSy28zlcShQkBQnQ+Fh+KPARjiHc1HdqL5RQilqPWFOGn7TMAgiAMw5kc7DcNsThm5WVVKa+tqimHHUUxyLPUBPdHus8n8X1HYkNYer3wpJBCciCNNznULcNw/DgQgbJI20nTcomIVGIRVbgHbHy/evDL00ri/szxZqKZD5fSsU6g4lG+igNzPKfhtvb1cw26kXDdwGh9TY/8AvabeVPp4Xgh/jdUBmkNKkM0uVC1pKkBKVDlMbH3+dvUrDrqf3Bdawsa0NPp49mnnbuaIvkoonv3IVetysq5ANrpeLx3KhoeVgFukjflt00sB20SorIJsEp9KQSCZm12MKk86XW3VKIJJED52TapcyDrpEgiwj1UdQD8/z91iD9UlQVCiDz1jW16NUXlNfGzqkUCAkqzl9oCD/jB+cG0eL4rUFKlTPr+Hm1g9mEmgpioKH+Y+Zs/MXqL1AjKn2Hm1iPBXy5WqlUcNsbVN7VjzF53Qyy7UuKb7WsIOQqOmVKT4e8WpV9JFU25qu4dUyQXyBTWeNrF3IVmxFQMBCfaapKVER/lQbIFT6T1ULrN90FVjGuoSWgioo7iq0oWXSjsglQp0pJV2reUAmcwjSzEa4L4hrm8lbie4GSYkh14jeD/sRM7Wcd48P+JN7YGq8B3jxkueipK1hppyou25nO3BQvNmSoupAJOckAAS4qAAEZaEWGYeD7S0ZMQrLeyELUekbjupcKKfh7xYeMTmcoXKdOUyJBcWjTQ9NrJzvGzjdei1N3RwjvpwgBsCvxVR06lKVpqhT5JEEHSd7Ef9B1/uKU5VekxiZC1krUaG5qemOqcpMoUCJG/xsbY9HzD9Yj/tzjxxOrykZZN7qbSBzSE5ttAY8LWxRYZH7qrmsxB3W3olnDD/AKRWIsK/2guu48F07TDy7uUmpxC7WkvNqCTKmGy3qvQ9+RlUCJyBdeXbzva+Lwrryq6rsV3k+apxlh9paELVJUEqhQIExIOo5mZtY+7eBXDK5blFz0uO8bvUinXahTIvFoJW65CnFr+yOdSilJJMkwCbZS8DOCV3tBtqkvdSAMqUu3gEiOYTkQkf0tNstHF+yYUN7Kqb9q5QPRvPIoLzbS+4peamczkoMq74kZU/HT3dY+vllx29X6hx9LbxT2awXCDyPJOh18duk2u7w/wF6ONNiuvp8ZXe61c4u5Dqe0q6twqfS6IB7NROxMcvOz8fuD0AaZ9dS7hBqsdHeUVNXg5OnRao25R18bW4mNkbnuB8VRmL2OykXsjnorYJw/xS9H2466/73TT3xdjZoagMPl5ptKYUzvuQ2tuTMEgxIINpJqPRr4euvqdp8WXlTPPQXVtOtkrMRMKSQPKzEwnxo9EzhZQO3bw9wZV3TSVLnaOsUN3JbQ4sD2jmcEnXfxNjtZ6YXCBlOejwXez+kgrbYTOv+Ym3Pz4DeYujeAD8VpR4nNkAOb6Li/vQiwRiivFVfHFrESkIBS0htFIkpB1jMWzOsnayJU/s8+FnZpSzxGxQ6ZUCo1NGkgGYgCmMxPPoPeZf9NzBraou7h3UqATIK6xCfuQbJ73pwhYIoMAMN8gXa1Sh8ki1hmFvY3LzrfAKu6d73Zstz52ShfH7PPhV+7mlUOLL9FWlsFTin6cgqEfwhoHUEjeyBjb0W7qwdf2E+H2B6mvqqO/aC+GHa+sT2qKd9bTZTn7NICRDe2nsm22fTVxHV1iKVdwXNRNrVBcWl50J00Htp1843sjXj6VmNL4vx8sN3Uh3DwfrqZxumMLIp3EqkKUQdF7TvYvgQ05jM4/JJksoP3QtUv7O/EjFM08jijdIXk1SuheGWRqPbOu2tkxPoLXxdlNeF2u4fw3fda9TvNN3k5mntl5iHZWkKBSTsP5iJhKRYa+PTB4kLwvTXndeMaMXi4+WXKVN2MAj/FBSTEazMaxoYlnVPpV8d6tyRjV1pJ1huip28x9yPAj61ushDN3k/IIF3u3A+qMI9CHi+aZijRgLACTTkqQ67UtFCiod+UmmUdTqJJjUSZstMehfxwcNOf3dwtpAyhLeXK/3UpCYjs2UxqCSBAsyKnj/AMdKztCvHWIBIiG3CgA+GUDobJT3FPjFeJIqMY4lcBTOU17x8J0O2nzsdvKHQlDySE7gKcrl9DzjJSXIu6K/G2D0hdUuqU41R1DhTmUFBKZyyBAHezEgJB9lMAUPoIPXepmovXilSoNOUkBN2wCR5u8x5b2r5X4vxu+r/tDEl4KAGvbXguDpzzGPGxFvGtFTPTfOKqBrImQl69Ggc0aTmVMaDx1snvZb7n1UmRHq5W4T6J2BqtTdFeXEgLQgKAbZp6ckAgToorjYcvytp70ReElNTkVnEGtbWJKKhinp2XUE7KCoMKHIxyFq6YbxK2mmReV34ruF1t1GdApr6pXHnE5oENJcLkTrqn87LNLxHcq2ygVCl9mSOaZg66kgEeW+/K3P1lHVzTZ4X5G9lrU74WsyyG5+SnPGfB7h1ii7Ka5sRcdcd1FLQVrV40yaKvo6dxupQFZVhTVNnSRmJkKnboLR9enAjgPTsP1BxTxHvepYQtxj17FdU412olSZT3QrvakEEGfGzTOPbvYbPbGqS4ANezEaT7UKMfOxBXEWhDvZdq0snTKt4tkb9UGNNSOUWeKkrAQXTOKnzKVosGBVLoEJYvWuaL61OpcOfSSpU7ADb32lfjIpp/gVgZp1UJeqF6xqDkcPx35HytEt+lq5sbV9OWZDrqozKnMTIAj7vDlJkyRx3ZvKo9HrAb9KlaXWqlTpJOXL9m4R5Ty+PKwuJR/SsPH/AK38N68s4qcBiWE/5j+FKkX0e37voeJFEbyu+mq30M1QaRVMJdQrMytSjkUCDEaeY2MWsXe98pTWUj7F03XTt09ShRRTU7bJJ1BEhO0aeHTSLU24ZYrql4mu2qpUst1zVQGlB/MpuSFAylIkjX5HQTFp/r7/AMT1jYp2a65UkfaZfVHEBX8MFUEjUg7fwaW3RESbr0EygCym648UhN30zDyuzNPTpQDnUUqygDSE9Bz+Vlq68e0NPVlDy3HEpbOYyNDIMTIJO2w+Vq502M8ZUSECruRFWB3CaN5KsoiR7WTSYnbysoXVjqrZce/fFxX22jIChKaBbpJBB2RmhUHrHnaXMlZsVFscL9wrIP8AFK4kZkoS8SBKRkIjTp08fumyVf3Eq77wu6tu8Xc86h5pbZ+02RGnLUaRaETjWpeSr1bBOKXs47uWhQiZ0ABWtMeI5aDTkC7i3EeSWOGuJVAOEZnDRpCvH+/5yOWnSy8RKVMU8I2TPrHKuiUUUDSu1Z+ybCU5u0ImAQOcT+HMWkXhXiS8sU3HfOCb1ojS3gmnVU0iiSULOgkKmQAS2YnYnpZgGsq27xUmvudy7X3oUmleWlSoOhAKCQenXTlZ2YLvd+570oa5ylS2hpwIIJ7xbXorTaQD8h0NqfEVBJiWFvbB+1bZ7P77Dmb6kW+BXE8Z4VLieFSx0v7ZlpIz2kjOZtviRb4EqIWvXKHFVR+8qhVRUPN5lq9mPAdQCD4jnuLMvGnBm/cc4hQ9hlyhbX2U1Hbr7FKdoMwZ3/QWnTj1cVJhjEAvGku6p7OsUqsS+kFTTZUZWM0QglWwnpG9kHA1U+L5YQ240n1kAkKkJESMpjbc+fXoqLFGYjRR11N74B+BI1B8xsV0HDuJ03ENBDXRfckaD8CRqD5g6FQ5TeiVj52O0v24G1Exl9ZcUQNdTCNBpZTpfQ8xK4QmsxbdrZKspDTS3IjfeNRa3rWFapaO1XWspChmkJKhGkHkddNdec+Jn+y1OVwp5R1mUtK056Sevl/EbVzidUTv9F1gwyl3sqnU/ocshKVVeO3SNSeyu4beEufUHpZRY9ELDAbAcxFfLqyTJSw0jbkBJtaZGGmEJzqedJCgogJTJJiBz+gOkW4TciEAhSHO/JiYKdgBtGv1vaBr6o+8pfZ9KNmqrTfonYZaWVGsvd5IPsqLYkf7v52UKT0fsNU/ZB3DNO4GNQVuLOYwPaP8Xlt8bWabuWlChNKoKToSXOe/4/WtjTdy0CSoFoe1oSkqmDr5+H52Xi6h27inFFTt2AUJXbgXDLBTODLgToAMtz06lCOUloz95s+rooblpwGkNmlHJLDSGgPIIQCdvn5w+P3PStrA7MAZZgJgqP5/Wth2brYBObswDzgAx5fW8WqymSTclWYmMZsm+w1ch0NZezmukVi405aDp9x95pd04ceAL9JXuneVKUvTTSSdenL32WhRUxIW29l5SOUaR0+tbdZGEAq7VB70HMYAHj9c7VHQOKutmaAkNOHcKJhv9ySFCSOwSmBqI1AsYbuW5m47G6G20mATkA08vP7uVlDtqFtBIrGUyZUE5QVeceHSwa6q6wCtV4ITzOVOogH8p+NmFI52pS8SwIuqjpGUhlqjSlOqUw2FRGnUSPrwsE5TNrgtMH+X2RHx9wsYVet3JgCsUtJOgAV3SNNJ0/r7yXdv2hR3cjxKCZkgGORk6g62M2id2QnVre6LP3UHQtSGlFSdhpqOu5H4WKO3C6opyJAO8ZiOh+Gm3Ww6sSop1do3SurChzdIG22xA8x8bBqxe+EZWqdrUCAFEwZ8InpYwoXdAgGtZ3RReE3nPZebbBkBJKtzP6eNtNYBeD6KhNZSHs1AhISoyR4z1sI5iyvjMCGysgJhEzI+e5/GwVRiW8XEyH1jUgSjQ7eHj99jtoXboD6xtk/P7ZcSUKzC87tUlOmjKiEjTYzHLfx99tK4j8SUISE3nd5WDObsBqYGk8vK0dKv+8icq7wdKSc0JVA+XX8LAOXi8pM+t9pJEJB3G+8/XyteZA4blUjMw7BSQOKnElpSQTc4QTP2jJOkzz6e7ewiuN+L2QFVTF1QcxkLCec6SnYA/UWi92rzqUAkEidCZ28zOwGuvKxN1RSrRqSkbAxHT6mx2w90B0o6BSpWcfsRBoJYo6AOEAKVIUJ027u2vjZrX3xjxXel21VEWqTtXEgpz06VshYUNCnIOh52Zay8ApAaCQInvR8dPrawQhOZAAUMw0Bkb6fKx2RBqrvkJRlXETH+RoMN3RTKEaN0CAAN+evT9bFHsfcSFLGW+WGFqOuSibiD5iwKgDopSjmkRmMgmR+FgVFKTq4paSe7Gnu/Tw+NgaBVje6MHFWOnB9piysGYf7JtCY6zA0/XXaxWovLEdQU9tiy/wDKiFZGqstlQiY7vn91uAXQUpCgEyZ028Z8/vtyEuocXGaTsUp9rTpznXzmzg3TINTtbmLrl7Xy8oHKS7eDqwdOYKiNvvOttUlBRLcU65QFUnNJk8geZ6j+ltLW4UQgqzAxruDMGIt3SuvocDhWowRuQc3L8OnP32dR6pXVS0ySU+p02hj2rZYPtyNO3ph/mKp99sslJQFhOkqk+stoumofK3AU9m0pcgITKtJkTOnyNjVTc2InnSpm4a/KYVpSOmTtvl152Qnb9vQKbp6C/KhDRECnp6hRbASBsEq8uXjpYNqkxHey1NtUt61OhHdp3nJgEaZZBOwInxtS5gBuSEct0tYpxtXPiZgl1d3VqQNIWyWxqT/N7o026zZHqMP3nTuqradqkYcLilkLrWklQJgiCQQY+YieihQcLOIN6PpF28PcSViglSvs7reWM0dcu420mY1nazkpfR343VQzUvCXFMNZc2e63G58swE9PztA1MQ3ePVOIXuHstKY9VcL7r/rbd80DT5CSsKrmSDBBMZVEjYaa+Nuq25qWuaT6xiG6kPQUrcQtaiobx3EEHbadJ+Mi0Xoo+kZeaQin4RX6hCJyl5LbQTrsM60mP69Ic9yehX6StVSClc4eu0aHe852960yUriIlIXO3jyO06jdW0zd5B6hSbTTONshUFVOF7tYb7J7FlGsxnX2bbxkEAjTs4ECdORjzBN2luWnGUX1UOZe6ot0x3CpnWI5/W1nF/s9PSFvNTa1ruVlKcpKam8yD/iCVIQuJJPlpzFlmk/ZncWKggV2NMNUjagZDa3lkHwHZwd99NQNLQ+0qJo1kCIKKpOzFUhuooGW0KTXVxCQMoLKTz3/vAZmJ0+dnDc3Fq98K3rT3jdNVXJqadYWy52LWUKSTCoVmze8QOlrY037Ma+mwlV48TrvQtRmGaNahm/3iJEk9DZMxF+zpuXDFCqvvvjIo9+EMU9zAKXrolOZ5QO3NJFgvxTD5Rkc64PSyPFRVzDmYLH4qHB6eHH2mZLNHjC9W2EiVdl6sjLMAaCnEan8+lm/iD0x+Ol/oLN4Y1vt9CFKyA15ak7QOxSjx66RtvaZrg9FDhJdKc19sX1f6jGdNTVdi0pRkghLSQeY0JIHjZZuX0ZOC90ldY3gxx9Di5DdXUuOxG4R0GsHlt52qNOEQm8cI9ArPh8Rfq5+/mqdsf2y444zujD1BSJrr8visZoqRTtQ4S6tailCVLdUYSMx1naY5C3tJcmG8V0tz0FCu4ni5TUrLLii82AVIQAYlQ5i1W+GXCvhZhDiBcGMU4Rum5m7qvFisXWqWtlFOlCge0KioJCQAfaEWtu/wCklwIYPe4s4ZJBBhu8EOH4JJNtyidBXMuPZAWVWRzUj7O1JQYw7icqg3UQr/7K3+CrYrCOKlIn1Sn22L+vyBFir/pR8CpAZx9T1OYSn1ajqXfeMjZ6WAX6T3CpwD1SsvurB1HY3DWxy1lTQHS1zkUzd3fVUjJO7QNR5WAsVPCJoWxvq+o/9FsHCy/XR9re1I3m6NqXr8R4WSXPSZwiof6phXF7+4GW6w3P/tHE2BX6Td25Qqm4b4sVmnKp31NAVB8X7SD6VnvD1TCGof7pSneHBKrvFkM1WJmxJCu5RncajdzqAbVmxxga98GY2vPD9L6xeKaZaFpqSlLYWVoCyN4AGaOZtPT/AKS95rSk0XC6sUVSP9Zvhhr/AMAXaqHG7h9jbivxDvXiS1d1JdDN7ertPULd4LeUhTLSUJUVhCArMEE7CIPO1aplpXgAO1VmlhqYjctKdSLvv0hI/wBTaXsErr2UHcHmrz+tjjWHb1qUBdRftzMzoUrvJCoPP2Z+haJmOEGOaZCUqv54ZBlRlrF6D8tvh8VFrgviaponKt++UuuU4HaN+sLV9nJ7+520BEaSD1Nqdqf8SuEVBP3VI5wuttwCpxVc6Ecsjrq9IP8AKj8bCfuW4WiXKjiHd7Ujf1d8GB5pHL62tF44NPPAl6tQhURISSofEHx+Fke/eEF7oeU5ddTX1lSEStptZSlwRqUDYK0iOc/GYfT7XUeXPvZTU2xw4aSpD/FBpa1CMrVMCJ8QpwWBdvnhPSJUl/GV5vKQdS1TMEGOUdrM9P6C1Z14dShYBaqMqVBCwXFdomdIIInQmOs6RYw1hdCnVKepMhnvZ1SUj+EHqJ6RGp8bSAi7KNpFLl+474aM1Ieu3EN4IQtvs3C/SElW2VGRtKlTuc22h11Fo9vPixciVrTSUd81SUHKFhumbS4BzAceSYPjrrrEGxWnuO7kKk0KAf4wo5ZPgTM6E6EcuekdruG7lhJpmG8hjMkwVpM8pGh5QdN9RFpBzNgFAxPKTjxKp3iS1hy9FOJBVDtVSIEDxQ6uN+li7vEyscTKML0YiCBUXupOUxpORhX1O1l5i56NLKltsU4UCQlWbKtI0kKH5aanewrVJRNlTdRT0JSD3ihsJIHOQN9ht8LPnaOiXKd3TUVxBxSs5qTDtxIJMgfvCpe93/m6RzB/G3CMe46qSDSsXAjLCu5d9S8oeEKdQeXSz0TQXeQFt9kpCBJOSU+Gk6aA+/3ACqpGgpCXVNO51EZ0yABGsGJn4fCz8xvZLlO7pkvYp4iVSM6bzo2FEQFMXFoUxvKqhfjuOVg6W9OKDJNSxiOrQp1tSF9nQ0SISqApMqbXuCefLY8n0UNtj7GrKm82Ud4kaK3zcvCZ326YtN3kOIDLqEoSJUEqUknfcanSN+k72WcHonEVuqYwRjqpSttWJL33yryro2gCf8SKcR7jYBVyYp0m9b/cBUZUq93U9Ng2pJPhFpCUllaA2h1LkwSYKUg7zIEQPvBsUUulpVQXSwUju65dI1PTmT8bLOegUTE0dUyH8PXu+Slyvr+zMEB29q5QCifF73e7xsWTw+bMO1VPRvISSYqC4/mMHUdotUc9OelpAVeF3lkOvusLGXRSklJjTYyZkT5/G3CL8uLtBMoTsC4nuxGpJEjl79dtLPnd0S5bB1TGRw1p1EH1G5BK9k3dT5iNdNUmDPjpPlYdrBaaAksIp2ERKizRtInXllSIj3naN7PE3vQq7zb5dBUDBToNDrJAkydPZHzkmu+Wm1JUgVaFH7Pvagxz3gQAddN+tnDnkJZWJCbw29TAJVfFfKoltuoPenkIIM6kRH52DXhRJKfWLwvFQX9p9pUq+ZnqOnMbQBZWXftRHbJSHz7MklKzroBMgnbXYR11sD++CSta6aoZW0Ncq8xSRz5wIB5nppzcZioEsCSncF0SnEreS4pYO+dR7x8NYI0Mc9ekW03hWhbC0JpAk94lJSSCkEEnczrzAIGtlJV5vOIWpCG3mykthOvtGQDI56fLwsXVV1jTZbZLK1ZSpKVNnQ6A7nYGOg231tINclmaNl1hDDtxXzxpw5T37Tsro3s6g2oANurbaUpCFA6EFaZy89iIMWn9jEl5YqvlzA1/YBr0XcU1qLxqqqmy0fZIVDXZrnvFQKY05KUIi1WcTuV1Wu76uhdcRVUj0h5iW1NrSEqSpJA0Vy0gz42dV6Yu4yY2ww/cZxqttpZQy+EsNNrdbIMoUtKQog7EyJEzzFuF4r4Rq8eq2VELm2DcozFwLDcnO3LoSQdjbVrdV4/x5wLW8UYlHV072WDA0ZnPBidcnmMDRZxIOxy6tab22h3DLNFhvjBRP07Yq7vbry42XSUF1pClQZ592NxzHW1o6PiLhFAz1d1OUpWAUn1dDiDM7BKirYHcazpBtVbDjTrt+XdVNLU1UUCnUrKVZVQW1dDJ1AgDkT5B6uVV4uPLTTKdXkJXl7clYSIAJBE76QNJ1EgkjthBmAaTey9cikMLA29/irP3XjnB1WhIZxDQU7RkZXkGmgiJMupHjqNJ18bOWlFDeQDtBX09Qkpkpb7N0AwRMp3G+418TtTF5+pUwXKt1LaHgMgeQEAhOg7pTBAHxB5wMum6ZapyJCnHSrs3GYCgYTqNATHdOnTnpEHUAJuHKy2vtu1XWTdQOSUtlQOVUJUmdf8ACOXmNJ00t0q70A5nKlSQpUkIcAIBVynyUPdp0tUCjvDiC0hTl2X1fA7NIzoTXfZo0MjIVb7ciIAiJss0/EHibdak071ZQuIU57VU1lCtCP8AZpH825Mkk66pFoGiI6ogrmnopl4jYeYqlU9VSvpFSiYzKSpQJgaeACR+PKyHhOmbqcQ3dR3kvMHXkodQlPdTB2kaakRtzNmfR8Uq281IZrrspXcy57dmvaSUgqEqLWYmOYgzy5TZy09Sh9wuMEJDsQpvQnnAjmJFrL4HS0r6drspc0gEbi4tcfDdYuKwurYJYY3FjntIDhu0kWBHmN1MVTfN51t+U+GV4YfXRrdearHHWJplU+VWQhR0JPcEeKhHMV0vcUdx4srEXQ32lLQV7jbB1IyoWYEx00nWz5expj+vu9+kavvJRAdn2/dS4UjQwvLJ2Osz42it9MPhJJcSlWQEnXvEhRHQmBPxtw3CnCtRw/JI6fKA5oFmFxDiCSXnMBY62AF7DquA/RzwTWcJyyyzljWua1uWNz3BzgXEyOzgWcb2AF7DS5VmsNYquqruhkKp3FLZRBjINDoFTM7AHnoDvZTViWlObLTOKBOoUUwPOBppGw+POEuHN9KFMaEujImQlJPeme7O/L7vi+fXTBKmkDmFFHdE6GT/AE8J3t1hpGbr2AVjraJ1uYoQCU/u5sAjm4JgA6kwNJP5baFjf9Q6c6adhMa6EmSRuPCSeXKzdSl1WchLsA6qSIPu68tRofDl0hS8ktPyATpl0E8j1PjHLzswpWBLxLylty/60qUS0hBEJB0Ma76ncxYuq+LyKoXVFABJgNaRPXroPgbJqFvpTLiG1BRGx0jTUcvrnvbajUkFJbSde6pK5I8uh/W0xTt7KJqHIyu86xeZKn1EKJEDRQ8JsCutqV5VhyrOWU6E6fdYLs1hWU5kJI/iSIHU/fbspA/hUpO0FcTIM2fkt7JxM7ugvWH1JKnVPqVEd9R39xsAVreT2sgpIAgqnTWOdjhZZWcnZEawClZAHU6nTe3K6YkmUmdCCe8PKRz+tbOIgOijzSUnLKlQFVMLP8qpBMTttJ+t7bS482kdmXCD/MmQNvKCPLlY0tlSVD7Qmdio6K8+v36WAXTuEBSU6b+0J003jUfl52mIwoGQoALcC8wfWMomcse/pHy/DjtnnJIWNdQZjwgjmI+ZsZTDUrFMCr2ZP36E/XPa3LimXQApteaRplAPwjXTxsuX2Tcwou44VJJKDmmJ0Jnr9dbcqq3ZKVIzQCnlGU+A2+vOxtLdJIOdKSoSBHmSTrblLTCSFNme7qMunhty38/CzhijzEALzfSlPb04Cd53mPH6+63K6wJEKCsxI0kfHXl8t7GFKbCwg9nIJ1PTkInzsEopz5EBBVJzKyyCfGBp7rSDFEyIquqg5gFZUgaxrE6GPrym2/XEFsqXyjQkAkwNd/rnYXKkKAbCAADGVMAAbaH3fW3DiFApUhzWdZM9Nf0/M2kGAKJfbVBqfZcSU5Uad0gJnT6n9OXK/VlpMRBEyG5392n1rbl5Li5WlaSgnMDmgdesfXlYB2kfCitJQlYOhB5kHXrM2nlUOYhCllQUlJAM6Qogzt+Q08OdgloZdSoJnX2SVExPhPy+Ns7GrbK1qqEmASFIke7pyPz87BlL+XKpxHOddOeo0+ps6RN1oNNJglaTAjQe/bf+vxDKCgGFk5QBlOm86dbDEVRzIKGgpAiTtMRy35WBWFIT9qwmJhScpk/HXl8rMLBMgVuOZjDoUR4gHaJmN9/CwROQ9nn1Om2/OZnryn4xYwuoaQQh6mASNAdh5wBHwiwSXmFZZS6uUQClJiI67dNP62fMAolpRZxwgLJC1DLBAZIJ8d425WFZX3iClMDUmDPUHXXexlLSVABgOhEAkrQe9POfCxynuta05WkZE7AKEQSfKfqLRzgBPyiUXBbUMzjDeY6q052yywm6k5R9nTnTfOPztlm5gT8or0wZwThinbS3TXFdzKEmUpRSIAB8gLHmrno2khKWmwBsA2ALZbLeVgdV0Gdy0qjbKoBKR4W0u7WtlLUQRbLZYrWA7ooe4bFAKp2Gj7BV5m2nG2kgKKJ3tlstLKAjNcTugHkJb1CUkCDqLaLh7PMEI3ja2Wy0BoUdutrpJvi8HKCgerkNpUW0khO3ztA2IfXMS1vrt617izJSlCEhKU+Q91stljUwvclWRskZvD1It9CFOOHOQCdOc62DduijUrOsLISAEpCvZH185tlstdCgVym56GrCqZ9suoUQlQdOcK23HPf5WEduW6KCpdpqK7KVoMLLQIZTJyxrtvJm2Wy1iJxA0KC9oO4XTLRynsilsBIMBOmv9bDVDWd1DMx2baRIG8yT99stljBxPVDygdEEpASoJUSqVRvGwKv+mwqWm3qBtRSe68tKZMkaCT56Wy2WK1QsgU0qIVqNEhQ7o/mj8PnY5RUaXVOsKWcvYrVoNe6kqH/hj3m2Wyz9VFFXbspkqKyCQJkddT7uXS2qOnZYrEvU6AlbUnWFBWmxHMRI99stlphN1SjeeGqOmfbWwrI2+2l5KAn2Aqe7PMSP68yBwzRvl1pSykqiFoSApJmMwPUWy2WcqKjPibgq6jcS8bNlSKth409UiARUhD3ZhU/wr00Vrpoc2hETON06aZBNOkl1OsaaSdumomR1tlstpUurBdZtTo/REXbwp2qdmKPMmodSiFKScpIKgoynX2R79fCxJGIH2aRtSW5z1HYyojMNN5jXYafdbLZa6AFTLijLd+VL7zLSm285SjvlM6qAO31NiNZfl4U/YKC05XwNEpCSAonnsYyjlbLZYgaFBzigXb/vM1DZTUHtErgLKUnTaNttefSwrNbeAqzRVNe68lbYdGuUCTEQPf8ALpNstliBosglxuunu1p3W3nahx77ZDQBMQFJGsjXn8h4yHVKfQ80TVOkuyEd49xMAwevPpvbLZaQaFG5RVAL1K04mEdogGBMA66767c/0t01VrQ2zSVEuKdcLedBDcRGpAGvlNstlnsErlGEMNKdLRTlJQRmbhPs6D8LctUak6B9UZtAQDsSOevLrbLZaKi4lCKu1vtkpQ6tvKJlAAOhJifed53sXCo76VuEKWAMypIlWokACPCLZbLPZNcrhLlMhwNOU5UaghvMFAQB10M7+HzsTvC8U0Ty1IaWtwqgqW4DMJ7ukaxHPW2Wy0gmRBu+UVqkKFClC4HeK5jfbQR7R2s5rvutN5NBYcSyAYhLST1EzuTpuSTtbLZZyooze2EVXXRfvB+vbqUrrEICAwUEFU6zmIO40jl5QIzdgpyUNPqBacRlMCdyPw+dstliN2UXbqHP3O1R4kYUh0yiqSlZCRK5cgmDI5HcEa+c2LouGuFqyo9USy8hQmSVJyxlzQAlKT8SYn3Wy2WqMO6skCy1VYCw5ddUlhDFSop3UmtfQFa80hcH+vW3bOFsOuMrccupp0AjuvkvTJ/xk8+kWy2WJcqIAXKLhwk0pCThagClOJRKGWwNdATKDO/XrtJsqP4eobppk1TFNSDNlkIpkp5x4jY9JtlstEbqZAXLK1vNpUVGUCRJkSdNtjsN+mkWab6lqvNWVQT2qgnRI0BII2HI9I91stlixbocvdNPGnES8rjvMYVoKRDqlIceNRVurcGilCAhJTHs8jtHS0fL4iuuvVbF5XO065SEoK6d0tJcAAPsqC43jfr1tlssOX7yND9xODhpxLN5X7+6mLmNOA22rtDU5iSZiQECYg+Oo1EWsaUtOAKWlUZUq7qoMnflHytlssN26kzZdCjStAfS4tMNhccySgHfyIHutrK4jVLypHdBIB5+NstlmTrpLaj2gz6A90aiNPAj6FttUwfUcrriYWUmSFAwT18rZbLJNc3W3KFxnKv1mQnomD13mxFbzqXBnWXCFhBzagmdzbLZaQCe65XUuJWhCsqtABpAjy62EpXy6tqnEhTgHfMGCQOUWy2WSQRu86J2mioNSFHMAAEERKlDqeSfnZNQ4tRJC4gpTGUEH5T87ZbLOkuXM63FJKh3RMx8bbU2kgkySVGJiNAP1+NstlkkFjl3KShxwvggAQnJpE7b2KlqXFJBSNRsnczbLZZimQSGO2StebKE92IkkgxMm2zTDKgpVAM6ETsT+Xztlssr6JINSHGioJdICdI98WGp23XEBZfmYyhSQQBvHQ2y2WinsEcRdilf3ryViNuzjoeR8bCfuanjK44sxochiZE856RbLZZnEp7BdMXDR1CFrS4+nKRopzNM/Cxw4bpKVwICyrInQx485Jm2Wyw3OKK1oWk3OwpwABoKWvJJaBgxvYJ+mQyjO0lCeSRB0IJ13tlssMON1INCLJRKUPJUU5yrTTQDbXr42Gp6P1l3slrSBJSO4DtP5Wy2We5TtAXTNBTPFxtTSBpmJyzMkD8vhYZq7aRJClJVBP8ACQDqAdZB62y2WGCVKwuhF0FJmMsyZ3J3tlstlpJ7L//Z" } ] } ] } ] -} \ No newline at end of file +} diff --git a/samples/js/node/notebooks/question-answering.nnb b/samples/js/node/notebooks/question-answering.nnb index 3b53ca163f0826..c1bf61063b5e15 100644 --- a/samples/js/node/notebooks/question-answering.nnb +++ b/samples/js/node/notebooks/question-answering.nnb @@ -17,62 +17,10 @@ { "language": "javascript", "source": [ - "const {\n exp,\n sum,\n tril,\n triu,\n argMax,\n reshape,\n getShape,\n downloadFile,\n extractValues,\n matrixMultiplication,\n} = require('../helpers.js');\nconst tokens = require('./tokens_bert.js');\n\nconst { addon: ov } = require('openvino-node'); \n" + "const {\n exp,\n sum,\n tril,\n triu,\n argMax,\n reshape,\n getShape,\n downloadFile,\n extractValues,\n matrixMultiplication,\n} = require('../helpers.js');\nconst Tokenizer = require('./tokenizer_bert.js');\n\nconst { addon: ov } = require('openvino-node');\n" ], "outputs": [] }, - { - "language": "markdown", - "source": [ - "## Download the Model" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "const baseArtifactsDir = '../../assets/models';\n\nconst modelName = 'bert-small-uncased-whole-word-masking-squad-int8-0002';\nconst modelXMLName = `${modelName}.xml`;\nconst modelBINName = `${modelName}.bin`;\n\nconst modelXMLPath = baseArtifactsDir + '/' + modelXMLName;\n\nconst baseURL = 'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/bert-small-uncased-whole-word-masking-squad-int8-0002/FP16-INT8/';\n\nawait downloadFile(baseURL + modelXMLName, modelXMLName, baseArtifactsDir);\nawait downloadFile(baseURL + modelBINName, modelBINName, baseArtifactsDir);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/models/bert-small-uncased-whole-word-masking-squad-int8-0002.bin'", - "" - ] - } - ] - } - ] - }, - { - "language": "markdown", - "source": [ - "## Download the Vocab" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "const baseImagesDir = '../../assets/text';\nconst imgUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/text/bert-uncased/vocab.txt';\n\nawait downloadFile(imgUrl, 'vocab.txt', baseImagesDir);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/text/vocab.txt'", - "" - ] - } - ] - } - ] - }, { "language": "markdown", "source": [ @@ -83,36 +31,10 @@ { "language": "typescript", "source": [ - "const core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\n\nconst _ppp = new ov.preprocess.PrePostProcessor(model);\n_ppp.input(0).tensor().setElementType(ov.element.f32);\n_ppp.input(1).tensor().setElementType(ov.element.f32);\n_ppp.input(2).tensor().setElementType(ov.element.f32);\n_ppp.input(3).tensor().setElementType(ov.element.f32);\n_ppp.build();\n\nconst compiledModel = await core.compileModel(model, 'CPU');\n\nconst inputs = compiledModel.inputs;\nconst outputs = compiledModel.outputs;\n\nconst inputSize = compiledModel.input(0).shape[1];\n" + "const modelXMLPath =\n '../../assets/models/bert-small-uncased-whole-word-masking-squad-0001.xml',\n\nconst core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\n\nconst _ppp = new ov.preprocess.PrePostProcessor(model);\n_ppp.input(0).tensor().setElementType(ov.element.f32);\n_ppp.input(1).tensor().setElementType(ov.element.f32);\n_ppp.input(2).tensor().setElementType(ov.element.f32);\n_ppp.build();\n\nconst compiledModel = await core.compileModel(model, 'AUTO');\n\nconst { inputs, outputs } = compiledModel;\nconst inputSize = compiledModel.input(0).shape[1];\n\nconsole.log('Input size:', inputSize);\n" ], "outputs": [] }, - { - "language": "javascript", - "source": [ - "console.log('=== Model Inputs:');\ninputs.forEach(i => console.log(`${i}`));\nconsole.log('=== Model Outputs:');\noutputs.forEach(o => console.log(`${o}`));\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "=== Model Inputs:", - "input_ids", - "attention_mask", - "token_type_ids", - "position_ids", - "=== Model Outputs:", - "output_s", - "output_e", - "" - ] - } - ] - } - ] - }, { "language": "markdown", "source": [ @@ -123,14 +45,14 @@ { "language": "javascript", "source": [ - "// The path to the vocabulary file.\nconst vocabFilePath = \"../../assets/text/vocab.txt\";\n\n// Create a dictionary with words and their indices.\nconst vocab = await tokens.loadVocabFile(vocabFilePath);\n\n// Define special tokens.\nconst clsToken = vocab[\"[CLS]\"];\nconst padToken = vocab[\"[PAD]\"];\nconst sepToken = vocab[\"[SEP]\"];\n\n// A function to load text from given urls.\nfunction loadContext(sources) {\n const input_urls = [];\n const paragraphs = [];\n \n for (source of sources) {\n paragraphs.push(source);\n\n // Produce one big context string.\n return paragraphs.join('\\n');\n }\n}\n" + "// Initialize BERT tokenizer\nconst tokenizer = await Tokenizer.load('../../assets/vocab/vocab.txt');\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "## Preprocessing\n\nThe input size in this case is 384 tokens long. The main input (`input_ids`) to used BERT model consists of two parts: question tokens and context tokens separated by some special tokens. \n\nIf `question + context` are shorter than 384 tokens, padding tokens are added. If `question + context` is longer than 384 tokens, the context must be split into parts and the question with different parts of context must be fed to the network many times. \n\nUse overlapping, so neighbor parts of the context are overlapped by half size of the context part (if the context part equals 300 tokens, neighbor context parts overlap with 150 tokens). You also need to provide the following sequences of integer values: \n\n- `attention_mask` - a sequence of integer values representing the mask of valid values in the input. \n- `token_type_ids` - a sequence of integer values representing the segmentation of `input_ids` into question and context. \n- `position_ids` - a sequence of integer values from 0 to 383 representing the position index for each input token. \n\nFor more information, refer to the **Input** section of [BERT model documentation](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/intel/bert-small-uncased-whole-word-masking-squad-int8-0002#input)." + "## Preprocessing\n\nThe input size in this case is 384 tokens long. The main input (`input_ids`) to used BERT model consists of two parts: question tokens and context tokens separated by some special tokens. \n\nIf `question + context` are shorter than 384 tokens, padding tokens are added. If `question + context` is longer than 384 tokens, the context must be split into parts and the question with different parts of context must be fed to the network many times. \n\nUse overlapping, so neighbor parts of the context are overlapped by half size of the context part (if the context part equals 300 tokens, neighbor context parts overlap with 150 tokens). You also need to provide the following sequences of integer values: \n\n- `attention_mask` - a sequence of integer values representing the mask of valid values in the input. \n- `token_type_ids` - a sequence of integer values representing the segmentation of `input_ids` into question and context.\n\nFor more information, refer to the **Input** section of [BERT model documentation](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/intel/bert-small-uncased-whole-word-masking-squad-int8-0002#input)." ], "outputs": [] }, @@ -158,14 +80,14 @@ { "language": "javascript", "source": [ - "// A function to add padding.\nfunction pad({ inputIds, attentionMask, tokenTypeIds }) {\n // How many padding tokens.\n const diffInputSize = inputSize - inputIds.length;\n\n if (diffInputSize > 0) {\n // Add padding to all the inputs.\n inputIds = inputIds.concat(Array(diffInputSize).fill(padToken));\n attentionMask = attentionMask.concat(Array(diffInputSize).fill(0));\n tokenTypeIds = tokenTypeIds.concat(Array(diffInputSize).fill(0));\n }\n\n return [inputIds, attentionMask, tokenTypeIds, diffInputSize];\n}\n" + "// A function to add padding.\nfunction pad({ inputIds, attentionMask, tokenTypeIds }) {\n // How many padding tokens.\n const diffInputSize = inputSize - inputIds.length;\n\n if (diffInputSize > 0) {\n // Add padding to all the inputs.\n inputIds = inputIds.concat(Array(diffInputSize).fill(tokenizer.padToken));\n attentionMask = attentionMask.concat(Array(diffInputSize).fill(0));\n tokenTypeIds = tokenTypeIds.concat(Array(diffInputSize).fill(0));\n }\n\n return [inputIds, attentionMask, tokenTypeIds, diffInputSize];\n}\n" ], "outputs": [] }, { "language": "javascript", "source": [ - "// A generator of a sequence of inputs.\nfunction* prepareInput(questionTokens, contextTokens) {\n // A length of question in tokens.\n const questionLen = questionTokens.length;\n // The context part size.\n const contextLen = inputSize - questionLen - 3;\n\n if (contextLen < 16)\n throw new Error('Question is too long in comparison to input size. No space for context');\n\n const inputLayerNames = inputs.map(i => i.toString());\n\n // Take parts of the context with overlapping by 0.5.\n const max = Math.max(1, contextTokens.length - contextLen);\n\n for (let start = 0; start < max; start += parseInt(contextLen / 2)) {\n // A part of the context.\n const partContextTokens = contextTokens.slice(start, start + contextLen);\n // The input: a question and the context separated by special tokens.\n let inputIds = [clsToken, ...questionTokens, sepToken, ...partContextTokens, sepToken];\n // 1 for any index if there is no padding token, 0 otherwise.\n let attentionMask = Array(inputIds.length).fill(1);\n // 0 for question tokens, 1 for context part.\n let tokenTypeIds = [...Array(questionLen + 2).fill(0), ...Array(partContextTokens.length + 1).fill(1)];\n\n let padNumber = 0;\n\n // Add padding at the end.\n [inputIds, attentionMask, tokenTypeIds, padNumber] = pad({ inputIds, attentionMask, tokenTypeIds });\n\n // Create an input to feed the model.\n const inputDict = {\n 'input_ids': new Float32Array(inputIds),\n 'attention_mask': new Float32Array(attentionMask),\n 'token_type_ids': new Float32Array(tokenTypeIds),\n };\n\n // Some models require additional position_ids.\n if (inputLayerNames.includes('position_ids')) {\n positionIds = inputIds.map((_, index) => index);\n inputDict['position_ids'] = new Float32Array(positionIds);\n }\n\n yield [inputDict, padNumber, start];\n }\n}\n" + "// A generator of a sequence of inputs.\nfunction* prepareInput(questionTokens, contextTokens) {\n // A length of question in tokens.\n const questionLen = questionTokens.length;\n // The context part size.\n const contextLen = inputSize - questionLen - 3;\n\n if (contextLen < 16)\n throw new Error('Question is too long in comparison to input size. No space for context');\n\n const inputLayerNames = inputs.map(i => i.toString());\n\n // Take parts of the context with overlapping by 0.5.\n const max = Math.max(1, contextTokens.length - contextLen);\n\n for (let start = 0; start < max; start += parseInt(contextLen / 2)) {\n // A part of the context.\n const partContextTokens = contextTokens.slice(start, start + contextLen);\n // The input: a question and the context separated by special tokens.\n let inputIds = [\n tokenizer.clsToken,\n ...questionTokens,\n tokenizer.sepToken,\n ...partContextTokens,\n tokenizer.sepToken,\n ];\n // 1 for any index if there is no padding token, 0 otherwise.\n let attentionMask = Array(inputIds.length).fill(1);\n // 0 for question tokens, 1 for context part.\n let tokenTypeIds = [...Array(questionLen + 2).fill(0), ...Array(partContextTokens.length + 1).fill(1)];\n\n let padNumber = 0;\n\n // Add padding at the end.\n [inputIds, attentionMask, tokenTypeIds, padNumber] = pad({ inputIds, attentionMask, tokenTypeIds });\n\n // Create an input to feed the model.\n const inputDict = {\n 'input_ids': new Float32Array(inputIds),\n 'attention_mask': new Float32Array(attentionMask),\n 'token_type_ids': new Float32Array(tokenTypeIds),\n };\n\n // Some models require additional position_ids.\n if (inputLayerNames.includes('position_ids')) {\n positionIds = inputIds.map((_, index) => index);\n inputDict['position_ids'] = new Float32Array(positionIds);\n }\n\n yield [inputDict, padNumber, start];\n }\n}\n" ], "outputs": [] }, @@ -186,7 +108,7 @@ { "language": "javascript", "source": [ - "function getBestAnswer(question, context) {\n // Convert the context string to tokens.\n const [contextTokens, contextTokensStartEnd] = tokens.textToTokens(context.toLowerCase(), vocab);\n // Convert the question string to tokens.\n const [questionTokens] = tokens.textToTokens(question.toLowerCase(), vocab);\n\n const results = [];\n // Iterate through different parts of the context.\n for ([networkInput, padding, startIdx] of prepareInput(questionTokens, contextTokens)) {\n // Get output layers.\n const outputStartKey = compiledModel.output('output_s');\n const outputEndKey = compiledModel.output('output_e');\n\n // OpenVINO inference.\n const inferRequest = compiledModel.createInferRequest();\n\n const transformedInput = {\n 'input_ids': new ov.Tensor(ov.element.f32, [1, 384], networkInput['input_ids']),\n 'attention_mask': new ov.Tensor(ov.element.f32, [1, 384], networkInput['attention_mask']),\n 'token_type_ids': new ov.Tensor(ov.element.f32, [1, 384], networkInput['token_type_ids']),\n 'position_ids': new ov.Tensor(ov.element.f32, [1, 384], networkInput['position_ids']),\n }\n\n inferRequest.infer(transformedInput);\n\n const resultStart = inferRequest.getTensor(outputStartKey).data;\n const resultEnd = inferRequest.getTensor(outputEndKey).data;\n\n // Postprocess the result, getting the score and context range for the answer.\n const scoreStartEnd = postprocess(resultStart,\n resultEnd,\n questionTokens,\n contextTokensStartEnd,\n padding,\n startIdx);\n results.push(scoreStartEnd);\n }\n\n // Find the highest score.\n const scores = results.map(r => r[0]);\n const maxIndex = scores.indexOf(Math.max(scores));\n\n const answer = results[maxIndex];\n // Return the part of the context, which is already an answer.\n return [context.slice(answer[1], answer[2]), answer[0]];\n}\n" + "function getBestAnswer(question, context) {\n // Convert the context string to tokens.\n const [contextTokens, contextTokensStartEnd] = tokenizer.tokenize(context.toLowerCase());\n // Convert the question string to tokens.\n const [questionTokens] = tokenizer.tokenize(question.toLowerCase());\n\n // Get output layers.\n const outputStartKey = compiledModel.output('output_s');\n const outputEndKey = compiledModel.output('output_e');\n\n const inferRequest = compiledModel.createInferRequest();\n\n const results = [];\n const preparedInput = prepareInput(questionTokens, contextTokens);\n\n // Iterate through different parts of the context.\n for ([networkInput, padding, startIdx] of preparedInput) {\n // OpenVINO inference\n inferRequest.infer({\n 'input_ids': new ov.Tensor(ov.element.f32, [1, inputSize], networkInput['input_ids']),\n 'attention_mask': new ov.Tensor(ov.element.f32, [1, inputSize], networkInput['attention_mask']),\n 'token_type_ids': new ov.Tensor(ov.element.f32, [1, inputSize], networkInput['token_type_ids']),\n });\n\n const resultStartData = inferRequest.getTensor(outputStartKey).data;\n const resultEndData = inferRequest.getTensor(outputEndKey).data;\n\n // Postprocess the result, getting the score and context range for the answer.\n const scoreStartEnd = postprocess(resultStartData,\n resultEndData,\n questionTokens,\n contextTokensStartEnd,\n padding,\n startIdx);\n\n results.push(scoreStartEnd);\n }\n\n // Find the highest score.\n const scores = results.map(r => r[0]);\n const maxIndex = argMax(scores);\n\n const answer = results[maxIndex];\n // Return the part of the context, which is already an answer.\n return [context.slice(answer[1], answer[2]), answer[0]];\n}\n" ], "outputs": [] }, @@ -200,23 +122,9 @@ { "language": "javascript", "source": [ - "function runQuestionAnswering(sources, exampleQuestion) {\n console.log(`Context: ${sources}`);\n const context = loadContext(sources);\n\n if (!context.length)\n return console.log('Error: Empty context or outside paragraphs');\n\n if (exampleQuestion) {\n const startTime = process.hrtime.bigint();\n const [answer, score] = getBestAnswer(exampleQuestion, context);\n const execTime = Number(process.hrtime.bigint() - startTime) / 1e9;\n\n console.log(`Question: ${exampleQuestion}`);\n console.log(`Answer: ${answer}`);\n console.log(`Score: ${score}`);\n console.log(`Time: ${execTime}s`);\n }\n}\n\nconst sources = [\"Computational complexity theory is a branch of the theory of computation in theoretical computer \" +\n \"science that focuses on classifying computational problems according to their inherent difficulty, \" +\n \"and relating those classes to each other. A computational problem is understood to be a task that \" +\n \"is in principle amenable to being solved by a computer, which is equivalent to stating that the \" +\n \"problem may be solved by mechanical application of mathematical steps, such as an algorithm.\"]\n\nrunQuestionAnswering(sources, 'What is the term for a task that generally lends itself to being solved by a computer?');\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "Context: Computational complexity theory is a branch of the theory of computation in theoretical computer science that focuses on classifying computational problems according to their inherent difficulty, and relating those classes to each other. A computational problem is understood to be a task that is in principle amenable to being solved by a computer, which is equivalent to stating that the problem may be solved by mechanical application of mathematical steps, such as an algorithm.", - "Score: 0.5286847737759395", - "Time: 0.045750747s", - "" - ] - } - ] - } - ] + "function runQuestionAnswering(sources, exampleQuestion) {\n const context = sources.join('\\n');\n\n if (!context.length)\n return console.log('Error: Empty context or outside paragraphs');\n\n if (exampleQuestion) {\n const startTime = process.hrtime.bigint();\n const [answer, score] = getBestAnswer(exampleQuestion, context);\n const execTime = Number(process.hrtime.bigint() - startTime) / 1e9;\n\n console.log(`Question: ${exampleQuestion}`);\n console.log(`Answer: ${answer}`);\n console.log(`Score: ${score}`);\n console.log(`Time: ${execTime}s`);\n }\n}\n\nconst context = [\n 'Computational complexity theory is a branch of the theory of computation in ' +\n 'theoretical computer science that focuses on classifying computational ' +\n 'problems according to their inherent difficulty and relating those classes ' +\n 'to each other. A computational problem is understood to be a task that is in ' +\n 'principle amenable to being solved by a computer, which is equivalent to ' +\n 'stating that the problem may be solved by mechanical application of ' +\n 'mathematical steps, such as an algorithm.',\n];\n\nconst question = 'What is the term for a task that generally lends itself to being solved by a computer?';\n\ntry {\n runQuestionAnswering(context, question);\n} catch (error) {\n console.error(error);\n}\n" + ], + "outputs": [] } ] -} +} \ No newline at end of file diff --git a/samples/js/node/notebooks/tokens_bert.js b/samples/js/node/notebooks/tokenizer_bert.js similarity index 79% rename from samples/js/node/notebooks/tokens_bert.js rename to samples/js/node/notebooks/tokenizer_bert.js index 10dc250abe8e51..b23bcbfc8519d2 100644 --- a/samples/js/node/notebooks/tokens_bert.js +++ b/samples/js/node/notebooks/tokenizer_bert.js @@ -1,10 +1,39 @@ const fs = require('node:fs/promises'); -exports.cleanWord = cleanWord; -exports.encodeByVoc = encodeByVoc; -exports.textToTokens = textToTokens; -exports.splitToWords = splitToWords; -exports.loadVocabFile = loadVocabFile; +class Tokenizer { + constructor(vocab, original) { + this.vocab = vocab; + this.original = original; + } + + get clsToken() { + return this.vocab["[CLS]"]; + } + + get padToken() { + return this.vocab["[PAD]"]; + } + + get sepToken() { + return this.vocab["[SEP]"]; + } + + tokenize(text) { + return textToTokens(text, this.vocab); + } + + detokenize(tokens) { + return tokens.map(t => this.original[t]).join(' '); + } + + static async load(path) { + const { vocab, original } = await loadVocabFile(path); + + return new Tokenizer(vocab, original); + } +} + +module.exports = Tokenizer; // Load vocabulary file for encoding async function loadVocabFile(vocabFileName) { @@ -17,7 +46,7 @@ async function loadVocabFile(vocabFileName) { vocab[token] = index; }); - return vocab; + return { vocab, original: lines }; } // Remove mark and control chars diff --git a/samples/js/node/notebooks/vision-background-removal.nnb b/samples/js/node/notebooks/vision-background-removal.nnb index 92e4da28d1b4e9..fea5ba9fc5dd68 100644 --- a/samples/js/node/notebooks/vision-background-removal.nnb +++ b/samples/js/node/notebooks/vision-background-removal.nnb @@ -10,38 +10,23 @@ { "language": "typescript", "source": [ - "const { cv } = require(\"opencv-wasm\");\nconst fs = require(\"fs\");\nconst { addon: ov } = require(\"openvino-node\");\nconst { display } = require(\"node-kernel\");\nconst {\n downloadFile,\n getImageData,\n transform,\n setShape,\n displayArrayAsImage,\n} = require(\"../helpers\");\n" + "const { addon: ov } = require('openvino-node');\nconst { display } = require('node-kernel');\n\nconst { transform } = require('../helpers');\nconst Image = require('../image');\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "## Download Images and Model" + "## Load and Compile Unet Model" ], "outputs": [] }, { "language": "typescript", "source": [ - "const baseArtifactsDir = \"../../assets/models\";\nconst baseImagesDir = \"../../assets/images\";\n\nconst modelXMLName = \"unet_ir_model.xml\";\nconst modelBINName = \"unet_ir_model.bin\";\nconst modelXMLPath = `${baseArtifactsDir}/${modelXMLName}`;\nconst modelBinPath = `${baseArtifactsDir}/${modelBINName}`;\n\nconsole.log(`Model XML path: ${modelXMLPath}`);\nconsole.log(`Model BIN path: ${modelBinPath}`);\n\nconst foregroundImgUrl =\n \"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_hollywood.jpg\";\nconst backgroundImgUrl =\n \"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/wall.jpg\";\nconst modelBaseURL =\n \"https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/vision-background-removal/\";\n\nawait downloadFile(foregroundImgUrl, \"coco_hollywood.jpg\", baseImagesDir);\nawait downloadFile(backgroundImgUrl, \"wall.jpg\", baseImagesDir);\n\nawait downloadFile(modelBaseURL + modelXMLName, modelXMLName, baseArtifactsDir);\nawait downloadFile(modelBaseURL + modelBINName, modelBINName, baseArtifactsDir);\n" + "const modelXMLPath = '../../assets/models/unet_ir_model.xml';\n\nconst core = new ov.Core();\n\n// Read and compile model\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'AUTO');\nconst inputLayer = compiledModel.input(0);\nconst outputLayer = compiledModel.output(0);\n\nconsole.log(`inputLayer: ${inputLayer}`);\nconsole.log(`outputLayer: ${outputLayer}`);\n" ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "Model XML path: ../assets/models/unet_ir_model.xml", - "Model BIN path: ../assets/models/unet_ir_model.bin", - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample-clone/ocr-node-sample/assets/images/coco_hollywood.jpg'", - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample-clone/ocr-node-sample/assets/images/wall.jpg'", - "" - ] - } - ] - } - ] + "outputs": [] }, { "language": "markdown", @@ -53,37 +38,10 @@ { "language": "typescript", "source": [ - "function normalizeImage(imageData, width, height) {\n // Mean and scale values\n const inputMean = [123.675, 116.28, 103.53];\n const inputScale = [58.395, 57.12, 57.375];\n\n const normalizedData = new Float32Array(imageData.length);\n const channels = 3;\n\n for (let i = 0; i < height; i++) {\n for (let j = 0; j < width; j++) {\n for (let c = 0; c < channels; c++) {\n const index = i * width * channels + j * channels + c;\n normalizedData[index] =\n (imageData[index] - inputMean[c]) / inputScale[c];\n }\n }\n }\n\n return normalizedData;\n}\n\nfunction removeBackground(mask, image) {\n // Iterate over the mask and set all background pixels to white\n for (let i = 0; i < mask.rows; i++) {\n for (let j = 0; j < mask.cols; j++) {\n if (mask.ucharPtr(i, j)[0] === 0) {\n image.ucharPtr(i, j)[0] = 255;\n image.ucharPtr(i, j)[1] = 255;\n image.ucharPtr(i, j)[2] = 255;\n }\n }\n }\n}\n\nfunction removeForeground(mask, image) {\n // Iterate over the mask and set all foreground pixels to black\n for (let i = 0; i < mask.rows; i++) {\n for (let j = 0; j < mask.cols; j++) {\n if (mask.ucharPtr(i, j)[0] === 1) {\n image.ucharPtr(i, j)[0] = 0;\n image.ucharPtr(i, j)[1] = 0;\n image.ucharPtr(i, j)[2] = 0;\n } else {\n image.ucharPtr(i, j)[0] = image.ucharPtr(i, j)[0];\n image.ucharPtr(i, j)[1] = image.ucharPtr(i, j)[1];\n image.ucharPtr(i, j)[2] = image.ucharPtr(i, j)[2];\n }\n }\n }\n}\n\nfunction combineImages(mask, fgImage, bgImage, newImage) {\n // Iterate over the mask and combine the foreground and background images\n for (let i = 0; i < mask.rows; i++) {\n for (let j = 0; j < mask.cols; j++) {\n if (mask.ucharPtr(i, j)[0] === 1) {\n newImage.ucharPtr(i, j)[0] = fgImage.ucharPtr(i, j)[0];\n newImage.ucharPtr(i, j)[1] = fgImage.ucharPtr(i, j)[1];\n newImage.ucharPtr(i, j)[2] = fgImage.ucharPtr(i, j)[2];\n } else {\n newImage.ucharPtr(i, j)[0] = bgImage.ucharPtr(i, j)[0];\n newImage.ucharPtr(i, j)[1] = bgImage.ucharPtr(i, j)[1];\n newImage.ucharPtr(i, j)[2] = bgImage.ucharPtr(i, j)[2];\n }\n }\n }\n}\n" - ], - "outputs": [] - }, - { - "language": "markdown", - "source": [ - "## Load and Compile Unet Model" + "// Details about this normalization:\n// https://docs.openvino.ai/2024/notebooks/vision-background-removal-with-output.html#load-and-pre-process-input-image\nfunction normalizeImage(imageData, width, height) {\n // Mean and scale values\n const inputMean = [123.675, 116.28, 103.53];\n const inputScale = [58.395, 57.12, 57.375];\n\n const normalizedData = new Float32Array(imageData.length);\n const channels = 3;\n\n for (let i = 0; i < height; i++) {\n for (let j = 0; j < width; j++) {\n for (let c = 0; c < channels; c++) {\n const index = i * width * channels + j * channels + c;\n\n normalizedData[index] =\n (imageData[index] - inputMean[c]) / inputScale[c];\n }\n }\n }\n\n return normalizedData;\n}" ], "outputs": [] }, - { - "language": "typescript", - "source": [ - "const core = new ov.Core();\n\n// Read and compile model\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, \"CPU\");\nconst inputLayer = compiledModel.input(0);\nconst outputLayer = compiledModel.output(0);\n\nconsole.log(`inputLayer: ${inputLayer}`);\nconsole.log(`outputLayer: ${outputLayer}`);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "inputLayer: x", - "outputLayer: test_0", - "" - ] - } - ] - } - ] - }, { "language": "markdown", "source": [ @@ -94,7 +52,7 @@ { "language": "typescript", "source": [ - "// Get Image data from the foreground image\n\nconst fgrImageData = await getImageData(`${baseImagesDir}/coco_hollywood.jpg`);\nconst inputImageMat = cv.matFromImageData(fgrImageData);\nconst originalImageDisplayMat = inputImageMat.clone();\n\n// Convert the image shape to a shape and a data type expected by the network\nconst [B, C, H, W] = inputLayer.shape;\nconst resizedImage = new cv.Mat();\n\ncv.cvtColor(inputImageMat, inputImageMat, cv.COLOR_BGR2RGB);\ncv.resize(inputImageMat, resizedImage, new cv.Size(W, H));\n\nlet inputImage = transform(\n resizedImage.data,\n { width: W, height: H },\n [0, 1, 2]\n);\n\ninputImage = normalizeImage(inputImage, W, H);\n\nconst tensorData = new Float32Array(inputImage);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, tensorData);\n\n" + "const foregroundImagePath = '../../assets/images/coco_hollywood.jpg';\n\n// Load foreground image\nconst originalImg = await Image.load(foregroundImagePath);\n\n// Resize image to a shape expected by the network\nconst [modelInputHeight, modelInputWidth] = inputLayer.shape.slice(2);\nconst resized = await originalImg.resize(modelInputWidth, modelInputHeight);\n\n// Create a tensor from the normalized input image\nconst transformed = transform(\n resized.rgb,\n {\n width: modelInputWidth,\n height: modelInputHeight\n },\n [0, 1, 2]\n);\nconst normalizedInputImage = normalizeImage(\n transformed,\n modelInputWidth,\n modelInputHeight,\n);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, normalizedInputImage);\n" ], "outputs": [] }, @@ -108,26 +66,9 @@ { "language": "typescript", "source": [ - "// Do inference\nconst inferRequest = compiledModel.createInferRequest();\nconst inferResult = await inferRequest.inferAsync([tensor]);\n\nconst { data } = inferResult[outputLayer];\nconst reshapedResult = setShape(data, [512, 512]);\n\n// Create a Mat from the reshaped result\nconst reshapedMat = cv.matFromArray(512, 512, cv.CV_32F, reshapedResult.flat());\n\n// Get the height and width of the original image\nconst height = inputImageMat.rows;\nconst width = inputImageMat.cols;\n\n// Resize the inference result to the original image size\nconst resizedResult = new cv.Mat();\ncv.resize(\n reshapedMat,\n resizedResult,\n new cv.Size(width, height),\n 0,\n 0,\n cv.INTER_LINEAR\n);\n\n// Convert the resized result to uint8\nresizedResult.convertTo(resizedResult, cv.CV_8U);\n\n// Create a Mat to store the background removed result\nconst bgRemovedResult = originalImageDisplayMat.clone();\n\nremoveBackground(resizedResult, bgRemovedResult);\n\ndisplayArrayAsImage(\n originalImageDisplayMat.data,\n originalImageDisplayMat.cols,\n originalImageDisplayMat.rows,\n display\n);\ndisplayArrayAsImage(\n bgRemovedResult.data,\n bgRemovedResult.cols,\n bgRemovedResult.rows,\n display\n);\n" + "const inferRequest = compiledModel.createInferRequest();\n const inferResult = await inferRequest.inferAsync([tensor]);\n const { data: resultData } = inferResult[outputLayer];\n\n // Normalize the result data from grayscale to RGB\n const rgbData = [];\n for (let i = 0; i < resultData.length; i += 1) {\n const value = resultData[i] * 255;\n\n rgbData.push(value, value, value, 255);\n }\n\n // Create image based on result data\n const [outputHeight, outputWidth] = outputLayer.shape.slice(2);\n const maskImg = await Image.fromArray(rgbData, outputWidth, outputHeight);\n\n // Resize the result mask to the original image size and save it\n const { width, height } = originalImg;\n const resizedMaskImg = await maskImg.resize(originalImg.width, originalImg.height);\n resizedMaskImg.display(display);\n\n // Remove the foreground from the original image\n const removedBgImg = Image.mask(originalImg, resizedMaskImg);\n removedBgImg.display(display);\n" ], - "outputs": [ - { - "items": [ - { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAKXA+IDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwC3iloxSgV7h5wmKKWjFABRS0YoAKKWigBMUtGKWgBKKWjFAhMUtLRQAlGKWigAoopcUAJRilooASilxS0AJRS0YoASilxRQAUUYpcUAJRS0UAJiilxRigBMUuKKWkIbS4paMUAJijFLijFACUUuKKAEpaXFFACUUtFACYopaKAEopaU0AJRS0UAJiiloxQAmKKXFGKAExRilxRigBKMUuKMUAJRS0YpAJijFLiimAmKMU6ikA2inUlABSUtLQA2inUlACYopaMUAJijFLijFACYopaKAEoxS4ooASilopgNop2KKQDcUYpcUUwExRilooATFFLiigBKTFOooGNxRS4oxQISilxRQAlJinUlAxMUYpcUUAJiilpKAEoxS0UAJRS0lABSU6kxTASilNFADaKdRQA3FFLRigBtFLRQAlJTqKAG0UuKKAEpKXFFADaKWigBKTFLRigBKKWigBaWiloGJRilxRQAYopaKACilxRigBMUtLRigBKMUtLigBKMUuKMUCExRinYooGJRilxRQISilxS4pAJRilooATFGKXFGKAEpaWigBMUYpcUYoATFFLiigBMUYpaWgQmKKWigBKMUuKMUAJiilxRigBMUU6igBtLS0YoASjFLijFACUUuKMUAGKKKWkAlFLRQAmKKWigBMUUtFACUYpaMUAJijFLRigBMUU6igBtFOooASijFFABRS0UAJikp1FADcUYpaXFADaMUtLQA3FGKWigBMUUtFACYopaKAExRS0YoGNpaKMUCEoxTqSgBKMUuKKAExRS0YoASilo70AJikxTsUlACUYpaKYCUUtGKBjaKdSYoASkxTsUUANxRTqTFACUlOxRQA2jFOIpKAExRS0UAJSU6igBtFLiigBMUlOoxQA2ilxRimA2ilooAbRTsUmKAEpMU7FJQAlJTqSgBMUUtFAC0tFLQMSlpaKAExS0UtACYoxS4pcUAJiilpaQCUYpcUYoASilpaBCUUuKKAExRilxRigBMUtLRQAmKMUuKWgBuKWlooASjFLijFACYoxS0tACUUtFIBMUUtFACYoxS0tADaMUuKKBCUtLRQAlGKWigBKKXFGKAExRS0UAJS4pcUUAJRiloxQAlFLRQAmKMUtGKAEopcUUAJiilxS4oAAMg0mKCxXAx1palPUp7ITFFLRimSJRilxRQAmKKXFFACUUtFACUUtGKAEopaKAG0U7FGKYDaMU6koASinUlACUUtGKAEoxS0YoASilxRQA3FFOpKAEoxS0UAJRS4ooASkxTsUYoAbRTqSgBKMUuKMUAJRS4ooASkp1JQAmKKXFGKBiUUtFACYpKdSUAJijFLiigBuKMU6imA2ilooAbijFOoxQA3FJTsUYoAbRS0YoAbRS0YoASkxTqSmAlJTqTFACYpKdSYoASilooAdRilooGFFGKXFIAoopaAExS0tFACYpaKWgBKKXFKBQIbS0tFACUYpcUYoASlpaKAEoxS4opAJiloooAKMUtGKAExRiloxQAlLRS0AJRilooASjFLRigQmKKXFFACUtLRQA3FLilxRQAlFLRigBKMUtFACUtLRQAmKMUtGKQCYoxS0UAJRS4ooASjFOxRigBuKKdRQAmKKWjFAD027TnrUZ61MreYpBQAL0NRVEN2XLZCUUtFWQJijFLRQAlGKXFFACUUtGKAEopaKAEopaKAEopaKAG0U6igBKSnUYoAbiilxS4oAbijFLRQAmKKWjFACUUuKKAG4oxS4paAG0YpaKAExRS0UwEopaMUANxRilooASilxRigBKSnUYoAbRilxRjmgBKKXFGKBjaKU0UAJiilxRQAlJTqTFACYopaKAEIopaSgBKMUtFACUmKXFGKYCYpKdSYoASkp2KTFACUUuKTFACUlOpKAEopcUUAOopaKBhRS4ooAMUUtFABiilxRQAmKWlxRigBKKWloASjFLRSASiloxQIKKWjFACUtFFABRS0YoASjFLRigBKMU7FFACYoxS4oxQAmKMUuKKBCYoxTsUUgExRilxRigBMUUuKMUAJijFLiigBMUU7FFACYopcUUAJijFLiigBKMUuKWgBtFLiloATFFLRQAlFLRigBKMUuKMUgExRilooGSx42NURHNQyXBglUBchuDU5OTkCojuy5LRDcUUuKMVZmJiilxRigBMUYpcUYoASjFLijFACYoxS0YpgJiilxRSASilopgJRS0UgEopaMUAJSU6imA3FGKdikpAJijFLiimAlGKWigBKMUuKKAEpMU6koASilxRQAlFLijFACUUtFADaKdRigBuKKXFFACYpKdRQA2ilxRigBKTFOxRimA00U7FJigYlJinUUANoxS4ooASkp1JigBKKWigBKSnUmKAEopcUUANxSYp1FADaTFOxRimA2kxTqKAG0UuKKAHYpaKWgYlLilxRikAlLRiloATFFLiigAopaMUCExS0tFACUYpaXFADcUtLRQAmKKWigBMUtLQBQAlGKXFGKAExRTqKQCUUuKKBCYopcUYoATFLiloxQAmKKWigBMUUuKMUAJiilxRigAxRS0UhiUUtGKAExRilxRigBMUUuKXFMQ2inUUhiUUtGKBCUUtGKAEopcUYoASjFLRQAmKKXFLQMqTI0kq7V4B61axip44yyttBPFREYJrKD1ZpPZDMUYp1JWpkJiinYpMUAJijFOxRigBuKMUtFACUYp2KKAG4oxS0YoATFGKWjFACYopaMUAJijFLiigBKMUtGKAEoxS0YoAbRS4ooATFGKXFGKAExRS4ooAbilxS0UANoxS0YoASilooASilopgNop2KSgBKKXFFACUUtJQAlFLRQAlGKXFGKAG0YpxpMUAJRiloxQA2iloxQMSkp1FMBtGKdSUAJSU6kxQAlFLRQA3FFLiigBuKMUuKKAG4oIpcYoxQA3FFOopgc0vipWB+TkdqI/ErSSLldozzXNzxKqghTk88UkOEbk5zXmfWJ73O50Yo719Ys4o0d5AN3vVq2uobpN8Lhh7V51M6EYc8dqt6bqUlmpWJ9q+9axxXdGboX2PQMUtcuuvXAj3gBwBVvS/EKXQbzx5bZ4zW8a8JdTOVGSN2lqOK4im+44NS1qmmZNWEpcUYpaBCYoxS0UAGKKXFFACUUtGKADFFLRQAmKKWjFACYpaWigBKMUtGKAExRilxRQAUYpaKAExRS4opAJilxRijFACYoxS0uKAExRilxRigBKMUtGKAExRS4oxQAlLS0UAJiiloxQAlGKXFGKBCUUuKMUhiYopaWgBuKWjFFAgxSUtLQMSijFOVGbO0ZxQBZtb5LWUr5ZZSMdOlV5iGlYgYBOcVXmniFqxVj5+7kfjUqksik9SKxppczZvVTUUFFLRWxziUUtFADaWlooASkp1FACUUtJQAlLRRQAlGKdSUAFJinYooAbijFLiigBMUUuKMUAJRS0YoAbRinUUwG4opcUYoATFGKXFFACUYpcUYoAbRTqKAG0Yp2KTFACUUuKMUAJSYp2KKAG0YpaMUAJRS0lACYoxS4oxQAlFLijFADaMUuKMUwEoxS0UAJSU6kxQAlFLijFAxuKMUtFACUUtFAhuKKXFFAxuKKWjFMBuKKdSUANopaDQA2inUUAedPM7wK23II5qq5K9GFbUTwWimF0BHY1FNDZSDPT6V4akeq0Y8cD3D7d3NXLe0ZAWZN4HXFTmaztEIjjJJ/iq9Z3cMMJyPlbpmm5BGKKAnWNwFBAPVTSzxhjmJwvcipbkRTqTCwL56VS8vZkuDkURGy7bX81iAY2yTWvZ+I5WUiVASOmK5wOGGBT7ZS84w2MetaRqyhsZygpbneaffC9i3kbT6GrgrhJLy6tnxG2F9RVqDWLlVJMwzXVDFK3vHPLD66HZUVx8Hie6BIeMEA9RUs3iSfcNsfymtPrMDP2Ejq8ilrgn1u9SUuH4J6VpW/ih2Ta6fNQsTBjdCR1RZRwWAqnLqltDN5buAaxftZmO8s289BVLUbV5gJc4I61jLGK9kjWOFdrs6hdTtWziQGpo7uGVcrIMfWuMt4Ci5Eu7dQIp9xKOy7Tkc9aSxmuqB4Xsd0OaAK5m38QTptSWEgDgtW5a6hBc/dcZ+tdUasZbM5pU5R3LdGKQuijJYAfWlVlcZVgR7VpcmwUUuKWgQlFLRigBMUYpcUUAJRilooATFGKdRigBKKXFFIBMUYpaMUAJS0UUAFFFLQAlFLRigBKKXFFACYoxS4ooASinYooAbRTqKAG4paWigBKKXFGKBCVFO0qxExEhqmpCMqwHXFTLbUuHxIyYypikMwO7PGK04Rm3RuOlRQ6czWTyyA8n5RmpYYjHEoYEGsoPU6atnEfiilxRW5xiYopcUYoASilxRigBKKWigBKKWigBMUUuKKAG0uKWigBtFOooAbRTqSgBKMUuKMUAJiilooASilooASilpKAEopcUUAJijFLRQAmKKXFFACYopaKBjcUYpaMUCEoxS4oxTAbRinUlACUYpcUUANoxS4oxQAmKKWigBKTFOooAbiilxRQMSkxTqTFACYopcUUANoxTsUYoAbiilxRTAbRTqTFAhKTFOpKBiYpMU6koEJSU7FJimMSilxRQBwgjjuULMrbuxrLl3RMV5Aq9a6mRGYioyBxUE94JGGUGVrwlc9V2IraOS5JCoSPcVYSby22OmQvbFW9MuAA7PhQeAKt/Z4ixBC4fvSchpaFeG0SVPMU7WPTmq9zb3W4bVDr3p1xKLSQIC2R+tW1uo5rUlGKsOtF2tRtJ6EDafFFAJGP7wjpUEcIlJzhcVdRYZ4VLsSR71QeFC77JeAemaUZXBoZLIAdgbIFReZzyKa6YOetMwc8GtEzJ3uW0OyJiw+U0glDqMkACrlhZG6jKzZCDpmoZbaCO9EB+6ehpcybLs7EBCyjC1YESQwg5w3pVxNICSKyv8AL1pbjSpJJw4YFR2pOS2HykllcCaMKVAde5plzqTQ742jyCOtNXTXSQP5u3BzgVLdyABVMOSP4qmyuVqZCXDqflyB1wa2bWVruM/uyBjis2SJriQGMAHOMVuWsX2W3UEjp1pztYIkCuHiaOdQoHGazmmeBiIGPXqDWrPDHPGyk8HuKzYLHy5z+93R+9EHbUUkmFxe3ZRUeVlyOxostWubd9qyFh71Dcxs9ztU7h0FQ/Z3SbDLhvWt41HvcycF2OpbxIsUaZUsx61eOuW4jRieWrlTZyGEOvzGqzWl5I2NuRWixMjN0IncyataxrkuOmaW01W1vCRG4z71whS4iYhwcj1pbe4dG/dD589jVrFO+qIeHR6RRisvR79ri2Anwrr15q3NfQoh2uCw6CupVItXuc7pyTtYllmjhxvcDPqagfUrVJNjSjP1rC1NJbkrLuJbP3QaiTTGmDNIpJI4Oa5JY2K2OmOFfU6cXUDAESDB6c06SeKJC7OMD3rlrizlMKKGKbT2NK9pPcWhTzmJHvS+uq2w/quu5srr1k0mwPmrv2uEKDvGDXFQaW9tIXkb5R61ofZ5LjaY3OB6Gk8bYawqZtahq8NnDuzknoKqWWstcwNI2BjPFVJNOF0hjflwKn07SntE2uQVPrWUsbK1y44aJnv4smF35apld2K3U1iGWMKrASkdKrS6JZzN5hAVwc5FV5dPtIbhZHmAIGOtCxsgeGibi3kShVkkXeR61Y3LgEsAD3rCFhDNJ5okzkdc1e+zLLbeUZjj1BprHtboTwi6M0FZW6MD9KXFZVtp8ttOGjuiy91Y1rbXAyVz7iuuniqc+pzTw849AooJwMkGjcPWt1JPZmLi1uFFIHUmncetCdwasJijGKR3WNSSas2tvLeWrToo2L6nk0nJLcai3sV8UUtFUSJiilxRigBKMUuKMUAISFGT0FCXlswPDBgDk4rYisGitkl8tXbOcZqrfRPM0hFuF2jnbjmuWpVu7I6IU7ashsop5rRtmSvJAJpjxSiMMw4HBqeCRrKDMYKgjkNSAs9sW3FsnJpQbvcqS90qUYpaK6zlExRilxRQISjFLRQAlFLiigBKKXFFACUYpaMUAJRS4ooATHNFLRQAlFLRQAlJTqKAG4opaMUAJijFLijFACYpKdijFADaKdRQA3FFLiimAlFLijFADcUU6igY3FFLijFACUUuKKBCUmKWigBMUUuKKAEpMU6koASjFLRQAhFJTqKAG0YpcUYpjEopTwOTTGcBcg0roYuKMVALoCQIVIz3qwOaEwasJRS4opiG4opaKAEpMU6kxQAlJTsUmKAEpMU6jFMBuKTFOpKAExRS4ooA8ptcefuALetW5IhKhcRlceorPsb54rkEp8pPPFdE3kXO0I/J5wK8SV0z1I2aMyJmHHRRVia5B2gE5WpLixK8I/bpWbLmMEMOaSSYaodPqDyTKXQYH61aEjTEBV25HaslJUdvnPTpV+Dfw8b5A9+lKatsJMVZZ7a42SA4/pTLh8y7YwcVakaSWMFlHPGcVHDAfObJzjtWPOk7lsRU4DFskdq0LeK0OJXGB7+tNMcKxAlMH1qSOSB18s42nrThO40XCPNQmF8Rj0qL7NbJ/r2y56E9aliRYLdvLPFULqaJ1DMCX7VUXfYbNKS4jjhAT5iKlt5jJEWcbfSucaSROmSD71J9smWPZu4NU4iUja83G4vjHaqs14+SdgKAdabbOJrb96Dn3qS2kjSNopB34BoGY8twygsDtzWppt09zaODliO9PutLS7UFPlxTNPtRHuSJ+QeRVNpolJpkUT3kaSEjgdM1Sgum8wu547gVstaSsXi34LCsltKmhk2FhzQmgaY37f8A6RlR0ORVqOd7uR96k54GO1ZjK9pIQwBrasbgRRLIsfX71D8hJ3HwRXcaNGT8uOCetS2SSlSWc5U4xVn7dbFTubt0rNk1AC4HlHaAefep1K0Gai1yJc+UxX1FVEk2E4BVjXQy3C+SrOASR6Vm3NnGxE+/aD2qlITRVS5uIm4lPPvUfnyrISXPXPWnvDEzDbJlvSiWAry6ke9Vclocb2cMWEjVYt9YuIzgyZ+tZJlCZGc1VFwEYkn8KlxuF7G81/NcTEebjPAra0hpo1ZXYMD3riI7gFtykhs10OmXTyMoQtnH4VE46WKi9Torq3FzE0Z4zWfptnc2N2MPuhPHJqCbVZYJ/LkRiAOWAq7HdBrcTDgdeazs0i7ps13jSJy/c81XlufSmWly99bvMV4Xge9ULjUUhDEpgj1qIxbdi2yOa8uPtRjUEBhwcV3dv4S0288PRy3EQeUx53nrnFcNb6jHdD+EEV67pSLJocCE/KYwD+Vbw00MZHH+HvDEN3ZSHzXG0lR+FZ8NmyarLp0J3uHxk16Lp2nQ6dC6QH5WJPXNcdp0G3xzPJ/eY/yFEooIyZdTwXISJWu2D46DpVe7sbjS2VJMvGxwGA6V0Ou3s1o9t5TYDOAak1vD6QWbGcDmlKEWCk0c3Pbz2sHnSJ+665qos8FwPkIJ9BXUawP+JCRjPy1zHhiwj/tlpCMgr0NRZp2TKvdXaGGJc5pSmASCa3dTt4RqkSBQqt1xTNT0cRND9nJAkOD7VcZ1Y35WS4we6OcJMpwvUetLBcXVuGhR2ER6gGtHUNAubFPMjdpM9cDpWSPtKtufOD0BFW8VUXxEqjDoaEYaQ4GPxpJD5bEHH4VR+1lWAYgVP5iOmSa0WPl1RDwkejJvMFG4VHG8cgwDg+lObavANarMI9UZ/U30Y/I9aMjtUaMUY9CCKiuLnYVwvHtVLHwbtYTwkrbmt9tjSwKtJIr5wGHQVGk0SxPHJfM1wxG3J71lSyySweXGwIzkgVk+TILwPOjCP+8vY1LqQk9GWqcktTr7uzkaFo7mcfMuQV4qDSXUaZIm9SQSMk9a59tSnmdoy7SAZCluuKv6QsBkDTsRnoM96uK0B7F2inOFDtg/LnimhlboQTXZzI4uUTFFOopkjaKXFFACUYp1JQAmKMUtGKAEoxS4oxQAmKMUuKMUAJRS4ooATFGKWjFACYpKdiigBMUmKdRimA3FFLS0ANxRilooASilxRigBtGKWjFACYopaMUAJRS0UCEpKWigYlFLijFACUYpaKBDcUU6kxQAlJinUUwG4opaMUDEopcU13WNcscCgApGYKMmqR1a2Z3jV8so7VHJqCJZPJIcHBxms3UiluUoMilummkdVfbjtVSS9AdVMnK+lc2NZ3XEmDgEnmm2135d4zH5s8ivLni29EdKp2R2Us7tbxygfLkCtROUB9RXP2epC7s3idcMoyBV6LVoIraMyNg9K9GnVi481zCSZp0YqpFqMMzYjO41bBBFbKSexFgpKUnvSEgDJIAp3EFNZgvU0nnxE4DCq14kkjLtIVR3pOVloNIt0Vlz6itouzO5vWktdVaeYJ5Zx61HtY3sVyPc1MUUF1HU01pUUZLACtbkWFxRimxzJLnYwbFPxRcBuKKXFFAHlUdpIV3KuQatWUj21wrMvHoavWbRvGkcbfNjpUsmm+e24uAfavEcr6M9RRtqi1dXEUsSgRYY9xWTqUHlwZyMmtmzszCm2Rt3oaTULNXiOBnjis00maNXRwBys2CD161q20kcKbQTz1pbi3CSlcAkdQKntoIkQSyLkjtWlWqlG7MYxsydXaQoiZ2jmgQSrcFskKTyaignMuo5I2xjoK0JpW8z5UymOtcnMzTcS4ilZAgA244NVraCQuI+nNaFm4ltTvOWB4qfyBERMxwMU41LR0Ha+pUvbhbREhD5Ydazr+9UxIVTBHWpr3TXnJnSTd3NYskUzuy4OFropJNEtsvNc74lIrYsxbLCGmUZxmubTd8qDmtGOF3zhu3Q1biCZenvVdgsK4APFaUNlFIEmYZcCodNtI/KBYAt3zWsoX7qkDFZtlJEOTGD6elVFuYonxtAYnk1LI6rId5PWsm9QmfMRJFCVxt2NKaVfORw2DWNqN663qsD0460FZ29eO9VxYTXExBJPPX0q0ktWS23sak1mlyImZW+YdqsWls9tHJG65XtU1uJIrUJ99lFPgMxWR5+BjiocmVYw5FHmMynHPSpX0+S4gFwi59qp3F0v2lscc10WkzhrQkjgVTbSuSkmZUMs/yxyqRt7GpdSJNupByPQUtwGv710i+Rl6dqt/ZZobVYmj3k/eJo5uo7GRYxC7lADbSBWyikL5Ug3D1NRWVtFBOSVAY1oYVsnK8VM5X2CMbI5y+01ElaQOMHtXPzsyseM47ium1ewuJ/mhYY9K517eZW2sp61rSemrM5oiiYkkgdK6fQJ28vbt5J9Kx7XTJZHyAQDxXV2MdvpsK5ILetKq1aw6cX1Lr7I0LSqD9RWPrd3thSOJtinrj0qzPqccrcDcoPNIUsdYQquQV61hHR3Zq9dEW9LuvJtI1RsqRzS6hYrfR53BT7VTvUbTrAeUPlXuaNJvHuIyW5x3ot9pBfoy5p+kRRqV3AHHJrux4q02y0dLb7QvnKm0KDzmuMkDeSzL2FcbdidpzIpyd1Om227kzske8+F7mW50nzJXLtzya5LTtUX/hYc8D/ACqCQCe54rN0XUdVsLMIk48th0ZelYutX0lrcCdeZXOS3fNUpJuwmmlc9g1nTpdSa2aB1AjkDHPORTtd+XSChIDcCvN/C3jTUSzRzS+Yg6bhyK2L7UZtRu4mkmbYpDbBwCaJTUdAjFvU6nW1c6AQiszbRwoyawvChY6m6urBgv8AEMVoJ4nVIgkluWCjFSWviHT5p1by/KYcEkYqeaLadx2aVirrdwF8QwR552k10sO25gic84rPltdK1K5W6LqZQMBgauWEawW8iI+4KTg5q0tbkslV1neWPg44qtPpkMjQZjXCnniqWh3Dy6jdq2eGreLDBPpVWuK9jkp/D0GpaxKqfJFGADt9aW68GR+Uxtrhw47E5q/oF0Lm/wBSx/BLt/QUujyTHXNQV5S0YYbVJ+7xU8keo+ZnFWunXg1CWDYzSJwQKNTXU9PG97GQxjqy84rv7FE/tS8cAbgR/KnWt7HqU11bPEMRNtOe/FSqcd2U5voeewSm4jEgyMjoati1lCZMTkHviq2r3EWn63JZxRnGeAOgr0SJoYNLSWRBtC5PFZqndspysjgIoFgkdgpy3UGpgQseSuR3FdZqVna32mNcxALhdwYVwwvo5IyY3yQORUTpuLKjJMs7LaQZCgE1F9mSPAjJHOeayYJJmuXdz+56jFWH120dTH5m1umaalUj8LE+V7mqyTtEdpD/AEp+jlYpJ2ugBheA1YWl6gbYysJDIm716V0BEWoQZ4yRwa3+tVI6S1MvYRa0ES4ikPyup9s1J1rlZdF1Gzvd8UheItnrW7DNNGAJVrthjab3OSWFn0LtFRC4XPzAj3qUMrDIIrqjVhPZmEqco7oKKWirIEoxS0YoASloooATFLijFGKAExRiloxTATFFLiigBKKWigYlFLRQAlFLRQIbiilxR2oGJiimrKrDqB7GkeeKP7zqPxouFh9FJvXbncMU0Sxs20OC3oDRcLD6SlxVDU9RTT4d7DtSbSV2CTbsi5JIsSF2OAK5q/8AE4jlMcAzjvWNeeJLidHHSM9KxstIdynArza+MvpTOylh7ayOpXxVKpAZM+9X4vFNqEzN8rVw248bmHFQ3Dbm5P0rGGLqLd3NJUIPY9DXxTp5IBkxmrtpq1tefccV5HO7KVxnn0q7a3E9uQ0bnNbxxsuqMnhl0PXhg9DTSyg4yM1wOn+J54JAs77l7+1Xv7fBuXkDZUjpXTHF02rmTw8k7HYKyt0OaWuVg1bcpkSTAPbNa9jqYkhLykcd61jWjLREypSRp4orGfxDAZQkfzc80469Cs20jC+tP2sO4vZT7GtRiqUWqwSOFzjPSrwIIyDVqSexDi1uJisXXWICDeVU9a2XkRMbmAzXP6/E5iyHGPSs6rtB2Lpr3kYcDfY7xivzK56mtTV4hJpLOASxXgCqkdmrWpJbEgFW7IzXcawEAKBg5rzZ1FTi+bqdag5OyOJs0QKxZWzU/nCJNzDGO+a6i90ZbGN5mwyH0Fc/e2sMluWVTivL9pzM1cXFBY33ySFGIY8AVPJLgJuYnuRWHBMltJlclc8itaS4ilRXJA9BWnNKLstiEk9zUh1P7IQwHJqxY6/KJnLsCp6VgO5EY7g1CzCLnBxWsMVUT3F7GLR2UniGKaIgnbjvXNTeJLiSRo95CA8GsmS6GSO1Up5ONw4rd4upII0Yo3LnxBKJEMROF61OniO6kIDsdtcr9p6cVKt5yAeKmVWq+pahFHY22qxyjDj5vetNL0QqrAAfSuGglJlT5uD3raa72RrGnzEdSa1pVpJasmcF0NufXyYz8pVu1Y114gkltpEJOfUdqZdjzGRV3cjnFZcgAPl7CCTSqVql7XIjBPobmna5JDBHFHkj+JjXa2lwJ7VZSRyK81it3tp4/vCJjzxXTm9WxtBCpLbhkV04eu0nzGU6V3ZHU719RRXDHWpgcBjRW31yIvYMzo0mtyJFUj8K29OmjlizJkOTVBNRUxNC6g9gcVWeeXO2Pp2rzpNvc7lpsdNJJGqg7wMdqguLgPAxHpWDb3TfaAJssB2rcW2LIG5VG6CptbcpO5y0WWvnZ8n1zU11IUUIBhD3rRuNJXzPOjc+4qoLf/SAjglKxqNSmuxCi0SWEcUtvvYjelJcXhWRYSvyn0qxPEqAiNdoIqtEIxqMSyHJxzmsnJbFk2nttuNuCF61evrhXhO3otOQR24dzgg8fSmtBCIHO7KmiDT3DoQafcedESCM9MVnXAuEaUCMEEkDFXrOKOOQFfuk1o4X7SRsyO1awajK0RW0OS+zXKL5vlla2NJtlceZJnOatamkzL5aJle2KXSI5IQy3CBR1Ga6HUurCS1LiIEuSiHjFSNvReDg1my30sd6fLjyCcZrVEiSoMryRzUllVt5Y7lDA96qhZBKylcg961Y444wV7GlkSMY2HFNOwrGFMs8BI2krVX+0HCbMbDnrXTssLLywJ9KqPYwPN+9QbKOa4nF9ChZ6kY1IY7jitEbruz3BsbqmjsLRGyiAjpUk/l2tt8iYx2qW0UlZanFajaTWtwSykoT1rb029gt7VAwJLdakubmOZlDJkd8inwraowbysq3tVyldWZKjZ3RqQtatiVQob1qvqOpy2kf+q3KeNwpJ7RHg2xhlJ6YqrLcyJavbzKGIHBrNItszX1BpJdynk9qY091CxJ3AHnmqKtsu14wSelaur3qtbRx+XtcY5xWrVmkQndXIo9QdZBls54xV+5SIGFsDB6muY83dKu0100ds1zaRJv9M0pLlCLuWJ7UxW++AjBFYdzdsE2HOc8mti9ulW3+zxHLIOa5ieUh8nO70op6hN2NbS3D3Cgjj3rpI7eNNzxIFJ5OO9chp81xc3EarGMKecV094LsxKYCAR1FTNWdgjsN1INc2rQ889qz9LkeC4FsI229zitm3WKS33SyZkA5FWLeOIYYjH4VPNZWK5eoy5ytpIV7CuLF2Y7ndnOGyRXePEksbqOhHeuXbQVjuvOZiFLcCik1rcJxbtY6OyvRPaqwGBjpVW90+PVYwu4qQetWIYVjjAU8YqZApzsI+tRezuit9GQaR4fGlh5Wl3bvWtZGxyRTYZMpskIPpSGLBOD1rNybepokktCf7Sm7aaaWQtuBH0qo8RAYnioEmTn5x+dPl7E3L4lZXJRsH2NXrXVbu2UqHOD1zWQgU/NnFSkGSPKtketGqC1zodN1hLCZ5JsfPzn3rSs9eikS5Mp25OU9xXnl/p91c7fLnYY7ZrTSZ7SzUyjdtHOKtVJJEOCZ0fha4htr28WR8NcSeYufcf8A1q37PTYrO+ubwTE+dgkE8DFebQava3EhG7Zt79MVbfUnmtXdb+URAYI3f1q41f5kQ4djtNHu4rrVNS8pw21wpwe+BS6PazQajfvIpCyPlT6jFcFotwNJZ72zmBVhuZM8Mea3k8fPdQyR29m32hVyQxGK1Uk9iGmtzG8QXEEeu3bHBlUjA/Cu51EsfCkhXIYwnGPpXkURkv8AVXvb+UebIdxUdK9NtvF+mpZRJcKyqflyVJBP+RUxsmypXaRcsgw8IJkHd5Hf1xXmWlItvMfPIDt/Ca9Ev/EtpJamGzG8OMbh0WuEv9BMkvnRzNlumD0qKkk9CoRe5oERmM7EBB7CuY1TR40m8wOUDnpXRWNq9rF5cku89s9anntYriErKM1ip8r0NHG61OctLS4sIiwxJG1bdpdNGijIUnoKpX7G3hEccg+XsaoS6gs8SkjbKnTFXZz1EnynZLcKY/nZR9apxahFcTvCcBl6e9Yc8Mt/ZJskKPjnmls4msrfM3zH+93qPZqw29Tfmb90wABOOKo2csrqfMyjA9KqW+qRx3BikJwehNXTc28svlA4J6Gi0o7C0ZbVpFbht3tVlZQeDwazll8slZGwB0arKSKVBPPuK6KWMqU/i1RlUw0J7aFw9DimRliuWpFlGOT0p0UiyrlelerSrwqfCzz6lGUNx9JS0e1bGIlLRiloAbijFOxRQA3FGKdikoGJijFLijFAhKMUuKKBiUhwBUVzcx2sReQ4ArldT8WoY2S1HPrWc6sYL3mXCnKWxvXWrQ26vk8rXOyeNY43ZQu70rkp7+e5LEyEk9RVRYcNljlj2rhni5P4Tqjh0tzem1+8uboOj7F64FRXOqXMzAvIePQ1m+VIp5XA9aUgEDnpXM6tRvc3UIrobA1+7WNYy3y+tWNJ1G4bUw5YkHrk1gMhcDmr+nzJA4LNitKdSTkrsmcI22PTY5UaMMGzxXIeJpnnfYJBtFJL4jigt/KiG4461yt3eySu7OST6V0VsTGS5UYUqLTuMeQICpwcUxLtcEdKzkmkcthc89KRzKCHKYHpXByq51F+S4QOOarvcAvkn8KqPMpPPBqREQkNmnygaEBDpuIHtTvNw3GKrB9vyjGKbI+1tyng0rDHBnklJPAq2mCBhjnvVBPNklCxjdnsK2rXSppFDt8vsaTdgRBBO0eVyeTWiupyRw+WO9PNmsUZBUH3qi8WWypAqFUfRjUbl1ZliZZAOtSTXi5yRwRWc8hVQDzVSW4kdtuCvpRGcrWKsbUd786nJwprcttdKqVJxxxXHJIq4DNyatbi7KQTj1pOpUWiZLiuqNm71eaU7Rng8GqN5qFxPgyMdoqnNOy4UH8arG5LAq7dKHUqNWbCy7F6LUJA+dxrTs9SZXWMMF3d65sHfyMjFWPOVAGHUd6ymm1ZlJ21O4WQSQGGQ7lbvWZdaYrRPHGcZFZUGtNHHtPNWoNTJAJbIJrFRkitGY0nh27eQqAAnrWjaaIYosSgELWwJ96/K3WneaE4c8EVpzSa1J9mkYZsWLnB+QGsvVgLdcB811E0caRFi2M1yPiGHaNyZK06esrMmSsjGa4+YZNNe43HZ1BqiwYZbPFNWRl3EGu1RRjc0di7cg8+lJ95h6n0qtbrLO4RASx7VZkt7y1J3wtntxRYpO5aiHlNuLcdhVuG7JmUE8VlqLjAaWJwPXFPWRw4CJ1PWocRpno1mIZLVW2gnHWmvp0Bl80R5P0qDSmK28Yc8YrVRCZARIAlcvNJvc1SSM6W7tocxypz24qN2jkCsqFvw6VryWdpJMGdQaUCOMsioAn0q+aSVrjsc+dNgYlueeaK3fKg9qKnml3CyOVS0ikI2ygSdxTUiaKcb+RmnCylhl56Z61c+RdozuYVvUqLoZJXGJCPPG1BnrWw80vk7QuSOlZ8E8TSYZcNmr7yApuUjjqKy9pdWZaGFlaAbzhj2rHN5suimARuwKmnupD8iL8zHArOEa29wxk3Mw5xWTloD2N6URyQHBBfqKwLgldXh3DaWGKsJMWPnI/A7VUmIvJhMSd0XJp8ztqhM05FeIGItuXrSW0jOzKfu+9U1uW8xSzfK1Sm5Pm+WinmqgubSJLZo2sXl3DHHy461YuZ0huUJPDDtVKeV41QqrEEYPtU/wBme7MKgbSByaVPmlK0ivI1VQSIrCsPxBK1uUZHIz2rdgRoYhH1x3rP1SwW8ADLmuxPXUbWhmaRceY481wciuhSPcmYxx71TstOtbZQdo3DtT7m78oqFYBSfuis6k0nZBFWWpLJx17daiIV0Pzc0iOxXcRkGplCsAQlNNsZSeGberIeKnJlXg85pZ7uOHCsuCelOhkRmBboelX0ETQQsY8g4NUNUmcrsA4B61oE8kK1QNAXUpIM0k+4Paxg295FHLtmXI9a3rQ2t5/qiPl7Vj39gPMG0YHc1YsII4lIQkOauVrXJjdM6hbmOGIkxB8DjArjb26aW5lYAqM9K1UuLoP5Qwwz1p9zp0E7gEFXI5NRFKLuVJuSMiGGznxKw/eird3ZxXtqNpAYdKifRwjEQzjcOxqtFI8SyrK/K8DFaPXVErTcqw+HrqSXPAX1rp7XTvstkytJ82OprL07UpFkw+TEO9bLSxXcGQSox1qZuT0ZUVFHOxEC5IJ3NnBNbFvodvKm+RAWNc5Mj2185yQpOVNaVrq1xC685X3qmnbQlNX1N220uKAkogUnuKxtWe4spyqyEqecVbj18+cfM+72FYGr6hJdXhYH5e1TCMnLUqclbQ1NEtZbyRpDKVI7VuqpjcpuDEelYGgXG1njkcKWHBFXrZI7a9d3uy2f4SamafMEXoa6bSwUnGaWSEYIx09aiLJvRgw9jTri6AjZy4OBUWZelhjuvKsQvHNNggzbsqSYBPDZqjFe21xNjb8zcYq1dxSJZ4hJXHenYSZLJZS+fGy3DYXqM9a047hNrKw4AxmuY+13YQKrAuOuatJeymwyFzMe1Eqd0NTNSXLhk3YBHFcbI09teyRsxJ3cYNWLjUL63JMnAPT2qlEhvb5W8whj0q4R5dyJyT2Oo0eeeZdk0Z2+prdgREG0dPSsVJxYW6iRs4qpLrbrLmMfJ61m48z0Kvbc6icIiF/Ss1LyK5JRW9sGsxNdDDDDcD1rOa/jjvvNhHyDqKybUNx3Nm60mOWOQRjDkZ4rCjuDbrJaykgYxitQ+IoDgKCCe9PuGs5oDP5YLFetXCom7ClFPVGJHMbcbY5GCjoM1fttRcSgr8rMME+tc/NLskJwRUlpcBpkDPxmunl0uRdbGtfQXf2kFIyR2Irc0yGdrZEulBRTkA1YR1eyWRACQOKzIk1WS6VmIEWegrHmbViralzW2lihVbeQID1FJoV1cMhEzbgvAo1HTzfbRvKkVYtLb7LCFGDjvU6ctiupZWyb7X9oErEHkrVpnHHFEEwVMcbvSpCN6EkAMOcVkykjG1mwa6tzJAuXHNcqHMEoE0ZB7g1satrk8FyYYsrjrnvVaKCbWoCXUKw6HHWuiF0tdjKWr0HR3zMAkZ2qehqyZkERjlmJZulYwt57CYpMpAHT0NPnlR3V1PQdKvQV2ak1h58asp+Yc5pLuREWMMSpH8VUBq0iEBeBiql3qJmdVx2rPnK0N6e6WSFV34THXNWLC7a3tiufM7j6VykUxlkKMSF7Vp6PLI1y3PyDjmkmmg6nSWt810xAGMdRV4TrAmXYLWFJfxwybIQAx6mmXcLXSCSSQlQOgNNJxd1oNtNWZtjWIiMqN6jqR2qza6lb3eQjcjtXLaTNFGzxRnK9wa0IraGK9SaOTYO4zwa7Y4uUdJHLLDRlqjowR60FlB5IzWJdawkLHCnj0qpNrYAV88mt44uDWuhhLDSR0+RjOeKQOrDIIrjbjxFLJGVDbeOoqtY69NCp3NuB4p/XKd7E/V5WOxn1G3gieRnGF6iqFv4msJ85fbg45rkrqfzmYu5+bnGaxZjIkmUXOT2rGeMaeiNo4ZNanra3ULRiQONp6c0/zoz0YV5nDqk8aqHckAdKmXXLpXDAnb6VTxq7E/Vj0jIxnNMkmSJSxYcVxM3iiR7Tbkq1Zn/CQzNIFZiRTljY20FHDO+pp69qZu7zyBJtjHUCuYvF8qTavIx1qe6kEsxcN8zU2SIGIFjljXDKq56s7IwUdEZvyqMoTk063heSXeSSBVy3hidxG3yjPWtCC3S2LqSCpqObQdilLxDzzVFizuFxgVsTRxlwFYYqhNHtYhSM0IGRquxRg5p+5dnIOadAqnCt1q2tuuwt1Woa1FykChGj5HNMSz8yTAGSasvsEfUYpsM3kuXJ47UldDSKwsGiuCfLwD1qCZCrEBfzrZF+GI3AGq9yiSSB8YBq1LUbRy1zbyK28oeTUEvnDAUHFdZPFEUwMEn1qD7CC6kbdvetPaonlMOyinuCFG41p/YpA6xmNiO9dPplpbwxbjGPrTL93jf92o+aolUtsVyoitLGztESRRlu+amuLsD7vAqi0U2wnkmlWGRceaDg1i3fVhYleRmiyT8tUZDl8oeKt3TBY/LzxWYQVU7TQi1ZIcxOfelnjM0ahcKaYm/qOaWadYQrOPwqkrMehSmt5Vbljk1JDeyWyGNgT6VsRQwXCo+8ZPamXGnRiXcOlUpJ6Mlx7GLI80uCDjdTo4iFG5smtf8As9ZCEA+lV7rS3j4V8U+ZE8tirvkQYxkGmMW2hWB5qaOCSBTv+b3qTK7M4/Os21cRUR9hxkkVYiuVXgHp1puUAJxkmmCJBkjjNGgI07S8ZZQQ3y5rovNWWIPgEVxcMZ8xVDEAnrXW2PlpbbQ2WxU7MtBLc27lVZuBUMljDfWsicYPANRSWRkJOcE1k3UtzanylkZVJotfVA/Mpz+GJIpyFbcnUkUn/COeeVSFeT1NdXaQ/wChea0uSRyDVNtTjhkEdsuT0NVCcpPUlwQ2y8PwWASRyDIB1p8+17gLIFI9cVEbuafcOgHXNMd/OgyjfPnmtxJErRxM5WQqI+2ayJ7VVv1WIDYT1qy6yTKFfIA71eSziWJGYnFTJ6BYuWdlKsRO/PFWoizIUQnIp2nOirtzwRipnjWFm8vv3rl5b6l9CuPODe4qwkkjcFDVcbzn5stV62uRFFiQfNRbUCHMY7UVKbu1z9w0U+VBc566mYyEMfkzUVtkTKNylCfxqylsJbeQOpYZ4aqlpakynORjpWnIuXQzd7k4jIvpGzhccVZBkAbbg8c0Iu2IqwyQepqN5vLbYh2lupNck21KxSEEDm4gfp9ah1UBWccbyOorYiVHWPcwyBVS8tDNISANq55q9W0kNmHbsLa3LlSSaekDpavcblxJ2rZisIpLMeYQwPpWO0ZnmezgyqqfWtne2qJsU7YpNOI8nI5rYhUxuJCoIUYFYPlz212JgAuxtuPWttbyO4O1WAAHes23C1hXRqQI08Lebgd6iaR0ICk7V7+tRxzeRHvkYGMirW4XMSSQJuC9fekve0LtciSa5w3lEkn1qa3e6UkzkH2qe1tJpnaVsIMcAVYeMoNrLyOc10LRBZlVozK+VByKoy25aZkIINag1DyCA0YGT19ahvJY2n89cY4zUOTWw7KxLYIIY/KI3HuTUOowzIwlt5AT3Wqk+qqkmY8AYqe2lZUa5lbIboprSEnLUV1sQuWnZQ6ZepEjdZAjEA1ELqFZi55+lPeZW/0gAkCtRE+8pcBdwqaaVbdhIWB9qqWmoQTuysnzD1qO8ZJ33BsAcYNCXcGxtxqKu/yqKpJeFZTmormGRGDJznvUcylFXcpB/vetWkSbtndwyNjo1XZJfMOBgHHWuYt1eU/ugcgdqsWv2troJ85Hcmp5R3JLsukrAkhvUVhSSvFI2WJyec12k+mpNBtz+8AzmucuNLaVzEcBvWqhJCkmXNFniuFNuqAk0+/L205gZiM9AKl0nQ/sDLLJLh+2Kv3VhHdy+eWy44qZNc3kUk7HK6rdxsY0XqopLFmuCFIJ+ldQug2PlmWWMMx7mr9rpNrEF8uIAmj2sUrB7Nt3MWCzheIiSLay9Ce9Yd/atDMz7cqPSu4ubVdrBG6VhmGaKVt0W5CcZpwqdRTp9DlrKVxdgjIxV9klmuMoTuJrpzptvEqyPCoOOoFZ9zb7JMWsZaRj19KpzUtUTycqsNMVxYW4aVi5PTFMin80MrI5J6cVftjJBMq3RBBHU9qvGLdbF4VQgegrNzsaKNzm4xLZTGd4yf7tbuh38tzbS/a1ABJ2g1jy37mRoyoLA4wa1tPS1gKG4lwx7dAKJ2a1FHR6DbtT8zJGfrUVu/lw8qQ3pWu8lrKWVJFNNe3g2hicMBS5tCuXUxNSs57i08xW3e1Yunrc216JBCxX3rqGuljm8qKNmU9SRxWdfzXFtvZ1AU9OKpTaViHFbjr/AFMCzKFMuR1rBXUowrBjk+lUZr64knKk9epqlJEkkwCyZY1jy3d2DZt/anIzGuR7Uu9lXJbGetVYJRbgK3U0s04k+Q9KThqFyeWeFSADljVq31HygEcEg/lWStmsuSSQexqCWd4T5bg8d6PZdgudIJIbmFkZBz0NUYbN/t6ovCMcZqrpN0z38YJ/dg8rXaT2MTKk0CHf14raEnFNMTV9TZtIoo7NI42zgc04OY/l2kGq2nyqxKOQrjtSajfLa8DLNWNnc1voNvJJzgRDnPJqaJpfL+brWFHqk9yzQoAHY8e1aO6W3sirOd47mqcbbk3NWJehJ+b61aVw4+dhntXIwanI8mxnO71zViLUpUufLY59KlwKUkamoQ2buBOq5PeqtxDDFEgik2beRiqF9dzXJZZYiqg8NVIsxwWlyB0Bp6pBuzVv5RJaguFk461zMq7Rg5G48VPcXbAGMMaogSPnG5wPTtSbeyJaA5UkglsVDJPtk3Ec4pJbiO1OZpVQH+8cVSn17T0JDsJOOiDOaajJ9CdEaMN2u055NXbW8W2jyHwxPPt9T2/GuQXVZJpP3MSxJ2Lckf5+lWxby3RV5ZiwwMAnI+gUccc/55raNCxLmdEdVt5pWJkII5JAJ/z+GaX/AISAW6FQ7EEcIV/mfz4xn8eKx1tC9qpiG0nLZU9Tj6+3p3zzT4rNI4Q9wMJGQc8gj1BYZ9/b69Rtyoi7EuPE11CzfZ1UE5BJUZPvwKpQ+NtQhmAnSKePPK42kfTHSs7Vpy8hCxGNAeA2M/UgdD/ntWG8pRufxp6Bc9W0/wAS22rxGE/upAOFIySfbnnnjt1pbmF5QRDKsiDrt4I/A+9eY2V8beZW3cE9Qa7xR/aGnPNC5aQpuYY5Y9fz9/8AGocIvcfMyd0dEOc8cc/59jVeOQctngdqsw6lF9mK3E6um0A7jyAQOf5ev9KzVmtwSEnVt3Qf5/zxUSpdilIutdLItXYbdprUug7Vz7QsgBVs55Faun6hLBCUJ6+tYTXKO5ILVwFZx35Bq7NJbxgcDpSROLhctxiobizMqEIai7bKKN75bfPEeO9VIZEjuELfMDWn/ZL+QCrHnqKqnQ5BOGLHaORW0bWswFnC5LICO9QZk3A5P0q+IGVSxOSOMVGI42Vsn5u1SnbQAjRiOFz3zSzFygyansCc+UcnPWrF9Zwxw5V8HsKT3H0MvsAW59ab5B6k5NQSLscbTVuBGkjJzgVV7CSEhiCyE5zVtWfytoA2mqJ3E/JzjrT4ZmXCM2TSfcaY65hKFSMkntTGUlwjccVsQAyqP3fTuamSyj83zHALdqlSuFjIht2zkLVswM67pFwKdcF4pHI7dqy5r2dvvEhKV3fQd0hLhfnOG+lQGd4xtzn3qKSdcEl+fSoHuV28U2nYhm7b6o0cIVjn0rQtglx+/eQfQ1yUM/mHBPFbOnybvl2kipaa3KjqbUcql9uAeaW8nRp1twoyRxSWNhOsvmnHl9hV0RQNKW2DzR3qeZI0UTAn026DsSNw9azbmJolJz9RWxd64Yrt4iMKOCa569vlluCFOVPNVBSbCXLYlt7gKp6EHvViG0S4l3NIG9q5ttQYs0Ma4yeta2mpKkYndzjPStZRa1M07mzFbKkoIGAKshBJISxyoqss7FMgdaDdNDj5c57VCV0Vctb/AChuXGRWZcXLyuSxzzU80++EuBj2qpEyuuZPlzRy23JeuxEZiG+Y5FRmRWBycCmXCrvO1s0za2wDIp6WFcXkHI6VKrBlwetRRxkZBPNSZG7GOlJq4IWLzMkDpWvpV44kKMufesxZyDwg2jvU6XW1g64XPUVLRSsa092EbAPOajmjjulDMMsOaou4kbcCcd6nWVkUEdKaTKsXLJC6ujNgYwBSQabHGzsGBb1qib5kbjioG1CRfuMeTzUq6E2jSOnkucS8HrViy0+KFWLHI96p2t8DGA/U1fgXzG5b5DVczFZE5MDtsCDHTOKtpDD5O1wABVNkWBC4HApI7nzScn8KV2Fi1FawRtlH/Ckug3y7TxUAmCOMrgVphBLb5UcH1pBYz4IGJDg96uNNbBgHxuFRyMLaPywOT3qiAjSbiDnNFrAafm2/9wflRVXC0UWGUkSdbUmOQYznBFVzLtQuD+87in3GsQmPykUbiOcVDEgFuWPO6nGErXIdnsHmySjIOBT4zHLcIkiFgTjNQykRws8Z+g9aiSdlKs5w56D0rlnq7itY050FrK6x5OFyMVHNJIul7kJZ261Te6d84OWzzmp4LoxpsKbsnNKD5X6j0GWaXgtWRTgDpmq1sjxPLKzZfpWneXxeFvJTyyFxj1rnprx4I1JRsk1vzuSshSRbkkglQRuMynpVGOA2+oiGTOCM0ktvcS+XNEh4OcmtiUKYkmZAZtuKVS0VYVr7jpWQ23l/wj9a0bO/hS1AQdsECse23TxMHI8wc7aZbM4VyUIbuKzhGSKTsdFZ3ztlYxkZ6ntVyVkuQzvLt2jp61z2nXflo4B+YnpUrfaJFIzhT1rWzZVxt1chhgcnopqkxneFgXwatw8z+WV3FeQ1SyxKiSbx1q6UnblJasZcduY9pkbJ61PI+YsGRtvpVX94s+GOF7Vp2FosshMmcjoDWqffQmz6FZUG3O8Ae9W7YsrCLepQnmkltrea48rOXz0WnNpnlSCIbkY9CavXuCTQX/lx3K+UAOOSKp3M4ZfL6H1q3cabPHGXZwzLUSD7S6LJHggVaaB3Es50ijKzjfxxU015BPAqBBjPJqGaz28pz6iquzy2xjj0o31Hqiz9rSyuk8rBGORW8LmE24mK7Cec1iR2UFyGkdwpA4FLPM62qxBtw6ZqWtRq50ME32iQbMYI6k1UvLdBJvfKkHqKwYrya3kUiQ8dga1YbyTUYnjZSoAyGNJpphdM0VS2u4o3WU5Q9M1bKQxFiDjIrmbO5FqXSRsHsa37MwpbhpJfNDjOTUSTRpFj5IxLb5EnuMUlsLyaRJM7Y17etJ/aVrEGVcALWPP4gneYpb8JmpSk9BtpdTWnvUM8kKA+bU4WNoAjttJGSTWHdq6QC4Vv3rdajV7xrYtPnaR1rRQuiHPU6FWFzG0UJDhepqtBdww3DR7QWQfNWVo1w9q0iRt971pkzsZmk3KrscVKg1oVz31JJbo3V0crtRm71pLcC0RY1Thu5rJRVWYeZ8wAzxVW5vJZbgLuIUHgVbS2M03uaWo6da/aoZg21s7m561Q1C1eWVWjkG08U+6dbizAZiJB71lGZ8Ebzx71PM0D1JS7wSFQ2CO4NXF1Z0iO+TJxisd1JXO/k96iuEKwfeyaFdsWxuw6q6EHIYZ5rTmurTUrRo2wpx1NcrZneqpjHrVyVQqhVODUO40yo9rFE7jgj1qKVbWOzZUi/eD+LvUN2ZQ3AJxTreJ5IWMp61STetxGaCZZF/eH6VbB2nGM4Gc0stj5boYlLeppDBOGO4bVx371bd2hWFW83Dg4APSrpgiuApcZxWG8EqyK6fcB5rZSZVjHUEilLXYqLXUtQ2dvEfMi4YVqWmr3EMYjABGe9YkU4BOCasLODy1Ll7lXRt3t3JIROvyMPTvVEam7TN5zdqoS3MsiiNDnB5zWhb6PJPhjk5HJpa9Q9CtDqHk3JdVHB4NXrrVJbmI5/SoX0N97gHFRNALZSpJYim/ImzK8Uv7wtzuq9bXDi7DYyRWQGdpSVHGa2LGdIiSy5OOtTKVtxxRrXF1vA3gfSsTU5kSb5Tt46VLLcljubGD0qhdmIAyTMqjH3nPH5dT+FVDV2Q35lNrkO24sFUdWJ4FY2ta8PINrZSOiZy5z948+lZ2q6n5sjxw8RZOOOtYj56nvXRCCRhKVyU3cjn58GnwEOwHeqgBYgdzU1qD9rjUdd1aEnW6XYqiiV9pbsAD8noT6/TB9MZzXRCO3SHZK4j+TH3znJPAI6egOB/SsGbU7fT4IoSWcqMsAerYHBP8Ah0+tY02rS3LtMzYbjaB0GPpQB3EeSrujBi6/dAAXOe5GOPftgVWleGSIySTvhDne5yMcd+ffr6DkZxXH2l/fyyERPkZ3MSpI5wP/ANQ6VcbUvMcLdsuFztfy8gHuSOc/r06UmCH3Wo23mMY7dGUjCKvJbHH9Of8A69YV3ezXLFtsSog+4kYAAz6kZP8A9f04q/eQuWFxiOVOzKQOPwwR+NRosFwhWSGcFj2dQPx+XJ/PtUlEcNnHcxebFtBAyVBziug8O6nBBIqyNhlG0D+97f5/pWefskMUYhstpVgWZXwzL3XPPJHQ4PNQQRr9oWYKVw3Y9PpTA7bUdKtr8GWFwHmG/J/i9x+f4/jXGarpF9pTeY8b+UGK7xyucev0zXSJrUcYCYeR8YG35eeO5609dQnkR1kjVkfrGVDA/UHP9KLhYz9M1Brq1jZ1wwGCT/FWkrBsg5qCO2soLrdFZQIxOOJWGfwzgVZaS0jKg+anc4wwH54rGcG3dFp2Rfs45uNoLCtJ0dgAq49aXTNQ08QlYrmF2xx8wBP4GrBuJVYgxjnnpXNJWepejFRWKBNuAO9NvIWAHlnPHNWGvVMW0gA0Q7pASB8p9aa7jaOeuiyIc8MetZ/PrW9qNk7ocY31lR2LGQBxwaaegrF7TECyhjjpRq8eDletTwWzQEAHIqwYVn+Y447GpvqM53yFRg78g1ZRo40OOc07U2TcLdEwAck1WHlSOqq+1VocmIkFuxt3AG1j3FMt9PmMisSdq8kmr3mDYMHipWnLERxnjvS5yrIs27Heqqvy+oqaeCRTyrZPTFPsIwqHIJwOvvT01dBcLGwHXHNT6F2M1rOZ8k5yfWqZ01pY2jmyvPUV1pmj8ws4AXHFVVaJ3IIG09KabWomjz+80rZIyxktjiqUdvJDlXU5969Jm023EDEqNx71i3tnE+cYyBWsa10RKnY5yzgjBDMMGuh0/wAuOULgHNYskbmXA6D0rT0udYbgGbnFTPVChozpZrtLSBUTJY9qZBGJYmnkOw+tSW8SahcB1Q4A796u3OkNNFsBwp6iue6Wh0Wb1PP9faBZCsZyx6kVzTrIHAwa7rVfD7xNwvy1imGKAlJY+TwGrqhUVtDGUHfUzbPRXlPnHjnvXWQWsTWHlkAECrGkWCXEah0Kxr39au3ptbVGCgEjtWU6jk7FxhZGLbeWshQ9qWYxM+Tjjisi9vitwREmCT2qXzgIw7kgmrtbUzbWxfukiW3OzGRWSMSgYPPpUd1dlUOG4IqjbTS7tw5WqUW1chs0HiVBk1Eq/NycCkW8xORIAc1Vu7hQSYzg01FvQTZeQp52Cfxq5BCjylG+ZW6H0rnYbg5ZmfmtbTLzy5FkYZQ9aHFxCLNO50owjCOSCKyLjNvIFY8V2Vrcx3CE4GMVQ1LR47uNpIVG+pi9dTRrsc4LzChUbJNSrcuFILcUtvozW0xaYcValgjUcJitG1shWZQM/Gd1SptkTJPzelRz2hU7o+R6UQo2cHhjSVidTRjyI92M1owNJ5QfO1RWSJmEe30rorNIrrTUB+Vqya1KRVlvJCuMZUVn/biJcjrVyaMrK0ZOVPAxUTWKEIFH1NJAbNl/pVsCw+atESFYAidRUNk0NvFHGFzkcmrDuin5BQlqUVNs0ikMvzClby1jAOMjrTxMwdielZ0+WkJ5xmmIu7k9qKo7x70UAYf2WNOZW57YqYTyW8IbYfLJwCapAySygLyWrXaX90lsU3IvWqd7e8ZpIgE0ZhMjNjHaohNDOnmYIepLlYxC2IgC3SqTuEkjLACPviufk7DTL9jaCaOSTPU4+lW9ohkEaxZx/FWbC6QmQwytsYZAqSK6dtiHdlj1NZuD1YNoffySOypFgMOpqe3gilCedhm9KqXEQWTCSHcevNNtrW63t84C/wAPvSgrrcEzUeRfIYgBVU4wKp3MmRCQCq9TmnQwCKF1ll3Sk5C5qG6nNzMEK4CjBFVGKctBtlKeT7PqAaCJmZh17Vot9pe3xhVz1NRq8SgbQCV6cU5g8kbyCT5V6rVNvoC2LVraKtr5jj5lHatBYmnlgRG2qR8xNZ9lfhYGXAbjinHUHFshcYPTinTblK7HoS3SeRfGKPBXruFI10iv5bDJrMlmfzNysct0pHWUncfmfrXSopLTqS22aUQWZ2maLKp0FX4reOazkuPM2Sdl9Kr2KzZVioCMBnNNvrR5Z8oW298cCm2pKzLSshNKmtrW8LsA8pPBPNaupzNJF5zFRjpg1zl08XCouCvcUxbh3+WR2IHQE07a3DmsrE/mSvkGU7j60+GcxYLYLDvWeZyJORwKswvDLGXcnirexBNPc7ZDJnr1qETJMGbjJrNNx5k7IeF7ZpsMreYV/h9RUaod0X9gJOXI/GleOWNEIy69zUJk2jnk1oWFzHGp835lPQGnzu4aFa3S3WdTJznrW080MRCqAFI7U7TotOlk8yRFBPTNUr6BI7pyj/J2FDmgtZBe+RLbMyrhh0qpDcOIdgzgUhnhQBS2TUQkO5gCNpojNdRWLAVXXLHLelRkxo2c4xVGQyISQSSaiFw+3DDJpTk+jCxus4lCksSBWkLqK6gWAfdA5rn7C5DShZchDWi80URxF9096nnk0PYdNFHGCYzjFZ5eNpOclqmFx5jlB3qpPZTwuWPINEZyb1FuTNqkdqSAu4mqE1yWYyqMFqRolZiTw3vVdyinaWwaG7sbViZLo7SHJyaejxSHYevrVaO2eRtyHinx27Cf371TihK5bFj5kgCuCuM0hsmaUJ0X1pIRJFIwBOKtF2YDnpU3aHYhMckCkhPlHcCqX2h/Ow4PtXRWkc11CYodvPXNZt1pUkE2Z2XPtRGa6jcdLldJA2WdfzqOWQAADgMaleHMbEN04FZ0lvdvKhVSVB6VcXcm5rW0bytsTHNMvbC5DAfeHtV+ys5dqOvBxyO9aaxxnCSNt9azT1LVjkPKEbBHp7ozxlEXp3q9qFg/2gyKQUB4NVyzR4U9atSsTYpmKVV6Y96sw2kskRfB6VbVRLGMjpVmS6+yxCNVHSh1LhYpWWmzswbacHvXUQy3EUIRFHyjk1kx6oyQgIoq/azM9uzM3J7VLbY9OhVutRkyxJwfaqdxIxRH6k9a0F2lwGjGCaukWqou9FDGou0wOZRV3/dPNV55jHLsz17Vs3wKtmLG0dxWUIBJMWdcnsatWe4+hALtIIZLicExRckE9T2H+e2a5O9vrnUZ2aWTrwE5x9MVo69ekhYkZjDHliFbGWzj8OB+XSqF1bw2unoXlCzEZZV6L7D/AD611Qiooxk7mZLEzcrlj25684/nVE8mpPOcFgGPPB9cU8I2do49fWrIEtojJLkfwDcfzqW3dEvRKAdiZbk9eKjHyZAPJ9O9AB3BOnr/AJ9aAHSyNI5YkkmgDOAccdKQ4DYGceuOam2/KD3piNjTIzJbMpb5VyTuJxn07Y/DP0p8kDyKYjHl/wCFWQ/MfUYBGPUdeR0qpbT+Wgj+VSxGDgFs98HgfqKstdgFnaVWjOAcMzDr/dwAfbr1/JDMsvcWcpXeykAjGOBnqMVpWl6txKBcRx5c4HbaPc+/T/8AVTnEepbRk7h0Tj5eevHHPHqaljs7fTys0zhDnoHGSMe2ev8AOgC/LpgCNIOEwOT6gc4/z6+mTWjtxAdzAE57c4HvU41tpv8AUphR1brtAxwPTjjj3q39sjKqXTa5U52HGB36/j+WPYooit5IRmRgN+MKSBz+nbp9TV1gYtztGABwMNz+nPP+eKy21W2t5S9vAm4cB2OfpjqPb8qsretcBdxV2OeG4OT1zjnP09D07oZl3l1PNfbRlAMbQpOce9XLaLzopJ5pEkSFlTaHB27s4LDrztPP06ZGUuUZwt4gbynfaZUAGXABwuO+MfnVW9Z7mKMRlIWSPYgbA3nIJy2BxkkjPC9M85LQG0moaTbSLNfuRbEgGOJQHxjOQOPy47896oP4ojhut+mSOsP91zkH8P61y9xYapLI3mWsxCZy4XKAA4zu6Yz3ziqhBibZkEjqR603qhXPWtP1GPV7H7TDgMp2yLn7p/wrSsrh1JQnOK4fwYtxFp9xK5IjnkXZnvtByf1/SulEzROSDXBP3ZtI6IK8bmrcysIy6jJqS2SN4UZ1G7rWUt3iP5hkVqW0yTxrhCOO1J66jS1ILx1ticNkms5bqRMjd1qzqiLEm9jWEJjK5ycYqLNvQT3JbgvLMXY4qrDC7yNtOQDyaWS6RUKOee1RR3SQREI/zNzVq9rEuxqqzRQuDz6Cm2tyDdR+YNq5/OqC3b7gSd39a3bSxs5ofNumCvjIGamVNxWpcfeOnkCNpo+zDBIrljaXUcjGeAgA53VtWuoRRERMw2r0Oa0buaB7AkuCW6VnFuOhrKKaucqJp3bO87R2NbdvJGbQMcFqyn8mNCC3zHsKZbTPCxA6H1rd6mK0Zfu76RYWRATxVTT7drp/3pJyOeKsQRNjzZCME9DWtaWrPloFU5FQ9EUtWcpqVqLNx5XIJxU+jWEz3DF4yRjjitS6geOVY7uAbd2Q1bWm7BGwUj2pTl7ug4x1H6dBLBEd6hQOlGoaj9nt9wOc1NdSCG2Y+ZziuUuZ/NXaWPBzWcI82pcpWJvt9xPLtkXMbHrVa+srYT+YxBxyPrV20ZZFCryFqhrIWWP9ycYPNaONtjPm0LFvqUJQIGCsKLu3a+cCJOW4JrmA4ikyc7q7bQJXmgH7rA9TUSXLqVBuWjMy18KNHdGSY70xmuT8QSLDeOqsAAcbRXr0kLC3LFwBjmvIPEkNs+tyeVJuGfmOe9XSk5SuxVYpR0MyFjKMMTSPkECM4HQ1oJEiBAmDQzwMxXADVvz6nKYbGYTZGSB3qyNjYJ6jrmrjbTJt2gA1BcWaMd6tuI6gGr57iKYdDIQMYNatopAVQMqajhsbdhvbC7e1acBiEfy49qUnc0hT6ss28z2/Ck4qxHqUyAmqO8j5j0zTXmG47TxWfKbaI0Zb4zpytQH5uD0ql57DGBxUnnquCW607W0Juicx+WRnnNRZiMnC8ipRKGlG7BAFNklUEnAFDJZCXMl2ihcL3rceTy7cR9B2xWKnEisozzmtJ51mQcDik9dBxEPK+ZvJx61ZhBdPvYzVbaioh6g9RVp5YwFKYHFLlGzXtGEcI3DJApksjtKCBgVnR6ikAxI2RRNq8LRZB5oSJNB5kWPLGs6XVYUVgoyaybnU3nG1ehrOkd1JP407Bc2DqLE5wKKwfPeijlJubVpGizZ3AkdBWjabnmclMqKx0VGkk2SDKHHFX7W4mR9i5KnqaxnNvYF2LNwoYsSowKzp4IxFu5xVyR1RzGxPz8k1A5j8gwb9vf61hzNWG13M+OBmHmtJiJewq1Fcw7lwfm7ZpzQb4VSMggdagKwQQxSbCz55xVq0r6kofM4M6hDvYnk5pXkeMMzHc69PaoIZohOf4SeVFPniWIvKSxZhlh2pqy0YgguEaZpG3ZPVjU3mTS7pYCu3ODmstZQZOOEI6VpwQtDZZB+VuTWijG92JMjuoZRJE4IAPU00zowMILbG+8RVe9uSGiTkoPQU6G8SQNFswxHFQ432GpdiawlAkMPr0J71qOjuyqQMCsaGy2tvLnevStGG4+Vi7HrgVMtNEWmr6kE0brMSx6dAK1bBoBkuctjNYr3GLxgfmHatKNAY2ZeGZafO4qzKVuhsWEqyRO0zDys8CsvU9eAZrO3XaAfve1UUadPlGcDsKSKFHkZpB8xHU10U5R6icm9hytuZST16024lXzAAaZ5DKhdicVTmhl3ggkg9K2TVyCWSR3APYdasQsGQRqeWqrscrt/iotoZlugT09abVwHT2bJIQpy1OWCSMDAzWiqIgJY7mNULq8dDhRxU+8NjZd5YDaasxxFkGWxVZLsTJtAG+po1I5ZuazlJok0rSO2BBmkYYPTNR6lPA74tmOKqM+3qaQQA/vN2ajn6s0jqrEKxYILZIqbdtHA4ppVs9cAU7cEx82c0OVwTsSMVwGPftRGkO7LcioWBdeopMBM8nNCKcrluUwgZTqahG6XAB6VXYsF9QKSCWUZbpzwK0TdjNs1o2WIjCA+pqSW4kKkFeO1Z8dy67iVJBNS/a/VeKlp7gQyWztJuPeoW0tpXwT83rWjBIJye1XBtVQRjdVajSMyHTbi3AXOfapJIfs+4spBPetOWQxQbxjd3zWfczm4VSSMUNXLM9pQCNrc96RnJkXDYHU1TuM+adhp6bmjyRntS5STTt7yWBi0bYqKa+ad2LsSR6mq8QdV5/I0YMpLAAEdqlRQakyYkHJwO9X4nURhkxn0rLiVhIQ3HFXrSVEGCc5NWk0C0NKyujBOS6g5qW6m8/cwUDNU4yrzZ3Ac1YuVJG2MZOM8USdnoNGewfaVY1QnZQw3DmpbiRxOM53dKjmQsAWoa6g3dE0Uw24Uc9qLlPNjCgnzDVSLdvyp4FTxsTIeefekkLoWtPhAt2WUgEHvWhG8cSY8wEVnC1LLkZJPXmqly5jkSJASzdqVriRqz3qlAo65qpcXRkIQsQVplraszZdvwqWe2iYZLfOOlF+g7kSXBGdzEg9qq3N+0VtOyD59p247GnC3kZmywNVL+2eGyZ2xydoLdM1pBJuwmzjdWmy4jHUcHGMYx6/karz39zI7kysQ46Z9adfgbtxzvbHfoKpGuwwLVpZTXhZ0IAGfmPc4zx/P2FJLKIyYl+90ZverWlPezXcUNnCZpAM7cA8A9z2Hv71DqemyaZciCWRHfGSV6frSur2ArpJtzjBbtUirhfc9aijGDk1ZhVWOWIH1BpiJI4SAZSCdo644FRNPmXKjODnkVLKwlQooOOgP+c/zqI20nl7wpI7ELxSAs7xKud21sYJOe/Xv/AJzTijsP3j5UYAyOfp7f5+tZisUfOeR0qwsjso3Fj6D/AA9BQMvRMsQVEXYG6yPzn6Dp/Orz6daTHe+oGZ8jPT0xj/J9MdayHaJVzLKS3ZF6fj61CLti4KKqYHAHH+f8/SgZ0uy2sxHIsfyjkEt93vnjGfy/pVY3Sv8ALuBGc8k5P4d/x+lUYb+aUeW0hUHpuOQfTihZFtjudMt/BhuM0hmjNbRIivvBcfw7ckcnuP5+mfap9nk2qq0D7WTL7hjqAR78cnPH5Vjf2leGSQrPh2wSec8d+nt+ntVdr252YNwADyOc49+Oh4FArm8WWNnV8q7JgS9cgHP164z+GeorOvZPl3AjK8DDVkGVgNqscnlvc0+SZpMgnIoaC4153I2bzt9BTEGGByMikZT1wcGrWm2hu9Qhh6Atlj6KOTQ9ELdnc2UTW+lWsLN8yoM+2ecfrV6CRghJORnGapQObmdgPuipGfypfIGSCfyrldrnQrpGrHFE8W4tz6VuF4rSwjVP9Y1YA810ComMfxV2unaak9pE9xtYha46jaehvFXOT1W4QW7CZckjrXEzTTTT7IiQoPWvYtQ8OWF3Fhm57AGuQuvCrWF+rxEGIcmtaNSKInBnJTWMuFAySR1qEWzwybHBPvXa3UcXygbQayzFG7NvAIHSumE79DFxMRI3LEpnjpWvb+awVWJ461JbxRLOFwMVpywIhyCNpXt2qZy6FQRnsQrEhs1NHcMQoLnGemazZ22NtRs88k0sLM0mA2fap5Ey+do3JI4yoL4x1JqylrDdqpik5WueN/tkaFs/StTRJ/8ASsDgEc1MouI00zZltsxCJX+bFaujQPaW2Glyaxr4PDKjKckmlS6nhJfdkelZSTa3KTSNzUIvPgbJDY5rn9JupotRkib7hOAKsQahI8rb8hcc1Tt4nOqbkbg859KajaLuHNd6GvrspgtVbPB7Vkxq91GsqR8Y596TxFvDxLvLAnoKkguhY2y7yFyOAaqKShcmT11L1jp0jfPFhEIwQaz9Rs1tSQTkse1Qf21cvuWI4XPap4b8KpkuBv2+tTdsSsUjoYnuopXbbGOo9a7e0S3trVFXGMVxzask0vHyr6VuLqsQsgyLuIHWoqalwaRqXvmm2bZyCOlecXvhXUJppZiqKpJOc1sXPiqfzxEnKg811FswurFW5yw701zUtR6TPHb3zrKTycEMvU1A85dM4/eY6iun8R6Pdy3xcRblPQKKwG0+6jJAt5Bg4+7XTFpq5zShZmY9y2MkncO1W7a9G0gLyRzW1p3g++v3DyReUh7sOauz+DPsLrmTcCee1Pnhew/ZStc5je0wKqpz3Oa0rO0nS0MjkYB9a17bRbZbrEh2p60y/lhjVoIeinH1olUT0RXK0rsgEytaMMCsxHdXJIPParcZG8BTx/EKGt9z8Hg96EydXqNUvJhVGSatw2QBDTodo6VJZRJazbmGR2rQklWVCO57UnLsWo21ZTlhAxJGowB0rPD+Y7hhj0rTuLebygIyeayxbTjO/kg01FNainuT2UZk3RuduO9PbfECAfxqKR1tyqrklhzSNP5nytwBS6grDjdMihM5qQyEKMGqwA69amA3HhSO1DFqyOR2Z+uc0wnC7as/Z+N3fvVa8jaGHz8HHShK5LTGNIsYz3qCeeQ4Xbx2NOjVnXcw4IzUiAEZbtVWsK5WxJRV0SJjpRS5l2AvtHFFL56cL396vw3CM20dD6VBDC8kUf3do55p81rHaTC4WXg/w1xyjG9pMaG3jhJSmPmK/LUEUf2lFJOJB1FS3ZR5VkyScc1BFcmGeQBMjHFQ1p7om9RryS27bc8H0qu4cjIJKH1NaAaKZC6p8+P4qozLMyYCjPcCrptWF0JIZLVHUyR73P3faprqRREZJBlScYrOsow10vnEgLVq/QMpWI5j6im4JSGloRJ5LptWMhuxqWWSQ2+wkgCqsDfZjhwxLdPar0MkMzgztsQDrnrT62EjJeSR51jQfpTfniYswOQeDUkEqW+tO+d0eDirLMJUYlQWPIrWStsJLqMF6GcFSRxg1dh/0mFivCr196qRRxSICy7SDzU8LxQSsASVI6Vi9ykPe0LbXiGcck1Lb3Mgl+YEg8YrPl1KSH5I14PGKlimfy1c5BHOaHF9QV0y/JHJGwliyV71IJEmO1htY96htmu3TORg9qcqPscyLjHQ0+VoZMMqpRzlB3qm7ZbPp2q7bSK6NjGcdDVaWMzKCihT3qoTa0Y+hXDbsnGDUqMwjyTTI4yAQT0qVApO0DJ71rz6DQqOSME02S3+0gKnXvSvJGCVHGPWkt7jad61Sm2PlQ1LQQvlhhu1ROWWQktx6VenJnKlR71GLdXYM3QdRRa7JcCp9oLLsxj0NPtHdpAjH5QamlSJW3Lg47VDHIocsEwRTcVbQErGyYYgDwDkViXbBbghVPy1fhm8xO4PpUcqNIWCp+NQotblOzK0TFwe2KQsXJGRjpS+S6RuM5akg06aC2NxITg9BSdl1IY4syAL1B71KuET3NRFlCgYOT61Tnu/JcAtye1VGNwNMykjAGR3p7IHiyB1qvbTjaC33WrSMcc0YRDgAZzRKajoCM+NzG+BxipmumMqlD93rSeQfPLcEYptvGTclnQgelZykhl27uEaMFmOCMcVRCxopJbr0FWZFTB29M9DVWRV4yKqE00O5lys7uRtIHqKu2xIj2gZ460FRuKjpSBuvbFU5CvYjkeRHKnNSRsdm4k1NAiTPvJ4p80SZwD8pqUykna4xR8hKnLEVDHG6nk1JJiJG2HoKpfaTxzzmi7JbL8bsCfmPNaNreGI4kbOe9Y5kby8qKVXkXbvHWjlbGmX70RG5LxtkN2qncSmMAk59BUckhE/I4x0oP7wlWOOOCapeYXCO4MgOwY/CrPlYIb1rPt2EGRuGc81qxYlAycinJWehS13NLR4jIr85NXY7JGuCzwjI6HFULJjAMxPhgelasN1NK2WUAetZu4bGdqFjIzlkGzH61lG2vZmUbcDua6i6lKMGbBBpFuIkjOQvI/KmkTZGRb2cYG04B781n+LHjt9JiTHBk5PsFPNaEyLJdMYXw3fmuP8cahIq2lpvy43O306D+taU4tzJk7I4+8YvcsTwe/1q74e0tdX1mG2kbbCPnlP+yOo/HgfjVlPCeptgyiOLnkO2SPriut8PaLFpkTYYPIeZJMYz7D2redRJaEwptvU6WODTdNtJDZwQwoF3OI0C5x6nqe9eQ6ve/b9TmnwAu4hQPSuz8SasbezkjibAkIXI7f5wfyrgNuBUUIvdlVmtkKuM88j2pxbA469vaod2OlGSxrcwJllOOefar8TNcoFAJAzkgbR9PQcc/h9KywrM21QSfYVsWTrBaSKZASRkkKOMYI56n+QznnqrAoXEKLgxZbnGcYB+g/X/OBC0pUFV79T60sr/vXAPy54PPSkKoMfMWbGT6D/ADxSGRhWY5qdInAyuG9dvJqI53LkZU9h3rqtIttPl09zK8ajOZFbrUTk4lRjc5hj2I6fpSGV9u0nOfXmt3VLObTdk8sCtaTuwi3ZDAD19PbPpVaOGyuiMBoi3HzdM/WhSK5L7Mx+tG3v2rZl0GVSdvanReD9eurY3VrpV7Pbjkyx27sn5gYqxcjMTIHuaQsT3re8PW2lR6uF8QrNHbbSF2ISN3+1j5sdemecds1uX/giwlLXenami2jMAihfM+uGzUSqRi7SBU21c4cZfaignn8zXYaJo7QxBpV/eyAEjuo9Kuad4Wt7R1lXdPMnzb34C49B9fX0rZjiWNkcnae+a56tZPSJpGm92VZbZ4AiRI2T1IFaNhYho8kDzFOcHvWnaCS4HyopVerCs+eeeLUnigQ/dySRwK5byk7G9kkWpriFUKkbGFW4Na2RBFcbAPWufuYLtVLzL87dD2rMCTxMfMkxnpg1sqXNoZ87TPVE1C3+yoUXdI2OKqatHI9v5oYhccis3wnDIcOXEvHA9K6+SPETCSMYx0Nc7XJLQ2vdHmlzYidlLMwx6GqjxfZYW2jcM1v6iY2nkWNSNp5AqmQGjQbNwY8+1dMKltTGUUc4gm8zzuQKstKwTKueeoJrpzaW0VuUdV5FYkmlo0bPGc88Cm6ikTy2MXUY5NgdFIBHUVmwX0sLhAMk967zTbaKSEQ3KBh71TvdMsYZn8uJRzxSjWS0aHydUzBhtXZTc53MO1XdLu5Eu/mXaDVmFFQsmQAac0MKTADAGKbldCtY3zdpcWwbHzKKqSXH+jsR94U22tt6ZikwB2NOWIvK0ci7fTHesoruXcXTbpZbadpuGHTNTaVdpNIw2456mlWzXBQKcEVHbxCJ9uMEdMVTaYlcsasdrISAfSuYvpJriTBJ46V1EwjkBWRvmA4rl9QLrKxQ5pW1sTMfYMYHAk5yav30QKLtPBrBtnYsfMJzmtG5uXMaqvIFaOPYSehVnjKvtQkn2rZ0yyuryyaLds4rJhkYMSqZPfir9rfXFu4OcDPSsZXuOMtdSew0D7PfmS6YFRXTW2oxR5jH3B0xWWJo72HJcgnrVg6WkcIlhmGMc80qkr7msfItLeQzzBVAJJxz2rRudPgNr5iqhI5rizcfZ7o/NznrVyDV5wkkZc7SeDWFmCqdGdFa3QP7ogDAqPUrJGjEpG4jnFc/LfGPDRHce5qxLrLlQwYYxypqo3TuVzrqUNQuQUKCEJg4zWTPACOFznuKs6nfK5BbjNVbW/QSbQPl9TWqbSuYznfQZBaRNuPIbOMVbSx3Nt6D1qGZo2lLRN7mrR1IQwhWAyR1pNyMyjMkkVx5ecgd6uLH5KiUEHI6GiSWOSNJSRWdLf4crzjNNtsZqRuzyLnjmr0lqnDYGcViLMZ1HJB7Vfjn2Q7Xcs1PmaQ+YrXdurN9wAisyWHzG2qMVrSzQiP72WqhlhKCF4PWtKbb3DcfFbLFFnOTUc0mwjHAoMuX9s9aqzTbmI681so3C9i2khddoPJ706VJTAInIaMGq0Ei/LxippnyCAxpWexaae43aowo4A4okt0MJCnDUEK8YyeaeeEAAyaLEtoxjb3m446UVtBZSOlFO5NiZg7RhI3wvTrStAWlClyxUdz1qm/nJgbCeecVKZzkvgg9K85Qb1ITuWJYTC4LN97qKgc4nJTt696W6kM8a7G+b371I+EgjYpliMGhKyGyKG6V2dWG1jximvL9mlBxz3qB2QkcbecE0+cokQUtuyPvVpGKuJEwgFwxuBgd8U2P9+rhWChe9WdLt2EDvIcZGBnvVe3KiZ1YBeaTdm/Ib0QgWJQZpWzxhRinW9oLyJ5JOf7oFOe6gklWF4wyg4zUd3qC2YCW6hR60R5r7DutzGEDtdyAggJwKtQyP5mwdcdTV20LSwuzlCzHOarOm2c/L0HWtnJPQLIcJQsRQj5s0yGQPPuPUVGUJjZmBwOlS6dEWVyV4Pepe9xJj/LR5CSB1pXRhuAPBpQUaTarfMKmjjYnk8VrdOwx32zyVRAfmxViK7klXymTtmqjWLSSeYDwD0q5FYzrIJN2RjoKzkocvmOKYW7fPkJtI61YwZDtVR+FFuIY2LPksTzmnyj5S8J/KsXJXsFhDbxRxjePmByasWyxKHkCgEjiq6h3QZI96iuJY8hVYjA6U7vYdmirqrGI4jQHPUiqlsHji+cdTVtGDN8xzg1ajCzHaVAxXXHSNgSuQxT+WQAM8Up8yYEqAFNSIFMhRgMCnzlQoVOKGuxfQpOixn5icVVM6LIwGcdq0GKldrrUcttDuXanSnF9yHqLaTROd2/BHatHeAofgKe9Z8VmrS4xya0f7NmkgyFPlr3NSy1KyKTyqzs0a/jUouPPCxykAVDOBBb4HUnkVnCcmTcRwK53G+pk3qaN3ZIF8xZPlXt61VRIZSSYxu7ZpVuFkYBjj2p7/J82KuEtPeC5LcRKLYBQA2KjsmkKbHYqD3pGkaUACoyJ45FYj5OlTJ30C5oIscblQ+T1pHuioKqvb71UoXAkJlHOeKmaYOSmNq+tZ8oCrIrkjNSbS8XABI71myN5b/Srtvcq0RVCR61WsdhqVitMShBI5bpUOWJI6etWrhUIGTyOc1QMju5CcqTW0HzK4IvxbII92cg1IjpNyBjFEcKi12xqC3U5qJpAi7SQrjnipvfY2UtLISVWHBHDcVQltiGOR9KtNcgIVPLnvVcvl8iXnHANaQTMmSwB44ycE+1SmV5VA2H6+lQwvJkDBbPpWjGgMDgk7sdBTbs9QRmssgZnJBHap7LMuTjOOtVSs3XoucDPetqw06WCIOzfM4+7U1Goq7BbkAskuTiFQZO9W4tMmjh3KvzDgip3B0+JmUAM3SkXU5bXBf8AeM44ArF1ZPYblqUfNktJSkvWtPTp3uFZlJ2CoLpo7uDcYv3hqCC9NgPLUY3dRVqfMgubU8iR2+6Xr/CKxnuHJJPAJp95qYliWPbkjoazpLsA46k8Uasdy6vJ8xOv1rm7nSLq58Tm+uSpgQqY1zndgcDHpmt62dRbv1OTUE84V1xkkValKOwmk9xzrK5OQSCcmoHkmVGRCeauQ3IGcN94d6g1JJ0t1aKGSXjJ8oZI/CiMujKucZrcsv2jY/Q/eGOPr/L9KyC5VsONw/Krup3bS3TKyupHUMuD+IrP3bhg9RXbFaHNLctCS0bpb/8AfTHj/P4daiZVUgmPbnpg8VFwOvT1FXrUs+U2+YPeqEVpp3cKhCJGOiooA/H1P1yaBM8MZWNyN+Cx9QDkA+2ecfT0qw0CgnA47iq0kLA7hz7UARj5myx6nknmrSqXcyRrmMkDr93jv/jVQ5BwRg+9T2k/kXCOeQDz0NAGpbaO13KkTOqCRgFl8wBEOR97j29uoOQKnubO90qKL7XEvJJSWEhhgY646fj+VQSX8alZYkeCTHVUKqfqOeff61rafr10wRBGbhBkHykySBzjBHtnvxntUts1hy9SGfU31bT0tJ5I5FUgmQj5o/cAYzjJp99Y6cttbf2YXDoCkgfJaQ56+3+FXn1bRZ9Q+3XFrEk5k83cijOTznB4/Sq0mpaDEHMFugbO5coMZHT8P8nnmpRryx7mjo3iKHw1fRCe0s9WDRBlDHPlMOSO4OPUj6V7N4H8b2/i21nkVfJkt5ArRFgSFI4P0PI/Cvna7ljuLdlhKBFYNsUYDde3Tj+p9TV7wx4juvC1x51tb+YHcM4JKll7j3HXHvTautBxnZ2ex7v4r+Hul+JJGnULa3h6yquVkPqw9f8AaHP1ryLVfCmo+GrsR3UbRh87JY2+WQD0I/kefavctD1y11vRo7u0k3xyLkHuvqCOxzxiotSitr+1eyvYllhcg4PYjuD2P+PvTjLuaSp3PDzql3DalQwJX+LHOPSsy41GSS4UvIce1dDrWkSaPqj275eBuY5CPvr/AI+tZ9t4Yupgr7SQx9OlZzUI+8YuM07G94YvLqV0ihjzCeT6muivNNET/axwWOChpND0hNOty7yFXC8CrUMhvrhYVfdjue1cEmnJtGy21M+7toLp4ozlQByKov4Ke6uN8UpC56Guzj0W3Qln5bHWpIY0hG0SihVXHYHBPcyND8PNpMxdpj06Vs3FxHLGYt4BIplwrGJnDEAd65pLmUaiCzZjB7mou5O7HpE0V0UFJGI3Fu9VW0ZIvl389cVJqPiyK1iMcKb3HGBWXZ+IXklLXCcn9K0tK1yfduM1CwkmuEEbnao+Y0lvZNHE2BwPWtnTpI7lHJKnJ4qpqbvCGWEgDvSUr6CaW5mQoyyln4xWfeSAXLM4+U0lzdSxxF2f8KjNyLmNRsOe9aqKJuiu8qGT7uB2qrMZGlUoflzyafdsqS7e1MjSQgEA7c1S0Mnds3Y5VitAe/tVlLvO0kYOOCabpkEM0RjfknpmrUthJCAGTcB0qedbGiWlwjvncbdn402ZljBkJ5NOGI4hhfrUrQQT25aQHbQG5lot3JOHMeUPQ1S1G0mLsVXk9q662ESW6BT8i+tYusarbJN5cMeW7ml1Bx0Oft7NiSGGMVchsXkkG87UpsU5dm/h71d82GSI/P8AvF7ClObWxDJAkNrEQqgg1RusI+7sR0pJNQVEwRkjpVS4d5yrjhTURve7JuWkvFjIGcA1Ze6dYCxlZV7DNZWU3BZBwBSqDckjedg6CnJJu47sjmm+YsG561PFeKYdo61UuY0jiOPvVDZSjz8uOB2quVNE3NWSUxxgjvSo+UOep6VHcSeYw7DFQuWGAp6VFh3ILwM8yxjmiL9wNsqgCrUanzNxUZ7VHfMssRBXDCrvfQRCVCMXV8p161WmuDI2O1NJxF3A71LapDKxVuh71fLYBPPbaqrnBpixyNLk9K1CtukQTA3L0NU5XVuTgGlfsMnileRhGq4I71JKWjGcknvUNvciKA5UZB4NSJcB3O8dahrUaICC2MZ9c1Y2yCPkcUxvK83kkKvNPN0JF2D7varuwvYpyLIV2oTnPWpJtOdYA4cNxzU4uY1Ox1GD0NWLe4tXV0due1W6khGLDvQ4INWdkr84wK0Y4EcnkdeKsfZztAUcetXz3KRkEBWUGpmzgbKnuLQyLuXqtQBfLHJwaFIGiTz8dqKZmP1FFFwuPhEyMCx4I71YmkWOEDYGPfjrVdHe4Kk5wvOKlW3luSJEPfhT7VxSWupnYjihJkw0ZC4yKvTXlvDGFlhOzbxx3pIZzGwjdN0mcEY6VRvWcXJVhlSM4NY8nNKzFqOWCzuFBB255psVgDMyOy+WPes7zWjZgAODkA1XS9mScszMN5456VtCEtkxxa6nQzosAjSJmPoKqNJ5rCEqFbuwpBeM9sFfBbsfSqjSkKUAO7P3qtJ9RyLaWyOuwk7lOcis27DSOUA/OrlpPNBIXyHHcU7Utjae11tCN6U4txlqHQzNNldfNQklc9fSr0UxR9kxyrdGqjaXCRaW2Dl3OT61M86zJEwXpwRTmndsbSsXL0CMKqAFGHUUsUqxAIp+8MGo45QwCP0NRiJhchQ4welTF2VmBcsrItcmTA+UVaELZIQZJNRWshhnffksBjg9aiTVNk7qyFeeMU43ZWhfWNwwibHHJqzDIUZtzc44FZlqZJpGeRyqnkE96uN5j4KAYA+9WNW6YDpWRjsEeGNUzcNCzIpy2fu+lWV3lxKWU7ODVFX23jEbXdjTik9AvY1rPy2dmcYO3iqjwvMzFFBA7irtvCfs7PKMZ7UsDxWYL4yG9ahT5XoO5kt+77URXBjyx6ntWw0UNyjNtHTIxWK0TGU714X9a66dRSQWJ/Lcp5y9zzT1XDZ3ZJp8b5hCdM9qlEabM9DWiaH0KrW8ssvI465qfy9kZJHPSmz3XkKQG5I4NVPtzyR8847Vnzu9ibl0u0YV1GWPStdb4x2qwuQDjJrHS6DRjMfzdqpXErhiXYqaqUn0C9i1dz+ZI54x2rFYSpIWKHb1Jq1FMrqd+eKYbluVIBUnpWaujN6kcOyW4yemOlWZjI6gKp2UsPkoxcAAkdKel2T8jDkdKmT1ugRXjZohls4Bq75jTR/Iu4Cq8zZABAwevtVuHy44htfBxilJ6DIGiZh5p+XA4FKFIi3yN1NMu7gAcHgVRe6Mg25O31oimwuWZYTLLuRs5q0oEEQiIwxqpbAxRCRs1K8rFlZu/Sh32ALg/wAPVsdKr2wiV8scH0qyVDSZU9utVHt5DKzkd+1VF2QXsTy3QjiIz19KqKxYgjDE9qlkhwVDHK96mdLe3jDKeSM4q4NJlKVmZl3DcK28HHpmqquXGZSFcHitb7fDeQCAsM54qe38MzSsJQMqfWuhtJajau9CxpDK1vtfAOeCa1mgjijL4yzdxVa20oWcbJN9QfSmw3WJyhwUBxXDN+9oAgaCcCJkGQflOKvRi5Vj5m0KB8pzVe9ntUeHjaCeTiq+q3y/ZGWFjmpknLQTZNeSEgGSRdpOOvSmlAyLJA64UdD3rItrK4u4lX5sHnOa6KzsoYrfAQtIowc0SXLpcSK9rdCUYMTZzyakeydphIYvkqwm23kG3aFJ5B9auGcOCpHHakppbFJmdJawsACNpFY13ZlZCYyGxWk9yZLxo41zVW5bZLtOPetKbkmNIgj3pD8ybcnr2qlcuYz5pIIzirVwsj2md/CHIrMlSSXy1AO09a6Iu+oNly2xhGByc9K3oJHYsVXgCubkie2KSryo4xU+sa5dWmlYt2MLsPvhQePqQcGplBzaURqSSuzD8VyXUkpFzfxtg/LbJnA/nn8awHiS6VpLa3ZPmC7Q275jk4HHTg46njvxUUpmnLysGYZ+Zz6/WpHla6+6saMzAlUwgz7DoOp/Ou6EeVWOeTu7jXs7mO285o8xrwxBBKem4dRntn/Gn2kxjYMAM9BVq1huGnluJ3YeXHLmdX+ZHVDtBPucLz1zgezI2TUXJhto4blUZ32kLG4AyTg8Kfpx0wBVMRM6lhvwCSPb+YxUapk5xn8RkVVE3HKuo4zxuzj6/wCNaNtIwt8y4EeeQSwH8j/gPxFSMjayE424KtgnkdKypYXhcqw5HpXR3EmxC0ZiSML/AAhfn/MDP4cj0FZN0yyLnA6cbFx+f6UJgygrYPQH61Iso2kbV56nHPWoaM4qhGn5ZvWy14nuZG+Yn1/r+NaFtaQWqho8G5XDIwJLDpjj/PX3rngaf5h2bQ2BUhex11xr9yihXlgV1O99gG9j/te/1zj86wNQuDLIZTKrYJUHfnI6j/H/ADiqMuJJmYH7zEikaGRRkxuB7ihJIpyctzp/Bvja98K6hlS0tjI2ZoM/+PL7/wA+n091sPF+lazpv2mG5jY7c4JwR7etfNNrZXd3KI7a3eRmOAAv9a73QfCS28IluZH+0sDuRWwo6cEd+tZVZKOtzoo1Wvdauj2W6i024sZWvESS3IyVbk+2Pf0xWFovk2UEkMsgcjoa52JXsoC8js5/h3uTj6Zqey1mOLKOgOeprjqTc9DeU1ck8RaowQLDwc9R6Uzw7OFuVPmEY5JrM1CZLi/IxiIjjFTWMT+Yoiwig8k1NtLGe7uehDUI2yF6Y5Y1SiESzPMDvPcZrIS+nMHlqqkKcFqqzaoLaRogcE9az5CuaxtNrau7QSx4UdMVzWrzqJwkGRk81au5A8COgAb9awrppMmQMCwPSrprUTYxRuZgeoqKWTK9cEelW4bqHY0hXDgYxWQ1w0pcsNpz0re9zOTLEOpzWikq546c1aTVpL8Eg4PvWa1u0wCouM96sWOjXKy9flJxmlaG4knuK6CWQq56dKsWrLGpEmK2r3S7e1tUJI8zFYVwpXcV5o51si1Ebci3cBsc96bBcKGEePlrKmeVXKjk1JE0khHGCKWo04nR2z7SCpwuetX73WTHEqI6saxImAhCvwad9kjk+YHFZuzepDfYfJq7vLg4C1pw6rAtuEI3Zrmri1ljfKjIqzCGWNT5ZppK5KZuSXPmINp2rWLcQB5WO7PvVgTblCr+NCQlnzkKPek5a3G7FZLYIobJYGmzIsS71PJ4PtUt5DLHtaNuAcnFUGkLswJ470tXqZSZXaREnznNXgrPESB+FZxaPzQnb1rXSaHyQIuW71UkCKUrEoFcYPrSW0u1yM8DvVuZY51w3FUpIVhwYzuBPNF01YbIbtvNkznC0/T4fO3ugyVp9wirFlVzxzUNnMYFYBsZ7VXTQRoOypHgr8xpPLZkBxUXmF489TUylpMHO0dxWdmBWnkePAH504yRkAsQSetSz24YbA+c9apW8INw0UnboTWkUmO5FP8AvGZI+mKqRzNESOprTFmN7gOARWbcgRyOvp3rRWehJNHdbmO4mnsqy49BVKGQMhOPm7VdsvNMwLR/L3zRypDvcl+zs6DYePSk8wxtg9aJbgwzlR0HpS2s0U02SKl9xloRsFDMuSaUrE6Aov1FSEJKvD4UVDxDEWRs1ncBlzCsg3KcVHBb4fcx6VLBIZJVyBgmrN9B5agx9xV83RgPTCBX4xV43iCIkrxjisuE7EVZBnHWpvNR43OMgdMUlIaYkd0zMTtxk0TmOVsMQrAcVHDdCMYdR7UPF9pmDIMetHUlsqmNs9KKueUo4J5+lFXcXMTLJuy/AZhwoFRJPN5rDO3HC4pLZi92xyAq9M1QubzMzhfvButcii27E3uW4buWOVnkAdu5qnd3bvcb85JHJpHfEXXOetViyfMG9OBW0Yq9yk2Ak3Nuc9KtXEUNwkTR4U561REOIS5Jx71Np5DzCM5PtTtroJdi1caZLCQFkDqwzkVAnmb9mMY710GEARPLy/QU9NOWMuX2lRzz1rL2vLuXY59YpQx2Pz3qGVvtET285K7eg9a6BIrJ7rBbbgZz2qtdabbPN5qMZN3QA1brJpXE0YPywyIFUbSMVO0DrIu05Vv0rQXTIZi0abo3zxuq22lskIQygMO570OrFjjF7sw0wLtQx5z0q1IY4Z1Zh0qveabPFcrKTlc9RTrxy6pGRgiqVnsJ6CrcEyswY7SalYbplcKSD1p8FvGY19e4FW2MYIEYLcdu1KTS0DdETBmcRliEHIq5auz27xg5K1TuGdZIsqQtT20qpHLxgnpmsJK6GiKSUR7liB3MfmqW309IJROzEE8gVUtWb+0CX+6fWtxh50QUjA7GqleySKSuWPOWQJCrjPWqdy8b3CRyfLTo4UjukPJbGKkuoIyUZuDmsbpOzCw8SrBH8sfX171XjkQ7mI9sGpmQvgFs7eRVfJy52ZOeK1hawNiSzojKAARntQs6YYA8+hqZVjaPcY8PVKYRlnA4PqKtWe2gm+wskYmUu65CjtVRUiBBOVGatWspSBtzZWqrxtJuMfTtVpu+pLLwZcZi5GOKq3i+bgsvIHJqKS5EEKIQQwply7SWxMb54oV7i5irGZHDBRwKqSzsh2981b0+YMjqTjjrVW4jRJc53E1qrXsKVmtAjklkkUDnHer8ZzJ7461Rs2cK5H0qT7SqkKPxpSWoloaCKxGHGR3NRtKElwucCmi5wF5pZNrA4HJ5zU2KFmcSIMd6rvJtj2BRx3qv5rRyjac1Xnu2MxAqoxZLkXjcsYsE8CnJcmRsZ9qyllYAhu9Swz+W2MdapwFfU2rebJ8s/nV2UZXEQ3EjkCs+xkjZtzCtQzRxbXXAJrCSsaIoOjyFfkIweat/2bHIhY53MMYpLq5UkNEfmotr1nGMZbtU3e6ErXMKTS5ob9I4kJUHOa9A02RbeCNHfPAz7VlQTAyHzE5xjNSOuzqeTWk6jkjRabF7WVPlMiOMP0PpXMSRNChMSFm7kVsyyCZFiPpxzVNAVcpn5e5qIsT1Maa5nkCgjKj1q7b/AOkRmYKCV42+tLe2cKxySQygkdiapo4jhUxylB1Ye9aOzWhB0VlDMttuPyM3b0FMe/FkCu/cScVXTVWNsNp3EDnNY97qDsm04B65rGMHJ6jbOjSye4PmPLgHlVqyrqibC25h1rl9O1aczKZHyo4xW3JmO6WVVzvHTNTKDTswTG28ge8clAADwarazKqsqeWRnnPrWwVgdBvQpJ7Vk6pDO0qkJvUd6ulJOZorlKdCbNZUkB3cCPNV9PuU2OZjtK8bT3qG4iZJSwyAwwPY0yy0m4muU3MSmeTXbZWswb10NDb/AGhgk7VXtWfrem3F95MacKvXP/6q3La0NjKY8bhnNaa7JWC7Rk1EW1LQTV0cxHY2tppitPaq6phFZjxn1A9f157jgY2p6datm8eYRwfd+VfmJ7j2/XHTtXb61ZpNpMkBk8h0PmI2QBkAjBz2wSPbr2rg76Q/8ImmC3N2YmywI+VF6Y7dPWt4ybMpKzsY0kkacRwo0YOQJBzj3Iwa2vDd/Gt3IsJstOujE3l3Eu9lb1QglhyM4OM5A55rGALEduOpqtKuxyK13ViTq9T0iRBJcWjgxAF9uFJHqfy/zxWGbmT5ArICDn5QqEe4wOBWcCw6E0DIPpSjGyswNCfU5i+CQdp+UBuB68iqUkjSNubrTMH/ACakkhMchj3K5UZO3ke/5flxVARUUpBzSYNABmlpVxuGcke1bHhvy/7ZtfN8tUVwzO3GMc9T0pN2VwMhWKMCOoNeneGCb/TEkd1lZTgNkcexGBirL6Np1zOk8dtENvIZBwfy4IrTs7KCyR/JjEe85IUYrCpO6NIpxJmjQ7AqBSh42jHNJMgifzAcZHNK9wqyBOhHWquoXf7vdg4HpXO43NblW7keeMqGOM8VRJkgTEnOO9KZ/PAER5NFzIwgClT7moS6CvctW8iSIxdgABkVbtJZDG21d6Y7VgvIRCZMHZjFW9KvJBINsmAeoNU4uxSep0thOXshtXYc8is+8YtcF2jw3ajzhLcAJlQDT7sjzkfIO0Vmir6lS5mkynzEH0zVRpB5qrIMMTwc0+R3uHJC4CmmSW0wkEkiADtVqJDd2LdJxujOfUCs2VY5G3byCDyBVmSVwQU5BPSqLuqynHU/eqkmJs0IZQkPmDJUdq19Jv0NyqknZ6e9YdtIjQFUBGDzUkL4m8xDgjpUyity1Kx0HiG7kLIAMAVjs2YyucE1JeTPeMGZvujkVXWMyZFJNWKXvMzpInLsyN0qrDNctcABDtB610EMEY6gZpHQKMqoFNSM5J3GRt5hAYHNWowVIUc4qO3BAZmHHrV1BDsAX7x71DYrjJD5se0daiSYxYUjJHanoggfcWzk1SublFm+Q5YnFSlclslkzKzMi7KSK4BR1kPPapWjdLcMTgY71mO6+bt656mqtcTLkVwksD7pCCO1RSJG9uQnamS/ZgERDtz1NRTTJbD5TuHShLXQRVuh5mxoF4Uc07Ti3ms2ePeoI7oGdtgOwip7ZipMgHGelaPYEXhuklBGAAeaeYQpbcwKmrQERiEsiheOgqlNKHdgqkKayQCNAnl4L/eqjcKA4AGNvpVqWZFQIR06Gqf2gRo7lck8VcUwL+lrC6FnfBHrVqQRO4KEADrisSFwAHU1eiAlcBXIz1pSjqBPIrTowg+9WLK1zBMRLGeTwRW8UktxiPA96jMYmGXYEDnmiMrA9Sqto6wLM743dqq36oSNoHStLzIpTslbCrVOaykkJ8v7ueDVReuoivpVvFLIfMIAWtcyRxfIADmsVo2tgRg781oWxbCmRevrTnvcaZlXZH2jqducGr8Cwxx4jH3u9T31miR79mc+lQqqiDPQ9hQ3dB1FSNsFQxOO1QmQplKljlCrvCk46mqN1dRyy4TvQkDLaSDy+ThgeMVJ9tY4Dnd6CkhRBDkDc2OlZwkbz2yuMHjNPluJsutJJMxHRafFcNAhTtUEDMzlD1PNTC2YYzyaTSBDyGmKnb9Kuk3CRqI156dKqxeZFMM/dq1bSs5bB4BpXaDcXy7s8mMZoq8Lg4++KKrnCxz4ugh+XhAMYPeoCfOn3qmABk1FdMARhOO1FvcusbAd+KjltqiGxXnJLKBgHvS27RC4SOXuOTVRpCJP3h6dKsnDIsgIJNXZIaZNcMRuhU5Gcg1FbRmOUur/ALwCiRAAXMgB7CohK8M4Yj5Djn1oS7FdToreaSC3+0zENjgCqs+py+fy37t6c12t/b7CNmBwPWqEihDGH6VzqKcm2W3bYnKSXXmKPuAdRW1a21ulpAfMIcc9ax7SWWYNDCuAeprU8rMaRdJAM5oqLRIcddRlzPGLgyjO5e3rS2l79v3iY7cdBULyKodZEC7RwfWsu3k2ySMThR2ojBNC5mmXJJGt7llLb1PFZ90jCYknOO1FxehiOAMVJGpunGGCkjvWqTjqyG7k1ltFu9z/AAqcYrRtp4Qm7gGQ8is9VHl/ZInBOcmprWJ7aYNcJ8oHB7GplDmRV7bE2qs3no8S5QDAqK2R5VeRwRxUFzPLOsZBwuScCr9qyyWwQdB196zcXGIluR2i+YQSuQnpV+G5JyxOEBxg0y1PkpI8aht3Wo8pNIFyFUc4qN2WnZGhhQ4kQ8+tOlmR1KnBPas+SeSMhAQF7GozIVmCq43Dkmh0rl8ysXynmIfLfawHSqvnvbjYfvdOaiVm3Ftx3E8Vfl8u4jTzMCQdxTjHlM20NVgCctliOlVHhLNhuMnJNPM0UJbB5HSm/bFPJ61ok0Tcj8kHKrnb0qIq0alQ2MVE1zJ52EOMmrZhMkTBmyx71Wq3EZr+W98iSvla0g9s/wC4RcN0zVF7LzJQWbGO9TCGJSCsvzUStoMWeNLdGSNRv9axLlSrEtzWncLIHzuyO9YF1PI07JyBnvWtJEyLL3AjhCp3qu8vzKe5PNOiiyAxNVpW+dh6VskiGavyhQS2c1KJTtzkccVlwkumR2q7bwtNjkjHWs3EabYx2ZpTtXmqbxMshz1rdvLaOFUETbpMfNUVtAhHmydO4NJTS1HymOWYkj0FXLOzEwyxPWtJYLRyTHFzu5zVo7Y+IQoz1olU6IfKkQpYtCu1eR39qgmnaNdhFX4LsJE7HvxWdLbyXbOyHp0qE7vUb8iOCc7wGyRV23uFjnAB6mstg8BCbTuB9KuSWNzBEty6gA/nTaRKub8cqsxLN06VG90WznlTWWs7vCcH5lHaoTPJGd7HC4o9nYu5dUyyylkJXB6ZqvfTvBhQ5BP3sVUOqFkG0ENnHFPZhI4Mg+Vh1quW24rjFtrie4AUkhhnFb8VlawwAXEZMgH3R61Np2lskaSrKuDzg0ahcZnVUAVl7msJTcnZDIlHmR7liCKOCDVeayttjGUrsAySKI7smZixLJ6Duax9UvSt0E2MsbH5gacYybsJtWJ7GzAVpMfu92U9xWhbaoodo2BZgflFVG1KDyFijAGBxVfTl868Z96qfWqcbpuQk+x0dpqRdm8xOh4rWlaCa1KBhvcc47Vi2oggkZ3cN7VNFeQxSsq/xciue2t4mkZWIrvQTFEDvLe+KjjWS2iVFU4z96ttJDJGpY5WpZYoJYACAMc12K7Wpo7MzmT5d3cjrU1vbrENxYE9SaewVyg6KasYSEHBwMVaEedeMvEn22VrC22+VEcPIM5Y+g9q5dr0tpEVjtPyXDzbsjHzKox0/wBn1rQ17T7iPVZXa1MMczkxg8jHuf51ikMjFWGCO1dcUrI5pXuPE7KR/Sl3CQ88mmsmFBqPoaYicLx2pccDIqESMO/508Teq/lSsBNbxRPc7ZUdowMlUYKfzIP8q0pk07aDCkls2MEpFvJ/FpP1AHWs+2IaR3yPu4GTyeRUkhYAfeOOm45/OkBXu9rTkq7MMdWTafyyagIFT3CeU5VgQw6qTyPr7+1RBGbkAn6CmIaBVhJGhikZGKsQACDgg5B/pUPGaXCyTqpcIpIBZug9zS3BHbeD9fubmU2Vy6vgZVzncf6H+f1rt5HKopB5rg9I8JahYXyXLX0MIXvCxYsPTkYxXYO7FByWx3rKdr6G6vbUmnHmneCA2MVnXbFLYxk5p6ykzdenWnXEQltzkgbqixRhvPDA0flfe706S6eUY/hqvfWPlOGDZPaq0U0nn7MexqXEm5qBy0AjwcdcUsWBKm0he3FQmdoTk9AKpSXBDrIG6mos2Vc7FjHGqfMORikMS4Kkkk85rItLtZPLB+Zga2QrBmI5BFY2tow5jPuZLe3wVf5geRVC+v5XRmVyVA6elX7uzSV+VGT0Irn9SFzbz/ZV5Vh2FbU7XC5Np9ySu6RxtbjHpV1Fsd4LDd65rnpIprZVI6VbhRhhjJuJHIq5JboalboaaqqmXysBOoHpTI5Auwk8k0+zgkuIXEa5zxUkenskeJlKsvANZN9wSuPlmAiIj5bvTLS5YEByDTfKIbZn5e5pjokcwEfTHJqIpFvuXXfMmVHFIXJX1qBnZGUKDircTI3VcYq2rEXuNDySR7WBCmiB3jBBGcHjNOFwQ20ruGeKWZ/3ZdgAAO1Z3JbIZHkbKls5qAxbVDk/MvaoDdmQnbwueDU0MgSYbvmA5571RLLcU8swUSL8i9jULQNJ57x7QB/DUN3dPyy/Kp6AVUjluEBfJAb9aIpiGFX2AscEdahuJyVG3kDrTldm3E9O9SeUBb5UHHertYCGFR5YYDk9q0YLWQBXbhBziqSONq7MHHUVdW7Yx7c9e1JtjRclngkByOgqNZYTbYAyfWsvczSODznpT4hMHCbTxU8oXLRRJkZecimSaeXtyEOD71px2SmESbwCRzVaWFkUr5hJPShS7BYzxYyrENvUVagdNm0/KQOtL9sNsVTG5scioZ5kyCiYLdabuxX7FlpjlQzfLTLiQqMpwpqGQxyhFRvnFW5bRo4VLnKkUthEMflPHuPAp5nWN1+Y7fSoCsa2zZbB7CmQNvBAGcetFrhceXiuZGJypzxRNLhlBbhaWZQiBsANiqtviedUbqTVJX1GXDcFgFDZX3qFcu7AHPai5hkgLfLlexFQ2ryAjIGc0ktALZcRQGNkH1rKsVQXLsybhnita4t3nhYow3Y6Vn6WkwlIdQMVUXoxsubdkpdThWFQz2wlcFGPuav3MAlRQq4OetJDcRCbyJIxhe471KlYVjIGY7vAb7oq9HeZcdKhudkd60m3CHpWczETllbGD0q+XmFsbU8jXGdgIx1p1upRN6Nk96ihu1iRSwwCOadbzb5XAHyHsKhjTJ9pPO8UUpt8nODRTuiro5/fv78ZwBSRMdzjHetGOyRVAK5b1pkcKCVlC9KqTTMuUY8PnMBgcLmqkL4k2HgBq3LaPAYuMccVni2RZ5DjOTQthtMr3AEjkqPlzVa5kk+RT91elak0EaFQD17VTvtpQKBgiqjvYLWJ45StvkHnFRyOpRWYkt/Ko4FLRZz8qikjuYhuV0yB3pcqBss2N1LBKBGCVbgmt+SQ/Z1mLDnisu2eBNOeRsAk/LxTTfRtpwjJ+ZelYTi5WNIWS1LeqASW8LRE7z1rHmR/MVFH1rVSTFspkIKleKzzIssmVJytOm+XSwpNFS6UA/dNS28Nw6eepwFFad9bxnT0cAB881VgSRWRJMiI+laRlzInl1J7SIgiYEZ7j1q40jywkOCepBNVUjaC4YxtlewrRhKXFu3mHGBwKynJpl8tjniJmlBVuM9KtC+dcRZx71oLYM6KyDDVkajBNBMsm3gHGKcWpuzJs1qamnXRJeBTkn1NW4lDzBZEw2eawrMStdKUXaepNdLCheTewxxUySTGtUMuZLYBlODnpSpHCQjbOfaoJLVGZiDlqkhm8mPaV5HQmpt2BNkc6mRmI4x2qGHzWlAHanw+a8zsRkGnMs8bjKYyeMVe2hO5BeiTzIyF+XPJpGmRG2EAj1rSuED2mCuWrmJo51kZFzxzVRtITVicyJ9oLbuB0FWI78mUKOo4qjaQq26aUnIHQ02OXF0FUDGatxFc3hEzIWfAzzWdMQhO05OatzXKso2nHFZrMRLk8rjk1nFX3LdiRbgjDAFjVeaJJGMjx8+gqSGZMsAafgLjc3B5rS3YDKlPlLgAjNU1AeUA85NX7+QMu5BkdKpLEwKmIFmzkit47Gb3La2VxGMp909KuW5YSCNxtB6mrNiJ7gE7QMDoasT2uVWTgFetZc+tmaJFeUosoCknPrRKSGXbyDVmNIZ5AjYyR1qRbNY5sEl8dKzckDTKwR1yynjHSoVdnmCg4JPWrSnbv4JOaqeU+0yBScGmJlqa0K4RG96WCN4rgA7sDuKmsbhXQBwOvU1qFwITsQDHespSa0GtdShHBH5cnmDc5bIJ7VZnQzwbZegHy4ps8illBABHWoDNtQDfgA9KizbuVcz5EEcm1STng4qGWANAWMvT+GtJo4HBkyd56isq4RvNOwE11xdyWVYI1in559jW1bvE5C7AQfWsWZTGwBHJHJojuZA6sh+UcY9aqUeZEp2OourgwqoVigxxg1UmuFndBIeR1OayrjUHmYBugFVmuGMqoAcnislSDmOgG0xYjIyDkYrmddu5nuQJcAjjGK0pFk08JNHJuDdRUltYWmos890Tu9KcWoPmYPXQybC1a9gyjHzPQVpjQJ7e2E32gh+pWpbeyNnen7MB5TevapmnYTt5snK9BRKbb02BJCWbSyQkzqdi8VoPCjQj7OckDqaYEViqROv7zls9qtpAqLt3AEelY82ppAmheRbeNDyB1NWPPBjCZzmoQAkWOue1NEDSPuTiumNzRlw8KpznFOzuY9+OlQSS/ZYS0hHTPNchqvi+e2kZbdYST053cfgeK0jBslySOwuAZIHRI0kcLny2I+cZwVPbkEjv1rzvxBa2qalHa2SmGGXDGNlJaM8jHPI+nv6YrofAEd9r+q3V3eXbmJAoZiee52r2HIB/L1ql49kNv49CQxhWgWL5WyckqH5787v1rqSSio9TLn3utDnb7RbzT4xJIm6FuFbHP4jtWYluXZQxALdAetekrNFd2HnXbKdxLMW+UR4PCgex6nufbFMXSLe4i8z7KqjOVZ1wT746j8cVUoNGfOmzgZbDytq7tzMMjHpSiyVbWWZm5XgZPfP/AOs1pTjOsvaRZBDtGMdAB7fr+AqG6sZI7fyAQwODn0IAz/n3FZXKWpiA4qTf5cmY2zg/Kf61O+m3CKzhdyDJJB7VAIGY4HJ9Bk1QNW3NLSZ7C2DT3QV5g37tWBIHHXA98Vp3XiiOW2liSMnfGyDCYGSCM5zXMmPaxDZBHUEYpNvOAOahxTd2S0rj3neQIpxhRhR6Vs6bpyXGmyhxhpB8rH17Ee3/ANesZonhIEiMpIz8wxkVtQ3TwadC5P7s5U8YxwcY/X8q0ik3YG9DqvC2onUNGS3kb9/bHymHfb/CfyyP+A10kiIqbV+9jpXkEmoSJezXFm724dtwCNj866PRPEd1cOI7h5HI7qoJxWUoa3RrGXRnYpCsIJ6u3U0swEkBKNyB0qBLkSxE4P1PWpoyh4B5xWRoc9qDttBYEE8fSqDKYJFk3feroLlUV/3oyGPHFZVzEJrkB1wi+lJvoKysV57hWjXIJOKp20ck5IVSQK15VTylxGMdjU1vttYiAoyRmlF2JG6chgUFxyDzW4JJ5MAEBcZzUVlai4y8uFJHSpXkRJfKU8gVlNpsTeuhYhjVEZmYH0zWdcwh/wB4cbieM1YtWDyvGxyfT0ptzbSSFjGxwvSpUS4uxjS2Ql5lOBngVHFp/l3GdxK1rzQP9lywAx1qtPGfJVo2JU9RT5mvdNNHqzStJEsoFjUDLck0l1eiVNm3kH061SZwIlZ+w6VItxDhQnf1qWkJpXuhTEr4YjBPWoJkO3aoAPY1b2huQ1NeACMkNk9RS2FqUXEoKqa04YAIl3cZ61DGh3qzYrQV1IwQDRJtoaXcoSQkuwUgAdDVC4MikIG3BjzzWvOytGYxw5rLYRwShZX96SZErDGi2p8qZB6mo2kCwj5fmqVrpHYgN8lEqq0HmLg54q15i3M+SQyLjNS5aQoi9OlVXUKDg85p/nCNVA+9VEl1rSOO3k+b5j0qC0eUgxkAjGKYLv8A0hFxkGtGzt8hixCgmp23BFMQLbvwMluarzMY5Nx4wcVp38EcTB1JIFZV7H5se6I896cHd3DYd5uJVeIbqv2cpYySyABsdK52J3hywbvWtZu05DZ/CqlHQlMvQXburgnGDxVd5rhZDI6naOBUsaL9qO4gegqQyNPJ5BUbajS4J2IbazabNwZAzE/dNJJBI7Hjp2q3Ghi3BRgL3pslxsiY5wSKOZjRlgNDN90itGCYvatLM+cdB6VnSzu0eOD70kG+RNu7A71VrgWm/wBIznAAGaiik8mXKD8KbvUHy1JyODikVX3fdOQeOKVhFwySTEs8Z+UcCohbYiNxu2kHpV2KchVDrt470yYpIhRDlT1FJOxSJFMhtwvD7hWW5xGSRhlNaIYxxjGQBWXeCUlpgpCU4gyS3vd8oAJAPWrEwjRw6uc9xmsiwfM+045NaNwcNtVcsapxsxXuixNcs0a7JOPSmW7gXGPvH3qs4KspZcA08iNblCj80rAh2sBHiDL1HXFZkNqZot4Y7s1uXCILR2LBs1nWQdELMB5eelNS93QHuPkTKLGwOauxwGAK6DPHIFOCSZUDBVhxx0qeKX+zkLSJ5qt+lQ3cnnRXN8+T8h/Kin/aEbnyDzz0FFKwucS3IKAjnAqBiiXpLD5KkVhHGMD5x1FR5WR+mDmteW+poidpPtChY+CKZGoy4bAYcVIqrbsSOd36VFEu5nznOelHkA0xqFJbkism7KzyYzgjsK3LmPyExjlqy51RTlV+aiO4MqWshiDDGR0xVqaCNowUQA9TmmWQDTsGXNX72AR2pkBwfanJ+9YRBNDH/Yu1/vg/KBWQNgICMSe4NTTXZEQHbFUo2LXAcd6tKzFJm5alngEbdQeKmjtg10vO31xVI7wg8pstV6xUjLzMQQetTZMtFq5ZFtGGdwB+73qsXNw0SRjAA5q6EjM2U+bNTW1j5dyXK8EdqzaUFqaNESW/loJFwQOuaq/2gokb5QM0ahdNCZETO3tisEykt7mphDm1Zk5WOwsrxARuYYI/KpJooLp933iORXOQbhAspJOTiuk02GNEDu+cisqkVDUIvmMoypaXBZ1+8cCrYvyZkQcKBV64isZ3AZAWFUr2S2R/LCjOOCKFJS6DehakYGDeq4z3pVtGubcOXqG2fbCIywI9DVkymG3ZsgDHSoba0Q9Bhtm8xVD4QDnFU7p7jzMpllXgCo7fVgqsrDcc9auNdI0QCAKW6k1dpLcWglq0otXNxwexqnM3lxeYgDds1dmlHliHr6moUETKsTDC5ojvcGZEhIiIYbd3NZX2hYbgvjPHFdNfac10wEZwBVeTQYChec7QOmO9bqpFLUlxbKsCmW3EobJbtT7mCSG32HG5xnipJ0NqkQj5UcYqK7uAMHneB3qLu+grJGZCXSTYwIY+taPlt5fKHp1qpLeQSSREcuOtbEckdzEEU4bFayel2EbFWG2haL5sDjpVeKOOKUhQMmrMNpcNcsgXtxVSSzcXgAY9fmpJ9Lgyz5zwEBTjd3FaiWks9qSWGDVFQvcbsdT6VZjuG6J90cAVnK/QpMq2NlMLppHJCocVrSNtUmIbmJ6UrSMAoI2hutU43kincqpOOlS25O4xZJRCwSRQrNTYJS8kkQ249ar3EnnuN/3ievpV2DT1jUTRk7gPmzTlZLUWtxILINLnfgDnFaQViDkjAFUBMTIFAwT1qWdWiBUS8kVm03uFylMSblc/xHFNnwshjyPrVOeeaNkdjkKeKht7oyyycZJNbKHUi5v28EbhWBzxzUVyihS5GBUCSPGuM7Tiq1zqBGU7YpRi7lXCaCOaD7w3HpWK0EtvlsfKprUtZ4pGJPJFSSyxOQrgYPSt02hPUyIpoyN8g5Bqe3kT7WjgjJOMGku4VBUEAKT2qN9Pu4lFxFCzRjkGqdmiTpW0oNF98E9c1XaFrQKHYHcelY7andpEvJGe1LPPcbY3Z92TzWCpy6sq6N2aMyW2UDBiOCKyoLOS7RppGIZWwPerA1SXykQbdgHPvSf2wGiAhUZHUAUkpLYeg6O0kVg7SHd9a27VMqpbr3NZUOp+ZCcxAMO9bNg/2iNenSqUW9zaNlsWZVUTI2eBTlcB93IzUhjD4B4xTCAJAW+76VskFzG8SXZgtSWAAPRm7+w964Q2bEmWUYLc4zzjv+P+NdvrVr9p1RGlx9lt1EshIyOvAP1Nc7qLh7hmAO5pCcHnuevrXRDRGMtWZ9ndXukXQutPuGilxj5eM/0//VS3Wq3V/wCIX1W6tvNkYguuQucAAc/QCk4A56Ef5/z7UwsBjPOaq9ibI39I1jTE1NmlVUaXDASqMxPk5wT2IxznOfrXSSXbMQ7lEQfd+cHI9a80mVZeG+nvTB9rtQxhmdUxkhWwCPpV899yXHsXjID4qd1cKDcPhiCR37V0UmhICJJLyZj1YKFUE+3XFcTFdSR3aT7Vd1bdg9Cfwrof+EtLpsmtWXj7yODg/QioFZmhqeniCGdkBK7CM+nHeuV0+RY7pizAAoRk/hXQf8JPaTxOrl03KVKsvqPbNcvbvFHKGmUsoXgA45qbdzR1G7N9AumVrqQqcj1qzpOReFkK+cEPlhuhP/6s1RkffIzcc+nSm7irZUkEHgim0Tua2qOwggjmKtcZJYg5wOnTt/8AWqg1wxtBDuJXdkDPTr/jVfJNJTWgWFAp8bvE4dGKsOhFMpc80xnYeHdXnlkWGV95PA+bmuuZGZQVwvtXm+guov1V1DI3Xk/0r0eJAoDAkLjPNYVVY0g77izoGhVBgkdzVQ2xKnHJ9aWe4eRioHAqJJJdmwHkmuaWpbswaANkEcDsKrNb3G/zFX5Bxya0TBJGiNnLHtUdzveMRrnPtQmKwiX0awDOSw9DVKCRmleQ5254qneM9tMF3YbHSkgld4+4ye1CgTbU6Oy2KJJhycVLBLI5JI2rnpWdbu8aDYcKetWlkZH5ztapZaQ8ktMwL5X0qeztAd5IAHYGse+vfsk3mY356KtVpPEdwqbyVX/Z9qapylqipS6MsXsclvcFGOVY8ULE+FKLup32k6nHHMkeRitKziKx5cYok+UT1KtqWBKstT4UNyeB2qwY1HKLznmmtCpIL96x5ru4EPmB/kAwKgeZo3wh5HrUkji24TBye9SxeTdLnbhx1p3ByM5rkq+9wSe9TzRxXca5TDEcEUy6hM4cBQAvpVa0uC7mINtK8c07dUZPcrPB9ncbzkHpSXcuxFjjzjviid91wVdskHOabdyeY2YxjjH1rRDvoUZWCxhs/MaZEstwzDOSOgptzGwVcnJFP09p0nMioWHfFW9iOogd4phvUgit3TgblSfM2n+7VG+cTCNgu1jwTViKNrWNX3AMaiWqKWhqyWxMDLJjGOKxY4N5YK2BUz6q8gMRbjpmli8qNN2Cw7ipScUF7mRc2ssW44yOxpLCZzLt3bcVtvNFMuGX5TVaWwgeRZbbgjr2q+e6sybdSEsY5ckk571YW4IOUH41FOgVueSB0pqSKEyfl4qdwJJLyR325pHdpEdSeQOKrqwZwQDipJiqkFDyetOwIYY9vJbt0p1uh+zO+/AzwKttBC1oGY/Mao70iiZAcii5RBbTbLvceee9b/2lVAlAXLdq5NXJuBg4Oa0jO20AHNOUbslM15p3kYfKOelEckVsPnXLmoN263V0+8o5qs05VsspaoUSrkrXzb+PuZ6VeYNqdoYogEUjrisy2KyzHcmFNX1udhEUY6dMU2rbB6mR9kNneYfOB3qws4M29h0/Wp9VWSePKLhh1qgqlEOTnirXvK4tieS8SU7WPHaoFyZMAZJPBNVLZg11hvu5xW/c6YEijaJtxbGCDQ/ddiW9CzZ2DSRkTkBT3pX0pEdTuOw9PQ1LELhbQLIAYx6Go3ea7AijO4L0NY63IcmyYCDy2jVlEg+lV3gHQyEjuDVZo5Y7orMrK3971qvcXUq3Kxsfl9cVSQkrvU0PJ9JiB6UUgVioO1ORRTHYrPMm8EDrVabcZQUOM1M0PkuxIOc4xToraScsQMAVd0lc2ZCpkXPzZA71atXzJgcnrmkMAjRoyR9aWFSVCpjPrUOd9hE1zLHnc5OewqkyLLGz9SOtTXa5kVDgkD71TSxx/YGKgBiOTST6gYcLsk7dQDVwu90PKGNqjnNUVmRN2/twK1ZYEh0Tzl4kfn8K3a9640Yc9sXJ54XsKhBgWTg47Y96m+0xLDycOKijSKfdtwWNXa+5O7H2935U/IJXPWtnzZLgDy1+Q+lYMcLlv909K6qxUC1WMKMnvWc/d1RSTRLbSC3blM8U23vpp75w/CAcVaOyMKGIPrmoZLm2klMUKgMRjNczlza2FK5FKIJT5QALsetJ/YMER82YjHpU0elOjK4fnrmmXguHbyUk3cVPM72ix201LELaYtu0SAFvSq93OsEOI1Iz0FV/sJtChdcNnJNW/sqTyIzscdQKbtuxIr2TTSYZlIb3qhd212ZnlAOFNdQslvkRghStOlUMAFUYPWkqtnsO1zjY9SkDGRwfl7U+41gyqNrHB4xWnqcdvEhDRYJ9K5eWARyb1B25rogoz1sJ3RvqGNorRgEn0q5bQTTW+JARisqC9jjCndgAdK1DqyNZfKcHsajVaFIWZ1gYgkk44qEXGZBt5NUb2dmCvnOe9OtAXXzCeBRy6XZnzam7aTNKdz8DpTbudzlOqg8UlqxePYE4Hems4+0Hjgc1DSbLK0+VA4yxOTntU2LScbvLyQMGlkQTTqqnBaoki8oSAHoetPoK5mS6Ixut0IwCc49K0rOzELH58uO1XFm8uABeWYdaSMiBPMYbs9eKHKTVg0FkheGIOD857io0haWYLtAGOtH2zcuHHB6VdgVnkVkT5QvNTstRrUztscUjxlflPUkd6WztyM7fXIq5dWplYFRgZ5qaGVYE8oICwHFF9NBbCCUSKY5U2kCojEzJmPlu9LczM0YO0bj6VnpfSRybWyuOopKL6FXJHtVlYIDhhyavAPDAyPgYHBqsJUceaikFqJ7sllD8rjFNpsTZCb5DkRqC+aZc3fmRsMfNiiO2CTRuCAHbmqtzCw1Aov3ScZ7VairksqXFwZcKozxVezEkdwxKnNadxbwW+QG+eiyuoViKyAbicA1pzaaE8qvqMkZ5oS+cOPWsqV2VmJySeK17oRRrhSQx5rNYM5LJtq6exTsRxGRFIVcHuauWtuZ5AXHy9uaiEo2AAZ/vGpLfcswXew7im2CNk6bDIF3EDBqeVZJEFrbv9R2qhJcmPaGbcO9IbvyW8xSyhu9YuDepWhQvNOvJbnoMD0qRYzbQKs0Qya1YLtJk/dZye5qObc5dWALY4qlJ7MWhzMkrJdELwh7VatHjgulwOG65qePTUaUmXh+oyaoXoWG4UITkVpo9CH3NmSWAS7c4U96u6fdLHwCRg1y8tw0zJuOAK6TRXifG5c5Hen8K1NISua7ahtbyicE96hfUCz+UvPvVO7iZrzeeAo4+lShFlRnVguBTvoaDNSnzGzDDZwvzLkA8/p398e/HLzozPIwILdFIHbpk/wDfOffNdTPZwG3T96rNsLtuI255z169Rkc8DHWqL2yzo8axBkiOW6b2zj0Psef8cDeKsjF7nJs59M5OAP0x/KmOwO7J4xnn04/rVq4sJolDMN6lvmKjp0/+KGPeqJRsITnB7+/t+YpiFRtzYJ68Ee9XoY1mVo8Z2oWwOu0f/W4qkFKlSRwOCe1b2lxm3KSAH93JvcHuoOSPy/PIpAZGoaUbTODkKFLHtzj8+T+n41Hp+mPe3kce1tm3zHwOdo6498c1tzBriKSZmzNcspHHCj5jnj1yv45rQtVjs73MKMyxoPlJAOCAPY9+3T3p3Cxk674Wazie5tTujQHegyehOWX2x6+jc9K5csRxXpNvetE48xPMijLHle/fjrjA6ccY9DWPd6Bp929y8DG3SNtqbfm3En9eSPTHHY8F+4HGdaK3r/RooYx5RIIOCSeO/wDk1jvbSKSNp45PFFwIaKKKYBRRRQB0nhiO2l80TICyEEH2P/6q68X4aXy8ZT3rz/Q7gwaii5+WX5D+PSupWMreKhLbQeawqrUuJtBVdDKrcDjFSBFjiDsOe1VopVCMEGFziluiyxrk8YrBI0Lnnb1Ug1WkhmVywYZbpVU3DJEFYYC0wX371SHzmk4tBcp6lt27pB8w6ms+G8CMI8cE8Vau3kvLj7OsbOzH5QoySfQVPpdjLLfJpj2bLOzhcMuGBPtW0fh2FFXZt2FuDGkj5YGtO4thsP7sgY4JFdrb/C7ULW2jeK+ikbblo3QgA+x5rC1ixu7a5NnOiqyjkA1hOEou7RpFxexx0mlRzzxhnKK7YZ/QV0p+HenatAkFrclWUcODk1jSPE8pjm4UHGRVnSbm50i8aaC8ZkPRSc4o9/ozSDjqmiebwpN4XC28svmRnkN0qNyhTCjmrlzfXesyESM8zAZwO1ZxiaNyGyCPWs5rWzepEo9thVDeXkdajdzjJprysGwvQVCzMiHcDg1CjqTsPuSoClQCT2qoLtRNtwF7GmukskiksQops9kX/eJwAMnPersjOQ6a/VlKKMAd6zpYcMJEyN1TW+yZivGR1q1HaLIuN/IPApp8pN0zONrLOd64+Uc571aNqEtQW64qaWMWjhTyD2FLOphjXfkq/QCqUtRnPPIN7K3TPFXbSeTT1zhSHqG+tGRDMQQAe9VppiURT1xxWjV0QnqalzIJ0yAPWmG4+0QAHhl4NUIZ2LotXVji8zrjPWpaKuVnTBOxc4NXY0mFv5iDcp6itGGyt2QAOBkcmqxf7HIYVJdTU819gsZsk6s4A4Ap8U5jXG48moZLeQSs4U4J6GqrM4lIY9KtJPQVzdiWEx7pOSe5p989pLECqhdoxxWYsplUJuwD0NRSkhhBuye5qeVg2TgeVtXjDdKSW3Ifcr545qO4lBjVe696kHOGJyMc0xWK0Vz85R34HFVbidUlZEORS3MKiQsnc8U2Wz2IsrE81okhFVH2ygk1oICY8hveqUcHmSkgEjtWlIg+yoMAFetOVgsaWmyq6bHI+Yc1pGygMRCsC2OKwbFwGJxkYq212wlQJ92sZRd9C0aDtbpZlQgEg4JFYiXDLPuU8g1oXCB4wyA5btWfLYSwuhYja1EBSNZZxJFknJPWsjUQUfCng1bt94bA5pl3DgqWGaqOjBbEumRxGwdJIxk87qQSyR5UOWRelUZLxreRogcKwp8U6rhh84HUUmnuS9UaVvqMhjwN209Q1XbeOe3ikm3gZ5A9axDfQm7DMNsJGMdKsPeReaoNwTD060OPQz8jVhvMwtcTxh3IwuR0qjOXubcvPCF2ngipVFmzhBK2MZXtToIJAsonU+V/ASamyuIoKke0fvXHHTcaKcySB2HmR9e4oq+ULnR3OlebhyQCTUi2ohhZSuM96VJmaTc7fKvekmuTJE2cc8CuJ870Z2aGNdQAORHz9aqxrN5pAXCqOa1fMieB12fvF4FQpII1bcvOOcGumF3ozNozXZpDt2kmrJANmfNyuB0qGCURzK8gyP4RVyRfOlzIRt649Kq1nYSMKW1H2QzcEZ6U+7upLm1jhHG1cUXWfOZQf3ecgVf0/TUktTNL36Ct5OyuK/Q5K7haGDkbie/pVW1meFiQetdHqix+Z9nRDzxWYmiyvL5anGelXGWmoW1IkumDgk/WuqtLpWhQKRjHNY8mjJb2peZW8zoAKXRVkNyEKsF6EHrUz5ZK5cXZm3eLJcuhiJx3xUkllHFEsjZVx3rShsxECYuvoaWWF52WN9uDXnuor6FtLcRLlPKVC+cjrTVSKN92ctUbWLwsETlBzmo5LyCMgMCT7VFk9ibloSrcPiVfYZodfITavJqh9vjYk4AVelTJcrJ8xPGO9VysRVlR5ZQQ2O5qY37xqIxz71mT3R85jnAJxUu+BAhDFietXy9xXNCVBeQEMwBHSsO9spCQmOPWrV1exxy4Vj9KoTatOzmILgetaU1JbAyOTTxDGHZ8gdqlgured/K6RgUSRSXVniLLMetLp+meVETNHyvU1pdWuxJCXBV1KIOFp0EoER5xg9KuWNsl3csgO1QOtPa1tra5XA3e1K6tYnl1uS208kTDB4IqWc5+VfvN3qKUZQkEAHtVYuV2tuyTUJBdmjBZSs6tvG4VFdK1uXyc561NaybRudyWxVO8bdnc3yk9alJ3G9gjmfYGBFW2kJhVTjHc1l+dGsQQdfWpIZjtdTyKqxKLqvGrqDhqsNfmNMJxWQZVVgM05WaRztBNJwTDmNaO+ZiqDkmrLOqnkfN3Nc8ssyvnkFTVhriZ4mbd0o5NQ5maauocsuGA4xTJreKV/M2gMRzmq8MckESS5yG5NXIY1vchmKY7ipasUik0vIVei9RVC9uMrgdFNat3YmByY+eK5+7favXkmtaaTBo0Ibr/AEbaeW6ioEuJwdxTduPBrLtrl9z46DitW2uAkSiU4IOVzTcbEXK11MfOkMxwdvAqlZXO+Py24YNwalv7hbi97Hjms+FlWc46ZrSMdBN6nQS2M91CSo3YGc1UWwuEPzIwXHHNbFnc/wDEtcqcECqa3szpsJyc9KiLexppYbbbUHlyINrdfar0aRxy79nygcE1EIcqJCMeoolmUgjPCCjcNCndOkl3uJwM9KuMybVyAVAzWPO4dsk4+lCXJA8sk/jV20JuaDXab0VMLk8Yq/IAiCRpAa51ruCP5iCWpYtQ2nLMWQ9jQ43EmX9akVYRKjnzDwMGufWV5eXOT61t/YX1JA6N16DNXk8PlI0UoMjrS54xWo2mzAt4wxG/NdfYC3jhjxwT1qhNaW1pIU+8zDp71bt9PuY1ErkbQMgGs5zUioqxZ1TLQHy8cjk1k218kSOrjJAOa0FuhcKUZMY6k1nalZww6TPc4+YAgH1zWtNaWZrd7lG21Np/llA2knPY47/TqcccDNdHZSwiya4kWMJwzb2xlj0wuc+hz2OR0HHB2sgEi5bIJwc9P/1V2ztE+l2q+csgnYLIqsCcAYOcHtuP4jqcc9hiUI4o7lEimdtzKWdQM5YLn17/AC/Tms+50tNzBQFDMVHPfJ/Xr+v1qVHLsryYwzbc5BGCRnA9ACvSryJJJcxxwECUyg8ScAZwPyYkCpGY0mnvAsTTKFUqpIYZAynf368e3rV6Jx9nkQqI3lTdG/A2jLYZs9uB+XpWvdLFqkTNkRJ5u0EEgrGPlz7gEfXrWVeLNkJDkTn5uFABGORjr6fXn8QDHt7soY2kUo8BJ2kcZDZ6dzzjHoB6VrxykxZll3xzEhipOXHBbnjGcMfx9q50qZdQiiKt12Hvx69PQ/kBXUR20kFqryAjdEcj06AfoBgfngUNCGyPJKEc5zOf3iYAAJHQDr0BBHrSxeSkKqXBLgAjHGSD19uB+IB4rOurwSTRuxCxgtgJ3GQe/wBf0HWqsN2zXMbSNtVcg4P04H6EfQcUhmksJuk80plCowTnLHBzx+Bz6AfSo5dJd7XK7SwJZ2GTt57Yzn+PIHcA8jFaemE/updzsfmO/bkJuOD37/MvfgnucjTjso3SIOUx5uPLDDAQDkEehdiO3Le9AHnGp6dLYy4dHA6ZYdTznGOo4rPr02802G8t5Irn5FbZGrZyx2kDcM+uOSPQ8V51eWslncNDIOV7iqEQUuKSnrVJCZJbgmaPb97cMfWvRXkFzGHRQpAG76153CpNzEo7sMfnXfWa7nyM7QeawrF0x8EMgUtg7B2pXkZ9quCAK047drhQsfyoOoo1GzMUccicr0PFY3VzZmQ4Eg2A5qs8KRFWGM1pQ7ZXO2Pjpk1bbSUv4QUxlTzVaCsdr8N9OsrPQbrXJo0kvHZkjyOUUcYHpmrfhzTLbUvHYv5R+9jBY4GBnsKd4b0xbS3s7J3zG4ywB716Ha2unWCDyFiRu5GMmu6PLGnaxz6uWppzTLDHkniuT8TaRb6pdW7su2Rep6bh6GtW8m3pv8wEIc4zWVrPiTQo7MNdTHzQPl2gkg/hWNRLltc0pp817HlPi3TbXTrxIY4vLdycnPBrmZ5JrSEs6uFxwSOK9HuNb0HV57c3UGZ42+QsOa5z4ja7ZxaX9kt44y0nAx2rhhPXlsdbirc1zmND8UPbXDkglWG3itaS+N1+/wAEA9q8+t3ZHVF6n0rr4JdkSAtlQMmnUpRUufqZe0k42ZeJdV3k8N0pkl8uzy2xVdb1GYh+AOlY9/O6SkqDg9KhQuyb9TXNxuBYNlRxioZLxlJVmOCOKybSZpQVJI55q2TG0qoThQOtNwsS31I4JgkxZQc5rZSRItsrMN5HSsyWNY3GwZHaoZZiFViec0mrkGszxyShnO4+1PjIkBPXB+UHtWbazAAuWxirFo5aV9zDnvSa0Bu+xdvIkubNg+MjoK5DUgIZ02HPHSunuZQgCK2cjpWLHp8dzct57FWJ4FXS03JepQtJNp3n71aMMZkb5jjJ6017YWcrcBsdKZ9pCRgtwT2q3rqgTsaqkLn5ztQVJDNDHbu7gFz0NZEd15kG0DgnmnSyAR4B4FZ8pSZekuFuIiCAPcVgTqUnKkk1djnBTA9elV7hhudsdquKs7A9hm87AvcUpSVHWaQkCqiSkEmrDTS3EPz8DtV2ZOhOCsyuAck9KmYk2nk/x1m27tGxK1fZyib/AOOpaswRWZnij8txyKdcXhltVjIHFVrmZ5JNzd6aqPK6oO9Ul1YF6xDbCeM0/erblc4PapIrNIQAZMnHQGs67JSXgEVK1ehWxZR2hkG08VfhXFsxYfOx4NZUUo2qW5wauT6gBt2jpQ1cC5BdOqFG5INNuXaeINu5U1QileaNyvDE5zViyLZKuc1PLbUQttdPHMMdTxzWnaRi81GKGYkK5xkVky2kv2tccKxzkdq2dOxDdQs+SEbk09LoNkaOveGbS0sZ5Vjbcq5EmTXnUU8wztBwe9e066Vl8KTyN90p3FeQ2drPGBuA2sODXRNJIyhdlz92tgDIMselR2MJuWdlHyJ1B71IbSXyNuN3piprRo7ZsBWGBhgeKwHJajpLhfJDRfeU9BV6e6vb+GGGLcqkc+9c9c3SefIsXAz0Fb+m3L5i39QByO1ElZXEJ/ZF4efLP/fRorUad9x/e9/Wio55diSUXYKFQw47VnS3oLZYldpxgVg22rIAFZsE96bcaosZ2r8/rVqnZnQ53N9bpD0JBJ60yVJY7kqHJ3CsuyuIbkKQ3zg/drbkliF6iv3UYpbSsNaxbIIVfz9rJk9q0WIihJYjdWZ9q8qSTn5icAH0qa0MlwzSOMxoM0JNyEjPnXfJwOScYrbtpmt4grr8mOKxpJQZw+QozUy6g858pRuNXU12JiJfOxlaWOIEj1qppsN5cXpuc7FHY1sRxuYWLgHjj2qK4/0exLxyfvPQUk9LFpFh7yFpER8MRVuGK1SQ3IUBj1rjYrnOWckPnvXaeCfL1bWDaSpuwmRkds1E6TeiBuyFl1GNZAqnFZ811KJQ4bCk4rV8daedNvk2IFix1HrXMNcJKiDJwtYOjyuwJ3RtTXxSEKTkGsp7sMXkZPlWonvo1g+Ugt71iS3kjO4dsKewqqdIGzSiuIJpTvcqPTNTJdp5m1XJArmzPtbj1q0lymOBj3rd0xGjLKC7Z6VWa4aNfaqu5mfcGz6U2QvswTxmhQJbsWXkMxBGSw6VsWVisoEk4IbtWFbTm3kVzHkV0kfiGBYh5iqO1TPmWkRrzLaSRWzIkaAZNXblPM2RwY5HzVzFxqaSTB4xkZ6VrWmopFbb2BDk81nyNK4OQ24im0/c44U+lRxNu2k4LGm6tqCyxbVfIxmseynMtzvMmOOBVxi2rsiT7G5JJEqFHHzetUg6kn1B71Vmuj5oXOTmkbdI/BxTURcxom7+XA4NQXcxKKCaorc5ZkPYVEl3uBy3TpTUAuaMRQw7m+9mpg4XLCsyFjKDz0py3RXjg4ocQRbkO6RSD17Vp2Qy7EEDArHgY+csnB5rRSOdDJOpAQjkGk0jSELli4nQWsnGWz1FQJEzWMku4BfSqck5W2f1JqNbwmDYMgHqKEiJO5q218DEEbJ9BT5b1ICQGwevFYSzMJBg4xVa8vzv2Hg+tHs7slM6O41NpICVfnGMViTHeBk5bqagSaPy0y/J7U25YowK9CKqMOUbbIoZfLkbPc1auLkSw5Bwyjis12xz609HLR1py9SQEjKxPciiMnOadtaQEqOntTlXMYHfvTuVys14J2WxKjqT0qrdXBhcNGcEDkVPEpNiWjAZu4zWOWZndJM5HrURWpXkbdrqkbACVs7uoqdrd5NxUnYeefSuftljFyu45UVtXWoCOIbGycYwKbjroO5HPAohLRtuK9qotNFj5/vVDHeSRMzHox5FVJWDSFlzgmqURXJZcFs9qWLa2VyRmoByeTVmIJsIYfjVMhbnTWF1DBEiqcbRzzWnPqOI1nicH1WuEklKy/KzEVJHevGynJI9K53RvqUpWOlWY29z9plXcGOQD2q/Hqhv3EJO1D6VgjU2vl8r5VUDvRaXCx3mY+SDzUunpruNS1Oxlsbcohj57cUt5py3Wl3FqP4kwPapbCRXtkaTCr1JqW9k8sBlPykdRTi3Hc6VG6PI2RoLhonG1kYhgexrrtH1OIWptjneSCrEcqAc545B64A7muTu5zcXs1wesrs/5nNSQSvHgLnGemetdqehzHTOsRnL8pFtwwyG3rhjx68ccdOM85qwgEzFSiB2RQxXjLYI298/ezzk4jPc1zkd8GnDSOmc/OQM7uc9O4zjqQTWpDfniRUSOOHc6pnJ68ISMAscleBwOmMCgo3XVdiR+ZhDGcoWx/E2Tjt9e/pSxQFIWLtltyyEktxtBG3Hrkk/kPUVm207RlTJJubI5OdzADGB9RuzzxgjqObVpqaIGDoJPLBYPydxz1OeSOGX8RnqcAGJqskFheyLHGRLlhlhySf4vY9fzA7VXl1OSdCgO1CScKcDGOn0xxVDVZXl1GR5DmQsc8+9RxI0vyxhiSOoGcc0gHTOQmQfc1BFvkkwT1Oat/YpZoGcAgLjAP8AEDgAj15yPb88T2NlIA021gq/8tOmD6/Udvw6cZAOo0NY/sSAFjiVT8pIBJ6j8MEk+i/gbzzLBbAGQNJJHh3Cnb0B69O+e/07VmxQ28ChgzK0QAJJweQBgDHoMH0BPWkn1GNofKEoKhSTnGGbJ5XHGMZI9AD68gCSTK0gkcuUDlB83QggjBHbAGeeoPpzzuowrKW6si/KjEdskZwPpk4q21wizuG3GFztdehIyQGz69/Tk/QwSxzxt5JAcEEYx7ZBHXH+H6sDBkgaI+opmcVumI3dhIm1TKoGMDDduSO/px7E44zhSIySMrDDKcEVSZLRa035tStwe8grvObfbHGOW7155bSGG4jlHVHVvyOa9Z0rTGvpsQAyTMfkSuaurs0pk2mRtDOA8mVYflXQiziazMcuCByD61teEPD1ta3EkmrxJ57f6sN0ArvvsGnyQ4EMZTHYVgoNm9rHgF7ZfZ7oKBhH5GKsWPnWcnyrlCeRXXeMdFtW1CIae6KUBZ0Brmo3X7QigZ5xSbaWguVpm5pr3I1awnD4gJKlK6K98IQ3+pNqUeo3Vu0ihSiP8px3wc81laVAtxrNrbxnKwqXb2NegR6aJUV2bao7Cu3Dvmp+8Z1/dn7pyV7owtrSQLrF3GwT75fIP1zXkeof2hJc5ubzftPC9Afwr6B1aG0msGtVkihRjtLsRyf8a8q8U+H0069Q7klUjIYccUV6ceTmJhUlexxmoSm3sWnDkSKMgjsa48S3eqXXmTM0hX1ro/EDqbZljbv0qDSLXda/u4zk9TisaFNuN0huaTs2UVeVJljFrtHdsVNcXkiPsU8d66FIMQMG2g479q5e7g3Ssd2cHtVzpyXxITknsyVb3aAGOTUV9etNsHYVEYTkcU82q7ct+FQoq4tRLSd45flwQaszzk42nnvWd5bQHO+kWYqvrzTcdbhc1luTGhJbPHFV5JTMQN3FUZJy0R9z1qRIfKgWVWzn3qeQNy6jbH25ODTZ7p4ZgY2wKrpI8sm1uoHWq0r7pQgOSOtPkFtsacF3K04c/Mc1oXcq4EuQGFYMbtEWA64p7XDPCN7cipcNQ2J7q7MjK+e9Q3DCQqV9Krl9yYoUsGHcVSjYVkWbWUghAMYPNXVlR2dcdqzDMItzDqaSKUryW69aTiCLMbAO4zgjpTrgpLajYct3qtIDGdxPBpiX6xJIhXO7oaOV7jXYhBIyewqwH3rgHgDoKq7w0XH41LDIq8HpVNC5S9axHytxXjNW73agUDGGFV4rmOO3b5hz2qjf3yylNuRtFRytyK8hJiGYKO3FSgPGy7VwwqlBNm4DMPpV+4nPlh1+9irceg0uo5nczK5b8KmvYw0kZPQ1nRXDAHcOSeKvzLIUTnPFS1ZgymwIdwvIFSRqiwF3OSO1NXCeYrD5z0qu7Hpn600iGX4bxcOEG2ptKnD3mW+7WOsgVsYq7A3lyjHANEkJHQ3vlhwY35xVVb4CZVXBINRIrStyflI61WeEQSAgknPWski9z0a/u1l8JEnGCvNeYy3RMnlI2VHSuguGuZdPhiEpEbdVzWVJpsIbcJNrDtW05J2JjBpFdb6ZHARWAA7ip21Az2zQmMbj1YVRlmdSwGDjjioFuGQkgdaFFESYnlhZVJfAJ61oW9wYLnZ5haL1FZx3PGelRKj5wCc96pq61JR1fnWh585/++qK5Iu4ON5496Kz9n5iIgpHFLyACOtBzmnCJ2AIVj9K2N7GhpcaxyiSQkHjpXRXYja9iIfaSowTWDYW1wo8x1O08c1s6nGBJAXyGCjGaxnui4p8ruGoWpk+a2O5wOa1tFieLQbhW5k71UaCb+0o4YQTujycCt63Q22iXGVw4yCDVJW0FbqcPfHJCg4OaS1ujDLjgHHJFU9VnHnEd+2KpwTsvXkn1qmrmfU6+3u2mjES9T3rSiso0gPmjJIrltKvI4ZQZFJ/CuuuBf21pHc3NpJFbyf6t2UgNWMlY6ILQw7nTI3uFI7HOK6fwVLBpuutcy/LHs25/Gueu7sBSY8Bz1zVjRFluJ1/eFSW596nmkncnl5nY6vx5eWurQosbnIbNcT/AGSY4M7+tdF4jtZvLUxjKrjOKxnndbXDZAx3pSk2wcOVHN30ZifA6Cs12JHNal4XcE9RmsqRxnBraCMmRE5p8eeR+dRE05ZCAQO9aiuKXIbgnFSeeBgY4qv3oKtRZAa6ajC0DROmCRwayfMJJBbNNIyOetMIxQopDcrllJ3TGDW20/l6YuTl2rng3y4qQzOyhSxwOgpSjcll+WVmh68471Wsrjy7lT2pjS5jKk1Gic5Gc0JKwJXNG4mzN5i/dBpFumcM4bGO1UwWwFbpT0QOxUGlZDUCdLlEgcnmRqScqgQjjIqnIpRiM5qRyZQoY8AcU7By6kkdy6ZAyfpTBOd3OcU1dy8IuT7Uwo+c7TRZD5TQjujGUIPer9xfyMUzJ+764rFgV5229MetLI5Ryh5AqHBNmidkbTXQls5GUfjVRbhXUAjBFVFuitsYAMAnrUiyxxuF4II5o5TK2tidJ/3u4jKiq0+Lq5IQgD3qzdT262/7vG6s1Rlg2cZ60RXUrlSdh7kLKFB6d6neYeUoPWoHhVQSDSDnAbpVbg4aiz/vMMp/Cp7Eb8I3TNQqgwcDNT20xglxtyaHtYpJItMfscrxEZVu+KrO/lMSOQaL25eRgAKpSSO55pKINpbF22vDC5YscVBO4dmkUn5jVcBvQ04kkYI6VVidGWFdYgCvU9c1JHMHDBh+NUlyx5zU8Y5OTjIosToRySDJA6UwNkYqRrZgeoOaYqEE8U9BNWAdetWI33e1V9vNWbUKGJbpiixK3JmjBQHvTGjBHI6VI77iABxTd/GMdKdiyqpKvwcVt6BGjXn7wZBrFkG1tw71s6KiyzqN2B61nUWg4K7Oud9oECdDUd7dyLpk6KcMkTEH04rOczQagqLvdD1ajU7rEMsOzbujYZ9eKyWtjZu2xxLYDLjjCimDJbAUs3t2pZTh/fFTWQDzICpK55G/aT9K61sYE9lptzdODEgx3LYwPw+nr/8Ar0ZLu3tIxFFJmQKFLJg8+uPUdvqTyck09WuZIWNorSxIoAKb/Tseev8AL0FYm4g5BNAzq47uFiYz8oU7uDnt69zgtk9/yNaIkjhs5TGQzyIDkNwynOe3UHIx7+vB4UTyA53Vbh1F1JUsQrDB7igY+7fdcFgOtbujQIscxZARuHQ8Y64P/fJ/M1zspBkViSMtkkfzrqbS6svtbC0ikjhhiAEbvvEw3dcg/e5Leg2t0xSAseX+6Id0jQcEYyZOuB37MT7e/AM8LGO3luJdqKGCqcAHjPcjjp7nuTjik88JOqbsMDuyMjAyACCf4u+exOOO/M63q/nbLW3bMUQwGxg/QY9sD8KYF/U/EEYjMUYzkbcAnp3J56kD/wDWea59tRkMxfuTn8apnJOScnvRjiiwi6L89CDjp+H+RV+11OM9dyOQRuB/X/H2OKxAvNWIQAw3DI+tAzpbcx3BTy5kiOOUwfmGeRnHp6nHT3xzmoKUvJFbJOepzz+ddDY24SxmlEhYMvKwsCwH94ruHAwRyCOenORzl3kyZJyfUrtJH0HFNMTIVHy/jX094BtIrf7VduoEoUKgPYYz/n6V8xJ2+tfSnwpvRq2jC5kcNIV2sM9xwaymryRUHozhvG/i3Ujcz2+8xskx2+WSCOa7GPxdJ4U0iztLuVrmWaLczFskH/8AXXJ+OtNjOu3sygcMT09q8yvdZu7p9s0zOE+VcnoBWfsnfU2VZWO90u4uL7X7i9FzJJ5pJdd2cAngV08UcEbpKByDkg15r4O1RrfUT1KsuGFd/FdxyOFZSQ3Ws6is7DU9U2a0HiS18MXVxqNwmQ64CjvXMa78bdSuQ1vpkZtkbjJOT+eKg8duraGVVMFCDn2ryxFLSjA5z3rqoS9yxhWtKdzrb7VdV1dInvNRmmCHKIW4U+oHr71vw6lqN3ao15cGQqu3cTya4+KTzI8D5SOorp9HgIsEkd8qexqcQ24kxWuhUmtxOkjjO0HnNXLS8S3RQq5XHQCk1CSQLshi4PXFUbAsshRuDnvW2BlZNGdePUdqGrAs0YjZSw61kO6pgZrZ1aFTbZYYYHg1iSQq/fkVeJvzWYUbWJY5FXJLZOOKid2l4B6VAVIGAeaZ5rqMkVy2NrrqWmiBAOcmongZRtx1oil+TceK0oHQxMGxkjjNFxXTMuRF+zncMYpqzEwrEelTyO6kxuowTVSQHdkcGmK6JPmjbeDQixp+8HLGkU4gbd1FQo7bwf4aAuLJIzPnOKbuJOM1FPuYkr0zTY9xHQ0WFLU0LMB5GD9AKjWURuxPQ9KSJJQDgHnjpUMkUyvho3+pWlYfQV2Jk6/LUiKCR6VA6snUc1JBcbQFK5OadhxsT3rsWVV4XFUSpB55qzI7O3I5pI7d532rge5pLQqxChwcHvT2cCtaPw9cCMyNImAM8VjzKUYr6GhNPYUk47iAs59qZIcHnrSCQgYBpwG9TnrVEpmhZeSwCCPc5HWrd9EohUAgMR0FZsEwgUbPvdzV6zljy7yfMx9aza1uaraxn8DG7ORVq3vN8y7zgDjFQXOJZMrgc9KZ5QKk9/WqtcnYuXQDTmQH5R0NUlyQ7A5zSG4dofKY8CkRlVcL+NCVhXTGFinBHNTRy5wSeaSeRSmByahRSxApkNWeh0lnMHh8rHJrVitI54QGHTrWRpkTll2r0Fa9w7RKoHyk9a5Zb6Gi2FvilvZJjJK9MVzdz5rSBmJw/pXQTQs8O+WTjHFZv2V5nAjwwArVOyEzIlAT5V61VIfOQOKvz28kbO7DGOMVSacBCuK0iYMdGrFgV5xT7qGaBxI5CluwqqkzL901JPJLclWkb7owKYER2560U3Z70UwPRYPDOl2yZlYOwHOasR22k2yHy1QE84Ncm82puQ7Fqme6kYDPUDmuVxb6neqkVsjR1C9gaMJEoUA9QKh1JBL5Tu/G3issyDBBHB61e1EbltgCcECrStYhvmuzq7WeO0vrR4wGMkeCfemahd/LOrkbieRTJ5ooLy3EMJCxx7jxWZq86yoJlPzSHkVqo9TNzdkmctc2n2i4dg3U8UselOozuGas+UQxGOavWUIkByuPrScgXKa/gjwzFqev2cc2HVX3uvqBXs3jxLV/DgtJFUMCuwe4rzTwXexaDqUmoSozRqmDtGcVsXmujxTfNcISIE4QGuWo25XN4Lqc3L4dtZxvORx2q/o2kQWIAzuYngmtNbLbwHyPSnfZ/Lw27pzUuVzSyIL+W3jDLLIowOhNcHrWoKzNEmNp6YFdLcWtrc6hJcTsTg4xniq2oro4ThBv9hVRaTuTJXVjzud5sYJIFUiCc5ro75VLsI0G2s17cvztArqizllAzaSrJtm7CpoLMsDuq7ohRZRwaXBxk9KtyWjhjtHAqMRsOCDRcqxWIoYcVY8pjxik8ls9KdxcpXGQKKstFgcjmm+X6Ci4uUjQZPIqxvAAwMVIluXHyjn0pfscm09BUtotKxA5xyOaSKby3LAVZFk2wkuOKgKbcjg0aBqROQ5J9aVU5GTxR+FHzY6GmK5dtHEe4Ej6miWddoVSCc81SwxPQ0/yyDnBx9KViuYewUHcr7T7VCSeSTmnMrE8DinR200oOxSQKZNyIEscdqmaI7MgjNRtDInVSKequV5zQK+uoiRup2kDnuaV4mjbB5PtS4ZiMnp0qWJZEcSFcgetK47oqu7EjIPFGcpVyQNK2RFjPtSC0ds/Ifpii6DcrpIyjg0gdg2ehqf7DMDgRvn2BqY6bcFd3lP9NpovEh3KjMWGSaYr4PNX10u7Zflt5SP92pV0mbYN1vL/AN8GjmihWbK0Do7hSvHrU7WRdnKLwO9bOk6ZEziOeCQZPUqa6xNBsGUKgYCuedeMWXGD7nmUUfO3bzUyWmTnHANdve+FxuP2cVTHhq6jiPyHcapYiD6jcVc5i+twoXy1PTkiqcStnmusTQNSRioh3D3qKbwxqJIKW+D1NX7eHchvscyY8McipoVQSdMjFbDeFtWkc/uCPxp8HhXUFlw8L/XNU61NLcFoYrjAU479ajfg11jeDruVNgXA96mh8AyOB5spB9qh4mmuoXOOKBl+tWLCRbacM2dtdzb+BrdFxK7MaePA1tvyXOPSs5Yum9BD9NuLa9t8KAGx3qW40m2mtZN+1nEbbSe3FWrLQYdPYmME59TV97BZcDaOeCPWuP2iUvdLTsjwqY/MPoKtWDlLiPleo5OOPfBFV7mNo5GjbqhKn8DipLIE3CbWIbPGFLHPtXrrYSDUCpuXHHBxx0FUSMVZuVIlYFtxzycYzVY00MSilpKYEqtuQrx7Vp6dO4aA52Bd4ZhwApGDx9Cf096yFJBBHBFbEDLJpDPhVKOFwM9P8SeePbFJgXr3VJGspThluLj5pZCSCcknAz+I4x3x783x2qxdPKx/ecZxjk8YGAPyqsOepoAcMUvFNHFLkk5PNIB3oM1PFt3DjNVwfepof9YvzEHPWgaOklMMGlkqy5YjjIAU44bHY+/tyeMVzc/Qggg+h9a3NRuv9FWD5w4GWG459cYPbnPQda55yc8UxMRfuivSvhhr0+mC7jjchQQ+O3Ix/SvNM10ngu6jg1+OGU/u7lTF+J5X9QB+NRPSLYQdnqdlq+qTX+oTeZgiTca8wliaS+kRRzuNekS2jS6mIkBJ5rlotKca4QUJ+c1mqvc2lFdB3hO1db5iyk47V26o8Ts4Pyk8+1XdL0K1it1kETB2HJxWmunwKuPLYiuOdfmexHN0Oav4Vu0KSfOh7GuL8QafFYXSeUm0MO1esm0tyeYzkVx3ju1i8q2ZI9rZ60Uqr5rCTuzDhsYLuwt2hwJ8/MPWuqtrS1hgQPlWjHTFZ+kWluk1tNswGjzj3rrxHHJED5QIIpzrNOyNqmkVI5O91S3gykKbyevFZCXUEmolJP3YcZDHjBrvvsFvnP2dc/Sud8WaRAbA3cabXi5471vhsVaoro5ajckZOoqTasqzCYjpiuf/AHpPMbD8K6zR0triBWbG8j8K1pPD7XABBUA9MCuzHT5LS7mVNvVI4NocpnHNQmFgNpXIrvV8JnJ3sTUw8JQEDO4muD6wja1+hwUGniaLduC4PTNSS2jQx53g+mK7+PwtbquAh/Onf8IlaN95Caj2+ocp5eyzscnkU0W5I3se/SvVE8JWK8mPIqT/AIRbTl6wD8qr60uwch5S0O/KpnpSxwPEoyM16t/wjtgowIF/KnroFirbhAMY9KPrXkVyHlsYcfLsHzetW3sLgQebmLA9Aa9An0KzklVhEAB2xSto8QUqsK49MUPELsNaHnRimEQw68c8Kf8AGoSLm5fGM7fRa9BTRZAxCxRgfSqw0C8ilZ4mT5j0xQq6C5wz6blNzkq3uKryW8UaDAO4d67+bw5f3DbmZfwqu/gy4m4klA+gqlXXULnBjjoOfem+cw6cGu6/4V+d/wA07EU1vAIDY89sVft4dwd+5xZvbkLtErY+tRLbyTqSEY/SvQY/A0ar/rM49QKcnhGZCdk4APtS9vDoPTqzzc2bhskY9jSxwM7Ywa9G/wCELjLAyTMT3q2vhCyix8mTQ8REEkeZeRtODyKdlgQqqSPpXqsXhLSwAxj59Kt/8I/pijAhX8qX1mJSS7nj5hI+YqVJ9akO3aqgdK9Xk8N6bIf9Uv5U3/hHdOTpEtJ4mPYHZHkDW8jucLgGpItPuXYBYmb6CvYU0OxwD5CZFWYtPtoyMQqPwo+srsJRj1Z4+mgahIwVbdsmtix8IXzyDzECjua9RW3hQ52gfhUuIVxk1LxDZajTOUi8PPDbiNMbh1NUNR8NX7qHiYHbyQe9d3mPqpphMRPL1kp2dymoHEW2mma1ZLtSCnWsm5a30/c9o2XHbNektFbyqwGOaxLzw3Dcg7AEz1xjmrVVX1Iko2sjyu+vpbuVnwVB7VnnI616U/ggdPNGDUQ8BRNnfNXQq8Ec/IedABTmnsfQ5zXoqeAbXJLTnFWY/A2nKCGbNDxEA5LnmHFFen/8ITpXpRS+sRDkEbRJniC7wDUJ8OPtA3Ln1roxbnplqRrZhggsa4ueSF7xzv8Awi4JBLfWrS6NDO6q3SLgVtiKT0NQwRn7TJnIp88n1Ki3Z3GX+LTcWjDF48A+lclb232zUEhPQnNdR4guxIixxYyBg1i6BFnVSGJ3Ktd7fLTbFLVGkPDtuOuadF4egVs5OPStwREgZzR5WPWvO5pdxKNjKuLRbfTbmOI43KeazPCEbfYpc54fH611ceIgR5ec+pqOOySMsYowgY5IFae0fJys0Ta1QCUIeal82MqQRkH0phg45GaVIyvG2s7j9pIqXGn203QFc9apnw9Yl9zZJ962xGxPCihomBHApqTH7SRgN4b09mJYEmlHhnTt6ny847VuMm09BQqNuzwKOeXcXOzIbQNMJ/49lH4U0+HdNIx5AGfatzyxjqM0qxKR94Uc0u4cz7nOyeGrEgqoxmqy+ELMDJYk11RhTP3qcsKY+9Rzy7hdnKx+EbAOWky3tV6HQtMhAUWyn3IreEEXdqUJCOtDnJ7sabXU52Xw3pkjbjbj8qiHhjS88Q105EGOTxUamLccL+NClJ6XE79znl8L6Yp/1WDQPCenMcmMn2rooxGy7nGDUyvDjilzS7i5fM5Y+ENLP/LKk/4QvTGYHya6v9xj5gaA8GcDOKOeXcOVHLHwXpuRtgFTp4UsE4+zLiuidolHGc0imLuxFHPJ9Q5InPf8Ilp27/j3XJqRfC1h0Nstb48jP3zQTFxhyaOZ9x8qMEeGNNU/8eqGpk0GyCkR2qD8K2dkB58ykDRqcBsUXfcLJGK3hzT8fNaoT9KaPDdgBj7On5Vunyz/AB0whFOAc0XfcOVGND4csATi2jFWV0GxC4MEf5Voq0a8mpPMXrt4ou+4KKRmf2HZKBiGP8ql/se0GCIox+FXzMrdE6UhYEfdpfMqyKP9m20fIRD+FKLOAHiJM/SrigBTkU5Y4zyS1FhcpUW1QDAjUfhUy20IXmJCfpUnkIx4Z6cLQAglmxTCxCIYB1gTP0oEMQJzGB9KsCzDHKkmkazYngtS0DlIGiQdBSHyxjK1N9hIOC5pDaYHJzSsLlIgy54UUxiWPGBUptkDc5qVbSEAlmpaC5SuqvjkrSZYHBAq4lpAx4bIpfsluzH5jRYfIVTnHQUzafQfnV5rOAHqSPrSraRe/wCdOyDkKQidhkKPzpxglY8YH41eESKQpOPxpzRw5zv6e9FkP2aKH2V+5FUtRmj06ISTPyfuqO9brJAqFieMeteV65qz3OpyM7/IrFU9AAa6cJh1VnrsiZxjFHA6kCuo3St1Ez/zNRRMQ2QwHuatawo/tKZh0kw4P4f45qlHw+CDg+hxmvRas7EDZGLMSTz6jvUdOkOWJplMoKKKKACr9jcxwK+6MvJ/AGxsBPGWBHzY6jkDOPpVCpoZdqspG4dcH1/z+NAEbszsWY5J5JpKVm3MSAAD2HagYNABR3oxijpSAcDirVoV81d/IzyAKqDmrlmA0y5UYz37+1JjLGpCSKYqV2qQMdP6cflWYePpV7UXd53Mh3HoGJ5xWeaZIoqSKR45FkRsOp3AjsRUfaheKGB7pZ2xmisdehwbW8tw+R/C38Q/Agj8KyNJhEniDewHLMcn61a+FOonUPCWoaVcNuFjMJIwTyqSZyB7bgT/AMCq/pFrENdnyPlTOK4KmjkjqWqubnAGBIAPam7hnHmfpVryIFPIpyrb5AIriuTylP5GzyB+Fc34usVubFCMsynjArsituuflFaWhafFfXpPlKyqO4rSinKaSC3LqeVG1MOi2DoCXUbSAK6iyjX7FGGyGxzkV2un6NEuvzwmFcRndjHTNUdcso7TUmVlA3DKgVtVpOMebzDm5lymCwXaADWR4kjV/D14B/cNb7JHnBzWL4oD/wBiyQWcDzTzfIqKMnNZUruasTKKSPOdBlZlQbjXqNtKgtowOuK4Ow8K63pKRvfaZPDESB5uNy/mOK9Ggt4o4UXZ822vTzGS5Iowox1YxJ/9kU/zcMCFqRRD1KdKUMgPC9fWvIudHKMLllyABTPPI42VOShPYYpDJHkcA5ouHKu5WZ3K8CnRu56rVhjGvPGKBOuOApouHKiB1Zz0/IUgDBtpHH0qYTMTyqij7QMkDbmi7HyogaLJGAT+FAjYj7pqVrhgOCvNM+2MoIJWnditEjKODkKfypCkvJUEfhVhbkMu7cM0C6Xu4/KldjtEgjWfBHelFvMQck5qf7Wg4BzTGussMEj8KNRWiQi1nxy5xR9lkI5Y1P8AaAcliQKj+1Ak7cmjULRGfZWHBP60fZCDkuAKesrE5ZTintIrADbzRYLRIDbqSMtQYVV/vZp5ZSccCgSLuwcZFMVkMZPm4phjAbcSasO53YBGDUec9SMUBYiJRemaYwVj0NWC6N8oApvAHy4qgsQMGUfKDihS5GQDxVnggfMBTkwVNAWKTPNnlTilCyMM7TVpnBbaOcUm7BIBIpXDlKuJcYwaPJl+8Fq0XYEYFIzsMdKeorIrGGUDhcE0bJemas7xkgsKheQDPzijULIjNvKG60ghfJzxUnmgAEt+tHnRn+LntQLQgMDlsA0z7M+4Et+FT+aqty1BkRuj4xT1FoJ5R96KUOMf60UUahoL5zOPuCpUDsMYAqslxkAbDmpC7BcjjNSK5LiTPbAqpErfaZTkDiplWQ+tQBZfPdR1xQtyktChPpP2q5SQPg5zj1qvo9ts125Ax8vFaF4k0l7Etm+TGuW9M1Do12EN5HMoE7PyfSvSUHUjyhK0UbawSs4VOSTwBUr2M6ybHIjbrzXPjxG2n3TQucyI2VPqKLrxKby7SS5ZkDcDIwKFg6a+JmDkzZeGRCRu3Y9KVYJw24N8pFR6fceYhPbsfWtAMK5K0VTk4pGkI3VyqLaU9WoFsTyWNWw+Ccc5pCcDjPNYXNOREItyBwxzTxZgjLMTUo3ADinbXI4ouPkRXNgvJ3mkFkp/jqyI5GIyMU4wvuGOlLmQciIPsKcfNyOtKbNM8Gp/s7nnvUi2pI64pcw+RFL7IoJweaQWbdc8Voi2YdDmlFscfeFLmH7NFEWuB604QgdVzV3yADyacFjA60uZlKCM9oAwwUpBbAL9ytAtEOQaY08I7mjmYckSkLZT/DSpbqp+5Vk3Fuvc5pjXkI4AouxcsRnlgfw1G8QzwuKlOoRAYCZpv28FciKnditEZ5QPVaPsikdM0G8Y9EFILyQZwFFGovdFFooHSnfZ4xgk4AqP7U4OGcVG1yWfO7IFPUPdLXlwnjHP0pPs6N0Aqv8Aav4jj8qRrxgQQRRqHulvyUX7wHNI1rGcEVV+1Ozc8+lSLNLkcce9GoaEy20YXBpxhUADBqu00qnOBTxNI2KWoaE6w9tvFBjCsPl61Es5GQ0mKbPMSAFkI96V2PQslWDAhBj0peWz8oFRQt8nzSEmmtMFkOWGPrRdi0Jh5gBximvLL5eCuRUH2yIEgNn8aBdRnjJNF5BdEkU7gdNtI88ndsVG0sJIBOBTHeI8A5p6iuiQynOWbJpPtBU9MmkjCdsfjT98KH5iM0xXAz7h93mkeTj7o/Go3mhB+8PwqM3UXTdk0IV0S+btxyOfSlMhI4FVTexKNpGWFOF2jjAOCKBcy7lkTMOvHtT1ldhjBqlvRh/rMGpElAIBmosO6LBc8g5NMDk5XGKUPCcjzuaTFuSMzGiwwYFhhj8v1rx7xJFJp2vXFvKMxM26PjqDXsJazBILNmsHxRoFnrenHYdt0gzG/v6V04ar7KeuxMo3R5HqNm0dtHcB96k4I7r6f1rMSQxy5Bxz2rUmSWEy2c6lHHDK36GspwQ2CORwa9SVm7oyQswO7J288/KQR+lQ1djgkuI9qrjAzkt2689h+OKrSR+W7LkHBxxSLI6KKKACnp0fB5x+lMqxZIkl3Gkr+WjHBbHTPFAFenou7uPxprAqxB4IOKdGSHGPpQA7YaAvOKuPASm9VyB1I7D39KrjAOSSKkBiRF3x/OtWzt9jhmIxtBBU+/rnFU1ulQDaoPrmtAsGsz5a7twyFPIY+wzww9RyM+jChDMu62+aQFAx/dGP07VWNSyj94SAQPQ9RUVMkD1pVNNzk04UAel/A++hh8fDT7lVaHUbWWDDHgMo8wH/AMcI/wCBV6lc6Tb2Guz20UikOcqxPQHsa+a9PvZNPv7e8hYrJBIsikHHIOa91EwkVJRlldQwOex5FYVpqHS9x38zXvnNjd/ZmVXBGQ69qh+2DGAn41mtyc5x+NMe7SIncwGPevPai2P2qRpeeTKiyHapIBPoK7zTdS0PRdNLC7jdyMn5gWJ/CvMUvlkTO4EGneeNuVAx0rSlNU3dITqJnoum+J9LD3N/NJskkIATvgdOKwfFGrwX9/HJatkBcE1yhnIY5H0oedi5VWHSrniHOPK0TzpHTaHJpj+YdUlKkH5QCRn8q37KbwrDcC5E67l+6rE8fhXmhuCJA34VK85HPalTq8myG6iZ6rdeLNJBSNWDxHqcVxOs3Vteak0to+yMjkDiuc+1SDkduxqNr9i+AM59KVSs5qzD2qRrtk8CUmgHb/Hk/WskXb4OFO6ovtk7cLHk96xug9suxsyOrDAbmkjCAHlifWsaSaVAC2Rz2py3rCAsAevSnoHtl2NqTYV+8fzpiNGDu7Dtmsdr6aQ/6oqMcmnwvJgnBAPShtC9t5Gt5iMCeahM0cbjPJrPkldE4bJzVGR5ZHO7PHSjcXtn2OhNxAJADwDU/wBptRj5Q2K5ORpmXAhdmHQ5qxYTs8mJo3SjQFWfY6D7Vb87VwPSo5ruMkFAKz1QmQ5+7mntwScdKSaH7WRO12VYEDJp5vy6LlQDVLIbgtg1E7Pvwh+XpQDqMvPdg8M3I7CiO6U5GeRVKTy0O0SDf3pm0Mw54HXFMj2kjVF8xUhsU1roDqeapRtEoXfu609kQs2QdvalcftJE32xQrHrQbzaAdvJqnJtWMhR1PFPj2+WA45ovYXtJFj7e2QRgU77RK6ZUDINV/KJH7tePenoQPu8HvS5h88h8l06xFivSoftUjFSCQGFWBbo3BbJPO2pQsaw7dgJ7e1LmDmb6lRrl0woOc0ov5FcAdDU6RquT5QOfWo1Q7yQq4WjmFeXcZJeyKAMd+tI0twME5Iqx5AmjzgDJ702KMqxDnK9sUvaDvLuVGkuX+7kE9qfiVUBZyTmrESHz2LJgD0NSAxbiuevPNNTYrO17lQlhyysxPQU1wxXLZUkdKskyBg2AaaSzOpIwB1NHtBamfHvwVZmzngmrScdV3VYkt1Uby2QenFQSOyMqBMEjqaOZisxksWDuyaRkXZnJxTpBI4UqMkH5qkKhlJCbTRzhZkQjiwPmoqQLOBwq/lRRzisTrIu3oCaXzwP4RVYOgGSpAFG6OQfKDVcqN+ZF0Xu0dBUFxfxxBpAfmb5RVWRJBjYOPWo5bczKPMIABpxsndh7Wxs6Ja/uZZ2+83JJrh7pL1/Ec32X/Ulss1dYss62/lrL8ntVJYooi6huX611LEKOqCVRNWJNJtLd2a5uogz4wCRUmv2VvqOmGOOMK6HcpHqKrhZETYsxC9uKYsErcm4bFYzqc0+e4lUSVi9pVygtosJwBhs9jWsbmMY2jIrChtUhQqkh55pwgbbkStSqSUxqpY3BcRDkGl+2x7ckiueCMjHMh21MI43XAkasuVDVZdjc+2pj1py3qE4xWAsZB/1jcU4rKmDuODRyor2yOgW9w3AzSf2gS2DxWA8sqkAPwaaDI7bt5osg9sjov7SjB+9StqkeBg5+lc4YTv4JNKqOmeD9aLRD2x0A1LA+9j60n9qYPy81gPbGT5i7Z9qVbYnAy1FkP2xuNqmT1AqF74jjeKzVtQshyTmg22cgE5pWQnVNEXTEcSrULs7P/rR7YqmtmF65z9acY2RfkU5zRYn2hcWdYwA7DdTvMA+bOQaoNEykOxzU6KWjOPyosL2ha3Y+YY5oMm/5RgVTUMScueKegBPzEn0o5Q9oTMgX+Ln60GWJcMfyqBrUtJuJPtzUpiwoHGfeiwvaMPtsBcgxH2OKaZjsOxQBStHxuwDSBSydQfpRYXOxBcERgmPJ9KabsynBjGPapzCAqktjjpQUGScjkUWDmkQefISAAVxQZrjdjJP4U9ZhGcFCeeuKV7ho3BK4H0o0FdkLTSuuOcg05LiRVwQakWaJk3BTk8/WmmfKkmP6UWDmIxKCcnOaGkkduAdvenCaFDllqT7VAX9AelOyFcrN5zEYkII64pVO0/O5arSy7yQEHNLLAEOWAGeaLBcr7NxBXOKTG3dzUylc/KwwKhfk8EEnih2QXI3JdAFODU0Q8uPcx5qMxgOdx5XpStnapYHFK6C4obJ6nHWmu+QMA596nzDw2eR2qESRiTLHPqKrQVyPJCDPc05YmbhaDNDJIArDFRFhHJ8hP4mi4XJDGUkOeWqQRgcswTimSSwiMMwO6mo6yoCEb2JphcEnRGKlgfrQ9xkfKO/FRSqiSr+7+b1qUzxRgcZx1pbD5mKw5DDg1M0mQpJxVU6hCyuqAlh3Iqb5JUDFuPQUw5mSlioPII9aXcoIUsDkcVAfK3fMxAUUglQSH0HQmi4XZj+IfDEGtoZI3WO5QcMB19jXl+p2FxY3LRXKbZAcE9j717eskMqfKmCTyRXO+MdNtr7R7iRISJ4ULq3Tpyc/hW9Gu4vlew1c8n851tvL7Z4/rUFW3AECn5B7AEn8f8APf8AAVSMdK9E0QlJS0lACilVirhh1ByKbTlG5gM4oAmu02zZBJyoPPXOOf1zUArfu9PEmnpL0dYwT+Arn6lSuJGxpiJMGSVJCmMkx8lffHp+lVri38kjcCM9A3BP4dRSWTHdho1ZB1JRTj6kjir97FG0W+JnZSMtvbgcevfmmMxyDnpgfWtLTojc28sQc5AJVQPveo/l/nrngZb5R+Jq/pUjw3QfBC45IH3ff86AKdwT3wTkkmq54GKtXkpeVt6jOe3+elVKAClzSUUwHV69oU11N4fsZBICrQqoz7cf0ryCvZPC4SHwxpyFct5W7n3JP9a5MX8KsRK3UtiORciSQHjOKcunC4QsXIzzg1I1xGGBKAuOlC3JdgcAE9BXB7xFoj4bSKFfwqCWfymGITjv7VcjDMw3dPWmXaNnZwXJ7dqFF2Ah3IQG5DHsaYZVViQpJPFSxxOpxIucd6aAYZWwMgc8UcoXIiheRRJEQCakMDEFVVgO2TU0spMXmBMkdM0yKS4kkBJRfanysLjRC+zYE3OKckDxr86D3FTjzQgy4U7sZpjyOcbjuJ7iq5QKpdFY/NgdOKkWPbzHgqw5NRvAgcEPknkikDBbUxry2fypWFckkhBQt5gIHSolKSMp/g/uihIZRFHHtJyTj3qYWbBkDcd+KTSDUmWONWGTx1waR2iUZ7etQCykExIlYh+MHtVk26L8mOgo0GRiOGOEsHBc9qZJMqKN0YAAznFWXt7cYZBhgtRtaxScBs5HzbqWgalVLxi7MIwSRxxUUuoqgYFQrg88VqxW6ONyryvAqGTTVe4IZQdwzyKVxWZShuiVLbcZoSdAWDEn1q8LLMbYUADoKjktYIUIHLdwfWlzBZkMewleuTViO2LynI2jHSoUWd3XKKq54qWT7R5hwwwvQinzoaRD9lWG4cyoDnpUBjIkJUc+gq15crENKc96Gk8tyFi+ZhzUufYLCrAxbJPB6CpWhIJyfxNR25LE7mIPbPap8M52kbg3U0rtlJKxF5KhBnB7g0/ySQpfAYmo5ECqQCSF6c0wNcFQ2cAd6G2LQmaQrKEUYx29aifapyxwc9BT1y8p35JXHNRTKZGKoOR1pa2BkqlcK4kwAetLJIquTkk+lNS3VfL7p1bPeiaDM+0qwTqCKEmIc11vBwOcdBSwJuPLcDqKjWDynZwucY4JqeOEq5YE4PJFHKxk0giRPvdD0BquZUDEE4HanNbl3Bjyc9arS20iHOP4uQarlsFxwkG8BGbGOTSod0hy3HY+tLbmJn+ddozjmo5YXRjwMZ+XntRYVx0Tv5jrMcen0p0k3lqY1OcD8qh80P8AMq8heTVSdp3g3D5uMYHegV2XPtG9NuelOkmQDJZcjvWQFuBhQCAR0NPNq8sJZwd3TiqUUHMzUdx5I2yKS3IpQJdh+YDAz9ayZoXEIi8wqV9atRRHbH+9ICjnNDikF7lvdIOCvP1oquQM/wDHwfzopaDsx6MXHC4z609XZRg4GKjBAO3pkZ+lAdiSox0qhExZ9u4MPpTSm5gHyc1VNw0LEFxx0GKkDyTqCWKgc8UriuW/LEabNvFQPED8wHFVGkdrnAkZwBUrmQxg7iMdqeoyVpAcdPTFLvRU+9xVczrlF25Y08u0knEeQKPURajVHXG7B7U8Mis2DnFVS0wIKgY7+1PKgHJ43U1YYOUwdwz3NOtjAykggVG9q0h+9xUbxiDABUCkBbWRBwSPY0C5VmEfUiqDb5YwFcLzTkiIcbmJajUC3IYt3Xr2pHdYlJA/GoHWJHUsDk1N9qt2HlsfYUhirL5g4PNSLKWOCuQO9QSEq/7sDHelDkqQoz6mnYLltboFiPLyvSkjmkDEKoA96oPOYuQCT2AqSKRpmBYFQetAJltp8biw5pPtbbsYHNQTxlQFRgQeKh2lUwp3GgGy0boljkjIpGuZCOOPrTILRJGBZ8P6GoLuCVZWy2VHYUxakxaViNx4NOkuRb4O761VSNpdqefirv2e3J2ud5pBqRRX8Zct2PSla/G7AIz2qR7aBuI1UkDioVt0cEnGRxikn3HZkU+qyRkdScZ4pU1CWeMEZ3H1pY7ZtjF8AA9xUsQiwqsRwcg4p3VgsJbXEyZEufamzPdNMTEMKemO9aG6Hbg44HehZUYAKcH0pahZGe4vip7t6UKLzGZnCr2Aqy1y+TGqNu61Tlju523I20D+FhTSYDjdzK4jVt2fappZLhlxtyai2sAoGC69an8yUIWOD607BYqmS4EfzjHPQVIJbg8qmQO9PdUkIbdtH86he8hT9yuSxO0gUWYiVLOSRt8jcntmm3O212lI2kbPSnmOQtmNzwMdakUMq/OwyOhqba6jIY55ZRnaUb0PapwlxKB5kpx2AoJ4JUrjvmlLMRtQgEj1oAiMUm7Ebg5qRLNlJB+bPoaprdJDMVCMecFq0o5w6NIVIIHFO2mrFdEa7N7Rs4DgelRSpPKu1ZeKTzwkLPxvJ7inS3g+zDCnOOwqH5FXVgELHA8wACqEzSq7YVnBbnipI2EECyEszk5waYZnlkxgj/Z9KtJktj0SFWwBtbHr3qQ2QmTdLOVxx8prOSIPL94r1ye1TriKPb5jH5snNDTFcsyaciIP9Jdl7CpY4kKBBK4HvUUe6RQiyEg9DjpUhuI7Jf3jbmzQ0O6Lf2ZNoJbd65qq725lwRz6003hlJW3xtPXPrUXlJDjeS0hOcCkojbJ0Csx2oFUnmrDC3hRWJ5z0FVnhRFDmQgEcg1Ebi2RjGz5HUNnpTsK5pGWB1PyjGOeKR0tpI1UrletUI7tJGKq6svQGrbrmEOTt9eeKVktx37EyyRRKDEgyPWud8YeJGg0SRGRPOuQYUHtj5j+A/UitSOWEwuqy/MTXEfECFo4tOJcMP3gAH/Aa1oKMppDUm9DjwoaIxqQzA8D+nvTbi3EUSsMjoBnnPGf5Gn2RCyq7EgBhkgZ+tMuzJvOc44IB9MDH5Zr1TQqHn2ptKTk0lMYUqttYMOoOaSigDpba8E+kPvYArGVP04/xrnH+8asW8xWGWP1H+f5VWPNTFWbEh8T7Gzj9K07ef7UyxOAxH3VPT8fU/Xj61kjrVyzuGtplkQfMvI9z2/ofwpjNC6tFjnZVILdBz0Ocdep571LZBLdWadD5bLhiuBwQD+XH+e93SbcX0haXIjUEbxzgbTk8+4z+eBnFN1BDEZI9irlSApGeD1+pGRz/s5yBSH0Oau1CzvtJKk8E1BViSMZKjpniq9MQUUUUwFHWvdUtYoBFbqwCRRhB9AMV4jZRedfW8X9+RV/MivcgyXPPlnd6muLFvZGc2VXIHyAAj19KkS2OFbrjt61JClut0BLwOhA71PJJbwruQ4TOCT61x8yRmVnmQuoG4KOpHap4Ut5opMOSwbIJNRw3ESqdyDb1JPeplZQnnRxrsY8gVPtEVYhmZ2DNCQy9CO9RR8BiMhmHQ1adYmVlXClj261V8+FH8rdlh+dPnQmOljBUhWbgciktttnKM4+YfxHpTHm81VMRJBOPekt7aeacGVdwJwPalzh1Jih8p180EE5GTTkyAFVgGI+bNJJZbSDESRg5U1JHZuih93y9dpo52Mi8yND5YxycFqb5aoSEAIJqZbR5XkHBHVfar1tZbHVWKttHB9Km7YWKaTO5RQh4P6VMYf4lBBzknPSpJUWQRrA/wAyn5sCmmaNGEeWDMucnpRqGg3yZQS4wR2xUZM6AhUDKf4j2NK93hAoPGaWaUrDhmI78UhkPktuLHcCBnAp6RCYhipDD070yWcoFDyH5hgMO1VmupbVkSOT7xOXNFhXSNFm8g7cksRkY7U7zwIhIW+71PrWYCZ42EkxXLfMe5qOG5jB8vJZCdoDGgfMahuofMjCH8e1Vrh43kYscN2NRTOkcgjAXeOmOmKXGVAkUHFHKHMyd5CWTAB47dKfNdpDCSVUyEfMBVB5DbRlVfBONo9KgLSEfviDmq2E2WmuWYb1ON35ClF2kpCshDd2FMhhVIVZMtnnaakU4X92nLdV9qAVySCRTGHwGwcGrMayIkbFSPM5AasISGKEgbhlzgHvV1biZIQruWYqcbuiincFqXWLo7qQjFuABQpErDJwg4I96r277JIy5BDL19Kj2AM4jOQOevWi4WZota5iZJlwSM7l9KmS2VGjeMjBHz5rOW4ma2lIfLqMBT3qGFrvJYkjfxtb1pXQWZtJFGH8zIKnjaelCyQiVmYYIOAKpNG7tsJ+XGAR2NAsJAAJZCd33eeaY+UvsLNz5qkZ4496sx+RIm0RqXQZB9ay0thFabywYBsbO+afGxIXZjgYOTyaaHYsSylkdRtUk8HtVG5MY3qzZYAYPrTblkaJVBG7Jbr0pfNXCGRFD44B70mwsUZolH7zcTg8AU55tw2HczYzyOlXkmtRKCyBc8mmTXkGWKRqd+FpWQcqXUoi6tkmCmNs98dKEk3GRljx3VauzLB5ZKRqpGBk01CsWWcKdw4C0WCyHQI7sFdV2nketMeFl3rgDPINRG4lAZVCqg5Y55p0Vy8m5XG4kfL609B6CBN1tmeMb88n2oIdiixxDbnj6UNNIhGF3qx5A7Ukc8oYPtZOSMDnIo0FeJMY1Bx5aflRVZtUjDEEgHPTFFFkF4iLMmNxG49OlS741wNo3E9RWRC0q7gXGDUqEhMAkjrnPSnZkXNEpCSzyBeKhe4iBZVTk9BWVPcMVIRWf6UsbxNGWKyK49TTcWK5oi6WJMiMKc+lMfUYzJsbA45qsJYpLRxuIYd2HeoIbJsmZn3t2pqPcLsvmeNIyzKAc8HFMhvw0jAYbHpTPs0s0YEjZAHTFR2lvHFO/lxspHHNUkg1LFzcXSGMwRFt33qsJKzEB/lOOBVWS5khYqGLBu/pUcUzhgThjnFHKFzS2g/M8pB+tVXhjlZm8zOPekluI0Uqy5J5NV5Gh8tGjxuzyKVguWFaJUVEfkdqnSeKNgG3Me5qnAHkkyEG09Gqyi/Z2LTsNp6UNXBCySwScoSPrUK7S3ypu5yTSNCk8hMZzgdqkAYALGmHAwc+tLlASWUxxkqpJNLFOV4PGRTCZNihwffApRsLAE4BHWqSAsWiE7nY7gTxxTyFViXbjsBWelxcWoAwPLyQCamOFjMsrbsnoKXKMejCZ2GSvbB61Y8oxLgA5qukiM4CjGec0PNIFJVt3PSjkVwuOMztJwhUL3p5LuuRnrzmmW7TFt2znvup8skjDBCqO1PlC4gt23Ehdu3p704eYq/dwT1qCOWTnfJk5wAKXz3csVGFQY570WAeqsSVVtueMg0wWzRtuWfnPbvVaO/2hQFyrHk4p0dyrhgqMAT1o5QuWZNUVJfLZCVUckDvUyXIKqzBNh6etZrzYURGPLMcg0iwGRd7vsGeFPrSaQrs0jJG0rc84+7R9riCkBDuHcVlStcg5CL8oxmrVjG21iCGOOcmmFy8LlmyRgLjqahOoIDgyKc0xo5ZJNuRg9QKoNo0j3RdmITGAtTddR6mncyRNEPJf94e4NQohfLvIQvQjNNtNNeNdsmVXOd2eaui1jERVfmUHOTRfUCGJTJ8uDgHjNOJty7nyv3h4HFOlQxZYPggZI9BSLKibJEYSD26ih3AhSV1dYsNx1bFOkkKhiSQvbNOvb8QDCRZLdDVZZWlmQSKGyKWoh0JaQYPIPOAauLATgklf6VPbwxg8AKQM1MhjIGTjnvUtlJEHkRwNtHO7mnPOEhJ257cCpWeJSvIJPTNQyzR9FVmBODjvSsxlR5ZZlDKFQ56GrdnHwTIwbj0p6LAwGcL6DvU5lt4ovuc+tNxYloVJLMSOMHaRzmhLWIEszH/ABpy3iOwCnBPrR9oU7vnUr057VNpIeg5TaBPLULk80eXFht0QI7GkRbZkLDAK9xSmRDCdsn3TzmhRbC6KxmaNSi25Ge1EMZlBEkSr7GmvORKXLD0FUG1GVbh2MbuOgAFPlYm0XmRoiQsaMCflp5+0KcrErEjGfSqcHmTqX3OhJ71pW6SorFnyAO9PUW5QuN0SN58oII+X2qr9mtJYFILMT6VpSTQSO8eOV6qRTdq2+w+VtLcnA6UXYFe3tIAw2xlGB5q+yqqkEEpjvQGEsykYCHj60+7t2cZSXjoR6UNXC5XEcZjwExjkcVxPjwoba0CEkCRv5V3kaxpGpLklRgMOhrkviHEqaLZsAQRcYP4qf8ACtcPFKomUtzgoVHkjjJznA/GpLsF4xIcEsOffv8AyP6VBb9G47duaszDzImbBVS2Ao5A/wA/rXqGxjkYJFJUzRkjPeoaoQuKSnU2gByNtP14NNoooAcoqaNdzAAZ5qKMZrT02DfOPl3bQTx/n6UgOm0tzaW6zHBQLnaRkMMEc/jt/Osm+meWQMzeZJty244z2/w/zzW+6Qmyh2EMYyGkGMBsjcefwFYGpEqSwGOMAkc0hmJK3zkDkA1WblifepZG3MxqGmhBRRRTA0dBj83xBpseM7rqMH/voV7qtsixOAcMeDXiPhZd3irS+MhblHI9gcn9BXt7z2zzmFTyv3gT37VwYz4kRIoyWISfDS5DAkEVXSwMcBaaRjE75A9a12KuAi4yAeKQfZ5rZQzfMrfdHpXDykXRSEUSoiMhZc4H0qcLHHA56fN8q98VbW3jaUbeA3GCfSobj7Hbz5ByRyQT+lLlC5EY0eKP5gHpUsLaCbzJlHmFclhRFd2gu1yuI/Wpg/myvhQw6Ae1FgREix8HKjJyQKuxbSqMEI38VB9ie4dBgAqOTnrU0ZggBE0gAAyMdBQo9xla5hZFcxksFxjHYUSKWZFLgEdqmnvooofKjZGcjn1rn7qea4kMkZy6jGwHrinyoTZsyTfY43yclumKhh1F7iBBHGcg5PvWZY3LXFopmQglz949BWu7/ZbKNlCYLZyp5FHKK4sYkKNcBNm5sDPBpfKDLkgcnv1NCyn7u7e74A/2RTLqdYYcZyVbauOtXZBcdb2ts4BLfIOntUciwxpsYEg/cc9CKz42nZJnRl37j8oPSiJ7lrULNklT8uRwanQdy2bdAfm3DuO9TtbosKs6qSDnB7VnxPPFE4myzMfl287auLcxG1EZkIlGMjHUUaAMdEmVH2KGY9B1pGs4lAMaAlhyMdKicLCTJFwG6L3Bq9FNGIlkZSHI59qFYCKCzVZFYQ4KqckdM1MLYxusTqckbjxTHlKsrhi3PyqO9JNfeb5hJbccfN3FVoBG5R97bAw3YOF5FJ+62YCEt/eI4qBphLZHY+CGwcdTSQFiuVbec7cVLkgV2ThISHO/Cr87HofoKW3nhSJpW656eoqtdW77pHUZUjG3vSGOXao2MNuM+gqebsPUlbyFPztg54HXApoMbKzE7vmwB7CnEq8pQhVZhkZ9qYsUruvlINm05OehpBexZeKOZcowA7rR9nRXCQnnHINNgLK/luq5AIJ9fepVh+SOSZx8wyMcfSmO4ySfZDs8ja+QAfWoZbgJG6u5jLEH1qUyJPAHLcw/eqrMoklCoAc/3qLi5mSJeOxjTdz0X1NNuJ7ueFHwwaPIOOtRRwyKollZXZG2rt7Vb3yRQlypJlwMCi7Fe+4yCWaOxdGUs7Px65pkSNIrZlKt1AalFvM0rOkg2owZj3FJPGxufLjdSAfvDvmn0EEETQzKZtoSRsHnOKbcW8TzN+/LKpJBX+VNjR2l8hmYZzknoDVz+zmW3WMOrAZ3H0P1p2ApYIYyRRgo3HXpT5YhxIhGRgke1NgjJsXDnKnJCKcZ96jAmDxgnKFfnLGkFy0zosTxqN2/BDDt9akmnCPGfkx935aph1MUWCSgciQEY6e9WrqBpFzDbBrdFDnHUmnqFyvdTRmFRP8AIOgx1NKzx7GkB4GFOB0NI1q7mOTOCwIBYcYqwiqtm4Vtzcbm96BFWO6e1DCMiRVPJPpTWuT5rlJQMchQepNNuEEc5jUkq3PA4NVZkKiJjDgIxyAO1MTZZMhY7vJHPNFPVUZQQrAEZAzRS1HdjHQbAQy5PIqW0tsRn7RJwx7U8ukUJk2j5eAtZs63dxKAjBVJx16VerQyw9qqzOY3wB0xUe1VCiRcnsTVqGzWJVEjs+OtWXt7e5O1R8oHNKMrDsY2yJpdyEBQfmWrcW1CwDA45HPanDSoFR8uwDccGq5sl8ttpYEDkk9a0umTaxoK0LM2JQCRwKa/7nCtyX71lxjaybioC9geasSlZyMOc9sHpRZBcmktBOpQOUHUU230/wAlt7uX3djVK51NrR0T7yAcnHelj1Q3OFzsI7mlZhdGiLXziX2D5e5qvJBa+fkvnHUDoKtwRSiEr52eM/WqNwJY+I7fIPcCkpLYGi2yRRAAsVHUYqG4iE0wBZigHGag+0XDNHvtjnvnpVmKeQ/uxAXbPWmgCO1EKFkY4AzimRXKG4c7vT8KT7RKzsGQBQcFfSrcVpHjzCq0SfYLEZl/eAg7h0/GomtGK5EmG9O1aD20CIrgHOc4FRPK5YosPzYzUczHYqG3YoyTZkIHbtT0gmZFUR4A9acfOyMDy/72e9DSSxpje2QM4ouwsTyRRoiMwPHBqrlVUlBnByasWl002UmRgQOMjrVh8l1QQjDcZxQpdx2KTSs0YA5zyMdqBGeV2vuC8Gp5JjFtXytu04PFSE73Em4hW6Yp84WM9Yp2lCsAE9auQRqrMgG4dyallZYV2OhZs5J9qqtJcbwIoSIj39afMK1iWSJHjZ/LULnjFV54nZN68bemO9WRp10EWEvgsd556VJBEDGyzNhuigGhMLMpXF1BBHGXjJIGRtHNVXJv9rxZZd3I/u1rSQ+apWEIxUfNmmx24ggkSMKsjjcxx3quYLXKC2+5fL3NgnlqtWdqlug3Nk+tQxyqCq7+/wCdPxKtxzgr2pXEkWGhWOX75GOc1WN5ullYZMecKaleHzVmaWTawwAtNS3gChJiwXrxSbHZki3RyVdu2R9KcbhY2TA+/wA7apXE0vmpHFDuQdz3FRyJOrx7sFsgqBxgelPQDUdtxZ5I8g/d96hnyqKyIqkjoKPMlmQwJGwJPBPrU5t3jljM53ditDaHYyZvtVxsEcO7/aPFEFpeLITKORzhewrZnLBFjWIqwPygd6mihAU4O2RhgknpUuXRCsRQttT5uuKJrdpAp/ixkYPapTbqEC5+Y5yfUU8lYW2M2C6/Liot5lMz3iRSvUntUsZJHCMp7ZFSJEN5Q87Tng8iknuljY7ScDtjrVJIke8ZYq4wTjHFU2MhSUOGDkYHcUNfOVZovTgY605NRjWNDMBncMgVSa6DM1I7jcBICBngjrUyWc7DYgxGWwdxqe5uUF0Sil1I3fL2plpfi4jlRciTIwCefrSuKxKtoYiSz7QzYKg4GKsQ28Zd4x8yEZLVA0olmeJkXOMLk9xT0mZJljCMdqckUKVgtqNkiEMYcANtYHOO1LO9qT5sIYh8Bs9jVhruGOGTdGSCMhT1zVJLqO5m2NGY0XG1AOvvRdASTOrhRF8hGOvSq9xdTQtvbhD096t3nCrEsYLKeRjkUz7P9pZI5IWwvTnihNILMz3nG/zQGIPRhV+OeOWGd2f5tn3vf0qVrIRrkEAxYUoB1qNbVZoCI1CEkscjGfpRdIVmNRwyo7kK+3AB4waa5MofDBHPBOalFldSYDRg7eQfSopdHumuDOWCxgcAdC1HMh8rC38uLELycDnI7n3rC8asJvDssbYLwypICPrt/wDZq6ePS9rKHXcHGcnsao+K9LtpPCuoyxKWljhDEjtgg/0qqc1zoai1qeQxAlTg81bBPkkDP97HYnH/ANeqURI5JIA/z/hWgqKYW65AwcDjqcn8gK9U3RTRNzHoQD3/ACqrcoFfgYq7u2M3JBJ5/CqMpyelCBkVBpaSmSJRSmkpjLlpFvxnvW1p6GNztXcT8oAGe3P6GqNkik9wMYA65rasYxHdo7LwCHOO2M/4ipKRdu7rZEUIDOQrDAxg5578jnqfQYxznmrybMLDGTn1/X+Vb2otI0ssbqd0Zwrtyep46+o/WuWvCd2D68D0FAiox4amU4/d+pptNCCiiimB03gSzN34mjIx+5jaTnp/d/8AZq9Be1eG9mkadmZTnGP0rlfhrboZtRut6CSJURQx7MST/wCgiu7exk85sybjL3HceleZinepYzkrkcLNJbvIwKSDkgGp4ZQiRyIp3Y785pAVS9EDKVVMhmx1461IpWOZfIKkA/mK5LEpIWZ5LhWQxEMq7t2cbap3NnILiMzKdhUDdn9auXDy+a6tgDOTu43ChI58Md4dD9wN6elMfKUZYHWUlcO2dqIPSnfamjYiAMSp5XHJq0YY4ozI0jBwc49/ShbeMMQgdSwyT0waB8o1r6S3mTerRqV3D2FUHmmmQYbKF8hc9av3DLcrmT/liuDk84FNVo4pImSMAE5PHak2JxMv7PNdXDtIvlhOOOCaYlndQq7RsWkXkYPTNbN35SykIS3Gc44qFjJFahgNrAHIb0ptk8qKyQssYjkBRiMnvinLBOuQ7loQMjFK11tXzirfvAPl65FXHBazKBgrueT1GKQ1YjtQIpHfz8lxw2O1RyQiKYkPlBk7sZyTT4rZIQuQXYEqeeAasRRiG8aIIHkOAV6jmgCJ5EidBbxglgN7etWLdo/O8s7iOWAx0qNIBNdzSK4jjV/udhV5IDK6yIQcLgqPSi9mFyoYxHMXj4B5JPNVkKRXBB3YcenpWjc2RtXD5PJzjsBUa5dXUZGAX5759KTmhlV0V4g4yxbjGMVZEAaLbgAMe/NK5Vp44mlIhVRk47+9STy77QQqoVkU89j6UcyC5TmDPGqRyKfJGTjrinC2T7KJRJ90Zyff1qtI0pkeZAGYYTavcd6sz2/lW0ggZmhkbaW/u8elNSEmVS2VaKGD5s5diKDujtxI0f3jlGzjNTwtJ5DRyKPNHHA5+tNjiIgMch+Unofr1FK4k2iSUM7RRR/xqCze9NmldojvXBjbDr6ip7aQeXwfmHRvRR1qnCkl4J5hG5OTyTjIH/1qLj1ZWRonheQ/wtwRzxVu3DzyIsTZ3ruwBwD71GYY1ZT80aiMnYw6/wCNWdNYLEPKI5TJPQk0XFazILszQMN4+c8KVGasSxm4iCooAxj6GmRyiRnIk3BW+QHt61fhYzP5Eagydc9Bk007j1K9vY24O2RSsjqCuOcmkQW8ZkeRGaROAMetOdDb3JMsm1QNrcHKGnsy2sqOTu3sdzHoaq1ugjPlMVsWUnhsMoPUn3q7AyJi3mfEci7kkA/i9KJLWOSRgoLAnJK9QfQGq5glnQxhcqjfLvPJOO1JPUEmWvs3kwM5AClOFH8bd6jkWCO3Vi4hZcMBjknsDUlu7TaZGk8ZVlyy9+nekklWW7F0+HRQAAVxk96bGYtxdt53+ry7ZXI+7n1qFtRmWSaRtzR4+ZAcY962LuzBtjLCx8ndwMcg+9QpY+dKE2L5McZb5up+tQ9ybMgaeFUUwAshj3gZ5FSi3Lxq7Fjay7T5qjJUn2qC1kgyY4Uw+5sIw+6O34VMbi5RHS1gaaOIhjzj5s8haI+YDlspIl8rcWEm5tzdDzV0hIYl2uVZiSy5xxVNpVs3R2mkCyNgIwyBn3pjo1+v7htpR9r7/r0p3DUs3lw11ujOfLGznGFwBxj1qayS2+xnyyzSkgsv17VHJZ3Ubt5vGEwSnKbajtLkiPybePa27f5hHBHp+FNt31HYmljBZfNRE2xllAPSqqFjb3CzIA6kbR/ept3LIJpZHcAlfmIGev8AjUcEc8szyFJFZX2KOvHbNDQisYbfccwTk55INFW/NuuwbFFLULMhMkUwKkkYPT1pFmt4x5bOAAcg980fZdkjEOx3AD2pzWUfyjaG9zXRzoY538yIBHG0dTmi3GYyqMGzyTVS4WVJhCkB8vuc0QR3uWXytkbjANJtWBXuXZlAgDOCMHkZ61H5cZh2b2XceMVWCXU8xikTC9Mk8UhtJQcecVZc49qlJDe5BdQR2rEYL5PJqub1Y5FVYj8vcDNW4oitsXuGZcnHPer9tdW/mCJYgcLySO1Ve2wrXM4BLuULHEQRyd44NXm03nCRrk88Cr0dzAo83aCvPIHpT01GCcIsPcnpUNy7DSRVihuYz8xBGOKdLvJKxrjI6+9WPtsRX5cdDj1yKha5kcoSoVepIqdRixRuoYXK4UDAxSfY4o5N0bEHOetFxepAsZlYAEnJNUzeQTJKyzHIx161SixNonYxQl3ZMnOfrVMX8sjCFIG3scg9gKt2qwsmCQ56nNL5kDyF2YIUU4YGjXsGhbt9xQbwA2Oakl/dRhipJPoKz49StnQ/vMHpn1q3DcA27Krg7uR3xUtMpNW0G3Ll7dm4Vh0yOaptJI6rIU56VbZFUq27JI5B7UyJhCjo7b93Iz2oTexLuRIwaUyFSGAzinSTy5iABI6gCiO7hvE+X5MdSR0qezSOWR1STcympejBFO6ubl3KRxZXr71FHczNIirGwYdiOK0JbK53Ha6qFPWkWG53qxKq27I45xTuNxI3llD7tgY8AjFPllm3RsB8hBPHY1caQeW4G0se4FKtwjRCKTGO5Wkrsp2RQiaZ4T5pYNuyoFNG4NucEMh6VLNeQRyInKbTnd2IqVbuK5mTjgNjJHWqvYSRBZ2kn2h5EfarckUCG6fzGLAqM4PSrYKCVioHXGabl2LxnnIyQKXO+g+UorpkaSCU5LOO56GpZE8uJDtG4ja3vVmQeYyhVcMuA2egp4t4pHdBzIg45o52HL2M5XNyuNnznpj2qS4YvP5cK5BUZHvV9baKK3bAYSZB9sVVNvAboMJghwScdzU8zC3cgh8zPlmPEinGe2KvSWu6VGlAJ7Y9fSqlqXmmcx7sKSCT/EK0Ig86hCQu0blPpVKQh8UsEjMiR4kQcD3qrKhMBaUN5it60iwOJXmL9XAJXqTS4b+0VXzcq5wSfTFPmuAklzsbfInAXOB1rOmvpriRnhjMakd60Wi8yWZWfcR8qn1pZYvs7RRxohPH4GpbDl6lU3juyqY2UKoXcRxzSahK8t5GVG3aMAmrsxhhTkllB3EY4NRhEEW5/uuQ2Qc4qWx8t9GZsP2qGCa4AYgttye9Wo4y4jVlDMV+Zs1ejczQHysFT19M1XglaNmiVBsB5brg+maCuVJEsNhai7VlyF25Ge1RfYYJJ2LgZYce9LG4nmLx4wBnGalnuIUzvjIKLgFTxVK5BDPZJCoiA2MVwSp71E1vAI4phGmYztPYmrAaWRQUjO9weW6Z7U23snwVkfOxd2AeR65odwJms41ljuCoJHIxyeackCrIXwGz9/2p9vGZZRtZfI+6OeSajV/s7GN23K+SS38qpIV0MaCGOElVPmEkH3BqwsSpGjpCgdRgZ6kU4SQeWtxuOM4J6jHSq1/eRAwBGBRflz/U0WC9i4LaJ1AUnzGO5jjvUYJKDClShw4x2NZcF3PLcymJ87AFAB/WrEF87ExFSjO4z34oVhcxqBYvnXjd0JHpSC3gjjZss5JPX+GoZ7lUmd44/LGPmJ549arx3LXVrKEc4Zd3ufYVLaK5jRtpTwE3YK45FU70zGLCrhYxufPQHtUljdLNHCnCsgyfcVIG89roOTuK/d/lRcV2xgE2UnkzlemBxjFV76Nrzw5qsYUBpYJAMj73yHFTx6lceYgkhLAcDcOMdM0rXESwgoSCvVcHkE9KuL1TFqfP8Z+brgVsW6jyHOARtK4zzkg1Qvbf7Hq91bZ/1Mzp9cEit2ww9nMgG+NVLE+hP/6x+IH0r2FqjZGHdAozjAG3jj/P+fxrNc1oXofzCSflPQGs5qaGxuaKSiqELQOtFFAGzpxO7oTzz/hXUWSocvJgjaMqTz+Xfv79O+M8rp52yduh612EHyWyyFeJF+Vd2M4+YD65x+f5SUjJ1RGcuwbaFJGAemTx/SuWuW+faGyBXQahJHsATOSAwz79R+WPxFc5JzIcevGKBMY3YU2lflzSVQgopKciF2CqCSfSgDp/A17Dbay8FxMI47iPapbpvBBXP6j8a9YkjCK8mSJIzu49MV4cNPkTHmbUOAwDHkgjI4FXILjUrZSIbq5RMcgMdv5dK4sRhvaS5kyXBs9geUPGsjtlnPl5H5U42C+dHFHvOentjvXJ+G/FqrZfYNTjBO/ek4X+L39P5V29pPHJG85cNIwURkNlfrxXBUpzg7MjltuUTBNJebJwGcP1J6rUshkhYBslUj61duZt+/AiICgb8ZZcHJxULTG5t1VfmickGTvjFSBmoLkR5YAhsuC2fyq2xuRAoILswGSPfpSLHLCWhyJyEG1gc59eKvtJBLp9vEWc3fLbgcKFB4x68UJMRk3Ikt7uC3GJASTJz+lE6SSSvMuFi42KGzxRNpbyyPLC0xRMsRjq3bJ9+aWWBontpFYoqx4MYH3jnrUvUWooBmaO3ug6Nncigc/jTHgkvJ5bYvnYcYBxn2q9I0jS+cBiZRhyV5KiieJTP5kceEIDnjBx3z9aGmUrdSg2mH7IkaYZ1yxycY54p0NlIjEvLtDJ0Pc1fult45EIDxpKA3zH26VI6GKMSOoaEqDk9Sx44puIijaQOEUKCSPmJIzk9qmF3HHNu2rGz/fJHOPSnrI0LjZKzMy7djLyo9aotcWz3KTmFnMQKuvXn1A9aSVh6WNefy44pLlYfkdchfXHU1Sa4RJnmiLFHTagXgBsdajvNXSO03OxkMo2Km3lR0qOyhXYImfEUUYZSDkM5HA/LNNxuK5L5s8qCO8uSFOOntyagjleTUEBlbazbgxGRt7ZqWO3dkMbpIYjmRd3qfSoreG7mtBHuP8Ao8eCccgEnAPtSUbAWHZGfEfLE4cHgnmnKAbREdCNpLfMeRnsaZbQSKQgykkiKGycnPtSSpdXjsJGMaCQ/LjBYbeKq2lwQkErGV1hVGdMcZ70kcM4eVw6NGGy/P3cH0pbb5opyqqI1IVjt5znGfpT7xW8yBAhV3GeRjd6H6HFHS4CmUfamuVlOyOPG0epPI+lVvllMb3BMfzFYz04ParMUao0kkiFVYhQO3rzTpEjurho51cMDkFRxz0NO1xFSCLEvkMxzuO5h3TrwatxwFooZVRxEcv7kD2+lWmtIXtrjy3Hmtwoz90AdafaWqvAhSbJC7CpP3B6/TNHKMp37rKbG4AYIwEeAOMA4NRpaLbXDpb/ADFWCBGzg55zmrMsi+VPCY1Zon554Xnt60WjiVgJmWPdl9wznPTmi2oEMdtaxJ5+4NIQSMD7ntQt35UonDFPkwQ3QHNWroQJZRohDHdsbb2x15qo86G52xwDay7QvZWHem9GDZJF9pmkVXiR/OOS4PX3+nNTXiHz0V1UKF2qo4HHeq1xd+XIiBsREhgAeUY/eH0pbmZ7rSru1hIcxqrGQdVBJGPegLiui2i+bCWO0fvNo4DEfrUf2gS3ccgUKFJOB3OME4p8s3mWn2IFY4EDNJIzYLemKz1vUCW5xuC7l8wjAxznn16Uth3LeoaohlW384bAoXK9hnn+dVpG32tvBDOXV3Ow4x3xWbFEl3d7y4Ql1bJJOVByBnt0rXusPHsVAGj+dJFOB1zwe9Jy1JV2XLqMLaxROHhlRT5mGzkjAz+VUxuMRaJgqZJ3c5cdeaklRhbKZIw0sjBmIfLKD2qMNiaTcr+UkRODxyD/ADpuzZSNRYIiVuDCC2DIojAAPHf2qpHeKlwrmMBCuEjU8ZHSqkMktkxdyWaZdqjB2jPYVXfLQiaRkH71gxU/MMdh/jTuD0Zcd3F6iyx+ZAWBweik9jVW5uWi3MihVUs3X34H1pkk9zACFUssybipIyAaktLeT7WPNicwuRgsOoxnGfzpbuwmiYXrwOolQsGQMyqfvZ6A+9V4LwyeUskZWB8s0adQe5qmYhGzyoGMMbA7CNxGR6d+KDFJDOFMnlpKhCCUcjPqe1VsTdsku5W+x3RR9xT5V2tztxjkdcZqVJ7q38tsiRmUNtR87sVUZHt78xMqyE/u1KchiRwc/wCelX9PtY0tpjIkzgkbjGMFMdxnkf8A1qBpNss/2jdj+Jl9gvSio9sh6Xa/jIKKOYZmR6jDKmSW2K20joasI4kVAFbA4Az61lrpzlBMZBGw46dakgE5HmM564APrVNEXNNrgDIA4BwTnpRNeRxx5LFh1BHpUbMLe0G5lVmIBDc7jUAsZbq288HDYyBj86El1KuyvcakZHU27gL/ABZ9angutzMJGLEkkgDkVDDpv2csChZWbaQRzip1D2ricwsx+5jrtHTmr0J1Kl1e3EoA8rIU5UnuKzDPdfbCjEojAhiO3eujkXzgihV2/wCrBPYetJPBBBGVZEd+QrEd6amkFmZtjdzTQGKAhiFIEY7n1rSnj3TiO0Voo1RSWxjnvRp8kNvCJRAgkwQ2OtWI7/7SHBTZ0Q+hI9aTqN7FK1iCPTYxE5MzCbIO4GpUshgnz2WPOMHqabM/+jMRwSxBwO/aq6TK6t5k7bi+1lPb2FTe4aDL2wmeEq8qnJG36VTOiTs5WKdDgZOTWzNY+SolibexYJhjnApZLbZKVXO/HzUrtdQKEYl0/IkIdMc0i3NrdERlC27ghauyRNJHsmX59pAwOoqKHR1JLKREkeDnPNAehHIbERnEDIQwwD2PTmrNne2kN/8AZwh5HDEcE+1FwkJlcQgKpGd55JOeasGe0jEaiIGUqDvx371LsUiS6kQOFIOH+6QOvtVURPI/7xfoKtNIJLd5OCyHgH0qvJdrb2zO+S55+gqLX2GyzFaL5OF2kZ6jt7Vagt7eIh92CcYwK56O7ea2eQHajNtUA981oxzzxfLMwBX5Tjp7GmqYJpmhMyuzKr5IbJ9DS/aY5mG9CCOARWPFPPMsuMoQ+C2OtKGYxzxxNsWMhT3J9cU/Zj5jWM8DOoCN5bEj8e1VY9ux0kj+cdQOelRWzXfkCFk3CRfkbpx60S28qCRYmMe5cE57iq5UhXuWXaJZP3yLlRzxmiS6kLbliQbSBx6etZTxXDwoyyF5jwR6Cr0MEbr5aSb3ijLyMe2KmyewczI57sPJFGWCAk/MBVtXZIxkh+mX71kzW5+1jewdFIZdvXntU10/lzSHPlFE6g9TT5A5jRM8sbJkbgzY47iq4uhHq7Iqsm75gTTYZIpoizO55GHI6U17aRo1A+bYeWHJ21MohdssyX3nSMqHnd8x7Yqs5jEwWL5Rjr61CbRiyeTKcMuSfX2qwLZIdhjJHQHPPPpWbhILXQpaVIdsTDIGeKI5pCGCsQcHI96YIzDKQD83cMc4zTprZoUyThs4+X396v2emovQilaTcBC7AjBP1qzFMzxqDHtZfnLHpUTRSi13xJmTurDtV6GDNlHKw+Uqd4Yj5SO1VyxtoNXKLxSszu02xgcjHepI2Kxq/J5+8RTpIpXkC4Hkhs5FSTW8XlqokZSCTuzx9KNARDDGDcNG5Yhh900s0McTLCp27T8y5q3Zxl0DuwRwD+8P8YFZwn+1XEhbbsDdW4zRokDYkwjhtrgrI0YYBuD1NMa4tWt0iidyScsQcc4qPVFWbegbAXHHUE02IW81u6vbCOQIMbe59aE7oXUtWkbmNEHBi6gd6lt3ZhIrhcMDkMKq28rWys53uUBO0DmnxlCHMYcSum47u1JsElY0IRIBGjyqFJzkc4xU0sMSySPG5aXcuCDxtNZ9uBHaSASsHOCrdcVFZ3EaXAd4pAsgwrZwM+4pcxadlqjTs2jN2zMcMe3v9KhuY/MO0OchtwPbrSWsKNPKpY/aApIOfvetQxea8PkKuJjJtB3dqdyAeKOOzePzCwUEkAe9JHa2+pWflIHKx4L+rVait1NzeQtkAsAGXpnHNVLUpZXscX3WZygUZ5Y+9CExIrUWbzmNW2k4wTyPrS2kaQly7HLFFV37Cr8kqw3kaA7hkksw++QKpzxXEjLNEv7rhmXOco3X8RSY9C15sPmywSttYgDg9T6Vdms/KiSKLa3l7dp7nPrWBZ3gS4topUBCnzS7DPC8DNX4L4XUpaMmWQjzPm6YHQD3o0QJpl+3t4Vt53QEhG2vx0J/nVW7lECGWNwrkdO4AFNtr0yQOm8oRIZJMj5evQ1Rup0TUkjuYfMEb5DN05/mKAua1pIr2y3cUhkQYEuTyo9KpreKklw6FyvmDc45Ge1VUCS3FusUmxJMiSNFwoweBxVqaWQBoreIeWsgdgo64PX36CgVzyTxcgj8W6gR0eQScHpuAP8AWrOiDzoWhILDcCOT3/lnAHrzxUHjCML4muNvRlU59TtGf1zT9AmSGWQlSXAyo7duPf0x6fQ17NN3gjWOxW1OEBWIyGVirHtnOT/P/Oaw3HJ+tdPqSsEuIiAArbsNnJJ7fTrz71zkwww4/OrRTIOlJTjSVYhKKKVfvD60gNqwQPJg8Ke+O2e3+e1de5K2pKxgZjyFY4xyeffjj/gXpXL6FGGvUMgJRSHbnoBXSJdF7Rppt20szADI6dMH2P5fjzBaOZ1M4LsATgAbhnjGMfSsUsTJuPXrWpq7DzyVbrng9vb/APV6Vk/dQ+/FMhkdFFFUAVr6YqRRGVyBu6E+ncfkayKtkMW2j7gwD6DApMaNpbqONGBCMx5J2A5+n5VFBeLuZAjMp52AZH+NUktZAoLDgjPB6D3FWrGaCF2jnjWRCeOeVP8AjUGiZqWt0lrcRzJGqkEHy5Bwfzq83ia50+YXscuXk4Ks5bIye31+v1rHitWkm/cEFP7pYA03V2BXayqzKAu9VAzj3/i7c+9Fk1qRI9G0PVo9cinubdPLPlnegPHHf2+n9Oa6PTkBtIUjHS3ErknPpmvDtD1ebRNQE8e7Yw2yJnG5T1FeyWt5bT6OLiAL9l8sOXAxxjp/jXnYikqcrrYwasOswIprr5NuSdrHr061FKslvctHGjvGeEJPK/8A1+9WYZ5ZFN4UEkk+Y1UNtXOOv6frWXHqIkitiWYuy7VPTax/wxXK3ZE3NPzhGpKzOYwC7DacEr1BpY5ZbiCQPFmRRlNx98j9KpPcbNPdFlbLuS+BnCt1z75FT3t4tnfSx3BDW5jRXii7YH3gfx5oTuFyy7NBl5ZS2V2umOCSM5BqNd/lrEEYt5ZMh3YyPQ+nFZy33nE24iIjCnLZzkdPwIFWLTUJPtjRqQ0BdZHJ79sfQ0X1DmHJPK9tHDKh2QEojHBNUkvpp41jSJmVV2tuPAbJ6fhzV2WOIXbOsUg3vnyyencgVHGtortFBcKIH3DDjayk8Zx39KBBYi4uUV5JtoUFd5GMgdBmnvHGtkzRSYudodc9Wyef0FMmnV5IxDHyV2SoOnoT9ajQIY1nPWM8oDnbjtSuFxuovBKEnRmRYyD5eMkvjkVZSS3mkjgVniUZZgO2B/n86rB47m0muTIjx26rK6OvB3Egj6jiqpuIriWaWIOnm/MOOGA7D0p+YmzXs7sKUjIDHy9jCRuOoPy/hTmla111N0m8XGSUTrsA5U/jWPZslxdxtFcRJtlAZpVIwMYwD35pYy8OoGSVVV4nbO7OVJ4BB9Ohp3dho2r26ge4hCoN8bAsT1c4z/Wmu8bwT3FzKY2TJxnlRjoKwLppJJElWZTK8bFeu4YOM+9WrVJL1IoLjdmRiArcFjg8+wwOlPULmhDuF7IZi7RDarsMZOe2KffXUgMdyU3OCVDdiAcAD+tZMZlvZpLWzXAZiVxwcKME9fyqJb0xbwxysjYlRuxz19s0LsFzZMiliN4aNgpK/wC2c8Y9qlmvoyI5IlKl48sdvTHUD3zWY1xbHT1Do22XdJbSA4IcdQfwyPyptrE9xIk007DdGu3OcEe57U1uF+hp2rkwBHZRHJKEaQ9fm6DPp1pyyC3jlgRfN8wny41POM4Bz+v0qpIYXltyn7uFuXjDZG8dD+op8WoGKSWb5VVFMSEjnaBjP5jFMosZkMVtc+dHIdrbkYc7hkdO/rSoWljjMjKkbg4YDGGP9D/Ss7R7g3KkImQFYBJD91wCQQfc4496lknL2drJLvRuFYAcIwzwfz/WjbUVyV3mspDHIiOjyYG0ZAP9PxpZoZLdzKih0YEnbyRjqcelKJdlrIPLYSSFhnZzlRlsfTNJGs4mkkywc7duxCvDAZIHbvkUmroNBdSEZ01n2MUlG2ORR0buf1qO2bMRt4mZJChkfZ14GMc9f/r1MUaSKG0hkd4Hwx3DHO3nA9Ce/tTd2dKmLxjzY1WIkL1XOf06UWHs7lC6tHuYSsbEhG+UfhUciTyxFJwAcCNIy20HoM/yq9p95DFPLbTLGFyAd+TgAHk/Uk/lT7lGnMvIKnLKMg8Kcn9KAtpcoQpHa2AjufMSffhXhGVOOoJp0DNJdC2dpPKXDKoPyht3TnsatT3KTfZn+ZUkHG0c4znd9c8VHAomspWinUOf9YXPB3cBf0FJjTQl2zyXU1x0iJdndRwn4UszTLpgkeTbvVdxTOHI/rUVztX9xdEgAbyq9NhGTg/57Uy9mjvfOELAWzxB9mD8rY44zzQJsZDPN9pjgfdyN6qSME5wMVLCYre9QMVWT5i6Fu+D+dZ7tJEsLM6lWj2EMOnfj65q1viuNPe3uFQyI+ImBBH49z9eaIiTLrXqhpDHCpgZCFAXPplj+Zqst1MiwmGaSMuu5ItxbHphfzGPeozbG1Dyhi0YOTC3KgEA7T/j9K1F+wpcfaYbUqmSSWO4AnB4J6Z9e1V1KTbGnTrOJmElxJI6viWNQVP3cgk/8B6deay7n7ZdXONkfkqo/djoFz39K2LwRXl7I6yhuN7RbcBAAclsdwP51mWkUbapeQZlRZ4PMgJPUBuATnkZ/PFD1B9iP7NKIJCJQZERZ4cHJA6kE/TJqMtdWyh9+2SaXrvPzqBnPB7Zq8sFxE0kTKwEceDkcsnrj2/LFZ7yPdz2wgikW1EbDei5bGB+mf5UJiNELoePntLtm7kE4JorL/sdX+dpZQzckBARn67qKXKPXsF3LviQMMK7ZO09MdjTVZ5ESFY8YG/J5AOakiUzxsGiIiLHGR1OeSfSpkaKIROgypbbnOASKttmavcLmJZ4kcnc6OGAHarSST2ACukbAKcEHJz9KhnTbDI+VR5GXbz97nqKr6g9xbShmJMgGHVR09Dj1o1HsTzvKoZi4YuOgbpTrdnjhbyZS7OMMnXdSLDDDZyNHIS0g3KXHPPr+NZ1pFcmZQsgVN+ARwMdWxSd2wNKN3n+TyN3l5JC8+1QT24kbDPgN8wYA8DFZ82pmx1LzUmYxsSGwcYHYZp010ZFV2lKxNkDA5cU+VobJJLO4ikKQSrsUAncf1poDrIscgYBwSGA6nt+FPMrXEsYgQCNQDjON3vmp4F+3QSzSsN+MRhScKB70rMVh0bMwuBcBllBDbF6YAqvIYp4luBEwcH5ievJ6/hUoiYTK7St5pQorDkGm3Ie0it5pVJi5XcP0NTZjeo+eObAzMojIzGc85qyryjT5JA2+QAIzd6rSTp5BilC7kThuuS3SrkMsdsz26kF3IZdwzmlZgkinHNdADzBnA2gnsfWrqi4W2eWQKyZCnb29OaUlI3kQAtGeWUjvVXz7h7cw+Uwi3ZIUdqF5laIkgcNcRZjClck7uSc1O9uHJYpgBNyOx96r2TE3tvIR8uRjf160TGSZ3DuxYyEELwuM8UcrHGSS1EeJYI3LYYPnYc9CewqtGYC7h337BtAz3xmpBP9otljaPaEkKrJjP41Bb2pE4mjcKUJCyEcN2NVZInmJ7bbEfObYU+Zo4yM4I6/jUzOXtGmP3C2WY042apKbeMKVbkyL/CDj+tSxW5hYxF1ETdQRwcD1o5kCI1u1nEsU6AADfujGMHtmoUkRHIztzgyEdqvxWcSROPMYs6kADue1UZ4bcKrvIGmUAZHTg9DS50GvU07a6GYYpGUY5ibpx71DdXsZjVgRy+3Ge5PJqOWxiMK3JfmUllXrgVFHYx5uMBgrKGdT1H+FU3cdxBaszmeMkOPlPNJa27xSuRKWZs+YAetWbe1H+lYdiq7SN3AxT0hRpTIEORngHsOoqL2GiJ1jFsHjUHy+Qe4Pamz2wmMzFTLHIPvY55H9KtIkFwTFGMR5BHOMnt+VW4omCGJ2PqGH6ijnG7GPNA8cKjLFgSGx344zTobK4csEcpvXHXrxzitQRF5VjkjIB+YsCBgU02rbpWlJUIc5qHJj5bmb9nkgto7dHGVzzU0QE8JSFm25DHHbFaLwRSOTKzMQvaiKMeWzWiBVKZw55NGpTjbqEsMM1srNgMV3FhxgiqkLrsKPKPmHJZvypDcOJFjlTjHGzv3prWsEkwDgqRyNuCSe1LmbILkcTycuR5e0KGBxilaP5Wjk2bU5JB4x0qsJruIsrFMJnrzuPpSFnJ+Zc5QYRepyaWoXJw9usfUquSOejUbwbdWMSjDYOPT1qLVTE8SBmVQWAUYx17U2JP9E5KgqNhPv0xTswuSRzmQMuQVDkL2wKmjtIAC0hGVO9QF6nsKyrdwhCM3zMQBxnir9g8jSsyMzEEqQ3AHpVKPclMGto5Y55nAWUHOzHHPtUctlbSxF0Y8LsKDqPQ0j28kzbQ2HZi0gHZfSp7dXjkkQrvTgrx2/wAKVh3s9h1vbRWe6OSPO8D73PTrzSNJBe6iCWVQyAjPGO1S6gslwYHtwPNyQyk8AYx+Xeq0Nksd7Gs8ahmVdpDZUggkj+VXtoxNsjTT4p7uWNZ2aNDkqBxzxSXlkkUzRu+AcqgHTaOKcyNawSzQMzMYUkAHQFT0P61BJPNd2RuHRsbmdCem3GCuanRCcrkdsZrR4zvUNnBjbuP8KvRSh7tbpfljJAYH1x61mpdbZBPJEjBh5fzDJUjoP1qcF4FSDeCJOCqjOSP8c/pS5iUy/KJEuCS5SMgyYXo2QP8A69ULyLfCWG7zcBlJ/n9c08l45DHINyxg4Kcnof1qK/DRJEv3NwDOv91ew+tHONifa5V8mRkyrJhty5Cbj1+tW7tHtLU/Zw0gKZHy9PXmmzwSWiZkO4M3IIwAo56Vea9SWdMoq5ZRtU8KvY00w1MWC1IcXCKQFUIGzn36VpYCLGYRgwHajL3bOTmnhY2vzb+UC6ugypxjPU+9Pjiitrp473EWCAMNgsexpoEV0klkWSIwkThiuduNw7sfXGCKtTWkItPOIE90ybAx5AGc/wBKkuJHSWKaQqNrtk5B+Xn+tNVmWziLDywwLPkZ5FUkMpSWD2t4iQKUidlQEn+dadvpxihgkUblWUr17DJ/I8VDJrNvMzMSHWJ9rlcYBGM7fXrViZiAt1aSI3yFJEZj06Zx2NCSBHkHxCCjxRmNFQGBOFHHVqzdJIS5Vm+53GM7vb6nFbPxBDvrNlI+MtahcDjkM3+NZGkDdPgICQATnHTv19v0zXrUXeCNY7Ghry/6WZM7hNGW6/7RGPfj/PSuUvBiXOMZ7eldNqkwngikjOVC7CfbCnn8QfrzXNXzZm6DOOfrWhT2KwpDS0lUSJTkGXUe9NqSEZmT60DOr0OHcxIYDIA3McAHkDJ/H9PfNXriJrO1aMEKGUhcnJOTnHt+Pr+UuiQKsSDndJIsWQdp5BwMnjHB/EdhTdakMSGMZLjJbC4JBAPPHXOe3BGako424w75AxwBj0qtJwoq1IBtLAd+melU5DkgUIkZRRRVAS2yeZcxpkDLDrWusEaTOr8rncqgdc/n7cVmWR2zbv7ozn0q0zedI5DFf6+1RLcuJHcskbfKRntg5qoJCHz19qmeNj0GR9OPrVZsZ4polmzaX68LnAxypNXXlNynMzCTHyng5P4YOPx/CuaDspBBIxWnYX5U7HOVPUHvRawXuMuLfyXI2nI65x1/DFdv4P1SGTQZ9MnLExyByu7rHkHp6bs5+ormNQjikiWXbsUjcMoF9sDBOcevtWbZ3ktheRXMJG6Nwyg9D9R6HpWdan7SFjOSuj2S0S4Q2+6URocqqkZC4OOT6EVUfTTDA6xOcmX93t/5ZjHOPz/KrunTxalBFeIztvQMsecAEjP5jgVekgt/sEzLcBTGQEB65Y4HPoMH9K8WzTsZNFFNO8oCCNgsuB8+cqVxgEg9veq11dSmfe6q0RGwMVzv4xgVr6dHE7wxSys8mGQY7lQC4z7gcVWS7t5I45b5yZCwWEdCXLbfzyRVWuFirJCIoYUkLI02X4+8oOAAaAXswpnBSO6O+M5zuXOPw5qLVYna88uN5UdXYEDofnwv5inGxa4W2geFi6gsE/A4/PFJrUVgs73zXMgkMmGdQGbp8vGPT/8AXUs0LIoSaJWlcK0AVsluMnd/OlMEFrBbmzhaOOORMrIc73J5we/f8qbcSuJhdLhzE7DaerjqPx7UWQ3poEG5WlikmO9JA8gHJGAM/nkVbvY4OJWZFlLeWyR/ezjPzfQd6qXK+frM8sCMIpkDEt/Cu0YB/HNJcoUuP3U2WZSAgHcgcE/ypjE1DZbnUIreJmWWXBVMg7Qo4/PNEFi7oJCqxGB/3cz85BH3WHbjpV+6NxE8QjRVEgVZD6MQvJ9uat2qJb29xayOzwoE756YA565NOw0k2UtCtbe+nzM58hVUmTG0E4JyPxwDVW3tTq+tT3RO22DbQS3seAO5GK0UiltPtFurHMiA7SozgtnP5flmrEU/kRqJ40dZg53Rj5hwN3I9cfpRohW6GXa2sTSSrJAoDIY4xjBBzxg9cc1bntzZQS3vL5uRGqocnAPHHXngZogvI/tke6L5hMF6dB2wPp/Omm+a6t57SJxGscuzCjkndkEntwBTT0AiWyazuba8t5ZBIrFo2ZAAp7qfY5/KqTFHtpt0AkZm88nryP4RxzzmhdTY7S6yCRSxYyNuVgDgN9BmoJdSaCdVtVUHI2lvuhs8gduw6+tA2WYikhjsGbejHJKp/qDzhsjkA5waZDZzKRGGMPcr/eJJzk+gHGaj+3iOGMxkxbz0U53HkjnuMitSO8luZr6WF0afLHyh1IJ5x6nP+NIS1KFkZZPOS0PnFSZNr8kqDzj+dQXImjtWkchQLkh5F5K59PzPFS2l4ltL5lmn71OIgFzle/5kVFJOxuGkii/d3DKssJG/wCbJB/HpR6C0SJIt1usczb14TzQpyWBPVffHbvV+3eDVLlop2dJWgZnVSQrN1Gewb+eKginVbF7bzoormRDsWQg+WP7qtnqf0yaz4plFxOPL/dbSn3sHcMAj3+9T2HsbCxTrHJc+YPMjkKhuQGdlOSB6YH8qbaysJRDEXDGQx+Zj73HIIPI71STUM2zEcKJfKQs/wA4xgnPr2HpUkGozQvHfRiKSWKQTMZTjfjDHpzzz+VOwI1ZjLLbvFEcMiiNuMeaPT2qvbqrw3Vrcs8cgDLDuHVdpyfwI7io7PUImt5WupPKimmcMIyG+Y/Nvz6HIq3emSAFJJllRTsQg4G1uQPrnp+NPYpx0uY9gknm3olyZ2XzFbHAVece+c5oubuOG1DoTFNtB2s3XOensRmrNyksEaHCeaR93bnZx2x1J9OelZS3fn2zg22+5gUHMgzgdcYOQOTU2uS9NCSKK5RzdnyzAyeXGhbP3uPw7flUv/Hrf2shkAEihtp+6SfUDjrUdtEDpwN6fIucNc2xY7DIxBwMZ6MRkHgfKfxfbzCQG1ltVSaXY8e8/KgJJ54yO3Sk00FjQNraXENxBLdRh9m1FckMABjGfXp9axrOwRQYnZQiAZ553Z/ya0r+Z7uRxPbw2ktqrbCh5kjySQOxwOQao2bo0twJvNQhQwlA+UjOASfqMelN66Ca1I7r91OrwOjRoCCSvI9Bn061IJENmnnRKZoiCgbGSC3CEdiOx+tEE4h0x3cOtwl1GSE6kBW7fVs/gKHLzqwdIwVIL3IkzucnIJ6/gfWlYRP9rw7xyQh1kDFHkJyV6fmM/pUn2iX7DHDKwjW5jVkAOc4BIAHqcDA+oqhMqGVJSGhiXMZ5JLyFCcc9zj2poJFhGTv+zAfOHHI5+Urnv7H060arcZceaeWeZA4LBVicqpVuMDB9+DzVuSGKC1kdpwWOTbgc7E7p+Z+lZKfaLqZBkEMkfm3AONwGBn1zjGeDyauvpupm2jhaF7XYZVAkB6DJGOO4xz7+4pq40mSQXjW9xIssvmGK2IyOxxgDPoQcj29wafYtHb6fG0hQxnJZXHY85+mMf98msezsb37dL8xEiKS29/ft7D+tJFNO1uqhJCjJ5eccAZ9SMZxgAH09qewrs3BZTTDzfsO7f82Q/Bzz60VnCa3QBftt0u3jC22QPod1FLmZWhShWZGeB3wwUvgHkHvT7tImt2RcqVAaNQe5rMlubqW889YjmThWxnj1q3dSyxWkcEjYdOA3XKnpj6V0XMy/alJLt1lZiydFOSqgDNSu4muxEzvl1xkA7mOeAfwqhaXsrXLwhldVh+cuB07fWnzXLi7DwbI1Q55GdvHTHtzUuRdtC2txKw2ADdjG3GCCp6/zqx9nR0iwQJJtqIx4PPJx71i2k81vK08rBgvzF8EnHPH41r2xkVoioVpAfMXIOQx6cnsKlzsCQxtJgMzTOnmPFzsA4OO9RJHBFK0JwPLfzFXsqnsK6GUPbR7XcR8l3kxnJqpHNaTqPL/eSEbgI1zn1qHNluPQPs9t9jmRFjO3bsLcblPUAVAI47UyoEVVjTITHQYq19vtRbiMRCMqdmCcnPrVO8UBI4pSJJNp83noOuM0JtkvQinmnSSOeCJGEShcYwefWia3kkgWDiSBXL4bjDYz+VLBfxTEGFQsjcHcM7sEdvWopDcyWk4RlIchypGeh5wfStNifQha1aCF5pHGZPn2Dnp0qa4RjHHJG25lHy+3aqd3cJMotx85Xnah+6oqrDqTJBskVjKr7PMPQDnp6+lFhM6m1eOSDcpO50+96GkjuGY7SoLojB1TqDjp78VjWU0oljb5AMbiOmT0BH+e1Ntrm4t5biRGwgZuScnPUH64qHuUmbkJ8/ZO0ihF3GPj5sLjk/nUhSG0+Ri2XIdt3Y45NZ9nqn2oH7NH5URGJA3JY5PPtzU8l7EHVJPnDKFJkOTnsc/lUOQ+hBC0YtZ4zGz5GVUk9+eBRb2O65iMsjKiocIB1Y/WnLOiTJlWjZSAAM4x3Oa0bhjHbSGKQEh92W53Htj0qWwVrGbHaTTXTCd/LIYZRTj6fXtUrlzI22JwFJXEp5DdPypup/a1h80HcAdrEY57ZA60RSXbIBIjkBzubGeR3pNXDYkWSYLbbpArxg/IowOnrWbN/pG9cE+c/O0dD/nmtO4gSRVmc/OykoAeWPqKihhkhMciYOxuOcg8+v501oJ6kMZb7VDDISdqNyOmB/8Arq+tzHJboUjIByGXqTg9T61LcWkeEwSpYnBUZJGe9MsNPaOKWJm3sQVYnPXHrVJhZ3KxugfNjGADtyD/ABVc82FrdF2ADOSB16VTgtDARd/dUgoEbkhjx+XNWZYw83lhcSFQpAbkHvUtjVy/Elubfe/zPjcCp4Bx3/SqLXSRM8eWLOMIB0+lQQbrVhDGxZJfuSjsOTnHqKsNpNxdzxTQxvNED8kmdoLAc8d6Vws2Mkv54LMkKyOVO75PukH7pq8syzWxMblmlwCOvQdqxLyWfUTtllZkwACrdACMn3z05qRb02dlBM0EnnAsjptPAPQn0707ju7F+2uJAo3hkkycN/dPv+lNgW6MrvMN0xJASPoR6/59ajhMpuoUkjUxOMPsI59wf6VNNIlrLblnB3NwzcFRzwaa1Vx201KzRNbndI7jqBgc57daRUM3lYyHAxv9/b8a03Et0ibVjMbcEucYJGeDUZecafFHLKQpITOwAqc9/wAqGkgsV1tpRB5kbHZtbeGOTn6Use6NoHlym5uQnVRjNWkmRZUZmymQHMY79tvbPsaHuI4ZZblwrttIGcHr0pXG7FWa1jYu9yT84DhW6Ejp9M0OQXuliQLJgD5hwQAOnvWl9niuLS2NxuTI2q+cjHv+NRCJY3BV4g7fMwk4AA/l0/Slcbj1MyG1bfG7YWVhnYevp1p0t6torRSqEjLbNx65ByMValeWOQeZEZdoZ8x8+xP0705bIlY0nTfMCQ2TlSpI/Oi5nYs2qx3MTOxjRkfkqc5U8U2W6iSN4dwEqhhheq/X/Cmx3Ef2uWFYQskOQwLnaAejDpkZyKn+whJXeNg8Izg9z6+/Wncpozdr+ZHDCzSoCcMw2ng9TUk0vmK8MYVZZGyBkYwCOR6d6WC4t7m3W4hST7QyFcHOGJPahPKUJGI33FMiYc/UfWp3FYnQW0qIr5QBgAQcjAPGcUl5psZiaCMHAQ/LnAfuSPx/lRZokCMYn2qiFnjyAAAcFjn6inzTeQUKTI6liD8p4Hpx9RVW0DTqZktmlkkZRJJA7ozAHO3A5J9scUsluI0jeLJAOF+bGQc5OPpVi9YLbysJNse1NwyTuOPu/qT+FVRdRSR29o0u5U2rvzn19ulSFkWrCG2trSNn3PuDEbgRlcYH61VuleVHEjARBeSPvHHP9BUxYNJMkRL+SuFUHjr0z+f5U03UV5dlI412kYbccADOMgn8aTDQl1KOSeSCRFbytibiSME+n5A1C1gHcXDzu45TI6EYOK07m3huLhoTOywxRgqh/iPTGf61BKPs9m6K4Cu4yBkkYGM/nVNXHZXII7iR4URiomBIdzxnaeOexxTZog90FEwEgzIqkhiwHPNRWqxRRAtcCBTIWcnnJ55ANVL+S5tLoPtfyzwo25ABHXA5XOOlUibaFiSeOcGBUAWQ7lIxuU4J6exBqvZa48YQagoYNmKPI9c5yPz5qna2dzeah57PCiRRggY2kg8EfkM1b1OKWKSeSIhre2I2b1ALAjk+3BNbJka7lzSdIjTTL+5dzuuZWEEJb5VJGc59f5VWieTyLi5O5lJ23KZ5HTbt/XnvU/nuLG3tpECzxwxbkXPXueP51Da20yRTrGSVcbODxwcjPfvRdXG2cR8QElW+s2k2/cdUKnnaCMZ9+axNJLecShOTwMfz/Cuo+JMBWPSLopgyCUF8feIKgj04IIrmtDG+72nvx1xx/X/OeBXo0PgRtDYlulaOe5ikcCTBYEHqQckDtgDNc5cnMmTwe9dfr8caPbzxnZu3xsueBnKkfr/OuNmOW561qi2R0UUVQhKmtP8Aj6j+tQ1b02PztSt4x1dwv50Ad5aW/wDo8UY6wlG46sw5/wDZsDsMjPArO19x9pdlYHoCQO44z9cjOD6+9b+n7pFVZF3CUmcsMFkB4H44xx1/DmuY1f8AcyyIOp5KkY45GMd8dM+xqUUzAueGxjtxjpVNuXarEhy+SRzzUCqQ34GmiWR0UtFMCSJS27FaMamOBlXknk1mRuUPFXWuAYgpG0gdqhlJleQtv/iqJsmpGbJ5PNNyOnFNCZFilBKnINKRj6U2qEa+n6gqQyQuXBYYADEL9eCOf8fwqjNIrTEqCAT3JP8AOq469cVNDFJdXKRRjc7sFA96TEz1XwfeRw6Lp8Uyl23b1C8Y5PU/QVs20ky+dG8pkhLIgO3AU84/Gq2k2EMOkzM4CxRwxiB3GedpH1xlf1FdBbzxWmjS2zRsUuU82YYUgAJuz6g8DrzwcV4k9ZtmVr6mVqSJb61apaXRUQR+ZsY9ySzZHYlcGoHWa9vZJHTL+d5sBwOcAlePqf0qS9llura4kktD59ywhSUqFCbgRnA5yAM/gadZ3cdjc2+6FgdyNcbl2gA9cd84zj6j0NJWE0Onu3dfPe0y0uCzYyUPU/SoBBL/AGqlwt00ksOTJGPlMSgEjr1A/rVywiS81ieFo5BJO3lMAflDEZT+Qz9TUFtJcXeobWgETRoySA5wUGM5OcknAAHv71KD1B32xyEMdskpIjHsxIyfoM097iJY45I02xgFXiK8qwJBP6GorhMQbjH5jvJ0T+8Rzx6cdO9X7oGQQ6fjbLDiIyFcEkZzkevX86fQehX1AJ9meW2YLAkm3AGcqMEAn/gRFULgskIyGjaOQqzKc53N6+w4/lXSrpn2C3h8yaO4Eq7vIwQFU4wx75xk1iRiDfbQSu7tI7BiAOGBPUY59B05xTYNO5oxCe6JmVJNkMeOU/uLxn8D+lQXE0cAmd52iE0SuAvTJBxgev8ASlglurWOaLzTJGvyl8kRsHUEZxnr7VIbKOKylBBd9oKyTDG7a2Mckjpg9qClFsrXM8kdw0jOpPlofMGMKpHb8OKlS3uP3zrtkxjbHnBGT19/TP8AhWVeXAksgkiBvNjJiVDgMxPT1yMHpU+66ifzBOzxFMb1BIPA+Tnpnk/lT6k9SrNJIkphuCxKYAdeCcZGMjvx+hNaMyW8NsFtppjcMBvd1ym0Hhg3U9eh5wDVOeT99HdK8MrgEYk/gKnJbjr3xj1FS2+pyXmlvcwbYrlHMpVo92FIB5zyRn/0L8mkJW6mQzpmTIG6WFgz5+Ugrxj8TTIrWW7slkaTzmt0AdAOQgI/xx69K0rezurAJNeKoEbPbyidxtdS5GcE9MEg47Yqe+s45YTfQmNYwQFSKQMzAAAsMckcjB9qWq2Dl0GWOlxX9mvluieQyND13EYPy+hyOfX5fTNQ3RitHurpAgnjVn+Vs5IOABgkHjP5dKTTp1jvmEsTrcGQBZA3D88kY6jpzVpbJZ2vGjuPMRpd6jcUUsCc7jg8Bcjgg5I9aNxpXRhzuqz+SwaJJSVzGepCjnOfpU1tFcmxm8kldgBkcenQd+vK0l1pkMIgQMRLK2xgfuqcchcdfTPvjtTL65lkvGjSF4pYdohB+U5AB5wOcljj6cdqpbaE6bMt3Mcb2qTM5QRyKcLj938oJ6ev9cdazriRlS1ljl3Fi3lpH8xwGXBJx7n8q6F4YnaVboKYcomFl2kktnrjnHf0BH1FCMF7mYRpsjjJ2tyrR7VzjJGDwPqcVKY7GdHNKl0EWM+WPnTjOSec57DrVmPSpdQmu3gbEEQV7ho03BIiM5AJBPGeB74zVS4nkW1guSPLSXdhQM4IJyOeozjn61NFqDuLkWkhglAEzKfmGQe+e2MdPT86QlbqWoFmtlQBoY2jCyq6DAI2nt378fWrc0st6scMpMlsyHcpY4CsueAB6fkRjvSyW9rJp1rd+WslmQUnKsdyAj7xIHyjLAjGQOnesy0ke01COUXDIIfmjTPTlc5PpgMcHoam422tC1Lfyz7jEymOFCUyMO+Ohx3xyD9KvabLa+YoFsss0kSxiM52yFsAsOR1XB69jUGp6e0zJLbvOVRWcRAjfFltxBP8QySc+gPPSpILW3jitZYnFsBJ++uFkLbeeu0dhuPTHQc5oYJO9wvls2uJHitQ628e3zS5+6rY45I6dB6HrTLdLZzFLaNOQww4KcBz26dRkf0pL4QyKyxRSRQlNwTksVJyWYjjPByOvB7Yp8FndpdeVDL9pheKSQFWZVCgrtPPRunUE4X1NGvQPiZHame2WOdmkcxYkAcg71TGQPUY6+1TabPbX0l0btvLUKoWVYyYvUJjoDljx7Y61bvdKSJcLqds7C3kjDuxVAyryAuOh6Z9TxWJY63HHLerELi1troOWt4pDsKnIAOTyAD/ADqkrBs9R2mWr3Guy201xAoiVCnmkpu5CrjPXk/kM1c1O2jsb+3hYRRLHvikER+WRBI/zZ9geOPTpisu4mFvdNMhyJYCssO0HcV3ZI7ccEZ5GfWn+RGdEjCrj9y2845YkADOOgGM9PcnNDsLTZGiq3k1msU8RkC3C3AdkbrsHJ/PAP4dafFO9/bTRyywtcCQpI8TblVm/jOM5wx6DpyOTVNbya41OdWAdAiFMqQqkqGABweSCCpOM/jUCSG7t/Mia4aKVwWj2DcFxlTuI+XOD9fxApNMq+hcv1TSY/mu0uTFJiSNAcybhz9AcjP0Ptl9v9peSF5bwzXOzI80s67mwSuCcjACgd6yLuW1t75ZIQpKMC5kIl3tg5JzyOoHXqO+eJ5YxDeRRiD7HFIplxjp2I49cg/QkUaW0Fe2qNa1xbSy3kwitIIiQXOSueeCTjJyBkDn9KoxytcXLWjKs2xW8jcdq4wx+6Op5J/L3rSi1pZ7SNpbaKTTSdrwyp1VQeR25JHOcjHQ9Sltc2MMiIIWPmnamY1GwYPDZ4yeP19sj02GYgt7aQB3guC7csUVME+3PSitl9D1d3Z47RPLYkr8qnjtz3oqedk8o0aYpliUqRFt4A/h/wAaj/sW1KqHRmRiR5eM9+Kf/aDW4Bcr5PynIGASeQB7VetdVhu0ikUo2xA3GAR1Bz9MUXZSSMkeH7RSX8lU+ZWOD29PxqV9MCKk0c5jHzRnYAG2njBqW7u2+0B0G7Dn92PTI5/wp+o3nmYiEGEzhW3ZJPGM470m2Oy3CGxj0+18iLNxjAy4zge1XFV/viJijYO0YyDn+VUYb02V4sbFTIpyx5JAx0P0OBVjcZSHheQzM+QwHG3sMHnNJsE0xqyx3c8o87EmMdPlz3BHrWZa2t3pepsWQRqBlXU9fcD0qw9vKruzTHe7l/MXtyM/oKnlMDuYSd2Od2SMcdvWrUrIUtQMcY0yUTJFNIZMogwCAcc59KLTyzJMWiDh+BuPIPT9Kj2Rz27rnkD5hzlsHI/HpTpFEdvHEo3hlyWRtxDZ/wAg/SlzNBuMXTFa5kWFGlJUOGV9pVsc4z17/nVZLSea5TbGIoyo8wjPUD+ZrYRZHiVSirMY9uQeOQf8KLe3ZZwZJnkYkK0SqPnGOvsQKbkxpaGPHprNf+TZRMkUh6Yzkge/bNWxpVtcTys4aMSKXZVAOCc8D05rQm+R5ZYBiKNAwDEZA/un375FNjWQxhpGAYsGARsgHPf8/wBKlyYJFOQRR6hvCLlc4AXO3A/x/nUX2VJUmljxlpAQvtjJ/EVrQSCK4kQR/vixGSM7lHOR61TEotgxYBZnkbaei+mR/ntSuw5RrW0Itgyjy9wKg45yMZOKihtHctFIuGDAIcckYrReeM27Js4UH92/TsOPfp+dAZHCugfe5zuyBtxn9c0miuVFeCApalYkMyM23B5JIHJx+NMFi0MQKkKpzIfMByOR8p9s4/CrEWoy2ryTPGowwZWUdQeCf0q7LdfaJjIY1MahQQTknI7j8qYKyKFwX+zbmUkEBCFA4Pv7UR3GNQbzWGyWPsec844/Cppbrzp1tlVEhYB0O35jnnFNjs7aR45JwTImXRFfnIPXn27UluO2pTupJXhijK8cYJH3cdD/APWqSGxu12yxj93AfmVh/reOcY/Cpr2xt91u8N0Gd5GcBx8uByvP86lnmmS1imkZS6gqSoIGDjkD07UWsTbuR6bK0d4/2g5icgrgn5R1x+Qqw94ipI3zTxE5Vc8hfWoY2MtuqRKGK9ZNnHJ5z+FEEUSSt8wZsBIArAZyec/rS1DZWGPILhlhKJEIxuK7jknqDmoZUjlvJF3sn3drqcbcgZ/Sqcsrki2sI/NeUrtIBBbOMgeuOlaMhgacqpC7ZVPODgFOadmK6ZBHmO8RLdiIw+RIOuc4/wAafJNPFMj8nY5MaluM5HQVNBGdPiliZQs8hOx2ONoBycfU8UNH5kEUojLZYE7ecE8g0WY9irbwb5op2TawLbwzfeHp9avSmN7JvlJb7pLdz6+xwBUsMQ85pjtWIrtBAHU9cfzzVa9nEdxOqSnPllIyRgFiPT16DH1osF7bEVhZjyZ0yEWNSWYc9+anmsbqWCNHETI0isSvOFH8h1qrIs1mjSRBnLqPMXaG75JwfSraSSNFE8QKHI3lgfmPcjPQHFC2HzdDRmuk06K4dpSXVR+72nBP8OO3Y9KzJ7jyw4kbelwokTAOAe/4003LEtIZiAnCq3G4Dv7U63hjvWDyu8UcAY7lX1PHHf8Axpbic76D0aPYxIYBmUqW5+bGKS4hAtjIZMSyg5U9jyRge+Ka3keWgldmihbGE+Useck/Qfyp8xJdS8SFezKTxt7/AJUWJuU5J5ftbKole3hCu7AHDNwv4dP0rSiR5ljQbVYsZMv1VRkLn6/0rMtZsSyIshO8q20tzjJwSP1p8Ltaia6nlR5pJDtD/wASgeh9f60JCUi9ayLdSyDzRhvl3qMFlB9B68U/Ihlkkj3lgu5lVuAB0H55rKtpLeK8Ee5CvJEin5eT909+oPHsKmdJERrZzGfPiD7lYjKkcc+gwKofM7FuSSG9tFu3TzQrCDKtyMndj+Z/CpmmY2sCqBEUY7X/ALyqep/MisSwd0uGiV1ZZMklc4xnHGeTwOKlW6drIxBowYLjYSM8AkkD6ZOM+1NC5i/BHdRNDPbMixlyFXOCo53Hn8qT7QJQPJiEYkZHMgYnPPPHbis8X0kccNv84m3SfMFHzgtnGc+xpyyqFhjVyrsmMPkfPyTnHQAYP0pbDUi1c2yXd60QLhI13sNpzgk7SaaInS5mmlcMqgiReCQCRuf8vpUVtBm2clz5kbJJuJ5YEcg+w4/CqqyGS2il+0TRyuSkjggbufmHuMdKOoKxobI0EbEnyGKhi/DNngnA9hj8arW9qby9luS0bKfliVcKpHcH8KYJJ/Of93mBY1EfOS5AHftkYqvFdTqZ3RFMKkg5AAVu4yPSjqJsu2hNtItvEp2MR8uCNmBz/OpwIBEq7VMQGSByzc8j261TMksj+ZLIIeqlcZbrnnnjjnpVm2tXnkvGmlSKGHKQ7kP7zdxtJ7GpBX2LDW65gmdUmSJzlCeRjtg9RnBz7UsN0LlS7R/ufNIO4cYI5APqOKoIA96Gnx59yNjFs8YI3DHfpj8DWn9rLzJF5CrDHGW8xBlipJ5PuMUFMbNb28dskUMaujcsSBnZjGATzzmkuUiFxG4kciIYQIfnPy9AfoP1qxPcWsUisgWIqwCnAwRxz7VXNnbz2i6gdSjjuWbLRkZIAORxnv6j2qrC9CrYRu9ksyr5XmyNDJ5i5IPAH4daLexRluJ4i8kzJgD7xGOoJ9vpTpp00yGR0F0TLy4Qg7wT1APGeBVS0n+xz77fzFhExbCLgDo2WHqDn8qVybltvKRxPFiUrtBDdNqgcfXPNNt7qNZ5SkrxB1JUvztG75gf8Khlv0eyPmAiMStI6MMAjJ5B7c449q4jU/F63TskcJVjkO+eeuQRWtKnOp8Ib7F34jXMkun6fE8AUJO5R1GAylVH9CfxrldEkjiui0n3MYYc9Dxn8zU3iDXp9VsLS3mRR5LFlI9SAPy46e9Z9guQzkqAM/hx/wDXr1aMXGPLI2gmlqb2u7biwiniQhzJlgR06H8+Qf19a4u4BWZlPY4rv7GAalp6K/yAEE4z2B/H8fp6CuL1qJodXuFZQpLbsAYHIzwK1W5bKFFJS0xCVoaIjPrFuq5BLYyO2Ris+ui8HW4m1kyFWYRRlsAZye1DBHXzyw/bAikiNegU8qMDIHqB+X6VyGrXRkupmZt+WPzHqfeuylWK2jMvmOXdWyzgBWOW7HqOMdun1x5/ekGTIORjn1/H3qeg2UyctUiR9SewIpkYDSYPcHH1xVpQFjB9RzQQygwAA9fSmtVl06kDOO4qqetNDQlLmkpaYxd3vinBjnjn2poxjoD+PNPVC33AxbHYd6QASVbleR2IqMnJz61IJ2HZT9VBpjuZGLEKCf7qgD8hTASuv8B6ZLcX9xqKxb47RMAnoHYED+RrlrS1mvbqK2t42kmlcIiKMlmJwAK9q8PabP4ds4NPjCxvExeVmTlpjwQTjptGMGuXFVOWFurJlbYu2VsFtZbeWRmN0zeREpAYRrnJOegJB69RipFijuoJriZzEYSvoA/B6j05XI5IxSTWdm0y3yKVmfcsjL/DjB4B4GN/Ax0/OpoIIbWaea6nJDxgqi/KqDIzjvnDDJNeUDa2K0UUc6FIIy1sigGGQlXYkc5Pfg8c9uadcw3VvcyrZym4dokWacrgRhc8D8D1zzg+tKkc10r+V/o8BkVEuZV3g5PJbHGPmPI6Cp4LVTHd2sLs9wsQjeVeDjkheR1Bz165pE30sVrRozDPFbOpMEx6KPmP3uCeQdxzxjgYollLRyyI2+ZkdVwcbZOT19STnt90Vbt7Yi0W6mhIm6NIxBDZIAIA7Db1+vXFVre28m8ljnhTyo0ZxON3BGTn13bcnHPQUxWuWSJWnluFQLBh1dRzltw6/gCf/rVXv5rjTITNKVS4LoGWQ7s7h8pBGehH+eaW3W7nlhXcpWGRQcRkpIMgdO5KlvpgZxVKaBmuFuJV/cuoVBg7uuOTzzkgY78+5ovYk07dp72KO5lhA3RmVZEQ7RlSeD3B3Y71Vlkgtby5BJMpV5AeoLcjA9MgDn3NNjsvLcwNdyRW6MxZFcsAh6Lt7Dnp9PSrMkUdxFHPONrfMy7UySvHAPAOd2c9OPQUNhuUWkke3dZXkbcsdtsYD7gXapwvGQBj8KfNeTS2/lBHJA3JIillJUjpxnOAeevB9aSKzjkRzLnz3YylPmAVj91SR16j057irkUqJDA8dvIWkjVQ7PtVAzcqU9QcjJz3oGnJnJ6w0sUKzYCwiQMk0b7grMScf99Z6Yz0xV6G4lltfsEKjzWxLHubg7eg9Qew9q1ZDbaZpVvZGzFzCHIl3EpjjPzHpgMcr7ntVOS13X8EUdtEqtNuinljy4YhzsZeDggckDrg+mW2O3Uosvk3kLSq8uFBLx8fJ8oOOuCen1FWViMV7FPGHtQE8lvtDgLMw3Ej8QB9TW2+ii4hgKXNuLqN45QrghZw6rlVbpx0we+frUF3aJb6cdPwGZnVrdxhhwBncR7np/tfWhuwuWyZk2ls93BEJ/30MhO1wfmUEkHCnsOvqOa0oFhEI4aeM5Dxy5DMo28t6HLHp71FCZ7a9ulvGPzN8yOeFPU9MA5wfyGKPLjOoTBSz+Q28I57EI3Htuz+IqeZjTsXrixtYrqS5VBbQJDk+Wu5ULfKzk9ePTI6+5NZEkixvK0cnmRxzMEAO9ZTu++noP8APrV2/uLN3VVQzxxowUDJQPjLE7fXjjB4JOOapxW9pFZLcIhKvIItqH+M5OATjIyw45PP4mr3Dd6Er2geWF0y5iBAbPXkctu9s9O3pWi0Q1G2Au3V4opgzTtt3rv5cgkY4I3YP+FZ8toot5EiN2XlKzKjuCWySeccDH9Tz0rQ02yeytZopY5du9ZoJgdvTIHPTBI6fl3wXG1roYlnYRW1xLcWubu2STmZSpCpz/ED3yoyBgnOfSn3KqJt6XSHcQsyqSfvArgj/ZHP4e/GrdWi3FgmZFwikSJEwXO8Z3HjG0sRnPY8dTVa+sD5hNouSWYCOJmcpkkjIPGARnqePSncLPlsjKvtMM9j9odmeOP5kRv4B8zNnsOVI4/vU7R9KuUMLtDFuYAtHhh5g3EcjOMgnpWul/EsQsJBKXDENtAbcmcg56jgMO5GastfmeB3O2NoyAAq5zx0yT6/oM9+FzMnS5iX8MujQRWyQRSxxnEu8GQMA2QhPZRnt6nniodHsJFs/tLRcYYtG4DKQTwR+AP48/TTu4LO2uppVsYvLf5n2kswLL5mWKnGznHOenAHaGzgiQSxRx/aEWb9y7ucKmACrY9QVx7+vOW2hW1FillmsovLliOXKOuGQHIIKqfwxwcDHtipo7Wye0nMxBtZIiWVGAY5OARkEAjOemfmz1qcwxxW6FHYMmFVHUBHyMHn1Hy9exPFVBPLDfyXCqpST+B3G0NuAbB7Asx/MdRU7MYyERzTtGrBDMgVixOY2HJyM9G6dyN3tVgWlpJqccMxjs4Y0kbzUKkscbsNngHOfwYdcYqtdsYpFlhjTyQilXY4LHPUg4J/h9eBjNJqLW5kilNuqecgQIWLptAwD1OeT9flbuDiosSdiSPTYDPbz2k6y7kDTi4UfKPlAwvQgbs4wOmTwKpyaUITLbBkdmOH8zDkfeBC9+/6ZrTspyVcJG7yjc4kkwDglgTs42jhhjvkY9qt3eb7iC2kMmElMjhCfkjXG4qvRjjHPByenOA9QuiM6V9q86YMwQXCM8zkEQjYFBCjCk54zgZHbioreF55SLQtDLAzGNGO4YPPGT0xkEHqCevIqy1u8tv/AKCzOVGRJcKQJBnOARkcq2c9qqzTXBvJp96JdmVXZVlbC/KN2ME9CScnPO3rmlqK+pcivFS2t7VJWjeNVIiUnYo5IPrgZIHtj1qnqejzXiT3aPNFLJ944PlrtO0AY9yOp6Hj3sSzM15IyGJncKBEh2oN+PlC5+U5yOnGMcVe0WE30E9leu6ooiYOpIDLnnP+0AOnHPOOMUK4730Mqz02DSb6J2Z2SSEgBOGLBgCOejfXjnPGObV6HvL8G4luYEldWy8e5VI5+U45HGeMjnqaiv5Zo50iklWZzI0illJLjaMkjsSMZ7444otZ7RIrlZWnuBIGCRgH92WxhwOhwQq4Bxkg445Svew0+gxPKEsk0M0pSRlZZA235GwQwGcg539B2HpU81mr2sEU7ytJMxVmyAu/cu1+TyCpx3PHaodIa3SKEXFlG23evk5YI2GYHp03dD06LxnFXbjxDcNpi28vmRTbg8M8bBdoAz5TFflJIycHk4Oc5zTtqCWjGYuYf3S6bDIqfKHMhBYDv93vRVKZ75p5CkpKliQd4GRn3Ofz5opWRJUltxNLNndNFtU89WVRgED15xVn7CkEJS1BEYHmKFPJ4zyO+M/pV9XjWJjwyouBmMZzzz75/wAav2zHy4y8MUU0xbcAeDwMYPb7uaq1y+VW3MlA7QGVo8M+eoOeKzzFcedHMsSCIjcCGzk4J457kAV0U3zxzxC5WNmj8z51OMYywHvjJGeCR2qBrZbOwDSMjHDp8ozs25JI+uBSsS0UIw3nm7ePyj5QAQ9gDnPPXtSBLqNlaeYB2YgKckEHJGMY5/wqeS7jW3jm2KwCBvnAIJxx7dB/KkW2F7D9puSYxGPNco3uORnqOo/KkhaASsi+SV2eWVfzV+UcZ4A/DmnLYM0MsauVmRuHZic9Sc+nA/Ws03+LoXEa4RQV2E4JXuT75OMVZudRKyxqihhIAXQLjf22fXA/M0xqS6mvYR7Y4kkZTuHzP0wQTgfnUJ8qxRnt/ldWGRj7/wA3Oc+2fwqlHePJp8lxM3lxIzNE+MM6g9x6Anj159KkdZGjisWgiKTRN+8f7zDK4OOoJ3D8faizuO9ti/d3HkzKYthj4P3Pu569OwP6Glf57qONvlmcCNJCchjxxu6DP8jVKfTZT9ntUmkZBt2SFyXGSeSR2wAAPapUZoHURxRvD5bDcxJyVPJJHBHp7iizGn3Ji4Mcyyx5cTjcAxb5eAP1oiS7/tK2jt9/lFlTCkZ6EgZPTkCqNvcXMpbzLVmjGCJR1wOfm9Rjn8BWvON3lzhfMzM+XyFAGMdBz7fhSsCfUlaJoYfPuPKU7zGiK2WUen4VgXBa5YRTvko2SAOgAPTHc/1qbfcRXIMmXVCw2xpkIOCvGfTBND3MMVxutSgViVcsnPy5GSPqSfyp2CTT2JpW+zaeUEwaYSiNnB9/07HNWbVLoQJMywYkc7FYgkgHBGPf39DULXMFvEQlqjCYurzONxcAY4Hp7+/FU5Eu0uLWPHlHzi+xTgKqkgdemc/kKfKgvY0JUikDxSxogMwSZHBAVeqn6cUxLx5YwZCYY95VgqcjBGDz04NSIGVbgyxSBSDCzhdwBBBXdj9Pxqpd2yxoyQ3EpjkR8l1AUnGCDz6ZPrn16UtAbYyO7kuX+xx3CllkLI4BUEEccHpjp1qa3a4ufMEjyLKvykKBhRjnPoScYz60sNsYbeOUiKQsqq8m8YLYO0eo44yRTZI5o7DDhxK8uI2YckoeFJ/A/gVosK+uoiiSaaNnAFtgfMoyAfl5/EnBFW5mInlE5XEW0bHBA29QAfrTLC9tk0g2rCROXSZs5aQkHJ/n9KIhFc28AuA0hTcrNDg7VHueScN0NJ+Q9CJdQCyz+W5dA4G0cYOOO9RwzJJuSWEyCYBIxkl1JODtI7gknH0pv9mFLkQwnzDKGTOApAHJJPsdv506O0QaWzrCzOS7xluue+COnT+XrUk6kI820ns4AjoIHKozjDAEn9c/lVOKbzdQCnCeacFgcgMOvJ9PStmOW7u7eZp2UrbLt8uRvmkJ64HXIC+/b8WnT4LZAzKAyMzYZdu04wc/iAQeOhp3DlIZ9TmvJWSULlA6NJwRkHGN2cDPzH6mqdnqciwygSbhHGUIbqWBxx68EY/GrMscUmJ1QSLOR5BZdu3DAYx0xjt36VTt57EKZo9OaNvMDSTNMSjMMLwmOAS2f/rUITTN2UzLYXKyTRNIqHARsnrjgjjIHH4CopRvnkeECQ4AnQr1B7+hIwffim/Z2vCq34kSCbdJJcxrtfpjyzzgqflH1qW4K2+oPNZqZLUKQ+RtLEdPcEd/Xgih66lqNyu8y21yPNlkOZCmWGBjcMgH6Gk/tGeQX9upcQMNit/usuSPxI/KnXt0kz5mCsBhgUGRjkgY79On4d6kuNLnFnbFExBKw8xY2zszg4bjrnjH1pLYlp30KK3T3b3EckYbZuZiwI3NgnAHfjFWI3jnsbogtGyhJEG7k8gkDB9RU9tFfTy20dswQwbSoxnKheTjpzz+XNRwaQrQvE9zPDINwDeUCueoxz+BHbPHSi2orOxnSmV54n8k/vB5MwDDGTjn6Zbr61faZRZwwvvMayKC2Qc5XgEg5/vCrSadLE7tLFlVbYQGwrHBO4evUj+lQw6c0SyyxqWt5MMkbHlW3HgdyMNgN3HHamgUWRKomshqCYWXfsBHACqcAkH1zio5zHc+XarInmrHuVSThhnPUe/vVi5sP9AisFUiKVG+VR8zeh9Tg4P41NZWtu9r5l8srXWMBnYZ2gbd2R1/+t2ougscyhlhuZWlt3AXKiNhgnnIIP1wfoTWilw6pbRmNm8hmUq3RUK/MAM84K/qatvaQPZW8U0jKjJI28N8ycnB+mOg/wAiC2tsXkb3RjYlmO+NuBnK59PejmFaxZvLaeOF43ZCEj3SKARuAUk4YdTyuV+lULcXP+kotvIw3oX4x5a7SDwcHHTpxVx5JLNp7eV2d1EYLOMnBJDY9OOMj1zTpjtt4XeXaqk72VMtxyCWzyO3OOBR0K3Zhy2skFy8lvau91By8qEnKqctkH7wAB5GehrVaCd7vyVYxT7WQhl5zgnJ/Dgn0IrWgsbNY2nRCXYpHIPlk+993bnnPPAXnnnIqpFcFNsa5kS2TzEye5wG2g88f0/ChtsbiiK1uFW8hjuIoVeLLM245dgCBnPOcADGKqXdu6acQiMjq4cYHzMR90AY7jH9auzOHj3CNGt3lMkORja4yMtgctjkHpj8apJdHUY74SuYJYQ4jwQQW2g4JPTgEjknIPpSVxMtqIFsFaSVs+cZJPJZdvlsu7IyQAcA8cdPpTpIGSG2a0ixJMgbfdcLvGSxyOfurkfhXLxDzWitrq5EdsoxGSTtdc9BnnGG710Npe29ra3VnJbb2BRoYpFZsEHcAOcexPoDVMFJMu3D2dzdQG0yUId3ZSSu4EYwDzx05Peg5WCaJHKtG+FdcFlcAkAE8dM9elZkk8VvttoEEVndbw0uDuUbsBW7fdbJ6denFW9UuEguIY1gjmWQ+bLCOGC8qWB7cH6jP4hIOYrxGa0EMnkveTJK+I4WO75sA/KD/nk1rXMTJKm4M3nqylCudmB6jvnrn3rMsIwLu2uJAYyC8jRY+8SGAwTnAJUkZPQ578Wry/t991YNFJayqCyuZASpxzuYD0A465J6nihgth+prcwWM1ulrmYSh8ufurnt7fIR/wDrqnBvmU+X5nlMCFYKCAxOBjPfp7VPfXWLfZYNG7TSFljb5hyN2OemQuPw7ZqBYprhIfNjdXkJGYzmUrgYHA+boepycr3NTcN2TXMssAV5UUbI33rgqQwI3ADp0B9O3c0tpPDeW1zCgEqRx4JyTjkgfMPUkcD1qW7hlngNreR+YqMYpJAQCCd3y7u5/ukgEgDPAOa+l2yW0M1vKsMxZVW3ljYKVK8nOAOMDtkc+pp2TK5bMpahpralY3NvZSOiOQdxXCYIyMjjDYPXvz0PXySdJLW5kikJVkYqevUHFew6xPeXE6Q21wDNIkkcpjB2KeCCcjO7BPUc4PPFc3ceDpPElut3BcwW97IxHkznAkcMquc9EGTu59cV24SpyaPYIrU4AtuTOT1q9p6OUcowXII46/55FVLq2ezHlyFGO7gocgjAwQe4OeKu6ZG0pRFY7ienp7/yr0U76mqNWC/Nt8hysWenXHf/ACKx/EU8N5Lb3UW7cybJMjqw7/r+lT6itxby7TyTlSAO/Of5/wAqy2y1rJE3LKdy+2Oo/Ln8KBlEUpoKkAEggHpx1pKoQV3XhfTzaacZJlYNcbZGXIBKfwjn1zn3/OuMs40lu4klz5ZbLY9Bya72yvGuJnZQBj+AAALx0AGOxx+XrwmNE2pTP++c9djJjLA9A3Xp3xz17elcFMxZmOcjOetdjdSAJcysWGI2C7QCBjOBnp1IB/znjZQoLAAjB70gZGn+vUdjViQnySp5PrVQMUlRh1FWDJngUEMjchVAzzUB554P0qSQ85AOOx9ajPJ4/KmihpGKKcRwKbTAXpUyP5YO5Dk9GVsf/r5xUaBidqruLcAY5z7UmSBjPB5xQAsrtJIWZyxPJY9TTKKvaRZf2jq1raE7RLIAxzjA78/Sk3ZXYHWeDNKubSSHW+ImiO+DcuenV/w5x17mu9XVZrnWHuL9yZEDDKICspGCM88A+3cmq0dilnFZwRyR+WIyYw6kMrEBipHtuXvyPQ4NOcxXrTtGwJuZV8xif9VGGLE89TkoMf414lWq5ybZjK99TY0yeW6AuFDyxywAISQgDFTjr7sV9yPpToI5LiJUkkh35GJGUEuSzKVDHnaANuOAeGrNtbm6vL+GGOIbmVvNYvjorDJJORjG0A8ZPqc1Dqs8086iztyrrbBkMJC7mbli+T94oQB0AI9eahWGpLqbc6x2uhm3gnZYyABGV3DauA2WPck/pxTLZWnMbzEhZcHIBOFXcR057rzz0I+lK5uEubKVAiFLsFTDlkDjCsrBlx02g8AgnA9zMLxo0is5d6xNDGiOrHMoUEyLxxwQOvpgehW421c07+6jt/lTAzu3RpwEbuPfnn1+b1xUUrRNbBiZGgMTt5ZJJy2R14xzx7ZrNSZb3VYp5ljeORHZWcDc2CuMg+23HPGfbNOuQk1oNPTLukhRnA4RQc9ckcjBznkZ9aVxXe4/T5jFJEs7n532KsSkpGc8HB5O3jPTOOvWpzZ2t5NKojJQbITsHMhONo7DgkHGARg8CqEbyx3QuRczhZF+c+UqurbD1B74J+nA74p2trHZ2z36eY0hQuIIhwzEcFvTvjHXcKfkF0WrpxP8wgaRjgBR1UswI3NjAByc+5z2oWxkcSIZWOyQIomP7zbkDrwPXHYgcdqpz31rbyNcX1rcRqHWOSSNmxCemfl+9yue/OO+Kda6sJtQihkaNnR0dZMlh1wWHPTnnv7+hsDkiaTTLJL9B50klrkiBDMqBjjA3DHIAII5H3hms65vmXVZFNvLHAihQZVJXdjGWOOvTK/pQ6T6TJNCBLMzLvKqfn2dTtYAFgCfbjNQzXZuIZ5JWBkjKOFXOG44GV6BsLg+47Gle4m1YueTPFbQpMzNbCJgssrgJLkqApQHoAR16eg5FRpfvHaSwwNlQQ0QEpLYyQCvT5cfkB7VFHLHd3sCpHtdwiSwhdoR+7kL/sBSf949xgUbq8WGK3vlgeSNVy3P3cjt7gE9sY/CnZiub2mQC+hnubq7SOWOU+WyqwLkcYAP3Oh4ODlc96kme0Nqs2p306iOQpEUPmYP3SCCdzfNnHPpiuThuYraCXZCWaUyMgGFJzu2jjnHDDHftW3509zpVxHIpJFwHIKAOoIVw5x23Mwz1GQOOKq2uxUZR6osXU0FvpNyC7xLIyKUmUbgwXlg2B7nHHUehxHa21wZN0aK8McTAvsLhVwT94/dzxgc8+vGB9NhiVppJHNu7K8kJbzQgwQc4zgbmAxnHLDPODsXuuW95o95JAm2CO1QKoONnmFoycA4xyx9B5QJ68LlL5FZtuxiLcOI7eDo6vhvlbBYjGCRyThQmD6flIytp0UMAljeJnZQcbmZnwSGXGB0wfo2OgpUuFmmclt8KhHkikA+98pJB54LA46Y4Paql1p7NY+RFIonGSls+5XCrzuAAOCeACccnjPFJGbatoaNxMLs2bxCBQrYLbyWcAHhgc4HfHXr6CmKwtU8lS5e3dVfGU80F9ofgZGCR1z15HWq0RunRWm3Q2+8zCR1AyAcFOenXOeh47MTUU0cMNpFMYvNjuD5crEbWCbztAJ4G5SeScfMORjJLXDm6mtNPcC6XbKDGsbCHY2VQ5B3Y6Y6Yz1zjHaqjXMTzD7VNP515JyUkIVotxXBJHBJCt9M9c0RX0F43kuoh2JGZmeMkZX7xXByAx3cYwSBgfdymo2qW1usq20ryqyJHcZGFUBYxkg4xweD1zxjnLQczsL9oNlBDC9tHLLxLGzEqg2gkA45OSDwM1EL60ubuZlid7c7jmGIr8u3J4BOFXJ7dFPeoZ7mE3McQ3vGFwAyqJEwwORjjPzgn2JpscSJe3OnXDB5LvcwkAykbkllzkZ6g8Y5DfgKURX1sWrqOdIXuLKSO4T7OQQgO9cDqcgBgOFBU5AwMHpTZb+K002zgmOQ8QKSmMEZUYCsBxgsCDyeCeABzLcNd2+oXH2W2ljjWIOzqWOWIG4jjHJDAjnPT0C1ZJo7nUme7igkgvNzpMXPJYEfMDnGWx8w4OEyRgU0lsxXF8+SdhPE8ciOqPEr4QCUchgTwQemc9c5xwarFrhdNlk+0ebDLCgkkOMwv+7yrA84YZ5xjGO5qH7StheTW7qxt5rZx5I+UYUEgjAByDtbAIzs7irVtKIUaK3BEZiXDsdzZYEcNnkjcBg9MnpyKNEhbogsvKlQQTsTeBVVJCARlip4yQBn5yeV474pmoZht4re6ljaYxKx3HdtIJJU9Nu1oyf91yee55NtLfNEgaERoTJGo3JIVHBAwApGScfd649pCkN2DNeI7W1xKLdt4wYxsUZUDryxPHt7ihWF5Fb7U0TreQXIkeRPNBUkeYhAVhgknK4U5PUK2ecZdbXkVyb++3sHRUjSQgnaVUFmbg8ENg9MFQfaq2pWt5baeZ7fCfZWbegjwxXuwHJyuX3c8fN2FLpiSLp7xyiCMTN5Z8yQiLdg84AyCfmfr2BAHFaJqwJa2Ny21RYI18+3FzJLuVsEpIJcDa+QCpH3hx1w2QrE1nARSy3NwFJkMflxt1IfCpgqeOrZBOMEc1mS+al4mWZUhIdd+I3dQyAttI6nIwOfX3q7LbTPoMupO7GSKYvEJRgMuSCCD03FM47F89aHG7DUekT3NpNL8rpP+8jZgB82QPwA+UY6HHBPFWSJVR9PjvZTtVpYpA4DgAcox7keowGDkkjPENlqWorIkGFZ3jbdHK+4FQB8p5wBkY54+YDtikfzJLyQsipdyATxr8oIc7iq7hwQenUHbJ0GDU+Qytd6nvght4Y2jRLz/j5KltobsQc4+/kkdwO5zUEssmnxNPHKSsKBZFC5BAckd8dVx2PXrWlcqdP1eGWdnWzG17kBRnjDDHIBO9SuQccdutZCw3EtqJI5pWtpUzIrsCGyWA475LEgkZPpxWita4nuaMA8yJoFg3SJvmMTMWCIfmUErjjdx06AHjg1Hrd2be3vYlbZ5xjWSOOQlTIME9AAWG4jnJGSAT1NQC70tYLqORhK0jN5W7JVTlFDEdMrgbR6N6YEotopbOadliuGnQZnjIOPm3MBuHynGc7eysSSM1PLZ3Y+Z2sdLayanJZwPF4fV42jUozX0qlgRwSAcD6CisSO/voYkij1OCJEUKsckUpZQOgJCYyPaijnQjVgjaWKJ5JIwis8LCRSw+UMQcDsBx6jsPW5BeQNbwupkDqDG4/LHPoWIz6e9FFR0KvYqWMhgEEshId5isyp1VMlQoPfAGfxPrVq7nWCO4jNvGRBAWTA6EAHH0II/KiikxrYoxcbIrjyxH9nLlWTK5BXgAdjmi8uoJZ0jhTZBKoDoBjIUgYHpyB/Oiilch6Ectn5RkghQFmjRlLN1JBJPf0GM496pxTNcyySx4Igkjk/1YHzHOPwyB+VFFC2KsWby2lutJmkODBHJHvyeRGZtu0j/D1q7PqVtc6lCTarHEIWjj2AAq5OVORyeQMk85bPtRRVfZFdllprxLeNk3bj5ZdQQFKnDYP4/lWfe7bYx28TYE4IcgdVBIJ9s9PpRRU9R9C+yMS22bY0kauVA+9u5we2Bu+vNLBqFs1p9hVQkjKIvNTdwf4ep68kZ9uaKKXUu9rELQCS4YGdv3kZjLjjlR1HHXgdfQ+tSTaXNPEYkkAJ2yI2ASCzEE8+5P4CiihBZEqaRcTXdsYgWUHy1VmGScfN7cnn8amnkjlj37PLZ4NjMOSDwp/AnNFFV0BqyIJrufTLIFo0WIuEmwATnb1PrnIJ6cjv1qMuz6cZVP7pwshU9EYrkZ55HaiioZPUfBbxNMyNIJVWRQQ4IB+U5/l/KpZrkwXU6SbnMcau3TDZBwc9mydvGfu/Q0UUuhdkQNFCr+csBhjZRwZC2MnqvTBIbBzU8L3EcZt1wjyu2WJz95BgNz06/nRRTvqOUUkVI7yKDS72aaMSJGoRwRhwQTja3bLEZ9ada3MTiJZZXKvGswKjIjyMkc+hwDwc4oooexlF6k6iRrhJljAkk2jDc4LdOc98qD7CqMFw0kMiTK0aXETiPod+w5YHHQelFFJLQYtwsZkltWtyYbYKyZbIXg5OPXJbn0Ud6kt0tPsRg+zCMhlKlACuQehB9F7+woopXDqaV7YrBpEl9DMH83aFQk8LsLZAxgd/zqh9vS8MduFzcKy5ZvQcYPrxn9KKKuSHNWasRQFUglihgDyl1WJnYkAFgp7992fw+laZlkMRKIjAuJmUcAkNjAz0xuGOv9aKKXQaehTaWZ5RdRMYYm2ypEWIES4K8Y74HT/Zq9Zzl9r4BkZhFtPzbweec4+6Oh75PoMlFKwky0szxkBrhJUO9fLeM/KVBAKkd8/TpVOP7JcBmi8yJo3SNQ5LFQW3bc55XkkDH4juUVRV9LBfs0JS5yiTyxDAjLBTz+YBDEdqgun+0zjch+zoSuVkwWBDegHbd+QooqXuS9HYdqTkPMbFYkLQrLGQCucLxj06+3f1qib6NYp41j8vyS3AH3WIySOfrx+XoCiktbhJlkQNdQSz3KlLhyHjZeiBuFbqcjoMY9TR9mlF3ckwwfZ4wGlRl+6OSCADjjLHAHJooqthrYlWOa+8uURqxhj3qnAAfI2n6gN+tKjxGAFAI925XZRkkhhu65Prx0PqKKKnoBmz2qQ2Mb2+77O7MgRjxvz0Pc5AXvgc1Zm03T9K1G5isXlnjjiEkiTAbZFXgsM5PqNpx17g8FFNbCto2Z1tpD3lnKGnWQLIWVgu3awcEqcdQQSf0zVu2MEaSqoacyXW5SePkY/Lz1wCTkHrxx1ooquhCdixDYFLYvMF2BmmAPIIwuV49Tmq1jZsby4u5XLSugR++MAEjPGclhzjv+FFFSWkMv7mKzDShzNIgE3zZGFztTtwQRjGeMdT1qG2nsNQQSTwGG7mfbHIGPPIPOO/XDcHrntRRSS6k31Lst1FZ2kquLiWYTfZ42LlVUk7TlVI5B9M55yehotYZLdYY5IQ0W9MyI20jkAsB9Oec9ec80UVUkrXLSIrzLQSQSyu3k7muCqgkqCF78HnaPzOO1RyXVzYzy2DD93NtVCgUFdzD1/x7Hg8UUUthEh0uWa8mu7Yurw4afzHGXbdtUjHQbQR1yCPQ4qWdYr2RY755UCzv8q4YCVMBlIOQU5Hbnd044KKd2U9LHBfEJo31q0jjUAeQXOGJyWdsnn1OT+NN0/TDBHEyAbgqsSRwcjI/mDntjtRRXsUF+7RojUhijbzDOqlAB5ZPzbRk8gEdgPbp9K47xCsSXLGBQqdAQMZBAPP50UVqiindIW0TT58YG+aHPqV2t/KQVnUUU0SixacTFvRTj+VdlploYLN2uUGJc7R97O3H9SPyoooY0RahHt01x5h3gZ78jsf8n+tctJ8vHeiilcGVpfv1Lbkgu/XYhP58fzOfwoop9BCHj1GaYWAG3H40UUyUMZSvWlCE444JxRRQMv2a41+2SMYBuU2AehYY/pVaUKsK4ypJOR1/wA//XooqeoDvsuNO+1s2N03louOuBlj+GV/M+lbfg8Wkd7cXF1HcybYtkawOFIduhJJ6AA8d6KKiprBiex17awbpbNZJB+7KsrKhITr5hUEjgeWpHf5QK2tPtma5mhY5kRkM+G5KjJYcghs59u9FFeTUSVrGSepKrXEN5dRXJMQuLdU8xCCPNVgobHcYO7oPzqK5uzZTG2mgjSYBpGCfemX7zAtyOx7d/yKKzaBlOK4EwuYbiJINrFzOrdAFOcBRx8uD0zkDg5Iq9Pe77hrlJtxktZDtBIKEBWxgjoTsxgn7hzjNFFBLk2VrNpZdPjnjj3OliHLb8ADhGbB9CyDA69e1a2hr8splTfPjLNkE/e3A5wOwUHjkg+xoooaKRDqazQsLuJt/wDy1IwABErEH3/i245+4p6VXE801v5l0zRQXWVHkSMPLUruDN06dBgEhRj5gAGKKSeo7JjrYSWDXNhcyNcSRF97lFJZG46HK5GCR257VBLpkenGS2NyIpwvzuEZgVw2zHzZXhT07t7UUVTJaL4vo1uY7dd8xwzOswAKDacDIOM7hzjrnNYcbW8rmbe8cs8ZiaPHDkoSuQOOo6+x9aKKEkOasi9pJjMk90Yd8iQFGkLZwuMFgD1bA6E4IYg8EiodVuontJbiCNYI3kMcaxrhAdpZsdwADt9eeMAZooqo62F0sJbafZ3EMEM9s8dxxH/o7hVR+Wyvc5Tkg8cHnLcEc0ljelTKzTxlYH2uQkiNgHgjPBLfTjrRRRLcfVIjneF43dgsqI7RtGVK7Pm3BeDjhc9M4J+hrQgguk0TVLSAxQRL5csJidsHDZdSGB45II6HPA60UVF7Ml7mSqtBezQrbKwKviYsNy5QkfUcH07E81qT6igmtbcRTx6gSI5y7hgQfmQ8HqQzZx6DpRRVWTF0Ift6iO4hMQk+yIJlRzuUljtzyDyDyPTP1y/TXXUI7V4nXy2xFtmQtsbBMZxnBHAB9i3TjBRTatsBWsElme9RLiSH7PuSZm5aEDk4K4z9xsdSCvUjBNmIvBBZy5KxiFYrvc52AbVaNyuDkrkcDPKHnHUoqnpIZm2OoJLe2luzk3E5EUeRkAONmzPU4J4J7Aegq1bStOPtTyOIpX2Rg9S3OFBzwu4uxB9cjmiiqaSSJT1J/NR7h7s+eluZdkzqQfmcBgAM5xubPbGDgHvXk0+YGS8cIiLIm187Tu3gbmKgE5z26beB0ooqWNamlHPbTWORZxPLPJJsmfkoAikj83K4xjjsKylvrhJpUlkaTypJ08xm5BA3KTxkjPmNgdMnbtbBoopxSsU9B8t0ZfnlDKQgZ4toBVlJycg4xv3fn6U15JNN0NnikuBKjJLGxIBUiMszAg8HaoYfT1JooqYK8rCXckuzIdTktopWYLHJuaElWj24JKljyVIPPBPNQL9nluUs5ETb9pZUklyATt5U7OgKZXp34wRklFCVnoHU1/J8y3mgmyjqyvLskIViT1242ggvHg443Me5FZzteXMNxDG43wzDMeAU8zIj6N1GSpzRRR0uMp3W57y08hXma3hYqFIjXaFyzEdOdrZUDjHGeM3NIgfzJACqwXKmJJEAEg2ht3JyRuV+ccc+1FFEnoEVqXGullWQMvlm4k+1K+wHDb1VzwcknPsfcioNPgF9NbqiMwmQ25jZzucuzRod2eAHw3Un5R1IoooguolqylaWa3MEplZzEsyibd1mBwSAM43Aow5wDuzxiponkZLdPtErx3MzRxmc53FcqORyOQQfXB5GckoqnuUt0Ij3SRqryszqAGPkqcn67hn8h9KKKKqyK5Uf/9k=" - } - ] - }, - { - "items": [ - { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAKXA+IDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD3+iiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAorI1PxV4f0UzLqWtWFrJCu94pLhRIBjP3M7icdABk9q47U/jn4I0/y/Iu7rUC2ci1tz8uMdfM29c9s9KAPSKhuru2sbZ7m8uIreBMb5ZnCKuTgZJ4HJFfOmtfHLxHrXm2+mQQaTavwHTMk4G3B+c8DnJyFBGBz68hcPe6vcm71C9mupnAJeeQuT2A69gP89aAPoa7+LvhK3ZVhubm767jBbsAmMdd+3OfbPSue1X4328ayppmlsW48uS5fA7Z3KvTv/F6d+K8fS1L2oMQ2k5bIPJOPr7enfOTT4rNY4Q9wMLGQc8gj1BYZ9/b69QAdfq/xv8TzR+XZx2dqwbPmRQlmPGMYcsMc+nbr1rP0/wCPfiyxHl3aWF+C+S8sJVwOOBsKj9M81wOrTl5CFiMaKeA2M/iB0P8AntWFJKUb+dAH074e+OGnaoBHfabLBKEGfJcNvfj7qnGBnPcnp9a6y0+I3hu6id2u5ICpxskhYsff5c9+PXj6Z+QLK+NvMrbjgnqDXdp/xMNPeaFy0hTcy45Y9fz68/407CufVFre2t9EZbO5huIwxUvDIHAI6jI71PXy/Ya49liZL54nVcLKkhV0BAycjkfr+uK9N8HfF2yvSLHxDdQQ3B2iK6VSqSdBhxyFbPOeFxn7oAyhnqdFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRWfrWu6Z4d05r/VrxLW2DBd7Aklj2AGSTwTgDoCe1AE2o6jZ6Tp89/f3CW9rAu6SVzgKP6nsB1JOBXzR8Rvi9qXiGWaz0q5mstLPy+WjbWlXBB3kcnIJyudvTrjJyfiX8SLrxnq8iwGWDSosLBblzhsZxIy5xvO49Og45xk+dsD1PegRKbuRz8+DT4WDsB3qoAWIHrU1qD9rjA67qBnWaXYqqiV9pbsAD8noT6/TB9Oua6MRW8cIWVxH8mPvnOSeAR09AcD+lYMup2+nwRQnc5UZYA9WwOCfb26fWsefVpbl2mZsNxtA4Ax9KAO3jyVd0IYuv3QAFznuRjj37YFVZXheMySTvhDne5yMcd+ffr6DkZxXIWt/fyyFYnyM7mJUkc4H/6h0q42peY4W7ZcLna/l5APckc5+vPTpQhMdd6jbeYxjgRlIwiryWxx/Tn/AOvWFd3stwxbbEqqOESMAAZ9SMn/AOv6cVfvIXZhPiOVOzKQOPwwR+NRosE6FXhnBJ7OoH4/Lk/nVCI4bOO5i82LaCBkqDmug8O6nBBIqyNh1G0D+97f5/pWcfskMUYhs9pVssyvhmXuueeSOhweahhjX7QswUrhux6fSgDttR0u2v1MsLgNMN+T/F7j8/x/GuM1XSL7Sm8x438oMV3jlc46Z+ma6OPWY4wEw8j4wNvHPHc9aeuoTyI6yRqyP1jKhgfqDn+lOwXPZPgh4xOveFzpN3ITeaeB5ZdsmSE9MZOTtPBwAACg716nXyZpky6LrEOoaZDHa3cL5SSOVgQe/GduCDyMcjI9a9O0z41TWxji1ewWdcKGmtzsYD+JiDwx9htGc/guUaZ7LRXL6X8RfCerBRBrVtG7JuKXB8ojpxlsDPPQE966ipGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQBheL/FVl4O8PTateq0gUhIol4MkhBIXPYcEk9gD16H5O8T+LNa8Yak91ql4zgsTFACwjiBx8qJ2+6AepOOSetdZ8YPGbeI/ETQW8ofTNPLRweWR87ZwzbuepXg/3QCMZNcHdW8NrYJvlCzEZZV6L7D/AD60xGZLCzcrlienPXnH86pHk0/zXG4BiM8H1xTwjZ2gY9fWkAlvGXlyP4BuP51LbuiXqygHYuW5PXimD5cgHk+lAU7gnT1oAdLI0shYkkmgc4zj2oOA2B09cc1Jt+UHvSGbOmIXtWUt8q5J3E4z6dsfhn6U+WB5FMRjyw+4rIfmPqMAjHqOvI6VUtrjy0EfyqWIwcAtnvg8D9RVhrofM7Sq0ZwDhmYdf7uAD7dev5MRll7izlK72VgCMY4GeoxWlaXi3EoFxHHlzgdto9z79P8A9VPcR6mFGTuHRBj5eevHHPHqakis7fTys0zhDnoHGSMe2eufzpgXptMAjaQcJgcn1A5x/n19MmtHbiA7mAJznjnA96sDW2m4gTCjgt12gY4Hpxxx71b+2RlVLptcqc7DjA79fx/LHsWIht5IQTIwG/GFJA5/Tt0/GrrAxbnaMADgYbn9Oef88VlnVba3lL28CbhwHY5+mOo9vyq0t61wF3FWY54bg5PXOOc/T0PTuwMq8up5r7aMoBjaFJzj3q7axCaKSeaRJEhZU2hwdu7OCw687Tz9OmRlt1GzBbxA3lO+0yoAMuADhcd8Y/Oqt4zXMUYjKQskexA2BvOQTlsDjJJGeF6Z5yQEbaajpNvIs1+5FsSAY4lAfGM5A4/Ljvz3qTS/iTfeHb4SaBdyx2m7LW0x3RPyCcoeASFA3DBx3FcLcafqkjt5lrMQmcuFygAOM7umM984qmR5TbMgkdcetK5TZ9m/D/xtb+OfD321ESK7gfyrqFWztbGQwHXae2fQjnGa6yvEP2btOuYtC1zVJGH2e6uI4Yl5zmNWLHpjB8wD/gJr2+pYkFFFFAwooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACue8da3P4d8E6rqlqubmGILEePld2CK3IIOCwOCOcYroa4X4wXcNv8ADbUIpZFWS5eKKEMDhnDh8e3yox59PwoA+TtVmy4jGMrxxjGMev5Gq89/cSO5MrEOOmfWlvlG7cSd7Y79BVM0AWrWymvCzoQAM/Me5xnj+fsKSWTyyY1+90ZvetDw/barqurW2n6RZvd3chASJVBz9SeFUdySAOTUXiDQL3w3q0umaiuy7iOJE9D/AFB6g9wc0CM9JNucY3dqlVcL7nrUUYwaswqrHLED6g0hj44SAZWBO0dccCo2nzJlecc8ipZWEqFEBx0B/wA5/nUJtpPL3hSR2IXimIs7xKud21sYJOe/Xv8A5zTijsP3j5UYAyOfp7f5+tZqsUbPQjpU4kdlG4sfQf4egoAvwssIEaLsDdZH5/IdP51dfTrSY731AzPkZ6emMf5PpjrWQ7RKMyykt2Ren4+tRC6YuCiqmBwBx/n/AD9KYHSbba0Ecix/KOQS33e+eMZ/L+lVmulf5dwIznknJ/D/AB+lUYb+aVfLaQoD03HIPpxQsi2xLOmW/gw3GaYjQmtYUjVy4Lj+Hbkjk9x/P0z7VYCeTaqpgfayZfcMdQCPfjk54/Ksb+0bwySFZ8OcEnJzx36e36e1Vze3OzBuAAeRznHvx0PAouBvFljZw+VdkwJcZyAc/Xrj9M9RWbeyfLuBHy8DDVlGQgYVjk8t7mnSSs5IJyKVwaGtM5XbvO30FNQYYHjIpGXvg4NbXhHw7P4p8V6botudrXMoDvx8kY5dsEjOFBOO+Md6QH1n8LtNm0n4Z6DazsrSG387K9AJGMgH4BwPwrrqZFFHBCkMMaxxRqFREGFUDgAAdBT6CgooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAryL4/X5g8O6VZkL5c1y8rEg5GxMD8PnP6V67Xz/wDtJarEtzoGmxyf6QiTTyJg8KxVUPpyVf8AL3oBnhF4xe4Yng9/rXR/Drwl/wAJr40stIkaVLQ7pbmSJMlI1GT9MnC5PQsOD0PT6R8AvGmq2KXU/wBg07eQRDeTMJMEA5IRWx1xgkHIOQK92+Hfw00z4fWMohl+26lOSJr549hKZ4RVydq9CeTk8noAADf0jw/4f8I6fMNMsLPTbZV3zSKAuVXJy7nkgZPLHgV8c+Mtc/4STxhqmrBpWinuXaISn5ljzhAeT0XA4OOMDivpX41+LT4a8DyWkDbbzVS1shx92PH7xuQR0IXsfnyOlfJ+3AoYgXrzyKeWwOOvb2qHdijJY0gJll4559qvxM1ygUAkDOSBtH09Bxz+H0rLCszbVBJ9hWvZOsFpIvmAkjJIUcYwRz1P8hnPPVQZQuIUXBjy3OM4wD9B/n+ghaUqNq9+p9aWV/3rAH5c8HnpSFUGPmLMRk+g/wA8UxEYVm5qdI3UZXDeu3rURzlSRlT0A710uk21jLYO0zxqM5kU9aqMbkzbSOdJ7H/9VIZX2kE5z681tanZTacEuJIVa0ndhEGyGAHr6e2fSqscVncsAA0Rbj5umfrV8l9mCZl9aNvftWxJoUwzt7U+Lwjrt1bG5tdLvZ7ccmWO3dk/MDFTyMrUxMge9G4mug8MWmi/27HF4nkuLaxII8yOEyBW7FgCGK/7vNesWvwA0fxBCNQ0LxpFLYyY2+XaCUKcA4z5oweRweR3pctlcL62PCBliqgE8/ma+rPgt8P28J6A2q6jC8esaigLxSxgNbxAkqnqCeGYcc7QQCvM3gr4KeHPCVzBqFw0mq6pCweOecbY4mBOGSMHGeRyxbBUEYr0qkAUUUUhhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABXlWofCafW/jHL4r1aWyn0QeW8dmd7O7pEqgMMBQu4FupzgAjBOPVaKACiiqmqXcthpN5eQWsl3LbwPKlvF96ZlUkIvB5JGBx3oA+Yvjzqc938SLi1kCiKzt4oEwOqlRJkn13OfTgCvLi5V8ONwrY8T6ndah4i1Ce9t2t7qW4kkniaMoUdmJZdp5GDxg81ibtw56imyUWhJanpB+bHj/AD+HWomCqQTHjPTB4qLgdenrV21LSZTb5g96QyvNO7hUIRIx0VFAH4+p+uTQJnhjKxuRvwWPqAcgH2zzj6elTtAoJwOO4qtJCwO4c+1ADPvNljnJ5J5q0ql3MiLmMkDryvHf/GqhyDgjB96mtZ/InRz0B56GgDUg0hruVImdUDsAsvmAInI6/l7dQcgVLc2l3pcUX2uJeSSksJDDjHXHT8fyqKS+jUrJGjwSY6qpVT9Rzz7/AFrTsdbumCIIzcIMg+UmSQOcYI9s9+M9q1hy9RNkM2pvqtgtpPJHIqkEuR8ye4AxnqaffWWmi2tf7MLh0BSUPktIc9fb/Crj6po8+ofbri2iScyebuRRnJ5zg8fpVaTUNDiDGC3TdncuUGMjp+H+TzzWvLHuStDT0fxDF4avYhNaWerBogyhjnyiOSO4OPUj6V7H4H8bweLbWdwnky28gVoiwJCkcH6dR+FfPVzNHcQMkRQIrBtijAbr26cf1PqaveG/EF14Yn8+1t94dwzjJUsvce46496cZ2dpbFeh7r4p+H2l+JJGnXba3h+9Kq5WQ+rD1/2hz9a8qu/D+r+D9Tjk3zWcxz5N1bSld3GDh1weh6cHB6V7PoeuWutaMl1ayb43XPuvqCPXPGKj1GO3v7V7K9iWWFyDg9iO4PY+/v71cqVylLuUPh38TNQ1DV10fxBMk7XLf6PdbUj2tj7jAAAg4wMDOSBznj12vlXUrG50DWgqSyKFYSW86na3HIII6MP0r6Z0DUW1bw9p2oSGIy3FukknlfdDkDcBycYbIxnjFctSDixu3Q0aKKKzEFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRXnHxn8X3nhbwjHHpl19m1C+kMayBTuSMD52VsYVslBzz8xxyMgA8R+Mclw/jzVlm1mK7RZhsijAUKNowhC5yV+6Se6nvxXBPEl0rSW1uyfMF2ht3zHJwOOnBx1PHfiopfNmLSkMRn5nPr9akeVrv7qxoWYZVMIMj26DqfzobuJKwj2Vylt55jzGvDEEEp6bh1Ge2f8AGltZjGwYAZ6VbtYrhp5Lid2Hlxy5nV/mR1Q7QT7nC89c4HsxGTUXzDbRw3KozvtIWNwBycHhT9OOmAKBsldSw34BJHt/MYqNUyc4z+IyKrCbjlWUcdt2cfX/ABrQtpGFuTKQI88glgP5H/D8xTJI2shONuCrYJ5HSsuWF4XKsOR6V0NxJsQtGYkjC/wBTv8AzAz+HI9BWXdMrrnA6cbFx+f6UMLlBW2nsfqKkEo2kbV56nHPWoaBSKNLyzetlrtPcyN8xPr/AF/GtC3tILZQyYNyPmRlJLDpjj/PX3rn80/edm0HAppknV3Gu3KKFeWBXU732Ab2P+17/XOPzrCv7hpXMhlVsEqDvzkdR/jVKXEkrsD95iRTWikUZKMB7im5OW4zpPCHjO98LahuQtLZSNmaDP8A48vof59Pp7fY+MNK1jTvtEVxGSFzgnBHtXz5oug6v4gv1s9H064vJ2x8sUeQoJAyx6KuSOSQBnk19BeBvgNZWFtDfeJ5pbi8dMvZQyFIo8g/KzKcuRkHggZH8Q5relX5VaWwmn0NJrSz1K3eOS2N0rgnyVQuxxz8oHOfpXoHhfSjonhmw05i5eKLL7yCQ7EswyOMAkge3rV6006ysAws7O3tg+N3kxKmcdM4FWairV59kNIKKKKxGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABXB/En4b/8LDTTVOrf2eLIyE4t/N379v8AtLjG33613lFAHz74v+F+meCfCsN9JqCXbq6wbJYSu5iWOUAJwQo7+jHceFryrU9PtWJu2mEcH3flX5ie49v1x07V9S/FWIz/AA01iMSiIlYiHPQYlQ8+x6Zr5UvnP/CLJgnm7MRBYEfKi9MduR60yG7MyHkjTiOFGjByBIOfxIwa2fDt/Gt1IsP2LT7oxN5dxJvZW9UIJYcjODjOQPWsYAkjqOOc1XkXa5FIrdWOp1HSXRZLi0cGIZfbhSR6n8v88ViG4k+QKyAg5+UKhHuMDgVngkdM0Dj2oFGLSszQm1KYvgkHaflAbp68iqUkjO249aZg+351JJCY5DHuVioydvI9/wAvy4oKIqKUg0mDQAtFKuNwzkj2rq/h1Jp0PjrR31W3t5bEXC+d577FX0ZieMKSGx0IGDwTQJuxyqsVYEdQa+xvhNry+IPhxpcvmRtNaJ9jmEakBTHwoOe5TYTjjJ7dBrP4F8JSSxSnwzpIeJiylbONeqlTnA+YYJ4OR0PUDGzZ2Vpp1qlrY2sNrbJnZDBGERckk4A4GSSfxoAnooooGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAHzJ8a/iNNrmqz+GrOPyrDTrhllZh88sy5Un2UZIHr1PYDyh7wvpMVltPyTvNuyMfMqj0/2fWus+Juh6jp3xC1k3lk1ut3eyzwE8rJGzEhgffv6cg9DXFHcjFXXBFILaDxOykZ7elBYSHnk010woNR9DQFibb6U7HTIqISMO+frThN6r+VBNia3jie52yo7RgZKowU/mQf5VpTJpwUGJJLZsYJSLeT+LSfqAOtZ1sQ0jvuGduBk8nkVJIWGPvHHTcc0wILra05KuzDHVk2n8smoCBU9wnlOVYEMOqk8j6+/tUQRmyQCfoKQDQKnSRoo5GVirEAAg8g5B/pUXGaXAknVS4RSQCzdB7mgNz6B+BXxIv8AUroeFNYmEwSItYzvnzPl5MZPQjaCQTjG3HORj3ivnv4WfB7XrDX9P8SahqEdnaRYmiS0m3SXAIBAPGBGwPPcgEYGcj6EplBRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUVWv7+00uwmvr6dILaFd0kjngD/Htjua+fPFfx71v7XLFocdraW4kzC7IJJWUcfNklRnrgDjpk9SAeofFK7s28Pf2cUjnv3YSxQF8EKMqxI6YILJz13H0r5j8Q21tHqKWlopihlwxjZSWjPIxzyPp7+mK6jwLLqnifW73UdT1CebhBJLK5Z26kKCc4xtH0AFZnjyQw+PlSGMK0CxfK2TyVD89+d360KW6exXRI52+0W8sIhLImYGyA2OfxHasxICzKCQC3QHqa9OWSG807zrtlO4lmLfKI8HhQPY9T3PtioRo9vPF5n2VVGcqzrgn3x1H44qOdNhKLR59LYeVtXduZhkfSlFmq2ssrNyvAye+f8A9ZrSnH/E5e0iyCHaMDsB9P1/AVDd2MkVv5IIYHBz6EAZ/wA/SrM7mMDin7zHJlGzg/Kf61M+nTqhcLuQZyQe1QiFmOAQT6DJoatuUaOkz2NsGnulV5g37tWBIHHXA98Vp3XieOW2liWMnfGyDCYGSCM5zmuaKBSQc5HYjFJt5wBzU2RDgm7se07ybVOMKMAelbOl6alzp0wYENIPlY9j2I9v/r1jNE8JAkRlJGfmGMity2uWttOikJ+Q5U8YxwcY/X8qG7IuKTdj6G+BHi8654ROi3cu6+0kiNQzfM0B+4eTk7eV9AAter18KprV3Y6tJf6TdT2EpbcjwSlGGRg8jnufzNe+/Cr4zXev39t4f8RRrJeTMwj1BSkYY9VR04GTyAV6naMZyaoD22iiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigDyz45a9Lpnhi106PCpfOzSuSOVj2nbjHUkqcg/w+hOPmw2bEmSUYLc4zzjv/n3r2v40W5u/HNoZo2S1tbFJJGcZV/nfGMZ4yccjt9M+U6i4e4ZgDuaQnB57nr60wM+yu73SLoXWn3DRS4x8pxn+n/6qdc6tc3viJ9Wu7bzZGILrkLnAAHP0AFN+UDnoR/n/PtTGYDGec9v8+tKyC9jotI1rS11NmlVY2mwwWVRmJ8nOCexGOc5z9a6SW6ZiHYoiDlfnByPWvMZVSThvp70xTd2u4wTuiYyQrYBH0qXDsVz33L7SA+K3dXCg3D4Ygkd+1dFNoSDDyXkzHqwUKoJ9uuK4iK6kjvEuCqu6tu56E/hXRf8JaZE2T2rLx95HBwfoR/WlqRZ9C5qmmi3jnZAWTYR9OO9cvYOEuWLMAChGT+FdD/wk9pPE6uXj3KVKsvqPbNcxbyRRShplLKF4AOOa0dRuz7B6jbkqbmQqcj1q1pWRdlkK+aEPlhuhP8A+rNUZH3yM3HPoOKbuKtlSQQeCKncGjW1R2EEMcxVrjJLEHOB9O3/ANaqLXL/AGIQbiVLZAz06/41WyTzSUWBKw4DIp0bvE4dGIYdCKYKXPNMo+gfgr8StZ1PXY/D2tXst9HPE5t5JcNIjrlzlj8xUjd1zghQMDNe+V8efCOW5T4l6KLVwkjT4LFGf5MHeMAHqpYZxgdSQASPsOgQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB4V8aLhD4ojWNo3ZbOONxgNsYtIQG9OGDY78eua8dnVmkkYEFuikDt0BP/fOffNd/4vvrfxHrt3qoKoJ/mVWwAEVdqg57lQu7BI64xmube2WZXjWIMkRy3Te2ceh9jz/jgNgcm8hHbPOAP0x/Ko3YEtk8Y7+nH9at3FhNGoZhvUthto6dP/ihj3qiyNtQnOD39/b8xSEKjZOCevBHvV2GNZg0eM7ULADrtH/1uKphSrKccDgntW7paG3KSAH93JvcHuoOSPy/PIpiMm/0s2nQ5ChSx7c4/Pk/p+NR6fpr3t3HHtbbt8x8DnaOuPfHNbUwa4iknZszXDKRxwo+Y549cr+Oa0LVY7O9zCrMsaD5SQDggewP5dPekkO5k674XNpG9zandGgO9Bk9Ccsvtj19G56VzBJHFekW940TgyJ5kUbMeU79+OuMDpxxj0rIu9B0+7e5eFjbpG21NvzbiT+vJHpjjseAL9zjOtFbt9o0UMYMRIIOCSeO/wDk1kPbyKSCp45PFAXIaKKKBhSikpRQB6/+z3Y21744lmlDCaxtnniK45LYjIPHTDnpjnFfT9fHvwd1s6J8TdIZnmEN3IbORYz97zBtTIyMgOUP4Z619hUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAVS1jUBpOiX+pGPzRaW0k5j3bd2xS2M9s461drh/jAwX4Va4Tn7kXT/rslAHzXa6mZ/llA2nOexx3+nU444Ga6OylhFk1xIsYThm3tjLHphc59DnscjoOOCtZAJF3NkE4Oen/6q7Z2ifSrVfOWQTsFkVWBOAMHOD23H8R1OOQZnxxR3KJFM7bmBZ1AzlgufXv8v05qhc6WmWCgAMxUfXJ/Xr+v1qRHLsryYwzbc5BGCRnA9ACvSryJI9xHFAQJTKDgScAZwPyYkCqJMaTT5IFiaZQqlVJ3DIGU7+/Xj29avxOPs8iFRG8qbo34G0ZbDNntwPwHpWtcrFqkTtkRJ5u0EEgrGPlz7gEfXrWXeLNkJDkTn5sBQARjkY/L68/igMaC7KGNpFKPASdpHGQc9O55xj0A9K145SYsyy745SQxUnLjgtzxjOGP4+1c8UMuoRRFW67D349enofyArp47aS3tleQEbojkenQD9AMD88CkDQ2R5JQjnOZz+8TAABI6AdegII9aWLyUhVS4JbAIxxkg9fbgfiAeKzru9EkyOxCxgtgJ3GQe/1/Qdaqw3bNcxtI21VyDg/TgfoR9BxTEaSwm5TzSmUKjBOcscHPH4HPoB9Kjl0l3tcrtLAlnYZO3ntjOf48gdwDyMVqaYSfKlLOx+Y79uQm7g9+/wAy9+Ce5yNKOyjZIg5THm48sEYCAcgj0LsR25b3oA851PTpbGXDo4HTLDqec4x1HFUK9LvNOhvLeSK5yivsjVs5Y7SBuGfpyR3zxXnl5avZ3DQyDlfSkUV8UoFFSJSbKSHQAmVMfe3DH1r73r4KiUm4jUd2H86+9aES9wooopgFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABWT4m0G38T+GtQ0W5O2O7hKB8E7G6q2ARnDAHGecYrWooA+EGVoZzG4wyEhgexrrtH1OIWptjneSCrEcqAc545B64A7muSupjcXks56yuzn8TmpIJXjIC5xnpnrR1EnodM6xNO0nKRbfmGQxdcMePXjjjpxnnNWUAmYqUQOyKGK8ZbBG3vn72ecnEZ7mucjvg04aR0yT85Azu5z07jOOpBNakN+eJFRI44dzhM5PXhCRgEnJXgcDoBgUwNyRF2JH5mEMZyhbH8TZOO317+lOigKQsXbJ3LISS3G0EbceuST+Q9RWbbXDRFWkk3MSOTncwAxgfUbs88YI6jm1Z6miBg6CTywWD8ncc9Tnkjhl/EZ6nABiarLBYX0ixxkS5YZYckn+L2PX8wO1VpdTknQoDtRiThTgYx0+mOKoarK8uoyPIcyFjnn3qOKNpTtjDEkcEDOOaBDpnITIPuagi3yScnqc1bFlLNbvIAQFxgH+IHABHrzke354nsbKQBptrBV/5adMH1+o7fh04yAdToaxixQBmO2VT8pIBJ6j8MEk+i/gb0kywWygyBpJI8O4U7egPXp3z3+nasyKC3gUMGZWiABJODyAMAY9Bg+gJ60T6jG0PlCUFQpJzjDNk8rjjGMkegB9eWMbJMrSLI5cpvKD5uhBBGCO2AM+4Ppzz2oQpKW6si/KrEY4yRnH4ZOKttcIs7htxhc7XXoSMkBs+vf05P0MEsc8beSQHBBGMfiCOuP8P1QGFJAYj6j1phOOnStzyjd2EibFMqgYwMN25I7+nHsTjjOFKrJIVIwynBFS0UmXdJ8qTV7RJ3dIjModkQMwGeSBkZPtkfUV9218EW8hhnjlHVHVvyNfdmlX39p6PZah5fl/areObZu3bdyhsZ79etNbE9S3RRRQMKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigD4MOAV7YUU0ZLYClj7dqJThx9KmswJJkBXKk8jftJ+lIlbE9lptzdODEgx3LYwPw+g7//AK9KS7t7SMRRSZkAA3Lg8+uPUdvqTyck0tWuZIWNorSxIoAKb/Tseev8vQVi7iDkE0xnVJdwtmM/KFO7g57evc4LZPf8jWkJI4rKUxkM8iA5DcMpznt1ByMe/rweFE0gOd1WodQkUlSxCsMH0p3AfdvuuCwHXpW7o0CKk5ZARuHQnGDzg/8AfJ/M1zsuDIrEkAtkkdfrXUWl1ZC7YWkUkcMMQAjd94mG7rkH73Jb0G1umKYiz5X7oh5EjQcEYyZOvHfs5Pt78AzwsY7eS4l2ooYKpwAeM9yOOnue5OOKTzwk4QNhgd2RkYGQAQT/ABd89iccd+Z1rV/O2Wtu2YohgNjB+gx7YH4Uhl/U/ECCMxRjORtwCencnnqQP/1nmsBtQkMxk7k5/GqZyTk8nuaMUgsXBfnoQcYx+H+RV+11OPvuRyCNwP6/4+xxWKF5qeIAMNwyPrTA6W3MdwyeXMkRxymD8wzyM49PU46e+Od1BSl3IpySD1bPP510FjbhLGaUSFgy8rCwLAf3iu4cDBHII56c5HO3ZJkyTk+pXaSPoOKTGiEfd/Gvufw0NvhXSBjGLKEY9PkFfDKdB9a+xvhRrp8QfDnS7iSYS3EKm2mwMbWQ4A/752/nQtiep2lFFFBQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAHwTL94fQVZsWKXEfK9Rycce+CKr3KNHI0bdUJU/gcVJZZNwm1iGzxhSxzQQgvypuXHHBxx0FUjVi5UiVgW3HPJxjNVzQUhKKWkoGSK2UK8e1amnzuGgOdgXeGYcAKRg8fQn9PeshSQQRwRWvCyyaSz4VSjheM9P8SeePbFAi/e6pI1jKcMtxcfNLISQTyTgZ/EcY749+b+lWLp5Sf3nGcY5PAAwB+VVhz1NACjFLxSDijOTk80CH+2ali27hxmoAfepof9YvzEHPWmgOkkMMOlEqy5YjjIAU44bGeD7+3J4xXNz9CCCD6H1rc1C5/0RYPnDgZb5jn1xg9uc9B1rn3PNJjEX7or6H/Ztu5ZNP8AENmZGMUUsEipngFw4JA99g/IV875r0v4G60uk/Eu1hkeJItQiktGeVsYJG9QDnqWRVA77sdcUkI+sKKKKZQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFYninxLaeGdIe6nkXz3BW3ixkyPjjjI4HGT/AFIrbr5h8Z+J5de8TXV67MIN5jt0J+5GpwOMnBPU44yTSk7IqMeZ2PLdSUpqN0rdRM/8zUUTENkMB7mrmtKBqkzDpJhx+XP65qhHw+CDg+hxmkjNqzsMkbcxJPJ7imU6Q5Y0yqKCiiigAq9Y3McCvujLyfwBsbATxlgR82Oo5Azj6VRqaKXarKRuHXB9f8/jQBG7M7FmOSeSaSlZtzEgAA9h2oGDQAlL3oxik6UCHg4qzaFfOXfyM8gCqg5q5ZqGmXKjGe/emhMsakJIpipXaCBjp/Tj8qzTx9Ku6i7vO5kO49AxOTiqBpDFFTW1zPaXMVzbyvFPC4kjkRiGRgcggjoQRmof4aFOKBH3L4W1yPxJ4V0zWYyh+126yOEBAV8YdefRgR+Fa9eM/s7+I3vvDF9oM7gtpsokgBIB8qQkkYxk4cMckn74HGBXs1DKQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFfJHxC0mXwz441DTpItlo8hmtcbiDE5yoBPXH3T7g8mvreuO+IvgK28d6GtuZPI1C23PZzknarHGVYf3TgZPUYB9imrgnZ3R8manZNHbxXIfepO0juvp/WspJDHLkHHPat7VNPvtJvbrR9Tt3truFtksUnY9QQe46EEcEdKwHBDYIwRwaUQlvdBMDvydvPPykEfpUVXY4JLhNqrjAzkt2689h+OKrSR+W7LkHBxxVAR0UUUAFPTo+Dzj9KZViyRJLuNJX8tGOC2OmeKAK9PRd3cfjTWBViDwQcU6MkOMUALsNAHOKuPASm9VyB1I7D39KrjAOSSKCRiRF3wOPrWrZW+xwzEY2ggqff1ziqa3SoBhc+ua0dwayPlru3DIU8hj7DPDD1HIz6MKbAyrrb5pAUDHHyjH6dqrmpZf8AWEgED0PUVFSGB60oNNzzSigD1T4A6qbH4kx2eMrqFrLBj0KjzAfyQj8a+qa+EdF1SbRNbsNVtwrTWdwk6K2cMVYHBxzg4wa+6LS6hvrOC7tn8yCeNZY3wRuVhkHn2NAImooooGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAHE/EX4c2HjzTAcrbavbqfs13jt12P6oT+IPI7g/J+v6JqGhapPYanbtBdwtskUj8iD3BHcfXvX3NXnXxi8HWfiPwXfaj5SLqWm27TxTZwSifMyHAORgNgevcAnKsS+6Pk7zmW22ds8f1qvVtwBbqfkHTgAk/j/nv+AqkY6U0UJSUtJQAopVYq4YdQcim05RuYDNAE12m2bIJOVB565xz+uariuovtCk/4Ry11RmQJKzwhc/MGjVCxPtiRcc9j+PL0EqVzX0xEmDJKkhQDJMfJX3x6fpVa4t/JYbgRnoG4J/DqKLJju+aNWTuSinH1JHFXr2KNot8TOykZbe2QOPXvzQMxyDnpx9a09OhNzbyxBjkAlVA+/wCo/l/nrnAZPyj8TWhpMjw3YfBC45IH3ff8/wBaAKU+e+Cckk1AeBirN5IXmbcoznt/npVWgApaSigY6vs74XahJqfwx8P3EmzctqIflzjEZMY/HCjPvmvjAV9j/CK2ktPhVoEcsbIzQvJhgQSHkZgfoQwP40C6nbUUUUDCiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArzf41eLV8N+CJbKIg3urBrWMHtHj943THQhfq4PavSK8E/aVHy+GT6fav8A2jSYnseE7Q0RjUhmB4A/l70k9uIolYZA4AzznjP8jT7IhZVdiQAwyQM/WmXZkLkHOOCAfTAx9cZpgioefam0pOTSUDClVtrBh2OaSigDpLa7E+kvvIBVCp+n+TXOv941Yt5isMkfqP8AP8qrHmgiKs2PifY2f6Vp28/2plicbiPuqemeOvqfrx9ayR1q3Z3DW0yyL95eR7nt/Q/hQUaF1aLHOyggv0HPQ5x16nnvUtkEt1Zp0by2XDEYHBAP5cf573dJtxfSlpciNQRvHOBtOTz7jP8AIZxTb9DEZIwir8pAUjPB6/UjI5/2c5Ap9BHN3ahZ22klSeCetQVPIgyVHTPFQUhoKKKKBijrX3npthDpWlWenW5YwWkCQRlzk7VUKM++BXwlZxedewRf35FX8zX3vQLqFFFFAwooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAK8M/aSglbT/D04jcwpLOjOFO0MwQgE+pCtj6H0r3OvIv2iuPh/p//AGFY/wD0VNQJ7HzfCB5I4yc5wPxqS7BeMSHBLAZx37/yP6VDb9G47duasTL5kRbBVS2Ao5A/z+uKYjIIwTSVKyEjPeoqRQuKSnU2gByNtP1602iigBVqaNSzAAZ5qNBk1pabBvnHy7tozx/n6UCOm0uQ2luspwUC52kZDDBHP47fzrJvpnlkDM3mSbctuOM/54/zzW+6Qmyh2EN5ZDSDGA2RuP8AIVgakxUlgMcYBI5qgMSVhvI6gGqx+8alkbczGoqkEFFFFAzS0CB7nxFpkEaM7yXUShVGSSWHAFfdlfEvw+jMnxF8NqoyRqVu35SA/wBK+2qBdQooooGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABXlP7QcXmfDiJucx6hEw/wC+XH9a9WrhvjBY/bvhZrarB50kMaTJhNxTa6lmHphd2T6ZoFLY+R4s7TjrVsE+SQM/3sds4/8Ar1SjJAyWIArQWNTC3XIGDgcdTk/kBTQimibmPQgHv+VVbhAj8DFXdxQtyQSefwqjKck8UMER0GlpKQxKKU0lAy5axb8Z71s6ehjc7V3E/KMDPbn9DVGyRc9DjGAOua2bGMR3aOy8A7zjtjP+IqkSXru6CRFCAz4VhgYwc89+Rz1PoMY5zzN5KDCRjJz6/r/Kt/UWkMssbqd0Zwrtyep46+o/WuWuyd20/gPQUAVSeGplOP3fqabUjQUUUUDPR/gZbmb4r6ZIP+WMc8h/79Mv/s1fW9fOX7NulQ3Gt63qr7TLaQRwxqVzjzCxLA9jiPH/AAI19G0CQUUUUDCiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArE8ZxNP4G8QRIMs+m3Kge5iYVt0UAz4LjPzdcCti3UeQ5wCNpXGeckGodc00aN4n1TSxJ5gs7uWAOVxvCOVzjPGQM9TWpYYks5kxvjVSxOOhP/1yPxA+lNErYw7oFGcYA28cf5/z+NZrmtC8D+YST8p7HtWe1DBDc0GkopFC0d6KKANjTid3c88/4V1FkqHLyYI2jKk8/l37+/TvjPK6e2JO3Q9a6+DK2yyFeJF+Vd2M4+YD65x/npaJMnVI2fewbaFJGAemTx/SuYuG+faGyB/n8q39QkQRgLnJAYZ9+o/LH4iudfmQ49eMVLAY3YU2lfljSUigopKcql2CqCSfSgD0v4GeIbfQfiCqXlzBb2t7bvA8s7hFUj515JABJXHPrjqa+sq+DxYSJjzCqHAYKx5IIyOB61t6P4j8S6BsGlazqNrEkglEaTN5Rbjkp91ugzkcinZk3sfatFeIeAfjeZUTT/F+RL8xGpRx8HuA6IOO43D2yOpr2m1u7a+tkubS4iuIJBlJYnDq3bgjg0hp3JqKKKBhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB8cfFKxfTfihr8LlSXujP8p4xIBIPx+aqeiAzQtCQWG4Ecnv/LOAPXnit342KR8WtY4xuWAj/vylc9oMyQyyEqS4GVHbtx7+mPT6GmiEVtThAViM7lYqx7Zzk/z/AM5rDYcn610+pKwW4jIACtuw2cknt9OvPvXOSjDDr1702NEOMUlOJzRUljaKKUfeFAjZsE3SYPyqe+O2e3+e1dc5K2pKxgZjztY4xyeffjj/AIF6VzGhxhr1DICUUh256AV0cd0XtWmmLbSzMoGR06YPsfy/Hm0Sc1qZwXYAnAA3DPGMY+lYxOX3Hr1rT1cr55Kt1zwe3t/+r0rK6IffipYEdFFFIoK1dNCRRNK+Bu6E/qPyNZVWiGLFR90YB9BgU0Jmut0kaNkIzHknYDn6flUcF4u5kWNmU87AAR/jVNLVwmWHB54PQe4qzZSwRM0c8SyITxzyp/xrRMg1bS6W1uI5kjVSCD5cg4P51v6Z4/1XwteHUdPuRunwJYZW3rIAxOCvHfPI5GThua5aG1Z5v3BBTrtLAGjV2BTayqzKAu9VC5x7j73bn3qJDR9XeAvHlj480qW7tYXt54H2zQM27bnO0hgOc4PHBBB7YJ6yvi3wP4uu/BPiaDVbdDIgyk0BcqJYz1Bx+Y9CAcHFfZGmalZ6xptvqOn3CXFpcIHilTow/HkHsQeQeDUlJlqiiigYUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB8y/tE/8AJQrA/wDUKj/9Gy157pJC3IZvudxjO72+pxXo37RCH/hPtObs2mIB+Esn+NedaQN0+AgLAAnOOnfr7fpmhEmhrykXZkJ3edGW6/7RGPfj/PSuUux+9zjGe3pXTarMJ4IpIzlQuwn2wp5/EH681zd62Zugzjn61TArCkJzS0lQUJTkGXUe9NqSIZmUe9MDqtEh3OSGAyANzHAB5Ayfx/T3zV64iaztWjBChlIUk5Jyc49vx9fyl0SBViQHO6SRYsg7TyDgZ6Y4P4jtTdakaKMxjJcZLYXBIYA88dc57cEZxVknG3GHfIGOAMelVn4Ue9WpFGCwHfpnpVSQ8gVAIZRRRQUS26eZcRpkDLDrWosKJK6vyudyqB1z+ftxWdZnbNu9BnPpVhm82RyGK/19quOxLGXBRG4Iz2wc1WDkNnr7VKyE9BkfTj61XPXik2CNe0vgML0AHIJq88v2lOZmEmPlPByfwwcfj+Fc0GZTkE8Vp2F8VbY5yp6g96L3C1hlxb+S5G05HXOOv4Yr6J/Z+8UxXnh2fw1PMxvLF3ngQoAPs7EZwR1w7NnP98Yz28Gv44pIll27FI3DKhfbAwTnHr7VU0fV77QdWttT064aG5tpBJG6nv6EdwQSCO4JFQHmfdNFUdG1a117RbLVbJiba7hWVMkZUEfdOCQGHQjPBBFXqZQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB83/tFpjxhpLjvYY6+kjf415lpJbziUJyeBj+f4V6v+0dauutaFeFAI3t5Ig/clWBI/DcPzryvQxvu9p78dccf1/zngUIhEt0rRTXMTuBJgsCD1IOSB2wBmucuTmTJ4Peuv1+ONHt7iM7N2+NlzwM5Uj9f51xspy3PWmxoZRRRSKEqa1/4+o/rUNW9Nj87UreMdXcL+dAHd2lt/o8UY6wlG46sw59P9rA7DIzwKztecfaHYMD0BIHccZ+uRnB9fet/T90iqsi7hKTOWHLIDwD9cY46/hzXMav+5lkQdTyVIxxyMY7+mfY1TEYF0MPjHbseKqHljU8hy+SRzzUKqQ34GpYkR0UtFBQ+Nc5xV9FZIWVeSeTWejlDxVtp8xhT8pA7VcWSyBywbnNRnJp7NnqeabkdOKlgiPFKCVORSkY57U2kUa+n6gqQyQuXBYYADEL9eCOf8fwqjPIrSkqCAT3JP86rjr1xVi1tri/vobW2ieW4mcRxxoMlmJwAAKTEz67+DWf+FTaFnrtm/wDRz13dVNK06HR9HstMt2doLO3jt4zIQWKooUZwBzgVbpgtEFFFFAwooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKK8p8RfHjQNJvJrPS7OfVJYZdjSrII4GGOSj/MWwePugHkg4xkAwP2kj/onhwY58y4OfwjrxnRJI4rotJymMMPY8Z/M11HxP+IsnjwacDpq2MdpvKqJvMYs2MktgcYUcY9eT242wXIZyVAGfw4/+vQiepva7tnsIp4kIcyZYEdOh/PkH9fWuMnBWZlPY4rvrGAajp6K/wAgBBOM9gfx/H6egrjNaiaHVrhWUKS27AGByM8CmxrcoUUlLSGJWhoqtJq9uq5BLYyO2eKz66HwdAJtZMhVmEUZbaozk9qEDOumkhF4FBIjXoFPKjaMgeoHr0/SuS1a6Ml1MzHflj83c+9djIsNtGZfNcu6tlnACsct2PUcY7dPrjz++IMuQe3Pr+PvTZPQp5y1SJHnJPYEUyJQ0mD3Bx9cVbUBYwfUc1LEUGAAB7+lNarEidSBnHcVWNMpCUuaSloGLn3xShjnjn2pBjHQH8eacqbvuBi2Ow70CFJKtyvI7EVGTk59akE7L2U/VQaY7mRixCgn+6AB+QoGJXsH7PvhePVvF1zrVwEaHSY1MaN1M0mQpwRggBXPUEHaa8ms7Se+u4bW2ieWeZxHGiDJZicAAdyTX2p4F8KQeDPCFjo0QRpo133MqgfvZm5ds4GRngZGdqqO1At9Do6KKKBhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAVNUtJb/SL2zguntJriB4o7iPO6JmUgOMEHIJz1HTrXxHq2nX2hatcabqMMltd277JI3ByPf3BHII4III4r7nrz74n/AAwh8f21tcW91HZ6naI6xyPECsykZCOR8wAYDB5xub5TmkI+TmbemST1q9p6OUcowXII46/55FT+JvDGqeENUl0rVoRHcI2VKklZE7Opxyp5x9CCAQQItMjaUoik5J6env8AyFMRqw3zW3yHKxZzjgkd/wDIrH8Q3EN5Lb3UW7cybJMjqw7/AK/pU+orcW8u08k5UgDvzn+f8qy2Ja1kibllO5fbHUflz+FMCkKU0FSACQQD0460lIoK7nwxp5tNPMkysGuNsjLkAlP4Rz65z7/nXG2caS3cSy58stlseg5Nd3ZXjXErsoAx/AAAB7ADHY4/l14aEyfUpn/fOf7jJjJB6BuvTvjnr29K4KZyzsc5BOetdjdSDZcSsWGI2C7QCBjOBnp1IB/znjZQoLAAjB70MQyP/XKOxqw5PlFTz7+tU1YpKrDsasmTPAqWBE5CqB3qE888H6VJIecgHHr61GeTx+VMaGkYopxHAptAxelTI/lg7kOT0ZWx/wDr5xUaBidqruLcAY5z7UmSBjPB5xQAsrtI5ZnLE8lj1NMorT8PaS+veItP0qNlV7u4SEMxwBuOMng0A9Ee0fAPwDM91H4yvWaKGEvHZRbSDKxUq0hJH3QGKjHU56Y5+hKq6bYQ6VpVnp1vu8i0gSCPccnaqhRn3wKtUCSCiiigYUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB8+ftH6fEureH79S/nTwzQMCRtCoykY465kbP0FeeafphgjiZANwVWJI4ORkfzBz2x2oooEakMUbGQzqpQAGMn5ioyeQCOwHt0+lcd4gWJbpjAoVOgIGMggHn86KKpiRUuUJ0TT58YXfLDn1K7W/9nFZ1FFSNE9rxKW9AcfjxXY6baGCzdrlBiXO0feztx/Uj8qKKaBjNQQLprjzD5gGR15HY/5P9a5WT5RjvRRQxFeT71S25ILv12IT+fH9QaKKTH0EPHqCaaWAG3H40UUkMYykHmgITjjgnFFFMC/ZrjXrZYxgG5TaB6Fhiq0gCwrjIJJyOv8An/69FFInqL9mxp32pmxul8tFx1wMsfwyv5n0r1P9nvSrTUPHtxc3UKymys2lhDdFkLqA3uQC3X1z1AoophufUdFFFBQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAf//Z" - } - ] - } - ] + "outputs": [] }, { "language": "markdown", @@ -139,26 +80,9 @@ { "language": "typescript", "source": [ - "// Get the background image data\nconst bgrImageData = await getImageData(`${baseImagesDir}/wall.jpg`);\nconst bgrImageMat = cv.matFromImageData(bgrImageData);\n\n// Resize the background image to the original image size\nconst resizedBgrImageMat = new cv.Mat();\ncv.resize(bgrImageMat, resizedBgrImageMat, new cv.Size(width, height));\n\n// Remove the foreground from the background image by\n// setting all foreground pixels to white\nremoveForeground(resizedResult, resizedBgrImageMat);\n\ndisplayArrayAsImage(\n resizedBgrImageMat.data,\n resizedBgrImageMat.cols,\n resizedBgrImageMat.rows,\n display\n);\n\n// create a new Mat to store the final image\nconst newImage = new cv.Mat(\n resizedBgrImageMat.rows,\n resizedBgrImageMat.cols,\n cv.CV_8UC3\n);\n\n// combine the foreground and background images to get the final image\ncombineImages(resizedResult, bgRemovedResult, resizedBgrImageMat, newImage);\n\ndisplayArrayAsImage(newImage.data, newImage.cols, newImage.rows, display);\n" + "const backgroundImagePath = '../../assets/images/wall.jpg';\n\n// Load the background image\nconst bgrImage = await Image.load(backgroundImagePath);\n\n// Resize the background image to the same size as the original image\nconst resizedBgrImage = bgrImage.resize(width, height);\n\n// Remove object from the background image\nconst removedFgImg = Image.mask(resizedBgrImage, resizedMaskImg.invert());\nremovedFgImg.display(display);\n\n// Combine the background and foreground images\nconst resultImg = Image.merge(removedBgImg, removedFgImg);\nresultImg.display(display);\n" ], - "outputs": [ - { - "items": [ - { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAKXA+IDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD3WlpKKxNBfwoFJS0CDvS0lJQA6k70oooAAaKKTpQMWiiigAooopiCiiigAooopgLRSUUgFooopgLRSUUAOzRSUUCFopKWgApaSigBaKBRQAUlLRRcAxSUtJTuAuaM0lFADs0UmcUA0ALRRRSAKKWkpgFLmkozQIWikpaYgooooGFFFFABS0lFAC0UlLmgAooooAKMUUUAGKKKKAEopaMUwEooxRQAUUUUAFFFFABRRRQAUUUUALmikooAWgUlFADqKbS9qAFoxSUZoELRSUtIApKWimAUUUlAC0UUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAVKOtJRWRoOozSUtAC0lFFAgpc0lFMBaKSigBaWkozQAuaKQUUALRRmigYUdqKO1AgooooAKXNJRQAtFJRmmAtFGaKAYtFJRQIWlpKKAFooooAWikooAWjFJS0AJiiloouAlFLRTASlzSUUALR3pKM0AOpKKKAFpKWkoAWikooEOoptLmgBaKTPFFMBaKKKACiiigApaSigBaKSjNAC0UUUAFFFFABSYpaKAEopaKYCUUuKSgAooooAKKKKACiiigAooooAKKKKAFozSUUALRSUUALRRmigBaKSigQtFJmigBaKKKACikooAWiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAKVLmkorE0FzS5ptLTAdRSCigQtFJ3paACiiimAUUUd6AClFJRQAtFGaKAFzRSUUgFNFJmlpgBoooxQAUUUUAFFFFAC0UlFMBaKKKBC0UlFAC0tNpaAFopKXNABmiiigBaKSigBaSiloASilpKYBQKKKAFopKWgAooooAO1FFFABRRRQAuaM0lFMQtLmm0ZoAdRTaXNAC0UlFAC0ZoooAKXNJRQAuaKSigBaKTNLmgAoozRQAUUUUAFJS0UAJRilopgJRS0mKACiiigAooooAKKKKACiiigAooooAKM0UUALmikooAWikooAXNFJS0AFLSZooAWikooAWikooELRSUUALRRSUALRRRQAUUUUAFFFFABRRRQAUUlLQAUUlFAC0UUlAC0UlFAC0UnFFAFKijiisTQXijNJRQAuaXNJS0AFLSUUxDqSkBpaAFopDRQAtJRRTELRRSUAKDS02lzQMWjNJmlFABmlBpKSgB3FFJmjNAC0UlFAC0UlFMBaKKKAFopKKAFooooEFLSUUALRSUtAC0UmaWgAooooAKKKKAFpKDRQAc0UUU7gFFGKKACiiloAKKKKYgoopKQxaKKKYBRRRQAUZoooAXNFJRQIWlptGaAHUUlFAC0UlLQAUZoooAKKKKAClpKKAFopM0ZoAWijNFABRRRQAUUUUAFJilooASilopgJRS0UAJRRiigAooooAKKKKACiiigAooooAKKKKACiiigBc0maKKADNLmkooAXNFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQBSpaSisSxaWm0vNABmiiloAWiko70AKKKTtRTAWjOKSloEL1o6UlFAC5paSimAtFJRQAtFFFAgooooAKWkozQMKKKKAFopKKAFoozRTEFLSUUDFooozQAUtJRQAtFJRQAtFFFAhc0UlFAC5oBpKKYC0UUUALRSUUALRSUtIAooopiCiiigYZozSUtAgpaTNGaYC0UlLQAUUUUAFFFFAwooooEFFFFAwooooAM0uaSigQtLmm0UALS02igB1FJmigBaKTNFAC0UlLQAUZoooAXNGaSigBaKSigBaKSigBaKSloAKKKKACiiigBMUtFFACUUtFMBKKKXFACUUuKKAEopaTFABRRRQAUUtJQAUUUUAFFFLQAlFFLigBKKKKACilxSUAFFLijFACUUuKKAEopaKAKNFJRWJYtLmkooAWjNJRQA6jvSUUwFzRRRQAtHekpaACikNLQIKKKMUAGaXNJRmmAtFIKXNIBc0UlFMBaSjNLQIKTNLSUDFzRmkopgLRSUtIAzS0lFMBaKSjNAhaXNJRQAtFJRQO4tFJS0AFFFFAhaSlzRQAUZoop3AM0ZpMUUALmlpKKAFozSUZoAWjNJmloEFFFFAC0lFFAC0UlFAC0ZpKWgQZopKKYDqKSigYtFJmigBaKTNLQAUUUUAFFFFABRRSE0ALRRRQMKKKKADNGaKKAFzRmkooELmlptFADqKbmlzQAtFIDRmgBaKTNLQAUUUUAFGaKKADNGaKKAFzSGiigApRSUUALRSUUALRSZozQAtFJmjNAC0UmaM0ALRikzRmgBaKTNGaAFxRSUZoAWikozQAtFJmigBaKSigBaKSigBc0UlFAFCjNAOaKyLFopKKAFpaSjNAC5opM0uaAFBopKBzTAWgUlKKQC0ZpKKAFpaQEUtMQUmKWigBOlLmiigAozRRTAKWkozQAtFJmigApaSimAtFFGRSAKM0UUwFzRSUUALRSZpaAFpM0UUALRQKKBBS0lFAC0UmaWgAozRRQAUUUUAFFFFMAopaSgAooooAWikooAXNLmkzSUCHUUmaKAFopKM0wFoozSUgFopKWgAooopgLRSUUCEdtiM2M4BOKEYlQSMZ7Ukv+pf/dNEZzGp9hSGPzRmkzRTAdTT3opD3oAfRTc0tAC0nejNGaAFpKM0tABRRRQAUUUnFAC0UUUAFFFFABRRRTGFFGaKBBmlzSUUgFzRmkopgLmjNJRQAtFJRQA7NFNozSAWlpuaM0AOopuaM0AOopuaM0AOopuaM0AOopuaM0AOpKTNFADqM03migB2aSkooAWjNJRQAuaM0lFAC5opKKAKGaX2pM0d6yLFzRmkooAdk0U2loAWikooAdmim5paAHZopM0UAKKWkzijOaAFo6UmaXNAgFLmkoFAxetGaSigB3Wik6UZpiCijtRQAtFBpKAFozSUZpgFLSUUALRSZpaACiiigBaKSimAtLmm5paAFzRSUUCHZoptFAWHUUmaM0ALRSZpaAClqtfXcVhYXF5MHMcEbStsGWwoycD6Cqfh/VjrWkpfFVXzGbCrzgBiB+OMUAatFAx/kUZHXNFgCijI9aM0AFFUL/V7TTrqzt7hijXbmONscBgOAfTNX6ACiiimAUUUUgClxSUZoAKKKKLgFLSUUAFLSUU7iFzRUEl5bQkiW4iQjqGcA9M/1rk7/wAWySo0UMYiB4LhyT+B7VLmkFjs6K4az8U3VvGUKpJly2WzwMdB7VuQa/HfW04RzaNGqnzWUOOvTH6UlNMDbk5jcf7JoiI8pMegrFu/EdrarCgfzt6gsw4wP8fb6Vlf27qCr5cIT5ydr8E4Hpz6etNzQG9qusrpbxK0Jk8zJzuxjGPY+tZy+LVKsTaYx0/e/wD1qwdQ1KfUxH5qjMYIzjGfw/CqRfYrLye1S22ykkdMfF0hU4tUVvXzCf6Uh8WTHGLaMdmyxNcrvOcbevvUqCWQ4CAE+9CuFkdN/wAJbNwBbR5AyTk0p8Vy5x9mj4x1Y1zggmzgLxjsak+zzf3e/YinZhodAviuZnx9njC5x941Z0zxDLe3kNu8KDzCcspPHBPT8K5cW8pIyv61d0hZLbU4JpNgRGO4s+ABggn9aLMWh3WeKonV7UXHkFm35I4UnkVWPiKw2tl3UZxkrz9cVgW6D+2YxA7hZCXRUYEgj+9kY9abl2EkdsDkAjoaKr24uwT9oeMjsEGMVYqkIWjNJRTAWjNJRQA6ikooAWikozQMWikzRmgBaKM0UAFJS0UAJS0UUwCiiikAUUUUAFFFFMAooopAFFFFABRRRQAUUUUwCiiikAUUUUAFFJS5oAKKM0lAC0UlLmgAoozRRcDPozSUuaxLFopM0UALRRRmmAUUZooAXNFJ70ZoAWlptLQAuaKSigB2aM0lGaAHZopuaXNAC5pRTc0ueaAHUlGaBQAUtJRmmIWik5oyKAFzRSZooAWikzRxTAWjNJS5oAKKSlzQAUtJmjNAxaKSigQ6ikophYWikooAWikpc460CFqOe4it4WmmkWOJerseBXGeMPE2raRqtjbWtoEtncSNccP5qjG5MfwcnBPX0xXJXviC71hpXvbkoq8xqM4xyCAB/X8T0pb7A9DX8X+M5ruKbT9NVxbvGQ8gXLSLzuAHYY6/j05FYmjavNZQLG7Si0L7I3RdhX16cHqD+PHFZt5dyXMnl2vSMBTMeDxgg5HXHb6DvzVdry6gnZrvfMGXBY/MDjp1FXFWE9T0CPVbsLuhvrh0I4cTcfqOD7e1LFrupFlQ3bjPGDJnAzjNcXbXDpcm4sLnPHEeeTjGME9uBwe3rT7zW9XnjSaJJ1mjkAkiDn5hg5G373JPv2qr+RNjt/7Y1AMQLuSXB7Z4GOvB98VG2u6ghDG7yeePMwD+vtXNx3V1cIqzMyzJw6iTJTjIB6ZOCPz71M4d5dxlk2D0bB/EE4xV8pNy5rF7fPDDM0jGSN/MjfzN5VgCQByfQdvr0roPDXjmDUFSDUWSC4PAkHCN9f7p/Tkd+K5C6smuU2yMWAGQdxXHbHWsqW3lglHmmTYhyJUXJGeuV/TNS4saZ7tke1LXio8V6xZ6c0FnebQxXy+cqmCCNp/hHGMHjB6c16rod5dz6ZaNqTW4vZohMY4FICqcepPr/hWdi7mpRRmjNIYUUmaKAFopKM0ALRSZrB8Qa8dOVYrZ0MzfeOMlB/LNJtIDYubuG0RWncIGO0E9zWLB4qtpZmWRDEmSFYnPHuK4651S7u1xJPLIuc4ZiRmqjyOQcis3N9BXL+ran9svnfBxnA57VntNg/KMGmrtCk8sT1odEI649sVnuF0L5pyD/KtK2C+R5jcZPGeKxlVvM2gkntW4xEeyPBwqgcKTVwRWhHI/71Qpix/ESVH86ltmeO6EhnRgv3V3A9v8aX7QcYCufwP9aaZiVOUYD+8SAB+tXZDvYkjiURDdnI4x+lNnBK8JgZz071agXEOeOcAd6jnJ2/T2rbl0M+bUzd2H5q5btmRcNjg/yrOd9sgHGatW8oEgJYAYOT+FQmUzSjOUJOSef51IPug8VnR3yDdnoOnvVo3EaxAtjFUpJkMnc8DAxz2ppd1Bwxzg4Occ1GssbqCGB57U/wCU5+lPcEyCS8cAJKuR/tDPWrAukmlSQFxKMhSAM84xjpgjH61Rlboc800qDhlOKycexaZ0emandxX628s6TRn753g447nt9K6rNcBptxDbXSG8hUqDwWUNj+v5V2keo2kkYf7RCATjJkGM0R0BlykzTUkSRAyOrqehU5FOzWghaKSigBaKSigBaKTNGaBC0UUUDClpKKAFopKM0ALRSZozQAtFJS5oAKM0lGaAFopKXNABRSUUALmikzS5oAKKTNGaAFopM0ZoAWikzRmgBaKSigBaKSigQtFJRSAWikozQAtFJmigdjP5ozRRWRYtFJRTGLmlpKM0CFo70lFAC0tJmigBaKSigQtLTaWgYtFJRQAtGaKM0ALRSUuaAFBopM80ZoAdRnmm5pc0wHZpKM0lAhcUuabmigBaKSkpgOopKKAFopM0UAOzRSUZoAWlpuaKYDqKSigBaOe1JS5oERtcwpPHA00azSAlIyw3MB1IHU1zviLxRFpnn2qShLpFR43GHDHdgow6g4HT37VkfELUvsVzaBNRkhfyzugizvZS3B9MEg5/3a8323N5IZXDKkikNJJ8zHnt3HSkk5bA3Y1tY8QNfSB5YwypLJJBCOdrMct9Mntz36ZrPZZbiQtdEordIx1x7+1SQW0cJby1AbuxA3fn2qfy0QbiQwz0U/5/WtYwIchUiIQIiBEHcDg/40SKEXy3AdO+R/Sia4lSweW3gikkDKu2RRg5Jznn2otpnlS4a4t4Yljm2IUTb8uOM8/X8qonUoSWTqd9nKQeuwnr9OxqSDUmhZEu4yGRgQ7DOD9D1/DNW5UgyGEuGI9v/iqhIdgVmQPH2LDP8uR+tKw7llJElnLI2IzKZJFjYjc3HXJPfHGfTgVq2MrT2wMjRtckkMqso2nPbJHQHqMng96537MkT7rScxEZOx87ckYJyO/1x0qMasbCY/bYmkORtxIVHQ5LY7dOMdutNOwrXO0eJkXoGGOMY4H9RQ0eQFKblPqMfyrIh8baY0Kb1CMRlgZQdv6Usnj+wVwNqMgweVJ9fQew/Or5hcrJptJifLxlomfnaBkN+nP4VNFc6vau4AeRX2+b5ecuFJIUjrjkjHOQx9aypfiHat88UUbE8Omxs/nmmTfEGxaM7bVQ2e6Hn9al2fQaUjvLLx5AiTnUYrgS78r5cZYKDwFwMkYx7966nSr9dT02G8QALICRgnsSO4B7eleK2PjFtSuY7SGxE00vyfKm3OfX5un19K9d0yf7BosMU95DPLGpbcAEBUscAD2HHrxzWUkkrlq/U2aM1zs3ieIoDCpVupDYx+lVrzxawVRaxjPO4uufpjn+dZc6KOrLAdSBWLqXiO2siY0HmyAkHBxtP5c1yV3rN/K7FrqQhx0VsDHpgVmSSu+4s5JJycmpc30Ea83iO9aZ5EnIJGBjtz2rHmuHmlZ5XZ3PUtyahyPUGoi5zgcntUDRN5mGA/SofPwSc9aAcH5mGPaniSMdAufoKBN2EMmzsee9NYuAGCn3pXmJwMYFIJXPUUaCv1LGnYmvowwAx8xOew7/AJ1feXfM7B2GOMBc/wBKz9L3fbJH5wsZA9skf4VZE4wSZefdkH8q0jsWibdnHzyfipH9KXdjn95jPXJP6f8A1qgE0TD5Jgw7nzcYPpxT4W8yVRkkZ7SlvzFUgehrp8tsuTk5NUJ7lj8q9jyasXVwLe3TIOQBmsSa5Z2Yrxu6nvWs5JaGSGTSr53XmojKVbrwewprOBjAHvmmrg8EDn865ZO5poTpKCc5596nE25SN3TpzxVF0K4KjIHXPNAYDqCp9qSbQaGhHOysMcBea0P7SDbflHIyeehrDMuw4/I0qyZHFUptbC5TTebcVIHT3p8UuBgnIrOWTAxzTjKykdwTVe0Yamn9owSCMrSjDkmMlT71neZubjPNWVlCxgE01K+4bGpY6pc6bJ+7c8jBV+VJ9ccV02j64168kd35MMgxtX7pJPbBPWuKW4QrtbBHvSKQSSpJH93PWn6DPUaM1w2neI7qyfZMXmizysjEsPof8iuwsr2O+tVnjDKGz8rdeDiqUr6CsWc0ZoopiFpM0UUxi0UlFAC0UlFAC0ZpKKAFopM0ZoAWl4ptLmgBaKTNGaAFoptLQAUUmaM0AOopuaKVwHUlJRTuAtGaSilcLC5ozSUUBYWjNJRQCFzRmkooAXNJmiigAooooAKKKKLgUKKbS1kaC0ZpM0A0AOzRSUZpiFzRSZpc0gFozxSZo7UwFpabmlzQIWikFGaAFzS5puaKBjqWm0UCHUUlGaAFpabRmgY6ikzRmgQuaWm0uaACikozTAWlptLQAtH1pKKAFpKKKYC0UlLQAUtJRQAtFJS0wFpD1X6/0NLVLVruew06S6t4EneLBKPJsG3PJzg9KQip4k0GDW9MmX7PG14sZ8iXaN4IOQoY9ASMenNeQp5gjyctx1C43f4V7rDJ5sKS4I3ANg15B4nEVj4m1GGGPcnmBzjHBdQxwPTJP6VUXaViZbGT5mFx5JHpnNMYuqMdozj0qNpQ21mJT2JH+FSF1dcbA3HBxmtSBsTSMD8xz+NPfDZ/ekfUE1XkmjQLn5fYcZpPMjZsfL/3yAR+NMCZSy4+bIz13f0qQ5DDaGBxySCf51BuGR8hyfoTU0YyCcbe9Ah8m0RbpFIYdWBxn9Kx9YiXyEMedznlFGcdeScAZ+ma0bxikKjcnX/P8QrA1tG/tO5AbICIBlicDA/xqZTUNbFRMvyAtyDnDdDkgCrf2cuMb4z/ANtF/wAaZcIwuZQcZ8wn144x+lAh+YncB9/HHt1/CiWJ5UrJaq5RHFHDGXG4Lzk9T/KknRNvykMR6Zp6qArAtg7en4ipntZJFbyo5HO4jCqT3FU60nPktoHmdT4GshbRyX7oNz/LExJOB0P9RXWy3LAY5x1yO9cjoV62naVbhrOR/MkZWYEjYM9Tx71s3mpCDy441Es8n3YtxBI7nODXPVjJy0BMutJnkUwSlQRnr2rNuLq9t0LS2UKMBu8s3Pz/AIAqM1ctQ13Yfa4sOvO3YGYE+mQPwqFSn2DmSHl88ZqJzkcH6nFRmV1JVwVYdVP/ANemCbOcZJqHvYNRsjMo5ORnrQhO0EDr2pWk+YNjI9qYbkZ+YD64qStSV+cbtuQaRcqoyQDnJqs021yeSKa8xOCp69qLisyyxkzxg+mKaWdCFyD+lUxcOzBc9ulTvveDcVII9aNws1ua2llhb3c5HGAMgdSM5/pQGwPvTfnJ/hUdiPK0ZXBZXkYnO3PfHp6CpA5x/rpf+/X/ANjWyVkihkSqgf5pBuYtxvHX8Ks25XzAVEjH+8Wbj88Zqv5hz/rpf+/X/wBjVq2LPMoYuwJ+8w2/pxVRWopPQNekCzRKB/Dn+dZIfeOCfpV/W3zfKM/dQfzNZjyEAdvepq6yZmtiY5545HrUYPz8rk1ElyWye2cZpvmY+bn8axbRST6lwEY6de1ACkbex/SqYuDu6Y+lSeeepz9afMOzRZwAu0nIFMIwAOp65FIjAjuM08DA4559adk9guRCRgfap1mzjGTmmBt2V2jr0o2bckHaDSt2He5NjIypNPWU8K/6Cq6kp0/MVKJkOB0PvQA4EDqTzVq3lC53enFUvNXkbQT6Gnqdy8DGD0oi9QNhGSQAE5z61Is11bqVgnkC/wB1XYfjwRzWTDcEMPm6dRV6OfseR61spKW4tjct/FV+qBXjgfYBuZ8hj29ev4V1ltdxXUCSRup3qDtDA446V50wVwMkH6HmpbS7m0+XzoCrY7EkfoDTs1sG56PmiuTsvFF1LPDHNHb7GcKzAkEDuetdSrq6hkYMp6EHINCYWH0UmaWquAUtJmkzQIdRSUUALRTc0ZouOwtLTc0uaLhYKXNNzRmi4DqKbRmi4C0UmaM0hi0UmaKAFopKKAFopKKAFopKKAFopKKAFoopKAFpaSigBaKSigAzRRRQBnUZpKO9Zljs0U2lzQAtLTaM0AO/GlptLmgQtApuaWgBTS0maBTAWikzRQAuaXNNpRQAtFIKKAHdKM+9NzRQA6ikzRmgBwozTaKAHZopM0UxC0UmaKAFzS02jmgB+aSkpaACjNGaSmA7NFJmjNAC0tNzQOtAGbr+sNo2nCeK2a5ndxHFEpxuY+pxgAAEkn0rzWTxd4tdz/pwiUqDxabQDnGPmTOc8cE13HjVd2mWv7suBcjOFLY+VuSARxXECEsgEkbHjH3XPfoOTiiCu2S3YBq3iMvHLJf3vmId2D5iIT2yAMEeueKxby81OUXcF5ql5IqQmVg0suCBzjDEZyPbFbTpISD5RxuGS0Eh/Prn/PrXMa5OLC+ugFDCeDyyAGXbkY/iGatqwk7k1jJcPBE5u/MZhsYNIQQpBB6oc/TJ61ZjtVgQrDIojZhhPOkTb1zwIsY6dKXRYJH0uGVQ6sykA7CeMnvn29KXWrm406zSeONp13hSG3KMGraJv0KVxpt60hMdwAqhRhSxVzxk5K5HfsfQcU0eYhdw8gXGN0qsM8em3+tbaRH7Ks0jOuQCoYE59vve49KzNUQRCVsYKqflIGR/48aLWC421LXCkee0ag5Lgtj9V/zkZpqRTSSgyNKEY4I3Dgex2807RYWurR3EJkUOQzBDxwPQ+/1q9ezXdnpst1b20gMajHyuAe3PNPzF5FFrZ3jRIrmUknPRdqj0A28n3/SmWUepG9ML5lUr1UHb0xnhRWhpxlubYOFZXkOGClhjnHr7Vo2PnpexOxkZAMY2Sfz9afLqF9Dn/OWeNFIcBvvsjc4z2yp9e/5ipP7LS/mMk0jCRgoGJIssRjg/KPTtnJ7VDobMdSCxeYzpGWAVWJ69flOa6PEhcj9+rEt8+ZvT+7/D/T8KXLfcL2MUaRAEk82VWcfNuDW+w5POcgEc49an0/TYDI0awWkrISNxSCYMDnqVZTyePcetainanDyk7RuZ2lHPqM5Ht74/KxYeXPK24gOpUqTIw9RwMYqox1FJ6FK+tGgsxLGgiuYArK6IVEajuQGweFHHvnFaFnB59sl0ZSzuEkYqAxBx0AwvH6+9MnEr2V+WzhQwj2gEkeWO477t3WixiefRrdFkKkeUdx/2HUkdv7pFa8upnzaGbbRlda1C3ErJEoV1iDMigsOSFGQOo6enpVbxIIpNNWRSz7JBjfIJMdRxkdQT1z/KtBIpYvEl0xUJDJAMDOORgdPzpmoaYZNFNtHvU+YWBIHdifUDHJo5dAvqaEYt1uxGEzKQzdwSMgHOeO4rN8PTiDQkdiGYTiMArtxucAcgju2c+1bKxobhd2PN2Hbg5O3IyfXHT9Kr2toujaYzOxMCyfN5eWA3Mq9PqR/hQ1qJPQqaiMXeWUglM8tnue9VPOXGMHnrxVvxEwinh2lCMEfKc9P/ANdYyMcFuM+9ebW0mzpirxLyyptPGfX5sYqG4jOchcg91PSoxLHtBYDj04pxmjKEAt83PWo6Bs9CBd0jFVJAHUmpvLj2bB37k803gqOSAOelKAM7ifpQkNsiiYIx6Ag4yc5pZX3FQpJyec02dFIyoOe+KfpkLTajD1IVtxJ9uf6UktbFJ9ToJ0WGCG3OAUAPLYPA/wDr1CWTu8Kj/afNFzcn7XJ8kRxgZaUD9KZ9rIyAtsD/ANdx/hXQwQjMhOBLbjPrg/1q5pTrLIjrE0eTkcjHX6/0FUTd5OPMtR/22z/StHSkCbdo4x0H3fw9qcFqTPYz9YkH9rTLyQNv/oIP9apSAlOGOO9TapJnVJ+4LAAfQCqO4oxyfl9qzqfEyEiwuxTjAHt2pXwRyADjiqm7J+Vhg0qyEDBOfXmsixfMZGyxC+9TRzhhg8+vFMR0Yj5flHTI6VL5aqQdg/KmkDa6khCsDg4NODiNVyc+tMBw2ATj0oJA4YHnvVWJuTkg89+vTrTQ4cFTkH0qJXQEAMfbqak+X0yf50bjQ/cwGDxS4DrxjdTFJJI/h96MbeVDYpWHcVVK8nrQHKOTmkDh/Y0jAjJDAipfkNPuWBKp64PuKkSby+AxIPrVMg4Gw4qUA4wSM0JsC5HP8wCnjv3q4sqN3znpWMSy/LyM96micoMk5/pVxm0DRrsivywGcdR1FS215dWE6Swt9w9hwfrjGaoQzluC3brmrQlPUD61tpIVzo9P8Uu06i/8hImBIeMNkenGTXQ297bXYJt50kxyQDyPqOtedMiycqArU+2v7vTrgvCRCXG3OMgj8c+lKzQ7npOaXNczY+Koimy/BWQfxoMqfyP8q3ba7hu4fNgkDpnGRxg+4PShSCxY3UZpuaM0wHZopuaWgBc0ZpKKAHZpM0lFADs0maSigBc0UlFAC5opKKAFozSUUALRmkzRQAuaKSlFAAKWgUUALRSFgO/60wzRggbqYElFUmlct94/QGkM74xuNK4F2lrPWRh0Y04zMf4qLgXaKo+Y/wDeNFFwI6KSjNZFjs8UZpuaKYDqKSigB1GaSimIWlzSUUALmikozQA7NFIKM0ALRmkooAdRSUCgBaKTNHNAh1FNzRuFADqM03INLmgB2aM03NLQAvFFNpaAFzxS03NGaAHZozTc0ZoAdmjNMpadwHZozSUUAO7UU2lFMDC8WoH023ygYi4GPlVjna3TcQM1wChI8fKque4Qrjvj5XA7V2PjbV9Ogso7SS6gNyJlYwblLAbW5IPQciuFaa1nk3C4VGRSF2NHgc54ww/X1PrVU+pEy/kttJTI3Eq4ic8j0w+R2rIvtMgaSWT7Gq/KMAxY7fj/ADq/PdWccmI5Idr5A+WLbj/vrj8a5XxO7XCRpYz7SrfOBMg4P+6xGOP1rS9tiLXOi0mGQabE8aq6KCTm13DgnqSwzjpTb/ULHTViluvkQthC1qMjv2PQ5/SqXh66t4LBR5vlTKzkMwhHfg8kEn61HqUWnXxEU8uEU7s/u1LH65Apt6hobEfkyJGyBQsiAhhAgBH3skbsnio7uw+1NGjJMp9Ft4wCDjH8XP4mqtrqVlDEio6sETaA6KO2Oc81R1q6a40wxWNy6TK4bjYntxgDPX+tEmC1Nq30tbXKJbmQEjPnWykg8Dj5sVHeTW9jD9rmAREK5dbbJB6Do30qKw1CP7DAlxLGZERA/mpHjO3nk8nvzz+tQX91Z3kDQyykwsAAAkfrnjbx196aYMltpY7uMSxktbs3Cta/KxHH976j/CrUsBaMKiyRvjIYW+GHXOPnx69qyrK80+0tPITeqqDz5Ck9c8kj61PeX1vJYXEcNyqsVZV2+WnPYjv1H8qcmhJEtppYhvA5EjsAQVe3TB44Jw3r/Wr7oCjN5S5A5ZYGHB4Of3nv/wDrzWDol35dmVuZSJVPzFzG2fl6/NzycjAFaD6jaMvRN+OcJGDn8qL2AtqrAF0h2Kep2vgn2xJ71oaUGa8bKyBtnGN59OeXNZLXtq3Rxl9wyUhzg+px/X1q/oU1hHdMxuLdQsfO14gOxxj8OxqotXJa0NSMpcXl1aEMTAqlyQCG3g9uvbBpJpUGh3E1mN5it3eNlwyllB6A++eM02fUtLhvnmEq+e6KrBm3ZA9Bg89e9Z9jqccVmtpbx3E+2RjmO3PIZj6/XtV3Isaj+SL6CIRgSSRuysq4BA29QT/tCqskd1YWOoScF1eaWHzmIBGMjvjHbt1qJ11G5uUljtUgIRliaRuFU4yNoIPYflU32DU5Yv3syRgH5Xjhzn3y2P5d6OYLE8tuk11b3iSxCSKNhiPOW3bf4lyB92ot9r5V1BfzrgzM6NI4V9mQRhg2QAfb+lMi0MXUAlknuJQQMM0hCnr/AHQAPzq7H4ftbd1ZbSNSSBllGTx2Y5P50hnP67d21/LE9tNJIELZLJxzj+IAZ6d6yXBRSAeD3Brq/EUMUcEYC4k355BB5B9evTtXLhCZCMFmHOa4a6983py0K/llhgHAp4j7cr71Z2YwXwOehpGQyKcIOBxjvWPKU5jVbCgKR7470wsVJJ5HXioJFeJgrRkE9PehZWJ2tkY9sUAl1HtKOxIzWpoAX7RIzHO2M8DsTWPt34zwM9K2NCG83ZUHIj+UDgnrVQWoXFeZXYttAJOSQiH+bUhk5yJlA/3Yh/U1MlpnpHKv/Ah/jUn2N8cK2P8Arof6CtLFXKxnYA4nQe26EVt6W2YlOCOOcnOTxWS1rhSXIx/tXDAfyq1YXEcMUwVogwXOBjJH8z1q4aMmeqMi8dLieWQN8rOSCPrVd/l4AyB0JquJGTCEAYFP84cjg1jJpu5nqiSSQIMbcjHaoGmAyAgAx6U10kDfIN3tUQSVsMUbH0rNmisSpcHcAGIAPPtV6GVgctgL3z1NZIyJsYxg8irIkY/xUIckmahZWG44z14yKDllxkMO9U1diuwkkdAakRtm0FgXFaJmb0JI0KAnFSq4Ix6GoxKmCSCc9qAu1coe/ei3YLkwdWJGSW7EUucY6/WoyVxkgDPpQWIUMpJx780WGTbhvOf/ANdNEqhtpU4yaRZlYZ709iDgEZBpWuO9heCvDfrUfII5OKYyMjZQrt9M9aXLo3Q47g9qgpMk3dj+poEyqxUZ+ppgG4Ekce1NZA3OcGkMtRz7+c4HtUwuiT97p09KzWEigAHpSCQkgMcDrRzMOU347kOMdDmpmm2qQ+CB2rFhlwvDEnHYVOZH4J71rGo7E2NEbH5Tr/dNXNP1S606XMTnaODG5O0/gD+tYkc5XpwR3Jp4uPMJ65NHOmtR6noFn4mtZwBOphfHJzlenrWpbXkF2heCQMAcHHavKvMbbnPFWrW/mgIaOZl/3HI/lRzsD1HNGa4mbxLcyQQpDIyOqAO+QS57mi01y5hglBlZyw+Xdzj8avmC52+aXNcrZa5PG5kuJSYsfccc/wDATjmn3Hi5FOIbYn3kbH6f/XqopvoS5JHT5ozXBT+O7uJ9pitF7gEknHr1qGL4gX8oDQ2UEy9SyqyqB67iefwBNVyMOdHoeaK4+Tx1FDGd9oJJAo+WOXqfbIzj3IH0rFu/iZdQybI9Lj56ZLMSfoMUckg5kelZpM153D471+4TdFoMZGeWmYwge/JJI/CtuLxbItuGurONZe4hmLL+ZUH9KORhzI6rNGa46Xxlch1KWVuIs8lpyW/LaP5mpU8Zbm2mzH1MuAf0NHIw5kdZmjNclL42RMrHYmRwBwJeF+vHFU5PF97JnBhhH+yu79T1/KjkYc6O5zSZrzuTW9Wk3EXUu0fxZ2/ouM1TOqXTnDXtwWP3Vy2T75BJ/Sq9kyedHqNRyXCR8dWHavNvtGs+UfMuHEGM4MrKPUE5JP8AKsS++IFjp0TCfxAHm6iO2BlJ9srgA/UilyD5j143TF8jgelPe7ATpg+vYV84X3xQvjIWsxeOhHBu5yB/3wh/9mrnbrx74hvRtfVZYYzkbYCI8f8AAh8x/Emp5bdSrn03f63pungtfalaW/f99OqH9TWdZeNvDN/O0Ftrdm0q/wALSbM/QtgH8M18vrc/aN0k0pkfOWZzktUti6f2sqKeCDxScQufWsVxFMm+GRJU6bkYMPzFP3V4P4O1a70vXZHjlJtjEQyqQMHI65PNblx8VL+11WVIbe2vLVGCBS21j6kN068dDT9nKzYlNXseuE0Bq4Of4o6Qnh641CNSLqJVItJXCl8sB8rDIPc+uAeK4a5+M1/dKypZpACOMSkAfkMn8xUqL3Kue7ZPvRXz4fieSc/2ahz3Mrc0VXKxXPoLNFJRWJoLmikzRmgBfrS5pM0lMB2aBSCjNAh1LkU3NGaAHUU2lzTAXNApjOBTCxPelcCfNJu9ahBozRcCUv6Um8+tMopASbz60m4+tNBooEOzmikzRmgB2aAfem5ozTAkDU4H3qHNOpgSZpN1MzilzQA7NLmmZxS5oAdmgU3NLQA7NGabmlzTAdRTc0uaAHCgc03NKDTA5Px/pEGo6VFcu2yaBtqNxjB7HJHoMc15a2h7PkDw7ySMsACPUffr2HxiR/wjsgLKoMijJfb69PevOo4yXIM6kYOA0oGR04OOaqmt0ZzZkR6BG5Z2eA8dQwxkc/3uT2ph8MyGTmKAegyOQf8Agf8AWugKlgxEgbG4FvMQkHHQ+ncfnSGKQwgDenIIKtGWOMjPJI/zxWtibmGfDO4ZjiUqBlgpGOvf959OlH/COIhJZU+8SAGA69x82TW2isrhJSXkPygbF3A9uQCPTp6gdSKc8uwndKqB/mypB3e5I659efqaaiDZix+HopUZQQxK7QA3PHXgMfWnjw7bMGeQRHGejPj26kY/OtA8s8fm5J5AboMjrz/TJqV5JHWNVuJS/cRyHGMfyPrjvg9aHFEpmM3h2IMGVrXH97zCRnvzup0fhhXQZaINtwBnHB9PmrTWaQqxW83LgfKLjlR+PJzTQ7qQvnEq+cKZccj/APXTsFzNbwxH5jMJIMYw2TyMfRv61ZGjTsjM80ZEm0ndvIZu3PrjpitJHkMays8/JHy+ZkfyHPfvgYPpU5naLmRsFRjcDz75zyevvQ0FznX8MHBlUwyxSSZ3L5jZx7gYxz6+tJJ4Xk25QYXO5SUfg/lXQjyyH25Y5KnIycdR0z3yOevpxVfYJEMsQIH3nATAHPfH8v8AIFEHIwT4VuInVo3QGNiysoOAfXNX9A0Rv7QZZlTYAZNu0/eyBnkgD86sTysFPCx/PnmMsD+G4VP4cbOoKHVWcodxKFc856ZziqjDUTlodLDo9nb5dAFX/ZUIT+IBH6irUEdtGG3x7s9GzyePXJ/QfypwaNcFFy2cnC44+q89u4qjq2pwaXpcl7d8pEQMDlmJ4AA49f8AIq3oSaM9yIo0ZIQScDJIzyOMYx+uR7VKB5ckjQgK2cFhwD9ex4rCudVj/wCEck1CGQMkcJuFCk8/LkfTtVWHxLLfeDLjVVUR3PkSjCgkK4B5H6Hn1qdmx7o3p5N1vcM7KB82Wz82PrnFPSRJ1jmi2vEyKUZSCCOcY7HtXFWN/d6h8PL4zuTcJDKjtu5YBc/j8pHvVvwPcmXw2kbMrmCVohnnA+9z/wB9YprXQGjQ8RFhY45Cq4IC8Lnp06g81yUsp25U8g112vB5NLuB3CBuuTgMOh7iuJBXdlySD61yYhe8aU9h++SQnPzfQ1Y3COMjLBj1zisu51vTrDzElvoIzGMshcbhxn7vWsO48c6VEco0s+7/AJ5x42/Xdj9KxUWU1c69nSTCufpxVe6aCGJppZ0iReWaRgqj8TwOTXmuoeONTu1eO2VLSNuMp8z4xgjcfzyACOOa5yeea5maaeV5ZW+88jFmPbkmtFTvuKz6Hplx450OGJUVprgN3hiwVPvux+npVAfE1rJLuOw04P5qhUkuJCAB3yq/iPve/tXn1FWoJDSsdXc/EHWp4QkSWluwOS8cRYn2+csMfh2otfiDq8ERSaCxu2LZDzQYI9vkKj9O9cpRTsh3PRrH4l2rMBf6RGnycyQoJCz8fwkrgdT1P49a27b4h6BNYzh7iW1bYVWOSBssccEbdwx9TXj1FFgPY47yK5jL288Uyg43RsGGcdMjv0pEkCuMnn8sV5BFNJBIJIZHjkHRkYgj8RXa6D4rt7hxDqzrFICuyYAhW/3vQ989OvTHOMqbWwXsjsdzIeRg+lSpcsuOcevFPSVWPzHd6HGaa0UUmBkowznBz+lQkRzLqiZmhlXLABqQpHHEx2kHIxms5X8qd1J6HqBU5uCuAkh9QaTKtbYsCRNw3EBscYpQNzgux9MiqUTkL93I71b83IwCQzdAOTQgkSN8nAOc96eHwBjGR271AsjBwMDB9adweQR16iqBeZcBwBgrj2pytjjGfes0yyK4V8Z+tWg7YHT65p3E0yUlVbOefWpEcZ5IPtmqckv3Q/J7mnICxUKc56VN9R20L25D3Bz2oYgJkcCpIdMvZFz5flju0hwAKe1vHbozz3kIQDllyePqcVShJ9BJ2M/7SwOepzUqy+auPun6ZrPv9Y8PWhz/AGtGT3VUJz+K7qpL4t0QBzBdNKyI742MuQqlsDI9v89aXsZdS+ZHQlc8EZ9aZ5D5G1C3tjNcS/xJcnEOmAKOMtKM/wDoNNTx/dT70SDyJCrHeGDcAE+gP45qlRXVhzNHfxWs24K22MHsxGfwHWrsVjK0X7zzFPckgA/jXk8Xj7XA5KX6xAk/diQHHpnGf1qtfeNfEkpIOrzqP9khR+grSNOCJbk2exSWllbwme4uwqd2d1VfzIrCvPGnhvTv3a3K3LjgrbwmTH0ZsD9a8YuL+6u5C91cTTv/AHpHLH9aiEvJ5OKr3Vsgs+rPSdR+JsxhZNOt/KHIDyydR/ujGD+NccPE+uz3Rnk1S8Zj/wBNmH8qxl5OSPpVy2Co4PIobuO1j0fQ/GsEFup1a+SV8fcigbcPYnhfyB+taTfE3SEwLZZI2B4d4yT9e/8AKvI4X2g7iMn1oeVQwAwSfSlGKWwM9Uj8dR38phTUgMqzElCmAASTnaOgBPr6ZqjJ4x0lpSZry+lYfxBMBvxyTj06fQdK86tSwmYfdzG4yTgcqR1psDAkKQf8avmYrI9Mj8W+HFUeWwiIOQ0lsX/nnH1HNTT+L7C7jCpqbO6gnCJIQAOT94cV5mEGOcjJyQRVvRY9x1FugjtSc+nzoP60KTFyo9Di17QIwDPqNw4Iy0aB8fnitOPxx4eQLHDJPEQfvCLoPTPPFeNXspV1VTwRzg0ls5AYtnB96OYOU9qm8a+HTiQam7EHaEWGTJJ7AEYrBk+J9koxHpc7p23TCMkfgDivO4/3t1DGAzjzQzBOuBycfhVAsQ464ocmHKj03/hZlsuSmjSncMEPd8f+g1DN8UpoSFt9JtQo/vuzH8+K86Zzkbc5pjse45Peldj5UddqvxG1++k/czrbQgD93GgOT9WBNUh468S7QBqbAe0SA/ntzXOu+wYYEZGRnv8A5xTBKNgUGk2x2R1Z8e+J4tudRD+zQRt/NasW/wARNaiMhaRwH5wk8q8/Tdj9K49pgQBnmkeZdqjIBHX1NAWRs3Ws6hrrs99dzSqONrMTj255/OqckSpGSAQfWoYr6JBypB+lJPdRSqoUnGfTmgAErfKTxj+tMkC7QV/H3qAyhj1/SpAU9z+BpDDJXgMcVYtJxDdCUVDwwyqMR6hTT1ilVdwglYeyHFFhHbadcotpc3RI8xFBjI7E9KwVvmkfAjKsSSSp5rOS7ktleNlwJFGAef5fX9asRXisURYZGY8YVSSc1pfSxKWpYv53axQeZlXwMdxjPFZqTiPKknp2rq7bwD4n1qGE22mM0DnMVw80QVlPc4b/ABqGL4d6rFK6XstrAAOCZ1XJz7/5+lT8WxWyMD7bjpI+O3B/xorpl8ABlDLqVqVIyCLmHn/x+ijlYuZH02KM03NGa5jYd0ozTd3pQGoAfnNGcU3dRuoAfRmmbqN1AD80Zpm40hb3ouIkLADmmFs0wnmjNADqKbQDSAfRmmilzzTAdRSUUAOopKWgQZoopaAClpKO9NALRmkpaYBmlzSUUALS0xmVELucKoyT6U37RH5ccilmWQgKVUnr046igCbNGcVkQeIrK5vPs0Xms28rnYcfX2H1rWo8xJ3FzS5ptLmgY6im5pc0wHZpc02jNMDD8X4OhgEkAzKCQSOx9K87m/dYbzEGUIY7guOf93jtXoPi+UppCIoYyNKNu1WPQHn5enUc15/JI8chYlkJBIw7KRz645rSluzOZP55ZfnlizzyWjz9MFaQKMgHYFz0KxcE/hUa3chLL55DZ6+ZI3H/AAEYH9KPMaXGbrIyMnfJ/hg1qQh1wwY7lb5UUKMOOncccdTj6fkfOvEus3dxdSRpK6QjgKpxke/+FegSH5AGeIgf3WcZ55HtXnGtQ4uJOAMGom9Bx3KOl67d6fdRFppHgBG6NmJGPb0P0r06O5jnQtu3ggEyM/UfyPUenU14+4+avTfD0rrpdkcFyIk2k5wOPQf54pwd0VNGndysI8uyyBvmPORnvj0Gc1WhlDEAgqfpwKfeOwGzLOpHQ9/qCetZsYffuYEgcg8DB/OmmRY6UyoZHdigwpHYK2OvT3YfgTx6+deJ/EeoG/eCCd4YxxlDtZueuetdSt1KU2rMdo+bG8jn+XYVwfiTnVd2MZX1z3pTvbQcFqa/hbxJd/bVsrtzMk3Cu/LA+hPcf59a7hnYIUBU5JLEqD+X4GvK9DXOtWfOP3q8+nP0P8q9OUFF3NKeF5JJOf04q6crrUmpGzIJFQkgk84yAcHBrQ0Bo/7QiQnnBGTz27kj+tUbuc79nm7hjOAxbH51f8NhUvINmMlm6bhj5T3A29q27GZ17Dox2A8dcD8sk/oa5D4k86BajdjdfoeM/wDPKT1rrVVWLODhvcY/UZH54rkPiEmdHtVxgfbEPznGPkf8KU17oQeoy1dn+GNxGTuARwe/8RP4VFoIA8Baum3CqsxGccnyx/8AW60lsQPhxNtJGVfjqB8zf5/KpfDkb3HgjVYo8mR1nVV6ZJiGOvHWo6v0L6fMj0Bl/wCES1uOMKsZilOF5H+rq94CydMlIYkrct2HHyrUHhqyuJND1O0f5PO3xhnYHG5QvUH/APXVmxbT/BmjSXWq3awRNMMllydxx8u0ZLcDPGe57GiOlmEtbo2vEF3FY+Hb+6uJYo4o4XJdgMBscYyMZzgAdzx3xXz94h8aahrTGGEi1sgNvlxIqNIMEEuVA6g8qPl6cE8l/jfxnc+LtWaRfNh0+PCwW7OcHGcOwzjedx6dBgZOMnlqym7suEbIKKKKksKKKKACiiigAooooAKKKKACiiigD0DwZq32ix+xu37+34Uk9Yz079jx0xjbXSO0hZScfLxkDOK8ft7iW1uEngcpKhyrDtXUWHji4iwl5brKPlHmRnawHckdCfbispQd7oXmdvhJ5CVYhj1yODUjBI2wcdPSsOz8RaRcSJsvFQldzLL+72+2Txn6Gtie8tIJVEk0YdsBVZvmYn261HKxNk6yq8fysAR2xSZYtlWII56dPWua1HxKsDBYYw7A5zuwD/jWFqWr6oLua2lnkjdJGRkQ4wQcEcdelNQY0elW9vc3IYxwMUHV3IVVPuxwB+dXUg0m1QvqOuWMAHVYnDt9OSM/8BzXlbWOsXEFpGtneT/usoiRM2BubsBxnFRNoetMQDpt0Pmx80ZUD8T0rRQSA9C1Txr4Ts2VbC2u7+aPpISUVvqTj9F/GuV1H4iavckraCKwQ/8APuuD+J71j3Hh/UYceakCDv8A6TE2PqAxIqynhGaREc6vpeGA2jznJ5z1AXjp39vUVdn0QaEtr4o1fySzz+bg/ekUMf16/jVyTxRrHnhor+e3GBlLZ/JU9OyYHetCLwLLp+gtqs93ZXlojDzoklK9WCqFP3ufoMDkZpYLPSRNuOlwPFtyVLOTwOxLH+VNQZLkjK1LVtUbSrS7l1G8l+0zTL887sAFEfqf9o1ixpeXW54oZJWbjKIST+Vd/Feb41iW3s1SJCsSJbphM4yemc/KOfakn1W/Wzn/ANKkULGxXZhcccngVXKJSONuvCmt/wBoXNvbaZdyrFKyLJ5ZwygnDfQjvVnTfCmrQ3bNfWEkEJglQvKQoUtGyqSSegYireoazqP9oTobiZIjI37tXYDBJ7Z54qhqF3PKuHkZlAyF/wABRyIfMyx/wgepxYMl1YohYgkynjHrx71Pb+E/scjTXN/ZSo0brtiYu3zIRkD1BPHuKc13FDbqIfusuY8DBKdvXHH1qs+oSyKvzZG3AJ9etPlRPNIqx6DYO4J1CUhWG5RBhsd++P8APStAaN4fYASTaiG2DBDJjJJGeR+mazRdfP8AKmwFQpORneO/06/nUvmyRwskqggEBT9PT86VkNORZGm+H7W0MjwahcSM/wAis6BCB6svIP4VV/4lcQyNFU8Y+e7c8888Y6VYtRC1v+/uBGznOGcDPpwaqHy4y4aVGAOAc8e3JqSgjvRb/c0uwIByFeIOfzbJNdNpmt6I+mmKTw3pxuG+UyrGimLcfvYKlmA5OFI449CObt4zdyARK0uOdqLk4/CtA+H7xdNvdVns5LWGIBYhJGRvJBLFeOcKGzxxnPHUCV2DLV1rkBvropo+jsHmY5lskc46Yzj8elV1v44YGul0vS2cEJuayTA6+mBn8O1ZkRimBZGyDyfUZ9asTs6+GnAJ4u0GAOPuN/8AWrVpGeppweIUifJ0rT4Gxw8MCRnpnBwp7454/qH/ANv6aApHh3TmfIJZyT+WAP8APrXHiYZBPOKeJDIzAZwalFWOxg1Twxc3DW8/h6OIEFg0U78BQT3Yfz/Oq8U+miSR7bTFSJlKgNJuJBIOGyMEDHT9TisHTNi3c2cki1nOSM8+U9P06eRrbEh+bOQCuP09M1ULX1Jnc1o00VkG/RYnbJJPmbQec/3fSmS2vh1zIP7ImXJyrC9bjOeMbcen5VGrOBIGO7nK80zfmL7pBJ6GrsuxF5E9laaVYxm8FszypKECNIc7W35OcjoB19SKs21xp0ERVNGtWAOcuu/8PmBP61QiQG2mkIXcJI1GT0GJP8/hUIceWdz7c8AkZxTjCIOTL8Gs6fYKd2i6bMT1MturHGegGMDrwcZ9zipo/FUExIttC0qDHdbXDD/vkgH8q5+eH96yu2eN3A7fjViOJLe2KoVz1I7jtinyK4czsJqEd9eXLX628LRSAjGzhSijOB2GCPzqi80btGvkxDbwD82Ov1raOr3NppwgjjhEEwkjywwc4Xd+m38q51sc+lZSWppHYkecyztK6RoNxO2Ncfz7U0JHK2WfKnsODUeR+FQnIyy4xnHWoZSHSSxZwqDI7nmpbW7ijf8Aep8noADVPAJOTjvk96cZYjHGoiCsudzbs7uako3xfWBKmK1hHHJOcn8alg1aCO2YTQGZ8/KwYKAPpjmueRkI5z+HU+1KXzkhsD+7T5hcp0lnqMM8qq6onpjNdE82lQRJ5pcTlWPkhSxbAB69gcg5rz60YGUDd8xPGSAPzNdzcTWC6As80aRXLSBbaQAnbk5I45xtzxzyKcJESjqOlezDjNmyucMo8zII9+AakgubSRkja0TzC2AQeg7dQaxeUumhSRnjDb1xnrxzg9KuxeUwaTco2AqOvpx+tWmJrU9e0nxBFb+CJLhTmSxtiCjqAeB8nA7EYrxu41jULxX+0X1wyO24xmVtmevC5wPwrWm1u3j8GX0QTNzJGIzIy4yPMXjrxxg/n71xTXLFjhs47jpURSitC9Wjc/t/V1+UazegDj/j4f8AxornfMH9wfkaKd0HKfXJbFNzTc0ZrhOgfmjNMpaAH5ozTc0ZoAdmgnimZxSZoAcWz0pKKKYC0CkpaQC0UlOFAg6UopMUtMBRSimkhQSxAA6k04fSgQufelFFI27Y2zG/HGemfemAjSIm0MyqWOFycZPoKdXI69BrN1ewRMYIYQrEPHKdpPYEcE4xmtDS9WYTixnczzLgtIsb7QD0Oec8+4pxV0Rz2dmjepaqXV6ts1urBszSBAAOcdz9P8agutcsLXaHuIwHfYGyCB6k8jApFXRpUuK5ey8VQwWO6/Zmm8tpuFx8gLc46gDb79RW3pmqW2q2MV1A+VkBOM+nB/ImnYFJMbrcs0GlSSwlgyEEhe46ehx169sVn3PiWEac13GrAIcN82DvBGVAI54OSfSuO1nxpdia705Ps89uHeMs8ZIYDoTnoc849R6Vx93f3N3bhGld4923buyBx1A/CpV2roxnVs7I9VXxvYC9nV5wYSieSFXB3H73Pt/Q+2eVi8WJawTRRLdbmwquXC9CCOAMY46DGfWuXSKPYIxOSSN27HA49Tjrn3qI299Axm81I2TLB425yMdD6+lW42dzn9u27XOoXXmudcvL4xykTLtYCUoRwu07h6Hse3pmt/QvErX2rwLJ9phaVgiW8bKyttGCT3UDGcZ9a8+sdREUbGVGZ5MqoDMo249VI44/x44qaPVLrSpTNaQj7TKMrJkkqBySf89iCaFZWLVR31Pd9wzjPPXFV4b6G4maFH/eKMsp4I5x0/CvP9L8T/YtKMhuori9u1BQqTJJngAY3N79cYOPlxW9Y6g1l4j1GK+uj5YiiZWnwm35ckDBweT1wP05Op08yOrorNg1m1uAzQv5iBtqsvR/p61Ol6jHaSu7HQNQVuW2dUUsxCqBkknAFYFz4w0+2vDblXcqCWK8/wCf/r1k+J/FkEEEtnAWeUf6zaOmO2fWvNZ9U8/LNEdwYng/e59KSu35GVSpy6I7nWfEMetQ4U/LGwIUDOPrisdpDLv2lsgY6ue+ePQVhWc5WX5W2KQeDzzx9DVuS+NpKWkk+U5B3McfXit4R5URGbmtTQmjxyTjOT94nHHr3pqsPdj6kn86hh1SG5QLE8ciDGQwl59cckD/AD1xUrS4jAkMbAIACplABz6c5/z6Vd2XYq6hey21lPNGQzopIDNgfj61yF0j6lcIJJIIHkGXaRiqKevJ5Nb2vyKfscjywQRJMxkJLZOBxlccD/H2NaPhDRUTS5dSjFjdTTTuUM5IZACNhAIJ4IYnkZz14FXWVKGH51dz00/r7zSEVu3Y8suIzFO8ZIJViCRyDg16N4fmP9gWpUHIQDgv2z6cdqZ4i8I2i6q95bNB5QCtJbuzD5sfMTtHcgngjr+FdDosVudMTbBHBHtICQ8AEMRyx5PTvt9mzWNFSmr2sTUkkYC3kN0zFJ18ztGmHkJ+h6/hmmmG5B81JY448/K5QZLDqD8wx1rXumLazPHI2IBAwVDH07/xEgjPru+tU7qRYraBDLlGJIDOExnHTg+3pWvJZXZl7RFUfupQjE5wDllPI4IOO3WuQ8Sqw1BCx5wecYzzXT3Vyy3jZlcgYA3ZHGBjgdPwqZJnl/d+YxyQcZc/yPSplG6Li+pxOisV1izIHPmr/OvTUYeWcEk46BiO361TSAKV2oWO3klWHOfc9fpU65BJaTAJ5whx+WaunGyJm02DqGXai8jHY/jxjmrnhvcb2IbWI3NyqN/dNU28sx8uucfLldxPPbmp9HYjV4wcqQ/Ycn9a1SM29DtU+8TuO38v/rVkeINMGsWSwRF2JlV8KR9OAcD+LtWqpwhJVhjnPUfTP9DVS/mlFs8lqAX8pioOOuODjn/CnJaERepTtNOjsfD40+WPzlJYNkHJySe44/EVZ022gt7JYodmyQkkLyMjg9OM8YOPSm6VLLd6VYySRSyStboZAigEMQPbA+hxU+laWLHT1tZZkKxb2DHqdzM2OvGM9Qe1RbUu+hi6z4ptPD2jXGp3MMzqkzW0MI48yRWYBcgYC/KTz0HqcCvBNc8Qal4hvWudQuGf5i0cIJ8uEHHCLngcD3OOcmtr4g+IxrviGSC2kD6fZO6W54O8k5d9wJyCw4I4wBxnOeSrCUr6G8Y21CiiipKCiiigAooooAKKKKACiiigAooooAKKKKACiiigAp8crxMGRsEc0yigDqtCj029hM179olki6wqwVScjHPJx19P8eq/4SWSK6le1sraB3kaSSVYgXdmJ3Els9Tk/jXJ+EbSeUXMi7EgbCtI7dxzjHU9e1dN/ZCrcSBb6GRsNJiFC+AF3EclfQ+vStI2tcylvYz9T1q8ufMV7mU7zh/mPI/lWNc6ndXZ3XF1LK3TdI5Y/ma6e407Sfs7vIbksTglZFGCATgDafTvmqLW2nRWMTjTkbzQSJHkcvwxHrt/h/u03cFYwHuPlAZzj0qP7UAFCuwI6811Uc8EaZOmaaVHQvbKzfj6/lUNzrFmn7s6Xpg24GRZoPx6VNyjWtvtF14HlgikLQLdxv8AOQoGByASeetR2VrcyHc0S+UVIybmIHO09fn45/TNV7jXmm8OPap9mjj3qBHCgXHOT0HsKqeYzaRDIGOWmcYz6Bcfzq0yLGmYbiCXzM2yLjkm6jIP5Mae6TNNJb3d1bW8XlvG+wu5+ZCOy4OM9M1lTQiaxDZO0DORTNSuX/tW9BBA+0Pwe3zHimxLUj1SGaXUCYGEivIwRtu0EckZyc5657cdTTJbRjDEZJlLM4UhVPC/3hnr7Dv7VM7SvCJFG5VyWJ6gHj+uKdcOWtQ2QQCCDUpXG3Ys6XZWEsFompXc8U8bBFSCJWHYZJZx2Hp6/jVlOlRTqsIu3VSSS8iAtzx8oQ44zxk8nrxzWgmMl9AGYKPNTJ7Abhk/lVVWYyxrwWDAEetFhmtG2kW0iu+kyTkDcBcXTEZ4/uBPTpW3B4l0+bSWtk0TTreQho2kigGWRkK4yctnkHO7qBXPzsBHKR8zYO3iqtrBILQ3DZCF9gGD1AB/rRKKWoRbYsaWtvITBApAY439R17jFbmjeKdR0JJVtJU2yEAh1yvGfy69vQViwWlxOG8uCWUB9uY0Lc+nHerz6D4geEtFoWpFMZyLR+fpxUXKsbL/ABH8RDJF2oU8cJ29OtEfjnUp4hDPcRywnKtE0CbdpGCBxxwSPxrmT4d8QJ8jaNqIfptNpJn+VPtvDmvidPP0bVIw2cFrOTt9RRcLG1NcWDSS/wDErslidsjCmMk45+6R78VZH9k/ZhbjSfk3+Yym5fG7AGd3UdxjpVFoJUleO4jkicfK0cibWB9x2NTBlIwGGfatjIi1GDw1CIo20+ZDKMkxysSvzEcZPP406fQ9Dgn2j7dtCqd3nxjqAcYKVnanCHltpHCkiA9R1PmPUmuXWdZu1jLmPzjtL/eIHAye5wPzqepZoxaJpcWXhu3WNkeNvOt/MOGBBOVZex9O1UBoUdu8ax6ujA9C1swAHbvnr7VWbUJDbsu/KEYAJ6VCl8yrk/MRgCmrE6muvh64OQms6bvdTsV2YDjv0z3FQJod4VZ0nsZsHpHMyZ/F1UfrS2DteNJNI4VYBnaq5LZOP8/hV9kjBwjcY78UJgyvZ6O9zZy2ck9vbvJKj7vNWTIAbj5c/wB6pB4YSO6AGp2OQc5ZnGcc/wBzFDP5f+rAJz1PeqkuqSwTEOm7cOhOOe38qd7IVi8nhJZJfMOu2G+Viix5bbkdADj9MelS3Hge+t9jW7wXDgcKs6jjHJ+bAxWGNTkmvY5IwEMZ+VlJO0+oPrVuXxJqExlDX1w4YANulY7vY880Xdx2F1fQNVGj2MEdhcSzRTzM6wr5pwwTbkpn+6a52bRtXQEtpF+ijjLWsgx+lbx1a4jgXbcSpwfl3HHXOcZwD706LxBebWSW9mYEdCxwB+dS1cadjlPIuFIDQyBm6Lt5/LrWzp3hXUbuCWT+zLuVWjPk7Ux82e+ccdeldHZeIL+8iud97cjbFuBSQjB3oO3sx46c1DPrl5BZzO1/MRKdh3kEH25/GkoXHzHISeE/EEblX0m7Bz1WFmB+hANV30DWFwG0u9/78N/hXRjxDfopAdAoJALQoxx9Sp/nTx4p1FXVgYCACBmCMZ/8d60nBD5zkJrS5tGAuIZIiRnBUjFM2sygqp54z1zXdy+KtTuYvKldCvUHByPXvWcb53idZYbdueWaEMfrnFHs0HOc7bWszzxpsZS7YBfgfmeK9K1DR7i68MQQ2cCXEkY3EKjEtnJYADPXCgVxyIEkFym1JEPDxoF5/wCAjH6Vu6Zrd5ev5FzeSOWz8zEbu3Q9qIQtK5MprcpjT9Td5FXTr9DLx5TW8gbI7dKnt9G8Qy7hHpdysbnLmYeUBjnnfj1FdFp+oixikAuZHCfKqzHcFzjoDxn6jjJx1q5LrYjRQJzEeeVVeT65x/KtY0Xtch1l2MqDwNrF/pt1DJDDEzRExF50HzgggYDZwRkZx/jXnGyY52wS/KMt8h4r0y98WXVpp0sG5tszbDgKDzznOD6dqzU8W3pYlribccDaHIx27VM4KOhcJtq9jg98mP8AUy/980V2knjjX1ldVZSoYgZVun50VnyruXdn0TmjNIDRmuE6BaWm5pN1MB+cUhamE0dqAHDmlJptAoAdS03NLmgBaB0pKUUCHZqlqOr2mlLGbpmXzCQu1Cc4FTXd5BY25nuZBHECAWPbNct4l1zR73SQouFaTcdhKD5TjHO4EjIPGB+Iob7EykkbWuaxHY6HLcpMI5GPloWXOGPbH0zXOad47EGnyNer5ku4LEF6Ee59Bg+prnZNXvGsP7Psw6RuhM0SJjKjvuxyD68enpWO0WoSBYXilgjVTIquPY/NjH15/Cq2uc06km7o6u58fXU4kQRxorHcgB+ZQD0PY8f5Fb+h+KZr3UPLvERQ0QKiJ9wX5iMuRwCeP09TXmQ025WGE7GKSHjJ+8CPT8vfnvWtDeQ2sMYOnot3CwYTgsCMDAHXGOPT14BOatWM1UnF3kz0LVtc1G1ubhbO2SaO327/AJWPBwM56ZBPT/A1UufEkpsp5GaKNpbVHjXcwwWzz2wPc4/SuJXX5rR3kid1WUhmSKUqBzz1zyR6569Koa9q7axPaRXO2MQIqRtbRYwD0GCc4+vT9ahxbVi/bp3ZLP4k1C5t47dpWdE3KCeF+lR/2ncRSxOvmxzqoYFG98DA5rn57e8tM7ydgfZsyecjPUdsVbIUxpMjyCRAQySfMGPcjjjoOP5VotDB663N+58U6ldTs1zcNjC42OQAwGFOOmeT6c1BJq0szSXZmO3JBDMxJzwTxzjBxz29awUuZYkO3ftVTjn8f5gflUkdxPFEjTNJEpBA2naPQ/gf5/jTS7Cak92bUl+LuzCA42LtAXoOpJP6fnTrTWLqzikslYpE8PlJtYgqM5OOR3Bz61kxRsFTzAEBw4YSA8AZPA5z09MfjxcuIbSSIBHb7WANqhFxg4x16f8A18cVTjclVOXRMynLSXDLJIGy2M5wcn2x1Oc1ZtowRGdjBh8rmTjJ7/hUb20VpM7NJuMjt8h6cf8A6/T1qa4uDDEiOwyFAyeQw9s//rqYx5Spz50lEfP50rO0bbFPQY4GOxP69/wpd8kUESyeYjNkkuOcjBH9earx6nMuGtgUUgq5BweoOQAcAcDjpVmNpmRUlUuqDOSNqkZ65BB9fwJp2IcWtwa1SRo32ZKk4KNuxznOPxoRQyhQQVHG48E5ODyR/nNIkKPbLIzjK/deSQDpkdh146e1VJLhorgbvpy24c9z1z17ik9HYai3qP8At8lrcyyLO58xzvjdjiQnPzHpn/6/1ps1++oTtI7uWJ3ORls9gOOgqK5mj2PH5a5cZAjHHXrz9D0qDTrC81CURWVq8sh5wP4fqc4H4mptdnTFXWp2WjeIdPs4ox5LNdRgsSCoHA6DJ6Y59c1rr4tEnlLFC6sxOSe+cHI7k59R/jWDaeDIrKPdrGprnGfItySdv+02OB26fjWut1HaRJa6XZpbxNwHR2LkeoY5z07Bh7itVBy3KScVZMyF0K7lneW6kS3jL8yXBIyP55+uKkOmaLblkQzX9xGAW3MEVOOpAGQPc8e9abtM0k8rs7LyhcsOhHOCTx7/AD/8B6VWeCNV+UY2vkZQAqP73QYPvtU8/erRU0kGidzJlu3g3FDHAu3BWMBN3GeCMZP0Y59KxHLmYu4ch8nLMScH8FJH51tapJamZzKVdz9cn+vb1NYlyXm4jiwMdB0/IcVDRrFk0DLG7NHtwB8xyR/TPcdauRXoknZY0G9VByrZOOB0xz26Virp1xcSKoQ7icKqjkmuz0bz9FtQiaJIofHmTfa4Szn1KHoOemTj68lx1BjtLnjN0IWOpfbE+cAxyptzxnsQOcZJHX3qS7020s9TsXjs4laZmLrGpIOD3OcHoeMYOON1XdWRW8QaSWVzuSYzOCcH5OuMH/a9OMcng1DqcTXGtaXI4IbnacrtAPqGU5/Aj6VqopWfmZObeiLWpZGj3DbjG3lkxKqhRGNvIHp+lM8PuzaNa+Uqbju6/Mc7j05PPXPfnpWtdwJd2s8TIw8xCp+QcAjH4j2J+jCsoXJ8O6LgI0JUhRPsGELHIJAxkAk9jnH3q12bbMr6FG7gY6yZwpAZAgZgoLHjnA5OO+Afes3VdMj1GON5bmXzlJO5WznnnnByc9fT+WdN4zu76Bo5JFEikq0iEg/w4Hf+nf14qS61cNaxxpKylCfmOMN6ZwO3OPrWbnFqxHOr6lW7laC42TD5lPMnUY9Pr3pthfbrmRZt4hU5U7Rzz3JP+fwqteSi7mZiu0kjkdCcdaqOnDRFpCp67R+XSsObXQuPL0Oq0/UYr6VYo9qkIxbO0YI7DJ57/mK0HDsOG6HOflz+hNcdbG4gkw58s/w7hgKP/r/0roVtEhhllfU4GcL8qtuOWz0+76Z/StYS01Hzbpjry/jsoFcbWZpNjKOoP07npV/R5xcanBJEM+XMFkCjKgAiuVW1TUrjcLpY3AH+s4BPsfWotOnjm1SC3jZk3zKnXkKW6e31pqo+Yhvqj3NUJG7iMdCznJ/L0+tQl7Zg8kHmXcwGBtHyde3Rf5059KjQNLdzvPj5grHgZGBjsCOvAqxc39paLDHLNDamUqscYPztk9B3/M4rZsZUig1KQMZmFuCTx/EBx6+v0rH8Z3ieHfCWo39uXku44VVHPQM7BQ3Oc43A4I5xiuj+028km5P3obG3LcN+A/xri/iremHwDepOFhF3JFFCgXBdg6vg45+6pOT6fSlLRDjq0fPdFFFcZ2BRViwsLrU76GysoWmuZm2pGvUn+g7kngDmi/sLrTL6ayvYWhuIW2vG3UH+oPUEcEHIoAr0UUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFS28JuLhIgcbjgn0Hc0AdnpTraaLZAsoyhZscclifxOCta1pHc2srSzQlFaFwvnYjDbkKjBfAPWqf9pyLHDDARDGY/LKRjaCuOhA47VTkQC+lWPEahFYBTxyoP9a1sZXLbRXU5kRjCqkg7mkzg/wDAckn86vHT8afBEmo2SFD8zMJSvLHoRH6EfjmshJl8hgH+YMDTrmSa6sRDalpJDjCx8sefQU5Kwo6miNHgJ2z67aBOuIo5GJ/AqP51Rn0m1OCNTikPfbAe3uTWM1rerJmUOCBtwRz9MVWaK5IJVXOPaoui7M6awsbaGCQ3NxHcwFwBBtZecEg7gwxwD0qZNSs7eIwrptuIkJZQS7kk4z95zg8Dp6VkeHrObVLe5gSaOJo2WT99KEXaAwOMnBOSvHXBOO9aB0K+WT53syFbkLfQH/2ei/YVjbvI9OFxLHHZ2/ko5UBXk2sM+z4I4qnePZSSzzzafayzzszuxkmHzE5J4k/zn8BNdaTdiWVxLaFPMJX/AE2Js5Jxzu9u/wDWqg0y/e6WKSOLG7aQk6OevZQxP6VWnUVmWYrq0sbeeO3srUxuSpdw0hK5zjDMRwQCOOwqhfyWlzGxFlbRnAAaNShUjvwRk/XPWmGOUuYRE7Ny2EjLcZx26c0xLK6uykcFpdS70Yr5cLEsQDgDjnkfoarSwtR6uiiU7IcSMXOYUbHXhcqcDk8D29BjO8m3VBKHk3KclC2QfqDnNW5LO8ijBmgniDDdiWB1GCeDyO5qGXS78SOiafduykZ227nr06DnqKTsNXLlskESWtxCHE2xixcggEll4GOhUDg56k962zqt9dW6wz3LTRpyqSgOqn2B6daxrXTL+YQIloweZflBdVz1POW+Xj1x0HrXRWnhPXCpZ7ERqT957iIAf+PfypxcepLUjFl8V+IbaQwW2sXkMKYRI1kwqKOAAO1VpvEWvSnD67f7sZ5unGf1q9c+Ddaa7nhgFheiLkyW19FsGf8AfZT7dKZ/wgnimRCE05MDksbuH+e+odrlq5jSa3rLEg6xfttOAftTnn861PCPibVbbxTZb7u4uEllWGSOWUvlWOOMk4IzkH1pf+EA8UFt39nIxPf7ZCf/AGerem+A/EFrqlpeSwwxJDIsrILqMuQCCQAG/wD1ZqWrrQZf1vVIJvFV59osYHjjuXjlCjY5CsR94EEnjvUUc+hzOAtrfQqc4KzBz+TZ/nWFBJJcM5OUnO6QllKkqRuJGeeRz+NWYhsUHIGfetW9TOxqXOm6TdxwB725Vo1KDMIPylywz83vTb7wu2savdTWN/ZRpK+Y47l2RsnORnbjqCBk9xSQkBxuP/16c0vLenQqRQ0Fzn9V8NatpEHn3MEJtsf62G5jdW5I4G7J+6e3GOazorO+nXEFheSYwT5cDtwyqV6A9Qyn8RXQ6tdx7IlC4ZoIs7T1JQEnj61l282VaJ0DqDjawBFTYu5atLDUbLSpbqWzkjtpgIzLMCgz1GM/eyR2oWdisZjO/wCTccHOc9KfJeWvkqJtOtWwON0KnHPTp0qpHc2ttMbqK0hE0YyqNGpjPGMFTwRQronRkx1CKOPa8saNjBDHHPpVWcrcRhxJGQM7ih3ValiH9ptKtvaQyGQuU+zROiH+6FKlcDGOmO9dFPrNwJry2ksbCFTGtvLDFaqER1G0lffg8nOM4GKbvsNJI5O3spxpl3cRlVjTazZGSckAD2Pze3GarxQeTIfNdfXA5z6V11rfpZ2D2502wniKgOJ4Sxcqcgk59RzjHGRxk1F9t8PzRN9t8NQefghWtLmWFVwPl+Xcc88Hn3wTU2aHoznrtCrRbirMyD5Vzx7fWqbhuo5BroL260a6vhLHpU0ECr8sUF2wJY4yWeRZMgc4AA6+3OjaWfhB4D5w8QruICAGAkckMScAEcdsU7iSOf0p1AuFdiB5eVHZjuXr+Gah1Ur9mRAynL/Lk46V3Gl6H4Sm0yfzNXuILychV86Bv3AB5+VCQ2QMctjkcA1ch+HHhu73PP4rDKU+QRiOMg++Wb16cfWjmVg5Xc8u5YDOA2OV9KcJFBZNo2DqSCea9Fj+FlmInKeLLd1U9fs3HTg8Ocinx/DSwGA/iu2U9Rm3OD+BajmVg5WeaiWQnKthc9kNNkmcMAsrcnuo/KvT5fh1paxFZfF9mjbh8/2cAAe4MlQD4Z6JMGB8dad9BAue5z/raXMPlPLpL2cFlKowPTg8frTjDcSqt0FGI8A7ONpr0SX4Y6AsoH/CdWZzjOyBeO3XzTW9pXw30W4Rbey8WRyXRG7alqHYr67Vfp05JI57VPNrqNrseWQX13HHIQ+VIwytzkfjT/t17t2NHJnhseiMAQf1/WvWn+EUEsW/+31Xeej2JjLDv96TNUZ/hrpVtL+/8UFIyoUs8aOMAYAPz9sYHXpitFfoZtLqeZ3cjuiEvkKRzkZBIzVJJ1Zw+ecdq6vxP4O/sPw9BdDWor+XzEjeKOMjy1wxySWPTG3gAYxXIgMqhW3ZA4zRO+7KiklodRFbLJCjkJllBOW/+vRU1nf262UAae2BEaggquRxRXlupK5ofQQoyKYWoyTVmwpalpBSjk9KYBUL3aR3HklXLBd5bb8oGccn19qjl1O2iaQB1cxjMgVl/dj1OSMCsPWl0+8vIZtR+zpBFiWGQyA+YoAJP93GTjHfBoS11MpzsvdLdj4otLu0ecwXMWzd8hhYk4zk5AxjIIz0yMZrdryyXxtaafcGytbXz4IW8wRGUlUUtyOQR+XIPfqK2IfGFkY5G0+3uYm5llWKAyx5xuI46EgE8beRk1rKnpdIwpYqMvdbuztnu7dLhbdp4lmYZWMuNxHrjrU1cRoIu7jUje3E73VuzhbcTw7ztYZDqwGEJXbnpnuOAK7eolHlsb06nOrgKjN1Et0tuxxIwyvv7CmX1s13Yz26zvA0iFRJH95c9xXiPia+1bSLuTR9Ula7ljZGt5t4AWMZ7Dnn0J45+tQn71hzk4q6R7D4l0y31HSWNxIUFuTKp3hRkA4ycGvE73Vx9o5TEMTKJAsn+sAP1zzz9M/SrFv8Q9W8qe1nkhltCAvkumV24xt/3a5c3TyAgAndkA5+Y5qlFqTfQwm1PodjZ6/HLGkVi0G8IsEUcsRd3xwRnB6/07V2EO2C0Tfpq7CywpI8Z3SA85VnbhcFedvJYDFcX4BvdH0e4urzVIWa9jwLaRZGBHBBHynHfrWn4k8XQ6klpHZNPHaBmYrOQSrkjDA5JwMZHfkjpjGko6pLqLmjCLbdyxqBSDdJLBcRy2+N25sEOQSVHAIXuPxx1Nc9eXN5dNNdSRja2SFUBeCSQT61Hc65PLbhp/NeU7Y2kY/LgdB+IHSo7e/ujtWK4kihEv8AGW2bvU9ugo5JXsYe2hJc2yK8MkkxKnOV6jk5/Kn216ZJDb7iEbJVnYAAe5OcD/Cr1uI0D3ceJoV6hoii+nGOcj8M+9RxWzXcqTNMPNmB4DYzzwMfhn06YrX2bWpzPExeliS4lQJGJLRZQrp5jo5I2jrgg+mOelQyRnzysVzHIdm8gkA4OR0/AfmKnjKiJJpmZgj+WWBAHqM55PQ1Z1KW2eRUtn+ZeSITLtxyMkOxx+GRz+dculzFVtbIybiG7jjMhiYleflHXvkY6/Wr6ShoYo3lWWQ5B+VcAkDr3z15J7/jVbUZGEUYRs/KA52gkHtgkfUcYqgm9YiBI+c525IDde2cenaoTSuje7qRRrW29RJ5koibb05+YgDhSfrnj1FEepGWQISAu4FlVcZH4VQivJcl/l+U9CCQ3tSoVcHahjU/8tDnaM+tF+xm4PXmNGciWAgyBSCMqXJ4yew9j+XaofMeN5CzjkYVs8H6Z6mq6SNCCr7juwGGeAOvFSPJE3lururxnKvsHBHrjr260/MI3S5XsQ3ThvMcIUkAVmIHABGeew/yKdbzSTygSiPZ0IbC447f4VYUQxlCjE/KRljnt04pC67gFgOQMLu5A/Lngf55oNlVVrWLUZAaYRbpN5ZQAdoIPTp0A68fpiq6wPqE6WlnDvcvt2RgFhgdfYc9enNa2j+GZb2RpJfMitsj5ipYvxnGD0we5NdXbG20VRbachDEktmMjOD65y2Ce5wPUU1C+5tTg92Zdj4MtrURT6vKpcL8lvGpOT6cDJOewrQl1GI2/wBm0uOKK3IwvlKMHHXp1/Dp3YVGUe7vJZrh2m5VghYFQpxgHjBHHUjHopPNTPDEkrb9zSEqOo4bAxnOSSOwI+ir1raMDa6jsQvGssQQj75YhcgkkHGRgHJ9wG/3h1pT5C/vt67HG3AP3sDpxnd36lsegqW4uEEuGbCN1z3I9T3xnvuP0qq2yRmbcyMD8yNkFug5zk49Bk49BT0QasguZAfmUviQ4GxhwMZ6hv0z/wABqFor24XaY2jT+E9P8P0x+NaMenxk5UFmz/y04bP17/0q4scavkct7t75zkDk+5pORVjnofD6PJym4k9DgE1ZXRY4zuEaEZOBnJ/Q1utOmAAkJ4wQyDgY+lNd1k+ZkXcOuR0/H+lQMxpdNuY9v9nxW5uCflM5woHrxzT7cajFuttXW0LeUXBtw+RhlAzkY7+uf1xFcrqE9xK8GsPbRZ/dp5COFGOc7jkn34rQSK5ks7QXDB51V1MhA5+ePOVy2DwBw3156VBXYpOyJb+2km8Q2k4Unyo2G4dBnOO38j+B5zO1uZ7y3uImVkAOMN25xgjt9PzPSpbYLJcecpBBUg9wRkZ4/Af56cbreqXemw2kFjI8U8FstvNDn725R8y9iRtPatXaKMHI6+0ad7y9V2ZgJwIgMcL5adAOnzFvQ1W1S509WjsL0RJJPyizj5dw788Z5rlv+FiOkVq7W43Fts0SMVjZOOc/ez+PHPWrurFPFHhaZ0nRUVfOCKoARlyPmJ5JwSM5FJyTWmpKkmcB4jsItP1GP7JcQTwthg1uc4Pp/L169aorcGTYY13PgRnJxgDk02VnglOwAqOpXqOMVSgY+c5aQfIO9cnNqWoqSv2L6yOhRXfO0naAf1qybhbcq0YDKxBcZ96zI5klAVypdfu+gqKecgGMMHYHufu/ShMn2d2bD3oefzJI12nB2Kc7ailugSAcru5XnknrWcJttuCMh8fmPeiOOaZfNYoDGRhRnIHtxincaprdlv7Q1wGZcKFHJ7Z/yKW2U2oNxKNrdUODg/8A1qaqBdqAL1AbPX8ac14gUhoVPVevGO1K4eSR0w+JWoQyagkMgeOW4LRSOMMidAPyArntS1iXVHgWS5dmgBVXfnjJx+PP6VlR2ymbcJRsA3Nn+VReU/2ktHyrc9OlDnJrUpQjfQ7fS/GmoaVo4to5/wDVZ3kDHBACD34XPHr7VR+Inih9Z8OaJb/IDIzzTANnDL8q/TIZj+Irmd93MRbIJHBYbY1BOT04Hrz+tReI7C705rOG9gkt5XiMgikBDBScAkdskHjrweMYzanJqxUKdpJmHW54Q0AeJfEtrpsjyxwNl5pI1yUQDP0GThcnoWHXpWhD8O9ce3tJbj7NZtdyiKGK4chySAQSFBx15BwR3ArpPAV9pfhPUtQiuYxczSOkUU+zBID4IwRkdQce3PbDjHXU0nUSWhf199C8N6pbDRrdYba0O2eJCQZiGD4ZzksOSOSccADAry/XNTfWdcvtRkaRjcTM6+YclVz8q/QDAA7AVueMNWi1DUbq5tsiC4mJjBUA4wMkgcelcpSlK+gUo/aYUUUVJqFFFFABRRRQAUUUUAFFFFABRRRQAUU5ApcB2KqepAzir8emxSgMl1lT32dP1ppXC5nV0ukaUYYGnn3oxBZgEDFVxnpke2ef5Utho8UVnNdtDLMF+VJsfIr5B47Z259fWtu1uVEUiIrNIYjt+UjnHvx+fp3q4xM5SIBHbxz70WQxgfKWbk898e2OlaUV3tRTHDahQQu828bNjgfeIz+Oe9QWltELaVridonHMaLCJAx9Cd67f1pJYo1nItpmktwWx5sexs5OOAWBHT0q0vIhmqmozBj5cjoSvJViO2O3sSKr33iHUtOjP2S+nikIyGEh4JOCf8+tFqbYRlLiKdyc7jHIq4HGOqnvn8xT7200t41kMV4Y0GGX7QhLHPY+XgfkaqWqskTHR3Zjf8Jp4jcHdrN6T/13b/GpP+Ey1vyyr6veA5DbvtDgjrnv9PyqQQ6AqtnTb+RiRtzdoABgZ6J9fzqCWHSCF/0OWNRnJE+SemOSMfp3rLlkauSIIL+VhmPHJyWx1I71t6pJFa6pfRRYEUc8ioB6BiBVawTSktPkg8yfcfmdjgLjqQCOc9OOx69n3Btby4e4lt0iDsCwiZ+meT8zNyR+HtVWZN0Vp77dbBVPLDoDzVjULrz7ya72cO5fb9Tmo9lpCFwGMalv9Zgk9AMkAZ6HPTOe1Nma3uHyUKkgZWJiBwAON248kZ5z14wKdmK6Y2OVTGHJ5PWo7acFijcqAQMjtnpU3k2SuYVN1GBjLO6t9cjaMZ9M8e9SfZdNWVPIurtUbPmb4kcj0x8y/rT+QfMYqxnPlqAB0wMUyRE8hZT95naMcDooU4H/AH3SrFF5AmW4nNwVI8vylKjBfjO8ZzhOfRz/AHebNxb2r6ZBDFeytKJWkObYIFDKAR/rGycqv5n0xRfyGkZTuqE7VXGM8AVrWV/JJoSQZYIszv14PCgD8MfrVez0FbyMhtWtLVyoZkmEp2jOOoQjqQPxz2rasvCzXGnw20evaQGiaSSTdM6Z+7gDKf7JqeYLHNySq+WkIJPPNV2mXACs6jrw1dFH4IlvB5sniPQIN+diG7JP6Lx/P2qKT4eakD+61vQJT2C32P5qKmTKSMOLVby2JSG7uFHoJDj8qv2GvaiZRt1G4DjnJkbipv8AhX3iBOVewaNskPHexkN+O6kn+H3iXTIzdvFbvB/EY7hT2ye/H1OKSYNF261yS/SSR0tvNdWQz+UBIFK7cZGOg45z6dOKHbR797C1tTc288jpFK5ZXQEtjcoABPB5Bx7Z78zIJrOaW1nQpNGxR1PY9MU/SYwdWhy+CMsnXqASBx3Jq3sJLU6YojT2qW95bSbnCbpN8GDzz864x/wLJz0q0un3lxdvb2kcdzKqBv8ARp45MjPbax59utZDr5sahSBj9Kz7jzoJyiyHnqDzVKL7kOVuhL4ntrm01UQTxujRwxIcrj51jXcPwPFZMLyA/KD1yTWr/beoWsrTQ3DrIwIL55OeT+frTraRpTLM4RpJovKkLxqwZOOMEY7A5xnIBznmpaZd0Z8yvtDs3Dc8VChZp1RSSzOMZ9c10upJDe6VZg2dtbDzXiL26FGcIqYzzyfmwT1PGc4qnaWuleTHHJZzFlDHzYblkc5IKnkEDHPQc5HcZM2bQ7og1O+V9Xu5YkKo07MBkcKSccirmZU1G9SfAmWdw427Pm3HPy9uc8VTm0mGSd5LctFH5o2IcNiPPv8AxAY6nnBya1ZYILzWry8lmNrDcztInmAsBuyTuK5PXHQH73saq7uS9hzPGAOmDWczr8x7Val07UXs2mhiia3ViBJ9ojGQCMnDMDjleo71lbbrzzB9mmZ1PKohYj8BTbQJMViOnHr1rSuFcCAZCgRKV78Ek5PvzWbsnEyq0Usb43YZCMDAbP0wQc+hFdLd6Lq91pWnXUGm3UqvbhGaJNxJDsF4GTjbt5IpX1HYzo5RGuQc+vFVL/VZ1tiseQCcMQOgqQWd/bqqXFlcx7vu74WXP0yKy759sQLL/FzilcaRVN67OQXPI6gUv2ho+RK+PrUOPMcbPm7dOKUwPu+b5c+tZ6lDlu5Vbhmz70pnuCMkswzxTRA245dcY4pm6RMgEccZoAeuoToR8xBz1r0Pwxr0cfgzxA87xCeOBDEzEKysSRuB4OckYxzz715i255AuMknA+tX52UeHnXeB/paAYXG7Cvk++3PP+8KE9UwaOxm8VXNzbljODleoGCfx9ayE1jbAsBkf92Op7855z/nisu3jje1O8HcMFRnOeOvNOKkRcfLjGC3WrUmQ4o6LRvEJt9TjuJIFuguS8TkYYEEemOM5/CunHjCzs0ddM8O6baeYu1sRqQ3HfAGR7V51YyJ9oc7TwvfrWjHKrv8rFtpx09qdk3di1Wx0v8Abtn38O6Bnv8A6CtFcuWXcfnYUVXLELs9/vr9bERlopJN7bcRjJFYJ8RpFfzwNqCR7HJWKZYwzDONobeBg54zz0ro7qF57Z40laJm4DocEfSvMda8N69ZakzQ3Je0MxMYMpLquPvEnnHXrkj34rgpqLdmGJnUhrHVHolrqybd9+0VoJZSlusrhGcYB5B6NnIx7VqySRQJvmkSNP7zsAPzNeP6t4jS8W3s2kZzbYjlkjQksdvI54BBz2wR6DOc+PXEsra3R97XMUpxPnnlslSMjA+9z+laqlcwePs+VK56de6xoGm/aL0XhR5HCNJHl1Y+gyCvvXNeK9T0fUr9bCS2SeC2j8+FYHKbpM4ZTj8OnvzmudvvFfmTwx2+m29rFFtZo4QIwwODlQAMHvzk8DtnOamp3KP9nQMiSSlxmMbsMQCCepyB0zVRorS5zVMbK7grWNXxDGmkJaS2wiSCdQbi0eIkqffLFtwxzyv8PvWNa+IbjS7kTaaY1O3DkKVHP3sr07/hgYrNa2aKVonn+dydu87ck9x198dKW3WOCOSWJ5EdcMh3bio6c4A74qo+6jOpLXnR2ukazAHlkug0ioDGMqHB4O3Cgjjg/wDfI9BW34b8aQRalcWN5dM6zTb4pZZPljVlBCZPpzya87ivbdhbRi12FZVlkkIPQjBA56YGcEcEn3J0rnT47xhcnU4IWt2ChsDJXruJyPYZ561cqabuyaWJdFqKPVZNcj1Wa4sNNuljfyWK3akMEYEA8f54IPcV5xrc9vbXwn16ximmkT97NbYdZASRvT5uuMduufTmpaakLd/tNj5kYE2yKXGUQg9y3I6+vc9c1DqMtxqGqzySSRZYFm2tlQScnA6DPP8AjWToJSTRt/aDa13MzXrG2acT28bLb5C4ZSuexPsOCfzqitlE8zyW5URIC2zeWKqO7HA9Rg4wa09annv5IXnlDkp1XIGO3bjP5kCshdTksHZIgFcxmJ2AGSM5zn1BxzS5Yx0CFWtUVluOtnDq6BCx5AC9a2rPRVurC8Zgpnt0wYo5lJyOQe4x9DknNYtg7GNhHKyxhgT83Q+uPX3rYuALOOG7+0vLdb94YksoBIB49cnpkVVOPUdefK+VblW00mRhG10ssabt0kS/eCDv0x/n3qwXS0mA3O9lIwkVCPvYyAR9Mt6Uybdcv9tnlWOGKQpH5Z2jcP7u7OVPXkcZ6dqozzzM+4EzQhMSOpBII74HQYH+ela3iuhyS9pO+peFy9iZII55nRyTIAwCsDzyMdcd+3NT26RzXZjhT7RFGwOVXDYY+mOeT36ZrBlcq1vNF5ik4fa+RgD0JArVsrsCVImhV1ZT5hbJIIOQB+QHI6mnGRNWlJR1L88sUDyPbndFlVCE7wxzyTngY7Zz+RqK5uPPmjlVAkJIBZQxUHHPUZ6/X8ahFw8RkP2Nt3lFAh+U49SPbH9Kl0+5L2CxEEBiC2FHzqD2Hcjn8qUrPQmHNH3mh9z5kcMrMFARQVIDDeM8MAwA257YHp1rLmSWRWMRw38SrnsB2xn/AOvmtW5XyYrxYXhurMFJGkA3FM4wcdvT64qq0sa28TONjBwUI6J2+g6Z6+lZSijppVGtUjNhDNGqFzjPzLk4X049a0EikhikkIClOibSSR1+mMf571PhJori4wqwRtuby/lVyexC8D8vWqs97IiLNCVZdxATnHGDgAj39al6bmznKbsiwi7o45I0lj54VgDu+mP0xVq2aR0KmHYvJYEOvTr7eorLjuoolcGQ4dAP3g4BwOpHPX/Pap5LqKeyRHnBKcqMkgH2547U7kyhJ7hcqyXPyZQdABnDc8AHp6d+1dbofhgzKl7emVV4KwlWPOP4u34dqu+GPC1xbRpdXPliXCsquC+wjvjIwfett/PuJJI1ld0H3u6t+H8h36ngc6xjY6adK2siJQJ32oCYtuR5Yzu9Pz7Dv14XBaWCFoXIfAwAHBOeD0ye54OCfoq4xU9umwbgWJJLdc5zyTnvk9z1x/dAFUbx0ikZwT3556nk8+p49z3IHFapWNm7kzzN57FFwwztGMnPfv1457+rDpVOW7csUihEhThuCVB9OBgj2GB7mq5mlulRJkMaBs/Kc7vTPqB6DAHpVny1iBYBSc9EBxn0xUtgo9yEQzSkyuz/AJYwf8KsrbjION2OCxGefw+lOUFlwSi56k4/lTtyIyjzAwBzuU4z+lTcqwu1QpyrZ4A64/lQdjE7gVHvyaGmQr8jFTnnLhs/pUXnwKMtIeD0VR+eaQyR2Cr+7aQY4GSOKiura4ubEyGWaNW4EijnjspII9exrMvbsLBcSRz/ADrG5jDDPzYOP1xXJ2etf2rf2tlq15dfZl3mf7TcN5QIBK/KTtUAgDAH88VEpa2Kir6nY2NhZvPb/ZNSllkRw00ZuwS3zDhlJYAdsgD8K07VtyWRXlXiYqQT3KkYBAPT24/Ws+2v/C6xw7YowFkAinW1ZUD5BG1sAZzjvWhFE8VpaIzjdEoBwNoxjg4HTrn/ADmumktDGqyZGkijWP5QA/PzEYXd/wDq+pry7x81x/wkRdMxgxqcHPIBPJ7Z+nHNd3q11dxW63tqPNWKQF4253KQRn8M59BiuZ1jxLpmqW9st3CHzesUcR4IgDAbSfUg5x9KKqurGClrscvDdWzS3W9XZ5EAhUKCd+Rnjp0zVL+1ZIrSW03OsbNllJx0zj/0I/nVO8kCTyeQGISQnBORjPBFR+aHB3gc81yOTNY01uSCYlQAcEcE9hUKozuzHrwxbHX2zUWTt2qcr0pxuvLAAUfQika8ttiaONogxZV3NxkHkDvUM0Thg4K/hTsggvI2d3TH5mmlvLQlMcnn0FISvctxbVXlAzA8k4oLqocbxgn5geh71Racc4YnPp0qYFZ4cEqGUE5x6c/0pi5H1LSTvPPI2Aeclj2qtJLuO0qSD19Kje6+UhemPxJqW1cRqCOZAN2c8UCceXUrYeOVccA8DPQ1b/eLhtwwi4NSjY37z7zK2QTyPpimgiWQpnYW7gfypCc7iQzywyRyLIVMbbgc/d61cGi6z4gvZ/EPkC9AlXdHtLM+FxkLjBAwuR/tLwc1IsNtax4WMyT5BLFgU7Hpj61b07WrrSYruSKcxS/K5CDKthxhTg8Dlv0rSNtmyfaLeIWOoXEF2l3fyyyX8LOLe3uQwClUJBOfcAY/Oqmr6xFf6pLqVvaGMzQkypjIWY9WGOmev5+9bWmeN9VtTLqM9hFO0pCLcNGQMAH5cj1479qwdUtobezbU7JgYLglTAygtE3p+a8HuPxFXLbQuKV9UcxqEoluuI1QKoHy5575/XtVSnSEtIxYYYkkj0NNrM6EFFFFAwooooAKKKKACiiigAooooAKKKKAClVmRgyMVYdCDgikooA6my8UXl/M8OpTmaSViyytgYPUg9uT+v6bOnu0kO4FSIndCSeSZIzj8vLJrz5WKsGUkMDkEdRXaJfGaxgYt/rEDOV9cc/rWkNTOSsbUWi308beasUDZI2z3MUTZHX5XYHj6U+DT5JkBSW12qcfvLuGM/kzgke9Zd9cTu8yEnEbnOOgJ4P8hUaB5YnkGckfyFapMzbNd0MMmxlVn25BSVHXqOpUketWm064lspQZbONsBh5l3Gox65LY9sdazIdoLRhlOV5OOmetU9Xm8q2aMkk7sfhnp/KnLRXEtXYtrol1cO2dV0iJOgJv4+PyJqYeGJFRSNa0WQ4PW+HHueP5Zrjd8jLgKfXrRFK65ADZPcmseZ3NeVHXR6bqMULwgWTQ7g5nhvoynHGDlhj72fXr6HDoNNnm8yLMKmNsNvuI0+uNzDd06Ln9RWLp1wUikLHjAHPbr3rXuJngdUxjgg5GDkfyrWKuZvQqXFpfTExx20rOieYVVdxC/3uO3I59xVcE/aEBziRQYzj7wzjj8ePrV+a8lbYA+c9/wA/8TV6/wBan/tm4ntppYv3jsu1yGUlsnnrk9/WhpsLoxrlXtLq5tXHzRSFW+oJFRIylWUnGf51pG5W5PmToHkJ3Fn5JPXn1p88sZuA6QworlXOyMABsHIA7DJPAwOnHAw7MLopqjCMY7HJB4pJJsKoPAHvWldyAvGzwRmWSQMzrldwxyu1SFA+gGPpUKiwLHz4p3GMfJKqjPHqp9z9fpiiz7CuivaieSK4dCohCgPnqckEY/EU1J33omCyZ5rWsv7Dhibzxf7NoB8udDznryg9+PfrV6Kw8NyLHOLjUY1ZiFfykYAjkjAIJPI/OknbdA0nsclHcsHYcDB5OKjlvJjMBg7Oma6CDRtAkmkWXX2typ2KG05nzj3Dn6fh2q4vhDSJxlPF9mT/ANNIWQfTk1MmUkcqZN3O8j1qC6mkMDhJWGVJxjrgZ/TrXXxeC4pyBb+ItIJyfleVk/8AZeat2Xw8gu7hY7zX9PEbHbm0mLS/hlQOT19s0rjsZ1jc6RdWFpFqWlrdIkRzOhZZucsDnIyBnoegzVW20/Rmu4pUbULPypFYHek+7BB6YXH45+lVmto4pri1WQyRQrMI5AMbxGjFT+JUUlpMjLmQhcijl6MLtGyNOsw8n2fU4ZeB5b3EUsAJyPVSvtywHJPaqWoaBqfmzzR2ck4R9rG1YThSOoJjJAPHQ4NQYIY/vMjORTomZm8wFi+CA+SCB6D8zWiTT3IbT3OflkQhDv65AB7mtKyYhVB6AVradq17p0KwW15MkKqUELv5ke0nJGxsrjk9qt2upW9vYC2/sbR5BhFEj2Sb8L0yRjP1PPvUPmuUrWMjUZHFtpxw20+YyE9CN+Mj8QR+FAYKAqjHHTFbs99Y6vJBFqNmIooFCRCxOwQqTk4Q5BP4joOvOebuQ1vNK0ZaZd2EzwxHbjmmvMHrsTw3G+VxjBBqGebJCFwu0ngc4pltFN5qtscOR9zaST7cDrT57G5QR3MlvIUkXcMDjHv1x0PXB4PpTutxWZZ0ny0gkkkUFnPp6VaYozjCqB14Ap2oQTaWkFpcWzRTGLcwJY5y7dM9Og4HHfuapwOVUlgQc0lZ6hqi5HCqKVjG0dMDj/PSrMMkysrebIHj+4Q7ZXqeDnjnP51TS52tz6Usl2pXIYCnYDqIfGGr6boKTxXjPL9pCASjd8m1ifrzt5NU5fiXIUb+0tC0u8YqctJF8w9OW3fl3rnLm4WWwtoQCQJWZjj5TwAB/P8ASsC9QhiitxkbgTwayaSNEegRfEbTZEHmeDdKfgLuwmcAYHWM9velk8a+Gpcs3hKxVsYxtUgf+OivOhGABj8hQe4/EVKSKPQY/E3gtXEknhpQ3okSsP1Yfypl34g8ETSID4ehYOoJMeU2Z7HHp7Z/GuBPIIAzTflB54pisd5L/wAIAXCixUqOQRLPjH4qP5Vrapq/gzXNDGlK8iyg5gdUdhE2MZGVz+A4OK8ul3MuQc0umzlHkD4wMY+UHP8AnijZpgdW/hPw/aP5UmsSTSLGJBtjby2yMhcr0JH5Z5wRitXTtJ8LjzJrix2x8bY2vGcAg88dfrn8BXHNfPknEnzLtZhjAAIPTFSG7RWZfn59T0/L8adkJ3Oh1jTdF0uyW703Ll5VGNpG0EHplifSsJ5Fjl67dx5GOKv6VexQ3KyXdul5BGpzBL90kggflnP4V1kPi/RSgS88K6cyhdg8uKMAJkHGCnTPOOKbl2RKXc4bzovVfzFFewQ6H4dlgjkbQNKRnUMVbTosrkdDx1oqfaILI2dZ1ZtHNpK8O61kl8uWXP8Aq89CfapdQW0uNIuboBZVMDMHjIywAJwD3rK1a/0XW9HjX7YrGdSYIg2DI23gY/H27V5gnjC+06yGim8Ef2WYmOaME5XsOB05riUW7we5tUmoq+5QU2j3TvJK8Mb5eGdU2jfgZXg9OenbjpmsK5vZJDOhugCj4ZRzuPTIIFdTdW8xkN1Kqi1dgDMqDMueCcH2HTpms6NLq922F3cXckWfJTzH3iML0IB6emAeK7Em0rHj80U22JBeSXen/aolDToQkigcOMdxj0z0x0pov47yVpWb7LHGu4jzASoA6JkYye2azxYvDLcW/mrHJFuOD3YcHn8KoyTyjEYU5QlTn+JTQ5scKEG9DSe6g1PUCwQhH5V5PlZmHoef8P5UwyRiNYvO2yAkFzjn8M4x9OvHFUYnA2ygRBc42Sds9sfnVlIxI7ykgMBhV6g+xzmpLlBL0LMV5cXTlVRMpzu/vEDHJzgDHWpJ2updO+0RPbyJENoXzQWXnuAee/X1+lL9kgmjR7aGSUeWPMD5GZOc7cce3Pb3qT+zbea4jS3cwQTFSY/NDFXxySvU9wOOv44rW1jG1NSH/wBstLp4t5kjWELlpF3Es+B1/wA9u1W9K10263Pn6iscpB3RmMFZRg9GAPzducdeM1yUsdxHLLBLuSQHDqB2Hr7VctZ0htzHJGGRm/1iAB+eoDenHfI70lNtms8LHluupvLPDND5zzRo4kJEfUqOcE4HI/wHtVO8tFurfz0bzJfur16AcADnOeeh69sCreg/Z1mmzdyQ3QYG3kiU/PknIOPbH61YmtiWlFugV0YhiTxgHBPv9cUTjpcwp1PZzsjLtrtTIsUkKZx5f7mPDc8fwjk5Ixx+VaCXKBZY5fNlDsScxlWAIHOPX9OODmqKwwoqXiyCWRZCsiMRx6DHX8f8K1WviJmlg3Kki7G3L8qgjGG49uDx1xzRHTcdWak/dRnahawRRxWsCCRQhdpozww9OuARg/mM4qpbw48gW21EYODJkAMMEEZPT73OT0qxI8cyNFJD5Tcl2x15HVfzqnDAVuxDs2JGjsgzgMSVB5GD/wDXqn8VyofDbqaDQxJLbhcTToNuC24MMcYHpjoeB/OpoX0uN42LFJpwUkSNWCK2TjI64IAA/M46VTKSwuEMXkofm+71H/6u/WkgiiublYzdRROTlFn+YNzjjj1p63MuVSVm2X9QubVbWJAolRVZS8Z2MDx94YOf0z7USbZY4dLs4/NeP5gA6AsxwT0JyeccHsPpVZo4obh7KOTLMuFV+hcZBUgDrnpn+tQq1olsHt0Mjsi4RWORj7wPv3z7Ur31ZCgkrK5etbBkV9zq07jy/szbhIrZzgggdgec8H0pFMJje3e38pvJXcSwOGB+8fTPToaZpqXF5AJFnZCshWMOzBixB2kMD0DA8D057VKl0GlaCa4WaXdtxHIBhhkdvvAg9apK4S5lddipdyEW8a27+UjjDc4D/wB3PGAMgn8vSmQ3gdxBNCzMfvMpyceowO3+TV6bTILi3kkLqoikBaANhh6N05BzjjH8qx0t447mRwTIABjaQCDx2Oee1ZTTUtTroOnKNluLcRytKchcNxgnkfhniu08BeHXuphqEzxSWkLfuUmXcrP/AHgB1x6569uK5vQNLk17Uvs3yxxL80lwxwVU4wMHjJ5wPr6V7HEbfTbJIrUKyRKEVBzn0BJ9+/506cbe8dkYvqPv7qeCHYl1GzODjapG0evBPTqfYH6VWto5LWyDBWZmHHGeep+UcEk9u59FFFvE11Kbmb5ZAcDnpkZx7HhT/wB8+9LNKiGSR2VViBOccKmB/iBjvwvrXRFWQ2yKa6EEbSSsBgZ3bgSQemCepOPxIzwoFYPnz3B8xpGIxkL2Tn/9XNPklN/deZuKrvyik8D1JOOp/wA9KlWGHcpzKGHYMeTUtjSEVGKr+8dx2B7fSp4i0bKVwCO5NTRKgyNgcc/MQCfbjNDRs0isV3KpGcIFyPTjpUlkjXEhXZNO+D1CnimmGMk7Wfnp0qOMNtGJW5HIBqfgSM4jYNzwO1IBj2zEMyowI6E8EfgKrG0WWQA88cA5U5+nWpZyZGyBKM4GBk8fQCqJtryRf+PduOhcdPzpAiaXSFAO141TbghgT1ry/QryRddsvtKqYEfLqi84AycdeePxr0K9tNUSEpBiIE5Owgfyrho/DrWeo/avNdipZsY46H0+tR9pMtPRnpS+MfDCf6M97IhPU4kP6gEdqt3M8U+nrc2RWa1YYVxggj7v8xz/ACrFsrjS7PTlllghUBRHL5oDZYduevqM1bOoafLaCKwe2ILcLCQQMnnvxyf1rppb2MKq0uaNuPlhfHykdPU4zXJ+O/Clu1gNUsxFamI/OqjAbJ4wB0OfzJ/Guq0sboiQQxjUhc9CR3rlbnx1ptzpdzYarbskxyjxEHHsc/5Na1kmuXqc9NtO6PNIblo2deMsPnz1Pt7VatZbBkEF9bjZJgtcRn50444zg9uPapdb0zTraV5bXUTLGygxKUIZiSM59OM/pWDOSsgAclO27riuF3i9TqSUtUSzxoQRAcgHGTxmqu1lX5uvYVOJQI+oHqMc1B5m5ix6+lQaK5JHuBy2doGakyWTAAwOuagDgsOTkd6flgdxOaAY+ONVl3Pt2E/dqVp9u4kBs/KNvp3/AJ1WBDvyeO4NIx4IC496BWvuWV8ppMMq884ABxTQ0qsUUZQ02NsR4Vfm6k+tRNcMH9B6UCs7lpZjFlFGT/WlidjcAuhLY4GeKpNKQ+5eoPercMMzBZCygdeT/n0oJastTRjbzEHzCM84QHOTV7Sr3TYVdtR05r5iMNmcoqD+9gDk9uT+FYVwr71aKTcemB2pIvOO5SpIA5A9KcW0ZKF1dM9KEuhaX4SjuEkAuSjNDFKTKFZmDcA8ZGAM8dPWvNZbm5mkIllLguZCCeAx6nHqalVLi6kjhkL/AN1UAy30xUeqRT6ZCjGC5heU/u5JUZcgddvQdx+dXOXMa06bW5l6ikkd9Isrq7jGSpzjgcfh09sVVoopG6CiiigAooooAKKKKACiiigAooooAKKKKACinxRSTOEjQsx7Ct6z0SzW3WS8maSU4PkoSoXrwTg57dMfjTSuJuxz6I0jBUUsx6ADJrsdBijgRfMg8xokZgpG7B2luR0IB/DGetbjW+g7IobazuY3bCsY5I13ZJ6/u/eqtp/Z9usiyW/mlh98yMCp9Rg4/MHpWsYGcppi3GuTtA8ISArkj/j3TngAk8deBz14pZkWC5igKKQ8SOTsAyWRWIx+NX47TTooIZY7edlfJIeZff0QH0PWi5u7OW4BbTY87Aq4lkyoAAGPm7Bcc+v0xeu6RGhWN6bSzlSMfKwAUdlOQc4/Aj8ad/wkd2unPDPBbXKjlRNAkhUnHzDcD7dPSq51LTJYnV7SVQ2eI5AMc8YyD0qd4dFitJJPPvoWlG5FaNHG3dgAtuU9vSh7aoa3Kh8V6gWm/wCJdpWSCBt06Bdvpj5OfxzUEfifUVcNJa6SRnJH9mwD+SUQafDMwLX8CRnPIZc44xwWHPtntUkeiRSOzteoF2gqSY8k9xjzPrg9+OmeMtCyxZ6n/aguPtNnabiQV8qER7RnPG3Hp3zU4t4Ly62Mkhkmk4JkYhdzD15P48++aZpGhXTzzpE0Kk/cWaVIy2PTLYzz0zWk+h6np13CLiFEbehH7+Ns/MBwA2SAeDx2rWLRnJO+hlXFjDOqiKQ253bmwNwOewHUY55yeo44ybNzZafcXMksKyWcbNkxr+/2/TJXv6/nTbiKS3kcSo0Zjba28YwfTn8ajZxtY7hwM5zVWTJu0RTaW0dy62t2lxHgFXwUyMDqD06+/Q/jFLFcCURi3fap27lGVPOM56e9WvKlhfLoyOowwYEEEdQR2NSRXe6JQ/IPX607DuR2pkitJzJFKGKMqtsJG7IOCe31PqPWsf7VJ5UztgEEAe3+c1rDUWWz8pmIV1BZAeDyp/min/gI9Krx3doqPthA3DDMqjB5HY9D05HNJJibRlm4IUxuSSw/I57/AMq6GGYJ4Y09dnJknIwOpO0D+VZ4a2nmKvFuDDhs4IP4duv51u2N3o8MYstR0uSW3ByjxzMGTPYDOMHnt3HpSd0ytGcwsgIcu2GDncVOefr3qczomEXrjGa6G5PgQgRppuqSN3la5CnPfjODn1x+VJA/guBCzadqjDbj/j4XDH145qbvqh2Xc5/duJIY5x17VLDdPEwIcnZ+lak+p+FfIcQaHeCUMdjm74I7ZGD+Qx9aoaNbaXq18IJrx7T5SSSvDAcnnPynGfXtTUhNGhEumiGbzY7qOVo2WNoSrZLIytu3HIHIPGf4umRilHoH2i9lTTdQRoo497SXq+VnkKAMbv7w6471VjmWUPtYlVPHNWLW8Nv9oIOVkiMZP/Alb+aik46jT01LFto2p3LmGCKCVtoJH2mJScjJxuYZx3AziomhurAvDd2M8Mi9RJGVOMnnntx19qZJNhSQwYFsVoW+s6hawLFa6jdwRDokU7Ko/AHFXyyWxPMjGSTDAcFifWrgQsmcGpWurHz/ADtQsFuiJTNJIsrRSPkDIJXgjgk8ZySc8mti4stHi1Oe2W8vEhSVk8zakg4PUYI46+tZ3aZejRh7iuGXg96S3YLNcuyEqIuWH8ILAZ/XH41pXekrA8i2+pWF0IwM4lEbHPoGwpx7Mfz4qKPS9Vit7xk0m4mSaHywY13/APLRGyNuc/d7fzwDTkiUncwy4F225j5IBLYGSRj9a1ILl0uLhEnOHYq+2TO7B4yR9761FotrE2o2iXAZWaZQUKkNgtj7vX9KoCOa3uQVDFQeCP8AChO4PY3bnX9QtVDm4EuM7RPGsgTdnOAwIB5xnrVe08QaSI0g1DQoJgoIWSCd4W5OSWKn5vb0HFZV3K+5ZHVghbHTrVSWRWIYrjIquRNC5mjtCnhsmJ4odTEJfLH7QmQmByFKnJzngsBjH4S3Gi6E7xuus3MCSJ5ipPaLKwBJABKFeePQ9RXGwXcpiTdnaOMYrY1CUx3MKkk5hjIOexUH+tTyj5jpE8FWeq6bcWth4gt5J4XWRfMhaNFVhjluTyQO3B4rF1D4YeKo5nSGG2vtjgFobhFwSAQD5mw5wR271X81RjOMetObUbm0gLWdzLBITw0blf5VLpy7jVRbA/w68VoSh0nDjnAuoTxz0w/PTtS2vw18V3TsTpUkKqCcysoyR265/pVG08QazYSySwajcRs5yxRz83GOfX/9XoKtx+L9diO86pdv0+/KX/Rs1Diy7lefwJ4qt423aLe4HBCR7/y25z+FZsvhfXrf/XaJqiD1aylA/Va6P/hYPiFcMt+3X/nkn/xNWU+JniPHzaiMf9cYx/7LS5WO6OVTQNZkQBNIvyGHygW75b6cVWbSr2yuGhu7S4t52PypPC0Zb6ButdpN8RtYlCtPPGxH8XlDOPwx+ldBa+NCtnZz20EKSztIJiIeONrKwJz2bkHrxVKDehLmkedLo+oNaLcJYXJtyeJhE2xsejYwanXw7rM2Hj0TUJY2XKmO3dsjqCMDmvRB4supC5HlqD97ES4P5j/OKgl1+7c7jeTRnGP3blARnphcCr9kyPao4G3VovNjlBjmU7WiZCpUj1yOtK+SrYOBjFaniG8kuZoJJZnfO45dix7etY7zqqBk5zyeama5XYtO6ue0Q6tHHDGjowZVAPyjrj60VnrayxqEjlZkUYUmReR2oqIwjYd0eGm4ns74FJp1MZ/dMw2suPbJx+dWre9jZpxfR+YZvnVwwGH+nTHPNbV0LG9mt1s9NhhDxDzgwZNr5OdhLcnHuc1VuIdMnhK2UM0/yna4hXII7HBz9Tx9KhJo4JYhSXK19xDcC3t9/wBmYFoyN7nnJU9j6E9K0LG6tb6yNr5kpvrqQZMxwpJ4HOT0yecZP1rn+UYpJGSi5/djufrikaSGKBjHuLhsAHpj19QaqMnaz2FKkpqy1L17o97pUxEpYySjBDjp/wACB/w/CsfZLKDIAzqgy46hR7mr39qX88TWr+ZLuYnMmWfJ6c9egNJcaVeRQW+0xstwMoiMSQR2I9RUvuaxcor32hfszBo0kt5lAUSPjuD0I68fnUsNsFnaSAS+RgDEufvNnHzAY7E9s4NQxvdAQPK8kSKpiD7Dyvce/Of/AK1Ty3cdxEsMSssiDDgJkNjoxxx+Y49TVWTMpOb90s6SIIBdpczSwTOA0QXO2QcjGPXpg8dDzRBLpgkmglijZxE2C/yDIBIAJPX8iSaqwW7x3McktuGUASYV+XOOM9eSTk/jWnHEgkWc2ywOVIj8mTYwPGCDnJP596uzsc9RxUrye/ZnNyytJKjOSzdz1OB0z+VW0ldVDIRuK4HzDgHt+VNt1i+1st4rRsSdxVBuB9Mcd+1XLa333VxGEWVIsMzK+Cg/vdORWKTO6copbElnEdpmd0WFMGQkk9T2Hrz7VPJqD20f2ZZVKqu0NFtJIYZAyQenGfyrN1FZBekrOJY2Abcpz1GeRng+1LFKFjVGhLRnO4qwyce3tnrVX0sckoJ+9ua2jTRafbTXKStLPcAh4wgIzzjByc/Tj+tT/aZJdQeO/JQyDMTrhAWPO5l7evHfsRxWIl3LbTzQRW5jSQEFXwrr17kcdsjv0qxbPKiQvbN90/NuIAHI6E478da0i0ZVaT3fUtblitd6YLDJViDgj2yOnB9OhrEby2uIfJ8xNuTjPO4kdOOOMetdSbOS/t5ZIZI4Y42EjqOVWQntxkg89eBjr0qpDotrHFLM8oMgJjjQAAFmOMrn3A54wM9aXK7hSqwgve3Zn/ZpxJdKLht/3HZ2PTdyT3xx0x/9eK5smM3nboi28MGVuCAcdBz/AJ7Vfu9PeDT4plkJmbIYKCSoGOp/H9KiLJa2zSKjMVB2kHjnORUNNbm0aqexWuN8bLJOTJNJ8oYA59e/Xt+dW0ubjSpEhtLlCpbahEnBbABJBABB9xRpcY1i3eFZoVnLqiJIpVsnPIIzkDGPxFVJoLFLZ4JpppLpGBKbeMdcDuR+I+lGoOSvyDJILyB7oncFdgJSFwC2cjJ6A59/bvU8FhNCsN4WKyOxMZ9NoyW/SoGuRAiwxyulvMBuVwVAIIwxXJ9OufWuit3RLKOCVh59mwj82Ah8gscY6ZHJ9e3FVGOmhNWcopGG001znmIMT5hkxjgnkYHYHHGO9WtJ0uTUrowqBuflmB+4B1JGef05NUrxIoHMciOrIfu9Bn246Ywelei+ArB9N0sXIaBJbwB2eXkqvYAZ/E/X2qILmep204KycdjoNO0+PRLBFtVjiUDceh3ADqSep4H6dsCn3P226uA8ownAjCngHPXHfGOn8R46CrYaWVsPNFIAMho03Y9Px7j3AB6065QurY2YVt3ynaBx2P049l56muiK1NZbFdQryddqhsHknPX/ABbnv857isfVczaiUWRmjXBePKlGYZxnjPGSOuM54rRvrgQR78qGcnCk9Txnj6Y+mFHY1liNZcfLGoQZJj9/qabEidfLKq7CKNhn5Y4wR+tSfKU5UMT6ooH5ioUR8gjOW7EVcFoVIc3MSdlyd3/1v/11m2i0hi+W0ZPlYbOBjAH5UrEHHyj2IxSmYksGnYqo6jAqKLEb5BBZucsucfnQBMHbheB74PFIUJQ/vJCe5XpSNckHI4HTgYqGa6kkJHnMdoz97/PrQCGhizHCMyLwMlRk+/zZqRbpo9qMIEHoHyR9azWEx2jAwzcFjgfmajg+2TzMsLmP5ecnhRx/XFIZpXFwssZIww9V3LWFeusiyfLIeP7pwORVi5W/jGHniZVPPXJrLy8iSZZWcDA2sQeo/Kktxkmnafp19rSR3kGNy7EkdTt3jnHUDJAPBz29eOxXRdGiZdot4p0BCOAq7R9FwCeK4B3MkhjY7VdQN3uc85/P6VNpFkl/qgTVtT1Bn3YijidUR/UZBPPsAD71a0egtzu4lFvhAw2rgZBznnGf6/jXkXjDQp4fEDNAyyi5JkXgIOT0HNet2ejQQxSR21zdsznKxzyiQD2HcD6k15p4x8Ma0damltoXuoGG8NEfu9sEdjxV1WpRT6mEIuM/I5C8sLy3WKSZG/eZxlgSSODwCTj69e1UDlyCTzWvqWja1o4Et/ZyxLn7/Vcnp8wyP1rPWWJYxgbieSCO9cj3OhN2K0gZWw2c05YTEyu+MHnB71YDAR/MCSx6GorguT0HHFIabehdEULsWKKeOOwoFlHGNzy/uz2HWq1rvMZRh05GRUhYom0/MBzkUzFqSdkx9xbQ4/dEpj171RdSi5JP0NWlcO5JxtHQUrpGVIJxnt6Ui4tx0Y22f91kj6Y7mpLiyLFWhIYEZIPUVFF8mUVuh4q2rbkOUZn6EY5FMUm07oqC2MUvOGx2HNNnmkVyDWjDaSPKMb2JIyka7mNaH2GZJGNvp8UB4BN3JH5n/j+APwAoSbKSb1Zzlqs88uI1LEda6DStIeaVWmdVtyGy7HGTjHyj+LHB7DtkEjOvZ6WY7RZbgRs24/Kksbjj1Kk9vWrM88DeVGgSVlLEdk24HY9vlHXA9jWsYdwduhEsFxbh2spFtYS+fPkYhn4xyygs3I7DAPpWTqWkprNxEs+pk3AyF2RGQMTjuSD0HpXR3ro1ozMTMWwAqg88469T9OOneqT6cwQSR6dJagHhyJAPf7x/lV2WxKb3OP1nw2uk2QnN4HfeE2bMbs5Oc9vp7HntWBXoHiUJ/YV2bdX2YTzHkIyTvHT07cDPTvjNef1nJWZrF3QUUUVJQUUUUAFFFFABRRRQAUUVZ09oV1CA3CK8W7DBjgDPGT7Dr+FAFau98NAX+nQMbiGPyQUdX3DkdMYB7Ee3WqmmW0N3fyQ3Wm20DIQyI8YTgg9SeOnPP+Fb8dkthE4V7QDBxHHIpA47bcgZ+vU1vCNtbmM5X0sQXemWMwR7m+kjcDCpFBv+Xnk5Yc5B49Oc9qdZ6Bps0ULLqcocEEr9kBBJHIz5nQY+p9BU81ijwFYb22eZlBxtkG3nkfc9fwqxpFnJDpiify4ZxK3yOcnZgYOVBHXdTtFsV2kVYrBYLmOaW4Mao25cQb8sOQPvDjr3/wAahaw01pnH9pTgArg/Y1GRnBP+t6+g7+oq7qDImUJilxk/u2fKn/gSjtn8xWbFE7OkpVghbgMuMgAnPOAehH1GO4zdkTqazfYsLbRXhMaKg894SmcKBwgZvzzWexhs9Rin82GdEdWxllxhs85X2965zWry7tr5XjYohO4DP4cjt06GuhEU15ZNcQW0ssYi27okLgEk4zjpycfl60rp+7cdmtRLLRICsyf2jZtIQQilmXcPXc6gZ9s07U9EuzpMDyLCyRJtLpdRN1J2g7WOOe596cbG5t9Ntrqa0ukVWYMxibA4QDJx65pqXcax5ctg8kj0zjii11uK9nsZ7eC9cSNAPsTs43Ki3kWf/QuD7HB9Aeall8DeLbaEyy6JdIgXcSF3ZHTtnmq8l/MieWjHYfX61p6Xq17H5kn2l1VUxwTypOD+HIrHk8zXmM220m9guzHdxzwShgpiYFWxg/w9euK0ZpmR2CnAB59TWgusMJ1nuFS5dCCpmLMVIJIIIII5Oev1qKG9054JoZrIed1WRJiu0cDBDbs9+eK2heJlKzMXzwWZNo78hAcVKHWHUiYVUtHIWjynOQcg+v51dS1tygd42Vgckq33huHbHHy7vXnHrxVNtvnUkJEZW2Ak/KMjP4Drzird92iU+xbvNVnvbwtcTb5RjdKoGSRjqcc9Mc54qFoopI2RfMQlQPvjBPr90nn07En2w7+y3+0rGgE0oTeyW7CXCjqfkJ9P85qqsisFYMNp6YNFosLyQSWjpJGEXemPmY4GDk8DnnjHPH09XQ+Hr1ovtSQtLbGYwq68729l+9jkc47iopriXyvlHzZ6VLI5i0yG4675XyPXhf8AP/6qTjrowv3B9LurZhcPazpBu2l3iZVU9hkjr7Ul2XSGZw5JUEjPsM1PpOu3llKxt7h41ZcNg/Kw9CDwRyeo71dknEGqQwQ2lnKbsRx+RNGixnci8k4yuSRkqR1pOTihpJnKwtlFLJyOmTzT2lO8nNdRA1qLr7FL4StIZmZ+JLmfDYYghSrEYByOOu2rUrxi2lgbwvp9vI0e1JHlkyhGcHpk549M9zWN2zWyOKacMOtUrgqWjBHJcceo5rtr/WbI2MkH/CIWkczIUSSOUttb+8MHPr39PoeUaQX2y1QeUiMHWNkBYnABO7qM4pajOtOl6TNYzNbNLaXITcqH95G2FGcnqCW3HsAMdT1gTRy7LFZX0chkKgLNE0RLZAHALL3PJYfgOak06fMV85wpW3IXJx1kRf5E1Wjlw7YJxkVaXYlvuMXTrmJXE0BDK5GVIdTgc4ZcjAweenvUEmY1dCGV1JypGCPz/GtX7bJGpSKRlyRna2OR0NWdPvVu9QA1OP7bHGs0oSU5DFY5HwT1AJycZxnnFaKTsZ2RzsCtdyJAjiOR5AqyOcBCTgE/SrKy4tiwI3ckg+3auisLjwwuqW+oLY3trJA6yeXFcB49ynIOGG7rjjd2qtb+GVvGVbG/s5kkd/LjnlMcuBnllxjkeh7/AFpKVndlON9jCu5Et70wRtJFEhwu8YPHHPvS+aHQtIFJBBz15FRa1Ex1u7IbzN0z4+Yc/MfT+lRRo8cZXkt1xTuK1jorTxdqscKwG/keFSD5c4Eq8dsMDgfSq+pXaag8ss8UaS3EnmeZEm052gY9MHGTxySTmseONwM7TnqealSZ2SOMr8yvk8cYppITbNeO08O3kURupdUACNmJJYnBY42lXKA/3sgr3GCO9G40CyeHfZaj82yRzBNEylMY2qHXcHY5OOFHHbOKzJD/AKaqScIORjgc1p2QxDsBGCOR3FQ9NivUsWPhTV7ywSW1tlnhBKlvORdp9CCQR65IxU0/hnWbYHzNIvDtIXdHEZATj1XP51QkuWEgAJ46nHH0q5H4m1KC4ldL64DMQWbzSc4xjPPsKdpNCvFFJjuBUj5lOCPQ1BNkW2CM85+ldbFr82oMGvrW1uSE2v5kSgvgk53YyD8zZI9vfNj7J4VvoZLi/W8sDnPlW5zG3svysQR7/wD6kpNIdk3oec/vGB2nOfXtTvKnUDJTnoCa7qTw74McFofE80ROSEeDJH/oP/16r3nhDw8qfJ4ztPMPQvEuP0kJFRddUXZ9DiWicDa7qP8AdbNSJbKxJWQD6jNdP/wgtlPtEHi7SndjwGDLn+dEvw61WBybXUbG46dJNpP4EVIPY50xFDjqT26ZqcTiSe3jt/MIiRhNFJKdqNkqw4A6gKcfme1dHY/D29vJtmoajaQruA/dShifb24+v0rVufhdDYzRPa+IoIpZf9Z9oQMrH25H5c01ITjc5eG+AzlxtAxhT09uSasLeoI/vcHnmtafwNZW0x+2eIbPzCcDy7fGOe3z9BVC10OwXVrqO8vpDZxnbC9qF3Pyeu4Y6emfxrbnb2RnypEP+hXClri3eeIfwpLsP4Nz/I1PDZ+GFFuZrC/YhCJVE+VLZODwQTxgduefYs1KHTNOuBbabJPLAqjLzbcseemO3Sqgn3DI6VMo33Gm1sdPHq2gLGqm21IYAGBMpA/EjNFcobpASPm/Kip5IhqVrkpNMVa6VWWMkIQQFIz94Ed+Twa1dPmjtLeO5IjSKSNsGKAsuRxzkEjqBxjkjOaxX1GSWHiEyAf69yTt5+g45PFTo9zCkEcrIsfy7gQXVUbhjnPB4U/hS32PGcHa0tB1tLDdS3kd0mHldWh+8H5J6BTt9OCO/HNNit4rW5UJF54GGlEhDZDHgqNoK4weSefanTxK8cjLKyMHwEPOTj17cU62EkcqNDCJPPDx+aR0AAJz+GfyzUpJaMpTbWmxW/sgQKJp7mHaylgUJ37sgEY9RkfUYqW8sUe1cpKfs4QO4LAE8jkdfm/zzTxCohgvpMSQwP5cuUI67QQT9CDn3qpcNbG7dIpWltnOIV2qfrlv4ef50NpDi5zabexPNaRBYMyPJ5OdiyHAC59c9+Dxgc+9X5L2GG3ktzbsZJZQEJ+6SeuTxzn17VStrpprGZriMGO1IjUqQd+7oG55A28H09aSzuY5o2DTpHKZCjAttKjaWHHXHGD+FUpLVGU6cpaS1sXhb3NlM4kmXejBkEXOPXOSCf5U2ba9mEcbELfuXPyOM9Qex6ng/n2qv5n2wvFbjbdLnfIH/wBbjAJGefQ45qUQ75pFvZRbSp8yboPPEh2lT0OODjg5wQc+lPmdiPZvm1diTXY7cSrBMkHnsSVn2hXZe24+vT6flWUllKbiKQ7408tgk0KseB/eAyR7gf3hVu6tpNsE13cSP8oVJCpULgcZz6Y7E/WoQn2m8W2m+TzdzAuigNnAOOB1KjHuPc1Ds2bUZOK3uVVtnRpWm+SHJKs4G5m6fUevNWredRcRpaqpKxkELkliBz9etTyRyXVizlxJGjFWDKdgyByB2PUZ9qpTefZ2VvdW9g0EbnaLhmwSwzwo7dOtNXT0Luqj1f8AkLPbSSlURgrPIF2fxk4449OPxq1EsFrPDbyS7QcghwVAz3Yjtk9M/jWYNQvrfVZJbhlMyu25c5Oe+D17VPeEzsl1cTFWmA+Re68HnOc9B19Ki/VFyg1aLehp5cBre0lzIFPmeU27zB7H0AGfaq5juI2zKREwj3p5n3Zcfw8Hg4xgH07VWsNam0i7drZE6/cUZVgO1Xb6/uLyUS3cKSQ8swjXAGewBPIBNVvszL2bi9iOW4nFtFI9wSTGFIzuIxU32q3gaKGedfJcCQop6d+COnX+f0qxDp0NzDbJJJETgsUSZTtP0459u+OM9Ky3u7MxLZ3CsCjMUPQZ7rkZznAzjHP503J3szNRT/4BQ1G0jivydPuBIiOXV1OcAc9D6VqaLFZ3MjzXU0qXcjONzRjyznoc9u46YOfrSpoEVzZPqNrIIk5DK8gI24J69MEggepB6VQtraGWV2Rd8HBJDD5fXgkZ+malK2ljpdRTg1fY0brSZ4oEg2AyRFmESZL4ycjdjGABn1579BmGc2yMQjAMdxwQcA+3P8q27dL0XmyzvPNLIZlEnmIN4xkjbzkKOfYGszU7cpDDdieITSjYIF4C9RwQeoOPzpOPVDo1dVGVncp2putd1m0sJJy0cswXcegDEbj+Q/SveVs7C3tmMdujN2EiE8/U5rzDwrqSX+pRXE0EX2qxRmefYBkbduWIBYnv+HYZr0O11Fpgjm6tHQk4ZOSfoRn+laKNonfTmtrWLUSgTMI4kbA5xx34XHbkdfUChlyR0wqAyO65GOCSRzwOD/3yvQVDYx3NxJLLbxDJUxuzN5aH5jkE4Jz9Af8ADR+xSOrw3N3EhfBkWJRnjnq+cjJz90VV7Ivc5ht+pXhnggcqh2RYBZgAep689z7k1bTSrtUjdrZ/LPJO3aT7c10a6PZQoN9uJABnM7FwPpuJA/DFTDywNsaRAegfFQ2Ukc7Fpl66sShUng5OePwpG0+aKRXDIpUZG4MeR9R61vzXCwYRmtoy3ABBbOf+BVzmo2lzrUkltaGIlFO+VvkSLoRnk9TwOv8AOhJsbaQ2WJB+4a4QOefuZYj6njFNWOBT8twzsW2jcB6Z7e1Ml0ixEkNxfXd1cOreWywYiUNychjklSc44HFMmj0e3t0eLSIjGXActNKzkYJyDuAB/CrUCOYdIVEm1fnKDJK1BLISpOcljyDwRj+VNuooY5P9BswkyjKr5rkyjdyGLE+x4x6UsciyRiRVUgrkAZ9CfWplGw4yuUZEYsfNZlCqcZyRn0/E1SeGQRq20ndyMD8P6VpPNNyQ4AbgjaKqSyuZFy7qAvRABzjr+dZmiKjLfncqhwijpJkD/PNQwRDEpl2l9p4UH1H+f88zG4lEUp+0kkkcP8xPXkf571HAk5k8tZn3TKBjZnfyGA9skDkU1uJlaUK7BRgHI5B+YckfSiGSKdWt8bGL43bSrfXn+eeKHQ+WTwQ3I49z3qOBlFwJUVsCQkHdnH0/wrS2pKLM2va5JcR2UepadbTq20NIrLISP7xKlcn2x1rsNLttWNx5l9rKXRZcMghwEPoHGP1FcPdyLcTtG7MkeQWC3KOpGQMeWBkD3J4rVsPEt5piPCsaXcCgEK5/eKv+y38Q9Aee2aUW1qEkmrEvxBT7DYwSySuE8wItsYt0L555JHsOCfoO9eazXVtqVzFDcRW1kFY5eOER8H1x6fSvTJfH7ahIbAaQ81q3EkM0Pnsw9PLyo/U9qxb3S/Dl44C6N4hijUlwghUBVPJ25OcccEk1M5OTFGCR59MttHvaC4kkxgfMmP61F5uegB4rtZdM0GKMpB4a1J2ONr3dwUGOxO0YI/EdOtT6YkOmYddA0eSVuQ07NN9OC5HfqPzrPluaWRwqbhJyAAeoqZY3uHZI1bpyq84rubu4vDLlrDw9AztlI1s/zJO38OtZ15q1/OyQNrEflBtvkWiuq56Y27Rn8aORisYFjpF1fbhaxCTacF2YIg+rMQo/E1ZOh26MRfazZxFfvLGHlZT9VXafwat2OCGFhvtz85AVpZDuPGSCBjvz0zUltI0d+qPMkUIJKiGIZXPtxk8Dv2qlAXmYi2WnGRItNS+vZsANPKqqg99i7iAPUv8AhV9NLis4y8s4fGN6IMA+oz1P6VrHVYrm1MQ1C/nJ4Alk2jP+7lv51lyRHcyu6xrvHC9T+Pbg/pVKKQPVkwSW5tEaKb7NaNw8aA5OO5TgMfqQapz2dpcXsvkJdyyM5O1ZlYZ9NoTP61qQxy2zrDGghUfMWm9fxBPoeB3pbm7YTyLd67cSLu4W3jeUAenzslN6ghLbT1j0YpcQ3VlI02RE8bKXUDnBbGBzxgHmhQkVsyAxwxDr3LHj8+ncj29KjRIvJd7G6Zy8m1mnhWN0wP4cMxwc/pTYwoKqpEkueDIAVBzwcfX1zVJXE2WDdhLeRYwttHt2o2755OR1PX+Q/WqXn2ZmCW080shbasRhwWb8HNXDDvjlK2r6pek7sMHIc5AO1VIY+uT+Qqgxv4JVE/huGzUsPnltWQrzjILc0XadgSVrnM+J9Wku72SxACw28hU4x8zDIJGO3XGP/wBWBV/WYJYNWufMjZBJI0ibh95SxwRVCspNt6msUraBRRRUjCiiigAooooAKKKKACiiigDf8N3h8yWzJO6RdyMOo2gkj8s89sV10a/YZZTJCTuiKMrDBG4FQQO/JrjdF0m4kniuy/lop3DDfMf/AK3NdbCxguGUsOe5PBrop3aszCdr3RNYWl4sjupjZcZb96AcBiOASCefT69Oa03MsZzKAFOF++pOT6jP1+lUoDNLBKsaPIxCk7FLYB5HT2yfpTBNbmHc80QdCBsLDdyCf6ChMVrlC5luLm5cRxys7ksEVSzY6n36ZNX57S4itIcwy7VUA74yMNjcRz3HPFRzMrRhoyMDv17VGk7pdR4OCDwR1454/GrV9xOxyWpws+sTIDkOwwMevSuwnmaIu4HKpgH05HI/z3rLuNZ1G6v41e5YRlwuBjdjoRu6+tWtsbKAMrkFQdxyAT759KmGjY5PYlg1q4QeUbh/LIxs3cY9MfjWi175N28Yt7Q7GK5e2jY8ZA6r/nr1qpLp0ULxwso3xErLhjl2HfJzjt0H8+FmSMzvIJDh8HzGQDJ78Angew/AcZp+aJ9GLM9l5qPLp1qoD7nKb138g9AwA6EcAfePtizb2enrahWluIsnk7RINvX7uV5yF6nHXgYFUpbR7iPAkh3gZXdIFB6/3sc9MAZ61V8q9iV98Uu0dWCnAPXFZ2Vy7sutpkjRealzbkscBGYqSRyeSAvf1qNdI1VYpfItJJZC6/6grMAMHj5SQc57ehpyyw/YYhlRIJWLAD5sELjPtkNj8aS2dZJ4IXIxJKqfN0IJAOfzrVRdtyG1cqPNc2sv72GRCGZcONvpkYPcHFW7aaO5W8B5aODfGM/xb0H8iR+NMa/Y8ySyNjIGXIxuGDznuOKYzeakIVRCqs7gIq8lgRzke5x6dsVV5E2iYN34jntL544UjdUOMtk8/wD1jW1aXKahZfaEgVmZGcq3ONoOefov5VymqTW01wzwwJEQTkKWIJz15JP61veFcPp74uY4pyXjihcFfM3LjhsbeScYyKw5/e1NXH3dC6s0E7qk1uwCkeYYThnAPvkA844XsPfMz21pfKiy3U9uAxEaeQrBBk9W3AtgYyQufQVnbrm0ndZIXjlRsEMuCParV7OrwxyMcMy8Dt71sknsQ7p6iPpJs7rYt1FJkhQVyC2VVwQrANggjsK1L9o7fULO78vzEgWJhyQGKop6/UAcVhTmWG4dJkAkiIi5XBGwBecd+OfetFdWubazlh8+RYWUhlVztOcg5HQ//XqLNoq6TMXUdYnubkzwNJEWbJO45wfft/8AXqst7epcsPtcrFDgFXPP+NW4rexnkVRDJFHGCS0TDJ59D15I/A1tweG9KvJEZbuS3t12mUsok/LGCeSvbuTkAVg07miaHWXiMWumlprUzTSN+6lkcheBzwuM/iSKtR+L7RY1MsJEv8QRQQPwOarX/gto7WO90e/tL+N8ZgVwk6MTgJ5bHLcnjGSfSuaRZLO4KTxYZThg3BHt7daak0DSOyktrCaJNt9NbNvzIGg3oQW/2Tk4XnGOSvUZ4lHhy8ubcyaXPZ33zfcin8uQD/aWQLj6ZNUMZBzxzwamgkCkHdjHcVty9UzJS6NFS8sNQ03c95aT24J2hpUIUn0B6H8CagsbgRzSPMWVPJlGQO5jYAdPUj/61bGpzTDUprdnZoon2CNmJUheBkHg9B1qL7RYXCRR3FjCm2QF3twYiyc5UqPl79QoPAp2kkK6uc41w4lHDbTjgHrVqG+eGI8biOct2rYfQdMmnH9nasmGYhY7tPLKjBPLjKn06Dmornw7fwwvm0kdQhYyQ4mQfVkJA/E0KVnqNxuV7i+mNujSP5wh5VJMMMA56Gliv4pUWGS1tiv3ywjCMCcAcptzwB1zzz1JrGaSVEYgHITDZ/nTvOJs4IVx/rM9MHJAzz+AolYFexp5t4r3YWcxAgMoHzEY55zjJ57d6ltkLSQxhI8ylVMu8bVzjlvQDPXHY1nXQe51iRAH2oFU/goH9K2YVWJQigfdwG9hTjG4pOxkS6dc3sck1vGZTbBBN5S7lUv93p75HscA4NVo/OtSVCmKQZSVW65/pWmbplvACdqMSMepBz/MA/UVeXWb3yZI7iV51clSk6iZecZ4cEDpRySWwc0WYMcbtucOWePJOTwajlvPMRCqfMMk454rsGvNAuSkd7pTWiLEMPp8gRnI4OUIK49+DWO1jZLMqwM/2YoMvMoDBsnIwucjp+OeKFLugt2Zntc3CwxyRH5m9qi1DW7lYY41KpIwy5AFbl74a1S7tLSXSLae/h8s820TP5Z3HhgBkH69c8Zrkb+K5h2w3Ebq6k71ZcEH+dZuXY0iu5XN5L5rSEknGOeaa8vmYAyD1JJ4pCuR0/Sk24GSKxuzSyHpcyRrt3HGc9avwajfQRSSQXDJ5g52nHFZvBx7e1SQ5bCHoe1NMTRsweJdWMsTNfT4QELl93B9jWrDqZvomW7nlldjgeY244Hrnt/9eubSNWYjnC9MdPzq3HGFYnzcv1OOP8itoXMpm1Leg3ck+5mmkJ8xiclye5PXNQS3b7Cu5gSBkBv8+lZuZHw4jfrjgHB+hqRY5FJ3Bw4wSpUk4/EVtczJYbpdj787t2ee1OgnWdGZegJyTVvTreOe3dpI43VzwuOMDvkc/rVh7WzghmcRiPdKWUIGIRT2GW5xxgH86xle5orMo/aI/wC5/OitWLQEMSF9btFcqNymNwQfTBGfzorH2sSuUwIdURbdrZY0ZGQZLHDL0Gfcd8VJI0qI0P2gkxny9pU5IHQc9vasexTJdHZIZsboyx4OD0Gfp/SmLc3Ml6xEreY2FLMeg6cmp5mcEsOm3Y6G2VDYXL3hYhQF3rnIGcjPPA64xULXEdtHCIZHMzsXKlmJRs4wOcbT16ZwayJWu4PNt2nDIy7XCvkZyDz+I7+lW0vlljmjks42EkYCAfejcDgjPOOORT576maouL7pj9Qe6kgZMHczbigyeR6j88delUmlnWE2sgA3DeXTjI/p6dqt2t80zCRolmlhdchwCGA4AK555PT/AANQX95c6i27yAAilR5Y4A569fT9Kl7GsI8vutCwG2ljc3E5SQgJgEDKjGDz1PH6VRnXzJx5JyuSSfSo0DzMittRh/Eenp/jWo8IspwskgaZRlHVc57g+h4NKzepo7Qe4mVe5hYqWjU4I7Bfw/yaupNiVbU/Z5VkYRq7McIMjnI9M981WlaOXT7eSORUkziSNIxwB/ETnJzx9eegFTWuiXlwt2ZIXWSJN4QuIyuexB9h61aujCcI2vJm6dUmn0t9PltkMJ+ZnD4ZDnhsY6Enp65rBlt0jsFE2W5JWQMd3+7zx74H156VJb2+o6NqEccjEME3+XDMuSMA4wchh6j29qtHUZpxe2V9DGxh+QGONYyp98cHp0IyeelN2kYpcj93Yy7cXG6QwfaXikjDMVPGB2f05Hf9anMM7xxD7QTISSimQvwT93Hbk9h1NX4v3ciCKBfK8sjAODI/Q8dD+PXtTkSOFLk38EEiTDYilgrRnB/i2nHc/h7U1CyCVfXRHP3kFxPK7BEBU4KhcCkkRY1RFkVuNrL/AHa35NFtU0+JFQvdKT5zJKW3nr+GB6deuawbqGCGUrCzuAuSMYx9f1qHFxOilVhU91PYRpJUmJWUgbSCwJ5/xrRjubudQAFVYMM0gYr8/bgd/wDA9qzJ02W4IMgjbhQ3H51oOklxGlxbyKCq7ZERtoOD2B5PahaIckrbGjZRRsglaUee+3ymLbQWz0Lf56VLeC2WO3trm38ueDJk2fc3DkEMD15PT047Vg2dy8EXlM222nkDPtAJGG6/h7V0MMd7rNtdTI/mJ5imaRmG5OvzcZIyOOmOK0ck1ock6bjO/QiktrW+u7qeNWYlkIeYgOnA+Y4+UA9+DjrU0kdnaWcCF58SRkts/hf+RGTnjnBpbW1t7pIprc3M18qEyxOQzOQTkAkcjGOx6VWgn8rVk+y/Z5VwsYRtzqCOmW+U5HHTH0Pc02FZzbV9i8C8kTxSSXLMFwpwVJ2jdgpyDnC9u2Tzisq5mBtFtS8Rsbk7xFGNuxweSVySD+hB4ro9J8LavqF6bq4kSyt2IJVQ7l+c4AbIIPXLegrrtN8K6Xp7tILd7mdjlp7g7mY+vOcGk9TqoYaVr3MDwPo8ltPcefb7LSUfNcMuzd7DJByOcH3P4d7HZWMrAxksi8ARgMMYxyDkflSJaqZATsP+8parzwRIB5skq4HB8r+WTTbud1OnyK1yWEWccZVoTg9sHH4AGpHvrYgoEwgHdWH/ANeqRMKqG+13IJ6ZjA/UvWVea5bWjNFHcNOR95VXGPr2/ImlqzTRG48mnRRGRVjO0Z+/J/jWJqeuIbcNbgxR9SVY5I79Saw31iKQ75J2BZigiLMQeDjHvx+tW00caZY2moa0cTkjy7BiMAZ+/Jg/XA49/SrUO5DkMNvqWq2ZnhVLWwPMd1cSbQxPTA5Y/Nu7c9aS/MEentZWM5kFvIZLmdhzM4A+b2XHA9APc0mt3zXtvMXVQu2NRg8KynbwPTAY+mDUHhq3uILK61WUBI0iIgLc+bJ6gdwvr2IHvVok2otNkk0uJLp/s8zbX8l0Jc4AA44xnaep79DUF7/Y9pAEeORwV+YSykFSVJx8mAfTmoLaW91S8UxK0jlVLyEkBBz8zN+GOevStK/sNMZJLiZWunwvyglE3DPcHJB49OnvimSZ07abcXcMBt2g85RtbzXbbnHGGJ/iAqqqC1Z4JImXClsHPGc4IPfkE/Q1fRNIubgZsgAieYJFmfK4+b5ctgdfSq92k5uGjLB8hRGwPysuAARn3B/GpmtBxauUJXjEa5BPJwBxVea4iVzE0AjVQQQTlsgc847n+dWHiDFRIpBUEZBAyeTVV7ZzG0jSIRnHzE9epP8An1rE20KYkRUeQQRAMcBpcnH0A/Cs+6mZ9nmRuMc5ZMLx069q0nsZZghjDsDwWCnGahm002q75pmZAPubcf1pAQzJJHDsaJ0YYJUgZ2EZ9B69R71AzM2ZHmmk+XBdjk9ec5zkVMJ0YBAvyx5JyecHGPwB7f7VFynktGm3KuwU4PtWydydiu9yZLmSPdaldu3AgCy9P7wXk/jzSbvskXlHKoQWZ2TcoJ49Pl+vJqw0yeX5fK8fK27hvpx174J6EVBFJIZMFye2V6jpStpoHqQWItcuYZ7WS64IEO9dqgHJYsqjOcdCatOst7utJYIRabvM3pIAQQP72fc8e/1pYPNhf7RIbqRWxtM9oUiyT8pDBuTweowRn0p5lCz8RxCRMOf3jYHfoGqUhtlG/jjhijWWyjKgbQTccngdQGzj3pYJA0ap5VoISoBUjJUDHfBI6duadqciTRRmRbcyNuLY8w+mOfektzHBFHsWHzSo4EYIGfqKdmK6JBMsqw+dLC+M4JQllBA6kjJ/E9ap20en294881xFOQr+XFGmCWKkL6dCQfw71N5jxgFpLZcHOBApx0yOR9KmOqwLa3EQubjzWmDB0hAGzByp+bgEkflQ0NMjkWKa5UzJKHRSfLlPl7M+vTn/AD7UxY40uoyZ7UBAcELvYcHvj1PXPapJ2FxLn7FOqGNfmkY45Gc5wByKZbReSykwWyEgsm/bIPTkHP600hXJ11eN8G51PVbm1hALwCMAbegG4yHAyQOnfsaZBHuuPNYLAA3G3Jbr2/wJFNtbaa8muVttTgXyInmk8lHX5F5bA2jPrj/CnRMfNVlAQbxh37e+OvX0HahIGx4EUxJSJnPQGRsAMPTGMfQk0qzOt6rXlxpcQVwxxbIXxycbkQ/TOe9VI5l83dH5lwoXbgJjnngc81Pbwo19Ck9pp0Adl3yXk7RkAnkkGQdueB9KcloKIx3uJbOZpL3+0JQNqSAyHyxg8fMB6moVO2PDtyP4YzznjgkjFWp1jlu3RIoIoTJ+7jtpUkAUnu6lstj1/wDrU2NnhhRI0S3VztaR1+Zjg555PYggccc1URSFvZ0+wtFeXr20Tk7oIY95YDoSu4Djn7xzXPpbWazD7LLPcMHBA8gIAMjqQ5A7/wCNat88NlYPM0Al2Id0sgJC5IGcDgdQMHNchc+Jb51aC2cQ2u7KxhRj8R0zWc3aRcNUa/iu7hOnpauyvceZkKrfdA78ceorjqc8jyuXkdnY9WY5JptZylzO5cVZWCiiipKCiiigAooooAKKKKACiiigD0XwL4kvRYvYC+nUwEPGglIGMYBxnt0+mBXQXWr6o8Mrvf3OWU7v3p5GDx16e3SvHY5ZIZBJE7I46MpwR+Nd54Z8aS3N1DY38FoZWyI7n7PEDnsCMY+hHfHXrW0JpaGU4Pc6R9X1ExNm9uS2CdxlbOfXr60ses6pFCgj1K8RABhVncYx6c029nRME2dnlmJz9lQbRxwMDGOfT8fSKORPsxla1t3WN1jKlpRuyG9JBj7taLuZkYuLl7k3T3Er3DHDSs5LNjjk9+Bj6cVKzKLOe4ktLaURoXIKFBjco4KFefm69f0xILrTySrac8QHQ29w2c++8Nx+WPfFOSS3urSax854FkVgrSQhlHKHlg2f4Oymh+gLfc5JJNPWRJmt183gnl9w55x82M/UY9q0DbpLHE6OfnAcl02hD1weST+X4U2bwxcSXmbW6srtMDHkzBS3yg/KsgVm64wAeQR2p88M1qEjuY5IHKj5ZVKHPTv9DSjYcrluSC6lld0SS4DSEb4lZyxJ64xu/EgVVnYxlA2cglTuPIPBx+VWIIiyGYtjaQuO5znn9K1ItTv4EULeTlV6K7llx/unI/Srd7aEKyepjxzcDAIIqIxSxXciOEYFdxOwZw4zjOM/dIH0rYS/t5WJu7G3ndsbmQGFup6bMAHn0PSqjixubwvKs9qhAGY283HHHBxnsOo6Vmrl6FRjFExby1UMOcIDnknuPc/hx0qBovOdJoDFbuv3RGHPIPXJYnPFW7uxZWLW1zBdxgbjsLK2OP4WAJOTjC5qW50i6sHLS2ssUSBEaTaSm8qCRu6Zyc4z3qroWpQNoGCAOqEt87PxGvPBzycdP4ajAKXiWTsPMfhQpznOQMf4dauJLtjl6tlGClRkgkEA/h1qvdz7bSQrIAyLnB5xgU7voxadTlrvQ9TM0jJYztGSSpEZ6ZrodLcW9jDZSx7WCHerjn15rl2vrwyFvtM/zHJxIRk10x09pTAPM2sUQO7sWO7aNxPGcZBxgHr7VlTdpNms1dK4jvvvYyJni+UBTncoA4A2njGPpViWC28pInDyXCq2X3hVYnocBTjjtz9fStf6fPZ3YRlOAEKyFWUOGB2kbgOCASPXFW05VGOPmXK/Qcc/iDWqSZm20WLsx6hdz3ILMZpXmkWbaG5IJ+bO1iSTz8vTOOaoXFjO9i8sSP5G3II5AxzgnoD7ZzV4riIY5J7Coopri3lZred4Z5BsEsTFWGfcYNNxaWgk1fUrWKpFoV9KciQSwRkH0O9jj8Y1qC2vCkZilbCISUPOecfyxXRXGpW86zrdxReVcgbzGoRty7sNuAx/Ec5BJ6ZFRReGdJ1CRBBrIgLsFUXkLKCT33JvAHu2Ky1W6NNGc9BezJeiZWYNuyDk8HOa6W+1O48UwwDV5GuPs/CEkBx0/ixk9P1NNPgjW2t4ZrO3jvbdjvWa0nSYE9SBtOc+2Kr2zJATG21WBCkehxTglJkybijRlt7O7VltHmt5VQt5cpEiMQMn5hgr09G57iqc9nexGJYBFctMP3QglBLdOMfeHUcED9DSGb/SI1BwfmBK8ZGKQOIgHjLDaw+79a0cLaIlSW7LGtuTrV3IqkFpWOCuCOe47GswBpATyG/lWnJfJ5gE6xzllK/vQW47YIII/A1UheBnk3DyS0m1DyyhSTwerccDjP07002lqS0mysoeGNsninwXLeY7ZYMQASOCPxHPfmrR0u61DzEsU+1NGu9kh+ZtpJAIThu3pwCM4yKoLa3NtKzXEMsfzFPnUjJUAMPqD1HbNHMrD5WWpWFzD5b4CgYB2LkdcZbG4n5j354znAFQSWEUt4jIFhh25cZLfOB1UY4B44JPfmnp0OTk9QalX0J7mlyphzMlufDtzZvNqMF1Bf2wYb5IZfmQsQMsrgNyW4wD746VHfQzWJjhlililUfMsiFG+uGANOaNJU2uisPRhkVqya7eLKLeWUXtskSL5d2omj5UN0bofpg8daFzRemo7xlqca/mLOu4kMDkZ/U1pKsjXsjBxsUYA9T3reDeGb07prCe0dWGWsrgkEHr8km4D8D3pNQ0jTFs55dM1yOZEG8RTwtG5H90HBDN9AKcZWeqE432Zz94hMgKEenFFtclUZByQen41IsUiRSedBJCFYHfIpCsGztIJA4IUkHvg46GmxIBmRcFHHBFP2iuJwZsaVqN3p0v2m0uGhl6AjB/MHg/jVnV/GV/cy28twbaSaJcBpIFdu397OOn8OKyY3Qxg54rN1STfNsxyVBJqJJPUuLexuR+NtWiaRopYFM3+sCWsY3n1Py1Xv8AX01OIpPpullmXDSJaIJPruxnP4/lXNBXT1A7ZpQMkZbkc1Ct2KbZ1mmazpFlEsJ8OaRMgXnzI8uT67nLHJ9sVrLrvhO5Yfa/CkUIAx/o7bc/gu2uAzgZ706K4aMN6470ckewuaR2Qfwjc3ggGhG2Q5PmzXM7KPfark/rW7f2HhTStVjivtF37bdGiktndSQxbO4bsHGB1Oee9ecQzTSSxgSbcEYbAyv/ANau5+JcguL60cMx2wKSdw2n5mwRj8famopzt5Bd2uEviLwrDCY18PQykEsWaCPd/wChEHt1NUV8T+H/ADA0Phizk3cFXhiK9eTyhIPbjiuRZkXB5JdcjPSq3nyJPhImZj0VF/z3qpJLoC1Ohkv7e51Caa3tILKI4AghG1R3OAKrXV4GhdiAQvI44NZUdw0U0iSja5wcZyBWzpcNvqN7FbXTqtuGLSZbAKgbjk9hwRmk5dQtqU2UTMZX2MzncSy5JJ9TRUCXEmxdsgC44Ht+dFRysOaRh7Q1ynlSk5Oct/COpOKtLiVmmHlxhB8yr1Y1FDcxRwSq1tG5eXOR94r6Z7etFpcW8GpRSXMW6ADDKSeAfT6cVild2Mp3aem34jtplUyxxgKSNx3DOO+B3p8gjWVQ0i+7HJH14qC7uYG1C4awUrbvj5WHTjn9aQM8cTxvsxnPJ6H2xU2Eo6JsuzLZB4FtWleFowWZhghsnI6DIz/k1nNJGpcbgW3ZD85I96unfMCWZmHTIHXtVG9hjDfISGJ4G3g/Sm00VCzdi350U0KlECBR+8OOfw/z/wDWns7mSOSGO2dss+9TtVjuHbn8azE3CMj5lABJx+tOML/u2hkYrENxdRgg57j8RRezuHs4u6Z28Mqazq11HOUjaRC5lZSXhPGAoPuRnPbODUN6s1pJ5b3b7poQCwYgyDtkZ9c4rCAJhjYSs1xIglkaX5cEE4wfTABz+FWdZa61CLT5buRRb4CLKoJYqeufbjI+pq1LSxwOh7yV9C0Yo38k3XnMlsnmERvtkyOByegP07dqqiS2jika0LfvDnY7bnGc9SOuD3AqC+u545GiuSkrybQ1yEO4r6cHB7H8BzUKwShX8iRWRQQz+oz0zST6ov2fu2bNNbqZ4LeDK7A2zLLh1HXAOfXPbP8AKrtzLJNuimwr27LkucZJH8Q+mATxmqUNvPstFSdXEwOyL7xDHBJ9iaZc28lxdSu+955HxIzkgZGADnpVanO4xvuW/wC0dl3F9hk8pEYLtd/3Z4PGcjAGcdB1OK5/UYpbW8e1cFJQSGZnznODnr0Iwc+9X2t18xbGV1t5GBKs6na2eAGP8PfnHerln4F17Xp/Pj0+SCAbQJ7tzGCAOMBhuYehAPAqZXdjtw0EtUYkiOT5ksrSRsCpJ7n2BrUisopLKExyGOOVSWBIchsgdCVySBnC5rudO+HNoiLHqmqPLgFjFbgBQccneRn8NvP1ruNI0rS9LANnHBC7Lnecs+Pdm5B5PGAaajobSpyk0eaw+ENW1HRbK2tLSZoIxkyXUezYcndyfmA5yMZq/pXwwuRP9qu9Zt7dfulIhuIHHRm246dcGvTZN0iBmnRgeAfMDf1pEt/MH3n+ioc07a3HHDxje+tzD07wboFtKJYjJdTqwbzpbg5yO+F2j9K2E0nToD+4tLaLJyRCuxWPqwUgE+55qO8ihtIme4iuFjHWQxOF/wC+iuP1rlr/AMUaNaA/Zy9y56bQQPxJ/pmi9zWNOMdlY7OaOCKP9/DBGh7t8g/MmqTz6em4eYGKjpE9eb3Xiq9unKW+IQxwqxdT+NWrbUFsrDdJOgOMYPXjPFWordg21sdRdeIbOEZWOXAPJdxnH4CuUvfGF7KGWOUQoegjGD+fWlstD1bWoI7mG2KWBba93MQkSqT975sbh9M1tWHhLw1BNGLu+uL+baWP2fCwnHbJG78RT5ewXtuccmt3jl/MkkkXh0JcnB5z/IVuaJ4b8Ra4srQwPbRGMES3KFFbkHgkZ7ZyB2rsTq+k6e0FtaaHYxxgkr5i+ZtPGcE9CcfpWTrXinUr3U/sPnGK1GRtj+UNimk2JySIre2tvCOoSqsovb/y9ou5ABFAxOQUQZJ4wdxP0HNVNV1KXV3Ek7HzJIsAg9SD/wDWxWQ1wralL9om+RZPmZj27ZrrdMsoNMsReygx3cYc2gYfdB6MQehznHtz6GtHFKKZnd3Hx21nbWET6hH5k8ce6aBzlN2SuCCMk/NyCccHOaqTXU+rQRxWw/eSOLeONflUKRhVAHAHT261kXr3F9qshxNM0qhyI1Lk9icD2Ga6LTrZtEsj5pAu3wrYIPlLjJGeRu9T04I5odrXBXJ7i9W2jg0+yCrBHKseQuDIemT3yc5q1cachEi6g7pG7BhFFy/APBP8PX3PPY1VsVe1s5718fa7mMvCv/PNccMPdv5fWmwPPd2C3N2ywRCTIkYE7wMfdA6/oPekNkEsumabxb6bEXdShMjvJgDpgMxxxnnrSPNC+noYCyqJQQSfmTg5BPfHFT319Yw2wnXT4pW3fK0rMevcgEA8ZqqstqYwEUIsg8xo933W5+7nt8rDvSktBJ6mU4bazq3Q7cfXNVp1VUUfMH53ZOM+laT+RIpG0LwTuJNVikbsziN3YLk/LwB0z+ZFYWNzN865RyI5H8qLkgHbkZ/qTUEt2HVvNO3+7xmtCQGNXG1VHRssBnv61VntdtujlV2yAshyDuAJB6e4NLoMz7U28dwTOpeJ0Kg5wyZ43Ad8VZuImcosjAMm3n+8B0/DHNRTWkDSqIUZWJGC3aiynja3dLggTwj92WH+uTOShOfvDOQfqPSqixSQq26uzcgowwV9+xzjg/8A16h8vLkZjyODknj2IHNXYZUdDJbkLnOcgdPcfz9OtUo7eV7y7VEbeyqTs7HaOmCdwrTYgW3i8uV2iaMDJVjDKJDg9tucnnB59KDLHAofMO8DBDQqc/pUNvbzDG+GyYiPmVcI6+g5xuPHvU89yVlEMkt5C7lioaLcCwwcFiRg456GkvMplC6mWWe3kM9qVK5kLWyk9RwAV9OlPSSFQiC4zCEwWWAA47cZ49T9ahupwZ4/NvrkRFd3+rDlenABcfzFWhIECot5M0YwSzrg9P7u4gH8e1KLB7CwXJS5V7e8kTJ+WRI8OoHbg+nvV2TXI1tZrVdU1CW5knU+c68qgDApneTyWGfp0qil3FdSGSC7nYRORlk2kkY9/wCtV2v7sTOkmoTLE7ZMTgOr4OepOR0HaqdrCTLqiQu4WGbaW/5adD+n+c1FJAwu42eKJDtJCl/mGBnlc5Gcd6dNaCaT7UlnfvFGpZljOSQMchtmOvt6VDbSrdBLu2t3QMhG2dwWHBBxwMnPpTv0FYVtXls2leKfSo3CsjKlmgchh8ykiPnIJBycdakihZ+VV2ZR/GcY757D/JqS3WXe7LPBaM5LKBGFc8dCyqTz9ceuKJFg3q80s8uRltxCEnPHJznt/nmpjoxy2IooXaTZO8KRbD8seHIJGO3H45zSpp1o6SBdON+yICodpMgkgcCMrxjHXPQ+vFbS7K4ijuI5IGhVJiMXbfMqnkHBG5h3yorQiYmJlLFS2TjblT+uabSaErohhhFvF5n2VYBIclFbHGPclvzrL1GOI3el+W00kvnYGOM52j7vOeeOvOa3vswmQ/I8wH3gSTjHX7oB/OqF0su6KMXKwIHXdCjY8xQfReD36kUr2H5mJ40uJreGO1/d7J2LHYysABg44yOpHQ8Yri66Txsgj1qJBHIoFuuC5B3ctyMdu3Pp+Nc3WM3eRrBWQUUUVJQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB1/hbXpLnUIrDVZbm4tsM0YV1DB+D8zEElSFx69MY79hI+nm0aNZ7pM4bCwrIC4/4GvGCf/r15hoayPrlmkcgiZpAC5QsFB68Dnpn/AOt1r0M6fdpKRDJbXAwcMkgUH8Hwa1hLTUymtS1FFCZI2gvVZJNyF5onjCjBBJADYH0J6j8LSaLqnl+bBbLdR5IDW0yP0GThc7vTt3FZlvbahFahpLC6WPJIYxNtPfrjHeql7eqVTLqyhyDGx4RsDJ9sgL+XtWvN2ZnbXYnkhls5mt51KSxna6HqD6VatNX1OwQx21xIkHOE4Kc9Rg8d+frSadq88zhGjju44o8slyglCqO4J5H4Grcl1pl2qRSQNYxoxYGAtJyQASVdv9kdCOh69mnfSwnoUodUgkkd7rT7aVTgERhoOMgnHlFQCcHnHetSNtEuPNZmvrUmTEaLtmRU445Ctnr1J/GsWe0gjmSOG8jnjmPEkcb4QbiPmyowe/8AWrEtlcWsKyXFtKkZOFkZfkP+63RvwJpaD1HzWAIMkd7aFOD88nlnJ7HdgZHPcjjrUMuj6mOtjOyHBDIpdSMZ+8uQePemyxfuTOrcKyrj65/wNIl1JbbZYJXjdc4ZCQQD1xiiz7hoUpi6ttPT9KsWd9dWkWLe5lhydxMbleQeOR+FbC6/JcFJNUsbXUhIpJaWIRynkj/WLg9uvNNu4NBuIGa0mvLWUNkRzhZA43fwlQNuB65+veqT7omy6My47l1nlnysksqssnnKJN4bG7O7PPHXqMnHWoJrKwuIhHi4hYptLLICGYD0I6Hjvxjoc1pnRLhzi0lt735SwW3lBk+nlnEhOOeFrG1GJ44irI6TxndsYEMCD0IPQ0nysa5kYa6UInZrlJ/s6jjysFs/l/StFi63JQB9qkL+Hb9Ksx6mjCRlX5lACh+QR0x9eKtWVxZSRtHcWrsCP9ZFLscdeOQQevUgnjGaSjy6oblzaMR4wMoBj5VDDPPQd/qKj3IrOWgUlshWX5Cuc9AOOp7g9MDFSXnli8eS3WTypXGxcFyOOpPqT+pp0YfDCSNkOFbDKRwRwaFZ6A3bUa0a5VI3OTnKyqcqMnbyuQcgcnAAPtzRbaVf3MCXMNlcTRucK0SGTkHvtzjpnB5x9akCt9rJXgFFGffLf4irlveXVlcGa3lkjl/vo5Un646/Q02pdGGnU5e6LrcGJiQyMQw7g/0rWt5+ACeR0rcTxT9rPlazZW+qQYIb7QgSYem2VACO3UE471Zh0vw5fK81nfyaaVUsIdQwytgdpVOFH+9k1F2nqirJrQw4NWudPLm1neJpiA7Rsyt8uccgjI5PWrh1SN7E2U1tBJb5JVSgDIeeVZcMDyepI5xjHFVNV0a80+ZXnhAiddySIQyspPDZHQemarIxWNS4+U/mKtJPUhtrQu/Z7KUK8DmKVV4WQ5ViFAxuHIJOT0x0HHWqiadcWoE/lsYHbbvyGQkgtgMMjPB46jBzio4T8xyM5PFSyM2wBw4R2BDEcZGefcjj8/eq5X3FdFe6XaocgnAOPzqK1DvKCqg8b3yccfTv1FXGicElfLeEksy7ArNx0GPlAz7d+/AqeGyhurmWO1kZHSJBFHOgjeVicEAE44AzgFjggDPWnzW3Fy32K73cltKskUrJKvzKynBB6cH8TV638Q3cdsYrpo7u23bzHejzAD6gnleg6EVlTIWkVJCAxRZNu4EgHpnHf27EVga/duqrax5x95z/ACH9fypScbXHFO9jtHbR52DQedYsx5RwZYlAHr97k/XrUcdjLJC8sKCeFeGeI7scZJK43Ae5AHB9K88tNYvLPaqvujHGxq7Cwu1uoYbmMlT1GDgqfr9azhJS0Rc423LwHzAq2UI7HrRqkXk3rRx5BTCMo4JIABP58/jUVzNcTzrLJceYeSQFG70w7Yyx4BGScZHrWlNFpuobZFu7u0ujksbrE0bEnu6AMPX/AFZHvWl2nqibLZGFFC25gznHQ89+1S+aI8LGx6E7u4PrVjWND1LREcyGKaF8MJ4XDoQenI5HtkDNU8n7GoRTvbjrkY+orSEk9iJRa3JrbU5s7xM0bMpUlGKkj046ium/tizu7OOW/wBDtZ42byWngPkyZ4PIUYzwf16ZriI7eRI3ZiFkUg7Seo71sOuNFsmWWUPJLJvj5C8BMEds8kflWU4prYqDszcew8M3F00drqc2nBpDsS6j3ptx2YHI5z96uabQ764nnlhktpYkYqm24UM4BxnacNjp1HfvVHWjL/aBLBlISNct3wij8uOv41nZaWcyeaFOQBhsAVk9EarU6RvCHiWRA0WlOxK5GHQn8s5qo/hPxPAD52i3mSN3yRFsD8M8UyDT7sMPszBm/wBiQbh+HWrsR1zSTvSe4Rz8pUZPyn36Y9qm7fUehmz6RqNqx+0afeQjH/LSBh+uKqtleGwv1rfi1jVoXWT7ZK8kUgkXzXdyGwRkbsjvz0zWwPHOsp5SzWzSICCyxxod5z3BXv0OPwxRdi0ONs7kiTEcYlIPRVJP6V3GuThjpttMDG0Glwkh+AOWB59RnB+lNf4kX9rLKU0eK3jYggSKwKttx1GO4rm9WvtR8SPFPfFJVXIUrEq8Enrjn86uEmpczJko2sQXdvDGGkEymPPGx8lh7VDZWlxJIsUlu0BfDpLcDy124JB3HA57etaFu06r5UCSpGECNg4yB6c1kXbNGFJQ7CT8rAjJ/wAa0k76kx7Hd6HH4cl0GSx1uKPzg7bJgCzAEfwlOmD2zg5/KppNlp2na/IiXNjeWhjwv2xHTPGGzxjgFjyDxnvisW3O23jUDACj+VSCfZwrEfQ1Hs7oanbQy7swx3k6LPDMqyMBLkNv5+9nvnrmir5MhYkSADP92itrrsPnOQlR5djpGI2C9Q33vepWKHC4cqRlztBKn2NQG73oSzEvxwvGachlufLLbY1J27uwPPXFefa4W01FhgkleUQo7gAt8oyQAev096H8yRlWbIVOPSrv2WSxLqbgyNtzxnBH4EGla7uEsYS0MfmREjcCwZlPqCMH8/601Yhyu7rUjE8FxaCIbxsGM8Z9veoomie4jiKhvLGAoHJY9T71BcXkksZVYBHEWJVQM8n3qa0tYZFiWVXjI5Z1bLH2wOnc9/pRuNx5Y6kCuzGQJKyJkkYBwx9KuxxSx7Y5G8o4B+YEnGOgHp0qdtOgmvxBY3Q+ziMSfOTv7fL0HPPp2qO4uJLG4dmhG9sgK4GVyeo24/T/AOtS1RKnGT5VuTxta29s7SKJnV2RfN+b5OMYB46+nTFSMbS40yN4PMeVMr5b5Kx5JPB6YPp71m4kC/fDSDLHBG0AHkk/j+NdFo3hXVtThM2l6fcskqYLyMIkJP8AEGf73f7uafQmdNy16mNBPFeTC1v5yiINsbAcIR6gdadbO0EctvE2UOcNjP4+1egaZ8FL+5kEl/qEYLD7trC0g/76OAPyrt9M+Gdnpibk06WZ8cEsgP1BBBH500u5UqN9Oh4tZ6PqM9sk1orBS3BbCqfxPFdhb+Gda1HUVurjCbwA8dvbs6sMYOSwC+/GefpXrNtp9zbFvJ0Z4+xdAoJ/LrT3hviQDYygAZIaPNUnYaw8L3krs5bT9CtNMKTiJUvFVU+0TRs8uAMfeAOPfbjNaBZH377jcB1YxTAH6fJzVq6SONwZ7V0J7lWAH/jtZ876fHgFohuGRi55x+VO7NlFJWQ9bm1jYA3cIP8AtLL/AFShrqFlLpc27duGwfyIFUp7iKJdypP5fByjqc/mMGqEjSXABWdooudxBDMPxxj9KaTYNpGtdagLK2NxK0flDuhD/wAvwrKj183gL264TdtHmdz9B/jUF/ZQzSGDzNyLCGnO7k/OoGfxasz7HeWz+XYYm/uR4LNk+g71pGK6mbk3sXtUvZoLSaOeXcxAIUHAwfUD6GlXSIHvLy3S3JUTPHFGepwcAc9+K1LXwLdSH+0fEl4lnacObeLmWTnpnovXtnr261qah4qsLCa5l0vT7aG5cF3uBGpkYkknJ+vrmna+wm7bs5/TfAs4lS9uE/s61PA8w5lJI/hHQfiePSr2qa5pPhu4hOm6VbyXJUI1zKPMfaMLx2B45xTNC1i81p5pbq5Z280Ehn+6CD0HbkD2/WuZ8X27Wt9GG5BJwf5VL0aTKTOg1TXb/V9Q+yT3JaBArbMAAHt069axLhnsdSWaByFIOF7ZIIqJTNFNbXW2Vxd267AiFizLkMABycAqfxrah01otUtJtaK20CES/Z8qznB4D54AOOg3HHYZzWysrGWtx2sQKmmxXMeciRXIPUE9v5is+5tL7VbuGXT7ZpZFAR9vAGQMEk8Cuivb/RZEbbpVtLEy4KmFSGwO3Gc/TFXCdlokUBMcW0bUUbQB6ADpWa90vczo9GsNAQXLQx3OpqwMk7HcsZPZFPGRwN2M9a5y61aQzu7FpGQnCk9cE8ZrTvLoQ3Twyh3WQBRjkg9R+HNT6XoUNreLd6wi7mBaO0fBIb/bAOPfHvz0INruS/MIEOmW3nK486YfIducw5yPpnhvpirWnWj6nOzTuwtFQF5M9W5G0epx+WOaZevJfXoldyi/M0j7d21Rg9M/h+IrSjlhS3VLVdtuM7E69epPqT60N6XEtxu4XGrSTygJaRJnb2OeFQe3H5CoLh7i/uGSWULEsLAu5wsYyMf/AFgKJI5jbWmmgAXBVTIST98qMg+mAOfTBqrqEslzcR2NjG3lbjnnlztILEnp/IUJBe42W80+Nls/sMlwE53Suy5I4+6p6c9Dn6024lgNwXECxsqBXRScFW6Ec8YbjHuKs+Ra28wD/wCkXG0twxCDPGOBk+ucj/GK6BkYMkLBtpQlCAu3GSceuWx9Ppy20KzKc0xn82Z0JySM4wN3Xr+ZqizOEAM6IrjoG+br39K07hRG/lsomiIDAFCje2fmI/8AHR9KY1y9vGcQSNGeisUKjjjkLn9a5mjoTMzyg6xlyVjHJZEJY8/zqubWckNsZyCMdTx19KuIbq5nAtS0JkOCQzlV/MkY9aQ2F65Be2XzRyHMEaEn1DYzj8akq5mSCVyYwsSyu/3mYAgc5+Un6du1V20+4w4lfOT0KEZ/EjH611sFheTRFbhBLnjlmbn6802fTZ2DKyqhJ64OfzI/pRYLnFvcSQ3EarF5jnCsiFdz46dCfm9/f89GL7PPA8qIhMm0fMcc+hP8Lc9OnfpWu3hGG5fdJMsbHq+N36AirD+FkEscqTyfaCMSyhtgnX/aBDZ7c+1VFvqS7GFLbLCyeadoOPlnU5K+zjkjjtgVn3cAOpWMkUJLBmP7mUMAMenJB+prtrfw/atGwivJ40zynmB0b2I24/PPSifQtOiVZZbZhuzhxuG78Bx+QFXewjko7WZZHwbmLGMhIyfxPIpRbSgnabhWJ++qnOfcf/Xrr7XTLNGLLbyy5Od72rNgc/7NaRt4LU7SqQOcFVkWOInp0BIpXaA8s022uZobyCAysn2luiklunXFbUWh6l5KiO3uiSPumAqM89816B9qkWMCV3Xrg+WzD80BpRLNIuY2kcAc+Wq/n85XFF3YDgT4X1SUMr6fMSVKgtOiY496dpHgrULW1RJbWFWG4fvLncuCT12gHPPUV3ix3TcnLDoVkIU/pvpTb3WV8lraDj5t8ZkJPsVKcdO1F2ByY8FyllaSW1R1ORs3vj8DVk+D8f629OG6rHGI8+/cfpXSPDOpxNcTEnnMIjI/JskVIbVWGXLTDHTzGjYe/wAopJ2A5mx8A6bZs5E9zJuYuyM6+nsua2U0DRrQK72tsu3oZT/8Ueatmz2tuUK3IK+ezSFfpuY1JDaJC2YikTNxmONV/pQxkCw6a7GRIIHf+9FCJD+YBNO3yKNsUNyw9Bxj/vrFWyBtw8hYjvuP/svFDNIVURusZ6nMe7+RFKwXPBPi5KJPF0AJXelkiugYEodznBwTg4IP0Irgq2PFOtnxF4mv9V27Vnk+QFcHYoCrkZPO1RnnrmsesmarYKKKKBhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAHR+CbA3+vELndDC0i4OMnIXn/vqvRE0W/B3RsIn7YkwfwrzTwhevZeKbFllMYlfyWOCRh+BuAIJAJB/DODjFe0mwncNJHcG4QdfJ2/+gkFx+IqokSMPOs2uA+rXMe3ooY8fgatyX7X8Xl6hYQagmAJJTB84H1HT9Ks/Yme2kQSN5bHcyNMcH8Cf5VSk06Ip8si7P7qqxP6jH61dybC2/h3R57i4fRr2WC5aMqbe4GBz1xwDj25rPuvDGuWuQ9l5yj/AJaQSKy/zyPxFXf7Mt2OGaV0/u7Aufxycfka17S+itYltYMuyqMJM4kZPbIC/lTT7CaODZZIGAlR4nPRWGDj1rU07UZ7Rv3c8qHa+CjH+6T2/rW7dhLtibqO0lXBXaUwFJ+h69OvpWK+l2cYZixEgPy7ZDtC4IIxg8nPXPHpT5nsKwjauboEXNvbTAn5iYgjNjplkwT19aQwaZNatMftMBV1RgpWQfNuIIBwf4fXv+FVYtOZ2Cpe2q56iZmTHJ77SPerEun31vp1wZbWVx5kTxvCDJGy7X3NlcjHK8nHX600xMlGnxTWdsLW9t5JlBQwu3lEAsSMFsAn5j0P0zVK5tbmzm2XMEkMmOFkUqT9KoG4Hl7gwwR2NWNO1S8sovIguXFqefIPKEnvtPHr+dXe2iJt1Zau1EcFk3Klrctkdz5jj+lI2ovNbxw3QS5hj4CzKGwOMgHqBxjgjvV19S03UEjS+szD5SFY2siE6knBDZyNxJ4x1qC60RjA82myNqFsmC7xRkPH/vJyR9Rke9JvuNeRnNodg0C3FrJ5G9ipgmYsm7IJKsBkAAjhv++jUy6PPBa/aGhcjdtV0+dCMEnDDIPT1qO5ONItjG4LLNMMZ74j6/57U20ubhFXE0sQJ3YVyATgjkdCcHH4kURTtoEmr6jlZkAx37VYgvltyGmhWeMgp5brkDIOCP7vPJK4PvVmCfTry4VNQSS2DAgT24BGd3Vo8dAP7pH0NTXnh65WNpbGa21C3T53aCVSUQdSyk5BA7DNVpsydd0QxR6fcEYee1YknzCRJGOThSMBgBwM5Y9eDU02kXiQmZI/tEA6zW7eYg+pH3f+BYqgsfyqBkHFS2t5PZ3Hm28skcyj5XjYqR+I7fXIolHsNPuNsIgb23JQSL5qkgjIxnp+VIqCNSEZl7cVeOpRmbzbixhlm2jM8TCCVm65O3KHrjJUk4zweKsQaEbm1MmmXiXcmWxZylY7gAHjA3EPxgnac+gqL63ZVuxh/IqmJol8s/eCqNp57qQQencVPM0c6SOYCqojPutY1XBO1V3KflK7icnIPz4GAAKfNZ3UMrRXNtJDIvJWRCjY9cHBp6xSraXLx7NmwLLu64LAjH4gVdk9UTdrcywUZilsXwdzbZkEbYDcY5IYkYPykn27lk0UwlmhliaOVX27JAQRjGRj86leNSvzAMp42kZH5VPZSW6Oxu7dbtCmzbIxBReMbSuMEYAGcgDIxg4qrSXmK6ZBCVjVUyMjNPkVSwJwQRtwa0rPRNP1S4WKz1KW2mKfIl3GHViAS3zrjHAJ5XtjNUprK7gRXubV4kVBJuJBXaWKg5UkckHHPPBGQQS1NPRi5XuivMN0Uq8qXTazgAsBz0JzjqaydW1GxvYIUbTLeK5iQK8kS7fN/wBo4OAfUAdcmtefY0bBmyp4471yt+YxKfLXAXjFZzSuXDYztQFqQogiZNvUscbq3fDLE2MgwQokIXJz2Fc3c/MM11Ohxm102FSisHcSSEZ8wKSv3cnaSFBxnHLVjHSehpJXjZmxdvNDcBbhWCtFGY+P4SikY/En8aiWaN2Zdw3DgKOpqIi51K9AN5FJLsCRxyYhZVUKiL82FJIA6Ej3qn9lktZomnjmidl3/vEK5HqM9R710RqK1jGUNbmgghW8DcxXKYKuhKsvccj86vtcGW8iuL1IrtFyGUqI2Ye7qNx69/as0gG7lZWEkYkOxx0wDxyO2MVOhLK2evp3q+RS1J5nEvf2fpN7p8cn2/7HdKVEkV4B5YJOMiVf4enBGe54BIjAmuPD9j5cSzQieZlkhdXbbtXO5QdygbCcnHGKqCYxyADjsSR0q1HJ5ZzFujZ0ZS8ZKMQeoJHXPvxUypytoxqcVujntYkaXUJGEbRREgpGf4BgYHvxjnvWbLtZQTkBTk4r0X7dY6g0cWvafb3Nuh3Pc20SwXA/hHKYVhkrxt4pU03wHe2v9m7dYtDOfMF5cJG3kMAQFLLxt/T3HUc81JaNG8WnqmcDBq0tpIjRbTtHAYfqa108cagQY9kZRuobJ/r/ACq3efDDW5JGfRXi1KAcrIksSs474Acj8ie30rlL/StS0efZqVhc2rA4AmiZA2PTI56j86z5rF8qN1fGFyk25oYxETgoinJ/E/QV0Fn4hs7lIhIiAMSCZeQo9+D7+1eczTqZEOAV7qOM1Is2843YGOA3Yf1q1NpkuKZ7RaT208COYklRvumE9ccduf0pTZ6VO21keMqOok5H0/XtXklhfNZusyEkEYdQSBjPQ+tdQnjG1tz5a2vnISC7knP4fT/63HWtFNdTNwfQ7I6NYgjZNI6L1V2DLj3GRxTE0axKv5gWXthIlUH8iM/niuXPjC3M4RLeQx7c7/Nz+hH9avJ4msmUPPITEOhL4I6fWtE4shxkYVyiw3U8SLtCSMuCOeDjmqzBg+cnbj8qfc38WoXE0giaF3bgxEAL74IwSeO47+taVvob38Mj6ddpcMDxbzIIZiOAMZYo3JPRvTucAlUjew1B7mbvoq0+ha0rsraNfZBwcQEj8wMH8KKOaIuSXY4BNyvv2bcHPWnz3EknO/LSEHAboBVcrgHOAQegpqHDHaob2IrgN7IupcxxM5J3k5AC8VYtblcOj8+YOCc8ew/lVKO2STHzKrHtzmr8dncTFLdUlaYjCRqhJ69MdfSmjOUYsgd/3g85CiNjAjGBxxn9KWVk3swD4HRlAz7Z/wD111tn8PNf1ONPNijtVAPzXMm0n0+Xlh+KjpXYaN8OtMsFP9p3Et+c58tIxEn0JB3H65H0qkmGvQ880DRtU1Ig2MEzZbZvI2oD2+c8A+2c+ld3YfCqScq+rX0zkncwiYDB/wB5ue3938a72za1soRDa20FvCAABEmD+PHP41aF8rj93NFGQf41/wAaqyQKmr83Uz9K8G+HdHkWW10xJZl5E053kH1H90/QiuhaRicokSP6hMn8yCaz1nlL4E8Tt1BIH+fyphnux1eNh/s5z+lMs0jcXDDEhLD1I3VGXDfdVSemWjxVJpBs8ySaRCOd25Rj80qnNqVvIq+RdXE7MMjY0ZQ/js/lTuBrtZxyIxMcL+pKAVl3Y06zQvNHbr7BBz+VYk/iW4F+bFpvMXyQ7Yj2mNiSMHrjp685HTpWbd6jFbwMjyKzkkc8807MTdi/L4jhYhbK1VEQ88BAR9etc54imuJ9NnlM7O+ElUf7SuMkemF3fgT61QuNXV4XQDgkjI7Vt+HPCes+KUdpFe3sGhdTNKm0NuQgFcjnHB4496pWDXcl/wCEmt5oI12MXAAZFHTpyP1p0cGr+JGNrpuntDBLjMr54HfnoOv6V0mm+G9D8PW5/tO7F9cAciFAqAjr1PNM1rxfJHp5t7FFtrMrtwoyTgcAn3GPypvUi6RYsNC0DwtayJeTi/u5UCSxqQw+8D/MDr6dKz5/HI0ye2i0q1gtrQPuZI13FwDzkn2rkF1OWdjulzkgqv5VSbEtywABxj3x6Vdra7k3bOr1PWpb+2W5leRmkAJBbJHt+FcrdXTE4JJO4jA9K05o1tvKtGk8x3jGFRckbxkADucMBjua0bHwDf3EhutYmTS7XbuZPvTMPZR93Pqef9k0XS0QkurIPCNxHDrE1uVbLwjaqjJZlPAAHXqeBXUa/othqJjfU737EisDjbufBIGMduSBzyPSltza6fbJYaakkFuOQoc73J/idhySccjOPbgVj+JpSmizsqqHRd4x/Dt5/pUy13KTNl9TstEsIrHSVkWGMt+9Z8uxOOScZHQenQVxGvXTXFyrEDao4A/P/GtiWJXCYYbWQOT/ADqsPDF5qAE8khtYs/IZI8+aOckAkcAHIPAPQE54Yl3HaDKLu0mFyeIdojz0Oc5/HGK6GCaRrcW/lyPdJGFeNFyQw4Oew57nAp2kR6bo0QhtrfzrljkyXGGKn8AB2B4/M4qd9UntrSVj5aIR5ZCqBsy3bH1olqFyK20+Cy+1X0+6W+KloUA+WBsDgnu3GMjp29Tg3Wo7r+EyMAM8t7ZFXry/MMauxA5xyevFS2WhtblNR1FVDDmCArn/AIEw/kOvr0wWu7EzRktXj0SVM4aYbyO4x0H5Z/OqdjKLa0SYlPMAPkqwz0OC359Pz7Vea7UWss07gJnBJ98f41mN5l1JGgThABEq+mSMf59KcdbikiK0klis5pzjdI5gDZ555Y4+mB+NTSNHZxIFVvNdQ7yMfuqSDjH0wSfwqxN5TK0ECqUtXLuTyJGABPp9B6dKLTT21yyluZLiW2GS3nxrnaQQc/pjpik5XGo2Fmls9KG6dRJdyrvji3qAg7F8/oPzrAWS5cytLdnLMDsXaf5YA6VX1zT9V1DWJTFPc7cBVeHADjHXJUEnnv6Y7ZrJh0W5tWMX2nUw2STIskPP1JXNSmirHQLIobZ5gLe7KP6mrZheG2WRinzMejZI4B5GOP8APpWYuhNt3SajNKDjdumdTn/gKr/OrS6BpcCL58cjzY3I0skjDZ0H3nIP5enpymxpDhf26uqNKmSdvLY59+eKlGr2a3DWY8k3MblXCP5pB6bdqFiCCDS/2RpxwW0y2GOjfZU/qtaSgm1ULJKQn3s7QgJ6YA6UhorrK+fls5ixGQyWcg/VlUfrUrXF4iI8lhd8kgSzyxoD04wCx4+lSxYYBxCzsONyt/TFTGZypRpoUPVAQDg8epA6UtQKwuZXGG8kA9hKzn9VT+tJJ57F5PMXywTtZomJC54y2/Hf071ZbUPJBje5kZgcMBlcfpiozfqQVjNwxPUnAz+VAFcM+75pWfPGFVB+u3P61MNphZZPtG/+H/S5F/Ndwz+VWFkldQEhn3dCrM2D71AbgxoQNsbD+IRh/wCZoAgk02Ey4kskdx/EyeZn8ef51oW0LozEwth+X6KWPTqfb2qpFqF2ZQzSCSMDBAQA49eB1qzHfTCMkRu3qSTigCYQSA4CJ+ZP/wBaphFKF42of9lRn881VTUZCCGRQf7xbj9KT7fyRJ8ynrhM4+h60rDLyRfJtkYyZORkjI/Lr+NIzQKCGVMD+9j+tZQEbSFt87KTwCP/AK9SeZAgx5byH3cf0p2Av+da7SuV2+m3ila6iYgBmY9sVm/bLfYVMAHGcqfmFNS8UT4USHsA2Cent/hRYLmuHGOARnuelU7i8aLBaWFB15ySfwpQZQ3K4XvvIqvIbREJjgid2JG5l25B68hTSAeL9GAIRpWAwGWI8fmagv8AWLiw0m9vjZuy29vJMAXC52qTjp7U+KUQqBHE2SMlA2QD2CkjJH1ya5v4jyXD+ANUEsTIMRYyP+mqdaTGlc8BooorI2CiiigAooooAKKKKACiiigAooooAKKKKACiiigAr6S+y7eGST+X86+ba+oXkJj3BCQf72AfyNVEiRmmJRyEB+p//XUIjwTnhe20Vck85nIQQgYGCWI+oxt6/jj3qNop1j/fHaM43hMA+wPIP4UxFaSyEqfu5w5PBQ/I3+fxNZM1qqsNgQMhzkjJX/CtxlBxg8juabLMzoscs7MifdTPApiOekTH33GfUnGagkhLKWBG3p1ranktoW+eREHTLcVDHH9tBa0hknRTtLRqSoPpnpmgDB+zNKwV3jiU8bnJ/oDUxt77Ti0thdzS7RmR7b7q/Xk/+PAfSr01tywZGVgxUoSMj64/Kq7xQRAFi6MRncEBGf8AvofnQmFiL+1orxlTVtJtrx8583PkyED1Yf4Vn3trpMVtJPZ6gUckbLedSSBnkblBB7EE+nvkWcxrK0ssRkkbnLykgn1IH+NRy3J3Fore3iOc4VN+P++y1VcmxQeG5tUR7qB4g5+UsOGHsfwog1BrW5kSNivmx7C4OCuGVuD6/L+pqxJqVybVrXEBgb76Laxrv+pC5P161lPbqSdrPDlgcKNy8euTkd+mfpT5nYOU6P8AthdQhW21aE3EcY+SeMhZgcYyWIO/gAc+nWoJtJHyNpdyt8jNgRx/69eP4o+v4rkYGc1hrcSQXOwjzIlGTKgOAM9TkZHUDnFXVdJGt5IZSGWUMrxsQysAT1HI6VUX2Ja7isDkBuCCRgj9Klt7mW2mWaCWSKVPuvGxUj8RW02tJextb63B9p3ABLqNFEyfNk/Nj5vofQ+uRQu9I+zrJPZXcV9bK20bFKyrkjrGeT1H3c++K0UnsyeXsXFvoNReCOW3jjnZ8SXEICls5xlOFJyRkjbwDnNOv9BvLa3+2wNFeWpyPOtW3gcZ+YYyvAOeOPWqGlg/a5EdSpWGbII5BEbf1FEF1PZzJNbTNFKhGGVsH6fT2qXvoUnpqRXColuNndQxDDBViMkH6HIplph08sqHHbdW02pwas+7WY3a5bav26A7XAyeXTG18AjsDhQAaZc+HLy1zcaf/wATK0PSW1XcfxQEsPwyB60ua24WvsMXxBeQWbxThb2Ftv7m6Zn29vkbO5DjjIP4VbtpdIurO5AvZdOaVAoivF3xZ3Ag+aoGB9VzyTWBqQkjhAeN0cSDcrqVI56EHBFNLlYB244ppJ6oTbW5bvdLvrBfMnhxDhWEysGjYHODuHHODVcIO45qW0u7mxZZbK6mtpMhi8L7ckHPI6N36g9TVs3+mXSSxXtsLe8kz5NzZAgFsAKrRFiuOpJXHTt3vmkibRZRUFEedXKtECQF4J3DaR9MMaS11a6snZ7a4kiLYDqD8rgZwGB4I5PUHqfWta50V4tGvrq0uYb+NWjCPFGQ4GMtuQ5KYHPORgHmuaBWSPIx+XWjmUh8rRsjUdInN5darZyZdFEcdntjRWHVip4Gc8449hXNXa+Gd7galcSlieRGQBzx1X86m1D5NPlIODtrj3wSRWE9DWKua17baFhPsupSOxYAho2x9egroYVC8bQFGAPbFcJgAgivU7rQLmJZJ7WRL60VA5uLfnCkZ+dPvIcAkgjjB5wM06MkpaiqRbWhkyiKT76KcnoRkU+3u7m1eNUkSe1RlLWtwDJG4XHy88gYAHHamSxExgryMfnUO0qBu6dcc8V0SipbmKk0dLdXGm+I9TtvIhTQZ2xHIHUPaucE53jBQ8BeRzkemTU1rw3rGit5l/YyiFnO2eE+bHwM9RyBgE/MB0NUobWXUJYbW3ZUnuHWGNn+6GYgDPB4yfSiy1jVtIS3jtNVvlFt/qy0pZQcEcI2VHBPGO9Z2lB2i9C7xkrtFZH892bzBIpGAFPSnzkm1zFuDqcDPatGTWzqeZdR02xuJ2YF7qFTbXDAE5JdPlZjnuuOPxpZLOzktopdP1FxI5Aaz1ALGye/nAiNvyXj8qpVf5kJ0+zMqa7KWcbbs7gI5PfkN/NajlvIbf7pyw/z1q5rdjfWOlPBPpN3GTMkj3BUNGFG8DBXIwS55zj5R1NcxNx1PNQ53L5LWNuHW9mx1JWUEHKsVYc9mHPYflXSWvxIuLuKK01S0tNSgjBAS+iV9xJGOcdRjGcdznNeehwfu5ziljcdD1rN2e5Sutj06G1+H2spKJNGbTrqTHMczMijuV5AB/4CR7VlX3wi1h7OO90qWC+gddyxq4WRM9iOh/A9e1c5a3i7kX/ZOMetbEPiLUdI8uTT72SNy+0LnKnPUEdMVPsuw1U1szkNR0nUtEuPs+oWstvLyuJVIB+h7/hVZ22qQcqQPukYzXsNn461O3i8hmgnQsZBHcR71BJJOOQepP0rlPEWmWes38FzZ2VtYQrEElhteN7g8sBjC544/GlytD5kziYWYcZxu606SdpMRNnbu+6fWtm50G500STQL58EbAMxXBx+vP406JZNPu3DboftUO3HHzKWBwQQf7vbHIFFnsF1uVldVaMY3Fs85qysizxkPgjzA4VucgA549jiieHYd/VhyMVLbRt9i8zDNG8kkY4GFKhCcd+d4z9BVkdDQXW9aRAo16/QAYCi6fA9utFZZZVYjK8cck0VryrsRzSMK00bUtSf/QLWWbPHyIcDPvXX6b8I/ElyA11CtmvVjOwH0IPQ/nXrYLgYCxqNu0BEEakf7qYX8wafFlYypRDnoRnP14rmVM2ucTp/wt02wnNxdXZnkXnHnIsf/juT+tdvpmjWltB5domnQ55Ki4UE/XjmpYrSSNS5BRGH+7n8jUTARAqYwB1+8ePwzVJJaCLq2TLgG4sol2/Mz3SAA+nBzSCygYA/2lpRJ/u3fX9Ky2nKnaOc9MVUmkLH52Ube7Hj9aAOgFjCfu6jpn+8bkGnDSLqZc27W1zn+GGden41yrTWzI+ArOvJKfKB+NUH1eSCJoop9mOVZQN2e3zCmoXFzI2p9Qht72WymjdLqFtrxlc7T7npjGDx6ioZNRZChSOMA9yM4rmfFepTnUNG1pyd15ZFJmHRpY3Zc/8AfIT9Kr2uvLKYo5cZY5Z89KLdOqGdOs819N5ck5KscEE4Hr0p19PDpds6quTGpYbRjNcquq3l3cSWuk2M01zKAoeJGIGc55/h6jniuvs/hxr+shH8Qaktnaj5vLhIZzkdCTwMfjR6CscPq0plmj1SDeRIpRiOPKK7CCT77yB9DVvR/CWt+IJAyq0ELf8ALSUHcfcAexzyQD2Jr0sSeFvDWmNp9jCLryW83LtvUv0yc9+B0HHasm78dXARjYrHbopA2xxjI/E5/TFWoN+hPOkSWnhDwz4UhWbVh9vu0GfLkwVU/wC73/Gq2pePbzVSbayjS3gztOM9O1cVq+r3eo3Cz3UjynlNxPIwf/r1oaR5MLK77SQDu9Cc/wD1qcUuom2JdRXsl4yO7+Wg4LH2qG81Ca30eexZVJZ0bJHIxnBH5mtLUdQiCNsGQeAPU9j/AFxVaz0G+8QTSGzWEID++mlcqkeCO4789PrTd2iUc3A3lyqzn5QOuOn+cV0mk+F73UZFubmVLPT5VV1aTG+TBPKJnJB45OB7mukSDwv4TgErRjVr8cGeZMqh7bVPAx7c+9Zmu+Jnulhu4mI+0LukI6bwSMfXAX86qKbE5Lob0+safoiL/ZtpBE6bY/OCKZWxgfMxHt2rHn1+aW5d97FJBgknrmuPlvpJCyyEndhue9S26XeoXMUduGaWQhUVerH0xVpRRNpPc7eDcHD9zSRaV/a7Nb3E3k2sheMzMQOoIIXJ5P6DvVq1iMdti7hBujGdkCHcpk2nAZhweewJ989KqHUGuH3l8kjAOAOB0GOgHsOKztdjbsXI/selQqlrB5skeAJ7jDsMegxtX8Bn3NUrzUZFtp7mWRnYvncxyTxSBjKGXPPQEc1Wh0qXXYGEM3k2y8vMykgYHKgZGW9s8d/dbME7opzXbNNFdwPhixMoLBQEXbgnJwBlzVrSX1HVLWZYoFntxIwNyxCxE7dwGSQDyAOPXtWxZ6bZ6YsZSMSSo29bmRF3jjHB/h7dMHjrUs+rzS3c0MjkkYCknqNo/wAarrZFPYjg0+LT5lur2SKa5z+5jHKR8ZB6YLfyx361Bf3buglxuIycZxUd3cg7N74wQcgdulTReVYYublVeVgHgicZ46B+fpx6dfSla+gr9Sa4g/4lvlzAh+GKdwecA/4fT0qrEv2WweRm+YjC4z9Of89jU8Usl7AWcEFtxYD65z+tIzm7tSh2IQNi5OAuOmT6e9VsS9SvFD5NpqExyRLORGo75PJ+nKj8ateINT0+y0y30OOR45ABK7xr93PODweefb9aryXcVtDLeJultbdRghMnbuHJA/M/ia5txBqN9Pcz3czeYQ24SFj7D5g5xjjr27dKhopPWxet5tNXBmvLnaT1z0H0qEzAp+5Y7yON68A/1pr2lgkeESaclc/NcvGAfTCFc/8A1qcqRqNvlptPZsvj8WJNIosxOHQq0bs56Edvw7/Srm2eOGOSbcWLFAjnARVA7ZyASeB7VTiP7kQLuMa/dQNhV/DoKsQiKONjJHuORhRIMfjjv0oYDo5iWLMLdPwLfnxU8UheVY/OTMjKg8uMjOTxxx35qASqh3LHGntjdn/vrNAuSjZTMeerRnaPyFIZYnWaN33s5CnBz2+tQFmPVR7/ADCpLlfKvJYy4k2n7/1AP9aQDbj5CT7ikMkKmZpJXZGkxvbAJPv0GPf8DQGEYxuk/wC+cURhmYgGMEHoZU/lmpZbZ4GMbmIMACQM9xkdqAEW5eP5l3gjo2eanLvcI0v7sgsQSF6nv/OokRMZLM5PZQF/U5/lUwdIInRIsqxDEyvnaR6YxQII5HUfLLg/3R2/CpFuJS2csW7nH9KjXdNwu18/3Ywf5UpWThAHOOCNwH8zQMcltI2SkL7V9F4Htn8akFvNwNoB9AM/ypkNlMlwLiMor+pO7I9CMcipH1C33FY2dVHbbS9AJlhZkKSHdGwwVZcgj8cfnVcabbKxyZME9C/+FAvEzuEg2H++2P0Ap8OoxAnJOewC5H86AJUtbWMDFujf7wz/ADqTzIgMBVT2XH8hVW+igmK3lvOpZ+JIHJOD6p6D1HQdqhR3g4yEdjwGAx+BNOyAsTCGUlpCysAMOv3vZT7fy/Qs+1JGMRIB7lev8zUyQSNgyb2PcEjB/KnvbQELnap/uDv/AFpXQ7ECS3cxyCwX2XP9ao61p6a5pF5pN2TtuYyocpyjdVbAI6MFOM84weK0yixgGNo9uOoXoffnj25qdZxwjPk5x1/pSYHyVRRRWJuFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFADohGZUErMsZYb2RdxA7kDIyfbIr6bYMq7trKoIG7bxk9smvmKvpjS9Qa90yz1BCYDcQpNsDZ27lBxnjpnrVImQNnbgKfqaajSxtlXKE9dp6/WpjHtQnzDu6kuBtP4DGP5e1QFiVMjREqpwWiBkXPpkD+YFMghWFY1Ybd7MSSSxxz6AEYqGWNZOCgwOgH+efxqeScq2Ft5n7fwr+jEGoSZZG+7CidwZTuH4BSP1oAq+QiH5IkX3C4pWD5B6EdKlEUrEh7hdvTCw849clj79qV9PMse4SvIAOkcmWX6gKp/HGKYFQJGZHkaFWd/vMcgn8jRNDaSskcEjJcM2BFK4G7jPDYA/A4+pqJ9LtifmR5F6/vJncfkxNJHaW8AKxW8Uef7igfyoAzLryUlkgZZWmjOGRUJx+Qqo8XmKR5Dgjp5gAz/P8AlW+0A28YxVdoAvt64pDMH7LIDuYIiD0Jc/0pRaW7nBlZHJAG8Hb+gz+H61ozQgHOc1VkjJUKASfSgZRe2a0ufMjJ3xMQskfIJ6H2IIPT0ODWi1/p+oJ5mpWrxXnIFzaRplvTchx9OOcZ6VVRpoCNobaT0Me5W+owQR060skfmzMHVY9w+8hAUHHXHP5DFNMTQ7UtPu7BBKQ09tx/pCxsFB9Gz91vY+ves1Z2BLRkh0OQw6g9eDVllubWVvIukKHgh0BRwOcMhGCM9jU6aTb3KE2d0sdyQAsMqiNGJ64JY+/ftWvP0M+W2pdi8QQXsxl1K1ErsfluYzsmCkYwxGFfjuR68U650nanm2NzFeQFS5KMN6AcncvUY74z68VhS2tzZOsV1C0UmONw+97g9xU+n6hd6bdx3dpK0c0RyrrgkfnS06DsWVPykdKdAzQSpPA7RSodyPG20g9Oo/KtZLzRtU06E3y/Yb8fJ59vHlJMY5dPU56j6knpVG7sJLBIZDJDNDPnypYZN6tjr9PoeaqLTdmJq2xPqGtrrKW8OuW7XRjYf6VC+yVV9x91+x598daqSaFNPA76RdQ6pGiK5jhyLiMH+9F1H4ZqpITnoAehqJGkWeKWKR45Y2zHIjFWU+xH8uhq3HsSpdxPN2lonykiffRhhlPuKjDlpwASDtOcHtx/9auouNdFytva6/bnUoDFHIZWwk8bMoY7WGOOQMcdPoBJB4X0fVbaWbQL+Vp41ObO6jBkznjDYXGenfP8pUmt0DSb0ObhuZ7S4SeGR45ouUdG2sv0IrSbULfWLiGDU7fMrsEW5tYwszMzZ5UcOST19zxmsueOSC58maJ4pAcMkilWH1B6VJDG7zjyVLMgL8Y6KCzdeOgNVJJoUbpljVPCsl7o01zo9/DeRBQWRiIpE5x8yseOnr61x6eFtUj2SPbxhcnKu/T3OOfyro0kyCpLeW4GQCR/Kkls5jA8drfz7HI4Dkkd+/Ss5U2+pUai2Ocg8PvBqkazyRPEMt8pIJx2wRn0/wAa6i2u7mxkd4Li4jaQFHMcm3ep7GsSGy1KHVlaQy3ShGAKgkqMZORW3Gg28gdM4oppK6Y5tt3NJtSt7+RTqNpDIMkvNaKLeds9SSBsc/7yj69aF8MtqSyy6Ndpf7Fy1vIvkzoO/wApyrY9Vbntmsp125KHGD0psTSIPOPOCVyHIbsePyHPqBWlrfCyL3+JFzS7W8i1rSxJb3VupvIP3k0LIAPMU5BI5HHWs6eILuUD866mx8Za7a2whiu0mjCBEiul+UADA+ZcMegyWJzWQtlFdozR31vBNtDPDev5eTjnZJ91uc4ztP8AOkm1rIdlb3TFQ9P0qdJCARnI70++sbnTXSHULd7V5BlPMHyv/ut0YdOQT1quMg49qE10BpmkmrXUdtLF5nmwSRmN7eUlo3GCMEZ4PPBHIrnLu2jnus25EQcktEC0ixcnjdjJHT8+prQPIIU49qdFaRR2XngEM87R5/3VQ/8AswqZJNlRdkY1ppV9qEkyWVrLcNCA0giXdtB+n+RUG0q3PX6VuNKbaVZUyJUOY5VJV4z/ALLDkH3FBMV5NFJd75Arl5SNoebJycvjJPXk561PI0NTTMeIgtjnr1q7dPiKJh13dffFW5rG1EwFmJvK2/8ALwy5UhefmGM9DjIXqBg1nzOs7pFCjSuW2qgXJz9B1qk1yu5NveHR6i+8u3OOAKuHU5yyYbgDAx69ay5rWe0umguIZIZB1WRSpx9DSLkDAbkHA96zUi3E6u11Z1/1bkMy4+tLZsLa7N5bvi4JJDyokoUZJIAYHAJJPGOprmradkmBLH5eRjvWg9yVba2NwHGPWrST0ZOqOoivPD9/bldT0NrecIEWbT7jAfHG4oSFU/gc96pabZaNLYXFp/bAtLkztJH9ugVFQEKvMikg5wvA9CcDmsNZZJjycY5OOKqtP/pRLSMiAEMVGceg/PFDgkroam76nUDwF4wkUOul2rK3IYXcWCPX79FYMep+KY41SLUb4xqAExKw47d6KyvPuP3T15tQjTlyGGeMCnDVY9gZLqDP90ZDfqBVU+FnYv5GrXKrnJAmkX+TCn3WjO9oC2pXcvQfKQD+fXj3NPQdyZ7tGJZ59vfc+T+tQvcRsMi7UjGSApzj6HmuZ1AarpM32gTQvak+Wyht0ke4EhsYAGcHkenPbOdLrlqbQqjNu27ixP3iapJdxcx0V/qiRHZC67iDl2OcEe3T865+TVj5zmS4LuP51iz39xqDbYQ5dj1U1ch0CJYS9zIzseTzjH45ovbYLPqOl8QB24Ylceves4ahc38rJEMZ6n2rptM8GS6koW2tSyk43vlV/PBz/wDrrutN8F+HfCccUuqSPczHnaq8Aj2GMgZHJo1vdhoZsngu88SeA9NsA7W11byyFd8JJ2MQDnkYPGRnrx6nF7RvhVoOkK02uXb3LDDfO/lL+QP8yRVm/wDiFBbxNaafbxw/3OxUfTGPX6e9crPrcs8ryXTu5I+ZSdxo5bycu4cx6JPquk6Fpk8miwW4iXndCBjcf8ivO/EHi7Vb+ACW6fY3UL8ox+FQ/wBpo+k3Ajdl8yeKMoVGCCHP4dKwLh2kZkzwpOAT0rRJIyd2x32yTHVjxg1DCJLhNsRI+YAHPBJ7VqWS27q0DxlMYJb1BHNJo9/aW0k8OfnJJUKNzOQRgBR1NVdhsZNzDPEod4/3ZZlDZ43AKSPw3L+dXrWK6vr2GxsoXmlfaQqLwMrkknsOOprt9J8KWk+hK2tx3NqPtb3CRNhXYMiAhuu3lOnUD9NhdTtNJtRb6TAlpbjnEY5J9STyT25qL9CzLsvBcOm2X27xLcJK6DItomITJ6bmxnH4Dp3rL1XxPJIFs7Jo4Yl4WOEbUUegA/nWpqV3MdM1AK+d8BcknJwvJ/8AHQ35153blo7h1x8+c49KuKVzN3Zp20M92GkYl05B565H/wBf9KoahbNbLbqeELsvHQE8gH8mP51oW0UiNGoHf7oG4sewA9a6+28IR3VkJtd8yKF5Ekjt0YiRioP3sfdyGIx159ejk+okjlND8OX/AIgQNBHEtvCxEt1I2EjwOQe54/8Arkda7JLKw0WJrTTB5lxkpcXUgHzr0wmOin2/XrTdS8Qw29qtnaqsFvEQqQxDaq84xx165rAS+mlSXIA3CplcpamlK7+arBVh8ojaIzwAO9ZNt5kaxxlSzABcDqT0/nVnSrS+ubn7PAPNkOScnhV9Sewrakey8MZdJBc3pztkYAbST0Udvr1x6d3AmaJLPRTZD7RqzrFtP/HuGBb1+bB4Ht1+lRXOpGZkWIhYhwAoAGPTjgD2rl9R1a6upy87jLHJ28A8Vf02C4vWjigjLZ6noFXuSewFNoUTeuGEcSF3Crg7ieOPf/Pesy8069mullVRDEy7fOuZBGqnucHkjjsD1H1rSlmt9OjaGGVrq6SOTyp3Ufu22HaVHrnHP9DiuKn1Ga5V/tLySSDABZiTj8alK2pW51D2djaMjzXK3VwBhdvEakHsOrH3PHtUIupdVdRMzNsJIJ689efyrEtoGaSzfPyvGXfPZgW/oF610+m2wtGW4A2Sj/VrIPlyf4iP5fjViY6aJbKAwszeYWBYdNuRgL/j7/SqkoMWn3Em7GcgAEc8fN/h+dSTM8wfMoWV1IDt25PNLBbTX17JFDDJ5JP7tWwDj37e9ZXutSraleS/ttM8Ktp00Lm5vDsBlVghjGMnI64z0Fc5FGsGEWSOQA/fUHn8Dj+QrT8S7xquxLdIGRFSMSSAkDAPGevXk5/lVNLC9nAZ4p5Dn5SCXz+WcUr3dyi7GkT2zrHIqt1zOm0n2B34H5VX8loeHCHPPysGz+IzUsel30ZTzbbyInJAeUqDx6DOakaKGNiHvR8vA2xbiT+dPQBse1sZdYx03MGAz+VbTaUbfTo7ieUlZMnai4z/AMCNYJSFpFYeY+Dn59uD+GP61qtrEjWKWjQw+WowCc5/PNDTsLQpyNETtjDKe25t36gCpLcRvKqluT0G3cfypouSqgJHbr6EQqT+ZGadLd3MsPlvJI0fpjikPQ1DbgIJJTMoc8M58sNj64zULSxx9Tb4z3yc/lkVGkv/ABKljKB2DEDLABFIznPsc/nVI28LDDTNyOTGuR+uKTWoy79tjMhHmeV7xQg/lkinXGoJcBF2MXRSN7YXd9R7fWqsaWyDHlyOR3Dhf0wat215ZrcwxS2MXlscE4dyO2euP0prsBEglfc/mxlcfdJUAdu3NWo7e9jiUiZ19HSEgf8AfQH86SSS4t5ZIUKJtO3EUQjGCP8AZAqHe4OW3Y74BOaQGlcZuLaD7VNl1TazBg2cHjIB9/0qBUhiiVVlUkfxKmP5mqX2gtz5ZUercCpbtXt0tX3KY7hW2sBkZHUZ9e/IFDA0Ip1C/fZ8dmfH6CmIlnDGHeFHJP8Aq1Y8D86zBKxzyOacshzkPz7UAaMltKhMnlokDHMZHPy5PHPOcf1qJLtLdcmCZyOQBz+mP5Zqs97bxxZlwMH77yYA/OmokdwWMJDAclkO5R9SOKQGiNQlm5ijjXtlnGfyJz+lWYrW8mYgXSksPkEY5b2HA59OayYrVi4IYDd93vWhDbTQgsjCT1weKNA1IDNBGxSSS5cqfm5A59x/9epVXTwVchCT1Ly9PYrn+lSXbx3siy3QWOYfel3A+Z9e/Ax7/pSSafa3A+RNvHUKR/OgYqz2IO2NYmJ7Imc/lSJrNobgWyS77g9IVHzn6L1NONhb+WEkhjkA6Bk3celPS7lhMaxLciNTkYBUL19cfSkB8p0UUVibhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAV9D+Gos+GdIbcx/wBDh6uRj5B2FfPFe4+CNct7nwTZvcT4mtQYHdVC/dPyKRwD8pTkc8jJOMU0TI67cUHBRSOpRQufyquW3EAZyTmqsOqtKg32NwJCAdiKGBBGeG+7jn1FK816/wDqLIhj186UKP8Ax3dVEk+GDfcyM/SosLuO4jPYUrxXbg+ZPboeQMRM3/swqsbCaXcH1K5XGcbY0wP/AB3P60CJmKthSAQelRNcwwShTcJHIOgLYP4VM2iwCFnwb1M5IkZ5SuAeSjEhe/TipIoViiURoiR9gowKAKEl/aPFloZZHJO2WGJuT7kKQ316+9RrHM5IdVTGcEtwfyya0WiUnOeab5S7ux/Ci4Gd5H3g7FT2KLuz6dSKhNuuORJn/fGP5f1rVdc8dB7VVaFs/eoGUUjETKTArAE5zySP+Bbl/Tt+NRm3e4n2oC528OVVDn0wDz7AVdl8lPvyIPqcVAZoDwrE+6qSPzApgZUoXeVeQ7h1B4oNvuUHqCOD61p3CwzsrXFt5wHTLle2OSCDx+NRGG2gBMLFFJ5XGQPxyM/kKAMxrMHFRvYqc4Of61qOylcAkn2GKrsCMjcR+VAWGG4meI218Dc25GACfnT3Vv6Hj6daz5dN+VWtJPNBByjgI4OcYxk57dOvar546k/iSRV7R763sZ3N1bJLC3PCAsp/HqD6UXEc9cQeVa2vXLxl8HqMuw/pTrS8udOlLwvhHA8xGUMkvs6ng/z9K7WXSNK8QR+fDcvGyLsLKAdvcZU49fxz171ymoaRNaI7GSMqq7hk43Kc4PPvxj14ppsCWM6RexD99Jp15ySJsNbN6YbO5B/vZ9M96rXmlX+mXCQ3lq8JZvlLDhsHna3Q9R0PcVlxkHAdgFPBOM4FWLjXr1ZrlIp0a3nYs6GNdjkdG2HhW9xgjjB4rRSsQ43L+tIsWpbI23RrDDggf9M1qgjy28iywuyMGypU4IrSjubLWMfbWi024jjWNLiFCbdwowN6ZyhwAMrlfYVU1DTb7TWjNzCRFKMxSoQ8co9UccH+fritIzVrMhxe5v2GtabqTuuv2cc7yqqC6RdsiAZwcqM9z09gQcU8+FhM17caXctc20YkjjjC7Zmbyg4A7N98Agc9eK5SJirlSMitKa4ksbaxNuzRNIrTs8Z2sx3lcE9wPKBweM0nDrEFPuZoyGKMCpU4II5B9DSjKuCrEEHsa6aLVdL8SSLBrsQtLpuBqUBwTxx5i9D9T+YqjrXhu90QecQtxZMQEu4TlDnGM+h+v4ZoU9bMHDS6K9nql1ZXUdzayiO5jyEkK5xkYOR9CfzrXS98P6xY3MuqJ/Zusu23z7ZJZISccPsGQASDnv3z3rmsggY7elXZ9q6PbgAZM8r7vX5Yx+m0/nTnFOwQbVy5feGdRg0438Xk31gE3/a7KTzE29zj72B3447mqMe1tFgG4MxuZQQDngLH/XIqKyvLmxleawu57Sd+skT4yccFl6Nj3B9q6WbxJY6/EYfEFqIL0/d1O0h37cdAy/ex7ZI+lZvmjuX7r2OWcFT2BHagSsoJIzkYrXvPD15Hai/sWj1Oy52zWh3Dg4OV6qfzrIGHOevr7VpGomZuNiaG6k8uKLfugSUSi1c7oWI6ZQ8Y57VvbPC+rlvtNt/Ylzgnz7c5hJ9CvAA79B9fXmmBQ46j3p9tIG1CySRd8RnQyJjduUHLcfQGiUI/EOMnsSX3h7UtNsheyQPLasoYSR4LBSSAWQZK9M55Xkc54quZY30aC2DMJUuJXkiYFWQkIBkdshaj8+R0QLKyIqlVCfdClgxG37uCQCeKmsLWO71tZL/URBbScErEdqHbjdtB5JIGf16YrN8y1epd4vYzpELcYpIIyqsNuMHA5rqLzwldGKSfTJoNQt1GT9mk3SIMDJKdeu7pnpnjpWHPZ3FnIVuAY2B+6wwQKfOmTyshHyKWOeTT47iW0vIru2laK4jOY5ozhl7cH+lMLgOsOCN3YjpQyHbggD1GarR6C1Rs3vjK4uraaPWLK1v4ZWBkURBJW+jjhTk9h3PrmqUmhaHdiSS01C6sHIykN5AZ1bJ4+eMFlAHXKnnHbkV4xEcebG7pzuCDnocfQZxz2+tSK/Jz37Go9mmy/aNIq/8ACM6r5RvLOGLU7cYLS6fKJtpwDhlHzqeehUVnxjzwzI3sfrW4yQSESMi+YvAkAG4D0z1x14rRk1Swns3GuW0l5I7bvtcRC3HJHylj/Dkk8eufUFckltqCnGRyq7owGPRR61UmR5VLxtjLheM55I6Htz3roLbTrO6ght49QkjvZMri6QCENnIzIOgwMcjrjsciObQb6x0lb27gMMfnJvQsN6jPDYz9044Pfr0o5ujHbqicRnHaimiSAgHcD75orosjnseoSIs0gLsSRnjJxU2nvHBAYlI2JnvnGTnmuTm1m4tSplYb3BJBXr2H4daXRdevZ7iez06ye+kAaRhEMhf949Bz61grJG1mbOtQLeQ3MSuBvEXmNnGB58WT+Wa4q40yyEwQr827p3Pt0613ujWCiDUJfFbQKl0F8u34YbUYPjHO7JUdMjpyc1Ztr7Q9PaSfStKjhuCeZ5yC2PwJOPx/Cly3Y0+Xc5/R/Bms3EamOyitoSOJJmwT9FXJ/PFdFBo+i6DqPk3l09zfEb/LZsbB2wABjvg9TzUcvi77aDA0z7fRAEBx7jmsVZIp9d0u8hVSCJbaQnk8Hepz35c/lVcrQKXMzevvFPlhkg3xQrkbYfkHPfPXv1ziuG1PXUmEgjZjIWzubOT+v862PEcTw6cZljCncAQcjn6VwJdWG5l+7yTmpvcEaOn3CiRjP8wPTPar9xOnmgAlAwwZByDkf/Xrn45ijh4wCFweehpkV208gihiknuBhVSIEkn/AB7U7obRu2oeWG7QIkyLCJnV2K9GAByD23Zx+FU7eZ3uUEcE8szHEcUal2kPoAOTXdeFPBsgsryfxFaSQW9xEqxRcCZvmDevy/dHB5+lb1veaXokLwaNZQ2m7hz95z/vOeW/PHbHenexLOTsfCUkcgl17VItNWX/AJdI2DzEZP3j91D+Z68DpXTWv9jaDDHLotnEtxIHVpH3M5IxyWPPOQcAge1clq1+PNWeUm5ck4aQcdfatPR71b/S7kEqJYZI2BAxwQ2R+gpWur9gvY0zdS3SuZZGkZx8zt2+mO3sKzbkmVUOT04GKtLN8gC4FRRWs17NHa2kLSynkKn8z6D3pAEEbXMb28pBEsUkfpwykf1rmtD0G+1hob2CMeRKgbzywEakgE5Oeoz0GTn6HHfWlhb6bcC4mkW6vowV8tDmGMkYIJ/iOCenA96p3moBLcMXVLaAeWiKuERfRVHAH0q1vclmhb6ZYeG4VliKXOo7f9ey52567R249OeeT6515rU11ITLIN+TwDwPYVn3d/JeLaum4q0SNk8ZAymf/HKkWxO8BVLbjwAM5ahhY5dna7uPJDYZ32/T0rrNM0Rr+KO4SQRWmTvnYZyeeF/vH9B3IrS0fw3baZqb3+ogPc8+XbLykee7+rc9Ogz37X9Uv5Jw7uRhV4A4AHtRugehkatqcGioLCxHlRlQ4Y8tIPViOpzkY6cdOhrlpLoXM7TOCT0GefWoZ7g3l4C4b7pCt24Pb3+b+VXNI0qe/lchCtsnMsuOEH+PoK3itDJkll4futZmikjdI7fLLLM/3YwuD+JO7gdT+eNy61C00vTm0/Tt22NsvK/LyH3I/HjoM/jWfqWtRw2n2DTl8q3Qkj5iSx7k/wCewrJsSbuYQMSC+AKVrjuy9p9y93fR5/iIXI6jJ60mkeG7q98yeYC3h+6Wnym73GRyPcDFbENlF4fbe4DXUo+RSARH1/M8fQY71QvHurlzLJIzbJVYFmznkZzSdmCdmasP2CwMTQxtPJBxmUAKTnOcZ57Y+mcVAZHb53Yktk5/Wq80g3sAcDJzmnQ3EK7JpCSUOI4ipBJH8R+mOPz9KgqwNMs0z20buJw+2UYG3ABIA7k9z9B6Vu3mtR+HdISKO336k6H7wIC+xP8AQc5/OqsF5Dpey7vYS0twzGIbCSi9N2Mc8/ofeud1lpdS1Ce4OViLYjBbjbgdu3NQ9itjJurq8u7l5ryZ5JCe54H0HYUqB9+4tjtk9aBamMnb5bD1GanhtnkwQefdcGknbYotSXn2yGKCcFUgBEbIM5+oJxUcUMCuSxeQnjAGP15p9xYT2UwinaIEjPyOSQPfjGfoTTVUdOTTvfUSVifzbcH93CzNjo77v5AV0Uem2r6ELtrfyZf7zhse2ATjGSOvrXNCTyBuD+X75xUT6rC7FWvVcjnAfcaT2Gi1IZQQVbn/AGQB/Kmq85J+Ybu+cN+p6UyGYyPtVJAMZ3Ouwfrg/lUsh2yqoZJEPVlLcfgQM0AJFb6jcTCG30+eWVwSF82MZHXPLfzpyH5F3bQ5GSA2RToGltLwTiIPb8ja6ld4I74P6c0ivEgZfs8RBYkJyAo9FwRx+dKwaEyFFXLng443AH8utO+2WNo3mz3CRAds7T+ZqIyM0YURQ7e37lWP5kZ/WmI0tspMRMa45EZ2/oDRYfoXIdXgmcJZ7CD8wckMD+PQfnUhvrnaDvcj/ZwP5mr1lKdU0KazuCSbMGeDd1JAPyj35Ix/te1ZJzG+zdGrYztLdKNmBI8ry43RqT6hQD+lI7F7OSBk+8yvkNyrKcggY+oPrUIPmPtkfag5BQEnP6Yq5YJbzS+XI7tnoQNv9TQFjMaG4LENdTOh6AhVx+KqKhNrvABLEZ53sXJ/Ou2uNMtIrWB4Y42ZmKvvLZXHY4OPpx+VUXYRTEJZ2qn+GRIQOPcgf1NK4zBsRcRXC3EWSqHgx8kevbIrrBeLqWnsZo8TKpB38rKP7pAz+f51Ql1mUP5bsJccZ3lh+GaqzXl1LwrSRg9PLGP/AK9G+4rl1zO1ukogbfj7qKrlfzIx+NRPHd70QyggkHlwpH/AQD/OqEYmlY+ZJLIynuNxFSbXRiFB3Dg7l24/OgC3c6DJe4M17GB2AR8/nvx+lLJ4cjh0+a4jurmZoDvNvGwG6PAzjjPHXrwO9JbPciQeZOyj+7gEY+taMWqpDMGViCvdkOP5UejGVLK303Z5kARx90/vTIPpySKv+agIjBUHGQn9cVSa4s0ndrW3jTzDuIQbUHqakS5hk4SSIsOwbPH6UrBc+UqKKKxNwooooAKKKKACiiigAooooAKKKKACiiigAooooAK9Q+Ec8xg1aDzpPJVonEYJKhiHBIHqQo59h6V5fXXfDbUf7P8AGdurGMR3SNbsX9xuXHuWVR+NNCex7csiBiADkVJvyOBx707dKxwECehY4/lk0mzAJMpY+gTI/PP9KozIpFYt0FGFH3z+GaeYo3GFR19GJ3Eflj/Peo54YY0U3CwvExCCRlA+bngqclf1HvTAa1xBEMeYifVsVXaRQwZPMIJxlELr+laMVlHEuECKDz8igCnxwbVIYk88H2oAy3klZ9ot8YONxbCn6YyfzAp/lvuHykr328H8+f5VqbADwtBQE0WFcyprN5VKfOFJyCz/ADAemVx/KqkmjqxxgP673ZgfwJrdbAOCfxqMYX7zBm9QKrlC5gx2MNrshMbRx7iS0ecg+gBO0/Tj61cn02OSMvZ3IuCuMx7CknTqF53fgTV6ZVZTkAehNZYZd21GMso7QqXIP0GTSasNMoMCT0H1qrIrAkba2PsMlxN58sJiJJDSF1jLZ7sp5Y59Rn3FVbm2eO48uGa3lQjIcsVP5YP8zSGZZD9O1RFnXnuO4q81tcno0QHpsJP6kU37OVbLySHH8PygH24Gf1oC5msHIyeDTMM2RtYnritMvsLFbePaRjDpu/nmmyPFPFscvCcYAzuQ9uMDI+mD9aAuUba8uLCdZoTtZT35B+oreTXdJvAf7Sh8uQ4OdokUEDqMDIPfkHr1rEudNuLdVd12q33SGDq30YZBqg6sg5U00Jlq603RzPCdMvYnfcHa1kWRBJz9xHIIGRx8x49TXKlZldEmhkilduEZcE84OPXkEfga2mHAO0cHIPoasXGoC7tFttTt1u1DZE5OJ1Hs/f8AHP8AKm/IDCDFXJyVcc4PetfTdcuLKNreGVRDK372CYB4pD/usCAfcD8+lLeeE7qZPt2kTvd2brlfMb5/Q84A4PHOD7VzkkLu7KwcOpwV6EYPpSTFY7qfSdN1ERTWFwNNup18yOxu5Q6SDJGUkByBwcBhn9KzNRMipp1vNG8ckNqVYMODmaVgQRweGHI7gjtWMtz5tuiTEOFUKM8gADitaz8Q+Yi2Gqwtd2CbjC4fE9vnk7GP3gTztbPtiqUrC5blJztbp+Nbfh7X9Qsb+GzhkEtvcyLC0E3zxtuOOh+v+Oap3miym3N3YSfbrMDLyQrlouOki9U+p496p6U7HXNPO7B+0xYJHQ7xzTlJNMSVmdfcw+HNem8uzZNE1IMVMMvFvIw64I4TngY6/wB2svWraaxsbCxubeWG4hEqyeZGVDHzWxtJ4YbdvIyOayLr9/NK8iDDuzEY45PpXQ6L4uvLKwex1SIavp/BSK5YF0UDGFY9x2zjHrV8slblFzRb1MHy1RAcfjTTIrNg4GOtMjulnupmBeGAy4ggljKuUONuCeDjJz9KeyIzBkYNnOQeDVXvsS0luWrXVbnS5jcWdxJBIV2l4zyQevXg/jXQ2d5pnipJ21K2h067iCF9QhXy4yp3BdylsAkgD1OOvYckY+uTj0HpViB2h0bU8AMJZbVTkA4A84/zUVMo3sNM0r3QNQsbU3skQkss5E0bq4VexfaWC/mR71l2cUVxfjaQdsM0mQeoWJz1weOP/wBXUFlq1/p88dxZXcsEqHI8tyFP1HQj61v/APCQ+HdTQC+0NbW8clZrzTxsKqykbvlwSegIIPBPXpSkpLzHFxZyO/5uAAPSnh88Y69a07rw3cozSaUX1OzUkebAu5wR13IPmHXrjH06DJJKNhwVYdVIwR+FUpJkuLRPHM8Z+RmU9MqSDj61aF/O9y1xNKJmcBWEyqcDIyVYqQrYBGSrfeJxms8cj2pwOOpzmk1cabRrQ2+kX0TvPcS6XfOBsWVd9uWx0Ei5KrnP3unqetVrzSb+wciWISRjkSwnehGcZJxwD2yBmnaTBDdara29zn7NJKolHYpnLfpmqVlKbZ/MijWPAwF2hlI9w2QfXJ5B5HPNSk1sVdNXZo6TCpsNYmfHy2qlSfeWPofpn86zg6g//qqWNo5b7deReZbkhgsIRHDep4+ce25eeetR3bWy3kNvakTSzFzsjOSg6qp4BLdew5HHGKadm7iautADB1JBPPB4qrqThbZNysQ0qKQPrViKQSAEbuRnaykHH0NQ36qVgVjz5wOPoCa0T0I6iKxHIp66hc28UlvBcSRxSAiSNWwrgjByOh4NMVgz/wAjTJVC5yAw9qrRi2Zsw+NtZghjhQwbY1CjNtH0Ax6UVh59FbH1oqPZRK5mejWei+HdKuC+rXkuvXa/dTaYoFx04zlhnPU4/wBnvT9U8Qtc/uo0aygKqq21pIY0VQPukLgEcnsB/OuYlmlkcuZCioMjnHtVPzy1wrHkDjNSootts2lvF88kABhx8oGfrSS3RgViGJ9/asz7TiUktjPHQ1K0yEE5yGFVciw1bn90x5yCeav+HXaQSIJCHguYZhnB+Q7kI/EstZCMZF2ZYjkkfStvwXplzfatM0MbtG0LqWRd20/wE9vvAHn0pvccLGzrln9oDKu6WVweATheO+D7Dsf0FcnbeGNQnuPKRJJJGf5IYUBLeucngcdehr07+yrazti+samqkZb7LZKC5Gehb/AcetUbnxclrZ/ZNIhNjACWZx99/wASM596wjF30KbSOetPhhMiR3XiHVYtMh+VjbQgvMR6HsD9A1dtptz4X8NWzQ6HYxodpLzyH5pPxPzNzg46DPBFcPNqbTv5kQJZjj5iSaZKzefC78kDOQudp7H88VbgkrkqbbsddqGuG7y8kjjjkN1HtxXET6yk2qIiqfLzjP8A9arMMt1emSJlO4LlR0A7Vmf2RdyagCkJYIRuPAA568kUtmUiTVElET4cMikMAF6Dp1/H0rW8I2Go39hqLafaRzOssXLHBAGSQv8AtHj8M1s2HhoiEXN4DaWpT5pGA3zE9Nq/j1PHpmtJ9at7O0Gn2EP2aL/pmfv/AFbufpVwi3exEpqO46HSbaz+bVptrn5vs0RDN9GYcDnPAyfpUerX8EFs0Nnm3hZQfLgGwtx1kYcsee571jy3zzS+Vv6fyp0MI3SlmDFkBCtyFzxn/PpTnT5VciNTmdhbe5keFAvyKSAB0wKztUEp0x4uCVfdxxWjbxiIHeylRzn0x61q2fh671CVpZ1+y2KjLzSjBb/dH9Tx9azeppYwtAt57uz0+JlZ2ZJEhQdThySR/wB9jnpxXaw29tocYaSaKXUgvybclYsnHHq3ucf44+pazb6FGtrpaIjbdvmhRvYZzy317dqxTqkqSwyhywLKGP16iteRy1JdS2x043RtlnLEnOWOTWN4mlmWCJISf3pIKgdcVduHkcRpGC0pOAAMkn0FWMW1ukc18UlnV9qRKMqOCeTjnp24+tTEUjndE8Ni6s2n1GSW3t95MJQjzG5GdoIxgkLkn0xznjQ1S+gjsm0u0PkwHIAU52ehJ6k5Gck5p+o6i8yyqsjNLgHJ7Z4H8ulYK6cftRg815AR5Yc8n0z9a0Wu5LZjfeuWOeD+lddFZReGYTLMgkv3X92rD/Ug9CfRj6DpTLXT4tBh866TffuAE3rxESRg4PRvT0+vSjPdPduTIxYA5JJ/rSb00GiZDJcSLLIxO0dM+p//AF/nTzG13FJboQGKk7mPC+me+M44AJp0ULvBauqiKNlkzI/cAptwO/8AF+nrTLi5tLXDptjjKZZy2S2ByT6ng8Um+4npqSJaJE+LmYzzIchEG1Nw6ADkuPrjnqKsadLp1pMk2oXML3mflia4RSAP4iDkgnrz/Osg6mbhFls7mCOZlG1ZDkjI5P5Hp/Trl/2dLK5P2mFmJy/z8k+p4zUvQqElLZnQ6hPeXF1NqV3Ym5kQBXnt5UdETOVUFmU4BY9uucZzWPJqqyHK2XA/56Sf4A/zpLuJobJYVuoyfODMi7gWAU4znjGTx+PtVQIxHYVnboakhnkk/hWNiOcHI+lXLS8NtIkigF1IILcrn6VTSHJB3c+wq/p+l/bZJSZCkUMTSyNjkKoycDvT2AS7u7jULs3F1O0jnHQBQAOMcYphQNgEEgepzSJGq9WOCalZA2Ogx6c0BoMESKpPlIMnqFrWAnfw08sUkaLFOPOyAGZSAFGe4yTx+PscxEZGyZgw9NgwKtMuAAbtD3KDdx+QxRYVymxbZnBI/wBgZx+VCtIAWXeRjHy/Kf8Ax7FTsVUgpnjqWH+cU0XUTkgSh2B5VPmP6UwJbe2d5lG4GMnkqCSPwIxn8au3OnxaffPbzyyXLhVIEa+UBkAjkliwx/sj61mpL52Tbwytt/h2Fcc471pas2o6tqsdxDbwoBAsbPNMybyM/NtC5yRjqO1JgQ+akchZI442HBIBbP4MSP0pXvJpPlLnb6Dj+VOj0O+Zg0txEgz9yJSxP4tj+VOGknesMl9iRR3baT7jAFTcZDGTI3EbOdp5cEY96mXTZTGJmjIiYcSDofxq5a20SrsMy3FwDlGceYYz2wx5H51k3NvLa6rNDIWJc+cpmbkI3IBP5j8KANqzs7SH5pbiAsOzleP1qyuo2QIiAaXn7sce7Hvg1gwPYiR/ODg57EEAf1rWXUtOjQDcNoHQxEn+VFhmsl0dk0fkM8cqYO/5TnsRg8EVSlWYRhPI83+9vXd+XzLVK51mzwgjXc56/IOfbmol19p38iGxkzjOd+wAdvmOAPzosInmhv15jSNUI6eSnH5g4/Os+W4nik2iaQD0SUhc/ReKtTazOxEbRRLkZJWUOF/FTyfpmqMy272zkS75CMgFMqD75Oadmg0Nmzu/7XtzZX08uAwaOSOX5kYHI4OQc+4we9TWyRRPLDOMzwviRZEAPscAng9RyfrkVzQlK4aIICOclSfp3GK0Ztcuri5t3NpE9wi7FYliX474IB7n2otcEzVlvxFuEaF3HVFQ/wA6p+Xe3QeY2kjRt1Ls2xf5Y/Gsi5mnaUvMYw2c8ybth9gSQp/KpbYG9usXM878feyXI/nRYLm1b6d5Y3yvboOuI8OfxIyR+VY3inxNp3hbTJ5nZTdyK4tUW3YiSQAdd235QSCSOg9TitW50/TrtRC0Dxns8btG35givnHxBqzazrM93vlaHOyBZWJZIx90HJPOOTz1JqJOxcVczKKKKyNQooooAKKKKACiiigAooooAKKKKACiiigAooooAKfFLJBKksTtHIjBkdDgqRyCD2NMooA+m/DmqR634dsNSARmniBk2KQoccOADzgMCPw71p8dlFeZfCbX3l0e60iXLmzPmwhBuby2JLAKBkgNyTz98Djiu+W/aUukVrcM6Y3K6eWcHPPz4yOO1aq1jJ6Mu5544pA0qMWjdo2xw6MQR+VVSL5+iW8ef7zlj+WB/OkFvdGT97efL6RQ7T+ZLfyp6CLxK3JySltPu4kjUtGw4+8mcqfcfUiqM2q2kUpS5Y27DODKNocDurHg05rCN2JeS4Yf3fOZR/47jP41OVAZmJRQ4IYBQoOfYcUXQWKj3pJTyrO9kV8bWFtIFPvuICge+cVWu7jV0wIdFlLMOC9zAB+OJD6e1aLSJnmQkAYxngVBJKP4VOPXNFwsZlvba5M+bi5s7RSP4U805+nAx2zuP0q8lqWLidbuQKduUnzu98IqsB+ePXvSF5MZEhHfikBk4O/PP40cwWJSLRIdtsluiHIDBFcnJ5yxyx/OopbhpgFcu3oCx4qO5JncSSSOJsABwRnjOMg8HqevPuKqNI2xjchSqDJlXjH/AAHJI/DI9TRcLFoxDBJAwfaoTGRxniljeORAyzb1YZBU5BqNphHHuZGX2ZuR/SkMjYkuy85HUkcGq8keAc4pjavaPKUDqX/uo4dj+C5NNkuZH4SzuS3TDJs/9CxQA1o8jg5+vSoJIcjoM/Sr9pbXVzLsJtogADgyhmOewHAJ/HvUbGG3dYruKXzsZKlwin3HB4/GgCgJpbb91HK6pJwyA8N7MOhqK7MEyZEAhfPLo5KY/wB09/owHtVqZowwZbOEHH8TO36FiP0pI7u4twTA625PUwoqH8wAaLjsZbaHfSJG9nFJeJIMloQSEPcMMDH1xj3rPks7wAFoniBbbmTCn8ic1rzyz3LkTyyvhi3zOep7/Wqsq/IwGARQBVtpr3TLmU212EBGCyFsNkduB+dad1qmm6jbWw1CCaO+jzi8gcd8csMHj14JOPes0Mdg3ZJ+lQyspHzITjjOKAIbzw9f2UEl0gF1p4H/AB82w3qOf4u6/lxWSsm4hlOCK6LSdZm0S9M1uolicbZI35Dqeoro9V8Gad4gshqvh3bazMMtauw2MSP4SCQDz0zj6UgOEtNSvLC7iurK5kt54jlZIyM+49x7Hiuq086V4tsrtry2j0u9tyha+hz5D7jtHmJnCZP8S9zk+h4/U7C70q8eC7tZrc5JCvzxnseh9Mir+mTrD4d1MbzukntwAD1GJDnHpwP0o3YF+5gksmNtOhSQqHXII3KRwwz2P/1uoqFSpXBGfSmW+pzfZPsplWS32nZFMu9YyeNyd0P0PPcVdis49RnmGledMsWC0DgCdOMn5Ry4H94Dv0FbKempk49iqFVwwfLLjAFNkJdIbUEMEJZXYlXTPJww6jvzTs5XK8j1Bpv2b7QJHB5jXzPy/wD11T2J6nT3fhbfZw3uhXD6vbudrqFCzRkHn5erYxzjn26mucd41tLnEgLtLEpUN02hyePx/nUmhX2oWurQGwlVJcPId5O3bGpkP44UgfX3pdW1G41S4tpL5I7mOAAGMMYnchSoJcdTyCemcY4pK687D09CqrB1znk9KYqAZY5Jbk1tzaDDc2LX2iX6TxxKDNb3REM8PHJbOFI685/OsOC4V8nPzKdrKeoPuKuMk0Q4tFhWaOZZFeRHT7rxyFHH0Yciuth17S9eiWHxJaRmfAVb6FDu4B5bHI/DI68AVyZIIGQKQjGcUpQUhxk0dJdeBLh4hcaHew30DZKozBXI9AejHr6VyU6SW9wYZo2ilUkMjjDL9R1rorLUbiz8NGOGeWIyag5JRyMhY4/6n9Ktw+KZp/8AkMWdpqkSj5RLEFkH0bp0rO0krml0zA05PO1O3h3mMu+AwUNjgnoev0qoo4ANdfaaLpVxex32h6u1rdFyI7G7jDAMyHgEHJGNwzyPr357UNHv9KWM3dtKiyAbGZSFbIzwT1PtQpag1oUzyMcA0yRVbBZVfB4JHP4HqKTdk8dKcG4wa03M9iaKdmlj85IbqNQg8u5Tc21CCFWUfOowNvUjB6dKtCTwpdTWFnqLXul3XLzTbRMmTn5VYN8q/VCcHB6ZrOZc9OtNMhHysoKnqDUuknsWp23Ok1bwJfadF9rs54b+12GQvGwVguM5weox6En2rmZYnSRo5UZHU4ZGGGH1B5FW7HxBeaLYvbQSXUto6uj2pkLLtKnOBj5cDuPTnNdPeeLIdWlkbVNDsr2DaCmxiJ0Pcbzxj3GPpSTnHRg1F7HG7U9DRW+bTwi7FhceJIwTnYIY22+2cHOPWinzhyFQFZVaNCMY+YbuQe3FVmVI22q4PsOamEEkki7FfzM4ATr+XrXWaF4C1TUpYXvLeW0tmzukk2qwI6Aqec59qtpLVkLU5AwuA2A24jj39q09I0K/1XcLW0knI5Z+iL/vMcKPoTXcRaFoOj3Ec8Nu2qTZ5lu3AiQf7KAfOfrx3q7feJXfyhENixMCvzenoowAPb880rNj5kjEtvDGjaDcwS6veS3s4+f7NbpsiX6seX9sbR61oan4wEkRhsohaoX3My9W/oM9e596wfGN2k/iV5I94eeGKR1HXJQY/TArnftDFgApB/OpWquwv2Ni71CSV3bzGJJyzMxJJPck9fxrPCmSQOXYMBxRGrs5O0DPQtyf/wBfFPLLkY5bGDiqsIRZAmd/B9c9adCGvXLE7AvzZLAYHGTk98ZNRqqscgDcfU8V03hvw881ykt/F5emujhnZirOeQAoHPBwc+1HQFuU7ONry/aytIZTLkr8keMc+px79Txx0rtrWxtfDWly3t60F5cqyqIyvyxk84xzk98nHsPVlxqdlpcbJpsEcSgBS38bY9T1P+elcfqOuG7c26tiMMDgHg8EY/MilGPM0hylZOxa1LWbnVLhriVzyc7QcCsxVJkZi2XXoPanxN8mDz6U94+Mk4xzmutK2xyb7jVh3OG55GelaFsk9zc/YrazlneRNxaMAfdZQMk/7xOPQGpdI0O9vis99JHZWbEeXPKcGUf7K55+vA+tbs1/BpafY9Ikfy/45CPmdu5JPbgcDFY1HfQ1guV3YkFpa6EGkvcXV/wxjUkRRYHGT/Ef8+9VPEWvzyqVJULtDRhRgKCoOAPXDcn61nX91PPmWVySThixyT/kVna40rRafKx/dm0UbR0JEkg/9BCVEUkXJuRlXczSlHOScVp6ajahdQWSRb/MPzk4IVeMtz0AA71Q0/TrrVLmO2tY2kkY8KOw9Sew967tEt9B006dbqHlU5nmxy7cZA9FyPxxmtW7E20HyvBYRmG2eR8jaZn+9J9fRenHfqcmsO9mMkqsRghhjB6cjNEt0XZSCWGTyT2otoZb69WCJd7NuHHHVSM/hUpWE7tk8/71hGEPD4GO/Tip1H2TLbCLkcD/AKZ8c49/ft255Ek2o2+gX0ce1ZLjftnl67M8Mq+mOeepPtxVa1l8+1inIwZUVsemR0qEy2irqiSyWMxJZpPLbbzzux8v64qGKK3itW1C8BNucmKIc+eev/fIPHv0q3dMBCbiRVNujBdpOPMbrj6Y6/gO+Rhz6gdUuI9vybMfd4CgdAB0AFJscUTRlNavLm8mZtyYRcMR2PH06VLe+HZ/sYuXWJLZFBHnF+vGCu11wegGcg56VY0uzstJ0MXOrSPFAoDGMqQ8jEHAA4/yD0Ark9S1i51JpFaSb7KSNsLMSMDHJ98ioZdr77EkFtNA+HaeQnHDE4PvjHH4VLum37FCoSScLgH/ABrKhhijUskaKCcn5QK1tNvGtYnKPiSVQqjkEKDyQOnXA59TSbbHGMY7Ia6vEV81DzyOM5FSLIrHhWA/3D/hT2KyNvLM5POW60EKAME8e3WhDH+ZsC4jkcnrtA4/MitK3vIodJvVBkW6kUKiHGCNwOGODgcdRn6Hoc5Hxg+2ck05dzn5FyPYZpgKmSSHfZkZAXnLfTjj3/SpQAQNpIb3FNZgkY3yKin+8wXP50pkRYfMDgqSR8gLnP0XJpAOiUg5ceZnqpOB+nP607UvsCXVrbLC/wBoaDzXbznI6kYAJx/PtjpTrSSAsJHYkYJ+6QP15/So9Smnv7+CXzlMcMbRqGHJG7K/kP50mCRCY4BMHWztyTjcxX09BjFW2upWBaPdHGTjaowP0qm8TupxJg9iF6UvlzSBRNeXMu08BnAA9htApXHYne6nlIaSRn292JOKZ55iBfftHc5qD7NDuxh2yf8Alo7MP1Na/wDZFpfeG7ic6fbGa32/P5ALbdw3cgccbqLtIDKGvRF9gumkb+4jFifwFP8A7beQqY7K5+XnbtZM+/zmnIAilQoCk5x7+tAAYnIAPbFO7EWU1i+kUyLZxQy+s0gJ6+xbH5Ut9fTX5sWJWFoEdJdnz79xyMZGFwR6Ecniq+3qTimn731oAkbInLLLOIyD+7LAf+PKqtn8fwqZr0vH5Rt7fy8YIMKsT9WbLfrVfuDnA/nSJIm8BiSpODtAJH60DLo1W9SHyY7mWOHtGjkAfgOKdDcS3sDafNKHV2DRmQbmVgcgKSflz0znvVIT2pn8oNuYDOzcA3Az0rV05rAxl3XLA5CyDaP16/WnqIzZopYJnglheOVGIYScHNRFnHbp1NXdSiRr43NvNE26NQ8e5vvgnJHUcgr/AN8n1qizTFf9WmfQvgD9KLiHwXjx7ljiiLHjMigkfnwKVria6k2C3csMZJUAH2A4qDy5C/zPAAp42KXB/E7cflVqaSGRMCFVAH99gf0IougsX9NuU8oNKIyEbHCgEfkP1zRdasc+XY+VHLzlZx989iGDfzz+FVYNMsb6FlhMi3AGQJT5it7c9uOvJGarJmNXiEIgYMQ8aptwwPPApMaNmLUXMxJSHI/idNjE9CeTj9fzrwDxNpZ0jxDeWoi8uHzC8ABLDyycrgnrgcE+oNeziMyJkZ4OCfesbxH4X/4SO0WNXCXMWTA54XJxkN7HA57foZmroqMrM8boqe8s7iwvJbS6iMU8TbXQ9j/Ue44NQVibBRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAdl8LrmaDx7ZxRPhLiOWOUYB3KELY9uVU8ele9/wCieev2mFHYA7TnDD3U9j0/LmvlqwvJNP1G2vYQrS28qyoHGQSpBGcduK+k7a9W8hhuofnt5o1kjcAjKsMg4PTrVxZEkabBdryROJYVGdxIDL7MP6gYPt0qv5x28DH15pYpWhkE0T7XXo3f8aikjjZnaCQLcv8AMLeQhUk9QmB8p74PBqvQkPMY9W/Ko8pljuBPfvUUdxNJGSllOhBAKSqIzk+zEVBdDUCi+THahic/vJGO38AuD+BqRl0EEcjn2OeabwpweP8ACs2G3vjIDdah8pzlLeML/wCPNu/kKd9gtGG6VZJzkgGeZnB+qk7f0oAfcX1pA4ElxED/AHQ43fgO9NEskiiSK2mdcdWxHx9GIz+FWwVW2McEcFu3RXjjCqPZlUcjp05A6ZqvHO7KizwtC7AkA8hgDjIP4dDg9OKYDHF43EYt0yOGYsxB9MDr+dMa3eQSGS9nV8jYbYLHt+u4MT+YqeTKk7sgeopobrnA9hQIqx2ESXa3jvJLcj78khGZD6sAACfw5wM5q04WRNrJFjpgRqPzwOaaeO3NMLfL7fSgYrMxYgsSR6mo3Uke9B6ZB4pN3yDIPpQIicBgcKM+9PE4lgW1v4jc26vuCM+GU+obqP8ADjionkCDJPTrx0prdMqdwIyPcUDJrjS4pFeTTJFkhj5dJpERgM9BnAwBwc46HBascncSRhvXB5q4rFfmUkY7ir9vqVlKBFqVtG3AAuFGJB9T3ot2AwigUYxwB0qGSLIGBjPf1reuvD7vGs9jOl7bnOGjI3dcYx3PHb8qy59A1RozJ5BiAPWWVIyPwZhRcDGkg+U/MPpmqskZAwhO316VvQ6HCiyNqGrQRYOQqfvSR9F6VFJb6ZHKyrdXE9vgFdqBGz3HOaFcDnZIzuyVxx+dS28stjKJ7aSWKUD78bY4966m0Xw1GgFwGZiDlgZTg/TA7fWobyTRTC409mhlz8rPbhgeuckknHT0x6Hs0mIv6bqFt4m8OyW2o6TaTyGUpG0Z8uX1yh2Ebs54XqQcjkk8bq/h2fQoGlS4E9ozjeEVlKMAfvDBXjJGQT36VbuIJ/s4lnVApJAZJVdRxxkA5X8QKTT9S1DR5hLayuiHmSLPySj0YdDS0t5j1OXE7KV2jjqfarMBWO6NxGpS4+9vRypT0II5Brp7bS/Dutm68yQ6ZcbQUlciOEnGcEHgdOgbJ6461iatpN7pEnnXKeZCSEFzCN0b8ZGH7nk+/H5PYNGXrfV7W/BGqRslyzcX1uuWYn/nrHwH/wB5dreu7pVlrDUrUebb2jXttcRMFubVTNGAR328qR0wwH0rlUlDAyI2NrjHtWxoepXmnTyS6feSWxHDhP4un4f/AK6ak+hLRY0Ni2sRq6Abra6B56H7PIf6U1MMnIGexre0S+0i71kXGsJJBdSbkW4tlwh3AqxdTkA4J5GBWZcaXcW7ztCpurONsC8txvi/EjO0+xrSMrXuZtXWhTIEihTkEDAOcEZ64PatVb+wurFbK/08SSfKv26PAuFx33cbh65yT9ay2A2Zxx15qMsA3NW4qRKbR0GoeFJbTToL21vY762lOEK8Sc8jK/1FYZDIWVgQy8EMCDUsc86xuomdEYfMVPOe2PQ+9dRc6/o/iCW2bWYpIY4IWSbYo378fKQwG7GM8epHBxUXlHTcqyZjz4Xwrpb5/wBZPck8f3Si/wBKyxJ8uK7SXw5puq6BZ2Xh3VVuXtTI6RzyDMhkIYjPG0+xA/CuKurW4065a1vIngmXqkgwR7/T3pRloNoUuSOOvaugtfGGoRxQ2t2lve2qNl1uohIe445APBP3gT71zYPBzSh85PHFW0pbkptbG9NpmkaxNLPp99Fpk7uSlvdELFjjoyjC9T07KPl5JGbqGi3+lASXcIMRJX7RAS8JOcY3YGD7HB9qpeYFIGBn0PStPRb+9bWNLtrfEqCTyvIZdymN2XeCv93Az7daTi46ormT3MpyAcZ5qNs4wR+NXLOwsSYEur57d2YI8rAGOMAHDEAc7iQOoxjJJzkW9T8P3unS48y3uV8n7Rm3csRGW27yuAQN3GeR70lOzswcL7FPT0Xdc5GcWsmBs3ZyMehx168fWoFJjJA6DrVzS7+OxF5PtDGS0eEDPUuVX+ufwrOYj33evtWkdbsiWyJRdPjv+dFQbov+emKKsnU95sLXQdACtplkHm73U2WbnrjccjPtj6VSvNauJpnV5ZDuHZuB7YHSsibVCzAO6opGOetYN5rDLJshBLtxnpWRWrNbULwljtyWzg5NU0LyLuUYweh9arwsD87sDx0PerlqiNG7qFI9zVbCsQ+LYpjNp15Ky7bi0CAL6qxX+QX8zXObdpD8nHtXXa/G1x4e02R8KInmQY64+U8n8ao6N4T1XVtrrAbe2/inuPlUfQdSf0PrUrd+ozLSQPGzP8uSD7VsaV4S1LVW85R5FqOPtEy4z/ug4z+g967bSfDejaVE0yJ/aF1EhbLsuAQOcLnj8cVna34tRrRJFcorDcuGzu/Lt7fzpiY+HR9D8OxGVgl1OBkvcYKof9lcevrzWHqHiK/vpn8mZlhzn5m5Y9jjp+ArJe8lvSWnZhGeQue/uKjeZVwo4DcDjFDdhpPqQ3eoTNIW8zk8ewrLzceaJIl3eUDM4I42INxP6Dr6471Kp3TqcDk9CcV2HhTQ9uj39/qgdYr6M28canBdDgkjPQcYB/8ArURb3KdrEGn2Fxeyi3tYzJIOuOAoPcnoBXS2un6RZPH9stzezq+0hwQin0VWwG7ct6dqYt4IYEs7GMW9uuQVjzz7k9Sfc01l4yByOQa0lO5io2M2+1l7q7ab5TI6jnqQp5259B6UnnrmPDY45FZEsSwybIduAzDgcAAnirEcsqRKjIzyEhVVVyzH2H9Ki5aRJqkha3bj8cVPpnh6+8QaVpuCI7YTztNOx+6n7rGPxDgCte10F1hSfXAYYiR5dop/eyH0P90cZ9cZ6YNTXmpNFbCPEcUSDEcEQwiD0x3Pv/KnurIV7bjXksfD9g9jpe8kgeZcMf3kxHqew9gAKyYLk3KySO7M46ljnGax727kmd8ZJ61raHDPf7LaO1m2eUd87JiNW3E8t9COPrjPZpWWpLdx1rZS3t9tXAUDLueFQep/z9KvXVzBb27WtiSVPEkzdZPw7DPYfqeanknt0sZrO0YmONlZ5SMGQ8jn25GBWPOhQcN17Y6Cs5Sbdi4pGfPEMkjI3HNXdMZX01AXZYISY3KrycMQFX3469gM+gMKxyXBkA4iUfPKRkJ6fU+g9s8AEhZbsi2jt4fLWBS2NgIByeSSepOBk8Z9BTvYLFbXrpbxtsDgRwERhE6ICoOP89cknml0TTA4llmYpFCu8qF+aU44UHHA9ap20DarrMtrGClsdrzTuPlBUENjJ7gL/ga7DWdUSO0+w6ezDy1wWh4z6jHU/wCPrU3stCjzfUtXn1nWpheNMgidlji8plVPXGQByepOCePQUsVtFMAVkkAx8pKgZ/WpLiEC6diuzcSAzDGf0puGK7BJgD05qF5mgCxkXnB2jqd2OKvuWkhjiXYsUS7V/vHuSfXknnjjtVaKFwuMk/hU6wyHqpX0yRzTAsRNZxrtkjlduhdHCZ/AhsfT9aVlhdvkM0YxjIcE59fu1EIpOMIxB5yamiTjjOO+VNFxDJbNbho1LPGB1Zcgt7k/4Yp8+jR2nks2+SOUFgZZGckZwPvE4zhuPapY7VpXLbWCjHO04z9a1/ENmltd29opEYhgXeGYb9x5JPTrkflUsZg+UkDbYUSMYx8igVMdjAYU4H3j1JFKLnTbV9tzdwqTwA0i8/kTU8t9pUUeE82XP8UUTv8AyHNAFVtu4lQQOwJyaTBIzjOOwNJLqsBO230y7cjpujMe7/vrFQNrV1NI0C6VHAVAJE03rnnC59D37UDLSt22nJNG8bSWbgj1qs895IApuIkQ9Ujgx/7NzTgI5AGbkgY4wP50ASiRVxy2PY5NdV4evLZbeS2V0MdzA6MHITrxjnrwD+A9K5aO4VdqbU2joGGfwp63DECRcbAdodeAD6CnpbURmWmqRyqscKNPIFG5bdHkIPfoCetasVvqUylo9PlVR3mYRY/76Of0pJLguNoaTOAGYvn+lR4DKAASFOcZ70hl5LNWUi4v4IH7KiNL/LA/WntoclzMscGpvEh6O9kw3H65wPpz+NUBls8cjoDUsE86DklCPujd0Oe386asxMgjsY4L28tZ7iW5MT7BvkwB8oPGwKc89yaVrW38sI0EbgHgyKHI/Fsmr2q3Nn/b8LWhjK3UAYpKNuxgR26HO7sP4fbi61jE0YeaaMDu0QJ/+sKWtguYytsj8pflj67RwM/SnbifvEn61NcwxRoHEZWJhgO/Ab8aILQzRLInzRH7pRS+fpjNK9gIc9+vakO7d1yKsb7aGUo8OoEggYSzkbH1O3APt1pZLmz3CNbeXdnjzT5efr1P4YB9qd7hYoTSJDCWckDgcAn9KnWMYxt68HIpzSzykJBDbRn+EqGlOfx21bRdaKbS20cYkVFj/LJNIY63s5InjkUAhSGwDirOq6ROq/2tFzDOP3oLj5XyBkex6Y7bR68VIJbuTdH+8uHJHLkkj8sDH4VfhstRj05rRJ9tu/3o3cvn8Wyf1qvmSY8cF2zAJExz0OP/AK1SPBdwWzvNA/lrzu2k98YBxzz2rSj0eaFCsd0sKH7wjyufqBTJNGjgjO64h+bqz9T+JNFwOS8T+H7LxFpo8mB49QjH7mYjZ/wB9xGV+nTr6g+Q3lnPYXktpdRmOeJtrqSDg/UcEehHBr3z7GvKrIpfsi5Yn8siuW8Z+HEvtJnunjEV3axNIjtkblXllPHPGce/pk1M431LhK2jPJaKKKxNQooooAKKKKACirD2ciafDekr5csskSjnOUCE57Y+cfrVegAooooAKKKKACiiigAooooAK9/8DyyX/gXTJJHIfyjEdvGFRii498KK8Ar374exvH4D0xXRlbEjYIIODIxB+hBBpomRtrplsrEyebJnqs0zyL/3yxI/SnxWtvbAiCGKBT2iQKD+AqckscVEzEDjnnBx296ZIuwJn0/OmMd3HUelRbHQkRSOSeSH5z+PapIzKR+8Rd/tTAbsUKARjHIAHSm5aRTuDDnkVKC3O5R9M0hI54oAQKd3JH1xSGNDt3gOB0Vi20/XBB/WmsxBoZ0jh3SyKqLxljgfnTAaRdy3B8qyMkQDFvKIJRQBzt3FmHXJAz0GD1ppARiAuHU4KsOhHUEdjUEeq2V00qWrm7dBllt0MuB77AcCnEy3twrGRrdwmxXbaRgdmXhuOeevPtiiwiQuTkEAVXLBc5OfSoB9sYv58SRrkgNG+8nj0OMVImB8rb2IIPGFyM9xz+eaADeQMEHHr6VXYFMNLOqxjnA4B+uamuYop1ZfK2g9CJXBA/Bv6VUXSbOLcy2sbMRy0g8wn8WyRQBA+pWe7EMwmYdogXK845I4HJxzSm8vfMXy7L7wyC8owfr1xVtbfyo9qfJznKcYPrxSLLaRl9yiNiPldCTHn/aHOP8AgOPp6HoMoSQahMm57iGAnqiJvx/wI4/lTDpy4/fT3M/OcM4X/wBBANab58pJWRkSQZUsPlbjPB6GoxjHyjPoeopARJPPDbmGEiOLOSqfLk+px1NVXZx1q0dvQketRtEOx4p3Aothuo+tVpFCjAHXrWk8eD1zVZg27oppDKTPtOChx64qtIPM/wD1VpTQiTABANRiDb1Uc0AU4Zp7WTfDI8bDupII/EVJLcx3Xn+cXE0rBmnJMhznJJBPXHpj9TU0tsByVYGoxBGQRtJoQGZwXZJcjBIDgYUj156VLDd6hpqSCxnaNGPzgchgOxz1Ht0PerrWEjKTHDI2BnAXqP60kck01lcpNplxI6zHdexxkmP1VlIAPf8AiB574p3EYr29rdSvNM0sE0jFi4QMuT1zg5A5PQH2FSSWbacm7cptZACsqtvDOMBunKgnJGR0+ua7Lw9ouhG5D312LwScqLhDbhRyOAfvnPGASPrXS3Wg6fb3ERt0ijSfCSQqAq3A7BhxuPoOvB96Em+gro8rBYkYPP1zkVe07VLvSrkzWkhRnXbIuAVkX0YHgijXtNbRNQuntoHFiGVkjB8zyweuSD8oBzjP881UsXS8YLFkyNwqdycZrW/RkNGnOljfzqNP22crLmRLmQiHdgH5Hwcc5GG+oIHTOvbO6069eG8geGXjAZeGHqp6MPcZFP3DPB61o6drMmmwtZTQreaZIPmtJTkA+qHqp+hFPVbC0e5lRMTGD6GnWzOJp1kVSgcY9xitJ9IF3dldEWWYPCk4tZWUSoGHQcjfjHYZ5HHespYLuC+uUuYHiZWEZDoVw6jJHPfBHFDmmJRtc0Id0dwtxC8kcy/cKuQB+Xt/Ouwh8a2MmnpY67ZPcIFwxkVZVc+3Qr+FcVGxDspyCPSpGclee/am4J7iUrHQTeHNB1e2E3hu6+z3B5+y3MrEMB6FhuH1PH865O5hm067e3uojFMvDRycH8Pb3q3HJJHPHMrbZE5Vx94fjXU2HiSxltvsniO0hvYScLIYQdmfTHK/Ufl3qbOK0KunucI6gnIzg+van6Z5javZRpIYSbqHa4XOMuMcHOTXcXvge3vrb7Z4XuvtEbHm1kcZBzyFc4z9DzXDyLdafqSr5ckV5DPGdjLhkYMCOCODxnkfpT5k4is0yRsFQwb5vUVoWOt6nYNA9vdOph4RX+dVHIIAPQEE9MH34FZ3J5yAe4HSkViFJrWyluRe2x0tzrmi6vFKuraVFHezbgt7ZxYKyM5Yuyknpxzy2MjkGszVfDN9pkT3STW97Z5Ki4tn3hcddw/h/kPXmswnPI6+tXNOv7rTbtLm1meOVOhViAevUdD1PX1qJU3H4SlUT+IxwlwwysDEHkEMOaK6NdevFUKbbSpCBgvJpcLM3uTjk+9FT7/Yr3e5rm0huFZ3GM5OfSq/9nbWyhJx3J6ipkjuBIFdgoGTyR6f/WrU0/Q7/UpxFbRGXBw79FX6n/J9Ka7i9DMRF+4w3Fec+1dRpHhy+1S3Ty1NvAOTNKOOfQdW/l71t2WgaV4eMc+qTi5u15CkZVT64A4+rUzUPFNxKSIWESg/KF6gfWhtvYNFuWBZaJoqxwfLd3EOT5k6h9rHrwBwfY449aoa1qkU6K0w3RxjID9AcY6dOlYUl67yEbvlzkA1aaPzkJPzArgZqr2VyGmUIdUmhujLlhIrHbjoQO2K5jUNsckdvJJuW2Zoi2euDj8uK3J44o3RY2PygfMDWFf23l6jdAj5ZCkmDyDlQSfxobuUlYQ3JygXO3GcipEhluLiK3hjaaZzhI4wSSfQVo+GPDdxq8yzyK0GnRnEtyTtB46KcHJ+g/8Ar93Zx6HoMM9tp0Jt2mRk+1O5MmSMAjJJ25wcZH40mU2Y2neFYNKZbnU3juLxeRarzFGf9o/xkenT69atXly8zMzksz9SaFmFxEH3AkjPB7VG6bmz7UIhu4+GMiMEr9DUkjBYiTgUMywxZZgFUc0+LRvtkLXusCWGwXlLUnY82CD83dV5HHBOOcAcjYWMCxsrnV9Zlh06DzQjAyTHiNMgdW6flz7V06SWPh8SfY2FxesuGuD0X1CDsPfv+lUrjW7mUmGNvKgUbUhj4RR0AGPas2aTjPU1rCnd2ZlOpZaFi+1svdrJLGxn8hSshbOfmIIA7fdBPrWPcX5mjiP3ZZHYN6gZwKt3Nt5kEVypkLIxjKgfLyMjP5NUuieHnvXN1cnyYVbmRjjnngZ6n27dT2y3ZNoF7yQzT9Hk1K8JZvLiUfvZWHCj/H2q5qOrQwo+l6fI628SlOvLHvnHUn9OnFXL++g8tLXT8LbqcqVP3/8Aaz3z61kvDGsyhUADckgd6i9y9iK3aTyJwwO5tpbjsrKx/QGpUC3K+Y0i29qHEZnYjBY9FUd2OenbqfdXkjhtpby5ZUskDK2fvTnH3F/qe1VbqWfVNOWaWJYsENbwpwIlByoH6Ek8k9ahvUpbajbvU41uHs7cNFaAkpEvGGGclj1LHjJ7dBWvoGjPqlo0880cFtC20EjG7jnH04/GuOjuI3kup5csWupPmjXoAx6fp9Ku20uszrEwmWC03mSFVRl49sjnoB3+poVrag3bZXO8tdP0rT4ktrZwlvnc0shHI9iR16fgfpWFNcWZmlVJHYBiMqjEkf8AAR1rInndLR457qdkwQFEuOD2x/n9ay/7SnnBSJmCgYByCT+OKh8tyo3a1NO5j06ZyjsyYXOSu0/iGwc1UiTTSoRLSdpem93QE89vT/PNU1OCckE+9XbGFp7lQGVDtOGZgig9sntzS6l20LTafbJB58whh+baIzcsWyOucRnnp0/Oqpe2jz5GfXLxk/qHH8vwqtqMzza1MjFJDHtX5GGxSQMgHHXpk8ngZzUZDxAZdcnuCD/I0Oy2BeZoJe8ENCoOPvRnBz9G3VFDczQzbhO7rk/JJtwR2zgA/wAqoQyyXUpSHaW7s8iqi/VvX2GasXcSQOVimWVQPvgYz+dAFpb6UMW83JxjOagu8anctPdolxIxG55VyTjGOvpgVXUbBuY5yewzipTcKozkY9aLjHXEclxn/SZkY9drd/fPWqf2O9iKlLwvg5YOoA/DHP8AKtAMuPvDngZ6/hTmU4AJUDPGTilZ9AIrdJFbMpRu5IzwKf5MDamw81zN9lhMi7CoQktxg9TyDnoc/iXQ3G2GWKFkLMpAbBPb16Us91c2uuvcWxjuIWhSM+fKzMu0k4VuBj5vQDuByaqwr6lw6U7zNMimC3Cgbpm29O+OeTUa28CH97dBsdfJiOD+ZFZ15FJqWoJLcv8AuhIC3luVk2DqMjj3H863LjVEwq21nAqA9GhByPz+n/16mwyutxZxNzFK3HGZNv6AZ/Wqz3JIC7yFznaDx+tJdIt2+SDFkjmHAIHccgj9Ks/8I5ZPotzqCHUJ3hRiUkm+RSBk5KqBwCDyRnHvijZXEUGuIt+1cuSfuqwz+pFW1sL4qWSCTB5+f5B+ZpFc20DW0GEjxgqnAb/ex1+pqM3Mk+0MzP7E5Ap2B3JyhjG15Yg/PG7P8sinwi3ZE80qGU5JRC+76kkVWaNZAPnIKtwY8DPt3qRU2/KPr1zTA0L26tdRsFsryyS5gQs0TGRojGWbJKhW4PPXr1HQ4qrHsgjSJTIQh73UvI9CN2D+Oc1EqcZP86UyDggAijQRds7uCwd2tbSzgeQ5ZorcKW+pGM1dOuzRtC8sBbcCCwYj8geP1rFDuTgDjtxU8UX2h1V2IbPyknpRoLY6y2dJytyjsMjblWyMfSuW1u0OnaxPsEkkM5Eyb49p54IHPI4HP+1XSWETW8CoQCM5DAY/OsTxjJqJs4pPIeS3tZfMMkIBxHtwdwznjOcnjAz2JE6Ior6X9u3SPbpgH5WBfJ+uM/0rTNq88QNyZwxG4qBkDvjpzXNR7ANyM3HQq3H6U5rqYKWldkQdSX6fWqsI6eysbaOR2E74B4z8uD9e9aG6J02mSMn0LCuItv8ASy32dvP2nB8tc88Z/mPz960bfSr6Zgoj8tcZ3ykAD6gZP6UtANe4imikLOJfI6jy5MKPyGf0qoZrIOfO81GOCVbDH6Hg0h0XUYGia3nspRn5t0kigfgUqVtO/tBpYjcwRSxMFbyo3kOcA4ywQjgg9MUXQWKd5dwxhfskjKM4YABT+mK4nxzrT2WjNArnzrzMa55IT+I9MHggdvvZ7V6N/YOmQRk3FxcPtHJMqop+o25/8erzD4uWFtYjRjbJMBJ5xLSyFsj93jAJ46+gqZS00KitTzOiiisTYKKKKACiiigAooooAKKKKACiiigAooooAKKKKACvqGy02O30yzstMUCa2hWJkc4E6qoAIxwH4GeADzn2+Xq+o47Z1VpJ7jc56JbqcD6O2D/47VRREhqNkcgqw4ZXBBB9CDTXAwdwBGc89jVe8glvwrXN3cLMo2lodqkr6EkHOPXrVB9CgcgS3F5KoGwh5+o564A9feq0JNISxF9gdWf0ByfyoaKdcyKtwQTgIwEYJHpuAP61LFLLBCIklYIOioqoB+CgD9Kg43MSp3HkmgCvczX8KZjsld8ZANymO3cE+vp2NVI49eut7G6sbVtvyxhDJnp3OMd/atHIHGBzSbx90ED6UahcbeWwuI4/MjkjBH8EzxlvckN/Liqo0mwikSUWcLuBjfIu9uv95sk1aN8lqhMqxyRk/OCvzAeq47/z96S4uolmSOOVzHMR5Lsv3uvBxnB/n1FFgTH7gwCscY6e1RkfOMbcd80yeJynyuQ2etNSMq3zEN70tAHXALxlUYIxHDYzioVLIgV2BPrjrVgEbiSVx04pGBDfIy7fpQMrHrgcn29Kd5m0kMME9qkm2Roc4OOSTUIkQgNjaD7UxCsQ+QeeOlRtGAMBAB6etTHYex474xTWVQAzMQKQyEgrC8Q/1TjDJ2I+ntUDqyMotfnJHzQuwH4qxP6H8znFTmeFyQrB2B5VDk5+gpMNjdFbyNnjnC4/Mg0XAqQXUF0CY3+ZfvI3DL+FK+0HDMAT6mry2/nAfbkDKgxGquSy564PGPpgio49Ms0ZnaOSTvtlkJ/ligCmUHsKoT3engtFLcQE9Cu4H9K1JrC2cqDbINvIHJH6k0kcMcP+qjRCB/AoH8qAKUaSToWgt5nXqHYBF/NiBTZ7XUYLaGaS3ihSf/Vl5d2fX7oxnnpkGrsoEg2mVlOe2P8AClE7xRsgk3Bl2uMcOORgjoev+GKAMN7e5JINyqn+6kQH8yaXyYQ0bOhlKD5hI7FWPqVGB+FbkVhZ3CFYLlLSXdxFdbniPPZx8yjr1z9ayYke4ikkiiaSGN9hmQHZnsMkAgn0NHqMV9SulQxQyC2jOcpbIIlP12gZ/GqMjOX3li7Y6k5q00R9PzqNo1B5HX0p3EQfadpyiKjlSGK5GRjB6e2atQaxeRxPGs5CP94SRpJn/vsH0H1qtJFk5UDPtTCABgg9fSndisdDZavNcQiCSeKIKcky/Om08EAEMVB+XgccdgKq3/g6zkY3FlfQRoq4GGDIxHXJUcZGOQOvbvWQRwAG24ORjr+Yp0N9LZzebA8kUmMbkYrn646ijfcNjOuibY+UYmKoxKOVwWUj/wDV39aSXeVI+8cHBXkGu/i1fRtVszbXvAkXY6zjex5yMP14PSsjXdCNgE1C12XFkUIeLAWRQR1OeG+owfbvVq6IumzkLm8ne3tGkYqyRxqGzyAFAGPy61fsNakkvIbq+jXVVjTyPJu3JJQnqGzkMOcH/wCtVY2brpMMYiYOmRtYYYYJ3cexB/WqMXlfeKkt0yKi+lmXbsdqdIsdX82fQJzmP72nXDDzo/8AdOSJB6d/qawpo5raUwzQTQuCeJImTp1xke9ZsU0kMqFHZWGCrocMPoa6ePxheXemy2GqRLqMRX928mfMiYDhsgjd+nU9c1cZNbEuKZjqz4ZiSKXzACofLHJ46g5FTW9vdXMxhgtmuQVBU2+JWPGSNq5PH9O3QRNFtfcBjHy89ODjmtYyTMnFo0NDuLiw1E3VvcyIyI7rH1V9iM21u5B2+ufSn+JfE83iTTVtr/SoZ5l5S6gYpJHgHCnJ+YHPrUWngm8AxwLe4YkenkuP61kt/eDED0pOKbY1JpI1hpKXtiLvSZXmVU3SwT4SeL146MPcflWWAVXDBlbuGBBH4UtvLLBKs0MrRSr0dTgitRdT0uc7tYt2UsCDdwNhwSepXBBpq8V3J0ZkAAcinbigGTXQt4Qubi3Nxo9zFqSAEsiYSVf+Ak8/gfwrnZI5FYpIrI6HDK6kFT6EHkU1JMHFoTfH3YUUeUp5wtFVcR7Jo/hKO5UT6pIpQthbeGTJYgjgt0+oGR710d5ew6ZZG009Y4pMcRpxsHuex9q5PT9cnvJ5tOiCMrwuI1BwyyBCc56KBgdDnrVezvYpUcxYOOVwOCD396ycddTS9lZC6lqRefEnzMfl3EmsmW7CKS5JJ6Cq+rtJ5oYdjkgdcVnOJLx1G3btOQO5qb3Go6XJZbuW4JCrtHcDnH41v2ksnkeXyW2ZUZyKyIEeST7PZwG4mOQ4XJIwRn5cZ6H8OK7HSvDUkS+dqMzW0OBiLcGkbvyeg/DORTW1gdjko9O1HVb54LW0aVt3O3hU/wB4np0rr5/CWl2V1FeX00t3OEQG3YgRblUD0zt46d/0p+oeJYNHtvs1nAlsvJCqQxb3PHX/ABrj73xRJexTYQq0v3nySfr7n61UU7CbvsbOu+NJIy1vajCAbcJwo7YGMcVyU+p3F1PvdiSPXms+WTcwBOc96sxhMs6jIRTnPT604iasdtpk/m2ETHuoBxjrj2p01w8dwkcMEkzOcDYCSOw4A5PtU2jaXdzrbpDbOLea3R1ZvuoxRScnPTcfr6A4re86HRLV7awlMlzIf31wRjPsPT269aJbiM17QaLZPq2poJruPaVtlYMluGIUM3ZmyR7DtnGayby+luXeR5GbnqTzT9XNxPpl4kbEu0TAZ53HGeaoxsJrdZVOcrux9aqPUiQsGSzfzqUgHK/hVe3zvOM5JxtAyT7CtdYIdPLG8RZLzgrbt92Pvl/U+i/ieoFac1jNxuOtoVt9Of7XuVZirxIpwzYzznsvJGep5x61nXd/NcjZv+VRhUUYRR6Adv8AOcmi7u5JZHdnJdjuLMck1QX935hZgF5Yk9hUN3d2aRVlZGja+VMHMYAVHdVUdFXOQPwGKVlWVXkeQw2kJ/eyDqT/AHV9+n0/IGvAjrdTqm8ROI2LoucZjHC54J4/DvV6a6S20+G2uQFjciNUPIXIPX1J7k9Sah+RdupjTRSeIr9dhWLToBsSJOhA5x/j9a0sS3BMUKGSZvTnb7n8j+VQ3V3Bo1iI7UFiV2JCigk7up6j/wDWRVNLqWGzJkkVQyYZ24GOcggdevfNTawbjG0tDqH7sz5KZNwrKoJHGOOfXOf51rfYlt7dnRN8wUYLHJ9AMmuJh1e5s1e3gmWSMEsshQg/Mc85PufarQ1m9kCsZNuByEJwffr1+mOlJ6rcaVtkaVxYaoVJcKVbooIOT6U6PQrkg7jEoznDZrNfU7+RCpuGUnowGSP6VWkM9xkXbtKg5GJWH14GAO9S0u5ombjWdtAwSeRN3Y7hxikEltAxliuHDxAsWXqTjgLxgnn9KxPs9vziCMjOcEZzxirCyseORjoRximrJ3JfM9jIg1KHbLNKjqWdpDwSeT+fcCr0NyZxGY7O4IboWTYD+LYq0u/++QPQHrUqKSwHHJ9eKkogAm3BCsMR6ZkkyoP/AAHP6U6Ky1O6kVY4bU5OM+Y5A9zhKsLbmdiFALA92AH61bFu9hLGxnwjkqXhcDHynuQeOOcZ4zVKzeopXtoZl7Y3lrMsQvbYvsBkxCxC5HGCSM/kO3rxDBaMpJluXfPZRtA/mf1ojuvtV7dyONrecUwcFhgADJHU4HXvVlvuYxmp2ZS1Wo0QRHO5dwPZmLD8jxTgkacpGgPqqgGnQLG8mJZfLTGc4z+HFNkCiRkXJjHRz0P4dqTYJIl3Ajk84oBqMIEYkFsZ+XDdqN20EhRk+gpgShzuwUIAH3uMfSnGTK7AxHtmofMbGAoPuaC6pkk9RRcLEyncyjfjB7Gt3TUkm0rV7KBt63FtyM4weVAz77vrxXLidHG4OoUdecVu+HTbzzTI87SRyrsYRLuUjnIJwQOcY/nT3Vgempkq7SoHC4BHApYmkWPLMgx1OcUltDfxJsGmSuE+Te80SDjgHDMCc4z071ejh1OQAxjTo1PVmldiv1AQD/x6jTcRVSbcpKqxIHHynBzU6swwXXBpLlLhmKLqIbjloINnP/AmbNRJYW7HbcS6hcA9VacIP/Iaqf1p3ESNOBy7BVP94iq8mpWkIIadFbGQHO3P0z1/Cuj0t4AEtFsITCB95kV2BBzkswLHn1NSXCS6ddy3H2iGzhnjVvN8jLb+hUEFQcYz0/iH1pNtArHOx3kzwmSO0uMbgFdozGGz2XdjNXczow22gkk6lWvIUx7HBbn/ABqhc3c80sn2y4ZwzFo13EAp/D1xk8du5Na2lw6bZwrdEveTEbgfKIRTjsCMD609bBoXrZfEbjDR6daoMBcyPK2Md+FFaUNxdWENwupXcM0E6GNo1tsJz3IJbPGRzx/Ksue81CdUeKLykJwGMqjJPQYJ61RmsL5lae6LKw5Cbwd34Cl6sZVg0vw9YXUktrHcsS+4RtJ+7HP9309qvu1rcN9o/sqzklQYR5UVmXHoCP5VmwMI5wJIyVXn3rdt7I3sZkM0g3KRheB+X+c0NJIWtx1u+pTuA0iovLYUCnNe265RWAkPMjBtuSO3ToadFo+xm3XDbf7qgKcenFR3ekWsVs0quIjGCxZiSMY79aV0MvW97FPJ5e4bgoJXOcA+tYXiFpdO1JNaEUjWqBVvArH5xnCsR04z27U7TLO+W98+dkET9Zd2MjHHbnoOP5V0sUmy4jVo1mhJxLHJyjDtkHg4OD0pvyEvMwV12K/nEFjGZHYAklvlAPfjrXn/AMX455I9JuCWkhDSoXC/KrEIQM+pwfyNd5d2Fvp/iea0lXbbTASWyp8qqO6nnjsccfe9K5P4rRSReEbJVDLbi9UKrnLD5Hx+HXvUy1RUfiPHaKKKxNgooooAKKKKACiiigAooooAKKKKACiiigAooooAfFFJPMkMMbSSuwVEQZLE8AADqa+qCWYfNjNfNXhcZ8W6KP8Ap+g/9GLX0uwUcAn6CrgZzKjoVYmq8wmVCYY1Zv8AabFaDxlhgVVeNh/FgiqZKKsck+WEojxjgqefy/8Ar03z/nwUcbvunHBqcHcBlTn0b/61IUyfug4/SkMrTQrMPnLj/dOKdGiIoVQv5VMyEnBJH0prlMhRyx6DufpQAm0H0xVSezh2NDtUxyZLR44J9eOh61O0wjZQ0cu5zhRtPzduD0696WCTzi4AVChwyu+SfoVyP1H0oAzg8+nw75WmuLQdWKZkix64+8PfrV22uI7m1WePIRhlSwxxVuaFGtmWOV1kI+V1UfKfoc5rnR4PsmjP2ieedySTubamT6KOBRfuBsrHGsYVQqr1AUYGaicFCAwYZ+7kEA/jS6fpFnpTE2KGF2XazCRiSPxPtVllLZLOSeuTSuMrhGZ2Rk2nGQWPB/EZrOk+2oWMVoh5J3eZk9cdAP8APetXYM8A/U00n2OD04PNMRU0+K4Mqzzywtlf9UIcfjyc+tXt8yHKTGLBHEahc+x9aQTyDYJVLxxklYyxGOeuR/XI9qZBBLPIcZkO4kBFOQOwPuPY+nToAYjhux4zkcd6D1GVB9T0pwAZQVYn0xTGyBgjvSAOSueMehqPKoeM/nT2UEdqi8tc4zz9aAIyuTwO9RmLJzVkA/hTe5BBoApsnfFGwqQSKsHAzwahLEEkniiwDHjWTBXAPt3q1puo3WmmRI41kgcYkikGVYf5/wAmq+do4OR7inefGU4UDHXmmBbbTrDWiz2UyWl2QcWLrhWPYI+eM+n6Cufu7KezmMVxFJDIOqSjB/8Ar9e1aE0tu8Z3Ovpg8/nTB4lgEJsmEV9Gx/1JRpCD0yu05B/yMUWAySo9KaUx2/GtG60u9lPn2DxQWrAEJetiX/vnJOPYjPuahl00pvUasjnHylLbnPoQSAPqCaQyg0fA+Xj2phjGCX6fStJtPh89WE1y0XGUd1GT35UDimNp9pjc0Ckr0zIzZ49yaLisZimON9xkXg5255rutAv5poJWmURwBMYlXG7PAGMdOPxrnYZzDHJFFviic/MqZRW/AUx5hHkI8uDyVGcU02JxuXfEXhnT7uGI6en2e62/wFiJOnKpgn15GOnQ9a4h4pbWVo7uMpKT8r9mHSujkupiyt5wcDkJKNy5yDnB+g/KrJnt9bn+z6lBFE7AKsikLke/GM//AKvQU3ZhaxyhhcFWO0qw4IYEf/W/GmxxqZGJJAXk4OOBz+tdk3gxLeIXGnXKuAMsk5DI3HGD269/XtXNSRyWuoCF08ifB+V+Ezjsen/16VmMZb3N1pd20iZimBw25c5xxgj/APVXV3epaNqWlWU+pQG1uLneouIYcsSgAOcfeHIwDnjIGOtcTcPLcXDvJjcxLfKAByc8Y4rUnkf/AIR/SImwMLM/vkykfyUVS3E0bOlaJdzXc7ad5epQS2sqK8EqqRuXjKsQVPbB98ZrnLmCS1uHtpo2jmjOGVhgqfcUltutbhLm3leOdDkSI5Vh+VbS+I4LqMW/iKzN7DnC3cY23MXuD0cZ7Ed+OlWpNambSZgr8pxniiRd6YAOM1vXHha6eGS80eWHU7AZO6A4kQf7UR+YH2GfwrBTBQsGDDqOetaRkpGbi0WrDUbnT7lLi3kaORe4xz7H1rsbTxPpPiDbZ+ILVd2wIt2OGU/73UeuDkepFcEWIzhaTedxBBHTpRKKYRk0ein4e28jF4ddXym5TNoxO3tyDg/hRXnu9uxP50VHLLuXzLsdhomqGPXrK5YfdmTJ28kZAIP4Vo6SptoprVyPNgkaKRQ2cMpwR+YrnLNHkuo8cAsFBPrXoWr6ah8UzfZoZ5PtarKDEjFSQApxgY/hBJ/2quWiXzH0Odkj8w+Y2c9lrS0Pwxeao4kA8i1HDSsOvsvr/nvxXQQ6PZWpEuo4l6EWsf8A7O2ee3A446kVDe6zcTOYcGK3XhI4CFCjsMccfnUKLYnK2hpull4e0e8exiDz2qb5WflicAnGenA4A7+lcbqPiKeb7sjckH5jzg8/hWrpt4H1mO1QNsmDB8jvjP49q87W68yziIPLIqnirT5XYm3NqT6vdtNN5u8sMY9h9KzUk28Z7806RSVUAd60dD8LX2vzYgASBWw8pzj6cVNy0rGXGHkf5eR2OK7XQPCMlzAt3qe6308gMF6PN349F9/y9R0Np4e0jwttllVLy6HMYYYSM+pGTkj8vypt5qs965aRz1yOflX6U15CbLl9q3mILaBNltGAscSjAAHSs9WL5BGMUxT82ck9iaUvgHBOKozJflC+o7g96yrK0MkzWFqm54sg9gqD+Jj2GO5rRt4JruTy1by4RkyTt92MDt7t6D/Jj1bxHBFA1ppqIkDYZpEXDOR3Y8ZPHX/9YYWuXozYaJpt/Jbt9o1GGEyGbj5FyA2wHpgE89fpXMC6nurkyyyFmcdye1JbSrIWEoYrMrRSgEjcrDaRSw2xiysjBnQ7WI6Eg0JDYq5V+Tnn8SamijgeR2usLZp/x8OegU9vcnoB70+2gMjvIwkEEf33QZJ/2V9WP6dT705byLVJLPTrS2kgtbdxIyODlcepPVj3JyTUNjSOhzCkaGGFYo0UJFGvRVHT8axdbie5sSFOCJEYN6EMD17dDVi8v44ImlkbCKDxWRLq1zqBWS3gdYI12IsigEncSSRjPX346c9aVuoXKN1p9zqF+9xFdbFLkhQmcdOnpwKzruxHmGM7pfLGc787V79e/wDjXSWyTfvJpWJVFJbCkk/Qd/wrIu7uCWC5uVEsboCvlE7uc/xDjHOMghh19KL6icbLQzYoiS0gC7DjB35wMdPQcDtU4ULgqwI9fSq0G9Y1V3LAfdz1xUwZN3DBeeBnJqDZKxaUoRzktTsbV2+tVWu7eHPmTxqf9psU9byFwxRt+ODs5IoDYsZQffIUetNE8OflcHt1qk9vBJGA+n3Em5gVkkhOevZmHI49x+dKbF4n8wacBuOQssuGHvsY/wCe1KwNo0dwOQMCnL24PHfHFVlN0Ysn7PGxJAQuWIx34GPXv27VOCwUAuwcdV2YB/OnYTZYiYqQVGT9KnSRjLCyrumViVJ6D5TyeR3wO/U8VlCO7aQk3YVf7qx/1z/SpYoiHLSSPKRzh8ADtjjGetNNDa0KcMUlvf36ySBv9JdeBjkHB6jPUHrVlZQWCBhuPQA81Eul2aAYjfAJP+tfHPtnFXIooIMlYowSOqjB7dx1/GpQ3foRFWB2GNiw6gjFDtdKq+Xa7wc5zKoIx7E1PkbuOacPU4oEkxkKSPOoYW8aYBLSO5Gfoq5rQMNlZlC8Xn748llO0bu/BJPHr3/A1URdz/u1LMfStG4cLpTROu2aGRSMr94MD/LZ+oo0G7maxh2hRG7MOfvcnjpxge9TecDHsMMSncW4zx04znkD+p9TUKkkZAHPOaR9wHXHrkUbAOjlFr/qI4o2ZvvJGobP+9jNPlvLib/XTSybegZiQPpVZVBHLH8O9OARVwOe/wB7mj0AnEhH3ckntTfMZn2lwCPXtUSKoGF9OlXLeG38tWnuDg5ygTnP157e1O7AjHyswBDY7jpVm2nt05ZZN/c8Y/IimRQiZ22s8UajO8gt/If4VCIyF3EN6Z6f56UJiNGHVGF8JhEDgYXB6j/GtK6kOpaO7RwlDFJvHyk/ewOo7k4AGOSa56OMhgF5+grqtOsj9j8i4nKJIccJhgSe3XnPQ4ND2sLqcZcJ54ML7trc/KeR71c0eaKJiLuSR4F/1e0/fx61asrfTvLna7mEfkStC2+TbtIOME5wfxFQGz0CR/8AQp5nkByRaxvKv4iMUr2HYkN1GjN5OoXq7eg3YAHpgU8XgaEM8pZlIwNxzn61at/Dqtbl1WJmYfL9oZ1IPTkAnH5ZpsGjalBC4iWyWZiV3+cxUL6geXnPtkfWnzA0Q5sUO7zZTMOcEcfrzVuLV4H2+V5rEHDbFyPpnjHWmHw5Isoea93gDgRwbSD/ALxY5/IVKuk6bLDsY3MkmfmeSXa+f+AgD9M/WhMTJZtVYQ5iifPQ9DgfhmqVxcSSMjGQW6qeHuJBtzzyRxzx05rej0+O1Kv5Fs0D58hxFyFBxhmYkluOue/FX/N+XbngdB6UnLsNIx5tIuSqPcCZmABU7dufwJB/WpES4tmKxWs1wPul2lQKfcZOf0rQ83dn5aUEAelK7HZGddWc19aqXAtrqIloJEO8qSMckjGMcYx/IVwPxWtZrfwXb/aZzLKdRTBB+Xb5cnsOf8816hzXI/Ey0+1eAdRxb+dLF5cqYTcUw67mHphd2T6Z7ZpO7Q1ufO9FFFZGoUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAdT8Odv/Ceabv3bf3uSucj90/I+nWvelm8qVYLiRN7rujdW+WQeoz0PsfQ9eteR/B/S4rnWb7UZCC1nEqIhXnc5PzA9sBSPcMfx9iYRSxhX2NGG3BWwVB9hWsNjKe5C777gbJ1K4+4gJb68VE9oZnGFuFBP3mmZQOO4Bz+lXtpYBcnZjjBp5bC4IzTEYy2VxGzbXQEjAdpHf9Dj+dQCy1JpSJNSRYN2B5Fvsfb2OWLAY+hrdcAjI4qp5u4uFVjtOOVxk1OozDj02G6uZEvLmeae2bZJB55VQ2OGwuG56gZxU66bZWMsl5FbsJOTuyzsMjBA5J6VoSGaaELIsaupypHzlPxwDj1HQ4FUVvGhvpbe8ZFYOAhVTiQf3lPofTqOQaNwC3hW9iS5lYmJ1BSE9AP9odz7dB79auKq4CgDaOw4ApREY2JQAo5ycdjTiFz1pAAXtQVI78elJk9qAGPViT70DArnmo3kA+UuoJ4GT1qUg4PIqu9pAZkmaPfImAGJPHOaBCLHKrkMFb3HFP2ALjIA9zimNdxhijJMDnAxExB+hApfOIChFLBjjJHQe4JoAa8LEDAH51EztE+7lXXlWXqD7VZDDklc++KZL0zjj1PamA37ZBqF+FmuUt7kIQfLQDzj2Lj19wPr0pbmJre58h1xIFDfKQQQe49qz5bmzhkVpLmASZwqj5mz0xgc0yTUYpOI7S8umwRhYXQc+jHAHTrmh+YF7YGHCkf0puzHbJqhNHraMPIMJicbh9okCvH/ALLbQQT9P0pDbao7fNqkaIO0dvlvzLY/Slddx2NDOwYwPpVOe7ghJ8yQIQMkswAA9+ajXRYGYNPcXlyQdw82cgfkuBT00qxiztsbbg7gxjDEH2J5FO6EJEZruMSwxvJEx4kVSUOOOGPH61K2nyAP5lzbwkcDL56+yg03YBkIAuOwGP5VH8roQGViDzg9KBlrTPD63credq0cowSIY/3bE+24H9M1Ru7SG2ufsd1Yz28oGf3kxbcD0IK4H5VLnDg4YY5AHGK1LbWRJCLW/jS7thnllzIp/vBu59+vvRYDCGmWIOfIRs92G7+dOjRDHi3ChT/dFba6IskMt1YXn2iIYIjl+V0GOh7E59QPrWK2WIZW2nrtC4/MetKwCLGVGS2eOnFKoUkbl49TzSkjOGODjJwOlBRSuAze3NDAJI4t3Hb8KhZQRnJHoaGwG+9k1GcjpQMa8Z7Emoym0Yxmp1VnXI7UjxSE8UAVWTcMMo5NQtbRhCAoI9COKueUOd7gUhVQOGXHvQBnI91YzLPaSNEyngKcD6HFbkOoafrFp9n1KC2M59zG2f7wxnJ+mPpWROeAqKWJ/iJwKqTwLnI4YHhgeQfUU0InvfDF5EhbTp1vYFywTDKQehwO/QVkeZP5cVvcqUmt1Y+Wy42gsW/HqTn3rorbxbqFpbJBMvnIilfNXIYjBALAHDEcdfT8a3rvRdJ8V2DXEKGFwcrcKCr8+oPUH0/WmL1PPN2TlT15qN3bdVjUtPu9Eu/Jvx50SfL5kL4PTjqMj6Ee1U1lDgsmWTJ5/wDrdqpSsJo0tN1i/wBLlMtlctC5wWZQCTjp1rofN8N+JYVN9cppGrBcvcRKEhmbgZKnAJ9cYPXOQM1xkbZbHUU/zCrHB4/z/ifzpuzEro1NY0S90OSMXSq0M3+puIm3RyfQ+vseazkYZOODn0q5pOs3Gly74UhkjOd1vcDdGwOQRyflBzzjrW0ugWniCym1Hw/mOVDtl02R9zI3fY2eR6BsE8/SqUrfEQ432Oe259KKryXDRSvHJBKjoxVlZGBBHUEY60VpdEcjPc9E8LWHhtPtl1Obu8jTbsUqERsHIX3I+tWbzxEQjLEnkxnk4wS/9T+Nc0dWSZjtYAhck+1Upr6J4nKS/IBgMO59qfJ3E5voLPr/AJlxJHIWVf4ccZ+p/P8AyapW2qCS+bdxlcbs5zWTd3u67K/wMPxxnv8AiKW0mjb5YkYOv8VXdCS6nS6LeSHVrKYIzuJxFnHQZx9etcvPumneNjskjldGJzsUhjwCeK6Hw080eq20SgFxOCAOpGRk4rvvI0nRGkeKOBbhwd6JEodz1+dgM9T3/Tmspxd00aJ2RwuieDze28V/qL/ZrEElzIpDSegVevPPP0wDmt7UNUSGzXTtPsxbaaq4IDYL+zfX09+Saj1TV57oBSwCryAONv0x9cfn6nPPy3BwqhsgnjPTP9apRM5Tvojbe9+36da3MhJwTFwD25Ht0YflUJIx02gdagscLpNuRHjF5JHngDLIpx/47n8KtStFGoRmA3naPVj6UpfEyrDA4YHLjC9cHp9ac48qIXN45t7XsuP3sn0H8I9zyew71pppL6TbyahdFSzMqxI2MJnJBPHzMPToDk81w+v6jcX94yPK0gB6senNK9wtqaWp6/HcpFaWsYhtFPEa55PqSep96yNzS3LRjlT90DFNtYlcorElQOW74q55fleWEBeVjhFXkknsB6mm2UlqTRPHApDnBPCkmtDSNBuE05JtQV7KxT5MyfLJIBwAqnkZA6kdORVyTT9P0KFL7U0E+rOCYIdwKQHHDHnkjrnnnp61yl3r2o6qFtpryWW3Thd7cn3Pqfc8+9RzblWudTdazF9otrGz2pEAYVjjJCgHoT6nO0k+xou7lNoaMAMyjkDFcdpJxrNrnJVHDYHXFdJJBJLa/wCrlFrHnzLtVynfGDwCODk9O1GlrikrHN6kt3qF9IscbGIAKrbxg5HXHfJ9PSrq2uohFgW7jhRcDKMCT+h7/wAu+ao32u/ZZDFZ3RdVUxuhiKAnPTAJHHrn8BWfHrt8u5UKhSOhQHFLmXclU+p1kFutsnmzahdScdHYYH4AVXnudNRJdrxSGX7/AApJ9s4ziuWaee4bMsrt7buPyqRBjp+lLmNOS+5qPcWf3YtNs0UEHiIHOPrx+nc+tMmvHuWBkWH5eFxEox+QqomMjJYLnkgZ4p2ccEVJViyJ5m+YyybsEbgx4z170m8+WELsVB4BPFRAnOAOfrTipzmi4DlQvKq5X5iBy2P1qxesst/IY8eWmETtwKfYLFHIbqeSIxxqzCN1ySQO2PTqM9xVK3LR26q5y2Ouc02CJDjBwRkdu5pUbIII696jY56UqsRxUjJcgDk4x2o3ZPFNKhlJKHNKI5OAVYegxQBLuUDLdKTcu7GKY0UvUgge5qRLZmjL71UKMklqEBI0EkcSSPC6o/3XKkBvoe9MJHODxVc3MG9F+0BtxwpVWbJ9OAetQpfK77TBcpxnLwOBj64x+tAGnayKlwpLELwCc4rY1e4jksreOM8oW34Jwc42+2eufqKwobySCQE6ZOBjnzXVAfphif0ovnkvnin81oZlZyyIcjBXCgnvyM8jjJ69KelhEqrI/EYXIGTxkAVEz734waECkMt1Ndyr/CsUoiX6kYOT78VDPCCR5FxeRR7iQpmUkf8AAtuccetAE6gtnlVCj/IqHzIVwTLHg9DuHP8An+tPgSC3IItYpJcY8ybMh/8AHjgH3GD71L/oplMn9m2bg4yjoWTI7hScZ/xoC5TlvoLVYzJLzLny1AJ3Y9K0bEX1y8kjaTqDQxArnydvzf3ckcdfrV+LWxFhBZxRIMAeV8uB0xx0qfXPOhs4763JbYgm+98hAYcE5yBg9fehXW4mxf7I1ORCxXThuIIXzZCU+uF2n8Klfw7e3CFZr21gTrthtmYj8Wf+laltcNIu7YSAcZGMH9asvIykbecsBjaT/wDqpNPuPQzbXQre1dfnupmUMTKZduSVx0XGMZ4xj8adHp1ok6t9llLKdytLM8mD6gFjitQByMlgPYDFBQb9xbp2/wA/jRcDOZLNblz5Vutw5GW2DccDA569B+lOMks5bZdIUHBXg4q1LaW7szSQo5bglhniq406ySN44raOIP8Ae8tAufyoVuwir50Clt94XjHG3oSfpkfyqaC9t1by1mLAZP0Huar3OiQCNmjdkbHAYiqw0KWSPKyDgnGG6+4I6VV11EaNzqMSqSCTnlSV4NZU16s5ZmcxqRhSOoPtTpf7Sto3jdBKhGDu+Y/nWY0avGXdiDnovUfpihNdAOjur+GHQJVuEeVQokjli5UN3BH+AqtoV9LcNtVxJCVyGJBI7j6iqFhcYtjGZ5AgbATy8ktng56jn8Kt+HbO3tnnt402yWjGLBcPhSAVwRx04PHUGhjR0W7GMjPvS9iOopCD60wbg5yevtUDF3AYCnOBg55NZHil8+E9a4PNhP8A+i2rSdlWJmlby0HLOXx+uay5PEOnxSkNcRIgJBkkkC5x/dHJP6U0mwPmeirF9BHa6hc28M4niilZElC7RIASA2O2etV6xNgooooAKKKKACiiigAooooAKKKKACiiigAooooA674batHpHjCJ55oIbeeGSGWWZtoUY3DkkAHcijn1r3oHPzbVx2zXyvWhp2u6rpO0afqNzboJPM8uOUhC3HJXoeg6jmqUrEyjc+nCSwpC244I4x6V5T4V+K2Atp4j/wBoi+RfxAZFH1GR7cdTXp9rcQ31utxazxzQtnbLE4dTzg4I4PINWmmZtWLBKgdqglYRsFLZLcgBTwKn2lgB0qGeFZ4jG27afQ4oAiOPUE/Sms5jHzgPFkFkYZU+5ppjSGBUZsLGoGS2OnrVe51Kxsh/pF3DEB3dwMClZsZfWGKfc1gxKqNxgd90g/3fUfr+dV1ZJOFPQ8isuPxBp8I822aeVsFla2hdj9cgYouEvNXIv7MtZXLDDx3CtEH5J3YAPJBHGMdaNANUDB5oJHQd6zol1h1IeWwC5IEil3LAEjOMKO2everNuY2jPmbp5P8Anp5pA/74ULj6EtSGTbO5PAHU0iq06EwJJOo6+Shf9Rx/+qrCXjQsJAIY2HIZIVQj8QM96Rr9pWO+WRi3O4kn9aAA2M0aAuY1GcfNICehPRcntj60yCzeRxM8sQgw2EVgJSR0+96nPb09abzuxngD0pGQSIyNkhhg09AIGt1WdWna9iY/dgnby84/3QCR7gke9V7nTrOaUym1jd85/elnA+gJIq1Da21wjw3bzQzRyZhuwS2FIzgjODznr6/jUt5A9gsk0j+ZaL1nCEBck4Dfhjnp7ihiK8EEMK4hhjiXH8CBc/lTyg/vHHvTmUEbkxjrkd6YQWU7B82OM8jNKyGNx+XcUhiGeOKlZkjQM+R9BQ0e4fLn8BigCH7vHHWmMCeAM/hVo2z46gHHGafHb7VG4qzeoHFOwrmdJESCpTI6EEdaih05YifKjEYPVRwK2nQbecZFRhQFAGeBjmjlFcxpg0eQVYkdgKaMD7wwT69q1Z4Nw6c9jVGVFWTBBzQ0NMhhuZ7ecPDvVxzuU9vf29jxRqcceoQs4iW2uQwYyQx4EnT74B9u1POMkq5qrNeW0fMs8akHoz0DGzytYW8Ml5EY4ZSFjlbADEjIyCcqcDPfvzwaFZJQdpB9xzUVvqNlNeqIoUvZRkiMweaMd+MHirtjoUbwzXLXKWBkk/c27KGiU9+jHaCR68c5BpAVYoXYne24fTFMZPmKbfl65PpTbj7bA2B5MhXIOxxhvTHOOvvn2qnN9vkB+e1UEZCjccfyp2GaLSKANoAwOwqubhh25/Oq8MTRKPOuJZj1O3Cf0P8An0qCazWYndvwRjiRh+PBxQInmv4UH7yVE+pxmqz3LupaC2uZUAyXSJto/HpVm0ht0t2ga4exyu0TQR7jIMj5ZACCV4z3JxUNxPa5kiNzGzDpJErEceuRn8aBlWSe9YALbeWCPvSuP5DNUWad2Iku4Uz2RCx/Wp57KZFLO24dnzmqm3Djk59KVx2GSKnkSIzvIXYHcTgjAPAx2Of0FPh13UrG2FtbXMkMa5A2tyBxxn8KR9rcBc+57VWeIjOWHB70Xa2Cy6mtaeIhK5XUIUfLEmRVAJz6r91vyH1q1deDoZbUXmm3qBpQeCAI3yenONv881y5UE1PZ3X2SQSIilgR95QQfqDwaFLuJxfQryQXenXzWt5C0cqE57gj1BHBHuKs71KnaRXZWdzpfiqwW3vNOto5UOIRHIUPv05A5PqOuQK57WvCWp6bdTzWcJubTzG8sQncQp5GQCT0/wA+rJ9TLZudo/PHWl05jFPNKrTROD8ssTFCwJzgkdRkf5wKpC4PmMkiMjD+E8c1qNtj0iyuFgjUtNNE0gYEvt2NyO2N/wCVO97ILDpLWOeR5ZLq4LuSzZkPU8+tFQ+ev9+it+Z9zK8u50kUztKY0BMQ4JbPTr1/CnXlwn2NRuCkHIU9Ov69qLS2ZpmjJVAF3Fm46dvrVeSB3kEcUYPb69ea0b0IWpWYiQg4YjACnGOOvH61teHtMutVuFit1Xrkk8BV9T6DjrW94e8BXB/07V3e2gUblhX/AFhH/soIGM9eeMGugutVs7PTRYaXb+Rbp1OSDn+p9z+lQn2KbSHWlvp3hxHa0H2rUiuPPc/JHkdEH9ep9ayrjUN5YnALHJxxk1mNqBa4Xcxz1AqB28wk4JHUe9aKNjKUmxtzdF5wOdo6k9KJiAiEchehxk/h6ml2I67cA4JOD61o6PoN5qz+aJBb2aHEl04+Uey56n9Pem3bcSVwsLO51HQbqzso83S3Ec8OS2wHkNuYAgfKT371tNHY2HlvGqTaptwbkjIjHcJn+YpNQ1Gz021XTNJJWD/lpL/FK3qT36D+Q4rHin+QI7l2PIHrWUpXdzZRE1SWWXS78Rs7PtSX73JZZEJPPfG7r61gTWzG8cHJP97sa6KSGOS1vEljBzbSgheCcISPxyKz4raa/ntYtGh8+aWPK/3UXpkk9KSY30Klv5iyLDHCXmkbCKOp/KtefVF8LT+VCYJ9Wb5ZmkUgWwYAgKD1yDnP07VuW0dh4VsZJI51u9RZcSXOMqnsoPbP19+mK4bV7MareJfwiUTuhV2diwfaRzz3w6dfSk5PoPYgvL+51F5HlkaSZzyxOSen+cVXtFdbgwhVaViQATjmtbS9N8qQyudxXgnBwK2rDREn1AQ2sM00rFDLNEOIkbk8n+LsPxPblWsx83QTwp4emublpdn7peHmZT857qvt2zVbxXf2tvbGG3lESsWVDHKG3AcZAH3R3zznrWn451s6Zof9naZE9rE5e3IO0Db0PB5zweffPWvJlmYyFiBLkEEnJ5x1z6jr+FQ3cLEwjAA8vLAHv2NSBD1xz71JbifyUlVpAVHBBOR6fSpIozu+Yg+zHFBQinK4xjA6+tSoe1K1tMCoHlcgnPnof5HijypUETAwnexBUS7imO5x9RigCQYAzinrhh2Hc81C8VwWBWSIAnkFSePzqUqxQDzG3Yx8uMfkc0ASDaCeeaUna2MnrioZEYHdCqIS2QpB2ipGU7RmGN3HQe/40BqO/tBbaznUytGHYIJGIXg9RnqcgYx05NVhqUEn+qeSYY5aGJpB+ag1cG4AKDgAdB0FPKngk0MNiqk7yA+VbSvzgBwIyf8AvsjH1NOhnuW4e2hibPKvIzFR74GM/jU6rg559qcSAQADk+1KzHcHecxgRSwxtnvCScf99Ujo02Qbu4RuMFQn49VNKcgH1pCGwAxA9SO9O4iK6sonjVXubqQuWUNJPj9EC9vX0/Coo9PsSVYRLKVPDSM0nP8AwImnXbF1VPMGRKG8s5xgDH59akjmVSAd3PsTTkrExle7TLDFi+4sc9aA5VgcnOcg570EjijcVTGQcUixzsWy7t77mPWmqyPlVILYBI7gfSmCO3kbc6I/YbucH1+tOhijiQLGoVPQU9xE8aKXAdginHzEZqSQWqnbHJIzdMMgA/nUD5yBHtzjkk9Pw70qocDPzEd8UAPwoYgciglTgfwjkc01VwCFA2+3SjAwaAQnOSAP1ragEN9p6W2poz2gyAqkBm49Txx/Xn1GZBbvM6xplmY9q6T+yIF0vy5Fb5WSaQA8nbyRnjHHGaLoTuVbOWKHQFePK+YnDNgnPPUjP86vaPeiS22sfmU7Tx196rQ2trc2N1DbeYqxXMwVXkLlcOwHP0A4rMhFxZzny7mJZDwIzIpJP0zRfuB14YDPJJ7UoclSWwACehrKt49TaJZLrcUYBgg2ox+m4g/jUpa9KBYbNuehmmUfntJx/nip0C5fdumTwenFBlVFJbt3rmtmvXkjiQW1vG23hrptsYOMZxHnuPWov+Efv7wEpqMKKSP3iWzlhyem5sHgDqO/tmnoBr3Os2IG1m38YwF61VTW5JJMJbsU9EBbiqsHgyLcDearqFwccqriJT/3yM/rWtH4f0lAQ1jHOWIJa5zOePQyFiKXMuiCxnTa0LgeSDEpPADyLzn8ahj0y6cNmOQYAJ+QgH6E8H8K6FI0t8RQxrHGBkJGNoH4Cpg5XGRznindgYlnpV3FIpMUgUZ++64/Q5/SrU+l304hk/tEW1zFH5e6OEOrqOVBzg8ZbuevXir0t4iAnOdvVR1qkuvWjXBhLsGJAAZcdfTvQlJhdEyW16YlSbUizDq8MCpn8G3Uh01ZXzLd3rgcFfOKA/8AfG2rIu42kKDHA+bJ6Uz7XAOsiqvYsRzRZhcrTeHdIupFkuLNZ2Tp57tJ/wChE5/GrH9l2McDLa2ltbuRhXjt0yp/Kny3cMCZZjjrx0plrepdRPLGcIpIOaXJ1aC585+NLSWy8aaxFOys7XTy5UAAhzvHA6cMOO1YVdZ8S23/ABB1Ns5yIT/5CSuTrJ7my2CiiikMKKKKACiiigAooooAKKKKACiiigAooooAKKKKACtbQPEmqeGrxrjTbjZvwJY2G5JADnDD8xkYIycEZrJooA+iPCXjC38X2UzwqLO6gz5qS5ZEzkqdwxkHB7cEfQnWurLVBsVtQiikYkGNIuQuSM8k+npXzt4c1248N63BqUC+ZsyskRYqJEPUEj8x1wQDg4r6KbxDZXOgQXunyeedu9Ej5V1HXBONpAzlcZyAO1awbexlJWZC/hu0mtpBeS3k8rOCjtdEAL3G1QBz/KktND0+wlWazsYopUAHmjJfOc5ySe4HStKC5ju7RJEZGVxu+U5FIY3JyjlR6dR+tJx7gmVZLqS3jLKMKMjg4A/+tVd3L/NEDv7kHI/+vVl4pAx3Kp78HmjgDIHOe1ICtJEZo1/ehccELlaSJgjCNEOPUnOaW8ZzGPLkjjYjH7z+lLFue2PmupZgeQMDH0NAyQ+YTgIGB7jjFAUhQOBn2pUJ2jO4+5p4UEfdyPrQAwEBeoOOM5owG64p/k5XoeeuOKRI97bTgHP8qBCY6DNTWs6+TNBPETDKNrqjYOPYik8pUOGYZ9DTmjLxKUhnZHICtGjYPPHIHuKdgIptImtrcz6az3UIG6SJ5U3DPpyAPocdePSqkEqSbgYiijn0Iz6jrmtS0Nzb35Do9pEV/eSOoO70G0Hnk/zqfUbLTJ5pJIBJHIzDc0D43cnJ+ZSM89xnpzTsK5nIscQwDx1G45p3mx43FhjpnPFSzadpnkFEjvi3Jybt1Az7LjNVks7C3k3Lp0TuCCHky7AgjByT14H5U7CuMlvbaNTm4jGeMbsn24pokLRmUQ3Owc7jC6gj1GQMj6VeF6BuxHCGZtxPkoTnuckHnioWubqXapv7hiCTtZ+CT7VQXKMuq+WAFtLpuuSLdzjjOScY6VG8+ott8qyOCeGMqgfXk5xx1x6eoq/IXkUo+XiVs7ZORnrkDmqvlTQxvKXluLdRvkkZvmUk9cHnHXPpkcUrgILObzcX2qIqN92KOMPznoDkHNF/ZWcqiKJ9QVuN0haMr/LNLFGTly6lj0YdMVK4yhH6imIpA21ovliyjl9WnZmJ/IgVltp1g0zSmzQFjkjkjrnvz+tbD2zuCGIx2wOlVjEEYq/WoaRSZASUf5ECDaVGxQvB6jio5Mkksck96n27TjjHtUTbQDxk0ttiirKVUZxVGe62ttAHPQM2D+AxWjIwzjnd7VGQGALYz1GRyKLgRKpKAtjOOgqLzVa6EO1tx55FWCctzkimBwpI54PpSAZIEU5IyR3HNRNNI1tLbxXEkUchBcIeGweMjoamcBl5B9qi8mNTkDB74oGUIl+wwT7pYQpBYLJlcgY+UHGO/A47+1VYh9pG94cKejK39DWvLHFJEySDPHUdqqTQIQpEaB1PEwBDn68/N/n8QDLlUICASc9PTFU3BLHIPFa8kttKmGIikHGDxu96qy2jAAqpA9exqWikyjtGM8fSgxqwJx0qVoWJxt744NRGPB9vY1NirjrdpIpMRleeBvxgZ9z0rpdN8Z/YrWK1mhRokz/qsqy5OT161y7xbVzuzz+NRFc8Y49j3qoycSXFM7n+ydG8bzNLFcR2koBZtsOJCPUjdg9un41keIvD81jFb2sEUUmw43RPw7HJ3FT0JGB1P3RgmsrTtSutNuVntmUOnqOvsfatlNa/tFvKuFiE75OTkqTnoc5PqeP5VpGzd3uZtNHLGC4VirQkEcEFTkUV6LFH4gMKGKefy9o2/uIm47ckZP1NFLXuF4ljSvD+o6xcmSBlisQ3N1ICFBJ5C/3j/wDqJFdNpz6FoQY2VrLe3adLqVV5b2B+6Pbr6+tGs608zfut8FtGo2JnDHGOTjgDjoPSuT1DUHWL9zjAHRRjHtXVZvc5+a2iOh17Xby61OyQsDaS2nmbk4BkDYcHnnGB19a57U7rznjSFpAgz07/AKUK0k9tYyEHZG8yEk93VCB/441EcByrEYx0FWlbQmW9yKG1LzBS52YySRj+tWZMxKRuBxxyamhWaSb7PbW7T3LnConJ/wA+9dJb6ZDoEaXN8I7jVMbki6pB7/7TA/570OaQlFsq6boltBDHfa3ujicZitQCHl9z3Cj/AD2zNeaobwLlVSGPiKGPhEx/X3rKvr+S6kkd2Lyv95zyc/57VTtXwxR2ITtWTldmqjoLcTGWcsSD/KmeZ5QXadu3GMUhDGQtjOT0z09qu6Vo7auXuJ5Db6ZCSZrhhjdgchfU478gVBoX/D6PqUjSBV+zxKxuZHbCgYIxkd/8+lNku9O8OaWdO0kEQN8skrnLy/U9hyePc1X1nxJbxxLpOlR/ZbCMAqrD5pD1yx6/55rlBNNfszMc844HFG4i3eX8t9IFVgsZOCc/nWlpskAs3jlGZDP+7G3+Eqc/T7o9+lUGtBDGgxwWz75qDUHKtFawT7Ll5QWO8/uxgjkDn+I//XqkS2rHSSYha2UTKqXAZXl2l1iIztBA5JJU8duO5AMevaxcaHpn2G3WSASgy7rkMkjgk/3R83IADHA5/Kt4W1K1j8ubW7kWthYtmBDuG+T1IGTkcfj9DXLeLPEM/iLWJLhpFECnEeBgY4HfnoBSneOwR1VzJW8unJL3U55zgyE4P51H85OSzH8aVFyMqpbv7YqQIdx3AE7uAOcj1qNXuaDwgVGXq+cZBzU6bUUclnPb0FN2cAZFP27fmYhcdWJ6UAXLOz8+5i8nzJcnLBkChMDJ5GeOOv6dqkkmSe4kmWNV3HA2qFzjua6KKK00Tww0wkEl7cYERClCM9+QemM9OdvTmuZZkYFUVVX0QdPam9xIccrgEgZpwPHNRjAApeQwJyPrSGStgjril+Uck4pp4ycDjpzTRE7EMdzKP7tFxkwUA7hgmpMkDnpUCsdxO7gdOn86kWQbTuxx1oQgDhhwevT1NLvCg7gcgZ4BP8qVDhjgZGaXzOTztA75oAZ5gUhT8vs2c1Io8w4yR61EkqxvvYx7Qc/O+4H86kbURcQsn2iJYj8oEe1V9B09yKdmBSkVriPTxE4VmV2mLEDHzHAxjkAd+vNXI1CA7iCc8YNVby3uLC8MOHmRY1CCD58A46Y7VYVGQfN5j4JIOFH06nv9P602TGSaLEal8/JuJ4H1pZrV4S24KQDtJz3po1m/t5CINLslIACM0rbuepJC9fxqvcXGr3DhpHs1OOQsTTE/i7D3qLruVqPbAZQF555z/wDWq3Dp95Ku/wAhgmM7pMIMfU1TuLW5WWxa7vr3FwrsIEVYU2gkZ+TGQSO3GK0IbaAPviigEwAG9lG7GemepppBcclgzlljuLaV0++kUyuw/Bc1N/ZciFSJCSRlUCsGPPoRx+P9c1dvLyU2oV51L90XjPP0qpZy2gcfallbJxu3HC8eg5ppCbLbXJlhaDEshjBdw0ojBH5Nx9BTIlRVEiWW1iu4CSNmxwMHkrke+BUV5YvAZHhR44wchiwzjPpUULyvA0b3G2NeQrnqfXFLcFY07Oa5uJtgupFUcZhtggz9CXxU91b3McBlluZnBBUiVgo57YUDNJp8q2di0s9yu1zuVUGD/niqd1ffaJA9u8zSLnBfGMU7MRXTT4rl5Hjgs0BxkyglWOMZwcjPHXFbKacIIFY3DhAORCwjQj3xWO8l2H3SSMDjndjP5GpkvHgTBklbI42YXB+vNFgFub4JKY4YGV8j/WOxx9RV/wDta4jcb4VwcdGBJ/DrWBJIHdmPyAg8nn/JqtudZQ0LsGHOAMfyp37jsXdcmvG0W9NtsWKCQTjbcIrnhXBCdSoXaOOc7uozWxH4gjl8poFzE4BDE84NV4zNdrGZb2WCaRDGWSHzIxnO0sBnoT1x6Zrl9KMtpLPZyQzKbWQqGuI9pKkkr8p6cY4H/wBejmu7MSjbY7kXhmRztlOP7pwMfUVbhl8yLeoOCPlzyDXKR3Vy9x8khYuehPWukspibdI5UMZAwDuyOP5cUNaAiItcQXMk74aLbhxjHToR+vH9eKmE4l3EhwinklTtYeoPQj6VMgyDhick++KZtiSFY/LCxBThAnH+ef1qQEkkQRtJMj+WvJxGW4HPQD+VVGRQrssZI4kQNGQEHOCAcZPsehqbTkul803E5YE4RduML+POc56mrDqRIssmxtnIznK/SgDJEKi6SOaZp3jXcSzFGGT1YAY4B9f/AK1sWEMtsi3MQlkAyMHB44HXHODVK+aRJoyrskLZQBWwAxPy/KOT+B/KpPIPyy3U0aIoMZ4wjgkZypyR0A5J4+tMClcRvaALIZJt5KjzGUDHUe2eOn60llqclhLJHNBsTOSAeRx29a1pNPshHI7YSI/MxY4X/wDVWUyaHBFJM9wJI422yeQC+z/eC5Kj3OB707vsKx5l8V7pLzxNZzRhgpsV+8MH/WSVwldn8S7ixuPEVq2nljCtmoJKsvO9z39iK4ysJbm8dgoooqSgooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACvZPhJrVveaRNoNy2+e1Zp4UeMFfLJHQ+zk5z/AHhjPbxurFhf3WmX0N7ZTvBcwtuSReoP9R2IPBHBpp2E1c+mr660KxkDWsgsncDzoDC4RWzjcCRtA5A7Z471cUL94MTn8q5c2i+K9Pi1GO9ia2uE3RI0CsVB/hYhjyDwcdx7Vq2Fo9lE0ct7Nc54G4bQv0xz+tb2jbQ5+Z31RqMwBJppXeeFzkZ4FV98ZIO0bgCoPfH1qXzZHJLOxz6sTSsVccEbe0YiKsv9/wCQfmcCqUpaU4WByFYZwdvPtkjP4VY2s3KnjvigqwA9PWlZBqMW3mMeYYogO6vLtIPcY/wz7EmoxFd+Vu32qNkhgGZynXrkDnPBHNSl9px37GqtpDi+kEH3nG54yT+8PqvX5gO3GQOORTSQXLC20nngveyzQk9I4lix145Lexz7Ed+LDCJI9kVso+XG93Zmyep64z+FQCV0c8fL0PtUwcg4LdeQaAGMrYXCqoVdowo6ehPU0rSyOPnlZ8dAxJ+lPxnBzkU1lQc4P4UXEN2k5IHymm7iMY6U0fONy5A9SKUHn19aBil+TUU7OYz5bAP15Gf61OVyOF4qMr/s/jRcRjXsN9LOrb38scARkr+PHNI6alB8sZluN/GRtOz67iK2cA8A8ikztOcA+oouwIF88shdlXj5lFS+cYGDoz7gcgjqKZK6EHoB71FGyAbNxRgejEZNMCK+mFvHDLFFiVm2yRqcKfQqMdc9eefxp0EyzoHQhlIBBB65q0E43q21v7wODWXN9l01J5/tEguJMu0ZQbHOR8xORhuR9cfkb7DLxzjjNQyBmY5RSvt1qGzvBc2izBkIP93+voaV7gkHAx+NK6CxHLbsCSpDVSljZkO04Jqw8hI5cj6VHkFcnqO4qX5FEJh+RRuJP86ieM7TkjFWWI2jB/SmEr3XOB6UhlcLjBAzx3NRyuu8LwGI4A71MU+csMg9+TUUkRJByVPTOBSAjZN5yc8EEAHFMOUJKnrzyc1J5b4GWyfpio2Ts3fnmgCAfNITyCB7gGk2ZB5NT7VUKuc+2akS2nkIEcEzNgkBY2JOPTjmi4yhIv7h49qOG7MuQD2I7j8CKoK+oRRspSOZI1LEM2GwOp/Kte5hu7ZkEthdqz/dLwlAfxbFU3jubg7XgjiXofNft+ANAEctrH/d2MBhlbOQff0qo1oqsGEsZXPPNajWunOUe7kuLh9o3LGfLAx/CDk5GPan20mlWS5h0lHfk7p5Wcf985AP40WC5jyfZFKqzqXPQKuTVbfbIzDyJiy5BUJmtmS4/eFoI1gycjy+Mfj1/DNQzTTySK8rmQqAql23ED0BPQUNCuOTRRc2KztJa2oIJ8t3Hm8f7PT8yOtSWdpo2nJ50l/dSTMmR5UO0rkfdOW6+uD+PWqWXOcsSPQ9qhk+pwadkFjeGpWoAEepXyIPuqUU4Hp0P8zRXO7H9D+VFVzrsLlR3E92bhc8njIOOKy5l8xsDJbuD+X41dj5iC9WX0NMW2Zpu2Dzj8a7TivYvRr9m0VVfHOoQqC/H/LKbp+X86v6Zot7r1y8Fq3k26cS3ZXKqfQDIyfoeO/vpaf4ct7rQA+rym2tTcpNGQ3zvtBxjHPO49OeOo5xa1HxJBaaeLSyh8i2jXaiDrjtnHSspS1dv60Nrdy0smneFLd4dNQS3PSW4mPzOfQYxx16Yx+dc/r9/bz6xaXCvvS8s8gfwhkYhgMYxjIH4Vzk2oXN/PI6yERlcEenNSXbIbLQXbK+TJcQEf3i2Gz+n61n5miXckw5YhE2rSvLbQToks0aOwyAxxwKZLcS4SK1haeaRgscajlvx/z1rT+xaT4fK3GtObvU2O7yF5WHHIAzxkZ6n8PSkkDYtvFYxQpfavI0dmTmO3VWMlyeSBgdF474z6gdWa94lmvFNpbR+RbDlYhGoA6YBAzyMfT+dZ+ra1e6jolvqK2UQkivDEGALfI445PcFAM/SsNUmlyzyhCp+Y9cnvT02DXqIbe4nuizHqfXpWzbW6WybiwGMEknoTVOAyIxDOGH4DBqe0hudYuvscI8tI0Msk3AIA4wmSAWzwPf6HDQn2Nm1sJ73zJlh3QwEea+SMH+6OMbvXPAHJ4rT1OwsLTS1v7u5W2hj++pBfdxnZGpOGY4wW7dT6izod7caMzaZbRKlpDGJWebPyk9uANxJyePz6VjavdXN68jyqEhLFzEhKqPfGc+/XrSl5Csjg9b1ZNYv/Mgtxa2sa7IYl6hQTgt2LHvis9LePcZNmfQkZ5/GulmutGgUhoirsMH95jP61nz3+lzSr5lw7dQvm3BbA/Ems7MvUoruHQFT0NKHjALAMxx25//AFVopNonliCE27yd2jiyR75Aq7H9ieRTI0rIvZkYD8+h/CnqMq6dptzcosjRKoY8KzdvU4zxXT6fprxZS9htUdxvT7OSxCrzn5lHOSPXqPXNZ8t3AsHl2z+USuUwOM+5HTv05qy+p262bia7kvbpl2NvGxOOm3aemSeue/XjFW6iZR1vUm1G6Fv8wtbcBUUsCCSATkYHrx9T1zk5gAVSsYGBximSW9wqjyZYZCckiTdheSeAO5zU6qQn3YI2P32RDlj9ST/KobGthh3Zx360A7XDsEZRzhqf5MRRhKZpcnA2yBMcg4+UZ/WtPR9OsWvIEvLOSZXkCBnnJxnJ+71PHqe1UkDZi3eoWcI2ho4xnklsmqx122DLADI7PwqhSd3oAB1rufElxpWjKLa00ayMknzbUjQKgPPPuR6EdcdiK5qXV7siTcYoPMHKQIseeh5x15A7npUq40zMj1OWTJg0+dmBxmXEag5/2iM/TFO+1apKAzNp8AOQVyzMv14AP51E96VbZJHtGcKoYEt/gKtLaW5Ky+Sm/g5IyRRqMWO1mYqZruZhkErGqxhgOx6nnnoafJpcFwh80zshOSjzNj6YzjFWQ23kkZ+tKsgK5GPrQIrpptkqBRaQfQIMYp8xk85ngjRA2S+MLknPPA5qRt/lEoAW7U2KJ0TBOT1JoCxKkkrBQZMYxnaMZ/nQ3VnMjMfTI5qNpGVEKYw3XIII/ClnSNyu5QXVT36/Ud6GwGJMSpdsAcDKg/5/GnRyB503yCNGGFBB7nqe/T2/OoSUOFmDBj2yeKUXUUjFVJLLjj0+tAG5dtu0jS1McXmAPgq2WCjHJz0zu6An8KggkVXTzTKFJ+cR4zjtgk+vtVsaLbxaZp1406I81uf4j8x4OOTgEc5x6GqWBx09qp7koklkQuREreX/AAl/vD8uKniicNtF4kS4zncBnj61ADlSOBn1OKRI2dwoAwTy3pQAbyJDIJ5Hc8nI4/PNN+csSwye5J5qWe2aI4DLgfxZFVQPm60XGWoUMjhEBZmOAo710NrpDQfOXEkgU/IE3DNc1G7fMVAZcfMCM8V0j+ILZLTFuJC+3AJXvjue9Du1oI0rWBBBlgfnPIdAuD6YxSTxwN8paNGI4BC/1Fc0msXMzDezHYNxOMZPOeg4H+NE2qhiNqrGR1JUHP1o5UxF24tWk3QoImbj94EwEz/tAj+VZd5YzWAMhmQEgr8vOfbkfypkmqyKjyNdqsZIzjCgH8Kz5dVsGB8y8Lgfd8tdxJ9B60ctho6nw7ORCrQu32hTkHdtC8HgnjHBJzUXiCxniu4Zp7gyT3RKESszMm3J29xxz3HLDjgmsiPWLa0lWS2truRjgGJYzyM9Tnr+PHHFXdVvpdW0ybYIre7Xy5rd3YMQy4wB2Xgc4x2B60W0uF0Zu51APQ+grTsrq/kjEUEH3TuEix5wfcnioYrnW7gLtl0WGQjLAh5Cox3OMZ696fGzTALN4kPysBItnsUY6/KNhOce5pq/9IC6l3qkkrRvG6sOyqAAOxoe8aGMC9u7aIYIDyTEM35EZqGew8P3ccYnEtwyKYz5txIVA7MAuAT7cfl0px2mhaf532XSkeSUDLTnftH+zuyR+eeAeoFPXqidCdPFOmQRqG1XzJMkCKGMsOB0yoPb3pz+LA7u0FtO4RMqWiKFjgn7p+YjA6gHHU4FT6ZqKxXGxbS2UFCg8uJUJX+724PvWvei0TS3lkgWNFQF1UY2gEE4x6YBGPQUr67f1+A7aaHEf8JRrAuGQWFspbnzEyWIHbGByBxj29KLK28Va/KEOoLb2quDI4jA4BBAwMZ6dM9+exqZP7EtfEVpDG0txGXzvlIZQSrHHHHYf56dNc63bwp9ns44ymMApwB+GKqd9o6ERT3kUJNF0WyjiM0ct9IBzLeyNMXP+6x2/pWdbXSJGYIo0tiGLGJVCA88HaMDpgZx2p73Ek0rvM4OTwBnNQO1lI6M8ih1O1DvA5PWpSsWeefEYlvEMBJzm1X/ANDeuQrsviLaTRarZ3LBvIlgKRsxGSVY546/xDnHNcbWEtzaOwUUUVJQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRT4opJ5UiiRpJHYKiIMlieAAO5oA99+G4/4oDTCP+mv/AKNeuo2c9OtVbS2TTdOtbKFmaK2jSFGY8kKABn3wKsh2bICmtlojB7ieWoJ4+uKeuB0/PFJngCoy+cj3wQKAJFb5yQQRjikkkUA9z3xUKsM5Rc4OOeDTwQw3Alc9c0DEcBgOx7c1CbfJGeQM4B561YYIRkso/GowY92E7jr2NCET2sLXW5Y1VZo0y0agYkA7gevr6+3FJ833GGCP0qJozkDIDA5HsacilbVGeV5G3ESM+MhiTtHAA55wPamA9XYEq2MClMigZNR8HAGSQOtPKDcGC9R3oAaJichYjwcZyKgaWbzQggwrAndnpz7VYbPTPAqEhucdehz3oASOaUsVkXtxj/HNSod33R/9aogGPU/TB603f823IB7jOaAJSeePyqM7s9CfpSGVVB3HI9qh3ghlDY7cd6VwsOklROWCgg8Bj3qBPJRtyqBk9AAP6ZpzojRgyYz6jrURAZxtIVRz83f6UuYdiZrocgDH41myu09zsiJQ45dSAc88H1qS8uIUUqxUr7kf41jNdWkUoLX1tCOvlHnH609R2NY+UGiiwkNyNqeZGAscnTJkH8PGTkDn0qCO68x5EZCjqehI59f/ANVZ8utQTstvBDLcO+FVUiJ3EnAxgHqeg96aGiv18iDTLu2kDjddBSAFOMhhnkDgjgHgjvU3WwzW6PjAOelOAxztJPoayPP11olSK1tywJG5shfqP8ioG03xDclluNXiiXPS3TrQ7LcRulovLLlk/DpWdPq9jArGeeKIg9Cw5/Ko5fDFjOIzObmZ1QBzJOW3N3PsOelWYdH0+1A8iytowO/lgn8zz/n60r+QzLk8Q2om2xJNKSMjZGxyDjHanpqGoXEsccOjTqZACHnYRqB6nNbaxRRKWVVVR6ACjYuORkEdSKLsDBVdVdm3p8yn/VxsuD9GOBn9B71cFrFZ6l/pl5LOjRZSFlAUE44LDqR6e4q75xC+WiEduT1qxZ30unvJJGscqsNssTjIZaNQY3+2mtFxaWNrbNwS8anJ+uTz+NU5tX1G4JVrqTa3UKdo/SppIZLm522dmzxn5nAb/VjrjHJOPXp7DvWubSW2ILgAMMq3Zh6g96L20QFSd3lzvO7PXPeoipA5P51MxPAYcDvimEHseKd7jKsiZ7Y+lQuhx368Vd2dxTXUkEflSuIokMaTkcEY/DrVhgV4x/8AWqMgj3+tADcrkFf1ocbw2QPypGUkk8ZpjE7eO3pTCwnlf7v6UUb2/vn86KQaHTQkFmKkMA20KPvHPYDqTXQ22k+Qy3F+iB1UPFZnnOTwZSDgDI+7+ueK2TZaJ4XBktokN4wxulYu/wDwIk8f7v54HFcxfak927oHESOcl8dSa6nNvY5lFItazrtzMASUeReCcbVT6DJrkdSuTJatMkhyeWyg/GrQZZcQyurlVJPHB98His2/doHSBOYieAwA4qHoWhLGcYEcfzMQCcdTXTHR59c0qxht2VHt73zCWBwEKYboOvTrgH1FZ+haK06l42ihtoot013MAFT/ABP+PvWpc6pDabLOyJ+zKmfMA+eXI5LH69qq3Rjb7EdxqNr4WjksNKMk184ImvJOWyc/LHxwB6//AFqx7LR5rstNfXTK0jZCuMk/lWlYWkl5qAvLhRtXmME5K59PbpW26RAE71Q9dzDpUu4lpuVZbG2i8Najb4KpGEmyScgiQdPz9+vrWBa2YgAlDsA3PI6+9dFeiaXRtVgR/wB4bKQocZ5XDDA69QK57RrS8vtHaPVZY7aRR5PlFyJM84LfK2BxyO/tmhLULla+Se4mSK3EqQld0k6KxD4ySFI+nXPXvWha3w0aBvssPlW7qCEcf6wZJ+bJPGCeMjjGQTTNYvTHDbWkoVbOMYMUbY+UZ6gqD8x9u5965rUdQmvpCzPj0AHAocuVaDSubJ8SXiyOY5EDv95zGGbjGOuegAA9Og4rMM8sjAuwnHY3CLJ+W4H1/wAmqVvGGI3sxb6VpLb+SrEk5xkZwKjmZpaxKNX1ARCNLyeNfRHKD8hioTLPLtMssj7QANzk4A6DmouM9RUqjcQowM8cnApb7isJtfdgPkHqCP8A69O2cck/X1pFOAeakDDABx+VFgERRg53fQmn4BJPIoz3NAYHNAEnmblVeMD2qBoIncMwJIOfvEfyp455pdpJwBz+dAy1aSRRbVSIrIWABUYA59K2FSaL7XfHY1lbx4LNMEw2Mnae7DgAHjqOc1kabbQ3UrxSvhwpCKX25Y8L9MkgAmtnxVZx6No62YWMy3c2WLOGcZ+Y8pkAHngn6Zqk7K5DXQ4+12T4LTYDZLNJO24Zxx+AwOo6VHd2aecF+0QTsy9EVmIAPXI4+nBq0AAQGwOP07003tukwiLkHGQMcfn0qdyyOPSYAVZ8uRz8wBbPuauYAIC42gc8dPpTGvLdVDmVVX1J4qJL+2mbEMiOemM4oHYtDHUelLjIyMn29ajRCMsSee2acSPlLMFJ460CJBu2/Ln86cp4Gf0qJGLBhhhjt61KxViWUNszwCcn86BkcsrKxUKWGM/KOf1pI7mKXkZB6AHqKa7rg/MRg4JPtVaS9s4HMbyJHISTjcMnH+elIC49uJI1AL5Ax15pCxgKIkZHOCFUH86ms7OZrZpi5jT5cfaiInbPopwzD3AP1q3MtpDBLZ3UCzStGSlzbzgrE2DgZ9eOQARyKdn0FcbcaiZU02LbDk2pO8ICzcp1c8nkHv8ATGDlgLDc4UnH05qrO0yPam0eDyYYjA3mp2LbsDA5PA5Yk5z9KpvY3Dom7UpFO4vmJVXg9j1xjBx9fpTk/eYlsbwkE4SJICJMZyvOf/r1JFaTvKFDbSeAhYD/AOvn/OKwF06AwmOa5v5V6lZJzjOevHNXtOA0lJV09ntxKR5hSRiWxnqSc9z+dFwsWJNqXMdq7TJOylynkPnaBknGPQVDJdwBSVt9SkYZ2bbbAb3O7BAPr+lTSXr4Pmz9OpdsVELiNx/rVK+xoswFtprq4ljitdFuXds7S91HHuODwBzknoOnJFXZNJ1GIK1zNY6YmFDxSs8jKCVBI2lQSBk7Rnpg4zkZ66jYGOX/AElTtXaTnJ+g9ap6xqjz3Km4kS1doVA8wgZBJYHnnPzd+cAVVu4r6lm+0tHt9n9v3Ew3ZbbZJGCvoGDbs474/CqX/CP6e00habUHhJzGrTAEDPG5tvzcccYp39s6bGI4Y7lZS2FUIC2fyqOXVJhzBpt/Mh6MsRxUuy/4cpF1NL0mExtHp6Foidskjs559cnH6VcjneEL9nIgGMfulCE/XHWs4vqbbRFpjcjJMk6Lj269akgj1Ga6SOcWltBn5pNzSFeM8AAZ7UroRaZ5HYhmY59agjHkS+Wzud2SM9h3qGK01JWAa6tkiPaNCT+uBViTQZNRlk33GYAh5MXIbHGcds4/D06076XFfoJFPayxSQsFcLLyz9eO2M4x7d6lknjg2q8qxKBgAkKB+FQQ+ELUXqC1vZYpWjJyBhcBipPOSOg4w3XrXSaf4F0W3aGeZLi4nQh/9Jm3gHOeVHyke2KTl5DSRzR1KxiVmNzHgds5JPt6/hUKa9bSu6wQXd06KWKwwkkAdSfQe9d4PDOg2cGV0uzG0ZDNCGP5mrNn9ktk8uDy48cFUG38/WhOT2QaHn9rqGpzSb08O6oydVKIw/8AHiuP0revLjW77SJrZNAu4DJGQJJLmLg/iQf0rRubq8TUCVvIthOUiXliPQj8DzWrA009sjzqA/dR2P1p3lbp+P8AmLQ8z1Pwxf6dFZX6GK3miK7zNKiID0wOcn/A/jWvaeFNcAfzL6ztEIO11DTuc9jnaBzkgjn65zXRPqfmu0UZhQxHgv6DrioPOe/b57rbD1CqMlvwFHLLuFzKg+H6yr/pmvXs6Z6W8axD17ls1ei8A6BbMZJ576X186625/FAtalm4iTZskLA/KWGP8muB1n4n2VtPNBZQyXbLIQZA4RCMdjyT6dAPepcV1b+8abexl/FTRtL0ddIXTbZYPN84yYd2LY2YJLE+przmt7xL4nm8SPbeZAIUtw20btxJbGecD0H61g1nK19DWN7ahRRRSGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAV3Xwu0Eapr8moOU8rTwrhWGcu2dpwRjA2k9QQcYriIopJ5kiijaSR2CoiDJYngADua+jfDWiJ4b8OWtgiB5FXfMyYPmSHljnAyOwzzgAdqqKuyZOyNGZnPlGN0kcMGKKRuI9smrHzKxADAHuDQuMqduD6d6OW5yOe461oZDN3ZSfrUKTCSWRTFKrIfvMnB+h71YI3DIPfHBpqxEZJUA/XrQBA0Ks28blY8Z20rPtZVWGV891X+eas+wX8KQYHPf60AIVHlkbRk84//AFVX8u5DAAxYAG4YIz9P8mrBIxknn0B6VF9pjaXYsyuf7iHLfpQA5kBHXn0NRNCnmKZVWVM528Aj6ZB/X+VQf23ZSErAJZ2GRiGJnwffA4qv/aN7cKY4dJuA/TfJ+6Vfc7sH8gaNt2BZt75VvmspmKtt3w7myWXj+LAyRnngfrxcMbDjeR/dPpWL/ZMjWG2a6f7WDvjcOxCvk9s+hIPHSqklr4iuWzNqFlbR5yqpG7n8SdtNyihpM6BZginzCoYHBGc1Vm1K3ix+8wSOo6VlNoLSEtd6pczHvsjVPrjgkD8alj0HTYeTEZT6uxP6UuZdgsTNrVuHIM68AYAOTVN/EEcshW2huLpsZKwJvwMZJNWltbWIEw2kWT1Kxj+tPYxyxlWZTGcAp0A/Wpu+wzPe/wBTcReVo07hx0MiKPoeeKijfWvOijfT1tY5pViLRyedIpJ4O0cHOcYB/Lvs7yFyDwe47Ux7mJYt0zK0akHeoJIP4dOaNQMI2GqSzywf275cqNtkjWDhOOQ27Dgg8dPoTSx+HZp4nW91W8lcj5VhlCqCQMZ4+vGD9e5feW8dg0cls5dCwL5YMAWIxuJOQOvI/I5NaMUjSW48uKSOTarbJAV4PQ89RwcHpTb7MLkGm6HpNigka1WaZcqZbnLk5GMYJxjBPb1z7W5Io0khlgQW/koUiW1ZowqkkkLg9CSTUcElytz5VzyGB8tgBhsc849vaonuT/aCRs21V6oQctnGD9KmyGXV1G8lkDO0q7RgM0pPHpUMM7y5LAJjj602ObzYyAjhgTwykZ96XYJGGYzu6gmmIlePL7myR/d7U13SIcrtHXhSR+lJhk9WpWj3FWYEFeQQaQCKyyoGwcHoSMUxl7HmkX5GIZ3bJyC2OKkOVYcA0ARYJGOwqM9RnirDHPPT2qAOjH17UAM6HjrVeaGGdysg+fGcg4Iq06Ky42/KRg5qv5ap0UAAUDFyyDeGbcOjKcEfiKrrsjSW5KyXW7kkPyc9S394gjtg8nJOMGwo+U7STnnmq8iPG5lhAyT8y4+9TA0n0Kd7H7ba/wClW395VZW4yCNpAPBHvxWK6bQegFa+k63LZONjK8XR4JCQDzyR6fUU6/stOu/OuNKuPInCtLLa3Gdp7kqx4H0z6fSkkMwOOO9NKnGVP4VMuJoVZWGcAkf1HsetROMHr+VFwGNypBH51E6dPTsakKk9QCR0oIyOelAEJUEEA8ewpjx568ipvKzyMYqME4IB/GgCHyv9kUVNuHcNRSA1tV1lpWxjj35FNjBmjJcMFIyR6VkTT7sLzvUckdM1Oki26wo772b5QoPzZ4xgDr/9cV0rsYO1iW8lSFgEchozu+U43e3Patyw0BngXVdYDW9jj5IGBEkvfAHbp9as2Om2nh6yOr6tCZ74/NDbsD+75yCx7Hocnp0HvRutfkn1F7me+Zg4VTHjhQQCAB2HP44NO3QRNqWsu9ukS28VvpUQBht1Uc5/vc+vOO55PIAFW0sn1G6hmm2qhH+rXgEDHH+fWsXy59U1BmZwY1IAx049vauxto49PgQP80hOOPU/5FLceyuTYKOFA2J/DxjPvmh32MuFLkMM8dM9P6VUuL2H7R5e4hwyjhOSx6KPU/59a1YrcI3nXM0QZZcCFZASowSob1JxzxgZHpmmlZE7leSD7JPDf3YCMSPKIRmaEEZ3EcDPTj8KZdyw2unzSqYWaONfLk8jYZSf4jwSWJJGD6+ladzrMaQwfarNrdY2KFgzNJ2wAAOeuMnH04rBuZHuLpbxGZIwP3cbt168njrz+ApPYfU5/U11CVGadp2MpJLOSeOy8+n+e+efZBF5arJli4Hzc8fWuo1V3ELSSsOByqvu/wAK56bZIoVyIlUhgenPXrWT3NESpII0BQbWHVsc0m9n65+pPWo/OGSAd2TnODt/A4qVNxIVFDE9NvzE0xirG7HPAH1qTb9M1Hubo2V+tKZNsZGR7nFADlBHHSpNwxgrz71UWaMN80yKR1DMKke6t4gu+4hQHpvcCgCzuBPFJvXOKSO3kMJdYbmdevmRQPIpz7opFSJp1zcSqILaWTgH5gI8fhIVP5CldANVg49fQ03EoOBsPXsce1aLaDqccW94YIlXqZpQP/QQ1Vkgu3lKZt2UcFkJP6lR/KlzAbWhRW96YxFHILqIFp3QZBj+bBxnHUgc+9ZnifUIbjXzDBFNAkCAFZCMZPcADgYAqzp0v9ks7mQb3YMSuH2kZxgY65J568np2z7uzjmvbjULiW7Jnk8xyFIDk++CeaqT2sC7lA7pEkjba3OQQBkioLmzjZmeSUnfknO3JPqcg1u6Xa2trLHa6tpjkXDHy7iZyBxyQwJwuB+f14rcim8N2YkggFrJkfOsMJlBPXBwD6dCahp9BnCWsVkCIwXu35xGqbyM+gUVpw6RqHD2mlmJeeZV8s/k2M/nXSy+ILdVaG3NwoH3Y1t2iH1y2B/X86xJtQlkIY2bycDAa5Axx3GKai2DaRTe01JT8z28ZPGGkUgf98bjTRa3juWkvLZNoLAqrsF/DjNXpZWkeLyrW3RD9/dMzED1xx78Z/Gmp54ZkZrbyZFw4VGJORyMlunXnGfYUWYXRkvDeCd4Jr9o23EKFtwhIHchs9v6VrWUv9nohRY5JFHMtwPN3e+xsoPwUfnzVH/hH4JZPNle4kcDGRJt/lU0Oi2tsc+QZG9ZWL/zpoVyzaajPpryvZXj2sko/ePFIUZvqRzjmmXGrS3bF7i9luGUdXkLnBP496etpHtKfZoNp5/1S/4VKyuM5bOevP40+lgM4X8ckhVVnZh1xC/H6VKruWAW1nOT1ZQv8yPSrLBgeT+vWoo1PmF2IVB05o1AfHaapcYFppFxcljjETIT0z03VDHHfO6u8UMSEZKySfP69gR/k12tnJbaV4fFxI6LNLCTFFPASCzkruViMEgY4B7nI6VzKlUiWOMnYigZLEk4HrQ00wRW+x3Hnu32qJY8/KnkFjj67hz+FX9PNnZWzxyRtdmRTuMhXHPp8pI/A1WVt5xkk+tOXzEBAIwRg8UeoDJtO052yluIVxjahzn3+bJ/p9KrNommOuw2hZcYw0r4P4Zxn8Ku4JXpgn3pQ3GGpWTAgt9NtLUiSC0t42XDK2wFgc9QTznmp2l8xVjc71TO1W6DPWm7iOnSl3DrwfeiytYB/nuI9oyB2IJ49hSK2CSck+uaaSvTv2pec4JG76015AO3ZOVYqo6jjmmOWK56/SjHHIrQtrW4vbTybdTsOfMLIAo9Pmzz+lAbFDOMdMitOwvfKVlklCIx+YKmWb8QM1UlsGgiLPJGWDbdqvk8d6fFaGREcPEp3ZGZMH6YzQmGhowWn224jdQ4YozMAAAAZHIyPX5QMnJOPQCte4j8q3HmTLG2PmfPCjHPWq7QxaQFumlXzJIF3L5nAI3HoeeSxPHr07nB1TWZNQSKAArGpDP0+c49PTv+ApiRZuNUlMDQQ3MoQY2ykhSPpjoPrmqun6hNaTsWJdVH33BOfpVm301Y4xJdhyuCW2DhMDPJ/pWbIVDOEY9flz2oUgsa8V+k92xuUiO8AfMM8fl/Ord5fG2DIm1sABUQg5/LpWDDPIm7ZIQD3Aq7bR3EjrKFMiD/AG8Z+tPQRNb27XFsrvAkcYGd7cFfqOD1/pUsWY5g0MTTqCMu0oLLx2HQfh1zya0YrXBEj7A3GEUnimyxWcyM0aRFFY7yg4z7471IWK2oxwrp91BDqItpJYnQSqdzoSCAwOc5Gc1843NrPZXMlvcxtHNGcMrdRX0DJFYsu392gVjhlU5x78D+vbmuS8QeEl8QbZEuEt5ogwWQoWVh2DEcgZ784yeDSnG5UJWPJqKvatpN3ouoyWN4gEichlOVdezKe4P/AOvB4qjWJsFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUVPY2j39/bWcboj3EqxK0hwoLEAEn05oA9G+FXhaSa7TxHPI0cULPHbxhceYSpVmJP8IyRx3B6YwfXHKgHLYH0qtpUWmadpttZwN9qW3iWJWactkKABwmPSrVxJa3dsI3tIkXsfLYMfxPNbKLRi3caZosgs49smiDzLiTFsrSA5JZVJHbv0HUfmKjilsbeVRF5MUyrgcBWwOfr75/GrHnPIRli2ehZiadhGfdahNbT+T9gvHk7COIlTz/e+7+tQfbdYb7uj4GON86A/jya12TzMjeyEeg6/nVeBohvWGQuQ5D5k3bT6e30o1Axby416VxFZW1vIrL85Lsmw9jkgZ/AYHHWprfTNZklja81OGKNcb47aHJYem5un5VtNDaXKxi8kaF4yPIniUllyfmU4PIPoQcc45qtaTySoQ0c0Ui8FZkKn9cZB65x/Wh3Ei8DGIUj8iBtowWaFSxPqSRUDXUoj8oMyovRRkAfQdBTdzl8cjB69jQYiecEmkO5EXZlwzsccYyajC8EF2/PpTyjL1pBjFSURCNg24yswzwHA4/ICng9Qec04gZ700qT6UADk7ef1ppJzyOKGQ9PSmnK8cGkA0gUhRCF8xVOOeRnFHO8NjJpoO7sQc0AHkwnnykBPP3QKZJGr5BUA8AsvBHvmpCOcYoKhTnHTtQBDMhMSpJmWPODxz/n6CqD2iC4DQyLHPuP70KxZhz8rjcARkjkDI9a05CSpx1NU45Ibo/Z5v3dynIXPzDHGQe9CAXUtN1aMWryxtbpIR+9ikDKw74I6cetJcRRmUyKgEjDCt2HORwePxq/Y6tLp4kguHa5sHBV4vLJJbn7o7Hp7fTrVO6vbSO5CxmVIXHyNMMZ9vr7HBppMLjIVfyUPmkk8kt/niptuH4JwR0rDk1R8yrZ20kiBvmZSB37CrVlqsN65RQ8bLyVdcfyosM1CdvbNVbwPJD+5Y5yOh7d6lIDnucehoIVBgKAOuBQBXtGeMFHY5X26ipjICcCM/1NHmbcnGAOtNWVWXcpbB9BSEJIkgxtGAfXnFMKyBiW27T3xUwkbPPI+tNYF/lPzZ9cUAV/M3NtQBiBnGaQ5C7iCD3HpSw2oglZ1LgH15qVgAehP0pjKnmrtywKjjlsc0rcgEcn0xUk0aMpDKpHow4qoZegKnjggDA//VSASW2inIfZhx0fHIP9aIfMRQJsiRGyHAx9CKkWQ4wWHvgVDPcqoHmSKqngZYCiwF7UtZu7/T0guLaC4aIAJMwIkA7gkHOPpjp3GQc++ghVEminJj27nDZXZyB1YAkc+nrwKge9jFq9wqtJCjbC0alsH04702yv7qSRTBaOkZ/im+VD37fSnp1Aao3fwMPc0m3PGKs6jbySXKGznWGHb84Ve+c4HoOcfhUUttCd4SeYo42kuQSOnQY46frRYCu+E5LAd+TVZ7+0Q5knj4OMBgea2c2TWMcD6dZu6jDTeWQze5AIHb0qtLOFt4YkjBih3eXGqDCFuTj6n+dIZEIywBFjfkHkEW7EH9KKnXXdSRQiiRVUYABHAopWfcV2YtqwW6iaZSzM2EUdz2/WvQ4odJ8FwxSagm/VHUMNqb/K4x8ueM4zz0/QUUV1wSZjLexmeK7pxf28hXFve2sZJGC3Jbk569a5p/DszXACyhUIBOWycUUVm1r8xxVjpLK2js7VCjbUXjI71p22nvqc6RSSvCsmN2zl2Gfug5wM9M9s0UVdid2O1qXTdIMTwQQfaSzKHMeCCpGUUhc8H+LjPbFZmteJJ7PTEt4Yv3wieckncV7E5Pvnjn2x3KKuyUbkXu7HGxeI3lvo4JJGW1lG4IUVckk5X5eOpOWxk89Mmum/t2COEqykN/CXYnPqSQM0UUviV2aWtaxm6pqsU0azMkRXbujVy2BzgEDB5z9Kx57LVrq/jWAxXjOm8IrEBF5/vY9z3NFFc1R8r0HeyuasWgayUJe0VADglp1AyOvTNPi8N6pKZHd7OOIHnEjswB79P8+lFFEfeVxKbZIvh54kZJtQaUowV2SPbtOenX264P0q/Y6TbLaGZZ33NHtY8jORgjI55Gc9u3OaKK1hFO5TFmstJtoiYbBb+YEBlZ3RNoJGTuY+3AGD+GKrLdanbWwh03TbKxQcYjuP5/u+TRRUzgk9ATKv2rVpW3XVzaFeuBvY/wAlFXYhrN0Eis9UVJGYgqlqq9v7xYn9e1FFDuluUMvNB12CALca1cy+Y/IaQsAPTmq0ejTbgwvbtF3FZF84AN/3yo9/zoorNN73E2x0FrLbWjAXtxNHgjdLISfT/wDVxU96lpFLHGlrC1xGSZHZFbD98fKMdBxzg5we9FFaL4bg4opzPPO2ZGAGdw4HX14/zzUbBzjL/nRRQtWA/wAp9pZpDgd6ktYVuJJEWdF2Ebtylj1x7elFFNK4yX7MUcc4UH6Z9+9XEtI0OQSc85NFFC1EKuxJDnO0HgHnNNd0/vHB7Yoop2AhLgHK5x060xjluv0ooqethjGyTzginRKAGCjLdh60UUIGbmr6vc32m2Vi8aR20cayRkJ8zj5kXLbj6NkYHJrFxtU7Sc0UVKlzaiTHJx07DrTyTtBBooqhjN5yOmPpT+qk4FFFCQDR03EhcdPehPn5HTPFFFSkMcVKMRwSKcQ3HIoopoT3EYHAyOaBjp04/OiihDDaw7DGeua6fTtNT7Irs4YsDyFwQD2BoopdBMx9evrqeW3WaQGN498cYjUCMHjAPU9O+KZpUVg9rJczs0iRdVGcD6+tFFXJaNkrY09ctrxLVjDLEliCuY1BDccY9MZArnQeAB9OaKKgvoOUkvgHnsK6fTNPEEKySxKJM5BByRRRRcll4oZDlsrhsjnP/wCqoIts0js00qrGdoG7IPuQRRRVLURcW3jIwo+QjtVd9M3o6fvVVjxyoUY9h9BRRWbdmM8m+LWix6beaZdrK7vcJJGwbGAEKkfj85/SvOKKKh7m0dgooopDCiiigAooooAKKKKACiiigAooooAKKKKACiiigArufhTY2174wZrmBJTb2zTxb+iuGQBseoyceh56gUUU47ilse4TRLNE0cg3KRyM02HcilXffg8Erg/j2P6UUVsjEkznIxkHg1G8VvIhjkgjZT1VkBBoop2ESRRpEfLiiVFH91QAKry28Vv5lzBFFHKeZGC43geuO/vRRS6gTOY5LfbtGzHTFUL24uYgl7I8s8qHZKHwFEQ5DZznI57c8+1FFDdgZYt7j7QrFk2sjlDnnPTkfmKmBPTGaKKVwGyqHHQ8VBuGcd6KKTKTF3Zximl+dtFFSNiLyenenHBJ4oopiRGdqngdPamsvzBj0oooYxANp4zTMAsFPJ60UUAIFAJxz7VUuLWG/jKTxBkzjBAPINFFIAjaWHMM7h0AyjgYOPQj1+lVdRVJ4njKozKpwSoJHIJKkj5TgDBHPFFFCBFW6tpLS/Gn6lCj7kMkNxkEyKDwSMcH1HtVDVLdLeSSdBKQrAhi+Gwfpj3oopoaK2narZ2+pSTyxFGfncn8XrkDA6V1N5dLBHGQpbf90D/PFFFMGNDBG3AkE9vXP/16lEIx82CSKKKkSI8EHtjrSNlBw3X1oooGVmvFWQozhSMkgA/TNUf7ft2n8mEzTys4RUUAZJ6DJxRRWjSUXIaHA6k8EpitII3jbZ5byc/oMY5PfsfxfZW2pNeRtfG0W1/iigB3H8SOPyoorDmbSJua2+yhcrHZREMMEy5fP4E4/Ss+W3tSZf8ARLcCRt7ARjBP0xRRVWVihm6KFCsKKg/uquB+lQu2Dz1oop2AiYg85I+lNJI7ceuaKKBDDz68+9MOQCM8d/WiigZkzeJNIgnkiku9siMVYeW5wQcHtRRRU3HY/9k=" - } - ] - }, - { - "items": [ - { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAKXA+IDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD3WlpKKxNBfwoFJS0CDvS0lJQA6k70oooAAaKKTpQMWiiigAooopiCiiigAooopgLRSUUgFooopgLRSUUAOzRSUUCFopKWgApaSigBaKBRQAUlLRRcAxSUtJTuAuaM0lFADs0UmcUA0ALRRRSAKKWkpgFLmkozQIWikpaYgooooGFFFFABS0lFAC0UlLmgAooooAKMUUUAGKKKKAEopaMUwEooxRQAUUUUAFFFFABRRRQAUUUUALmikooAWgUlFADqKbS9qAFoxSUZoELRSUtIApKWimAUUUlAC0UUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAVKOtJRWRoOozSUtAC0lFFAgpc0lFMBaKSigBaWkozQAuaKQUUALRRmigYUdqKO1AgooooAKXNJRQAtFJRmmAtFGaKAYtFJRQIWlpKKAFooooAWikooAWjFJS0AJiiloouAlFLRTASlzSUUALR3pKM0AOpKKKAFpKWkoAWikooEOoptLmgBaKTPFFMBaKKKACiiigApaSigBaKSjNAC0UUUAFFFFABSYpaKAEopaKYCUUuKSgAooooAKKKKACiiigAooooAKKKKAFozSUUALRSUUALRRmigBaKSigQtFJmigBaKKKACikooAWiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAKVLmkorE0FzS5ptLTAdRSCigQtFJ3paACiiimAUUUd6AClFJRQAtFGaKAFzRSUUgFNFJmlpgBoooxQAUUUUAFFFFAC0UlFMBaKKKBC0UlFAC0tNpaAFopKXNABmiiigBaKSigBaSiloASilpKYBQKKKAFopKWgAooooAO1FFFABRRRQAuaM0lFMQtLmm0ZoAdRTaXNAC0UlFAC0ZoooAKXNJRQAuaKSigBaKTNLmgAoozRQAUUUUAFJS0UAJRilopgJRS0mKACiiigAooooAKKKKACiiigAooooAKM0UUALmikooAWikooAXNFJS0AFLSZooAWikooAWikooELRSUUALRRSUALRRRQAUUUUAFFFFABRRRQAUUlLQAUUlFAC0UUlAC0UlFAC0UnFFAFKijiisTQXijNJRQAuaXNJS0AFLSUUxDqSkBpaAFopDRQAtJRRTELRRSUAKDS02lzQMWjNJmlFABmlBpKSgB3FFJmjNAC0UlFAC0UlFMBaKKKAFopKKAFooooEFLSUUALRSUtAC0UmaWgAooooAKKKKAFpKDRQAc0UUU7gFFGKKACiiloAKKKKYgoopKQxaKKKYBRRRQAUZoooAXNFJRQIWlptGaAHUUlFAC0UlLQAUZoooAKKKKAClpKKAFopM0ZoAWijNFABRRRQAUUUUAFJilooASilopgJRS0UAJRRiigAooooAKKKKACiiigAooooAKKKKACiiigBc0maKKADNLmkooAXNFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQBSpaSisSxaWm0vNABmiiloAWiko70AKKKTtRTAWjOKSloEL1o6UlFAC5paSimAtFJRQAtFFFAgooooAKWkozQMKKKKAFopKKAFoozRTEFLSUUDFooozQAUtJRQAtFJRQAtFFFAhc0UlFAC5oBpKKYC0UUUALRSUUALRSUtIAooopiCiiigYZozSUtAgpaTNGaYC0UlLQAUUUUAFFFFAwooooEFFFFAwooooAM0uaSigQtLmm0UALS02igB1FJmigBaKTNFAC0UlLQAUZoooAXNGaSigBaKSigBaKSigBaKSloAKKKKACiiigBMUtFFACUUtFMBKKKXFACUUuKKAEopaTFABRRRQAUUtJQAUUUUAFFFLQAlFFLigBKKKKACilxSUAFFLijFACUUuKKAEopaKAKNFJRWJYtLmkooAWjNJRQA6jvSUUwFzRRRQAtHekpaACikNLQIKKKMUAGaXNJRmmAtFIKXNIBc0UlFMBaSjNLQIKTNLSUDFzRmkopgLRSUtIAzS0lFMBaKSjNAhaXNJRQAtFJRQO4tFJS0AFFFFAhaSlzRQAUZoop3AM0ZpMUUALmlpKKAFozSUZoAWjNJmloEFFFFAC0lFFAC0UlFAC0ZpKWgQZopKKYDqKSigYtFJmigBaKTNLQAUUUUAFFFFABRRSE0ALRRRQMKKKKADNGaKKAFzRmkooELmlptFADqKbmlzQAtFIDRmgBaKTNLQAUUUUAFGaKKADNGaKKAFzSGiigApRSUUALRSUUALRSZozQAtFJmjNAC0UmaM0ALRikzRmgBaKTNGaAFxRSUZoAWikozQAtFJmigBaKSigBaKSigBc0UlFAFCjNAOaKyLFopKKAFpaSjNAC5opM0uaAFBopKBzTAWgUlKKQC0ZpKKAFpaQEUtMQUmKWigBOlLmiigAozRRTAKWkozQAtFJmigApaSimAtFFGRSAKM0UUwFzRSUUALRSZpaAFpM0UUALRQKKBBS0lFAC0UmaWgAozRRQAUUUUAFFFFMAopaSgAooooAWikooAXNLmkzSUCHUUmaKAFopKM0wFoozSUgFopKWgAooopgLRSUUCEdtiM2M4BOKEYlQSMZ7Ukv+pf/dNEZzGp9hSGPzRmkzRTAdTT3opD3oAfRTc0tAC0nejNGaAFpKM0tABRRRQAUUUnFAC0UUUAFFFFABRRRTGFFGaKBBmlzSUUgFzRmkopgLmjNJRQAtFJRQA7NFNozSAWlpuaM0AOopuaM0AOopuaM0AOopuaM0AOopuaM0AOpKTNFADqM03migB2aSkooAWjNJRQAuaM0lFAC5opKKAKGaX2pM0d6yLFzRmkooAdk0U2loAWikooAdmim5paAHZopM0UAKKWkzijOaAFo6UmaXNAgFLmkoFAxetGaSigB3Wik6UZpiCijtRQAtFBpKAFozSUZpgFLSUUALRSZpaACiiigBaKSimAtLmm5paAFzRSUUCHZoptFAWHUUmaM0ALRSZpaAClqtfXcVhYXF5MHMcEbStsGWwoycD6Cqfh/VjrWkpfFVXzGbCrzgBiB+OMUAatFAx/kUZHXNFgCijI9aM0AFFUL/V7TTrqzt7hijXbmONscBgOAfTNX6ACiiimAUUUUgClxSUZoAKKKKLgFLSUUAFLSUU7iFzRUEl5bQkiW4iQjqGcA9M/1rk7/wAWySo0UMYiB4LhyT+B7VLmkFjs6K4az8U3VvGUKpJly2WzwMdB7VuQa/HfW04RzaNGqnzWUOOvTH6UlNMDbk5jcf7JoiI8pMegrFu/EdrarCgfzt6gsw4wP8fb6Vlf27qCr5cIT5ydr8E4Hpz6etNzQG9qusrpbxK0Jk8zJzuxjGPY+tZy+LVKsTaYx0/e/wD1qwdQ1KfUxH5qjMYIzjGfw/CqRfYrLye1S22ykkdMfF0hU4tUVvXzCf6Uh8WTHGLaMdmyxNcrvOcbevvUqCWQ4CAE+9CuFkdN/wAJbNwBbR5AyTk0p8Vy5x9mj4x1Y1zggmzgLxjsak+zzf3e/YinZhodAviuZnx9njC5x941Z0zxDLe3kNu8KDzCcspPHBPT8K5cW8pIyv61d0hZLbU4JpNgRGO4s+ABggn9aLMWh3WeKonV7UXHkFm35I4UnkVWPiKw2tl3UZxkrz9cVgW6D+2YxA7hZCXRUYEgj+9kY9abl2EkdsDkAjoaKr24uwT9oeMjsEGMVYqkIWjNJRTAWjNJRQA6ikooAWikozQMWikzRmgBaKM0UAFJS0UAJS0UUwCiiikAUUUUAFFFFMAooopAFFFFABRRRQAUUUUwCiiikAUUUUAFFJS5oAKKM0lAC0UlLmgAoozRRcDPozSUuaxLFopM0UALRRRmmAUUZooAXNFJ70ZoAWlptLQAuaKSigB2aM0lGaAHZopuaXNAC5pRTc0ueaAHUlGaBQAUtJRmmIWik5oyKAFzRSZooAWikzRxTAWjNJS5oAKKSlzQAUtJmjNAxaKSigQ6ikophYWikooAWikpc460CFqOe4it4WmmkWOJerseBXGeMPE2raRqtjbWtoEtncSNccP5qjG5MfwcnBPX0xXJXviC71hpXvbkoq8xqM4xyCAB/X8T0pb7A9DX8X+M5ruKbT9NVxbvGQ8gXLSLzuAHYY6/j05FYmjavNZQLG7Si0L7I3RdhX16cHqD+PHFZt5dyXMnl2vSMBTMeDxgg5HXHb6DvzVdry6gnZrvfMGXBY/MDjp1FXFWE9T0CPVbsLuhvrh0I4cTcfqOD7e1LFrupFlQ3bjPGDJnAzjNcXbXDpcm4sLnPHEeeTjGME9uBwe3rT7zW9XnjSaJJ1mjkAkiDn5hg5G373JPv2qr+RNjt/7Y1AMQLuSXB7Z4GOvB98VG2u6ghDG7yeePMwD+vtXNx3V1cIqzMyzJw6iTJTjIB6ZOCPz71M4d5dxlk2D0bB/EE4xV8pNy5rF7fPDDM0jGSN/MjfzN5VgCQByfQdvr0roPDXjmDUFSDUWSC4PAkHCN9f7p/Tkd+K5C6smuU2yMWAGQdxXHbHWsqW3lglHmmTYhyJUXJGeuV/TNS4saZ7tke1LXio8V6xZ6c0FnebQxXy+cqmCCNp/hHGMHjB6c16rod5dz6ZaNqTW4vZohMY4FICqcepPr/hWdi7mpRRmjNIYUUmaKAFopKM0ALRSZrB8Qa8dOVYrZ0MzfeOMlB/LNJtIDYubuG0RWncIGO0E9zWLB4qtpZmWRDEmSFYnPHuK4651S7u1xJPLIuc4ZiRmqjyOQcis3N9BXL+ran9svnfBxnA57VntNg/KMGmrtCk8sT1odEI649sVnuF0L5pyD/KtK2C+R5jcZPGeKxlVvM2gkntW4xEeyPBwqgcKTVwRWhHI/71Qpix/ESVH86ltmeO6EhnRgv3V3A9v8aX7QcYCufwP9aaZiVOUYD+8SAB+tXZDvYkjiURDdnI4x+lNnBK8JgZz071agXEOeOcAd6jnJ2/T2rbl0M+bUzd2H5q5btmRcNjg/yrOd9sgHGatW8oEgJYAYOT+FQmUzSjOUJOSef51IPug8VnR3yDdnoOnvVo3EaxAtjFUpJkMnc8DAxz2ppd1Bwxzg4Occ1GssbqCGB57U/wCU5+lPcEyCS8cAJKuR/tDPWrAukmlSQFxKMhSAM84xjpgjH61Rlboc800qDhlOKycexaZ0emandxX628s6TRn753g447nt9K6rNcBptxDbXSG8hUqDwWUNj+v5V2keo2kkYf7RCATjJkGM0R0BlykzTUkSRAyOrqehU5FOzWghaKSigBaKSigBaKTNGaBC0UUUDClpKKAFopKM0ALRSZozQAtFJS5oAKM0lGaAFopKXNABRSUUALmikzS5oAKKTNGaAFopM0ZoAWikzRmgBaKSigBaKSigQtFJRSAWikozQAtFJmigdjP5ozRRWRYtFJRTGLmlpKM0CFo70lFAC0tJmigBaKSigQtLTaWgYtFJRQAtGaKM0ALRSUuaAFBopM80ZoAdRnmm5pc0wHZpKM0lAhcUuabmigBaKSkpgOopKKAFopM0UAOzRSUZoAWlpuaKYDqKSigBaOe1JS5oERtcwpPHA00azSAlIyw3MB1IHU1zviLxRFpnn2qShLpFR43GHDHdgow6g4HT37VkfELUvsVzaBNRkhfyzugizvZS3B9MEg5/3a8323N5IZXDKkikNJJ8zHnt3HSkk5bA3Y1tY8QNfSB5YwypLJJBCOdrMct9Mntz36ZrPZZbiQtdEordIx1x7+1SQW0cJby1AbuxA3fn2qfy0QbiQwz0U/5/WtYwIchUiIQIiBEHcDg/40SKEXy3AdO+R/Sia4lSweW3gikkDKu2RRg5Jznn2otpnlS4a4t4Yljm2IUTb8uOM8/X8qonUoSWTqd9nKQeuwnr9OxqSDUmhZEu4yGRgQ7DOD9D1/DNW5UgyGEuGI9v/iqhIdgVmQPH2LDP8uR+tKw7llJElnLI2IzKZJFjYjc3HXJPfHGfTgVq2MrT2wMjRtckkMqso2nPbJHQHqMng96537MkT7rScxEZOx87ckYJyO/1x0qMasbCY/bYmkORtxIVHQ5LY7dOMdutNOwrXO0eJkXoGGOMY4H9RQ0eQFKblPqMfyrIh8baY0Kb1CMRlgZQdv6Usnj+wVwNqMgweVJ9fQew/Or5hcrJptJifLxlomfnaBkN+nP4VNFc6vau4AeRX2+b5ecuFJIUjrjkjHOQx9aypfiHat88UUbE8Omxs/nmmTfEGxaM7bVQ2e6Hn9al2fQaUjvLLx5AiTnUYrgS78r5cZYKDwFwMkYx7966nSr9dT02G8QALICRgnsSO4B7eleK2PjFtSuY7SGxE00vyfKm3OfX5un19K9d0yf7BosMU95DPLGpbcAEBUscAD2HHrxzWUkkrlq/U2aM1zs3ieIoDCpVupDYx+lVrzxawVRaxjPO4uufpjn+dZc6KOrLAdSBWLqXiO2siY0HmyAkHBxtP5c1yV3rN/K7FrqQhx0VsDHpgVmSSu+4s5JJycmpc30Ea83iO9aZ5EnIJGBjtz2rHmuHmlZ5XZ3PUtyahyPUGoi5zgcntUDRN5mGA/SofPwSc9aAcH5mGPaniSMdAufoKBN2EMmzsee9NYuAGCn3pXmJwMYFIJXPUUaCv1LGnYmvowwAx8xOew7/AJ1feXfM7B2GOMBc/wBKz9L3fbJH5wsZA9skf4VZE4wSZefdkH8q0jsWibdnHzyfipH9KXdjn95jPXJP6f8A1qgE0TD5Jgw7nzcYPpxT4W8yVRkkZ7SlvzFUgehrp8tsuTk5NUJ7lj8q9jyasXVwLe3TIOQBmsSa5Z2Yrxu6nvWs5JaGSGTSr53XmojKVbrwewprOBjAHvmmrg8EDn865ZO5poTpKCc5596nE25SN3TpzxVF0K4KjIHXPNAYDqCp9qSbQaGhHOysMcBea0P7SDbflHIyeehrDMuw4/I0qyZHFUptbC5TTebcVIHT3p8UuBgnIrOWTAxzTjKykdwTVe0Yamn9owSCMrSjDkmMlT71neZubjPNWVlCxgE01K+4bGpY6pc6bJ+7c8jBV+VJ9ccV02j64168kd35MMgxtX7pJPbBPWuKW4QrtbBHvSKQSSpJH93PWn6DPUaM1w2neI7qyfZMXmizysjEsPof8iuwsr2O+tVnjDKGz8rdeDiqUr6CsWc0ZoopiFpM0UUxi0UlFAC0UlFAC0ZpKKAFopM0ZoAWl4ptLmgBaKTNGaAFoptLQAUUmaM0AOopuaKVwHUlJRTuAtGaSilcLC5ozSUUBYWjNJRQCFzRmkooAXNJmiigAooooAKKKKLgUKKbS1kaC0ZpM0A0AOzRSUZpiFzRSZpc0gFozxSZo7UwFpabmlzQIWikFGaAFzS5puaKBjqWm0UCHUUlGaAFpabRmgY6ikzRmgQuaWm0uaACikozTAWlptLQAtH1pKKAFpKKKYC0UlLQAUtJRQAtFJS0wFpD1X6/0NLVLVruew06S6t4EneLBKPJsG3PJzg9KQip4k0GDW9MmX7PG14sZ8iXaN4IOQoY9ASMenNeQp5gjyctx1C43f4V7rDJ5sKS4I3ANg15B4nEVj4m1GGGPcnmBzjHBdQxwPTJP6VUXaViZbGT5mFx5JHpnNMYuqMdozj0qNpQ21mJT2JH+FSF1dcbA3HBxmtSBsTSMD8xz+NPfDZ/ekfUE1XkmjQLn5fYcZpPMjZsfL/3yAR+NMCZSy4+bIz13f0qQ5DDaGBxySCf51BuGR8hyfoTU0YyCcbe9Ah8m0RbpFIYdWBxn9Kx9YiXyEMedznlFGcdeScAZ+ma0bxikKjcnX/P8QrA1tG/tO5AbICIBlicDA/xqZTUNbFRMvyAtyDnDdDkgCrf2cuMb4z/ANtF/wAaZcIwuZQcZ8wn144x+lAh+YncB9/HHt1/CiWJ5UrJaq5RHFHDGXG4Lzk9T/KknRNvykMR6Zp6qArAtg7en4ipntZJFbyo5HO4jCqT3FU60nPktoHmdT4GshbRyX7oNz/LExJOB0P9RXWy3LAY5x1yO9cjoV62naVbhrOR/MkZWYEjYM9Tx71s3mpCDy441Es8n3YtxBI7nODXPVjJy0BMutJnkUwSlQRnr2rNuLq9t0LS2UKMBu8s3Pz/AIAqM1ctQ13Yfa4sOvO3YGYE+mQPwqFSn2DmSHl88ZqJzkcH6nFRmV1JVwVYdVP/ANemCbOcZJqHvYNRsjMo5ORnrQhO0EDr2pWk+YNjI9qYbkZ+YD64qStSV+cbtuQaRcqoyQDnJqs021yeSKa8xOCp69qLisyyxkzxg+mKaWdCFyD+lUxcOzBc9ulTvveDcVII9aNws1ua2llhb3c5HGAMgdSM5/pQGwPvTfnJ/hUdiPK0ZXBZXkYnO3PfHp6CpA5x/rpf+/X/ANjWyVkihkSqgf5pBuYtxvHX8Ks25XzAVEjH+8Wbj88Zqv5hz/rpf+/X/wBjVq2LPMoYuwJ+8w2/pxVRWopPQNekCzRKB/Dn+dZIfeOCfpV/W3zfKM/dQfzNZjyEAdvepq6yZmtiY5545HrUYPz8rk1ElyWye2cZpvmY+bn8axbRST6lwEY6de1ACkbex/SqYuDu6Y+lSeeepz9afMOzRZwAu0nIFMIwAOp65FIjAjuM08DA4559adk9guRCRgfap1mzjGTmmBt2V2jr0o2bckHaDSt2He5NjIypNPWU8K/6Cq6kp0/MVKJkOB0PvQA4EDqTzVq3lC53enFUvNXkbQT6Gnqdy8DGD0oi9QNhGSQAE5z61Is11bqVgnkC/wB1XYfjwRzWTDcEMPm6dRV6OfseR61spKW4tjct/FV+qBXjgfYBuZ8hj29ev4V1ltdxXUCSRup3qDtDA446V50wVwMkH6HmpbS7m0+XzoCrY7EkfoDTs1sG56PmiuTsvFF1LPDHNHb7GcKzAkEDuetdSrq6hkYMp6EHINCYWH0UmaWquAUtJmkzQIdRSUUALRTc0ZouOwtLTc0uaLhYKXNNzRmi4DqKbRmi4C0UmaM0hi0UmaKAFopKKAFopKKAFopKKAFopKKAFoopKAFpaSigBaKSigAzRRRQBnUZpKO9Zljs0U2lzQAtLTaM0AO/GlptLmgQtApuaWgBTS0maBTAWikzRQAuaXNNpRQAtFIKKAHdKM+9NzRQA6ikzRmgBwozTaKAHZopM0UxC0UmaKAFzS02jmgB+aSkpaACjNGaSmA7NFJmjNAC0tNzQOtAGbr+sNo2nCeK2a5ndxHFEpxuY+pxgAAEkn0rzWTxd4tdz/pwiUqDxabQDnGPmTOc8cE13HjVd2mWv7suBcjOFLY+VuSARxXECEsgEkbHjH3XPfoOTiiCu2S3YBq3iMvHLJf3vmId2D5iIT2yAMEeueKxby81OUXcF5ql5IqQmVg0suCBzjDEZyPbFbTpISD5RxuGS0Eh/Prn/PrXMa5OLC+ugFDCeDyyAGXbkY/iGatqwk7k1jJcPBE5u/MZhsYNIQQpBB6oc/TJ61ZjtVgQrDIojZhhPOkTb1zwIsY6dKXRYJH0uGVQ6sykA7CeMnvn29KXWrm406zSeONp13hSG3KMGraJv0KVxpt60hMdwAqhRhSxVzxk5K5HfsfQcU0eYhdw8gXGN0qsM8em3+tbaRH7Ks0jOuQCoYE59vve49KzNUQRCVsYKqflIGR/48aLWC421LXCkee0ag5Lgtj9V/zkZpqRTSSgyNKEY4I3Dgex2807RYWurR3EJkUOQzBDxwPQ+/1q9ezXdnpst1b20gMajHyuAe3PNPzF5FFrZ3jRIrmUknPRdqj0A28n3/SmWUepG9ML5lUr1UHb0xnhRWhpxlubYOFZXkOGClhjnHr7Vo2PnpexOxkZAMY2Sfz9afLqF9Dn/OWeNFIcBvvsjc4z2yp9e/5ipP7LS/mMk0jCRgoGJIssRjg/KPTtnJ7VDobMdSCxeYzpGWAVWJ69flOa6PEhcj9+rEt8+ZvT+7/D/T8KXLfcL2MUaRAEk82VWcfNuDW+w5POcgEc49an0/TYDI0awWkrISNxSCYMDnqVZTyePcetainanDyk7RuZ2lHPqM5Ht74/KxYeXPK24gOpUqTIw9RwMYqox1FJ6FK+tGgsxLGgiuYArK6IVEajuQGweFHHvnFaFnB59sl0ZSzuEkYqAxBx0AwvH6+9MnEr2V+WzhQwj2gEkeWO477t3WixiefRrdFkKkeUdx/2HUkdv7pFa8upnzaGbbRlda1C3ErJEoV1iDMigsOSFGQOo6enpVbxIIpNNWRSz7JBjfIJMdRxkdQT1z/KtBIpYvEl0xUJDJAMDOORgdPzpmoaYZNFNtHvU+YWBIHdifUDHJo5dAvqaEYt1uxGEzKQzdwSMgHOeO4rN8PTiDQkdiGYTiMArtxucAcgju2c+1bKxobhd2PN2Hbg5O3IyfXHT9Kr2toujaYzOxMCyfN5eWA3Mq9PqR/hQ1qJPQqaiMXeWUglM8tnue9VPOXGMHnrxVvxEwinh2lCMEfKc9P/ANdYyMcFuM+9ebW0mzpirxLyyptPGfX5sYqG4jOchcg91PSoxLHtBYDj04pxmjKEAt83PWo6Bs9CBd0jFVJAHUmpvLj2bB37k803gqOSAOelKAM7ifpQkNsiiYIx6Ag4yc5pZX3FQpJyec02dFIyoOe+KfpkLTajD1IVtxJ9uf6UktbFJ9ToJ0WGCG3OAUAPLYPA/wDr1CWTu8Kj/afNFzcn7XJ8kRxgZaUD9KZ9rIyAtsD/ANdx/hXQwQjMhOBLbjPrg/1q5pTrLIjrE0eTkcjHX6/0FUTd5OPMtR/22z/StHSkCbdo4x0H3fw9qcFqTPYz9YkH9rTLyQNv/oIP9apSAlOGOO9TapJnVJ+4LAAfQCqO4oxyfl9qzqfEyEiwuxTjAHt2pXwRyADjiqm7J+Vhg0qyEDBOfXmsixfMZGyxC+9TRzhhg8+vFMR0Yj5flHTI6VL5aqQdg/KmkDa6khCsDg4NODiNVyc+tMBw2ATj0oJA4YHnvVWJuTkg89+vTrTQ4cFTkH0qJXQEAMfbqak+X0yf50bjQ/cwGDxS4DrxjdTFJJI/h96MbeVDYpWHcVVK8nrQHKOTmkDh/Y0jAjJDAipfkNPuWBKp64PuKkSby+AxIPrVMg4Gw4qUA4wSM0JsC5HP8wCnjv3q4sqN3znpWMSy/LyM96micoMk5/pVxm0DRrsivywGcdR1FS215dWE6Swt9w9hwfrjGaoQzluC3brmrQlPUD61tpIVzo9P8Uu06i/8hImBIeMNkenGTXQ297bXYJt50kxyQDyPqOtedMiycqArU+2v7vTrgvCRCXG3OMgj8c+lKzQ7npOaXNczY+Koimy/BWQfxoMqfyP8q3ba7hu4fNgkDpnGRxg+4PShSCxY3UZpuaM0wHZopuaWgBc0ZpKKAHZpM0lFADs0maSigBc0UlFAC5opKKAFozSUUALRmkzRQAuaKSlFAAKWgUUALRSFgO/60wzRggbqYElFUmlct94/QGkM74xuNK4F2lrPWRh0Y04zMf4qLgXaKo+Y/wDeNFFwI6KSjNZFjs8UZpuaKYDqKSigB1GaSimIWlzSUUALmikozQA7NFIKM0ALRmkooAdRSUCgBaKTNHNAh1FNzRuFADqM03INLmgB2aM03NLQAvFFNpaAFzxS03NGaAHZozTc0ZoAdmjNMpadwHZozSUUAO7UU2lFMDC8WoH023ygYi4GPlVjna3TcQM1wChI8fKque4Qrjvj5XA7V2PjbV9Ogso7SS6gNyJlYwblLAbW5IPQciuFaa1nk3C4VGRSF2NHgc54ww/X1PrVU+pEy/kttJTI3Eq4ic8j0w+R2rIvtMgaSWT7Gq/KMAxY7fj/ADq/PdWccmI5Idr5A+WLbj/vrj8a5XxO7XCRpYz7SrfOBMg4P+6xGOP1rS9tiLXOi0mGQabE8aq6KCTm13DgnqSwzjpTb/ULHTViluvkQthC1qMjv2PQ5/SqXh66t4LBR5vlTKzkMwhHfg8kEn61HqUWnXxEU8uEU7s/u1LH65Apt6hobEfkyJGyBQsiAhhAgBH3skbsnio7uw+1NGjJMp9Ft4wCDjH8XP4mqtrqVlDEio6sETaA6KO2Oc81R1q6a40wxWNy6TK4bjYntxgDPX+tEmC1Nq30tbXKJbmQEjPnWykg8Dj5sVHeTW9jD9rmAREK5dbbJB6Do30qKw1CP7DAlxLGZERA/mpHjO3nk8nvzz+tQX91Z3kDQyykwsAAAkfrnjbx196aYMltpY7uMSxktbs3Cta/KxHH976j/CrUsBaMKiyRvjIYW+GHXOPnx69qyrK80+0tPITeqqDz5Ck9c8kj61PeX1vJYXEcNyqsVZV2+WnPYjv1H8qcmhJEtppYhvA5EjsAQVe3TB44Jw3r/Wr7oCjN5S5A5ZYGHB4Of3nv/wDrzWDol35dmVuZSJVPzFzG2fl6/NzycjAFaD6jaMvRN+OcJGDn8qL2AtqrAF0h2Kep2vgn2xJ71oaUGa8bKyBtnGN59OeXNZLXtq3Rxl9wyUhzg+px/X1q/oU1hHdMxuLdQsfO14gOxxj8OxqotXJa0NSMpcXl1aEMTAqlyQCG3g9uvbBpJpUGh3E1mN5it3eNlwyllB6A++eM02fUtLhvnmEq+e6KrBm3ZA9Bg89e9Z9jqccVmtpbx3E+2RjmO3PIZj6/XtV3Isaj+SL6CIRgSSRuysq4BA29QT/tCqskd1YWOoScF1eaWHzmIBGMjvjHbt1qJ11G5uUljtUgIRliaRuFU4yNoIPYflU32DU5Yv3syRgH5Xjhzn3y2P5d6OYLE8tuk11b3iSxCSKNhiPOW3bf4lyB92ot9r5V1BfzrgzM6NI4V9mQRhg2QAfb+lMi0MXUAlknuJQQMM0hCnr/AHQAPzq7H4ftbd1ZbSNSSBllGTx2Y5P50hnP67d21/LE9tNJIELZLJxzj+IAZ6d6yXBRSAeD3Brq/EUMUcEYC4k355BB5B9evTtXLhCZCMFmHOa4a6983py0K/llhgHAp4j7cr71Z2YwXwOehpGQyKcIOBxjvWPKU5jVbCgKR7470wsVJJ5HXioJFeJgrRkE9PehZWJ2tkY9sUAl1HtKOxIzWpoAX7RIzHO2M8DsTWPt34zwM9K2NCG83ZUHIj+UDgnrVQWoXFeZXYttAJOSQiH+bUhk5yJlA/3Yh/U1MlpnpHKv/Ah/jUn2N8cK2P8Arof6CtLFXKxnYA4nQe26EVt6W2YlOCOOcnOTxWS1rhSXIx/tXDAfyq1YXEcMUwVogwXOBjJH8z1q4aMmeqMi8dLieWQN8rOSCPrVd/l4AyB0JquJGTCEAYFP84cjg1jJpu5nqiSSQIMbcjHaoGmAyAgAx6U10kDfIN3tUQSVsMUbH0rNmisSpcHcAGIAPPtV6GVgctgL3z1NZIyJsYxg8irIkY/xUIckmahZWG44z14yKDllxkMO9U1diuwkkdAakRtm0FgXFaJmb0JI0KAnFSq4Ix6GoxKmCSCc9qAu1coe/ei3YLkwdWJGSW7EUucY6/WoyVxkgDPpQWIUMpJx780WGTbhvOf/ANdNEqhtpU4yaRZlYZ709iDgEZBpWuO9heCvDfrUfII5OKYyMjZQrt9M9aXLo3Q47g9qgpMk3dj+poEyqxUZ+ppgG4Ekce1NZA3OcGkMtRz7+c4HtUwuiT97p09KzWEigAHpSCQkgMcDrRzMOU347kOMdDmpmm2qQ+CB2rFhlwvDEnHYVOZH4J71rGo7E2NEbH5Tr/dNXNP1S606XMTnaODG5O0/gD+tYkc5XpwR3Jp4uPMJ65NHOmtR6noFn4mtZwBOphfHJzlenrWpbXkF2heCQMAcHHavKvMbbnPFWrW/mgIaOZl/3HI/lRzsD1HNGa4mbxLcyQQpDIyOqAO+QS57mi01y5hglBlZyw+Xdzj8avmC52+aXNcrZa5PG5kuJSYsfccc/wDATjmn3Hi5FOIbYn3kbH6f/XqopvoS5JHT5ozXBT+O7uJ9pitF7gEknHr1qGL4gX8oDQ2UEy9SyqyqB67iefwBNVyMOdHoeaK4+Tx1FDGd9oJJAo+WOXqfbIzj3IH0rFu/iZdQybI9Lj56ZLMSfoMUckg5kelZpM153D471+4TdFoMZGeWmYwge/JJI/CtuLxbItuGurONZe4hmLL+ZUH9KORhzI6rNGa46Xxlch1KWVuIs8lpyW/LaP5mpU8Zbm2mzH1MuAf0NHIw5kdZmjNclL42RMrHYmRwBwJeF+vHFU5PF97JnBhhH+yu79T1/KjkYc6O5zSZrzuTW9Wk3EXUu0fxZ2/ouM1TOqXTnDXtwWP3Vy2T75BJ/Sq9kyedHqNRyXCR8dWHavNvtGs+UfMuHEGM4MrKPUE5JP8AKsS++IFjp0TCfxAHm6iO2BlJ9srgA/UilyD5j143TF8jgelPe7ATpg+vYV84X3xQvjIWsxeOhHBu5yB/3wh/9mrnbrx74hvRtfVZYYzkbYCI8f8AAh8x/Emp5bdSrn03f63pungtfalaW/f99OqH9TWdZeNvDN/O0Ftrdm0q/wALSbM/QtgH8M18vrc/aN0k0pkfOWZzktUti6f2sqKeCDxScQufWsVxFMm+GRJU6bkYMPzFP3V4P4O1a70vXZHjlJtjEQyqQMHI65PNblx8VL+11WVIbe2vLVGCBS21j6kN068dDT9nKzYlNXseuE0Bq4Of4o6Qnh641CNSLqJVItJXCl8sB8rDIPc+uAeK4a5+M1/dKypZpACOMSkAfkMn8xUqL3Kue7ZPvRXz4fieSc/2ahz3Mrc0VXKxXPoLNFJRWJoLmikzRmgBfrS5pM0lMB2aBSCjNAh1LkU3NGaAHUU2lzTAXNApjOBTCxPelcCfNJu9ahBozRcCUv6Um8+tMopASbz60m4+tNBooEOzmikzRmgB2aAfem5ozTAkDU4H3qHNOpgSZpN1MzilzQA7NLmmZxS5oAdmgU3NLQA7NGabmlzTAdRTc0uaAHCgc03NKDTA5Px/pEGo6VFcu2yaBtqNxjB7HJHoMc15a2h7PkDw7ySMsACPUffr2HxiR/wjsgLKoMijJfb69PevOo4yXIM6kYOA0oGR04OOaqmt0ZzZkR6BG5Z2eA8dQwxkc/3uT2ph8MyGTmKAegyOQf8Agf8AWugKlgxEgbG4FvMQkHHQ+ncfnSGKQwgDenIIKtGWOMjPJI/zxWtibmGfDO4ZjiUqBlgpGOvf959OlH/COIhJZU+8SAGA69x82TW2isrhJSXkPygbF3A9uQCPTp6gdSKc8uwndKqB/mypB3e5I659efqaaiDZix+HopUZQQxK7QA3PHXgMfWnjw7bMGeQRHGejPj26kY/OtA8s8fm5J5AboMjrz/TJqV5JHWNVuJS/cRyHGMfyPrjvg9aHFEpmM3h2IMGVrXH97zCRnvzup0fhhXQZaINtwBnHB9PmrTWaQqxW83LgfKLjlR+PJzTQ7qQvnEq+cKZccj/APXTsFzNbwxH5jMJIMYw2TyMfRv61ZGjTsjM80ZEm0ndvIZu3PrjpitJHkMays8/JHy+ZkfyHPfvgYPpU5naLmRsFRjcDz75zyevvQ0FznX8MHBlUwyxSSZ3L5jZx7gYxz6+tJJ4Xk25QYXO5SUfg/lXQjyyH25Y5KnIycdR0z3yOevpxVfYJEMsQIH3nATAHPfH8v8AIFEHIwT4VuInVo3QGNiysoOAfXNX9A0Rv7QZZlTYAZNu0/eyBnkgD86sTysFPCx/PnmMsD+G4VP4cbOoKHVWcodxKFc856ZziqjDUTlodLDo9nb5dAFX/ZUIT+IBH6irUEdtGG3x7s9GzyePXJ/QfypwaNcFFy2cnC44+q89u4qjq2pwaXpcl7d8pEQMDlmJ4AA49f8AIq3oSaM9yIo0ZIQScDJIzyOMYx+uR7VKB5ckjQgK2cFhwD9ex4rCudVj/wCEck1CGQMkcJuFCk8/LkfTtVWHxLLfeDLjVVUR3PkSjCgkK4B5H6Hn1qdmx7o3p5N1vcM7KB82Wz82PrnFPSRJ1jmi2vEyKUZSCCOcY7HtXFWN/d6h8PL4zuTcJDKjtu5YBc/j8pHvVvwPcmXw2kbMrmCVohnnA+9z/wB9YprXQGjQ8RFhY45Cq4IC8Lnp06g81yUsp25U8g112vB5NLuB3CBuuTgMOh7iuJBXdlySD61yYhe8aU9h++SQnPzfQ1Y3COMjLBj1ziqFxqVrZKRJcxxgDozAfpWNceLtNhJxK0pPaNf8axUX2Kep07OkmFc/TioZhGqM5lCgcnccD864mTxZe3TFYI0t4zxuPzN0/L9KYYp71xNcXDyMQD87Z59MDj1/zzWip33FZrY6z+2NPKmIyFgOSVQ8e/Tn8KSDxRDp0VwscZkaZdoDkgAepx+PH9eK55LUvagxDaTlsg8k4+vt6d85NPis1jhD3AwsZBzyCPUFhn39vr1FqCQ0rE194vulBS3t4V5+8VLH9c/yrPt/HV5A4W5tLO5Xdklo8Nj0GMD9KytWnLyELEY0U8BsZ/EDof8APasKSUo386dkO563pXiew1LINvDFLj7rRBix9uR3rcW/ga0uEjlHEeAqqRknIHHb8a8Ssr428ytuOCeoNd2n/Ew095oXLSFNzLjlj1/Prz/jTUQvc2HdwO3sahSQK4yefyxVeHU4hbkTzqybQMlvmAIHP8vX+lQ2t/ZvIqzTruzlSM4/H0P6VhKk1sF7I1dzIeRg+lSpcsuOcevFPSVWPzHd6HGaa0UUmBkowznBz+lQkRzLqiZmhlXLABqQpHHEx2kHIxms5X8qd1J6HqBU5uCuAkh9QaTKtbYsCRNw3EBscYpQNzgux9MiqUTkL93I71b83IwCQzdAOTQgkSN8nAOc96eHwBjGR271AsjBwMDB9adweQR16iqBeZcBwBgrj2pytjjGfes0yyK4V8Z+tWg7YHT65p3E0yUlVbOefWpEcZ5IPtmqckv3Q/J7mnICxUKc56VN9R20L25D3Bz2oYgJkcCpIdMvZFz5flju0hwAKe1vHbozz3kIQDllyePqcVShJ9BJ2M/7SwOepzUqy+auPun6ZrPv9Y8PWhz/AGtGT3VUJz+K7qpL4t0QBzBdNKyI742MuQqlsDI9v89aXsZdS+ZHQlc8EZ9aZ5D5G1C3tjNcS/xJcnEOmAKOMtKM/wDoNNTx/dT70SDyJCrHeGDcAE+gP45qlRXVhzNHfxWs24K22MHsxGfwHWrsVjK0X7zzFPckgA/jXk8Xj7XA5KX6xAk/diQHHpnGf1qtfeNfEkpIOrzqP9khR+grSNOCJbk2exSWllbwme4uwqd2d1VfzIrCvPGnhvTv3a3K3LjgrbwmTH0ZsD9a8YuL+6u5C91cTTv/AHpHLH9aiEvJ5OKr3Vsgs+rPSdR+JsxhZNOt/KHIDyydR/ujGD+NccPE+uz3Rnk1S8Zj/wBNmH8qxl5OSPpVy2Co4PIobuO1j0fQ/GsEFup1a+SV8fcigbcPYnhfyB+taTfE3SEwLZZI2B4d4yT9e/8AKvI4X2g7iMn1oeVQwAwSfSlGKWwM9Uj8dR38phTUgMqzElCmAASTnaOgBPr6ZqjJ4x0lpSZry+lYfxBMBvxyTj06fQdK86tSwmYfdzG4yTgcqR1psDAkKQf8avmYrI9Mj8W+HFUeWwiIOQ0lsX/nnH1HNTT+L7C7jCpqbO6gnCJIQAOT94cV5mEGOcjJyQRVvRY9x1FugjtSc+nzoP60KTFyo9Di17QIwDPqNw4Iy0aB8fnitOPxx4eQLHDJPEQfvCLoPTPPFeNXspV1VTwRzg0ls5AYtnB96OYOU9qm8a+HTiQam7EHaEWGTJJ7AEYrBk+J9koxHpc7p23TCMkfgDivO4/3t1DGAzjzQzBOuBycfhVAsQ464ocmHKj03/hZlsuSmjSncMEPd8f+g1DN8UpoSFt9JtQo/vuzH8+K86Zzkbc5pjse45Peldj5UddqvxG1++k/czrbQgD93GgOT9WBNUh468S7QBqbAe0SA/ntzXOu+wYYEZGRnv8A5xTBKNgUGk2x2R1Z8e+J4tudRD+zQRt/NasW/wARNaiMhaRwH5wk8q8/Tdj9K49pgQBnmkeZdqjIBHX1NAWRs3Ws6hrrs99dzSqONrMTj255/OqckSpGSAQfWoYr6JBypB+lJPdRSqoUnGfTmgAErfKTxj+tMkC7QV/H3qAyhj1/SpAU9z+BpDDJXgMcVYtJxDdCUVDwwyqMR6hTT1ilVdwglYeyHFFhHbadcotpc3RI8xFBjI7E9KwVvmkfAjKsSSSp5rOS7ktleNlwJFGAef5fX9asRXisURYZGY8YVSSc1pfSxKWpYv53axQeZlXwMdxjPFZqTiPKknp2rq7bwD4n1qGE22mM0DnMVw80QVlPc4b/ABqGL4d6rFK6XstrAAOCZ1XJz7/5+lT8WxWyMD7bjpI+O3B/xorpl8ABlDLqVqVIyCLmHn/x+ijlYuZH02KM03NGa5jYd0ozTd3pQGoAfnNGcU3dRuoAfRmmbqN1AD80Zpm40hb3ouIkLADmmFs0wnmjNADqKbQDSAfRmmilzzTAdRSUUAOopKWgQZoopaAClpKO9NALRmkpaYBmlzSUUALS0xmVELucKoyT6U37RH5ccilmWQgKVUnr046igCbNGcVkQeIrK5vPs0Xms28rnYcfX2H1rWo8xJ3FzS5ptLmgY6im5pc0wHZpc02jNMDD8X4OhgEkAzKCQSOx9K87m/dYbzEGUIY7guOf93jtXoPi+UppCIoYyNKNu1WPQHn5enUc15/JI8chYlkJBIw7KRz645rSluzOZP55ZfnlizzyWjz9MFaQKMgHYFz0KxcE/hUa3chLL55DZ6+ZI3H/AAEYH9KPMaXGbrIyMnfJ/hg1qQh1wwY7lb5UUKMOOncccdTj6fkfOvEus3dxdSRpK6QjgKpxke/+FegSH5AGeIgf3WcZ55HtXnGtQ4uJOAMGom9Bx3KOl67d6fdRFppHgBG6NmJGPb0P0r06O5jnQtu3ggEyM/UfyPUenU14+4+avTfD0rrpdkcFyIk2k5wOPQf54pwd0VNGndysI8uyyBvmPORnvj0Gc1WhlDEAgqfpwKfeOwGzLOpHQ9/qCetZsYffuYEgcg8DB/OmmRY6UyoZHdigwpHYK2OvT3YfgTx6+deJ/EeoG/eCCd4YxxlDtZueuetdSt1KU2rMdo+bG8jn+XYVwfiTnVd2MZX1z3pTvbQcFqa/hbxJd/bVsrtzMk3Cu/LA+hPcf59a7hnYIUBU5JLEqD+X4GvK9DXOtWfOP3q8+nP0P8q9OUFF3NKeF5JJOf04q6crrUmpGzIJFQkgk84yAcHBrQ0Bo/7QiQnnBGTz27kj+tUbuc79nm7hjOAxbH51f8NhUvINmMlm6bhj5T3A29q27GZ17Dox2A8dcD8sk/oa5D4k86BajdjdfoeM/wDPKT1rrVVWLODhvcY/UZH54rkPiEmdHtVxgfbEPznGPkf8KU17oQeoy1dn+GNxGTuARwe/8RP4VFoIA8Baum3CqsxGccnyx/8AW60lsQPhxNtJGVfjqB8zf5/KpfDkb3HgjVYo8mR1nVV6ZJiGOvHWo6v0L6fMj0Bl/wCES1uOMKsZilOF5H+rq94CydMlIYkrct2HHyrUHhqyuJND1O0f5PO3xhnYHG5QvUH/APXVzTI7PwnpMk+o3KxI0oJ3LkhsD5do+9wM8Z7+hojpZhLW5ta5OltoN9NJJEiLC53NjaGxx1GM5xwev414j4m8Yy3kS21oywoq4ZokVDJ1HO0D1xgce3emeOPGs/iTUXWIvHYx/LFGWPOM/MR03cn6Dj3rimB6nvWc3dlwjYlN3I5+fBp8LB2A71UALED1qa1B+1xgdd1QWdZpdiqqJX2luwAPyehPr9MH065roxFbxwhZXEfyY++c5J4BHT0BwP6Vgy6nb6fBFCdzlRlgD1bA4J9vbp9ax59WluXaZmw3G0DgDH0oA7ePJV3Qhi6/dAAXOe5GOPftgVVleF4zJJO+EOd7nIxx359+voORnFcha39/LIVifIzuYlSRzgf/AKh0q42peY4W7ZcLna/l5APckc5+vPTpQhMdd6jbeYxjgRlIwiryWxx/Tn/69YV3ey3DFtsSqo4RIwABn1Iyf/r+nFX7yF2YT4jlTsykDj8MEfjUaLBOhV4ZwSezqB+Py5P51QiOGzjuYvNi2ggZKg5roPDupwQSKsjYdRtA/ve3+f6VnH7JDFGIbPaVbLMr4Zl7rnnkjocHmoYY1+0LMFK4bsen0oA7bUdLtr9TLC4DTDfk/wAXuPz/AB/GuM1XSL7Sm8x438oMV3jlc46Z+ma6OPWY4wEw8j4wNvHPHc9aeuoTyI6yRqyP1jKhgfqDn+lOwXJfDeqG9sUDgiZBt3H+JfX8/wCla7tIWUnHy8ZAziufhgtLa8EkFnBG+7GRKwz+Gdv6Vp/2pDFgMHBIyWHIH8qxlTd7oVy5hJ5CVYhj1yODUjBI2wcdPSqcNzbO6mKeNw3GAwB/I81YnvLSCVRJNGHbAVWb5mJ9utZOLG2TrKrx/KwBHbFJli2VYgjnp09a5rUfEqwMFhjDsDnO7AP+NYWpavqgu5raWeSN0kZGRDjBBwRx16U1BjR6Vb29zchjHAxQdXchVU+7HAH51dSDSbVC+o65YwAdVicO305Iz/wHNeVtY6xcQWka2d5P+6yiJEzYG5uwHGcVE2h60xAOm3Q+bHzRlQPxPStFBID0LVPGvhOzZVsLa7v5o+khJRW+pOP0X8a5XUfiJq9yStoIrBD/AM+64P4nvWPceH9Rhx5qQIO/+kxNj6gMSKsp4RmkRHOr6XhgNo85yec9QF46d/b1FXZ9EGhLa+KNX8ks8/m4P3pFDH9ev41ck8Uax54aK/ntxgZS2fyVPTsmB3rQi8Cy6foLarPd2V5aIw86JJSvVgqhT97n6DA5GaWCz0kTbjpcDxbclSzk8DsSx/lTUGS5IytS1bVG0q0u5dRvJftM0y/PO7ABRH6n/aNYsaXl1ueKGSVm4yiEk/lXfxXm+NYlt7NUiQrEiW6YTOMnpnPyjn2pJ9Vv1s5/9KkULGxXZhcccngVXKJSONuvCmt/2hc29tpl3KsUrIsnlnDKCcN9CO9WdN8KatDds19YSQQmCVC8pChS0bKpJJ6BiKt6hrOo/wBoTobiZIjI37tXYDBJ7Z54qhqF3PKuHkZlAyF/wFHIh8zLH/CB6nFgyXViiFiCTKeMevHvU9v4T+xyNNc39lKjRuu2Ji7fMhGQPUE8e4pzXcUNuoh+6y5jwMEp29ccfWqz6hLIq/NkbcAn160+VE80irHoNg7gnUJSFYblEGGx374/z0rQGjeH2AEk2ohtgwQyYySRnkfpms0XXz/KmwFQpORneO/06/nUvmyRwskqggEBT9PT86VkNORZGm+H7W0MjwahcSM/yKzoEIHqy8g/hVX/AIlcQyNFU8Y+e7c8888Y6VYtRC1v+/uBGznOGcDPpwaqHy4y4aVGAOAc8e3JqSgjvRb/AHNLsCAchXiDn82yTXTaZreiPppik8N6cbhvlMqxopi3H72CpZgOThSOOPQjm7eM3cgEStLjnai5OPwrQPh+8XTb3VZ7OS1hiAWISRkbyQSxXjnChs8cZzx1Aldgy1da5Ab66KaPo7B5mOZbJHOOmM4/HpVdb+OGBrpdL0tnBCbmskwOvpgZ/DtWZEYpgWRsg8n1GfWrE7OvhpwCeLtBgDj7jf8A1q1aRnqacHiFInydK0+BscPDAkZ6ZwcKe+OeP6h/9v6aApHh3TmfIJZyT+WAP8+tceJhkE84p4kMjMBnBqUVY7GDVPDFzcNbz+Ho4gQWDRTvwFBPdh/P86rxT6aJJHttMVImUqA0m4kEg4bIwQMdP1OKwdM2LdzZySLWc5Izz5T0/Tp5GtsSH5s5AK4/T0zVQtfUmdzWjTRWQb9Fidskk+ZtB5z/AHfSmS2vh1zIP7ImXJyrC9bjOeMbcen5VGrOBIGO7nK80zfmL7pBJ6GrsuxF5E9laaVYxm8FszypKECNIc7W35OcjoB19SKs21xp0ERVNGtWAOcuu/8AD5gT+tUIkBtppCF3CSNRk9BiT/P4VCHHlnc+3PAJGcU4wiDky/BrOn2CndoumzE9TLbqxxnoBjA68HGfc4qaPxVBMSLbQtKgx3W1ww/75IB/Kufnh/esrtnjdwO341YjiS3tiqFc9SO47Yp8iuHM7CahHfXly1+tvC0UgIxs4Uoozgdhgj86ovNG7Rr5MQ28A/Njr9a2jq9zaacII44RBMJI8sMHOF3fpt/KudbHPpWUlqaR2JHnMs7SukaDcTtjXH8+1NCRytlnyp7Dg1HkfhUJyMsuMZx1qGUh0ksWcKgyO55qW1u4o3/ep8noADVPAJOTjvk96cZYjHGoiCsudzbs7uako3xfWBKmK1hHHJOcn8alg1aCO2YTQGZ8/KwYKAPpjmueRkI5z+HU+1KXzkhsD+7T5hcp0lnqMM8qq6onpjNdE82lQRJ5pcTlWPkhSxbAB69gcg5rz60YGUDd8xPGSAPzNdzcTWC6As80aRXLSBbaQAnbk5I45xtzxzyKcJESjqOlezDjNmyucMo8zII9+AakgubSRkja0TzC2AQeg7dQaxeUumhSRnjDb1xnrxzg9KuxeUwaTco2AqOvpx+tWmJrU9e0nxBFb+CJLhTmSxtiCjqAeB8nA7EYrxu41jULxX+0X1wyO24xmVtmevC5wPwrWm1u3j8GX0QTNzJGIzIy4yPMXjrxxg/n71xTXLFjhs47jpURSitC9Wjc/t/V1+UazegDj/j4f/Giud8wf3B+Rop3Qcp9clsU3NNzRmuE6B+aM0yloAfmjNNzRmgB2aCeKZnFJmgBxbPSkoopgLQKSlpALRSU4UCDpSikxS0wFFKKaSFBLEADqTTh9KBC596UUUjbtjbMb8cZ6Z96YCNIibQzKpY4XJxk+gp1cjr0Gs3V7BExghhCsQ8cp2k9gRwTjGa0NL1ZhOLGdzPMuC0ixvtAPQ55zz7inFXRHPZ2aN6lqpdXq2zW6sGzNIEAA5x3P0/xqC61ywtdoe4jAd9gbIIHqTyMCkVdGlS4rl7LxVDBY7r9maby2m4XHyAtzjqANvv1FbemapbarYxXUD5WQE4z6cH8iadgUkxutyzQaVJLCWDIQSF7jp6HHXr2xWfc+JYRpzXcasAhw3zYO8EZUAjng5J9K47WfGl2JrvTk+zz24d4yzxkhgOhOehzzj1HpXH3d/c3duEaV3j3bdu7IHHUD8KlXaujGdWzsj1VfG9gL2dXnBhKJ5IVcHcfvc+39D7Z5WLxYlrBNFEt1ubCq5cL0II4AxjjoMZ9a5dIo9gjE5JI3bscDj1OOufeojb30DGbzUjZMsHjbnIx0Pr6VbjZ3Of27btc6hdea51y8vjHKRMu1gJShHC7TuHoex7ema39C8StfavAsn2mFpWCJbxsrK20YJPdQMZxn1rz6x1ERRsZUZnkyqgMyjbj1Ujjj/Hjipo9UutKlM1pCPtMoysmSSoHJJ/z2IJoVlYtVHfU933DOM89cVXhvobiZoUf94oyyngjnHT8K8/0vxP9i0oyG6iuL27UFCpMkmeABjc3v1xg4+XFb1jqDWXiPUYr66PliKJlafCbflyQMHB5PXA/Tk6nTzI6uis2DWbW4DNC/mIG2qy9H+nrU6XqMdpK7sdA1BW5bZ1RSzEKoGSScAVgXPjDT7a8NuVdyoJYrz/n/wCvWT4n8WQQQS2cBZ5R/rNo6Y7Z9a81n1Tz8s0R3BieD97n0pK7fkZVKnLojudZ8Qx61DhT8sbAhQM4+uKx2kMu/aWyBjq57549BWFZzlZflbYpB4PPPH0NW5L42kpaST5TkHcxx9eK3hHlREZua1NCaPHJOM5P3iccevemqw92PqSfzqGHVIblAsTxyIMZDCXn1xyQP89cVK0uIwJDGwCAAqZQAc+nOf8APpV3ZdirqF7LbWU80ZDOikgM2B+PrXIXSPqVwgkkggeQZdpGKop68nk1va/Ip+xyPLBBEkzGQktk4HGVxwP8fY1o+ENFRNLl1KMWN1NNO5QzkhkAI2EAgnghieRnPXgVdZUoYfnV3PTT+vvNIRW7djyy4jMU7xkglWIJHIODXo3h+Y/2BalQchAOC/bPpx2pniLwjaLqr3ls0HlAK0lu7MPmx8xO0dyCeCOv4V0OixW50xNsEcEe0gJDwAQxHLHk9O+32bNY0VKavaxNSSRgLeQ3TMUnXzO0aYeQn6Hr+GaaYbkHzUljjjz8rlBksOoPzDHWte6YtrM8cjYgEDBUMfTv/ESCM+u761TupFitoEMuUYkgM4TGcdOD7ela8lldmXtEVR+6lCMTnAOWU8jgg47da5DxKrDUELHnB5xjPNdPdXLLeNmVyBgDdkcYGOB0/CpkmeX935jHJBxlz/I9KmUbouL6nE6KxXWLMgc+av8AOvTUYeWcEk46BiO361TSAKV2oWO3klWHOfc9fpU65BJaTAJ5whx+WaunGyJm02DqGXai8jHY/jxjmrnhvcb2IbWI3NyqN/dNU28sx8uucfLldxPPbmp9HYjV4wcqQ/Ycn9a1SM29DtU+8TuO38v/AK1ZHiDTBrFksERdiZVfCkfTgHA/i7VqqcISVYY5z1H0z/Q1Uv5pRbPJagF/KYqDjrjg45/wpyWhEXqU7TTo7Hw+NPlj85SWDZBycknuOPxFWdNtoLeyWKHZskJJC8jI4PTjPGDj0pulSy3elWMkkUskrW6GQIoBDED2wPocVPpWlix09bWWZCsW9gx6nczNjrxjPUHtUW1LvoZd/wCIbbR9LutRuI5THDI8EcfA3urMMAjoOCfYfTFeKa1ruo+ILtpbycldxMcIJ2R57KvboB36d+tbvj/XRf6k0FvIWsbVnEZDDLMWO4k+5zg+mCOprmrq3htbBN8oWYjLKvRfYf59axk76GsV1MyWFm5XLE9OevOP51SPJp/muNwDEZ4PrinhGztAx6+tQWJbxl5cj+Abj+dS27ol6soB2LluT14pg+XIB5PpQFO4J09aAHSyNLIWJJJoHOM49qDgNgdPXHNSbflB70hmzpiF7VlLfKuSdxOM+nbH4Z+lPlgeRTEY8sPuKyH5j6jAIx6jryOlVLa48tBH8qliMHALZ74PA/UVYa6HzO0qtGcA4ZmHX+7gA+3Xr+TEZZe4s5Su9lYAjGOBnqMVpWl4txKBcRx5c4HbaPc+/T/9VPcR6mFGTuHRBj5eevHHPHqakis7fTys0zhDnoHGSMe2eufzpgXptMAjaQcJgcn1A5x/n19MmtHbiA7mAJznjnA96sDW2m4gTCjgt12gY4Hpxxx71b+2RlVLptcqc7DjA79fx/LHsWIht5IQTIwG/GFJA5/Tt0/GrrAxbnaMADgYbn9Oef8APFZZ1W2t5S9vAm4cB2OfpjqPb8qtLetcBdxVmOeG4OT1zjnP09D07sDKvLqea+2jKAY2hSc496u2sQmiknmkSRIWVNocHbuzgsOvO08/TpkZbdRswW8QN5TvtMqADLgA4XHfGPzqreM1zFGIykLJHsQNgbzkE5bA4ySRnhemeckBG2mo6TbyLNfuRbEgGOJQHxjOQOPy47896w9T1LTry4DWsbBF4VmOGFYtxp+qSO3mWsxCZy4XKAA4zu6Yz3ziqZHlNsyCR1x60rlNnWabZ6XMpnuXubgLnESkIucjCk8noc8Yrov+ElkiupXtbK2gd5GkklWIF3ZidxJbPU5P41zHha0upYppDhLd/wCOQ4GQew6nn0Hauh/shVuJAt9DI2GkxChfAC7iOSvofXpVxS3Mm2zP1PWry58xXuZTvOH+Y8j+VY1zqd1dndcXUsrdN0jlj+Zrp7jTtJ+zu8huSxOCVkUYIBOANp9O+aotbadFYxONORvNBIkeRy/DEeu3+H+7Q7grGA9x8oDOcelR/agAoV2BHXmuqjngjTJ0zTSo6F7ZWb8fX8qhudYs0/dnS9MG3AyLNB+PSpuUa1t9ouvA8sEUhaBbuN/nIUDA5AJPPWo7K1uZDuaJfKKkZNzEDnaevz8c/pmq9xrzTeHHtU+zRx71AjhQLjnJ6D2FVPMZtIhkDHLTOMZ9AuP51aZFjTMNxBL5mbZFxyTdRkH8mNPdJmmkt7u6treLy3jfYXc/MhHZcHGemaypoRNYhsnaBnIpmpXL/wBq3oIIH2h+D2+Y8U2JakeqQzS6gTAwkV5GCNt2gjkjOTnPXPbjqaZLaMYYjJMpZnCkKp4X+8M9fYd/apnaV4RIo3KuSxPUA8f1xTrhy1qGyCAQQalK427FnS7KwlgtE1K7ninjYIqQRKw7DJLOOw9PX8asp0qKdVhF26qSSXkQFuePlCHHGeMnk9eOa0ExkvoAzBR5qZPYDcMn8qqqzGWNeCwYAj1osM1o20i2kV30mScgbgLi6YjPH9wJ6dK24PEunzaS1smiadbyENG0kUAyyMhXGTls8g53dQK5+dgI5SPmbB28VVtYJBaG4bIQvsAweoAP9aJRS1CLbFjS1t5CYIFIDHG/qOvcYrc0bxTqOhJKtpKm2QgEOuV4z+XXt6CsWC0uJw3lwSygPtzGhbn0471efQfEDwlotC1IpjORaPz9OKi5VjZf4j+Ihki7UKeOE7enWiPxzqU8QhnuI5YTlWiaBNu0jBA444JH41zJ8O+IE+RtG1EP02m0kz/Kn23hzXxOnn6NqkYbOC1nJ2+oouFjamuLBpJf+JXZLE7ZGFMZJxz90j34qyP7J+zC3Gk/Jv8AMZTcvjdgDO7qO4x0qi0EqSvHcRyROPlaORNrA+47GpgykYDDPtWxkRajB4ahEUbafMhlGSY5WJX5iOMnn8adPoehwT7R9u2hVO7z4x1AOMFKztThDy20jhSRAeo6nzHqTXLrOs3axlzH5x2l/vEDgZPc4H51PUs0YtE0uLLw3brGyPG3nW/mHDAgnKsvY+naqA0KO3eNY9XRgeha2YADt3z19qrNqEht2XflCMAE9KhS+ZVyfmIwBTVidTXXw9cHITWdN3up2K7MBx36Z7ioE0O8Ks6T2M2D0jmZM/i6qP1pbB2vGkmkcKsAztVclsnH+fwq+yRg4RuMd+KEwZXs9He5s5bOSe3t3klR93mrJkANx8uf71SDwwkd0ANTscg5yzOM45/uYoZ/L/1YBOep71Ul1SWCYh03bh0Jxz2/lTvZCsXk8JLJL5h12w3ysUWPLbcjoAcfpj0qW48D31vsa3eC4cDhVnUcY5PzYGKwxqck17HJGAhjPyspJ2n1B9aty+JNQmMoa+uHDABt0rHd7Hnmi7uOwur6Bqo0exgjsLiWaKeZnWFfNOGCbclM/wB01zs2jaugJbSL9FHGWtZBj9K3jq1xHAu24lTg/LuOOuc4zgH3p0XiC82skt7MwI6FjgD86lq407HKeRcKQGhkDN0Xbz+XWtnTvCuo3cEsn9mXcqtGfJ2pj5s984469K6Oy8QX95Fc7725G2LcCkhGDvQdvZjx05qGfXLyCzmdr+YiU7DvIIPtz+NJQuPmOQk8J+II3KvpN2DnqsLMD9CAarvoGsLgNpd7/wB+G/wrox4hv0UgOgUEgFoUY4+pU/zp48U6irqwMBABAzBGM/8AjvWk4IfOchNaXNowFxDJESM4KkYpm1mUFVPPGeua7uXxVqdzF5UroV6g4OR696zjfO8TrLDbtzyzQhj9c4o9mg5znba1meeNNjKXbAL8D8zxXpWoaPcXXhiCGzgS4kjG4hUYls5LAAZ64UCuORAkguU2pIh4eNAvP/ARj9K3dM1u8vX8i5vJHLZ+ZiN3boe1EIWlcmU1uUxp+pu8irp1+hl48preQNkdulT2+jeIZdwj0u5WNzlzMPKAxzzvx6iui0/URYxSAXMjhPlVZjuC5x0B4z9Rxk461cl1sRooE5iPPKqvJ9c4/lWsaL2uQ6y7GVB4G1i/026hkhhiZoiYi86D5wQQMBs4IyM4/wAa842THO2CX5RlvkPFemXviy6tNOlg3NtmbYcBQeec5wfTtWani29LEtcTbjgbQ5GO3apnBR0LhNtXscHvkx/qZf8Avmiu0k8ca+srqrKVDEDKt0/Ois+Vdy7s+ic0ZpAaM1wnQLS03NJupgPzikLUwmjtQA4c0pNNoFADqWm5pc0ALQOlJSigQ7NUtR1e00pYzdMy+YSF2oTnAqa7vILG3M9zII4gQCx7ZrlvEuuaPe6SFFwrSbjsJQfKcY53AkZB4wPxFDfYmUkja1zWI7HQ5blJhHIx8tCy5wx7Y+ma5zTvHYg0+Rr1fMl3BYgvQj3PoMH1Nc7Jq941h/Z9mHSN0JmiRMZUd92OQfXj09Kx2i1CQLC8UsEaqZFVx7H5sY+vP4VW1zmnUk3dHV3Pj66nEiCONFY7kAPzKAeh7Hj/ACK39D8UzXuoeXeIihogVET7gvzEZcjgE8fp6mvMhptysMJ2MUkPGT94Een5e/PetaG8htYYwdPRbuFgwnBYEYGAOuMcenrwCc1asZqpOLvJnoWra5qNrc3C2dsk0dvt3/Kx4OBnPTIJ6f4Gqlz4klNlPIzRRtLao8a7mGC2ee2B7nH6VxK6/NaO8kTuqykMyRSlQOeeueSPXPXpVDXtXbWJ7SK52xiBFSNraLGAegwTnH16frUOLasX7dO7JZ/EmoXNvHbtKzom5QTwv0qP+07iKWJ182OdVDAo3vgYHNc/Pb3lpneTsD7NmTzkZ6jtirZCmNJkeQSICGST5gx7kccdBx/KtFoYPXW5v3PinUrqdmubhsYXGxyAGAwpx0zyfTmoJNWlmaS7Mx25IIZmJOeCeOcYOOe3rWClzLEh279qqcc/j/MD8qkjuJ4okaZpIlIIG07R6H8D/P8AGml2E1J7s2pL8XdmEBxsXaAvQdSSf0/OnWmsXVnFJZKxSJ4fKTaxBUZyccjuDn1rJijYKnmAIDhwwkB4AyeBznp6Y/Hi5cQ2kkQCO32sAbVCLjBxjr0/+vjiqcbkqpy6JmU5aS4ZZJA2WxnODk+2OpzmrNtGCIzsYMPlcycZPf8ACo3torSZ2aTcZHb5D04//X6etTXFwYYkR2GQoGTyGHtn/wDXUxjylTnzpKI+fzpWdo22KegxwMdif17/AIUu+SKCJZPMRmySXHORgj+vNV49TmXDWwKKQVcg4PUHIAOAOBx0qzG0zIqSqXVBnJG1SM9cgg+v4E07EOLW4NapI0b7MlScFG3Y5znH40IoZQoIKjjceCcnB5I/zmkSFHtlkZxlfuvJIB0yOw68dPaqklw0VwN305bcOe56569xSejsNRb1H/b5LW5lkWdz5jnfG7HEhOfmPTP/ANf602a/fUJ2kd3LE7nIy2ewHHQVFczR7Hj8tcuMgRjjr15+h6VBp1heahKIrK1eWQ84H8P1OcD8TU2uzpirrU7LRvEOn2cUY8lmuowWJBUDgdBk9Mc+ua118WiTylihdWYnJPfODkdyc+o/xrBtPBkVlHu1jU1zjPkW5JO3/abHA7dPxrXW6jtIktdLs0t4m4Do7FyPUMc56dgw9xWqg5blJOKsmZC6FdyzvLdSJbxl+ZLgkZH88/XFSHTNFtyyIZr+4jALbmCKnHUgDIHuePetN2maSeV2dl5QuWHQjnBJ49/n/wCA9KrPBGq/KMbXyMoAVH97oMH32qefvVoqaSDRO5ky3bwbihjgXbgrGAm7jPBGMn6Mc+lYjlzMXcOQ+TlmJOD+Ckj862tUktTM5lKu5+uT/Xt6msS5LzcRxYGOg6fkOKho1iyaBljdmj24A+Y5I/pnuOtXIr0STssaDeqg5VsnHA6Y57dKxV064uJFUIdxOFVRyTXZ6N5+i2oRNEkUPjzJvtcJZz6lD0HPTJx9eS46gx2lzxm6ELHUvtifOAY5U254z2IHOMkjr71Jd6baWep2Lx2cStMzF1jUkHB7nOD0PGMHHG6rurIreINJLK53JMZnBOD8nXGD/tenGOTwah1OJrjWtLkcENztOV2gH1DKc/gR9K1UUrPzMnNvRFrUsjR7htxjbyyYlVQojG3kD0/SmeH3ZtGtfKVNx3dfmOdx6cnnrnvz0rWu4Eu7WeJkYeYhU/IOARj8R7E/RhWULk+HdFwEaEqQon2DCFjkEgYyASexzj71a7NtmV9CjdwMdZM4UgMgQMwUFjxzgcnHfAPvWbqumR6jHG8tzL5ykncrZzzzzg5Oevp/LOm8Z3d9A0ckiiRSVaRCQf4cDv8A07+vFSXWrhrWONJWUoT8xxhvTOB25x9azc4tWI51fUq3crQXGyYfMp5k6jHp9e9NsL7dcyLNvEKnKnaOee5J/wA/hVa8lF3MzFdpJHI6E461UdOGiLSFT12j8ulYc2uhceXodVp+oxX0qxR7VIRi2dowR2GTz3/MVoOHYcN0Oc/Ln9Ca462NxBJhz5Z/h3DAUf8A1/6V0K2iQwyyvqcDOF+VW3HLZ6fd9M/pWsJaaj5t0x15fx2UCuNrM0mxlHUH6dz0q/o84uNTgkiGfLmCyBRlQARXKrapqVxuF0sbgD/WcAn2PrUWnTxzapBbxsyb5lTryFLdPb601UfMQ31R7mqEjdxGOhZzk/l6fWoS9sweSDzLuYDA2j5Ovbov86c+lRoGlu53nx8wVjwMjAx2BHXgVYub+0tFhjlmhtTKVWOMH52yeg7/AJnFbNjKkUGpSBjMwtwSeP4gOPX1+lZ3ieaPRfDl9dwF5LpIQEY9AzEAHJz0yDjHbFbn2m3kk3J+9DY25bhvwH+Ncx4/unj8IXYlCQi4KRxLjbuIZW5/AHk+nelLYa1aPAtVmy4jGMrxxjGMev5Gq89/cSO5MrEOOmfWlvlG7cSd7Y79BVM1xnYWrWymvCzoQAM/Me5xnj+fsKSWTyyY1+90ZveruirqF7qEFrp1s1xcHgIFBzz3PYe56cmotZ0e50O/ayuyPPX74HY/1Hv3p2driur2KSSbc4xu7VKq4X3PWooxg1ZhVWOWIH1BqRj44SAZWBO0dccCo2nzJlecc8ipZWEqFEBx0B/zn+dQm2k8veFJHYheKYizvEq53bWxgk579e/+c04o7D94+VGAMjn6e3+frWarFGz0I6VOJHZRuLH0H+HoKAL8LLCBGi7A3WR+fyHT+dXX060mO99QMz5GenpjH+T6Y61kO0SjMspLdkXp+PrUQumLgoqpgcAcf5/z9KYHSbba0Ecix/KOQS33e+eMZ/L+lVmulf5dwIznknJ/D/H6VRhv5pV8tpCgPTccg+nFCyLbEs6Zb+DDcZpiNCa1hSNXLguP4duSOT3H8/TPtVgJ5NqqmB9rJl9wx1AI9+OTnj8qxv7RvDJIVnw5wScnPHfp7fp7VXN7c7MG4AB5HOce/HQ8Ci4G8WWNnD5V2TAlxnIBz9euP0z1FZt7J8u4EfLwMNWUZCBhWOTy3uadJKzkgnIpXBoa0zldu87fQU1BhgeMikZe+Dg1PZQeddxIc7c5b6d6W4M7O0dbPTbVCwyE+ft82c4/IitC0jubWVpZoSitC4XzsRhtyFRgvgHrVR9VmIjSNvLV0KsE+UMuOQ2Oue9UpEAvpVjxGoRWAU8cqD/WtbEXLbRXU5kRjCqkg7mkzg/8BySfzq8dPxp8ESajZIUPzMwlK8sehEfoR+OayEmXyGAf5gwNOuZJrqxENqWkkOMLHyx59BTkrCjqaI0eAnbPrtoE64ijkYn8Co/nVGfSbU4I1OKQ99sB7e5NYzWt6smZQ4IG3BHP0xVZorkglVc49qi6LszprCxtoYJDc3EdzAXAEG1l5wSDuDDHAPSpk1Kzt4jCum24iQllBLuSTjP3nODwOnpWR4es5tUt7mBJo4mjZZP30oRdoDA4ycE5K8dcE471oHQr5ZPnezIVuQt9Af8A2ei/YVjbvI9OFxLHHZ2/ko5UBXk2sM+z4I4qnePZSSzzzafayzzszuxkmHzE5J4k/wA5/ATXWk3YllcS2hTzCV/02Js5Jxzu9u/9aqDTL97pYpI4sbtpCTo569lDE/pVadRWZZiurSxt547eytTG5Kl3DSErnOMMxHBAI47CqF/JaXMbEWVtGcABo1KFSO/BGT9c9aYY5S5hETs3LYSMtxnHbpzTEsrq7KRwWl1LvRivlwsSxAOAOOeR+hqtLC1Hq6KJTshxIxc5hRsdeFypwOTwPb0GM7ybdUEoeTcpyULZB+oOc1bks7yKMGaCeIMN2JYHUYJ4PI7moZdLvxI6Jp927KRnbbuevToOeopOw1cuWyQRJa3EIcTbGLFyCASWXgY6FQODnqT3rbOq311brDPctNGnKpKA6qfYHp1rGtdMv5hAiWjB5l+UF1XPU85b5ePXHQetdFaeE9cKlnsRGpP3nuIgB/49/KnFx6ktSMWXxX4htpDBbaxeQwphEjWTCoo4AA7VWm8Ra9KcPrt/uxnm6cZ/Wr1z4N1prueGAWF6IuTJbX0WwZ/32U+3Smf8IJ4pkQhNOTA5LG7h/nvqHa5auY0mt6yxIOsX7bTgH7U55/OtTwj4m1W28U2W+7uLhJZVhkjllL5VjjjJOCM5B9aX/hAPFBbd/ZyMT3+2Qn/2erem+A/EFrqlpeSwwxJDIsrILqMuQCCQAG//AFZqWrrQZf1vVIJvFV59osYHjjuXjlCjY5CsR94EEnjvUUc+hzOAtrfQqc4KzBz+TZ/nWFBJJcM5OUnO6QllKkqRuJGeeRz+NWYhsUHIGfetW9TOxqXOm6TdxwB725Vo1KDMIPylywz83vTb7wu2savdTWN/ZRpK+Y47l2RsnORnbjqCBk9xSQkBxuP/ANenNLy3p0KkUNBc5/VfDWraRB59zBCbbH+thuY3VuSOBuyfuntxjms6Kzvp1xBYXkmME+XA7cMqlegPUMp/EV0OrXceyJQuGaCLO09SUBJ4+tZdvNlWidA6g42sARU2LuWrSw1Gy0qW6ls5I7aYCMyzAoM9RjP3skdqFnYrGYzv+TccHOc9KfJeWvkqJtOtWwON0KnHPTp0qpHc2ttMbqK0hE0YyqNGpjPGMFTwRQronRkx1CKOPa8saNjBDHHPpVWcrcRhxJGQM7ih3ValiH9ptKtvaQyGQuU+zROiH+6FKlcDGOmO9dFPrNwJry2ksbCFTGtvLDFaqER1G0lffg8nOM4GKbvsNJI5O3spxpl3cRlVjTazZGSckAD2Pze3GarxQeTIfNdfXA5z6V11rfpZ2D2502wniKgOJ4Sxcqcgk59RzjHGRxk1F9t8PzRN9t8NQefghWtLmWFVwPl+Xcc88Hn3wTU2aHoznrtCrRbirMyD5Vzx7fWqbhuo5BroL260a6vhLHpU0ECr8sUF2wJY4yWeRZMgc4AA6+3OjaWfhB4D5w8QruICAGAkckMScAEcdsU7iSOf0p1AuFdiB5eVHZjuXr+Gah1Ur9mRAynL/Lk46V3Gl6H4Sm0yfzNXuILychV86Bv3AB5+VCQ2QMctjkcA1ch+HHhu73PP4rDKU+QRiOMg++Wb16cfWjmVg5Xc8u5YDOA2OV9KcJFBZNo2DqSCea9Fj+FlmInKeLLd1U9fs3HTg8Ocinx/DSwGA/iu2U9Rm3OD+BajmVg5WeaiWQnKthc9kNNkmcMAsrcnuo/KvT5fh1paxFZfF9mjbh8/2cAAe4MlQD4Z6JMGB8dad9BAue5z/raXMPlPLpL2cFlKowPTg8frTjDcSqt0FGI8A7ONpr0SX4Y6AsoH/CdWZzjOyBeO3XzTW9pXw30W4Rbey8WRyXRG7alqHYr67Vfp05JI57VPNrqNrseWQX13HHIQ+VIwytzkfjT/ALde7djRyZ4bHojAEH9f1r1p/hFBLFv/ALfVd56PYmMsO/3pM1Rn+GulW0v7/wAUFIyoUs8aOMAYAPz9sYHXpitFfoZtLqeZ3cjuiEvkKRzkZBIzVJJ1Zw+ecdq6vxP4O/sPw9BdDWor+XzEjeKOMjy1wxySWPTG3gAYxXIgMqhW3ZA4zRO+7KiklodRFbLJCjkJllBOW/8Ar0VNZ39utlAGntgRGoIKrkcUV5bqSuaH0EKMimFqMk1ZsKWpaQUo5PSmAVC92kdx5JVywXeW2/KBnHJ9fao5dTtomkAdXMYzIFZf3Y9TkjArD1pdPvLyGbUfs6QRYlhkMgPmKACT/dxk4x3waEtdTKc7L3S3Y+KLS7tHnMFzFs3fIYWJOM5OQMYyCM9MjGa3a8sl8bWmn3BsrW18+CFvMERlJVFLcjkEflyD36itiHxhZGORtPt7mJuZZVigMsecbiOOhIBPG3kZNayp6XSMKWKjL3W7s7Z7u3S4W3aeJZmGVjLjcR6461NXEaCLu41I3txO91bs4W3E8O87WGQ6sBhCV256Z7jgCu3qJR5bG9Opzq4CozdRLdLbscSMMr7+wpl9bNd2M9us7wNIhUSR/eXPcV4j4mvtW0i7k0fVJWu5Y2RrebeAFjGew559CeOfrUJ+9Yc5OKukew+JdMt9R0ljcSFBbkyqd4UZAOMnBrxO91cfaOUxDEyiQLJ/rAD9c88/TP0qxb/EPVvKntZ5IZbQgL5LplduMbf92uXN08gIAJ3ZAOfmOapRak30MJtT6HY2evxyxpFYtBvCLBFHLEXd8cEZwev9O1dhDtgtE36auwssKSPGd0gPOVZ24XBXnbyWAxXF+Ab3R9HuLq81SFmvY8C2kWRgRwQR8px361p+JPF0OpJaR2TTx2gZmKzkEq5IwwOScDGR35I6YxpKOqS6i5owi23csagUg3SSwXEctvjdubBDkElRwCF7j8cdTXPXlzeXTTXUkY2tkhVAXgkkE+tR3OuTy24afzXlO2NpGPy4HQfiB0qO3v7o7ViuJIoRL/GW2bvU9ugo5JXsYe2hJc2yK8MkkxKnOV6jk5/Kn216ZJDb7iEbJVnYAAe5OcD/AAq9biNA93HiaFeoaIovpxjnI/DPvUcVs13KkzTDzZgeA2M88DH4Z9OmK19m1qczxMXpYkuJUCRiS0WUK6eY6OSNo64IPpjnpUMkZ88rFcxyHZvIJAODkdPwH5ip4yoiSaZmYI/llgQB6jOeT0NWdSltnkVLZ/mXkiEy7ccjJDscfhkc/nXLpcxVbWyMm4hu44zIYmJXn5R175GOv1q+koaGKN5VlkOQflXAJA6989eSe/41W1GRhFGEbPygOdoJB7YJH1HGKoJvWIgSPnOduSA3XtnHp2qE0ro3u6kUa1tvUSeZKIm29OfmIA4Un6549RRHqRlkCEgLuBZVXGR+FUIryXJf5flPQgkN7UqFXB2oY1P/AC0Odoz60X7Gbg9eY0ZyJYCDIFIIypcnjJ7D2P5dqh8x43kLOORhWzwfpnqarpI0IKvuO7AYZ4A68VI8kTeW6u6vGcq+wcEeuOvbrT8wjdLlexDdOG8xwhSQBWYgcAEZ57D/ACKdbzSTygSiPZ0IbC447f4VYUQxlCjE/KRljnt04pC67gFgOQMLu5A/Lngf55oNlVVrWLUZAaYRbpN5ZQAdoIPTp0A68fpiq6wPqE6WlnDvcvt2RgFhgdfYc9enNa2j+GZb2RpJfMitsj5ipYvxnGD0we5NdXbG20VRbachDEktmMjOD65y2Ce5wPUU1C+5tTg92Zdj4MtrURT6vKpcL8lvGpOT6cDJOewrQl1GI2/2bS44orcjC+UowcdenX8OndhUZR7u8lmuHablWCFgVCnGAeMEcdSMeik81M8MSStv3NISo6jhsDGc5JI7Aj6KvWtowNrqOxC8ayxBCPvliFyCSQcZGAcn3Ab/AHh1pT5C/vt67HG3AP3sDpxnd36lsegqW4uEEuGbCN1z3I9T3xnvuP0qq2yRmbcyMD8yNkFug5zk49Bk49BT0QasguZAfmUviQ4GxhwMZ6hv0z/wGoWivbhdpjaNP4T0/wAP0x+NaMenxk5UFmz/AMtOGz9e/wDSrixxq+Ry3u3vnOQOT7mk5FWOeh8Po8nKbiT0OATVldFjjO4RoRk4Gcn9DW606YACQnjBDIOBj6U13WT5mRdw65HT8f6VAzGl025j2/2fFbm4J+UznCgevHNPtxqMW621dbQt5RcG3D5GGUDORjv65/XEVyuoT3Erwaw9tFn92nkI4UY5zuOSffitBIrmSztBcMHnVXUyEDn5485XLYPAHDfXnpUFdik7Ilv7aSbxDaThSfKjYbh0Gc47fyP4HnM7W5nvLe4iZWQA4w3bnGCO30/M9Kltgslx5ykEFSD3BGRnj8B/npxut6pd6bDaQWMjxTwWy280OfvblHzL2JG09q1doowcjr7Rp3vL1XZmAnAiAxwvlp0A6fMW9DVbVLnT1aOwvREkk/KLOPl3DvzxnmuW/wCFiOkVq7W43Fts0SMVjZOOc/ez+PHPWrurFPFHhaZ0nRUVfOCKoARlyPmJ5JwSM5FJyTWmpKkmcB4jsItP1GP7JcQTwthg1uc4Pp/L169aorcGTYY13PgRnJxgDk02VnglOwAqOpXqOMVSgY+c5aQfIO9cnNqWoqSv2L6yOhRXfO0naAf1qybhbcq0YDKxBcZ96zI5klAVypdfu+gqKecgGMMHYHufu/ShMn2d2bD3oefzJI12nB2Kc7ailugSAcru5XnknrWcJttuCMh8fmPeiOOaZfNYoDGRhRnIHtxincaprdlv7Q1wGZcKFHJ7Z/yKW2U2oNxKNrdUODg//WpqoF2oAvUBs9fxpzXiBSGhU9V68Y7Urh5JHTD4lahDJqCQyB45bgtFI4wyJ0A/ICue1LWJdUeBZLl2aAFVd+eMnH48/pWVHbKZtwlGwDc2f5VF5T/aS0fKtz06UOcmtSlCN9Dt9L8aahpWji2jn/1Wd5AxwQAg9+Fzx6+1WfFniZdX8J6eXwJGffMobIGAwBx26Z/EVwW+7mItkEjgsNsagnJ6cD15/Wna7b3+kQ21tdwPbyTJ5oikyG28gEr2B5688HtjNqcmrFRp2kmYl4xe4Yng9/rWv4O0AeJfEttp8jOlvzJO6DJCL29snC57Z79K0I/hzrhitJLprW0e7mEUcU0h3kkA5IUEDr3OR3rovAWoaZ4U1DUIbmNbmWR0jim2YJw+CMEZHBBx7euMOMdfeNJ1EloaPiKbRdC1i2/se2SGC1IWeCFdpmIYPhjzn05z2GK818R6tJrev3l/ISfMkO0E5wvQD8sD8K2vE+qR32pXeoW2Vt5Jj5eVAOcdSB0PH6d65PbgUpSuKlHqwXrzyKeWwOOvb2qHdijJY1BqTLLxzz7VfiZrlAoBIGckDaPp6Djn8PpWWFZm2qCT7CteydYLSRfMBJGSQo4xgjnqf5DOeeqgyhcQouDHlucZxgH6D/P9BC0pUbV79T60sr/vWAPy54PPSkKoMfMWYjJ9B/nimIjCs3NTpG6jK4b129aiOcqSMqegHeul0m2sZbB2meNRnMinrVRjcmbaRzpPY/8A6qQyvtIJzn15ra1Oym04JcSQq1pO7CINkMAPX09s+lVY4rO5YABoi3HzdM/Wr5L7MEzL60be/atiTQphnb2p8XhHXbq2Nza6Xez245Msdu7J+YGKnkZWpiZA96NxNa2lRWNveOdXt5XRYz5ca8K7/wC0w5AxnoOuM4GTT7zTLZ7ktbFoIW+6pPmD0OG7ilyuwr62McZYqoBPP5mul0zSzDAZZi6Nt3NtUMQMZxjI9s80ljpMUVnNd+TLKFAVJiPlV8g8ds4zxz/Wtq1uVEUiIrNIYjt+UjnHvx+fp3qoxJlIgEdvHPvRZDGB8pZuTz3x7Y6VpRXe1FMcNqFBC7zbxs2OB94jP4571BaW0QtpWuJ2iccxosIkDH0J3rt/WklijWci2maS3BbHmx7Gzk44BYEdPSrS8iGaqajMGPlyOhK8lWI7Y7exIqvfeIdS06M/ZL6eKQjIYSHgk4J/z60WpthGUuIp3JzuMcirgcY6qe+fzFPvbTS3jWQxXhjQYZftCEsc9j5eB+RqpaqyRMdHdmN/wmniNwd2s3pP/Xdv8ak/4TLW/LKvq94DkNu+0OCOue/0/KpBDoCq2dNv5GJG3N2gAGBnon1/OoJYdIIX/Q5Y1GckT5J6Y5Ix+nesuWRq5Iggv5WGY8cnJbHUjvW3qkkVrql9FFgRRzyKgHoGIFVrBNKS0+SDzJ9x+Z2OAuOpAI5z047Hr2fcG1vLh7iW3SIOwLCJn6Z5PzM3JH4e1VZk3RWnvt1sFU8sOgPNWNQuvPvJrvZw7l9v1Oaj2WkIXAYxqW/1mCT0AyQBnoc9M57U2Zre4fJQqSBlYmIHAA43bjyRnnPXjAp2YrpjY5VMYcnk9ajtpwWKNyoBAyO2elTeTZK5hU3UYGMs7q31yNoxn0zx71J9l01ZU8i6u1Rs+ZviRyPTHzL+tP5B8xirGc+WoAHTAxTJETyFlP3mdoxwOihTgf8AfdKsUXkCZbic3BUjy/KUqMF+M7xnOE59HP8Ad5s3FvavpkEMV7K0olaQ5tggUMoBH+sbJyq/mfTFF/IaRlO6oTtVcYzwBWtZX8kmhJBlgizO/Xg8KAPwx+tV7PQVvIyG1a0tXKhmSYSnaM46hCOpA/HPatqy8LNcafDbR69pAaJpJJN0zpn7uAMp/smp5gsc3JKr5aQgk881XaZcAKzqOvDV0UfgiW8HmyeI9Ag352Ibsk/ovH8/aopPh5qQP7rW9AlPYLfY/moqZMpIw4tVvLYlIbu4UegkOPyq/Ya9qJlG3UbgOOcmRuKm/wCFfeIE5V7Bo2yQ8d7GQ347qSf4feJdMjN28Vu8H8RjuFPbJ78fU4pJg0XbrXJL9JJHS2811ZDP5QEgUrtxkY6DjnPp04odtHv3sLW1NzbzyOkUrlldAS2NygAE8HkHHtnvzMgms5pbWdCk0bFHU9j0xT9JjB1aHL4IyydeoBIHHcmrewktTpiiNPapb3ltJucJuk3wYPPPzrjH/AsnPSrS6feXF29vaRx3MqoG/wBGnjkyM9trHn261kOvmxqFIGP0rPuPOgnKLIeeoPNUovuQ5W6Evie2ubTVRBPG6NHDEhyuPnWNdw/A8VkwvID8oPXJNav9t6haytNDcOsjAgvnk55P5+tOtpGlMszhGkmi8qQvGrBk44wRjsDnGcgHOealpl3RnzK+0OzcNzxUKFmnVFJLM4xn1zXS6kkN7pVmDZ21sPNeIvboUZwipjPPJ+bBPU8Zziqdpa6V5McclnMWUMfNhuWRzkgqeQQMc9BzkdxkzZtDuiDU75X1e7liQqjTswGRwpJxyKuZlTUb1J8CZZ3Djbs+bcc/L25zxVObSYZJ3kty0UfmjYhw2I8+/wDEBjqecHJrVlggvNavLyWY2sNzO0ieYCwG7JO4rk9cdAfvexqru5L2HM8YA6YNZzOvzHtVqXTtRezaaGKJrdWIEn2iMZAIycMwOOV6jvWVtuvPMH2aZnU8qiFiPwFNtAkxWI6cevWtK4VwIBkKBEpXvwSTk+/NZuycTKrRSxvjdhkIwMBs/TBBz6EV0t3our3WladdQabdSq9uEZok3EkOwXgZONu3kilfUdjOjlEa5Bz68VUv9VnW2Kx5AJwxA6CpBZ39uqpcWVzHu+7vhZc/TIrLvn2xAsv8XOKVxpFU3rs5Bc8jqBS/aGj5Er4+tQ48xxs+bt04pTA+75vlz61nqUOW7lVuGbPvSme4IySzDPFNEDbjl1xjimbpEyARxxmgB66hOhHzEHPWvQ/DGvRx+DPEDzvEJ44EMTMQrKxJG4Hg5yRjHPPvXmLbnkC4yScD61fnZR4edd4H+loBhcbsK+T77c8/7woT1TBo7GbxVc3NuWM4OV6gYJ/H1rITWNsCwGR/3Y6nvznnP+eKy7eON7U7wdwwVGc546804qRFx8uMYLdatSZDijotG8Qm31OO4kgW6C5LxORhgQR6Y4zn8K6ceMLOzR10zw7ptp5i7WxGpDcd8AZHtXnVjIn2hztPC9+taMcqu/ysW2nHT2p2Td2LVbHS/wBu2ffw7oGe/wDoK0Vy5Zdx+dhRVcsQuz3++v1sRGWikk3ttxGMkVgnxGkV/PA2oJHsclYpljDMM42ht4GDnjPPSujuoXntnjSVombgOhwR9K8x1rw3r1lqTNDcl7QzExgykuq4+8SecdeuSPfiuCmot2YYmdSGsdUeiWurJt337RWgllKW6yuEZxgHkHo2cjHtWrJJFAm+aRI0/vOwA/M14/q3iNLxbezaRnNtiOWSNCSx28jngEHPbBHoM5z49cSytrdH3tcxSnE+eeWyVIyMD73P6VqqVzB4+z5Urnp17rGgab9ovReFHkcI0keXVj6DIK+9c14r1PR9Sv1sJLZJ4LaPz4VgcpukzhlOPw6e/Oa52+8V+ZPDHb6bb2sUW1mjhAjDA4OVAAwe/OTwO2c5qanco/2dAyJJKXGYxuwxAIJ6nIHTNVGitLnNUxsruCtY1fEMaaQlpLbCJIJ1BuLR4iSp98sW3DHPK/w+9Y1r4huNLuRNppjU7cOQpUc/eyvTv+GBis1rZopWief53J27ztyT3HX3x0pbdY4I5JYnkR1wyHduKjpzgDviqj7qM6ktedHa6RrMAeWS6DSKgMYyocHg7cKCOOD/AN8j0FbfhvxpBFqVxY3l0zrNNvillk+WNWUEJk+nPJrzuK9t2FtGLXYVlWWSQg9CMEDnpgZwRwSfcnSudPjvGFydTgha3YKGwMleu4nI9hnnrVyppu7JpYl0Woo9Vk1yPVZriw026WN/JYrdqQwRgQDx/ngg9xXnGtz29tfCfXrGKaaRP3s1th1kBJG9Pm64x2659OalpqQt3+02PmRgTbIpcZRCD3Lcjr69z1zUOoy3GoarPJJJFlgWba2VBJycDoM8/wCNZOglJNG39oNrXczNesbZpxPbxstvkLhlK57E+w4J/OqK2UTzPJblREgLbN5Yqo7scD1GDjBrT1qee/kheeUOSnVcgY7duM/mQKyF1OSwdkiAVzGYnYAZIznOfUHHNLljHQIVa1RWW462cOroELHkAL1ras9FW6sLxmCme3TBijmUnI5B7jH0OSc1i2DsY2EcrLGGBPzdD649feti4As44bv7S8t1v3hiSygEgHj1yemRVU49R158r5VuVbTSZGEbXSyxpu3SRL94IO/TH+ferBdLSYDc72UjCRUI+9jIBH0y3pTJt1y/22eVY4YpCkflnaNw/u7s5U9eRxnp2qjPPMz7gTNCExI6kEgjvgdBgf56VreK6HJL2k76l4XL2JkgjnmdHJMgDAKwPPIx1x37c1PbpHNdmOFPtEUbA5VcNhj6Y55PfpmsGVyrW80XmKTh9r5GAPQkCtWyuwJUiaFXVlPmFskgg5AH5AcjqacZE1aUlHUvzyxQPI9ud0WVUITvDHPJOeBjtnP5Gorm48+aOVUCQkgFlDFQcc9Rnr9fxqEXDxGQ/Y23eUUCH5Tj1I9sf0qXT7kvYLEQQGILYUfOoPYdyOfypSs9CYc0feaH3PmRwyswUBFBUgMN4zwwDADbntgenWsuZJZFYxHDfxKuewHbGf8A6+a1blfJivFheG6swUkaQDcUzjBx29PriqrSxrbxM42MHBQjonb6Dpnr6VlKKOmlUa1SM2EM0aoXOM/MuThfTj1rQSKSGKSQgKU6JtJJHX6Yx/nvU+EmiuLjCrBG25vL+VXJ7ELwPy9aqz3siIs0JVl3EBOccYOACPf1qXpubOcpuyLCLujjkjSWPnhWAO76Y/TFWrZpHQqYdi8lgQ69Ovt6isuO6iiVwZDh0A/eDgHA6kc9f89qnkuop7JEecEpyoySAfbnjtTuTKEnuFyrJc/JlB0AGcNzwAenp37V1uh+GDMqXt6ZVXgrCVY84/i7fh2q74Y8LXFtGl1c+WJcKyq4L7CO+MjB96238+4kkjWV3Qfe7q34fyHfqeBzrGNjpp0rayIlAnfagJi25HljO70/PsO/XhcFpYIWhch8DAAcE54PTJ7ng4J+irjFT26bBuBYkkt1znPJOe+T3PXH90AVRvHSKRnBPfnnqeTz6nj3PcgcVqlY2buTPM3nsUXDDO0Yyc9+/Xjnv6sOlU5btyxSKESFOG4JUH04GCPYYHuarmaW6VEmQxoGz8pzu9M+oHoMAelWfLWIFgFJz0QHGfTFS2Cj3IRDNKTK7P8AljB/wqytuMg43Y4LEZ5/D6U5QWXBKLnqTj+VO3IjKPMDAHO5TjP6VNyrC7VCnKtngDrj+VB2MTuBUe/JoaZCvyMVOecuGz+lRefAoy0h4PRVH55pDJHYKv7tpBjgZI4qK6tri5sTIZZo1bgSKOeOykgj17Gsy9uwsFxJHP8AOsbmMMM/Ng4/XFcnZ61/at/a2WrXl19mXeZ/tNw3lAgEr8pO1QCAMAfzxUSlrYqKvqdjY2Fm89v9k1KWWRHDTRm7BLfMOGUlgB2yAPwrTtW3JZFeVeJipBPcqRgEA9Pbj9az7a/8LrHDtijAWQCKdbVlQPkEbWwBnOO9aEUTxWlojON0SgHA2jGODgdOuf8AOa6aS0MarJkaSKNY/lAD8/MRhd3/AOr6mvLvHzXH/CRF0zGDGpwc8gE8ntn6cc13erXV3Fbre2o81YpAXjbncpBGfwzn0GK5nWPEumapb2y3cIfN6xRxHgiAMBtJ9SDnH0oqq6sYKWuxy8N1bNLdb1dnkQCFQoJ35GeOnTNUv7VkitJbTc6xs2WUnHTOP/Qj+dU7yQJPJ5AYhJCcE5GM8EVH5ocHeBzzXI5M1jTW5IJiVABwRwT2FQqjO7MevDFsdfbNRZO3apyvSnG68sABR9CKRry22Jo42iDFlXc3GQeQO9QzROGDgr+FOyCC8jZ3dMfmaaW8tCUxyefQUhK9y3FtVeUDMDyTiguqhxvGCfmB6HvVFpxzhic+nSpgVnhwSoZQTnHpz/SmLkfUtJO888jYB5yWPaq0ku47SpIPX0qN7r5SF6Y/EmpbVxGoI5kA3ZzxQJx5dSth45VxwDwM9DVv94uG3DCLg1KNjfvPvMrZBPI+mKaCJZCmdhbuB/KkJzuJDPLDJHIshUxtuBz93rV8aVrmv3sviHy/tojkUbApZmwOoXGCFOM/7y8HNKsNtax4WMyT5BLFgU7Hpj61b07WrrSYruSKcxS/K5CDKthxhTg8Dlv0rSNtmyfaJ6xCx1C4gu0u7+WWS/hZxb29yGAUqhIJz7gDH51U1fWIr/VJdSt7QxmaEmVMZCzHqwx0z1/P3ra0zxvqtqZdRnsIp2lIRbhoyBgA/LkevHftWHqNrbwW39pWjj7PcttaEqC8LZ5Hp1BweMj8RVvbQuKV9UYOoTFljAQKhXkDoff27fkKzy5V8ONwq1e3O+XYAwUDA3DB/KqW7cOeoqGbxWhaElqekH5seP8AP4daiYKpBMeM9MHiouB16etXbUtJlNvmD3pDK807uFQhEjHRUUAfj6n65NAmeGMrG5G/BY+oByAfbPOPp6VO0CgnA47iq0kLA7hz7UAM+82WOcnknmrSqXcyIuYyQOvK8d/8aqHIOCMH3qa1n8idHPQHnoaANSDSGu5UiZ1QOwCy+YAicjr+Xt1ByBUtzaXelxRfa4l5JKSwkMOMdcdPx/KopL6NSskaPBJjqqlVP1HPPv8AWtOx1u6YIgjNwgyD5SZJA5xgj2z34z2rWHL1E2Qzam+q2C2k8kciqQS5HzJ7gDGepp99ZaaLa1/swuHQFJQ+S0hz19v8KuPqmjz6h9uuLaJJzJ5u5FGcnnODx+lVpNQ0OIMYLdN2dy5QYyOn4f5PPNa8se5K0NPR/EMXhq9iE1pZ6sGiDKGOfKI5I7g49SPpXsfgfxvB4ttZ3CeTLbyBWiLAkKRwfp1H4V89XM0dxAyRFAisG2KMBuvbpx/U+pq94b8QXXhifz7W33h3DOMlSy9x7jrj3pxnZ2lsV6Huvin4faX4kkaddtreH70qrlZD6sPX/aHP1ryPVvCV74euRFeQ7Fk+5IhykmPQ+3oefavbtD1y11rRkurWTfG65919QR654xUeox29/avZXsSywuQcHsR3B7H39/erlSuUpdzxW4vrm4SQ3MzSuSW3sfU5P61Fp7tJDuBUiJ3QknkmSM4/Lyya0NX0uTR9Ue3cloW5ikI++v8AiO9ZNzM8SiJ9wOPm4wTWLg1oyJ2TujTi0W+njbzVigbJG2e5iibI6/K7A8fSnwafJMgKS2u1Tj95dwxn8mcEj3rLvrid3mQk4jc5x0BPB/kKjQPLE8gzkj+QppMybNd0MMmxlVn25BSVHXqOpUketWm064lspQZbONsBh5l3Gox65LY9sdazIdoLRhlOV5OOmetU9Xm8q2aMkk7sfhnp/KnLRXEtXYtrol1cO2dV0iJOgJv4+PyJqYeGJFRSNa0WQ4PW+HHueP5Zrjd8jLgKfXrRFK65ADZPcmseZ3NeVHXR6bqMULwgWTQ7g5nhvoynHGDlhj72fXr6HDoNNnm8yLMKmNsNvuI0+uNzDd06Ln9RWLp1wUikLHjAHPbr3rXuJngdUxjgg5GDkfyrWKuZvQqXFpfTExx20rOieYVVdxC/3uO3I59xVcE/aEBziRQYzj7wzjj8ePrV+a8lbYA+c9/z/wATV6/1qf8Atm4ntppYv3jsu1yGUlsnnrk9/WhpsLoxrlXtLq5tXHzRSFW+oJFRIylWUnGf51pG5W5PmToHkJ3Fn5JPXn1p88sZuA6QworlXOyMABsHIA7DJPAwOnHAw7MLopqjCMY7HJB4pJJsKoPAHvWldyAvGzwRmWSQMzrldwxyu1SFA+gGPpUKiwLHz4p3GMfJKqjPHqp9z9fpiiz7CuivaieSK4dCohCgPnqckEY/EU1J33omCyZ5rWsv7Dhibzxf7NoB8udDznryg9+PfrV6Kw8NyLHOLjUY1ZiFfykYAjkjAIJPI/OknbdA0nsclHcsHYcDB5OKjlvJjMBg7Oma6CDRtAkmkWXX2typ2KG05nzj3Dn6fh2q4vhDSJxlPF9mT/00hZB9OTUyZSRypk3c7yPWoLqaQwOElYZUnGOuBn9OtdfF4LinIFv4i0gnJ+V5WT/2Xmrdl8PILu4WO81/TxGx25tJi0v4ZUDk9fbNK47GdY3OkXVhaRalpa3SJEczoWWbnLA5yMgZ6HoM1VttP0ZruKVG1Cz8qRWB3pPuwQemFx+OfpVZraOKa4tVkMkUKzCOQDG8RoxU/iVFJaTIy5kIXIo5ejC7RsjTrMPJ9n1OGXgeW9xFLACcj1Ur7csByT2qlqGgan5s80dnJOEfaxtWE4UjqCYyQDx0ODUGCGP7zIzkU6JmZvMBYvggPkggeg/M1ok09yG09zn5ZEIQ7+uQAe5rSsmIVQegFa2nate6dCsFteTJCqlBC7+ZHtJyRsbK45PardrqVvb2Atv7G0eQYRRI9km/C9MkYz9Tz71D5rlK1jI1GRxbaccNtPmMhPQjfjI/EEfhQGCgKoxx0xW7PfWOryQRajZiKKBQkQsTsEKk5OEOQT+I6Drznm7kNbzStGWmXdhM8MR245przB67E8NxvlcYwQahnmyQhcLtJ4HOKZbRTearbHDkfc2kk+3A60+exuUEdzJbyFJF3DA4x79cdD1weD6U7rcVmWdJ8tIJJJFBZz6elWmKM4wqgdeAKdqEE2lpBaXFs0Uxi3MCWOcu3TPToOBx37mqcDlVJYEHNJWeoaouRwqilYxtHTA4/wA9KswyTKyt5sgeP7hDtlep4OeOc/nVNLna3PpSyXalchgKdgOoh8YavpugpPFeM8v2kIBKN3ybWJ+vO3k1Tl+JchRv7S0LS7xipy0kXzD05bd+XeucubhZbC2hAJAlZmOPlPAAH8/0rAvUIYorcZG4E8GsmkjRHoEXxG02RB5ng3Sn4C7sJnAGB1jPb3pZPGvhqXLN4SsVbGMbVIH/AI6K86EYAGPyFB7j8RUpIo9Bj8TeC1cSSeGlDeiRKw/Vh/KmXfiDwRNIgPh6Fg6gkx5TZnscentn8a4E8ggDNN+UHnimKx3kv/CAFwosVKjkESz4x+Kj+Va2qav4M1zQxpSvIsoOYHVHYRNjGRlc/gODivLpdzLkHNLps5R5A+MDGPlBz/nijZpgdW/hPw/aP5UmsSTSLGJBtjby2yMhcr0JH5Z5wRitXTtJ8LjzJrix2x8bY2vGcAg88dfrn8BXHNfPknEnzLtZhjAAIPTFSG7RWZfn59T0/L8adkJ3Oh1jTdF0uyW703Ll5VGNpG0EHplifSsJ5Fjl67dx5GOKv6VexQ3KyXdul5BGpzBL90kggflnP4V1kPi/RSgS88K6cyhdg8uKMAJkHGCnTPOOKbl2RKXc4bzovVfzFFewQ6H4dlgjkbQNKRnUMVbTosrkdDx1oqfaILI2dZ1ZtHNpK8O61kl8uWXP+rz0J9ql1BbS40i5ugFlUwMweMjLAAnAPesrVr/Rdb0eNftisZ1JgiDYMjbeBj8fbtXmCeML7TrIaKbwR/ZZiY5owTlew4HTmuJRbvB7m1Sair7lBTaPdO8krwxvl4Z1TaN+BleD056duOmawrm9kkM6G6AKPhlHO49MggV1N1bzGQ3UqqLV2AMyoMy54JwfYdOmazo0ur3bYXdxdyRZ8lPMfeIwvQgHp6YB4rsSbSsePzRTbYkF5Jd6f9qiUNOhCSKBw4x3GPTPTHSmi/jvJWlZvssca7iPMBKgDomRjJ7ZrPFi8Mtxb+asckW44PdhwefwqjJPKMRhTlCVOf4lNDmxwoQb0NJ7qDU9QLBCEflXk+VmYeh5/wAP5UwyRiNYvO2yAkFzjn8M4x9OvHFUYnA2ygRBc42Sds9sfnVlIxI7ykgMBhV6g+xzmpLlBL0LMV5cXTlVRMpzu/vEDHJzgDHWpJ2updO+0RPbyJENoXzQWXnuAee/X1+lL9kgmjR7aGSUeWPMD5GZOc7cce3Pb3qT+zbea4jS3cwQTFSY/NDFXxySvU9wOOv44rW1jG1NSH/2y0uni3mSNYQuWkXcSz4HX/PbtVvStdNutz5+orHKQd0ZjBWUYPRgD83bnHXjNclLHcRyywS7kkBw6gdh6+1XLWdIbcxyRhkZv9YgAfnqA3px3yO9JTbZrPCx5brqbyzwzQ+c80aOJCRH1KjnBOByP8B7VTvLRbq389G8yX7q9egHAA5znnoevbAq3oP2dZps3ckN0GBt5IlPz5JyDj2x+tWJrYlpRboFdGIYk8YBwT7/AFxROOlzCnU9nOyMu2u1MixSQpnHl/uY8Nzx/COTkjHH5VoJcoFljl82UOxJzGVYAgc49f044OaorDCipeLIJZFkKyIxHHoMdfx/wrVa+ImaWDcqSLsbcvyqCMYbj24PHXHNEdNx1ZqT91GdqFrBFHFawIJFCF2mjPDD064BGD+YziqlvDjyBbbURg4MmQAwwQRk9Pvc5PSrEjxzI0UkPlNyXbHXkdV/OqcMBW7EOzYkaOyDOAxJUHkYP/16p/FcqHw26mg0MSS24XE06DbgtuDDHGB6Y6HgfzqaF9LjeNixSacFJEjVgitk4yOuCAAPzOOlUyksLhDF5KH5vu9R/wDq79aSCKK5uVjN1FE5OUWf5g3OOOPWnrcy5VJWbZf1C5tVtYkCiVFVlLxnYwPH3hg5/TPtRJtljh0uzj814/mADoCzHBPQnJ5xwew+lVmjihuHso5Msy4VX6FxkFSAOuemf61CrWiWwe3QyOyLhFY5GPvA+/fPtSvfVkKCSsrl61sGRX3OrTuPL+zNuEitnOCCB2B5zwfSkUwmN7d7fym8ldxLA4YH7x9M9OhpmmpcXkAkWdkKyFYw7MGLEHaQwPQMDwPTntUqXQaVoJrhZpd23EcgGGGR2+8CD1qkrhLmV12Kl3IRbxrbv5SOMNzgP/dzxgDIJ/L0pkN4HcQTQszH7zKcnHqMDt/k1em0yC4t5JC6qIpAWgDYYejdOQc44x/KsdLeOO5kcEyAAY2kAg8djnntWU01LU66DpyjZbi3EcrSnIXDcYJ5H4Z4rtPAXh17qYahM8UlpC37lJl3Kz/3gB1x6569uK5vQNLk17Uvs3yxxL80lwxwVU4wMHjJ5wPr6V7HEbfTbJIrUKyRKEVBzn0BJ9+/506cbe8dkYvqPv7qeCHYl1GzODjapG0evBPTqfYH6VWto5LWyDBWZmHHGeep+UcEk9u59FFFvE11Kbmb5ZAcDnpkZx7HhT/3z70s0qIZJHZVWIE5xwqYH+IGO/C+tdEVZDbIproQRtJKwGBnduBJB6YJ6k4/EjPCgVg+fPcHzGkYjGQvZOf/ANXNPklN/deZuKrvyik8D1JOOp/z0qVYYdynMoYdgx5NS2NIRUYqv7x3HYHt9KniLRspXAI7k1NEqDI2Bxz8xAJ9uM0NGzSKxXcqkZwgXI9OOlSWSNcSFdk074PUKeKaYYyTtZ+enSo4w20YlbkcgGp+BIziNg3PA7UgGPbMQzKjAjoTwR+AqsbRZZADzxwDlTn6dalnJkbIEozgYGTx9AKom2vJF/49246Fx0/OkCJpdIUA7XjVNuCGBPWvL9CvJF12y+0qpgR8uqLzgDJx154/GvQr201RISkGIgTk7CB/KuGj8OtZ6j9q812KlmxjjofT61H2ky09GelL4x8MJ/oz3siE9TiQ/qAR2q3czxT6etzZFZrVhhXGCCPu/wAxz/KsWyuNLs9OWWWCFQFEcvmgNlh256+ozVs6hp8toIrB7YgtwsJBAyee/HJ/WumlvYwqrS5o24+WF8fKR09TjNcn478KW7WA1SzEVqYj86qMBsnjAHQ5/Mn8a6rSxuiJBDGNSFz0JHeuVufHWm3Ol3NhqtuyTHKPEQcexz/k1rWSa5epz0207o80huWjZ14yw+fPU+3tVq1lsGQQX1uNkmC1xGfnTjjjOD249ql1vTNOtpXltdRMsbKDEpQhmJIzn04z+lYM5KyAByU7buuK4XeL1OpJS1RLPGhBEByAcZPGaq7WVfm69hU4lAj6geoxzUHmbmLHr6VBorkke4HLZ2gZqTJZMADA65qAOCw5OR3p+WB3E5oBj441WXc+3YT92pWn27iQGz8o2+nf+dVgQ78njuDSMeCAuPegVr7llfKaTDKvPOAAcU0NKrFFGUNNjbEeFX5upPrUTXDB/QelArO5aWYxZRRk/wBaWJ2NwC6EtjgZ4qk0pD7l6g96twwzMFkLKB15P+fSglqy1NGNvMQfMIzzhAc5NXtKvdNhV21HTmvmIw2ZyioP72AOT25P4VhXCvvVopNx6YHaki847lKkgDkD0pxbRkoXV0z0oS6FpfhKO4SQC5KM0MUpMoVmYNwDxkYAzx09a81lubmaQiWUuC5kIJ4DHqcepqVUuLqSOGQv/dVAMt9MVLc2s+nQmd7e5iJwY5ZYyoJ9sjBrScuY1hBpakOtC6S2hiuL2PCqu22VicfX3+vTp7VlvEl0rSW1uyfMF2ht3zHJwOOnBx1PHfiopfNmLSkMRn5nPr9akeVrv7qxoWYZVMIMj26DqfzqW7m0VZWEeyuUtvPMeY14YgglPTcOoz2z/jS2sxjYMAM9Kt2sVw08lxO7Dy45czq/zI6odoJ9zheeucD2YjJqL5hto4blUZ32kLG4A5ODwp+nHTAFA2SupYb8Akj2/mMVGqZOcZ/EZFVhNxyrKOO27OPr/jWhbSMLcmUgR55BLAfyP+H5imSRtZCcbcFWwTyOlZcsLwuVYcj0robiTYhaMxJGF/gCnf8AmBn8OR6Csu6ZXXOB042Lj8/0oYXKCttPY/UVIJRtI2rz1OOetQ0CkUaXlm9bLXae5kb5ifX+v41oW9pBbKGTBuR8yMpJYdMcf56+9c/mn7zs2g4FNMk6u4125RQrywK6ne+wDex/2vf65x+dYV/cNK5kMqtglQd+cjqP8apS4kldgfvMSKa0UijJRgPcU3Jy3GdJ4Q8Z3vhbUNyFpbKRszQZ/wDHl9D/AD6fT2+x8YaVrGnfaIriMkLnBOCPavm+KCeVv3UTN2zjj866XT7C0tLcSSTO9y2MqjFQPY8YNdFKs0rS2Jfke7Xv9kvps0uoeU1rjLCQjj0wex9MeteOQywzanPOY/PCbvLJUtkKpwSp4xwOvv71rXUumXkcEBTUGYfKD9oTnJ9Nh9vyFZ1p/Z9usiyW/mlh98yMCp9Rg4/MHpVTlz7Izcr7i3GuTtA8ISArkj/j3TngAk8deBz14pZkWC5igKKQ8SOTsAyWRWIx+NX47TTooIZY7edlfJIeZff0QH0PWi5u7OW4BbTY87Aq4lkyoAAGPm7Bcc+v0xGu6QtCsb02lnKkY+VgAo7Kcg5x+BH407/hI7tdOeGeC2uVHKiaBJCpOPmG4H26elVzqWmSxOr2kqhs8RyAY54xkHpU7w6LFaSSeffQtKNyK0aONu7ABbcp7elD21Q1uVD4r1AtN/xLtKyQQNunQLt9MfJz+Oagj8T6irhpLXSSM5I/s2AfySiDT4ZmBa/gSM55DLnHGOCw59s9qkj0SKR2dr1Au0FSTHknuMeZ9cHvx0zxloWWLPU/7UFx9ps7TcSCvlQiPaM54249O+anFvBeXWxkkMk0nBMjELuYevJ/Hn3zTNI0K6eedImhUn7izSpGWx6ZbGeema0n0PU9Ou4RcQojb0I/fxtn5gOAGyQDweO1axaM5J30Mq4sYZ1URSG3O7c2BuBz2A6jHPOT1HHGTZubLT7i5klhWSzjZsmNf3+36ZK9/X86bcRSW8jiVGjMbbW3jGD6c/jUbONrHcOBnOaqyZN2iKbS2juXW1u0uI8Aq+CmRgdQenX36H8YpYrgSiMW77VO3coyp5xnPT3q15UsL5dGR1GGDAggjqCOxqSK73RKH5B6/WnYdyO1MkVpOZIpQxRlVthI3ZBwT2+p9R61j/apPKmdsAggD2/zmtYaiy2flMxCuoLIDweVP80U/wDAR6VXju7RUfbCBuGGZVGDyOx6HpyOaSTE2jLNwQpjcklh+Rz3/lXQwzBPDGnrs5Mk5GB1J2gfyrPDW08xV4twYcNnBB/Dt1/Ot2xu9HhjFlqOlyS24OUeOZgyZ7AZxg89u49KTumVozmFkBDl2wwc7ipzz9e9TmdEwi9cYzXQ3J8CECNNN1SRu8rXIU578Zwc+uPypIH8FwIWbTtUYbcf8fC4Y+vHNTd9UOy7nP7txJDHOOvapYbp4mBDk7P0rUn1Pwr5DiDQ7wShjsc3fBHbIwfyGPrVDRrbS9WvhBNePafKSSV4YDk85+U4z69qakJo0Il00QzebHdRytGyxtCVbJZGVt245A5B4z/F0yMUo9A+0XsqabqCNFHHvaS9Xys8hQBjd/eHXHeqscyyh9rEqp45qxa3ht/tBByskRjJ/wCBK381FJx1GnpqWLbRtTuXMMEUErbQSPtMSk5GTjcwzjuBnFRNDdWBeG7sZ4ZF6iSMqcZPPPbjr7UySbCkhgwLYrQt9Z1C1gWK11G7giHRIp2VR+AOKvlktieZGMkmGA4LE+tXAhZM4NStdWPn+dqFgt0RKZpJFlaKR8gZBK8EcEnjOSTnk1sXFlo8Wpz2y3l4kKSsnmbUkHB6jBHHX1rO7TL0aMPcVwy8HvSW7BZrl2QlRFyw/hBYDP64/GtK70lYHkW31KwuhGBnEojY59A2FOPZj+fFRR6XqsVveMmk3EyTQ+WDGu//AJaI2Rtzn7vb+eAackSk7mGXAu23MfJAJbAySMfrWpBculxcIk5w7FX2yZ3YPGSPvfWotFtYm1G0S4DKzTKChUhsFsfd6/pVARzW9yCoYqDwR/hQncHsbtzr+oWqhzcCXGdonjWQJuznAYEA84z1qvaeINJEaQahoUEwUELJBO8LcnJLFT83t6Disq7lfcsjqwQtjp1qpLIrEMVxkVXImhczR2hTw2TE8UOpiEvlj9oTITA5ClTk5zwWAxj8JbjRdCd43XWbmBJE8xUntFlYAkgAlCvPHoeorjYLuUxJuztHGMVsahKY7mFSScwxkHPYqD/Wp5R8x0ieCrPVdNuLWw8QW8k8LrIvmQtGiqwxy3J5IHbg8Vi6h8MPFUczpDDbX2xwC0Nwi4JAIB8zYc4I7d6r+aoxnGPWnNqNzaQFrO5lgkJ4aNyv8ql05dxqotgf4deK0JQ6ThxzgXUJ456Yfnp2pbX4a+K7p2J0qSFVBOZWUZI7dc/0qjaeINZsJZJYNRuI2c5Yo5+bjHPr/wDq9BVuPxfrsR3nVLt+n35S/wCjZqHFl3K8/gTxVbxtu0W9wOCEj3/ltzn8KzZfC+vW/wDrtE1RB6tZSgfqtdH/AMLB8Qrhlv26/wDPJP8A4mrKfEzxHj5tRGP+uMY/9lpcrHdHKpoGsyIAmkX5DD5QLd8t9OKrNpV7ZXDQ3dpcW87H5UnhaMt9A3Wu0m+I2sShWnnjYj+LyhnH4Y/SugtfGhWzs57aCFJZ2kExEPHG1lYE57NyD14qlBvQlzSPOl0fUGtFuEsLk25PEwibY2PRsYNTr4d1mbDx6JqEsbLlTHbu2R1BGBzXog8WXUhcjy1B+9iJcH8x/nFQS6/dudxvJozjH7tygIz0wuBV+yZHtUcDbq0XmxygxzKdrRMhUqR65HWlfJVsHAxitTxDeSXM0EkszvnccuxY9vWsd51VAyc55PNTNcrsWndXPaIdWjjhjR0YMqgH5R1x9aKz1tZY1CRysyKMKTIvI7UVEYRsO6PDTcT2d8Ck06mM/umYbWXHtk4/OrVvexs04vo/MM3zq4YDD/Tpjnmtq6FjezW62emwwh4h5wYMm18nOwluTj3Oaq3EOmTwlbKGaf5TtcQrkEdjg5+p4+lQk0cEsQpLla+4huBb2+/7MwLRkb3POSp7H0J6VoWN1a31kbXzJTfXUgyZjhSTwOcnpk84yfrXP8oxSSMlFz+7Hc/XFI0kMUDGPcXDYAPTHr6g1UZO1nsKVJTVlqXr3R73SpiJSxklGCHHT/gQP+H4Vj7JZQZAGdUGXHUKPc1e/tS/nia1fzJdzE5kyz5PTnr0BpLjSryKC32mNluBlERiSCOxHqKl9zWLlFe+0L9mYNGklvMoCiR8dwehHXj86lhtgs7SQCXyMAYlz95s4+YDHYntnBqGN7oCB5XkiRVMQfYeV7j35z/9ap5buO4iWGJWWRBhwEyGx0Y44/McepqrJmUnN+6WdJEEAu0uZpYJnAaILnbIORjHr0weOh5ogl0wSTQSxRs4ibBf5BkAkAEnr+RJNVYLd47mOSW3DKAJMK/LnHGevJJyfxrTjiQSLObZYHKkR+TJsYHjBBzkn8+9XZ2Oeo4qV5Pfszm5ZWklRnJZu56nA6Z/KraSuqhkI3FcD5hwD2/Km26xfa2W8Vo2JO4qg3A+mOO/arltb77q4jCLKkWGZlfBQf3unIrFJndOUUtiSziO0zO6LCmDISSep7D159qnk1B7aP7MsqlVXaGi2kkMMgZIPTjP5Vm6isgvSVnEsbANuU56jPIzwfalilCxqjQlozncVYZOPb2z1qr6WOSUE/e3NbRpotPtprlJWlnuAQ8YQEZ5xg5Ofpx/Wp/tMkuoPHfkoZBmJ1wgLHncy9vXjv2I4rES7ltp5oIrcxpICCr4V169yOO2R36VYtnlRIXtm+6fm3EADkdCcd+OtaRaMqtJ7vqWtyxWu9MFhkqxBwR7ZHTg+nQ1iN5bXEPk+Ym3JxnncSOnHHGPWupNnJf28skMkcMcbCR1HKrIT24yQeevAx16VUh0W1jilmeUGQExxoAACzHGVz7gc8YGetLldwpVYQXvbsz/ALNOJLpRcNv+47Ox6buSe+OOmP8A68VzZMZvO3RFt4YMrcEA46Dn/Par93p7wafFMshMzZDBQSVAx1P4/pURZLW2aRUZioO0g8c5yKhprc2jVT2K1xvjZZJyZJpPlDAHPr369vzq2lzcaVIkNpcoVLbUIk4LYAJIIAIPuKNLjGsW7wrNCs5dURJFKtk55BGcgYx+IqpNBYpbPBNNNJdIwJTbxjrgdyPxH0o1ByV+QZJBeQPdE7grsBKQuAWzkZPQHPv7d6ngsJoVhvCxWR2JjPptGS36VA1yIEWGOV0t5gNyuCoBBGGK5Pp1z610Vu6JZRwSsPPs2EfmwEPkFjjHTI5Pr24qox00JqzlFIw2mmuc8xBifMMmMcE8jA7A44x3q1pOlyaldGFQNz8swP3AOpIzz+nJqleJFA5jkR1ZD93oM+3HTGD0r0XwFYPpuli5DQJLeAOzy8lV7ADP4n6+1RBcz1O2nBWTjsdBp2nx6JYItqscSgbj0O4AdST1PA/TtgU+5+23VwHlGE4EYU8A56474x0/iPHQVbDSyth5opABkNGm7Hp+Pce4APWnXKF1bGzCtu+U7QOOx+nHsvPU10RWprLYrqFeTrtUNg8k56/4tz3+c9xWPquZtRKLIzRrgvHlSjMM4zxnjJHXGc8Vo31wII9+VDOThSep4zx9MfTCjsayxGsuPljUIMkx+/1NNiROvllVdhFGwz8scYI/WpPlKcqGJ9UUD8xUKI+QRnLdiKuC0KkObmJOy5O7/wCt/wDrrNtFpDF8toyfKw2cDGAPypWIOPlHsRilMxJYNOxVR1GBUUWI3yCCzc5Zc4/OgCYO3C8D3weKQoSh/eSE9yvSka5IORwOnAxUM11JISPOY7Rn73+fWgENDFmOEZkXgZKjJ9/mzUi3TR7UYQIPQPkj61msJjtGBhm4LHA/M1HB9snmZYXMfy85PCjj+uKQzSuLhZYyRhh6ruWsK9dZFk+WQ8f3TgcirFyt/GMPPEyqeeuTWXl5Ekyys4GBtYg9R+VJbjJNO0/Tr7WkjvIMbl2JI6nbvHOOoGSAeDnt68diui6NEy7RbxToCEcBV2j6LgE8VwDuZJDGx2q6gbvc55z+f0qbSLJL/VAmranqDPuxFHE6oj+oyCefYAH3q1o9BbndxKLfCBhtXAyDnPOM/wBfxryLxhoU8PiBmgZZRckyLwEHJ6DmvW7PRoIYpI7a5u2ZzlY55RIB7DuB9Sa808Y+GNaOtTS20L3UDDeGiP3e2COx4q6rUop9TCEXGfkcheWF5brFJMjfvM4ywJJHB4BJx9evaqBy5BJ5rX1LRta0cCW/s5Ylz9/quT0+YZH61nrLEsYwNxPJBHeuR7nQm7FaQMrYbOacsJiZXfGDzg96sBgI/mBJY9DUVwXJ6DjikNNvQuiKF2LFFPHHYUCyjjG55f3Z7DrVa13mMow6cjIqQsUTafmA5yKZi1JOyY+4tocfuiUx696oupRckn6GrSuHck42joKV0jKkE4z29KRcW46MbbP+6yR9MdzUlxZFirQkMCMkHqKii+TKK3Q8VbVtyHKMz9CMcimKTad0VBbGKXnDY7DmmzzSK5BrRhtJHlGN7EkZSNdzGtD7DMkjG30+KA8Am7kj8z/x/AH4AUJNlJN6s5y1WeeXEaliOtdBpWkPNKrTOq25DZdjjJxj5R/Fjg9h2yCRnXs9LMdostwI2bcflSWNxx6lSe3rVmeeBvKjQJKyliOybcDse3yjrgexrWMO4O3QiWC4tw7WUi2sJfPnyMQz8Y5ZQWbkdhgH0rNvtOTVpUjn1ItOTx5cfmbvbJIPT2rfvXRrRmYmYtgBVB55x16n6cdO9Un05ggkj06S1APDkSAe/wB4/wAqtpbEpvcqHTrPTrArcRxylCFTPYeuPXr6n1J6Vh6np9qxN20wjg+78q/MT3Ht+uOnaumvYFntZ0tnaJdvzyykckHP4A8DAzgc84zXJ3zn/hFkwTzdmIgsCPlRemO3I9aiSsUpMyHkjTiOFGjByBIOfxIwa2fDt/Gt1IsP2LT7oxN5dxJvZW9UIJYcjODjOQPWsYAkjqOOc1XkXa5FQabqx1Oo6S6LJcWjgxDL7cKSPU/l/nisQ3EnyBWQEHPyhUI9xgcCs8Ejpmgce1AoxaVmaE2pTF8Eg7T8oDdPXkVSkkZ23HrTMH2/OpJITHIY9ysVGTt5Hv8Al+XFBRFRSkGkwaAFopVxuGcke1amh7RqMRYIoXLbycEEDIOe3OKFuJuyMxWKsCOoNegaGov7SCU3UKhVIfeT1H0B5x64HFUtLUXWoyx3VlBBImCgdQnBUjknjp/npW4tmLKN/wB5a85wiSK2OP8AZyOee/U1vCKWtzGbv0K93pljMEe5vpI3AwqRQb/l55OWHOQePTnPanWegabNFCy6nKHBBK/ZAQSRyM+Z0GPqfQVPNYo8BWG9tnmZQcbZBt55H3PX8KsaRZyQ6Yon8uGcSt8jnJ2YGDlQR13U7RbC7SKsVgsFzHNLcGNUbcuIN+WHIH3hx17/AONQtYaa0zj+0pwAVwfsajIzgn/W9fQd/UVd1BkTKExS4yf3bPlT/wACUds/mKzYonZ0lKsELcBlxkAE55wD0I+ox3GbsidTWb7FhbaK8JjRUHnvCUzhQOEDN+eaz2MNnqMU/mwzojq2MsuMNnnK+3vXOa1eXdtfK8bFEJ3AZ/DkdunQ10IimvLJriC2lljEW3dEhcAknGcdOTj8vWldP3bjs1qJZaJAVmT+0bNpCCEUsy7h67nUDPtmnanol2dJgeRYWSJNpdLqJupO0Haxxz3PvTjY3NvpttdTWl0iqzBmMTYHCAZOPXNNS7jWPLlsHkkemccUWutxXs9jPbwXriRoB9idnG5UW8iz/wChcH2OD6A81LL4G8W20Jll0S6RAu4kLuyOnbPNV5L+ZE8tGOw+v1rT0vVr2PzJPtLqqpjgnlScH8ORWPJ5mvMZttpN7BdmO7jnglDBTEwKtjB/h69cVozTMjsFOADz6mtBdYYTrPcKly6EFTMWYqQSQQQQRyc9frUUN7pzwTQzWQ87qsiTFdo4GCG3Z788VtC8TKVmYvngsybR35CA4qUOsOpEwqpaOQtHlOcg5B9fzq6lrblA7xsrA5JVvvDcO2OPl3evOPXiqbbfOpISIytsBJ+UZGfwHXnFW77tEp9i3earPe3ha4m3yjG6VQMkjHU456Y5zxULRRSRsi+YhKgffGCfX7pPPp2JPth39lv9pWNAJpQm9kt2EuFHU/IT6f5zVVZFYKwYbT0waLRYXkgktHSSMIu9MfMxwMHJ4HPPGOePp6uh8PXrRfakhaW2MxhV153t7L97HI5x3FRTXEvlfKPmz0qWRzFpkNx13yvkevC/5/8A1UnHXRhfuD6XdWzC4e1nSDdtLvEyqp7DJHX2pLsukMzhySoJGfYZqfSddvLKVjb3Dxqy4bB+Vh6EHgjk9R3q7JOINUhghtLOU3Yjj8iaNFjO5F5JxlckjJUjrScnFDSTOVhbKKWTkdMnmntKd5Oa6iBrUXX2KXwlaQzMz8SXM+GwxBClWIwDkcddtWpXjFtLA3hfT7eRo9qSPLJlCM4PTJzx6Z7msbtmtkcU04YdapXBUtGCOS449RzXbX+s2RsZIP8AhELSOZkKJJHKW2t/eGDn17+n0PKNIL7ZaoPKRGDrGyAsTgAnd1GcUtRnWnS9JmsZmtmltLkJuVD+8jbCjOT1BLbj2AGOp6wJo5dlisr6OQyFQFmiaIlsgDgFl7nksPwHNSadPmK+c4UrbkLk46yIv8iarRy4dsE4yKtLsS33GLp1zEriaAhlcjKkOpwOcMuRgYPPT3qCTMauhDK6k5UjBH5/jWr9tkjUpFIy5IztbHI6GrOn3q3eoAanH9tjjWaUJKchiscj4J6gE5OM4zzitFJ2M7I52BWu5EgRxHI8gVZHOAhJwCfpVlZcWxYEbuSQfbtXRWFx4YXVLfUFsb21kgdZPLiuA8e5TkHDDd1xxu7VWt/DK3jKtjf2cySO/lxzymOXAzyy4xyPQ9/rSUrO7Kcb7GFdyJb3pgjaSKJDhd4weOOfel80OhaQKSCDnryKi1qJjrd2Q3mbpnx8w5+Y+n9KijR44yvJbrincVrHRWni7VY4VgN/I8KkHy5wJV47YYHA+lV9Su01B5ZZ4o0luJPM8yJNpztAx6YOMnjkknNY8cbgZ2nPU81KkzskcZX5lfJ44xTSQm2a8dp4dvIojdS6oAEbMSSxOCxxtKuUB/vZBXuMEd6NxoFk8O+y1H5tkjmCaJlKYxtUOu4OxyccKOO2cVmSH/TVSThByMcDmtOyGIdgIwRyO4qHpsV6lix8KaveWCS2tss8IJUt5yLtPoQSCPXJGKmn8M6zbA+ZpF4dpC7o4jICceq5/OqElywkABPHU44+lXI/E2pQXErpfXAZiCzeaTnGMZ59hTtJoV4opMdwKkfMpwR6GoJsi2wRnnP0rrYtfm1Bg19a2tyQm1/MiUF8EnO7GQfmbJHt75sfZPCt9DJcX63lgc58q3OY29l+ViCPf/8AUlJpDsm9Dzn94wO05z69qd5U6gZKc9ATXdSeHfBjgtD4nmiJyQjwZI/9B/8Ar1XvPCHh5U+TxnaeYeheJcfpISKi66ouz6HEtE4G13Uf7rZqRLZWJKyAfUZrp/8AhBbKfaIPF2lO7HgMGXP86Jfh1qsDk2uo2Nx06SbSfwIqQexzpiKHHUnt0zU4nEk9vHb+YREjCaKSU7UbJVhwB1AU4/M9q6Ox+Ht7eTbNQ1G0hXcB+6lDE+3tx9fpWrc/C6Gxmie18RQRSy/6z7QgZWPtyPy5pqQnG5y8N8BnLjaBjCnp7ck1YW9QR/e4PPNa0/gaytpj9s8Q2fmE4Hl2+Mc9vn6CqFrodgurXUd5fSGzjO2F7ULufk9dwx09M/jW3O3sjPlSIf8AQrhS1xbvPEP4Ul2H8G5/kanhs/DCi3M1hfsQhEqifKlsnB4IJ4wO3PPsWalDpmnXAttNknlgVRl5tuWPPTHbpVQT7hkdKmUb7jTa2Onj1bQFjVTbakMADAmUgfiRmiuUN0gJHzflRU8kQ1K1yUmmKtdKrLGSEIICkZ+8CO/J4NaunzR2lvHckRpFJG2DFAWXI45yCR1A4xyRnNYr6jJLDxCZAP8AXuSdvP0HHJ4qdHuYUgjlZFj+XcCC6qjcMc54PCn8KW+x4zg7WloOtpYbqW8jukw8rq0P3g/JPQKdvpwR345psVvFa3KhIvPAw0okIbIY8FRtBXGDyTz7U6eJXjkZZWRg+Ah5ycevbinWwkjlRoYRJ54ePzSOgABOfwz+WalJLRlKba02K39kCBRNPcw7WUsChO/dkAjHqMj6jFS3lij2rlJT9nCB3BYAnkcjr83+eaeIVEMF9JiSGB/LlyhHXaCCfoQc+9VLhrY3bpFK0ts5xCu1T9ct/Dz/ADobSHFzm029iea0iCwZkeTyc7FkOAFz6578HjA596vyXsMNvJbm3YySygIT90k9cnjnPr2qlbXTTWMzXEYMdqRGpUg793QNzyBt4Pp60lncxzRsGnSOUyFGBbaVG0sOOuOMH8KpSWqMp05S0lrYvC3ubKZxJMu9GDIIuceuckE/yps217MI42IW/cufkcZ6g9j1PB/PtVfzPtheK3G26XO+QP8A63GASM8+hxzUoh3zSLeyi2lT5k3QeeJDtKnoccHHBzgg59KfM7EezfNq7Emux24lWCZIPPYkrPtCuy9tx9en0/KspLKU3EUh3xp5bBJoVY8D+8Bkj3A/vCrd1bSbYJru4kf5QqSFSoXA4zn0x2J+tQhPtN4ttN8nm7mBdFAbOAccDqVGPce5qHZs2oycVvcqrbOjStN8kOSVZwNzN0+o9eatW86i4jS1VSVjIIXJLEDn69ankjkurFnLiSNGKsGU7BkDkDseoz7VSm8+zsre6t7BoI3O0XDNglhnhR26daaunoXdVHq/8hZ7aSUqiMFZ5Auz+MnHHHpx+NWolgtZ4beSXaDkEOCoGe7Edsnpn8azBqF9b6rJLcMpmV23LnJz3wevap7wmdkuriYq0wHyL3Xg85znoOvpUX6ouUGrRb0NPLgNb2kuZAp8zym3eYPY+gAz7VXMdxG2ZSImEe9PM+7Lj+Hg8HGMA+naq1hrU2kXbtbInX7ijKsB2q7fX9xeSiW7hSSHlmEa4Az2AJ5AJqt9mZezcXsRy3E4topHuCSYwpGdxGKm+1W8DRQzzr5LgSFFPTvwR06/z+lWIdOhuYbZJJIicFiiTKdp+nHPt3xxnpWW93ZmJbO4VgUZih6DPdcjOc4GcY5/Om5O9mZqKf8AwChqNpHFfk6fcCREcurqc4A56H0rU0WKzuZHmuppUu5GcbmjHlnPQ57dx0wc/WlTQIrmyfUbWQRJyGV5ARtwT16YJBA9SD0qhbW0Msrsi74OCSGHy+vBIz9M1KVtLHS6inBq+xo3WkzxQJBsBkiLMIkyXxk5G7GMADPrz36DMM5tkYhGAY7jgg4B9uf5Vt26XovNlneeaWQzKJPMQbxjJG3nIUc+wNZmp25SGG7E8QmlGwQLwF6jgg9QcfnSceqHRq6qMrO5TtTda7rNpYSTlo5Zgu49AGI3H8h+le8rZ2FvbMY7dGbsJEJ5+pzXmHhXUkv9SiuJoIvtVijM8+wDI27csQCxPf8ADsM16Ha6i0wRzdWjoScMnJP0Iz/StFG0TvpzW1rFqJQJmEcSNgc4478LjtyOvqBQy5I6YVAZHdcjHBJI54HB/wC+V6CobGO5uJJZbeIZKmN2ZvLQ/McgnBOfoD/ho/YpHV4bm7iQvgyLEozxz1fORk5+6Kq9kXucw2/UrwzwQOVQ7IsAswAPU9ee59yatppV2qRu1s/lnknbtJ9ua6NdHsoUG+3EgAzmdi4H03Egfhiph5YG2NIgPQPiobKSOdi0y9dWJQqTwcnPH4UjafNFIrhkUqMjcGPI+o9a35rhYMIzW0ZbgAgtnP8AwKuc1G0udakktrQxEop3yt8iRdCM8nqeB1/nQk2NtIbLEg/cNcIHPP3MsR9TximrHAp+W4Z2LbRuA9M9vamS6RYiSG4vru6uHVvLZYMRKG5OQxySpOccDimTR6Pb26PFpERjLgOWmlZyME5B3AA/hVqBHMOkKiTavzlBklaglkJUnOSx5B4Ix/Km3UUMcn+g2YSZRlV81yZRu5DFifY8Y9KWORZIxIqqQVyAM+hPrUyjYcZXKMiMWPmsyhVOM5Iz6fiapPDII1baTu5GB+H9K0nmm5IcANwRtFVJZXMi5d1AXogA5x1/OszRFRlvzuVQ4RR0kyB/nmoYIhiUy7S+08KD6j/P+eZjcSiKU/aSSSOH+YnryP8APeo4EnMnlrM+6ZQMbM7+QwHtkgcimtxMrShXYKMA5HIPzDkj6UQyRTq1vjYxfG7aVb68/wA88UOh8snghuRx7nvUcDKLgSorYEhIO7OPp/hWltSUWZte1yS4jso9S062nVtoaRWWQkf3iVK5PtjrXYaXbasbjzL7WUuiy4ZBDgIfQOMfqK4e7kW4naN2ZI8gsFuUdSMgY8sDIHuTxWrYeJbzTEeFY0u4FAIVz+8Vf9lv4h6A89s0otrUJJNWJfiCn2GxglklcJ5gRbYxboXzzySPYcE/Qd681murbUrmKG4itrIKxy8cIj4Prj0+lemS+P21CQ2A0h5rVuJIZofPZh6eXlR+p7Vi3ul+HLxwF0bxDFGpLhBCoCqeTtyc444JJqZycmKMEjz6ZbaPe0FxJJjA+ZMf1qLzc9ADxXay6ZoMUZSDw1qTscbXu7goMdidowR+I6dan0xIdMw66Bo8krchp2ab6cFyO/UfnWfLc0sjhU3CTkAA9RUyxvcOyRq3TlV5xXc3dxeGXLWHh6BnbKRrZ/mSdv4dazrzVr+dkgbWI/KDbfItFdVz0xt2jP40cjFYwLHSLq+3C1iEm04LswRB9WYhR+JqydDt0Yi+1mziK/eWMPKyn6qu0/g1bscEMLDfbn5yArSyHceMkEDHfnpmpLaRo79UeZIoQSVEMQyufbjJ4HftVKAvMxFstOMiRaal9ezYAaeVVVB77F3EAepf8KvppcVnGXlnD4xvRBgH1Gep/StY6rFc2piGoX85PAEsm0Z/3ct/OsuSI7mV3WNd44Xqfx7cH9KpRSB6smCS3NojRTfZrRuHjQHJx3KcBj9SDVOeztLi9l8hLuWRnJ2rMrDPptCZ/WtSGOW2dYY0EKj5i03r+IJ9DwO9Lc3bCeRbvXbiRd3C28bygD0+dkpvUEJbaesejFLiG6spGmyInjZS6gc4LYwOeMA80KEitmQGOGIde5Y8fn07ke3pUaJF5LvY3TOXk2s08KxumB/DhmODn9KbGFBVVIklzwZACoOeDj6+uapK4mywbsJbyLGFto9u1G3fPJyOp6/yH61S8+zMwS2nmlkLbViMOCzfg5q4Yd8cpW1fVL0ndhg5DnIB2qpDH1yfyFUGN/BKon8Nw2alh88tqyFecZBbmi7TsCStcxvE2rs0sthEV2xtiRl9R2HbFYD3hfSYrLafknebdkY+ZVHp/s+tX9asbhdSkkMBjjncmInoR65/nWOdyMVdcEVjJvmdzWKXKh4nZSM9vSgsJDzyaa6YUGo+hqR2JtvpTsdMiohIw75+tOE3qv5UE2JreOJ7nbKjtGBkqjBT+ZB/lWlMmnBQYkktmxglIt5P4tJ+oA61nWxDSO+4Z24GTyeRUkhYY+8cdNxzTAgutrTkq7MMdWTafyyagIFT3CeU5VgQw6qTyPr7+1RBGbJAJ+gpANAqdJGijkZWKsQACDyDkH+lRcZpcCSdVLhFJALN0HuaA3Oh8P3kty1zE5/esobeOCcZyT+Gcn2zzya6aNfsMspkhJ3RFGVhgjcCoIHfk1zOnaJc2d+X+0x/JlW8pjzzyOQMiuhhYwXDKWHPcng10wu1ZmU7J3RNYWl4sjupjZcZb96AcBiOASCefT69Oa03MsZzKAFOF++pOT6jP1+lUoDNLBKsaPIxCk7FLYB5HT2yfpTBNbmHc80QdCBsLDdyCf6ChMVrlC5luLm5cRxys7ksEVSzY6n36ZNX57S4itIcwy7VUA74yMNjcRz3HPFRzMrRhoyMDv17VGk7pdR4OCDwR1454/GrV9xOxyWpws+sTIDkOwwMevSuwnmaIu4HKpgH05HI/wA96y7jWdRur+NXuWEZcLgY3Y6EbuvrVrbGygDK5BUHccgE++fSpho2OT2JYNauEHlG4fyyMbN3GPTH41ote+TdvGLe0OxiuXto2PGQOq/569aqS6dFC8cLKN8RKy4Y5dh3yc47dB/PhZkjM7yCQ4fB8xkAye/AJ4HsPwHGafmifRizPZeajy6daqA+5ym9d/IPQMAOhHAH3j7Ys29np62oVpbiLJ5O0SDb1+7lechepx14GBVKW0e4jwJId4GV3SBQev8Aexz0wBnrVXyr2JX3xS7R1YKcA9cVnZXLuy62mSNF5qXNuSxwEZipJHJ5IC9/Wo10jVVil8i0klkLr/qCswAwePlJBznt6GnLLD9hiGVEglYsAPmwQuM+2Q2PxpLZ1knghcjEkqp83QgkA5/OtVF23IbVyo81zay/vYZEIZlw42+mRg9wcVbtpo7lbwHlo4N8Yz/FvQfyJH40xr9jzJLI2MgZcjG4YPOe44pjN5qQhVEKqzuAiryWBHOR7nHp2xVXkTaJg3fiOe0vnjhSN1Q4y2Tz/wDWNbVpcpqFl9oSBWZkZyrc42g55+i/lXKapNbTXDPDAkRBOQpYgnPXkk/rW94Vw+nvi5jinJeOKFwV8zcuOGxt5JxjIrDn97U1cfd0LqzQTuqTW7AKR5hhOGcA++QDzjhew98zPbWl8qLLdT24DERp5CsEGT1bcC2BjJC59BWduubSd1kheOVGwQy4I9qtXs6vDHIxwzLwO3vWySexDunqI+kmzuti3UUmSFBXILZVXBCsA2CCOwrUv2jt9Qs7vy/MSBYmHJAYqinr9QBxWFOZYbh0mQCSIiLlcEbAF5x345960V1a5trOWHz5FhZSGVXO05yDkdD/APXqLNoq6TMXUdYnubkzwNJEWbJO45wfft/9eqy3t6lyw+1ysUOAVc8/41bit7GeRVEMkUcYJLRMMnn0PXkj8DW3B4b0q8kRlu5Le3XaZSyiT8sYJ5K9u5OQBWDTuaJodZeIxa6aWmtTNNI37qWRyF4HPC4z+JIq1H4vtFjUywkS/wAQRQQPwOarX/gto7WO90e/tL+N8ZgVwk6MTgJ5bHLcnjGSfSuaRZLO4KTxYZThg3BHt7daak0DSOyktrCaJNt9NbNvzIGg3oQW/wBk5OF5xjkr1GeJR4cvLm3Mmlz2d9833Ip/LkA/2lkC4+mTVDGQc8c8GpoJApB3Yx3FbcvVMyUujRUvLDUNN3PeWk9uCdoaVCFJ9Aeh/AmoLG4Ec0jzFlTyZRkDuY2AHT1I/wDrVsanNMNSmt2dmiifYI2YlSF4GQeD0HWovtFhcJFHcWMKbZAXe3BiLJzlSo+Xv1Cg8CnaSQrq5zjXDiUcNtOOAetWob54YjxuI5y3ath9B0yacf2dqyYZiFju08sqME8uMqfToOaiufDt/DC+bSR1CFjJDiZB9WQkD8TQpWeo3G5XuL6Y26NI/nCHlUkwwwDnoaWK/ilRYZLW2K/fLCMIwJwBym3PAHXPPPUmsZpJURiAchMNn+dO84mzghXH+sz0wckDPP4CiVgV7Gnm3ivdhZzECAygfMRjnnOMnnt3qW2QtJDGEjzKVUy7xtXOOW9AM9cdjWddB7nWJEAfagVT+Cgf0rZhVYlCKB93Ab2FOMbik7GRLp1zexyTW8ZlNsEE3lLuVS/3envkexwDg1Wj861JUKYpBlJVbrn+laZumW8AJ2oxIx6kHP8AMA/UVeXWb3yZI7iV51clSk6iZecZ4cEDpRySWwc0WYMcbtucOWePJOTwajlvPMRCqfMMk454rsGvNAuSkd7pTWiLEMPp8gRnI4OUIK49+DWO1jZLMqwM/wBmKDLzKAwbJyMLnI6fjnihS7oLdmZ7XNwsMckR+ZvaotQ1u5WGONSqSMMuQBW5e+GtUu7S0l0i2nv4fLPNtEz+Wdx4YAZB+vXPGa5G/iuYdsNxG6upO9WXBB/nWbl2NIruVzeS+a0hJJxjnmmvL5mAMg9SSeKQrkdP0pNuBkisbs0sh6XMka7dxxnPWr8Go30EUkkFwyeYOdpxxWbwce3tUkOWwh6HtTTE0bMHiXVjLEzX0+EBC5fdwfY1qw6mb6Jlu55ZXY4HmNuOB657f/Xrm0jVmI5wvTHT86txxhWJ83L9Tjj/ACK2hcymbUt6DdyT7maaQnzGJyXJ7k9c1BLdvsK7mBIGQG/z6Vm5kfDiN+uOAcH6GpFjkUncHDjBKlSTj8RW1zMlhul2Pvzu3Z57U6CdZ0Zl6AnJNW9Ot457d2kjjdXPC44wO+Rz+tWHtbOCGZxGI90pZQgYhFPYZbnHGAfzrGV7misyj9oj/ufzorVi0BDEhfW7RXKjcpjcEH0wRn86Kx9rErlMCHVEW3a2WNGRkGSxwy9Bn3HfFSSNKiND9oJMZ8vaVOSB0HPb2rHsUyXR2SGbG6MseDg9Bn6f0pi3NzJesRK3mNhSzHoOnJqeZnBLDpt2OhtlQ2Fy94WIUBd65yBnIzzwOuMVC1xHbRwiGRzM7FypZiUbOMDnG09emcGsiVruDzbdpwyMu1wr5Gcg8/iO/pVtL5ZY5o5LONhJGAgH3o3A4IzzjjkU+e+pmqLi+6Y/UHupIGTB3M24oMnkeo/PHXpVJpZ1hNrIANw3l04yP6enardrfNMwkaJZpYXXIcAhgOACueeT0/wNQX95c6i27yAAilR5Y4A569fT9Kl7GsI8vutCwG2ljc3E5SQgJgEDKjGDz1PH6VRnXzJx5JyuSSfSo0DzMittRh/Eenp/jWo8IspwskgaZRlHVc57g+h4NKzepo7Qe4mVe5hYqWjU4I7Bfw/yaupNiVbU/Z5VkYRq7McIMjnI9M981WlaOXT7eSORUkziSNIxwB/ETnJzx9eegFTWuiXlwt2ZIXWSJN4QuIyuexB9h61aujCcI2vJm6dUmn0t9PltkMJ+ZnD4ZDnhsY6Enp65rBlt0jsFE2W5JWQMd3+7zx74H156VJb2+o6NqEccjEME3+XDMuSMA4wchh6j29qtHUZpxe2V9DGxh+QGONYyp98cHp0IyeelN2kYpcj93Yy7cXG6QwfaXikjDMVPGB2f05Hf9anMM7xxD7QTISSimQvwT93Hbk9h1NX4v3ciCKBfK8sjAODI/Q8dD+PXtTkSOFLk38EEiTDYilgrRnB/i2nHc/h7U1CyCVfXRHP3kFxPK7BEBU4KhcCkkRY1RFkVuNrL/drfk0W1TT4kVC90pPnMkpbeev4YHp165rBuoYIZSsLO4C5IxjH1/WocXE6KVWFT3U9hGklSYlZSBtILAnn/ABrRjubudQAFVYMM0gYr8/bgd/8AA9qzJ02W4IMgjbhQ3H51oOklxGlxbyKCq7ZERtoOD2B5PahaIckrbGjZRRsglaUee+3ymLbQWz0Lf56VLeC2WO3trm38ueDJk2fc3DkEMD15PT047Vg2dy8EXlM222nkDPtAJGG6/h7V0MMd7rNtdTI/mJ5imaRmG5OvzcZIyOOmOK0ck1ock6bjO/QiktrW+u7qeNWYlkIeYgOnA+Y4+UA9+DjrU0kdnaWcCF58SRkts/hf+RGTnjnBpbW1t7pIprc3M18qEyxOQzOQTkAkcjGOx6VWgn8rVk+y/Z5VwsYRtzqCOmW+U5HHTH0Pc02FZzbV9i8C8kTxSSXLMFwpwVJ2jdgpyDnC9u2Tzisq5mBtFtS8Rsbk7xFGNuxweSVySD+hB4ro9J8LavqF6bq4kSyt2IJVQ7l+c4AbIIPXLegrrtN8K6Xp7tILd7mdjlp7g7mY+vOcGk9TqoYaVr3MDwPo8ltPcefb7LSUfNcMuzd7DJByOcH3P4d7HZWMrAxksi8ARgMMYxyDkflSJaqZATsP+8parzwRIB5skq4HB8r+WTTbud1OnyK1yWEWccZVoTg9sHH4AGpHvrYgoEwgHdWH/wBeqRMKqG+13IJ6ZjA/UvWVea5bWjNFHcNOR95VXGPr2/ImlqzTRG48mnRRGRVjO0Z+/J/jWJqeuIbcNbgxR9SVY5I79Saw31iKQ75J2BZigiLMQeDjHvx+tW00caZY2moa0cTkjy7BiMAZ+/Jg/XA49/SrUO5DkMNvqWq2ZnhVLWwPMd1cSbQxPTA5Y/Nu7c9aS/MEentZWM5kFvIZLmdhzM4A+b2XHA9APc0mt3zXtvMXVQu2NRg8KynbwPTAY+mDUHhq3uILK61WUBI0iIgLc+bJ6gdwvr2IHvVok2otNkk0uJLp/s8zbX8l0Jc4AA44xnaep79DUF7/AGPaQBHjkcFfmEspBUlScfJgH05qC2lvdUvFMStI5VS8hJAQc/Mzfhjnr0rSv7DTGSS4mVrp8L8oJRNwz3ByQePTp74pkmdO2m3F3DAbdoPOUbW81225xxhif4gKqqgtWeCSJlwpbBzxnOCD35BP0NX0TSLm4GbIAInmCRZnyuPm+XLYHX0qvdpObhoywfIURsD8rLgAEZ9wfxqZrQcWrlCV4xGuQTycAcVXmuIlcxNAI1UEEE5bIHPOO5/nVh4gxUSKQVBGQQMnk1Ve2cxtI0iEZx8xPXqT/n1rE20KYkRUeQQRAMcBpcnH0A/Cs+6mZ9nmRuMc5ZMLx069q0nsZZghjDsDwWCnGahm002q75pmZAPubcf1pAQzJJHDsaJ0YYJUgZ2EZ9B69R71AzM2ZHmmk+XBdjk9ec5zkVMJ0YBAvyx5JyecHGPwB7f7VFynktGm3KuwU4PtWydydiu9yZLmSPdaldu3AgCy9P7wXk/jzSbvskXlHKoQWZ2TcoJ49Pl+vJqw0yeX5fK8fK27hvpx174J6EVBFJIZMFye2V6jpStpoHqQWItcuYZ7WS64IEO9dqgHJYsqjOcdCatOst7utJYIRabvM3pIAQQP72fc8e/1pYPNhf7RIbqRWxtM9oUiyT8pDBuTweowRn0p5lCz8RxCRMOf3jYHfoGqUhtlG/jjhijWWyjKgbQTccngdQGzj3pYJA0ap5VoISoBUjJUDHfBI6duadqciTRRmRbcyNuLY8w+mOfektzHBFHsWHzSo4EYIGfqKdmK6JBMsqw+dLC+M4JQllBA6kjJ/E9ap20en294881xFOQr+XFGmCWKkL6dCQfw71N5jxgFpLZcHOBApx0yOR9KmOqwLa3EQubjzWmDB0hAGzByp+bgEkflQ0NMjkWKa5UzJKHRSfLlPl7M+vTn/PtTFjjS6jJntQEBwQu9hwe+PU9c9qknYXEufsU6oY1+aRjjkZznAHIpltF5LKTBbISCyb9sg9OQc/rTSFcnXV43wbnU9VubWEAvAIwBt6AbjIcDJA6d+xpkEe6481gsADcbcluvb/AkU21tprya5W21OBfIieaTyUdfkXlsDaM+uP8ACnRMfNVlAQbxh37e+OvX0HahIGx4EUxJSJnPQGRsAMPTGMfQk0qzOt6rXlxpcQVwxxbIXxycbkQ/TOe9VI5l83dH5lwoXbgJjnngc81Pbwo19Ck9pp0Adl3yXk7RkAnkkGQdueB9KcloKIx3uJbOZpL3+0JQNqSAyHyxg8fMB6moVO2PDtyP4YzznjgkjFWp1jlu3RIoIoTJ+7jtpUkAUnu6lstj1/8ArU2NnhhRI0S3VztaR1+Zjg555PYggccc1URSFvZ0+wtFeXr20Tk7oIY95YDoSu4Djn7xzXPpbWazD7LLPcMHBA8gIAMjqQ5A7/41sXCLFbEpbee4UgySZIUkgZwOAOQOcisG912SyQ28RR485EQP7vrnOFIrOej1Lhqjq7SOW48wIEkaNWDRnBEg4G3PTkEj8vWuN8Q21tHqKWlopihlwxjZSWjPIxzyPp7+mK6L4dwXes6ldXNxcFYEChscdSTtUdByAf8A9dUvHkhh8fKkMYVoFi+VsnkqH5787v1rN1Lt3WhsklFROdvtFvLCISyJmBsgNjn8R2rMSAsygkAt0B6mvTlkhvNO867ZTuJZi3yiPB4UD2PU9z7YqEaPbzxeZ9lVRnKs64J98dR+OKy502OUWjz6Ww8rau7czDI+lKLNVtZZWbleBk98/wD6zWlOP+Jy9pFkEO0YHYD6fr+AqG7sZIrfyQQwODn0IAz/AJ+lWZ3MYHFP3mOTKNnB+U/1qZ9OnVC4Xcgzkg9qhELMcAgn0GTQ1bco0dJnsbYNPdKrzBv3asCQOOuB74rTuvE8cttLEsZO+NkGEwMkEZznNc0UCkg5yOxGKTbzgDmpsiHBN3Y9p3k2qcYUYA9K2dL01LnTpgwIaQfKx7HsR7f/AF6xmieEgSIykjPzDGRW5bXLW2nRSE/IcqeMY4OMfr+VDdkXFJux1ng3xBfHSzbrezo1uwHlrIQAQMK2Bx0GP+Amte61fVHhld7+5yynd+9PIweOvT26V5TNqDrfT3FmzwLI24BWx+ddv4Y8Wy30qWl3bWrTDO2cW0Y7cZ4/l7VvCatZmU4Pc3X1fUTE2b25LYJ3GVs59evrSx6zqkUKCPUrxEAGFWdxjHpzTb2dEwTZ2eWYnP2VBtHHAwMY59Px9Io5E+zGVrW3dY3WMqWlG7Ib0kGPu1qu5mRi4uXuTdPcSvcMcNKzks2OOT34GPpxUrMos57iS0tpRGhcgoUGNyjgoV5+br1/TEgutPJKtpzxAdDb3DZz77w3H5Y98U5JLe6tJrHzngWRWCtJCGUcoeWDZ/g7KaH6At9zkkk09ZEma3XzeCeX3DnnHzYz9Rj2rQNukscTo5+cByXTaEPXB5JP5fhTZvDFxJeZtbqyu0wMeTMFLfKD8qyBWbrjAB5BHanzwzWoSO5jkgcqPllUoc9O/wBDSjYcrluSC6lld0SS4DSEb4lZyxJ64xu/EgVVnYxlA2cglTuPIPBx+VWIIiyGYtjaQuO5znn9K1ItTv4EULeTlV6K7llx/unI/Srd7aEKyepjxzcDAIIqIxSxXciOEYFdxOwZw4zjOM/dIH0rYS/t5WJu7G3ndsbmQGFup6bMAHn0PSqjixubwvKs9qhAGY283HHHBxnsOo6Vmrl6FRjFExby1UMOcIDnknuPc/hx0qBovOdJoDFbuv3RGHPIPXJYnPFW7uxZWLW1zBdxgbjsLK2OP4WAJOTjC5qW50i6sHLS2ssUSBEaTaSm8qCRu6Zyc4z3qroWpQNoGCAOqEt87PxGvPBzycdP4ajAKXiWTsPMfhQpznOQMf4dauJLtjl6tlGClRkgkEA/h1qvdz7bSQrIAyLnB5xgU7voxadTlrvQ9TM0jJYztGSSpEZ6ZrodLcW9jDZSx7WCHerjn15rl2vrwyFvtM/zHJxIRk10x09pTAPM2sUQO7sWO7aNxPGcZBxgHr7VlTdpNms1dK4jvvvYyJni+UBTncoA4A2njGPpViWC28pInDyXCq2X3hVYnocBTjjtz9fStf6fPZ3YRlOAEKyFWUOGB2kbgOCASPXFW05VGOPmXK/Qcc/iDWqSZm20WLsx6hdz3ILMZpXmkWbaG5IJ+bO1iSTz8vTOOaoXFjO9i8sSP5G3II5AxzgnoD7ZzV4riIY5J7Coopri3lZred4Z5BsEsTFWGfcYNNxaWgk1fUrWKpFoV9KciQSwRkH0O9jj8Y1qC2vCkZilbCISUPOecfyxXRXGpW86zrdxReVcgbzGoRty7sNuAx/Ec5BJ6ZFRReGdJ1CRBBrIgLsFUXkLKCT33JvAHu2Ky1W6NNGc9BezJeiZWYNuyDk8HOa6W+1O48UwwDV5GuPs/CEkBx0/ixk9P1NNPgjW2t4ZrO3jvbdjvWa0nSYE9SBtOc+2Kr2zJATG21WBCkehxTglJkybijRlt7O7VltHmt5VQt5cpEiMQMn5hgr09G57iqc9nexGJYBFctMP3QglBLdOMfeHUcED9DSGb/SI1BwfmBK8ZGKQOIgHjLDaw+79a0cLaIlSW7LGtuTrV3IqkFpWOCuCOe47GswBpATyG/lWnJfJ5gE6xzllK/vQW47YIII/A1UheBnk3DyS0m1DyyhSTwerccDjP07002lqS0mysoeGNsninwXLeY7ZYMQASOCPxHPfmrR0u61DzEsU+1NGu9kh+ZtpJAIThu3pwCM4yKoLa3NtKzXEMsfzFPnUjJUAMPqD1HbNHMrD5WWpWFzD5b4CgYB2LkdcZbG4n5j354znAFQSWEUt4jIFhh25cZLfOB1UY4B44JPfmnp0OTk9QalX0J7mlyphzMlufDtzZvNqMF1Bf2wYb5IZfmQsQMsrgNyW4wD746VHfQzWJjhlililUfMsiFG+uGANOaNJU2uisPRhkVqya7eLKLeWUXtskSL5d2omj5UN0bofpg8daFzRemo7xlqca/mLOu4kMDkZ/U1pKsjXsjBxsUYA9T3reDeGb07prCe0dWGWsrgkEHr8km4D8D3pNQ0jTFs55dM1yOZEG8RTwtG5H90HBDN9AKcZWeqE432Zz94hMgKEenFFtclUZByQen41IsUiRSedBJCFYHfIpCsGztIJA4IUkHvg46GmxIBmRcFHHBFP2iuJwZsaVqN3p0v2m0uGhl6AjB/MHg/jVnV/GV/cy28twbaSaJcBpIFdu397OOn8OKyY3Qxg54rN1STfNsxyVBJqJJPUuLexuR+NtWiaRopYFM3+sCWsY3n1Py1Xv9fTU4ik+m6WWZcNIlogk+u7Gc/j+Vc0FdPUDtmlAyRluRzUK3YptnWaZrOkWUSwnw5pEyBefMjy5Prucscn2xWsuu+E7lh9r8KRQgDH+jttz+C7a4DOBnvTorhow3rjvRyR7C5pHZB/CNzeCAaEbZDk+bNczso99quT+tbt/YeFNK1WOK+0Xftt0aKS2d1JDFs7huwcYHU55715xDNNJLGBJtwRhsDK/wD1q7n4lyC4vrRwzHbApJ3DafmbBGPx9qainO3kF3a4S+IvCsMJjXw9DKQSxZoI93/oRB7dTVFfE/h/zA0Phizk3cFXhiK9eTyhIPbjiuRZkXB5JdcjPSq3nyJPhImZj0VF/wA96qSS6AtToZL+3udQmmt7SCyiOAIIRtUdzgCq11eBoXYgELyOODWVHcNFNIko2ucHGcgVs6XDb6jexW106rbhi0mWwCoG45PYcEZpOXULalNlEzGV9jM53EsuSSfU0VAlxJsXbIAuOB7fnRUcrDmkYe0Ncp5UpOTnLfwjqTirS4lZph5cYQfMq9WNRQ3MUcEqtbRuXlzkfeK+me3rRaXFvBqUUlzFugAwykngH0+nFYpXdjKd2npt+I7aZVMscYCkjcdwzjvgd6fII1lUNIvuxyR9eKgu7mBtQuGsFK274+Vh045/WkDPHE8b7MZzyeh9sVNhKOibLsy2QeBbVpXhaMFmYYIbJyOgyM/5NZzSRqXG4Ft2Q/OSPerp3zAlmZh0yB17VRvYYw3yEhieBt4P0ptNFQs3Yt+dFNCpRAgUfvDjn8P8/wD1p7O5kjkhjtnbLPvU7VY7h25/GsxNwjI+ZQAScfrTjC/7toZGKxDcXUYIOe4/EUXs7h7OLumdvDKms6tdRzlI2kQuZWUl4TxgKD7kZz2zg1DerNaSeW92+6aEAsGIMg7ZGfXOKwgCYY2ErNcSIJZGl+XBBOMH0wAc/hVnWWutQi0+W7kUW+AiyqCWKnrn24yPqatS0scDoe8lfQtGKN/JN15zJbJ5hEb7ZMjgcnoD9O3aqokto4pGtC37w52O25xnPUjrg9wKgvrueORorkpK8m0NchDuK+nBwex/Ac1CsEoV/IkVkUEM/qM9M0k+qL9n7tmzTW6meC3gyuwNsyy4dR1wDn1z2z/KrtzLJNuimwr27LkucZJH8Q+mATxmqUNvPstFSdXEwOyL7xDHBJ9iaZc28lxdSu+955HxIzkgZGADnpVanO4xvuW/7R2XcX2GTykRgu13/dng8ZyMAZx0HU4rn9Riltbx7VwUlBIZmfOc4OevQjBz71fa3XzFsZXW3kYEqzqdrZ4AY/w9+cd6uWfgXXten8+PT5IIBtAnu3MYIA4wGG5h6EA8Cpld2O3DQS1RiSI5PmSytJGwKknufYGtSKyiksoTHIY45VJYEhyGyB0JXJIGcLmu5074c2iIseqao8uAWMVuAFBxyd5Gfw28/Wu40jStL0sA2ccELsud5yz492bkHk8YBpqOhtKnKTR5rD4Q1bUdFsra0tJmgjGTJdR7Nhyd3J+YDnIxmr+lfDC5E/2q71m3t1+6UiG4gcdGbbjp1wa9Nk3SIGadGB4B8wN/WkS38wfef6KhzTtrcccPGN763MPTvBugW0oliMl1OrBvOluDnI74XaP0rYTSdOgP7i0tosnJEK7FY+rBSAT7nmo7yKG0iZ7iK4WMdZDE4X/vorj9a5a/8UaNaA/Zy9y56bQQPxJ/pmi9zWNOMdlY7OaOCKP9/DBGh7t8g/MmqTz6em4eYGKjpE9eb3Xiq9unKW+IQxwqxdT+NWrbUFsrDdJOgOMYPXjPFWordg21sdRdeIbOEZWOXAPJdxnH4CuUvfGF7KGWOUQoegjGD+fWlstD1bWoI7mG2KWBba93MQkSqT975sbh9M1tWHhLw1BNGLu+uL+baWP2fCwnHbJG78RT5ewXtuccmt3jl/MkkkXh0JcnB5z/ACFbmieG/EWuLK0MD20RjBEtyhRW5B4JGe2cgdq7E6vpOntBbWmh2McYJK+YvmbTxnBPQnH6Vk614p1K91P7D5xitRkbY/lDYppNickiK3trbwjqEqrKL2/8vaLuQARQMTkFEGSeMHcT9BzVTVdSl1dxJOx8ySLAIPUg/wD1sVkNcK2pS/aJvkWT5mY9u2a63TLKDTLEXsoMd3GHNoGH3QejEHoc5x7c+hrRxSimZ3dx8dtZ21hE+oR+ZPHHumgc5TdkrggjJPzcgnHBzmqk11Pq0EcVsP3kji3jjX5VCkYVQBwB09utZF69xfarIcTTNKociNS5PYnA9hmui062bRLI+aQLt8K2CD5S4yRnkbvU9OCOaHa1wVye4vVto4NPsgqwRyrHkLgyHpk98nOatXGnIRIuoO6RuwYRRcvwDwT/AA9fc89jVWxV7WznvXx9ruYy8K/881xww92/l9abA893YLc3bLBEJMiRgTvAx90Dr+g96Q2QSy6ZpvFvpsRd1KEyO8mAOmAzHHGeetI80L6ehgLKolBBJ+ZODkE98cVPfX1jDbCddPilbd8rSsx69yAQDxmqqy2pjARQiyDzGj3fdbn7ue3ysO9KS0EnqZThtrOrdDtx9c1WnVVRR8wfndk4z6VpP5EikbQvBO4k1WKRuzOI3dguT8vAHTP5kVhY3M3zrlHIjkfyouSAduRn+pNQS3YdW807f7vGa0JAY1cbVUdGywGe/rVWe1226OVXbICyHIO4AkHp7g0ugzPtTbx3BM6l4nQqDnDJnjcB3xVm4iZyiyMAybef7wHT8Mc1FNaQNKohRlYkYLdqLKeNrd0uCBPCP3ZYf65M5KE5+8M5B+o9KqLFJCrbq7NyCjDBX37HOOD/APXqHy8uRmPI4OSePYgc1dhlR0MluQuc5yB09x/P061Sjt5XvLtURt7KpOzsdo6YJ3CtNiBbeLy5XaJowMlWMMokOD225yecHn0oMscCh8w7wMENCpz+lQ29vMMb4bJiI+ZVwjr6DnG48e9Tz3JWUQyS3kLuWKhotwLDBwWJGDjnoaS8ymULqZZZ7eQz2pUrmQtbKT1HABX06U9JIVCILjMITBZYADjtxnj1P1qG6nBnj82+uREV3f6sOV6cAFx/MVaEgQKi3kzRjBLOuD0/u7iAfx7UosHsLBclLlXt7yRMn5ZEjw6gduD6e9XZNcjW1mtV1TUJbmSdT5zryqAMCmd5PJYZ+nSqKXcV1IZILudhE5GWTaSRj3/rVdr+7EzpJqEyxO2TE4Dq+DnqTkdB2qnawky6okLuFhm2lv8Alp0P6f5zUUkDC7jZ4okO0kKX+YYGeVzkZx3p01oJpPtSWd+8UalmWM5JAxyG2Y6+3pUNtKt0Eu7a3dAyEbZ3BYcEHHAyc+lO/QVhW1eWzaV4p9KjcKyMqWaByGHzKSI+cgkHJx1qSKFn5VXZlH8ZxjvnsP8AJqS3WXe7LPBaM5LKBGFc8dCyqTz9ceuKJFg3q80s8uRltxCEnPHJznt/nmpjoxy2IooXaTZO8KRbD8seHIJGO3H45zSpp1o6SBdON+yICodpMgkgcCMrxjHXPQ+vFbS7K4ijuI5IGhVJiMXbfMqnkHBG5h3yorQiYmJlLFS2TjblT+uabSaErohhhFvF5n2VYBIclFbHGPclvzrL1GOI3el+W00kvnYGOM52j7vOeeOvOa3vswmQ/I8wH3gSTjHX7oB/OqF0su6KMXKwIHXdCjY8xQfReD36kUr2H5kOtytaRSFmiC5JAVgQD1wCD1+n6Vx5s2JMkowW5xnnHf8Az712ep2CyXsJdWFrAgmlL8g4OAOB0Jx61g6i4e4ZgDuaQnB57nr61lN3ZrDRGfZXd7pF0LrT7hopcY+U4z/T/wDVTrnVrm98RPq13bebIxBdchc4AA5+gApvygc9CP8AP+famMwGM857f59aiyKvY6LSNa0tdTZpVWNpsMFlUZifJzgnsRjnOc/WuklumYh2KIg5X5wcj1rzGVUk4b6e9MU3druME7omMkK2AR9Klw7Fc99y+0gPit3VwoNw+GIJHftXRTaEgw8l5Mx6sFCqCfbriuIiupI7xLgqrurbuehP4V0X/CWmRNk9qy8feRwcH6Ef1pakWfQuappot452QFk2EfTjvXL2DhLlizAAoRk/hXQ/8JPaTxOrl49ylSrL6j2zXMW8kUUoaZSyheADjmtHUbs+weo25Km5kKnI9ataVkXZZCvmhD5YboT/APqzVGR98jNxz6Dim7irZUkEHgip3Bo1tUdhBDHMVa4ySxBzgfTt/wDWqi1y/wBiEG4lS2QM9Ov+NVsk80lFgSsOAyKdG7xOHRiGHQimClzzTKO58M60b2WO11SSaaJdzIEkVW3cZySpJBxjt7V0kj6ebRo1nukzhsLCsgLj/ga8YJ/+vXnnhr59Wij3KnmcbypbbwecA81250+7SUiGS2uBg4ZJAoP4Pg1rCWmpjOOpaiihMkbQXqskm5C80TxhRggkgBsD6E9R+FpNF1Ty/Ngtluo8kBraZH6DJwud3p27isy3ttQitQ0lhdLHkkMYm2nv1xjvVS9vVKpl1ZQ5BjY8I2Bk+2QF/L2rXm7MztrsTyQy2czW86lJYztdD1B9KtWmr6nYIY7a4kSDnCcFOeoweO/P1pNO1eeZwjRx3ccUeWS5QShVHcE8j8DVuS60y7VIpIGsY0YsDAWk5IAJKu3+yOhHQ9ezTvpYT0KUOqQSSO91p9tKpwCIw0HGQTjyioBODzjvWpG2iXHmszX1qTJiNF2zIqccchWz16k/jWLPaQRzJHDeRzxzHiSON8INxHzZUYPf+tWJbK4tYVkuLaVIycLIy/If91ujfgTS0HqPmsAQZI720KcH55PLOT2O7AyOe5HHWoZdH1MdbGdkOCGRS6kYz95cg8e9Nli/cmdW4VlXH1z/AIGkS6kttssErxuucMhIIB64xRZ9w0KUxdW2np+lWLO+urSLFvcyw5O4mNyvIPHI/CthdfkuCkmqWNrqQkUktLEI5TyR/rFwe3Xmm3cGg3EDNaTXlrKGyI5wsgcbv4SoG3A9c/XvVJ90TZdGZcdy6zyz5WSWVWWTzlEm8Njdndnnjr1GTjrUE1lYXEQjxcQsU2llkBDMB6EdDx34x0Oa0zolw5xaS2978pYLbygyfTyziQnHPC1jajE8cRVkdJ4zu2MCGBB6EHoaT5WNcyMNdKETs1yk/wBnUceVgtn8v6VosXW5KAPtUhfw7fpVmPU0YSMq/MoAUPyCOmPrxVqyuLKSNo7i1dgR/rIpdjjrxyCD16kE8YzSUeXVDcubRiPGBlAMfKoYZ56Dv9RUe5FZy0CktkKy/IVznoBx1PcHpgYqS88sXjyW6yeVK42LguRx1J9Sf1NOjD4YSRshwrYZSOCODQrPQG7ajWjXKpG5yc5WVTlRk7eVyDkDk4AB9uaLbSr+5gS5hsriaNzhWiQycg99ucdM4POPrUgVvtZK8Aooz75b/EVct7y6srgzW8skcv8AfRypP1x1+hptS6MNOpy90XW4MTEhkYhh3B/pWtbz8AE8jpW4nin7WfK1myt9UgwQ32hAkw9NsqAEduoJx3qzDpfhy+V5rO/k00qpYQ6hhlbA7SqcKP8Aeyai7T1RVk1oYcGrXOnlzazvE0xAdo2ZW+XOOQRkcnrVw6pG9ibKa2gkt8kqpQBkPPKsuGB5PUkc4xjiqmq6NeafMrzwgROu5JEIZWUnhsjoPTNVkYrGpcfKfzFWknqQ21oXfs9lKFeBzFKq8LIcqxCgY3DkEnJ6Y6DjrVRNOuLUCfy2MDtt35DISQWwGGRng8dRg5xUcJ+Y5GcnipZGbYA4cI7AhiOMjPPuRx+fvVcr7iuivdLtUOQTgHH51Fah3lBVQeN75OOPp36irjROCSvlvCSWZdgVm46DHygZ9u/fgVPDZQ3VzLHayMjpEgijnQRvKxOCACccAZwCxwQBnrT5rbi5b7Fd7uS2lWSKVklX5lZTgg9OD+Jq9b+IbuO2MV00d3bbt5jvR5gB9QTyvQdCKypkLSKkhAYosm3cCQD0zjv7diKwNfu3VVtY84+85/kP6/lSk42uOKd7HaO2jzsGg86xZjyjgyxKAPX73J+vWo47GWSF5YUE8K8M8R3Y4ySVxuA9yAOD6V55aaxeWe1VfdGONjV2FhdrdQw3MZKnqMHBU/X61nCSloi5xtuXgPmBVsoR2PWjVIvJvWjjyCmEZRwSQACfz5/Gorma4nnWWS48w8kgKN3ph2xljwCMk4yPWtKaLTdQ2yLd3dpdHJY3WJo2JPd0AYev+rI960u09UTZbIwooW3MGc46Hnv2qXzRHhY2PQnd3B9asaxoepaIjmQxTQvhhPC4dCD05HI9sgZqnk/Y1CKd7cdcjH1FaQknsRKLW5NbanNneJmjZlKkoxUkenHUV039sWd3Zxy3+h2s8bN5LTwHyZM8HkKMZ4P69M1xEdvIkbsxCyKQdpPUd62HXGi2TLLKHklk3x8heAmCO2eSPyrKcU1sVB2ZuPYeGbi6aO11ObTg0h2JdR70247MDkc5+9XNNod9cTzywyW0sSMVTbcKGcA4ztOGx06jv3qjrRl/tAlgykJGuW74RR+XHX8azstLOZPNCnIAw2AKyeiNVqdI3hDxLIgaLSnYlcjDoT+Wc1Ufwn4ngB87RbzJG75Ii2B+GeKZBp92GH2Zgzf7Eg3D8OtXYjrmknek9wjn5SoyflPv0x7VN2+o9DNn0jUbVj9o0+8hGP8AlpAw/XFVWyvDYX61vxaxq0LrJ9sleSKQSL5ru5DYIyN2R356ZrYHjnWU8pZrZpEBBZY40O857gr36HH4YouxaHG2dyRJiOMSkHoqkn9K7jXJwx022mBjaDS4SQ/AHLA8+ozg/Smv8SL+1llKaPFbxsQQJFYFW246jHcVzerX2o+JHinvikqrkKViVeCT1xz+dXCTUuZkyUbWILu3hjDSCZTHnjY+Sw9qhsrS4kkWKS3aAvh0luB5a7cEg7jgc9vWtC3adV8qBJUjCBGwcZA9OayLtmjCkodhJ+VgRk/41pJ31Jj2O70OPw5LoMljrcUfnB22TAFmAI/hKdMHtnBz+VTSbLTtO1+REubG8tDHhftiOmeMNnjHALHkHjPfFYtudtvGoGAFH8qkE+zhWI+hqPZ3Q1O2hl3ZhjvJ0WeGZVkYCXIbfz97PfPXNFXyZCxIkAGf7tFbXXYfOchKjy7HSMRsF6hvve9SsUOFw5UjLnaCVPsagN3vQlmJfjheM05DLc+WW2xqTt3dgeeuK8+1wtpqLDBJK8ohR3ABb5RkgA9fp70P5kjKs2QqcelXfssliXU3BkbbnjOCPwINK13cJYwloY/MiJG4FgzKfUEYP5/1pqxDld3WpGJ4Li0EQ3jYMZ4z7e9RRNE9xHEVDeWMBQOSx6n3qC4vJJYyqwCOIsSqgZ5PvU1pawyLEsqvGRyzq2WPtgdO57/Sjcbjyx1IFdmMgSVkTJIwDhj6Vdjilj2xyN5RwD8wJOMdAPTpU7adBNfiCxuh9nEYk+cnf2+XoOefTtUdxcSWNw7NCN7ZAVwMrk9Rtx+n/wBalqiVOMnyrcnja1t7Z2kUTOrsi+b83ycYwDx19OmKkY2lxpkbweY8qZXy3yVjySeD0wfT3rNxIF++GkGWOCNoAPJJ/H8a6LRvCuranCZtL0+5ZJUwXkYRISf4gz/e7/dzT6EzpuWvUxoJ4ryYWt/OURBtjYDhCPUDrTrZ2gjlt4myhzhsZ/H2r0DTPgpf3Mgkv9QjBYfdtYWkH/fRwB+Vdvpnwzs9MTcmnSzPjglkB+oIII/Oml3KlRvp0PFrPR9RntkmtFYKW4LYVT+J4rsLfwzrWo6it1cYTeAHjt7dnVhjByWAX34zz9K9ZttPubYt5OjPH2LoFBP5dae8N8SAbGUADJDR5qk7DWHhe8ldnLafoVpphScRKl4qqn2iaNnlwBj7wBx77cZrQLI+/fcbgOrGKYA/T5OatXSRxuDPauhPcqwA/wDHaz530+PALRDcMjFzzj8qd2bKKSsh63NrGwBu4Qf9pZf6pQ11Cyl0ubdu3DYP5ECqU9xFEu5Un8vg5R1OfzGDVCRpLgArO0UXO4ghmH44x+lNJsG0jWutQFlbG4laPyh3Qh/5fhWVHr5vAXt1wm7aPM7n6D/GoL+yhmkMHmbkWENOd3J+dQM/i1Zn2O8tn8uwxN/cjwWbJ9B3rSMV1M3JvYvapezQWk0c8u5iAQoOBg+oH0NKukQPeXluluSomeOKM9Tg4A578VqWvgW6kP8AaPiS8SztOHNvFzLJz0z0Xr2z17da1NQ8VWFhNcy6Xp9tDcuC73AjUyMSSTk/X1zTtfYTdt2c/pvgWcSpe3Cf2dangeYcykkfwjoPxPHpV7VNc0nw3cQnTdKt5LkqEa5lHmPtGF47A8c4pmhaxea080t1cs7eaCQz/dBB6DtyB7frXM+L7drW+jDcgk4P8ql6NJlJnQaprt/q+ofZJ7ktAgVtmAAD26detYlwz2OpLNA5CkHC9skEVEpmimtrrbK4u7ddgRCxZlyGAA5OAVP41tQ6a0WqWk2tFbaBCJfs+VZzg8B88AHHQbjjsM5rZWVjLW47WIFTTYrmPORIrkHqCe38xWfc2l9qt3DLp9s0sigI+3gDIGCSeBXRXt/osiNt0q2liZcFTCpDYHbjOfpirhOy0SKAmOLaNqKNoA9AB0rNe6XuZ0ejWGgILloY7nU1YGSdjuWMnsinjI4G7Getc5datIZ3di0jIThSeuCeM1p3l0Ibp4ZQ7rIAoxyQeo/Dmp9L0KG1vFu9YRdzAtHaPgkN/tgHHvj356EG13JfmECHTLbzlcedMPkO3OYc5H0zw30xVrTrR9TnZp3YWioC8mercjaPU4/LHNMvXkvr0Su5RfmaR9u7aowemfw/EVpRywpbqlqu23GdidevUn1J9aG9LiW43cLjVpJ5QEtIkzt7HPCoPbj8hUFw9xf3DJLKFiWFgXc4WMZGP/rAUSRzG2tNNAAuCqmQkn75UZB9MAc+mDVXUJZLm4jsbGNvK3HPPLnaQWJPT+QoSC9xst5p8bLZ/YZLgJzuldlyRx91T056HP1ptxLAbguIFjZUCuik4Kt0I54w3GPcVZ8i1t5gH/0i42luGIQZ4xwMn1zkf4xXQMjBkhYNtKEoQF24yTj1y2Pp9OW2hWZTmmM/mzOhOSRnGBu69fzNUWZwgBnRFcdA3zde/pWncKI38tlE0RAYAoUb2z8xH/jo+lMa5e3jOIJGjPRWKFRxxyFz+tczR0JmZ5QdYy5KxjksiEsef51XNrOSG2M5BGOp46+lXEN1czgWpaEyHBIZyq/mSMetIbC9cgvbL5o5DmCNCT6hsZx+NSVczJBK5MYWJZXf7zMAQOc/KT9O3aq7afcYcSvnJ6FCM/iRj9a62CwvJoitwglzxyzNz9eabPps7BlZVQk9cHP5kf0osFzi3uJIbiNVi8xzhWRCu58dOhPze/v+ejF9nngeVEQmTaPmOOfQn+FuenTv0rXbwjDcvukmWNj1fG79ARVh/CyCWOVJ5PtBGJZQ2wTr/tAhs9ufaqi31JdjCltlhZPNO0HHyzqclfZxyRx2wKz7uAHUrGSKElgzH9zKGAGPTkg/U121v4ftWjYRXk8aZ5TzA6N7EbcfnnpRPoWnRKsstsw3Zw43Dd+A4/ICrvYRyUdrMsj4NzFjGQkZP4nkUotpQTtNwrE/fVTnPuP/AK9dfa6ZZoxZbeWXJzve1ZsDn/ZrSNvBanaVSBzgqsixxE9OgJFK7QHlmm21zNDeQQGVk+0t0Ukt064rai0PUvJUR290SR90wFRnnvmvQPtUixgSu69cHy2YfmgNKJZpFzG0jgDny1X8/nK4ou7AcCfC+qShlfT5iSpUFp0THHvTtI8Faha2qJLawqw3D95c7lwSeu0A556iu8WO6bk5YdCshCn9N9Kbe6yvktbQcfNvjMhJ9ipTjp2ouwOTHguUsrSS2qOpyNm98fgasnwfj/W3pw3VY4xHn37j9K6R4Z1OJriYk85hEZH5NkipDaqwy5aYY6eY0bD3+UUk7AczY+AdNs2cie5k3MXZGdfT2XNbKaBo1oFd7W2Xb0Mp/wDijzVs2e1tyhW5BXz2aQr9NzGpIbRIWzEUiZuMxxqv9KGMgWHTXYyJBA7/AN6KESH8wCadvkUbYoblh6DjH/fWKtkDbh5CxHfcf/ZeKGaQqojdYz1OY938iKVgueZeOZmGojcqbvJRSpw20ksRnBPrnHfj1zXnk6s0kjAgt0UgdugJ/wC+c++a7fxBLBq9/Nfb1zIN2GwAFAwAQe+Nu4AkYGB1rDe2WZXjWIMkRy3Te2ceh9jz/jgZvc1WxybyEds84A/TH8qjdgS2Txjv6cf1q3cWE0ahmG9S2G2jp0/+KGPeqLI21Cc4Pf39vzFIBUbJwT14I96uwxrMGjxnahYAddo/+txVMKVZTjgcE9q3dLQ25SQA/u5N7g91ByR+X55FMRk3+lm06HIUKWPbnH58n9PxqPT9Ne9u449rbdvmPgc7R1x745ramDXEUk7Nma4ZSOOFHzHPHrlfxzWharHZ3uYVZljQfKSAcED2B/Lp70kh3MnXfC5tI3ubU7o0B3oMnoTll9sevo3PSuYJI4r0i3vGicGRPMijZjynfvx1xgdOOMelZF3oOn3b3LwsbdI22pt+bcSf15I9McdjwBfucZ1ordvtGihjBiJBBwSTx3/yayHt5FJBU8cnigLkNFFFAwpRSUooA67wPZrfXM6KgM0S71OccHj/AD9a7hNFvwd0bCJ+2JMH8K868GXz2PiW12yiNZ28liylh83AyAQSM4/+v0r2M2E7hpI7g3CDr5O3/wBBILj8RVRIktTDzrNrgPq1zHt6KGPH4Grcl+1/F5eoWEGoJgCSUwfOB9R0/SrP2JntpEEjeWx3MjTHB/An+VUpNOiKfLIuz+6qsT+ox+tXcmwtv4d0ee4uH0a9lguWjKm3uBgc9ccA49uaz7rwxrlrkPZeco/5aQSKy/zyPxFXf7Mt2OGaV0/u7Aufxycfka17S+itYltYMuyqMJM4kZPbIC/lTT7CaODZZIGAlR4nPRWGDj1rU07UZ7Rv3c8qHa+CjH+6T2/rW7dhLtibqO0lXBXaUwFJ+h69OvpWK+l2cYZixEgPy7ZDtC4IIxg8nPXPHpT5nsKwjauboEXNvbTAn5iYgjNjplkwT19aQwaZNatMftMBV1RgpWQfNuIIBwf4fXv+FVYtOZ2Cpe2q56iZmTHJ77SPerEun31vp1wZbWVx5kTxvCDJGy7X3NlcjHK8nHX600xMlGnxTWdsLW9t5JlBQwu3lEAsSMFsAn5j0P0zVK5tbmzm2XMEkMmOFkUqT9KoG4Hl7gwwR2NWNO1S8sovIguXFqefIPKEnvtPHr+dXe2iJt1Zau1EcFk3Klrctkdz5jj+lI2ovNbxw3QS5hj4CzKGwOMgHqBxjgjvV19S03UEjS+szD5SFY2siE6knBDZyNxJ4x1qC60RjA82myNqFsmC7xRkPH/vJyR9Rke9JvuNeRnNodg0C3FrJ5G9ipgmYsm7IJKsBkAAjhv++jUy6PPBa/aGhcjdtV0+dCMEnDDIPT1qO5ONItjG4LLNMMZ74j6/57U20ubhFXE0sQJ3YVyATgjkdCcHH4kURTtoEmr6jlZkAx37VYgvltyGmhWeMgp5brkDIOCP7vPJK4PvVmCfTry4VNQSS2DAgT24BGd3Vo8dAP7pH0NTXnh65WNpbGa21C3T53aCVSUQdSyk5BA7DNVpsydd0QxR6fcEYee1YknzCRJGOThSMBgBwM5Y9eDU02kXiQmZI/tEA6zW7eYg+pH3f+BYqgsfyqBkHFS2t5PZ3Hm28skcyj5XjYqR+I7fXIolHsNPuNsIgb23JQSL5qkgjIxnp+VIqCNSEZl7cVeOpRmbzbixhlm2jM8TCCVm65O3KHrjJUk4zweKsQaEbm1MmmXiXcmWxZylY7gAHjA3EPxgnac+gqL63ZVuxh/IqmJol8s/eCqNp57qQQencVPM0c6SOYCqojPutY1XBO1V3KflK7icnIPz4GAAKfNZ3UMrRXNtJDIvJWRCjY9cHBp6xSraXLx7NmwLLu64LAjH4gVdk9UTdrcywUZilsXwdzbZkEbYDcY5IYkYPykn27lk0UwlmhliaOVX27JAQRjGRj86leNSvzAMp42kZH5VPZSW6Oxu7dbtCmzbIxBReMbSuMEYAGcgDIxg4qrSXmK6ZBCVjVUyMjNPkVSwJwQRtwa0rPRNP1S4WKz1KW2mKfIl3GHViAS3zrjHAJ5XtjNUprK7gRXubV4kVBJuJBXaWKg5UkckHHPPBGQQS1NPRi5XuivMN0Uq8qXTazgAsBz0JzjqaydW1GxvYIUbTLeK5iQK8kS7fN/2jg4B9QB1ya159jRsGbKnjjvXK35jEp8tcBeMVnNK5cNjO1AWpCiCJk29Sxxurd8MsTYyDBCiQhcnPYVzdz8wzXU6HGbXTYVKKwdxJIRnzApK/dydpIUHGcctWMdJ6GkleNmbF280NwFuFYK0UZj4/hKKRj8SfxqJZo3Zl3DcOAo6moiLnUr0A3kUkuwJHHJiFlVQqIvzYUkgDoSPeqf2WS1miaeOaJ2Xf+8Qrkeoz1HvXRGorWMZQ1uaCCFbwNzFcpgq6Eqy9xyPzq+1wZbyK4vUiu0XIZSojZh7uo3Hr39qzSAbuVlYSRiQ7HHTAPHI7YxU6EsrZ6+ner5FLUnmcS9/Z+k3unxyfb/sd0pUSRXgHlgk4yJV/h6cEZ7ngEiMCa48P2PlxLNCJ5mWSF1dtu1c7lB3KBsJyccYqoJjHIAOOxJHSrUcnlnMW6NnRlLxkoxB6gkdc+/FTKnK2jGpxW6Oe1iRpdQkYRtFESCkZ/gGBge/GOe9Zsu1lBOQFOTivRft1jqDRxa9p9vc26Hc9zbRLBcD+EcphWGSvG3ilTTfAd7a/wBm7dYtDOfMF5cJG3kMAQFLLxt/T3HUc81JaNG8WnqmcDBq0tpIjRbTtHAYfqa108cagQY9kZRuobJ/r/Krd58MNbkkZ9FeLUoBysiSxKzjvgByPyJ7fSuUv9K1LR59mpWFzasDgCaJkDY9MjnqPzrPmsXyo3V8YXKTbmhjEROCiKcn8T9BXQWfiGzuUiEiIAxIJl5Cj34Pv7V5zNOpkQ4BXuo4zUizbzjdgY4Ddh/WrU2mS4pntFpPbTwI5iSVG+6YT1xx25/SlNnpU7bWR4yo6iTkfT9e1eSWF81m6zISQRh1BIGM9D611CeMbW3Plra+chILuSc/h9P/AK3HWtFNdTNwfQ7I6NYgjZNI6L1V2DLj3GRxTE0axKv5gWXthIlUH8iM/niuXPjC3M4RLeQx7c7/ADc/oR/WryeJrJlDzyExDoS+COn1rROLIcZGFcosN1PEi7QkjLgjng45qswYPnJ24/Kn3N/FqFxNIImhd24MRAC++CMEnjuO/rWlb6G9/DI+nXaXDA8W8yCGYjgDGWKNyT0b07nAJVI3sNQe5m76KtPoWtK7K2jX2QcHEBI/MDB/CijmiLkl2OATcr79m3Bz1p89xJJzvy0hBwG6AVXK4BzgEHoKahwx2qG9iK4DeyLqXMcTOSd5OQAvFWLW5XDo/PmDgnPHsP5VSjtkkx8yqx7c5q/HZ3ExS3VJWmIwkaoSevTHX0pozlGLIHf94POQojYwIxgccZ/SllZN7MA+B0ZQM+2f/wBddbZ/DzX9TjTzYo7VQD81zJtJ9Pl5Yfio6V2GjfDrTLBT/adxLfnOfLSMRJ9CQdx+uR9KpJhr0PPNA0bVNSINjBM2W2byNqA9vnPAPtnPpXd2HwqknKvq19M5J3MImAwf95ue3938a72za1soRDa20FvCAABEmD+PHP41aF8rj93NFGQf41/xqrJAqavzdTP0rwb4d0eRZbXTElmXkTTneQfUf3T9CK6FpGJyiRI/qEyfzIJrPWeUvgTxO3UEgf5/KmGe7HV42H+znP6UyzSNxcMMSEsPUjdUZcN91VJ6ZaPFUmkGzzJJpEI53blGPzSqc2pW8ir5F1cTswyNjRlD+Oz+VO4Gu1nHIjExwv6koBWXdjTrNC80duvsEHP5ViT+JbgX5sWm8xfJDtiPaY2JIweuOnrzkdOlZt3qMVvAyPIrOSRzzzTsxN2L8viOFiFsrVURDzwEBH161zniKa4n02eUzs74SVR/tK4yR6YXd+BPrVC41dXhdAOCSMjtW34c8J6z4pR2kV7ewaF1M0qbQ25CAVyOccHjj3qlYNdyX/hJreaCNdjFwAGRR06cj9adHBq/iRja6bp7QwS4zK+eB356Dr+ldJpvhvQ/D1uf7TuxfXAHIhQKgI69TzTNa8XyR6ebexRbazK7cKMk4HAJ9xj8qb1IukWLDQtA8LWsiXk4v7uVAksakMPvA/zA6+nSs+fxyNMntotKtYLa0D7mSNdxcA85J9q5BdTlnY7pc5IKr+VUmxLcsAAcY98elXa2u5N2zq9T1qW/tluZXkZpACQWyR7fhXK3V0xOCSTuIwPStOaNbbyrRpPMd4xhUXJG8ZAA7nDAY7mtGx8A39xIbrWJk0u127mT70zD2Ufdz6nn/ZNF0tEJLqyDwjcRw6xNblWy8I2qoyWZTwAB16ngV1Gv6LYaiY31O9+xIrA427nwSBjHbkgc8j0pbc2un2yWGmpJBbjkKHO9yf4nYcknHIzj24FY/iaUpos7Kqh0XeMfw7ef6VMtdykzZfU7LRLCKx0lZFhjLfvWfLsTjknGR0Hp0FcRr101xcqxA2qOAPz/AMa2JYlcJhhtZA5P86rDwxeagBPJIbWLPyGSPPmjnJAJHAByDwD0BOeGJdx2gyi7tJhcniHaI89DnOfxxiuhgmka3Fv5cj3SRhXjRckMODnsOe5wKdpEem6NEIba3865Y5Mlxhip/AAdgePzOKnfVJ7a0lY+WiEeWQqgbMt2x9aJahcittPgsvtV9PulvipaFAPlgbA4J7txjI6dvU4N1qO6/hMjADPLe2RV68vzDGrsQOccnrxUtlobW5TUdRVQw5ggK5/4Ew/kOvr0wWu7EzRktXj0SVM4aYbyO4x0H5Z/OqdjKLa0SYlPMAPkqwz0OC359Pz7Vea7UWss07gJnBJ98f41mN5l1JGgThABEq+mSMf59KcdbikiK0klis5pzjdI5gDZ555Y4+mB+NTSNHZxIFVvNdQ7yMfuqSDjH0wSfwqxN5TK0ECqUtXLuTyJGABPp9B6dKLTT21yyluZLiW2GS3nxrnaQQc/pjpik5XGo2Fmls9KG6dRJdyrvji3qAg7F8/oPzrAWS5cytLdnLMDsXaf5YA6VX1zT9V1DWJTFPc7cBVeHADjHXJUEnnv6Y7ZrJh0W5tWMX2nUw2STIskPP1JXNSmirHQLIobZ5gLe7KP6mrZheG2WRinzMejZI4B5GOP8+lZi6E23dJqM0oON26Z1Of+Aqv86tLoGlwIvnxyPNjcjSySMNnQfecg/l6enKbGkOF/bq6o0qZJ28tjn354qUavZrcNZjyTcxuVcI/mkHpt2oWIIINL/ZGnHBbTLYY6N9lT+q1pKCbVQskpCfeztCAnpgDpSGiusr5+WzmLEZDJZyD9WVR+tStcXiIjyWF3ySBLPLGgPTjALHj6VLFhgHELOw43K39MVMZnKlGmhQ9UBAODx6kDpS1ArC5lcYbyQD2ErOf1VP60knnsXk8xfLBO1miYkLnjLb8d/TvVltQ8kGN7mRmBwwGVx+mKjN+pBWM3DE9ScDP5UAVwz7vmlZ88YVUH67c/rUw2mFlk+0b/AOH/AEuRfzXcM/lVhZJXUBIZ93QqzNg+9QG4MaEDbGw/iEYf+ZoAgk02Ey4kskdx/EyeZn8ef51oW0LozEwth+X6KWPTqfb2qpFqF2ZQzSCSMDBAQA49eB1qzHfTCMkRu3qSTigCYQSA4CJ+ZP8A9aphFKF42of9lRn881VTUZCCGRQf7xbj9KT7fyRJ8ynrhM4+h60rDLyRfJtkYyZORkjI/Lr+NIzQKCGVMD+9j+tZQEbSFt87KTwCP/r1J5kCDHlvIfdx/SnYC/51rtK5Xb6beKVrqJiAGZj2xWb9st9hUwAcZyp+YU1LxRPhRIewDYJ6e3+FFgua4cY4BGe56VTuLxosFpYUHXnJJ/ClBlDcrhe+8iq8htEQmOCJ3YkbmXbkHryFNIB4v0YAhGlYDAZYjx+ZqO61Se00+5u2tHKQwvLy4X7qk+ntRFKIVAjibJGSgbIB7BSRkj65NYfjqaf/AIQnUzNE6DYmOP8AbXrQO1zyy11Mz/LKBtOc9jjv9OpxxwM10dlLCLJriRYwnDNvbGWPTC5z6HPY5HQccFayASLubIJwc9P/ANVds7RPpVqvnLIJ2CyKrAnAGDnB7bj+I6nHOJuZ8cUdyiRTO25gWdQM5YLn17/L9OaoXOlplgoADMVH1yf16/r9akRy7K8mMM23OQRgkZwPQAr0q8iSPcRxQECUyg4EnAGcD8mJAqiTGk0+SBYmmUKpVSdwyBlO/v149vWr8Tj7PIhURvKm6N+BtGWwzZ7cD8B6VrXKxapE7ZESebtBBIKxj5c+4BH161l3izZCQ5E5+bAUAEY5GPy+vP4oDGguyhjaRSjwEnaRxkHPTuecY9APSteOUmLMsu+OUkMVJy44Lc8Yzhj+PtXPFDLqEURVuuw9+PXp6H8gK6eO2kt7ZXkBG6I5Hp0A/QDA/PApA0NkeSUI5zmc/vEwAASOgHXoCCPWli8lIVUuCWwCMcZIPX24H4gHis67vRJMjsQsYLYCdxkHv9f0HWqsN2zXMbSNtVcg4P04H6EfQcUxGksJuU80plCowTnLHBzx+Bz6AfSo5dJd7XK7SwJZ2GTt57Yzn+PIHcA8jFammEnypSzsfmO/bkJu4Pfv8y9+Ce5yNKOyjZIg5THm48sEYCAcgj0LsR25b3oA851PTpbGXDo4HTLDqec4x1HFUK9LvNOhvLeSK5yivsjVs5Y7SBuGfpyR3zxXnl5avZ3DQyDlfSkUV8UoFFSJSbKSHQAmVMfe3DH1r6K+y7eGST+X86+d4lJuI1Hdh/Ovpd5CY9wQkH+9gH8jVQM57maYlHIQH6n/APXUIjwTnhe20Vck85nIQQgYGCWI+oxt6/jj3qNop1j/AHx2jON4TAPsDyD+FUSVpLISp+7nDk8FD8jf5/E1kzWqqw2BAyHOSMlf8K3GUHGDyO5psszOixyzsyJ91M8CmI56RMffcZ9ScZqCSEspYEbenWtqeS2hb55EQdMtxUMcf20FrSGSdFO0tGpKg+memaAMH7M0rBXeOJTxucn+gNTG3vtOLS2F3NLtGZHtvur9eT/48B9KvTW3LBkZWDFShIyPrj8qrvFBEAWLoxGdwQEZ/wC+h+dCYWIv7WivGVNW0m2vHznzc+TIQPVh/hWfe2ukxW0k9nqBRyRst51JIGeRuUEHsQT6e+RZzGsrSyxGSRucvKSCfUgf41HLcncWit7eI5zhU34/77LVVybFB4bm1RHuoHiDn5Sw4Yex/CiDUGtbmRI2K+bHsLg4K4ZW4Pr8v6mrEmpXJtWtcQGBvvotrGu/6kLk/XrWU9upJ2s8OWBwo3Lx65OR36Z+lPmdg5To/wC2F1CFbbVoTcRxj5J4yFmBxjJYg7+ABz6dagm0kfI2l3K3yM2BHH/r14/ij6/iuRgZzWGtxJBc7CPMiUZMqA4Az1ORkdQOcVdV0ka3khlIZZQyvGxDKwBPUcjpVRfYlruKwOQG4IJGCP0qW3uZbaZZoJZIpU+68bFSPxFbTa0l7G1vrcH2ncAEuo0UTJ82T82Pm+h9D65FC70j7Osk9ldxX1srbRsUrKuSOsZ5PUfdz74rRSezJ5excW+g1F4I5beOOdnxJcQgKWznGU4UnJGSNvAOc06/0G8trf7bA0V5anI861beBxn5hjK8A5449aoaWD9rkR1KlYZsgjkERt/UUQXU9nMk1tM0UqEYZWwfp9Pape+hSempFcKiW42d1DEMMFWIyQfocimWmHTyyocdt1bTanBqz7tZjdrltq/boDtcDJ5dMbXwCOwOFABplz4cvLXNxp//ABMrQ9JbVdx/FASw/DIHrS5rbha+wxfEF5BZvFOFvYW2/ubpmfb2+Rs7kOOMg/hVu2l0i6s7kC9l05pUCiK8XfFncCD5qgYH1XPJNYGpCSOEB43RxINyupUjnoQcEU0uVgHbjimknqhNtblu90u+sF8yeHEOFYTKwaNgc4O4cc4NVwg7jmpbS7ubFllsrqa2kyGLwvtyQc8jo3fqD1NWzf6ZdJLFe2wt7yTPk3NkCAWwAqtEWK46klcdO3e+aSJtFlFQUR51cq0QJAXgncNpH0wxpLXVrqydntriSItgOoPyuBnAYHgjk9Qep9a1rnRXi0a+urS5hv41aMI8UZDgYy25Dkpgc85GAea5oFZI8jH5daOZSHytGyNR0ic3l1qtnJl0URx2e2NFYdWKngZzzjj2Fc1dr4Z3uBqVxKWJ5EZAHPHVfzqbUPk0+Ug4O2uPfBJFYT0NYq5rXttoWE+y6lI7FgCGjbH16CuhhULxtAUYA9sVwmACCK9TutAuYlkntZEvrRUDm4t+cKRn50+8hwCSCOMHnAzToySlqKpFtaGTKIpPvopyehGRT7e7ubV41SRJ7VGUta3AMkbhcfLzyBgAcdqZLETGCvIx+dQ7SoG7p1xzxXRKKluYqTR0t1cab4j1O28iFNBnbEcgdQ9q5wTneMFDwF5HOR6ZNTWvDesaK3mX9jKIWc7Z4T5sfAz1HIGAT8wHQ1ShtZdQlhtbdlSe4dYY2f7oZiAM8HjJ9KLLWNW0hLeO01W+UW3+rLSllBwRwjZUcE8Y71naUHaL0LvGSu0Vkfz3ZvMEikYAU9KfOSbXMW4OpwM9q0ZNbOp5l1HTbG4nZgXuoVNtcMATkl0+VmOe644/Glks7OS2il0/UXEjkBrPUAsbJ7+cCI2/JePyqlV/mQnT7MyprspZxtuzuAjk9+Q381qOW8ht/unLD/PWrmt2N9Y6U8E+k3cZMySPcFQ0YUbwMFcjBLnnOPlHU1zE3HU81DncvktY24db2bHUlZQQcqxVhz2Yc9h+VdJa/Ei4u4orTVLS01KCMEBL6JX3EkY5x1GMZx3Oc156HB+7nOKWNx0PWs3Z7lK62PTobX4faykok0ZtOupMcxzMyKO5XkAH/gJHtWVffCLWHs473SpYL6B13LGrhZEz2I6H8D17VzlreLuRf9k4x61sQ+ItR0jy5NPvZI3L7Qucqc9QR0xU+y7DVTWzOQ1HSdS0S4+z6hay28vK4lUgH6Hv+FVnbapBypA+6RjNew2fjrU7eLyGaCdCxkEdxHvUEkk45B6k/SuU8RaZZ6zfwXNnZW1hCsQSWG143uDywGMLnjj8aXK0PmTOJhZhxnG7rTpJ2kxE2du77p9a2bnQbnTRJNAvnwRsAzFcHH68/jTolk0+7cNuh+1Q7ccfMpYHBBB/u9scgUWewXW5WV1VoxjcWzzmrKyLPGQ+CPMDhW5yADnj2OKJ4dh39WHIxUttG32LzMM0bySRjgYUqEJx353jP0FWR0NBdb1pECjXr9ABgKLp8D260VlllViMrxxyTRWvKuxHNIwrTRtS1J/9AtZZs8fIhwM+9dfpvwj8SXIDXUK2a9WM7AfQg9D+detguBgLGo27QEQRqR/uphfzBp8WVjKlEOehGc/XiuZUza5xOn/C3TbCc3F1dmeRececix/+O5P612+maNaW0Hl2iadDnkqLhQT9eOalitJI1LkFEYf7ufyNRMBECpjAHX7x4/DNUkloIurZMuAbiyiXb8zPdIAD6cHNILKBgD/aWlEn+7d9f0rLacqdo5z0xVSaQsfnZRt7seP1oA6AWMJ+7qOmf7xuQacNIuplzbtbXOf4YZ16fjXKtNbMj4Cs68kp8oH41QfV5IImiin2Y5VlA3Z7fMKahcXMjan1CG3vZbKaN0uoW2vGVztPuemMYPHqKhk1FkKFI4wD3IziuZ8V6lOdQ0bWnJ3XlkUmYdGljdlz/wB8hP0qva68spijlxljlnz0ot06oZ06zzX03lyTkqxwQTgevSnX08Ol2zqq5MalhtGM1yq6reXdxJa6TYzTXMoCh4kYgZznn+HqOeK6+z+HGv6yEfxBqS2dqPm8uEhnOR0JPAx+NHoKxw+rSmWaPVIN5EilGI48orsIJPvvIH0NW9H8Ja34gkDKrQQt/wAtJQdx9wB7HPJAPYmvSxJ4W8NaY2n2MIuvJbzcu29S/TJz34HQcdqybvx1cBGNisduikDbHGMj8Tn9MVag36E86RJaeEPDPhSFZtWH2+7QZ8uTBVT/ALvf8aral49vNVJtrKNLeDO04z07VxWr6vd6jcLPdSPKeU3E8jB/+vWhpHkwsrvtJAO70Jz/APWpxS6ibYl1FeyXjI7v5aDgsfaobzUJrfR57FlUlnRskcjGcEfma0tR1CII2wZB4A9T2P8AXFVrPQb7xBNIbNYQgP76aVyqR4I7jvz0+tN3aJRzcDeXKrOflA646f5xXSaT4XvdRkW5uZUs9PlVXVpMb5ME8omckHjk4Hua6RIPC/hOAStGNWvxwZ5kyqHttU8DHtz71ma74me6WG7iYj7Qu6QjpvBIx9cBfzqopsTkuhvT6xp+iIv9m2kETptj84IplbGB8zEe3asefX5pbl33sUkGCSeua4+W+kkLLISd2G571Lbpd6hcxR24ZpZCFRV6sfTFWlFE2k9zt4NwcP3NJFpX9rs1vcTeTayF4zMxA6gghcnk/oO9WrWIx22LuEG6MZ2QIdymTacBmHB57An3z0qodQa4feXySMA4A4HQY6Aew4rO12Nuxcj+x6VCqWsHmyR4AnuMOwx6DG1fwGfc1SvNRkW2nuZZGdi+dzHJPFIGMoZc89ARzVaHSpddgYQzeTbLy8zKSBgcqBkZb2zx391swTuinNds00V3A+GLEygsFARduCcnAGXNWtJfUdUtZligWe3EjA3LELETt3AZJAPIA49e1bFnptnpixlIxJKjb1uZEXeOMcH+Ht0weOtSz6vNLdzQyOSRgKSeo2j/ABqutkU9iODT4tPmW6vZIprnP7mMcpHxkHpgt/LHfrUF/du6CXG4jJxnFR3dyDs3vjBByB26VNF5Vhi5uVV5WAeCJxnjoH5+nHp19KVr6Cv1JriD/iW+XMCH4Yp3B5wD/h9PSqsS/ZbB5Gb5iMLjP05/z2NTxSyXsBZwQW3FgPrnP60jObu1KHYhA2Lk4C46ZPp71WxL1K8UPk2moTHJEs5Eajvk8n6cqPxq14g1PT7LTLfQ45HjkAErvGv3c84PB559v1qvJdxW0Mt4m6W1t1GCEydu4ckD8z+Jrm3EGo309zPdzN5hDbhIWPsPmDnGOOvbt0qGik9bF63m01cGa8udpPXPQfSoTMCn7ljvI43rwD/WmvaWCR4RJpyVz81y8YB9MIVz/wDWpypGo2+Wm09my+PxYk0iizE4dCrRuznoR2/Dv9KubZ44Y5JtxYsUCOcBFUDtnIBJ4HtVOI/uRAu4xr91A2FX8OgqxCIo42Mke45GFEgx+OO/ShgOjmJYswt0/At+fFTxSF5Vj85MyMqDy4yM5PHHHfmoBKqHcscae2N2f++s0C5KNlMx56tGdo/IUhlidZo3fezkKcHPb61AWY9VHv8AMKkuV8q8ljLiTafv/UA/1pANuPkJPuKQyQqZmkldkaTG9sAk+/QY9/wNAYRjG6T/AL5xRGGZiAYwQehlT+WalltngYxuYgwAJAz3GR2oARbl4/mXeCOjZ5qcu9wjS/uyCxBIXqe/86iRExkszk9lAX9Tn+VTB0gidEiyrEMTK+dpHpjFAgjkdR8suD/dHb8KkW4lLZyxbucf0qNd03C7Xz/djB/lSlZOEAc44I3AfzNAxyW0jZKQvtX0Xge2fxqQW83A2gH0Az/KmQ2UyXAuIyiv6k7sj0IxyKkfULfcVjZ1UdttL0AmWFmQpId0bDBVlyCPxx+dVxptsrHJkwT0L/4UC8TO4SDYf77Y/QCnw6jECck57ALkfzoAlS1tYwMW6N/vDP8AOpPMiAwFVPZcfyFVb6KCYreW86ln4kgck4PqnoPUdB2qFHeDjIR2PAYDH4E07ICxMIZSWkLKwAw6/e9lPt/L9Cz7UkYxEgHuV6/zNTJBI2DJvY9wSMH8qe9tAQudqn+4O/8AWldDsQJLdzHILBfZc/1qpq1kusaXd6XdZ23MZQMUGVbqDgHsQD156GtAosYBjaPbjqF6H3549uanWccIz5Ocdf6UgPlplaGcxuMMhIYHsa67R9TiFqbY53kgqxHKgHOeOQeuAO5rkrqY3F5LOesrs5/E5qSCV4yAucZ6Z61j1Nk9DpnWJp2k5SLb8wyGLrhjx68ccdOM85qygEzFSiB2RQxXjLYI298/ezzk4jPc1zkd8GnDSOmSfnIGd3OencZx1IJrUhvzxIqJHHDucJnJ68ISMAk5K8DgdAMCmBuSIuxI/MwhjOULY/ibJx2+vf0p0UBSFi7ZO5ZCSW42gjbj1ySfyHqKzba4aIq0km5iRyc7mAGMD6jdnnjBHUc2rPU0QMHQSeWCwfk7jnqc8kcMv4jPU4AMTVZYLC+kWOMiXLDLDkk/xex6/mB2qtLqck6FAdqMScKcDGOn0xxVDVZXl1GR5DmQsc8+9RxRtKdsYYkjggZxzQIdM5CZB9zUEW+STk9TmrYspZrd5ACAuMA/xA4AI9ecj2/PE9jZSANNtYKv/LTpg+v1Hb8OnGQDqdDWMWKAMx2yqflJAJPUfhgkn0X8DekmWC2UGQNJJHh3Cnb0B69O+e/07VmRQW8ChgzK0QAJJweQBgDHoMH0BPWifUY2h8oSgqFJOcYZsnlccYxkj0APryxjZJlaRZHLlN5QfN0IIIwR2wBn3B9Oee1CFJS3VkX5VYjHGSM4/DJxVtrhFncNuMLna69CRkgNn17+nJ+hgljnjbySA4IIxj8QR1x/h+qAwpIDEfUetMJx06VueUbuwkTYplUDGBhu3JHf049iccZwpVZJCpGGU4IqWiky3pmx9UtlkJCGVQxC5IGew719GsGVd21lUEDdt4ye2TXzVbyGGeOUdUdW/I19K2d00ttBdRuYjIivgHPBGcZqo7GctyJs7cBT9TTUaWNsq5QnrtPX61MY9qE+Yd3UlwNp/AYx/L2qAsSpkaIlVOC0QMi59MgfzAqiSFYVjVht3sxJJLHHPoARioZY1k4KDA6Af55/Gp5JyrYW3mft/Cv6MQahJlkb7sKJ3BlO4fgFI/WgCr5CIfkiRfcLilYPkHoR0qURSsSHuF29MLDzj1yWPv2pX08yx7hK8gA6RyZZfqAqn8cYpgVAkZkeRoVZ3+8xyCfyNE0NpKyRwSMlwzYEUrgbuM8NgD8Dj6mon0u2J+ZHkXr+8mdx+TE0kdpbwArFbxR5/uKB/KgDMuvJSWSBllaaM4ZFQnH5CqjxeYpHkOCOnmADP8/5VvtANvGMVXaAL7euKQzB+yyA7mCIg9CXP9KUWlu5wZWRyQBvB2/oM/h+taM0IBznNVZIyVCgEn0oGUXtmtLnzIyd8TELJHyCeh9iCD09Dg1otf6fqCeZqVq8V5yBc2kaZb03IcfTjnGelVUaaAjaG2k9DHuVvqMEEdOtLJH5szB1WPcPvIQFBx1xz+QxTTE0O1LT7uwQSkNPbcf6QsbBQfRs/db2Pr3rNWdgS0ZIdDkMOoPXg1ZZbm1lbyLpCh4IdAUcDnDIRgjPY1Omk29yhNndLHckALDKojRieuCWPv37Vrz9DPltqXYvEEF7MZdStRK7H5bmM7JgpGMMRhX47kevFOudJ2p5tjcxXkBUuSjDegHJ3L1GO+M+vFYUtrc2TrFdQtFJjjcPve4PcVPp+oXem3cd3aStHNEcq64JH50tOg7FlT8pHSnQM0EqTwO0UqHcjxttIPTqPyrWS80bVNOhN8v2G/Hyefbx5STGOXT1Oeo+pJ6VRu7CSwSGQyQzQz58qWGTerY6/T6Hmqi03ZiatsT6hra6ylvDrlu10Y2H+lQvslVfcfdfseffHWqkmhTTwO+kXUOqRoiuY4ci4jB/vRdR+GaqSE56AHoaiRpFnilikeOWNsxyIxVlPsR/Loatx7EqXcTzdpaJ8pIn30YYZT7iow5acAEg7TnB7cf/AFq6i410XK29rr9udSgMUchlbCTxsyhjtYY45Axx0+gEkHhfR9VtpZtAv5WnjU5s7qMGTOeMNhcZ6d8/ylSa3QNJvQ5uG5ntLhJ4ZHjmi5R0bay/QitJtQt9YuIYNTt8yuwRbm1jCzMzNnlRw5JPX3PGay545ILnyZonikBwySKVYfUHpUkMbvOPJUsyAvxjooLN146A1UkmhRumWNU8KyXujTXOj38N5EFBZGIikTnHzKx46evrXHp4W1SPZI9vGFycq79Pc45/KujSTIKkt5bgZAJH8qSWzmMDx2t/PscjgOSR379KzlTb6lRqLY5yDw+8GqRrPJE8Qy3ykgnHbBGfT/Guotru5sZHeC4uI2kBRzHJt3qexrEhstSh1ZWkMt0oRgCoJKjGTkVtxoNvIHTOKKaSumObbdzSbUre/kU6jaQyDJLzWii3nbPUkgbHP+8o+vWhfDLakssujXaX+xctbyL5M6Dv8pyrY9Vbntmsp125KHGD0psTSIPOPOCVyHIbsePyHPqBWlrfCyL3+JFzS7W8i1rSxJb3VupvIP3k0LIAPMU5BI5HHWs6eILuUD866mx8Za7a2whiu0mjCBEiul+UADA+ZcMegyWJzWQtlFdozR31vBNtDPDev5eTjnZJ91uc4ztP86SbWsh2VvdMVD0/Sp0kIBGcjvT76xudNdIdQt3tXkGU8wfK/wDut0YdOQT1quMg49qE10BpmkmrXUdtLF5nmwSRmN7eUlo3GCMEZ4PPBHIrnLu2jnus25EQcktEC0ixcnjdjJHT8+prQPIIU49qdFaRR2XngEM87R5/3VQ/+zCpkk2VF2RjWmlX2oSTJZWstw0IDSCJd20H6f5FQbSrc9fpW40ptpVlTIlQ5jlUlXjP+yw5B9xQTFeTRSXe+QK5eUjaHmycnL4yT15OetTyNDU0zHiILY569au3T4iiYdd3X3xVuaxtRMBZibytv/Lwy5UhefmGM9DjIXqBg1nzOs7pFCjSuW2qgXJz9B1qk1yu5NveHR6i+8u3OOAKuHU5yyYbgDAx69ay5rWe0umguIZIZB1WRSpx9DSLkDAbkHA96zUi3E6u11Z1/wBW5DMuPrS2bC2uzeW74uCSQ8qJKFGSSAGBwCSTxjqa5q2nZJgSx+XkY71oPclW2tjcBxj1q0k9GTqjqIrzw/f25XU9Da3nCBFm0+4wHxxuKEhVP4HPeqWm2WjS2Fxaf2wLS5M7SR/boFRUBCrzIpIOcLwPQnA5rDWWSY8nGOTjiqrT/wClEtIyIAQxUZx6D88UOCSuhqbvqdQPAXjCRQ66XasrchhdxYI9fv0Vgx6n4pjjVItRvjGoATErDjt3orK8+4/dPXm1CNOXIYZ4wKcNVj2BkuoM/wB0ZDfqBVU+FnYv5GrXKrnJAmkX+TCn3WjO9oC2pXcvQfKQD+fXj3NPQdyZ7tGJZ59vfc+T+tQvcRsMi7UjGSApzj6HmuZ1AarpM32gTQvak+Wyht0ke4EhsYAGcHkenPbOdLrlqbQqjNu27ixP3iapJdxcx0V/qiRHZC67iDl2OcEe3T865+TVj5zmS4LuP51iz39xqDbYQ5dj1U1ch0CJYS9zIzseTzjH45ovbYLPqOl8QB24Ylceves4ahc38rJEMZ6n2rptM8GS6koW2tSyk43vlV/PBz/+uu603wX4d8JxxS6pI9zMedqrwCPYYyBkcmjW92GhmyeC7zxJ4D02wDtbXVvLIV3wknYxAOeRg8ZGevHqcXtG+FWg6QrTa5dvcsMN87+Uv5A/zJFWb/4hQW8TWmn28cP9zsVH0xj1+nvXKz63LPK8l07uSPmUncaOW8nLuHMeiT6rpOhaZPJosFuIl53QgY3H/IrzvxB4u1W/gAlun2N1C/KMfhUP9po+k3Ajdl8yeKMoVGCCHP4dKwLh2kZkzwpOAT0rRJIyd2x32yTHVjxg1DCJLhNsRI+YAHPBJ7VqWS27q0DxlMYJb1BHNJo9/aW0k8OfnJJUKNzOQRgBR1NVdhsZNzDPEod4/wB2WZQ2eNwCkj8Ny/nV61iur69hsbKF5pX2kKi8DK5JJ7Djqa7fSfClpPoStrcdzaj7W9wkTYV2DIgIbrt5Tp1A/TYXU7TSbUW+kwJaW45xGOSfUk8k9uai/Qsy7LwXDptl9u8S3CSugyLaJiEyem5sZx+A6d6y9V8TySBbOyaOGJeFjhG1FHoAP51qaldzHTNQCvnfAXJJycLyf/HQ35153blo7h1x8+c49KuKVzN3Zp20M92GkYl05B565H/1/wBKoahbNbLbqeELsvHQE8gH8mP51oW0UiNGoHf7oG4sewA9a6+28IR3VkJtd8yKF5Ekjt0YiRioP3sfdyGIx159ejk+okjlND8OX/iBA0EcS28LES3UjYSPA5B7nj/65HWuySysNFia00weZcZKXF1IB869MJjop9v1603UvEMNvarZ2qrBbxEKkMQ2qvOMcdeuawEvppUlyANwqZXKWppSu/mqwVYfKI2iM8ADvWTbeZGscZUswAXA6k9P51Z0q0vrm5+zwDzZDknJ4VfUnsK2pHsvDGXSQXN6c7ZGAG0k9FHb69cendwJmiSz0U2Q+0as6xbT/wAe4YFvX5sHge3X6VFc6kZmRYiFiHACgAY9OOAPauX1HVrq6nLzuMscnbwDxV/TYLi9aOKCMtnqegVe5J7AU2hRN64YRxIXcKuDuJ449/8APesy8069mullVRDEy7fOuZBGqnucHkjjsD1H1rSlmt9OjaGGVrq6SOTyp3Ufu22HaVHrnHP9DiuKn1Ga5V/tLySSDABZiTj8alK2pW51D2djaMjzXK3VwBhdvEakHsOrH3PHtUIupdVdRMzNsJIJ689efyrEtoGaSzfPyvGXfPZgW/oF610+m2wtGW4A2Sj/AFayD5cn+Ij+X41YmOmiWygMLM3mFgWHTbkYC/4+/wBKqSgxafcSbsZyAARzx83+H51JMzzB8yhZXUgO3bk80sFtNfXskUMMnkk/u1bAOPft71le61KtqV5L+20zwq2nTQubm8OwGVWCGMYycjrjPQVzkUawYRZI5AD99QefwOP5CtPxLvGq7Et0gZEVIxJICQMA8Z69eTn+VU0sL2cBninkOflIJfP5ZxSvd3KLsaRPbOsciq3XM6bSfYHfgflVfyWh4cIc8/KwbP4jNSx6XfRlPNtvIickB5SoPHoM5qRooY2Ie9Hy8DbFuJP509AGx7Wxl1jHTcwYDP5VtNpRt9OjuJ5SVkydqLjP/AjWCUhaRWHmPg5+fbg/hj+taraxI1ilo0MPlqMAnOfzzQ07C0KcjRE7Ywyntubd+oAqS3Ebyqpbk9Bt3H8qaLkqoCR26+hEKk/mRmnS3dzLD5bySNH6Y4pD0NQ24CCSUzKHPDOfLDY+uM1C0scfU2+M98nP5ZFRpL/xKljKB2DEDLABFIznPsc/nVI28LDDTNyOTGuR+uKTWoy79tjMhHmeV7xQg/lkinXGoJcBF2MXRSN7YXd9R7fWqsaWyDHlyOR3Dhf0wat215ZrcwxS2MXlscE4dyO2euP0prsBEglfc/mxlcfdJUAdu3NWo7e9jiUiZ19HSEgf99AfzpJJLi3lkhQom07cRRCMYI/2QKh3uDlt2O+ATmkBpXGbi2g+1TZdU2swYNnB4yAff9KgVIYolVZVJH8Spj+Zql9oLc+WVHq3AqW7V7dLV9ymO4VtrAZGR1GfXvyBQwNCKdQv32fHZnx+gpiJZwxh3hRyT/q1Y8D86zBKxzyOacshzkPz7UAaMltKhMnlokDHMZHPy5PHPOcf1qJLtLdcmCZyOQBz+mP5Zqs97bxxZlwMH77yYA/OmokdwWMJDAclkO5R9SOKQGiNQlm5ijjXtlnGfyJz+lWYrW8mYgXSksPkEY5b2HA59OayYrVi4IYDd93vWhDbTQgsjCT1weKNA1IDNBGxSSS5cqfm5A59x/8AXqVV08FXIQk9S8vT2K5/pUl28d7Ist0FjmH3pdwPmfXvwMe/6Ukmn2twPkTbx1CkfzoGKs9iDtjWJieyJnP5UiazaG4Fsku+4PSFR85+i9TTjYW/lhJIY5AOgZN3HpT0u5YTGsS3IjU5GAVC9fXH0pAfL5wCvbCimjJbAUsfbtRKcOPpU1mBJMgK5Unkb9pP0rA1WxPZabc3TgxIMdy2MD8PoO//AOvSku7e0jEUUmZAANy4PPrj1Hb6k8nJNLVrmSFjaK0sSKACm/07Hnr/AC9BWLuIOQTTGdUl3C2Yz8oU7uDnt69zgtk9/wAjWkJI4rKUxkM8iA5DcMpznt1ByMe/rweFE0gOd1WodQkUlSxCsMH0p3AfdvuuCwHXpW7o0CKk5ZARuHQnGDzg/wDfJ/M1zsuDIrEkAtkkdfrXUWl1ZC7YWkUkcMMQAjd94mG7rkH73Jb0G1umKYiz5X7oh5EjQcEYyZOvHfs5Pt78AzwsY7eS4l2ooYKpwAeM9yOOnue5OOKTzwk4QNhgd2RkYGQAQT/F3z2Jxx35nWtX87Za27ZiiGA2MH6DHtgfhSGX9T8QIIzFGM5G3AJ6dyeepA//AFnmsBtQkMxk7k5/GqZyTk8nuaMUgsXBfnoQcYx+H+RV+11OPvuRyCNwP6/4+xxWKF5qeIAMNwyPrTA6W3MdwyeXMkRxymD8wzyM49PU46e+Od1BSl3IpySD1bPP510FjbhLGaUSFgy8rCwLAf3iu4cDBHII56c5HO3ZJkyTk+pXaSPoOKTGiEfd/Gvo+wg/0K25biNODIfl49BXzinQfWvePDWvW994Us7u5lCTImx3THLKSACO/bnryOT0qomb3Oi3FBwUUjqUULn8qrltxAGck5qrDqrSoN9jcCQgHYihgQRnhvu459RSvNev/qLIhj186UKP/Hd1MCfDBvuZGfpUWF3HcRnsKV4rtwfMnt0PIGImb/2YVWNhNLuD6lcrjONsaYH/AI7n9aBEzFWwpAIPSomuYYJQpuEjkHQFsH8KmbRYBCz4N6mckSM8pXAPJRiQvfpxUkUKxRKI0RI+wUYFAFCS/tHiy0MsjknbLDE3J9yFIb69feo1jmckOqpjOCW4P5ZNaLRKTnPNN8pd3Y/hRcDO8j7wdip7FF3Z9OpFQm3XHIkz/vjH8v61quueOg9qqtC2fvUDKKRiJlJgVgCc55JH/Aty/p2/GozbvcT7UBc7eHKqhz6YB59gKuy+Sn35EH1OKgM0B4VifdVJH5gUwMqULvKvIdw6g8UG33KD1BHB9a07hYZ2Vri284DplyvbHJBB4/GojDbQAmFiik8rjIH45GfyFAGY1mDio3sVOcHP9a1HZSuAST7DFV2BGRuI/KgLDDcTPEba+BubcjABPzp7q39Dx9OtZ8um/KrWknmgg5RwEcHOMYyc9unXtV88dSfxJIq9o99b2M7m6tklhbnhAWU/j1B9KLiOeuIPKtbXrl4y+D1GXYf0p1peXOnSl4XwjgeYjKGSX2dTwf5+ldrLpGleII/PhuXjZF2FlAO3uMqcev456965TUNImtEdjJGVVdwycblOcHn34x68U02BLGdIvYh++k0685JE2Gtm9MNncg/3s+me9VrzSr/TLhIby1eEs3ylhw2Dztboeo6HuKy4yDgOwCngnGcCrFxr16s1ykU6NbzsWdDGuxyOjbDwre4wRxg8VopWIcbl/WkWLUtkbbo1hhwQP+ma1QR5beRZYXZGDZUqcEVpR3NlrGPtrRabcRxrGlxChNu4UYG9M5Q4AGVyvsKqahpt9prRm5hIilGYpUIeOUeqOOD/AD9cVpGatZkOL3N+w1rTdSd11+zjneVVQXSLtkQDODlRnuensCDinnwsJmvbjS7lrm2jEkccYXbMzeUHAHZvvgEDnrxXKRMVcqRkVpTXEljbWJt2aJpFadnjO1mO8rgnuB5QODxmk4dYgp9zNGQxRgVKnBBHIPoaUZVwVYgg9jXTRarpfiSRYNdiFpdNwNSgOCeOPMXofqfzFUda8N3uiDziFuLJiAl3Ccoc4xn0P1/DNCnrZg4aXRXs9UurK6jubWUR3MeQkhXOMjByPoT+da6Xvh/WLG5l1RP7N1l22+fbJLJCTjh9gyACQc9++e9c1kEDHb0q7PtXR7cADJnlfd6/LGP02n86c4p2CDauXL7wzqMGnG/i8m+sAm/7XZSeYm3ucfewO/HHc1Rj2tosA3BmNzKCAc8BY/65FRWV5c2MrzWF3PaTv1kifGTjgsvRse4PtXSzeJLHX4jD4gtRBen7up2kO/bjoGX72PbJH0rN80dy/dexyzgqewI7UCVlBJGcjFa954evI7UX9i0ep2XO2a0O4cHByvVT+dZAw5z19fatI1EzNxsTQ3UnlxRb90CSiUWrndCxHTKHjHPat7Z4X1ct9ptv7EucE+fbnMJPoV4AHfoPr680wKHHUe9PtpA2oWSSLviM6GRMbtyg5bj6A0ShH4hxk9iS+8PalptkL2SB5bVlDCSPBYKSQCyDJXpnPK8jnPFVzLG+jQWwZhKlxK8kTAqyEhAMjtkLUfnyOiBZWRFUqoT7oUsGI2/dwSATxU1hax3etrJf6iILaTglYjtQ7cbtoPJJAz+vTFZvmWr1LvF7GdIhbjFJBGVVhtxg4HNdReeEroxST6ZNBqFuoyfs0m6RBgZJTr13dM9M8dKw57O4s5CtwDGwP3WGCBT50yeVkI+RSxzyafHcS2l5Fd20rRXEZzHNGcMvbg/0phcB1hwRu7EdKGQ7cEAeozVaPQWqNm98ZXF1bTR6xZWt/DKwMiiIJK30ccKcnsO59c1Sk0LQ7sSSWmoXVg5GUhvIDOrZPHzxgsoA65U847civGIjjzY3dOdwQc9Dj6DOOe31qRX5Oe/Y1Hs02X7RpFX/AIRnVfKN5ZwxanbjBaXT5RNtOAcMo+dTz0Kis+MeeGZG9j9a3GSCQiRkXzF4EgA3AemeuOvFaMmqWE9m41y2kvJHbd9riIW45I+Usf4cknj1z6grkkttQU4yOVXdGAx6KPWqkyPKpeNsZcLxnPJHQ9ue9dBbadZ3UENvHqEkd7JlcXSAQhs5GZB0GBjkdcdjkRzaDfWOkre3cBhj85N6FhvUZ4bGfunHB79elHN0Y7dUTiM47UU0SQEA7gffNFdFkc9j1CRFmkBdiSM8ZOKm0944IDEpGxM984yc81yc2s3FqVMrDe4JIK9ew/DrS6Lr17PcT2enWT30gDSMIhkL/vHoOfWsFZI2szZ1qBbyG5iVwN4i8xs4wPPiyfyzXFXGmWQmCFfm3dO59unWu90awUQahL4raBUugvl2/DDajB8Y53ZKjpkdOTmrNtfaHp7ST6VpUcNwTzPOQWx+BJx+P4UuW7Gny7nP6P4M1m4jUx2UVtCRxJM2Cfoq5P54rooNH0XQdR8m8unub4jf5bNjYO2AAMd8Hqeajl8XfbQYGmfb6IAgOPcc1irJFPrul3kKqQRLbSE8ng71Oe/Ln8qrlaBS5mb194p8sMkG+KFcjbD8g57569+ucVw2p66kwkEbMZC2dzZyf1/nWx4jieHTjMsYU7gCDkc/SuBLqw3Mv3eSc1N7gjR0+4USMZ/mB6Z7VfuJ080AEoGGDIOQcj/69c/HMUcPGAQuDz0NMiu2nkEUMUk9wMKqRAkk/wCPandDaN21Dyw3aBEmRYRM6uxXowAOQe27OPwqnbzO9ygjgnlmY4jijUu0h9ABya7rwp4NkFleT+IrSSC3uIlWKLgTN8wb1+X7o4PP0ret7zS9EheDRrKG03cOfvOf95zy3547Y7072JZydj4SkjkEuvapFpqy/wDLpGweYjJ+8fuofzPXgdK6a1/sbQYY5dFs4luJA6tI+5nJGOSx55yDgED2rktWvx5qzyk3LknDSDjr7Vp6Perf6XcglRLDJGwIGOCGyP0FK11fsF7GmbqW6VzLI0jOPmdu30x29hWbckyqhyenAxVpZvkAXAqKK1mvZo7W0haWU8hU/mfQe9IAgja5je3lIIlikj9OGUj+tc1oeg32sNDewRjyJUDeeWAjUkAnJz1Gegyc/Q4760sLfTbgXE0i3V9GCvlocwxkjBBP8RwT04HvVO81AJbhi6pbQDy0RVwiL6Ko4A+lWt7ks0LfTLDw3CssRS51Hb/r2XO3PXaO3HpzzyfXOvNamupCZZBvyeAeB7Cs+7v5LxbV03FWiRsnjIGUz/45Ui2J3gKpbceABnLUMLHLs7Xdx5IbDO+36eldZpmiNfxR3CSCK0yd87DOTzwv94/oO5FaWj+G7bTNTe/1EB7nny7ZeUjz3f1bnp0Ge/a/ql/JOHdyMKvAHAA9qN0D0MjVtTg0VBYWI8qMqHDHlpB6sR1OcjHTjp0NctJdC5naZwSegzz61DPcG8vAXDfdIVu3B7e/zfyq5pGlT38rkIVtk5llxwg/x9BW8VoZMksvD91rM0UkbpHb5ZZZn+7GFwfxJ3cDqfzxuXWoWml6c2n6du2xtl5X5eQ+5H48dBn8az9S1qOG0+wacvlW6EkfMSWPcn/PYVk2JN3MIGJBfAFK1x3Ze0+5e7vo8/xELkdRk9aTSPDd1e+ZPMBbw/dLT5Td7jI5HuBitiGyi8PtvcBrqUfIpAIj6/mePoMd6oXj3Vy5lkkZtkqsCzZzyM5pOzBOzNWH7BYGJoY2nkg4zKAFJznOM89sfTOKgMjt87sSWyc/rVeaQb2AOBk5zTobiFdk0hJKHEcRUgkj+I/THH5+lQVYGmWaZ7aN3E4fbKMDbgAkAdye5+g9K3bzWo/DukJFHb79SdD94EBfYn+g5z+dVYLyHS9l3ewlpbhmMQ2ElF6bsY55/Q+9c7rLS6lqE9wcrEWxGC3G3A7duah7FbGTdXV5d3LzXkzySE9zwPoOwpUD79xbHbJ60C1MZO3y2HqM1PDbPJgg8+64NJO2xRakvPtkMUE4KpACI2QZz9QTio4oYFcli8hPGAMfrzT7iwnsphFO0QJGfkckge/GM/Qmmqo6cmne+okrE/m24P7uFmbHR33fyAroo9NtX0IXbW/ky/3nDY9sAnGMkdfWuaEnkDcH8v3zion1WF2Kteq5HOA+40nsNFqQyggq3P8AsgD+VNV5yT8w3d84b9T0pkMxkfaqSAYzuddg/XB/KpZDtlVQySIerKW4/AgZoASK31G4mENvp88srgkL5sYyOueW/nTkPyLu2hyMkBsinQNLaXgnEQe35G11K7wR3wf05pFeJAy/Z4iCxITkBR6Lgjj86Vg0JkKKuXPBxxuAP5dad9ssbRvNnuEiA7Z2n8zURkZowoih29v3KsfzIz+tMRpbZSYiY1xyIzt/QGiw/QuQ6vBM4Sz2EH5g5IYH8eg/OpDfXO0He5H+zgfzNXrKU6poU1ncEk2YM8G7qSAflHvyRj/a9qyTmN9m6NWxnaW6UbMCR5XlxujUn1CgH9KR2L2ckDJ95lfIblWU5BAx9QfWoQfMfbI+1ByCgJOf0xVywS3ml8uR3bPQgbf6mgLGY0NwWIa6mdD0BCrj8VUVCbXeACWIzzvYuT+ddtcaZaRWsDwxxszMVfeWyuOxwcfTj8qouwimISztVP8ADIkIHHuQP6mlcZg2IuIrhbiLJVDwY+SPXtkV1gvF1LT2M0eJlUg7+VlH90gZ/P8AOqEusyh/LdhLjjO8sPwzVWa8upeFaSMHp5Yx/wDXo33FcuuZ2t0lEDb8fdRVcr+ZGPxqJ47veiGUEEg8uFI/4CAf51QjE0rHzJJZGU9xuIqTa6MQoO4cHcu3H50AW7nQZL3BmvYwOwCPn89+P0pZPDkcOnzXEd1czNAd5t42A3R4GccZ469eB3pLZ7kSDzJ2Uf3cAjH1rRi1VIZgysQV7shx/Kj0YypZW+m7PMgCOPun96ZB9OSRV/zUBEYKg4yE/riqTXFmk7ta28aeYdxCDag9TUiXMMnCSRFh2DZ4/SlYLny3L94fQVZsWKXEfK9Rycce+CKr3KNHI0bdUJU/gcVJZZNwm1iGzxhSxzWJogvypuXHHBxx0FUjVi5UiVgW3HPJxjNVzQUhKKWkoGSK2UK8e1amnzuGgOdgXeGYcAKRg8fQn9PeshSQQRwRWvCyyaSz4VSjheM9P8SeePbFAi/e6pI1jKcMtxcfNLISQTyTgZ/EcY749+b+lWLp5Sf3nGcY5PAAwB+VVhz1NACjFLxSDijOTk80CH+2ali27hxmoAfepof9YvzEHPWmgOkkMMOlEqy5YjjIAU44bGeD7+3J4xXNz9CCCD6H1rc1C5/0RYPnDgZb5jn1xg9uc9B1rn3PNJjEX7or1T4VXcrWGpWvmP5cciSBAcjLAg8f8BFeVZrsfhtqS6f4uiRygS7RoCW7E8rj3JUD8aI7ks9nWRAxAByKk35HA496dulY4CBPQscfyyaTZgEmUsfQJkfnn+lWSRSKxboKMKPvn8M08xRuMKjr6MTuI/LH+e9RzwwxopuFheJiEEjKB83PBU5K/qPemA1riCIY8xE+rYqu0ihgyeYQTjKIXX9K0YrKOJcIEUHn5FAFPjg2qQxJ54PtQBlvJKz7Rb4wcbi2FP0xk/mBT/LfcPlJXvt4P58/yrU2AHhaCgJosK5lTWbyqU+cKTkFn+YD0yuP5VUk0dWOMB/Xe7MD+BNbrYBwT+NRjC/eYM3qBVcoXMGOxhtdkJjaOPcSWjzkH0AJ2n6cfWrk+mxyRl7O5FwVxmPYUk6dQvO78CavTKrKcgD0JrLDLu2oxllHaFS5B+gyaTVhplBgSeg+tVZFYEjbWx9hkuJvPlhMRJIaQusZbPdlPLHPqM+4qrc2zx3HlwzW8qEZDlip/LB/maQzLIfp2qIs689x3FXmtrk9GiA9NhJ/Uim/ZyrZeSQ4/h+UA+3Az+tAXM1g5GTwaZhmyNrE9cVpl9hYrbx7SMYdN38802R4p4tjl4TjAGdyHtxgZH0wfrQFyjbXlxYTrNCdrKe/IP1Fbya7pN4D/aUPlyHBztEiggdRgZB78g9etYlzptxbqruu1W+6QwdW+jDINUHVkHKmmhMtXWm6OZ4Tpl7E77g7WsiyIJOfuI5BAyOPmPHqa5UrMrok0MkUrtwjLgnnBx68gj8DW0w4B2jg5B9DVi41AXdottqdut2obInJxOo9n7/jn+VN+QGEGKuTkq45we9a+m65cWUbW8MqiGVv3sEwDxSH/dYEA+4H59KW88J3UyfbtIne7s3XK+Y3z+h5wBweOcH2rnJIXd2Vg4dTgr0IwfSkmKx3U+k6bqIimsLgabdTr5kdjdyh0kGSMpIDkDg4DDP6VmaiZFTTreaN45IbUqwYcHM0rAgjg8MOR3BHasZbnzbdEmIcKoUZ5AAHFa1n4h8xFsNVha7sE3GFw+J7fPJ2MfvAnna2fbFUpWFy3KTna3T8a2/D2v6hY38NnDIJbe5kWFoJvnjbccdD9f8AHNU7zRZTbm7sJPt1mBl5IVy0XHSReqfU8e9U9Kdjrmnndg/aYsEjod45pykmmJKzOvuYfDmvTeXZsmiakGKmGXi3kYdcEcJzwMdf7tZetW01jY2Fjc28sNxCJVk8yMqGPmtjaTww27eRkc1kXX7+aV5EGHdmIxxyfSuh0XxdeWVg9jqkQ1fT+CkVywLooGMKx7jtnGPWr5ZK3KLmi3qYPlqiA4/GmmRWbBwMdaZHdLPdTMC8MBlxBBLGVcocbcE8HGTn6U9kRmDIwbOcg8GqvfYlpLctWuq3OlzG4s7iSCQrtLxnkg9evB/Guhs7zTPFSTtqVtDp13EEL6hCvlxlTuC7lLYBJAHqcdew5Ix9cnHoPSrEDtDo2p4AYSy2qnIBwB5x/moqZRvYaZpXugahY2pvZIhJZZyJo3Vwq9i+0sF/Mj3rLs4ori/G0g7YZpMg9QsTnrg8cf8A6uoLLVr/AE+eO4sruWCVDkeW5Cn6joR9a3/+Eh8O6mgF9oa2t45KzXmnjYVVlI3fLgk9AQQeCevSlJSXmOLizkd/zcAAelPD54x161p3Xhu5Rmk0ovqdmpI82BdzgjruQfMOvXGPp0GSSUbDgqw6qRgj8KpSTJcWieOZ4z8jMp6ZUkHH1q0L+d7lriaUTM4CsJlU4GRkqxUhWwCMlW+8TjNZ45HtTgcdTnNJq402jWht9Ivonee4l0u+cDYsq77ctjoJFyVXOfvdPU9arXmk39g5EsQkjHIlhO9CM4yTjgHtkDNO0mCG61W1t7nP2aSVRKOxTOW/TNUrKU2z+ZFGseBgLtDKR7hsg+uTyDyOealJrYq6auzR0mFTYaxM+PltVKk+8sfQ/TP51nB1B/8A1VLG0ct9uvIvMtyQwWEIjhvU8fOPbcvPPWo7trZbyG3tSJpZi52RnJQdVU8AluvYcjjjFNOzdxNXWgBg6kgnng8VV1Jwtsm5WIaVFIH1qxFIJACN3IztZSDj6Gob9VKwKx584HH0BNaJ6EdRFYjkU9dQubeKS3guJI4pARJGrYVwRg5HQ8GmKwZ/5GmSqFzkBh7VWjFszZh8bazBDHChg2xqFGbaPoBj0orDz6K2PrRUeyiVzM9Gs9F8O6VcF9WvJdeu1+6m0xQLjpxnLDOepx/s96fqniFrn91GjWUBVVW2tJDGiqB90hcAjk9gP51zEs0sjlzIUVBkc49qp+eWuFY8gcZqVFFttm0t4vnkgAMOPlAz9aSW6MCsQxPv7VmfacSklsZ46GpWmQgnOQwqrkWGrc/umPOQTzV/w67SCRBIQ8FzDMM4PyHchH4llrIRjIuzLEckj6Vt+C9Mub7Vpmhjdo2hdSyLu2n+Ant94A8+lN7jhY2dcs/tAZV3SyuDwCcLx3wfYdj+grk7bwxqE9x5SJJJIz/JDCgJb1zk8Djr0Nenf2VbWdsX1jU1UjLfZbJQXIz0Lf4Dj1qjc+LktbP7JpEJsYASzOPvv+JGc+9YRi76FNpHPWnwwmRI7rxDqsWmQ/KxtoQXmI9D2B+gau20258L+GrZodDsY0O0l55D80n4n5m5wcdBngiuHm1Np38yIEsxx8xJNMlZvPhd+SBnIXO09j+eKtwSVyVNt2Ou1DXDd5eSRxxyG6j24riJ9ZSbVERVPl5xn/61WYZbq9MkTKdwXKjoB2rM/si7k1AFISwQjceABz15IpbMpEmqJKInw4ZFIYAL0HTr+PpWt4RsNRv7DUW0+0jmdZYuWOCAMkhf9o8fhmtmw8NEQi5vAbS1KfNIwG+YnptX8ep49M1pPrVvZ2g0+wh+zRf9Mz9/6t3P0q4RbvYiU1HcdDpNtZ/Nq021z832aIhm+jMOBzngZP0qPVr+CC2aGzzbwsoPlwDYW46yMOWPPc96x5b55pfK39P5U6GEbpSzBiyAhW5C54z/AJ9Kc6fKrkRqczsLb3MjwoF+RSQAOmBWdqglOmPFwSr7uOK0beMRA72UqOc+mPWtWz8PXeoStLOv2WxUZeaUYLf7o/qePrWb1NLGFoFvPd2enxMrOzJIkKDqcOSSP++xz04rtYbe20OMNJNFLqQX5NuSsWTjj1b3OP8AHH1LWbfQo1tdLREbbt80KN7DOeW+vbtWKdUlSWGUOWBZQx+vUVryOWpLqW2OnG6Nss5Yk5yxyaxvE0sywRJCT+9JBUDrirtw8jiNIwWlJwABkk+gqxi2t0jmviks6vtSJRlRwTycc9O3H1qYikc7onhsXVm0+oyS29vvJhKEeY3IztBGMEhck+mOc8aGqX0Edk2l2h8mA5ACnOz0JPUnIzknNP1HUXmWVVkZpcA5PbPA/l0rBXTj9qMHmvICPLDnk+mfrWi13JbMb71yxzwf0rrorKLwzCZZkEl+6/u1Yf6kHoT6MfQdKZa6fFoMPnXSb79wAm9eIiSMHB6N6en16UZ7p7tyZGLAHJJP9aTemg0TIZLiRZZGJ2jpn1P/AOv86eY2u4pLdCAxUncx4X0z3xnHABNOihd4LV1URRssmZH7gFNuB3/i/T1plxc2lrh02xxlMs5bJbA5J9TweKTfcT01JEtEifFzMZ5kOQiDam4dAByXH1xz1FWNOl060mSbULmF7zPyxNcIpAH8RByQT15/nWQdTNwiy2dzBHMyjashyRkcn8j0/p1y/wCzpZXJ+0wsxOX+fkn1PGal6FQkpbM6HUJ7y4uptSu7E3MiAK89vKjoiZyqgsynALHt1zjOax5NVWQ5Wy4H/PST/AH+dJdxNDZLCt1GT5wZkXcCwCnGc8YyePx9qqBGI7Cs7dDUkM8kn8KxsRzg5H0q5aXhtpEkUAupBBblc/SqaQ5IO7n2FX9P0v7bJKTIUihiaWRschVGTgd6ewCXd3cahdm4up2kc46AKABxjjFMKBsAgkD1OaRI1XqxwTUrIGx0GPTmgNBgiRVJ8pBk9QtawE7+GnlikjRYpx52QAzKQAoz3GSePx9jmIjI2TMGHpsGBVplwADdoe5QbuPyGKLCuU2LbM4JH+wM4/KhWkALLvIxj5flP/j2KnYqpBTPHUsP84pouonJAlDsDyqfMf0pgS29s7zKNwMZPJUEkfgRjP41dudPi0++e3nlkuXCqQI18oDIBHJLFhj/AGR9azUl87Jt4ZW2/wAOwrjnHetLVm1HVtVjuIbeFAIFjZ5pmTeRn5toXOSMdR2pMCHzUjkLJHHGw4JALZ/BiR+lK95NJ8pc7fQcfyp0eh3zMGluIkGfuRKWJ/Fsfypw0k71hkvsSKO7bSfcYAqbjIYyZG4jZztPLgjHvUy6bKYxM0ZETDiQdD+NXLW2iVdhmW4uAcozjzDGe2GPI/Osm5t5bXVZoZCxLnzlMzchG5AJ/MfhQBtWdnaQ/NLcQFh2crx+tWV1GyBEQDS8/djj3Y98GsGB7ESP5wcHPYggD+tay6lp0aAbhtA6GIk/yosM1kujsmj8hnjlTB3/ACnPYjB4IqlKswjCeR5v97eu78vmWqVzrNnhBGu5z1+Qc+3NRLr7Tv5ENjJnGc79gA7fMcAfnRYRPNDfrzGkaoR08lOPzBx+dZ8txPFJtE0gHokpC5+i8Vam1mdiI2iiXIySsocL+Knk/TNUZlt3tnIl3yEZAKZUH3yc07NBobNnd/2vbmyvp5cBg0ckcvzIwORwcg59xg96mtkiieWGcZnhfEiyIAfY4BPB6jk/XIrmhKVw0QQEc5Kk/TuMVoza5dXFzbubSJ7hF2KxLEvx3wQD3PtRa4JmrLfiLcI0LuOqKh/nVPy726DzG0kaNupdm2L/ACx+NZFzNO0peYxhs55k3bD7AkhT+VS2wN7dYuZ534+9kuR/OiwXNq307yxvle3QdcR4c/iRkj8qrajqVjpUMjOfnYN5ZSBsOwxxlgvqOmaludP067UQtA8Z7PG7Rt+YIry7V9SMl+4EjtDGxSLzGLEKD3yaznLlRrThzux59qSlNRulbqJn/maiiYhshgPc1c1pQNUmYdJMOPy5/XNUI+HwQcH0OM1ihtWdhkjbmJJ5PcUynSHLGmVRQUUUUAFXrG5jgV90ZeT+ANjYCeMsCPmx1HIGcfSqNTRS7VZSNw64Pr/n8aAI3ZnYsxyTyTSUrNuYkAAHsO1AwaAEpe9GMUnSgQ8HFWbQr5y7+RnkAVUHNXLNQ0y5UYz3700JljUhJFMVK7QQMdP6cflWaePpV3UXd53Mh3HoGJycVQNIYoqWCaSCZJonZJY2Do6nBUjkEVF/DQpxQI+ndE1CLV9EstQQKRcRK5A6A45H4HI/Cr3HZRXm/wAKNaebQLrTpBI/2KTegRNxEb5OMAZPzBjnnrXcrftKXSK1uGdMbldPLODnn58ZHHatUkZsu5544pA0qMWjdo2xw6MQR+VVSL5+iW8ef7zlj+WB/OkFvdGT97efL6RQ7T+ZLfyp6AXiVuTklLafdxJGpaNhx95M5U+4+pFUZtVtIpSlyxt2GcGUbQ4HdWPBpzWEbsS8lww/u+cyj/x3GfxqcqAzMSihwQwChQc+w4ougsVHvSSnlWd7Ir42sLaQKffcQFA984qtd3GrpgQ6LKWYcF7mAD8cSH09q0WkTPMhIAxjPAqCSUfwqceuaLhYzLe21yZ83FzZ2ikfwp5pz9OBjtncfpV5LUsXE63cgU7cpPnd74RVYD88evekLyYyJCO/FIDJwd+efxo5gsSkWiQ7bZLdEOQGCK5OTzljlj+dRS3DTAK5dvQFjxUdyTO4kkkcTYADgjPGcZB4PU9efcVUaRtjG5ClUGTKvGP+A5JH4ZHqaLhYtGIYJIGD7VCYyOM8UsbxyIGWberDIKnINRtMI49zIy+zNyP6UhkbEl2XnI6kjg1XkjwDnFMbV7R5SgdS/wDdRw7H8FyabJcyPwlnclumGTZ/6FigBrR5HBz9elQSQ5HQZ+lX7S2urmXYTbRAAHBlDMc9gOAT+Peo2MNu6xXcUvnYyVLhFPuODx+NAFATS237qOV1SThkB4b2YdDUV2YJkyIBC+eXRyUx/unv9GA9qtTNGGDLZwg4/iZ2/QsR+lJHd3FuCYHW3J6mFFQ/mADRcdjLbQ76RI3s4pLxJBktCCQh7hhgY+uMe9Z8lneAAtE8QLbcyYU/kTmteeWe5cieWV8MW+Zz1Pf61VlX5GAwCKAKttNe6Zcym2uwgIwWQthsjtwPzrTutU03Uba2GoQTR30ecXkDjvjlhg8evBJx71mhjsG7JP0qGVlI+ZCccZxQBDeeHr+ygkukAutPA/4+bYb1HP8AF3X8uKyVk3EMpwRXRaTrM2iXpmt1EsTjbJG/IdT1FdHqvgzTvEFkNV8O7bWZhlrV2GxiR/CQSAeemcfSkBwlpqV5YXcV1ZXMlvPEcrJGRn3HuPY8V1WnnSvFtldteW0el3tuULX0OfIfcdo8xM4TJ/iXucn0PH6nYXelXjwXdrNbnJIV+eM9j0PpkVf0ydYfDupjed0k9uAAeoxIc49OB+lG7Av3MElkxtp0KSFQ65BG5SOGGex/+t1FQqVK4Iz6Uy31Ob7J9lMqyW+07Ipl3rGTxuTuh+h57irsVnHqM8w0rzpliwWgcATpxk/KOXA/vAd+grZT01MnHsVQquGD5ZcYApshLpDaghghLK7Eq6Z5OGHUd+adnK5XkeoNN+zfaBI4PMa+Z+X/AOuqexPU6e78Lb7OG90K4fV7dztdQoWaMg8/L1bGOcc+3U1zjvGtpc4kBdpYlKhum0OTx+P86k0K+1C11aA2EqpLh5DvJ27Y1Mh/HCkD6+9Lq2o3GqXFtJfJHcxwAAxhjE7kKVBLjqeQT0zjHFJXXnYenoVVYOuc8npTFQDLHJLcmtubQYbmxa+0S/SeOJQZre6Ihnh45LZwpHXnP51hwXCvk5+ZTtZT1B9xVxkmiHFosKzRzLIryI6fdeOQo4+jDkV1sOvaXr0Sw+JLSMz4CrfQod3APLY5H4ZHXgCuTJBAyBSEYzilKCkOMmjpLrwJcPELjQ72G+gbJVGYK5HoD0Y9fSuSnSS3uDDNG0UqkhkcYZfqOtdFZajcWfhoxwzyxGTUHJKORkLHH/U/pVuHxTNP/wAhiztNUiUfKJYgsg+jdOlZ2klc0umYGnJ52p28O8xl3wGChscE9D1+lVFHABrr7TRdKuL2O+0PV2tbouRHY3cYYBmQ8Ag5IxuGeR9e/Paho9/pSxm7tpUWQDYzKQrZGeCep9qFLUGtCmeRjgGmSKrYLKr4PBI5/A9RSbsnjpTg3GDWm5nsTRTs0sfnJDdRqEHl3Kbm2oQQqyj51GBt6kYPTpVoSeFLqaws9Ra90u65eabaJkyc/KrBvlX6oTg4PTNZzLnp1ppkI+VlBU9Qal0k9i1O250mreBL7TovtdnPDf2uwyF42CsFxnOD1GPQk+1czLE6SNHKjI6nDIwww+oPIq3Y+ILzRbF7aCS6ltHV0e1MhZdpU5wMfLgdx6c5rp7zxZDq0sjapodlewbQU2MROh7jeeMe4x9KSc46MGovY43anoaK3zaeEXYsLjxJGCc7BDG232zg5x60U+cOQqArKrRoRjHzDdyD24qsypG21XB9hzUwgkkkXYr+ZnACdfy9a6zQvAWqalLC95by2ls2d0km1WBHQFTznPtVtJashanIGFwGwG3Ece/tWnpGhX+q7ha2kk5HLP0Rf95jhR9Ca7iLQtB0e4jnht21SbPMt24ESD/ZQD5z9eO9Xb7xK7+UIhsWJgV+b09FGAB7fnmlZsfMkYlt4Y0bQbmCXV7yW9nHz/ZrdNkS/Vjy/tjaPWtDU/GAkiMNlELVC+5mXq39Bnr3PvWD4xu0n8SvJHvDzwxSOo65KDH6YFc79oYsAFIP51K1V2F+xsXeoSSu7eYxJOWZmJJJ7knr+NZ4UySBy7BgOKI1dnJ2gZ6FuT/+vinllyMctjBxVWEIsgTO/g+uetOhDXrlidgX5slgMDjJye+Mmo1VWOQBuPqeK6bw34eea5SW/i8vTXRwzsxVnPIAUDng4OfajoC3KdnG15ftZWkMplyV+SPGOfU49+p446V21rY2vhrS5b29aC8uVZVEZX5YyecY5ye+Tj2Hqy41Oy0uNk02COJQApb+Nsep6n/PSuP1HXDdubdWxGGBwDweCMfmRSjHmaQ5SsnYtalrNzqlw1xK55OdoOBWYqkyMxbLr0HtT4m+TB59Ke8fGScY5zXWlbY5N9xqw7nDc8jPStC2Se5ufsVtZyzvIm4tGAPusoGSf94nHoDUukaHe3xWe+kjsrNiPLnlODKP9lc8/XgfWt2a/g0tPsekSP5f8chHzO3ckntwOBisajvoawXK7sSC0tdCDSXuLq/4YxqSIosDjJ/iP+feqniLX55VKkqF2howowFBUHAHrhuT9azr+6nnzLK5JJwxY5J/yKztcaVotPlY/uzaKNo6EiSQf+ghKiKSLk3IyruZpSjnJOK09NRtQuoLJIt/mH5ycEKvGW56AAd6oafp11qlzHbWsbSSMeFHYepPYe9d2iW+g6adOt1DyqczzY5duMgei5H44zWrdibaD5XgsIzDbPI+RtMz/ek+vovTjv1OTWHezGSVWIwQwxg9ORmiW6LspBLDJ5J7UW0Mt9erBEu9m3DjjqpGfwqUrCd2yef96wjCHh8DHfpxU6j7JlthFyOB/wBM+Oce/v27c8iSbUbfQL6OPaslxv2zy9dmeGVfTHPPUn24qtay+faxTkYMqK2PTI6VCZbRV1RJZLGYks0nltt553Y+X9cVDFFbxWraheAm3OTFEOfPPX/vkHj36VbumAhNxIqm3Rgu0nHmN1x9MdfwHfIw59QOqXEe35NmPu8BQOgA6ACk2OKJoymtXlzeTM25MIuGI7Hj6dKlvfDs/wBjFy6xJbIoI84v14wV2uuD0AzkHPSrGl2dlpOhi51aR4oFAYxlSHkYg4AHH+QegFcnqWsXOpNIrSTfZSRthZiRgY5PvkVDLtffYkgtpoHw7TyE44YnB98Y4/Cpd02/YoVCSThcA/41lQwxRqWSNFBOT8oFa2m3jWsTlHxJKoVRyCFB5IHTrgc+ppNtjjGMdkNdXiK+ah55HGcipFkVjwrAf7h/wp7FZG3lmcnnLdaCFAGCePbrQhj/ADNgXEcjk9doHH5kVpW95FDpN6oMi3UihUQ4wRuBwxwcDjqM/Q9DnI+MH2zkmnLuc/IuR7DNMBUySQ77MjIC85b6cce/6VKACBtJDe4prMEjG+RUU/3mC5/OlMiLD5gcFSSPkBc5+i5NIB0SkHLjzM9VJwP05/Wnal9gS6tbZYX+0NB5rt5zkdSMAE4/n2x0p1pJAWEjsSME/dIH68/pUepTT39/BL5ymOGNo1DDkjdlfyH86TBIhMcAmDrZ25JxuYr6egxirbXUrAtHujjJxtUYH6VTeJ3U4kwexC9KXy5pAomvLmXaeAzgAew2gUrjsTvdTykNJIz7e7EnFM88xAvv2juc1B9mh3Yw7ZP/AC0dmH6mtf8Asi0vvDdxOdPtjNb7fn8gFtu4buQOON1F2kBlDXoi+wXTSN/cRixP4Cn/ANtvIVMdlc/Lzt2smff5zTkARSoUBSc49/WgAMTkAHtindiLKaxfSKZFs4oZfWaQE9fYtj8qW+vpr82LErC0COkuz59+45GMjC4I9COTxVfb1JxTT9760ASNkTlllnEZB/dlgP8Ax5VVs/j+FTNel4/KNvb+XjBBhVifqzZb9ar9wc4H86RJE3gMSVJwdoBI/WgZdGq3qQ+THcyxw9o0cgD8BxTobiW9gbT5pQ6uwaMyDcysDkBST8uemc96pCe1M/lBtzAZ2bgG4GelaunNYGMu65YHIWQbR+vX609RGbNFLBM8EsLxyoxDCTg5qIs47dOpq7qUSNfG5t5om3RqHj3N98E5I6jkFf8Avk+tUWaYr/q0z6F8AfpRcQ+C8ePcscURY8ZkUEj8+BStcTXUmwW7lhjJKgA+wHFQeXIX+Z4AFPGxS4P4nbj8qtTSQyJgQqoA/vsD+hFF0Fi/ptynlBpRGQjY4UAj8h+uaLrVjny7Hyo5ecrOPvnsQwb+efwqrBpljfQssJkW4AyBKfMVvbntx15IzVZMxq8QhEDBiHjVNuGB54FJjRsxai5mJKQ5H8TpsYnoTycfr+deQeJUfT/EFzA6YgkYyRAHI2ntnvjpXo4jMiZGeDgn3rN1rw9/btr5JIWZctDIeAp9z6VFSPMi4S5Xc8v1OyaO3iuQ+9SdpHdfT+tZSSGOXIOOe1bN3DPayT6fdIY5VO10Jzz2IPcd8isVwQ2CMEcGsF2NZWvdBMDvydvPPykEfpUVXY4JLhNqrjAzkt2689h+OKrSR+W7LkHBxxVAR0UUUAFPTo+Dzj9KZViyRJLuNJX8tGOC2OmeKAK9PRd3cfjTWBViDwQcU6MkOMUALsNAHOKuPASm9VyB1I7D39KrjAOSSKCRiRF3wOPrWrZW+xwzEY2ggqff1ziqa3SoBhc+ua0dwayPlru3DIU8hj7DPDD1HIz6MKbAyrrb5pAUDHHyjH6dqrmpZf8AWEgED0PUVFSGB60oNNzzSigDvvhLqAtfF7WrLuF7bvEvP3WXD5/JWH417b/onnr9phR2AO05ww91PY9Py5r5f0y+k03UrW+i5e3lWUA8A4OcHHY9K+kYrnz1SZAGikUMjDuCMg1cWRI0GC7XkicSwqM7iQGX2Yf1AwfbpVfzjt4GPrzSxStDIJon2uvRu/41FJHGzO0EgW5f5hbyEKknqEwPlPfB4NV6CDzGPVvyqPKZY7gT371FHcTSRkpZToQQCkqiM5PsxFQXQ1Aovkx2oYnP7yRjt/ALg/gakZdBBHI59jnmm8KcHj/Cs2G3vjIDdah8pzlLeML/AOPNu/kKd9gtGG6VZJzkgGeZnB+qk7f0oAfcX1pA4ElxED/dDjd+A700SySKJIraZ1x1bEfH0YjP4VbBVbYxwRwW7dFeOMKo9mVRyOnTkDpmq8c7sqLPC0LsCQDyGAOMg/h0OD04pgMcXjcRi3TI4ZizEH0wOv50xrd5BIZL2dXyNhtgse367gxP5ip5MqTuyB6imhuucD2FAirHYRJdreO8ktyPvySEZkPqwAAJ/DnAzmrThZE2skWOmBGo/PA5pp47c0wt8vt9KBiszFiCxJHqajdSR70HpkHik3fIMg+lAiJwGBwoz708TiWBbW/iNzbq+4Iz4ZT6huo/w44qJ5AgyT068dKa3TKncCMj3FAya40uKRXk0yRZIY+XSaREYDPQZwMAcHOOhwWrHJ3EkYb1weauKxX5lJGO4q/b6lZSgRalbRtwALhRiQfU96LdgMIoFGMcAdKhkiyBgYz39a3rrw+7xrPYzpe25zhoyN3XGMdzx2/KsufQNUaMyeQYgD1llSMj8GYUXAxpIPlPzD6ZqrJGQMITt9elb0Ohwosjahq0EWDkKn70kfRelRSW+mRysq3VxPb4BXagRs9xzmhXA52SM7slccfnUtvLLYyie2klilA+/G2OPeuptF8NRoBcBmYg5YGU4P0wO31qG8k0UwuNPZoZc/Kz24YHrnJJJx09Meh7NJiL+m6hbeJvDsltqOk2k8hlKRtGfLl9codhG7OeF6kHI5JPG6v4dn0KBpUuBPaM43hFZSjAH7wwV4yRkE9+lW7iCf7OJZ1QKSQGSVXUccZAOV/ECk0/UtQ0eYS2sroh5kiz8ko9GHQ0tLeY9TlxOyldo46n2qzAVjujcRqUuPvb0cqU9CCOQa6e20vw7rZuvMkOmXG0FJXIjhJxnBB4HToGyeuOtYmraTe6RJ51ynmQkhBcwjdG/GRh+55Pvx+T2DRl631e1vwRqkbJcs3F9brlmJ/56x8B/wDeXa3ru6VZaw1K1Hm29o17bXETBbm1UzRgEd9vKkdMMB9K5VJQwMiNja4x7VsaHqV5p08kun3klsRw4T+Lp+H/AOumpPoS0WNDYtrEaugG62ugeeh+zyH+lNTDJyBnsa3tEvtIu9ZFxrCSQXUm5FuLZcIdwKsXU5AOCeRgVmXGl3Fu87QqbqzjbAvLcb4vxIztPsa0jK17mbV1oUyBIoU5BAwDnBGeuD2rVW/sLqxWyv8ATxJJ8q/bo8C4XHfdxuHrnJP1rLYDZnHHXmoywDc1bipEptHQah4UltNOgvbW9jvraU4QrxJzyMr/AFFYZDIWVgQy8EMCDUsc86xuomdEYfMVPOe2PQ+9dRc6/o/iCW2bWYpIY4IWSbYo378fKQwG7GM8epHBxUXlHTcqyZjz4Xwrpb5/1k9yTx/dKL/SssSfLiu0l8OabqugWdl4d1Vbl7UyOkc8gzIZCGIzxtPsQPwrirq1uNOuWtbyJ4Jl6pIMEe/096UZaDaFLkjjr2roLXxhqEcUNrdpb3tqjZdbqISHuOOQDwT94E+9c2Dwc0ofOTxxVtKW5KbWxvTaZpGsTSz6ffRaZO7kpb3RCxY46MowvU9Oyj5eSRm6hot/pQEl3CDESV+0QEvCTnGN2Bg+xwfaqXmBSBgZ9D0rT0W/vW1jS7a3xKgk8ryGXcpjdl3gr/dwM+3Wk4uOqK5k9zKcgHGeajbOMEfjVyzsLEmBLq+e3dmCPKwBjjABwxAHO4kDqMYySc5FvU/D97p0uPMt7lfJ+0Zt3LERltu8rgEDdxnke9JTs7MHC+xT09F3XORnFrJgbN2cjHocdevH1qBSYyQOg61c0u/jsReT7QxktHhAz1LlV/rn8KzmI993r7VpHW7IlsiUXT47/nRUG6L/AJ6YoqydT3mwtdB0AK2mWQebvdTZZueuNxyM+2PpVK81q4mmdXlkO4dm4HtgdKyJtULMA7qikY561g3msMsmyEEu3GelZFas1tQvCWO3JbODk1TQvIu5RjB6H1qvCwPzuwPHQ96uWqI0buoUj3NVsKxD4timM2nXkrLtuLQIAvqrFf5BfzNc5t2kPyce1ddr8bXHh7TZHwoieZBjrj5Tyfxqjo3hPVdW2usBt7b+Ke4+VR9B1J/Q+tSt36jMtJA8bM/y5IPtWxpXhLUtVbzlHkWo4+0TLjP+6DjP6D3rttJ8N6NpUTTIn9oXUSFsuy4BA5wuePxxWdrfi1GtEkVyisNy4bO78u3t/OmJj4dH0Pw7EZWCXU4GS9xgqh/2Vx6+vNYeoeIr++mfyZmWHOfmblj2OOn4Csl7yW9JadmEZ5C57+4qN5lXCjgNwOMUN2Gk+pDd6hM0hbzOTx7CsvNx5okiXd5QMzgjjYg3E/oOvrjvUqndOpwOT0JxXYeFND26Pf3+qB1ivozbxxqcF0OCSM9BxgH/AOtRFvcp2sQafYXF7KLe1jMkg644Cg9yegFdLa6fpFk8f2y3N7Or7SHBCKfRVbAbty3p2pi3ghgSzsYxb265BWPPPuT1J9zTWXjIHI5BrSU7mKjYzb7WXurtpvlMjqOepCnnbn0HpSeeuY8NjjkVkSxLDJsh24DMOBwACeKsRyypEqMjPISFVVXLMfYf0qLlpEmqSFrduPxxU+meHr7xBpWm4IjthPO007H7qfusY/EOAK17XQXWFJ9cBhiJHl2in97IfQ/3Rxn1xnpg1Neak0VsI8RxRIMRwRDCIPTHc+/8qe6shXtuNeSx8P2D2Ol7ySB5lwx/eTEep7D2AArJguTcrJI7szjqWOcZrHvbuSZ3xknrWtocM9/sto7WbZ5R3zsmI1bcTy30I4+uM9mlZakt3HWtlLe321cBQMu54VB6n/P0q9dXMFvbta2JJU8STN1k/DsM9h+p5qeSe3Sxms7RiY42VnlIwZDyOfbkYFY86FBw3XtjoKzlJt2LikZ88QySMjcc1d0xlfTUBdlghJjcqvJwxAVffjr2Az6AwrHJcGQDiJR88pGQnp9T6D2zwASFluyLaO3h8tYFLY2AgHJ5JJ6k4GTxn0FO9gsVteulvG2wOBHARGETogKg4/z1ySeaXRNMDiWWZikUK7yoX5pTjhQccD1qnbQNqusy2sYKWx2vNO4+UFQQ2MnuAv8Aga7DWdUSO0+w6ezDy1wWh4z6jHU/4+tTey0KPN9S1efWdamF40yCJ2WOLymVU9cZAHJ6k4J49BSxW0UwBWSQDHykqBn9akuIQLp2K7NxIDMMZ/Sm4YrsEmAPTmoXmaALGRecHaOp3Y4q+5aSGOJdixRLtX+8e5J9eSeeOO1VooXC4yT+FTrDIeqlfTJHNMCxE1nGu2SOV26F0cJn8CGx9P1pWWF2+QzRjGMhwTn1+7UQik4wjEHnJqaJOOM475U0XEMls1uGjUs8YHVlyC3uT/hinz6NHaeSzb5I5QWBlkZyRnA+8TjOG49qljtWlcttYKMc7TjP1rX8Q2aW13b2ikRiGBd4Zhv3Hkk9OuR+VSxmD5SQNthRIxjHyKBUx2MBhTgfePUkUoudNtX23N3CpPADSLz+RNTy32lRR4TzZc/xRRO/8hzQBVbbuJUEDsCcmkwSM4zjsDSS6rATtt9Mu3I6bozHu/76xUDa1dTSNAulRwFQCRNN655wufQ9+1Ay0rdtpyTRvG0lm4I9arPPeSAKbiJEPVI4Mf8As3NOAjkAZuSBjjA/nQBKJFXHLY9jk11Xh68tlt5LZXQx3MDowchOvGOevAP4D0rlo7hV2ptTaOgYZ/CnrcMQJFxsB2h14APoKeltRGZaapHKqxwo08gUblt0eQg9+gJ61qxW+pTKWj0+VVHeZhFj/vo5/SkkuC42hpM4AZi+f6VHgMoABIU5xnvSGXks1ZSLi/ggfsqI0v8ALA/WntoclzMscGpvEh6O9kw3H65wPpz+NUBls8cjoDUsE86DklCPujd0Oe386asxMgjsY4L28tZ7iW5MT7BvkwB8oPGwKc89yaVrW38sI0EbgHgyKHI/Fsmr2q3Nn/b8LWhjK3UAYpKNuxgR26HO7sP4fbi61jE0YeaaMDu0QJ/+sKWtguYytsj8pflj67RwM/SnbifvEn61NcwxRoHEZWJhgO/Ab8aILQzRLInzRH7pRS+fpjNK9gIc9+vakO7d1yKsb7aGUo8OoEggYSzkbH1O3APt1pZLmz3CNbeXdnjzT5efr1P4YB9qd7hYoTSJDCWckDgcAn9KnWMYxt68HIpzSzykJBDbRn+EqGlOfx21bRdaKbS20cYkVFj/ACyTSGOt7OSJ45FAIUhsA4qzqukTqv8Aa0XMM4/eguPlfIGR7HpjttHrxUglu5N0f7y4ckcuSSPywMfhV+Gy1GPTmtEn227/AHo3cvn8Wyf1qvmSY8cF2zAJExz0OP8A61SPBdwWzvNA/lrzu2k98YBxzz2rSj0eaFCsd0sKH7wjyufqBTJNGjgjO64h+bqz9T+JNFwOX8Q6JZa9p4CQSJex/wCpmxsI/wBlt5GR/L+flOradc6deSQXSbZUO1iDkH0IPcV7f9jXlVkUv2RcsT+WRXO+LNEF5otzJJGI54ImljZsgkLyR09j+NROF9UOLaPJ/OZbbZ2zx/Wq9W3AFup+QdOACT+P+e/4CqRjpWSNxKSlpKAFFKrFXDDqDkU2nKNzAZoAmu02zZBJyoPPXOOf1zVcV0l7pDNosF6SBuBTHfKKpP8A6EP1rm6CYyua+mIkwZJUkKAZJj5K++PT9KrXFv5LDcCM9A3BP4dRRZMd3zRqydyUU4+pI4q9exRtFviZ2UjLb2yBx69+aBmOQc9OPrWnp0JubeWIMcgEqoH3/Ufy/wA9c4DJ+Ufia0NJkeG7D4IXHJA+77/n+tAFKfPfBOSSagPAxVm8kLzNuUZz2/z0qrQAUtJRQMdX0F4Qc3vgzTJDKxZoAhIPTb8ox+VfPgr6C8DQmHwRpaFSD5RfH+8xb+tNESNVdMtlYmTzZM9VmmeRf++WJH6U+K1t7YEQQxQKe0SBQfwFTkljiomYgcc84OO3vTELsCZ9PzpjHdx1HpUWx0JEUjknkh+c/j2qSMykfvEXf7UwG7FCgEYxyAB0puWkU7gw55FSgtzuUfTNISOeKAECndyR9cUhjQ7d4DgdFYttP1wQf1prMQaGdI4d0siqi8ZY4H50wGkXctwfKsjJEAxbyiCUUAc7dxZh1yQM9Bg9aaQEYgLh1OCrDoR1BHY1BHqtldNKlq5u3QZZbdDLge+wHApxMt7cKxka3cJsV22kYHZl4bjnnrz7YosIkLk5BAFVywXOTn0qAfbGL+fEka5IDRvvJ49DjFSJgfK29iCDxhcjPcc/nmgA3kDBBx6+lV2BTDSzqsY5wOAfrmprmKKdWXytoPQiVwQPwb+lVF0mzi3MtrGzEctIPMJ/FskUAQPqVnuxDMJmHaIFyvOOSOBycc0pvL3zF8uy+8MgvKMH69cVbW38qPanyc5ynGD68Uiy2kZfcojYj5XQkx5/2hzj/gOPp6HoMoSQahMm57iGAnqiJvx/wI4/lTDpy4/fT3M/OcM4X/0EA1pvnyklZGRJBlSw+VuM8HoajGMfKM+h6ikBEk88NuYYSI4s5Kp8uT6nHU1VdnHWrR29CR61G0Q7HincCi2G6j61WkUKMAdetaTx4PXNVmDbuimkMpM+04KHHriq0g8z/wDVWlNCJMAEA1GINvVRzQBThmntZN8MjxsO6kgj8RUktzHdef5xcTSsGackyHOckkE9cemP1NTS2wHJVgajEEZBG0mhAZnBdklyMEgOBhSPXnpUsN3qGmpILGdo0Y/OByGA7HPUe3Q96utYSMpMcMjYGcBeo/rSRyTTWVyk2mXEjrMd17HGSY/VWUgA9/4gee+KdxGK9va3UrzTNLBNIxYuEDLk9c4OQOT0B9hUklm2nJu3KbWQArKrbwzjAbpyoJyRkdPrmuy8PaLoRuQ99di8EnKi4Q24UcjgH75zxgEj610t1oOn29xEbdIo0nwkkKgKtwOwYcbj6DrwfehJvoK6PKwWJGDz9c5FXtO1S70q5M1pIUZ12yLgFZF9GB4Io17TW0TULp7aBxYhlZIwfM8sHrkg/KAc4z/PNVLF0vGCxZMjcKncnGa1v0ZDRpzpY386jT9tnKy5kS5kIh3YB+R8HHORhvqCB0zr2zutOvXhvIHhl4wGXhh6qejD3GRT9wzwetaOnazJpsLWU0K3mmSD5rSU5APqh6qfoRT1WwtHuZUTExg+hp1sziadZFUoHGPcYrSfSBd3ZXRFlmDwpOLWVlEqBh0HI34x2GeRx3rKWC7gvrlLmB4mVhGQ6FcOoyRz3wRxQ5piUbXNCHdHcLcQvJHMv3CrkAfl7fzrsIfGtjJp6WOu2T3CBcMZFWVXPt0K/hXFRsQ7Kcgj0qRnJXnv2puCe4lKx0E3hzQdXthN4buvs9wefstzKxDAehYbh9Tx/OuTuYZtOu3t7qIxTLw0cnB/D296txySRzxzK22ROVcfeH411Nh4ksZbb7J4jtIb2EnCyGEHZn0xyv1H5d6mzitCrp7nCOoJyM4Pr2p+meY2r2UaSGEm6h2uFzjLjHBzk13F74Ht762+2eF7r7RGx5tZHGQc8hXOM/Q81w8i3Wn6kq+XJFeQzxnYy4ZGDAjgjg8Z5H6U+ZOIrNMkbBUMG+b1FaFjrep2DQPb3TqYeEV/nVRyCAD0BBPTB9+BWdyecgHuB0pFYhSa1spbkXtsdLc65ourxSrq2lRR3s24Le2cWCsjOWLspJ6cc8tjI5BrM1XwzfaZE90k1ve2eSouLZ94XHXcP4f5D15rMJzyOvrVzTr+6027S5tZnjlToVYgHr1HQ9T19aiVNx+EpVE/iMcJcMMrAxB5BDDmiujXXrxVCm20qQgYLyaXCzN7k45PvRU+/wBivd7mubSG4VncYzk59Kr/ANnbWyhJx3J6ipkjuBIFdgoGTyR6f/WrU0/Q7/UpxFbRGXBw79FX6n/J9Ka7i9DMRF+4w3Fec+1dRpHhy+1S3Ty1NvAOTNKOOfQdW/l71t2WgaV4eMc+qTi5u15CkZVT64A4+rUzUPFNxKSIWESg/KF6gfWhtvYNFuWBZaJoqxwfLd3EOT5k6h9rHrwBwfY449aoa1qkU6K0w3RxjID9AcY6dOlYUl67yEbvlzkA1aaPzkJPzArgZqr2VyGmUIdUmhujLlhIrHbjoQO2K5jUNsckdvJJuW2Zoi2euDj8uK3J44o3RY2PygfMDWFf23l6jdAj5ZCkmDyDlQSfxobuUlYQ3JygXO3GcipEhluLiK3hjaaZzhI4wSSfQVo+GPDdxq8yzyK0GnRnEtyTtB46KcHJ+g/+v3dnHoegwz22nQm3aZGT7U7kyZIwCMknbnBxkfjSZTZjad4Vg0pludTeO4vF5FqvMUZ/2j/GR6dPr1q1eXLzMzOSzP1JoWYXEQfcCSM8HtUbpubPtQiG7j4YyIwSv0NSSMFiJOBQzLDFlmAVRzT4tG+2Qte6wJYbBeUtSdjzYIPzd1XkccE45wByNhYwLGyudX1mWHToPNCMDJMeI0yB1bp+XPtXTpJY+HxJ9jYXF6y4a4PRfUIOw9+/6VSuNbuZSYY28qBRtSGPhFHQAY9qzZpOM9TWsKd3ZmU6lloWL7Wy92sksbGfyFKyFs5+YggDt90E+tY9xfmaOI/dlkdg3qBnAq3c23mQRXKmQsjGMqB8vIyM/k1S6J4ee9c3VyfJhVuZGOOeeBnqfbt1PbLdk2gXvJDNP0eTUrwlm8uJR+9lYcKP8farmo6tDCj6Xp8jrbxKU68se+cdSf06cVcv76Dy0tdPwtupypU/f/2s98+tZLwxrMoVAA3JIHeovcvYit2k8icMDubaW47Kysf0BqVAtyvmNItvahxGZ2IwWPRVHdjnp26n3V5I4baW8uWVLJAytn705x9xf6ntVW6ln1TTlmliWLBDW8KcCJQcqB+hJPJPWob1KW2o271ONbh7O3DRWgJKRLxhhnJY9Sx4ye3QVr6Boz6paNPPNHBbQttBIxu45x9OPxrjo7iN5LqeXLFrqT5o16AMen6fSrttLrM6xMJlgtN5khVUZePbI56Ad/qaFa2oN22VzvLXT9K0+JLa2cJb53NLIRyPYkden4H6VhTXFmZpVSR2AYjKoxJH/AR1rInndLR457qdkwQFEuOD2x/n9ay/7SnnBSJmCgYByCT+OKh8tyo3a1NO5j06ZyjsyYXOSu0/iGwc1UiTTSoRLSdpem93QE89vT/PNU1OCckE+9XbGFp7lQGVDtOGZgig9sntzS6l20LTafbJB58whh+baIzcsWyOucRnnp0/Oqpe2jz5GfXLxk/qHH8vwqtqMzza1MjFJDHtX5GGxSQMgHHXpk8ngZzUZDxAZdcnuCD/ACNDstgXmaCXvBDQqDj70Zwc/Rt1RQ3M0M24Tu65PySbcEds4AP8qoQyyXUpSHaW7s8iqi/VvX2GasXcSQOVimWVQPvgYz+dAFpb6UMW83JxjOagu8anctPdolxIxG55VyTjGOvpgVXUbBuY5yewzipTcKozkY9aLjHXEclxn/SZkY9drd/fPWqf2O9iKlLwvg5YOoA/DHP8q0Ay4+8OeBnr+FOZTgAlQM8ZOKVn0Ait0kVsylG7kjPAp/kwNqbDzXM32WEyLsKhCS3GD1PIOehz+JdDcbYZYoWQsykBsE9vXpSz3Vza669xbGO4haFIz58rMy7SThW4GPm9AO4HJqrCvqXDpTvM0yKYLcKBumbb07455NRrbwIf3t0Gx18mI4P5kVnXkUmpagkty/7oSAt5blZNg6jI49x/Oty41RMKttZwKgPRoQcj8/p/9epsMrrcWcTcxStxxmTb+gGf1qs9ySAu8hc52g8frSXSLdvkgxZI5hwCB3HII/SrP/COWT6Lc6gh1Cd4UYlJJvkUgZOSqgcAg8kZx74o2VxFBriLftXLkn7qsM/qRVtbC+Klkgkwefn+QfmaRXNtA1tBhI8YKpwG/wB7HX6mozcyT7QzM/sTkCnYHcnKGMbXliD88bs/yyKfCLdkTzSoZTklEL7vqSRVZo1kA+cgq3BjwM+3epFTb8o+vXNMDQvbq11GwWyvLJLmBCzRMZGiMZZskqFbg89evUdDiqseyCNIlMhCHvdS8j0I3YP45zUSpxk/zpTIOCACKNBF2zu4LB3a1tLOB5Dlmitwpb6kYzV067NG0LywFtwILBiPyB4/WsUO5OAOO3FTxRfaHVXYhs/KSelGgtjrLZ0nK3KOwyNuVbIx9K5bW7Q6drE+wSSQzkTJvj2nnggc8jgc/wC1XSWETW8CoQCM5DAY/OsTxjJqJs4pPIeS3tZfMMkIBxHtwdwznjOcnjAz2JE6Ior6X9u3SPbpgH5WBfJ+uM/0rTNq88QNyZwxG4qBkDvjpzXNR7ANyM3HQq3H6U5rqYKWldkQdSX6fWqsI6eysbaOR2E74B4z8uD9e9aG6J02mSMn0LCuItv9LLfZ28/acHy1zzxn+Y/P3rRt9KvpmCiPy1xnfKQAPqBk/pS0A17iKaKQs4l8jqPLkwo/IZ/Sqhmsg587zUY4JVsMfoeDSHRdRgaJreeylGfm3SSKB+BSpW07+0GliNzBFLEwVvKjeQ5wDjLBCOCD0xRdBYp3l3DGF+ySMozhgAFP6YrkfF+smx0iRCd010DEisc8EfMx+gP5kV339g6ZBGTcXFw+0ckyqin6jbn/AMerzX4r6fbWKaS1skwD+cSZZC2R8mMAnjv2BqZytHQaSPPNoaIxqQzA8Afy96Se3EUSsMgcAZ5zxn+Rp9kQsquxIAYZIGfrTLsyFyDnHBAPpgY+uM1ibIqHn2ptKTk0lAwpVbawYdjmkooA6S2uxPpL7yAVQqfp/k1zr/eNWLeYrDJH6j/P8qrHmgiKs2PifY2f6Vp28/2plicbiPuqemeOvqfrx9ayR1q3Z3DW0yyL95eR7nt/Q/hQUaF1aLHOyggv0HPQ5x16nnvUtkEt1Zp0by2XDEYHBAP5cf573dJtxfSlpciNQRvHOBtOTz7jP8hnFNv0MRkjCKvykBSM8Hr9SMjn/ZzkCn0Ec3dqFnbaSVJ4J61BU8iDJUdM8VBSGgooooGKOtfUNtYRx2Nra6ZGqzQRiNo2OBMqgAEdg/A9M85Pp8yWcXnXsEX9+RV/M19Nx2zqrST3G5z0S3U4H0dsH/x2qiZy3Go2RyCrDhlcEEH0INNcDB3AEZzz2NV7yCW/Ctc3dwsyjaWh2qSvoSQc49etUH0KByBLcXkqgbCHn6jnrgD196rQRpCWIvsDqz+gOT+VDRTrmRVuCCcBGAjBI9NwB/WpYpZYIREkrBB0VFVAPwUAfpUHG5iVO48k0AV7ma/hTMdkrvjIBuUx27gn19OxqpHHr11vY3Vjatt+WMIZM9O5xjv7Vo5A4wOaTePuggfSjULjby2FxHH5kckYI/gmeMt7khv5cVVGk2EUiSizhdwMb5F3t1/vNkmrRvktUJlWOSMn5wV+YD1XHf8An70lxdRLMkccrmOYjyXZfvdeDjOD/PqKLAmP3BgFY4x09qjI+cY2475pk8TlPlchs9aakZVvmIb3paAOuAXjKowRiOGxnFQqWRArsCfXHWrAI3EkrjpxSMCG+Rl2/SgZWPXA5Pt6U7zNpIYYJ7VJNsjQ5wcckmoRIhAbG0H2piFYh8g88dKjaMAYCAD09amOw9jx3ximsqgBmYgUhkJBWF4h/qnGGTsR9PaoHVkZRa/OSPmhdgPxVif0P5nOKnM8LkhWDsDyqHJz9BSYbG6K3kbPHOFx+ZBouBUguoLoExv8y/eRuGX8KV9oOGYAn1NXlt/OA+3IGVBiNVcllz1weMfTBFRx6ZZozO0cknfbLIT/ACxQBTKD2FUJ7vTwWiluICehXcD+lak1hbOVBtkG3kDkj9SaSOGOH/VRohA/gUD+VAFKNJJ0LQW8zr1DsAi/mxAps9rqMFtDNJbxQpP/AKsvLuz6/dGM89Mg1dlAkG0yspz2x/hSid4o2QSbgy7XGOHHIwR0PX/DFAGG9vckkG5VT/dSID+ZNL5MIaNnQylB8wkdirH1KjA/CtyKws7hCsFylpLu4iutzxHns4+ZR165+tZMSPcRSSRRNJDG+wzIDsz2GSAQT6Gj1GK+pXSoYoZBbRnOUtkESn67QM/jVGRnL7yxdsdSc1aaI+n51G0ag8jr6U7iIPtO05RFRypDFcjIxg9PbNWoNYvI4njWchH+8JI0kz/32D6D61WkiycqBn2phAAwQevpTuxWOhstXmuIRBJPFEFOSZfnTaeCACGKg/LwOOOwFVb/AMHWcjG4sr6CNFXAwwZGI65KjjIxyB17d6yCOAA23ByMdfzFOhvpbObzYHkikxjcjFc/XHUUb7hsZ10TbHyjExVGJRyuCykf/q7+tJLvKkfeODgryDXfxavo2q2Ztr3gSLsdZxvY85GH68HpWRruhGwCaha7LiyKEPFgLIoI6nPDfUYPt3q1dEXTZyFzeTvb2jSMVZI41DZ5ACgDH5dav2GtSSXkN1fRrqqxp5Hk3bkkoT1DZyGHOD/9aqxs3XSYYxEwdMjawwwwTu49iD+tUYvK+8VJbpkVF9LMu3Y7U6RY6v5s+gTnMf3tOuGHnR/7pyRIPTv9TWFNHNbSmGaCaFwTxJEydOuMj3rNimkhlQo7KwwVdDhh9DXTx+MLy702Ww1SJdRiK/u3kz5kTAcNkEbv06nrmrjJrYlxTMdWfDMSRS+YAVD5Y5PHUHIqa3t7q5mMMFs1yCoKm3xKx4yRtXJ4/p26CJotr7gMY+Xnpwcc1rGSZk4tGhodxcWGom6t7mRGRHdY+qvsRm2t3IO31z6U/wAS+J5vEmmrbX+lQzzLyl1AxSSPAOFOT8wOfWotPBN4BjgW9wxI9PJcf1rJb+8GIHpScU2xqTSRrDSUvbEXekyvMqpulgnwk8Xrx0Ye4/KssAquGDK3cMCCPwpbeWWCVZoZWilXo6nBFai6npc53axbspYEG7gbDgk9SuCDTV4ruTozIAA5FO3FAMmuhbwhc3FubjR7mLUkAJZEwkq/8BJ5/A/hXOyRyKxSRWR0OGV1IKn0IPIpqSYOLQm+Puwoo8pTzhaKq4j2TR/CUdyon1SRShbC28MmSxBHBbp9QMj3ro7y9h0yyNpp6xxSY4jTjYPc9j7Vyen65PeTzadEEZXhcRqDhlkCE5z0UDA6HPWq9nexSo5iwccrgcEHv71k466ml7KyF1LUi8+JPmY/LuJNZMt2EUlyST0FV9XaTzQw7HJA64rOcSXjqNu3acgdzU3uNR0uSy3ctwSFXaO4HOPxrftJZPI8vktsyozkVkQI8kn2ezgNxMchwuSRgjPy4z0P4cV2OleGpIl87UZmtocDEW4NI3fk9B+GcimtrA7HJR6dqOq3zwWto0rbudvCp/vE9OldfP4S0uyuory+mlu5wiA27ECLcqgemdvHTv8ApT9Q8SwaPbfZrOBLZeSFUhi3ueOv+Ncfe+KJL2KbCFWl+8+ST9fc/WqinYTd9jZ13xpJGWt7UYQDbhOFHbAxjiuSn1O4up97sSR681nyybmAJznvVmMJlnUZCKc56fWnETVjttMn82wiY91AOMdce1OmuHjuEjhgkmZzgbASR2HAHJ9qm0bS7udbdIbZxbzW6OrN91GKKTk56bj9fQHFb3nQ6JavbWEpkuZD++uCMZ9h6e3XrRLcRmvaDRbJ9W1NBNdx7StsrBktwxChm7M2SPYds4zWTeX0ty7yPIzc9Seafq5uJ9MvEjYl2iYDPO44zzVGNhNbrKpzld2PrVR6kSFgyWb+dSkA5X8Kr2+d5xnJONoGSfYVrrBDp5Y3iLJecFbdvux98v6n0X8T1ArTmsZuNx1tCtvpz/a9yrMVeJFOGbGec9l5Iz1POPWs67v5rkbN/wAqjCoowij0A7f5zk0Xd3JLI7s5LsdxZjkmqC/u/MLMAvLEnsKhu7uzSKsrI0bXypg5jACo7qqjoq5yB+AxSsqyq8jyGG0hP72QdSf7q+/T6fkDXgR1up1TeInEbF0XOMxjhc8E8fh3q9NdJbafDbXICxuRGqHkLkHr6k9yepNQ/Iu3UxpopPEV+uwrFp0A2JEnQgc4/wAfrWliW4JihQyTN6c7fc/kfyqG6u4NGsRHagsSuxIUUEnd1PUf/rIqml1LDZkySKoZMM7cDHOQQOvXvmptYNxjaWh1D92Z8lMm4VlUEjjHHPrnP861vsS29uzom+YKMFjk+gGTXEw6vc2avbwTLJGCWWQoQfmOecn3PtVoazeyBWMm3A5CE4Pv16/THSk9VuNK2yNK4sNUKkuFKt0UEHJ9KdHoVyQdxiUZzhs1mvqd/IhU3DKT0YDJH9KrSGe4yLt2lQcjErD68DAHepaXc0TNxrO2gYJPIm7sdw4xSCS2gYyxXDh4gWLL1JxwF4wTz+lYn2e35xBGRnOCM54xVhZWPHIx0I4xTVk7kvmexkQalDtlmlR1LO0h4JPJ/PuBV6G5M4jMdncEN0LJsB/FsVaXf/fIHoD1qVFJYDjk+vFSUQATbghWGI9MySZUH/gOf0p0Vlqd1Iqxw2pycZ8xyB7nCVYW3M7EKAWB7sAP1q2Ld7CWNjPhHJUvC4GPlPcg8cc4zxmqVm9RSvbQzL2xvLWZYhe2xfYDJiFiFyOMEkZ/IdvXiGC0ZSTLcu+eyjaB/M/rRHdfar27kcbW84pg4LDAAGSOpwOverLfcxjNTsylqtRogiOdy7gezMWH5HinBI05SNAfVVANOgWN5MSy+WmM5xn8OKbIFEjIuTGOjnofw7UmwSRLuBHJ5xQDUYQIxILYz8uG7UbtoJCjJ9BTAlDndgoQAPvcY+lOMmV2BiPbNQ+Y2MBQfc0F1TJJ6ii4WJlO5lG/GD2NbumpJNpWr2UDb1uLbkZxg8qBn33fXiuXE6ONwdQo684rd8Om3nmmR52kjlXYwiXcpHOQTggc4x/OnurA9NTJV2lQOFwCOBSxNIseWZBjqc4pLaG/iTYNMlcJ8m95okHHAOGYE5xnp3q9HDqcgBjGnRqerNK7FfqAgH/j1Gm4iqk25SVViQOPlODmp1ZhguuDSXKXDMUXUQ3HLQQbOf8AgTNmoksLdjtuJdQuAeqtOEH/AJDVT+tO4iRpwOXYKp/vEVXk1K0hBDTorYyA525+mev4V0elvAAlothCYQPvMiuwIOclmBY8+pqS4SXTruW4+0Q2cM8at5vkZbf0KggqDjGen8Q+tJtoFY52O8meEyR2lxjcArtGYw2ey7sZq7mdGG20EknUq15CmPY4Lc/41QubueaWT7ZcM4Zi0a7iAU/h64yeO3cmtbS4dNs4VuiXvJiNwPlEIpx2BGB9aetg0L1sviNxho9OtUGAuZHlbGO/CitKG4urCG4XUruGaCdDG0a22E57kEtnjI54/lWXPeahOqPFF5SE4DGVRknoME9aozWF8ytPdFlYchN4O78BS9WMqwaX4esLqSW1juWJfcI2k/djn+76e1X3a1uG+0f2VZySoMI8qKzLj0BH8qzYGEc4EkZKrz71u29kb2MyGaQblIwvA/L/ADmhpJC1uOt31KdwGkVF5bCgU5r23XKKwEh5kYNtyR26dDTotH2M264bb/dUBTj04qO70i1itmlVxEYwWLMSRjHfrSuhl63vYp5PL3DcFBK5zgH1rC8QtLp2pJrQika1QKt4FY/OM4ViOnGe3anaZZ3y3vnzsgifrLuxkY47c9Bx/Kulik2XEatGs0JOJY5OUYdsg8HBwelN+Ql5mCuuxX84gsYzI7AEkt8oB78da4f4spK9nptwWaSISOm4D5VJA4z6nB/Ku0u7C30/xPNaSrttpgJLZU+VVHdTzx2OOPvelct8UYpIfClkqhltxeKFVzlh8j4/r3qZaxGlqeWQgeSOMnOcD8akuwXjEhwSwGcd+/8AI/pUNv0bjt25qxMvmRFsFVLYCjkD/P64rI1MgjBNJUrISM96ipFC4pKdTaAHI20/XrTaKKAFWpo1LMABnmo0GTWlpsG+cfLu2jPH+fpQI6bS5DaW6ynBQLnaRkMMEc/jt/Osm+meWQMzeZJty244z/nj/PNb7pCbKHYQ3lkNIMYDZG4/yFYGpMVJYDHGASOaoDElYbyOoBqsfvGpZG3MxqKpBBRRRQM0NCiM2v6dEASWuoxgf7wr6aJZh82M183+EYzL4w0dV/5/Im/Jgf6V9JMFHAJ+gq4GU9yo6FWJqvMJlQmGNWb/AGmxWg8ZYYFVXjYfxYIqmJFWOSfLCUR4xwVPP5f/AF6b5/z4KON33Tjg1ODuAypz6N/9akKZP3QcfpSGVpoVmHzlx/unFOjREUKoX8qmZCTgkj6U1ymQo5Y9B3P0oATaD6YqpPZw7Gh2qY5Mlo8cE+vHQ9anaYRsoaOXc5wo2n5u3B6de9LBJ5xcAKhQ4ZXfJP0K5H6j6UAZwefT4d8rTXFoOrFMyRY9cfeHv1q7bXEdzarPHkIwypYY4q3NCjWzLHK6yEfK6qPlP0Oc1zo8H2TRn7RPPO5JJ3NtTJ9FHAov3A2VjjWMKoVV6gKMDNROChAYMM/dyCAfxpdP0iz0pibFDC7LtZhIxJH4n2qyylslnJPXJpXGVwjM7IybTjILHg/iM1nSfbULGK0Q8k7vMyeuOgH+e9auwZ4B+pppPscHpweaYipp8VwZVnnlhbK/6oQ4/Hk59avb5kOUmMWCOI1C59j60gnkGwSqXjjJKxliMc9cj+uR7UyCCWeQ4zIdxICKcgdgfcex9OnQAxHDdjxnI470HqMqD6npTgAygqxPpimNkDBHekAclc8Y9DUeVQ8Z/OnsoI7VF5a5xnn60ARlcngd6jMWTmrIB/Cm9yCDQBTZO+KNhUgkVYOBng1CWIJJPFFgGPGsmCuAfbvVrTdRutNMiRxrJA4xJFIMqw/z/k1XztHByPcU7z4ynCgY680wLbadYa0WeymS0uyDixdcKx7BHzxn0/QVz93ZT2cxiuIpIZB1SUYP/wBfr2rQmlt3jO519MHn86YPEsAhNkwivo2P+pKNIQemV2nIP+RiiwGSVHpTSmO341o3Wl3sp8+weKC1YAhL1sS/985Jx7EZ9zUMumlN6jVkc4+Upbc59CCQB9QTSGUGj4Hy8e1MMYwS/T6VpNp8PnqwmuWi4yjuoye/KgcUxtPtMbmgUlemZGbPHuTRcVjMUxxvuMi8HO3PNd1oF/NNBK0yiOAJjEq43Z4Axjpx+Nc7DOYY5Iot8UTn5lTKK34CmPMI8hHlweSozimmxONy74i8M6fdwxHT0+z3W3+AsRJ05VME+vIx06HrXEPFLaytHdxlJSflfsw6V0cl1MWVvODgchJRuXOQc4P0H5VZM9vrc/2fUoIonYBVkUhcj34xn/8AV6Cm7MLWOUMLgqx2lWHBDAj/AOt+NNjjUyMSSAvJwccDn9a7JvBiW8QuNOuVcAZZJyGRuOMHt17+vauakjktdQELp5E+D8r8JnHY9P8A69KzGMt7m60u7aRMxTA4bcuc44wR/wDqrq7vUtG1LSrKfUoDa3FzvUXEMOWJQAHOPvDkYBzxkDHWuJuHluLh3kxuYlvlAA5OeMcVqTyP/wAI/pETYGFmf3yZSP5KKpbiaNnStEu5rudtO8vUoJbWVFeCVVI3LxlWIKntg++M1zlzBJa3D200bRzRnDKwwVPuKS23WtwlzbyvHOhyJEcqw/KtpfEcF1GLfxFZm9hzhbuMbbmL3B6OM9iO/HSrUmtTNpMwV+U4zxRIu9MAHGa3rjwtdPDJeaPLDqdgMndAcSIP9qI/MD7DP4VgpgoWDBh1HPWtIyUjNxaLVhqNzp9ylxbyNHIvcY59j612Np4n0nxBts/EFqu7YEW7HDKf97qPXByPUiuCLEZwtJvO4ggjp0olFMIyaPRT8PbeRi8Our5Tcpm0Ynb25Bwfworz3e3Yn86Kjll3L5l2Ow0TVDHr1lcsPuzJk7eSMgEH8K0dJU20U1q5HmwSNFIobOGU4I/MVzlmjyXUeOAWCgn1r0LV9NQ+KZvs0M8n2tVlBiRipIAU4wMfwgk/7VXLRL5j6HOyR+YfMbOey1paH4YvNUcSAeRajhpWHX2X1/z34roIdHsrUiXUcS9CLWP/ANnbPPbgccdSKhvdZuJnMODFbrwkcBChR2GOOPzqFFsTlbQ03Sy8PaPePYxB57VN8rPyxOATjPTgcAd/SuN1HxFPN92RuSD8x5wefwrV028D6zHaoG2TBg+R3xn8e1edrdeZZxEHlkVTxVp8rsTbm1J9Xu2mm83eWGMew+lZqSbeM9+adIpKqAO9aOh+Fr7X5sQAJArYeU5x9OKm5aVjLjDyP8vI7HFdroHhGS5gW71Pdb6eQGC9Hm78ei+/5eo6G08PaR4W2yyql5dDmMMMJGfUjJyR+X5U281We9ctI565HPyr9Ka8hNly+1bzEFtAmy2jAWOJRgADpWerF8gjGKYp+bOSexNKXwDgnFUZkvyhfUdwe9ZVlaGSZrC1Tc8WQewVB/Ex7DHc1o28E13J5at5cIyZJ2+7GB2929B/kx6t4jgiga001ESBsM0iLhnI7seMnjr/APrDC1y9GbDRNNv5LdvtGowwmQzcfIuQG2A9MAnnr9K5gXU91cmWWQszjuT2pLaVZCwlDFZlaKUAkblYbSKWG2MWVkYM6HaxHQkGhIbFXKvyc8/iTU0UcDyO11hbNP8Aj4c9Ap7e5PQD3p9tAZHeRhIII/vugyT/ALK+rH9Op96ct5Fqklnp1pbSQWtu4kZHByuPUnqx7k5JqGxpHQ5hSNDDCsUaKEijXoqjp+NYutxPc2JCnBEiMG9CGB69uhqxeX8cETSyNhFB4rIl1a51ArJbwOsEa7EWRQCTuJJIxnr78dOetK3ULlG60+51C/e4iutilyQoTOOnT04FZ13YjzDGd0vljOd+dq9+vf8AxrpLZJv3k0rEqikthSSfoO/4VkXd3BLBc3KiWN0BXyid3Of4hxjnGQQw6+lF9RONloZsURJaQBdhxg784GOnoOB2qcKFwVYEevpVaDesaq7lgPu564qYMm7hgvPAzk1BslYtKUI5yWp2Nq7fWqrXdvDnzJ41P+02Ket5C4Yo2/HB2ckUBsWMoPvkKPWmieHPyuD261Se3gkjAfT7iTcwKySQnPXszDkce4/OlNi8T+YNOA3HIWWXDD32Mf8APalYG0aO4HIGBTl7cHjvjiqym6MWT9njYkgIXLEY78DHr37dqnBYKAXYOOq7MA/nTsJssRMVIKjJ+lTpIxlhZV3TKxKk9B8p5PI74HfqeKyhHdtISbsKv91Y/wCuf6VLFEQ5aSR5SOcPgAdscYz1ppobWhThikt7+/WSQN/pLrwMcg4PUZ6g9asrKCwQMNx6AHmol0uzQDEb4BJ/1r459s4q5FFBBkrFGCR1UYPbuOv41KG79CIqwOwxsWHUEYodrpVXy7XeDnOZVBGPYmp8jdxzTh6nFAkmMhSR51DC3jTAJaR3Iz9FXNaBhsrMoXi8/fHksp2jd34JJ49e/wCBqoi7n/dqWY+laNw4XSmidds0MikZX7wYH+Wz9RRoN3M1jDtCiN2Yc/e5PHTjA96m84GPYYYlO4txnjpxnPIH9T6moVJIyAOec0j7gOuPXIo2AdHKLX/URxRszfeSNQ2f97GafLeXE3+umlk29AzEgfSqyqCOWP4d6cAirgc9/vc0egE4kI+7kk9qb5jM+0uAR69qiRVAwvp0q5bw2/lq09wcHOUCc5+vPb2p3YEY+VmAIbHcdKs209unLLJv7njH5EUyKETO21nijUZ3kFv5D/CoRGQu4hvTPT/PShMRow6owvhMIgcDC4PUf41pXUh1LR3aOEoYpN4+Un72B1HcnAAxyTXPRxkMAvP0FdVp1kfsfkXE5RJDjhMMCT26856HBoe1hdTjLhPPBhfdtbn5TyPermjzRRMRdySPAv8Aq9p+/j1q1ZW+neXO13MI/IlaFt8m3aQcYJzg/iKgNnoEj/6FPM8gOSLWN5V/ERilew7Ehuo0ZvJ1C9Xb0G7AA9MCni8DQhnlLMpGBuOc/WrVv4dVrcuqxMzD5ftDOpB6cgE4/LNNg0bUoIXES2SzMSu/zmKhfUDy859sj60+YGiHNih3ebKZhzgjj9eatxavA+3yvNYg4bYuR9M8Y60w+HJFlDzXu8AcCODaQf8AeLHP5CpV0nTZYdjG5kkz8zyS7Xz/AMBAH6Z+tCYmSzaqwhzFE+eh6HA/DNUri4kkZGMgt1U8PcSDbnnkjjnjpzW9Hp8dqVfyLZoHz5DiLkKDjDMxJLcdc9+Kv+b8u3PA6D0pOXYaRjzaRclUe4EzMACp27c/gSD+tSIlxbMVitZrgfdLtKgU+4yc/pWh5u7Py0oIA9KV2OyM66s5r61UuBbXURLQSId5UkY5JGMY4xj+Qri/iVYTR+B2a5mMs0d5GylT8uMEeg55PSvRua5j4g2v2nwLqiiLe6RiRcDJXawJP5A/hmlK9gPn6LO0461bBPkkDP8Aex2zj/69UoyQMliAK0FjUwt1yBg4HHU5P5AVmjQpom5j0IB7/lVW4QI/AxV3cULckEnn8KoynJPFDBEdBpaSkMSilNJQMuWsW/Ge9bOnoY3O1dxPyjAz25/Q1RskXPQ4xgDrmtmxjEd2jsvAO847Yz/iKpEl67ugkRQgM+FYYGMHPPfkc9T6DGOc8zeSgwkYyc+v6/yrf1FpDLLG6ndGcK7cnqeOvqP1rlrsndtP4D0FAFUnhqZTj936mm1I0FFFFAzsPhnbibxrbSNu2wRySEr1Hylc/huz+Fe5LN5UqwXEib3XdG6t8sg9RnofY+h69a8q+D1jFNqGp3r8vAkaKOf4ixP/AKDXrbCKWMK+xow24K2CoPsK0hsYy+Ihd99wNk6lcfcQEt9eKie0MzjC3Cgn7zTMoHHcA5/Sr20sAuTsxxg08thcEZqgMZbK4jZtroCRgO0jv+hx/OoBZak0pEmpIsG7A8i32Pt7HLFgMfQ1uuARkcVU83cXCqx2nHK4yanUZhx6bDdXMiXlzPNPbNskg88qobHDYXDc9QM4qddNsrGWS8it2EnJ3ZZ2GRggck9K0JDNNCFkWNXU5Uj5yn44Bx6jocCqK3jQ30tveMisHAQqpxIP7yn0Pp1HING4BbwrexJcysTE6gpCegH+0O59ug9+tXFVcBQBtHYcAUoiMbEoAUc5OOxpxC560gAL2oKkd+PSkye1ADHqxJ96BgVzzUbyAfKXUE8DJ61KQcHkVXe0gMyTNHvkTADEnjnNAhFjlVyGCt7jin7AFxkAe5xTGu4wxRkmBzgYiYg/QgUvnEBQilgxxkjoPcE0ANeFiBgD86iZ2ifdyrryrL1B9qshhySuffFMl6Zxx6ntTAb9sg1C/CzXKW9yEIPloB5x7Fx6+4H16UtzE1vc+Q64kChvlIIIPce1Z8tzZwyK0lzAJM4VR8zZ6YwOaZJqMUnEdpeXTYIwsLoOfRjgDp1zQ/MC9sDDhSP6U3Zjtk1Qmj1tGHkGExONw+0SBXj/ANltoIJ+n6UhttUdvm1SNEHaO3y35lsfpSuu47GhnYMYH0qnPdwQk+ZIEIGSWYAAe/NRrosDMGnuLy5IO4ebOQPyXAp6aVYxZ22NtwdwYxhiD7E8indCEiM13GJYY3kiY8SKpKHHHDHj9albT5AH8y5t4SOBl89fZQabsAyEAXHYDH8qj+V0IDKxB5welAy1pnh9buVvO1aOUYJEMf7tifbcD+mao3dpDbXP2O6sZ7eUDP7yYtuB6EFcD8qlzhwcMMcgDjFaltrIkhFrfxpd2wzyy5kU/wB4N3Pv196LAYQ0yxBz5CNnuw3fzp0aIY8W4UKf7orbXRFkhlurC8+0RDBEcvyugx0PYnPqB9axWyxDK209doXH5j1pWARYyoyWzx04pVCkjcvHqeaUkZwxwcZOB0oKKVwGb25oYBJHFu47fhULKCM5I9DQ2A33smozkdKBjXjPYk1GU2jGM1Oqs65HakeKQnigCqybhhlHJqFraMIQFBHoRxVzyhzvcCkKqBwy496AM5HurGZZ7SRomU8BTgfQ4rch1DT9YtPs+pQWxnPuY2z/AHhjOT9MfSsic8BUUsT/ABE4FVJ4FzkcMDwwPIPqKaET3vhi8iQtp063sC5YJhlIPQ4HfoKyPMn8uK3uVKTW6sfLZcbQWLfj1Jz710Vt4t1C0tkgmXzkRSvmrkMRggFgDhiOOvp+Nb13ouk+K7BriFDC4OVuFBV+fUHqD6frTF6nnm7Jyp681G7tuqxqWn3eiXfk3486JPl8yF8Hpx1GR9CPaqayhwWTLJk8/wD1u1UpWE0aWm6xf6XKZbK5aFzgsygEnHTrXQ+b4b8Swqb65TSNWC5e4iUJDM3AyVOAT64weucgZrjI2y2Oop/mFWODx/n/ABP503ZiV0amsaJe6HJGLpVaGb/U3ETbo5PofX2PNZyMMnHBz6Vc0nWbjS5d8KQyRnO63uBujYHII5Pyg55x1raXQLTxBZTaj4fzHKh2y6bI+5kbvsbPI9A2CefpVKVviIcb7HPbc+lFV5LhopXjkglR0YqysjAgjqCMdaK0uiORnueieFrDw2n2y6nN3eRpt2KVCI2DkL7kfWrN54iIRliTyYzycYJf+p/GuaOrJMx2sAQuSfaqU19E8TlJfkAwGHc+1Pk7ic30Fn1/zLiSOQsq/wAOOM/U/n/k1SttUEl827jK43Zzmsm7vd12V/gYfjjPf8RS2k0bfLEjB1/iq7oSXU6XRbyQ6tZTBGdxOIs46DOPr1rl59007xsdkkcroxOdikMeATxXQ+Gnmj1W2iUAuJwQB1IyMnFd95Gk6I0jxRwLcODvRIlDuevzsBnqe/6c1lOLumjROyOF0Tweb23iv9Rf7NYgkuZFIaT0Cr1555+mAc1vahqiQ2a6dp9mLbTVXBAbBf2b6+nvyTUeqavPdAKWAVeQBxt+mPrj8/U55+W4OFUNkE8Z6Z/rVKJnKd9Ebb3v2/TrW5kJOCYuAe3I9ujD8qhJGOm0DrUFjhdJtyI8YvJI88AZZFOP/Hc/hVqVoo1CMwG87R6sfSlL4mVYYHDA5cYXrg9PrTnHlRC5vHNva9lx+9k+g/hHueT2HetNNJfSbeTULoqWZlWJGxhM5IJ4+Zh6dAcnmuH1/Ubi/vGR5WkAPVj05pXuFtTS1PX47lIrS1jENop4jXPJ9ST1PvWRuaW5aMcqfugYptrErlFYkqBy3fFXPL8rywgLyscIq8kk9gPU02yktSaJ44FIc4J4Uk1oaRoNwmnJNqCvZWKfJmT5ZJAOAFU8jIHUjpyKuSafp+hQpfamgn1ZwTBDuBSA44Y88kdc889PWuUu9e1HVQttNeSy26cLvbk+59T7nn3qObcq1zqbrWYvtFtY2e1IgDCscZIUA9CfU52kn2NF3cptDRgBmUcgYrjtJONZtc5Ko4bA64rpJIJJbX/Vyi1jz5l2q5TvjB4BHByenajS1xSVjm9SW71C+kWONjEAFVt4wcjrjvk+npV1bXUQiwLdxwouBlGBJ/Q9/wCXfNUb7Xfsshis7ouqqY3QxFATnpgEjj1z+ArPj12+XcqFQpHQoDilzLuSqfU6yC3W2TzZtQupOOjsMD8AKrz3Omoku14pDL9/hST7ZxnFcs089w2ZZXb23cflUiDHT9KXMacl9zUe4s/uxabZooIPEQOcfXj9O59aZNePcsDIsPy8LiJRj8hVRMZGSwXPJAzxTs44IqSrFkTzN8xlk3YI3Bjxnr3pN58sIXYqDwCeKiBOcAc/WnFTnNFwHKheVVyvzEDlsfrVi9ZZb+Qx48tMInbgU+wWKOQ3U8kRjjVmEbrkkgdsenUZ7iqVuWjt1Vzlsdc5psESHGDgjI7dzSo2QQR171Gxz0pVYjipGS5AHJxjtRuyeKaVDKSUOaURycAqw9BigCXcoGW6Um5d2MUxopepBA9zUiWzNGX3qoUZJLUICRoJI4kkeF1R/uuVIDfQ96YSOcHiq5uYN6L9oDbjhSqs2T6cA9ahS+V32mC5TjOXgcDH1xj9aANO1kVLhSWIXgE5xWxq9xHJZW8cZ5QtvwTg5xt9s9c/UVhQ3kkEgJ0ycDHPmuqA/TDE/pRfPJfPFP5rQzKzlkQ5GCuFBPfkZ5HGT16U9LCJVWR+IwuQMnjIAqJn3vxg0IFIZbqa7lX+FYpREv1Iwcn34qGeEEjyLi8ij3EhTMpI/wCBbc449aAJ1BbPKqFH+RUPmQrgmWPB6HcOf8/1p8CQW5BFrFJLjHmTZkP/AI8cA+4wfepf9FMpk/s2zcHGUdCyZHcKTjP+NAXKct9BarGZJeZc+WoBO7HpWjYi+uXkkbSdQaGIFc+Tt+b+7kjjr9avxa2IsILOKJBgDyvlwOmOOlT6550NnHfW5LbEE33vkIDDgnOQMHr70K63E2L/AGRqciFiunDcQQvmyEp9cLtP4VK/h29uEKzXtrAnXbDbMxH4s/8AStS2uGkXdsJAOMjGD+tWXkZSNvOWAxtJ/wD1Umn3HoZtroVvauvz3UzKGJlMu3JK46LjGM8Yx+NOj060SdW+yyllO5WlmeTB9QCxxWoA5GSwHsBigoN+4t07f5/Gi4GcyWa3LnyrdbhyMtsG44GBz16D9KcZJZy2y6QoOCvBxVqW0t3ZmkhRy3BLDPFVxp1kkbxxW0cQf73loFz+VCt2EVfOgUtvvC8Y429CT9Mj+VTQXtureWsxYDJ+g9zVe50SARs0bsjY4DEVWGhSyR5WQcE4w3X3BHSquuojRudRiVSQSc8qSvBrKmvVnLMzmNSMKR1B9qdL/aVtG8boJUIwd3zH86zGjV4y7sQc9F6j9MUJroB0d1fww6BKtwjyqFEkcsXKhu4I/wABVbQr6W4bariSErkMSCR3H1FULC4xbGMzyBA2Anl5JbPBz1HP4Vb8O2dvbPPbxptktGMWC4fCkArgjjpweOoNDGjot2MZGfel7EdRSEH1pg3Bzk9faoGLuAwFOcDBzyazPEKmfw5qsSKS0lnKg+pQirzsqxM0reWg5Zy+P1zWXJ4h0+KUhriJEBIMkkgXOP7o5J/SmotoGfOUZ+brgVsW6jyHOARtK4zzkg1QvIYrbVrqGCUSwxzMsb4wHUHg+2RzW7YYks5kxvjVSxOOhP8A9cj8QPpWSLWxh3QKM4wBt44/z/n8azXNaF4H8wkn5T2Pas9qGNDc0GkopFC0d6KKANjTid3c88/4V1FkqHLyYI2jKk8/l37+/TvjPK6e2JO3Q9a6+DK2yyFeJF+Vd2M4+YD65x/npaJMnVI2fewbaFJGAemTx/SuYuG+faGyB/n8q39QkQRgLnJAYZ9+o/LH4iudfmQ49eMVLAY3YU2lfljSUigopKcql2CqCSfSgDtfhfq8OmeKTHcTRxQ3ULRlpDhdw+Yc/gR+Ne4g5+bauO2a+ZhYSJjzCqHAYKx5IIyOB61p2Gq61pe0WWo3kKK28IJCUz7r0PTv1q43RnKN3dH0USWFIW3HBHGPSvOvC3xFMka2mu5EvOLtE4PsygcfUf8A169AhkS5iWaGRZI2GVdCGBHt2qkyCYlQO1QSsI2ClsluQAp4FT7SwA6VDPCs8RjbdtPocUDIjj1BP0prOYx84DxZBZGGVPuaaY0hgVGbCxqBktjp61XudSsbIf6RdwxAd3cDApWbGX1hin3NYMSqjcYHfdIP931H6/nVdWSThT0PIrLj8QafCPNtmnlbBZWtoXY/XIGKLhLzVyL+zLWVyww8dwrRB+Sd2ADyQRxjHWjQDVAweaCR0Hes6JdYdSHlsAuSBIpdywBIzjCjtnr3qzbmNoz5m6eT/np5pA/74ULj6EtSGTbO5PAHU0iq06EwJJOo6+Shf9Rx/wDqqwl40LCQCGNhyGSFUI/EDPeka/aVjvlkYtzuJJ/WgANjNGgLmNRnHzSAnoT0XJ7Y+tMgs3kcTPLEIMNhFYCUkdPvepz29PWm87sZ4A9KRkEiMjZIYYNPQCBrdVnVp2vYmP3YJ28vOP8AdAJHuCR71XudOs5pTKbWN3zn96WcD6AkirUNrbXCPDdvNDNHJmG7BLYUjOCM4POevr+NS3kD2CyTSP5lovWcIQFyTgN+GOenuKGIrwQQwriGGOJcfwIFz+VPKD+8ce9OZQRuTGOuR3phBZTsHzY4zyM0rIY3H5dxSGIZ44qVmSNAz5H0FDR7h8ufwGKAIfu8cdaYwJ4Az+FWjbPjqAccZp8dvtUbirN6gcU7CuZ0kRIKlMjoQR1qKHTliJ8qMRg9VHAradBt5xkVGFAUAZ4GOaOUVzGmDR5BViR2ApowPvDBPr2rVng3Dpz2NUZUVZMEHNDQ0yGG5nt5w8O9XHO5T29/b2PFGpxx6hCziJba5DBjJDHgSdPvgH27U84ySrmqs15bR8yzxqQejPQMbPK1hbwyXkRjhlIWOVsAMSMjIJypwM9+/PBoVklB2kH3HNRW+o2U16oihS9lGSIzB5ox34weKu2OhRvDNctcpYGST9zbsoaJT36MdoJHrxzkGkBVihdid7bh9MUxk+Ypt+Xrk+lNuPtsDYHkyFcg7HGG9Mc46++faqc32+QH57VQRkKNxx/KnYZotIoA2gDA7Cq5uGHbn86rwxNEo864lmPU7cJ/Q/59Kgms1mJ3b8EY4kYfjwcUCJ5r+FB+8lRPqcZqs9y7qWgtrmVAMl0ibaPx6VZtIbdLdoGuHscrtE0Ee4yDI+WQAgleM9ycVDcT2uZIjcxsw6SRKxHHrkZ/GgZVknvWAC23lgj70rj+QzVFmndiJLuFM9kQsf1qeeymRSztuHZ85qptw45OfSlcdhkip5EiM7yF2B3E4IwDwMdjn9BT4dd1KxthbW1zJDGuQNrcgccZ/Ckfa3AXPue1VniIzlhwe9F2tgsuprWniISuV1CFHyxJkVQCc+q/db8h9atXXg6GW1F5pt6gaUHggCN8npzjb/PNcuVBNT2d19kkEiIpYEfeUEH6g8GhS7icX0K8kF3p181reQtHKhOe4I9QRwR7irO9Sp2kV2Vnc6X4qsFt7zTraOVDiERyFD79OQOT6jrkCue1rwlqem3U81nCbm08xvLEJ3EKeRkAk9P8+rJ9TLZudo/PHWl05jFPNKrTROD8ssTFCwJzgkdRkf5wKpC4PmMkiMjD+E8c1qNtj0iyuFgjUtNNE0gYEvt2NyO2N/5U73sgsOktY55Hlkurgu5LNmQ9Tz60VD56/wB+it+Z9zK8u50kUztKY0BMQ4JbPTr1/CnXlwn2NRuCkHIU9Ov69qLS2ZpmjJVAF3Fm46dvrVeSB3kEcUYPb69ea0b0IWpWYiQg4YjACnGOOvH61teHtMutVuFit1Xrkk8BV9T6DjrW94e8BXB/07V3e2gUblhX/WEf+yggYz154wa6C61Wzs9NFhpdv5FunU5IOf6n3P6VCfYptIdaW+neHEdrQfatSK489z8keR0Qf16n1rKuNQ3licAscnHGTWY2oFrhdzHPUCoHbzCTgkdR71oo2MpSbG3N0XnA52jqT0omICIRyF6HGT+HqaXYjrtwDgk4PrWjo+g3mrP5okFvZocSXTj5R7Lnqf096bdtxJXCws7nUdBurOyjzdLcRzw5LbAeQ25gCB8pPfvW00djYeW8apNqm3BuSMiMdwmf5ik1DUbPTbVdM0klYP8AlpL/ABSt6k9+g/kOKx4p/kCO5djyB61lKV3c2URNUlll0u/EbOz7Ul+9yWWRCTz3xu6+tYE1sxvHByT/AHuxropIY5LW8SWMHNtKCF4JwhI/HIrPitpr+e1i0aHz5pY8r/dRemST0pJjfQqW/mLIsMcJeaRsIo6n8q159UXwtP5UJgn1ZvlmaRSBbBgCAoPXIOc/TtW5bR2HhWxkkjnW71FlxJc4yqeyg9s/X36YrhtXsxqt4l/CJRO6FXZ2LB9pHPPfDp19KTk+g9iC8v7nUXkeWRpJnPLE5J6f5xVe0V1uDCFVpWJABOOa1tL03ypDK53FeCcHArasNESfUBDawzTSsUMs0Q4iRuTyf4uw/E9uVazHzdBPCnh6a5uWl2ful4eZlPznuq+3bNVvFd/a29sYbeURKxZUMcobcBxkAfdHfPOetafjnWzpmh/2dpkT2sTl7cg7QNvQ8HnPB5989a8mWZjIWIEuQQScnnHXPqOv4VDdwsTCMADy8sAe/Y1IEPXHPvUluJ/JSVWkBUcEE5Hp9KkijO75iD7McUFCKcrjGMDr61Kh7UrW0wKgeVyCc+eh/keKPKlQRMDCd7EFRLuKY7nH1GKAJBgDOKeuGHYdzzULxXBYFZIgCeQVJ4/OpSrFAPMbdjHy4x+RzQBINoJ55pSdrYyeuKhkRgd0KohLZCkHaKkZTtGYY3cdB7/jQGo7+0FtrOdTK0YdggkYheD1GepyBjHTk1WGpQSf6p5JhjloYmkH5qDVwbgAoOAB0HQU8qeCTQw2KqTvID5VtK/OAHAjJ/77Ix9TToZ7luHtoYmzyryMxUe+BjP41Oq4OefanEgEAA5PtSsx3B3nMYEUsMbZ7wknH/fVI6NNkG7uEbjBUJ+PVTSnIB9aQhsAMQPUjvTuIiurKJ41V7m6kLllDST4/RAvb19PwqKPT7ElWESylTw0jNJz/wACJp12xdVTzBkShvLOcYAx+fWpI5lUgHdz7E05KxMZXu0ywxYvuLHPWgOVYHJznIOe9BI4o3FUxkHFIsc7Fsu7e+5j1pqsj5VSC2ASO4H0pgjt5G3OiP2G7nB9frToYo4kCxqFT0FPcRPGilwHYIpx8xGakkFqp2xySM3TDIAP51A+cgR7c45JPT8O9KqHAz8xHfFAD8KGIHIoJU4H8I5HNNVcAhQNvt0owMGgEJzkgD9a2oBDfaeltqaM9oMgKpAZuPU8cf159RmQW7zOsaZZmPauk/siBdL8uRW+VkmkAPJ28kZ4xxxmi6E7lWzlih0BXjyvmJwzYJzz1Iz/ADq9o96JLbax+ZTtPHX3qtDa2tzY3UNt5irFczBVeQuVw7Ac/QDisyEXFnOfLuYlkPAjMikk/TNF+4HXhgM8kntShyVJbAAJ6Gsq3j1NolkutxRgGCDajH6biD+NSlr0oFhs256GaZR+e0nH+eKnQLl926ZPB6cUGVUUlu3eua2a9eSOJBbW8bbeGum2xg4xnEee49ai/wCEfv7wEpqMKKSP3iWzlhyem5sHgDqO/tmnoBr3Os2IG1m38YwF61VTW5JJMJbsU9EBbiqsHgyLcDearqFwccqriJT/AN8jP61rR+H9JQENYxzliCWucznj0MhYilzLogsZ02tC4HkgxKTwA8i85/GoY9MunDZjkGACfkIB+hPB/CuhSNLfEUMaxxgZCRjaB+AqYOVxkc54p3YGJZ6VdxSKTFIFGfvuuP0Of0q1Ppd9OIZP7RFtcxR+XujhDq6jlQc4PGW7nr14q9LeIgJznb1UdapLr1o1wYS7BiQAGXHX070JSYXRMltemJUm1Isw6vDAqZ/Bt1IdNWV8y3d64HBXzigP/fG2rIu42kKDHA+bJ6Uz7XAOsiqvYsRzRZhcrTeHdIupFkuLNZ2Tp57tJ/6ETn8asf2XYxwMtraW1u5GFeO3TKn8qfLdwwJlmOOvHSmWt6l1E8sZwikg5pcnVoLnz/44t2tPHGqoxBLTeZkADO4Bs4HA69KfogM0LQkFhuBHJ7/yzgD154qz8Sst49v2P8aREHPUeWtUtBmSGWQlSXAyo7duPf0x6fQ1l1LjsVtThAViM7lYqx7Zzk/z/wA5rDYcn610+pKwW4jIACtuw2cknt9OvPvXOSjDDr1702UiHGKSnE5oqSxtFFKPvCgRs2CbpMH5VPfHbPb/AD2rrnJW1JWMDMedrHGOTz78cf8AAvSuY0OMNeoZASikO3PQCujjui9q00xbaWZlAyOnTB9j+X482iTmtTOC7AE4AG4Z4xjH0rGJy+49etaerlfPJVuueD29v/1elZXRD78VLAjooopFBWrpoSKJpXwN3Qn9R+RrKq0QxYqPujAPoMCmhM11ukjRshGY8k7Ac/T8qjgvF3MixsynnYACP8appauEyw4PPB6D3FWbKWCJmjniWRCeOeVP+NaJkGraXS2txHMkaqQQfLkHB/OtWHxfe6NP9ut5gWm4eORiwYZJ6cd8/meeawIbVnm/cEFOu0sAaNXYFNrKrMoC71ULnHuPvdufepkM9r8N+JrfxRYvPbFbaRMhxL8yp1IyR649P0wTeurLVBsVtQiikYkGNIuQuSM8k+npXgfhnX5/Desx3sab0+7JEWIDL3/xHuBXv0uv2k2gwXdixmfYJEROVde5BONuFzkYzwB2qoNvRENWepA/hu0mtpBeS3k8rOCjtdEAL3G1QBz/ACpLTQ9PsJVms7GKKVAB5oyXznOcknuB0rSguY7u0SRGRlcbvlORSGNyco5UenUfrSce40yrJdSW8ZZRhRkcHAH/ANaq7uX+aIHf3IOR/wDXqy8UgY7lU9+DzRwBkDnPakBWkiM0a/vQuOCFytJEwRhGiHHqTnNLeM5jHlyRxsRj95/Sli3PbHzXUswPIGBj6GgZIfMJwEDA9xxigKQoHAz7UqE7Rncfc08KCPu5H1oAYCAvUHHGc0YDdcU/ycr0PPXHFIke9tpwDn+VAhMdBmprWdfJmgniJhlG11RsHHsRSeUqHDMM+hpzRl4lKQzsjkBWjRsHnjkD3FOwEU2kTW1uZ9NZ7qEDdJE8qbhn05AH0OOvHpVSCVJNwMRRRz6EZ9R1zWpaG5t78h0e0iK/vJHUHd6DaDzyf51PqNlpk80kkAkjkZhuaB8buTk/MpGee4z05p2FczkWOIYB46jcc07zY8biwx0znipZtO0zyCiR3xbk5N26gZ9lxmqyWdhbybl06J3BBDyZdgQRg5J68D8qdhXGS3ttGpzcRjPGN2T7cU0SFozKIbnYOdxhdQR6jIGR9KvC9A3YjhDM24nyUJz3OSDzxULXN1LtU39wxBJ2s/BJ9qoLlGXVfLAC2l03XJFu5xxnJOMdKjefUW2+VZHBPDGVQPryc44649PUVfkLyKUfLxK2dsnIz1yBzVXypoY3lLy3Fuo3ySM3zKSeuDzjrn0yOKVwEFnN5uL7VEVG+7FHGH5z0ByDmi/srOVRFE+oK3G6QtGV/lmlijJy5dSx6MOmKlcZQj9RTEUgba0XyxZRy+rTszE/kQKy206waZpTZoCxyRyR1z35/Wth7Z3BDEY7YHSqxiCMVfrUNIpMgJKP8iBBtKjYoXg9RxUcmSSWOSe9T7dpxxj2qJtoB4yaW2xRVlKqM4qjPdbW2gDnoGbB/AYrRkYZxzu9qjIDAFsZ6jI5FFwIlUlAWxnHQVF5qtdCHa2488irBOW5yRTA4Ukc8H0pAMkCKckZI7jmommka2lt4riSKOQguEPDYPGR0NTOAy8g+1ReTGpyBg98UDKES/YYJ90sIUgsFkyuQMfKDjHfgcd/aqsQ+0je8OFPRlb+hrXljikiZJBnjqO1VJoEIUiNA6niYAhz9efm/wA/iAZcqhAQCTnp6Yqm4JY5B4rXkltpUwxEUg4weN3vVWW0YAFVIHr2NS0UmUdoxnj6UGNWBOOlStCxONvfHBqIx4Pt7GpsVcdbtJFJiMrzwN+MDPueldLpvjP7FaxWs0KNEmf9VlWXJyevWuXeLaud2efxqIrnjHHse9VGTiS4pnc/2To3jeZpYriO0lALNthxIR6kbsHt0/GsjxF4fmsYre1giik2HG6J+HY5O4qehIwOp+6ME1ladqV1ptys9syh09R19j7VsprX9ot5VwsQnfJyclSc9DnJ9Tx/KtI2bu9zNpo5YwXCsVaEgjggqciivRYo/EBhQxTz+XtG39xE3HbkjJ+popa9wvEsaV4f1HWLkyQMsViG5upAQoJPIX+8f/1Eium059C0IMbK1lvbtOl1Kq8t7A/dHt19fWjWdaeZv3W+C2jUbEzhjjHJxwBx0HpXJ6hqDrF+5xgDooxj2rqs3uc/NbRHQ69rt5danZIWBtJbTzNycAyBsODzzjA6+tc9qd15zxpC0gQZ6d/0oVpJ7axkIOyN5kJJ7uqED/xxqI4DlWIxjoKtK2hMt7kUNqXmClzsxkkjH9asyZiUjcDjjk1NCs0k32e2t2nuXOFROT/n3rpLfTIdAjS5vhHcapjckXVIPf8A2mB/z3oc0hKLZV03RLaCGO+1vdHE4zFagEPL7nuFH+e2ZrzVDeBcqqQx8RQx8ImP6+9ZV9fyXUkjuxeV/vOeTn/Paqdq+GKOxCdqycrs1UdBbiYyzliQf5UzzPKC7Tt24xikIYyFsZyementV3StHbVy9xPIbfTISTNcMMbsDkL6nHfkCoNC/wCH0fUpGkCr9niVjcyO2FAwRjI7/wCfSmyXeneHNLOnaSCIG+WSVzl5fqew5PHuar6z4kt44l0nSo/sthGAVVh80h65Y9f881ygmmv2ZmOeccDijcRbvL+W+kCqwWMnBOfzrS02SAWbxyjMhn/djb/CVOfp90e/SqDWghjQY4LZ981BqDlWitYJ9ly8oLHef3YwRyBz/Ef/AK9UiW1Y6STELWyiZVS4DK8u0usRGdoIHJJKnjtx3IBj17WLjQ9M+w26yQCUGXdchkkcEn+6Pm5AAY4HP5VvC2pWsflza3ci1sLFswIdw3yepAycjj8foa5bxZ4hn8RaxJcNIogU4jwMDHA789AKU7x2COquZK3l05Je6nPOcGQnB/Oo/nJyWY/jSouRlVLd/bFSBDuO4AndwBzketRq9zQeECoy9XzjIOanTaijks57egpuzgDIp+3b8zELjqxPSgC5Z2fn3MXk+ZLk5YMgUJgZPIzxx1/TtUkkyT3Ekyxqu44G1QucdzXRRRWmieGGmEgkvbjAiIUoRnvyD0xnpzt6c1zLMjAqiqq+iDp7U3uJDjlcAkDNOB45qMYAFLyGBOR9aQyVsEdcUvyjknFNPGTgcdOaaInYhjuZR/douMmCgHcME1Jkgc9KgVjuJ3cDp0/nUiyDad2OOtCEAcMOD16eppd4UHcDkDPAJ/lSocMcDIzS+ZyedoHfNADPMCkKfl9mzmpFHmHGSPWoklWN97GPaDn533A/nUjaiLiFk+0RLEflAj2qvoOnuRTswKUitcR6eInCsyu0xYgY+Y4GMcgDv15q5GoQHcQTnjBqreW9xYXhhw8yLGoQQfPgHHTHarCoyD5vMfBJBwo+nU9/p/WmyYyTRYjUvn5NxPA+tLNavCW3BSAdpOe9NGs39vIRBpdkpAARmlbdz1JIXr+NV7i41e4cNI9mpxyFiaYn8XYe9RddytR7YDKAvPPOf/rVbh0+8lXf5DBMZ3SYQY+pqncWtysti13fXuLhXYQIqwptBIz8mMgkduMVoQ20AffFFAJgAN7KN2M9M9TTSC45LBnLLHcW0rp99Ipldh+C5qb+y5EKkSEkjKoFYMefQjj8f65q7eXkptQrzqX7ovGefpVSzltA4+1LK2TjduOF49BzTSE2W2uTLC0GJZDGC7hpRGCPybj6CmRKiqJEstrFdwEkbNjgYPJXI98CoryxeAyPCjxxg5DFhnGfSooXleBo3uNsa8hXPU+uKW4Kxp2c1zcTbBdSKo4zDbBBn6Evip7q3uY4DLLczOCCpErBRz2woGaTT5Vs7FpZ7ldrncqoMH/PFU7q++0SB7d5mkXOC+MYp2YiumnxXLyPHBZoDjJlBKscYzg5GeOuK2U04QQKxuHCAciFhGhHvisd5LsPukkYHHO7GfyNTJePAmDJK2RxswuD9eaLALc3wSUxwwMr5H+sdjj6ir/9rXEbjfCuDjowJP4dawJJA7sx+QEHk8/5NVtzrKGhdgw5wBj+VO/cdi7rk142i3pttixQSCcbbhFc8K4ITqVC7Rxznd1Ga2I/EEcvlNAuYnAIYnnBqvGZrtYzLeywTSIYyyQ+ZGM52lgM9CeuPTNcvpRltJZ7OSGZTayFQ1xHtJUklflPTjHA/wDr0c13ZiUbbHci8MyOdspx/dOBj6ircMvmRb1BwR8ueQa5SO6uXuPkkLFz0J610llMTbpHKhjIGAd2Rx/LihrQERFriC5knfDRbcOMY6dCP14/rxUwnEu4kOEU8kqdrD1B6EfSpkGQcMTkn3xTNsSQrH5YWIKcIE4/zz+tSAkkiCNpJkfy15OIy3A56AfyqoyKFdljJHEiBoyAg5wQDjJ9j0NTacl0vmm4nLAnCLtxhfx5znPU1YdSJFlk2Ns5Gc5X6UAZIhUXSRzTNO8a7iWYowyerADHAPr/APWtiwhltkW5iEsgGRg4PHA645wapXzSJNGVdkhbKAK2AGJ+X5RyfwP5VJ5B+WW6mjRFBjPGEcEjOVOSOgHJPH1pgUriN7QBZDJNvJUeYygY6j2zx0/WkstTksJZI5oNiZyQDyOO3rWtJp9kI5HbCRH5mLHC/wD6qymTQ4IpJnuBJHG22TyAX2f7wXJUe5wPend9hWPOPijdJeeJLOeLcFNigGRg/wCskrB0khbkM33O4xnd7fU4rZ+Istlca7ZyaeSYfsoTLKy5YMx7+zCsfSBunwEBYAE5x079fb9M1hLc1jsaGvKRdmQnd50Zbr/tEY9+P89K5S7H73OMZ7eldNqswngikjOVC7CfbCnn8QfrzXN3rZm6DOOfrQyisKQnNLSVBQlOQZdR702pIhmZR70wOq0SHc5IYDIA3McAHkDJ/H9PfNXriJrO1aMEKGUhSTknJzj2/H1/KXRIFWJAc7pJFiyDtPIOBnpjg/iO1N1qRoozGMlxkthcEhgDzx1zntwRnFWScbcYd8gY4Ax6VWfhR71akUYLAd+melVJDyBUAhlFFFBRLbp5lxGmQMsOtaiwokrq/K53KoHXP5+3FZ1mds270Gc+lWGbzZHIYr/X2q47EsZcFEbgjPbBzVYOQ2evtUrIT0GR9OPrVc9eKTYI17S+AwvQAcgmrzy/aU5mYSY+U8HJ/DBx+P4VzQZlOQTxWnYXxVtjnKnqD3ovcLWGXFv5LkbTkdc46/hivWvhVq0F5o0ui3DB57VzNDG6gjyyQePXDE5/3hXmt/HFJEsu3YpG4ZUL7YGCc49faqGnajdaTqEF7ZymOWFw6EH9CPQjg+oJqU7MUkfSF9daFYyBrWQWTuB50BhcIrZxuBI2gcgds8d6uKF+8GJz+VczJY/8JPYxX63kRt7iPfEj24JVTyAx3Hkd8dxWnYWj2UTRy3s1zngbhtC/THP610WjZWMVJ31RqMwBJppXeeFzkZ4FV98ZIO0bgCoPfH1qXzZHJLOxz6sTSsVccEbe0YiKsv8Af+QfmcCqUpaU4WByFYZwdvPtkjP4VY2s3KnjvigqwA9PWlZBqMW3mMeYYogO6vLtIPcY/wAM+xJqMRXflbt9qjZIYBmcp165A5zwRzUpfacd+xqraQ4vpBB95xueMk/vD6r1+YDtxkDjkU0kFywttJ54L3ss0JPSOJYsdeOS3sc+xHfiwwiSPZFbKPlxvd2ZsnqeuM/hUAldHPHy9D7VMHIOC3XkGgBjK2FwqqFXaMKOnoT1NK0sjj55WfHQMSfpT8Zwc5FNZUHOD+FFxDdpOSB8ppu4jGOlNHzjcuQPUilB59fWgYpfk1FOzmM+WwD9eRn+tTlcjheKjK/7P40XEY17DfSzq29/LHAEZK/jxzSOmpQfLGZbjfxkbTs+u4itnAPAPIpM7TnAPqKLsCBfPLIXZV4+ZRUvnGBg6M+4HII6imSuhB6Ae9RRsgGzcUYHoxGTTAivphbxwyxRYlZtskanCn0KjHXPXnn8adBMs6B0IZSAQQeuatBON6ttb+8Dg1lzfZdNSef7RILiTLtGUGxzkfMTkYbkfXH5G+wy8c44zUMgZmOUUr7dahs7wXNoswZCD/d/r6Gle4JBwMfjSugsRy27AkqQ1UpY2ZDtOCasPISOXI+lR5BXJ6juKl+RRCYfkUbiT/OonjO05IxVliNowf0phK91zgelIZXC4wQM8dzUcrrvC8BiOAO9TFPnLDIPfk1FJESQclT0zgUgI2TecnPBBABxTDlCSp688nNSeW+Blsn6YqNk7N355oAgHzSE8gge4BpNmQeTU+1VCrnPtmpEtp5CBHBMzYJAWNiTj045ouMoSL+4ePajhuzLkA9iO4/AiqCvqEUbKUjmSNSxDNhsDqfyrXuYbu2ZBLYXas/3S8JQH8WxVN47m4O14I4l6HzX7fgDQBHLax/3djAYZWzkH39KqNaKrBhLGVzzzWo1rpzlHu5Li4faNyxnywMfwg5ORj2p9tJpVkuYdJR35O6eVnH/AHzkA/jRYLmPJ9kUqrOpc9Aq5NVt9sjMPImLLkFQma2ZLj94WgjWDJyPL4x+PX8M1DNNPJIryuZCoCqXbcQPQE9BQ0K45NFFzYrO0lraggny3cebx/s9PzI61JZ2mjacnnSX91JMyZHlQ7SuR905br64P49apZc5yxI9D2qGT6nBp2QWN4alagAR6lfIg+6pRTgenQ/zNFc7sf0P5UVXOuwuVHcT3ZuFzyeMg44rLmXzGwMlu4P5fjV2PmIL1ZfQ0xbZmm7YPOPxrtOK9i9Gv2bRVV8c6hCoL8f8spun5fzq/pmi3uvXLwWreTbpxLdlcqp9AMjJ+h47++lp/hy3utAD6vKba1Nyk0ZDfO+0HGMc87j0546jnFrUfEkFpp4tLKHyLaNdqIOuO2cdKylLV2/rQ2t3LSyad4Ut3h01BLc9JbiY/M59BjHHXpjH51z+v39vPrFpcK+9LyzyB/CGRiGAxjGMgfhXOTahc388jrIRGVwR6c1JdshstBdsr5MlxAR/eLYbP6frWfmaJdyTDliETatK8ttBOiSzRo7DIDHHApktxLhIrWFp5pGCxxqOW/H/AD1rT+xaT4fK3GtObvU2O7yF5WHHIAzxkZ6n8PSkkDYtvFYxQpfavI0dmTmO3VWMlyeSBgdF474z6gdWa94lmvFNpbR+RbDlYhGoA6YBAzyMfT+dZ+ra1e6jolvqK2UQkivDEGALfI445PcFAM/SsNUmlyzyhCp+Y9cnvT02DXqIbe4nuizHqfXpWzbW6WybiwGMEknoTVOAyIxDOGH4DBqe0hudYuvscI8tI0Msk3AIA4wmSAWzwPf6HDQn2Nm1sJ73zJlh3QwEea+SMH+6OMbvXPAHJ4rT1OwsLTS1v7u5W2hj++pBfdxnZGpOGY4wW7dT6izod7caMzaZbRKlpDGJWebPyk9uANxJyePz6VjavdXN68jyqEhLFzEhKqPfGc+/XrSl5Csjg9b1ZNYv/Mgtxa2sa7IYl6hQTgt2LHvis9LePcZNmfQkZ5/GulmutGgUhoirsMH95jP61nz3+lzSr5lw7dQvm3BbA/Ems7MvUoruHQFT0NKHjALAMxx25/8A1VopNonliCE27yd2jiyR75Aq7H9ieRTI0rIvZkYD8+h/CnqMq6dptzcosjRKoY8KzdvU4zxXT6fprxZS9htUdxvT7OSxCrzn5lHOSPXqPXNZ8t3AsHl2z+USuUwOM+5HTv05qy+p262bia7kvbpl2NvGxOOm3aemSeue/XjFW6iZR1vUm1G6Fv8AMLW3AVFLAgkgE5GB68fU9c5OYAFUrGBgcYpklvcKo8mWGQnJIk3YXkngDuc1OqkJ92CNj99kQ5Y/Uk/yqGxrYYd2cd+tAO1w7BGUc4an+TEUYSmaXJwNsgTHIOPlGf1rT0fTrFryBLyzkmV5AgZ5ycZyfu9Tx6ntVJA2Yt3qFnCNoaOMZ5JbJqsddtgywAyOz8KoUnd6AAda7nxJcaVoyi2tNGsjJJ821I0CoDzz7kehHXHYiual1e7Ik3GKDzBykCLHnoecdeQO56VKuNMzI9TlkyYNPnZgcZlxGoOf9ojP0xTvtWqSgMzafADkFcszL9eAD+dRPelW2SR7RnCqGBLf4CrS2luSsvkpv4OSMkUajFjtZmKma7mYZBKxqsYYDsep556GnyaXBcIfNM7ITko8zY+mM4xVkNt5JGfrSrICuRj60CK6abZKgUWkH0CDGKfMZPOZ4I0QNkvjC5JzzwOakbf5RKAFu1NiidEwTk9SaAsSpJKwUGTGMZ2jGf50N1ZzIzH0yOajaRlRCmMN1yCCPwpZ0jcruUF1U9+v1HehsBiTEqXbAHAyoP8An8adHIHnTfII0YYUEHuep79Pb86hJQ4WYMGPbJ4pRdRSMVUksuOPT60Abl227SNLUxxeYA+CrZYKMcnPTO7oCfwqCCRVdPNMoUn5xHjOO2CT6+1WxotvFpmnXjTojzW5/iPzHg45OARznHoapYHHT2qnuSiSWRC5ESt5f8Jf7w/Lip4onDbReJEuM53AZ4+tQA5UjgZ9TikSNncKAME8t6UAG8iQyCeR3PJyOPzzTfnLEsMnuSealntmiOAy4H8WRVUD5utFxlqFDI4RAWZjgKO9dDa6Q0HzlxJIFPyBNwzXNRu3zFQGXHzAjPFdI/iC2S0xbiQvtwCV747nvQ7taCNK1gQQZYH5zyHQLg+mMUk8cDfKWjRiOAQv9RXNJrFzMw3sx2DcTjGTznoOB/jRNqoYjaqxkdSVBz9aOVMRduLVpN0KCJm4/eBMBM/7QI/lWXeWM1gDIZkBIK/Lzn25H8qZJqsio8jXarGSM4woB/Cs+XVbBgfMvC4H3fLXcSfQetHLYaOp8OzkQq0Lt9oU5B3bQvB4J4xwSc1F4gsZ4ruGae4Mk90ShErMzJtydvccc9xyw44JrIj1i2tJVktra7kY4BiWM8jPU56/jxxxV3Vb6XVtMm2CK3u18ua3d2DEMuMAdl4HOMdgetFtLhdGbudQD0PoK07K6v5IxFBB907hIsecH3J4qGK51u4C7ZdFhkIywIeQqMdzjGevenxs0wCzeJD8rASLZ7FGOvyjYTnHuaav/SAupd6pJK0bxurDsqgADsaHvGhjAvbu2iGCA8kxDN+RGahnsPD93HGJxLcMimM+bcSFQOzALgE+3H5dKcdpoWn+d9l0pHklAy0537R/s7skfnngHqBT16onQnTxTpkEahtV8yTJAihjLDgdMqD296c/iwO7tBbTuETKloihY4J+6fmIwOoBx1OBU+maisVxsW0tlBQoPLiVCV/u9uD71r3otE0t5ZIFjRUBdVGNoBBOMemARj0FK+u39fgO2mhxH/CUawLhkFhbKW58xMliB2xgcgcY9vSiytvFWvyhDqC29qrgyOIwOAQQMDGenTPfnsamT+xLXxFaQxtLcRl875SGUEqxxxx2H+enTXOt28KfZ7OOMpjAKcAfhiqnfaOhEU95FCTRdFso4jNHLfSAcy3sjTFz/usdv6VnW10iRmCKNLYhixiVQgPPB2jA6YGcdqe9xJNK7zODk8AZzUDtZSOjPIodTtQ7wOT1qUrFnF/EJSb+xlzktGwzn0I/xrD0kt5xKE5PAx/P8K6H4gwbE0yXnDGUBm6kfL+mc4/GsHQxvu9p78dccf1/zngVhLcuGxLdK0U1zE7gSYLAg9SDkgdsAZrnLk5kyeD3rr9fjjR7e4jOzdvjZc8DOVI/X+dcbKctz1pMtDKKKKRQlTWv/H1H9ahq3psfnalbxjq7hfzoA7u0tv8AR4ox1hKNx1Zhz6f7WB2GRngVna84+0OwYHoCQO44z9cjOD6+9b+n7pFVZF3CUmcsOWQHgH64xx1/DmuY1f8AcyyIOp5KkY45GMd/TPsapiMC6GHxjt2PFVDyxqeQ5fJI55qFVIb8DUsSI6KWigofGuc4q+iskLKvJPJrPRyh4q20+Ywp+UgdquLJZA5YNzmozk09mz1PNNyOnFSwRHilBKnIpSMc9qbSKNfT9QVIZIXLgsMABiF+vBHP+P4VRnkVpSVBAJ7kn+dVx164qeCCW8u44IULyyMERVGSSegpCZ758PwR4F0wng7X6f8AXRq6TZz061VtbdNO0+2s4Sxit40iQseSFAAz74FWQ7NkBTW60RiJ5agnj64p64HT88UmeAKjL5yPfBAoAkVvnJBBGOKSSRQD3PfFQqwzlFzg454NPBDDcCVz1zQMRwGA7HtzUJt8kZ5AzgHnrVhghGSyj8ajBj3YTuOvY0IRPawtdbljVVmjTLRqBiQDuB6+vr7cUnzfcYYI/SomjOQMgMDkexpyKVtUZ5XkbcRIz4yGJO0cADnnA9qYD1dgSrYwKUyKBk1HwcAZJA608oNwYL1HegBomJyFiPBxnIqBpZvNCCDCsCd2enPtVhs9M8CoSG5x16HPegBI5pSxWRe3GP8AHNSod33R/wDWqIBj1P0wetN3/NtyAe4zmgCUnnj8qjO7PQn6UhlVQdxyPaod4IZQ2O3HelcLDpJUTlgoIPAY96gTyUbcqgZPQAD+mac6I0YMmM+o61EQGcbSFUc/N3+lLmHYma6HIAx+NZsrtPc7IiUOOXUgHPPB9akvLiFFKsVK+5H+NYzXVpFKC19bQjr5R5x+tPUdjWPlBoosJDcjanmRgLHJ0yZB/Dxk5A59KgjuvMeRGQo6noSOfX/9VZ8utQTstvBDLcO+FVUiJ3EnAxgHqeg96aGiv18iDTLu2kDjddBSAFOMhhnkDgjgHgjvU3WwzW6PjAOelOAxztJPoayPP11olSK1tywJG5shfqP8ioG03xDclluNXiiXPS3TrQ7LcRulovLLlk/DpWdPq9jArGeeKIg9Cw5/Ko5fDFjOIzObmZ1QBzJOW3N3PsOelWYdH0+1A8iytowO/lgn8zz/AJ+tK/kMy5PENqJtsSTSkjI2Rscg4x2p6ahqFxLHHDo06mQAh52EagepzW2sUUSllVVUegAo2LjkZBHUii7AwVXVXZt6fMp/1cbLg/RjgZ/Qe9XBaxWepf6ZeSzo0WUhZQFBOOCw6kenuKu+cQvlohHbk9asWd9Lp7ySRrHKrDbLE4yGWjUGN/tprRcWlja2zcEvGpyfrk8/jVObV9RuCVa6k2t1CnaP0qaSGS5udtnZs8Z+ZwG/1Y64xyTj16ew71rm0ltiC4ADDKt2YeoPei9tEBUnd5c7zuz1z3qIqQOT+dTMTwGHA74phB7Hine4yrIme2PpULocd+vFXdncU11JBH5UriKJDGk5HBGPw61YYFeMf/WqMgj3+tADcrkFf1ocbw2QPypGUkk8ZpjE7eO3pTCwnlf7v6UUb2/vn86KQaHTQkFmKkMA20KPvHPYDqTXQ22k+Qy3F+iB1UPFZnnOTwZSDgDI+7+ueK2TZaJ4XBktokN4wxulYu//AAIk8f7v54HFcxfak927oHESOcl8dSa6nNvY5lFItazrtzMASUeReCcbVT6DJrkdSuTJatMkhyeWyg/GrQZZcQyurlVJPHB98His2/doHSBOYieAwA4qHoWhLGcYEcfzMQCcdTXTHR59c0qxht2VHt73zCWBwEKYboOvTrgH1FZ+haK06l42ihtoot013MAFT/E/4+9alzqkNpss7In7MqZ8wD55cjksfr2qrdGNvsR3Go2vhaOSw0oyTXzgia8k5bJz8sfHAHr/APWrHstHmuy019dMrSNkK4yT+VaVhaSXmoC8uFG1eYwTkrn09ulbbpEATvVD13MOlS7iWm5VlsbaLw1qNvgqkYSbJJyCJB0/P36+tYFrZiACUOwDc8jr710V6JpdG1WBH/eGykKHGeVwwwOvUCue0a0vL7R2j1WWO2kUeT5RciTPOC3ytgccjv7ZoS1C5WvknuJkitxKkJXdJOisQ+MkhSPp1z171oWt8NGgb7LD5Vu6ghHH+sGSfmyTxgnjI4xkE0zWL0xw21pKFWzjGDFG2PlGeoKg/Mfbufeua1HUJr6Qsz49ABwKHLlWg0rmyfEl4sjmORA7/ecxhm4xjrnoAAPToOKzDPLIwLsJx2NwiyfluB9f8mqVvGGI3sxb6VpLb+SrEk5xkZwKjmZpaxKNX1ARCNLyeNfRHKD8hioTLPLtMssj7QANzk4A6DmouM9RUqjcQowM8cnApb7isJtfdgPkHqCP/r07ZxyT9fWkU4B5qQMMAHH5UWARFGDnd9CafgEk8ijPc0Bgc0ASeZuVV4wPaoGgidwzAkg5+8R/Knjnml2knAHP50DLVpJFFtVIishYAFRgDn0rYVJovtd8djWVvHgs0wTDYydp7sOAAeOo5zWRpttDdSvFK+HCkIpfbljwv0ySACa2fFVnHo2jrZhYzLdzZYs4Zxn5jymQAeeCfpmqTsrkNdDj7XZPgtNgNks0k7bhnHH4DA6jpUd3Zp5wX7RBOzL0RWYgA9cjj6cGrQABAbA4/TvTTe26TCIuQcZAxx+fSp3LI49JgBVny5HPzAFs+5q5gAgLjaBzx0+lMa8t1UOZVVfUniokv7aZsQyI56Yzigdi0MdR6UuMjIyfb1qNEIyxJ57ZpxI+UswUnjrQIkG7b8ufzpyngZ/SokYsGGGGO3rUrFWJZQ2zPAJyfzoGRyysrFQpYYz8o5/WkjuYpeRkHoAeopruuD8xGDgk+1VpL2zgcxvIkchJONwycf56UgLj24kjUAvkDHXmkLGAoiRkc4IVQfzqazs5mtmmLmNPlx9qIids+inDMPcA/Wrcy2kMEtndQLNK0ZKXNvOCsTYOBn145ABHIp2fQVxtxqJlTTYtsOTak7wgLNynVzyeQe/0xg5YCw3OFJx9OaqztMj2ptHg8mGIwN5qdi27AwOTwOWJOc/Sqb2Nw6Ju1KRTuL5iVV4PY9cYwcfX6U5P3mJbG8JBOEiSAiTGcrzn/wCvUkVpO8oUNtJ4CFgP/r5/zisBdOgMJjmub+VepWSc4znrxzV7TgNJSVdPZ7cSkeYUkYlsZ6knPc/nRcLFiTalzHau0yTspcp5D52gZJxj0FQyXcAUlbfUpGGdm22wG9zuwQD6/pU0l6+D5s/TqXbFRC4jcf61SvsaLMBbaa6uJY4rXRbl3bO0vdRx7jg8Ac5J6DpyRV2TSdRiCtczWOmJhQ8UrPIyglQSNpUEgZO0Z6YOM5Geuo2Bjl/0lTtXaTnJ+g9ap6xqjz3Km4kS1doVA8wgZBJYHnnPzd+cAVVu4r6lm+0tHt9n9v3Ew3ZbbZJGCvoGDbs474/CqX/CP6e00habUHhJzGrTAEDPG5tvzcccYp39s6bGI4Y7lZS2FUIC2fyqOXVJhzBpt/Mh6MsRxUuy/wCHKRdTS9JhMbR6ehaInbJI7OefXJx+lXI53hC/ZyIBjH7pQhP1x1rOL6m20RaY3IyTJOi49uvWpII9RmukjnFpbQZ+aTc0hXjPAAGe1K6EWmeR2IZmOfWoIx5Evls7ndkjPYd6hitNSVgGurZIj2jQk/rgVYk0GTUZZN9xmAIeTFyGxxnHbOPw9OtO+lxX6CRT2ssUkLBXCy8s/XjtjOMe3epZJ44NqvKsSgYAJCgfhUEPhC1F6gtb2WKVoycgYXAYqTzkjoOMN1610mn+BdFt2hnmS4uJ0If/AEmbeAc55UfKR7YpOXkNJHNHUrGJWY3MeB2zkk+3r+FQpr1tK7rBBd3TopYrDCSQB1J9B713g8M6DZwZXS7MbRkM0IY/mas2f2S2Ty4PLjxwVQbfz9aE5PZBoef2uoanNJvTw7qjJ1UojD/x4rj9K3ry41u+0ia2TQLuAyRkCSS5i4P4kH9K0bm6vE1AlbyLYTlIl5Yj0I/A81qwNNPbI86gP3Udj9ad5W6fj/mLQ8z1Pwxf6dFZX6GK3miK7zNKiID0wOcn/A/jWvaeFNcAfzL6ztEIO11DTuc9jnaBzkgjn65zXRPqfmu0UZhQxHgv6DrioPOe/b57rbD1CqMlvwFHLLuFzKg+H6yr/pmvXs6Z6W8axD17ls1ei8A6BbMZJ576X186625/FAtalm4iTZskLA/KWGP8muK1X4h2kE8sFqklyyyEGQNtQjHY8k+napaXVv7xpt7GZ8TNH0vR7LTU0y1WHzZHaTDuxbAGMlifU1yeiSRxXRaTlMYYex4z+Zq14p8QSa1DbLJF5flEkfNuOT71l2C5DOSoAz+HH/16zdr6Fxv1N7Xds9hFPEhDmTLAjp0P58g/r61xk4KzMp7HFd9YwDUdPRX+QAgnGewP4/j9PQVxmtRNDq1wrKFJbdgDA5GeBQyluUKKSlpDErQ0VWk1e3VcglsZHbPFZ9dD4OgE2smQqzCKMttUZye1CBnXTSQi8CgkRr0CnlRtGQPUD16fpXJatdGS6mZjvyx+bufeuxkWG2jMvmuXdWyzgBWOW7HqOMdun1x5/fEGXIPbn1/H3psnoU85apEjzknsCKZEoaTB7g4+uKtqAsYPqOaliKDAAA9/SmtViROpAzjuKrGmUhKXNJS0DFz74pQxzxz7UgxjoD+PNOVN33AxbHYd6BCklW5XkdiKjJyc+tSCdl7KfqoNMdzIxYhQT/dAA/IUDErvvhZorX2uTaixQR2KjG7rvbOCOMcAH8xXDW1tLd3EcEKF5ZGCIoGSSTgCvovw5osfhvw9bWEab3UbpnTH7xzyxzxn0GewA7VUVdkTfQ0Jmc+UY3SRwwYopG4j2yasfMrEAMAe4NC4yp24Pp3o5bnI57jrWhmM3dlJ+tQpMJJZFMUqsh+8ycH6HvVgjcMg98cGmrERklQD9etAEDQqzbxuVjxnbSs+1lVYZXz3Vf55qz7BfwpBgc9/rQAhUeWRtGTzj/8AVVfy7kMADFgAbhgjP0/yasEjGSefQHpUX2mNpdizK5/uIct+lADmQEdefQ1E0KeYplVZUznbwCPpkH9f5VB/bdlISsAlnYZGIYmfB98Diq/9o3twpjh0m4D9N8n7pV9zuwfyBo23YFm3vlW+aymYq23fDubJZeP4sDJGeeB+vFwxsON5H90+lYv9kyNYbZrp/tYO+Nw7EK+T2z6Eg8dKqSWviK5bM2oWVtHnKqkbufxJ203KKGkzoFmCKfMKhgcEZzVWbUreLH7zBI6jpWU2gtIS13qlzMe+yNU+uOCQPxqWPQdNh5MRlPq7E/pS5l2CxM2tW4cgzrwBgA5NU38QRyyFbaG4umxkrAm/Axkk1aW1tYgTDaRZPUrGP609jHLGVZlMZwCnQD9am77DM97/AFNxF5WjTuHHQyIo+h54qKN9a86KN9PW1jmlWItHJ50ikng7Rwc5xgH8u+zvIXIPB7jtTHuYli3TMrRqQd6gkg/h05o1AwjYapLPLB/bvlyo22SNYOE45DbsOCDx0+hNLH4dmnidb3VbyVyPlWGUKoJAxnj68YP17l95bx2DRyWzl0LAvlgwBYjG4k5A68j8jk1oxSNJbjy4pI5NqtskBXg9Dz1HBwelNvswuQaboek2KCRrVZplyplucuTkYxgnGME9vXPtbkijSSGWBBb+ShSJbVmjCqSSQuD0JJNRwSXK3PlXPIYHy2AGGxzzj29qie5P9oJGzbVXqhBy2cYP0qbIZdXUbyWQM7SrtGAzSk8elQwzvLksAmOPrTY5vNjICOGBPDKRn3pdgkYZjO7qCaYiV48vubJH93tTXdIhyu0deFJH6UmGT1alaPcVZgQV5BBpAIrLKgbBwehIxTGXseaRfkYhndsnILY4qQ5VhwDQBFgkY7Coz1GeKsMc89PaoA6MfXtQAzoeOtV5oYZ3KyD58ZyDgirTorLjb8pGDmq/lqnRQABQMXLIN4Ztw6MpwR+IquuyNJbkrJdbuSQ/Jz1Lf3iCO2Dyck4wbCj5TtJOeearyI8bmWEDJPzLj71MDSfQp3sfttr/AKVbf3lVlbjII2kA8Ee/FYrptB6AVr6Trctk42MrxdHgkJAPPJHp9RTr+y0678640q48icK0strcZ2nuSrHgfTPp9KSQzA44700qcZU/hUy4mhVlYZwCR/Uex61E4wev5UXAY3KkEfnUTp09OxqQqT1AJHSgjI56UAQlQQQDx7CmPHnryKm8rPIxiowTggH8aAIfK/2RRU24dw1FIDW1XWWlbGOPfkU2MGaMlwwUjJHpWRNPuwvO9RyR0zU6SLbrCjvvZvlCg/NnjGAOv/1xXSuxg7WJbyVIWARyGjO75Tjd7c9q3LDQGeBdV1gNb2OPkgYESS98Adun1qzY6baeHrI6vq0Jnvj80NuwP7vnILHsehyenQe9G61+SfUXuZ75mDhVMeOFBAIAHYc/jg07dBE2pay726RLbxW+lRAGG3VRzn+9z6847nk8gAVbSyfUbqGabaqEf6teAQMcf59axfLn1TUGZnBjUgDHTj29q7G2jj0+BA/zSE449T/kUtx7K5Ngo4UDYn8PGM++aHfYy4UuQwzx0z0/pVS4vYftHl7iHDKOE5LHoo9T/n1rVitwjedczRBllwIVkBKjBKhvUnHPGBkemaaVkTuV5IPsk8N/dgIxI8ohGZoQRncRwM9OPwpl3LDa6fNKphZo418uTyNhlJ/iPBJYkkYPr6Vp3OsxpDB9qs2t1jYoWDM0nbAAA564ycfTisG5ke4ulvEZkjA/dxu3XryeOvP4Ck9h9Tn9TXUJUZp2nYykks5J47Lz6f57559kEXlqsmWLgfNzx9a6jVXcQtJKw4HKq+7/AArnptkihXIiVSGB6c9etZPc0RKkgjQFBtYdWxzSb2frn6k9aj84ZIB3ZOc4O38DipU3EhUUMT02/MTTGKsbsc8AfWpNv0zUe5ujZX60pk2xkZHucUAOUEcdKk3DGCvPvVRZow3zTIpHUMwqR7q3iC77iFAem9wKALO4E8Um9c4pI7eQwl1huZ16+ZFA8inPuikVImnXNxKogtpZOAfmAjx+EhU/kKV0A1WDj19DTcSg4Gw9exx7VotoOpxxb3hgiVepmlA/9BDVWSC7eUpm3ZRwWQk/qVH8qXMBtaFFb3pjEUcguogWndBkGP5sHGcdSBz71meJ9QhuNfMMEU0CQIAVkIxk9wAOBgCrOnS/2SzuZBvdgxK4faRnGBjrknnryenbPu7OOa9uNQuJbsmeTzHIUgOT74J5qpPawLuUDukSSNtrc5BAGSKgubONmZ5JSd+Sc7ck+pyDW7pdra2ssdrq2mORcMfLuJnIHHJDAnC4H5/XityKbw3ZiSCAWsmR86wwmUE9cHAPp0JqGn0GcJaxWQIjBe7fnEapvIz6BRWnDpGocPaaWYl55lXyz+TYz+ddLL4gt1Vobc3CgfdjW3aIfXLYH9fzrEm1CWQhjZvJwMBrkDHHcYpqLYNpFN7TUlPzPbxk8YaRSB/3xuNNFreO5aS8tk2gsCquwX8OM1ellaR4vKtbdEP390zMQPXHHvxn8aannhmRmtvJkXDhUYk5HIyW6decZ9hRZhdGS8N4J3gmv2jbcQoW3CEgdyGz2/pWtZS/2eiFFjkkUcy3A83d77Gyg/BR+fNUf+Efglk82V7iRwMZEm3+VTQ6La2xz5Bkb1lYv/OmhXLNpqM+mvK9lePaySj948UhRm+pHOOaZcatLdsXuL2W4ZR1eQucE/j3p62ke0p9mg2nn/VL/hUrK4zls568/jT6WAzhfxySFVWdmHXEL8fpUqu5YBbWc5PVlC/zI9KssGB5P69aijU+YXYhUHTmjUB8dpqlxgWmkXFyWOMRMhPTPTdUMcd87q7xQxIRkrJJ8/r2BH+TXa2cltpXh8XEjos0sJMUU8BILOSu5WIwSBjgHucjpXMqVSJY4ydiKBksSTgetDTTBFb7Hcee7faoljz8qeQWOPruHP4Vf082dlbPHJG12ZFO4yFcc+nykj8DVZW3nGST605fMQEAjBGDxR6gMm07TnbKW4hXGNqHOff5sn+n0qs2iaY67DaFlxjDSvg/hnGfwq7glemCfelDcYalZMCC3020tSJILS3jZcMrbAWBz1BPOeanaXzFWNzvVM7VboM9abuI6dKXcOvB96LK1gH+e4j2jIHYgnj2FIrYJJyT65ppK9O/al5zgkbvrTXkA7dk5ViqjqOOaY5Yrnr9KMccitC2tbi9tPJt1Ow58wsgCj0+bPP6UBsUM4x0yK07C98pWWSUIjH5gqZZvxAzVSWwaCIs8kZYNt2q+Tx3p8VoZERw8SndkZkwfpjNCYaGjBafbbiN1DhijMwAAABkcjI9flAyck49AK17iPyrceZMsbY+Z88KMc9artDFpAW6aVfMkgXcvmcAjceh55LE8evTucHVNZk1BIoACsakM/T5zj09O/4CmJFm41SUwNBDcyhBjbKSFI+mOg+uaq6fqE1pOxYl1UffcE5+lWbfTVjjEl2HK4JbYOEwM8n+lZshUM4Rj1+XPahSCxrxX6T3bG5SI7wB8wzx+X86t3l8bYMibWwAFRCDn8ulYMM8ibtkhAPcCrttHcSOsoUyIP8Abxn609BE1vbtcWyu8CRxgZ3twV+o4PX+lSxZjmDQxNOoIy7SgsvHYdB+HXPJrRitcESPsDcYRSeKbLFZzIzRpEUVjvKDjPvjvUhYgv0gSyuYYtQFu8kTKJAdzoSMZBznIzmvny6hnsbuS3nDRyIcEEGvdJIrFl2/u0CscMqnOPfgf17c1ymu+FV14K6zpBLEGCuyFlYdgxHIGQOecZPBqZxuVGVmeYs29MknrV7T0co5RguQRx1/zyKZq2k3WiXL2d2E8wNwUJKsvZge4Of8cEYqXTI2lKIpOSenp7/yFZGid9TVhvmtvkOViznHBI7/AORWP4huIbyW3uot25k2SZHVh3/X9Kn1Fbi3l2nknKkAd+c/z/lWWxLWskTcsp3L7Y6j8ufwpgUhSmgqQASCAenHWkpFBXc+GNPNpp5kmVg1xtkZcgEp/COfXOff8642zjSW7iWXPllstj0HJru7K8a4ldlAGP4AAAPYAY7HH8uvDQmT6lM/75z/AHGTGSD0DdenfHPXt6VwUzlnY5yCc9a7G6kGy4lYsMRsF2gEDGcDPTqQD/nPGyhQWABGD3oYhkf+uUdjVhyfKKnn39apqxSVWHY1ZMmeBUsCJyFUDvUJ554P0qSQ85AOPX1qM8nj8qY0NIxRTiOBTaBi9KmR/LB3Icnoytj/APXzio0DE7VXcW4Axzn2pMkDGeDzigBZXaRyzOWJ5LHqaZRV/R7D+1NYtLLcEE0oQsTgAdzmgTdlc9F+FvhaTzk8QXOUjTclsndyRgufbBIHrz6c+quVAOWwPpUGnLpllYwW1vi4WFFRf35YYAwOFI9KsXElrd2wje0iRex8tgx/E81so2Mb9RpmiyCzj2yaIPMuJMWytIDkllUkdu/QdR+YqOKWxt5VEXkxTKuBwFbA5+vvn8asec8hGWLZ6FmJp2Az7rUJrafyfsF48nYRxEqef733f1qD7brDfd0fAxxvnQH8eTWuyeZkb2Qj0HX86rwNEN6wyFyHIfMm7afT2+lGoGLeXGvSuIrK2t5FZfnJdk2HsckDP4DA461Nb6ZrMksbXmpwxRrjfHbQ5LD03N0/KtpobS5WMXkjQvGR5E8SksuT8ynB5B9CDjnHNVrSeSVCGjmikXgrMhU/rjIPXOP60O4kXgYxCkfkQNtGCzQqWJ9SSKga6lEflBmVF6KMgD6DoKbucvjkYPXsaDETzgk0h3Ii7MuGdjjjGTUYXggu359KeUZetIMYqSiIRsG3GVmGeA4HH5AU8HqDzmnEDPemlSfSgAcnbz+tNJOeRxQyHp6U05Xjg0gGkCkKIQvmKpxzyM4o53hsZNNB3diDmgA8mE8+UgJ5+6BTJI1fIKgHgFl4I981IRzjFBUKc46dqAIZkJiVJMyx5weOf8/QVQe0QXAaGRY59x/ehWLMOflcbgCMkcgZHrWnISVOOpqnHJDdH7PN+7uU5C5+YY4yD3oQC6lpurRi1eWNrdJCP3sUgZWHfBHTj1pLiKMymRUAkYYVuw5yODx+NX7HVpdPEkFw7XNg4KvF5ZJLc/dHY9Pb6dap3V7aR3IWMypC4+RphjPt9fY4NNJhcZCr+Sh80knklv8APFTbcPwTgjpWHJqj5lWztpJEDfMykDv2FWrLVYb1yih42Xkq64/lRYZqE7e2aq3geSH9yxzkdD271KQHPc49DQQqDAUAdcCgCvaM8YKOxyvt1FTGQE4EZ/qaPM25OMAdaasqsu5S2D6CkISRJBjaMA+vOKYVkDEtt2nviphI2eeR9aawL/Kfmz64oAr+ZubagDEDOM0hyF3EEHuPSlhtRBKzqXAPrzUrAA9CfpTGVPNXblgVHHLY5pW5AI5PpipJo0ZSGVSPRhxVQy9AVPHBAGB/+qkAkttFOQ+zDjo+OQf60Q+YigTZEiNkOBj6EVIshxgsPfAqGe5VQPMkVVPAywFFgL2pazd3+npBcW0Fw0QASZgRIB3BIOcfTHTuMg599BCqJNFOTHt3OGyuzkDqwBI59PXgVA97GLV7hVaSFG2Fo1LYPpx3ptlf3UkimC0dIz/FN8qHv2+lPTqA1Ru/gYe5pNueMVZ1G3kkuUNnOsMO35wq985wPQc4/CopbaE7wk8xRxtJcgkdOgxx0/WiwFd8JyWA78mqz39ohzJPHwcYDA81s5smsY4H06zd1GGm8shm9yAQO3pVaWcLbwxJGDFDu8uNUGELcnH1P86QyIRlgCLG/IPIIt2IP6UVOuu6kihFEiqowACOBRSs+4rsxbVgt1E0ylmZsIo7nt+tehxQ6T4Lhik1BN+qOoYbU3+Vxj5c8Zxnnp+goorrgkzGW9jM8V3Ti/t5CuLe9tYySMFuS3Jz161zT+HZmuAFlCoQCctk4oorNrX5jirHSWVtHZ2qFG2ovGR3rTttPfU50ikleFZMbtnLsM/dBzgZ6Z7Zooq7E7sdrUum6QYnggg+0lmUOY8EFSMopC54P8XGe2KzNa8ST2emJbwxfvhE85JO4r2JyffPHPtjuUVdko3Ivd2ONi8RvLfRwSSMtrKNwQoq5JJyvy8dSctjJ56ZNdN/bsEcJVlIb+EuxOfUkgZoopfErs0ta1jN1TVYpo1mZIiu3dGrlsDnAIGDzn6Vjz2WrXV/GsBivGdN4RWICLz/AHse57miiuao+V6DvZXNWLQNZKEvaKgBwS06gZHXpmnxeG9UlMju9nHEDziR2YA9+n+fSiiiPvK4lNskXw88SMk2oNKUYK7JHt2nPTr7dcH6VfsdJtltDMs77mj2seRnIwRkc8jOe3bnNFFawincpizWWk20RMNgt/MCAys7om0EjJ3MfbgDB/DFVlutTtrYQ6bptlYoOMR3H8/3fJooqZwSegJlX7Vq0rbrq5tCvXA3sf5KKuxDWboJFZ6oqSMxBVLVV7f3ixP69qKKHdLcoZeaDrsEAW41q5l8x+Q0hYAenNVo9Gm3Bhe3aLuKyL5wAb/vlR7/AJ0UVmm97ibY6C1ltrRgL24mjwRulkJPp/8Aq4qe9S0iljjS1ha4jJMjsith++PlGOg45wc4PeiitF8NwcUU5nnnbMjADO4cDr68f55qNg5xl/zoooWrAf5T7SzSHA71JawrcSSIs6LsI3blLHrj29KKKaVxkv2Yo45woP0z796uJaRocgk55yaKKFqIVdiSHOdoPAPOaa7p/eOD2xRRTsBCXAOVzjp1pjHLdfpRRU9bDGNknnBFOiUAMFGW7D1oooQM3NX1e5vtNsrF40jto41kjIT5nHzIuW3H0bIwOTWLjap2k5ooqVLm1EmOTjp2HWnknaCDRRVDGbzkdMfSn9VJwKKKEgGjpuJC46e9CfPyOmeKKKlIY4qUYjgkU4huORRRTQnuIwOBkc0DHTpx+dFFCGG1h2GM9c10+naan2RXZwxYHkLggHsDRRS6CZj69fXU8tus0gMbx744xGoEYPGAep6d8UzSorB7WS5nZpEi6qM4H19aKKuS0bJWxp65bXiWrGGWJLEFcxqCG44x6YyBXOg8AD6c0UVBfQcpJfAPPYV0+maeIIVkliUSZyCDkiiii5LLxQyHLZXDZHOf/wBVQRbZpHZppVWM7QN2QfcgiiiqWoi4tvGRhR8hHaq76ZvR0/eqrHjlQox7D6Ciis27MZ5d8U9HSwudLmWRnacSqQccBduP/QjWTp+mGCOJkA3BVYkjg5GR/MHPbHaiioe5pFWRqQxRsZDOqlAAYyfmKjJ5AI7Ae3T6Vx3iBYlumMChU6AgYyCAefzoopsaKlyhOiafPjC75Yc+pXa3/s4rOooqRonteJS3oDj8eK7HTbQwWbtcoMS52j72duP6kflRRTQMZqCBdNceYfMAyOvI7H/J/rXKyfKMd6KKGIryfeqW3JBd+uxCfz4/qDRRSY+gh49QTTSwA24/GiikhjGUg80BCcccE4oopgX7Nca9bLGMA3KbQPQsMVWkAWFcZBJOR1/z/wDXoopE9Rfs2NO+1M2N0vlouOuBlj+GV/M+ldx8JLGG58TT3EqBzb25aMHsxYDP5Z/OiiqjuKTume0TRLNE0cg3KRyM02HcilXffg8Erg/j2P6UUVsjMkznIxkHg1G8VvIhjkgjZT1VkBBoop2ESRRpEfLiiVFH91QAKry28Vv5lzBFFHKeZGC43geuO/vRRS6gTOY5LfbtGzHTFUL24uYgl7I8s8qHZKHwFEQ5DZznI57c8+1FFDdgZYt7j7QrFk2sjlDnnPTkfmKmBPTGaKKVwGyqHHQ8VBuGcd6KKTKTF3Zximl+dtFFSNiLyenenHBJ4oopiRGdqngdPamsvzBj0oooYxANp4zTMAsFPJ60UUAIFAJxz7VUuLWG/jKTxBkzjBAPINFFIAjaWHMM7h0AyjgYOPQj1+lVdRVJ4njKozKpwSoJHIJKkj5TgDBHPFFFCBFW6tpLS/Gn6lCj7kMkNxkEyKDwSMcH1HtVDVLdLeSSdBKQrAhi+Gwfpj3oopoaK2narZ2+pSTyxFGfncn8XrkDA6V1N5dLBHGQpbf90D/PFFFMGNDBG3AkE9vXP/16lEIx82CSKKKkSI8EHtjrSNlBw3X1oooGVmvFWQozhSMkgA/TNUf7ft2n8mEzTys4RUUAZJ6DJxRRWjSUXIaHA6k8EpitII3jbZ5byc/oMY5PfsfxfZW2pNeRtfG0W1/iigB3H8SOPyoorDmbSJua2+yhcrHZREMMEy5fP4E4/Ss+W3tSZf8ARLcCRt7ARjBP0xRRVWVihm6KFCsKKg/uquB+lQu2Dz1oop2AiYg85I+lNJI7ceuaKKBDDz68+9MOQCM8d/WiigZmy63psUrxvc4dGKsNjcEfhRRRSuOx/9k=" - } - ] - } - ] + "outputs": [] } ] } \ No newline at end of file diff --git a/samples/js/node/optical_character_recognition/README.md b/samples/js/node/optical_character_recognition/README.md index 130566ca0bcd4c..c397bb3c846db3 100644 --- a/samples/js/node/optical_character_recognition/README.md +++ b/samples/js/node/optical_character_recognition/README.md @@ -1,6 +1,10 @@ # Optical Character Recognition Node.js Sample -Run: +Run sample: ```bash -node hello_reshape_ssd.js *path_to_detection_model_file* *path_to_recognition_model_file* *path_to_img* AUTO -``` \ No newline at end of file +node optical-character-recognition.js ../../assets/models/horizontal-text-detection-0001.xml ../../assets/models/text-recognition-resnet-fc.xml ../../assets/images/intel_rnb.jpg AUTO +``` +Where: +```bash +node optical-character-recognition.js *path_to_detection_model_file* *path_to_recognition_model_file* *path_to_img* *device* +``` diff --git a/samples/js/node/optical_character_recognition/optical-character-recognition.js b/samples/js/node/optical_character_recognition/optical-character-recognition.js index 5e371c1975a993..a5665afc67fa84 100644 --- a/samples/js/node/optical_character_recognition/optical-character-recognition.js +++ b/samples/js/node/optical_character_recognition/optical-character-recognition.js @@ -1,14 +1,10 @@ -const { addon: ov } = require('openvino-node'); -const fs = require('node:fs'); const path = require('node:path'); -const { createCanvas, ImageData } = require('canvas'); -const { cv } = require('opencv-wasm'); -const { - transform, - getImageData, - argMax, - setShape, -} = require('../helpers.js'); +const { addon: ov } = require('openvino-node'); + +const Image = require('../image.js'); +const { transform, argMax, setShape } = require('../helpers.js'); + +const OUTPUT_PATH = './output/'; if (require.main === module) { // Parsing and validation of input arguments @@ -32,110 +28,91 @@ if (require.main === module) { } async function main(detModelXMLPath, recModelXMLPath, imagePath, deviceName) { - // Initialize OpenVINO core and load the detection mode + // Initialize OpenVINO Core const core = new ov.Core(); + // Load the detection model const detModel = await core.readModel(detModelXMLPath); const detCompiledModel = await core.compileModel(detModel, deviceName); const detInputLayer = detCompiledModel.input(0); const detOutputLayer = detCompiledModel.output('boxes'); - const imageData = await getImageData(imagePath); - const inputImageMat = cv.matFromImageData(imageData); + const img = await Image.load(imagePath); // Resize the image to meet network input size - const [, , H, W] = detInputLayer.shape; - const resizedImage = new cv.Mat(); - cv.cvtColor(inputImageMat, inputImageMat, cv.COLOR_RGBA2RGB); - cv.cvtColor(inputImageMat, inputImageMat, cv.COLOR_BGR2RGB); - cv.resize(inputImageMat, resizedImage, new cv.Size(W, H)); + const [, , detInputHeight, detInputWidth] = detInputLayer.shape; + const resizedImg = img.resize(detInputWidth, detInputHeight); // Prepare input tensor - const inputImage = transform( - resizedImage.data, - { width: W, height: H }, + const inputImageTransformedData = transform( + resizedImg.rgb, + { width: detInputWidth, height: detInputHeight }, [0, 1, 2], ); - const tensorData = new Float32Array(inputImage); + const tensorData = new Float32Array(inputImageTransformedData); const tensor = new ov.Tensor(ov.element.f32, detInputLayer.shape, tensorData); + // Run inference on the detection model const detInferRequest = detCompiledModel.createInferRequest(); - const detResult = await detInferRequest.inferAsync([tensor]); - const boundingBoxesArray = extractBoundingBoxes(detResult[detOutputLayer]); + // Load the recognition model const recModel = await core.readModel(recModelXMLPath); const recModelCompiled = await core.compileModel(recModel, deviceName); - const recInputLayer = recModelCompiled.input(0); - const recOutputLayer = recModelCompiled.output(0); + const recInferRequest = recModelCompiled.createInferRequest(); - // Process each bounding box and run inference on the recognition model - const [, , height, width] = recInputLayer.shape; // Calculate ratios - const { ratioX, ratioY } = calculateRatios(inputImageMat, resizedImage); + const [ratioX, ratioY] = + [img.width / detInputWidth, img.height / detInputHeight]; + const boundingBoxesArray = extractBoundingBoxes(detResult[detOutputLayer]); + // Resize bounding boxes to the original image size + const boundingBoxesOriginalSizeArray = boundingBoxesArray.map(box => + [...multiplyByRatio(ratioX, ratioY, box), box[4]]); - // Convert image to grayscale - const grayscaleImage = convertToGrayscale(inputImageMat); + // Process each bounding box and run inference on the recognition model + const boxesWithAnnotations = []; + for (let i = 0; i < boundingBoxesOriginalSizeArray.length; i++) { + const box = boundingBoxesOriginalSizeArray[i]; + const [xMin, yMin, xMax, yMax] = box; + const croppedImg = img.crop(xMin, yMin, xMax - xMin, yMax - yMin); + await croppedImg.save(OUTPUT_PATH + `cropped_image_${i}.jpg`); - const annotations = []; - const croppedImages = []; + const annotation = + await performTextRecognition(recModel, recInferRequest, croppedImg); - for (let i = 0; i < boundingBoxesArray.length; i++) { - const crop = boundingBoxesArray[i]; - const [xMin, yMin, xMax, yMax] = multiplyByRatio(ratioX, ratioY, crop).map( - Math.floor, - ); - const cropRect = new cv.Rect(xMin, yMin, xMax - xMin, yMax - yMin); - const croppedImage = grayscaleImage.roi(cropRect); - - try { - const preprocessedCrop = resizeAndConvertCropToModelInput(croppedImage, [ - width, - height, - ]); - const tensorData = new Float32Array(preprocessedCrop); - const tensor = new ov.Tensor( - ov.element.f32, - Int32Array.from(recInputLayer.shape), - tensorData, - ); - - await inferAsyncProcess( - tensor, - recModelCompiled, - recOutputLayer, - i, - annotations, - ); - - croppedImages.push(cropImage(inputImageMat, xMin, yMin, xMax, yMax)); - } catch(error) { - console.error('Error during preprocessing:', error); - } - - croppedImage.delete(); + boxesWithAnnotations.push({ box, annotation }); + + console.log(`Box ${i}: [${box.join(',')}], Annotation: '${annotation}'`); } - grayscaleImage.delete(); + const annotatedImg = await putAnnotationsOnTheImage( + img, + boxesWithAnnotations, + { threshold: 0.3, confLabels: false }, + ); + const savePath = path.join(OUTPUT_PATH, 'output_image.jpg'); + await annotatedImg.save(savePath); + console.log(`The result was saved to ${savePath}`); +} - const boxesWithAnnotations = boundingBoxesArray.map((box, index) => ({ - box, - annotation: annotations[index], - })); +async function performTextRecognition(model, inferenceRequest, img) { + const inputLayerShape = model.input(0).shape; + const outputLayer = model.output(0); - logBoxesWithAnnotations(boxesWithAnnotations); + const [,, inputHeight, inputWidth] = inputLayerShape; + const resizedImg = img.resize(inputWidth, inputHeight); - convertResultToImage( - inputImageMat, - resizedImage, - boxesWithAnnotations, - { threshold: 0.3, confLabels: true }, - './assets/results/output_image.jpg', + // Convert image to grayscale and create tensor + const tensor = new ov.Tensor( + ov.element.f32, + inputLayerShape, + new Float32Array(resizedImg.grayscale), ); - croppedImages.forEach((croppedImage, i) => { - const savePath = `./assets/results/cropped_image_${i}.jpg`; - saveImage(croppedImage, savePath); - }); + const result = await inferenceRequest.inferAsync([tensor]); + const recognitionResults = extractRecognitionResults(result[outputLayer]); + const annotation = parseAnnotations(recognitionResults); + + return annotation; } // Function to extract bounding boxes from the model output @@ -147,56 +124,17 @@ function extractBoundingBoxes(output) { return setShape(boxes, [numberOfBoxes, foldingCoefficient]); } -// Function to calculate the ratios for the image -function calculateRatios(originalImage, resizedImage) { - const realY = originalImage.rows; - const realX = originalImage.cols; - const resizedY = resizedImage.rows; - const resizedX = resizedImage.cols; - const ratioX = realX / resizedX; - const ratioY = realY / resizedY; - - return { ratioX, ratioY }; -} - -// Function to convert the image to grayscale -function convertToGrayscale(originalImage) { - const grayscaleImage = new cv.Mat(); - cv.cvtColor(originalImage, grayscaleImage, cv.COLOR_BGR2GRAY); - - return grayscaleImage; -} - // Function to adjust bounding box coordinates by a given ratio function multiplyByRatio(ratioX, ratioY, box) { - const scaleShape = (shape, idx) => - idx % 2 ? Math.max(shape * ratioY, 10) : shape * ratioX; - - return box.map(scaleShape); -} - -// Function to resize and convert a crop to the recognition model input format -function resizeAndConvertCropToModelInput(crop, netShape) { - const [netWidth, netHeight] = netShape; + const scaleShape = (shape, idx) => { + const position = idx % 2 + ? Math.max(shape * ratioY, 10) + : shape * ratioX; - // Resize the crop to the network's input shape - const tempImg = new cv.Mat(); - cv.resize(crop, tempImg, new cv.Size(netWidth, netHeight)); - - // Create the reshaped buffer - const reshapedBuffer = new Uint8Array(netHeight * netWidth); - let index = 0; - - for (let i = 0; i < netHeight; i++) { - for (let j = 0; j < netWidth; j++) { - reshapedBuffer[index++] = tempImg.ucharPtr(i, j)[0]; - } + return Math.floor(position); } - // Clean up - tempImg.delete(); - - return reshapedBuffer; + return box.map(scaleShape); } // Function to extract recognition results from the model output @@ -219,204 +157,49 @@ function parseAnnotations(recognitionResults) { // Stop if end character is encountered if (parsedLetter === letters[0]) break; + annotation.push(parsedLetter); } return annotation.join(''); } -// Function to crop the image based on the bounding box coordinates -function cropImage(originalImage, xMin, yMin, xMax, yMax) { - xMin = Math.max(0, xMin); - yMin = Math.max(0, yMin); - xMax = Math.min(originalImage.cols, xMax); - yMax = Math.min(originalImage.rows, yMax); - if (xMin >= xMax || yMin >= yMax) { - throw new Error('Invalid crop coordinates'); - } - const roi = originalImage.roi( - new cv.Rect(xMin, yMin, xMax - xMin, yMax - yMin), - ); - const cropped = new cv.Mat(); - roi.copyTo(cropped); - roi.delete(); - - return cropped; -} - -// Get Text size -function getTextSize(text, fontFace, fontScale) { - const canvas = createCanvas(200, 200); - const ctx = canvas.getContext('2d'); - const adjustedFontScale = fontScale * 35; - ctx.font = `${adjustedFontScale}px ${fontFace}`; - const metrics = ctx.measureText(text); - const width = metrics.width; - const height = - metrics.actualBoundingBoxAscent + metrics.actualBoundingBoxDescent; - - return { width, height }; -} - -/* The convertResultToImage function visualizes object detection - results on an image by drawing bounding boxes around detected - objects and optionally adding labels to them. */ - -function convertResultToImage( - bgrImage, - resizedImage, - boxesWithAnnotations, - options, - savePath, -) { +// Takes original image and bounding boxes with annotations +// and returns the image with annotations +async function putAnnotationsOnTheImage(img, boxesWithAnnotations, options) { const defaultOptions = { threshold: 0.3, confLabels: true }; const { threshold, confLabels } = Object.assign(defaultOptions, options); - const colors = { - red: [255, 0, 0, 255], - green: [0, 255, 0, 255], - white: [255, 255, 255, 255], - }; - const [realY, realX] = [bgrImage.rows, bgrImage.cols]; - const [resizedY, resizedX] = [resizedImage.rows, resizedImage.cols]; - const [ratioX, ratioY] = [realX / resizedX, realY / resizedY]; - - const rgbImage = new cv.Mat(); - cv.cvtColor(bgrImage, rgbImage, cv.COLOR_BGR2RGB); + let finalImage = img; - boxesWithAnnotations.forEach(({ box, annotation }) => { + for (const item of boxesWithAnnotations) { + const { box, annotation } = item; const conf = box[box.length - 1]; - if (conf < threshold) return; - - const [xMin, yMin, xMax, yMax] = multiplyByRatio(ratioX, ratioY, box); - - cv.rectangle( - rgbImage, - new cv.Point(xMin, yMin), - new cv.Point(xMax, yMax), - colors.green, - 3, - ); + if (conf < threshold) continue; - if (!confLabels) return; + const [xMin, yMin, xMax, yMax] = box; + const yOffset = 10; - const text = `${annotation}`; - const fontScale = 0.8; - const thickness = 1; - const { width: textW, height: textH } = getTextSize( - text, - 'Arial', - fontScale, + finalImage = finalImage.drawRect( + xMin, yMin, + xMax - xMin, yMax - yMin, + { color: 'green', width: 3 }, ); - const imageCopy = rgbImage.clone(); - - cv.rectangle( - imageCopy, - new cv.Point(xMin, yMin - textH - 10), - new cv.Point(xMin + textW, yMin - 10), - colors.white, - cv.FILLED, + finalImage = finalImage.drawText( + annotation, + xMin, yMin - yOffset, + { font: '30px Arial' }, ); - cv.addWeighted(imageCopy, 0.4, rgbImage, 0.6, 0, rgbImage); - cv.putText( - rgbImage, - text, - new cv.Point(xMin, yMin - 10), - cv.FONT_HERSHEY_SIMPLEX, - fontScale, - colors.red, - thickness, - cv.LINE_AA, - ); - - imageCopy.delete(); - }); - - const saveDir = path.dirname(savePath); - if (!fs.existsSync(saveDir)) { - fs.mkdirSync(saveDir, { recursive: true }); - } - try { - saveImage(rgbImage, savePath); - } catch(e) { - console.log(`Error occurred while saving ----> ${e}`); - } - - return rgbImage; -} + if (!confLabels) continue; -// Infer async helper function - -async function inferAsyncProcess( - tensor, - recModelCompiled, - recOutputLayer, - i, - annotations, -) { - // Create infer request - const inferRequest = recModelCompiled.createInferRequest(); - - // Define the completion callback function - function completionCallback(outputTensor, i, annotations) { - const recognitionResults = extractRecognitionResults(outputTensor); - const annotation = parseAnnotations(recognitionResults); - annotations.push(annotation); - } - - // Start inference in asynchronous mode - try { - const result = await inferRequest.inferAsync([tensor]); - completionCallback(result[recOutputLayer], i, annotations); - } catch(error) { - console.error('Error during inference:', error); - } -} - -// Log boudning boxes with annotations -function logBoxesWithAnnotations(boxesWithAnnotations) { - boxesWithAnnotations.forEach((item, i) => { - const { box, annotation } = item; - console.log(`Box ${i}: [${box}], Annotation: ${annotation}`); - }); -} - -function saveImage(rgbImage, savePath) { - const canvas = createCanvas(rgbImage.cols, rgbImage.rows); - const ctx = canvas.getContext('2d'); - const componentsPerPixel = - rgbImage.data.length / (rgbImage.cols * rgbImage.rows); - const imgDataArr = []; - - if (componentsPerPixel === 1) { - for (const val of rgbImage.data) { - imgDataArr.push(val, val, val, 255); - } - } else if (componentsPerPixel === 3) { - for (let i = 0; i < rgbImage.data.length; i++) { - if (i % 3 === 0) imgDataArr.push(255); - imgDataArr.push(rgbImage.data[i]); - } - } - - const imageData = new ImageData( - new Uint8ClampedArray(imgDataArr), - rgbImage.cols, - rgbImage.rows, - ); - ctx.putImageData(imageData, 0, 0); - - const dataURL = canvas.toDataURL('image/jpeg'); - const base64Data = dataURL.replace(/^data:image\/jpeg;base64,/, ''); - const imageBuffer = Buffer.from(base64Data, 'base64'); - - const saveDir = path.dirname(savePath); - if (!fs.existsSync(saveDir)) { - fs.mkdirSync(saveDir, { recursive: true }); + finalImage = finalImage.drawText( + conf.toFixed(2), + xMin, yMax + 2 * yOffset, + { font: '20px Arial' }, + ); } - fs.writeFileSync(savePath, imageBuffer); - console.log('Image saved successfully!', savePath); + return finalImage; } diff --git a/samples/js/node/package-lock.json b/samples/js/node/package-lock.json index fe003eaac14935..96a013fb0435c7 100644 --- a/samples/js/node/package-lock.json +++ b/samples/js/node/package-lock.json @@ -7,14 +7,14 @@ "": { "name": "openvino-node-demo", "version": "1.0.0", + "hasInstallScript": true, "license": "Apache-2.0", "devDependencies": { + "@napi-rs/canvas": "^0.1.59", "@tensorflow/tfjs-node": "^4.19.0", "args": "^5.0.3", - "canvas": "^2.11.2", "eslint": "^8.39.0", "https-proxy-agent": "^7.0.2", - "opencv-wasm": "^4.3.0-10", "openvino-node": "^2024.4.0" }, "engines": { @@ -173,6 +173,180 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.59.tgz", + "integrity": "sha512-3vUtQ8DzYcz9xy86UUe8OfDiXNuuLB9zFAUs5N/I2GpkY/MWBJ2M7w5FqH380oC44IzYOWaOMLWCPfNZBsbBww==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.59", + "@napi-rs/canvas-darwin-arm64": "0.1.59", + "@napi-rs/canvas-darwin-x64": "0.1.59", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.59", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.59", + "@napi-rs/canvas-linux-arm64-musl": "0.1.59", + "@napi-rs/canvas-linux-x64-gnu": "0.1.59", + "@napi-rs/canvas-linux-x64-musl": "0.1.59", + "@napi-rs/canvas-win32-x64-msvc": "0.1.59" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.59.tgz", + "integrity": "sha512-p4rRL9KIDz57Z+gKLpemX36DB7fVVHmY4DtesMGrnjx4gSBUM2M7LNzbzf4o3oPZGDiHMY0vnvNHR4dKfszNeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.59.tgz", + "integrity": "sha512-+8s06WxcM9ilv9PVOl57hvasbwKWMfrrNAYknqMPCn4jpc4XDcLbrM5LTZGhhptlv9jQ9DmHfZ978/xInsMYXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.59.tgz", + "integrity": "sha512-6kziJHjXdxduYK2L2uuwjEIYoPJednKq+C81MCm3fPobXE4HBKs0JGXwq3GkWNe340U340vmagwXiFi6muEy+g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.59.tgz", + "integrity": "sha512-eCkyS7jojNmaUPaVFdNjAyS0R3isrJtUfRf1vRP6K50GRuHso3vwQRbZBPKM71qHdjPDylfaQc5H6/M7epyD+w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.59.tgz", + "integrity": "sha512-1u4++lbsolP1MAPViuDoZmgmDLKlV0iJnlHN2dfwgbu3t53P0l3jIT1oCIAiWil0OlrWtDF24JbY7LUUGH5aHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.59.tgz", + "integrity": "sha512-eqevZ2kWPxeAnvhxl7U5tf6AiMnhlO4w2Hci79WQkfeirqQG6RRM4Jnxbh9iO3jkAnnOXmM4r+S3UrOcfIx1Rg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.59.tgz", + "integrity": "sha512-F+T63RnLt0qYUXhbOpaome3vIWLW4xoQRmhTnkKDzOtBSnKVP7sCM6E5/5tByOFCR3fTj4ksMeeHy8zJScEExA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.59.tgz", + "integrity": "sha512-HhUgpTGQUR2VRslEC5Idf6s0hhamJiVlEh2k3AG9XXOwX6fg0xXkqm84DPiOCLzsO5bqtJEo+rh03BUSDcf53g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.59.tgz", + "integrity": "sha512-bYMiZJsKPkU7HEoYI5E0alOSV1EkaigY4VEgGHPK9W/qGMmNFsxdbURQqa5h3zbhZTK5QRSdYYqowcTEYVIlug==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -774,21 +948,6 @@ "node": ">=6" } }, - "node_modules/canvas": { - "version": "2.11.2", - "resolved": "https://registry.npmjs.org/canvas/-/canvas-2.11.2.tgz", - "integrity": "sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==", - "dev": true, - "hasInstallScript": true, - "dependencies": { - "@mapbox/node-pre-gyp": "^1.0.0", - "nan": "^2.17.0", - "simple-get": "^3.0.3" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -924,18 +1083,6 @@ } } }, - "node_modules/decompress-response": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", - "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", - "dev": true, - "dependencies": { - "mimic-response": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -1745,18 +1892,6 @@ "node": ">= 0.6" } }, - "node_modules/mimic-response": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", - "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", - "dev": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -1830,12 +1965,6 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, - "node_modules/nan": { - "version": "2.20.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.20.0.tgz", - "integrity": "sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==", - "dev": true - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -1908,12 +2037,6 @@ "wrappy": "1" } }, - "node_modules/opencv-wasm": { - "version": "4.3.0-10", - "resolved": "https://registry.npmjs.org/opencv-wasm/-/opencv-wasm-4.3.0-10.tgz", - "integrity": "sha512-EWmWLUzp2suoc6N44Y4ouWT85QwvShx23Q430R+lp6NyS828bjQn6mCgA3NJ6Z/S59aaTeeu+RhqPQIJIYld1w==", - "dev": true - }, "node_modules/openvino-node": { "version": "2024.4.0", "resolved": "https://registry.npmjs.org/openvino-node/-/openvino-node-2024.4.0.tgz", @@ -2272,37 +2395,6 @@ "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true }, - "node_modules/simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/simple-get": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.1.tgz", - "integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==", - "dev": true, - "dependencies": { - "decompress-response": "^4.2.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", diff --git a/samples/js/node/package.json b/samples/js/node/package.json index 683312fab6742b..d392a72143de03 100644 --- a/samples/js/node/package.json +++ b/samples/js/node/package.json @@ -2,17 +2,18 @@ "name": "openvino-node-demo", "version": "1.0.0", "license": "Apache-2.0", + "type": "commonjs", "devDependencies": { + "@tensorflow/tfjs-node": "^4.19.0", "args": "^5.0.3", - "canvas": "^2.11.2", "eslint": "^8.39.0", "https-proxy-agent": "^7.0.2", - "opencv-wasm": "^4.3.0-10", - "@tensorflow/tfjs-node": "^4.19.0", - "openvino-node": "^2024.4.0" + "openvino-node": "^2024.4.0", + "@napi-rs/canvas": "^0.1.59" }, "scripts": { - "lint": "eslint ." + "lint": "eslint .", + "postinstall": "node ./fetch-samples-assets.js" }, "engines": { "node": ">=21.0.0" diff --git a/samples/js/node/vision_background_removal/README.md b/samples/js/node/vision_background_removal/README.md index cdefc0c4186cbb..a9a3e3270eae45 100644 --- a/samples/js/node/vision_background_removal/README.md +++ b/samples/js/node/vision_background_removal/README.md @@ -1,6 +1,10 @@ # Vision Background Removal Node.js Sample -Run: +Run sample: ```bash -node vision_background_removal.js *path_to_model_file* *path_to_foreground_image* *path_to_background_image* AUTO -``` \ No newline at end of file +node vision_background_removal.js ../../assets/models/unet_ir_model.xml ../../assets/images/coco_hollywood.jpg ../../assets/images/wall.jpg AUTO +``` +Where: +```bash +node vision_background_removal.js *path_to_model_file* *path_to_foreground_image* *path_to_background_image* *device* +``` diff --git a/samples/js/node/vision_background_removal/vision_background_removal.js b/samples/js/node/vision_background_removal/vision_background_removal.js index 2487371c597008..aabaad1a65b80c 100644 --- a/samples/js/node/vision_background_removal/vision_background_removal.js +++ b/samples/js/node/vision_background_removal/vision_background_removal.js @@ -1,12 +1,10 @@ -const { cv } = require('opencv-wasm'); -const fs = require('node:fs').promises; -const path = require('node:path'); const { addon: ov } = require('openvino-node'); -const { createCanvas, ImageData } = require('canvas'); -const { getImageData, transform, setShape } = require('../helpers'); + +const { transform } = require('../helpers'); +const Image = require('../image'); if (require.main === module) { -// Parsing and validation of input arguments + // Parsing and validation of input arguments if (process.argv.length !== 6) throw new Error( `Usage: ${process.argv[1]} ` + @@ -15,21 +13,23 @@ if (require.main === module) { ); const unetModelPath = process.argv[2]; - const foreGroundImage = process.argv[3]; - const backGroundImage = process.argv[4]; + const foregroundImagePath = process.argv[3]; + const backgroundImagePath = process.argv[4]; const deviceName = process.argv[5]; try { - main(unetModelPath, foreGroundImage, backGroundImage, deviceName); + main(unetModelPath, foregroundImagePath, backgroundImagePath, deviceName); } catch(error) { console.error('Error occurred', error); } } +module.exports = main; + async function main( unetModelPath, - foreGroundImage, - backGroundImage, + foregroundImagePath, + backgroundImagePath, deviceName, ) { const core = new ov.Core(); @@ -38,104 +38,81 @@ async function main( const model = await core.readModel(unetModelPath); const compiledModel = await core.compileModel(model, deviceName); - // Get the names of input and output layers. + // Get the names of input and output layers const inputLayer = compiledModel.input(0); const outputLayer = compiledModel.output(0); - // Get Image data from the foreground image - const imageData = await getImageData(foreGroundImage); - const inputImageMat = cv.matFromImageData(imageData); - - // Convert the image shape to a shape and a data type expected by the network - const [, , H, W] = inputLayer.shape; - const resizedImage = new cv.Mat(); - cv.cvtColor(inputImageMat, inputImageMat, cv.COLOR_BGR2RGB); - cv.resize(inputImageMat, resizedImage, new cv.Size(W, H)); + // Load foreground image + const originalImg = await Image.load(foregroundImagePath); - const inputImage = transform( - resizedImage.data, - { width: W, height: H }, - [0, 1, 2], - ); - - // Normalize the input image Mat - const normalizedInputImage = normalizeImage(inputImage, W, H); + // Resize image to a shape expected by the network + const [, , modelInputHeight, modelInputWidth] = inputLayer.shape; + const resized = await originalImg.resize(modelInputWidth, modelInputHeight); // Create a tensor from the normalized input image - const tensorData = new Float32Array(normalizedInputImage); - const tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, tensorData); + const transformed = transform( + resized.rgb, + { + width: modelInputWidth, + height: modelInputHeight + }, + [0, 1, 2] + ); + const normalizedInputImage = normalizeImage( + transformed, + modelInputWidth, + modelInputHeight, + ); + const tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, normalizedInputImage); // Do inference const inferRequest = compiledModel.createInferRequest(); const inferResult = await inferRequest.inferAsync([tensor]); + const { data: resultData } = inferResult[outputLayer]; - const { data } = inferResult[outputLayer]; - const reshapedResult = setShape(data, [512, 512]); - - // Create a Mat from the reshaped result - const reshapedMat = cv.matFromArray( - 512, - 512, - cv.CV_32F, - reshapedResult.flat(), - ); - - // Get the height and width of the original image - const height = inputImageMat.rows; - const width = inputImageMat.cols; - - // Resize the inference result to the original image size - const resizedResult = new cv.Mat(); - cv.resize( - reshapedMat, - resizedResult, - new cv.Size(width, height), - 0, - 0, - cv.INTER_LINEAR, - ); - - // Convert the resized result to uint8 - resizedResult.convertTo(resizedResult, cv.CV_8U); - - // Create a Mat to store the background removed result - const bgRemovedResult = inputImageMat.clone(); + // Normalize the result data from grayscale to RGB + const rgbData = []; + for (let i = 0; i < resultData.length; i += 1) { + const value = resultData[i] * 255; - removeBackground(resizedResult, bgRemovedResult); + rgbData.push(value, value, value, 255); + } - // Save the background removed result - await saveImage(bgRemovedResult, './bg_removed_result.jpg'); + // Create image based on result data + const [outputHeight, outputWidth] = outputLayer.shape.slice(2); + const maskImg = await Image.fromArray(rgbData, outputWidth, outputHeight); - // Get the background image data - const bgrImageData = await getImageData(backGroundImage); - const bgrImageMat = cv.matFromImageData(bgrImageData); + // Resize the result mask to the original image size and save it + const { width, height } = originalImg; + const resizedMaskImg = await maskImg.resize(originalImg.width, originalImg.height); + const maskImagePath = './out_mask.jpg'; + await resizedMaskImg.save(maskImagePath); + console.log(`The mask image was saved to '${maskImagePath}'`); - // Resize the background image to the original image size - const resizedBgrImageMat = new cv.Mat(); - cv.cvtColor(bgrImageMat, bgrImageMat, cv.COLOR_BGR2RGB); - cv.resize(bgrImageMat, resizedBgrImageMat, new cv.Size(width, height)); + // Remove the foreground from the original image + const removedBgImg = Image.mask(originalImg, resizedMaskImg); - // Remove the foreground from the background image by - // setting all foreground pixels to white - removeForeground(resizedResult, resizedBgrImageMat); + // Load the background image + const bgrImage = await Image.load(backgroundImagePath); - // Save the foreground removed from the background image - await saveImage(resizedBgrImageMat, './fg_removed_from_background.jpg'); + // Resize the background image to the same size as the original image + const resizedBgrImage = bgrImage.resize(width, height); - // create a new Mat to store the final image - const newImage = new cv.Mat( - resizedBgrImageMat.rows, - resizedBgrImageMat.cols, - cv.CV_8UC3, - ); + // Remove object from the background image + const removedFgImg = Image.mask(resizedBgrImage, resizedMaskImg.invert()); - // combine the foreground and background images to get the final image - combineImages(resizedResult, bgRemovedResult, resizedBgrImageMat, newImage); + // Combine the background and foreground images + const resultImg = Image.merge(removedBgImg, removedFgImg); // Save the final image - await saveImage(newImage, './background_changed_image.jpg'); + const outputImagePath = './out_bgr_changed_image.jpg'; + await resultImg.save(outputImagePath); + console.log(`The result image was saved to '${outputImagePath}'`); + console.log('The background was successfully changed'); } +// Details about this normalization: +// https://docs.openvino.ai/2024/notebooks/vision-background-removal-with-output.html#load-and-pre-process-input-image function normalizeImage(imageData, width, height) { // Mean and scale values const inputMean = [123.675, 116.28, 103.53]; @@ -148,6 +125,7 @@ function normalizeImage(imageData, width, height) { for (let j = 0; j < width; j++) { for (let c = 0; c < channels; c++) { const index = i * width * channels + j * channels + c; + normalizedData[index] = (imageData[index] - inputMean[c]) / inputScale[c]; } @@ -156,93 +134,3 @@ function normalizeImage(imageData, width, height) { return normalizedData; } - -function removeBackground(mask, image) { - // Iterate over the mask and set all background pixels to white - for (let i = 0; i < mask.rows; i++) { - for (let j = 0; j < mask.cols; j++) { - if (mask.ucharPtr(i, j)[0] === 0) { - image.ucharPtr(i, j)[0] = 255; - image.ucharPtr(i, j)[1] = 255; - image.ucharPtr(i, j)[2] = 255; - } - } - } -} - -function removeForeground(mask, image) { - // Iterate over the mask and set all foreground pixels to black - for (let i = 0; i < mask.rows; i++) { - for (let j = 0; j < mask.cols; j++) { - if (mask.ucharPtr(i, j)[0] === 1) { - image.ucharPtr(i, j)[0] = 0; - image.ucharPtr(i, j)[1] = 0; - image.ucharPtr(i, j)[2] = 0; - } else { - image.ucharPtr(i, j)[0] = image.ucharPtr(i, j)[0]; - image.ucharPtr(i, j)[1] = image.ucharPtr(i, j)[1]; - image.ucharPtr(i, j)[2] = image.ucharPtr(i, j)[2]; - } - } - } -} - -function combineImages(mask, fgImage, bgImage, newImage) { - // Iterate over the mask and combine the foreground and background images - for (let i = 0; i < mask.rows; i++) { - for (let j = 0; j < mask.cols; j++) { - if (mask.ucharPtr(i, j)[0] === 1) { - newImage.ucharPtr(i, j)[0] = fgImage.ucharPtr(i, j)[0]; - newImage.ucharPtr(i, j)[1] = fgImage.ucharPtr(i, j)[1]; - newImage.ucharPtr(i, j)[2] = fgImage.ucharPtr(i, j)[2]; - } else { - newImage.ucharPtr(i, j)[0] = bgImage.ucharPtr(i, j)[0]; - newImage.ucharPtr(i, j)[1] = bgImage.ucharPtr(i, j)[1]; - newImage.ucharPtr(i, j)[2] = bgImage.ucharPtr(i, j)[2]; - } - } - } -} - -async function saveImage(rgbImage, savePath) { - const canvas = createCanvas(rgbImage.cols, rgbImage.rows); - const ctx = canvas.getContext('2d'); - const componentsPerPixel = - rgbImage.data.length / (rgbImage.cols * rgbImage.rows); - const imgDataArr = []; - - if (componentsPerPixel === 1) { - for (const val of rgbImage.data) { - imgDataArr.push(val, val, val, 255); - } - } else if (componentsPerPixel === 3) { - for (let i = 0; i < rgbImage.data.length; i += 3) { - imgDataArr.push( - rgbImage.data[i + 2], // Red - rgbImage.data[i + 1], // Green - rgbImage.data[i], // Blue - 255, // Alpha - ); - } - } - - const imageData = new ImageData( - new Uint8ClampedArray(imgDataArr), - rgbImage.cols, - rgbImage.rows, - ); - ctx.putImageData(imageData, 0, 0); - - const dataURL = canvas.toDataURL('image/jpeg'); - const base64Data = dataURL.replace(/^data:image\/jpeg;base64,/, ''); - const imageBuffer = Buffer.from(base64Data, 'base64'); - - const saveDir = path.dirname(savePath); - try { - await fs.mkdir(saveDir, { recursive: true }); - await fs.writeFile(savePath, imageBuffer); - console.log('Image saved successfully!', savePath); - } catch(error) { - console.error('Error saving image:', error); - } -} From 892eea46bc96d79a3ca57e0f737f97c0f73e9341 Mon Sep 17 00:00:00 2001 From: Roman Kazantsev Date: Tue, 5 Nov 2024 13:21:51 +0400 Subject: [PATCH 02/93] [TF FE] Provide better support for TopKV2 and stabilize test (#27374) **Details:** Provide better validation for TopKV2 and stabilize test **Ticket:** TBD --------- Signed-off-by: Kazantsev, Roman --- .../tensorflow_common/src/op/top_k.cpp | 23 ++++-- .../tensorflow_tests/test_tf_TopKV2.py | 70 +++++++++++-------- 2 files changed, 58 insertions(+), 35 deletions(-) diff --git a/src/frontends/tensorflow_common/src/op/top_k.cpp b/src/frontends/tensorflow_common/src/op/top_k.cpp index c916fe5b746327..4d3bfdcf64bcaf 100644 --- a/src/frontends/tensorflow_common/src/op/top_k.cpp +++ b/src/frontends/tensorflow_common/src/op/top_k.cpp @@ -3,6 +3,7 @@ // #include "common_op_table.hpp" +#include "openvino/op/convert.hpp" #include "openvino/op/topk.hpp" using namespace std; @@ -14,22 +15,35 @@ namespace tensorflow { namespace op { NamedOutputVector translate_top_k_base_op(const NodeContext& node, const ov::Output& k_input, - int min_input_size) { + int min_input_size, + const ov::element::Type& index_type = ov::element::i32) { default_op_checks(node, min_input_size, {"TopK", "TopKV2", "TOPK_V2"}); auto input = node.get_input(0); // retrieve k attribute bool sorted = node.get_attribute("sorted", true); + auto topk_index_type = index_type; + if (index_type == ov::element::i16) { + // v11::TopK supports only int32 and int64 output index type + topk_index_type = ov::element::i32; + } auto top_k = make_shared(input, k_input, -1, ov::op::v11::TopK::Mode::MAX, sorted ? v11::TopK::SortType::SORT_VALUES : v11::TopK::SortType::SORT_INDICES, - ov::element::i32, + topk_index_type, true); + auto values = top_k->output(0); + auto indices = top_k->output(1); + if (index_type != topk_index_type) { + // satisfy the requested output index type + indices = make_shared(indices, index_type)->output(0); + } set_node_name(node.get_name(), top_k); - return {{"values", top_k->output(0)}, {"indices", top_k->output(1)}}; + return {{"values", values}, {"indices", indices}}; } + NamedOutputVector translate_top_k_op(const NodeContext& node) { // retrieve k attribute auto k = node.get_attribute("k"); @@ -39,8 +53,9 @@ NamedOutputVector translate_top_k_op(const NodeContext& node) { NamedOutputVector translate_top_k_v2_op(const NodeContext& node) { default_op_checks(node, 2, {"TopKV2", "TOPK_V2"}); + auto index_type = node.get_attribute("index_type", ov::element::i32); auto k_input = node.get_input(1); - return translate_top_k_base_op(node, k_input, 1); + return translate_top_k_base_op(node, k_input, 1, index_type); } } // namespace op } // namespace tensorflow diff --git a/tests/layer_tests/tensorflow_tests/test_tf_TopKV2.py b/tests/layer_tests/tensorflow_tests/test_tf_TopKV2.py index 08540a1d7fec0e..23d5c6bf2c23fe 100644 --- a/tests/layer_tests/tensorflow_tests/test_tf_TopKV2.py +++ b/tests/layer_tests/tensorflow_tests/test_tf_TopKV2.py @@ -1,58 +1,66 @@ # Copyright (C) 2018-2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import platform - import numpy as np +import platform import pytest import tensorflow as tf from common.tf_layer_test_class import CommonTFLayerTest +rng = np.random.default_rng(235436) + class TestTopKV2(CommonTFLayerTest): def _prepare_input(self, inputs_info): # generate elements so that the input tensor may contain repeating elements - assert 'input:0' in inputs_info, "Test error: inputs_info must contain `input`" - x_shape = inputs_info['input:0'] + assert 'input:0' in inputs_info, "Test error: inputs_info must contain `x`" + input_shape = inputs_info['input:0'] inputs_data = {} - inputs_data['input:0'] = np.random.randint(-10, 10, x_shape) + # need to generate data with repeated elements so the range is narrow from 30 elements + if np.issubdtype(self.input_type, np.floating): + inputs_data['input:0'] = rng.integers(-15, 15, input_shape).astype(self.input_type) + elif np.issubdtype(self.input_type, np.signedinteger): + inputs_data['input:0'] = rng.integers(-15, 15, input_shape).astype(self.input_type) + else: + inputs_data['input:0'] = rng.integers(0, 30, input_shape).astype(self.input_type) return inputs_data - def create_topk_v2_net(self, input_shape, input_type, k, sorted, is_first_output, is_second_output): + def create_topk_v2_net(self, input_shape, input_type, k, k_type, sorted, index_type): + self.input_type = input_type tf.compat.v1.reset_default_graph() with tf.compat.v1.Session() as sess: input = tf.compat.v1.placeholder(input_type, input_shape, 'input') - k = tf.constant(k, dtype=tf.int32, shape=[]) - topk = tf.raw_ops.TopKV2(input=input, k=k, sorted=sorted) - if is_first_output: - tf.identity(topk[0], name='topk_values') - if is_second_output: - tf.identity(topk[1], name='unique_indices') + k = tf.constant(k, dtype=k_type, shape=[]) + topk = tf.raw_ops.TopKV2(input=input, k=k, sorted=sorted, index_type=index_type) + tf.identity(topk[0], name='topk_values') + tf.identity(topk[1], name='unique_indices') tf.compat.v1.global_variables_initializer() tf_net = sess.graph_def - ref_net = None return tf_net, None - test_basic = [ - dict(input_shape=[10], input_type=tf.float32, k=5, sorted=True, is_first_output=True, is_second_output=False), - dict(input_shape=[2, 3, 10], input_type=tf.int32, k=10, sorted=True, is_first_output=True, - is_second_output=False), - dict(input_shape=[4, 12], input_type=tf.float32, k=10, sorted=True, is_first_output=True, - is_second_output=True), - # Expect stable mode implementation for sort_type=indices in OpenVINO. See 101503 - pytest.param(dict(input_shape=[5, 10], input_type=tf.int32, k=8, sorted=False, is_first_output=True, - is_second_output=True), marks=pytest.mark.xfail(reason="101503")), - ] - - @pytest.mark.parametrize("params", test_basic) + @pytest.mark.parametrize('input_shape', [[30], [4, 30], [4, 2, 70]]) + @pytest.mark.parametrize('input_type', [np.float16, np.float32, np.float64, + np.int32, np.uint8, np.int16, np.int8, np.int64, + np.uint16, np.uint32, np.uint64]) + @pytest.mark.parametrize('k', [5, 10, 20]) + @pytest.mark.parametrize('k_type', [np.int16, np.int32, np.int64]) + @pytest.mark.parametrize('sorted', [True, + # TensorFlow has undefined behaviour (or poorly described in the spec) + # for sorted = False + ]) + @pytest.mark.parametrize('index_type', [np.int16, np.int32, np.int64]) @pytest.mark.precommit @pytest.mark.nightly - @pytest.mark.xfail(condition=platform.system() in ('Linux', 'Darwin') and platform.machine() in ('arm', 'armv7l', - 'aarch64', - 'arm64', 'ARM64'), - reason='Ticket - 126314, 122716') - def test_topk_v2_basic(self, params, ie_device, precision, ir_version, temp_dir, use_legacy_frontend): - self._test(*self.create_topk_v2_net(**params), + def test_topk_v2(self, input_shape, input_type, k, k_type, sorted, index_type, + ie_device, precision, ir_version, + temp_dir, use_legacy_frontend): + if ie_device == 'GPU' and input_type == np.uint8: + pytest.skip('156587: Check correct_layout_selected failed for input uint8 on GPU') + if platform.machine() in ['arm', 'armv7l', 'aarch64', 'arm64', 'ARM64'] and \ + input_type in [np.int32, np.uint8, np.int16, np.int8, np.int64, + np.uint16, np.uint32, np.uint64]: + pytest.skip('156588: Segmenation fault or OpenVINO hangs for TopKV2 on ARM') + self._test(*self.create_topk_v2_net(input_shape, input_type, k, k_type, sorted, index_type), ie_device, precision, ir_version, temp_dir=temp_dir, use_legacy_frontend=use_legacy_frontend) From 6878482f702cf92be6391147cb30da528e9dc543 Mon Sep 17 00:00:00 2001 From: Vladislav Golubev Date: Tue, 5 Nov 2024 10:25:09 +0100 Subject: [PATCH 03/93] [Transformations][CPU] LoraFusion: convert on states support (#27340) ### Details: - *This PR extends LoraFusion transformation: convert on states support is added* - *The corresponding transformation tests are extended* - *Fixed a critical bug in the memory reuse between the LoRA subgraph and the main CPU graph* ### Tickets: - CVS-155112 - CVS-153035 --------- Co-authored-by: Maksim Kutakov --- .../lora_subgraph_fusion.cpp | 31 ++- .../lora_subgraph_fusion.cpp | 57 +++++- .../include/openvino/util/weights_path.hpp | 4 +- src/plugins/intel_cpu/src/nodes/lora.cpp | 22 ++- src/plugins/intel_cpu/src/nodes/lora.h | 6 +- .../src/common/lora_pattern.cpp | 177 +++++++++++------- 6 files changed, 207 insertions(+), 90 deletions(-) diff --git a/src/common/transformations/src/transformations/common_optimizations/lora_subgraph_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/lora_subgraph_fusion.cpp index 366ce00894242e..95b754062c13b8 100644 --- a/src/common/transformations/src/transformations/common_optimizations/lora_subgraph_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/lora_subgraph_fusion.cpp @@ -9,6 +9,7 @@ #include "itt.hpp" #include "openvino/op/add.hpp" +#include "openvino/op/convert.hpp" #include "openvino/op/convolution.hpp" #include "openvino/op/matmul.hpp" #include "openvino/op/multiply.hpp" @@ -26,12 +27,19 @@ ov::pass::LoraSubgraphFusion::LoraSubgraphFusion() { auto lora_input_m = any_input(); auto transpose_const1_m = wrap_type(consumers_count(1)); auto transpose1_m = optional({lora_input_m, transpose_const1_m}, consumers_count(1)); + auto read_value1_m = wrap_type(); - auto matmul1_m = wrap_type({transpose1_m, read_value1_m}, consumers_count(1)); + auto convert1_m = optional(read_value1_m, consumers_count(1)); + auto matmul1_m = wrap_type({transpose1_m, convert1_m}, consumers_count(1)); + auto read_value2_m = wrap_type(); - auto multiply_m = wrap_type({matmul1_m, read_value2_m}, consumers_count(1)); + auto convert2_m = optional(read_value2_m, consumers_count(1)); + auto multiply_m = wrap_type({matmul1_m, convert2_m}, consumers_count(1)); + auto read_value3_m = wrap_type(); - auto matmul2_m = wrap_type({multiply_m, read_value3_m}, consumers_count(1)); + auto convert3_m = optional(read_value3_m, consumers_count(1)); + auto matmul2_m = wrap_type({multiply_m, convert3_m}, consumers_count(1)); + auto transpose_const2_m = wrap_type(consumers_count(1)); auto transpose2_m = optional({matmul2_m, transpose_const2_m}, consumers_count(1)); auto main_flow_m = wrap_type({lora_input_m, any_input()}); @@ -41,11 +49,14 @@ ov::pass::LoraSubgraphFusion::LoraSubgraphFusion() { const auto& pattern_map = m.get_pattern_value_map(); const auto& lora_input = pattern_map.at(lora_input_m); const auto& matmul1 = pattern_map.at(matmul1_m); - const auto& read_value1 = pattern_map.at(read_value1_m); + const auto& state_1 = + pattern_map.count(convert1_m) ? pattern_map.at(convert1_m) : pattern_map.at(read_value1_m); const auto& multiply = pattern_map.at(multiply_m); - const auto& read_value2 = pattern_map.at(read_value2_m); + const auto& state_2 = + pattern_map.count(convert2_m) ? pattern_map.at(convert2_m) : pattern_map.at(read_value2_m); const auto& matmul2 = pattern_map.at(matmul2_m); - const auto& read_value3 = pattern_map.at(read_value3_m); + const auto& state_3 = + pattern_map.count(convert3_m) ? pattern_map.at(convert3_m) : pattern_map.at(read_value3_m); const auto& main_flow = pattern_map.at(main_flow_m); const auto& add = pattern_map.at(add_m); @@ -70,15 +81,15 @@ ov::pass::LoraSubgraphFusion::LoraSubgraphFusion() { pattern_map.count(transpose1_m) ? pattern_map.at(transpose1_m).get_node()->input(0) : matmul1.get_node()->input(0), matmul1.get_node()->input(1), - find_connected_input(multiply.get_node(), read_value2.get_node()), + find_connected_input(multiply.get_node(), state_2.get_node()), matmul2.get_node()->input(1), }; const ov::OutputVector external_connections{ main_flow, lora_input, - read_value1, - read_value2, - read_value3, + state_1, + state_2, + state_3, }; ov::ParameterVector subgraph_parameters; diff --git a/src/common/transformations/tests/common_optimizations/lora_subgraph_fusion.cpp b/src/common/transformations/tests/common_optimizations/lora_subgraph_fusion.cpp index 6557f763c6b368..2294668f49a6c5 100644 --- a/src/common/transformations/tests/common_optimizations/lora_subgraph_fusion.cpp +++ b/src/common/transformations/tests/common_optimizations/lora_subgraph_fusion.cpp @@ -20,21 +20,25 @@ using namespace ov; static constexpr auto netType = ov::element::f32; -std::pair create_states(const std::vector& shapes) { - ov::OutputVector read_values; +std::pair create_states(const std::vector& shapes, + const ov::element::Type& states_precision = netType) { + ov::OutputVector state_outs; ov::SinkVector assigns; size_t idx = 0; auto create_state = [&](const ov::PartialShape& shape) { - auto variable = - std::make_shared(ov::op::util::VariableInfo{shape, netType, std::to_string(idx++)}); + auto variable = std::make_shared( + ov::op::util::VariableInfo{shape, states_precision, std::to_string(idx++)}); auto read_value = std::make_shared(variable); auto assign = std::make_shared(read_value, variable); - read_values.push_back(read_value); assigns.push_back(assign); + if (states_precision == netType) + state_outs.push_back(read_value); + else + state_outs.push_back(std::make_shared(read_value, netType)); }; for (const auto& shape : shapes) create_state(shape); - return std::make_pair(read_values, assigns); + return std::make_pair(state_outs, assigns); } std::shared_ptr create_lora_subgraph(const ov::Output& main_flow, @@ -134,6 +138,47 @@ TEST_F(LoraSubgraphFusionMatMulTests, StandardPattern) { } } +TEST_F(LoraSubgraphFusionMatMulTests, StandardPatternWithConvert) { + { + auto param_lora = std::make_shared(netType, shape_x); + auto param_w = std::make_shared(netType, shape_w); + auto main_mm = std::make_shared(param_lora, param_w, false, true); + main_mm->set_friendly_name("main_mm"); + auto states = create_states({shape_state_1, shape_state_2, shape_state_3}, ov::element::f16); + auto lora_subgraph = create_lora_subgraph(main_mm, param_lora, states.first, false); + lora_subgraph->set_friendly_name("lora_subgraph"); + model = std::make_shared(OutputVector{lora_subgraph, main_mm}, + states.second, + ParameterVector{param_lora, param_w}); + } + { + auto param_lora = std::make_shared(netType, shape_x); + auto param_w = std::make_shared(netType, shape_w); + auto main_mm = std::make_shared(param_lora, param_w, false, true); + main_mm->set_friendly_name("main_mm"); + + auto inner_param_lora = std::make_shared(netType, shape_x); + auto inner_state_1 = std::make_shared(netType, shape_state_1); + auto inner_state_2 = std::make_shared(netType, shape_state_2); + auto inner_state_3 = std::make_shared(netType, shape_state_3); + auto inner_param_mm = std::make_shared(netType, main_mm->get_output_partial_shape(0)); + + ov::OutputVector states_outs{inner_state_1, inner_state_2, inner_state_3}; + auto lora_subgraph = create_lora_subgraph(inner_param_mm, inner_param_lora, states_outs, false); + lora_subgraph->set_friendly_name("lora_subgraph"); + ov::ParameterVector inner_params{inner_param_mm, inner_param_lora, inner_state_1, inner_state_2, inner_state_3}; + auto inner_model = std::make_shared(OutputVector{lora_subgraph}, inner_params); + + auto states = create_states({shape_state_1, shape_state_2, shape_state_3}, ov::element::f16); + ov::OutputVector lora_inputs{main_mm, param_lora, states.first[0], states.first[1], states.first[2]}; + auto lora = std::make_shared(lora_inputs, inner_model); + lora->set_friendly_name("lora_subgraph"); + + model_ref = + std::make_shared(OutputVector{lora, main_mm}, states.second, ParameterVector{param_lora, param_w}); + } +} + TEST_F(LoraSubgraphFusionMatMulTests, ReshaffledEltwiseInputs) { { auto param_lora = std::make_shared(netType, shape_x); diff --git a/src/common/util/include/openvino/util/weights_path.hpp b/src/common/util/include/openvino/util/weights_path.hpp index db97484be98d35..11215d8eee1304 100644 --- a/src/common/util/include/openvino/util/weights_path.hpp +++ b/src/common/util/include/openvino/util/weights_path.hpp @@ -11,5 +11,5 @@ namespace util { bool validate_weights_path(std::string& weights_path); -} // namespace ov -} // namespace util +} // namespace util +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/lora.cpp b/src/plugins/intel_cpu/src/nodes/lora.cpp index 2c69bc347b6139..52509cdfc44a13 100644 --- a/src/plugins/intel_cpu/src/nodes/lora.cpp +++ b/src/plugins/intel_cpu/src/nodes/lora.cpp @@ -47,8 +47,14 @@ void LoRA::selectOptimalPrimitiveDescriptor() { std::vector inConfs; std::vector graphInputConfig; - for (size_t i = 0; i < getParentEdges().size(); i++) { - auto desc = getParentOutputMemDesc(getParentEdgeAt(i)); + auto mainInputDesc = getParentOutputMemDesc(getParentEdgeAt(0)); + auto mainInputPrc = mainInputDesc->getPrecision(); // we have to align precision across all the inputs + + inConfs.emplace_back(mainInputDesc); + graphInputConfig.emplace_back(node::Input::InputConfig{mainInputDesc, true}); + + for (size_t i = 1; i < getParentEdges().size(); i++) { + auto desc = getParentOutputMemDesc(getParentEdgeAt(i))->cloneWithNewPrecision(mainInputPrc); inConfs.emplace_back(desc); graphInputConfig.emplace_back(node::Input::InputConfig{desc, true}); } @@ -87,7 +93,10 @@ void LoRA::createPrimitive() { std::vector inputMemory; for (size_t i = 0; i < getOriginalInputsNumber(); i++) { - inputMemory.emplace_back(getSrcMemoryAtPort(i)); + auto srcEdgeMem = getSrcMemoryAtPort(i); + auto mem = std::make_shared(getEngine(), srcEdgeMem->getDescPtr(), srcEdgeMem->getMemoryBlock()); + subgraphMemoryPtrs.push_back(mem); + inputMemory.emplace_back(std::move(mem)); } CPU_NODE_ASSERT(getOriginalOutputsNumber() == m_graph.GetOutputNodesMap().size(), @@ -105,6 +114,13 @@ void LoRA::executeDynamicImpl(dnnl::stream strm) { execute(strm); } +void LoRA::prepareParams() { + for (size_t i = 0; i < getOriginalInputsNumber(); i++) { + // since the external and internal descriptors are compatible, we may pass the descriptor + subgraphMemoryPtrs[i]->redefineDesc(getSrcMemoryAtPort(i)->getDescPtr()); + } +} + } // namespace node } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/lora.h b/src/plugins/intel_cpu/src/nodes/lora.h index 89a1bc15c2bf17..27701daf9034f2 100644 --- a/src/plugins/intel_cpu/src/nodes/lora.h +++ b/src/plugins/intel_cpu/src/nodes/lora.h @@ -21,18 +21,16 @@ class LoRA : public Node { return getType() == Type::LoRA; } - bool needPrepareParams() const override { - return false; - } - void getSupportedDescriptors() override{}; void selectOptimalPrimitiveDescriptor() override; void createPrimitive() override; + void prepareParams() override; void execute(dnnl::stream) override; void executeDynamicImpl(dnnl::stream strm) override; private: std::shared_ptr m_body; + std::vector subgraphMemoryPtrs; Graph m_graph; }; diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/lora_pattern.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/lora_pattern.cpp index 4f4b05ef56750c..20620e42ceddc7 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/lora_pattern.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/lora_pattern.cpp @@ -16,10 +16,85 @@ constexpr auto t4_name = "lora/MatMul.B"; constexpr auto t5_name = "lora/MatMul.alpha"; constexpr auto t6_name = "lora/MatMul.A"; constexpr auto netType = ov::element::f32; + +enum class StatesPolicy { + EMPTY_TENSORS, + RANDOM_TENSORS, + UNDEFINED +}; +std::ostream &operator<<(std::ostream& os, StatesPolicy states_policy) { + switch (states_policy) { + case StatesPolicy::EMPTY_TENSORS: + return os << "empty_tensors"; + case StatesPolicy::RANDOM_TENSORS: + return os << "random_tensors"; + case StatesPolicy::UNDEFINED: + return os << "undefined"; + default: + OPENVINO_THROW("Unexpected states policy"); + } +} + } // namespace -class LoraPatternBaseCPUTest : public SubgraphBaseTest { +using LoraPatternParams = std::tuple; // states filling policy + +class LoraPatternBaseCPUTest : public SubgraphBaseTest, public testing::WithParamInterface { +public: +static std::string getTestCaseName(testing::TestParamInfo obj) { + ov::element::Type states_precision; + StatesPolicy states_policy; + std::tie(states_precision, states_policy) = obj.param; + + std::ostringstream result; + result << "states_precision=" << states_precision << "_states_policy=" << states_policy; + return result.str(); + } + + void SetUp() override { + targetDevice = ov::test::utils::DEVICE_CPU; + std::tie(states_precision, states_policy) = this->GetParam(); + init_function(); + } + protected: + virtual void init_function() = 0; + + std::pair create_states(const std::vector& shapes, + const std::vector names) { + ov::OutputVector state_outs; + ov::SinkVector assigns; + auto create_state = [&](const ov::PartialShape& shape, const std::string name) { + auto variable = std::make_shared( + ov::op::util::VariableInfo{shape, states_precision, name}); + auto read_value = std::make_shared(variable); + auto assign = std::make_shared(read_value, variable); + assigns.push_back(assign); + if (states_precision == netType) + state_outs.push_back(read_value); + else + state_outs.push_back(std::make_shared(read_value, netType)); + }; + OPENVINO_ASSERT(shapes.size() == names.size()); + for (size_t i = 0; i < shapes.size(); ++i) + create_state(shapes[i], names[i]); + return std::make_pair(state_outs, assigns); + } + + void run_test() { + switch (states_policy) { + case StatesPolicy::EMPTY_TENSORS: + run_test_empty_tensors(); + break; + case StatesPolicy::RANDOM_TENSORS: + run_test_random_tensors(); + break; + default: + OPENVINO_THROW("Unexpected states policy: ", states_policy); + } + } + void run_test_empty_tensors() { compile_model(); inferRequest = compiledModel.create_infer_request(); @@ -92,7 +167,7 @@ class LoraPatternBaseCPUTest : public SubgraphBaseTest { using ov::test::utils::InputGenerateData; const auto& shape = stateShapes.at(item.get_name()); auto tensor = - ov::test::utils::create_and_fill_tensor(netType, shape, InputGenerateData{0, 10, 1, i}); + ov::test::utils::create_and_fill_tensor(states_precision, shape, InputGenerateData{0, 10, 1, i}); item.set_state(tensor); auto itr = std::find_if(refStates.begin(), refStates.end(), [&](const ov::VariableState& state) { return state.get_name() == item.get_name(); @@ -112,17 +187,18 @@ class LoraPatternBaseCPUTest : public SubgraphBaseTest { auto tx_result_ref = inferRequestRef.get_tensor(outputs[0]); auto tz_result_ref = inferRequestRef.get_tensor(outputs[1]); - ov::test::utils::compare(tx_result, tx_result_ref, 1e-4, 1e-4); - ov::test::utils::compare(tz_result, tz_result_ref, 1e-4, 1e-4); + ov::test::utils::compare(tx_result_ref, tx_result, 1e-4, 1e-4); + ov::test::utils::compare(tz_result_ref, tz_result, 1e-4, 1e-4); } } + + StatesPolicy states_policy = StatesPolicy::UNDEFINED; + ov::element::Type states_precision = ov::element::undefined; }; class LoraPatternMatmulCPUTest : public LoraPatternBaseCPUTest { -public: - void SetUp() override { - targetDevice = ov::test::utils::DEVICE_CPU; - +protected: + void init_function() override { ov::PartialShape shape_x = {-1, -1, K}; ov::PartialShape shape_w = {N, K}; @@ -133,25 +209,12 @@ class LoraPatternMatmulCPUTest : public LoraPatternBaseCPUTest { auto tx = std::make_shared(param_y, param_w, false, true); // LoRA parameters from states - auto variable_t4 = std::make_shared( - ov::op::util::VariableInfo{ov::PartialShape({N, -1}), netType, t4_name}); - auto t4 = std::make_shared(variable_t4); - auto t4_assign = std::make_shared(t4, variable_t4); - - auto variable_t5 = std::make_shared( - ov::op::util::VariableInfo{ov::PartialShape({1, -1}), netType, t5_name}); - auto t5 = std::make_shared(variable_t5); - auto t5_assign = std::make_shared(t5, variable_t5); - - auto variable_t6 = std::make_shared( - ov::op::util::VariableInfo{ov::PartialShape({-1, K}), netType, t6_name}); - auto t6 = std::make_shared(variable_t6); - auto t6_assign = std::make_shared(t6, variable_t6); + auto states = create_states({{N, -1}, {1, -1}, {-1, K}}, {t4_name, t5_name, t6_name}); // Apply LoRA parameters to the current activations - auto t5810 = std::make_shared(param_y, t6, false, true); - auto t5811 = std::make_shared(t5810, t5); - auto t5812 = std::make_shared(t5811, t4, false, true); + auto t5810 = std::make_shared(param_y, states.first[2], false, true); + auto t5811 = std::make_shared(t5810, states.first[1]); + auto t5812 = std::make_shared(t5811, states.first[0], false, true); // Mix LoRA part into normally computed activations after the "main" MatMul auto tz = std::make_shared(tx, t5812); @@ -160,19 +223,17 @@ class LoraPatternMatmulCPUTest : public LoraPatternBaseCPUTest { auto result_z = std::make_shared(tz); function = std::make_shared(ov::ResultVector({result_x, result_z}), - ov::SinkVector({t4_assign, t5_assign, t6_assign}), + states.second, ov::ParameterVector({param_y, param_w})); } -protected: static constexpr size_t K = 563ul; // Weights matrix K dimension static constexpr size_t N = 2048ul; // Weights matrix N dimension }; class LoraPatternConvolutionCPUTest : public LoraPatternBaseCPUTest { public: - void SetUp() override { - targetDevice = ov::test::utils::DEVICE_CPU; + void init_function() override { ov::PartialShape shape_x = {-1, num_channels, -1, -1}; auto param_y = std::make_shared(netType, shape_x); @@ -189,29 +250,16 @@ class LoraPatternConvolutionCPUTest : public LoraPatternBaseCPUTest { num_channels); // LoRA parameters from states - auto variable_t4 = std::make_shared( - ov::op::util::VariableInfo{ov::PartialShape({num_channels, -1}), netType, t4_name}); - auto t4 = std::make_shared(variable_t4); - auto t4_assign = std::make_shared(t4, variable_t4); - - auto variable_t5 = std::make_shared( - ov::op::util::VariableInfo{ov::PartialShape({1, -1}), netType, t5_name}); - auto t5 = std::make_shared(variable_t5); - auto t5_assign = std::make_shared(t5, variable_t5); - - auto variable_t6 = std::make_shared( - ov::op::util::VariableInfo{ov::PartialShape({-1, num_channels}), netType, t6_name}); - auto t6 = std::make_shared(variable_t6); - auto t6_assign = std::make_shared(t6, variable_t6); + auto states = create_states({{num_channels, -1}, {1, -1}, {-1, num_channels}}, {t4_name, t5_name, t6_name}); // LoRA pattern with additional Transposes to move channel dimensions into positions where MatMul can be applied auto t4940 = std::make_shared(ov::element::i32, ov::Shape{4}, std::vector{2, 3, 0, 1}); auto t4941 = std::make_shared(param_y, t4940); - auto t4942 = std::make_shared(t4941, t6, false, true); - auto t4943 = std::make_shared(t4942, t5); - auto t4944 = std::make_shared(t4943, t4, false, true); + auto t4942 = std::make_shared(t4941, states.first[2], false, true); + auto t4943 = std::make_shared(t4942, states.first[1]); + auto t4944 = std::make_shared(t4943, states.first[0], false, true); auto t4945 = std::make_shared(ov::element::i32, ov::Shape{4}, std::vector{2, 3, 0, 1}); @@ -224,7 +272,7 @@ class LoraPatternConvolutionCPUTest : public LoraPatternBaseCPUTest { auto result_z = std::make_shared(tz); function = std::make_shared(ov::ResultVector({result_x, result_z}), - ov::SinkVector({t4_assign, t5_assign, t6_assign}), + states.second, ov::ParameterVector({param_y})); } @@ -232,35 +280,34 @@ class LoraPatternConvolutionCPUTest : public LoraPatternBaseCPUTest { static constexpr size_t num_channels = 64ul; }; -TEST_F(LoraPatternMatmulCPUTest, smoke_LoRA_CPU_MatMul_empty) { +TEST_P(LoraPatternMatmulCPUTest, CompareWithRefs) { targetStaticShapes = {{{{1, 20, K}}, {{N, K}}}}; - run_test_empty_tensors(); + run_test(); CPUTestUtils::CheckNumberOfNodesWithType(compiledModel, "LoRA", 1); CPUTestUtils::CheckNumberOfNodesWithType(compiledModel, "MatMul", 1); } -TEST_F(LoraPatternConvolutionCPUTest, smoke_LoRA_CPU_Conv_empty) { +TEST_P(LoraPatternConvolutionCPUTest, CompareWithRefs) { targetStaticShapes = {{{1, num_channels, 10, 15}}}; - run_test_empty_tensors(); + run_test(); CPUTestUtils::CheckNumberOfNodesWithType(compiledModel, "LoRA", 1); CPUTestUtils::CheckNumberOfNodesWithType(compiledModel, "MatMul", 0); } -TEST_F(LoraPatternMatmulCPUTest, smoke_LoRA_CPU_MatMul_random) { - GTEST_SKIP(); - targetStaticShapes = {{{{1, 20, K}}, {{N, K}}}}; - run_test_random_tensors(); - CPUTestUtils::CheckNumberOfNodesWithType(compiledModel, "LoRA", 1); - CPUTestUtils::CheckNumberOfNodesWithType(compiledModel, "MatMul", 1); -} +const ov::element::TypeVector states_precisions {ov::element::f32, ov::element::f16}; +const std::vector states_policies {StatesPolicy::EMPTY_TENSORS, StatesPolicy::RANDOM_TENSORS}; -TEST_F(LoraPatternConvolutionCPUTest, smoke_LoRA_CPU_Conv_random) { - GTEST_SKIP(); - targetStaticShapes = {{{1, num_channels, 10, 15}}}; - run_test_random_tensors(); - CPUTestUtils::CheckNumberOfNodesWithType(compiledModel, "LoRA", 1); - CPUTestUtils::CheckNumberOfNodesWithType(compiledModel, "MatMul", 0); -} +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_LoRA_CPU_MatMul, LoraPatternMatmulCPUTest, + ::testing::Combine( + ::testing::ValuesIn(states_precisions), + ::testing::ValuesIn(states_policies)), + LoraPatternBaseCPUTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_LoRA_CPU_Conv, LoraPatternConvolutionCPUTest, + ::testing::Combine( + ::testing::ValuesIn(states_precisions), + ::testing::ValuesIn(states_policies)), + LoraPatternBaseCPUTest::getTestCaseName); } // namespace test } // namespace ov \ No newline at end of file From 7c73c50ab843ba7e166a3273ff7bdc5643b081c4 Mon Sep 17 00:00:00 2001 From: Karol Blaszczak Date: Tue, 5 Nov 2024 10:34:39 +0100 Subject: [PATCH 04/93] [DOCS] unhide columns one more change mstr (#27396) --- docs/sphinx_setup/_static/js/openVinoDataTables.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sphinx_setup/_static/js/openVinoDataTables.js b/docs/sphinx_setup/_static/js/openVinoDataTables.js index ea663394de5004..c65748065b2ad2 100644 --- a/docs/sphinx_setup/_static/js/openVinoDataTables.js +++ b/docs/sphinx_setup/_static/js/openVinoDataTables.js @@ -4,7 +4,7 @@ $(document).ready(function () { if(pageTitle.includes('Most Efficient Large Language Models for AI PC')) { columnDefs= [ - { "visible": false, "targets": [1] } + { "visible": false, "targets": [3,4,6] } ] } else From c2e42fab2c08ccd37304f5bddc1b62f5228abfb3 Mon Sep 17 00:00:00 2001 From: Sebastian Golebiewski Date: Tue, 5 Nov 2024 11:58:21 +0100 Subject: [PATCH 05/93] [DOCS] Updating notebooks (#27401) Updating the `Interactive Tutorials` section. --- docs/nbdoc/consts.py | 2 +- .../3D-pose-estimation-with-output.rst | 210 +++---- ...-segmentation-point-clouds-with-output.rst | 4 +- .../action-recognition-webcam-with-output.rst | 9 +- ...on-recognition-webcam-with-output_22_0.png | 3 + docs/notebooks/all_notebooks_paths.txt | 2 + ...-lightweight-text-to-image-with-output.rst | 53 +- docs/notebooks/async-api-with-output.rst | 25 +- .../async-api-with-output_23_0.png | 4 +- docs/notebooks/auto-device-with-output.rst | 50 +- .../auto-device-with-output_27_0.png | 4 +- .../auto-device-with-output_28_0.png | 4 +- docs/notebooks/catvton-with-output.rst | 360 ++++++++++++ ...p-zero-shot-classification-with-output.rst | 1 - .../convert-to-openvino-with-output.rst | 38 +- .../convnext-classification-with-output.rst | 2 +- ...segmentation-quantize-nncf-with-output.rst | 68 +-- ...ntation-quantize-nncf-with-output_37_1.png | 4 +- ...ddcolor-image-colorization-with-output.rst | 56 +- .../depth-anything-v2-with-output.rst | 47 +- docs/notebooks/depth-anything-with-output.rst | 32 +- .../detectron2-to-openvino-with-output.rst | 180 +++--- ...etectron2-to-openvino-with-output_22_0.jpg | 4 +- ...etectron2-to-openvino-with-output_22_0.png | 4 +- ...etectron2-to-openvino-with-output_32_0.jpg | 4 +- ...etectron2-to-openvino-with-output_32_0.png | 4 +- .../distil-whisper-asr-with-output.rst | 3 +- ...rt-sequence-classification-with-output.rst | 70 ++- ...ly-2-instruction-following-with-output.rst | 43 +- ...micrafter-animating-images-with-output.rst | 99 ++-- docs/notebooks/efficient-sam-with-output.rst | 68 +-- .../efficient-sam-with-output_17_1.png | 4 +- .../efficient-sam-with-output_25_1.png | 4 +- .../efficient-sam-with-output_36_1.png | 4 +- .../encodec-audio-compression-with-output.rst | 18 +- .../fast-segment-anything-with-output.rst | 20 +- docs/notebooks/florence2-with-output.rst | 39 +- .../florence2-with-output_18_0.png | 4 +- .../flux.1-image-generation-with-output.rst | 213 ++++--- ...ux.1-image-generation-with-output_16_1.jpg | 3 + ...ux.1-image-generation-with-output_16_1.png | 3 + ...ux.1-image-generation-with-output_20_1.jpg | 3 - ...ux.1-image-generation-with-output_20_1.png | 3 - .../freevc-voice-conversion-with-output.rst | 30 +- docs/notebooks/gpu-device-with-output.rst | 122 +--- .../grounded-segment-anything-with-output.rst | 50 +- .../hello-segmentation-with-output.rst | 4 +- .../hugging-face-hub-with-output.rst | 55 +- ...lassification-quantization-with-output.rst | 52 +- docs/notebooks/instant-id-with-output.rst | 1 + docs/notebooks/internvl2-with-output.rst | 20 +- docs/notebooks/jina-clip-with-output.rst | 14 +- .../knowledge-graphs-conve-with-output.rst | 28 +- ...modal-large-language-model-with-output.rst | 30 +- ...-large-language-model-with-output_29_1.jpg | 4 +- ...-large-language-model-with-output_29_1.png | 4 +- ...-large-language-model-with-output_48_1.png | 4 +- ...l-large-language-model-with-output_8_0.jpg | 4 +- ...l-large-language-model-with-output_8_0.png | 4 +- .../language-quantize-bert-with-output.rst | 64 +-- ...stency-models-optimum-demo-with-output.rst | 110 +--- ...y-models-optimum-demo-with-output_15_1.jpg | 4 +- ...y-models-optimum-demo-with-output_15_1.png | 4 +- ...cy-models-optimum-demo-with-output_8_1.jpg | 4 +- ...cy-models-optimum-demo-with-output_8_1.png | 4 +- ...a-multimodal-chatbot-genai-with-output.rst | 28 +- ...multimodal-chatbot-optimum-with-output.rst | 27 +- ...lm-agent-functioncall-qwen-with-output.rst | 36 +- .../llm-agent-rag-llamaindex-with-output.rst | 24 +- .../notebooks/llm-agent-react-with-output.rst | 107 ++-- docs/notebooks/meter-reader-with-output.rst | 2 +- ...nicpm-v-multimodal-chatbot-with-output.rst | 311 ++--------- .../mobileclip-video-search-with-output.rst | 163 +++--- ...bilevlm-language-assistant-with-output.rst | 22 +- .../music-generation-with-output.rst | 20 +- ...o-llava-multimodal-chatbot-with-output.rst | 385 +------------ .../object-detection-with-output.rst | 413 +++----------- .../object-detection-with-output_13_0.png | 3 + .../object-detection-with-output_19_0.png | 3 - docs/notebooks/openvino-api-with-output.rst | 16 +- docs/notebooks/openvoice-with-output.rst | 77 +-- ...ical-character-recognition-with-output.rst | 312 ++--------- ...haracter-recognition-with-output_13_0.png} | 0 ...character-recognition-with-output_23_0.png | 3 + ...haracter-recognition-with-output_25_0.jpg} | 0 ...haracter-recognition-with-output_25_0.png} | 0 ...aracter-recognition-with-output_25_10.jpg} | 0 ...aracter-recognition-with-output_25_10.png} | 0 ...haracter-recognition-with-output_25_2.jpg} | 0 ...haracter-recognition-with-output_25_2.png} | 0 ...haracter-recognition-with-output_25_4.jpg} | 0 ...haracter-recognition-with-output_25_4.png} | 0 ...haracter-recognition-with-output_25_6.jpg} | 0 ...haracter-recognition-with-output_25_6.png} | 0 ...haracter-recognition-with-output_25_8.jpg} | 0 ...haracter-recognition-with-output_25_8.png} | 0 ...character-recognition-with-output_26_0.png | 3 - .../optimize-preprocessing-with-output.rst | 10 +- .../paddle-ocr-webcam-with-output.rst | 6 +- .../paddle-ocr-webcam-with-output_30_0.png | 4 +- ...to-openvino-classification-with-output.rst | 18 +- .../parler-tts-text-to-speech-with-output.rst | 80 +-- .../notebooks/person-tracking-with-output.rst | 57 +- .../person-tracking-with-output_25_0.png | 4 +- docs/notebooks/phi-3-vision-with-output.rst | 51 +- docs/notebooks/photo-maker-with-output.rst | 47 +- docs/notebooks/pixart-with-output.rst | 51 +- docs/notebooks/pixtral-with-output.rst | 41 +- .../pose-estimation-with-output_22_0.png | 4 +- .../pytorch-onnx-to-openvino-with-output.rst | 6 +- ...training-quantization-nncf-with-output.rst | 104 ++-- ...uantization-aware-training-with-output.rst | 78 +-- ...on-sparsity-aware-training-with-output.rst | 358 ++++++------ .../pytorch-to-openvino-with-output.rst | 14 +- docs/notebooks/qwen2-audio-with-output.rst | 35 +- docs/notebooks/qwen2-vl-with-output.rst | 69 +-- .../rmbg-background-removal-with-output.rst | 18 +- .../segment-anything-2-image-with-output.rst | 29 +- ...-shot-image-classification-with-output.rst | 13 +- ...-image-classification-with-output_24_1.png | 4 +- .../speculative-sampling-with-output.rst | 516 +++++++---------- ...tion-quantization-wav2vec2-with-output.rst | 160 +++--- ...hbrain-emotion-recognition-with-output.rst | 74 +-- docs/notebooks/stable-audio-with-output.rst | 109 ++-- ...e-cascade-image-generation-with-output.rst | 20 +- ...cade-image-generation-with-output_29_2.jpg | 4 +- ...cade-image-generation-with-output_29_2.png | 4 +- ...table-diffusion-ip-adapter-with-output.rst | 43 +- ...-diffusion-ip-adapter-with-output_22_1.png | 4 +- ...-diffusion-ip-adapter-with-output_25_0.png | 4 +- ...-diffusion-ip-adapter-with-output_28_0.png | 4 +- ...fusion-torchdynamo-backend-with-output.rst | 3 +- docs/notebooks/style-transfer-with-output.rst | 2 +- .../style-transfer-with-output_25_0.png | 4 +- .../table-question-answering-with-output.rst | 59 +- ...classification-to-openvino-with-output.rst | 2 +- ...e-segmentation-to-openvino-with-output.rst | 2 +- ...mentation-to-openvino-with-output_39_0.png | 4 +- ...ject-detection-to-openvino-with-output.rst | 8 +- ...detection-to-openvino-with-output_38_0.png | 4 +- ...uantization-aware-training-with-output.rst | 54 +- ...tflite-selfie-segmentation-with-output.rst | 2 +- ...e-selfie-segmentation-with-output_33_0.png | 4 +- .../tflite-to-openvino-with-output.rst | 20 +- docs/notebooks/typo-detector-with-output.rst | 13 +- .../vision-background-removal-with-output.rst | 8 +- .../vision-monodepth-with-output.rst | 6 +- docs/notebooks/wav2lip-with-output.rst | 74 +-- .../whisper-asr-genai-with-output.rst | 39 +- ...isper-subtitles-generation-with-output.rst | 19 +- ...uerstchen-image-generation-with-output.rst | 4 +- ...ov11-instance-segmentation-with-output.rst | 60 +- ...instance-segmentation-with-output_46_0.png | 4 +- ...yolov11-keypoint-detection-with-output.rst | 58 +- ...11-keypoint-detection-with-output_43_0.png | 4 +- .../yolov11-object-detection-with-output.rst | 60 +- ...ov11-object-detection-with-output_43_0.png | 4 +- .../yolov8-object-detection-with-output.rst | 524 +++++++++++++++--- ...lov8-object-detection-with-output_16_1.jpg | 3 - ...lov8-object-detection-with-output_16_1.png | 4 +- ...lov8-object-detection-with-output_38_1.jpg | 3 - ...lov8-object-detection-with-output_38_1.png | 3 - ...lov8-object-detection-with-output_43_1.png | 3 + ...lov8-object-detection-with-output_61_0.jpg | 3 - ...lov8-object-detection-with-output_61_0.png | 3 - ...lov8-object-detection-with-output_67_0.png | 3 - ...lov8-object-detection-with-output_70_0.png | 3 + ...lov8-object-detection-with-output_76_0.png | 3 + ...olov8-object-detection-with-output_9_1.jpg | 3 - ...olov8-object-detection-with-output_9_1.png | 3 - ...olov8-object-detection-with-output_9_3.png | 3 + .../yolov9-optimization-with-output.rst | 60 +- .../yolov9-optimization-with-output_36_0.png | 4 +- 173 files changed, 3393 insertions(+), 4145 deletions(-) create mode 100644 docs/notebooks/action-recognition-webcam-with-output_files/action-recognition-webcam-with-output_22_0.png create mode 100644 docs/notebooks/catvton-with-output.rst create mode 100644 docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.jpg create mode 100644 docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.png delete mode 100644 docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.jpg delete mode 100644 docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.png create mode 100644 docs/notebooks/object-detection-with-output_files/object-detection-with-output_13_0.png delete mode 100644 docs/notebooks/object-detection-with-output_files/object-detection-with-output_19_0.png rename docs/notebooks/optical-character-recognition-with-output_files/{optical-character-recognition-with-output_16_0.png => optical-character-recognition-with-output_13_0.png} (100%) create mode 100644 docs/notebooks/optical-character-recognition-with-output_files/optical-character-recognition-with-output_23_0.png rename docs/notebooks/optical-character-recognition-with-output_files/{optical-character-recognition-with-output_28_0.jpg => optical-character-recognition-with-output_25_0.jpg} (100%) rename docs/notebooks/optical-character-recognition-with-output_files/{optical-character-recognition-with-output_28_0.png => optical-character-recognition-with-output_25_0.png} (100%) rename docs/notebooks/optical-character-recognition-with-output_files/{optical-character-recognition-with-output_28_10.jpg => optical-character-recognition-with-output_25_10.jpg} (100%) rename docs/notebooks/optical-character-recognition-with-output_files/{optical-character-recognition-with-output_28_10.png => optical-character-recognition-with-output_25_10.png} (100%) rename docs/notebooks/optical-character-recognition-with-output_files/{optical-character-recognition-with-output_28_2.jpg => optical-character-recognition-with-output_25_2.jpg} (100%) rename docs/notebooks/optical-character-recognition-with-output_files/{optical-character-recognition-with-output_28_2.png => optical-character-recognition-with-output_25_2.png} (100%) rename docs/notebooks/optical-character-recognition-with-output_files/{optical-character-recognition-with-output_28_4.jpg => optical-character-recognition-with-output_25_4.jpg} (100%) rename docs/notebooks/optical-character-recognition-with-output_files/{optical-character-recognition-with-output_28_4.png => optical-character-recognition-with-output_25_4.png} (100%) rename docs/notebooks/optical-character-recognition-with-output_files/{optical-character-recognition-with-output_28_6.jpg => optical-character-recognition-with-output_25_6.jpg} (100%) rename docs/notebooks/optical-character-recognition-with-output_files/{optical-character-recognition-with-output_28_6.png => optical-character-recognition-with-output_25_6.png} (100%) rename docs/notebooks/optical-character-recognition-with-output_files/{optical-character-recognition-with-output_28_8.jpg => optical-character-recognition-with-output_25_8.jpg} (100%) rename docs/notebooks/optical-character-recognition-with-output_files/{optical-character-recognition-with-output_28_8.png => optical-character-recognition-with-output_25_8.png} (100%) delete mode 100644 docs/notebooks/optical-character-recognition-with-output_files/optical-character-recognition-with-output_26_0.png delete mode 100644 docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.jpg delete mode 100644 docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.jpg delete mode 100644 docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.png create mode 100644 docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_43_1.png delete mode 100644 docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.jpg delete mode 100644 docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.png delete mode 100644 docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_67_0.png create mode 100644 docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_70_0.png create mode 100644 docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_76_0.png delete mode 100644 docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.jpg delete mode 100644 docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.png create mode 100644 docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_3.png diff --git a/docs/nbdoc/consts.py b/docs/nbdoc/consts.py index 75c8111bafcc6b..bfad4b042e5359 100644 --- a/docs/nbdoc/consts.py +++ b/docs/nbdoc/consts.py @@ -6,7 +6,7 @@ repo_owner = "openvinotoolkit" repo_name = "openvino_notebooks" repo_branch = "tree/main" -artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20241022220806/dist/rst_files/" +artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20241104220807/dist/rst_files/" blacklisted_extensions = ['.xml', '.bin'] notebooks_repo = "https://github.com/openvinotoolkit/openvino_notebooks/blob/latest/" notebooks_binder = "https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=" diff --git a/docs/notebooks/3D-pose-estimation-with-output.rst b/docs/notebooks/3D-pose-estimation-with-output.rst index e5e8e3813b5173..f39aa93b36851d 100644 --- a/docs/notebooks/3D-pose-estimation-with-output.rst +++ b/docs/notebooks/3D-pose-estimation-with-output.rst @@ -93,7 +93,7 @@ Lab instead.** .. code:: ipython3 - %pip install pythreejs "openvino-dev>=2024.0.0" "opencv-python" "torch" "onnx<1.16.2" --extra-index-url https://download.pytorch.org/whl/cpu + %pip install pythreejs "openvino>=2024.4.0" "opencv-python" "torch" "tqdm" --extra-index-url https://download.pytorch.org/whl/cpu .. parsed-literal:: @@ -101,85 +101,75 @@ Lab instead.** Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cpu Collecting pythreejs Using cached pythreejs-2.4.2-py3-none-any.whl.metadata (5.4 kB) - Collecting openvino-dev>=2024.0.0 - Using cached openvino_dev-2024.4.0-16579-py3-none-any.whl.metadata (16 kB) + Collecting openvino>=2024.4.0 + Using cached openvino-2024.4.0-16579-cp38-cp38-manylinux2014_x86_64.whl.metadata (8.3 kB) Collecting opencv-python Using cached opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (20 kB) Collecting torch Using cached https://download.pytorch.org/whl/cpu/torch-2.4.1%2Bcpu-cp38-cp38-linux_x86_64.whl (194.9 MB) - Collecting onnx<1.16.2 - Using cached onnx-1.16.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (16 kB) - Requirement already satisfied: ipywidgets>=7.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (8.1.5) + Collecting tqdm + Using cached tqdm-4.66.6-py3-none-any.whl.metadata (57 kB) + Requirement already satisfied: ipywidgets>=7.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (8.1.5) Collecting ipydatawidgets>=1.1.1 (from pythreejs) Using cached ipydatawidgets-4.3.5-py2.py3-none-any.whl.metadata (1.4 kB) Collecting numpy (from pythreejs) Using cached numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.6 kB) - Requirement already satisfied: traitlets in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (5.14.3) - Requirement already satisfied: defusedxml>=0.7.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (0.7.1) - Collecting networkx<=3.1.0 (from openvino-dev>=2024.0.0) - Using cached networkx-3.1-py3-none-any.whl.metadata (5.3 kB) - Collecting openvino-telemetry>=2023.2.1 (from openvino-dev>=2024.0.0) + Requirement already satisfied: traitlets in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (5.14.3) + Collecting openvino-telemetry>=2023.2.1 (from openvino>=2024.4.0) Using cached openvino_telemetry-2024.1.0-py3-none-any.whl.metadata (2.3 kB) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (24.1) - Requirement already satisfied: pyyaml>=5.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (6.0.2) - Requirement already satisfied: requests>=2.25.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (2.32.0) - Collecting openvino==2024.4.0 (from openvino-dev>=2024.0.0) - Using cached openvino-2024.4.0-16579-cp38-cp38-manylinux2014_x86_64.whl.metadata (8.3 kB) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2024.4.0) (24.1) Collecting filelock (from torch) Using cached filelock-3.16.1-py3-none-any.whl.metadata (2.9 kB) - Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (4.12.2) + Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (4.12.2) Collecting sympy (from torch) Using cached sympy-1.13.3-py3-none-any.whl.metadata (12 kB) - Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1.4) + Collecting networkx (from torch) + Using cached https://download.pytorch.org/whl/networkx-3.2.1-py3-none-any.whl (1.6 MB) + Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1.4) Collecting fsspec (from torch) Using cached fsspec-2024.10.0-py3-none-any.whl.metadata (11 kB) - Collecting protobuf>=3.20.2 (from onnx<1.16.2) - Using cached protobuf-5.28.2-cp38-abi3-manylinux2014_x86_64.whl.metadata (592 bytes) Collecting traittypes>=0.2.0 (from ipydatawidgets>=1.1.1->pythreejs) Using cached traittypes-0.2.1-py2.py3-none-any.whl.metadata (1.0 kB) - Requirement already satisfied: comm>=0.1.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (0.2.2) - Requirement already satisfied: ipython>=6.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (8.12.3) - Requirement already satisfied: widgetsnbextension~=4.0.12 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (4.0.13) - Requirement already satisfied: jupyterlab-widgets~=3.0.12 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (3.0.13) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (3.4.0) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (3.10) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (2.2.3) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (2024.8.30) - Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch) (2.1.5) + Requirement already satisfied: comm>=0.1.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (0.2.2) + Requirement already satisfied: ipython>=6.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (8.12.3) + Requirement already satisfied: widgetsnbextension~=4.0.12 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (4.0.13) + Requirement already satisfied: jupyterlab-widgets~=3.0.12 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (3.0.13) + Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch) (2.1.5) + INFO: pip is looking at multiple versions of networkx to determine which version is compatible with other requirements. This could take a while. + Collecting networkx (from torch) + Using cached networkx-3.1-py3-none-any.whl.metadata (5.3 kB) Collecting mpmath<1.4,>=1.1.0 (from sympy->torch) Using cached https://download.pytorch.org/whl/mpmath-1.3.0-py3-none-any.whl (536 kB) - Requirement already satisfied: backcall in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.0) - Requirement already satisfied: decorator in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (5.1.1) - Requirement already satisfied: jedi>=0.16 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.19.1) - Requirement already satisfied: matplotlib-inline in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.1.7) - Requirement already satisfied: pickleshare in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.5) - Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (3.0.48) - Requirement already satisfied: pygments>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.18.0) - Requirement already satisfied: stack-data in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.6.3) - Requirement already satisfied: pexpect>4.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (4.9.0) - Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.8.4) - Requirement already satisfied: ptyprocess>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.0) - Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.13) - Requirement already satisfied: executing>=1.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.1.0) - Requirement already satisfied: asttokens>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.4.1) - Requirement already satisfied: pure-eval in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.3) - Requirement already satisfied: six>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (1.16.0) + Requirement already satisfied: backcall in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.0) + Requirement already satisfied: decorator in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (5.1.1) + Requirement already satisfied: jedi>=0.16 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.19.1) + Requirement already satisfied: matplotlib-inline in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.1.7) + Requirement already satisfied: pickleshare in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.5) + Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (3.0.48) + Requirement already satisfied: pygments>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.18.0) + Requirement already satisfied: stack-data in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.6.3) + Requirement already satisfied: pexpect>4.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (4.9.0) + Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.8.4) + Requirement already satisfied: ptyprocess>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.0) + Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.13) + Requirement already satisfied: executing>=1.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.1.0) + Requirement already satisfied: asttokens>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.4.1) + Requirement already satisfied: pure-eval in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.3) + Requirement already satisfied: six>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (1.16.0) Using cached pythreejs-2.4.2-py3-none-any.whl (3.4 MB) - Using cached openvino_dev-2024.4.0-16579-py3-none-any.whl (4.7 MB) Using cached openvino-2024.4.0-16579-cp38-cp38-manylinux2014_x86_64.whl (42.6 MB) Using cached opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (62.5 MB) - Using cached onnx-1.16.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (15.9 MB) + Using cached tqdm-4.66.6-py3-none-any.whl (78 kB) Using cached ipydatawidgets-4.3.5-py2.py3-none-any.whl (271 kB) - Using cached networkx-3.1-py3-none-any.whl (2.1 MB) Using cached numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB) Using cached openvino_telemetry-2024.1.0-py3-none-any.whl (23 kB) - Using cached protobuf-5.28.2-cp38-abi3-manylinux2014_x86_64.whl (316 kB) Using cached filelock-3.16.1-py3-none-any.whl (16 kB) Using cached fsspec-2024.10.0-py3-none-any.whl (179 kB) + Using cached networkx-3.1-py3-none-any.whl (2.1 MB) Using cached sympy-1.13.3-py3-none-any.whl (6.2 MB) Using cached traittypes-0.2.1-py2.py3-none-any.whl (8.6 kB) - Installing collected packages: openvino-telemetry, mpmath, traittypes, sympy, protobuf, numpy, networkx, fsspec, filelock, torch, openvino, opencv-python, onnx, openvino-dev, ipydatawidgets, pythreejs - Successfully installed filelock-3.16.1 fsspec-2024.10.0 ipydatawidgets-4.3.5 mpmath-1.3.0 networkx-3.1 numpy-1.24.4 onnx-1.16.1 opencv-python-4.10.0.84 openvino-2024.4.0 openvino-dev-2024.4.0 openvino-telemetry-2024.1.0 protobuf-5.28.2 pythreejs-2.4.2 sympy-1.13.3 torch-2.4.1+cpu traittypes-0.2.1 + Installing collected packages: openvino-telemetry, mpmath, traittypes, tqdm, sympy, numpy, networkx, fsspec, filelock, torch, openvino, opencv-python, ipydatawidgets, pythreejs + Successfully installed filelock-3.16.1 fsspec-2024.10.0 ipydatawidgets-4.3.5 mpmath-1.3.0 networkx-3.1 numpy-1.24.4 opencv-python-4.10.0.84 openvino-2024.4.0 openvino-telemetry-2024.1.0 pythreejs-2.4.2 sympy-1.13.3 torch-2.4.1+cpu tqdm-4.66.6 traittypes-0.2.1 Note: you may need to restart the kernel to use updated packages. @@ -228,41 +218,31 @@ Download the model -We use ``omz_downloader``, which is a command line tool from the -``openvino-dev`` package. ``omz_downloader`` automatically creates a -directory structure and downloads the selected model. - .. code:: ipython3 - # directory where model will be downloaded - base_model_dir = "model" + from notebook_utils import download_file + import tarfile - # model name as named in Open Model Zoo - model_name = "human-pose-estimation-3d-0001" - # selected precision (FP32, FP16) - precision = "FP32" - BASE_MODEL_NAME = f"{base_model_dir}/public/{model_name}/{model_name}" - model_path = Path(BASE_MODEL_NAME).with_suffix(".pth") - onnx_path = Path(BASE_MODEL_NAME).with_suffix(".onnx") + # directory where model will be downloaded + base_model_dir = Path("model") + + download_file( + "https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2022.1/human-pose-estimation-3d-0001/human-pose-estimation-3d.tar.gz", + directory=base_model_dir, + ) - ir_model_path = Path(f"model/public/{model_name}/{precision}/{model_name}.xml") - model_weights_path = Path(f"model/public/{model_name}/{precision}/{model_name}.bin") + ckpt_file = base_model_dir / "human-pose-estimation-3d-0001.pth" - if not model_path.exists(): - download_command = f"omz_downloader " f"--name {model_name} " f"--output_dir {base_model_dir}" - ! $download_command + if not ckpt_file.exists(): + with tarfile.open(base_model_dir / "human-pose-estimation-3d.tar.gz") as f: + f.extractall(base_model_dir) + .. parsed-literal:: - ################|| Downloading human-pose-estimation-3d-0001 ||################ - - ========== Downloading model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.tar.gz - - - ========== Unpacking model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.tar.gz - + model/human-pose-estimation-3d.tar.gz: 0%| | 0.00/17.6M [00:00 1 or self.sliding_window is not None: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:808: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:861: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! encoder_states = () if output_hidden_states else None - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:813: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:866: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if output_hidden_states: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:836: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:889: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if output_hidden_states: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:839: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:892: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not return_dict: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:935: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:988: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not return_dict: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:1426: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:1486: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not return_dict: @@ -351,13 +352,13 @@ suitable. This function repeats part of ``AmusedPipeline``. .. parsed-literal:: - /tmp/ipykernel_2578393/3779428577.py:34: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /tmp/ipykernel_498025/3779428577.py:34: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! shape=shape.tolist(), - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/autoencoders/vq_model.py:144: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/autoencoders/vq_model.py:144: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not force_not_quantize: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:147: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:147: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if hidden_states.shape[0] >= 64: @@ -490,7 +491,7 @@ And insert wrappers instances in the pipeline: .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -696,7 +697,7 @@ model. .. parsed-literal:: - INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, onnx, openvino + INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, openvino @@ -707,7 +708,7 @@ model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -760,17 +761,17 @@ model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) @@ -794,7 +795,7 @@ Demo generation with quantized pipeline .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -878,7 +879,7 @@ a rough estimate of generation quality. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:43: UserWarning: Metric `InceptionScore` will save all extracted features in buffer. For large datasets this may lead to large memory footprint. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:43: UserWarning: Metric `InceptionScore` will save all extracted features in buffer. For large datasets this may lead to large memory footprint. warnings.warn(\*args, \*\*kwargs) # noqa: B028 @@ -890,7 +891,7 @@ a rough estimate of generation quality. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/image/inception.py:175: UserWarning: std(): degrees of freedom is <= 0. Correction should be strictly less than the reduction factor (input numel divided by output numel). (Triggered internally at ../aten/src/ATen/native/ReduceOps.cpp:1808.) + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/image/inception.py:175: UserWarning: std(): degrees of freedom is <= 0. Correction should be strictly less than the reduction factor (input numel divided by output numel). (Triggered internally at ../aten/src/ATen/native/ReduceOps.cpp:1808.) return kl.mean(), kl.std() @@ -908,7 +909,7 @@ a rough estimate of generation quality. .. parsed-literal:: Quantized pipeline Inception Score: 11.0730562210083 - Quantization speed-up: 2.09x + Quantization speed-up: 2.08x Interactive inference diff --git a/docs/notebooks/async-api-with-output.rst b/docs/notebooks/async-api-with-output.rst index e8f5b80d429d81..9f9130a4fe0db2 100644 --- a/docs/notebooks/async-api-with-output.rst +++ b/docs/notebooks/async-api-with-output.rst @@ -61,13 +61,6 @@ Imports %pip install -q "openvino>=2023.1.0" %pip install -q opencv-python "matplotlib>=3.4" - -.. parsed-literal:: - - Note: you may need to restart the kernel to use updated packages. - Note: you may need to restart the kernel to use updated packages. - - .. code:: ipython3 import cv2 @@ -119,14 +112,12 @@ the person in each frame of the video. ################|| Downloading person-detection-0202 ||################ - ========== Downloading model/intel/person-detection-0202/FP16/person-detection-0202.xml - + ========== Retrieving model/intel/person-detection-0202/FP16/person-detection-0202.xml from the cache - ========== Downloading model/intel/person-detection-0202/FP16/person-detection-0202.bin + ========== Retrieving model/intel/person-detection-0202/FP16/person-detection-0202.bin from the cache - Select inference device ~~~~~~~~~~~~~~~~~~~~~~~ @@ -348,8 +339,8 @@ Test performance in Sync Mode .. parsed-literal:: Source ended - average throuput in sync mode: 63.79 fps - + average throuput in sync mode: 55.59 fps + Async Mode ~~~~~~~~~~ @@ -487,8 +478,8 @@ Test the performance in Async Mode .. parsed-literal:: Source ended - average throuput in async mode: 106.83 fps - + average throuput in async mode: 75.17 fps + Compare the performance ~~~~~~~~~~~~~~~~~~~~~~~ @@ -630,5 +621,5 @@ Test the performance with ``AsyncInferQueue`` .. parsed-literal:: - average throughput in async mode with async infer queue: 145.36 fps - + average throughput in async mode with async infer queue: 103.81 fps + diff --git a/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png b/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png index f3492582efc67f..3694e16797b727 100644 --- a/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png +++ b/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:84a9b6aab4a04feb319b3243644da6837b3b894122657a8f6639fa604e3b48dd -size 29468 +oid sha256:fd71c3c8d066b8a16264c215b773c6200e981634ac24d04db99b0288d1ea1cca +size 30409 diff --git a/docs/notebooks/auto-device-with-output.rst b/docs/notebooks/auto-device-with-output.rst index 28d894eb72b22e..2ebcbe7d80deb2 100644 --- a/docs/notebooks/auto-device-with-output.rst +++ b/docs/notebooks/auto-device-with-output.rst @@ -197,16 +197,16 @@ By default, ``compile_model`` API will select **AUTO** as .. parsed-literal:: - [22:37:15.4888]I[plugin.cpp:421][AUTO] device:CPU, config:LOG_LEVEL=LOG_INFO - [22:37:15.4888]I[plugin.cpp:421][AUTO] device:CPU, config:PERFORMANCE_HINT=LATENCY - [22:37:15.4888]I[plugin.cpp:421][AUTO] device:CPU, config:PERFORMANCE_HINT_NUM_REQUESTS=0 - [22:37:15.4888]I[plugin.cpp:421][AUTO] device:CPU, config:PERF_COUNT=NO - [22:37:15.4888]I[plugin.cpp:426][AUTO] device:CPU, priority:0 - [22:37:15.4888]I[schedule.cpp:17][AUTO] scheduler starting - [22:37:15.4888]I[auto_schedule.cpp:181][AUTO] select device:CPU - [22:37:15.5995]I[auto_schedule.cpp:346][AUTO] Device: [CPU]: Compile model took 110.638049 ms - [22:37:15.5996]I[auto_schedule.cpp:112][AUTO] device:CPU compiling model finished - [22:37:15.5997]I[plugin.cpp:454][AUTO] underlying hardware does not support hardware context + [22:41:57.1267]I[plugin.cpp:421][AUTO] device:CPU, config:LOG_LEVEL=LOG_INFO + [22:41:57.1268]I[plugin.cpp:421][AUTO] device:CPU, config:PERFORMANCE_HINT=LATENCY + [22:41:57.1268]I[plugin.cpp:421][AUTO] device:CPU, config:PERFORMANCE_HINT_NUM_REQUESTS=0 + [22:41:57.1268]I[plugin.cpp:421][AUTO] device:CPU, config:PERF_COUNT=NO + [22:41:57.1268]I[plugin.cpp:426][AUTO] device:CPU, priority:0 + [22:41:57.1268]I[schedule.cpp:17][AUTO] scheduler starting + [22:41:57.1269]I[auto_schedule.cpp:181][AUTO] select device:CPU + [22:41:57.2582]I[auto_schedule.cpp:346][AUTO] Device: [CPU]: Compile model took 131.300219 ms + [22:41:57.2583]I[auto_schedule.cpp:112][AUTO] device:CPU compiling model finished + [22:41:57.2584]I[plugin.cpp:454][AUTO] underlying hardware does not support hardware context Successfully compiled model without a device_name. @@ -219,8 +219,8 @@ By default, ``compile_model`` API will select **AUTO** as .. parsed-literal:: - Deleted compiled_model[22:37:15.6060]I[schedule.cpp:308][AUTO] scheduler ending - + Deleted compiled_model + [22:41:57.2639]I[schedule.cpp:308][AUTO] scheduler ending Explicitly pass AUTO as device_name to Core::compile_model API @@ -378,7 +378,7 @@ executed on CPU until GPU is ready. .. parsed-literal:: - Time to load model using AUTO device and get first inference: 0.15 seconds. + Time to load model using AUTO device and get first inference: 0.12 seconds. .. code:: ipython3 @@ -553,12 +553,12 @@ Loop for inference and update the FPS/Latency every Compiling Model for AUTO device with THROUGHPUT hint Start inference, 6 groups of FPS/latency will be measured over 10s intervals - throughput: 184.25fps, latency: 31.12ms, time interval: 10.02s - throughput: 184.19fps, latency: 31.80ms, time interval: 10.00s - throughput: 183.00fps, latency: 32.00ms, time interval: 10.01s - throughput: 183.37fps, latency: 31.91ms, time interval: 10.01s - throughput: 178.30fps, latency: 32.90ms, time interval: 10.01s - throughput: 182.80fps, latency: 32.08ms, time interval: 10.01s + throughput: 179.70fps, latency: 32.12ms, time interval: 10.00s + throughput: 183.61fps, latency: 31.86ms, time interval: 10.01s + throughput: 183.96fps, latency: 31.88ms, time interval: 10.01s + throughput: 183.98fps, latency: 31.91ms, time interval: 10.00s + throughput: 183.26fps, latency: 31.98ms, time interval: 10.01s + throughput: 183.40fps, latency: 32.01ms, time interval: 10.00s Done @@ -604,12 +604,12 @@ Loop for inference and update the FPS/Latency for each Compiling Model for AUTO Device with LATENCY hint Start inference, 6 groups fps/latency will be out with 10s interval - throughput: 139.34fps, latency: 6.64ms, time interval: 10.00s - throughput: 141.45fps, latency: 6.63ms, time interval: 10.00s - throughput: 141.42fps, latency: 6.63ms, time interval: 10.01s - throughput: 141.70fps, latency: 6.62ms, time interval: 10.01s - throughput: 130.57fps, latency: 7.22ms, time interval: 10.00s - throughput: 141.61fps, latency: 6.62ms, time interval: 10.01s + throughput: 130.56fps, latency: 7.18ms, time interval: 10.00s + throughput: 142.51fps, latency: 6.61ms, time interval: 10.01s + throughput: 142.47fps, latency: 6.62ms, time interval: 10.00s + throughput: 142.46fps, latency: 6.61ms, time interval: 10.00s + throughput: 142.63fps, latency: 6.61ms, time interval: 10.00s + throughput: 142.73fps, latency: 6.60ms, time interval: 10.00s Done diff --git a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png index af14e6c0ac24c7..cc037738f18096 100644 --- a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png +++ b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:05bc2541cede3c32f2f9acff3f93d19572aab80a5bb7ac3e6c77bb397817bb54 -size 25899 +oid sha256:1bedd8ff3e65a23fb4af380958a261d0916d2e0134b9426652a2779bdc06d6de +size 26887 diff --git a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png index e12bb8a48eaa0e..21be57ac89d68d 100644 --- a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png +++ b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b1e2e56c28c18b5d31607518b846e78c58e76b2b59a9ee083d02e0d8f8ec2b52 -size 40077 +oid sha256:ed1ab24c30040707a36155169f4aaa91a5bff6cb48a2c5d10401ecbd87ca6f54 +size 40117 diff --git a/docs/notebooks/catvton-with-output.rst b/docs/notebooks/catvton-with-output.rst new file mode 100644 index 00000000000000..a7a9a04359f338 --- /dev/null +++ b/docs/notebooks/catvton-with-output.rst @@ -0,0 +1,360 @@ +Virtual Try-On with CatVTON and OpenVINO +======================================== + +Virtual try-on methods based on diffusion models achieve realistic +try-on effects but replicate the backbone network as a ReferenceNet or +leverage additional image encoders to process condition inputs, +resulting in high training and inference costs. `In this +work `__, authors rethink the necessity +of ReferenceNet and image encoders and innovate the interaction between +garment and person, proposing CatVTON, a simple and efficient virtual +try-on diffusion model. It facilitates the seamless transfer of in-shop +or worn garments of arbitrary categories to target persons by simply +concatenating them in spatial dimensions as inputs. The efficiency of +the model is demonstrated in three aspects: 1. Lightweight network. Only +the original diffusion modules are used, without additional network +modules. The text encoder and cross attentions for text injection in the +backbone are removed, further reducing the parameters by 167.02M. 2. +Parameter-efficient training. We identified the try-on relevant modules +through experiments and achieved high-quality try-on effects by training +only 49.57M parameters (∼5.51% of the backbone network’s parameters). 3. +Simplified inference. CatVTON eliminates all unnecessary conditions and +preprocessing steps, including pose estimation, human parsing, and text +input, requiring only garment reference, target person image, and mask +for the virtual try-on process. Extensive experiments demonstrate that +CatVTON achieves superior qualitative and quantitative results with +fewer prerequisites and trainable parameters than baseline methods. +Furthermore, CatVTON shows good generalization in in-the-wild scenarios +despite using open-source datasets with only 73K samples. + +Teaser image from `CatVTON +GitHub `__ |teaser| + +In this tutorial we consider how to convert and run this model using +OpenVINO. + + +**Table of contents:** + + +- `Prerequisites <#prerequisites>`__ +- `Convert the model to OpenVINO + IR <#convert-the-model-to-openvino-ir>`__ +- `Compiling models <#compiling-models>`__ +- `Interactive demo <#interactive-demo>`__ + +Installation Instructions +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This is a self-contained example that relies solely on its own code. + +We recommend running the notebook in a virtual environment. You only +need a Jupyter server to start. For details, please refer to +`Installation +Guide `__. + +.. |teaser| image:: https://github.com/Zheng-Chong/CatVTON/blob/edited/resource/img/teaser.jpg?raw=true + +Prerequisites +------------- + + + +.. code:: ipython3 + + import platform + + + if platform.system() == "Darwin": + %pip install -q "numpy<2.0.0" + %pip install -q "openvino>=2024.4" + %pip install -q "torch>=2.1" "diffusers>=0.29.1" torchvision opencv_python --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q fvcore "pillow" "tqdm" "gradio>=4.36" "omegaconf==2.4.0.dev3" av pycocotools cloudpickle scipy accelerate "transformers>=4.27.3" + + +.. parsed-literal:: + + Note: you may need to restart the kernel to use updated packages. + Note: you may need to restart the kernel to use updated packages. + Note: you may need to restart the kernel to use updated packages. + + +.. code:: ipython3 + + import requests + + + r = requests.get( + url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", + ) + open("notebook_utils.py", "w").write(r.text) + + r = requests.get( + url="https://raw.githubusercontent.com/aleksandr-mokrov/openvino_notebooks/refs/heads/catvton/utils/cmd_helper.py", + ) + open("cmd_helper.py", "w").write(r.text) + + + + +.. parsed-literal:: + + 741 + + + +.. code:: ipython3 + + from cmd_helper import clone_repo + + + clone_repo("https://github.com/Zheng-Chong/CatVTON.git", "3b795364a4d2f3b5adb365f39cdea376d20bc53c") + + + + +.. parsed-literal:: + + PosixPath('CatVTON') + + + +Convert the model to OpenVINO IR +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +OpenVINO supports PyTorch models via conversion to OpenVINO Intermediate +Representation (IR). `OpenVINO model conversion +API `__ +should be used for these purposes. ``ov.convert_model`` function accepts +original PyTorch model instance and example input for tracing and +returns ``ov.Model`` representing this model in OpenVINO framework. +Converted model can be used for saving on disk using ``ov.save_model`` +function or directly loading on device using ``core.complie_model``. + +``ov_catvton_helper.py`` script contains helper function for models +downloading and models conversion, please check its content if you +interested in conversion details. + +To download checkpoints and load models, just call the helper function +``download_models``. It takes care about it. Functions +``convert_pipeline_models`` and ``convert_automasker_models`` will +convert models from pipeline and ``automasker`` in OpenVINO format. + +The original pipeline contains VAE encoder and decoder and UNET. +|CatVTON-overview| + +The ``automasker`` contains ``DensePose`` with +``detectron2.GeneralizedRCNN`` model and ``SCHP`` (``LIP`` and ``ATR`` +version). + +.. |CatVTON-overview| image:: https://github.com/user-attachments/assets/e35c8dab-1c54-47b1-a73b-2a62e6cdca7c + +.. code:: ipython3 + + from pathlib import Path + + from ov_catvton_helper import download_models, convert_pipeline_models, convert_automasker_models + + + MODEL_DIR = Path("models") + VAE_ENCODER_PATH = MODEL_DIR / "vae_encoder.xml" + VAE_DECODER_PATH = MODEL_DIR / "vae_decoder.xml" + UNET_PATH = MODEL_DIR / "unet.xml" + DENSEPOSE_PROCESSOR_PATH = MODEL_DIR / "densepose_processor.xml" + SCHP_PROCESSOR_ATR = MODEL_DIR / "schp_processor_atr.xml" + SCHP_PROCESSOR_LIP = MODEL_DIR / "schp_processor_lip.xml" + + + pipeline, mask_processor, automasker = download_models(MODEL_DIR) + convert_pipeline_models(pipeline, VAE_ENCODER_PATH, VAE_DECODER_PATH, UNET_PATH) + convert_automasker_models(automasker, DENSEPOSE_PROCESSOR_PATH, SCHP_PROCESSOR_ATR, SCHP_PROCESSOR_LIP) + + +.. parsed-literal:: + + Note: switching to '3b795364a4d2f3b5adb365f39cdea376d20bc53c'. + + You are in 'detached HEAD' state. You can look around, make experimental + changes and commit them, and you can discard any commits you make in this + state without impacting any branches by switching back to a branch. + + If you want to create a new branch to retain commits you create, you may + do so (now or later) by using -c with the switch command. Example: + + git switch -c + + Or undo this operation with: + + git switch - + + Turn off this advice by setting config variable advice.detachedHead to false + + HEAD is now at 3b79536 Update default base model path + + + +.. parsed-literal:: + + Fetching 10 files: 0%| | 0/10 [00:00= 64: + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_2d_condition.py:1111: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if dim % default_overall_up_factor != 0: + + +Compiling models +---------------- + + + +Select device from dropdown list for running inference using OpenVINO. + +.. code:: ipython3 + + import openvino as ov + + from notebook_utils import device_widget + + + core = ov.Core() + + device = device_widget() + + device + + + + +.. parsed-literal:: + + Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') + + + +``get_compiled_pipeline`` and ``get_compiled_automasker`` functions +defined in ``ov_catvton_helper.py`` provides convenient way for getting +the pipeline and the ``automasker`` with compiled ov-models that are +compatible with the original interface. It accepts the original pipeline +and ``automasker``, inference device and directories with converted +models as arguments. Under the hood we create callable wrapper classes +for compiled models to allow interaction with original pipelines. Note +that all of wrapper classes return ``torch.Tensor``\ s instead of +``np.array``\ s. And then insert wrappers instances in the pipeline. + +.. code:: ipython3 + + from ov_catvton_helper import get_compiled_pipeline, get_compiled_automasker + + + pipeline = get_compiled_pipeline(pipeline, core, device, VAE_ENCODER_PATH, VAE_DECODER_PATH, UNET_PATH) + automasker = get_compiled_automasker(automasker, core, device, DENSEPOSE_PROCESSOR_PATH, SCHP_PROCESSOR_ATR, SCHP_PROCESSOR_LIP) + +Interactive inference +--------------------- + + + +Please select below whether you would like to use the quantized models +to launch the interactive demo. + +.. code:: ipython3 + + from gradio_helper import make_demo + + + output_dir = "output" + demo = make_demo(pipeline, mask_processor, automasker, output_dir) + try: + demo.launch(debug=False) + except Exception: + demo.launch(debug=False, share=True) + + +.. parsed-literal:: + + Running on local URL: http://127.0.0.1:7860 + + To create a public link, set `share=True` in `launch()`. + + + + + + + diff --git a/docs/notebooks/clip-zero-shot-classification-with-output.rst b/docs/notebooks/clip-zero-shot-classification-with-output.rst index 3da831e6d9d0dd..fd572a83ffb834 100644 --- a/docs/notebooks/clip-zero-shot-classification-with-output.rst +++ b/docs/notebooks/clip-zero-shot-classification-with-output.rst @@ -729,7 +729,6 @@ up of the dynamic quantized models. Interactive demo ---------------- - Now, it is your turn! You can provide your own image and comma-separated list of labels for zero-shot classification. diff --git a/docs/notebooks/convert-to-openvino-with-output.rst b/docs/notebooks/convert-to-openvino-with-output.rst index 8ec32adef0e04a..2baaf0043e7f04 100644 --- a/docs/notebooks/convert-to-openvino-with-output.rst +++ b/docs/notebooks/convert-to-openvino-with-output.rst @@ -163,7 +163,7 @@ NLP model from Hugging Face and export it in ONNX format: .. code:: ipython3 from transformers import AutoModelForSequenceClassification, AutoTokenizer - from transformers.onnx import export, FeaturesManager + import torch ONNX_NLP_MODEL_PATH = MODEL_DIRECTORY_PATH / "distilbert.onnx" @@ -172,37 +172,27 @@ NLP model from Hugging Face and export it in ONNX format: # initialize tokenizer tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english") - # get model onnx config function for output feature format sequence-classification - model_kind, model_onnx_config = FeaturesManager.check_supported_model_or_raise(hf_model, feature="sequence-classification") - # fill onnx config based on pytorch model config - onnx_config = model_onnx_config(hf_model.config) - - # export to onnx format - export( - preprocessor=tokenizer, - model=hf_model, - config=onnx_config, - opset=onnx_config.default_onnx_opset, - output=ONNX_NLP_MODEL_PATH, - ) + if not ONNX_NLP_MODEL_PATH.exists(): + inputs = tokenizer("Hi, how are you?", return_tensors="pt") + input_names = list(inputs.keys()) + dynamic_axes = {input_name: {0: "batch_size", 1: "seq_length"} for input_name in input_names} + torch.onnx.export( + hf_model, args=dict(inputs), input_names=input_names, output_names=["logits"], dynamic_axes=dynamic_axes, f=ONNX_NLP_MODEL_PATH, opset_version=14 + ) + print(f"ONNX model exported to {ONNX_NLP_MODEL_PATH}") .. parsed-literal:: - 2024-10-22 22:40:21.522113: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-22 22:40:21.555890: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-04 22:48:30.842642: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-04 22:48:30.876775: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-22 22:40:22.084160: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:215: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. - mask, torch.tensor(torch.finfo(scores.dtype).min) - - + 2024-11-04 22:48:31.539454: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: - (['input_ids', 'attention_mask'], ['logits']) - + ONNX model exported to model/distilbert.onnx Fetch @@ -670,7 +660,7 @@ frameworks conversion guides. .. parsed-literal:: - 2024-10-22 22:40:40.138322: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. + 2024-11-04 22:48:47.716205: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. Skipping registering GPU devices... diff --git a/docs/notebooks/convnext-classification-with-output.rst b/docs/notebooks/convnext-classification-with-output.rst index 1204ea2c17f106..6e1c039f7013c6 100644 --- a/docs/notebooks/convnext-classification-with-output.rst +++ b/docs/notebooks/convnext-classification-with-output.rst @@ -192,7 +192,7 @@ And print results Predicted Class: 281 Predicted Label: n02123045 tabby, tabby cat - Predicted Probability: 0.5351971983909607 + Predicted Probability: 0.4661690592765808 Convert the model to OpenVINO Intermediate representation format diff --git a/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst b/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst index 6f003fb71d75fb..30778bafc8e884 100644 --- a/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst +++ b/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst @@ -154,10 +154,10 @@ Imports .. parsed-literal:: - 2024-10-22 22:41:03.560708: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-22 22:41:03.596278: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-04 22:49:10.827255: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-04 22:49:10.861330: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-22 22:41:04.191138: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-04 22:49:11.454332: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -223,7 +223,7 @@ notebook `__. .. parsed-literal:: - /tmp/ipykernel_2583350/1592321960.py:3: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /tmp/ipykernel_503635/1592321960.py:3: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. state_dict = torch.load(state_dict_file, map_location=torch.device("cpu")) @@ -444,7 +444,7 @@ this notebook. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x_e.shape[-i - 1] != x_0.shape[-i - 1]: @@ -526,18 +526,18 @@ Convert quantized model to OpenVINO IR model and save it. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:340: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:340: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! return self._level_low.item() - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:348: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:348: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! return self._level_high.item() - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x_e.shape[-i - 1] != x_0.shape[-i - 1]: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: Tensor-likes are not close! - Mismatched elements: 249255 / 262144 (95.1%) - Greatest absolute difference: 3.8265769481658936 at index (0, 0, 126, 353) (up to 1e-05 allowed) - Greatest relative difference: 18364.59337143498 at index (0, 0, 305, 451) (up to 1e-05 allowed) + Mismatched elements: 245783 / 262144 (93.8%) + Greatest absolute difference: 3.1180567741394043 at index (0, 0, 474, 435) (up to 1e-05 allowed) + Greatest relative difference: 16087.83647354372 at index (0, 0, 37, 224) (up to 1e-05 allowed) _check_trace( @@ -663,7 +663,7 @@ be run in the notebook with ``! benchmark_app`` or [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 8.83 ms + [ INFO ] Read model took 8.85 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [1,1,512,512] @@ -677,7 +677,7 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.final_conv/aten::_convolution/Add) : f32 / [...] / [1,1,512,512] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 250.74 ms + [ INFO ] Compile model took 253.47 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -714,17 +714,17 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 51.21 ms + [ INFO ] First inference took 56.51 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 392 iterations - [ INFO ] Duration: 15027.48 ms + [ INFO ] Count: 406 iterations + [ INFO ] Duration: 15019.48 ms [ INFO ] Latency: - [ INFO ] Median: 34.96 ms - [ INFO ] Average: 38.10 ms - [ INFO ] Min: 34.49 ms + [ INFO ] Median: 35.01 ms + [ INFO ] Average: 36.77 ms + [ INFO ] Min: 34.63 ms [ INFO ] Max: 48.05 ms - [ INFO ] Throughput: 26.09 FPS + [ INFO ] Throughput: 27.03 FPS .. code:: ipython3 @@ -750,7 +750,7 @@ be run in the notebook with ``! benchmark_app`` or [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 11.01 ms + [ INFO ] Read model took 10.78 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [1,1,512,512] @@ -764,7 +764,7 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.final_conv/aten::_convolution/Add) : f32 / [...] / [1,1,512,512] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 238.95 ms + [ INFO ] Compile model took 250.08 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model49 @@ -801,17 +801,17 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 30.03 ms + [ INFO ] First inference took 29.09 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 952 iterations - [ INFO ] Duration: 15015.31 ms + [ INFO ] Count: 938 iterations + [ INFO ] Duration: 15008.12 ms [ INFO ] Latency: - [ INFO ] Median: 15.55 ms - [ INFO ] Average: 15.57 ms - [ INFO ] Min: 15.23 ms - [ INFO ] Max: 17.26 ms - [ INFO ] Throughput: 63.40 FPS + [ INFO ] Median: 15.77 ms + [ INFO ] Average: 15.80 ms + [ INFO ] Min: 15.47 ms + [ INFO ] Max: 17.13 ms + [ INFO ] Throughput: 62.50 FPS Visually Compare Inference Results @@ -905,7 +905,7 @@ seed is displayed to enable reproducing specific runs of this cell. .. parsed-literal:: - Visualizing results with seed 1729629747 + Visualizing results with seed 1730757034 @@ -988,8 +988,8 @@ performs inference, and displays the results on the frames loaded in .. parsed-literal:: - Loaded model to AUTO in 0.18 seconds. - Total time for 68 frames: 2.32 seconds, fps:29.79 + Loaded model to AUTO in 0.15 seconds. + Total time for 68 frames: 2.36 seconds, fps:29.25 References diff --git a/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png b/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png index f7c163c1c77604..5aa37909b71cf7 100644 --- a/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png +++ b/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5d329fc21292f69aeee8164a3f805f2e8e61369c42eab565ebafe555cf6d1a1c -size 381131 +oid sha256:894600de56af211d4cc3e64ee092b5a62d1b0158c51048d17accadddea0f046e +size 382725 diff --git a/docs/notebooks/ddcolor-image-colorization-with-output.rst b/docs/notebooks/ddcolor-image-colorization-with-output.rst index b2a76fd1a0ded8..409d2495e2fea6 100644 --- a/docs/notebooks/ddcolor-image-colorization-with-output.rst +++ b/docs/notebooks/ddcolor-image-colorization-with-output.rst @@ -105,12 +105,12 @@ Prerequisites .. parsed-literal:: Cloning into 'DDColor'... - remote: Enumerating objects: 233, done. - remote: Counting objects: 100% (76/76), done. - remote: Compressing objects: 100% (42/42), done. - remote: Total 233 (delta 54), reused 34 (delta 34), pack-reused 157 (from 1) - Receiving objects: 100% (233/233), 13.34 MiB | 641.00 KiB/s, done. - Resolving deltas: 100% (80/80), done. + remote: Enumerating objects: 241, done. + remote: Counting objects: 100% (84/84), done. + remote: Compressing objects: 100% (49/49), done. + remote: Total 241 (delta 57), reused 37 (delta 35), pack-reused 157 (from 1) + Receiving objects: 100% (241/241), 14.10 MiB | 21.95 MiB/s, done. + Resolving deltas: 100% (83/83), done. @@ -131,7 +131,7 @@ Prerequisites .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) @@ -412,10 +412,10 @@ Perform model quantization .. parsed-literal:: - 2024-10-22 22:45:07.339219: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-22 22:45:07.378241: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-04 22:52:53.152561: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-04 22:52:53.191342: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-22 22:45:07.784302: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-04 22:52:53.595160: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -523,7 +523,7 @@ Tool + @@ -307,13 +304,13 @@ loading on device using ``core.complie_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2_layers/patch_embed.py:73: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2_layers/patch_embed.py:73: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}" - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2_layers/patch_embed.py:74: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2_layers/patch_embed.py:74: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}" - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2.py:183: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2.py:183: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if npatch == N and w == h: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dpt.py:147: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dpt.py:147: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! out = F.interpolate(out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True) @@ -405,7 +402,7 @@ range. .. parsed-literal:: - + @@ -627,7 +624,7 @@ Run inference on video .. parsed-literal:: - Processed 60 frames in 13.44 seconds. Total FPS (including video processing): 4.46.Inference FPS: 10.42 + Processed 60 frames in 13.24 seconds. Total FPS (including video processing): 4.53.Inference FPS: 10.68 Video saved to 'output/Coco Walking in Berkeley_depth_anything.mp4'. @@ -654,7 +651,7 @@ Run inference on video .. parsed-literal:: Showing video saved at - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/output/Coco Walking in Berkeley_depth_anything.mp4 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/output/Coco Walking in Berkeley_depth_anything.mp4 If you cannot see the video in your browser, please click on the following link to download the video @@ -787,10 +784,10 @@ quantization code below may take some time. .. parsed-literal:: - 2024-10-22 22:53:01.689628: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-22 22:53:01.723459: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-04 23:01:18.047102: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-04 23:01:18.080343: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-22 22:53:02.312695: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-04 23:01:18.654050: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -916,10 +913,10 @@ data. .. parsed-literal:: - Processed 60 frames in 12.93 seconds. Total FPS (including video processing): 4.64.Inference FPS: 12.78 + Processed 60 frames in 12.60 seconds. Total FPS (including video processing): 4.76.Inference FPS: 13.12 Video saved to 'output/Coco Walking in Berkeley_depth_anything_int8.mp4'. Showing video saved at - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/output/Coco Walking in Berkeley_depth_anything.mp4 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/output/Coco Walking in Berkeley_depth_anything.mp4 If you cannot see the video in your browser, please click on the following link to download the video @@ -999,9 +996,9 @@ Tool =4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (4.12.2) - Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (1.13.3) - Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1) - Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1.4) - Requirement already satisfied: fsspec in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (2024.9.0) - Requirement already satisfied: numpy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchvision) (1.23.5) - Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchvision) (10.4.0) - Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch) (2.1.5) - Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch) (1.3.0) + Requirement already satisfied: torch in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.4.1+cpu) + Requirement already satisfied: torchvision in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (0.19.1+cpu) + Requirement already satisfied: opencv-python in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.10.0.84) + Requirement already satisfied: wheel in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (0.44.0) + Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.16.1) + Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (4.12.2) + Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (1.13.3) + Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1) + Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1.4) + Requirement already satisfied: fsspec in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (2024.9.0) + Requirement already satisfied: numpy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchvision) (1.23.5) + Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchvision) (10.4.0) + Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch) (2.1.5) + Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch) (1.3.0) Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cpu Collecting git+https://github.com/facebookresearch/detectron2.git - Cloning https://github.com/facebookresearch/detectron2.git to /tmp/pip-req-build-we1e_5gi + Cloning https://github.com/facebookresearch/detectron2.git to /tmp/pip-req-build-9ds1xx43 .. parsed-literal:: - Running command git clone --filter=blob:none --quiet https://github.com/facebookresearch/detectron2.git /tmp/pip-req-build-we1e_5gi + Running command git clone --filter=blob:none --quiet https://github.com/facebookresearch/detectron2.git /tmp/pip-req-build-9ds1xx43 .. parsed-literal:: @@ -125,88 +125,88 @@ Install required packages for running model Resolved https://github.com/facebookresearch/detectron2.git to commit 8d85329aed8506ea3672e3e208971345973ea761 Preparing metadata (setup.py): started Preparing metadata (setup.py): finished with status 'done' - Requirement already satisfied: Pillow>=7.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (10.4.0) - Requirement already satisfied: black in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (24.3.0) - Collecting cloudpickle (from detectron2==0.6) - Using cached cloudpickle-3.1.0-py3-none-any.whl.metadata (7.0 kB) - Collecting fvcore<0.1.6,>=0.1.5 (from detectron2==0.6) - Using cached fvcore-0.1.5.post20221221-py3-none-any.whl + Requirement already satisfied: Pillow>=7.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (10.4.0) + Requirement already satisfied: black in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (24.3.0) + Requirement already satisfied: cloudpickle in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (3.1.0) + Requirement already satisfied: fvcore<0.1.6,>=0.1.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (0.1.5.post20221221) Collecting hydra-core>=1.1 (from detectron2==0.6) Using cached hydra_core-1.3.2-py3-none-any.whl.metadata (5.5 kB) Collecting iopath<0.1.10,>=0.1.7 (from detectron2==0.6) - Using cached iopath-0.1.9-py3-none-any.whl.metadata (370 bytes) - Requirement already satisfied: matplotlib in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (3.7.5) - Requirement already satisfied: omegaconf<2.4,>=2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (2.3.0) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (24.1) - Collecting pycocotools>=2.0.2 (from detectron2==0.6) - Using cached pycocotools-2.0.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.1 kB) - Requirement already satisfied: tabulate in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (0.9.0) - Requirement already satisfied: tensorboard in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (2.12.3) - Requirement already satisfied: termcolor>=1.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (2.4.0) - Requirement already satisfied: tqdm>4.29.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (4.66.5) - Collecting yacs>=0.1.8 (from detectron2==0.6) - Using cached yacs-0.1.8-py3-none-any.whl.metadata (639 bytes) - Requirement already satisfied: numpy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from fvcore<0.1.6,>=0.1.5->detectron2==0.6) (1.23.5) - Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from fvcore<0.1.6,>=0.1.5->detectron2==0.6) (6.0.2) - Requirement already satisfied: antlr4-python3-runtime==4.9.* in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.1->detectron2==0.6) (4.9.3) - Requirement already satisfied: importlib-resources in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.1->detectron2==0.6) (6.4.5) - Collecting portalocker (from iopath<0.1.10,>=0.1.7->detectron2==0.6) - Using cached portalocker-2.10.1-py3-none-any.whl.metadata (8.5 kB) - Requirement already satisfied: contourpy>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (1.1.1) - Requirement already satisfied: cycler>=0.10 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (0.12.1) - Requirement already satisfied: fonttools>=4.22.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (4.54.1) - Requirement already satisfied: kiwisolver>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (1.4.7) - Requirement already satisfied: pyparsing>=2.3.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (3.1.4) - Requirement already satisfied: python-dateutil>=2.7 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (2.9.0.post0) - Requirement already satisfied: click>=8.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (8.1.7) - Requirement already satisfied: mypy-extensions>=0.4.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (1.0.0) - Requirement already satisfied: pathspec>=0.9.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (0.12.1) - Requirement already satisfied: platformdirs>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (4.3.6) - Requirement already satisfied: tomli>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (2.0.2) - Requirement already satisfied: typing-extensions>=4.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (4.12.2) - Requirement already satisfied: absl-py>=0.4 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (1.4.0) - Requirement already satisfied: grpcio>=1.48.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (1.67.0) - Requirement already satisfied: google-auth<3,>=1.6.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (2.35.0) - Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (1.0.0) - Requirement already satisfied: markdown>=2.6.8 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (3.7) - Requirement already satisfied: protobuf>=3.19.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (3.20.3) - Requirement already satisfied: requests<3,>=2.21.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (2.32.3) - Requirement already satisfied: setuptools>=41.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (44.0.0) - Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (0.7.2) - Requirement already satisfied: werkzeug>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (3.0.4) - Requirement already satisfied: wheel>=0.26 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (0.44.0) - Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (5.5.0) - Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (0.4.1) - Requirement already satisfied: rsa<5,>=3.1.4 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (4.9) - Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard->detectron2==0.6) (2.0.0) - Requirement already satisfied: zipp>=3.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from importlib-resources->hydra-core>=1.1->detectron2==0.6) (3.20.2) - Requirement already satisfied: importlib-metadata>=4.4 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from markdown>=2.6.8->tensorboard->detectron2==0.6) (8.5.0) - Requirement already satisfied: six>=1.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from python-dateutil>=2.7->matplotlib->detectron2==0.6) (1.16.0) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (3.4.0) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (3.10) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (2.2.3) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (2024.8.30) - Requirement already satisfied: MarkupSafe>=2.1.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from werkzeug>=1.0.1->tensorboard->detectron2==0.6) (2.1.5) - Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (0.6.1) - Requirement already satisfied: oauthlib>=3.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard->detectron2==0.6) (3.2.2) + Using cached https://download.pytorch.org/whl/iopath-0.1.9-py3-none-any.whl (27 kB) + Requirement already satisfied: matplotlib in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (3.7.5) + Collecting omegaconf<2.4,>=2.1 (from detectron2==0.6) + Using cached omegaconf-2.3.0-py3-none-any.whl.metadata (3.9 kB) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (24.1) + Requirement already satisfied: pycocotools>=2.0.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (2.0.7) + Requirement already satisfied: tabulate in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (0.9.0) + Requirement already satisfied: tensorboard in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (2.12.3) + Requirement already satisfied: termcolor>=1.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (2.4.0) + Requirement already satisfied: tqdm>4.29.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (4.66.6) + Requirement already satisfied: yacs>=0.1.8 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (0.1.8) + Requirement already satisfied: numpy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from fvcore<0.1.6,>=0.1.5->detectron2==0.6) (1.23.5) + Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from fvcore<0.1.6,>=0.1.5->detectron2==0.6) (6.0.2) + Requirement already satisfied: antlr4-python3-runtime==4.9.* in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.1->detectron2==0.6) (4.9.3) + Requirement already satisfied: importlib-resources in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.1->detectron2==0.6) (6.4.5) + Requirement already satisfied: portalocker in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath<0.1.10,>=0.1.7->detectron2==0.6) (2.10.1) + Requirement already satisfied: contourpy>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (1.1.1) + Requirement already satisfied: cycler>=0.10 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (0.12.1) + Requirement already satisfied: fonttools>=4.22.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (4.54.1) + Requirement already satisfied: kiwisolver>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (1.4.7) + Requirement already satisfied: pyparsing>=2.3.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (3.1.4) + Requirement already satisfied: python-dateutil>=2.7 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (2.9.0.post0) + Requirement already satisfied: click>=8.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (8.1.7) + Requirement already satisfied: mypy-extensions>=0.4.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (1.0.0) + Requirement already satisfied: pathspec>=0.9.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (0.12.1) + Requirement already satisfied: platformdirs>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (4.3.6) + Requirement already satisfied: tomli>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (2.0.2) + Requirement already satisfied: typing-extensions>=4.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (4.12.2) + Requirement already satisfied: absl-py>=0.4 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (1.4.0) + Requirement already satisfied: grpcio>=1.48.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (1.67.1) + Requirement already satisfied: google-auth<3,>=1.6.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (2.35.0) + Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (1.0.0) + Requirement already satisfied: markdown>=2.6.8 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (3.7) + Requirement already satisfied: protobuf>=3.19.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (3.20.3) + Requirement already satisfied: requests<3,>=2.21.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (2.32.3) + Requirement already satisfied: setuptools>=41.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (44.0.0) + Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (0.7.2) + Requirement already satisfied: werkzeug>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (3.0.6) + Requirement already satisfied: wheel>=0.26 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (0.44.0) + Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (5.5.0) + Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (0.4.1) + Requirement already satisfied: rsa<5,>=3.1.4 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (4.9) + Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard->detectron2==0.6) (2.0.0) + Requirement already satisfied: zipp>=3.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from importlib-resources->hydra-core>=1.1->detectron2==0.6) (3.20.2) + Requirement already satisfied: importlib-metadata>=4.4 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from markdown>=2.6.8->tensorboard->detectron2==0.6) (8.5.0) + Requirement already satisfied: six>=1.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from python-dateutil>=2.7->matplotlib->detectron2==0.6) (1.16.0) + Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (3.4.0) + Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (3.10) + Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (2.2.3) + Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (2024.8.30) + Requirement already satisfied: MarkupSafe>=2.1.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from werkzeug>=1.0.1->tensorboard->detectron2==0.6) (2.1.5) + Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (0.6.1) + Requirement already satisfied: oauthlib>=3.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard->detectron2==0.6) (3.2.2) Using cached hydra_core-1.3.2-py3-none-any.whl (154 kB) - Using cached iopath-0.1.9-py3-none-any.whl (27 kB) - Using cached pycocotools-2.0.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (439 kB) - Using cached yacs-0.1.8-py3-none-any.whl (14 kB) - Using cached cloudpickle-3.1.0-py3-none-any.whl (22 kB) - Using cached portalocker-2.10.1-py3-none-any.whl (18 kB) + Using cached omegaconf-2.3.0-py3-none-any.whl (79 kB) Building wheels for collected packages: detectron2 Building wheel for detectron2 (setup.py): started Building wheel for detectron2 (setup.py): finished with status 'done' - Created wheel for detectron2: filename=detectron2-0.6-cp38-cp38-linux_x86_64.whl size=8313552 sha256=23ceb6e5b734ecc530172b613be139d732deaa2e962d5a8bc940e6b23a85309d - Stored in directory: /tmp/pip-ephem-wheel-cache-65iaghs7/wheels/19/ac/65/e48e5e4ec2702274d927c5a6efb75709b24014371d3bb778f2 + Created wheel for detectron2: filename=detectron2-0.6-cp38-cp38-linux_x86_64.whl size=8313237 sha256=7cd84a15a89de76a7ab5b648f2fb7ebff63b7e43ffc90c7f19a568d16858de8a + Stored in directory: /tmp/pip-ephem-wheel-cache-uvptv5zg/wheels/19/ac/65/e48e5e4ec2702274d927c5a6efb75709b24014371d3bb778f2 Successfully built detectron2 - Installing collected packages: yacs, portalocker, cloudpickle, iopath, hydra-core, pycocotools, fvcore, detectron2 - Successfully installed cloudpickle-3.1.0 detectron2-0.6 fvcore-0.1.5.post20221221 hydra-core-1.3.2 iopath-0.1.9 portalocker-2.10.1 pycocotools-2.0.7 yacs-0.1.8 - Requirement already satisfied: openvino>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2024.4.0) - Requirement already satisfied: numpy<2.1.0,>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (1.23.5) - Requirement already satisfied: openvino-telemetry>=2023.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (2024.1.0) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (24.1) + Installing collected packages: omegaconf, iopath, hydra-core, detectron2 + Attempting uninstall: omegaconf + Found existing installation: omegaconf 2.4.0.dev3 + Uninstalling omegaconf-2.4.0.dev3: + Successfully uninstalled omegaconf-2.4.0.dev3 + Attempting uninstall: iopath + Found existing installation: iopath 0.1.10 + Uninstalling iopath-0.1.10: + Successfully uninstalled iopath-0.1.10 + Successfully installed detectron2-0.6 hydra-core-1.3.2 iopath-0.1.9 omegaconf-2.3.0 + Requirement already satisfied: openvino>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2024.4.0) + Requirement already satisfied: numpy<2.1.0,>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (1.23.5) + Requirement already satisfied: openvino-telemetry>=2023.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (2024.1.0) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (24.1) Define helpers for PyTorch model initialization and conversion diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg index 6160562199f757..f5b1d98eea3213 100644 --- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg +++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b6475f9155bca7152327255fb36bac32dea6f10aa834b6a91d46d4ecc718be0f -size 58279 +oid sha256:0df4e94924f81aab66086702d85a461f463078f0d06f67b1fe5d46ad8480aa91 +size 58652 diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png index 77ac4e5e0c1fd8..f676b44edd1d9a 100644 --- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png +++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b98f4e2e56bb62e6d7c68e536d05695f16b412d4bb8a502f5ced3c466716fe91 -size 508620 +oid sha256:b5a857cd060d740290ccc65aec47252aad9f41c665dc2808195c3185248977e8 +size 509376 diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg index 98087b7c1f10f2..67719cdcbd66b0 100644 --- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg +++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:02cb8226997c872abaee3c70c1ab90c8e6ac078adb81ec6d0721ece6049e7af3 -size 54825 +oid sha256:ddc40900fddf1a115903c4e200899306060114348bf2ca82fbb4d7d92a885b09 +size 53897 diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png index e317a3a3005ef6..af63ef41697b47 100644 --- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png +++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:940552cfe8d62878a7b2fb827529df2ef6ef4fa135f82f578362142650e0d751 -size 457741 +oid sha256:d1276209027e5aac72e4bb6f39f4494d2a807ee4bd85054a1285b0832e4515b9 +size 460797 diff --git a/docs/notebooks/distil-whisper-asr-with-output.rst b/docs/notebooks/distil-whisper-asr-with-output.rst index 2cdecfe17a19d4..53950226a210c7 100644 --- a/docs/notebooks/distil-whisper-asr-with-output.rst +++ b/docs/notebooks/distil-whisper-asr-with-output.rst @@ -133,7 +133,8 @@ using tokenizer. "distil-whisper/distil-small.en", ], "Whisper": [ - "openai/whisper-large-v3-turbo" "openai/whisper-large-v3", + "openai/whisper-large-v3-turbo", + "openai/whisper-large-v3", "openai/whisper-large-v2", "openai/whisper-large", "openai/whisper-medium", diff --git a/docs/notebooks/distilbert-sequence-classification-with-output.rst b/docs/notebooks/distilbert-sequence-classification-with-output.rst index 5f069551357b6d..862079f68aeeb7 100644 --- a/docs/notebooks/distilbert-sequence-classification-with-output.rst +++ b/docs/notebooks/distilbert-sequence-classification-with-output.rst @@ -47,31 +47,31 @@ Imports .. parsed-literal:: Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cpu - Requirement already satisfied: openvino>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2024.4.0) - Requirement already satisfied: transformers in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.45.2) - Requirement already satisfied: torch>=2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.4.1+cpu) - Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.66.5) - Requirement already satisfied: numpy<2.1.0,>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (1.23.5) - Requirement already satisfied: openvino-telemetry>=2023.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (2024.1.0) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (24.1) - Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (3.16.1) - Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.26.1) - Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (6.0.2) - Requirement already satisfied: regex!=2019.12.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2024.9.11) - Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2.32.3) - Requirement already satisfied: safetensors>=0.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.4.5) - Requirement already satisfied: tokenizers<0.21,>=0.20 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.20.1) - Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (4.12.2) - Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (1.13.3) - Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (3.1) - Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (3.1.4) - Requirement already satisfied: fsspec in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (2024.9.0) - Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch>=2.1) (2.1.5) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.4.0) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.10) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2.2.3) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2024.8.30) - Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch>=2.1) (1.3.0) + Requirement already satisfied: openvino>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2024.4.0) + Requirement already satisfied: transformers in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.46.1) + Requirement already satisfied: torch>=2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.4.1+cpu) + Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.66.6) + Requirement already satisfied: numpy<2.1.0,>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (1.23.5) + Requirement already satisfied: openvino-telemetry>=2023.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (2024.1.0) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (24.1) + Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (3.16.1) + Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.26.2) + Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (6.0.2) + Requirement already satisfied: regex!=2019.12.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2024.9.11) + Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2.32.3) + Requirement already satisfied: safetensors>=0.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.4.5) + Requirement already satisfied: tokenizers<0.21,>=0.20 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.20.2) + Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (4.12.2) + Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (1.13.3) + Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (3.1) + Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (3.1.4) + Requirement already satisfied: fsspec in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (2024.9.0) + Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch>=2.1) (2.1.5) + Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.4.0) + Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.10) + Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2.2.3) + Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2024.8.30) + Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch>=2.1) (1.3.0) Note: you may need to restart the kernel to use updated packages. @@ -110,6 +110,15 @@ model from Hugging Face. checkpoint = "distilbert-base-uncased-finetuned-sst-2-english" model = AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path=checkpoint) + +.. parsed-literal:: + + 2024-11-04 23:18:47.102633: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-04 23:18:47.135966: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2024-11-04 23:18:47.793551: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + + Initializing the Tokenizer -------------------------- @@ -166,10 +175,15 @@ optimal execution on end-point target devices. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4779: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. + + +.. parsed-literal:: + + [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:215: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. - mask, torch.tensor(torch.finfo(scores.dtype).min) + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. OpenVINO™ Runtime uses the `Infer diff --git a/docs/notebooks/dolly-2-instruction-following-with-output.rst b/docs/notebooks/dolly-2-instruction-following-with-output.rst index 9f6857b608d962..3b276a1725f0aa 100644 --- a/docs/notebooks/dolly-2-instruction-following-with-output.rst +++ b/docs/notebooks/dolly-2-instruction-following-with-output.rst @@ -136,6 +136,8 @@ documentation `__. .. code:: ipython3 import os + from pathlib import Path + import requests os.environ["GIT_CLONE_PROTECTION_ACTIVE"] = "false" @@ -144,12 +146,17 @@ documentation `__. %pip install -q "diffusers>=0.16.1" "transformers>=4.33.0" "torch>=2.1" "nncf>=2.10.0" "onnx<1.16.2" "gradio>=4.19" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "git+https://github.com/huggingface/optimum-intel.git" - import requests - r = requests.get( - url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", - ) - open("notebook_utils.py", "w").write(r.text) + utility_files = ["notebook_utils.py", "cmd_helper.py"] + + for utility in utility_files: + local_path = Path(utility) + if not local_path.exists(): + r = requests.get( + url=f"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/{local_path.name}", + ) + with local_path.open("w") as f: + f.write(r.text) Convert model using Optimum-CLI tool ------------------------------------ @@ -227,7 +234,7 @@ sacrifice of the model size and inference latency. .. code:: ipython3 - from IPython.display import Markdown, display + from IPython.display import display import ipywidgets as widgets prepare_int4_model = widgets.Checkbox( @@ -272,6 +279,7 @@ sacrifice of the model size and inference latency. .. code:: ipython3 from pathlib import Path + from cmd_helper import optimum_cli model_id = "databricks/dolly-v2-3b" model_path = Path("dolly-v2-3b") @@ -284,36 +292,19 @@ sacrifice of the model size and inference latency. def convert_to_fp16(): if (fp16_model_dir / "openvino_model.xml").exists(): return - fp16_model_dir.mkdir(parents=True, exist_ok=True) - export_command_base = "optimum-cli export openvino --model {} --task text-generation-with-past --weight-format fp16".format(model_id) - export_command = export_command_base + " " + str(fp16_model_dir) - display(Markdown("**Export command:**")) - display(Markdown(f"`{export_command}`")) - ! $export_command + optimum_cli(model_id, fp16_model_dir, additional_args={"weight-format": "fp16"}) def convert_to_int8(): if (int8_model_dir / "openvino_model.xml").exists(): return - int8_model_dir.mkdir(parents=True, exist_ok=True) - export_command_base = "optimum-cli export openvino --model {} --task text-generation-with-past --weight-format int8".format(model_id) - export_command = export_command_base + " " + str(int8_model_dir) - display(Markdown("**Export command:**")) - display(Markdown(f"`{export_command}`")) - ! $export_command + optimum_cli(model_id, int8_model_dir, additional_args={"weight-format": "int8"}) def convert_to_int4(): if (int4_model_dir / "openvino_model.xml").exists(): return - int4_model_dir.mkdir(parents=True, exist_ok=True) - export_command_base = "optimum-cli export openvino --model {} --task text-generation-with-past --weight-format int4 --ratio 1.0 --group-size 128".format( - model_id - ) - export_command = export_command_base + " " + str(int4_model_dir) - display(Markdown("**Export command:**")) - display(Markdown(f"`{export_command}`")) - ! $export_command + optimum_cli(model_id, int4_model_dir, additional_args={"weight-format": "int4"}) if prepare_fp16_model.value: diff --git a/docs/notebooks/dynamicrafter-animating-images-with-output.rst b/docs/notebooks/dynamicrafter-animating-images-with-output.rst index 584c3442c94af2..992c346194e31c 100644 --- a/docs/notebooks/dynamicrafter-animating-images-with-output.rst +++ b/docs/notebooks/dynamicrafter-animating-images-with-output.rst @@ -189,10 +189,10 @@ Prerequisites remote: Counting objects: 100% (153/153), done. remote: Compressing objects: 100% (99/99), done. remote: Total 335 (delta 97), reused 54 (delta 54), pack-reused 182 (from 1) - Receiving objects: 100% (335/335), 72.41 MiB | 22.40 MiB/s, done. + Receiving objects: 100% (335/335), 72.41 MiB | 20.85 MiB/s, done. Resolving deltas: 100% (123/123), done. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images @@ -282,7 +282,7 @@ We will use model for 256x256 resolution as example. Also, models for .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/huggingface_hub/file_download.py:834: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/huggingface_hub/file_download.py:834: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`. For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder. warnings.warn( @@ -300,8 +300,12 @@ We will use model for 256x256 resolution as example. Also, models for .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) + 2024-11-04 23:23:38.980054: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-04 23:23:39.013901: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2024-11-04 23:23:39.616188: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -384,6 +388,17 @@ Convert CLIP text encoder del cond_stage_model gc.collect(); + +.. parsed-literal:: + + WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. + + +.. parsed-literal:: + + WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. + + Convert CLIP image encoder ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -409,43 +424,43 @@ resolutions. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/utils/image.py:226: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/utils/image.py:226: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input.numel() == 0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:573: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:573: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if size == input_size: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:579: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:579: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! antialias = antialias and (max(factors) > 1) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:581: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:581: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if antialias: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:584: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:584: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! sigmas = (max((factors[0] - 1.0) / 2.0, 0.001), max((factors[1] - 1.0) / 2.0, 0.001)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:589: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:589: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! ks = int(max(2.0 * 2 * sigmas[0], 3)), int(max(2.0 * 2 * sigmas[1], 3)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:589: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:589: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! ks = int(max(2.0 * 2 * sigmas[0], 3)), int(max(2.0 * 2 * sigmas[1], 3)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/gaussian.py:55: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/gaussian.py:55: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. sigma = tensor([sigma], device=input.device, dtype=input.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/gaussian.py:55: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/gaussian.py:55: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! sigma = tensor([sigma], device=input.device, dtype=input.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/core/check.py:78: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/core/check.py:78: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x_shape_to_check[i] != dim: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/kernels.py:92: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/kernels.py:92: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. mean = tensor([[mean]], device=sigma.device, dtype=sigma.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:101: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:101: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if len(mean.shape) == 0 or mean.shape[0] == 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if len(std.shape) == 0 or std.shape[0] == 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:107: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:107: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if mean.shape and mean.shape[0] != 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:108: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:108: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if mean.shape[0] != data.shape[1] and mean.shape[:2] != data.shape[:2]: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:112: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:112: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if std.shape and std.shape[0] != 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:113: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:113: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if std.shape[0] != data.shape[1] and std.shape[:2] != data.shape[:2]: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:116: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:116: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. mean = torch.as_tensor(mean, device=data.device, dtype=data.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:117: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:117: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. std = torch.as_tensor(std, device=data.device, dtype=data.dtype) @@ -474,7 +489,7 @@ Convert AE encoder .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/ae_modules.py:67: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/ae_modules.py:67: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! w_ = w_ * (int(c)**(-0.5)) @@ -518,15 +533,15 @@ Convert Diffusion U-Net model .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:556: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:556: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if l_context == 77 + t*16: ## !!! HARD CODE here - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:205: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:205: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if batch_size: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:232: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:232: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if self.use_temporal_conv and batch_size: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:76: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:76: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert x.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:99: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:99: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert x.shape[1] == self.channels @@ -913,14 +928,14 @@ Run OpenVINO pipeline inference .. parsed-literal:: Seed set to 234 - /tmp/ipykernel_2590985/2451984876.py:25: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at ../torch/csrc/utils/tensor_numpy.cpp:206.) + /tmp/ipykernel_511478/2451984876.py:25: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at ../torch/csrc/utils/tensor_numpy.cpp:206.) img_tensor = torch.from_numpy(image).permute(2, 0, 1).float().to(model.device) .. parsed-literal:: - start: man fishing in a boat at sunset 2024-10-22 23:16:18 - Saved in man_fishing_in_a_boat_at_sunset.mp4. Time used: 196.25 seconds + start: man fishing in a boat at sunset 2024-11-04 23:26:56 + Saved in man_fishing_in_a_boat_at_sunset.mp4. Time used: 206.55 seconds .. code:: ipython3 @@ -1170,14 +1185,6 @@ quantization time. INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino -.. parsed-literal:: - - 2024-10-22 23:45:51.693284: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-22 23:45:51.735392: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-22 23:45:52.354791: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - - .. parsed-literal:: @@ -1357,8 +1364,8 @@ Let’s run the optimized pipeline .. parsed-literal:: - start: man fishing in a boat at sunset 2024-10-23 00:47:13 - Saved in man_fishing_in_a_boat_at_sunset.mp4. Time used: 98.40 seconds + start: man fishing in a boat at sunset 2024-11-05 00:58:08 + Saved in man_fishing_in_a_boat_at_sunset.mp4. Time used: 97.78 seconds .. code:: ipython3 @@ -1463,9 +1470,9 @@ models, we use median inference time on calibration subset. .. parsed-literal:: - FP32 latency: 195.358 - INT8 latency: 97.265 - Performance speed up: 2.009 + FP32 latency: 201.526 + INT8 latency: 96.036 + Performance speed up: 2.098 Interactive inference diff --git a/docs/notebooks/efficient-sam-with-output.rst b/docs/notebooks/efficient-sam-with-output.rst index 8d725c4594afc6..b50b82341f4af8 100644 --- a/docs/notebooks/efficient-sam-with-output.rst +++ b/docs/notebooks/efficient-sam-with-output.rst @@ -108,9 +108,9 @@ Prerequisites remote: Counting objects: 100% (85/85), done. remote: Compressing objects: 100% (33/33), done. remote: Total 424 (delta 76), reused 52 (delta 52), pack-reused 339 (from 1) - Receiving objects: 100% (424/424), 262.14 MiB | 24.94 MiB/s, done. + Receiving objects: 100% (424/424), 262.14 MiB | 23.37 MiB/s, done. Resolving deltas: 100% (246/246), done. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM .. code:: ipython3 @@ -377,23 +377,23 @@ disk using ``openvino.save_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:220: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:220: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if ( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:241: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:241: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert ( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:163: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:163: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! size = int(math.sqrt(xy_num)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert size * size == xy_num - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:166: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:166: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if size != h or size != w: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:251: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:251: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert x.shape[2] == num_patches - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if num_pts > self.decoder_max_num_input_points: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:92: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:92: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! elif num_pts < self.decoder_max_num_input_points: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:126: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:126: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if output_w > 0 and output_h > 0: @@ -640,10 +640,10 @@ architecture type, we should specify ``transformer`` in ``model_type``. .. parsed-literal:: - 2024-10-23 01:04:24.934254: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:04:24.966235: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 01:15:40.935673: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:15:40.968460: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:04:25.612813: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:15:41.606156: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -810,7 +810,7 @@ models, we use ``bencmark_app``. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 30.59 ms + [ INFO ] Read model took 30.24 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] batched_images (node: batched_images) : f32 / [...] / [?,?,?,?] @@ -830,7 +830,7 @@ models, we use ``bencmark_app``. [ INFO ] ***NO_NAME*** (node: aten::reshape/Reshape_3) : f32 / [...] / [?,?,?,?,?] [ INFO ] ***NO_NAME*** (node: aten::reshape/Reshape_2) : f32 / [...] / [?,?,?] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 1397.68 ms + [ INFO ] Compile model took 1388.43 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -871,17 +871,17 @@ models, we use ``bencmark_app``. [ INFO ] Fill input 'batched_point_labels' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in full mode (inputs filling are included in measurement loop). - [ INFO ] First inference took 822.58 ms + [ INFO ] First inference took 798.46 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 51 iterations - [ INFO ] Duration: 16272.65 ms + [ INFO ] Count: 49 iterations + [ INFO ] Duration: 16827.30 ms [ INFO ] Latency: - [ INFO ] Median: 1835.79 ms - [ INFO ] Average: 1847.61 ms - [ INFO ] Min: 1271.60 ms - [ INFO ] Max: 2300.40 ms - [ INFO ] Throughput: 3.13 FPS + [ INFO ] Median: 2025.54 ms + [ INFO ] Average: 1991.09 ms + [ INFO ] Min: 816.09 ms + [ INFO ] Max: 2176.67 ms + [ INFO ] Throughput: 2.91 FPS .. code:: ipython3 @@ -907,7 +907,7 @@ models, we use ``bencmark_app``. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 43.92 ms + [ INFO ] Read model took 43.95 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] batched_images (node: batched_images) : f32 / [...] / [?,?,?,?] @@ -927,7 +927,7 @@ models, we use ``bencmark_app``. [ INFO ] ***NO_NAME*** (node: aten::reshape/Reshape_3) : f32 / [...] / [?,?,?,?,?] [ INFO ] ***NO_NAME*** (node: aten::reshape/Reshape_2) : f32 / [...] / [?,?,?] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 1623.93 ms + [ INFO ] Compile model took 1607.96 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -968,17 +968,17 @@ models, we use ``bencmark_app``. [ INFO ] Fill input 'batched_point_labels' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in full mode (inputs filling are included in measurement loop). - [ INFO ] First inference took 578.24 ms + [ INFO ] First inference took 596.94 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] [ INFO ] Count: 55 iterations - [ INFO ] Duration: 15797.98 ms + [ INFO ] Duration: 15959.69 ms [ INFO ] Latency: - [ INFO ] Median: 1695.87 ms - [ INFO ] Average: 1683.49 ms - [ INFO ] Min: 550.40 ms - [ INFO ] Max: 1833.79 ms - [ INFO ] Throughput: 3.48 FPS + [ INFO ] Median: 1701.74 ms + [ INFO ] Average: 1692.86 ms + [ INFO ] Min: 653.76 ms + [ INFO ] Max: 1817.85 ms + [ INFO ] Throughput: 3.45 FPS Interactive segmentation demo @@ -1308,7 +1308,7 @@ Interactive segmentation demo .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam Running on local URL: http://127.0.0.1:7860 To create a public link, set `share=True` in `launch()`. diff --git a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_17_1.png b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_17_1.png index c6c2e7d72e694a..9f65fa9db4554a 100644 --- a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_17_1.png +++ b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_17_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ee7d364aeabe9a2787785e4d0ee5bb8951f530f90a0a625c9f31047bdc157b59 -size 1261144 +oid sha256:9368b1fbd458d1e022a768f24e689af0fd6e5dacc98a920f45d3fc0f63062567 +size 1259373 diff --git a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_25_1.png b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_25_1.png index ff03bff6360a90..7c0716600906a1 100644 --- a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_25_1.png +++ b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_25_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:40a049edc89f5339af256b80e0fc99740d9e3b6d3159e6e9d09e78683d24e0fe -size 1261722 +oid sha256:22f0e5bfd74e7426218d2bd007f9219433556530ddb10f33b9706398eb7cd370 +size 1263404 diff --git a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_36_1.png b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_36_1.png index 766db096877cba..0a717e2c9aa38d 100644 --- a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_36_1.png +++ b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_36_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f4fcf4eecf855fa1a9ca72c0dbf5c070e11a4a520873971202b592bc65dd2ccc -size 1261487 +oid sha256:d1863ccc9483f6cbd60768b311d104ee68692c3a7181e06da4bc751b52cf0ca1 +size 1262535 diff --git a/docs/notebooks/encodec-audio-compression-with-output.rst b/docs/notebooks/encodec-audio-compression-with-output.rst index 394c6e5b92e9bf..7f0e153ffa4a55 100644 --- a/docs/notebooks/encodec-audio-compression-with-output.rst +++ b/docs/notebooks/encodec-audio-compression-with-output.rst @@ -142,7 +142,7 @@ bandwidth. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. WeightNorm.apply(module, name, dim) @@ -302,7 +302,7 @@ similar as possible to the original. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. WeightNorm.apply(module, name, dim) @@ -402,13 +402,13 @@ with ``ov.save_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:60: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:60: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! ideal_length = (math.ceil(n_frames) - 1) * stride + (kernel_size - padding_total) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! max_pad = max(padding_left, padding_right) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:89: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:89: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if length <= max_pad: @@ -428,11 +428,11 @@ with ``ov.save_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:358: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:358: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. quantized_out = torch.tensor(0.0, device=q_indices.device) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:359: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results). + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:359: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results). for i, indices in enumerate(q_indices): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert (padding_left + padding_right) <= x.shape[-1] diff --git a/docs/notebooks/fast-segment-anything-with-output.rst b/docs/notebooks/fast-segment-anything-with-output.rst index c29a58c33e3490..e0f20e0f79974b 100644 --- a/docs/notebooks/fast-segment-anything-with-output.rst +++ b/docs/notebooks/fast-segment-anything-with-output.rst @@ -158,7 +158,7 @@ model and generate a segmentation map. .. parsed-literal:: - 100%|██████████| 138M/138M [00:02<00:00, 56.5MB/s] + 100%|██████████| 138M/138M [00:02<00:00, 67.7MB/s] @@ -170,8 +170,8 @@ model and generate a segmentation map. .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/fast-segment-anything/coco_bike.jpg: 768x1024 37 objects, 662.7ms - Speed: 3.8ms preprocess, 662.7ms inference, 766.0ms postprocess per image at shape (1, 3, 768, 1024) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/fast-segment-anything/coco_bike.jpg: 768x1024 37 objects, 728.3ms + Speed: 3.1ms preprocess, 728.3ms inference, 768.2ms postprocess per image at shape (1, 3, 768, 1024) The model returns segmentation maps for all the objects on the image. @@ -216,8 +216,8 @@ tracing. The FastSAM model itself is based on YOLOv8 model. OpenVINO: starting export with openvino 2024.4.0-16579-c3152d32c9c-releases/2024/4... OpenVINO: export success ✅ 6.2s, saved as 'FastSAM-x_openvino_model/' (276.1 MB) - Export complete (9.2s) - Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/fast-segment-anything + Export complete (9.1s) + Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/fast-segment-anything Predict: yolo predict task=segment model=FastSAM-x_openvino_model imgsz=1024 Validate: yolo val task=segment model=FastSAM-x_openvino_model imgsz=1024 data=ultralytics/datasets/sa.yaml Visualize: https://netron.app @@ -321,8 +321,8 @@ pipeline. .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/fast-segment-anything/coco_bike.jpg: 1024x1024 42 objects, 558.8ms - Speed: 5.7ms preprocess, 558.8ms inference, 34.6ms postprocess per image at shape (1, 3, 1024, 1024) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/fast-segment-anything/coco_bike.jpg: 1024x1024 42 objects, 504.9ms + Speed: 5.8ms preprocess, 504.9ms inference, 31.6ms postprocess per image at shape (1, 3, 1024, 1024) One can observe the converted model outputs in the next cell, they is @@ -643,9 +643,9 @@ calibration dataset to measure the performance. .. parsed-literal:: - Segmented in 23 seconds - Resulting in 5.57 fps - That is 3.0 times faster! + Segmented in 22 seconds + Resulting in 5.82 fps + That is 3.14 times faster! Try out the converted pipeline diff --git a/docs/notebooks/florence2-with-output.rst b/docs/notebooks/florence2-with-output.rst index 4cca2d85a2bd6c..e929a95fb182c1 100644 --- a/docs/notebooks/florence2-with-output.rst +++ b/docs/notebooks/florence2-with-output.rst @@ -100,10 +100,10 @@ available model. By default, we will use .. parsed-literal:: - 2024-10-23 01:17:38.434215: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:17:38.467940: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 01:28:54.034484: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:28:54.069316: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:17:39.118965: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:28:54.728430: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -193,49 +193,49 @@ pipeline. .. parsed-literal:: - SECURITY.md: 0%| | 0.00/2.66k [00:00 1 or self.sliding_window is not None: /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/chkpt/modeling_florence2.py:1205: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! is_causal = True if self.is_causal and attention_mask is None and tgt_len > 1 else False diff --git a/docs/notebooks/florence2-with-output_files/florence2-with-output_18_0.png b/docs/notebooks/florence2-with-output_files/florence2-with-output_18_0.png index a1484a3eb4b6d0..37d11a47fd30c9 100644 --- a/docs/notebooks/florence2-with-output_files/florence2-with-output_18_0.png +++ b/docs/notebooks/florence2-with-output_files/florence2-with-output_18_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f5808e660c685e4518f0a83232c2b9c9e88263e05ecd05891f769d263df4c642 -size 259656 +oid sha256:d85b3df68708172ed849a9e182bdec6a94f0174643833bd8cc7184ac0d090fae +size 259636 diff --git a/docs/notebooks/flux.1-image-generation-with-output.rst b/docs/notebooks/flux.1-image-generation-with-output.rst index 62549bd074d4a6..fe0c47899c3601 100644 --- a/docs/notebooks/flux.1-image-generation-with-output.rst +++ b/docs/notebooks/flux.1-image-generation-with-output.rst @@ -26,7 +26,11 @@ using OpenVINO. - `Prerequisites <#prerequisites>`__ - `Select model <#select-model>`__ - `Convert model with OpenVINO <#convert-model-with-openvino>`__ -- `Compress model weights <#compress-model-weights>`__ + + - `Convert model using Optimum + Intel <#convert-model-using-optimum-intel>`__ + - `Compress model weights <#compress-model-weights>`__ + - `Run OpenVINO model inference <#run-openvino-model-inference>`__ - `Interactive demo <#interactive-demo>`__ @@ -47,8 +51,9 @@ Prerequisites .. code:: ipython3 - %pip install -q "gradio>=4.19" "torch>=2.1" "transformers" "nncf>=2.12.0" "diffusers>=0.30.0" "opencv-python" "pillow" "peft>=0.7.0" --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q "gradio>=4.19" "torch>=2.1" "transformers" "nncf>=2.12.0" "diffusers>=0.31.0" "opencv-python" "pillow" "peft>=0.7.0" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "sentencepiece" "protobuf" + %pip install -q "git+https://github.com/huggingface/optimum-intel.git" %pip install -qU "openvino>=2024.4.0" .. code:: ipython3 @@ -56,9 +61,9 @@ Prerequisites import requests from pathlib import Path - if not Path("flux_helper.py").exists(): - r = requests.get(url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/flux.1-image-generation/flux_helper.py") - open("flux_helper.py", "w").write(r.text) + if not Path("cmd_helper.py").exists(): + r = requests.get(url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/cmd_helper.py") + open("cmd_helper.py", "w").write(r.text) if not Path("gradio_helper.py").exists(): r = requests.get(url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/flux.1-image-generation/gradio_helper.py") @@ -100,25 +105,19 @@ FLUX.1-dev version using widget bellow. .. code:: ipython3 - from flux_helper import get_model_selector + import ipywidgets as widgets - model_selector = get_model_selector() - model_selector - - -.. parsed-literal:: - - INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino + model_ids = ["black-forest-labs/FLUX.1-schnell", "black-forest-labs/FLUX.1-dev"] + + model_selector = widgets.Dropdown( + options=model_ids, + default=model_ids[0], + description="Model:", + ) - -.. parsed-literal:: - - 2024-08-13 17:30:13.543036: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-08-13 17:30:13.544738: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used. - 2024-08-13 17:30:13.579013: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-08-13 17:30:14.449873: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + model_selector + @@ -169,40 +168,44 @@ The pipeline consists of four important parts: - Transformer for step-by-step denoising latent image representation. - Autoencoder (VAE) for decoding latent space to image. -We will use ``convert_flux`` helper function defined in -`flux_helper.py `__ that create original PyTorch model -and convert each part of pipeline using ``ov.convert_model``. - -.. code:: ipython3 +Convert model using Optimum Intel +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - from flux_helper import convert_flux - - # uncomment the line to see model conversion code - # ??convert_flux -.. code:: ipython3 - model_dir = convert_flux(model_selector.value) +For convenience, we will use OpenVINO integration with HuggingFace +Optimum. `Optimum +Intel `__ is the +interface between the Transformers and Diffusers libraries and the +different tools and libraries provided by Intel to accelerate end-to-end +pipelines on Intel architectures. +Among other use cases, Optimum Intel provides a simple interface to +optimize your Transformers and Diffusers models, convert them to the +OpenVINO Intermediate Representation (IR) format and run inference using +OpenVINO Runtime. ``optimum-cli`` provides command line interface for +model conversion and optimization. -.. parsed-literal:: +General command format: - ✅ black-forest-labs/FLUX.1-schnell model already converted and can be found in FLUX.1-schnell - +.. code:: bash -.. code:: ipython3 + optimum-cli export openvino --model --task - from flux_helper import TRANSFORMER_PATH, VAE_DECODER_PATH, TEXT_ENCODER_PATH, TEXT_ENCODER_2_PATH - - model_dict = { - "transformer": model_dir / TRANSFORMER_PATH, - "text_encoder": model_dir / TEXT_ENCODER_PATH, - "text_encoder_2": model_dir / TEXT_ENCODER_2_PATH, - "vae": model_dir / VAE_DECODER_PATH, - } +where task is task to export the model for, if not specified, the task +will be auto-inferred based on the model. You can find a mapping between +tasks and model classes in Optimum TaskManager +`documentation `__. +Additionally, you can specify weights compression using +``--weight-format`` argument with one of following options: ``fp32``, +``fp16``, ``int8`` and ``int4``. Fro int8 and int4 +`nncf `__ will be used for +weight compression. More details about model export provided in `Optimum +Intel +documentation `__. Compress model weights ----------------------- +~~~~~~~~~~~~~~~~~~~~~~ @@ -215,14 +218,16 @@ where the size of weights is relatively larger than the size of activations, for example, Large Language Models (LLM). Compared to INT8 compression, INT4 compression improves performance even more, but introduces a minor drop in prediction quality. We will use -`NNCF `__ for weight -compression. +`NNCF `__ integration to +``optimum-cli`` tool for weight compression. .. code:: ipython3 - from flux_helper import weight_compression_widget - - to_compress = weight_compression_widget() + to_compress = widgets.Checkbox( + value=True, + description="Weight compression", + disabled=False, + ) to_compress @@ -237,112 +242,98 @@ compression. .. code:: ipython3 - import nncf - import openvino as ov - import gc + from pathlib import Path - compression_args = {"mode": nncf.CompressWeightsMode.INT4_SYM, "group_size": 64, "ratio": 1.0} + model_id = model_selector.value - int4_model_dict = {} + model_base_dir = Path(model_id.split("/")[-1]) + additional_args = {} if to_compress.value: - core = ov.Core() - - for model_name, model_path in model_dict.items(): - int4_path = model_path.parent / (model_path.stem + "_int4.xml") - if not int4_path.exists(): - print(f"⌛ {model_path.stem} compression started") - print( - f"Compression parameters:\n\tmode = {compression_args['mode']}\n\tratio = {compression_args['ratio']}\n\tgroup_size = {compression_args['group_size']}" - ) - model = core.read_model(model_path) - compressed_model = nncf.compress_weights(model, **compression_args) - ov.save_model(compressed_model, int4_path) - print(f"✅ {model_path.stem} compression finished") - del compressed_model - del model - gc.collect() - print(f"Compressed {model_path.stem} can be found in {int4_path}") - int4_model_dict[model_name] = int4_path + model_dir = model_base_dir / "INT4" + additional_args.update({"weight-format": "int4", "group-size": "64", "ratio": "1.0"}) + else: + model_dir = model_base_dir / "FP16" + additional_args.update({"weight-format": "fp16"}) +.. code:: ipython3 -.. parsed-literal:: - - Compressed transformer can be found in FLUX.1-schnell/transformer/transformer_int4.xml - Compressed text_encoder can be found in FLUX.1-schnell/text_encoder/text_encoder_int4.xml - Compressed text_encoder_2 can be found in FLUX.1-schnell/text_encoder_2/text_encoder_2_int4.xml - Compressed vae_decoder can be found in FLUX.1-schnell/vae/vae_decoder_int4.xml + from cmd_helper import optimum_cli + if not model_dir.exists(): + optimum_cli(model_id, model_dir, additional_args=additional_args) Run OpenVINO model inference ---------------------------- -``OVFluxPipeline`` class defined in ``flux_helper.py`` provides -convenient way for running model. It accepts directory with converted -model and inference device as arguments. +``OVDiffusionPipeline`` from Optimum Intel provides ready-to-use +interface for running Diffusers models using OpenVINO. It supports +various models including Stable Diffusion, Stable Diffusion XL, LCM, +Stable Diffusion v3 and Flux. Similar to original Diffusers pipeline, +for initialization, we should use ``from_preptrained`` method providing +model id from HuggingFace hub or local directory (both original PyTorch +and OpenVINO models formats supported, in the first case model class +additionally will trigger model conversion). .. code:: ipython3 - from flux_helper import get_pipeline_selection_option + from notebook_utils import device_widget - use_compressed = get_pipeline_selection_option(int4_model_dict) - use_compressed + device = device_widget(default="CPU", exclude=["NPU"]) + device .. parsed-literal:: - Checkbox(value=True, description='Use compressed models') + Dropdown(description='Device:', options=('CPU', 'AUTO'), value='CPU') .. code:: ipython3 - from flux_helper import OVFluxPipeline, init_pipeline # noqa: F401 + import ipywidgets as widgets - # uncomment the line to see model pipeline - # ??OVFluxPipeline - -.. code:: ipython3 - - from notebook_utils import device_widget + model_available = (model_base_dir / "INT4").is_dir() + use_quantized_models = widgets.Checkbox( + value=model_available, + description="Use compressed models", + disabled=not model_available, + ) - device = device_widget(default="CPU", exclude=["NPU"]) - device + use_quantized_models .. parsed-literal:: - Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') + Checkbox(value=True, description='Use compressed models') .. code:: ipython3 - ov_pipe = init_pipeline(model_dir, model_dict if not use_compressed.value else int4_model_dict, device.value) - - -.. parsed-literal:: - - Models compilation - ✅ transformer - Done! - ✅ text_encoder - Done! - ✅ text_encoder_2 - Done! + from optimum.intel.openvino import OVDiffusionPipeline - -.. parsed-literal:: - - You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers + model_dir = model_base_dir / "INT4" if use_quantized_models.value else model_base_dir / "FP16" + ov_pipe = OVDiffusionPipeline.from_pretrained(model_dir, device=device.value) + .. parsed-literal:: - ✅ vae - Done! + 2024-10-28 18:12:30.714636: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-10-28 18:12:30.727116: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:477] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered + WARNING: All log messages before absl::InitializeLog() is called are written to STDERR + E0000 00:00:1730124750.741387 52454 cuda_dnn.cc:8310] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered + E0000 00:00:1730124750.745955 52454 cuda_blas.cc:1418] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered + 2024-10-28 18:12:30.761443: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers .. code:: ipython3 @@ -365,7 +356,7 @@ model and inference device as arguments. -.. image:: flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.png +.. image:: flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.png diff --git a/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.jpg b/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.jpg new file mode 100644 index 00000000000000..6ad8e593ef5115 --- /dev/null +++ b/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5caeca2e3ae8e99146bb786321cbad04584523b693e3cbdba2a5f8c7f0dbb096 +size 13490 diff --git a/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.png b/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.png new file mode 100644 index 00000000000000..7cb1e2903a902f --- /dev/null +++ b/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0de23e409e068f39c1e0e568b921174c80b0c42e29a1473eeeba54a91b88aaef +size 113356 diff --git a/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.jpg b/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.jpg deleted file mode 100644 index 7223074e225dd8..00000000000000 --- a/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8923d4a7ce04ed66bb58c28ab4450594a0340b97a15245fc594e669d33b574ef -size 14369 diff --git a/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.png b/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.png deleted file mode 100644 index 83b60f80d75c13..00000000000000 --- a/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:fab1fa5bf3f71d6bb6f505658da6765d37295b7b9618560d5f7c0c344e2a3896 -size 116250 diff --git a/docs/notebooks/freevc-voice-conversion-with-output.rst b/docs/notebooks/freevc-voice-conversion-with-output.rst index a3c914a44b36a7..fe2ac780f5cca6 100644 --- a/docs/notebooks/freevc-voice-conversion-with-output.rst +++ b/docs/notebooks/freevc-voice-conversion-with-output.rst @@ -104,7 +104,7 @@ Check if FreeVC is installed and append its path to ``sys.path`` remote: Counting objects: 100% (74/74), done. remote: Compressing objects: 100% (47/47), done. remote: Total 131 (delta 43), reused 27 (delta 27), pack-reused 57 (from 1) - Receiving objects: 100% (131/131), 15.28 MiB | 3.81 MiB/s, done. + Receiving objects: 100% (131/131), 15.28 MiB | 17.50 MiB/s, done. Resolving deltas: 100% (43/43), done. @@ -134,8 +134,8 @@ Check if FreeVC is installed and append its path to ``sys.path`` Downloading... From: https://drive.google.com/uc?id=12-cB34qCTvByWT-QtOcZaqwwO21FLSqU&confirm=t&uuid=a703c43c-ccce-436c-8799-c11b88e9e7e4 - To: /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/WavLM-Large.pt - 100%|██████████| 1.26G/1.26G [00:26<00:00, 48.4MB/s] + To: /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/WavLM-Large.pt + 100%|██████████| 1.26G/1.26G [00:32<00:00, 38.5MB/s] .. code:: ipython3 @@ -239,7 +239,7 @@ Models initialization .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. warnings.warn("torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.") @@ -288,7 +288,7 @@ Inference .. parsed-literal:: - 2it [00:03, 1.72s/it] + 2it [00:04, 2.03s/it] Result audio files should be available in ‘outputs/freevc’ @@ -360,13 +360,13 @@ Converting to OpenVINO’s IR format. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:495: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:495: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert embed_dim == self.embed_dim - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:496: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:496: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert list(query.size()) == [tgt_len, bsz, embed_dim] - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:500: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:500: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert key_bsz == bsz - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:502: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:502: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert src_len, bsz == value.shape[:2] @@ -581,12 +581,12 @@ function to OpenVINO IR format. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1102: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1102: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: Tensor-likes are not close! - Mismatched elements: 25909 / 25920 (100.0%) - Greatest absolute difference: 0.713585376739502 at index (0, 0, 4783) (up to 1e-05 allowed) - Greatest relative difference: 22806.258007743752 at index (0, 0, 19024) (up to 1e-05 allowed) + Mismatched elements: 25915 / 25920 (100.0%) + Greatest absolute difference: 1.3485908806324005 at index (0, 0, 24258) (up to 1e-05 allowed) + Greatest relative difference: 8204.075456053068 at index (0, 0, 5777) (up to 1e-05 allowed) _check_trace( @@ -645,7 +645,7 @@ And now we can check inference using only IR models. .. parsed-literal:: - 2it [00:01, 1.30it/s] + 2it [00:01, 1.31it/s] Result audio files should be available in ‘outputs/freevc’ and you can @@ -707,7 +707,7 @@ Result audio: diff --git a/docs/notebooks/gpu-device-with-output.rst b/docs/notebooks/gpu-device-with-output.rst index 2b35846b766bd1..732cc297aa9531 100644 --- a/docs/notebooks/gpu-device-with-output.rst +++ b/docs/notebooks/gpu-device-with-output.rst @@ -21,13 +21,7 @@ Working with GPUs in OpenVINO™ - `Compiling a Model on GPU <#compiling-a-model-on-gpu>`__ - - `Download and Convert a Model <#download-and-convert-a-model>`__ - - - `Download and unpack the - Model <#download-and-unpack-the-model>`__ - - `Convert the Model to OpenVINO IR - format <#convert-the-model-to-openvino-ir-format>`__ - + - `Download a Model <#download-a-model>`__ - `Compile with Default Configuration <#compile-with-default-configuration>`__ - `Reduce Compile Time through Model @@ -119,10 +113,7 @@ Install required packages .. code:: ipython3 - %pip install -q "openvino-dev>=2024.0.0" "opencv-python" "tqdm" - %pip install -q "tensorflow-macos>=2.5; sys_platform == 'darwin' and platform_machine == 'arm64' and python_version > '3.8'" # macOS M1 and M2 - %pip install -q "tensorflow>=2.5; sys_platform == 'darwin' and platform_machine != 'arm64' and python_version > '3.8'" # macOS x86 - %pip install -q "tensorflow>=2.5; sys_platform != 'darwin' and python_version > '3.8'" + %pip install -q "openvino>=2024.4.0" "opencv-python" "tqdm" "huggingface_hub" Checking GPUs with Query Device ------------------------------- @@ -305,8 +296,8 @@ properties. We can easily use one for compiling and running models with OpenVINO `GPU plugin `__. -Download and Convert a Model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download a Model +~~~~~~~~~~~~~~~~ @@ -317,19 +308,9 @@ was trained on `Common Objects in Context categories of object. For details, see the `paper `__. -Download and unpack the Model -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - - -Use the ``download_file`` function from the ``notebook_utils`` to -download an archive with the model. It automatically creates a directory -structure and downloads the selected model. This step is skipped if the -package is already downloaded. - .. code:: ipython3 - import tarfile + import huggingface_hub as hf_hub from pathlib import Path # Fetch `notebook_utils` module @@ -340,95 +321,18 @@ package is already downloaded. ) open("notebook_utils.py", "w").write(r.text) - from notebook_utils import download_file # A directory where the model will be downloaded. base_model_dir = Path("./model").expanduser() - model_name = "ssdlite_mobilenet_v2" - archive_name = Path(f"{model_name}_coco_2018_05_09.tar.gz") - - # Download the archive - downloaded_model_path = base_model_dir / archive_name - if not downloaded_model_path.exists(): - model_url = f"http://download.tensorflow.org/models/object_detection/{archive_name}" - download_file(model_url, downloaded_model_path.name, downloaded_model_path.parent) - - # Unpack the model - tf_model_path = base_model_dir / archive_name.with_suffix("").stem / "frozen_inference_graph.pb" - if not tf_model_path.exists(): - with tarfile.open(downloaded_model_path) as file: - file.extractall(base_model_dir) - - - -.. parsed-literal:: - - model/ssdlite_mobilenet_v2_coco_2018_05_09.tar.gz: 0%| | 0.00/48.7M [00:00`__. - -.. code:: ipython3 - - from openvino.tools.mo.front import tf as ov_tf_front - - precision = "FP16" + model_name = "ssdlite_mobilenet_v2_fp16" - # The output path for the conversion. - model_path = base_model_dir / "ir_model" / f"{model_name}_{precision.lower()}.xml" + ov_model_path = base_model_dir / model_name / f"{model_name}.xml" - trans_config_path = Path(ov_tf_front.__file__).parent / "ssd_v2_support.json" - pipeline_config = base_model_dir / archive_name.with_suffix("").stem / "pipeline.config" + if not (ov_model_path).exists(): + hf_hub.snapshot_download("katuni4ka/ssdlite_mobilenet_v2_fp16", local_dir=base_model_dir) - model = None - if not model_path.exists(): - model = ov.tools.mo.convert_model( - input_model=tf_model_path, - input_shape=[1, 300, 300, 3], - layout="NHWC", - transformations_config=trans_config_path, - tensorflow_object_detection_api_pipeline_config=pipeline_config, - reverse_input_channels=True, - ) - ov.save_model(model, model_path, compress_to_fp16=(precision == "FP16")) - print("IR model saved to {}".format(model_path)) - else: - print("Read IR model from {}".format(model_path)) - model = core.read_model(model_path) - - -.. parsed-literal:: - - [ WARNING ] The Preprocessor block has been removed. Only nodes performing mean value subtraction and scaling (if applicable) are kept. - - -.. parsed-literal:: - - IR model saved to model/ir_model/ssdlite_mobilenet_v2_fp16.xml - + model = core.read_model(ov_model_path) Compile with Default Configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -484,7 +388,7 @@ following: core.set_property({props.cache_dir(): cache_folder}) # Compile the model as before - model = core.read_model(model=model_path) + model = core.read_model(ov_model_path) compiled_model = core.compile_model(model, device) print(f"Cache enabled (first time) - compile time: {time.time() - start}s") @@ -502,13 +406,13 @@ compile times with caching enabled and disabled as follows: start = time.time() core = ov.Core() core.set_property({props.cache_dir(): "cache"}) - model = core.read_model(model=model_path) + model = core.read_model(model=ov_model_path) compiled_model = core.compile_model(model, device) print(f"Cache enabled - compile time: {time.time() - start}s") start = time.time() core = ov.Core() - model = core.read_model(model=model_path) + model = core.read_model(ov_model_path) compiled_model = core.compile_model(model, device) print(f"Cache disabled - compile time: {time.time() - start}s") diff --git a/docs/notebooks/grounded-segment-anything-with-output.rst b/docs/notebooks/grounded-segment-anything-with-output.rst index 51522e791e04a4..232629422b14e0 100644 --- a/docs/notebooks/grounded-segment-anything-with-output.rst +++ b/docs/notebooks/grounded-segment-anything-with-output.rst @@ -124,16 +124,16 @@ segmentation you can select vanilla ``SAM``. Cloning into 'GroundingDINO'... remote: Enumerating objects: 379, done. remote: Counting objects: 100% (190/190), done. - remote: Compressing objects: 100% (81/81), done. - remote: Total 379 (delta 135), reused 109 (delta 109), pack-reused 189 (from 1) - Receiving objects: 100% (379/379), 14.03 MiB | 18.28 MiB/s, done. + remote: Compressing objects: 100% (79/79), done. + remote: Total 379 (delta 136), reused 111 (delta 111), pack-reused 189 (from 1) + Receiving objects: 100% (379/379), 14.03 MiB | 20.95 MiB/s, done. Resolving deltas: 100% (194/194), done. Cloning into 'EfficientSAM'... remote: Enumerating objects: 424, done. remote: Counting objects: 100% (85/85), done. remote: Compressing objects: 100% (33/33), done. remote: Total 424 (delta 76), reused 52 (delta 52), pack-reused 339 (from 1) - Receiving objects: 100% (424/424), 262.14 MiB | 30.07 MiB/s, done. + Receiving objects: 100% (424/424), 262.14 MiB | 24.44 MiB/s, done. Resolving deltas: 100% (246/246), done. @@ -222,6 +222,10 @@ GroundingDINO imports .. parsed-literal:: + 2024-11-05 01:34:53.765709: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:34:53.988314: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2024-11-05 01:34:54.760718: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers UserWarning: Failed to load custom C++ ops. Running on CPU mode Only! @@ -362,6 +366,30 @@ Convert GroundingDINO to OpenVINO IR format TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + + +.. parsed-literal:: + + output layer_id 0 is nan + num_nan 230400, num_inf 0 + output layer_id 1 is nan + num_nan 230400, num_inf 0 + output layer_id 2 is nan + num_nan 230400, num_inf 0 + output layer_id 3 is nan + num_nan 230400, num_inf 0 + output layer_id 4 is nan + num_nan 230400, num_inf 0 + output layer_id 5 is nan + num_nan 230400, num_inf 0 + WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. + + +.. parsed-literal:: + + [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. Run OpenVINO optimized GroundingDINO @@ -504,15 +532,6 @@ class, but the inference will be done using OpenVINO optimized model. boxes_filt, pred_phrases, logits_filt = get_ov_grounding_output(ov_compiled_grounded_dino, pil_image, classes_prompt, BOX_THRESHOLD, TEXT_THRESHOLD) - -.. parsed-literal:: - - 2024-10-23 01:22:04.926963: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:22:04.966234: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:22:05.582957: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - - Convert predicted boxes to supervision box detections format .. code:: ipython3 @@ -573,11 +592,6 @@ segmentation. First of all let’s convert ``SAM`` model to OpenVINO IR. ov_efficient_sam = core.read_model(ov_efficient_sam_path) -.. parsed-literal:: - - WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. - - .. parsed-literal:: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! diff --git a/docs/notebooks/hello-segmentation-with-output.rst b/docs/notebooks/hello-segmentation-with-output.rst index e490cf48533277..6ddc0e3b0aa78b 100644 --- a/docs/notebooks/hello-segmentation-with-output.rst +++ b/docs/notebooks/hello-segmentation-with-output.rst @@ -188,7 +188,7 @@ is provided. .. parsed-literal:: - + @@ -215,7 +215,7 @@ Do Inference .. parsed-literal:: - + diff --git a/docs/notebooks/hugging-face-hub-with-output.rst b/docs/notebooks/hugging-face-hub-with-output.rst index e17d63a031b445..a92f8cd18fba31 100644 --- a/docs/notebooks/hugging-face-hub-with-output.rst +++ b/docs/notebooks/hugging-face-hub-with-output.rst @@ -132,6 +132,10 @@ tutorials `__. To disable this warning, you can either: - Avoid using `tokenizers` before the fork if possible - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) - 2024-10-23 01:23:29.312809: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:23:29.347879: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:23:29.943155: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Initialize and Convert the Model Automatically using OVModel class @@ -377,11 +384,6 @@ inference run. - This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model). -.. parsed-literal:: - - WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. - - Convert model using Optimum CLI interface ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -434,7 +436,7 @@ Full list of supported arguments available via ``--help`` .. parsed-literal:: - 2024-10-23 01:23:44.450300: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:37:12.161579: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT usage: optimum-cli export openvino [-h] -m MODEL [--task TASK] [--framework {pt,tf}] [--trust-remote-code] [--weight-format {fp32,fp16,int8,int4,mxfp4}] @@ -465,20 +467,20 @@ Full list of supported arguments available via ``--help`` --task TASK The task to export the model for. If not specified, the task will be auto-inferred based on the model. Available tasks depend on the model, but are among: - ['audio-xvector', 'text-to-audio', 'fill-mask', - 'feature-extraction', 'text-generation', 'zero-shot- - image-classification', 'text-to-image', 'text2text- - generation', 'zero-shot-object-detection', 'automatic- - speech-recognition', 'text-classification', 'semantic- - segmentation', 'masked-im', 'image-to-text', - 'sentence-similarity', 'object-detection', - 'inpainting', 'audio-frame-classification', 'image-to- - image', 'token-classification', 'question-answering', - 'audio-classification', 'image-segmentation', 'image- - classification', 'mask-generation', 'multiple-choice', - 'depth-estimation']. For decoder models, use `xxx- - with-past` to export the model using past key values - in the decoder. + ['mask-generation', 'image-classification', 'fill- + mask', 'audio-xvector', 'audio-frame-classification', + 'sentence-similarity', 'multiple-choice', 'automatic- + speech-recognition', 'text-to-image', 'token- + classification', 'image-to-text', 'image- + segmentation', 'question-answering', 'depth- + estimation', 'semantic-segmentation', 'feature- + extraction', 'text-generation', 'zero-shot-object- + detection', 'text-to-audio', 'zero-shot-image- + classification', 'object-detection', 'text2text- + generation', 'audio-classification', 'image-to-image', + 'masked-im', 'inpainting', 'text-classification']. For + decoder models, use `xxx-with-past` to export the + model using past key values in the decoder. --framework {pt,tf} The framework to use for the export. If not provided, will attempt to use the local checkpoint's original framework or what is available in the environment. @@ -583,10 +585,11 @@ compression: .. parsed-literal:: - 2024-10-23 01:23:49.987001: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:37:17.680673: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Some weights of the model checkpoint at cardiffnlp/twitter-roberta-base-sentiment-latest were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] - This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model). - This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model). + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. Tokenizer won't be converted. diff --git a/docs/notebooks/image-classification-quantization-with-output.rst b/docs/notebooks/image-classification-quantization-with-output.rst index b5b33fdfc852e7..7bf7172f720588 100644 --- a/docs/notebooks/image-classification-quantization-with-output.rst +++ b/docs/notebooks/image-classification-quantization-with-output.rst @@ -102,9 +102,9 @@ Model preparation stage has the following steps: Cloning into 'pytorch-cifar-models'... remote: Enumerating objects: 282, done. remote: Counting objects: 100% (281/281), done. - remote: Compressing objects: 100% (96/96), done. - remote: Total 282 (delta 135), reused 269 (delta 128), pack-reused 1 (from 1) - Receiving objects: 100% (282/282), 9.22 MiB | 24.58 MiB/s, done. + remote: Compressing objects: 100% (94/94), done. + remote: Total 282 (delta 135), reused 275 (delta 130), pack-reused 1 (from 1) + Receiving objects: 100% (282/282), 9.22 MiB | 9.58 MiB/s, done. Resolving deltas: 100% (135/135), done. @@ -176,7 +176,7 @@ Preprocessing for model obtained from training .. parsed-literal:: - 100%|██████████| 170498071/170498071 [00:07<00:00, 23490143.19it/s] + 100%|██████████| 170498071/170498071 [00:07<00:00, 22536051.32it/s] .. parsed-literal:: @@ -248,10 +248,10 @@ about supported parameters can be found on this .. parsed-literal:: - 2024-10-23 01:24:29.479834: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:24:29.511386: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 01:37:57.500572: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:37:57.532367: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:24:30.071901: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:37:58.074631: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -419,7 +419,7 @@ Tool `__ - `Interactive demo <#interactive-demo>`__ + Installation Instructions ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/notebooks/internvl2-with-output.rst b/docs/notebooks/internvl2-with-output.rst index 602bfe84590494..ed67209a0303eb 100644 --- a/docs/notebooks/internvl2-with-output.rst +++ b/docs/notebooks/internvl2-with-output.rst @@ -279,7 +279,7 @@ documentation self.max_seq_len_cached: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/qwen2/modeling_qwen2.py:324: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/qwen2/modeling_qwen2.py:324: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/qwen2/modeling_qwen2.py:339: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/qwen2/modeling_qwen2.py:339: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): @@ -531,7 +531,7 @@ Let’s check model capabilities in answering questions about image: .. parsed-literal:: - The image depicts a red panda, a large feline with a distinctive, reddish-brown coat and white face and chest. It is peeking over what appears to be a wooden platform or platform made for panda viewing in captivity. The background is filled with greenery, indicating that the photo was likely taken in a conservatory or wildlife park where penguins or seabirds are displayed. + The image shows a red panda lying on its side, partially wrapped in a wooden structure, possibly a container or log. The red panda appears to be looking at the camera with large, expressive eyes, displaying an endearing and lively appearance. The background consists of a portion of the red panda's habitat environment, which appears to be a tree and some greenery. Interactive demo ---------------- diff --git a/docs/notebooks/jina-clip-with-output.rst b/docs/notebooks/jina-clip-with-output.rst index 466e3be5d03fd1..1cdb2e1d286245 100644 --- a/docs/notebooks/jina-clip-with-output.rst +++ b/docs/notebooks/jina-clip-with-output.rst @@ -106,11 +106,11 @@ weights, using ``from_pretrained`` method. .. parsed-literal:: - 2024-10-23 01:28:42.177313: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:28:42.211816: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 01:41:58.578137: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:41:58.612620: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:28:42.875342: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + 2024-11-05 01:41:59.276782: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) @@ -301,7 +301,7 @@ loading on device using ``core.complie_model``. .. parsed-literal:: WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4713: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4713: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/jinaai/jina-bert-flash-implementation/b78d1595de294f13ffe7b19d6cd63892a6e4e7a4/mha.py:333: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! softmax_scale = self.softmax_scale or 1.0 / math.sqrt(q.shape[-1]) @@ -820,8 +820,8 @@ approximately estimate the speed up of the dynamic quantized models. .. parsed-literal:: - Performance speed up for text model: 1.560 - Performance speed up for vision model: 1.435 + Performance speed up for text model: 1.978 + Performance speed up for vision model: 1.428 Gradio demo diff --git a/docs/notebooks/knowledge-graphs-conve-with-output.rst b/docs/notebooks/knowledge-graphs-conve-with-output.rst index b9e4f56ebb05af..de9115fd9ab4a8 100644 --- a/docs/notebooks/knowledge-graphs-conve-with-output.rst +++ b/docs/notebooks/knowledge-graphs-conve-with-output.rst @@ -233,7 +233,7 @@ Download Model Checkpoint .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/knowledge-graphs-conve/models/conve.pt') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/knowledge-graphs-conve/models/conve.pt') @@ -395,7 +395,7 @@ typical to use metrics such as Mean Reciprocal Rank, Hits@10 etc. .. parsed-literal:: - Average time taken for inference: 0.6894171237945557 ms + Average time taken for inference: 0.7430613040924072 ms Mean accuracy of the model on the test dataset: 0.875 @@ -534,7 +534,7 @@ select device from dropdown list for running inference using OpenVINO .. parsed-literal:: - Average time taken for inference: 0.8764564990997314 ms + Average time taken for inference: 1.0752081871032715 ms Mean accuracy of the model on the test dataset: 0.10416666666666667 @@ -553,7 +553,7 @@ Determine the platform specific speedup obtained through OpenVINO graph optimiza .. parsed-literal:: - Speedup with OpenVINO optimizations: 0.79 X + Speedup with OpenVINO optimizations: 0.69 X Benchmark the converted OpenVINO model using benchmark app @@ -598,7 +598,7 @@ inference can also be obtained by looking at the benchmark app results. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 4.34 ms + [ INFO ] Read model took 4.36 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] e1 (node: e1) : i64 / [...] / [] @@ -614,7 +614,7 @@ inference can also be obtained by looking at the benchmark app results. [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: aten::softmax/Softmax) : f32 / [...] / [1,271] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 72.42 ms + [ INFO ] Compile model took 54.92 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -653,17 +653,17 @@ inference can also be obtained by looking at the benchmark app results. [ INFO ] Fill input 'rel' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 10000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 1.79 ms + [ INFO ] First inference took 1.66 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 94740 iterations - [ INFO ] Duration: 10002.15 ms + [ INFO ] Count: 94596 iterations + [ INFO ] Duration: 10001.69 ms [ INFO ] Latency: - [ INFO ] Median: 1.08 ms - [ INFO ] Average: 1.08 ms - [ INFO ] Min: 0.74 ms - [ INFO ] Max: 9.80 ms - [ INFO ] Throughput: 9471.96 FPS + [ INFO ] Median: 1.07 ms + [ INFO ] Average: 1.09 ms + [ INFO ] Min: 0.79 ms + [ INFO ] Max: 8.57 ms + [ INFO ] Throughput: 9458.00 FPS Conclusions diff --git a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output.rst b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output.rst index c18464f77585b2..b6a7a971fef3be 100644 --- a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output.rst +++ b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output.rst @@ -85,7 +85,7 @@ Install requirements .. parsed-literal:: - Requirement already satisfied: pip in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (24.2) + Requirement already satisfied: pip in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (24.3.1) Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -152,10 +152,10 @@ example `__ .. parsed-literal:: - 2024-10-23 01:31:37.373532: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:31:37.408388: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 01:44:54.753766: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:44:54.788691: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:31:37.931462: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:44:55.309895: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -374,11 +374,11 @@ Vision model accept ``pixel_values`` and returns ``image_embeds``. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4713: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4713: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:465: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:465: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:505: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:505: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): @@ -406,7 +406,7 @@ Convert Image To Text Projection model .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:165: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at aten/src/ATen/core/TensorBody.h:489.) + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:165: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at aten/src/ATen/core/TensorBody.h:489.) if a.grad is not None: @@ -541,13 +541,13 @@ generated text by ``AutoProcessor``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:804: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:804: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if max_pos > self.weights.size(0): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:1113: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:1113: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input_shape[-1] > 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:920: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:920: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attention_mask.size() != (batch_size, 1, seq_length, src_len): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:1206: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:1206: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: @@ -1389,9 +1389,9 @@ pipelines, we use mean inference time on 7 samples. .. parsed-literal:: - FP32 pipeline: 2.696 seconds - Optimized pipeline: 1.137 seconds - Performance speed-up: 2.372 + FP32 pipeline: 2.746 seconds + Optimized pipeline: 1.140 seconds + Performance speed-up: 2.409 Interactive inference diff --git a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.jpg b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.jpg index 9673054cc1b8e0..2310cb001b0c6b 100644 --- a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.jpg +++ b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c45f05bcfa118a3d7e9024cf83963a4bb5504cb97f669c0c8261c469bab6dc75 -size 121985 +oid sha256:9ca596f09c0f6c0dafa4aca0fbe7974941301cfcbc6bcb3a8c4255774c347d0b +size 123320 diff --git a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.png b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.png index a5be9e2155f6cc..91289c35d7c60c 100644 --- a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.png +++ b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2fc77a26e4c6d30ca9d21fe47ddc25624662d566f6c98ac1c160637fbcb2363b -size 1151151 +oid sha256:56d06f7d654939feda627f67196b813de9b38a718acba9f5daed59a43314829f +size 1150807 diff --git a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_48_1.png b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_48_1.png index 0da4a480d88cc3..d98f56141b1252 100644 --- a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_48_1.png +++ b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_48_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9cfd18faba3aef57349c70b157fabd4dc2827a8bc6074252e717a67a9f9d488a -size 1148752 +oid sha256:0d7f8506e5f1bd369debee273b45c601d05901af4937d8cc976f985cd4a81fed +size 1149292 diff --git a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.jpg b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.jpg index e992bdf66a9da7..b53344f52b7396 100644 --- a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.jpg +++ b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f70d76f01ee01d54713ed161de8d09713816b4f3fd7df99a5ff96c4c5e0b1bf0 -size 124539 +oid sha256:edd5a47baf47ae90532b47bc5ee05e8503b7d1deda59d956a354688ed949c8b5 +size 121605 diff --git a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.png b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.png index ec9e6a6277d4f0..2edc9a038ff8c3 100644 --- a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.png +++ b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f4dc2d892ef69df259eff4b46527d5e4382457b9b381a1e3b35b22d1d4312da4 -size 1150974 +oid sha256:aa184084b598dac717e99fe9677f1fe9dd4f6b85ec123c075d4109c75b134841 +size 1150675 diff --git a/docs/notebooks/language-quantize-bert-with-output.rst b/docs/notebooks/language-quantize-bert-with-output.rst index 64f49c2b314811..21ecfe511f1b76 100644 --- a/docs/notebooks/language-quantize-bert-with-output.rst +++ b/docs/notebooks/language-quantize-bert-with-output.rst @@ -101,10 +101,10 @@ Imports .. parsed-literal:: - 2024-10-23 01:38:12.900514: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:38:12.934654: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 01:51:49.197259: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:51:49.231710: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:38:13.484434: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:51:49.783615: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -211,7 +211,7 @@ PyTorch model formats are supported: .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4713: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4713: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( @@ -247,7 +247,7 @@ tokenizer from HuggingFace. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884 warnings.warn( @@ -505,9 +505,9 @@ Frames Per Second (FPS) for images. .. parsed-literal:: - PyTorch model on CPU: 0.068 seconds per sentence, SPS: 14.67 - IR FP32 model in OpenVINO Runtime/AUTO: 0.020 seconds per sentence, SPS: 49.46 - OpenVINO IR INT8 model in OpenVINO Runtime/AUTO: 0.009 seconds per sentence, SPS: 108.19 + PyTorch model on CPU: 0.068 seconds per sentence, SPS: 14.68 + IR FP32 model in OpenVINO Runtime/AUTO: 0.020 seconds per sentence, SPS: 49.24 + OpenVINO IR INT8 model in OpenVINO Runtime/AUTO: 0.009 seconds per sentence, SPS: 108.47 Finally, measure the inference performance of OpenVINO ``FP32`` and @@ -548,7 +548,7 @@ in OpenVINO. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 18.86 ms + [ INFO ] Read model took 19.11 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,?] @@ -559,7 +559,7 @@ in OpenVINO. [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: 'input_ids': [1,128], '63': [1,128], 'token_type_ids': [1,128] - [ INFO ] Reshape model took 5.46 ms + [ INFO ] Reshape model took 5.55 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: [ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,128] @@ -568,7 +568,7 @@ in OpenVINO. [ INFO ] Model outputs: [ INFO ] logits (node: __module.classifier/aten::linear/Add) : f32 / [...] / [1,2] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 373.39 ms + [ INFO ] Compile model took 344.20 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -609,17 +609,17 @@ in OpenVINO. [ INFO ] Fill input 'token_type_ids' with random values [Step 10/11] Measuring performance (Start inference synchronously, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 24.05 ms + [ INFO ] First inference took 22.90 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 5730 iterations - [ INFO ] Duration: 120000.60 ms + [ INFO ] Count: 6485 iterations + [ INFO ] Duration: 120011.48 ms [ INFO ] Latency: - [ INFO ] Median: 20.64 ms - [ INFO ] Average: 20.84 ms - [ INFO ] Min: 19.84 ms - [ INFO ] Max: 31.70 ms - [ INFO ] Throughput: 47.75 FPS + [ INFO ] Median: 18.09 ms + [ INFO ] Average: 18.41 ms + [ INFO ] Min: 17.32 ms + [ INFO ] Max: 26.49 ms + [ INFO ] Throughput: 54.04 FPS .. code:: ipython3 @@ -646,27 +646,27 @@ in OpenVINO. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 25.60 ms + [ INFO ] Read model took 24.93 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,?] - [ INFO ] 63 , attention_mask (node: attention_mask) : i64 / [...] / [1,?] + [ INFO ] attention_mask , 63 (node: attention_mask) : i64 / [...] / [1,?] [ INFO ] token_type_ids (node: token_type_ids) : i64 / [...] / [1,?] [ INFO ] Model outputs: [ INFO ] logits (node: __module.classifier/aten::linear/Add) : f32 / [...] / [1,2] [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: 'input_ids': [1,128], '63': [1,128], 'token_type_ids': [1,128] - [ INFO ] Reshape model took 7.46 ms + [ INFO ] Reshape model took 7.14 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: [ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,128] - [ INFO ] 63 , attention_mask (node: attention_mask) : i64 / [...] / [1,128] + [ INFO ] attention_mask , 63 (node: attention_mask) : i64 / [...] / [1,128] [ INFO ] token_type_ids (node: token_type_ids) : i64 / [...] / [1,128] [ INFO ] Model outputs: [ INFO ] logits (node: __module.classifier/aten::linear/Add) : f32 / [...] / [1,2] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 1067.96 ms + [ INFO ] Compile model took 1080.21 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -707,15 +707,15 @@ in OpenVINO. [ INFO ] Fill input 'token_type_ids' with random values [Step 10/11] Measuring performance (Start inference synchronously, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 17.45 ms + [ INFO ] First inference took 16.00 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 13316 iterations - [ INFO ] Duration: 120006.19 ms + [ INFO ] Count: 13181 iterations + [ INFO ] Duration: 120003.10 ms [ INFO ] Latency: - [ INFO ] Median: 8.92 ms - [ INFO ] Average: 8.92 ms - [ INFO ] Min: 7.66 ms - [ INFO ] Max: 14.16 ms - [ INFO ] Throughput: 110.96 FPS + [ INFO ] Median: 8.93 ms + [ INFO ] Average: 9.01 ms + [ INFO ] Min: 7.68 ms + [ INFO ] Max: 12.00 ms + [ INFO ] Throughput: 109.84 FPS diff --git a/docs/notebooks/latent-consistency-models-optimum-demo-with-output.rst b/docs/notebooks/latent-consistency-models-optimum-demo-with-output.rst index 3707d3f30d5f09..a0bce9d85c7196 100644 --- a/docs/notebooks/latent-consistency-models-optimum-demo-with-output.rst +++ b/docs/notebooks/latent-consistency-models-optimum-demo-with-output.rst @@ -9,7 +9,7 @@ and time required to generate an image for provided prompt. The notebook can be also used on other Intel hardware with minimal or no modifications. -|image0| +.. image:: https://github.com/openvinotoolkit/openvino_notebooks/assets/10940214/1858dae4-72fd-401e-b055-66d503d82446 Optimum Intel is an interface from Hugging Face between both diffusers and transformers libraries and various tools provided by Intel to @@ -48,8 +48,6 @@ need a Jupyter server to start. For details, please refer to `Installation Guide `__. -.. |image0| image:: https://github.com/openvinotoolkit/openvino_notebooks/assets/10940214/1858dae4-72fd-401e-b055-66d503d82446 - Prerequisites ~~~~~~~~~~~~~ @@ -67,9 +65,7 @@ Install required packages .. parsed-literal:: Note: you may need to restart the kernel to use updated packages. - Note: you may need to restart the kernel to use updated packages. - Note: you may need to restart the kernel to use updated packages. - + .. code:: ipython3 @@ -112,8 +108,12 @@ this .. parsed-literal:: - CPU: Intel(R) Core(TM) i9-10920X CPU @ 3.50GHz - + CPU: Intel(R) Core(TM) Ultra 7 155H + GNA.GNA_SW: GNA_SW + GNA.GNA_HW: GNA_HW + GPU: Intel(R) Arc(TM) Graphics (iGPU) + NPU: Intel(R) AI Boost + Using full precision model in CPU with ``LatentConsistencyModelPipeline`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -132,14 +132,6 @@ https://huggingface.co/docs/diffusers/en/api/pipelines/latent_consistency_models pipeline = LatentConsistencyModelPipeline.from_pretrained("SimianLuo/LCM_Dreamshaper_v7") -.. parsed-literal:: - - 2024-10-23 01:44:02.155955: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:44:02.191160: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:44:02.863862: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - - .. parsed-literal:: @@ -172,6 +164,15 @@ https://huggingface.co/docs/diffusers/en/api/pipelines/latent_consistency_models del pipeline gc.collect(); + + + +.. parsed-literal:: + + 345 + + + Select inference device for text-to-image generation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -190,15 +191,6 @@ Select inference device for text-to-image generation device - - - -.. parsed-literal:: - - Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') - - - Running inference using Optimum Intel ``OVLatentConsistencyModelPipeline`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -228,74 +220,6 @@ and there is no need to do it manually ov_pipeline.reshape(batch_size=1, height=512, width=512, num_images_per_prompt=1) - -.. parsed-literal:: - - Keyword arguments {'subfolder': '', 'trust_remote_code': False} are not expected by LatentConsistencyModelPipeline and will be ignored. - - - -.. parsed-literal:: - - Loading pipeline components...: 0%| | 0/7 [00:00 by passing `safety_checker=None`. Ensure that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered results in services or applications open to the public. Both the diffusers team and Hugging Face strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling it only for use-cases that involve analyzing network behavior or auditing its results. For more information, please have a look at https://github.com/huggingface/diffusers/pull/254 . - - - - -.. parsed-literal:: - - OVLatentConsistencyModelPipeline { - "_class_name": "OVLatentConsistencyModelPipeline", - "_diffusers_version": "0.31.0", - "_name_or_path": "/tmp/tmpz3tx8cvr", - "feature_extractor": [ - "transformers", - "CLIPImageProcessor" - ], - "image_encoder": [ - null, - null - ], - "requires_safety_checker": true, - "safety_checker": [ - null, - null - ], - "scheduler": [ - "diffusers", - "LCMScheduler" - ], - "text_encoder": [ - "optimum.intel.openvino.modeling_diffusion", - "OVModelTextEncoder" - ], - "tokenizer": [ - "transformers", - "CLIPTokenizer" - ], - "unet": [ - "optimum.intel.openvino.modeling_diffusion", - "OVModelUnet" - ], - "vae": [ - "optimum.intel.openvino.modeling_diffusion", - "OVModelVae" - ] - } - - - .. code:: ipython3 ov_pipeline.to(device.value) diff --git a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.jpg b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.jpg index a19eb369148e10..25ee04df1c536d 100644 --- a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.jpg +++ b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8ccbcf6c99b8d75b1c683375852111ce17983fbea50bd0b0bb294d06bbb317fb -size 38768 +oid sha256:7167bdd42f946c71dcff8f7c7f35e48df5434ed2c54b56505da9c61d25a0d740 +size 79839 diff --git a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.png b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.png index d27692124393df..d89126405ceff7 100644 --- a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.png +++ b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b936a832e23c30affb1988828f17a16e988b42413ff7dfe4381fb723e068ed97 -size 456597 +oid sha256:7f216ac91b5737006142f8426f0716c9ab8e3ce5ddea222987c69e8a4f11ed3e +size 1034680 diff --git a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.jpg b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.jpg index e320b91964889a..f0f1cd042ef9b5 100644 --- a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.jpg +++ b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0ab792ded8330c36072ec13fcfb0e0896ce3014413145167fd7a7c5b570919a4 -size 37604 +oid sha256:900339d0d0c9d4b8e25a5b72a159ca2c966f7e51dc112c30a7ac8636a01b1dd4 +size 81729 diff --git a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.png b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.png index e7a5d16aa88eb2..e083c8da654356 100644 --- a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.png +++ b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f440a9e7cb3d6a9f6609165516be7e5025e26752004ca56baf570f49840af03c -size 470802 +oid sha256:89e0bc6c3f710c16cb1f81011848f41ad350e43ef9dc77961353897aa1d5b6b6 +size 1042433 diff --git a/docs/notebooks/llava-multimodal-chatbot-genai-with-output.rst b/docs/notebooks/llava-multimodal-chatbot-genai-with-output.rst index 435e7eaf62c53b..d035645fb27291 100644 --- a/docs/notebooks/llava-multimodal-chatbot-genai-with-output.rst +++ b/docs/notebooks/llava-multimodal-chatbot-genai-with-output.rst @@ -117,10 +117,25 @@ Install required dependencies .. code:: ipython3 + from pathlib import Path + import requests + %pip install -q "torch>=2.1.0" "torchvision" "torchaudio" --index-url https://download.pytorch.org/whl/cpu %pip install -q "git+https://github.com/eaidova/optimum-intel.git@ea/minicpmv" %pip install -q "nncf>=2.13.0" "sentencepiece" "tokenizers>=0.12.1" "transformers>=4.45.0" "gradio>=4.36" - %pip install -q -U --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly openvino_tokenizers openvino openvino-genai + %pip install -q -U --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly openvino-tokenizers openvino openvino-genai + + + utility_files = ["notebook_utils.py", "cmd_helper.py"] + + for utility in utility_files: + local_path = Path(utility) + if not local_path.exists(): + r = requests.get( + url=f"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/{local_path.name}", + ) + with local_path.open("w") as f: + f.write(r.text) Convert and Optimize Model -------------------------- @@ -172,13 +187,13 @@ documentation =2.1.0" "torchvision" "torchaudio" --index-url https://download.pytorch.org/whl/cpu %pip install -q "git+https://github.com/eaidova/optimum-intel.git@ea/minicpmv" %pip install -q "nncf>=2.13.0" "sentencepiece" "tokenizers>=0.12.1" "transformers>=4.45.0" "gradio>=4.36" - %pip install -q -U --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly openvino_tokenizers openvino openvino-genai + %pip install -q -U --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly openvino-tokenizers openvino openvino-genai + + utility_files = ["notebook_utils.py", "cmd_helper.py"] + + for utility in utility_files: + local_path = Path(utility) + if not local_path.exists(): + r = requests.get( + url=f"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/{local_path.name}", + ) + with local_path.open("w") as f: + f.write(r.text) Convert and Optimize Model -------------------------- @@ -172,13 +186,13 @@ documentation `__, -`blog `__, -`GitHub `__, and -`Documentation `__. +``Qwen2.5`` as LLM in agent pipeline. + +* **qwen2.5-3b-instruct/qwen2.5-7b-instruct/qwen2.5-14b-instruct** - + Qwen2.5 is the latest series of Qwen large language models. Comparing + with Qwen2, Qwen2.5 series brings significant improvements in coding, + mathematics and general knowledge skills. Additionally, it brings + long-context and multiple languages support including Chinese, English, + French, Spanish, Portuguese, German, Italian, Russian, Japanese, Korean, + Vietnamese, Thai, Arabic, and more. For more details, please refer to + `model_card `__, + `blog `__, + `GitHub `__, and + `Documentation `__. .. code:: ipython3 import ipywidgets as widgets - + llm_model_ids = ["Qwen/Qwen2.5-3B-Instruct", "Qwen/Qwen2.5-7B-Instruct", "Qwen/qwen2.5-14b-instruct"] - + llm_model_id = widgets.Dropdown( options=llm_model_ids, value=llm_model_ids[0], description="Model:", disabled=False, ) - + llm_model_id @@ -146,9 +147,9 @@ Vietnamese, Thai, Arabic, and more. For more details, please refer to .. code:: ipython3 from pathlib import Path - + llm_model_path = llm_model_id.value.split("/")[-1] - + if not Path(llm_model_path).exists(): !optimum-cli export openvino --model {llm_model_id.value} --task text-generation-with-past --trust-remote-code --weight-format int4 --group-size 128 --ratio 1.0 --sym {llm_model_path} @@ -160,9 +161,9 @@ Select inference device for LLM .. code:: ipython3 from notebook_utils import device_widget - + llm_device = device_widget("CPU", exclude=["NPU"]) - + llm_device @@ -226,15 +227,15 @@ guide `__ import openvino.properties as props import openvino.properties.hint as hints import openvino.properties.streams as streams - + import json import json5 import torch - + tokenizer = AutoTokenizer.from_pretrained(llm_model_path, trust_remote_code=True) - + ov_config = {hints.performance_mode(): hints.PerformanceMode.LATENCY, streams.num(): "1", props.cache_dir(): ""} - + llm = OVModelForCausalLM.from_pretrained( llm_model_path, device=llm_device.value, @@ -242,7 +243,7 @@ guide `__ config=AutoConfig.from_pretrained(llm_model_path, trust_remote_code=True), trust_remote_code=True, ) - + llm.generation_config.top_k = 1 llm.generation_config.max_length = 2000 @@ -260,31 +261,31 @@ received from tool calling.. class StopSequenceCriteria(StoppingCriteria): """ This class can be used to stop generation whenever a sequence of tokens is encountered. - + Args: stop_sequences (`str` or `List[str]`): The sequence (or list of sequences) on which to stop execution. tokenizer: The tokenizer used to decode the model outputs. """ - + def __init__(self, stop_sequences, tokenizer): if isinstance(stop_sequences, str): stop_sequences = [stop_sequences] self.stop_sequences = stop_sequences self.tokenizer = tokenizer - + def __call__(self, input_ids, scores, **kwargs) -> bool: decoded_output = self.tokenizer.decode(input_ids.tolist()[0]) return any(decoded_output.endswith(stop_sequence) for stop_sequence in self.stop_sequences) - - + + def text_completion(prompt: str, stop_words) -> str: im_end = "<|im_end|>" if im_end not in stop_words: stop_words = stop_words + [im_end] streamer = TextStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True) - + stopping_criteria = StoppingCriteriaList([StopSequenceCriteria(stop_words, tokenizer)]) input_ids = torch.tensor([tokenizer.encode(prompt)]) generate_kwargs = dict( @@ -297,7 +298,7 @@ received from tool calling.. output = tokenizer.decode(output, errors="ignore") assert output.startswith(prompt) output = output[len(prompt) :].replace("<|endoftext|>", "").replace(im_end, "") - + for stop_str in stop_words: idx = output.find(stop_str) if idx != -1: @@ -339,13 +340,13 @@ parameter should be a sequence of messages that contains the .. code:: ipython3 TOOL_DESC = """{name_for_model}: Call this tool to interact with the {name_for_human} API. What is the {name_for_human} API useful for? {description_for_model} Parameters: {parameters}""" - + PROMPT_REACT = """Answer the following questions as best you can. You have access to the following APIs: - + {tools_text} - + Use the following format: - + Question: the input question you must answer Thought: you should always think about what to do Action: the action to take, should be one of [{tools_name_text}] @@ -354,9 +355,9 @@ parameter should be a sequence of messages that contains the ... (this Thought/Action/Action Input/Observation can be repeated zero or more times) Thought: I now know the final answer Final Answer: the final answer to the original input question - + Begin! - + Question: {query}""" Meanwhile we have to create function for consolidate the tools @@ -381,9 +382,9 @@ information and conversation history into the prompt template. raise NotImplementedError tools_text.append(tool) tools_text = "\n\n".join(tools_text) - + tools_name_text = ", ".join([tool_info["name_for_model"] for tool_info in list_of_tool_info]) - + messages = [{"role": "system", "content": "You are a helpful assistant."}] for i, (query, response) in enumerate(chat_history): if list_of_tool_info: @@ -397,9 +398,9 @@ information and conversation history into the prompt template. messages.append({"role": "user", "content": query}) if response: messages.append({"role": "assistant", "content": response}) - + prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False, return_tensors="pt") - + return prompt Create parser @@ -493,7 +494,7 @@ execute them according to the output of LLM. return str(ret) elif tool_name == "image_gen": import urllib.parse - + tool_args = tool_args.replace("(", "").replace(")", "") prompt = json5.loads(tool_args)["prompt"] prompt = urllib.parse.quote(prompt) @@ -503,11 +504,11 @@ execute them according to the output of LLM. ) else: raise NotImplementedError - - + + def llm_with_tool(prompt: str, history, list_of_tool_info=()): chat_history = [(x["user"], x["bot"]) for x in history] + [(prompt, "")] - + planning_prompt = build_input_text(chat_history, list_of_tool_info) text = "" while True: @@ -522,7 +523,7 @@ execute them according to the output of LLM. else: text += output break - + new_history = [] new_history.extend(history) new_history.append({"user": prompt, "bot": text}) @@ -537,7 +538,7 @@ Run agent history = [] query = "get the weather in London, and create a picture of Big Ben based on the weather information" - + response, history = llm_with_tool(prompt=query, history=history, list_of_tool_info=tools) diff --git a/docs/notebooks/meter-reader-with-output.rst b/docs/notebooks/meter-reader-with-output.rst index 7f539abf025ebb..c1317625880917 100644 --- a/docs/notebooks/meter-reader-with-output.rst +++ b/docs/notebooks/meter-reader-with-output.rst @@ -637,7 +637,7 @@ bounds of input batch size. .. parsed-literal:: - + diff --git a/docs/notebooks/minicpm-v-multimodal-chatbot-with-output.rst b/docs/notebooks/minicpm-v-multimodal-chatbot-with-output.rst index f6a22b6f160760..98f1217902a587 100644 --- a/docs/notebooks/minicpm-v-multimodal-chatbot-with-output.rst +++ b/docs/notebooks/minicpm-v-multimodal-chatbot-with-output.rst @@ -61,9 +61,15 @@ Prerequisites .. parsed-literal:: - Note: you may need to restart the kernel to use updated packages. - Note: you may need to restart the kernel to use updated packages. - + WARNING: Error parsing dependencies of torchsde: .* suffix can only be used with `==` or `!=` operators + numpy (>=1.19.*) ; python_version >= "3.7" + ~~~~~~~^ + Note: you may need to restart the kernel to use updated packages. + WARNING: Error parsing dependencies of torchsde: .* suffix can only be used with `==` or `!=` operators + numpy (>=1.19.*) ; python_version >= "3.7" + ~~~~~~~^ + Note: you may need to restart the kernel to use updated packages. + .. code:: ipython3 @@ -107,7 +113,16 @@ details.
+.. raw:: html + + + Click here for more detailed explanation of conversion steps + +.. raw:: html + + + MiniCPM-V2.6 is autoregressive transformer generative model, it means that each next model step depends from model output from previous step. The generation approach is based on the assumption that the probability @@ -205,11 +220,12 @@ Let’s convert each model part. .. parsed-literal:: - 2024-10-23 01:47:25.606377: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:47:25.640217: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-10-07 09:57:53.402018: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-10-07 09:57:53.403877: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used. + 2024-10-07 09:57:53.440490: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:47:26.161344: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - + 2024-10-07 09:57:54.270302: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + .. code:: ipython3 @@ -222,157 +238,7 @@ Let’s convert each model part. ⌛ openbmb/MiniCPM-V-2_6 conversion started. Be patient, it may takes some time. ⌛ Load Original model - - - -.. parsed-literal:: - - Fetching 24 files: 0%| | 0/24 [00:00 self.max_seq_len_cached: - + .. parsed-literal:: - ✅ Language model successfully converted - ⌛ Convert Image embedding model ✅ Image embedding model successfully converted - ⌛ Convert Resamler model - - -.. parsed-literal:: - - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/ckpt/resampler.py:421: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - assert embed_dim == embed_dim_to_check, \ - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/ckpt/resampler.py:428: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - assert head_dim * num_heads == embed_dim, f"embed_dim {embed_dim} not divisible by num_heads {num_heads}" - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/ckpt/resampler.py:434: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - assert key.shape == value.shape, f"key shape {key.shape} does not match value shape {value.shape}" - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/ckpt/resampler.py:520: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - assert key_padding_mask.shape == (bsz, src_len), \ - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/ckpt/resampler.py:539: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - q_scaled = q / math.sqrt(E) - - -.. parsed-literal:: - - ✅ Resampler model successfully converted ✅ openbmb/MiniCPM-V-2_6 model sucessfully converted. You can find results in MiniCPM-V-2_6 - + Compress Language Model Weights to 4 bits ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -459,12 +288,21 @@ applied using `NNCF `__.
-Click here for more details about weight compression Weight compression -aims to reduce the memory footprint of a model. It can also lead to -significant performance improvement for large memory-bound models, such -as Large Language Models (LLMs). LLMs and other models, which require -extensive memory to store the weights during inference, can benefit from -weight compression in the following ways: +.. raw:: html + + + +Click here for more details about weight compression + +.. raw:: html + + + +Weight compression aims to reduce the memory footprint of a model. It +can also lead to significant performance improvement for large +memory-bound models, such as Large Language Models (LLMs). LLMs and +other models, which require extensive memory to store the weights during +inference, can benefit from weight compression in the following ways: - enabling the inference of exceptionally large models that cannot be accommodated in the memory of the device; @@ -547,33 +385,14 @@ documentation `__ - `Select model <#select-model>`__ @@ -62,9 +63,9 @@ Prerequisites .. code:: ipython3 from pathlib import Path - + repo_dir = Path("./ml-mobileclip") - + if not repo_dir.exists(): !git clone https://github.com/apple/ml-mobileclip.git @@ -76,15 +77,15 @@ Prerequisites remote: Counting objects: 100% (95/95), done. remote: Compressing objects: 100% (66/66), done. remote: Total 95 (delta 38), reused 85 (delta 28), pack-reused 0 (from 0) - Unpacking objects: 100% (95/95), 469.11 KiB | 3.91 MiB/s, done. + Unpacking objects: 100% (95/95), 469.11 KiB | 3.13 MiB/s, done. .. code:: ipython3 %pip install -q "./ml-mobileclip" --no-deps - + %pip install -q "clip-benchmark>=1.4.0" "datasets>=2.8.0" "open-clip-torch>=2.20.0" "timm>=0.9.5" "torch>=1.13.1" "torchvision>=0.14.1" --extra-index-url https://download.pytorch.org/whl/cpu - + %pip install -q "openvino>=2024.0.0" "gradio>=4.19" "matplotlib" "Pillow" "altair" "pandas" "opencv-python" "tqdm" "matplotlib>=3.4" @@ -141,9 +142,9 @@ comparison purposes, you can select different models among: .. code:: ipython3 import ipywidgets as widgets - + model_dir = Path("checkpoints") - + supported_models = { "MobileCLIP": { "mobileclip_s0": { @@ -184,17 +185,17 @@ comparison purposes, you can select different models among: "image_size": 224, }, "clip-vit-b-16": { - "image_name": "ViT-B-16", + "model_name": "ViT-B-16", "pretrained": "openai", "image_size": 224, }, "clip-vit-l-14": { - "image_name": "ViT-L-14", + "model_name": "ViT-L-14", "pretrained": "datacomp_xl_s13b_b90k", "image_size": 224, }, "clip-vit-h-14": { - "image_name": "ViT-H-14", + "model_name": "ViT-H-14", "pretrained": "laion2b_s32b_b79k", "image_size": 224, }, @@ -212,8 +213,8 @@ comparison purposes, you can select different models among: }, }, } - - + + model_type = widgets.Dropdown(options=supported_models.keys(), default="MobileCLIP", description="Model type:") model_type @@ -229,13 +230,13 @@ comparison purposes, you can select different models among: .. code:: ipython3 available_models = supported_models[model_type.value] - + model_checkpoint = widgets.Dropdown( options=available_models.keys(), default=list(available_models), description="Model:", ) - + model_checkpoint @@ -250,15 +251,15 @@ comparison purposes, you can select different models among: .. code:: ipython3 import requests - + r = requests.get( url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", ) - + open("notebook_utils.py", "w").write(r.text) - + from notebook_utils import download_file, device_widget - + model_config = available_models[model_checkpoint.value] Run model inference @@ -293,8 +294,8 @@ Prepare image gallery import matplotlib.pyplot as plt import numpy as np from PIL import Image - - + + def visualize_result(images: List, query: str = "", selected: List[int] = None): """ Utility function for visualization classification results @@ -322,8 +323,8 @@ Prepare image gallery mask = np.ones_like(np.array(images[idx])) a.imshow(mask, "jet", interpolation="none", alpha=0.75) return fig - - + + images_urls = [ "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/282ce53e-912d-41aa-ab48-2a001c022d74", "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/9bb40168-82b5-4b11-ada6-d8df104c736c", @@ -333,17 +334,17 @@ Prepare image gallery image_names = ["red_panda.png", "cat.png", "raccoon.png", "dog.png"] sample_path = Path("data") sample_path.mkdir(parents=True, exist_ok=True) - + images = [] for image_name, image_url in zip(image_names, images_urls): image_path = sample_path / image_name if not image_path.exists(): download_file(image_url, filename=image_name, directory=sample_path) images.append(Image.open(image_path).convert("RGB").resize((640, 420))) - + input_labels = ["cat"] text_descriptions = [f"This is a photo of a {label}" for label in input_labels] - + visualize_result(images, "image gallery"); @@ -390,7 +391,7 @@ preprocessing utilities from PIL import Image import mobileclip import open_clip - + # instantiate model model_name = model_config["model_name"] pretrained = model_config["pretrained"] @@ -407,7 +408,7 @@ preprocessing utilities .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) @@ -426,8 +427,8 @@ Perform search image_tensor = torch.stack([preprocess(image) for image in images]) text = tokenizer(text_descriptions) - - + + with torch.no_grad(): # calculate image embeddings image_encoding_start = time.perf_counter() @@ -439,22 +440,22 @@ Perform search text_features = model.encode_text(text) text_encoding_end = time.perf_counter() print(f"Text encoding took {text_encoding_end - text_encoding_start:.3} ms") - + # normalize embeddings image_features /= image_features.norm(dim=-1, keepdim=True) text_features /= text_features.norm(dim=-1, keepdim=True) - + # calcualte similarity score image_probs = (100.0 * text_features @ image_features.T).softmax(dim=-1) selected_image = [torch.argmax(image_probs).item()] - + visualize_result(images, input_labels[0], selected_image); .. parsed-literal:: - Image encoding took 0.136 ms - Text encoding took 0.0123 ms + Image encoding took 0.114 ms + Text encoding took 0.0113 ms @@ -481,8 +482,8 @@ be used separately. Let’s convert each part to OpenVINO. import types import torch.nn.functional as F - - + + def se_block_forward(self, inputs): """Apply forward pass.""" b, c, h, w = inputs.size() @@ -498,12 +499,12 @@ be used separately. Let’s convert each part to OpenVINO. import openvino as ov import gc - + ov_models_dir = Path("ov_models") ov_models_dir.mkdir(exist_ok=True) - + image_encoder_path = ov_models_dir / f"{model_checkpoint.value}_im_encoder.xml" - + if not image_encoder_path.exists(): if "mobileclip_s" in model_name: model.image_encoder.model.conv_exp.se.forward = types.MethodType(se_block_forward, model.image_encoder.model.conv_exp.se) @@ -516,23 +517,23 @@ be used separately. Let’s convert each part to OpenVINO. ov.save_model(ov_image_encoder, image_encoder_path) del ov_image_encoder gc.collect() - + text_encoder_path = ov_models_dir / f"{model_checkpoint.value}_text_encoder.xml" - + if not text_encoder_path.exists(): model.forward = model.encode_text ov_text_encoder = ov.convert_model(model, example_input=text, input=[-1, text.shape[1]]) ov.save_model(ov_text_encoder, text_encoder_path) del ov_text_encoder gc.collect() - + del model gc.collect(); .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/mobileclip/modules/common/transformer.py:125: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/mobileclip/modules/common/transformer.py:125: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if seq_len != self.num_embeddings: @@ -549,9 +550,9 @@ Select device for image encoder .. code:: ipython3 core = ov.Core() - + device = device_widget() - + device @@ -608,17 +609,17 @@ Perform search print(f"Text encoding took {text_encoding_end - text_encoding_start:.3} ms") image_features /= image_features.norm(dim=-1, keepdim=True) text_features /= text_features.norm(dim=-1, keepdim=True) - + image_probs = (100.0 * text_features @ image_features.T).softmax(dim=-1) selected_image = [torch.argmax(image_probs).item()] - + visualize_result(images, input_labels[0], selected_image); .. parsed-literal:: - Image encoding took 0.0297 ms - Text encoding took 0.00513 ms + Image encoding took 0.0294 ms + Text encoding took 0.00498 ms @@ -654,14 +655,14 @@ models can require different optimal threshold for search. ) from open_clip.transform import image_transform from typing import Optional - - + + current_device = device.value current_model = image_encoder_path.name.split("_im_encoder")[0] - + available_converted_models = [model_file.name.split("_im_encoder")[0] for model_file in ov_models_dir.glob("*_im_encoder.xml")] available_devices = list(core.available_devices) + ["AUTO"] - + download_file( "https://storage.openvinotoolkit.org/data/test_data/videos/car-detection.mp4", directory=sample_path, @@ -671,8 +672,8 @@ models can require different optimal threshold for search. directory=sample_path, filename="coco.mp4", ) - - + + def get_preprocess_and_tokenizer(model_name): if "mobileclip" in model_name: resolution = supported_models["MobileCLIP"][model_name]["image_size"] @@ -693,10 +694,10 @@ models can require different optimal threshold for search. resize_size = model_configs[model_name]["image_size"] preprocess = image_transform((resize_size, resize_size), is_train=False, resize_mode="longest") tokenizer = open_clip.get_tokenizer(model_configs[model_name]["model_name"]) - + return preprocess, tokenizer - - + + def run( path: str, text_search: str, @@ -715,7 +716,7 @@ models can require different optimal threshold for search. global tokenizer global ov_compiled_image_encoder global ov_compiled_text_encoder - + if current_model != model_name or device != current_device: ov_compiled_image_encoder = core.compile_model(ov_models_dir / f"{model_name}_im_encoder.xml", device) ov_compiled_text_encoder = core.compile_model(ov_models_dir / f"{model_name}_text_encoder.xml", device) @@ -725,7 +726,7 @@ models can require different optimal threshold for search. # Load video dataset = LoadVideo(path, transforms=preprocess, vid_stride=stride) dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=0) - + # Get image query features if image_search: image = preprocess(image_search).unsqueeze(0) @@ -745,11 +746,11 @@ models can require different optimal threshold for search. for image, orig, frame, timestamp in dataloader: with torch.no_grad(): image_features = torch.from_numpy(ov_compiled_image_encoder(image)[0]) - + image_features /= image_features.norm(dim=-1, keepdim=True) probs = query_features.cpu().numpy() @ image_features.cpu().numpy().T probs = probs[0] - + # Save frame similarity values df = pd.DataFrame( { @@ -759,15 +760,15 @@ models can require different optimal threshold for search. } ) res = pd.concat([res, df]) - + # Check if frame is over threshold for i, p in enumerate(probs): if p > thresh: matches.append(to_pil_image(orig[i])) matches_probs.append(p) - + print(f"Frames: {frame.tolist()} - Probs: {probs}") - + # Create plot of similarity values lines = ( alt.Chart(res) @@ -778,16 +779,16 @@ models can require different optimal threshold for search. ) ).properties(width=600) rule = alt.Chart().mark_rule(strokeDash=[6, 3], size=2).encode(y=alt.datum(thresh)) - + selected_frames = np.argsort(-1 * np.array(matches_probs))[:20] matched_sorted_frames = [matches[idx] for idx in selected_frames] - + return ( lines + rule, matched_sorted_frames, ) # Only return up to 20 images to not crash the UI - - + + class LoadVideo(Dataset): def __init__(self, path, transforms, vid_stride=1): self.transforms = transforms @@ -795,27 +796,27 @@ models can require different optimal threshold for search. self.cur_frame = 0 self.cap = cv2.VideoCapture(path) self.total_frames = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT) / self.vid_stride) - + def __getitem__(self, _): # Read video # Skip over frames for _ in range(self.vid_stride): self.cap.grab() self.cur_frame += 1 - + # Read frame _, img = self.cap.retrieve() timestamp = self.cap.get(cv2.CAP_PROP_POS_MSEC) - + # Convert to PIL img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) img = Image.fromarray(np.uint8(img)) - + # Apply transforms img_t = self.transforms(img) - + return img_t, to_tensor(img), self.cur_frame, timestamp - + def __len__(self): return self.total_frames @@ -837,15 +838,15 @@ models can require different optimal threshold for search. if not Path("gradio_helper.py").exists(): r = requests.get(url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/mobileclip-video-search/gradio_helper.py") open("gradio_helper.py", "w").write(r.text) - + from gradio_helper import make_demo, Option - + demo = make_demo( run=run, model_option=Option(choices=available_converted_models, value=model_checkpoint.value), device_option=Option(choices=available_devices, value=device.value), ) - + try: demo.launch(debug=False) except Exception: @@ -858,7 +859,7 @@ models can require different optimal threshold for search. .. parsed-literal:: Running on local URL: http://127.0.0.1:7860 - + To create a public link, set `share=True` in `launch()`. diff --git a/docs/notebooks/mobilevlm-language-assistant-with-output.rst b/docs/notebooks/mobilevlm-language-assistant-with-output.rst index 02efe16d9c0f4a..5902537e3026a5 100644 --- a/docs/notebooks/mobilevlm-language-assistant-with-output.rst +++ b/docs/notebooks/mobilevlm-language-assistant-with-output.rst @@ -119,13 +119,13 @@ Import required packages .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( - 2024-10-23 01:57:03.532418: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:57:03.567584: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 02:02:06.143728: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 02:02:06.177889: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:57:04.078609: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + 2024-11-05 02:02:06.679118: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( @@ -342,15 +342,15 @@ compression instead of INT8 weight compression. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:595: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:595: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input_shape[-1] > 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:119: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:119: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if seq_len > self.max_seq_len_cached: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:348: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:348: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:355: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:355: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:365: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:365: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): @@ -416,7 +416,7 @@ compression instead of INT8 weight compression. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:165: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at aten/src/ATen/core/TensorBody.h:489.) + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:165: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at aten/src/ATen/core/TensorBody.h:489.) if a.grad is not None: diff --git a/docs/notebooks/music-generation-with-output.rst b/docs/notebooks/music-generation-with-output.rst index 566aa1c87a941c..4adc89b9ff79e7 100644 --- a/docs/notebooks/music-generation-with-output.rst +++ b/docs/notebooks/music-generation-with-output.rst @@ -124,13 +124,13 @@ Imports .. parsed-literal:: - 2024-10-23 01:59:20.725760: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:59:20.759494: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 02:04:23.419260: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 02:04:23.453089: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:59:21.367845: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + 2024-11-05 02:04:24.059462: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( @@ -170,11 +170,11 @@ generate a text-conditioned music sample. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/huggingface_hub/file_download.py:797: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/huggingface_hub/file_download.py:797: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`. warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. warnings.warn("torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.") @@ -229,7 +229,7 @@ vocabulary. It helps the model understand the context of a sentence. @@ -655,7 +655,7 @@ We can now infer the pipeline backed by OpenVINO models. diff --git a/docs/notebooks/nano-llava-multimodal-chatbot-with-output.rst b/docs/notebooks/nano-llava-multimodal-chatbot-with-output.rst index 082b6613456e28..337458e35bbf0c 100644 --- a/docs/notebooks/nano-llava-multimodal-chatbot-with-output.rst +++ b/docs/notebooks/nano-llava-multimodal-chatbot-with-output.rst @@ -53,7 +53,7 @@ Prerequisites %pip install -q "torch>=2.1" "transformers>=4.40" "accelerate" "pillow" "gradio>=4.26" "tqdm" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "nncf>=2.13" - %pip install -q -U --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly openvino_tokenizers[transformers] "openvino>=2024.4.0" + %pip install -q -U --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly "openvino-tokenizers[transformers]" "openvino>=2024.4.0" %pip install -q "git+https://github.com/eaidova/optimum-intel.git@ea/minicpmv" @@ -64,9 +64,10 @@ Prerequisites Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - openvino-dev 2024.4.0 requires openvino==2024.4.0, but you have openvino 2024.5.0.dev20241014 which is incompatible. openvino-genai 2024.4.0.0 requires openvino_tokenizers~=2024.4.0.0.dev, but you have openvino-tokenizers 2024.5.0.0.dev20241022 which is incompatible. Note: you may need to restart the kernel to use updated packages. + ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. + mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.17.2+cpu which is incompatible. Note: you may need to restart the kernel to use updated packages. @@ -145,13 +146,13 @@ Download PyTorch model .. parsed-literal:: - example_1.png: 0%| | 0.00/200k [00:00 1 or self.sliding_window is not None) and self.is_causal: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/exporters/onnx/model_patcher.py:307: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/exporters/onnx/model_patcher.py:306: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/qnguyen3/nanoLLaVA/1ea99cffcf50a27c5f06fe5d22a07046aba0bffe/modeling_llava_qwen2.py:939: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/qnguyen3/nanoLLaVA/13d60cec183a86755afed64da495fcc2c382ea80/modeling_llava_qwen2.py:939: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if seq_len > self.max_seq_len_cached: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/cache_utils.py:432: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/cache_utils.py:432: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results. elif len(self.key_cache[layer_idx]) == 0: # fills previously skipped layers; checking for tensor causes errors - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/qnguyen3/nanoLLaVA/1ea99cffcf50a27c5f06fe5d22a07046aba0bffe/modeling_llava_qwen2.py:1499: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/qnguyen3/nanoLLaVA/13d60cec183a86755afed64da495fcc2c382ea80/modeling_llava_qwen2.py:1499: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - Unexpectedly found already patched module while applying ModuleExtension during PyTorch model conversion. Result of the conversion maybe broken. Depending on the exact issue it may lead to broken original model. OpenVINO and OpenVINO Tokenizers versions are not binary compatible. OpenVINO version: 2024.5.0-16993 OpenVINO Tokenizers version: 2024.5.0.0 @@ -579,18 +250,18 @@ documentation + File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/bin/optimum-cli", line 10, in sys.exit(main()) - File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/commands/optimum_cli.py", line 208, in main + File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/commands/optimum_cli.py", line 208, in main service.run() - File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/commands/export/openvino.py", line 349, in run + File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/commands/export/openvino.py", line 349, in run main_export( - File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/exporters/openvino/__main__.py", line 416, in main_export + File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/exporters/openvino/__main__.py", line 416, in main_export core = Core() - File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino_tokenizers/__init__.py", line 53, in new_core_init + File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino_tokenizers/__init__.py", line 53, in new_core_init self.add_extension(str(_ext_path)) # Core.add_extension doesn't support Path object RuntimeError: Exception from src/inference/src/cpp/core.cpp:158: - Cannot add extension. Cannot find entry point to the extension library. This error happened: Cannot load library '/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino_tokenizers/lib/libopenvino_tokenizers.so': /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino_tokenizers/lib/libopenvino_tokenizers.so: undefined symbol: _ZNK2ov4Node17can_constant_foldERKSt6vectorINS_6OutputIS0_EESaIS3_EE + Cannot add extension. Cannot find entry point to the extension library. This error happened: Cannot load library '/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino_tokenizers/lib/libopenvino_tokenizers.so': /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino_tokenizers/lib/libopenvino_tokenizers.so: undefined symbol: _ZNK2ov4Node17can_constant_foldERKSt6vectorINS_6OutputIS0_EESaIS3_EE @@ -709,12 +380,12 @@ image encoder model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/quantization/quantize_model.py:432: FutureWarning: `CompressWeightsMode.INT8` is deprecated. Please, use `CompressWeightsMode.INT8_ASYM` as value instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/quantization/quantize_model.py:432: FutureWarning: `CompressWeightsMode.INT8` is deprecated. Please, use `CompressWeightsMode.INT8_ASYM` as value instead. warning_deprecated( - 2024-10-23 02:04:33.280788: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 02:04:33.314985: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 02:09:38.791476: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 02:09:38.825207: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 02:04:33.946816: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 02:09:39.427301: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -915,8 +586,8 @@ can use the same tokenizer and image processor that provided with model. Question: Describe this image in detail Answer: - The image portrays a charming and playful scene featuring a white lama. This adorable creature has a playful expression, with its eyes sparkling with joy and its nose in a playful smile. It is adorned with cute, tiny eyes that add a playful touch to its face. The lama's ears are also quite noticeable, with one of them sporting a tiny pink button. The lama's body is covered in fluffy, white fur, and it has its hind legs visible, adding a sense of movement to the image. - The lama is surrounded by a vivid display of fire. The flames are bright and lively, with some areas appearing more intense + The image features a small, adorable white lamb standing amidst a fire. The lamb's fur is fluffy and white, and it is adorned with tiny black eyes that are bright and lively. The lamb's face is cute, with a small black nose and a small mouth. It seems like the lamb is looking straight at the camera, making it appear even more adorable. + The lamb's right ear is visible, and it is white and pink. The lamb's right eye is also black and pink. The lamb's face is quite detailed, with the nose and mouth visible. There are also details like the lamb's right foot, which is white Interactive demo diff --git a/docs/notebooks/object-detection-with-output.rst b/docs/notebooks/object-detection-with-output.rst index b1db0093158982..a34f72f5d8ff1e 100644 --- a/docs/notebooks/object-detection-with-output.rst +++ b/docs/notebooks/object-detection-with-output.rst @@ -2,12 +2,10 @@ Live Object Detection with OpenVINO™ ==================================== This notebook demonstrates live object detection with OpenVINO, using -the `SSDLite -MobileNetV2 `__ -from `Open Model -Zoo `__. Final part -of this notebook shows live inference results from a webcam. -Additionally, you can also upload a video file. +the `Ultralytics +YOLOv8 `__. Final part of +this notebook shows live inference results from a webcam. Additionally, +you can also upload a video file. **NOTE**: To use this notebook with a webcam, you need to run the notebook on a computer with a webcam. If you run the notebook on a @@ -25,13 +23,12 @@ Additionally, you can also upload a video file. - `The Model <#the-model>`__ - - `Download the Model <#download-the-model>`__ - - `Convert the Model <#convert-the-model>`__ + - `Download and convert the + Model <#download-and-convert-the-model>`__ - `Load the Model <#load-the-model>`__ - `Processing <#processing>`__ - - `Process Results <#process-results>`__ - `Main Processing Function <#main-processing-function>`__ - `Run <#run>`__ @@ -62,8 +59,8 @@ Install requirements .. code:: ipython3 - %pip install -q "openvino-dev>=2024.0.0" - %pip install -q tensorflow + %pip install -q "openvino>=2024.4.0" + %pip install -q "ultralytics==8.3.0" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q opencv-python requests tqdm # Fetch `notebook_utils` module @@ -79,10 +76,6 @@ Install requirements .. parsed-literal:: Note: you may need to restart the kernel to use updated packages. - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - magika 0.5.1 requires numpy<2.0,>=1.24; python_version >= "3.8" and python_version < "3.9", but you have numpy 1.23.5 which is incompatible. - mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.17.2+cpu which is incompatible. - supervision 0.24.0 requires numpy<1.23.3,>=1.21.2; python_full_version <= "3.10.0", but you have numpy 1.23.5 which is incompatible. Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -102,17 +95,10 @@ Imports .. code:: ipython3 - import collections - import tarfile - import time from pathlib import Path - - import cv2 - import numpy as np - from IPython import display + import gc import openvino as ov - from openvino.tools.mo.front import tf as ov_tf_front - from openvino.tools import mo + from ultralytics import YOLO import notebook_utils as utils @@ -121,92 +107,53 @@ The Model -Download the Model -~~~~~~~~~~~~~~~~~~ - +Download and convert the Model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Use the ``download_file``, a function from the ``notebook_utils`` file. -It automatically creates a directory structure and downloads the -selected model. This step is skipped if the package is already -downloaded and unpacked. The chosen model comes from the public -directory, which means it must be converted into OpenVINO Intermediate -Representation (OpenVINO IR). - - **NOTE**: Using a model other than ``ssdlite_mobilenet_v2`` may - require different conversion parameters as well as pre- and - post-processing. .. code:: ipython3 # A directory where the model will be downloaded. - base_model_dir = Path("model") - - # The name of the model from Open Model Zoo - model_name = "ssdlite_mobilenet_v2" - archive_name = Path(f"{model_name}_coco_2018_05_09.tar.gz") - model_url = f"https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2022.1/{model_name}/{archive_name}" + # The name of the model + model_name = "yolov8n" - # Download the archive - downloaded_model_path = base_model_dir / archive_name - if not downloaded_model_path.exists(): - utils.download_file(model_url, downloaded_model_path.name, downloaded_model_path.parent) + det_model_path = Path(f"{model_name}_openvino_model/{model_name}.xml") - # Unpack the model - tf_model_path = base_model_dir / archive_name.with_suffix("").stem / "frozen_inference_graph.pb" - if not tf_model_path.exists(): - with tarfile.open(downloaded_model_path) as file: - file.extractall(base_model_dir) - + # export model to OpenVINO format using Ultralytics API + if not det_model_path.exists(): + pt_model = YOLO(f"{model_name}.pt") + pt_model.export(format="openvino", dynamic=True, half=True) + del pt_model + gc.collect() .. parsed-literal:: - model/ssdlite_mobilenet_v2_coco_2018_05_09.tar.gz: 0%| | 0.00/48.7M [00:00`__ -(``mo.convert_model`` function). If the model has been already -converted, this step is skipped. - -.. code:: ipython3 - - precision = "FP16" - # The output path for the conversion. - converted_model_path = Path("model") / f"{model_name}_{precision.lower()}.xml" - - # Convert it to IR if not previously converted - trans_config_path = Path(ov_tf_front.__file__).parent / "ssd_v2_support.json" - if not converted_model_path.exists(): - ov_model = mo.convert_model( - tf_model_path, - compress_to_fp16=(precision == "FP16"), - transformations_config=trans_config_path, - tensorflow_object_detection_api_pipeline_config=tf_model_path.parent / "pipeline.config", - reverse_input_channels=True, - ) - ov.save_model(ov_model, converted_model_path) - del ov_model + Downloading https://github.com/ultralytics/assets/releases/download/v8.3.0/yolov8n.pt to 'yolov8n.pt'... .. parsed-literal:: - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. - In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. - Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html + 100%|██████████| 6.25M/6.25M [00:00<00:00, 25.9MB/s] .. parsed-literal:: - [ WARNING ] The Preprocessor block has been removed. Only nodes performing mean value subtraction and scaling (if applicable) are kept. + Ultralytics 8.3.0 🚀 Python-3.8.10 torch-2.2.2+cpu CPU (Intel Core(TM) i9-10920X 3.50GHz) + YOLOv8n summary (fused): 168 layers, 3,151,904 parameters, 0 gradients, 8.7 GFLOPs + + PyTorch: starting from 'yolov8n.pt' with input shape (1, 3, 640, 640) BCHW and output shape(s) (1, 84, 8400) (6.2 MB) + + OpenVINO: starting export with openvino 2024.4.0-16579-c3152d32c9c-releases/2024/4... + OpenVINO: export success ✅ 1.3s, saved as 'yolov8n_openvino_model/' (6.4 MB) + + Export complete (1.5s) + Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/object-detection-webcam + Predict: yolo predict task=detect model=yolov8n_openvino_model imgsz=640 half + Validate: yolo val task=detect model=yolov8n_openvino_model imgsz=640 data=coco.yaml half + Visualize: https://netron.app Load the Model @@ -241,208 +188,44 @@ best performance. For that purpose, just use ``AUTO``. .. code:: ipython3 - # Read the network and corresponding weights from a file. - model = core.read_model(model=converted_model_path) - # Compile the model for CPU (you can choose manually CPU, GPU etc.) - # or let the engine choose the best available device (AUTO). - compiled_model = core.compile_model(model=model, device_name=device.value) - - # Get the input and output nodes. - input_layer = compiled_model.input(0) - output_layer = compiled_model.output(0) - - # Get the input size. - height, width = list(input_layer.shape)[1:3] - -Input and output layers have the names of the input node and output node -respectively. In the case of SSDLite MobileNetV2, there is 1 input and 1 -output. - -.. code:: ipython3 - - input_layer.any_name, output_layer.any_name - - - - -.. parsed-literal:: - - ('image_tensor:0', 'detection_boxes:0') - - - -Processing ----------- - - - -Process Results -~~~~~~~~~~~~~~~ - - - -First, list all available classes and create colors for them. Then, in -the post-process stage, transform boxes with normalized coordinates -``[0, 1]`` into boxes with pixel coordinates ``[0, image_size_in_px]``. -Afterward, use `non-maximum -suppression `__ -to reject overlapping detections and those below the probability -threshold (0.5). Finally, draw boxes and labels inside them. - -.. code:: ipython3 - - # https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/ - classes = [ - "background", - "person", - "bicycle", - "car", - "motorcycle", - "airplane", - "bus", - "train", - "truck", - "boat", - "traffic light", - "fire hydrant", - "street sign", - "stop sign", - "parking meter", - "bench", - "bird", - "cat", - "dog", - "horse", - "sheep", - "cow", - "elephant", - "bear", - "zebra", - "giraffe", - "hat", - "backpack", - "umbrella", - "shoe", - "eye glasses", - "handbag", - "tie", - "suitcase", - "frisbee", - "skis", - "snowboard", - "sports ball", - "kite", - "baseball bat", - "baseball glove", - "skateboard", - "surfboard", - "tennis racket", - "bottle", - "plate", - "wine glass", - "cup", - "fork", - "knife", - "spoon", - "bowl", - "banana", - "apple", - "sandwich", - "orange", - "broccoli", - "carrot", - "hot dog", - "pizza", - "donut", - "cake", - "chair", - "couch", - "potted plant", - "bed", - "mirror", - "dining table", - "window", - "desk", - "toilet", - "door", - "tv", - "laptop", - "mouse", - "remote", - "keyboard", - "cell phone", - "microwave", - "oven", - "toaster", - "sink", - "refrigerator", - "blender", - "book", - "clock", - "vase", - "scissors", - "teddy bear", - "hair drier", - "toothbrush", - "hair brush", - ] - - # Colors for the classes above (Rainbow Color Map). - colors = cv2.applyColorMap( - src=np.arange(0, 255, 255 / len(classes), dtype=np.float32).astype(np.uint8), - colormap=cv2.COLORMAP_RAINBOW, - ).squeeze() + core = ov.Core() - def process_results(frame, results, thresh=0.6): - # The size of the original frame. - h, w = frame.shape[:2] - # The 'results' variable is a [1, 1, 100, 7] tensor. - results = results.squeeze() - boxes = [] - labels = [] - scores = [] - for _, label, score, xmin, ymin, xmax, ymax in results: - # Create a box with pixels coordinates from the box with normalized coordinates [0,1]. - boxes.append(tuple(map(int, (xmin * w, ymin * h, (xmax - xmin) * w, (ymax - ymin) * h)))) - labels.append(int(label)) - scores.append(float(score)) + def load_model(det_model_path, device): + compiled_model = compile_model(det_model_path, device) + det_model = YOLO(det_model_path.parent, task="detect") - # Apply non-maximum suppression to get rid of many overlapping entities. - # See https://paperswithcode.com/method/non-maximum-suppression - # This algorithm returns indices of objects to keep. - indices = cv2.dnn.NMSBoxes(bboxes=boxes, scores=scores, score_threshold=thresh, nms_threshold=0.6) + if det_model.predictor is None: + custom = {"conf": 0.25, "batch": 1, "save": False, "mode": "predict"} # method defaults + args = {**det_model.overrides, **custom} + det_model.predictor = det_model._smart_load("predictor")(overrides=args, _callbacks=det_model.callbacks) + det_model.predictor.setup_model(model=det_model.model) - # If there are no boxes. - if len(indices) == 0: - return [] + det_model.predictor.model.ov_compiled_model = compiled_model + return det_model - # Filter detected objects. - return [(labels[idx], scores[idx], boxes[idx]) for idx in indices.flatten()] + def compile_model(det_model_path, device): + det_ov_model = core.read_model(det_model_path) - def draw_boxes(frame, boxes): - for label, score, box in boxes: - # Choose color for the label. - color = tuple(map(int, colors[label])) - # Draw a box. - x2 = box[0] + box[2] - y2 = box[1] + box[3] - cv2.rectangle(img=frame, pt1=box[:2], pt2=(x2, y2), color=color, thickness=3) + ov_config = {} + if device != "CPU": + det_ov_model.reshape({0: [1, 3, 640, 640]}) + if "GPU" in device or ("AUTO" in device and "GPU" in core.available_devices): + ov_config = {"GPU_DISABLE_WINOGRAD_CONVOLUTION": "YES"} + det_compiled_model = core.compile_model(det_ov_model, device, ov_config) + return det_compiled_model - # Draw a label name inside the box. - cv2.putText( - img=frame, - text=f"{classes[label]} {score:.2f}", - org=(box[0] + 10, box[1] + 30), - fontFace=cv2.FONT_HERSHEY_COMPLEX, - fontScale=frame.shape[1] / 1000, - color=color, - thickness=1, - lineType=cv2.LINE_AA, - ) - return frame + det_model = load_model(det_model_path, device.value) + + +.. parsed-literal:: + + Ultralytics 8.3.0 🚀 Python-3.8.10 torch-2.2.2+cpu CPU (Intel Core(TM) i9-10920X 3.50GHz) + Loading yolov8n_openvino_model for OpenVINO inference... + Using OpenVINO LATENCY mode for batch=1 inference... + Main Processing Function ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -454,8 +237,18 @@ file. .. code:: ipython3 + from IPython import display + import cv2 + import numpy as np + + # Main processing function to run object detection. - def run_object_detection(source=0, flip=False, use_popup=False, skip_first_frames=0): + def run_object_detection( + source=0, + flip=False, + use_popup=False, + skip_first_frames=0, + ): player = None try: # Create a video player to play with target fps. @@ -466,7 +259,6 @@ file. title = "Press ESC to Exit" cv2.namedWindow(winname=title, flags=cv2.WINDOW_GUI_NORMAL | cv2.WINDOW_AUTOSIZE) - processing_times = collections.deque() while True: # Grab the frame. frame = player.next() @@ -483,43 +275,10 @@ file. fy=scale, interpolation=cv2.INTER_AREA, ) - - # Resize the image and change dims to fit neural network input. - input_img = cv2.resize(src=frame, dsize=(width, height), interpolation=cv2.INTER_AREA) - # Create a batch of images (size = 1). - input_img = input_img[np.newaxis, ...] - - # Measure processing time. - - start_time = time.time() # Get the results. - results = compiled_model([input_img])[output_layer] - stop_time = time.time() - # Get poses from network results. - boxes = process_results(frame=frame, results=results) - - # Draw boxes on a frame. - frame = draw_boxes(frame=frame, boxes=boxes) - - processing_times.append(stop_time - start_time) - # Use processing times from last 200 frames. - if len(processing_times) > 200: - processing_times.popleft() - - _, f_width = frame.shape[:2] - # Mean processing time [ms]. - processing_time = np.mean(processing_times) * 1000 - fps = 1000 / processing_time - cv2.putText( - img=frame, - text=f"Inference time: {processing_time:.1f}ms ({fps:.1f} FPS)", - org=(20, 40), - fontFace=cv2.FONT_HERSHEY_COMPLEX, - fontScale=f_width / 1000, - color=(0, 0, 255), - thickness=1, - lineType=cv2.LINE_AA, - ) + input_image = np.array(frame) + detections = det_model(input_image, verbose=False) + frame = detections[0].plot() # Use this workaround if there is flickering. if use_popup: @@ -592,22 +351,10 @@ Run the object detection: -.. image:: object-detection-with-output_files/object-detection-with-output_19_0.png +.. image:: object-detection-with-output_files/object-detection-with-output_13_0.png .. parsed-literal:: Source ended - -References ----------- - - - -1. `SSDLite - MobileNetV2 `__ -2. `Open Model - Zoo `__ -3. `Non-Maximum - Suppression `__ diff --git a/docs/notebooks/object-detection-with-output_files/object-detection-with-output_13_0.png b/docs/notebooks/object-detection-with-output_files/object-detection-with-output_13_0.png new file mode 100644 index 00000000000000..1c548a0445636e --- /dev/null +++ b/docs/notebooks/object-detection-with-output_files/object-detection-with-output_13_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3fca049d156843f0976e9386ece07d1846e92efcacc5c559de140e7733b67b3 +size 171752 diff --git a/docs/notebooks/object-detection-with-output_files/object-detection-with-output_19_0.png b/docs/notebooks/object-detection-with-output_files/object-detection-with-output_19_0.png deleted file mode 100644 index fb2b3b2abdf69f..00000000000000 --- a/docs/notebooks/object-detection-with-output_files/object-detection-with-output_19_0.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1d300cd81ce6f44a5ee2ded6f14b739b0a8ecd8d5d2e487743cd085b92b3533b -size 175076 diff --git a/docs/notebooks/openvino-api-with-output.rst b/docs/notebooks/openvino-api-with-output.rst index a837b88ac6976b..b2b4c8c0f04fdd 100644 --- a/docs/notebooks/openvino-api-with-output.rst +++ b/docs/notebooks/openvino-api-with-output.rst @@ -201,7 +201,7 @@ notebooks. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') @@ -250,7 +250,7 @@ points to the filename of an ONNX model. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/segmentation.onnx') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/segmentation.onnx') @@ -310,7 +310,7 @@ without any conversion step. Pass the filename with extension to .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/inference.pdiparams') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/inference.pdiparams') @@ -354,7 +354,7 @@ TensorFlow models saved in frozen graph format can also be passed to .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.pb') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.pb') @@ -497,7 +497,7 @@ Information about the inputs and outputs of the model are in .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') @@ -703,7 +703,7 @@ produced data as values. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') @@ -892,7 +892,7 @@ input shape. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/segmentation.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/segmentation.bin') @@ -1044,7 +1044,7 @@ the cache. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') diff --git a/docs/notebooks/openvoice-with-output.rst b/docs/notebooks/openvoice-with-output.rst index b73dd8059aa65b..2ee11fcded84dc 100644 --- a/docs/notebooks/openvoice-with-output.rst +++ b/docs/notebooks/openvoice-with-output.rst @@ -100,10 +100,11 @@ Clone repository and install requirements Cloning into 'OpenVoice'... remote: Enumerating objects: 438, done. remote: Total 438 (delta 0), reused 0 (delta 0), pack-reused 438 (from 1) - Receiving objects: 100% (438/438), 3.84 MiB | 19.30 MiB/s, done. + Receiving objects: 100% (438/438), 3.84 MiB | 21.51 MiB/s, done. Resolving deltas: 100% (207/207), done. ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.17.2+cpu which is incompatible. + tensorflow 2.12.0 requires numpy<1.24,>=1.22, but you have numpy 1.24.4 which is incompatible. torchvision 0.17.2+cpu requires torch==2.2.2, but you have torch 2.4.1 which is incompatible. Note: you may need to restart the kernel to use updated packages. @@ -242,9 +243,9 @@ True .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. WeightNorm.apply(module, name, dim) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/api.py:36: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/api.py:36: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. checkpoint_dict = torch.load(ckpt_path, map_location=torch.device(self.device)) @@ -258,9 +259,9 @@ True .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/wavmark/__init__.py:16: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/wavmark/__init__.py:16: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. checkpoint = torch.load(resume_path, map_location=torch.device('cpu')) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/api.py:36: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/api.py:36: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. checkpoint_dict = torch.load(ckpt_path, map_location=torch.device(self.device)) @@ -410,40 +411,40 @@ documentation 0 No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:283: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:283: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert ( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:346: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:346: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! pad_length = max(length - (self.window_size + 1), 0) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:347: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:347: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! slice_start_position = max((self.window_size + 1) - length, 0) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:349: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:349: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if pad_length > 0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:114: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:114: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if torch.min(inputs) < left or torch.max(inputs) > right: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:119: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:119: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if min_bin_width * num_bins > 1.0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:121: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:121: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if min_bin_height * num_bins > 1.0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:171: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:171: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert (discriminant >= 0).all() - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes: - %3293 : Float(1, 2, 43, strides=[86, 43, 1], requires_grad=0, device=cpu) = aten::randn(%3288, %3289, %3290, %3291, %3292) # /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86:0 - %5559 : Float(1, 192, 152, strides=[29184, 1, 192], requires_grad=0, device=cpu) = aten::randn_like(%m_p, %5554, %5555, %5556, %5557, %5558) # /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86:0 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes: + %3293 : Float(1, 2, 43, strides=[86, 43, 1], requires_grad=0, device=cpu) = aten::randn(%3288, %3289, %3290, %3291, %3292) # /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86:0 + %5559 : Float(1, 192, 153, strides=[29376, 1, 192], requires_grad=0, device=cpu) = aten::randn_like(%m_p, %5554, %5555, %5556, %5557, %5558) # /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86:0 This may cause errors in trace checking. To disable trace checking, pass check_trace=False to torch.jit.trace() _check_trace( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: - The values for attribute 'shape' do not match: torch.Size([1, 1, 38656]) != torch.Size([1, 1, 39680]). + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: + The values for attribute 'shape' do not match: torch.Size([1, 1, 39424]) != torch.Size([1, 1, 38656]). _check_trace( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 2. of the traced function does not match the corresponding output of the Python function. Detailed error: - The values for attribute 'shape' do not match: torch.Size([1, 1, 151, 43]) != torch.Size([1, 1, 155, 43]). + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 2. of the traced function does not match the corresponding output of the Python function. Detailed error: + The values for attribute 'shape' do not match: torch.Size([1, 1, 154, 43]) != torch.Size([1, 1, 151, 43]). _check_trace( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 3. of the traced function does not match the corresponding output of the Python function. Detailed error: - The values for attribute 'shape' do not match: torch.Size([1, 1, 151]) != torch.Size([1, 1, 155]). + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 3. of the traced function does not match the corresponding output of the Python function. Detailed error: + The values for attribute 'shape' do not match: torch.Size([1, 1, 154]) != torch.Size([1, 1, 151]). _check_trace( - 2024-10-23 02:08:53.401718: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 02:13:33.268258: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -476,16 +477,16 @@ documentation )`. (Triggered internally at ../aten/src/ATen/native/SpectralOps.cpp:836.) + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/modules/module.py:1562: UserWarning: A window was not provided. A rectangular window will be applied,which is known to cause spectral leakage. Other windows such as torch.hann_window or torch.hamming_window can are recommended to reduce spectral leakage.To suppress this warning and use a rectangular window, explicitly set `window=torch.ones(n_fft, device=)`. (Triggered internally at ../aten/src/ATen/native/SpectralOps.cpp:836.) return forward_call(\*args, \*\*kwargs) @@ -713,7 +714,7 @@ Load speaker embeddings .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/functional.py:666: UserWarning: stft with return_complex=False is deprecated. In a future pytorch release, stft will return complex tensors for all inputs, and return_complex=False will raise an error. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/functional.py:666: UserWarning: stft with return_complex=False is deprecated. In a future pytorch release, stft will return complex tensors for all inputs, and return_complex=False will raise an error. Note: you can still call torch.view_as_real on the complex output to recover the old return format. (Triggered internally at ../aten/src/ATen/native/SpectralOps.cpp:873.) return _VF.stft(input, n_fft, hop_length, win_length, window, # type: ignore[attr-defined] @@ -868,7 +869,7 @@ And finally, run voice tone conversion with OpenVINO optimized model @@ -886,7 +887,7 @@ And finally, run voice tone conversion with OpenVINO optimized model @@ -1075,7 +1076,7 @@ voice tone conversion online. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/gradio/components/dropdown.py:100: UserWarning: The `max_choices` parameter is ignored when `multiselect` is False. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/gradio/components/dropdown.py:100: UserWarning: The `max_choices` parameter is ignored when `multiselect` is False. warnings.warn( diff --git a/docs/notebooks/optical-character-recognition-with-output.rst b/docs/notebooks/optical-character-recognition-with-output.rst index 9efa8e8e7b87f2..7dae2290312e68 100644 --- a/docs/notebooks/optical-character-recognition-with-output.rst +++ b/docs/notebooks/optical-character-recognition-with-output.rst @@ -9,16 +9,9 @@ which shows only text detection. The `horizontal-text-detection-0001 `__ and -`text-recognition-resnet `__ +`text-recognition-0014 `__ models are used together for text detection and then text recognition. -In this tutorial, Open Model Zoo tools including Model Downloader, Model -Converter and Info Dumper are used to download and convert the models -from `Open Model -Zoo `__. For more -information, refer to the -`model-tools `__ tutorial. - **Table of contents:** @@ -26,7 +19,6 @@ information, refer to the - `Imports <#imports>`__ - `Settings <#settings>`__ - `Download Models <#download-models>`__ -- `Convert Models <#convert-models>`__ - `Select inference device <#select-inference-device>`__ - `Object Detection <#object-detection>`__ @@ -61,15 +53,12 @@ Guide =2024.0.0" "onnx<1.16.2" torch torchvision pillow opencv-python "matplotlib>=3.4" --extra-index-url https://download.pytorch.org/whl/cpu + # Install openvino package + %pip install -q "openvino>=2024.4.0" pillow opencv-python "matplotlib>=3.4" .. parsed-literal:: - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.17.2+cpu which is incompatible. - torchaudio 2.4.1+cpu requires torch==2.4.1, but you have torch 2.2.2+cpu which is incompatible. Note: you may need to restart the kernel to use updated packages. @@ -111,7 +100,7 @@ Settings model_dir = Path("model") precision = "FP16" detection_model = "horizontal-text-detection-0001" - recognition_model = "text-recognition-resnet-fc" + recognition_model = "text-recognition-0014" model_dir.mkdir(exist_ok=True) @@ -120,274 +109,35 @@ Download Models -The next cells will run Model Downloader to download the detection and -recognition models. If the models have been downloaded before, they will -not be downloaded again. +The next cells will download the detection and recognition models. If +the models have been downloaded before, they will not be downloaded +again. .. code:: ipython3 - download_command = ( - f"omz_downloader --name {detection_model},{recognition_model} --output_dir {model_dir} --cache_dir {model_dir} --precision {precision} --num_attempts 5" - ) - display(Markdown(f"Download command: `{download_command}`")) - display(Markdown(f"Downloading {detection_model}, {recognition_model}...")) - !$download_command - display(Markdown(f"Finished downloading {detection_model}, {recognition_model}.")) - - detection_model_path = (model_dir / "intel/horizontal-text-detection-0001" / precision / detection_model).with_suffix(".xml") - recognition_model_path = (model_dir / "public/text-recognition-resnet-fc" / precision / recognition_model).with_suffix(".xml") - - - -Download command: -``omz_downloader --name horizontal-text-detection-0001,text-recognition-resnet-fc --output_dir model --cache_dir model --precision FP16 --num_attempts 5`` - - - -Downloading horizontal-text-detection-0001, text-recognition-resnet-fc… - - -.. parsed-literal:: - - ################|| Downloading horizontal-text-detection-0001 ||################ - - ========== Downloading model/intel/horizontal-text-detection-0001/FP16/horizontal-text-detection-0001.xml - - - ========== Downloading model/intel/horizontal-text-detection-0001/FP16/horizontal-text-detection-0001.bin - - - ################|| Downloading text-recognition-resnet-fc ||################ - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/model.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/weight_init.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/heads/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/heads/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/heads/fc_head.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/heads/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/body.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/component.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/sequences/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/sequences/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/sequences/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/bricks/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/bricks/bricks.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/bricks/builder.py - + from notebook_utils import download_ir_model - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/bricks/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/resnet.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/enhance_modules/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/enhance_modules/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/enhance_modules/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/utils/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/utils/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/utils/conv_module.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/utils/fc_module.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/utils/norm.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/utils/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/utils/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/utils/common.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/utils/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/utils/config.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/configs/resnet_fc.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/ckpt/resnet_fc.pth - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/addict-2.4.0-py3-none-any.whl + detection_model_url = f"https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{detection_model}/{precision}/{detection_model}.xml" + recognition_model_url = ( + f"https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{recognition_model}/{precision}/{recognition_model}.xml" + ) - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/heads/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/sequences/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/component.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/bricks/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/enhance_modules/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/utils/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/utils/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/utils/config.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/utils/config.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/utils/config.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/utils/config.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/utils/config.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/resnet.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/resnet.py - ========== Unpacking model/public/text-recognition-resnet-fc/vedastr/addict-2.4.0-py3-none-any.whl + detection_model_path = download_ir_model(detection_model_url, model_dir / detection_model / precision) + recognition_model_path = download_ir_model(recognition_model_url, model_dir / recognition_model / precision) -Finished downloading horizontal-text-detection-0001, -text-recognition-resnet-fc. - - -.. code:: ipython3 - - ### The text-recognition-resnet-fc model consists of many files. All filenames are printed in - ### the output of Model Downloader. Uncomment the next two lines to show this output. - - # for line in download_result: - # print(line) - -Convert Models --------------- - - - -The downloaded detection model is an Intel model, which is already in -OpenVINO Intermediate Representation (OpenVINO IR) format. The text -recognition model is a public model which needs to be converted to -OpenVINO IR. Since this model was downloaded from Open Model Zoo, use -Model Converter to convert the model to OpenVINO IR format. - -The output of Model Converter will be displayed. When the conversion is -successful, the last lines of output will include -``[ SUCCESS ] Generated IR version 11 model.`` - -.. code:: ipython3 - - convert_command = f"omz_converter --name {recognition_model} --precisions {precision} --download_dir {model_dir} --output_dir {model_dir}" - display(Markdown(f"Convert command: `{convert_command}`")) - display(Markdown(f"Converting {recognition_model}...")) - ! $convert_command - - - -Convert command: -``omz_converter --name text-recognition-resnet-fc --precisions FP16 --download_dir model --output_dir model`` - +.. parsed-literal:: + model/horizontal-text-detection-0001/FP16/horizontal-text-detection-0001.bin: 0%| | 0.00/3.70M [00:… -Converting text-recognition-resnet-fc… .. parsed-literal:: - ========== Converting text-recognition-resnet-fc to ONNX - Conversion to ONNX command: /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/omz_tools/internal_scripts/pytorch_to_onnx.py --model-path=/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/omz_tools/models/public/text-recognition-resnet-fc --model-path=model/public/text-recognition-resnet-fc --model-name=get_model --import-module=model '--model-param=file_config=r"model/public/text-recognition-resnet-fc/vedastr/configs/resnet_fc.py"' '--model-param=weights=r"model/public/text-recognition-resnet-fc/vedastr/ckpt/resnet_fc.pth"' --input-shape=1,1,32,100 --input-names=input --output-names=output --output-file=model/public/text-recognition-resnet-fc/resnet_fc.onnx - - ONNX check passed successfully. - - ========== Converting text-recognition-resnet-fc to IR (FP16) - Conversion command: /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=model/public/text-recognition-resnet-fc/FP16 --model_name=text-recognition-resnet-fc --input=input '--mean_values=input[127.5]' '--scale_values=input[127.5]' --output=output --input_model=model/public/text-recognition-resnet-fc/resnet_fc.onnx '--layout=input(NCHW)' '--input_shape=[1, 1, 32, 100]' --compress_to_fp16=True - - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. - In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. - Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html - [ INFO ] Generated IR will be compressed to FP16. If you get lower accuracy, please consider disabling compression explicitly by adding argument --compress_to_fp16=False. - Find more information about compression to FP16 at https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_FP16_Compression.html - [ SUCCESS ] Generated IR version 11 model. - [ SUCCESS ] XML file: /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/optical-character-recognition/model/public/text-recognition-resnet-fc/FP16/text-recognition-resnet-fc.xml - [ SUCCESS ] BIN file: /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/optical-character-recognition/model/public/text-recognition-resnet-fc/FP16/text-recognition-resnet-fc.bin - + model/text-recognition-0014/FP16/text-recognition-0014.bin: 0%| | 0.00/17.4M [00:00 + @@ -384,7 +384,7 @@ may be specified is input data .. parsed-literal:: - + @@ -422,7 +422,7 @@ then such conversion will be added explicitly. .. parsed-literal:: - + @@ -636,6 +636,6 @@ Compare performance .. parsed-literal:: - IR model in OpenVINO Runtime/CPU with manual image preprocessing: 0.0154 seconds per image, FPS: 65.10 - IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0142 seconds per image, FPS: 70.53 + IR model in OpenVINO Runtime/CPU with manual image preprocessing: 0.0153 seconds per image, FPS: 65.39 + IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0166 seconds per image, FPS: 60.23 diff --git a/docs/notebooks/paddle-ocr-webcam-with-output.rst b/docs/notebooks/paddle-ocr-webcam-with-output.rst index e5c426d1920f33..9f7510cd5efe96 100644 --- a/docs/notebooks/paddle-ocr-webcam-with-output.rst +++ b/docs/notebooks/paddle-ocr-webcam-with-output.rst @@ -75,6 +75,8 @@ Guide =1.22, but you have numpy 1.24.4 which is incompatible. Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -207,7 +209,7 @@ Download the Model for Text **Detection** .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-no… + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-no… .. parsed-literal:: @@ -253,7 +255,7 @@ Download the Model for Text **Recognition** .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-no… + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-no… .. parsed-literal:: diff --git a/docs/notebooks/paddle-ocr-webcam-with-output_files/paddle-ocr-webcam-with-output_30_0.png b/docs/notebooks/paddle-ocr-webcam-with-output_files/paddle-ocr-webcam-with-output_30_0.png index 43d5bfb39ed4e8..38a0d5d593351b 100644 --- a/docs/notebooks/paddle-ocr-webcam-with-output_files/paddle-ocr-webcam-with-output_30_0.png +++ b/docs/notebooks/paddle-ocr-webcam-with-output_files/paddle-ocr-webcam-with-output_30_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e5f67b60f77e15fb0b7908ada4ff64ab506accc23f579c81193175e5044f8f3b -size 593679 +oid sha256:ac7efd85f2c50b0a189dbf00c0cd2252f362e6469cd014d8f255c53172152c3d +size 591373 diff --git a/docs/notebooks/paddle-to-openvino-classification-with-output.rst b/docs/notebooks/paddle-to-openvino-classification-with-output.rst index 576878a578a323..25feb9293ee93a 100644 --- a/docs/notebooks/paddle-to-openvino-classification-with-output.rst +++ b/docs/notebooks/paddle-to-openvino-classification-with-output.rst @@ -89,11 +89,11 @@ Imports .. parsed-literal:: - --2024-10-23 02:12:15-- http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb + --2024-11-05 02:15:59-- http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb Resolving proxy-dmz.intel.com (proxy-dmz.intel.com)... 10.241.208.166 Connecting to proxy-dmz.intel.com (proxy-dmz.intel.com)|10.241.208.166|:911... connected. Proxy request sent, awaiting response... 404 Not Found - 2024-10-23 02:12:15 ERROR 404: Not Found. + 2024-11-05 02:16:00 ERROR 404: Not Found. dpkg: error: cannot access archive 'libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb': No such file or directory @@ -124,8 +124,8 @@ Imports .. parsed-literal:: - 2024-10-23 02:12:17 INFO: Loading faiss with AVX512 support. - 2024-10-23 02:12:17 INFO: Successfully loaded faiss with AVX512 support. + 2024-11-05 02:16:02 INFO: Loading faiss with AVX512 support. + 2024-11-05 02:16:02 INFO: Successfully loaded faiss with AVX512 support. Settings @@ -209,7 +209,7 @@ inference on that image, and then show the top three prediction results. .. parsed-literal:: - [2024/10/23 02:12:43] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead. + [2024/11/05 02:16:41] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead. Labrador retriever, 0.75138 German short-haired pointer, 0.02373 Great Dane, 0.01848 @@ -275,7 +275,7 @@ clipping values. .. parsed-literal:: - 2024-10-23 02:12:44 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). + 2024-11-05 02:16:42 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). .. parsed-literal:: @@ -287,7 +287,7 @@ clipping values. .. parsed-literal:: - + @@ -462,7 +462,7 @@ Note that many optimizations are possible to improve the performance. .. parsed-literal:: - PaddlePaddle model on CPU: 0.0073 seconds per image, FPS: 137.19 + PaddlePaddle model on CPU: 0.0074 seconds per image, FPS: 134.37 PaddlePaddle result: Labrador retriever, 0.75138 @@ -523,7 +523,7 @@ select device from dropdown list for running inference using OpenVINO .. parsed-literal:: - OpenVINO IR model in OpenVINO Runtime (AUTO): 0.0026 seconds per image, FPS: 382.97 + OpenVINO IR model in OpenVINO Runtime (AUTO): 0.0027 seconds per image, FPS: 373.31 OpenVINO result: Labrador retriever, 0.74909 diff --git a/docs/notebooks/parler-tts-text-to-speech-with-output.rst b/docs/notebooks/parler-tts-text-to-speech-with-output.rst index 3bbb67d2232c99..323959aa17e8ef 100644 --- a/docs/notebooks/parler-tts-text-to-speech-with-output.rst +++ b/docs/notebooks/parler-tts-text-to-speech-with-output.rst @@ -9,7 +9,7 @@ with synthetic annotations `__ by Dan Lyth and Simon King, from Stability AI and Edinburgh University respectively. -|image0| +.. image:: https://images.squarespace-cdn.com/content/v1/657816dfbefe0533e8a69d9a/30c96e25-acc5-4019-acdd-648da6142c4c/architecture_v3.png?format=2500w Text-to-speech models trained on large-scale datasets have demonstrated impressive in-context learning capabilities and naturalness. However, @@ -53,8 +53,6 @@ need a Jupyter server to start. For details, please refer to `Installation Guide `__. -.. |image0| image:: https://images.squarespace-cdn.com/content/v1/657816dfbefe0533e8a69d9a/30c96e25-acc5-4019-acdd-648da6142c4c/architecture_v3.png?format=2500w - Prerequisites ------------- @@ -69,23 +67,6 @@ Prerequisites %pip install -q "openvino>=2024.2.0" %pip install -q git+https://github.com/huggingface/parler-tts.git "gradio>=4.19" transformers "torch>=2.2" --extra-index-url https://download.pytorch.org/whl/cpu - -.. parsed-literal:: - - Note: you may need to restart the kernel to use updated packages. - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - googleapis-common-protos 1.65.0 requires protobuf!=3.20.0,!=3.20.1,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0.dev0,>=3.20.2, but you have protobuf 3.19.6 which is incompatible. - mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.17.2+cpu which is incompatible. - onnx 1.16.1 requires protobuf>=3.20.2, but you have protobuf 3.19.6 which is incompatible. - paddlepaddle 2.6.2 requires protobuf>=3.20.2; platform_system != "Windows", but you have protobuf 3.19.6 which is incompatible. - tensorflow 2.12.0 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 3.19.6 which is incompatible. - tensorflow-datasets 4.9.2 requires protobuf>=3.20, but you have protobuf 3.19.6 which is incompatible. - tensorflow-metadata 1.14.0 requires protobuf<4.21,>=3.20.3, but you have protobuf 3.19.6 which is incompatible. - torchvision 0.17.2+cpu requires torch==2.2.2, but you have torch 2.4.1+cpu which is incompatible. - visualdl 2.5.3 requires protobuf>=3.20.0, but you have protobuf 3.19.6 which is incompatible. - Note: you may need to restart the kernel to use updated packages. - - Load the original model and inference ------------------------------------- @@ -114,19 +95,6 @@ Load the original model and inference audio_arr = generation.cpu().numpy().squeeze() sf.write("parler_tts_out.wav", audio_arr, model.config.sampling_rate) - -.. parsed-literal:: - - 2024-10-23 02:13:22.641328: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 02:13:22.675982: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - Flash attention 2 is not installed - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. - WeightNorm.apply(module, name, dim) - You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers - The attention mask is not set and cannot be inferred from input because pad token is same as eos token.As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results. - - .. code:: ipython3 import IPython.display as ipd @@ -140,10 +108,10 @@ Load the original model and inference - + @@ -191,13 +159,6 @@ and Decoder (``ParlerTTSDecoder``). Lets convert them one by one. text_encoder_ov_model = convert(model.text_encoder, TEXT_ENCODER_OV_PATH, example_input) - -.. parsed-literal:: - - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4664: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead - warnings.warn( - - The Decoder Model performs in generation pipeline and we can separate it into two stage. In the first stage the model generates ``past_key_values`` into output for the second stage. In the second @@ -232,17 +193,6 @@ stage the model produces tokens during several runs. decoder_1_ov_model = convert(DecoderStage1Wrapper(model.decoder.model.decoder), DECODER_STAGE_1_OV_PATH, example_input) - -.. parsed-literal:: - - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/parler_tts/modeling_parler_tts.py:253: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - if seq_len > self.weights.size(0): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/parler_tts/modeling_parler_tts.py:1599: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - if sequence_length != 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/parler_tts/modeling_parler_tts.py:802: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - if attn_output.size() != (bsz, self.num_heads, tgt_len, self.head_dim): - - .. code:: ipython3 DECODER_STAGE_2_OV_PATH = Path("models/decoder_stage_2_ir.xml") @@ -308,7 +258,7 @@ Select device from dropdown list for running inference using OpenVINO. .. parsed-literal:: - Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') + Dropdown(description='Device:', index=4, options=('CPU', 'GPU.0', 'GPU.1', 'GPU.2', 'AUTO'), value='AUTO') @@ -410,10 +360,10 @@ and run inference. - + @@ -456,27 +406,13 @@ Interactive inference demo = make_demo(fn=infer) try: - demo.queue().launch(debug=False) + demo.queue().launch(debug=True) except Exception: - demo.queue().launch(share=True, debug=False) + demo.queue().launch(share=True, debug=True) # if you are launching remotely, specify server_name and server_port # demo.launch(server_name='your server name', server_port='server port in int') # Read more in the docs: https://gradio.app/docs/ - -.. parsed-literal:: - - Running on local URL: http://127.0.0.1:7860 - - To create a public link, set `share=True` in `launch()`. - - - - - - - - .. code:: ipython3 # please uncomment and run this cell for stopping gradio interface diff --git a/docs/notebooks/person-tracking-with-output.rst b/docs/notebooks/person-tracking-with-output.rst index 2b07305615f09e..653a9b376edf7e 100644 --- a/docs/notebooks/person-tracking-with-output.rst +++ b/docs/notebooks/person-tracking-with-output.rst @@ -128,7 +128,7 @@ Guide =2024.0.0" + %pip install -q "openvino>=2024.0.0" %pip install -q opencv-python requests scipy tqdm "matplotlib>=3.4" @@ -189,16 +189,6 @@ We will use pre-trained models from OpenVINO’s `Open Model Zoo `__ to start the test. -Use ``omz_downloader``, which is a command-line tool from the -``openvino-dev`` package. It automatically creates a directory structure -and downloads the selected model. This step is skipped if the model is -already downloaded. The selected model comes from the public directory, -which means it must be converted into OpenVINO Intermediate -Representation (OpenVINO IR). - - **NOTE**: Using a model outside the list can require different pre- - and post-processing. - In this case, `person detection model `__ is deployed to detect the person in each frame of the video, and @@ -207,60 +197,39 @@ model `__, -``person-reidentification-retail-xxx`` from `Reidentification Models -list `__), -replace the name of the model in the code below. - .. code:: ipython3 + from notebook_utils import download_ir_model + # A directory where the model will be downloaded. base_model_dir = "model" precision = "FP16" # The name of the model from Open Model Zoo detection_model_name = "person-detection-0202" - download_command = ( - f"omz_downloader " f"--name {detection_model_name} " f"--precisions {precision} " f"--output_dir {base_model_dir} " f"--cache_dir {base_model_dir}" - ) - ! $download_command - detection_model_path = f"model/intel/{detection_model_name}/{precision}/{detection_model_name}.xml" + download_det_model_url = ( + f"https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{detection_model_name}/{precision}/{detection_model_name}.xml" + ) + detection_model_path = download_ir_model(download_det_model_url, Path(base_model_dir) / detection_model_name / precision) reidentification_model_name = "person-reidentification-retail-0287" + download_reid_model_url = f"https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{reidentification_model_name}/{precision}/{reidentification_model_name}.xml" - download_command = ( - f"omz_downloader " f"--name {reidentification_model_name} " f"--precisions {precision} " f"--output_dir {base_model_dir} " f"--cache_dir {base_model_dir}" - ) - ! $download_command + reidentification_model_path = download_ir_model(download_reid_model_url, Path(base_model_dir) / reidentification_model_name / precision) - reidentification_model_path = f"model/intel/{reidentification_model_name}/{precision}/{reidentification_model_name}.xml" .. parsed-literal:: - ################|| Downloading person-detection-0202 ||################ - - ========== Downloading model/intel/person-detection-0202/FP16/person-detection-0202.xml + model/person-detection-0202/FP16/person-detection-0202.bin: 0%| | 0.00/3.47M [00:00`__ #### -Table of contents: +precision using `NNCF `__ + +**Table of contents:** - `Prerequisites <#prerequisites>`__ - `Select Model <#select-model>`__ @@ -56,11 +57,6 @@ install required packages and setup helper functions. .. parsed-literal:: - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - descript-audiotools 0.7.2 requires protobuf<3.20,>=3.9.2, but you have protobuf 5.28.2 which is incompatible. - open-clip-torch 2.22.0 requires protobuf<4, but you have protobuf 5.28.2 which is incompatible. - tensorflow 2.12.0 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 5.28.2 which is incompatible. - tensorflow-metadata 1.14.0 requires protobuf<4.21,>=3.20.3, but you have protobuf 5.28.2 which is incompatible. Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -264,10 +260,10 @@ documentation 1 or self.sliding_window is not None) and self.is_causal: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py:444: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! seq_len = seq_len or torch.max(position_ids) + 1 /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py:445: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if seq_len > self.original_max_position_embeddings: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. op1 = operator(\*args, \*\*kwargs) + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/cache_utils.py:443: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results. + elif len(self.key_cache[layer_idx]) == 0: # fills previously skipped layers; checking for tensor causes errors /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py:683: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py:690: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py:702: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:165: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at aten/src/ATen/core/TensorBody.h:489.) + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:165: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at aten/src/ATen/core/TensorBody.h:489.) if a.grad is not None: @@ -505,12 +516,6 @@ arguments. For running model we will use ``generate`` method. generate_ids = model.generate(**inputs, eos_token_id=processor.tokenizer.eos_token_id, **generation_args) -.. parsed-literal:: - - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/auto/image_processing_auto.py:513: FutureWarning: The image_processor_class argument is deprecated and will be removed in v4.42. Please use `slow_image_processor_class`, or `fast_image_processor_class` instead - warnings.warn( - - .. parsed-literal:: Answer: diff --git a/docs/notebooks/photo-maker-with-output.rst b/docs/notebooks/photo-maker-with-output.rst index fe49eb9217065e..1685e9c05ea9f1 100644 --- a/docs/notebooks/photo-maker-with-output.rst +++ b/docs/notebooks/photo-maker-with-output.rst @@ -94,13 +94,13 @@ Clone PhotoMaker repository .. parsed-literal:: Cloning into 'PhotoMaker'... - remote: Enumerating objects: 303, done. - remote: Counting objects: 100% (148/148), done. - remote: Compressing objects: 100% (95/95), done. - remote: Total 303 (delta 130), reused 53 (delta 53), pack-reused 155 (from 1) - Receiving objects: 100% (303/303), 10.23 MiB | 29.77 MiB/s, done. - Resolving deltas: 100% (162/162), done. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/photo-maker/PhotoMaker + remote: Enumerating objects: 306, done. + remote: Counting objects: 100% (151/151), done. + remote: Compressing objects: 100% (98/98), done. + remote: Total 306 (delta 132), reused 53 (delta 53), pack-reused 155 (from 1) + Receiving objects: 100% (306/306), 10.24 MiB | 23.03 MiB/s, done. + Resolving deltas: 100% (164/164), done. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/photo-maker/PhotoMaker Note: switching to '1e78aa6514c11a84ef1be27b56c7c72d6c70f8fc'. You are in 'detached HEAD' state. You can look around, make experimental @@ -119,7 +119,7 @@ Clone PhotoMaker repository Turn off this advice by setting config variable advice.detachedHead to false HEAD is now at 1e78aa6 Update README.md - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/photo-maker + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/photo-maker Install required packages @@ -133,11 +133,9 @@ Install required packages .. parsed-literal:: ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - descript-audiotools 0.7.2 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.3 which is incompatible. - mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.17.2+cpu which is incompatible. paddleclas 2.5.2 requires gast==0.3.3, but you have gast 0.4.0 which is incompatible. paddleclas 2.5.2 requires opencv-python==4.6.0.66, but you have opencv-python 4.10.0.84 which is incompatible. - supervision 0.24.0 requires numpy<1.23.3,>=1.21.2; python_full_version <= "3.10.0", but you have numpy 1.24.4 which is incompatible. + parler-tts 0.2.1 requires protobuf>=4.0.0, but you have protobuf 3.20.3 which is incompatible. tensorflow 2.12.0 requires numpy<1.24,>=1.22, but you have numpy 1.24.4 which is incompatible. Note: you may need to restart the kernel to use updated packages. @@ -199,10 +197,10 @@ PhotoMaker to generate the original PhotoMaker pipeline. .. parsed-literal:: - 2024-10-23 02:19:25.748160: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 02:19:25.783265: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 02:22:09.727876: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 02:22:09.761823: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 02:19:26.449413: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 02:22:10.482979: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. code:: ipython3 @@ -391,9 +389,12 @@ output(text embeddings) which will be the input for U-Net model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4664: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/photo-maker/PhotoMaker/photomaker/model.py:84: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:243: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if not interpolate_pos_encoding and (height != self.image_size or width != self.image_size): + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/photo-maker/PhotoMaker/photomaker/model.py:84: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert class_tokens_mask.sum() == stacked_id_embeds.shape[0], f"{class_tokens_mask.sum()} != {stacked_id_embeds.shape[0]}" @@ -468,9 +469,9 @@ sequence of latent text embeddings. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:86: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:88: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input_shape[-1] > 1 or self.sliding_window is not None: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: @@ -574,15 +575,15 @@ original Stable Diffusion XL model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_2d_condition.py:1103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_2d_condition.py:1103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if dim % default_overall_up_factor != 0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:136: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:136: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:145: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:145: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:146: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:146: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if hidden_states.shape[0] >= 64: diff --git a/docs/notebooks/pixart-with-output.rst b/docs/notebooks/pixart-with-output.rst index ef008d1d82ee52..c1c9a4b4e8ec57 100644 --- a/docs/notebooks/pixart-with-output.rst +++ b/docs/notebooks/pixart-with-output.rst @@ -118,10 +118,10 @@ directly in latent space, achieving super fast inference with few steps. .. parsed-literal:: - 2024-10-23 02:27:23.824587: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 02:27:23.860019: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 02:30:04.644117: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 02:30:04.680089: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 02:27:24.531762: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 02:30:05.360275: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -130,17 +130,17 @@ directly in latent space, achieving super fast inference with few steps. Loading pipeline components...: 0%| | 0/5 [00:00. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 +.. parsed-literal:: + Loading checkpoint shards: 0%| | 0/4 [00:00. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 + Some weights of the model checkpoint were not used when initializing PixArtTransformer2DModel: + ['caption_projection.y_embedding'] @@ -229,8 +229,9 @@ Convert text encoder .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4664: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. Convert transformer @@ -271,11 +272,11 @@ Convert transformer .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/embeddings.py:219: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/embeddings.py:219: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if self.height != height or self.width != width: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/attention_processor.py:682: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/attention_processor.py:682: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if current_length != target_length: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/attention_processor.py:697: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/attention_processor.py:697: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attention_mask.shape[0] < batch_size * head_size: @@ -300,9 +301,9 @@ Convert VAE decoder .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:146: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:146: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if hidden_states.shape[0] >= 64: @@ -448,7 +449,7 @@ And insert wrappers instances in the pipeline: .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'PixArtAlphaPipeline' object attribute is deprecated. Please access '_execution_device' over 'PixArtAlphaPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'PixArtAlphaPipeline' object attribute is deprecated. Please access '_execution_device' over 'PixArtAlphaPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -563,7 +564,7 @@ To collect intermediate model inputs for calibration we should customize .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'PixArtAlphaPipeline' object attribute is deprecated. Please access '_execution_device' over 'PixArtAlphaPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'PixArtAlphaPipeline' object attribute is deprecated. Please access '_execution_device' over 'PixArtAlphaPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -1621,16 +1622,16 @@ pipelines. Loading pipeline components...: 0%| | 0/5 [00:00=4.43.0, but you have transformers 4.45.2 which is incompatible. Note: you may need to restart the kernel to use updated packages. @@ -156,21 +153,21 @@ documentation 0.19. Please downgrade to tokenizers version <= 0.19 to export tokenizers to OpenVINO. + Exporting tokenizers to OpenVINO is not supported for tokenizers version > 0.19 and openvino version <= 2024.4. Please downgrade to tokenizers version <= 0.19 to export tokenizers to OpenVINO. INFO:nncf:Statistics of the bitwidth distribution: ┍━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑ │ Num bits (N) │ % all parameters (layers) │ % ratio-defining parameters (layers) │ @@ -488,7 +485,7 @@ documentation target_length: + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/cache_utils.py:443: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results. elif len(self.key_cache[layer_idx]) == 0: # fills previously skipped layers; checking for tensor causes errors - Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32) .. parsed-literal:: diff --git a/docs/notebooks/qwen2-vl-with-output.rst b/docs/notebooks/qwen2-vl-with-output.rst index 0ff882bb537ef2..a6f49a58ff0f25 100644 --- a/docs/notebooks/qwen2-vl-with-output.rst +++ b/docs/notebooks/qwen2-vl-with-output.rst @@ -139,10 +139,10 @@ using widget bellow: .. parsed-literal:: - 2024-10-23 04:32:38.810057: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 04:32:38.845114: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 04:37:28.225170: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 04:37:28.260034: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 04:32:39.399370: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 04:37:28.814599: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -335,26 +335,44 @@ documentation target_length: + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/cache_utils.py:443: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results. + elif len(self.key_cache[layer_idx]) == 0: # fills previously skipped layers; checking for tensor causes errors .. parsed-literal:: + ✅ Language model successfully converted ⌛ Weights compression with int4_asym mode started INFO:nncf:Statistics of the bitwidth distribution: ┍━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑ │ Num bits (N) │ % all parameters (layers) │ % ratio-defining parameters (layers) │ ┝━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥ - │ 8 │ 1% (1 / 130) │ 0% (0 / 129) │ + │ 8 │ 15% (1 / 197) │ 0% (0 / 196) │ ├────────────────┼─────────────────────────────┼────────────────────────────────────────┤ - │ 4 │ 99% (129 / 130) │ 100% (129 / 129) │ + │ 4 │ 85% (196 / 197) │ 100% (196 / 196) │ ┕━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙ @@ -374,31 +392,15 @@ documentation =3.9.2, but you have protobuf 3.20.3 which is incompatible. mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.19.1+cpu which is incompatible. - parler-tts 0.2 requires transformers<=4.43.3,>=4.43.0, but you have transformers 4.45.2 which is incompatible. + parler-tts 0.2.1 requires protobuf>=4.0.0, but you have protobuf 3.20.3 which is incompatible. Note: you may need to restart the kernel to use updated packages. Collecting iopath>=0.1.10 Using cached iopath-0.1.10-py3-none-any.whl - Requirement already satisfied: pillow>=9.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (10.4.0) - Requirement already satisfied: hydra-core>=1.3.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (1.3.2) - Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath>=0.1.10) (4.66.5) - Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath>=0.1.10) (4.12.2) - Requirement already satisfied: portalocker in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath>=0.1.10) (2.10.1) - Requirement already satisfied: omegaconf<2.4,>=2.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (2.3.0) - Requirement already satisfied: antlr4-python3-runtime==4.9.* in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (4.9.3) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (24.1) - Requirement already satisfied: importlib-resources in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (6.4.5) - Requirement already satisfied: PyYAML>=5.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from omegaconf<2.4,>=2.2->hydra-core>=1.3.2) (6.0.2) - Requirement already satisfied: zipp>=3.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from importlib-resources->hydra-core>=1.3.2) (3.20.2) + Requirement already satisfied: pillow>=9.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (10.4.0) + Requirement already satisfied: hydra-core>=1.3.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (1.3.2) + Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath>=0.1.10) (4.66.6) + Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath>=0.1.10) (4.12.2) + Requirement already satisfied: portalocker in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath>=0.1.10) (2.10.1) + Requirement already satisfied: omegaconf<2.4,>=2.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (2.3.0) + Requirement already satisfied: antlr4-python3-runtime==4.9.* in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (4.9.3) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (24.1) + Requirement already satisfied: importlib-resources in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (6.4.5) + Requirement already satisfied: PyYAML>=5.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from omegaconf<2.4,>=2.2->hydra-core>=1.3.2) (6.0.2) + Requirement already satisfied: zipp>=3.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from importlib-resources->hydra-core>=1.3.2) (3.20.2) Installing collected packages: iopath Attempting uninstall: iopath Found existing installation: iopath 0.1.9 @@ -204,10 +203,10 @@ Clone and install segment-anything-2 .. parsed-literal:: env: SAM2_BUILD_CUDA=0 - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/sam2-image-segmentation/sam2 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/sam2-image-segmentation/sam2 ERROR: Package 'sam-2' requires a different Python: 3.8.10 not in '>=3.10.0' Note: you may need to restart the kernel to use updated packages. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/sam2-image-segmentation + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/sam2-image-segmentation .. code:: ipython3 diff --git a/docs/notebooks/siglip-zero-shot-image-classification-with-output.rst b/docs/notebooks/siglip-zero-shot-image-classification-with-output.rst index 298a30bf7b6d7a..50aa71af0082b8 100644 --- a/docs/notebooks/siglip-zero-shot-image-classification-with-output.rst +++ b/docs/notebooks/siglip-zero-shot-image-classification-with-output.rst @@ -126,10 +126,10 @@ tokenizer and preparing the images. .. parsed-literal:: - 2024-10-23 04:42:12.224179: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 04:42:12.258186: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 04:45:37.694278: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 04:45:37.728953: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 04:42:12.920282: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 04:45:38.384577: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Run PyTorch model inference @@ -265,8 +265,9 @@ object ready to load on the device and start making predictions. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4779: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. Run OpenVINO model @@ -610,7 +611,7 @@ model are similar to the PyTorch model. .. parsed-literal:: - [{'dog': 0.99}, {'horse': 0.0}, {'cat': 0.0}, {'wolf': 0.0}, {'frog': 0.0}] + [{'dog': 0.99}, {'horse': 0.0}, {'cat': 0.0}, {'wolf': 0.0}, {'tiger': 0.0}] @@ -685,7 +686,7 @@ approximately estimate the speed up of the dynamic quantized models. .. parsed-literal:: - Performance speed up: 2.019 + Performance speed up: 2.491 Interactive inference diff --git a/docs/notebooks/siglip-zero-shot-image-classification-with-output_files/siglip-zero-shot-image-classification-with-output_24_1.png b/docs/notebooks/siglip-zero-shot-image-classification-with-output_files/siglip-zero-shot-image-classification-with-output_24_1.png index 76d6a6178cf64e..90392474870369 100644 --- a/docs/notebooks/siglip-zero-shot-image-classification-with-output_files/siglip-zero-shot-image-classification-with-output_24_1.png +++ b/docs/notebooks/siglip-zero-shot-image-classification-with-output_files/siglip-zero-shot-image-classification-with-output_24_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2f3af06f927b6f5e643d37869bbe32651d9a0f23270d5f3426cebbc2f02765cb -size 580998 +oid sha256:1e64834a97582f6009d2c6f15fd109cba0bdc0d9577d3a7f0f31a4d9f4519d44 +size 581000 diff --git a/docs/notebooks/speculative-sampling-with-output.rst b/docs/notebooks/speculative-sampling-with-output.rst index 127f61ab35085c..a88201eb73453c 100644 --- a/docs/notebooks/speculative-sampling-with-output.rst +++ b/docs/notebooks/speculative-sampling-with-output.rst @@ -1,5 +1,5 @@ -Text Generation via Speculative Sampling, KV Caching, and OpenVINO™ -=================================================================== +Text Generation via Speculative Decoding and OpenVINO™ +====================================================== As model sizes grow, Generative AI implementations require significant inference resources. This not only increases the cost per generation @@ -18,42 +18,49 @@ summary, inference optimizations for text generation are essential to reduce costs and power consumption, while also improving the accuracy and speed of text generation. -Another necessary condition is that the optimizations are compatible -with each other. That is, implementing a certain optimization should not -preclude other optimizations. There are several levels of optimizations -that can provide significant speedup without “bumping into each other” -in a way that will compromise overall efficiency. +Speculative decoding (or +`assisted-generation `__) +is a recent technique, that allows to speed up token generation when an +additional smaller draft model is used alongside with the main model. -For details on this method, please refer to the paper by Chen et al, -http://arxiv.org/abs/2302.01318. Additionally, there’s an interesting -proof of correctness of speculative sampling (showing that the original -distribution is preserved) by Leviathan et al, -http://arxiv.org/abs/2211.17192 +Speculative decoding works the following way. The draft model predicts +the next K tokens one by one in an autoregressive manner, while the main +model validates these predictions and corrects them if necessary. We go +through each predicted token, and if a difference is detected between +the draft and main model, we stop and keep the last token predicted by +the main model. Then the draft model gets the latest main prediction and +again tries to predict the next K tokens, repeating the cycle. -Our blog article describing this implementation with OpenVino is -available at openvino.ai +This approach reduces the need for multiple infer requests to the main +model, enhancing performance. For instance, in more predictable parts of +text generation, the draft model can, in best-case scenarios, generate +the next K tokens that exactly match the target. In that case they are +validated in a single inference request to the main model (which is +bigger, more accurate but slower) instead of running K subsequent +requests. More details can be found in the original +`paper `__. + +|image0| + +In this tutorial we consider how to apply Speculative decoding using +OpenVINO GenAI. **Table of contents:** - `Prerequisites <#prerequisites>`__ +- `Prepare models <#prepare-models>`__ - `Select inference device <#select-inference-device>`__ -- `Create autoregressive and speculative forms of sampling with KV - Cache - support <#create-autoregressive-and-speculative-forms-of-sampling-with-kv-cache-support>`__ - - - `Setup imports <#setup-imports>`__ - - `Prepare autoregressive - sampling <#prepare-autoregressive-sampling>`__ - - `Prepare speculative sampling <#prepare-speculative-sampling>`__ - -- `Main generation function <#main-generation-function>`__ +- `Run target model without speculative + decoding <#run-target-model-without-speculative-decoding>`__ +- `Run Speculative decoding + pipeline <#run-speculative-decoding-pipeline>`__ - - `Download and Convert Model <#download-and-convert-model>`__ ### - Installation Instructions +Installation Instructions +~~~~~~~~~~~~~~~~~~~~~~~~~ This is a self-contained example that relies solely on its own code. @@ -62,29 +69,64 @@ need a Jupyter server to start. For details, please refer to `Installation Guide `__. +.. |image0| image:: https://github.com/user-attachments/assets/eb999dea-d98b-42bb-835e-28d3054e1a84 + Prerequisites ------------- -First, we should install the `Hugging Face -Optimum `__ library -accelerated by OpenVINO integration. The Hugging Face Optimum Intel API -is a high-level API that enables us to convert and quantize models from -the Hugging Face Transformers library to the OpenVINO™ IR format. For -more details, refer to the `Hugging Face Optimum Intel -documentation `__. +First, we should install the `OpenVINO +GenAI `__ for running +model inference. + +|image01| + +OpenVINO™ GenAI is a library of the most popular Generative AI model +pipelines, optimized execution methods, and samples that run on top of +highly performant `OpenVINO +Runtime `__. + +This library is friendly to PC and laptop execution, and optimized for +resource consumption. It requires no external dependencies to run +generative models as it already includes all the core functionality +(e.g. tokenization via openvino-tokenizers). -We will also need to install transformers (HuggingFace) and some other -useful modules. +.. |image01| image:: https://media.githubusercontent.com/media/openvinotoolkit/openvino.genai/refs/heads/master/src/docs/openvino_genai.svg .. code:: ipython3 - %pip install -Uq pip - %pip uninstall -q -y optimum optimum-intel - %pip install --pre -Uq "openvino>=2024.2.0" openvino-tokenizers[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly - %pip install -q --upgrade transformers "torch>=2.1" "torchvision" "gradio>=4.19" accelerate "onnx<1.16.2" ipywidgets --extra-index-url https://download.pytorch.org/whl/cpu - %pip install -q "git+https://github.com/huggingface/optimum-intel.git" + %pip install --pre -Uq "openvino>=2024.4.0" openvino-tokenizers openvino-genai huggingface_hub --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + +Prepare models +-------------- + + + +As example, we will use already converted LLMs from `OpenVINO +collection `__, +but in case, if you want run own models, you should convert them using +`Hugging Face +Optimum `__ +library accelerated by OpenVINO integration. More details about model +preparation can be found in `OpenVINO LLM inference +guide `__ + +.. code:: ipython3 + + from pathlib import Path + import huggingface_hub as hf_hub + + draft_model_id = "OpenVINO/dolly-v2-3b-int4-ov" + target_model_id = "OpenVINO/dolly-v2-7b-int8-ov" + + draft_model_path = Path(draft_model_id.split("/")[-1]) + target_model_path = Path(target_model_id.split("/")[-1]) + + if not draft_model_path.exists(): + hf_hub.snapshot_download(draft_model_id, local_dir=draft_model_path) + if not target_model_path.exists(): + hf_hub.snapshot_download(target_model_id, local_dir=target_model_path) Select inference device ~~~~~~~~~~~~~~~~~~~~~~~ @@ -97,336 +139,184 @@ OpenVINO. .. code:: ipython3 import requests - + r = requests.get( url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", ) open("notebook_utils.py", "w").write(r.text) - + from notebook_utils import device_widget - - device = device_widget() - - device + device = device_widget(default="CPU", exclude=["NPU", "AUTO"]) + device -.. parsed-literal:: - Dropdown(description='Device:', options=('CPU', 'GPU.0', 'GPU.1', 'AUTO'), value='CPU') +.. parsed-literal:: + Dropdown(description='Device:', options=('CPU',), value='CPU') -Create autoregressive and speculative forms of sampling with KV Cache support ------------------------------------------------------------------------------ +Run target model without speculative decoding +--------------------------------------------- -Text generation is often done in an autoregressive fashion. We will all -support a KV cache (aka Past Value Cache) in the code. Note that we are -using greedy sampling. We do not adjust other text generation parameters -(e.g. temperature) so keep this illustration of speculative sampling as -simple and understandable as possible. -Setup imports -~~~~~~~~~~~~~ +OpenVINO GenAI provides easy-to-use API for running text generation. +Firstly we will create pipeline with ``LLMPipeline``. ``LLMPipeline`` is +the main object used for decoding. You can construct it straight away +from the folder with the converted model. It will automatically load the +``main model``, ``tokenizer``, ``detokenizer`` and default +``generation configuration``. After that we will configure parameters +for decoding. Then we just run ``generate`` method and get the output in +text format. We do not need to encode input prompt according to model +expected template or write post-processing code for logits decoder, it +will be done easily with LLMPipeline. +To obtain intermediate generation results without waiting until when +generation is finished, we will write streamer function. .. code:: ipython3 + import openvino_genai import time - import numpy as np - import openvino as ov -Prepare autoregressive sampling -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + pipe = openvino_genai.LLMPipeline(target_model_path, device.value) + config = openvino_genai.GenerationConfig() + config.max_new_tokens = 100 -.. code:: ipython3 + def streamer(subword): + print(subword, end="", flush=True) + # Return flag corresponds whether generation should be stopped. + # False means continue generation. + return False - def autoregressive_sampling_with_pkv(input, model, N=30): - input_ids, attention_mask = input.input_ids, input.attention_mask - seq_len = input_ids.shape[-1] - position_ids = np.arange(0, seq_len, dtype=np.int64).reshape([-1, seq_len]) - - # in all subsequent inferences we feed tokens one by one, - # but for the first one we feed the whole encoded prompt - request = model.create_infer_request() - request.infer((input_ids, attention_mask, position_ids, np.array([0]))) - next_token = np.argmax(request.results["logits"][:, -1]).reshape([1]) - - all_tokens = [] - all_tokens.extend(input_ids[0]) - all_tokens.append(next_token[0]) - - while seq_len < N: - input_ids = next_token.reshape([1, 1]) - attention_mask = np.concatenate((attention_mask, np.array([1]).reshape([1, 1])), axis=1) - position_ids = np.array([attention_mask.shape[1]]).reshape([1, 1]) - - request.infer((input_ids, attention_mask, position_ids, np.array([0]))) - next_token = np.argmax(request.results["logits"][:, -1]) - all_tokens.append(next_token) - seq_len += 1 - - return all_tokens - -Prepare speculative sampling -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - -- Step 1: With speculative sampling, we first generate K samples from - the draft model (in an autoregressive manner). -- Step 2: These are now candidates to examine using the main model - (step 2) using a batch size of K. -- Step 3: We go through each K predicted tokens, and if tokens differ, - we stop and keep the last token predicted by the main model. -- Step 4: We update KV-cache dropping keys & values for differing - tokens and repeat Step 1. -.. code:: ipython3 + start_time = time.perf_counter() + pipe.generate(["Sun is yellow because"], config, streamer=streamer) + end_time = time.perf_counter() - def update_state(request, seq_len): - for state in request.query_state(): - old_seq_len = state.state.shape[2] - if seq_len >= old_seq_len: - continue - # After the inference request, key/values have shape [BATCH_SIZE, seq_len + K, vocab_size]. - # Increment the sequence length by the number of matched tokens, and - # trim the KV cache to match the new sequence length. - state.state = ov.Tensor(state.state.data[:, :, :seq_len]) - - - def speculative_sampling_with_pkv(input, draft_model, main_model, K, N=30, **kwargs): - input_ids, attention_mask = input.input_ids, input.attention_mask - # seq_len number of key/values or number of already processed input tokens - seq_len = input_ids.shape[-1] - position_ids = np.arange(0, seq_len, dtype=np.int64).reshape([-1, seq_len]) - - draft_request = draft_model.create_infer_request() - draft_request.infer((input_ids, attention_mask, position_ids, np.array([0]))) - - main_request = main_model.create_infer_request() - main_request.infer((input_ids, attention_mask, position_ids, np.array([0]))) - first_token = np.argmax(main_request.results["logits"][:, -1]).reshape([1]) - - all_tokens = [] - all_tokens.extend(input_ids[0]) - all_tokens.append(first_token[0]) - - accum_draft_tokens = [] - while seq_len < N: - next_token = first_token - for i in range(K): - input_ids = next_token.reshape([1, 1]) - attention_mask = np.concatenate((attention_mask, np.array([1]).reshape([1, 1])), axis=1) - position_ids = np.array([attention_mask.shape[1]]).reshape([1, 1]) - - draft_request.infer((input_ids, attention_mask, position_ids, np.array([0]))) - next_token = np.argmax(draft_request.results["logits"][:, -1]) - accum_draft_tokens.append(next_token) - - # main model will give also K out tokens - # feed the same first token to the main model and do not give the last token generated by the draft - input_ids = np.concatenate((first_token.reshape([1]), accum_draft_tokens[:-1])).reshape([1, -1]) - attention_mask = np.ones((1, seq_len + K)) - position_ids = np.arange(seq_len, seq_len + K, dtype=np.int64).reshape([1, -1]) - - main_request.infer((input_ids, attention_mask, position_ids, np.array([0]))) - next_tokens = np.argmax(main_request.results["logits"], axis=-1)[0] - - # if disagrees from the very beggining then context will be expanded only for one element - # all elements match then context will be expanded to K elements - for disagree_idx, (t1, t2) in enumerate(zip(accum_draft_tokens, next_tokens)): - if t1 != t2: - break - - first_token = next_tokens[disagree_idx] - all_tokens.extend(next_tokens[: disagree_idx + 1]) - seq_len += disagree_idx + 1 - - # cut key/values depending on the position where disagreement starts - update_state(draft_request, seq_len) - update_state(main_request, seq_len) - - attention_mask = np.ones((1, seq_len)) - accum_draft_tokens = [] - all_tokens.extend(accum_draft_tokens) - return all_tokens - -Main generation function ------------------------- - - - -Download and Convert Model -~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - -Optimum Intel can be used to load optimized models from the `Hugging -Face Hub `__ and -create pipelines to run an inference with OpenVINO Runtime using Hugging -Face APIs. For speculative decoding we need to manually update states, -therefore we will use directly openvino inference api, and optimum only -for model conversion. >To download Llama-2-7b-chat-hf, you will need to -accept license agreement. You must be a registered user in Hugging -Face Hub. Please visit HuggingFace model -`card `__, -carefully read terms of usage and click accept button. You will need to -use an access token for the code below to run. For more information on -access tokens, refer to this section of the documentation. -.. code:: ipython3 +.. parsed-literal:: - from pathlib import Path - - main_model_id = "meta-llama/Llama-2-7b-chat-hf" - main_model_path = Path("Llama-2-7b-chat-hf") - draft_model_id = "TinyLlama/TinyLlama-1.1B-Chat-v1.0" - draft_model_path = Path("TinyLlama-1.1B-Chat-v1.0") - - from transformers import AutoTokenizer - - main_tokenizer = AutoTokenizer.from_pretrained(main_model_id) - draft_tokenizer = AutoTokenizer.from_pretrained(draft_model_id) + it is made of gas. The gas is heated to a high temperature and then cooled. The gas is yellow because it has a band of light called the "Bondeson Pendulum Effect." The Bondeson Pendulum Effect is caused by the light waves bouncing off of the gas molecules. The light waves bounce off of the gas molecules in different ways, some of the light waves get scattered, and some of the light waves get reflected. The light waves that get scattered and reflected combine to .. code:: ipython3 - # In order for speculative sampling to work, both main and draft tokenizers should be the same. - token_test_txt = "text to ensure tokenizers work the same, as of 2024" - tokens_1 = draft_tokenizer(token_test_txt, return_tensors="pt").input_ids - tokens_2 = main_tokenizer(token_test_txt, return_tensors="pt").input_ids - - assert all((tokens_1 - tokens_2)[0] == 0) + import gc -.. code:: ipython3 + print(f"Generation time: {end_time - start_time:.2f}s") + del pipe + gc.collect(); - if not main_model_path.exists(): - !optimum-cli export openvino --model $main_model_id --weight-format fp16 {main_model_path} - if not draft_model_path.exists(): - !optimum-cli export openvino --model $draft_model_id --weight-format fp16 {draft_model_path} -Infer directly using OpenVINO Inference Pipeline +.. parsed-literal:: -.. code:: ipython3 + Generation time: 18.44s - core = ov.Core() - draft_ov_model = core.read_model(draft_model_path / "openvino_model.xml") - draft_model = core.compile_model(draft_ov_model, device_name=device.value) - - main_ov_model = core.read_model(main_model_path / "openvino_model.xml") - main_model = core.compile_model(main_ov_model, device_name=device.value) -.. code:: ipython3 +Run Speculative decoding pipeline +--------------------------------- - def main( - prompt: str, - n_tokens_to_generate: int = 75, - K: int = 5, - seed: int = 5555, - ): - # seed numpy rng - np.random.seed(seed) - tokenized = main_tokenizer(prompt, return_tensors="pt") - - def run_autoregressive_sampling_fn(decode_fn, tokenized, **kwargs): - start = time.perf_counter() - output_ids = decode_fn(tokenized, **kwargs) - text = main_tokenizer.decode(output_ids, skip_special_tokens=True) - elapsed_time = time.perf_counter() - start - return text, elapsed_time - - def run_speculative_sampling_fn(decode_fn, input_ids, **kwargs): - start = time.perf_counter() - output_ids = decode_fn(input_ids, **kwargs) - text = main_tokenizer.decode(output_ids, skip_special_tokens=True) - elapsed_time = time.perf_counter() - start - return text, elapsed_time - - autoregressive_text, autoregressive_time = run_autoregressive_sampling_fn( - autoregressive_sampling_with_pkv, - tokenized, - model=main_model, - N=n_tokens_to_generate, - ) - - speculative_text, speculative_time = run_speculative_sampling_fn( - speculative_sampling_with_pkv, - tokenized, - main_model=main_model, - draft_model=draft_model, - N=n_tokens_to_generate, - K=K, - ) - - # Format results for output in gradio - out = "\n" + "Autoregressive Decode" + "\n" + "---------------------" + "\n" - out = out + f"Time = {autoregressive_time:.2f}s" + "\n" + f"Text = {autoregressive_text}" + "\n" - out = out + "\n" + "Speculative Decode" + "\n" + "------------------" + "\n" - out = out + f"Time = {speculative_time:.2f}s" + "\n" + f"Text = {speculative_text}" - return out + + +To enable Speculative decoding in ``LLMPipeline,`` we should +additionally provide the ``draft_model`` structure and +``SchedulerConfig`` for resource management. + +|image011| + +As shown in the figure above, speculative decoding works by splitting +the generative process into two stages. In the first stage, a fast, but +less accurate draft model (AKA assistant) autoregressively generates a +sequence of tokens. In the second stage, a large, but more accurate +target model conducts parallelized verification over the generated draft +tokens. This process allows the target model to produce multiple tokens +in a single forward pass and thus accelerate autoregressive decoding. +The success of speculative decoding largely hinges on the speculation +lookahead (SL), i.e. the number of tokens produced by the draft model in +each iteration. The straightforward method, based on `Leviathan et +al. `__, uses a static value of the +speculation lookahead and involves generating a constant number of +candidate tokens at each speculative iteration. You can adjust the +number of candidates using ``num_assistant_tokens`` parameter in +generation config. If the assistant model’s confidence in its prediction +for the current token is lower than this threshold, the assistant model +stops the current token generation iteration is not yet reached. + +.. |image011| image:: https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/dynamic_speculation_lookahead/spec_dec_diagram.png .. code:: ipython3 - res = main("Alan Turing was a", n_tokens_to_generate=100) - print(res) + scheduler_config = openvino_genai.SchedulerConfig() + # cache params + scheduler_config.cache_size = 2 + scheduler_config.block_size = 16 if "GPU" in device.value else 32 + + draft_model = openvino_genai.draft_model(draft_model_path, device.value) + + pipe = openvino_genai.LLMPipeline(target_model_path, device.value, draft_model=draft_model, scheduler_config=scheduler_config) + + config = openvino_genai.GenerationConfig() + config.max_new_tokens = 100 + config.num_assistant_tokens = 5 + start_time = time.perf_counter() + result = pipe.generate(["Sun is yellow because"], config, streamer=streamer) + end_time = time.perf_counter() .. parsed-literal:: - 2024-04-17 10:21:41.642283: I tensorflow/core/util/port.cc:111] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-04-17 10:21:41.644834: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used. - 2024-04-17 10:21:41.677055: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered - 2024-04-17 10:21:41.677093: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered - 2024-04-17 10:21:41.677119: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered - 2024-04-17 10:21:41.683198: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used. - 2024-04-17 10:21:41.683977: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-04-17 10:21:42.477656: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + it is made of gas. The gas is heated to a high temperature and then cooled. The gas changes from a hot gas to a cold gas and then from a cold gas to a hot gas. The gas is very hot when it changes from a hot gas to a cold gas and very cold when it changes from a cold gas to a hot gas. When the gas changes from a hot gas to a cold gas it becomes yellow. When the gas changes from a cold gas to a hot gas it + +.. code:: ipython3 + + print(f"Generation time: {end_time - start_time:.2f}s") .. parsed-literal:: - - Autoregressive Decode - --------------------- - Time = 44.39s - Text = Alan Turing was a British mathematician, computer scientist, and codebreaker who played a pivotal role in cracking the German Enigma code during World War II. He was also a pioneer in the field of artificial intelligence and made significant contributions to the development of computer science. - - Turing was born on June 23, 1912, in London, England. He was educated at Cambridge University, where he earned a degree in mathematics in - - Speculative Decode - ------------------ - Time = 22.96s - Text = Alan Turing was a British mathematician, computer scientist, and codebreaker who played a pivotal role in cracking the German Enigma code during World War II. He was also a pioneer in the field of artificial intelligence and made significant contributions to the development of computer science. - - Turing was born on June 23, 1912, in London, England. He was educated at Cambridge University, where he earned a degree in mathematics in 1 + Generation time: 15.62s +Alternative approach, Dynamic Speculative Decoding, described in the +`paper `__ is based on heuristics and +adjusts the number of candidate tokens for the next iteration based on +the acceptance rate of the current iteration. If all speculative tokens +are correct, the number of candidate tokens increases; otherwise, it +decreases. For adjusting number of tokens +``assistant_confidence_threshold`` parameters should be used. If the +assistant model’s confidence in its prediction for the current token is +lower than this threshold, the assistant model stops the current token +generation iteration, even if the number of ``num_assistant_tokens`` is +not yet reached. You can find more details in this `blog +post `__. + .. code:: ipython3 - if not Path("gradio_helper.py").exists(): - r = requests.get(url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/speculative-sampling/gradio_helper.py") - open("gradio_helper.py", "w").write(r.text) - - from gradio_helper import make_demo - - demo = make_demo(fn=main) - - try: - demo.launch(debug=False) - except Exception: - demo.launch(share=True, debug=False) - # If you are launching remotely, specify server_name and server_port - # EXAMPLE: `demo.launch(server_name='your server name', server_port='server port in int')` - # To learn more please refer to the Gradio docs: https://gradio.app/docs/ + config = openvino_genai.GenerationConfig() + config.max_new_tokens = 100 + config.assistant_confidence_threshold = 0.05 + start_time = time.perf_counter() + result = pipe.generate(["Sun is yellow because"], config, streamer) + end_time = time.perf_counter() + + +.. parsed-literal:: + + it is made of gas. The gas is heated to a high temperature and then cooled. The gas changes from a hot gas to a cold gas and then from a cold gas to a hot gas. The gas is very hot when it changes from a hot gas to a cold gas and very cold when it changes from a cold gas to a hot gas. The gas is very light and can float in the air. When the gas cools it becomes a liquid. The Sun is a huge sphere of .. code:: ipython3 - # please uncomment and run this cell for stopping gradio interface - # demo.close() + print(f"Generation time: {end_time - start_time:.2f}s") + + +.. parsed-literal:: + + Generation time: 17.97s + diff --git a/docs/notebooks/speech-recognition-quantization-wav2vec2-with-output.rst b/docs/notebooks/speech-recognition-quantization-wav2vec2-with-output.rst index be35e6d00f9293..b4a9cf92289306 100644 --- a/docs/notebooks/speech-recognition-quantization-wav2vec2-with-output.rst +++ b/docs/notebooks/speech-recognition-quantization-wav2vec2-with-output.rst @@ -57,47 +57,47 @@ Guide =0.11.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (1.5.0) - Requirement already satisfied: torch>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.4.1+cpu) - Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (3.16.1) - Requirement already satisfied: numpy>=1.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (1.24.4) - Requirement already satisfied: pyarrow>=15.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (17.0.0) - Requirement already satisfied: dill<0.3.9,>=0.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (0.3.8) - Requirement already satisfied: pandas in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (2.0.3) - Requirement already satisfied: requests>=2.32.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (2.32.3) - Requirement already satisfied: tqdm>=4.66.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (4.66.5) - Requirement already satisfied: xxhash in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (3.5.0) - Requirement already satisfied: multiprocess<0.70.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (0.70.16) - Requirement already satisfied: fsspec<=2024.9.0,>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets) (2024.9.0) - Requirement already satisfied: aiohttp in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (3.10.10) - Requirement already satisfied: huggingface-hub>=0.23.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (0.26.1) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (24.1) - Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (6.0.2) - Requirement already satisfied: lightning-utilities>=0.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchmetrics>=0.11.0) (0.11.8) - Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchmetrics>=0.11.0) (4.12.2) - Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1.0) (1.13.3) - Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1.0) (3.1) - Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1.0) (3.1.4) - Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (2.4.3) - Requirement already satisfied: aiosignal>=1.1.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (1.3.1) - Requirement already satisfied: attrs>=17.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (24.2.0) - Requirement already satisfied: frozenlist>=1.1.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (1.4.1) - Requirement already satisfied: multidict<7.0,>=4.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (6.1.0) - Requirement already satisfied: yarl<2.0,>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (1.15.2) - Requirement already satisfied: async-timeout<5.0,>=4.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (4.0.3) - Requirement already satisfied: setuptools in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from lightning-utilities>=0.8.0->torchmetrics>=0.11.0) (75.2.0) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (3.4.0) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (3.10) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (2.2.3) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (2024.8.30) - Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch>=2.1.0) (2.1.5) - Requirement already satisfied: python-dateutil>=2.8.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pandas->datasets) (2.9.0.post0) - Requirement already satisfied: pytz>=2020.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pandas->datasets) (2024.2) - Requirement already satisfied: tzdata>=2022.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pandas->datasets) (2024.2) - Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch>=2.1.0) (1.3.0) - Requirement already satisfied: six>=1.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0) - Requirement already satisfied: propcache>=0.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from yarl<2.0,>=1.12.0->aiohttp->datasets) (0.2.0) + Requirement already satisfied: datasets in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (3.1.0) + Requirement already satisfied: torchmetrics>=0.11.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (1.5.1) + Requirement already satisfied: torch>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.4.1+cpu) + Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (3.16.1) + Requirement already satisfied: numpy>=1.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (1.24.4) + Requirement already satisfied: pyarrow>=15.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (17.0.0) + Requirement already satisfied: dill<0.3.9,>=0.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (0.3.8) + Requirement already satisfied: pandas in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (2.0.3) + Requirement already satisfied: requests>=2.32.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (2.32.3) + Requirement already satisfied: tqdm>=4.66.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (4.66.6) + Requirement already satisfied: xxhash in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (3.5.0) + Requirement already satisfied: multiprocess<0.70.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (0.70.16) + Requirement already satisfied: fsspec<=2024.9.0,>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets) (2024.9.0) + Requirement already satisfied: aiohttp in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (3.10.10) + Requirement already satisfied: huggingface-hub>=0.23.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (0.26.2) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (24.1) + Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (6.0.2) + Requirement already satisfied: lightning-utilities>=0.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchmetrics>=0.11.0) (0.11.8) + Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchmetrics>=0.11.0) (4.12.2) + Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1.0) (1.13.3) + Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1.0) (3.1) + Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1.0) (3.1.4) + Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (2.4.3) + Requirement already satisfied: aiosignal>=1.1.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (1.3.1) + Requirement already satisfied: attrs>=17.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (24.2.0) + Requirement already satisfied: frozenlist>=1.1.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (1.5.0) + Requirement already satisfied: multidict<7.0,>=4.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (6.1.0) + Requirement already satisfied: yarl<2.0,>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (1.15.2) + Requirement already satisfied: async-timeout<5.0,>=4.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (4.0.3) + Requirement already satisfied: setuptools in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from lightning-utilities>=0.8.0->torchmetrics>=0.11.0) (75.3.0) + Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (3.4.0) + Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (3.10) + Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (2.2.3) + Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (2024.8.30) + Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch>=2.1.0) (2.1.5) + Requirement already satisfied: python-dateutil>=2.8.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pandas->datasets) (2.9.0.post0) + Requirement already satisfied: pytz>=2020.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pandas->datasets) (2024.2) + Requirement already satisfied: tzdata>=2022.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pandas->datasets) (2024.2) + Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch>=2.1.0) (1.3.0) + Requirement already satisfied: six>=1.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0) + Requirement already satisfied: propcache>=0.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from yarl<2.0,>=1.12.0->aiohttp->datasets) (0.2.0) Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -116,6 +116,15 @@ Imports from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor + +.. parsed-literal:: + + 2024-11-05 04:52:12.108210: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 04:52:12.142263: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2024-11-05 04:52:12.803683: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + + Settings -------- @@ -163,9 +172,16 @@ IR). .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4779: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. + + +.. parsed-literal:: + + [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py:871: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py:872: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_output.size() != (bsz, self.num_heads, tgt_len, self.head_dim): @@ -272,14 +288,6 @@ steps: INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino -.. parsed-literal:: - - 2024-10-23 04:48:23.521146: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 04:48:23.553631: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 04:48:24.182318: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - - .. parsed-literal:: @@ -500,7 +508,7 @@ quantized model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:62: FutureWarning: Importing `WordErrorRate` from `torchmetrics` was deprecated and will be removed in 2.0. Import `WordErrorRate` from `torchmetrics.text` instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:62: FutureWarning: Importing `WordErrorRate` from `torchmetrics` was deprecated and will be removed in 2.0. Import `WordErrorRate` from `torchmetrics.text` instead. _future_warning( @@ -570,23 +578,23 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 17.90 ms + [ INFO ] Read model took 18.07 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: - [ INFO ] input_values , 45 (node: input_values) : f32 / [...] / [?,?] + [ INFO ] 45 , input_values (node: input_values) : f32 / [...] / [?,?] [ INFO ] Model outputs: [ INFO ] logits (node: __module.lm_head/aten::linear/Add) : f32 / [...] / [?,1..,32] [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: '45': [1,30480] - [ INFO ] Reshape model took 4.29 ms + [ INFO ] Reshape model took 4.26 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: - [ INFO ] input_values , 45 (node: input_values) : f32 / [...] / [1,30480] + [ INFO ] 45 , input_values (node: input_values) : f32 / [...] / [1,30480] [ INFO ] Model outputs: [ INFO ] logits (node: __module.lm_head/aten::linear/Add) : f32 / [...] / [1,95,32] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 493.63 ms + [ INFO ] Compile model took 495.05 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -623,17 +631,17 @@ models. [ INFO ] Fill input '45' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 69.32 ms + [ INFO ] First inference took 69.60 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 5400 iterations - [ INFO ] Duration: 120141.91 ms + [ INFO ] Count: 5406 iterations + [ INFO ] Duration: 120228.72 ms [ INFO ] Latency: - [ INFO ] Median: 131.56 ms - [ INFO ] Average: 133.32 ms - [ INFO ] Min: 88.10 ms - [ INFO ] Max: 314.58 ms - [ INFO ] Throughput: 44.95 FPS + [ INFO ] Median: 131.74 ms + [ INFO ] Average: 133.26 ms + [ INFO ] Min: 107.29 ms + [ INFO ] Max: 340.52 ms + [ INFO ] Throughput: 44.96 FPS .. code:: ipython3 @@ -660,7 +668,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 24.51 ms + [ INFO ] Read model took 23.99 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] 45 , input_values (node: input_values) : f32 / [...] / [?,?] @@ -669,14 +677,14 @@ models. [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: '45': [1,30480] - [ INFO ] Reshape model took 5.99 ms + [ INFO ] Reshape model took 6.04 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: [ INFO ] 45 , input_values (node: input_values) : f32 / [...] / [1,30480] [ INFO ] Model outputs: [ INFO ] logits (node: __module.lm_head/aten::linear/Add) : f32 / [...] / [1,95,32] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 1234.19 ms + [ INFO ] Compile model took 1203.24 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -713,15 +721,15 @@ models. [ INFO ] Fill input '45' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 56.91 ms + [ INFO ] First inference took 53.37 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 7974 iterations - [ INFO ] Duration: 120125.59 ms + [ INFO ] Count: 8004 iterations + [ INFO ] Duration: 120086.07 ms [ INFO ] Latency: - [ INFO ] Median: 88.71 ms - [ INFO ] Average: 90.24 ms - [ INFO ] Min: 68.79 ms - [ INFO ] Max: 212.19 ms - [ INFO ] Throughput: 66.38 FPS + [ INFO ] Median: 88.68 ms + [ INFO ] Average: 89.86 ms + [ INFO ] Min: 68.86 ms + [ INFO ] Max: 235.63 ms + [ INFO ] Throughput: 66.65 FPS diff --git a/docs/notebooks/speechbrain-emotion-recognition-with-output.rst b/docs/notebooks/speechbrain-emotion-recognition-with-output.rst index 55e3b31765b191..42783b7d598b01 100644 --- a/docs/notebooks/speechbrain-emotion-recognition-with-output.rst +++ b/docs/notebooks/speechbrain-emotion-recognition-with-output.rst @@ -53,25 +53,6 @@ Installations %pip install -q "transformers>=4.30.0" "huggingface_hub>=0.8.0" "SoundFile" %pip install -q "openvino>=2024.1.0" - -.. parsed-literal:: - - Note: you may need to restart the kernel to use updated packages. - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - altair 5.4.1 requires typing-extensions>=4.10.0; python_version < "3.13", but you have typing-extensions 4.9.0 which is incompatible. - descript-audiotools 0.7.2 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.3 which is incompatible. - detectron2 0.6 requires iopath<0.1.10,>=0.1.7, but you have iopath 0.1.10 which is incompatible. - mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.19.1+cpu which is incompatible. - modelscope-studio 0.5.0 requires gradio<5.0,>=4.0, but you have gradio 3.43.1 which is incompatible. - openvino-dev 2024.4.0 requires openvino==2024.4.0, but you have openvino 2024.5.0.dev20241014 which is incompatible. - parler-tts 0.2 requires transformers<=4.43.3,>=4.43.0, but you have transformers 4.45.2 which is incompatible. - tensorflow 2.12.0 requires numpy<1.24,>=1.22, but you have numpy 1.24.4 which is incompatible. - typeguard 4.3.0 requires typing-extensions>=4.10.0, but you have typing-extensions 4.9.0 which is incompatible. - Note: you may need to restart the kernel to use updated packages. - Note: you may need to restart the kernel to use updated packages. - Note: you may need to restart the kernel to use updated packages. - - Imports ~~~~~~~ @@ -85,6 +66,12 @@ Imports import openvino as ov + +.. parsed-literal:: + + torchvision is not available - cannot save figures + + Prepare base model ~~~~~~~~~~~~~~~~~~ @@ -116,42 +103,6 @@ SpeechBrain codebase. source="speechbrain/emotion-recognition-wav2vec2-IEMOCAP", pymodule_file="custom_interface.py", classname="CustomEncoderWav2vec2Classifier" ) - - -.. parsed-literal:: - - config.json: 0%| | 0.00/1.84k [00:00=2.2" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "stable-audio-tools" "nncf>=2.12.0" --extra-index-url https://download.pytorch.org/whl/cpu - %pip install -q "openvino>=2024.3.0" + if platform.system() == "Darwin": + %pip install -q "numpy>=1.26,<2.0.0" "pandas>2.0.2" + else: + %pip install -q "numpy>=1.26" "pandas>2.0.2" + %pip install -q "openvino>=2024.4.0" Load the original model and inference ------------------------------------- @@ -108,15 +114,28 @@ Load the original model and inference model, model_config = get_pretrained_model("stabilityai/stable-audio-open-1.0") + +.. parsed-literal:: + + model_config.json: 0%| | 0.00/4.17k [00:00=t0 but got ta=0.29999998211860657 and t0=0.3. + /home/ea/work/py311/lib/python3.11/site-packages/torchsde/_brownian/brownian_interval.py:599: UserWarning: Should have ta>=t0 but got ta=0.29999998211860657 and t0=0.3. warnings.warn(f"Should have ta>=t0 but got ta={ta} and t0={self._start}.") @@ -210,7 +235,7 @@ Load the original model and inference @@ -287,7 +312,7 @@ documentation =t0 but got ta=0.29999998211860657 and t0=0.3. + /home/ea/work/py311/lib/python3.11/site-packages/torchsde/_brownian/brownian_interval.py:599: UserWarning: Should have ta>=t0 but got ta=0.29999998211860657 and t0=0.3. warnings.warn(f"Should have ta>=t0 but got ta={ta} and t0={self._start}.") @@ -679,7 +685,7 @@ and run inference. @@ -736,8 +742,3 @@ Interactive inference # If you are launching remotely, specify server_name and server_port # EXAMPLE: `demo.launch(server_name='your server name', server_port='server port in int')` # To learn more please refer to the Gradio docs: https://gradio.app/docs/ - -.. code:: ipython3 - - # please uncomment and run this cell for stopping gradio interface - # demo.close() diff --git a/docs/notebooks/stable-cascade-image-generation-with-output.rst b/docs/notebooks/stable-cascade-image-generation-with-output.rst index 969ed4e4dbe336..8bd358958e47b6 100644 --- a/docs/notebooks/stable-cascade-image-generation-with-output.rst +++ b/docs/notebooks/stable-cascade-image-generation-with-output.rst @@ -56,13 +56,8 @@ Prerequisites .. parsed-literal:: ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - descript-audiotools 0.7.2 requires protobuf<3.20,>=3.9.2, but you have protobuf 5.28.3 which is incompatible. mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.19.1+cpu which is incompatible. - open-clip-torch 2.22.0 requires protobuf<4, but you have protobuf 5.28.3 which is incompatible. s3fs 2024.10.0 requires fsspec==2024.10.0.*, but you have fsspec 2024.9.0 which is incompatible. - tensorflow 2.12.0 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 5.28.3 which is incompatible. - tensorflow-metadata 1.14.0 requires protobuf<4.21,>=3.20.3, but you have protobuf 5.28.3 which is incompatible. - wandb 0.15.4 requires protobuf!=4.21.0,<5,>=3.12.0; python_version < "3.9" and sys_platform == "linux", but you have protobuf 5.28.3 which is incompatible. Note: you may need to restart the kernel to use updated packages. @@ -85,10 +80,10 @@ Load and run the original pipeline .. parsed-literal:: - 2024-10-23 04:56:18.757348: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 04:56:18.791430: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 05:00:13.157963: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 05:00:13.192172: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 04:56:19.346484: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 05:00:13.880563: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -257,11 +252,12 @@ here, we always use fixed shapes in conversion by using an .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4779: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:88: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:88: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input_shape[-1] > 1 or self.sliding_window is not None: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: @@ -310,7 +306,7 @@ here, we always use fixed shapes in conversion by using an .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_stable_cascade.py:548: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_stable_cascade.py:548: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if skip is not None and (x.size(-1) != skip.size(-1) or x.size(-2) != skip.size(-2)): diff --git a/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.jpg b/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.jpg index ff7c5a51fa8623..240c88c559da36 100644 --- a/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.jpg +++ b/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8ea7494f5ec5eee5a399e31abce1aafc5b2df1e446414a4c8ece593b465608e2 -size 81605 +oid sha256:8f0845e937cadada4b26550b58858292fd42eebd18367b454b1401143e263338 +size 85254 diff --git a/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.png b/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.png index 81e228f915393f..349c8fa79af81f 100644 --- a/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.png +++ b/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:33518ccf125cee029c73380add209aae63208485ac340655ac3123d0e19c0426 -size 1629985 +oid sha256:d44cddd44f25458a49a6bf4a70431bbd39eecdc67d015c2c88aafa644f7c8ff9 +size 1644717 diff --git a/docs/notebooks/stable-diffusion-ip-adapter-with-output.rst b/docs/notebooks/stable-diffusion-ip-adapter-with-output.rst index 9f84e3178267ff..15c490942998d6 100644 --- a/docs/notebooks/stable-diffusion-ip-adapter-with-output.rst +++ b/docs/notebooks/stable-diffusion-ip-adapter-with-output.rst @@ -75,8 +75,6 @@ Prerequisites ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. botocore 1.35.36 requires urllib3<1.27,>=1.25.4; python_version < "3.10", but you have urllib3 2.2.3 which is incompatible. - tensorflow 2.12.0 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 5.28.3 which is incompatible. - wandb 0.15.4 requires protobuf!=4.21.0,<5,>=3.12.0; python_version < "3.9" and sys_platform == "linux", but you have protobuf 5.28.3 which is incompatible. Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -197,10 +195,10 @@ Additionally, LCM requires using LCMScheduler for efficient generation. .. parsed-literal:: - 2024-10-23 05:07:50.575524: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 05:07:50.610025: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 05:11:37.931642: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 05:11:37.965522: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 05:07:51.160950: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 05:11:38.628517: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -211,7 +209,7 @@ Additionally, LCM requires using LCMScheduler for efficient generation. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/feature_extraction_clip.py:28: FutureWarning: The class CLIPFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please use CLIPImageProcessor instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/feature_extraction_clip.py:28: FutureWarning: The class CLIPFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please use CLIPImageProcessor instead. warnings.warn( @@ -293,8 +291,11 @@ extractor as input and returns image embeddings. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4779: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:243: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if not interpolate_pos_encoding and (height != self.image_size or width != self.image_size): U-net @@ -355,17 +356,17 @@ Model predicts the ``sample`` state for the next step. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_2d_condition.py:1111: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_2d_condition.py:1111: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if dim % default_overall_up_factor != 0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/embeddings.py:1801: FutureWarning: You have passed a tensor as `image_embeds`.This is deprecated and will be removed in a future release. Please make sure to update your script to pass `image_embeds` as a list of tensors to suppress this warning. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/embeddings.py:1801: FutureWarning: You have passed a tensor as `image_embeds`.This is deprecated and will be removed in a future release. Please make sure to update your script to pass `image_embeds` as a list of tensors to suppress this warning. deprecate("image_embeds not a list", "1.0.0", deprecation_message, standard_warn=False) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:136: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:136: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:145: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:145: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:147: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:147: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if hidden_states.shape[0] >= 64: @@ -443,16 +444,16 @@ image in pipeline, we can discuss it in inference examples. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes: - %2506 : Float(1, 4, 64, 64, strides=[16384, 4096, 64, 1], requires_grad=0, device=cpu) = aten::randn(%2500, %2501, %2502, %2503, %2504, %2505) # /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/torch_utils.py:81:0 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes: + %2506 : Float(1, 4, 64, 64, strides=[16384, 4096, 64, 1], requires_grad=0, device=cpu) = aten::randn(%2500, %2501, %2502, %2503, %2504, %2505) # /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/torch_utils.py:81:0 This may cause errors in trace checking. To disable trace checking, pass check_trace=False to torch.jit.trace() _check_trace( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: Tensor-likes are not close! - Mismatched elements: 10395 / 16384 (63.4%) - Greatest absolute difference: 0.0018606185913085938 at index (0, 1, 63, 63) (up to 1e-05 allowed) - Greatest relative difference: 0.005050937208143127 at index (0, 3, 63, 59) (up to 1e-05 allowed) + Mismatched elements: 10398 / 16384 (63.5%) + Greatest absolute difference: 0.0014954805374145508 at index (0, 1, 0, 63) (up to 1e-05 allowed) + Greatest relative difference: 0.004146566421115057 at index (0, 3, 63, 59) (up to 1e-05 allowed) _check_trace( @@ -498,9 +499,9 @@ hidden states. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:88: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:88: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input_shape[-1] > 1 or self.sliding_window is not None: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: diff --git a/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_22_1.png b/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_22_1.png index 9e41d1a02b25ee..b202db52895198 100644 --- a/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_22_1.png +++ b/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_22_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0d5fa26a0d01a01557d2baf61229065adafcf06e97d796f49cad83cfeef8c42e -size 964520 +oid sha256:f9454f039912f132f1c278b4d58fddc6b737c8a6d56ddbc5358142ad1e9f987c +size 980237 diff --git a/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_25_0.png b/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_25_0.png index 45f0dc6e8e9e10..299ec38e04f457 100644 --- a/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_25_0.png +++ b/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_25_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3b0f7041fd5c84ecde36f1036e957a5f64eb12507270cd300392042c8a703f6d -size 961384 +oid sha256:8e9b89ad55b5255e86b4a7e7d40984fa3e8ca7d64dc5cff8ca1367dcc0de5a72 +size 927436 diff --git a/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_28_0.png b/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_28_0.png index 8b9cea0390f80a..9f1e4c4d220755 100644 --- a/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_28_0.png +++ b/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_28_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e434bcc6a4403efc3f9272a49882d42ea354dae2de6231c7d71458ea79fd6de9 -size 593289 +oid sha256:46120a84ef8334f29241e07bd43c8313977cd2318ce36a951f1f8ed3c626ba84 +size 598424 diff --git a/docs/notebooks/stable-diffusion-torchdynamo-backend-with-output.rst b/docs/notebooks/stable-diffusion-torchdynamo-backend-with-output.rst index e03a4ab614c769..151a5a5c30588d 100644 --- a/docs/notebooks/stable-diffusion-torchdynamo-backend-with-output.rst +++ b/docs/notebooks/stable-diffusion-torchdynamo-backend-with-output.rst @@ -117,8 +117,7 @@ torch.compile it goes through the following steps: 1. Graph acquisition - the model is rewritten as blocks of subgraphs that are either: - compiled by TorchDynamo and “flattened”, - - falling back to the eager-mode, due to unsupported Python constructs (like control-flow - code). + - falling back to the eager-mode, due to unsupported Python constructs (like control-flow code). 2. Graph lowering - all PyTorch operations are decomposed into their constituent kernels specific to the chosen backend. diff --git a/docs/notebooks/style-transfer-with-output.rst b/docs/notebooks/style-transfer-with-output.rst index dbbe6395574657..9b902aeb79f7a7 100644 --- a/docs/notebooks/style-transfer-with-output.rst +++ b/docs/notebooks/style-transfer-with-output.rst @@ -193,7 +193,7 @@ OpenVINO Intermediate Representation (IR) with ``FP16`` precision. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/style-transfer-webcam/model/mosaic-9.onnx') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/style-transfer-webcam/model/mosaic-9.onnx') diff --git a/docs/notebooks/style-transfer-with-output_files/style-transfer-with-output_25_0.png b/docs/notebooks/style-transfer-with-output_files/style-transfer-with-output_25_0.png index d9cfe3e6360a7a..69ebcfdd60f395 100644 --- a/docs/notebooks/style-transfer-with-output_files/style-transfer-with-output_25_0.png +++ b/docs/notebooks/style-transfer-with-output_files/style-transfer-with-output_25_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0565650551a7561fb5450ec915c747ffad759c8cba258eec1d1cd2d30f85c812 -size 81718 +oid sha256:0aa8c4fc4dd201212c4b63e6436c2a5e9cd6f7994271244bfabe91a5cef7e675 +size 81862 diff --git a/docs/notebooks/table-question-answering-with-output.rst b/docs/notebooks/table-question-answering-with-output.rst index 88edfa22333b86..e1cad24a567f4b 100644 --- a/docs/notebooks/table-question-answering-with-output.rst +++ b/docs/notebooks/table-question-answering-with-output.rst @@ -80,10 +80,10 @@ Prerequisites .. parsed-literal:: - 2024-10-23 05:11:15.997414: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 05:11:16.031769: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 05:15:04.432298: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 05:15:04.467903: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 05:11:16.587915: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 05:15:05.158075: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Use ``TapasForQuestionAnswering.from_pretrained`` to download a @@ -219,57 +219,58 @@ function to serialize the result of conversion. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4779: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1571: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1571: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. self.indices = torch.as_tensor(indices) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1572: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1572: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. self.num_segments = torch.as_tensor(num_segments, device=indices.device) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1674: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1674: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. batch_size = torch.prod(torch.tensor(list(index.batch_shape()))) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1750: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1750: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. [torch.as_tensor([-1], dtype=torch.long), torch.as_tensor(vector_shape, dtype=torch.long)], dim=0 - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1753: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1753: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! flat_values = values.reshape(flattened_shape.tolist()) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1755: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1755: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! out = torch.zeros(int(flat_index.num_segments), dtype=torch.float, device=flat_values.device) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1763: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1763: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor(index.batch_shape(), dtype=torch.long), - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1764: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1764: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor([index.num_segments], dtype=torch.long), - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1765: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1765: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor(vector_shape, dtype=torch.long), - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1770: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1770: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! output_values = segment_means.clone().view(new_shape.tolist()).to(values.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1701: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1701: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. batch_shape = torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1705: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1705: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. num_segments = torch.as_tensor(num_segments) # create a rank 0 tensor (scalar) containing num_segments (e.g. 64) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1716: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1716: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! new_shape = [int(x) for x in new_tensor.tolist()] - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1719: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1719: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. multiples = torch.cat([batch_shape, torch.as_tensor([1])], dim=0) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1720: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1720: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! indices = indices.repeat(multiples.tolist()) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:282: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:282: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor(self.config.max_position_embeddings - 1, device=device), position - first_position - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1231: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1231: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. indices=torch.min(row_ids, torch.as_tensor(self.config.max_num_rows - 1, device=row_ids.device)), - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1236: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1236: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. indices=torch.min(column_ids, torch.as_tensor(self.config.max_num_columns - 1, device=column_ids.device)), - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1928: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1928: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. column_logits += CLOSE_ENOUGH_TO_LOG_ZERO * torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1933: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1933: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. column_logits += CLOSE_ENOUGH_TO_LOG_ZERO * torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1969: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1969: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. labels_per_column, _ = reduce_sum(torch.as_tensor(labels, dtype=torch.float32, device=labels.device), col_index) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1992: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1992: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor(labels, dtype=torch.long, device=labels.device), cell_index - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1999: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1999: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. column_mask = torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2024: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2024: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. selected_column_id = torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2029: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2029: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. selected_column_mask = torch.as_tensor( diff --git a/docs/notebooks/tensorflow-classification-to-openvino-with-output.rst b/docs/notebooks/tensorflow-classification-to-openvino-with-output.rst index b2efda5f0aa31a..45eb4a96573be1 100644 --- a/docs/notebooks/tensorflow-classification-to-openvino-with-output.rst +++ b/docs/notebooks/tensorflow-classification-to-openvino-with-output.rst @@ -338,5 +338,5 @@ performance. .. parsed-literal:: - IR model in OpenVINO Runtime/CPU: 0.0010 seconds per image, FPS: 995.15 + IR model in OpenVINO Runtime/CPU: 0.0010 seconds per image, FPS: 959.64 diff --git a/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output.rst b/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output.rst index 0c0357fa707f9c..2410d1657c9b1c 100644 --- a/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output.rst +++ b/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output.rst @@ -391,7 +391,7 @@ Read the image, resize and convert it to the input shape of the network: .. parsed-literal:: - + diff --git a/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output_files/tensorflow-instance-segmentation-to-openvino-with-output_39_0.png b/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output_files/tensorflow-instance-segmentation-to-openvino-with-output_39_0.png index d91e735a1d6467..4940e220329fbf 100644 --- a/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output_files/tensorflow-instance-segmentation-to-openvino-with-output_39_0.png +++ b/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output_files/tensorflow-instance-segmentation-to-openvino-with-output_39_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8492fcb14f954229b35321f809f600bc9c9fb445cf44cf1cab1dbae5473e2d78 -size 391367 +oid sha256:69ea6c54eefea173c120ff5666caa829b5a008727ebd3a83f7e10d1986de07bd +size 394182 diff --git a/docs/notebooks/tensorflow-object-detection-to-openvino-with-output.rst b/docs/notebooks/tensorflow-object-detection-to-openvino-with-output.rst index 970ff5dd5278ae..11dc1675dcfeb4 100644 --- a/docs/notebooks/tensorflow-object-detection-to-openvino-with-output.rst +++ b/docs/notebooks/tensorflow-object-detection-to-openvino-with-output.rst @@ -189,7 +189,7 @@ from TensorFlow Hub: .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-object-detection-to-openvino/od-model/tf/faster_rcnn_resnet50_v1_640x640.tar.gz') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-object-detection-to-openvino/od-model/tf/faster_rcnn_resnet50_v1_640x640.tar.gz') @@ -368,7 +368,7 @@ Load and save an image: .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-object-detection-to-openvino/data/coco_bike.jpg') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-object-detection-to-openvino/data/coco_bike.jpg') @@ -396,7 +396,7 @@ Read the image, resize and convert it to the input shape of the network: .. parsed-literal:: - + @@ -697,7 +697,7 @@ Zoo `__: .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-object-detection-to-openvino/data/coco_91cl.txt') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-object-detection-to-openvino/data/coco_91cl.txt') diff --git a/docs/notebooks/tensorflow-object-detection-to-openvino-with-output_files/tensorflow-object-detection-to-openvino-with-output_38_0.png b/docs/notebooks/tensorflow-object-detection-to-openvino-with-output_files/tensorflow-object-detection-to-openvino-with-output_38_0.png index e3868b416ee4e5..41c96a6eeb7d0e 100644 --- a/docs/notebooks/tensorflow-object-detection-to-openvino-with-output_files/tensorflow-object-detection-to-openvino-with-output_38_0.png +++ b/docs/notebooks/tensorflow-object-detection-to-openvino-with-output_files/tensorflow-object-detection-to-openvino-with-output_38_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dfdccf9cf18ab90165dbca452027d6afdd51d2f355b8cdc45d7431f9e88c5782 -size 392207 +oid sha256:63da618f6999e82c3f9e461cb912450186fbb00bc247ccd311a0dfd0bbc825b3 +size 390893 diff --git a/docs/notebooks/tensorflow-quantization-aware-training-with-output.rst b/docs/notebooks/tensorflow-quantization-aware-training-with-output.rst index 8fe9423b3dcc53..4a4285ca97cc9a 100644 --- a/docs/notebooks/tensorflow-quantization-aware-training-with-output.rst +++ b/docs/notebooks/tensorflow-quantization-aware-training-with-output.rst @@ -127,10 +127,10 @@ models will be stored. .. parsed-literal:: - 2024-10-23 05:14:10.399928: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 05:14:10.434579: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 05:17:59.724629: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 05:17:59.759583: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 05:14:11.041876: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 05:18:00.367471: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -140,7 +140,7 @@ models will be stored. Downloading data from https://storage.openvinotoolkit.org/repositories/nncf/openvino_notebook_ckpts/305_resnet18_imagenette_fp32_v1.h5 134604992/134604992 [==============================] - 2s 0us/step Absolute path where the model weights are saved: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-quantization-aware-training/model/ResNet-18_fp32.h5 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-quantization-aware-training/model/ResNet-18_fp32.h5 Dataset Preprocessing @@ -175,13 +175,13 @@ Download and prepare Imagenette 160px dataset. .. parsed-literal:: - 2024-10-23 05:14:16.300917: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. + 2024-11-05 05:18:05.378970: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. Skipping registering GPU devices... - 2024-10-23 05:14:16.433616: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_2' with dtype string and shape [1] - [[{{node Placeholder/_2}}]] - 2024-10-23 05:14:16.434383: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_0' with dtype string and shape [1] - [[{{node Placeholder/_0}}]] - 2024-10-23 05:14:16.493317: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. + 2024-11-05 05:18:05.496251: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_3' with dtype int64 and shape [1] + [[{{node Placeholder/_3}}]] + 2024-11-05 05:18:05.496608: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_1' with dtype string and shape [1] + [[{{node Placeholder/_1}}]] + 2024-11-05 05:18:05.549189: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. @@ -325,15 +325,15 @@ model and a training pipeline. .. parsed-literal:: - 2024-10-23 05:14:17.636634: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_2' with dtype string and shape [1] - [[{{node Placeholder/_2}}]] - 2024-10-23 05:14:17.637423: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_3' with dtype int64 and shape [1] - [[{{node Placeholder/_3}}]] + 2024-11-05 05:18:06.706690: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_4' with dtype int64 and shape [1] + [[{{node Placeholder/_4}}]] + 2024-11-05 05:18:06.707393: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_0' with dtype string and shape [1] + [[{{node Placeholder/_0}}]] .. parsed-literal:: - 4/4 [==============================] - 1s 250ms/sample - loss: 0.9807 - acc@1: 0.8220 + 4/4 [==============================] - 1s 241ms/sample - loss: 0.9807 - acc@1: 0.8220 Accuracy of FP32 model: 0.822 @@ -379,13 +379,13 @@ scenario and requires only 3 modifications. .. parsed-literal:: - 2024-10-23 05:14:20.291889: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_2' with dtype string and shape [1] + 2024-11-05 05:18:09.325779: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_2' with dtype string and shape [1] [[{{node Placeholder/_2}}]] - 2024-10-23 05:14:20.292274: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_1' with dtype string and shape [1] - [[{{node Placeholder/_1}}]] - 2024-10-23 05:14:21.242391: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. - 2024-10-23 05:14:21.895899: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. - 2024-10-23 05:14:30.320766: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. + 2024-11-05 05:18:09.326170: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_0' with dtype string and shape [1] + [[{{node Placeholder/_0}}]] + 2024-11-05 05:18:10.310403: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. + 2024-11-05 05:18:10.931815: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. + 2024-11-05 05:18:19.063128: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. Evaluate the new model on the validation set after initialization of @@ -411,7 +411,7 @@ demonstrated here. .. parsed-literal:: - 4/4 [==============================] - 1s 306ms/sample - loss: 0.9766 - acc@1: 0.8120 + 4/4 [==============================] - 1s 305ms/sample - loss: 0.9766 - acc@1: 0.8120 Fine-tune the Compressed Model @@ -446,10 +446,10 @@ training pipeline are required. Here is a simple example. Accuracy of INT8 model after initialization: 0.812 Epoch 1/2 - 101/101 [==============================] - 49s 418ms/step - loss: 0.7134 - acc@1: 0.9299 + 101/101 [==============================] - 49s 416ms/step - loss: 0.7134 - acc@1: 0.9299 Epoch 2/2 - 101/101 [==============================] - 42s 415ms/step - loss: 0.6807 - acc@1: 0.9489 - 4/4 [==============================] - 1s 144ms/sample - loss: 0.9760 - acc@1: 0.8160 + 101/101 [==============================] - 42s 414ms/step - loss: 0.6807 - acc@1: 0.9489 + 4/4 [==============================] - 1s 141ms/sample - loss: 0.9760 - acc@1: 0.8160 Accuracy of INT8 model after fine-tuning: 0.816 @@ -559,10 +559,10 @@ Please select a benchmarking device using the dropdown list: .. parsed-literal:: Benchmark FP32 model (IR) - [ INFO ] Throughput: 2786.08 FPS + [ INFO ] Throughput: 2758.24 FPS Benchmark INT8 model (IR) - [ INFO ] Throughput: 10924.61 FPS + [ INFO ] Throughput: 11317.45 FPS Show Device Information for reference. diff --git a/docs/notebooks/tflite-selfie-segmentation-with-output.rst b/docs/notebooks/tflite-selfie-segmentation-with-output.rst index 9acc434aa435f6..cc8a203e4f15a9 100644 --- a/docs/notebooks/tflite-selfie-segmentation-with-output.rst +++ b/docs/notebooks/tflite-selfie-segmentation-with-output.rst @@ -143,7 +143,7 @@ Download pretrained model and test image .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/tflite-selfie-segmentation/selfie_multiclass_256x256.tflite') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/tflite-selfie-segmentation/selfie_multiclass_256x256.tflite') diff --git a/docs/notebooks/tflite-selfie-segmentation-with-output_files/tflite-selfie-segmentation-with-output_33_0.png b/docs/notebooks/tflite-selfie-segmentation-with-output_files/tflite-selfie-segmentation-with-output_33_0.png index d0b2b849bf51c2..882a9d712dd7b0 100644 --- a/docs/notebooks/tflite-selfie-segmentation-with-output_files/tflite-selfie-segmentation-with-output_33_0.png +++ b/docs/notebooks/tflite-selfie-segmentation-with-output_files/tflite-selfie-segmentation-with-output_33_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e2963c9b4c7dacdca6b1bbe9ea3ef832a19ba962c0a9980363722083e66b7358 -size 14233 +oid sha256:2cda2d6f5f72e6e7c24898ead882b6b26b696d1d1c61ff589d08fd55d98357ef +size 14403 diff --git a/docs/notebooks/tflite-to-openvino-with-output.rst b/docs/notebooks/tflite-to-openvino-with-output.rst index dcd7a98897f334..a70e10853c0dc7 100644 --- a/docs/notebooks/tflite-to-openvino-with-output.rst +++ b/docs/notebooks/tflite-to-openvino-with-output.rst @@ -274,7 +274,7 @@ GPU. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 9.48 ms + [ INFO ] Read model took 10.03 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] images (node: images) : f32 / [...] / [1,224,224,3] @@ -288,7 +288,7 @@ GPU. [ INFO ] Model outputs: [ INFO ] Softmax (node: 61) : f32 / [...] / [1,1000] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 157.24 ms + [ INFO ] Compile model took 165.61 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: TensorFlow_Lite_Frontend_IR @@ -325,15 +325,15 @@ GPU. [ INFO ] Fill input 'images' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 7.14 ms + [ INFO ] First inference took 6.93 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 17436 iterations - [ INFO ] Duration: 15008.41 ms + [ INFO ] Count: 14982 iterations + [ INFO ] Duration: 15006.90 ms [ INFO ] Latency: - [ INFO ] Median: 4.99 ms - [ INFO ] Average: 5.03 ms - [ INFO ] Min: 2.84 ms - [ INFO ] Max: 24.60 ms - [ INFO ] Throughput: 1161.75 FPS + [ INFO ] Median: 5.47 ms + [ INFO ] Average: 5.91 ms + [ INFO ] Min: 3.02 ms + [ INFO ] Max: 34.80 ms + [ INFO ] Throughput: 998.34 FPS diff --git a/docs/notebooks/typo-detector-with-output.rst b/docs/notebooks/typo-detector-with-output.rst index 5c03dc870f8a69..94431f153ae72d 100644 --- a/docs/notebooks/typo-detector-with-output.rst +++ b/docs/notebooks/typo-detector-with-output.rst @@ -108,10 +108,10 @@ Imports .. parsed-literal:: - 2024-10-23 05:18:31.129817: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 05:18:31.163652: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 05:23:42.228206: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 05:23:42.262729: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 05:18:31.823748: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 05:23:42.928779: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Methods @@ -255,8 +255,7 @@ your model. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. - op1 = operator(\*args, \*\*kwargs) + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. Load the tokenizer @@ -372,7 +371,7 @@ Let’s run a demo using the Hugging Face Optimum API. [Input]: I have been stuying for my math exam all week, but I'm stil not very confidet that I will pass it, because there are so many formuals to remeber. [Detected]: I have been stuying for my math exam all week, but I'm stil not very confidet that I will pass it, because there are so many formuals to remeber. ---------------------------------------------------------------------------------------------------------------------------------- - Time elapsed: 0.14796948432922363 + Time elapsed: 0.14881014823913574 2. Converting the model to OpenVINO IR @@ -618,5 +617,5 @@ Let’s run a demo using the converted OpenVINO IR model. [Input]: I have been stuying for my math exam all week, but I'm stil not very confidet that I will pass it, because there are so many formuals to remeber. [Detected]: I have been stuying for my math exam all week, but I'm stil not very confidet that I will pass it, because there are so many formuals to remeber. ---------------------------------------------------------------------------------------------------------------------------------- - Time elapsed: 0.10040116310119629 + Time elapsed: 0.10274600982666016 diff --git a/docs/notebooks/vision-background-removal-with-output.rst b/docs/notebooks/vision-background-removal-with-output.rst index 63e02d9c787aaf..4e79038a213347 100644 --- a/docs/notebooks/vision-background-removal-with-output.rst +++ b/docs/notebooks/vision-background-removal-with-output.rst @@ -195,7 +195,7 @@ next cell loads the model and the pre-trained weights. Downloading... From: https://drive.google.com/uc?id=1W8E4FHIlTVstfRkYmNOjbr0VDXTZm0jD To: <_io.BufferedWriter name='model/u2net_lite/u2net_lite.pth'> - 100%|██████████| 4.68M/4.68M [00:00<00:00, 34.0MB/s] + 100%|██████████| 4.68M/4.68M [00:00<00:00, 33.7MB/s] .. parsed-literal:: @@ -225,7 +225,7 @@ next cell loads the model and the pre-trained weights. .. parsed-literal:: - /tmp/ipykernel_2664614/1036642300.py:7: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /tmp/ipykernel_586189/1036642300.py:7: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. net.load_state_dict(state_dict=torch.load(model_path, map_location="cpu")) @@ -252,7 +252,7 @@ OpenVINO IR format. Executing the following command may take a while. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/vision-background-removal/model/u2net.py:23: UserWarning: `nn.functional.upsample` is deprecated. Use `nn.functional.interpolate` instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/vision-background-removal/model/u2net.py:23: UserWarning: `nn.functional.upsample` is deprecated. Use `nn.functional.interpolate` instead. src = F.upsample(src,size=tar.shape[2:],mode='bilinear') @@ -341,7 +341,7 @@ Load the OpenVINO IR model to OpenVINO Runtime and do inference. .. parsed-literal:: - Inference finished. Inference time: 0.107 seconds, FPS: 9.31. + Inference finished. Inference time: 0.106 seconds, FPS: 9.42. Visualize Results diff --git a/docs/notebooks/vision-monodepth-with-output.rst b/docs/notebooks/vision-monodepth-with-output.rst index ea81314fe49df7..dfb20643e991e7 100644 --- a/docs/notebooks/vision-monodepth-with-output.rst +++ b/docs/notebooks/vision-monodepth-with-output.rst @@ -309,7 +309,7 @@ original image shape. .. parsed-literal:: - /tmp/ipykernel_2664839/2076527990.py:15: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead. + /tmp/ipykernel_586416/2076527990.py:15: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead. cmap = matplotlib.cm.get_cmap(colormap) @@ -508,7 +508,7 @@ Do Inference on a Video and Create Monodepth Video .. parsed-literal:: - Processed 60 frames in 24.30 seconds. Total FPS (including video processing): 2.47.Inference FPS: 46.34 + Processed 60 frames in 25.55 seconds. Total FPS (including video processing): 2.35.Inference FPS: 48.61 Monodepth Video saved to 'output/Coco%20Walking%20in%20Berkeley_monodepth.mp4'. @@ -535,7 +535,7 @@ Display Monodepth Video .. parsed-literal:: Showing monodepth video saved at - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/vision-monodepth/output/Coco%20Walking%20in%20Berkeley_monodepth.mp4 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/vision-monodepth/output/Coco%20Walking%20in%20Berkeley_monodepth.mp4 If you cannot see the video in your browser, please click on the following link to download the video diff --git a/docs/notebooks/wav2lip-with-output.rst b/docs/notebooks/wav2lip-with-output.rst index 3f6b510cf00374..c70015d2614d8d 100644 --- a/docs/notebooks/wav2lip-with-output.rst +++ b/docs/notebooks/wav2lip-with-output.rst @@ -140,7 +140,7 @@ Download example files. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/data_video_sun_5s.mp4') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/data_video_sun_5s.mp4') @@ -177,7 +177,7 @@ and will convert both model in OpenVINO format. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/ov_wav2lip_helper.py:43: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/ov_wav2lip_helper.py:43: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. model_weights = torch.load(path_to_detector) @@ -200,7 +200,7 @@ and will convert both model in OpenVINO format. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/ov_wav2lip_helper.py:16: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/ov_wav2lip_helper.py:16: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage) @@ -263,7 +263,7 @@ python API and converted OpenVINO models. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/Wav2Lip/audio.py:100: FutureWarning: Pass sr=16000, n_fft=800 as keyword args. From version 0.10 passing these as positional arguments will result in an error + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/Wav2Lip/audio.py:100: FutureWarning: Pass sr=16000, n_fft=800 as keyword args. From version 0.10 passing these as positional arguments will result in an error return librosa.filters.mel(hp.sample_rate, hp.n_fft, n_mels=hp.num_mels, @@ -286,14 +286,14 @@ python API and converted OpenVINO models. 0%| | 0/8 [00:00 #0:0 (mpeg4 (native) -> h264 (libx264)) Stream #0:0 -> #0:1 (pcm_s16le (native) -> aac (native)) Press [q] to stop, [?] for help - [libx264 @ 0x56537dc66840] -qscale is ignored, -crf is recommended. - [libx264 @ 0x56537dc66840] using SAR=1/1 - [libx264 @ 0x56537dc66840] using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX FMA3 BMI2 AVX2 AVX512 - [libx264 @ 0x56537dc66840] profile High, level 3.1 - [libx264 @ 0x56537dc66840] 264 - core 155 r2917 0a84d98 - H.264/MPEG-4 AVC codec - Copyleft 2003-2018 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=24 lookahead_threads=4 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=250 keyint_min=25 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=23.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00 + [libx264 @ 0x55edc3fbd840] -qscale is ignored, -crf is recommended. + [libx264 @ 0x55edc3fbd840] using SAR=1/1 + [libx264 @ 0x55edc3fbd840] using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX FMA3 BMI2 AVX2 AVX512 + [libx264 @ 0x55edc3fbd840] profile High, level 3.1 + [libx264 @ 0x55edc3fbd840] 264 - core 155 r2917 0a84d98 - H.264/MPEG-4 AVC codec - Copyleft 2003-2018 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=24 lookahead_threads=4 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=250 keyint_min=25 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=23.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00 Output #0, mp4, to 'results/result_voice.mp4': Metadata: encoder : Lavf58.29.100 @@ -349,25 +349,25 @@ python API and converted OpenVINO models. encoder : Lavc58.54.100 aac frame= 123 fps=0.0 q=-1.0 Lsize= 621kB time=00:00:05.06 bitrate=1005.8kbits/s speed=10.8x video:573kB audio:43kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.827166% - [libx264 @ 0x56537dc66840] frame I:1 Avg QP:22.24 size: 31028 - [libx264 @ 0x56537dc66840] frame P:75 Avg QP:22.01 size: 6954 - [libx264 @ 0x56537dc66840] frame B:47 Avg QP:25.58 size: 718 - [libx264 @ 0x56537dc66840] consecutive B-frames: 38.2% 27.6% 14.6% 19.5% - [libx264 @ 0x56537dc66840] mb I I16..4: 14.0% 83.9% 2.1% - [libx264 @ 0x56537dc66840] mb P I16..4: 1.3% 3.3% 0.1% P16..4: 37.8% 8.2% 6.4% 0.0% 0.0% skip:43.0% - [libx264 @ 0x56537dc66840] mb B I16..4: 0.2% 0.7% 0.0% B16..8: 27.9% 0.4% 0.1% direct: 0.2% skip:70.6% L0:43.9% L1:54.2% BI: 1.9% - [libx264 @ 0x56537dc66840] 8x8 transform intra:73.3% inter:77.1% - [libx264 @ 0x56537dc66840] coded y,uvDC,uvAC intra: 56.9% 72.4% 8.1% inter: 11.4% 13.0% 0.2% - [libx264 @ 0x56537dc66840] i16 v,h,dc,p: 20% 23% 9% 48% - [libx264 @ 0x56537dc66840] i8 v,h,dc,ddl,ddr,vr,hd,vl,hu: 25% 23% 36% 3% 3% 2% 2% 3% 3% - [libx264 @ 0x56537dc66840] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu: 39% 14% 14% 4% 6% 7% 4% 9% 3% - [libx264 @ 0x56537dc66840] i8c dc,h,v,p: 42% 25% 29% 4% - [libx264 @ 0x56537dc66840] Weighted P-Frames: Y:0.0% UV:0.0% - [libx264 @ 0x56537dc66840] ref P L0: 74.2% 10.4% 11.1% 4.3% - [libx264 @ 0x56537dc66840] ref B L0: 86.1% 11.2% 2.8% - [libx264 @ 0x56537dc66840] ref B L1: 98.3% 1.7% - [libx264 @ 0x56537dc66840] kb/s:953.36 - [aac @ 0x56537dc68140] Qavg: 121.673 + [libx264 @ 0x55edc3fbd840] frame I:1 Avg QP:22.24 size: 31028 + [libx264 @ 0x55edc3fbd840] frame P:75 Avg QP:22.01 size: 6954 + [libx264 @ 0x55edc3fbd840] frame B:47 Avg QP:25.58 size: 718 + [libx264 @ 0x55edc3fbd840] consecutive B-frames: 38.2% 27.6% 14.6% 19.5% + [libx264 @ 0x55edc3fbd840] mb I I16..4: 14.0% 83.9% 2.1% + [libx264 @ 0x55edc3fbd840] mb P I16..4: 1.3% 3.3% 0.1% P16..4: 37.8% 8.2% 6.4% 0.0% 0.0% skip:43.0% + [libx264 @ 0x55edc3fbd840] mb B I16..4: 0.2% 0.7% 0.0% B16..8: 27.9% 0.4% 0.1% direct: 0.2% skip:70.6% L0:43.9% L1:54.2% BI: 1.9% + [libx264 @ 0x55edc3fbd840] 8x8 transform intra:73.3% inter:77.1% + [libx264 @ 0x55edc3fbd840] coded y,uvDC,uvAC intra: 56.9% 72.4% 8.1% inter: 11.4% 13.0% 0.2% + [libx264 @ 0x55edc3fbd840] i16 v,h,dc,p: 20% 23% 9% 48% + [libx264 @ 0x55edc3fbd840] i8 v,h,dc,ddl,ddr,vr,hd,vl,hu: 25% 23% 36% 3% 3% 2% 2% 3% 3% + [libx264 @ 0x55edc3fbd840] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu: 39% 14% 14% 4% 6% 7% 4% 9% 3% + [libx264 @ 0x55edc3fbd840] i8c dc,h,v,p: 42% 25% 29% 4% + [libx264 @ 0x55edc3fbd840] Weighted P-Frames: Y:0.0% UV:0.0% + [libx264 @ 0x55edc3fbd840] ref P L0: 74.2% 10.4% 11.1% 4.3% + [libx264 @ 0x55edc3fbd840] ref B L0: 86.1% 11.2% 2.8% + [libx264 @ 0x55edc3fbd840] ref B L1: 98.3% 1.7% + [libx264 @ 0x55edc3fbd840] kb/s:953.36 + [aac @ 0x55edc3fbf140] Qavg: 121.673 diff --git a/docs/notebooks/whisper-asr-genai-with-output.rst b/docs/notebooks/whisper-asr-genai-with-output.rst index 45cc8291b8acc2..3f4339e01b3acc 100644 --- a/docs/notebooks/whisper-asr-genai-with-output.rst +++ b/docs/notebooks/whisper-asr-genai-with-output.rst @@ -75,8 +75,8 @@ Prerequisites .. code:: ipython3 - %pip install -q "transformers>=4.35" "torch>=2.3" "torchvision>=0.18.1" "onnx>=1.16.1" --extra-index-url https://download.pytorch.org/whl/cpu - %pip install -q "git+https://github.com/huggingface/optimum-intel.git" + %pip install -q "torch>=2.3" "torchvision>=0.18.1" --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q "transformers>=4.45" "git+https://github.com/huggingface/optimum-intel.git" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q --pre -U "openvino" "openvino-tokenizers" "openvino-genai" --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly %pip install -q datasets "gradio>=4.0" "soundfile>=0.12" "librosa" "python-ffmpeg<=1.0.16" %pip install -q "nncf>=2.13.0" "jiwer" @@ -91,6 +91,12 @@ Prerequisites url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", ) open("notebook_utils.py", "w").write(r.text) + + if not Path("cmd_helper.py").exists(): + r = requests.get( + url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/cmd_helper.py", + ) + open("cmd_helper.py", "w").write(r.text) Load PyTorch model ------------------ @@ -266,7 +272,7 @@ Let’s check how to work the ``transcribe`` task. .. parsed-literal:: - /home/labuser/work/notebook/genai_whisper/lib/python3.10/site-packages/transformers/models/whisper/generation_whisper.py:496: FutureWarning: The input name `inputs` is deprecated. Please make sure to use `input_features` instead. + /home/labuser/work/notebook/whisper_new/lib/python3.10/site-packages/transformers/models/whisper/generation_whisper.py:496: FutureWarning: The input name `inputs` is deprecated. Please make sure to use `input_features` instead. warnings.warn( @@ -349,8 +355,8 @@ be found in the `paper `__. Result: The blog is our tool that is prefilled to encourage collaboration and develop the learning of the students and to attract a normal school class. -Download and convert model to OpenVINO IR via Optimum Intel CLI ---------------------------------------------------------------- +Convert model to OpenVINO IR via Optimum Intel CLI +-------------------------------------------------- @@ -381,20 +387,13 @@ documentation `__. import logging import nncf - import os - from IPython.display import display, Markdown + from cmd_helper import optimum_cli nncf.set_log_level(logging.ERROR) model_path = Path(model_id.value.split("/")[1]) - export_command = f"optimum-cli export openvino --model {model_id.value} --library transformers --task automatic-speech-recognition-with-past --framework pt {str(model_path)}" - - display(Markdown("**Export command:**")) - display(Markdown(f"`{export_command}`")) - - exit_code = os.system(export_command) - if exit_code != 0: - raise Exception("Failed to load and convert model!") + optimum_cli(model_id.value, model_path) + print(f"✅ {model_id.value} model converted and can be found in {model_path}") Run inference OpenVINO model with WhisperPipeline ------------------------------------------------- @@ -604,9 +603,9 @@ Compare performance PyTorch vs OpenVINO .. parsed-literal:: - Mean torch openai/whisper-tiny generation time: 0.624s - Mean openvino openai/whisper-tiny generation time: 0.344s - Performance openai/whisper-tiny openvino speedup: 1.814 + Mean torch openai/whisper-tiny generation time: 0.564s + Mean openvino openai/whisper-tiny generation time: 0.311s + Performance openai/whisper-tiny openvino speedup: 1.815 Quantization @@ -934,7 +933,7 @@ for Word Error Rate. .. parsed-literal:: - Whole pipeline performance speedup: 1.499 + Whole pipeline performance speedup: 1.350 Whisper transcription word accuracy. Original model: 82.88%. Quantized model: 84.13%. Accuracy drop: -1.25%. @@ -960,7 +959,7 @@ upload button) or record using your microphone. pipe = ov_quantized_pipe if to_quantize.value else ov_pipe - gr_pipeline = GradioPipeline(pipe, multilingual=(not model_id.value.endswith(".en")), quantized=to_quantize.value) + gr_pipeline = GradioPipeline(pipe, model_id.value, quantized=to_quantize.value) demo = make_demo(gr_pipeline) diff --git a/docs/notebooks/whisper-subtitles-generation-with-output.rst b/docs/notebooks/whisper-subtitles-generation-with-output.rst index 8bfbfa8a6e86a3..b1e16ffa40c626 100644 --- a/docs/notebooks/whisper-subtitles-generation-with-output.rst +++ b/docs/notebooks/whisper-subtitles-generation-with-output.rst @@ -80,7 +80,7 @@ Install dependencies. %pip install -q "nncf>=2.13.0" %pip install -q --pre -U "openvino" "openvino-tokenizers" "openvino-genai" --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly - %pip install -q "python-ffmpeg<=1.0.16" "ffmpeg" "moviepy" "onnx!=1.16.2" "git+https://github.com/huggingface/optimum-intel.git" "torch>=2.1" --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q "python-ffmpeg<=1.0.16" "ffmpeg" "moviepy" "transformers>=4.45" "git+https://github.com/huggingface/optimum-intel.git" "torch>=2.1" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q -U "yt_dlp>=2024.8.6" soundfile librosa jiwer %pip install -q "gradio>=4.19" @@ -94,6 +94,12 @@ Install dependencies. url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", ) open("notebook_utils.py", "w").write(r.text) + + if not Path("cmd_helper.py").exists(): + r = requests.get( + url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/cmd_helper.py", + ) + open("cmd_helper.py", "w").write(r.text) Instantiate model ----------------- @@ -184,10 +190,12 @@ documentation `__. .. code:: ipython3 + from cmd_helper import optimum_cli + model_dir = model_id.value.split("/")[-1] if not Path(model_dir).exists(): - !optimum-cli export openvino -m {model_id.value} {model_dir} --weight-format fp16 + optimum_cli(model_id.value, model_dir) Prepare inference pipeline -------------------------- @@ -218,13 +226,6 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 - import requests - - r = requests.get( - url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", - ) - open("notebook_utils.py", "w").write(r.text) - from notebook_utils import device_widget device = device_widget(default="CPU", exclude=["NPU"]) diff --git a/docs/notebooks/wuerstchen-image-generation-with-output.rst b/docs/notebooks/wuerstchen-image-generation-with-output.rst index 342ddbafdc7edb..237bfb8b4bd943 100644 --- a/docs/notebooks/wuerstchen-image-generation-with-output.rst +++ b/docs/notebooks/wuerstchen-image-generation-with-output.rst @@ -229,7 +229,8 @@ parameter to generate a less memory-demanding model. Text encoder model has 2 inputs: -- ``input_ids``: vector of tokenized input sentence. Default tokenizer vector length is 77. +- ``input_ids``: vector of tokenized + input sentence. Default tokenizer vector length is 77. - ``attention_mask``: vector of same length as ``input_ids`` describing the attention mask. @@ -285,6 +286,7 @@ Decoder pipeline Decoder pipeline consists of 3 parts: decoder, text encoder and VQGAN. Decoder model is the WuerstchenDiffNeXt UNet decoder. Inputs are: + - ``x``: sample - ``r``: timestep - ``effnet``: interpolation block diff --git a/docs/notebooks/yolov11-instance-segmentation-with-output.rst b/docs/notebooks/yolov11-instance-segmentation-with-output.rst index 9d9b5e6bd15c0e..9cbca9fe4f1c37 100644 --- a/docs/notebooks/yolov11-instance-segmentation-with-output.rst +++ b/docs/notebooks/yolov11-instance-segmentation-with-output.rst @@ -143,7 +143,7 @@ Import required utility functions. The lower cell will download the .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg') @@ -216,14 +216,14 @@ Let us consider the examples: .. parsed-literal:: - 100%|██████████| 5.90M/5.90M [00:00<00:00, 25.1MB/s] + 100%|██████████| 5.90M/5.90M [00:00<00:00, 25.2MB/s] .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 480x640 3 bicycles, 2 cars, 1 motorcycle, 1 dog, 66.4ms - Speed: 1.8ms preprocess, 66.4ms inference, 2.8ms postprocess per image at shape (1, 3, 480, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 480x640 3 bicycles, 2 cars, 1 motorcycle, 1 dog, 69.6ms + Speed: 2.0ms preprocess, 69.6ms inference, 4.8ms postprocess per image at shape (1, 3, 480, 640) @@ -260,7 +260,7 @@ preserve dynamic shapes in the model. OpenVINO: export success ✅ 2.0s, saved as 'yolo11n-seg_openvino_model/' (6.0 MB) Export complete (2.2s) - Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization + Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization Predict: yolo predict task=segment model=yolo11n-seg_openvino_model imgsz=640 half Validate: yolo val task=segment model=yolo11n-seg_openvino_model imgsz=640 data=/ultralytics/ultralytics/cfg/datasets/coco.yaml half Visualize: https://netron.app @@ -345,8 +345,8 @@ Test on single image .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 3 bicycles, 2 cars, 1 dog, 23.2ms - Speed: 3.6ms preprocess, 23.2ms inference, 3.8ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 3 bicycles, 2 cars, 1 dog, 23.2ms + Speed: 1.8ms preprocess, 23.2ms inference, 3.8ms postprocess per image at shape (1, 3, 640, 640) @@ -668,8 +668,8 @@ on the image. .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 2 bicycles, 2 cars, 1 dog, 17.7ms - Speed: 2.1ms preprocess, 17.7ms inference, 3.8ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 2 bicycles, 2 cars, 1 dog, 11.2ms + Speed: 1.9ms preprocess, 11.2ms inference, 3.4ms postprocess per image at shape (1, 3, 640, 640) @@ -728,7 +728,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 20.05 ms + [ INFO ] Read model took 19.61 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [?,3,?,?] @@ -738,7 +738,7 @@ models. [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: 'x': [1,3,640,640] - [ INFO ] Reshape model took 8.82 ms + [ INFO ] Reshape model took 8.69 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: [ INFO ] x (node: x) : u8 / [N,C,H,W] / [1,3,640,640] @@ -746,7 +746,7 @@ models. [ INFO ] ***NO_NAME*** (node: __module.model.23/aten::cat/Concat_8) : f32 / [...] / [1,116,8400] [ INFO ] input.255 (node: __module.model.23.cv4.2.1.act/aten::silu_/Swish_46) : f32 / [...] / [1,32,160,160] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 387.38 ms + [ INFO ] Compile model took 390.55 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -783,17 +783,17 @@ models. [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 36.42 ms + [ INFO ] First inference took 36.99 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 1788 iterations - [ INFO ] Duration: 15050.22 ms + [ INFO ] Count: 1806 iterations + [ INFO ] Duration: 15074.73 ms [ INFO ] Latency: - [ INFO ] Median: 49.80 ms - [ INFO ] Average: 50.35 ms - [ INFO ] Min: 33.27 ms - [ INFO ] Max: 104.15 ms - [ INFO ] Throughput: 118.80 FPS + [ INFO ] Median: 49.77 ms + [ INFO ] Average: 49.90 ms + [ INFO ] Min: 24.69 ms + [ INFO ] Max: 66.18 ms + [ INFO ] Throughput: 119.80 FPS .. code:: ipython3 @@ -819,7 +819,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 29.13 ms + [ INFO ] Read model took 29.59 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [1,3,640,640] @@ -837,7 +837,7 @@ models. [ INFO ] ***NO_NAME*** (node: __module.model.23/aten::cat/Concat_8) : f32 / [...] / [1,116,8400] [ INFO ] input.255 (node: __module.model.23.cv4.2.1.act/aten::silu_/Swish_46) : f32 / [...] / [1,32,160,160] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 594.13 ms + [ INFO ] Compile model took 592.55 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -874,17 +874,17 @@ models. [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 27.63 ms + [ INFO ] First inference took 24.82 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 3714 iterations - [ INFO ] Duration: 15026.92 ms + [ INFO ] Count: 3702 iterations + [ INFO ] Duration: 15040.97 ms [ INFO ] Latency: - [ INFO ] Median: 23.95 ms - [ INFO ] Average: 24.14 ms - [ INFO ] Min: 17.70 ms - [ INFO ] Max: 39.05 ms - [ INFO ] Throughput: 247.16 FPS + [ INFO ] Median: 24.15 ms + [ INFO ] Average: 24.25 ms + [ INFO ] Min: 20.10 ms + [ INFO ] Max: 43.05 ms + [ INFO ] Throughput: 246.13 FPS Other ways to optimize model diff --git a/docs/notebooks/yolov11-instance-segmentation-with-output_files/yolov11-instance-segmentation-with-output_46_0.png b/docs/notebooks/yolov11-instance-segmentation-with-output_files/yolov11-instance-segmentation-with-output_46_0.png index 011e13859e8a3e..737d0af981d026 100644 --- a/docs/notebooks/yolov11-instance-segmentation-with-output_files/yolov11-instance-segmentation-with-output_46_0.png +++ b/docs/notebooks/yolov11-instance-segmentation-with-output_files/yolov11-instance-segmentation-with-output_46_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e393b8edd8a20af1aaab6b69932bf0e715bf4f9c35814dd766dfa74ae27f2ae4 -size 493271 +oid sha256:f7d6739facd4204cf94a8bb5d471ea56a2ce3eab4dcd1e3807ba40c8e43baa44 +size 502582 diff --git a/docs/notebooks/yolov11-keypoint-detection-with-output.rst b/docs/notebooks/yolov11-keypoint-detection-with-output.rst index 97c549e6751dd0..cf94af9ab1e794 100644 --- a/docs/notebooks/yolov11-keypoint-detection-with-output.rst +++ b/docs/notebooks/yolov11-keypoint-detection-with-output.rst @@ -143,7 +143,7 @@ Import required utility functions. The lower cell will download the .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg') @@ -214,14 +214,14 @@ Let us consider the examples: .. parsed-literal:: - 100%|██████████| 5.97M/5.97M [00:00<00:00, 25.3MB/s] + 100%|██████████| 5.97M/5.97M [00:00<00:00, 21.1MB/s] .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg: 480x640 1 person, 59.6ms - Speed: 2.1ms preprocess, 59.6ms inference, 0.9ms postprocess per image at shape (1, 3, 480, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg: 480x640 1 person, 62.6ms + Speed: 1.8ms preprocess, 62.6ms inference, 0.9ms postprocess per image at shape (1, 3, 480, 640) @@ -258,7 +258,7 @@ preserve dynamic shapes in the model. OpenVINO: export success ✅ 2.0s, saved as 'yolo11n-pose_openvino_model/' (6.0 MB) Export complete (2.1s) - Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization + Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization Predict: yolo predict task=pose model=yolo11n-pose_openvino_model imgsz=640 half Validate: yolo val task=pose model=yolo11n-pose_openvino_model imgsz=640 data=/ultralytics/ultralytics/cfg/datasets/coco-pose.yaml half Visualize: https://netron.app @@ -338,8 +338,8 @@ ready to check model prediction. Loading yolo11n-pose_openvino_model for OpenVINO inference... Using OpenVINO LATENCY mode for batch=1 inference... - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg: 640x640 1 person, 19.9ms - Speed: 2.3ms preprocess, 19.9ms inference, 0.9ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg: 640x640 1 person, 20.1ms + Speed: 2.8ms preprocess, 20.1ms inference, 1.0ms postprocess per image at shape (1, 3, 640, 640) @@ -686,8 +686,8 @@ on the image. Loading yolo11n-pose_openvino_model for OpenVINO inference... Using OpenVINO LATENCY mode for batch=1 inference... - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg: 640x640 1 person, 28.0ms - Speed: 2.0ms preprocess, 28.0ms inference, 1.1ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg: 640x640 1 person, 31.4ms + Speed: 2.1ms preprocess, 31.4ms inference, 1.0ms postprocess per image at shape (1, 3, 640, 640) @@ -748,7 +748,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 19.55 ms + [ INFO ] Read model took 19.20 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [?,3,?,?] @@ -764,7 +764,7 @@ models. [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.model.23/aten::cat/Concat_9) : f32 / [...] / [1,56,8400] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 329.35 ms + [ INFO ] Compile model took 338.50 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -801,17 +801,17 @@ models. [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 33.67 ms + [ INFO ] First inference took 33.80 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 17280 iterations - [ INFO ] Duration: 120049.75 ms + [ INFO ] Count: 17082 iterations + [ INFO ] Duration: 120040.01 ms [ INFO ] Latency: - [ INFO ] Median: 40.74 ms - [ INFO ] Average: 41.55 ms - [ INFO ] Min: 24.20 ms - [ INFO ] Max: 98.48 ms - [ INFO ] Throughput: 143.94 FPS + [ INFO ] Median: 40.53 ms + [ INFO ] Average: 42.03 ms + [ INFO ] Min: 30.12 ms + [ INFO ] Max: 171.39 ms + [ INFO ] Throughput: 142.30 FPS .. code:: ipython3 @@ -838,7 +838,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 28.39 ms + [ INFO ] Read model took 28.43 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [1,3,640,640] @@ -854,7 +854,7 @@ models. [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.model.23/aten::cat/Concat_9) : f32 / [...] / [1,56,8400] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 556.59 ms + [ INFO ] Compile model took 573.24 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -891,17 +891,17 @@ models. [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 30.44 ms + [ INFO ] First inference took 25.90 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 5160 iterations - [ INFO ] Duration: 15031.11 ms + [ INFO ] Count: 5124 iterations + [ INFO ] Duration: 15036.37 ms [ INFO ] Latency: - [ INFO ] Median: 34.62 ms - [ INFO ] Average: 34.76 ms - [ INFO ] Min: 25.46 ms - [ INFO ] Max: 51.89 ms - [ INFO ] Throughput: 343.29 FPS + [ INFO ] Median: 34.72 ms + [ INFO ] Average: 35.02 ms + [ INFO ] Min: 25.74 ms + [ INFO ] Max: 52.00 ms + [ INFO ] Throughput: 340.77 FPS Compare accuracy of the Original and Quantized Models diff --git a/docs/notebooks/yolov11-keypoint-detection-with-output_files/yolov11-keypoint-detection-with-output_43_0.png b/docs/notebooks/yolov11-keypoint-detection-with-output_files/yolov11-keypoint-detection-with-output_43_0.png index f4e1de3947dc95..24dd29e6b7751f 100644 --- a/docs/notebooks/yolov11-keypoint-detection-with-output_files/yolov11-keypoint-detection-with-output_43_0.png +++ b/docs/notebooks/yolov11-keypoint-detection-with-output_files/yolov11-keypoint-detection-with-output_43_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:44dd0441baec6d73d3cc7552c26d2878c806514edacb3da332031733686c654b -size 507432 +oid sha256:498642bae277603810b461dcd81b4e67eb3cce1e0f1099f346b8d832dde4720f +size 504317 diff --git a/docs/notebooks/yolov11-object-detection-with-output.rst b/docs/notebooks/yolov11-object-detection-with-output.rst index d987f4148e7265..b446d6d3c1d41e 100644 --- a/docs/notebooks/yolov11-object-detection-with-output.rst +++ b/docs/notebooks/yolov11-object-detection-with-output.rst @@ -141,7 +141,7 @@ Import required utility functions. The lower cell will download the .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg') @@ -207,14 +207,14 @@ Let us consider the examples: .. parsed-literal:: - 100%|██████████| 5.35M/5.35M [00:00<00:00, 23.2MB/s] + 100%|██████████| 5.35M/5.35M [00:00<00:00, 24.0MB/s] .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 480x640 2 bicycles, 2 cars, 2 dogs, 78.0ms - Speed: 2.3ms preprocess, 78.0ms inference, 0.9ms postprocess per image at shape (1, 3, 480, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 480x640 2 bicycles, 2 cars, 2 dogs, 80.3ms + Speed: 2.2ms preprocess, 80.3ms inference, 1.0ms postprocess per image at shape (1, 3, 480, 640) @@ -250,8 +250,8 @@ preserve dynamic shapes in the model. OpenVINO: starting export with openvino 2024.5.0-16993-9c432a3641a... OpenVINO: export success ✅ 1.8s, saved as 'yolo11n_openvino_model/' (5.4 MB) - Export complete (1.9s) - Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization + Export complete (2.0s) + Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization Predict: yolo predict task=detect model=yolo11n_openvino_model imgsz=640 half Validate: yolo val task=detect model=yolo11n_openvino_model imgsz=640 data=/usr/src/ultralytics/ultralytics/cfg/datasets/coco.yaml half Visualize: https://netron.app @@ -330,8 +330,8 @@ ready to check model prediction for object detection. Loading yolo11n_openvino_model for OpenVINO inference... Using OpenVINO LATENCY mode for batch=1 inference... - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 1 bicycle, 2 cars, 1 dog, 18.7ms - Speed: 2.0ms preprocess, 18.7ms inference, 1.0ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 1 bicycle, 2 cars, 1 dog, 19.9ms + Speed: 1.8ms preprocess, 19.9ms inference, 0.9ms postprocess per image at shape (1, 3, 640, 640) @@ -645,8 +645,8 @@ on the image. .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 1 bicycle, 2 cars, 1 dog, 19.7ms - Speed: 1.8ms preprocess, 19.7ms inference, 0.9ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 1 bicycle, 2 cars, 1 dog, 18.1ms + Speed: 1.8ms preprocess, 18.1ms inference, 1.0ms postprocess per image at shape (1, 3, 640, 640) @@ -708,7 +708,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 18.73 ms + [ INFO ] Read model took 18.52 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [?,3,?,?] @@ -724,7 +724,7 @@ models. [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.model.23/aten::cat/Concat_7) : f32 / [...] / [1,84,8400] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 316.96 ms + [ INFO ] Compile model took 317.51 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -761,17 +761,17 @@ models. [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 30.27 ms + [ INFO ] First inference took 30.05 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 18480 iterations - [ INFO ] Duration: 120044.94 ms + [ INFO ] Count: 18396 iterations + [ INFO ] Duration: 120062.14 ms [ INFO ] Latency: - [ INFO ] Median: 38.09 ms - [ INFO ] Average: 38.84 ms - [ INFO ] Min: 20.45 ms - [ INFO ] Max: 97.13 ms - [ INFO ] Throughput: 153.94 FPS + [ INFO ] Median: 38.20 ms + [ INFO ] Average: 39.02 ms + [ INFO ] Min: 22.85 ms + [ INFO ] Max: 99.30 ms + [ INFO ] Throughput: 153.22 FPS .. code:: ipython3 @@ -798,7 +798,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 26.48 ms + [ INFO ] Read model took 26.57 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [1,3,640,640] @@ -814,7 +814,7 @@ models. [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.model.23/aten::cat/Concat_7) : f32 / [...] / [1,84,8400] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 556.87 ms + [ INFO ] Compile model took 533.02 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -851,17 +851,17 @@ models. [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 31.34 ms + [ INFO ] First inference took 29.56 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 5208 iterations - [ INFO ] Duration: 15031.88 ms + [ INFO ] Count: 5148 iterations + [ INFO ] Duration: 15046.93 ms [ INFO ] Latency: - [ INFO ] Median: 34.21 ms - [ INFO ] Average: 34.44 ms - [ INFO ] Min: 18.94 ms - [ INFO ] Max: 52.48 ms - [ INFO ] Throughput: 346.46 FPS + [ INFO ] Median: 34.51 ms + [ INFO ] Average: 34.87 ms + [ INFO ] Min: 24.28 ms + [ INFO ] Max: 69.01 ms + [ INFO ] Throughput: 342.13 FPS Next steps diff --git a/docs/notebooks/yolov11-object-detection-with-output_files/yolov11-object-detection-with-output_43_0.png b/docs/notebooks/yolov11-object-detection-with-output_files/yolov11-object-detection-with-output_43_0.png index 02e6e38bb12a7b..d5c2464840ad65 100644 --- a/docs/notebooks/yolov11-object-detection-with-output_files/yolov11-object-detection-with-output_43_0.png +++ b/docs/notebooks/yolov11-object-detection-with-output_files/yolov11-object-detection-with-output_43_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0904f4ceeae66d4c516656b23389b16d93bc601214a4c9b8d1ee30f7b5366526 -size 572206 +oid sha256:4e54004e6e28473d61aacba64acb407291de3581ae9a0b792a3d8f96bd394f08 +size 572509 diff --git a/docs/notebooks/yolov8-object-detection-with-output.rst b/docs/notebooks/yolov8-object-detection-with-output.rst index 6b7dacd839a587..16648e4559f618 100644 --- a/docs/notebooks/yolov8-object-detection-with-output.rst +++ b/docs/notebooks/yolov8-object-detection-with-output.rst @@ -121,7 +121,7 @@ Install necessary packages. .. code:: ipython3 %pip install -q "openvino>=2024.0.0" "nncf>=2.9.0" - %pip install -q "torch>=2.1" "torchvision>=0.16" "ultralytics==8.3.0" onnx tqdm opencv-python --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q "torch>=2.1" "torchvision>=0.16" "ultralytics==8.2.24" onnx tqdm opencv-python --extra-index-url https://download.pytorch.org/whl/cpu Import required utility functions. The lower cell will download the ``notebook_utils`` Python module from GitHub. @@ -161,7 +161,7 @@ Import required utility functions. The lower cell will download the .. parsed-literal:: - PosixPath('/home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/data/coco_bike.jpg') + PosixPath('/home/akash/intel/openvino_notebooks/notebooks/yolov8-optimization/data/coco_bike.jpg') @@ -194,10 +194,9 @@ Let us consider the examples: from PIL import Image from ultralytics import YOLO - DET_MODEL_NAME = "yolo11n" + DET_MODEL_NAME = "yolov8n" - det_model = YOLO(f"{DET_MODEL_NAME}.pt") - det_model.to("cpu") + det_model = YOLO(models_dir / f"{DET_MODEL_NAME}.pt") label_map = det_model.model.names res = det_model(IMAGE_PATH) @@ -206,14 +205,24 @@ Let us consider the examples: .. parsed-literal:: + Downloading https://github.com/ultralytics/assets/releases/download/v8.2.0/yolov8n.pt to 'models/yolov8n.pt'... - image 1/1 /home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/data/coco_bike.jpg: 480x640 2 bicycles, 2 cars, 2 dogs, 101.6ms - Speed: 3.1ms preprocess, 101.6ms inference, 1.4ms postprocess per image at shape (1, 3, 480, 640) +.. parsed-literal:: + + 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 6.25M/6.25M [00:02<00:00, 2.29MB/s] + + +.. parsed-literal:: + image 1/1 /home/akash/intel/openvino_notebooks/notebooks/yolov8-optimization/data/coco_bike.jpg: 480x640 2 bicycles, 2 cars, 1 dog, 82.9ms + Speed: 2.4ms preprocess, 82.9ms inference, 475.6ms postprocess per image at shape (1, 3, 480, 640) -.. image:: yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.png + + + +.. image:: yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_3.png @@ -230,10 +239,27 @@ preserve dynamic shapes in the model. .. code:: ipython3 # object detection model - det_model_path = Path(f"{DET_MODEL_NAME}_openvino_model/{DET_MODEL_NAME}.xml") + det_model_path = models_dir / f"{DET_MODEL_NAME}_openvino_model/{DET_MODEL_NAME}.xml" if not det_model_path.exists(): det_model.export(format="openvino", dynamic=True, half=True) + +.. parsed-literal:: + + Ultralytics YOLOv8.2.24 🚀 Python-3.8.10 torch-2.1.0+cu121 CPU (Intel Core(TM) i9-10980XE 3.00GHz) + + PyTorch: starting from 'models/yolov8n.pt' with input shape (1, 3, 640, 640) BCHW and output shape(s) (1, 84, 8400) (6.2 MB) + + OpenVINO: starting export with openvino 2024.3.0-16041-1e3b88e4e3f-releases/2024/3... + OpenVINO: export success ✅ 1.7s, saved as 'models/yolov8n_openvino_model/' (6.4 MB) + + Export complete (3.1s) + Results saved to /home/akash/intel/openvino_notebooks/notebooks/yolov8-optimization/models + Predict: yolo predict task=detect model=models/yolov8n_openvino_model imgsz=640 half + Validate: yolo val task=detect model=models/yolov8n_openvino_model imgsz=640 data=coco.yaml half + Visualize: https://netron.app + + Verify model inference ~~~~~~~~~~~~~~~~~~~~~~ @@ -297,7 +323,6 @@ ready to check model prediction for object detection. det_model.predictor.inference = infer det_model.predictor.model.pt = False - det_model res = det_model(IMAGE_PATH) Image.fromarray(res[0].plot()[:, :, ::-1]) @@ -306,8 +331,8 @@ ready to check model prediction for object detection. .. parsed-literal:: - image 1/1 /home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/data/coco_bike.jpg: 640x640 1 bicycle, 2 cars, 1 dog, 16.9ms - Speed: 3.7ms preprocess, 16.9ms inference, 1.7ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /home/akash/intel/openvino_notebooks/notebooks/yolov8-optimization/data/coco_bike.jpg: 640x640 2 bicycles, 2 cars, 1 dog, 16.1ms + Speed: 3.4ms preprocess, 16.1ms inference, 1.8ms postprocess per image at shape (1, 3, 640, 640) @@ -368,16 +393,245 @@ evaluation function. zip_ref.extractall(OUT_DIR / "coco/images") + +.. parsed-literal:: + + /home/akash/intel/NNCF/nncf/examples/post_training_quantization/openvino/yolov8/datasets/val2017.zip: 0%| … + + +.. parsed-literal:: + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + + .. parsed-literal:: - '/home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/datasets/val2017.zip' already exists. - '/home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/datasets/coco2017labels-segments.zip' already exists. + '/home/akash/intel/NNCF/nncf/examples/post_training_quantization/openvino/yolov8/datasets/coco2017labels-segments.zip' already exists. .. parsed-literal:: - /home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/datasets/coco.yaml… + /home/akash/intel/NNCF/nncf/examples/post_training_quantization/openvino/yolov8/datasets/coco.yaml: 0%| … Define validation function @@ -520,7 +774,7 @@ validator class instance. .. parsed-literal:: - val: Scanning /home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/datasets/coco/labels/val2017.cache... 4952 images, 48 backgrounds, 0 corrupt: 100%|██████████| 5000/5000 [00:00\ **Note**: Model evaluation is time consuming +process and can take several minutes, depending on the hardware. For +reducing calculation time, we define ``num_samples`` parameter with +evaluation subset size, but in this case, accuracy can be noncomparable +with originally reported by the authors of the model, due to validation +subset difference. *To validate the models on the full dataset set +``NUM_TEST_SAMPLES = None``.* + +.. code:: ipython3 + + NUM_TEST_SAMPLES = 300 + +.. code:: ipython3 + + fp_det_stats = test(det_ov_model, core, det_data_loader, det_validator, num_samples=NUM_TEST_SAMPLES) + + + +.. parsed-literal:: + + 0%| | 0/300 [00:00 ([1,640,640,3], [N,H,W,C], f32) convert layout [N,C,H,W]: ([1,640,640,3], [N,H,W,C], f32) -> ([1,3,640,640], [N,C,H,W], f32) @@ -1348,7 +1704,7 @@ Now, we can skip these preprocessing steps in detect function: -.. image:: yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.png +.. image:: yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_70_0.png @@ -1527,7 +1883,7 @@ Run the object detection: -.. image:: yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_67_0.png +.. image:: yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_76_0.png .. parsed-literal:: diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.jpg b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.jpg deleted file mode 100644 index 7a742941b54c17..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:394895a09cc413a6343edb2a6f6242258701b6911c09d8405eacc7c27f6fac62 -size 110819 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.png index 6d4bfe012b6187..703c1375a98784 100644 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.png +++ b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7b18cc2e60a802c059cd5cc05598a06c4affd959216e03674eeb68e822295843 -size 913173 +oid sha256:e85f0434876997adc7cf2152fa667b4e271817a443a75ca350d284aee48f9145 +size 908034 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.jpg b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.jpg deleted file mode 100644 index 40ac3a8da0dc2a..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:90d9079b877726f30f154885745b36d8bc281f4970ce10f86529b9739a56ac72 -size 112567 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.png deleted file mode 100644 index e7c4fb31035d98..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6fad172495a797bc753d19d54bd4afaf8f08ed36dfd1b9761df7bfa948673aab -size 911538 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_43_1.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_43_1.png new file mode 100644 index 00000000000000..44e7623ec662d1 --- /dev/null +++ b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_43_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6778bdd31a39722d8ce3fb9ddea6383e451ab1b32cf23fad23b556bd89c99707 +size 907722 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.jpg b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.jpg deleted file mode 100644 index 202af866d221dd..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:02d585dcd341e2580baf166b6f35337d2da17561a251fb5475041e2a20d6e558 -size 110078 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.png deleted file mode 100644 index d0d2028049c4f3..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:44c1e80aa859828b9aae3240fb544b483df8dbd8b7c7bca29479a2a0714612c2 -size 929124 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_67_0.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_67_0.png deleted file mode 100644 index cdae11c722ca65..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_67_0.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2c6c2609efd786cba1d9baac015d06040939ff8d1e7644f67d10d6df5c171a62 -size 569438 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_70_0.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_70_0.png new file mode 100644 index 00000000000000..3b025ea9323f25 --- /dev/null +++ b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_70_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b44f35cf238dcd208fcebe73bb388947f5528a167b511c8098d040b05d9bb819 +size 931491 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_76_0.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_76_0.png new file mode 100644 index 00000000000000..6d628536486d93 --- /dev/null +++ b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_76_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc0d406144e3b07ae6bef629cd7b4973424847958b9865e7133bd0f0c7deeb5c +size 534154 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.jpg b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.jpg deleted file mode 100644 index bcf00f3d50dcb5..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cc502c95e9f5526befe662fcc79a6a91d4f5ff0cfce09165436e91f4ef8224ad -size 113613 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.png deleted file mode 100644 index e886ccd29b47f9..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2c7bb27d4264dbc22e0006e42fb2f1619992aeac7220232119578e42a0d8a083 -size 904352 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_3.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_3.png new file mode 100644 index 00000000000000..5073fa9f3eca09 --- /dev/null +++ b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46d38bd078c6eeda706640d0e35907a9a504bf78f9dd8dbf2961de865c294802 +size 907783 diff --git a/docs/notebooks/yolov9-optimization-with-output.rst b/docs/notebooks/yolov9-optimization-with-output.rst index 39690ea292721d..b132215f390216 100644 --- a/docs/notebooks/yolov9-optimization-with-output.rst +++ b/docs/notebooks/yolov9-optimization-with-output.rst @@ -21,7 +21,6 @@ The tutorial consists of the following steps: - Compare performance of the FP32 and quantized models. - Run optimized model inference on video - **Table of contents:** - `Prerequisites <#prerequisites>`__ @@ -58,7 +57,6 @@ Guide =2023.3.0" "nncf>=2.8.1" "opencv-python" "matplotlib>=3.4" "seaborn" "pandas" "scikit-learn" "torch" "torchvision" "tqdm" --extra-index-url https://download.pytorch.org/whl/cpu @@ -93,9 +91,9 @@ Prerequisites Cloning into 'yolov9'... remote: Enumerating objects: 781, done. remote: Total 781 (delta 0), reused 0 (delta 0), pack-reused 781 (from 1) - Receiving objects: 100% (781/781), 3.27 MiB | 19.93 MiB/s, done. + Receiving objects: 100% (781/781), 3.27 MiB | 16.41 MiB/s, done. Resolving deltas: 100% (331/331), done. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9 Get PyTorch model @@ -133,7 +131,7 @@ applicable for other models from YOLO V9 family. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9/model/gelan-c.pt') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9/model/gelan-c.pt') @@ -188,11 +186,11 @@ using ``ov.save_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9/models/experimental.py:243: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9/models/experimental.py:243: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. ckpt = torch.load(attempt_download(w), map_location='cpu') # load Fusing layers... Model summary: 387 layers, 25288768 parameters, 0 gradients, 102.1 GFLOPs - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9/models/yolo.py:108: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9/models/yolo.py:108: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! elif self.dynamic or self.shape != shape: @@ -573,10 +571,10 @@ asymmetric quantization of activations. .. parsed-literal:: - 2024-10-23 05:32:14.632848: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 05:32:14.668324: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 05:37:29.322072: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 05:37:29.357249: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 05:32:15.276616: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 05:37:29.968269: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -675,7 +673,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 26.16 ms + [ INFO ] Read model took 26.54 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] images (node: x) : f32 / [...] / [?,3,?,?] @@ -687,7 +685,7 @@ models. [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: 'images': [1,3,640,640] - [ INFO ] Reshape model took 7.97 ms + [ INFO ] Reshape model took 8.17 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: [ INFO ] images (node: x) : u8 / [N,C,H,W] / [1,3,640,640] @@ -697,7 +695,7 @@ models. [ INFO ] xi.3 (node: __module.model.22/aten::cat/Concat_1) : f32 / [...] / [1,144,40,40] [ INFO ] xi (node: __module.model.22/aten::cat/Concat) : f32 / [...] / [1,144,20,20] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 475.37 ms + [ INFO ] Compile model took 499.10 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -734,17 +732,17 @@ models. [ INFO ] Fill input 'images' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 182.45 ms + [ INFO ] First inference took 170.42 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 228 iterations - [ INFO ] Duration: 15670.39 ms + [ INFO ] Count: 222 iterations + [ INFO ] Duration: 15614.22 ms [ INFO ] Latency: - [ INFO ] Median: 412.82 ms - [ INFO ] Average: 410.86 ms - [ INFO ] Min: 309.65 ms - [ INFO ] Max: 431.51 ms - [ INFO ] Throughput: 14.55 FPS + [ INFO ] Median: 412.60 ms + [ INFO ] Average: 419.41 ms + [ INFO ] Min: 210.55 ms + [ INFO ] Max: 885.97 ms + [ INFO ] Throughput: 14.22 FPS .. code:: ipython3 @@ -769,7 +767,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 41.07 ms + [ INFO ] Read model took 41.88 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] images (node: x) : f32 / [...] / [1,3,640,640] @@ -791,7 +789,7 @@ models. [ INFO ] xi.3 (node: __module.model.22/aten::cat/Concat_1) : f32 / [...] / [1,144,40,40] [ INFO ] xi (node: __module.model.22/aten::cat/Concat) : f32 / [...] / [1,144,20,20] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 943.07 ms + [ INFO ] Compile model took 946.19 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -828,17 +826,17 @@ models. [ INFO ] Fill input 'images' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 64.84 ms + [ INFO ] First inference took 68.41 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 720 iterations - [ INFO ] Duration: 15158.98 ms + [ INFO ] Count: 726 iterations + [ INFO ] Duration: 15191.13 ms [ INFO ] Latency: - [ INFO ] Median: 119.92 ms - [ INFO ] Average: 125.90 ms - [ INFO ] Min: 80.89 ms - [ INFO ] Max: 277.84 ms - [ INFO ] Throughput: 47.50 FPS + [ INFO ] Median: 121.43 ms + [ INFO ] Average: 125.10 ms + [ INFO ] Min: 56.89 ms + [ INFO ] Max: 305.84 ms + [ INFO ] Throughput: 47.79 FPS Run Live Object Detection diff --git a/docs/notebooks/yolov9-optimization-with-output_files/yolov9-optimization-with-output_36_0.png b/docs/notebooks/yolov9-optimization-with-output_files/yolov9-optimization-with-output_36_0.png index d6b48379756722..ccb15ab8382b86 100644 --- a/docs/notebooks/yolov9-optimization-with-output_files/yolov9-optimization-with-output_36_0.png +++ b/docs/notebooks/yolov9-optimization-with-output_files/yolov9-optimization-with-output_36_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:014be25aa9fb5ebd46a7f69afca67c33d8059694ca54bc2e32c0abfa460cb825 -size 496427 +oid sha256:13788e6c93a735cf601513bf18c5e6cae29819bdff2bd506fa90a4d34ae4bcc5 +size 496199 From 84e683737918e8169ec0fe05132c5157ece7754c Mon Sep 17 00:00:00 2001 From: Alexander Kozlov Date: Tue, 5 Nov 2024 15:24:20 +0400 Subject: [PATCH 06/93] Added E2E LLM test (#26914) Co-authored-by: Alina Kladieva --- tests/llm/accuracy_conformance.py | 116 ++++++++++++++++++++++++++++++ tests/llm/requirements.txt | 6 ++ 2 files changed, 122 insertions(+) create mode 100644 tests/llm/accuracy_conformance.py create mode 100644 tests/llm/requirements.txt diff --git a/tests/llm/accuracy_conformance.py b/tests/llm/accuracy_conformance.py new file mode 100644 index 00000000000000..41015d7664ecc2 --- /dev/null +++ b/tests/llm/accuracy_conformance.py @@ -0,0 +1,116 @@ +import gc +import logging +import os +import shutil +import tempfile + +import pytest +import whowhatbench as wwb +from optimum.intel.openvino import (OVModelForCausalLM, + OVWeightQuantizationConfig) +from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +MODEL_IDS = [ + "TinyLlama/TinyLlama-1.1B-Chat-v1.0", + "Qwen/Qwen2-0.5B-Instruct", +] +DEVICES = [ + "CPU", + "GPU", +] +NUMBER_OF_SAMPLES = 15 +METRIC_OF_INTEREST = "similarity" + +REFERENCES = { + "llama": {"INT8": 0.95, "INT4": 0.95}, + "qwen2": {"INT8": 0.77, "INT4": 0.77}, +} +ACCURACY_THRESHOLDS = { + "INT8": 0.05, + "INT4": 0.05, +} + +tmp_dir = tempfile.mkdtemp() + + +def init_test_scope(): + test_scope = [] + + for model_id in MODEL_IDS: + logger.info(f"Downloading and quantizing model: {model_id}") + model = AutoModelForCausalLM.from_pretrained(model_id) + tokenizer = AutoTokenizer.from_pretrained(model_id) + model_type = model.config.model_type + model_path = os.path.join(tmp_dir, model_type) + model.save_pretrained(model_path) + tokenizer.save_pretrained(model_path) + + ov_model = OVModelForCausalLM.from_pretrained(model_path, load_in_8bit=True) + ov_model_path = model_path = os.path.join(tmp_dir, model_type + "_ov") + ov_model.save_pretrained(ov_model_path) + tokenizer.save_pretrained(ov_model_path) + del ov_model + gc.collect() + + quantization_config = OVWeightQuantizationConfig(bits=4, ratio=0.5) + quantized_model = OVModelForCausalLM.from_pretrained( + model_path, quantization_config=quantization_config + ) + quantized_model_path = model_path = os.path.join( + tmp_dir, model_type + "_ov_int4" + ) + quantized_model.save_pretrained(quantized_model_path) + tokenizer.save_pretrained(quantized_model_path) + del quantized_model + gc.collect() + + set_seed(42) + evaluator = wwb.Evaluator( + base_model=model, tokenizer=tokenizer, num_samples=NUMBER_OF_SAMPLES + ) + gt_path = os.path.join(tmp_dir, model_type + "_gt.json") + evaluator.dump_gt(gt_path) + [ + test_scope.append((ov_model_path, model_type, "INT8", gt_path, device)) + for device in DEVICES + ] + [ + test_scope.append((ov_model_path, model_type, "INT4", gt_path, device)) + for device in DEVICES + ] + + return test_scope + + +def teardown_module(): + logger.info("Remove models") + shutil.rmtree(tmp_dir) + + +test_scope = init_test_scope() + + +@pytest.mark.parametrize( + ("model_path", "model_type", "precision", "gt_data", "device"), + test_scope, +) +def test_accuracy_conformance(model_path, model_type, precision, gt_data, device): + target_model = OVModelForCausalLM.from_pretrained(model_path, device=device) + tokenizer = AutoTokenizer.from_pretrained(model_path) + + evaluator = wwb.Evaluator( + base_model=None, + tokenizer=tokenizer, + gt_data=gt_data, + num_samples=NUMBER_OF_SAMPLES, + ) + + set_seed(42) + _, all_metrics = evaluator.score(target_model) + metric = all_metrics[METRIC_OF_INTEREST].values[0] + abs_metric_diff = abs(REFERENCES[model_type][precision] - metric) + print(metric, REFERENCES[model_type][precision], model_type, precision) + assert abs_metric_diff <= ACCURACY_THRESHOLDS[precision] diff --git a/tests/llm/requirements.txt b/tests/llm/requirements.txt new file mode 100644 index 00000000000000..2d1912890910f5 --- /dev/null +++ b/tests/llm/requirements.txt @@ -0,0 +1,6 @@ +-c ../constraints.txt +--extra-index-url https://download.pytorch.org/whl/cpu +optimum-intel +nncf +whowhatbench @ git+https://github.com/openvinotoolkit/openvino.genai.git#subdirectory=tools/who_what_benchmark +pytest \ No newline at end of file From eb80f9b89c95c04b9e4b5ed459285fa075b7159e Mon Sep 17 00:00:00 2001 From: Anastasia Kuporosova Date: Tue, 5 Nov 2024 14:24:13 +0100 Subject: [PATCH 07/93] Use pyOv tests requirements (#27252) ### Details: - move dependency for torchvision preprocessing to the part where it tests - update numpy upperbound for testing PyOV & numpy 2.1 ### Tickets: - *ticket-id* --- .github/workflows/job_python_unit_tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/job_python_unit_tests.yml b/.github/workflows/job_python_unit_tests.yml index 1dd6ebbfc204d8..64be9ef4bbcc44 100644 --- a/.github/workflows/job_python_unit_tests.yml +++ b/.github/workflows/job_python_unit_tests.yml @@ -111,10 +111,7 @@ jobs: run: | # To enable pytest parallel features python3 -m pip install pytest-xdist[psutil] - # For torchvision to OpenVINO preprocessing converter - python3 -m pip install -r ${INSTALL_TEST_DIR}/python/preprocess/torchvision/requirements.txt - - # TODO: replace with Python API tests requirements + python3 -m pip install -r ${INSTALL_TEST_DIR}/bindings/python/requirements_test.txt python3 -m pip install -r ${INSTALL_TEST_DIR}/mo/requirements_dev.txt # @@ -158,6 +155,9 @@ jobs: - name: Install Python Layer tests dependencies run: | + # For torchvision to OpenVINO preprocessing converter + python3 -m pip install -r ${INSTALL_TEST_DIR}/python/preprocess/torchvision/requirements.txt + # layer test requirements python3 -m pip install -r ${LAYER_TESTS_INSTALL_DIR}/requirements.txt @@ -267,7 +267,7 @@ jobs: if: ${{ fromJSON(inputs.affected-components).Python_API.test }} run: | python3 -m pip uninstall -y numpy - python3 -m pip install "numpy>=2.0.0,<2.1.0" + python3 -m pip install "numpy>=2.0.0,<2.2.0" python3 -m pip install -r ${INSTALL_TEST_DIR}/bindings/python/requirements_test.txt # for 'template' extension export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}:$LD_LIBRARY_PATH From 7171a7dd9baa726e7f2849a981a7666ca81e1f3c Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Tue, 5 Nov 2024 14:29:33 +0100 Subject: [PATCH 08/93] [c-api] PPP add pad step (#27353) ### Details: - Add image padding support in PrePostProcessor for C API ### Continuation of PR: - #26730 ### Tickets: - CVS-152252 --------- Co-authored-by: panhaiqi --- .../c/include/openvino/c/ov_prepostprocess.h | 32 +++++++++ src/bindings/c/src/ov_prepostprocess.cpp | 26 +++++++ src/bindings/c/tests/ov_preprocess_test.cpp | 70 +++++++++++++++++++ src/common/util/src/weights_path.cpp | 1 - 4 files changed, 128 insertions(+), 1 deletion(-) diff --git a/src/bindings/c/include/openvino/c/ov_prepostprocess.h b/src/bindings/c/include/openvino/c/ov_prepostprocess.h index ea2a82943dd683..d7cbbdc26eb5cb 100644 --- a/src/bindings/c/include/openvino/c/ov_prepostprocess.h +++ b/src/bindings/c/include/openvino/c/ov_prepostprocess.h @@ -93,6 +93,18 @@ typedef enum { RESIZE_NEAREST //!< nearest algorithm } ov_preprocess_resize_algorithm_e; +/** + * @enum ov_padding_mode_e + * @ingroup ov_prepostprocess_c_api + * @brief This enum contains enumeration for padding mode. + */ +typedef enum { + CONSTANT = 0, //!< Pads with given constant value. + EDGE, //!< Pads with tensor edge values. + REFLECT, //!< Pads with reflection of tensor data along axis. Values on the edges are not duplicated. + SYMMETRIC //!< Pads similar like `REFLECT` but values on the edges are duplicated. +} ov_padding_mode_e; + /** * @brief Create a ov_preprocess_prepostprocessor_t instance. * @ingroup ov_prepostprocess_c_api @@ -512,3 +524,23 @@ ov_preprocess_input_model_info_set_layout(ov_preprocess_input_model_info_t* prep */ OPENVINO_C_API(ov_status_e) ov_preprocess_prepostprocessor_build(const ov_preprocess_prepostprocessor_t* preprocess, ov_model_t** model); + +/** + * @brief Add pad preprocess operation. Extends an input tensor on edges with constants. + * + * @param preprocess_input_process_steps A pointer to the ov_preprocess_preprocess_steps_t. + * @param pads_begin Number of padding elements to add at the beginning of each axis. + * @param pads_begin_size Pads begin size (number of axes). + * @param pads_end Number of padding elements to add at the end of each axis. + * @param pads_end_size Pads end size (number of axes). + * @param value Value to be populated in the padded area (mode=CONSTANT) + * @param mode Padding mode. + */ +OPENVINO_C_API(ov_status_e) +ov_preprocess_preprocess_steps_pad(const ov_preprocess_preprocess_steps_t* preprocess_input_process_steps, + const int* const pads_begin, + size_t pads_begin_size, + const int* const pads_end, + size_t pads_end_size, + float value, + ov_padding_mode_e mode); diff --git a/src/bindings/c/src/ov_prepostprocess.cpp b/src/bindings/c/src/ov_prepostprocess.cpp index 616883dd54e74c..8448e9daeb10b1 100644 --- a/src/bindings/c/src/ov_prepostprocess.cpp +++ b/src/bindings/c/src/ov_prepostprocess.cpp @@ -24,6 +24,12 @@ const std::map color_format_map {ov_color_format_e::RGBX, ov::preprocess::ColorFormat::RGBX}, {ov_color_format_e::BGRX, ov::preprocess::ColorFormat::BGRX}}; +const std::map padding_mode_map = { + {ov_padding_mode_e::CONSTANT, ov::preprocess::PaddingMode::CONSTANT}, + {ov_padding_mode_e::EDGE, ov::preprocess::PaddingMode::EDGE}, + {ov_padding_mode_e::REFLECT, ov::preprocess::PaddingMode::REFLECT}, + {ov_padding_mode_e::SYMMETRIC, ov::preprocess::PaddingMode::SYMMETRIC}}; + #define GET_OV_COLOR_FARMAT(a) \ (color_format_map.find(a) == color_format_map.end() ? ov::preprocess::ColorFormat::UNDEFINED \ : color_format_map.at(a)) @@ -524,3 +530,23 @@ ov_status_e ov_preprocess_prepostprocessor_build(const ov_preprocess_prepostproc return ov_status_e::OK; } + +ov_status_e ov_preprocess_preprocess_steps_pad(const ov_preprocess_preprocess_steps_t* preprocess_input_process_steps, + const int* const pads_begin, + size_t pads_begin_size, + const int* const pads_end, + size_t pads_end_size, + float value, + ov_padding_mode_e mode) { + if (!preprocess_input_process_steps) { + return ov_status_e::INVALID_C_PARAM; + } + try { + std::vector vec_begin(pads_begin, pads_begin + pads_begin_size); + std::vector vec_end(pads_end, pads_end + pads_end_size); + preprocess_input_process_steps->object->pad(vec_begin, vec_end, value, padding_mode_map.at(mode)); + } + CATCH_OV_EXCEPTIONS + + return ov_status_e::OK; +} diff --git a/src/bindings/c/tests/ov_preprocess_test.cpp b/src/bindings/c/tests/ov_preprocess_test.cpp index 94252bd3a3cbb9..aeb7a097af1352 100644 --- a/src/bindings/c/tests/ov_preprocess_test.cpp +++ b/src/bindings/c/tests/ov_preprocess_test.cpp @@ -1,6 +1,8 @@ // Copyright (C) 2018-2024 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // +#include + #include "ov_test.hpp" class ov_preprocess_test : public ::testing::Test { @@ -596,3 +598,71 @@ TEST_F(ov_preprocess_test, ov_preprocess_prepostprocessor_for_nv12_input) { ov_layout_free(layout); } + +TEST_F(ov_preprocess_test, ov_preprocess_preprocess_steps_pad_constant) { + OV_EXPECT_OK(ov_preprocess_prepostprocessor_create(model, &preprocess)); + EXPECT_NE(nullptr, preprocess); + + OV_EXPECT_OK(ov_preprocess_prepostprocessor_get_input_info(preprocess, &input_info)); + EXPECT_NE(nullptr, input_info); + + OV_EXPECT_OK(ov_preprocess_input_info_get_tensor_info(input_info, &input_tensor_info)); + EXPECT_NE(nullptr, input_tensor_info); + + ov_shape_t shape; + int64_t dims[4] = {1, 1, 225, 230}; + OV_EXPECT_OK(ov_shape_create(4, dims, &shape)); + + OV_EXPECT_OK(ov_tensor_create(ov_element_type_e::F32, shape, &tensor)); + OV_EXPECT_OK(ov_preprocess_input_tensor_info_set_from(input_tensor_info, tensor)); + + OV_EXPECT_OK(ov_preprocess_input_info_get_preprocess_steps(input_info, &input_process)); + EXPECT_NE(nullptr, input_process); + + constexpr auto pads_begin = std::array{0, 1, 2, 0}; + constexpr auto pads_end = std::array{0, 1, 0, -3}; + + OV_EXPECT_OK(ov_preprocess_preprocess_steps_pad(input_process, + pads_begin.data(), + pads_begin.size(), + pads_end.data(), + pads_end.size(), + 1.0f, + ov_padding_mode_e::CONSTANT)); + OV_EXPECT_OK(ov_preprocess_prepostprocessor_build(preprocess, &ppp_model)); + EXPECT_NE(nullptr, ppp_model); +} + +TEST_F(ov_preprocess_test, ov_preprocess_preprocess_steps_pad_edge) { + OV_EXPECT_OK(ov_preprocess_prepostprocessor_create(model, &preprocess)); + EXPECT_NE(nullptr, preprocess); + + OV_EXPECT_OK(ov_preprocess_prepostprocessor_get_input_info(preprocess, &input_info)); + EXPECT_NE(nullptr, input_info); + + OV_EXPECT_OK(ov_preprocess_input_info_get_tensor_info(input_info, &input_tensor_info)); + EXPECT_NE(nullptr, input_tensor_info); + + ov_shape_t shape; + int64_t dims[4] = {1, 2, 225, 230}; + OV_EXPECT_OK(ov_shape_create(4, dims, &shape)); + + OV_EXPECT_OK(ov_tensor_create(ov_element_type_e::F32, shape, &tensor)); + OV_EXPECT_OK(ov_preprocess_input_tensor_info_set_from(input_tensor_info, tensor)); + + OV_EXPECT_OK(ov_preprocess_input_info_get_preprocess_steps(input_info, &input_process)); + EXPECT_NE(nullptr, input_process); + + constexpr auto pads_begin = std::array{0, 0, 2, 0}; + constexpr auto pads_end = std::array{0, 1, 0, -3}; + + OV_EXPECT_OK(ov_preprocess_preprocess_steps_pad(input_process, + pads_begin.data(), + pads_begin.size(), + pads_end.data(), + pads_end.size(), + 1.0f, + ov_padding_mode_e::EDGE)); + OV_EXPECT_OK(ov_preprocess_prepostprocessor_build(preprocess, &ppp_model)); + EXPECT_NE(nullptr, ppp_model); +} diff --git a/src/common/util/src/weights_path.cpp b/src/common/util/src/weights_path.cpp index 9cf2336f064dd0..56d0b56d643fa8 100644 --- a/src/common/util/src/weights_path.cpp +++ b/src/common/util/src/weights_path.cpp @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 // - #include "openvino/util/weights_path.hpp" bool ov::util::validate_weights_path(std::string& weights_path) { From 59ddd74211f0ccec4171a036c010aa162bc1cff8 Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Tue, 5 Nov 2024 14:30:37 +0100 Subject: [PATCH 09/93] [core] Remove LSTM sequence v0 (#27330) ### Details: - Remove LSTM Sequence v0 operator ### Tickets: - CVS-156182 --------- Signed-off-by: Pawel Raasz --- .../src/openvino/runtime/opset1/__init__.py | 1 - .../python/src/openvino/runtime/opset1/ops.py | 89 ----- .../src/openvino/runtime/opset2/__init__.py | 1 - .../src/openvino/runtime/opset3/__init__.py | 1 - .../src/openvino/runtime/opset4/__init__.py | 1 - .../python/tests/test_graph/test_create_op.py | 258 ------------- .../include/openvino/op/lstm_sequence.hpp | 97 ----- src/core/include/openvino/opsets/opset1.hpp | 2 - .../include/openvino/opsets/opset1_tbl.hpp | 1 - src/core/include/openvino/opsets/opset2.hpp | 2 - .../include/openvino/opsets/opset2_tbl.hpp | 1 - src/core/include/openvino/opsets/opset3.hpp | 2 - .../include/openvino/opsets/opset3_tbl.hpp | 1 - .../include/openvino/reference/sequences.hpp | 125 ------ .../include/lstm_sequence_shape_inference.hpp | 26 -- src/core/src/op/lstm_sequence.cpp | 149 -------- src/core/src/pass/low_latency.cpp | 21 +- src/core/tests/opset.cpp | 6 +- src/core/tests/type_prop/lstm_sequence.cpp | 359 ------------------ src/core/tests/type_prop/rnn_seq_base.cpp | 17 +- src/core/tests/visitors/op/lstm_sequence.cpp | 60 --- src/frontends/onnx/frontend/src/op/lstm.cpp | 48 +-- .../models/lstm_bdir_short_input_seq.prototxt | 235 ------------ .../lstm_fwd_with_clip_peepholes.prototxt | 191 ---------- .../onnx/tests/onnx_import_rnn.in.cpp | 110 ------ .../onnx/tests/runtime/ie/unit_test.manifest | 3 - src/plugins/intel_cpu/src/nodes/rnn.cpp | 44 +-- .../src/shape_inference/shape_inference.cpp | 1 - .../pass/rnn_sequences_optimization.cpp | 4 +- .../lstm_seq_shape_inference_test.cpp | 178 --------- .../template/backend/evaluates_map.cpp | 2 - .../template/backend/ops/ops_evaluates.hpp | 8 +- .../template/backend/ops/sequences.cpp | 108 ------ .../template/backend/opset_int_tbl.hpp | 1 - .../functional/op_reference/lstm_sequence.cpp | 86 ----- .../src/op_impl_check/single_op_graph.cpp | 16 - 36 files changed, 37 insertions(+), 2218 deletions(-) delete mode 100644 src/frontends/onnx/tests/models/lstm_bdir_short_input_seq.prototxt delete mode 100644 src/frontends/onnx/tests/models/lstm_fwd_with_clip_peepholes.prototxt diff --git a/src/bindings/python/src/openvino/runtime/opset1/__init__.py b/src/bindings/python/src/openvino/runtime/opset1/__init__.py index 5bfa42f43f26b9..ca7e1aef385b2a 100644 --- a/src/bindings/python/src/openvino/runtime/opset1/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset1/__init__.py @@ -54,7 +54,6 @@ from openvino.runtime.opset1.ops import logical_xor from openvino.runtime.opset1.ops import lrn from openvino.runtime.opset1.ops import lstm_cell -from openvino.runtime.opset1.ops import lstm_sequence from openvino.runtime.opset1.ops import matmul from openvino.runtime.opset1.ops import max_pool from openvino.runtime.opset1.ops import maximum diff --git a/src/bindings/python/src/openvino/runtime/opset1/ops.py b/src/bindings/python/src/openvino/runtime/opset1/ops.py index c100a6c2db2cb4..54f32d404336d4 100644 --- a/src/bindings/python/src/openvino/runtime/opset1/ops.py +++ b/src/bindings/python/src/openvino/runtime/opset1/ops.py @@ -1532,95 +1532,6 @@ def lstm_cell( return _get_node_factory_opset1().create("LSTMCell", node_inputs, attributes) -@deprecated(version="2025.0", message="Use lstm_sequence from opset 5") -@nameable_op -def lstm_sequence( - X: NodeInput, - initial_hidden_state: NodeInput, - initial_cell_state: NodeInput, - sequence_lengths: NodeInput, - W: NodeInput, - R: NodeInput, - B: NodeInput, - hidden_size: int, - direction: str, - activations: Optional[List[str]] = None, - activations_alpha: Optional[List[float]] = None, - activations_beta: Optional[List[float]] = None, - clip: float = 0.0, - name: Optional[str] = None, -) -> Node: - """Return a node which performs LSTMSequence operation. - - :param X: The input tensor. Shape: [batch_size, seq_length, input_size]. - :param initial_hidden_state: The hidden state tensor. - Shape: [batch_size, num_directions, hidden_size]. - :param initial_cell_state: The cell state tensor. - Shape: [batch_size, num_directions, hidden_size]. - :param sequence_lengths: Specifies real sequence lengths for each batch element. - Shape: [batch_size]. Integer type. - :param W: Tensor with weights for matrix multiplication operation with input portion of data. - Shape: [num_directions, 4*hidden_size, input_size]. - :param R: The tensor with weights for matrix multiplication operation with hidden state. - Shape: [num_directions, 4*hidden_size, hidden_size]. - :param B: The tensor with biases. - Shape: [num_directions, 4*hidden_size]. - :param hidden_size: Specifies hidden state size. - :param direction: Specifies if the RNN is forward, reverse, or bidirectional. - :param activations: The list of three activation functions for gates. - :param activations_alpha: The list of alpha parameters for activation functions. - :param activations_beta: The list of beta parameters for activation functions. - :param clip: Specifies bound values [-C, C] for tensor clipping performed before activations. - :param name: An optional name of the output node. - - :return: The new node represents LSTMSequence. Node outputs count: 3. - """ - if activations is None: - activations = ["sigmoid", "tanh", "tanh"] - if activations_alpha is None: - activations_alpha = [] - if activations_beta is None: - activations_beta = [] - - node_inputs = as_nodes( - X, - initial_hidden_state, - initial_cell_state, - sequence_lengths, - W, - R, - B, - name=name, - ) - - # P - nGraph additional input, no such input in the OV spec - peepholes_count = 3 # nGraph default - if direction.lower() == "bidirectional": - num_directions = 2 - else: - num_directions = 1 - peepholes_shape = [num_directions, peepholes_count * hidden_size] - peepholes_array = np.zeros(peepholes_shape) # nGraph default - data_dtype = get_dtype(node_inputs[0].get_output_element_type(0)) - default_p = make_constant_node(peepholes_array, dtype=data_dtype) - node_inputs.append(default_p) - - weights_format = "fico" # OV LSTMWeightsFormat, no such attribute in the OV spec - input_forget = False # nGraph default, no such attribute in the OV spec - - attributes = { - "hidden_size": hidden_size, - "direction": direction.lower(), - "activations": activations, - "activations_alpha": activations_alpha, - "activations_beta": activations_beta, - "clip": clip, - "weights_format": weights_format, - "input_forget": input_forget, - } - return _get_node_factory_opset1().create("LSTMSequence", node_inputs, attributes) - - @nameable_op def matmul( data_a: NodeInput, diff --git a/src/bindings/python/src/openvino/runtime/opset2/__init__.py b/src/bindings/python/src/openvino/runtime/opset2/__init__.py index 34d0d9b6737709..6624149e157e9e 100644 --- a/src/bindings/python/src/openvino/runtime/opset2/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset2/__init__.py @@ -56,7 +56,6 @@ from openvino.runtime.opset1.ops import logical_xor from openvino.runtime.opset1.ops import lrn from openvino.runtime.opset1.ops import lstm_cell -from openvino.runtime.opset1.ops import lstm_sequence from openvino.runtime.opset1.ops import matmul from openvino.runtime.opset1.ops import max_pool from openvino.runtime.opset1.ops import maximum diff --git a/src/bindings/python/src/openvino/runtime/opset3/__init__.py b/src/bindings/python/src/openvino/runtime/opset3/__init__.py index 964acb15b34bbc..5bd68912cae807 100644 --- a/src/bindings/python/src/openvino/runtime/opset3/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset3/__init__.py @@ -65,7 +65,6 @@ from openvino.runtime.opset1.ops import logical_xor from openvino.runtime.opset1.ops import lrn from openvino.runtime.opset1.ops import lstm_cell -from openvino.runtime.opset1.ops import lstm_sequence from openvino.runtime.opset1.ops import matmul from openvino.runtime.opset1.ops import max_pool from openvino.runtime.opset1.ops import maximum diff --git a/src/bindings/python/src/openvino/runtime/opset4/__init__.py b/src/bindings/python/src/openvino/runtime/opset4/__init__.py index bf57172bed40e2..e7cef8eb216d17 100644 --- a/src/bindings/python/src/openvino/runtime/opset4/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset4/__init__.py @@ -70,7 +70,6 @@ from openvino.runtime.opset1.ops import logical_xor from openvino.runtime.opset1.ops import lrn from openvino.runtime.opset4.ops import lstm_cell -from openvino.runtime.opset1.ops import lstm_sequence from openvino.runtime.opset1.ops import matmul from openvino.runtime.opset1.ops import max_pool from openvino.runtime.opset1.ops import maximum diff --git a/src/bindings/python/tests/test_graph/test_create_op.py b/src/bindings/python/tests/test_graph/test_create_op.py index 98d0ec3583882c..f86ea4a18a8ca1 100644 --- a/src/bindings/python/tests/test_graph/test_create_op.py +++ b/src/bindings/python/tests/test_graph/test_create_op.py @@ -315,264 +315,6 @@ def test_lstm_cell_operator(dtype): assert list(node_param.get_output_shape(1)) == [1, 128] -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -@pytest.mark.parametrize("op_name", ["lstm", "lstmOpset1"]) -def test_lstm_cell_operator_opset1(dtype, op_name): - batch_size = 1 - input_size = 16 - hidden_size = 128 - - x_shape = [batch_size, input_size] - h_t_shape = [batch_size, hidden_size] - c_t_shape = [batch_size, hidden_size] - w_shape = [4 * hidden_size, input_size] - r_shape = [4 * hidden_size, hidden_size] - b_shape = [4 * hidden_size] - - parameter_x = ov.parameter(x_shape, name="X", dtype=dtype) - parameter_h_t = ov.parameter(h_t_shape, name="H_t", dtype=dtype) - parameter_c_t = ov.parameter(c_t_shape, name="C_t", dtype=dtype) - parameter_w = ov.parameter(w_shape, name="W", dtype=dtype) - parameter_r = ov.parameter(r_shape, name="R", dtype=dtype) - parameter_b = ov.parameter(b_shape, name="B", dtype=dtype) - - node_default = ov_opset1.lstm_cell( - parameter_x, parameter_h_t, parameter_c_t, parameter_w, parameter_r, parameter_b, hidden_size, name=op_name, - ) - - assert node_default.get_type_name() == "LSTMCell" - assert node_default.get_friendly_name() == op_name - assert node_default.get_output_size() == 2 - assert list(node_default.get_output_shape(0)) == [1, 128] - assert list(node_default.get_output_shape(1)) == [1, 128] - - activations = ["tanh", "Sigmoid", "RELU"] - activation_alpha = [1.0, 2.0, 3.0] - activation_beta = [3.0, 2.0, 1.0] - clip = 0.5 - - node_param = ov_opset1.lstm_cell( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - activations, - activation_alpha, - activation_beta, - clip, - ) - - assert node_param.get_type_name() == "LSTMCell" - assert node_param.get_output_size() == 2 - assert list(node_param.get_output_shape(0)) == [1, 128] - assert list(node_param.get_output_shape(1)) == [1, 128] - - -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -@pytest.mark.parametrize("op_name", ["lstm", "lstmOpset1"]) -def test_lstm_sequence_operator_bidirectional_opset1(dtype, op_name): - batch_size = 1 - input_size = 16 - hidden_size = 128 - num_directions = 2 - seq_length = 2 - - x_shape = [batch_size, seq_length, input_size] - h_t_shape = [batch_size, num_directions, hidden_size] - c_t_shape = [batch_size, num_directions, hidden_size] - seq_len_shape = [batch_size] - w_shape = [num_directions, 4 * hidden_size, input_size] - r_shape = [num_directions, 4 * hidden_size, hidden_size] - b_shape = [num_directions, 4 * hidden_size] - - parameter_x = ov.parameter(x_shape, name="X", dtype=dtype) - parameter_h_t = ov.parameter(h_t_shape, name="H_t", dtype=dtype) - parameter_c_t = ov.parameter(c_t_shape, name="C_t", dtype=dtype) - parameter_seq_len = ov.parameter(seq_len_shape, name="seq_len", dtype=np.int32) - parameter_w = ov.parameter(w_shape, name="W", dtype=dtype) - parameter_r = ov.parameter(r_shape, name="R", dtype=dtype) - parameter_b = ov.parameter(b_shape, name="B", dtype=dtype) - - direction = "BIDIRECTIONAL" - with pytest.warns(DeprecationWarning): - node = ov_opset1.lstm_sequence( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_seq_len, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - direction, - name=op_name, - ) - - assert node.get_type_name() == "LSTMSequence" - assert node.get_friendly_name() == op_name - assert node.get_output_size() == 3 - - activations = ["RELU", "tanh", "Sigmoid"] - activation_alpha = [1.0, 2.0, 3.0] - activation_beta = [3.0, 2.0, 1.0] - clip = 1.22 - - with pytest.warns(DeprecationWarning): - node_param = ov_opset1.lstm_sequence( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_seq_len, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - direction, - activations, - activation_alpha, - activation_beta, - clip, - ) - - assert node_param.get_type_name() == "LSTMSequence" - assert node_param.get_output_size() == 3 - - -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_lstm_sequence_operator_reverse_opset1(dtype): - batch_size = 2 - input_size = 4 - hidden_size = 3 - num_directions = 1 - seq_length = 2 - - x_shape = [batch_size, seq_length, input_size] - h_t_shape = [batch_size, num_directions, hidden_size] - c_t_shape = [batch_size, num_directions, hidden_size] - seq_len_shape = [batch_size] - w_shape = [num_directions, 4 * hidden_size, input_size] - r_shape = [num_directions, 4 * hidden_size, hidden_size] - b_shape = [num_directions, 4 * hidden_size] - - parameter_x = ov.parameter(x_shape, name="X", dtype=dtype) - parameter_h_t = ov.parameter(h_t_shape, name="H_t", dtype=dtype) - parameter_c_t = ov.parameter(c_t_shape, name="C_t", dtype=dtype) - parameter_seq_len = ov.parameter(seq_len_shape, name="seq_len", dtype=np.int32) - parameter_w = ov.parameter(w_shape, name="W", dtype=dtype) - parameter_r = ov.parameter(r_shape, name="R", dtype=dtype) - parameter_b = ov.parameter(b_shape, name="B", dtype=dtype) - - direction = "REVERSE" - with pytest.warns(DeprecationWarning): - node_default = ov_opset1.lstm_sequence( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_seq_len, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - direction, - ) - - assert node_default.get_type_name() == "LSTMSequence" - assert node_default.get_output_size() == 3 - - activations = ["RELU", "tanh", "Sigmoid"] - activation_alpha = [1.0, 2.0, 3.0] - activation_beta = [3.0, 2.0, 1.0] - clip = 1.22 - with pytest.warns(DeprecationWarning): - node_param = ov_opset1.lstm_sequence( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_seq_len, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - direction, - activations, - activation_alpha, - activation_beta, - clip, - ) - - assert node_param.get_type_name() == "LSTMSequence" - assert node_param.get_output_size() == 3 - - -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_lstm_sequence_operator_forward_opset1(dtype): - batch_size = 2 - input_size = 4 - hidden_size = 3 - num_directions = 1 - seq_length = 2 - - x_shape = [batch_size, seq_length, input_size] - h_t_shape = [batch_size, num_directions, hidden_size] - c_t_shape = [batch_size, num_directions, hidden_size] - seq_len_shape = [batch_size] - w_shape = [num_directions, 4 * hidden_size, input_size] - r_shape = [num_directions, 4 * hidden_size, hidden_size] - b_shape = [num_directions, 4 * hidden_size] - - parameter_x = ov.parameter(x_shape, name="X", dtype=dtype) - parameter_h_t = ov.parameter(h_t_shape, name="H_t", dtype=dtype) - parameter_c_t = ov.parameter(c_t_shape, name="C_t", dtype=dtype) - parameter_seq_len = ov.parameter(seq_len_shape, name="seq_len", dtype=np.int32) - parameter_w = ov.parameter(w_shape, name="W", dtype=dtype) - parameter_r = ov.parameter(r_shape, name="R", dtype=dtype) - parameter_b = ov.parameter(b_shape, name="B", dtype=dtype) - - direction = "forward" - with pytest.warns(DeprecationWarning): - node_default = ov_opset1.lstm_sequence( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_seq_len, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - direction, - ) - - assert node_default.get_type_name() == "LSTMSequence" - assert node_default.get_output_size() == 3 - - activations = ["RELU", "tanh", "Sigmoid"] - activation_alpha = [2.0] - activation_beta = [1.0] - clip = 0.5 - with pytest.warns(DeprecationWarning): - node = ov_opset1.lstm_sequence( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_seq_len, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - direction, - activations, - activation_alpha, - activation_beta, - clip, - ) - - assert node.get_type_name() == "LSTMSequence" - assert node.get_output_size() == 3 - - def test_gru_cell_operator(): batch_size = 1 input_size = 16 diff --git a/src/core/include/openvino/op/lstm_sequence.hpp b/src/core/include/openvino/op/lstm_sequence.hpp index 4d68b5ceb7f9c6..28f5a2b1b8bfaf 100644 --- a/src/core/include/openvino/op/lstm_sequence.hpp +++ b/src/core/include/openvino/op/lstm_sequence.hpp @@ -17,103 +17,6 @@ namespace ov { namespace op { -namespace v0 { - -/// -/// \brief Class for lstm sequence node. -/// -/// \note It follows notation and equations defined as in ONNX standard: -/// https://github.com/onnx/onnx/blob/master/docs/Operators.md#LSTM -/// -/// \sa LSTMCell, RNNCell, GRUCell -/// -/// -/// \ingroup ov_ops_cpp_api -class OPENVINO_API OPENVINO_DEPRECATED( - "The class ov::op::v0::LSTMSequence is deprecated. It will be removed in 2025.0 release. Use " - "ov::op::v5::LSTMSequence instead.") LSTMSequence : public util::RNNCellBase { -public: - OPENVINO_OP("LSTMSequence", "opset1", util::RNNCellBase); - LSTMSequence() = default; - - using direction = RecurrentSequenceDirection; - - size_t get_default_output_index() const override { - return no_default_index(); - } - explicit LSTMSequence(const Output& X, - const Output& initial_hidden_state, - const Output& initial_cell_state, - const Output& sequence_lengths, - const Output& W, - const Output& R, - const Output& B, - const Output& P, - const std::int64_t hidden_size, - const direction lstm_direction, - LSTMWeightsFormat weights_format = LSTMWeightsFormat::IFCO, - const std::vector activations_alpha = {}, - const std::vector activations_beta = {}, - const std::vector activations = {"sigmoid", "tanh", "tanh"}, - const float clip_threshold = 0, - const bool input_forget = false); - - explicit LSTMSequence(const Output& X, - const Output& initial_hidden_state, - const Output& initial_cell_state, - const Output& sequence_lengths, - const Output& W, - const Output& R, - const Output& B, - const std::int64_t hidden_size, - const direction lstm_direction, - LSTMWeightsFormat weights_format = LSTMWeightsFormat::IFCO, - const std::vector& activations_alpha = {}, - const std::vector& activations_beta = {}, - const std::vector& activations = {"sigmoid", "tanh", "tanh"}, - const float clip_threshold = 0, - const bool input_forget = false); - - void validate_and_infer_types() override; - bool visit_attributes(AttributeVisitor& visitor) override; - - std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; - - std::vector get_activations_alpha() const { - return m_activations_alpha; - } - std::vector get_activations_beta() const { - return m_activations_beta; - } - std::vector get_activations() const { - return m_activations; - } - float get_clip_threshold() const { - return m_clip; - } - direction get_direction() const { - return m_direction; - } - void set_direction(const direction& dir) { - m_direction = dir; - } - std::int64_t get_hidden_size() const { - return m_hidden_size; - } - bool get_input_forget() const { - return m_input_forget; - } - LSTMWeightsFormat get_weights_format() const { - return m_weights_format; - } - -private: - direction m_direction; - bool m_input_forget; - LSTMWeightsFormat m_weights_format; -}; -} // namespace v0 - namespace v5 { /// /// \brief Class for lstm sequence node. diff --git a/src/core/include/openvino/opsets/opset1.hpp b/src/core/include/openvino/opsets/opset1.hpp index 80ea1b517e4080..e2a7a159299ad3 100644 --- a/src/core/include/openvino/opsets/opset1.hpp +++ b/src/core/include/openvino/opsets/opset1.hpp @@ -9,9 +9,7 @@ namespace ov { namespace opset1 { #define _OPENVINO_OP_REG(a, b) using b::a; -OPENVINO_SUPPRESS_DEPRECATED_START #include "openvino/opsets/opset1_tbl.hpp" -OPENVINO_SUPPRESS_DEPRECATED_END #undef _OPENVINO_OP_REG } // namespace opset1 } // namespace ov diff --git a/src/core/include/openvino/opsets/opset1_tbl.hpp b/src/core/include/openvino/opsets/opset1_tbl.hpp index 089df5896e664c..3008e450065926 100644 --- a/src/core/include/openvino/opsets/opset1_tbl.hpp +++ b/src/core/include/openvino/opsets/opset1_tbl.hpp @@ -88,7 +88,6 @@ _OPENVINO_OP_REG(LogicalOr, ov::op::v1) _OPENVINO_OP_REG(LogicalXor, ov::op::v1) _OPENVINO_OP_REG(LRN, ov::op::v0) _OPENVINO_OP_REG(LSTMCell, ov::op::v0) -_OPENVINO_OP_REG(LSTMSequence, ov::op::v0) _OPENVINO_OP_REG(MatMul, ov::op::v0) _OPENVINO_OP_REG(MaxPool, ov::op::v1) _OPENVINO_OP_REG(Maximum, ov::op::v1) diff --git a/src/core/include/openvino/opsets/opset2.hpp b/src/core/include/openvino/opsets/opset2.hpp index 3c0888c0a88487..bcdaa6a1bc354b 100644 --- a/src/core/include/openvino/opsets/opset2.hpp +++ b/src/core/include/openvino/opsets/opset2.hpp @@ -9,9 +9,7 @@ namespace ov { namespace opset2 { #define _OPENVINO_OP_REG(a, b) using b::a; -OPENVINO_SUPPRESS_DEPRECATED_START #include "openvino/opsets/opset2_tbl.hpp" -OPENVINO_SUPPRESS_DEPRECATED_END #undef _OPENVINO_OP_REG } // namespace opset2 } // namespace ov diff --git a/src/core/include/openvino/opsets/opset2_tbl.hpp b/src/core/include/openvino/opsets/opset2_tbl.hpp index 1d7c55ea74549e..e14887aaced94b 100644 --- a/src/core/include/openvino/opsets/opset2_tbl.hpp +++ b/src/core/include/openvino/opsets/opset2_tbl.hpp @@ -57,7 +57,6 @@ _OPENVINO_OP_REG(LogicalOr, ov::op::v1) _OPENVINO_OP_REG(LogicalXor, ov::op::v1) _OPENVINO_OP_REG(LRN, ov::op::v0) _OPENVINO_OP_REG(LSTMCell, ov::op::v0) -_OPENVINO_OP_REG(LSTMSequence, ov::op::v0) _OPENVINO_OP_REG(MatMul, ov::op::v0) _OPENVINO_OP_REG(MaxPool, ov::op::v1) _OPENVINO_OP_REG(Maximum, ov::op::v1) diff --git a/src/core/include/openvino/opsets/opset3.hpp b/src/core/include/openvino/opsets/opset3.hpp index 29cd23b89c1111..a4e99a21e1c643 100644 --- a/src/core/include/openvino/opsets/opset3.hpp +++ b/src/core/include/openvino/opsets/opset3.hpp @@ -9,9 +9,7 @@ namespace ov { namespace opset3 { #define _OPENVINO_OP_REG(a, b) using b::a; -OPENVINO_SUPPRESS_DEPRECATED_START #include "openvino/opsets/opset3_tbl.hpp" -OPENVINO_SUPPRESS_DEPRECATED_END #undef _OPENVINO_OP_REG } // namespace opset3 } // namespace ov diff --git a/src/core/include/openvino/opsets/opset3_tbl.hpp b/src/core/include/openvino/opsets/opset3_tbl.hpp index 3c34a6999f2696..5c16414a02e987 100644 --- a/src/core/include/openvino/opsets/opset3_tbl.hpp +++ b/src/core/include/openvino/opsets/opset3_tbl.hpp @@ -60,7 +60,6 @@ _OPENVINO_OP_REG(LogicalOr, ov::op::v1) _OPENVINO_OP_REG(LogicalXor, ov::op::v1) _OPENVINO_OP_REG(LRN, ov::op::v0) _OPENVINO_OP_REG(LSTMCell, ov::op::v0) -_OPENVINO_OP_REG(LSTMSequence, ov::op::v0) _OPENVINO_OP_REG(MatMul, ov::op::v0) _OPENVINO_OP_REG(MaxPool, ov::op::v1) _OPENVINO_OP_REG(Maximum, ov::op::v1) diff --git a/src/core/reference/include/openvino/reference/sequences.hpp b/src/core/reference/include/openvino/reference/sequences.hpp index af1946fcbe72c9..42b4083793b63b 100644 --- a/src/core/reference/include/openvino/reference/sequences.hpp +++ b/src/core/reference/include/openvino/reference/sequences.hpp @@ -380,131 +380,6 @@ void lstm_sequence(const char* X, } } -template -void lstm_sequence_v1(const char* X, - const Shape& X_shape, - const char* H, - const Shape& H_shape, - const char* C, - const Shape& C_shape, - const char* seq_lengths, - const Shape& seq_lengths_shape, - const char* W, - const Shape& W_shape, - const char* R, - const Shape& R_shape, - const char* B, - const Shape& B_shape, - const char* P, - const Shape& P_shape, - char* Y, - char* Ho, - char* Co, - const std::string& activation_f, - const std::string& activation_g, - const std::string& activation_h, - float clip, - const ov::op::LSTMWeightsFormat weight_format, - bool input_forget, - op::RecurrentSequenceDirection direction) { - OutputVector results; - if (direction == op::RecurrentSequenceDirection::FORWARD || direction == op::RecurrentSequenceDirection::REVERSE) { - CellArgs args; - args.activation_f = activation_f; - args.activation_g = activation_g; - args.activation_h = activation_h; - args.clip = clip; - args.weight_format = weight_format; - args.input_forget = input_forget; - std::vector inputs = {X, seq_lengths, H, C, W, R, B, P}; - std::vector outputs = {Y, Ho, Co}; - std::vector shapes = {X_shape, seq_lengths_shape, H_shape, C_shape, W_shape, R_shape, B_shape, P_shape}; - cell_pass(CellType::LSTM_v1, - inputs, - shapes, - outputs, - args, - direction == op::RecurrentSequenceDirection::REVERSE); - } else if (direction == op::RecurrentSequenceDirection::BIDIRECTIONAL) { - // Split bidirectional case to forward + reverse passes. - // split inputs - std::vector> H_split(2, std::vector(sizeof(T) * shape_size(H_shape) / 2)); - std::vector> C_split(2, std::vector(sizeof(T) * shape_size(C_shape) / 2)); - std::vector> W_split(2, std::vector(sizeof(T) * shape_size(W_shape) / 2)); - std::vector> R_split(2, std::vector(sizeof(T) * shape_size(R_shape) / 2)); - std::vector> B_split(2, std::vector(sizeof(T) * shape_size(B_shape) / 2)); - std::vector> P_split(2, std::vector(sizeof(T) * shape_size(P_shape) / 2)); - char* h_pointers[2] = {H_split[0].data(), H_split[1].data()}; - char* c_pointers[2] = {C_split[0].data(), C_split[1].data()}; - char* w_pointers[2] = {W_split[0].data(), W_split[1].data()}; - char* r_pointers[2] = {R_split[0].data(), R_split[1].data()}; - char* b_pointers[2] = {B_split[0].data(), B_split[1].data()}; - char* p_pointers[2] = {P_split[0].data(), P_split[1].data()}; - reference::split(H, H_shape, sizeof(T), 1, 2, h_pointers); - reference::split(C, C_shape, sizeof(T), 1, 2, c_pointers); - reference::split(W, W_shape, sizeof(T), 0, 2, w_pointers); - reference::split(R, R_shape, sizeof(T), 0, 2, r_pointers); - reference::split(B, B_shape, sizeof(T), 0, 2, b_pointers); - reference::split(P, P_shape, sizeof(T), 0, 2, p_pointers); - std::vector forward_res_y(sizeof(T) * H_shape[0] * H_shape[2] * X_shape[1]); - std::vector reverse_res_y(sizeof(T) * H_shape[0] * H_shape[2] * X_shape[1]); - std::vector> forward_res(2, std::vector(sizeof(T) * H_shape[0] * H_shape[2])); - std::vector> reverse_res(2, std::vector(sizeof(T) * H_shape[0] * H_shape[2])); - - CellArgs args; - args.activation_f = activation_f; - args.activation_g = activation_g; - args.activation_h = activation_h; - args.clip = clip; - args.weight_format = weight_format; - args.input_forget = input_forget; - std::vector shapes = {X_shape, seq_lengths_shape, H_shape, C_shape, W_shape, R_shape, B_shape, P_shape}; - // update H,C,W,R,B,P shapes after split - shapes[2][1] = 1; - shapes[3][1] = 1; - for (size_t i = 4; i < shapes.size(); ++i) { - shapes[i][0] = 1; - } - // forward pass - cell_pass( - CellType::LSTM_v1, - {X, seq_lengths, h_pointers[0], c_pointers[0], w_pointers[0], r_pointers[0], b_pointers[0], p_pointers[0]}, - shapes, - {forward_res_y.data(), forward_res[0].data(), forward_res[1].data()}, - args, - false); - // reverse pass - cell_pass( - CellType::LSTM_v1, - {X, seq_lengths, h_pointers[1], c_pointers[1], w_pointers[1], r_pointers[1], b_pointers[1], p_pointers[1]}, - shapes, - {reverse_res_y.data(), reverse_res[0].data(), reverse_res[1].data()}, - args, - true); - - // Stack together respective outputs from both forward and reverse passes. - std::vector in_shapes_y = {{H_shape[0], 1, X_shape[1], H_shape[2]}, - {H_shape[0], 1, X_shape[1], H_shape[2]}}; - std::vector in_shapes_h_c = {{H_shape[0], 1, H_shape[2]}, {H_shape[0], 1, H_shape[2]}}; - Shape output_shape_y{H_shape[0], 2, X_shape[1], H_shape[2]}; - Shape output_shape_h_c{H_shape[0], 2, H_shape[2]}; - - reference::concat({forward_res_y.data(), reverse_res_y.data()}, Y, in_shapes_y, output_shape_y, 1, sizeof(T)); - reference::concat({forward_res[0].data(), reverse_res[0].data()}, - Ho, - in_shapes_h_c, - output_shape_h_c, - 1, - sizeof(T)); - reference::concat({forward_res[1].data(), reverse_res[1].data()}, - Co, - in_shapes_h_c, - output_shape_h_c, - 1, - sizeof(T)); - } -} - template void gru_sequence(const char* X, const Shape& X_shape, diff --git a/src/core/shape_inference/include/lstm_sequence_shape_inference.hpp b/src/core/shape_inference/include/lstm_sequence_shape_inference.hpp index c2b5f1389dee7b..8b0daeb27663fb 100644 --- a/src/core/shape_inference/include/lstm_sequence_shape_inference.hpp +++ b/src/core/shape_inference/include/lstm_sequence_shape_inference.hpp @@ -7,32 +7,6 @@ namespace ov { namespace op { -namespace v0 { -OPENVINO_SUPPRESS_DEPRECATED_START -template -std::vector> shape_infer(const LSTMSequence* op, const std::vector& input_shapes) { - OPENVINO_SUPPRESS_DEPRECATED_END - constexpr auto num_gates = 4; - constexpr auto num_state_nodes = 2; - const auto output_shapes = - rnn::seq_base_shape_infer(op, input_shapes, num_gates, num_state_nodes, op->get_direction()); - // Validate rank and dimension for P input (the input doesn't exists in the next version of LSTM or other RNN based - // ops) The checks are compatible with the original restrictions of the v0::LSTMSequence - const auto& hidden_size = output_shapes[0][3]; - if (input_shapes.size() > 7 && input_shapes[7].is_static() && hidden_size.is_static()) { - const auto& p_pshape = input_shapes[7]; - NODE_VALIDATION_CHECK(op, p_pshape.rank().compatible(2), "Input tensor P should have rank equal 2."); - NODE_VALIDATION_CHECK(op, - p_pshape[1].compatible(hidden_size * (num_gates - 1)), - "Inorrect shape of P input. Second dimension is: ", - p_pshape[1], - ", expected: ", - hidden_size.get_length() * (num_gates - 1), - "."); - } - return output_shapes; -} -} // namespace v0 namespace v5 { template std::vector> shape_infer(const LSTMSequence* op, const std::vector& input_shapes) { diff --git a/src/core/src/op/lstm_sequence.cpp b/src/core/src/op/lstm_sequence.cpp index 4d9afa58f469b3..3f70b74e5ef22a 100644 --- a/src/core/src/op/lstm_sequence.cpp +++ b/src/core/src/op/lstm_sequence.cpp @@ -10,155 +10,6 @@ #include "openvino/op/util/recurrent_sequence.hpp" namespace ov { -OPENVINO_SUPPRESS_DEPRECATED_START -op::v0::LSTMSequence::LSTMSequence(const Output& X, - const Output& initial_hidden_state, - const Output& initial_cell_state, - const Output& sequence_lengths, - const Output& W, - const Output& R, - const Output& B, - const Output& P, - const std::int64_t hidden_size, - const LSTMSequence::direction lstm_direction, - LSTMWeightsFormat weights_format, - const std::vector activations_alpha, - const std::vector activations_beta, - const std::vector activations, - const float clip_threshold, - const bool input_forget) - : RNNCellBase({X, initial_hidden_state, initial_cell_state, sequence_lengths, W, R, B, P}, - hidden_size, - clip_threshold, - activations, - activations_alpha, - activations_beta), - m_direction(lstm_direction), - m_input_forget(input_forget), - m_weights_format(weights_format) { - constructor_validate_and_infer_types(); -} - -op::v0::LSTMSequence::LSTMSequence(const Output& X, - const Output& initial_hidden_state, - const Output& initial_cell_state, - const Output& sequence_lengths, - const Output& W, - const Output& R, - const Output& B, - const std::int64_t hidden_size, - const LSTMSequence::direction lstm_direction, - LSTMWeightsFormat weights_format, - const std::vector& activations_alpha, - const std::vector& activations_beta, - const std::vector& activations, - const float clip_threshold, - const bool input_forget) - : op::v0::LSTMSequence( - X, - initial_hidden_state, - initial_cell_state, - sequence_lengths, - W, - R, - B, - Constant::create(element::f32, - Shape{(lstm_direction == LSTMSequence::direction::BIDIRECTIONAL ? 2UL : 1UL), - 3UL * static_cast(hidden_size)}, - std::vector{0.f}), - hidden_size, - lstm_direction, - weights_format, - activations_alpha, - activations_beta, - activations, - clip_threshold, - input_forget) {} - -bool op::v0::LSTMSequence::visit_attributes(AttributeVisitor& visitor) { - OV_OP_SCOPE(v0_LSTMSequence_visit_attributes); - visitor.on_attribute("hidden_size", m_hidden_size); - visitor.on_attribute("activations", m_activations); - visitor.on_attribute("activations_alpha", m_activations_alpha); - visitor.on_attribute("activations_beta", m_activations_beta); - visitor.on_attribute("clip", m_clip); - visitor.on_attribute("direction", m_direction); - - visitor.on_attribute("input_forget", m_input_forget); - visitor.on_attribute("weights_format", m_weights_format); - return true; -} - -std::shared_ptr op::v0::LSTMSequence::clone_with_new_inputs(const OutputVector& new_args) const { - OV_OP_SCOPE(v0_LSTMSequence_clone_with_new_inputs); - check_new_args_count(this, new_args); - if (new_args.size() == 8) { - return std::make_shared(new_args.at(0), // X - new_args.at(1), // initial_hidden_state - new_args.at(2), // initial_cell_state - new_args.at(3), // sequence_lengths - new_args.at(4), // W - new_args.at(5), // R - new_args.at(6), // B - new_args.at(7), // P - m_hidden_size, - m_direction, - m_weights_format, - m_activations_alpha, - m_activations_beta, - m_activations, - m_clip, - m_input_forget); - } else if (new_args.size() == 7) { - return std::make_shared(new_args.at(0), // X - new_args.at(1), // initial_hidden_state - new_args.at(2), // initial_cell_state - new_args.at(3), // sequence_lengths - new_args.at(4), // W - new_args.at(5), // R - new_args.at(6), // B - m_hidden_size, - m_direction, - m_weights_format, - m_activations_alpha, - m_activations_beta, - m_activations, - m_clip, - m_input_forget); - } else { - OPENVINO_THROW("Incorrect number of new arguments"); - } -} - -void op::v0::LSTMSequence::validate_and_infer_types() { - OV_OP_SCOPE(v0_LSTMSequence_validate_and_infer_types); - auto result_et = element::dynamic; - - // Validate input types and save result for output type - NODE_VALIDATION_CHECK(this, - element::Type::merge(result_et, result_et, get_input_element_type(0)) && - element::Type::merge(result_et, result_et, get_input_element_type(1)) && - element::Type::merge(result_et, result_et, get_input_element_type(2)) && - element::Type::merge(result_et, result_et, get_input_element_type(4)) && - element::Type::merge(result_et, result_et, get_input_element_type(5)) && - element::Type::merge(result_et, result_et, get_input_element_type(6)), - "Element types for X, initial_hidden_state, initial_cell_state, W, R and B inputs do " - "not match."); - - // Mark inputs which are relevant to output parameters - for (size_t i = 0; i <= 6; ++i) - set_input_is_relevant_to_shape(i); - - const auto input_shapes = ov::util::get_node_input_partial_shapes(*this); - auto output_shapes = shape_infer(this, input_shapes); - - // Set output size, type and shape - set_output_type(0, result_et, output_shapes[0]); - set_output_type(1, result_et, output_shapes[1]); - set_output_type(2, result_et, output_shapes[2]); -} -OPENVINO_SUPPRESS_DEPRECATED_END - bool op::v5::LSTMSequence::visit_attributes(AttributeVisitor& visitor) { OV_OP_SCOPE(v5_LSTMSequence_visit_attributes); visitor.on_attribute("direction", m_direction); diff --git a/src/core/src/pass/low_latency.cpp b/src/core/src/pass/low_latency.cpp index 7ef4fcd5e84cd8..4b9f606b3e4b28 100644 --- a/src/core/src/pass/low_latency.cpp +++ b/src/core/src/pass/low_latency.cpp @@ -158,26 +158,7 @@ std::vector> process_sequence(const std::sha std::shared_ptr cell; std::vector> new_assigns; bool unroll = false; - OPENVINO_SUPPRESS_DEPRECATED_START - if (auto lstm_seq_v0 = std::dynamic_pointer_cast(op)) { - unroll = need_unroll(op); - new_assigns = replace_with_memory(op, {1, 2}, m_use_const_initializer, to); - if (unroll) { - auto inputs = prepare_inputs(op, 3, to); - cell = to.make(inputs[0], - inputs[1], - inputs[2], - inputs[3], - inputs[4], - inputs[5], - lstm_seq_v0->get_hidden_size(), - lstm_seq_v0->get_activations(), - lstm_seq_v0->get_activations_alpha(), - lstm_seq_v0->get_activations_beta(), - lstm_seq_v0->get_clip_threshold()); - } - OPENVINO_SUPPRESS_DEPRECATED_END - } else if (auto lstm_seq_v5 = std::dynamic_pointer_cast(op)) { + if (auto lstm_seq_v5 = std::dynamic_pointer_cast(op)) { unroll = need_unroll(op); new_assigns = replace_with_memory(op, {1, 2}, m_use_const_initializer, to); if (unroll) { diff --git a/src/core/tests/opset.cpp b/src/core/tests/opset.cpp index 3a2a590de43edf..65c79dc9432439 100644 --- a/src/core/tests/opset.cpp +++ b/src/core/tests/opset.cpp @@ -61,9 +61,9 @@ TEST_P(OpsetTests, opset_dump) { INSTANTIATE_TEST_SUITE_P(opset, OpsetTests, - testing::Values(OpsetTestParams{ov::get_opset1, 110}, - OpsetTestParams{ov::get_opset2, 112}, - OpsetTestParams{ov::get_opset3, 127}, + testing::Values(OpsetTestParams{ov::get_opset1, 109}, + OpsetTestParams{ov::get_opset2, 111}, + OpsetTestParams{ov::get_opset3, 126}, OpsetTestParams{ov::get_opset4, 137}, OpsetTestParams{ov::get_opset5, 145}, OpsetTestParams{ov::get_opset6, 152}, diff --git a/src/core/tests/type_prop/lstm_sequence.cpp b/src/core/tests/type_prop/lstm_sequence.cpp index 698206e25a5562..80da26340fd144 100644 --- a/src/core/tests/type_prop/lstm_sequence.cpp +++ b/src/core/tests/type_prop/lstm_sequence.cpp @@ -89,42 +89,6 @@ shared_ptr lstm_seq_direction_initialization(const recurre return lstm_sequence; } - -OPENVINO_SUPPRESS_DEPRECATED_START -shared_ptr lstm_seq_v0_tensor_initialization(const recurrent_sequence_parameters& param) { - auto batch_size = param.batch_size; - auto seq_length = param.seq_length; - auto input_size = param.input_size; - auto num_directions = param.num_directions; - auto hidden_size = param.hidden_size; - auto et = param.et; - - const auto X = make_shared(et, PartialShape{batch_size, seq_length, input_size}); - const auto initial_hidden_state = - make_shared(et, PartialShape{batch_size, num_directions, hidden_size}); - const auto initial_cell_state = - make_shared(et, PartialShape{batch_size, num_directions, hidden_size}); - const auto sequence_lengths = make_shared(et, PartialShape{batch_size}); - const auto W = make_shared(et, PartialShape{num_directions, hidden_size * 4, input_size}); - const auto R = make_shared(et, PartialShape{num_directions, hidden_size * 4, hidden_size}); - const auto B = make_shared(et, PartialShape{num_directions, hidden_size * 4}); - const auto P = make_shared(et, PartialShape{num_directions, hidden_size * 3}); - - const auto lstm_sequence = make_shared(); - - lstm_sequence->set_argument(0, X); - lstm_sequence->set_argument(1, initial_hidden_state); - lstm_sequence->set_argument(2, initial_cell_state); - lstm_sequence->set_argument(3, sequence_lengths); - lstm_sequence->set_argument(4, W); - lstm_sequence->set_argument(5, R); - lstm_sequence->set_argument(6, B); - lstm_sequence->set_argument(7, P); - - return lstm_sequence; -} -OPENVINO_SUPPRESS_DEPRECATED_END - } // namespace TEST(type_prop, lstm_sequence_forward) { @@ -173,56 +137,6 @@ TEST(type_prop, lstm_sequence_forward) { EXPECT_EQ(lstm_sequence->get_output_shape(2), (Shape{batch_size, num_directions, hidden_size})); } -TEST(type_prop, lstm_sequence_v0_forward) { - const size_t batch_size = 8; - const size_t num_directions = 1; - const size_t seq_length = 6; - const size_t input_size = 4; - const size_t hidden_size = 128; - - const auto X = make_shared(element::f32, Shape{batch_size, seq_length, input_size}); - const auto initial_hidden_state = - make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); - const auto initial_cell_state = - make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); - const auto sequence_lengths = make_shared(element::i32, Shape{batch_size}); - const auto W = make_shared(element::f32, Shape{num_directions, 4 * hidden_size, input_size}); - const auto R = make_shared(element::f32, Shape{num_directions, 4 * hidden_size, hidden_size}); - const auto B = make_shared(element::f32, Shape{num_directions, 4 * hidden_size}); - const auto P = make_shared(element::f32, Shape{num_directions, 3 * hidden_size}); - - const auto lstm_direction = op::RecurrentSequenceDirection::FORWARD; - - OPENVINO_SUPPRESS_DEPRECATED_START - const auto lstm_sequence = make_shared(X, - initial_hidden_state, - initial_cell_state, - sequence_lengths, - W, - R, - B, - P, - hidden_size, - lstm_direction); - - EXPECT_EQ(lstm_sequence->get_hidden_size(), hidden_size); - EXPECT_EQ(lstm_sequence->get_direction(), op::RecurrentSequenceDirection::FORWARD); - EXPECT_TRUE(lstm_sequence->get_activations_alpha().empty()); - EXPECT_TRUE(lstm_sequence->get_activations_beta().empty()); - EXPECT_EQ(lstm_sequence->get_activations()[0], "sigmoid"); - EXPECT_EQ(lstm_sequence->get_activations()[1], "tanh"); - EXPECT_EQ(lstm_sequence->get_activations()[2], "tanh"); - EXPECT_EQ(lstm_sequence->get_clip_threshold(), 0.f); - EXPECT_EQ(lstm_sequence->get_output_element_type(0), element::f32); - EXPECT_EQ(lstm_sequence->outputs().size(), 3); - EXPECT_EQ(lstm_sequence->get_output_shape(0), (Shape{batch_size, num_directions, seq_length, hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(1), element::f32); - EXPECT_EQ(lstm_sequence->get_output_shape(1), (Shape{batch_size, num_directions, hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(2), element::f32); - EXPECT_EQ(lstm_sequence->get_output_shape(2), (Shape{batch_size, num_directions, hidden_size})); - OPENVINO_SUPPRESS_DEPRECATED_END -} - TEST(type_prop, lstm_sequence_bidirectional) { const size_t batch_size = 24; const size_t num_directions = 2; @@ -273,68 +187,6 @@ TEST(type_prop, lstm_sequence_bidirectional) { EXPECT_EQ(lstm_sequence->get_output_shape(2), (Shape{batch_size, num_directions, hidden_size})); } -TEST(type_prop, lstm_sequence_v0_bidirectional) { - const size_t batch_size = 24; - const size_t num_directions = 2; - const size_t seq_length = 12; - const size_t input_size = 8; - const size_t hidden_size = 256; - const bool input_forget = true; - const ov::op::LSTMWeightsFormat weights_format = ov::op::LSTMWeightsFormat::FICO; - const float clip_threshold = 3.5f; - - const auto X = make_shared(element::f32, Shape{batch_size, seq_length, input_size}); - const auto initial_hidden_state = - make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); - const auto initial_cell_state = - make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); - const auto sequence_lengths = make_shared(element::i32, Shape{batch_size}); - const auto W = make_shared(element::f32, Shape{num_directions, 4 * hidden_size, input_size}); - const auto R = make_shared(element::f32, Shape{num_directions, 4 * hidden_size, hidden_size}); - const auto B = make_shared(element::f32, Shape{num_directions, 4 * hidden_size}); - const auto P = make_shared(element::f32, Shape{num_directions, 3 * hidden_size}); - - const auto lstm_direction = opset5::LSTMSequence::direction::BIDIRECTIONAL; - const std::vector activations_alpha = {2.7f, 7.0f, 32.367f}; - const std::vector activations_beta = {0.0f, 5.49f, 6.0f}; - const std::vector activations = {"tanh", "sigmoid", "sigmoid"}; - - OPENVINO_SUPPRESS_DEPRECATED_START - const auto lstm_sequence = make_shared(X, - initial_hidden_state, - initial_cell_state, - sequence_lengths, - W, - R, - B, - hidden_size, - lstm_direction, - weights_format, - activations_alpha, - activations_beta, - activations, - clip_threshold, - input_forget); - - EXPECT_EQ(lstm_sequence->get_hidden_size(), hidden_size); - EXPECT_EQ(lstm_sequence->get_direction(), opset5::LSTMSequence::direction::BIDIRECTIONAL); - EXPECT_EQ(lstm_sequence->get_activations_alpha(), activations_alpha); - EXPECT_EQ(lstm_sequence->get_activations_beta(), activations_beta); - EXPECT_EQ(lstm_sequence->get_activations()[0], "tanh"); - EXPECT_EQ(lstm_sequence->get_activations()[1], "sigmoid"); - EXPECT_EQ(lstm_sequence->get_activations()[2], "sigmoid"); - EXPECT_EQ(lstm_sequence->get_clip_threshold(), 3.5f); - EXPECT_EQ(lstm_sequence->get_input_forget(), true); - EXPECT_EQ(lstm_sequence->get_weights_format(), ov::op::LSTMWeightsFormat::FICO); - EXPECT_EQ(lstm_sequence->get_output_element_type(0), element::f32); - EXPECT_EQ(lstm_sequence->get_output_shape(0), (Shape{batch_size, num_directions, seq_length, hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(1), element::f32); - EXPECT_EQ(lstm_sequence->get_output_shape(1), (Shape{batch_size, num_directions, hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(2), element::f32); - EXPECT_EQ(lstm_sequence->get_output_shape(2), (Shape{batch_size, num_directions, hidden_size})); - OPENVINO_SUPPRESS_DEPRECATED_END -} - TEST(type_prop, lstm_sequence_dynamic_batch_size) { recurrent_sequence_parameters param; @@ -573,214 +425,3 @@ TEST(type_prop, lstm_sequence_invalid_input_direction_num_mismatch) { check_error(op::RecurrentSequenceDirection::FORWARD, 2); check_error(op::RecurrentSequenceDirection::REVERSE, 2); } - -OPENVINO_SUPPRESS_DEPRECATED_START -TEST(type_prop, lstm_sequence_v0_dynamic_num_directions) { - recurrent_sequence_parameters param; - - param.batch_size = 24; - param.num_directions = Dimension::dynamic(); - param.seq_length = 12; - param.input_size = 8; - param.hidden_size = 256; - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - lstm_sequence->validate_and_infer_types(); - - EXPECT_EQ(lstm_sequence->get_output_partial_shape(0), - (PartialShape{param.batch_size, 1, param.seq_length, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(1), (PartialShape{param.batch_size, 1, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(2), (PartialShape{param.batch_size, 1, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(0), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(1), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(2), param.et); -} - -TEST(type_prop, lstm_sequence_v0_dynamic_seq_length) { - recurrent_sequence_parameters param; - - param.batch_size = 24; - param.num_directions = 1; - param.seq_length = Dimension::dynamic(); - param.input_size = 8; - param.hidden_size = 256; - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - lstm_sequence->validate_and_infer_types(); - - EXPECT_EQ(lstm_sequence->get_output_partial_shape(0), - (PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(1), - (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(2), - (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(0), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(1), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(2), param.et); -} - -TEST(type_prop, lstm_sequence_v0_dynamic_hidden_size) { - recurrent_sequence_parameters param; - - param.batch_size = 24; - param.num_directions = 1; - param.seq_length = 12; - param.input_size = 8; - param.hidden_size = Dimension::dynamic(); - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - lstm_sequence->validate_and_infer_types(); - - EXPECT_EQ(lstm_sequence->get_output_partial_shape(0), - (PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(1), - (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(2), - (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(0), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(1), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(2), param.et); -} - -TEST(type_prop, lstm_sequence_v0_dynamic_inputs) { - recurrent_sequence_parameters param; - - param.batch_size = Dimension::dynamic(); - param.input_size = Dimension::dynamic(); - param.hidden_size = Dimension::dynamic(); - param.num_directions = Dimension::dynamic(); - param.seq_length = Dimension::dynamic(); - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - lstm_sequence->validate_and_infer_types(); - - EXPECT_EQ(lstm_sequence->get_output_partial_shape(0), - (PartialShape{param.batch_size, 1, param.seq_length, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(1), (PartialShape{param.batch_size, 1, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(2), (PartialShape{param.batch_size, 1, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(0), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(1), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(2), param.et); -} - -TEST(type_prop, lstm_sequence_v0_invalid_input_dimension) { - recurrent_sequence_parameters param; - - param.batch_size = 24; - param.num_directions = 2; - param.seq_length = 12; - param.input_size = 8; - param.hidden_size = 256; - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - auto invalid_rank0_tensor = make_shared(param.et, PartialShape{}); - - // Validate invalid rank0 tensor for all inputs: X, initial_hidden_state, initial_cell_state W, - // R, B - for (size_t i = 0; i < lstm_sequence->get_input_size(); i++) { - lstm_sequence = lstm_seq_v0_tensor_initialization(param); - lstm_sequence->set_argument(i, invalid_rank0_tensor); - ASSERT_THROW(lstm_sequence->validate_and_infer_types(), ov::AssertFailure) - << "LSTMSequence node was created with invalid data."; - } -} - -TEST(type_prop, lstm_sequence_v0_input_dynamic_rank) { - recurrent_sequence_parameters param; - param.batch_size = 24; - param.num_directions = 1; - param.seq_length = 12; - param.input_size = 8; - param.hidden_size = 256; - param.et = element::f32; - - auto op = lstm_seq_tensor_initialization(param); - auto dynamic_tensor = make_shared(param.et, PartialShape::dynamic(Rank::dynamic())); - - for (size_t i = 0; i < op->get_input_size(); i++) { - auto op = lstm_seq_v0_tensor_initialization(param); - op->set_argument(i, dynamic_tensor); - op->validate_and_infer_types(); - if (i == 0) { // X input - EXPECT_EQ(op->get_output_partial_shape(0), - (PartialShape{param.batch_size, param.num_directions, -1, param.hidden_size})); - } else { - EXPECT_EQ(op->get_output_partial_shape(0), - (PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size})); - } - EXPECT_EQ(op->get_output_partial_shape(1), - (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); - EXPECT_EQ(op->get_output_partial_shape(2), - (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); - - EXPECT_EQ(op->get_output_element_type(0), param.et); - EXPECT_EQ(op->get_output_element_type(1), param.et); - } -} - -TEST(type_prop, lstm_sequence_v0_invalid_input_direction) { - recurrent_sequence_parameters param; - - param.batch_size = 24; - param.num_directions = 3; - param.seq_length = 12; - param.input_size = 8; - param.hidden_size = 256; - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - try { - lstm_sequence->validate_and_infer_types(); - FAIL() << "LSTMSequence node was created with invalid data."; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING( - error.what(), - std::string("Dimension `num_directions` doesn't match to other inputs or `direction` attribute")); - } - - param.num_directions = 2; // 2 is also not allowed for default 'm_direction' = FORWARD - lstm_sequence = lstm_seq_v0_tensor_initialization(param); - try { - lstm_sequence->validate_and_infer_types(); - FAIL() << "LSTMSequence node was created with invalid data."; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING( - error.what(), - std::string("Dimension `num_directions` doesn't match to other inputs or `direction` attribute")); - } -} - -TEST(type_prop, lstm_sequence_v0_invalid_input_P) { - recurrent_sequence_parameters param; - - param.batch_size = 24; - param.num_directions = 1; - param.seq_length = 12; - param.input_size = 8; - param.hidden_size = 256; - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - auto P = make_shared(element::f32, PartialShape{param.hidden_size * 5}); - lstm_sequence->set_argument(7, P); - try { - lstm_sequence->validate_and_infer_types(); - FAIL() << "LSTMSequence node was created with invalid data."; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Input tensor P should have rank equal 2"); - } - P = make_shared(element::f32, PartialShape{param.num_directions, param.hidden_size * 5}); - lstm_sequence->set_argument(7, P); - try { - lstm_sequence->validate_and_infer_types(); - FAIL() << "LSTMSequence node was created with invalid data."; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Inorrect shape of P input. Second dimension is: 1280, expected: 768"); - } - OPENVINO_SUPPRESS_DEPRECATED_END -} diff --git a/src/core/tests/type_prop/rnn_seq_base.cpp b/src/core/tests/type_prop/rnn_seq_base.cpp index 9b3c1b7f8f25df..db38622333d67f 100644 --- a/src/core/tests/type_prop/rnn_seq_base.cpp +++ b/src/core/tests/type_prop/rnn_seq_base.cpp @@ -83,11 +83,7 @@ class RNNSeqBaseTest : public TypePropOpTest { return std::make_shared(X, H_t, sequence_lengths, W, R, B, p.hidden_size.get_max_length(), p.direction); } - OPENVINO_SUPPRESS_DEPRECATED_START - template < - typename T = TOp, - typename std::enable_if::value || std::is_same::value, - bool>::type = true> + template ::value, bool>::type = true> std::shared_ptr make_rnn_seq_based_op(RNNSeqParams& p, bool use_default_ctor = false) { p.gates_count = 4; p.outputs_size = 3; @@ -109,19 +105,12 @@ class RNNSeqBaseTest : public TypePropOpTest { op->set_direction(p.direction); op->set_hidden_size(p.hidden_size.get_max_length()); auto inputs = OutputVector{X, H_t, C_t, sequence_lengths, W, R, B}; - if (ov::is_type(op)) { - const auto P = - make_shared(p.et, - PartialShape{p.num_directions, p.hidden_size * (p.gates_count - 1)}); - inputs.push_back(P); - } op->set_arguments(inputs); op->validate_and_infer_types(); return op; } return std::make_shared(X, H_t, C_t, sequence_lengths, W, R, B, p.hidden_size.get_max_length(), p.direction); } - OPENVINO_SUPPRESS_DEPRECATED_END }; TYPED_TEST_SUITE_P(RNNSeqBaseTest); @@ -284,9 +273,7 @@ REGISTER_TYPED_TEST_SUITE_P(RNNSeqBaseTest, interval_symbols_dims_shape_infer_REVERSE, interval_symbols_dims_shape_infer_BIDIRECTIONAL); -OPENVINO_SUPPRESS_DEPRECATED_START -using RNNSeqBaseTypes = Types; -OPENVINO_SUPPRESS_DEPRECATED_END +using RNNSeqBaseTypes = Types; INSTANTIATE_TYPED_TEST_SUITE_P(type_prop, RNNSeqBaseTest, RNNSeqBaseTypes); } // namespace rnn_seq_test diff --git a/src/core/tests/visitors/op/lstm_sequence.cpp b/src/core/tests/visitors/op/lstm_sequence.cpp index 0d386ba7bc4ccf..f9819d9ae84bc9 100644 --- a/src/core/tests/visitors/op/lstm_sequence.cpp +++ b/src/core/tests/visitors/op/lstm_sequence.cpp @@ -61,63 +61,3 @@ TEST(attributes, lstm_sequence_op) { EXPECT_EQ(g_lstm_sequence->get_clip(), lstm_sequence->get_clip()); EXPECT_EQ(g_lstm_sequence->get_direction(), lstm_sequence->get_direction()); } - -OPENVINO_SUPPRESS_DEPRECATED_START -TEST(attributes, lstm_sequence_v1_op) { - NodeBuilder::opset().insert(); - - const size_t batch_size = 4; - const size_t num_directions = 2; - const size_t seq_length = 8; - const size_t input_size = 16; - const size_t hidden_size = 64; - - const auto X = make_shared(element::f32, Shape{batch_size, seq_length, input_size}); - const auto initial_hidden_state = - make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); - const auto initial_cell_state = - make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); - const auto sequence_lengths = make_shared(element::i32, Shape{batch_size}); - const auto W = make_shared(element::f32, Shape{num_directions, 4 * hidden_size, input_size}); - const auto R = - make_shared(element::f32, Shape{num_directions, 4 * hidden_size, hidden_size}); - const auto B = make_shared(element::f32, Shape{num_directions, 4 * hidden_size}); - const auto P = make_shared(element::f32, Shape{num_directions, 3 * hidden_size}); - - const auto lstm_direction = op::RecurrentSequenceDirection::BIDIRECTIONAL; - const ov::op::LSTMWeightsFormat weights_format = ov::op::LSTMWeightsFormat::FICO; - const std::vector activations_alpha = {1, 2, 3}; - const std::vector activations_beta = {4, 5, 6}; - const std::vector activations = {"tanh", "sigmoid", "tanh"}; - const float clip_threshold = 0.5f; - const bool input_forget = true; - - const auto lstm_sequence = make_shared(X, - initial_hidden_state, - initial_cell_state, - sequence_lengths, - W, - R, - B, - P, - hidden_size, - lstm_direction, - weights_format, - activations_alpha, - activations_beta, - activations, - clip_threshold, - input_forget); - NodeBuilder builder(lstm_sequence, {X, initial_hidden_state, initial_cell_state, sequence_lengths, W, R, B, P}); - auto g_lstm_sequence = ov::as_type_ptr(builder.create()); - - EXPECT_EQ(g_lstm_sequence->get_hidden_size(), lstm_sequence->get_hidden_size()); - EXPECT_EQ(g_lstm_sequence->get_activations(), lstm_sequence->get_activations()); - EXPECT_EQ(g_lstm_sequence->get_activations_alpha(), lstm_sequence->get_activations_alpha()); - EXPECT_EQ(g_lstm_sequence->get_activations_beta(), lstm_sequence->get_activations_beta()); - EXPECT_EQ(g_lstm_sequence->get_clip_threshold(), lstm_sequence->get_clip_threshold()); - EXPECT_EQ(g_lstm_sequence->get_direction(), lstm_sequence->get_direction()); - EXPECT_EQ(g_lstm_sequence->get_input_forget(), lstm_sequence->get_input_forget()); - EXPECT_EQ(g_lstm_sequence->get_weights_format(), lstm_sequence->get_weights_format()); -} -OPENVINO_SUPPRESS_DEPRECATED_END diff --git a/src/frontends/onnx/frontend/src/op/lstm.cpp b/src/frontends/onnx/frontend/src/op/lstm.cpp index 6b934db7d13071..1c0efbbe6b9454 100644 --- a/src/frontends/onnx/frontend/src/op/lstm.cpp +++ b/src/frontends/onnx/frontend/src/op/lstm.cpp @@ -211,41 +211,19 @@ ov::OutputVector lstm(const ov::frontend::onnx::Node& node) { LSTMAttributes attributes{node}; std::shared_ptr lstm_sequence; - if ((input_map.at(LSTMInput::LSTM_INPUT_P).get_names() != std::unordered_set({"P_blank"})) || - (attributes.m_input_forget == true)) { - OPENVINO_SUPPRESS_DEPRECATED_START - lstm_sequence = std::make_shared(input_map.at(LSTMInput::LSTM_INPUT_X), - input_map.at(LSTMInput::LSTM_INPUT_INIT_H), - input_map.at(LSTMInput::LSTM_INPUT_INIT_C), - input_map.at(LSTMInput::LSTM_INPUT_SEQ_LENGTHS), - input_map.at(LSTMInput::LSTM_INPUT_W), - input_map.at(LSTMInput::LSTM_INPUT_R), - input_map.at(LSTMInput::LSTM_INPUT_B), - input_map.at(LSTMInput::LSTM_INPUT_P), - attributes.m_hidden_size, - attributes.m_direction, - ov::op::LSTMWeightsFormat::FICO, - attributes.m_activation_alpha, - attributes.m_activation_beta, - attributes.m_activations, - attributes.m_clip_threshold, - attributes.m_input_forget); - OPENVINO_SUPPRESS_DEPRECATED_END - } else { - lstm_sequence = std::make_shared(input_map.at(LSTMInput::LSTM_INPUT_X), - input_map.at(LSTMInput::LSTM_INPUT_INIT_H), - input_map.at(LSTMInput::LSTM_INPUT_INIT_C), - input_map.at(LSTMInput::LSTM_INPUT_SEQ_LENGTHS), - input_map.at(LSTMInput::LSTM_INPUT_W), - input_map.at(LSTMInput::LSTM_INPUT_R), - input_map.at(LSTMInput::LSTM_INPUT_B), - attributes.m_hidden_size, - attributes.m_direction, - attributes.m_activation_alpha, - attributes.m_activation_beta, - attributes.m_activations, - attributes.m_clip_threshold); - } + lstm_sequence = std::make_shared(input_map.at(LSTMInput::LSTM_INPUT_X), + input_map.at(LSTMInput::LSTM_INPUT_INIT_H), + input_map.at(LSTMInput::LSTM_INPUT_INIT_C), + input_map.at(LSTMInput::LSTM_INPUT_SEQ_LENGTHS), + input_map.at(LSTMInput::LSTM_INPUT_W), + input_map.at(LSTMInput::LSTM_INPUT_R), + input_map.at(LSTMInput::LSTM_INPUT_B), + attributes.m_hidden_size, + attributes.m_direction, + attributes.m_activation_alpha, + attributes.m_activation_beta, + attributes.m_activations, + attributes.m_clip_threshold); const auto Y = lstm_sequence->output(0); const auto Y_h = lstm_sequence->output(1); diff --git a/src/frontends/onnx/tests/models/lstm_bdir_short_input_seq.prototxt b/src/frontends/onnx/tests/models/lstm_bdir_short_input_seq.prototxt deleted file mode 100644 index 57b723e44b6914..00000000000000 --- a/src/frontends/onnx/tests/models/lstm_bdir_short_input_seq.prototxt +++ /dev/null @@ -1,235 +0,0 @@ -ir_version: 5 -graph { - node { - input: "X" - input: "W" - input: "R" - input: "B" - input: "sequence_lens" - input: "initial_h" - input: "initial_c" - input: "P" - output: "Y" - output: "Y_h" - output: "" - name: "node1" - op_type: "LSTM" - attribute { - name: "direction" - s: "bidirectional" - type: STRING - } - attribute { - name: "input_forget" - i: 0 - type: INT - } - attribute { - name: "activations" - strings: "sigmoid" - strings: "tanh" - strings: "tanh" - strings: "sigmoid" - strings: "tanh" - strings: "tanh" - type: STRINGS - } - attribute { - name: "hidden_size" - i: 2 - type: INT - } - attribute { - name: "clip" - f: 9999 - type: FLOAT - } - doc_string: "LSTM" - domain: "" - } - input { - name: "X" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "W" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 8 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "R" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 8 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "B" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 16 - } - } - } - } - } - input { - name: "sequence_lens" - type { - tensor_type { - elem_type: 6 - shape { - dim { - dim_value: 1 - } - } - } - } - } - input { - name: "initial_h" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "initial_c" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "P" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 6 - } - } - } - } - } - output { - name: "Y" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - output { - name: "Y_h" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } -} -opset_import { - domain: "" - version: 7 -} diff --git a/src/frontends/onnx/tests/models/lstm_fwd_with_clip_peepholes.prototxt b/src/frontends/onnx/tests/models/lstm_fwd_with_clip_peepholes.prototxt deleted file mode 100644 index 2a218f089240a5..00000000000000 --- a/src/frontends/onnx/tests/models/lstm_fwd_with_clip_peepholes.prototxt +++ /dev/null @@ -1,191 +0,0 @@ -ir_version: 4 -producer_name: "OpenVINO ONNX Frontend" -graph { - node { - input: "X" - input: "W" - input: "R" - input: "B" - input: "" - input: "" - input: "" - input: "P" - output: "Y" - output: "Y_h" - output: "Y_c" - op_type: "LSTM" - attribute { - name: "clip" - f: 0.10000000149011612 - type: FLOAT - } - attribute { - name: "direction" - s: "forward" - type: STRING - } - attribute { - name: "hidden_size" - i: 2 - type: INT - } - attribute { - name: "input_forget" - i: 0 - type: INT - } - } - name: "compute_graph" - input { - name: "X" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "W" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 1 - } - dim { - dim_value: 8 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "R" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 1 - } - dim { - dim_value: 8 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "B" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 1 - } - dim { - dim_value: 16 - } - } - } - } - } - input { - name: "P" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 1 - } - dim { - dim_value: 6 - } - } - } - } - } - output { - name: "Y" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - output { - name: "Y_h" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 1 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - output { - name: "Y_c" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 1 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } -} -opset_import { - version: 7 -} diff --git a/src/frontends/onnx/tests/onnx_import_rnn.in.cpp b/src/frontends/onnx/tests/onnx_import_rnn.in.cpp index 5979466849d5e6..b61fd1d36f99e0 100644 --- a/src/frontends/onnx/tests/onnx_import_rnn.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_rnn.in.cpp @@ -202,72 +202,6 @@ OPENVINO_TEST(${BACKEND_NAME}, onnx_model_lstm_bidir_mixed_seq_const) { test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 1); } -OPENVINO_TEST(${BACKEND_NAME}, onnx_model_lstm_fwd_with_clip_peepholes) { - auto model = convert_model("lstm_fwd_with_clip_peepholes.onnx"); - - auto test_case = ov::test::TestCase(model, s_device); - test_case.add_input({-0.455351f, -0.276391f, -0.185934f, -0.269585f}); // X - test_case.add_input({-0.494659f, // W - 0.0453352f, - -0.487793f, - 0.417264f, - -0.0175329f, - 0.489074f, - -0.446013f, - 0.414029f, - -0.0091708f, - -0.255364f, - -0.106952f, - -0.266717f, - -0.0888852f, - -0.428709f, - -0.283349f, - 0.208792f}); // W - test_case.add_input({0.146626f, - -0.0620289f, - -0.0815302f, - 0.100482f, - -0.219535f, - -0.306635f, - -0.28515f, - -0.314112f, - -0.228172f, - 0.405972f, - 0.31576f, - 0.281487f, - -0.394864f, - 0.42111f, - -0.386624f, - -0.390225f}); // R - - test_case.add_input({0.381619f, - 0.0323954f, - -0.14449f, - 0.420804f, - -0.258721f, - 0.45056f, - -0.250755f, - 0.0967895f, - 0.0f, - 0.0f, - 0.0f, - 0.0f, - 0.0f, - 0.0f, - 0.0f, - 0.0f}); // B - test_case.add_input({0.2345f, 0.5235f, 0.4378f, 0.3475f, 0.8927f, 0.3456f}); // P - - test_case.add_expected_output(Shape{2, 1, 1, 2}, - {-0.02280854f, 0.02744377f, -0.03516197f, 0.03875681f}); // Y_data - test_case.add_expected_output(Shape{1, 1, 2}, {-0.03516197f, 0.03875681f}); // Y_h_data - test_case.add_expected_output(Shape{1, 1, 2}, {-0.07415761f, 0.07395997f}); // Y_c_data - - // We have to enlarge tolerance bits to 3 - it's only one bit more than default value. - // The discrepancies may occur at most on 7th decimal position. - test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 3); -} - OPENVINO_TEST(${BACKEND_NAME}, onnx_model_lstm_fwd_mixed_seq) { auto model = convert_model("lstm_fwd_mixed_seq.onnx"); @@ -397,50 +331,6 @@ OPENVINO_TEST(${BACKEND_NAME}, onnx_model_lstm_fwd_large_batch_no_clip) { test_case.run(); } -OPENVINO_TEST(${BACKEND_NAME}, onnx_model_lstm_bdir_short_input_seq_peepholes) { - auto model = convert_model("lstm_bdir_short_input_seq.onnx"); - - auto test_case = ov::test::TestCase(model, s_device); - - // X - test_case.add_input({-0.455351f, -0.276391f, -0.185934f, -0.269585f}); - // W - test_case.add_input( - {-0.494659f, 0.0453352f, -0.487793f, 0.417264f, -0.0175329f, 0.489074f, -0.446013f, 0.414029f, - -0.0091708f, -0.255364f, -0.106952f, -0.266717f, -0.0888852f, -0.428709f, -0.283349f, 0.208792f, - -0.494659f, 0.0453352f, -0.487793f, 0.417264f, -0.0175329f, 0.489074f, -0.446013f, 0.414029f, - -0.0091708f, -0.255364f, -0.106952f, -0.266717f, -0.0888852f, -0.428709f, -0.283349f, 0.208792f}); - // R - test_case.add_input({0.146626f, -0.0620289f, -0.0815302f, 0.100482f, -0.219535f, -0.306635f, -0.28515f, - -0.314112f, -0.228172f, 0.405972f, 0.31576f, 0.281487f, -0.394864f, 0.42111f, - -0.386624f, -0.390225f, 0.146626f, -0.0620289f, -0.0815302f, 0.100482f, -0.219535f, - -0.306635f, -0.28515f, -0.314112f, -0.228172f, 0.405972f, 0.31576f, 0.281487f, - -0.394864f, 0.42111f, -0.386624f, -0.390225f}); - // B - test_case.add_input({0.381619f, 0.0323954f, -0.14449f, 0.420804f, -0.258721f, 0.45056f, -0.250755f, - 0.0967895f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 0.381619f, 0.0323954f, -0.14449f, 0.420804f, -0.258721f, - 0.45056f, -0.250755f, 0.0967895f, 0.0f, 0.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 0.0f, 0.0f}); - // sequence_lens - test_case.add_input({1}); - // initial_h - test_case.add_input({0.0f, 0.0f, -0.0306872f, 0.028035f}); - // initial_c - test_case.add_input({0.0f, 0.0f, -0.07243599f, 0.0467052f}); - // P - test_case.add_input( - {0.2345f, 0.5235f, 0.4378f, 0.3475f, 0.8927f, 0.3456f, 0.2345f, 0.5235f, 0.4378f, 0.3475f, 0.8927f, 0.3456f}); - - // Y - test_case.add_expected_output(Shape{2, 2, 1, 2}, - {-0.0251062f, 0.0561262f, -0.0318928f, 0.0762679f, 0.0f, 0.0f, 0.0f, 0.0f}); - // Y_h - test_case.add_expected_output(Shape{2, 1, 2}, {-0.0251062f, 0.0561262f, -0.0318928f, 0.0762679f}); - - test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 3); -} - OPENVINO_TEST(${BACKEND_NAME}, onnx_model_lstm_mixed_seq_reverse) { auto model = convert_model("lstm_mixed_seq_reverse.onnx"); diff --git a/src/frontends/onnx/tests/runtime/ie/unit_test.manifest b/src/frontends/onnx/tests/runtime/ie/unit_test.manifest index b2e87a2588ec65..3eadb698ca0a03 100644 --- a/src/frontends/onnx/tests/runtime/ie/unit_test.manifest +++ b/src/frontends/onnx/tests/runtime/ie/unit_test.manifest @@ -121,9 +121,6 @@ onnx_model_lp_norm_default_dynamic onnx_instance_normalization_dynamic # Legacy tests with unsupported features from opset4 LSTM/GRU/RNN -# Peepholes input unsupported -onnx_model_lstm_fwd_with_clip_peepholes -onnx_model_lstm_bdir_short_input_seq_peepholes # Activation function hardsigmoid is not supported onnx_model_gru_fwd_activations_relu_hardsigmoid onnx_model_lstm_fwd_hardsigmoid_activation diff --git a/src/plugins/intel_cpu/src/nodes/rnn.cpp b/src/plugins/intel_cpu/src/nodes/rnn.cpp index cab8bb3ad46325..679907c2d9cf28 100644 --- a/src/plugins/intel_cpu/src/nodes/rnn.cpp +++ b/src/plugins/intel_cpu/src/nodes/rnn.cpp @@ -33,8 +33,6 @@ static rnn_direction ieDirection2dnnl(const std::shared_ptr& op) ov::op::RecurrentSequenceDirection direction = ov::op::RecurrentSequenceDirection::FORWARD; if (ov::is_type(op)) { direction = ov::as_type_ptr(op)->get_direction(); - } else if (ov::is_type(op)) { - direction = ov::as_type_ptr(op)->get_direction(); } else if (ov::is_type(op)) { direction = ov::as_type_ptr(op)->get_direction(); } else if (ov::is_type(op)) { @@ -75,14 +73,13 @@ static dnnl::algorithm ie2dnnl(const std::shared_ptr& op) { else return dnnl::algorithm::vanilla_augru; } else if (one_of(op->get_type_info(), - ov::op::v0::LSTMCell::get_type_info_static(), - ov::op::v4::LSTMCell::get_type_info_static(), - ov::op::v0::LSTMSequence::get_type_info_static(), - ov::op::v5::LSTMSequence::get_type_info_static())) { + ov::op::v0::LSTMCell::get_type_info_static(), + ov::op::v4::LSTMCell::get_type_info_static(), + ov::op::v5::LSTMSequence::get_type_info_static())) { return dnnl::algorithm::vanilla_lstm; } else if (one_of(op->get_type_info(), - ov::op::v0::RNNCell::get_type_info_static(), - ov::op::v5::RNNSequence::get_type_info_static())) { + ov::op::v0::RNNCell::get_type_info_static(), + ov::op::v5::RNNSequence::get_type_info_static())) { return dnnl::algorithm::vanilla_rnn; } else { OPENVINO_THROW("Operation ", @@ -201,16 +198,15 @@ bool RNNKey::operator==(const RNNKey& rhs) const { bool RNN::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { if (!one_of(op->get_type_info(), - ov::op::v3::GRUCell::get_type_info_static(), - ov::op::internal::AUGRUCell::get_type_info_static(), - ov::op::internal::AUGRUSequence::get_type_info_static(), - ov::op::v0::LSTMCell::get_type_info_static(), - ov::op::v4::LSTMCell::get_type_info_static(), - ov::op::v0::RNNCell::get_type_info_static(), - ov::op::v5::GRUSequence::get_type_info_static(), - ov::op::v0::LSTMSequence::get_type_info_static(), - ov::op::v5::LSTMSequence::get_type_info_static(), - ov::op::v5::RNNSequence::get_type_info_static())) { + ov::op::v3::GRUCell::get_type_info_static(), + ov::op::internal::AUGRUCell::get_type_info_static(), + ov::op::internal::AUGRUSequence::get_type_info_static(), + ov::op::v0::LSTMCell::get_type_info_static(), + ov::op::v4::LSTMCell::get_type_info_static(), + ov::op::v0::RNNCell::get_type_info_static(), + ov::op::v5::GRUSequence::get_type_info_static(), + ov::op::v5::LSTMSequence::get_type_info_static(), + ov::op::v5::RNNSequence::get_type_info_static())) { errorMessage = "Unsupported sequence operation."; return false; } @@ -239,9 +235,7 @@ bool RNN::isSupportedOperation(const std::shared_ptr& op, std::s errorMessage = "Node expects 6 inputs. Actual: " + std::to_string(op->get_input_size()); return false; } - } else if (one_of(op->get_type_info(), - ov::op::v0::LSTMSequence::get_type_info_static(), - ov::op::v5::LSTMSequence::get_type_info_static())) { + } else if (one_of(op->get_type_info(), ov::op::v5::LSTMSequence::get_type_info_static())) { if (op->get_input_size() != 7) { errorMessage = "Node expects 7 inputs. Actual: " + std::to_string(op->get_input_size()); return false; @@ -264,7 +258,6 @@ bool RNN::isSupportedOperation(const std::shared_ptr& op, std::s if (one_of(rnnCellBase->get_type_info(), ov::op::v0::LSTMCell::get_type_info_static(), ov::op::v4::LSTMCell::get_type_info_static(), - ov::op::v0::LSTMSequence::get_type_info_static(), ov::op::v5::LSTMSequence::get_type_info_static())) { if (rnnCellBase->get_activations() != std::vector{"sigmoid", "tanh", "tanh"}) { errorMessage = "Not supported activation functions"; @@ -294,9 +287,6 @@ bool RNN::isSupportedOperation(const std::shared_ptr& op, std::s if (auto gru_seq = ov::as_type_ptr(op)) { direction = gru_seq->get_direction(); seqLenIdx = 2; - } else if (auto lstm_seq = ov::as_type_ptr(op)) { - direction = lstm_seq->get_direction(); - seqLenIdx = 3; } else if (auto lstm_seq = ov::as_type_ptr(op)) { direction = lstm_seq->get_direction(); seqLenIdx = 3; @@ -441,9 +431,7 @@ RNN::RNN(const std::shared_ptr& op, const GraphContext::CPtr context) } else if (op->get_type_info() == ov::op::internal::AUGRUSequence::get_type_info_static()) { sIdx = 2; wIdx = 3; rIdx = 4; bIdx = 5; aIdx = 6; yIdx = 0; hoIdx = 1; - } else if (one_of(op->get_type_info(), - ov::op::v0::LSTMSequence::get_type_info_static(), - ov::op::v5::LSTMSequence::get_type_info_static())) { + } else if (one_of(op->get_type_info(), ov::op::v5::LSTMSequence::get_type_info_static())) { sIdx = 3; wIdx = 4; rIdx = 5; bIdx = 6; yIdx = 0; hoIdx = 1; coIdx = 2; } diff --git a/src/plugins/intel_cpu/src/shape_inference/shape_inference.cpp b/src/plugins/intel_cpu/src/shape_inference/shape_inference.cpp index 1921169f83afd7..8e2e794eacc939 100644 --- a/src/plugins/intel_cpu/src/shape_inference/shape_inference.cpp +++ b/src/plugins/intel_cpu/src/shape_inference/shape_inference.cpp @@ -532,7 +532,6 @@ const IStaticShapeInferFactory::TRegistry IStaticShapeInferFactory::registry{ _OV_OP_SHAPE_INFER_MASK_REG(opset1::GroupConvolutionBackpropData, ShapeInferPaddingTA, util::bit::mask(2)), _OV_OP_SHAPE_INFER_MASK_REG(opset1::Interpolate, ShapeInferTA, util::bit::mask(1)), _OV_OP_SHAPE_INFER_MASK_REG(opset1::LSTMCell, ShapeInferTA, util::bit::mask()), - _OV_OP_SHAPE_INFER_MASK_REG(opset1::LSTMSequence, ShapeInferTA, util::bit::mask()), _OV_OP_SHAPE_INFER_MASK_REG(opset1::MatMul, ShapeInferTA, util::bit::mask()), _OV_OP_SHAPE_INFER_MASK_REG(opset1::MaxPool, ShapeInferPaddingTA, util::bit::mask()), _OV_OP_SHAPE_INFER_MASK_REG(opset1::NonMaxSuppression, ShapeInferTA, util::bit::mask(2)), diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/rnn_sequences_optimization.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/rnn_sequences_optimization.cpp index 72ca09a8152298..916e7b21ffa484 100644 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/rnn_sequences_optimization.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/rnn_sequences_optimization.cpp @@ -124,14 +124,12 @@ ov::intel_cpu::OptimizeRNNSequenceTransposes::OptimizeRNNSequenceTransposes() { ov::intel_cpu::OptimizeLSTMSequenceTransposes::OptimizeLSTMSequenceTransposes() { MATCHER_SCOPE(OptimizeLSTMSequenceTransposes); - auto lstmSequenceNgraph = ov::pass::pattern::wrap_type(); + auto lstmSequenceNgraph = ov::pass::pattern::wrap_type(); ov::matcher_pass_callback callback = [](ov::pass::pattern::Matcher &m) { auto checkSequence = [](const std::shared_ptr& node) { if (auto lstm5 = ov::as_type_ptr(node)) { return lstm5->get_direction() != ov::op::RecurrentSequenceDirection::BIDIRECTIONAL; - } else if (auto lstm1 = ov::as_type_ptr(node)) { - return lstm1->get_direction() != ov::op::RecurrentSequenceDirection::BIDIRECTIONAL; } else { return false; } diff --git a/src/plugins/intel_cpu/tests/unit/shape_inference_test/lstm_seq_shape_inference_test.cpp b/src/plugins/intel_cpu/tests/unit/shape_inference_test/lstm_seq_shape_inference_test.cpp index 067477e146f509..a8b4160f405fd1 100644 --- a/src/plugins/intel_cpu/tests/unit/shape_inference_test/lstm_seq_shape_inference_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/shape_inference_test/lstm_seq_shape_inference_test.cpp @@ -9,184 +9,6 @@ using namespace ov; using namespace ov::intel_cpu; -class LSTMSequenceV0StaticShapeInferenceTest : public OpStaticShapeInferenceTest { -protected: - void SetUp() override { - this->output_shapes = ShapeVector(3); - } -}; - -TEST_F(LSTMSequenceV0StaticShapeInferenceTest, default_ctor) { - constexpr size_t batch_size = 2; - constexpr size_t input_size = 3; - constexpr size_t hidden_size = 5; - constexpr size_t seq_len = 4; - constexpr size_t num_directions = 1; - constexpr size_t gates_count = 4; - - const auto op = make_op(); - - input_shapes = {StaticShape{batch_size, seq_len, input_size}, // X - StaticShape{batch_size, num_directions, hidden_size}, // H_t - StaticShape{batch_size, num_directions, hidden_size}, // C_t - StaticShape{batch_size}, // seq_lengths - StaticShape{num_directions, gates_count * hidden_size, input_size}, // W - StaticShape{num_directions, gates_count * hidden_size, hidden_size}, // R - StaticShape{num_directions, gates_count * hidden_size}, // B - StaticShape{num_directions, (gates_count - 1) * hidden_size}}; // P - - output_shapes = shape_inference(op.get(), input_shapes); - - EXPECT_EQ(output_shapes.size(), 3); - EXPECT_EQ(output_shapes[0], StaticShape({batch_size, num_directions, seq_len, hidden_size})); - EXPECT_EQ(output_shapes[1], StaticShape({batch_size, num_directions, hidden_size})); - EXPECT_EQ(output_shapes[2], StaticShape({batch_size, num_directions, hidden_size})); -} - -TEST_F(LSTMSequenceV0StaticShapeInferenceTest, FORWARD_without_P) { - constexpr size_t batch_size = 2; - constexpr size_t input_size = 3; - constexpr size_t hidden_size = 5; - constexpr size_t seq_len = 4; - constexpr size_t num_directions = 1; - constexpr size_t gates_count = 4; - - constexpr auto direction = op::RecurrentSequenceDirection::FORWARD; - - const auto X = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto H_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto C_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto seq_lengths = std::make_shared(element::f32, PartialShape::dynamic(1)); - const auto W = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto R = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto B = std::make_shared(element::f32, PartialShape::dynamic(2)); - - const auto op = make_op(X, H_t, C_t, seq_lengths, W, R, B, hidden_size, direction); - - input_shapes = {StaticShape{batch_size, seq_len, input_size}, // X - StaticShape{batch_size, num_directions, hidden_size}, // H_t - StaticShape{batch_size, num_directions, hidden_size}, // C_t - StaticShape{batch_size}, // seq_lengths - StaticShape{num_directions, gates_count * hidden_size, input_size}, // W - StaticShape{num_directions, gates_count * hidden_size, hidden_size}, // R - StaticShape{num_directions, gates_count * hidden_size}}; // B - - output_shapes = shape_inference(op.get(), input_shapes); - - EXPECT_EQ(output_shapes.size(), 3); - EXPECT_EQ(output_shapes[0], StaticShape({batch_size, num_directions, seq_len, hidden_size})); - EXPECT_EQ(output_shapes[1], StaticShape({batch_size, num_directions, hidden_size})); - EXPECT_EQ(output_shapes[2], StaticShape({batch_size, num_directions, hidden_size})); -} - -TEST_F(LSTMSequenceV0StaticShapeInferenceTest, FORWARD_with_P) { - constexpr size_t batch_size = 2; - constexpr size_t input_size = 3; - constexpr size_t hidden_size = 5; - constexpr size_t seq_len = 4; - constexpr size_t num_directions = 1; - constexpr size_t gates_count = 4; - - constexpr auto direction = op::RecurrentSequenceDirection::FORWARD; - - const auto X = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto H_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto C_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto seq_lengths = std::make_shared(element::f32, PartialShape::dynamic(1)); - const auto W = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto R = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto B = std::make_shared(element::f32, PartialShape::dynamic(2)); - const auto P = std::make_shared(element::f32, PartialShape::dynamic(2)); - - const auto op = make_op(X, H_t, C_t, seq_lengths, W, R, B, P, hidden_size, direction); - - input_shapes = {StaticShape{batch_size, seq_len, input_size}, // X - StaticShape{batch_size, num_directions, hidden_size}, // H_t - StaticShape{batch_size, num_directions, hidden_size}, // C_t - StaticShape{batch_size}, // seq_lengths - StaticShape{num_directions, gates_count * hidden_size, input_size}, // W - StaticShape{num_directions, gates_count * hidden_size, hidden_size}, // R - StaticShape{num_directions, gates_count * hidden_size}, // B - StaticShape{num_directions, (gates_count - 1) * hidden_size}}; // P - - output_shapes = shape_inference(op.get(), input_shapes); - - EXPECT_EQ(output_shapes.size(), 3); - EXPECT_EQ(output_shapes[0], StaticShape({batch_size, num_directions, seq_len, hidden_size})); - EXPECT_EQ(output_shapes[1], StaticShape({batch_size, num_directions, hidden_size})); - EXPECT_EQ(output_shapes[2], StaticShape({batch_size, num_directions, hidden_size})); -} - -TEST_F(LSTMSequenceV0StaticShapeInferenceTest, REVERSE) { - constexpr size_t batch_size = 2; - constexpr size_t input_size = 3; - constexpr size_t hidden_size = 5; - constexpr size_t seq_len = 4; - constexpr size_t num_directions = 1; - constexpr size_t gates_count = 4; - - constexpr auto direction = op::RecurrentSequenceDirection::REVERSE; - - const auto X = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto H_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto C_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto seq_lengths = std::make_shared(element::f32, PartialShape::dynamic(1)); - const auto W = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto R = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto B = std::make_shared(element::f32, PartialShape::dynamic(2)); - - const auto op = make_op(X, H_t, C_t, seq_lengths, W, R, B, hidden_size, direction); - - input_shapes = {StaticShape{batch_size, seq_len, input_size}, // X - StaticShape{batch_size, num_directions, hidden_size}, // H_t - StaticShape{batch_size, num_directions, hidden_size}, // C_t - StaticShape{batch_size}, // seq_lengths - StaticShape{num_directions, gates_count * hidden_size, input_size}, // W - StaticShape{num_directions, gates_count * hidden_size, hidden_size}, // R - StaticShape{num_directions, gates_count * hidden_size}}; // B - - output_shapes = shape_inference(op.get(), input_shapes); - EXPECT_EQ(output_shapes.size(), 3); - EXPECT_EQ(output_shapes[0], StaticShape({batch_size, num_directions, seq_len, hidden_size})); - EXPECT_EQ(output_shapes[1], StaticShape({batch_size, num_directions, hidden_size})); - EXPECT_EQ(output_shapes[2], StaticShape({batch_size, num_directions, hidden_size})); -} - -TEST_F(LSTMSequenceV0StaticShapeInferenceTest, BIDIRECTIONAL) { - constexpr size_t batch_size = 2; - constexpr size_t input_size = 3; - constexpr size_t hidden_size = 5; - constexpr size_t seq_len = 4; - constexpr size_t num_directions = 2; - constexpr size_t gates_count = 4; - - constexpr auto direction = op::RecurrentSequenceDirection::BIDIRECTIONAL; - - const auto X = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto H_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto C_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto seq_lengths = std::make_shared(element::f32, PartialShape::dynamic(1)); - const auto W = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto R = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto B = std::make_shared(element::f32, PartialShape::dynamic(2)); - - const auto op = make_op(X, H_t, C_t, seq_lengths, W, R, B, hidden_size, direction); - - input_shapes = {StaticShape{batch_size, seq_len, input_size}, // X - StaticShape{batch_size, num_directions, hidden_size}, // H_t - StaticShape{batch_size, num_directions, hidden_size}, // C_t - StaticShape{batch_size}, // seq_lengths - StaticShape{num_directions, gates_count * hidden_size, input_size}, // W - StaticShape{num_directions, gates_count * hidden_size, hidden_size}, // R - StaticShape{num_directions, gates_count * hidden_size}}; // B - - output_shapes = shape_inference(op.get(), input_shapes); - EXPECT_EQ(output_shapes.size(), 3); - EXPECT_EQ(output_shapes[0], StaticShape({batch_size, num_directions, seq_len, hidden_size})); - EXPECT_EQ(output_shapes[1], StaticShape({batch_size, num_directions, hidden_size})); - EXPECT_EQ(output_shapes[2], StaticShape({batch_size, num_directions, hidden_size})); -} - class LSTMSequenceV5StaticShapeInferenceTest : public OpStaticShapeInferenceTest { protected: void SetUp() override { diff --git a/src/plugins/template/backend/evaluates_map.cpp b/src/plugins/template/backend/evaluates_map.cpp index 134decae3badce..3c9e008ccb5ee0 100644 --- a/src/plugins/template/backend/evaluates_map.cpp +++ b/src/plugins/template/backend/evaluates_map.cpp @@ -105,7 +105,6 @@ std::vector get_signal_size(const ov::TensorVector& inputs, size_t num_ } ov::runtime::interpreter::EvaluatorsMap& ov::runtime::interpreter::get_evaluators_map() { - OPENVINO_SUPPRESS_DEPRECATED_START static runtime::interpreter::EvaluatorsMap evaluatorsMap{ #define _OPENVINO_OP_REG(NAME, NAMESPACE) {NAMESPACE::NAME::get_type_info_static(), evaluate_node}, @@ -113,6 +112,5 @@ ov::runtime::interpreter::EvaluatorsMap& ov::runtime::interpreter::get_evaluator #undef _OPENVINO_OP_REG }; - OPENVINO_SUPPRESS_DEPRECATED_END return evaluatorsMap; } diff --git a/src/plugins/template/backend/ops/ops_evaluates.hpp b/src/plugins/template/backend/ops/ops_evaluates.hpp index 54d2e0cb7c8c63..e63c7aa8b4f9e4 100644 --- a/src/plugins/template/backend/ops/ops_evaluates.hpp +++ b/src/plugins/template/backend/ops/ops_evaluates.hpp @@ -77,12 +77,6 @@ extern template bool evaluate_node(std::shared_ptr(std::shared_ptr node, - ov::TensorVector& outputs, - const ov::TensorVector& inputs); -OPENVINO_SUPPRESS_DEPRECATED_END - extern template bool evaluate_node(std::shared_ptr node, ov::TensorVector& outputs, const ov::TensorVector& inputs); @@ -555,4 +549,4 @@ extern template bool evaluate_node(std::shared_pt extern template bool evaluate_node(std::shared_ptr node, ov::TensorVector& outputs, - const ov::TensorVector& inputs); \ No newline at end of file + const ov::TensorVector& inputs); diff --git a/src/plugins/template/backend/ops/sequences.cpp b/src/plugins/template/backend/ops/sequences.cpp index 633ec5bcab3329..c8e57c846c8b73 100644 --- a/src/plugins/template/backend/ops/sequences.cpp +++ b/src/plugins/template/backend/ops/sequences.cpp @@ -54,44 +54,6 @@ bool evaluate(const std::shared_ptr& op, return true; } -namespace lstm_seq_v1 { -OPENVINO_SUPPRESS_DEPRECATED_START -template -inline void evaluate(const std::shared_ptr& op, - ov::TensorVector& outputs, - const ov::TensorVector& inputs) { - OPENVINO_SUPPRESS_DEPRECATED_END - using T1 = typename ov::element_type_traits::value_type; - using T2 = typename ov::element_type_traits::value_type; - ov::reference::lstm_sequence_v1(static_cast(inputs[0].data()), - inputs[0].get_shape(), - static_cast(inputs[1].data()), - inputs[1].get_shape(), - static_cast(inputs[2].data()), - inputs[2].get_shape(), - static_cast(inputs[3].data()), - inputs[3].get_shape(), - static_cast(inputs[4].data()), - inputs[4].get_shape(), - static_cast(inputs[5].data()), - inputs[5].get_shape(), - static_cast(inputs[6].data()), - inputs[6].get_shape(), - static_cast(inputs[7].data()), - inputs[7].get_shape(), - static_cast(outputs[0].data()), - static_cast(outputs[1].data()), - static_cast(outputs[2].data()), - op->get_activations()[0], - op->get_activations()[1], - op->get_activations()[2], - op->get_clip_threshold(), - op->get_weights_format(), - op->get_input_forget(), - op->get_direction()); -} -} // namespace lstm_seq_v1 - namespace lstm_seq_v5 { template inline void evaluate(const std::shared_ptr& op, @@ -124,27 +86,6 @@ inline void evaluate(const std::shared_ptr& op, } } // namespace lstm_seq_v5 -OPENVINO_SUPPRESS_DEPRECATED_START -template -bool evaluate(const std::shared_ptr& op, - ov::TensorVector& outputs, - const ov::TensorVector& inputs) { - OPENVINO_SUPPRESS_DEPRECATED_END - switch (inputs[3].get_element_type()) { - case ov::element::i64: - case ov::element::u64: - lstm_seq_v1::evaluate(op, outputs, inputs); - break; - case ov::element::i32: - case ov::element::u32: - lstm_seq_v1::evaluate(op, outputs, inputs); - break; - default: - return false; - } - return true; -} - template bool evaluate(const std::shared_ptr& op, ov::TensorVector& outputs, @@ -402,55 +343,6 @@ bool evaluate_node(std::shared_ptr node, } } -OPENVINO_SUPPRESS_DEPRECATED_START -template <> -bool evaluate_node(std::shared_ptr node, - ov::TensorVector& outputs, - const ov::TensorVector& inputs) { - auto element_type = node->get_output_element_type(0); - if (ov::is_type(node) || ov::is_type(node)) - element_type = node->get_input_element_type(1); - - switch (element_type) { - case ov::element::boolean: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::bf16: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::f16: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::f64: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::f32: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::i4: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::i8: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::i16: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::i32: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::i64: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::u1: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::u4: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::u8: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::u16: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::u32: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - case ov::element::u64: - return evaluate(ov::as_type_ptr(node), outputs, inputs); - default: - OPENVINO_THROW(std::string("Unhandled data type ") + node->get_element_type().get_type_name() + - std::string("in evaluate_node()")); - } -} -OPENVINO_SUPPRESS_DEPRECATED_END - template <> bool evaluate_node(std::shared_ptr node, ov::TensorVector& outputs, diff --git a/src/plugins/template/backend/opset_int_tbl.hpp b/src/plugins/template/backend/opset_int_tbl.hpp index 5f4e5737fcb567..42dfa2797031f1 100644 --- a/src/plugins/template/backend/opset_int_tbl.hpp +++ b/src/plugins/template/backend/opset_int_tbl.hpp @@ -22,7 +22,6 @@ _OPENVINO_OP_REG(HardSigmoid, op::v0) _OPENVINO_OP_REG(Interpolate, op::v0) _OPENVINO_OP_REG(LRN, op::v0) _OPENVINO_OP_REG(LSTMCell, op::v0) -_OPENVINO_OP_REG(LSTMSequence, op::v0) _OPENVINO_OP_REG(MVN, op::v0) _OPENVINO_OP_REG(NormalizeL2, op::v0) _OPENVINO_OP_REG(PriorBox, op::v0) diff --git a/src/plugins/template/tests/functional/op_reference/lstm_sequence.cpp b/src/plugins/template/tests/functional/op_reference/lstm_sequence.cpp index fe9ad2ddde743f..999c7f5c6692ac 100644 --- a/src/plugins/template/tests/functional/op_reference/lstm_sequence.cpp +++ b/src/plugins/template/tests/functional/op_reference/lstm_sequence.cpp @@ -241,90 +241,10 @@ class ReferenceLSTMSequenceTest : public testing::TestWithParam, public CommonReferenceTest { -public: - void SetUp() override { - legacy_compare = true; - auto params = GetParam(); - function = CreateFunction(params); - inputData = {params.X.data, - params.H_t.data, - params.C_t.data, - params.S_t.data, - params.W.data, - params.R.data, - params.B.data, - params.P.data}; - refOutData = {params.Y.data, params.Ho.data, params.Co.data}; - } - - static std::string getTestCaseName(const testing::TestParamInfo& obj) { - auto param = obj.param; - std::ostringstream result; - result << "iType=" << param.iType << "_"; - result << "xShape=" << param.X.shape << "_"; - result << "htShape=" << param.H_t.shape << "_"; - result << "ctShape=" << param.C_t.shape << "_"; - result << "stShape=" << param.S_t.shape << "_"; - result << "wShape=" << param.W.shape << "_"; - result << "rShape=" << param.R.shape << "_"; - result << "bShape=" << param.B.shape << "_"; - result << "pShape=" << param.P.shape << "_"; - result << "YShape=" << param.Y.shape << "_"; - result << "hoShape=" << param.Ho.shape << "_"; - result << "coShape=" << param.Co.shape << "_"; - result << "clip=" << param.clip << "_"; - result << "input_forget=" << param.input_forget << "_"; - result << "LSTMdirection=" << param.lstm_direction; - if (!param.testcaseName.empty()) - result << "_" << param.testcaseName; - - return result.str(); - } - -private: - static std::shared_ptr CreateFunction(const LSTMSequenceV1Params& params) { - const auto X = std::make_shared(params.X.type, params.X.shape); - const auto H_t = std::make_shared(params.H_t.type, params.H_t.shape); - const auto C_t = std::make_shared(params.C_t.type, params.C_t.shape); - const auto S_t = std::make_shared(params.S_t.type, params.S_t.shape); - const auto W = std::make_shared(params.W.type, params.W.shape); - const auto R = std::make_shared(params.R.type, params.R.shape); - const auto B = std::make_shared(params.B.type, params.B.shape); - const auto P = std::make_shared(params.P.type, params.P.shape); - - const auto lstm_sequence = - std::make_shared(X, - H_t, - C_t, - S_t, - W, - R, - B, - P, - params.hiddenSize, - params.lstm_direction, - ov::op::LSTMWeightsFormat::FICO, - std::vector{}, - std::vector{}, - std::vector{"sigmoid", "tanh", "tanh"}, - params.clip, - params.input_forget); - - auto function = - std::make_shared(lstm_sequence->outputs(), ParameterVector{X, H_t, C_t, S_t, W, R, B, P}); - return function; - } -}; - TEST_P(ReferenceLSTMSequenceTest, CompareWithRefs) { Exec(); } -TEST_P(ReferenceLSTMSequenceV1Test, CompareWithRefs) { - Exec(); -} - template std::vector generateParams() { using T = typename element_type_traits::value_type; @@ -3481,10 +3401,4 @@ INSTANTIATE_TEST_SUITE_P(smoke_LSTMSequence_With_Hardcoded_Refs, ReferenceLSTMSequenceTest, testing::ValuesIn(generateCombinedParams()), ReferenceLSTMSequenceTest::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P(smoke_LSTMSequence_With_Hardcoded_Refs, - ReferenceLSTMSequenceV1Test, - testing::ValuesIn(generateV1CombinedParams()), - ReferenceLSTMSequenceV1Test::getTestCaseName); - } // namespace diff --git a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/single_op_graph.cpp b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/single_op_graph.cpp index bcb259cd49b60f..fb192e0ac40223 100644 --- a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/single_op_graph.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/single_op_graph.cpp @@ -2017,22 +2017,6 @@ std::shared_ptr generateRNNCellBase(const std::shared_ptr ov::op::RecurrentSequenceDirection::FORWARD); ov::ResultVector results{std::make_shared(gru_sequence)}; return std::make_shared(results, params, "GRUSequence"); - } else if (ov::is_type(node)) { - ov::ParameterVector params{std::make_shared(ov::element::f32, ov::Shape{{5, 10, 10}}), - std::make_shared(ov::element::f32, ov::Shape{{5, 1, 10}}), - std::make_shared(ov::element::f32, ov::Shape{{5, 1, 10}}), - std::make_shared(ov::element::i64, ov::Shape{5})}; - - const auto W = std::make_shared(utils::create_and_fill_tensor(ov::element::f32, ov::Shape{1, 40, 10})); - const auto R = std::make_shared(utils::create_and_fill_tensor(ov::element::f32, ov::Shape{1, 40, 10})); - const auto B = std::make_shared(utils::create_and_fill_tensor(ov::element::f32, ov::Shape{1, 40})); - const auto P = std::make_shared(utils::create_and_fill_tensor(ov::element::f32, ov::Shape{1, 30})); - RNNCellBaseNode = std::make_shared(params.at(0), params.at(1), params.at(2), params.at(3), - W, R, B, 10, ov::op::RecurrentSequenceDirection::FORWARD); - ov::ResultVector results{std::make_shared(RNNCellBaseNode->output(0)), - std::make_shared(RNNCellBaseNode->output(1)), - std::make_shared(RNNCellBaseNode->output(2))}; - return std::make_shared(results, params, "LSTMSeq1BaseGraph"); } else if (ov::is_type(node)) { ov::ParameterVector params{std::make_shared(ov::element::f32, ov::Shape{{5, 10, 10}}), std::make_shared(ov::element::f32, ov::Shape{{5, 1, 10}}), From 2725b163623bca9d3ba37eb1261e21cb9812031e Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Tue, 5 Nov 2024 14:30:58 +0100 Subject: [PATCH 10/93] [core] Remove legacy tensor name (#27307) ### Details: - Remove legacy name from tensor descriptor ### Related PR: - #26855 ### Tickets: - CVS-156182 --------- Signed-off-by: Pawel Raasz --- .../include/transformations/utils/utils.hpp | 16 +++------ .../control_flow/unroll_if.cpp | 7 ---- .../control_flow/unroll_tensor_iterator.cpp | 25 +------------- .../src/transformations/convert_precision.cpp | 6 ---- .../convert_maxpool_downgrade.cpp | 4 --- .../transpose_sinking/ts_utils.cpp | 6 ---- .../unroll_tensor_iterator_test.cpp | 32 ++++++++--------- .../tests/utils/convert_precision.cpp | 10 ------ .../openvino/core/descriptor_tensor.hpp | 9 ----- .../openvino/core/descriptor/tensor.hpp | 3 -- src/core/src/descriptor/tensor.cpp | 13 ------- src/core/src/graph_util.cpp | 20 ----------- src/core/src/node.cpp | 4 --- src/core/src/node_output.cpp | 19 ----------- src/core/src/preprocess/preprocess_impls.cpp | 7 ---- .../onnx/frontend/src/core/graph.cpp | 4 --- src/plugins/intel_cpu/src/utils/serialize.cpp | 34 +++++-------------- .../src/subgraph_simple.cpp | 8 ----- 18 files changed, 28 insertions(+), 199 deletions(-) diff --git a/src/common/transformations/include/transformations/utils/utils.hpp b/src/common/transformations/include/transformations/utils/utils.hpp index fed812ecc864ca..6bcc50e29b3733 100644 --- a/src/common/transformations/include/transformations/utils/utils.hpp +++ b/src/common/transformations/include/transformations/utils/utils.hpp @@ -64,18 +64,10 @@ inline bool has_decompression_converts(const std::shared_ptr& f OPENVINO_DEPRECATED("Plugins should use ov::ISyncInferRequest::find_port") inline std::string create_ie_output_name(const Output& output) { - std::string out_name; - OPENVINO_SUPPRESS_DEPRECATED_START - auto tensor_name = ov::descriptor::get_ov_tensor_legacy_name(output.get_tensor()); - OPENVINO_SUPPRESS_DEPRECATED_END - if (!tensor_name.empty()) { - out_name = std::move(tensor_name); - } else { - const auto& prev_layer = output.get_node_shared_ptr(); - out_name = prev_layer->get_friendly_name(); - if (prev_layer->get_output_size() != 1) { - out_name += "." + std::to_string(output.get_index()); - } + const auto& prev_layer = output.get_node_shared_ptr(); + auto out_name = prev_layer->get_friendly_name(); + if (prev_layer->get_output_size() != 1) { + out_name += "." + std::to_string(output.get_index()); } return out_name; } diff --git a/src/common/transformations/src/transformations/control_flow/unroll_if.cpp b/src/common/transformations/src/transformations/control_flow/unroll_if.cpp index 6ef5536495e663..14d28a3a3fcfa3 100644 --- a/src/common/transformations/src/transformations/control_flow/unroll_if.cpp +++ b/src/common/transformations/src/transformations/control_flow/unroll_if.cpp @@ -54,14 +54,7 @@ bool ov::pass::UnrollIf::run_on_model(const std::shared_ptr& f) { } for (const auto& output_desc : output_descriptions) { std::shared_ptr result = body->get_results()[output_desc->m_body_value_index]; - const auto& in_value = result->input_value(0); - // set output name to Tensor to store it for openvino to cnn conversion - OPENVINO_SUPPRESS_DEPRECATED_START - ov::descriptor::set_ov_tensor_legacy_name( - in_value.get_tensor(), - op::util::create_ie_output_name(if_node->output(output_desc->m_output_index))); - OPENVINO_SUPPRESS_DEPRECATED_END for (const auto& input : if_node->output(output_desc->m_output_index).get_target_inputs()) { input.replace_source_output(result->get_input_source_output(0)); } diff --git a/src/common/transformations/src/transformations/control_flow/unroll_tensor_iterator.cpp b/src/common/transformations/src/transformations/control_flow/unroll_tensor_iterator.cpp index 431527ade998a1..dde1ab35898b67 100644 --- a/src/common/transformations/src/transformations/control_flow/unroll_tensor_iterator.cpp +++ b/src/common/transformations/src/transformations/control_flow/unroll_tensor_iterator.cpp @@ -115,21 +115,6 @@ bool ov::pass::UnrollTensorIterator::run_on_model(const std::shared_ptrget_output_descriptions()) { - // we need to insert tensor_name to the outputs of TensorIterator if they directly connected to - // Results ops. It's necessary to save original TensorIterator name when we use CNNNetwork. - auto insert_tensor_name = [&](const ov::Output& ti_output, const ov::Output& insert_to) { - auto target_inputs = ti_output.get_target_inputs(); - if (target_inputs.empty() || - std::any_of(target_inputs.begin(), target_inputs.end(), [](const ov::Input& target_inp) { - return ov::as_type(target_inp.get_node()) != nullptr; - })) { - OPENVINO_SUPPRESS_DEPRECATED_START - ov::descriptor::set_ov_tensor_legacy_name(insert_to.get_tensor(), - ov::op::util::create_ie_output_name(ti_output)); - OPENVINO_SUPPRESS_DEPRECATED_END - } - }; - if (const auto& concat_desc = std::dynamic_pointer_cast(desc)) { if (!concat_desc) { @@ -155,9 +140,6 @@ bool ov::pass::UnrollTensorIterator::run_on_model(const std::shared_ptr(to_concat, concat_desc->m_axis); copy_runtime_info(sub_graph_op, concat); - // set output name to Tensor to store it for openvino to cnn conversion - insert_tensor_name(sub_graph_op->output(concat_desc->m_output_index), concat->output(0)); - // connect the Concat layer to the corresponding TI outputs for (auto& input : sub_graph_op->output(concat_desc->m_output_index).get_target_inputs()) { input.replace_source_output(concat); @@ -167,8 +149,6 @@ bool ov::pass::UnrollTensorIterator::run_on_model(const std::shared_ptr result = body_functions[0]->get_results().at(concat_desc->m_body_value_index); const auto& input_to_res = result->get_input_source_output(0); - // set output name to Tensor to store it for openvino to cnn conversion - insert_tensor_name(sub_graph_op->output(concat_desc->m_output_index), input_to_res); for (auto& input : sub_graph_op->output(concat_desc->m_output_index).get_target_inputs()) { input.replace_source_output(input_to_res); @@ -179,11 +159,8 @@ bool ov::pass::UnrollTensorIterator::run_on_model(const std::shared_ptrm_iteration; iter = iter >= 0 ? iter : num_iter - 1; - std::shared_ptr result = - body_functions[iter]->get_results()[output_desc->m_body_value_index]; - const auto& in_value = result->input_value(0); + auto result = body_functions[iter]->get_results()[output_desc->m_body_value_index]; - insert_tensor_name(sub_graph_op->output(output_desc->m_output_index), in_value); for (const auto& input : sub_graph_op->output(output_desc->m_output_index).get_target_inputs()) { input.replace_source_output(result->get_input_source_output(0)); } diff --git a/src/common/transformations/src/transformations/convert_precision.cpp b/src/common/transformations/src/transformations/convert_precision.cpp index 3ab2c694be40ef..d8fd21699a5c20 100644 --- a/src/common/transformations/src/transformations/convert_precision.cpp +++ b/src/common/transformations/src/transformations/convert_precision.cpp @@ -332,12 +332,6 @@ bool convert_function_precision(const std::shared_ptr& f, auto& convert_output_tensor = convert->get_output_tensor(0); convert_output_tensor.set_names(result_input.get_names()); - OPENVINO_SUPPRESS_DEPRECATED_START - const auto& legacy_name = ov::descriptor::get_ov_tensor_legacy_name(result_input.get_tensor()); - if (!legacy_name.empty()) { - ov::descriptor::set_ov_tensor_legacy_name(convert_output_tensor, legacy_name); - } - OPENVINO_SUPPRESS_DEPRECATED_END result_input.set_names({}); result->input(0).replace_source_output(convert->output(0)); diff --git a/src/common/transformations/src/transformations/op_conversions/convert_maxpool_downgrade.cpp b/src/common/transformations/src/transformations/op_conversions/convert_maxpool_downgrade.cpp index 50a49202742bde..0e0865b3845f48 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_maxpool_downgrade.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_maxpool_downgrade.cpp @@ -57,10 +57,6 @@ ov::pass::ConvertMaxPool8ToMaxPool1::ConvertMaxPool8ToMaxPool1() { ov::copy_runtime_info(maxpool_v8_node, maxpool_v1_node); maxpool_v8_node->clear_control_dependencies(); - OPENVINO_SUPPRESS_DEPRECATED_START - ov::descriptor::set_ov_tensor_legacy_name(maxpool_v1_node->output(0).get_tensor(), out_name); - OPENVINO_SUPPRESS_DEPRECATED_END - return true; }; diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_utils.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_utils.cpp index ba950ad93bf501..ae40ca869c7dcc 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_utils.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_utils.cpp @@ -122,12 +122,6 @@ void SwapOutputNames(Output output1, Output output2) { const auto node2_output_names = output2.get_names(); output2.set_names(output1.get_names()); output1.set_names(node2_output_names); - - OPENVINO_SUPPRESS_DEPRECATED_START - const auto node2_legacy_output_names = get_ov_tensor_legacy_name(output2.get_tensor()); - set_ov_tensor_legacy_name(output2.get_tensor(), get_ov_tensor_legacy_name(output1.get_tensor())); - set_ov_tensor_legacy_name(output1.get_tensor(), node2_legacy_output_names); - OPENVINO_SUPPRESS_DEPRECATED_END } void SwapFriendlyNames(const NodePtr& node1, const NodePtr& node2) { diff --git a/src/common/transformations/tests/control_flow/unroll_tensor_iterator_test.cpp b/src/common/transformations/tests/control_flow/unroll_tensor_iterator_test.cpp index 9ff2be2e754e95..fb4891d1590cc7 100644 --- a/src/common/transformations/tests/control_flow/unroll_tensor_iterator_test.cpp +++ b/src/common/transformations/tests/control_flow/unroll_tensor_iterator_test.cpp @@ -491,14 +491,12 @@ TEST(TransformationTests, UnrollTensorIteratorLSTMCellSingleIterationSingleItera ASSERT_TRUE(res.first) << res.second; } -void collect_legacy_tensor_names(const std::shared_ptr& model, std::vector& holder) { +void collect_tensor_names(const std::shared_ptr& model, + std::vector>& holder) { for (const auto& op : model->get_ordered_ops()) { for (const auto& out : op->outputs()) { - OPENVINO_SUPPRESS_DEPRECATED_START - auto tensor_name = ov::descriptor::get_ov_tensor_legacy_name(out.get_tensor()); - OPENVINO_SUPPRESS_DEPRECATED_END - if (!tensor_name.empty() && ov::as_type_ptr(op)) - holder.emplace_back(tensor_name); + if (!out.get_tensor().get_names().empty() && ov::as_type_ptr(op)) + holder.emplace_back(out.get_tensor().get_names()); } } } @@ -538,8 +536,8 @@ TEST(TransformationTests, CheckTensorNamesAfterConvertToTIAndUnrolling) { f = std::make_shared(NodeVector{Y_out, Ho}, ParameterVector{X, Y}); } - std::vector names_before; - collect_legacy_tensor_names(f, names_before); + std::vector> names_before; + collect_tensor_names(f, names_before); pass::Manager m; m.register_pass(); @@ -548,8 +546,8 @@ TEST(TransformationTests, CheckTensorNamesAfterConvertToTIAndUnrolling) { m.run_passes(f); OV_ASSERT_NO_THROW(check_rt_info(f)); - std::vector names_after; - collect_legacy_tensor_names(f, names_after); + std::vector> names_after; + collect_tensor_names(f, names_after); EXPECT_EQ(names_before, names_after); } @@ -605,8 +603,8 @@ TEST(TransformationTests, CheckTensorNamesAfterUnrolling) { f = std::make_shared(NodeVector{res_ti_1, res_ti_2}, ParameterVector{X, Y, Z}); } - std::vector names_before; - collect_legacy_tensor_names(f, names_before); + std::vector> names_before; + collect_tensor_names(f, names_before); pass::Manager m; m.register_pass(); @@ -614,11 +612,9 @@ TEST(TransformationTests, CheckTensorNamesAfterUnrolling) { m.run_passes(f); OV_ASSERT_NO_THROW(check_rt_info(f)); - std::vector names_after; - collect_legacy_tensor_names(f, names_after); + std::vector> names_after; + collect_tensor_names(f, names_after); - EXPECT_NE(names_before, names_after); - EXPECT_EQ(names_after.size(), 2); - EXPECT_EQ(names_after[0], "TensorIterator.0"); - EXPECT_EQ(names_after[1], "TensorIterator.1"); + ASSERT_EQ(names_after.size(), 0); + EXPECT_EQ(names_before, names_after); } diff --git a/src/common/transformations/tests/utils/convert_precision.cpp b/src/common/transformations/tests/utils/convert_precision.cpp index 2aa4d4d2fac9e9..318f15ab1a64dc 100644 --- a/src/common/transformations/tests/utils/convert_precision.cpp +++ b/src/common/transformations/tests/utils/convert_precision.cpp @@ -2304,11 +2304,6 @@ TEST(TransformationTests, ConvertPrecisionExplicitConvertsSingleNodeMultipleOutp split->get_output_tensor(0).add_names({"split:0"}); split->get_output_tensor(1).add_names({"split:1"}); split->get_output_tensor(2).add_names({"split:2"}); - OPENVINO_SUPPRESS_DEPRECATED_START - ov::descriptor::set_ov_tensor_legacy_name(split->get_output_tensor(0), "legacy_split:0"); - ov::descriptor::set_ov_tensor_legacy_name(split->get_output_tensor(1), "legacy_split:1"); - ov::descriptor::set_ov_tensor_legacy_name(split->get_output_tensor(2), "legacy_split:2"); - OPENVINO_SUPPRESS_DEPRECATED_END model = make_shared(split->outputs(), ParameterVector{param_1}); type_to_fuse_map empty_type_to_fuse_map = {}; @@ -2345,11 +2340,6 @@ TEST(TransformationTests, ConvertPrecisionExplicitConvertsSingleNodeMultipleOutp ASSERT_EQ("split.0", results[0]->get_input_node_ptr(0)->get_friendly_name()); ASSERT_EQ("split.1", results[1]->get_input_node_ptr(0)->get_friendly_name()); ASSERT_EQ("split.2", results[2]->get_input_node_ptr(0)->get_friendly_name()); - OPENVINO_SUPPRESS_DEPRECATED_START - ASSERT_EQ("legacy_split:0", ov::descriptor::get_ov_tensor_legacy_name(results[0]->get_input_tensor(0))); - ASSERT_EQ("legacy_split:1", ov::descriptor::get_ov_tensor_legacy_name(results[1]->get_input_tensor(0))); - ASSERT_EQ("legacy_split:2", ov::descriptor::get_ov_tensor_legacy_name(results[2]->get_input_tensor(0))); - OPENVINO_SUPPRESS_DEPRECATED_END } TEST(TransformationTests, ConvertPrecisionExplicitConvertsMultiSubgraphs) { diff --git a/src/core/dev_api/openvino/core/descriptor_tensor.hpp b/src/core/dev_api/openvino/core/descriptor_tensor.hpp index 4c7c0ad8c3c972..9418183a1189fc 100644 --- a/src/core/dev_api/openvino/core/descriptor_tensor.hpp +++ b/src/core/dev_api/openvino/core/descriptor_tensor.hpp @@ -14,14 +14,5 @@ void set_element_type(Tensor& tensor, const element::Type& elemenet_type); // To change Tensor type please change the Parameter type. OPENVINO_API void set_tensor_type(Tensor& tensor, const element::Type& element_type, const PartialShape& pshape); - -OPENVINO_DEPRECATED("get_ov_tensor_legacy_name() is deprecated. Please don't use this function.") -OPENVINO_API -std::string get_ov_tensor_legacy_name(const Tensor& tensor); - -OPENVINO_DEPRECATED("set_ov_tensor_legacy_name() is deprecated. Please don't use this function.") -OPENVINO_API -void set_ov_tensor_legacy_name(Tensor& tensor, const std::string& tensor_name); - } // namespace descriptor } // namespace ov diff --git a/src/core/include/openvino/core/descriptor/tensor.hpp b/src/core/include/openvino/core/descriptor/tensor.hpp index edf2e690e8c860..a1cac56fa64e01 100644 --- a/src/core/include/openvino/core/descriptor/tensor.hpp +++ b/src/core/include/openvino/core/descriptor/tensor.hpp @@ -98,14 +98,11 @@ class OPENVINO_API Tensor { PartialShape m_partial_shape; ov::Tensor m_lower_value, m_upper_value; TensorSymbol m_value_symbol; - std::string m_legacy_name; std::unordered_set m_names; std::unordered_set::const_iterator m_name_it; RTMap m_rt_info; - friend OPENVINO_API std::string get_ov_tensor_legacy_name(const Tensor& tensor); - friend OPENVINO_API void set_ov_tensor_legacy_name(Tensor& tensor, const std::string& tensor_name); friend OPENVINO_API void set_element_type(Tensor& tensor, const element::Type& elemenet_type); friend OPENVINO_API void set_tensor_type(Tensor& tensor, const element::Type& element_type, diff --git a/src/core/src/descriptor/tensor.cpp b/src/core/src/descriptor/tensor.cpp index 31509605003fdb..ae3f7c6e77cd4f 100644 --- a/src/core/src/descriptor/tensor.cpp +++ b/src/core/src/descriptor/tensor.cpp @@ -116,18 +116,9 @@ void ov::descriptor::Tensor::clone_from(const ov::descriptor::Tensor& old) { m_lower_value = old.get_lower_value(); m_upper_value = old.get_upper_value(); m_value_symbol = old.get_value_symbol(); - m_legacy_name = old.m_legacy_name; m_rt_info = old.get_rt_info(); } -std::string ov::descriptor::get_ov_tensor_legacy_name(const ov::descriptor::Tensor& tensor) { - return tensor.m_legacy_name; -} - -void ov::descriptor::set_ov_tensor_legacy_name(ov::descriptor::Tensor& tensor, const std::string& tensor_name) { - tensor.m_legacy_name = tensor_name; -} - void ov::descriptor::set_tensor_type(ov::descriptor::Tensor& tensor, const element::Type& element_type, const PartialShape& pshape) { @@ -148,10 +139,6 @@ std::ostream& ov::descriptor::operator<<(std::ostream& out, const ov::descriptor names += ", "; names += name; } - OPENVINO_SUPPRESS_DEPRECATED_START - if (names.empty()) - names = get_ov_tensor_legacy_name(tensor); - OPENVINO_SUPPRESS_DEPRECATED_END out << "Tensor(" << names << ")"; return out; } diff --git a/src/core/src/graph_util.cpp b/src/core/src/graph_util.cpp index 17780f7751d52e..4a8f4598f55cc3 100644 --- a/src/core/src/graph_util.cpp +++ b/src/core/src/graph_util.cpp @@ -273,36 +273,16 @@ bool replace_output_update_name(Output output, const Output& replace }); }; - bool preserve_legacy_output_name = false; if (has_result_consumers(output)) { - preserve_legacy_output_name = true; if (output.get_node()->get_output_size() != 1 || replacement.get_node()->get_output_size() != 1 || is_type(replacement.get_node()) || has_result_consumers(replacement)) { return false; } - } - - OPENVINO_SUPPRESS_DEPRECATED_START - if (preserve_legacy_output_name) { replacement.get_node()->set_friendly_name(output.get_node()->get_friendly_name()); - // Update output tensor name - const auto& output_tensor_name = ov::descriptor::get_ov_tensor_legacy_name(output.get_tensor()); - if (!output_tensor_name.empty()) { - ov::descriptor::set_ov_tensor_legacy_name(replacement.get_tensor(), output_tensor_name); - } else { - ov::descriptor::set_ov_tensor_legacy_name(replacement.get_tensor(), output.get_node()->get_friendly_name()); - } } - // Save replacement tensor name before replacement as they will be overridden by the output tensor name - const auto tensor_name = ov::descriptor::get_ov_tensor_legacy_name(replacement.get_tensor()); - output.replace(replacement); - // Restore back original replacement tensor name - ov::descriptor::set_ov_tensor_legacy_name(replacement.get_tensor(), tensor_name); - OPENVINO_SUPPRESS_DEPRECATED_END - copy_runtime_info({replacement.get_node_shared_ptr(), output.get_node_shared_ptr()}, replacement.get_node_shared_ptr()); return true; diff --git a/src/core/src/node.cpp b/src/core/src/node.cpp index 8b9936b5496e7c..ec9197a5a337cb 100644 --- a/src/core/src/node.cpp +++ b/src/core/src/node.cpp @@ -157,10 +157,6 @@ std::shared_ptr ov::Node::copy_with_new_inputs( } for (size_t i = 0; i < get_output_size(); i++) { clone->get_output_tensor(i).set_names(get_output_tensor(i).get_names()); - OPENVINO_SUPPRESS_DEPRECATED_START - ov::descriptor::set_ov_tensor_legacy_name(clone->get_output_tensor(i), - ov::descriptor::get_ov_tensor_legacy_name(get_output_tensor(i))); - OPENVINO_SUPPRESS_DEPRECATED_END } return clone; } diff --git a/src/core/src/node_output.cpp b/src/core/src/node_output.cpp index 3608abe13f3953..a2ea865b2ec84e 100644 --- a/src/core/src/node_output.cpp +++ b/src/core/src/node_output.cpp @@ -73,25 +73,6 @@ void Output::replace(const Output& replacement) { input.replace_source_output(replacement); } replacement.get_tensor_ptr()->add_names(get_tensor_ptr()->get_names()); - OPENVINO_SUPPRESS_DEPRECATED_START - // In legacy API we rely on output port tensor name and use it as an input or output name for the model - // Due to m_name is just a string, and we can't store multiple aliases for single output port we have to - // handle two situations during replacement: - // 1. When we replace consumers to Parameter output port we can't change its name, so we skip this part - // 2. In other cases when we replace consumers to another output port we should set name. For example: - // if we eliminate Node2 from Node1->Node2->Result we have to set Node2 output port name to Node1 - // output port name, so the output name for model won't be changed. - // But there are some cases when output name can not be preserved, so the replacement shouldn't be used: - // 1. Parameter->Node->Result - if we eliminate Node we will lose output name - // 2. Node1-->Node2->Result - if we eliminate Node2 we will lose Result output name - // `->Result - // In both of these cases please use replace_output_update_name() method which automatically prevents the - // replacement for cases when we can not preserve input/output names of model. - if (!is_type(replacement.get_node())) { - ov::descriptor::set_ov_tensor_legacy_name(replacement.get_tensor(), - ov::descriptor::get_ov_tensor_legacy_name(get_tensor())); - } - OPENVINO_SUPPRESS_DEPRECATED_END ov::copy_output_runtime_info({*this, replacement}, {replacement}); } diff --git a/src/core/src/preprocess/preprocess_impls.cpp b/src/core/src/preprocess/preprocess_impls.cpp index 13a4c6f1353312..cbe18a78beb575 100644 --- a/src/core/src/preprocess/preprocess_impls.cpp +++ b/src/core/src/preprocess/preprocess_impls.cpp @@ -385,13 +385,6 @@ void OutputInfo::OutputInfoImpl::build(ov::ResultVector& results) { std::to_string(result->get_input_source_output(0).get_index())); } - OPENVINO_SUPPRESS_DEPRECATED_START - const auto tensor_name = ov::descriptor::get_ov_tensor_legacy_name(result->get_input_tensor(0)); - if (!tensor_name.empty()) { - ov::descriptor::set_ov_tensor_legacy_name(node.get_tensor(), tensor_name); - } - OPENVINO_SUPPRESS_DEPRECATED_END - // Reset friendly name of input node to avoid names collision // when there is at a new node inserted by post-processing steps // If no new nodes are inserted by post-processing, then we need to preserve friendly name of input diff --git a/src/frontends/onnx/frontend/src/core/graph.cpp b/src/frontends/onnx/frontend/src/core/graph.cpp index ad755c958703cb..80ec9c6c9f390c 100644 --- a/src/frontends/onnx/frontend/src/core/graph.cpp +++ b/src/frontends/onnx/frontend/src/core/graph.cpp @@ -446,10 +446,6 @@ void Graph::set_friendly_names(const Node& onnx_node, const ov::OutputVector& ov // null node does not have tensor if (!ov::op::util::is_null(ov_subgraph_outputs[i])) { ov_subgraph_outputs[i].get_tensor().set_names({onnx_node.output(static_cast(i))}); - OPENVINO_SUPPRESS_DEPRECATED_START - ov::descriptor::set_ov_tensor_legacy_name(ov_subgraph_outputs[i].get_tensor(), - onnx_node.output(static_cast(i))); - OPENVINO_SUPPRESS_DEPRECATED_END } } } diff --git a/src/plugins/intel_cpu/src/utils/serialize.cpp b/src/plugins/intel_cpu/src/utils/serialize.cpp index f7fd337afa932e..814e8d19311a8c 100644 --- a/src/plugins/intel_cpu/src/utils/serialize.cpp +++ b/src/plugins/intel_cpu/src/utils/serialize.cpp @@ -20,12 +20,7 @@ void ModelSerializer::operator<<(const std::shared_ptr& model) { auto serialize_info = [&](std::ostream& stream) { pugi::xml_document xml_doc; pugi::xml_node root = xml_doc.append_child("cnndata"); - pugi::xml_node outputs = root.append_child("outputs"); - for (const auto& out : model->get_results()) { - auto out_node = outputs.append_child("out"); - const auto name = ov::descriptor::get_ov_tensor_legacy_name(out->input_value(0).get_tensor()); - out_node.append_attribute("name").set_value(name.c_str()); - } + root.append_child("outputs"); xml_doc.save(stream); }; @@ -44,26 +39,15 @@ ModelDeserializer::ModelDeserializer(std::istream& model_stream, ModelBuilder fn } } -void ModelDeserializer::set_info(pugi::xml_node& root, std::shared_ptr& model) { - pugi::xml_node outputs = root.child("outputs"); - auto nodes_it = outputs.children("out").begin(); - size_t size = model->outputs().size(); - for (size_t i = 0lu; i < size; ++nodes_it, i++) { - std::string name = nodes_it->attribute("name").value(); - if (name.empty()) - continue; - auto result = model->output(i).get_node_shared_ptr(); - ov::descriptor::set_ov_tensor_legacy_name(result->input_value(0).get_tensor(), name); - } -} + void ModelDeserializer::set_info(pugi::xml_node& root, std::shared_ptr& model) {} -void ModelDeserializer::operator>>(std::shared_ptr& model) { - if (auto mmap_buffer = dynamic_cast(m_istream.rdbuf())) { - auto buffer = mmap_buffer->get_buffer(); - process_mmap(model, buffer); - } else { - process_stream(model); - } + void ModelDeserializer::operator>>(std::shared_ptr& model) { + if (auto mmap_buffer = dynamic_cast(m_istream.rdbuf())) { + auto buffer = mmap_buffer->get_buffer(); + process_mmap(model, buffer); + } else { + process_stream(model); + } } void ModelDeserializer::process_mmap(std::shared_ptr& model, diff --git a/src/tests/ov_helpers/ov_snippets_models/src/subgraph_simple.cpp b/src/tests/ov_helpers/ov_snippets_models/src/subgraph_simple.cpp index 12758a90c07652..b15007aca13e0c 100644 --- a/src/tests/ov_helpers/ov_snippets_models/src/subgraph_simple.cpp +++ b/src/tests/ov_helpers/ov_snippets_models/src/subgraph_simple.cpp @@ -218,15 +218,11 @@ std::shared_ptr EltwiseTwoResultsFunction::initOriginal() const { auto relu = std::make_shared(hswish); relu->set_friendly_name("relu"); - OPENVINO_SUPPRESS_DEPRECATED_START auto& out_tensor0 = add->get_output_tensor(0); - ov::descriptor::set_ov_tensor_legacy_name(out_tensor0, "add_out"); out_tensor0.set_names({"add_out", "y0"}); auto& out_tensor1 = relu->get_output_tensor(0); - ov::descriptor::set_ov_tensor_legacy_name(out_tensor1, "relu_out"); out_tensor1.set_names({"relu_out", "y1"}); - OPENVINO_SUPPRESS_DEPRECATED_END auto res0 = std::make_shared(add); res0->set_friendly_name("res0"); @@ -257,15 +253,11 @@ std::shared_ptr EltwiseTwoResultsFunction::initReference() const { std::make_shared(NodeVector{relu}, ParameterVector{indata2})); subgraph1->set_friendly_name("relu"); - OPENVINO_SUPPRESS_DEPRECATED_START auto& out_tensor0 = subgraph0->get_output_tensor(0); - ov::descriptor::set_ov_tensor_legacy_name(out_tensor0, "add_out"); out_tensor0.set_names({"add_out", "y0"}); auto& out_tensor1 = subgraph1->get_output_tensor(0); - ov::descriptor::set_ov_tensor_legacy_name(out_tensor1, "relu_out"); out_tensor1.set_names({"relu_out", "y1"}); - OPENVINO_SUPPRESS_DEPRECATED_END auto res0 = std::make_shared(subgraph0->output(0)); res0->set_friendly_name("res0"); From fd2d6dcc2e9131f1b507adcd5413eefea915d190 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 5 Nov 2024 15:16:01 +0100 Subject: [PATCH 11/93] [GHA] Set commit action versions (#27384) ### Details: - Use commit action versions as security requirement ### Tickets: - *ticket-id* --- .github/actions/handle_docker/action.yml | 8 ++++---- .github/actions/setup_python/action.yml | 2 +- .github/actions/smart-ci/action.yml | 6 +++--- .github/dockerfiles/docker_tag | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/actions/handle_docker/action.yml b/.github/actions/handle_docker/action.yml index 204e4266729c34..f79bc06afa9f5c 100644 --- a/.github/actions/handle_docker/action.yml +++ b/.github/actions/handle_docker/action.yml @@ -27,17 +27,17 @@ runs: using: 'composite' steps: - name: Checkout head - uses: actions/checkout@v4 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Checkout base - uses: actions/checkout@v4 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: ref: ${{ github.base_ref || github.event.merge_group.base_ref }} sparse-checkout: ${{ inputs.dockerfiles_root_dir }}/docker_tag path: base - name: Install Python dependencies - uses: py-actions/py-dependency-install@v4 + uses: py-actions/py-dependency-install@30aa0023464ed4b5b116bd9fbdab87acf01a484e # v4.1.0 with: path: "${{ github.action_path }}/requirements.txt" update-setuptools: "false" @@ -45,7 +45,7 @@ runs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 with: driver-opts: | image=${{ inputs.registry }}/dockerio/moby/buildkit:buildx-stable-1 diff --git a/.github/actions/setup_python/action.yml b/.github/actions/setup_python/action.yml index 0d9138bc643d2a..80e22866170535 100644 --- a/.github/actions/setup_python/action.yml +++ b/.github/actions/setup_python/action.yml @@ -56,7 +56,7 @@ runs: - if: ${{ runner.os == 'macOS' || runner.os == 'Windows' || (runner.os == 'Linux' && runner.arch != 'ARM64' && steps.check_python.outputs.installed == 'false' ) }} name: Setup Python ${{ inputs.version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: ${{ inputs.version }} env: diff --git a/.github/actions/smart-ci/action.yml b/.github/actions/smart-ci/action.yml index 4d772c8f0eeb03..ec65d2f1e9e82c 100644 --- a/.github/actions/smart-ci/action.yml +++ b/.github/actions/smart-ci/action.yml @@ -65,7 +65,7 @@ runs: using: "composite" steps: - name: Wait for labeler to finish - uses: lewagon/wait-on-check-action@v1.3.1 + uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc # v1.3.4 if: ${{ github.event_name == 'pull_request' }} with: ref: ${{ github.event.pull_request.head.sha }} @@ -74,13 +74,13 @@ runs: wait-interval: 10 - name: checkout components file - uses: actions/checkout@v4 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: sparse-checkout: .github/components.yml sparse-checkout-cone-mode: false - name: Install Python dependencies - uses: py-actions/py-dependency-install@v4 + uses: py-actions/py-dependency-install@30aa0023464ed4b5b116bd9fbdab87acf01a484e # v4.1.0 with: path: "${{ github.action_path }}/requirements.txt" update-setuptools: "false" diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag index cdd92779f658e7..5a4f7795ea4a44 100644 --- a/.github/dockerfiles/docker_tag +++ b/.github/dockerfiles/docker_tag @@ -1 +1 @@ -pr-27380 +pr-27384 From 7daf8009109f46596cb018607dc1451d3290518a Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 5 Nov 2024 16:13:53 +0100 Subject: [PATCH 12/93] [GHA] Switch coverity pipeline to internal docker image (#27385) ### Details: - Moved pipeline to ubuntu 20.04 image - Enabled sccache - ### Tickets: -156658 --- .github/workflows/coverity.yml | 80 +++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 11 deletions(-) diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index d87de4257e0270..9797414cde56c8 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -25,7 +25,56 @@ env: TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }} jobs: + + Smart_CI: + runs-on: ubuntu-latest + outputs: + affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + changed_components: "${{ steps.smart_ci.outputs.changed_components }}" + skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" + steps: + - name: checkout action + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + sparse-checkout: .github/actions/smart-ci + + - name: Get affected components + id: smart_ci + uses: ./.github/actions/smart-ci + with: + repository: ${{ github.repository }} + pr: ${{ github.event.number }} + commit_sha: ${{ github.sha }} + ref_name: ${{ github.ref_name }} + component_pattern: "category: (.*)" + repo_token: ${{ secrets.GITHUB_TOKEN }} + skip_when_only_listed_labels_set: 'docs' + skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg' + + Docker: + needs: Smart_CI + runs-on: aks-linux-4-cores-16gb-docker-build + container: + image: openvinogithubactions.azurecr.io/docker_build:0.2 + volumes: + - /mount:/mount + outputs: + images: "${{ steps.handle_docker.outputs.images }}" + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: ./.github/actions/handle_docker + id: handle_docker + with: + images: | + ov_build/ubuntu_20_04_x64 + registry: 'openvinogithubactions.azurecr.io' + dockerfiles_root_dir: '.github/dockerfiles' + changed_components: ${{ needs.smart_ci.outputs.changed_components }} + Build: + needs: Docker timeout-minutes: 150 defaults: run: @@ -33,7 +82,10 @@ jobs: runs-on: aks-linux-16-cores-32gb if: ${{ github.repository_owner == 'openvinotoolkit' }} container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} + volumes: + - /mount:/mount + options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING env: DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input CMAKE_BUILD_TYPE: 'Release' @@ -43,13 +95,16 @@ jobs: OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib BUILD_DIR: /__w/openvino/openvino/openvino_build COVERITY_TOOL_DIR: /__w/openvino/openvino/coverity_tool + CMAKE_CXX_COMPILER_LAUNCHER: sccache + CMAKE_C_COMPILER_LAUNCHER: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: 1 + SCCACHE_SERVER_PORT: 35555 + SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt + SCCACHE_LOG: warn + SCCACHE_AZURE_KEY_PREFIX: coverity_lin + SCCACHE_CACHE_SIZE: 50G steps: - - name: Install git - run: | - apt-get update - apt-get install --assume-yes --no-install-recommends git ca-certificates - - name: Clone OpenVINO uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -86,7 +141,9 @@ jobs: # # Build # - + - name: Clean sccache stats + run: ${SCCACHE_PATH} --zero-stats + - name: CMake configure - OpenVINO run: | cmake \ @@ -108,12 +165,13 @@ jobs: - name: Cmake build - OpenVINO with Coverity run: ${COVERITY_TOOL_DIR}/cov-analysis*/bin/cov-build --dir ${BUILD_DIR}/cov-int cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + + - name: Show sccache stats + run: ${SCCACHE_PATH} --show-stats - name: Pack Artefacts - run: | - pushd ${BUILD_DIR} - tar -cvf - cov-int | pigz > openvino.tgz - popd + run: tar -cvf - cov-int | pigz > openvino.tgz + working-directory: ${{ env.BUILD_DIR }} - name: Submit artefacts run: | From 0a3f439953766ff279cc8fcf5a65c3c3039a8c32 Mon Sep 17 00:00:00 2001 From: Roman Kazantsev Date: Tue, 5 Nov 2024 19:57:52 +0400 Subject: [PATCH 13/93] [TF FE] Check OV and FW outputs shapes and types in layer tests (#27375) **Details:** Check OV and FW outputs shapes and types in layer tests **Ticket:** TBD --------- Signed-off-by: Kazantsev, Roman --- .../src/op/tensor_list_operations.cpp | 9 +- .../tensorflow_common/src/op/tobool.cpp | 52 ++++--- .../src/op/unique_with_counts.cpp | 6 +- .../layer_tests/common/tf_layer_test_class.py | 139 +++++++++++++++++- .../tensorflow_tests/test_tf_Equal.py | 7 +- .../tensorflow_tests/test_tf_ToBool.py | 18 ++- 6 files changed, 192 insertions(+), 39 deletions(-) diff --git a/src/frontends/tensorflow_common/src/op/tensor_list_operations.cpp b/src/frontends/tensorflow_common/src/op/tensor_list_operations.cpp index 5cbf4c92b8a35c..e082be4943977c 100644 --- a/src/frontends/tensorflow_common/src/op/tensor_list_operations.cpp +++ b/src/frontends/tensorflow_common/src/op/tensor_list_operations.cpp @@ -199,8 +199,13 @@ OutputVector translate_tensor_list_length_op(const NodeContext& node) { auto tensor_list_shape = make_shared(input_handle, element::i32); auto list_length = make_shared(tensor_list_shape, zero_const, one_const, one_const); - set_node_name(node.get_name(), list_length); - return {list_length}; + // output of TensorListLength must be a scalar + // after Slice operation it is a 1D tensor with one element + auto scalar_shape = make_shared(element::i32, Shape{0}, std::vector{}); + auto list_length_scalar = make_shared(list_length, scalar_shape, false); + + set_node_name(node.get_name(), list_length_scalar); + return {list_length_scalar}; } OutputVector translate_tensor_list_concat_v2_op(const NodeContext& node) { diff --git a/src/frontends/tensorflow_common/src/op/tobool.cpp b/src/frontends/tensorflow_common/src/op/tobool.cpp index 9f1082d5df87da..9025a35f549799 100644 --- a/src/frontends/tensorflow_common/src/op/tobool.cpp +++ b/src/frontends/tensorflow_common/src/op/tobool.cpp @@ -9,6 +9,8 @@ #include "openvino/op/logical_and.hpp" #include "openvino/op/logical_or.hpp" #include "openvino/op/not_equal.hpp" +#include "openvino/op/range.hpp" +#include "openvino/op/reduce_logical_and.hpp" #include "openvino/op/reduce_prod.hpp" #include "openvino/op/select.hpp" #include "openvino/op/shape_of.hpp" @@ -22,47 +24,49 @@ namespace tensorflow { namespace op { OutputVector translate_tobool_op(const NodeContext& node) { // (rank(x) == 0 && x != 0) || (rank > 0 && ReduceProd(ShapeOf(x))) > 0 - default_op_checks(node, 1, {"ToBool"}); auto x = node.get_input(0); // prepare auxiliary zero and zero constants of the same type as the inputs - auto zero = create_same_type_const_scalar(x, 0); - auto zero_2 = make_shared(element::i32, Shape{}, 0); - auto true_const = make_shared(element::boolean, Shape{}, true); + auto zero_x = create_same_type_const_scalar(x, 0); + auto zero_i64 = make_shared(element::i64, Shape{}, 0); + auto one_i64 = make_shared(element::i64, Shape{}, 1); auto false_const = make_shared(element::boolean, Shape{}, false); // compute a mask to get rank(x) == 0 - auto x_rank = compute_subgraph_scalar_rank(x, element::i32); + auto x_rank = compute_subgraph_scalar_rank(x, element::i64, true); + // 1. try to evaluate if it satisfy non-zero scalar input // compute rank(x) == 0 - auto is_zero = make_shared(x_rank, zero_2); - + auto is_rank_zero = make_shared(x_rank, zero_i64); // compute mask to get x != 0 - auto is_not_zero = make_shared(x, zero); - + auto is_x_not_zero = make_shared(x, zero_x)->output(0); // compute (rank(x) == 0 && x != 0) - auto logical_and = make_shared(is_zero, is_not_zero); - // compute rank(x) > 0 - auto greater_than_zero = make_shared(x_rank, zero_2); + auto scalar_cond = make_shared(is_rank_zero, is_x_not_zero)->output(0); + // generate reduce_axes + auto reduce_axes = make_shared(zero_i64, x_rank, one_i64); + scalar_cond = make_shared(scalar_cond, reduce_axes, false); + // correct result for empty tensor, for which scalar_cond is still equal to True + scalar_cond = make_shared(is_rank_zero, scalar_cond, false_const); + // 2. try to evaluate if it is non-scalar input tensor and not empty tensor + // compute rank(x) > 0 + auto rank_greater_than_zero = make_shared(x_rank, zero_i64); // compute ShapeOf(x) - auto cond_shape = make_shared(x, element::i32); + auto x_shape = make_shared(x, element::i64); // compute ReduceProd(ShapeOf(x))) and axis - auto axis = make_shared(element::i32, Shape{}, 0); - auto reduce_prod = make_shared(cond_shape, axis); - + auto reduce_axis = make_shared(element::i32, Shape{}, 0); + auto num_elems = make_shared(x_shape, reduce_axis, false); // compute ReduceProd(ShapeOf(x))) > 0 - auto greater_than__zero_2 = make_shared(reduce_prod, zero_2); + auto num_elems_greater_than_zero = make_shared(num_elems, zero_i64); // compute (rank > 0 && ReduceProd(ShapeOf(x))) > 0 - auto logical_and_2 = make_shared(greater_than_zero, greater_than__zero_2); - - auto logical_or = make_shared(logical_and, logical_and_2); + // it will be a scalar + auto non_scalar_tensor_not_empty = make_shared(rank_greater_than_zero, num_elems_greater_than_zero); - auto tobool = make_shared(logical_or, true_const, false_const); - set_node_name(node.get_name(), tobool); - return tobool->outputs(); + auto to_bool = make_shared(scalar_cond, non_scalar_tensor_not_empty); + set_node_name(node.get_name(), to_bool); + return to_bool->outputs(); } } // namespace op } // namespace tensorflow } // namespace frontend -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/src/frontends/tensorflow_common/src/op/unique_with_counts.cpp b/src/frontends/tensorflow_common/src/op/unique_with_counts.cpp index 7c5c0a196aa018..d3ca3156ed1347 100644 --- a/src/frontends/tensorflow_common/src/op/unique_with_counts.cpp +++ b/src/frontends/tensorflow_common/src/op/unique_with_counts.cpp @@ -6,6 +6,7 @@ #include "openvino/op/unique.hpp" using namespace std; +using namespace ov; using namespace ov::op; namespace ov { @@ -19,8 +20,9 @@ OutputVector translate_unique_with_counts_op(const NodeContext& node) { // get input 'x' from node and node name auto x = node.get_input(0); auto node_name = node.get_name(); + auto out_idx = node.get_attribute("out_idx", element::i32); - auto unique = make_shared(x, false, ov::element::i32, ov::element::i32); + auto unique = make_shared(x, false, out_idx, out_idx); set_node_name(node_name, unique); // story 'y', 'idx', and 'count' outputs from Unique in separate variables @@ -34,4 +36,4 @@ OutputVector translate_unique_with_counts_op(const NodeContext& node) { } // namespace op } // namespace tensorflow } // namespace frontend -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/tests/layer_tests/common/tf_layer_test_class.py b/tests/layer_tests/common/tf_layer_test_class.py index 99ade1737721fa..98f47af542ab63 100644 --- a/tests/layer_tests/common/tf_layer_test_class.py +++ b/tests/layer_tests/common/tf_layer_test_class.py @@ -1,14 +1,18 @@ # Copyright (C) 2018-2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -from common.layer_test_class import CommonLayerTest +import numpy as np +from common.layer_utils import InferAPI +from common.utils.common_utils import generate_ir_python_api +from common.utils.tf_utils import save_to_pb from common.utils.tf_utils import summarize_graph - from common.utils.tflite_utils import get_tflite_results, save_pb_to_tflite -from common.utils.tf_utils import save_to_pb +from pathlib import Path + +class CommonTFLayerTest: + input_model_key = "input_model" -class CommonTFLayerTest(CommonLayerTest): def prepare_tf_inputs(self, inputs_dict): input = dict() for key in inputs_dict.keys(): @@ -62,3 +66,130 @@ def get_framework_results(self, inputs_dict, model_path): else: # get results from tflite return get_tflite_results(self.use_legacy_frontend, inputs_dict, model_path) + + def _test(self, framework_model, ref_net, ie_device, precision, ir_version, temp_dir, + use_legacy_frontend=False, infer_timeout=60, **kwargs): + model_path = self.produce_model_path(framework_model=framework_model, save_path=temp_dir) + self.use_legacy_frontend = use_legacy_frontend + + compress_to_fp16 = False if precision == 'FP32' else True + + if use_legacy_frontend: + mo_params = {self.input_model_key: model_path, + "output_dir": temp_dir, + "compress_to_fp16": compress_to_fp16, + "model_name": 'model'} + + if 'input_shapes' in kwargs and len(kwargs['input_shapes']): + input_shapes_str = [] + for ishape in kwargs['input_shapes']: + input_shapes_str.append('[' + ','.join([str(i) for i in ishape]) + ']') + mo_params.update(dict(input_shape=','.join(input_shapes_str))) + + if 'input_names' in kwargs and len(kwargs['input_names']): + mo_params.update(dict(input=','.join(kwargs['input_names']))) + mo_params["use_legacy_frontend"] = True + else: + # pack input parameters for convert_model of OVC + # that are different from MO + mo_params = {"input_model": model_path, + "output_dir": temp_dir, + "compress_to_fp16": compress_to_fp16 + } + + if 'input_shapes' in kwargs and 'input_names' in kwargs: + input_shapes = kwargs['input_shapes'] + input_names = kwargs['input_names'] + assert len(input_shapes) == len(input_names) + input_dict = {} + for input_name, input_shape in zip(input_names, input_shapes): + input_dict[input_name] = input_shape + mo_params.update(dict(input=input_dict)) + elif 'input_names' in kwargs: + mo_params.update(dict(input=kwargs['input_names'])) + elif 'input_shapes' in kwargs: + mo_params.update(dict(input=kwargs['input_shapes'])) + + exit_code, stderr = generate_ir_python_api(**mo_params) + assert not exit_code, ( + "IR generation failed with {} exit code: {}".format(exit_code, stderr)) + + path_to_xml = Path(temp_dir, 'model.xml') + path_to_bin = Path(temp_dir, 'model.bin') + + config = None + # GPU default execution precision is FP16, so if we want to check FP32 inference + # we need to set explicit precision hint + if ie_device == 'GPU' and precision == 'FP32': + config = {'INFERENCE_PRECISION_HINT': 'f32'} + + ie_engine = InferAPI(model=path_to_xml, + weights=path_to_bin, + device=ie_device, + use_legacy_frontend=use_legacy_frontend) + # Prepare feed dict + if 'kwargs_to_prepare_input' in kwargs and kwargs['kwargs_to_prepare_input']: + inputs_dict = self._prepare_input(ie_engine.get_inputs_info(precision), + kwargs['kwargs_to_prepare_input']) + else: + inputs_dict = self._prepare_input(ie_engine.get_inputs_info(precision)) + + # Infer using OpenVINO + infer_res = ie_engine.infer(input_data=inputs_dict, infer_timeout=infer_timeout, config=config) + + # Infer using the original framework (TensorFlow or TFLite) + fw_res = self.get_framework_results(inputs_dict=inputs_dict, model_path=model_path) + + if 'custom_eps' in kwargs and kwargs['custom_eps'] is not None: + custom_eps = kwargs['custom_eps'] + else: + if precision == 'FP32': + custom_eps = 1e-4 + else: + custom_eps = 5e-2 + # Compare Ie results with Framework results + assert self.compare_ie_results_with_framework(infer_res=infer_res, framework_res=fw_res, + framework_eps=custom_eps), \ + "Comparing with Framework failed: ie_res={}; framework_res={}.".format(infer_res, + fw_res) + + # Feed dict for each input is filled with random number. + # It is possible to redefine this function and generate your own input + def _prepare_input(self, inputs_dict): + for input in inputs_dict.keys(): + inputs_dict[input] = np.random.randint(-10, 10, inputs_dict[input]).astype(np.float32) + return inputs_dict + + def compare_ie_results_with_framework(self, infer_res, framework_res, framework_eps): + is_ok = True + from common.utils.common_utils import allclose + for framework_out_name in framework_res: + if framework_out_name not in infer_res and len(infer_res) == 1: + ov_res = list(infer_res.values())[0] + else: + ov_res = infer_res[framework_out_name] + + fw_res = np.array(framework_res[framework_out_name]) + + assert fw_res.dtype == ov_res.dtype or \ + ov_res.dtype.type == str or \ + ov_res.dtype.type == np.str_, 'Outputs types are different: ' \ + 'OpenVINO output type - {}, ' \ + 'Framework output type - {}'.format(ov_res.dtype, fw_res.dtype) + assert fw_res.shape == ov_res.shape, 'Outputs shapes are different: ' \ + 'OpenVINO output shape - {}, ' \ + 'Framework output shape - {}'.format(ov_res.shape, fw_res.shape) + + if not allclose(ov_res, fw_res, + atol=framework_eps, + rtol=framework_eps): + is_ok = False + if ov_res.dtype != bool: + diff = np.array(abs(ov_res - fw_res)).max() + print("Max diff is {}".format(diff)) + else: + print("Boolean results are not equal") + else: + print("Accuracy validation successful!\n") + print("absolute eps: {}, relative eps: {}".format(framework_eps, framework_eps)) + return is_ok diff --git a/tests/layer_tests/tensorflow_tests/test_tf_Equal.py b/tests/layer_tests/tensorflow_tests/test_tf_Equal.py index 8c0b496a7b4c42..4db05e00b589ea 100644 --- a/tests/layer_tests/tensorflow_tests/test_tf_Equal.py +++ b/tests/layer_tests/tensorflow_tests/test_tf_Equal.py @@ -1,9 +1,8 @@ # Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import platform - import numpy as np +import platform import pytest import tensorflow as tf from common.tf_layer_test_class import CommonTFLayerTest @@ -233,7 +232,7 @@ def create_equal_net(self, x_shape, y_shape): tf.compat.v1.reset_default_graph() with tf.compat.v1.Session() as sess: x = tf.compat.v1.placeholder(tf.string, x_shape, 'x') - y = tf.compat.v1.placeholder(tf.string, x_shape, 'y') + y = tf.compat.v1.placeholder(tf.string, y_shape, 'y') tf.raw_ops.Equal(x=x, y=y) tf.compat.v1.global_variables_initializer() tf_net = sess.graph_def @@ -253,6 +252,8 @@ def create_equal_net(self, x_shape, y_shape): def test_equal_str(self, x_shape, y_shape, ie_device, precision, ir_version, temp_dir, use_legacy_frontend): + if x_shape == [] and y_shape == []: + pytest.skip("156746: EqualStr operation outputs 1D tensor for two input scalars") if ie_device == 'GPU' or run_in_jenkins(): pytest.skip("operation extension is not supported on GPU") self._test(*self.create_equal_net(x_shape=x_shape, y_shape=y_shape), diff --git a/tests/layer_tests/tensorflow_tests/test_tf_ToBool.py b/tests/layer_tests/tensorflow_tests/test_tf_ToBool.py index 1beab798e14d1f..789cc820502cdf 100644 --- a/tests/layer_tests/tensorflow_tests/test_tf_ToBool.py +++ b/tests/layer_tests/tensorflow_tests/test_tf_ToBool.py @@ -13,10 +13,16 @@ def _prepare_input(self, inputs_info): x_shape = inputs_info['x:0'] inputs_data = {} inputs_data['x:0'] = np.random.randint(-10, 10, x_shape).astype(np.float32) - + if self.input_shape == [] and self.with_zero: + inputs_data['x:0'] = np.array(0, dtype=np.float32) + elif self.input_shape == [] and not self.with_zero: + inputs_data['x:0'] = np.array(1, dtype=np.float32) + return inputs_data - def create_tobool_net(self, input_shape, input_type): + def create_tobool_net(self, input_shape, input_type, with_zero=None): + self.with_zero = with_zero + self.input_shape = input_shape self.input_type = input_type tf.compat.v1.reset_default_graph() # Create the graph and model @@ -29,6 +35,10 @@ def create_tobool_net(self, input_shape, input_type): return tf_net, None test_data_basic = [ + dict(input_shape=[], input_type=np.float32, with_zero=True), + dict(input_shape=[], input_type=np.float32, with_zero=False), + dict(input_shape=[0], input_type=np.float32), + dict(input_shape=[4, 2, 0], input_type=np.float32), dict(input_shape=[10, 20], input_type=np.float32), dict(input_shape=[2, 3, 4], input_type=np.float32), ] @@ -37,7 +47,7 @@ def create_tobool_net(self, input_shape, input_type): @pytest.mark.precommit @pytest.mark.nightly def test_to_bool_basic(self, params, ie_device, precision, ir_version, temp_dir, - use_legacy_frontend): + use_legacy_frontend): self._test(*self.create_tobool_net(**params), ie_device, precision, ir_version, temp_dir=temp_dir, - use_legacy_frontend=use_legacy_frontend) \ No newline at end of file + use_legacy_frontend=use_legacy_frontend) From c81a17f465d1d0a680c539c72f9e43ff4126dec3 Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Tue, 5 Nov 2024 17:21:16 +0100 Subject: [PATCH 14/93] [core] Remove deprecated element Type ctor (#27333) ### Details: - Removed deprecated and not used element Type constructor ### Tickets: - CVS-156179 --- .../openvino/core/type/element_type.hpp | 3 -- src/core/src/type/element_type.cpp | 45 ------------------- 2 files changed, 48 deletions(-) diff --git a/src/core/include/openvino/core/type/element_type.hpp b/src/core/include/openvino/core/type/element_type.hpp index e579e8ba83210b..2af2fd272a890d 100644 --- a/src/core/include/openvino/core/type/element_type.hpp +++ b/src/core/include/openvino/core/type/element_type.hpp @@ -129,9 +129,6 @@ class OPENVINO_API Type { // Return element type in string representation std::string to_string() const; - OPENVINO_DEPRECATED("This constructor is deprecated. It will be removed in 2025.0") - Type(size_t bitwidth, bool is_real, bool is_signed, bool is_quantized, const std::string& cname); - private: Type_t m_type{Type_t::undefined}; }; diff --git a/src/core/src/type/element_type.cpp b/src/core/src/type/element_type.cpp index c0fad90285e3f6..3e664335033d80 100644 --- a/src/core/src/type/element_type.cpp +++ b/src/core/src/type/element_type.cpp @@ -164,51 +164,6 @@ std::vector ov::element::Type::get_known_types() { return rc; } -ov::element::Type::Type(size_t bitwidth, - bool is_real, - bool is_signed, - bool is_quantized, - const std::string& /* cname */) { - const ElementTypes::ElementsMap elements_map{ - {ov::element::Type_t::undefined, - {std::numeric_limits::max(), false, false, false, "undefined", "undefined"}}, - {ov::element::Type_t::dynamic, {0, false, false, false, "dynamic", "dynamic"}}, - {ov::element::Type_t::boolean, {8, false, true, false, "char", "boolean"}}, - {ov::element::Type_t::bf16, {16, true, true, false, "bfloat16", "bf16"}}, - {ov::element::Type_t::f16, {16, true, true, false, "float16", "f16"}}, - {ov::element::Type_t::f32, {32, true, true, false, "float", "f32"}}, - {ov::element::Type_t::f64, {64, true, true, false, "double", "f64"}}, - {ov::element::Type_t::i4, {4, false, true, true, "int4_t", "i4"}}, - {ov::element::Type_t::i8, {8, false, true, true, "int8_t", "i8"}}, - {ov::element::Type_t::i16, {16, false, true, false, "int16_t", "i16"}}, - {ov::element::Type_t::i32, {32, false, true, true, "int32_t", "i32"}}, - {ov::element::Type_t::i64, {64, false, true, false, "int64_t", "i64"}}, - {ov::element::Type_t::u1, {1, false, false, false, "uint1_t", "u1"}}, - {ov::element::Type_t::u2, {2, false, false, false, "uint2_t", "u2"}}, - {ov::element::Type_t::u3, {3, false, false, false, "uint3_t", "u3"}}, - {ov::element::Type_t::u4, {4, false, false, false, "uint4_t", "u4"}}, - {ov::element::Type_t::u6, {6, false, false, false, "uint6_t", "u6"}}, - {ov::element::Type_t::u8, {8, false, false, true, "uint8_t", "u8"}}, - {ov::element::Type_t::u16, {16, false, false, false, "uint16_t", "u16"}}, - {ov::element::Type_t::u32, {32, false, false, false, "uint32_t", "u32"}}, - {ov::element::Type_t::u64, {64, false, false, false, "uint64_t", "u64"}}, - {ov::element::Type_t::nf4, {4, false, false, true, "nfloat4", "nf4"}}, - {ov::element::Type_t::f8e4m3, {8, true, true, true, "f8e4m3", "f8e4m3"}}, - {ov::element::Type_t::f8e5m2, {8, true, true, true, "f8e5m2", "f8e5m2"}}, - {ov::element::Type_t::string, {8 * sizeof(std::string), false, false, false, "string", "string"}}, - {ov::element::Type_t::f4e2m1, {4, true, true, true, "f4e2m1", "f4e2m1"}}, - {ov::element::Type_t::f8e8m0, {4, true, true, true, "f8e8m0", "f8e8m0"}}, - }; - for (const auto& t : elements_map) { - const TypeInfo& info = t.second; - if (bitwidth == info.m_bitwidth && is_real == info.m_is_real && is_signed == info.m_is_signed && - is_quantized == info.m_is_quantized) { - m_type = t.first; - return; - } - } -} - ov::element::Type::Type(const std::string& type) : Type(type_from_string(type)) {} std::string ov::element::Type::c_type_string() const { From 1221a97c7213bd6cb3eb3e64595ecceb4b021653 Mon Sep 17 00:00:00 2001 From: Alina Kladieva Date: Tue, 5 Nov 2024 20:35:56 +0100 Subject: [PATCH 15/93] Bump OV version 2025.0 (#27412) Signed-off-by: Alina Kladieva --- cmake/packaging/debian.cmake | 1 + cmake/packaging/rpm.cmake | 1 + src/core/include/openvino/core/version.hpp | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmake/packaging/debian.cmake b/cmake/packaging/debian.cmake index 0eb020a5584f1c..59b312963c180d 100644 --- a/cmake/packaging/debian.cmake +++ b/cmake/packaging/debian.cmake @@ -98,6 +98,7 @@ macro(ov_cpack_settings) 2024.2.0 2024.3.0 2024.4.0 + 2024.5.0 ) ov_check_conflicts_versions(conflicting_versions) diff --git a/cmake/packaging/rpm.cmake b/cmake/packaging/rpm.cmake index cf915ee852417e..a4a63c35858bf9 100644 --- a/cmake/packaging/rpm.cmake +++ b/cmake/packaging/rpm.cmake @@ -86,6 +86,7 @@ macro(ov_cpack_settings) 2024.2.0 2024.3.0 2024.4.0 + 2024.5.0 ) ov_check_conflicts_versions(conflicting_versions) diff --git a/src/core/include/openvino/core/version.hpp b/src/core/include/openvino/core/version.hpp index db1595f7addcb6..79f688795ce37c 100644 --- a/src/core/include/openvino/core/version.hpp +++ b/src/core/include/openvino/core/version.hpp @@ -19,8 +19,8 @@ * @brief Defines OpenVINO patch version */ -#define OPENVINO_VERSION_MAJOR 2024 -#define OPENVINO_VERSION_MINOR 5 +#define OPENVINO_VERSION_MAJOR 2025 +#define OPENVINO_VERSION_MINOR 0 #define OPENVINO_VERSION_PATCH 0 namespace ov { From 9204a20d24aabd9276d26e150572981996090ec4 Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Tue, 5 Nov 2024 18:55:44 +0100 Subject: [PATCH 16/93] [core] Remove deprecated members of Concat operator (#27339) ### Details: - Remove deprecated members of Concat operator: * get_concatenation_axis * set_concatenation_axis ### Tickets: - CVS-156181 --- src/core/include/openvino/op/concat.hpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/core/include/openvino/op/concat.hpp b/src/core/include/openvino/op/concat.hpp index 0e8fa67c54dfae..d32171c9facbc1 100644 --- a/src/core/include/openvino/op/concat.hpp +++ b/src/core/include/openvino/op/concat.hpp @@ -36,17 +36,6 @@ class OPENVINO_API Concat : public Op { std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; - /// \return The concatenation axis. - OPENVINO_DEPRECATED("The function get_concatenation_axis() is deprecated. Will be removed in 2025.0 release. Use " - "get_axis() instead.") - int64_t get_concatenation_axis() const { - return m_concat_axis; - } - OPENVINO_DEPRECATED("The function set_concatenation_axis() is deprecated. Will be removed in 2025.0 release. Use " - "set_axis() instead.") - void set_concatenation_axis(int64_t concatenation_axis) { - m_concat_axis = concatenation_axis; - } /// \return The concatenation axis. int64_t get_axis() const { return m_axis; From 41b2093a1db2846d68bc1591044dcb728b5e9fd9 Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Tue, 5 Nov 2024 19:02:29 +0100 Subject: [PATCH 17/93] [docs] Update LSTM sequence specification (version update) (#27355) ### Details: - Align LSTM sequence version in specification, after LSTM sequence v0 removal from OV opset. ### Blocked by: - #27330 ### Tickets: - CVS-156182 --- .../operation-sets/available-opsets/opset1.rst | 3 --- .../operation-sets/available-opsets/opset10.rst | 3 +-- .../operation-sets/available-opsets/opset11.rst | 3 +-- .../operation-sets/available-opsets/opset12.rst | 3 +-- .../operation-sets/available-opsets/opset13.rst | 3 +-- .../operation-sets/available-opsets/opset14.rst | 2 +- .../operation-sets/available-opsets/opset15.rst | 2 +- .../operation-sets/available-opsets/opset2.rst | 3 --- .../operation-sets/available-opsets/opset3.rst | 2 -- .../operation-sets/available-opsets/opset5.rst | 4 +--- .../operation-sets/available-opsets/opset6.rst | 3 +-- .../operation-sets/available-opsets/opset7.rst | 3 +-- .../operation-sets/available-opsets/opset8.rst | 4 +--- .../operation-sets/available-opsets/opset9.rst | 3 +-- .../sequence/{lstm-sequence-1.rst => lstm-sequence-5.rst} | 5 ++--- 15 files changed, 13 insertions(+), 33 deletions(-) rename docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/{lstm-sequence-1.rst => lstm-sequence-5.rst} (98%) diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset1.rst index 55b29d30fa8502..1ce95152c20c90 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset1.rst @@ -66,7 +66,6 @@ Table of Contents * :doc:`LogicalXor <../operation-specs/logical/logical-xor-1>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-1>` * :doc:`Maximum <../operation-specs/arithmetic/maximum-1>` @@ -122,5 +121,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset10.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset10.rst index 04f151a3e26b84..56397ecd852305 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset10.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset10.rst @@ -107,7 +107,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-8>` @@ -192,4 +192,3 @@ Table of Contents * :doc:`Unique <../operation-specs/movement/unique-10>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset11.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset11.rst index 6ae5882e2e4fd7..8a918bde06ebb1 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset11.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset11.rst @@ -107,7 +107,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-8>` @@ -192,4 +192,3 @@ Table of Contents * :doc:`Unique <../operation-specs/movement/unique-10>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset12.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset12.rst index 6e6ddd61ba1e0e..06cfaa7f8f6a20 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset12.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset12.rst @@ -108,7 +108,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-8>` @@ -193,4 +193,3 @@ Table of Contents * :doc:`Unique <../operation-specs/movement/unique-10>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset13.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset13.rst index 87396dfd542203..6bcea43cdeb103 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset13.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset13.rst @@ -113,7 +113,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-8>` @@ -201,4 +201,3 @@ Table of Contents * :doc:`Unique <../operation-specs/movement/unique-10>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset14.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset14.rst index 05131a907d15b4..eb967a6fc94e11 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset14.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset14.rst @@ -115,7 +115,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-14>` diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset15.rst index 1dbb0a9101bb99..12368033a1fa20 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset15.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset15.rst @@ -120,7 +120,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-14>` diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset2.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset2.rst index 5ea1c3bde54018..41170ad2036a25 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset2.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset2.rst @@ -68,7 +68,6 @@ Table of Contents * :doc:`LogicalXor <../operation-specs/logical/logical-xor-1>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-1>` * :doc:`Maximum <../operation-specs/arithmetic/maximum-1>` @@ -128,5 +127,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset3.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset3.rst index fcce876e6944df..9b2992db18cc36 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset3.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset3.rst @@ -76,7 +76,6 @@ Table of Contents * :doc:`LogicalXor <../operation-specs/logical/logical-xor-1>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-1>` * :doc:`Maximum <../operation-specs/arithmetic/maximum-1>` @@ -144,4 +143,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset5.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset5.rst index 0c0ae150b1d21d..fdada28b6bdc06 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset5.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset5.rst @@ -86,7 +86,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-1>` * :doc:`Maximum <../operation-specs/arithmetic/maximum-1>` @@ -162,5 +162,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset6.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset6.rst index 8d626c71be2d89..d587370800aa9e 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset6.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset6.rst @@ -93,7 +93,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-1>` * :doc:`Maximum <../operation-specs/arithmetic/maximum-1>` @@ -168,4 +168,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset7.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset7.rst index 7dd5d8fbba14d7..6de9bccf5921b7 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset7.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset7.rst @@ -96,7 +96,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-1>` * :doc:`Maximum <../operation-specs/arithmetic/maximum-1>` @@ -172,4 +172,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset8.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset8.rst index 0bb5b336f9e2f2..f35299c9f4c97a 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset8.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset8.rst @@ -101,7 +101,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-8>` @@ -183,5 +183,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset9.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset9.rst index ebb5dfdbdc6555..2421e4b9bc53c1 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset9.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset9.rst @@ -105,7 +105,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-8>` @@ -189,4 +189,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst similarity index 98% rename from docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-1.rst rename to docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst index dcc21ca5321451..c00b4c819cc66a 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst @@ -3,10 +3,10 @@ LSTMSequence .. meta:: - :description: Learn about LSTMSequence-1 - a sequence processing operation, which + :description: Learn about LSTMSequence-5 - a sequence processing operation, which can be performed on seven required input tensors. -**Versioned name**: *LSTMSequence-1* +**Versioned name**: *LSTMSequence-5* **Category**: *Sequence processing* @@ -145,4 +145,3 @@ A single cell in the sequence is implemented in the same way as in :doc:`LSTM Ce - From 7cf2bbb83910abda73689b27a418b91dff43e5c2 Mon Sep 17 00:00:00 2001 From: Pavel Durandin Date: Tue, 5 Nov 2024 22:19:05 +0400 Subject: [PATCH 18/93] [GPU] clean oneDNN cache by release memory method (#27403) ### Details: - Partial fix for https://jira.devtools.intel.com/browse/CVS-156022 - explicit call for release memory is required ### Tickets: - *ticket-id* --- .../intel_gpu/include/intel_gpu/plugin/compiled_model.hpp | 2 ++ src/plugins/intel_gpu/src/plugin/compiled_model.cpp | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/compiled_model.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/compiled_model.hpp index 640cc82c2c8c2a..0d78b480b733b0 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/compiled_model.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/compiled_model.hpp @@ -61,6 +61,8 @@ class CompiledModel : public ov::ICompiledModel { const std::vector>& get_graphs() const; std::shared_ptr get_graph(size_t n) const; + void release_memory() override; + private: RemoteContextImpl::Ptr m_context; ExecutionConfig m_config; diff --git a/src/plugins/intel_gpu/src/plugin/compiled_model.cpp b/src/plugins/intel_gpu/src/plugin/compiled_model.cpp index 527e08f07432ef..18e7a88fc42f3e 100644 --- a/src/plugins/intel_gpu/src/plugin/compiled_model.cpp +++ b/src/plugins/intel_gpu/src/plugin/compiled_model.cpp @@ -293,5 +293,13 @@ std::shared_ptr CompiledModel::create_sync_infer_request( return std::make_shared(std::static_pointer_cast(shared_from_this())); } + +void CompiledModel::release_memory() { +#ifdef ENABLE_ONEDNN_FOR_GPU + auto capacity = dnnl::get_primitive_cache_capacity(); + dnnl::set_primitive_cache_capacity(0); + dnnl::set_primitive_cache_capacity(capacity); +#endif +} } // namespace intel_gpu } // namespace ov From 7666e39cf30d53f8b2e93bc31558b7b5624cf43c Mon Sep 17 00:00:00 2001 From: Roman Lyamin Date: Wed, 6 Nov 2024 09:40:06 +0400 Subject: [PATCH 19/93] [GPU] Disable crop fusing (#27417) ### Details: - *Disable crop fusing that was enabled in the https://github.com/openvinotoolkit/openvino/pull/26951* ### Tickets: - *[156367](https://jira.devtools.intel.com/browse/CVS-156367)* --- .../graph/graph_optimizer/prepare_primitive_fusing.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp index 39cbc1aa89b4e2..c323109850c489 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp @@ -736,8 +736,6 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { should_fuse |= input.is_type(); - should_fuse |= input.is_type(); - bool legacy_fusion = activation_node.get_dependencies().size() == 1 && !input.can_be_optimized() && !activation_node.is_constant() && @@ -922,8 +920,7 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { (parents[i].first->is_type()) || (parents[i].first->is_type() && reduce_supports_fusings(parents[i].first->as())) || - (parents[i].first->is_type()) || - (parents[i].first->is_type()); + (parents[i].first->is_type()); } // Disable fusion to a node on constant path when second input is in data flow @@ -1063,6 +1060,9 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { fused_node->get_input_pshape().rbegin()->is_dynamic(); if (is_fc_lora || is_conv_lora || is_gemm_lora) { + if (!can_fuse_parents[peer_idx]) { + return; + } std::swap(peer_node, fused_node); } } From 8a3277fa7121509148a7a03c19e53cae1b1712d4 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 6 Nov 2024 09:03:50 +0100 Subject: [PATCH 20/93] [GHA] Changed android runner (#27414) ### Details: - Increased RAM ### Tickets: - *156361* --------- Co-authored-by: Alina Kladieva --- .github/workflows/android_x64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android_x64.yml b/.github/workflows/android_x64.yml index 1cdb2023784979..47ea6d2cd90ffe 100644 --- a/.github/workflows/android_x64.yml +++ b/.github/workflows/android_x64.yml @@ -73,7 +73,7 @@ jobs: defaults: run: shell: bash - runs-on: aks-linux-16-cores-32gb + runs-on: aks-linux-16-cores-64gb container: image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_android }} volumes: From cd15be95e3bb2b3581661180a851aaeeb4977642 Mon Sep 17 00:00:00 2001 From: Anastasia Kuporosova Date: Wed, 6 Nov 2024 09:43:46 +0100 Subject: [PATCH 21/93] [PyOV] Test for export-import via file (#27320) ### Details: - add a test where exported model saved to file and then imported from file - fix a problem with `inc_ref` and `dec_ref` when python callbacks are passed ### Tickets: - *ticket-id* --- .../python/src/pyopenvino/utils/utils.cpp | 23 +++++++--- .../python/tests/test_graph/test_custom_op.py | 4 +- .../python/tests/test_graph/test_manager.py | 6 +-- .../tests/test_runtime/test_compiled_model.py | 43 ++++++++++++++++++- .../python/tests/test_runtime/test_core.py | 16 +++---- .../python/tests/test_runtime/test_model.py | 6 +-- .../test_transformations/test_compression.py | 4 +- .../test_transformations/test_offline_api.py | 30 ++++++------- .../test_public_transformations.py | 10 ++--- src/bindings/python/tests/utils/helpers.py | 13 ++++-- 10 files changed, 105 insertions(+), 50 deletions(-) diff --git a/src/bindings/python/src/pyopenvino/utils/utils.cpp b/src/bindings/python/src/pyopenvino/utils/utils.cpp index e165c2e00b4808..c747e2d3b81166 100644 --- a/src/bindings/python/src/pyopenvino/utils/utils.cpp +++ b/src/bindings/python/src/pyopenvino/utils/utils.cpp @@ -258,20 +258,31 @@ std::map properties_to_any_map(const std::map(property_value)) { OPENVINO_THROW("The value type of ov::cache_encryption_callbacks property is expected list"); } + auto property_list = property_value.cast(); + // Wrapped to sp due-to we need to hold GIL upon destruction of python function + auto py_encrypt = std::shared_ptr(new py::function(std::move(property_list[0])), + [](py::function* py_encrypt) { + py::gil_scoped_acquire acquire; + delete py_encrypt; + }); + auto py_decrypt = std::shared_ptr(new py::function(std::move(property_list[1])), + [](py::function* py_decrypt) { + py::gil_scoped_acquire acquire; + delete py_decrypt; + }); + std::function encrypt_func = - [property_value](const std::string& in_str) -> std::string { + [py_encrypt](const std::string& in_str) -> std::string { // Acquire GIL, execute Python function py::gil_scoped_acquire acquire; - auto _list = property_value.cast(); - return _list[0](in_str).cast(); + return (*py_encrypt)(in_str).cast(); }; std::function decrypt_func = - [property_value](const std::string& in_str) -> std::string { + [py_decrypt](const std::string& in_str) -> std::string { // Acquire GIL, execute Python function py::gil_scoped_acquire acquire; - auto _list = property_value.cast(); - return _list[1](in_str).cast(); + return (*py_decrypt)(in_str).cast(); }; ov::EncryptionCallbacks encryption_callbacks{encrypt_func, decrypt_func}; properties_to_cpp[property.first] = encryption_callbacks; diff --git a/src/bindings/python/tests/test_graph/test_custom_op.py b/src/bindings/python/tests/test_graph/test_custom_op.py index 5a2e33c7ee5783..8643844e2c54fd 100644 --- a/src/bindings/python/tests/test_graph/test_custom_op.py +++ b/src/bindings/python/tests/test_graph/test_custom_op.py @@ -12,7 +12,7 @@ from openvino.runtime import DiscreteTypeInfo import openvino.runtime.opset14 as ops -from tests.utils.helpers import create_filename_for_test +from tests.utils.helpers import create_filenames_for_ir class CustomOp(Op): @@ -108,7 +108,7 @@ def visit_attributes(self, visitor): # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request @pytest.fixture def prepared_paths(request, tmp_path): - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path) yield xml_path, bin_path diff --git a/src/bindings/python/tests/test_graph/test_manager.py b/src/bindings/python/tests/test_graph/test_manager.py index b5fce8cc09b8d7..ff72ef43158d6e 100644 --- a/src/bindings/python/tests/test_graph/test_manager.py +++ b/src/bindings/python/tests/test_graph/test_manager.py @@ -12,7 +12,7 @@ from openvino.runtime.passes import Manager, Serialize, ConstantFolding, Version from tests.test_graph.util import count_ops_of_type -from tests.utils.helpers import create_filename_for_test, compare_models +from tests.utils.helpers import create_filenames_for_ir, compare_models def create_model(): @@ -51,7 +51,7 @@ def test_constant_folding(): # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request @pytest.fixture def prepare_ir_paths(request, tmp_path): - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path) yield xml_path, bin_path @@ -138,7 +138,7 @@ def test_serialize_pass_mixed_args_kwargs_v2(prepare_ir_paths): # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request def test_serialize_pass_wrong_num_of_args(request, tmp_path): - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path) pass_manager = Manager() with pytest.raises(TypeError) as e: diff --git a/src/bindings/python/tests/test_runtime/test_compiled_model.py b/src/bindings/python/tests/test_runtime/test_compiled_model.py index d6909fa94711d3..9e17497fc22081 100644 --- a/src/bindings/python/tests/test_runtime/test_compiled_model.py +++ b/src/bindings/python/tests/test_runtime/test_compiled_model.py @@ -2,6 +2,7 @@ # Copyright (C) 2018-2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +import os import pytest import numpy as np @@ -14,6 +15,7 @@ generate_relu_compiled_model_with_config, encrypt_base64, decrypt_base64, + create_filenames_for_ir, create_filename_for_test) from openvino import Model, Shape, Core, Tensor, serialize from openvino.runtime import ConstOutput @@ -97,6 +99,43 @@ def test_export_import_advanced(device): assert np.argmax(res[new_compiled.outputs[0]]) == 531 +# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request +@pytest.fixture +def prepare_blob_path(request, tmp_path): + filename = create_filename_for_test(request.node.name) + path_to_blob = tmp_path / str(filename + ".blob") + yield path_to_blob + + os.remove(path_to_blob) + + +def test_export_import_via_file(prepare_blob_path, device): + import io + + core = Core() + + if props.device.Capability.EXPORT_IMPORT not in core.get_property(device, props.device.capabilities): + pytest.skip(f"{core.get_property(device, props.device.full_name)} plugin due-to export, import model API isn't implemented.") + + compiled_model = generate_relu_compiled_model(device) + + user_stream = io.BytesIO() + + compiled_model.export_model(user_stream) + path_to_blob = prepare_blob_path + + with open(path_to_blob, "wb") as f_w: + f_w.write(user_stream.getbuffer()) + + with open(path_to_blob, "rb") as f_r: + new_compiled = core.import_model(f_r.read(), device) + + img = generate_image() + res = new_compiled.infer_new_request({"data": img}) + + assert np.argmax(res[new_compiled.outputs[0]]) == 531 + + @pytest.mark.parametrize("input_arguments", [[0], ["data"], []]) def test_get_input(device, input_arguments): compiled_model = generate_relu_compiled_model(device) @@ -250,7 +289,7 @@ def test_direct_infer(device, shared_flag): # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request def test_compiled_model_after_core_destroyed(request, tmp_path, device): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path) model = get_relu_model() serialize(model, xml_path, bin_path) with open(bin_path, "rb") as f: @@ -267,7 +306,7 @@ def test_compiled_model_after_core_destroyed(request, tmp_path, device): def test_compiled_model_from_buffer_in_memory(request, tmp_path, device): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path) model = get_relu_model() serialize(model, xml_path, bin_path) with open(bin_path, "rb") as f: diff --git a/src/bindings/python/tests/test_runtime/test_core.py b/src/bindings/python/tests/test_runtime/test_core.py index 8c15c2a32a05c5..d147ce2d6bcab2 100644 --- a/src/bindings/python/tests/test_runtime/test_core.py +++ b/src/bindings/python/tests/test_runtime/test_core.py @@ -28,7 +28,7 @@ get_relu_model, plugins_path, compare_models, - create_filename_for_test, + create_filenames_for_ir, get_model_with_template_extension, ) @@ -77,7 +77,7 @@ def test_core_class(device): ]) def test_compile_model(request, tmp_path, device_name): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path) relu_model = get_relu_model() serialize(relu_model, xml_path, bin_path) model = core.read_model(model=xml_path, weights=bin_path) @@ -97,7 +97,7 @@ def get_model(): @pytest.fixture def get_model_path(request, tmp_path): - xml_path, _ = create_filename_for_test(request.node.name, tmp_path, True) + xml_path, _ = create_filenames_for_ir(request.node.name, tmp_path, True) serialize(get_relu_model(), xml_path) return Path(xml_path) @@ -130,7 +130,7 @@ def test_compact_api(model_type, device_name, config, request): # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request def test_read_model_from_ir(request, tmp_path): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path) relu_model = get_relu_model() serialize(relu_model, xml_path, bin_path) model = core.read_model(model=xml_path, weights=bin_path) @@ -143,7 +143,7 @@ def test_read_model_from_ir(request, tmp_path): # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request def test_read_model_from_tensor(request, tmp_path): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path, is_xml_path=True, is_bin_path=True) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path, is_xml_path=True, is_bin_path=True) relu_model = get_relu_model() serialize(relu_model, xml_path, bin_path) arr = np.ones(shape=(10), dtype=np.int8) @@ -164,7 +164,7 @@ def test_read_model_with_wrong_input(): # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request def test_read_model_as_path(request, tmp_path): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path, True, True) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path, True, True) relu_model = get_relu_model() serialize(relu_model, xml_path, bin_path) @@ -181,7 +181,7 @@ def test_read_model_as_path(request, tmp_path): # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request def test_read_model_from_buffer(request, tmp_path): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path) relu_model = get_relu_model() serialize(relu_model, xml_path, bin_path) with open(bin_path, "rb") as f: @@ -195,7 +195,7 @@ def test_read_model_from_buffer(request, tmp_path): # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request def test_model_from_buffer_valid(request, tmp_path): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path) relu_model = get_relu_model() serialize(relu_model, xml_path, bin_path) with open(bin_path, "rb") as f: diff --git a/src/bindings/python/tests/test_runtime/test_model.py b/src/bindings/python/tests/test_runtime/test_model.py index 62b1eac9da3865..0ae592b2d1dff5 100644 --- a/src/bindings/python/tests/test_runtime/test_model.py +++ b/src/bindings/python/tests/test_runtime/test_model.py @@ -30,7 +30,7 @@ from tests.utils.helpers import ( generate_add_model, generate_model_with_memory, - create_filename_for_test, + create_filenames_for_ir, ) @@ -618,7 +618,7 @@ def check_rt_info(model): assert model.get_rt_info(["optimization", "test"]) core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path) input_shape = PartialShape([1]) param = ops.parameter(input_shape, dtype=np.float32, name="data") relu1 = ops.relu(param, name="relu1") @@ -701,7 +701,7 @@ def check_rt_info(model): assert rt_info_val in ["float_empty", "nodes", "type", "directed"] core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path) input_shape = PartialShape([1]) param = ops.parameter(input_shape, dtype=np.float32, name="data") relu1 = ops.relu(param, name="relu1") diff --git a/src/bindings/python/tests/test_transformations/test_compression.py b/src/bindings/python/tests/test_transformations/test_compression.py index fa46b6d227f1e3..d2754fd29d9c70 100644 --- a/src/bindings/python/tests/test_transformations/test_compression.py +++ b/src/bindings/python/tests/test_transformations/test_compression.py @@ -9,7 +9,7 @@ from openvino.runtime.opset13 import add, multiply import openvino as ov -from tests.utils.helpers import create_filename_for_test +from tests.utils.helpers import create_filenames_for_ir def make_constant(values, transposed): @@ -38,7 +38,7 @@ def make_model(add_consts, mul_consts): def get_constants(model, request, tmp_path) -> List[Constant]: - model_fname, _ = create_filename_for_test(request.node.name, tmp_path) + model_fname, _ = create_filenames_for_ir(request.node.name, tmp_path) ov.save_model(model, model_fname) core = ov.Core() restored_model = core.read_model(model_fname) diff --git a/src/bindings/python/tests/test_transformations/test_offline_api.py b/src/bindings/python/tests/test_transformations/test_offline_api.py index cd336493b58246..e265cef4635988 100644 --- a/src/bindings/python/tests/test_transformations/test_offline_api.py +++ b/src/bindings/python/tests/test_transformations/test_offline_api.py @@ -18,7 +18,7 @@ from openvino import Model, PartialShape, Core, serialize, save_model import openvino.runtime as ov -from tests.utils.helpers import create_filename_for_test, compare_models, _compare_models +from tests.utils.helpers import create_filenames_for_ir, compare_models, _compare_models def get_relu_model(): @@ -171,10 +171,10 @@ def test_fused_names_cleanup(): def prepare_test_model_for_serialize(request, tmp_path, is_path_xml, is_path_bin): - xml_path, bin_path = create_filename_for_test(request.node.name, - tmp_path, - is_path_xml, - is_path_bin) + xml_path, bin_path = create_filenames_for_ir(request.node.name, + tmp_path, + is_path_xml, + is_path_bin) shape = [100, 100, 2] parameter_a = ov.opset8.parameter(shape, dtype=np.float32, name="A") parameter_b = ov.opset8.parameter(shape, dtype=np.float32, name="B") @@ -267,10 +267,10 @@ def test_compress_model_transformation(): ) def test_version_default(request, tmp_path, is_path_xml, is_path_bin): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, - tmp_path, - is_path_xml, - is_path_bin) + xml_path, bin_path = create_filenames_for_ir(request.node.name, + tmp_path, + is_path_xml, + is_path_bin) shape = [100, 100, 2] parameter_a = ov.opset8.parameter(shape, dtype=np.float32, name="A") parameter_b = ov.opset8.parameter(shape, dtype=np.float32, name="B") @@ -297,10 +297,10 @@ def test_version_default(request, tmp_path, is_path_xml, is_path_bin): ], ) def test_serialize_default_bin(request, tmp_path, is_path_xml, is_path_bin): - xml_path, bin_path = create_filename_for_test(request.node.name, - tmp_path, - is_path_xml, - is_path_bin) + xml_path, bin_path = create_filenames_for_ir(request.node.name, + tmp_path, + is_path_xml, + is_path_bin) model = get_relu_model() serialize(model, xml_path) assert os.path.exists(bin_path) @@ -311,7 +311,7 @@ def test_serialize_default_bin(request, tmp_path, is_path_xml, is_path_bin): # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request def test_version_ir_v10(request, tmp_path): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path) shape = [100, 100, 2] parameter_a = ov.opset8.parameter(shape, dtype=np.float32, name="A") parameter_b = ov.opset8.parameter(shape, dtype=np.float32, name="B") @@ -332,7 +332,7 @@ def test_version_ir_v10(request, tmp_path): # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request def test_version_ir_v11(request, tmp_path): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = create_filenames_for_ir(request.node.name, tmp_path) shape = [100, 100, 2] parameter_a = ov.opset8.parameter(shape, dtype=np.float32, name="A") parameter_b = ov.opset8.parameter(shape, dtype=np.float32, name="B") diff --git a/src/bindings/python/tests/test_transformations/test_public_transformations.py b/src/bindings/python/tests/test_transformations/test_public_transformations.py index 429bc6c192acc4..a10fea786b9770 100644 --- a/src/bindings/python/tests/test_transformations/test_public_transformations.py +++ b/src/bindings/python/tests/test_transformations/test_public_transformations.py @@ -17,7 +17,7 @@ ) from tests.test_transformations.utils.utils import count_ops, get_relu_model -from tests.utils.helpers import create_filename_for_test, compare_models +from tests.utils.helpers import create_filenames_for_ir, compare_models def get_model(): @@ -132,10 +132,10 @@ def test_low_latency2(): ) def test_serialize_pass(request, tmp_path, is_path_xml, is_path_bin): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, - tmp_path, - is_path_xml, - is_path_bin) + xml_path, bin_path = create_filenames_for_ir(request.node.name, + tmp_path, + is_path_xml, + is_path_bin) model = get_relu_model() diff --git a/src/bindings/python/tests/utils/helpers.py b/src/bindings/python/tests/utils/helpers.py index 2ea00484e9840c..c14bd6e5e779da 100644 --- a/src/bindings/python/tests/utils/helpers.py +++ b/src/bindings/python/tests/utils/helpers.py @@ -303,7 +303,14 @@ def generate_abs_compiled_model_with_data(device, ov_type, numpy_dtype): return compiled_model, request, tensor1, array1 -def create_filename_for_test(test_name, tmp_path, is_xml_path=False, is_bin_path=False): +def create_filename_for_test(test_name): + python_version = str(sys.version_info.major) + "_" + str(sys.version_info.minor) + filename = test_name.replace("test_", "").replace("[", "_").replace("]", "_") + filename = filename + "_" + python_version + return filename + + +def create_filenames_for_ir(test_name, tmp_path, is_xml_path=False, is_bin_path=False): """Return a tuple with automatically generated paths for xml and bin files. :param test_name: Name used in generating. @@ -311,9 +318,7 @@ def create_filename_for_test(test_name, tmp_path, is_xml_path=False, is_bin_path :param is_bin_path: True if bin file should be pathlib.Path object, otherwise return string. :return: Tuple with two objects representing xml and bin files. """ - python_version = str(sys.version_info.major) + "_" + str(sys.version_info.minor) - filename = test_name.replace("test_", "").replace("[", "_").replace("]", "_") - filename = filename + "_" + python_version + filename = create_filename_for_test(test_name) path_to_xml = tmp_path / Path(filename + ".xml") path_to_bin = tmp_path / Path(filename + ".bin") _xml = path_to_xml if is_xml_path else str(path_to_xml) From 362ebe99aa07549976d38daec17d8ace4041f69f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:10:05 +0000 Subject: [PATCH 22/93] Bump codecov/codecov-action from 4.5.0 to 4.6.0 (#26882) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4.5.0 to 4.6.0.
Release notes

Sourced from codecov/codecov-action's releases.

v4.6.0

What's Changed

... (truncated)

Commits
  • b9fd7d1 chore(release):4.6.0 (#1587)
  • 6f7612c fix: bump eslint parser deps (#1586)
  • 26c7e28 build(deps): bump actions/checkout from 4.1.7 to 4.2.0 (#1583)
  • 6f744f7 build(deps): bump github/codeql-action from 3.26.8 to 3.26.9 (#1584)
  • 543c3d4 chore: fix typo of OSS (#1578)
  • e379426 build(deps-dev): bump @​vercel/ncc from 0.38.1 to 0.38.2 (#1577)
  • 42656e4 build(deps): bump github/codeql-action from 3.26.7 to 3.26.8 (#1575)
  • 2296b6b build(deps-dev): bump eslint from 8.57.0 to 8.57.1 (#1571)
  • bd77bc3 build(deps): bump github/codeql-action from 3.26.6 to 3.26.7 (#1569)
  • 180b964 build(deps-dev): bump @​types/jest from 29.5.12 to 29.5.13 (#1567)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=codecov/codecov-action&package-manager=github_actions&previous-version=4.5.0&new-version=4.6.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index db5ba3de1a3c85..6cb0b2c5b6233c 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -138,6 +138,6 @@ jobs: lcov --capture --directory ${{ github.workspace }}/. --output-file coverage.info genhtml coverage.info --output-directory coverage-report - name: Collect coverage - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0 + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 with: verbose: true From b6fe65f01ad71c9298fe658da5978371ffbc2e3c Mon Sep 17 00:00:00 2001 From: Alexandra Sidorova Date: Wed, 6 Nov 2024 13:12:15 +0400 Subject: [PATCH 23/93] [Snippets] Fixed LoopManager::update_loop_ports (#27300) ### Details: - *To remind, `LoopPort` is expression port connected to another expression port which is not in the same Loop. It's like entry (of exit) point of the Loop. It means that some expression port cannot be port of the Loop if all consumers (or sources) are from the same Loop. However, the method `LoopManager::update_loop_ports` sometimes creates these situation. This PR fixes this method. The screenshot below describes this situation: red loop is inner loop and blue loops is outer loop. However, some of output ports of this Loop is inside (green question sign) - invalid situation which is fixed by these changes.* image - *Added the corresponding checks to validate pass* - *Remove parts in `init_is_incremented` which handle invalid case by `is_incremented=false`.* ### Tickets: - *CVS-156299* --- .../snippets/lowered/pass/init_loops.hpp | 2 +- .../lowered/pass/validate_unified_loops.hpp | 7 +++ .../include/snippets/utils/loop_utils.hpp | 6 ++ .../snippets/src/lowered/loop_manager.cpp | 52 ++++++++++++------ .../snippets/src/lowered/pass/init_loops.cpp | 50 ++--------------- .../lowered/pass/validate_unified_loops.cpp | 55 +++++++++++++++---- src/common/snippets/src/utils/loop_utils.cpp | 9 +++ 7 files changed, 107 insertions(+), 74 deletions(-) diff --git a/src/common/snippets/include/snippets/lowered/pass/init_loops.hpp b/src/common/snippets/include/snippets/lowered/pass/init_loops.hpp index 169dbd30e35cc2..e94e1977974716 100644 --- a/src/common/snippets/include/snippets/lowered/pass/init_loops.hpp +++ b/src/common/snippets/include/snippets/lowered/pass/init_loops.hpp @@ -25,7 +25,7 @@ class InitLoops : public Pass { bool run(LinearIR& linear_ir) override; private: - static void update_compile_parameters(const UnifiedLoopInfoPtr& loop_info, size_t loop_id); + static void update_compile_parameters(const UnifiedLoopInfoPtr& loop_info); }; } // namespace pass diff --git a/src/common/snippets/include/snippets/lowered/pass/validate_unified_loops.hpp b/src/common/snippets/include/snippets/lowered/pass/validate_unified_loops.hpp index 80c1b7be19d1f0..d78aaaa668363e 100644 --- a/src/common/snippets/include/snippets/lowered/pass/validate_unified_loops.hpp +++ b/src/common/snippets/include/snippets/lowered/pass/validate_unified_loops.hpp @@ -6,6 +6,9 @@ #include "pass.hpp" +#include "snippets/lowered/loop_manager.hpp" + + namespace ov { namespace snippets { namespace lowered { @@ -27,6 +30,10 @@ class ValidateUnifiedLoops : public Pass { OPENVINO_RTTI("ValidateUnifiedLoops", "Pass") ValidateUnifiedLoops() = default; bool run(LinearIR& linear_ir) override; + +private: + static void validate_loop_infos(const LoopManagerPtr& loop_manager); + static void validate_loop_port_presence(const LinearIR& linear_ir); }; } // namespace pass diff --git a/src/common/snippets/include/snippets/utils/loop_utils.hpp b/src/common/snippets/include/snippets/utils/loop_utils.hpp index b0597008adc1c4..c9f9412ff5c574 100644 --- a/src/common/snippets/include/snippets/utils/loop_utils.hpp +++ b/src/common/snippets/include/snippets/utils/loop_utils.hpp @@ -21,6 +21,12 @@ void update_data_pointer_shifts(const ov::snippets::lowered::UnifiedLoopInfoPtr& * @brief Updates work amount and updates data pointer shifts of the provided "loop_info" */ void update_runtime_parameters(const ov::snippets::lowered::UnifiedLoopInfoPtr& loop_info); +/** + * @brief Check if the passed expression port should be port of the Loop with ID `loop_id`: + * the target expression port should be connected to an expression from another Loop (missed in the loop with ID `loop_id`), + */ +bool should_be_loop_port(const ov::snippets::lowered::ExpressionPort& port, size_t loop_id); + } // namespace utils } // namespace snippets } // namespace ov \ No newline at end of file diff --git a/src/common/snippets/src/lowered/loop_manager.cpp b/src/common/snippets/src/lowered/loop_manager.cpp index 21f4ecc83c57b0..f0e5306c5878bc 100644 --- a/src/common/snippets/src/lowered/loop_manager.cpp +++ b/src/common/snippets/src/lowered/loop_manager.cpp @@ -11,6 +11,7 @@ #include "openvino/core/graph_util.hpp" #include "openvino/core/type.hpp" +#include "snippets/utils/loop_utils.hpp" #include "snippets/itt.hpp" @@ -349,30 +350,45 @@ void LoopManager::fuse_loop_ports(std::vector& output_ports, } void LoopManager::update_loop_ports(const ExpressionPtr& expr) { - auto output_ports = expr->get_output_ports(); - for (size_t i = 0; i < expr->get_input_count(); ++i) { - const auto& source = expr->get_input_port_connector(i)->get_source(); - const auto common_outer_loop_ids = get_common_outer_loops(expr, source.get_expr()); - // The source output port can have several consumers (including the current expr) that can be potential output ports - // So we should verify on the possible future output ports - size_t count_of_common_outer_loops = common_outer_loop_ids.size(); - for (const auto& source_consumer : source.get_connected_ports()) { - if (source_consumer.get_expr() == expr) + auto update_ports = [&](const ov::snippets::lowered::ExpressionPort& connected_port) { + const auto is_output = connected_port.get_type() == ExpressionPort::Output; + // Iterate through all Loops of the connected expression + for (const auto& loop_id : connected_port.get_expr()->get_loop_ids()) { + const auto& loop_info = get_loop_info(loop_id); + // If the connected expression port is not Loop port - nothing to update + // If the target expression is not from the same Loop - nothing to update + if (!loop_info->is_loop_port(connected_port) || !is_loop_id_found(expr, loop_id)) continue; - count_of_common_outer_loops = std::min(count_of_common_outer_loops, get_common_outer_loops(source.get_expr(), source_consumer.get_expr()).size()); - } - replace_loop_ports({common_outer_loop_ids.cbegin(), common_outer_loop_ids.cbegin() + count_of_common_outer_loops}, source, output_ports); - // Save previous port - if (count_of_common_outer_loops != common_outer_loop_ids.size()) { - output_ports.insert(output_ports.begin(), source); - replace_loop_ports({common_outer_loop_ids.cbegin() + count_of_common_outer_loops, common_outer_loop_ids.cend()}, source, output_ports); + + std::vector new_ports; + // Check if some ports of target expression must be Loop port + const auto target_expr_ports = is_output ? expr->get_output_ports() : expr->get_input_ports(); + for (const auto& port : target_expr_ports) { + if (utils::should_be_loop_port(port, loop_id)) + new_ports.push_back(port); + } + // Leave the connected expression port as Loop port if needed + if (utils::should_be_loop_port(connected_port, loop_id)) + new_ports.push_back(connected_port); + + // Nothing should be updated + if (new_ports.size() == 1 && new_ports.front() == connected_port) + continue; + + loop_info->replace_with_new_ports(connected_port, new_ports); } + }; + + // The case with parent loops: source -> target expr + for (size_t i = 0; i < expr->get_input_count(); ++i) { + update_ports(expr->get_input_port_connector(i)->get_source()); } - const auto input_ports = expr->get_input_ports(); + + // The case with child loops: target expr -> consumers for (size_t i = 0; i < expr->get_output_count(); ++i) { const auto& consumers = expr->get_output_port_connector(i)->get_consumers(); for (const auto& consumer : consumers) { - replace_loop_ports(get_common_outer_loops(expr, consumer.get_expr()), consumer, input_ports); + update_ports(consumer); } } } diff --git a/src/common/snippets/src/lowered/pass/init_loops.cpp b/src/common/snippets/src/lowered/pass/init_loops.cpp index aa7d0ab042e1a4..69d336094f1a14 100644 --- a/src/common/snippets/src/lowered/pass/init_loops.cpp +++ b/src/common/snippets/src/lowered/pass/init_loops.cpp @@ -16,46 +16,10 @@ namespace lowered { namespace pass { namespace { -inline void init_is_incremented(LoopPort& port, size_t loop_id) { +inline void init_is_incremented(LoopPort& port) { const auto& expr = port.expr_port->get_expr(); - const auto& expr_loops = expr->get_loop_ids(); if (!std::dynamic_pointer_cast(expr->get_node())) { port.is_incremented = false; - } else if (expr_loops.back() != loop_id) { - // Note: LoopPort connected to Buffer between two loops should not be incremented in the outermost loop - // Consider the example below: - // Store; Loop ids [0,1,2,3] - // Buffer; Loop ids [0,1] - // Load; Loop ids [0,1,4,5] - // Store is output port of Loop-1, but it should be incremented only in Loop-2 and Loop-3. Similar with Load. - auto is_ignored = [=](const ExpressionPtr& target_expr) { - if (ov::is_type(target_expr)) { - const auto& target_loops = target_expr->get_loop_ids(); - const auto i_max = std::min(expr_loops.size(), target_loops.size()); - for (size_t i = 0; i < i_max && expr_loops[i] == target_loops[i]; i++) { - if (target_loops[i] == loop_id) - return true; - } - } - return false; - }; - if (port.expr_port->get_type() == ExpressionPort::Type::Output) { - const auto& out_connector = expr->get_output_port_connector(port.expr_port->get_index()); - for (const auto& consumer : out_connector->get_consumers()) { - if (is_ignored(consumer.get_expr())) { - port.is_incremented = false; - return; - } - } - } else if (port.expr_port->get_type() == ExpressionPort::Type::Input) { - const auto& in_connector = expr->get_input_port_connector(port.expr_port->get_index()); - if (is_ignored(in_connector->get_source().get_expr())) { - port.is_incremented = false; - return; - } - } else { - OPENVINO_THROW("Unexpected LoopPort type"); - } } } @@ -71,11 +35,11 @@ inline int64_t get_data_size(const LoopPort& loop_port) { } } // namespace -void InitLoops::update_compile_parameters(const UnifiedLoopInfoPtr& loop_info, size_t loop_id) { +void InitLoops::update_compile_parameters(const UnifiedLoopInfoPtr& loop_info) { OPENVINO_ASSERT(loop_info != nullptr, "UnifiedLoopInfo is nullptr, nothing to update"); loop_info->iterate_through_infos( - [loop_id](LoopPort& loop_port, UnifiedLoopInfo::LoopPortDesc& ptr_shifts_params) { - init_is_incremented(loop_port, loop_id); + [](LoopPort& loop_port, UnifiedLoopInfo::LoopPortDesc& ptr_shifts_params) { + init_is_incremented(loop_port); ptr_shifts_params.data_size = get_data_size(loop_port); }); } @@ -85,12 +49,10 @@ bool InitLoops::run(LinearIR& linear_ir) { if (linear_ir.empty()) return false; - const auto& loop_manager = linear_ir.get_loop_manager(); - const auto& loops = loop_manager->get_map(); + const auto& loops = linear_ir.get_loop_manager()->get_map(); for (const auto& loop : loops) { - const auto& loop_id = loop.first; const auto& loop_info = ov::as_type_ptr(loop.second); - update_compile_parameters(loop_info, loop_id); + update_compile_parameters(loop_info); ov::snippets::utils::update_runtime_parameters(loop_info); } diff --git a/src/common/snippets/src/lowered/pass/validate_unified_loops.cpp b/src/common/snippets/src/lowered/pass/validate_unified_loops.cpp index bdfb8896405847..ec43f02d28792f 100644 --- a/src/common/snippets/src/lowered/pass/validate_unified_loops.cpp +++ b/src/common/snippets/src/lowered/pass/validate_unified_loops.cpp @@ -7,6 +7,7 @@ #include "snippets/itt.hpp" #include "snippets/lowered/linear_ir.hpp" #include "snippets/lowered/loop_manager.hpp" +#include "snippets/utils/loop_utils.hpp" #include "snippets/utils/utils.hpp" namespace ov { @@ -14,14 +15,7 @@ namespace snippets { namespace lowered { namespace pass { -bool ValidateUnifiedLoops::run(LinearIR& linear_ir) { - OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::ValidateUnifiedLoops") - if (linear_ir.empty()) - return false; - - const auto& loop_manager = linear_ir.get_loop_manager(); - const auto& loops = loop_manager->get_map(); - +void ValidateUnifiedLoops::validate_loop_infos(const LoopManagerPtr& loop_manager) { // Already validated vectors of Loop IDs std::set> validated_nested_loops; auto is_already_verified = [&validated_nested_loops](const std::vector& ids) { @@ -66,10 +60,9 @@ bool ValidateUnifiedLoops::run(LinearIR& linear_ir) { validated_nested_loops.insert(loop_ids); }; - for (const auto& pair : loops) { + for (const auto& pair : loop_manager->get_map()) { const auto& loop_info = ov::as_type_ptr(pair.second); - OPENVINO_ASSERT(loop_info, - "ValidateUnifiedLoops expects only UnifiedLoopInfo in LoopManager"); + OPENVINO_ASSERT(loop_info, "ValidateUnifiedLoops expects only UnifiedLoopInfo in LoopManager"); loop_info->iterate_through_ports(validate_loop_port); // Validate that iteration dimnsion is broadcastable @@ -88,6 +81,46 @@ bool ValidateUnifiedLoops::run(LinearIR& linear_ir) { OPENVINO_ASSERT(unique_dimensions.size() <= 1, "Loop ports have incompatible dimensions, by which the loop iterates"); } +} + +void ValidateUnifiedLoops::validate_loop_port_presence(const LinearIR& linear_ir) { + auto validate_loop_port = [](const ExpressionPort& expr_port, const LoopInfoPtr& loop_info, size_t loop_id) { + if (utils::should_be_loop_port(expr_port, loop_id)) { + OPENVINO_ASSERT(loop_info->is_loop_port(expr_port), + "Expression port with idx ", expr_port.get_index(), " with node ", + expr_port.get_expr()->get_node()->get_friendly_name(), " is not Loop port but should be!"); + } else { + OPENVINO_ASSERT(!loop_info->is_loop_port(expr_port), + "Expression port with idx ", expr_port.get_index(), " with node ", + expr_port.get_expr()->get_node()->get_friendly_name(), " is Loop port but should not be!"); + } + }; + + const auto& loop_manager = linear_ir.get_loop_manager(); + for (const auto& expr : linear_ir) { + const auto& op = expr->get_node(); + if (ov::is_type(op)) + continue; + + for (const auto& loop_id : expr->get_loop_ids()) { + const auto& loop_info = loop_manager->get_loop_info(loop_id); + + for (size_t i = 0; i < expr->get_input_count(); ++i) + validate_loop_port(expr->get_input_port(i), loop_info, loop_id); + + for (size_t i = 0; i < expr->get_output_count(); ++i) + validate_loop_port(expr->get_output_port(i), loop_info, loop_id); + } + } +} + +bool ValidateUnifiedLoops::run(LinearIR& linear_ir) { + OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::ValidateUnifiedLoops") + if (linear_ir.empty()) + return false; + + validate_loop_infos(linear_ir.get_loop_manager()); + validate_loop_port_presence(linear_ir); return true; } diff --git a/src/common/snippets/src/utils/loop_utils.cpp b/src/common/snippets/src/utils/loop_utils.cpp index dabd129fce451d..3d6b274c7613a8 100644 --- a/src/common/snippets/src/utils/loop_utils.cpp +++ b/src/common/snippets/src/utils/loop_utils.cpp @@ -82,6 +82,15 @@ void update_runtime_parameters(const UnifiedLoopInfoPtr& loop_info) { update_data_pointer_shifts(loop_info); } +bool should_be_loop_port(const ov::snippets::lowered::ExpressionPort& port, size_t loop_id) { + const auto& connected_ports = port.get_connected_ports(); + return std::any_of(connected_ports.cbegin(), connected_ports.cend(), + [&](const ExpressionPort& connected_port) { + const auto& loops = connected_port.get_expr()->get_loop_ids(); + return std::find(loops.cbegin(), loops.cend(), loop_id) == loops.cend(); + }); +} + } // namespace utils } // namespace snippets } // namespace ov \ No newline at end of file From d8befa1a5f6f803a711b356c5b0b2ea93f9ada53 Mon Sep 17 00:00:00 2001 From: Roman Lyamin Date: Wed, 6 Nov 2024 14:22:21 +0400 Subject: [PATCH 24/93] [GPU] Added convolution_gpu_b_fs_zyx_fsv16_imad shape agnostic kernel (#26404) ### Tickets: - *[148550](https://jira.devtools.intel.com/browse/CVS-148550)* --- .../src/graph/impls/ocl/convolution.cpp | 4 + .../src/graph/impls/ocl/convolution.hpp | 3 +- src/plugins/intel_gpu/src/graph/program.cpp | 25 +--- .../convolution_gpu_b_fs_zyx_fsv16_imad.cl | 7 +- .../intel_gpu/src/kernel_selector/jitter.cpp | 3 +- ...convolution_kernel_b_fs_zyx_fsv16_imad.cpp | 68 +++++++-- .../convolution_kernel_b_fs_zyx_fsv16_imad.h | 17 +-- .../convolution/convolution_kernel_base.h | 15 ++ .../unit/test_cases/convolution_gpu_test.cpp | 138 ++++++++++++++++++ 9 files changed, 233 insertions(+), 47 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/convolution.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/convolution.cpp index e8043fa9fe90a9..cda7d8f1a4cedc 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/convolution.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/convolution.cpp @@ -32,6 +32,10 @@ struct convolution_impl : typed_primitive_impl_ocl { if (is_dynamic()) { auto& kernel_selector = kernel_selector_t::Instance(); auto kernel_impl = kernel_selector.GetImplementation(_kernel_data.kernelName); + + const kernel_impl_params* impl_params = reinterpret_cast(ib.getKernelImplParams()); + _kernel_data.params = std::make_shared(get_kernel_params(*impl_params, true)); + kernel_impl->GetUpdateDispatchDataFunc(_kernel_data); } } diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/convolution.hpp b/src/plugins/intel_gpu/src/graph/impls/ocl/convolution.hpp index 69ef9f0f8a2a7c..1771da5a5a63ba 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/convolution.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/convolution.hpp @@ -55,7 +55,8 @@ struct ConvolutionImplementationManager : public ImplementationManager { static const std::vector supported_dyn_formats = { format::bfyx, format::bfzyx, - format::b_fs_yx_fsv16 + format::b_fs_yx_fsv16, + format::b_fs_zyx_fsv16 }; if (!one_of(input_fmt.value, supported_dyn_formats) || !one_of(output_fmt.value, supported_dyn_formats)) diff --git a/src/plugins/intel_gpu/src/graph/program.cpp b/src/plugins/intel_gpu/src/graph/program.cpp index 07fad4873659cd..7a66d32795c17c 100644 --- a/src/plugins/intel_gpu/src/graph/program.cpp +++ b/src/plugins/intel_gpu/src/graph/program.cpp @@ -1796,15 +1796,9 @@ void program::save(cldnn::BinaryOutputBuffer& ob) const { for (auto& impl_id : impl_ids) { std::string type_name = get_node_ptr(impl_id)->get_selected_impl()->m_manager->get_type_info().name; ob << type_name; - if (get_node_ptr(impl_id)->get_selected_impl()->is_onednn()) { - ob << true; - auto params = get_node_ptr(impl_id)->get_kernel_impl_params(); - ob.setKernelImplParams(params.get()); - ob << get_node_ptr(impl_id)->selected_impl; - } else { - ob << false; - ob << get_node_ptr(impl_id)->selected_impl; - } + auto params = get_node_ptr(impl_id)->get_kernel_impl_params(); + ob.setKernelImplParams(params.get()); + ob << get_node_ptr(impl_id)->selected_impl; ob << get_node_ptr(impl_id)->get_selected_impl()->get_cached_kernel_ids(kernels_cache); } } @@ -1930,15 +1924,10 @@ void program::load(cldnn::BinaryInputBuffer& ib) { ib >> type_name; ov::DiscreteTypeInfo type(type_name.c_str()); auto impl_manager = p_node.type()->get(type); - bool is_onednn; - ib >> is_onednn; - if (is_onednn) { - auto params = p_node.get_kernel_impl_params(); - ib.setKernelImplParams(params.get()); - ib >> p_node.selected_impl; - } else { - ib >> p_node.selected_impl; - } + + auto params = p_node.get_kernel_impl_params(); + ib.setKernelImplParams(params.get()); + ib >> p_node.selected_impl; p_node.selected_impl->m_manager = impl_manager.get(); diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/convolution_gpu_b_fs_zyx_fsv16_imad.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/convolution_gpu_b_fs_zyx_fsv16_imad.cl index fc5f2c18fe7efb..07486c1b9a1498 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/convolution_gpu_b_fs_zyx_fsv16_imad.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/convolution_gpu_b_fs_zyx_fsv16_imad.cl @@ -54,6 +54,7 @@ REQD_SUB_GROUP_SIZE(SIMD) __attribute__((reqd_work_group_size(1, 1, FEATURE_SLM_SPLIT * SIMD))) KERNEL(convolution_gpu_b_fs_zyx_fsv16_imad)( + OPTIONAL_SHAPE_INFO_ARG const __global INPUT0_TYPE *conv_input, __global OUTPUT_TYPE *output, const __global FILTER_TYPE *weights @@ -606,11 +607,15 @@ KERNEL(convolution_gpu_b_fs_zyx_fsv16_imad)( __attribute__((opencl_unroll_hint(OUT_BLOCK_WIDTH))) for (uint ow = 0; ow < OUT_BLOCK_WIDTH; ow++) { + #if !IS_DYNAMIC #if OUTPUT_SIZE_X % OUT_BLOCK_WIDTH != 0 if (out_x + OUT_BLOCK_WIDTH > OUTPUT_SIZE_X && ow >= OUTPUT_SIZE_X % OUT_BLOCK_WIDTH) break; #endif - + #else + if (OUTPUT_SIZE_X % OUT_BLOCK_WIDTH != 0 && out_x + OUT_BLOCK_WIDTH > OUTPUT_SIZE_X && ow >= OUTPUT_SIZE_X % OUT_BLOCK_WIDTH) + break; + #endif if (out_f_g < FILTER_OFM_NUM) { output[dst_index + ow * FSV + oh * OUTPUT_Y_PITCH * FSV + od * OUTPUT_Z_PITCH * FSV] = result[ofb][od][oh][ow]; } diff --git a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp index 33d13429fdcf3f..716e64937ec9e5 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp @@ -364,7 +364,8 @@ JitDefinitions DataTensorJitConstant::GetDefinitions() const { if (_tensor.GetLayout() == DataLayout::bf || _tensor.GetLayout() == DataLayout::bfyx || _tensor.GetLayout() == DataLayout::bfzyx || _tensor.GetLayout() == DataLayout::bfwzyx || _tensor.GetLayout() == DataLayout::bfuwzyx || _tensor.GetLayout() == DataLayout::bfvuwzyx || - _tensor.GetLayout() == DataLayout::b_fs_yx_fsv16 || _tensor.GetLayout() == DataLayout::b_fs_yx_fsv32) { + _tensor.GetLayout() == DataLayout::b_fs_yx_fsv16 || _tensor.GetLayout() == DataLayout::b_fs_yx_fsv32 || + _tensor.GetLayout() == DataLayout::b_fs_zyx_fsv16) { definitions.push_back({_name + "_X_PITCH", "1"}); definitions.push_back({_name + "_Y_PITCH", dims_padded.x()}); definitions.push_back({_name + "_Z_PITCH", toVectorMulString({dims_padded.x(), dims_padded.y()})}); diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_b_fs_zyx_fsv16_imad.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_b_fs_zyx_fsv16_imad.cpp index 38ed9e42d3718f..352470913e4751 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_b_fs_zyx_fsv16_imad.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_b_fs_zyx_fsv16_imad.cpp @@ -55,7 +55,10 @@ namespace kernel_selector { Convolution_kernel_b_fs_zyx_fsv16_imad::BlockParams Convolution_kernel_b_fs_zyx_fsv16_imad::GetBlockParams(const convolution_params& params) const { - size_t max_block_width = getOutBlock_X(params.outputs[0].X().v, params.stride.x, params.filterSize.x, params.dilation.x); + size_t max_block_width = 1; + if (!params.outputs[0].X().is_dynamic) { + max_block_width = getOutBlock_X(params.outputs[0].X().v, params.stride.x, params.filterSize.x, params.dilation.x); + } size_t max_in_block_width = (max_block_width - 1) * params.stride.x + (params.filterSize.x - 1) * params.dilation.x + 1; size_t block_width = max_block_width; @@ -90,7 +93,9 @@ Convolution_kernel_b_fs_zyx_fsv16_imad::GetBlockParams(const convolution_params& size_t max_slm_split = params.engineInfo.maxWorkGroupSize / simd; // TGLU exceptions related to SLM usage - if (params.engineInfo.deviceType == dev_type::integrated_gpu && params.engineInfo.computeUnitsCount == 96) { + if (params.is_shape_agnostic) { + max_slm_split = 2; + } else if (params.engineInfo.deviceType == dev_type::integrated_gpu && params.engineInfo.computeUnitsCount == 96) { bool split_exception_1 = params.outputs[0].X().v == 3 && params.outputs[0].Y().v == 3 && params.outputs[0].Z().v == 1 && params.outputs[0].Feature().v == 512; bool split_exception_2 = params.outputs[0].X().v == 5 && params.outputs[0].Y().v == 5 && params.outputs[0].Z().v == 1 @@ -118,13 +123,16 @@ Convolution_kernel_b_fs_zyx_fsv16_imad::GetBlockParams(const convolution_params& } } + size_t max_d = params.outputs[0].Z().is_dynamic ? 1 : 16; + size_t max_h = params.outputs[0].Y().is_dynamic ? 1 : 16; + for (size_t split = 1; split <= max_slm_split; split *= 2) { for (size_t temp_block_features = simd; temp_block_features <= simd * 2; temp_block_features += simd) { - for (size_t d = 1; d < 16; ++d) { - if (params.outputs[0].Z().v % d) + for (size_t d = 1; d < max_d; ++d) { + if (d != 1 && params.outputs[0].Z().v % d) continue; - for (size_t h = 1; h < 16; ++h) { - if (params.outputs[0].Y().v % h) + for (size_t h = 1; h < max_h; ++h) { + if (h != 1 && params.outputs[0].Y().v % h) continue; bool c_ifm_mul = CeilDiv(params.weights.IFM().v, fsv) % split == 0; @@ -174,6 +182,10 @@ Convolution_kernel_b_fs_zyx_fsv16_imad::GetBlockParams(const convolution_params& } float Convolution_kernel_b_fs_zyx_fsv16_imad::EstimateBlockParamsRatio(const convolution_params& params, const BlockParams& block) const { + if (params.has_dynamic_outputs()) { + return -10.f; + } + float occupancy_by_logic_size = static_cast(params.outputs[0].LogicalSize() / static_cast(params.engineInfo.maxThreadsPerDevice)); bool increase_max_reg_pressure = occupancy_by_logic_size >= 595.f; bool twice_increase_max_reg_pressure = occupancy_by_logic_size >= 595.f * 2.f; @@ -373,6 +385,7 @@ ParamsKey Convolution_kernel_b_fs_zyx_fsv16_imad::GetSupportedKey() const { k.EnableQuantization(QuantizationType::SYMMETRIC); k.EnableQuantization(QuantizationType::ASYMMETRIC_DATA); k.EnableDilation(); + k.EnableDynamicShapesSupport(); return k; } @@ -450,10 +463,15 @@ JitConstants Convolution_kernel_b_fs_zyx_fsv16_imad::GetJitConstants(const convo ConvolutionKernelBase::DispatchData Convolution_kernel_b_fs_zyx_fsv16_imad::SetDefault(const convolution_params& params, int) const { + const BlockParams& block_params = GetBlockParams(params); + return CalcDispatchDataWithBlockParams(params, block_params); +} // SetDefault + +ConvolutionKernelBase::DispatchData Convolution_kernel_b_fs_zyx_fsv16_imad::CalcDispatchDataWithBlockParams(const convolution_params& params, + const BlockParams& block_params) const { DispatchData dispatchData; const auto& output = params.outputs[0]; const auto& weights = params.weights; - auto block_params = GetBlockParams(params); dispatchData.gws[0] = CeilDiv(output.X().v, block_params.output_block_width); dispatchData.gws[1] = CeilDiv(output.Y().v, block_params.output_block_height) * CeilDiv(output.Z().v, block_params.output_block_depth); @@ -466,17 +484,24 @@ ConvolutionKernelBase::DispatchData Convolution_kernel_b_fs_zyx_fsv16_imad::SetD dispatchData.cldnnStyle = {0, 0, 0, 0, 0}; dispatchData.gemmStyle = {0, 0, 0, 0, 0, 0}; - + dispatchData.blockParams = { block_params.output_block_width, block_params.output_block_height, + block_params.output_block_depth, block_params.output_block_features, + block_params.input_block_width, block_params.input_block_height, + block_params.input_block_depth, block_params.feature_slm_split }; return dispatchData; -} // SetDefault +} KernelsPriority Convolution_kernel_b_fs_zyx_fsv16_imad::GetKernelsPriority(const Params& params) const { const auto& p = static_cast(params); - if (static_cast(p.weights.IFM().v) / static_cast(Align(p.weights.IFM().v, fsv)) < 0.5f) + if (!p.is_shape_agnostic) { + if (static_cast(p.weights.IFM().v) / static_cast(Align(p.weights.IFM().v, fsv)) < 0.5f) + return FORCE_PRIORITY_4; + else + return FORCE_PRIORITY_2; + } else { return FORCE_PRIORITY_4; - else - return FORCE_PRIORITY_2; + } } bool Convolution_kernel_b_fs_zyx_fsv16_imad::Validate(const Params& params) const { @@ -507,4 +532,23 @@ bool Convolution_kernel_b_fs_zyx_fsv16_imad::Validate(const Params& params) cons return true; } + +void Convolution_kernel_b_fs_zyx_fsv16_imad::GetUpdateDispatchDataFunc(KernelData& kd) const { + const auto& prim_params = static_cast(*kd.params); + const auto& dynamicDispatchData = SetDefault(prim_params); + + kd.update_dispatch_data_func = [this, dynamicDispatchData](const Params& params, KernelData& kd) { + const auto& prim_params = static_cast(params); + const auto& dispatchData = CalcDispatchDataWithBlockParams(prim_params, dynamicDispatchData.blockParams); + OPENVINO_ASSERT(kd.kernels.size() == 1, "[GPU] Invalid kernels size for update dispatch data func"); + kd.kernels[0].params.workGroups.global = dispatchData.gws; + kd.kernels[0].params.workGroups.local = dispatchData.lws; + kd.kernels[0].skip_execution = KernelData::SkipKernelExecution(prim_params); + + kd.internalBufferSizes.clear(); + kd.internalBufferSizes.push_back(prim_params.inputs[0].PhysicalSizeInBytes()); + kd.internalBufferDataType = prim_params.inputs[0].GetDType(); + }; +} + } // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_b_fs_zyx_fsv16_imad.h b/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_b_fs_zyx_fsv16_imad.h index c2da5e66982d11..adb5125873a657 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_b_fs_zyx_fsv16_imad.h +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_b_fs_zyx_fsv16_imad.h @@ -12,6 +12,7 @@ namespace kernel_selector { class Convolution_kernel_b_fs_zyx_fsv16_imad : public ConvolutionKernelBase { public: using Parent = ConvolutionKernelBase; + using BlockParams = DispatchData::BlockParams; Convolution_kernel_b_fs_zyx_fsv16_imad() : ConvolutionKernelBase("convolution_gpu_b_fs_zyx_fsv16_imad") {} virtual ~Convolution_kernel_b_fs_zyx_fsv16_imad() {} @@ -24,6 +25,7 @@ class Convolution_kernel_b_fs_zyx_fsv16_imad : public ConvolutionKernelBase { bool Validate(const Params& params) const override; JitConstants GetJitConstants(const convolution_params& params, const DispatchData& dispatchData) const override; DispatchData SetDefault(const convolution_params& params, int autoTuneIndex = -1) const override; + void GetUpdateDispatchDataFunc(KernelData& kd) const override; bool NeedPaddedInput() const override { return true; } WeightsLayout GetPreferredWeightsLayout(const convolution_params& p) const override { return p.groups > 1 ? WeightsLayout::g_os_is_zyx_osv16_isv16 : WeightsLayout::os_is_zyx_osv16_isv16; @@ -35,24 +37,11 @@ class Convolution_kernel_b_fs_zyx_fsv16_imad : public ConvolutionKernelBase { FusedOpType::ACTIVATION }; } - struct BlockParams { - size_t output_block_width; - size_t output_block_height; - size_t output_block_depth; - - size_t output_block_features; - - size_t input_block_width; - size_t input_block_height; - size_t input_block_depth; - - size_t feature_slm_split; - }; - BlockParams GetBlockParams(const convolution_params& params) const; float EstimateBlockParamsRatio(const convolution_params& params, const BlockParams& block) const; float EstimateRegPressure(const convolution_params& params, const BlockParams& block) const; float EstimateOccupancy(const convolution_params& params, const BlockParams& block) const; float EstimateSLMUsage(const convolution_params& params, const BlockParams& block) const; + DispatchData CalcDispatchDataWithBlockParams(const convolution_params& params, const BlockParams& block_params) const; }; } // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_base.h b/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_base.h index f8d1b3bf0de956..ccdd4941cdd1d0 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_base.h +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/convolution/convolution_kernel_base.h @@ -36,8 +36,23 @@ class ConvolutionKernelBase : public WeightBiasKernelBase { size_t globalWorkSizeDZ; }; + struct BlockParams { + size_t output_block_width; + size_t output_block_height; + size_t output_block_depth; + + size_t output_block_features; + + size_t input_block_width; + size_t input_block_height; + size_t input_block_depth; + + size_t feature_slm_split; + }; + CLDNNStyle cldnnStyle; GEMMStyle gemmStyle; + BlockParams blockParams; }; std::string GetAutoTuneOptions(int autoTuneIndex) const; diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/convolution_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/convolution_gpu_test.cpp index 4f9c31064e9026..5d01d448dcfc64 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/convolution_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/convolution_gpu_test.cpp @@ -11056,3 +11056,141 @@ INSTANTIATE_TEST_SUITE_P(smoke, conv_dyn_test, { ov::Shape{1, 32, 91}, ov::Shape{32, 1, 1, 11}, ov::Strides{1}, ov::Strides{1}, ov::CoordinateDiff{0}, ov::CoordinateDiff{0}, 32 }, { ov::Shape{1, 64, 16, 16}, ov::Shape{64, 1, 1, 3, 3}, ov::Strides{1, 1}, ov::Strides{1, 1}, ov::CoordinateDiff{0, 0}, ov::CoordinateDiff{0, 0}, 64 }, })); + + +struct conv_dyn_3d_test_params { + ov::Shape in_shape; + ov::Shape wei_shape; + ov::Strides stride; + ov::Strides dilation; + ov::CoordinateDiff pad_begin; + ov::CoordinateDiff pad_end; + uint32_t groups; + bool is_caching_test; +}; + +class conv_dyn_3d_test : public testing::TestWithParam {}; + +TEST_P(conv_dyn_3d_test, convolution_gpu_b_fs_zyx_fsv16_imad_quantized) { + auto& engine = get_test_engine(); + auto p = GetParam(); + + auto calculate_ref = [&](memory::ptr input, memory::ptr weights, + memory::ptr a_zp, memory::ptr compensation, ExecutionConfig config) { + auto in_layout = input->get_layout(); + + topology topology_ref( + input_layout("input", in_layout), + data("weights", weights), + data("a_zp", a_zp), + data("compensation", compensation), + convolution("conv", input_info("input"), "weights", no_bias, "", "a_zp", "compensation", + p.groups, p.stride, p.dilation, p.pad_begin, p.pad_end, false, data_types::f32)); + + network network_ref(engine, topology_ref, config); + network_ref.set_input_data("input", input); + + auto outputs_ref = network_ref.execute(); + + return outputs_ref.at("conv").get_memory(); + }; + + ov::PartialShape dyn_input_pshape; + for (size_t i = 0; i < p.in_shape.size(); ++i) { + dyn_input_pshape.emplace_back(ov::Dimension()); + } + dyn_input_pshape[1] = p.in_shape[1]; + + auto in_layout = layout{dyn_input_pshape, data_types::u8, format::b_fs_zyx_fsv16}; + auto input = engine.allocate_memory({ p.in_shape, data_types::u8, format::b_fs_zyx_fsv16 }); + auto weights = engine.allocate_memory({p.wei_shape, data_types::i8, format::bfzyx}); + + auto a_zp_shape = ov::Shape(p.in_shape.size(), 1); + a_zp_shape[1] = p.in_shape[1]; + auto a_zp = engine.allocate_memory({ a_zp_shape, data_types::u8, format::bfyx }); + + auto compensation = engine.allocate_memory({ a_zp_shape, data_types::f32, format::bfyx }); + + tests::random_generator rg(GET_SUITE_NAME); + VF input_rnd = rg.generate_random_1d(ov::shape_size(p.in_shape), 0, 10); + VF weights_rnd = rg.generate_random_1d(ov::shape_size(p.wei_shape), -5, 5); + VF a_zp_rnd = rg.generate_random_1d(ov::shape_size(a_zp_shape), 1, 5); + VF compensation_rnd = rg.generate_random_1d(ov::shape_size(a_zp_shape), -5, 5); + + set_values(input, input_rnd); + set_values(weights, weights_rnd); + set_values(a_zp, a_zp_rnd); + set_values(compensation, compensation_rnd); + + topology topology( + input_layout("input", in_layout), + data("weights", weights), + data("a_zp", a_zp), + data("compensation", compensation), + convolution("conv", input_info("input"), "weights", no_bias, "", "a_zp", "compensation", + p.groups, p.stride, p.dilation, p.pad_begin, p.pad_end, false, data_types::f32)); + + ExecutionConfig config = get_test_default_config(engine); + ov::intel_gpu::ImplementationDesc conv_impl = { format::b_fs_zyx_fsv16, "convolution_gpu_b_fs_zyx_fsv16_imad", impl_types::ocl }; + config.set_property(ov::intel_gpu::force_implementations(ov::intel_gpu::ImplForcingMap{ { "conv", conv_impl } })); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + config.set_property(ov::enable_profiling(true)); + + cldnn::network::ptr network = get_network(engine, topology, config, get_test_stream_ptr(), p.is_caching_test); + network->set_input_data("input", input); + + auto inst = network->get_primitive("conv"); + auto impl = inst->get_impl(); + ASSERT_TRUE(impl != nullptr); + ASSERT_TRUE(impl->is_dynamic()); + + auto outputs = network->execute(); + + auto output_memory = outputs.at("conv").get_memory(); + + auto output_memory_ref = calculate_ref(input, weights, a_zp, compensation, config); + + cldnn::mem_lock output_ptr(output_memory, get_test_stream()); + cldnn::mem_lock output_ptr_ref(output_memory_ref, get_test_stream()); + + ASSERT_EQ(outputs.at("conv").get_layout(), output_memory_ref->get_layout()); + for (size_t i = 0; i < output_ptr.size(); i++) { + ASSERT_EQ(output_ptr[i], output_ptr_ref[i]); + } + + { + // Change original shape for the second run + auto new_shape = p.in_shape; + new_shape[2] += 4; + + auto input = engine.allocate_memory({ new_shape, data_types::u8, format::b_fs_zyx_fsv16 }); + + VF input_rnd = rg.generate_random_1d(ov::shape_size(p.in_shape), 0, 10); + set_values(input, input_rnd); + + network->set_input_data("input", input); + auto outputs = network->execute(); + + auto output_memory = outputs.at("conv").get_memory(); + auto output_memory_ref = calculate_ref(input, weights, a_zp, compensation, config); + + cldnn::mem_lock output_ptr(output_memory, get_test_stream()); + cldnn::mem_lock output_ptr_ref(output_memory_ref, get_test_stream()); + + ASSERT_EQ(outputs.at("conv").get_layout(), output_memory_ref->get_layout()); + for (size_t i = 0; i < output_ptr.size(); i++) { + ASSERT_EQ(output_ptr[i], output_ptr_ref[i]); + } + } +} + +INSTANTIATE_TEST_SUITE_P(smoke, conv_dyn_3d_test, + testing::ValuesIn(std::vector{ + { ov::Shape{1, 16, 5, 5, 5}, ov::Shape{2, 16, 3, 3, 3}, ov::Strides{1, 1, 1}, ov::Strides{1, 1, 1}, ov::CoordinateDiff{0, 0, 0}, ov::CoordinateDiff{0, 0, 0}, 1, false}, + { ov::Shape{1, 16, 5, 5, 5}, ov::Shape{2, 16, 3, 3, 3}, ov::Strides{1, 1, 1}, ov::Strides{1, 1, 1}, ov::CoordinateDiff{0, 0, 0}, ov::CoordinateDiff{0, 0, 0}, 1, true}, + { ov::Shape{2, 32, 30, 30, 30}, ov::Shape{16, 32, 10, 10, 10}, ov::Strides{1, 1, 1}, ov::Strides{1, 1, 1}, ov::CoordinateDiff{0, 0, 0}, ov::CoordinateDiff{0, 0, 0}, 1, false }, + { ov::Shape{2, 32, 30, 30, 30}, ov::Shape{16, 32, 10, 10, 10}, ov::Strides{1, 1, 1}, ov::Strides{1, 1, 1}, ov::CoordinateDiff{0, 0, 0}, ov::CoordinateDiff{0, 0, 0}, 1, true }, + { ov::Shape{1, 16, 5, 5, 5}, ov::Shape{2, 16, 3, 3, 3}, ov::Strides{2, 2, 2}, ov::Strides{1, 1, 1}, ov::CoordinateDiff{0, 0, 0}, ov::CoordinateDiff{0, 0, 0}, 1, false }, + { ov::Shape{1, 16, 5, 5, 5}, ov::Shape{2, 16, 3, 3, 3}, ov::Strides{1, 1, 1}, ov::Strides{1, 1, 1}, ov::CoordinateDiff{1, 1, 1}, ov::CoordinateDiff{1, 1, 1}, 1, false }, + { ov::Shape{1, 16, 5, 5, 5}, ov::Shape{16, 1, 1, 3, 3, 3}, ov::Strides{1, 1, 1}, ov::Strides{1, 1, 1}, ov::CoordinateDiff{0, 0, 0}, ov::CoordinateDiff{0, 0, 0}, 16, false } +})); From 05c365c5e4fdc78a5d7137963fad233c36da972e Mon Sep 17 00:00:00 2001 From: Przemyslaw Wysocki Date: Wed, 6 Nov 2024 12:00:55 +0100 Subject: [PATCH 25/93] [PyOV] Migrate benchmark_app from `setup.py` to `pyproject.toml` (#27305) ### Details: - Added `--no-deps` because `pip wheel .` with `pyproject.toml` would save the `numpy` wheel in cwd - Since requirements are added to the benchmark_app wheel itself, when installing it they'll be pulled from PyPI anyway ### Tickets: - CVS-156008 --------- Signed-off-by: p-wysocki Co-authored-by: Anastasia Kuporosova Co-authored-by: Michal Lukaszewski --- tools/benchmark_tool/pyproject.toml | 31 +++++++++++++++++++ tools/benchmark_tool/requirements.txt | 1 - tools/benchmark_tool/setup.py | 43 --------------------------- 3 files changed, 31 insertions(+), 44 deletions(-) create mode 100644 tools/benchmark_tool/pyproject.toml delete mode 100644 tools/benchmark_tool/requirements.txt delete mode 100644 tools/benchmark_tool/setup.py diff --git a/tools/benchmark_tool/pyproject.toml b/tools/benchmark_tool/pyproject.toml new file mode 100644 index 00000000000000..61fe23ffbbf539 --- /dev/null +++ b/tools/benchmark_tool/pyproject.toml @@ -0,0 +1,31 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# Use this configuration file to create a wheel with OpenVINO™ Python* benchmark tool: +# $ python -m pip wheel . --no-deps + +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "benchmark_tool" +version = "0.0.0" +description = "OpenVINO™ Python* tools package" +readme = "README.md" +authors = [ + {name = "Intel® Corporation", email = "openvino_pushbot@intel.com"} +] +urls = {homepage = "https://github.com/openvinotoolkit/openvino"} +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + "License :: OSI Approved :: Apache Software License" +] +requires-python = ">=3.9" +dependencies = [ + "numpy>=1.16.6,<2.2.0", +] + +[project.scripts] +benchmark_app = "openvino.tools.benchmark.main:main" diff --git a/tools/benchmark_tool/requirements.txt b/tools/benchmark_tool/requirements.txt deleted file mode 100644 index 5594ff31cd35c5..00000000000000 --- a/tools/benchmark_tool/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -numpy>=1.16.6,<2.1.0 diff --git a/tools/benchmark_tool/setup.py b/tools/benchmark_tool/setup.py deleted file mode 100644 index 98f2a369a60360..00000000000000 --- a/tools/benchmark_tool/setup.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2018-2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -""" -Use this script to create a wheel with OpenVINO™ Python* tools: - -$ python -m pip wheel . -""" -from setuptools import setup, find_packages - - -with open('requirements.txt', 'r', encoding='utf-8') as f: - raw_contents = f.readlines() - reqs = [line.strip() for line in raw_contents] - -with open('README.md', 'r', encoding='utf-8') as f: - long_description = f.read() - -setup( - name='benchmark_tool', - version='0.0.0', - author='Intel® Corporation', - license='OSI Approved :: Apache Software License', - author_email='openvino_pushbot@intel.com', - url='https://github.com/openvinotoolkit/openvino', - description='OpenVINO™ Python* tools package', - long_description=long_description, - long_description_content_type='text/markdown', - entry_points={ - 'console_scripts': [ - 'benchmark_app = openvino.tools.benchmark.main:main'], - }, - classifiers=[ - 'Programming Language :: Python :: 3', - 'Operating System :: OS Independent', - ], - packages=find_packages(), - install_requires=reqs, - data_files=[('.', ['requirements.txt'])], - python_requires='>=3.9', -) From eabe5282511563d3fc4325f25234e338b0d8a28a Mon Sep 17 00:00:00 2001 From: Bogdan Pereanu Date: Wed, 6 Nov 2024 14:17:17 +0200 Subject: [PATCH 26/93] [NPU] Fix compiler clang build (#27405) ### Details: - *Change from define to constexpr and remove unused code* - *Fix clang build* ### Tickets: - *EISW-146067* --------- Signed-off-by: Bogdan Pereanu --- .../intel_npu/src/al/include/intel_npu/prefix.hpp | 10 +++------- .../src/compiler_adapter/src/ze_graph_ext_wrappers.cpp | 1 - .../utils/include/intel_npu/utils/zero/zero_types.hpp | 4 ++-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/prefix.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/prefix.hpp index 9be19d67454d82..f80390ef61ca7e 100644 --- a/src/plugins/intel_npu/src/al/include/intel_npu/prefix.hpp +++ b/src/plugins/intel_npu/src/al/include/intel_npu/prefix.hpp @@ -11,9 +11,9 @@ namespace intel_npu { // // Prefix for ReadValue and Assign operations in compiler. // -#define READVALUE_PREFIX std::string("vpux_ie_read_value_") -#define ASSIGN_PREFIX std::string("vpux_ie_assign_") -#define SHAPE_TENSOR_PREFIX std::string("vpux_ie_shape_") +constexpr std::string_view READVALUE_PREFIX = "vpux_ie_read_value_"; +constexpr std::string_view ASSIGN_PREFIX = "vpux_ie_assign_"; +constexpr std::string_view SHAPE_TENSOR_PREFIX = "vpux_ie_shape_"; inline bool isStateInputName(const std::string& name) { return !name.compare(0, READVALUE_PREFIX.length(), READVALUE_PREFIX); @@ -25,8 +25,4 @@ inline bool isShapeTensorName(const std::string& name) { return !name.compare(0, SHAPE_TENSOR_PREFIX.length(), SHAPE_TENSOR_PREFIX); } -inline std::string stateOutputToStateInputName(const std::string& name) { - return READVALUE_PREFIX + name.substr(ASSIGN_PREFIX.length()); -} - } // namespace intel_npu diff --git a/src/plugins/intel_npu/src/compiler_adapter/src/ze_graph_ext_wrappers.cpp b/src/plugins/intel_npu/src/compiler_adapter/src/ze_graph_ext_wrappers.cpp index c094838cc22ef0..095f470e15e02f 100644 --- a/src/plugins/intel_npu/src/compiler_adapter/src/ze_graph_ext_wrappers.cpp +++ b/src/plugins/intel_npu/src/compiler_adapter/src/ze_graph_ext_wrappers.cpp @@ -396,7 +396,6 @@ ze_graph_handle_t ZeGraphExtWrappers::getGraphHandle( const uint32_t& flags) const { ze_graph_handle_t graphHandle; - _logger.info("compileIR Using extension version: %s", typeid(TableExtension).name()); createGraph(std::move(serializedIR), buildFlags, flags, &graphHandle); return graphHandle; diff --git a/src/plugins/intel_npu/src/utils/include/intel_npu/utils/zero/zero_types.hpp b/src/plugins/intel_npu/src/utils/include/intel_npu/utils/zero/zero_types.hpp index 032dd867306004..188da886f5d9e4 100644 --- a/src/plugins/intel_npu/src/utils/include/intel_npu/utils/zero/zero_types.hpp +++ b/src/plugins/intel_npu/src/utils/include/intel_npu/utils/zero/zero_types.hpp @@ -70,7 +70,7 @@ struct ze_graph_dditable_ext_decorator final { } ~ze_graph_dditable_ext_decorator() = default; - inline const uint32_t version() const { + inline uint32_t version() const { return _driverExtVersion; } @@ -202,7 +202,7 @@ struct ze_command_queue_npu_dditable_ext_decorator final { _commandQueueExtVersion(commandQueueExtVersion) {} ~ze_command_queue_npu_dditable_ext_decorator() = default; - inline const uint32_t version() const { + inline uint32_t version() const { return _commandQueueExtVersion; } From 6c715998b36012c26d59e73fe86fca00fee761d3 Mon Sep 17 00:00:00 2001 From: Przemyslaw Wysocki Date: Wed, 6 Nov 2024 13:48:23 +0100 Subject: [PATCH 27/93] [PyOV] Use `python -m build` only for old `pip` versions (#27429) ### Details: - An alternative to https://github.com/openvinotoolkit/openvino/pull/27426 - The functionality branching based on pip version has recently been removed in https://github.com/openvinotoolkit/openvino/pull/27190 - This fix has originated from discussion at https://github.com/openvinotoolkit/openvino/pull/27190/files#r1830707810 - Fix for issues with NPU building on Windows machines - A disadvantage is that we need to have `build` as a dependency no matter what, there's no environment marker for `pip` version which would allow us to install it conditionally ### Tickets: - EISW-146038 Signed-off-by: p-wysocki --- src/bindings/python/wheel/CMakeLists.txt | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/bindings/python/wheel/CMakeLists.txt b/src/bindings/python/wheel/CMakeLists.txt index 72828a155dd865..d943c3caa21e52 100644 --- a/src/bindings/python/wheel/CMakeLists.txt +++ b/src/bindings/python/wheel/CMakeLists.txt @@ -100,14 +100,31 @@ set(openvino_wheel_path "${openvino_wheels_output_dir}/${openvino_wheel_name}") # create target for openvino.wheel # -# for --config-setting explanation see https://github.com/pypa/setuptools/issues/2491 -set(wheel_build_command - ${Python3_EXECUTABLE} -m build "${CMAKE_CURRENT_SOURCE_DIR}" +execute_process(COMMAND ${Python3_EXECUTABLE} -m pip --version + OUTPUT_VARIABLE pip_version OUTPUT_STRIP_TRAILING_WHITESPACE) + +string(REGEX MATCH "pip[ ]+([\\.0-9]*)" pip_version "${pip_version}") +set(pip_version ${CMAKE_MATCH_1}) + +if(pip_version VERSION_GREATER_EQUAL 22.0) + set(wheel_build_command + ${Python3_EXECUTABLE} -m pip wheel + --no-deps + --wheel-dir ${openvino_wheels_output_dir} + --verbose + --build-option --build-number=${WHEEL_BUILD} + --build-option --plat-name=${PLATFORM_TAG} + "${CMAKE_CURRENT_SOURCE_DIR}") +else() + # for --config-setting explanation see https://github.com/pypa/setuptools/issues/2491 + set(wheel_build_command + ${Python3_EXECUTABLE} -m build "${CMAKE_CURRENT_SOURCE_DIR}" --outdir ${openvino_wheels_output_dir} --config-setting=--build-option=--build-number=${WHEEL_BUILD} --config-setting=--build-option=--plat-name=${PLATFORM_TAG} --config-setting=--quiet --wheel) +endif() add_custom_command(OUTPUT ${openvino_wheel_path} COMMAND ${setup_py_env} ${wheel_build_command} From 95d6be201f9cbb2e41c5d58d05f6638e84c71606 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 6 Nov 2024 15:01:33 +0100 Subject: [PATCH 28/93] [GHA] WA parallel make issue (#27422) ### Details: - To address the issue mentioned [here](https://unix.stackexchange.com/questions/316644/is-make-j-with-no-argument-dangerous), we need to set the number of parallel jobs explicitly ### Tickets: - *156361* --- .github/workflows/android_arm64.yml | 2 +- .github/workflows/android_x64.yml | 4 ++-- .github/workflows/coverity.yml | 2 +- .github/workflows/dev_cpu_linux_snippets_libxsmm.yml | 2 +- .github/workflows/job_build_linux.yml | 10 +++++----- .github/workflows/job_build_windows.yml | 4 ++-- .github/workflows/job_onnx_runtime.yml | 2 +- .github/workflows/linux_conditional_compilation.yml | 4 ++-- .github/workflows/linux_riscv.yml | 4 ++-- .github/workflows/mac.yml | 6 +++--- .github/workflows/mac_arm64.yml | 6 +++--- .github/workflows/ubuntu_22.yml | 2 +- .github/workflows/windows_conditional_compilation.yml | 4 ++-- 13 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/android_arm64.yml b/.github/workflows/android_arm64.yml index fca16f2848f7bb..e0954871f4b51e 100644 --- a/.github/workflows/android_arm64.yml +++ b/.github/workflows/android_arm64.yml @@ -169,7 +169,7 @@ jobs: run: ${SCCACHE_PATH} --zero-stats - name: Cmake - build - run: cmake --build ${BUILD_DIR} --parallel + run: cmake --build ${BUILD_DIR} --parallel $(nproc) - name: Show ccache stats run: ${SCCACHE_PATH} --show-stats diff --git a/.github/workflows/android_x64.yml b/.github/workflows/android_x64.yml index 47ea6d2cd90ffe..b0b46c662abdbb 100644 --- a/.github/workflows/android_x64.yml +++ b/.github/workflows/android_x64.yml @@ -73,7 +73,7 @@ jobs: defaults: run: shell: bash - runs-on: aks-linux-16-cores-64gb + runs-on: aks-linux-16-cores-32gb container: image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_android }} volumes: @@ -144,7 +144,7 @@ jobs: run: ${SCCACHE_PATH} --zero-stats - name: Cmake - build - run: cmake --build ${BUILD_DIR} --parallel + run: cmake --build ${BUILD_DIR} --parallel $(nproc) - name: Show ccache stats run: ${SCCACHE_PATH} --show-stats diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 9797414cde56c8..5e3aa3f9bf0fc2 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -164,7 +164,7 @@ jobs: popd - name: Cmake build - OpenVINO with Coverity - run: ${COVERITY_TOOL_DIR}/cov-analysis*/bin/cov-build --dir ${BUILD_DIR}/cov-int cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + run: ${COVERITY_TOOL_DIR}/cov-analysis*/bin/cov-build --dir ${BUILD_DIR}/cov-int cmake --build ${BUILD_DIR} --parallel $ENV:NUMBER_OF_PROCESSORS --config ${{ env.CMAKE_BUILD_TYPE }} - name: Show sccache stats run: ${SCCACHE_PATH} --show-stats diff --git a/.github/workflows/dev_cpu_linux_snippets_libxsmm.yml b/.github/workflows/dev_cpu_linux_snippets_libxsmm.yml index 2f6d646f8e271d..ba458da5d3ec1a 100644 --- a/.github/workflows/dev_cpu_linux_snippets_libxsmm.yml +++ b/.github/workflows/dev_cpu_linux_snippets_libxsmm.yml @@ -143,7 +143,7 @@ jobs: run: ${SCCACHE_PATH} --zero-stats - name: Cmake build - OpenVINO - run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + run: cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: Show sccache stats run: ${SCCACHE_PATH} --show-stats diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index e491388a4b3606..3964f049be2abb 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -168,7 +168,7 @@ jobs: run: ${SCCACHE_PATH} --zero-stats - name: Cmake build - OpenVINO - run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + run: cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: Show sccache stats run: ${SCCACHE_PATH} --show-stats @@ -210,7 +210,7 @@ jobs: -DPython3_EXECUTABLE=$python_exec \ -DCPACK_GENERATOR=DEB \ ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target package + cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} --target package - name: Cmake & Build - OpenVINO Contrib if: ${{ inputs.build-contrib }} @@ -221,7 +221,7 @@ jobs: -DENABLE_WHEEL=OFF \ -S ${OPENVINO_REPO} \ -B ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: CMake configure, build and install - OpenVINO JS API if: ${{ fromJSON(inputs.affected-components).JS_API && inputs.build-js }} @@ -230,7 +230,7 @@ jobs: -DCPACK_GENERATOR=NPM \ -DENABLE_SYSTEM_TBB=OFF \ -DENABLE_WHEEL=OFF - cmake --build ${BUILD_DIR} --parallel + cmake --build ${BUILD_DIR} --parallel $(nproc) cmake --install ${BUILD_DIR} --prefix ${INSTALL_DIR_JS} - name: Build RPM packages @@ -243,7 +243,7 @@ jobs: -DENABLE_WHEEL=OFF \ -DENABLE_TESTS=OFF \ ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel --target package --verbose + cmake --build ${BUILD_DIR} --parallel $(nproc) --target package --verbose # # Upload build artifacts and logs diff --git a/.github/workflows/job_build_windows.yml b/.github/workflows/job_build_windows.yml index 0567109e0dea46..4ea82590e7abd5 100644 --- a/.github/workflows/job_build_windows.yml +++ b/.github/workflows/job_build_windows.yml @@ -164,7 +164,7 @@ jobs: run: '& ccache --zero-stats' - name: Cmake build - OpenVINO - run: cmake --build ${{ env.BUILD_DIR }} --config ${{ env.CMAKE_BUILD_TYPE }} --parallel --verbose + run: cmake --build ${{ env.BUILD_DIR }} --config ${{ env.CMAKE_BUILD_TYPE }} --parallel $ENV:NUMBER_OF_PROCESSORS --verbose - name: Show ccache stats run: '& ccache --show-stats' @@ -201,7 +201,7 @@ jobs: -DCPACK_GENERATOR=NPM ` -DENABLE_SYSTEM_TBB=OFF ` -DENABLE_WHEEL=OFF - cmake --build ${{ env.BUILD_DIR }} --parallel + cmake --build ${{ env.BUILD_DIR }} --parallel $ENV:NUMBER_OF_PROCESSORS cmake --install ${{ env.BUILD_DIR }} --config ${{ env.CMAKE_BUILD_TYPE }} --prefix ${{ env.INSTALL_DIR_JS }} # diff --git a/.github/workflows/job_onnx_runtime.yml b/.github/workflows/job_onnx_runtime.yml index 966d258a2fc609..df50c4f3e2ad3c 100644 --- a/.github/workflows/job_onnx_runtime.yml +++ b/.github/workflows/job_onnx_runtime.yml @@ -96,7 +96,7 @@ jobs: --config RelWithDebInfo \ --use_openvino CPU \ --build_shared_lib \ - --parallel \ + --parallel $(nproc) \ --skip_tests \ --compile_no_warning_as_error \ --allow_running_as_root \ diff --git a/.github/workflows/linux_conditional_compilation.yml b/.github/workflows/linux_conditional_compilation.yml index acb7e1271d7a34..27f54da6ecdc60 100644 --- a/.github/workflows/linux_conditional_compilation.yml +++ b/.github/workflows/linux_conditional_compilation.yml @@ -169,7 +169,7 @@ jobs: - name: Cmake build - CC COLLECT run: | cmake --build ${BUILD_DIR} --parallel 8 --config ${{ env.CMAKE_BUILD_TYPE }} - cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target sea_itt_lib + cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} --target sea_itt_lib - name: Show sccache stats run: ${SCCACHE_PATH} --show-stats @@ -182,7 +182,7 @@ jobs: - name: Build C++ samples - OpenVINO build tree run: | cmake -G "${{ env.CMAKE_GENERATOR }}" -DOpenVINO_DIR=${BUILD_DIR} -S ${INSTALL_DIR}/samples/cpp -B ${BUILD_DIR}/cpp_samples - cmake --build ${BUILD_DIR}/cpp_samples --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target hello_query_device + cmake --build ${BUILD_DIR}/cpp_samples --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} --target hello_query_device - name: Build C samples - OpenVINO install tree run: ${INSTALL_DIR}/samples/c/build_samples.sh -i ${INSTALL_DIR} -b ${BUILD_DIR}/c_samples diff --git a/.github/workflows/linux_riscv.yml b/.github/workflows/linux_riscv.yml index c450a5d30768e4..85b0db8c36294e 100644 --- a/.github/workflows/linux_riscv.yml +++ b/.github/workflows/linux_riscv.yml @@ -213,13 +213,13 @@ jobs: source ${OPENVINO_BUILD_DIR}/dependencies/deactivate_conanbuild.sh - name: Cmake - Build - run: cmake --build ${OPENVINO_BUILD_DIR} --parallel + run: cmake --build ${OPENVINO_BUILD_DIR} --parallel $(nproc) - name: Show ccache stats run: ccache --show-stats - name: Cmake - Install - run: cmake --build ${OPENVINO_BUILD_DIR} --parallel --target install + run: cmake --build ${OPENVINO_BUILD_DIR} --parallel $(nproc) --target install - name: Build OpenVINO C++ samples run: | diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 5492ad40aa17b4..c587c5ad7323b3 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -159,7 +159,7 @@ jobs: -B ${{ env.BUILD_DIR }} - name: Cmake build - OpenVINO - run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + run: cmake --build ${{ env.BUILD_DIR }} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: Show ccache stats run: ccache --show-stats @@ -186,7 +186,7 @@ jobs: -DOPENVINO_EXTRA_MODULES=${{ env.OPENVINO_CONTRIB_REPO }}/modules/custom_operations \ -S ${{ env.OPENVINO_REPO }} \ -B ${{ env.BUILD_DIR }} - cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + cmake --build ${{ env.BUILD_DIR }} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: CMake configure, build and install - OpenVINO JS API if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API @@ -196,7 +196,7 @@ jobs: -S ${{ env.OPENVINO_REPO }} \ -B ${{ env.BUILD_DIR }} - cmake --build ${{ env.BUILD_DIR }} --parallel + cmake --build ${{ env.BUILD_DIR }} --parallel $(nproc) cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR_JS }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake # diff --git a/.github/workflows/mac_arm64.yml b/.github/workflows/mac_arm64.yml index 8100b74734ab17..0708a844fe6b8b 100644 --- a/.github/workflows/mac_arm64.yml +++ b/.github/workflows/mac_arm64.yml @@ -159,7 +159,7 @@ jobs: -B ${{ env.BUILD_DIR }} - name: Cmake build - OpenVINO - run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + run: cmake --build ${{ env.BUILD_DIR }} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: Show ccache stats run: ccache --show-stats @@ -187,7 +187,7 @@ jobs: -DOPENVINO_EXTRA_MODULES=${{ env.OPENVINO_CONTRIB_REPO }}/modules/custom_operations \ -S ${{ env.OPENVINO_REPO }} \ -B ${{ env.BUILD_DIR }} - cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + cmake --build ${{ env.BUILD_DIR }} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: CMake configure, build and install - OpenVINO JS API if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API @@ -197,7 +197,7 @@ jobs: -S ${{ env.OPENVINO_REPO }} \ -B ${{ env.BUILD_DIR }} - cmake --build ${{ env.BUILD_DIR }} --parallel + cmake --build ${{ env.BUILD_DIR }} --parallel $(nproc) cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR_JS }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake # diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 92178fce7f5054..f4caec8b2458a0 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -477,7 +477,7 @@ jobs: -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ -S ${OPENVINO_CONTRIB_REPO}/modules/nvidia_plugin \ -B ${NVIDIA_BUILD_DIR} - cmake --build ${NVIDIA_BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --verbose -- ov_nvidia_func_tests ov_nvidia_unit_tests + cmake --build ${NVIDIA_BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} --verbose -- ov_nvidia_func_tests ov_nvidia_unit_tests - name: Show ccache stats run: ${SCCACHE_PATH} --show-stats diff --git a/.github/workflows/windows_conditional_compilation.yml b/.github/workflows/windows_conditional_compilation.yml index d93c737606a2b4..6a5846b514dbd7 100644 --- a/.github/workflows/windows_conditional_compilation.yml +++ b/.github/workflows/windows_conditional_compilation.yml @@ -193,7 +193,7 @@ jobs: - name: Build C++ samples - OpenVINO build tree run: | cmake -G "${{ env.CMAKE_GENERATOR }}" -DOpenVINO_DIR=${{ env.BUILD_DIR }} -S ${{ env.INSTALL_DIR }}/samples/cpp -B ${{ env.BUILD_DIR }}/cpp_samples - cmake --build ${{ env.BUILD_DIR }}/cpp_samples --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target hello_query_device + cmake --build ${{ env.BUILD_DIR }}/cpp_samples --parallel $ENV:NUMBER_OF_PROCESSORS --config ${{ env.CMAKE_BUILD_TYPE }} --target hello_query_device - name: Build C samples - OpenVINO install tree run: | @@ -331,7 +331,7 @@ jobs: -B ${{ env.BUILD_DIR }} - name: Cmake build - CC ON - run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target benchmark_app --verbose + run: cmake --build ${{ env.BUILD_DIR }} --parallel $ENV:NUMBER_OF_PROCESSORS --config ${{ env.CMAKE_BUILD_TYPE }} --target benchmark_app --verbose - name: List bin files shell: cmd From 42d4377a97e025d34c8f38b9504b85151b5473e9 Mon Sep 17 00:00:00 2001 From: Sergey Shlyapnikov Date: Wed, 6 Nov 2024 19:47:51 +0400 Subject: [PATCH 29/93] [GPU] Fix accuracy issue in PagedAttention kernel for large prompts (#27433) ### Details: - Fix accuracy issue in PagedAttention kernel for large prompts (4K/8K tokens) by correcting index calculation in sub_group_broadcast function to ensure accurate data broadcasting within the subgroup ### Tickets: - [CVS-154881](https://jira.devtools.intel.com/browse/CVS-154881) --- .../intel_gpu/src/kernel_selector/cl_kernels/pa_sdpa_opt.cl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/pa_sdpa_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/pa_sdpa_opt.cl index a3bdd7e12dcd49..00c43829d02ea7 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/pa_sdpa_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/pa_sdpa_opt.cl @@ -436,7 +436,7 @@ KERNEL(pa_sdpa_finalization_stage)( partition_num * HEAD_SIZE + head_size_idx; OUTPUT_TYPE out_val = tmp_out[tmp_out_offset]; - acc += TO_SOFTMAX_ACCUMULATOR_TYPE(out_val) * TO_SOFTMAX_ACCUMULATOR_TYPE(sub_group_broadcast(exp_sum[partition_num / SUBGROUP_SIZE], partition_num)) / TO_SOFTMAX_ACCUMULATOR_TYPE(global_sum); + acc += TO_SOFTMAX_ACCUMULATOR_TYPE(out_val) * TO_SOFTMAX_ACCUMULATOR_TYPE(sub_group_broadcast(exp_sum[partition_num / SUBGROUP_SIZE], partition_num % SUBGROUP_SIZE)) / TO_SOFTMAX_ACCUMULATOR_TYPE(global_sum); } const uint out_offset = seq_idx * (HEADS_NUM * HEAD_SIZE) + head_num_idx * HEAD_SIZE + From 33cba008c50ddac49efc12b15cb933f4feeacba5 Mon Sep 17 00:00:00 2001 From: Alexey Smirnov Date: Wed, 6 Nov 2024 17:24:18 +0000 Subject: [PATCH 30/93] [NPUW] Support i16/u16 constants during partitioning (#27437) --- .../intel_npu/src/plugin/npuw/partitioning/partitioning.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/plugins/intel_npu/src/plugin/npuw/partitioning/partitioning.cpp b/src/plugins/intel_npu/src/plugin/npuw/partitioning/partitioning.cpp index 99705fef30e8a8..dbc86c5062da9e 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/partitioning/partitioning.cpp +++ b/src/plugins/intel_npu/src/plugin/npuw/partitioning/partitioning.cpp @@ -1287,6 +1287,8 @@ void Partitioner::saveRepeatedConstants(const std::string& func_name) { HANDLE_CASE(boolean, bool); HANDLE_CASE(i4, int8_t); HANDLE_CASE(u4, uint8_t); + HANDLE_CASE(i16, int16_t); + HANDLE_CASE(u16, uint16_t); HANDLE_CASE(i32, int); HANDLE_CASE(i64, int64_t); HANDLE_CASE(f16, uint16_t); From 9226bbe7dc108af10bf40b022f09b605ada081bb Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 7 Nov 2024 09:42:19 +0100 Subject: [PATCH 31/93] [GHA] Coverity build regression fix (#27445) ### Details: - Fixed regression introduced by https://github.com/openvinotoolkit/openvino/pull/27422 ### Tickets: - *ticket-id* --- .github/workflows/coverity.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 5e3aa3f9bf0fc2..e26bd66e097e33 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -164,7 +164,7 @@ jobs: popd - name: Cmake build - OpenVINO with Coverity - run: ${COVERITY_TOOL_DIR}/cov-analysis*/bin/cov-build --dir ${BUILD_DIR}/cov-int cmake --build ${BUILD_DIR} --parallel $ENV:NUMBER_OF_PROCESSORS --config ${{ env.CMAKE_BUILD_TYPE }} + run: ${COVERITY_TOOL_DIR}/cov-analysis*/bin/cov-build --dir ${BUILD_DIR}/cov-int cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: Show sccache stats run: ${SCCACHE_PATH} --show-stats From 5fad8053be6aeb954844aef8511ac0610ba96227 Mon Sep 17 00:00:00 2001 From: Sergey Shlyapnikov Date: Thu, 7 Nov 2024 14:30:59 +0400 Subject: [PATCH 32/93] [GPU] Enable KV-cache compression by default for non-systolic platforms (#27410) ### Details: - Enable KV-cache compression by default for non-systolic platforms --- .../cl_kernels/dynamic_quantize_gpu_kv_cache.cl | 6 +++--- .../dynamic_quantize_kernel_opt_kv_cache.cpp | 3 +++ src/plugins/intel_gpu/src/runtime/execution_config.cpp | 5 +++++ .../functional/subgraph_tests/dynamic/kv_cache_sdpa.cpp | 5 ++++- .../tests/unit/test_cases/dynamic_quantize_gpu_test.cpp | 7 +++---- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_kv_cache.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_kv_cache.cl index 22a2f03c94564a..b0e363169e9e4d 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_kv_cache.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_kv_cache.cl @@ -83,11 +83,11 @@ KERNEL(dynamic_quantize_gpu_kv_cache)( #if ASYMMETRIC_QUANTIZATION min_value = work_group_reduce_min(min_value); max_value = work_group_reduce_max(max_value); - OUTPUT1_TYPE scale = (OUTPUT1_TYPE)((CHAR_MAX - CHAR_MIN) / (max_value - min_value)); - OUTPUT1_TYPE zp = (OUTPUT1_TYPE)(-min_value * scale) - CHAR_MAX; + ACCUMULATOR_TYPE scale = (ACCUMULATOR_TYPE)((CHAR_MAX - CHAR_MIN) / (max_value - min_value)); + ACCUMULATOR_TYPE zp = (ACCUMULATOR_TYPE)(-min_value * scale) - CHAR_MAX; #else max_value = work_group_reduce_max(max_value); - OUTPUT1_TYPE scale = 127.0h / max_value; + ACCUMULATOR_TYPE scale = 127.0h / max_value; #endif #ifdef APPEND_MODE diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt_kv_cache.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt_kv_cache.cpp index d0c99484e3f52e..8f7537eeeb5d7d 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt_kv_cache.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt_kv_cache.cpp @@ -141,6 +141,9 @@ JitConstants DynamicQuantizeKernelKVCache::GetJitConstants(const dynamic_quantiz jit.AddConstant(MakeJitConstant("ASYMMETRIC_QUANTIZATION", params.use_asymmetric_quantization)); jit.AddConstant(MakeJitConstant("GROUP_SCALES_WITH_ZP", params.combine_scales_and_zp)); + // Use FP32 accumulator type for scale/zp calculation + jit.Merge(MakeTypeJitConstants(Datatype::F32, "ACCUMULATOR")); + bool rearrange_scales_order = false; const auto& scales_output_order = params.scales_output_order; if (!scales_output_order.empty()) { diff --git a/src/plugins/intel_gpu/src/runtime/execution_config.cpp b/src/plugins/intel_gpu/src/runtime/execution_config.cpp index 7c8e55cddfe593..44758f73289edb 100644 --- a/src/plugins/intel_gpu/src/runtime/execution_config.cpp +++ b/src/plugins/intel_gpu/src/runtime/execution_config.cpp @@ -246,6 +246,11 @@ void ExecutionConfig::apply_user_properties(const cldnn::device_info& info) { set_property(ov::intel_gpu::queue_type(QueueTypes::in_order)); } + // Enable KV-cache compression by default for non-systolic platforms + if (!is_set_by_user(ov::hint::kv_cache_precision) && !info.supports_immad) { + set_property(ov::hint::kv_cache_precision(ov::element::i8)); + } + user_properties.clear(); } diff --git a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/kv_cache_sdpa.cpp b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/kv_cache_sdpa.cpp index 16db9d89c28b4d..2563fe535a93d9 100644 --- a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/kv_cache_sdpa.cpp +++ b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/kv_cache_sdpa.cpp @@ -47,8 +47,11 @@ class SDPAWithKVCacheTest : public ::testing::Test, public ::testing::WithParamI ov::AnyMap properties = {ov::hint::inference_precision(ov::element::f16), ov::intel_gpu::hint::enable_sdpa_optimization(true)}; - if (p.compressed) + if (p.compressed) { properties.emplace(ov::hint::kv_cache_precision(ov::element::i8)); + } else { + properties.emplace(ov::hint::kv_cache_precision(ov::element::undefined)); + } const size_t n_heads = 16; const size_t n_features = 64; diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp index 5a78360eb1f6d8..c0e317ff6ce915 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp @@ -121,11 +121,10 @@ class dynamic_quantization_gpu_tests: public ::testing::Test { auto outputs = network->execute(); auto output_mem = outputs.begin()->second.get_memory(); - cldnn::mem_lock output_ptr (output_mem, get_test_stream()); + cldnn::mem_lock output_ptr (output_mem, get_test_stream()); auto ref_output_mem = get_ref_results(); - cldnn::mem_lock output_ptr_ref (ref_output_mem, get_test_stream()); - + cldnn::mem_lock output_ptr_ref (ref_output_mem, get_test_stream()); size_t count = 0; float max_diff = 0.f; float avg = 0.f; @@ -135,7 +134,7 @@ class dynamic_quantization_gpu_tests: public ::testing::Test { max_diff = abs_diff; avg += abs_diff; count++; - OPENVINO_ASSERT(abs_diff < 1); + ASSERT_LE(abs_diff, 1); } GPU_DEBUG_LOG << "---> count: " << count << ", max_diff:" << max_diff << ", avg_diff: " << (avg/count) << std::endl; } From ab71e5ba54e60684e0e69680fee55af1899d707b Mon Sep 17 00:00:00 2001 From: Alexandra Sidorova Date: Thu, 7 Nov 2024 15:06:40 +0400 Subject: [PATCH 33/93] [Snippets] Added dynamic memory sharing support (#27175) ### Details: - *Added memory inplace support for Buffer with different precision (it's supported only when output Buffer has data size not more than input Buffer) - it's needed for BF16 and INT8 MHA* - *Added memory inplace support for dynamic Buffers with the marking pass "MarkInvariantShapePath"* ### Tickets: - *154732, 150148* ### Prerequisites: - [x] https://github.com/openvinotoolkit/openvino/pull/27169 - [x] https://github.com/openvinotoolkit/openvino/pull/27300 --- .../lowered/expressions/buffer_expression.hpp | 1 + .../include/snippets/lowered/loop_info.hpp | 17 +- .../lowered/pass/define_buffer_clusters.hpp | 64 ++-- .../pass/mark_invariant_shape_path.hpp | 65 ++++ .../lowered/pass/set_buffer_reg_group.hpp | 55 ++- .../snippets/include/snippets/utils/utils.hpp | 6 + .../snippets/src/lowered/expression.cpp | 9 +- .../lowered/expressions/buffer_expression.cpp | 9 +- src/common/snippets/src/lowered/loop_info.cpp | 27 +- .../src/lowered/pass/allocate_buffers.cpp | 3 +- .../lowered/pass/define_buffer_clusters.cpp | 341 ++++++++---------- .../pass/mark_invariant_shape_path.cpp | 128 +++++++ .../src/lowered/pass/set_buffer_reg_group.cpp | 104 +++--- .../snippets/src/pass/mha_tokenization.cpp | 11 +- .../x64/pass/lowered/set_tpp_leading_dim.cpp | 11 +- .../x64/lowered/buffer_allocation.cpp | 94 +++-- 16 files changed, 571 insertions(+), 374 deletions(-) create mode 100644 src/common/snippets/include/snippets/lowered/pass/mark_invariant_shape_path.hpp create mode 100644 src/common/snippets/src/lowered/pass/mark_invariant_shape_path.cpp diff --git a/src/common/snippets/include/snippets/lowered/expressions/buffer_expression.hpp b/src/common/snippets/include/snippets/lowered/expressions/buffer_expression.hpp index 3dcd98ef0a95fd..9174d9866db503 100644 --- a/src/common/snippets/include/snippets/lowered/expressions/buffer_expression.hpp +++ b/src/common/snippets/include/snippets/lowered/expressions/buffer_expression.hpp @@ -38,6 +38,7 @@ class BufferExpression : public Expression { size_t get_offset() const { return m_offset; } size_t get_allocation_size() const { return m_allocation_size; } size_t get_byte_size() const; + ov::element::Type get_data_type() const; void set_reg_group(size_t reg_group) { m_reg_group = reg_group; } void set_cluster_id(size_t cluster) { m_cluster_id = cluster; } diff --git a/src/common/snippets/include/snippets/lowered/loop_info.hpp b/src/common/snippets/include/snippets/lowered/loop_info.hpp index cc66f5f6ffcc95..23e1f14a8b7f5e 100644 --- a/src/common/snippets/include/snippets/lowered/loop_info.hpp +++ b/src/common/snippets/include/snippets/lowered/loop_info.hpp @@ -211,13 +211,20 @@ class UnifiedLoopInfo : public LoopInfo { int64_t data_size = 0; bool is_dynamic() const; + bool is_static() const; + + friend bool operator==(const LoopPortDesc& lhs, const LoopPortDesc& rhs); + friend bool operator!=(const LoopPortDesc& lhs, const LoopPortDesc& rhs); }; // The structure describes full information about port // - TODO [140365] : UnifiedLoopInfo should have the map of LoopPorts and LoopDesc as class field // instead of the separate vectors with descriptors. struct LoopPortInfo { - LoopPort port; - LoopPortDesc desc; + LoopPortInfo() = default; + LoopPortInfo(LoopPort port_, LoopPortDesc desc_) : port(std::move(port_)), desc(std::move(desc_)) {} + + LoopPort port = {}; + LoopPortDesc desc = {}; }; UnifiedLoopInfo() = default; @@ -367,6 +374,12 @@ class UnifiedLoopInfo : public LoopInfo { caller(m_output_ports[i], m_output_port_descs[i]); } + /** + * @brief Return loop port info of an expression port + * @param expr_port - expression port. + */ + LoopPortInfo get_loop_port_info(const ExpressionPort& expr_port); + protected: /** * @brief Clone LoopPortDesc[actual_port_idx] `new_count` times and insert on the place of current desc diff --git a/src/common/snippets/include/snippets/lowered/pass/define_buffer_clusters.hpp b/src/common/snippets/include/snippets/lowered/pass/define_buffer_clusters.hpp index 1597eaa2377a50..312abb02abf7b5 100644 --- a/src/common/snippets/include/snippets/lowered/pass/define_buffer_clusters.hpp +++ b/src/common/snippets/include/snippets/lowered/pass/define_buffer_clusters.hpp @@ -6,6 +6,8 @@ #include "pass.hpp" +#include "snippets/lowered/loop_info.hpp" + namespace ov { namespace snippets { namespace lowered { @@ -45,7 +47,7 @@ class DefineBufferClusters : public RangedPass { private: using BufferCluster = std::set; using BufferClusters = std::vector; - using BufferPorts = std::unordered_map>; + using BufferMap = std::unordered_map; /** * @brief Finds Buffer cluster in set of clusters which contains the target expression with Buffer * @param target target expression with Buffer op @@ -58,76 +60,72 @@ class DefineBufferClusters : public RangedPass { * @param target_expr expression with target op - LoopEnd or MemoryAccess op * @return boolean value */ - bool is_direct_buffer(const BufferExpressionPtr& buffer_expr, const ExpressionPtr& target_expr) const; + static bool is_direct_buffer(const BufferExpressionPtr& buffer_expr, const ExpressionPtr& target_expr); /** * @brief Creates new buffer cluster if buffer_exprs is missed in clusters. If buffer_exprs is already in clusters, do nothing * @param buffer_expr expression with Buffer op */ void create_new_cluster(const BufferExpressionPtr& buffer_expr); + /** + * @brief Add buffers to the existing clusters + * @param existing_cluster existing clusters + * @param buffers buffers which will be added to the existing cluster + */ + static void add_buffers_to_cluster(BufferCluster& existing_cluster, const std::set& buffers); /** * @brief Returns common ID of cluster if all buffer inside have the same Buffer ID. Otherwise returns the default value SIZE_MAX * that means that Buffers in cluster have different IDs. * @param cluster set of Buffer expressions - cluster * @return common buffer ID or SIZE_MAX - size value */ - size_t get_cluster_buffer_id(const BufferCluster& cluster) const; + static size_t get_cluster_buffer_id(const BufferCluster& cluster); /** * @brief Analyzes Loop: if Loop has Buffer ops on inputs and outputs, Loop can read and write from/to the same memory. + * @param loop_manager loop manager * @param expr_it iterator of Linear IR which refers to the expression with LoopEnd */ - void parse_loop(const LinearIR::constExprIt& expr_it); + void parse_loop(const LoopManagerPtr& loop_manager, const LinearIR::constExprIt& expr_it); /** * @brief Analyzes full MemoryAccess op: if the op has Buffer ops on I/O, the op can read and write from/to the same memory. * @param expr expression with full MemoryAccess op */ void parse_memory_access_op(const ExpressionPtr& expr); /** - * @brief Gets input outputs buffers of Loop - * @param loop_expr expression with LoopEnd op - * @return unordered map [Expression -> set of input ports] which represents input Buffers of Loop + * @brief Find all direct buffers that are connected to the current Loop + * @param loop_info current unified loop info + * @param loop_expr the target LoopEnd expression + * @return input and output buffer maps */ - BufferPorts get_input_buffers(const ExpressionPtr& loop_expr) const; - /** - * @brief Gets output buffers of Loop - * @param loop_expr expression with LoopEnd op - * @return unordered map [Expression -> set of input ports] which represents output Buffers of Loop - */ - BufferPorts get_output_buffers(const ExpressionPtr& loop_expr) const; + static std::pair get_direct_buffers(const UnifiedLoopInfoPtr& loop_info, const ExpressionPtr& loop_expr); /** * @brief Analyzes nested Loops: unite nested buffer clusters if they can reproduce `window` sliding - * @param input_buffers unordered map [Expression -> set of input ports] which represents input Buffers of Loop - * @param output_buffers unordered map [Expression -> set of output ports (one)] which represents output Buffers of Loop + * @param loop_manager loop manager + * @param input_buffers unordered map [Expression -> LoopPortInfo] which represents input Buffers of Loop + * @param output_buffers unordered map [Expression -> LoopPortInfo] which represents output Buffers of Loop * @param outer_loop_end_expr_it iterator of Linear IR which refers to the expression with outer LoopEnd */ - void parse_nested_loops(const BufferPorts& input_buffers, const BufferPorts& output_buffers, const LinearIR::constExprIt& outer_loop_end_expr_it); + void parse_nested_loops(const LoopManagerPtr& loop_manager, const BufferMap& input_buffers, const BufferMap& output_buffers, + const LinearIR::constExprIt& outer_loop_end_expr_it); /** - * @brief Finds the last connected Loop to the target Buffer and returns the corresponding finalization offset + * @brief Finds the last connected Loop to the target Buffer and init the corresponding loop port info + * @param loop_manager loop manager * @param buffer_expr expression with Buffer op - * @return finalization offset - int64_t value - */ - int64_t get_buffer_finalization_offset(const BufferExpressionPtr& buffer_expr) const; - /** - * @brief Check if two Buffer expressions are connected to the same Loop. Set common LoopEnd as `loop` parameter and - * indexes of Loop ports `up_idx` and `down_idx` if Buffers are really neighbours - * @param up expression with upper Buffer op - * @param down expression with lower Buffer op - * @param loop expression with common LoopEnd op - * @param up_idx the reference to port index of upper Buffer op to the Loop - * @param down_idx the reference to port index of lower Buffer op to the Loop - * @return Return True if the Buffers are connected to the same Loop + * @param port_info target loop port info to be initialized + * @return status - True if loop port has been found. Otherwise, return false - not connected to the Loop. */ - static bool are_buffer_neighbours(const BufferExpressionPtr& up, const BufferExpressionPtr& down, ExpressionPtr& loop, - size_t& up_idx, size_t& down_idx); + static bool init_buffer_last_loop_port_info(const LoopManagerPtr& loop_manager, const BufferExpressionPtr& buffer_expr, + UnifiedLoopInfo::LoopPortInfo& port_info); /** * @brief Unite clusters + * @param loop_manager loop manager * @param inner_cluster_it iterator to inner cluster - buffer cluster is in the loop * @param outer_cluster buffer clusters with buffers outside the Loop * @param outer_buffer target Buffer from outer_cluster * @param is_outer_up true if outer buffer is upper in Linear IR than inner Buffers * @return Return True if clusters have been united */ - bool unite_nested_clusters(const BufferClusters::iterator& inner_cluster_it, BufferCluster& outer_cluster, + bool unite_nested_clusters(const LoopManagerPtr& loop_manager, const BufferClusters::iterator& inner_cluster_it, BufferCluster& outer_cluster, const BufferExpressionPtr& outer_buffer, bool is_outer_up); BufferClusters m_clusters; diff --git a/src/common/snippets/include/snippets/lowered/pass/mark_invariant_shape_path.hpp b/src/common/snippets/include/snippets/lowered/pass/mark_invariant_shape_path.hpp new file mode 100644 index 00000000000000..6a31a697baca77 --- /dev/null +++ b/src/common/snippets/include/snippets/lowered/pass/mark_invariant_shape_path.hpp @@ -0,0 +1,65 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "pass.hpp" + +namespace ov { +namespace snippets { +namespace lowered { +namespace pass { + +/** + * @interface MarkInvariantShapePath + * @brief The helper pass for BufferAllocation pipeline: + * - Many buffer-relates passes (SetBufferRegGroup, DefineBufferClusters) depend on loop pointer increments. + * The increments are unknown in dynamic case, so these passes can't set reg groups and clusters efficiently. + * The current pass marks expressions port which will have the same shape. The shape and layout means + * the same loop pointer arithmetic in runtime. + * @ingroup snippets + */ +class MarkInvariantShapePath: public RangedPass { +public: + OPENVINO_RTTI("MarkInvariantShapePath", "RangedPass") + MarkInvariantShapePath() = default; + + /** + * @brief Apply the pass to the Linear IR + * @param linear_ir the target Linear IR + * @return status of the pass + */ + bool run(LinearIR& linear_ir, lowered::LinearIR::constExprIt begin, lowered::LinearIR::constExprIt end) override; + + /** + * @brief Returns ID (color) of the current Invariant Shape path for the passed port. + * Ports which have the same IDs of the paths - will have the same shapes in runtime. + * Note: if passed port is input port, the method returns value for source of port connector + * for the passed port. Because the shape is created by output ports of expressions. + * @param port target expression port + * @return ID + */ + static size_t getInvariantPortShapePath(const ExpressionPort& port); + +private: + /** + * @brief Sets ID (color) of the current Invariant Shape path for the passed output port. + * Ports which have the same IDs of the paths - will have the same shapes in runtime. + * @param port target expression port + * @param value ID of the path (color) + */ + static void SetInvariantPortShapePath(const ExpressionPort& port, size_t value); + + /** + * @brief Return runtime info for the passed expression port + * @param port target expression port + * @return runtime info map + */ + static ov::RTMap& get_rt_info(const ExpressionPort& port); +}; + +} // namespace pass +} // namespace lowered +} // namespace snippets +} // namespace ov diff --git a/src/common/snippets/include/snippets/lowered/pass/set_buffer_reg_group.hpp b/src/common/snippets/include/snippets/lowered/pass/set_buffer_reg_group.hpp index cba3f28856be42..dc22ce4beff1a0 100644 --- a/src/common/snippets/include/snippets/lowered/pass/set_buffer_reg_group.hpp +++ b/src/common/snippets/include/snippets/lowered/pass/set_buffer_reg_group.hpp @@ -6,6 +6,7 @@ #include "pass.hpp" +#include "snippets/lowered/loop_info.hpp" #include "snippets/utils/utils.hpp" namespace ov { @@ -38,34 +39,19 @@ class SetBufferRegGroup: public RangedPass { * @param linear_ir the target Linear IR * @return status of the pass */ - bool run(LinearIR& linear_ir, lowered::LinearIR::constExprIt begin, lowered::LinearIR::constExprIt end) override; - - struct ShiftPtrParams { - ShiftPtrParams() = default; - ShiftPtrParams(int64_t ds, int64_t pi, int64_t fo) : data_size(ds), ptr_increment(pi), finalization_offset(fo) {} - int64_t data_size = 0; - int64_t ptr_increment = 0; - int64_t finalization_offset = 0; - - inline bool is_static() const { - return !utils::is_dynamic_value(ptr_increment) && !utils::is_dynamic_value(finalization_offset); - } - - friend bool operator==(const ShiftPtrParams& lhs, const ShiftPtrParams& rhs); - friend bool operator!=(const ShiftPtrParams& lhs, const ShiftPtrParams& rhs); - }; + bool run(LinearIR& linear_ir, LinearIR::constExprIt begin, LinearIR::constExprIt end) override; /** - * @brief Check if two Buffers can be in one register group by ShiftPtrParams < data_size, ptr_increment, finalization_offset > - * @param lhs Data pointer shift params for first Buffer - * @param rhs Data pointer shift params for second Buffer + * @brief Check if two Buffers can be in one register group by LoopDesc < data_size, ptr_increment, finalization_offset > + * @param lhs LoopPortInfo (Port and Data pointer shift params for first Buffer) + * @param rhs LoopPortInfo (Port and Data pointer shift params for second Buffer) * @return Returns True if params are valid to reuse one register. Otherwise returns False */ - static bool can_be_in_one_group(const ShiftPtrParams& lhs, const ShiftPtrParams& rhs); + static bool can_be_in_one_reg_group(const UnifiedLoopInfo::LoopPortInfo& lhs, const UnifiedLoopInfo::LoopPortInfo& rhs); private: using BufferPool = std::vector; - using BufferMap = std::map; + using BufferMap = std::map; /** * @brief Get Buffer Index in Buffer set @@ -76,11 +62,14 @@ class SetBufferRegGroup: public RangedPass { static size_t get_buffer_idx(const BufferExpressionPtr& target, const BufferPool& pool); /** * @brief Create adjacency matrix for Buffer system. See comment in the method for more details. - * @param linear_ir the target Linear IR + * @param loop_manager the loop manager + * @param begin begin iterator + * @param end end iterator * @param pool set of Buffers from the Linear IR * @return adjacency matrix where True value means that Buffers are adjacent and cannot have the same ID */ - static std::vector create_adjacency_matrix(lowered::LinearIR::constExprIt begin, lowered::LinearIR::constExprIt end, const BufferPool& pool); + static std::vector create_adjacency_matrix(const LoopManagerPtr& loop_manager, LinearIR::constExprIt begin, LinearIR::constExprIt end, + const BufferPool& pool); /** * @brief Algorithm of Graph coloring where vertices are Buffers * @param buffers set of Buffers from the Linear IR @@ -99,25 +88,23 @@ class SetBufferRegGroup: public RangedPass { * @param buffers set of Buffers from the Linear IR * @param adj Target adjacency matrix */ - static void update_adj_matrix(const std::pair& lhs, - const std::pair& rhs, - const BufferPool& buffers, + static void update_adj_matrix(const BufferMap::value_type& lhs, const BufferMap::value_type& rhs, const BufferPool& buffers, std::vector& adj); + /** * @brief Check if two Buffers are adjacent and cannot have the same ID - * @param lhs Pair where first value is Expression with first Buffer and second value is data pointer shift params for it - * @param rhs Pair where first value is Expression with second Buffer and second value is data pointer shift params for it + * @param lhs LoopPortInfo (Port and Data pointer shift params for first Buffer) + * @param rhs LoopPortInfo (Port and Data pointer shift params for second Buffer) * @return Returns True if they are adjacent, otherwise returns False */ - static bool are_adjacent(const std::pair& lhs, - const std::pair& rhs); + static bool are_adjacent(const BufferMap::value_type& lhs, const BufferMap::value_type& rhs); /** - * @brief Find all buffers that are connected to the current LoopEnd - * @param loop_end_expr expression of the target LoopEnd - * @return buffer map [buffer expr -> ShiftDataPtrs] + * @brief Find all buffers that are connected to the current Loop + * @param loop_info current unified loop info + * @return buffer map */ - static BufferMap get_buffer_loop_neighbours(const ExpressionPtr& loop_end_expr); + static BufferMap get_buffer_loop_neighbours(const UnifiedLoopInfoPtr& loop_info); /** * @brief Find all buffers that are inside the current Loop. * @param loop_end_it expression iterator in LinearIR of the target LoopEnd diff --git a/src/common/snippets/include/snippets/utils/utils.hpp b/src/common/snippets/include/snippets/utils/utils.hpp index f7e584d48a905c..ff4646f24d03b7 100644 --- a/src/common/snippets/include/snippets/utils/utils.hpp +++ b/src/common/snippets/include/snippets/utils/utils.hpp @@ -91,6 +91,12 @@ static inline auto rnd_up(const T lhs, const U rhs) -> decltype(div_up(lhs, rhs) return div_up_res * rhs; } +static inline bool is_planar_layout(const std::vector& order) { + for (size_t i = 0; i < order.size(); ++i) + if (order[i] != i) return false; + return true; +} + inline bool is_dynamic_vdims(const VectorDims& shape) { return std::any_of(shape.cbegin(), shape.cend(), [](size_t v){ return is_dynamic_value(v); }); } diff --git a/src/common/snippets/src/lowered/expression.cpp b/src/common/snippets/src/lowered/expression.cpp index aaa71612cef706..1952b93017aab5 100644 --- a/src/common/snippets/src/lowered/expression.cpp +++ b/src/common/snippets/src/lowered/expression.cpp @@ -170,11 +170,6 @@ ExpressionPtr Expression::clone() const { } bool Expression::visit_attributes(AttributeVisitor &visitor) { - auto is_planar_layout = [](const std::vector& layout) { - for (size_t i = 0; i < layout.size(); ++i) - if (layout[i] != i) return false; - return true; - }; auto subtensor2str = [](const VectorDims& subtensor) { std::stringstream ss; for (size_t i = 0; i < subtensor.size(); ++i) { @@ -203,7 +198,7 @@ bool Expression::visit_attributes(AttributeVisitor &visitor) { subtensors.emplace_back("in_subtensor_" + std::to_string(i), subtensor2str(subtensor)); const auto& layout = desc->get_layout(); - if (!layout.empty() && !is_planar_layout(layout)) + if (!layout.empty() && !utils::is_planar_layout(layout)) layouts.emplace_back("in_layout_" + std::to_string(i), layout); in_reg_types.emplace_back(regTypeToStr(desc->get_reg().type)); @@ -220,7 +215,7 @@ bool Expression::visit_attributes(AttributeVisitor &visitor) { subtensors.emplace_back("out_subtensor_" + std::to_string(i), subtensor2str(subtensor)); const auto& layout = desc->get_layout(); - if (!layout.empty() && !is_planar_layout(layout)) + if (!layout.empty() && !utils::is_planar_layout(layout)) layouts.emplace_back("out_layout_" + std::to_string(i), layout); out_reg_types.emplace_back(regTypeToStr(desc->get_reg().type)); diff --git a/src/common/snippets/src/lowered/expressions/buffer_expression.cpp b/src/common/snippets/src/lowered/expressions/buffer_expression.cpp index acc742ff196407..a8b3bb2034b105 100644 --- a/src/common/snippets/src/lowered/expressions/buffer_expression.cpp +++ b/src/common/snippets/src/lowered/expressions/buffer_expression.cpp @@ -25,12 +25,15 @@ ExpressionPtr BufferExpression::clone() const { } bool BufferExpression::visit_attributes(AttributeVisitor &visitor) { + Expression::visit_attributes(visitor); auto allocation_size = utils::value2str(m_allocation_size); auto offset = utils::value2str(m_offset); + auto prc = get_data_type(); visitor.on_attribute("allocation_size", allocation_size); visitor.on_attribute("offset", offset); visitor.on_attribute("reg_group", m_reg_group); visitor.on_attribute("cluster_id", m_cluster_id); + visitor.on_attribute("data_type", prc); return true; } @@ -38,9 +41,13 @@ bool BufferExpression::is_defined() const { return !utils::is_dynamic_value(m_allocation_size); } +ov::element::Type BufferExpression::get_data_type() const { + return get_node()->get_output_element_type(0); +} + size_t BufferExpression::get_byte_size() const { if (is_defined()) - return m_allocation_size * get_node()->get_output_element_type(0).size(); + return m_allocation_size * get_data_type().size(); return utils::get_dynamic_value(); } diff --git a/src/common/snippets/src/lowered/loop_info.cpp b/src/common/snippets/src/lowered/loop_info.cpp index 954cc180a1527b..1c856869878b80 100644 --- a/src/common/snippets/src/lowered/loop_info.cpp +++ b/src/common/snippets/src/lowered/loop_info.cpp @@ -99,7 +99,7 @@ template<> std::vector::iterator LoopInfo::find_loop_port(const ExpressionPort& expr_port) { auto& ports = expr_port.get_type() == ExpressionPort::Input ? m_input_ports : m_output_ports; const auto it = std::find_if(ports.begin(), ports.end(), - [&expr_port](const LoopPort& port) { return *port.expr_port.get() == expr_port; }); + [&expr_port](const LoopPort& port) { return *port.expr_port == expr_port; }); return it; } @@ -176,6 +176,19 @@ bool UnifiedLoopInfo::LoopPortDesc::is_dynamic() const { return utils::is_dynamic_value(ptr_increment) || utils::is_dynamic_value(finalization_offset); } +bool UnifiedLoopInfo::LoopPortDesc::is_static() const { + return !is_dynamic(); +} + +bool operator==(const UnifiedLoopInfo::LoopPortDesc& lhs, const UnifiedLoopInfo::LoopPortDesc& rhs) { + if (&lhs == &rhs) + return true; + return lhs.ptr_increment == rhs.ptr_increment && lhs.finalization_offset == rhs.finalization_offset && lhs.data_size == rhs.data_size; +} +bool operator!=(const UnifiedLoopInfo::LoopPortDesc& lhs, const UnifiedLoopInfo::LoopPortDesc& rhs) { + return !(rhs == lhs); +} + UnifiedLoopInfo::UnifiedLoopInfo(size_t work_amount, size_t increment, const std::vector& entries, const std::vector& exits, const SpecificIterationHandlers& handlers) @@ -321,6 +334,18 @@ void UnifiedLoopInfo::sort_ports() { reorder(m_output_ports, m_output_port_descs); } +UnifiedLoopInfo::LoopPortInfo UnifiedLoopInfo::get_loop_port_info(const ExpressionPort& expr_port) { + OPENVINO_ASSERT(is_loop_port(expr_port), "Failed get_loop_port: expr_port is not a loop port"); + const auto is_input = expr_port.get_type() == ExpressionPort::Input; + const auto& ports = is_input ? m_input_ports : m_output_ports; + const auto& descs = is_input ? m_input_port_descs : m_output_port_descs; + const auto it = std::find_if(ports.begin(), ports.end(), + [&expr_port](const LoopPort& port) { return *port.expr_port == expr_port; }); + const auto index = static_cast(std::distance(ports.cbegin(), it)); + OPENVINO_ASSERT(index < ports.size() && index < descs.size(), "LoopPortInfo has not been found!"); + return {ports[index], descs[index]}; +} + void UnifiedLoopInfo::replace_with_cloned_descs(size_t actual_port_idx, size_t new_count, bool is_input) { auto& descs = is_input ? m_input_port_descs : m_output_port_descs; std::vector target_shifts(new_count, descs[actual_port_idx]); diff --git a/src/common/snippets/src/lowered/pass/allocate_buffers.cpp b/src/common/snippets/src/lowered/pass/allocate_buffers.cpp index f76c4097b38f38..0269c65109b701 100644 --- a/src/common/snippets/src/lowered/pass/allocate_buffers.cpp +++ b/src/common/snippets/src/lowered/pass/allocate_buffers.cpp @@ -12,7 +12,7 @@ #include "snippets/lowered/pass/define_buffer_clusters.hpp" #include "snippets/lowered/pass/normalize_buffer_reg_groups.hpp" #include "snippets/lowered/pass/propagate_buffer_offset.hpp" -#include "snippets/pass/tokenization.hpp" +#include "snippets/lowered/pass/mark_invariant_shape_path.hpp" #include "snippets/itt.hpp" #include "snippets/utils/utils.hpp" @@ -30,6 +30,7 @@ bool AllocateBuffers::run(lowered::LinearIR& linear_ir, lowered::LinearIR::const PassPipeline pipeline; pipeline.register_pass(); if (m_is_optimized_mode) { + pipeline.register_pass(); pipeline.register_pass(); pipeline.register_pass(); pipeline.register_pass(buffer_scratchpad_size); diff --git a/src/common/snippets/src/lowered/pass/define_buffer_clusters.cpp b/src/common/snippets/src/lowered/pass/define_buffer_clusters.cpp index c43b5d63a358c6..e8132d62be0cc9 100644 --- a/src/common/snippets/src/lowered/pass/define_buffer_clusters.cpp +++ b/src/common/snippets/src/lowered/pass/define_buffer_clusters.cpp @@ -4,24 +4,42 @@ #include "snippets/lowered/pass/define_buffer_clusters.hpp" +#include "snippets/lowered/pass/mark_invariant_shape_path.hpp" #include "snippets/lowered/pass/set_buffer_reg_group.hpp" +#include "snippets/lowered/loop_manager.hpp" #include "snippets/snippets_isa.hpp" #include "snippets/utils/utils.hpp" #include "snippets/itt.hpp" + namespace ov { namespace snippets { namespace lowered { namespace pass { -using ShiftPtrParams = SetBufferRegGroup::ShiftPtrParams; +namespace { + +// Find Loops which are connected to the current `buffer_expr` (consumer of Buffer is port of these Loops) +std::vector get_connected_loops(const BufferExpressionPtr& buffer_expr, const ExpressionPtr& consumer_expr) { + // [133463] Remove it please + if (ov::is_type(consumer_expr->get_node())) + return {}; + const auto& buffer_loops_ids = buffer_expr->get_loop_ids(); + const auto& consumer_loop_ids = consumer_expr->get_loop_ids(); + OPENVINO_ASSERT(buffer_loops_ids.size() <= consumer_loop_ids.size(), "Buffer with consumer are in incorrect loops"); + const auto mismatched_its = std::mismatch(buffer_loops_ids.begin(), buffer_loops_ids.end(), consumer_loop_ids.begin()); + return {mismatched_its.second, consumer_loop_ids.cend()}; +} +} // namespace + +using LoopPortInfo = UnifiedLoopInfo::LoopPortInfo; DefineBufferClusters::BufferClusters::iterator DefineBufferClusters::find_cluster_by_expr(const BufferExpressionPtr& target) { return std::find_if(m_clusters.begin(), m_clusters.end(), [&target](const BufferCluster& cluster) { return cluster.count(target) > 0; }); } -bool DefineBufferClusters::is_direct_buffer(const BufferExpressionPtr& buffer_expr, const ExpressionPtr& target_expr) const { +bool DefineBufferClusters::is_direct_buffer(const BufferExpressionPtr& buffer_expr, const ExpressionPtr& target_expr) { return buffer_expr && buffer_expr->get_loop_ids() == target_expr->get_loop_ids(); } @@ -33,7 +51,16 @@ void DefineBufferClusters::create_new_cluster(const BufferExpressionPtr& buffer_ } } -size_t DefineBufferClusters::get_cluster_buffer_id(const BufferCluster& cluster) const { +void DefineBufferClusters::add_buffers_to_cluster(BufferCluster& existing_cluster, const std::set& buffers) { + existing_cluster.insert(buffers.cbegin(), buffers.cend()); + // All buffers in one cluster must be only static or dynamic (no mixes). + if (std::any_of(existing_cluster.cbegin(), existing_cluster.cend(), [](const BufferExpressionPtr& buffer) { return !buffer->is_defined(); })) { + for (const auto& buffer : existing_cluster) + buffer->set_allocation_size(utils::get_dynamic_value()); + } +} + +size_t DefineBufferClusters::get_cluster_buffer_id(const BufferCluster& cluster) { OPENVINO_ASSERT(!cluster.empty(), "Buffer cluster is empty!"); const auto id = cluster.cbegin()->get()->get_reg_group(); if (std::all_of(cluster.cbegin(), cluster.cend(), [&id](const BufferExpressionPtr& expr) { return expr->get_reg_group() == id; })) { @@ -42,141 +69,121 @@ size_t DefineBufferClusters::get_cluster_buffer_id(const BufferCluster& cluster) return SIZE_MAX; } -DefineBufferClusters::BufferPorts DefineBufferClusters::get_input_buffers(const ExpressionPtr& loop_expr) const { - BufferPorts input_buffers; - - const auto loop_end = ov::as_type_ptr(loop_expr->get_node()); - const auto in_count = loop_end->get_input_num(); - const auto& connectors = loop_expr->get_input_port_connectors(); - - // Input Buffers - for (size_t i = 0; i < in_count; ++i) { - const auto& source_expr = ov::as_type_ptr(connectors[i]->get_source().get_expr()); - if (!is_direct_buffer(source_expr, loop_expr)) +std::pair DefineBufferClusters::get_direct_buffers(const UnifiedLoopInfoPtr& loop_info, + const ExpressionPtr& loop_expr) { + BufferMap input_buffers; + const auto& loop_inputs = loop_info->get_input_ports_info(); + for (const auto& port_info : loop_inputs) { + const auto& buffer_expr = ov::as_type_ptr(port_info.port.expr_port->get_port_connector_ptr()->get_source().get_expr()); + if (!is_direct_buffer(buffer_expr, loop_expr)) + continue; + if (input_buffers.count(buffer_expr) > 0) { + const auto& port_desc = port_info.desc; + OPENVINO_ASSERT(input_buffers[buffer_expr].desc == port_desc, + "Invalid data pointer shifts: If Buffer has several consumers, this consumers must have the same shifts or zero"); continue; - // Save as input Buffer - const auto ret = input_buffers.insert(std::make_pair(source_expr, std::set{ i })).second; - if (!ret) - input_buffers[source_expr].insert(i); + } + input_buffers[buffer_expr] = port_info; } - return input_buffers; -} - -DefineBufferClusters::BufferPorts DefineBufferClusters::get_output_buffers(const ExpressionPtr& loop_expr) const { - BufferPorts output_buffers; - const auto loop_end = ov::as_type_ptr(loop_expr->get_node()); - const auto in_count = loop_end->get_input_num(); - const auto out_count = loop_end->get_output_num(); - const auto& connectors = loop_expr->get_input_port_connectors(); - - for (size_t i = in_count; i < in_count + out_count; ++i) { - for (const auto& consumer : connectors[i]->get_consumers()) { - const auto& consumer_expr = ov::as_type_ptr(consumer.get_expr()); - if (!is_direct_buffer(consumer_expr, loop_expr)) + BufferMap output_buffers; + const auto& loop_outputs = loop_info->get_output_ports_info(); + for (const auto& port_info : loop_outputs) { + const auto& consumer_inputs = port_info.port.expr_port->get_port_connector_ptr()->get_consumers(); + for (const auto& consumer_input : consumer_inputs) { + const auto& buffer_expr = ov::as_type_ptr(consumer_input.get_expr()); + if (!is_direct_buffer(buffer_expr, loop_expr)) continue; - // Save as output Buffer - output_buffers[consumer_expr] = { i }; + OPENVINO_ASSERT(output_buffers.count(buffer_expr) == 0, "Only one Buffer can be on node output!"); + output_buffers[buffer_expr] = port_info; } } - return output_buffers; + + return std::make_pair(input_buffers, output_buffers); } -void DefineBufferClusters::parse_loop(const LinearIR::constExprIt& expr_it) { +void DefineBufferClusters::parse_loop(const LoopManagerPtr& loop_manager, const LinearIR::constExprIt& expr_it) { const auto& expr = *expr_it; - const auto loop_end = ov::as_type_ptr(expr->get_node()); - const auto& ptr_increments = loop_end->get_ptr_increments(); - const auto& final_offsets = loop_end->get_finalization_offsets(); - const auto& data_sizes = loop_end->get_element_type_sizes(); + const auto& loop_end = ov::as_type_ptr(expr->get_node()); + const auto& loop_info = loop_manager->get_loop_info(loop_end->get_id()); - // [ Expression -> Port indexes ] - const auto input_buffers = get_input_buffers(expr); - const auto output_buffers = get_output_buffers(expr); + BufferMap input_buffers, output_buffers; + std::tie(input_buffers, output_buffers) = get_direct_buffers(loop_info, expr); for (const auto& in : input_buffers) create_new_cluster(in.first); std::set visited_buffers; for (const auto& out : output_buffers) { - const auto output_buffer_expr = out.first; - const auto output_buffer_port_idx = *(out.second.cbegin()); // Output port is always one + const auto& output_buffer_expr = out.first; + const auto& output_buffer_port_info = out.second; bool has_been_added = false; for (const auto& in : input_buffers) { const auto& input_buffer_expr = in.first; + const auto& input_buffer_port_info = in.second; if (visited_buffers.count(input_buffer_expr) > 0) continue; - // If allocated sizes of buffers are unkown on compilation stage (dynamic), - // we cannot be sure that they're will be the same in runtime. - if (!input_buffer_expr->is_defined()|| !output_buffer_expr->is_defined()) + // Memory can be reused if reading and writing are executed proportionally: + // - output buffer can have precision with data size less than input buffer + if ((input_buffer_expr->get_data_type().size() < output_buffer_expr->get_data_type().size())) continue; - // Memory can be reused if reading and writing are executed proportionally: - // - the same reading/writing order - // - the same buffer memory sizes - if ((input_buffer_expr->get_byte_size() != output_buffer_expr->get_byte_size()) || - (input_buffer_expr->get_output_port_descriptor(0)->get_layout() != output_buffer_expr->get_input_port_descriptor(0)->get_layout())) + const auto in_path = MarkInvariantShapePath::getInvariantPortShapePath(*input_buffer_port_info.port.expr_port); + const auto out_path = MarkInvariantShapePath::getInvariantPortShapePath(*output_buffer_port_info.port.expr_port); + // - Memory can be reused if there are the same loop pointer increments (data size, final offsets, ptr increments). + // For that, loop ports with buffers should be on the same shape-path and have the same value of `is_incremented`. + if (in_path != out_path || input_buffer_port_info.port.is_incremented != output_buffer_port_info.port.is_incremented) continue; - // Also memory can be reused if there are the same ShiftPtrParams (data size, final offsets, ptr increments) - const auto& input_buffer_ports = in.second; - for (const auto& input_buffer_port_idx : input_buffer_ports) { - const auto input_params = - ShiftPtrParams(data_sizes[input_buffer_port_idx], ptr_increments[input_buffer_port_idx], final_offsets[input_buffer_port_idx]); - const auto output_params = - ShiftPtrParams(data_sizes[output_buffer_port_idx], ptr_increments[output_buffer_port_idx], final_offsets[output_buffer_port_idx]); - - // If data pointer shift parameters are unknown on model compilation stage (dynamic), - // we cannot be sure that these data pointers will be proportionally shifted in runtime. - if (input_params.is_static() && output_params.is_static() && input_params == output_params) { - const auto cluster_it = find_cluster_by_expr(input_buffer_expr); - OPENVINO_ASSERT(cluster_it != m_clusters.end(), "Buffer on inputs of Loop must be already saved in clusters"); - // Add to the existing cluster - has_been_added = cluster_it->insert(output_buffer_expr).second; - OPENVINO_ASSERT(has_been_added, "Buffer has not been saved in cluster"); - // Remove input buffer because we have already use its memory - visited_buffers.insert(input_buffer_expr); - break; - } + // - Memory can be shared if Buffer has the same allocation size. + if (input_buffer_expr->is_defined() && output_buffer_expr->is_defined()) { + if (input_buffer_expr->get_allocation_size() != output_buffer_expr->get_allocation_size()) + continue; + } else { + // If allocation sizes are undefined, we can check if they have the same allocation sizes in runtime: + // - they should calculate allocation size using the common algorithm from `BufferExpression::init_allocation_size`. + if (!utils::everyone_is(BufferExpression::get_type_info_static(), input_buffer_expr->get_type_info(), output_buffer_expr->get_type_info())) + continue; } - if (has_been_added) break; + + const auto cluster_it = find_cluster_by_expr(input_buffer_expr); + OPENVINO_ASSERT(cluster_it != m_clusters.end(), "Buffer on inputs of Loop must be already saved in clusters"); + // Add to the existing cluster + add_buffers_to_cluster(*cluster_it, {output_buffer_expr}); + // Remove input buffer because we have already use its memory + visited_buffers.insert(input_buffer_expr); + has_been_added = true; + break; } if (!has_been_added) { - m_clusters.push_back(BufferCluster{output_buffer_expr}); + create_new_cluster(output_buffer_expr); } } // Check Buffers inside to possible memory reusing using `window` sliding - parse_nested_loops(input_buffers, output_buffers, expr_it); + parse_nested_loops(loop_manager, input_buffers, output_buffers, expr_it); } -void DefineBufferClusters::parse_nested_loops(const BufferPorts& input_buffers, const BufferPorts& output_buffers, - const LinearIR::constExprIt& outer_loop_end_expr_it) { +void DefineBufferClusters::parse_nested_loops(const LoopManagerPtr& loop_manager, const BufferMap& input_buffers, + const BufferMap& output_buffers, const LinearIR::constExprIt& outer_loop_end_expr_it) { if (input_buffers.empty() && output_buffers.empty()) return; - // The inner Buffer can reuse memory of the outer Buffer using `window` sliding only if: - // - The finalization offset of the latest Loop connected to the inner Buffer is equal to pointer increment of outer Buffer to emulate `window` sliding - // - This outer Buffer should have the same Buffer ID as inner to move data ptr of inner Buffer after each outer Loop iteration. - // It's needed because all Loops reset data pointers of connected Buffer after full work. - // To avoid rewriting of outer Buffer data we have to have the same Buffer ID (GPR) to proportionally shift pointers both Buffers. - - auto can_be_data_ptr_proportionally_shifted = [](int64_t outer_buffer_ptr_increment, int64_t outer_buffer_data_size, - int64_t inner_buffer_final_offsets, int64_t inner_buffer_data_size) { - // If data pointer shift parameters are unknown on model compilation stage (dynamic), - // we cannot be sure that these data pointers will be proportionally shifted in runtime. - if (utils::is_dynamic_value(outer_buffer_ptr_increment) || utils::is_dynamic_value(inner_buffer_final_offsets)) + auto can_be_data_ptr_proportionally_shifted = [](const LoopPortInfo& outer_port_info, const LoopPortInfo& inner_port_info) { + // Outer Buffer ptr should be shifted to emulate "window" sliding + const auto& outer_desc = outer_port_info.desc; + if (!outer_port_info.port.is_incremented || (!utils::is_dynamic_value(outer_desc.ptr_increment) && outer_desc.ptr_increment == 0)) return false; - return (outer_buffer_ptr_increment != 0) && - ((inner_buffer_data_size * inner_buffer_final_offsets * -1) == outer_buffer_ptr_increment * outer_buffer_data_size); - }; - const auto outer_loop_end = ov::as_type_ptr(outer_loop_end_expr_it->get()->get_node()); - const auto outer_loop_begin = outer_loop_end->get_loop_begin(); - const auto& outer_ptr_increments = outer_loop_end->get_ptr_increments(); - const auto& outer_data_sizes = outer_loop_end->get_element_type_sizes(); + OPENVINO_ASSERT(inner_port_info.port.expr_port && outer_port_info.port.expr_port, "Expression ports are nullptr!"); + // we can be sure that these data pointers will be proportionally shifted if they're on the same invariant shape path + return MarkInvariantShapePath::getInvariantPortShapePath(*inner_port_info.port.expr_port) == + MarkInvariantShapePath::getInvariantPortShapePath(*outer_port_info.port.expr_port); + }; + const auto outer_loop_begin = ov::as_type_ptr(outer_loop_end_expr_it->get()->get_node())->get_loop_begin(); for (auto it = std::reverse_iterator(outer_loop_end_expr_it); (*it)->get_node() != outer_loop_begin; ++it) { const auto& inner_expr = *it; if (const auto inner_buffer_expr = ov::as_type_ptr(inner_expr)) { @@ -185,9 +192,12 @@ void DefineBufferClusters::parse_nested_loops(const BufferPorts& input_buffers, const auto inner_cluster_id = get_cluster_buffer_id(*inner_cluster_it); if (inner_cluster_id == SIZE_MAX) continue; - const auto final_offset = get_buffer_finalization_offset(inner_buffer_expr); + // If inner Buffer is not connected to the Loop - `window` sliding effect is not possible + LoopPortInfo final_loop_info; + if (!init_buffer_last_loop_port_info(loop_manager, inner_buffer_expr, final_loop_info)) + continue; - auto unite = [&](const BufferPorts& ports, const bool is_input) { + auto unite = [&](const BufferMap& ports, const bool is_input) { bool applied = false; for (const auto& port : ports) { const auto cluster_it = find_cluster_by_expr(port.first); @@ -196,17 +206,15 @@ void DefineBufferClusters::parse_nested_loops(const BufferPorts& input_buffers, if (cluster_it == inner_cluster_it) continue; // Buffer from one cluster must be only defined (with known allocation_size) or dynamic (with unknown allocation_size) if (inner_buffer_expr->is_defined() != port.first->is_defined()) continue; - - bool can_be_reused = true; - for (const auto idx : port.second) { - can_be_reused = can_be_reused && - can_be_data_ptr_proportionally_shifted(outer_ptr_increments[idx], outer_data_sizes[idx], - final_offset, inner_buffer_expr->get_node()->get_element_type().size()); - } - if (!can_be_reused) - continue; - - applied = unite_nested_clusters(inner_cluster_it, *cluster_it, port.first, is_input); + // The inner Buffer can reuse memory of the outer Buffer using `window` sliding only if: + // - The finalization offset of the latest Loop connected to the inner Buffer is equal to + // pointer increment of outer Buffer to emulate `window` sliding + // - This outer Buffer should have the same Buffer ID as inner to move data ptr of inner Buffer after each outer Loop iteration. + // It's needed because all Loops reset data pointers of connected Buffer after full work. + // To avoid rewriting of outer Buffer data we have to have the same Buffer ID (GPR) to proportionally shift pointers both Buffers. + if (!can_be_data_ptr_proportionally_shifted(port.second, final_loop_info)) continue; + + applied = unite_nested_clusters(loop_manager, inner_cluster_it, *cluster_it, port.first, is_input); if (applied) break; } return applied; @@ -218,101 +226,66 @@ void DefineBufferClusters::parse_nested_loops(const BufferPorts& input_buffers, } } -int64_t DefineBufferClusters::get_buffer_finalization_offset(const BufferExpressionPtr& buffer_expr) const { - auto index = [](const std::vector& loop_inputs, const PortConnectorPtr& buffer_out) { - const auto it = std::find(loop_inputs.cbegin(), loop_inputs.cend(), buffer_out); - OPENVINO_ASSERT(it != loop_inputs.cend(), "Buffer output PortConnector has not been found in target LoopEnd inputs"); - return std::distance(loop_inputs.cbegin(), it); +bool DefineBufferClusters::init_buffer_last_loop_port_info(const LoopManagerPtr& loop_manager, const BufferExpressionPtr& buffer_expr, + UnifiedLoopInfo::LoopPortInfo& port_info) { + auto get_direct_loop_for_buffer_out = [&](const BufferExpressionPtr& buffer_expr, const ExpressionPtr& consumer_expr) -> UnifiedLoopInfoPtr { + const auto inner_loops = get_connected_loops(buffer_expr, consumer_expr); + if (inner_loops.empty()) + return nullptr; + return loop_manager->get_loop_info(inner_loops.front()); }; - int64_t final_offset = 0; + + bool found = false; double last_loop_exec_order = -1 * std::numeric_limits::max(); const auto& buffer_outs = buffer_expr->get_output_port_connectors(); for (const auto& buffer_out : buffer_outs) { const auto consumers = buffer_out->get_consumers(); for (const auto& consumer : consumers) { - const auto consumer_expr = consumer.get_expr(); - const auto loop_end = ov::as_type_ptr(consumer_expr->get_node()); - if (loop_end && consumer_expr->get_loop_ids() == buffer_expr->get_loop_ids()) { - const auto loop_order = consumer_expr->get_exec_num(); + if (const auto& direct_loop = get_direct_loop_for_buffer_out(buffer_expr, consumer.get_expr())) { + const auto loop_order = direct_loop->get_output_ports().back().expr_port->get_expr()->get_exec_num(); if (loop_order > last_loop_exec_order) { - const auto& loop_inputs = consumer_expr->get_input_port_connectors(); - final_offset = loop_end->get_finalization_offsets()[index(loop_inputs, buffer_out)]; + OPENVINO_ASSERT(direct_loop->is_loop_port(consumer), "Consumer of Buffer from another loop must be loop port"); + port_info = direct_loop->get_loop_port_info(consumer); last_loop_exec_order = loop_order; + found = true; } } } } - return final_offset; + return found; } -bool DefineBufferClusters::unite_nested_clusters(const BufferClusters::iterator& inner_cluster_it, - BufferCluster& outer_cluster, - const BufferExpressionPtr& outer_buffer, bool is_outer_up) { +bool DefineBufferClusters::unite_nested_clusters(const LoopManagerPtr& loop_manager, const BufferClusters::iterator& inner_cluster_it, + BufferCluster& outer_cluster, const BufferExpressionPtr& outer_buffer, bool is_outer_up) { for (const auto& inner_buffer : *inner_cluster_it) { - ExpressionPtr common_loop_end_expr = nullptr; - size_t outer_idx = SIZE_MAX, inner_idx = SIZE_MAX; - const auto& up_buffer = is_outer_up ? outer_buffer : inner_buffer; - const auto& down_buffer = is_outer_up ? inner_buffer : outer_buffer; - auto& up_idx = is_outer_up ? outer_idx : inner_idx; - auto& down_idx = is_outer_up ? inner_idx : outer_idx; - if (are_buffer_neighbours(up_buffer, down_buffer, common_loop_end_expr, up_idx, down_idx)) { - const auto common_loop_end = ov::as_type_ptr(common_loop_end_expr->get_node()); - const auto& inner_ptr_increments = common_loop_end->get_ptr_increments(); - const auto& inner_final_offsets = common_loop_end->get_finalization_offsets(); - const auto& inner_data_sizes = common_loop_end->get_element_type_sizes(); - if (SetBufferRegGroup::can_be_in_one_group({ inner_data_sizes[up_idx], inner_ptr_increments[up_idx], inner_final_offsets[up_idx] }, - { inner_data_sizes[down_idx], inner_ptr_increments[down_idx], inner_final_offsets[down_idx] })) { - for (const auto& inner_buffer : *inner_cluster_it) - inner_buffer->set_reg_group(outer_buffer->get_reg_group()); - - outer_cluster.insert(inner_cluster_it->cbegin(), inner_cluster_it->cend()); - m_clusters.erase(inner_cluster_it); - return true; + const auto& upper_buffer = is_outer_up ? outer_buffer : inner_buffer; + const auto& lower_buffer = is_outer_up ? inner_buffer : outer_buffer; + + const auto& lower_buffer_source = lower_buffer->get_input_port_connector(0)->get_source(); + const auto& upper_buffer_consumers = upper_buffer->get_output_port_connector(0)->get_consumers(); + for (const auto& upper_buffer_consumer : upper_buffer_consumers) { + const auto& connected_loops = get_connected_loops(upper_buffer, upper_buffer_consumer.get_expr()); + for (const auto& loop_id : connected_loops) { + const auto& common_loop_info = loop_manager->get_loop_info(loop_id); + if (!common_loop_info->is_loop_port(lower_buffer_source) || !common_loop_info->is_loop_port(upper_buffer_consumer)) + continue; + + const auto upper_port_desc = common_loop_info->get_loop_port_info(upper_buffer_consumer); + const auto lower_port_desc = common_loop_info->get_loop_port_info(lower_buffer_source); + if (SetBufferRegGroup::can_be_in_one_reg_group(upper_port_desc, lower_port_desc)) { + for (const auto& inner_buffer : *inner_cluster_it) + inner_buffer->set_reg_group(outer_buffer->get_reg_group()); + + add_buffers_to_cluster(outer_cluster, *inner_cluster_it); + m_clusters.erase(inner_cluster_it); + return true; + } } } } return false; } -bool DefineBufferClusters::are_buffer_neighbours(const BufferExpressionPtr& up, const BufferExpressionPtr& down, ExpressionPtr& loop, - size_t& up_idx, size_t& down_idx) { - auto find_input = [&down](const PortConnectorPtr& in) { - return in->get_source().get_expr() == down; - }; - auto find_output = [&down](const PortConnectorPtr& in) { - const auto consumers = in->get_consumers(); - return std::any_of(consumers.cbegin(), consumers.cend(), - [&down](const ExpressionPort& port) { return port.get_expr() == down; }); - }; - auto find = [&](const std::vector::const_iterator& begin, - const std::vector::const_iterator& end, - const std::vector::const_iterator& orig_begin, - const ExpressionPort& loop_port, - bool is_input) -> bool { - const auto in_buffer_it = is_input ? std::find_if(begin, end, find_input) - : std::find_if(begin, end, find_output); - if (in_buffer_it != end) { - up_idx = loop_port.get_index(); - down_idx = std::distance(orig_begin, in_buffer_it); - loop = loop_port.get_expr(); - return true; - } - return false; - }; - for (const auto& out : up->get_output_port_connectors()) { - for (const auto& buffer_consumer : out->get_consumers()) { - const auto buffer_consumer_expr = buffer_consumer.get_expr(); - const auto loop_end = ov::as_type_ptr(buffer_consumer_expr->get_node()); - if (!loop_end) - continue; - const auto& loop_inputs = buffer_consumer_expr->get_input_port_connectors(); - if (find(loop_inputs.cbegin(), loop_inputs.cbegin() + loop_end->get_input_num(), loop_inputs.cbegin(), buffer_consumer, true)) return true; - if (find(loop_inputs.cbegin() + loop_end->get_input_num(), loop_inputs.cend(), loop_inputs.cbegin(), buffer_consumer, false)) return true; - } - } - return false; -} - void DefineBufferClusters::parse_memory_access_op(const ExpressionPtr& expr) { const auto ma = std::dynamic_pointer_cast(expr->get_node()); // TODO: Some full MemoryAccess ops can have inplace inputs and outputs in general. @@ -340,7 +313,7 @@ bool DefineBufferClusters::run(lowered::LinearIR& linear_ir, lowered::LinearIR:: const auto& expr = *expr_it; const auto op = expr->get_node(); if (ov::is_type(op)) { - parse_loop(expr_it); + parse_loop(linear_ir.get_loop_manager(), expr_it); continue; } diff --git a/src/common/snippets/src/lowered/pass/mark_invariant_shape_path.cpp b/src/common/snippets/src/lowered/pass/mark_invariant_shape_path.cpp new file mode 100644 index 00000000000000..b32056d4e32a57 --- /dev/null +++ b/src/common/snippets/src/lowered/pass/mark_invariant_shape_path.cpp @@ -0,0 +1,128 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + + +#include "snippets/lowered/pass/mark_invariant_shape_path.hpp" + +#include "snippets/lowered/expressions/buffer_expression.hpp" +#include "snippets/op/memory_access.hpp" +#include "snippets/snippets_isa.hpp" +#include "snippets/utils/utils.hpp" +#include "snippets/itt.hpp" + +namespace ov { +namespace snippets { +namespace lowered { +namespace pass { + +namespace { + +// Specific value to mark ports which doesn't affect output shape of broadcastable ops. +// For example, ops with output scalar shape or Horizon ops. +static const size_t NOT_AFFECTING_PATH = SIZE_MAX; + +static bool is_shape_broadcastable_op(const ExpressionPtr& expr) { + return expr->get_node()->get_autob() != ov::op::AutoBroadcastType::NONE; +} + +static bool is_not_affecting_op(const ExpressionPtr& expr) { + const auto& node = expr->get_node(); + return ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node); +} + +static bool is_affecting_op(const ExpressionPtr& expr) { + const auto& node = expr->get_node(); + return ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node); +} +} // namespace + +size_t MarkInvariantShapePath::getInvariantPortShapePath(const ExpressionPort& port) { + auto& rt = get_rt_info(port); + const auto rinfo = rt.find("InvariantShapePath"); + OPENVINO_ASSERT(rinfo != rt.end(), "Invariant path for this expression port has not been marked!"); + return rinfo->second.as(); +} + +void MarkInvariantShapePath::SetInvariantPortShapePath(const ExpressionPort& port, size_t value) { + OPENVINO_ASSERT(port.get_type() == ExpressionPort::Output, "SetInvariantPortShapePath can be used only for output port"); + auto& rt = get_rt_info(port); + rt["InvariantShapePath"] = value; +} + +ov::RTMap& MarkInvariantShapePath::get_rt_info(const ExpressionPort& port) { + const auto& source_port = port.get_type() == ExpressionPort::Input ? port.get_port_connector_ptr()->get_source() : port; + const auto& node = source_port.get_expr()->get_node(); + const auto port_idx = source_port.get_index(); + OPENVINO_ASSERT(port_idx < node->get_output_size(), "Node has incompatible port count with the expression"); + return node->output(port_idx).get_rt_info(); +} + +bool MarkInvariantShapePath::run(lowered::LinearIR& linear_ir, lowered::LinearIR::constExprIt begin, lowered::LinearIR::constExprIt end) { + OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::MarkInvariantShapePath"); + + bool modified = false; + + // Shape -> color + std::map colored_shapes; + + size_t color_path = 0; + + auto merge_paths = [&color_path](size_t lhs, size_t rhs) { + if (lhs == rhs || rhs == NOT_AFFECTING_PATH) return lhs; + if (lhs == NOT_AFFECTING_PATH) return rhs; + return ++color_path; + }; + + for (auto expr_it = begin; expr_it != end; ++expr_it) { + const auto& expr = *expr_it; + if (ov::is_type(expr->get_node())) + continue; + + for (size_t out_idx = 0; out_idx < expr->get_output_count(); ++out_idx) { + const auto& out_shape = expr->get_output_port_descriptor(out_idx)->get_shape(); + size_t current_color_path; + if (colored_shapes.count(out_shape)) { + current_color_path = colored_shapes.at(out_shape); + } else if (!utils::is_dynamic_vdims(out_shape) && ov::shape_size(out_shape) == 1) { + current_color_path = NOT_AFFECTING_PATH; + } else { + if (is_affecting_op(expr)) { + current_color_path = ++color_path; + } else if (is_not_affecting_op(expr)) { + current_color_path = NOT_AFFECTING_PATH; + } else if (is_shape_broadcastable_op(expr)) { + current_color_path = NOT_AFFECTING_PATH; + for (size_t in_idx = 0; in_idx < expr->get_input_count(); ++in_idx) { + const auto input_path = getInvariantPortShapePath(expr->get_input_port(in_idx)); + current_color_path = merge_paths(current_color_path, input_path); + } + } else { + current_color_path = expr->get_input_count() > 0 ? getInvariantPortShapePath(expr->get_input_port(0)) + : ++color_path; + } + + if (!utils::is_dynamic_vdims(out_shape)) + colored_shapes[out_shape] = current_color_path; + } + + SetInvariantPortShapePath(expr->get_output_port(out_idx), current_color_path); + modified = true; + } + } + + return modified; +} + +} // namespace pass +} // namespace lowered +} // namespace snippets +} // namespace ov diff --git a/src/common/snippets/src/lowered/pass/set_buffer_reg_group.cpp b/src/common/snippets/src/lowered/pass/set_buffer_reg_group.cpp index 9bdb5e8ef3a9dc..46248b9c277818 100644 --- a/src/common/snippets/src/lowered/pass/set_buffer_reg_group.cpp +++ b/src/common/snippets/src/lowered/pass/set_buffer_reg_group.cpp @@ -4,8 +4,10 @@ #include "snippets/lowered/pass/set_buffer_reg_group.hpp" +#include "snippets/lowered/pass/mark_invariant_shape_path.hpp" #include "snippets/lowered/linear_ir.hpp" -#include "snippets/snippets_isa.hpp" +#include "snippets/lowered/loop_manager.hpp" +#include "snippets/lowered/expressions/buffer_expression.hpp" #include "snippets/itt.hpp" namespace ov { @@ -19,54 +21,44 @@ inline size_t index(size_t col_num, size_t row, size_t col) { } } // namespace -bool operator==(const SetBufferRegGroup::ShiftPtrParams& lhs, const SetBufferRegGroup::ShiftPtrParams& rhs) { - if (&lhs == &rhs) - return true; - return lhs.ptr_increment == rhs.ptr_increment && lhs.finalization_offset == rhs.finalization_offset && lhs.data_size == rhs.data_size; -} -bool operator!=(const SetBufferRegGroup::ShiftPtrParams& lhs, const SetBufferRegGroup::ShiftPtrParams& rhs) { - return !(rhs == lhs); -} - size_t SetBufferRegGroup::get_buffer_idx(const BufferExpressionPtr& target, const BufferPool& pool) { const auto iter = std::find(pool.cbegin(), pool.cend(), target); OPENVINO_ASSERT(iter != pool.cend(), "Buffer wasn't find in Buffer system of Subgraph"); return std::distance(pool.cbegin(), iter); } -bool SetBufferRegGroup::can_be_in_one_group(const ShiftPtrParams& lhs, const ShiftPtrParams& rhs) { - // If data pointer shift parameters are unknown on model compilation stage (dynamic), - // we cannot be sure that these data pointers will be proportionally shifted. - // Then we force `false` value here to set unique registers for these buffers - const auto are_static = lhs.is_static() && rhs.is_static(); - const auto equal_ptr_params_shifting = lhs.ptr_increment == rhs.ptr_increment && lhs.finalization_offset == rhs.finalization_offset; - const auto equal_element_type_sizes = lhs.data_size == rhs.data_size; - return are_static && equal_ptr_params_shifting && (equal_element_type_sizes || (lhs.ptr_increment == 0 && lhs.finalization_offset == 0)); +bool SetBufferRegGroup::can_be_in_one_reg_group(const UnifiedLoopInfo::LoopPortInfo& lhs_info, + const UnifiedLoopInfo::LoopPortInfo& rhs_info) { + const auto equal_element_type_sizes = lhs_info.desc.data_size == rhs_info.desc.data_size; + OPENVINO_ASSERT(lhs_info.port.expr_port && rhs_info.port.expr_port, "Expression ports are nullptr!"); + const auto equal_invariant_shape_paths = + MarkInvariantShapePath::getInvariantPortShapePath(*lhs_info.port.expr_port) == + MarkInvariantShapePath::getInvariantPortShapePath(*rhs_info.port.expr_port); + const auto equal_is_incremented = lhs_info.port.is_incremented == rhs_info.port.is_incremented; + return equal_invariant_shape_paths && equal_is_incremented && + (equal_element_type_sizes || !lhs_info.port.is_incremented || (lhs_info.desc.ptr_increment == 0 && lhs_info.desc.finalization_offset == 0)); } -bool SetBufferRegGroup::are_adjacent(const std::pair& lhs, - const std::pair& rhs) { +bool SetBufferRegGroup::are_adjacent(const BufferMap::value_type& lhs, const BufferMap::value_type& rhs) { const auto& lhs_ids = lhs.first->get_loop_ids(); const auto& rhs_ids = rhs.first->get_loop_ids(); const auto equal_loop_ids = lhs_ids == rhs_ids; if (equal_loop_ids) { // Buffers are connected to the same Loop and have the same outer Loops - return !can_be_in_one_group(lhs.second, rhs.second); + return !can_be_in_one_reg_group(lhs.second, rhs.second); } else { // Buffers are connected to the same Loop, but one of Buffers - inside this Loop, another - outside - // Buffers are adjacent if outer Buffer has not zero data shift params + // Buffers are adjacent if outer Buffer has non-zero data shift params if (lhs_ids.size() == rhs_ids.size()) // If the count of outer Loops are equal, it means that outer loops are already different return true; const auto& outer_buffer = lhs_ids.size() < rhs_ids.size() ? lhs : rhs; const auto count_outer_loops = std::min(lhs_ids.size(), rhs_ids.size()); const auto are_outer_loops_the_same = lhs_ids.size() != rhs_ids.size() && std::equal(rhs_ids.cbegin(), rhs_ids.cbegin() + count_outer_loops, lhs_ids.cbegin()); - const auto outer_buffer_has_zero_shifts = outer_buffer.second.ptr_increment == 0 && outer_buffer.second.finalization_offset == 0; + const auto outer_buffer_has_zero_shifts = outer_buffer.second.desc.ptr_increment == 0 && outer_buffer.second.desc.finalization_offset == 0; return !(are_outer_loops_the_same && outer_buffer_has_zero_shifts); } } -void SetBufferRegGroup::update_adj_matrix(const std::pair& lhs, - const std::pair& rhs, - const BufferPool& buffers, +void SetBufferRegGroup::update_adj_matrix(const BufferMap::value_type& lhs, const BufferMap::value_type& rhs, const BufferPool& buffers, std::vector& adj) { const auto size = buffers.size(); const auto lhs_idx = get_buffer_idx(lhs.first, buffers); @@ -80,7 +72,8 @@ void SetBufferRegGroup::update_adj_matrix(const std::pair SetBufferRegGroup::create_adjacency_matrix(LinearIR::constExprIt begin, LinearIR::constExprIt end, const BufferPool& pool) { +std::vector SetBufferRegGroup::create_adjacency_matrix(const LoopManagerPtr& loop_manager, LinearIR::constExprIt begin, LinearIR::constExprIt end, + const BufferPool& pool) { // The sync point to check for adjacency is Loop because only in Loop we increment pointers. // So if some Buffers in the one Loop have conflict (cannot be inplace: the different ptr increment and data sizes) // they are called as adjacent @@ -91,10 +84,12 @@ std::vector SetBufferRegGroup::create_adjacency_matrix(LinearIR::constExpr for (auto expr_it = begin; expr_it != end; expr_it++) { const auto &expr = *expr_it; - if (!ov::is_type(expr->get_node())) + const auto& loop_end = ov::as_type_ptr(expr->get_node()); + if (!loop_end) continue; - const auto buffer_loop_neighbours = get_buffer_loop_neighbours(expr); + const auto& loop_info = loop_manager->get_loop_info(loop_end->get_id()); + const auto buffer_loop_neighbours = get_buffer_loop_neighbours(loop_info); const auto buffers_loop_inside = get_buffer_loop_inside(expr_it); for (auto buffer_it = buffer_loop_neighbours.cbegin(); buffer_it != buffer_loop_neighbours.cend(); ++buffer_it) { // If Buffers, that are connected to the same Loop, have not proportionally ptr shift params for this Loop - these Buffers are adjacent @@ -113,47 +108,33 @@ std::vector SetBufferRegGroup::create_adjacency_matrix(LinearIR::constExpr return adj; } -SetBufferRegGroup::BufferMap SetBufferRegGroup::get_buffer_loop_neighbours(const ExpressionPtr& loop_end_expr) { - const auto& loop_end = ov::as_type_ptr(loop_end_expr->get_node()); - const auto input_count = loop_end->get_input_num(); - const auto output_count = loop_end->get_output_num(); - - const auto& ptr_increments = loop_end->get_ptr_increments(); - const auto& finalization_offsets = loop_end->get_finalization_offsets(); - const auto& data_sizes = loop_end->get_element_type_sizes(); - +SetBufferRegGroup::BufferMap SetBufferRegGroup::get_buffer_loop_neighbours(const UnifiedLoopInfoPtr& loop_info) { BufferMap buffer_neighbours; - for (size_t i = 0; i < input_count; ++i) { - const auto& parent_output = loop_end_expr->get_input_port_connector(i)->get_source().get_expr(); + + const auto& loop_inputs = loop_info->get_input_ports_info(); + for (const auto& port_info : loop_inputs) { + const auto& parent_output = port_info.port.expr_port->get_port_connector_ptr()->get_source().get_expr(); if (const auto buffer_expr = ov::as_type_ptr(parent_output)) { if (buffer_neighbours.count(buffer_expr) > 0) { - OPENVINO_ASSERT(buffer_neighbours[buffer_expr].ptr_increment == ptr_increments[i] && - buffer_neighbours[buffer_expr].finalization_offset == finalization_offsets[i], + const auto& port_desc = port_info.desc; + OPENVINO_ASSERT(buffer_neighbours[buffer_expr].desc == port_desc, "Invalid data pointer shifts: If Buffer has several consumers, this consumers must have the same shifts or zero"); continue; } - buffer_neighbours[buffer_expr] = { data_sizes[i], ptr_increments[i], finalization_offsets[i] }; + buffer_neighbours[buffer_expr] = port_info; } } - for (size_t i = input_count; i < input_count + output_count; ++i) { - // The consumers of the corresponding Store ops - const auto consumer_inputs = loop_end_expr->get_input_port_connector(i)->get_consumers(); - size_t buffer_count = 0; - size_t loop_count = 0; + + const auto& loop_outputs = loop_info->get_output_ports_info(); + for (const auto& port_info : loop_outputs) { + const auto& consumer_inputs = port_info.port.expr_port->get_port_connector_ptr()->get_consumers(); for (const auto& consumer_input : consumer_inputs) { const auto& child_expr = consumer_input.get_expr(); - if (const auto buffer_expr = ov::as_type_ptr(child_expr)) { - buffer_neighbours[buffer_expr] = { data_sizes[i], ptr_increments[i], finalization_offsets[i] }; - buffer_count++; - } else if (ov::is_type(child_expr->get_node())) { - loop_count++; - } - } - if (buffer_count > 0) { - OPENVINO_ASSERT((buffer_count == 1) && (buffer_count + loop_count == consumer_inputs.size()), - "Loop output must have not more than 1 Buffer"); + if (const auto buffer_expr = ov::as_type_ptr(child_expr)) + buffer_neighbours[buffer_expr] = port_info; } } + return buffer_neighbours; } @@ -164,9 +145,9 @@ SetBufferRegGroup::BufferMap SetBufferRegGroup::get_buffer_loop_inside(const Lin for (auto it = std::reverse_iterator(loop_end_it); (*it)->get_node() != loop_begin; ++it) { const auto& inner_expr = *it; if (const auto buffer_expr = ov::as_type_ptr(inner_expr)) { - // Set default zero values since it's not used for adjacency definition in case with Buffers in Loop + // Set default value (zeroes) since it's not used for adjacency definition in case with Buffers in Loop if (inner_buffers.count(buffer_expr) == 0) - inner_buffers[buffer_expr] = { 0, 0, 0 }; + inner_buffers[buffer_expr] = UnifiedLoopInfo::LoopPortInfo(); } } return inner_buffers; @@ -219,6 +200,7 @@ auto SetBufferRegGroup::coloring(BufferPool& buffers, std::vector& adj) -> bool SetBufferRegGroup::run(LinearIR& linear_ir, lowered::LinearIR::constExprIt begin, lowered::LinearIR::constExprIt end) { OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::SetBufferRegGroup") + // Identify Buffers using Graph coloring algorithm. BufferPool buffer_pool = linear_ir.get_buffers(); // For the better coloring Buffers should be stored in the order of execution numbers @@ -226,7 +208,7 @@ bool SetBufferRegGroup::run(LinearIR& linear_ir, lowered::LinearIR::constExprIt [](const BufferExpressionPtr& lhs, const BufferExpressionPtr& rhs) { return lhs->get_exec_num() < rhs->get_exec_num(); }); // Creation of Adj matrix - auto adj = create_adjacency_matrix(begin, end, buffer_pool); + auto adj = create_adjacency_matrix(linear_ir.get_loop_manager(), begin, end, buffer_pool); // Graph coloring algorithm const auto color_groups = coloring(buffer_pool, adj); diff --git a/src/common/snippets/src/pass/mha_tokenization.cpp b/src/common/snippets/src/pass/mha_tokenization.cpp index c42eb08b82bd4a..beb465ab3a3fbe 100644 --- a/src/common/snippets/src/pass/mha_tokenization.cpp +++ b/src/common/snippets/src/pass/mha_tokenization.cpp @@ -268,16 +268,11 @@ ov::snippets::pass::TokenizeMHASnippets::TokenizeMHASnippets(const SnippetsToken const auto pattern_rank = matmul0->get_output_partial_shape(0).size(); - const auto ops_count_before_softmax = ordered_ops.size(); auto interm_op = matmul0->get_output_target_inputs(0).begin()->get_node()->shared_from_this(); // Add supported operations which are between MatMul0 and Softmax to ordered_ops if (!update_intermediate_supported_ops(interm_op, ordered_ops, hidden_virtual_ports_count, potential_body_params_count)) return false; - // If before Softmax there is Eltwise ops, there will be one more Buffer - if (ops_count_before_softmax != ordered_ops.size() && interm_op->get_output_partial_shape(0).rbegin()->is_dynamic()) - uniqie_buffer_reg_group_count++; - std::shared_ptr reshape0 = nullptr; if (!tokenize_reshape_around_softmax(interm_op, reshape0, ordered_ops)) return false; @@ -295,10 +290,6 @@ ov::snippets::pass::TokenizeMHASnippets::TokenizeMHASnippets(const SnippetsToken if (axis != rank.get_length() - 1 || interm_op->get_output_target_inputs(0).size() != 1) return false; - // Softmax need one buffer at least - if (interm_op->get_output_partial_shape(0).rbegin()->is_dynamic()) - uniqie_buffer_reg_group_count++; - ordered_ops.push_back(interm_op); interm_op = interm_op->get_output_target_inputs(0).begin()->get_node()->shared_from_this(); @@ -333,7 +324,7 @@ ov::snippets::pass::TokenizeMHASnippets::TokenizeMHASnippets(const SnippetsToken // The Loop will have one Buffer with the same shape both on input and output. // Need to check for precision to get if we need one more register for Buffer const auto matmul0_prc = op::Brgemm::get_output_type(matmul0->get_input_element_type(0), matmul0->get_input_element_type(1)); - if (matmul1->get_input_element_type(0).size() != matmul0_prc.size() || matmul1->get_input_partial_shape(0).is_dynamic()) { + if (matmul1->get_input_element_type(0).size() != matmul0_prc.size()) { uniqie_buffer_reg_group_count++; } diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.cpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.cpp index 4f38eddc2bde0f..2a85714a792655 100644 --- a/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.cpp +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.cpp @@ -16,13 +16,6 @@ namespace tpp { namespace pass { namespace { using ExpressionPort = snippets::lowered::ExpressionPort; -bool is_planar_layout(const std::vector& layout) { - for (size_t i = 0; i < layout.size(); i++) { - if (layout[i] != i) - return false; - } - return true; -} // Note: Buffer is directly connected to the port if it remains in the same loops with the port's expression // Directly connected Buffers store data densely, so strides are defined by subternsor dims // Indirectly connected Buffers (with loops between the expr and Buffer) store data according @@ -81,12 +74,12 @@ size_t get_leading_dim(ExpressionPort port, const snippets::lowered::LoopManager subtensor[idx] = shape[shape.size() - i]; } } - OPENVINO_ASSERT(!full_dim_substituted || is_planar_layout(layout), + OPENVINO_ASSERT(!full_dim_substituted || ov::snippets::utils::is_planar_layout(layout), "Only planar layouts are supported for FULL_DIM substitution"); if (has_directly_connected_buffer(port, loop_mngr)) { shape = port_desc->get_subtensor(); - OPENVINO_ASSERT(is_planar_layout(layout), "Only planar layouts are supported for Buffers"); + OPENVINO_ASSERT(ov::snippets::utils::is_planar_layout(layout), "Only planar layouts are supported for Buffers"); const auto rank_diff = static_cast(layout.size()) - static_cast(shape.size()); if (rank_diff > 0) layout.erase(layout.end() - rank_diff, layout.end()); diff --git a/src/plugins/intel_cpu/tests/unit/snippets_transformations/x64/lowered/buffer_allocation.cpp b/src/plugins/intel_cpu/tests/unit/snippets_transformations/x64/lowered/buffer_allocation.cpp index 6dad1d4772f531..e31a8bebb95758 100644 --- a/src/plugins/intel_cpu/tests/unit/snippets_transformations/x64/lowered/buffer_allocation.cpp +++ b/src/plugins/intel_cpu/tests/unit/snippets_transformations/x64/lowered/buffer_allocation.cpp @@ -36,21 +36,24 @@ using BRGEMM_TYPE = intel_cpu::brgemm_utils::BRGEMM_TYPE; */ typedef std::tuple< - bool, // Optimized pipeline - bool, // With SplitLoops opt - size_t, // Expected Buffer size in bytes - size_t, // Expected unique Buffer reg group count - size_t // Expected unique Buffer cluster count + std::vector, // Input shapes + bool, // Optimized pipeline + bool, // With SplitLoops opt + size_t, // Expected Buffer size in bytes + size_t, // Expected unique Buffer reg group count + size_t // Expected unique Buffer cluster count > BufferAllocationCPUParams; class BufferAllocationCPUTest : public testing::TestWithParam { public: using VectorDims = ov::snippets::VectorDims; static std::string getTestCaseName(testing::TestParamInfo obj) { + std::vector shapes; bool is_optimized, with_split_loops; size_t expected_size, expected_reg_group_count, expected_cluster_count; - std::tie(is_optimized, with_split_loops, expected_size, expected_reg_group_count, expected_cluster_count) = obj.param; + std::tie(shapes, is_optimized, with_split_loops, expected_size, expected_reg_group_count, expected_cluster_count) = obj.param; std::ostringstream result; + result << "Shapes=" << ov::test::utils::partialShape2str(shapes) << "_"; result << "Opt=" << ov::test::utils::bool2str(is_optimized) << "_"; result << "Split=" << ov::test::utils::bool2str(with_split_loops) << "_"; result << "ExpBufferSize=" << expected_size << "_"; @@ -61,9 +64,11 @@ class BufferAllocationCPUTest : public testing::TestWithParamGetParam(); + std::vector shapes; + std::tie(shapes, m_is_buffer_optimized, m_with_split_loops, m_expected_size, + m_expected_reg_group_count, m_expected_cluster_count) = this->GetParam(); - const auto body = GetModel(); + const auto body = GetModel(shapes); m_linear_ir = ov::snippets::lowered::LinearIR(body, std::make_shared()); m_linear_ir.set_loop_depth(m_loop_depth); // When Subgraph::control_flow_transformations become public method, @@ -105,7 +110,7 @@ class BufferAllocationCPUTest : public testing::TestWithParam GetModel() const = 0; + virtual std::shared_ptr GetModel(const std::vector& shapes) const = 0; void MarkOp(const std::shared_ptr& node, const std::vector& subtensor) const { for (const auto& input : node->inputs()) @@ -131,20 +136,16 @@ class BufferAllocationCPUTest : public testing::TestWithParam GetModel() const override { + std::shared_ptr GetModel(const std::vector& shapes) const override { const auto subtensor_scalar = std::vector{1}; const auto subtensor_power = std::vector{1, ov::snippets::utils::get_full_dim_value()}; const auto subtensor_full = std::vector(2, ov::snippets::utils::get_full_dim_value()); // Dims are selected in order to have blocking loops by each dim - const size_t m = 1024; - const size_t k = 1024; - const size_t n1 = 128; - const size_t n2 = 256; - - const auto parameter0 = std::make_shared(ov::element::f32, ov::PartialShape({1, 12, m, k})); - const auto parameter1 = std::make_shared(ov::element::f32, ov::PartialShape({1, n1, 12, k})); - const auto parameter2 = std::make_shared(ov::element::f32, ov::PartialShape({1, 12, n1, n2})); + OPENVINO_ASSERT(shapes.size() == 3, "Incorrect count of input shapes"); + const auto parameter0 = std::make_shared(ov::element::f32, shapes[0]); + const auto parameter1 = std::make_shared(ov::element::f32, shapes[1]); + const auto parameter2 = std::make_shared(ov::element::f32, shapes[2]); const auto order = std::vector{0, 2, 3, 1}; const auto load_reshape = std::make_shared(parameter1, 1, 0, order); @@ -186,20 +187,15 @@ class MHAFP32BufferAllocationTest : public BufferAllocationCPUTest { class MHABF16AMXBufferAllocationTest : public BufferAllocationCPUTest { protected: - std::shared_ptr GetModel() const override { + std::shared_ptr GetModel(const std::vector& shapes) const override { const auto subtensor_scalar = std::vector{1}; const auto subtensor_power = std::vector{1, ov::snippets::utils::get_full_dim_value()}; const auto subtensor_full = std::vector(2, ov::snippets::utils::get_full_dim_value()); - // Dims are selected in order to have blocking loops by each dim - const size_t m = 1024; - const size_t k = 1024; - const size_t n1 = 128; - const size_t n2 = 256; - - const auto parameter0 = std::make_shared(ov::element::bf16, ov::PartialShape({1, 12, m, k})); - const auto parameter1 = std::make_shared(ov::element::bf16, ov::PartialShape({1, n1, 12, k})); - const auto parameter2 = std::make_shared(ov::element::bf16, ov::PartialShape({1, 12, n1, n2})); + OPENVINO_ASSERT(shapes.size() == 3, "Incorrect count of input shapes"); + const auto parameter0 = std::make_shared(ov::element::bf16, shapes[0]); + const auto parameter1 = std::make_shared(ov::element::bf16, shapes[1]); + const auto parameter2 = std::make_shared(ov::element::bf16, shapes[2]); const auto order = std::vector{0, 2, 3, 1}; const auto load_reshape = std::make_shared(parameter1, 1, 0, order); @@ -264,8 +260,17 @@ TEST_P(MHABF16AMXBufferAllocationTest, BufferAllocationCPU) { namespace BufferAllocationCPUTest_Instances { +std::vector static_shapes = { + { 1, 12, 1024, 1024 }, {1, 128, 12, 1024 }, {1, 12, 128, 256 }, +}; + +std::vector dynamic_shapes = { + { -1, -1, -1, -1 }, { -1, -1, -1, -1 }, { -1, -1, -1, -1 }, +}; + INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHANotOptimizedWSplit, MHAFP32BufferAllocationTest, ::testing::Combine( + ::testing::Values(static_shapes), ::testing::Values(false), ::testing::Values(true), ::testing::Values(591360), // Each Buffer has own allocated memory @@ -275,6 +280,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHANotOptimizedWSplit, INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHAOptimizedWSplit, MHAFP32BufferAllocationTest, ::testing::Combine( + ::testing::Values(static_shapes), ::testing::Values(true), ::testing::Values(true), ::testing::Values(573440), // (Buffer before brgemm) + (between brgemms) + (after brgemm) @@ -284,6 +290,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHAOptimizedWSplit, MHA INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHANotOptimizedWOSplit, MHAFP32BufferAllocationTest, ::testing::Combine( + ::testing::Values(static_shapes), ::testing::Values(false), ::testing::Values(false), ::testing::Values(2622976), // Each Buffer has own allocated memory @@ -293,6 +300,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHANotOptimizedWOSplit, INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHAOptimizedWOSplit, MHAFP32BufferAllocationTest, ::testing::Combine( + ::testing::Values(static_shapes), ::testing::Values(true), ::testing::Values(false), ::testing::Values(1572864), // (between brgemms) + (Buffer before brgemm0 and after brgemm1) @@ -302,6 +310,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHAOptimizedWOSplit, MH INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHABF16AMXNotOptimizedWSplit, MHABF16AMXBufferAllocationTest, ::testing::Combine( + ::testing::Values(static_shapes), ::testing::Values(false), ::testing::Values(true), ::testing::Values(713984), @@ -311,15 +320,17 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHABF16AMXNotOptimizedW INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHABF16AMXOptimizedWSplit, MHABF16AMXBufferAllocationTest, ::testing::Combine( + ::testing::Values(static_shapes), ::testing::Values(true), ::testing::Values(true), ::testing::Values(524288), ::testing::Values(3), - ::testing::Values(8)), + ::testing::Values(7)), BufferAllocationCPUTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHABF16AMXNotOptimizedWOSplit, MHABF16AMXBufferAllocationTest, ::testing::Combine( + ::testing::Values(static_shapes), ::testing::Values(false), ::testing::Values(false), ::testing::Values(2491648), @@ -329,11 +340,32 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHABF16AMXNotOptimizedW INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHABF16AMXOptimizedWOSplit, MHABF16AMXBufferAllocationTest, ::testing::Combine( + ::testing::Values(static_shapes), ::testing::Values(true), ::testing::Values(false), - ::testing::Values(1409024), + ::testing::Values(1671168), + ::testing::Values(3), + ::testing::Values(7)), + BufferAllocationCPUTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHAOptimizedWSplit_Dynamic, MHAFP32BufferAllocationTest, + ::testing::Combine( + ::testing::Values(dynamic_shapes), + ::testing::Values(true), + ::testing::Values(true), + ::testing::Values(0), // no static clusters + ::testing::Values(2), // (Buffer before brgemm0 and after brgemm1) + (between brgemms) + ::testing::Values(3)), // (Buffer before brgemm0) + (between brgemms) + (after brgemm1) + BufferAllocationCPUTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_BufferAllocation_MHABF16AMXOptimizedWSplit_Dynamic, MHABF16AMXBufferAllocationTest, + ::testing::Combine( + ::testing::Values(dynamic_shapes), + ::testing::Values(true), + ::testing::Values(true), + ::testing::Values(32768), // only WSP buffers ::testing::Values(3), - ::testing::Values(8)), + ::testing::Values(7)), BufferAllocationCPUTest::getTestCaseName); } // namespace BufferAllocationCPUTest_Instances From 696bddee349dd584996f73e81017cf9c2515f2c9 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Thu, 7 Nov 2024 19:11:18 +0800 Subject: [PATCH 34/93] [CPU] Roll back deconv 3rd rank layout order for non AMX (#27446) ### Details: - *temporal WA limiting Deconv acb Layout to avx512_core_amx_fp16 platform before oneDNN fix the performance regression issue of related primitive creation* ### Tickets: - *CVS-156640* --- src/plugins/intel_cpu/src/nodes/deconv.cpp | 44 ++++++++++++++-------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/deconv.cpp b/src/plugins/intel_cpu/src/nodes/deconv.cpp index cb340afc029304..2ee858e730c900 100644 --- a/src/plugins/intel_cpu/src/nodes/deconv.cpp +++ b/src/plugins/intel_cpu/src/nodes/deconv.cpp @@ -418,24 +418,38 @@ std::pair Deconvolution::makeDummyInOutShape() { return {inShape.getStaticDims(), outShape.getStaticDims()}; } -std::vector Deconvolution::getAvailableFormatsForDims(const Shape &dims) const { - if (dims.getRank() == 0) +std::vector Deconvolution::getAvailableFormatsForDims(const Shape& dims) const { + if (dims.getRank() == 0) { return {memory::format_tag::x}; - else if (dims.getRank() == 1) + } else if (dims.getRank() == 1) { return {memory::format_tag::x}; - else if (dims.getRank() == 2) + } else if (dims.getRank() == 2) { return {memory::format_tag::nc}; - else if (dims.getRank() == 3) - return {memory::format_tag::ncw, - memory::format_tag::nCw8c, - memory::format_tag::nCw16c, - memory::format_tag::nwc}; - else if (dims.getRank() == 4) - return {memory::format_tag::nchw, memory::format_tag::nChw8c, - memory::format_tag::nChw16c, memory::format_tag::nhwc }; - else if (dims.getRank() == 5) - return {memory::format_tag::ncdhw, memory::format_tag::nCdhw8c, - memory::format_tag::nCdhw16c, dnnl::memory::format_tag::ndhwc }; + } else if (dims.getRank() == 3) { + // Ticket 156640 + if (impl::cpu::x64::mayiuse(impl::cpu::x64::avx512_core_amx_fp16)) { + return {memory::format_tag::ncw, + memory::format_tag::nCw8c, + memory::format_tag::nCw16c, + memory::format_tag::nwc}; + } else { + return {memory::format_tag::tnc, + memory::format_tag::ntc, + memory::format_tag::ncw, + memory::format_tag::nCw8c, + memory::format_tag::nCw16c}; + } + } else if (dims.getRank() == 4) { + return {memory::format_tag::nchw, + memory::format_tag::nChw8c, + memory::format_tag::nChw16c, + memory::format_tag::nhwc}; + } else if (dims.getRank() == 5) { + return {memory::format_tag::ncdhw, + memory::format_tag::nCdhw8c, + memory::format_tag::nCdhw16c, + dnnl::memory::format_tag::ndhwc}; + } return {memory::format_tag::any}; } From a9ddd47ac02dea96b06bf364e2a5a22078a681ba Mon Sep 17 00:00:00 2001 From: Denis Orlov Date: Thu, 7 Nov 2024 13:45:19 +0000 Subject: [PATCH 35/93] [GHA] Increase timeout for PyTorch Models tests in precommit (#27449) ### Tickets: - 155619 --- .github/workflows/job_pytorch_models_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/job_pytorch_models_tests.yml b/.github/workflows/job_pytorch_models_tests.yml index 2959728e39dee7..d52b819981d821 100644 --- a/.github/workflows/job_pytorch_models_tests.yml +++ b/.github/workflows/job_pytorch_models_tests.yml @@ -22,7 +22,7 @@ permissions: read-all jobs: PyTorch_Models_Tests: name: PyTorch Models tests - timeout-minutes: ${{ inputs.model_scope == 'precommit' && 40 || 400 }} + timeout-minutes: ${{ inputs.model_scope == 'precommit' && 45 || 400 }} runs-on: ${{ inputs.runner }} container: ${{ fromJSON(inputs.container) }} defaults: From 98f6ea5fdea0416f17e319e6291029c2d59af14a Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 7 Nov 2024 14:58:01 +0100 Subject: [PATCH 36/93] [GHA][Coverity] Disable sccache (#27450) ### Details: - sccache doesn't properly work with Coverity ### Tickets: - *ticket-id* --- .github/workflows/coverity.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index e26bd66e097e33..5a08ec084dadac 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -85,7 +85,6 @@ jobs: image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} volumes: - /mount:/mount - options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING env: DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input CMAKE_BUILD_TYPE: 'Release' @@ -95,14 +94,6 @@ jobs: OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib BUILD_DIR: /__w/openvino/openvino/openvino_build COVERITY_TOOL_DIR: /__w/openvino/openvino/coverity_tool - CMAKE_CXX_COMPILER_LAUNCHER: sccache - CMAKE_C_COMPILER_LAUNCHER: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: 1 - SCCACHE_SERVER_PORT: 35555 - SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt - SCCACHE_LOG: warn - SCCACHE_AZURE_KEY_PREFIX: coverity_lin - SCCACHE_CACHE_SIZE: 50G steps: - name: Clone OpenVINO @@ -141,8 +132,6 @@ jobs: # # Build # - - name: Clean sccache stats - run: ${SCCACHE_PATH} --zero-stats - name: CMake configure - OpenVINO run: | @@ -165,9 +154,6 @@ jobs: - name: Cmake build - OpenVINO with Coverity run: ${COVERITY_TOOL_DIR}/cov-analysis*/bin/cov-build --dir ${BUILD_DIR}/cov-int cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - - - name: Show sccache stats - run: ${SCCACHE_PATH} --show-stats - name: Pack Artefacts run: tar -cvf - cov-int | pigz > openvino.tgz From 5c834609fcf9d7d5bf3ce67208dccd5588e03df6 Mon Sep 17 00:00:00 2001 From: Maxim Vafin Date: Thu, 7 Nov 2024 16:01:39 +0100 Subject: [PATCH 37/93] [PT FE] Inherit signature from forward while patching (#27413) ### Details: - *Inherit signature from forward while patching* ### Tickets: - *ticket-id* --------- Signed-off-by: Maxim Vafin --- .../python/src/openvino/frontend/pytorch/patch_model.py | 3 +++ tests/layer_tests/py_frontend_tests/test_torch_frontend.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/bindings/python/src/openvino/frontend/pytorch/patch_model.py b/src/bindings/python/src/openvino/frontend/pytorch/patch_model.py index fb8f70e2a566bc..0d1c4803e3115f 100644 --- a/src/bindings/python/src/openvino/frontend/pytorch/patch_model.py +++ b/src/bindings/python/src/openvino/frontend/pytorch/patch_model.py @@ -4,6 +4,7 @@ # flake8: noqa # mypy: ignore-errors +import functools import logging import torch from openvino.frontend.pytorch import ModuleExtension @@ -70,6 +71,8 @@ def new_forward(*args, **kwargs): Trampoline.stashed_kwargs = kwargs return extension.convert(m, Trampoline.apply, *args, **kwargs) + # make signature of new_forward same as of forward + new_forward = functools.wraps(m.forward)(new_forward) setattr(m, orig_forward_name, m.forward) m.forward = new_forward diff --git a/tests/layer_tests/py_frontend_tests/test_torch_frontend.py b/tests/layer_tests/py_frontend_tests/test_torch_frontend.py index faee72bb5d938a..b659c1735d8453 100644 --- a/tests/layer_tests/py_frontend_tests/test_torch_frontend.py +++ b/tests/layer_tests/py_frontend_tests/test_torch_frontend.py @@ -687,6 +687,7 @@ def test_patched_16bit_model_converts(): from openvino.frontend.pytorch import patch_model from openvino import convert_model, compile_model import copy + import inspect from transformers.pytorch_utils import Conv1D class ModelWithLinear(torch.nn.Module): @@ -716,6 +717,9 @@ def forward(self, x1, x2): model_fp16 = copy.deepcopy(model_ref).half() patch_model.__make_16bit_traceable(model_fp16) + # verify torch.nn.Linear signature after patching + signature = inspect.signature(model_ref.branch1[0].forward).parameters + assert ["input"] == list(signature) # the approach with patching only works for node with no grad with torch.no_grad(): converted_model = convert_model(model_fp16, example_input=example) From 8edb1501d3472944f131df2ecb8d60246c7dac6a Mon Sep 17 00:00:00 2001 From: Ivan Novoselov Date: Thu, 7 Nov 2024 15:29:49 +0000 Subject: [PATCH 38/93] Sns enable lp kn blocking (#26957) ### Details: - *Perform weights repacking outside of the blocking cycles* - *Functionally enable blocking for I8,U8 and BF16* - *The blocking will be temporary disabled until blocking heuristic is updated in 156014* ### Tickets: - *154729* --- src/common/snippets/src/lowered/loop_port.cpp | 6 +- .../snippets/cpu_runtime_configurator.cpp | 65 ++++++++++- .../snippets/cpu_runtime_configurator.hpp | 13 +++ .../snippets/x64/jit_kernel_emitter.cpp | 10 +- .../snippets/x64/kernel_executors/brgemm.cpp | 12 +- .../x64/kernel_executors/brgemm_copy_b.cpp | 2 +- src/plugins/intel_cpu/src/nodes/subgraph.cpp | 9 +- .../snippets/x64/op/brgemm_utils.cpp | 5 - .../snippets/x64/op/brgemm_utils.hpp | 12 +- .../adjust_brgemm_copy_b_loop_ports.cpp | 107 ++++++++++++++++++ .../adjust_brgemm_copy_b_loop_ports.hpp | 37 ++++++ .../x64/pass/lowered/brgemm_cpu_blocking.cpp | 83 ++++++-------- .../x64/pass/lowered/brgemm_cpu_blocking.hpp | 4 - .../x64/lowered/brgemm_blocking.cpp | 89 +++++++-------- 14 files changed, 329 insertions(+), 125 deletions(-) create mode 100644 src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.cpp create mode 100644 src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.hpp diff --git a/src/common/snippets/src/lowered/loop_port.cpp b/src/common/snippets/src/lowered/loop_port.cpp index 990b2801beccb8..52f59bb1fa4d35 100644 --- a/src/common/snippets/src/lowered/loop_port.cpp +++ b/src/common/snippets/src/lowered/loop_port.cpp @@ -30,7 +30,7 @@ std::shared_ptr LoopPort::clone_with_new_expr(const ExpressionPtr& new bool operator==(const LoopPort& lhs, const LoopPort& rhs) { if (&lhs == &rhs) return true; - return lhs.expr_port == rhs.expr_port && lhs.is_incremented == rhs.is_incremented && lhs.dim_idx == rhs.dim_idx; + return *lhs.expr_port == *rhs.expr_port && lhs.is_incremented == rhs.is_incremented && lhs.dim_idx == rhs.dim_idx; } bool operator!=(const LoopPort& lhs, const LoopPort& rhs) { @@ -38,8 +38,8 @@ bool operator!=(const LoopPort& lhs, const LoopPort& rhs) { } bool operator<(const LoopPort& lhs, const LoopPort& rhs) { - return (lhs.expr_port < rhs.expr_port) || - (lhs.expr_port == rhs.expr_port && + return (*lhs.expr_port < *rhs.expr_port) || + (*lhs.expr_port == *rhs.expr_port && (lhs.is_incremented < rhs.is_incremented || (lhs.is_incremented == rhs.is_incremented && lhs.dim_idx < rhs.dim_idx))); } diff --git a/src/plugins/intel_cpu/src/emitters/snippets/cpu_runtime_configurator.cpp b/src/plugins/intel_cpu/src/emitters/snippets/cpu_runtime_configurator.cpp index 56c2c75dae9bc2..1c3d283ab673b1 100644 --- a/src/plugins/intel_cpu/src/emitters/snippets/cpu_runtime_configurator.cpp +++ b/src/plugins/intel_cpu/src/emitters/snippets/cpu_runtime_configurator.cpp @@ -7,6 +7,9 @@ #include "snippets/lowered/loop_manager.hpp" #include "snippets/utils/utils.hpp" +#ifndef OPENVINO_ARCH_ARM64 +#include "transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.hpp" +#endif namespace ov { namespace intel_cpu { @@ -36,10 +39,36 @@ std::string CPURuntimeConfig::to_string() const { CPURuntimeConfigurator::CPURuntimeConfigurator() : ov::snippets::RuntimeConfigurator(std::make_shared()) { } +void CPURuntimeConfigurator::initialization(const ov::snippets::lowered::LinearIRCPtr& linear_ir) { + RuntimeConfigurator::initialization(linear_ir); + if (linear_ir->is_dynamic()) { + loopPortsAdjuster = BrgemmCopyBLoopPortsAdjuster(linear_ir); + } +} + void CPURuntimeConfigurator::update(const ov::snippets::lowered::LinearIRCPtr& linear_ir) { - RuntimeConfigurator::update(linear_ir); + m_config->master_shape = linear_ir->get_master_shape(); + if (linear_ir->is_dynamic()) { + update_loop_info(linear_ir); + } + + if (!m_optimizer.optimize()) { + // If the optimization was not applied, offsets are updated using shapes from descriptors + auto shapes = extract_shapes(); + update_data_offsets(shapes, extract_layouts()); + m_latest_shapes = std::move(shapes); + } if (linear_ir->is_dynamic()) + loopPortsAdjuster.optimize(); + + // Update KernelExecutor Table should be before `update_buffer_scratchpad_size` + // because `ComputeAllocationSize` depends on subtensors which are updated in the table + get_kernel_executor_table()->update_state(linear_ir); + update_buffer_scratchpad_size(linear_ir); + + if (linear_ir->is_dynamic()) { update_loop_args(linear_ir); + } } void CPURuntimeConfigurator::update_tensor_rank(const ov::snippets::VectorDims& master_shape) { @@ -72,6 +101,40 @@ void CPURuntimeConfigurator::update_loop_args(const ov::snippets::lowered::Linea } } } +#ifdef OPENVINO_ARCH_ARM64 +CPURuntimeConfigurator::BrgemmCopyBLoopPortsAdjuster::BrgemmCopyBLoopPortsAdjuster(const ov::snippets::lowered::LinearIRCPtr& linear_ir) { +} + +void CPURuntimeConfigurator::BrgemmCopyBLoopPortsAdjuster::optimize() { +} +#else +CPURuntimeConfigurator::BrgemmCopyBLoopPortsAdjuster::BrgemmCopyBLoopPortsAdjuster(const ov::snippets::lowered::LinearIRCPtr& linear_ir) { + const auto& pass = std::make_shared(); + pass->run(*linear_ir); + const auto& affected_uni_loops = pass->get_affected_loops(); + const auto& loop_map = linear_ir->get_loop_manager()->get_map(); + for (const auto& p : loop_map) { + if (const auto& exp_loop = ov::as_type_ptr(p.second)) { + const auto& uni_loop = exp_loop->get_unified_loop_info(); + if (affected_uni_loops.count(uni_loop)) + m_affected_uni2exp_map[uni_loop].push_back(exp_loop); + } + } +} + +void CPURuntimeConfigurator::BrgemmCopyBLoopPortsAdjuster::optimize() { + for (const auto& p : m_affected_uni2exp_map) { + const auto& uni_loop = p.first; + const auto& exp_loops = p.second; + snippets::RuntimeConfigurator::LoopInfoRuntimeParamsMap initialized_info; + if (intel_cpu::pass::AdjustBrgemmCopyBLoopPorts::update_loop_info(uni_loop)) { + initialized_info[uni_loop] = get_loop_runtime_params(uni_loop); + for (const auto& exp_loop : exp_loops) + update_expanded_loop_info(exp_loop, initialized_info); + } + } +} +#endif } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/emitters/snippets/cpu_runtime_configurator.hpp b/src/plugins/intel_cpu/src/emitters/snippets/cpu_runtime_configurator.hpp index 00bc676678d189..d8ef9772e813ff 100644 --- a/src/plugins/intel_cpu/src/emitters/snippets/cpu_runtime_configurator.hpp +++ b/src/plugins/intel_cpu/src/emitters/snippets/cpu_runtime_configurator.hpp @@ -44,6 +44,7 @@ class CPURuntimeConfigurator : public ov::snippets::RuntimeConfigurator { * @param linear_ir LinearIR */ void init_tensor_rank(const ov::snippets::lowered::LinearIRCPtr& linear_ir) const override; + void initialization(const ov::snippets::lowered::LinearIRCPtr& linear_ir) override; /** * @brief Calculate Loop parameters of Loop emitters and update these values in CPURuntimeConfig * @param linear_ir LinearIR @@ -51,6 +52,18 @@ class CPURuntimeConfigurator : public ov::snippets::RuntimeConfigurator { void update_loop_args(const ov::snippets::lowered::LinearIRCPtr& linear_ir) const; static const size_t rank6D; + + class BrgemmCopyBLoopPortsAdjuster { + public: + BrgemmCopyBLoopPortsAdjuster() = default; + BrgemmCopyBLoopPortsAdjuster(const ov::snippets::lowered::LinearIRCPtr& linear_ir); + + void optimize(); + + private: + std::unordered_map> m_affected_uni2exp_map; + } loopPortsAdjuster; }; } // namespace intel_cpu diff --git a/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_kernel_emitter.cpp b/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_kernel_emitter.cpp index 4a27c8e17150e8..476123355abe70 100644 --- a/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_kernel_emitter.cpp +++ b/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_kernel_emitter.cpp @@ -180,16 +180,16 @@ void jit_kernel_static_emitter::init_data_pointers(const std::vectormov(data_ptr_regs[i], h->ptr[reg_runtime_params + GET_OFF(dst_ptrs) + (i - num_inputs) * sizeof(void*)]); init_ptr_with_offset(data_ptr_regs[i], data_offsets[i], reg_tmp); } - // a rare case when num_params is maximal, so we have no spare gprs - // * Static case: we can use reg_runtime_params as the last reg_tmp for the last iteration (and corrupt it), since - // it won't be used anymore - // * Dynamic case: we will need reg_runtime_params to pass runtime args to LoopScheduler, so we have to - // push a reg on the stack, and restore it value afterward + // A rare case when num_params is maximal, so we have no spare gprs + // Note that we need to push-pop runtime params because some kernels might need them even in the static case + // (e.g. brgemm emitter for amx tile configuration access) if (last_iter_explicitly) { + h->push(reg_runtime_params); h->mov(data_ptr_regs[i], h->ptr[reg_runtime_params + GET_OFF(dst_ptrs) + (i - num_inputs) * sizeof(void*)]); reg_tmp = reg_runtime_params; // can corrupt reg_runtime_params, since we won't use it anymore init_ptr_with_offset(data_ptr_regs[i], data_offsets[i], reg_tmp); + h->pop(reg_runtime_params); } } diff --git a/src/plugins/intel_cpu/src/emitters/snippets/x64/kernel_executors/brgemm.cpp b/src/plugins/intel_cpu/src/emitters/snippets/x64/kernel_executors/brgemm.cpp index 6f1f4ab93aeda9..fad1be5a5d1289 100644 --- a/src/plugins/intel_cpu/src/emitters/snippets/x64/kernel_executors/brgemm.cpp +++ b/src/plugins/intel_cpu/src/emitters/snippets/x64/kernel_executors/brgemm.cpp @@ -246,7 +246,7 @@ void BrgemmKernelExecutor::update_config(const ov::snippets::lowered::Expression // Note: We check `is_incremented` attribute only for not incremented ports because // this `is_incremented = true` can be changed by `CleanRepeatedDataPointerShifts` optimization auto check_port = [&](const ov::snippets::lowered::LoopPort& p) { return p.dim_idx == 0; }; - OPENVINO_ASSERT(in_ports.size() == 2 && !in_ports.front().is_incremented && std::all_of(in_ports.cbegin(), in_ports.cend(), check_port) && + OPENVINO_ASSERT(in_ports.size() >= 2 && !in_ports.front().is_incremented && std::all_of(in_ports.cbegin(), in_ports.cend(), check_port) && out_ports.size() == 1 && check_port(out_ports.back()), "Incorrect Loop by Brgemm dimension N"); N = current_expanded_loop_info->get_increment(); @@ -269,7 +269,7 @@ void BrgemmKernelExecutor::update_config(const ov::snippets::lowered::Expression // Quick validation check: Should we check that port is really Brgemm port? // Note: We check `is_incremented` attribute only for not incremented ports because // this `is_incremented = true` can be changed by `CleanRepeatedDataPointerShifts` optimization - OPENVINO_ASSERT(in_ports.size() == 2 && in_ports.front().dim_idx == 0 && in_ports.back().dim_idx == 1 && + OPENVINO_ASSERT(in_ports.size() >= 2 && in_ports.front().dim_idx == 0 && in_ports.back().dim_idx == 1 && out_ports.size() == 1 && !out_ports.front().is_incremented, "Incorrect Loop by Brgemm dimension K"); K = current_expanded_loop_info->get_increment(); @@ -286,7 +286,7 @@ void BrgemmKernelExecutor::update_config(const ov::snippets::lowered::Expression OV_CPU_JIT_EMITTER_ASSERT(brgemm_node, "Got invalid node type in update_config"); // In case of data repacking LDB is chosen in accordance with repacking buffer size if (with_repacking(brgemm_node->get_type())) - LDB = brgemm_utils::repacking::compute_out_leading_dim(N, brgemm_node->get_input_element_type(1)); + LDB = brgemm_utils::repacking::compute_LDB(LDB, brgemm_node->get_input_element_type(1)); config.update(DIM_CAST(M), DIM_CAST(N), DIM_CAST(K), LDA, LDB, LDC, beta); } @@ -303,6 +303,8 @@ void BrgemmKernelExecutor::execute(const BrgemmKernelExecutor* executor, call_ar } cpu::x64::brgemm_kernel_params_t brgemm_p; + // Note: compensations should be applied only once, so we do it only on the first iteration, when beta == 0 + size_t is_with_comp = config.get_beta() == 0 && config.is_with_comp(); brgemm_p.batch = nullptr; // default value brgemm_p.ptr_A = args->A; @@ -311,8 +313,8 @@ void BrgemmKernelExecutor::execute(const BrgemmKernelExecutor* executor, call_ar brgemm_p.ptr_D = args->C; brgemm_p.ptr_buf = args->scratch; brgemm_p.ptr_bias = nullptr; - brgemm_p.do_post_ops = static_cast(config.is_with_comp()); - brgemm_p.do_apply_comp = static_cast(config.is_with_comp()); + brgemm_p.do_post_ops = is_with_comp; + brgemm_p.do_apply_comp = is_with_comp; brgemm_p.skip_accm = 0; brgemm_p.BS = 1; // default value OV_CPU_JIT_EMITTER_ASSERT(kernel->compiled_kernel, "has nullptr kernel"); diff --git a/src/plugins/intel_cpu/src/emitters/snippets/x64/kernel_executors/brgemm_copy_b.cpp b/src/plugins/intel_cpu/src/emitters/snippets/x64/kernel_executors/brgemm_copy_b.cpp index 17f8923ae9867b..cc79458c7c4c64 100644 --- a/src/plugins/intel_cpu/src/emitters/snippets/x64/kernel_executors/brgemm_copy_b.cpp +++ b/src/plugins/intel_cpu/src/emitters/snippets/x64/kernel_executors/brgemm_copy_b.cpp @@ -305,7 +305,7 @@ void BrgemmCopyBKernelExecutor::update_config(const ov::snippets::lowered::Expre init(N_dim, N_blk, 0); const auto& brg_weight_etype = expr->get_node()->get_input_element_type(0); - const auto LDB = brgemm_utils::repacking::compute_out_leading_dim(N_dim, brg_weight_etype); + const auto LDB = brgemm_utils::repacking::compute_LDB(N_dim, brg_weight_etype); const auto copy_B_wei_stride = ov::snippets::utils::get_dim_stride(expr->get_input_port(0), config.is_transposed_B() ? 0 : 1) * brg_weight_etype.size(); config.update(N_dim, N_blk, K_dim, K_blk, copy_B_wei_stride, LDB); diff --git a/src/plugins/intel_cpu/src/nodes/subgraph.cpp b/src/plugins/intel_cpu/src/nodes/subgraph.cpp index 05a1e5a958771b..ee24dd66493204 100644 --- a/src/plugins/intel_cpu/src/nodes/subgraph.cpp +++ b/src/plugins/intel_cpu/src/nodes/subgraph.cpp @@ -17,11 +17,10 @@ #include "snippets/pass/positioned_pass.hpp" #include "snippets/pass/canonicalization.hpp" #include "snippets/pass/analyze_broadcastable_inputs.hpp" -#include "snippets/lowered/linear_ir.hpp" -#include "snippets/lowered/pass/optimize_domain.hpp" #include "snippets/lowered/pass/insert_loops.hpp" #include "snippets/lowered/pass/mark_loops.hpp" #include "snippets/lowered/pass/insert_buffers.hpp" +#include "snippets/lowered/pass/init_loops.hpp" #include "transformations/defs.hpp" #include "transformations/cpu_opset/common/pass/convert_to_swish_cpu.hpp" #include "transformations/snippets/common/pass/mul_add_to_fma.hpp" @@ -38,6 +37,7 @@ #include "transformations/snippets/x64/pass/brgemm_to_brgemm_cpu.hpp" #include "transformations/snippets/x64/pass/enforce_precision.hpp" #include "transformations/snippets/x64/shape_inference.hpp" +#include "transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.hpp" #endif #include "utils/cpu_utils.hpp" @@ -54,6 +54,7 @@ std::mutex err_print_lock; #endif #ifdef SNIPPETS_LIBXSMM_TPP +#include "snippets/lowered/pass/optimize_domain.hpp" #include "transformations/tpp/x64/pass/brgemm_to_brgemm_tpp.hpp" #include "transformations/tpp/x64/pass/eltwise_to_eltwise_tpp.hpp" #include "transformations/tpp/x64/pass/scalar_to_scalar_tpp.hpp" @@ -683,6 +684,10 @@ Subgraph::ControlFlowPasses Subgraph::getControlFlowPasses() const { SNIPPETS_REGISTER_PASS_RELATIVE(Place::After, ov::snippets::lowered::pass::MarkLoops, ov::intel_cpu::pass::BrgemmCPUBlocking); + + SNIPPETS_REGISTER_PASS_RELATIVE(Place::After, ov::snippets::lowered::pass::InitLoops, + ov::intel_cpu::pass::AdjustBrgemmCopyBLoopPorts); + SNIPPETS_REGISTER_PASS_RELATIVE(Place::After, ov::snippets::lowered::pass::InsertLoops, ov::intel_cpu::pass::FuseLoadStoreConvert); SNIPPETS_REGISTER_PASS_RELATIVE(Place::Before, ov::snippets::lowered::pass::InsertBuffers, diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_utils.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_utils.cpp index 844ec338b8a83b..adc215ef1d9900 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_utils.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_utils.cpp @@ -6,7 +6,6 @@ #include "dnnl_extension_utils.h" #include "emitters/utils.hpp" -#include "snippets/utils/utils.hpp" #include "transformations/snippets/x64/op/brgemm_copy_b.hpp" #include "utils/general_utils.h" @@ -76,10 +75,6 @@ size_t get_elems_in_vec(const ov::element::Type& precision) { } namespace repacking { -size_t compute_out_leading_dim(const size_t n_block, const ov::element::Type& precision) { - return std::max(n_block, compute_inner_n_block(precision)); -} - size_t compute_inner_n_block(const ov::element::Type& precision) { switch (precision) { case element::i8: return 64; diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_utils.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_utils.hpp index bc627c59920c4b..aeb5b22cd56129 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_utils.hpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_utils.hpp @@ -8,6 +8,7 @@ #include "openvino/core/type/element_type.hpp" #include "openvino/core/dimension.hpp" #include "snippets/lowered/expression.hpp" +#include "snippets/utils/utils.hpp" namespace ov { namespace intel_cpu { @@ -42,14 +43,19 @@ size_t compute_vnni_factor(const ov::element::Type& precision); size_t get_elems_in_vec(const ov::element::Type& precision); namespace repacking { +/// \brief Computes inner N block size used by OneDNN implementation. Depends on tensor precision +size_t compute_inner_n_block(const ov::element::Type& precision); /** * @brief Computes leading dimension (LDB) which must be used in brgemm and brgemm_copy_b emitters * @param n_block N block size shared between BrgemmCPU and BrgemmCopyB node * @param precision tensor precision */ -size_t compute_out_leading_dim(const size_t n_block, const ov::element::Type& precision); -/// \brief Computes inner N block size used by OneDNN implementation. Depends on tensor precision -size_t compute_inner_n_block(const ov::element::Type& precision); +template::value || std::is_same::value), bool>::type> +T compute_LDB(T n_block, const ov::element::Type& precision) { + return snippets::utils::is_dynamic_value(n_block) ? + n_block : + std::max(n_block, static_cast(compute_inner_n_block(precision))); +} } // namespace repacking } // namespace brgemm_utils } // namespace intel_cpu diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.cpp new file mode 100644 index 00000000000000..c421e5cc2a4805 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.cpp @@ -0,0 +1,107 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/itt.hpp" + +#include "adjust_brgemm_copy_b_loop_ports.hpp" +#include "snippets/utils/utils.hpp" +#include "snippets/lowered/loop_manager.hpp" +#include "snippets/lowered/expressions/buffer_expression.hpp" +#include "transformations/snippets/x64/op/brgemm_copy_b.hpp" +#include "transformations/snippets/x64/op/brgemm_cpu.hpp" + +namespace ov { +namespace intel_cpu { + +bool pass::AdjustBrgemmCopyBLoopPorts::update_loop_info(const std::shared_ptr& loop_info) { + OPENVINO_ASSERT(loop_info, "Invalid loop info pointer"); + bool modified = false; + auto caller = [&](snippets::lowered::LoopPort &loop_port, + snippets::lowered::UnifiedLoopInfo::LoopPortDesc &loop_desc) { + const auto& p = *loop_port.expr_port; + if (p.get_type() == snippets::lowered::ExpressionPort::Input && + p.get_index() == 1) { + const auto& node = p.get_expr()->get_node(); + if (auto brg = as_type_ptr(node)) { + const auto precision = node->get_input_element_type(1); + /* + * The BrgemmCopyB operation repacks the weights in the following way: + * 1) VNNI format is applied: KN4k for I8/U8, or KN2k for BF16 + * 2) Zero padding is applied if N4k < 256 or N2k < 64 + */ + if (brgemm_utils::with_repacking(brg->get_type()) && + precision != element::f32 && loop_port.is_incremented) { + // K blocking loop: account for zero padding + if (loop_port.dim_idx == 1) { + const auto ptr_incr = loop_desc.ptr_increment; + const auto blocked_shape_ptr_inc = brgemm_utils::repacking::compute_LDB(ptr_incr, precision); + if (ptr_incr != 0 && ptr_incr != blocked_shape_ptr_inc) { + loop_desc.ptr_increment = blocked_shape_ptr_inc; + OPENVINO_ASSERT(loop_desc.finalization_offset % ptr_incr == 0, + "Can't rescale finalization offsets"); + loop_desc.finalization_offset = loop_desc.ptr_increment * + (loop_desc.finalization_offset / ptr_incr); + } + // N blocking loop: account for the VNNI format + } else if (loop_port.dim_idx == 0) { + auto k_blk_size = static_cast(brgemm_utils::compute_vnni_factor(precision)); + loop_desc.ptr_increment = snippets::utils::dynamic_safe_mul(loop_desc.ptr_increment, k_blk_size); + loop_desc.finalization_offset = snippets::utils::dynamic_safe_mul(loop_desc.finalization_offset, k_blk_size); + } else { + OPENVINO_THROW("Unexpected loop port dimension index in AdjustBrgemmCopyBLoopPorts"); + } + modified = true; + } + } + } + }; + loop_info->iterate_through_infos(caller); + return modified; +} + +bool pass::AdjustBrgemmCopyBLoopPorts::run(const snippets::lowered::LinearIR& linear_ir) { + OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::AdjustBrgemmCopyBLoopPorts") + + bool modified = false; + + for (const auto& expr : linear_ir) { + const auto& node = expr->get_node(); + if (!is_type(node)) + continue; + const auto& repacking_loop_ids = expr->get_loop_ids(); + const auto& child_ports = expr->get_output_port(0).get_connected_ports(); + OPENVINO_ASSERT(child_ports.size() == 1 && + is_type(child_ports.begin()->get_expr()), + "BrgemmCopyB should have one BufferExpression child"); + auto grandchild_ports = child_ports.begin()->get_expr()->get_output_port(0).get_connected_ports(); + for (const auto& target_port : grandchild_ports) { + const auto& port_node = target_port.get_expr()->get_node(); + if (!is_type(port_node)) { + OPENVINO_ASSERT(is_type(port_node), + "Invalid grandchild of BrgemmCopyB"); + continue; + } + const auto &brgemm_loop_ids = target_port.get_expr()->get_loop_ids(); + // Continue if there is no blocking loop + if (brgemm_loop_ids.empty() && repacking_loop_ids.empty()) + continue; + OPENVINO_ASSERT(brgemm_loop_ids.size() > repacking_loop_ids.size(), "Invalid BrgemmCopyB loop configuration"); + const auto &loop_manager = linear_ir.get_loop_manager(); + for (auto i = repacking_loop_ids.size(); i < brgemm_loop_ids.size(); i++) { + const auto &loop = loop_manager->get_loop_info(brgemm_loop_ids[i]); + auto uni_loop = ov::as_type_ptr(loop); + if (!uni_loop) + uni_loop = ov::as_type_ptr(loop)->get_unified_loop_info(); + if (!m_affected_loops.count(uni_loop) && update_loop_info(uni_loop)) { + m_affected_loops.insert(uni_loop); + modified = true; + } + } + } + } + + return modified; +} +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.hpp new file mode 100644 index 00000000000000..5c65c7a0282823 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.hpp @@ -0,0 +1,37 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "snippets/lowered/pass/pass.hpp" +#include "snippets/lowered/loop_info.hpp" + +namespace ov { +namespace intel_cpu { +namespace pass { + +/** + * @interface AdjustBrgemmCopyBLoopPorts + * @brief BrgemmCopyB is located outside of blocking loops and repacks input data into a blocked layout. + * This layout should be accounted for when we increment BrgemmCopyB data pointers. This pass + * Finds loop ports connected to BrgemmCopyB and sets appropriate pointer increments. + * @ingroup snippets + */ +class AdjustBrgemmCopyBLoopPorts: public snippets::lowered::pass::Pass { +public: + AdjustBrgemmCopyBLoopPorts() = default; + OPENVINO_RTTI("AdjustBrgemmCopyBLoopPorts", "Pass"); + bool run(const snippets::lowered::LinearIR& linear_ir); + bool run(snippets::lowered::LinearIR& linear_ir) override { + return run(const_cast(linear_ir)); + } + static bool update_loop_info(const snippets::lowered::UnifiedLoopInfoPtr& uni_loop_info); + const std::unordered_set& get_affected_loops() { return m_affected_loops; } +private: + std::unordered_set m_affected_loops; +}; + +} // namespace pass +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_cpu_blocking.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_cpu_blocking.cpp index 51565537c43568..9b3009284e09e8 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_cpu_blocking.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_cpu_blocking.cpp @@ -44,18 +44,6 @@ LinearIR::constExprIt BrgemmCPUBlocking::move_new_memory_buffer(LinearIR& linear return std::prev(brgemm_it); } -LinearIR::constExprIt BrgemmCPUBlocking::get_loop_begin_pos(LinearIR& linear_ir, const LinearIR::constExprIt& brgemm_it, const ExpressionPtr& copy_b_expr) { - auto loop_begin_it = brgemm_it; - const auto& brgemm_expr = *brgemm_it; - const auto brgemm = ov::as_type_ptr(brgemm_expr->get_node()); - OPENVINO_ASSERT(brgemm, "get_loop_begin_pos must be called only for BrgemmCPU expression"); - if (with_amx(brgemm->get_type())) - loop_begin_it = move_new_memory_buffer(linear_ir, brgemm_it); - if (copy_b_expr) - loop_begin_it = linear_ir.find(copy_b_expr); - return loop_begin_it; -} - size_t BrgemmCPUBlocking::get_default_n_blk(size_t n) const { return dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core) ? 64 : 24; } @@ -66,6 +54,8 @@ std::tuple BrgemmCPUBlocking::get_blocking_params(const size_t m_blk, n_blk, k_blk; std::tie(m_blk, n_blk, k_blk) = BrgemmBlockingBase::get_blocking_params(brgemm_expr); + // Note: K,N blocking is functionally enabled, need to turn it on after blocking heuristic is updated to cover + // the low precision cases (ticket: 156014) if (with_repacking(brgemm->get_type())) { n_blk = get_full_dim_value(); k_blk = get_full_dim_value(); @@ -88,47 +78,48 @@ bool BrgemmCPUBlocking::mark_blocking_loops(LinearIR& linear_ir, const auto brgemm = ov::as_type_ptr(brgemm_expr->get_node()); const auto type = brgemm->get_type(); - if (stand_alone(type)) - return ov::snippets::lowered::pass::BrgemmBlockingBase::mark_blocking_loops(linear_ir, brgemm_it, m_block, n_block, k_block); + auto res = ov::snippets::lowered::pass::BrgemmBlockingBase::mark_blocking_loops(linear_ir, brgemm_it, m_block, n_block, k_block); - brgemm_expr->get_input_port_descriptor(0)->set_subtensor({m_block, k_block}); - brgemm_expr->get_input_port_descriptor(1)->set_subtensor({k_block, n_block}); - brgemm_expr->get_output_port_descriptor(0)->set_subtensor({m_block, n_block}); + if (stand_alone(type)) + return res; const auto copy_b_expr = linear_ir.get_expr_by_node(brgemm->get_brgemm_copy()); - copy_b_expr->get_input_port_descriptor(0)->set_subtensor({k_block, n_block}); - copy_b_expr->get_output_port_descriptor(0)->set_subtensor({k_block, n_block}); + const ov::snippets::VectorDims full_subtensor(2, get_full_dim_value()); + copy_b_expr->get_input_port_descriptor(0)->set_subtensor(full_subtensor); + copy_b_expr->get_output_port_descriptor(0)->set_subtensor(full_subtensor); + + if (with_amx(type)) { + move_new_memory_buffer(linear_ir, brgemm_it); + auto buffer_it = std::prev(brgemm_it); + buffer_it->get()->set_loop_ids(brgemm_expr->get_loop_ids()); + } + + const auto& loop_manager = linear_ir.get_loop_manager(); if (with_compensations(type)) { - const ov::snippets::VectorDims compensations_subtensor{1, n_block}; + const ov::snippets::VectorDims compensations_subtensor{1, get_full_dim_value()}; OPENVINO_ASSERT(brgemm_expr->get_input_count() == 3, "Brgemm must have 3 inputs in case of compensations."); - brgemm_expr->get_input_port_descriptor(2)->set_subtensor(compensations_subtensor); + const auto& compens_port = brgemm_expr->get_input_port(2); + compens_port.get_descriptor_ptr()->set_subtensor(compensations_subtensor); copy_b_expr->get_output_port_descriptor(1)->set_subtensor(compensations_subtensor); - } - const auto& loop_manager = linear_ir.get_loop_manager(); - if (!is_full_dim_value(k_block)) { - const auto loop_begin = get_loop_begin_pos(linear_ir, brgemm_it, copy_b_expr); - const std::vector entries{LoopPort(brgemm_expr->get_input_port(0), true, 0), - LoopPort(copy_b_expr->get_input_port(0), true, 1)}; - const std::vector exits{LoopPort(brgemm_expr->get_output_port(0), false)}; - mark_k_blocking(loop_manager, loop_begin, std::next(brgemm_it), entries, exits, k_block); - } - if (!is_full_dim_value(n_block)) { - const auto loop_begin = get_loop_begin_pos(linear_ir, brgemm_it, copy_b_expr); - const std::vector entries{LoopPort(brgemm_expr->get_input_port(0), false), - LoopPort(copy_b_expr->get_input_port(0), true)}; - const std::vector exits{LoopPort(brgemm_expr->get_output_port(0), true)}; - mark_n_blocking(loop_manager, loop_begin, std::next(brgemm_it), entries, exits, n_block); - } - if (!is_full_dim_value(m_block)) { - const bool include_repacking = !is_full_dim_value(k_block) || !is_full_dim_value(n_block); - const auto loop_begin = get_loop_begin_pos(linear_ir, brgemm_it, include_repacking ? copy_b_expr : nullptr); - const auto b_input_port = include_repacking ? copy_b_expr->get_input_port(0) : brgemm_expr->get_input_port(1); - std::vector entries{LoopPort(brgemm_expr->get_input_port(0), true), LoopPort(b_input_port, false)}; - if (!include_repacking && with_compensations(type)) - entries.emplace_back(brgemm_expr->get_input_port(2), false); - const std::vector exits{LoopPort(brgemm_expr->get_output_port(0), true)}; - mark_m_blocking(loop_manager, loop_begin, std::next(brgemm_it), entries, exits, m_block); + const auto& loop_ids = brgemm_expr->get_loop_ids(); + size_t i = 0; + LoopInfoPtr loop_info = nullptr; + auto update_loop_info = [&](LoopPort&& new_port) { + OPENVINO_ASSERT(i < loop_ids.size(), "Attempt to access invalid loop id"); + loop_info = loop_manager->get_loop_info(loop_ids[i++]); + const auto& in_ports = loop_info->get_input_ports(); + OPENVINO_ASSERT(in_ports.size() > 1, "Invalid number of input loop ports"); + loop_info->replace_with_new_ports(in_ports[1], {in_ports[1], new_port}); + }; + if (!is_full_dim_value(m_block)) + update_loop_info({compens_port, false, 1}); + + if (!is_full_dim_value(n_block)) + update_loop_info({compens_port, true, 0}); + + if (!is_full_dim_value(k_block)) + update_loop_info({compens_port, false, 1}); } return true; } diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_cpu_blocking.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_cpu_blocking.hpp index 22429a6b0c98fb..0fee4e80454f9a 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_cpu_blocking.hpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_cpu_blocking.hpp @@ -40,10 +40,6 @@ class BrgemmCPUBlocking : public ov::snippets::lowered::pass::BrgemmBlocking get_blocking_params(const ov::snippets::lowered::ExpressionPtr& brgemm_expr) const override; diff --git a/src/plugins/intel_cpu/tests/unit/snippets_transformations/x64/lowered/brgemm_blocking.cpp b/src/plugins/intel_cpu/tests/unit/snippets_transformations/x64/lowered/brgemm_blocking.cpp index 89f2e06c14a9fa..738afba6a101f9 100644 --- a/src/plugins/intel_cpu/tests/unit/snippets_transformations/x64/lowered/brgemm_blocking.cpp +++ b/src/plugins/intel_cpu/tests/unit/snippets_transformations/x64/lowered/brgemm_blocking.cpp @@ -81,43 +81,6 @@ void create_brgemm_loop_infos(const LinearIRPtr& linear_ir, BrgemmBlockingBase::get_default_blocking_loop_handlers(m, m_block))); } } - -void create_brgemm_with_copy_b_loop_infos(const LinearIRPtr& linear_ir, - const ExpressionPtr& brgemm_expr, - const ExpressionPtr& copy_b_expr, - size_t m = 0, size_t m_blk = 0, - size_t k = 0, size_t k_blk = 0, - size_t n = 0, size_t n_blk = 0) { - const bool k_block = k != 0 && k_blk != 0; - const bool n_block = k != 0 && k_blk != 0; - const bool m_block = m != 0 && m_blk != 0; - if (k_block) { - const auto loop_info = - std::make_shared(k, k_blk, - std::vector{LoopPort(brgemm_expr->get_input_port(0)), - LoopPort(copy_b_expr->get_input_port(0), true, 1)}, - std::vector{LoopPort(brgemm_expr->get_output_port(0), false)}, - get_k_loop_handlers(k, k_block)); - linear_ir->get_loop_manager()->add_loop_info(loop_info); - } - if (n_block) { - linear_ir->get_loop_manager()->add_loop_info( - std::make_shared(n, n_blk, - std::vector{LoopPort(brgemm_expr->get_input_port(0), false), - LoopPort(copy_b_expr->get_input_port(0))}, - std::vector{LoopPort(brgemm_expr->get_output_port(0))}, - BrgemmBlockingBase::get_default_blocking_loop_handlers(n, n_block))); - } - if (m_block) { - const auto& second_input_port = k_block || n_block ? copy_b_expr->get_input_port(0) : brgemm_expr->get_input_port(1); - linear_ir->get_loop_manager()->add_loop_info( - std::make_shared(m, m_blk, - std::vector{LoopPort(brgemm_expr->get_input_port(0), true, 1), - LoopPort(second_input_port, false, 1)}, - std::vector{LoopPort(brgemm_expr->get_output_port(0), true, 1)}, - BrgemmBlockingBase::get_default_blocking_loop_handlers(m, m_block))); - } -} } // namespace class BrgemmBlockingTest : public LoweredPassTestsF { @@ -230,6 +193,8 @@ TEST_F(BrgemmCPUBlockingTest, BlockingIsNotNeeded) { } TEST_F(BrgemmCPUBlockingTest, WithDataRepacking) { + // Skipped because K,N blocking is disabled until heuristic is updated (ticket: 156014) + GTEST_SKIP(); const ov::Dimension::value_type m = 384; const ov::Dimension::value_type k = 1024; const ov::Dimension::value_type n = 384; @@ -257,14 +222,16 @@ TEST_F(BrgemmCPUBlockingTest, WithDataRepacking) { auto brgemm = linear_ir_ref->push_node(data_a.second, copy_b.second, BRGEMM_TYPE::REPACKING_ONLY); const auto& brgemm_expr = *brgemm.first; - init_expr_descriptors(brgemm_expr, {{m_blk, full_dim}, {full_dim, full_dim}, {m_blk, full_dim}}); - create_brgemm_with_copy_b_loop_infos(linear_ir_ref, brgemm_expr, copy_b_expr, m, m_blk); - brgemm_expr->set_loop_ids({0}); + init_expr_descriptors(brgemm_expr, {{m_blk, k_blk}, {k_blk, n_blk}, {m_blk, n_blk}}); + create_brgemm_loop_infos(linear_ir_ref, brgemm_expr, m, m_blk, k, k_blk, n, n_blk); + brgemm_expr->set_loop_ids({2, 1, 0}); auto result = linear_ir_ref->push_node(brgemm.second); } } TEST_F(BrgemmCPUBlockingTest, WithCompensations) { + // Skipped because K,N blocking is disabled until heuristic is updated (ticket: 156014) + GTEST_SKIP(); const ov::Dimension::value_type m = 384; const ov::Dimension::value_type k = 1024; const ov::Dimension::value_type n = 384; @@ -292,14 +259,27 @@ TEST_F(BrgemmCPUBlockingTest, WithCompensations) { const auto& copy_b_n = copy_b.second; auto brgemm = linear_ir_ref->push_node(data_a.second, copy_b_n->output(0), copy_b_n->output(1), BRGEMM_TYPE::WITH_COMPENSATIONS); const auto& brgemm_expr = *brgemm.first; - init_expr_descriptors(brgemm_expr, {{m_blk, full_dim}, {full_dim, full_dim}, {1, full_dim}, {m_blk, full_dim}}); - create_brgemm_loop_infos(linear_ir_ref, brgemm_expr, m, m_blk); - brgemm_expr->set_loop_ids({0}); + init_expr_descriptors(brgemm_expr, {{m_blk, k_blk}, {k_blk, n_blk}, {1, full_dim}, {m_blk, n_blk}}); + create_brgemm_loop_infos(linear_ir_ref, brgemm_expr, m, m_blk, k, k_blk, n, n_blk); + + const auto loop_manager = linear_ir_ref->get_loop_manager(); + auto update_loop_info = [&loop_manager](size_t loop_id, LoopPort&& new_port) { + const auto& loop_info = loop_manager->get_loop_info(loop_id); + const auto& in_ports = loop_info->get_input_ports(); + loop_info->replace_with_new_ports(in_ports[1], {in_ports[1], new_port}); + }; + const auto& compens_port = brgemm_expr->get_input_port(2); + update_loop_info(1, {compens_port, true, 0}); + update_loop_info(0, {compens_port, false, 1}); + + brgemm_expr->set_loop_ids({2, 1, 0}); auto result = linear_ir_ref->push_node(brgemm.second); } } TEST_F(BrgemmCPUBlockingTest, AMX) { + // Skipped because K,N blocking is disabled until heuristic is updated (ticket: 156014) + GTEST_SKIP(); const ov::Dimension::value_type m = 384; const ov::Dimension::value_type k = 1024; const ov::Dimension::value_type n = 384; @@ -322,16 +302,25 @@ TEST_F(BrgemmCPUBlockingTest, AMX) { auto data_b = linear_ir_ref->push_node(precision, input_shape_b); auto copy_b = linear_ir_ref->push_node(data_b.second, precision, BRGEMM_TYPE::REPACKING_ONLY); const auto copy_b_expr = *copy_b.first; - init_expr_descriptors(copy_b_expr, {{full_dim, full_dim}, {full_dim, full_dim}}); + init_expr_descriptors(copy_b_expr, {{full_dim, full_dim}, + {full_dim, full_dim}}); - auto scratch = linear_ir_ref->push_node(ov::Shape{BrgemmCPU::SCRATCH_BYTE_SIZE}); - scratch.first->get()->set_loop_ids({0}); + auto scratch = linear_ir_ref->push_node(ov::Shape {BrgemmCPU::SCRATCH_BYTE_SIZE}); + scratch.first->get()->set_loop_ids({2, 1, 0}); - auto brgemm = linear_ir_ref->push_node(data_a.second, copy_b.second, scratch.second, BRGEMM_TYPE::WITH_AMX); + auto brgemm = linear_ir_ref->push_node(data_a.second, copy_b.second, scratch.second, + BRGEMM_TYPE::WITH_AMX); const auto& brgemm_expr = *brgemm.first; - init_expr_descriptors(brgemm_expr, {{m_blk, full_dim}, {full_dim, full_dim}, get_default_subtensor(), {m_blk, full_dim}}); - create_brgemm_with_copy_b_loop_infos(linear_ir_ref, brgemm_expr, copy_b_expr, m, m_blk); - brgemm_expr->set_loop_ids({0}); + init_expr_descriptors(brgemm_expr, {{m_blk, k_blk}, {k_blk, n_blk}, get_default_subtensor(), {m_blk, n_blk}}); + create_brgemm_loop_infos(linear_ir_ref, brgemm_expr, m, 0, k, k_blk, n, n_blk); + + std::vector entries {LoopPort(brgemm_expr->get_input_port(0), true, 1), + LoopPort(brgemm_expr->get_input_port(1), false, 1)}; + std::vector exits {LoopPort(brgemm_expr->get_output_port(0), true, 1)}; + auto handlers = BrgemmBlockingBase::get_default_blocking_loop_handlers(m, m_blk); + linear_ir_ref->get_loop_manager()-> + add_loop_info(std::make_shared(m, m_blk, entries, exits, handlers)); + brgemm_expr->set_loop_ids({2, 1, 0}); auto result = linear_ir_ref->push_node(brgemm.second); } } From f33a637d30484ea1404f9648b739e35fb43883cd Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Thu, 7 Nov 2024 07:50:28 -0800 Subject: [PATCH 39/93] Improve SIT to compose a batch tensor as a model input make up from a packet of different images (#27341) ### Details: - A new parameter `override_model_batch_size` was added several PRs ago. The purpose of this parameter was is to take a picture/binary and "stretch" it to fit a batched tensor as an input of a batched model providing that we could run and validate batched models - The current PR enhance this idea and allows us to set distinguished images/binaries in a batched tensor input so that we could validate correctness of processing of different lines of batch during our inference invocations. SIT now uses "|" separator to discern different images inside input images array as an command line argument. The batched images filing algorithm is the following: 1) If passed images amount is not enough to fill up an entire batch, a last image will be propagated and copied to remnant lines of N-batch. 2) If passed images array is larger than an batched tensor, then only N images will be used up and the rest of images will be thrown away Example: ``` single-image-test --device CPU --network resnet50.xml --mode classification --top_k 5 --il NCHW --ol NC --iml NCHW --oml NC --ip FP16 --op FP16 --override_model_batch_size 3 --input "dog.bmp|00000001.JPEG|00000002.JPEG" ``` gives us ``` ... Load input #0 from [dog.bmp,00000001.JPEG,00000002.JPEG] as f16 [N,C,H,W] [3,3,224,224] Fill up all input batch slices planes up to 3 with image data from the array: [0/3] Fill input batch slices planes starting from index 1 up to 3 with image data from the array: [1/3] Fill input batch slices planes starting from index 2 up to 3 with image data from the array: [2/3] ``` ### Tickets: - E####145303 --------- Co-authored-by: Maksim Doronin --- .../tools/common/include/tensor_utils.hpp | 8 + .../tools/common/include/tools_helpers.hpp | 16 +- .../tools/common/src/tensor_utils.cpp | 24 +++ .../intel_npu/tools/compile_tool/main.cpp | 18 +- .../tools/single-image-test/main.cpp | 179 ++++++++++++++---- 5 files changed, 197 insertions(+), 48 deletions(-) diff --git a/src/plugins/intel_npu/tools/common/include/tensor_utils.hpp b/src/plugins/intel_npu/tools/common/include/tensor_utils.hpp index 87b2301a7ae4fb..18d431c157d2dd 100644 --- a/src/plugins/intel_npu/tools/common/include/tensor_utils.hpp +++ b/src/plugins/intel_npu/tools/common/include/tensor_utils.hpp @@ -68,5 +68,13 @@ std::vector> parseTensorsAsFP32(const std::map& tensors, const ov::Layout& layout); + +/** + * @brief Split a batched tensor on several non-batched tensors having the same shapes and precisions. + * + * @param tensors The source non-batched tensors + * @return The merged batched tensor + */ +std::list splitBatchedTensor(const ov::Tensor &tensor, const ov::Layout& layout, size_t parts); } // namespace utils } // namespace npu diff --git a/src/plugins/intel_npu/tools/common/include/tools_helpers.hpp b/src/plugins/intel_npu/tools/common/include/tools_helpers.hpp index e9743594ad8711..304f16c29d9a87 100644 --- a/src/plugins/intel_npu/tools/common/include/tools_helpers.hpp +++ b/src/plugins/intel_npu/tools/common/include/tools_helpers.hpp @@ -109,6 +109,7 @@ void setModelBatch(std::shared_ptr& model, uint32_t batch = 1) { if (batch == 1) { return; } + std::cout << "Configuring model batch: " << batch << std::endl; for (auto&& item : model->get_parameters()) { auto shape = item->get_partial_shape(); auto rank = shape.rank(); @@ -175,7 +176,20 @@ void reshape(ov::OutputVector inputsInfo, InputsInfo& infoMap, std::shared_ptrget_layout(); + std::cout << " " << param->get_friendly_name() << " : " << param->get_element_type() << " / " + << param->get_layout().to_string() << " / " << param->get_partial_shape().to_string() << std::endl; + } + std::cout << "Network outputs:" << std::endl; + for (auto&& result : network.get_results()) { + std::cout << " " << result->get_friendly_name() << " : " << result->get_element_type() << " / " + << result->get_layout().to_string() << " / " << result->get_output_partial_shape(0).to_string() + << std::endl; } } diff --git a/src/plugins/intel_npu/tools/common/src/tensor_utils.cpp b/src/plugins/intel_npu/tools/common/src/tensor_utils.cpp index 32616b86135243..c2b4902497777e 100644 --- a/src/plugins/intel_npu/tools/common/src/tensor_utils.cpp +++ b/src/plugins/intel_npu/tools/common/src/tensor_utils.cpp @@ -492,5 +492,29 @@ ov::Tensor joinTensors(const std::list& tensors, const ov::Layout& l } return out; } + +std::list splitBatchedTensor(const ov::Tensor &tensor, const ov::Layout& layout, size_t parts) { + if (!parts) { + OPENVINO_THROW("Cannot split tensor on parts: ", parts); + } + auto pivotShape = tensor.get_shape(); + if (!ov::layout::has_batch(layout)) { + OPENVINO_THROW("Cannot split tensor: has no batch_idx in layout", layout.to_string()); + } + auto pivotPrecision = tensor.get_element_type(); + if (pivotShape[ov::layout::batch_idx(layout)] % parts != 0) { + OPENVINO_THROW("Cannot split tensor with batch size: ", pivotShape[ov::layout::batch_idx(layout)], " on: ", parts ," equal tensors"); + } + pivotShape[ov::layout::batch_idx(layout)] /= parts; + std::list ret; + const auto *inputBuffer = tensor.data(); + for (size_t i = 0; i < parts; i ++) { + ov::Tensor out(pivotPrecision, pivotShape); + memcpy(out.data(), inputBuffer, out.get_byte_size()); + inputBuffer += out.get_byte_size(); + ret.push_back(std::move(out)); + } + return ret; +} } // namespace utils } // namespace npu diff --git a/src/plugins/intel_npu/tools/compile_tool/main.cpp b/src/plugins/intel_npu/tools/compile_tool/main.cpp index 7a088d1afc69e2..47178833508182 100644 --- a/src/plugins/intel_npu/tools/compile_tool/main.cpp +++ b/src/plugins/intel_npu/tools/compile_tool/main.cpp @@ -296,21 +296,6 @@ void configurePrePostProcessing(std::shared_ptr& model, const std::st model = preprocessor.build(); } -void printInputAndOutputsInfoShort(const ov::Model& network) { - std::cout << "Network inputs:" << std::endl; - for (auto&& param : network.get_parameters()) { - auto l = param->get_layout(); - std::cout << " " << param->get_friendly_name() << " : " << param->get_element_type() << " / " - << param->get_layout().to_string() << " / " << param->get_partial_shape().to_string() << std::endl; - } - std::cout << "Network outputs:" << std::endl; - for (auto&& result : network.get_results()) { - std::cout << " " << result->get_friendly_name() << " : " << result->get_element_type() << " / " - << result->get_layout().to_string() << " / " << result->get_output_partial_shape(0).to_string() - << std::endl; - } -} - inline std::string fileNameNoExt(const std::string& filepath) { auto pos = filepath.rfind('.'); if (pos == std::string::npos) @@ -456,6 +441,9 @@ int main(int argc, char* argv[]) { std::cout << "Configuring model pre & post processing" << std::endl; configurePrePostProcessing(model, FLAGS_ip, FLAGS_op, FLAGS_iop, FLAGS_il, FLAGS_ol, FLAGS_iol, FLAGS_iml, FLAGS_oml, FLAGS_ioml); + if (FLAGS_shape.empty()) { + setModelBatch(model, FLAGS_override_model_batch_size); + } std::cout << "Printing Input and Output Info from model" << std::endl; printInputAndOutputsInfoShort(*model); auto timeBeforeLoadNetwork = std::chrono::steady_clock::now(); diff --git a/src/plugins/intel_npu/tools/single-image-test/main.cpp b/src/plugins/intel_npu/tools/single-image-test/main.cpp index 5658c18650243b..2454e9afd40e0d 100644 --- a/src/plugins/intel_npu/tools/single-image-test/main.cpp +++ b/src/plugins/intel_npu/tools/single-image-test/main.cpp @@ -183,6 +183,21 @@ std::vector splitStringList(const std::string& str, char delim) { return out; } +std::string to_string(const std::vector& c) { + std::stringstream stream; + std::string ret; + if (!c.empty()) { + stream << "["; + for (const auto &elem : c) { + stream << elem << ","; + } + ret = stream.str(); + ret.pop_back(); + ret += "]"; + } + return ret; +} + std::map parseArgMap(std::string argMap) { argMap.erase(std::remove_if(argMap.begin(), argMap.end(), ::isspace), argMap.end()); @@ -386,7 +401,23 @@ void convertBufferType(OutT* destination, const InT* source, size_t numberOfElem }); } -void cvToOV(const cv::Mat& cvImg, const ov::Tensor& tensor, const ov::Shape& shape, const ov::Layout& layout, +struct BatchIndexer { + const size_t index = 0; + const size_t size = 1; + + BatchIndexer(size_t lineIndex = 0, size_t lineCount = 1) : index(lineIndex), size(lineCount) { + OPENVINO_ASSERT(index < size, "Inconsistent parameters used for " + "BatchIndexer construction, lineIndex: ", index, + " must be lesser than lineCount: ", size); + } + + std::string to_string() const { + std::stringstream sstream; + sstream << "[" << index << "/" << size << "]"; + return sstream.str(); + } +}; +void cvToOV(const cv::Mat& cvImg, const BatchIndexer &cvImgInBatch, const ov::Tensor& tensor, const ov::Shape& shape, const ov::Layout& layout, const std::string& colorFormat) { const ov::element::Type& precision = tensor.get_element_type(); @@ -449,8 +480,12 @@ void cvToOV(const cv::Mat& cvImg, const ov::Tensor& tensor, const ov::Shape& sha if (layout == ov::Layout("NHWC")) { const auto dataBuffer = reinterpret_cast(tensor.data()); - - cv::Mat out(static_cast(H), static_cast(W), cvType, dataBuffer); + cv::Mat auxOut(static_cast(H), static_cast(W), cvType); + cv::Mat tensorOut(static_cast(H), static_cast(W), cvType, dataBuffer); + // only a first image from an input image array fills an original input tensor up. + // Subsequent images (if exist) will fill batch slices of the input tensor + // by its number in the input array respectively + cv::Mat &out = (cvImgInBatch.index == 0 ? tensorOut : auxOut); if (precision == ov::element::Type_t::f16) { const auto inPtr = in.ptr(); @@ -466,13 +501,35 @@ void cvToOV(const cv::Mat& cvImg, const ov::Tensor& tensor, const ov::Shape& sha in.copyTo(out); } - for (size_t n = 1; n < N; ++n) { + // Being called sequentially with ascending `cvImgInBatch.index` value, + // it fills up rest of the batched tensor by + // a last requested image data until its ending from a batched slice position + // determined by parameter 'cvImgInBatch.index', so that filling N batched tensor + // by array of images size M, where M < N, will make up + // The final batched tensor will comprise + // [imgIdx_0, imgIdx_1,..., imgIdx_M, imgIdx_M,...,imgIdx_M] as its slices + if (cvImgInBatch.index == 0 && N != 1) { + std::cout << "Fill up all input batch slices up to " << N + << " with image data from the array: [" + << cvImgInBatch.to_string() << std::endl; + } + for (size_t n = std::max(1, cvImgInBatch.index); n < N; ++n) { + if (n == std::max(1, cvImgInBatch.index) && cvImgInBatch.index >= 1) { + std::cout << "Fill input batch slices starting from index " + << n << " up to " << N << " with image data from the array: " + << cvImgInBatch.to_string() << std::endl; + } cv::Mat batch(static_cast(H), static_cast(W), cvType, dataBuffer + n * (out.size().area() * out.elemSize())); out.copyTo(batch); } } else if (layout == ov::Layout("NCHW")) { - auto tensorPlanes = ovToCV(tensor, shape, layout, 0); + ov::Tensor auxTensor(precision, shape); + const ov::Tensor &outTensor = (cvImgInBatch.index == 0 ? tensor : auxTensor); + // only a first image from an input image array fills an original input tensor up. + // Subsequent images (if exist) will fill batch slices of the input tensor + // by its number in the input array respectively + auto tensorPlanes = ovToCV(outTensor, shape, layout, 0); if (!(precision == ov::element::Type_t::f16 || precision == ov::element::Type_t::bf16)) { @@ -495,7 +552,24 @@ void cvToOV(const cv::Mat& cvImg, const ov::Tensor& tensor, const ov::Shape& sha } } - for (size_t n = 1; n < N; ++n) { + // Being called sequentially with ascending `cvImgInBatch.index` value, + // it fills up rest of the batched tensor by + // a last requested image data until its ending from a batched slice position + // determined by parameter 'cvImgInBatch.index', so that filling N batched tensor + // by array of images size M, where M < N, will make up + // The final batched tensor will comprise + // [imgIdx_0, imgIdx_1,..., imgIdx_M, imgIdx_M,...,imgIdx_M] as its slices + if (cvImgInBatch.index == 0 && N != 1) { + std::cout << "Fill up all input batch slices planes up to " << N + << " with image data from the array: " + << cvImgInBatch.to_string() << std::endl; + } + for (size_t n = std::max(1, cvImgInBatch.index); n < N; ++n) { + if (n == std::max(1, cvImgInBatch.index) && cvImgInBatch.index >= 1) { + std::cout << "Fill input batch slices planes starting from index " + << n << " up to " << N << " with image data from the array: " + << cvImgInBatch.to_string() << std::endl; + } const auto batchPlanes = ovToCV(tensor, shape, layout, n); OPENVINO_ASSERT(batchPlanes.size() == tensorPlanes.size()); @@ -654,27 +728,28 @@ std::string cleanName(std::string&& name) { return std::move(name); } -ov::Tensor loadImage(const ov::element::Type& precision, const ov::Shape& shape, const ov::Layout& layout, - const std::string& filePath, const std::string& colorFormat) { - const auto frame = cv::imread(filePath, cv::IMREAD_COLOR); - OPENVINO_ASSERT(!frame.empty(), "Failed to open input image file ", filePath); - +ov::Tensor loadImages(const ov::element::Type& precision, const ov::Shape& shape, const ov::Layout& layout, + const std::vector& filePaths, const std::string& colorFormat) { const ov::Tensor tensor(precision, shape); + for (size_t fileIndex = 0; fileIndex != filePaths.size(); fileIndex++) { + const auto &filePath = filePaths[fileIndex]; + const auto frame = cv::imread(filePath, cv::IMREAD_COLOR); + OPENVINO_ASSERT(!frame.empty(), "Failed to open input image file ", filePath); - cvToOV(frame, tensor, shape, layout, colorFormat); - + cvToOV(frame, BatchIndexer{fileIndex, filePaths.size()}, tensor, shape, layout, colorFormat); + } return tensor; } -ov::Tensor loadBinary(const ov::element::Type& modelPrecision, const ov::Shape& shape, const ov::Layout& layout, - const std::string& filePath, const ov::element::Type& dataPrecision) { +void loadBinary(const std::string& filePath, const BatchIndexer &fileSourceInBatch, ov::Tensor &requestedTensor, + const ov::element::Type& modelPrecision, const ov::Shape& shape, + const ov::Layout& layout, const ov::element::Type& dataPrecision) { std::ifstream binaryFile(filePath, std::ios_base::binary | std::ios_base::ate); OPENVINO_ASSERT(binaryFile, "Failed to open input binary file: ", filePath); const auto fileSize = binaryFile.tellg(); binaryFile.seekg(0, std::ios_base::beg); OPENVINO_ASSERT(binaryFile.good(), "While reading a file an error is encountered"); const size_t fileBytes = static_cast(fileSize); - ov::Tensor requestedTensor(modelPrecision, shape); const size_t reqTensorBytes = static_cast(requestedTensor.get_byte_size()); if (dataPrecision != modelPrecision && dataPrecision != ov::element::Type_t::undefined) { @@ -688,7 +763,7 @@ ov::Tensor loadBinary(const ov::element::Type& modelPrecision, const ov::Shape& std::cout << "File contains " << fileBytes << " bytes, but it expected to be: " << inputTensor.get_byte_size() << " while converting precision from " << dataPrecision << " to " << modelPrecision - << ". Check whether it is possible to batch loading " << std::endl; + << ". Check whether it is possible to fit it into batch loading " << std::endl; OPENVINO_ASSERT(ov::layout::has_batch(layout), "Input layout has no batch dimenstion: ", layout.to_string()); size_t N = shape[ov::layout::batch_idx(layout)]; @@ -703,12 +778,20 @@ ov::Tensor loadBinary(const ov::element::Type& modelPrecision, const ov::Shape& const ov::Tensor convertedPrecisionTensor(modelPrecision, debatchedInputTensorShape); npu::utils::convertTensorPrecision(inputDebatchedTensor, convertedPrecisionTensor); std::list tensorsToJoin; - std::generate_n(std::back_inserter(tensorsToJoin), N, [&convertedPrecisionTensor]() { - return convertedPrecisionTensor; - }); + std::list tensorsFromSplit = npu::utils::splitBatchedTensor(requestedTensor, layout, N); + // Constitute a new bathed tensor of size N from parts of it + // enumerated by indices from the interval [0...fileSourceInBatch.index], + // where fileSourceInBatch.index < N + // The rest parts of the new tensor [fileSourceInBatch.index+1...N] + // will be filled up by same content of an image of `fileSourceInBatch.index` + std::copy_n(tensorsFromSplit.begin(), std::min(fileSourceInBatch.index, N), std::back_inserter(tensorsToJoin)); + if (fileSourceInBatch.index < N) { + std::generate_n(std::back_inserter(tensorsToJoin), N - fileSourceInBatch.index, [&convertedPrecisionTensor]() { + return convertedPrecisionTensor; + }); + } requestedTensor = npu::utils::joinTensors(tensorsToJoin, layout); } - } else { if (fileBytes == reqTensorBytes) { binaryFile.read(reinterpret_cast(requestedTensor.data()), @@ -716,22 +799,40 @@ ov::Tensor loadBinary(const ov::element::Type& modelPrecision, const ov::Shape& } else { std::cout << "File contains " << fileBytes << " bytes, but it expected to be: " << reqTensorBytes << " when datatypes match. " - << ". Check whether it is possible to batch loading " << std::endl; + << "Check whether it is possible to fit it into batch loading " << std::endl; OPENVINO_ASSERT(ov::layout::has_batch(layout), "Input layout has no batch dimenstion: ", layout.to_string()); size_t N = shape[ov::layout::batch_idx(layout)]; OPENVINO_ASSERT(fileBytes * N == reqTensorBytes, "File contains ", fileBytes, " bytes, but ", reqTensorBytes, " in batch size ", N, " expected"); - // duplicate a binary into tensor memory if the tensor batched - for (size_t n = 0; n < N; ++n) { + if (fileSourceInBatch.index == 0 && N != 1) { + std::cout << "Fill up all input batch slices up to " << N + << " with binary data from the array: " + << fileSourceInBatch.to_string() << std::endl; + } + for (size_t n = std::max(0, fileSourceInBatch.index); n < N; ++n) { + if (n == std::max(1, fileSourceInBatch.index) && fileSourceInBatch.index >= 1) { + std::cout << "Fill input batch slices starting from index " + << n << " up to " << N + << " with binary data from the data sources array: " + << fileSourceInBatch.to_string() << std::endl; + } binaryFile.seekg(0, std::ios_base::beg); binaryFile.read(reinterpret_cast(requestedTensor.data()) + fileBytes * n, static_cast(fileBytes)); } } } +} +ov::Tensor loadBinaries(const ov::element::Type& modelPrecision, const ov::Shape& shape, const ov::Layout& layout, + const std::vector& filePaths, const ov::element::Type& dataPrecision) { + ov::Tensor requestedTensor(modelPrecision, shape); + for (size_t fileIndex = 0; fileIndex != filePaths.size(); fileIndex++) { + const auto &filePath = filePaths[fileIndex]; + loadBinary(filePath, BatchIndexer{fileIndex, filePaths.size()}, requestedTensor, modelPrecision, shape, layout, dataPrecision); + } return requestedTensor; } @@ -752,12 +853,12 @@ ov::Tensor loadBinary(const ov::element::Type& modelPrecision, const ov::Shape& * @return The tensor containing the loaded data. */ ov::Tensor loadInput(const ov::element::Type& modelPrecision, const ov::Shape& shape, const ov::Layout& layout, - const std::string& filePath, const std::string& colorFormat, + const std::vector& filePaths, const std::string& colorFormat, const ov::element::Type& dataPrecision = ov::element::Type_t::undefined) { if (isImage(shape, layout) && !FLAGS_img_as_bin) { - return loadImage(modelPrecision, shape, layout, filePath, colorFormat); + return loadImages(modelPrecision, shape, layout, filePaths, colorFormat); } else { - return loadBinary(modelPrecision, shape, layout, filePath, dataPrecision); + return loadBinaries(modelPrecision, shape, layout, filePaths, dataPrecision); } } @@ -1719,11 +1820,20 @@ static int runSingleImageTest() { std::map outModelLayouts = parseLayoutRegex(FLAGS_oml); std::vector inputFilesPerCase; - std::vector> inputFilesForOneInfer; + using FilesPerInput = std::vector; + using FilesForModelInputs = std::vector; + std::vector inputFilesForOneInfer; inputFilesPerCase = splitStringList(FLAGS_input, ';'); for (const auto& images : inputFilesPerCase) { - inputFilesForOneInfer.push_back(splitStringList(images, ',')); + std::vector filesPerModel = splitStringList(images, ','); + FilesForModelInputs entireModelFiles; + entireModelFiles.reserve(filesPerModel.size()); + for (auto &&filesPerInput : filesPerModel) { + // from now on each input of a model support multiple image files as content of a batched input + entireModelFiles.push_back(splitStringList(filesPerInput, '|')); + } + inputFilesForOneInfer.push_back(std::move(entireModelFiles)); } std::vector inputBinPrecisionStrPerCase; @@ -1896,8 +2006,13 @@ static int runSingleImageTest() { } } + if (FLAGS_shape.empty()) { + setModelBatch(model, FLAGS_override_model_batch_size); + } std::cout << "Compile model" << std::endl; - compiledModel = core.compile_model(ppp.build(), FLAGS_device); + model = ppp.build(); + printInputAndOutputsInfoShort(*model); + compiledModel = core.compile_model(model, FLAGS_device); } else { std::cout << "Import network " << FLAGS_network << std::endl; @@ -1947,7 +2062,7 @@ static int runSingleImageTest() { for (size_t numberOfTestCase = 0; numberOfTestCase < inputFilesPerCase.size(); ++numberOfTestCase) { const auto inputsInfo = compiledModel.inputs(); const auto outputsInfo = compiledModel.outputs(); - std::vector inputFiles = inputFilesForOneInfer[numberOfTestCase]; + const FilesForModelInputs &inputFiles = inputFilesForOneInfer[numberOfTestCase]; OPENVINO_ASSERT(inputFiles.size() == inputsInfo.size(), "Number of input files ", inputFiles.size(), " doesn't match network configuration ", inputsInfo.size()); @@ -1984,7 +2099,7 @@ static int runSingleImageTest() { inputDescriptors.emplace(inputInfo.get_any_name(), TensorDescriptor{precision, shape, dataShape, inputLayout}); - std::cout << "Load input #" << inputInd << " from " << inputFiles[inputInd] << " as " << precision + std::cout << "Load input #" << inputInd << " from " << to_string(inputFiles[inputInd]) << " as " << precision << " " << inputLayout.to_string() << " " << shape << std::endl; const ov::Tensor tensor = From b416fb02d97896f8198287c8a976cc8e1978ff62 Mon Sep 17 00:00:00 2001 From: Alina Kladieva Date: Thu, 7 Nov 2024 17:48:10 +0100 Subject: [PATCH 40/93] [GHA] Enable Fedora trigger in post-commit (#27463) We need that to have build artifacts stored from pushes Signed-off-by: Alina Kladieva --- .github/workflows/fedora_29.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fedora_29.yml b/.github/workflows/fedora_29.yml index 3f685502747a19..f3b101327f76dc 100644 --- a/.github/workflows/fedora_29.yml +++ b/.github/workflows/fedora_29.yml @@ -5,7 +5,7 @@ on: merge_group: push: branches: - # - master + - master - 'releases/**' concurrency: From fb5b5ed0036c1bff4753c70364541600889841db Mon Sep 17 00:00:00 2001 From: Chon Ming Lee Date: Fri, 8 Nov 2024 14:34:54 +0800 Subject: [PATCH 41/93] [GPU] Add check to fallback to permute_ref if different format for oneDNN (#27402) ### Details: If the input and output format are not the same in permute before oneDNN convolution, permute_kernel_f_y_axes doesn't support it. Need to fallback to permute_ref ### Tickets: - *CVS-155933* --- .../permute/permute_kernel_f_y_axes.cpp | 6 ++ .../unit/test_cases/permute_gpu_test.cpp | 70 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/permute/permute_kernel_f_y_axes.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/permute/permute_kernel_f_y_axes.cpp index 69756b02e2d9ff..deb25d17618347 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/permute/permute_kernel_f_y_axes.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/permute/permute_kernel_f_y_axes.cpp @@ -214,6 +214,8 @@ bool PermuteKernel_f_y_axes::Validate(const Params& p) const { const auto& params = dynamic_cast(p); const auto& in = params.inputs[0]; const auto in_layout = in.GetLayout(); + const auto& out = params.outputs[0]; + const auto& out_layout = out.GetLayout(); const auto feature_div = GetDivisor(in.Feature().v); const auto y_div = GetDivisor(in.Y().v); @@ -227,6 +229,10 @@ bool PermuteKernel_f_y_axes::Validate(const Params& p) const { return false; } + if (in_layout != out_layout) { + return false; + } + // Accept only supported blocked layouts and SIMD sizes. if (!SimpleLayout(in_layout)) { const auto feature_block_size = GetFeatureBlockSize(params); diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/permute_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/permute_gpu_test.cpp index 9688ebe227071f..db9777a3a613d9 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/permute_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/permute_gpu_test.cpp @@ -2175,6 +2175,76 @@ TEST(permute_gpu_f32_dynamic, bfyx_0_2_3_1) { } } +TEST(permute_f_y_axes_fallback, b_fs_yx_fsv16) { + constexpr size_t array_size = 128; + + auto& engine = get_test_engine(); + if (!engine.get_device_info().supports_immad) + return; + + auto input_layout_static = layout{ov::PartialShape{1, 8, 16, 1}, data_types::f32, format::bfyx}; + auto input = engine.allocate_memory(input_layout_static); + + std::vector input_data; + input_data.reserve(array_size); + for (size_t i = 0; i < array_size; ++i) + input_data.push_back(static_cast(i)); + + auto weights = engine.allocate_memory({ data_types::f32, format::bfyx, { 8, 16, 1, 1 } }); + + std::vector weights_data; + weights_data.reserve(array_size); + for (size_t i = 0; i < array_size; ++i) + weights_data.push_back(static_cast(1.0)); + + set_values(weights, weights_data); + set_values(input, input_data); + + auto impl_desc_onednn = ov::intel_gpu::ImplementationDesc{format::b_fs_yx_fsv16, "", impl_types::onednn}; + auto impl_forcing_map = ov::intel_gpu::ImplForcingMap{{"conv", impl_desc_onednn}}; + + topology topology; + topology.add(input_layout("input", input_layout_static)); + topology.add(permute("permute", input_info("input"), { 0, 2, 1, 3 })); + topology.add(data("weights", weights)); + topology.add(convolution("conv", input_info("permute"), "weights", "", 1, {1,1}, {1,1}, {0,0}, {0,0}, false)); + + ExecutionConfig config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::force_implementations(impl_forcing_map)); + + network network(engine, topology, config); + network.set_input_data("input", input); + auto outputs = network.execute(); + ASSERT_EQ(outputs.size(), size_t(1)); + ASSERT_EQ(outputs.begin()->first, "conv"); + + auto output = outputs.begin()->second.get_memory(); + + float answers[] = { + 120.f, 120.f, 120.f, 120.f, 120.f, 120.f, 120.f, 120.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 376.f, 376.f, 376.f, 376.f, 376.f, 376.f, 376.f, 376.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 632.f, 632.f, 632.f, 632.f, 632.f, 632.f, 632.f, 632.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 888.f, 888.f, 888.f, 888.f, 888.f, 888.f, 888.f, 888.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 1144.f, 1144.f, 1144.f, 1144.f, 1144.f, 1144.f, 1144.f, 1144.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 1400.f, 1400.f, 1400.f, 1400.f, 1400.f, 1400.f, 1400.f, 1400.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 1656.f, 1656.f, 1656.f, 1656.f, 1656.f, 1656.f, 1656.f, 1656.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 1912.f, 1912.f, 1912.f, 1912.f, 1912.f, 1912.f, 1912.f, 1912.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + }; + + cldnn::mem_lock output_ptr(output, get_test_stream()); + for (size_t i = 0; i < array_size; i++) { + ASSERT_FLOAT_EQ(answers[i], output_ptr[i]); + } +} + class permute_bfzyx_to_bfyxz: public TiledPermuteTest {}; INSTANTIATE_TEST_SUITE_P(, permute_bfzyx_to_bfyxz, From 5e6fb3c157cc0231f7ba6aae4d546da93beaaba2 Mon Sep 17 00:00:00 2001 From: yuanxion <96522341+yuanxion@users.noreply.github.com> Date: Fri, 8 Nov 2024 15:47:57 +0800 Subject: [PATCH 42/93] [GPU] Fix one hot seg fault when the second input node is not a Constant node (#27442) ### Details: - *Get the 'depth' value from 'Select' node instead of a Constant node* This happens when the second input of **OneHot** node is not a **Constant** node (**Select** node in this PR). ![image](https://github.com/user-attachments/assets/1bef5fdd-ae42-4916-9ba1-46b8d397e34b) ### Tickets: - *[CVS-155564](https://jira.devtools.intel.com/browse/CVS-155564)* --------- Signed-off-by: yuan.xiong --- .../include/intel_gpu/primitives/one_hot.hpp | 15 ++++++++ .../src/graph/include/one_hot_inst.h | 2 +- src/plugins/intel_gpu/src/graph/one_hot.cpp | 20 ++++++++-- src/plugins/intel_gpu/src/graph/program.cpp | 4 +- .../intel_gpu/src/plugin/ops/one_hot.cpp | 37 ++++++++++++------- 5 files changed, 58 insertions(+), 20 deletions(-) diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/one_hot.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/one_hot.hpp index c66bbedc80ee0c..d47a128c62c434 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/primitives/one_hot.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/one_hot.hpp @@ -54,6 +54,21 @@ struct one_hot : public primitive_base { , on_value(on_value) , off_value(off_value) {} + /// @brief onehot with depth from Select node + one_hot(const primitive_id& id, + const input_info& input, + const input_info& input_depth, + const tensor& shape, + const data_types output_dt, + const int64_t& one_hot_axis, + const float& on_value = 1.0f, + const float& off_value = 0.0f) + : primitive_base(id, {input, input_depth}, 1, {optional_data_type{output_dt}}) + , shape(shape) + , one_hot_axis(one_hot_axis) + , on_value(on_value) + , off_value(off_value) {} + /// @brief Constructs one-hot primitive layer. /// @param id An identifier of new primitive. /// @param input An identifier of primitive which is an input for newly created one-hot primitive. diff --git a/src/plugins/intel_gpu/src/graph/include/one_hot_inst.h b/src/plugins/intel_gpu/src/graph/include/one_hot_inst.h index 782751e261e51a..2e5dc81a3d4049 100644 --- a/src/plugins/intel_gpu/src/graph/include/one_hot_inst.h +++ b/src/plugins/intel_gpu/src/graph/include/one_hot_inst.h @@ -22,7 +22,7 @@ struct typed_program_node : typed_program_node_base { support_padding_all(true); } program_node& input() const { return get_dependency(0); } - std::vector get_shape_infer_dependencies() const override { return {}; } + std::vector get_shape_infer_dependencies() const override { return {1}; } }; using one_hot_node = typed_program_node; diff --git a/src/plugins/intel_gpu/src/graph/one_hot.cpp b/src/plugins/intel_gpu/src/graph/one_hot.cpp index ede34e8e0935b0..bb90e3bf1276b5 100644 --- a/src/plugins/intel_gpu/src/graph/one_hot.cpp +++ b/src/plugins/intel_gpu/src/graph/one_hot.cpp @@ -66,11 +66,23 @@ std::vector one_hot_inst::calc_output_layouts(const one_hot_node& /*node }; int64_t depth = desc->depth; + auto& memory_deps = impl_param.memory_deps; + + std::unordered_map const_data = {}; + if (depth != 0) { + auto depth_tensor = ov::Tensor(ov::element::i64, ov::Shape{1}, static_cast(&depth)); + const_data[1] = depth_tensor; + } else if (memory_deps.count(1) > 0) { + auto depth_mem = memory_deps.at(1); + + cldnn::mem_lock depth_lock(depth_mem, impl_param.get_stream()); + auto depth_ptr = depth_lock.data(); + + // update depth_tensor if depth value comes from memory_deps instead of Constant node + auto depth_tensor = make_tensor(depth_mem->get_layout(), depth_ptr); + const_data[1] = depth_tensor; + } - auto depth_tensor = ov::Tensor(ov::element::i64, ov::Shape{1}, static_cast(&depth)); - std::unordered_map const_data = { - {1, depth_tensor} - }; std::vector output_shapes = ov::op::v1::shape_infer(&op, input_shapes, ov::make_tensor_accessor(const_data)); return {{output_shapes[0], dt, format::get_default_format(output_shapes[0].size())}}; diff --git a/src/plugins/intel_gpu/src/graph/program.cpp b/src/plugins/intel_gpu/src/graph/program.cpp index 7a66d32795c17c..2bfaac84134387 100644 --- a/src/plugins/intel_gpu/src/graph/program.cpp +++ b/src/plugins/intel_gpu/src/graph/program.cpp @@ -436,7 +436,7 @@ void program::prepare_nodes(topology const& topology) { } } -// add node's dependecies from its primitive dependencies +// add node's dependencies from its primitive dependencies void program::add_node_dependencies(program_node* node) { auto deps = node->get_primitive()->dependencies(); // add pointers to node's dependencies @@ -453,7 +453,7 @@ void program::add_node_dependencies(program_node* node) { } /* helper method for program constructor from list of nodes which - copies src_node dependecies to the destination node dest_node dependencies. + copies src_node dependencies to the destination node dest_node dependencies. But only to those which appaer in this program implementation nodes_map */ void program::copy_node_dependencies(program_node* dest_node, program_node* src_node) { if (dest_node->get_primitive()->id != src_node->get_primitive()->id) { diff --git a/src/plugins/intel_gpu/src/plugin/ops/one_hot.cpp b/src/plugins/intel_gpu/src/plugin/ops/one_hot.cpp index 9a7f8697d0b736..09e9d81db088df 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/one_hot.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/one_hot.cpp @@ -7,7 +7,6 @@ #include "transformations/utils/utils.hpp" #include "openvino/op/one_hot.hpp" - #include "intel_gpu/primitives/one_hot.hpp" namespace ov { @@ -49,21 +48,33 @@ static void CreateOneHotOp(ProgramBuilder& p, const std::shared_ptrcast_vector()[0]; - auto out_pshape = op->get_output_partial_shape(0); cldnn::tensor out_tensor = out_pshape.is_static() ? tensor_from_dims(out_pshape.to_shape()) : cldnn::tensor{}; - auto oneHotPrim = cldnn::one_hot(layerName, - inputs[0], - out_tensor, - cldnn::element_type_to_data_type(op->get_output_element_type(0)), - axis, - depth, - on_value, - off_value); - - p.add_primitive(*op, oneHotPrim); + if (depth_value_node) { + int64_t depth = depth_value_node->cast_vector()[0]; + auto oneHotPrim = cldnn::one_hot(layerName, + inputs[0], + out_tensor, + cldnn::element_type_to_data_type(op->get_output_element_type(0)), + axis, + depth, + on_value, + off_value); + + p.add_primitive(*op, oneHotPrim); + } else { + auto oneHotPrim = cldnn::one_hot(layerName, + inputs[0], + inputs[1], + out_tensor, + cldnn::element_type_to_data_type(op->get_output_element_type(0)), + axis, + on_value, + off_value); + + p.add_primitive(*op, oneHotPrim); + } } REGISTER_FACTORY_IMPL(v1, OneHot); From 9ff42cde8951ff3e70d6f512c032e1ce21bf800b Mon Sep 17 00:00:00 2001 From: Valentin Rusu Date: Fri, 8 Nov 2024 11:12:38 +0200 Subject: [PATCH 43/93] [Benchmark App] Reject -nireq > -niter when using sync API (#27460) ### Details: While investigating the problem raised by the E-117106 ticket, turned out that, in fact, a problem lies within the Benchmark App. When choosing the sync API, it is not possible to run more inference requests than specified iterations. Later on, if `-pc` flag was given, the app still tries to display the profiling data for the never launched inferences. This PR detects this situation and rejects the application execution after writing an explanatory message to the console output. Changes are applied for both C++ and Python versions of the benchmark app. ### Tickets: - CVS-156823 Co-authored-by: Vladimir Zlobin --- samples/cpp/benchmark_app/main.cpp | 4 ++++ tools/benchmark_tool/openvino/tools/benchmark/main.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp index 2cfd15b77afb6e..4dcc1e82924efd 100644 --- a/samples/cpp/benchmark_app/main.cpp +++ b/samples/cpp/benchmark_app/main.cpp @@ -60,6 +60,10 @@ bool parse_and_check_command_line(int argc, char* argv[]) { if (FLAGS_api != "async" && FLAGS_api != "sync") { throw std::logic_error("Incorrect API. Please set -api option to `sync` or `async` value."); } + if (FLAGS_api == "sync" && FLAGS_nireq > FLAGS_niter) { + throw std::logic_error( + "Number of iterations should be greater than number of infer requests when using sync API."); + } if (!FLAGS_hint.empty() && FLAGS_hint != "throughput" && FLAGS_hint != "tput" && FLAGS_hint != "latency" && FLAGS_hint != "cumulative_throughput" && FLAGS_hint != "ctput" && FLAGS_hint != "none") { throw std::logic_error("Incorrect performance hint. Please set -hint option to" diff --git a/tools/benchmark_tool/openvino/tools/benchmark/main.py b/tools/benchmark_tool/openvino/tools/benchmark/main.py index ad2839d217fcad..c77b50a7fd4721 100644 --- a/tools/benchmark_tool/openvino/tools/benchmark/main.py +++ b/tools/benchmark_tool/openvino/tools/benchmark/main.py @@ -49,6 +49,9 @@ def arg_not_empty(arg_value,empty_value): raise Exception("Cannot set precision for a compiled model. " \ "Please re-compile your model with required precision.") + if args.api_type == "sync" and args.number_infer_requests > args.number_iterations: + raise Exception("Number of infer requests should be less than or equal to number of iterations in sync mode.") + return args, is_network_compiled def main(): From bb99d6723ba3e875158ddec63cf4df26affe14ed Mon Sep 17 00:00:00 2001 From: Wilson Seok Date: Fri, 8 Nov 2024 02:43:02 -0800 Subject: [PATCH 44/93] [GPU] Don't select add_fusing_type::sum when dependency is input_layout in loop body network (#27443) ### Details: - Don't select add_fusing_type::sum when dependency is input_layout in loop body network. sum op could contaminate memory buffer. ### Tickets: - 154020 --- src/plugins/intel_gpu/src/graph/program_helpers.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/src/graph/program_helpers.cpp b/src/plugins/intel_gpu/src/graph/program_helpers.cpp index 256ba493311b7c..51e5d8510739e2 100644 --- a/src/plugins/intel_gpu/src/graph/program_helpers.cpp +++ b/src/plugins/intel_gpu/src/graph/program_helpers.cpp @@ -117,7 +117,8 @@ add_fusing_type onednn_add_fusing_helpers::get_add_fusing_type( && p_layout.data_padding == d_layout.data_padding && dep_node.get_users().size() == 1 && !dep_node.is_constant() - && !p_node.is_type()) { + && !p_node.is_type() + && !(dep_node.get_program().is_body_program() && dep_node.is_type())) { return add_fusing_type::sum; } else if (p_layout.get_tensor() == d_layout.get_tensor()) { return add_fusing_type::binary_per_tensor; From 406034de7d38f0d076cb88618499b6bb860f617d Mon Sep 17 00:00:00 2001 From: Karol Blaszczak Date: Fri, 8 Nov 2024 12:03:16 +0100 Subject: [PATCH 45/93] [DOCS] benchmark data tweaks (#27424) Co-authored-by: Andrzej Kopytko --- .../benchmarks_files/data/graph-data-ov.json | 1462 ++++++++--------- .../benchmarks_files/graph-config.json | 10 +- .../_static/download/supported_models.csv | 3 +- docs/sphinx_setup/index.rst | 2 +- 4 files changed, 741 insertions(+), 736 deletions(-) diff --git a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json index 0bb38199b997ca..7576fd96148554 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json +++ b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json @@ -3,7 +3,7 @@ "Platform": "Intel® Celeron® 6305E CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -37,7 +37,7 @@ "Platform": "Intel® Core™ i3-8100 CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -71,7 +71,7 @@ "Platform": "Intel® Core™ i5-10500TE CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -105,7 +105,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -139,7 +139,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -173,7 +173,7 @@ "Platform": "Intel® Core™ i5-13600K CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -207,7 +207,7 @@ "Platform": "Intel® Core™ i5-8500 CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -241,7 +241,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -275,7 +275,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -309,7 +309,7 @@ "Platform": "Intel® Core™ i7-12700H CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -343,7 +343,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -377,7 +377,7 @@ "Platform": "Intel® Core™ i7-1360P CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -411,7 +411,7 @@ "Platform": "Intel® Core™ i7-8700T CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -445,7 +445,7 @@ "Platform": "Intel® Core™ i9-10900TE CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -479,7 +479,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -513,7 +513,7 @@ "Platform": "Intel® Xeon® W1290P CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -547,7 +547,7 @@ "Platform": "Intel® Xeon® E-2124G CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -581,7 +581,7 @@ "Platform": "Intel® Xeon® Gold 5218T CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -615,7 +615,7 @@ "Platform": "Intel® Xeon® Platinum 8280 CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -649,7 +649,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -683,7 +683,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -717,7 +717,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -751,7 +751,7 @@ "Platform": "Intel® Xeon® Gold 6238L CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -785,7 +785,7 @@ "Platform": "Intel® Xeon® Silver 4316 CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -819,7 +819,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -853,7 +853,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -887,7 +887,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1023,7 +1023,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1057,7 +1057,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1091,7 +1091,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1125,7 +1125,7 @@ "Platform": "Intel® Celeron® 6305E iGPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1159,7 +1159,7 @@ "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1193,7 +1193,7 @@ "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1227,7 +1227,7 @@ "Platform": "Intel® Core™ i7-1185G7 iGPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1261,7 +1261,7 @@ "Platform": "Intel® Core™ i7-1185GRE iGPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1295,7 +1295,7 @@ "Platform": "Intel® Core™ i7-12700H iGPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1329,7 +1329,7 @@ "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1363,7 +1363,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1397,7 +1397,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1431,7 +1431,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1465,7 +1465,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1499,7 +1499,7 @@ "Platform": "Intel® Celeron® 6305E CPU+iGPU", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -1533,7 +1533,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1567,7 +1567,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1601,7 +1601,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1635,7 +1635,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1669,7 +1669,7 @@ "Platform": "Intel® Core™ i7-12700H CPU+iGPU", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1703,7 +1703,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1737,7 +1737,7 @@ "Platform": "Intel® Core™ i7-1360P CPU+iGPU", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1771,7 +1771,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1805,7 +1805,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1839,7 +1839,7 @@ "Platform": "Intel® Celeron® 6305E CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1873,7 +1873,7 @@ "Platform": "Intel® Core™ i3-8100 CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1907,7 +1907,7 @@ "Platform": "Intel® Core™ i5-10500TE CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1941,7 +1941,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1975,7 +1975,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2009,7 +2009,7 @@ "Platform": "Intel® Core™ i5-13600K CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2043,7 +2043,7 @@ "Platform": "Intel® Core™ i5-8500 CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2077,7 +2077,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2111,7 +2111,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2145,7 +2145,7 @@ "Platform": "Intel® Core™ i7-12700H CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2179,7 +2179,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2213,7 +2213,7 @@ "Platform": "Intel® Core™ i7-1360P CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2247,7 +2247,7 @@ "Platform": "Intel® Core™ i7-8700T CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2281,7 +2281,7 @@ "Platform": "Intel® Core™ i9-10900TE CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2315,7 +2315,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2349,7 +2349,7 @@ "Platform": "Intel® Xeon® W1290P CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2383,7 +2383,7 @@ "Platform": "Intel® Xeon® E-2124G CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2417,7 +2417,7 @@ "Platform": "Intel® Xeon® Gold 5218T CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2451,7 +2451,7 @@ "Platform": "Intel® Xeon® Platinum 8280 CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2485,7 +2485,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2519,7 +2519,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2553,7 +2553,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2587,7 +2587,7 @@ "Platform": "Intel® Xeon® Gold 6238L CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2621,7 +2621,7 @@ "Platform": "Intel® Xeon® Silver 4316 CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2655,7 +2655,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2689,7 +2689,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2723,7 +2723,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2859,7 +2859,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2893,7 +2893,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2927,7 +2927,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2961,7 +2961,7 @@ "Platform": "Intel® Atom® x7425E iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2995,7 +2995,7 @@ "Platform": "Intel® Atom® X6425E iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3029,7 +3029,7 @@ "Platform": "Intel® Celeron® 6305E iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3063,7 +3063,7 @@ "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3097,7 +3097,7 @@ "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3131,7 +3131,7 @@ "Platform": "Intel® Core™ i7-1185G7 iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3165,7 +3165,7 @@ "Platform": "Intel® Core™ i7-1185GRE iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3199,7 +3199,7 @@ "Platform": "Intel® Core™ i7-12700H iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3233,7 +3233,7 @@ "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3267,7 +3267,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3301,7 +3301,7 @@ "Platform": "Intel® Processor N100 iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3335,7 +3335,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3369,7 +3369,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3403,7 +3403,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3437,7 +3437,7 @@ "Platform": "Intel® Celeron® 6305E CPU+iGPU", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -3471,7 +3471,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3505,7 +3505,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3539,7 +3539,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3573,7 +3573,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3607,7 +3607,7 @@ "Platform": "Intel® Core™ i7-12700H CPU+iGPU", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3641,7 +3641,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3675,7 +3675,7 @@ "Platform": "Intel® Core™ i7-1360P CPU+iGPU", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3709,7 +3709,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3743,7 +3743,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3777,7 +3777,7 @@ "Platform": "Intel® Atom® x7425E CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3811,7 +3811,7 @@ "Platform": "Intel® Atom® X6425E CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3845,7 +3845,7 @@ "Platform": "Intel® Celeron® 6305E CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3879,7 +3879,7 @@ "Platform": "Intel® Core™ i3-8100 CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3913,7 +3913,7 @@ "Platform": "Intel® Core™ i5-10500TE CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3947,7 +3947,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3981,7 +3981,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4015,7 +4015,7 @@ "Platform": "Intel® Core™ i5-13600K CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4049,7 +4049,7 @@ "Platform": "Intel® Core™ i5-8500 CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4083,7 +4083,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4117,7 +4117,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4151,7 +4151,7 @@ "Platform": "Intel® Core™ i7-12700H CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4185,7 +4185,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4219,7 +4219,7 @@ "Platform": "Intel® Core™ i7-1360P CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4253,7 +4253,7 @@ "Platform": "Intel® Core™ i7-8700T CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4287,7 +4287,7 @@ "Platform": "Intel® Core™ i9-10900TE CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4321,7 +4321,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4355,7 +4355,7 @@ "Platform": "Intel® Processor N100 CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4389,7 +4389,7 @@ "Platform": "Intel® Xeon® W1290P CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4423,7 +4423,7 @@ "Platform": "Intel® Xeon® E-2124G CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4457,7 +4457,7 @@ "Platform": "Intel® Xeon® Gold 5218T CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4491,7 +4491,7 @@ "Platform": "Intel® Xeon® Platinum 8280 CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4525,7 +4525,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4559,7 +4559,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4593,7 +4593,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4627,7 +4627,7 @@ "Platform": "Intel® Xeon® Gold 6238L CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4661,7 +4661,7 @@ "Platform": "Intel® Xeon® Silver 4316 CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4695,7 +4695,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4729,7 +4729,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4763,7 +4763,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4899,7 +4899,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4933,7 +4933,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4967,7 +4967,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5001,7 +5001,7 @@ "Platform": "Intel® Atom® x7425E iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5035,7 +5035,7 @@ "Platform": "Intel® Atom® X6425E iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5069,7 +5069,7 @@ "Platform": "Intel® Celeron® 6305E iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5103,7 +5103,7 @@ "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5137,7 +5137,7 @@ "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5171,7 +5171,7 @@ "Platform": "Intel® Core™ i7-1185G7 iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5205,7 +5205,7 @@ "Platform": "Intel® Core™ i7-1185GRE iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5239,7 +5239,7 @@ "Platform": "Intel® Core™ i7-12700H iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5273,7 +5273,7 @@ "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5307,7 +5307,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5341,7 +5341,7 @@ "Platform": "Intel® Processor N100 iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5375,7 +5375,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5409,7 +5409,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5443,7 +5443,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5477,7 +5477,7 @@ "Platform": "Intel® Atom® x7425E CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -5511,7 +5511,7 @@ "Platform": "Intel® Atom® X6425E CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -5545,7 +5545,7 @@ "Platform": "Intel® Celeron® 6305E CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -5579,7 +5579,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5613,7 +5613,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5647,7 +5647,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5681,7 +5681,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5715,7 +5715,7 @@ "Platform": "Intel® Core™ i7-12700H CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5749,7 +5749,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5783,7 +5783,7 @@ "Platform": "Intel® Core™ i7-1360P CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5817,7 +5817,7 @@ "Platform": "Intel® Processor N100 CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -5851,7 +5851,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5885,7 +5885,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5919,7 +5919,7 @@ "Platform": "Intel® Celeron® 6305E CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5953,7 +5953,7 @@ "Platform": "Intel® Core™ i3-8100 CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5987,7 +5987,7 @@ "Platform": "Intel® Core™ i5-10500TE CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6021,7 +6021,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6055,7 +6055,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6089,7 +6089,7 @@ "Platform": "Intel® Core™ i5-13600K CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6123,7 +6123,7 @@ "Platform": "Intel® Core™ i5-8500 CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6157,7 +6157,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6191,7 +6191,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6225,7 +6225,7 @@ "Platform": "Intel® Core™ i7-12700H CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6259,7 +6259,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6293,7 +6293,7 @@ "Platform": "Intel® Core™ i7-1360P CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6327,7 +6327,7 @@ "Platform": "Intel® Core™ i7-8700T CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6361,7 +6361,7 @@ "Platform": "Intel® Core™ i9-10900TE CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6395,7 +6395,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6429,7 +6429,7 @@ "Platform": "Intel® Xeon® W1290P CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6463,7 +6463,7 @@ "Platform": "Intel® Xeon® E-2124G CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6497,7 +6497,7 @@ "Platform": "Intel® Xeon® Gold 5218T CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6531,7 +6531,7 @@ "Platform": "Intel® Xeon® Platinum 8280 CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6565,7 +6565,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6599,7 +6599,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6633,7 +6633,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6667,7 +6667,7 @@ "Platform": "Intel® Xeon® Gold 6238L CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6701,7 +6701,7 @@ "Platform": "Intel® Xeon® Silver 4316 CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6735,7 +6735,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6769,7 +6769,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6803,7 +6803,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6939,7 +6939,7 @@ "Platform": "Intel® Celeron® 6305E iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6973,7 +6973,7 @@ "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7007,7 +7007,7 @@ "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7041,7 +7041,7 @@ "Platform": "Intel® Core™ i7-1185G7 iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7075,7 +7075,7 @@ "Platform": "Intel® Core™ i7-1185GRE iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7109,7 +7109,7 @@ "Platform": "Intel® Core™ i7-12700H iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7143,7 +7143,7 @@ "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7177,7 +7177,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7211,7 +7211,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7245,7 +7245,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7279,7 +7279,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7313,7 +7313,7 @@ "Platform": "Intel® Celeron® 6305E CPU+iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -7347,7 +7347,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7381,7 +7381,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7415,7 +7415,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7449,7 +7449,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7483,7 +7483,7 @@ "Platform": "Intel® Core™ i7-12700H CPU+iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7517,7 +7517,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7551,7 +7551,7 @@ "Platform": "Intel® Core™ i7-1360P CPU+iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7585,7 +7585,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7619,7 +7619,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7653,7 +7653,7 @@ "Platform": "Intel® Atom® x7425E CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7687,7 +7687,7 @@ "Platform": "Intel® Atom® X6425E CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7721,7 +7721,7 @@ "Platform": "Intel® Celeron® 6305E CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7755,7 +7755,7 @@ "Platform": "Intel® Core™ i3-8100 CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7789,7 +7789,7 @@ "Platform": "Intel® Core™ i5-10500TE CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7823,7 +7823,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7857,7 +7857,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7891,7 +7891,7 @@ "Platform": "Intel® Core™ i5-13600K CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7925,7 +7925,7 @@ "Platform": "Intel® Core™ i5-8500 CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7959,7 +7959,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7993,7 +7993,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8027,7 +8027,7 @@ "Platform": "Intel® Core™ i7-12700H CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8061,7 +8061,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8095,7 +8095,7 @@ "Platform": "Intel® Core™ i7-1360P CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8129,7 +8129,7 @@ "Platform": "Intel® Core™ i7-8700T CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8163,7 +8163,7 @@ "Platform": "Intel® Core™ i9-10900TE CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8197,7 +8197,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8231,7 +8231,7 @@ "Platform": "Intel® Processor N100 CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8265,7 +8265,7 @@ "Platform": "Intel® Xeon® W1290P CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8299,7 +8299,7 @@ "Platform": "Intel® Xeon® E-2124G CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8333,7 +8333,7 @@ "Platform": "Intel® Xeon® Gold 5218T CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8367,7 +8367,7 @@ "Platform": "Intel® Xeon® Platinum 8280 CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8401,7 +8401,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8435,7 +8435,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8469,7 +8469,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8503,7 +8503,7 @@ "Platform": "Intel® Xeon® Gold 6238L CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8537,7 +8537,7 @@ "Platform": "Intel® Xeon® Silver 4316 CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8571,7 +8571,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8605,7 +8605,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8639,7 +8639,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8775,7 +8775,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8809,7 +8809,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8843,7 +8843,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8877,7 +8877,7 @@ "Platform": "Intel® Atom® x7425E iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8911,7 +8911,7 @@ "Platform": "Intel® Atom® X6425E iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8945,7 +8945,7 @@ "Platform": "Intel® Celeron® 6305E iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8979,7 +8979,7 @@ "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9013,7 +9013,7 @@ "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9047,7 +9047,7 @@ "Platform": "Intel® Core™ i7-1185G7 iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9081,7 +9081,7 @@ "Platform": "Intel® Core™ i7-1185GRE iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9115,7 +9115,7 @@ "Platform": "Intel® Core™ i7-12700H iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9149,7 +9149,7 @@ "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9183,7 +9183,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9217,7 +9217,7 @@ "Platform": "Intel® Processor N100 iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9251,7 +9251,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9285,7 +9285,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9319,7 +9319,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9353,7 +9353,7 @@ "Platform": "Intel® Atom® x7425E CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -9387,7 +9387,7 @@ "Platform": "Intel® Atom® X6425E CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -9421,7 +9421,7 @@ "Platform": "Intel® Celeron® 6305E CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -9455,7 +9455,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9489,7 +9489,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9523,7 +9523,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9557,7 +9557,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9591,7 +9591,7 @@ "Platform": "Intel® Core™ i7-12700H CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9625,7 +9625,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9659,7 +9659,7 @@ "Platform": "Intel® Core™ i7-1360P CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9693,7 +9693,7 @@ "Platform": "Intel® Processor N100 CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -9727,7 +9727,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9761,7 +9761,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9795,7 +9795,7 @@ "Platform": "Intel® Atom® x7425E CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9829,7 +9829,7 @@ "Platform": "Intel® Atom® X6425E CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9863,7 +9863,7 @@ "Platform": "Intel® Celeron® 6305E CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9897,7 +9897,7 @@ "Platform": "Intel® Core™ i3-8100 CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9931,7 +9931,7 @@ "Platform": "Intel® Core™ i5-10500TE CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9965,7 +9965,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9999,7 +9999,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10033,7 +10033,7 @@ "Platform": "Intel® Core™ i5-13600K CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10067,7 +10067,7 @@ "Platform": "Intel® Core™ i5-8500 CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10101,7 +10101,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10135,7 +10135,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10169,7 +10169,7 @@ "Platform": "Intel® Core™ i7-12700H CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10203,7 +10203,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10237,7 +10237,7 @@ "Platform": "Intel® Core™ i7-1360P CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10271,7 +10271,7 @@ "Platform": "Intel® Core™ i7-8700T CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10305,7 +10305,7 @@ "Platform": "Intel® Core™ i9-10900TE CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10339,7 +10339,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10373,7 +10373,7 @@ "Platform": "Intel® Processor N100 CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10407,7 +10407,7 @@ "Platform": "Intel® Xeon® W1290P CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10441,7 +10441,7 @@ "Platform": "Intel® Xeon® E-2124G CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10475,7 +10475,7 @@ "Platform": "Intel® Xeon® Gold 5218T CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10509,7 +10509,7 @@ "Platform": "Intel® Xeon® Platinum 8280 CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10543,7 +10543,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10577,7 +10577,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10611,7 +10611,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10645,7 +10645,7 @@ "Platform": "Intel® Xeon® Gold 6238L CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10679,7 +10679,7 @@ "Platform": "Intel® Xeon® Silver 4316 CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10713,7 +10713,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10747,7 +10747,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10781,7 +10781,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10917,7 +10917,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10951,7 +10951,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10985,7 +10985,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11019,7 +11019,7 @@ "Platform": "Intel® Atom® x7425E iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11053,7 +11053,7 @@ "Platform": "Intel® Atom® X6425E iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11087,7 +11087,7 @@ "Platform": "Intel® Celeron® 6305E iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11121,7 +11121,7 @@ "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11155,7 +11155,7 @@ "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11189,7 +11189,7 @@ "Platform": "Intel® Core™ i7-1185G7 iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11223,7 +11223,7 @@ "Platform": "Intel® Core™ i7-1185GRE iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11257,7 +11257,7 @@ "Platform": "Intel® Core™ i7-12700H iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11291,7 +11291,7 @@ "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11325,7 +11325,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11359,7 +11359,7 @@ "Platform": "Intel® Processor N100 iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11393,7 +11393,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11427,7 +11427,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11461,7 +11461,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11495,7 +11495,7 @@ "Platform": "Intel® Atom® x7425E CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -11529,7 +11529,7 @@ "Platform": "Intel® Atom® X6425E CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -11563,7 +11563,7 @@ "Platform": "Intel® Celeron® 6305E CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -11597,7 +11597,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11631,7 +11631,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11665,7 +11665,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11699,7 +11699,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11733,7 +11733,7 @@ "Platform": "Intel® Core™ i7-12700H CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11767,7 +11767,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11801,7 +11801,7 @@ "Platform": "Intel® Core™ i7-1360P CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11835,7 +11835,7 @@ "Platform": "Intel® Processor N100 CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -11869,7 +11869,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11903,7 +11903,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11937,7 +11937,7 @@ "Platform": "Intel® Atom® x7425E CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11971,7 +11971,7 @@ "Platform": "Intel® Atom® X6425E CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12005,7 +12005,7 @@ "Platform": "Intel® Celeron® 6305E CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12039,7 +12039,7 @@ "Platform": "Intel® Core™ i3-8100 CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12073,7 +12073,7 @@ "Platform": "Intel® Core™ i5-10500TE CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12107,7 +12107,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12141,7 +12141,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12175,7 +12175,7 @@ "Platform": "Intel® Core™ i5-13600K CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12209,7 +12209,7 @@ "Platform": "Intel® Core™ i5-8500 CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12243,7 +12243,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12277,7 +12277,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12311,7 +12311,7 @@ "Platform": "Intel® Core™ i7-12700H CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12345,7 +12345,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12379,7 +12379,7 @@ "Platform": "Intel® Core™ i7-1360P CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12413,7 +12413,7 @@ "Platform": "Intel® Core™ i7-8700T CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12447,7 +12447,7 @@ "Platform": "Intel® Core™ i9-10900TE CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12481,7 +12481,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12515,7 +12515,7 @@ "Platform": "Intel® Processor N100 CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12549,7 +12549,7 @@ "Platform": "Intel® Xeon® W1290P CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12583,7 +12583,7 @@ "Platform": "Intel® Xeon® E-2124G CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12617,7 +12617,7 @@ "Platform": "Intel® Xeon® Gold 5218T CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12651,7 +12651,7 @@ "Platform": "Intel® Xeon® Platinum 8280 CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12685,7 +12685,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12719,7 +12719,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12753,7 +12753,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12787,7 +12787,7 @@ "Platform": "Intel® Xeon® Gold 6238L CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12821,7 +12821,7 @@ "Platform": "Intel® Xeon® Silver 4316 CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12855,7 +12855,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12889,7 +12889,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12923,7 +12923,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13059,7 +13059,7 @@ "Platform": "Intel® Atom® x7425E iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13093,7 +13093,7 @@ "Platform": "Intel® Atom® X6425E iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13127,7 +13127,7 @@ "Platform": "Intel® Celeron® 6305E iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13161,7 +13161,7 @@ "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13195,7 +13195,7 @@ "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13229,7 +13229,7 @@ "Platform": "Intel® Core™ i7-1185G7 iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13263,7 +13263,7 @@ "Platform": "Intel® Core™ i7-1185GRE iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13297,7 +13297,7 @@ "Platform": "Intel® Core™ i7-12700H iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13331,7 +13331,7 @@ "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13365,7 +13365,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13399,7 +13399,7 @@ "Platform": "Intel® Processor N100 iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13433,7 +13433,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13467,7 +13467,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13501,7 +13501,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13535,7 +13535,7 @@ "Platform": "Intel® Atom® x7425E CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -13569,7 +13569,7 @@ "Platform": "Intel® Atom® X6425E CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -13603,7 +13603,7 @@ "Platform": "Intel® Celeron® 6305E CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -13637,7 +13637,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13671,7 +13671,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13705,7 +13705,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13739,7 +13739,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13773,7 +13773,7 @@ "Platform": "Intel® Core™ i7-12700H CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13807,7 +13807,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13841,7 +13841,7 @@ "Platform": "Intel® Core™ i7-1360P CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13875,7 +13875,7 @@ "Platform": "Intel® Processor N100 CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -13909,7 +13909,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13943,7 +13943,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13977,7 +13977,7 @@ "Platform": "Intel® Atom® x7425E CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14011,7 +14011,7 @@ "Platform": "Intel® Atom® X6425E CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14045,7 +14045,7 @@ "Platform": "Intel® Celeron® 6305E CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14079,7 +14079,7 @@ "Platform": "Intel® Core™ i3-8100 CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14113,7 +14113,7 @@ "Platform": "Intel® Core™ i5-10500TE CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14147,7 +14147,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14181,7 +14181,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14215,7 +14215,7 @@ "Platform": "Intel® Core™ i5-13600K CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14249,7 +14249,7 @@ "Platform": "Intel® Core™ i5-8500 CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14283,7 +14283,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14317,7 +14317,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14351,7 +14351,7 @@ "Platform": "Intel® Core™ i7-12700H CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14385,7 +14385,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14419,7 +14419,7 @@ "Platform": "Intel® Core™ i7-1360P CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14453,7 +14453,7 @@ "Platform": "Intel® Core™ i7-8700T CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14487,7 +14487,7 @@ "Platform": "Intel® Core™ i9-10900TE CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14521,7 +14521,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14555,7 +14555,7 @@ "Platform": "Intel® Processor N100 CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14589,7 +14589,7 @@ "Platform": "Intel® Xeon® W1290P CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14623,7 +14623,7 @@ "Platform": "Intel® Xeon® E-2124G CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14657,7 +14657,7 @@ "Platform": "Intel® Xeon® Gold 5218T CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14691,7 +14691,7 @@ "Platform": "Intel® Xeon® Platinum 8280 CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14725,7 +14725,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14759,7 +14759,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14793,7 +14793,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14827,7 +14827,7 @@ "Platform": "Intel® Xeon® Gold 6238L CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14861,7 +14861,7 @@ "Platform": "Intel® Xeon® Silver 4316 CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14895,7 +14895,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14929,7 +14929,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14963,7 +14963,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15099,7 +15099,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15133,7 +15133,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15167,7 +15167,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15201,7 +15201,7 @@ "Platform": "Intel® Atom® x7425E iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15235,7 +15235,7 @@ "Platform": "Intel® Atom® X6425E iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15269,7 +15269,7 @@ "Platform": "Intel® Celeron® 6305E iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15303,7 +15303,7 @@ "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15337,7 +15337,7 @@ "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15371,7 +15371,7 @@ "Platform": "Intel® Core™ i7-1185G7 iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15405,7 +15405,7 @@ "Platform": "Intel® Core™ i7-1185GRE iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15439,7 +15439,7 @@ "Platform": "Intel® Core™ i7-12700H iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15473,7 +15473,7 @@ "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15507,7 +15507,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15541,7 +15541,7 @@ "Platform": "Intel® Processor N100 iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15575,7 +15575,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15609,7 +15609,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15643,7 +15643,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15677,7 +15677,7 @@ "Platform": "Intel® Atom® x7425E CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -15711,7 +15711,7 @@ "Platform": "Intel® Atom® X6425E CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -15745,7 +15745,7 @@ "Platform": "Intel® Celeron® 6305E CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -15779,7 +15779,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15813,7 +15813,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15847,7 +15847,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15881,7 +15881,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15915,7 +15915,7 @@ "Platform": "Intel® Core™ i7-12700H CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15949,7 +15949,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15983,7 +15983,7 @@ "Platform": "Intel® Core™ i7-1360P CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16017,7 +16017,7 @@ "Platform": "Intel® Processor N100 CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -16051,7 +16051,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16085,7 +16085,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16119,7 +16119,7 @@ "Platform": "Intel® Celeron® 6305E CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16153,7 +16153,7 @@ "Platform": "Intel® Core™ i3-8100 CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16187,7 +16187,7 @@ "Platform": "Intel® Core™ i5-10500TE CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16221,7 +16221,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16255,7 +16255,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16289,7 +16289,7 @@ "Platform": "Intel® Core™ i5-13600K CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16323,7 +16323,7 @@ "Platform": "Intel® Core™ i5-8500 CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16357,7 +16357,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16391,7 +16391,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16425,7 +16425,7 @@ "Platform": "Intel® Core™ i7-12700H CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16459,7 +16459,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16493,7 +16493,7 @@ "Platform": "Intel® Core™ i7-1360P CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16527,7 +16527,7 @@ "Platform": "Intel® Core™ i7-8700T CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16561,7 +16561,7 @@ "Platform": "Intel® Core™ i9-10900TE CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16595,7 +16595,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16629,7 +16629,7 @@ "Platform": "Intel® Xeon® W1290P CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16663,7 +16663,7 @@ "Platform": "Intel® Xeon® E-2124G CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16697,7 +16697,7 @@ "Platform": "Intel® Xeon® Gold 5218T CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16731,7 +16731,7 @@ "Platform": "Intel® Xeon® Platinum 8280 CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16765,7 +16765,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16799,7 +16799,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16833,7 +16833,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16867,7 +16867,7 @@ "Platform": "Intel® Xeon® Gold 6238L CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16901,7 +16901,7 @@ "Platform": "Intel® Xeon® Silver 4316 CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16935,7 +16935,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16969,7 +16969,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17003,7 +17003,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17139,7 +17139,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17173,7 +17173,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17207,7 +17207,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17241,7 +17241,7 @@ "Platform": "Intel® Atom® x7425E iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17275,7 +17275,7 @@ "Platform": "Intel® Atom® X6425E iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17309,7 +17309,7 @@ "Platform": "Intel® Celeron® 6305E iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17343,7 +17343,7 @@ "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17377,7 +17377,7 @@ "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17411,7 +17411,7 @@ "Platform": "Intel® Core™ i7-1185G7 iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17445,7 +17445,7 @@ "Platform": "Intel® Core™ i7-1185GRE iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17479,7 +17479,7 @@ "Platform": "Intel® Core™ i7-12700H iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17513,7 +17513,7 @@ "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17547,7 +17547,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17581,7 +17581,7 @@ "Platform": "Intel® Processor N100 iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17615,7 +17615,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17649,7 +17649,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17683,7 +17683,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17717,7 +17717,7 @@ "Platform": "Intel® Celeron® 6305E CPU+iGPU", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -17751,7 +17751,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17785,7 +17785,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17819,7 +17819,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17853,7 +17853,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17887,7 +17887,7 @@ "Platform": "Intel® Core™ i7-12700H CPU+iGPU", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17921,7 +17921,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17955,7 +17955,7 @@ "Platform": "Intel® Core™ i7-1360P CPU+iGPU", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17989,7 +17989,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18023,7 +18023,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18057,7 +18057,7 @@ "Platform": "Intel® Atom® x7425E CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18091,7 +18091,7 @@ "Platform": "Intel® Atom® X6425E CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18125,7 +18125,7 @@ "Platform": "Intel® Celeron® 6305E CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18159,7 +18159,7 @@ "Platform": "Intel® Core™ i3-8100 CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18193,7 +18193,7 @@ "Platform": "Intel® Core™ i5-10500TE CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18227,7 +18227,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18261,7 +18261,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18295,7 +18295,7 @@ "Platform": "Intel® Core™ i5-13600K CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18329,7 +18329,7 @@ "Platform": "Intel® Core™ i5-8500 CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18363,7 +18363,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18397,7 +18397,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18431,7 +18431,7 @@ "Platform": "Intel® Core™ i7-12700H CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18465,7 +18465,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18499,7 +18499,7 @@ "Platform": "Intel® Core™ i7-1360P CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18533,7 +18533,7 @@ "Platform": "Intel® Core™ i7-8700T CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18567,7 +18567,7 @@ "Platform": "Intel® Core™ i9-10900TE CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18601,7 +18601,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18635,7 +18635,7 @@ "Platform": "Intel® Processor N100 CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18669,7 +18669,7 @@ "Platform": "Intel® Xeon® W1290P CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18703,7 +18703,7 @@ "Platform": "Intel® Xeon® E-2124G CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18737,7 +18737,7 @@ "Platform": "Intel® Xeon® Gold 5218T CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18771,7 +18771,7 @@ "Platform": "Intel® Xeon® Platinum 8280 CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18805,7 +18805,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18839,7 +18839,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18873,7 +18873,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18907,7 +18907,7 @@ "Platform": "Intel® Xeon® Gold 6238L CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18941,7 +18941,7 @@ "Platform": "Intel® Xeon® Silver 4316 CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18975,7 +18975,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19009,7 +19009,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19145,7 +19145,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19179,7 +19179,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19213,7 +19213,7 @@ "Platform": "Intel® Atom® x7425E iGPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19247,7 +19247,7 @@ "Platform": "Intel® Atom® X6425E iGPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19281,7 +19281,7 @@ "Platform": "Intel® Celeron® 6305E iGPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19315,7 +19315,7 @@ "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19349,7 +19349,7 @@ "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19383,7 +19383,7 @@ "Platform": "Intel® Core™ i7-1185G7 iGPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19417,7 +19417,7 @@ "Platform": "Intel® Core™ i7-1185GRE iGPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19451,7 +19451,7 @@ "Platform": "Intel® Core™ i7-12700H iGPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19485,7 +19485,7 @@ "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19519,7 +19519,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19553,7 +19553,7 @@ "Platform": "Intel® Processor N100 iGPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19587,7 +19587,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19621,7 +19621,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19655,7 +19655,7 @@ "Platform": "Intel® Atom® x7425E CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -19689,7 +19689,7 @@ "Platform": "Intel® Atom® X6425E CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -19723,7 +19723,7 @@ "Platform": "Intel® Celeron® 6305E CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -19757,7 +19757,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19791,7 +19791,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19825,7 +19825,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19859,7 +19859,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19893,7 +19893,7 @@ "Platform": "Intel® Core™ i7-12700H CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19927,7 +19927,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19961,7 +19961,7 @@ "Platform": "Intel® Core™ i7-1360P CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -19995,7 +19995,7 @@ "Platform": "Intel® Processor N100 CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -20029,7 +20029,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20063,7 +20063,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20097,7 +20097,7 @@ "Platform": "Intel® Atom® x7425E CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20131,7 +20131,7 @@ "Platform": "Intel® Atom® X6425E CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20165,7 +20165,7 @@ "Platform": "Intel® Celeron® 6305E CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20199,7 +20199,7 @@ "Platform": "Intel® Core™ i3-8100 CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20233,7 +20233,7 @@ "Platform": "Intel® Core™ i5-10500TE CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20267,7 +20267,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20301,7 +20301,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20335,7 +20335,7 @@ "Platform": "Intel® Core™ i5-13600K CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20369,7 +20369,7 @@ "Platform": "Intel® Core™ i5-8500 CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20403,7 +20403,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20437,7 +20437,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20471,7 +20471,7 @@ "Platform": "Intel® Core™ i7-12700H CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20505,7 +20505,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20539,7 +20539,7 @@ "Platform": "Intel® Core™ i7-1360P CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20573,7 +20573,7 @@ "Platform": "Intel® Core™ i7-8700T CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20607,7 +20607,7 @@ "Platform": "Intel® Core™ i9-10900TE CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20641,7 +20641,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20675,7 +20675,7 @@ "Platform": "Intel® Processor N100 CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20709,7 +20709,7 @@ "Platform": "Intel® Xeon® W1290P CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20743,7 +20743,7 @@ "Platform": "Intel® Xeon® E-2124G CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20777,7 +20777,7 @@ "Platform": "Intel® Xeon® Gold 5218T CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20811,7 +20811,7 @@ "Platform": "Intel® Xeon® Platinum 8280 CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20845,7 +20845,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20879,7 +20879,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20913,7 +20913,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20947,7 +20947,7 @@ "Platform": "Intel® Xeon® Gold 6238L CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -20981,7 +20981,7 @@ "Platform": "Intel® Xeon® Silver 4316 CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21015,7 +21015,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21049,7 +21049,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21083,7 +21083,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21219,7 +21219,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21253,7 +21253,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21287,7 +21287,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21321,7 +21321,7 @@ "Platform": "Intel® Atom® x7425E iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21355,7 +21355,7 @@ "Platform": "Intel® Atom® X6425E iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21389,7 +21389,7 @@ "Platform": "Intel® Celeron® 6305E iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21423,7 +21423,7 @@ "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21457,7 +21457,7 @@ "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21491,7 +21491,7 @@ "Platform": "Intel® Core™ i7-1185G7 iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21525,7 +21525,7 @@ "Platform": "Intel® Core™ i7-1185GRE iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21559,7 +21559,7 @@ "Platform": "Intel® Core™ i7-12700H iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21593,7 +21593,7 @@ "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21627,7 +21627,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21661,7 +21661,7 @@ "Platform": "Intel® Processor N100 iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21695,7 +21695,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21729,7 +21729,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21763,7 +21763,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21797,7 +21797,7 @@ "Platform": "Intel® Atom® x7425E CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -21831,7 +21831,7 @@ "Platform": "Intel® Atom® X6425E CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -21865,7 +21865,7 @@ "Platform": "Intel® Celeron® 6305E CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -21899,7 +21899,7 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21933,7 +21933,7 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -21967,7 +21967,7 @@ "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22001,7 +22001,7 @@ "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22035,7 +22035,7 @@ "Platform": "Intel® Core™ i7-12700H CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22069,7 +22069,7 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22103,7 +22103,7 @@ "Platform": "Intel® Core™ i7-1360P CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22137,7 +22137,7 @@ "Platform": "Intel® Processor N100 CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -22171,7 +22171,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22205,7 +22205,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22239,7 +22239,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "chatglm2-6b", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22273,7 +22273,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "chatglm2-6b", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22307,7 +22307,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "chatglm2-6b", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22341,7 +22341,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "chatglm2-6b", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22375,7 +22375,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "chatglm2-6b", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22511,7 +22511,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "chatglm2-6b", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22545,7 +22545,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "chatglm2-6b", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22579,7 +22579,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "chatglm2-6b", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22613,7 +22613,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "falcon-7b-instruct", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22647,7 +22647,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "falcon-7b-instruct", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22681,7 +22681,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "falcon-7b-instruct", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22715,7 +22715,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "falcon-7b-instruct", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22749,7 +22749,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "falcon-7b-instruct", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22885,7 +22885,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "falcon-7b-instruct", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22919,7 +22919,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "falcon-7b-instruct", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22953,7 +22953,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "falcon-7b-instruct", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -22987,7 +22987,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "llama-2-7b-chat", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23021,7 +23021,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "llama-2-7b-chat", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23055,7 +23055,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "llama-2-7b-chat", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23089,7 +23089,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "llama-2-7b-chat", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23123,7 +23123,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "llama-2-7b-chat", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23225,7 +23225,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "llama-2-7b-chat", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23259,7 +23259,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "llama-2-7b-chat", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23293,7 +23293,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "llama-2-7b-chat", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23327,7 +23327,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "llama-3-8b", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23361,7 +23361,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "llama-3-8b", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23395,7 +23395,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "llama-3-8b", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23429,7 +23429,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "llama-3-8b", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23463,7 +23463,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "llama-3-8b", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23599,7 +23599,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "llama-3-8b", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23633,7 +23633,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "llama-3-8b", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23667,7 +23667,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "llama-3-8b", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23701,7 +23701,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "mistral-7b-v0.1", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23735,7 +23735,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "mistral-7b-v0.1", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23769,7 +23769,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "mistral-7b-v0.1", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23803,7 +23803,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "mistral-7b-v0.1", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23837,7 +23837,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "mistral-7b-v0.1", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -23973,7 +23973,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "mistral-7b-v0.1", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24007,7 +24007,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "mistral-7b-v0.1", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24041,7 +24041,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "mistral-7b-v0.1", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24075,7 +24075,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "phi-3-mini-4k-instruct", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24109,7 +24109,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "phi-3-mini-4k-instruct", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24143,7 +24143,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "phi-3-mini-4k-instruct", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24177,7 +24177,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "phi-3-mini-4k-instruct", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24211,7 +24211,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", "Model": "phi-3-mini-4k-instruct", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24347,7 +24347,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "phi-3-mini-4k-instruct", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24381,7 +24381,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", "Model": "phi-3-mini-4k-instruct", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24415,7 +24415,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "phi-3-mini-4k-instruct", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24449,7 +24449,7 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "stable-diffusion-v1-5", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24483,7 +24483,7 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "stable-diffusion-v1-5", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24517,7 +24517,7 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "stable-diffusion-v1-5", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24551,7 +24551,7 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "stable-diffusion-v1-5", "Checked": "true", - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24653,7 +24653,7 @@ "Platform": "Intel® Core™ i7-1360P iGPU-only", "Model": "stable-diffusion-v1-5", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24687,7 +24687,7 @@ "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", "Model": "stable-diffusion-v1-5", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24721,7 +24721,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24753,9 +24753,9 @@ }, { "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", - "Model": "bert-large-uncased", + "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24789,7 +24789,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24823,7 +24823,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24857,7 +24857,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24891,7 +24891,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24925,7 +24925,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24959,7 +24959,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -24993,7 +24993,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25027,7 +25027,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25061,7 +25061,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25095,7 +25095,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25127,9 +25127,9 @@ }, { "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "bert-large-uncased", + "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25163,7 +25163,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25197,7 +25197,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25231,7 +25231,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25265,7 +25265,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25299,7 +25299,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25333,7 +25333,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25367,7 +25367,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25401,7 +25401,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25435,7 +25435,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25469,7 +25469,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25501,9 +25501,9 @@ }, { "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", - "Model": "bert-large-uncased", + "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25537,7 +25537,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25571,7 +25571,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25605,7 +25605,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25639,7 +25639,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25673,7 +25673,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25707,7 +25707,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25741,7 +25741,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25775,7 +25775,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25809,7 +25809,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25843,7 +25843,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", "Model": "bert-base-cased", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25875,9 +25875,9 @@ }, { "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", - "Model": "bert-large-uncased", + "Model": "bert-large-uncased-whole-word-masking-squad-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25911,7 +25911,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", "Model": "efficientdet-d0", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25945,7 +25945,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", "Model": "mask_rcnn_resnet50_atrous_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -25979,7 +25979,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", "Model": "mobilenet-v2", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -26013,7 +26013,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", "Model": "resnet-50", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -26047,7 +26047,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", "Model": "ssd_mobilenet_v1_coco", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -26081,7 +26081,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", "Model": "ssd-resnet34-1200", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -26115,7 +26115,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", "Model": "unet-camvid-onnx-0001", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -26149,7 +26149,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", "Model": "yolo_v5m", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -26183,7 +26183,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", "Model": "yolo_v8n", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -26217,7 +26217,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "chatglm2-6b", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -26251,7 +26251,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "falcon-7b-instruct", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -26283,9 +26283,9 @@ }, { "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "llama-2-7b-chat-hf", + "Model": "llama-2-7b-chat", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -26319,7 +26319,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "llama-3-8b", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -26353,7 +26353,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "mistral-7b-v0.1", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -26387,7 +26387,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "phi-3-mini-4k-instruct", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -26421,7 +26421,7 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", "Model": "stable-diffusion-v1-5", "Checked": "true", - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ diff --git a/docs/sphinx_setup/_static/benchmarks_files/graph-config.json b/docs/sphinx_setup/_static/benchmarks_files/graph-config.json index 6fb8d19e1a1adf..9cd8176af26235 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/graph-config.json +++ b/docs/sphinx_setup/_static/benchmarks_files/graph-config.json @@ -80,7 +80,10 @@ "platformTypes": { "name": "ietype", "data": [ - "Client Platforms (Intel® Core™)" + "Intel® Core™, CPU-only", + "Intel® Core™, iGPU-only", + "Intel® Core™, NPU-only", + "Intel® Core™, CPU+iGPU" ] }, "platforms": { @@ -97,7 +100,10 @@ "models": { "name": "networkmodel", "data": [ - "bert-base-cased" + "bert-base-cased", + "resnet-50", + "yolo_v8n", + "llama-3-8B" ] }, "parameters": { diff --git a/docs/sphinx_setup/_static/download/supported_models.csv b/docs/sphinx_setup/_static/download/supported_models.csv index 1c7db3ab6bfc9d..87ea37b0f207c3 100644 --- a/docs/sphinx_setup/_static/download/supported_models.csv +++ b/docs/sphinx_setup/_static/download/supported_models.csv @@ -344,8 +344,7 @@ mask_rcnn_resnet101_atrous_coco,Instance Segmentation,tf,FP16-INT8,+,+, mask_rcnn_resnet50_atrous_coco,Instance Segmentation,tf,FP16,+,+, mask_rcnn_resnet50_atrous_coco,Instance Segmentation,tf,FP16-INT8,+,+, mask_rcnn_resnet50_atrous_coco,Instance Segmentation,tf,FP32,+,+, -MaskRCNN-12,Object Detection,onnx,FP16,,,+ -MaskRCNN-12,Object Detection,onnx,FP32,+,+,+ +MaskRCNN-12,Object Detection,onnx,FP32,+,+, mbart-large-50-many-to-one-mmt,Natural Language Processing,pytorch,intel-optimum default,,+, Meta-Llama-3-8B,Large Language Model,pytorch,intel-optimum default,,+, Meta-Llama-3-8B-Instruct,Large Language Model,pytorch,intel-optimum default,,+, diff --git a/docs/sphinx_setup/index.rst b/docs/sphinx_setup/index.rst index 47dc76bf16cf29..9d376877b51d08 100644 --- a/docs/sphinx_setup/index.rst +++ b/docs/sphinx_setup/index.rst @@ -124,7 +124,7 @@ Places to Begin Cloud-ready deployments for microservice applications. - .. button-link:: openvino-workflow/openvino-workflow/model-server/ovms_what_is_openvino_model_server.html + .. button-link:: openvino-workflow/model-server/ovms_what_is_openvino_model_server.html :color: primary :outline: From 704de6e74443f28b53f4ca755240fd425ab2830a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Krzemi=C5=84ski?= Date: Fri, 8 Nov 2024 12:03:59 +0100 Subject: [PATCH 46/93] [Opset] Opset16 initialization (#27458) ### Details: - Initialization of OV Opset16 ### Tickets: - 156876, Finalization 156877 --- docs/sphinx_setup/api/ie_python_api/api.rst | 6 ++++++ .../src/openvino/runtime/opset16/__init__.py | 8 ++++++++ .../python/src/openvino/runtime/opset16/ops.py | 12 ++++++++++++ .../test_transformations/test_pattern_ops.py | 2 +- src/core/include/openvino/opsets/opset.hpp | 5 +++++ src/core/include/openvino/opsets/opset16.hpp | 15 +++++++++++++++ src/core/include/openvino/opsets/opset16_tbl.hpp | 16 ++++++++++++++++ src/core/src/opsets/opset.cpp | 14 +++++++++++++- src/core/tests/op.cpp | 1 + src/core/tests/opset.cpp | 4 +++- .../include/openvino/frontend/extension/op.hpp | 2 +- src/plugins/template/src/plugin.cpp | 1 + .../src/op_impl_check/single_op_graph.cpp | 1 + 13 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 src/bindings/python/src/openvino/runtime/opset16/__init__.py create mode 100644 src/bindings/python/src/openvino/runtime/opset16/ops.py create mode 100644 src/core/include/openvino/opsets/opset16.hpp create mode 100644 src/core/include/openvino/opsets/opset16_tbl.hpp diff --git a/docs/sphinx_setup/api/ie_python_api/api.rst b/docs/sphinx_setup/api/ie_python_api/api.rst index efbe25ee40bcde..6fc754da1246b7 100644 --- a/docs/sphinx_setup/api/ie_python_api/api.rst +++ b/docs/sphinx_setup/api/ie_python_api/api.rst @@ -119,6 +119,12 @@ OpenVINO Python API openvino.runtime.opset15 +.. autosummary:: + :toctree: _autosummary + :template: custom-module-template.rst + + openvino.runtime.opset16 + .. autosummary:: :toctree: _autosummary :template: custom-module-template.rst diff --git a/src/bindings/python/src/openvino/runtime/opset16/__init__.py b/src/bindings/python/src/openvino/runtime/opset16/__init__.py new file mode 100644 index 00000000000000..bf155b81312090 --- /dev/null +++ b/src/bindings/python/src/openvino/runtime/opset16/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# New operations added in Opset16 + +# Operators from previous opsets +# TODO (ticket: 156877): Add previous opset operators at the end of opset16 development diff --git a/src/bindings/python/src/openvino/runtime/opset16/ops.py b/src/bindings/python/src/openvino/runtime/opset16/ops.py new file mode 100644 index 00000000000000..0825ccb9d0f487 --- /dev/null +++ b/src/bindings/python/src/openvino/runtime/opset16/ops.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Factory functions for ops added to openvino opset16.""" +from functools import partial + +from openvino.runtime.opset_utils import _get_node_factory + +_get_node_factory_opset16 = partial(_get_node_factory, "opset16") + +# -------------------------------------------- ops ------------------------------------------------ diff --git a/src/bindings/python/tests/test_transformations/test_pattern_ops.py b/src/bindings/python/tests/test_transformations/test_pattern_ops.py index 041bf83764e265..24b28061582c68 100644 --- a/src/bindings/python/tests/test_transformations/test_pattern_ops.py +++ b/src/bindings/python/tests/test_transformations/test_pattern_ops.py @@ -189,7 +189,7 @@ def test_pattern_optional_root(): def test_wrap_type_pattern_type(): - last_opset_number = 15 + last_opset_number = 16 for i in range(1, last_opset_number + 1): WrapType(f"opset{i}.Parameter") WrapType(f"opset{i}::Parameter") diff --git a/src/core/include/openvino/opsets/opset.hpp b/src/core/include/openvino/opsets/opset.hpp index 7ab8c43fcec6ab..524729793f0569 100644 --- a/src/core/include/openvino/opsets/opset.hpp +++ b/src/core/include/openvino/opsets/opset.hpp @@ -172,6 +172,11 @@ const OPENVINO_API OpSet& get_opset14(); * @ingroup ov_opset_cpp_api */ const OPENVINO_API OpSet& get_opset15(); +/** + * @brief Returns map of available opsets + * @ingroup ov_opset_cpp_api + */ +const OPENVINO_API OpSet& get_opset16(); /** * @brief Returns map of available opsets * @ingroup ov_opset_cpp_api diff --git a/src/core/include/openvino/opsets/opset16.hpp b/src/core/include/openvino/opsets/opset16.hpp new file mode 100644 index 00000000000000..0e2c6c5452f661 --- /dev/null +++ b/src/core/include/openvino/opsets/opset16.hpp @@ -0,0 +1,15 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/op/ops.hpp" + +namespace ov { +namespace opset16 { +#define _OPENVINO_OP_REG(a, b) using b::a; +#include "openvino/opsets/opset16_tbl.hpp" +#undef _OPENVINO_OP_REG +} // namespace opset16 +} // namespace ov diff --git a/src/core/include/openvino/opsets/opset16_tbl.hpp b/src/core/include/openvino/opsets/opset16_tbl.hpp new file mode 100644 index 00000000000000..82aa9629263e2a --- /dev/null +++ b/src/core/include/openvino/opsets/opset16_tbl.hpp @@ -0,0 +1,16 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#ifndef _OPENVINO_OP_REG +# warning "_OPENVINO_OP_REG not defined" +# define _OPENVINO_OP_REG(x, y) +#endif + +// Previous opsets operators +// TODO (ticket: 156877): Add remaining operators from opset15 at the end of opset16 development +_OPENVINO_OP_REG(Parameter, ov::op::v0) +_OPENVINO_OP_REG(Convert, ov::op::v0) +_OPENVINO_OP_REG(ShapeOf, ov::op::v3) + +// New operations added in opset16 diff --git a/src/core/src/opsets/opset.cpp b/src/core/src/opsets/opset.cpp index ca219f1c68ecd3..f2490010e9dc50 100644 --- a/src/core/src/opsets/opset.cpp +++ b/src/core/src/opsets/opset.cpp @@ -113,7 +113,8 @@ const std::map>& ov::get_availabl _OPENVINO_REG_OPSET(opset12), _OPENVINO_REG_OPSET(opset13), _OPENVINO_REG_OPSET(opset14), - _OPENVINO_REG_OPSET(opset15)}; + _OPENVINO_REG_OPSET(opset15), + _OPENVINO_REG_OPSET(opset16)}; #undef _OPENVINO_REG_OPSET return opset_map; } @@ -288,3 +289,14 @@ const ov::OpSet& ov::get_opset15() { }); return opset; } + +const ov::OpSet& ov::get_opset16() { + static OpSet opset; + static std::once_flag flag; + std::call_once(flag, [&]() { +#define _OPENVINO_OP_REG(NAME, NAMESPACE) opset.insert(); +#include "openvino/opsets/opset16_tbl.hpp" +#undef _OPENVINO_OP_REG + }); + return opset; +} diff --git a/src/core/tests/op.cpp b/src/core/tests/op.cpp index cf65eecc16cd4c..08fa7a97584e41 100644 --- a/src/core/tests/op.cpp +++ b/src/core/tests/op.cpp @@ -67,4 +67,5 @@ TEST(op, opset_multi_thread) { doTest(ov::get_opset13); doTest(ov::get_opset14); doTest(ov::get_opset15); + doTest(ov::get_opset16); } diff --git a/src/core/tests/opset.cpp b/src/core/tests/opset.cpp index 65c79dc9432439..d667de21263f5a 100644 --- a/src/core/tests/opset.cpp +++ b/src/core/tests/opset.cpp @@ -14,6 +14,7 @@ #include "openvino/opsets/opset13.hpp" #include "openvino/opsets/opset14.hpp" #include "openvino/opsets/opset15.hpp" +#include "openvino/opsets/opset16.hpp" #include "openvino/opsets/opset2.hpp" #include "openvino/opsets/opset3.hpp" #include "openvino/opsets/opset4.hpp" @@ -75,7 +76,8 @@ INSTANTIATE_TEST_SUITE_P(opset, OpsetTestParams{ov::get_opset12, 178}, OpsetTestParams{ov::get_opset13, 186}, OpsetTestParams{ov::get_opset14, 188}, - OpsetTestParams{ov::get_opset15, 199}), + OpsetTestParams{ov::get_opset15, 199}, + OpsetTestParams{ov::get_opset16, 3}), OpsetTestNameGenerator{}); class MyOpOld : public ov::op::Op { diff --git a/src/frontends/common/include/openvino/frontend/extension/op.hpp b/src/frontends/common/include/openvino/frontend/extension/op.hpp index 55fa919447406a..9022b9f801f800 100644 --- a/src/frontends/common/include/openvino/frontend/extension/op.hpp +++ b/src/frontends/common/include/openvino/frontend/extension/op.hpp @@ -25,7 +25,7 @@ inline const ov::OpSet& get_opset_by_name(const std::string& opset_name) { if (opsets.find(opset_name) != opsets.end()) return opsets.at(opset_name)(); if (opset_name.empty() || opset_name == "latest") { - return ov::get_opset15(); + return ov::get_opset15(); // TODO (ticket: 156877): Update to 16 at the end of opset16 development } else { FRONT_END_GENERAL_CHECK(false, "Unsupported opset name: ", opset_name); } diff --git a/src/plugins/template/src/plugin.cpp b/src/plugins/template/src/plugin.cpp index ee885f67e188b5..f66df99c7b1c43 100644 --- a/src/plugins/template/src/plugin.cpp +++ b/src/plugins/template/src/plugin.cpp @@ -221,6 +221,7 @@ ov::SupportedOpsMap ov::template_plugin::Plugin::query_model(const std::shared_p #include "openvino/opsets/opset13_tbl.hpp" #include "openvino/opsets/opset14_tbl.hpp" #include "openvino/opsets/opset15_tbl.hpp" +#include "openvino/opsets/opset16_tbl.hpp" // clang-format on #undef _OPENVINO_OP_REG return op_super_set.contains_type(node->get_type_info()); diff --git a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/single_op_graph.cpp b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/single_op_graph.cpp index fb192e0ac40223..e311f532a2497b 100644 --- a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/single_op_graph.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/single_op_graph.cpp @@ -2189,6 +2189,7 @@ OpGenerator getOpGeneratorMap() { #include "openvino/opsets/opset13_tbl.hpp" #include "openvino/opsets/opset14_tbl.hpp" #include "openvino/opsets/opset15_tbl.hpp" +#include "openvino/opsets/opset16_tbl.hpp" #undef _OPENVINO_OP_REG }; return opGeneratorMap; From 3826706b881ac46aade97fceda78cad5b3f33cc2 Mon Sep 17 00:00:00 2001 From: Andrzej Kopytko Date: Fri, 8 Nov 2024 13:16:56 +0100 Subject: [PATCH 47/93] Docs new benchmark grapf with two y axis (#27480) ### Details: - *item1* - *...* ### Tickets: - *ticket-id* --- .../about-openvino/performance-benchmarks.rst | 6 +- .../data/graph-data-ovms-genai.json | 96 ++ .../benchmarks_files/graph-config.json | 41 +- docs/sphinx_setup/_static/html/modalLLM.html | 95 ++ docs/sphinx_setup/_static/js/graphs.js | 179 +--- docs/sphinx_setup/_static/js/graphsLLM.js | 889 ++++++++++++++++++ docs/sphinx_setup/conf.py | 1 + 7 files changed, 1121 insertions(+), 186 deletions(-) create mode 100644 docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms-genai.json create mode 100644 docs/sphinx_setup/_static/html/modalLLM.html create mode 100644 docs/sphinx_setup/_static/js/graphsLLM.js diff --git a/docs/articles_en/about-openvino/performance-benchmarks.rst b/docs/articles_en/about-openvino/performance-benchmarks.rst index ed9d39aaf8b9e6..75d1882b8cee89 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks.rst @@ -13,7 +13,7 @@ Performance Benchmarks Efficient LLMs for AI PC Performance Information F.A.Q. OpenVINO Accuracy - Getting Performance Numbers + Getting Performance Numbers This page presents benchmark results for the @@ -59,12 +59,12 @@ implemented in your solutions. Click the buttons below to see the chosen benchma .. grid-item:: .. button-link:: # - :class: ovms-toolkit-benchmark-llm + :class: ovms-toolkit-benchmark-llm-result :color: primary :outline: :expand: - :material-regular:`bar_chart;1.4em` OVMS for GenAI (coming soon) + :material-regular:`bar_chart;1.4em` OVMS for GenAI (incoming) diff --git a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms-genai.json b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms-genai.json new file mode 100644 index 00000000000000..f96fb11e6b029d --- /dev/null +++ b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms-genai.json @@ -0,0 +1,96 @@ +[ + { + "Platform": "Intel® Xeon® Platinum 8580", + "Model": "mistralai/Mistral-7B-v0.1", + "PlatformType": "None", + "Parameters": { + "Vllm": { + "Precisions": [ + { + "Throughput": { + "0.2": "350.06", + "0.6": "486.89", + "0.8": "575.92", + "2.0": "778.07" + } + }, + { + "Latency": { + "0.2": "60.93", + "0.6": "91.63", + "0.8": "113.61", + "2.0": "240.25" + } + } + ] + }, + "Ovms": { + "Precisions": [ + { + "Throughput": { + "0.2": "90.98", + "0.6": "266.24", + "0.8": "351.63", + "2.0": "195.16" + } + }, + { + "Latency": { + "0.2": "54.9", + "0.6": "78.78", + "0.8": "95.78", + "2.0": "352.23" + } + } + ] + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8530", + "Model": "mistralai/Mistral-7B-v0.1", + "PlatformType": "None", + "Parameters": { + "Vllm": { + "Precisions": [ + { + "Throughput": { + "0.2": "350.06", + "0.6": "486.89", + "0.8": "575.92", + "2.0": "778.07" + } + }, + { + "Latency": { + "0.2": "60.93", + "0.6": "91.63", + "0.8": "113.61", + "2.0": "240.25" + } + } + ] + }, + "Ovms": { + "Precisions": [ + { + "Throughput": { + "0.2": "90.98", + "0.6": "266.24", + "0.8": "351.63", + "2.0": "195.16" + } + }, + { + "Latency": { + "0.2": "54.9", + "0.6": "78.78", + "0.8": "95.78", + "2.0": "352.23" + } + } + ] + } + } + } +] \ No newline at end of file diff --git a/docs/sphinx_setup/_static/benchmarks_files/graph-config.json b/docs/sphinx_setup/_static/benchmarks_files/graph-config.json index 9cd8176af26235..29fbe714094b74 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/graph-config.json +++ b/docs/sphinx_setup/_static/benchmarks_files/graph-config.json @@ -1,4 +1,3 @@ - { "PrecisionsMap": { "INT4": "int4", @@ -9,7 +8,9 @@ "FP32_OV": "fp32_ov", "FP32_OVMS": "fp32_ovms", "INT8_OV": "int8_ov", - "INT8_OVMS": "int8_ovms" + "INT8_OVMS": "int8_ovms", + "THROUGHPUT": "Throughput", + "LATENCY": "Latency" }, "ParametersMap": { "Throughput": "throughput", @@ -71,6 +72,30 @@ "data": null, "color": "#00536a", "label": "INT8 OVMS" + }, + "Vllm": { + "Latency": { + "data": null, + "color": "#FDB2BC", + "label": "Latency" + }, + "Throughput": { + "data": null, + "color": "#91CDF7", + "label": "Throughput" + } + }, + "Ovms": { + "Latency": { + "data": null, + "color": "#CDB2BC", + "label": "Latency" + }, + "Throughput": { + "data": null, + "color": "#12CDF7", + "label": "Throughput" + } } }, "Filters": [ @@ -80,6 +105,7 @@ "platformTypes": { "name": "ietype", "data": [ + "None", "Intel® Core™, CPU-only", "Intel® Core™, iGPU-only", "Intel® Core™, NPU-only", @@ -88,8 +114,7 @@ }, "platforms": { "name": "platform", - "data": [ - ] + "data": [] }, "platformFilters": { "name": "coretype", @@ -109,15 +134,15 @@ "parameters": { "name": "parameters", "data": [ - "Throughput", - "Latency" + "Vllm", + "Ovms" ] }, "precision": { "name": "precision", "data": [ - "INT4", - "INT8" + "THROUGHPUT", + "LATENCY" ] } } diff --git a/docs/sphinx_setup/_static/html/modalLLM.html b/docs/sphinx_setup/_static/html/modalLLM.html new file mode 100644 index 00000000000000..e3395a16931188 --- /dev/null +++ b/docs/sphinx_setup/_static/html/modalLLM.html @@ -0,0 +1,95 @@ +
+
+

Benchmark Graph Builder

+ × +
+
+ + +
+
+
\ No newline at end of file diff --git a/docs/sphinx_setup/_static/js/graphs.js b/docs/sphinx_setup/_static/js/graphs.js index f8146c1580c58e..4d621ce0780261 100644 --- a/docs/sphinx_setup/_static/js/graphs.js +++ b/docs/sphinx_setup/_static/js/graphs.js @@ -1,173 +1,4 @@ -// =================== GENERAL OUTPUT CONFIG ========================= - -class Filter { - - // param: GraphData[], networkModels[] - static FilterByNetworkModel(graphDataArr, networkModels) { - const optionMap = new Map(); - networkModels.map((model) => graphDataArr.filter((graphData => graphData.Model === model))) - .flat(1) - .forEach(item => optionMap.set(item.Platform, item)); - return Array.from(optionMap.values()); - } - - // param: GraphData[], ieType - static ByIeTypes(graphDataArr, ieTypes) { - const optionMap = new Map(); - graphDataArr - .filter(graphData => ieTypes.includes(graphData.PlatformType)) - .forEach(item => optionMap.set(item.Platform, item)); - return Array.from(optionMap.values()); - } - - // param: GraphData[], ieType, networkModels - static ByTypesAndModels(graphDataArr, ieTypes, models) { - const optionMap = new Map(); - graphDataArr - .filter(graphData => ieTypes.includes(graphData.PlatformType)) - .filter(graphData => models.includes(graphData.Model)) - .forEach(item => optionMap.set(item.Platform, item)); - return Array.from(optionMap.values()); - } - - // param: GraphData[], clientPlatforms - static ByIeKpis(graphDataArr, clientPlatforms) { - var kpis = [] - clientPlatforms.forEach((platformName) => { - graphDataArr.filter((data) => { - if (data.Platform.includes(platformName)) { - for (var key in data.Parameters) { - if (!kpis.includes(key)) kpis.push(key) - } - } - }) - }) - return kpis; - } - - // param: GraphData[] - static getParameters(graphDataArr) { - var parameters = [] - graphDataArr.filter((data) => { - for (var key in data.Parameters) { - if (!parameters.includes(Graph.capitalizeFirstLetter(key))) parameters.push(Graph.capitalizeFirstLetter(key)) - } - }) - return parameters; - } - - // param: GraphData[] - static getIeTypes(graphDataArr) { - var kpis = [] - graphDataArr.filter((data) => { - for (var key in data.Parameters) { - if (!kpis.includes(Graph.capitalizeFirstLetter(key))) kpis.push(Graph.capitalizeFirstLetter(key)) - } - }) - return kpis; - } - - // param: GraphData[], clientPlatforms[] - static ByClientPlatforms(graphDataArr, platformsArr) { - return graphDataArr.filter((data) => { - return platformsArr.includes(data.Platform) - }); - } - - // param: GraphData[], coreTypes[] - static FilterByCoreTypes(graphDataArr, coreTypes) { - if (coreTypes) { - return graphDataArr.filter((data) => coreTypes.includes(data.PlatformType)); - } - return graphDataArr; - } -} - -class Modal { - static getPrecisionsLabels(graphDataArr) { - var kpis = [] - graphDataArr.filter((data) => { - for (var key in data.Parameters) { - data.Parameters[key].Precisions.forEach((key) => { - Object.keys(key).forEach((key) => { - if (!kpis.includes(key.toUpperCase())) kpis.push(key.toUpperCase()) - }); - }) - } - }) - return kpis; - } - - static getPrecisions(appConfig, labels) { - return labels.map((label) => { - var prec = appConfig.PrecisionsMap[label]; - if (prec !== undefined) { - return prec; - } - else { - return "no name"; - } - }); - } -} - - -class Graph { - // functions to get unique keys - static getNetworkModels(graphDataArr) { - return Array.from(new Set(graphDataArr.map((obj) => obj.Model))); - } - static getIeTypes(graphDataArr) { - return Array.from(new Set(graphDataArr.map((obj) => obj.PlatformType))); - } - static getCoreTypes(graphDataArr) { - return Array.from(new Set(graphDataArr.map((obj) => obj.ieType))); - } - - // param: GraphData[] - static getPlatformNames(graphDataArr) { - return graphDataArr.map((data) => data.Platform); - } - - // param: GraphData[], parameterName: string, precisions: list - static getDatabyParameter(graphDataArr, parameterName, precisions) { - var array = []; - graphDataArr.forEach((item) => { - if (item.Parameters[parameterName] !== undefined) { - array.push(item.Parameters[parameterName].Precisions); - } - else { - var obj = {}; - precisions.forEach((prec) => { - obj[prec] = 0; - }) - array.push([obj]) - } - }) - return array; - - } - - // this returns an object that is used to ender the chart - static getGraphConfig(parameterName, item, precisions, appConfig) { - return { - chartTitle: Graph.capitalizeFirstLetter(parameterName), - iconClass: parameterName + '-icon', - unit: item.Parameters[parameterName]?.Unit, - datasets: precisions.map((precision) => appConfig.PrecisionData[precision]), - }; - } - static capitalizeFirstLetter(string) { - return string.charAt(0).toUpperCase() + string.slice(1); - } -} - -class ChartDisplay { - constructor(mode, numberOfCharts) { - this.mode = mode; - this.numberOfChartsInRow = numberOfCharts; - } -} +// =================== ADDITIONAL OUTPUT CONFIG ========================= $(document).ready(function () { @@ -186,7 +17,6 @@ $(document).ready(function () { $('.graph-chart-title-header').off('click').on('click', (event) => { var parent = event.target.parentElement; - if ($(parent).children('.chart-wrap,.empty-chart-container').is(":visible")) { $(parent).children('.chart-wrap,.empty-chart-container').hide(); $(parent).children('.chevron-right-btn').show(); @@ -702,11 +532,10 @@ $(document).ready(function () { chartContainer.append(chartWrap); var graphConfigs = parameters.map((parameter) => { var groupUnit = model[0]; - var kpiData = Graph.getDatabyParameter(model, appConfig.ParametersMap[parameter], precisions); - var config = Graph.getGraphConfig(appConfig.ParametersMap[parameter], groupUnit, precisions, JSON.parse(JSON.stringify(appConfig))); + var kpiData = Graph.getDatabyParameterOld(model, appConfig.ParametersMap[parameter], precisions); + var config = Graph.getGraphConfigOld(appConfig.ParametersMap[parameter], groupUnit, precisions, JSON.parse(JSON.stringify(appConfig))); precisions.forEach((precision, index) => { - config.datasets[index].data = kpiData.map(tData => tData[0][precision] - ); + config.datasets[index].data = kpiData.map(tData => tData[0][precision]); }); return config; }); diff --git a/docs/sphinx_setup/_static/js/graphsLLM.js b/docs/sphinx_setup/_static/js/graphsLLM.js new file mode 100644 index 00000000000000..4dbc0313e2a133 --- /dev/null +++ b/docs/sphinx_setup/_static/js/graphsLLM.js @@ -0,0 +1,889 @@ +// =================== GENERAL OUTPUT CONFIG ========================= + +class Filter { + + // param: GraphData[], networkModels[] + static FilterByNetworkModel(graphDataArr, networkModels) { + const optionMap = new Map(); + networkModels.map((model) => graphDataArr.filter((graphData => graphData.Model === model))) + .flat(1) + .forEach(item => optionMap.set(item.Platform, item)); + return Array.from(optionMap.values()); + } + + // param: GraphData[], ieType + static ByIeTypes(graphDataArr, ieTypes) { + const optionMap = new Map(); + graphDataArr + .filter(graphData => ieTypes.includes(graphData.PlatformType)) + .forEach(item => optionMap.set(item.Platform, item)); + return Array.from(optionMap.values()); + } + + // param: GraphData[], ieType, networkModels + static ByTypesAndModels(graphDataArr, ieTypes, models) { + const optionMap = new Map(); + graphDataArr + .filter(graphData => ieTypes.includes(graphData.PlatformType)) + .filter(graphData => models.includes(graphData.Model)) + .forEach(item => optionMap.set(item.Platform, item)); + return Array.from(optionMap.values()); + } + + // param: GraphData[], clientPlatforms + static ByIeKpis(graphDataArr, clientPlatforms) { + var kpis = [] + clientPlatforms.forEach((platformName) => { + graphDataArr.filter((data) => { + if (data.Platform.includes(platformName)) { + for (var key in data.Parameters) { + if (!kpis.includes(key)) kpis.push(key) + } + } + }) + }) + return kpis; + } + + // param: GraphData[] + static getParameters(graphDataArr) { + var parameters = [] + graphDataArr.filter((data) => { + for (var key in data.Parameters) { + if (!parameters.includes(Graph.capitalizeFirstLetter(key))) parameters.push(Graph.capitalizeFirstLetter(key)) + } + }) + return parameters; + } + + // param: GraphData[] + static getIeTypes(graphDataArr) { + var kpis = [] + graphDataArr.filter((data) => { + for (var key in data.Parameters) { + if (!kpis.includes(Graph.capitalizeFirstLetter(key))) kpis.push(Graph.capitalizeFirstLetter(key)) + } + }) + return kpis; + } + + // param: GraphData[], clientPlatforms[] + static ByClientPlatforms(graphDataArr, platformsArr) { + return graphDataArr.filter((data) => { + return platformsArr.includes(data.Platform) + }); + } + + // param: GraphData[], coreTypes[] + static FilterByCoreTypes(graphDataArr, coreTypes) { + if (coreTypes) { + return graphDataArr.filter((data) => coreTypes.includes(data.PlatformType)); + } + return graphDataArr; + } +} + +class Modal { + static getPrecisionsLabels(graphDataArr) { + var kpis = [] + graphDataArr.filter((data) => { + for (var key in data.Parameters) { + data.Parameters[key].Precisions.forEach((key) => { + Object.keys(key).forEach((key) => { + if (!kpis.includes(key.toUpperCase())) kpis.push(key.toUpperCase()) + }); + }) + } + }) + return kpis; + } + + static getPrecisions(appConfig, labels) { + return labels.map((label) => { + var prec = appConfig.PrecisionsMap[label]; + if (prec !== undefined) { + return prec; + } + else { + return "no name"; + } + }); + } +} + +class Graph { + // functions to get unique keys + static getNetworkModels(graphDataArr) { + return Array.from(new Set(graphDataArr.map((obj) => obj.Model))); + } + static getIeTypes(graphDataArr) { + return Array.from(new Set(graphDataArr.map((obj) => obj.PlatformType))); + } + static getCoreTypes(graphDataArr) { + return Array.from(new Set(graphDataArr.map((obj) => obj.ieType))); + } + + // param: GraphData[] + static getPlatformNames(graphDataArr) { + return graphDataArr.map((data) => data.Platform); + } + + // param: GraphData[], engine: string, precisions: list + static getDatabyParameter(graphDataArr, engine, array) { + if (!Array.isArray(array[engine])) { + array[engine] = []; + } + array[engine].push(graphDataArr.Parameters[engine].Precisions); + return array; + } + + // this returns an object that is used to ender the chart + static getGraphConfig(engine, precisions, appConfig) { + return { + chartTitle: 'Throughput vs Latency', + iconClass: 'latency-icon', + datasets: precisions.map((precision) => appConfig.PrecisionData[engine][precision]), + unit: "None" + }; + } + // param: GraphData[], parameterName: string, precisions: list + static getDatabyParameterOld(graphDataArr, parameterName, precisions) { + var array = []; + graphDataArr.forEach((item) => { + if (item.Parameters[parameterName] !== undefined) { + array.push(item.Parameters[parameterName].Precisions); + } + else { + var obj = {}; + precisions.forEach((prec) => { + obj[prec] = 0; + }) + array.push([obj]) + } + }) + return array; + + } + + // this returns an object that is used to ender the chart + static getGraphConfigOld(parameterName, item, precisions, appConfig) { + return { + chartTitle: Graph.capitalizeFirstLetter(parameterName), + iconClass: parameterName + '-icon', + unit: item.Parameters[parameterName]?.Unit, + datasets: precisions.map((precision) => appConfig.PrecisionData[precision]), + }; + } + static capitalizeFirstLetter(string) { + return string.charAt(0).toUpperCase() + string.slice(1); + } +} + +class ChartDisplay { + constructor(mode, numberOfCharts) { + this.mode = mode; + this.numberOfChartsInRow = numberOfCharts; + } +} + +$(document).ready(function () { + $('.ovms-toolkit-benchmark-llm-result').on('click', () => showModal("graph-data-ovms-genai.json")); + function clickBuildGraphsLLM(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions) { + renderData(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions); + $('.modal-footer').show(); + $('#modal-display-graphs').show(); + $('.edit-settings-btn').off('click').on('click', (event) => { + $('#modal-configure-graphs').show(); + $('#modal-display-graphs').hide(); + $('.modal-footer').hide(); + $('.chart-placeholder').empty(); + }); + + $('.graph-chart-title-header').off('click').on('click', (event) => { + var parent = event.target.parentElement; + + if ($(parent).children('.chart-wrap,.empty-chart-container').is(":visible")) { + $(parent).children('.chart-wrap,.empty-chart-container').hide(); + $(parent).children('.chevron-right-btn').show(); + $(parent).children('.chevron-down-btn').hide(); + } else { + $(parent).children('.chart-wrap,.empty-chart-container').show(); + $(parent).children('.chevron-down-btn').show(); + $(parent).children('.chevron-right-btn').hide(); + } + }); + } + + function hideModal() { + $('#graphModal').remove(); + $('body').css('overflow', 'auto'); + } + + function showModal(file) { + $('body').css('overflow', 'hidden'); + + fetch('../_static/benchmarks_files/data/' + file) + .then((response) => response.json()) + .then((jsonData) => { + fetch('../_static/benchmarks_files/graph-config.json') + .then((configResponse) => configResponse.json()) + .then((appConfig) => { + renderModal(jsonData, appConfig) + }) + }); + } + + function getSelectedNetworkModels() { + return $('.models-column input:checked, .platforms-column input:checked').not('[data-networkmodel="Select All"]').map(function () { + return $(this).data('networkmodel'); + }).get(); + } + + function getSelectedIeTypes() { + return $('.ietype-column input:checked').map(function () { + return $(this).data('ietype'); + }).get(); + } + + function getSelectedClientPlatforms() { + return $('.platforms-column input:checked').map(function () { + return $(this).data('platform'); + }).get(); + } + + function getSelectedKpis() { + return $('.kpi-column input:checked').map(function () { + return $(this).data('kpi'); + }).get(); + } + + function getSelectedPrecisions() { + return $('.precisions-column input:checked').map(function () { + return $(this).data('precision'); + }).get(); + } + + function validateSelections() { + if (getSelectedNetworkModels().length > 0 + && getSelectedIeTypes() + && getSelectedClientPlatforms().length > 0 + && getSelectedKpis().length > 0) { + if (getSelectedPrecisions().length > 0) { + $('#build-graphs-btn').prop('disabled', false); + return; + } + $('#build-graphs-btn').prop('disabled', true); + return; + } + $('#build-graphs-btn').prop('disabled', true); + } + + function renderModal(graph, appConfig) { + new Graph(graph); + var networkModels = Graph.getNetworkModels(graph); + var ieTypes = Graph.getIeTypes(graph); + fetch('../_static/html/modalLLM.html').then((response) => response.text()).then((text) => { + + // generate and configure modal container + var modal = $('
'); + modal.attr('id', 'graphModal'); + modal.addClass('modal'); + var modalContent = $(text); + modalContent.attr('id', 'graphModalContent'); + modalContent.addClass('modal-content'); + modal.append(modalContent); + + const models = networkModels.map((networkModel) => createCheckMark(networkModel, 'networkmodel')); + modal.find('.models-column').append(models); + + const selectAllModelsButton = createCheckMark('', 'networkmodel'); + modal.find('.models-selectall').append(selectAllModelsButton); + + const selectAllPlatformsButton = createCheckMark('', 'platform'); + modal.find('.platforms-selectall').append(selectAllPlatformsButton); + + const precisions = Modal.getPrecisionsLabels(graph).map((precision) => createCheckMark(precision, 'precision', false)); + modal.find('.precisions-column').append(precisions); + + selectAllCheckboxes(precisions); + disableAllCheckboxes(precisions); + + const selectAllTypesButton = createCheckMark('', 'ietype') + modal.find('.ietype-selectall').append(selectAllTypesButton); + + const iefilter = ieTypes.map((ieType) => createCheckMark(ieType, 'ietype')); + modal.find('.ietype-column').append(iefilter); + + modal.find('#modal-display-graphs').hide(); + modal.find('.ietype-column input').first().prop('checked', true); + + const kpiLabels = Filter.getParameters(graph).map((parameter) => createCheckMark(parameter, 'kpi', false)); + modal.find('.kpi-column').append(kpiLabels); + + $('body').prepend(modal); + + preselectDefaultSettings(graph, modal, appConfig); + + //is not generic solution :( + if (appConfig.DefaultSelections.platformTypes?.data?.includes('Select All')) { + selectAllCheckboxes(iefilter); + + }; + renderClientPlatforms(graph, modal); + + $('.clear-all-btn').on('click', clearAll); + $('#build-graphs-btn').on('click', () => { + $('#modal-configure-graphs').hide(); + clickBuildGraphsLLM(graph, appConfig, getSelectedNetworkModels(), getSelectedIeTypes(), getSelectedClientPlatforms(), getSelectedKpis(), Modal.getPrecisions(appConfig, getSelectedPrecisions())); + }); + $('.modal-close').on('click', hideModal); + $('.close-btn').on('click', hideModal); + + modal.find('.ietype-selectall input').on('click', function () { + if ($(this).prop('checked')) + selectAllCheckboxes(iefilter); + else deSelectAllCheckboxes(iefilter); + }); + + modal.find('.models-selectall input').on('click', function () { + if ($(this).prop('checked')) selectAllCheckboxes(models); + else deSelectAllCheckboxes(models); + + renderClientPlatforms(graph, modal) + }); + + modal.find('.platforms-selectall input').on('click', function () { + if ($(this).prop('checked')) + renderClientPlatforms(graph, modal) + else { + var enabledPlatforms = modal.find('.platforms-column .checkmark-container'); + deSelectCheckbox(enabledPlatforms); + }; + + }); + + modal.find('.models-column input').on('click', function () { + if (!$(this)[0].checked) { + deSelectCheckbox(selectAllModelsButton); + } + }); + + + modal.find('.ietype-column input').on('click', function () { + if (!$(this)[0].checked) { + deSelectCheckbox(selectAllTypesButton); + } + }); + + modal.find('.models-column input').on('click', () => renderClientPlatforms(graph, modal)); + modal.find('.ietype-column input').on('click', () => renderClientPlatforms(graph, modal)); + modal.find('.ietype-selectall input').on('click', () => renderClientPlatforms(graph, modal)); + modal.find('.platforms-column').on('click', () => enableParmeters(graph, getSelectedClientPlatforms())); + + modal.find('.kpi-column input').on('click', validatePrecisionSelection); + modal.find('input').on('click', validateSelections); + + var modalFilters = document.getElementById("modal-filters"); + + var showFiltersButton = document.getElementById("filters"); + showFiltersButton.onclick = function () { + modalFilters.style.display = "block"; + } + + var closeFiltersButton = document.getElementsByClassName("close-filters")[0]; + closeFiltersButton.onclick = function () { + modalFilters.style.display = "none"; + } + + window.onclick = function (event) { + if (event.target == modalFilters) { + modalFilters.style.display = "none"; + } + } + }); + } + + function validatePrecisionSelection() { + const precisions = $('.precisions-column').find('input') + precisions.prop('disabled', false); + } + + function clearAll() { + $('.modal-content-grid-container input:checkbox').each((index, object) => $(object).prop('checked', false)); + validatePrecisionSelection(); + validateSelections(); + } + + function preselectDefaultSettings(graph, modal, appConfig) { + + const defaultSelections = appConfig.DefaultSelections; + + selectDefaultPlatformType(defaultSelections.platformTypes, graph, modal); + + applyPlatformFilters(defaultSelections.platformFilters, modal, graph); + + clearAllSettings(defaultSelections); + + validateSelections(); + validatePrecisionSelection(); + } + + function selectDefaultPlatformType(platformTypes, graph, modal) { + if (!platformTypes) return; + + const type = platformTypes.data[0]; + $(`input[data-ietype="${type}"]`).prop('checked', true); + renderClientPlatforms(graph, modal); + } + + function applyPlatformFilters(platformFilters, modal, graph) { + if (!platformFilters) return; + + const filters = modal.find('.selectable-box-container').children('.selectable-box'); + filters.removeClass('selected'); + + platformFilters.data.forEach(selection => { + filters.filter(`[data-${platformFilters.name}="${selection}"]`).addClass('selected'); + }); + + renderClientPlatforms(graph, modal); + } + + function clearAllSettings(defaultSelections) { + clearAll(); + Object.keys(defaultSelections).forEach(setting => { + const { name, data } = defaultSelections[setting]; + data.forEach(selection => { + $(`input[data-${name}="${selection}"]`).prop('checked', true); + }); + }); + } + + function filterClientPlatforms(graph, ietypes) { + return Filter.ByIeTypes(graph, ietypes); + } + + function filterPlatforms(graph, ietypes, models) { + return Filter.ByTypesAndModels(graph, ietypes, models); + } + + function renderClientPlatforms(graph, modal) { + var fPlatforms = filterClientPlatforms(graph, getSelectedIeTypes()); + var platformNames = Graph.getPlatformNames(fPlatforms); + $('.platforms-column .checkmark-container').remove(); + + const clientPlatforms = platformNames.map((platform) => createCheckMark(platform, 'platform', true)); + + var enabledPlatforms = filterPlatforms(graph, getSelectedIeTypes(), getSelectedNetworkModels()); + enableCheckBoxes(clientPlatforms, enabledPlatforms); + modal.find('.platforms-column').append(clientPlatforms); + + enableParmeters(graph, getSelectedClientPlatforms()); + modal.find('.platforms-column input').on('click', validateSelections); + } + + function enableParmeters(graph, clientPlatforms) { + var allKpis = Filter.getParameters(graph); + + allKpis.forEach((kpi) => { + $(`input[data-kpi="${Graph.capitalizeFirstLetter(kpi)}"]`).prop('disabled', true); + }) + + var kpis = Filter.ByIeKpis(graph, clientPlatforms); + kpis.forEach((kpi) => { + $(`input[data-kpi="${Graph.capitalizeFirstLetter(kpi)}"]`).prop('disabled', false); + }) + } + + function createCheckMark(itemLabel, modelLabel, disabled) { + const item = $('