From 90793aab546beedfd805eddd8fd2d807b3862d53 Mon Sep 17 00:00:00 2001 From: Nadav Elyahu <88962733+nelyahu@users.noreply.github.com> Date: Fri, 3 May 2024 23:22:29 +0300 Subject: [PATCH 01/25] re-introduce: stage3: efficient compute of scaled_global_grad_norm (#5493) reverting previous revert of this feature: https://github.com/nelyahu/DeepSpeed/commit/bc48371c5e1fb8fd70fc79285e66201dbb65679b in addition, bug fix for offload mode. --- deepspeed/runtime/zero/stage3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/runtime/zero/stage3.py b/deepspeed/runtime/zero/stage3.py index c6ff216edfcb..13ca29c9fceb 100644 --- a/deepspeed/runtime/zero/stage3.py +++ b/deepspeed/runtime/zero/stage3.py @@ -1409,7 +1409,7 @@ def complete_grad_norm_calculation_for_cpu_offload(self, params): norm_is_nan = total_norm.isnan() inf_or_nan = norm_is_nan.logical_or(norm_is_inf) - err = torch.tensor(-1.0, device=self.device, dtype=torch.float) + err = torch.tensor(-1.0, device=inf_or_nan.device, dtype=torch.float) total_norm = inf_or_nan * err + inf_or_nan.logical_not() * total_norm return total_norm From 0fc19b6a320cf8aa0a5f6c2b1fa310bae9a70d94 Mon Sep 17 00:00:00 2001 From: harygo2 <168085522+harygo2@users.noreply.github.com> Date: Tue, 7 May 2024 08:05:54 +0800 Subject: [PATCH 02/25] Fix crash when creating Torch tensor on NPU with device=get_accelerator().current_device() (#5464) Creating a Torch tensor with the parameter `device=get_accelerator().current_device()` can result in a crash when using an NPU. This issue arises because the `current_device` API across all accelerators is expected to return a device id as an integer, according to the [interface docs.](https://github.com/microsoft/DeepSpeed/blob/fa8458b1a80d6ba55091b17f092de19bbf95eb3d/docs/_tutorials/accelerator-abstraction-interface.md?plain=1#L52C1-L56C103) However, specifying `device` as an interger when creating tensors by default directs Torch to use the CUDA backend, which leads to crash on NPUs (and potentially other accelerators as well). To resolve this, we should use `get_accelerator().current_device_name()` instead, which returns the correct device identifier strings such as `"npu:0", "cuda:0", or "xpu:0"`. This API provides the appropriate context needed for creating tensors on specific hardware accelerators. I also notice that `device=get_accelerator().current_device()` is used across several files under deepspeed/inference, and may also lead to crash on other accelerators. --------- Co-authored-by: Olatunji Ruwase Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> --- deepspeed/linear/optimized_linear.py | 2 +- deepspeed/runtime/fp16/fused_optimizer.py | 2 +- deepspeed/runtime/utils.py | 8 +++++--- tests/unit/moe/test_moe.py | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/deepspeed/linear/optimized_linear.py b/deepspeed/linear/optimized_linear.py index 138bd493ffc7..e982785a8122 100644 --- a/deepspeed/linear/optimized_linear.py +++ b/deepspeed/linear/optimized_linear.py @@ -85,7 +85,7 @@ def __init__(self, self.bias = bias self.lora_config = lora_config self.quantization_config = quantization_config - device = get_accelerator().current_device() if device is None else device + device = get_accelerator().current_device_name() if device is None else device assert self.lora_config is not None, "DSOptimizedLinear requires a LoRA config" self.zero_shards = self.lora_config.base_weight_sharding diff --git a/deepspeed/runtime/fp16/fused_optimizer.py b/deepspeed/runtime/fp16/fused_optimizer.py index bf1693307ea7..49093bb73c8f 100755 --- a/deepspeed/runtime/fp16/fused_optimizer.py +++ b/deepspeed/runtime/fp16/fused_optimizer.py @@ -241,7 +241,7 @@ def _get_norm_mask_idx(self, group): group_mask_idx_list.append([grad_flat_st_idx, grad_flat_en_idx]) grad_flat_st_idx = grad_flat_en_idx - return torch.tensor(group_mask_idx_list, device=get_accelerator().current_device()) + return torch.tensor(group_mask_idx_list, device=get_accelerator().current_device_name()) def step(self, closure=None): """ diff --git a/deepspeed/runtime/utils.py b/deepspeed/runtime/utils.py index 7744b2ee8b98..2c01c3475a70 100755 --- a/deepspeed/runtime/utils.py +++ b/deepspeed/runtime/utils.py @@ -171,7 +171,7 @@ def get_norm_with_moe_layers_fast(all_groups_norm, group): # This implementation standardizes the grad_norm across ranks. A more precise implementation can be found in 'get_norm_with_moe_layers'. # Need to allreduce (avg) the norms across different ranks because moe params will not be synced during allreduce scaled_norm = all_groups_norm * 1.0 / float(dist.get_world_size(group=group)) - scaled_norm_tensor = torch.tensor(scaled_norm, device=get_accelerator().current_device(), dtype=torch.float) + scaled_norm_tensor = torch.tensor(scaled_norm, device=get_accelerator().current_device_name(), dtype=torch.float) dist.all_reduce(scaled_norm_tensor, group=group) all_groups_norm = scaled_norm_tensor.item() #print(f"old = {all_groups_norm_old} and new = {all_groups_norm} at rank: {deepspeed.comm.get_rank()}") @@ -424,9 +424,11 @@ def get_flattened_grad_norm(parameters, norm_type=2, mpu=None, grad_norm_mask=No # # mask_tensor_ = torch.zeros_like(p, device=p.device, dtype=bool) # # for mask_idx in grad_norm_mask[idx]: # # mask_tensor_[mask_idx[0]:mask_idx[1]] = True - cum_sum_pairs = torch.tensor([1, -1], device=get_accelerator().current_device(), + cum_sum_pairs = torch.tensor([1, -1], device=get_accelerator().current_device_name(), dtype=p.dtype).repeat(grad_norm_mask[idx].shape[0], 1) - mask_tensor = torch.zeros(p.shape[0] + 1, device=get_accelerator().current_device(), dtype=p.dtype) + mask_tensor = torch.zeros(p.shape[0] + 1, + device=get_accelerator().current_device_name(), + dtype=p.dtype) mask_tensor = mask_tensor.scatter_(0, grad_norm_mask[idx].view(-1), cum_sum_pairs.view(-1)).cumsum(0).bool()[:-1] diff --git a/tests/unit/moe/test_moe.py b/tests/unit/moe/test_moe.py index d39f9fe3d651..fdff9430a4e6 100644 --- a/tests/unit/moe/test_moe.py +++ b/tests/unit/moe/test_moe.py @@ -177,7 +177,7 @@ class TestTopk(DistributedTest): world_size = 2 def test(self): - device = get_accelerator().current_device() + device = get_accelerator().current_device_name() if dist.get_rank() == 0: logits = torch.rand(2, 2, device=device) elif dist.get_rank() == 1: From 0b224edcf7d83713b95ad6b989694a8bdf01809e Mon Sep 17 00:00:00 2001 From: BacharL Date: Wed, 8 May 2024 12:53:25 +0300 Subject: [PATCH 03/25] Fix compile wrapper (#5455) compile wrapper will inherit from user module class and copy it's __dict__ This should resolve most issues in #5383 except potential extra user forward hooks. @tohtana @loadams Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> Co-authored-by: Olatunji Ruwase Co-authored-by: Masahiro Tanaka <81312776+tohtana@users.noreply.github.com> --- deepspeed/runtime/compiler.py | 163 ++++++++++++++++--------------- deepspeed/runtime/engine.py | 7 -- deepspeed/runtime/pipe/engine.py | 4 +- 3 files changed, 83 insertions(+), 91 deletions(-) diff --git a/deepspeed/runtime/compiler.py b/deepspeed/runtime/compiler.py index b5e4e33425d0..66fe29fbbea2 100644 --- a/deepspeed/runtime/compiler.py +++ b/deepspeed/runtime/compiler.py @@ -83,84 +83,85 @@ def validate_enabled(cls, field_value, values): return field_value -class CompiledModuleWrapper(torch.nn.Module): - - def __init__(self, module, compile_config: Union[CompileConfig, None] = None): - super().__init__() - - assert is_compile_supported(), "torch.compile is not supported on this version of PyTorch." - - modules = self.__dict__.get('_modules') - modules['wrapped'] = module - self.__dict__['wrapped'] = module - self._is_compiled = False - self._backend = get_backend_fn(compile_config.backend) - self._compile_kwargs = compile_config.kwargs - self._compiler_fn = None - - def __getattr__(self, name): - return getattr(self.__dict__['wrapped'], name) - - def set_backend(self, backend: Union[str, Callable]): - """Set the backend for torch.compile. - - Args: - backend (Union[str, Callable]): backend name or a function that takes a torch.nn.Module and returns a compiled module. - You can directly pass a function that works as a backend. - See also `backend` field in `CompileConfig` for more details. - """ - self._backend = get_backend_fn(backend) - - def set_torch_compile_kwargs(self, kwargs: Dict[str, Union[str, Any]]) -> None: - """Set kwargs for torch.compile. Kwargs that are set in DeepSpeed config will be overwritten. - You can also pass a backend name with "backend" key to change the backend. - - Args: - kwargs (Dict[str, Union[str, Any]]): kwargs passed to torch.compile. - """ - - if "backend" in kwargs: - raise ValueError("backend cannot be set as compile kwargs. Use set_backend instead.") - self._compile_kwargs.update(kwargs) - - def set_compiler_fn(self, compiler_fn: Callable) -> None: - """Set a function to be used for compiling the module. - This function should take a torch.nn.Module as input and return a compiled module. - Note that other compile options are ignored when a compiler_fn is set. - - Example: - ```python - def my_compiler_fn(module: torch.nn.Module): - ... - return torch.compile(module, ...) - - engine.set_compiler_fn(my_compiler_fn) - ``` - """ - self._compiler_fn = compiler_fn - - def forward(self, *args, **kwargs) -> Any: - if not self.is_compiled: - if self._compiler_fn is None: - self.__dict__['wrapped'] = torch.compile(self.wrapped, backend=self._backend, **self._compile_kwargs) - else: - self.__dict__['wrapped'] = self._compiler_fn(self.wrapped) - self._is_compiled = True - - return self.__dict__['wrapped'](*args, **kwargs) - - @property - def is_compiled(self) -> bool: - return self._is_compiled - - @property - def backend(self) -> Union[str, Callable]: - return self._backend - - @property - def torch_compile_kwargs(self) -> Dict[str, Any]: - return self._compile_kwargs - - @property - def compiler_fn(self) -> Union[Callable, None]: - return self._compiler_fn +def CompiledModuleWrapper(mod, compile_config: Union[CompileConfig, None] = None): + + class wrapper(mod.__class__): + + def __init__(self, module, compile_config: Union[CompileConfig, None] = None): + self.__dict__ = module.__dict__.copy() + + assert is_compile_supported(), "torch.compile is not supported on this version of PyTorch." + + self.__dict__['wrapped'] = module + self._is_compiled = False + self._backend = get_backend_fn(compile_config.backend) + self._compile_kwargs = compile_config.kwargs + self._compiler_fn = None + + def set_backend(self, backend: Union[str, Callable]): + """Set the backend for torch.compile. + + Args: + backend (Union[str, Callable]): backend name or a function that takes a torch.nn.Module and returns a compiled module. + You can directly pass a function that works as a backend. + See also `backend` field in `CompileConfig` for more details. + """ + self._backend = get_backend_fn(backend) + + def set_torch_compile_kwargs(self, kwargs: Dict[str, Union[str, Any]]) -> None: + """Set kwargs for torch.compile. Kwargs that are set in DeepSpeed config will be overwritten. + You can also pass a backend name with "backend" key to change the backend. + + Args: + kwargs (Dict[str, Union[str, Any]]): kwargs passed to torch.compile. + """ + + if "backend" in kwargs: + raise ValueError("backend cannot be set as compile kwargs. Use set_backend instead.") + self._compile_kwargs.update(kwargs) + + def set_compiler_fn(self, compiler_fn: Callable) -> None: + """Set a function to be used for compiling the module. + This function should take a torch.nn.Module as input and return a compiled module. + Note that other compile options are ignored when a compiler_fn is set. + + Example: + ```python + def my_compiler_fn(module: torch.nn.Module): + ... + return torch.compile(module, ...) + + engine.set_compiler_fn(my_compiler_fn) + ``` + """ + self._compiler_fn = compiler_fn + + def forward(self, *args, **kwargs) -> Any: + if not self.is_compiled: + if self._compiler_fn is None: + self.__dict__['wrapped'] = torch.compile(self.wrapped, + backend=self._backend, + **self._compile_kwargs) + else: + self.__dict__['wrapped'] = self._compiler_fn(self.wrapped) + self._is_compiled = True + + return self.__dict__['wrapped'](*args, **kwargs) + + @property + def is_compiled(self) -> bool: + return self._is_compiled + + @property + def backend(self) -> Union[str, Callable]: + return self._backend + + @property + def torch_compile_kwargs(self) -> Dict[str, Any]: + return self._compile_kwargs + + @property + def compiler_fn(self) -> Union[Callable, None]: + return self._compiler_fn + + return wrapper(mod, compile_config) diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py index 9a2b943b0992..34263444c1b7 100644 --- a/deepspeed/runtime/engine.py +++ b/deepspeed/runtime/engine.py @@ -469,13 +469,6 @@ def __getattr__(self, name): return getattr(self, name) elif name in dir(_module): return getattr(_module, name) - elif isinstance(_module, CompiledModuleWrapper): - try: - return getattr(_module, name) - except AttributeError: - raise AttributeError( - f"None of {type(self).__name__}, CompiledModuleWrapper, or the wrapped model has the attribute '{name}'" - ) else: raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") diff --git a/deepspeed/runtime/pipe/engine.py b/deepspeed/runtime/pipe/engine.py index 1dda7f1aad32..be8fe1a368c6 100644 --- a/deepspeed/runtime/pipe/engine.py +++ b/deepspeed/runtime/pipe/engine.py @@ -67,9 +67,7 @@ class PipelineEngine(DeepSpeedEngine): def __init__(self, has_bool_tensors=False, *super_args, **super_kwargs): super().__init__(*super_args, **super_kwargs) - assert isinstance(self.module, PipelineModule) \ - or (hasattr(self.module, 'wrapped') and isinstance(self.module.wrapped, PipelineModule)), \ - "model must base PipelineModule" + assert isinstance(self.module, PipelineModule), "model must base PipelineModule" assert self.zero_optimization_stage( ) < ZeroStageEnum.gradients, "ZeRO-2 and ZeRO-3 are incompatible with pipeline parallelism" From 3dd7ccff8103be60c31d963dd2278d43abb68fd1 Mon Sep 17 00:00:00 2001 From: Yejing-Lai Date: Thu, 9 May 2024 06:04:02 +0800 Subject: [PATCH 04/25] enable phi3_mini autotp (#5501) This PR aims to enable phi3 mini autotp. Phi3 mini uses chunk MLP. We adjust this linear layer weight order to support this model. Please kindly review~ Thanks! --------- Co-authored-by: Lev Kurilenko <113481193+lekurile@users.noreply.github.com> --- deepspeed/module_inject/auto_tp.py | 9 +++-- deepspeed/module_inject/fusedqkv_utils.py | 40 ++++++++++++++++++++++- deepspeed/module_inject/replace_module.py | 6 +++- deepspeed/module_inject/tp_shard.py | 10 ++++++ 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/deepspeed/module_inject/auto_tp.py b/deepspeed/module_inject/auto_tp.py index 88f7086518e8..4944e1954e37 100644 --- a/deepspeed/module_inject/auto_tp.py +++ b/deepspeed/module_inject/auto_tp.py @@ -13,7 +13,7 @@ from deepspeed import comm as dist from .layers import LinearAllreduce, LinearLayer, LmHeadLinearAllreduce from deepspeed.accelerator import get_accelerator -from .fusedqkv_utils import require_tp_fused_qkvw, prepare_tp_fused_qkvw +from .fusedqkv_utils import require_tp_fused_qkvw, prepare_tp_fused_qkvw, shard_chunk_mlp from deepspeed.module_inject.tp_shard import get_shard_size, get_shard_size_list @@ -133,7 +133,8 @@ def is_load_module(module): load_layers = [nn.Linear, nn.Embedding, nn.LayerNorm] load_layer_names = [ "LPLayerNorm", "SharedEmbedding", "OPTLearnedPositionalEmbedding", "LlamaRMSNorm", "FalconLinear", - "MistralRMSNorm", "T5LayerNorm", "MixtralRMSNorm" + "MistralRMSNorm", "T5LayerNorm", "MixtralRMSNorm", "Phi3RotaryEmbedding", "Phi3SuScaledRotaryEmbedding", + "Phi3RMSNorm" ] return module.__class__ in load_layers or module._get_name() in load_layer_names @@ -328,6 +329,10 @@ def _replace(self, child, name, conv_linear_layer): # For mixtral-7x8b, need to skip MoE gate linear replace. if name == "block_sparse_moe.gate": return child + # for phi3. + if 'gate_up_proj' in name: + weight, bias = shard_chunk_mlp(child.weight.data, child.bias, dist.get_rank(), dist.get_world_size()) + return LinearLayer(weight=weight, bias=bias) if name in self.all_reduce_linears: # if conv_linear_layer [weight_shape[1], weight_shape[0] // mp_size] # else [weight_shape[0], weight_shape[1] // mp_size] diff --git a/deepspeed/module_inject/fusedqkv_utils.py b/deepspeed/module_inject/fusedqkv_utils.py index cf087c16da8a..33d36fbfae54 100644 --- a/deepspeed/module_inject/fusedqkv_utils.py +++ b/deepspeed/module_inject/fusedqkv_utils.py @@ -4,7 +4,7 @@ # DeepSpeed Team import torch from deepspeed.utils.logging import warning_once -from deepspeed.module_inject.tp_shard import get_shard_size, get_shard_size_list, get_num_kv_heads, get_n_embd +from deepspeed.module_inject.tp_shard import get_shard_size, get_shard_size_list, get_num_kv_heads, get_n_embd, get_num_attention_heads def split_by_qkvlist_and_refuse(qkv_list, split_size, split_dim=0, cat_dim=0): @@ -42,6 +42,7 @@ def prepare_tp_fused_qkvw(module, src, mp_size, gpu_index): "FalconDecoderLayer": 'bloomtype', "GPTBigCodeBlock": 'bigcodetype', "DecoderLayer": 'glmtype', + "Phi3DecoderLayer": "phi3type" } def _codegen_type_transpose(input, mp_size, codegen_mp_num=4): @@ -93,6 +94,20 @@ def _bigcode_type_transpose(input, mp_size): split_q = q.split(get_shard_size_list(shape[0], mp_size), dim=0) return torch.cat((split_q[gpu_index], kv), dim=0) + def _phi3_type_transpose(input, mp_size): + num_kv_heads = get_num_kv_heads() + num_heads = get_num_attention_heads() + hidden_size = input.shape[1] + head_dim = hidden_size // num_heads + q_pos = input.shape[0] - 2 * num_kv_heads * head_dim + q = input[:q_pos] + k = input[q_pos:q_pos + num_kv_heads * head_dim] + v = input[q_pos + num_kv_heads * head_dim:] + split_q = q.split(get_shard_size_list(q.shape[0], mp_size), dim=0) + split_k = k.split(get_shard_size_list(k.shape[0], mp_size), dim=0) + split_v = v.split(get_shard_size_list(v.shape[0], mp_size), dim=0) + return torch.cat((split_q[gpu_index], split_k[gpu_index], split_v[gpu_index]), dim=0) + def _transpose_fused_qkvw(src, mp_size, fused_qkv_type=None, module=None): # suppose num_heads=n, q(n)_w means the n-th q head linear weight, the weight format are as following @@ -110,6 +125,8 @@ def _transpose_fused_qkvw(src, mp_size, fused_qkv_type=None, module=None): return _qwen_type_transpose(src, mp_size, module) elif fused_qkv_type == 'bigcodetype': return _bigcode_type_transpose(src, mp_size) + elif fused_qkv_type == 'phi3type': + return _phi3_type_transpose(src, mp_size) raise ValueError("unknown fused_qkv_type") @@ -123,3 +140,24 @@ def _transpose_fused_qkvw(src, mp_size, fused_qkv_type=None, module=None): warning_once(f"Unrecognized fusedkqv weight type, default to using bloom type," f"please check in prepare_tp_fused_qkvw() to avoid potential calculation errors") return _bloom_type_transpose(src, mp_size) + + +# For phi3 with chunk mlp, adjust the weight order. +def shard_chunk_mlp( + weight, + bias, + rank, + world_size, +): + weight_gate, weight_states = weight.chunk(2, dim=0) + total_size = weight_gate.shape[0] + split_weight_gate = weight_gate.split(get_shard_size_list(total_size, world_size, "mlp"), dim=0) + split_weight_states = weight_states.split(get_shard_size_list(total_size, world_size, "mlp"), dim=0) + shard_weight = torch.cat((split_weight_gate[rank], split_weight_states[rank]), dim=0) + if bias is not None: + bias_gate, bias_states = bias.chunk(2, dim=0) + split_bias_gate = bias_gate.split(get_shard_size_list(total_size, world_size, "mlp"), dim=0) + split_bias_states = bias_states.split(get_shard_size_list(total_size, world_size, "mlp"), dim=0) + return shard_weight, torch.cat((split_bias_gate[rank], split_bias_states[rank]), dim=0) + + return shard_weight, None diff --git a/deepspeed/module_inject/replace_module.py b/deepspeed/module_inject/replace_module.py index e1703562d180..3029a79698dc 100644 --- a/deepspeed/module_inject/replace_module.py +++ b/deepspeed/module_inject/replace_module.py @@ -16,7 +16,7 @@ from .auto_tp import AutoTP, ReplaceWithTensorSlicing, Loading from deepspeed import comm as dist -from deepspeed.module_inject.tp_shard import set_num_kv_heads, set_n_embd +from deepspeed.module_inject.tp_shard import set_num_kv_heads, set_n_embd, set_num_attention_heads from .load_checkpoint import load_model_with_checkpoint import time @@ -290,6 +290,10 @@ def replace_wo_policy(module, all_reduce_linears, prefix="", state_dict=None): # 4.2 set n_embd set_n_embd(n_embd) + # 4.3 set attention_heads + if hasattr(model_config, 'num_attention_heads'): + set_num_attention_heads(getattr(model_config, 'num_attention_heads')) + # 5. Set linear policies _autotp.update_linear_policies() diff --git a/deepspeed/module_inject/tp_shard.py b/deepspeed/module_inject/tp_shard.py index 79c19b5f1272..6758c7a657f6 100644 --- a/deepspeed/module_inject/tp_shard.py +++ b/deepspeed/module_inject/tp_shard.py @@ -12,6 +12,11 @@ def set_num_kv_heads(num): num_kv_heads = num +def set_num_attention_heads(num): + global num_attention_heads + num_attention_heads = num + + def set_n_embd(num): global n_embd n_embd = num @@ -22,6 +27,11 @@ def get_num_kv_heads(): return num_kv_heads +def get_num_attention_heads(): + global num_attention_heads + return num_attention_heads + + def get_shard_size(total_size, mp_size, name=None, rank=None): global num_kv_heads last_linear = ["lm_head", "embed_out"] From df4ef0ab698a6ad071b4aad282690eac690b16fd Mon Sep 17 00:00:00 2001 From: BacharL Date: Fri, 10 May 2024 23:53:55 +0300 Subject: [PATCH 05/25] Fused adam for HPU (#5500) Co-authored-by: Olatunji Ruwase --- op_builder/hpu/fused_adam.py | 94 ++++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 9 deletions(-) diff --git a/op_builder/hpu/fused_adam.py b/op_builder/hpu/fused_adam.py index d77228317ddb..5acb121668e3 100644 --- a/op_builder/hpu/fused_adam.py +++ b/op_builder/hpu/fused_adam.py @@ -4,10 +4,88 @@ # DeepSpeed Team -from .builder import CPUOpBuilder +try: + # is op_builder from deepspeed or a 3p version? this should only succeed if it's deepspeed + # if successful this also means we're doing a local install and not JIT compile path + from op_builder import __deepspeed__ # noqa: F401 # type: ignore + from op_builder.builder import OpBuilder +except ImportError: + from deepspeed.ops.op_builder.builder import OpBuilder +try: + import torch + import math +except ImportError as e: + pass -class FusedAdamBuilder(CPUOpBuilder): + +class HPUFusedAdam: + htcore = None + is_lazy_mode = None + + @staticmethod + def multi_tensor_adam(chunk_size, noop_flag_buffer, tensor_lists, lr, beta1, beta2, epsilon, step, adam_w_mode, + bias_correction, weight_decay, *args): + + if HPUFusedAdam.htcore is None: + from habana_frameworks.torch import core as htcore + from habana_frameworks.torch.utils.internal import is_lazy + HPUFusedAdam.htcore = htcore + HPUFusedAdam.is_lazy_mode = is_lazy() + + htcore = HPUFusedAdam.htcore + + htcore.step_closure._mark_step_if_lazy() + step_size = lr + if bias_correction: + bias_correction1 = 1.0 - pow(beta1, step) + bias_correction2 = 1.0 - pow(beta2, step) + step_size = step_size * math.sqrt(bias_correction2) / bias_correction1 + + neg_step = -step_size + neg_step_t = (torch.tensor([neg_step], dtype=torch.float, + requires_grad=False).to(tensor_lists[1][0].dtype).to(tensor_lists[1][0].device, + non_blocking=True)) + + weight_decay = weight_decay if adam_w_mode else 0 + + # since lr is fed into the kernel as tensor, perform the scalar multiplication of wd here + # NOTE: TODO if lr is updated every step, then we need to convert it as tensor and + # perform weight decay unconditonally. + modified_wd = 1.0 - weight_decay * lr + + if HPUFusedAdam.is_lazy_mode: + torch.ops.hpu.optimizer_adamw( + tensor_lists[0], + tensor_lists[1], + tensor_lists[2], + tensor_lists[3], + neg_step_t, + beta1, + beta2, + epsilon, + modified_wd, + ) + else: + modified_wd_t = (torch.tensor([modified_wd], dtype=torch.float, requires_grad=False).to( + tensor_lists[1][0].dtype).to(tensor_lists[1][0].device, non_blocking=True)) + torch.ops.hpu.optimizer_adamw( + tensor_lists[0], + tensor_lists[1], + tensor_lists[2], + tensor_lists[3], + neg_step_t, + beta1, + beta2, + epsilon, + modified_wd_t, + modified_wd != 1.0, + ) + + htcore.step_closure._mark_step_if_lazy() + + +class FusedAdamBuilder(OpBuilder): BUILD_VAR = "DS_BUILD_FUSED_ADAM" NAME = "fused_adam" @@ -18,12 +96,10 @@ def absolute_name(self): return f'deepspeed.ops.adam.{self.NAME}_op' def sources(self): - return ['csrc/cpu/adam/fused_adam.cpp', 'csrc/adam/cpu_adam_impl.cpp'] - - def cxx_args(self): - args = super().cxx_args() - args += ['-DENABLE_BFLOAT16'] - return args + return [] def include_paths(self): - return ['csrc/includes'] + return [] + + def load(self, verbose=True): + return HPUFusedAdam From 4696afd27bd54abe0ee313048f176133402e2497 Mon Sep 17 00:00:00 2001 From: YiSheng5 Date: Tue, 14 May 2024 02:26:11 +0800 Subject: [PATCH 06/25] [manifest] update mainfest to add hpp file in csrc. (#5522) Update the mainfest to cover hpp file in csrc. --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index ab79573ef96c..85e00695d648 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,7 +3,7 @@ include deepspeed/inference/v2/kernels/ragged_ops/libs/*.so include deepspeed/inference/v2/kernels/cutlass_ops/libs/*.so recursive-include requirements *.txt recursive-include deepspeed *.cpp *.h *.cu *.hip *.tr *.cuh *.cc *.json -recursive-include csrc *.cpp *.h *.cu *.tr *.cuh *.cc +recursive-include csrc *.cpp *.h *.hpp *.cu *.tr *.cuh *.cc recursive-include op_builder *.py recursive-include benchmarks *.py recursive-include accelerator *.py From 3a7f3aa8498582860cfdaca6d1e19a968964ea76 Mon Sep 17 00:00:00 2001 From: Yejing-Lai Date: Tue, 14 May 2024 04:10:53 +0800 Subject: [PATCH 07/25] enable phi2 autotp (#5436) This PR aims to enable phi2 model autotp. --------- Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> Co-authored-by: Olatunji Ruwase --- deepspeed/module_inject/auto_tp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deepspeed/module_inject/auto_tp.py b/deepspeed/module_inject/auto_tp.py index 4944e1954e37..5e0aa0286a3a 100644 --- a/deepspeed/module_inject/auto_tp.py +++ b/deepspeed/module_inject/auto_tp.py @@ -307,6 +307,8 @@ def tp_parser(model): # Mixtral-7x8b used w2*act(w1*w3) linear. need to replace w2 to linearallreduce. elif 'w2' in layer and 'Mixtral' in str(type(module)): gem_list = gem_list + [layer] + elif "self_attn.dense" in layer and "Phi" in str(type(module)): + gem_list = gem_list + [layer] layer_list = [] if gem_list != []: From 82ce4ae815d6cab4ffea97427dfa1e0b0ff59f93 Mon Sep 17 00:00:00 2001 From: Logan Adams <114770087+loadams@users.noreply.github.com> Date: Mon, 13 May 2024 16:45:50 -0700 Subject: [PATCH 08/25] Switch pynvml to nvidia-ml-py (#5529) Fixes: #5517 Link to PyPI for nvidia-ml-py [here](https://pypi.org/project/nvidia-ml-py/) showing usage remaining the same as previous pynvml package. --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 80c9f9b3287a..05f88337f3a9 100755 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,10 +1,10 @@ hjson ninja numpy +nvidia-ml-py packaging>=20.0 psutil py-cpuinfo pydantic -pynvml torch tqdm From 62ca317829f833c7a83ef84b9ef8533728749c08 Mon Sep 17 00:00:00 2001 From: Logan Adams <114770087+loadams@users.noreply.github.com> Date: Mon, 13 May 2024 20:20:21 -0700 Subject: [PATCH 09/25] Switch from double quotes to match single quotes (#5530) --- deepspeed/module_inject/auto_tp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/module_inject/auto_tp.py b/deepspeed/module_inject/auto_tp.py index 5e0aa0286a3a..3429ceb0a4ee 100644 --- a/deepspeed/module_inject/auto_tp.py +++ b/deepspeed/module_inject/auto_tp.py @@ -307,7 +307,7 @@ def tp_parser(model): # Mixtral-7x8b used w2*act(w1*w3) linear. need to replace w2 to linearallreduce. elif 'w2' in layer and 'Mixtral' in str(type(module)): gem_list = gem_list + [layer] - elif "self_attn.dense" in layer and "Phi" in str(type(module)): + elif 'self_attn.dense' in layer and 'Phi' in str(type(module)): gem_list = gem_list + [layer] layer_list = [] From ebf82e8f3ad6d51d49d115e54a11ae4597ff36fb Mon Sep 17 00:00:00 2001 From: YiSheng5 Date: Wed, 15 May 2024 00:28:18 +0800 Subject: [PATCH 10/25] [manifest] update mainfest to add hpp file in deepspeed. (#5533) Hi @loadams, Could you please help to review this pr? After add hpp files in csrc, we found sometimes the hpp headers will still be excluded from the op src packaging, so we add the hpp file in deepspeed to make sure the hpp header in deepspeed package, to ensure jit load to compile the xpu/fused_adam ops in 0.14.2. --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 85e00695d648..8d84aee0faf4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,7 +2,7 @@ include *.txt README.md include deepspeed/inference/v2/kernels/ragged_ops/libs/*.so include deepspeed/inference/v2/kernels/cutlass_ops/libs/*.so recursive-include requirements *.txt -recursive-include deepspeed *.cpp *.h *.cu *.hip *.tr *.cuh *.cc *.json +recursive-include deepspeed *.cpp *.h *.hpp *.cu *.hip *.tr *.cuh *.cc *.json recursive-include csrc *.cpp *.h *.hpp *.cu *.tr *.cuh *.cc recursive-include op_builder *.py recursive-include benchmarks *.py From 488a823f64d436b3cce8936928780e8825b18862 Mon Sep 17 00:00:00 2001 From: Aliaksandr Kuzmik <98702584+alexkuzmik@users.noreply.github.com> Date: Wed, 15 May 2024 18:04:44 +0200 Subject: [PATCH 11/25] New integration - CometMonitor (#5466) This PR introduces a new monitoring option - `CometMonitor` which comes up as an official integration with [CometML](https://www.comet.com/site/). The new monitor is covered with unit tests. Notes: * We've updated `docs/code-docs/source/monitor.rst` but it doesn't look used anymore * We've updated the "Monitoring Module" section name in `config-json.md` to be generic so the next integration won't require updating it. --------- Co-authored-by: Boris Feld Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> --- deepspeed/monitor/comet.py | 92 +++++++++++++++++++++++++++ deepspeed/monitor/config.py | 69 +++++++++++++++++++- deepspeed/monitor/monitor.py | 6 ++ deepspeed/monitor/utils.py | 13 ++++ docs/_data/navigation.yml | 2 +- docs/_pages/config-json.md | 37 ++++++++++- docs/_tutorials/monitor.md | 15 ++++- docs/assets/images/comet_monitor.png | Bin 0 -> 258147 bytes docs/code-docs/source/monitor.rst | 5 ++ requirements/requirements-dev.txt | 1 + tests/unit/monitor/test_monitor.py | 67 +++++++++++++++++++ 11 files changed, 298 insertions(+), 9 deletions(-) create mode 100644 deepspeed/monitor/comet.py create mode 100644 docs/assets/images/comet_monitor.png diff --git a/deepspeed/monitor/comet.py b/deepspeed/monitor/comet.py new file mode 100644 index 000000000000..d8bc4017800f --- /dev/null +++ b/deepspeed/monitor/comet.py @@ -0,0 +1,92 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +from typing import TYPE_CHECKING, Any, Tuple, List, Dict, Optional + +from .utils import check_comet_availability +from .monitor import Monitor + +import deepspeed.comm as dist + +if TYPE_CHECKING: + import comet_ml + from .config import CometConfig + +Name = str +Value = Any +GlobalSamples = int +Event = Tuple[Name, Value, GlobalSamples] + + +class CometMonitor(Monitor): + + def __init__(self, comet_config: "CometConfig"): + super().__init__(comet_config) + check_comet_availability() + import comet_ml + + self.enabled = comet_config.enabled + self._samples_log_interval = comet_config.samples_log_interval + self._experiment: Optional["comet_ml.ExperimentBase"] = None + + if self.enabled and dist.get_rank() == 0: + self._experiment = comet_ml.start( + api_key=comet_config.api_key, + project=comet_config.project, + workspace=comet_config.workspace, + experiment_key=comet_config.experiment_key, + mode=comet_config.mode, + online=comet_config.online, + ) + + if comet_config.experiment_name is not None: + self._experiment.set_name(comet_config.experiment_name) + + self._events_log_scheduler = EventsLogScheduler(comet_config.samples_log_interval) + + @property + def experiment(self) -> Optional["comet_ml.ExperimentBase"]: + return self._experiment + + @property + def samples_log_interval(self) -> int: + return self._samples_log_interval + + def write_events(self, event_list: List[Event]) -> None: + if not self.enabled or dist.get_rank() != 0: + return None + + for event in event_list: + name = event[0] + value = event[1] + engine_global_samples = event[2] + + if self._events_log_scheduler.needs_logging(name, engine_global_samples): + self._experiment.__internal_api__log_metric__( + name=name, + value=value, + step=engine_global_samples, + ) + + +class EventsLogScheduler: + + def __init__(self, samples_log_interval: int): + self._samples_log_interval = samples_log_interval + self._last_logged_events_samples: Dict[str, int] = {} + + def needs_logging(self, name: str, current_sample: int) -> bool: + if name not in self._last_logged_events_samples: + self._last_logged_events_samples[name] = current_sample + return True + + last_logged_sample = self._last_logged_events_samples[name] + samples_delta = current_sample - last_logged_sample + + if samples_delta >= self._samples_log_interval: + self._last_logged_events_samples[name] = current_sample + return True + + return False diff --git a/deepspeed/monitor/config.py b/deepspeed/monitor/config.py index 5a8ca6ecf5cd..d422d3b1b9bb 100644 --- a/deepspeed/monitor/config.py +++ b/deepspeed/monitor/config.py @@ -3,12 +3,14 @@ # DeepSpeed Team +from typing import Optional + from deepspeed.pydantic_v1 import root_validator from deepspeed.runtime.config_utils import DeepSpeedConfigModel def get_monitor_config(param_dict): - monitor_dict = {key: param_dict.get(key, {}) for key in ("tensorboard", "wandb", "csv_monitor")} + monitor_dict = {key: param_dict.get(key, {}) for key in ("tensorboard", "wandb", "csv_monitor", "comet")} return DeepSpeedMonitorConfig(**monitor_dict) @@ -60,12 +62,75 @@ class CSVConfig(DeepSpeedConfigModel): """ Name for the current job. This will become a new directory inside `output_path`. """ +class CometConfig(DeepSpeedConfigModel): + """ + Sets parameters for Comet monitor. For logging data Comet uses + experiment object. + https://www.comet.com/docs/v2/api-and-sdk/python-sdk/reference/Experiment/ + """ + + enabled: bool = False + """ Whether logging to Comet is enabled. Requires `comet_ml` package is installed. """ + + samples_log_interval: int = 100 + """ Metrics will be submitted to Comet after processing every `samples_log_intervas` samples""" + + project: Optional[str] = None + """ + Comet project name. Can be set through .comet.config file or environment variable COMET_PROJECT_NAME + https://www.comet.com/docs/v2/guides/experiment-management/configure-sdk/#explore-comet-configuration-options + """ + + workspace: Optional[str] = None + """ + Comet workspace name. Can be set through .comet.config file or environment variable COMET_WORKSPACE + https://www.comet.com/docs/v2/guides/experiment-management/configure-sdk/#explore-comet-configuration-options + """ + + api_key: Optional[str] = None + """ + Comet API key. Can be set through .comet.config file or environment variable COMET_API_KEY + https://www.comet.com/docs/v2/guides/experiment-management/configure-sdk/#explore-comet-configuration-options + """ + + experiment_name: Optional[str] = None + """ + The name for comet experiment to be used for logging. + Can be set through .comet.config file or environment variable COMET_EXPERIMENT_NAME + https://www.comet.com/docs/v2/guides/experiment-management/configure-sdk/#explore-comet-configuration-options + """ + + experiment_key: Optional[str] = None + """ + The key for comet experiment to be used for logging. Must be an alphanumeric string whose length is between 32 and 50 characters. + Can be set through .comet.config or environment variable COMET_EXPERIMENT_KEY + https://www.comet.com/docs/v2/guides/experiment-management/configure-sdk/#explore-comet-configuration-options + """ + + online: Optional[bool] = None + """ + If True, the data will be logged to Comet server, otherwise it will be stored locally in offline experiment + Defaults to True. + """ + + mode: Optional[str] = None + """ + Control how the Comet experiment is started, 3 options are possible.: + - "get": Continue logging to an existing experiment identified by the `experiment_key` value. + - "create": Always creates of a new experiment, useful for HPO sweeps. + - "get_or_create" (default): Starts a fresh experiment if required, or persists logging to an existing one. + """ + + class DeepSpeedMonitorConfig(DeepSpeedConfigModel): """Sets parameters for various monitoring methods.""" tensorboard: TensorBoardConfig = {} """ TensorBoard monitor, requires `tensorboard` package is installed. """ + comet: CometConfig = {} + """ Comet monitor, requires `comet_ml` package is installed """ + wandb: WandbConfig = {} """ WandB monitor, requires `wandb` package is installed. """ @@ -75,5 +140,5 @@ class DeepSpeedMonitorConfig(DeepSpeedConfigModel): @root_validator def check_enabled(cls, values): values["enabled"] = values.get("tensorboard").enabled or values.get("wandb").enabled or values.get( - "csv_monitor").enabled + "csv_monitor").enabled or values.get("comet") return values diff --git a/deepspeed/monitor/monitor.py b/deepspeed/monitor/monitor.py index 5a32b8bbcadd..e7e26dc483d9 100644 --- a/deepspeed/monitor/monitor.py +++ b/deepspeed/monitor/monitor.py @@ -24,6 +24,7 @@ def write_events(self, event_list): from .wandb import WandbMonitor from .tensorboard import TensorBoardMonitor from .csv_monitor import csvMonitor +from .comet import CometMonitor class MonitorMaster(Monitor): @@ -33,6 +34,7 @@ def __init__(self, monitor_config): self.tb_monitor = None self.wandb_monitor = None self.csv_monitor = None + self.comet_monitor = None self.enabled = monitor_config.enabled if dist.get_rank() == 0: @@ -42,6 +44,8 @@ def __init__(self, monitor_config): self.wandb_monitor = WandbMonitor(monitor_config.wandb) if monitor_config.csv_monitor.enabled: self.csv_monitor = csvMonitor(monitor_config.csv_monitor) + if monitor_config.comet.enabled: + self.comet_monitor = CometMonitor(monitor_config.comet) def write_events(self, event_list): if dist.get_rank() == 0: @@ -51,3 +55,5 @@ def write_events(self, event_list): self.wandb_monitor.write_events(event_list) if self.csv_monitor is not None: self.csv_monitor.write_events(event_list) + if self.comet_monitor is not None: + self.comet_monitor.write_events(event_list) diff --git a/deepspeed/monitor/utils.py b/deepspeed/monitor/utils.py index 265fc9811553..f5530e8532e1 100644 --- a/deepspeed/monitor/utils.py +++ b/deepspeed/monitor/utils.py @@ -3,6 +3,8 @@ # DeepSpeed Team +from packaging import version as pkg_version + def check_tb_availability(): try: @@ -22,3 +24,14 @@ def check_wandb_availability(): 'If you want to use wandb logging, please `pip install wandb` and follow the instructions at https://docs.wandb.ai/quickstart' ) raise + + +def check_comet_availability(): + try: + import comet_ml + comet_version = pkg_version.parse(comet_ml.__version__) + if comet_version < pkg_version.Version("3.41.0"): + raise ImportError("`comet_ml` must have at least version 3.41.0") + except ImportError: + print('If you want to use comet logging, please `pip install "comet_ml>=3.41.0"`') + raise diff --git a/docs/_data/navigation.yml b/docs/_data/navigation.yml index 217d56c14812..3bd3e451ab49 100755 --- a/docs/_data/navigation.yml +++ b/docs/_data/navigation.yml @@ -41,7 +41,7 @@ lnav: - title: 'Flops Profiler' url: /docs/config-json/#flops-profiler - title: 'Monitoring' - url: /docs/config-json/#monitoring-module-tensorboard-wandb-csv + url: /docs/config-json/#monitoring-module - title: 'Communication Logging' url: /docs/config-json/#communication-logging - title: 'Model Compression' diff --git a/docs/_pages/config-json.md b/docs/_pages/config-json.md index abe314cbb1a6..adb2f1679ea0 100755 --- a/docs/_pages/config-json.md +++ b/docs/_pages/config-json.md @@ -1139,15 +1139,16 @@ DeepSpeed Data Efficiency Library includes two techniques: curriculum learning a | ---------------------------------------------------------------------------------------------------------------------------- | ------- | | List of which step to change difficulty level. One of the `schedule_config` when the `fixed_discrete` schedule_type is used. | N/A | -### Monitoring Module (TensorBoard, WandB, CSV) +### Monitoring Module **Note:** Deepspeed logs to TensorBoard through PyTorch. Logging to TensorBoard requires that the `tensorboard` package is installed (read more in the [PyTorch documentation](https://pytorch.org/docs/1.8.0/tensorboard.html)). {: .notice--warning} **Note:** Logging to WandB requires that the `wandb` package is installed (read more in the [WandB documentation](https://docs.wandb.ai/quickstart)). {: .notice--warning} +**Note:** Logging to Comet requires that the `comet_ml` package is installed (read more in the [Comet documentation](https://www.comet.com/docs/v2/guides/quickstart/#1-install-and-configure-the-comet-ml-sdk)). +{: .notice--warning} - -Deepspeed's Monitor module can log training details into a [Tensorboard](https://www.tensorflow.org/tensorboard)-compatible file, to [WandB](https://wandb.ai/site), or to simple CSV files. Below is an overview of what DeepSpeed will log automatically. +Deepspeed's Monitor module can log training details into a [Tensorboard](https://www.tensorflow.org/tensorboard)-compatible file, to [WandB](https://wandb.ai/site), to [Comet](https://www.comet.com/site/?utm_source=deepseed&utm_medium=docs&utm_content=docs) or to simple CSV files. Below is an overview of what DeepSpeed will log automatically. | Field | Description |Conditions | | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----- | @@ -1201,6 +1202,36 @@ Example of **wandb** configuration: } ``` +**comet**: [dictionary] + +| Fields | Value | Default | +|--- |--- |--- | +| enabled | Whether logging to [Comet](https://www.comet.com/site/) is enabled. | `false` | +| workspace | Comet workspace name. | `None` | +| project | Comet project name. | `None` | +| samples_log_interval | Metrics will be submitted to Comet after processing every `samples_log_intervas` samples. | `100` | +| experiment_name | The name for comet experiment to be used for logging. | `None` | +| api_key | Comet API key. It's not recommended to save the Comet API Key in code. | `None` | +| experiment_key | The key for comet experiment to be used for logging. Must be an alphanumeric string whose length is between 32 and 50 characters. | `None` | +| online | If True, the data will be logged to Comet server, otherwise it will be stored locally in offline experiment. Default is `True`. | `None` | +| mode | Control how the Comet experiment is started. "get": Continue logging to an existing experiment identified by the `experiment_key` value. "create": Always creates of a new experiment, useful for HPO sweeps. "get_or_create" (default): Starts a fresh experiment if required, or persists logging to an existing one. | `None` | + + +Example of **comet** configuration: + +```json +"comet": { + "enabled": true, + "workspace": "my_workspace", + "project": "my_project", + "samples_log_interval": 50, + "experiment_name": "llama-fine-tuning", + "experiment_key": "0c4a1c4a90664f2a8084e600b19a9d7", + "online": false, + "mode": "get", +} +``` + **csv_monitor**: [dictionary] | Fields | Value |Default | diff --git a/docs/_tutorials/monitor.md b/docs/_tutorials/monitor.md index a9c111f8eeec..572e3f4558a7 100644 --- a/docs/_tutorials/monitor.md +++ b/docs/_tutorials/monitor.md @@ -11,7 +11,7 @@ In this tutorial, we introduce the DeepSpeed Monitor and provide examples of its ## Overview -Monitoring model and system metrics during training is vital to ensure hardware resources are fully utilized. The DeepSpeed Monitor enables live logging of metrics through one or more monitoring backends such as PyTorch's [TensorBoard](https://pytorch.org/docs/1.8.0/tensorboard.html), [WandB](https://docs.wandb.ai/quickstart), and simple CSV files. +Monitoring model and system metrics during training is vital to ensure hardware resources are fully utilized. The DeepSpeed Monitor enables live logging of metrics through one or more monitoring backends such as PyTorch's [TensorBoard](https://pytorch.org/docs/1.8.0/tensorboard.html), [WandB](https://docs.wandb.ai/quickstart), [Comet](https://www.comet.com/site/?utm_source=deepseed&utm_medium=docs&utm_content=tutorial) and simple CSV files. Below is a live monitoring view for TensorBoard: @@ -21,16 +21,20 @@ Below is a live monitoring view for WandB: ![WandB Example Output](/assets/images/wandb_monitor.PNG){: .align-center} +Below is a live monitoring view for Comet: + +![CometML Example Output](/assets/images/comet_monitor.png){: .align-center} + ## Usage -The DeepSpeed Monitor is configured within the deepspeed [configuration file](/docs/config-json/#monitoring-module-tensorboard-wandb-csv). DeepSpeed will automatically monitor key training metrics, including those tracked with the `wall_clock_breakdown` configuration option. In addition, users can log their own custom events and metrics. +The DeepSpeed Monitor is configured within the deepspeed [configuration file](/docs/config-json/#monitoring-module). DeepSpeed will automatically monitor key training metrics, including those tracked with the `wall_clock_breakdown` configuration option. In addition, users can log their own custom events and metrics. - [Automatic Monitoring](#automatic-monitoring) - [Custom Monitoring](#custom-monitoring) ### Automatic Monitoring -When using DeepSpeed for model training, the Monitor can be configured in the DeepSpeed [configuration file](/docs/config-json/#monitoring-module-tensorboard-wandb-csv). No explicit API calls are needed to use the Monitor. The Monitor can be enabled by adding the following field to DeepSpeed's configuration json file. Refer to [Monitoring](/docs/config-json/#monitoring-module-tensorboard-wandb-csv) for details. +When using DeepSpeed for model training, the Monitor can be configured in the DeepSpeed [configuration file](/docs/config-json/#monitoring-module). No explicit API calls are needed to use the Monitor. The Monitor can be enabled by adding the following field to DeepSpeed's configuration json file. Refer to [Monitoring](/docs/config-json/#monitoring-module) for details. ```json { @@ -45,6 +49,11 @@ When using DeepSpeed for model training, the Monitor can be configured in the De "group": "my_group", "project": "my_project" } + "comet": { + "enabled": true, + "project": "my_project", + "experiment_name": "my_experiment" + } "csv_monitor": { "enabled": true, "output_path": "output/ds_logs/", diff --git a/docs/assets/images/comet_monitor.png b/docs/assets/images/comet_monitor.png new file mode 100644 index 0000000000000000000000000000000000000000..83564cd5f1eb7812badba79e0eb15f7d6bb6d5d8 GIT binary patch literal 258147 zcmdqIbyOWs*DeTw;2PZB-5nAjNP;8~+}+*v1PCq%cMb0D?(RVkZU?sm2ZzZ!ckXw8 z_kF+lZ`Q0?vsbUxRozw9wW_=Je)h9>hksL&MMoh8k2r?&@LWYzAXt?_g)f=3?S(W@hhV>EL<+ z*De0H5#zraNjaMtxmr2cQ~t2BGlMbl_F$#t=A*Rnu%hJVfjP4EB2Lz2HjN!9+r0PH23nH zLbJ$~9lO@1HQVL|kU5LDOWrp|#YsA}r5<$M4+{A2i7dA75Gmp6R+6}%x(uqluF{33 zMZrQAQ=%3jkSy>OGaDP*XX!mD@He4|L^zy zRTIMbgGXy4_p`E#Qd6MtA?vC`Fm2hwxX83tEwG_>RhD;UVs=K&)ppcJ3}E(@!D!$2h*! zEe9cEE*+B^E`-(+8 z^vMH`s{OmFcHM|}^m|3hBf{ETk)&4B13omdYY+T}t0WX`!#wlQ08yAbytDg$b2cCT zjiP3$w!01dL(lvgN+(!{1LysDrVr_<`-y|XykTc7!~4Tj_5N@!hf_a?#YYn0Bh=2& z-WcKiWc_>)dl|d=vLxRYhIs2h4m{W;sU)lsJPDMHFT9WbL)zin+rI>pxqlXu~_>_|H{>iKa0(OT-n!~lQmAf$=b;e5>U;yA$YKKK3Z32jYis5IA3641*>ikzHBIwF5 zTbnRz#l0%@JJNYIY`pKH3xl7qtRxB`gG=W2#KggE(e=x5XRP<$sPDnPAAoalzO0ybJ{MN~Iucp*6&4=xH_@I;j3E=0pZnYF9#Q}ck)cxlL0owl-g@U}CoF0+Fu z0DMTYwiTb-7fUc?|w=2lem!(SiiU%W-bx9K|Cr4D4`Zaic zEZGXo<-D<1M{=Q_Hn$_+I9v1QnUXq8&&IQA(rGjHFV60@+;(i?8?RC2x3?VBG`lc< zpdau)dZuQksxv)jd7>@qC*bzvIJDLVKKHGy0}4%u4U4w{?k7}^ntPp-Wgm8EwU+#G z7yW?#YNVh<%|*Ex2rkeu3#>e}dR9R2YjzUzw7n_Le<0&fJkG6Rce-@r&PMF`M8{e* zW${Uvys8evLPV?%Pb_4~~+h}7A=&U?}WHxa|s)p!47BlhC`KNY64xrm9BHGTC5 z{z?S~Xu9A#n+|fdk>ni6W#jwpb_a>0cD-NC5LU~f>e387qLt-*>MY#R36#Qb_QbFR?qg(OxD_Z<5pwvH%>J-94GkNMKG=0^U;Rv!MOR-MfF9 zd7=3Qm!k`yiAXmb8w$7lYiSYcYENF4>0U;7m{<;!jnvfal&$+vn@O;PFWFD)Sy$H` z@CNO~ZLGuA@2C90UJO^#wg!*UJLlAS{iO|5dQcM^*2h02JLcW=jCN(I` z+T`G}h(d0|_2)apd$+?WzLen)WswIK%{^Gu9bcDk)~y%MruxtMak6KCHtc1o{uqw! zEYma?1WMCj4OSK=Y8#L^#*X56^2U})z6lA9kPk+pndj-#sNfbgCwCNn_B4l^(OpRf zaqS1q2D&=DQI}5sr3Cif zxPph~n%Pbt?S)t{)5Dd*=vuT5RBI(p-y@DJYrIqRW08gApTDRzP4sS8WSt7XV|#wF zez^@CG^bgv&8hSmV|F@U>7R~Kmu@W0gOu+Imfu0vnH=}+j6RL&BSl+i3cMw(>hCsX zNF91*OME5*t+cs_SRrJ*Wjmr&BwKy4F|I1XMMYa3WP zFVf4@gL_wUfq!o2>tR6?S(W~78aUyL{eE<-0@wQFcT1ekkraZ zPocQolCg)vG01dG!gy8OGlui+^CY}C;LV(^?Ha&1#w_QrGhb*fsJx-)tfPVRcB5Qw zuWB62$@V~kt3C|A7;{zUkf&}8Pbz`z5Mpf-Z$`9ud_th;((>|0?!LFKI@Rpw4id;) z_ns5@gG6XWK{QhL2kw;l{3967XXWC$*r&aWiRTHLG_j)ZKfI;9ISY5RSP^2j!FB{~ zMh&al_gqVgA3_jnfO^mZlf#Y3H}JX~RkilRFzIL9b*5o;g#Q-F*gVDV%6lOXkkj-1U1^apDN7_weE@&)wH{*q5%yFjWd#?Aqob1kEC9x@0c91$a)| z8@}GYL(rET zn6-^3u&b`Iz%L{cYo9ICL8IE!AXbEH`r?XrAY{pXvC$@C(iIU7E1^2KFy4)Z+vu_uRD@m zPa5|F4b8H+V#~A4*DiAEl`vS0I-TVk_S}Yw;kq4vTsbvl&$Nf@C6l~duS?jw z9d>LA&+i0gESw*+aPNB9W93cyr&NB3laTB5lPyi*SfczW1gWA{i^P z7mb}_*nIh7+3kOWbVJ`FMG6<;?D-TLWV#}P^&}DMwx{6cx!dfUWL0<(CMenY5+2e0 zGu|&Jq2Z45ePZaJi4FKh=ki$XmP0mKnKaz*e!8M*2UbpB}+!6(D_SPcZSpC_L+Hz^_GKv0dbi0ceEL{4?+3 z1^iTLWoD3GJLdKc#_wo=of+{L@X4w^!BdQ@`bE&CI01Isi-?Lwal%5ANFKQ=&0?p- zv9^9~#ryT93-j5cGt(>rgOfjlO3fr+K}gh5ELtfleMb4wMn(wB3zui5tluEuc2&HM zPLXwC{N4asm2Ms79~t^T%2IDZ<{*8^}H6E>===sjIBe^V2DYkK)iHyl8( zDH>$dc*BPNM7jYJasc1VU4PyK3l)(Z1KJ>JJz)ejM=wso0GT9|mz-$+se%DfJR$`M z_wHf&cTI4#7v+`}k*85FxLSdSFU^+PDF!;MW18c??N)5X<^t@q@!&&dXB|XrFjB*b zP5UK{&~2O?Jva5;y)<-7c(;~>!|+iu!!kb;&v@L+&ajCFeYVl4>sb%51kk)+X%{@? z|6t26i3;$IY_1VDV;f?xw35fxX(o`YS8u2=fv0|e)Fbc@b}=XHIi%0d*PZ zZX(&3QTBhAUOb1xRr~cPY6BirrUVLa{n@JwG8H z+z#ub;PKy5h|6Z;Q`K5&6&u^sym>g=#VorJ0&#<-`!-Js`i-g2N3*BvVSoiz2}YIyMTUpY>vGSwgA^UGh6$0H(5)yI3Q#!R+vZAMB!IS z6%_6pS>q^|2&l!)h?fir0@$$Thl&2Grse%H=6yxBL>pho>5Z2aK!QageYlq z*snDHZ?8H`qW6}}VCS!?d#Pt5MQ|iFJURd1+s~0}WBF|#fj64cPDfaW2jysS3i9yk z8gXMWsjO}+Wg{Qgf+}y&6n_)(gkaF7=OpA8WX;Y_4Uj}X64|MgHdsRhA4gY!8Uo(a z8Cor7hGlmif@aZvaXM;xu(CAQvzsw|w&(E=Fhc7%d?tqb#AZc;`C+N}RI{N0oo3DR zXB0mFZ|hW_ne5Ks_t{ptGv^BADxJ9kS<{#%x4*M0KzRK%?hH8)Xa1@^NOaO2oR9w0 za!bMz!7^QTOBC*$E}y8xdYA)5UHa}wc~faIA?G>c#cg_tuaiA!jbz1jAmvb;%+d%Z z(D-oOo&lBLo|dhRVivu$CHZBcUYkqx1Y_X+DJ>scP(PW;Ny&_yB_fGG2#T{nRkDZ~ z$472)R+dr*eK-efZw7o?uS;t#vR$2qbzCH8DgM29SI!mImEsn9(Nx24#^D|$5Rg}^ zJ1bC@T&gVgn5+O$Ko?BmPh{duW4 z%F619VjT%PE@Zjh0@KLI2;j=|PyPL`@*b|`DaAiK{*Uti|7o%R2bjF-^>&Yx*V*WQ zCo-|I#m^a#M!}B7a8|$ z^?WZIgI{k2Vz+NKI<%^Mipn~7*&Y%t0%Q*^3-AyGk^x+H-1 z_tDdv&%AJ;&5=+cbe8PfMaI6+4!I!*-E}QHNF>^sJB`ByLJefFGVaFL)NF+t{Du4w zf3#q**yMAtt@nj~2sDC9lsY7PX>B}|zg+`8CSdluK)hm<21Ye&B#0^aB}R?23-Hw)V)Jh!5w% z`e|MJVA=%063uV-cX(Kg$<}1OX4_JJiw#`;qbwHow%19oAB$3z&|N;)O=`!_zUy0C z&lC2la~xUMAG~7hIU`?LoNcU}_#yUTG{1+@)LX4Lt4=@j5YBhkb+w+LlvhdY)6wg1 z1Ra(j{pKJY({r%Ptx0anjh(#M7+fHb!NJB;1eTv0>w7QE1)m%^FI|#L%gWC0Ubm+{ znVdDQ5Gc{Ex=?)nJij}ak=DuEUN^%5A;iJ^G%(U?y7Xi8Ulp=X1N$6tT2J=(e*0wL z1Tad={-W=YB9rO4GeZpX`?Uf;r(dOSBj86JNZO+=mzA10Mp7N%Gd5509*_Mswu&=#e7w9e z>?bXbU54dsW+L_*nAUpMMx~Bsg#r&-x}v-+VOP{Po>~h;J$T^S)o;Gm%Rdxm3q$zp zZ<3w}U5@-%D>s0rdSewCh#v^ZKDjOhtUW8Bw7T8!xixY?lJXWF`^Ws9hRReoaQ@fF zZGo7QfqBk;Fcz28T$}L4(>@^b_k3f_I!4iR=dZ!rm+ltq!fy8{7FqJ$eTuI(`r>zg z%_{4`5rzMga6~ew)qT#XY3UYlio#C7@z$AI{MPrD(?lUL<&~A6>VBsv`u0bx19GgO z?+q14z>wQo$Hc#0>=+19OS}^UZi)wG*t2x6zY=e;n_GE6{ z1KN*j9Yvv*lCipSzCc*THC@8=LQe(t3jZFb!lvLLzR=WHFMLE)vRe&WGmMV$5onN? z5r|?v*4mc9d401w*XQGI6+L(yvE?<*)?4D9t8`raU^`WDqT_{2{YPf0NB8)ijzodu zVI*xKJ9rXgN2}Z!1QuBOu{1AwYj+FtnQ9cc*$WYAzrHLUm-mLyzw7D>lCRF>x%CXW}NK+sb& zya9f>W3jL1JnA^cv#2*UqDPi`GJ5YU(HWWX)|qnuw83CC`XoAI#^FM*Y~qh@K?PWx`h zb-A~Wa`4Z*v=u4jsQ70bkH@;3b;mO)u^gm5D6Q(Ak4Ke)u{3PK!6{;^tkD&*iD@`# zS!C1s5LGQf57EUxR%@@$X4MH#N4)2cz^qU81o7$jqWxKr7MtT|ZT)=R&xXOm_g*Q4x!vt`U+=i^)5ivNaV8Od85u ziI-^!vM1U90!(|;EPHkI&rd=QuVSFS^%S4l6B7|-4)>+Efe<84&Uw70Pxy>i^!tG)1vS-O9);7rP>PRt#ZY}@Uc*rX~1egq; zqII+|+&%9is8=g8Ftr9SiJPc;$7D=Y$kOxJtlsM4#VKEtTwJbX53S=1@zEgM(dX%) z{XL1Fx#c)fH4=0Q^;AhU8hqMt8<|4gB>hW?(nj~KCe7l=&a51&EoaA=Q$Bfqwso|mG9*J8W%Mo--P2=KG%D}>0X;s~McbH7~)?LkST71(gh3F!Ss zLm(k8N39>7P$)Qpx9_G*O`GLxvkQq@Bh^mrJGE%SlHl#Kep6FC z_$|%vk=k=Zf)j=oX*AQB(3*(brNCD10114sJ?9Ka^SGg*XN~{G1>|PRi&ZP-EAVDA zT?oWwl`7`+`-aw3Lm09ylKiVfjg^VQkKJQ-xBx!N?BJz8>;iYXMmF2j_an*$TN&H( zJHx?6s`wFToV0s{#d7B~4;@wLz?~|yR8SF&Eo>Ssd&x$XZ8E(*J?Dfv*#BP7Y(;*Q z+i_(QJkyyKu9R}`Y=WuT`YdRs)uqB;+|D_i)9<#f2;bc3xUOdI>n^ygdzByGZ|}T@ z&}LvvRaS{j5kw&Vitzj#oz!g1V1r3?1OjOSGS-w-G?G3D^o3H@(ytC>^`9*fjv5#- zRPKOs*A=_^BTFC?t`SV#ZoD|Bs*48=co>K0{n0Oz4R#jWZ91J+?I5Kwdq1aVJ>2S@ zaZ^J7*@l(L|CtML=ZfG~hrMFQI{!v4b)XS$>khkDXWGmBt~zZh(LZi^_0D9@&jeXF zVf)DDv z-Sn0a_wVotx2A&$Tpk*7N6i`))>9|THKe&+kLMMLy{X7(sKZH|5gi;V%3p#j04EqB z&?u@7KT$Xbzc{>57EsdFQBc&WjyT)?jELtgB5>9Uqk1n`2N7qf%lOME$7q>_hFK?H z#HBrE-ET`AW%QcayQ|y%$>kqS(R$S5psO}nk+CM5J(_Ar&*7`9ErtAAkBFy_2|tT| zM{vM8H8QX(Hcz#?qRzlMxpU;6%Iqo6p z!6zwC?5yJL&ZHtLW2_;vOfOw5RzA5?EGwUd1~Y(+26^?ChX|dB{D<`G50n(~-qW*} zG=HBZ?LehgR;`+}d!zkN7$yYvo{vmJ9(Rm(b%woxh149bf+@Nc@QXZkrU!x2&QItJ zy@)-MeZRn7&aMTrom+pKS?xTqijcgDMecK{rp{dySjs0ki*WPQx`*|uU&Ydeq{@ki zaF2`QBIt7mgDFV$TS+gQel{2Sg8VZFd#W^eJP>~-8M!Q8o&_P3pFxIU2&%y6$UrGCOVX2D0 z0cb?_=r@gNTRsoM=?}Q__WULO)?f-rJz&j&U{9Y}c*mBW+5^U2awenh{y{`l2%*5R zet&h+mzBC_!3QqImY(YC+836HG&ckB-61f~QCvRj>i1Wca$^FaL49~cpRAJJ;?8jX zL3Su9GaxrN(YV7>f49cg-Wl_gVNG7$3Y&uTLrQ#Y@r`sEn*rH=FH_RVn>K9IyoCe3 z$SJ?EusT$v3Cg8;XI{62+)_6>@ynfy{6)5Xs)5Hmz7>@+D#b{rpg_2t8ND z*Pc;WzMo2}a@r7l6R}6m$k}&Q5z2uJ$K}<23or2_!%X}#F)?p!K2rb7JXYXiAHRPn z+*8xVVRC{@+R!O<_U^t34mFy@a9v&fMGbviRen{^IrSnj z5Fi&F>l6pasdN3~`UmU9-vk@~MY zuZOa%EBh67zK`YMqE9m>3ZBPL*A(+9v8{VF=4)?+IOq2acj9+=<$$XW%578SYgvVE zFcb$YhBNZSkgT#owqSH!!vIUe4{d9zl0fqv#nIY|t^)lA9zde5^vg{wLb%g57AlKFDYr?yvv*AYuWE z)CGXRn)beN1xe!%H_0&Z#T)&gb}?pFUHLXAT$irY>4qvd{wLGmaZlF?u?kB+1_ydJ zi|q>7y)Iw4-A(I{>5ImB)Di(aDR>;+{lXxms#EQL=TlR^mvZN@nkE#E#;(�ltcyWbLcV^dLhi*!VUv*hY}gUb3OzZsr8vL)T=te|FuB*9{8-{ zh8)Z%Cve7yTqTjRDEldNzOBr${%|H^=b^eIBj5YOs~L@J)uGi@y6T-b+`$J2KXV$#c#{L*LK0%z6CWhh#hZk<;wH}fTY2x% z(l6@MU=7+co_EGVNnz;DuQ`+v+6G9RY#FJ{449Vn3RvC0{n=2+q)7F-zn*)+51f># z^7$=Py=h$!{eNRye`8FW;RWQkr}Pyjrwwrr!;%r%$upk!>gNE``_mPg1@)F60eP{@m4Efw*)t z?W8$;X}vr!7kx8T9s1RFzqMdunpreE@^&48!Z@(|g1BYZ3oW7%t5$!fL9l_P|TD%Y~3GS(JkeY*haD&v+&n+~~;i?Us+ znI!lx#Ph&1I?D9kz!K#n(`KhJI7pWUC;QSIx=!y45S)S-B)!4CYB0EDyTszy?aVT; zu4TiNpQMUI4CkD(bP*ezJ}@XC z@kx>c4uAR+NU?aa7vj_2uuZ=#;YkCr#D?pPry#EMI#V*vecS-*MoSdS`u_~uC`Gb|_rn%?%lQbb0Q32__ z>y;COtNHM8A0_Ci=$AVukIUV4}-dX?g=bNDdNsGfeDBs;IX%4eV@Uvu`)DeQsfL8 z%nFZaO%)kN@0rFMKOD9<`lvvgg}6EDCU$CS%Oa5h58PUL5AlMoR{d&fntfRPj#gtH z+qGaKagTOTB`Y&4LLc9wnz_ScvEA<5!$|IL3aU+ZbHTkuPe;OE2RH3?Y-R1dK|3g>% zzON5I=c<8r4=<~HLy^s5O}Aj^|7`Zsh0q5g7JBE+En-m6)3rsR< zpr^twoY6m-%n@_trgRjpoHe;SWi<5Cg%-}WvotmkOpt**;!^vh@XGsYvG?$m8|Rzy z^Ze6*GiscMve8^_TYe>jmN$XZEVlI9nK*yu9gMP`A9by^iyyjGjvpRGe6(njufNoy zM*hDvV_bLUvmM69Ls4Sv27O<2k2r%t6=Wwp+&jfXfe%Y07g>PLb3q@V!~{<$cW6mDV!qP`nq2$S{+#n-&M|?|gQPgdkqoYOzl>fP zD43=1o93Iz1K?DJGnq}^B@tvykSVS?r^PvD1ZbPU4O0~~hv!zZcd=0{QN1A&98E^u zgd?=e_OZU{FqEdIa&Ab6x}g7Zwd*_6AX#o58FeR!J=@7x*GJ2q`4#ya?Ux4^&(Rm) zilHt${&REUvP}l(eaH1#$A;YeB>Un^Fk+#Qt6r(Q1=&Piu2#I-p_@F$?gZGMc z6at)Gf1psRD*ti7w9p1Ai_oBr6m@03zh`9L)|WrY3@4@VPZ%9s4Lu^@1(eOUVmtK7B7%dU+Hj$5 z@^1iS(5kw9+Xq@<)H~O3jnXm*dZ#2p_fw1}OL34TZ9^pfh~dj}E#g4z$*-BmGU$;) zMP+P~4{F|HVM9NWsVTT_=WPf1t$Qr=Y-);h0@izdM9)dU@og~Z;n+cn@UppAm)~6a z;j;&c#s+u9bg}-|O>_dD+-Vqg5~uh1Is*4!{FBZwZH>TZ46;|-%}1tquuw@$P`NG2 z#5GNy@pTCZ6eyq`_{@Cr*Ew9*60i8phjr4-!FlvLSTR142QLDZB-|mg<@C}}vh#@W z-$=yBlHC^4(KlT3l%&|4m=4XeG4$h;c3vOEIXJt+x*d!}yYkcrgTM*A z$CO$&8~d{i440OBAIQ6AB&RHZSAEs_o{;1b?PGSEAJO$JZ|^tts4XhVAgNbzNuvxtTVRQ zWYnudc+Jhlcbi|OaInD_t;RL~aC0!$#e5e@G%-ptsw1$p2;iHMKWV7nUIC@Wc`;Q_ zP^c?A(#F?v&~cSGa_ER>PWC4h%?e$4A-sn<8;0x!WwgBH1xPEuw>gz#7u)<2!T{*{^*Q1GWYMV}UP(Z2^|4pFL0`w75x0ctyo8*LXalm{Ab~HI5|bHQ z-T><@TBe21ph#a!o$rfh=9`}^acnA{-^#~P&@Z;jvJWVPBC2V=*h;B`;?f33@U?S{V-=k&y0TTt);a9Ct8)(SJ)KK`VDADiJD__X=OK4|qDAyMV=KPg5{5M{zM1pP1xm zGTA2MdQ*KuNsyqQ^8Blv}B+xG|C>y+PZS$2$_-Sz|VuT`&*U zZuTaUh%&pviHXVB@Hk&ra4xid?KLk@E(!XBXCI;mTdv-M^yRWBXhre&@LNiK{a90J zzHH&AGxUZLk9r7_>+;EgAr2gw7{?`%Xw!CBMrY_(dB6kVQkU*SO}`zb1chL8*HCsA z#!I^Lw~weW&Y1hyD^<3L__*YScAA_pbad{|6=o*)t?Tw^AqY`%`(rc@D@3$XE44<@ zVz8SD;X*I`UY(Bt^84=RP|(s3@xg7!4|<>x^9|Tv4w-vPgA z($&Syk&6@1#n*CBdClIcv1S(Bjb(n<2_J7Jq9FQ@2J;hm>NfLW`%QZOhD8Aw0`!q} zHhn^nFVt8)b{rdvC@IP+Za8+LB4(GLGipx0_0J{oztP_k2&=K@4jP*KaVP0#`wZ=P zT=N3?nDYvn;yQKRPmAQLe74c;qoFNG7B@v+Ha&mRKnImOW9Z}e%w9wW6QonB4r`-; zjn?2Pd?eCmL3uHI*gk-X(h$9NS)p zKm_@l(4e)boo9OOUA09|{l%3nfJ9yW`r`$O^5$g<{Y3)s2g4r$>M#Zy8+NgF5M3Rw-uXG|pELNdzhuOx8g7vl9zE&i*^&0n1PBrmthSGrm&U)Y))HtD zlC(#i=s#sM1WxxxWW`epaz!bBr*hB7>)wo~!j0om8Ov~4j@1}B7|%~b|4G(P#;=Pv zIhP?pceFvc94}{VB$2I9vOQ_g5wMa#ZvOS6`2Hq&q%EI%NRGPS$V!Rb5blZjfI)}o z)m_mDD5*l;nkDp}#_I$R2$N;QbJ3YBxqYq{6`rB-YN(arAPzw82Ul#5`e)y2UfLkg z4Q9WTk@;|GzLbSXId`{2hF|EUUiT_I@@?UoiYXO7^Dp{yr6K;H+GMg!Z>5-gB z)sX9Ki(QjMLSi&LR|_|FFQNGXo(HDoC^PgT;U`e|gAoiX%`CJZqjS5j*t--)gYmvH z9zHP!Rm7f&2Fa+v`3h(E->hul5+oW!A;x#lk}84G@We393Dw>1v;L_`R99v>wxAde zI+HOje7i7L zl#G^5OEVTQS%Q}%VcxEXznmr#vfV$xXkOK?jxsg5P!k?BsBSf2-FN#E)xjI0qW-Lr z%N~2lCg+&8=s)8LulDcw>ouaZ()u?f_88^NDd&{_7r!PAgVfr9tjfXlF+38}T!;~c z59`MjKvQ7L#K_si-v2J`1)6M5dXM6x@wL zHpM_oTk<*43c zm3j$Eh&2G#F_}z;d|D*wo3@)<2)mvVsSMz8jW;AOFFTLmFL4ie5+(tK^>&V~H3CO2 z`HMTxd92%}f4NT{bKH=O@h8%)$^7_EFXT>=XUAe60N+@%h|PG}vh3%HlTlZ>ENM-6 zMnxIVXv0s`iVtkFgMW`;$oFY+sT%U9nCN~G(|oOSn)59oG)(UeBgX+gg8yopr#itF zSa7${e0llR;QFA^UHqQ1)i!-jbz8w=-TTn(EZP-=mEwIx$&gbIF{{a>cO7-8snVD|%@inEzTvzuzt^9%dM7w7SgwZZvp^x36<^O2!=dgJJ)3Yvju_*uH@2b~ zNN+?}FYe%gN4Ldo-F^Xm(t7-PC*jwVUeWeTlcHM~Tdsz?mo!Ck)eLjH@jXLNpWm*Z z)b=Xk9PcDmQi|gnXiHC_i?qh9{6kY(BTu{Il4>OQtihjXlrc(D-r`9Cj ziuH|ri`#@-Z4VSlMcP5{wnCuMEcEU>WZrxH^puTVnvN}dPv!A17?5%;*|K5NX9 zqy-Q#F6deA;m~M92PH*+wtAZ<{B;9O|NfqYGeuBNPv?t5$HPF1Kxn=#^EA#Fz{s^@ z15%OEus9x z<5R^o)SK4%`|N*;?{wn{wiB73dPitCAl;W7oxze*Z-OHdMtj}-H<)XTFU67bDFV!; zwFDG7wlN6K0rLRtX#1;6O?dhLffWeQsSTR5qX ztrV5(bf1cDvz13cWNx;M@PV+`n-qZvh`6;Rq$7&$530M`dQp%>yO1p()*PCp9c4*D zVrQt0mb?ob|E5YE=5FmYT7#uk&mD&_vck}m?F2o|mDB9$S|^Z3HZnXqoVI%lshgag zlCg$W&keL$Z9GGVoS2;K%n9ZUs60k9f%j5B)Jk956)wLDF38QdLRxmK=W|S85$3>ay#$_6mIPRned7D6Ovf z`Ev9JS?-JZN|ea`M>1_Hn z%?($~3-dD_B<=Qe^7%T?>(4^v%65v~CLFiC5BNg#{+VThBdsr$#s0uyJhv)1-a0EJ zdmPhb_u-ZeC(S^ADiE;Qp9JvF$e$8Pc&=1$8Q+`HWo&yhADYk~siQ+pyS>bL0Hq9H zzW6Iz#hgC~RooAgt=*rBifgG%Ju`Vgws09qU1{ZM`P2+_IDWJMXptqYq%lTpQDkY! z`k8{_(?lOJULW*@auTD!P}$8Q@Jz3(LtK7JgNR&&X?$AahO)UfgpWE@&ieyz;$P07`Yc|({nidh+-wG2!&!2!&7Gw@y^N-D)1hA+T;SmPvMM)q zr@Rx3NpTP|e878aa?`1>@2Mo->a#<&_Zx#prP;F4Y8HV}9{`C?&a{Eva>cHbzZ69)N8s z_4^Ux`7MY!n#l&_Gs+#Cbu0_awW8ivFI|t2@lCuaeJF<7`1{~1Y}H6@O#=RKKJ_D% zr3|Kub;StuQ^g2dk^pAZ79iDgFr81K0*?r?vHhrhmycBO#Z3P*7l1fB6)*p~n@d0E zTYo>#E^QS7cci+&<&}?HhvUVUY|X?TD+UJsW}K~`QaGAoKIqCFiw!Rs0E7PSE;Q#q z-%nl|)#pDB4u98wJz+l2{4?^sm_a~Uh&Xp%9G3a3^}`Z~3%^5a`~|qAx-x@1qKYaf zoSo2}Gte;G%zd`*Joe_m+ARKxAtZ$`m-s2tv2SL#)c;le_5NCNOd_-2ubX1LZi9h~ z@!hxKzV5fUr-&xf>&+eypSsQX?(5-A<(({L7AAnthh}l^P^I9Y^kT-1l1@8lwN*yE z#p_sKzKrvu`Mb7wTKWI;SiT$J3`!vJe7=}8P>|pp*a{-{+CCR9>iE0CrioLXJ=nTE ze_MHlZct%VMqk~VaBXk^A@1MBfYWws*BHsXm-j)p3IL8N@K+K>FCHboP?#rvKvdi+ zzUBw;8F074{-+-hIc^hcciSgFF>(08_fu48b+y?Ii_XgMqAQw{u00K%%P#4fM8M?2 z(Xpd#nD`apof!V^OQDl^7Qy~2c-K{g>(38GcAfxO#G$l%>BP_FRBgONE~<-9e^c$> z1)&D{-uQ}P5(r!#%?Z*0s&@&x!|N<<@Yfk8C6e2qw?Vj4@)eh@u27t@uQ9_ zz(xEjf3Q8GYoW0f-L#TtIP`2)YHyq&fA9+p zYv!{k$5i#-y_UJ-G@mMki9nHs@@wP-1h>Iz-1p48)b^6pVzQy)qiu$S#y~V~IZR48{k4KI~ zn2kx+q1TSr+HIsslcrxGI4*6X8$xOiupt^n#5{O>s@_>fW! zycrd99^Xk6`XJr+UU<5$Og>-OSzM;jxfPgE^kZJMB~Vbn#P*wEZb+R)0}kRZHWl0? z8N9PSiSG0uopu_*R&QccB48)>)_pYg63_Uwr9GbGsm<$NJ`b1PQj_JonhTd%jhvAZ)v z6?L3yE};)XvD1XwtaqLdro+6FyD_(a8>F;<;|}-3d?3#!e{ZS~Hp7-`}W8G9ViI z_J9vS=(6P}DdfLdKz$KkS`HBB`89Egu5hc@S5u(Fn5uVJrWk1YX)1~MBy zKB}7K$7b|h-v0-4Zy8nj?zIUPQbVj4{Mwue8eM^6(L0<6ek)il8}2Ke?+h3Kz9b{) z8~s1@S=*xox8_UcCTjoDtRu9oub#Msd@V3jx6ga4zPR3P$r6xUmzcXAe3iTQj`V=3=qs?7ir>W_DaHsDO z?UvD~Zl0+h*N0CGRIe_e!4QU}cehXCIL!gKO*Ugy!3!7tI;wVVtof(u-S2IbiRV@K zO`^Bu*xuLoP{hw64?bODd(R=|yv3h*gNtmJlUaFO?j>?JVC`OfNu$!RjGUZE9G7cs z6wSQEaec-iG#^)Rn%{2Tl!BR#JVrsQ?=w#4(!@op{SW}{ad=gPz0NdJ`=_W~*`};@ z%_FZs-u;5&>+aF&8OTo zm4ZUq;blC4t%Q&2{{~KRb99-_`WX~TZ^F?VFxanWxP}*K;z9)BSF4}~9x2~b5@onS9S*eMzez75MAFa%!i-*A9plU`86st(+6Zquei%yu2vJqF0sBB2gm~J2EGG#0{ux~ zLw*8ZCVBWYZ>anPd?%B=#043|M3V6y2wo>%G1sdC+sHcw^{k21TOeAFUy!1uGlppu z^#i$$r|)-(P(4{$jYA`>k34xU?Z^`rkk3{#Kj?2_XDWQaVN6FhRhE~48rdkl-ZZ3G zYqt!aBY~^5BGxbWxV>n7SqD8s=SJpQOzLjD!@mQI)mH?D)+2ZxQSdXaI2L4N$aiA2 zag$m?gDjGZ?)7IUE!^y1_?wLese6x7Z+Le^BTNxC4B>gxB~ynVV51f5sf9?(paUlO zg3H%R^B04YlvTV~*Gwc;72#Oe*v@n<8=N{GRL~swG{xPN6}?7)H$qSNi0@B-(0=V_@5ihl!&1h{U}?Xjk>` z30i!^7`|II2mw3Qdkj14ETi85!&CC0aaG!@xj7D^&Kb1Oe3B=xP3@l8LxJC?Q9xjO z{&UiP!AdnIE)JNu5Vaq;6SW)C*D3mwXM$o+a%5uEbPr$oYTL@dmBa>>9pRGZtl}t# zLdS5g6nnX#@M6XxxY2Z_O^d9dAs9cY6X)}^vjKFp{;H&jLYti!Em5jh7yR$wKa85M zt4egsyAL^i3l-#4;@p(>r(z)5;&sybtd7^BBd2|v=jQ>pwAB- zGoBhT?T!VocckN4_HTh(K6HHHHwc9+cPbQ!e0OC>d>YHv+-pUoR=^`Zvj6&{N8OrU zt859YT9f=g9r0IU*8FS9f4%ir&aX%!)_>(FlriR$|0~CwHF)-)2KgU}iouw)f8~_S zVJgJ`E2jbbr}=*tJ#D?&T|QiS=08jQze|L;d=&#R>Hie!qdmaOR4S}j=Q3mdB|Rv! z{Qr+e_-`fnf7Kj0BFm@8d}0;k-PoU&8>N>9cw_&w0-gV-%m4p@M*X*qDM-7v>GOtS z@Z!LzQfaxm`S|hzN!LdD&A075lF3HfGGGt1{29~kSLjZZ#U(NJSWq@B=WGeC%n~3vU){A61e3DN+5R#_znb4V;WKJDE=jsM`%Z(rTU;R13x#P7-sQfuAfh2^kvBN*1k^g@6+Rnf4w~2G+RhE zc_+73PpGaHVCV?{alU2W_E!v50WWc-?P`BFP5&M^^n=*qv!8Mei1W2zx!|wYDnUUAE9si z141}0K{uIBcE)@tM$=_>NfeNW-j<=)vWVZr7lEB+I4L)-5k8$LHp>fBL=&9MLL$w# zh#LPIiOv+X_zGi_vbw;Zw~nXHm0}tO1|;uk_&cRKEp_bVyS@salz4nB)N+GL2HhhK>$Lss%wTM=NVKH6t6PfG31_CsmA zk`9v4L$(}bKxW5sM$cb+N{7OE4yRFj>nMlqKOH!8LEkNLvqBg6b<DOis-7F3eRY$|f!oA} ziIKf=^!qCZB;oMA2m8g~vt6k;p_MqqyC-wEco(AGqKj|kkBJu_yGl1!nY~TISa|1k zvAyI-M-I|%J*|L0tizu%*sk2I#&G`)(ID|(Casf^CfAm$K>nthiG>DNeY^uH%QsPo zd`Icc3_!>*ECl3X4ARbHTV8u7hm168=#SAFoW^g~b?ZB}pSHn{{SJ%lsB`VIT8!Z* z%bUDeXvLf$aDrz2ZwPSp@A7UnAHBgt_C6mi&;xzXJ?nP|Q{S{RM#dSp1tcU1lK)j4X#ZDx+_Lo%x84LN zB7Q2;ULh7369VGpD~j&EL~$^v*c#oidlKb$RQP2t;Mg_ple!zx#JgT^Vd5;Zf5uZd z!&H&}%j2Mkxj$I#Rj;PAyD79q|G%Lni>BIM*^_(hw-vy_yWO~M#*2@gNW@-YrqcWb zJcG@1#&b2PG0|Dcm-SP z2t0xVw8HpY+*x3rB=^!}CgoL^5r!>xk})0&@dazDYOFH#OJjSeg$)fk&jYeGF80#h z6PmE5ha}@y%z+C2)9I(zcxSoWa=RDzRq$R>z_=CvnT#Ud0Ckwi#@OP0JXQ0ly?5M- z|9yHI6MmZ;F&rX6Po#+=w~%j37J#a~xo1)4!g~0iWS!pO z^TthA4UR~Qb9$q90Fm)#@_m!eW5y(ytV8TIaM)4&|n2z}T2%CngyC3(+t%)u(2 z;Ib+@!m@#`^{Hvxcv`y7(4k7VjBUD~&Cgfs`Buh>VI|SFV>gB(FHe?VxXy0yCH{t7 z(6?VaXwIv4L{Z$qWT*+q>-qOX-hs2K5hcA-87K4$J#!u(qnVF5OFLgR8 z9B~(8&?k)-xVmHY(E+W#2-9VbB|J6+I()&JjngCawZ)m{%AX-J5?$63o;u{ldRC{e zY~CTga#r3feL=%dmlH86(uBgBi|ljG)6g9Vzqbb^UWxCzAGB1D zB+X|ptA&JPn*m}kAdWDTB)O@sm^*a zs5g&K;C{8vWuAoZDGd_c_bXye*|_-7q)EJS<_&Mmtvz7l#&hfBzr$Fcc9nS${jJ;e z9xH47SP6I3Q)o2m2+{;d6h_Q12M1G4Y>5aK$;rtKZ;$%>o=U>?(im5r?19(NMr_JI zeycLtOdFDqYxi#}$ILfC2TWr3W}g}o4z`Eb>|{M^9Xhy1Ovn9t(!S%50RFCV9fXU> zd;<$F)6aJVP52VL`R25goIKT+C)D^*<~-#6>do|Ec=3;?1TJ0`5dC6IX6*7vb^{EE zjg0O99YX#TT}y<$T~G`rxtPpanaogpR%40OS>wcc(&^4LF|__8;xt8~$(EDkueYZD zCIM@tBM@?ax1S3KH)58S9-B7cfc3s=xvr)u%#tB9F$M8BV%8$$FUPxMN9SS9`3TXf z$Xt~WM3dd;61|lhPuza_!V-p|^uwJk$Rt|V^=!FhNJXCOkw7Y?#W#$Vh|6)z;2{;_ zXl7|_bDff`DrT#tfQPLsR1_dpRGwSXyPOTAwW-@$m*nGRvS0mC;u1ua8AMF6RTO2tE#xW>aVl zJFkpX+9bwy&}I=FLj&6Wmebp zg$vsIlB)vIhq;YWFiNj0;|Y%vUz{p>Qkv#B3uKa2s&xOHpO~>5N+YH2gFBjcmSk?C zCRZCx7)W%u1CgQ$_8{7Uz-QE$hWtRz@bohSk_iwrGX=ArHYe*Aj2lH(e=!PkI{3Cm zdvdVr?OQsQ;_{8XbO#7(G+47Y^5@pT%9&Abskf%fwSc{UcbKr0mo>KVxl(OAQEGX+ z0j8aCSD}GT7Ph_20=P)MPIo?gvRwRZP-oxdCWYvEd7PccaEfK)-igt*M|!O#6dqhb z9fmxXm8i@+Q3UWc@N#mZz*0RTp&Xo7MCRKOj1;9`;w-H3{@Nu8=(M`;a6aP&kDc`YEVev~c=fl5&7v_l6MFGS zgf;*`_^$NsK=ttAa=6%}MCmgZ66u~qw1SA1=g(nXt0DeGL8$n0l1Ke^JWtEipU`F9 z!#oO;Q226V02%Y?5yuLH{0xz`hwjfMG$!l4hF4L13HWAmuTR&jZ{(|4ILqA8JpnSNpd?o*l0ek_%K&4 zQT$l(=E@BAF@0&!gpPm_PhxvtQ_PnzRfU&xd#)S4;NC1NHEk8(V(w_6QJI9>P1Xpo z0>DvlNctRCfrPHh>M0`9&L=Z!1J z3=KSok{hgbKt>>$RH~AM)1wr5duv!d2^5p3!_0mK<<=DAKtd3re*Y2$W4Q)x0m{(z zp+!T5Yx{CyFYhxW2OPByU?gy)j;cbE+{gBuR%JLMNqYZ|VqYP4J^Y&qe7?=IL>Rz3 zNj%&y-!d3}jI5wG5OzRWX(DZKwt&bU+9&#D+|V<(Aim|G#pU<@%>ji8M%);&wx=tP zA`Sa5#>Jps2bo;LJTroy^gOA(l_%(`K%eBaN?yvtrJxIEt$-#^_>ErxE?mYKNw{=y z19gh82h+?~p^!O+H=nJ2^y$Lm5%m95d*h}mP0We{bS{AP zoiE#&R9v{zet5MC2vbF`y}_U)31iyo^V`Q$r?T5&TiDQRbS4K6rRM*hR-+BSMO-3Jj`&}d47p-oMS@ap%Jy3(#gtb3ZORby!5Ma zY*C%1wT7DBuYbi!5M8cC#($o<25L<4@P#;2d>-a-he<{Nbyhy@Z!7Hg*I~(}h>BRk z4otpylpTDaady&l(Gat@HX7d56XWS2Ot=ocfdhT?-zMqwdVkfiB)2fvXAEtD?G5eC z)TM%Uj>-fKfF#9ve>{he7@L{E!08gOOx)y8u_9I$mcK!poWl&qe)|&`tLT01&q+Z# z(7azpJ6%2)XBoOJD8YVR>r0D;gKm#h@cR`HQ~`t%<82L-`Lu`K(ZJ7PcnD70zMlo{ zZPiKs6VmBIUxFP5%u)r=tj6B^A10IJzJ=7&3aOEN?4^V;*ZN_$td7w#kEL9z6ZLuA z5}Nd~jIOH-vG@>3Y*Yl^tm7_ONmhw7c}?&|19pS5$rHoMbh67mVdaSf1GEl(?qHxC zK{D2)kgsv-a!h{HpZA2|X1bR7-Xp$60O5MVoRu`0Q){hXTegQf#7RU%<}z2AZD7MK z2x;1Vsp_b64#YJbMey-_Uu?;`o;0u=&=E;#p3?z(px|0}`486GgK$h-y6i^7&mv>A zM2q=C?moxEl-6-ZjHnyqeps?qCNJzJ`icVLjPYTDed|u9WsJn{zXkJ6_`f_0Z(hK} zLZH{QdRaPHSX3e9Z1(ez^J|B3uf%y1$;ob;Oiu4u6+{e(L}4eK$Kl| zxZKqHHk$Xhr0f`^z4-U8WLO=?gN$%lV*bzO(@D0M&Gvk6-=Sjv%%2x5wd^|4>3bp( zx3ug!uLI#ab?Fq*2ZphqNGdU>mGo%nx3sNO_|kKQJFsHs+O<`D=m?YrAcxDe`r(=1 z#)ifSeQ4%zHRiJz$d?9hYab^QblLjo`F{JOh8io`UG^CXJpp&yQi7@=&)_Wo>fr2c zlVq7bXH4vp-(f%TnR{c#s4`>j6lRQ!V}OM2h~tAfSJ|GD5_qcC@GlKMY7;{Cb;jx= zyBSNBeA22+ggSUCDgmR2+weN8?-}*ZuSn`^Tqs#JzF|g;!@!+k>lsXqJ21F1m834WCzhe@WKHN^IjK*eYtWJ!%@-Q|^x8E5T+r82a z6?|>eoyXB+1@lRRd<#Kp@;wx+NSW{8U9G1<@K17&Mqg_?P|qTB;Y<|kn~&_IKWuU3 z4_h2OyohfCo}P7~9M+iW$^}C%rL%$&gV^Mcz*9R{lxnizV74Es^J;HYoxA*EMc`%p zknPfoCBhhQXy?#Po@~EeBfBJLEK)+S;T9f5W0`NWc}kZ#_!6GClN}X4)y_;i9+HmnXSjyk!1y2jxLPH8J+CI^{MZ` zBN6LKeAJ9TRaB-oto72;TB>HYZIwM5oxwgJz5yEbc2X#7JwE zWv@o*b;E1h-sO!7wtP}BagdmqHf@14lOEynpPTb~P;0Wa`fZV9F?k}11F;TovXdM+ z6@6m$L)|Ym90f@kayzScg|x!%h_l^{V|+&PCif1_Xa68EDN}b;8MB{XVv?3M#pyB4 zxenl^Usv}ZVU?@%(TO>ibd<&5TqenBGzHZk6?AD7YsrH<*Z`? zu(RGoc!5-^nn+Jji{TAUFWOpKQ5;-hT;k|mWJ_z}EX(_-@WL$HXKMDQ6Wzccs+tmK z?Z{ypFTT^;Bd8eTMt{iiFl$-PgYtkE9@dZq{C6FAJ|5?C3Wm9s`|fF=kRgp?umvi?WCkUAb#aL%;Qj)8rY%Qi#4O%% zuCoJc6$Yc3d_;ZT7c912kFR5}@~=KKDfN^h%?@mCAqQ-)yGC*7>sgE#c{c;Z5*8%8 z`}qowfDbCmMsMd>7}m2T>>TIki1rKcvzKuJdu-~|>nC)!Kic1gNQ<6ZFYneId%L&q z$4X$06&OgPGa_JO*>JRI@+4E3Y66dMEbU1efaz*N>TCPU(KWV%AFD7e!Q@`Qh}Ki& zH(P6$UY+)_)B7gp%#9~r-Jz-;?!_YAz;ZgGhw_Xd*73;-YLlJ=q_fo^G06^%wJZ-a z+mri>`UtxGny?@^nl5+i%U-hC7#WoX>9Tf^w{sQa()7ZZRTe-7%TYWyEE-+DWCA#nLrHHIfGSIYt_wJo{l4Gp1e?neVj_nh1p^RtXO7i|x zoD0 z`vvU4K%&5tia*5B4KH7^xDVA<4Rkye4-7gmYpfkD^<@``G;3+rR#BGq3f|;E$?Gpk znWZe;Y(eDJR=H+`7zIodS?a|!fewhKHP)7AbXjW%19Ln2?sLEz#y@5_D@`)|o;I^>#*RU2GM&$4-kTcn@Pm#@ zpl75b#eG3Dj2u{Sk^CBJSwGS^sOfqsHQ4pBK**EXpVm`eb(k?Ef3>3XqC4p@(-^U! ze4eyif3Sq_i#h=(=#muX5fUjv;68Y0SXn8OH+NJctcTxU&mIXAaqF*B9Y0oj_$`8&i z%s4^`+j;=MVyl~HZ-eq#;{(f|z7NEg!d4uOzYi?tAkG|p_Xu)pDEWZ-0G0=8;B_Qq zlNY^JdYTl6w)jy3G4X43A%jaETkE~CfC?;1A>SCUQ^(JOa{CZ&!Ae=i4NIR6$+P$=qdvr}fc%oh}vOhuX^rqQp$6_q;ln z(`Y0(BE2HY8m(Ihuil!Xi8`0>Rj9*-KAEKcjn&Iz>f+MNSaG#jmt)tMOZ0-bFD+7O zn!t>LkzIJ~9%X)@WtwxT0XGYTI5o6ImC4sk9b5+8`RWI z-##sR`VtFGTwFKni_zgFB{J!4ORX1=82;hpXUJ~ED9vDu%#{eep~Oon!Lyw z5yF@->}_9`W?w!|Da$>ysVjjoVX{yui^6=<)`n<>PuItG+k0W0kENP4}m^*A&79czfcLT>q-&T53EmFnb)5OTo*n zb)<7ZuUf7Dts9ub&!4^(5~#-EfQ@2IO9Z;`)4aU)Hmui`b!pUMBX_xQZyBiGWt)@@ z&Dc7yKl+_2w2?agFjDZ+WI!*X?N04Af5x<98hra3>Frc2Y+bX~VC$hmg8aO^4l8vd zYLwlC`KUpH$(|kI`ES%A1$lwfs1D$`I8o%!BkRI{vRX+KTISSGh{PEO**Ncntpvw6 z^-+Cr3i3OY)~HKy6vRL1tiKrDzAkh&$Mx#Gg~f8VHCp^_tBO|)xEYZ)L=+fFrMzKR zO=t_}``FmS7k?CIMJlqifxlJha-VaN5^yN@{B3|aN#MZyW=*2u$KT>~R=hFO4A%P$ zs8ut_)bAAuOrZz%{o$n;?taJEZZ^;$*ESZDRv1&eqZ6*8&9RCWq{B{Y*-`ZR@PvaJ z1b4GIVt&jqQ)Mxw9=lRwZGeUfOb|$i*k%6~&ua5`jl8z&l;LtEQe6ew9H_@52XaVtxkspikmogo-$4 zoTgY^Tcar%XyOKvq1X=vX!k14v| zB@^}1E)FGq)4gz_;&B%5wVrGGQ+eilVD1?t81*k8SVqH22a{i{dJqYy9i?m}f3>l-wbD$}VSI(^ ze9|f|t;})A_X=<9vUj4jFh4n3ZVAxKv1t(?6j(_h{V4ouX?m|*GxN{(Tp6y-6PNQ~ zO2XgDGhNuri*h(eH<5o}V0b`gr+NNqM(3ALaQ>$d1lir*bBb!6=2+otY_*0d%~U-1 zkd~KIBxq4^&lbW#=ub3kU0vE34O_>rI4Rlyfu0{=$f`qbx97L3++;HKZkex9;=8?y zC>+fWpOS{$oTgj%I_S*%<}wX_|eg(!}%*_KiXUAID`; z)ZP?$Kkm(mD&mK+T_*dDyHNRAyIe*8FRjAH!O7fKToz-%%&w`~Q(SerW&G&6M%T9G zx`KM~{7Sz0Z4lRbecmq%sed-q%t%IHZ${uQ*4v?yAZ#YBbZ`A&QHAC!k+RQgh?bK? zT3iGnY`PY@6NYkuFZx<2&v)>4wk(4vh$2*O^b=+ACn-v;w`(H&mYYYa+vSy3yPy&I zw?{i)Z7&oUh^Yd29#03nJPOw8!#WNd+25DWRx#EmpEq&|hMH|OBbqQ+E&vAx?V`RA z_cq5ifF2@+1K}1I35ak$YuzUld2MN!Y4$2Io#v!HN5BUzOBfjBDOc407RYuIjE!Ok z70+YG|BZO!%W@p_ek3bw4409Glg7E{EEghH$X?G-DJ&)5kikzEVEKGVCa!MTBdR`~ay1;IL6c++KkC4=Oz7xlBn6oawVUHF8@MZs8k?mmPSe9wK>3DCm+H?yW*b@V)4AY}K`!7FFb^*mqd zr>VT3r`(*Q2jjBBJLwy5H1Lr^UE-qiaP>jaV!Ndd0qw71czFl?>N4QT(c-zFyLuE$ z^OKq6V@zIyzFZhq9TxC56S?t!<7Iw_>eBhzA&gKudxPG{*Gx-|}At9zNonZk;P>qpb4KiJys zec*gO*iio8fDkuto(|~F*@D`CY4Z~%80>m->*5&1q4B=~p+I7%D&ms33x(EwjNNqH z%|8Tz9JM>G&U^s0osI;^2`*MYuXu>l1#aLy;?h%R-`dX%Zc@KfAmLDE=J4i(op$=1 z0BZ5Cc`>gABnm)EcaCTv#|ehsQ+<%__8qP=$pHPp0~0-P?jV@s#+NW8D`r^QM{Ii* zi04!BD?RbwY(S8c`z=D;gI>e}Tcrb%)#lu^$P+Qd;Cl|%&VT2G}J7H?gNTlbl z zxILYB)@60@$jHbQJ|;8726v~w?3BXOX>ZT}!8#>wt4EwZD`xd(fALyrxyxTzxx52a zQ)I^TmQ?*S5V&U$WviCvv>rswwb2KFnxf9vT z&Lz94IvIVD*AAFY8$#M?Hw+6HO4kpc4{T>vTMLzwczTYY;rGw(ovmaWnu-kuPTN|{ za({^ZBb%9x;?*Yh#A0~ilO=49)*6`jit{#Y`(6+-F>y4z{6DAw!@PJ4pdh7^~QA?EF%E!paBWLK%LiMc?IpyrH?~!64q& z7_xjqI@RfgC+xB=J#}xLQ~hDHMc)~IXiYi~0_6N!_`J{ef9e4GQiz_vrm3t-o_as`4@wBxZ!o{rI z%HYygB)sMG52IEfwEm9BTy<9Nq-C{L4q063+HK%TxnlYH^(1Sr1b{bk5FJVQJbaSU z;Lg{CVLbhw6Q2u~V_ux&Bz;}rBlyOD#cBSI5$2yATAc63+sx<(B`1<~F9jq(b z-rD=+{nDhkeXTt>WIPfwOe|(>#VI8v%s2mFmGbOd{aACWk}x>)vIn`@Ar&jH;^qKW z6#=;)zrp!){S$@zO@#!$CiwnFs--u1evO4h9)?0@T;YTBkaaGX5HhMEbarubN?e@%#L4VdJ}7Vd z`L^8kouu{}VVf-oC6UFKMj-?V#Wk1Leev!o?>k{8zgB#CoP+(cq4yyX%WOCO6PpeZ zD2x&P%%?i;1)M*&a;G?D+ivGB-HyvjP5KkMe=kJIQw??#G%osus;d2_E+^R>A44iE zf&=n5IR~EgFs3hGXsGTV7*l{8KQrs$SirnTzaYqMHt60qm()W}w{$?1Qd4rM$+WCQ zZZGW)9J`7)# z!8^KG1SkHAC zRPY8UXsCk9Oq!3P(DGOL0YE;s+K0g>J0q56-hJ0oFQhQA^ z1UVM_mQa9;`yvO)kUJ4BX9kA*-_&OW2fg&L?yue&>BcTsxP`Zw!HQ_w9RjVehutIYR68%Uo^znFt<`Nq=^TCCt_$Kt zG`q=V-*er^@m&Iw5u++wY0?Vpyy-tnOytoB#dvrk%$Q$8-Z3Kw`{rhql%;op)gTQ5 zi+T6qiMz|6orpFwEo#M?mg|Q8cd;UI@zR=1Kq>tuhtucWe{K;G<3)#Smd%!lu7?lb zE4rWBe=qAWI$?XeT8yuYHaa?aW;ehU#6UiRv%Yi6JP;s}w_rv-a zJ6y1hpE}JEP!kl4;E2h&{AGwlMiCO^PQls}ci+`uCy=)Fi4RdB$v#x4!UaaV$}fA^ zcAFT4*`bVzny>{AL(azKe`w3B97KkyvcPudQF6Cpwn61TL5I^h%@meK{jhT%Fh1gz zNqo4vj!;nWqP7qHqJ&4iFVKLcssyRU#aYufMn%bWwKjHsC#U7YCqrTsGHOva_(;hu zCzZ1x5+d`>F{=Kj8SLjhT4lhcygH~)zXEP$4m2FHf!yzQ5HvMm-L0hwd}D$jiv8B_ zWRJW_Me;5;Zudy@5pelx8wN3N%q=G{-Bal?>Sklt4(oeh$Wl&yM6Nl#aL`^e-j8N* zi=l{~h_&s!S_zxraX91t45RO9tfAX3_*<%d?o-#JZ7EHajkRe@!L-%nDaMG^ddv_? zHOE28lpjF>5M>Ufr49@SSDi49lxfs10VELYZoTKtOhlL(w%`V59Rc44Xk1VDOAVEW z^!=dE>Sj3ShgT^|A3Vomi*5imC9n_!E{s>*Ue3S+P5o7o-9YH(yo2Ny5Ixo|LC5kn zY*p6kbm-cgwB^`1j}MDN+|%uRF&E+evdhqQGv_vyIJ^hsGaKXQx zKmGX*ZEHWCt~1_IM?Lb={d2vu5Zs)Vf8ps>^h_&yXmXv0o`$)s#YN@w`+dgx*gqF? z@2<|V{vep;jn9ZUnJQGh$#Vi+Pxm$*Ewn+aMqs(SJ|>(DPIvy~&Pu!I;#_fl?e9pa z)-ys6(wa$O>ZHGvj}>wPNiNU*gjvZP5as6OukneUDf{UG0S%|&ovrCwT{v#X@4==9 zV?!XpT^5}S5p?=={^Jg>JiGQzFSRQ-4!J9_T8@T10L&sLfG|F!R)Qb1LP#hW`H01! z+n4oE@Z5Qy)_EiI5(rYmg1Y&EhMJ0f5eBYNmj$yL5^|buHmsqM3lfTX0trVY4=jEy zZ0)cQWVRjnl^h(G{Ht%zG2tv28rs|gqb}^1!gv~_kdaU1G=9zy_|a9?^qAjOdNa@? zphy}&F{1;L%?UOua*KO?nOQ!yjrdNftk}3Y?F`8co%ij0jz21HQVK{VEJ6#P`s7MU zd#wxqg>=!pTep-GO4bNX`Jv>y%T5h66257#F3T27qh0fbI^2+v?T3inqIy5RL9eMS z2dx};wBRtNckh@r@`QXsk~%*zzCM>_yFSv8WgxRB@kzg-In&Wv6?*11^ic$!EmVH& zL_u!1k|DYD+`g1$r@Z)+<6}dUI{a{C^mtoQ0`SYS;k-Qlz{%voO1)M?(=qmV^bAkt zj8^`JW3VfWxLC6!b1&bRXj=H;%$J&<0lsEuZOgT+;qc6%7aauGP*emz;~NBTyIfB_@rr#_lxgIWhhMm@Jsgll%dM!)24E= z)9S-K5W@i);xcXDfSwm@#!*f+9wRu0mKG};C8I8$gW*1}I5X$2fh8_)ku7li2BJr@ zAV+40BL4mxjK&4e4g<3Lan}sDbZVcg%QsX+{$IlT(6k*{PKn>VA_z?Of43m4d-8^| z>mY7_BnP9JckZ8uU%JNuCV=(pe?oSAGo#6)<36BP%b1fHyfv+(X(hfUS%1G!6wz7+ z%*Ou2E)R!6+)Bc!A~u+k_}IIFjjzO)J`bPZufbFH>|<|1N3IedRQG8qqxcDpVUNV_ zRHPmkKazXTUR{?BE-AzVF({>LI2~I--Dnb5PZ4HqQ_~+X6HcN%Pp_mM*I`0jnNu*Kc2P=bh`POPt6Q z_}eEtHbxY%zqtVO6YK|-8q;zo_<1-Csb3#fSt}x;2#i7`9&<{`Q}(Z)x~Fa$L)8Gx*oA-|lX%UeWAsQTX5Ky>1 z&gBJ04;eE4QE3Eox(vQul@%=1KG5l3EJ^P}2ZJM1ymwmiHVE~tJ5#XUe~kmTc3}D3 z)0Yp%I&8dxd|sFed#KxJFe%LqMV+>YIDZNF`ibJ#GM?m68&8(N$~hey?&d&SE`n$+ zJsVH#;1IPt0!MsOMX$()DjNxS4{9(j8i^DBY-1+wXN(xs(&Za7U{cbvUmxmFFFk7Y zg{5+LUDsTrDP&3 z;N(D@*%HEzh;O_492Mx^fS?(Yb z2Y!f|V&;Gqoi{bPU6(Cg2}=^-7Q+@d*qk3!VuAI$m_0}nc833cpd*Juti5=M-?Mz| z?R)1{9jIu^s57(vM@b)3v6&WuE^RO0Jw&g)GOW!EXuo24LIuzA;3K|kUx5Lvp2Eti z+9Q5IxU((OF?9F9#}8i4bC;>dT;h|F?P>DVPhOr^SH}nf;$`$i0TLX%L-_=pm1|?E z@5`z}+I+Dax3&sbd-hD)>Cw+g8?o31!Ex}0hKBMf34;k1+&yRClR8fP@Eh=yEDe^z z#~z5NGD}p2x0hPdgl*^zjds5(7$BK3Lp-NH+^37Yy;-u>omstX%|!D*xq)=%zT%jZ zb38oY^LUI&P!-p1ereL{^_?;%`Q9r!u>NJV^d%4n9&;RlOXM?N$*FQtX#&9)UdUF# zBU5(&BLDXG!T=PJc(8!U=mkV{2wwshBUZk;NWh)ic=XUtpDxQMJ_$s0Mg6b22BO^P zd4Mno=%tOcCAvs*DJbSGygnFiW4=9OeRC)X5faz5m5`p$5mQ1!(u}ByJV3@%2@BSx zhv$Q7e!o%O(C5v(@?zwsfry&+jO|S%|C0`~Cr|GP<2-QlaZ2rvnDl(QJ0r4LdcdvI zD7fS?sP~F|YP`V>l@Qz29JgjOHvT)ShF8_$5Xp25FI(I`%?y#E7(o(GV`g1b zrfRsvW9KEL)#Js(dV7>5KI0b~-epi*r(0Xe$y=QyYiGGNrj1f#@*Y3fn8jo9K#x)~ z;QWSRGUyQ%7DrQtBKctq0TrI<`;!d!d+jt|f7r;$zMMSTehi_1J72pu!{*R|-Gb%8 z{fv?yRzP3WjvQ4G8QU^+`lV_@?7IPLlo(Fq4(+FddpVW;K`ed0Q6>asOqkGW{ccqh zbY|@C57R_l*scJvs{zd`tpaj+l=6Fd_cy&E>l=A(C7A-&m%G z=U76d0=h6gFWt!iqdMMp8ZD9lEQeR2H|M^-m%CJ5<0#M<&Z_H+(<;o&%P)$#7EkN- z7mV7i7uF&|Xf@^&GoH9^WN=VaM6(H-x)3h!mnIKSfzihM1W=4uopYjs=KDW1nM-I4 z-rf1Hxz@*1Qi$1jxzh(UNB9R@(?FFXCxgW-=TbQFqJ<|cHQT(hbzlu#f8q5uFJf{O z1Kzqe7^|6-lQ7vLuG;adCb};B$GX+X_UflOwEber2~qYvp@<_#Er-lmgntbwqu6jGjCkl|&7L$;viuXo~6oK%58DcHU7%}aaU4v-aF|5%Y3@B=;&##K< zc(m5+3N2yZ0R*I|VVp_3;gY7ty$FV$N-A}z_UrS+d5%qv4JlT~n-BoG+JQUQx@Ync zA#54!%8-XyriOO_(2IjNt74JzpB+yW#wn;y;9YFnis+4-$Z6(Rp2sF?e6K|rp?CIvl#>X#W7RUG(Q3-TrT=V)w4{Liz+@n(SO#Csmj+8I_VRH+b=Ryh)sk zP*k(Q7{1|R3+(kE-<^KW8wi^*?m4~Q0!#C8G$GE{V(5e@INi_)Muboa34kez-KW53 z-Ly@iqICEOqrS^$nbRZmB+2p62j+bfu%u5$x~h5YY=%h8l%L`c5#)Kmu8YX9kdEGv zx^ty$mtBpmXiG_iP7(;E&|VesqCkTeAhl_yxW$OFDTkS(?ib){yM-h4erxu2?ow6# zs*6Ts=Oxy0qHL_K@tnZz%E7LA&(t*DnU;R7Z3N=&sDbzN{VcoG{9fh4bXPSH*bl#5 z>S1zy;*K~N|11>$`rrh7B_{B=pW=BqUHi)Ca7AYi^}FQ>%v$RP)~77l;L!cVGwYrQ z*yo}%u^Q0R|Fu1ivubrbR0Vy=2ZIs$6Ufdvt+$?0(D z$+YefQ!PS2<6Y~IuZOc?UKln;>xyHG{}+32!4+4tG>k&<1W9lwcnI#!kPzGg1Pku& z?j!_vcP0rI9D?iME(z}LFbqDx0COkjywCHT^RD{??)}y`i^cBg+1@~e+5%kF>cf>qH}@Z+D|2X`oK%eX?2c7o9gD_LB#-((Y`k90 z1io1h5p?r;wy<%0JezWLg(88DXFFERYKu*6ZgH@rX>J8PjAs5D$LU=CReT!`1qD^s z*A#htd^SYy&xfC)Upe`=s3HL;2J|468~LgN$r2~(9MrB)rUKBzU&vr#JQL@W%Wz3J z=09SIq!L=P--t<{2DBLf@jfwE?OOj-18VHDI?=U8qWRbvB$jX(1i&%D!#P~&9ni!z_M7SzY?QICroDVol*p?pIW`1a3)w7g zh9<)~MdYZw-m{3Phqg_8mX|ARqyb*ZD~l~_op6aOB)H!s?uT*TUGtN{|40^0GgsGFAKxEAr=F_({yl1rqEYssYSBg{#qIZW zpP~lk<~!2;jV7|f$@$oh35m(jm(E#SaA`|VLcr8`)U`=8u7{-9&Ig#h>2X4y%V-!B z-rK4Ie>BH}tM0tuv{`NAgyc@^d;@DDpHL#y2E0zswuy%oqGQo@S|Pd!$=djzn+m0) zT8j@Ujig>^7{jq&TJ0$z_N9nrbACavUoB;U=14RBaQfcx;1d#zn?_8CCB~91cqzc0 za4cjACu3|ZFk-*gXTGr*$)9>P$P^bcXGEkv_^roi9;*V{jG+xNynJ*fD|>v@_GECU zWx4eaGvrC=mk4t4zYm9B10@xzEwc%JC~*A8I(=HN+x^;O`gzWqPi#JZ{L{%!w-FGZ75>g*&4M?uMQ7i6g~@lf{99qr_p)=I7=GUbHto zsQ$M92+G7+SQ`4+mI~D_c7~0*`!;gnxWVM;ePAEmK;$L!LKBKbelf{yExAH|JT6h$7EV>PTO|m6X<{!_kska6L0?Dl@y~g^_m}rS zWLZ5yxlULQ(|FRD6&YOp{0VvP)63pCMlP`?htfPbR%u>v1(lNMN;oau9$`pVaF=nX z`8kh$b!M(;nO^-$HLYbiF->KJu@V!3OupJ1eqpV2Lf`QAQBfyHCR=^VG{{eH(Sf00 z(eF!^K`#uNxqKN#8y>^UKQqWQ*%FiZ-_bOrNp(xh^KS?#jzR2qc2espX0<+w%GedR z@l91qk=nBqd>xj4t1zbObkY_;_zu;?rj7-QGTucc4-ZE45cv&0n=J0EOFeZKqw1(;ywtaH zvc9TKS(jfL?&^Q{?P%keKdjBwte^SVL#Skp52ytk7eiWmBqzgXilFr0Bd~&1;k8L5 z;1x`SLv1;gsLQjOQh9rI%_%?PL%mN|6)6>-cKpqQ4%W1f&( zc{UbwX#yKpNMmE~v#xVJ`y~LMG5iN^+swj@@Z^4E+I9>9$Tp-8Z3)aB(ThMBPpTgA;RySuY3itHiZDx|sF zcG86Rouv;R?HTgT?xgZ-6&6PW*^vr1!UZz#I2}PNJ)Z^VYFx%%H+<}l0H+Se(u+=h zkQSI5j2v|aDfSPPWi;YGq5Dq8&64U;!w$4E6{M%-{YS_`MalK^1=tDaVYk}l6w!f>84Rk|tb zILtv)%O46d`V}dSktnlvLxcSqP$cRDVK4Y$RDD0|Qi`=@5``Z~4 z^I>sJ_T8KMmS4KcL5`(mc2Nx3V_}OPI6XGxWpu@BbtPIHWR{j=AMq9zBmxJa8&~99 zq)4C=rZRL=9068BdnGQ}jPmB~zv4Pt@o{ zJpJO9oBZljt{`q=PN0iAdq&Aen*S4WL-J~4PWv!xX^pPxW4t)Ilk4}u!;Er2csY&w z1?*v53ajehSIJjw07mz^es1?A`NG+~2eh(Z$5 zz)+A7#c`d6&WAuH9jU+$#On@~-a!=wGq*d%3JSkqwo{tL`!)ldtgSf^CwfotPoFOm zr{w<(`6U}eFiNNeVd{tV3zor`04E6`@O#ozT*9;$#Q4sWFfa5)m-&k0m7s{9=dVM< z1N#w=u-MT=RA7d$MDd<(hfnjZZhP0wWQd8B7ddDB4H85l#|Ar4TshTEM^{%E>C-1> z0H#}8X{k6NajJ%)TE+8_`*JdD^e2>B-EW^Pn&0(_Fj}M%rhib^1pgL?@>+0_R_l%- zrs9DAzD#vX`Qz4yxPNj5AtsW@8~N7ORYf%Ut?fT7$p4f8vRNk@|16xpN{cc7FZa?- zS`w=NCuIH&q(p)Ig#Vh&zh_oQ7CQY`cmG@q#$Nr0Q2zID9As$u{{!UzQ?%jzgRB4h zBbTlApZmpMegEGf{{@M6@BV)YWHF%7^vN7;5tj$7<~Ufze;3?GIeQ zQmXc=aq={Fcn}h*(6PSob&^KC-7QuF`yKDI*ONC>+j+%~)qi#W_jOQsD`h9HlF|?B z%_rRRAT|VL{;Sh}UAL`<_bzG#IcehO27J^Kb=Y2~)6&to(lrDenf(VH6_{JZ3m!rR z)5kI}XE&K=WcE51ThfgK$Gl)6G=Pz0yCq&9OOZ&+V1F&Z0|P~?(%uQ{arq-`%3K!r zr9YgwW5MWC-%Ret0Rb7l1{7bjTGWnP5pWmQ_Vihz0Y-x7jNE3fWFPu2AS<%jiY-)l zZM}J)_=Nfl<$}f_B;N?T&&Z;-b@Z+GJ<=rev?I#<#DTjV#oPTdG29mRcb-&l7r$)u zVhH@+BjTnU9Wq~lQOxVUe?rTx&u99?CV-9`C9jId+r1qL zCrvt{>N#no*=k!}$0`b@R#hbx39ZX^l}Qx!}HStybI#)v-! z*N+xK?a7U+y2>Yc0p4Q#s=sm2QK&@g%ti;hu&IAt@X>sW$YJk*1#+(Ww?;3oz>rXK z%|s@}x#ODVe@q2f(-n`l-OiW?uG@UmROT9kKxqs&&pHtXdcTV{MD6ZR#j$1I=n9

38j*tYZUe{MM{*&J=6g;ZtV**a>*vgCp!F`yRX*r222KMLBEt>n+|r za_7EdrM&m1<$M_EIyFLgiGK>U*U#(fW;7XX>flsCGQw)T$SqLLQ~B_Y;S}WtH%X80 z3rnN&%gNg0)Mv!i&Nd9ZOwe9=G`+M@=Gq^42O7)-cEj{XV`yyptC6svR2?^yxD#H^ zWJCbix7$R?nTdq&4zyWys`)~y=hwKGHT}LbuAM5&liLW!cAu}Y@uXRvUTP;^Qi*3C zP%Up)4jV5_&bY(PUUT>FPOySk!{zkPNy&)Jn&4T{&N`o>9N7F^$N&t5yjP`iVxro$ z`{2;>n4CAZN@KL%=bjqemQSVznlarvyW)8VZ@RmF90keZ+0jz0jT?7?dNR2^kT#0_ z8U9{4e+q*oA2c<`KF1LGw>`RX>?d}P%^Ar<9dm1R+3lBVvL`?{VS7eC(Ky*4Q-m0d zPe}T!G;qYrU#5T3!+u#KVu?`Z80`VU5Po7nMJ7BQ&+)NW%IpVf7_fVIT#pzG1*jgI z^!5ny(jB|gC@sC>;MhqV$SYnS=-yf!pW%qXfHb37y`S28y*DE~YM|{i>Z%Bd90g{% zAETq`id~-0T{Z`in~6$xWu^P<=UZEjlfSXIdy4;nU8${CQd+uuCCz#pMB~6fC-#NI zW$y(gVdo;kVW2-rNk)G;KJp;n+pq6qjEq?{?*ml;nEj0j!zJZlZ`s|8rItpzS1Wy+ z>FKq$dbFQHn9puzTmT!`6lxur&Ahb{En83}6`^K>Mqwz2VH@1Sm80d;K$Y-Nm7<=o zV&z^>yjIOlwB<+<^5dd&7VvC~4WS6673fEmJaOct3EH$o=)HLW&dET?-Z$NA0>D=- zA%t0g`dmwVNW6?(cjntQB*B~`%3xg9Xy?rwsX{}LR32|4>v)-+HB;%Fvu}eJ60UC? z!Y$G*?d*1(51-)TWO`jLR6)hcNwfd4|#&iLrnTp?17P zwT+BVr;h;#9k$3mKGe=Lh6;9GVev#G!s^B2b+u%tmkG}$(e`FP-}rWKzb*QM}2HaGe8ye$G9nD-k^%8B$qvTYl)!}rDq z9BWk==TbvWM101f~{uSr*qt}t}%v{0E&a(4xSMM zLKKyI2>Muo`;VEOn_q`wt?!*b2|);kuJ`?rD?%LFfajIjPDR-kix*tairU>6Ulk*z zq-oY=m^vt!!&#Kbf1Ogidt6M{&CrIXCT~A8)qVV(etg|5l#wj!Jo(Br%Qv}RdXlAg zEDnX}eM5S?G|_S3g_>o%?p$Zc0Ex6b_j7FHuI27A^ONV&q|JoH$e%28kf|Hmplr_? zNt~;PaU}5as!>m899%vCad7XKYr0%y5nxX6-&wqy?@NYI9O@*kT`J&kT4pF zJ$~~}y4l*{to)AcH%p9oC~X$741u6XptT7D5Qw^U%y;x|RXjN&I@a{W$cWZziR11{ z#nE8kswx13WU9?)`S#qh5S1oJ1-& z!dX;cGa%Pv#uV)j_zN5{BVkZLm*HpzN>7Blw1CbbmNjXb>$Mz{XP>NpGizGX3NC&) zztRl#m;p3kcn{f3D4?q2lj8Tdr=LQRH7mrFj2_;oci#np-lh2nB|A|_Dn(=_#RDx) zg0fbZFCJVC4RaJXX9T&pS@1DLb;b4=qx70#GV#UD7q*{;IZ_X_ufD!X1lT!Gg$(fr zvv@HkH?n6RJ1m$>G$xe?YGn9I`g4DDhhh9@|OtB%o zohK)(#~hoAIrEYBpCRSZMPwABGjs^o9f3BIMD8vxG^ zL31@%pwOm}?_>#C!`od{c-ot<7h9r8(j1wrf*8{Vg=F;k5jN7VOojyto^7BMG`q%R zZCj=~#is{zG2es2vx-|pS8piXK+%>zc0_-=in>lM%U?E<{Ioga;hBY$_=w~Cai1-! z6aMFs^8JfdL#&^JRrYut)ye5i#|SHY^F*~08YORsAV zW3ycRF_~+`F?end1A7+_7st1vI~oyQ6yVTN!~V-4E$69Z0_TaKFFty9#1pxX{13$| zeRX-o;xdyzhD&X=s#ZD!yjJI}C2`hc`!d_0UCiMi0dgGXx~gCmwWfK6H!t|5qf4(7 zulikA7wY1rn3%CpWL(UA@j0$*ZZ>?{YH5sV1^o|AkJ}p)scR7Xv9`$p#aO&vsRTW@-xm!(4w7p@|ct$I)&5iNf6vXs6Z?D;I zC@;%1UZS@+eR*_CTXt&FPfv;;vUSv>Zr$j`D*T522AGNBlDjkKHnup6G2I_u`+Ti% z$4{gqIupSRumvoQW!0a!2^M%N2qzzqwdU}%YHOG%o2$E#)&JB#k7D~GEA)~p>AJx* z@D&e1;G3HbCThHlY<8~$$J111Cr-34zXh>Rx0AA;oy|?7P9wcu9>XgL5p`pUQ?itc zAIx&r_CRB|pKR00nfr~xz{Z2!u%od_`8MTh5S`jhL#pIUE{#>wS`+_ZqX^#8q07?wAb@5QcA>-9-Dj!n?gNW{)xOhQ=rTuGaPqu(q61TVVK%UrD z+VJ_y>sJ3m59dJe3&Qf@^`ZFV0@X|zgoW=7eUN zEHbYZzB{jShCOg!+i6UAFlu1u>9j_8;&QF3Q#5r{9MVhnUTh zY;{cQzEK}EJL0o#TiyeWBJP)yQsd`>-W#`?y^lVGSncX^IxT6Op5AiuJUySl{xyJ@ zRZtwd!jSKV1u;pX^Ay!VZ*@47gv0FQ8G>bSvME*IeJGZY91h}XGAjvLqEVq|TptYx z27#9#k?x_%zX}hMJV@{k#zINlMejY{-9*OfdzsLNR}+wcVA51 zt-hI&zUMV3ua)s+czE=484Jq69ze~z@tDgxGQjm#T&S5C@|bdDVnT%QTGBnebamC} z=XFyTU0$+1IqRcxGBc$lEVtQsyh!_bFeRnbc85O64)cZ;TC`QSQK%Xac)i{8e%<8O zVOdu5YzjX_uJkYr7F)6X!Gp1o2>Rt{k^*6=iffK{WZXcc;en_V;5v>W))@7xKwMrIn{A>uae^41a9uY45MK+z9qY_s)}OVS!zl-gk3+*(*a}qX33h!>0F> zKo7gSpZGJ{`v8(pw(i!?j~U#g1sIfHE+9Ww@zy~*nzKlx$_TcA@0E`Z_n5^O zmn>)28VIF5=;>VOvIWqy&ewRN>AelakysFW{He^A*)FZ}o9R4QsinH6#Ts`I9+i5e$OU&5iX2z)+}+;tWWx6ko3LX4 zFNvD@1co^16d)n;F0666Nh^I`1!?RjYg4n)Bld43A0Q3?ur0&#dG)FQ|##v zeJxh~gh(B?o+OHE4?*D|$6=#9e=KZv?P=FNHnY1+3t3{F79Ma7V$(dPJ(_xJDr+<{ zpJyT|`#n&%vtQUqa5b?nt`C)TOvk>$@8PY63^%KAjE=VQ^I~{4!|G~v_%fXwrYKOt zK z2YVGeb~}`X31i$herD8=cbOux(ZhYSKa+BCq&{Nt8xdDLd|^`rJm%!R`v4Y~a6H;# zTWt!DbootK?+(vI+(sMwLf-N#eZHSn%|UAb;R`RBP`UUY=0{;pv^G!ViK`ias*`q# zy6IcAh5R#jhbsW$Y^vx)El?xBIg{=DtN!cq+!nW>S=}IUgbRX1ftH5!Soj*G&`s%x z^}9zL_XgB&0SMP9wcv*ofJWL7N;wIg2NtA43!*oX7Rnrw1q~-C|w!|N_jNG_63+enot%tjona0n%nFC%4Fs< z^MZjnZQId%rdWusXrBdPPf(`JV|J$bs9r4n>&$Obu+4?dZ{H)2z)f7Sm*QmQ-d++E zN*fHh*HM|r?GM^a?{I0)%68iL;4cq&i+;^5Bu8p2v-$bqkg2i{fFl?> zjL2{X+9`C@}2_wb=8mFjcUQA>m{n@ zdsqcthUXS4l0D)m3`TGtw)YQD!If?2sXJPfXtHN{Sy(k}mTy0cl^flQ1x>Jg{W^>y zfp-!iv@Fp%-+fJlM)(0?s?+BrG~uJcicUSDuc}VU(WbFFEc19K-qcRTB7#auq`uS$ zLK3+ugWfMFO>#OhVRbgfx5LEW;Y3z8X6D{-a4Z9N)g;w>x?hLF2HA1wOBCD9YWrLQ z`P%Zdkx*wuf%N!cI1Cme3MT>Udg|&ezZ;F7(xVIV=kAy64d6Kowdv@qFRmZ$3SKwP z_u>nF&P-0m6!OQRL}VIsLG~e1!Ec$)hSv23Z=cnC5oY<=k+J{;(^bufP~bT_8T3`Z zyt|BI+UluV*l;`Bnhp3;6!WqR^N+20{``rfhvjA z=>4dNILOwJ#W$)KoLG(ZA|24sq~ zI?=PTeSs=5JNWzNi)T$xbQS=xt*h+^HyW!CO`Tq zV@(Qv^=&;EXDp#djlM-@=tFY)*k+pkAOu*6M$C5S|Qiaa*9Is1*K5#TS;Y!ooo ze_S=>?Mmc?m)ARL)K8ULA}Y8L5)tLfA5iI_^Mw)89x8T`8J+ETi}H{UuC^kAiZ>jc z-X%njGn)*F2sp-X{PMz8mzhPMkFmT#oZ|KK)||;bJ>%SHPjXt*c1o|HFSE;yK$fp!DG1GqPoFYE`YpELA>oGu;RUw`DrXy(ZR0$r696~SXk zf^&893w{KIC6-sfyxr7oLUe{1_HyLSsZ5^%hRGvAcO}Xdgo^|!hRS9h*#XSREcd7= z*1-yDw6k_GSXRz-z67jpd;C1_HCU!(^-dADv@mGt&<T!j51XNg|rhEUNbAISbYfp+n^lZA*8fZk@LgDdRgS z_IS^qGD02>X4${3${D~e#e`Ay6j(~qz209oN}E@4gk(G~&ZYdi-LvUCpf^^8*Ny5| zd<=*pmz-NIU`$avee=*lRQqTqC(P)cfKvJr;Ps-J233wiAghG8a_6xMkKw+tanLEd zt1ONNtA4L#d+RU&$j?t(Bwxz%3Ui9x%-!KLdFS2?jG2ZB$3zL_s>SJ@dq0jsHdS@K zwLV1qR-8MVm1?UrzH4Db zvkrn9@t0{4<5O~n!h$Z@16%g?D%p5mnIE)zK-FJ1!RtGYp3hlEKQ@4?#B(OTNoD69 z)^CpuOI5jdwpiu3@>3GyJER!7M7Kr{X=Nj|FMdn-*?O0D~fa3#fm!X|vweVgvx z;j%(>KbP<20;QUB8>lcUZoB7|;J`wu4v-aYT)P6k%Ww6DJmC4*&4_e}oV7`%4mXX+ zPO&>7$BP4H-^gctzvmzIIroTC5s#0?5Tuo7pO1YXZ+mW}H+CS|M}prmCxFOhiw#OaIQ>E?k%783=zh>=P&KM5KYw!3f!DUVf<6k0K{U zV&=0IIv}#kNX}JNYl``vDlB&EFCi+z`~;bYWBBXRqO{oAM_NW;*wBQ5B^@l( zu;3uvIklhTri0-c#4CQ47e*g@&)L=JQcco8b(U+*j*|ZRl=du4pr)8%DO_*Dm(%oV zr;CC^2HHw*R@E&h45YlZQv#YTuyXk6Qo*5xT<(}C*XF(jxO5eC1TREaD!-%(E^nO$M#tStU%xDt}Pg{c*)3*sJ)E8452bO}%q&#WD% z>;*o;EV_XyRPOB#7NSFXjWs)>3KSEU8!udIwckHU5Xc(+w9|I~L7 zFM#%Z9f-)|ZN5x$4QsrxDm!CUL5%YT`R@Jk+^C_ha%(Hxv(jp&ABk+wJi*g7B1Iff z(nA)6zFp-HRCrl9F@y@x@9C~DOd)h&fW|9ct8N}rDvIq8wi_wQue8tKgB&od5yYkY zOqnXxPH=+rhw)a?p;TqFiv~a93zFxB>eiy;bd>e@F`FJz6XEv0OFsUax=pd8j0&`H;K_4`-UniMVm4Izsr_t_uX1-wRw$cM;U@ znMTG5+5x(rVLPr}k?e}`W*?U#eG4R8b{=8i!4;X%558orGTjBbd7qsvi0M+jOc97U z5VRD~U(5e$rsUu`;0Fo9l$5ve2CY#L5V-A!L=f(}Lh#(6aA#*?P>aLms@ubH2w)>} zoOV*SFlmy<#zrLZ=I%f1g;60@I1&L%+ zbOZ|1hZRB^#}6AXZXbA`$tj9w#(YE3Ybw@3oa49Z3~4q8rFOCJF(ms;Vbe|QRG*$1 z@rI*LhUN8Kr9&NN2lW^%xQ^0Zx#j6~A7c=|h{Q__3?vqe`9U>(paX(unGG!YY-3=! zjHKg-p_s>zqN?uHl+uN`ci5@rVIzBG;abK#x zzaO3^zIPD)brvLnffr@6sr`iQ5f~FW1WbK8iF+2?8`BGC! z)}IPo)F$h<%m|13*3m)SRxilvie;gq?1B|=rpVXpz*TH~@}Hn3^_)iU=36h6h6c}EZ%(SZ_3zi~%>}Yj z3ex6lYCpqh?%0=xJ^AcfISe+vPi5gUjg^rOA8P3fUWP*?UhLe_sM5CUF7Saz_JBcM zZMV_h;8@MxfdNj$R4Tbr^?o_Pu=-@Ov_N^1X_6Gvq#%t{|evyv!b9k4=ED+uhfxeZNH(*;d=h7rZbpBR;P0i;kX<)iFoU8b7a;l~ymcT3B=o@YQi; zc1V*HLhbRS*L}ZNI@87fBFy{ACnPE}5doXap)@XeWpI}KKQa9#Yph^#uCFD^=ytaA z9Ih?AvzK%SM}o!~p)01{t@v_^fbavjSe(18nugtL-R^cgkjhS{Qiso!4mzL7>yPrx zaX65<)5RmT*^06a8a=!D`ez*}KV)!P?Y`H8G^7Sd_lcfznac$Sk&`Iit*@`5*5gu% z{Q4#pE>Kb?Z)-cJ^xvVk;wrcJ7!Cf}oe^A^yBK23|5Y<>`M+8*)RQIucX%jyhkl0t zm0>cQ`~SpluqeS#<(R+AF}&5(BubR~V$J7US4NN6O{k!Vx87x((9|UKxy!gL#b%iU zQCnX*8@Cj@fJoX|FgrsW0QMVkoS9^xnjFs+Eh#CXWnqcTAjRPQ6CUuRqOvlMsMiSt z6H{10K|#;!k)}WI#g?LisF9-f4+G_p&ktKXI;!@Ao~%@8Wx2=P9@i=$wYWzDhWSf0 zwX|g3zI|I}mHj7L-6Au7H?FR((G7x*e=>tDu5R>#`;QSnsxOHycU+^~+D5v6 zIbLp_3b#ZI^Z*kRJ%oXQq1tV4+|tSJUxTM2V~_ox!P9R!-z`#@CS|#!zN33#YisLo z{U^U#T!Jx|9Aeywas@+6q>X<@DXZnSmr^`WGc_Ua?MgbEQvZkhHg@|euqDg_2 zqJqNr($dnQBZ_}FFm&7etAU;4UkwnT%^-CW{L_tGRAi*hwVZ7~7}5P?;!IA{O2!}1 zh@59F%?18){rHU|373{UmKyw84pGfqu-m)&?|Xm$p4M6Z_q`$?>4^pZKZTIOkIB=v z3Oe_|wLKWXH!2m#U)QZg47dyE2!cX%v{s{Fv5uRe;WXwaS*2QR{}p&FX)#^Wj>H>W z?w8#@HdPB*YX#@QofDxz6+3KMfI`Ojp+VCN2Jc@m2fo{jhf1*rF<@@!R;8`21{d7)0|D z;P%R5q!HXGeH+#{3P0iz2&L~W#I{)8X3_t}Jh%v0?rFPU^a%8ViiC~bBTvEF#~{@D zO2#_m7PpWW;xGz-NSUTNKq$O)1Y*{oSwE&z??c9Uzk?%?VoyI=h-YPGIG*yGr9M`_ zBeHJ-)cG@3ujHv`@q{ixrsn~dZ+$ZFpTMgJ^YwmE$Ki%_f)!1vi4*zWUP}8x>puXH zjdp28Ov2rs*AgHm>w!4Un7Qi~pC?7;d9E;9u`v9l>hj6U78~}X?+-)dq$8U5ZlD2ap zNf&DunM;KvjX{VQ#Bu*htVv$uG9}w~-f#PwwG4TGnPo6T&5CQYyo;UlA|!tP{rgd? zR86EDkA5T-nt6=idua5ao)N5Vno%N-R|OE;5#TFWb}#2n0>6N3;S&(_^ahbjD=CE& z&uJoHAwvD{1w9~OcHQHpqy+Vu$Fh|yV9GS<(eUi`+y5|Z)q)A*+V9Kz(mUsXyb^EMu! zxnmIA0a9+RaV+Qfc=%r3G?-draPeF(EzL0$;gAiENSJqe6{32G%WTkD2PYb7u|77T zX;h&cv058BuE)?N4*IhnuHuovkb1y1$Dyx z+}MBbbBvR57cA_wgGw?;NZ1SgT=CCbVln(SRENDEIkRs+&Z1o^+TA@9(R(~&xrLA* zJ6rk!R1Q{Z7p}xU#w=wYVD^H6?2qDJoREy$@PI?6Wr�d<)`Gn4NL8p6tJ>*r(MbId#*T;5Dfu7mXAvPy z0~>DL(ebE<7kb7mm0VkG0;!PcEANN_?QPd^fX{<4wccF1E?gvQl^eY068wLF;JCi;S4xdnCVo8RPt4#ch(2k1WzubQbXc!We( zoxL50i|(JZ=)!yFI9i`u7urZbe?-3Bd@pkaXBD;+;69mdTp$!pIl4ptdK!FX`=lTz zh-zv=ezgkubEHd!Mmjn}o^a|C%%C&OYwpbOu}ti0(G=b2wn_=WTQhcDa?O}Qxd@#N zdW-bA=28WVYE=&DtR!Z{_Dz`b4W#_8jwU&LwgXGO+EsjMYJ4lN{BluHQ~j+zfL`gb zhgC;itmT2>jM90TxjfBK5J46*TwJS8?gJh$*cY!aIO*I~Rcuec-AZ;z%KCaj6f9=; z#ujDMZ*)4&u0}q70OgN0J&+pzVH6gzwyqmYMgy_AEUPSx{@|v#+4ZgsVh^b@@Z4Bs zK!p0LN3Cz)Bo%7|aI)!lKV!^}5Ou>3z!mGxF!#OsVzJ?h?{{gX`tjps^s7%Q>}zg6 zv@x{;M`yE4>ZbypFtWPpkO+oFhOp{fDfvbV_{Q^$9;Q7y<4(#^a|jHT~tz{ z`Jr-SL(~l`PGKjxE%x1ba~jYnQ$dI`UHI8-0;upwk_K5ae`uzeRK3#yC6|&{mm6q% z*#KfaV zJ8%Fxf{W;OwavGxnlVEWNsUDbr8A^2( ztF>HFj9*gxB(1R$x|vYaJ=e$OtmLrp8+eBvli7Qv0j09*>2ACEWxp!yAjU#dWwL;= zgX)K4ehyNia|pn7Gh{8;zR;NrvpOFm>ygJ2cvu$LHePZty7%jV`Ew(o;jNCGKXcmN z(=MFc=hqS}@VUP&Io4^-`yPd~f=~+FRY6+O8RbQi5CWi_`&?zdK`cSC3rsC?2~-!K zPQtZ!6`S$_b$^yKq(obFb2<4)sFI8-?@v7CcBB|7@%0}t!dX~7nQ zl%&Gwj0<&YG@wi*zydS<-i??961m5!IFt+Q8L?BeJ4y+p<^@T_U52#Zla8CME^F(a zzae8Nb&k? z^aT56sz>1mPbJGvVHj{C08GgQtyxPp5Y7^HnkYEG1zya@$={bcZnscXiH}i5OD*3T z6Qh_0cT4+2(+l;llkLdT-)%u}7VRAai(+DU^S7&5=pTw&Gg4m^zVea)`Pk5J$g$bb zhdnoP8|+VDsORlJ9-Fd`a21Q95R2;Kr_h0_d4T*G(f4|;6TqVZ0v!qIx;@r>|WX*QO!#<$8WBde86TR9zmG-Bv6 zfOurl^z%W^pFS=Y+@f@nN>5l&)a}A$;t6>m@HWZ7c$a0LHLu$P<(`d-Td|tD?}jTq z<-_68m7qjFM`~3dyQ25@Cwr|c(g}92vCXtItZmLgT0#A)%DzIb6Y{{TL4##C`6rOk zfVfa;92#vQFY5R<=oF`H8{ z0Y7;o%|g=y5@Yn<`6=s6Oq(w;;NdqmhvMQOLHtgFHc3XP(TL^sFcQg{STryQ@vzdl z)A=dZEW>bgRh_LR&E1{T)7sE49GP9J5Kv+j-fOcBxNhfpF49c-!_v)+ z!PMaVnMf_Xt!9I>=d~cuUeMlq=R<~*``?XC4@u@v^ZeqDyTh~)BCRZfj&P4=Aogvn*+bnV-dm{gt zBHp|_nvWZNu&pAbX=9B?{s0}`5-!dV-Xumk2 z@<8SxLuaQL#(iE8rj3O?y4hcbVh+j~+p3brWGd#ipCpt4`y73R>u?Bp;8sao8QHhq zVJJc7zC3tR^<4&)?1qMNp59Elel~z{9sjf&BBw-B&$zidct^};xoRf-p^qVn$NR}t zEqdc4*00D4vnOUuRxT&OEo&$}x4Y1=-^E#9}o5>U%cb{$cTpo4~ zz94asjRh}D6FMMYFVjf-l*bjgJ#_jm702w1Ll|qF-*9k>U-|tE87hdU8n_$@Po5>) zC#AcfsLja1FAac|4e0bk{c^1U*gdycu*1Lw2ANAVOqZckqVhnfQjw|(GdntFu2jF> z#qH_Vt1q@ts$;>dA+q{`G*1)a^zs}2X4a_8EYHrsL;YAvWoV+SC((St5m}<7NfBoj=PL$HB zt}w8W;=8#yrMF|&FZ5gtm*2e|^&=QDW_R;q)H-Kg-I-gM0joMxcxvh2`q*?0WfmFd zoENIizY4Nwhv{DZx(hK>fN@0Ve5U%8fz&mQ0goo46iO@OJXX|x3I6%J0F)pex8T!tL1hP;A!uBaJ|=iWOa^=UJJ5% zJLNITcPGa@JXO?sdRiZuGK2JbqWer<5|KMJ%J4N9v^QXS#xauie5>LPjvI<;v*xpr zM?%RIVGOYV4Vh_5qi@AMUaln-c<$~jH@c{$9{{tTCifIfBsBrS zZQ^fTn`QjZ=Br{PI&GM&ZO-lJ4CWrq(Ha#6>Gp^ck;Oyq&ONYw=2>N!gqp?~18cp#R8s0}WTA^4=Gm^KfS zq9Vpx8?*n1ytfLAqiezd2bTng;2KD94ek=$9fF47?(QDk2@WB_Ex5b8ySux~Y?JS| zmCN0$<(Z3Vrl+U+be*cV>b$2ad2FqC!RcT;z`f~G;I|HIlNEFuWl_=&T7Pl}(hsOm zxQWuVuAXW+rHOzKG312&;qzZ+71B@aqdFz!!zA=KTKU_$nu($Jyt`5Eq8<+4rVXGF z<%0!%PK=FgCx2)lCCwM0U>H67cqWZdTDj>>K|B%;O%b1{QQCUa>pj%co74>v3HLYX zQ^L*voRaiAV%HMzAZvHFcRl*yfmoA>0YR22hC8l~R#D+Fe+gz1^kY(JnR|5eihtU^ zXWS@U$c{cdz29>9JtApr_L<7nx&#h%*Q)@eLrO@P4##zcH4sCwbZAbn zZtY!ggb~YEm6-*y=|8mt*^HLFwl;2{;u92=W2D_$-5}I=Ti!|p8cTHuKtlCIQ5YPP zojj~Re*5(t&`=39^X>38RbGRaY32rA>L)DItjvnMUP87wnEG<*=zNPKqpN+NG0~>XOwp}R_64#2cIMY3-x5|}W$Nt*pbe3kd(Uw%vjO0s@bkkwkB7bA z71A};thjp}PNrfaQr%;Un>=}l)&$HGVUv~U#Pvp{bnrUrF1~|^iCX=3(}UQcWkdIa z>bf*301&r-Jsk;4MCdeb)g0bAa(M`<%!-_+f*n5$GB>JB%E5esm#COsOaqr9Q^-Vw z`ug(9FWAIhV&WXgJUDVjCrxEw(Qb^_cYo7zMyg~M>QVxth)k(4IJ9gH#Z)KmxpLe|X8dC@BjiH+VZeiWY#et0SJjXq_5KC08>x*a-@%D0c-t|Z== z@8s`zE>g3P4j$81AR~;!bv|a*n#p^~Y$nlm19ioEZPR+4%LQ&VKl9P7s<(Bdtqe7J z!fpz|ah{}5PC<(|n!v6$Wi5J*W~aw~^@Tc6ocCj!J9WV;Fo+VKG8~QG>mBG+KtjzN z8RGsBqtp6_>6%MYbC`6*(%jNH-@LfN*Am&OSu%D?w6Ku><5|LoiGQOPP4f||?xb3`039Sh{ z!q(a(mRR0YqsM;XIv4ejI~)?lVeH@-Y>fWV&2%eaj@lOwW>!Pm`_9w*Ynoq= zCHT^kxin~Gx*!VT@`6Hk1LjBcLVVc!uu)ErM_%7%E3}QKdVF%R4PnLN7kO+0K z+#AmRP%W6d0v4lsIK$XO%h}C7aLI2V4-l82%-h;@`=JQuDOGw5<4%E5_!3}4UcYiL zp8G}7qNvruDc^q4Y?1H)59)@$6_twqSAbQ38^MhN@KnheS{wXklbY@4wlOiihI&x= z%qp*M_fq0*qa%bF;V7^JxXpieV4dAoFe(u(h3^Q z6Nrh-yW0^*bCt5K=R*yqTdO0CKbEozB9X-N)0-$H3&?J|NkEwi?^m{V#H*T%~2*bBd8lk@<6GRgL5Y`?cD z+F-3nsB)0T%LLx{l)EzmVr;lC-}@QpfBx~^4@jZ2{PY8A(DlB+!j3%vAqg|~vCHDb ztF-)@cn1JbopYu3wCC|kS6)mMhU3Dz2B1c2^%|0-rVZwfKyw!|ciwN`7%%2R;DALZ zpWFbKh7V4hSKqld!WR##B$m8;#WEwF;OTOv!?(zC3c()r<}n-mdIQbOXi z(fMkBXLS>Tx#z8Kb^W(8n)m1hS$o&B8aD+j`Qra*okxVKS?`9J_5c#`i1A8hrjbPN zH|7yy8KYr5Nj?U+uc>P4-H)`(IA4dVwixITBQo20&jJ3%pAaz}+MmoHe5=}VhaO~& zj=xZ*2ea6VQpp*$wn0e&c~e`WBrQ}!2}4e9y;Xk^78oU^{hAY)6Hj_+@=|t#(w6T9 zNIdLNx!P?0U)0;0j`I3T;87*3l~&iWx{AtTf9zNSou`r6s#5&}Di^LzN-tKZjR{ZlT+ZqwC#_~~Cmb2ct@-$cbfOe;dnjYU?{3U0|)e51KSY3piPvqfVV~H{6$L#CM6ZJn@`#3jirR!dq5eVW6 z!eQY`G#up_jx(IPi5Sg4#OL3-e`|Z*D*F+bRdPJ>Qb2lj-yqL+acf0)MNWuIxAlX+ zyLa4hQ}*gtBR7zrC+-Y`fKHO2`{RMI@mn%WLDqf$TkSBq`d)wG$~w;r53kFsGS_<` zDajnfNP?O5r{0oAg6jmQvQ{e?V0)bM6|zPzi~;^EiGd1?f?A9G4D={HVvOG!M|zIOi^5W!dqj8yDYd0wVxsNk~_F1MS>&%+YH2hWt!to{Cx#8!V4 zLG01hk+hm?u856(7zEFM7!HxfSB?S#)}3N%y)Z!20GsDy=E<75ZnKY99xEH%<_jPF zbAT(D9oxq{!ws!j&g}kaZaB0%V)2V^W7r~oQAbQT1BHZQqux@BoRWd9!OhI<$yWf4 znpe;MT>)VHF z(sDQ9N3yJ0zf0!u;^5`x1m&LrxA96ZAiRxjr1)bo-AMc8`h;}$crnknHVU7qe6>Kj z<&I}%vF^t}VNV{)TgTI$IwoUQcVeoKw`)r=!h1Lv_Ub6ZA9J0ETZy`uY?gYGVu8d| z_@bPAufUO;a@FHzP1&=Q$2)+Y0o8Lbhz_X%EJ2v;`FIHa8TVTz5d}VsbT6|mrI`jd6&UP;VsDSjU@yOMpv_zAV zj{m0yqU8Hh|L+lRpkaAq{>=pdg!u34|G!+G|1Swjxx$n*G~5d+CfF9eGy2>$$cyJ^ z^Fbw@P8NhU28ZQ@zl1OOOmP=Wwk;YHE0#4!YRvz0Fty?$L(I2$2Am%}%EseSv(Cp_ zw1fV94Q;maa%yC0j8F8>r*OuxDgS+ruOVxr^Xzl&&fbi$Vi7Yyc8Szy<(`pOdb$0n zjr89~&c%E~o_8#v8aG7`aC0Pj%SSJ*w!)rHiH^-)`7{5#CSoesC5B_Ba&Nj`r$^#( zBvAUzWLD@uC&WgEH&1$U#imSUKu2zkthj@PRf~1R@{O63Jhq$O*PBal@lA{9IbZ!> zzw|zz^Yu#v6){sa3CDUXPoKm2y`h0c^Vbzqq5^HH30w2q6Zl4F?*F;FT#T-jeLg}7 z88f-aIdL=aTtkE))TFr7v5cmf^~+^NeTkygeUDoIeWr>9<-92QTzb5qL7l?|%+>w` zbvL`oWk=@u;#uI+T986lb*_(`?zT67Lu7WXS1LmGZwpkz!$R!jTYl@x>#!H;`w<5} zZ`Xis+sV4;@9n{Sq2n@5XPK9j;DO^IoE4jhTDoV#sOE$onOa%$ZVA0#2E8Xh37`Ko z36hF#bCdDFx+Zj-Pv3H^HII7Vb8W{ zm#1piz-RC6TJMmvV-27^tWInRpL=Rrpfv^#Z@TqcDbjcxvFUQAaD?1X$I!6vY)sb46twp$f^ z(_xJ$5XQLqQgigv%;e$^vyRtY(ohl3An&c*SApl<8gWv&mo?MGtC0=0+KAtWF?_23 zXhcEv>tmxQtccG|FkdlL+))?ejjO*Z+MEZ-qksOS2P*)~OLQz z?*)AJ8ZTXJUf5vs#O9;bde+QO4Bgx~Q*{POjqkZ62T0GivZd?3;u*-6SVj)N@$Rvf zdSLl&k59yt>_aB+wSmlc(KC@(2`5 z@ShWu+n6{PLjJlOe&rIN$ytkFSsC)rrxJD-1_IL3QJmk}aouYoSw>LvC?nYi7mvv? zO(B`QFR+VdKCL``x3jWaH`X#FmQa=Z)fRS%t-Uj&e&oN93^ukk z-oBYO|0yTNOmWYv^QH1fdaxOk1f3EF&@B^S_-#ta0QsV>n3(CLJDd<=uXp>S&9eQn zeS4}y4=BRn^;`)qK9yoDUj^9?`TfF`@HJ+z_i*&u+C)|~8e+uV+_V)nS&NcdL zF5i^RiE@ULev-{Lt&u%0 z;m?mWW9*809dxUzErXn0ok5h7#-vmR>8qKjvZg=e-rpwiq;i2&giwARL z8s|^S?xxkI{?;hk{=~C|bP5>*VDrz(cP|cu8968eUBn0ux#@@;gr^&2qDbEqc4K zU|+$H?H1KuwiP2>nPt_LNM&pkw5r%XY_IOzES7Hi1t1N1U%;>_PZ!a^KiXK~49zMA-m7WHEZjccswPl#Q=##x~%cWmckW( zHFjt^2^DWxsd4q2>q{a&XB$wiaoOWO9hMSjZT*> z+j+8j2f5i-oK%7B*pa_bzEI6Cw6&OjSHHhud(h%Y2sO@)o-n4wU?3D8SV@e!Siv6z z_#uc&qBq%UQJ6uVj_ATa@c}{_COoQI(h~L&CW%eC{=ips&c_MUaNJ~}0n?v2&PL9K zHSzK*ZH2xgz)mXxX|2xfX^V{rb}8|}w2`hW9f#BLV4&iNjnnWV(CI`s)K{sOwFHU0 zdg&zp{VP$OPz~~%G?Cb7f_?pUV&>U0KW3a)w+k$^uQpOK5mbzJusW<8*j$XRGC5U#DaIoZFqPC8W`k%kGePa)Zq^S$*Mr+ zZpS{bvUV?=Jg?9O>khW9|j_lSgkQf@Q;GOwD@MdjHrTO{y|Rip6d*Y}GvrKMV< z)x7b)EHSv!r4WGb_c((D4J9u;xbk~dp@X0AP*yXu%QMsQKn{%QVm+vub#nRpRa1-Z zbJ!FBcO}sq%E$orLWl`vzqku*4z=jpjxokAf2!I~TO9jojRegN89`TsQ4ng_HRr|C zNEvko1EtaO$FH&!NBnH^UDKV4oQB4{zNL3gMYCp)lR4*Nsi8$}V&(C_4Wy#lIP|KZ=}KUlXjGmHb6BQKa!AL zJQg=obf5SwVRmkN{&e;+YVx*9X|B$c<;SMjtS8Z++K05kDDY-2G$h1t*&kz19gR+T z?s^};)T*`@*p4}pPg>(jSN#q*^iJvgE&2jY}i|2r(2y@*!_Y-x- zF5zq7JYQ- zu@QaY45qM-uV~;(KZJu@i|8!6u)Po5Pyh-S8@X5e9M9O*Xb;X=(HSpRhyVMa0zAgN7LDD=hb>yJQuoYzf?FkeH-Dv1*$SWTiXCT0a;wuOovwm^IqPb5PzHeu3}LH z@pWU5=-_i%T&#qJ1L7Dpn`d1@?p4KGsn8-B#K)Y2!JqllB8Q$e2~kpo;s@i+_Zpts zZsSFRvNcu(g=$XC+f4me2MB-pWe$1n%W{fe(L>Wy8Y@*}lRG4|YZR*LSH5MB4_mB2 zHp`wk##2+I`vBOOa~W7=nK*w>FCr;KMYD>C0YM%1GQz$7O?Ds4M~}J_HooF$T8P@0 z(7KF-VIiqf_Sd=LHOwQkm3n;AZW$?u8wb==jx?alW-aNiL+&v287Z88y@ecH5=h<)LAcP9Gq*|ikusdUB2Ko1rtT3-nQ?X!q7B;IExQQj^J zQqkwBu4nQL91J5{#TCPX`Z-NKdY*D==+{S(H>-Iu4-V;E3ck> zc*esJgteQkzi3xX$?XHht*ZCZ;b=0J^F@XFQC07J0m{UL$M4Utu=0&QM(K369{p6? zx}NnRB{+l8kEi-dC6E+0(5Za)=qfYSpHSjPxiDM}!_EdKO7aV>1*Q|bU3Q`vzB)Rq z96D{2F~|vSwx2xR9S#e=IJC~|^jtsWhootl1>(d5^4YiW`JL_rHLbIz2}4F5M9Pm*-RT9K_~XKQ~3 zKag5kLcgBCp8uT@MbyL>!Bay!}~~WduKz4tW4Ac z)%whR(sBs9waW?9AN%QvV9CjxK9N?#_wy*7Xi;KthrR*hea_<9Fs?sg##Vb>SZUAE zPT-x(ic&OOMEeip%&>mlqo4^yy^spkeH_e0y0$oLV&62i?~|o<$dyt`_(uEeWu4nE z_v%k~&FaxCZ#QKL{S{|p`a(VowbRYIl%M$rx*TBOu545!yNz#iR4x`eqV)6|%XK#d zzByOkWIvBvG{R~5j4(JNe)d1D8tiNH8vBudYc}ug zk(|D~y}XrclbMYN!r$?t=FYFBMF6p3GrakQi+zJge-K|90Wn$ro+TSEF zg8lpMM@-4(T*7Miy*oEWY{{;K^JdepNc$wbkuL`v&IRj#4^k}BYQK47k9GmI5HmgD z<*ce4%f>);JUHI-QMRYu-9BKFcYH0VSjO66jYbu2wsL(v1`T<($dmsW3;63!F%l;_ zfAfo9HtDyOxtag^WS%Sd$DKCMX2KLA9H!pk4KI!s59*5msy{5S8RRVkZn!1)jh=eH zUi>Mi&zq6djD2kkG(&867}ji%Vh_Xof2%QwB}`g3qcCXjPj95J@&yaqpR}XofyT@# zjsgScm`v$Uj07S=Lh^6n=^kj%3!!ySJ7{5jE$8W0jVN846^EwF7dv2A zM%ZrT zLA}Z=`8sgdDhdfx@hkXn|GCN6Hwto0H2D>=cnYfe;t?VFWfK;CeX9v8zslW)EdQ~^07c)zy#INJ*&kPTg1}U zIw~aJU4xYMzrwr@)m^K>f90m+9JpNdy*Vdt^yL3KNM^FQdU*4UN_ZWQnpGfBj=M4T zfn;oI5qOw7s*Q60a~(WD+Uw6|op$ix6<~1wEqDw|_JMapD&9;di^x>-P;|PKC(D6`uJjue(5!smXASj~ zHAaY}4*`W1vYLVtk+iQp_MdpaWENP6j*wUIwm>2ARmY3FfLTlN*?isv zTBUGuB{6z98qd2RWROVO-sF~Q3--OFqNy%asYl*$iN(+8F z>#lIgGvl^rhz;-71zc9Og|sWaLI={gu6s1RG7d!LF2U^hlY16hvCWF;UC8+~XBzb> zn8wfCD35Q92pyq-&{j*#QtS0D>ZW_Af<`xCTbTu+i>|0mY@t=tg2sBx(`(Tt%I?os zxPR>&*PVJZZ#~Ni6?iS%)@wdyqa`kG^F)5n%T^wC)+%N7iaaZQ%&xX_)X8(@t(u{W z9E_{qF|*``m_DuJ!Szj;L{sN{Ut;-9$fcJ7b-_bPa}jm@!3yPfSeutitu}1+&9l{V z)Qk>nwXI6+$$MP0W4LH%u-uX(aAzAeq3>RrsF9)&UG66^xk3!2H9B3jF9=_Nu5^7mlQ_EY}S%6{2mQO;nwgKDTB){qTm3aJYcAQSclURHbym>Es|xhlmK~C*|ME@%s7ZY?2(>Y9 z>yP*-AHh3yONf%a^*ce%%W$q>jL5;yeN8i*QLJ_Ak^wbOV}Ob@il9^wxz5_Ak2P46 zo^`VmBz+WyMivMt+=%??Spj{0mKu2&Pz%P)rmOwMid=0)HT8Z(Wh`LgcPoQ!sf-cj z@;-~Mxn`bjElSH?9a4!Uch&Fek-_Ij5>{{Il5LvaSJgmUu=%xl@vV;=jNd-J%Sn$d z8TP{Y&bT-fz8>OreiXP3+Y)Y%(ZDA?51czg4Y!Jc-xg|Y)kU0Th+Q8#rT9wz<5>qXz7_OvewY&HD1lqJR}8qB*Uj>LNLnH48SgIq z>m=s`0PiFznTsB4&nUcdi8GbrjBbe_WXe)A%~_8Aba7EPEf_CchLinf7oQ=vGc6TR!UejV~T*l*{_3p+C`4s8Wo?3Rrmwl%jG? zC7UG~U>3d>p_Da{Jjs#WEK+P4u+hYD*!l)Mf!@2VxhPx3kVta+7)F=-j_?BBvI+Ku z0!oSKBdD*@D`KtCevOAXRM^ZdSybKkx9s5XVek=wB8F(z;J$h!!J%w>nRwO!6~Tv& zYd4j;W^bFNAiFbH-T(&|{Ue)5+hz21SdNh&wnrQ-iF&M8_jY`IRXdL7i^YgA>*=>7 z1bM^1aqa9E4ojNdh@?N!sCPyZecGhNFyK ze&2=QaC5A-jMdskOYF^2lzW<4MA5am_hk|?w*rZf7>S?l2_0w691;s9xqAD}bYBEXcPF6RuvbeHEMA zZ0lzlx*K-5N-+v+B4?8~&rUzP%s$I+1md(lKrF0|+H286!d;I}a$xb7%pN)nSMR)+ z1x+Vg?=sT;<8iOeLxpI~U>9MSb7u^NST)(1zlrq!S@2w-CTHH7tK8Bf*OTP+KjsVl z3o_y{i(*rhPz5RBPoF5Zphn572}wl7Whmv%SAsNahwkr~>FsnO<#OM?vYh3c6_{K8taXZD#)y+Ex~@+W%aOABI%z>z&Q`jG@d zvaRMv#ZV?$3f4;x8;(=5UiCEl0YpPiX!h{Kpv`)eW~97V*I!4Z8Gb0eRfiVYE53ze zhq$EDt6dt!`ofi>fFx@^zoxFiY}f37Sny*u$|YfntM;)fxQHwh>AX}>lw`K`;|(9= z&LjG+2SS(k>5yd?FJH$wO##9$4HS}YPQD=nX>*-Z9^~vEqz*`d)TzO9w{KQv!B^BPmW*$ak^e3zzY~=ZG z37{5tu1Q0)es=Eq&(w6z&AxOk+Puw5(I0>sSv)&s_Yw;s%$K@rggBbj%CjZ-uF3Gi zHPXxdFGVnziSP_YjjvB5OoA|^e>7j?~we*DW~0=UR9gk^#;q9U7<+( z^zUF!qhWvf4dv&F-a@V_`>IkIQP!%tLuH)B?}Xp}sMXl<1fd5+2P~OrnQtm{RP~^p zF3o~3PVeTeGnA0q=^uH1D@hDLNMCaw^&(1f>DE?@w3o6qPgg1F&9)5Pdoqo3?wW_Y zmSD9@<~vG7sOF7+u8ENjVHJew6#qO1sOg59!9P(|j+bnNu7pf7m};STFM)dgCNs42 zc;E8Sx9j`;q+*#(*m$=p?~A7 z_;_(v!eIQIo?)xu)1?OWn`$DkTLwI(wz%|1j>x7J6&xpqyW z(%|N8qbn|>FM`3TV6QLo*}{54D~}&3r>)6L9di1kdHlmq4Zs`t_5DZkmIci`7oLYT zbBpEF1)NI`wJK~<$}tl54fhP}f)jqA0E56EBJ8)*`mz|xi5M@P<5aG4hF>9?-)F4(CE<mF?_yv-~ZN%f4(%_)VdeeeWKI&VRsKl4VP3b}rFdf?2Gu8$$lc|A+A z>Y22%an`1@mN_E#>Q8ZOv^vW_E6l2y*VBm@62~cCkM?aEF;1Ox88z;3sXR+!_DO%` zD;B7kj%BU3Y?oGC!^GX*g1X_dLe2@|TZw@5rceh9rS-`t?tv|4D9Cmh|0akaxo| z-!Umo&69Kn7V}pMTFKbwr>~8pE7<`g5BBBCagVy?Q@uLec zvKOZlq#Pv5w}odRFSfzPYOq06N!A~jE9>Ty8}WrV;PX=fQ0sA%mT9Q6gMEq;OVuu_ z^4P+AU+7)=42A7JqVRIHbHwR6KW*9Uu=Jjnufc3T{k#cxl!1_`#8jI+?JIQRRD)sU zrrgwt5{j^63m-WaBXe!ZgMqcMDa4eM!7{X9o=r(7Cj}Q_CSfhbmS|hLHEoz9FJ_IT z>3LDd{vkO>9>L=M{^bcC2bNi5mkD_llV< zR2~jqIIBPBd3@aPAZS4R2qiEyVAG+xJx-wH(mRT_Vpn9D4yB;G+48Q}Tq9?uCLw+1 zm}f>LVanoi9aG>_XH~zlA`rv|*)UrvVedcsBcjpumW-qN@lz+=zADTg z7`A{3Z<@D`*w@32Bm;>(>TQzhQ=e}x2j3Pm<<0;I1Ds*6a^k&N+hJ;PEAcn;lOr22 z@$T84!|4~n_L)!s+v;019!wR5f&HGwnOML%G@_LdxiZ|$h2yX9M3@cd5B((YK(o8| zKV!hhn$M(5{&|0^Ue;Mmv^hsMPBbL0?Ax=y(^`@e`Mgu9d0X+Ke#jPb!`s7F2!w!v zZg(Zf+%XVIYP!7z3yOY-Mn-F*19X9ITg&3t?;qKQ&VmhaIyiHUh-l6iz7$5#ke)sp zY;hv^;r0i>rFZCtE_jNnbx7FQ^E=?Yf_U=>GPnA1CuGo6=NljlKWU2OL=lQK!={3S z1C&`W@axs=FJfV{VX{LeAQF*3a0wWv{aQ_PaQb1CtJ)lMyH5$qj4=!HO=hfp_cl)k9`6Om~2j)=+X+7c=8`y zlPOfU==sUN(<5XzB9S-OK>>fGa%mKA4k$vu(NVn0K5YBv`-dE+#w4mf4zYYrD9oZUvc}+Hf8}7vlzYt}~ zjP2LzM~9|&qNR@6NNS{2_lbSX^=R!3E&4Fu(uKos7~_@1EnAr~x=2~`Zf6j^V5zy& zxdpCvJ$juo$z0%ttU{nqj%Ih;AevQdyC<~^clHSe{ty!*Pj^q%^s!9|Gc>@9_n)WZm8+vXN!NX#$=Wh(Utm z+3_+ATA=EW)D*BXrh>412I0_318?Ut1p*F9PF*om6nw?42j78~TnT%!S51;UD1clH zsGk?KVhhHyFOH#RQrv2`m=76CBo>{i4y3P2l#xenh3RL{Rcezd(S}qTpS_x`enb<$ z@BK0ue*p73%0T92H8e_SfDi($(08VZDAt)VUuoZ9mHYdPsE*CF{3eK%EysXe7InUDs)VeL-$aF{qq?ALUQ zB5%GbJW@4Fy5NL+Dg_gNz<#x9u;uvB;0}f^F^%ARH`*p%USx{&39Msc$^EKF0cIrk zL4pBM0xOqrG}5^0j;dFjyCcDxwV3THEbG)9XcNt zrClWH1#LFL{>FE|;SnK@LR7!1xUdW)?Zl0U~kjp3@_)KO4D4s{16uqd+ViaJ^ z0*r?&ZQQ?S5xHW6P!}AaE=Cw;cgI&=m>qC5C$Zs+(;!CoQrEf974m@!x-gBe%EhJZ zS=Fiv*jH#ieyG@5_>NZWb?w%mCa^?^4V~8K$#wddhI1WJzgW#fRZ6&*($?A*ebi%j zl2dQUiw)yN0bB9~`jAx+iKOXXRou0=L)+tBe4!fY)`vR8 z4*~;(2)Ad0BnK;gDcl6HwdyExC17Np_htThSJZ910;`b|Z%s(WSH7B*d|3y7BUtKY ziD9|M;^iUWfG;s*Azzgbu^|LR<+O8 z=ej4s=Y_>5irl3b)OO)$nin9y$twUj{%0bQM=SWB(?2;|eYcg?sCl|MbP7>d+!~2; z?6(0uB2!1ot^ynk$`whCH?h`z?_o)py;ujY2_672fwbwHc>mE(R+E$$+>gpz!*@8o z3A(~AqXq(X?@DvZ1gnmx_&2p&W1KSEB|`*YmH_)vI@y^n1WyT<;a_AD%#zAu}0;yH2}v0qkz>t zwZlL<-2{ktF#F*bmf{MIi7(<|r7Yi>b6uEGihJM|AwdD<&H&67EIgR6FM(lyR*)4$ ztWF8KP-)eVmWH_edMs zAmHmIz(>rx&B7oR^LgmmwgQDHbGphG$@}LRMTc!Vi#EXXPYD~$m$zUN;q8~ zyRGzmy++0JRvVO7NV0O`6Mg&eTb2vlUo-T+c552&^+c};tKBKr=nK*PO`BZ^Gmzvq zKgo7{o040}b-fu8#sljiBaj&vfG)}+&;eol*epZV!O8j--4!SIB2$d&R=j%yu70O` zqQ-a`nYnRr-s870>~bGSp{|V!{atj^uc#%gJ`RpteI^8j==se~ZF-(#)>F);(9CC33YeHmXMwZw`)gDwV_F$!3|Z&h$GR z_lgU$+)PFo{}LZeLDR)ecLiAUfI0)A#Mbf300yHNe;H7d!<|N4&1fE|9O_C5&i7ZT zy?H$TXCsY2{memY)F}|-ElMbmE{H!1nI&4#d6St;G>6#C>nB%*O*X52j0N(_L$`3h z1$J;na=)XpE;blscvK%o^?mUnvSQp$YXN0nuO2@&`Q~Y|PYrWAVo3vf0ZZi^&-YPu zQAn*Fnpcaoex12Pf$)ob1EO4L0|fDN16+sph0K<;yKouv$9eNN=A`kkt5&tbB zkPEDU9g5FF6Orbj5nu=eE;_g$VBuwan;zGZ?VV^v^t`9E8iUiYS#|6Q;~KY80g$B8 z8^Vc9wTlBn&w>FAwD15faCGsQ+5*#1~{`ILaHkIo1h!x)I zj;N`31>yRvHH)qE6OnGj*$LQJ^AdSU*r=1`;X^)aSooP(uV*}np1~u|t1L4U*-j3e zq|kGUmTO`+7pv7xAYHS@D`FO#2Qm<3fcCJcjv-u~`zgj2Y9P&C6;i-becY7x9=p1uH5c_HR%#8` zQ9IB4XORsvFvjCiNL)~gD<=8DtDk0UUyk5Fz(ulwb+0-C(8b6i3>r_!TFM(Py1^ee zusrX%z{r!jj@!=X;ZG~bplb8;z0=vRV`+o7!{OsN-=1~D*l59raq6mWgeNZ}CVhw4 z8|yGcTHv+q1f-=Dl$TqlK=?uU%}Fn0F5*~PuDFvScE z-hU4Zb9cviM&6}!OlzF7SyAk*mqPmI;M-;s#bcvOkC*YNUCuAO1OC#$Pg@*yWv(rg zcE^KY|C$}wx{*g*tvCi5Pkw+EFfv37es1#Ng1}6dE}&+^8U&in4a+{h1C z)o>@cApz)0e;kon&Ko&^CG~5Jmj;rc4AwW;;Wr>0A1ane;qfNTDYLdL|Hkt7-soNy z%6iETE12;-nvu&Fn9}D?p1K)KwSpx&pxLZRlI_TV+d6Q7`5GpGGsqG|$PqGPW`!*^ z-_$>my7Vq1-HurUP?i#aRn2}H{oMej-b%YNZYIOIvWboceD;&0i|7kKkhc^l0YkT7 z5WSk^41If$8h!kLD}2)Oo6&M9QY`7Q&Nsj^DBDzr^F$`ZFD$c3E&%G0G#&Sz$<`4A zx)kkxV^(J;wx6JR^9-{R3Wn-egQT*qBDxvt@Oc+dbY~Gz`>bBZwmA~)z846XWTBFW zk6Qk}J6N{>CV>akNe&Ad-X}O1|6bvXlwbRwLgeXy&_)oN`f8WZ6&qO1y&jN*!64m! z3+TFpjjsC7TGt|1{L&@^Kr|mffu~YF%OCRj8E@Ubo)gdh&KFAwf)%8;PGy4?9vsh) zTPuLJcOcNDyQtV$d{3vUs$*4X+c)2q34t4^J>&#rx{E+W9eg8Gkx zhjRliZ|@v_sa%k_(r(iKKvuU70kRx*vVH)t@#oq2nQy=V0bCDek^_FEQ;4$Jv{A5V za4h(IhZaZ`h{;slKD(ixk=JDtxxvcJeFLI`=(qPxqGFjC8^N}@Br{m2yB8VH;!a3azRe}C| zIH!opkF=2wU=a~Uyls*vlx8^JH9TANty(ad`_GCIo>)H8%G|@DtO_{2_Q4I9hBL~) zd3R+l{!m6EsK?oHQ(()d8Hjp7R&W7>Dr_zi+6_TBhqOs}ukW`LJS7^f!*ikVfYNSk zBjz@!3*0m1e)uPk;69=b0^Dr40Kft~fX&lviX+MZ2#c}f?dxA=huBX0f}b zAExpd9WW(&tW`NxOZ7vvI?w@}229duj#DmR@(`SlMgW#Ky|{NEHT0PdV3MX8g8^Fa z-A8!WX?%F=4yeC_@@)!#ia-#sXxMf8uAt>+nJ$*4VR;}mU)Zb?6L@~#{A2PvJV^im zLImsp%0f$Ezy6?0buXjiot^tFdTtZ^U%yXhZHJapkcCy$1ySHZzA82mQg-pRdl!lm z|7LNw6ffD~wnCYtabGI!*7mc%K5K1}OTA0ls&8r%2TI%;zW{Fdv1{|F?=B3JJq zGAX1oA?6(uN*6#}@btFGz@&-igRFeEpTD}8&=&>V&1&FeF!(|LZ+A7rrkgWQW?ZrX z!UFUh1~-W{Q1cb21yy;y24G#(WRX>cE3{!VPWJ=TPQ zI@im0h705;y+E6ST%U1uKAH78j_F9X6xiWpKxk}XhZ<1pz|eMcTzR++tXFD)#&{&E zcs(BU?UD}Oij!USoPMsQm%RC%?+ik;V2IiTDW>MkzYp0V47f|jrz5bZ;bmw1)EdEka%(UCUN0CWbOPx^s7gt>2*sNZ*p zA;}j+1g-?B@=A3ipNoaprFW!ZGpt;I*@KB9>U+OqI@J$}1B$*{T`sqLFdHqPdm{Mq;ukH;KJDK>l`*Vt!PIA_C0t72dcG3LZq3zbygD zQm?jcNYM>J^AEFbT}~)q8iD+tukV2&!UOWaO?+=89Vmq$PJ>+M@FSeJS2V3_UID4X z`fk26o9+Z8j8eeD0dUO$c;KXpqYR#+4p@yI1a)DpZ!3|j_*va*c8~y~x1Bl9X-GQu zHwO~d*UPC?b+HlBzrIuls)$VI6)2;38{9ye3_Sybm$ok|l`xXX`n}nU8!kKG zw1Tv*Kjc4ell3|)yt;gJ%%(wx-4s|KTqqC4l8G**4wm4))K!Hh~MoW0o3hz)I8gfQ)54)5;_D*Ss%~ zQCTFEwK=~^3qdTdwo$p?{^}7K0db{CLwRZbyvQls|Vr0r2KPN1P~PMPWtabS>k@NlL%XyOFno{SX=+a2NoD%mFC0$pJL^wpIMM}#+$1)y0WDX` zqV4b>H1ZgmysVtaYrl{B%g?QSWG8FCCJDXM9pdpe9B5i29;{S@YvA>;ZtD3}Z|v#TIhhS4ysgFmw4(cQ8xw8MmNzqD)&o?v@^2ya z|EsFzR#zTnnpT9kog*YiAwamleebGO>3sY*-yb+Y#Xgek3S4|7*QxV?T%gzZ`qg*oV~7Cr8V@$y zyomN2!7A=6CLSwOASG0N!*zB=1{^3lG6SK;mkYq2fyeDx1KxU{N9VT{XU~J5Qr9*6 zuk)z=_<|A13-5T%SKf*7Zb3?Y$~fn8)~0SS>x5};zgWF81^QAq9(Wh@68&Sa*ZcIR zw9tF)sJ_g89y$HS333265*O?Qj$Yp5km%g0)CEA2L0vfOu>CS{S1E&hn1CQ4#HvCD z;lV>2SF9{^=Uc0a8$>A%fnUxWQ)s&fbtK%Ty^O9S9OQjp&;r&gUC`R9H~~Z?ENh+>t4l z*e%%QIB$hNw@>HpYpzZkM0-VUnFYD`QBWj54`sIMS|18od;|*(9(+X*&qBIZ?&7vi zTMHq@igJ07$yS>phkfpo+$&JVnXS!91Ym?bFD$WDnfkIab%RW{ZF_$4u+6(d)FP72 z`}%!If&gI;K@}Rz*Dzi#J$)EU<^nYwf5@}^v-FYDOkl=hzWZ2l!LC!e&s~;ypM@*E zY1aPiKX1IP*PqtTPnRZ&tEti%m`et>?_|tNs)h6osT=6_yz04~^h_txR0{ZxlEg<1 zL&eg_o+F~!iInRYt|9!?<`~-|NdcRx0jo4Cv=9xA>RkQ|XDwTRczR3+ALBHDat zXF1z`c;pQ>c??tMDz}vC71aOH4&QP|q+LF9q$zpqqBM8UUydVX|JH|4;qQAyL!7SV zI$ZtzF)F3UeGGZR7$|4>`&)P?UzE4K+{Duv{~ZSd`T)GnMFl zZv$%07Nr&P>zkt3^$od8$gxfJIZN81xtyir|6<$_SD_R+$wHU+4njIW(Hh|*>mOzt z@z5--i*NUH%%f>v*}*Ofncq3IJ{UayX!t^$9^^>tVWeUBk0c~FPa`)v6+d!{9akAk zhL;$(HriTHJiDZiog=@uYs>_PBcGb*X=3`hPX}FTv_RP2My0M@l=|50=LsnB*yRQ<=aCE-%88g%W33OPoF+ z;7tPhV&I`P=@8L9Mo;4PIKiO$fb-y5)qfR8X*bia)6GYqGRSRoI|$x{n`*ab{=hi6 zHLDcbH8$Z%G?7ud83;qTnIVX>{n+I3k5cl8&JbA4&QCNo=pjirlwwZx7K9{z#f_Vn zO^7e{OxfzI7TQjGcjwY-nMfkE=UhU#_ujfu#*#veyrI77cY!y#VjHIs zOm+0Wk!7&AA|iTJulICx49=15dR~9CkRHqK_WpBET$Ny$wm#&QqYNl7>S0a<(gaDU+r7i6#jT;Uke0tC^3FvT@PYIlrB%(q!JND%YLM!V4dL<2dPl zK>k}85w8Z6TO7SF{HLPBStniS5OVcp3P#`~*+2dGV^}-D(e{;mBBH@+h)T!zo=Rt< zz~oo+!x`AF=F#}D#clI)X#U!4&CkrNK*;K!?~Gs$1E`>G%}kY=jLTk1ex(*(Sq~%s zQnE+i!0Hs@*A{WKOp$n^CV4}p8rKX1wv}_tl^5~yNYPfy zt~{BmqQu}Vq4+k$j_(azXw@5EJ_c=3V~h>3TB?>Z`7F4?P?Dn7`Q%33a8s4mp@=Xk zXetk}`jU@>`*-Z9KfsdzOFH|n*O7M!qL#gG{4b5pcQTe?E8agMs>B$)A;#@=hg)Fnqjh71V>^K9CRW5A^U+<$8){Qi~w%cfP zet7Fs{%nDW)8!_V?w1P5@IjAiOpVX1V%m+5#~df*4Ng&y>y`QQw!IOJ2B{sfTAl)b zCqDt&&2w}p9p&G0m;ZVIEnM*?h$hMih5cTgWsIa+yKpy~Ei%mfF4z{gN5=s5a z%Q@A>Z;;AYmX=33WOo%z~#IgocVa(B!ulz%X#MGUwk-#NyzwdBNMXvb@4QfMqrQWKc6=R^mn}-?Hg> z?HNPU>Ib##M|{$=<5u@=hM$~3`Xkwt5ojnq|C3Ss0;Gqk6A7-$O05YyjI{pmavY3b zU(v|nQKETQ>FO9tOlb0H+^;Fr^8YAB7Dj8qU+jQB<@pO^)$amJGg_OS2YT941xVzS z9L;fuE?E-UGMuWyye_7=nPx#Q9_;3EKMXj;Uj(K4XtWZB&@F?DSD;FEpamXa#34;f zA(iehrj${fxboitYF!*cv40OMK zy4IQcN_MialI@bkFwFES~CQkxq-)3FU{lC0f+M;b2(-1J;O#Ubx?+}PD|7}g) zKy2dKX0oQx>sjrL#(Eb1ThZLN-Hl;_h}L-WPCyZq?aY5kt!525j{f@nfVKo5ao`q96hVZYv1F-% zxMok-iTdkT*erHU;FwoSZ${Ayd-d?1a`buK^CF<;3StGf=8sA0sc)RCGi#-(jiG5;v7q$?-_lTFNIuZF5=P+utJ6oI*pyi=|bT+dK) zYhbX@SjD32>JiuBL149!cK)fbZ1zullE>y6@3 zj~C@Q&;mn{gH{(q6UA5TcXgtc)-l0*&nrMQQD2h@dWm^)4aUi=37HLdm*>lW2is08X8A&8 z6(5Q@tFKQ3j6j!Z01@=nkDGH2HR%f4jV_<1v)}LJ>eQ6$Cgzx`@(bZi!GqsaH(sd$ z68-LumHqu9G6>yap;9{6ioU+$+-WehR^knZS&SyZ|Bkj^hts}$t{W&O%E58 zgsYdue6g1nno>8AGqo?WRa3{df74UET#q2#Kbzk4e~K+y0N-O47{QqL$<@QI{U#A3J0$ z)o)AWV7=nW^3x%n?k!+Bf}NR9hNDwE*39s4v;U=O&QeEtrhfvPfx&1{3@2)}s4v#N1ZGnNBL*>%+m z6-LII^G#yMYmG(P}3) zq^7A8H}+UI3)&PPPDW2$A`DW$&1X!j&4n+_;A$EE`2OPz!K`Erz5YZY)l!Xk^r6tz z3v`=sPK&!Y&dFh(bz&ys+2O5av#Bn^p!vVZjUbu1Y*f@)SH=PTj0uCq5jpU zcw*Tf(;A683nDnu-ehJO=&rP#OS$Zc`$P)I>jAF~Vf3&*}je@o!57M5zn`2RsW z1qABwOQM!}g-3==9C$`uN~gBr$bIy)RA6#vv6T;oYSklCjbhACUQ3W3Z1;q$dXCz} zn-dTqDdP!0P6vi)p^vSxTDW^crE6abd+{awWEt>gZr}YB0ED|#7(3G+Iv{Fn)pj8A zX0ocAz{pE0-l_g09@+*&Zp4lE8cuBarV(nW5)4Aa2n;2`FT&L@M`gg)_4f`Z&SU#e zh`FTa@2v^9**0*sskxjQVzr9p(MnEVXBQJOjF#&`e>R`S!c`C;1@mIHT5+c0rM@kZ z7{BMsDomi0^7)4o>quhPYm3SSf@wYhSa$_~=anR$LR>tV5! z4Les=s1p{p%8a$Tm&CIrAO5{_4rz6QITw~`#^y#eNVjlc=XnDCI%4 z$g4#zYu;c3|j0GDTdt6aWA382^-fV92) zBx>GhKwzYWot#p@uOR>cUfdO5zqvuzP?`>C4Pd!H4VK4x{*900 zLr{%J7V5`Ro)^P+TA#ElUL)*vB!ZcMuv{XvV1;iPbebM11_mEt_p(OqMxF|`@$68g zGUPlvf2b!W6(=cMFKt>M#rB-z05{|?I@*pku=c4AI5sfbikxVkJ#bnd^C@mTP3F6h z$D>DU7r#Z@z;G6QGvxkJ`kL5pp9jR#`29-OWw%a3Uz6=s%qz zeu-J%pP*W%fFF9p>6 zRv6fHYWzRWen%LO`_re}ccGLf85)Xzg7mGnXfc}B%t{kxm;HMU?d0De^4ZLePASII z4*iEBz8Zdev4C+q`9(4&qqpZ6(H#xnB<6ifQ!M*N+i(dHb0J&Ekg{moJ(zmk$lGxg z-C>lpLfA3UjWG?8r7C8bg81nRd4noL$>HQ_q~G_>mmM)nMM&XM!(&N9v`|g8IY8=T z-Gj!VY5_xja_*?H;yof9sboEx{`uIsAGHKwrG2F@YkGxiU_NKfY+&Vs%U6=oM!&e$ z3x!aWYnA2nza{l7?HosJhG;}flOZ1m^NKgh;6%}E7kp6(&*Qp8vaqxZSTU-c&0q9F zM-Y6G#(YzC31la4FaT$= zPo<8)AAGWE-ZBf|xN!&9^+Oa>Z5O~(F3hstZ<D>ms7%GWrpGWHWVb!_EFV*IxcDsJv-i~}S> zOr~A##f3MK|cLXEG5tnUH zGi0>5OM5nLplt(cfW#dVW|dK;q0`85qJqCYq1|K&@Wc=f-x1E;5jNgcD}~g^mfDN0 zmj}%%=lMopIH5C#mmU-XI)YOi?NRNNGUtV@c zpVMt0pfVzHR7xlda#U*;7p=&c#A39={?SSj=0;vLPw#$s9*D}SOn;ef!&w+JW}N+A+*tnM>0CHX zW!2|L!t67ONA{RH0w$SzrBxULUnt?$70C%>vJ5uJ`7g#A}htIWe zv_?U6;_YdRcLrMJ!U1a1%)g?=i%|~nnw7_8EInA|2~-gcZC@-u zJ5^w4?Y}0GbRv9k_0hdMH7j2$V(SX-yQwntRYjL!1j8>MnUU)r1ae;X9_(u|Bel`y zn~s|Slkj6bg7UO^2V}Yx-}hxpe+cU7Q-19T2HwW%SVd7kZqn!%bblJD#s!SIGmdmo zoV<3U?2?&xmhj@fr&4!f3?yYCe8$1vAVxom6!H?(D9S}-04X*1`bbeK96lvyBvGl85Gn9s>!(7u&uf zSE{gta2`)Aey+FeLtbMfuG}^)XnX2FpIa&aMszY~e0ns}#(9?~G9YG2P5rC+kT*me z_rIOYv{HX5A{Ds8l0VmyLwr|ae>1)VC7bBPt8b&6A`U?3fPqQXtrfvH^qMqwLV=qwuE#P{ye`VAb@9x#m zZ589Gj|!EGZy{Whgj^#EP1q$Pxd3#62DJ4kg*PGXtMA=9|3qbL=bNwhL<_j1ZgnYk zSa6A!iku3aOxRBxTO0-(X;wM@M}u0aQG)H<(LzCk&&rI}rAApOAww~|8b8yjp^~`5t|9%kcHQ|tyGAxQY27YwAVN0n4{^~WcAD(Wef8Hdf8Z%o; z15zN9mPaMccZHD(;y7#jrJ3{@s)QuFvqRJ}wgI{_lbN;Y`F-`SFL`C#iu)UpVd;u^ zb(ZaKQ2kX+Mo7z{O(?#Zh@9SzZj!7B>e+X5ZKbHg`J)IcFR2_UaS(F-zS%J3TU3_$uiitB;rfU>W(di}rf%ier)r;t zz;la^b(lZ9(3gzAa0gN|jmV*-loL^BlsvAsem*K{*)ogfBMJH7i-I1yd>!S-t)jrG zkhS=zKpyGd^7*?V-v2zA;>BY=t6+XP0`2G>8(T$S`jQ-$Xd$XtgeyC5CxBItqv}rGB&(Lo>I<2R$rL<%N?1dBd9Q2e6hw(qgs}~N)wp;Nszmpx^szLR+6%`;W2M6 za@eqs7V}hW-A2UoeMZx^!>qXWQ`B{qwt|$llo>)FC#K?zzwIItQ@*~LtHJDJmnK_< zHw?3=jrM=+x29X`gmo}~|5d1Or1mJE14F6E$8XC zk>r|IW1EicK3ZRrwlo&}UUEm+y7u%Z?4(2tyr{YG*?8bsVcdXEHSU9xmb1Yk!&q6tuj$ z4EblEqn!GYW&C0>D^o5Xfl-WS+%;TvMcN_s&EB}CjOWx?9}t-A^}6b^ z)C&ajgT~gW?ocnoECoLw59N6X1f&E#;9B7$(`i!%z0bSKd8)9uW2kn z@APO?ywI|g3`4z#q70#FfBBM}cIN;1=W{wt!+2JHtCM(3ffi^>} z`_WZ9T$!e-l8kE8hF=^}Wxi%KtQzmwtJOHT1{WQ3V>g-ID{KkJ_2nwf4c=cz9m0 z|2?ihXx4cN`hYsW?#mzk_JaJhXMJvDeZhVEUxejDFWCD}>8OAhDy zJ4De_^Oh5z_Y!)qLxstjhVU18*xr~O|Gy}UiFGky;k4mBa5-z#w1=VOq zl))Q8@ljTmfKC4)r{m}9OT9+16B72IRDDcf+us2m+dm7s+)N^R%zJ*2VHr|u{G_#O zRCcGIQVi)EJy6W6$8~AWRHAen9Gg9x+(^YpD=^&}X-RFA@Qerl`#QOTT)Ws93^{>! z5uAP-X7i*;qlb6D26^4EG1vM1l1^6H@$3ycc6O$8%yJ;jMtyy9FBqQt%NxnI zrrNzA-`8_9v}-Sf|7Zb7jTae9B6QDwt64fRm8(i@A;rplOniV<{F9))3PI#fyL7tx z`5G?~Ow>5?)ifP9QI5EJS2)FahuqzqvBS2NdtNY83FHM4%2Sr3|9D)?)Bk29y;vNXwlkF5}qv)j1z?!zwt+1 zZN4L@%6y^z+B%uy>(Cb;WGY*y&X9WRQ}wZsK;^89165494ofxAdBSH`NR9BRkten# z_&o5Q%zMr?hq=pT71$=BI)8u_x$pmZrfVg?xSarMwsTkJ(ev&OSL;v5T-@898&B9a z=^ca{U1l7S_0fqZ>r$jubCNU9bUQf}H@1u@pWe+(gg7>!UL>9~m>O2Ty$j#)oz_$@ zW$p@3%Qq#Mde=#8wCs=!M2O37Tm$}xgk{p?HM+-c-P%!1v--zx{$B{i!4~n#352j@ zj6!mZ!`Yjbo>zX?w$7(>%N$!&T#VMV+$TebA#wSQU<-y(gl@MJ!g|&(k|skkJjs)9 ze#$x{{p@Vc_`05~dpM6wH)S?~jW;sFAEhlF>~MiN#?X_W3LQ`CEF+y5Cb(DUO7^sA z#!oMEHKjh(zdmxz`#=E1GU)ATUx;VJ1Kf>2@efR2tu4e|Dgw>*|Hz*+dOEEf0AM672{CGF?mA2Qlgz{5#{cPaz}u(0+SqSS}K@PQ;YjfmVWj|Ih?Ex-wdnrCpjpKZ#+6-g7BS zc0=im`E;Ys2duw$2VKy4+E8V9e!U*l?!Z9R5S9e4|5a}UOZUDf#Sua7lWXj9aop6$ zM4iI>|`#aH-wfukIl_Kv$;WBU>DX>%r@)&y6@ z*AEDEk9N6`jj^S7QRH-xtZlFG9j@2OE2;vvf+#=}BoEupuNu|XkQVAJ1t0ZUeV)k4 z((=WJB3wOXgP^*H$h9q#TH?8?7^JOd*9#eB*d2v7O8d{n0k=CDUYvD8lvliCY+IXp za(=a{4>C^VGWnux{J*|mf$eEQ$ZwEfqEE|pQSIKvD#FR=^bejNI`4a}AM?6=Q_k2d z{!6!#8cz}gwfD~Fu)*cmTZ=C>YNuGGv?&y6S0g0zl2+GYz9E;#pA$2j_@yhL&3O|t zRM{2Tpd8bka6rHgEf@}h-H5Wvj4u~j=GQE_!7((T*crVMtVt%GKIX;YOpVg4>h}CJ z?dajCUauDsNd<>I!S8KCC!`9o%Lkx(O*1G^e?86(hbr6_nhJs#wWH>+JOEeBlSMI? z+kfVa3;Ld59SDNg@$TOQWHx-OR7e_9p;gCt3#clhZuSb>44+?rvg*Gu*XQVxg!Mn1 zzWG1OnP5f|EgHUQYu{pEURp^rpJK50xBny(fuS-=2ArhhAl3v#lyZHHnCAfrW0-`l zr-RUo($isFUmLg%9ukl{i6FjoR)-Nz%MHQMf>yQUGMmTTiVa?4);eGa5}z zS;|?|F}WhG=MXN=c)hDHU%pcvTRO|HHX=ge0X5L4AE-)ivs>I!8lA&Q#r>&DZl`79 z%Av$0{uQj8tWBEhE%{_<(>$71`9=K<%TMEnqv5ET<(Nqvl|X{*w@*p+IN|cb%7ifa zHxs&)vvBOwr%XaKW|9dpPWK(Rs~$42`v7vd86J62JDjd`R_7ThYCkVZJaU zGL!w^+lu?y%2j`F1q$0hFCV4(8z9ffa3?_VL4cmzeZ{&o=od8Yi7hbWSFP)0A@;vZ zi@J7oRru6peZF^_MA3MEugPL#im7n5_@zKwcd_8JBW2^NmIyi2*En?Ju-qSr_>R!* zLMI(0bvq2cCq%zjUq5~|djIvF=G%tqrzMT>8=mW|TjZr{rJh??l@HR!Jqe4JT`H^F z^Sa;N`Fpf(Exh^G(C9nBZOeYI#Ek~DH1BgwSYnb(f~dLv2e#ww%^JFKocUlk-*{() zwk8!xuo6^?iF*tD9GFrh5@FZh#lsT6bLAmTy0#b+vvuR9i{z!4gZJ)F3~W0vE*64@ zrl9&%vcq+2zmjpKnz^}!`8ymexJyrIS+(Gmng4#86A4IDEPb0{$ zff5N{a#M-)5Zowo=2_!`Kd}lQeMWmYaJ!{{cj{-Jim)+A8q&CfsAV2zo~i5%s&By+ zX{O@RCUT@*CAN;$^3|C8rJONJEt4Ge`cp+qp`oXBa(S%IylZ`-+;gj5drTi`niU4P z!3u9FU@xX-%H5R8{zf~>)ALrefxtK10=`Oz#+s*9pT>l8tkBq1m9fREILw8evaSN_ z`0zzD1ZMwgX76R{JqOCanMwg^JLv%FYgptql4#e}Mz-z)g867w@cFiC?7K6w58Nm8 ze6(%zG3ZVA?rj_4ZLgSnPb;TaJ+%^;F%0&!WmF}@-tR@v+|u)NDutH8K_9%bCW@Z8 zvMT8+_UU&m+-Nm&;ELZ$;n~@17SbrbjCFgRDG(s#@Or$K=IafM5)4qjh0nNd$akTg zP?6Vo3HZX8GEZ77V?jVIq@>)lR#co<*^jlsB=Be*si zBPKYOQY;Jn1?bvIeyf=hXjd(8+*+8PHR9DuGyZT*n@ z6~~~I#%(v+tZX%Y)x}Ka3k>q=1c3D`yw*Wcc)0^EoJox*Jk<-qPF=^=sWU{^3Kg`hf5eY@=_{LRUs`ap6R66nIUZ)ClXOiss(Gq~Sp zz33tf6<1qHa?HA5m^n7BN6jG#{H$t8gT*1Ioi_rC)55>5>45t$3w5DFG5xRhevtelwlkd*Yzh$9rh4mtP1N<`AC;f!yc z8cNJ*#jMr1sU(n)KIVN1(w2QM0`LW>BJab2@$6oxgEG`e8Ip4V{St@F30&EPi}u{M z$AfbvtKR~zZ+}IE??w~gJ8_>degikL{;7-Bl{d0v;zov8bWBRbuQxI0n1g~4exgKw z~yt@PMW%Zr7Bb3}Zg4|##c#}THA8-6@pNsr*L`LkIjRa5@WNlA%@KAVc zV(^FL=D&!>39h{?it!&X;Wu$;iV?l%;!I5)sQ_`&eBR*W11`~b<5gw9m!V``R<)Mw_)+%Z3 z9`TBK6kVq1ghzkpyJ4Y6$$hPKa)&lKXs4~I8!bCtx@WNo9|9!J94Wje2Yt^UveQit z?b+0a8&0k?g^n~&o<&eSm@7ppdeS{gC%&v0Dpz;@jof}J(Pjp&O;MCMU|1}(g;`)K zP+1JJmx;nuy76c+CkEQ92TvS8)Fp3|j^2kmWHK?(M^_ffITGePvR|4q4Z*rgZMf`T z^y`Emy15s>b(LMW>5JP~n*1s1GE2Knm!<-HFsw&`vPnH~dYHR{^k|0mB`_~cVy8g) zPpyXUyr)9$L`wGa=JBpb7fppm2Rc#EL1Q-{@syiH(v$CrlZ%cVpyXhp?^p=p5-L*= z*x&E+7B?bT{dx_!Yv(%jOp?h|CajN&+q$SCUMG zw69^nLYNg%Aj_}5F$$#vw&iv~A>}=Y6{SB9Lgo>|;K`e$iY1qDN{5tMR+$1jOk`F* zUK*w;i!<^5F1JW8L8j8{540=Jo{1|TCRR;#W!rw>`{)RuzOdiyQysoN??Ka(Gz?39i z1ii;on;)IDtOToN7G*MF{Tp*CpPvz`NR~RA0xFzw{h%YtTsfiTx6S$)m?Bb z@Vyu#qbGeiRpG!qy)dw$s4(=_ekaPtt!t0msl!0kdwA=umFbV}9wu*d4em0?^GV+o z3%cKn9=?PQY)9vLGR{kmW0?8i5eHqSX_K3#BZN^<`S~f_FJe=-$@}XFP4N?kHi!}~ z8%KdfW0mvkPtYxuaIw+n?+W#=`HOm_klC>VSs3<79kr`EDjE+m(24hZ+%~T-%;5YL zk>R-(cU=YcWzYS`kcA;bL%dmUV_ki2kUTPHIN!bSy&8-vEXOh$+Oz=YLTt>!YHlDq z>JMC}C;h;GxRwt3V-LclmYuSM2b2A=moi3&rK`XMa<^t@Yk&+#Uww8I*Vx#xuoX

!sg1P3r2ML#)wB|iRB!iAt_zv(VsB;EW$yLDC!^07#?@qqOng;B3 zeWz*tT#G>s!Vuy=$4D9L1EKRC^A7c_v3h>!A+-jv-$u7kp@#~;N7k4a&xPms0_Xc1 z8Srg%TNN^v8rwI)_1;vFSi;pcDCqNUVGR&7&{vf3sONb+U*b>9K34DIq8Bjrul^)! zdo@q0L=JZngS<#_fB1e68TSq_T(JpWsDZte7 zMOY+!=W(7j|1n`bAqFJ=vglZIg2nlcmN}^F?Y7jhX(^$8Wex9c>D+zt131Kh#G}E; zU5djn(cGr|UY6jkB-gtGgEWHNG3qxrg}ib_@zn|J4|?I`OxK`j>Q-)3|ICFGbVw}> z8jx?{h6gG{-_X~9JEgH~ z!4o_eh1DVESZ)JeC`RG75E=q~KNyMp-tlaM8=1dM-f_sd*2yJ-0Ag+The zvSNR}PLv6r(70>}!4y^@&OOA)pVb)K;#7X+j@AMP3Ul?D;s7_+PC=eq4Wg5nke6}>1{%}}LD{xNc1_vt*okgflu!vHN zV}wpUURyA{s7JtVI1{lAg?K3HTxtj-oUS&xM~#fUTs~eeyq$qviMt`vbks- z3|E@~A|th|oF?Sy7#gT`k{-rApNcWAkwNi(M%T0^8X`ivtm>IObfARwz7yoWi4*$x zjXwmP+3|oBitmsYU|~QbRG@oj=-df!I>uiNYN=VypU1dK{4c@pXLx2mB8mGM4fxoc zIxbMC!u4zARCVACMG4c~dB|LbNnzFh6jbxx`0}&K8;Y;Yh6c?#&z(|cp;Ey%`&tg*b97>t_X#%|NpS%5pSmsIydI`^Ssj|*)Q3?@P0LsiVg{XsM8v~;`$J|f|GXSHG@-MYuEKqV7pbw(q^27pSoc8 zfJ}Uw^XV4>61xyGkVZeLHBCEkMYYK~?S-7)!TSb{)FrBn@Z3u#WIEkPHprd%d>@&+ zJ1<@wuqomS`4OomY&t}5a=Jc{;6ryE$!hj|WlZ;S!23k4qAQvJqtWtNZGHCTK9oRg zDgVV)j2XCAs&;s;>q+V3Q0qG$haq}kr^sWgq%x+0qfCvjA7Hl4NNY!5)uZJhYScrA zi>AFg->4@Yb7i7_7NYL!Vp^2CP!%Gk|NoNlnB~ofq?cmgCZFfKW7o~@-8a7|78u3! zsEHXUx0`}%R@Y!O?FDOzx7E31&-_@eZ#eko%j{&kGRX8_+eN>Fp&DaNYFRERwZ$N% z(Lgf!u_m#p+5w8koLL&2Dl_7Or(SZd+ZcR7l!73?$)`n~R|o;s&!lzNCYUKo(6Ft3 z!q`s*`hSC%h5K(g?RIo|;`(w{#OvFQIM&*WQ&#RZlr1!+6%~PrhFwWNgA7O1Bw@eC z$L=%BJkJx@-bfoTjg}GI&@LuLvS|AVM*Pq2_h}!-@D7- zFR{nGAj44H=#(3W8R1YK*t-|n3?J`BFC;q9gVw~TT#*Yo0qvpy78?3titkdtq z4o%<|KGU&9(gCHLamD$m#>Tb|+DO|up;OEs4N(Ro??lU@LrBwuQOL_v)Uil$FJfKoP&4NYO*5pK~ul$SKBM^dDZYiiAr**C}r~kA0n;PP;@} zBTA%r)Zq2Qq-Q9gsL;5aLcJ10FPgR$Oqe4bS zoj3#>;FX$v=rQ94`#YCEsE>3mKU(rB$;PYp*)fgiC`?{J3A@-{vTKi;OfBka+Y=JA z$56t#@@#}%A%{w}x>y(p9uJI5)1EgWR^4~kfA|>t7DOzCs(K+#d=T7cjkM=BQV3A< zJUj?&F(*mX@*bzWox;mhMfI~f*gv6F<|)xPm~;9fhR8m{l8eES_=X=&i3^u>4zW06X&GL+p@n?w zdl8uxWal0T-8#ETLPwIt?5dD0a8&3SuFk~Cm)sgDnHOXnNbE=xtO$i3JkhrJz@PR0 z&WIV^FRA)fwdG2ZBgbYy?9BiAd2srK(B_}FG4oDdux*;}gnp$95Z->fVd^y~l6<>7 z$a!6DKHNs+kAyoeY3D0#ujKJvm!@Rs>{YcH+i#FYb-+G@6U~Z4h&e^Exe3?;LxvOV zvNQ}t%l+<_)18nT#EmI%7K&aG#*>>6)gZ8{X?{m~hvBe=9>U6qE0XG5&cNL%of!kg zM*_^*`pkoXq>m^NzK2-HWYSm=rD+pEItKRy=BryFA#GE6Qa0KT5O9JCH{{VYJ+U;; z?e1%-HKT}Lp|g#Bu0Mwj9t_>n5Tw$SmDRN(Ld#F%a<<}kB2paa^t^6yqNfAP-HJw) z*Wzy8==t9*-jF%{8m$Q|lw(RC_i7MF!SHZUz{rN-UKIUAJ?x$2rtPacy(L zPvAz*C^(v4D3Y|F=jrqzT|Uc|*_ZXjvLniC%^Y21Y0{Xprb0?<9>|bbX^LsPo6LHU zq?n3aYkl`;f;Rv{g%6u=apjWNOI71vz!Bd=6%U2h5eSI}dGkn2#B8AzrJF*cZO>Qw0@w$XBzU zWvp1#_zvA~1utb%&`Leb$BXMPNe$Yc(MNzC!y}M~=T~{)q?xf;u=|#*?n-e_lr)?+aG2EUy?oHI!mYKCHU03nM zKJdgOa^jU;jp0Y#^U9)wny5uqjvE~QzYr)M81vqimvth38Par1`@{G1Z)?~Zr~N^~ zBiymdS+DLe20Za{%nQ=R$qN_?Bg_4${e_GR*Y-{y5sk;OlbD`^hmM<~S~3F(mgUTK zr+y`bA1$FaISTZoA*IC;jPB-*fG#0;7Lhg8y(i^O9YSn!WI>qg+_lrIFq7>+JkBOf zrJa?;H{vGPD`Tg$QcQtjJTs4g`+;~r-&s&UzZB%Ve8$b;I%&duqX`ymNB81)85V6a z7|39Vg3}FLex}yPJ~DpGUK{&U5r-n0ZcaKMm0`Wp^G=s{e)WxCjcu(2Ax5w=Sj?3+ zy!l#+ps0pC$|^>@vKy>C?ZjIsovHj?j#$)i?8Gcv@7jmNRbZQ^q5j@9$@Z};dXqj6 z*tS%~ZZcI1`|=@ezO=bo5%G{aG!jU{(BCsIDb^Xy3gCRRy_g#Wq026m$G!W+^a6wH zMEG88kK*uoc=wj)zi1OTXtAmv@(*}L_~w?~!`Du)AvN8=NC(iGrbE;E^w1P1$SKn( zrW@k{EfA}sS)Uvn%>Q7Y2Irm!rN?-0LzANCLyj1t=Ey3?CX)oS7QX@!4%6xtGBiQ< z8Ljyqc@Y}+X`-K8^q7IfJct+Hh-hEn4rGk3ZuV$dgQZ6J@d2O~`ubu_N(`yL5HhZ1 z89F0K`WL23qKK2N|MJaevd^Ezg&o&m@z|zHWd)$E)>U?*6UHz3o4%6cTj%_1_ z#fm1?spUHHUoIv2jmA|r7FNo26d(YX(~S>)AJfYm*~X70^5$bFCii;(nkmLetCdrW zeDZJU??qt_*ex zUcxV&=B6{vrkP}%0ZlWQHH&NWG%hWTC=%vHyAA=Nnh8?!4}w4*=_UnTE)2{Ps89{*^D?|@Fm zSmp{8u0pH~N8+IubcZ)slC$|iT3+@fD9HtOt6oydv_{5CJI0bx;#{v6&_2guVO>>1 zR9x>%6^zVP3s2#gPP>0I7~3Rw;JIXL+mcddD18(zD`{eTyK>-pO7Bme|D{u~Ll z{;s8`Fi1b_o)eR(qn6F6I|lE+0X_iUrxlmo6*TxXk)Q{~Go`Sv$T%p2%}VYYfM*qg zbagzTz5@sXW-l3ys>^2R~)MlkG?-m|Td zwzOy_`*14;g3gbnXgkM`6W>mFKV%Xl487Rfko8$AmH}oz(F@n`9x<-{fia2cB_@S= zQx?>@_i!R?)Pw1npo3C0gYV;S+$#JkxQ?e5%QY)APlIB) zIqu5}mIHu4(D{ns_0*`O`aAM$YH)#SR?Jrrg7MsSwMQ&h z7vmzb^;yd|J)%;FP@TiQHEEm@N*93MV9;l@aoM{&cvIX0}}lbXA{n@**e4$|yrK_s3c1GF(IUgM=6bj#r2 zt5}F!UjUP-Qmw67tO=5D8h?Zrka2OF9=xQ76y}6T{()5E#^_W7(!|MEj_~Dz86?VP znQjOtZ+#|2@1o9&nQiHlUAAgCJ)O0uW)`YsM>6pvqGeQ8^ z4oLLG!f$H#he>K^ zsPfT#;kiUC1Y;1?c#abmCU$g?7x;&1GOctHB~8`(KW3mHVCqV2It}qUjD_F^&#;hhy<^&%XPCY z-s}6!{|UYbbi)E{^*F;hH(Yj^Ma3x)h&DTkDF0QJ2EH*R1n@dc?qdg#JOUG~rdfU* zb(cvO1-kVGNrRwO;#$lKez9@NH&uL)bm{^C@J>9*UgKDi=ENKeI@6Dvs{b&ciUw$2 zTX&5g*!)T!7s>2*d7&^ccb7Fjha>m+HV@TOPqIUuPvFbrN>KFQb;ncCU65UptPtg( zo%m7bYsl%Co|1)XV#cJH@$G(0f=oW}^PZ$K;Qs}KDRNFGzZRF9S9b3r>7e4Jo*!l^ z$G(eICW+?|j};H6cM+`Uf~i)NF=hVYcVJ3d5qDXR=vHl^VL6zLZ;o2h6p3MbTX0zA zB5oN5gtwxwPkTlQIPIEl+zNH@PaMcAtv-u`8a^R#-*gjY?~25?s>FmT*@+hm`%43n zYKW!>+$?iSaeYG*^%do>v@EpW!{ZI#6C(E_gt%(_ z1WUUfDxOkxTehzEy~e&-cvR{MiHPcj+JKm^Wmr@1YmM)wjB3dnd#k1=5)+@DGn7-_ zO1t6ZVu3{vW2h|v;NsRH+G0pBC+#SLT@1a-Lot26G-oKMNbP7){Q)ITOC|&?ZR`x7 z5fW26FXmgX%RhdmD7&(vgm7~5-66e6Lp7&TUf9r~es?j36O#h4@5SpFYk%z@7m~yt zy2Z=7qxhqQ@#h+zu(dTp@Q%|fafUmT-1cBO?Z=aHp z$QrL8F^zocezRP#7lA&t3gSu-4A}GsW(p2j|Hev$e(_xB{4naoG(Jy;{FnbcGz`KIl>6-u&gW#tT>JF0~V!I19^7G%@W9^Qr2yP#q@9w)NlGkRAUV25+vOS`+ya^q)_kbO$DaHg`~^!}gbU1{8eL zm-Cwh6H01vtbskxHs(Ls7A-gisZKi~FRkX#VDyYPaGCYwr%4T_edRdhJr`BFHFmWM z(-jgQegO~Fg_HF_bJruyjNxo;bi$ZoI@#|{fmVcR^<3GYN+S#@-AuIj^-$mDktZJC zNe0`GnF1pJA$yd!U$?ZZ?X?u(M@zjsu=Q2$k{2lH9?f8mQsTj3Iarg%!OLxQBk4wUTucyrsjO3Y@!(qpiy+q5KAtmLM6c z`WS5L8HhtzG~kXeZ+LSGFk6>5%lMP&g&<`QYGqbK`u53vO3vM?)TOsAw1sc<@>@}E zK>OZF7X>2hWvKlny&9TSs_(tn0;OZR2;tWdI_+ezZY0vwS z7V&5Az`a9tBUfQ*nr>oTq2PTGdy6|{S{f$=TGaGG;Hixq;{0Z&5BWfdD$Skm(!5$` zvYi;eSAcncGK2{+V)=I@{||Y?7xN zl+kvl*eb0tc)W$0tDnQM>wr3>5=*VY%Xwno^}3v5DjHYqh^GpM9OE+@Ql^c!%!eIX z8UJw{^T4IFzcndRv1gti8{+mxY^I_6$1@JX>XnNAoWhq+PY@pq^UCdOVXcG&s~P{^6RUnpXbpfDq1TKRuL5<-{s}lvGLlEoLE{fYF(*oU@sI0^z2WyQ zck}V{X>sQLihtTW?|qN6UbvX6YB`hXKDW(M}y5m-YaB|)b(jt+sfXNnR5)S1uQF2-2;k~nKr{@(H^UH0f z{FDMisU;T4=rUTEQ=Av)=&9hm_%;;+uW#!z0J0yeJKmWAxyJW z#VKjlWBWciH$)|3$bBS{0@O~=pBGM#5{X@|W8uv6Pj5yTZupkomJ=u^yn}W%mtVPC zD*!?Q`hrh{;|2G8?VwH9%$j`kreh4+2F_E%kGrc|;Inghoofi%*u|NodDBbFgu~)Gu|F8iL4SA0{0~>g%=EI z96fG`7~g5GFgNN&v`@i#g$Y9RqC_SrYzq3J_${@u6o}T;{K%>Fv+72^D-PLz)| z;|-WiCH_pU+yxMbhk1fG8!7(2JeKlK3?8 zN2E1~Tn{pFPs*MxiHY}XvJ{;`*PY0-BC^Hl-XDo1pPsVehzl}MUro7~I|8XE@t?ex z%7mLU=a57`+k3S<+04&*C)C@fNk0T0q_Zyq$3hC~(-hMr0+P9OyUwjkGuQThMHY`z z#{s8%=8SZYabLhck<>hMPC{Q=^im^+)#HAZYN;Rf9eQZlpEyrrK=`t$8}>fkY+OBc zPMI5(9)hwSm|b-Vb=^$izNV)q#3gk1nbjK&weIf~&%yo;1(27I`^4#-%pjlLo-XL1 z0L1QSPYXu{iO_j_!fMi`i6k5B{PL*4T!GSarhGlU-Hsq(2=DFQ40DWb4e4hJtH5bez_vmw_ zj!03vXi?1I)1}A*Gp3sO1Loeb3~eH?@$i5l167HjK(caV#a+#BE z9#r;_LIP3%R+Kcq28{`jaWv2)Xba|lk1KNlBi)5@bsY5$LX;|7s^e(bx1hW6fl}s# zoA6#DGfksONH1#3e+RYYs7#U6WQCl{+GY#}s|9=Iy+HEF0oFbl7zOaGeoM==ix0$- zt0_7i#%k*`N~`^z+jX=40O|yTtp(hGipMw#bEUw@~q8%y$j$gn74Xj2Y^VVUb)mxsOvcY0PjUvZat@X#cE6X6jd z{0gv;o=TRAjy5vPveTah4;mB&@?@P%jovdHNYo4;2YVkwD`8O}-Q( zpT^b~B~#t7YdJC@KsX5Rlf_&gsGO&k(-c&S7rJz!bPhkhNSr!xsv3<*Uty{!P*rg* z%7XO({B6Hm?HUi4s7pIkpF%ktsobhKZv%P}bh#bbrA!Ar5XniZ5Ma)}69>=|IyxPE zA!j%w2UKD`W}=kWQ(nJd%E(6#Vp8zB0d-AVV}N&Aml<$2@jlJvvoifGe@P$rr7v5? z6UlGXO4ZlCm6>VtrhAVpbL>kQ!S@$hwBrxgA7cFfU zN5l9^zO+|$Rp@4qxR-af7vzl}Lm6*uUY#dszwRoM#F<`uhA}hIJdY@%6b+lr03&6U zwL$<8wAK(?kbePR^#>L)8h*w>2w$=he|$exwA$@?Iax)heD*s*Pa^$Sip|~9x(=ZN zyQ|R2KYr`UgmWBfe334bR4j{)=#vjZ(&HkNYS>d0;fou_-80>MV8A3-jU(ny2;R{o zS70A6h%2X!i6|m{cmKRY0`j##9gW%-Y!;ft#!R^c#q+9yEu#CRC#eB7od9AWBmt7K zAJg5x1^I;KUCXai-`aTWdCoPO-cPHNWvt8oZ|`OrNp;O$;&@fiQ6dhNE2GLfKjT&lRQr>>lzG-JNT_B7sZM76WfuM}bv z-6+t$a)BT`Xeo6Bm0~uHgCw3Up&hZ%>RP);bPHj8dgyd)ae_P}rHmhXiyV`qN+hp+ z;szIDF&LtbnPTQ$sx+k4aqThla)Q~oZNjJ%T#)>0<@Vdx>R}G^+MRsmw*ZkzcFO#4 zetZFpB`i&8GfAKH|X)( zS-@+SOV=iK95v24-yD`PUXU zy(BWK5S~Mp$4!U`HMvii*T`tvN zx4bSAmoSqI2&?R(L|R*|%AbXM!(ZiV9dMrTL+jzqodXLO`H$SLh9Qw@D|N8w) z=gESgAXacBgDP;+0PYAzbDmXuGW&aK?^5!*z~atNkZ~;^a^rERW+flb9WqX4TJ%c` z!W3%+Mr$k)Jq*WSCN|jg1#i5B(S-S)+Un3=MW{fI)^1}7HCcR*sF4x5Y2NbDDsA&`+rQeIJ*K1iA*$HJ}6NPh==pb_BA+SuKzZ3{xn~r0pu* z^Vy19#H`|bti;@5Jh zdD`=;+b=XhJ(Jy#ZlYII@}$u}pLrqwi#h)NSQg27NFXh0nTEO+e)IMTVpj6>*JL=7 zb=Q)wB4N)Z8=c^}#qP_f65&JACZ@OlDLkfQN!B0!5Vo{)W2}YUw2sfqPwar7%1L&S z8N=WC`1dBl)`5jbIlt;JbzY6h1tewu0<;l#T3!)CozYy=rO#*$gt@BRiSk~e&(+?T z+B|IXN#SKNSI9m#=Y7!<_xc4x+j->;hO_$M zs6HW(0|o)w8t%x;OeR?@`1(jfYCmj_DM?>UA_!&JMIo%xf4h`H4^8U3^Nn^cM6vDI zEkxMb;2kt+RPL9!jZclYV(|bnz;UwrM&XgWoj81-7js+uf?&Y-X1!t8SOlTPXhgCO zkVbO(Y{MpNXJck@t21#u2xRP8-mJ{amcxcX4k(G%4ncyafI$g)arX8Pu*GYz#pa4O zh5y@q{vHCZ|2*2n5AJrW>?`rdy|!iIY)m01PCDg+$~(DUcq3qa@SeIsS`bO*muBBT zTK$4pXPImIm03?<_IwV(-z4Z4-Xy==*a=rEwEBMVuk&q0Nh!hKr52E3s?8<#_mhN0 zHkg2l$io@#K8+D4^?pQ-8f+~puH7m+C9VY=+q%r5~|HuU{&U7x^fltcLP$EWlJN6&vm72t$r0WWw?l z`;#sM1Fcm^bT%(RF+5F*sc6R*QN+hJtPtQ7#S0jEy^Ro9S2@)lK@Ki6O^whmZCaBA z3@iR}(QOvVd>g_+M|?|1^04AhuLjn2;VD8TQd*0B`Z2ZNUO>qas%kyj zO|kZOF)AF>#m-o)Xum+m&Lm->{>SzR``CE<<#x4q>{RFGk?Le-HOR;N#RTdiq`hfh+Jt3N^Au=@p;-v{&D@JR4 zXEEQf3|bPyrl|c{A^9VgxJ{!&Q~DNpA7d4MaMUM9?m-<}R$GK@2YOZT0GX6BAEy6X z?&3UZ>;3pBHg$qbc9wKivg|uzyaM6U*7c;fy0uJ!-dD%+h-o^I&4EDtu2@aR(^#JI zKlo}>6n?k$Mf#|5^Zo5Zs-rLv=z8j$-Mfrz7#kkBNH7{wS+YMVncMV&_?JYXLLtTZ zLO_V44ap;+7?TqGXb#ko7reO5P(WY|N-n)gpu`Lc{v-p7bY<1MnC0Bwc3>k4th0)O z@-*Z9EB#k=2}l(1;TQa0VG@Pe6JC6%SS15&!WTBxyj9Q$oQsc*LI6?D!g$1w+O9tv z)6nlP+@n&U`h&)876!l}|DXs>j>S*-2PA7p&hUgIvqz}q4ZG{cy*8h{S6#SedFbYp z_6tU0iC$r8Ut&koqAq^ZFJPGi`pZZEH4|He&n&cegWN=?Vm>J`i}sRWiz*c<TB;1cnh=gC{xVd>tU1>UTu=|nx`j1rYA-;p*SYpLi(c1n-6f|K0yg_ zGw$-3PeS>+q{WfR2*0X>?(w~9wU1c@qy3ZJ{$>&&>{|!lCqJm1IU7YQu%}AjbZhz( zO+?%M^#tp#c*9mt4(7eq&fIDro@1dgkdH_WrtGX9vG0H5U#f;0!*hKLX{#HfRp$r+ zq&>nGEmO%@Pzn8?Ni@h838J!^h?Dg{#kn@OI%$VLCh8hT-~p-ykaj~wFj<9of8`p^ zQ+|Y*U8oO)(5f0V9+JIgSO=oT#i8NNLU6 zbFrI%<^Q!$ZSUg{oA2N+ll*T0#IH!&!JysIHRIhkf^K^RTGC+_`8_X%o*{RkBYoKZ zHyPnCdcOOuUAC$*yZ#&-UyCfqoyNc=5b69?E?+Q}sDtYSuo=zxnQmD`0fDF=Tt?#?ofwfU|FwPp>%&OM?_b|x8A$Isl>TD)e&8l} zNUah`|oqZY{E- zl^~lH?#>U^kt!2{aGt*Y{sw0(a&XKamol4X9wp(MoP7@VRNX6~fOn{7qhCE&m*8>R zP~Cn)+=={br=_JG$%9EU8fNZKFo+m63uNdf&<^COM|O<+yOpx-TWaNRG$xa!$@J8W zCd1{#I-Y zYnWoa10&3|x8!UR!d4#pDfavE#vpSJp2vCT+_)kiOy3P0yfNM0aE=Qwk~)a(E-J;t-Wp#&jx2%iD~#eWWbbxA zO}{F&xFRrZZ4GwbHZT<3M{cGK@-wsI{v}Wu*MFKSM$DG{bwf>@@m8?G=Ud?8LxB?8 zMhA|Uu1-VILS^vhMC{6DC!}>$@CRlmXr3og=kx*#LdOgy{mB$oi_bbzygoR&r>XlakivYl4yj)ruMvlY=M}(Hmy3T z^0y~dkMkqY7gCZhkLH3};opHm1Lzo_aR$OlW-y0nLH7T1w^!E+5I25$#+XY&6 zHS(B@RT$p`poFQ<+ue+0sW+Io*XbWN=3w9<{e+RbsEKK%Ds^;J)lOfXyAX$-tw~e8 zf^ix(kUH}kp;`P*zId&yUEylDJajmJ#$i$udx$QQ+<^l^kSOxVhn}TV?-6WeF#iNY z9bYQR`zs0xkbj{4WXx-NM3dMRst^!5UeefSD(SmM*ce`MjXHe{wcT*a-JIs9kTtxq zX4msi9y-9gTA0?LKuY#`dI@4X$yY-2Crkber}QQ|ml6-lX4gCY8jZ!s?r=YpaVbwq zos?Ug{cNCcuU0;)zsu47dG{~D_nQfw_<7{9&BDmcJEi*21`LLSY%k^>=aX)N)|Bj5 z%EDwrNm$6z886;fpVyEWm+MMs8uIQVI7?ERppb8eX!76xq*b0C+c_GMt-kLiyp%S{ zHxVD2XW17*y0HqKc?X}?5m z9*|541wkykf>5v6+!R=HUi>(J-X(Gp>;a`qnI zZxNI&(--w{b?Y^}w1MgZH~PkPRf;l=BWm`v+*s|5zo4BnXy)`00EMIK{-pZnsP}uO zDTlpI<}io71LsF~s@{AFYxZJK_OPfn+{1Z8u3Y6&UoAd0E@!-2ofvE-L;m!2g~CFu z)T1qG>O>6ZlMQyZV-3y^58e)U8;nCn1rmW@+rArEn+DFfYW7eFb>G%-LeLRvC+Q89 z&d4(mLx;WvA2v5sGM-1m``ur2xu~v`mHVeao3c>~@{!u~dUj(--p#LWvKq!9dH}n@01xMG z*2frxZ(;?5+mGKX{R+~g*HMvPpvo`R{?K_uD#gR)tk9)u_>MPZ8=w046X*WQH35pr zDeJ=>B(|4EU&1(NWp!hi-3NZc!&roO{pd7w20HuL78cpVcTn6d*Vc#`I-Lg!^_3>w z%I8!9;;VcdlAhxXWPz6uULl$~otGVDd6zW|anD}Zkx%2Q*lGl~Z}D=6@4UbZXV`3d zK%%-2;QIB&Z-&z=Q4qLS?cyK^{6rlB!)B# z*)9UyX1tYTZl*EuoRL0K3hH;CH_4v~A?oG+B2&E2MYLf16 zvERw#h^;lsI`sR~d|Kx?f14q}O&J~M&9jv@@3O7wx4N+GY3=*?{yh0oD@oszHM!ey z58o%s$yqB}{f=fp=!QS8n*7biNKPj;XT*CkqzOfB zTr8I8-Dv+g-TflxsW*C_XA&}*PRDTg8kKf1>3qHRQ(&sI-Zy$jcc*?ytHbns^~;S zYjdOAJIVi}sD*8E^poL+^4ZL<6I66im5nZ^j2)%){o}s#X{4;m$iDxmzi_ik!Dn`) z^6tN`hcLd-|LWLML(|_7=-!{T zgFOG`Ua)-iDqy9??Y94E?^128^zS1n+|J+9^6eK3nItZ2S|(fMv$AMSC!R-)KXwg= z+$RCINCpzm{ZU!>OZk%~+7y#%vl=TnVUB?DF+=Oq58h9E<4r!|1fiM9GD^304K!P`WBEDJlM1DqhxppJ**rL>|}W;T55=eA@=~WN!K29p&+A)BP;< z{xB<)K@5R;pQeDQO|ydP&vm{UgkX$vBI1!X4fE8B!B&_GRM0g#KH(kE{g7h0u|Rx~bKd?|(K03^PJdo`Ldr zyULb7sxltUd&$EalXaCkci}$CzVa6m8W9AM`Wvy~c<$hj=&8%SbXCPMG32mc88Y&k z;JS@|G?kE$V2cnuOL{+@q&=Oi^B;Et>!#j+nGl5Ae@&-2e>Q>5RT`pqpJ&!$E*$Zj)W7=A*d_#x|5k%_nn?UA>jmpJuk4--^AnetdfU`t{B%^v;}< z={);?+<#ADUt`N5^N%&IE3)`nAT&(JB9>(5tgr~Auwy9{>T_%_9r%=&6?jS_r}M#} zwpIB3Z1QUSxa5HNhTdRuj2$)~7I=eq>fc->i4i*U$7c%`3~AGCma-f)p?1M9@+ht_ z9#IS5K1*E8)j$<_dLT$dXQcV`8C)MbMr0rlh5Z@|+lGF)D%O6d`o~}ZR>(~o_LBbn zFr)19Dq6^rY};z%1i|sfO1|+D_W9x9?d8iD{2V{a7oQGiDyNgKiPLaW26%L@V6O-1 z_C~bq`86s_;qUKVwEt?5oFhoW;?J4JU3(=l7ms({xXNud^G!9L0Gr+jv=j=*&C-R5qAVX zaaSt%!S%_Gb>r|_y*zDn%Pg_R#nOLl&})D&b~Hd|slFFzV(P3j`A=&7ZqLc^2Y2S$ zct|34t;XYyJ8#}KIrKymT$>?RwVedax*dmv%zU?|8OHfE zJMjO-3y{_;pg*SnyYu~THL>5<1P9{cF{Jjo{Vm;RtWmFjmS6}pUR1r9STUe4r2@7^ zffrVVvXauIGK%0yvf4$m!GB+b;kbX=(Er}a;HePu6p@Cn5ugx3zeW0OT$d#Om$3c)`0Hm{md0fo9!}rCwobfn4+mTV!b;6z2a6S{XM`Oz{Z8ohp4;DUkck z6%k0-#s8jrPvy4Pd1kWQ`VM?$Y>dgN%4!?mppvj5Btq(%nbM#l%0X&0Eiw6rd{m#Ij3kO1sO#d~aKlk+cI@5LUb z-7iK{gQkXQh=8Bwnbt`CTdSc8{^+igG3=w~o#h3jl1Z}K{~D-hRswpOvL8S7{ttUD zMl1XutNxn=dFn>d|7Paj<0_uj$ooUc{YfF_kmzA&3sK37Sc*o?E zTVnA5XoGL&{Z73fSYc5cv-1BGc2oBG)gL)yGj+xmttalnkR5EtcD|Qeul!3NK`reK zoq2vt>|j}6YnL2nQxMA8ob$P5L3>(b`-_fF)Gc#SLTyR@lMN1%?|e6QGk#8*g-YRP ziagZY{7a_7O>7K%sjFEFZ9sDPW|(Do_taF(>P1V$QaP-RpvZC8Q;2Jv1%)i%JT^BI zS-y{b84i(s@yIhT{I=3r3P*WZfI;W`z9_mh*N5PhKSd{aFn7nIClfz zU8>z)cI$XvPma~w1pa0i!f5PFa9Q?iAD>J#89_>Pcsm;{M1febGPkA#r6yl68e{(a zEdnwhY|1w*sbjo?WscZ7u)xO`&DY0*ww2?_uYgAKup~`hFB!#r?0B#=BQ)H9_3T^L z;;XOCDGLmz%({WKH$?xm&8^#>o~ICxyaVO-H=1k&(8C zl)pC_?@*mZG@kGj?n>hY6&I`XJyNUsEjFLa(bkt3<-Qf}7zLrFAuOL;q zsm{J79wwly{2K77H0=Cj(?N?+WVm8klT`PKB3}*NRf|`w{H??sZg?ou`2wP;fKbqD z3Tkjw1cfhOuI!AJP~irwg&McSe7EaWD6SdJ5D*h5J-OOhJNNfL>pbQ*O{f%@MU zka3fTJM&hod1SRM8!FF%1WBh!ngWn>lJYs3Ow& zTrmj=SAJ{cf9g=|Bgc8M@*AN{?c94lk@fv)PRcr0VC{6X7eQC@-399xF9!PzvbGAc zHdDv(8omZ7f%y912Yr>-xIKJ_de!_3N=1Dr1b-Rj@0&`;*ktySDW@CjO_5FRB zhW1Inxzp&~x4D-z=%b)VY&<uO0X&J`)&b?*WCW{)ru1^yN>PVO{kc&esl@ufQO*Vz- zDem7KW8Y8xK*-9;`7XHdKRU1L)`q-5z|9+C=AB##9CfILtXq^uz%FWhj8qPPc? zO;?_^wbD-w6bXa!zegsi(MCoW+llY~^zaP1s+%Zgq4jgZ`nWP8V7NXi<_QY6pa&F! zj0>oAX2aBF1`5$ydG0W5rXvNU$GqNl`zOgGpYuG8=!HHKk+Pr?$Cr+vhbAN_zN5`v z#7t8y0z^hNJ}7gu$_bv4{KW24`NpP%<3r=l|SZ!CZ}AC$+uj>iu!Cc|DM zI!(3d#P+(NPr5VI3_P))ZIO?MxrVj=h-Lhaka(K@5hVE|)L}G$t4GGb8v1^%JxTWs zEOmySxLbwDr31DJu+KJE@47Li4)E=c7yqO4lPvEA=hD$p0ypA9xm$kC8R`kKFeO&+JOe81l{oFR#_RRnGX)`y>3vn?sx=W7BY@)qv>2HgA zN~G2zTI=|Yo}%`{_7yA>t{sHW^YEG!H%A7bFs!D?ar@#Zmp6tl)y!4huc!fqvcClF z1PvMf<$?XOWK{b%FmaFtcOUmB^sBz#jjuk?N%5x$(<1uukOj;eyVwH z?0!8r`;NE-$*ZEtPeu%-MJFoh)5oEU9N09l$nWzqyH){zh_HiTac(r~w;lmp#s3d` zZy6QG^7RWtI0+FX0RjX~a0w9Hf+SdQcS~?5xDF(NK!OH$26u-6X0QYT1a})?2=0T= z0E4{EIp;qAbMA9L+;!LecHdgSDyplytM}fu_paaG)vX?7F<3#58tCJ+I_94-h11MF z3ppKh!Gkrb+M8T`7uWxhKEqxVruxiHO(3^B+EVRUg!H+@^sO}hGE7xOlr7FLyv}j$ zlgr!!_1YrrpUhnyg_c^K9@&ah=MotrquiWW?$4_2za0*&V$v(C*YCXbyO*Ev3g#Xu z>~FYCwCUMPDvmI!MW2>T^!vTTru3{Ne)72c_Co(E%>K@!2H1xu&e{`#CBd;oZ#H#x zv+k%7I}MK}OB0DY0z$&8y6GF(-U%5q95w-L2yCQ+gzLT?SJGmd|!w z%Z+F`5mdUWVip8-|Hb9Mo*Y>2q6|EkAY(9)&;2q&@r&e(Pf#r#9p3lfZ{IO4(}RN(#kck0Od)LIS8s7Fsb4dA-j5}x?MU=`pg-*Qum{5U z+~0@Q(2n??U`}e;4Gigbg?B$wnZbG*%CKY5ZO?0(YqehY(r1jvVGdx_^sng&{nrl$NO$+^k&=kBY-p&XTt zH<9JP=XI^BvaBAa>=v0KiEpW6-?GM|_u4R^IorldEQrW<_9uh=oT{>U_l#Zin6B~6 z(TXgQ=&MRGyL7lh_THIsGN0F-BN2S&mJU*#3&%(K5bkvRhi@251L@+{R0^{$@7BcP zXpsyQ{N5|%Jx_cWOn`1K`uKPBYD?+C+of5*o3kxsaIbe&^Uw8qpVhFh05GhIuTXhc{&T#!~#~>c!Xhg$(B= zFTWfMP7xK~B4{A7G-869x05vQQ}_@L(NV>{QD~!b9V}&Uy`Q)C!b*?ECotg4?IyFk z`T22k{E`E&RLQdxHORy>FNJCq?e8+BE2Drn&)VuX>UzuDxI;5q|eyfZ3 zwOBvm3Ly!j`mOLaNRHe8WY7ETOeSwGzVqAOiW+K;>X2!Oo+*AX>lNs0*%vipZ1^ji zX&YD174G#Exnwe8y2}d^qNyaLu<1q(w^28q)#V8{sUzX&YJ~e86R~eMw&PSMM-bAtgt@(va++N03 zH!gcp4<4NRjl%!oRsZc_M|ICph?08s0~;1;Fvq$Bc!2T$*Uz5!Kow)xk~+tDqS(@s zNYzzGbo6QC32lIqkx^)(Q&j8g{J^xIQh^3PVF1vc@6K3jV)9CP$Xz1???21s8_n|$ za_S0RxI|gLJPsUE(}rAri3$(j{_fQV(pMCeGtD0U;J{;=s(iI_tKOeRt_UqD8~sb< zw?@uCg)CMDkHaKS0GOV5RPSFYf_1PMcvceqo{R#XX>ub;h>_^kh9mfIz4j;R^D2$P zxzMctV#BD@{~Jr||4XpLfII;5cak~I-z|nXEVSZ$AfSdf*oumutq{XJwkHc5Zw!fw zv>Xqq*W8T<0g3F2>ZGn!1Q_R-3y99(|D(vp4f);k?522E2YZ_J;FI{mT|hWvS||xT z&mX=7uXp;zJxFCJy*kHCj`4Oe6kH}Jdv~y6didRb!g2nadAe}@UDMM1jw5tl50xNa z45*@K$4%J+Y|n!+0y2P8`^5{FBSEnLIkxd9`lgaqPWe>c(NPY-*^ZK`iQG_#4yVs`(_NQeKtikm|ouzDRb>n)>G@cd)LUx&7$ zDM0AS3AYaEfYo7bpMDd*2sq{ya$v@28scc^U-uxXe-(+F_>B*#(q%dB$lKi&R^@Pb z`4UD~#5ls<$l)DaA7FSuR%MXgI`v=9Yp%GKql9 zsff8$i)LvMKj#V`cxM--Q~&Vh30ZwVRz*|iJUdAP@_fFYE&71v{$7)?_uw0vR>mOI z+O+pcW%p;=U53W%?dr-2jh4Hs>`A=nlhye2%noW@&xNMGg}9*7^0zZ?w^umke@(vU z%RUZ=9-Lz{fNu#o3FT35lO3+qoslHS-Syn<5tNHw>NOeh5Z!f))DTDC3lQKFLn^G70~{|%m7|G*Auxpy-rYA|0zemw#T$F^KdNmXxs37JX(0y4OcsU z=(I;>$ELBI)Ytfuo{1p5VTgKEWwMngj*pw)f8_L5(&&7abVS@DZ%_lbC8gW%}$1h z)&xeHb@0~AV#5Do-974m<+f!s8jzw~$$~M;)p~9X zFHQT9>DMr`PEcNxy4xcKUB|uLlKA%dt>6A0wBxJV#SX2v>yeL9;PzckJej?DOR#8T zM2u1@pX=Vx;)xQb(};-kOSiZe`S$t|#^({|HCpruMbp;{xk6q`CsigB8fWlWwe6)t z04lK4U17Js5WI`7&I7^JsoVrjS1)d5USx?92e{z0Eb^Qe zKwP_lriMwIFa~Ok_75W$=dEc^l04WjBrih$jwV*5MdtrkG!ee{2jnq{itjhDxUtfz z?$l0h-{gIvQd-@UXy!K`g2mV4+I18f%%kL%Rvzk4;aS^6+k6RQ+HIy$(@_!r{=Wc8 z=bf#d)j#GvTPHl+Xb-Dwy_8Eiyj`0-yJ;4ZIr%mh;IQN4)d`n$gjgb~dc-48r5GdU zw&}M$lNsotPT>%h7|+T zg_8K58b(GcmDr%W8Ml4}i~G|bpAhT_&E7O0uU9bP( z?zoQHxh>w4OW#rP5+p=~Lqla#z`r~Z^@VZy+|cKA(5OO$-D<)3DwFDUuxjZo3(%37 zZ|ALQv0t#KndO!qeH@}rLcwqwU&7W;F*r;2Ze{TJ#u*u_Y!d9aMb^Yw4v`Vj3`d*b zBj)Rds13XgA1`v=3l@>xd$Nw3TQx+j(&=i#A`)reO)!s^da>gsXV!fv$g+iqesvE} zR`kuq>E(rBsY%<9u&K!iZv{cn{pof0i2L|fOP3N$QTBt2F(^u$&66pK)&YzOsnG2gZXX>zO zl$O9%08HNbc-b%0$2Zs77!wQESi2YseBp9N%zT@zvIOM-r1+ksY}-2XMt0l7pL&LmcHOj8=~{~%hT8lQ*AlH67cUK z$(eZ`omA1%ERlT}&&uuSf5g_*^gB_>Lh!wS3FJe?!uO92z4(O!)|pH$<`o@aPm^45 zRQByI9A#Eo?k7bnBf(_vOVbh<^cs&0>5 zuBs%p96yS{*`$4iIb5DbEVa)sy|J_}sSNR^sfaMyb?P$WaxrKQQ)t#}k6lt!MlcNR zKn6X%mWZx1NR7nu zYlYCqa|K3wQ2ULR^0w+I;@3q-itiHysuDbtgnyxIn0F#xL!xABjjw(Zz||wzk)gW zWgzi8KyYNecX8u?0>O?zLLSs4=4`XWjTNEOFeE;ux7we9X0;6(EKXznDem>=N9`b; z&ynAAwt4wRU;1#Jb9jmQ9P3D(#4jRU)MnTF{&^xIo;xp!bd5t6!}v)mBEBW+2qyA% zbO8p;2rytqQ3HlLjW*Bv%rGzN0xXGm9jLmu2i)f7Tvj@KNS+T_D1jRbz zl?TM+q;}AHd67GyfDex94-vIu?cGP#`&(NybMD<(Q!xm~Zr{Vg$8mg1YcS`L_i%Pmf!@2wYn}bcu`!?0vG)Nk9?>{tCYdihHTY`3^@dlz zv$@sWxM1SuP^LEs-KQFS*+BRtqeNf8jVbOr{`=)J=HZq9b@BRBz1iQPTj&&lGf;bn z^Y5;RAphyW*I(42j?9Lo%fBv1R8t;8LTs-^<7^-Jy;zLBdp*6k*Zp*7H(b(t*@%XX zt?;^SUg^5R)UqksOmuR8>rgM`$`5^x4>&T|JQWl_OVO(9p_SO`2Y3{MQ6{gl&anv5 z5CeFUIb4=K5X8i#bF!FNIo0xHy>W5B*VN||xTPxQeU+&0RgtOjNGibGc1oYcvq%f0 zIAzx?n+0PORLah)OW5LMTWRw1{9@2&B_B@kMV{fWUH=cbv+Jm71NDKZ^7rS{wC~6C z`Jbnrlf)gxot`8kP*BoxXGuF1%X4dz~p-H+CK~Yb{K8u@q2%{llaKTG&=u zE+Rft7@cm8w7OgQ?l5Ca-v~d0NA58E{AGVt3{_|Y)`FKaXZIRMaccV+?Ki#`Y4eta zR?}~yrqc2!!pEJ+z0l9l68vYGq)?ToO#NFkZh9F4vE#`d5+<<(n=3dDrJsKzcfm=u zrvvvtRS&R4h#|)Rk*NXt+$c04b+Tp5@K;!Sc;L?U{(R+9 zf<24BpyZ2_pJ_y%fs2r%9!!;>ph+5*M|eK6gNl;1L^P;k0C8l#A!@o_Zdz&W*hrY6 z0=%&rXORU(a`IW1%UbIzqMhw6(*SC2yKen>%IJj2R!IqN)vdY#BNu4ut`(P(pW zihdWWcf0L$p9+7*d1ethtEjDpBeH~`v{QSm0uPeSa}$nwi9ZL2&iCgjHW8bMC52w` z#dTV*jV?^tZSG%)*~U||TOXvgwI@hWD7apJf4b4QSVhFymeS1?t=8&5+U8UC0U<#v z9|A&tQ&=Cg(vMzL+Ijr;`h;i4Ep(C<^l&PF>iS`?mBm3&^S+5N-plVnyKa6HvNnN8 zwF;&1+h$LWBi+0Y2=_EKW@p8xSVZ+vaKCilg8@IQMKt7O*pp;n3Ee41j_z}@aAzsp z-?Tovd>{t3miEG=M~R{~>6eq6Qa-gjL2vt!>$b#@(GD$-=)!mp4*%eFVCY`t_>P`W z8>;r{n>l52>uc@u0HH_~Qr;~K-IxS@WNkKpez^UT;GmaW`rOfD!CJfK^!qn0$5a&( zviZ{WTAE3vIn-3l2$W=_ofnQ>MxfyWZ`>FZuPUht%Be$hxLf z#>6YZ$mN0+Lhr+wC9ZOM&{A=M&3SDz5pVE4U9`oJo!#2B&Elxpecj2SZK?>y-6gk; zi$z%lyN{eM?nM%DQF761i9NUGoZDc>=cHGR%GhZ3Xx+|MU*&7hXcqfxz zEfURXlwH?}7qJ=7GcOd8p!{~gKKXE0J});t$Ne4uJrmo#@YJSWS8H#k!5lE)jniU^ zrFZ-YESG`TMwTuegeEef$9P( z%v|<(g*EabGM+PY@btjTt)q=ht=S$wZtv9eaB_B?z>(RWj5E9OhlIzckLTim|C&cY zZJrG?+XI2|n%aQ90QSL$tIg-H2=7>T?#){P$78*wi2FCXKi|XtwA;k%E=gEVTA57( z$e9FSK-~K0UjnN*O)Z1#drl67cTVK9R)Sz?iGiv|fJVUsG>QPAQ5av3*!SH-?a3Jg zrLz5HLT&;XrYXr%kGsqQwzDLnKw_w2~IoToSx#HN0-* z*KxCFSHsc*%`X%5x1VdqMfv=ou|So)9cez;29CTWbN6o(&`Qw~T~P>o55?GtVU&UhoK~UM&X<{_I;G zICvwiy+7$z>z!}GJg%d8yydD8=QqrL_nvLbX4UxstQBTY?6xp1?P?ITG|qXC0Zr8( znIwCs=$FHjdECU99U@NJWgh60yo}B+#%KN{&@CG-|CFTZYrbC_5&W&}&u4G%h2hy)hg zvY2DMQE`*kh_L(ZH;&;J#nlMrd2BAAG0EQTUK_q%y=}VQj=~oJq?py+z`CB4OuZTZ zu5OjXjO-hwr`UfQpTTlA3QleqMz>pYM0T*;I)K7rf$QS1tDR~h_W5C}wNj>&eEktgd18?>~RJDS5+5qM58;UE+SOz$Wp7y#lS6l)2N zpheR!D(VXVNHJgbp#I(#P?)};8t{#*B-_oE)`>!Mfi@1;ZCTLj#p)C~jx35Po;y8| znxt*v-Yo{;#ZC)GXMbp#Y!3WnZ#r7N7Zn`{KxV3fh{dz&AiSp$WqQrq$ft zviDIIE6L$LhtaCsS2Gz8XEVwk(L-PU5#^KDwU1s=n`{+H>s)>O2W{ZD_Ib)6G3X{@ zVfFd{3^uWE=t2!woiE6+HiFRSoDquXnmMNOh# zQt42Lw~`#9H3EPHw5UXqL08TIhLJb z%2AqpS#NXXYzPz4nGHd-l8M4b>MXg|xlo*rwd<^-)SEKB?|Iky3WtsNEcUKsNgwH4 zEVd<2*y4FbzWLaDTgmr5YfG2vTfMtk5yEXgUwl?Sp65W$bAinpvU3B|S5?UT?v&QX zu+#BcVNw2!a30OgQRFXYsd94+3f3bo>OS4}$71P84>NeU+^|8wF$e&c-RVUBjA8)b zalEqRi9iu#`5!xTqql?%gs-kO7)-?cL)dYV~#Glk9Xl_4gU-tFvxe8 z3Fsqv>01Peou14q=;D2Iop~|3Fkp_D#inbr9t?jlQr$>fgh-Q^ z(7&X}n@IJu%&PWJ59N1gw#=A7>AQ6}H-Xg0-+c6$Qi3#oHRo};(F6qF?(5s|n|U?( z?+69+Hythw`%$9IfyYTb4I!MX*<_PHkaT&IC2s#C7XVU3atEcl5(D+=4yT9@_2s7ucOKzJUwZLvrE9|vJL1n+jDPkgDL~)O>D9m| z)DF9-CW+qulzDawT#=P0^2Xo4|5e(*QxTbCmz|(P^RVL~t6zq3$zyK~m-5ZYH4P-8W?<06#LaV&auY zR6>kmXV!O)#Qg3YidnZ>J4dF9NSV_!u*#%(2|{gJ=u;_6WgH)?^CkLw&lqF{Au_eEb7@DpSx&0w;8ZsIBYftYcZ~p3a_G zY@m8=4aUOW5~8J3%q{!=6ldtAX1jc0c_ zFh};TBB~Acex>|cHw~u2xBk-&sO{5(Ns5)6@^n^3+&r3lOVrBl z$_`MIoZ8-KK~1XV-Eqf`>tSe^-Q9ZuG=DJZ-1Y3U;DPx09eM))XKi~hI5S)00#%ra z+#aMMgpNP!LGpl=aE3}?=ndVD*;EQOt!kn3vvCbL!2iaT^gg1(qCjo2@D2b$OevJwY>K5fJDfehgxgUJO6oc6}cl%*FW!Q@3Cg$XMgi3Z}jw% z?dCe2Uj~C2@bmNEfAC;?zjb97Xp)@Ta6tUUGk1JdB&30%sI|2f!7hf4y>YzB zCa+;*^94*VT$+>f+Nr@%r&L=uSN+wiS62l0fRi^+Tie^Ko=vOnB9?pm|7ySj zXaD=&SigAze*CMCKfuy|HGCK%zof<*ri2=CYU}Fut-AK?dSXPWfIeBXycCf=y%Q&;Dn7^OmXt0;WEHJ4{Aix zpX1)3$QYRInE|Nrp<=P81(o>cWJv1EhrvWRQ*W9%&3IC@bak1N`X-Q|UpH#vL&e0% z--F#S$be2ED8j}+a01V{JK^CA9LK8Tc>Y{zVq#(!k@?q6e5rf>$4#b2{C3L$hHTv8 zOj5jq6F3j_|4?D^f?VWf9EH!HQwAau+Ynd#*F4Ha!wq-4A5>%61kLWEyzTCdVBA<&zitnmMY&%gr%7yVD!|KAb12n6!|azVyt3<0pD4qBt_IXTfW2GZ>|S5)g2YZ2yjZSk6h-S6en`ndc7`hdXNed4G8I0a zJ~&3rsTP-a+Q+%Ozhl2n)vzzM+I(G%N=%RMBPmW^WIew+zDGd>-{hQiU)f#Q6z*ma z;|VP5=cohE9XBTi5*2=y$jBil)w%u8feb(JX?au5aSd-)v`5T6m51#nvcfnmA81f`F?r~ue%jQbKgs)GWdR*Z9HYCl)_&kt@ zFYf?8WbCSioZTCxcN<`vhZ%6SK@}!n5!&_s?n2RnxB5=?Q!xuMC5V$moojlz*$Ta) z`28askL_Q6l{P2q>Z#`)7?02SN=;X-I_##r+ds^zpX4HiTib zgq_IUXX&B5#*fyH-i2DzSP}}e$;U~i1*gwOsJ9kY#dFcuP~|&8uQXdlb?!*16`J3O zD-7o^n~UCb6mC)tAJ?fgu9T8f7;kIS)<3dRjIB_7vig}(Jk15Vnh2O^^V!xU``+Vf ziphpWxdt&Qbi^06{tqE+RqD%?!y(3@9^+Mqn_C#`6 zgDAyGx_-p*+uOy`G1_XBggEr8%NOSzg^$l+AhsBJ{Jzt#OnAJCVWTT4aoRNnT8tAK zdNcCE_yWdGv^l@jSP%VbEhE^p^)d=^f-}wf!sH-ky6ll5(w3J2QC*?#Rbxv=V&~mK z3Wm*+oApt<23H5{=axTg#9|W49?C~u7?lTDhaN`jD3GQRRj6pMapXRpstn@x>03NG zKidmWxZ1w05WR2GunzXxo`WCm5c|?QL|q9nBmA(v!|7F{uY98`d>9nR?s&X6B@`E& zS4^p|?3vtBjoLhBB!*bHq{2Gx4{Bc<V6ZCl^nT@fitXfr6)9NQZME!;z8TgO)FC$$BqAp_7i*!_pXcX(jj!zdP2CRl zxrKX<&DyCxa~@(!P^icWU9F&`^D4y+*Xn;-%chbZZ-_5rw?*dfnW7$~&J<)=ktp&$ zam&InkLV768mXgX*xKCK>H1GCg5#!p;D+& zrTda0 zMfdvo@lQ2nRlCzq1vaxZ8DUL2iaa~`ad*3>`JAJqZmT+Jn;bL_20kLO85G?!DXj&k zITdV4rx;xlCy0xixb(ldgSGMQj*QIKkr=KB#CcbcQ${uz;yvOo9#@5W(o;ju=`pN%~TJWC_J zm{P(Ir@h#360q+$uf+4LYGNc+{a}&~T!eLB;Alu0|sEblQ`CMBx0ZnVsDamD=xKBwjbMUcNb~84B z+2hz7GVcCz-2`;1UQFba{#0=>6z+fSs!T1YK+H**e~(yT`;1v)yuKrj-&{|QFvO(T zE;Q39oqcRt50qibGUbIF)okeMjRBV?IPa?cI8b1v(_wwW%O^N7`GS7ZXUv_f{46Nb zJz{bW8*@CEXqLe<6Q9nls2`UzaZ#z+lPJd4P}qkMTFxtuUMX?!U5sr{6NqThA$%#M zQGksv0aG}HoLBn5jaI((kz@zQ7x$)G>NaMxqLgdqlUceGso6vrPgKbZeI%$VVjJ9! zQ5EJTTh^L~%C+$ch=+LVcyb%#N(6SOET zYoo{Jgg0vh#by>2-kW?R)((5NRx)Jh!%1SXAAariq$&GPrTs_m@_q9i~e1%`(Z9>35q zmBxv{4I9hBq2c6$&ENLXtj@|&h?xMT!gIZZZ#>#ZRvBVbiNeQ&#O%j2mc)su@&R%%x$<|dJo!kCd020nU%jMn2%QqDC9eRk294O8a-kX z^9=Lg=Do&oxoA)7F;@M@f^6Du&Xnj~EsTu)RTeYhPqqPfNuSVM{=lC1pl0_{qExHh zBGICR=rWC-NJ5(f>F_)XN1ruT%h?-&=v2_|j&G{Z4h-!pBF8{d{j5`W5^LAngz5Bg zqSs2l$5wQc74~}ONS2zKRLqifwb}>?3k^w?kqXdfvJ3Mbr>I$LiF?0N@8Xa;O$+dT?iq4ZqxFq2hI-TNcw!X z%dIacr##frTPXU6y z^nmM8pO*0>x_) ze)Lr!t#Tro%zCqR`;+$mn(t}?|7;dASzxbB{m^Tvt))mQ*}I&GO~7_$ul8iH-YlVs z;mPwb`7e&i6!RWxrPs|Ww{b@6PIbeNPuoXEne9)O3VN-oMUQPix?eWmHgFx1#uv^k zv{bA;+5GSowzv3vUT`GmQ-f>#whgKM0XOR|!KKSghka?yyxuUPW3 z5}eyjW0)lxDiVKb6m@9GX5utNu0>MHV3w2k^5DVcLF>q9J>+~Lx_1y6SWV7F5pEc| zpr1>26y$L^ukX9VOzdIXgPl<45!m;w$Vc^M8EzZ#bIn6MG0ZVlK7?nY#oqZN0e(ga zE*Y2FB)KhSo6BZ%HUYL4>a|&o%WF|Nmrb_7=I}hCbUZOhm$k{EaqViyM6CXO`fJa! ziR*yhHqaxtM(w(zFAkbcW9H$!2{Zu!4Xa+W6B25#fq(&2#Ee!@l_PY03SXl4C#G4{@M#ylrCaS7YRNQvSVg?$K^=+uIn&?hDBo5(5G~tkr z>shzK!Bo4|?axjLBnhl8lg0)-&UFs18!cp$Y(oh}XW^ta2RqrWJ7SZb*mPJ~-K9lk zqXmvlhL_F@S0OnW%CnxO%{euZh{1%1fyG3j1WtY(DozV4T0Z$~>0<69WQQouzFEhq zh5Yrvgo;1nlm)ML-iZZl&lEEc_RPM9$AxDU(wRd298*sg06GlDjMf8mm>HRwn(gJb zuUkEt@NMX=C=0!oi{_wW9vk%VU28GNZz2mUvqlINeSG4zS=>#!f%>WGi#cnkYbgnO z2iV!Rr_O|LS9Xm3Dlc!XBXEJW#l}}(^Fd@#%J`PY_JCj=+;^LUtp3IYZjoqJ_OcX% zL#0f3UP%~`mz``2+RZ-@cAZSIjHJlPE0!3IKZq&cu^6u^PDwS&EGN7^EHhe^;zPlK zZ&Pb9G-zrp6SHgPq43yt$j960uO7BPsb;bC4e4U$yKtm7^jMRLM^raTBs3qp59-Q(_GCm?o+og>w z=_Hid*lln#yuua^`oRT>1JLu@cnu@VNQQgZXL<%`n=*~@%l(8akWCd-Y2@*x&4 z`)aGs48e6D<7xzEKY53jRt+Ym7pq+B^rW1l54BNSGNMq*9+%6b=c3ENHfR;^@Z`Lj z)pT{>K@F#nb3=V#m3E=5wVk|OnNqBoll+wR?4;e8NkeShP@prM`_5&^_BrbT{_Z{7 zUZsJl(Q0kVpJHgDY9dsi2W)(n+HFV=g{H5x3{p>7)rNdFIHIZUCsBFw z3wZ^t^G2K1E6kgYM*fWJpT(&HY_RO3r9k$*_rasIgTtqK@?nz~teE&~6Uud5tc?Ri z-=%b?oZ{@9(@{N_)&*E&-t%b16YWS#GnK6;0&8IqR!JW%wVzasU0aCBzFJHqKtCG( zHQU7L?3~!0dVX7Q#Y5TLTLrP^B**EQ8SpjLLpL>6ilH-~ZMB2Cm)-M?d5)p~kiocB z)Mh-e3;v2;mQ9%Hb(s9bMN`m8ap}RwX5vEk-WO_Papsn_Z|V)Le)0Xv-iM(=fth*{ z+AQ`Vrq*i_fFIMnXh{H!6?ZEVSp6qrj&aTV^(^Pabe@z$Ok;}W0zruI><+$FWGG+$ znf;Wquovag3+2qT8M8L20*So*bt(pVw}6b}-RKxww2iGnt*2C8Y-^jE{DL#<>)Mb! z%dM-rMED03&t|$@^y@MnP)zA+C23L7IBODvZn>d3=cLU0s@G{(4ljBv-fyQl7ZSWL zeXID9Z%u$PU63%>_M35S%}%PekJ1^~VPbZ#Vr*23-`7FC#0-!0WJ=spqDdA^npE7& z*t4qh>vXLaFFDUjkt0n>Hl^R{#92gzHusz&Z>)Uq>T!0XVvt9#I#ZI_5a?FvVX{K6 zo!0)ArokzYHs+G?Rp*RVo|~Ru%&|1_dT&g`zw^Cu9G}K=ca-Mqw%S<}Z-k|Yd!6iR zx9(=9>L5+;s%xLg+T`u)=6ZX)!Nz6E9XKR07GapHPdGHg$uLZcFkerlIZ0J-)$f{yRIICQ=1dbh{d4&My4 z={X4LRK0F%>3OGR8h5Nb7M~0VdX!57Ra~xV84;HwYAIhTe%M*F13#l6O`U03xFhXt zbkZgShs_m#bWu!Y#yRzw;Tao9XG7;xu3l7fOid`?PZ}o|dpFbr--7SY8?MrHFM`~0+Ul%XOa>C~8GfeHKkU5-T zC6}K*eY&9j6b-nL#)yDN>no0qD`p%#eeMGH5{sRs2oR4P*OLYEmE|YbbWMkIaK0+2 zP!z{k_bAW~Cm+ppZx;0FY<8N8J%{H)IeH zn#$6=#bJ&14Y|R3e`n{0we5`73p%X<)fxk-S;Rw0%;WN@vC^;ve>8QkLOrWA?#bx2 zalVS`%7#eMz}XK4Xyw_MBEKt@`!CCI?FgAzMuVPBUJF`^GOz?h+b2{r!AJh1$>ou6 zg6gmF*fQ^#b11~Jli0W8i1(#nGfb!Tp~!M#scP^j5+*ek!gp;wlsAkWOY$63u77^% zMaUMoF77ycnur+uIyr~C&N49B>oWz-7s*|!0O!)GZUhc~_FbrYE5F{UG4YjKXx*Qa zX!(PHnXN&>_g6}*WP`W*hMUuD+f5=EbS&BLm#`34c{#mz~&o5o|(^XPpcMrm^GOw6Z%`s(sV z_d#W507}p_)?_yai{p1L)38i?=>btF*h`l$s`W{EA7d5`UNB3&x)ap?yd`Tcq`BoI z*CEPt+k$hg*mEU$(cEg|;8Ah!f_B!>;ZfbrJfX)x#sfa*2$?OiQd`33qo+Ayy+d5% zg@MM|JA_Pj2dZNv>$|u>kVG;G@B-9UcJMD!ZvqO(&s(?w4#vOtQv+%-^-hHkpYN3kv8+%87M{m8S^h zIMz(iXvwF|D!P@}ev;+G25rj6PwM>Y{}tynnd+1$u(T5#w|R{kfr zBm(h^%!i5<3P2%GvE-#+Y5DniFp(X75^(E>o2{-=XKFu{)u(GY;F+izmuaBS^ zpx{+m*y+_YvCJtjO)5JMw#YZ7EoG)X)APX{Hwmr;*toOq0Fn9p4G}3QVwE z$Kli%^Xjk1qsVg-1XxX=&Q|s^Z(tRw6h+_Ql@PJ@1raK4jatq)E!#24Rdz0BQ|>9f zlas;dgqA6eBx|((3fo+s?)Bqxebig(+4x+q=-K!H>z&V(sYD6lBL(@w_I0DN*~i(k zpcXM*_6E6aMCcd*`pU}Cu9m!EvNREWQ&4#bDQqwLh|^yVZurD+HELtYH8gVQrPNC$ z@kXRZyL5_Xj9;QjaT8}S{8i$saY25E@}(uET5fZ+fNc}MNkQ(JuU`i@wEti| zo-32ys&4d-^Jr7xJR2q>_p#e~7UHT=kJH0FRR~Mc`JvT4b@EzWT!urGta#5XM83`I})CvXIz3%g?P-A zSjkpWHf89Y?QLURt(?2Yk2m|Wu(3XH#WdhDxuRrSdvC&@!|YXAqQgO-BQ9IlXbnmy z0zoo9Ai8+}d7=ENT{Zb~O^x^6BBgVM%AW=D`=c7OdIVge5`$ORFhM5bUtCgCoe4ftkQD6wg7RL$|o_4 zyM)c8N+9U8*BX2gOc*UOzJ;PBgs-CFf=3_UQyo@b-gwUd;5Cr=Hr?dM;X3K>LNk5n z45N4-)f5iuOy+ehn>_FG{7EFZD=<*rKS|kk8ELX(9S3qKTRInl^AEaA74){+wc3CFL<-CYfDV7ZeW|d zHi(Wykj*&UI@;-omx`CpL?BLGH`3u{1*?f}uFdNkMro9b;jMnZQDMx%uuP ztu=ZMh<6cJE5-MXMQq-NMydP23@Xb(ra(Z+&1&IkEws}#SnQ4uM!)sf>|-8{DbogR z_8QZXG3Gr0xT<&G7jWeUb7aiOM%jLuECcr^Zl4%&1e$?5{1W-p(etS+aSUVADZW+D z<=7YeWx{*&zhPG^K;1${&n5}6J~)Lpi@k`c1+T{4#O$-k0C&~z&zu`mdMy9H^Dc@4 zrzS7AN-N!a7?V!=FEa}yuH5UKmp4m}5DDdeCW&j; z0f}9wI}fT7$65Af3L!PW+o79|amdl-hB{Z~c3pyssiEsC?I_rhiD0Mw)u=GVUZ>!X z>c!ia7a8Vc0zdP&55W0{e6(0uhJLG&Ui|)gWg6X##_mmyqHW_9^D+iPXv{9MY5#lR zG0N{+$6K$cEHT7%P}ewSf0tzWHim#ba40}tLwQ&5{9GF9bcOf>zoqPQ8%iaHg6Ldx z=*Av*=ufeq$0lapvH?e}EUoM!rj`#;sEzh?QXsX)c?F1s3z9KgqgzcJY`IXCN=lG3Padkn}lKx9z5 z2EE$U;SZ%(+$U&3sV3BLYU@Gwb{J!VRw>&JaFf) zQn(G%R2h_|5 zMEMj4`X=|rU8=ho%S`J1NDCtqYwWrCFTDpRRKsM`o~rO@8!q*|AYCH ztbb+>?Mv$Gr?g3Jpva|$xI;3{@x;2xeWSRK@s`(xyQ=&aFAu5AH=cA2@kW5xfi5os zGT&Xi>^Sv_;{F3G3%iXNJIn9?C`akP^Pr8S_;NQR`ocx;Urd)I<2|oAYHK-)amp0f z{4_PSdQsas1<2#y1?-woY%@y$s;XAY9;@Y>3aBrB8Q7vzHj|xmzxevMg(c^&;bd&b zhMO}Qg}{S(^_=VA)}5=@jFfdd>(RCo>rO)$k%R23x)s&);gS929_69!>O#SyyuqW2 z|GLd@+#3i#hU|EhAhcC{F@9UAwjTc4vXbrFZawDs;t-e5dE-?}Ba^ANBjn~J8_gsR z^bDK`j4(Q{wU}~BnAElpymHd^E1$aaE-7*%IKJ=kV$yAm|J;CEs+lJc`X+iIt)jGj z$etZh_~~KOXx#JIPnOqCH4eDuo zEgvXZgUn2BMcJ&AsYH@XD6Hko!?#!2_|5xSlxZZSt@b=0+;o;K?1vrInFQ=o{*?ph z&MW&_&%l<=YpZUizkxKye>>A|~TJ$L>o(Ug6e0 zA_-EPWsoZ?m=K?^)gf^yZyM#RhR%}_Xs#19j4m&mj4&M-Vaa9iDz0yV7*HN6YRz-W z%InG$pXFpTA7xI|?B0W|e`F1dVbHd}mYeCFIm}m3O(P){RM51tvWXwqmb!m4g73G> z5}#^kR-nCZvPC7Mmtx1!7i8OTrz2ov4$qz!9G_amczN$3mODF-x*GuzP(Zq+yK@K;X{4o* z28Zq;{yq4-&;Ne^|DDT=Yi67?XUE$2y4StdK4;?7^wsqJaJf=&lIo5uyVbl%5;CQV(gUK+WcGejkvE*iL<5mst{>rLwU)Bhehse|R}0xCqWP z_|BQw&bX^e-X9qG%<<|DHqDo+i z@0?ah{Bxh*DQFCgmrk(G+}I#0p5^4W_7{P#le>8`ae9GpM3 z@)$|EbAG+B_71hwj#T$(uI}8w#q41fO@-)`xa%lHUF?Q9IzhsmK?Z?~rhK7Fd-wTU zCMNdwF_~nhaq7`Di8`hp$Hgy-=bJ!2gZPQ99$JF?^Jt&m;%2?##m;zZmF&4#rxjoL-~ik-VJwK z8G>;YCoa)?uq}&T#;#0QSf~BW;*kk{u6JiEfdp|PassGaGdK0Bvp#j+og8c#zjxB& z`LvV9>*P3=;P}$W)k@1f{C*H6bKr_g!_bcA`m-k6H=na4I6!}O@6Ga4&UA$j6hCfc zI?BPPb(DYnf!Se)@!4Db@Own#`%Z`}Kr#X!+%4#kyRG;g7T|s9Y5EhbEM#i5=8SLi z;f`YkZ{ly~bxv-c?_+Pd!Pkdp+`SuylJj0}jd;&_x?0)8xIQ#`gk{;NSll!{(U6-U z_P#!slQU_27p;@q#A`JfqZ)U|ZnW{#wWzUxaDeN@>$32%>yOu&2KaTe>9AwvU0?wRGoQ(|5J(T*EV6+ z=UvOIki^1WCTHiJ2-PaB-`zYowC==}d{-mB_ zOLA0-HGjHb>~UC{_0;9$>YiZJ>dB94&+4Zo=YyJf3hCJxdWqwRQc?)S$SO&0Ku4!6 z-(TD(3Q4Sdl9>^;V}Ndbe`R^*YsSo{ zi+Fb=BrYx#35;DxQiMp`OE+g?`hn%0vfIWkCVovEV^>o9w>}foM&zNY^#=n?>~nsM3I}=P{)BjKx8!l6~X^B zxsk1^Vd=ht2G4)>NBTd_{47wPDO=JwJ>-x0eZ=8F^iK`*@XKQu2O65(_=8(onGsr0 zw*CxAAh%9L-q=Tg~LfaOj8WYPdw1xl#5BMW}#-w|_*pf^l}>gROtzMRxpTCXA9 zRTB?0*p#~~*Iwz%r&Eg(%>LHhrm~gx2zv?;~?XcYIj+k7(-J+6L%gautrPR9Nrj_sV4lsHk$-H7^i`Kir9mhITFpUJeh$D52g34_s-) z`**5@r08Fo*_;^X|Fv&{lmMqZkQv7VGN((_pu$k_N~7 zH5VT5Js*K@Ar3z=shE4ilJ!i-!Z%%wcX@9wh-Q;?@*@k!5U(Q7df$Ge#UqE%NzzQ$ z*||kRBLCmO2bBgCsMB;XZ6+TY{aIPq19XQVd((iN^M8=_zZBicpFf0%hzPKrMV*Iu z!Ip6vs$qGfm5AXL^UE(QJ4Mu>XCyPGV?F>2P*^-iBQs7hZc>gnFeh9-F1dJXp(j`5 zoT2;*7#TSy!GA{s|6Cx_

KY&y|2Q~}`Ge+zFJ7MR$-F93@EUA32B)pC*x>ASf4 zi%jXT7095`uJ^@6=Lad?y+xl5ktDKzVTds7iyYnt*8k?MU**z*c4w^PSU*rL%DAnL z7bEx|(33G!VL8nh@yLzmTUu!sRPIz#2r<}~Yz!%XSr6i!|KHdZK>T-5vD3ds_jes0 zvHlOv^DmG!IB@(IhIPG7NBjohMN1}sA<9kieCJrV!W>jRHH?eeRX^Ds1i{4z+w%{$ zq^dXVJ-zx4wRrPdMngQnjqTz77xXkX?3L4L{Ed3tzSb(+H|u8XCF*!we4F20iHQm$iIw= z#=Xn@@uSv;7QVn1)Bgrda_loXxdf#XE?TceLnpV_Kd$8G5BlaI3>p<|hPdtT##%>5 zfVxZgExfleUcHVpW}EnGVD`3wJq`-Xc=|5?6>e;A>NDF~_MFY+8s&%%1I5(BV~aLn z%04GxZc2z1lnwm{$Z;Tr1mNJN`}F|{tJh|n)O9d{$C^5sIH_26(3J>=jvofNpde0_{q_ehrx|RFRSjf=d4VC5Ye7qv8tL zlIASK$3K1WZ+So|{vS1!Z?Xe*dZw|T9j@L;+fQW%^g32b>yqkjs=KF}a?2L}15a`_ zv4i!i2P4H+{(ESz=(v)hw|hJ?lDU%yRy2{mE$-fqCB)C7BboHC>F2N}+jO5=9xmOJ z10`KOlO_=_AJfPFuiUuae(o`h?VUm_6$VI4rz>W%0N?oUD&zX}ck9?3*xm%L;z#yP zaqMGW$*2NHnyO3gohPg+d=F){0>oqKRT5nd3s%cGCp)xqqT73tpmaL)3p;=Q2MaE9 z+7Q1@L;@q&So?3!?_?1%c1eZUxiVZrLCWnIAO9j{{FQZ>ut9|Hr}=}@uWobaeJ?&d zu9*c%N63+j2%G9mbd!$8cn$S5%ePr2wGlNhqQ8VrSXVjP6bnx^_@9vE2=~Ddz-A$7 zz*KPOclHQ|T1$Ut8x?>ZVB)8;ny2>&aeb-79I1fe@mkeMn zgj~}I_EM&XKG#UwMS%|e;r@jpHx)yT%E%s(pD%_+>RRAAS2Iy)JFYpSTU2sgTw-K9 z-dMxxj3!NlhGzPS`qyN6urSuIgkM7!@a0qiIXbb}#Avx@(aA39FRLl1%}#@fl51(} zY~Px-=G)R@los1@RDW|BGTa_n@iFo2OAh|?)17pGKSBI`CKC}nhEK)XF~+$@Z@)h8 zQFjWPL*dhR!n!!!B}zD+8mWw-zIq;u$D2%O)-+E)B%Z(HA+KM+MvRM+U%FI>^8Rn5 zz+YByY37;`b@UpM7>^HWm6i(fzNIIuQw41ucjQHM=ioNo zTdq2+xIgp(r|e$vAGK#3Qe&D0GX!n?s=uZ?&s3f?Be|ugMDAT4mk58>%p?7b)P$qw zNiHSz84{TU383W+s)g3qgi4e!-BT?xVg5UCRv1%h5n!ZVbpf+^E59A+{3kj};roYT z*O}ael!Z!;aisdt#S4r_&(jV-onff4Kh9XG9j|4!;iKobH!#MD?b)q5(r?- zC*p0O3lZoOB~(<`4(BNtKUlzi@o{KYbyQ>SoY*F?k@5UJWpWEHBY8+;9%1e1Te@7g z-~5fWFLhc_GAa6Ej#fUD=djeBIOJ3BLl4j%B(gayILcWQFqc-3u%n9VVk&e$)mG}W zge@s^$SM?+d8EXysy1sbQTD1UL3@ifV4u+j zED?!2g(c5S8RWS_iY0zk5={I5&bHI?bLi6u5E_>-Dv^3KI9<=FRfZLI&cPZn^JKt5 zSKIEuX&i=;X(LKlS5eK&SPU7!+*Y|HH_>Uve>{9b?GJ`cnRhp0G=CbSBm=iM-kZy- zeS!wNh8w(u9lGJjNX)`EE`pNo^Tg5rGf$%bHBaCPg;pE9*x4TL6{geb#r>S((vBjU zJ_qcE*X*?H5cRV5e(Br|#~!K3R4yWm;P}P9T;|^mcgbe$NaI3XGRkLh%qPQqau{fj zCQ7DBOIDa`VhH>{WIPC<;qhMq%5!X-Uq+QFIbf?2JbJFlLXRN!Tz` zO%y+s;Lk$^r<2W-eoS7d6~oW>F?DIWFz_RSvty_W1ZC@t{7Tk4eer$`!uJ|+TJ!^$jQOsD0 z{Ky0{^K>hNO0MPW>wVfg;Vt*nC2@UaFR6zb!ZYXch*YPwf)h7f@05!Erxu_Yedc@X z6b(fsu~S&@EDOc=!fxD#+8C#YHT3ip9?pWe1;O7QkCx7_=l7Jhbl^i+A1?Eu-l9yt z?I)&!nou!dQnplOEW{D86#`4a%5LF+_3u0m&s7N`ThECiMb@J3;Ia@B^bzghwcGREzTSSw=O0eHSV{RgLsaB&Pr}o~@{XiNSZS;cAh=@!H4?+@4>^^ff;km#25Q@2yxWI_IJmshhc> z)Qo2(9bW#>PuZx+y?bY_?6vJFg?V`n{m<9gCF&(PwV6APU$x*Sa!PhxKfh@OwtHNt zylrAel`Ij)zBP}msgw;TY49@Z`yv3z_nu8dMxt&wjjjWhdm`Y`H}E{qNvg|>cD>)i zR18^PDDbM|ZLCYqtvCvJs6w8sybc7n19r_|W-s7@ZhfZbSUb6oJV%MDEU5HSLrOzvb~gt992D;v2a;0Irz__H2bH27rnTBqAUR(=g!i<`ZG!g*$()*oA6 zh7Ofn#@3-iIN$Tnx8c|kW9Az4L=8Ft&O6VYC3`(Qi+0HITq>n^KBo1t(T_#V+ubhy zjc)KHkO>-OgoAhdcOZh!O@%f+gZt$ ztai8!WO`b2?C=y3>(0Rh6d{jeq>A!4VfihF%~8}&3orbxp2U2C^MF}baz^QcZFr$V zWWpZ4l{5awb0g+~LH_CO;0Ng}<=a^gIdqa1TS;2IANQy@k=$NJHQv(HfA1>$P!)@< zv`<)3qv=yru3bJXU$dlyetL`Kjp2`C=FZ@K^B?Slo9BkEO*4%ihcpVT--@TEF5qU8 z&AvXg8gGoK)Mv-iPLiwwi zd-*vo>Wi#0Uc_=sMPEJaa zF7ak!>-A2NUVEMZeU%5?BmjeW%PyTpKyfB;!3mhh9O?oN>QE4(ag!GB02H- z^dpV^ZPwf$DL}E}istm^FZzo65ea!lm`lq3^S#;hx3+To=auY-FvYgL(2N}++>K{P zzk4TW@F8?(`x5Fmzcqd=*o&BL+vKM%I09HDR2@|)BD}C-M&2DHrAAj$ zW*AqZohV+h?A2z-xi}U^P&=L)noM{%()ZcbMtPzNiJwdTpOvAay41Y;W4X@*=r1x~ zuPbmCJn`iXc(O7ssQh@J?MDP}I2chH(xjT{W~kxJC~7{gD?>=x$rr9^HCY(H|F}74 z9+8P8fY%MudGoTOg2?h$X``?33O1!lArQ0GlTSNU$5E5~Aza7o>HOU^F)~W zJrMzlCHp8mbXws@)-E2O2A}tSZOLyuWd6}tJJDpoSrXD{X4GQ#yucHivgOi^i=&m7 zF=y?nvuT*t11E0s2JPVY%YB-Asjuu@VaHn1%zeK|5Tsva=G%!rfa*IOhDuC#L&m~3 z>OUVh`Tvm9<17ehb|Wd=@xtm|Yk_Y9UBpnm1JWP9ch~7nGEkoQ zBec1^r6C`oDgR9In5mNztKd8Ub4Z6{dU+g+!5z^3!kKHA4s6ce5Me|&4howumh(>{ zjJq5~+$Qkvpg@ie?rmr=VCHF^({s)tOX((4Q}3wO{vl}(e3lux*MS$4G4^#?v$WuR z&Uevwd3)NKmZd(6`hmZpF4{~R&8yVbGt}M4XxyIVJKOxK?~}l@40cCvipY2|o4G0D z>Y5?DwDZy%eWbtLOO)kaauCz=z?3apX2vfiQ0J4-AkjjK+ax`RqvWzAjwwUhWSIJWjg)!xrD$TaAL?5U3!No%$b z)A;W&l+8vdQ$_8sS;^PrT+mN^A9y`S___H_HfEiaouH1G{I4$ryPrN0T^zw>KJLE( zOILPn+ERV59i08to6T#;*Q;Zx zCxY+&X$tz_%ZnInR=pGPIS+^*|HlxE8wlw1N4TzajME2Aby58?n_n0$>dmyz7wR5T zt7Rr=GIF`&v)>m%yMdaw-E1W&yHZK*`7UI>`mHvMKbh84bJaDhP_*c2!v6Wg`5LSh z>KmTK!qRyt2Z}Lj7W)R87GC0+ zyNP7dwtE%+r)%_FW;6is&Fl39^~2j{sxobeCtZi}g2?QIHa*xX8VqVgx&J!|d*3!O zjY608E!VE>%2=JFG^moDz{8SsMLL;I=ONcSNv{n(LzMMzF)s~Ui7=DJv%inv$BDfa z$bbnVgSmNwwyHd_>)+S}nNXw{#_o+i+2W+n^-yfyDM#Mr-j$R;rdtkVfg~TF5#Uo( z+B`8ui=9DYWMvexz*||Ji9YHpGFG3|OxlN@Fvms~A%*gfH#!(ASCvOm|F#`3Mh2VHDGY$J6raDNnOB`Q&+)hXfykr-_aZGJ1Ln$|!)Z!sTg z>!6f#eI;^seF^oT4Whgz9GgQMb|GsUd5kx2A?mk2xPQ!!T_8vhvEM90694C2PRqIM z)nrCFl`_u;npw1#O=xW$W@ect!|^JHF>IpZ`Zc3J=8Rt{n7SY$RXXM->&cocNMxOb zxaZ0<@+5S<8@rmJIakyP!&JhwmV34IEyl3KhlWv!92u)*?bEp^)nerBr)v^|`sh8& zQj2o}vKPnB!$n@0-Zj`(g zc~+Wbyo`gdHyG<@PSHn2)<3m@w_TzVyd!42%Ia)Sgb)|3N8ZTd3szHutQBCh$P0(y z5lx{3jNGeKh9Of!d?8x-uxK4Wd zZ1k-vAL4g7kF$zR(wEKqe-@oXcXy_FKn-r_=c|}qt5kMGe|GI!p86B#WOly%-HVdB z6EsRX-#VJDc?R-p!atI5F#rTJ6fIWiMh28zBPVy;r<=H&l@vp~SH$W|g=ziyT0;O! z{+x!p&TPTR@BBD`w*UF`HbMJzky`e5?F1rB#xtn9!<-O$>w_`y^I@D)=gQD=_N&P_`<>tq1>4)wL` zJkl!d_?ne&2uh2dx4WNXl3#O;rM`lyOzaBm92{eF(s7i(>kR_phQAgx=rHMU5sYfw zr)y+jrJH<{?dPAWtn1Ee63f%#QNZBS3BUka$;v$nA}fCW<}0SU&J#7T7TlS6z-rL0 zT8&j)KeZH%3b;bm>DU0lTwvW?{s({l2^>i zX7GWLOG)_Xw^B~OH}T*{k@OpZr`}k_!j+jj?1annjYiEkaT<)Pi8CgfDQxjoH<-YQ zEZlgbTPNUS4(MYkm}6gZz6@P)WH`=+)8{geC!;(ReH+O+jXK0x8st)c$0L{ER_+Nq zedzv+ck05~pP2>adeVYnGqXrZXrT$4(90D+MJpWxK0NPdmvIoXzDVVIHCgAm!=0%M{F=?lQkIAO z8{N|zIJfcC6L6aO2o5Ss%JGQBEnv!6reSS~F6DP7QrvDfaJS3iox*gk@GtJl#?f75PJ)SesD;em4SYO;1px>I2$Q!I2PcX z!;}eXNBQxt)uT>Kj{NJcqVm*vlAe!B$dI#ZUc7^mhvg^z{@!ow-FDL5`Wx}=3Ae5p z!Phl8m9wZ>{jEoW0UHWj?2g#dc=Fzd%i!=#&|yJ9E<2;nw ze6QE(6ctPXp_}zxuOV+->oP@I&nx=JEa0{Ks*7d#E%SZqAd4@$6;-pdYF)`W4*M55pK<*0R?K?1%IvEJj4OgA({wBJ6-Ak)fPo3xjZ zckC1}sdxJ`&v+~$<6&7d^*tIFW{d^kDYQn3%t<7>ox7z&uS_Cpw~UtII34P@Qi?}o z1(zOY)`OPDG~78P^STkthewH>iw8**(w_4{=)8NEF%i&Mi=xcC)(LR1x<$I20EW_i z9!Ssm3z>&_rSfEp>7+$6x13;`pvNq<@g*u6a@jx-Sk>p7GlXe1*u5SqsftBv38PZ6jR>>jH2Wa4H=%tQWS`r4s|g4-x6sVJ zIlYk0BwO<(&W~;ol!vVguamNvOtI+8*`1MjFVv;;tS4G;)^7W4xXOpZ?x9nIX)*l( zxRhe*z~T;g_h|;;^2-734Ega!n6QB|lb#ok^^mNZaEw#<$YZSb=?6P!DG;ssMvQ`OWM5hz7co^QM5l({mOs zcyIxt2>uDVM)A!M%bW@>EPO@sqj-ep(}=otf>M6GgA4!RkFbGgYZHNvDlJLFI1A0k zk@MhIU-|d$=Q7g^xt8c?R9?jRG6GEQ7!k&ikqtSnrrHQ5Rc7e^dRo919XmIS)2DK{ zVyZn7yV!cgQl9!~ka8?{l+dso$^XU+rIRuUK2hPY@Z>IL&mMH2lnG1ry!#5% zw+a1Z*3d!!i?yBS-qkK)%8BqFi<6Mg@u=35QCY-u@!rw;_)~AC>C_D?s6B!wg)rps z1y`J#GD+hl=X@Y(p`{@=_G(My+;_HWuasTM=q_RRP(}6qK(qn6W;0k&qh#^sTskW} z^=s$j3*5u0TGOv$z-93e+)#qxAhKMQGXp(rFTSoA{lp1Fe9oEt;50e2cmdCO-GqC# zg!J?B8!MlcsgEg5M zLcTdiKj;Ir+NvEAY3X|huDGixdvPwMeskcAdJTUVc~A5K zMUTV6GyEm{NwERWAR^5SJ5fvpMf8-{p$um|3~xev`y0YkHFy2ISK6jwb82C2P>T*9 zP0{*ND8!1XU;nUg()b5^j%nT0*N5=v>vocI)30CJ#}Pvy74SRiQ1LkkkGhXs))gj?V_Of|)R&6ve(6@58L0>iDvlzS@55Es{2#GH$ip%s!3d|!1a(QE%ETWrX?w1H zi3_X_cq02KRF;+!9r@Ebosbh+%5q2teEk#<7{A@~kBlWHQEiXcUTCWdWxX`j_E&+0 zHXd?yER>+w@T$nkPDn`?_c#%{9a__|B9JvwjJ&teAUeS|aC_^}7&cF|)L>2DKrMUT zM?HLiW1@b()clUT8U+Mfg$SaulB<_d{0r-*FFgX%oAuHh6UQ%~S5*^5L6;byODog| z)ftX7HMEJL@LUCrRmh^`&4)az;S zed+i4w|}JzS&k_mu931jiK==Vm+@vaL*W!_lABMNVmZMk7R;Cp#Qm8zy`d_rfh9&- ztfGIoP@Y2;$(^6+^XZ4}%{l|38KS8i(q?TGPB?J=O4fyviVoFe??3pxxbc z2~O6~8Z8#`n28)eOsrt&d3%az_7mb#L3LzzjkJ8soP1gDeZ%@vT0G}dla1h22P03B z2L&`g6zV#_oFKxI^-?T{C;<`=XGrfBH5hybQgbT5Lh(6w%q28Rle2(w)4XKb5rrPR zH=fu_DO|W9JRr`D@S6%f9k26jmv9SYF0rV^^+R!KFOI62P*#a6_7Y+%d2Z#`qd_KeNgM6Ve zY?>&VDTvI*@8J^+V4-W*+s*IzXD1CHos@UhNg+Fwk0ji#J@~uvq!*!15S_;;5+S0d zA#$qM?>IV~CXh|gdcS&eA=3lgSR{W;D5EXDc$`?CQ*Y9v7;31Fm1BZ_R2sK~` zNXodSW?ofYrEy};WYBOGdPT37g|1Y0#(ts?&{ZR2JnDD~2v)m`hf)soilg418V4A; z$_GoB-H)O}fXGBqF@c0>4OAWt5?7@en^GrfaHM~G~Byfi41k&{1$*Y zOYSaJHA{l!O$xV)TM%O5=cL95$_>8^Zcy&)d&oBz#O<33glSMD`5fmw!%Y%+PA5Ll zvGgYm=n8Gwk|?692zCi3gG?A6Q2f%$(kFS&aDK|yi5Zj$V+$%#BGYa%OC+WFco$E; zp_B#_lrOi#dTWLf?|27W9LBd^a54X5Os2uk*M_~0mB@q~<*irl|S+uc-a5})7${?ln{G9$M z|GcQn!3HG$sN=1+km7#h7?!`aCwArzn;$ceoO;XW9FG$7R1kvJHnn-BkKpumxzAC8 zH_Gapv-Mg+=#UP1*s=TyZ}yXs0KHL0|9<*zR65~PgGYs$6U;@!_z4noAh^Rp?{vI2 zCMPxg*;}8FoYuq{e0V?v_>*6=10$8ispMXSnFB5i<&o7zSMMw{_t zpbNNc<0#haFM!k;+y;dr`WpAjl}t<&BwTvX6kMJ?kbA_Md@Au#edKeBi`XR5wFh&t zfuKa$p2S==*@T^rL!>YXh>Q+-kRXU;h9&82oe?t6^n{vbhRrlW2L<#fbbXzKK3$Y> z)721LK~W~KTVHSdv-N)MyEz){;6;}v%4SYRgAP)XKid0&=uM>jH~K!JCG;#wlFtMN zjcoXRE0(Fgha8eo{(WQp(P?T5arS^>g`TE<)G9M+i7+lm4On#!4h?=o6QI`hB!@`| zm3EycTY=)F6c0d;#d{<~UE;bgK6QzP1B`wXw<*JrXOmU3@8_zHp$#nom*pta*RQI=@gYoC2l78S z4hb*aGIief@Yb`a8a0!}O$zIky-kL%J;wBIphAdyhWc2i4_^}{HFVfat|mRb2gnnU z!4?}Ygo4aV{lH3hqiH7Q`PlL>ey$GZfsSRgK11~~kXcak`hyKu$;4v=nZdvUT`SO~ zT*{`itHI%ABcX<<=TEN^NuXI6#<3Ji3fOR&lGbv2(4QCR8KatU15IfSLzX7*7sD{n z&Dxj)vNCmwGPS~n{Xh)jwgaUa_l2YWATz;q?>d>bg)T7wnke4;B4oGWBYguv1_7E7 zQPi2}(QmKqC@}P(cID}OQ#MQMo;!?F7p}qE(}!Ir&keNzpq9G~qP{}`KW>cL!PSfe zd}Qc$BhDhogp{BB2-z-F*@KFZXt=hb0%NWfG00dWlQ`hd+z|@ef*U^OAZaMQhh{cs zcV>7dRs4EIOy%WX|9qDGnEiZ!;c6&6tq&o60oJtSd#nJsG98WY62+_ZGZH1n}(24DhJ@>IEWB|0iJui2O|DtEDp29>cMR0>C-m{ffxhs5>AKD zju?~ZwHol>*=z4_I^cgW_>JAu}FvV^kOY*XJri^T9QPsMZLbF{tv&F3J2fXL1MQNSp0c17If?qxc;52trXA%84RgDW3H4V~+g zo-Yu0&*uCDc@w`Hz30F&b6 zOKKX!Vo+N0scNt1zEz;N30ppmlrPiidGf5R{l*K22TE9bZ7@*#8A!ZHFGCMws}K!R zzB=?p2*{Hf4G2NzriRl57LNumzh;Quz{^_uStrGRkXf_{rVjYfjy?1xFj+u2>wx!T z$ewRgm#5DWcobkdog1Yhv30!;!O6B|UwZq6v`pyK0R09EmFSZCIXFB4u4+SX7&GRC*>^J2cu2&uetrz>(7$tbb{f4x~#^GD%;kx;V5RHFqMglMl zBqHrH_PmvMVZ9MCt209z!JrlnbBN4e%zB7wDwKc_*ccMXC185D-kC`Ig)sbi(PkVy zQKCn23HE{O{v5_Yb1S&1H&Tb_*8|p6g;xOv7SKEtvo4qVN3`@OqmsGieO+(1RUzdk zo5&V9s_8&(Q1RX`0Yb#3z%Y~}mZtUIN!3Nx+UahG^SJ}+d@{yD#3;~6s68xs0R1mN z*+Ny2c>>s8rWPV3;HM&6m#CD@2kAEx*a%Wo#1QHO;*NzAc0qmB1 zWT1C^7>-;;To&9cF`CSV(4-ut3rMM!wF&5cpP_Sz&;dA9kki8Jqzw_S<3s4x7vu8e z?OY%S=(a;cun>{n;n+^dUq+4vqS#lhkSZ(Pk!4ZR?#~0*;AEg%BeKG)p{eGS?-6g^fb-QG@<#lJP|O{{M@Z^K zD^3PZG{t$&ppEUPr%EP**<|{YX)F@QF^lDmFR8jGd<;xrF?i~=J7j{a*H1>7)zvFv z?;&!!L^Ut}tCG_MB-5jEtq|$~$jSk*3`D?MB6^lY!g2?<@?jBQ#tvH$0MJY=5D1B4 z4%seC6@51dU25Dx7K>eM@I=Pdk#-xr2ZW(Dj662q29)@BF_ksY3%X{guNG>ThdClY zQ9M@_eb7I&=zUG1f^}b6txWhnq~5;y$K@}PX3;;La1;pD>p>}ptygE)2bu2B8qd2a zB+okpRH{#LhICN0A0XCVroE5f@?|?h*NBfQ*qaq4YP~tdD$ZM#n!chyv_GU!+^`x8 zRBM1@Ot40XFxA5e+;gj$n1sFXjLAX510T4xC30cZQtQFM(?o*B=Cl6kilZG|*WEz| zKO)P~wd;bZyF;915^aqG|lIi`8+^&&aM)s~hY8YKA?1+Y;9ALS|hk&das;78zA72u~1kFfK4$+g# zJ`rKkH#?9#u%vktDxXQsE`302?_X(G_v=FStzh&)2atSvi7suhV$++6K(;`lk27F9 zB*J(M)fg5Z43-l_3O@AN!sq_|mCI3+(|USmSyBzF`?!Iy{`vA_82j{Z_do~3XSlUU zb4wuK)M79`0WJDvsQLC?iT6XAWOcy>lt#u?`_nMZ6p9$rI@C5RG#gDP(SunYns~)T z2YoJ7)o={|8y`kwWZy>8cBQ-Rz!eEi-aRx zr~*JBJ2P)Y0VW}2cg~UkW)5l(xj2gO-ju5pcPiEy z{5I#qE>JO>_|OZ5_JQ9znHB;@%|;DD5Z4hugR`~fZT^F$QE4V51>O4eK^p@-Y}}~h z^kCf~QUqnX{gqQI%;F@#u}gL;w z5rssd9s0HK`97)K4&h)1G1p_+oH^gs$7TLbmt3+vcKI{-V!bc?w?|C&u9mrpItE~P zLk3_!@U^dxiKdjob>{#g5XqIqrg|L$u7P?3GC5^O?T+oyuO~oKOBHA!G9#R2K{a`p z8W-0G^bZ6mVr-K!*T>hzXB;tgElK}a90mBSHbmdG@mr6vCcO3_1<)5j`-l+!Ej7!) zaipwF5+uUW#*!m2#A56E z@Y%wiFOUz&&AZd^mRdfpWE zYiI1HiPrDUsO5ls5t&RCaKP~R3b3xpYQ=>!U>pH8*3Y#uTtazRI%gBqZq6Z0GZhI{ zz>6=ny5lk>Q0tnYfk5K<0PTD*GY1~fa)6&ho3Nu^YPfl;t@_DWG>=waE9&XY+}Bn& z&Ii=0NO{zj7N{1JYt5>Xw%-}>$OpYe(pH<5J)aTIs#yyWUh`Bk z*rDV^rz;#nTS$)hOt$0CCx|tOtJ2O6TOg2pE0{seF6q|lW?$n%YQjG>m-XyxV-aLJ z4SDLVXXicJV>gpWe>yJ`HE2r=1jWkk#BotGVlaTrr#L96ZnM9!b1>v>p67*R4b(Vg zVSbVRHAHADRU%wD0Q=czZQS!hd<*J>-~*TT&WLbH zr^tkMV!_514-${45$IcLfUCA~Wh9J?0XpN}W1b`An~)!iPIq;+K!3J*$}m;yLIzN^ zleb(@_2w8yg>?V>&%L7^Y>>Fz6FDV9gfaZSm|&HG9O-Q)6MQ}^+&-&#I^=_FUH9M( z{9~BB9Yv4W%Is>xx+?i68OFV9FaAMVTu_V7FM~&X=!kG>6ddyyjBx)IM;Uf5njnx~ z1yc#3JD`~u!0nK?W0?iC6)dc>y^d*ajMAb(9+dR7CYCmM=xOx9k!|_O&I7|qAZe0m zbvW)Pf|MD`*2Gn`zRUizw0Sl44UU3<8ol@_Uk}$1X2AhmxO<_OInL1MJUjo)Eue{K zXkpOh)^7)I7I^pcN0*y5`5a&n-@3QIK8~x03)iP4Y5F15NF7B%{-An`KRhBL=F_Lq z!q;!Gfu-^f3SI?$d~Ay5;TVUR_cUM$uLCn!dc*7(f9g4Yu`3kJaHeHM%w?V)h@jNe zY31Iw@1Y}x+%OIUzJ~;mc4Pwv=0aLfA!0RX`6;rA3bS>e2oNgUyt|n(Um9Ug=KSOe zFd?V!!kyXB@L?@FSZr1gvSjDrh>7{x9Ip*3;Sfyf#+p0kXm6awyP4=SlAL;0%*ZF{ zqB3#{KzS;YJPxz9+(5qLy-8x_FRKcAP-K@;Ximo*FCdh(n~O+H(SO~4PC<{zvI5^- zX-k66mNpsY2or1kcv`1P7rRC<8R^7zWg ziZHJqGy?u;{-&lK&NxZ9|Li(@dRt@)C;@7YbA~O>kLHtVX*>(nb54aQ`Q~_t)}Lli zd%ZR+BM^<-U$sF?e<_Cy<#H=NX&g#tPk~N3T)*lTh{Rjx+==u7Rh%yLG?TxJLC&|EUq=j&(6&B_*fd&oHdh$BuuD522)NK$T1K+FRYEnBAVisPS|CvB-?j6qVkGSJ5i zCBJcJ$mGeo@rk&L5?Jzy_}!$Kb+3KQTP#tU$ljH-U@Sq?oTQfwFoBHoppT4xQN7)m zWkp3AY<<^wt1dZ4Ym|}z>LC1<@#d;O{)K0r5 ztOWdCzm}bvi%f$bhqxW}^gR%L@^_ApDo=L4)1Zm1Vhf#as28g-d=%y)^F3R(`D2;M znSz!CzD(?;WpSb)H91uXFuluHcP9)_{qOjEpq_y$CyjiBbbj+U`w+QHusP zb2e>h<7?taFf)|Gmc1s=HWIv2W@g5hzak19ilV+CdY(UEEc+opHQ(?l8Z@c4>;zpw zRA{f)mVXjFP2qMDvYxR|E|(0kJ1%uXyEMH0!Pd8B7@KIuki86D5k-XHFAV>J0zam6 zqF93;KOCOg_rtW!%*^2DTs5R%W&OWRrCu*b8&}VIwi4X3VC%M&hWq)^mvFKL1HaZG zMKL1?w%_a0V!aGq-K=0jJq0du8yNlVx+B9ycef>t!Mq9m+uFjD%)T#z``^9|mZLyy z_P8=`OD{h2pX_v`MUFZSWtdJO9bNVO+7PlAKJa3Uj-$F6*C`RR3J;S1r*7Yhidy;E z%J0I*sY?|Q;qDJoJD0mLTF6Ie#u#+8S5%aK{A7<5g9r#o{AUa%rX|5P#Vp=In9Jcb zyT7*LH6eo)PN8*x?Dps2YEv?yo=0x6iQ>$N@sel#rSs!iB@PEc5G0>fzlH)MwrV)H z;Yx&(p4&h|n)IJ73m;;W0;)3z#^sOD_dQ*jNX5*tW4#R3pL67mpDEiai8l%#*uM-O zLDm?*HP2}rP&7LoOju|o!K%kg3cZ@cIU7Gzdu*YUUxR_zPA4Vh>SETN{XnD2M3RQs5p6i4UW;ke5e(35Q;)PBWD8{x--}u zD~ev={sr2OK^N4#%a>hkFM9MLWbBB+W}yTbB7b@}v#_Q%lln34wb8%ZD?3ybq+n7B z;ugzPGn@70>3A3kLtXIQewhTQ;m29?>Pu6lwXaKFT~SWtf`cZ1_(KsIEO-Mn*gwSo zhpMjti>mF`CIo3|kr2V58|iKsy1P3Cq#LC{YUq|`=2B$gZuqzF`QGoGpKD*h zTrjhrSb48|?Z**gNrT3fs9D#UKeKu7UZuW&1P%G_fsk>x^5sXAn;^vTArj;Ac=WXL zpMk@^YrH%OqxNy*dHW+S;A> zMf!Q{)lJ1wA$<7s-e#4do9N3JKL$3DZKTk*IfO|fonAL zjSn{!0o%562{xa7;GaK=m6eMIBLu^$s*@4{{SdefG${YK-(|WQ;OJYwDHZBE*Gg`# z#YIt#FW33%^7VwPO~&TU_kYAjTSX$6+QLA~YCXh-Fw>VTa7b#qc5h7@ZS6A+4SS_e zA^ru|Km8k_QRW1XBIv*}_XPuXIOtd)Y!P0@Lf2(ZJo)h%mye78m6xa6_ZA=Qv9sFi z&VN3oa(t&P^lpC=jVH(l1ix% z`i4i(f5u7e`+s^3yJP+b_`p))on=uG5zjXYo|KmETT`2fm9}OHoEFKp{8hT&afFPh zpeJ=4Jq}=QFYF(|Hy@Rp#Rje#9;)26(0Jkh&POS|ivKO}^B<=N%=jNeOP=FO|4}q> zIwxT|Dt=E9w{tfdXbgI8Kz@q-q2)viMIcw3fB&F@d8!auHlpPb^z|S$xFI)hS}1ak zSto9?%2fPulsl)$=e4spkErm6PM>f8Oab_|8SsB@0azh$3X(Ofkd@X`XJQVU-$J$& z7sK>-Y3qp?U1rPmb)&Wg6KcMkt!IPRe^w>YU$3}s+rY@n_0`u)F9@bU4p!RAA>Xtt{b zWPTk{Z4b_TPa;S<^_+~aFwsMUQh!l?Swwqh!)b~1%Gy3IVdtf~=<|>AAA9r6idI^+ z#yT|`v6!=V)67>+P2$>LienQiBsJ1>+T?rw?!ObKDA_1hlxhKLT^FtIM6oBWfu}U4 zwyDAQVC%5Zy#ZlJUYy|HW!ro8w+F}eOn?3I-P%UyhH+7Ga1TNsSD8hUgkXb31F>xL zHN^eR|EO;0o@9xr`7x967_o6*IH@Xb(t9H3d+&npzfI5P`w^QTze-WmZ3Q~~-IzZg zP!ZqMzQ1yZ*Eh%jgtj*2J0p%zo0){#oEW^O-Gfft#IKm~*Nu8-Iw!e)Xq%ncQguza z`U66ypL=ZK7wM~)9d0WaLJ=j|gQuy5|91ti`}p%**Bw~id);!lxB0eyGD8G@t(OqI zdC`AhUhVl$+s(bwu+-bG8u38I_f%~LXSoA=ZA2@)3}951rtGm= z;+7|K5%O;~Y5NK6`0(J+UC^z+cYN#Xrj|v3kKfxLt>4dg1LE^s@xSfBEGuaH{8!n5 zk6*$?hhCl92y2JmHZ|rwhdgSLP)NZ@lb5l;!z$Gk^xLp$W}p?3%A$sRweujXuA%>V zKs}2Us%oNg@CtbPPttk#{jjRB<;^?vY1k_rKZzN48+%klba%e8H06?VmZ+4M{!qj| z?*R{f!p*C9x7WOQyBG~QlAJOD(N;y)nZ<+-T%Zefv|&3Y-1CzV)^;6wIJdHV1`Xwi z_YH*b!toz8fey-lzKV_`yn>5ziV06;Q;dLCwP)KJtu$4&GMedti$7Uz8gLc{QA_0q z$PF(rvm;N4r#>)`o6i$JKi}R|%DFmB>}qOGB515R;@q2B3^cNFdZLj8Db+LDQTokk;^IB>+X<2()2SUk6V?W< zUOi;&72`q!7}!HY=z-n-15=MKeWJ{-ieis5S`_9aKvcTU4+#-qy=MMrMzC(Y(u z_s)Y7NxnUfSCP4~jt1~1RR7TR zA2|Xt2*#v9q6!y7MZXd)@zOliSBj9hzK#rx4^a6a7Qap*agda2>yX}8o`h)*J3Q#u z=B_$RJjOctgj!3>!h<5$tY`nPfPLk1*qi6bqryRg10ETn49Pd?*VdwV&WNM%eZu-I zvLv3h)_L++5WQ9MbTTQ;HXihf99T!<=rS4L7 zUG}bc$#z|zYr+JIr{|+Vw?v8EjmPFdLgJzOeGsOvW)|0enxz997Mw7hdP(E6tKhSs zm&JQ3NwxBqLeJC8eXUP%rr4CcWFR3;@>fV2-R!hVxIK86wEAtE2&#&@n6c~P%z{cv zLReiT{zsVI_=!^U*x23JB0r_CW3*Up%so9z@(*tYpbZDAE?4wq$keAd7ZWL$&Kem! zF2$7l&*yx+u0H-$ad({)+~7zff}M|H`!`8y!A1nZM#*HIep$B;j*30y7~m@lR~(LG z6qEC$g06h=8My(gLf2eW!aZgcqNMc=3;lumYx?dw9OjD=CuWOq_k{LmU)>vO=7%-^ z27T?29b01bsohOFgXAbeHj}~A&)pZtFn>57OTCW?BK=_v!K^>IAEAZVPqd}>o`2Aw7|l&*xw`1rhBvadaI+2zid#JLiz9q zXB#R@j^Y}RHdE#qNHh+PTO513^+(z-jYoTM6-9qiWNyqP3HuvD_cVSxgCW3D-IiOVUQHnxHkSeApyHlsy|kxNy_0| zM)@MgsuMm!)%g^4W0^_Jdn!-gQ+%K(7FL${^!fC7@}1JgJ7jpMz`Eo`O{h5D-0_0?Jbn7)Lm`>Ocm&`A2iWr|Kn9t`UQV}&hg<@R#M#| zx+yEAd@2@g@q@Qb+1E7t;jJW%K4uT)UpH5&jF}}#MJl8Bc}1{MuONe=6{m&e=lwbt znpXu;UiN~ub!i~zJ`ZLX@uc1omrK@D$2)1boNKaTwfrg~QaErVX!s_s@#o`(=uC~S z@Gvu4&ws9$R^mS(cJdVZo>T!vT^72 z)>v=$bdaXickNhnk_W}&`NpjAsAJbq^}=}Bca1RbXm@Y^>HhFnH20ggy56wnZ*2M@ z;={kxtzUvTY**n%_-@E`AL|w@Exq3RwVe6d7N7R)s zlJLcu+aUJ;C>Eezce~p^z}7H-78J5v|C9)#5?%@+WjpvO3>mM;*neO*!`R8-fxVi! zN_JTENap1Z;?!Sw_0Sgi({$u+(^mGqUUa{SxsgObaJ{(iwTJmpx++^4>`xtkxMhj-0#YZY&YGO$MZXLWySHUz z;)*#Ka&r(gGvnPAZ+D_Z)e+!Bh2;SwTG8{{pTECI~?-K2-tI&Y0&5+O73DtC6 zvV3h<%-82QcOCZey5&DQ28wZZGLS2w%!RA*v|nt|>ucq|9hC2}e!PKtn&S3gcBg1K zFV@%iU(4f!u_A15lbg|}u}6UFHe8GatHmr*JJsg49bP{6ng8k^MQm=C^FDH?%C@y{ zxrn~&SkB)ZcSB>;KUyh6V5Veq8MLr$)XVZl-iR}K~&!sh2T#R`-4Xa7nmUdan_!#vard$ z!^5V{hd{aH2*jQkCot)-LiMkSLF;+Ea8jgTf7fxKQ2gR;5ZHqx9J0FRXQ#any~+<_o}JL5-%ylZj_=9vQ+0FU9UtD7 z!$nMCu3LBGUVezV4e^ps#nF2U!Bo^%&gGs)$0=sNZWlQFgq?bQmF#Ym64kLU#r6ny+Fsk+?otBtzdRSB`TlGg){uMG`1_ zE30nQ&6OuXeVR=wtu@4n&uhcNUiKd+Ll@rah7Gs#5aHnHxgQRuIXsn(KTu^M^a#QH znQ6l~@j~aYV|qmQGr;K{e7Q5x^%$*dyzX8o5)`5XrM-P!w)^HO?dyNd?RWSvqpG5B zF;~&QG*yMkH+)xjcY_^Q)L0(I+i0fbU+Fy6f0dV6yq|t&G$Ozdh}DBz@Kwp*Qmu<0oUdaKa2`(fXrCoH2W=yBS6-+;TQSTpy|YC(=GXyrQlxHF zZc+vtgE*w)$XQ0@m6t3wNtvv#8%^0Vtg)X4b~F~GA8-U4uD8K(hj|xD?Ir~YggDS@XqBoUd@M^A*o$sx|j;HYAO{icxce5U=z+mLq;JlQV4 zGX!DgtM#h-$RS8;3dSTbYz_F89>eQzlJmfV;K=^-?4k$NH?k{*)9HWu(@9<@Mfs!=a3XUb@?qt)r@Nbq?eU;L;6dL;pFR7K zbd=lRGB23@)EnvZxBu?LiwJQws4hVL>75{)yd*Vad~ec!>Z4hve6;Sfl$IrCN&Vq=xApQG3x+B>zbOE0H&%RgY zfQfw|^99aSjLtK4qRV^#NY?+3@*yZA4 zZqZ?5@-Nz82#++_IzKSOueHeNuB2JzX5YidLgVZVb0IyfVeZ|0Q;YxS{YEKeuYEj{ zi?_djE%Ue=NlwWUApTiQ+w@=L?X`h2&Y))Kz=Wul&o{4hu5wJt@8)`3#D&b-p{;)v zFUF)R%jz&dK8yvneRD2fF8>+>=MCee?E)kZ+>ysQQ~uyA_%98(y(0GPxJIa)m? z26c?!*Z5HPjqU6=CL#6%5$1>HT19j9|w)&BH`k&J9@LXX4&+G4c z=;C9AV|TpyAglh9u0{cMqobX;Du~C}AUceH6yf{(ZBYGSwW7T=pE*IFT0F?VZ>#f< zCxVj9mt3FSbAKhwLA66p!us0npaWbep<3cs)fp(}@d-=StrmL8s`KjG*dUx9hk9`l z6q4cg_c%Tj*4%YZr4QcEW+X4}!VL|Do^jL75z!VG`0y2_UjjwsaA@pbAxAmuib7vN z!qIu0i?2y;KFP4_J`h)~K~e4oF?>QrVh289xou zcsQpEw$etN>M4=$l#`wp`_U3s0rUJTSax+rk=Y#}_Tp3*!S=&8xYY$i;z?^|HJ91x z2jMBXHPx0b$T&QMS0B3|F4=d`Oh7`O#Nmp41_=^}9SgIc@R&T(;|~JcmE9d65c2CZ z;a4RH3SiotWWO+aJCo_7KR{Q4arD7ulxMvz6 zc@R(@Dz@0X$V&B}K3{U28ujae?06DStlKjAj_`RutsPb!LYEw#D(t&g=xUO$Np}s+ z%hvj>nYUUiOkA%x_*@UK8aG}Ne^~Vrdq1_llR9}p3K2rW5%73#KF7Ir$7|nZ7u^wX z8&+tfZHL73&g{&J>&|3m{%P8-(M0RwNB!c}fr;j<0IZV$%R5I9%hQNf(Nfi`=(bOn zoI3CR70=`wg&@6~eGpWfT*xEkHZ!o)sE$0tc0&4pZb?=oZ!d<0hvLB z`~v}4Nim#`Dz>3T~-8VK37&Cq+j+%roCnop18BL=H{wKRXZ?eeQkEJ zS2@Y@WF7W~A|PQJe>tY7n+hXa{0`Q5&wS$PPDpg=rBcIJpi!!CG6^F+Phw!D2SWI7 z7(N2Kt#~Pge+J}dh%i$5la0qw;e#HKXM)t;Z#d@{M_Q3DsfeH6T00UrK<2Sya<6Da0 zG@eaj?4n=8nl-t0D-SwEqVmSj&_8@I{U8AUR+qQuO0kEEDd2Gb>tM25!ll&0tC73= zXckG|G3a^LX0rDzMdLmFZfp*VM^wqqZ&Zh?!O7@D^Q}#rdqIW&`tOmh0LV1C5(3$w z-d7;p>xQ&_zLSHK8%1D?`tmA9bHwAR_YAgmPE|;wA#9P*4MOh^gVcD)=?FN*LoH8wMG!N z^)@`JVUBlTZbGdy4|yyLj1nQ{|LIvycJiY_v1L*(Z;yWlzDBW#Vq#1 z(ok3{)Tp^Yd5jlB@BQQR%<2qO0(;9GXyC(ev zHxF0qvniPE=4}n9@T+DCC0_kn)~1$Lh8H7L^7z@LFNHbfJXKx;f=2G#l+__VKW+8d zuN+(9Pbvz$>CLzKWI@#sb~uRo+k&5O)?4PZDSpi<8L(|g1)cB-YX3ZM`=da+EA(FUVNGD_X@pMRb8veUa>FO2 zI@?~q*r_HMp5Qwvi0tfxk0P}GAp=7vXXYzBzX=e`INU?Y(ghBme8h}WH*cyuVS@M} z#{U4W|6{{;I?umO`}}Xbhj!7WNJ9Y??^b=Hc~unTmXKYn+%-)j%xi}3FT)omfjhz_ z@BKq}3#=Ts{|Edv^+wAgS9SbdnC4a(4U*1Q_MALFd8llEZ#_vAmzW45vW(KawwPg1$#Sm!v(grWwi3}!*;5oJqaZlm8#vc z?fML7dVCV&AwkE7(J*@UzjC3Vyx#tPN`v`gNFqKsKdJQL(x=gq{`~a|0OvHt+(;sO z#yck^f$AJh;Ob>1;WXwx#7+P9DddBsTA^e|Om6=MRp%VXC&Pae`_3;~ihLxOa_zgD zK`~~Q#s1zSYxPydceNX+)!4Hw6>`4HmWah*6qcr{$T?`FMT{ zlhTy6==E@Sh&4CJdMX3h%S)*4K6Oz<_M;|uCGb?F9)|+^HjKNv7bJQjx;Dp>tkcdc z13SK4RQmf?l@P*}il}t@-lm(>-rJK_g}ySsm2pkZH=u5_U`1gb+`aVO3raNj#)D>J zeExm=ul$WtJ*^j00qyM}z>^Q)DWF!``(E!qO#YcIClz_~hwKM5ztZ4af)Rt;@dtIs z$+knGIiH;`o%fN5^|oFwyE$H2m}WBhsV8q}sh}+X8V6!r40p(4@NXLM6fZQ7<7(jZ}N&62WlSYBg*x=9BTQ?odu=EGBOH@nAO7y^s!`{QqijV-UWNT_9AgZ!Fyv4GP>)((Ti z95l}oYFkB;Mr=yZdAH@K0{wsMDZu<0q9ti-ql`)=?Cl<4Pk&0Olu-{Kz*7atGw4wI z0_3kF3k5AFD&7^=BDFs*G-=;O0KFF-RCZH~@-$kqi6|`;6BA5eIeE(=Q`x-xUc&4u z{u));y@=rq_RK*vMv5c1hTn$TGQq`?4*}QU!xZeO-_6=QZ_e2PT+{tri{w+(8)IFQ z&T_hu?blhi=Fwd>zMijX$`w(jeuo+R)E(i4`KC1xmYeoJQP||q!wPxEK-LwruTsW& zAqVZ);7}61_##aqxjM`X4ts|6V>h&arr`UY76i6>;*zT^c#u&Z-dUH{e`NkMOmt45 zRDs65BG<4ZG8OJ;K|(F%%l`{P80PXc^K-R9QQRUa>{J(%;f`1cyZ?Ma2N@%kB)fh2N%-J>=ii1*q`AOYxO|AcPfBr+YtWlus=q1-4c~pdu#f zMINB!TcUruKcMZy|1QD#%3_*0jgN6N)5lnT-DMMQInasBaPv(YU(JW+jD^nAq&iSx2Y2a4Y5OOKm$PcbynMj|6ig;rTyx$02Bh5?p30srbp&mo*^o3&l!w{#8V zi)qm3B_$y)G#$qH61D(gglxmA-->{A!NIJ@PM6M|&TkWtj)I_8BE&_-s){OQp{c*p ztw?2nb;^374KXP|biB6cCI1`ak(W1$Q9h)WyXoiaK!x%e_5TtY|uSkD#dBLPWi8T|UJ41e(3!UjpJHhKv^ENv|23_#%wz zL=Z>J%RuZaP)b!b$xV1^DwBz`{a{TK&Sl8JGPjY=mw1Y{#925`6-r74l;*r9=f#C6 zJ_9)h1dW-}&&jtZ+gh@zn4JpkZ)?Vz=nEH?i1R=Rtzo|M7EIBtJY>7$I)0JDVN3qpyb!Xa3IIMMtTWi zpMlaaQlv~?{hHS8oKWdwO0GLZ|iL{eyK9*LI~BA)wzxT?0ax-6q<3rW4=vX z$TuUd!5{ha)n&izdB}wiPYgj&*v@u+r=LK9>tE#UEaXt&zcD;edF)^DlV9h3D~1I* z5O@$(@~^zgYVz1-_y!3pkf(Wu;l*}@Oy#_%@$ap5AHhPTHnRL33S&I$DjDhA5hP4w ze(Bi>Rw2@7YFGCVxU7IW*0Wf(hV@V(GUw=53BKY)Td$_8ad&KrDx%UGqoa!X@TyU8 zM89oTNU~7tvg_d|*2$R)ifS3ibXyN>z4_q2e3uzmzjUfPF>7Sp>#$o_Hxn+yvs9n! z-*)_E6W?LRMQh|G-{iXXgdOO|BR*FMhn6erZYmUl%wjRCRip!QcXuu*o4sUJHMl3W zG#ac(hXIs(RmIlp!2tNUF85Y1>DAjyhah}2p|YnFxzDqRAkZp$T`gBE?RCptwN&1*ZyXP7CaR`ww~ba5*7dMYZ5ZJFr{ z40E)x=k(JxT@%CAJ1cqvt=6AXEY;htmoER|GK# z=>OL%Gml8dD~w@m>>O?tmA$_`{_4IAMZS7!N|0qcMv_#vsq8J4(yYQ6CC`**ZTZ5s zGaY?hAd6OlH%|0P+nZ$NLt!z3Ets5qW*|0?<4dD^d3re_l5mvr#?Y|bLzCVLC7a$) zaQvU&csG;^Xq!$JJiK(cvO3>CGyNcPOmCnHyj^EoByXIyEK&*{DgAXFS4s zQPTC3qVeo&vr(y5fCrF~!eo4(PJvjbkCv>si1ycn6?p|p%I-uT;r%3~r%FLaF+MFB zR85VR@vf*uM%6u7Bucf)M+4S`XvK!wow;5 zSiQr}a1eEqAb5tlo_=D&EXOf^K1(-63ZgwGsY*@3bE~FT452>RsE6 zj+YpGdrgGMH}Y(uw9G@=4NGn@->EWXAaMn}yKRMYy;};r@Fdh=xhcblP#>#Z?<3C& z!wRgb`Ur%m_S$`E-Qki>k{c)hMy}^;5O;+^I;dAFJMwb6q@}c?KiteN`sP%vODQ)? zs6}0B?{fRPWYtE?&N_p`7oIS+q&x}d2!yJ2g=vq8=|)Ej%KFGLk>v)zIQr#Qsj#ML z{F1p!aliIhalyD3Ee8(sA=WBn;%{_P3io=idX^^myR@qNSsFVvy@*;gS|x{kJ@G>b zO4&XDRL?hRmEu5VT-!8+6e_~*P`_wC7mL|k%KXa(IPvuLGgcWRj9z`I?n$#jTkcS- zqu2p}S+T~ic|+MEHl}{B=z3DkMi?4|qxEQy^GY}Yf*ZSDSJ%|7mn7Zk)O|4yIP{6 zdDxe%2WNu9bOMnlW`yI}^ul{tRnXTdi=ax^0C(GoxKuqla1PO+Mu^sVjxK`apwxn` zqn-sAJSvH%b^a}?c9IbTv zao0aZQ%^!=SnU!lJrvQREWXyUm(-VOQSozxA+3(V6gR5 zbkMA_Q;8Sjn>-{0E*iZD6*VjONdz;`2jX{Z$5=csuF=+onx9vVZz;>}2Gdusq)P!T zbU~Wr_&7kpHusJb=wzyrWjtkC9|Vf~4LHyO{Q9{DYrsv}-NObA6#Xyktq(LJA>-4IwiOXixhqGOO#`->d){Di5u#R&jRS$Vjm+$Wy_ zMV+#Ed1c_tk3FCu@*>74hcoD#u~MMtk}?eMGRtSqT984F5Xvo+FGtMJzE@nIFt9kG z=1@k*bbfDt$XsPq^L5xav5)k#n|_|BSbA_QEWHkNi|R$kmU+b^EBy6)B|EJ^F=Fgi z)4?dnev!mQzJ{|`S>C4t8L6R!A|(WWFH&_Ncp&_ICwK$SW(n*;p&i0Y`XLThTyn)n z<$=aqp;mnXl9eyMyOHgm*Vrrnh(gT7m8VCPyG;!Y_UHbp#f`wSADA^-p%fyop|bk7L(lD>=QgjQDiu_8?-AAm?(rbd0GT~rOKLd zxcWv6I7#38i1ZTkAwj*a|m4G^uKl*h<4w_e1%M#4h{9r zZMY=X;;K6=vyk;}Nyq-iM}hq-*IR|C`=Mg&($1+o6roPzQCyokdN7XQ0!UvyVCaAt zppt>@mI696tGH5l`$-K_f5OlRViz<*6^ea)#q|X?|5=vU;QkmNGF=VHQzt_vAfor6 zuYe1QMEO9k<{+NVDk;mtSLGQQ$ z#3z6GXnHp)L+VxDLST7SErJM|J(;v*z6l@wXujpAU7(JsC4xi*LQP1DC|17H{O!rB zccVa6Ql=zba1*0jurF_%z_|g`Gm{i^ibZZi`+BvpT(ZWl;)cWG8>(6R%JUGGkwt9g zh@QeFs`KjiknuG1k^+k1mfW~^f6_IPu#1muB-%Zzr8K3-N0}8$;LZ6A;gyV*A+*%L z#k`=1p!3~r%~t~voUDHAuQH1B;secz_lPvE%$aP4+f)OHQ2L?w5VEl|nlmX%2nllJ zP8Z$a&$;7k5_m~~gj7<3Yg*mezWVi0V9uVka$oO;tKpb>y^V6Ws=Zbf$L3gcG;QyM z?04}~pshWngV@V7UP<5CiHm#uqP*R7Zi7|BP zIg+z4qrApZdxS#?Qp&l4!w_wSin#Kf*LG8Bc@j(aSESv5paztf9Y?PU5PF?)FEuVy zimPrOAaL_Nr4ziG@i`FKOiy2Iw9SutO^R0$O9$b!bA%9>Wj=@Oc*&&}c#R6QsP+K# z!Iu6;)yY)S^W+v8DQjne2N^s7sw*NbSSK>TLTu6qq~K|mEp71aII*H#pY7OrC?iZX z>s`G`k#+ykbm)st1n9eo28`cb3s2FsxcSS3D?^_GlfJ*s98dIBsSkcf^A}jg+Y`9Q zJ&lMn^WE2+0{;QL?koZ*{tr>h2r;lXaPI@C4gk5>XQE>8!(g1`-)vm34X{|a7VX{b zW|DL%NunTsr4`)%O}u$7W0LHnjdYfLk*tg5xp^k+eXU}os#Q@Yg&$p3FVTueUPv^f zO7}Nh@d~-f_NIAP0q_id7i4)e1>2N|YMK1W%hI|?2H*$0$H#Mm_Oq-__-m4u43_Ui zI1T_POOC?W!4~{-U+w;x4dwAhgZ#Fu!6a0>uJAIu{Xkk7=gZu@gyT^>C1;L+VF#nC zw|0zFCqCS#cR0rx2$%U~pt_<5(T6h=cxstO8+};GVi{2SZx+w@Y8CZv+7NBs(hMN@ zq0@KrJCHY_3@ZW<=RL%}#4Z@a!)rL#BidU^sMV}@46ztqe7^M#%nzve3>K-YZ(vHQLfXeh5h1Se}uJ!a!>hDL>WhVyt!1O0_ub z=DKk^RdyCAcO~W3d)UCfp;xYQX^Cs6tYB!)B**G&FW}_~`Z+UUwg`@c#=EZqm}zr! z5VcfT6jsTJia?=0x8(CeIX`>^3tM@rHQ85Xay37ftYU3?il^sUzLVy0?hb(JvRa>> zmnfY^;FSFBp65(Z=w}!RQSUaAI*Lnoxpr08?2v8+UNDSUa6~ zC$BxFTo9*rv|Yer%+l^*<~aOGv)a0Z6D+4|pVmO_K35E@UnSkpSEXFKW;xG3%nH&x z-~r0B81@xT7$+8U+m-HN7pXve(Ht3^@!g6bs#3dFZ=Acpl&z^Cx!864cUwUs+~3$s z;i>1vy=e8?&2MO{o>#5bE%}<|GD&tvTp$FBJ?Q6J=P1Z%ID{<>y>C%HU;H#~1(ef6 zY-ekCe{@WF!^90+Hy%M_Mrm-^>j6~ z*{LI9$^KcOoyGU{^Xb>RpYh8`{{n)n|0g74M*H*uMCV~c@pN0kB>5-qwC~87Q5rsC z0FRh^fvQ^O+X3$rs4kj>v{y+I(T?nu0~Hi2p*~N&HS*=wn*jBc77&pOveQ5JFsV9n zt#lLv@x|;6Hw%Eu;p9w$o zFex2U)c)9$52~}K12bT$FnfZKZ9ap`a%=o`$2@#`LWPuOh`~Cbzqu~~fAOsygBs4h z&bB^St}u^5hHv3e_2@1v#6N}|hV19l9)vnasVev|gr*4kr6c%vPyd9mED;AX{70?w zAFJ4N653FoFOb-R6bi_6uhW!3Le}Lx{~KsgsUl#IyU1z4Q~r!J8!}UUi=h(0A#>kj zn5p+a8$=^d;ci6^;L2u@mw&>pV|)xYvE%rTv>s^E_l<^$?)_UB4O+7G>Dv35APz0C znw+w+ng-{f@z_OcUDg+fwzRaeSH)o4sqL(u(jG~Vf^TNuXajfR>#}jY3T{NyV_s}n z+S04Z8HT`O08Z?W#@j8SGRrAj`CFLfghF}L#i_hB^xNR!I2&5GQyTf|bCD&& zB?6%vfvLpR`(SD))MKjMFQwq)Jwb3ywD+nQhfR*7DvaN~9o*V|swPft;vNBw(peqn z3gv`6;gTAR)MI5vMJ#ily?Bn-?J0`Bnvg6CPJ=7$IPUF~QD1w^+7?n%HH#aHe zxbl@5`y0|>@q|oJrn!^y%B~`0+Ijc=L4pQsVKo6Y12!D_b}9M(qde`EE@8!It{W=7 z8^4p&&x1=hf#G?|$~jAaKEo>TIqe;CDDd{h1}Hph{* zH8wFu8B12j_z+A(DoE(E)L>YG_`?gKyzR)h6+CP_S980c5K-l_zQp|>t?@QB{M#Y% zZ%rl{mh8-s8EhSy3!lu)Xoza+3#U%g3Xo(KvEQyI7Kcvc!u>UuV~>H+T{}HLH-Hn! z*eadey2=wS09YJ}Rt&TWx=@@k*II@)UY4Lo_TlR6k%8qh=NwIXm)LRo5%k#8L1TUN zVN#hN@3rKwUOGt~Oj#j45*Tkjn93>rglw47FydcS@Z-^V^exG0lEz09=%?r5N6-PB zlNi#WQXy9ESlT%DWA<6YL0i=`6quPr&AA1$Maaq?VwFL$EdnvDB>p%pM1FA){XrQP#Y`dZvy+C%%c{J_jetz@aRBT+I_w(AC+OlIs_+R*t zAY5XZy&FUm*-*Z)sy`*a$BL%3t+@exhzCGNt&#hMKu1{HwS6Ykm8T!!g4&Y_#o=DB zS3UvhRRvh<+a$pMDb?K0jBd5YKh<#R)zV6xbdl5?-VTZpv&k}Tm+enkbI`1+2%523 zc}U0?`JrjaDrd1)3?lD0>8q}2n4CzN080~Y~M5j%n zxakceV%~w3bsDPtz{i6Y&X<>|uE!1nry!^cIv_jmYqCtK=CTs8kzrY~2ESND#`C<1 zKQA~G`Cf4P;KlgC*C4T2U%v3Qz^Sk80^>o@6UcP4OPr7+i8-zPst&22gEpDD+@I9j z0@IJiinSsr5`-GG|dMT&;u{a;M|9^Iir2Cel&{@*|y4MRuHR zGZY|I)LsH2B}$1XB*Qr`k#!*I-zaFB0nAZn{@d`d_LBw?7yy&_X)5QHnWN;m%4t^x znI6KGtyHeJ991kPfcZX`K^oLVbE$duL{ko^C+9_2frw7EX~+8&(ebJ&XSE~Bu_to3jZCEsQ#io zX{gv*U##p%p&ok#NdUFgW8f$Vlx+3lq>HBx!kyY;I{5%DmZv4CG7+fSI@j6GGUWaQ zjMY`3S^^5=i#{6XBTEQX=ER@gPpNt~gwb@2EYbRc%f;=oSKKr@cdapDmJAXfQM9#D z-BqQR&&O~EwQvSrSJ_FzX6P08v|*R5am2{;vTCs97^b_}O%ak??Nt@B0To%U1;P#^ zn4oV2X*9uDFXDy9Uph#z`TQB18!|J9PKLHrS^E?;;ZxbtH9gN^sf=7i}5e1yncvYgK>e zM+GjbCP8o+XOQrW&m_x1b9}KnP$3zcs3b&S@Op}l*wHMVn)G{XWyN4j(1{X zf8hQb868+Q007vmI`N@Q)j>%HQvOVR93UFbwlyzLQ5OKuVJar1Nb!=Bq{?%}6O?MGdfnk4@ zFt1giYT3SpAfAMlNLAeTA?zG%$B8(?Yc`#It;EKQ8q$961~M6it{)#OG2@RAeiX6R z;l>tS!20MjJPYv?I$}>-mt@=zRsx(2zIu1vv!`_*O*@#k=@nZtyQ+)d2w~E_TnQ}3 zke67cee|kI5gWyenza!~{^mz=E!y#*I-CMjFoSJb?hp#g9A4tB2B&OSlZB-27vSGBhmPWXA$?`{joxiTWv{wXd_7AvGY)V$w#S8_)O!l?GylVv>zeSKmubz1tUcFbjhEEAC1?E6c_kw}aWEOs@BttI?&m=Kv-!m+= z-PnF8@H5wvPD|JSqv|Vz;@X;M2M8M69fG@CfZ*;nxVyV2XmEEYxD8HlcSvw2Lm*gi z2n5%>lY75g?>&A{MHNspXL|SUUcGv){`pLv4?ETBJaG&nZ2%=Pk&M9e%R3rGn3wAY zn!0E$dd1am^a<%_R>@JAr}(1P-yRPYa{&%u$Pp7fP>b9^a|PIfZD0(8G|_<{?{4_> zn`t0K@L~P9$gjU~6XUG}(QJ=%Lyn>(fxAh!YvXN#7KNIo`i_b+d z4X5t!DlK^cvyJPp%~q9+Nk-lLqh~R7;wQIUX>$J;Y0vb5Hpd;0KPY)`I`p zFK_>n=+GJ6oo>6l#G3LXN50;%)~_0rJao$zW4^3(tK2mOAx@X>08masZR%IdDLjm1 z@ulL#V@^JMrxrytygA%`G%cP`DKEVB+{)t5ATZvo$}d4}aHIFGlq5U1O<03pEDG3T zG-)E&yQQ+=N)P2;u>67wr@YeJx~2 zXlVC(KajDQQM>52AHd|?E8;w{^IKn?*C(_P(h+3Bjfn4btk@p3PxE0%(F+8P4J1cU z^E^g^IS`F<9sfWmBKE|A8H-l{_fN@~RfHKka`#J{g6gi|{~>w}p^kGdgbG;eyKd4e ztjgOlUkHA;)^EnWg%!EoYsZenih zHdICJ_W1HiOtgWRS1=t(i!nhWW_h3I{&sBRTbxRy6jDW|;6e4`9H4}|`*iF2=#B!^o$(>KyXz^x?^o3V^LF%lGr7$_YyZTBy}NEx z_vta|NN=X5uY!k-G14JKiFRvgeS2LhH7n>06^S>g^M4oUk2$7FKo5{hf+vDLu|lrW zl5t`{vhzyAcsllmjE!>qrm=>gS*UYy$I|ef8_Wu?#C4{W3?#?#Z#qtl2GD;kn%H^| zA*_bM0Y@03@#&gyjBFYl zRcPA$egzq0eE}*-a&GklS;C^r&lYW!#!riVk%sQgd*8;!RMECZep02YeJWXhpZAN= zZGt(~+cTR%H*;hi1LAwah}QRLMY`sV>$;6Bp69OtKsxNkQm|A@Mi59Pc}6`lRK$rp zle;mnB;-?C^gCs`=?71w!#GTfpYQL^=o;gJq&)P_&IMqWSr-7jYFTzqr-M$rf*Hr< z4pqNY@@Rl<7d4g_V-1=`v#YDUYzs^c(PA=<_1`Ff>;FNa)>!dvm?8+LZAC*mJQXO1 zBWvHkZg(k)2@;~UOuUWM`FZk7k+y1XrUa{y>QJRG_3-nVp2RBivruP+PU$P}UpQV% z@7G3PoY&374pgSSlE3u#Mpy#VXSBTh7zc1qmksfWgWj)w--(sM%S572Sw%Zv!VcAQm~M`vWy(S zx7kRmETBcE)vH^*l&Mk?@*y}W6%dOO_giW%n<;qai19TXsJlhU;(8kupNl4RYQJ%R zSj=hjndVwxFu|^KWSssJV;vpsA)__;ynLZNKo!TU;H8l{vGk`ekzlEQe{U))!&_sETAK?Nc3L*sB`n~635Iy zgSrR_9~Z!*)4#TIgiMrgQz52= zSP6ShYiYO>&3P^16<t?pH?8Kiog|@{^ zfXtC#G(97&k+bT|w?9e!W3hg;6o6%GQCqOM73)4=7t2PuCHP?=hqQFpb8e5>U;+fM=3;kcjfH$7J6^Ly62T= z*Ll_3iLLJ6;(1UaZQZ@D8zj3aSi2Yw?JKl9!)rgc1wps6irGg!T=GMs$08dp%35p- ze2MAif1IMQt~N-I$uCIQ{@4f&sC=YE2Q)nYxfn&78%jt2Zv#0g9+L907`O(D&E8nw zFDYSvahH`t#(mU9B-7Kl{K^r1*FQ##-qi;?jd)V_ibYTO_qsI_p>|1ORfk%dD*`UP z;+Oi=TgfrHpmDZ}qs3)ZAihC@+<`8IhGD9Sglmk{hBN+)*siRqb) zwuD%&Uy{=_uJ?%FG_e>LZ+(hX(g|mx9Ilu{iwQ`;M!xMVzNZ^tAcjXtS;0hqOGUC- z^;k;7Tdk5dZ4~_3NA+H!19?;&tIn-G?Nu1)g}Sv5->&tiC}F8wKUib z2W)U70HNX=L8{ZKIy6*hsQ%KV?DR#SoN;f3ON+`{On6x_VL+^*p(LA8YM|M9 z1-z_&ZbfMYQV?oAdoSDBdtE}=oiPNX!xSxX`6^4lrTkY&xb!W4sfa!&OqBUpE8%$w z9b*}bxImXy84>)fYXWVCuXT$Tcihkr1BY+t;5-D-*JE|cI5@!B9&;p3bgcxN2}sah z+-VZ?xp=iIHf5E=oaToY?{XO_s7~&Y>cvh=avhFOUVU}|PSUB@HgVDn79~yhMw({s zqjBox`_Elte$GIt4C!HfQ+0inb2sY4Rd@Q61t#@qFkI?@z3S(sgh;N3KN7HX+^`Ej z#ad-WdZA@p;(pCWIoxv&3Su`!$KH|``1MrcdfCIj6Dkxud{6=iB?uEw*Q=7bh@K-p z9fV*q$!3()>_+?#R@^H=R{Xz`45&k-xN;dp3bZx?HSP9P&uX~pwAs^FkIf$u%JM)X zc8bPtzG5f@y_R1q!dO*Qu>UAm;y_5$|H*nG9mHehj_K7hLX|%0cS$8+uF>Bjs^=Lq zhAUKL%aKA{76VpnsYz4$nNOkD`5Onv!2$QZ5KRCcK+Bn`0iXp6MJja`66wJhR`}uK zegY8^8K;)ecT>a+^3&e}FWMRIr&YfehZUEqvhJ#DhDvBud+)6ZiS*0UP`kzU-uUju z>uMo61LGrE>G06sTq6NzHGV;^$sF<#D~Pk-x8(@88q8PJVb9VU3R^4>2%Y4W6#{;5 z^HY=%!|L`dW49?N*vBXiA0F@kbAjOeC#hoBQuC*f?ZSsO6kvNodH zp8QW{T4Qzp6*t*PU<8wpp)K1`1fcHHtko?0>6v(n_Fvpb(i^0?@M{g~)u>D;f-40} zEo*>5Vu2uaoUPOVbG7HyO-}2sv?Z5>)`MX;Yo@tc?kiE5q zwGB&7YKygaO}B7}jRS^=V(YT&F2$2D_wkBsS--7a(Nu*?Z1)x)OOO*AyO_J=hURN1 z5Y$!}PD{yFL|b;iD*McL{4Fil&}C8JhTp08MkWhL?+exE%(DJ)_3a7maRRtXGM-*t zW4!@3Fo!%G?(N&y9X}kHD6!qy|l!%ij=G+vodYqfL3F~j=)|~^yN0hSE+GdFqb>_CIoGlLeDl+1u^KZ;`uJ2N^+!5IR zj1ax3Vk8dTU(O6&AXX;)w{QEhbO(BQZ}A@p4;&j26eK^oh?0tXT`AhTEuWIR%U_6m zJ&@{H=1p`wV0lxXzCCd&cXJm}{=K^dL9}SPu{^8XAs8;v0l374ya^1`eRjrP0rEEL zU7_=df9?zkWyu&5hUy5?!-!KV13l$&FGYmT#h6l2{&rytiSBz5uXtSap$7;yhQ@DT zL3I!6s6RTylvM610j}zE2f(SCNt6j2m&R6SDK|aK`%A+6U3&w7pr5BR3)1jVU@AW0 z4ko9@t~_(f!gD%UBgv|;t;&Ba8Spq3vn5SNpH0^4AV7$w7O4s|#&J%IoWb>6rW zDsXXEsA&E=JkR1BC5k7A=f`L32-hh*&P5F7o{UO`7PHdJJmekNEWwc;lYUlOI&2j$ z@-LDlr7fJ3#JT0sf20Rdh-E+qELdetgW2`OjDWhsS2@r{UH**&6D++>=EOhzM!nbu*0x(4WJ?$5 z7P}9b(PWC@enjRN0Q4UO5CA6v35HZ0~-$)IYCLk3VsRCQNe}2d+PiZKm zd!6kb3*qA4qJiNw>`Z<0^{zVidbEU4JUyc=TZibQE`@5e#g$tsi&@z<9p~6^Jxg`O zAdizmcv+q4*uE1k*mv{YQ2}I_8)1WFT~dRldf7r?SCd3N&u5L4Z2;-`?U&pszY@wn zkZ=a}CjLaDzpQs#kVk{LaoZ3xRb~LsqkwWChTGx-IF#)KIbuoa@fYz%rAI&3sH~|%i+5~gL|y9^2G1WE;1(&(EsRvN?+J1u?tVlK&Wa1wNcbrwxymYi}7VZt-{rW7$f zcaS{YP8UzmbaXCw3(dg!B!@a-t?a2^3`kf_lTJKpf!z$LWd)AAH6V}GRoW-l){6ES zliNKaLV~aBhIzzjeC94-`x|9rkyFcb>+=YaUlaZMN-^wurS-?}N};7?c@D8$y?^ z3EjOf-AP@z0}}Ig4zu~+c42g`*u1fAMM^q+TOotmjh_n4mvX16t<7?{=U(n0nnSm& zBCRnHw=*23xt%?!SAcFYefYvZvbPNTo9Qo$An_foBjr6YYPczQHfP*gc~PrF6CH*V zL5wD}r6u<-D7)+082?}f^R)Y(^ShS=Y*7+*QMmA-2Mud-DWJL4ID3&Y$y@=~YB0Yt z!=AWxrS}Fn5V#$`!-Ve9gB@yNRPuijRfHEZ^t0E+eWRkv*Q~cxT1jPLpTAb#ujRKg zUsI6{rD0@0)ia@qRd*-n)Q8kAd)~Gp5zAVIo7YQ&;aeREX5RfAPI7tnVxS2WJ?P~A z0g$C?{1^VJNBq?QkScL4n|>>AGqQph?MKtI0buxmAeezOl^D)!@qE_fm-%M(zuHhaFg~KGHIJAYk>D0ax2m{&cnEoqIYj z!G{0R3#Fd}ur4A%+C7TSInDv>c%=;a1J% zSCkQaW`{CBAYidHjp`SnprG>OoxFR|3SS&1vTN!nBCx>JzC5=Z4Wpx<_w>1%J@MNY zm*CWh1Ea{LrR{58Fp7Qka~pUo`w?Ib6%;V5od4GG0_K9fEA1@rrgDgeu_C*EDW<6i z5CSPS!lQ4Kr8$cpCNZg=4Ew{b1HE1y$GayjjuTvlq)%n~?fp`z zlfJ%fkyc?DuDvv7X&i+FSz)Ep+!sDerltw2;#4Q5)G3OC?w71Coc*6Mk`;!LI{x%b zxeqOZY6C@7$FpDAi3;Hj-ry>v^4a}<-q}sfE0xcEp+XCIM}a}!e1HG8;PP#S-S4Bv z+REB7-VUXK-KM&pT71V_A`5M3ao9<=|kr;P+=dB_`GkY3?Jm z`cXe`RUM-9#=R_NmCFcys=W&iizxGTR?N7Y%yVhptg>;8%=;Tj3GpASSrEr-=w``v zVVJhP(N*S1K}v&xcFrN@vUKiP&4;7()?M$+X7)CKvPW3L!~I|YWoY<;4P7@3N%-$n0y{RzAASO_m;!SKC= z$tlvwp>OUJH{tJ?%2I1BN$|#`wUY@?-ttY6$OqbK#%C@XU)V=&3XO$&jE^u$G@DJ? zOicXm=dj0@4%e*t_#Z!x#b)h;RS%s(PC@n-=tddl>A>S%cE4Kba|&HM#c*guwViQl zlymKP8fc-NUfJ%T`Z!U%UJ+;%jgsk{X`6c}?VW+^SaN8G?y~>-)u1`V+sKl62mUfH zOIUqisL!Jfg#=cgw=uQi$xIwr1VJEuhDPGhKNIG#Bks|9=0yjTGC=RA*0AkII?PHC zn^1A_Y;Ql8$qlp?YFCH#_3TSKC*Y!{qOo&q&SpZ}zdP45S(LK({f1w;3Hoj^@aM&Z z+gr;{v|wSOIVorDbgc6an$w4p2@3pGp@$X>@7&fIDcY7z|AgP4|CMpBY_kA7)c&Ww z3wH(c(H7har5NPJ;sQcr)SXP!!4lhMmu99Leu#0!uLyBWqo{f8ZejFzOYKJ{8|#?f)3GPPw}H;D#= z310H^ttMI)P5NcpA#=9-2%kBh9>y&uY|~6zM$xOv$_^#%HFT|nW6$pC(;4r6M`1+v zUWK2krdE0V6zHm>0z=*jKUK+uv0()L9E!$y*}RNtNE1n`!Z;B`z{@q#h zYXwu~_Y)`H8{35Chyo!^PX}%FX4Cbmg<{x})-(I#r-Mu)a*anCOsS*Hsuo@{FiVN6 zu5IYmvE<|$nkdKiPEhmaPCo11(#NxghHrPx@OeU1+ZRloD_UsKEwa`RBpashyUR|ot?0AVI~#n z#ZMT?!HrQKY5~=du;@9iuogZi@i~7CS0sS}NrxA$p&qbrjj!n2{%*>K7B|65*niV2 z3Mgd?VY2RYtz0cX)NCGme-bhqO}WIjRZT4-{M%R4mAo$7VmOTpzVF2e$Iw~y=EQ+a zqk5yH1rxscGJ6Q?<)x` zfJ?yt0#;D0`4?QV&Nur+q|-{bumUMtrcX)4pxrAB7o%16nD3pxSHClBu{^pxox+sJ zw>XH$i^Cc8;r*1$?8H?A{_rPC_#zdoVEMVzqE0Y2t3WLlUYdU>q2&8h5pM3mdj4am zZ%(mRAWot~g6*vxXSfMQGg%XEQC2hXib&F`!4xE5mLbb|SbcNUPDC^w$|Z99TY~sGLX^TNd%FkI!n=pB6Ixornb@c!Ewa%X z$V!;v%MJEsAv-U{@+fCL3#ed#BsU+070xx9M{jY|)=Y9d!w#5UA~Vy)X~OQTw1WFY zO27-;*;gxy3?vn=st4^XU|9B6VF)6f3v%P3fs4RS{aTEgt>w99*vD7Td23D@y0)r8 zftujWQ&!`XN_%nbzH^kghtn9;<9xxN7+M+&y+1nFV*`mas!>9m?gfj@GnEN&7oe{c z2OuBtKjWf?Ssg-74w)satrW(|3dwLRJWoBZ9b`?^7bxwD%2cZj&VT9@Juj>tU~Ej2 zmY7!PwwXB?@CRa1k)%-2mRh6z=}69iz%wYyd>!D7;dR!M980Y%&B$PaXFlo3eOkSr z!RO(H+Zi2iKjOlpj2Kxfnv`MXETDmN-k_Puj)TOcJbzEg)3 z5NAG?S$_kqtcfE6sB^7Ej`l!989bpL%}(|Y1GfcOYU}hikt(Q{lSyuFO8^UO2ctHC zsu*|Q?eA%e{umZ!)w7j=yj#@Qd#V>*9!pG~6E%=1Vb%gW`-Z?gV%!Z5dM>@!KYu~g zKWF7qQ~4(Z(@+5qjDO7&1TDHN@6XYMy?;>aUVTQ<@`LsvUW*vj0$u{Q-(H-g)fhGu ztc#`1n>Q7N>6DW=_vg?4>CsY{ z!;tFoB~8{PE$Pq^Y#~!xCTVGeJC;4QeolSb7PaHy^J_*}k%xQR>Yp_%k+2N;W_=i~1;;W|~X`&8n<6vH@=R4#>YYQE-DhBD8=R#DMVFApDhSY1un zl6{ZiBQH^3i=+}jvUQ;0D%IAqMM4O?GNZh>gL@z;Zr#Q6fSBm;vvZlvg`T6$&3I^+ zzp)R-KoVU+8%GHe`Rv?9^|Ad2>Fp52;r8$Xv{244@ay_mxn^Q~c44l*qL0L3E3xXz+U~{1MQAQG1;n;+j(YP9>9` zS;zTSe*K>^Ze3~pZnj)@Fkg#doyfIlo7t`{aLq=WwHOI$Ji9^?y<7Xulhrh8!r5n- zk)2ABlI;?!^9vF`oDLnQ7ZFG$Uy|RG`agK_76u(S1gxo0YGtcCeyl8Ao-aQAsMQsw z$d+I{$IaCaDX>VDcMvkTji=yd(kx+O%cp?79l(uECh)ZN` zJ=Ej{ir2kfQk`=?h`>66U;B>tUwM9)vHu)C=7==@Y#|6=xs%Bld*FQf;LE0P4`ZQ` z0{5$Yc%hD)n3qt)L6j7bs!xH=ilyi2ibvci5muPq#O)ennc)ph7EJ(GZxTlnUXh3v zrE_)t{NlHMllK6}&Na~PL>l*VN{+OSJnPrhLFTH%q4(PlcaY&PG0WcdpWm>Ra0<73 zZs`*B1r3L~B-p*d<7Fu=e1uawPRC zds~xZgRD{eHt45T=uM1ASTB4zlDVF_PQg!&QaUYRsG6!j~=P zo5VvuOWYcyed1L$j8ZisK$hnn?o8nD9K{)?qwzW*;L8Y8psVs#YIbEiKC@N@LnlgM zr$-EuY7AvH1eUFhBIwU}tlNGhpG=>sbUMdHI|Su}$7EvL3qBJ;xr-Omw0brXwqs|E zbrI(Qa0f#3FxWW?gIc}sH^mWv%pR)PN2X7xPs`hXd|gsTyO_d{-xyG=kz>R^MrHbV zX`qxPD39#0K-%$+>O&?rZD~0J)PF6gKQfcg-?ULKi${Fm92PDoN1{zgz1}7Bt9QY_ z$>)QAr5Vh>g8W$FLZNLN!ZEM+e#GgGJb4LcM^lQ|h>5?K5@YIBu%~LrHc7IcE@Mg{ zyDe$2ba^%^#-=^L8u~>vH(v}-BZD^~@KlCDuX=CB1)L+9i2LE64+{40tj_S9B4MOW zOTo0fIGYlMG1_S;@2S!(xWXePC)Js%TENl!qZEX0I~almv)s8fSxU8%xb6*Kbq|_M z<)(g)HCl+o#m^V3BUhsk{1K=bxT)80?VSwiuyaPQk6FZ`nkB5A|5VvpN%1+$A(r-` zk!4VNj$HMX+tmOU7EP?SvZ{LXX8a6+aoMC}hN&k!6s-RQ8f4CjK93VAr3=;n=w4Ew zS={m%C-W)?6`_`CZL3}aujOi=->ExL__|=ZA$X5H$U3wO`EK7vzgqHJp;O`-nZAX# z;=yXKX>(jnAa?It@~l(_yb01nY;@9jreof`o1J4pQTHZV)Jf!5y^dQUB@OnDGC{7r zCw#R`#S=ee%S*hxTI8sNiSN0^67EENf?;@N18g0y5Dgs4E9Be#6JpRvTE1F*L?oh= z?+0hvkeOc1Xy$Smke z@LO{Y%S~rQjG9#cE29{Qu4{c*{#Q|9kN6QjCS<#rUoNy?FQ@Ce&-ix-_OHD>SFRw# zs|o6vLE=%Al0)%63u1olaO=gg+Ii5%)aYZEDl2*thKu-Q(O+l?cV^%$Bj3ok1%cG_ z;a>=ZsF~KpNZ6$CHjrgwz2Oa@1w0{cq;%!N=B3gL0r}^3C!vmr9UuhTH1N_GP6YGK)>n#!o<$E-gwq{|Y5Y5Kl4QF_zC zGyw)d@=kCuan5pa*>RVcfhp}H-ZZF|%;}H;F7U2rHh;?6sqRzY_m~lW`&flnn^!aR z2?bYI=Sk^EAB}uNLJbIVIlPLoee3%9j+hs-Jx@uTu$5$LQ!;gRoWd7F#g*Uw{(b%! zM_A`n-@9JHDm{3+7twnEs6%qG3Wslmsqko$vMrhz{dlmF`3WL75J1C3nWLBy|*bMe`_eSFA_L(Qup^2 zqAQ1M0saQiKYGHJ4KO7!Rv&Cykzgb0n912Jz7Ee`Mi+?cWver-walGqW`sEGRSs~N zaV$b*z>B`2Xa8gvsUEbT^f!Wvi#k0aOYCR;y{Tdkg&LDxc6_7ObpBfZ{0^??e6;^9 zV`cc|#ij(}5#ZHV)RNB7?n8$nctLNeQt|n5Y<0}j8(o!O4x}~S)Tf!ab@3HsdG|~% zkYC5=a%{i&Y2w%}Y~j%1veic4MgL<;sf1QI7h&;C^*|wrqp< zR7lhq!748!cP{gd&1vnqcea}l6?)0f&o8we2`P>*0^W*nh=fv1v4ncF-{&kzWFoh0 zn&|zvXksBF^Dm_Db6OtdY#%@D-8!sguB)!~jxH2=F)Ww28lgPc$g3B$6~Ta-o72YD z!YEkJw^JdrowG9-Ac1_bl2z4+*A9yIbC< z$A^}dS0L+Dis05yn=ZlXVAHn8%}YPfxLhO8z@cLtCjoSa!F~AYL2{oHG(NF3y0Z96 z4r?qH0y+1kY5$H^yIfC0WEOAn0~tYH=$HKLG*NP0uSDn%_5>@Z6Uba-1OmOaVd?tl z;VWD{lAsG$3d1@aSNM#Ny3{}qiU<|<-@3gAg;fmWmc145X%Y{gf;hzrWQ0FvyiXFM zytyddzkoGA<>Aebn6VM__zV+F!D<|o>)^A9bZU~ih9-s<$Nq3S;+B~MaViK-^P^zFdlKBh93`Xkvl);S2!|+z^M5 zP#Cfxfk4bvULcwmJXW`F;cUYo3ir~jtA^dJ(>h*AThvtB+Oq-8GSmcKCsw-y|?;_yEBn=T z2^h7B5ShJK@obV3AWDLaE$IfxBRR`9!O zv9zn_k3=MAF3hytxFgVb5%;a0bPKmTRuion+tq^RBTs5(NxwNc1HIbNR>OdK6^1d} zvY&T`w=?nV4}{b&Xt5p7*FO%H_Xg~p{(NZtU576)Pa{BzAQv$!c#@foAY16amPlad zKnWF@gO1tXHtIL;80JyK6q@ucACIn=#;f+R$H|<2NM85>TS%n$90w8(Y0TPkC8R#9 z&ZpPV$obgDol6_{b|%6)kznX`jjnqtMm*c*Ffv83^)6b|sIZN}B=kWo?fjk>k&R%Q zwNd=_AWV7G&WPiD!tqt}@>*U76e*SQWd-^9!4Jek%}xD~b&X-sYHQJ4)@;1iit7X@ zU5#Vk2qeVKiogbUhggHA8nWqjy=%|6G(|m3GUb5>s z_@&4Thk`XY^ON<*rK9kf71+KXpxC9^Gbr9!Up0J!C*$7RA?JlX8@0OD_^Q@fVP3!H z>$L2u{9GqKr;U*;QgkA?2Zv0(Dd_XV_3Hwm)3-Ev_UDqkzgsB(sr+aQHZITTu)H?5 z-c`2Z@9bfpDkpBoDkNXe8lPOI74s;3`|wYHst~LtG;b)Y_6x=ocDR1mo|KiM$_L8{ zQE1IxvH3Z6|D~AMye1>Cbgqp!ZmG%n1{Ke0rx{MAxh~eRgy}|Jp-|bFYOTshKwDNCq3c3y!i~9GFINzL~T0f+ZIEn(MiX zUx}qPEN5RN8Z`Hix+lQMqzZg0HLcb$cp_@zFrSik&><9-^*n!infz74zA^BzIirQl zjH}i7$Ll?!xy8}|R(<@OsiF>nKCRJ!iK+1!Zd+FJoh+NV1P-XM6caglPLl4aNF-_k z8DW~&v2hJW$(dVC#`3LtN%JG_qe}Cmk}{f+%}Pre?&-Uci*nJKtSPZSj&Q#(n+FPn zG)QLmZd-PE^e3R>jk2FihJSuUAa=b}rpt=2tp|?qrk12HiQ$_OrWSjv?-@7HfTNbB zR_}wiTnPo|JH_%_B>4B2%|s8YPRC2*;Y=+C7BmC*_6VO+JE)$FEme@8z!|7 z)jJ9Xhfa97nom4dwxSKtW=E3~dZpc&>N!y}EaGG3MA7%pvuh6ig*;UP>HP<9-o#MB zS5HK7XqUcx!+x+ql&}v8!1Oa{RAtS$yPeB6f@f`7Bb77m{y)j&r$ z$sKW)8Fi|;djKOAAW&(N7Mxhq)zb)9q>N=qtZs@FVWm{AmR0&WeOJnrQD03e0c%(& zb+Y_C01e~^C%K>OA^YI^!<`!?;!FD}fxcH0&{M0AKnwB8yvA9VFF|wrMXV1X!|vOq zdL2nqO@w`*tkxG&S4`VXBc{b=>BVjFu{iK|WYLeuGZu)iX-O=*GSI zsZ<>XRz}kr4K6zO=BiuYnaEb#(oL=CNjoN6pVaAWV?LFHz4{)z2E!Hfp)(hs&}|`f z8PR4}(6dyLBlk%SmyzWTF)TEsvC^z)IDxE-sh0SjrWo62khoB^Pq^tlF>vV- z!{%zk1+!vIlv8ldTeZ%{svNdBJX3r=rutct17Xh?>|YM%{^z)=tD4KjAFyn}%pN&W zn!qAR{8K}53UC~ceycR-f)VXnLObqHZZ|i3oApjXq!F??toX`tyQ7cV2<1^Nw>Z_R zR&eIC>Zm#jQcRwfVPfOFoB?5toE}Gd_o%IR45q25gqj7|DkuSt<#w zfgM@&g0*HbwMvr+TvW>NAN4y$bd=4Bkpy3D(X~omSC=jyCgt1&HQReOy}3xgi=#u9 zu{gnuT}MEjUhp1GO>7l$*5dWYWK2PdXdz&Wxw#u?9Q8AK_Qx|>bHbKMNV>4|dR?Q= z~L@L z&t;4I8)-5FUke3EFv%KqX!joBn{So3(HionTkqew3OH8!(snP|SlY6t=rYTc;Nzbl zNhi;KKZ(Rfs`Bd&Bs3^>eupvLp*51H2>zg7+v3#H!dd(9Oi@#sfe#fa^hy#0Mz&zR zSdF(y|6s-cj5!WiiePOMx*Jku+ZPT`wrwBqr*&4`(7O^mR=x6u>UK!I0iS3Gfgqz5 zlKt5&R~mn$b>?c^m38%)W9)b6M=84Lh1+<*ERNPtET7vk3WiPR32apyE*7ljDx-A} zT@k*OeZG^clw?IITEA1JEgj#Kk3vbyKuJ=)UV&6)WhG6K_2PRhmpk8~Q`zoX6DJOb zcDHQ;HMt~;*B>wc48!Y$8uF(k4BjF(io=mx9Si^K6hroMW7o8SL*m zUY<0FQo5_awRsQOukF?eG2I~uc_Ot& z<)UMw3(NXptAf9uLW`?IAxvVLsosD9xf_#y#u0OsbjZHtUEX@5T>3-FrKu{@-VQED&ilbbl;3 z{YL!>)d;n)(tbt4c@Y}wM^?>8e;!xQq%T2K-|tiFXsco( zNkYN-P><`p=^mIe2=?qiLUznwU}`t_vc zJo!qwgAfU1u-quaK9j?|JxHnHS(H`KDq(N7^t`L~Xr_mrzeDw_z&@43yEzUR z2}<32SpAdMi;WXqBC+$)5YS{BeQ-7&b`2N^s|)=-@QzT*3xk97{`GxaJeB&HGaWHG z9QhlyidLmEUH|Tipa&t?>iD40Y#3YN6wbDuc4Kv_@~Ib{l(!;`y(1=?`4(N{JOS18g_bN}x7SOIyhc%E{zrb<% zsQ^wF1W&PP$+PmfY-A*Wf5+ zxr6{;jP9QYVXuST2bPc$w|}$8DnL+etQ#&p1G_9 z6H}sAzH#@p^hJL~+TM+?mEPT_jtxeRh-Vd*8r9eMUo2BVcV>yUFxA9m4Z#LJYUWxy zxIZGp7nQJca-M5PU6izFTTJo3-u<#@lh4wtGq(*q&R=HNtgdVIOh&#H-c2~Zh#Q@5 zd0ZR}31ooa=JB_!{h;=+rkf`r5?RJNMN`7~RKLKMw!#buZri+r;P% z>Ki5qX=l*;6aL$w2ZXZtl%?oXM$M|&hWe}26poZ~!KZ$0CIenvXK^9Vbu@MhO&R-Q6o}SI?Hmd4r2TKJ~?aAVct&EVxEhcTMO{;_c@5DiZ=U6>(aUn965J!qPsmfe9@LkEbbezaWE z%*l;sLAjhw480?!JPHloh3l1t2ZJRK_ezUy#MLSKG_gdrp`-u4_Af^)zSaMYoDU}F zMRv~&1%gjSa8$MWxZ1Fq)hu2mO(V$Q+1klZdLS~Jd2tXH_XRZL=SHguD#*MPE)V3W zDQ<3XPXGOkYN&`Q_ZEK?F-7Vamlp?luc=z{$=B)!z(U^^cHLUk6qersmcy#m#0M_d zmxGoHN|;=P8pl#{HLRC?BtgHgT7&po?3YLr0jt%h0AjtlGbQyw3H!z|0D!f<$qV@( z&YBOi{^EU5U=o0+V=PkJMUnBp`D5h@$9TRAOs0M2-)iRJ_Lr&P$4=$Sb(^)ew+3sx z5GFyx0Z7-YY9g=mySYfWIHpZiHClllp&;y@eX|-&?C@iy^rq?N$Lk9L|R>JEq` zb^u(DJEu=>DG`UM%((NaAcf{r~__#n$k=?BoFw9O-Kix4gSg4SQ-xYN7L$F>>a7%-M!_NYG-PwDfRT=vqtX z+SAd3D%bnXt;?5DTU_6rZyx_`zCRBUJ>s3%FbBrU{_3lJq*&GY$RG4bBSI1I+L>_% zy$iAj@a(<>l(E+nH7c)!e8_%68fW*C8oMJoOcD5*ue5}EPw86hD!r2ui~1ejY^ z7a=RR9$9@9wwd^}x^-e_GB~2$eBX zDQVMTzya4^4G12Q7k|}tF@bZ!+LGp_vY=~~?^~80Y1(M&u`dkG9ZC7%U;vi%aNr9c zEvQV==SZuh`(gO6v%HtKL{QK3hZz`9f7F)b^!TM!Tu#Clo89dH#b;DhWmk4eVP#UU zIgS_mke-_S6s4@o6610#q@I{T!mr1Ybt((kL~k}u$vUR|1{yCegPaLd`7~U+w#7%{ z5&VM+S7Im}xTM9$J9pm>h)%1?O~nGTF%r?`NInCgA1hks< zqf0m*ZtfGQzIW`AAW$TEAC3?`+rX&~D#GUAiw{%7;l~;E_xjTIFSe6+(~fb|*b4`P zMXY$M4fh_4{>@*Vp8GzKjIp=F#nM6D4$>np7K;W1Oui+m_D+uEbvgG7ESj#BXI`9D z@Cux%CmRV6dDMMMFk=L%#kLjrk``d(B)rIq4$?=eNKBE29IhC~eD5J?bUO}QGUYmv{@|jEiq!-Y_Sp6(hcoNf*B{J!bu$T4UldA;SEQNI9OK-45j#!uK zr|GC>sY(3*2u4Xa`@}h3GDs*sfWW{>|LyGJQpBL)IvwA68I@PqUm_rSE-d~}U5DRlKMO>JHI^2?x4Rlong&3@ zP}Z5v>{~;^i+qEIlU5RrZ>=OG%g)2jnKwywNAbQd*Tj~pUFdBrz34dc1r|E-{7-fc z=mTKjNU&so#EyL}^v4Blqh@NXEcUG5NO8aDU~x^i&5mE<_V@N|1>#ngAVnKX2f9_& zW8>i=xbm(E^x35yUa^i9#@lJQJU>Kyu{++2{gB+Hml-fSpGHWBkV}!^C*OvRSgyN;ql_^k$0TE4~{$L6cpUl;h}b!yWjYef?JJe zAatGLG_>u`ellO{)$8EzFqvDM6{H)1M;dh_{c)a>ZY2Z7sPPFB ztceIm>Epfud@P&BW=5~J9YOt9^3tm}Pi{&t)0VhpFoZmCR%@ceLcCregrQ^M1kDJR zI0o{#I9$q|`RXFe6{@9`GUR{D6yQB7Fi0nx^4z?{XbdRWBimE^@@Y|zo)d;T{(Xjd$8B-~BWHv&hh=Ilz;?KPZCI!=CFQ)GgxOO;D zC*Vay^uGDtJBi71S($hL*k;SoVtJ3IER zXeZ1LSIKwb^U+A+8-yrCa*Ad#BG}Y#MJ7zDW@~>$KC*uGH1zQxLa1OAxZX(hJ@`fD z7l$-Dkkxy@7WcBk_Tk~%{@rfN*u`GgLgPXC;rP#7qW@hYQfZ~Qv~QsS#FnkLCyX|d zT{EWetZ#q2p9TeBYBoPww$z?k&F9NvgD;Z)M?k7%KbkWMK_?Rl<=n@+-qrNnSP{C| zQ!%}2leM@?Gb4OwN4oK}zT@j7%Q_k$)OCG&+Hp|ePKt<;59{>=OD5m(U4@Iq+96if+dv+=0&e0| zdY7S`$c@oR#0rb)2DW!R|9vFtgFxPb^oZ7Y@HLm7)JlNI|0XTk#WU^?B)k2$54>zl zfmNtnUd#b&rjmou0ECzQ-qY~z%5R;>4kpj2h$P(&=u9KuDW*pOrsg!H-XE^oYSaVDpG4GW-IawKDpk3OTU+B9Zx|YMDp-X$G;2O`$=;#ldrmLp?41T&Z85u~(rUjH`BHx4e==!n~t3s_oj2{w36i;4Z z^RI*^rPmwvK&sX~K*me$ot`&}!bE(xES#oCLSS zPZvsiK`b(TywLLePQDzFvSF8+r7E%74|VY-asc53go!1fWOa zH;v1@O^nN(v*gI@Hc|!=yw5PhvkO86^Z~k6^V)W$m&tG61LLz4r6>1l5Y|`Sk^TgQ zIUfSJs&;XxC48`&C$j%EsEa$ zA&1$%V@`9Ey~b4Q+?RzpnwJD~ieSTZDz6GVry15@ujwej)Y7fofC-XG> zFdv+ex3Bg+H~lgfdA;8psxNzQosbql^vf-E3*0oZ^-9v>`?MevVDH=KlvdcfB6=u) zcbuoG52tcITTHnrA)V7x!LaC!1EMQL6HdjM4CiZ*6@Gxm8&RJuofMQS>jqo?l|4v! zSmN(z&vIxt+?&p)nc9Cju&7mp_h+A7S&Ru?9gnu41t2|lrxxAh5=BX_E-tz}Ko{PF zyWx&&YaLuBGE$^`xZF6%4t3*=GRG0Kw5OyLGsg-^t4N$!z8K4VWbV zGT;_Ub?25_1Z$@HCp225K_O7K3_dDiMe>C?{8WkuqzuV|)O6egq7Q$X47VS$AC-XY z!HSKkf1S`C&!RFe&h`?Rf^lV*^X9bMGaiOM`JD?u%T!GFH8%N~Tv`33E-#g2*EVR_ z^AP{5XDzF%S1=_bZlTgXM)di;AdNoe(A})A`-=E#>OW^rLb?b1#{cOw$NsVP z#+ON_w8-yiWU=))+9mqmfSP16nPAX5L66ykyHo`A{$y5_B$;W8L#{3ZOF*%et2F>{EE_Wu%_*}+7E`Z?XDAKYyJIi}v?H#om}^Yv?} zeDFH??M(6roBJbl?sZm3xms~)|8p4fb@2C$^A=twMTIKZ^oKBg1y6VJaJq!$SKPfs z*A)DTL?-heT&lK%)U{4nj40K@g7_1j)zY`1_9s?c++b&t?yGuAhu%qldvDtV|4ClF z{qH#{t6aJ3tup*$wG+-@5BJ}qtX?yID*HwcG*nh}a{kJ9Ib7Xt;H<18N?M6ZSR1ZX z3o+`bTsJNwySBbAKUKszh~QUE{exo6c5l;CP*Wvkz7eXCF7%E($19Q|GLrF zP|HuBQ{HEO{u!u!l+O7(x%XEUzhK6pJ=G<8TBtq+LDh0qXebrtPu{BRZX@{F&99Ff zvB2NCz=I;e(BGVnIu=15CP2;nU;T2lyLNy5@+;TwwiT3WART?qOrmWTFvUXW&G1j3 zSQ6HM0EiL@L+o)a|Ku7E(h}AD$Hw~)2su=pzf<>qwfyT7PVxsZ|G-rQ=J+Rt`Ct9K zek~gO@A3QH^fo6d;`6^0U zDp79No}kOpf4r|`gNC;?f{!P7)7mPQuG}a$ehI1ANB`H&|AzPX@&5-S@*C9O$N$|R zyD_tiX|+7YZyEi(`JR?@&^YNF&j@xj<_Ua0j3Hx!3Ooqwg&NKR?~S#-#-|NMp`fhw z$cjY=wI!G_E$`q{IHSnE`y)fJ2O*?E|GR0u&6%(g@gc>ex(K%WBgp%o*p9y&{~ABw zQxU&SS#fvb0JUkVi$EePX*7>h(8ga(nKPd>OSXrKgp`C#@rVY01oFlN!Xg!@UUZ?H zRkN^m*Us3yb2McPQ!-3*G7B^-OMN}askYd2L_;6N(PEoxhmP`?03Z5F1<%`*f-WP&QQmsj7+RZtfBI$-1B zr3DJJLGBrJ9@|r#3ANhzD)?TFNR}Zljne{1&rL0s7g`O&STNdM?7^OR8q0ejb*2Z) zYg&3G`rW1ic)ST`UDvuw1P@BTaIUSd)mPoUqGT|j$6thp$*TE#-?buU?+(+}o?a8d zt_!T3Ym&IGX!qLtNW67!%CR3jKvw;9ci zjw|>2I{mw^EwSX(&0ndg3k@Q%saZe5H#I~~n{bO5>TxFsnbSHwJ@K-Zw-E2A!Vgzp zW0Q%OwO3{ozdr1pV&LF0>I+Z(TInc8)k1@x5zXv~M5^T+voo|=I z#Zmc?CV^CMR9vRhwn-GR2#UDwe z`~|1BZAaq78B{aX72T^-83PSM@*Ufbj9agA#dR9!4Qq>brK6?*&o0jBBZo9l zE|p5GUBHS|-*+r?QFR^`JG5_IInQ41uV5@mzX0ArVOAl7rbr8jt}bD@ylxoz+TVJU z)incSrgLIXf4r6$;9WsxHNn=Ma;&b_a=@a2w8F-*rwZiB!1$@m8G7LFnjqq>2V^%p zTS=eO*V;$Aj=x%r;go3u@5m8DBv9x^!cXMRR_<5AS08@*B*bd7>??B7F0Z9Gj=uyL z+LCm0qr`b0^|IogQcd^ba*k?IiMyo82V!hP02xT^0PB=DqrnM5CPuGdBw#VX@;U{l z$jekIaYJ&BlU*Rh=_pxQ0M6B)z8Qe?cX{o@b6mF~tGkPW5s;gNpz7jjL)*f1aVpaU zjI-UoA7E6DKwR|V21Gsgrr+V^zLO{OI3OiNu6tSn%)P;352x3cCjT77lS--!18m{3o=-Mg*s`}J{dXWdC5q4{|2MiN^epGu!M;<B9J+;=eTL52fdCy*!!Y9YD0TXmNr^S5SIUeK!K^It< zTa51$!tQ}!+3rBUD1roX_pF7rH3q&dHWbVDL=w=?ovpXP&(2}^%j4MND{w!1SjmN|fL&E5 zlYw%<g;wuY*b`mZwX zLE{jwlDcKn8IpMH5_O&oSDG#&Yh`nJpoHjH0Kvs2#az@Fph)uO>ikQ>#7bA551rel z(kk`!gBk1D10IU*I{iW=r97bcD#i1-GXsf7M!@7;!;WM7OO#aatCpoVk$ul6MYUmI zJGJ)4sfsUKWTSh_jBvaSvp+?*tn}jps!k`NC0jH+yY6cb(GR-&-ejfx)bFtu_g^>( z)n9T%1haoVt-!9@9G7xLW|ja09zxzzc2Df3f6fTGGh=K!xAhS`8c>3dk0ufFYy#~! z&u_thICuekmw3$tRGd29T&A1TS^Dai8U;Pq&m(U+eJNz2txOfNQSLR z35>;i276tasr-xOb6#4W;Mc2?ih8H_;JW-vZf-al5x{Kw%srez<5}l$uu&P;V^thN z3U4H?2m<$f3i}T40Xd7Qm$-bH&6O)K|@x6BcHu_kb4^7GMqG`9|xD?UOee6A}MMhIQ zLi~Uv+RM~^G5eD0`!O2yLvp>xhg0;Y8yv~CNV2eL@Ugwrftzok*Qp8o$n7i(U7B2! z7=byMi9%~qd7XN94XI@3Ib6CyX19EVTM?i-9~9!?%ji#<1i?*J3IsyJ&CZ zxv5;}Q<+|;f>M!OCUwN*GudhIcibP64Y5?eY)ER9SiA7cBkBe(D}kDGQdSC`IbEyBzPQ^r|OrbD;}VmuyIa_k^X`U-KNIbsHN>ZRi zUp-h*))uCih`i@#l;2w}?tMt|@ajc(jSqHRvBfIrMx;#9s%-irvSP4vIU!s*MJ0q4 z(((!EZ~oWbwF2G-M{zBJENmIYwaqB5m4m)w@jo$Ou@{KYoG1gAOt_#Ph)4u!a5~tX ziGU-SzDF+fz534Vi6#=f5~IO|IgxO;wI@BYVJvzv0kR#?qio;5@cx6X=&v9}id;_nFW8sjGx(Jb{7f42p_XFK9Vn?@cGgAaOJa+;{b~Sv2emGa*>NKb z``|1%U7+G}hjqUyeX@Stc(!R-VfJz;uIQ|YXj!vaqEaM+P_ay^Ya9EkNf&n`m^P;- zmC}ljUm@)wplj{TwG8pd$m^y%ml*QqZR{ut)X`f zsTq~V+Cw-=-Sc?TUbWQknPF_J3y*YF*|UCk>)XiQ-Qhn7L0lXd7UxaJtXseEI|wAQ zi!A062%%hlGbSvYr9cvGGH0o4g~zQRO#XKP=x~+MbGOdOhQm!-P=kdWAEHZsxV^+J zx<0?MScsjK7zLhW!7Sm$UL!bm%<*QoZWnyZ25SkM?lR#u9(HUe)Fa*KQC2~HmBWs^+76$g z`kl6w1CJ~L25Cj!+}0blgU63X)&#%OoK3fkwoV56Ma9(DPhzP9vlo{+f1FQP7TKfy zo`9V+isqf3XJ82I&n{86E_iL=fj4~cT(9n>1B>8-pHnHdpA3G8=vVVhQ%K`^Vmx@% z#0iMNls(j&^@HG&(1(@s?stb=)^Hw&fQ)*0=??a0KehRidD096$;l_|ziSF6nGDDvdp*M2C)PbWBuxN)bjy(1#!VRURS92PDC zm`faoyuUgU{IXN@Z8`}*tMUf@ohkge$f2guUbJPdz;2D|ps{b-*+G6}(PVMU`*XFCg;HI*c>Kotyz@$Rk?vd3M`D`c_x<@=^y_Xquh-#+de1~2)LwOj0l!M(w|)q=MUf1tX##_`+{&w2=0cA_qBpE9GbPocJ^@=nu%E zOdNRUFY?d>$j4Gkl{A$;-8sZ+=y_j;9`prqt>SxpB&kLvx9Co*V!_GwV>(cikMcze zI5JRoUJ7XUR~UDP;O?h5bM~^KI*5b>wSu!1Ar5A4YIcWVp6N`b))!@(ZQ<&I{*LBd$iXCxxoUM7ro^5RD0f8Tu=e8oDtQyJBw1Fu7jAxe75!=Y6ZhQaqV&^S!L@h-eP zY~x$Bl+!dh?A~btf>!O7wyj)pSgPgS4`d=6?zquQ0t^hT)>H*N z>L%yaO;xK^w__XLfQf_h_V_|$lg~gs?K{z8pXiwRvn`-wT8`O?lWT$;V!p6>OfS_# zhoqUq;ccm}@3sOxH$^d%X4RtZ+}!+{u2`b?>?1Clj?o?ToY}|AbRO9v&Nnm{I@S^5 z-GCb~ z5$bKwlV%c}@=UV)DlEYS_4IB4cCri%v&8Ugx7*Vb! z7U914xj--n2R%6&Dz-6t`-g@`6Py{iaOJIiP*f$5ijw)_ zHx1Y*ch1%s0a-}4*(dBK5a5IAMccPh|hAr&k1}Pxy*$d6lpBp022+3r}cls-|?hgZ8iT01YsEp^^;VeIK8mwRbB2UxlSj=QvPgh4jvJ2nY^ zIfNM4V8-*oWHM)ya1}#e$?%2wyp>^YH~qkF{(tp}hSq)#FDfoBKJ0D>0IY&8k|kU( z*()_68KS9rg^i6m`&TOxgh><=`&4N=ksLz<=>p16WpExm*c=k@zdDrC_dV>QB*Q?p zI7A7|Cn~hZMx6?w_QiFwc1N9MnR@0BA zGx2Y)`11C-VESMb-fe4}8fglJZ=b7{PW&1+-%3_DVP2F{LtFH*6CL*#<0(cLTioBT ztgf#9eMN6?@9O^k!v9?#q>+;98XE4d45-FlfRwbg8Bu_(Cqs*VL1tz18!`=_s9%sp zfBg8dsH7x}9fGQ6)(&FNn(JeJ??c@l_+!@1^ zUxNfWHv0z@&y2!oAph$)vc}r`RWkCi6nG&_%*<U3k1qlyhEPe||1Jk){{!lZprl0ck01peYS??198pUH++77wt-L6- zELEE!zXUajEhzBOk?TH4oG7;&VxA|+$lxdFA4 zp%du?te}q{>v@TOVURMT>?i(RsYI<&MM&tcc3p+EqR+ILSXi!(v_M%|*(Xn)d?z$P zH4#9J0w*CUsn3=Vv~&Jzv=I%upr3@RS5HB45QFO1CFee>?g4sSPhM8m-<0D1%>$79 zpEAI=Grf@#_1lXL6f-Si;Dt!AC502{-?ki$VuA)eHbIIg4e^kll@7{JfA)YTM2{LR z>Y}nTHEC&SsgT>zc4bBpVM$q;E&Spa6#GEn&mM*Zhx4$`0y>vIceI0VF?B!}*fI29 zL~d>K&mb*t3NK=yRg97|xb{02KuAA?O@4&t_<8X{!%l+z@D8Lx-C$E~0N@z*iVmiG z{ZS*{piq2HRO7~AQslpSMxAZ;H;?4GFL5$ibo&lu8eFROBqr#zI(%B6rK(g4q&+Xq zY|B%1D@A^lOForJF%k5MU!H4<-#F3#Q=AS>xy3!_?U|%kE8Nh&&(r(vM6)fNTtox% zu+0Dco8DQg(RiyRT%sq8+d;XMJXmA8;Y#u#zn<1q6pe`z5eDDns$zo+vhR;HpWK(61*P}pIu8-I1TLKP# z&1QUOw&A>pK4Ahqd@WI5$gMZKIZ(X*yhLG&i)aU*RGx)w=(LLBhO^Q6KuUHYp)l|1 zFjpg<)g?cecQr)-GYRI*t&!zd$L6Oe#|^0&;!tCR3=E$I@Zsrv&ITKbWPQX$Fbt2> z6(;&0Cf*2J#%IWest3^eXauWFz|_WK_hw!SZ55Q>=1;t^lS=IRUc6xOUf~TiyPnR%*M@(ffQ% zS0H45gK`G77&4Rn-7r4E$*jZ*6nxwXiY$Fm9Nm(CRg7=bVb*4$Q8p4aVW?4pjP!OM zR7&`=~}iILpD~l ze&QV1k(D6|nYDtd!$v?0l+eXKHswkKm@te|=tf~6RQ&Rj`!KPCRem|lR8apWjFhk( zxW7LFD$L;(0W?*Y1~YFR8Ewf~HZ5b!@?Iof!dD}K$eXQjb#FnR&no`h3dUG6M@jkR z^o_K|NDev!Q)M%0-At_<))`o}0ry+j(MA>Xw(YyI)FWZC=ihyINj2HQ}A zHp;>Z7qQh{*dCknyq9b^b8IawU0oe#@!Tb@5WhGsg-^ocNMoKN^cVEp8|%fE*A(LJ zc}%4W)({Z(j9c~2Vm3`@Z^DS?~NFU|%Qu|jV%ZSJEHi~Mpk()o@VTL~T9?u;_ zGri;+q9e_%REv13o-An21w(2K8gmn_uoE^+bN{_47B`hM8iM^HBDP1-cnO!vCOY6R zI@&k2*IuU%C-C8plt-#QaQD5_79k=*<@<4EK+U8OUDpYUCg+U#f=J92o>EG7r|NGVI0{S5?TiJt2i+)0q>p z_vrqO`={j7goaYN_HAfNpotumcFP*g{iWj)Jey@$_l}GUk(6Rtr$W0nkGnJccf}np zy0!q2lw<~(M&ty@Q|JMLG=#xEPnj81uD8COQTuYO zej88J$K~U$HmOdPMWJ(4yxjUy)XFhvev55ZLLj<5kI@#kHwj9=)zsCo`{#8l4pktsWm>QgSk? znLh`&0aqLwp!9EZ3kRWdthON91PjZ}0R)I~y&4R0-LT4 zY%}4l!XL_^7w|M?9Dd+Ui(QHYxO#kN3*^CsD1!2Pr|mT!_lu40UNXvn@!!prz%O{3 zln6x!!Wy@C5yx*cV7;E_xcTZ4qaV&gOFWUB3rOi__BSs^>3@R7}!)AzI~{;`8iHor*EJQVdz@ zxRV>RL`8cpEL>w14$ri`R)-W53Fy&L;%VS>QwRApw1Dx=shYyHu`xr6hR&Z@I-Ztc^=8(kQXA9uPU zMu8v(d!)b3L}MH4*j^RFW(nFz$?Q|U-l9|7L`-i}D-&D#T6jE<+3py9N^#tOFK&aj zZqEk;&DH52>G2qEBP~f;Lb9s|`J(P9f-{WV8MO>-qlarMny9qCBePnTIv599%BE>= z{M90FHaDaoNCuPC`v>ghNnU@dnmbySVXk`r06cD=H1}7kIw1whNHs-}3hxd_7oU z8e+o6R3?y6FKD{DRJKYJSIDpWZ|kGiZzvbax*+*hbEkI_glI`WJfR{~_7SE5qvzl3 z1ADhJvoHbvT8i_+vKQ9C`B}aBzulW!2V?yGCq#7w9ICOXri!}4M^4<3J>HAMPeeHc zr3Gz95{w-l+Put;cT6?VyDB0wAkUj)*4zrbDggCm#FTWU=6w`aJz1rx;k{VpYiV{a zlvO8QupwoNXf(K=dB)BvUCpq6ogh0mv3S}lZwor&uJzQ8y14O;%J5vKniHF>sGS%| z7=vPll=pNT-kIu;10EEEPnI!q^I)a$?F~B1j0L@0?ohjFRQKlYQxtFGexrD0QO#E6 z$F3^}20f5)n^29mzESt*>KcT`Zfr0ti6G>hcl(ArI!1W_9ipvP$6G3@w@Zz|(8gC>Q|JdF?B&2B<=%*<;P}TwvhdcLhe(02Mh%<<6yy;B<9_i;3;f~HBnU!N&@)>rF z7?AQ-74xd0%5c)cvO<+yW`W{ zfX+H;Lp_I|4X3uDiYDz=IgtF03EImoO`ZwF)myIS-Nq zPlvD!U)Q>2N@4hsO{#iRVM7h|o%W#UU-pT|w^gHoh7;K?_(ER)CiB?a=oHs?fr%nb zOJyNg28d7VbSKC5zU>X(Sd9LXvE5Q**R~Y-hT>eZOOGs)X|FmR2F1B~8J6p8jWdSv|j(;g6ebuB{(dUj8IRb7#@G(5%?cZgkNp<&kcaUCadS*0fzEgQ@O&i#HiAa^#xt#m>u= z#>uHwa>f*6-_+QX<(G#etUbCI=;~$7*b%C$KeEGWN$5}(g;uWwy=OV-#PEI0rmR304>?pLn1` zEfd2Z5eE-Pf|4yjhODs3fd#r;_G*ey193-Tt+3GA zO2cpA&E|?m{0Lq_mFg7 zRISFIRGmNWld3SEWzQpFRS$YR^?qP-k8^|l=I zmKm7?(G@#nYP>~`CM$tdf4+Rm?fIFQw(wXV;HY|jqQzBTuIDhnx>SN)PKU*(U?~p4 zLM3c$ydCyHv8w`Ek*RWKE$38pk`w-W4rgm-lzJ8u**HDUv+36l(b#5x60JYH}WD>DQ8(kbbM7=5ZVD+6bjuzB7)Ah<}ESff3?;KQ?Kd0p8y}@^E)`yVHMKKr>Dxua+LUdu`Qg> zs-0hXUsi4D^Qt40k#A>dySxenOFTJ(>-@6R;dg-r|@F_fRfbOD)PWwYj`0%_h%bNQXQy?Ml zq_2f>AEMuiJn*pp^Yh%dqLDZ!;&B*pJq+p-d(u&zd{Y#dve)}<+b2k(F$KYkkHye@ zB8Mpx>-t%<_~OFMJBpdjTqQ{|LTg2RHZ@*0p~ZZA7nxYWjCv?jiHWi395L$jy~6Z{ z=nbE=SNmkLj+O^v$5$q*6FEL|y@r$K8uGZ4e3$)YAkQ-q{U$_0pYgeZU^Y#y(YL3p z7q)G?s-3ms3k*A}jRY3u7ZIss{i!<7W|VulgkIr75XsY`l_fII+guhbx73&uLG{!; zVJJ5p5h0;^7&W&FL3bBV)BLIa0u#6Jgp@MS;HR(iE_<$hp}oU1qHXtW_h=|g5`NFw zT5g)$n-0u&DO0%W+&c{TOu+`MXyxObRMfe6(9pC%1?qDjVX^THoY~ju-g=MeRvEW4 z@BbGXvC#Id4;^hT6y*e}paKN(2V)LtH0;2ydhc}2d!?$Fu;L7%vtD+}0)39$!02{8 zN2*sC#<)~vvDX3mcyb@V1=3;$G4eSTlm%X&fKkuFGQGJ=XPB1jaf9$qP>>z}7XPyP z#NB#lr0^u;QrFYnv*I{Spv1j*c(m-Ju>B}52Kmi2H{;hf+VdW7m#~b>DRgt#g~AMf z`7rjDaf&b?>M_g4lzoz_Gb!QuvET(fuho(=aG!9Qek)CjX*wpbrmirVl^=%FP5g-BymjpDQGd7S2KPAuf_# z+B0zM0L!UU!p+HmM=4a6$^kE9b7cz$#nE4cQwVBL&v|_8=5H4NRQj5H!C#`%SI2X_ zvEZ{2JVqi)HP&{4xsvPcVQ8RPag$q5ah&slHhti3FME-SVR*ygo0*qHK?t@u*cWA@ z@bi<*R=+^&W0O9W=@b^q9dYTELWNO``+GwVkQf(crPo(8b{spgNx#nSsRP;Np!>Le&ifvaZTZArjy<@bL)C$GD2D8#uTfC`*G)^ zjo`S4$7in}v9^E)Tz)JG)IYeC}Pcbx(P)(?V zFovY+b46XJ!UvNutEqv^Hu{ZXVkX09hmRsPc~FYTEc|Fx)2d4ovw8|8$a`e350f9P zHPON=RQ5KubPiue9l_|TVFM%cE0X%q1?1HXt9WFqMf{zbQTS2g^v2r0R%I8!pg1B| z{_ac)McQm7zJ_Hwl}*<=*PBStc&mL4XWheafN#Vq+D&^p4?qrOX{cw_{&?!(%EMzpz)OVjjH*aoG1L(Q6A4jU*nhaxW(?<|j zyID;J-93eJt#mr@4`oyoI&lH=Ls_=IR*AZDwunVH)V5*fWPOITw-`qy^WxA)R^_wg zf_ICgru}F&FQ#Q3;J^r1P&-R+S8GO27I zTTG{N#T9$&P-3oqTCFv^;ZI&xkyZ-0xQkua{LDA1u?6p^A?!oRzwNGSsC}WyxN$r% z?UeOG+pxf9D)^Nk#h`nu!`>dK<5tDh!BT3;GMx=A=IZDWylfs4bK9NbYnSR5syPXl#owjC5yMbQy>$gjS?i zr?bSZPSkbuwvDb|6LmrO_;t*ZMWpUyaK)c)d9xa5=HN(tewJ&RaAX(?)Qmz<YP-m7Ti1vYvYu;cb)oI7J%7#Nce3`aJu(eG0!h|M|r%>wgIStME8;0 zDlX`qy;sWo<%}WR!XdoOl;+7)4R6h3=@%sSbBzv~ z)0V_jT}@%q&V}rT!_Fg8O=5Wf-PSD^G@1i4C zCT3`7VhS34QZ#&b>)A*hl~WCJuU_ZR>tC~@;$Q>G*Z0pa`FYpWCtl9^WBnqi^3w^J zamryS?bO2H(1>N5(u?nZ=?lBXQ_Q55I7(ZCD^(Gvy18ZidX?WN#Gt8=z4-n|VbPR% zVp6W&UeHJoW4?F!h=E}VA^r6e!6PP7$AyoL!K$tNXq*23L>|=th_s3Nl9NnUjYcBR zR-zlvmcxa4us~BTCq46j)4h!v3gYncxplyu#N2H;n^B&R1Z&QuCl`42Q*#JD9*cP)P&4C8=>6n-}t5=>E|6h(jM$7xrbD~1iR$(L(~(5 zd04&} z1=3tP)?aXE1E;CJ*gj|6N{&@wZqEw}%Xl?6|JUtsqq(9(VV16;I=8Xh7d2mGa9`|b zm8FO^H9ZbKM|*YGaeV2baI4$>L-w-F#aKE#kFMkPTYm}+3WDNZ8K3Xa{~TP zoEIJt0{vLUV|=s&HkSn>yL0d8I^UMUE;npkTa)fAiwCHww8h)ZA_oMu${Ta_njM`B zxcaLrLOcJ+1N3WN|Ng(=JS4M1!(*&&=NE91#w&MK5dSNbn+gw2AMKZh-}{IP>~5~V zeY$>sV>PNr(T~NhIbYoe^KQ?#ducW2Tu4-SI3>d~A#9hfT!$Yza(4QOl~m|V1vQJe zit8)ud}dBbG_}r2ertm~k7}AN=M1O_Zlo+!24TENdihJ{2rs#3ZjRK?(W8cAsKu>m z#_u+il>QKS8~GiAlL`r;j{hnzCf@vy(h~l43F`F!#VGdwt`O<}qmcUlx4;BQ;w4hU zHXjO=$HvA+;bp7R0QP7?_MQT6-@i_1LN=`@3q=D(8lx%xPw!hg*epZc!H~54pP35* zY=J1kjD$jm3jRnlfS?iq{=oasT!nWZz5WQrE;6DL7~bUt^Zk*u;Q48I6oY6g9C(h} z2~N~v(|D+!lZJP3Ib7u0`HwaO?tcMWs`wltn+tQhz!2|#^_3AMi@#i^|4l-n;ht?B zuFe&hzV?W`(8CaXKg^?YEVhtn8~=XhklenVrm_ux2gZzE^SPVo;T6`D9z0q={lqQB zz}q^TNUTb4Ms7xwN2oZ3%;JyajQ~qe-{-*2x@Xt0Hl$`Qm_90c)P=o#s8^ku_q;g( zO?=LIR879s+qKDXCNa7EnPz@(b+x*eSK8H#sLIUlZFpO45SqEs4P?)zud}h!Dd5>s z7yu<&jrx=Vy^kLb$Zbw6tIjkJu%+^ew7qW7WF2ZQ4WyHKniM%kD(NXwL`QIr)0flE}hCiHRv=xzS9mj0!g6?|JVmHp0sZhO; zS{udW95A@2Rap0uY30dADI2fHlwEbhH(s}I4}-8|IRi)FFh#mpop&n7Lww|&D?D_~ z9T@p^Oin>IyWdH0c|d-DM&^q=I^J5F?~&(}#1&49w`c25t8yx-63{aM7Mx+d#cH!f zSd_XICgyZx|;x-$r_`!Q_h%siaa2p>;FVy^%J;G`QT)}M83*b^ZI4W z{*m~n>G?HZZC2cGa{^xY2%2AV>GzG)#2Xr`Ep+7aAWp5qc=@$jMJ0-q=EkXq+iJGZZb7TBl=P zU!Bl71b~XPV$-}XHs|0>jkX2r*L#oVl1NG4X2!MuB34H8cOc4h~8`n^NzQz9g=ZIE1Q(n+6+8ysdWwPt#~`$-=}Lo zZf%uHR~Uvz#C+6v$+Mrxx$R*UyDrYB7F*#3Z<%Rt9 z+%wij;OS`^3S^+KOTLMPc#*iMh@#9=Nc_#bH>LAf>tMFxyvjEJ^T_M9K=0M}W?h0H zo35r*HCqA2DBDb{wGAm7U50}Adj(0r5d+{F!D$%XnpQ8qu=O;*{h z7T1Q40K%$M=BB)S_3jBr%g#N%4Uz^#CH^>L(9}4>G|PCISOucevyW|Tp;Q$IH*p-rESNek3=W$p{5-9?VJOR_-n>eSeyCB$23>8GUI(Td2a4g=IZr0 z@U37PGVTq3d60W#<;rg&td&d>VA>P;T)f*WEgxB70}G2#jzBt_?=g-}9s((KF;?iU zevof@pHDdu+EG?4IXQ(wZOAm z;FF6TUQ7l}PSR_clUnJ-Ce!;QOW+Bvx#GsF#`cD(`BrDw-=VG)JWe5X*t{zr zrd2T(PHdCJE7@7@6qfpM!r6t&Hz&B^RzC%t-UC^CU%S1cm#c-BV&Ftj%Zi_%AnTM(VYQIqkkDcWB(nTy zv+n0%T{gwo5P{Qnp+IM7F2-zJuy_nq;Wyz?^mJPbR&pQ*Q zji^dH`L<$T@YhWRhKU(g5GH4*_|4FS-9sb|*S!o+TCD@L8({FR&5JYc8P%QcUTZXo z`O1*<$q%<6-YR<%PcjRO{vT|;1zZ(d7d8$^ONlf{h;)ObG}7JO0@B^xASvA_Al=>F zt$=iwfV6b@-$(C#-}nE0-}#-(J>Z#{J$u%A)_R_`_Ke4$7|p3-zDYM>p;@n79r^8E z`=td5zh2I_V%Ajfd}?Labdw2*nW+(0tc%{)qWBAsv=kH+S*R@$FXgj@h?m9D{(o;* zvYkmQ4p%2c$Rsh^Wp=z61U`00%MNbPo@RdW=DhKct?h9_cbGnEHjm$A_|75M%24^W z`McI`@qWv1lZhTX+1d%5-eE@3rWPXkSXU8TeH18)OEboG>wU?!=4K!JAG=E@Bk~w7 zS>dh(90}!iX$m5p*>c&XHe$x2a6OLT2iH&3J09z-O&LF-o9J38LH3;x8&@ycUpcX^rchGJu8{Fax3ou`_RH-PF?c-eP1N%FelSHlefLv0GWTuMqU-*Y>>V~2xu|2AY-wEf z=Yi~%e1Do4T8i&iWRQ&NM7jGRhNergJcFIC?pJ2!?MSQQ{$?D}gE9MN8Z#VT-e7MD z*4-ZNFil)e+U?OSj2vuzX=7zz%DH^ocABIuChWtnw^y?kBV8}CN%-=9X1wGM7(-@}*jFvO_pO-O_ca-D zsnrm@V@uSxiJSk)G^Ee+ynontRjaI2+vbi^x8lLOseW3w^xfHcat*b$AucX5_Y9Plm5uWF_2X;58TWt)@BU@e|Dg%$|cgAZ5&APVK3j$V)!P=-XPZz{>T=V$=SuDV?hD7mCI#R5nT0iHujROP{ItC6?%?kan$73C zpND_{VCOskDXrD?MTv!TLhZh(L8K2qL`drKIP_xcA-s8mmu-8V^9x-rczkWWF-}Ox z-z0RQ==#ld;`CZ=^*Xs>bxIdx5{6&^oKo_-=M2K7TEt){gKm zfHpT0vmfqJTlSZXeEZe2PJ)d$*IQ~hMlGONG#Ccmd=;I}i;vda^qh+Jm88Ev^b3N4 z7=g#a2&VQqk=*&ZBU{@c{&T}+?5VZ&O-&GZG6_x%;%HagL_*^id|M{?29VjSZBwWM z4qZ}gR{6FjytgL7&;NYz)@2E`F6!e`ZcwMWif7ll-1C{2Cz#=6RU7J%`u1LW*~l|V zjb6JV>P8j0@m-MV62EfzGciv6=etwbS9{Td{}=AfsyhCdH~s;!CdRkK0+^2nEv7|l zCow$sIn}ZgPmz%)zpQ`GLPPjLS)5P@1b(WOoVO9R<6~c?q@`7rrl!YCzY+jkeepuL zKpvbL=6}b8+as__==flT8t;vv)uc0m} z!~Xb{F?{QlZY7?f^=omg4!!9W!$XNrntcb3-0mxK$k+aT@u10CfG+XrAD$#r+6BJ# zpufoWhuybasc`GnF!th8nfL#G6Y~Ebmnfjq-0!J8Dxbyw{`~|4ur8h#5jNLTJ!4G1 zzZA1Y|KT=Hl!TW=Q%3>Fe7tJDL>!+JHXfh~%UHtN;LIqV3{z$tV$nKbu{n{5gl0N`HQXa#Fn* zGhg~=CCci5ml{l3t7K}pCe3W{#S|l!J$)rn>yX7^1o-!KKkXaD&fai{K%30zh@_CO!VomNrBnaQ+?glE?)_Vg%Y`pC}WnN?8qnC?x2r>YF}c@sIk@X$Fxe$=Ux7 zdZA$LXPUs}z|yr|P%b9Xy{pe+VI0==<A%^f&p&!pZ_jc(nWre3LNB1Dh14Obv_7S{D zoc)eO`8R29zaGx?*U!LjOc`fWH}x3N{Qp?mAAfrvwCM;r%w396{+|DkW$n+Vg4&Hd zMQ?Ir=c|hTB6)Fy9iao4WNHsT0e}U5a^P{yJFAS%|I7gXzf*xHASx^WJ$_1CYa)k` z*n1im+C5`RmE4S->H-=nGrlAw_3sux5W&I-sLtSm{19@d-vXXj_$2D?`Kb44D`u~A z8Ksk!tmhjS=M0KpNKupF!MuAeEK!{j(tko&~%Wn7Hd^V3Ua8mEBD_?#@y!ZJX7JaZ8bh&&o`AEpW5ZWVhXY2OX&$oo#E?&O8@X|up6{=Ed70m2qR zG8B)-Zu{ZJ)J+e>r4uB#rdBR*mQjdacC57Uy~f{uPnyxgo17 zZl6ukzgB=D%x?p^$|{|%MtAvRTx|yAM=eHp6x07qWFESdL|L+wh3q4i9`BZ@B{O?c z&&wc`Kq4vWhKIb^1r3iMGnDuH%W6TGOK#}f#v;o{@wrkK|L%M+<+Dq6kPIA^i}5*< zKuV2o*KGZWz-!|}^`n8d2!syqTc3YUAuftY?w~9#jmYM+i+_*|@tHfJY&gZ2ANMcb z7}BKFcQ6JxI}&NAva}9gzcb)cy?DK^C;Oj^{Q?;n#opFt{_l2ybMXkyjVs?JE~5%T zVBddDXWq_hHF0{Y-k+wp_w8Wb=b-7hj9;$%zFcZ?#exQ}xw)@A)r0!HC%&(bBx4&q zz|6X#n+dvIbd2YXVE@-03YV$K@(}(zWZB|umu!bQiXpl2%D#7sQy&H6JIthK6f$~x zDmd2_yq$=|L@DW-9MXv5a-=Mbsj{Rh64#%-zq&ayB=M=qwVqqVN8!%M$as4$mYI8W z=X|ixlw}>WjapokztTIJqe6&0QLs_SO(ilfG}WPehN%mSE9uKp{_q z%yK4Y>lHeZoU><1Pvb%OdEpf$!oyOizHN8?RF@(icIRN?w33ko?YK<)xofpl@#TdC zcoZM3;iyQ614Z=^mW_&j&yWAX%%2JpQ5fHzc=P>>Bx1Tk4BDjN8GzpLL&n)C(3xC4 zW-rWyZ#9iDLtq8awv3bTwumAmtVw6e$i(B8`dwcME4{Oub~Jpo;9Y&cwmWM#G#gOm zv%J2hLc^}b_>VmV#a2Op=RY%ySkhjV&5*|tPFFNEGeGIlDTmRkAn{V#KoTwdRciPH zF@Jdi#yV+?yivk#J`wAOqMhk?7Ry|zLjxSM<>F4T3*Kj~P=s470>(Yk+D{I?Z>eMY?8$vO~?_PF`s+=4CmJpqog{0{LLkS z#$v^>fPJQ;y<>5|F4rHCiUj4IM zmOPSU=zpz<@37?ZM3k_U2nihyW?R7xDUc$q6`Ykqrl%99DJLq25F;dJp7z~*!e8w| znT%!=`cX%qlOV7{{_hNb{tQqjmKfIJO7icl4+R`l#4Ui|Z+6jnPD*sUpat~MG-Q7H zB@?M&2K{VUHAE{V@#T=_8-t6>jCH zCvND0keN}&i&vB^+dWSFGz*JJ2Wv#6l!3>C7dHLMJh2Fk3*60}f@<=Sif?Fuk>!U= z_WbV7d_C8K@MvX!@6*(EWqh`-i9j&4GiJiqmFXa(^|dlMP_^AM=1w9LaluqmdalC2 zKjXPy_T1=hz$%fp-~$%jnu;`0*YyeAm8{EGNgSo;P}jyfVl&eJ*hZ=qzY?!`7h*R7X7Ti&s5c?|N^DA76HRVzt9^<85=Q{_sySF#tYrfSpO`y?L+OzJ_wEd3 zs}6hX#P7D?)WYT6ISFRoQ4aY6?E*-k!Nd#Rx>U%5>dVQ@B^WQ34s>zPX&dM?pGLPYCBL|ZhZdIhg6$v~=lx{rchfw^YDTh;Q!*Q4W!-}Uct zuoTXMRm(Rd&mQ%POXi(}^Vdh1fJeNDJN?1|flJQ;BaKa?>bkpMk^|A(aC!MhSpQ~v z_J2^FnY%p=U!b)%)NMgoq+0}vDsl;T=PY?dWS){L@~1{WArs>8BWb*{@pO#1yp6c7 z+x`k7$P!qaOjMLc9F&vS#VASFqbdyl9p1rP6~sQ zz%zH%Y{UFBqJO8w8Zs@hLUAIp$}~%_leqjzzFO!zP)EE_lCXX^01Ut%firkzV&GXB zQ&8+W&%R}Us=*!bW*Nf=RVC@r>5#|NR|#t;ok?O|7q)dGk7i^(G*IizmWs!+>rR$L zOvGm!8p@@!yLlmVAM8s_f~N6`^Ekeb*M2bkq-!P&#rou9^%3^byZHrgVwU(@iw(QiD{o=B^#eAxO7@uR(?{E zmn<8wZurg8fHNX-Iw!DnZ3f@;WP^<9a^^!B;r4F<;t}R-FI_x|h&j7}o*fbXFWuWa zB6|1Ew!4MV$1~*oy!RgQ4tiPUQMp4*?d_5kV|tR-Sb>_j&Al$^RCipv;_y*xk3UeD|cWOF~TZ0#zJRLP>-R#Y^`& zoTJIn{b}!`atZ4nh+PjK&nxRct5{?RpT}Xa*0(zvu=%*$XPy$nr*=;_Aux`}>H@%+ z_JbXGdflhZfY_^1m1rn%IOsu+;EIET|CK=IGes;6)U*?&hj%nlB}P#A@-5xx;{2I5 zqcozWR7B{km0DOOuGy74`;-n96zFg3A9ow?4yb2+;?7uqn+m^*}vRxy1{{YOhw;<;~#AV4C1)-PFi+kJ--U2OP z@r5M7z zZjLCx;R=uNH0bs}QXbzKlpaf2yijh9OS?Vqd)vMOx6-V1zmJ_S++SUSzE~!rTu5iADsPnC3#5%rWAFXV=U87uPVXqjs=@3h)C-> ze-d;2h$4DjNY4rd^?eMdL4EnFCzq6WJWn4o4io1AoktV-RTrg3=Ed>fv28L zlGV{91QJ0hOnvNp!o=B)YjPT@d&sBdHcXZkwBI%|^MU8?y=O|9N7m9&{AMt}cJQTm zdH-f@7#U5VT<(4OJQF#jYK7*ptSPE-bU*jW zXglM*hqh(^1ezYkz!bhI+avnVJ#XFXo^lshBONi%bxL1k8nsSw=Vx!@c`bM_uL~Ii zxO#ewCWR|zI`d|!2b@1`b$--e@ICvz!hffT`*^qA&w^AsJPW5MD0VPvb*~v+(-Ffl zTAf_%HK+Wd|NO4y*8AsEguZ>N)x{-0U@rJV)fIZ0VQ_soTw^>-bv(E{QSiLz_YXaC z05e3~b!@xnZG-xZkKwA!vL}%L_6+GbI9+R zZMpjfQ<*W(rIaSA%B5(cuB-kIrLXUU-?D(bHTTb+MXD!lyd}QC&NNFL`;~ZIGEvF! z%vcIKQT+ZxjVYP_aByKtLcO9Tb{1QxC`dlh%_MKt6TIkylI!x02|J@R<~^N=58mN3 zaS!@U$*dUVBJaI-JxJY^R~<_DvE3m-k;Ub2VS#x?<6r8Gx?)FP*@8DWUSW2>IefHf|4XdA2RbkJH9~4N7@mcn#o{v%kGl_jb1G` zEH2bJZ^j_b=$lheC&>MjZQlToHz%VtI{uvG3V=D3Q77rp8S|d!V>+MnLYV9x?J75 zU9?-He>nm9ry-}geLMNbXAp<^~dfCwebNy)NQ67~)}Z5u-kW8?MlHWTfi z@a?EUC1Qfl;1S-VwLoppbEwh19MC#=R+7m zy{~`vxesq{b%JVI0uk&UGJ0uTn%Dgdbg)52@XY(47dPG?R|l4a*%m}leo|&)o>`qy z0rR1ftM5J8T4qyRq~Wjk{B$I&jv2b2JEyQ0Fp?wUNE$5b$zwO;mCTw9e3P7uSfise zTov1e$R?6(ptz^hi;Z~k`-e?gmix0yhlFO&c64Y#Rgnn;9qREet4Sje&mVvJAaJNO z$G&s(IxNl+Iu7;yd1d6Q3;Wx<=k?aY)Ngcp#LoIDp;vgs`l@9w9M%#y;es}Zbd&~< z?L@E`{IOiJpMK_@4a&C8i1x6>g!qT%jp-{*;-}K|KTvAi9tufT5F^fXos|1NXTj`g3(3!dUVxq$Jj6?5c+X~!`> z-nC)Q%G#yi7DI%YB#mm7k_lzH`J4l?2nJMhUcFb%P zuG2fH?S!+f7RR8|);PI&qW9V@o@@4F)^D@LjT2E#EFRQh=e4E>-S;Q?c6(L`-^ZTq z`o>`!#Fe|6E%Y``@%zOt7%ys78Tvm3nZ?UKL=Ne_8?s!Mss9XMDPXsCi^sHXG7#)N|B_%J$9y70HNuS+$ zFMmE0(;zaB4A_RFr>nW}Y$GWINW-;=Gm!6TN|vz3s&#NVb5v7K7e26J-B*n7veY__ zTA&$P96}9bA-JB?!olYgt7*}&SpsWtP@}_~#x4y9+P(nk0Gc}m5B_%4MJj%S2~Lx! zH0EX8n%&*7j*jwPxB8=61V-?`JUJc<@{v!WWukpZ|Mo1USsyXVw2sRMHh|Y+iCH+1 zH;9I9waGB(VsH>I+yGG^a@a2@)<1oCmw@U7%ZY;Tl_oUGKrY+p%U8=`tNOTIASgUu zC9x)K;hp|?!(gZRtMOgZlc^;!3OneqM7|G>D0bQx_}NWcAGiTF`#zq-B*`_<2pm*U zhm%4Hczt;$&}gV7#+;MS=5Nj|M#0;#>xJiO*{#7j;2%F`->byy(@=xDx}bm(!*Hig zK*Y56a86d5Whfpk8bMTo=8qSedSg26LV40HM8e-7hJq#<&x^+S)^OS~!S4l>Rry)| zZ?D17?QITByGtdnsQ)Q8i zybYDvZ8Z0xUw0NHyR#B!$KAdnE{ynvSX8uo-i=v;$?=iz%;l+#-oBWwI%Y%JV4sO2 zQDJe+TG?~xrOG+fWYJ_rKG_#M2eIAez>ml4`&cR@c#psUh(Ln8k?K-gOe-{GDCHcs zQUf3OWVyeePU3-axanOy$uQ&(`0=><)fIa!|T9C}QY7@Zbkuq_rBik)V!u_>GR zdNMK&$F1@(w|5~~F>Fn-BNM@xSg8-4y%9$gi&A8WMs-(E8^4Xv> z)dH;x0wG>Gu&+a8i>ZH-NY3xYC*Ao|C*AF6QFb8`W0sA?dA;~XjwrVH*!m52kk}I| zosT7@WN#*5?Pt7jp{lghw8Oe~XSZ?(6S+e#RYKRI!gg0aI1svIAm!Avg}1Tq-7IcN z8z#Lr6A5?fEMOa@w#;uvR%+u67hg43Eq*e57&6N=C?2akUiyiI0wJ!zmsK%ip9-m}XkNB+n@6SeKM>tF4R zMCM@|GL|~rG5Wk)!SkO~7x+UqiGZQ?-|m`)WaOCWX6-qG-C1!|C@3E0 zzg!v!qQK21tQBRM!%~ZKlFEe6&eY<4iMmXb`ZVl0`#r zL~&GHUn$IB^gVZ-=snHH!cODi!ZHA6OxR1g-IS-)OoxU<6n>MPAZ7caUSV zEE@;u^3;EZ4SlG4<|L}o>d+hN6$*{MlHxvozfpz*w!x_#(#{*aF3q4u4=U@#x}SkW+YO>xUvlN3u98_o9poWA;9>Idb#+ zUe#edHLiL;e+SRe=7N0j4Cni|zPamQLOb`blaI5lY@?*BU)f4GBdQCJB-Yk~5zmO= zTb+6RfG3YM2m5T$5Ze-?f2y9#sV!gh;R#z*+vzTy{RBK{qHGKzx*M9OqOP{J!=g}u ztor59nbbuF@ZBFDD(AFch=E|rZt+a|6yHDn2rxLHSM?sd)UMxUXM`wZ1gep%VQTU$ z=P&(Adh*gU@6!~3u#@p3sP>K~=+*Xoyz>ZpE;llmHo3qrCLFJCl;2+#U`U@rB8nhp zgj6XcD+f82n-$N`iU?}5x=gb`H`8o9+E@`1D7#vg5ToR0WnEZMr~UW3@)-l_j7)=t zpV-MKf&hHVHS{OOE}SE@a($ydn%Iwq1prZ*vlPwFcF47W_k;>DMf-jK@5Pl6e0NHE z`YPHx#mgzq^wYYM50$3*FKi_ZgFyb@_^$p*f!3y7iZitX{wL_l&#VZz&6sZ6VI*JQ z@pR^+NsltslV)w$1B`c-gFBfVGnupJ3G*rtD(WRS)7~OQC51m7ukAbSXA!W`FY2m$ z>Ifk<;5@I7E+~dK)1J_L(Onp4Bhz7-K#Gx#0f_ZIQ!N*#$))Yv_tOayr;1|Mycn4Kd@T%f&Kctufl z(by=kl5n{?Tfx|!edV;4jnHQhD_PM;rMk}N&d-UURJbQWZTO0`CfLv3+(328s@V*OUW_5@ zrtm4CD{hB{Ou6jE-;R`3Z_Ia7N%Oby?0SqiFK5I&YV%?JK<;|7V=us%d8p7GwcSLS zrvc4&E(_B1?t-Ldd&MGD0w{5UT(`sb%@;rYn-TY$dzqo08Y20nEo(M+ztKERN&$ZcSDJ8 zylx=zv^>kEH~JiDyLC|?S5dE}(@-#S{h6fQ^)57%Rv?W=6ixKTpwdDE3TKK zT2V1>nEiir|3-tW&Mz^ybmle%eDP3a~u<>JlrTKCi$=>!9wO=25P zZpo9yZJn+qi2;zUtQ`#@Ox}WMDxc;8wx^sL?cEtocft=Er%FXglbUv5j6VHxQ%1S! z08e}cx{P2EPJ>6t3&yD&p-&oU~+9PHzQb`r_gzR=eIChPTP%AWRX6?(^wq(H*{gJeuQX{!tgY@RTgI}Q2diKeq4*5mPn=v zB@R{hgVyT&XrmbY!^pgIG?)q!b{nTeL*CNxq1e}k$G@H481h61MN)FF@nVoH8S1LXQzP$CbW*M~R)j2*{5yvz*##uJZl9q#4G z((P>o`76wu|H=JoololXIn;_r)$&cIZ zqRNE~bvYB1Nci9&@l(PW3%5DWfU7D|I}Q#_C+yRxb#GD+zeb(`0VzE5xa+Hv^9HSx z#ffo+02qr3qSG#$@9Q8vhe+(w1&bK6c8C;Q6kFt>Wm{73-Cg+jYiM!RM;$kCZR^&2 z^{l%O=^RhHg_tsMUbI(okaWy{g%>P^cLqr0RJ7 zU{Z0gf# z_#G+`V)7pPR6ITOKKAK&u|)?`w0Bgmf{ko6cZ6icVLDd6f4G-VHnfe>C@KSCnC-3^ zL;9?7Hx8z>R3a2@cC>agaGzPJ)Q?GD%}>8|CLowXZK-)FP~Y0JkHn=e_iM7xYRCLa z2)sTf9NH6F2U$QYzbvOR(W=GsxRC{rLb`GMR3YJI>j20HX<>NOs>7a0H<$4#7~XCNyG z(JSd%75LekAzX7tJk9WwWbRv-Ji~>7XfUcH@~Bkf&EJj#+Epub6sT?VoOkak304Xx{V!rY+o0a-7x z(=1vwMz{bPhzgM4r-7mc$eUb*97=@~<>@^%6Uby6KOx7{Gc|OXe>a?+#{i*mAwWj# zq79_huSPZji_zg2fF{)yny{XtIgm&l2m(NkcR_#{fG72**<%+xU&cZnfgYX#777^n zOAOtS`yxO_OrNB*~`j~nWwkoPw zkP!%J#|mYv1IaKFGx?f5zj@ihBpeP9(rqmTtFBf4pDM*NLm-x~tvJICpK{M;bp~n^ zd?k|jtYoohIcd&Jw;CDGz=G9;GW?jPY|x)sa(*)Sh5<7B;Oc@l1Q3>cTVJvP@Bsvz z$xUzYy{I{x`RUk2ADFYQ#B-fV^S-rtrx;0ds@ud7WHLd8040|ioVst^4G&HjK)*9~ zZ$cIDDMLcHF@GzlMedS?WN7P**KzYRGBW8SRr;os0TM=_f-TYcW?C?}+i@u#5qaT7=QgpsaH3hyxH1IIHF~bqO~?B$iEN_{ zzrJRnZ>{bQ$+orb8cMC6@eUX%~PzmIbH{)Tm`rMzA%x39?v;c`Y z_H`hDT*J6|G)R*WMP4)DgAkp1VLZ@@32H`hs9 zc$I~{$mx<1Anv<^U+zZb2^{tZBdqp&{ zVawu~Le=7cp)PiPEUx+Q(KXg`AVAYtE-%ZgK7qnX*0R6w!p(Xpe2SaL-UgtjnHw#S z3)+W<=q?$(H})W(X}tTWhU{JxxednXN1HTqHh+Tv8)yac!i7a_G%A0N-nNhzq=K8Q z#U^Y)G9C1FL*8(i2m!=%5?2A$C@Pl*@;+n@@URIWH`{4B0sj~C@A1QTH|aZ3#hgcJhT)yfsM zDIvaSe_ZVQr|)m@?OV{jA|W5o<;9MZFMNUBo~>q(t2#!rtY^J-`9dI4@C6fMJdaf& zWk*cWw{|Di4LF~<0~pR=Z46A-k1~fY*)XccM7GVCrER&G2P3_oh65so{bqT-hQkf+ zNs%7M(N(s~pm#?1S(=jWR%HIzf}ZU>DlnV#ctNVKbip6V7tOdKoMizod|jufLDvk6gN9B4(s$Pdx;4e+ZSO2IEJvOGJmZ7<~V>@Yp~G7CB(@S z&Nz7IJ>PVk#tb1847NneO@JCS`@7>Csp*jJ~$^#-Q| zDG664;Qf%Of&p>a%SV>~jA*ao(@{q~?C;&aKY}5-DjYlhga{I)%8a<;25XL5@yX?5 zQ*8BbpIcKlsH(cv=6N)rpo1iT_`P@03{)RumDZT*xo)FH`%@IEwH79hwJKL_`%{@W z{B5IwSab%4;iBe}x5Lj!^td;ANccIQD^V%@s5MF(ER@f#bbLAnEWgnt6@dUXK}ymO zPZ)#^u~v=k9)_oV$Xr)?X)#HpfgQec;kan-ifhbkCF(c<>#$1?D(pV>b@)%eG&vtQ zRNYRXMlAn8WVt@tfq#XkR|K12%em$`D=z}_|L@^YSpuY4=#Wei2pth-bBa4G>p=fZ zLDBkh_Eq_6w42D1;I750>hrLlQXA$-~nZxZL7f~HYdHzpn6i3%GWMtUFNZ~}2c z_xtKPO}^3`OmWUeFbKkl5gq@8si__>OF!&YJ)hdW8|R`JOT0mzv)DR0j1g5Dv%hwFblx9))6R0GQM#ljArix^r_2eGDnLC!B}Zz#T|aLzEu8DGgnulf zO*9MeVL5s0i9~1;G_~sW)@8N>)R4vWMt|Q?B+j=7`xIxTnXy{lP~svnXi)CBDq|7` z;xT(BQfX;rkKy;OihZvA+8=ARKkNo*`5eF#$LJe!Urq!e*wO~e(TxWNPnEt$3M1>dQXKVcE1`?-? z2LmK9dg&MYeJ79sIirD#KR%F2eTh$U2F(Rkc=Q|!OXp02`U?AAl$=$9>Z;G9+EW}! z%V;63&7`FHgiR=@CKtX98a++Vi8`uE5PN~d1H&T6&Ecofc}wX&_xtHOLL*9@l;%?}f-&5voy?x>1K0#y(a8&)){Dug6pkYe7Sy zw=SjFd&7KEvAh2|_GM1}Igoc9Pq#b1LqVD1ra+2u-(YmhCES3}@lsOHD)8s)Phz(^ zz@&qP3c50c*+5bVNVzm~!(=4tQ=P&CsWni%202s}7_0&BGO_9k(;0W`xeUfSY(dca z)|CsO4ZrX9Sg&)4Xr<|gcO#U)T@;RHHi@;AdDJ4r|I=F>I~4X;QLHyzZKRoujya&p zV9!`nU#+0Cv3(V&0x9WabjqhDy`QFZxW91KL$xAwpkV*Q#FCLM4d5NnE{elk`YBSozPUR()t&O*#Fi>+l_?}>NsW)cfrA88A5NKYcQ?}~W=vourWua6 z1V$1^YY9+)QoaBJS0_m9w&pLSoXfGO1Q$oVK8pB3C44YBYmcK_8f9eOtjBJ;p8swN z>XPr#SwN-i2Vw+vohf#TJ*aS}CpSZCji9k}mVUVFs}>ppVLT9r6n8iz8B8{RAl67Z z=WD{5>hP@wQ@Og5z6^ee7aclNsBY6(q8Y{0o2n9%{RBu`iJe_71-)vYfROm~0|TT? zodLsEp0XRnIlQq=O9GljG zubT*$TRABvjvN9r{~ z5!y-alUulp<$5*DRK`SX=Y7^XEzAgL@n|1mLoIQ3sJjZ{=*idxt(d45-8q78`fw5& zQ9!aTSTWS}mQk&N6_j0j)OH02WK~^K)*m>dM~Y+Hv}i-I?l4 zcI%*)=(Sn=GQlI#4YZja;A~TGS{}YTS&RCTa>TNMPjjNQHEiXc0kVhM;@VUvNOi(C z6RV_Tua>~eUYGy7dh=TSzWr*AvjI2sEF+fwfCbh>F6a*VS%yT1U4Nvj?nlz`Jwefa z!upF<21Xf{Yrv{X=98**C)(q2b<4NNTS4$$W`R1F6%r-VSUt~8UPzJPZIQ+=nPuQ0HOKFbyeulFaEr*+7g@J>N^ye+E{ zz`OyiyY8ldd;c{51h5aC;_s1^ZSa7n_1Js|ttV9%ZcreU2o((!mM*f!V7<5exWMdt z`pwJB*6|5YF?C*#ePAjdQQw)h^E&+o1M;@_Uoc%c8SIVaoh8hlXUYPX&lrJX?GATT zDGTgDLIY_GAlndndj0~0#>37mi>f#+|EW(zS4WS#$|ih1Y9GzICphE=?sj|JHd9EP zbtSz#QW2&cWx19x`i>lpu>0ITs0Xw=L=4HRG@o3CLq)%SoA{~mi#A*{s0OHyI~oC5 zA-%*a>?ZDdoV~A*9!G!;!l=ZRwq@oae0J%P5wb_L2seh3L#YjYCMH49UIA<y+!(qet`=}u9z(h)d8h{e8wC64JIL!===8Pwu3%=q~S!wxFas71WeD!zh zsZAYEqJJeie8oQ!ojI+F{`zgvb#D~tvUH*?LVa)xOPtQ{kMilutTHqq0E~kDeGlMkj#sg zBh?_wfSQHWTrNXzr|I-Ye8h4kK^-ts1SUi&uyVNl_mDZH&5uF}E(xZ9m`c0%rtwY( zkxBi=fOGHDEHs^}VU%{Vr`ie5L=P=QPCdL~@$mZ1Rq z@&Okmr6xx1pi5hIEF;5kY{?|`ZcG$9!Gd;jeke*29;Xu$!Ve!UXE&1@l5;1`1%Wnx zA`A-=?M9F)@$0UK%BEOAx#{9u_oPn)iyKt}i@ClENwg^pHAb z^3`wC|9Ke=J!4&W-mF1dF>N%@FE~N<2Z0Bl?YXQ#;g0?ZkTqqTUSHG^$kNbwb*(40(41Cf`xP&c=4N4(u-olc-!bWlBd$88X7Ov7*&k9pefsM zM)J+zy?={3j~?r#uO3`O*=LLAHCeLlDeQAV(rsDkWvRa2^=}q_@=Vg%#_D6PmxUgj z8v9Zj=XnPwFG?Jr$(~8r!+>I=C#)4hv5v+3eLo8e7Jo@+B=$B(X>sK`sHx~{srwDM zHjrZ-ySIC-$9eJAm#J#jFocX=jGBC1UoFQFnP`~^1-%mZyT8gt_#nlICrky}#sXE^ zR#jdPqI(yDP0WiqgB1UouQZr z(rUW%uZ`_? z<}cf4FF$P|;-@|(LUj!y>(53qD24PuyN@9cectC;MeRzrKE z?H}>fmp>|NC(-;6VmoowQ=$2DMsd>>0}Wn2q-ilicla7)e~~KdJi}xF9q$aU-5I|` zvSZ^8Ozxp~lLb>_R#D&k;1s=Weko6n8I-igSJ+>Lf=Pq(;u)HB0e<3POPSUQZ^A=a zq`m|K-t#Kv`vr{{>uA_Oteq_xXSG%_K7!21w31Th?BC?URvoaU>qN zbY6qKQb=9HHnPI^BURG}5N9yp{xPJkBKazBm@ay?OjE>yW}v5Roj^JEj5AXq&jiV( zmxwWV$l?c8!gt2Z!)VrFL01V3B(S8g#Y-9vTqK-e8}jJOvsr|GOT|#!V(i>rdjl!% zbk=w#CZz1SiLPb$@75f-rrzMGiC_!P%-&w}AK#CzI_VUow*7)$)(IB&i3B}Ssw59q&osAD)g}g+Mqs2t-tFdgAhvVw%S#=#!?uZd2eLosC{xgHNk4(I z3%u4;IDz@lN>&Ptvw=hbd)bqW-uJFYMrdzj60P|WcyG=(HtHt*6wI7S$aFHzJKq*< zTC3Bc$@R({UpLJ-AHC58te?GK+|^DdsvPbnr6W>Y5vJ-6_6<7fhRwVxmG1Gal^d8H z7!}0cxv^d8mNF4v4xLfy!3bC!Cddo=H1#COGkVkeP0Yf<;cw~xiB^&X{~ftvJHUT~ zBZbaJ6O%f5>wG18N?-Q@AI$oRHqZt!X$B_$#XWWR-nxFhXZ&?oppzJqh$~{S1NQ~y z%FXJqW8v2c0A4#0A4R-&_m7xY&GloHe7PBE6W2d$o-a~Es4&<2K@ zu|ygDIlQj`{r$^p^%&oHB7u0>z&lp2R8>Z zG_6n7Ylbft4?}dmoOZ1ftWyOm3kIELq6HJgTZ!*`y+o3t_7Y!o5_eO;Rc@kf}u2QsJ zJvDyEdl1NV_4TFj@j{R*d+N9tgb;~v^GYbWapc+NF0!1NAb!9B_0s}+%&zWH=cYsI z>cYd%ZFWT=hnYSbc-ThRfEOx=7L_n;B8f_&=RidK+fD2cmtp;`3002Pvif zGGu*n>w4`QX{eEStF$tNuLXN?zkfVK&`t%KW4fJEZQhltr{5P#uCFq3Z1G)^L! zsUY6re92{#p#Cql=0y3An&z;(DF;-Yc&& zbM%j}7Q6hLU#Dsn(q%8?h+0U>o#cAjv9US&^}{FkUmV8qZk$N1#0$cXJv-=BCqLOE z)n$|!8k|wP(!#8Emb)4OR@WV z_rWsBBu$I-Fl#J%2RH_jo3-ji(~Qn5+~7$Du-ao;bA@yqgU;x7N$RAW5@;Ap`d%ELXIhu^_wmwhCEj`YRl42D-tb29 z+KGNPF2L>7t?cbgJ^NBwiD=wV>3H*0x_Hm*i3Zg6rY#tFIcs4dky_l&dLiPJTG0JH zNREiQ^&$xeiRU9}ius)~#--#458`Nw|fGont!b$osJ*~tVi)Dj0i7oWLzq^6sI&;e(fA@#}!az??E-s{U}NIW!- z8fQl#`l|?LTxU%43HgCMCj%CqPvX#CVJA#B_8H%_UYKzlVDN&X>^ash(l_4&a)-|X z1QvXchjEv*zAyfl<)cr;^VTv`gN)cuSLCit=aQfG{Z<;fBBj;%l=8TINp6p8dVUF4c;` z)6#CLlRGIV>L#6CZRCNs1ivLG6ue^5{fZJyIsH4hefi{&q4wnH<1#o zu<04U@(YH=biFzj$o~XB~_h(Fx+vz5H^ifH8 z{cF-J*z85o80~!U1=$E?xt(hPA#5#@P+$e!Oh&7Vm>rJmD4{86eSbmrNIff5w8CD zC$_vrvWG4;0;@+CF$sWx8!^J$}}8RnVt`uBz}EmpC@W!^+E~hp2=vGbo94TDfrC}SqQ&> zpLLzlKRw{WpfTG1LF_=PZj%2~qWYE}^^L1ug8r{sN~S8-C-KyzC)re`0rZuhP~nrW z$N$vIshgK&QyqwrbRsR??h6%iH5+x~yE})gImAbC*n|11^i8e$*4_#1rVOO<-PC;7 z4Oek@^hL~3oa^Iy=m&2;(KgfI9CiB6h=3`&Mr4XOZafWDS3;G4cb@Q1DjP?&WV|iX zq@a4mVw11kuy$*!Bw}|U`R0@2q)SPSD@tW;e>P@@S?zP01UApNE?uU5$6LwcMDI|B zm1K(Y5yQX@ZD~ty?bLOG`E~>caJOQ>i0i@;EQm`xG+el&!)_8G!h2;9O)-`R+ojV8 zqG$g}M=MV9Q0I^=tBmM1+Bz|I#N=A^EgfoHY)dB^;L)8g9rEApop)P}=Oc!W{NIdL zrhe5~-{Iu3(3%e@VlTl8eNT}^^=ekWV#DJMER4eIWa_-~yXLA0*-i7R>1twXp)8BS zQ6Y@fiy#gKgNkZH}PZ3KO^`&+>XXxE%zqD+|qfBO_y2 zSPU@`V1|e{Jl(;;0u@Z+|AY~7nOP$ zgzQQiGp@(75;j_#diVAUhxtdf`_oJw-lweI%#kncQ42qdMroWa(S`b0XQF>h6|ii4 z#^<%Ut3sVOWG=zwZr1l?C0#h@Xyl1@@E|Y+W))NTQ<~$T!jLWNCzTj=BX^A}Igcwp zp^~JIZ1eNzt16cWK@kAxM`BgD+IJo1?+@#Kqa1fnnI*rB*EzY(#o}>{Rfc&vMMO@+ zx{Z(xs~O^3C5a|6L+So`Wca;@$rH#?Xdc!GX$#f2u;IZyA(OFgKNo>u`;810h1V+i zpKl_2v6BxJKsK~0Ff#Qf)zA!@X5Sz$6wayYi;=j`PuY*KY^dstQzjn|jUK#LHd*g| zX9UD#iEhPEU3|kaI51hXjrTCjca`2Klw@>i0e6WO>;#b$(=|)k;8?x&1Ush}_?0Es z-i>mLC0-zAW{*=TrU&sBx`hn#fuqL8Cx~aSTpT+w8!ME3ydqsw<3f7yHnxy7QaSpq zcCVpGU*bWIkHie&c1{l3Yw@!mjwm%h>Y>Zk)JX>kE87MylvBTYhUL@kmOthVRte6= z*bf}qw%ZYAq3qMDl@1ko?`VaeF&!rEo;@j_eNw^MF9jqvMkg_0q$?0=_#r6U*a0u z4V-Iq7LSFhqYgzP-QJo7y$&gbWH9;KYqQ@}yMj4eRKB?GE2d!cO?_=@k;}as5>$Gx z6Y^X_xW~kT890nJA4W9(Mc?bC3^JB&)Ae%$|9jUnCed!4j1Ppfyu&F(Ohc@Av1tXmj?%ug}KKdyv_dawn!6%lhFec=%V>-3~Gcxw7iT3F0Kj}i6XG9 z_ok;@j&9#RTuyzy?jb^L0eG_7twSh@aauH6Jz76?gt+AasEE4C)NIDazxt#D+&dP~ z0MAM4g5NJgHzkn^;%nW^W>40s24tt__Me$7#3z_l%NhA8l(2|u5!LLcu$P$}YH$$S zw5bjZ1ZdkjJAZrryw#1K%Ls8ZpKf6i?#X5i6H#QCBy7`-w~X^wX$B0#X4|%CvA88ah)F)g>L z{H_~2@<>8$rToIc{955jF862LE8N0uf#)v<0=|9_QXr5MeVtia7p_KRwEe{-Y2^mR4jrrNjC+#{ChDDruAw_uU$GqDs<8K;8Q;red}4QQ7%MZHhpl|%B>6l&iug}^6^5FZjWqO?>H_WjCd{*S=u8fxl)Y&MCtohZQ6X?h znD%HC6GOeasBAA!+K$7$$eQkDmJZSJ=t%Pp2kV6R0*`3SK!DhjyAibmS~GMZyf8eY z=jv?F7G@~8ofRT{0TN39m4g>un?bWw1P~xQY_iO zT7Z;i4G`T^E?9uzS9<4S&}V~5_D+4AX?&QFApVPiUN8f#^blS-n>(B{i{Q;9Gs!Ws zUdv*!abUTgKb{@j^{-jjw{2k(CBYNy-3fiW2@>1{S)15yUTkC*Wd%$V2y( z4(Wt%4hXhLcdOq3vR_-lQGX`ZSH!>#2Yz$Vl>L;&uKPZ|N}8|M)Q-qgYQw{kKT2f&+_nDNGO$zni|_$cHP%nTls-@Q{7zr*PJPF?%2%W zNX20u1_`Iv~1OC)z#za;2O-ZE%8G%Il;gO!IQ@7I(A|~1Il}oFeUxY6#ee0-O zp9y2KPEFfl+CSOcyQ2}sQw6(&!2F8N^S8S9x$LoB2}OJ850ckRdG^cdCGB7P;`Xrq znoKzKPfl`2zu*aU+hY{@V#g#_w1&AEvHd0LR)0L2m#=O2KJ4URW^!hi%p61?l_r3e``B?L$M&Yevff_k<(slipvxv=X3= zi8w7LH6z7ToDrwxlHy&{ZP(?D2qgjSwmF3Q}OvEH6F7 z0bKggR0mUX%x$Aj3e~Do(X>t)@dh~y^Mj`XtsGr6YFU}6%d%nGV(D_L`$p-PSz`55 z070%*HAt3)h~UwjfwjAXzVs=wUHhU4wDcGz4Vjd{`j#q~Wv7iz+&H_4^Tc0GJus=i zG$OKB?#Gd=>fL^N<>q+ln;GK&45)$bj{#wi&o!>*wZjuO&Z_;qd$(Ju#Ze~V@XTrkzG zjSdru&PMwBDXc+v%~rFt#L?T(SV~#|pA{8I0aE-au_$Rwf-xT^ZwEaw8?wnn~;hrn;!1>bi&M^cO(gN6pKJm%nBX z;R!{mj~5H+3-)irTy~}mOy>RzvWAHK8JfY5_G3(I6K+G@f2sA$v~Bj+zD$xtbS6hI z2B3~~q0*C@y|8w_$f7AE^Rkqy&~4L*06rO+JlX8&m=a2wG;33$x@6Jsa)w*nlU(A< z9q@rppRlsfRt()BteL@-xCT{e3;PLx;|Ob2M>uDBgLGK3Ck1d;=_uc%`YJhTyb0xN zM4!0H|FD4d(eJU1=I-% znqulx? zJc$F^nWKQ}ys9Q$=91n%HJ^YNU8eT|E%t0($0FG8y=^F_w)0t+U1Qd#ak?&sq2^N2{BFD zC~5zH>yaUNRdL>qVeQSADgz*@p39$RN*+c+g6s@D03_2!Y$_AvtO;0ahgDGb2>O|k zkYckceRFEQJVlD`S&t6Sp~4x{B|)OwPA$oE^xW?zazzRTuVsj3P;klkqL z9Z0yvF8;WW>#1wiGHb!dVoW$UVXGX@^p~!i`V9wV{T9`!TsL%VNdG8V8Ga>7Z!&;SteN^IfV4LVrAYp}i* zNF5JE);`u+KEK#$J3jd3;l<&>!hieZnvJ%h_V>__v|mU_Bqh`}lue)$(%FfbxZujB zwha{?YZVCPuKRRDgiJfXg(KH^A2}LQTlT8d)#XJ;R8Qa-_0y>9JoghZq4^*=!d+Rv zjDxmIii4$9d<=t#I`y7fCB}_X z=xtVA{V;~RmO>V{NRCHuBp=j0uhj}TXCI2ZoPCeiHIhL6|2B^5oQgZt23^}QSa1L- zH;@ULE-YmG(tNG?%+)vyp+!-Yo9a>&l{EHJx=<12H{ z&Qh1kKF)wiAIm*@y1?7)h>3y0S3>c^6Ytyac39n-p%la8#WL_9vd_8X^`l>w` ze>E6i(uDUJ3x`9M4lzhkcrT|zEpa?VHhlm^JE@;|-O6?Skx_@UqtZ#NtpYZ!0QCHQ6 z2CCJ)mE&iC=8Sr2gYC=8*91>GtXj*qkOh{uo1JYjgwfZ%!3debO>?7F^nM0)2c(`W zd#dj9oH9M8L6v9Q)HJB!4~}m>gBT;8OIE@KaPk4xX(e#Yv3>*QEmLjO{rspem2MO8 z<5x_r#jq@8xn6F27ofVbANk zO2e7(%OR<l~ZQGw`Lr^%(+pg^9*g`zEeCV6VV8B_c8&tJ;D*XST3)vgc*(Z zl1(?XWd;H0j|ZMd&R!ar9i%0_+xi93SHURQxaqXa(M^%50G>AG)jq7yz~^GJ`m(V= zyHJ>Rp^?Mr@(!+Jad*oXw>(n(el)7e6b^y3JBO8rwd?af(P80u%RnkAyysBbmUz(9 zQ@gwD;C=;nUQv6=??jW?UnQHnoN!P0Hu6e0B@Im2nEtNpGund*4Whnasnsosp5USA zj)iR%D!cGIf~iQ_Ktc6k7V&lJ)w*CUi6Cyn=hKh!3mvy90~bP1ieYam(;c-mQ*lz! zc+*dA4ed!z{1x<>w}`bN-@CSR*xp#BN;TOaKYEm>1@*rSYvj|1VdzIVV3O$*kixiN|9ZFeuWLA`};|M1TCo?m5E{!piNB(v)-igLL!;^Aa zTC>1D2kLeUZ7{3~{yh|q!wW3WGVP%C^5r%r@y-T$j3pXW86QsQNQqhtRp-F0%>0QC zSKcXfo%=}u6L4~A9rK8*1o=gtB-d=A?%#?qg+N)&?*72oo3gpc@)sT^e^+4MD*cTV z!3vP7bHYNJ)R)G&*wQmg8kWoX3bhprvkJ#fqt}{5Gxw)iVzZw7@XjNX5SiX0Uc0Vg z%h`>ORIqYC$gMG(&5Y)ot@qQ)7jSe#ea2=v=cs?m z;1Q$o>HQ*tXPDP>d3dRJ8=HAFqS?jAx4ts}F&d=*p2R;akYf0zW15QxDnF}jLafP! z)PfIkG0TMjV#9FE)6F%WdK9FvndLUiuJ^~8mqzQes6i;lh`g(*8;C`;d^6~_C=nO_ zK85Sj&Yl}L@x-U%bc%@Ot|m<>f!%$j7Fa`y_ZxAGx{kXYpVIq5#|Y8o??2Q_&z6fdD&TLpYJ-;(&BUDVsO2dxpJNjJtlI~j5kOQXsl z{IRI$?j{Qx5cUSCqnw7qn926BUX^EI3+V8q%+}CnJTp|}^*|BoA;B-VMjK-><}P1g zS-Y8KeF-9P$7Ns3oEZq<>nhS*sM^A^;Bv50!edgVNZhdpWw0{CMW;y&&>Ax)XYYQp z6)R@+-T(xT%4iruFuK)3U9i(WZs|6SrXfnvN7x<8-(+xxduvS~O|bc+$Vz#(r=B9F zt6YvxysCb3zS&F3+l`T_MBP=snc0{iR2QpTJ|adY4x7v!z2MxYc(M4MrL0h|Za$^- zkY%j@&_snfB#SD~PCphZ9<3qF0KL~pja)kVL@&hcTH;?u<=(|wmLkTC zSu9FU)A?0-uiEbx#@MTYQ_gxBd&SS4hRclEfJqLSv}>D2gw_OGgVcp?+INh|Tq_;b z54dUaZ;#Ft$8(I@ff7j{W!Esw5Y<7-4B7xVl-BwA*sWpf7vhGNcj##z=rp3n1y>)M zf@cmr|3?r?6V$x^IEPDz+2Q9cJ$u5Zt>`^K%aLSV#2K5L(ly?)cym@9bKdjHpGct>Z|-&FH}LYyG41l8hgk2h@uQH?!(y-0K1@zEb9BL|FLB}3aVc4R;a5pE zF+hT^*wA&~zT;M71yKp&ZA8V#KXTnTv%;xs_hXPA^6UcZU4)jP;H5seQ+i?74V*t4 z%oS}b98NrdHTRD8o15I-+%;EwIyaD%%uF~tV`1SJgt3rXnJ(OeS-T@n(ll1SW44C;(xd#Z1I~1{0(<(L!6V{rsL`%IBN9qmk9g2zWP9u827DMNq4(6b? z>ZPN7eVitN@wkRG;ZPfL;mv)k&ZBRf-cJ90^t$6;m0AdEw@PO`!75v*rR7$|n+YhN ze+9`(hv8YA>TV_wvD><;1l4PGt?TYsR~8OOuf1jrumGLr<5tb}z)!liqR9s9NtI|C zyqZQi*Tcw_d$WE%%}3MII+x1Ps~pDs^d&3Ly+6ch(HhH~i*7*HYJ;$Cbh`;g&m5Hr z1iQ;txigMMcveqUTveuKe((XKr!8O_l>eg@mUfm=MMjF06euvxd!tx5`A>Uo7UtA!!OrfeqIgmExE_;Z% z%4x#)K5y_h$?vmJDJL&?Ol0jX4cE_$#S|0<+Ru6i;zme&QhrS5lrFt#@54!2eUWs~ zt;QC-gOF=YtU~l>KNn}_UW}qs^u_tGQoy!cNzqD7c*?!wsB@i z1m@j=-`WGnl)`{quqg%H0X<`-^lE6w)aSA{B2gMxHvS;LbA62a%W%ceW6Ff3H(MwY zOcfU@|Ho%&@9O@frtMrtgA@3$l%p3vbre$#?9A=w4Bzjr_{a{S=o~gj+bHFNsA|Nj zs8vBHy(Rqupe}c@VW1}h2P^t@+pk3sHFyms#X4hM|GJc1h_>h(@n+N2GquXhpImS6 z5e@H9${OM1wUzu{nIOCN3~FCDZSY#_C2Rb2{wsLa(n^>6bvq-W;Y9@Z`kUWZR`=cG zo!2LoPv}rzUGmW0-XNvV8~T zIm&dwK|Ou31~2BWHBV$I>-yaY(*te8Vj- z&u7)T@e+R-Lk;waFnO{h?QU|RRBZ9ES(b^AqV44gTfGV`aia5MhGKh$;j}|nZLU1v z=?OI-^V`8>IK^Uh5MyLA;b7kDl;n3d5_)N2T6XtW4_5&2}v(2?y*OH9A>R(bbs^Te63oV1ban|I4ZEaUG!ih!mWu z=!Ud+ihzGlD^IptOVfS;9Hri!XA;yj+N_5LugU8tw6)6#7~f7Z&U|Kh6u5jehfwx+ z4tF^9YhzI1NG2(oB1D*=NKYnUWxJlkkAigQtqQTIZBBA=luEh^Yc8fuC zFL&KAqTlw5>}hSLTb`&^raRnMC6$)hVZ04vrY%r36zP3}>0Z6XujXTibJSM?iEsUc z#b(>O#J~Mu?Bz{%)VDXlXdAWGe}$#6q-4pO_Q016+v?b^kt2I@_u7sJxp&l3;tx!H z750TDY`i=R{~HE!FJyARzK=%Ie^cgux9IF8qc>5w5g-apgL|#WTjehTShYkMc0!US zSSrwl%o{G03E~hFairb}R9_7aS_uI~5D*&h%yB091srQzS2f&Kf61)`z)+wGsMb83 zh!`(OI4uuQU6(TUi~Qa4;l!N9#4i7N#;>v1aa(?Z-rO9Y&ZWHwqV~d#rdQi7^`hud z{Bqw)=%9$vVe)XD{#CobbdLMFk>|Cf(ky-0_R%eK;1^Zov}U}Zv+C4)3W`^s&o`Ic zleeh@v5ub&Ii4#@9x=f)KXe%?gWW%+d~RzeigSW}oolc7zFqgW@g`cs7UZSWMucXT zAb-T@9+IEEv=RAtrP4!u2o49$EG9npEd=cL3p2&Ba#*hsR+Pbd1#g*-SJ*f&`l{u2 zBSEB?0I0)ei2wAos53v(+u_A6zyJ&xRn*fehE!(L8H|r6)b`pDQ7JLGm3S=qzw!K> z!l`QF0FGJFud1s|?e57o9uQDE>u`VT1Igl_DKgDpv=+*HTJ`q!HtQ$aUp7nv4`_!bouY2vnbwb0&dkcA-V1_c+0iTa zfkb(QKWL%beF}y_)~W7#-10WJKYnF#Mie}?9EF-yU&f6s46L?qK~}^fBs9Hyq~&r@ zslMmjsa%w{N!5$i4_DjX<|M$UDk~}EV9>i$V-#>r;lf;S`rS~&^vjB8YShLxvswLH zi@7@*n0Kg+pSPbrW@VSW%C*>ZYyp^#T5o(nZWd^H>fBU|kLhJL4CR(TYEU&*2%wqf z+LyOxNc@b=@d2reC>rD27*(0yZ0rwt(+|NUA-n7;7eD|Js~-n>Gy`>t#7vOnG5{qo zXv)j8mK}-j*Uq85iAixm9t}1CsZY1DitM2MR6i0SZJ}Ltp?rlG(1P=0#{N-C#8b5! zxod1{Y%9h-v_7n|Qc%r2F;eu4!1@pCGqwUu`8sMuBEf41CZSa>g-goyrv zD^%B<%TLCBd+H=iH8h5Ff-EMnT;KTqV%dfB_ThhyxLWsTBp+{bl z*hF!`h%HudoVIjZ1N0g|(TkN0p~YcZi%&Oo%(CwXYnz_s*P?NcA*MXsIF&hvN@C48 zwYPp<(YQXadaHTmaHDYtkhPvqzd25J{+Z}kgM40L95))ch5HLRc;%*Xbs~Lr#AGRE z^e3uEnQq+ZQX?tNu%nw29Z>Qbv=qkF4~1lzp!f?4mHs5?zph5KO=hd_Nuh(w+k}MC zh%18VJ1t)l5oX0MZ`W|9KkF4RQVPw$jlBw+hPv;k=pX+lGNH4Rflx{lq^)e$W3w6O zQumlRwcsxYP2e`Vjg>Gr4AFA9mI-8*5`h?4GiK2&upAN0hlWQc0RHwy-cOM=);m>> zyc46L^MDk8TMnF(gT{AqX3buwxCKe7UQUIa*}z_|i4PeHbNPF#MP|iC^F`RJ?HN;6_xi6 zy;*z5S0+tPUQhx2W2IN0Or45_>3rS<6P?Te!Ru7>T>Ccs$lrm4uk6%DPO~O0g%65S znc|wlP1uUo1hJXmgkAgNzXINJt@H$>|1F`+xn8wt(_`kx>qE0ZYXe)BurpBvKsM!| z$ty2vlGj#MpxhpEW3_X#)QYm?WM{nLBM&{}Lh5`>zEvX;O259)Cvs#Jsm z#O}ctz>TR7($&!acutkfjrf6E9tYmg5GiVMo)tU1l%!rdCnW+v)%# z=o_;u{z7E+6c@4v$mDEZ69gdG9dhzW#{Ed`Z0IKB^mS!eNZ&5OYH8tD+fURo7$V1*qw(aV+kq>OrExT_HXl=1}%;payQ0Jz@E=V`FCf-aUf zYBX6#`}2Oh$hMN02^5N8_v|BDZ#+Ln#_M*dq|=_r6oiE;LTLj89%S#^jBv}t^B=*n zcvDR`Yjr20-A4r$Y-va&2!|?2lI4H5q5zg~wz>EWMx330Q$tCun<>%UzS7lCT@DGX z)EP*1$@_hOtyiD4>dZs)o-VrQIIDIa;6P*h8ae5&Dd&@NUaLxk((QxUy=6HFGe-P0 z+(gqdItG(F)0R31IEzwmV_2;w#^lC6>rr)!a=|TcM)RX};7{Q!3!m{C#E)DQ4d8O7 zg)L6ZJz8N{1cmNBDh!5&R~f+3RZAvT-T}gRi*F~AoX@PA!$o@ZwF1q~7#2`Y>vX&Y z-6%)1C2j6R_UHTX5?*@YTd(VDpGDT`8KzS`6R0WS+hb^R%>_d!dE1Va+@%uaN)RCQ z9o<;zw6!>A#53Z)vIg|yL`;%%VdjMx1)>L(if1gb`R@MJ^wFZBP^|o6dAn6`(O*L* zsQVa%fq-LJ_3BWKXE4vXe)}D!8*kmc*_{%hq)Xj4mwd-gJ0P0f%I778!28CDJHN2x31Fbl8{ycDu*gnP-62TyHv_HF_R= zLn7a7cO>pT>A3#zwU}rEZ9h__gTJf{I1>kk}7>NgR z$`L|cFtk;~nXMgYqF%s71uipKa{31)L{(G94go0=OdTuLY4mNO z1ezQOh~$E{Wb)%T+$G^Xqrxkktx4(c&PLF26)quL0Jvas@D|lRrLV06ksSN_RzT%k z)dlSThvWK#z18J81!Ynk#&5D8YZ`>kk`+hXpk)X^&)%DU^+VoOre7Nr3J1)?@ena8 zJlR~AX=g-HiY!ZwC}A{@D25NJQ^m3v6X%zX$`ebJ*&KS}k$+gt3M_9Tq0SQCXTf3$ z6MrH!XP?r!5mCLim30}Be%Lt!f<5_)oGfH(Eb?SeX&WGD-Rd5FMaK+@k;(iC5cDdW zQw2jGQq)3^u>+SERT)13Oq6fBhsRAH#qxe)c3P# zr>9GqWXOQWg8q!26{$`28)sbkhbak_;sVbk&dc)58ufF5{N}hA-7_p#i8%@tho8}d zdW(;{$zIS+mYP(&Os!`y5KR9oGPFP?H3bmM;MY+E>zUz*nzla?x&M}+ugB#K04C(T zgQx5Zm2~?dOV<*CbHJJI$ehDvw`UTWLlL+KGkH@HR}boW1-&W$b-x|;5)$8s$K5TA z%}}`1zVtd)x`g-x`k(Z_VLvmBiis@G*p(wNmUBC4opG2IlNtX{zj%gwbl;Oi7cCm% zz;eY(MSF%_)@2DAF_o?wp7i_DAqq&P7DN41QmLh!cBg5`0ky?YpN?bKLP!p0J4hGG zbxAUa+(?lK`U}}4kKX2KDh~295;aSR##bKs5m|e3t|bkp$?0-b+cuNBNhW>V*oNr0 zs`=+Kkxo;MYAeCFSa=82_pv$S=ZKi8+DGP1BS6OTGW}n@?)Ce8Z+1C`ae7T4rl2z zxglyFZ=LF(mOPbWr4uI>tq*IohSdYNuA{mHT?n<)s<+?%3IfK;p}^50kC-1{e#T0{ zc?nQM_{+qz%k@qjuC6Af;Rh#YuCZmRj;Kn*Xr`a-R>Bc~^Mbkdr{xlYKQ83MGTbeY z`g=dsE$;<{&_SdX=S4C<(s_bwa@$@%wv0|mBCH2eyc@%^wpFD8-PL}Ns3^Fs+0ztC zHL-_S7B+Yxv1>t&K}f>6LAVuCXKPt`6f;cWa0g4EGU(uoL1Y;il0|WH;2{=^AUJ;= zlRb(0IbL98<~WiDd$81^kOr;+iPwC1cHAV0D_Ic~2)6;p)lR~V_>3!?WtoRj%%Gmf zlTO^{qd*eqCjKyFWYC{8(~(bEvCYns&)1~K&DNl||I~gx1*V-{lM1=(s#B*vI`ZG; zI!9YL{$3|MEg@;|te?5*mZvqoGUPH?@y2!ZLpf6LZ`UQNle(Ff<*|Cg@P+1!&?atp zP7Umm`98H&yt>&Mo%G#sg2^Vw0iL;SoI~sH6^-p#-`}qX7ph@7fa|7_j*;MPG{sOG z%XwL%X``mkk2u?OgTA4he*cFb{`kr^l#WtH`^j@# zayZ*Y_^n7~GN3Xe15%;EmD100Ar$o5wvb{+K~WkMRf$O=v$>{CU zlw%U$FRF(r$Yj@O&BBXk$*UlTzM4HK0Srx8i7fVQ6Se?a{x%>(2}2h(EHI0TxYQDv_|hdr60|Y0H<)yaB4hNQCXb+h2|@C+^*C)scu& zjA3Ia3))SM)^vbniiEJp*aGcRVHKzDJbj)lbS1^h`+1^9SNwB50Wf`tKB}z-aV8p~ zx02Q`wkBxN|IRA0KQWmE5H<(F+++j3`jYiV0ow(bQ1^mysL288oAO@GfD2!T|S5)!OoDy)nxlxnHYG+|3uwoHEZyfVBI*@*q3OOYsJenq}o zBYK93UPlRB3V>5Ayev?%yJKMVl;Z!Sgo!wRIplGNnBf<1F#gc&Pnn8vlj3GrS#Es% z_C9Pu^j!@bnc%og#ki2S>R)&gdDCFbJ2FV$F!btJgh2>ZN0=Z`xni8OgR+-MqtW8G zd^{}wo>zR+LTQtm8zD*-+v)lGJGs-qrPEys8L12H(@yFRvA6F=@Sp7|e*O98yJq4~ zd-gbkvhWf_;?s80c2y{QHUm@#ug5qdT8ZzM($u~vWWAr9z0Jwl6x6PxPhR?R>5Jwc z=z-qRXnAmYaiwxkM(0LzP9j!O}YVxsrJ~_7^=R$|E|NvTiukW{kCgk zz&>N5kBLu6JGC34|s=`nq zbwOmJbTibX!?Gj5f93EEE8Y%I5w=io7`fz-k0}Q zxtE++Xf2GV;3Dh^XjfQ>Q-1m_`R+KmlY34K!xSWg9BgZpqR4)c9PQ?V{_^XuF5}+= z=l`^<{!sKmId-35<^@a*LvD!8TB$%C)e6Wps%%wzGv$_-Dkoki2Z+OlEOmDJ9p5G2 zl?F0%U+E#_x^_UtS4ttM1D3L`ix=VZ$QkS}dMhefG#gVrdLf&$*1UQJ5EyC7eRD14 zH;jA_;oO*+`?64JE&+u0-&TfoUOUuiVay|5%IZaCW$oe@o#e?ya^Y&C7*;*LtIVgH z%uPfT(9fzZlVC6B#5oW+nI?TQ)}2#>9$0=Zj;|)sYbTSdV0&8zg`M&JX|eJ_tF+Q| zw9gb+MbCGyH2)ga|J%eDe}vrnA^T?cyVj_`ZoY4y6dWzat8$7*KL2e;mVdFV_W=wH zRsZT^!f|EGy|evjA?~{Uo!OulX`cz+ZF7Q~EqDhh*S>{9nHCx{a zn-%rU-#r7d?BKjZ`phA!#ipeDwVj3-ASf@W2j~aXu>tK8Qs^xp`f!H~;vORt1@Slh zh-r$pE&#RoCWZZn6`8P0J<(iat89M6IvWxBb~4;8Z%%Aek`p_aPZj4>1w3OIp1@cD zvvO>Gt_m{i-5>~dYp>F5b!5)Z*os8xmxJc+&D?|BNCAQt<}Aaw{5ZLsmFi+wdyj#y1?B5g&0p9?aQZgt=u?NJt!7EuV`2!Vw&ng z0dbrNxt}F2BIbT&hw#D)kwrQEnbJy%2}#C<n}4k+>9l(5pF!6SJ^0omb__LY@;L>@*i9q&L33OyNDv;}u`1xtEuuQF!L1 zn%Bksbtn1HwizeHBe8JX|19}8JKL1VxxKVQ`~`K&s`v~+iO=hFOU@AtW%-*vq&e-XR; zy_lJEK5@>>IK&DLWEUr<79@5jFy(JO4?s&-h?E}=C8Jur^b~MFZfk7J{a(Cr+-mIQ z(?{AweC4MucWuic1_PTL)YFFAy)eWdX&@1R51|?X!H2*q^ zs*`HAw#f?t-xl{PU61>m8ygx9?Tg!qV12~cSIc0KQ5oCN>t^AWNxc@w)#yYn5|0i4 zQT$d!U8@i9dL-;0@^*%+tgGd7df%oQ;XNAy4c@h61;ioY`hAygrE}jw#5Zz-t`|&6 zc%O+78M?b83RAOhV*pZ-sm$s<3ya<1+~0%H-9u#r7J~Hm5PihAai{3wEW$ z8XrS>;E{c4>BBL+($WzbysxZ&kN=VEO=M$F8|yvnsE8XxhBP-F;{WS5H8@3`K}{a# z{P~^ib?}mbq=YOlLCVr$B{^1|}ZH>m_2T0!aUE*!r8t9idA$4fNmHDX^yMEq>al+9b7O47u5 z#LqyF8#%cr_^hD>s|>&0o;6s{wWBdwGzj~z!HA1MlqRY(kr#%(8|?yX*9BQTV*+k; zfnvY;Tb3xPB~?2DYt`Dq1M@6=n4k!G%obB6S$nUy5v!rTc#lC;nE57wjEN^J&S=SU zxh!6_^vk%W&8lwGdsQk8Y^uy-u6N5bn=!g#-^(qo=jd!{Gn(MTSniJ~BWIcJ6xvTq z!_yIc5%=)^cI2Uk+m%4ULDVN@%zUi2Pt&zMaq6GwTKptwGj_K6(*nKFZZw`wEHl@UVt|6@9v`X-gv2FFd5wqFzb9_dWHG zBICE+!P-|JS3OPQhx47h3DeWs-gPVpB6mj50Hf|FjXB({@Qli&kzS16BD{J<zQ#bhdhskNw@NSLad|eJIh1 ztlp)!%7{2Ew-+&WscD5?uI^v07*$}Sy)B|H3OB^MoZhm!Jx{Jt;#(l23`tCFg(E}T zZU*WetI%uex(E~`OrJPcXOw2u_~M$c-3hcW8e>bX^?ZWW*SXSSt(#Iy)SnGvTn$f7 z2B~HwdQsLO|E`1)F3#T|-)pB+#1V784nFGQiCf97S%W7u#V4fG)39?wv##i|*{xmr zkNv3BJK`+H;nka!QoHLfa8J4xexF(RXHN`Io4P!hv*y{3mLQ*5J=YUY5W#h(|HC_| z7Rp)D2!Uf93iHVRaZ#TAqw=}Kt2-YR_vvh9xfQYQnIqvW71FKADKcgZtI6SB{k zvCQJsOJuY`Br10167pA{HE6;Fs<5~VbX@v3u&9rYx&=H;&et!`hphI#rUt6BNJDwBi>I{95x{`@@|& z8bK4BzOl-FN2v=rQ#||uA61a+{dNDwB=gzO5`U>}z#{g2^|cR>Oy)Ie zHOGT-EE=QgZ)`8Iw$?R+>9RExeQ(l%c&L8O6niz^1A8qJr=|=0b^Xm-YU#%iyGPJ* z4n+e0hCG?fdD6e%_>E(mv@a4!$;Xu4`NEB3WKlaD-#?LReqV{)r%hLd#7}N~Db#UM zhV_&4vv+ozX(@gL zHt-BhvhsR93^LulVXb2m@ERtxJkLZ0L z0wi`-*C&eCp9J!u-2|syn@rN$VnnmoP48b}phTM9t@TM_FD~M)bdjX;Jc4k&A{5PU z!20R?szvnC$NUOn&!z({1y@Zb9O5Igr5EqkUcQY>7@n}&zcxDG*=QoWS^a<6XeKr- zT&k`BYcEcNM=;$|{-!C?s?i^LUZrRZ&c3P@l^X3KG0=K|rj(IJGF#b=X#I)rWakb6V; zVcjnryb^cK0Uf0Ts*W$^>y$X;b=bH&`^*}zqxw9`kFQz8%6X@r^=t~Myw&7*CX>r1 zcch`>`3gj>@BAn=`f5ghX)xGm#g43pDfVV;rd@mpwlik% zh$FY7JQq1rKM)Ry>#L2|tDf7e$O;;2S;oE9vk~ps1Wy(Cy%TwqhLfWGoj6|q#obf3AyWBE*|PUeMuE=k^~-0e+sUhhOe zb~Nihm1Au?E{+JT`ZtQbpvH5&o7d*%gjmu-ZD$K007~bDa}tzY#?JQ|p^*@=-ptPpw$65$R$7_~B3IC^B7h?no#1 zuVCYMWId4!vb^{9I#>%=V_ICG4iy(#86DAurSR`hY0|Bg0|F=<2k+%BorQ%}}YdFL=8g*M`KaWl%^FSk~ni zP5MZzfdLN{31O*U7Qx_8Aj(@P{#e&@*xv^HnWf~h2A7tX`MJ~#1z22Y<$}LgxPuEr zNfz3X1|Dho&2rchL5>nM4%VT~f}$JR@Oj!g+yJp)K&`b^~FPnYqrU5Tqw z{}lAejI?a?Z=0f#OS7f?8W)#v46Dwr^oA6`jZf}f;fKe@gNs@hBoG%dZ>=i#*Q;46 zUjLveu@x>|G0uAMGckv4!#p~aRA*9G6w>*#RUYG~X#c`i3ZkAEo}54Fg0@npwIBnE z@9K>!yHv_bM|^*Nfqb;xFW3R%{)m&8gyozhD*r8y3i;23GDDBwk{C}z>`7_7jPoh< zj={}G^NV~=0SVCR$Gp-FEqs>uVD6&#kha?JX3Z9KU1pJ8p+OLf z;dYd+rCrX`|MEOkWwq7ZnyoQc-LD6%<|DY_KEDav97Tk_XG_ai8?8mAO-HYn#^&l) zi89=o&z4%io{;Fre|rHg_g=b2tA9NqlrQOIu?@@B*ThKt7yH5+4P3c%p_2aRD{V;D zF^YdSE-d;l)o%JU^;fe`J&_L&<<2X2bd!%EX#$=Q@E>fNYL0Tl7zSHzj5OyOh+gF`Acg!1jhHt59E0s!~NzxId&3VT)Ob@^Y zjoF$+DpmUV!uB2hr_WyvPH)a$n^Fa&G(}+odH-fKCPDb>1PFgbWVJi?tQj(X%09E# z*>rl#6EfK1KnlqM4y&F|$m0hhh$<{^bU)O@Po?;Cbh5fE=ENw36o0}0?)sLWh)@)i z=8l>`%xo%PZ-kw$4)|^!sc0N-vP}EjZ*G*4BLZy^?Qp&A_03+6<-BT9XqbC$X!H!= zmFTA%U@F2SwL254Ek7bYr~8NmGBbl{Nxr>b^SYnG>iLgHneKUvhfvASjMz-$gK2VD z^eGenB#p$3<$@j#U>PIchv-;`lP)TqzX@`9A0$WnE%-HwM4N&+g-tl>OA|Y>m_!h7!t36*D~L;ZYfp?F85fz$b~M+w04oyFhk@5*A|)8IpEy z3=5WVs!lqbk#j(BUq#^vh?jZ%aJlOon$_6uISp(Qrf{N3{6m$&mgpJl%Wa*&j-d1-S^ykO{fC5EyQK)#s+Tk zL~B-VkfBSNr@;nKlN$dmO8nnW-qU3^1jtLg`uLzB80Ou0$8ozgFnjbYkAnF|D zQXiuWTLYErH4YSRoA zU(AoccIls*poyuD`{ggtf(2rdC_Ijq8xJTi?^CS)_`bYXn6?%4UM5}LRk?SDe3bbe z?k#}jGq9~)eAfgD+D#FZJHPKiJuNKQt^VnU)G(cW>*h{FeC2Y?p=c}Z8XL;{^APw+ z7h=Jylz-hc6t;U0445=-&*#QYXOJOqZcrjP?3V1#E#*D`wVBxG#KdCf#j*qh!{;JO z`QyM5e2-qU6t5L|vMJHg03>Y|^~Jkr3H)A9aX&WdUP-)jdZ8)OKff7Km~Bfr#|9{A zt7JdKn%r&NhhJ`hBv!~+6x<{F(N(O$GYa#}x)v!ieMFdx%AmUn ziPmNB{DxhC*M$rU#di4LB+Mwq%p{-jQECcNi47!vJ;QK*%f;YFSQGo9FBdL!^PEsBEjAVFWW&EveRm7py_my;;(Pwf_3$1$s6!I zZm$|_m)jroxHh2H-Vjpn11o{RW}9IR7Y*ne!H^z{$IvM^!IJBF&ZD*NM}T9|Vm08i zevY02Ifbz1;F`g^#OENw|uZndYtjjQ(I^6_b1ajYA(z-`(FRWt#Lfj z2knDP3$6Q?g=vAN&p0^atC*Jd8Nc=Sk0!mri!YidZ8@E(2t3KkF+a1>o3U9m+ReW7 z^h2qmR=^|*msyHlSa%s6vH#7);%CCBH31i+L{{rx<$W@^GTYk2u)oJ{~c?|1W?DT-2X*;^C1bu-}v zuupHb+xDfY53MV`C+A9p#UF(|he*c6ZeaVh?YgQSYTpdruNEz=C{?g~=!+Ob9_=#} zQ$-mWL_3=6yu?v|k-+z$MW>jHd!LK@j%8{3$w*h_$uye9~ljPD8FGm98E~l9b!c%hgXV{xOmXka%7CqM} z@9RPMdTh)U9dG1p>iDL~pI@xwtpIUv>)L=Bm=;e&x_K840^0dPQMoCjmL{_T_)rSaEo-q-1W zq(6=5o0{r{sDAD_x$2XriUW19nCo3}tkw}%d;tG=osoL;x#yZI#1dV!@jm+)f#~ML zGXj>)*5Ez>AtE|St39-U4{y@{g599U0dECa5}!MS3sAvP^!pU{X) zMOmOSbL+pE9G#9`z7EkZR6S5245dduYD;6S-E!n1Uia)0W>BV5-}R@FrmBLoKB^$e zJ3sE0r(8$PQH^{7>V8WSE&3m(5_Jo7sRM+$KBgdZD=qKh>_&?3_Ulu_^??hZF~CJS zNzYB_<8&w9*O41bq&+%mp7U!ZPqaFYSVH;qjiiaaXk$XSkzSxZ{kNZol8oVH4q+10 zdkdWr>`793^!m3X9i1mOb_AlUF&?-7yR++m*#H+hV#|f%8o9BSuL)zgd!!ZE(Yl=b zV&NZ_2G0{+$e-DGEuF8o_yA%-FhJBtaOj>?o9{f04`}$E+eK4A{5@i&b+rLKGAC9u zDYY#E9h!c;ujC65{%lkQ zSGPD4!V%wdZqKA{0qu@bEY}QhwK}tzzA22`qnA=w8=3ePEJ-w+8f=$-{mk_b_3}=K z@oD;_qrZ%ueKECPMPOb`!gb5bKXqWUsdwQBH3Q6OINa|?Q8$r@J-CK z=m%Pg(*}22FDYcKX&E@LPS!q1Q;>6|rLO3o0bw0hbHDr${}b<5xHku~4UHFDS!?BBn=cj%zJyqs1!?$Wn=1_~_?$1|*#px;%OC*d!5eb^dY?8j1( zy1>PxP{T0}KmX2^^I{k~NbW4;=FpF-Diwu(PuxQ0bTiIqu(&$ddTUcV&SmC1pzsu| zWEyu6eFYY;@fe2;gvtW8C!wCNcRB99KX~T*y}Fh8jKA?)z^J*>@(Br6@5jzHG(aj> zDg~^9qDQ%i2seAaTQGjHHw7|bZQJyraT5;lX(+j;xxa|s9m-jR`mEGG_Nl7BRWJCr zQTstzwV_nC7St<%D`R*#Eoe`WaK_l>Ke_N-a3m5P;_cq!=rWmvw+dDr&bPNzH~Wc1 zv{>g04ae@6T`FFo#^>z!6`L)=g3c5It)+cX?K9lD`U^KmGNkiwjg$8)(=VMxWpa$p z^B~=7-f!(+E;0%Ji>y+SIZ~gmdj*Vf?G_jIG*Vu_{p&jwgT;T(ALhO?c6b}vufD;` z9FY@9mry{PPE72ja!s1ds)uw>G{3-0^$lZu$8rIdtn&ENVahgsM+)Ng#g|i?lPeQi zkt@plNp@9?(D&E%+^Jzppb4Rl#$!D*?khNO;p^5a43r%Yr=+#b@FB`OCnvJOKe%EO zMLrLJD~@8_@B(@3!q9#4&b1o$JT#kaka(VuHF61bE3CZ{qwE_!z2= zTdJFv=#7y1-$e4?ZumV{V5Wpz7P`gG?Z++N`zn}cJ^K}D6=Zh#4 z_|vQY9Oj=AYSCkE{`cVd=#4fkEf;Q3)@iA-^aK9xaXh?Z<0DpQPsx>NCqcYOB?0m0yUL_i}O|Eb-b) zadB~Z89|-A94Rm0crfl_;1ZcJw^~x53uZbruaX%!z#)iQ zJ=h$*T;A;EahzF`$@`}BD{b$32r$=I!DGN_0?gS*h7QZ3sWOn?<@TU{b6Ap?U2Sdj zv=rJ%uv>cd)aoY7`3X%3DIhVx8j-36s~zYl<*m%nr=;=Kj;o4h&ornbmQR`HgGvuv zi8!sjUx9u1tDq19{H~$JVgJ+mge94HEt6M%X(JNC_gAjgf4fCn0>Q7}=3w`E3J5wq z<+Qp&^3khPv)#X3FCzvG&K?`z+@pH>Toiwdb&fDM+V-eHd=M@h*k;A(Z=u}^dY=#q zbwRXPOC|zssac=*o4{^|pyq`%q*J*5e>R1uj&)4~|HfsPv^o5R*B#U4LTkSvN>JBI z%#?sDQ49V;=%Ch-(SM3|vl1okZ8pd}XSOIN!Db(AOO$EJ>jAq*00JLdWeW8lt-7xg zk$Z<*t*82;we&UmcFzF{wcaO<-lqK(Se-lQxI}!$Yy$LqMFYFoJ|c6(q#U+-dv73v ziFs*c_da#L>iKk|AHa6RaT>?t{E@2({=5(q!*`q!M1PXg3~SAzY)ft%v`_3^Zyu*T zuXf#go_khhXJ||c`YB32#2N20WG~@sUO0|y~|0h-Jz!S z;)?yIV)JmLi}w{nuam_*a;3Yo2-DXSX>} zG1qUuQBP4PWahnaQYOAbi-T`53cry)a`#&NF1(t|M^7b1C+@T}Wt)&m?=(N5|J_}( z3S0*dZm>!Ebyu~Xe0!hnFdyVS9)IG0J(%XvPp59(xOqQgH{Y;>&R>49D3Wh&enobt zGyhKNPDi?E|4&o>bSf&Vh9L&p2``+qpmd^j2GuaAYj<}$0Y6goDQ6W@UNiyZf4;Wp za}@-U{^V?g7DP(eA*(dvRF4o1i#rON$tsqcffMKLh3R_rIA=t~rEvHT{=G2@&}htr zId!ND;TTn4E2*1G*-9l@soi_KdHrQb9)XoYLmONh1Sb75(FtB9SNp!sbV9eP)ax}c)$+`Bi5yRZiQo7gf1*NQV4-MN- zR`0YIn$=t8&wTH>&Zxd6`kjZ~I-m7ku-8-7r5~rViN@*`W2}?TQ)|fn1_O0{_k-|1 zub8U;;>*HHebWArZj)ORjMI4ULEq97*?vBON}-{GN6+p zH*VQJ+>|`QFB*+)T2`>98g8c9x<=Prm`w7=%Ux>bL1VN133(=(;A>Ew5z zR#)Zi5(kcQ?AHCBtq6_5m9Hh3c1;f%@+F&J)t{jn_ue8%X(SoLA~-c0v;;e6f3G?3 zNF>Tc&Ge`5jmX&Hn|#{^jFdh35ga^>Le2(gqfDm>Tghu)cF%%n`moB6m90;e@lTZY zd2)Zet$-S`SjRQUR(uiNDPmuPK_$#kQ^UVYjnSw57nMknG6_OF>^A5)^~~+3JSG@lbo!9+tYrLXJDdMjqhd#96O1Ek{NdJ zqOu)CE8|J$awA{j(*Cm;{}XG#pT;WZY=|-Whw~wQVdgYPd#d{L!_;xovA_auO|;@J z$~t>#V*Ji`#bWk<`7H3md8moCX>bHC@#w~hf3L(AWe;(Ail9|)M7qgXf8WNAOiDu0 zeXYd&KIV*;7H!aU=H-tMKe=o_udcA3gEx1ZVjRC{Z*1`nrT;`_S$`39J*RNqQ$r9# z*`2K}V?+J%;O>tZi@C(rr*Tr}9xGJks@=r<2Is8T4_CFvJc}QX7Ex8`#w>0hD61DOGT z{r>G&xN+oYi?ese?TBaRX=t(%@y$)d;MxZ@Ha-q1!J8G!`C0R@#S0it&{_A)H`JFH z!{h?Kq$%%Jans+b$981&2D))ew)Q%867E~3XxO_XOb{&?dJOQZWwDX-tEBW{Yz@dj z$3ZY&_r*uveO6n5$mdfZjrZlv7#EU@7l}KW8joX=k0tGVcL;*GqmA09By6idn~fEJ zoPt3soFC7rFP&7SoSo0;{7|9fnTjH^UibHh4M->EnjX1n*C$$1QcmfdSQg8LX88)X zTHlUO_C^Nig9Ma##HMLqsRpJ=itv|S9CudO6H7ZJ* zZIlE9$;QLjUb|2XHzmFS)45e2`zZRBuE5pbJo+Y?kzI*ggMS^9a+cg7yIGx-cq&(} zY+GcBet@7bMIL!OMhM6!mBZZ>{PysW6!2$SF#?=fA@3v8;hT%*a5wiw5%h?I5-xLv zSvw~CkH1zvx2eN&Yb$oXTZt29Q{Hfx4#Y=>znvn+Z)X!|&I_@=&LL4{l{t0tEU~ps zk~$@fDTJ;u$$5OuPi-Z;RywFkuGzKcuW?PHa2~J2H?6xq4y*}Vh3B*>i5!)vxTN+* zjPH#4^WHvgjBY?LTPvY=?PwU>86S40;rfvOWp>ZXDNm&1d?>xfqqxR{mc+}=BnV?n zc`I-g8Vd3aj?z6H>F#)ZH^(Tz~ruZRDMfoQ~xxvBH(R033hy6}s z^{}p2cWy^j(Q}Vz#Cn}L=H~{VhYmZ=-_XtCTR#nw=WosM!VDJq5~s^-n*8GjrI;PK z;wC5c(0wNPZl@1kS+tYCz_75#S<;C`edh1@BD~slJ7YKmuGn)DIM%paH=I^T8(o>R z@@s;4devD$0H)UQBUju^igT<3+{LRl+0@%6{CmSy^t72c>D%=-=OjHo^>%Q`FO` z0s?w)4JLM?wM2Xld7C(4yM*E0e!y%2I*>2$7KXvWA$ zAvx)$osLTYYsL)rYAu<|KS%~?*NgvB{VxAO2`sXfgR~FYnqL#`-bJ6yN|C;aD{jo_ ztO{~2YB3fp;9t;Dj38LKN;}-`Zzd-po5EbP0~JLEt&4M)diQ>!Kq3yyW*oqkLvlzB*0us=Ok*It$Dnr)3Y-X=kXH! zf;y2)h7mmlQsc}KA~{5?HkFw-$sOB>uIq+}NlmtUp2S*24&Be7hI`%AxRtt8G^cqb z#Y^9ofJxDBeDbaV-#7b9!|A!aZXa%d%)0a_6S7nsh5iHl`gTj(vWP32bVKr_2-S!c z>82{<{6J>nZ3Rn871?dauY%;*F^%Z39oc}5 z#u=vr0MVW66dFJGkr{$(g1``=?CKF~!b=wg4Z?3BV#1#k?+O>|HaVnsBlafj)BYq{ zf9)uGQJ-^Z~QVW}L zlpI+4pd#fr zh#rH%D^t;ROeFUb_64|PndnYcnX`Dbm4vg3YH0b?ou5HM7hEyAzd#1VGJ(PoUW^AQ zfl}R$Av(+MYR}6aTIN1kkTj8kjDxFP`v{Jo;}69-yKvcx0jgctP|&X8E(EJ6h|j@~3rAWIfxk!caMF z$uh22f2UB2y(~`3dq01@BQCRKxlo%FJ;&)$q;+n_XZyLLP~u#07ngPzx1?OZif64L zSzc8w6Z64SB$|!3B#7RZiebLxOHt>4{J%mky>W z3w1iz+&e#s?!Kf)fOEC!AJ~>|wFQnPI(>MPk-E_s7n9n`L0k-zwKnz>`J204(f-BF zZS_W^iurJ}{~u9-pp|xR=1vyDo9X`NNqoF#RP)!z5}h|2B94=ZwP@qD5!+c$W)Z<3 ziL9B^Wly!^JAQiJld%sVvCwXuZp=XLs!UxgYDfaXc^@tUoc;VIMYyPY^(Fx~z$PJD z!K0C^7R?87mC}3BcQ74P-o9EjGD=XD(qF(CG+zKcRJBrEkDBux8jL&`pG;`U{^bJ! zSf&8-r`0aAqGLECjK6WV^dxZ&gL{qApuk%=`Z4ye-#SITk>j)@D^@S?jM2woX+OS| zev<1!q^6Rci7pf3?OCd-y2CSqtZxl1vU*<=SBf*P3oBJ)qTP0ZstCJ^pCkJVL86&4 z<80%JcKD*l)Dof{?V!K&5x~UQ1EB-VO8X)c2#>I`xo82^!N}T<=ly@({2UGs=sx(- zMBz515xZr%-6&3+oGAYz#7`+f32r*uZJjjxtjEMKT_XmxL-^Y)>-Bo?hz1Gk{i|Lk z;dgcImVkU$IXvJR26R?FFI9h%(xwg=a148QF<7WQ_(YlR%FVf%g;|}6(3xiSW03pZ zt_`mjrAYH@b2WuB71$}L#;5S!F*x1n4YBt=4<80tI|`rp2ft~Y=8hQ`WD(P#!@z%3 z2@+{;9-6F^OQRkvLquX4&>1(+O&$iCnO`?552J^k_skp8hJhV#Hd|_6Ial!jMh()c zh+*|yzQO#;sGTezGRXs&!)fU;{n!?YuD4DTCRs@4oGA*2d$ZY5W!p0<8YO<^PZ(x! zA|ek9TCDCAqaEzkOO#1wG5~4b*+)@v7cy)9G-}#&r>N+tI7|D|^rHc1c=#Y&2_11kvp{^~*=W)T)^(@S@ zOh?-q=uCk1lISw4I~&yOL^jNQnrT!cIGHFt2byLf7 zZS^oMD0^)S+(DPcElc_68{VsiQ%i3drk^G{djrLnPm!WvU+qzhWmd@MEM+GMNQyWr z%A`O#ZUWpeay+%4w5V69^zgh+b5ZR)u#AD`t*jVFo`7Jz3Wmb(hzS)sCbB*g_%!D` zxd$faT$Ni;FF3&~czu>`W6EFrz9ou;ynd;C`gQJ`N{pl)|dF5dL>Eg?}vhhi7 zf1xl`*`AB%o-9{nVj}_0>Ejvo;!e%b~9R)Z}FKCYGRs%d#e9)oeujUI||o9VZq=3 zBhpc4y~iVoJHMcxZ*|D5$XgUa(~p@`K>BZZroDlkkE*{mX+ilb3?@#waHM5q5>_IE z{|iXl>usO+ofoZgQkHmK9dL_dw;tkpHGwPlou(sWuW_h{>#TV46aO_d3^_NGN&)^# zA^vtfc2;|XqEgd-VDrQ!3@Lcx-F@Ep^VZP)GkS~ zPA-qpuwIuts5yjSC^mR+fD!5MC9r(D2Pj{`lQbubQ~pm~6K9cRl-`UrWh zKP}QB;+A3?Xf2bAO>47*>NxS|D3d>v4&sxObj#`m-HV%3_@&Jwu}zhF7{RE@A4|2w zQt-20@+j3s0N3f0X@(qlhl-V=^EudDfMZxj*wtD!Z;x&1XaYPmud^$SWWjqYoXOaN z7V5?&*i!@4=druOSrn!=4k3&8d%F{7eP@@~4>WUo0$y(4y70-^4+q)n)y6Y{`kK*D z$mQf`kgNe}V4hzTVanYTv1j#Pc^IB?hLw{0S*HD%>7m=7kK_K#jVT6biK*ga)E8E~ z*%Mt!zj9Q(RHg@9_6r!5L{!6?<2h7I4>gOJ({^;Dd9y2zXQlyy-6$D?JfYDtU8@K` zBQjL{7+My%Q}AijfE9@juxfP`{|GnQz%S~1UZwCrTiiV-WGX0;Lkx=^)~tnl$^lA2 zc*K5&YWlB*u*%jM6$L8uA8J`EZd)<_v*)TUbdD`~Gcy1P)jeB~-JvDB3ePH^=ZlRD zB`SK_nKZ}~M*-ZG@;(Xhri9xbHAs=}OhTLT_{Ve9%dKOEIVyms*TKGykgF}UN^-aj z*Dij^i-$3J&Y>^DEb%j#fg7P}oP3ZBzlp3ClQq`jN!#U1( zR8kKMci#d#vJravs~xgkSEq5c%l@zKHC+ApIYDo09Qs4!h8Cv>dSqcr^W>(M1L&O$ zrCXtfYk1gh#>ev9E+QZmQzmZEB=qLS z9k`&K7k(#CX4G7TSygW559jz;?E{X_Y0`VswC~8HD{pl&aw*}H2Rm7XPQ;@bf#Ts} z-Ldo77Ecd6gubkInSrEceXQN@TI2)3LNUK5v*yAphj`T6EI>Di$xGzRxG0V}x)SiX znT~c~A@9ZDZnRt+n`ju!kF&sXb*q(0yD!s+AcNVg)g+xRK9Uz)}jcH@>q? z0OrZR{5dX=yu9DIWpy^4@-{Ww|3tVHat}q77hj`>G0v!;;dHWvBd;$ZQTf2}}-TI{9w1=F~z-#_0+kHnAu84mtd-7jHO zsvOTCeOW91>5*?pRp8DAiO_87YQPU0eZiBHc|nG>x@z~5s{tL2Q1}tQ-~vb}Zh;*; zQO#Iq1I<$PWQM;0S=%itzazk(6}lQtR=*2Of~mV))gY)gu%a~hlqm9CC%!N%2c_YA z{5-y0w7ix@z)hCe%ig_$-vrj^=Iq@O6~9c9(lk^Hnc;$&(_)Uj4_y*&7ggsw4^3pK zrHi?f+KjT7{S{}oA1h8Cj-xjktF`gr=pzM@7XO=kHRs|WLu4ux7bCZ}tT@XBCD?}w zS6h<8?vGK1m4it3opam*L7bM1#l1z)qHMg6XmVi0TDXx|LogO_fhNs)g#yEW6P^ChV6vy__Mysnp{aIh2KQEI_Rg0 z^R6Ic%kM|^qDHbt`~(=ZUA`5Id${i?+W7&-Up|{ZXK!7~j3_2tC?*iJ7HW(j(@m3P zAoItS1G$Z`i8G9fnz`y#(8IgJKPtF?QMZ^;aukk&1<-o{67ZqTP)*ks5ear#_33Bq z3r?*73dqOo6J%<;`Z(uy#+ly9b#UIvBb(8M{ZJRS`9AFR;V*WvigWRB5$_y>LrUbL z2DeRnR1@$}*yw4$VCphF(bwkxoOPZyH%{!SH4ckBKCi8n^t7{SaAPxPm`Vn$ab1M6 zrrr{a_+7>lxnY(3u#{P$^!bqS3kvrA7-u6ig_iHaQUDEClI_40Zp|EUd%-X~i^lFe zme@r?<-|I#G^2-6<;)6WY1sQ$FDY9llZzfiUzQ}5rsDB)-+Xv#Qre2KjgkXwIaY9Y| zq9D=c|GM?@n({8sP199^oej~j?*!--M{xL8$G=qwwQTM>|C_yl$nQB^@2U)-zMmyq zua#vUj?MtsZQb3auv0M#!OscID(^uTr-RZ#hgXa4+VHf<PXncu1ekAUqTk1cj>RCJKaQqb4}h6E|Y=si0(W zIXVXvWj`sW0)Lw{G2|G?RH^!wN!1o7D_9*Ef~=i{Qt$AVo7i+)p`DK%Prtpvu@=f2G z1;ArYUM`?_qMFdW6U@6yE<46o_p(U=a++$D*_yIp!fGuYo$S*F?j!b|`HjQ645?KY z-M^EATX#>ee*ppVzGIk0XQY3_0@LnRgbv4T2G!~EgxEZ~e_N%hu(DDSg>2c2VY|&= zC{{0WjMcw%lsc%IZXR>qh`ps+*yXcWUSameuED3c3Wi=|O-zALY}T*3T++t9Zx;&f zl}fHT{695&jXlsJ3`1(q0fw~S`YlGwiK93qM46d{UK1M^-8w1Y(;Yx{YjKBrgewiK zB?6`1*Otp}gLydF37oRqJi}8f9;>_wFeNXu`_zPL7#l57e0+TM^Q&syEsN69JZNsI zyLLJW!F*T`V&%MS)f<^NP>y%Ne zi-{ted!wUsz`i!{xcKm`-!L^zhW{euVt6J#zW;NJiN4z7?5 z(362TP9+0W0|b|yjYOrM`mYt|XETOK>O7e(=3DIL)0p7$DVh3*E9DKL&5CvDVIhIO z!sxa;o_HSl_rB4oJ-s%?!y)K7y(Buyt;W1LXTjnWzn=ZiqO9%4lz(EQ9FPP9A_d2I zUzM^-RYr&60$aCc6#em69>b5YERl{2SyKBO+ASBlbX9*zJN3CoQE!TbT1Y%#l8^FA zL?%cqs|e!eI4>#RSNq2u^5}uuqm=Z{DtxL(JyVNDS}AH<^B7&@GVosAoO_ByimdAN z)L0(_&93HxZ(&JgyBBwjR6Z2#;^h#RBrY)Z3T=tahpEmbfp!_#B66(HbsQY7Jm5Ay zn*l?@L7@m(AH!#}VK1_(i6>djx-BN=U2Q#^ZPU*s(oAiNnmnGU%v->ase(GQ3->o- z^13vX`rYRvssgJCMg)ip^5nh@ByfM|=E6t`nxAzWoMN3jOzTd{2QENz)d{=n$&B&O z8bd3yeBSJh5ALOwFh%(n7qILJ?njmmut5I4^MpvL5!zLUq!Dn(pV|N-^?kzUy8YR zikXvRZ2uRJVPktwhlDqRdyi}5_iAE$ggyr1huTwH4Dxq84-b8h!ARBqMl8UIH4#eZ zOa4aPAbZ=GdOFu4&2NeSM11neEt{oPbUyl!9K_^hFc)m@vCfhQ{FqT&Mdu?Dv~9~z z9K!_gIr$U6eXy;YM<$>n*OT4QEAG-mh}801!OG`3vZ_Q0dW&^SB-e7M^!_In+K*6W z!K<>)#}non9dT2ryA4p-e#Y4AeEo5Ay3+7FGq)7fSAc)OJIB$;-633ZIa#GXVBHIOBqMxY(i@qBZ<;b@=bPt^JLL4 zv+sHXOAfu^e-!8#7!1<}9&eQW|HJUVV}dY(HS?brJmMv2p`SJ^nl1sLj|lLnt*e6@LABbUgiH+-gQScm3?c}0Y?;mqa)ye;HVS{%>p7I zASlv1p+t)GDkz~NARt|ebZLaO+JEb)RHdfI?_)3)2_UboV#wpboy~(?)_6fzld% zPZ65NjxgVX+txxK%=G?N5nN?a(f}$qct77}>&}`Ims{;M{IDNZUo*>c50&K&|QoW-p=gB@rg5svzq~tI&nm(@pvz+6-QHK7k@i&>}PV1!k!9 zpcs*(aG`PXDm=PhfD(22sJ#RXrDG%<2)$QQ-9#tfe9Rb0?KfohiJY{42Yf>GlqwWg zX>wWsAKI;yf7t%E3 z&OK7)%CBRR0)`}P!z4@Jl5A5&On5?X_i?vWpFmZ8tAat}R1*j!NHW@o%zQe00`a{6 zPrj0Q@fCGAg&O-K*a~hP9=+eN2!tkpv`a?@GYhNxf8jY-TAyZ88C*Z`6I}kvK)3l{ zjbLeHzbwPDXopO?1~8VWikM?8i>g4L2!O-Crr%ZPe$|Yvh1OKB+ z;O~F<@#XWJng++eP7S&On)|Oa@*Kde{#ouJE&~$yXIYd6Si66f{OFXwr(ONH@Rsn4 z(O=5+XH6eoU`YP6ywJPO7^{sPGNkEoWB(Gx5$PRD2q*}NkS>k})kx#kGqgbQ7bHJhy8U>}Gv zh?@Nllhsc0FlpBBnd@{OVsn|zjBwxe*Da>H`ZY^m&~xV|YE!eZ z5QnJ{XBR=Sj?<{9f&MVKsazCN%z6kZ#$YXZLk}Z!UeAvIp%maPL*JV0?1z)|O?0_aF7;pz%ylLkiSW|#Q|^#QhQwzr zeS=AvRiAzS!L2tWnU)*Uez71aK0nC0#Ykeh-e&e!C)e$g3yk?nS^Y#AriBrr)t}pK zl{pEqXnWsuWHg@I9@)kfBRDv07i}%3Z>nXF#M|%JNrc%~v)}R>y_GCnVLN?%e{W4Y zW9N%BUW|i5qgebQw}V1vQe=I{Zcx2iaFNr zn8OU$OcMyvv2WGdq@tx_qTs(4+wJ3esT^Pz{|Vs-h$r9fB~Dg@0ZmXwdPbwRLV|<%P_~Qp>Gb> z!?n|z4zt%UGBKaW&1C#tbDZ?!?{x19f2dIZ;T2X?vHC17JTcy$%{I+p9?_+xG2gx1 z#)@^ZcGAx)nJDG9lbnWCu87+!Pqw)e;gACaVyml%6{Kv|req_XS+1if<7!vh+LtO9 zm=Y~QLVk(WHp*vb*Gj)(mvT-bRn93z*Z~Y`e*0lOx87v4|8&r29&HnI0@us~p3_<* zUvou3QsW?2NXE33g}h18In4<)n6=h}vgNB$0}8fGvh(vWwnaBvo2rFV_rAI+VvBaB ztW}r%-CA1DQS`dfzr}Q1Y+E6iBU8%_goN%mK+UHQEv96j#LKRj_|8}+#@*1&{E5P$ zZD81Jl54fY$5e(3_Z!#x4 zmYPtM@2n%{UImwl%g!aXVR87>AoqZen%;u$0eUav8J99g$XguHpO4(n-jP3_=%&7} zZ7QraqHvhX;qq@D=l7m)1Dmcwi|GLw?BwrQ?vdmXMpNdyuX>kO0)?-uq&ex5F6d{S z8C^D|SLi7PtXAeVg8=+zj78;p{?;)y71`YO8rk z8LAK;=<06MqFQcD>48BNX)zFi5g;-GiM1<|!F2-x`IS30C0#j#ZULZw4y( zI4g(pcuuE#)8FN!KNscRkQgfE=eT*vyu7pd@YeLcGE{mY?VOCpawMV;-kMA-6ZVhw z7C%RN?dL5l)>f@9c00Yiuuy9Uwvs+f$hdH=>MHPF;K~}z5INr6m^tQ+SS)lWyt-$g zIUCBat!u`<5WQ`mADWr&M%&wpZxYrYZ?X*~YOa`?LWEYVOJU02J_rQNvG!Ti!Z1`< z&_s=TOSpnGBM#xiIH6BpKylUI$5zS?u()sEa@5l|!-oT_`79C9<%*CUF#KlCIYEq9 zNZZAunDGwGDyKP$^#z(1AePLk#nCY@x$@|?&6<}cu%aQ!z>ZILSdt$MVRYDWZReLx z&IxtH?fa;M;+l!Nhn{0-ljc6Dq4#UOBk#+gvcntt`I*X$57bafsvR4xFgGnxf+IB| zS9w|?2GCIEdIf|}$xz=KR;lgBbNBs2RAs`@tIlDE$;KwyGr)Oym-%~o!MA@a$@>a= z_o&_gYMn4-%-}R?`$nbQ6GRW(s4RaM=C4rnDxJIKhL~g~rdy4#61eo>0oNu4623>oaYrQkAimSN- zybRY47dT;#hBbV@oO8nk7rOPuI7ls3&b}yAzL#G|VRScI?x!O^)>I6v8t^eO5KETP za$1}3)c)YMB`{hq*~hJVUb}G`{^I~~+Yd**$rB_8_%|Yg;avB>Wa|;sq2_BuSqUfglM zG_?FpQX;@L5zC(sn5(%q3{5(NN^ftbihb+gOXneMUjI|-mt|OkR#xOG)7Q%!S*3uH z&rSA!<%tzm|1ItY{(_7efoWR)*!UE5%u|xzJ*B3TDk|F{2sNn{PJCrBo2dwhdiO}w zIeYE9+A1|p`^zyw2X$!aw(5srI|HGb2*}L!7(m!8{b4}#@U~d>%6ZQ!Bh9FVnM5!M zdFu78so;f{LRwi2BNNLym0w7jxvQ&S4&L2a^DeE zZqmR&Ig6E>cyeYC6gq2lrnkq=a46S5kP0=LU@cbu&v_JaSQidqED;*#$(QI!&D08~ zg>#DQ`>c4`XJ_iPHN(j}y@Z<@sp7)(55odpf=A)tj@q1fEt!|{4vf9YyiCOSe>e== zmhqxz`}JCpo!h3yhU|>A5FzAhFLpT`m!6{ANu-7+?Z&F$uO; zy;{%c+}(+RgATs3PVY|zjlS1&L&$SgnHL|2O?P>$_nw80=?TlVRf42Y(gNQOZ29SZ zgikrR`|-C|)8(9F6~tY&4al($8C)L5)q0pFTh1eFmv;yRhWFIpq{;kVY$B#ShUC?x zz)*%wV!Hd-z}k~hAko_o+dLZ}<56BTiZ9qoZbH)7_`Uf6PD1yv|GcBZ^E&g4X<3yh zSe2n6Zsh4>T;`KgU_ZZ?-(G!P{2Bg5m#`W8yw9rMILo0Wq4*Kz^P2XDy%zvnh$lV) zYpO;V9T)PYBR$N*$%h}c01%N$sSvzs{2MvfCiCa^PtI-~5_0k%pT5o%pf!3G_XtvH zh_GFx_4)zh-No@?;U$`LsaR*njVZ}+I7Gp#nk-tL##@noco4*e5EN?@Z+CFUH#uU0 z4=ZO-aM-GumLG(`5!ZDAUXnN}+lk%WuIK^#hzfiPKgC;MKRh98R#w&l&uP`iJlBs1 za<*tl%8QQP_uqZUt#rVtxP~@`DJrSP%{wpcFxk2RT%NDvHdd_z<>4;-sN#0b z8pSx>i|>km{BGP+*wKb)EF+fbwsS0tSF4Ve4~FAcbAKo1B4T8MI|Vxwz@{mv2zE^l zIHb-qIamT^jPlC9f248?*tf8~FCxmcB^8!@cnbk5bf?}?bQ6kfY=m&e%#Y}QVx3(y ztJrKnqSkBfD<)@;NJv21FNu##$tn#FTbU;AOpNlT{UV-!jj(XZBP(R_v7=gNng8ao zeRr~x8V9Z7fp%6X--0cXN{l zN__W$ds2tXx@^^OiUvU~j9)GvbBDft+GJlTWOw!~-1>E~Cz&}5ga!*e{+qpXSNJ9H zB)c`<{S_JcsV60D!q!s@(da1O-En`Pc~FSJ;G@8h)^>R$?l@Q4kkuA_hL~r@L+=~F zrw6qA?(RCCy?HBUh5LMN`6e6V+g&e<*S(QdqgH0ZT_9raN@otPxKdO!ex^ICj=Knlx1P~#2PU;A!{g6r9 zk7pMJ1eDV4;Ktans5|1o7Ra=!$s?O;nGM*~g%{%=3.41.0 deepspeed-kernels ; sys_platform == 'linux' docutils<0.18 future diff --git a/tests/unit/monitor/test_monitor.py b/tests/unit/monitor/test_monitor.py index 3e04bebfb6c1..d4b3cf43921d 100644 --- a/tests/unit/monitor/test_monitor.py +++ b/tests/unit/monitor/test_monitor.py @@ -7,10 +7,14 @@ from deepspeed.monitor.wandb import WandbMonitor from deepspeed.monitor.csv_monitor import csvMonitor from deepspeed.monitor.config import DeepSpeedMonitorConfig +from deepspeed.monitor.comet import CometMonitor from unit.common import DistributedTest +from unittest.mock import Mock, patch from deepspeed.runtime.config import DeepSpeedConfig +import deepspeed.comm as dist + class TestTensorBoard(DistributedTest): world_size = 2 @@ -97,3 +101,66 @@ def test_empty_csv_monitor(self): assert csv_monitor.enabled == defaults.enabled assert csv_monitor.output_path == defaults.output_path assert csv_monitor.job_name == defaults.job_name + + +class TestCometMonitor(DistributedTest): + world_size = 2 + + def test_comet_monitor(self): + import comet_ml + mock_experiment = Mock() + mock_start = Mock(return_value=mock_experiment) + + config_dict = { + "train_batch_size": 2, + "comet": { + "enabled": True, + "samples_log_interval": 42, + "workspace": "some-workspace", + "project": "some-project", + "api_key": "some-api-key", + "experiment_name": "some-experiment-name", + "experiment_key": "some-experiment-key", + "mode": "get_or_create", + "online": True + } + } + + ds_config = DeepSpeedConfig(config_dict) + + with patch.object(comet_ml, "start", mock_start): + comet_monitor = CometMonitor(ds_config.monitor_config.comet) + + assert comet_monitor.enabled is True + assert comet_monitor.samples_log_interval == 42 + + # experiment should be initialized via comet_ml.start only if rank == 0 + if dist.get_rank() == 0: + mock_start.assert_called_once_with( + api_key="some-api-key", + project="some-project", + workspace="some-workspace", + experiment_key="some-experiment-key", + mode="get_or_create", + online=True, + ) + + mock_experiment.set_name.assert_called_once_with("some-experiment-name") + assert comet_monitor.experiment is mock_experiment + else: + mock_start.assert_not_called() + + def test_empty_comet(self): + import comet_ml + mock_start = Mock() + + config_dict = {"train_batch_size": 2, "comet": {}} + ds_config = DeepSpeedConfig(config_dict) + + with patch.object(comet_ml, "start", mock_start): + comet_monitor = CometMonitor(ds_config.monitor_config.comet) + + defaults = DeepSpeedMonitorConfig().comet + assert comet_monitor.enabled == defaults.enabled + assert comet_monitor.samples_log_interval == defaults.samples_log_interval + mock_start.assert_not_called() From 23173faa4b571c308bcc0d929c8c18905371ed63 Mon Sep 17 00:00:00 2001 From: Nadav Elyahu <88962733+nelyahu@users.noreply.github.com> Date: Wed, 15 May 2024 21:06:36 +0300 Subject: [PATCH 12/25] Improve _configure_optimizer() final optimizer log (#5528) Was providing the optimizer name which was configured, and not optimizer that was actually taking place after this function processing. This is not always aligned. Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> --- deepspeed/runtime/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py index 34263444c1b7..13f335cae6d5 100644 --- a/deepspeed/runtime/engine.py +++ b/deepspeed/runtime/engine.py @@ -1263,7 +1263,7 @@ def _configure_optimizer(self, client_optimizer, model_parameters): else: self.optimizer = basic_optimizer - log_dist("DeepSpeed Final Optimizer = {}".format(self.optimizer_name()), ranks=[0]) + log_dist("DeepSpeed Final Optimizer = {}".format(self.optimizer.__class__.__name__), ranks=[0]) self.compression_scheduler = self._configure_compression_scheduler() self.quantizer = self._configure_quantization() From 7f55b20f3ef409a5fd9c443e31b099af48121043 Mon Sep 17 00:00:00 2001 From: vikram singh shekhawat Date: Thu, 16 May 2024 06:04:25 +0530 Subject: [PATCH 13/25] Enhance testing: Skip fused_optimizer tests if not supported. (#5159) Enhance testing: Skip fused_optimizer tests if not supported. Added condition check to skip fused_optimizer tests if FusedAdam and FusedLamb are not supported by the accelerator. This enhancement ensures that the tests are appropriately skipped when the hardware configuration does not support these optimizers, preventing potential issues. Details: - Introduced a condition check to determine support for FusedAdam and FusedLamb. - If not supported, fused_optimizer tests are skipped to improve test reliability. - Improved compatibility and stability across different hardware configurations. --------- Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> --- tests/unit/elasticity/test_elastic.py | 3 +++ tests/unit/ops/adam/test_cpu_adam.py | 4 +++- tests/unit/ops/adam/test_hybrid_adam.py | 4 +++- .../runtime/half_precision/test_dynamic_loss_scale.py | 2 ++ tests/unit/runtime/half_precision/test_fp16.py | 9 ++++++++- tests/unit/runtime/test_ds_initialize.py | 6 ++++++ 6 files changed, 25 insertions(+), 3 deletions(-) diff --git a/tests/unit/elasticity/test_elastic.py b/tests/unit/elasticity/test_elastic.py index 63633a51914b..92e1520b2c7c 100644 --- a/tests/unit/elasticity/test_elastic.py +++ b/tests/unit/elasticity/test_elastic.py @@ -150,6 +150,7 @@ def test_proper_mbsz(ds_config): class TestNonElasticBatchParams(DistributedTest): world_size = 2 + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") def test(self): config_dict = { "train_batch_size": 2, @@ -182,6 +183,7 @@ def test(self): class TestNonElasticBatchParamsWithOverride(DistributedTest): world_size = 2 + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") def test(self): if not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME]: pytest.skip("This op had not been implemented on this system.", allow_module_level=True) @@ -215,6 +217,7 @@ def test(self): class TestElasticConfigChanged(DistributedTest): world_size = 2 + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") def test(self): config_dict = { "train_batch_size": 2, diff --git a/tests/unit/ops/adam/test_cpu_adam.py b/tests/unit/ops/adam/test_cpu_adam.py index 9a6ff6689446..785cf786acc3 100644 --- a/tests/unit/ops/adam/test_cpu_adam.py +++ b/tests/unit/ops/adam/test_cpu_adam.py @@ -11,7 +11,7 @@ import deepspeed from deepspeed.accelerator import get_accelerator from deepspeed.ops.adam import FusedAdam -from deepspeed.ops.op_builder import CPUAdamBuilder +from deepspeed.ops.op_builder import CPUAdamBuilder, FusedAdamBuilder from unit.common import DistributedTest if not deepspeed.ops.__compatible_ops__[CPUAdamBuilder.NAME]: @@ -62,6 +62,8 @@ class TestCPUAdam(DistributedTest): set_dist_env = False @pytest.mark.skipif(not get_accelerator().is_available(), reason="only supported in CUDA environments.") + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedAdamBuilder.NAME], + reason="FusedAdam is not compatible") def test_fused_adam_equal(self, dtype, model_size): if ("amd" in pytest.cpu_vendor) and (dtype == torch.half): pytest.skip("cpu-adam with half precision not supported on AMD CPUs") diff --git a/tests/unit/ops/adam/test_hybrid_adam.py b/tests/unit/ops/adam/test_hybrid_adam.py index c7ef4890b322..9003e02588c1 100644 --- a/tests/unit/ops/adam/test_hybrid_adam.py +++ b/tests/unit/ops/adam/test_hybrid_adam.py @@ -12,7 +12,7 @@ import deepspeed from deepspeed.accelerator import get_accelerator from deepspeed.ops.adam import FusedAdam, DeepSpeedCPUAdam -from deepspeed.ops.op_builder import CPUAdamBuilder +from deepspeed.ops.op_builder import CPUAdamBuilder, FusedAdamBuilder from unit.common import DistributedTest if not deepspeed.ops.__compatible_ops__[CPUAdamBuilder.NAME]: @@ -43,6 +43,8 @@ class TestHybridAdam(DistributedTest): set_dist_env = False @pytest.mark.skipif(not get_accelerator().is_available(), reason="only supported in CUDA environments.") + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedAdamBuilder.NAME], + reason="FusedAdam is not compatible") def test_hybrid_adam_equal(self, dtype, model_size): if ("amd" in pytest.cpu_vendor) and (dtype == torch.half): pytest.skip("cpu-adam with half precision not supported on AMD CPUs") diff --git a/tests/unit/runtime/half_precision/test_dynamic_loss_scale.py b/tests/unit/runtime/half_precision/test_dynamic_loss_scale.py index f350e08e68a7..38c539c1cc6c 100644 --- a/tests/unit/runtime/half_precision/test_dynamic_loss_scale.py +++ b/tests/unit/runtime/half_precision/test_dynamic_loss_scale.py @@ -10,6 +10,7 @@ import numpy as np from unit.common import DistributedTest from unit.simple_model import SimpleModel +from deepspeed.ops.op_builder import FusedLambBuilder def run_model_step(model, gradient_list): @@ -152,6 +153,7 @@ def test_some_overflow(self): assert optim.cur_iter == expected_iteration +@pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") class TestUnfused(DistributedTest): world_size = 1 diff --git a/tests/unit/runtime/half_precision/test_fp16.py b/tests/unit/runtime/half_precision/test_fp16.py index 5b300053d2a8..cf7a1d8a8183 100644 --- a/tests/unit/runtime/half_precision/test_fp16.py +++ b/tests/unit/runtime/half_precision/test_fp16.py @@ -12,7 +12,7 @@ from unit.simple_model import SimpleModel, SimpleOptimizer, random_dataloader, SimpleMoEModel, sequence_dataloader from deepspeed.utils.torch import required_torch_version from deepspeed.accelerator import get_accelerator -from deepspeed.ops.op_builder import CPUAdamBuilder +from deepspeed.ops.op_builder import CPUAdamBuilder, FusedLambBuilder from deepspeed.moe.utils import split_params_into_different_moe_groups_for_optimizer try: @@ -22,7 +22,11 @@ _amp_available = False amp_available = pytest.mark.skipif(not _amp_available, reason="apex/amp is not installed") +if torch.half not in get_accelerator().supported_dtypes(): + pytest.skip(f"fp16 not supported, valid dtype: {get_accelerator().supported_dtypes()}", allow_module_level=True) + +@pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") class TestLambFP32GradClip(DistributedTest): world_size = 2 @@ -55,6 +59,7 @@ def test(self): model.step() +@pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") class TestLambFP16(DistributedTest): world_size = 2 @@ -231,6 +236,7 @@ def mock_unscale_and_clip_grads(grads_groups_flat, total_norm, apply_scale=True) engine.backward(loss) engine.step() + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") @pytest.mark.parametrize("fused_lamb_legacy", [(False), (True)]) def test_lamb_gradnorm(self, monkeypatch, fused_lamb_legacy: bool): if not get_accelerator().is_fp16_supported(): @@ -495,6 +501,7 @@ def test_adam_basic(self): model.backward(loss) model.step() + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") def test_lamb_basic(self): if not get_accelerator().is_fp16_supported(): pytest.skip("fp16 is not supported") diff --git a/tests/unit/runtime/test_ds_initialize.py b/tests/unit/runtime/test_ds_initialize.py index 169096a6d4e5..9ff99f169f7a 100644 --- a/tests/unit/runtime/test_ds_initialize.py +++ b/tests/unit/runtime/test_ds_initialize.py @@ -20,6 +20,7 @@ from deepspeed.runtime.utils import see_memory_usage from deepspeed.utils.torch import required_torch_version from deepspeed.accelerator import get_accelerator +from deepspeed.ops.op_builder import FusedAdamBuilder @pytest.mark.parametrize('zero_stage', [0, 3]) @@ -67,6 +68,9 @@ def test(self, optimizer_type): def _optimizer_callable(params) -> Optimizer: return AdamW(params=params) + if (optimizer_type is None) and (not deepspeed.ops.__compatible_ops__[FusedAdamBuilder.NAME]): + pytest.skip("FusedAdam is not compatible") + hidden_dim = 10 model = SimpleModel(hidden_dim) @@ -95,6 +99,8 @@ def _optimizer_callable(params) -> Optimizer: class TestConfigOptimizer(DistributedTest): world_size = 1 + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedAdamBuilder.NAME], + reason="FusedAdam is not compatible") def test(self, client_parameters): ds_config = {"train_batch_size": 1, "optimizer": {"type": "Adam", "params": {"lr": 0.001}}} From 8e4f6e48db8e38f5a9fb02dc8851c33a818ebbf4 Mon Sep 17 00:00:00 2001 From: Zixu Wang <61218792+foin6@users.noreply.github.com> Date: Fri, 17 May 2024 01:46:52 +0800 Subject: [PATCH 14/25] Skip the UT cases that use unimplemented op builders. (#5372) Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> Co-authored-by: Logan Adams --- tests/unit/elasticity/test_elastic.py | 13 +++++++------ .../ops/accelerators/test_accelerator_backward.py | 4 ++++ .../ops/accelerators/test_accelerator_forward.py | 4 ++++ tests/unit/ops/lion/test_cpu_lion.py | 7 ++++--- tests/unit/ops/lion/test_lion.py | 2 ++ .../half_precision/test_dynamic_loss_scale.py | 3 ++- tests/unit/runtime/half_precision/test_fp16.py | 14 ++++++++++---- 7 files changed, 33 insertions(+), 14 deletions(-) diff --git a/tests/unit/elasticity/test_elastic.py b/tests/unit/elasticity/test_elastic.py index 92e1520b2c7c..1f7cbbbca214 100644 --- a/tests/unit/elasticity/test_elastic.py +++ b/tests/unit/elasticity/test_elastic.py @@ -12,7 +12,7 @@ from deepspeed.ops.op_builder import FusedAdamBuilder, FusedLambBuilder if not deepspeed.ops.__compatible_ops__[FusedAdamBuilder.NAME]: - pytest.skip("This op had not been implemented on this system.", allow_module_level=True) + pytest.skip("This op has not been implemented on this system.", allow_module_level=True) @pytest.fixture @@ -150,7 +150,8 @@ def test_proper_mbsz(ds_config): class TestNonElasticBatchParams(DistributedTest): world_size = 2 - @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], + reason="FusedLambBuilder has not been implemented on this system.") def test(self): config_dict = { "train_batch_size": 2, @@ -183,10 +184,9 @@ def test(self): class TestNonElasticBatchParamsWithOverride(DistributedTest): world_size = 2 - @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], + reason="FusedLambBuilder has not been implemented on this system.") def test(self): - if not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME]: - pytest.skip("This op had not been implemented on this system.", allow_module_level=True) config_dict = { "train_batch_size": 2, "steps_per_print": 1, @@ -217,7 +217,8 @@ def test(self): class TestElasticConfigChanged(DistributedTest): world_size = 2 - @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], + reason="FusedLambBuilder has not been implemented on this system.") def test(self): config_dict = { "train_batch_size": 2, diff --git a/tests/unit/ops/accelerators/test_accelerator_backward.py b/tests/unit/ops/accelerators/test_accelerator_backward.py index 48e5fbbe7475..4b1b392e933a 100644 --- a/tests/unit/ops/accelerators/test_accelerator_backward.py +++ b/tests/unit/ops/accelerators/test_accelerator_backward.py @@ -9,12 +9,14 @@ import random import copy import os +import deepspeed from torch import nn from deepspeed import DeepSpeedTransformerLayer, DeepSpeedTransformerConfig from deepspeed.accelerator import get_accelerator from unit.modeling import BertConfig, BertLayerNorm, BertEncoder as BertEncoderPostln from unit.modelingpreln import BertEncoder as BertEncoderPreln from unit.common import DistributedTest, is_rocm_pytorch +from deepspeed.ops.op_builder import TransformerBuilder if torch.half not in get_accelerator().supported_dtypes(): pytest.skip(f"fp16 not supported, valid dtype: {get_accelerator().supported_dtypes()}", allow_module_level=True) @@ -257,6 +259,8 @@ class TestCUDABackward(DistributedTest): #This is to flush denorms in forward pass. Please refer to https://github.com/pytorch/pytorch/blob/main/docs/source/notes/numerical_accuracy.rst#reduced-precision-fp16-and-bf16-gemms-and-convolutions-on-amd-instinct-mi200-devices os.environ['ROCBLAS_INTERNAL_FP16_ALT_IMPL'] = '1' + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[TransformerBuilder.NAME], + reason="TransformerBuilder has not been implemented on this system.") def test_backward(self, is_preln, use_fp16, batch_size, hidden_size, seq_len, heads, num_layers, atol): # Only run fp16 test cases on devices with FP16 capability. if not get_accelerator().is_fp16_supported() and (use_fp16 is True or is_preln is False): diff --git a/tests/unit/ops/accelerators/test_accelerator_forward.py b/tests/unit/ops/accelerators/test_accelerator_forward.py index ee9464f63aa1..e2f4ac177f1b 100644 --- a/tests/unit/ops/accelerators/test_accelerator_forward.py +++ b/tests/unit/ops/accelerators/test_accelerator_forward.py @@ -8,12 +8,14 @@ import pytest import random import copy +import deepspeed from torch import nn from unit.modelingpreln import BertEncoder as BertEncoderPreln from unit.modeling import BertLayerNorm, BertConfig, BertEncoder as BertEncoderPostln from deepspeed import DeepSpeedTransformerLayer, DeepSpeedTransformerConfig from deepspeed.accelerator import get_accelerator from unit.common import DistributedTest +from deepspeed.ops.op_builder import TransformerBuilder if torch.half not in get_accelerator().supported_dtypes(): pytest.skip(f"fp16 not supported, valid dtype: {get_accelerator().supported_dtypes()}", allow_module_level=True) @@ -260,6 +262,8 @@ def test_forward(self, batch_size, hidden_size, seq_len, heads, num_layers, is_p class TestCUDAForwardSmallBatchSize(DistributedTest): world_size = 1 + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[TransformerBuilder.NAME], + reason="TransformerBuilder has not been implemented on this system.") def test_forward_with_small_bsz(self, batch_size, small_bsz, hidden_size, seq_len, heads, num_layers, is_preln, use_fp16): # Only run fp16 test cases on devices with FP16 capability. diff --git a/tests/unit/ops/lion/test_cpu_lion.py b/tests/unit/ops/lion/test_cpu_lion.py index 61a069af3257..7d40a98f35b9 100644 --- a/tests/unit/ops/lion/test_cpu_lion.py +++ b/tests/unit/ops/lion/test_cpu_lion.py @@ -14,9 +14,6 @@ from deepspeed.ops.op_builder import CPULionBuilder from unit.common import DistributedTest -if not deepspeed.ops.__compatible_ops__[CPULionBuilder.NAME]: - pytest.skip("cpu-lion is not compatible", allow_module_level=True) - pytest.cpu_vendor = get_cpu_info()["vendor_id_raw"].lower() @@ -62,6 +59,8 @@ class TestCPULion(DistributedTest): set_dist_env = False @pytest.mark.skipif(not get_accelerator().is_available(), reason="only supported in CUDA environments.") + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[CPULionBuilder.NAME], + reason="CPULionBuilder has not been implemented on this system.") def test_fused_lion_equal(self, dtype, model_size): if ("amd" in pytest.cpu_vendor) and (dtype == torch.half): pytest.skip("cpu-lion with half precision not supported on AMD CPUs") @@ -84,6 +83,8 @@ def test_fused_lion_equal(self, dtype, model_size): class TestCPULionGPUError(DistributedTest): + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[CPULionBuilder.NAME], + reason="CPULionBuilder has not been implemented on this system.") def test_cpu_lion_gpu_error(self): model_size = 64 from deepspeed.ops.lion import DeepSpeedCPULion diff --git a/tests/unit/ops/lion/test_lion.py b/tests/unit/ops/lion/test_lion.py index b2c3ac2f52df..507ff72ea51a 100644 --- a/tests/unit/ops/lion/test_lion.py +++ b/tests/unit/ops/lion/test_lion.py @@ -12,6 +12,7 @@ from unit.common import DistributedTest from unit.simple_model import SimpleModel from deepspeed.accelerator import get_accelerator +from deepspeed.ops.op_builder import CPULionBuilder if torch.half not in get_accelerator().supported_dtypes(): pytest.skip(f"fp16 not supported, valid dtype: {get_accelerator().supported_dtypes()}", allow_module_level=True) @@ -27,6 +28,7 @@ class TestLionConfigs(DistributedTest): world_size = 1 reuse_dist_env = True + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[CPULionBuilder.NAME], reason="CPULionBuilder has not been implemented on this system.") def test(self, optimizer, zero_offload, diff --git a/tests/unit/runtime/half_precision/test_dynamic_loss_scale.py b/tests/unit/runtime/half_precision/test_dynamic_loss_scale.py index 38c539c1cc6c..4b263172261c 100644 --- a/tests/unit/runtime/half_precision/test_dynamic_loss_scale.py +++ b/tests/unit/runtime/half_precision/test_dynamic_loss_scale.py @@ -153,7 +153,8 @@ def test_some_overflow(self): assert optim.cur_iter == expected_iteration -@pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") +@pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], + reason="FusedLambBuilder has not been implemented on this system.") class TestUnfused(DistributedTest): world_size = 1 diff --git a/tests/unit/runtime/half_precision/test_fp16.py b/tests/unit/runtime/half_precision/test_fp16.py index cf7a1d8a8183..dba15a969459 100644 --- a/tests/unit/runtime/half_precision/test_fp16.py +++ b/tests/unit/runtime/half_precision/test_fp16.py @@ -26,10 +26,11 @@ pytest.skip(f"fp16 not supported, valid dtype: {get_accelerator().supported_dtypes()}", allow_module_level=True) -@pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") class TestLambFP32GradClip(DistributedTest): world_size = 2 + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], + reason="FusedLambBuilder has not been implemented on this system.") def test(self): if not get_accelerator().is_fp16_supported(): pytest.skip("fp16 is not supported") @@ -59,10 +60,11 @@ def test(self): model.step() -@pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") class TestLambFP16(DistributedTest): world_size = 2 + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], + reason="FusedLambBuilder has not been implemented on this system.") def test__basic(self): if not get_accelerator().is_fp16_supported(): pytest.skip("fp16 is not supported") @@ -90,6 +92,8 @@ def test__basic(self): model.backward(loss) model.step() + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], + reason="FusedLambBuilder has not been implemented on this system.") def test_empty_grad(self): if not get_accelerator().is_fp16_supported(): pytest.skip("fp16 is not supported") @@ -236,8 +240,9 @@ def mock_unscale_and_clip_grads(grads_groups_flat, total_norm, apply_scale=True) engine.backward(loss) engine.step() - @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") @pytest.mark.parametrize("fused_lamb_legacy", [(False), (True)]) + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], + reason="FusedLambBuilder has not been implemented on this system.") def test_lamb_gradnorm(self, monkeypatch, fused_lamb_legacy: bool): if not get_accelerator().is_fp16_supported(): pytest.skip("fp16 is not supported") @@ -501,7 +506,8 @@ def test_adam_basic(self): model.backward(loss) model.step() - @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], reason="lamb is not compatible") + @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedLambBuilder.NAME], + reason="FusedLambBuilder has not been implemented on this system") def test_lamb_basic(self): if not get_accelerator().is_fp16_supported(): pytest.skip("fp16 is not supported") From d3dd8e74546f95a1eb6369dc6ec9671dd393e540 Mon Sep 17 00:00:00 2001 From: Ramya Ramineni <62723901+rraminen@users.noreply.github.com> Date: Thu, 16 May 2024 20:57:00 -0500 Subject: [PATCH 15/25] rocblas -> hipblas changes for ROCm (#5401) Fixes https://github.com/microsoft/DeepSpeed/issues/4989 In addition to this PR, below changes are required to build below extensions successfully. Please note that not all unit tests for these extensions will pass with this PR. More details on the unit test results are below. These unit tests are skipped in CI anyway, so they will not break the CI. - transformer_inference - quantizer - random_ltd - https://github.com/pytorch/pytorch/pull/121030 - https://github.com/microsoft/DeepSpeed/pull/5402 Unit test results (rocm/pytorch:rocm6.1_ubuntu20.04_py3.9_pytorch_2.1.2) on MI200: **transformer_inference:** pytest --color=yes --durations=0 --verbose -s -m "inference_ops" -rF -n 4 unit/ops/transformer/inference Before this PR: ==== 674 failed, 622 skipped, 8 warnings, 1728 errors in 123.66s (0:02:03) ===== After this PR: ========== 555 failed, 983 passed, 1486 skipped, 8 warnings in 14.35s ========== **quantizer:** pytest --color=yes --durations=0 --verbose -s -m "inference_ops" -rF -n 4 unit/ops/quantizer Before this PR: ==== 244 failed, 8 warnings in 48.02s ==== After this PR: ===== 187 failed, 57 passed, 8 warnings in 14.74s ==== I could not find random_ltd related unit tests to run. --------- Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> Co-authored-by: Logan Adams --- csrc/includes/cublas_wrappers.h | 14 ++- csrc/includes/feed_forward.h | 10 +- csrc/includes/gemm_test.h | 32 +++-- csrc/includes/strided_batch_gemm.h | 13 ++- csrc/transformer/cublas_wrappers.cu | 109 ++++++++++++++++-- .../transformer/inference/csrc/pt_binding.cpp | 52 ++++++--- .../includes/inference_cublas_wrappers.h | 93 +++++++++++++-- .../core_ops/blas_kernels/blas_utils.h | 34 +++++- .../gated_activation_kernels_cuda.cu | 2 +- 9 files changed, 293 insertions(+), 66 deletions(-) diff --git a/csrc/includes/cublas_wrappers.h b/csrc/includes/cublas_wrappers.h index b57ff79923fc..2721fb990c7e 100644 --- a/csrc/includes/cublas_wrappers.h +++ b/csrc/includes/cublas_wrappers.h @@ -17,6 +17,7 @@ #include #endif #include +#include int cublas_gemm_ex(cublasHandle_t handle, cublasOperation_t transa, @@ -29,7 +30,9 @@ int cublas_gemm_ex(cublasHandle_t handle, const float* A, const float* B, float* C, -#ifdef __HIP_PLATFORM_AMD__ +// TODO HIP: Remove backward compatibility for torch<=2.0 in future +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo algo = rocblas_gemm_algo_standard); #else cublasGemmAlgo_t algo = CUBLAS_GEMM_DEFAULT); @@ -46,7 +49,8 @@ int cublas_gemm_ex(cublasHandle_t handle, const __half* A, const __half* B, __half* C, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo algo = rocblas_gemm_algo_standard); #else cublasGemmAlgo_t algo = CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -67,7 +71,8 @@ int cublas_strided_batched_gemm(cublasHandle_t handle, int stride_B, int stride_C, int batch, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo algo = rocblas_gemm_algo_standard); #else cublasGemmAlgo_t algo = CUBLAS_GEMM_DEFAULT); @@ -88,7 +93,8 @@ int cublas_strided_batched_gemm(cublasHandle_t handle, int stride_B, int stride_C, int batch, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo algo = rocblas_gemm_algo_standard); #else cublasGemmAlgo_t algo = CUBLAS_GEMM_DEFAULT_TENSOR_OP); diff --git a/csrc/includes/feed_forward.h b/csrc/includes/feed_forward.h index 46e3ba748d52..d2056403d265 100644 --- a/csrc/includes/feed_forward.h +++ b/csrc/includes/feed_forward.h @@ -48,7 +48,9 @@ class FeedForward { weights, input_ptr, out, -#ifdef __HIP_PLATFORM_AMD__ +// TODO HIP: Remove backward compatibility for torch<=2.0 in future +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo(config_.gemm_algos[0])); #else cublasGemmAlgo_t(config_.gemm_algos[0])); @@ -77,7 +79,8 @@ class FeedForward { input_ptr, out_grad, weights_grad, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo(config_.gemm_algos[1])); #else cublasGemmAlgo_t(config_.gemm_algos[1])); @@ -94,7 +97,8 @@ class FeedForward { weights, out_grad, inp_grad_out, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo(config_.gemm_algos[2])); #else cublasGemmAlgo_t(config_.gemm_algos[2])); diff --git a/csrc/includes/gemm_test.h b/csrc/includes/gemm_test.h index 278515174523..de5b55cd3df1 100644 --- a/csrc/includes/gemm_test.h +++ b/csrc/includes/gemm_test.h @@ -67,7 +67,9 @@ class GemmTest { B, A, C, -#ifdef __HIP_PLATFORM_AMD__ +// TODO HIP: Remove backward compatibility for torch<=2.0 in future +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) static_cast(algo)); #else static_cast(algo)); @@ -86,7 +88,8 @@ class GemmTest { A, C, B, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) static_cast(algo)); #else static_cast(algo)); @@ -105,7 +108,8 @@ class GemmTest { B, C, A, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) static_cast(algo)); #else static_cast(algo)); @@ -121,8 +125,11 @@ class GemmTest { float fast_latency = (std::numeric_limits::max)(); int fast_algo = 0; -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) for (int algo = (int)rocblas_gemm_algo_standard; algo <= (int)rocblas_gemm_algo_standard; +#elif defined(__HIP_PLATFORM_AMD__) + for (int algo = (int)HIPBLAS_GEMM_DEFAULT; algo <= (int)HIPBLAS_GEMM_DEFAULT; #else for (int algo = (int)CUBLAS_GEMM_DEFAULT_TENSOR_OP; algo <= (int)CUBLAS_GEMM_ALGO15_TENSOR_OP; @@ -211,7 +218,8 @@ class StridedGemmTest { stride_b, stride_c, bsz, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) static_cast(algo)); #else static_cast(algo)); @@ -245,7 +253,8 @@ class StridedGemmTest { stride_b, stride_c, bsz, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) static_cast(algo)); #else static_cast(algo)); @@ -276,7 +285,8 @@ class StridedGemmTest { stride_b, stride_c, bsz, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) static_cast(algo)); #else static_cast(algo)); @@ -292,11 +302,17 @@ class StridedGemmTest { float fast_latency = (std::numeric_limits::max)(); int fast_algo = 0; -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) for (int algo = (int)rocblas_gemm_algo_standard; algo <= (int)rocblas_gemm_algo_standard; +#else +#ifdef __HIP_PLATFORM_AMD__ + for (int algo = (int)CUBLAS_GEMM_DEFAULT_TENSOR_OP; + algo <= (int)CUBLAS_GEMM_DEFAULT_TENSOR_OP; #else for (int algo = (int)CUBLAS_GEMM_DEFAULT_TENSOR_OP; algo <= (int)CUBLAS_GEMM_ALGO15_TENSOR_OP; +#endif #endif algo++) { int warm_up = 5; diff --git a/csrc/includes/strided_batch_gemm.h b/csrc/includes/strided_batch_gemm.h index 86d1e3dea11a..9767fcf589b8 100644 --- a/csrc/includes/strided_batch_gemm.h +++ b/csrc/includes/strided_batch_gemm.h @@ -77,7 +77,9 @@ class StridedBatchGemm { stride_b, stride_c, bsz, -#ifdef __HIP_PLATFORM_AMD__ +// TODO HIP: Remove backward compatibility for torch<=2.0 in future +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo(_config.gemm_algos[0])); #else cublasGemmAlgo_t(_config.gemm_algos[0])); @@ -105,7 +107,8 @@ class StridedBatchGemm { stride_b, stride_c, _config.batch_size, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo(_config.gemm_algos[0])); #else cublasGemmAlgo_t(_config.gemm_algos[0])); @@ -149,7 +152,8 @@ class StridedBatchGemm { stride_b, stride_c, bsz, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo(_config.gemm_algos[1])); #else cublasGemmAlgo_t(_config.gemm_algos[1])); @@ -178,7 +182,8 @@ class StridedBatchGemm { stride_b, stride_c, bsz, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo(_config.gemm_algos[2])); #else cublasGemmAlgo_t(_config.gemm_algos[2])); diff --git a/csrc/transformer/cublas_wrappers.cu b/csrc/transformer/cublas_wrappers.cu index 7821a8759ab0..d982e65b8a81 100644 --- a/csrc/transformer/cublas_wrappers.cu +++ b/csrc/transformer/cublas_wrappers.cu @@ -5,7 +5,9 @@ #include "cublas_wrappers.h" -#ifdef __HIP_PLATFORM_AMD__ +// TODO HIP: Remove backward compatibility for torch<=2.0 in future +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) int cublas_gemm_ex(rocblas_handle handle, rocblas_operation transa, rocblas_operation transb, @@ -33,7 +35,8 @@ int cublas_gemm_ex(cublasHandle_t handle, cublasGemmAlgo_t algo) #endif { -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_status status = rocblas_gemm_ex(handle, transa, transb, @@ -67,20 +70,39 @@ int cublas_gemm_ex(cublasHandle_t handle, k, (const void*)alpha, (const void*)A, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif (transa == CUBLAS_OP_N) ? m : k, (const void*)B, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif (transb == CUBLAS_OP_N) ? k : n, (const void*)beta, C, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif m, +#if defined(__HIP_PLATFORM_AMD__) && defined(HIPBLAS_V2) + HIPBLAS_COMPUTE_32F, +#elif defined(__HIP_PLATFORM_AMD__) + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif algo); #endif -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) if (status != rocblas_status_success) { #else if (status != CUBLAS_STATUS_SUCCESS) { @@ -96,7 +118,8 @@ int cublas_gemm_ex(cublasHandle_t handle, return 0; } -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) int cublas_gemm_ex(rocblas_handle handle, rocblas_operation transa, rocblas_operation transb, @@ -124,7 +147,8 @@ int cublas_gemm_ex(cublasHandle_t handle, cublasGemmAlgo_t algo) #endif { -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_status status = rocblas_gemm_ex(handle, transa, transb, @@ -158,20 +182,39 @@ int cublas_gemm_ex(cublasHandle_t handle, k, (const void*)alpha, (const void*)A, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_16F, +#else CUDA_R_16F, +#endif (transa == CUBLAS_OP_N) ? m : k, (const void*)B, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_16F, +#else CUDA_R_16F, +#endif (transb == CUBLAS_OP_N) ? k : n, (const void*)beta, (void*)C, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_16F, +#else CUDA_R_16F, +#endif m, +#if defined(__HIP_PLATFORM_AMD__) && defined(HIPBLAS_V2) + HIPBLAS_COMPUTE_32F, +#elif defined(__HIP_PLATFORM_AMD__) + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif algo); #endif -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) if (status != rocblas_status_success) { #else if (status != CUBLAS_STATUS_SUCCESS) { @@ -187,7 +230,8 @@ int cublas_gemm_ex(cublasHandle_t handle, return 0; } -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) int cublas_strided_batched_gemm(rocblas_handle handle, int m, int n, @@ -223,7 +267,8 @@ int cublas_strided_batched_gemm(cublasHandle_t handle, cublasGemmAlgo_t algo) #endif { -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_status status = rocblas_gemm_strided_batched_ex(handle, op_A, @@ -263,24 +308,43 @@ int cublas_strided_batched_gemm(cublasHandle_t handle, k, alpha, A, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif (op_A == CUBLAS_OP_N) ? m : k, stride_A, B, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif (op_B == CUBLAS_OP_N) ? k : n, stride_B, beta, C, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif m, stride_C, batch, +#if defined(__HIP_PLATFORM_AMD__) && defined(HIPBLAS_V2) + HIPBLAS_COMPUTE_32F, +#elif defined(__HIP_PLATFORM_AMD__) + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif algo); #endif -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) if (status != rocblas_status_success) { #else if (status != CUBLAS_STATUS_SUCCESS) { @@ -297,7 +361,8 @@ int cublas_strided_batched_gemm(cublasHandle_t handle, return 0; } -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) int cublas_strided_batched_gemm(rocblas_handle handle, int m, int n, @@ -333,7 +398,8 @@ int cublas_strided_batched_gemm(cublasHandle_t handle, cublasGemmAlgo_t algo) #endif { -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_status status = rocblas_gemm_strided_batched_ex(handle, op_A, @@ -373,24 +439,43 @@ int cublas_strided_batched_gemm(cublasHandle_t handle, k, alpha, A, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_16F, +#else CUDA_R_16F, +#endif (op_A == CUBLAS_OP_N) ? m : k, stride_A, B, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_16F, +#else CUDA_R_16F, +#endif (op_B == CUBLAS_OP_N) ? k : n, stride_B, beta, C, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_16F, +#else CUDA_R_16F, +#endif m, stride_C, batch, +#if defined(__HIP_PLATFORM_AMD__) && defined(HIPBLAS_V2) + HIPBLAS_COMPUTE_32F, +#elif defined(__HIP_PLATFORM_AMD__) + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif algo); #endif -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) if (status != rocblas_status_success) { #else if (status != CUBLAS_STATUS_SUCCESS) { diff --git a/csrc/transformer/inference/csrc/pt_binding.cpp b/csrc/transformer/inference/csrc/pt_binding.cpp index b7277d1e1678..1b9f91cd9c88 100644 --- a/csrc/transformer/inference/csrc/pt_binding.cpp +++ b/csrc/transformer/inference/csrc/pt_binding.cpp @@ -163,7 +163,9 @@ at::Tensor einsum_sec_sm_ecm(at::Tensor& Q, at::Tensor& W) (T*)W.data_ptr(), (T*)Q.data_ptr(), (T*)O.data_ptr(), -#ifdef __HIP_PLATFORM_AMD__ +// TODO HIP: Remove backward compatibility for torch<=2.0 in future +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -216,7 +218,8 @@ void attention_unfused(at::Tensor& prev_key_cont, seq_len * k, seq_len * soft_len, bsz * heads, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -253,7 +256,8 @@ void attention_unfused(at::Tensor& prev_key_cont, seq_len * soft_len, seq_len * k, bsz * heads, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -388,7 +392,8 @@ void attention_unfused(T* prev_key_cont, seq_len * k, seq_len * soft_len, bsz * heads, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -421,7 +426,8 @@ void attention_unfused(T* prev_key_cont, seq_len * soft_len, seq_len * k, bsz * heads, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -886,7 +892,8 @@ void quantized_gemm(void* output, weight16, (T*)input, (T*)output, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -931,7 +938,8 @@ at::Tensor qkv_unfused_cublas(at::Tensor& output, (T*)weight.data_ptr(), workspace, (T*)output.data_ptr(), -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -1003,7 +1011,8 @@ std::vector ds_rms_qkv(at::Tensor& input, (T*)weight.data_ptr(), (T*)rms_norm.data_ptr(), (T*)output.data_ptr(), -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -1089,7 +1098,8 @@ void quantized_gemm(at::Tensor& output, (T*)weight16.data_ptr(), (T*)input.data_ptr(), (T*)output.data_ptr(), -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -1135,7 +1145,8 @@ at::Tensor ds_linear_layer(at::Tensor& input, (T*)weight.data_ptr(), (T*)input_cont.data_ptr(), (T*)output.data_ptr(), -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -1353,7 +1364,8 @@ at::Tensor ds_vector_matmul(at::Tensor& input, (T*)weight.data_ptr(), (T*)input.data_ptr(), (T*)output.data_ptr(), -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -1439,7 +1451,8 @@ at::Tensor mlp_unfused_cublas(at::Tensor& output, (T*)weight.data_ptr(), inp_norm, intermediate, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -1483,7 +1496,8 @@ at::Tensor mlp_unfused_cublas(at::Tensor& output, (T*)weight1.data_ptr(), intermediate, (T*)output.data_ptr(), -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -1617,7 +1631,8 @@ std::vector ds_rms_mlp_gemm(at::Tensor& input, (T*)weight_interm.data_ptr(), (T*)inp_norm.data_ptr(), intermediate_ptr, -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -1680,7 +1695,8 @@ std::vector ds_rms_mlp_gemm(at::Tensor& input, (T*)weight_out.data_ptr(), intermediate_ptr, (T*)output.data_ptr(), -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard, #else CUBLAS_GEMM_DEFAULT_TENSOR_OP, @@ -1742,7 +1758,8 @@ at::Tensor fused_gemm_gelu(at::Tensor& input, (T*)weight.data_ptr(), (T*)input.data_ptr(), (T*)intermediate.data_ptr(), -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); @@ -1776,7 +1793,8 @@ at::Tensor fused_gemm_gelu(at::Tensor& input, (T*)weight_out.data_ptr(), (T*)intermediate.data_ptr(), (T*)output.data_ptr(), -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_gemm_algo_standard); #else CUBLAS_GEMM_DEFAULT_TENSOR_OP); diff --git a/csrc/transformer/inference/includes/inference_cublas_wrappers.h b/csrc/transformer/inference/includes/inference_cublas_wrappers.h index 640751b12c8f..40c3e443941d 100644 --- a/csrc/transformer/inference/includes/inference_cublas_wrappers.h +++ b/csrc/transformer/inference/includes/inference_cublas_wrappers.h @@ -18,7 +18,9 @@ #endif #include -#ifdef __HIP_PLATFORM_AMD__ +// TODO HIP: Remove backward compatibility for torch<=2.0 in future +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) int cublas_gemm_ex(rocblas_handle handle, rocblas_operation transa, rocblas_operation transb, @@ -49,7 +51,8 @@ int cublas_gemm_ex(cublasHandle_t handle, #endif { const int ldb = (b_stride == -1) ? ((transb == CUBLAS_OP_N) ? k : n) : b_stride; -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_status status = rocblas_gemm_ex(handle, transa, transb, @@ -83,20 +86,39 @@ int cublas_gemm_ex(cublasHandle_t handle, k, (const void*)alpha, (const void*)A, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif (transa == CUBLAS_OP_N) ? m : k, (const void*)B, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif ldb, (const void*)beta, C, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif m, +#if defined(__HIP_PLATFORM_AMD__) && defined(HIPBLAS_V2) + HIPBLAS_COMPUTE_32F, +#elif defined(__HIP_PLATFORM_AMD__) + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif algo); #endif -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) if (status != rocblas_status_success) { #else if (status != CUBLAS_STATUS_SUCCESS) { @@ -113,7 +135,8 @@ int cublas_gemm_ex(cublasHandle_t handle, } template -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) int cublas_gemm_ex(rocblas_handle handle, rocblas_operation transa, rocblas_operation transb, @@ -144,7 +167,8 @@ int cublas_gemm_ex(cublasHandle_t handle, #endif { const int ldb = (b_stride == -1) ? ((transb == CUBLAS_OP_N) ? k : n) : b_stride; -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) constexpr auto rocblas_dtype_16 = std::is_same::value ? rocblas_datatype_f16_r : rocblas_datatype_bf16_r; rocblas_status status = rocblas_gemm_ex(handle, @@ -171,8 +195,12 @@ int cublas_gemm_ex(cublasHandle_t handle, algo, 0, 0); +#else +#ifdef __HIP_PLATFORM_AMD__ + constexpr auto cublas_dtype_16 = std::is_same::value ? HIPBLAS_R_16F : HIPBLAS_R_16B; #else constexpr auto cublas_dtype_16 = std::is_same::value ? CUDA_R_16F : CUDA_R_16BF; +#endif cublasStatus_t status = cublasGemmEx(handle, transa, transb, @@ -190,11 +218,18 @@ int cublas_gemm_ex(cublasHandle_t handle, (void*)C, cublas_dtype_16, m, +#if defined(__HIP_PLATFORM_AMD__) && defined(HIPBLAS_V2) + HIPBLAS_COMPUTE_32F, +#elif defined(__HIP_PLATFORM_AMD__) + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif algo); #endif -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) if (status != rocblas_status_success) { #else if (status != CUBLAS_STATUS_SUCCESS) { @@ -210,7 +245,8 @@ int cublas_gemm_ex(cublasHandle_t handle, return 0; } -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) int cublas_strided_batched_gemm(rocblas_handle handle, int m, int n, @@ -246,7 +282,8 @@ int cublas_strided_batched_gemm(cublasHandle_t handle, cublasGemmAlgo_t algo) #endif { -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_status status = rocblas_gemm_strided_batched_ex(handle, op_A, @@ -286,24 +323,43 @@ int cublas_strided_batched_gemm(cublasHandle_t handle, k, alpha, A, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif (op_A == CUBLAS_OP_N) ? m : k, stride_A, B, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif (op_B == CUBLAS_OP_N) ? k : n, stride_B, beta, C, +#ifdef __HIP_PLATFORM_AMD__ + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif m, stride_C, batch, +#if defined(__HIP_PLATFORM_AMD__) && defined(HIPBLAS_V2) + HIPBLAS_COMPUTE_32F, +#elif defined(__HIP_PLATFORM_AMD__) + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif algo); #endif -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) if (status != rocblas_status_success) { #else if (status != CUBLAS_STATUS_SUCCESS) { @@ -321,7 +377,8 @@ int cublas_strided_batched_gemm(cublasHandle_t handle, } template -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) int cublas_strided_batched_gemm(rocblas_handle handle, int m, int n, @@ -357,7 +414,8 @@ int cublas_strided_batched_gemm(cublasHandle_t handle, cublasGemmAlgo_t algo) #endif { -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) constexpr auto rocblas_dtype_16 = std::is_same::value ? rocblas_datatype_f16_r : rocblas_datatype_bf16_r; rocblas_status status = @@ -390,8 +448,12 @@ int cublas_strided_batched_gemm(cublasHandle_t handle, algo, 0, 0); +#else +#ifdef __HIP_PLATFORM_AMD__ + constexpr auto cublas_dtype_16 = std::is_same::value ? HIPBLAS_R_16F : HIPBLAS_R_16B; #else constexpr auto cublas_dtype_16 = std::is_same::value ? CUDA_R_16F : CUDA_R_16BF; +#endif cublasStatus_t status = cublasGemmStridedBatchedEx(handle, op_A, op_B, @@ -413,11 +475,18 @@ int cublas_strided_batched_gemm(cublasHandle_t handle, m, stride_C, batch, +#if defined(__HIP_PLATFORM_AMD__) && defined(HIPBLAS_V2) + HIPBLAS_COMPUTE_32F, +#elif defined(__HIP_PLATFORM_AMD__) + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif algo); #endif -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) if (status != rocblas_status_success) { #else if (status != CUBLAS_STATUS_SUCCESS) { diff --git a/deepspeed/inference/v2/kernels/core_ops/blas_kernels/blas_utils.h b/deepspeed/inference/v2/kernels/core_ops/blas_kernels/blas_utils.h index c02cc76905e0..294db7528699 100644 --- a/deepspeed/inference/v2/kernels/core_ops/blas_kernels/blas_utils.h +++ b/deepspeed/inference/v2/kernels/core_ops/blas_kernels/blas_utils.h @@ -55,7 +55,9 @@ class BlasContext { enum class BlasType { FP32, FP16, BF16 }; -#ifdef __HIP_PLATFORM_AMD__ +// TODO HIP: Remove backward compatibility for torch<=2.0 in future +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_operation get_trans_op(bool do_trans) { return (do_trans) ? rocblas_operation_transpose : rocblas_operation_none; @@ -76,9 +78,15 @@ cublasOperation_t get_trans_op(bool do_trans) { return (do_trans) ? CUBLAS_OP_T cublasDataType_t get_datatype(BlasType type) { switch (type) { +#ifdef __HIP_PLATFORM_AMD__ + case BlasType::FP32: return HIPBLAS_R_32F; + case BlasType::FP16: return HIPBLAS_R_16F; + case BlasType::BF16: return HIPBLAS_R_16B; +#else case BlasType::FP32: return CUDA_R_32F; case BlasType::FP16: return CUDA_R_16F; case BlasType::BF16: return CUDA_R_16BF; +#endif default: throw std::runtime_error("Unsupported BlasType"); } } @@ -99,7 +107,8 @@ int blas_gemm_ex(void* C, const float* beta, BlasType type) { -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_operation_t transa_op = get_trans_op(transa); rocblas_operation_t transb_op = get_trans_op(transb); @@ -151,11 +160,18 @@ int blas_gemm_ex(void* C, C, abc_type, ldc, +#if defined(__HIP_PLATFORM_AMD__) && defined(HIPBLAS_V2) + HIPBLAS_COMPUTE_32F, +#elif defined(__HIP_PLATFORM_AMD__) + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif CUBLAS_GEMM_DEFAULT_TENSOR_OP); #endif -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) if (status != rocblas_status_success) { #else if (status != CUBLAS_STATUS_SUCCESS) { @@ -190,7 +206,8 @@ int blas_strided_batched_gemm(void* C, int batch, BlasType type) { -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) rocblas_operation_t transa_op = get_trans_op(transa); rocblas_operation_t transb_op = get_trans_op(transb); @@ -253,11 +270,18 @@ int blas_strided_batched_gemm(void* C, ldc, stride_C, batch, +#if defined(__HIP_PLATFORM_AMD__) && defined(HIPBLAS_V2) + HIPBLAS_COMPUTE_32F, +#elif defined(__HIP_PLATFORM_AMD__) + HIPBLAS_R_32F, +#else CUDA_R_32F, +#endif CUBLAS_GEMM_DEFAULT_TENSOR_OP); #endif -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) && \ + ((TORCH_VERSION_MAJOR < 2) || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0)) if (status != rocblas_status_success) { #else if (status != CUBLAS_STATUS_SUCCESS) { diff --git a/deepspeed/inference/v2/kernels/core_ops/gated_activations/gated_activation_kernels_cuda.cu b/deepspeed/inference/v2/kernels/core_ops/gated_activations/gated_activation_kernels_cuda.cu index cfa62f94596a..fc14b1831361 100644 --- a/deepspeed/inference/v2/kernels/core_ops/gated_activations/gated_activation_kernels_cuda.cu +++ b/deepspeed/inference/v2/kernels/core_ops/gated_activations/gated_activation_kernels_cuda.cu @@ -17,7 +17,7 @@ constexpr int access_size = 16; constexpr int threads = 1024; template -float gated_act_fn(float x, float y); +DS_D_INLINE float gated_act_fn(float x, float y); template <> DS_D_INLINE float gated_act_fn(float x, float y) From 76c9c69fb1ca053b83f703662ed654ad03f44e4f Mon Sep 17 00:00:00 2001 From: Ramya Ramineni <62723901+rraminen@users.noreply.github.com> Date: Fri, 17 May 2024 15:35:58 -0500 Subject: [PATCH 16/25] Rocm warp size fix (#5402) This PR enables building the below extensions for AMD GPUs with warp size 32. - transformer_inference - quantizer - random_ltd This PR works stand-alone for torch version <=2.0. For the latest versions, https://github.com/microsoft/DeepSpeed/pull/5401 is required to be merged in addition to this PR. Unit test results (rocm/pytorch:rocm6.1_ubuntu20.04_py3.9_pytorch_2.1.2) on NAVI3x: **transformer_inference:** pytest --color=yes --durations=0 --verbose -s -m "inference_ops" -rF -n 4 unit/ops/transformer/inference Before this PR: ===== 674 failed, 622 skipped, 8 warnings, 1728 errors in 69.37s (0:01:09) ===== After this PR: ========== 476 failed, 1062 passed, 1486 skipped, 8 warnings in 9.31s ========== **quantizer:** pytest --color=yes --durations=0 --verbose -s -m "inference_ops" -rF -n 4 unit/ops/quantizer Before this PR: ==== 244 failed, 8 warnings in 30.53s ==== After this PR: ====== 186 failed, 58 passed, 8 warnings in 8.89s ====== I could not find random_ltd related unit tests to run. Fixes: https://github.com/microsoft/DeepSpeed/issues/4753 https://github.com/microsoft/DeepSpeed/issues/5474 https://github.com/ROCm/DeepSpeed/issues/68 cc: @jithunnair-amd --------- Co-authored-by: rraminen@amd.com Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> --- csrc/includes/ds_kernel_utils.h | 2 +- csrc/random_ltd/token_sort.cu | 2 +- .../inference/csrc/apply_rotary_pos_emb.cu | 12 ++----- .../v2/kernels/includes/ds_kernel_utils.h | 2 +- op_builder/builder.py | 36 +++++++++++++++++++ 5 files changed, 41 insertions(+), 13 deletions(-) diff --git a/csrc/includes/ds_kernel_utils.h b/csrc/includes/ds_kernel_utils.h index 8e4888109fcd..f8b16ee6a315 100644 --- a/csrc/includes/ds_kernel_utils.h +++ b/csrc/includes/ds_kernel_utils.h @@ -23,7 +23,7 @@ used throughout the codebase. #ifdef __HIP_PLATFORM_AMD__ // constexpr variant of warpSize for templating -constexpr int hw_warp_size = 64; +constexpr int hw_warp_size = ROCM_WAVEFRONT_SIZE; #define HALF_PRECISION_AVAILABLE = 1 #include #include diff --git a/csrc/random_ltd/token_sort.cu b/csrc/random_ltd/token_sort.cu index 3049471cfe34..3c1dff49429f 100644 --- a/csrc/random_ltd/token_sort.cu +++ b/csrc/random_ltd/token_sort.cu @@ -16,7 +16,7 @@ constexpr int mem_vals = granularity / sizeof(int32_t); constexpr int max_buffer_size = (threads + 1) * mem_vals; #ifdef __HIP_PLATFORM_AMD__ -constexpr int warp_size = 64; +constexpr int warp_size = ROCM_WAVEFRONT_SIZE; #else constexpr int warp_size = 32; #endif diff --git a/csrc/transformer/inference/csrc/apply_rotary_pos_emb.cu b/csrc/transformer/inference/csrc/apply_rotary_pos_emb.cu index a06dbb48fd33..25a494111c54 100644 --- a/csrc/transformer/inference/csrc/apply_rotary_pos_emb.cu +++ b/csrc/transformer/inference/csrc/apply_rotary_pos_emb.cu @@ -99,17 +99,9 @@ __global__ void apply_rotary_pos_half(T* mixed_query, rope_theta, \ max_out_tokens); -#ifdef __HIP_PLATFORM_AMD__ +#if defined(__HIP_PLATFORM_AMD__) and ROCM_WAVEFRONT_SIZE == 64 #define LAUNCH_FOR_ALIGNMENT(ALIGNMENT) \ - if (threads_per_head == 4) { \ - LAUNCH_ROT_POS_EMB_HALF(4, ALIGNMENT); \ - } else if (threads_per_head == 8) { \ - LAUNCH_ROT_POS_EMB_HALF(8, ALIGNMENT); \ - } else if (threads_per_head == 16) { \ - LAUNCH_ROT_POS_EMB_HALF(16, ALIGNMENT); \ - } else if (threads_per_head == 32) { \ - LAUNCH_ROT_POS_EMB_HALF(32, ALIGNMENT); \ - } else if (threads_per_head == 64) { \ + if (threads_per_head == 64) { \ LAUNCH_ROT_POS_EMB_HALF(64, ALIGNMENT); \ } else { \ assert(false); \ diff --git a/deepspeed/inference/v2/kernels/includes/ds_kernel_utils.h b/deepspeed/inference/v2/kernels/includes/ds_kernel_utils.h index 8e4888109fcd..f8b16ee6a315 100644 --- a/deepspeed/inference/v2/kernels/includes/ds_kernel_utils.h +++ b/deepspeed/inference/v2/kernels/includes/ds_kernel_utils.h @@ -23,7 +23,7 @@ used throughout the codebase. #ifdef __HIP_PLATFORM_AMD__ // constexpr variant of warpSize for templating -constexpr int hw_warp_size = 64; +constexpr int hw_warp_size = ROCM_WAVEFRONT_SIZE; #define HALF_PRECISION_AVAILABLE = 1 #include #include diff --git a/op_builder/builder.py b/op_builder/builder.py index 8dc825c7926d..18c130221b0e 100644 --- a/op_builder/builder.py +++ b/op_builder/builder.py @@ -107,6 +107,8 @@ def assert_no_cuda_mismatch(name=""): class OpBuilder(ABC): _rocm_version = None + _rocm_gpu_arch = None + _rocm_wavefront_size = None _is_rocm_pytorch = None _is_sycl_enabled = None _loaded_ops = {} @@ -229,6 +231,32 @@ def installed_rocm_version(): OpBuilder._rocm_version = (int(ROCM_MAJOR), int(ROCM_MINOR)) return OpBuilder._rocm_version + @staticmethod + def get_rocm_gpu_arch(): + if OpBuilder._rocm_gpu_arch: + return OpBuilder._rocm_gpu_arch + rocm_gpu_arch_cmd = "/opt/rocm/bin/rocminfo | grep -o -m 1 'gfx.*'" + try: + result = subprocess.check_output(rocm_gpu_arch_cmd, shell=True) + rocm_gpu_arch = result.decode('utf-8').strip() + except subprocess.CalledProcessError: + rocm_gpu_arch = "" + OpBuilder._rocm_gpu_arch = rocm_gpu_arch + return OpBuilder._rocm_gpu_arch + + @staticmethod + def get_rocm_wavefront_size(): + if OpBuilder._rocm_wavefront_size: + return OpBuilder._rocm_wavefront_size + rocm_wavefront_size_cmd = "/opt/rocm/bin/rocminfo | grep -Eo -m1 'Wavefront Size:[[:space:]]+[0-9]+' | grep -Eo '[0-9]+'" + try: + result = subprocess.check_output(rocm_wavefront_size_cmd, shell=True) + rocm_wavefront_size = result.decode('utf-8').strip() + except subprocess.CalledProcessError: + rocm_wavefront_size = "32" + OpBuilder._rocm_wavefront_size = rocm_wavefront_size + return OpBuilder._rocm_wavefront_size + def include_paths(self): ''' Returns list of include paths, relative to root of deepspeed package (i.e., DeepSpeed/deepspeed) @@ -520,6 +548,8 @@ def jit_load(self, verbose=True): if self.is_rocm_pytorch(): cxx_args.append("-D__HIP_PLATFORM_AMD__=1") + os.environ["PYTORCH_ROCM_ARCH"] = self.get_rocm_gpu_arch() + cxx_args.append('-DROCM_WAVEFRONT_SIZE=%s' % self.get_rocm_wavefront_size()) op_module = load(name=self.name, sources=self.strip_empty_entries(sources), @@ -650,6 +680,12 @@ def builder(self): if self.is_rocm_pytorch(): compile_args['cxx'].append("-D__HIP_PLATFORM_AMD__=1") + #cxx compiler args are required to compile cpp files + compile_args['cxx'].append('-DROCM_WAVEFRONT_SIZE=%s' % self.get_rocm_wavefront_size()) + #nvcc compiler args are required to compile hip files + compile_args['nvcc'].append('-DROCM_WAVEFRONT_SIZE=%s' % self.get_rocm_wavefront_size()) + if self.get_rocm_gpu_arch(): + os.environ["PYTORCH_ROCM_ARCH"] = self.get_rocm_gpu_arch() cuda_ext = ExtensionBuilder(name=self.absolute_name(), sources=self.strip_empty_entries(self.sources()), From 49df8d8da01e378fb1a0685cbcd4350fd0445801 Mon Sep 17 00:00:00 2001 From: Max Kovalenko <75629718+deepcharm@users.noreply.github.com> Date: Mon, 20 May 2024 15:48:56 +0300 Subject: [PATCH 17/25] Optimize zero3 fetch params using all_reduce (#5420) * Use all_reduce instead of all_gather to fetch module parameters. This improves performance by reducing the overhead of concatenation and slicing, which are no longer required. * Instead, all tensors views are created prior to the collective (all_reduce), so upon its completion only the parameter status is updated. * The behavior is enabled via a new boolean flag under the section "zero_optimization": { "stage3_use_all_reduce_for_fetch_params": true } * By default the optimization is not enabled. Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> Co-authored-by: Olatunji Ruwase --- deepspeed/runtime/zero/config.py | 7 + .../runtime/zero/partition_parameters.py | 181 ++++++++++++------ 2 files changed, 125 insertions(+), 63 deletions(-) diff --git a/deepspeed/runtime/zero/config.py b/deepspeed/runtime/zero/config.py index 76583c129cb9..2089d59dbce4 100644 --- a/deepspeed/runtime/zero/config.py +++ b/deepspeed/runtime/zero/config.py @@ -20,6 +20,7 @@ "stage": [0|1|2], "stage3_max_live_parameters" : 1000000000, "stage3_max_reuse_distance" : 1000000000, + "stage3_use_all_reduce_for_fetch_params": [true|false], "allgather_partitions": [true|false], "use_multi_rank_bucket_allreduce": [true|false], "allgather_bucket_size": 500000000, @@ -234,6 +235,12 @@ class DeepSpeedZeroConfig(DeepSpeedConfigModel): this option is enabled and then saves the fp16 model weights. """ + use_all_reduce_for_fetch_params: bool = Field(False, alias="stage3_use_all_reduce_for_fetch_params") + """ + Use all_reduce op when fetching module parameters at stage3. This improves performance by reducing + the overhead of concatenation and slicing on the host. + """ + stage3_gather_fp16_weights_on_model_save: bool = Field(False, deprecated=True, new_param="gather_16bit_weights_on_model_save") diff --git a/deepspeed/runtime/zero/partition_parameters.py b/deepspeed/runtime/zero/partition_parameters.py index c8099791f882..b2909145c50c 100755 --- a/deepspeed/runtime/zero/partition_parameters.py +++ b/deepspeed/runtime/zero/partition_parameters.py @@ -23,7 +23,7 @@ from deepspeed.utils import groups import deepspeed -from ..utils import see_memory_usage +from ..utils import see_memory_usage, get_only_unique_item from deepspeed.runtime.zero.config import DeepSpeedZeroConfig from deepspeed.runtime.zero.utils import assert_ints_same_as_other_ranks, is_zero_param from deepspeed.runtime.zero.offload_config import OffloadDeviceEnum @@ -716,6 +716,31 @@ def wait(self) -> None: handle.wait() +class AllReduceCoalescedHandle: + + def __init__(self, handle, params: List[Parameter]) -> None: + self.handle = handle + self.params = params + self.complete = False + + for param in self.params: + if param.ds_status != ZeroParamStatus.INFLIGHT: + raise RuntimeError(f"expected param {param.ds_summary()} to not be available") + + @instrument_w_nvtx + def wait(self) -> None: + if self.complete: + return + + instrument_w_nvtx(self.handle.wait)() + + for param in self.params: + assert param.ds_status == ZeroParamStatus.INFLIGHT, f"expected param {param.ds_summary()} to be inflight" + param.ds_status = ZeroParamStatus.AVAILABLE + + self.complete = True + + class QuantizationInfo: # a placeholder object to store all quant related vars used in handles def __init__(self) -> None: @@ -1003,6 +1028,11 @@ def __init__( if not self.use_all_gather_into_tensor: logger.info(f"all_gather_into_tensor API is not available in torch {torch.__version__}") + self.use_all_reduce_for_fetch_params = get_config_default(DeepSpeedZeroConfig, + "use_all_reduce_for_fetch_params") + if _ds_config is not None: + self.use_all_reduce_for_fetch_params = _ds_config.zero_config.use_all_reduce_for_fetch_params + def _update_persist_config(self, ds_config): Init.apply_param_persistence = True Init.param_persistence_threshold = ds_config.zero_config.param_persistence_threshold @@ -1250,75 +1280,99 @@ def all_gather_coalesced(params: Iterable[Parameter], return AllGatherHandle(handle, param, quantization=quant_info) else: - if not quantize: - dtype_params = defaultdict(list) - for p in params: - dtype_params[p.ds_tensor.dtype].append(p) - handles = [] - for dtype, params in dtype_params.items(): - handles.append(_all_gather_dtype(dtype, params, world_size, rank_in_group, ds_process_group)) + if self.use_all_reduce_for_fetch_params and not quantize and not use_secondary_tensor: + # Use all_reduce instead of all_gather to fetch the module params + flat_buffer_size = sum(p.ds_numel_aligned for p in params) + flat_tensor = torch.zeros(flat_buffer_size, + dtype=get_only_unique_item(p.ds_tensor.dtype for p in params), + device=get_accelerator().current_device_name(), + requires_grad=False) + start_param = 0 + for param in params: + param.data = flat_tensor.narrow(0, start_param, param.ds_numel).view(param.ds_shape) + start = start_param + param.ds_tensor.ds_numel * self.get_partition_rank() + flat_tensor.narrow(0, start, param.ds_tensor.ds_numel).copy_(param.ds_tensor) - return MultipleAllGatherHandles(handles) + start_param += param.ds_numel + handle = dist.all_reduce(flat_tensor, group=ds_process_group, async_op=True) + + return AllReduceCoalescedHandle(handle=handle, params=params) else: - partition_sz = sum(p.ds_tensor.ds_numel for p in params) + if not quantize: + dtype_params = defaultdict(list) + for p in params: + dtype_params[p.ds_tensor.dtype].append(p) + handles = [] + for dtype, params in dtype_params.items(): + handles.append( + _all_gather_dtype(dtype, params, world_size, rank_in_group, ds_process_group)) - if use_secondary_tensor: - partition_sz = sum(p.ds_tensor.ds_numel * p.ds_secondary_tensor_num_of_groups for p in params) + return MultipleAllGatherHandles(handles) - flat_tensor = torch.empty(partition_sz * world_size, - dtype=torch.int8, - device=get_accelerator().current_device_name(), - requires_grad=False) - - if use_secondary_tensor: - if hasattr(params[0].ds_secondary_tensor, "ds_quant_scale"): - quantized_param = instrument_w_nvtx(torch.cat)([ - p.ds_secondary_tensor.data.to(get_accelerator().current_device_name()) for p in params - ]) - scales = instrument_w_nvtx(torch.cat)([ - p.ds_secondary_tensor.ds_quant_scale.to(get_accelerator().current_device_name()) - for p in params - ]) - else: - quantized_param, scales = self.quantizer_module.quantize( - instrument_w_nvtx(torch.cat)([ - p.ds_secondary_tensor.to(get_accelerator().current_device_name()) for p in params - ])) else: - if hasattr(params[0].ds_tensor, "ds_quant_scale"): - quantized_param = instrument_w_nvtx(torch.cat)( - [p.ds_tensor.data.to(get_accelerator().current_device_name()) for p in params]) - scales = instrument_w_nvtx(torch.cat)([ - p.ds_tensor.ds_quant_scale.to(get_accelerator().current_device_name()) for p in params - ]) + partition_sz = sum(p.ds_tensor.ds_numel for p in params) + + if use_secondary_tensor: + partition_sz = sum(p.ds_tensor.ds_numel * p.ds_secondary_tensor_num_of_groups + for p in params) + + flat_tensor = torch.empty(partition_sz * world_size, + dtype=torch.int8, + device=get_accelerator().current_device_name(), + requires_grad=False) + + if use_secondary_tensor: + if hasattr(params[0].ds_secondary_tensor, "ds_quant_scale"): + quantized_param = instrument_w_nvtx(torch.cat)([ + p.ds_secondary_tensor.data.to(get_accelerator().current_device_name()) + for p in params + ]) + scales = instrument_w_nvtx(torch.cat)([ + p.ds_secondary_tensor.ds_quant_scale.to(get_accelerator().current_device_name()) + for p in params + ]) + else: + quantized_param, scales = self.quantizer_module.quantize( + instrument_w_nvtx(torch.cat)([ + p.ds_secondary_tensor.to(get_accelerator().current_device_name()) + for p in params + ])) else: - quantized_param, scales = self.quantizer_module.quantize( - instrument_w_nvtx(torch.cat)( - [p.ds_tensor.to(get_accelerator().current_device_name()) for p in params])) - quant_scale_buffer = torch.empty( - scales.numel() * world_size, - dtype=torch.float32, - device=get_accelerator().current_device_name(), - requires_grad=False, - ) - handle = _dist_allgather_fn(quantized_param, flat_tensor, ds_process_group) - quant_handle = _dist_allgather_fn(scales, quant_scale_buffer, ds_process_group) - quant_info = QuantizationInfo() - quant_info.quantized_param = flat_tensor - quant_info.backend = self.quantizer_module - quant_info.quant_handle = quant_handle - quant_info.scale_buffer = quant_scale_buffer - quant_info.partition_sz = partition_sz - quant_info.world_size = world_size - return AllGatherCoalescedHandle( - allgather_handle=handle, - params=params, - partitions=None, - world_size=world_size, - use_secondary_tensor=use_secondary_tensor, - quantization=quant_info, - ) + if hasattr(params[0].ds_tensor, "ds_quant_scale"): + quantized_param = instrument_w_nvtx(torch.cat)( + [p.ds_tensor.data.to(get_accelerator().current_device_name()) for p in params]) + scales = instrument_w_nvtx(torch.cat)([ + p.ds_tensor.ds_quant_scale.to(get_accelerator().current_device_name()) + for p in params + ]) + else: + quantized_param, scales = self.quantizer_module.quantize( + instrument_w_nvtx(torch.cat)( + [p.ds_tensor.to(get_accelerator().current_device_name()) for p in params])) + quant_scale_buffer = torch.empty( + scales.numel() * world_size, + dtype=torch.float32, + device=get_accelerator().current_device_name(), + requires_grad=False, + ) + handle = _dist_allgather_fn(quantized_param, flat_tensor, ds_process_group) + quant_handle = _dist_allgather_fn(scales, quant_scale_buffer, ds_process_group) + quant_info = QuantizationInfo() + quant_info.quantized_param = flat_tensor + quant_info.backend = self.quantizer_module + quant_info.quant_handle = quant_handle + quant_info.scale_buffer = quant_scale_buffer + quant_info.partition_sz = partition_sz + quant_info.world_size = world_size + return AllGatherCoalescedHandle( + allgather_handle=handle, + params=params, + partitions=None, + world_size=world_size, + use_secondary_tensor=use_secondary_tensor, + quantization=quant_info, + ) def partition(param_list=None, hierarchy=0, has_been_updated=False): cls = param @@ -1554,6 +1608,7 @@ def _partition_param(self, param, buffer=None, has_been_updated=False): param.ds_tensor.ds_numel = partition_size param.ds_tensor.status = PartitionedParamStatus.AVAILABLE param.ds_tensor.final_location = final_location + param.ds_numel_aligned = tensor_size start = partition_size * self.get_partition_rank() end = start + partition_size From 69af3611679609cde7a3100483c75f8e8d6937e1 Mon Sep 17 00:00:00 2001 From: Liran Bachar Date: Mon, 20 May 2024 15:50:20 +0300 Subject: [PATCH 18/25] CPUAdam fp16 and bf16 support (#5409) Hi. Please review the following changes I added support for BF16 to cpu adam. BF16, FP16 and float are supported at compilation time. the correct template is called at runtime according to input params dtype. --------- Co-authored-by: Olatunji Ruwase --- csrc/adagrad/cpu_adagrad.cpp | 207 +++++++---------- csrc/adam/cpu_adam.cpp | 3 - csrc/adam/cpu_adam_impl.cpp | 254 ++++++++------------- csrc/common/custom_cuda_kernel.cu | 44 ---- csrc/includes/cpu_adagrad.h | 150 +++--------- csrc/includes/cpu_adam.h | 173 +++----------- csrc/includes/cpu_lion.h | 162 +++---------- csrc/includes/custom_cuda_layers.h | 3 - csrc/includes/simd.h | 136 +++++++++-- csrc/lion/cpu_lion.cpp | 3 - csrc/lion/cpu_lion_impl.cpp | 211 +++++++---------- deepspeed/ops/adagrad/cpu_adagrad.py | 17 +- deepspeed/ops/adam/cpu_adam.py | 23 +- deepspeed/ops/lion/cpu_lion.py | 20 +- op_builder/builder.py | 1 + op_builder/cpu/builder.py | 6 +- op_builder/cpu_adagrad.py | 20 +- op_builder/cpu_adam.py | 21 +- op_builder/cpu_lion.py | 25 +- op_builder/hpu/builder.py | 6 +- op_builder/hpu/cpu_adam.py | 5 - tests/perf/adam_test1.py | 4 +- tests/unit/common.py | 8 +- tests/unit/ops/adagrad/test_cpu_adagrad.py | 4 +- tests/unit/ops/adam/test_cpu_adam.py | 33 +-- tests/unit/ops/adam/test_hybrid_adam.py | 6 +- tests/unit/ops/lion/test_cpu_lion.py | 6 +- 27 files changed, 530 insertions(+), 1021 deletions(-) delete mode 100644 csrc/common/custom_cuda_kernel.cu diff --git a/csrc/adagrad/cpu_adagrad.cpp b/csrc/adagrad/cpu_adagrad.cpp index 563255176500..5790e79e2bc2 100644 --- a/csrc/adagrad/cpu_adagrad.cpp +++ b/csrc/adagrad/cpu_adagrad.cpp @@ -5,55 +5,38 @@ #include "cpu_adagrad.h" #include +#include #include +#include #include #include #include -#if defined(__ENABLE_CUDA__) -#include -#include "cublas_v2.h" -#include "cuda.h" -#include "curand.h" -#include "custom_cuda_layers.h" -#endif +using namespace std::string_literals; static std::unordered_map> s_optimizers; // C++ interface -void Adagrad_Optimizer::Step_1(float* _params, - float* grads, - float* _exp_avg_sq, - size_t _param_size, - ds_half_precision_t* dev_params, - bool half_precision) +template +void Adagrad_Optimizer::Step_1(ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg_sq, + size_t _param_size) { size_t rounded_size = 0; #if defined(__AVX512__) or defined(__AVX256__) - Step_AVX<1>( - &rounded_size, _params, grads, _exp_avg_sq, _param_size, dev_params, half_precision); + Step_AVX<1>(&rounded_size, _params, grads, _exp_avg_sq, _param_size); #endif if (_param_size > rounded_size) { float step_size = -1 * _alpha; - ds_half_precision_t* grads_cast_h; - ds_half_precision_t* params_cast_h; - if (half_precision) { - grads_cast_h = reinterpret_cast(grads); - params_cast_h = reinterpret_cast(_params); - } for (size_t t = rounded_size; t < _param_size; t += TILE) { size_t copy_size = TILE; if ((t + TILE) > _param_size) copy_size = _param_size - t; size_t offset = copy_size + t; -#if defined(__ENABLE_CUDA__) - if ((t / TILE) >= 2) { cudaStreamSynchronize(_streams[_buf_index]); } -#elif defined(__ENABLE_CANN__) - if ((t / TILE) >= 2) { aclrtSynchronizeStream(_streams[_buf_index].stream()); } -#endif #pragma omp parallel for for (size_t k = t; k < offset; k++) { - float grad = half_precision ? (float)grads_cast_h[k] : grads[k]; - float param = half_precision ? (float)params_cast_h[k] : _params[k]; + float grad = (float)grads[k]; + float param = (float)_params[k]; float momentum = grads[k]; float variance = _exp_avg_sq[k]; if (_weight_decay > 0) { grad = param * _weight_decay + grad; } @@ -64,58 +47,30 @@ void Adagrad_Optimizer::Step_1(float* _params, grad += _eps; grad = momentum / grad; param = grad * step_size + param; -#if defined(__ENABLE_CUDA__) or defined(__ENABLE_CANN__) - if (dev_params) _doubled_buffer[_buf_index][k - t] = param; -#endif - if (half_precision) - params_cast_h[k] = (ds_half_precision_t)param; - else - _params[k] = param; + _params[k] = param; // STORE UPDATE TERM TO GRAD'S MEMORY grads[k] = grad * step_size; _exp_avg_sq[k] = variance; } -#if defined(__ENABLE_CUDA__) - if (dev_params) { - launch_param_update( - _doubled_buffer[_buf_index], dev_params + t, (copy_size), _streams[_buf_index]); - _buf_index = !_buf_index; - } -#elif defined(__ENABLE_CANN__) - if (dev_params) { - size_t memcpy_size = copy_size * sizeof(_doubled_buffer[_buf_index][0]); - aclrtMemcpy(dev_params + t, - memcpy_size, - _doubled_buffer[_buf_index], - memcpy_size, - aclrtMemcpyKind::ACL_MEMCPY_HOST_TO_DEVICE); - - _buf_index = !_buf_index; - } -#endif } } } -void Adagrad_Optimizer::Step_4(float* _params, - float* grads, - float* _exp_avg_sq, - size_t _param_size, - ds_half_precision_t* dev_params, - bool half_precision) +template +void Adagrad_Optimizer::Step_4(ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg_sq, + size_t _param_size) { size_t rounded_size = 0; #if defined(__AVX512__) or defined(__AVX256__) - Step_AVX<4>( - &rounded_size, _params, grads, _exp_avg_sq, _param_size, dev_params, half_precision); + Step_AVX<4>(&rounded_size, _params, grads, _exp_avg_sq, _param_size); #endif if (_param_size > rounded_size) Step_1((_params + rounded_size), (grads + rounded_size), (_exp_avg_sq + rounded_size), - (_param_size - rounded_size), - (dev_params != nullptr ? (dev_params + rounded_size) : dev_params), - half_precision); + (_param_size - rounded_size)); } int create_adagrad_optimizer(int optimizer_id, @@ -149,25 +104,77 @@ int create_adagrad_optimizer(int optimizer_id, return 0; } -void Adagrad_Optimizer::Step_8(float* _params, - float* grads, - float* _exp_avg_sq, - size_t _param_size, - ds_half_precision_t* dev_params, - bool half_precision) +template +void Adagrad_Optimizer::Step_8(ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg_sq, + size_t _param_size) { size_t rounded_size = 0; #if defined(__AVX512__) or defined(__AVX256__) - Step_AVX<8>( - &rounded_size, _params, grads, _exp_avg_sq, _param_size, dev_params, half_precision); + Step_AVX<8>(&rounded_size, _params, grads, _exp_avg_sq, _param_size); #endif if (_param_size > rounded_size) Step_4((_params + rounded_size), (grads + rounded_size), (_exp_avg_sq + rounded_size), - (_param_size - rounded_size), - (dev_params != nullptr ? (dev_params + rounded_size) : dev_params), - half_precision); + (_param_size - rounded_size)); +} + +template +void step_invoker(std::shared_ptr opt, + void* _params, + void* grads, + void* _exp_avg_sq, + size_t _param_size) +{ + opt->Step_8((ds_params_percision_t*)(_params), + (ds_params_percision_t*)(grads), + (ds_state_precision_t*)(_exp_avg_sq), + _param_size); +} + +std::map, + std::function, void*, void*, void*, size_t)>> + invokers; + +// Fill map with template functions for each type +template +void create_invoker() +{ + invokers[std::tuple(c10::CppTypeToScalarType(), + c10::CppTypeToScalarType())] = + step_invoker; +} +struct InvokerInitializer { + InvokerInitializer() + { + create_invoker(); + create_invoker(); + create_invoker(); + create_invoker(); + create_invoker(); + } +} _invoker_initializer; + +void invoke(std::shared_ptr opt, + torch::Tensor& params, + torch::Tensor& grads, + torch::Tensor& exp_avg_sq, + size_t param_size) +{ + c10::ScalarType params_type = at::typeMetaToScalarType(params.options().dtype()); + c10::ScalarType state_type = at::typeMetaToScalarType(exp_avg_sq.options().dtype()); + + auto it = invokers.find(std::tuple(params_type, state_type)); + if (it == invokers.end()) { + throw std::runtime_error("Adagrad optimizer with param type "s + + c10::toString(params_type) + " and state type "s + + c10::toString(state_type) + + " is not supported on current hardware"s); + } + + it->second(opt, params.data_ptr(), grads.data_ptr(), exp_avg_sq.data_ptr(), param_size); } int ds_adagrad_step(int optimizer_id, @@ -183,58 +190,13 @@ int ds_adagrad_step(int optimizer_id, auto grads_c = grads.contiguous(); auto exp_avg_sq_c = exp_avg_sq.contiguous(); - float* params_ptr = (float*)params_c.data_ptr(); - float* grads_ptr = (float*)grads_c.data_ptr(); - float* exp_avg_sq_ptr = (float*)exp_avg_sq_c.data_ptr(); - std::shared_ptr opt = std::static_pointer_cast(s_optimizers[optimizer_id]); opt->IncrementStep(step); opt->update_state(lr, epsilon, weight_decay); - opt->Step_8(params_ptr, grads_ptr, exp_avg_sq_ptr, params_c.numel()); -#if defined(__ENABLE_CUDA__) or defined(__ENABLE_CANN__) - opt->SynchronizeStreams(); -#endif - return 0; -} + invoke(opt, params_c, grads_c, exp_avg_sq_c, params_c.numel()); -int ds_adagrad_step_plus_copy(int optimizer_id, - size_t step, - float lr, - float epsilon, - float weight_decay, - torch::Tensor& params, - torch::Tensor& grads, - torch::Tensor& exp_avg_sq, - torch::Tensor& gpu_params) -{ -#if defined(__ENABLE_CUDA__) or defined(__ENABLE_CANN__) - auto params_c = params.contiguous(); - auto gpu_params_c = gpu_params.contiguous(); - auto exp_avg_sq_c = exp_avg_sq.contiguous(); - auto grads_c = grads.contiguous(); - - float* params_ptr = (float*)params_c.data_ptr(); - float* grads_ptr = (float*)grads_c.data_ptr(); - ds_half_precision_t* gpu_params_ptr = (ds_half_precision_t*)gpu_params_c.data_ptr(); - float* exp_avg_sq_ptr = (float*)exp_avg_sq_c.data_ptr(); - - std::shared_ptr opt = - std::static_pointer_cast(s_optimizers[optimizer_id]); - opt->IncrementStep(step); - opt->update_state(lr, epsilon, weight_decay); - opt->Step_8(params_ptr, - grads_ptr, - exp_avg_sq_ptr, - params_c.numel(), - gpu_params_ptr, - (params.options().dtype() == at::kHalf)); - - opt->SynchronizeStreams(); -#else - assert(false); -#endif return 0; } @@ -248,9 +210,6 @@ int destroy_adagrad_optimizer(int optimizer_id) PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("adagrad_update", &ds_adagrad_step, "DeepSpeed CPU Adagrad update (C++)"); - m.def("adagrad_update_copy", - &ds_adagrad_step_plus_copy, - "DeepSpeed CPU Adagrad update and param copy (C++)"); m.def("create_adagrad", &create_adagrad_optimizer, "DeepSpeed CPU Adagrad (C++)"); m.def("destroy_adagrad", &destroy_adagrad_optimizer, "DeepSpeed CPU Adagrad destroy (C++)"); } diff --git a/csrc/adam/cpu_adam.cpp b/csrc/adam/cpu_adam.cpp index 96809827f3e1..263c443cb4d4 100644 --- a/csrc/adam/cpu_adam.cpp +++ b/csrc/adam/cpu_adam.cpp @@ -8,9 +8,6 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("adam_update", &ds_adam_step, "DeepSpeed CPU Adam update (C++)"); - m.def("adam_update_copy", - &ds_adam_step_plus_copy, - "DeepSpeed CPU Adam update and param copy (C++)"); m.def("create_adam", &create_adam_optimizer, "DeepSpeed CPU Adam (C++)"); m.def("destroy_adam", &destroy_adam_optimizer, "DeepSpeed CPU Adam destroy (C++)"); } diff --git a/csrc/adam/cpu_adam_impl.cpp b/csrc/adam/cpu_adam_impl.cpp index 9a4a8d956519..15d4e74d69d5 100644 --- a/csrc/adam/cpu_adam_impl.cpp +++ b/csrc/adam/cpu_adam_impl.cpp @@ -5,42 +5,29 @@ #include #include +#include #include +#include #include #include #include #include "cpu_adam.h" -#if defined(__ENABLE_CUDA__) -#include -#include "cublas_v2.h" -#include "cuda.h" -#include "curand.h" -#include "custom_cuda_layers.h" -#endif - +using namespace std::string_literals; static std::unordered_map> s_optimizers; // C++ interface -void Adam_Optimizer::Step_1(float* _params, - float* grads, - float* _exp_avg, - float* _exp_avg_sq, - size_t _param_size, - ds_half_precision_t* dev_params, - bool half_precision) +template +void Adam_Optimizer::Step_1(ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg, + ds_state_precision_t* _exp_avg_sq, + size_t _param_size) { size_t rounded_size = 0; #if defined(__AVX512__) or defined(__AVX256__) - Step_AVX<1>(&rounded_size, - _params, - grads, - _exp_avg, - _exp_avg_sq, - _param_size, - dev_params, - half_precision); + Step_AVX<1>(&rounded_size, _params, grads, _exp_avg, _exp_avg_sq, _param_size); #endif if (_param_size > rounded_size) { float betta1_minus1 = 1 - _betta1; @@ -48,26 +35,15 @@ void Adam_Optimizer::Step_1(float* _params, float step_size = -1 * _alpha / _bias_correction1; float w_decay = -1 * _alpha * _weight_decay; - ds_half_precision_t* grads_cast_h; - ds_half_precision_t* params_cast_h; - if (half_precision) { - grads_cast_h = reinterpret_cast(grads); - params_cast_h = reinterpret_cast(_params); - } for (size_t t = rounded_size; t < _param_size; t += TILE) { size_t copy_size = TILE; if ((t + TILE) > _param_size) copy_size = _param_size - t; size_t offset = copy_size + t; -#if defined(__ENABLE_CUDA__) - if ((t / TILE) >= 2) { cudaStreamSynchronize(_streams[_buf_index]); } -#elif defined(__ENABLE_CANN__) - if ((t / TILE) >= 2) { aclrtSynchronizeStream(_streams[_buf_index].stream()); } -#endif #pragma omp parallel for for (size_t k = t; k < offset; k++) { - float grad = half_precision ? (float)grads_cast_h[k] : grads[k]; - float param = half_precision ? (float)params_cast_h[k] : _params[k]; + float grad = (float)grads[k]; + float param = (float)_params[k]; float momentum = _exp_avg[k]; float variance = _exp_avg_sq[k]; if (_weight_decay > 0 && !_adamw_mode) { grad = param * _weight_decay + grad; } @@ -83,66 +59,31 @@ void Adam_Optimizer::Step_1(float* _params, grad = momentum / grad; if (_weight_decay > 0 && _adamw_mode) { param += w_decay * param; } param = grad * step_size + param; -#if defined(__ENABLE_CUDA__) or defined(__ENABLE_CANN__) - if (dev_params) _doubled_buffer[_buf_index][k - t] = param; -#endif - if (half_precision) - params_cast_h[k] = (ds_half_precision_t)param; - else - _params[k] = param; + _params[k] = param; _exp_avg[k] = momentum; _exp_avg_sq[k] = variance; } -#if defined(__ENABLE_CUDA__) - if (dev_params) { - launch_param_update( - _doubled_buffer[_buf_index], dev_params + t, (copy_size), _streams[_buf_index]); - - _buf_index = !_buf_index; - } -#elif defined(__ENABLE_CANN__) - if (dev_params) { - size_t memcpy_size = copy_size * sizeof(_doubled_buffer[_buf_index][0]); - aclrtMemcpy(dev_params + t, - memcpy_size, - _doubled_buffer[_buf_index], - memcpy_size, - aclrtMemcpyKind::ACL_MEMCPY_HOST_TO_DEVICE); - - _buf_index = !_buf_index; - } -#endif } } } -void Adam_Optimizer::Step_4(float* _params, - float* grads, - float* _exp_avg, - float* _exp_avg_sq, - size_t _param_size, - ds_half_precision_t* dev_params, - bool half_precision) +template +void Adam_Optimizer::Step_4(ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg, + ds_state_precision_t* _exp_avg_sq, + size_t _param_size) { size_t rounded_size = 0; #if defined(__AVX512__) or defined(__AVX256__) - Step_AVX<4>(&rounded_size, - _params, - grads, - _exp_avg, - _exp_avg_sq, - _param_size, - dev_params, - half_precision); + Step_AVX<4>(&rounded_size, _params, grads, _exp_avg, _exp_avg_sq, _param_size); #endif if (_param_size > rounded_size) Step_1((_params + rounded_size), (grads + rounded_size), (_exp_avg + rounded_size), (_exp_avg_sq + rounded_size), - (_param_size - rounded_size), - (dev_params != nullptr ? (dev_params + rounded_size) : dev_params), - half_precision); + (_param_size - rounded_size)); } int create_adam_optimizer(int optimizer_id, @@ -185,33 +126,86 @@ int create_adam_optimizer(int optimizer_id, return 0; } -void Adam_Optimizer::Step_8(float* _params, - float* grads, - float* _exp_avg, - float* _exp_avg_sq, - size_t _param_size, - ds_half_precision_t* dev_params, - bool half_precision) +template +void Adam_Optimizer::Step_8(ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg, + ds_state_precision_t* _exp_avg_sq, + size_t _param_size) { size_t rounded_size = 0; #if defined(__AVX512__) or defined(__AVX256__) - Step_AVX<8>(&rounded_size, - _params, - grads, - _exp_avg, - _exp_avg_sq, - _param_size, - dev_params, - half_precision); + Step_AVX<8>(&rounded_size, _params, grads, _exp_avg, _exp_avg_sq, _param_size); #endif if (_param_size > rounded_size) Step_4((_params + rounded_size), (grads + rounded_size), (_exp_avg + rounded_size), (_exp_avg_sq + rounded_size), - (_param_size - rounded_size), - (dev_params != nullptr ? (dev_params + rounded_size) : dev_params), - half_precision); + (_param_size - rounded_size)); +} + +template +void step_invoker(std::shared_ptr opt, + void* _params, + void* grads, + void* _exp_avg, + void* _exp_avg_sq, + size_t _param_size) +{ + opt->Step_8((ds_params_percision_t*)(_params), + (ds_params_percision_t*)(grads), + (ds_state_precision_t*)(_exp_avg), + (ds_state_precision_t*)(_exp_avg_sq), + _param_size); +} + +std::map, + std::function, void*, void*, void*, void*, size_t)>> + invokers; + +// Fill map with template functions for each type +template +void create_invoker() +{ + invokers[std::tuple(c10::CppTypeToScalarType(), + c10::CppTypeToScalarType())] = + step_invoker; +} +struct InvokerInitializer { + InvokerInitializer() + { + create_invoker(); + create_invoker(); + create_invoker(); + create_invoker(); + create_invoker(); + } +} _invoker_initializer; + +void invoke(std::shared_ptr opt, + torch::Tensor& params, + torch::Tensor& grads, + torch::Tensor& exp_avg, + torch::Tensor& exp_avg_sq, + size_t param_size) +{ + c10::ScalarType params_type = at::typeMetaToScalarType(params.options().dtype()); + c10::ScalarType state_type = at::typeMetaToScalarType(exp_avg.options().dtype()); + + auto it = invokers.find(std::tuple(params_type, state_type)); + if (it == invokers.end()) { + throw std::runtime_error("Adam optimizer with param type "s + c10::toString(params_type) + + " and state type "s + c10::toString(state_type) + + " is not supported on current hardware"s); + } + + it->second(opt, + params.data_ptr(), + grads.data_ptr(), + exp_avg.data_ptr(), + exp_avg_sq.data_ptr(), + param_size); } int ds_adam_step(int optimizer_id, @@ -232,75 +226,13 @@ int ds_adam_step(int optimizer_id, auto exp_avg_c = exp_avg.contiguous(); auto exp_avg_sq_c = exp_avg_sq.contiguous(); - // assert(params.options().dtype() == grads.options().dtype()); - - float* params_ptr = (float*)params_c.data_ptr(); - float* grads_ptr = (float*)grads_c.data_ptr(); - float* exp_avg_ptr = (float*)exp_avg_c.data_ptr(); - float* exp_avg_sq_ptr = (float*)exp_avg_sq_c.data_ptr(); - std::shared_ptr opt = std::static_pointer_cast(s_optimizers[optimizer_id]); opt->IncrementStep(step, beta1, beta2); opt->update_state(lr, epsilon, weight_decay, bias_correction); - opt->Step_8(params_ptr, - grads_ptr, - exp_avg_ptr, - exp_avg_sq_ptr, - params_c.numel(), - nullptr, - (params.options().dtype() == at::kHalf)); + invoke(opt, params_c, grads_c, exp_avg_c, exp_avg_sq_c, params_c.numel()); -#if defined(__ENABLE_CUDA__) or defined(__ENABLE_CANN__) - opt->SynchronizeStreams(); -#endif - return 0; -} - -int ds_adam_step_plus_copy(int optimizer_id, - size_t step, - float lr, - float beta1, - float beta2, - float epsilon, - float weight_decay, - bool bias_correction, - torch::Tensor& params, - torch::Tensor& grads, - torch::Tensor& exp_avg, - torch::Tensor& exp_avg_sq, - torch::Tensor& device_params) -{ -#if defined(__ENABLE_CUDA__) or defined(__ENABLE_CANN__) - auto params_c = params.contiguous(); - auto device_params_c = device_params.contiguous(); - auto exp_avg_c = exp_avg.contiguous(); - auto exp_avg_sq_c = exp_avg_sq.contiguous(); - auto grads_c = grads.contiguous(); - - float* params_ptr = (float*)params_c.data_ptr(); - float* grads_ptr = (float*)grads_c.data_ptr(); - ds_half_precision_t* device_params_ptr = (ds_half_precision_t*)device_params_c.data_ptr(); - float* exp_avg_ptr = (float*)exp_avg_c.data_ptr(); - float* exp_avg_sq_ptr = (float*)exp_avg_sq_c.data_ptr(); - - std::shared_ptr opt = - std::static_pointer_cast(s_optimizers[optimizer_id]); - opt->IncrementStep(step, beta1, beta2); - opt->update_state(lr, epsilon, weight_decay, bias_correction); - opt->Step_8(params_ptr, - grads_ptr, - exp_avg_ptr, - exp_avg_sq_ptr, - params_c.numel(), - device_params_ptr, - (params.options().dtype() == at::kHalf)); - - opt->SynchronizeStreams(); -#else - assert(false); -#endif return 0; } diff --git a/csrc/common/custom_cuda_kernel.cu b/csrc/common/custom_cuda_kernel.cu deleted file mode 100644 index f46bf303125c..000000000000 --- a/csrc/common/custom_cuda_kernel.cu +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -// DeepSpeed Team - -#include "custom_cuda_layers.h" - -__global__ void param_update_kernel(const float* input, __half* output, int size) -{ - int id = blockIdx.x * blockDim.x + threadIdx.x; - - if (id < size) { output[id] = (__half)input[id]; } -} - -void launch_param_update(const float* input, __half* output, int size, cudaStream_t stream) -{ - int threads = 1024; - - dim3 grid_dim((size - 1) / threads + 1); - dim3 block_dim(threads); - - param_update_kernel<<>>(input, output, size); -} - -__global__ void param_update_kernel_half(const float* input, __half* output, int size) -{ - int id = blockIdx.x * blockDim.x + threadIdx.x; - __half2* output_cast = reinterpret_cast<__half2*>(output); - if (id < size) { - float input_f = input[id]; - __half2* input_h = reinterpret_cast<__half2*>(&input_f); - output_cast[id] = *input_h; - } -} - -void launch_param_update_half(const float* input, __half* output, int size, cudaStream_t stream) -{ - int threads = 1024; - size /= 2; - dim3 grid_dim((size - 1) / threads + 1); - dim3 block_dim(threads); - - param_update_kernel_half<<>>(input, output, size); -} diff --git a/csrc/includes/cpu_adagrad.h b/csrc/includes/cpu_adagrad.h index e60984d64b76..c06d3a6b35e9 100644 --- a/csrc/includes/cpu_adagrad.h +++ b/csrc/includes/cpu_adagrad.h @@ -9,84 +9,35 @@ // https://stackoverflow.com/questions/4913922/possible-problems-with-nominmax-on-visual-c #include +#include #include #include "simd.h" -#if defined(__ENABLE_CUDA__) -#include -#include -#include "cuda.h" -#include "custom_cuda_layers.h" -typedef __half ds_half_precision_t; -#elif defined(__ENABLE_CANN__) -#include "acl/acl.h" -#include "torch_npu/csrc/core/npu/NPUStream.h" -typedef c10::Half ds_half_precision_t; -#else -typedef unsigned short ds_half_precision_t; -#endif - -#define STEP(SPAN) \ - void Step_##SPAN(float* _params, \ - float* grads, \ - float* _exp_avg_sq, \ - size_t _param_size, \ - ds_half_precision_t* dev_param = nullptr, \ - bool half_precision = false); +#define STEP(SPAN) \ + template \ + void Step_##SPAN(ds_params_percision_t* _params, \ + ds_params_percision_t* grads, \ + ds_state_precision_t* _exp_avg_sq, \ + size_t _param_size); class Adagrad_Optimizer { public: Adagrad_Optimizer(float alpha = 1e-2, float eps = 1e-8, float weight_decay = 0) : _alpha(alpha), _eps(eps), _weight_decay(weight_decay) { -#if defined(__ENABLE_CUDA__) - cudaMallocHost((void**)_doubled_buffer, TILE * sizeof(float)); - cudaMallocHost((void**)(_doubled_buffer + 1), TILE * sizeof(float)); - - _streams[0] = TrainingContext::Instance().GetCurrentStream(); - _streams[1] = TrainingContext::Instance().GetNewStream(); - _buf_index = false; -#elif defined(__ENABLE_CANN__) - aclrtMallocHost((void**)_doubled_buffer, TILE * sizeof(float)); - aclrtMallocHost((void**)(_doubled_buffer + 1), TILE * sizeof(float)); - - _buf_index = false; -#endif - } - ~Adagrad_Optimizer() - { -#if defined(__ENABLE_CUDA__) - cudaFreeHost(_doubled_buffer[0]); - cudaFreeHost(_doubled_buffer[1]); -#elif defined(__ENABLE_CANN__) - aclrtFreeHost(_doubled_buffer[0]); - aclrtFreeHost(_doubled_buffer[1]); -#endif } + ~Adagrad_Optimizer() {} #if defined(__AVX512__) or defined(__AVX256__) - template + template void Step_AVX(size_t* rounded_size, - float* _params, - float* grads, - float* _exp_avg_sq, - size_t param_size, - ds_half_precision_t* dev_param = nullptr, - bool half_precision = false); + ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg_sq, + size_t param_size); #endif STEP(1) STEP(4) STEP(8) -#if defined(__ENABLE_CUDA__) - inline void SynchronizeStreams() - { - for (int i = 0; i < 2; i++) cudaStreamSynchronize(_streams[i]); - } -#elif defined(__ENABLE_CANN__) - inline void SynchronizeStreams() - { - for (int i = 0; i < 2; i++) aclrtSynchronizeStream(_streams[i].stream()); - } -#endif inline void IncrementStep(size_t step) { _step++; @@ -107,29 +58,22 @@ class Adagrad_Optimizer { float _betta1_t; float _betta2_t; size_t _step; - -#if defined(__ENABLE_CUDA__) - bool _buf_index; - float* _doubled_buffer[2]; - cudaStream_t _streams[2]; -#elif defined(__ENABLE_CANN__) - float* _doubled_buffer[2]; - c10_npu::NPUStream _streams[2] = {c10_npu::getCurrentNPUStream(), - c10_npu::getNPUStreamFromPool()}; - bool _buf_index; -#endif }; #if defined(__AVX512__) or defined(__AVX256__) -template +template void Adagrad_Optimizer::Step_AVX(size_t* rounded_size, - float* _params, - float* grads, - float* _exp_avg_sq, - size_t _param_size, - ds_half_precision_t* dev_params, - bool half_precision) + ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg_sq, + size_t _param_size) { +#if !defined(__AVX512__) + if (std::is_same_v || + std::is_same_v) { + return; + } +#endif size_t new_rounded_size = 0; AVX_Data eps_4; eps_4.data = SIMD_SET(_eps); @@ -145,24 +89,19 @@ void Adagrad_Optimizer::Step_AVX(size_t* rounded_size, size_t copy_size = TILE; if ((t + TILE) > new_rounded_size) copy_size = new_rounded_size - t; size_t offset = copy_size + t; -#if defined(__ENABLE_CUDA__) - if ((t / TILE) >= 2) { cudaStreamSynchronize(_streams[_buf_index]); } -#elif defined(__ENABLE_CANN__) - if ((t / TILE) >= 2) { aclrtSynchronizeStream(_streams[_buf_index].stream()); } -#endif #pragma omp parallel for for (size_t i = t; i < offset; i += SIMD_WIDTH * span) { AVX_Data grad_4[span]; - simd_load(grad_4, grads + i, half_precision); + simd_load(grad_4, grads + i); AVX_Data momentum_4[span]; - simd_load(momentum_4, grads + i, false); + simd_load(momentum_4, grads + i); AVX_Data variance_4[span]; - simd_load(variance_4, _exp_avg_sq + i, false); + simd_load(variance_4, _exp_avg_sq + i); AVX_Data param_4[span]; - simd_load(param_4, _params + i, half_precision); + simd_load(param_4, _params + i); if (_weight_decay > 0) { simd_fma(grad_4, param_4, weight_decay4, grad_4); } @@ -172,38 +111,9 @@ void Adagrad_Optimizer::Step_AVX(size_t* rounded_size, simd_div(grad_4, momentum_4, grad_4); simd_fma(param_4, grad_4, step_size_4, param_4); - simd_store(_params + i, param_4, half_precision); -#if defined(__ENABLE_CUDA__) or defined(__ENABLE_CANN__) - if (dev_params) { - simd_store(_doubled_buffer[_buf_index] + (i - t), param_4, half_precision); - } -#endif - simd_store(_exp_avg_sq + i, variance_4, false); - } -#if defined(__ENABLE_CUDA__) - if (dev_params) { - if (half_precision) - launch_param_update_half( - _doubled_buffer[_buf_index], dev_params + t, copy_size, _streams[_buf_index]); - else - launch_param_update( - _doubled_buffer[_buf_index], dev_params + t, copy_size, _streams[_buf_index]); - - _buf_index = !_buf_index; + simd_store(_params + i, param_4); + simd_store(_exp_avg_sq + i, variance_4); } -#elif defined(__ENABLE_CANN__) - if (dev_params) { - size_t memcpy_size = copy_size * sizeof(_doubled_buffer[_buf_index][0]); - if (half_precision) memcpy_size /= 2; - aclrtMemcpy(dev_params + t, - memcpy_size, - _doubled_buffer[_buf_index], - memcpy_size, - aclrtMemcpyKind::ACL_MEMCPY_HOST_TO_DEVICE); - - _buf_index = !_buf_index; - } -#endif } *rounded_size = new_rounded_size; } diff --git a/csrc/includes/cpu_adam.h b/csrc/includes/cpu_adam.h index b1a104b2571d..faf99020aee5 100644 --- a/csrc/includes/cpu_adam.h +++ b/csrc/includes/cpu_adam.h @@ -13,29 +13,13 @@ #include #include "simd.h" -#if defined(__ENABLE_CUDA__) -#include -#include -#include "cuda.h" -#include "custom_cuda_layers.h" -typedef __half ds_half_precision_t; -#elif defined(__ENABLE_CANN__) -#include "acl/acl.h" -#include "torch_npu/csrc/core/npu/NPUStream.h" -typedef c10::Half ds_half_precision_t; -#else -#include -typedef unsigned short ds_half_precision_t; -#endif - -#define STEP(SPAN) \ - void Step_##SPAN(float* _params, \ - float* grads, \ - float* _exp_avg, \ - float* _exp_avg_sq, \ - size_t _param_size, \ - ds_half_precision_t* dev_param = nullptr, \ - bool half_precision = false); +#define STEP(SPAN) \ + template \ + void Step_##SPAN(ds_params_percision_t* _params, \ + ds_params_percision_t* grads, \ + ds_state_precision_t* _exp_avg, \ + ds_state_precision_t* _exp_avg_sq, \ + size_t _param_size); class Adam_Optimizer { public: @@ -55,56 +39,21 @@ class Adam_Optimizer { _step(0), _adamw_mode(adamw_mode) { -#if defined(__ENABLE_CUDA__) - cudaMallocHost((void**)_doubled_buffer, TILE * sizeof(float)); - cudaMallocHost((void**)(_doubled_buffer + 1), TILE * sizeof(float)); - - _streams[0] = TrainingContext::Instance().GetCurrentStream(); - _streams[1] = TrainingContext::Instance().GetNewStream(); - _buf_index = false; -#elif defined(__ENABLE_CANN__) - aclrtMallocHost((void**)_doubled_buffer, TILE * sizeof(float)); - aclrtMallocHost((void**)(_doubled_buffer + 1), TILE * sizeof(float)); - - _buf_index = false; -#endif - } - ~Adam_Optimizer() - { -#if defined(__ENABLE_CUDA__) - cudaFreeHost(_doubled_buffer[0]); - cudaFreeHost(_doubled_buffer[1]); -#elif defined(__ENABLE_CANN__) - aclrtFreeHost(_doubled_buffer[0]); - aclrtFreeHost(_doubled_buffer[1]); -#endif } + ~Adam_Optimizer() {} #if defined(__AVX512__) or defined(__AVX256__) - template + template void Step_AVX(size_t* rounded_size, - float* _params, - float* grads, - float* _exp_avg, - float* _exp_avg_sq, - size_t param_size, - ds_half_precision_t* dev_param = nullptr, - bool half_precision = false); + ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg, + ds_state_precision_t* _exp_avg_sq, + size_t param_size); #endif STEP(1) STEP(4) STEP(8) -#if defined(__ENABLE_CUDA__) - inline void SynchronizeStreams() - { - for (int i = 0; i < 2; i++) cudaStreamSynchronize(_streams[i]); - } -#elif defined(__ENABLE_CANN__) - inline void SynchronizeStreams() - { - for (int i = 0; i < 2; i++) aclrtSynchronizeStream(_streams[i].stream()); - } -#endif inline void IncrementStep(size_t step, float beta1, float beta2) { if (beta1 != _betta1 || beta2 != _betta2) { @@ -154,32 +103,24 @@ class Adam_Optimizer { float _bias_correction2; bool _adamw_mode; - -#if defined(__ENABLE_CUDA__) - float* _doubled_buffer[2]; - cudaStream_t _streams[2]; - bool _buf_index; -#elif defined(__ENABLE_CANN__) - float* _doubled_buffer[2]; - c10_npu::NPUStream _streams[2] = {c10_npu::getCurrentNPUStream(), - c10_npu::getNPUStreamFromPool()}; - bool _buf_index; -#endif }; #if defined(__AVX512__) or defined(__AVX256__) -template +template void Adam_Optimizer::Step_AVX(size_t* rounded_size, - float* _params, - float* grads, - float* _exp_avg, - float* _exp_avg_sq, - size_t _param_size, - ds_half_precision_t* dev_params, - bool half_precision) + ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg, + ds_state_precision_t* _exp_avg_sq, + size_t _param_size) { +#if !defined(__AVX512__) + if (std::is_same_v || + std::is_same_v) { + return; + } +#endif size_t new_rounded_size = 0; - int rshft = half_precision ? 1 : 0; AVX_Data betta1_4; betta1_4.data = SIMD_SET(_betta1); @@ -212,24 +153,19 @@ void Adam_Optimizer::Step_AVX(size_t* rounded_size, size_t copy_size = TILE; if ((t + TILE) > new_rounded_size) copy_size = new_rounded_size - t; size_t offset = copy_size + t; -#if defined(__ENABLE_CUDA__) - if ((t / TILE) >= 2) { cudaStreamSynchronize(_streams[_buf_index]); } -#elif defined(__ENABLE_CANN__) - if ((t / TILE) >= 2) { aclrtSynchronizeStream(_streams[_buf_index].stream()); } -#endif #pragma omp parallel for for (size_t i = t; i < offset; i += SIMD_WIDTH * span) { AVX_Data grad_4[span]; - simd_load(grad_4, grads + (i >> rshft), half_precision); + simd_load(grad_4, grads + i); AVX_Data momentum_4[span]; - simd_load(momentum_4, _exp_avg + i, false); + simd_load(momentum_4, _exp_avg + i); AVX_Data variance_4[span]; - simd_load(variance_4, _exp_avg_sq + i, false); + simd_load(variance_4, _exp_avg_sq + i); AVX_Data param_4[span]; - simd_load(param_4, _params + (i >> rshft), half_precision); + simd_load(param_4, _params + i); if (_weight_decay > 0 && !_adamw_mode) { simd_fma(grad_4, param_4, weight_decay4, grad_4); @@ -250,39 +186,10 @@ void Adam_Optimizer::Step_AVX(size_t* rounded_size, simd_fma(param_4, grad_4, step_size_4, param_4); - simd_store(_params + (i >> rshft), param_4, half_precision); -#if defined(__ENABLE_CUDA__) or defined(__ENABLE_CANN__) - if (dev_params) { - simd_store(_doubled_buffer[_buf_index] + (i - t), param_4, half_precision); - } -#endif - simd_store(_exp_avg + i, momentum_4, false); - simd_store(_exp_avg_sq + i, variance_4, false); + simd_store(_params + i, param_4); + simd_store(_exp_avg + i, momentum_4); + simd_store(_exp_avg_sq + i, variance_4); } -#if defined(__ENABLE_CUDA__) - if (dev_params) { - if (half_precision) - launch_param_update_half( - _doubled_buffer[_buf_index], dev_params + t, copy_size, _streams[_buf_index]); - else - launch_param_update( - _doubled_buffer[_buf_index], dev_params + t, copy_size, _streams[_buf_index]); - - _buf_index = !_buf_index; - } -#elif defined(__ENABLE_CANN__) - if (dev_params) { - size_t memcpy_size = copy_size * sizeof(_doubled_buffer[_buf_index][0]); - if (half_precision) memcpy_size /= 2; - aclrtMemcpy(dev_params + t, - memcpy_size, - _doubled_buffer[_buf_index], - memcpy_size, - aclrtMemcpyKind::ACL_MEMCPY_HOST_TO_DEVICE); - - _buf_index = !_buf_index; - } -#endif } *rounded_size = new_rounded_size; } @@ -310,18 +217,4 @@ int ds_adam_step(int optimizer_id, torch::Tensor& exp_avg, torch::Tensor& exp_avg_sq); -int ds_adam_step_plus_copy(int optimizer_id, - size_t step, - float lr, - float beta1, - float beta2, - float epsilon, - float weight_decay, - bool bias_correction, - torch::Tensor& params, - torch::Tensor& grads, - torch::Tensor& exp_avg, - torch::Tensor& exp_avg_sq, - torch::Tensor& gpu_params); - int destroy_adam_optimizer(int optimizer_id); diff --git a/csrc/includes/cpu_lion.h b/csrc/includes/cpu_lion.h index 34c29eec47db..62b304923222 100644 --- a/csrc/includes/cpu_lion.h +++ b/csrc/includes/cpu_lion.h @@ -13,28 +13,12 @@ #include #include "simd.h" -#if defined(__ENABLE_CUDA__) -#include -#include -#include "cuda.h" -#include "custom_cuda_layers.h" -typedef __half ds_half_precision_t; -#elif defined(__ENABLE_CANN__) -#include "acl/acl.h" -#include "torch_npu/csrc/core/npu/NPUStream.h" -typedef c10::Half ds_half_precision_t; -#else -#include -typedef unsigned short ds_half_precision_t; -#endif - -#define STEP(SPAN) \ - void Step_##SPAN(float* _params, \ - float* grads, \ - float* _exp_avg, \ - size_t _param_size, \ - ds_half_precision_t* dev_param = nullptr, \ - bool half_precision = false); +#define STEP(SPAN) \ + template \ + void Step_##SPAN(ds_params_percision_t* _params, \ + ds_params_percision_t* grads, \ + ds_state_precision_t* _exp_avg, \ + size_t _param_size); class Lion_Optimizer { public: @@ -44,55 +28,21 @@ class Lion_Optimizer { float weight_decay = 0) : _alpha(alpha), _betta1(betta1), _betta2(betta2), _weight_decay(weight_decay), _step(0) { -#if defined(__ENABLE_CUDA__) - cudaMallocHost((void**)_doubled_buffer, TILE * sizeof(float)); - cudaMallocHost((void**)(_doubled_buffer + 1), TILE * sizeof(float)); - - _streams[0] = TrainingContext::Instance().GetCurrentStream(); - _streams[1] = TrainingContext::Instance().GetNewStream(); - _buf_index = false; -#elif defined(__ENABLE_CANN__) - aclrtMallocHost((void**)_doubled_buffer, TILE * sizeof(float)); - aclrtMallocHost((void**)(_doubled_buffer + 1), TILE * sizeof(float)); - - _buf_index = false; -#endif - } - ~Lion_Optimizer() - { -#if defined(__ENABLE_CUDA__) - cudaFreeHost(_doubled_buffer[0]); - cudaFreeHost(_doubled_buffer[1]); -#elif defined(__ENABLE_CANN__) - aclrtFreeHost(_doubled_buffer[0]); - aclrtFreeHost(_doubled_buffer[1]); -#endif } + ~Lion_Optimizer() {} #if defined(__AVX512__) or defined(__AVX256__) - template + template void Step_AVX(size_t* rounded_size, - float* _params, - float* grads, - float* _exp_avg, - size_t param_size, - ds_half_precision_t* dev_param = nullptr, - bool half_precision = false); + ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg, + size_t param_size); #endif STEP(1) STEP(4) STEP(8) -#if defined(__ENABLE_CUDA__) - inline void SynchronizeStreams() - { - for (int i = 0; i < 2; i++) cudaStreamSynchronize(_streams[i]); - } -#elif defined(__ENABLE_CANN__) - inline void SynchronizeStreams() - { - for (int i = 0; i < 2; i++) aclrtSynchronizeStream(_streams[i].stream()); - } -#endif + inline void IncrementStep(size_t step, float beta1, float beta2) { _step++; @@ -114,31 +64,23 @@ class Lion_Optimizer { float _betta2; float _weight_decay; size_t _step; - -#if defined(__ENABLE_CUDA__) - float* _doubled_buffer[2]; - cudaStream_t _streams[2]; - bool _buf_index; -#elif defined(__ENABLE_CANN__) - float* _doubled_buffer[2]; - c10_npu::NPUStream _streams[2] = {c10_npu::getCurrentNPUStream(), - c10_npu::getNPUStreamFromPool()}; - bool _buf_index; -#endif }; #if defined(__AVX512__) or defined(__AVX256__) -template +template void Lion_Optimizer::Step_AVX(size_t* rounded_size, - float* _params, - float* grads, - float* _exp_avg, - size_t _param_size, - ds_half_precision_t* dev_params, - bool half_precision) + ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg, + size_t _param_size) { +#if !defined(__AVX512__) + if (std::is_same_v || + std::is_same_v) { + return; + } +#endif size_t new_rounded_size = 0; - int rshft = half_precision ? 1 : 0; constexpr float neg1 = -1.0f; AVX_Data neg1_4; @@ -169,21 +111,17 @@ void Lion_Optimizer::Step_AVX(size_t* rounded_size, size_t copy_size = TILE; if ((t + TILE) > new_rounded_size) copy_size = new_rounded_size - t; size_t offset = copy_size + t; -#if defined(__ENABLE_CUDA__) - if ((t / TILE) >= 2) { cudaStreamSynchronize(_streams[_buf_index]); } -#elif defined(__ENABLE_CANN__) - if ((t / TILE) >= 2) { aclrtSynchronizeStream(_streams[_buf_index].stream()); } -#endif + #pragma omp parallel for for (size_t i = t; i < offset; i += SIMD_WIDTH * span) { AVX_Data grad_4[span]; - simd_load(grad_4, grads + (i >> rshft), half_precision); + simd_load(grad_4, grads + i); AVX_Data momentum_4[span]; - simd_load(momentum_4, _exp_avg + i, false); + simd_load(momentum_4, _exp_avg + i); AVX_Data param_4[span]; - simd_load(param_4, _params + (i >> rshft), half_precision); + simd_load(param_4, _params + i); AVX_Data tmp_4[span]; @@ -201,38 +139,9 @@ void Lion_Optimizer::Step_AVX(size_t* rounded_size, simd_mul(momentum_4, momentum_4, betta2_4); simd_fma(momentum_4, grad_4, betta2_minus1_4, momentum_4); - simd_store(_params + (i >> rshft), param_4, half_precision); -#if defined(__ENABLE_CUDA__) or defined(__ENABLE_CANN__) - if (dev_params) { - simd_store(_doubled_buffer[_buf_index] + (i - t), param_4, half_precision); - } -#endif - simd_store(_exp_avg + i, momentum_4, false); - } -#if defined(__ENABLE_CUDA__) - if (dev_params) { - if (half_precision) - launch_param_update_half( - _doubled_buffer[_buf_index], dev_params + t, copy_size, _streams[_buf_index]); - else - launch_param_update( - _doubled_buffer[_buf_index], dev_params + t, copy_size, _streams[_buf_index]); - - _buf_index = !_buf_index; + simd_store(_params + i, param_4); + simd_store(_exp_avg + i, momentum_4); } -#elif defined(__ENABLE_CANN__) - if (dev_params) { - size_t memcpy_size = copy_size * sizeof(_doubled_buffer[_buf_index][0]); - if (half_precision) memcpy_size /= 2; - aclrtMemcpy(dev_params + t, - memcpy_size, - _doubled_buffer[_buf_index], - memcpy_size, - aclrtMemcpyKind::ACL_MEMCPY_HOST_TO_DEVICE); - - _buf_index = !_buf_index; - } -#endif } *rounded_size = new_rounded_size; } @@ -255,15 +164,4 @@ int ds_lion_step(int optimizer_id, torch::Tensor& grads, torch::Tensor& exp_avg); -int ds_lion_step_plus_copy(int optimizer_id, - size_t step, - float lr, - float beta1, - float beta2, - float weight_decay, - torch::Tensor& params, - torch::Tensor& grads, - torch::Tensor& exp_avg, - torch::Tensor& gpu_params); - int destroy_lion_optimizer(int optimizer_id); diff --git a/csrc/includes/custom_cuda_layers.h b/csrc/includes/custom_cuda_layers.h index 265eb7b12444..21f19749d4cf 100644 --- a/csrc/includes/custom_cuda_layers.h +++ b/csrc/includes/custom_cuda_layers.h @@ -272,9 +272,6 @@ void launch_fuse_transpose_bias_kernel(const T* inp, int cols, cudaStream_t stream); -void launch_param_update(const float* input, __half* output, int size, cudaStream_t stream); -void launch_param_update_half(const float* input, __half* output, int size, cudaStream_t stream); - void launch_token_sort(int32_t* indices, int layers, int batch_size, diff --git a/csrc/includes/simd.h b/csrc/includes/simd.h index 59237b0261c1..73e41216a652 100644 --- a/csrc/includes/simd.h +++ b/csrc/includes/simd.h @@ -13,6 +13,19 @@ #define TILE (128 * 1024 * 1024) #if defined(__AVX512__) or defined(__AVX256__) +template +inline T readAs(const void* src) +{ + T res; + std::memcpy(&res, src, sizeof(T)); + return res; +} +template +inline void writeAs(void* dst, const T& val) +{ + std::memcpy(dst, &val, sizeof(T)); +} + #define ROUND_DOWN(size, step) ((size) & ~((step)-1)) #if defined(__AVX512__) @@ -30,11 +43,52 @@ #define SIMD_XOR(x, y) _mm512_xor_ps(x, y) #define SIMD_WIDTH 16 -#define SIMD_LOAD2(x, h) \ - ((h) ? _mm512_cvtph_ps(_mm256_castps_si256(_mm256_loadu_ps(x))) : _mm512_loadu_ps(x)) -#define SIMD_STORE2(x, d, h) \ - ((h) ? _mm256_store_ps(x, _mm256_castsi256_ps(_mm512_cvtps_ph(d, _MM_FROUND_TO_NEAREST_INT))) \ - : _mm512_storeu_ps(x, d)) +static __m512 load_16_bf16_as_f32(const void* data) +{ + __m256i a = readAs<__m256i>(data); // use memcpy to avoid aliasing + __m512i b = _mm512_cvtepu16_epi32(a); // convert 8 u16 to 8 u32 + __m512i c = _mm512_slli_epi32(b, 16); // logical shift left of all u32 by + // 16 bits (representing bf16->f32) + return readAs<__m512>(&c); // use memcpy to avoid aliasing +} + +static void store_16_f32_as_bf16_nearest(__m512 v, void* data) +{ + __m512i u32 = readAs<__m512i>(&v); + + // flow assuming non-nan: + + // uint32_t rounding_bias = ((U32 >> 16) & 1) + UINT32_C(0x7FFF); + __m512i b = _mm512_srli_epi32(u32, 16); + __m512i lsb_mask = _mm512_set1_epi32(0x00000001); + __m512i c = _mm512_and_si512(b, lsb_mask); + __m512i bias_constant = _mm512_set1_epi32(0x00007fff); + __m512i rounding_bias = _mm512_add_epi32(c, bias_constant); + + // uint16_t res = static_cast((U32 + rounding_bias) >> 16); + __m512i d = _mm512_add_epi32(u32, rounding_bias); + __m512i e = _mm512_srli_epi32(d, 16); + __m256i non_nan_res = _mm512_cvtusepi32_epi16(e); + + // handle nan (exp is all 1s and mantissa != 0) + // if ((x & 0x7fffffffU) > 0x7f800000U) + __m512i mask_out_sign = _mm512_set1_epi32(0x7fffffff); + __m512i non_sign_bits = _mm512_and_si512(u32, mask_out_sign); + __m512i nan_threshold = _mm512_set1_epi32(0x7f800000); + __mmask16 nan_mask = _mm512_cmp_epi32_mask(non_sign_bits, nan_threshold, _MM_CMPINT_GT); + + // mix in results with nans as needed + __m256i nans = _mm256_set1_epi16(0x7fc0); + __m256i res = _mm256_mask_mov_epi16(non_nan_res, nan_mask, nans); + + writeAs(data, res); +} +#define SIMD_LOAD_BF16(x) load_16_bf16_as_f32(x) +#define SIMD_STORE_BF16(x, d) store_16_f32_as_bf16_nearest(d, x) + +#define SIMD_LOAD_FP16(x) _mm512_cvtph_ps(_mm256_castps_si256(_mm256_loadu_ps(x))) +#define SIMD_STORE_FP16(x, d) \ + _mm256_store_ps(x, _mm256_castsi256_ps(_mm512_cvtps_ph(d, _MM_FROUND_TO_NEAREST_INT))) #define INTV __m256i #elif defined(__AVX256__) @@ -52,11 +106,11 @@ #define SIMD_XOR(x, y) _mm256_xor_ps(x, y) #define SIMD_WIDTH 8 -#define SIMD_LOAD2(x, h) \ - ((h) ? _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*)(x))) : _mm256_loadu_ps(x)) -#define SIMD_STORE2(x, d, h) \ - ((h) ? _mm_store_ps(x, _mm_castsi128_ps(_mm256_cvtps_ph(d, _MM_FROUND_TO_NEAREST_INT))) \ - : _mm256_storeu_ps(x, d)) +#define SIMD_LOAD_BF16(x) static_assert(false && "AVX256 does not support BFloat16") +#define SIMD_STORE_BF16(x, d) static_assert(false && "AVX256 does not support BFloat16") +#define SIMD_LOAD_FP16(x) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*)x)) +#define SIMD_STORE_FP16(x, d) \ + _mm_store_ps(x, _mm_castsi128_ps(_mm256_cvtps_ph(d, _MM_FROUND_TO_NEAREST_INT))) #define INTV __m128i #endif @@ -70,20 +124,66 @@ union AVX_Data { // float data_f[16]; }; -template -inline void simd_store(float* dst, AVX_Data* src, bool half_precision) +template +inline typename std::enable_if_t, void> simd_store(T* dst, + AVX_Data* src) { - size_t width = (half_precision ? SIMD_WIDTH / 2 : SIMD_WIDTH); + size_t width = SIMD_WIDTH; #pragma unroll - for (size_t i = 0; i < span; ++i) { SIMD_STORE2(dst + width * i, src[i].data, half_precision); } + for (size_t i = 0; i < span; ++i) { SIMD_STORE_FP16((float*)(dst + width * i), src[i].data); } } -template -inline void simd_load(AVX_Data* dst, float* src, bool half_precision) + +template +inline typename std::enable_if_t, void> simd_store(T* dst, + AVX_Data* src) { - size_t width = (half_precision ? SIMD_WIDTH / 2 : SIMD_WIDTH); +#ifdef __AVX512__ + size_t width = SIMD_WIDTH; #pragma unroll - for (size_t i = 0; i < span; ++i) { dst[i].data = SIMD_LOAD2(src + width * i, half_precision); } + for (size_t i = 0; i < span; ++i) { SIMD_STORE_BF16((float*)(dst + width * i), src[i].data); } +#else + throw std::runtime_error("AVX512 required for BFloat16"); +#endif +} + +template +inline typename std::enable_if_t, void> simd_store(T* dst, AVX_Data* src) +{ + size_t width = SIMD_WIDTH; +#pragma unroll + for (size_t i = 0; i < span; ++i) { SIMD_STORE(dst + width * i, src[i].data); } } + +template +inline typename std::enable_if_t, void> simd_load(AVX_Data* dst, + T* src) +{ + size_t width = SIMD_WIDTH; +#pragma unroll + for (size_t i = 0; i < span; ++i) { dst[i].data = SIMD_LOAD_FP16((float*)(src + width * i)); } +} + +template +inline typename std::enable_if_t, void> simd_load(AVX_Data* dst, + T* src) +{ +#ifdef __AVX512__ + size_t width = SIMD_WIDTH; +#pragma unroll + for (size_t i = 0; i < span; ++i) { dst[i].data = SIMD_LOAD_BF16((float*)(src + width * i)); } +#else + throw std::runtime_error("AVX512 required for BFloat16"); +#endif +} + +template +inline typename std::enable_if_t, void> simd_load(AVX_Data* dst, T* src) +{ + size_t width = SIMD_WIDTH; +#pragma unroll + for (size_t i = 0; i < span; ++i) { dst[i].data = SIMD_LOAD(src + width * i); } +} + template inline void simd_fma(AVX_Data* dst, AVX_Data* src_m_l, AVX_Data src_m_r, AVX_Data* src_a) { diff --git a/csrc/lion/cpu_lion.cpp b/csrc/lion/cpu_lion.cpp index a0562eac9c4a..c5cf3e9e9235 100644 --- a/csrc/lion/cpu_lion.cpp +++ b/csrc/lion/cpu_lion.cpp @@ -8,9 +8,6 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("lion_update", &ds_lion_step, "DeepSpeed CPU Lion update (C++)"); - m.def("lion_update_copy", - &ds_lion_step_plus_copy, - "DeepSpeed CPU Lion update and param copy (C++)"); m.def("create_lion", &create_lion_optimizer, "DeepSpeed CPU Lion (C++)"); m.def("destroy_lion", &destroy_lion_optimizer, "DeepSpeed CPU Lion destroy (C++)"); } diff --git a/csrc/lion/cpu_lion_impl.cpp b/csrc/lion/cpu_lion_impl.cpp index 28314cf5b6e1..85896ba86e19 100644 --- a/csrc/lion/cpu_lion_impl.cpp +++ b/csrc/lion/cpu_lion_impl.cpp @@ -6,34 +6,28 @@ #include #include #include +#include #include +#include #include #include #include #include "cpu_lion.h" -#if defined(__ENABLE_CUDA__) -#include -#include "cublas_v2.h" -#include "cuda.h" -#include "curand.h" -#include "custom_cuda_layers.h" -#endif - +using namespace std::string_literals; static std::unordered_map> s_optimizers; // C++ interface -void Lion_Optimizer::Step_1(float* _params, - float* grads, - float* _exp_avg, - size_t _param_size, - ds_half_precision_t* dev_params, - bool half_precision) +template +void Lion_Optimizer::Step_1(ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg, + size_t _param_size) { size_t rounded_size = 0; #if defined(__AVX512__) or defined(__AVX256__) - Step_AVX<1>(&rounded_size, _params, grads, _exp_avg, _param_size, dev_params, half_precision); + Step_AVX<1>(&rounded_size, _params, grads, _exp_avg, _param_size); #endif if (_param_size > rounded_size) { float betta1_minus1 = 1 - _betta1; @@ -41,26 +35,15 @@ void Lion_Optimizer::Step_1(float* _params, float alpha = _alpha; float after_decay = 1 - alpha * _weight_decay; - ds_half_precision_t* grads_cast_h; - ds_half_precision_t* params_cast_h; - if (half_precision) { - grads_cast_h = reinterpret_cast(grads); - params_cast_h = reinterpret_cast(_params); - } for (size_t t = rounded_size; t < _param_size; t += TILE) { size_t copy_size = TILE; if ((t + TILE) > _param_size) copy_size = _param_size - t; size_t offset = copy_size + t; -#if defined(__ENABLE_CUDA__) - if ((t / TILE) >= 2) { cudaStreamSynchronize(_streams[_buf_index]); } -#elif defined(__ENABLE_CANN__) - if ((t / TILE) >= 2) { aclrtSynchronizeStream(_streams[_buf_index].stream()); } -#endif #pragma omp parallel for for (size_t k = t; k < offset; k++) { - float grad = half_precision ? (float)grads_cast_h[k] : grads[k]; - float param = half_precision ? (float)params_cast_h[k] : _params[k]; + float grad = (float)grads[k]; + float param = (float)_params[k]; float momentum = _exp_avg[k]; float tmp = momentum * _betta1; tmp = grad * betta1_minus1 + tmp; @@ -74,56 +57,28 @@ void Lion_Optimizer::Step_1(float* _params, } momentum = momentum * _betta2; momentum = grad * betta2_minus1 + momentum; -#if defined(__ENABLE_CUDA__) or defined(__ENABLE_CANN__) - if (dev_params) _doubled_buffer[_buf_index][k - t] = param; -#endif - if (half_precision) - params_cast_h[k] = (ds_half_precision_t)param; - else - _params[k] = param; + _params[k] = param; _exp_avg[k] = momentum; } -#if defined(__ENABLE_CUDA__) - if (dev_params) { - launch_param_update( - _doubled_buffer[_buf_index], dev_params + t, (copy_size), _streams[_buf_index]); - - _buf_index = !_buf_index; - } -#elif defined(__ENABLE_CANN__) - if (dev_params) { - size_t memcpy_size = copy_size * sizeof(_doubled_buffer[_buf_index][0]); - aclrtMemcpy(dev_params + t, - memcpy_size, - _doubled_buffer[_buf_index], - memcpy_size, - aclrtMemcpyKind::ACL_MEMCPY_HOST_TO_DEVICE); - - _buf_index = !_buf_index; - } -#endif } } } -void Lion_Optimizer::Step_4(float* _params, - float* grads, - float* _exp_avg, - size_t _param_size, - ds_half_precision_t* dev_params, - bool half_precision) +template +void Lion_Optimizer::Step_4(ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg, + size_t _param_size) { size_t rounded_size = 0; #if defined(__AVX512__) or defined(__AVX256__) - Step_AVX<4>(&rounded_size, _params, grads, _exp_avg, _param_size, dev_params, half_precision); + Step_AVX<4>(&rounded_size, _params, grads, _exp_avg, _param_size); #endif if (_param_size > rounded_size) Step_1((_params + rounded_size), (grads + rounded_size), (_exp_avg + rounded_size), - (_param_size - rounded_size), - (dev_params != nullptr ? (dev_params + rounded_size) : dev_params), - half_precision); + (_param_size - rounded_size)); } int create_lion_optimizer(int optimizer_id, @@ -162,24 +117,76 @@ int create_lion_optimizer(int optimizer_id, return 0; } -void Lion_Optimizer::Step_8(float* _params, - float* grads, - float* _exp_avg, - size_t _param_size, - ds_half_precision_t* dev_params, - bool half_precision) +template +void Lion_Optimizer::Step_8(ds_params_percision_t* _params, + ds_params_percision_t* grads, + ds_state_precision_t* _exp_avg, + size_t _param_size) { size_t rounded_size = 0; #if defined(__AVX512__) or defined(__AVX256__) - Step_AVX<8>(&rounded_size, _params, grads, _exp_avg, _param_size, dev_params, half_precision); + Step_AVX<8>(&rounded_size, _params, grads, _exp_avg, _param_size); #endif if (_param_size > rounded_size) Step_4((_params + rounded_size), (grads + rounded_size), (_exp_avg + rounded_size), - (_param_size - rounded_size), - (dev_params != nullptr ? (dev_params + rounded_size) : dev_params), - half_precision); + (_param_size - rounded_size)); +} + +template +void step_invoker(std::shared_ptr opt, + void* _params, + void* grads, + void* _exp_avg, + size_t _param_size) +{ + opt->Step_8((ds_params_percision_t*)(_params), + (ds_params_percision_t*)(grads), + (ds_state_precision_t*)(_exp_avg), + _param_size); +} + +std::map, + std::function, void*, void*, void*, size_t)>> + invokers; + +// Fill map with template functions for each type +template +void create_invoker() +{ + invokers[std::tuple(c10::CppTypeToScalarType(), + c10::CppTypeToScalarType())] = + step_invoker; +} +struct InvokerInitializer { + InvokerInitializer() + { + create_invoker(); + create_invoker(); + create_invoker(); + create_invoker(); + create_invoker(); + } +} _invoker_initializer; + +void invoke(std::shared_ptr opt, + torch::Tensor& params, + torch::Tensor& grads, + torch::Tensor& exp_avg, + size_t param_size) +{ + c10::ScalarType params_type = at::typeMetaToScalarType(params.options().dtype()); + c10::ScalarType state_type = at::typeMetaToScalarType(exp_avg.options().dtype()); + + auto it = invokers.find(std::tuple(params_type, state_type)); + if (it == invokers.end()) { + throw std::runtime_error("Lion optimizer with param type "s + c10::toString(params_type) + + " and state type "s + c10::toString(state_type) + + " is not supported on current hardware"s); + } + + it->second(opt, params.data_ptr(), grads.data_ptr(), exp_avg.data_ptr(), param_size); } int ds_lion_step(int optimizer_id, @@ -196,67 +203,13 @@ int ds_lion_step(int optimizer_id, auto grads_c = grads.contiguous(); auto exp_avg_c = exp_avg.contiguous(); - // assert(params.options().dtype() == grads.options().dtype()); - - float* params_ptr = (float*)params_c.data_ptr(); - float* grads_ptr = (float*)grads_c.data_ptr(); - float* exp_avg_ptr = (float*)exp_avg_c.data_ptr(); - std::shared_ptr opt = std::static_pointer_cast(s_optimizers[optimizer_id]); opt->IncrementStep(step, beta1, beta2); opt->update_state(lr, weight_decay); - opt->Step_8(params_ptr, - grads_ptr, - exp_avg_ptr, - params_c.numel(), - nullptr, - (params.options().dtype() == at::kHalf)); + invoke(opt, params_c, grads_c, exp_avg_c, params_c.numel()); -#if defined(__ENABLE_CUDA__) or defined(__ENABLE_CANN__) - opt->SynchronizeStreams(); -#endif - return 0; -} - -int ds_lion_step_plus_copy(int optimizer_id, - size_t step, - float lr, - float beta1, - float beta2, - float weight_decay, - torch::Tensor& params, - torch::Tensor& grads, - torch::Tensor& exp_avg, - torch::Tensor& gpu_params) -{ -#if defined(__ENABLE_CUDA__) or defined(__ENABLE_CANN__) - auto params_c = params.contiguous(); - auto gpu_params_c = gpu_params.contiguous(); - auto exp_avg_c = exp_avg.contiguous(); - auto grads_c = grads.contiguous(); - - float* params_ptr = (float*)params_c.data_ptr(); - float* grads_ptr = (float*)grads_c.data_ptr(); - ds_half_precision_t* gpu_params_ptr = (ds_half_precision_t*)gpu_params_c.data_ptr(); - float* exp_avg_ptr = (float*)exp_avg_c.data_ptr(); - - std::shared_ptr opt = - std::static_pointer_cast(s_optimizers[optimizer_id]); - opt->IncrementStep(step, beta1, beta2); - opt->update_state(lr, weight_decay); - opt->Step_8(params_ptr, - grads_ptr, - exp_avg_ptr, - params_c.numel(), - gpu_params_ptr, - (params.options().dtype() == at::kHalf)); - - opt->SynchronizeStreams(); -#else - assert(false); -#endif return 0; } diff --git a/deepspeed/ops/adagrad/cpu_adagrad.py b/deepspeed/ops/adagrad/cpu_adagrad.py index c356a52777f2..dbde6d95f652 100755 --- a/deepspeed/ops/adagrad/cpu_adagrad.py +++ b/deepspeed/ops/adagrad/cpu_adagrad.py @@ -34,7 +34,7 @@ def __setstate__(self, state): group.setdefault('amsgrad', False) @torch.no_grad() - def step(self, closure=None, fp16_param_groups=None): + def step(self, closure=None): """Update the model parameters. .. note:: @@ -46,8 +46,6 @@ def step(self, closure=None, fp16_param_groups=None): Args: closure (callable, optional): closure to compute the loss. Defaults to ``None``. - fp16_param_groups: FP16 GPU parameters to update. Performing the - copy here reduces communication time. Defaults to ``None``. Returns: loss: if ``closure`` is provided. Otherwise ``None``. @@ -94,16 +92,7 @@ def step(self, closure=None, fp16_param_groups=None): sparse_exp_avg_sq.values()) p[sparse_param.indices()] = sparse_param.values() state['exp_avg_sq'][sparse_exp_avg_sq.indices()] = sparse_exp_avg_sq.values() - if fp16_param_groups is not None: - fp16_param_groups[group_id][param_id][sparse_param.indices()] = sparse_param.values() else: - if fp16_param_groups is not None: - self.ds_opt_adagrad.adagrad_update_copy(self.opt_id, state['step'], group['lr'], group['eps'], - group['weight_decay'], p.data, p.grad.data, - state['exp_avg_sq'], - fp16_param_groups[group_id][param_id].data) - else: - self.ds_opt_adagrad.adagrad_update(self.opt_id, state['step'], group['lr'], group['eps'], - group['weight_decay'], p.data, p.grad.data, - state['exp_avg_sq']) + self.ds_opt_adagrad.adagrad_update(self.opt_id, state['step'], group['lr'], group['eps'], + group['weight_decay'], p.data, p.grad.data, state['exp_avg_sq']) return loss diff --git a/deepspeed/ops/adam/cpu_adam.py b/deepspeed/ops/adam/cpu_adam.py index 10b8c15f970b..e0a72a494257 100755 --- a/deepspeed/ops/adam/cpu_adam.py +++ b/deepspeed/ops/adam/cpu_adam.py @@ -107,7 +107,7 @@ def __setstate__(self, state): group.setdefault('amsgrad', False) @torch.no_grad() - def step(self, closure=None, fp16_param_groups=None): + def step(self, closure=None): """Update the model parameters. .. note:: @@ -119,8 +119,6 @@ def step(self, closure=None, fp16_param_groups=None): Args: closure (callable, optional): closure to compute the loss. Defaults to ``None``. - fp16_param_groups: FP16 GPU parameters to update. Performing the - copy here reduces communication time. Defaults to ``None``. Returns: loss: if ``closure`` is provided. Otherwise ``None``. @@ -134,13 +132,6 @@ def step(self, closure=None, fp16_param_groups=None): # intended device for step device = torch.device('cpu') - # converting the fp16 params to a group of parameter - if type(fp16_param_groups) is list: - if type(fp16_param_groups[0]) is not list: - fp16_param_groups = [fp16_param_groups] - elif fp16_param_groups is not None: - fp16_param_groups = [[fp16_param_groups]] - for group_id, group in enumerate(self.param_groups): for param_id, p in enumerate(group['params']): @@ -169,13 +160,7 @@ def step(self, closure=None, fp16_param_groups=None): state['step'] += 1 beta1, beta2 = group['betas'] - if fp16_param_groups is not None: - self.ds_opt_adam.adam_update_copy(self.opt_id, state['step'], group['lr'], beta1, beta2, - group['eps'], group['weight_decay'], group['bias_correction'], - p.data, p.grad.data, state['exp_avg'], state['exp_avg_sq'], - fp16_param_groups[group_id][param_id].data) - else: - self.ds_opt_adam.adam_update(self.opt_id, state['step'], group['lr'], beta1, beta2, group['eps'], - group['weight_decay'], group['bias_correction'], p.data, p.grad.data, - state['exp_avg'], state['exp_avg_sq']) + self.ds_opt_adam.adam_update(self.opt_id, state['step'], group['lr'], beta1, beta2, group['eps'], + group['weight_decay'], group['bias_correction'], p.data, p.grad.data, + state['exp_avg'], state['exp_avg_sq']) return loss diff --git a/deepspeed/ops/lion/cpu_lion.py b/deepspeed/ops/lion/cpu_lion.py index a91a00643873..03342a3fcd34 100755 --- a/deepspeed/ops/lion/cpu_lion.py +++ b/deepspeed/ops/lion/cpu_lion.py @@ -69,7 +69,7 @@ def __setstate__(self, state): group.setdefault('amsgrad', False) @torch.no_grad() - def step(self, closure=None, fp16_param_groups=None): + def step(self, closure=None): """Update the model parameters. .. note:: @@ -81,8 +81,6 @@ def step(self, closure=None, fp16_param_groups=None): Args: closure (callable, optional): closure to compute the loss. Defaults to ``None``. - fp16_param_groups: FP16 GPU parameters to update. Performing the - copy here reduces communication time. Defaults to ``None``. Returns: loss: if ``closure`` is provided. Otherwise ``None``. @@ -96,13 +94,6 @@ def step(self, closure=None, fp16_param_groups=None): # intended device for step device = torch.device('cpu') - # converting the fp16 params to a group of parameter - if type(fp16_param_groups) is list: - if type(fp16_param_groups[0]) is not list: - fp16_param_groups = [fp16_param_groups] - elif fp16_param_groups is not None: - fp16_param_groups = [[fp16_param_groups]] - for group_id, group in enumerate(self.param_groups): for param_id, p in enumerate(group['params']): @@ -131,11 +122,6 @@ def step(self, closure=None, fp16_param_groups=None): state['step'] += 1 beta1, beta2 = group['betas'] - if fp16_param_groups is not None: - self.ds_opt_lion.lion_update_copy(self.opt_id, state['step'], group['lr'], beta1, beta2, - group['weight_decay'], p.data, p.grad.data, state['exp_avg'], - fp16_param_groups[group_id][param_id].data) - else: - self.ds_opt_lion.lion_update(self.opt_id, state['step'], group['lr'], beta1, beta2, - group['weight_decay'], p.data, p.grad.data, state['exp_avg']) + self.ds_opt_lion.lion_update(self.opt_id, state['step'], group['lr'], beta1, beta2, + group['weight_decay'], p.data, p.grad.data, state['exp_avg']) return loss diff --git a/op_builder/builder.py b/op_builder/builder.py index 18c130221b0e..4c4978c29575 100644 --- a/op_builder/builder.py +++ b/op_builder/builder.py @@ -545,6 +545,7 @@ def jit_load(self, verbose=True): nvcc_args.append("-DBF16_AVAILABLE") nvcc_args.append("-U__CUDA_NO_BFLOAT16_OPERATORS__") nvcc_args.append("-U__CUDA_NO_BFLOAT162_OPERATORS__") + nvcc_args.append("-U__CUDA_NO_BFLOAT16_CONVERSIONS__") if self.is_rocm_pytorch(): cxx_args.append("-D__HIP_PLATFORM_AMD__=1") diff --git a/op_builder/cpu/builder.py b/op_builder/cpu/builder.py index d2bc8eacfa25..d881842ad0b1 100644 --- a/op_builder/cpu/builder.py +++ b/op_builder/cpu/builder.py @@ -30,7 +30,11 @@ def builder(self): return cpp_ext def cxx_args(self): - return ['-O3', '-g', '-Wno-reorder'] + args = ['-O3', '-g', '-Wno-reorder'] + CPU_ARCH = self.cpu_arch() + SIMD_WIDTH = self.simd_width() + args += [CPU_ARCH, '-fopenmp', SIMD_WIDTH] + return args def libraries_args(self): return [] diff --git a/op_builder/cpu_adagrad.py b/op_builder/cpu_adagrad.py index d3f163f7464a..c05f71488950 100644 --- a/op_builder/cpu_adagrad.py +++ b/op_builder/cpu_adagrad.py @@ -3,7 +3,6 @@ # DeepSpeed Team -import os from .builder import TorchCPUOpBuilder @@ -18,26 +17,11 @@ def absolute_name(self): return f'deepspeed.ops.adagrad.{self.NAME}_op' def sources(self): - if self.build_for_cpu: - return ['csrc/adagrad/cpu_adagrad.cpp'] - - return ['csrc/adagrad/cpu_adagrad.cpp', 'csrc/common/custom_cuda_kernel.cu'] + return ['csrc/adagrad/cpu_adagrad.cpp'] def libraries_args(self): args = super().libraries_args() - if self.build_for_cpu: - return args - - if not self.is_rocm_pytorch(): - args += ['curand'] return args def include_paths(self): - import torch - if self.build_for_cpu: - CUDA_INCLUDE = [] - elif not self.is_rocm_pytorch(): - CUDA_INCLUDE = [os.path.join(torch.utils.cpp_extension.CUDA_HOME, "include")] - else: - CUDA_INCLUDE = [] - return ['csrc/includes'] + CUDA_INCLUDE + return ['csrc/includes'] diff --git a/op_builder/cpu_adam.py b/op_builder/cpu_adam.py index 7c34c4ce43a1..7f4c0847a8c4 100644 --- a/op_builder/cpu_adam.py +++ b/op_builder/cpu_adam.py @@ -3,7 +3,6 @@ # DeepSpeed Team -import os from .builder import TorchCPUOpBuilder @@ -18,27 +17,11 @@ def absolute_name(self): return f'deepspeed.ops.adam.{self.NAME}_op' def sources(self): - if self.build_for_cpu: - return ['csrc/adam/cpu_adam.cpp', 'csrc/adam/cpu_adam_impl.cpp'] - - return ['csrc/adam/cpu_adam.cpp', 'csrc/adam/cpu_adam_impl.cpp', 'csrc/common/custom_cuda_kernel.cu'] + return ['csrc/adam/cpu_adam.cpp', 'csrc/adam/cpu_adam_impl.cpp'] def libraries_args(self): args = super().libraries_args() - if self.build_for_cpu: - return args - - if not self.is_rocm_pytorch(): - args += ['curand'] - return args def include_paths(self): - import torch - if self.build_for_cpu: - CUDA_INCLUDE = [] - elif not self.is_rocm_pytorch(): - CUDA_INCLUDE = [os.path.join(torch.utils.cpp_extension.CUDA_HOME, "include")] - else: - CUDA_INCLUDE = [] - return ['csrc/includes'] + CUDA_INCLUDE + return ['csrc/includes'] diff --git a/op_builder/cpu_lion.py b/op_builder/cpu_lion.py index 5c16d10ebb44..9a60d99773b3 100644 --- a/op_builder/cpu_lion.py +++ b/op_builder/cpu_lion.py @@ -3,7 +3,6 @@ # DeepSpeed Team -import os from .builder import TorchCPUOpBuilder @@ -18,31 +17,11 @@ def absolute_name(self): return f'deepspeed.ops.lion.{self.NAME}_op' def sources(self): - if self.build_for_cpu: - return ['csrc/lion/cpu_lion.cpp', 'csrc/lion/cpu_lion_impl.cpp'] - - return ['csrc/lion/cpu_lion.cpp', 'csrc/lion/cpu_lion_impl.cpp', 'csrc/common/custom_cuda_kernel.cu'] + return ['csrc/lion/cpu_lion.cpp', 'csrc/lion/cpu_lion_impl.cpp'] def libraries_args(self): args = super().libraries_args() - if self.build_for_cpu: - return args - - if not self.is_rocm_pytorch(): - args += ['curand'] - return args def include_paths(self): - import torch - if self.build_for_cpu: - CUDA_INCLUDE = [] - elif not self.is_rocm_pytorch(): - CUDA_INCLUDE = [os.path.join(torch.utils.cpp_extension.CUDA_HOME, "include")] - else: - CUDA_INCLUDE = [ - os.path.join(torch.utils.cpp_extension.ROCM_HOME, "include"), - os.path.join(torch.utils.cpp_extension.ROCM_HOME, "include", "rocrand"), - os.path.join(torch.utils.cpp_extension.ROCM_HOME, "include", "hiprand"), - ] - return ['csrc/includes'] + CUDA_INCLUDE + return ['csrc/includes'] diff --git a/op_builder/hpu/builder.py b/op_builder/hpu/builder.py index 3c86128fffd6..c176a586ba49 100644 --- a/op_builder/hpu/builder.py +++ b/op_builder/hpu/builder.py @@ -31,7 +31,11 @@ def builder(self): return cpp_ext def cxx_args(self): - return ['-O3', '-g', '-Wno-reorder'] + args = ['-O3', '-g', '-Wno-reorder'] + CPU_ARCH = self.cpu_arch() + SIMD_WIDTH = self.simd_width() + args += [CPU_ARCH, '-fopenmp', SIMD_WIDTH] + return args def libraries_args(self): return [] diff --git a/op_builder/hpu/cpu_adam.py b/op_builder/hpu/cpu_adam.py index 2f3b7aefe705..58eea2698ebb 100644 --- a/op_builder/hpu/cpu_adam.py +++ b/op_builder/hpu/cpu_adam.py @@ -20,11 +20,6 @@ def absolute_name(self): def sources(self): return ['csrc/adam/cpu_adam.cpp', 'csrc/adam/cpu_adam_impl.cpp'] - def cxx_args(self): - args = super().cxx_args() - args += ['-DENABLE_BFLOAT16'] - return args - def libraries_args(self): args = super().libraries_args() return args diff --git a/tests/perf/adam_test1.py b/tests/perf/adam_test1.py index b35477afb4fe..bde1d53e5179 100755 --- a/tests/perf/adam_test1.py +++ b/tests/perf/adam_test1.py @@ -6,12 +6,10 @@ import torch from deepspeed.ops.adam import DeepSpeedCPUAdam import time -from deepspeed.accelerator import get_accelerator device = 'cpu' model_size = 1 * 1024**3 param = torch.nn.Parameter(torch.ones(model_size, device=device)) -param_fp16 = torch.nn.Parameter(torch.ones(model_size, dtype=torch.half, device=get_accelerator().device_name(0))) optimizer = DeepSpeedCPUAdam([param]) #torch.set_num_threads(128) @@ -19,7 +17,7 @@ avg = 0 for i in range(100): start = time.time() - optimizer.step(fp16_param_groups=[param_fp16]) + optimizer.step() stop = time.time() avg += (stop - start) param.grad = torch.ones(model_size, device=device) * 2 diff --git a/tests/unit/common.py b/tests/unit/common.py index a2593e703aef..58bb26ca18b4 100644 --- a/tests/unit/common.py +++ b/tests/unit/common.py @@ -82,8 +82,12 @@ def set_accelerator_visible(): if match: num_accelerators += 1 elif get_accelerator().device_name() == 'hpu': - hl_smi = subprocess.check_output(['hl-smi', "-L"]) - num_accelerators = re.findall(r"Module ID\s+:\s+(\d+)", hl_smi.decode()) + try: + hl_smi = subprocess.check_output(['hl-smi', "-L"]) + num_accelerators = re.findall(r"Module ID\s+:\s+(\d+)", hl_smi.decode()) + except FileNotFoundError: + sim_list = subprocess.check_output(['ls', '-1', '/dev/accel']) + num_accelerators = re.findall(r"accel(\d+)", sim_list.decode()) num_accelerators = sorted(num_accelerators, key=int) os.environ["HABANA_VISIBLE_MODULES"] = ",".join(num_accelerators) elif get_accelerator().device_name() == 'npu': diff --git a/tests/unit/ops/adagrad/test_cpu_adagrad.py b/tests/unit/ops/adagrad/test_cpu_adagrad.py index 99e934e2efda..0c675ecd6a85 100644 --- a/tests/unit/ops/adagrad/test_cpu_adagrad.py +++ b/tests/unit/ops/adagrad/test_cpu_adagrad.py @@ -18,8 +18,8 @@ def check_equal(first, second, atol=1e-2, verbose=False): - x = first.detach().numpy() - y = second.detach().numpy() + x = first.detach().float().numpy() + y = second.detach().float().numpy() if verbose: print("x = {}".format(x.flatten())) print("y = {}".format(y.flatten())) diff --git a/tests/unit/ops/adam/test_cpu_adam.py b/tests/unit/ops/adam/test_cpu_adam.py index 785cf786acc3..851485440428 100644 --- a/tests/unit/ops/adam/test_cpu_adam.py +++ b/tests/unit/ops/adam/test_cpu_adam.py @@ -21,8 +21,8 @@ def check_equal(first, second, atol=1e-2, verbose=False): - x = first.detach().numpy() - y = second.detach().numpy() + x = first.detach().float().numpy() + y = second.detach().float().numpy() print("ATOL", atol) if verbose: print("x = {}".format(x.flatten())) @@ -43,7 +43,7 @@ def _compare_optimizers(model_size, param1, optimizer1, param2, optimizer2): check_equal(param1.float().norm(), param2.float().cpu().norm(), atol=tolerance, verbose=True) -@pytest.mark.parametrize('dtype', [torch.half, torch.float], ids=["fp16", "fp32"]) +@pytest.mark.parametrize('dtype', [torch.half, torch.bfloat16, torch.float], ids=["fp16", "bf16", "fp32"]) @pytest.mark.parametrize('model_size', [ (64), @@ -65,6 +65,9 @@ class TestCPUAdam(DistributedTest): @pytest.mark.skipif(not deepspeed.ops.__compatible_ops__[FusedAdamBuilder.NAME], reason="FusedAdam is not compatible") def test_fused_adam_equal(self, dtype, model_size): + if dtype not in get_accelerator().supported_dtypes(): + pytest.skip(f"dtype {dtype} not supported in current accelerator") + if ("amd" in pytest.cpu_vendor) and (dtype == torch.half): pytest.skip("cpu-adam with half precision not supported on AMD CPUs") @@ -91,6 +94,8 @@ def test_fused_adam_equal(self, dtype, model_size): def test_torch_adamw_equal(self, dtype, model_size): if get_accelerator().is_available(): + if dtype == torch.half: + pytest.skip("torch.optim.AdamW with half precision inf/nan output.") if ("amd" in pytest.cpu_vendor) and (dtype == torch.half): pytest.skip("cpu-adam with half precision not supported on AMD CPUs") ref_param_device = get_accelerator().device_name() @@ -99,20 +104,20 @@ def test_torch_adamw_equal(self, dtype, model_size): pytest.skip("torch.optim.AdamW with half precision only supported in CUDA environments.") ref_param_device = 'cpu' - from deepspeed.ops.adam import DeepSpeedCPUAdam + from deepspeed.ops.adam import DeepSpeedCPUAdam - cpu_data = torch.randn(model_size, device='cpu').to(dtype) - cpu_param = torch.nn.Parameter(cpu_data) - ref_param = torch.nn.Parameter(cpu_data.to(ref_param_device)) + cpu_data = torch.randn(model_size, device='cpu').to(dtype) + cpu_param = torch.nn.Parameter(cpu_data) + ref_param = torch.nn.Parameter(cpu_data.to(ref_param_device)) - cpu_optimizer = DeepSpeedCPUAdam([cpu_param]) - ref_optimizer = torch.optim.AdamW([ref_param]) + cpu_optimizer = DeepSpeedCPUAdam([cpu_param]) + ref_optimizer = torch.optim.AdamW([ref_param]) - _compare_optimizers(model_size=model_size, - param1=cpu_param, - optimizer1=cpu_optimizer, - param2=ref_param, - optimizer2=ref_optimizer) + _compare_optimizers(model_size=model_size, + param1=cpu_param, + optimizer1=cpu_optimizer, + param2=ref_param, + optimizer2=ref_optimizer) class TestCPUAdamGPUError(DistributedTest): diff --git a/tests/unit/ops/adam/test_hybrid_adam.py b/tests/unit/ops/adam/test_hybrid_adam.py index 9003e02588c1..652090d5b9d5 100644 --- a/tests/unit/ops/adam/test_hybrid_adam.py +++ b/tests/unit/ops/adam/test_hybrid_adam.py @@ -22,8 +22,8 @@ def check_equal(first, second, atol=1e-2, verbose=False): - x = first.detach().numpy() - y = second.detach().numpy() + x = first.detach().float().numpy() + y = second.detach().float().numpy() print("ATOL", atol) if verbose: print("x = {}".format(x.flatten())) @@ -32,7 +32,7 @@ def check_equal(first, second, atol=1e-2, verbose=False): np.testing.assert_allclose(x, y, err_msg="param-update mismatch!", atol=atol) -@pytest.mark.parametrize('dtype', [torch.half, torch.float], ids=["fp16", "fp32"]) +@pytest.mark.parametrize('dtype', [torch.half, torch.bfloat16, torch.float], ids=["fp16", "bf16", "fp32"]) @pytest.mark.parametrize('model_size', [8, 16]) class TestHybridAdam(DistributedTest): world_size = 1 diff --git a/tests/unit/ops/lion/test_cpu_lion.py b/tests/unit/ops/lion/test_cpu_lion.py index 7d40a98f35b9..dce027e286fb 100644 --- a/tests/unit/ops/lion/test_cpu_lion.py +++ b/tests/unit/ops/lion/test_cpu_lion.py @@ -18,8 +18,8 @@ def check_equal(first, second, atol=1e-2, verbose=False): - x = first.detach().numpy() - y = second.detach().numpy() + x = first.detach().float().numpy() + y = second.detach().float().numpy() print("ATOL", atol) if verbose: print("x = {}".format(x.flatten())) @@ -40,7 +40,7 @@ def _compare_optimizers(model_size, param1, optimizer1, param2, optimizer2): check_equal(param1.float().norm(), param2.float().cpu().norm(), atol=tolerance, verbose=True) -@pytest.mark.parametrize('dtype', [torch.half, torch.float], ids=["fp16", "fp32"]) +@pytest.mark.parametrize('dtype', [torch.half, torch.bfloat16, torch.float], ids=["fp16", "bf16", "fp32"]) @pytest.mark.parametrize('model_size', [ (64), From 1d8196736f3f53dd754803f42c23fa1010c49d04 Mon Sep 17 00:00:00 2001 From: shiyang-weng Date: Mon, 20 May 2024 22:52:44 +0800 Subject: [PATCH 19/25] Fix the TypeError for XPU Accelerator (#5531) Fixing following error /datadisk2/wengshiy/llm.devkit/DeepSpeed/deepspeed/runtime/utils.py return get_accelerator().FloatTensor(float(v)).detach() TypeError: new(): data must be a sequence (got float) cuda accelerator modified the interface for fixing warning: https://github.com/microsoft/DeepSpeed/commit/177dc14331a64e61f6dcce2c4b8071576bcb22db --------- Co-authored-by: Olatunji Ruwase --- accelerator/xpu_accelerator.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/accelerator/xpu_accelerator.py b/accelerator/xpu_accelerator.py index 9c4a9c903f96..3fed89d7200f 100644 --- a/accelerator/xpu_accelerator.py +++ b/accelerator/xpu_accelerator.py @@ -7,6 +7,7 @@ from deepspeed.accelerator.abstract_accelerator import DeepSpeedAccelerator import intel_extension_for_pytorch as ipex # noqa: F401 # type: ignore import oneccl_bindings_for_pytorch # noqa: F401 # type: ignore +import functools class XPU_Accelerator(DeepSpeedAccelerator): @@ -191,31 +192,31 @@ def supported_dtypes(self): @property def BFloat16Tensor(self): - return torch.xpu.BFloat16Tensor + return functools.partial(torch.tensor, dtype=torch.bfloat16, device=self._name) @property def ByteTensor(self): - return torch.xpu.ByteTensor + return functools.partial(torch.tensor, dtype=torch.uint8, device=self._name) @property def DoubleTensor(self): - return torch.xpu.DoubleTensor + return functools.partial(torch.tensor, dtype=torch.double, device=self._name) @property def FloatTensor(self): - return torch.xpu.FloatTensor + return functools.partial(torch.tensor, dtype=torch.float, device=self._name) @property def HalfTensor(self): - return torch.xpu.HalfTensor + return functools.partial(torch.tensor, dtype=torch.half, device=self._name) @property def IntTensor(self): - return torch.xpu.IntTensor + return functools.partial(torch.tensor, dtype=torch.int, device=self._name) @property def LongTensor(self): - return torch.xpu.LongTensor + return functools.partial(torch.tensor, dtype=torch.long, device=self._name) def pin_memory(self, tensor, align_bytes=1): if align_bytes == 1: From 695d79ea0664247ed4ef31425f529a7cf0ff561b Mon Sep 17 00:00:00 2001 From: shiyang-weng Date: Tue, 21 May 2024 23:01:05 +0800 Subject: [PATCH 20/25] Fix RuntimeError for moe on XPU: tensors found at least two devices (#5519) There is following error on XPU while unit testing "DeepSpeed/tests/unit/moe/test_moe.py" DeepSpeed/deepspeed/moe/sharded_moe.py line 223, in top1gating RuntimeError: Expected all tensors to be on the same device, but found at least two devices, xpu:0 and cpu! Fix it by device conversion. --------- Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> --- deepspeed/moe/sharded_moe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/moe/sharded_moe.py b/deepspeed/moe/sharded_moe.py index bd2782279c01..96eab5e2ab17 100644 --- a/deepspeed/moe/sharded_moe.py +++ b/deepspeed/moe/sharded_moe.py @@ -220,7 +220,7 @@ def top1gating(logits: Tensor, tp = 1 if groups.mpu is None else bwc_tensor_model_parallel_world_size(mpu=groups.mpu) new_capacity = torch.ceil(new_capacity / tp).mul(tp).to(new_capacity.dtype) # Make sure the capacity value does not exceed the number of tokens. - capacity = min(new_capacity, torch.tensor(mask1.size(0))) + capacity = min(new_capacity, torch.tensor(mask1.size(0)).to(new_capacity.device)) # Compute l_aux me = torch.mean(gates, dim=0) From 0a1740386f6a3fef1b655e27e0d8d0bc40879021 Mon Sep 17 00:00:00 2001 From: Liran Bachar Date: Tue, 21 May 2024 18:01:20 +0300 Subject: [PATCH 21/25] Remove synchronize calls from allgather params (#5516) Co-authored-by: Olatunji Ruwase --- deepspeed/runtime/zero/partition_parameters.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/deepspeed/runtime/zero/partition_parameters.py b/deepspeed/runtime/zero/partition_parameters.py index b2909145c50c..09e72a695db3 100755 --- a/deepspeed/runtime/zero/partition_parameters.py +++ b/deepspeed/runtime/zero/partition_parameters.py @@ -56,7 +56,7 @@ def __init__(self, param: Parameter) -> None: self.__param = param def wait(self) -> None: - if not get_accelerator().is_synchronized_device(): + if not get_accelerator().resolves_data_dependency(): get_accelerator().current_stream().synchronize() self.__param.ds_status = ZeroParamStatus.AVAILABLE @@ -82,7 +82,7 @@ def wait(self) -> None: if self.__complete: return - if not get_accelerator().is_synchronized_device(): + if not get_accelerator().resolves_data_dependency(): get_accelerator().current_stream().synchronize() for param in self.__params: assert param.ds_status == ZeroParamStatus.INFLIGHT, f"expected param {param.ds_summary()} to be inflight" @@ -1737,7 +1737,8 @@ def _allgather_param(self, param, async_op=False, hierarchy=0): f'After allocate allgather param {debug_param2name_id_shape_status(param)} {aligned_param_size} {partition_size} ', force=False) - get_accelerator().synchronize() + if not get_accelerator().resolves_data_dependency(): + get_accelerator().synchronize() print_rank_0( f"{'--'* hierarchy}----allgather param with {debug_param2name_id_shape_status(param)} partition size={partition_size}" @@ -1870,7 +1871,8 @@ def _allgather_params_coalesced(self, param_list, hierarchy=0, quantize=False): param.data = gathered_tensor.narrow(0, 0, param.ds_numel).view(param.ds_shape).data # guarantee the communication to be completed - get_accelerator().synchronize() + if not get_accelerator().resolves_data_dependency(): + get_accelerator().synchronize() return None From 5b314f4e6b87f9f55426a6bc35ff0d1cca88a552 Mon Sep 17 00:00:00 2001 From: Max Kovalenko <75629718+deepcharm@users.noreply.github.com> Date: Tue, 21 May 2024 20:17:06 +0300 Subject: [PATCH 22/25] Avoid overwrite of compiled module wrapper attributes (#5549) **Fix overwriting of the compiled wrapper class attributes by those of the wrapped class itself: Copy only those attributes which are not already present in the wrapper.** In the current implementation of the `CompiledModuleWrapper` the wrapper attributes (eg `forward` method) are overwritten by `self._dict_ = module._dict_.copy()`: ``` def CompiledModuleWrapper(mod, compile_config: Union[CompileConfig, None] = None): class wrapper(mod.__class__): def __init__(self, module, compile_config: Union[CompileConfig, None] = None): self.__dict__ = module.__dict__.copy() ``` This causes the `wrapper`'s `forward` method not being called and, consequently, the wrapped module not compiled. Instead, the wrapped module `forward` method is being called as illustrated in the diagram below (a real scenario from Deespeed-Chat): ![compiled_module_wrapper_bug](https://github.com/microsoft/DeepSpeed/assets/75629718/00eeb3d1-927c-49c7-84ab-f882821cc452) The proposed fix copies only those attributes which are not present in the wrapper class, thus implementing the desired inheritance quality of the wrapper. Attached is a simple reproducer of the problem. [compiled_module_wrapper_bug.zip](https://github.com/microsoft/DeepSpeed/files/15378282/compiled_module_wrapper_bug.zip) Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> --- deepspeed/runtime/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/runtime/compiler.py b/deepspeed/runtime/compiler.py index 66fe29fbbea2..4f828d978613 100644 --- a/deepspeed/runtime/compiler.py +++ b/deepspeed/runtime/compiler.py @@ -88,7 +88,7 @@ def CompiledModuleWrapper(mod, compile_config: Union[CompileConfig, None] = None class wrapper(mod.__class__): def __init__(self, module, compile_config: Union[CompileConfig, None] = None): - self.__dict__ = module.__dict__.copy() + self.__dict__ = {k: module.__dict__[k] for k in module.__dict__ if not k in self.__class__.__dict__} assert is_compile_supported(), "torch.compile is not supported on this version of PyTorch." From 975290ae6586c42a4fafe578e84a6e95af793961 Mon Sep 17 00:00:00 2001 From: Zihan Zhao Date: Wed, 22 May 2024 05:37:27 +0800 Subject: [PATCH 23/25] Small typos in functions set_none_gradients_to_zero (#5557) change from "zero_like" to "zeros_like" --- deepspeed/runtime/zero/stage3.py | 2 +- deepspeed/runtime/zero/stage_1_and_2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deepspeed/runtime/zero/stage3.py b/deepspeed/runtime/zero/stage3.py index 13ca29c9fceb..3f43e865fa72 100644 --- a/deepspeed/runtime/zero/stage3.py +++ b/deepspeed/runtime/zero/stage3.py @@ -1575,7 +1575,7 @@ def set_none_gradients_to_zero(self, i, partition_id): for param_id in self.is_grad_computed[i][partition_id]: param = self.param_dict[param_id] if param.grad is None: - param.grad = torch.zero_like(param) + param.grad = torch.zeros_like(param) ######################Reduction Related Methods############################## diff --git a/deepspeed/runtime/zero/stage_1_and_2.py b/deepspeed/runtime/zero/stage_1_and_2.py index 225c085f6f2b..3d5ff5e6b43e 100755 --- a/deepspeed/runtime/zero/stage_1_and_2.py +++ b/deepspeed/runtime/zero/stage_1_and_2.py @@ -1474,7 +1474,7 @@ def set_none_gradients_to_zero(self, i, partition_id): for param_id in self.is_grad_computed[i][partition_id]: param = self.param_dict[param_id] if param.grad is None: - param.grad = torch.zero_like(param) + param.grad = torch.zeros_like(param) ######################Reduction Related Methods############################## def allreduce_bucket(self, bucket, rank=None, log=None, divide=True, process_group=None): From 29903925cf281e41742b6e5baba2daf6b83d2b76 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Wed, 22 May 2024 05:58:47 +0800 Subject: [PATCH 24/25] Adapt doc for #4405 (#5552) ditto Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> --- deepspeed/comm/comm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/comm/comm.py b/deepspeed/comm/comm.py index 4f826edab3d6..85b7fab2c548 100644 --- a/deepspeed/comm/comm.py +++ b/deepspeed/comm/comm.py @@ -618,7 +618,7 @@ def init_distributed(dist_backend=None, auto_mpi_discovery Optional (bool). if distributed environment variables are not set, attempt to discover them from MPI distributed_port: Optional (int). torch distributed backend port verbose: Optional (bool). verbose logging - timeout: Optional (timedelta). Timeout for operations executed against the process group. Default value equals 30 minutes. + timeout: Optional (timedelta). Timeout for operations executed against the process group. The default value of 30 minutes can be overridden by the environment variable `DEEPSPEED_TIMEOUT`. init_method: Optional (string). Torch distributed, URL specifying how to initialize the process group. Default is “env://” if no init_method or store is specified. config: Optional (dict). DeepSpeed configuration for setting up comms options (e.g. Comms profiling) rank: Optional (int). The current manually specified rank. Some init_method like “tcp://” need the rank and world_size as well (see: https://pytorch.org/docs/stable/distributed.html#tcp-initialization) From 263bfe2892c1ee6285076214bb5e3898c35e78f3 Mon Sep 17 00:00:00 2001 From: Logan Adams <114770087+loadams@users.noreply.github.com> Date: Wed, 22 May 2024 09:08:51 -0700 Subject: [PATCH 25/25] Update to HF_HOME from TRANSFORMERS_CACHE (#4816) Addresses the following warning: ``` /tmp/actions-runner/_work/DeepSpeed/DeepSpeed/unit-test-venv/lib/python3.8/site-packages/transformers/utils/hub.py:123: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. ``` and the code on the transformers side is [here](https://github.com/huggingface/transformers/blob/1a585c1222a56bcaecc070966d558d4a9d862e83/src/transformers/utils/hub.py#L86C1-L96C81). --- .github/workflows/cpu-inference.yml | 4 ++-- .github/workflows/cpu-torch-latest.yml | 4 ++-- .github/workflows/setup-venv/action.yml | 2 +- tests/unit/inference/test_checkpoint_sharding.py | 2 +- tests/unit/inference/test_inference.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cpu-inference.yml b/.github/workflows/cpu-inference.yml index 38dd9bd3efef..d91034270eec 100644 --- a/.github/workflows/cpu-inference.yml +++ b/.github/workflows/cpu-inference.yml @@ -97,5 +97,5 @@ jobs: unset TORCH_CUDA_ARCH_LIST # only jit compile for current arch cd tests # LOCAL_SIZE=2 enforce CPU to report 2 devices, this helps run the test on github default runner - LOCAL_SIZE=2 COLUMNS=240 TRANSFORMERS_CACHE=~/tmp/transformers_cache/ TORCH_EXTENSIONS_DIR=./torch-extensions pytest -m 'seq_inference' unit/ - LOCAL_SIZE=2 COLUMNS=240 TRANSFORMERS_CACHE=~/tmp/transformers_cache/ TORCH_EXTENSIONS_DIR=./torch-extensions pytest -m 'inference_ops' -m 'inference' unit/ + LOCAL_SIZE=2 COLUMNS=240 HF_HOME=~/tmp/hf_home/ TORCH_EXTENSIONS_DIR=./torch-extensions pytest -m 'seq_inference' unit/ + LOCAL_SIZE=2 COLUMNS=240 HF_HOME=~/tmp/hf_home/ TORCH_EXTENSIONS_DIR=./torch-extensions pytest -m 'inference_ops' -m 'inference' unit/ diff --git a/.github/workflows/cpu-torch-latest.yml b/.github/workflows/cpu-torch-latest.yml index 5727ff2e1cde..213421590ad6 100644 --- a/.github/workflows/cpu-torch-latest.yml +++ b/.github/workflows/cpu-torch-latest.yml @@ -50,5 +50,5 @@ jobs: run: | unset TORCH_CUDA_ARCH_LIST # only jit compile for current arch cd tests - TRANSFORMERS_CACHE=/tmp/transformers_cache/ pytest $PYTEST_OPTS -n 4 unit/ --torch_ver="2.3" - TRANSFORMERS_CACHE=/tmp/transformers_cache/ pytest $PYTEST_OPTS -m 'sequential' unit/ --torch_ver="2.3" + HF_HOME=/tmp/hf_home/ pytest $PYTEST_OPTS -n 4 unit/ --torch_ver="2.3" + HF_HOME=/tmp/hf_home/ pytest $PYTEST_OPTS -m 'sequential' unit/ --torch_ver="2.3" diff --git a/.github/workflows/setup-venv/action.yml b/.github/workflows/setup-venv/action.yml index ce2c458b9e57..9a88e0651860 100644 --- a/.github/workflows/setup-venv/action.yml +++ b/.github/workflows/setup-venv/action.yml @@ -22,7 +22,7 @@ runs: - id: set-env-vars run: | echo TEST_DATA_DIR=/blob/ >> $GITHUB_ENV - echo TRANSFORMERS_CACHE=/blob/transformers_cache/ >> $GITHUB_ENV + echo HF_HOME=/blob/hf_home/ >> $GITHUB_ENV echo TORCH_EXTENSIONS_DIR=./torch-extensions/ >> $GITHUB_ENV echo TORCH_CACHE=/blob/torch_cache/ >> $GITHUB_ENV echo HF_DATASETS_CACHE=/blob/datasets_cache/ >> $GITHUB_ENV diff --git a/tests/unit/inference/test_checkpoint_sharding.py b/tests/unit/inference/test_checkpoint_sharding.py index 564b3fab6bf4..5bae9a151a27 100644 --- a/tests/unit/inference/test_checkpoint_sharding.py +++ b/tests/unit/inference/test_checkpoint_sharding.py @@ -110,7 +110,7 @@ def write_checkpoints_json(model_name, class_tmpdir): cached_repo_dir = snapshot_download( model_name, local_files_only=is_offline_mode(), - cache_dir=os.getenv("TRANSFORMERS_CACHE", None), + cache_dir=os.getenv("HF_HOME", None), ignore_patterns=["*.safetensors", "*.msgpack", "*.h5"], ) file_list = [str(entry) for entry in Path(cached_repo_dir).rglob("*.[bp][it][n]") if entry.is_file()] diff --git a/tests/unit/inference/test_inference.py b/tests/unit/inference/test_inference.py index 4e203a71db60..36003319856c 100644 --- a/tests/unit/inference/test_inference.py +++ b/tests/unit/inference/test_inference.py @@ -84,7 +84,7 @@ class ModelInfo: def _hf_model_list() -> List[ModelInfo]: """ Caches HF model list to avoid repeated API calls """ - cache_dir = os.getenv("TRANSFORMERS_CACHE", "~/.cache/huggingface") + cache_dir = os.getenv("HF_HOME", "~/.cache/huggingface") cache_file_path = os.path.join(cache_dir, "DS_model_cache.pkl") cache_expiration_seconds = 60 * 60 * 24 # 1 day